[llvm] [ASan] Convert pointer-pair operands based on type (PR #218494)

[via llvm-commits]

https://github.com/im-lunex updated https://github.com/llvm/llvm-project/pull/218494

>From e0cbdd99feddf9f70e7d15177eb9b0e08df219e8 Mon Sep 17 00:00:00 2001
From: im-lunex <thisissamir04 at gmail.com>
Date: Tue, 25 Aug 2026 00:56:51 +0600
Subject: [PATCH] [ASan] Convert pointer-pair operands based on type

---
 .../Instrumentation/AddressSanitizer.cpp      | 18 +++++--
 .../asan-detect-invalid-pointer-pair.ll       | 48 +++++++++++++++++++
 2 files changed, 61 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index 06d60003631d0..0d7ed944dc5a9 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -1698,6 +1698,14 @@ bool AddressSanitizer::GlobalIsLinkerInitialized(GlobalVariable *G) {
   return true;
 }
 
+static Value *convertToIntptr(IRBuilder<> &IRB, Value *V, Type *IntptrTy) {
+  if (V->getType()->isPtrOrPtrVectorTy())
+    return IRB.CreatePointerCast(V, IntptrTy);
+  assert(V->getType()->isIntOrIntVectorTy() &&
+         "unexpected pointer-pair operand type");
+  return IRB.CreateZExtOrTrunc(V, IntptrTy);
+}
+
 bool AddressSanitizer::instrumentPointerComparisonOrSubtraction(
     Instruction *I, RuntimeCallInserter &RTCI) {
   IRBuilder<> IRB(I);
@@ -1715,11 +1723,11 @@ bool AddressSanitizer::instrumentPointerComparisonOrSubtraction(
     for (unsigned Index = 0, NumElements = VTy->getNumElements();
          Index != NumElements; ++Index) {
       Value *ScalarParam[2] = {
-          IRB.CreatePointerCast(
-              IRB.CreateExtractElement(Param[0], IRB.getInt32(Index)),
+          convertToIntptr(
+              IRB, IRB.CreateExtractElement(Param[0], IRB.getInt32(Index)),
               IntptrTy),
-          IRB.CreatePointerCast(
-              IRB.CreateExtractElement(Param[1], IRB.getInt32(Index)),
+          convertToIntptr(
+              IRB, IRB.CreateExtractElement(Param[1], IRB.getInt32(Index)),
               IntptrTy)};
       RTCI.createRuntimeCall(IRB, F, ScalarParam);
     }
@@ -1727,7 +1735,7 @@ bool AddressSanitizer::instrumentPointerComparisonOrSubtraction(
   }
 
   for (Value *&P : Param)
-    P = IRB.CreatePointerCast(P, IntptrTy);
+    P = convertToIntptr(IRB, P, IntptrTy);
   RTCI.createRuntimeCall(IRB, F, Param);
   return true;
 }
diff --git a/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll b/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll
index 7632a79436662..6c340f64e4ced 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll
@@ -71,3 +71,51 @@ define <2 x i1> @mycmp_vector(<2 x ptr> %p, <2 x ptr> %q) sanitize_address {
   %z = icmp ult <2 x i64> %x, %y
   ret <2 x i1> %z
 }
+
+define i32 @mysub_ptrtoint_trunc(ptr %p, ptr %q) sanitize_address {
+; ALL-LABEL: @mysub_ptrtoint_trunc
+; NOSUB-NOT: call void @__sanitizer_ptr_sub
+; SUB: [[P:%[0-9A-Za-z]+]] = ptrtoint ptr %p to i32
+; SUB: [[Q:%[0-9A-Za-z]+]] = ptrtoint ptr %q to i32
+  %x = ptrtoint ptr %p to i32
+  %y = ptrtoint ptr %q to i32
+  %z = sub i32 %x, %y
+; SUB: [[XP:%[0-9A-Za-z]+]] = zext i32 [[P]] to i64
+; SUB: [[XQ:%[0-9A-Za-z]+]] = zext i32 [[Q]] to i64
+; SUB: call void @__sanitizer_ptr_sub(i64 [[XP]], i64 [[XQ]])
+  ret i32 %z
+}
+
+define <2 x i32> @mysub_vector_ptrtoint_trunc(<2 x ptr> %p, <2 x ptr> %q) sanitize_address {
+; ALL-LABEL: @mysub_vector_ptrtoint_trunc
+; NOSUB-NOT: call void @__sanitizer_ptr_sub
+  %x = ptrtoint <2 x ptr> %p to <2 x i32>
+  %y = ptrtoint <2 x ptr> %q to <2 x i32>
+; SUB: [[X0:%[0-9A-Za-z]+]] = extractelement <2 x i32> %x, i32 0
+; SUB: [[ZX0:%[0-9A-Za-z]+]] = zext i32 [[X0]] to i64
+; SUB: [[Y0:%[0-9A-Za-z]+]] = extractelement <2 x i32> %y, i32 0
+; SUB: [[ZY0:%[0-9A-Za-z]+]] = zext i32 [[Y0]] to i64
+; SUB: call void @__sanitizer_ptr_sub(i64 [[ZX0]], i64 [[ZY0]])
+; SUB: [[X1:%[0-9A-Za-z]+]] = extractelement <2 x i32> %x, i32 1
+; SUB: [[ZX1:%[0-9A-Za-z]+]] = zext i32 [[X1]] to i64
+; SUB: [[Y1:%[0-9A-Za-z]+]] = extractelement <2 x i32> %y, i32 1
+; SUB: [[ZY1:%[0-9A-Za-z]+]] = zext i32 [[Y1]] to i64
+; SUB: call void @__sanitizer_ptr_sub(i64 [[ZX1]], i64 [[ZY1]])
+  %z = sub <2 x i32> %x, %y
+  ret <2 x i32> %z
+}
+
+define i128 @mysub_ptrtoint_widen(ptr %p, ptr %q) sanitize_address {
+; ALL-LABEL: @mysub_ptrtoint_widen
+; NOSUB-NOT: call void @__sanitizer_ptr_sub
+; SUB: [[P:%[0-9A-Za-z]+]] = ptrtoint ptr %p to i128
+; SUB: [[Q:%[0-9A-Za-z]+]] = ptrtoint ptr %q to i128
+  %x = ptrtoint ptr %p to i128
+  %y = ptrtoint ptr %q to i128
+; SUB-NOT: zext i128
+; SUB: [[XP:%[0-9A-Za-z]+]] = trunc i128 [[P]] to i64
+; SUB: [[XQ:%[0-9A-Za-z]+]] = trunc i128 [[Q]] to i64
+; SUB: call void @__sanitizer_ptr_sub(i64 [[XP]], i64 [[XQ]])
+  %z = sub i128 %x, %y
+  ret i128 %z
+}