[compiler-rt] [llvm] [ASan] Correctly handle vectorized pointer sub/cmp for `invalid-pointer-pair` (PR #213546)

[Thurston Dang via llvm-commits]

================
@@ -1698,16 +1698,38 @@ bool AddressSanitizer::GlobalIsLinkerInitialized(GlobalVariable *G) {
   return true;
 }
 
-void AddressSanitizer::instrumentPointerComparisonOrSubtraction(
+bool AddressSanitizer::instrumentPointerComparisonOrSubtraction(
     Instruction *I, RuntimeCallInserter &RTCI) {
   IRBuilder<> IRB(I);
   FunctionCallee F = isa<ICmpInst>(I) ? AsanPtrCmpFunction : AsanPtrSubFunction;
   Value *Param[2] = {I->getOperand(0), I->getOperand(1)};
-  for (Value *&i : Param) {
-    if (i->getType()->isPointerTy())
-      i = IRB.CreatePointerCast(i, IntptrTy);
+
+  if (const auto *Ty = Param[0]->getType(); Ty->isVectorTy()) {
+    const auto *VTy = dyn_cast<FixedVectorType>(Ty);
+    // Skip scalable vectors
+    if (!VTy)
+      return false;
+
+    assert(Param[1]->getType()->isVectorTy() &&
+           VTy->getElementCount() ==
+               cast<VectorType>(Param[1]->getType())->getElementCount() &&
+           "invalid vector pointer pair instrumentation operands");
+    for (unsigned Index = 0, NumElements = VTy->getNumElements();
+         Index != NumElements; ++Index) {
+      Value *ScalarParam[2] = {
+          IRB.CreateExtractElement(Param[0], IRB.getInt32(Index)),
+          IRB.CreateExtractElement(Param[1], IRB.getInt32(Index))};
+      for (Value *&P : ScalarParam)
+        P = IRB.CreatePointerCast(P, IntptrTy);
----------------
thurstond wrote:

Nit: a for loop over two elements is of marginal benefit. It might be simpler to fold the CreatePointerCasts into the preceding lines.

https://github.com/llvm/llvm-project/pull/213546