[llvm] [msan][NFCI] Replace unnecessary shadow cast with assertion (PR #186498)

[Thurston Dang via llvm-commits]

https://github.com/thurstond updated https://github.com/llvm/llvm-project/pull/186498

>From 54acb650a458c9abe944d458cd2ccacac7490857 Mon Sep 17 00:00:00 2001
From: Thurston Dang <thurston at google.com>
Date: Fri, 13 Mar 2026 18:25:25 +0000
Subject: [PATCH 1/4] [msan][NFCI] Replace unnecessary shadow cast with
 assertion

Fabian Wolff pointed out that #176031 made the output of CreateIntCast
unused.

Upon closer inspection, the CreateIntCasts are unnecessary, because the
arguments to handleBitwiseAnd/visitOr are integers or vectors of integers, for which
the shadow types are the same as the original types. This patch removes
the unnecessary if and shadow cast, and adds in an assertion.
---
 .../Instrumentation/MemorySanitizer.cpp       | 32 +++++++++++++------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index 991dd2e2d4db7..7a8cf5fe1e926 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -2620,15 +2620,23 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   //    S = (S1 & S2) | (V1 & S2) | (S1 & V2)
   Value *handleBitwiseAnd(IRBuilder<> &IRB, Value *V1, Value *V2, Value *S1,
                           Value *S2) {
+    // "The two arguments to the ‘and’ instruction must be integer or vector
+    //  of integer values. Both arguments must have identical types."
+    //
+    // We enforce this condition for all callers to handleBitwiseAnd(); callers
+    // with non-integer types should call CreateIntCast() themselves.
+    assert(V1->getType()->isIntOrIntVectorTy());
+    assert(V2->getType()->isIntOrIntVectorTy());
+
+    // Conveniently, getShadowTy() of Int/IntVector returns the original type.
+    assert(V1->getType() == V2->getType());
+    assert(V1->getType() == S1->getType());
+    assert(V2->getType() == S2->getType());
+
     Value *S1S2 = IRB.CreateAnd(S1, S2);
     Value *V1S2 = IRB.CreateAnd(V1, S2);
     Value *S1V2 = IRB.CreateAnd(S1, V2);
 
-    if (V1->getType() != S1->getType()) {
-      V1 = IRB.CreateIntCast(V1, S1->getType(), false);
-      V2 = IRB.CreateIntCast(V2, S2->getType(), false);
-    }
-
     return IRB.CreateOr({S1S2, V1S2, S1V2});
   }
 
@@ -2662,10 +2670,16 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     Value *S2 = getShadow(&I, 1);
     Value *V1 = I.getOperand(0);
     Value *V2 = I.getOperand(1);
-    if (V1->getType() != S1->getType()) {
-      V1 = IRB.CreateIntCast(V1, S1->getType(), false);
-      V2 = IRB.CreateIntCast(V2, S2->getType(), false);
-    }
+
+    // "The two arguments to the ‘or’ instruction must be integer or vector
+    //  of integer values. Both arguments must have identical types."
+    assert(V1->getType()->isIntOrIntVectorTy());
+    assert(V2->getType()->isIntOrIntVectorTy());
+
+    // Conveniently, getShadowTy() of Int/IntVector returns the original type.
+    assert(V1->getType() == V2->getType());
+    assert(V1->getType() == S1->getType());
+    assert(V2->getType() == S2->getType());
 
     Value *NotV1 = IRB.CreateNot(V1);
     Value *NotV2 = IRB.CreateNot(V2);

>From 0aaa5447fb233c09416dbb23ab20e907a4a92ed1 Mon Sep 17 00:00:00 2001
From: Thurston Dang <thurston at google.com>
Date: Fri, 13 Mar 2026 19:45:23 +0000
Subject: [PATCH 2/4] Recommend CreateAppToShadowCast

---
 llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index 7a8cf5fe1e926..a6e17a28a100b 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -2624,7 +2624,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     //  of integer values. Both arguments must have identical types."
     //
     // We enforce this condition for all callers to handleBitwiseAnd(); callers
-    // with non-integer types should call CreateIntCast() themselves.
+    // with non-integer types should call CreateAppToShadowCast() themselves.
     assert(V1->getType()->isIntOrIntVectorTy());
     assert(V2->getType()->isIntOrIntVectorTy());
 

>From 00a07c5888848002383fda34f34738c790725cce Mon Sep 17 00:00:00 2001
From: Thurston Dang <thurston at google.com>
Date: Fri, 13 Mar 2026 19:46:58 +0000
Subject: [PATCH 3/4] Move assert to under correct comment

---
 llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index a6e17a28a100b..3e9310d6915e2 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -2627,9 +2627,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     // with non-integer types should call CreateAppToShadowCast() themselves.
     assert(V1->getType()->isIntOrIntVectorTy());
     assert(V2->getType()->isIntOrIntVectorTy());
+    assert(V1->getType() == V2->getType());
 
     // Conveniently, getShadowTy() of Int/IntVector returns the original type.
-    assert(V1->getType() == V2->getType());
     assert(V1->getType() == S1->getType());
     assert(V2->getType() == S2->getType());
 
@@ -2675,9 +2675,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     //  of integer values. Both arguments must have identical types."
     assert(V1->getType()->isIntOrIntVectorTy());
     assert(V2->getType()->isIntOrIntVectorTy());
+    assert(V1->getType() == V2->getType());
 
     // Conveniently, getShadowTy() of Int/IntVector returns the original type.
-    assert(V1->getType() == V2->getType());
     assert(V1->getType() == S1->getType());
     assert(V2->getType() == S2->getType());
 

>From fca24a3273c64b4c5bdabe7dc2ef277e8b607b5b Mon Sep 17 00:00:00 2001
From: Thurston Dang <thurston at google.com>
Date: Fri, 13 Mar 2026 19:48:46 +0000
Subject: [PATCH 4/4] Remove redundant check

---
 llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index 3e9310d6915e2..f7d383a26dcab 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -2626,7 +2626,6 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     // We enforce this condition for all callers to handleBitwiseAnd(); callers
     // with non-integer types should call CreateAppToShadowCast() themselves.
     assert(V1->getType()->isIntOrIntVectorTy());
-    assert(V2->getType()->isIntOrIntVectorTy());
     assert(V1->getType() == V2->getType());
 
     // Conveniently, getShadowTy() of Int/IntVector returns the original type.
@@ -2674,7 +2673,6 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     // "The two arguments to the ‘or’ instruction must be integer or vector
     //  of integer values. Both arguments must have identical types."
     assert(V1->getType()->isIntOrIntVectorTy());
-    assert(V2->getType()->isIntOrIntVectorTy());
     assert(V1->getType() == V2->getType());
 
     // Conveniently, getShadowTy() of Int/IntVector returns the original type.