[llvm] 5a194c1 - [msan] Sharpen instrumentation of Intrinsic::{ctlz,cttz} (#145609)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 25 09:30:03 PDT 2025
Author: Thurston Dang
Date: 2025-06-25T09:29:59-07:00
New Revision: 5a194c1fd97b10fdbdbd8ada85372d978c9ff3c4
URL: https://github.com/llvm/llvm-project/commit/5a194c1fd97b10fdbdbd8ada85372d978c9ff3c4
DIFF: https://github.com/llvm/llvm-project/commit/5a194c1fd97b10fdbdbd8ada85372d978c9ff3c4.diff
LOG: [msan] Sharpen instrumentation of Intrinsic::{ctlz,cttz} (#145609)
The current instrumentation of Intrinsic::{ctlz,cttz} has false positives. For example, consider `ctlz(0001 11??)` whereby `0` and `1` denotes initialized bits (with concrete values of 0 and 1 respectively) and `?` denotes an uninitialized bit. The result (of 3) is well-defined and the shadow ought to be fully initialized, but the current instrumentation marks it as fully uninitialized.
This patch improves the fidelity of the instrumentation by comparing the number of leading (for ctlz; trailing for cttz) zeros in the concrete value and the shadow.
This patch also renames the function from 'handleCountZeroes' to 'handleLeadingTrailingCountZeros', to clarify that the intrinsics handled do not count all the zeros (unlike `llvm.ctpop`, which counts all the 1s).
Added:
Modified:
llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
llvm/test/Instrumentation/MemorySanitizer/count-zeroes.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index a6f9992383cd3..1a76898bd61c6 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -3294,22 +3294,51 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
setOrigin(&I, getOrigin(Op));
}
- void handleCountZeroes(IntrinsicInst &I) {
+ // Uninitialized bits are ok if they appear after the leading/trailing 0's
+ // and a 1. If the input is all zero, it is fully initialized iff
+ // !is_zero_poison.
+ //
+ // e.g., for ctlz, with little-endian, if 0/1 are initialized bits with
+ // concrete value 0/1, and ? is an uninitialized bit:
+ // - 0001 0??? is fully initialized
+ // - 000? ???? is fully uninitialized (*)
+ // - ???? ???? is fully uninitialized
+ // - 0000 0000 is fully uninitialized if is_zero_poison,
+ // fully initialized otherwise
+ //
+ // (*) TODO: arguably, since the number of zeros is in the range [3, 8], we
+ // only need to poison 4 bits.
+ //
+ // OutputShadow =
+ // ((ConcreteZerosCount >= ShadowZerosCount) && !AllZeroShadow)
+ // || (is_zero_poison && AllZeroSrc)
+ void handleCountLeadingTrailingZeros(IntrinsicInst &I) {
IRBuilder<> IRB(&I);
Value *Src = I.getArgOperand(0);
+ Value *SrcShadow = getShadow(Src);
- // Set the Output shadow based on input Shadow
- Value *BoolShadow = IRB.CreateIsNotNull(getShadow(Src), "_mscz_bs");
+ Value *False = IRB.getInt1(false);
+ Value *ConcreteZerosCount = IRB.CreateIntrinsic(
+ I.getType(), I.getIntrinsicID(), {Src, /*is_zero_poison=*/False});
+ Value *ShadowZerosCount = IRB.CreateIntrinsic(
+ I.getType(), I.getIntrinsicID(), {SrcShadow, /*is_zero_poison=*/False});
+
+ Value *CompareConcreteZeros = IRB.CreateICmpUGE(
+ ConcreteZerosCount, ShadowZerosCount, "_mscz_cmp_zeros");
+
+ Value *NotAllZeroShadow =
+ IRB.CreateIsNotNull(SrcShadow, "_mscz_shadow_not_null");
+ Value *OutputShadow =
+ IRB.CreateAnd(CompareConcreteZeros, NotAllZeroShadow, "_mscz_main");
// If zero poison is requested, mix in with the shadow
Constant *IsZeroPoison = cast<Constant>(I.getOperand(1));
if (!IsZeroPoison->isZeroValue()) {
Value *BoolZeroPoison = IRB.CreateIsNull(Src, "_mscz_bzp");
- BoolShadow = IRB.CreateOr(BoolShadow, BoolZeroPoison, "_mscz_bs");
+ OutputShadow = IRB.CreateOr(OutputShadow, BoolZeroPoison, "_mscz_bs");
}
- Value *OutputShadow =
- IRB.CreateSExt(BoolShadow, getShadowTy(Src), "_mscz_os");
+ OutputShadow = IRB.CreateSExt(OutputShadow, getShadowTy(Src), "_mscz_os");
setShadow(&I, OutputShadow);
setOriginForNaryOp(I);
@@ -4726,7 +4755,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
break;
case Intrinsic::ctlz:
case Intrinsic::cttz:
- handleCountZeroes(I);
+ handleCountLeadingTrailingZeros(I);
break;
case Intrinsic::masked_compressstore:
handleMaskedCompressStore(I);
diff --git a/llvm/test/Instrumentation/MemorySanitizer/count-zeroes.ll b/llvm/test/Instrumentation/MemorySanitizer/count-zeroes.ll
index 73e047e68ddc6..c51dc1a373629 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/count-zeroes.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/count-zeroes.ll
@@ -9,10 +9,14 @@ define i64 @test_ctlz_i64_zeropoison(i64 %v) #0 {
; CHECK-LABEL: @test_ctlz_i64_zeropoison(
; CHECK-NEXT: [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
; CHECK-NEXT: call void @llvm.donothing()
-; CHECK-NEXT: [[_MSCZ_BS:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT: [[_MSCZ_BZP:%.*]] = icmp eq i64 [[V:%.*]], 0
-; CHECK-NEXT: [[_MSCZ_BS1:%.*]] = or i1 [[_MSCZ_BS]], [[_MSCZ_BZP]]
-; CHECK-NEXT: [[_MSCZ_OS:%.*]] = sext i1 [[_MSCZ_BS1]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.ctlz.i64(i64 [[V:%.*]], i1 false)
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.ctlz.i64(i64 [[TMP1]], i1 false)
+; CHECK-NEXT: [[_MSCZ_CMP_ZEROS:%.*]] = icmp uge i64 [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[_MSCZ_SHADOW_NOT_NULL:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT: [[_MSCZ_MAIN:%.*]] = and i1 [[_MSCZ_CMP_ZEROS]], [[_MSCZ_SHADOW_NOT_NULL]]
+; CHECK-NEXT: [[_MSCZ_BZP:%.*]] = icmp eq i64 [[V]], 0
+; CHECK-NEXT: [[_MSCZ_BS:%.*]] = or i1 [[_MSCZ_MAIN]], [[_MSCZ_BZP]]
+; CHECK-NEXT: [[_MSCZ_OS:%.*]] = sext i1 [[_MSCZ_BS]] to i64
; CHECK-NEXT: [[RES:%.*]] = call i64 @llvm.ctlz.i64(i64 [[V]], i1 true)
; CHECK-NEXT: store i64 [[_MSCZ_OS]], ptr @__msan_retval_tls, align 8
; CHECK-NEXT: ret i64 [[RES]]
@@ -24,9 +28,13 @@ define i64 @test_ctlz_i64_nozeropoison(i64 %v) #0 {
; CHECK-LABEL: @test_ctlz_i64_nozeropoison(
; CHECK-NEXT: [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
; CHECK-NEXT: call void @llvm.donothing()
-; CHECK-NEXT: [[_MSCZ_BS:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT: [[_MSCZ_OS:%.*]] = sext i1 [[_MSCZ_BS]] to i64
-; CHECK-NEXT: [[RES:%.*]] = call i64 @llvm.ctlz.i64(i64 [[V:%.*]], i1 false)
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.ctlz.i64(i64 [[V:%.*]], i1 false)
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.ctlz.i64(i64 [[TMP1]], i1 false)
+; CHECK-NEXT: [[_MSCZ_CMP_ZEROS:%.*]] = icmp uge i64 [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[_MSCZ_SHADOW_NOT_NULL:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT: [[_MSCZ_MAIN:%.*]] = and i1 [[_MSCZ_CMP_ZEROS]], [[_MSCZ_SHADOW_NOT_NULL]]
+; CHECK-NEXT: [[_MSCZ_OS:%.*]] = sext i1 [[_MSCZ_MAIN]] to i64
+; CHECK-NEXT: [[RES:%.*]] = call i64 @llvm.ctlz.i64(i64 [[V]], i1 false)
; CHECK-NEXT: store i64 [[_MSCZ_OS]], ptr @__msan_retval_tls, align 8
; CHECK-NEXT: ret i64 [[RES]]
;
@@ -39,10 +47,14 @@ define <2 x i64> @test_ctlz_v2i64_zeropoison(<2 x i64> %v) #0 {
; CHECK-LABEL: @test_ctlz_v2i64_zeropoison(
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
; CHECK-NEXT: call void @llvm.donothing()
-; CHECK-NEXT: [[_MSCZ_BS:%.*]] = icmp ne <2 x i64> [[TMP1]], zeroinitializer
-; CHECK-NEXT: [[_MSCZ_BZP:%.*]] = icmp eq <2 x i64> [[V:%.*]], zeroinitializer
-; CHECK-NEXT: [[_MSCZ_BS1:%.*]] = or <2 x i1> [[_MSCZ_BS]], [[_MSCZ_BZP]]
-; CHECK-NEXT: [[_MSCZ_OS:%.*]] = sext <2 x i1> [[_MSCZ_BS1]] to <2 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[V:%.*]], i1 false)
+; CHECK-NEXT: [[TMP3:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[TMP1]], i1 false)
+; CHECK-NEXT: [[_MSCZ_CMP_ZEROS:%.*]] = icmp uge <2 x i64> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[_MSCZ_SHADOW_NOT_NULL:%.*]] = icmp ne <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT: [[_MSCZ_MAIN:%.*]] = and <2 x i1> [[_MSCZ_CMP_ZEROS]], [[_MSCZ_SHADOW_NOT_NULL]]
+; CHECK-NEXT: [[_MSCZ_BZP:%.*]] = icmp eq <2 x i64> [[V]], zeroinitializer
+; CHECK-NEXT: [[_MSCZ_BS:%.*]] = or <2 x i1> [[_MSCZ_MAIN]], [[_MSCZ_BZP]]
+; CHECK-NEXT: [[_MSCZ_OS:%.*]] = sext <2 x i1> [[_MSCZ_BS]] to <2 x i64>
; CHECK-NEXT: [[RES:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[V]], i1 true)
; CHECK-NEXT: store <2 x i64> [[_MSCZ_OS]], ptr @__msan_retval_tls, align 8
; CHECK-NEXT: ret <2 x i64> [[RES]]
@@ -54,9 +66,13 @@ define <2 x i64> @test_ctlz_v2i64_nozeropoison(<2 x i64> %v) #0 {
; CHECK-LABEL: @test_ctlz_v2i64_nozeropoison(
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
; CHECK-NEXT: call void @llvm.donothing()
-; CHECK-NEXT: [[_MSCZ_BS:%.*]] = icmp ne <2 x i64> [[TMP1]], zeroinitializer
-; CHECK-NEXT: [[_MSCZ_OS:%.*]] = sext <2 x i1> [[_MSCZ_BS]] to <2 x i64>
-; CHECK-NEXT: [[RES:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[V:%.*]], i1 false)
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[V:%.*]], i1 false)
+; CHECK-NEXT: [[TMP3:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[TMP1]], i1 false)
+; CHECK-NEXT: [[_MSCZ_CMP_ZEROS:%.*]] = icmp uge <2 x i64> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[_MSCZ_SHADOW_NOT_NULL:%.*]] = icmp ne <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT: [[_MSCZ_MAIN:%.*]] = and <2 x i1> [[_MSCZ_CMP_ZEROS]], [[_MSCZ_SHADOW_NOT_NULL]]
+; CHECK-NEXT: [[_MSCZ_OS:%.*]] = sext <2 x i1> [[_MSCZ_MAIN]] to <2 x i64>
+; CHECK-NEXT: [[RES:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[V]], i1 false)
; CHECK-NEXT: store <2 x i64> [[_MSCZ_OS]], ptr @__msan_retval_tls, align 8
; CHECK-NEXT: ret <2 x i64> [[RES]]
;
@@ -69,10 +85,14 @@ define i64 @test_cttz_i64_zeropoison(i64 %v) #0 {
; CHECK-LABEL: @test_cttz_i64_zeropoison(
; CHECK-NEXT: [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
; CHECK-NEXT: call void @llvm.donothing()
-; CHECK-NEXT: [[_MSCZ_BS:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT: [[_MSCZ_BZP:%.*]] = icmp eq i64 [[V:%.*]], 0
-; CHECK-NEXT: [[_MSCZ_BS1:%.*]] = or i1 [[_MSCZ_BS]], [[_MSCZ_BZP]]
-; CHECK-NEXT: [[_MSCZ_OS:%.*]] = sext i1 [[_MSCZ_BS1]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.cttz.i64(i64 [[V:%.*]], i1 false)
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.cttz.i64(i64 [[TMP1]], i1 false)
+; CHECK-NEXT: [[_MSCZ_CMP_ZEROS:%.*]] = icmp uge i64 [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[_MSCZ_SHADOW_NOT_NULL:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT: [[_MSCZ_MAIN:%.*]] = and i1 [[_MSCZ_CMP_ZEROS]], [[_MSCZ_SHADOW_NOT_NULL]]
+; CHECK-NEXT: [[_MSCZ_BZP:%.*]] = icmp eq i64 [[V]], 0
+; CHECK-NEXT: [[_MSCZ_BS:%.*]] = or i1 [[_MSCZ_MAIN]], [[_MSCZ_BZP]]
+; CHECK-NEXT: [[_MSCZ_OS:%.*]] = sext i1 [[_MSCZ_BS]] to i64
; CHECK-NEXT: [[RES:%.*]] = call i64 @llvm.cttz.i64(i64 [[V]], i1 true)
; CHECK-NEXT: store i64 [[_MSCZ_OS]], ptr @__msan_retval_tls, align 8
; CHECK-NEXT: ret i64 [[RES]]
@@ -84,9 +104,13 @@ define i64 @test_cttz_i64_nozeropoison(i64 %v) #0 {
; CHECK-LABEL: @test_cttz_i64_nozeropoison(
; CHECK-NEXT: [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
; CHECK-NEXT: call void @llvm.donothing()
-; CHECK-NEXT: [[_MSCZ_BS:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT: [[_MSCZ_OS:%.*]] = sext i1 [[_MSCZ_BS]] to i64
-; CHECK-NEXT: [[RES:%.*]] = call i64 @llvm.cttz.i64(i64 [[V:%.*]], i1 false)
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.cttz.i64(i64 [[V:%.*]], i1 false)
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.cttz.i64(i64 [[TMP1]], i1 false)
+; CHECK-NEXT: [[_MSCZ_CMP_ZEROS:%.*]] = icmp uge i64 [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[_MSCZ_SHADOW_NOT_NULL:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT: [[_MSCZ_MAIN:%.*]] = and i1 [[_MSCZ_CMP_ZEROS]], [[_MSCZ_SHADOW_NOT_NULL]]
+; CHECK-NEXT: [[_MSCZ_OS:%.*]] = sext i1 [[_MSCZ_MAIN]] to i64
+; CHECK-NEXT: [[RES:%.*]] = call i64 @llvm.cttz.i64(i64 [[V]], i1 false)
; CHECK-NEXT: store i64 [[_MSCZ_OS]], ptr @__msan_retval_tls, align 8
; CHECK-NEXT: ret i64 [[RES]]
;
@@ -99,10 +123,14 @@ define <2 x i64> @test_cttz_v2i64_zeropoison(<2 x i64> %v) #0 {
; CHECK-LABEL: @test_cttz_v2i64_zeropoison(
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
; CHECK-NEXT: call void @llvm.donothing()
-; CHECK-NEXT: [[_MSCZ_BS:%.*]] = icmp ne <2 x i64> [[TMP1]], zeroinitializer
-; CHECK-NEXT: [[_MSCZ_BZP:%.*]] = icmp eq <2 x i64> [[V:%.*]], zeroinitializer
-; CHECK-NEXT: [[_MSCZ_BS1:%.*]] = or <2 x i1> [[_MSCZ_BS]], [[_MSCZ_BZP]]
-; CHECK-NEXT: [[_MSCZ_OS:%.*]] = sext <2 x i1> [[_MSCZ_BS1]] to <2 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> [[V:%.*]], i1 false)
+; CHECK-NEXT: [[TMP3:%.*]] = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> [[TMP1]], i1 false)
+; CHECK-NEXT: [[_MSCZ_CMP_ZEROS:%.*]] = icmp uge <2 x i64> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[_MSCZ_SHADOW_NOT_NULL:%.*]] = icmp ne <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT: [[_MSCZ_MAIN:%.*]] = and <2 x i1> [[_MSCZ_CMP_ZEROS]], [[_MSCZ_SHADOW_NOT_NULL]]
+; CHECK-NEXT: [[_MSCZ_BZP:%.*]] = icmp eq <2 x i64> [[V]], zeroinitializer
+; CHECK-NEXT: [[_MSCZ_BS:%.*]] = or <2 x i1> [[_MSCZ_MAIN]], [[_MSCZ_BZP]]
+; CHECK-NEXT: [[_MSCZ_OS:%.*]] = sext <2 x i1> [[_MSCZ_BS]] to <2 x i64>
; CHECK-NEXT: [[RES:%.*]] = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> [[V]], i1 true)
; CHECK-NEXT: store <2 x i64> [[_MSCZ_OS]], ptr @__msan_retval_tls, align 8
; CHECK-NEXT: ret <2 x i64> [[RES]]
@@ -114,9 +142,13 @@ define <2 x i64> @test_cttz_v2i64_nozeropoison(<2 x i64> %v) #0 {
; CHECK-LABEL: @test_cttz_v2i64_nozeropoison(
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
; CHECK-NEXT: call void @llvm.donothing()
-; CHECK-NEXT: [[_MSCZ_BS:%.*]] = icmp ne <2 x i64> [[TMP1]], zeroinitializer
-; CHECK-NEXT: [[_MSCZ_OS:%.*]] = sext <2 x i1> [[_MSCZ_BS]] to <2 x i64>
-; CHECK-NEXT: [[RES:%.*]] = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> [[V:%.*]], i1 false)
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> [[V:%.*]], i1 false)
+; CHECK-NEXT: [[TMP3:%.*]] = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> [[TMP1]], i1 false)
+; CHECK-NEXT: [[_MSCZ_CMP_ZEROS:%.*]] = icmp uge <2 x i64> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[_MSCZ_SHADOW_NOT_NULL:%.*]] = icmp ne <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT: [[_MSCZ_MAIN:%.*]] = and <2 x i1> [[_MSCZ_CMP_ZEROS]], [[_MSCZ_SHADOW_NOT_NULL]]
+; CHECK-NEXT: [[_MSCZ_OS:%.*]] = sext <2 x i1> [[_MSCZ_MAIN]] to <2 x i64>
+; CHECK-NEXT: [[RES:%.*]] = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> [[V]], i1 false)
; CHECK-NEXT: store <2 x i64> [[_MSCZ_OS]], ptr @__msan_retval_tls, align 8
; CHECK-NEXT: ret <2 x i64> [[RES]]
;
More information about the llvm-commits
mailing list