[llvm] [ScalarizeMaskedMemIntrin] Preserve metadata on constant-mask scalarization (PR #206661)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 30 00:04:34 PDT 2026
https://github.com/ayrai-gb created https://github.com/llvm/llvm-project/pull/206661
`ScalarizeMaskedMemIntrin` lowers `llvm.masked.{load,store,gather,scatter}` into scalar memory operations when the target lacks native masked-memory support. Each of `scalarizeMaskedLoad`, `scalarizeMaskedStore`, `scalarizeMaskedGather`, and `scalarizeMaskedScatter` has three lowering paths:
* An all-true shortcut that emits a single vector load/store.
* A predicated path that emits a single conditional vector load/store.
* A constant-mask fast path that emits one scalar load/store per active lane.
The all-true and predicated paths preserve the original intrinsic's metadata via `copyMetadata(*CI)`. The constant-mask fast path does not: the per-lane scalar loads/stores are created without any metadata, silently dropping everything attached to the intrinsic, including `!range`, `!tbaa`, `!nontemporal`, `!alias.scope`/`!noalias`, `!invariant.load`, etc.
This is a missed optimization rather than a correctness issue - the loss of metadata is the conservative direction but it is observable. For example, a `!range` on a `masked.load` may enable an `InstCombine` fold that no longer fires after scalarization, and an `!nontemporal` store loses its streaming-store hint, causing the X86 backend to emit a cached `MOV` instead of `MOVNT*`.
Copy the intrinsic metadata onto each scalar load/store created by the four constant-mask fast paths, bringing them in line with the existing all-true and predicated paths.
Added `test/Transforms/ScalarizeMaskedMemIntrin/X86/preserve-metadata-const-mask.ll`, covering `load`, `store`, `gather`, and `scatter` with a constant mask and checking that `!range`, `!nontemporal`, `!alias.scope`, and `!noalias` are preserved on the scalarized accesses.
Note: the dynamic per-element (bit-test) paths have the same omission and are left for a follow-up; this patch is intentionally scoped to the constant-mask fast paths reported here.
Found via @jlebar's X86 LLVM bug-hunt / FuzzX effort:
* https://github.com/SemiAnalysisAI/FuzzX/tree/master/x86/bugs/180-scalarize-masked-mem-drops-metadata-const-mask
cc @jlebar
>From 061a6addb68784bbeac42955b7497f6ae723e302 Mon Sep 17 00:00:00 2001
From: Ayush Rai <your-email at amd.com>
Date: Tue, 30 Jun 2026 10:49:36 +0530
Subject: [PATCH] [ScalarizeMaskedMemIntrin] Preserve metadata on constant-mask
fast paths
---
.../Scalar/ScalarizeMaskedMemIntrin.cpp | 9 ++-
.../X86/preserve-metadata-const-mask.ll | 56 +++++++++++++++++++
2 files changed, 63 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/preserve-metadata-const-mask.ll
diff --git a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
index da9ceb4f440e5..663b11a20c028 100644
--- a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
+++ b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
@@ -186,6 +186,7 @@ static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence,
continue;
Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Gep, AdjustedAlignVal);
+ Load->copyMetadata(*CI);
VResult = Builder.CreateInsertElement(VResult, Load, Idx);
}
CI->replaceAllUsesWith(VResult);
@@ -352,7 +353,9 @@ static void scalarizeMaskedStore(const DataLayout &DL, bool HasBranchDivergence,
continue;
Value *OneElt = Builder.CreateExtractElement(Src, Idx);
Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
- Builder.CreateAlignedStore(OneElt, Gep, AdjustedAlignVal);
+ StoreInst *Store =
+ Builder.CreateAlignedStore(OneElt, Gep, AdjustedAlignVal);
+ Store->copyMetadata(*CI);
}
CI->eraseFromParent();
return;
@@ -497,6 +500,7 @@ static void scalarizeMaskedGather(const DataLayout &DL,
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
LoadInst *Load =
Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx));
+ Load->copyMetadata(*CI);
VResult =
Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx));
}
@@ -635,7 +639,8 @@ static void scalarizeMaskedScatter(const DataLayout &DL,
Value *OneElt =
Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
- Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
+ StoreInst *Store = Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
+ Store->copyMetadata(*CI);
}
CI->eraseFromParent();
return;
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/preserve-metadata-const-mask.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/preserve-metadata-const-mask.ll
new file mode 100644
index 0000000000000..f7b4cb5de8187
--- /dev/null
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/preserve-metadata-const-mask.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S %s -passes=scalarize-masked-mem-intrin -mtriple=x86_64-linux-gnu | FileCheck %s
+
+; The constant-mask scalarization fast paths must propagate the metadata of the
+; original masked intrinsic onto the scalar loads/stores they create, matching
+; the all-true and predicated paths.
+
+define <2 x i64> @masked_load_const_mask(ptr %p, <2 x i64> %passthru) {
+; CHECK-LABEL: @masked_load_const_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 8, !range [[RNG0:![0-9]+]], !alias.scope [[META1:![0-9]+]], !noalias [[META1]], !nontemporal [[META4:![0-9]+]]
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP2]], i64 0
+; CHECK-NEXT: ret <2 x i64> [[TMP3]]
+;
+ %ret = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr %p, i32 8, <2 x i1> <i1 true, i1 false>, <2 x i64> %passthru), !range !0, !nontemporal !1, !alias.scope !2, !noalias !2
+ ret <2 x i64> %ret
+}
+
+define void @masked_store_const_mask(ptr %p, <2 x i64> %val) {
+; CHECK-LABEL: @masked_store_const_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i64> [[VAL:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 0
+; CHECK-NEXT: store i64 [[TMP1]], ptr [[TMP2]], align 8, !alias.scope [[META1]], !noalias [[META1]], !nontemporal [[META4]]
+; CHECK-NEXT: ret void
+;
+ call void @llvm.masked.store.v2i64.p0(<2 x i64> %val, ptr %p, i32 8, <2 x i1> <i1 true, i1 false>), !nontemporal !1, !alias.scope !2, !noalias !2
+ ret void
+}
+
+define <2 x i64> @masked_gather_const_mask(<2 x ptr> %ptrs, <2 x i64> %passthru) {
+; CHECK-LABEL: @masked_gather_const_mask(
+; CHECK-NEXT: [[PTR0:%.*]] = extractelement <2 x ptr> [[PTRS:%.*]], i64 0
+; CHECK-NEXT: [[LOAD0:%.*]] = load i64, ptr [[PTR0]], align 8, !range [[RNG0]], !alias.scope [[META1]], !noalias [[META1]], !nontemporal [[META4]]
+; CHECK-NEXT: [[RES0:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[LOAD0]], i64 0
+; CHECK-NEXT: ret <2 x i64> [[RES0]]
+;
+ %ret = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> %ptrs, i32 8, <2 x i1> <i1 true, i1 false>, <2 x i64> %passthru), !range !0, !nontemporal !1, !alias.scope !2, !noalias !2
+ ret <2 x i64> %ret
+}
+
+define void @masked_scatter_const_mask(<2 x ptr> %ptrs, <2 x i64> %val) {
+; CHECK-LABEL: @masked_scatter_const_mask(
+; CHECK-NEXT: [[ELT0:%.*]] = extractelement <2 x i64> [[VAL:%.*]], i64 0
+; CHECK-NEXT: [[PTR0:%.*]] = extractelement <2 x ptr> [[PTRS:%.*]], i64 0
+; CHECK-NEXT: store i64 [[ELT0]], ptr [[PTR0]], align 8, !alias.scope [[META1]], !noalias [[META1]], !nontemporal [[META4]]
+; CHECK-NEXT: ret void
+;
+ call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> %val, <2 x ptr> %ptrs, i32 8, <2 x i1> <i1 true, i1 false>), !nontemporal !1, !alias.scope !2, !noalias !2
+ ret void
+}
+
+!0 = !{i64 0, i64 100}
+!1 = !{i32 1}
+!2 = !{!3}
+!3 = distinct !{!3, !4}
+!4 = distinct !{!4}
More information about the llvm-commits
mailing list