[llvm] [ScalarizeMaskedMemIntrin] Preserve metadata during scalarization (PR #218753)
Yonah Goldberg via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 25 13:50:32 PDT 2026
https://github.com/YonahGoldberg updated https://github.com/llvm/llvm-project/pull/218753
>From 9e8cbe5d20c9bac6494ddc003158a8e78c3785b5 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 25 Aug 2026 18:40:01 +0000
Subject: [PATCH 1/3] support metadata
---
.../Scalar/ScalarizeMaskedMemIntrin.cpp | 112 ++++-
.../ScalarizeMaskedMemIntrin/X86/metadata.ll | 425 ++++++++++++++++++
2 files changed, 523 insertions(+), 14 deletions(-)
create mode 100644 llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/metadata.ll
diff --git a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
index da9ceb4f440e5..5eeac40f15275 100644
--- a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
+++ b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
@@ -28,6 +28,7 @@
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Metadata.h"
#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
@@ -111,6 +112,64 @@ static unsigned adjustForEndian(const DataLayout &DL, unsigned VectorWidth,
return DL.isBigEndian() ? VectorWidth - 1 - Idx : Idx;
}
+static void copyMemCacheHint(Instruction &Dest, const Instruction &Source,
+ unsigned SourcePtrOperand,
+ unsigned DestPtrOperand) {
+ MDNode *CacheHint = Source.getMetadata(LLVMContext::MD_mem_cache_hint);
+ // These intrinsics have a single memory operand.
+ if (!CacheHint || CacheHint->getNumOperands() != 2)
+ return;
+
+ auto *OperandNo = mdconst::extract<ConstantInt>(CacheHint->getOperand(0));
+ if (OperandNo->getZExtValue() != SourcePtrOperand)
+ return;
+
+ Metadata *DestOperandNo = ConstantAsMetadata::get(ConstantInt::get(
+ Type::getInt32Ty(Dest.getContext()), DestPtrOperand));
+ Dest.setMetadata(
+ LLVMContext::MD_mem_cache_hint,
+ MDNode::get(Dest.getContext(),
+ {DestOperandNo, CacheHint->getOperand(1)}));
+}
+
+static void copyMetadataForMemoryAccess(Instruction &Dest,
+ const Instruction &Source,
+ unsigned SourcePtrOperand,
+ unsigned DestPtrOperand) {
+ // Only propagate metadata that is valid on each constituent memory access.
+ // In particular, do not copy metadata whose meaning is tied to the call,
+ // such as !prof or !callsite.
+ Dest.copyMetadata(
+ Source,
+ {LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope,
+ LLVMContext::MD_noalias, LLVMContext::MD_nontemporal,
+ LLVMContext::MD_mem_parallel_loop_access, LLVMContext::MD_access_group,
+ LLVMContext::MD_annotation, LLVMContext::MD_nosanitize,
+ LLVMContext::MD_mmra, LLVMContext::MD_noalias_addrspace});
+ copyMemCacheHint(Dest, Source, SourcePtrOperand, DestPtrOperand);
+}
+
+static void copyMetadataForScalarizedLoad(LoadInst &Dest,
+ const Instruction &Source,
+ unsigned SourcePtrOperand) {
+ copyMetadataForMemoryAccess(Dest, Source, SourcePtrOperand,
+ Dest.getPointerOperandIndex());
+
+ // !range applies element-wise to vectors, so the same range describes each
+ // scalar result. The other metadata here also describes the loaded result.
+ Dest.copyMetadata(Source,
+ {LLVMContext::MD_fpmath, LLVMContext::MD_range,
+ LLVMContext::MD_invariant_load});
+}
+
+static void copyMetadataForScalarizedStore(StoreInst &Dest,
+ const Instruction &Source,
+ unsigned SourcePtrOperand) {
+ copyMetadataForMemoryAccess(Dest, Source, SourcePtrOperand,
+ Dest.getPointerOperandIndex());
+}
+
+
// Translate a masked load intrinsic like
// <16 x i32 > @llvm.masked.load( <16 x i32>* %addr,
// <16 x i1> %mask, <16 x i32> %passthru)
@@ -165,7 +224,7 @@ static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence,
// Short-cut if the mask is all-true.
if (isa<Constant>(Mask) && cast<Constant>(Mask)->isAllOnesValue()) {
LoadInst *NewI = Builder.CreateAlignedLoad(VecType, Ptr, AlignVal);
- NewI->copyMetadata(*CI);
+ copyMetadataForScalarizedLoad(*NewI, *CI, /*SourcePtrOperand=*/0);
NewI->takeName(CI);
CI->replaceAllUsesWith(NewI);
CI->eraseFromParent();
@@ -186,6 +245,7 @@ static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence,
continue;
Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Gep, AdjustedAlignVal);
+ copyMetadataForScalarizedLoad(*Load, *CI, /*SourcePtrOperand=*/0);
VResult = Builder.CreateInsertElement(VResult, Load, Idx);
}
CI->replaceAllUsesWith(VResult);
@@ -208,7 +268,7 @@ static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence,
Builder.SetInsertPoint(CondBlock->getTerminator());
LoadInst *Load = Builder.CreateAlignedLoad(VecType, Ptr, AlignVal,
CI->getName() + ".cond.load");
- Load->copyMetadata(*CI);
+ copyMetadataForScalarizedLoad(*Load, *CI, /*SourcePtrOperand=*/0);
BasicBlock *PostLoad = ThenTerm->getSuccessor(0);
Builder.SetInsertPoint(PostLoad, PostLoad->begin());
@@ -267,6 +327,7 @@ static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence,
Builder.SetInsertPoint(CondBlock->getTerminator());
Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Gep, AdjustedAlignVal);
+ copyMetadataForScalarizedLoad(*Load, *CI, /*SourcePtrOperand=*/0);
Value *NewVResult = Builder.CreateInsertElement(VResult, Load, Idx);
// Create "else" block, fill it in the next iteration
@@ -336,7 +397,9 @@ static void scalarizeMaskedStore(const DataLayout &DL, bool HasBranchDivergence,
if (isa<Constant>(Mask) && cast<Constant>(Mask)->isAllOnesValue()) {
StoreInst *Store = Builder.CreateAlignedStore(Src, Ptr, AlignVal);
Store->takeName(CI);
- Store->copyMetadata(*CI);
+ copyMetadataForScalarizedStore(*Store, *CI, /*SourcePtrOperand=*/1);
+ // This is a one-to-one replacement, so the assignment link remains valid.
+ Store->copyMetadata(*CI, LLVMContext::MD_DIAssignID);
CI->eraseFromParent();
return;
}
@@ -352,7 +415,9 @@ static void scalarizeMaskedStore(const DataLayout &DL, bool HasBranchDivergence,
continue;
Value *OneElt = Builder.CreateExtractElement(Src, Idx);
Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
- Builder.CreateAlignedStore(OneElt, Gep, AdjustedAlignVal);
+ StoreInst *Store =
+ Builder.CreateAlignedStore(OneElt, Gep, AdjustedAlignVal);
+ copyMetadataForScalarizedStore(*Store, *CI, /*SourcePtrOperand=*/1);
}
CI->eraseFromParent();
return;
@@ -373,7 +438,9 @@ static void scalarizeMaskedStore(const DataLayout &DL, bool HasBranchDivergence,
StoreInst *Store = Builder.CreateAlignedStore(Src, Ptr, AlignVal);
Store->takeName(CI);
- Store->copyMetadata(*CI);
+ copyMetadataForScalarizedStore(*Store, *CI, /*SourcePtrOperand=*/1);
+ // This is a one-to-one replacement, so the assignment link remains valid.
+ Store->copyMetadata(*CI, LLVMContext::MD_DIAssignID);
CI->eraseFromParent();
ModifiedDT = true;
@@ -425,7 +492,9 @@ static void scalarizeMaskedStore(const DataLayout &DL, bool HasBranchDivergence,
Builder.SetInsertPoint(CondBlock->getTerminator());
Value *OneElt = Builder.CreateExtractElement(Src, Idx);
Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
- Builder.CreateAlignedStore(OneElt, Gep, AdjustedAlignVal);
+ StoreInst *Store =
+ Builder.CreateAlignedStore(OneElt, Gep, AdjustedAlignVal);
+ copyMetadataForScalarizedStore(*Store, *CI, /*SourcePtrOperand=*/1);
// Create "else" block, fill it in the next iteration
BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
@@ -497,6 +566,7 @@ static void scalarizeMaskedGather(const DataLayout &DL,
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
LoadInst *Load =
Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx));
+ copyMetadataForScalarizedLoad(*Load, *CI, /*SourcePtrOperand=*/0);
VResult =
Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx));
}
@@ -556,6 +626,7 @@ static void scalarizeMaskedGather(const DataLayout &DL,
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
LoadInst *Load =
Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx));
+ copyMetadataForScalarizedLoad(*Load, *CI, /*SourcePtrOperand=*/0);
Value *NewVResult =
Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx));
@@ -635,7 +706,8 @@ static void scalarizeMaskedScatter(const DataLayout &DL,
Value *OneElt =
Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
- Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
+ StoreInst *Store = Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
+ copyMetadataForScalarizedStore(*Store, *CI, /*SourcePtrOperand=*/1);
}
CI->eraseFromParent();
return;
@@ -689,7 +761,8 @@ static void scalarizeMaskedScatter(const DataLayout &DL,
Builder.SetInsertPoint(CondBlock->getTerminator());
Value *OneElt = Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
- Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
+ StoreInst *Store = Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
+ copyMetadataForScalarizedStore(*Store, *CI, /*SourcePtrOperand=*/1);
// Create "else" block, fill it in the next iteration
BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
@@ -745,8 +818,10 @@ static void scalarizeMaskedExpandLoad(const DataLayout &DL,
} else {
Value *NewPtr =
Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, MemIndex);
- InsertElt = Builder.CreateAlignedLoad(EltTy, NewPtr, AdjustedAlignment,
- "Load" + Twine(Idx));
+ LoadInst *Load = Builder.CreateAlignedLoad(
+ EltTy, NewPtr, AdjustedAlignment, "Load" + Twine(Idx));
+ copyMetadataForScalarizedLoad(*Load, *CI, /*SourcePtrOperand=*/0);
+ InsertElt = Load;
ShuffleMask[Idx] = Idx;
++MemIndex;
}
@@ -804,6 +879,7 @@ static void scalarizeMaskedExpandLoad(const DataLayout &DL,
Builder.SetInsertPoint(CondBlock->getTerminator());
LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Ptr, AdjustedAlignment);
+ copyMetadataForScalarizedLoad(*Load, *CI, /*SourcePtrOperand=*/0);
Value *NewVResult = Builder.CreateInsertElement(VResult, Load, Idx);
// Move the pointer if there are more blocks to come.
@@ -874,7 +950,9 @@ static void scalarizeMaskedCompressStore(const DataLayout &DL,
Value *OneElt =
Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));
Value *NewPtr = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, MemIndex);
- Builder.CreateAlignedStore(OneElt, NewPtr, AdjustedAlignment);
+ StoreInst *Store =
+ Builder.CreateAlignedStore(OneElt, NewPtr, AdjustedAlignment);
+ copyMetadataForScalarizedStore(*Store, *CI, /*SourcePtrOperand=*/1);
++MemIndex;
}
CI->eraseFromParent();
@@ -924,7 +1002,9 @@ static void scalarizeMaskedCompressStore(const DataLayout &DL,
Builder.SetInsertPoint(CondBlock->getTerminator());
Value *OneElt = Builder.CreateExtractElement(Src, Idx);
- Builder.CreateAlignedStore(OneElt, Ptr, AdjustedAlignment);
+ StoreInst *Store =
+ Builder.CreateAlignedStore(OneElt, Ptr, AdjustedAlignment);
+ copyMetadataForScalarizedStore(*Store, *CI, /*SourcePtrOperand=*/1);
// Move the pointer if there are more blocks to come.
Value *NewPtr;
@@ -1004,9 +1084,11 @@ static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI,
continue;
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
LoadInst *Load = Builder.CreateLoad(EltTy, Ptr, "Load" + Twine(Idx));
+ copyMetadataForScalarizedLoad(*Load, *CI, /*SourcePtrOperand=*/0);
Value *Update =
CreateHistogramUpdateValue(cast<IntrinsicInst>(CI), Load, Inc);
- Builder.CreateStore(Update, Ptr);
+ StoreInst *Store = Builder.CreateStore(Update, Ptr);
+ copyMetadataForScalarizedStore(*Store, *CI, /*SourcePtrOperand=*/0);
}
CI->eraseFromParent();
return;
@@ -1026,9 +1108,11 @@ static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI,
Builder.SetInsertPoint(CondBlock->getTerminator());
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
LoadInst *Load = Builder.CreateLoad(EltTy, Ptr, "Load" + Twine(Idx));
+ copyMetadataForScalarizedLoad(*Load, *CI, /*SourcePtrOperand=*/0);
Value *UpdateOp =
CreateHistogramUpdateValue(cast<IntrinsicInst>(CI), Load, Inc);
- Builder.CreateStore(UpdateOp, Ptr);
+ StoreInst *Store = Builder.CreateStore(UpdateOp, Ptr);
+ copyMetadataForScalarizedStore(*Store, *CI, /*SourcePtrOperand=*/0);
// Create "else" block, fill it in the next iteration
BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/metadata.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/metadata.ll
new file mode 100644
index 0000000000000..79d5d54f7102b
--- /dev/null
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/metadata.ll
@@ -0,0 +1,425 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals
+; RUN: opt -S %s -passes=scalarize-masked-mem-intrin -mtriple=x86_64-linux-gnu | FileCheck %s
+
+define <2 x i64> @masked_load_variable(ptr %p, <2 x i1> %mask, <2 x i64> %passthru) {
+; CHECK-LABEL: @masked_load_variable(
+; CHECK-NEXT: [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-NEXT: [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT: br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK: cond.load:
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 0
+; CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 8, !tbaa [[TBAA4:![0-9]+]], !range [[RNG7:![0-9]+]], !invariant.load [[META8:![0-9]+]], !alias.scope [[META9:![0-9]+]], !noalias [[META9]], !nontemporal [[META12:![0-9]+]], !llvm.mem.parallel_loop_access [[META13:![0-9]+]], !llvm.access.group [[META13]], !annotation [[META14:![0-9]+]], !nosanitize [[META8]], !mmra [[META15:![0-9]+]], !noalias.addrspace [[META16:![0-9]+]], !mem.cache_hint [[META17:![0-9]+]]
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP4]], i64 0
+; CHECK-NEXT: br label [[ELSE]]
+; CHECK: else:
+; CHECK-NEXT: [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP5]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0
+; CHECK-NEXT: br i1 [[TMP7]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK: cond.load1:
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[P]], i32 1
+; CHECK-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP8]], align 8, !tbaa [[TBAA4]], !range [[RNG7]], !invariant.load [[META8]], !alias.scope [[META9]], !noalias [[META9]], !nontemporal [[META12]], !llvm.mem.parallel_loop_access [[META13]], !llvm.access.group [[META13]], !annotation [[META14]], !nosanitize [[META8]], !mmra [[META15]], !noalias.addrspace [[META16]], !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP9]], i64 1
+; CHECK-NEXT: br label [[ELSE2]]
+; CHECK: else2:
+; CHECK-NEXT: [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP10]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-NEXT: ret <2 x i64> [[RES_PHI_ELSE3]]
+;
+ %ret = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr %p, i32 8, <2 x i1> %mask, <2 x i64> %passthru), !tbaa !3, !prof !16, !range !12, !invariant.load !14, !alias.scope !6, !noalias !6, !nontemporal !11, !llvm.mem.parallel_loop_access !9, !llvm.access.group !9, !annotation !13, !nosanitize !14, !mmra !10, !noalias.addrspace !15, !mem.cache_hint !0
+ ret <2 x i64> %ret
+}
+
+define <2 x i64> @masked_load_constant(ptr %p, <2 x i64> %passthru) {
+; CHECK-LABEL: @masked_load_constant(
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 8, !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP2]], i64 1
+; CHECK-NEXT: ret <2 x i64> [[TMP3]]
+;
+ %ret = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr %p, i32 8, <2 x i1> <i1 false, i1 true>, <2 x i64> %passthru), !mem.cache_hint !0
+ ret <2 x i64> %ret
+}
+
+define <2 x i64> @masked_load_all_true(ptr %p) {
+; CHECK-LABEL: @masked_load_all_true(
+; CHECK-NEXT: [[RET:%.*]] = load <2 x i64>, ptr [[P:%.*]], align 8, !range [[RNG7]], !mem.cache_hint [[META17]]
+; CHECK-NEXT: ret <2 x i64> [[RET]]
+;
+ %ret = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr %p, i32 8, <2 x i1> <i1 true, i1 true>, <2 x i64> poison), !prof !16, !range !12, !mem.cache_hint !0
+ ret <2 x i64> %ret
+}
+
+define void @masked_store_variable(ptr %p, <2 x i1> %mask, <2 x i64> %value) {
+; CHECK-LABEL: @masked_store_variable(
+; CHECK-NEXT: [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-NEXT: [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT: br i1 [[TMP2]], label [[COND_STORE:%.*]], label [[ELSE:%.*]]
+; CHECK: cond.store:
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[VALUE:%.*]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 0
+; CHECK-NEXT: store i64 [[TMP3]], ptr [[TMP4]], align 8, !tbaa [[TBAA4]], !alias.scope [[META9]], !noalias [[META9]], !nontemporal [[META12]], !llvm.mem.parallel_loop_access [[META13]], !llvm.access.group [[META13]], !annotation [[META14]], !nosanitize [[META8]], !mmra [[META15]], !noalias.addrspace [[META16]], !mem.cache_hint [[META19:![0-9]+]]
+; CHECK-NEXT: br label [[ELSE]]
+; CHECK: else:
+; CHECK-NEXT: [[TMP5:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ne i2 [[TMP5]], 0
+; CHECK-NEXT: br i1 [[TMP6]], label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
+; CHECK: cond.store1:
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i64> [[VALUE]], i64 1
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[P]], i32 1
+; CHECK-NEXT: store i64 [[TMP7]], ptr [[TMP8]], align 8, !tbaa [[TBAA4]], !alias.scope [[META9]], !noalias [[META9]], !nontemporal [[META12]], !llvm.mem.parallel_loop_access [[META13]], !llvm.access.group [[META13]], !annotation [[META14]], !nosanitize [[META8]], !mmra [[META15]], !noalias.addrspace [[META16]], !mem.cache_hint [[META19]]
+; CHECK-NEXT: br label [[ELSE2]]
+; CHECK: else2:
+; CHECK-NEXT: ret void
+;
+ call void @llvm.masked.store.v2i64.p0(<2 x i64> %value, ptr %p, i32 8, <2 x i1> %mask), !tbaa !3, !prof !16, !alias.scope !6, !noalias !6, !nontemporal !11, !llvm.mem.parallel_loop_access !9, !llvm.access.group !9, !annotation !13, !nosanitize !14, !mmra !10, !noalias.addrspace !15, !mem.cache_hint !1
+ ret void
+}
+
+define void @masked_store_constant(ptr %p, <2 x i64> %value) {
+; CHECK-LABEL: @masked_store_constant(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i64> [[VALUE:%.*]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 1
+; CHECK-NEXT: store i64 [[TMP1]], ptr [[TMP2]], align 8, !mem.cache_hint [[META19]]
+; CHECK-NEXT: ret void
+;
+ call void @llvm.masked.store.v2i64.p0(<2 x i64> %value, ptr %p, i32 8, <2 x i1> <i1 false, i1 true>), !mem.cache_hint !1
+ ret void
+}
+
+define void @masked_store_all_true(ptr %p, <2 x i64> %value) {
+; CHECK-LABEL: @masked_store_all_true(
+; CHECK-NEXT: store <2 x i64> [[VALUE:%.*]], ptr [[P:%.*]], align 8, !DIAssignID [[DIASSIGNID20:![0-9]+]], !mem.cache_hint [[META19]]
+; CHECK-NEXT: ret void
+;
+ call void @llvm.masked.store.v2i64.p0(<2 x i64> %value, ptr %p, i32 8, <2 x i1> <i1 true, i1 true>), !DIAssignID !26, !mem.cache_hint !1
+ ret void
+}
+
+define <2 x i64> @masked_gather_variable(<2 x ptr> %p, <2 x i1> %mask, <2 x i64> %passthru) {
+; CHECK-LABEL: @masked_gather_variable(
+; CHECK-NEXT: [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-NEXT: [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT: br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK: cond.load:
+; CHECK-NEXT: [[PTR0:%.*]] = extractelement <2 x ptr> [[P:%.*]], i64 0
+; CHECK-NEXT: [[LOAD0:%.*]] = load i64, ptr [[PTR0]], align 8, !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[RES0:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[LOAD0]], i64 0
+; CHECK-NEXT: br label [[ELSE]]
+; CHECK: else:
+; CHECK-NEXT: [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[RES0]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ne i2 [[TMP3]], 0
+; CHECK-NEXT: br i1 [[TMP4]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK: cond.load1:
+; CHECK-NEXT: [[PTR1:%.*]] = extractelement <2 x ptr> [[P]], i64 1
+; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[PTR1]], align 8, !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[RES1:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[LOAD1]], i64 1
+; CHECK-NEXT: br label [[ELSE2]]
+; CHECK: else2:
+; CHECK-NEXT: [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[RES1]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-NEXT: ret <2 x i64> [[RES_PHI_ELSE3]]
+;
+ %ret = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> %p, i32 8, <2 x i1> %mask, <2 x i64> %passthru), !mem.cache_hint !0
+ ret <2 x i64> %ret
+}
+
+define <2 x i64> @masked_gather_constant(<2 x ptr> %p, <2 x i64> %passthru) {
+; CHECK-LABEL: @masked_gather_constant(
+; CHECK-NEXT: [[PTR1:%.*]] = extractelement <2 x ptr> [[P:%.*]], i64 1
+; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[PTR1]], align 8, !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[RES1:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[LOAD1]], i64 1
+; CHECK-NEXT: ret <2 x i64> [[RES1]]
+;
+ %ret = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> %p, i32 8, <2 x i1> <i1 false, i1 true>, <2 x i64> %passthru), !mem.cache_hint !0
+ ret <2 x i64> %ret
+}
+
+define void @masked_scatter_variable(<2 x ptr> %p, <2 x i1> %mask, <2 x i64> %value) {
+; CHECK-LABEL: @masked_scatter_variable(
+; CHECK-NEXT: [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-NEXT: [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT: br i1 [[TMP2]], label [[COND_STORE:%.*]], label [[ELSE:%.*]]
+; CHECK: cond.store:
+; CHECK-NEXT: [[ELT0:%.*]] = extractelement <2 x i64> [[VALUE:%.*]], i64 0
+; CHECK-NEXT: [[PTR0:%.*]] = extractelement <2 x ptr> [[P:%.*]], i64 0
+; CHECK-NEXT: store i64 [[ELT0]], ptr [[PTR0]], align 8, !mem.cache_hint [[META19]]
+; CHECK-NEXT: br label [[ELSE]]
+; CHECK: else:
+; CHECK-NEXT: [[TMP3:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ne i2 [[TMP3]], 0
+; CHECK-NEXT: br i1 [[TMP4]], label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
+; CHECK: cond.store1:
+; CHECK-NEXT: [[ELT1:%.*]] = extractelement <2 x i64> [[VALUE]], i64 1
+; CHECK-NEXT: [[PTR1:%.*]] = extractelement <2 x ptr> [[P]], i64 1
+; CHECK-NEXT: store i64 [[ELT1]], ptr [[PTR1]], align 8, !mem.cache_hint [[META19]]
+; CHECK-NEXT: br label [[ELSE2]]
+; CHECK: else2:
+; CHECK-NEXT: ret void
+;
+ call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> %value, <2 x ptr> %p, i32 8, <2 x i1> %mask), !mem.cache_hint !1
+ ret void
+}
+
+define void @masked_scatter_constant(<2 x ptr> %p, <2 x i64> %value) {
+; CHECK-LABEL: @masked_scatter_constant(
+; CHECK-NEXT: [[ELT1:%.*]] = extractelement <2 x i64> [[VALUE:%.*]], i64 1
+; CHECK-NEXT: [[PTR1:%.*]] = extractelement <2 x ptr> [[P:%.*]], i64 1
+; CHECK-NEXT: store i64 [[ELT1]], ptr [[PTR1]], align 8, !mem.cache_hint [[META19]]
+; CHECK-NEXT: ret void
+;
+ call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> %value, <2 x ptr> %p, i32 8, <2 x i1> <i1 false, i1 true>), !mem.cache_hint !1
+ ret void
+}
+
+define <2 x i64> @masked_expandload_variable(ptr %p, <2 x i1> %mask, <2 x i64> %passthru) {
+; CHECK-LABEL: @masked_expandload_variable(
+; CHECK-NEXT: [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-NEXT: [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT: br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK: cond.load:
+; CHECK-NEXT: [[TMP3:%.*]] = load i64, ptr [[P:%.*]], align 1, !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP3]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[P]], i32 1
+; CHECK-NEXT: br label [[ELSE]]
+; CHECK: else:
+; CHECK-NEXT: [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP4]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT: [[PTR_PHI_ELSE:%.*]] = phi ptr [ [[TMP5]], [[COND_LOAD]] ], [ [[P]], [[TMP0]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0
+; CHECK-NEXT: br i1 [[TMP7]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK: cond.load1:
+; CHECK-NEXT: [[TMP8:%.*]] = load i64, ptr [[PTR_PHI_ELSE]], align 1, !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP8]], i64 1
+; CHECK-NEXT: br label [[ELSE2]]
+; CHECK: else2:
+; CHECK-NEXT: [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP9]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-NEXT: ret <2 x i64> [[RES_PHI_ELSE3]]
+;
+ %ret = call <2 x i64> @llvm.masked.expandload.v2i64.p0(ptr %p, <2 x i1> %mask, <2 x i64> %passthru), !mem.cache_hint !0
+ ret <2 x i64> %ret
+}
+
+define <2 x i64> @masked_expandload_constant(ptr %p, <2 x i64> %passthru) {
+; CHECK-LABEL: @masked_expandload_constant(
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 0
+; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[TMP1]], align 1, !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[RES1:%.*]] = insertelement <2 x i64> poison, i64 [[LOAD1]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i64> [[RES1]], <2 x i64> [[PASSTHRU:%.*]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT: ret <2 x i64> [[TMP2]]
+;
+ %ret = call <2 x i64> @llvm.masked.expandload.v2i64.p0(ptr %p, <2 x i1> <i1 false, i1 true>, <2 x i64> %passthru), !mem.cache_hint !0
+ ret <2 x i64> %ret
+}
+
+define void @masked_compressstore_variable(ptr %p, <2 x i1> %mask, <2 x i64> %value) {
+; CHECK-LABEL: @masked_compressstore_variable(
+; CHECK-NEXT: [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-NEXT: [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT: br i1 [[TMP2]], label [[COND_STORE:%.*]], label [[ELSE:%.*]]
+; CHECK: cond.store:
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[VALUE:%.*]], i64 0
+; CHECK-NEXT: store i64 [[TMP3]], ptr [[P:%.*]], align 1, !mem.cache_hint [[META19]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[P]], i32 1
+; CHECK-NEXT: br label [[ELSE]]
+; CHECK: else:
+; CHECK-NEXT: [[PTR_PHI_ELSE:%.*]] = phi ptr [ [[TMP4]], [[COND_STORE]] ], [ [[P]], [[TMP0:%.*]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ne i2 [[TMP5]], 0
+; CHECK-NEXT: br i1 [[TMP6]], label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
+; CHECK: cond.store1:
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i64> [[VALUE]], i64 1
+; CHECK-NEXT: store i64 [[TMP7]], ptr [[PTR_PHI_ELSE]], align 1, !mem.cache_hint [[META19]]
+; CHECK-NEXT: br label [[ELSE2]]
+; CHECK: else2:
+; CHECK-NEXT: ret void
+;
+ call void @llvm.masked.compressstore.v2i64.p0(<2 x i64> %value, ptr %p, <2 x i1> %mask), !mem.cache_hint !1
+ ret void
+}
+
+define void @masked_compressstore_constant(ptr %p, <2 x i64> %value) {
+; CHECK-LABEL: @masked_compressstore_constant(
+; CHECK-NEXT: [[ELT1:%.*]] = extractelement <2 x i64> [[VALUE:%.*]], i64 1
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 0
+; CHECK-NEXT: store i64 [[ELT1]], ptr [[TMP1]], align 1, !mem.cache_hint [[META19]]
+; CHECK-NEXT: ret void
+;
+ call void @llvm.masked.compressstore.v2i64.p0(<2 x i64> %value, ptr %p, <2 x i1> <i1 false, i1 true>), !mem.cache_hint !1
+ ret void
+}
+
+define void @masked_histogram_variable(<2 x ptr> %p, i64 %inc, <2 x i1> %mask) {
+; CHECK-LABEL: @masked_histogram_variable(
+; CHECK-NEXT: [[MASK0:%.*]] = extractelement <2 x i1> [[MASK:%.*]], i64 0
+; CHECK-NEXT: br i1 [[MASK0]], label [[COND_HISTOGRAM_UPDATE:%.*]], label [[ELSE:%.*]]
+; CHECK: cond.histogram.update:
+; CHECK-NEXT: [[PTR0:%.*]] = extractelement <2 x ptr> [[P:%.*]], i64 0
+; CHECK-NEXT: [[LOAD0:%.*]] = load i64, ptr [[PTR0]], align 8, !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[LOAD0]], [[INC:%.*]]
+; CHECK-NEXT: store i64 [[TMP1]], ptr [[PTR0]], align 8, !mem.cache_hint [[META19]]
+; CHECK-NEXT: br label [[ELSE]]
+; CHECK: else:
+; CHECK-NEXT: [[MASK1:%.*]] = extractelement <2 x i1> [[MASK]], i64 1
+; CHECK-NEXT: br i1 [[MASK1]], label [[COND_HISTOGRAM_UPDATE1:%.*]], label [[ELSE2:%.*]]
+; CHECK: cond.histogram.update1:
+; CHECK-NEXT: [[PTR1:%.*]] = extractelement <2 x ptr> [[P]], i64 1
+; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[PTR1]], align 8, !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[LOAD1]], [[INC]]
+; CHECK-NEXT: store i64 [[TMP2]], ptr [[PTR1]], align 8, !mem.cache_hint [[META19]]
+; CHECK-NEXT: br label [[ELSE2]]
+; CHECK: else2:
+; CHECK-NEXT: ret void
+;
+ call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %p, i64 %inc, <2 x i1> %mask), !mem.cache_hint !0
+ ret void
+}
+
+define void @masked_histogram_constant(<2 x ptr> %p, i64 %inc) {
+; CHECK-LABEL: @masked_histogram_constant(
+; CHECK-NEXT: [[PTR1:%.*]] = extractelement <2 x ptr> [[P:%.*]], i64 1
+; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[PTR1]], align 8, !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[LOAD1]], [[INC:%.*]]
+; CHECK-NEXT: store i64 [[TMP1]], ptr [[PTR1]], align 8, !mem.cache_hint [[META19]]
+; CHECK-NEXT: ret void
+;
+ call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %p, i64 %inc, <2 x i1> <i1 false, i1 true>), !mem.cache_hint !0
+ ret void
+}
+
+define <2 x float> @masked_load_fpmath(ptr %p, <2 x i1> %mask, <2 x float> %passthru) {
+; CHECK-LABEL: @masked_load_fpmath(
+; CHECK-NEXT: [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2
+; CHECK-NEXT: [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT: br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK: cond.load:
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[P:%.*]], i32 0
+; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[TMP3]], align 4, !fpmath [[META21:![0-9]+]]
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x float> [[PASSTHRU:%.*]], float [[TMP4]], i64 0
+; CHECK-NEXT: br label [[ELSE]]
+; CHECK: else:
+; CHECK-NEXT: [[RES_PHI_ELSE:%.*]] = phi <2 x float> [ [[TMP5]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0
+; CHECK-NEXT: br i1 [[TMP7]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK: cond.load1:
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[P]], i32 1
+; CHECK-NEXT: [[TMP9:%.*]] = load float, ptr [[TMP8]], align 4, !fpmath [[META21]]
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x float> [[RES_PHI_ELSE]], float [[TMP9]], i64 1
+; CHECK-NEXT: br label [[ELSE2]]
+; CHECK: else2:
+; CHECK-NEXT: [[RES_PHI_ELSE3:%.*]] = phi <2 x float> [ [[TMP10]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-NEXT: ret <2 x float> [[RES_PHI_ELSE3]]
+;
+ %ret = call <2 x float> @llvm.masked.load.v2f32.p0(ptr %p, i32 4, <2 x i1> %mask, <2 x float> %passthru), !fpmath !17
+ ret <2 x float> %ret
+}
+
+define <2 x i64> @masked_load_debug(ptr %p, <2 x i1> %mask, <2 x i64> %passthru) !dbg !23 {
+; CHECK-LABEL: @masked_load_debug(
+; CHECK-NEXT: [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2, !dbg [[DBG24:![0-9]+]]
+; CHECK-NEXT: [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1, !dbg [[DBG24]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0, !dbg [[DBG24]]
+; CHECK-NEXT: br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]], !dbg [[DBG24]]
+; CHECK: cond.load:
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 0, !dbg [[DBG24]]
+; CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 8, !dbg [[DBG24]]
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP4]], i64 0, !dbg [[DBG24]]
+; CHECK-NEXT: br label [[ELSE]], !dbg [[DBG24]]
+; CHECK: else:
+; CHECK-NEXT: [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP5]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ], !dbg [[DBG24]]
+; CHECK-NEXT: [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2, !dbg [[DBG24]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0, !dbg [[DBG24]]
+; CHECK-NEXT: br i1 [[TMP7]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]], !dbg [[DBG24]]
+; CHECK: cond.load1:
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[P]], i32 1, !dbg [[DBG24]]
+; CHECK-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP8]], align 8, !dbg [[DBG24]]
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP9]], i64 1, !dbg [[DBG24]]
+; CHECK-NEXT: br label [[ELSE2]], !dbg [[DBG24]]
+; CHECK: else2:
+; CHECK-NEXT: [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP10]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ], !dbg [[DBG24]]
+; CHECK-NEXT: ret <2 x i64> [[RES_PHI_ELSE3]], !dbg [[DBG25:![0-9]+]]
+;
+ %ret = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr %p, i32 8, <2 x i1> %mask, <2 x i64> %passthru), !dbg !24
+ ret <2 x i64> %ret, !dbg !25
+}
+
+declare <2 x i64> @llvm.masked.load.v2i64.p0(ptr, i32, <2 x i1>, <2 x i64>)
+declare void @llvm.masked.store.v2i64.p0(<2 x i64>, ptr, i32, <2 x i1>)
+declare <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr>, i32, <2 x i1>, <2 x i64>)
+declare void @llvm.masked.scatter.v2i64.v2p0(<2 x i64>, <2 x ptr>, i32, <2 x i1>)
+declare <2 x i64> @llvm.masked.expandload.v2i64.p0(ptr, <2 x i1>, <2 x i64>)
+declare void @llvm.masked.compressstore.v2i64.p0(<2 x i64>, ptr, <2 x i1>)
+declare void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr>, i64, <2 x i1>)
+declare <2 x float> @llvm.masked.load.v2f32.p0(ptr, i32, <2 x i1>, <2 x float>)
+
+!llvm.dbg.cu = !{!20}
+!llvm.module.flags = !{!18, !19}
+
+!0 = !{i32 0, !2}
+!1 = !{i32 1, !2}
+!2 = !{!"test.cache.hint", !"value"}
+!3 = !{!4, !4, i64 0}
+!4 = !{!"long", !5, i64 0}
+!5 = !{!"ScalarizeMaskedMemIntrin metadata"}
+!6 = !{!7}
+!7 = distinct !{!7, !8, !"scope"}
+!8 = distinct !{!8, !"domain"}
+!9 = distinct !{}
+!10 = !{!"test", !"relaxed"}
+!11 = !{i32 1}
+!12 = !{i64 0, i64 10}
+!13 = !{!"test.annotation"}
+!14 = !{}
+!15 = !{i32 1, i32 2}
+!16 = !{!"branch_weights", i32 10}
+!17 = !{float 2.5}
+!18 = !{i32 2, !"Dwarf Version", i32 4}
+!19 = !{i32 1, !"Debug Info Version", i32 3}
+!20 = distinct !DICompileUnit(language: DW_LANG_C, file: !21, producer: "test", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!21 = !DIFile(filename: "metadata.ll", directory: "/")
+!22 = !DISubroutineType(types: !14)
+!23 = distinct !DISubprogram(name: "masked_load_debug", scope: !21, file: !21, line: 1, type: !22, scopeLine: 1, unit: !20)
+!24 = !DILocation(line: 2, column: 1, scope: !23)
+!25 = !DILocation(line: 3, column: 1, scope: !23)
+!26 = distinct !DIAssignID()
+;.
+; CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(read) }
+; CHECK: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(argmem: write) }
+; CHECK: attributes #[[ATTR2:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) }
+; CHECK: attributes #[[ATTR3:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(argmem: read) }
+; CHECK: attributes #[[ATTR4:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(write) }
+;.
+; CHECK: [[META0:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C, file: [[META1:![0-9]+]], producer: "test", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+; CHECK: [[META1]] = !DIFile(filename: "{{.*}}metadata.ll", directory: {{.*}})
+; CHECK: [[META2:![0-9]+]] = !{i32 2, !"Dwarf Version", i32 4}
+; CHECK: [[META3:![0-9]+]] = !{i32 1, !"Debug Info Version", i32 3}
+; CHECK: [[TBAA4]] = !{[[META5:![0-9]+]], [[META5]], i64 0}
+; CHECK: [[META5]] = !{!"long", [[META6:![0-9]+]], i64 0}
+; CHECK: [[META6]] = !{!"ScalarizeMaskedMemIntrin metadata"}
+; CHECK: [[RNG7]] = !{i64 0, i64 10}
+; CHECK: [[META8]] = !{}
+; CHECK: [[META9]] = !{[[META10:![0-9]+]]}
+; CHECK: [[META10]] = distinct !{[[META10]], [[META11:![0-9]+]], !"scope"}
+; CHECK: [[META11]] = distinct !{[[META11]], !"domain"}
+; CHECK: [[META12]] = !{i32 1}
+; CHECK: [[META13]] = distinct !{}
+; CHECK: [[META14]] = !{!"test.annotation"}
+; CHECK: [[META15]] = !{!"test", !"relaxed"}
+; CHECK: [[META16]] = !{i32 1, i32 2}
+; CHECK: [[META17]] = !{i32 0, [[META18:![0-9]+]]}
+; CHECK: [[META18]] = !{!"test.cache.hint", !"value"}
+; CHECK: [[META19]] = !{i32 1, [[META18]]}
+; CHECK: [[DIASSIGNID20]] = distinct !DIAssignID()
+; CHECK: [[META21]] = !{float 2.500000e+00}
+; CHECK: [[META22:![0-9]+]] = distinct !DISubprogram(name: "masked_load_debug", scope: [[META1]], file: [[META1]], line: 1, type: [[META23:![0-9]+]], scopeLine: 1, spFlags: DISPFlagDefinition, unit: [[META0]])
+; CHECK: [[META23]] = !DISubroutineType(types: [[META8]])
+; CHECK: [[DBG24]] = !DILocation(line: 2, column: 1, scope: [[META22]])
+; CHECK: [[DBG25]] = !DILocation(line: 3, column: 1, scope: [[META22]])
+;.
>From ffeebc6d4bc5754e56c77ba9069bf1a02286eeda Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 25 Aug 2026 19:39:42 +0000
Subject: [PATCH 2/3] format
---
.../Scalar/ScalarizeMaskedMemIntrin.cpp | 31 +++++++++----------
1 file changed, 14 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
index 5eeac40f15275..c9bdc06a62ab2 100644
--- a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
+++ b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
@@ -124,12 +124,11 @@ static void copyMemCacheHint(Instruction &Dest, const Instruction &Source,
if (OperandNo->getZExtValue() != SourcePtrOperand)
return;
- Metadata *DestOperandNo = ConstantAsMetadata::get(ConstantInt::get(
- Type::getInt32Ty(Dest.getContext()), DestPtrOperand));
- Dest.setMetadata(
- LLVMContext::MD_mem_cache_hint,
- MDNode::get(Dest.getContext(),
- {DestOperandNo, CacheHint->getOperand(1)}));
+ Metadata *DestOperandNo = ConstantAsMetadata::get(
+ ConstantInt::get(Type::getInt32Ty(Dest.getContext()), DestPtrOperand));
+ Dest.setMetadata(LLVMContext::MD_mem_cache_hint,
+ MDNode::get(Dest.getContext(),
+ {DestOperandNo, CacheHint->getOperand(1)}));
}
static void copyMetadataForMemoryAccess(Instruction &Dest,
@@ -139,13 +138,13 @@ static void copyMetadataForMemoryAccess(Instruction &Dest,
// Only propagate metadata that is valid on each constituent memory access.
// In particular, do not copy metadata whose meaning is tied to the call,
// such as !prof or !callsite.
- Dest.copyMetadata(
- Source,
- {LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope,
- LLVMContext::MD_noalias, LLVMContext::MD_nontemporal,
- LLVMContext::MD_mem_parallel_loop_access, LLVMContext::MD_access_group,
- LLVMContext::MD_annotation, LLVMContext::MD_nosanitize,
- LLVMContext::MD_mmra, LLVMContext::MD_noalias_addrspace});
+ Dest.copyMetadata(Source,
+ {LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope,
+ LLVMContext::MD_noalias, LLVMContext::MD_nontemporal,
+ LLVMContext::MD_mem_parallel_loop_access,
+ LLVMContext::MD_access_group, LLVMContext::MD_annotation,
+ LLVMContext::MD_nosanitize, LLVMContext::MD_mmra,
+ LLVMContext::MD_noalias_addrspace});
copyMemCacheHint(Dest, Source, SourcePtrOperand, DestPtrOperand);
}
@@ -157,9 +156,8 @@ static void copyMetadataForScalarizedLoad(LoadInst &Dest,
// !range applies element-wise to vectors, so the same range describes each
// scalar result. The other metadata here also describes the loaded result.
- Dest.copyMetadata(Source,
- {LLVMContext::MD_fpmath, LLVMContext::MD_range,
- LLVMContext::MD_invariant_load});
+ Dest.copyMetadata(Source, {LLVMContext::MD_fpmath, LLVMContext::MD_range,
+ LLVMContext::MD_invariant_load});
}
static void copyMetadataForScalarizedStore(StoreInst &Dest,
@@ -169,7 +167,6 @@ static void copyMetadataForScalarizedStore(StoreInst &Dest,
Dest.getPointerOperandIndex());
}
-
// Translate a masked load intrinsic like
// <16 x i32 > @llvm.masked.load( <16 x i32>* %addr,
// <16 x i1> %mask, <16 x i32> %passthru)
>From b77e7fe6820ec923bd5028bc4d8782ecd97b1920 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 25 Aug 2026 20:50:12 +0000
Subject: [PATCH 3/3] support full tbaa
---
.../Scalar/ScalarizeMaskedMemIntrin.cpp | 122 +++++++++++-----
.../ScalarizeMaskedMemIntrin/X86/metadata.ll | 132 +++++++++---------
2 files changed, 154 insertions(+), 100 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
index c9bdc06a62ab2..be5e58125081f 100644
--- a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
+++ b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
@@ -131,28 +131,41 @@ static void copyMemCacheHint(Instruction &Dest, const Instruction &Source,
{DestOperandNo, CacheHint->getOperand(1)}));
}
-static void copyMetadataForMemoryAccess(Instruction &Dest,
- const Instruction &Source,
- unsigned SourcePtrOperand,
- unsigned DestPtrOperand) {
+static void copyMetadataForMemoryAccess(
+ Instruction &Dest, const Instruction &Source, const DataLayout &DL,
+ unsigned SourcePtrOperand, unsigned DestPtrOperand, Type *AccessType,
+ bool IsWholeAccess, std::optional<size_t> ByteOffset) {
// Only propagate metadata that is valid on each constituent memory access.
// In particular, do not copy metadata whose meaning is tied to the call,
// such as !prof or !callsite.
Dest.copyMetadata(Source,
- {LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope,
- LLVMContext::MD_noalias, LLVMContext::MD_nontemporal,
+ {LLVMContext::MD_nontemporal,
LLVMContext::MD_mem_parallel_loop_access,
LLVMContext::MD_access_group, LLVMContext::MD_annotation,
- LLVMContext::MD_nosanitize, LLVMContext::MD_mmra,
- LLVMContext::MD_noalias_addrspace});
+ LLVMContext::MD_nosanitize, LLVMContext::MD_mmra});
+
+ AAMDNodes AANodes = Source.getAAMetadata();
+ if (IsWholeAccess)
+ Dest.setAAMetadata(AANodes);
+ else if (ByteOffset)
+ Dest.setAAMetadata(AANodes.adjustForAccess(*ByteOffset, AccessType, DL));
+ else {
+ // The packed address is runtime-dependent. The other AA metadata remains
+ // applicable, but !tbaa.struct cannot be adjusted to a known byte range.
+ AANodes.TBAAStruct = nullptr;
+ Dest.setAAMetadata(AANodes);
+ }
copyMemCacheHint(Dest, Source, SourcePtrOperand, DestPtrOperand);
}
static void copyMetadataForScalarizedLoad(LoadInst &Dest,
const Instruction &Source,
- unsigned SourcePtrOperand) {
- copyMetadataForMemoryAccess(Dest, Source, SourcePtrOperand,
- Dest.getPointerOperandIndex());
+ const DataLayout &DL,
+ unsigned SourcePtrOperand,
+ std::optional<size_t> ByteOffset) {
+ copyMetadataForMemoryAccess(Dest, Source, DL, SourcePtrOperand,
+ Dest.getPointerOperandIndex(), Dest.getType(),
+ Dest.getType() == Source.getType(), ByteOffset);
// !range applies element-wise to vectors, so the same range describes each
// scalar result. The other metadata here also describes the loaded result.
@@ -162,9 +175,14 @@ static void copyMetadataForScalarizedLoad(LoadInst &Dest,
static void copyMetadataForScalarizedStore(StoreInst &Dest,
const Instruction &Source,
- unsigned SourcePtrOperand) {
- copyMetadataForMemoryAccess(Dest, Source, SourcePtrOperand,
- Dest.getPointerOperandIndex());
+ const DataLayout &DL,
+ unsigned SourcePtrOperand,
+ std::optional<size_t> ByteOffset) {
+ copyMetadataForMemoryAccess(
+ Dest, Source, DL, SourcePtrOperand, Dest.getPointerOperandIndex(),
+ Dest.getValueOperand()->getType(),
+ Dest.getValueOperand()->getType() == Source.getOperand(0)->getType(),
+ ByteOffset);
}
// Translate a masked load intrinsic like
@@ -195,9 +213,9 @@ static void copyMetadataForScalarizedStore(StoreInst &Dest,
// br label %else2
//
// else2: ; preds = %else, %cond.load1
-// %res.phi.else3 = phi <16 x i32> [ %9, %cond.load1 ], [ %res.phi.else, %else ]
-// %10 = extractelement <16 x i1> %mask, i32 2
-// br i1 %10, label %cond.load4, label %else5
+// %res.phi.else3 = phi <16 x i32> [ %9, %cond.load1 ], [ %res.phi.else, %else
+// ] %10 = extractelement <16 x i1> %mask, i32 2 br i1 %10, label %cond.load4,
+// label %else5
//
static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence,
CallInst *CI, DomTreeUpdater *DTU,
@@ -221,7 +239,8 @@ static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence,
// Short-cut if the mask is all-true.
if (isa<Constant>(Mask) && cast<Constant>(Mask)->isAllOnesValue()) {
LoadInst *NewI = Builder.CreateAlignedLoad(VecType, Ptr, AlignVal);
- copyMetadataForScalarizedLoad(*NewI, *CI, /*SourcePtrOperand=*/0);
+ copyMetadataForScalarizedLoad(*NewI, *CI, DL, /*SourcePtrOperand=*/0,
+ std::nullopt);
NewI->takeName(CI);
CI->replaceAllUsesWith(NewI);
CI->eraseFromParent();
@@ -242,7 +261,9 @@ static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence,
continue;
Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Gep, AdjustedAlignVal);
- copyMetadataForScalarizedLoad(*Load, *CI, /*SourcePtrOperand=*/0);
+ copyMetadataForScalarizedLoad(
+ *Load, *CI, DL, /*SourcePtrOperand=*/0,
+ Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
VResult = Builder.CreateInsertElement(VResult, Load, Idx);
}
CI->replaceAllUsesWith(VResult);
@@ -265,7 +286,8 @@ static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence,
Builder.SetInsertPoint(CondBlock->getTerminator());
LoadInst *Load = Builder.CreateAlignedLoad(VecType, Ptr, AlignVal,
CI->getName() + ".cond.load");
- copyMetadataForScalarizedLoad(*Load, *CI, /*SourcePtrOperand=*/0);
+ copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
+ std::nullopt);
BasicBlock *PostLoad = ThenTerm->getSuccessor(0);
Builder.SetInsertPoint(PostLoad, PostLoad->begin());
@@ -324,7 +346,9 @@ static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence,
Builder.SetInsertPoint(CondBlock->getTerminator());
Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Gep, AdjustedAlignVal);
- copyMetadataForScalarizedLoad(*Load, *CI, /*SourcePtrOperand=*/0);
+ copyMetadataForScalarizedLoad(
+ *Load, *CI, DL, /*SourcePtrOperand=*/0,
+ Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
Value *NewVResult = Builder.CreateInsertElement(VResult, Load, Idx);
// Create "else" block, fill it in the next iteration
@@ -394,7 +418,8 @@ static void scalarizeMaskedStore(const DataLayout &DL, bool HasBranchDivergence,
if (isa<Constant>(Mask) && cast<Constant>(Mask)->isAllOnesValue()) {
StoreInst *Store = Builder.CreateAlignedStore(Src, Ptr, AlignVal);
Store->takeName(CI);
- copyMetadataForScalarizedStore(*Store, *CI, /*SourcePtrOperand=*/1);
+ copyMetadataForScalarizedStore(*Store, *CI, DL, /*SourcePtrOperand=*/1,
+ std::nullopt);
// This is a one-to-one replacement, so the assignment link remains valid.
Store->copyMetadata(*CI, LLVMContext::MD_DIAssignID);
CI->eraseFromParent();
@@ -414,7 +439,9 @@ static void scalarizeMaskedStore(const DataLayout &DL, bool HasBranchDivergence,
Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
StoreInst *Store =
Builder.CreateAlignedStore(OneElt, Gep, AdjustedAlignVal);
- copyMetadataForScalarizedStore(*Store, *CI, /*SourcePtrOperand=*/1);
+ copyMetadataForScalarizedStore(
+ *Store, *CI, DL, /*SourcePtrOperand=*/1,
+ Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
}
CI->eraseFromParent();
return;
@@ -435,7 +462,8 @@ static void scalarizeMaskedStore(const DataLayout &DL, bool HasBranchDivergence,
StoreInst *Store = Builder.CreateAlignedStore(Src, Ptr, AlignVal);
Store->takeName(CI);
- copyMetadataForScalarizedStore(*Store, *CI, /*SourcePtrOperand=*/1);
+ copyMetadataForScalarizedStore(*Store, *CI, DL, /*SourcePtrOperand=*/1,
+ std::nullopt);
// This is a one-to-one replacement, so the assignment link remains valid.
Store->copyMetadata(*CI, LLVMContext::MD_DIAssignID);
@@ -491,7 +519,9 @@ static void scalarizeMaskedStore(const DataLayout &DL, bool HasBranchDivergence,
Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
StoreInst *Store =
Builder.CreateAlignedStore(OneElt, Gep, AdjustedAlignVal);
- copyMetadataForScalarizedStore(*Store, *CI, /*SourcePtrOperand=*/1);
+ copyMetadataForScalarizedStore(
+ *Store, *CI, DL, /*SourcePtrOperand=*/1,
+ Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
// Create "else" block, fill it in the next iteration
BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
@@ -563,7 +593,8 @@ static void scalarizeMaskedGather(const DataLayout &DL,
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
LoadInst *Load =
Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx));
- copyMetadataForScalarizedLoad(*Load, *CI, /*SourcePtrOperand=*/0);
+ copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
+ /*ByteOffset=*/0);
VResult =
Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx));
}
@@ -623,7 +654,8 @@ static void scalarizeMaskedGather(const DataLayout &DL,
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
LoadInst *Load =
Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx));
- copyMetadataForScalarizedLoad(*Load, *CI, /*SourcePtrOperand=*/0);
+ copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
+ /*ByteOffset=*/0);
Value *NewVResult =
Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx));
@@ -704,7 +736,9 @@ static void scalarizeMaskedScatter(const DataLayout &DL,
Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
StoreInst *Store = Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
- copyMetadataForScalarizedStore(*Store, *CI, /*SourcePtrOperand=*/1);
+ copyMetadataForScalarizedStore(*Store, *CI, DL,
+ /*SourcePtrOperand=*/1,
+ /*ByteOffset=*/0);
}
CI->eraseFromParent();
return;
@@ -759,7 +793,9 @@ static void scalarizeMaskedScatter(const DataLayout &DL,
Value *OneElt = Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
StoreInst *Store = Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
- copyMetadataForScalarizedStore(*Store, *CI, /*SourcePtrOperand=*/1);
+ copyMetadataForScalarizedStore(*Store, *CI, DL,
+ /*SourcePtrOperand=*/1,
+ /*ByteOffset=*/0);
// Create "else" block, fill it in the next iteration
BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
@@ -817,7 +853,9 @@ static void scalarizeMaskedExpandLoad(const DataLayout &DL,
Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, MemIndex);
LoadInst *Load = Builder.CreateAlignedLoad(
EltTy, NewPtr, AdjustedAlignment, "Load" + Twine(Idx));
- copyMetadataForScalarizedLoad(*Load, *CI, /*SourcePtrOperand=*/0);
+ copyMetadataForScalarizedLoad(
+ *Load, *CI, DL, /*SourcePtrOperand=*/0,
+ MemIndex * DL.getTypeAllocSize(EltTy).getFixedValue());
InsertElt = Load;
ShuffleMask[Idx] = Idx;
++MemIndex;
@@ -876,7 +914,8 @@ static void scalarizeMaskedExpandLoad(const DataLayout &DL,
Builder.SetInsertPoint(CondBlock->getTerminator());
LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Ptr, AdjustedAlignment);
- copyMetadataForScalarizedLoad(*Load, *CI, /*SourcePtrOperand=*/0);
+ copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
+ std::nullopt);
Value *NewVResult = Builder.CreateInsertElement(VResult, Load, Idx);
// Move the pointer if there are more blocks to come.
@@ -949,7 +988,9 @@ static void scalarizeMaskedCompressStore(const DataLayout &DL,
Value *NewPtr = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, MemIndex);
StoreInst *Store =
Builder.CreateAlignedStore(OneElt, NewPtr, AdjustedAlignment);
- copyMetadataForScalarizedStore(*Store, *CI, /*SourcePtrOperand=*/1);
+ copyMetadataForScalarizedStore(
+ *Store, *CI, DL, /*SourcePtrOperand=*/1,
+ MemIndex * DL.getTypeAllocSize(EltTy).getFixedValue());
++MemIndex;
}
CI->eraseFromParent();
@@ -1001,7 +1042,8 @@ static void scalarizeMaskedCompressStore(const DataLayout &DL,
Value *OneElt = Builder.CreateExtractElement(Src, Idx);
StoreInst *Store =
Builder.CreateAlignedStore(OneElt, Ptr, AdjustedAlignment);
- copyMetadataForScalarizedStore(*Store, *CI, /*SourcePtrOperand=*/1);
+ copyMetadataForScalarizedStore(*Store, *CI, DL, /*SourcePtrOperand=*/1,
+ std::nullopt);
// Move the pointer if there are more blocks to come.
Value *NewPtr;
@@ -1081,11 +1123,14 @@ static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI,
continue;
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
LoadInst *Load = Builder.CreateLoad(EltTy, Ptr, "Load" + Twine(Idx));
- copyMetadataForScalarizedLoad(*Load, *CI, /*SourcePtrOperand=*/0);
+ copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
+ /*ByteOffset=*/0);
Value *Update =
CreateHistogramUpdateValue(cast<IntrinsicInst>(CI), Load, Inc);
StoreInst *Store = Builder.CreateStore(Update, Ptr);
- copyMetadataForScalarizedStore(*Store, *CI, /*SourcePtrOperand=*/0);
+ copyMetadataForScalarizedStore(*Store, *CI, DL,
+ /*SourcePtrOperand=*/0,
+ /*ByteOffset=*/0);
}
CI->eraseFromParent();
return;
@@ -1105,11 +1150,14 @@ static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI,
Builder.SetInsertPoint(CondBlock->getTerminator());
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
LoadInst *Load = Builder.CreateLoad(EltTy, Ptr, "Load" + Twine(Idx));
- copyMetadataForScalarizedLoad(*Load, *CI, /*SourcePtrOperand=*/0);
+ copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
+ /*ByteOffset=*/0);
Value *UpdateOp =
CreateHistogramUpdateValue(cast<IntrinsicInst>(CI), Load, Inc);
StoreInst *Store = Builder.CreateStore(UpdateOp, Ptr);
- copyMetadataForScalarizedStore(*Store, *CI, /*SourcePtrOperand=*/0);
+ copyMetadataForScalarizedStore(*Store, *CI, DL,
+ /*SourcePtrOperand=*/0,
+ /*ByteOffset=*/0);
// Create "else" block, fill it in the next iteration
BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/metadata.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/metadata.ll
index 79d5d54f7102b..59b1895deb92d 100644
--- a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/metadata.ll
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/metadata.ll
@@ -33,20 +33,20 @@ define <2 x i64> @masked_load_variable(ptr %p, <2 x i1> %mask, <2 x i64> %passth
define <2 x i64> @masked_load_constant(ptr %p, <2 x i64> %passthru) {
; CHECK-LABEL: @masked_load_constant(
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 1
-; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 8, !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 8, !tbaa [[TBAA19:![0-9]+]], !mem.cache_hint [[META17]]
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP2]], i64 1
; CHECK-NEXT: ret <2 x i64> [[TMP3]]
;
- %ret = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr %p, i32 8, <2 x i1> <i1 false, i1 true>, <2 x i64> %passthru), !mem.cache_hint !0
+ %ret = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr %p, i32 8, <2 x i1> <i1 false, i1 true>, <2 x i64> %passthru), !tbaa.struct !27, !mem.cache_hint !0
ret <2 x i64> %ret
}
define <2 x i64> @masked_load_all_true(ptr %p) {
; CHECK-LABEL: @masked_load_all_true(
-; CHECK-NEXT: [[RET:%.*]] = load <2 x i64>, ptr [[P:%.*]], align 8, !range [[RNG7]], !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[RET:%.*]] = load <2 x i64>, ptr [[P:%.*]], align 8, !range [[RNG7]], !tbaa.struct [[TBAA_STRUCT21:![0-9]+]], !mem.cache_hint [[META17]]
; CHECK-NEXT: ret <2 x i64> [[RET]]
;
- %ret = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr %p, i32 8, <2 x i1> <i1 true, i1 true>, <2 x i64> poison), !prof !16, !range !12, !mem.cache_hint !0
+ %ret = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr %p, i32 8, <2 x i1> <i1 true, i1 true>, <2 x i64> poison), !tbaa.struct !27, !prof !16, !range !12, !mem.cache_hint !0
ret <2 x i64> %ret
}
@@ -59,7 +59,7 @@ define void @masked_store_variable(ptr %p, <2 x i1> %mask, <2 x i64> %value) {
; CHECK: cond.store:
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[VALUE:%.*]], i64 0
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 0
-; CHECK-NEXT: store i64 [[TMP3]], ptr [[TMP4]], align 8, !tbaa [[TBAA4]], !alias.scope [[META9]], !noalias [[META9]], !nontemporal [[META12]], !llvm.mem.parallel_loop_access [[META13]], !llvm.access.group [[META13]], !annotation [[META14]], !nosanitize [[META8]], !mmra [[META15]], !noalias.addrspace [[META16]], !mem.cache_hint [[META19:![0-9]+]]
+; CHECK-NEXT: store i64 [[TMP3]], ptr [[TMP4]], align 8, !tbaa [[TBAA4]], !alias.scope [[META9]], !noalias [[META9]], !nontemporal [[META12]], !llvm.mem.parallel_loop_access [[META13]], !llvm.access.group [[META13]], !annotation [[META14]], !nosanitize [[META8]], !mmra [[META15]], !noalias.addrspace [[META16]], !mem.cache_hint [[META22:![0-9]+]]
; CHECK-NEXT: br label [[ELSE]]
; CHECK: else:
; CHECK-NEXT: [[TMP5:%.*]] = and i2 [[SCALAR_MASK]], -2
@@ -68,7 +68,7 @@ define void @masked_store_variable(ptr %p, <2 x i1> %mask, <2 x i64> %value) {
; CHECK: cond.store1:
; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i64> [[VALUE]], i64 1
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[P]], i32 1
-; CHECK-NEXT: store i64 [[TMP7]], ptr [[TMP8]], align 8, !tbaa [[TBAA4]], !alias.scope [[META9]], !noalias [[META9]], !nontemporal [[META12]], !llvm.mem.parallel_loop_access [[META13]], !llvm.access.group [[META13]], !annotation [[META14]], !nosanitize [[META8]], !mmra [[META15]], !noalias.addrspace [[META16]], !mem.cache_hint [[META19]]
+; CHECK-NEXT: store i64 [[TMP7]], ptr [[TMP8]], align 8, !tbaa [[TBAA4]], !alias.scope [[META9]], !noalias [[META9]], !nontemporal [[META12]], !llvm.mem.parallel_loop_access [[META13]], !llvm.access.group [[META13]], !annotation [[META14]], !nosanitize [[META8]], !mmra [[META15]], !noalias.addrspace [[META16]], !mem.cache_hint [[META22]]
; CHECK-NEXT: br label [[ELSE2]]
; CHECK: else2:
; CHECK-NEXT: ret void
@@ -81,19 +81,19 @@ define void @masked_store_constant(ptr %p, <2 x i64> %value) {
; CHECK-LABEL: @masked_store_constant(
; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i64> [[VALUE:%.*]], i64 1
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 1
-; CHECK-NEXT: store i64 [[TMP1]], ptr [[TMP2]], align 8, !mem.cache_hint [[META19]]
+; CHECK-NEXT: store i64 [[TMP1]], ptr [[TMP2]], align 8, !tbaa [[TBAA19]], !mem.cache_hint [[META22]]
; CHECK-NEXT: ret void
;
- call void @llvm.masked.store.v2i64.p0(<2 x i64> %value, ptr %p, i32 8, <2 x i1> <i1 false, i1 true>), !mem.cache_hint !1
+ call void @llvm.masked.store.v2i64.p0(<2 x i64> %value, ptr %p, i32 8, <2 x i1> <i1 false, i1 true>), !tbaa.struct !27, !mem.cache_hint !1
ret void
}
define void @masked_store_all_true(ptr %p, <2 x i64> %value) {
; CHECK-LABEL: @masked_store_all_true(
-; CHECK-NEXT: store <2 x i64> [[VALUE:%.*]], ptr [[P:%.*]], align 8, !DIAssignID [[DIASSIGNID20:![0-9]+]], !mem.cache_hint [[META19]]
+; CHECK-NEXT: store <2 x i64> [[VALUE:%.*]], ptr [[P:%.*]], align 8, !tbaa.struct [[TBAA_STRUCT21]], !DIAssignID [[DIASSIGNID23:![0-9]+]], !mem.cache_hint [[META22]]
; CHECK-NEXT: ret void
;
- call void @llvm.masked.store.v2i64.p0(<2 x i64> %value, ptr %p, i32 8, <2 x i1> <i1 true, i1 true>), !DIAssignID !26, !mem.cache_hint !1
+ call void @llvm.masked.store.v2i64.p0(<2 x i64> %value, ptr %p, i32 8, <2 x i1> <i1 true, i1 true>), !tbaa.struct !27, !DIAssignID !26, !mem.cache_hint !1
ret void
}
@@ -105,7 +105,7 @@ define <2 x i64> @masked_gather_variable(<2 x ptr> %p, <2 x i1> %mask, <2 x i64>
; CHECK-NEXT: br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
; CHECK: cond.load:
; CHECK-NEXT: [[PTR0:%.*]] = extractelement <2 x ptr> [[P:%.*]], i64 0
-; CHECK-NEXT: [[LOAD0:%.*]] = load i64, ptr [[PTR0]], align 8, !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[LOAD0:%.*]] = load i64, ptr [[PTR0]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META17]]
; CHECK-NEXT: [[RES0:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[LOAD0]], i64 0
; CHECK-NEXT: br label [[ELSE]]
; CHECK: else:
@@ -115,25 +115,25 @@ define <2 x i64> @masked_gather_variable(<2 x ptr> %p, <2 x i1> %mask, <2 x i64>
; CHECK-NEXT: br i1 [[TMP4]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
; CHECK: cond.load1:
; CHECK-NEXT: [[PTR1:%.*]] = extractelement <2 x ptr> [[P]], i64 1
-; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[PTR1]], align 8, !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[PTR1]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META17]]
; CHECK-NEXT: [[RES1:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[LOAD1]], i64 1
; CHECK-NEXT: br label [[ELSE2]]
; CHECK: else2:
; CHECK-NEXT: [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[RES1]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
; CHECK-NEXT: ret <2 x i64> [[RES_PHI_ELSE3]]
;
- %ret = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> %p, i32 8, <2 x i1> %mask, <2 x i64> %passthru), !mem.cache_hint !0
+ %ret = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> %p, i32 8, <2 x i1> %mask, <2 x i64> %passthru), !tbaa.struct !27, !mem.cache_hint !0
ret <2 x i64> %ret
}
define <2 x i64> @masked_gather_constant(<2 x ptr> %p, <2 x i64> %passthru) {
; CHECK-LABEL: @masked_gather_constant(
; CHECK-NEXT: [[PTR1:%.*]] = extractelement <2 x ptr> [[P:%.*]], i64 1
-; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[PTR1]], align 8, !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[PTR1]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META17]]
; CHECK-NEXT: [[RES1:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[LOAD1]], i64 1
; CHECK-NEXT: ret <2 x i64> [[RES1]]
;
- %ret = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> %p, i32 8, <2 x i1> <i1 false, i1 true>, <2 x i64> %passthru), !mem.cache_hint !0
+ %ret = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> %p, i32 8, <2 x i1> <i1 false, i1 true>, <2 x i64> %passthru), !tbaa.struct !27, !mem.cache_hint !0
ret <2 x i64> %ret
}
@@ -146,7 +146,7 @@ define void @masked_scatter_variable(<2 x ptr> %p, <2 x i1> %mask, <2 x i64> %va
; CHECK: cond.store:
; CHECK-NEXT: [[ELT0:%.*]] = extractelement <2 x i64> [[VALUE:%.*]], i64 0
; CHECK-NEXT: [[PTR0:%.*]] = extractelement <2 x ptr> [[P:%.*]], i64 0
-; CHECK-NEXT: store i64 [[ELT0]], ptr [[PTR0]], align 8, !mem.cache_hint [[META19]]
+; CHECK-NEXT: store i64 [[ELT0]], ptr [[PTR0]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META22]]
; CHECK-NEXT: br label [[ELSE]]
; CHECK: else:
; CHECK-NEXT: [[TMP3:%.*]] = and i2 [[SCALAR_MASK]], -2
@@ -155,12 +155,12 @@ define void @masked_scatter_variable(<2 x ptr> %p, <2 x i1> %mask, <2 x i64> %va
; CHECK: cond.store1:
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <2 x i64> [[VALUE]], i64 1
; CHECK-NEXT: [[PTR1:%.*]] = extractelement <2 x ptr> [[P]], i64 1
-; CHECK-NEXT: store i64 [[ELT1]], ptr [[PTR1]], align 8, !mem.cache_hint [[META19]]
+; CHECK-NEXT: store i64 [[ELT1]], ptr [[PTR1]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META22]]
; CHECK-NEXT: br label [[ELSE2]]
; CHECK: else2:
; CHECK-NEXT: ret void
;
- call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> %value, <2 x ptr> %p, i32 8, <2 x i1> %mask), !mem.cache_hint !1
+ call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> %value, <2 x ptr> %p, i32 8, <2 x i1> %mask), !tbaa.struct !27, !mem.cache_hint !1
ret void
}
@@ -168,10 +168,10 @@ define void @masked_scatter_constant(<2 x ptr> %p, <2 x i64> %value) {
; CHECK-LABEL: @masked_scatter_constant(
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <2 x i64> [[VALUE:%.*]], i64 1
; CHECK-NEXT: [[PTR1:%.*]] = extractelement <2 x ptr> [[P:%.*]], i64 1
-; CHECK-NEXT: store i64 [[ELT1]], ptr [[PTR1]], align 8, !mem.cache_hint [[META19]]
+; CHECK-NEXT: store i64 [[ELT1]], ptr [[PTR1]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META22]]
; CHECK-NEXT: ret void
;
- call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> %value, <2 x ptr> %p, i32 8, <2 x i1> <i1 false, i1 true>), !mem.cache_hint !1
+ call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> %value, <2 x ptr> %p, i32 8, <2 x i1> <i1 false, i1 true>), !tbaa.struct !27, !mem.cache_hint !1
ret void
}
@@ -200,19 +200,19 @@ define <2 x i64> @masked_expandload_variable(ptr %p, <2 x i1> %mask, <2 x i64> %
; CHECK-NEXT: [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP9]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
; CHECK-NEXT: ret <2 x i64> [[RES_PHI_ELSE3]]
;
- %ret = call <2 x i64> @llvm.masked.expandload.v2i64.p0(ptr %p, <2 x i1> %mask, <2 x i64> %passthru), !mem.cache_hint !0
+ %ret = call <2 x i64> @llvm.masked.expandload.v2i64.p0(ptr %p, <2 x i1> %mask, <2 x i64> %passthru), !tbaa.struct !27, !mem.cache_hint !0
ret <2 x i64> %ret
}
define <2 x i64> @masked_expandload_constant(ptr %p, <2 x i64> %passthru) {
; CHECK-LABEL: @masked_expandload_constant(
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 0
-; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[TMP1]], align 1, !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[TMP1]], align 1, !tbaa [[TBAA4]], !mem.cache_hint [[META17]]
; CHECK-NEXT: [[RES1:%.*]] = insertelement <2 x i64> poison, i64 [[LOAD1]], i64 1
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i64> [[RES1]], <2 x i64> [[PASSTHRU:%.*]], <2 x i32> <i32 2, i32 1>
; CHECK-NEXT: ret <2 x i64> [[TMP2]]
;
- %ret = call <2 x i64> @llvm.masked.expandload.v2i64.p0(ptr %p, <2 x i1> <i1 false, i1 true>, <2 x i64> %passthru), !mem.cache_hint !0
+ %ret = call <2 x i64> @llvm.masked.expandload.v2i64.p0(ptr %p, <2 x i1> <i1 false, i1 true>, <2 x i64> %passthru), !tbaa.struct !27, !mem.cache_hint !0
ret <2 x i64> %ret
}
@@ -224,7 +224,7 @@ define void @masked_compressstore_variable(ptr %p, <2 x i1> %mask, <2 x i64> %va
; CHECK-NEXT: br i1 [[TMP2]], label [[COND_STORE:%.*]], label [[ELSE:%.*]]
; CHECK: cond.store:
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[VALUE:%.*]], i64 0
-; CHECK-NEXT: store i64 [[TMP3]], ptr [[P:%.*]], align 1, !mem.cache_hint [[META19]]
+; CHECK-NEXT: store i64 [[TMP3]], ptr [[P:%.*]], align 1, !mem.cache_hint [[META22]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[P]], i32 1
; CHECK-NEXT: br label [[ELSE]]
; CHECK: else:
@@ -234,12 +234,12 @@ define void @masked_compressstore_variable(ptr %p, <2 x i1> %mask, <2 x i64> %va
; CHECK-NEXT: br i1 [[TMP6]], label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
; CHECK: cond.store1:
; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i64> [[VALUE]], i64 1
-; CHECK-NEXT: store i64 [[TMP7]], ptr [[PTR_PHI_ELSE]], align 1, !mem.cache_hint [[META19]]
+; CHECK-NEXT: store i64 [[TMP7]], ptr [[PTR_PHI_ELSE]], align 1, !mem.cache_hint [[META22]]
; CHECK-NEXT: br label [[ELSE2]]
; CHECK: else2:
; CHECK-NEXT: ret void
;
- call void @llvm.masked.compressstore.v2i64.p0(<2 x i64> %value, ptr %p, <2 x i1> %mask), !mem.cache_hint !1
+ call void @llvm.masked.compressstore.v2i64.p0(<2 x i64> %value, ptr %p, <2 x i1> %mask), !tbaa.struct !27, !mem.cache_hint !1
ret void
}
@@ -247,10 +247,10 @@ define void @masked_compressstore_constant(ptr %p, <2 x i64> %value) {
; CHECK-LABEL: @masked_compressstore_constant(
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <2 x i64> [[VALUE:%.*]], i64 1
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 0
-; CHECK-NEXT: store i64 [[ELT1]], ptr [[TMP1]], align 1, !mem.cache_hint [[META19]]
+; CHECK-NEXT: store i64 [[ELT1]], ptr [[TMP1]], align 1, !tbaa [[TBAA4]], !mem.cache_hint [[META22]]
; CHECK-NEXT: ret void
;
- call void @llvm.masked.compressstore.v2i64.p0(<2 x i64> %value, ptr %p, <2 x i1> <i1 false, i1 true>), !mem.cache_hint !1
+ call void @llvm.masked.compressstore.v2i64.p0(<2 x i64> %value, ptr %p, <2 x i1> <i1 false, i1 true>), !tbaa.struct !27, !mem.cache_hint !1
ret void
}
@@ -260,35 +260,35 @@ define void @masked_histogram_variable(<2 x ptr> %p, i64 %inc, <2 x i1> %mask) {
; CHECK-NEXT: br i1 [[MASK0]], label [[COND_HISTOGRAM_UPDATE:%.*]], label [[ELSE:%.*]]
; CHECK: cond.histogram.update:
; CHECK-NEXT: [[PTR0:%.*]] = extractelement <2 x ptr> [[P:%.*]], i64 0
-; CHECK-NEXT: [[LOAD0:%.*]] = load i64, ptr [[PTR0]], align 8, !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[LOAD0:%.*]] = load i64, ptr [[PTR0]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META17]]
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[LOAD0]], [[INC:%.*]]
-; CHECK-NEXT: store i64 [[TMP1]], ptr [[PTR0]], align 8, !mem.cache_hint [[META19]]
+; CHECK-NEXT: store i64 [[TMP1]], ptr [[PTR0]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META22]]
; CHECK-NEXT: br label [[ELSE]]
; CHECK: else:
; CHECK-NEXT: [[MASK1:%.*]] = extractelement <2 x i1> [[MASK]], i64 1
; CHECK-NEXT: br i1 [[MASK1]], label [[COND_HISTOGRAM_UPDATE1:%.*]], label [[ELSE2:%.*]]
; CHECK: cond.histogram.update1:
; CHECK-NEXT: [[PTR1:%.*]] = extractelement <2 x ptr> [[P]], i64 1
-; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[PTR1]], align 8, !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[PTR1]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META17]]
; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[LOAD1]], [[INC]]
-; CHECK-NEXT: store i64 [[TMP2]], ptr [[PTR1]], align 8, !mem.cache_hint [[META19]]
+; CHECK-NEXT: store i64 [[TMP2]], ptr [[PTR1]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META22]]
; CHECK-NEXT: br label [[ELSE2]]
; CHECK: else2:
; CHECK-NEXT: ret void
;
- call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %p, i64 %inc, <2 x i1> %mask), !mem.cache_hint !0
+ call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %p, i64 %inc, <2 x i1> %mask), !tbaa.struct !27, !mem.cache_hint !0
ret void
}
define void @masked_histogram_constant(<2 x ptr> %p, i64 %inc) {
; CHECK-LABEL: @masked_histogram_constant(
; CHECK-NEXT: [[PTR1:%.*]] = extractelement <2 x ptr> [[P:%.*]], i64 1
-; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[PTR1]], align 8, !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[PTR1]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META17]]
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[LOAD1]], [[INC:%.*]]
-; CHECK-NEXT: store i64 [[TMP1]], ptr [[PTR1]], align 8, !mem.cache_hint [[META19]]
+; CHECK-NEXT: store i64 [[TMP1]], ptr [[PTR1]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META22]]
; CHECK-NEXT: ret void
;
- call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %p, i64 %inc, <2 x i1> <i1 false, i1 true>), !mem.cache_hint !0
+ call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %p, i64 %inc, <2 x i1> <i1 false, i1 true>), !tbaa.struct !27, !mem.cache_hint !0
ret void
}
@@ -300,7 +300,7 @@ define <2 x float> @masked_load_fpmath(ptr %p, <2 x i1> %mask, <2 x float> %pass
; CHECK-NEXT: br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
; CHECK: cond.load:
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[P:%.*]], i32 0
-; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[TMP3]], align 4, !fpmath [[META21:![0-9]+]]
+; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[TMP3]], align 4, !fpmath [[META24:![0-9]+]]
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x float> [[PASSTHRU:%.*]], float [[TMP4]], i64 0
; CHECK-NEXT: br label [[ELSE]]
; CHECK: else:
@@ -310,7 +310,7 @@ define <2 x float> @masked_load_fpmath(ptr %p, <2 x i1> %mask, <2 x float> %pass
; CHECK-NEXT: br i1 [[TMP7]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
; CHECK: cond.load1:
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[P]], i32 1
-; CHECK-NEXT: [[TMP9:%.*]] = load float, ptr [[TMP8]], align 4, !fpmath [[META21]]
+; CHECK-NEXT: [[TMP9:%.*]] = load float, ptr [[TMP8]], align 4, !fpmath [[META24]]
; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x float> [[RES_PHI_ELSE]], float [[TMP9]], i64 1
; CHECK-NEXT: br label [[ELSE2]]
; CHECK: else2:
@@ -323,28 +323,28 @@ define <2 x float> @masked_load_fpmath(ptr %p, <2 x i1> %mask, <2 x float> %pass
define <2 x i64> @masked_load_debug(ptr %p, <2 x i1> %mask, <2 x i64> %passthru) !dbg !23 {
; CHECK-LABEL: @masked_load_debug(
-; CHECK-NEXT: [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2, !dbg [[DBG24:![0-9]+]]
-; CHECK-NEXT: [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1, !dbg [[DBG24]]
-; CHECK-NEXT: [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0, !dbg [[DBG24]]
-; CHECK-NEXT: br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]], !dbg [[DBG24]]
+; CHECK-NEXT: [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2, !dbg [[DBG27:![0-9]+]]
+; CHECK-NEXT: [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1, !dbg [[DBG27]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0, !dbg [[DBG27]]
+; CHECK-NEXT: br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]], !dbg [[DBG27]]
; CHECK: cond.load:
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 0, !dbg [[DBG24]]
-; CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 8, !dbg [[DBG24]]
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP4]], i64 0, !dbg [[DBG24]]
-; CHECK-NEXT: br label [[ELSE]], !dbg [[DBG24]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 0, !dbg [[DBG27]]
+; CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 8, !dbg [[DBG27]]
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP4]], i64 0, !dbg [[DBG27]]
+; CHECK-NEXT: br label [[ELSE]], !dbg [[DBG27]]
; CHECK: else:
-; CHECK-NEXT: [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP5]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ], !dbg [[DBG24]]
-; CHECK-NEXT: [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2, !dbg [[DBG24]]
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0, !dbg [[DBG24]]
-; CHECK-NEXT: br i1 [[TMP7]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]], !dbg [[DBG24]]
+; CHECK-NEXT: [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP5]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ], !dbg [[DBG27]]
+; CHECK-NEXT: [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2, !dbg [[DBG27]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0, !dbg [[DBG27]]
+; CHECK-NEXT: br i1 [[TMP7]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]], !dbg [[DBG27]]
; CHECK: cond.load1:
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[P]], i32 1, !dbg [[DBG24]]
-; CHECK-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP8]], align 8, !dbg [[DBG24]]
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP9]], i64 1, !dbg [[DBG24]]
-; CHECK-NEXT: br label [[ELSE2]], !dbg [[DBG24]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[P]], i32 1, !dbg [[DBG27]]
+; CHECK-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP8]], align 8, !dbg [[DBG27]]
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP9]], i64 1, !dbg [[DBG27]]
+; CHECK-NEXT: br label [[ELSE2]], !dbg [[DBG27]]
; CHECK: else2:
-; CHECK-NEXT: [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP10]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ], !dbg [[DBG24]]
-; CHECK-NEXT: ret <2 x i64> [[RES_PHI_ELSE3]], !dbg [[DBG25:![0-9]+]]
+; CHECK-NEXT: [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP10]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ], !dbg [[DBG27]]
+; CHECK-NEXT: ret <2 x i64> [[RES_PHI_ELSE3]], !dbg [[DBG28:![0-9]+]]
;
%ret = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr %p, i32 8, <2 x i1> %mask, <2 x i64> %passthru), !dbg !24
ret <2 x i64> %ret, !dbg !25
@@ -389,6 +389,9 @@ declare <2 x float> @llvm.masked.load.v2f32.p0(ptr, i32, <2 x i1>, <2 x float>)
!24 = !DILocation(line: 2, column: 1, scope: !23)
!25 = !DILocation(line: 3, column: 1, scope: !23)
!26 = distinct !DIAssignID()
+!27 = !{i64 0, i64 8, !3, i64 8, i64 8, !28}
+!28 = !{!29, !29, i64 0}
+!29 = !{!"second long", !5, i64 0}
;.
; CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(read) }
; CHECK: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(argmem: write) }
@@ -415,11 +418,14 @@ declare <2 x float> @llvm.masked.load.v2f32.p0(ptr, i32, <2 x i1>, <2 x float>)
; CHECK: [[META16]] = !{i32 1, i32 2}
; CHECK: [[META17]] = !{i32 0, [[META18:![0-9]+]]}
; CHECK: [[META18]] = !{!"test.cache.hint", !"value"}
-; CHECK: [[META19]] = !{i32 1, [[META18]]}
-; CHECK: [[DIASSIGNID20]] = distinct !DIAssignID()
-; CHECK: [[META21]] = !{float 2.500000e+00}
-; CHECK: [[META22:![0-9]+]] = distinct !DISubprogram(name: "masked_load_debug", scope: [[META1]], file: [[META1]], line: 1, type: [[META23:![0-9]+]], scopeLine: 1, spFlags: DISPFlagDefinition, unit: [[META0]])
-; CHECK: [[META23]] = !DISubroutineType(types: [[META8]])
-; CHECK: [[DBG24]] = !DILocation(line: 2, column: 1, scope: [[META22]])
-; CHECK: [[DBG25]] = !DILocation(line: 3, column: 1, scope: [[META22]])
+; CHECK: [[TBAA19]] = !{[[META20:![0-9]+]], [[META20]], i64 0}
+; CHECK: [[META20]] = !{!"second long", [[META6]], i64 0}
+; CHECK: [[TBAA_STRUCT21]] = !{i64 0, i64 8, [[TBAA4]], i64 8, i64 8, [[TBAA19]]}
+; CHECK: [[META22]] = !{i32 1, [[META18]]}
+; CHECK: [[DIASSIGNID23]] = distinct !DIAssignID()
+; CHECK: [[META24]] = !{float 2.500000e+00}
+; CHECK: [[META25:![0-9]+]] = distinct !DISubprogram(name: "masked_load_debug", scope: [[META1]], file: [[META1]], line: 1, type: [[META26:![0-9]+]], scopeLine: 1, spFlags: DISPFlagDefinition, unit: [[META0]])
+; CHECK: [[META26]] = !DISubroutineType(types: [[META8]])
+; CHECK: [[DBG27]] = !DILocation(line: 2, column: 1, scope: [[META25]])
+; CHECK: [[DBG28]] = !DILocation(line: 3, column: 1, scope: [[META25]])
;.
More information about the llvm-commits
mailing list