[llvm-branch-commits] [llvm] [ScalarizeMaskedMemIntrin] Share memory metadata propagation (PR #243676)
Yonah Goldberg via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Nov 18 01:13:37 PST 2026
https://github.com/YonahGoldberg updated https://github.com/llvm/llvm-project/pull/243676
>From f9b7392d6413a1ead86fe70059f2f7052751e571 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 18 Nov 2026 01:04:57 +0000
Subject: [PATCH 1/2] [ScalarizeMaskedMemIntrin] Add memory metadata tests
---
llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/metadata.ll | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/metadata.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/metadata.ll
index 22142eecf3e3c6..40e69c79960e4d 100644
--- a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/metadata.ll
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/metadata.ll
@@ -26,7 +26,7 @@ define <2 x i64> @masked_load_variable(ptr %p, <2 x i1> %mask, <2 x i64> %passth
; CHECK-NEXT: [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP10]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
; CHECK-NEXT: ret <2 x i64> [[RES_PHI_ELSE3]]
;
- %ret = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr %p, i32 8, <2 x i1> %mask, <2 x i64> %passthru), !tbaa !3, !prof !16, !range !12, !invariant.load !14, !alias.scope !6, !noalias !6, !nontemporal !11, !llvm.mem.parallel_loop_access !9, !llvm.access.group !9, !annotation !13, !nosanitize !14, !mmra !10, !noalias.addrspace !15, !mem.cache_hint !0
+ %ret = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr %p, i32 8, <2 x i1> %mask, <2 x i64> %passthru), !tbaa !3, !prof !16, !range !12, !invariant.load !14, !alias.scope !6, !noalias !6, !nontemporal !11, !llvm.mem.parallel_loop_access !9, !llvm.access.group !9, !annotation !13, !nosanitize !14, !mmra !10, !noalias.addrspace !15, !mem.cache_hint !0, !pcsections !999
ret <2 x i64> %ret
}
@@ -392,6 +392,8 @@ declare <2 x float> @llvm.masked.load.v2f32.p0(ptr, i32, <2 x i1>, <2 x float>)
!27 = !{i64 0, i64 8, !3, i64 8, i64 8, !28}
!28 = !{!29, !29, i64 0}
!29 = !{!"second long", !5, i64 0}
+
+!999 = !{!"metadata_section"}
;.
; CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(read) }
; CHECK: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(argmem: write) }
>From 889258765027b039b42705092758303a039c9805 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 18 Nov 2026 01:05:03 +0000
Subject: [PATCH 2/2] [ScalarizeMaskedMemIntrin] Share memory metadata
propagation
---
.../Scalar/ScalarizeMaskedMemIntrin.cpp | 132 ++++--------------
.../ScalarizeMaskedMemIntrin/X86/metadata.ll | 121 ++++++++--------
2 files changed, 88 insertions(+), 165 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
index d8fd2603dd1f3f..431037b3a0cc33 100644
--- a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
+++ b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
@@ -16,6 +16,7 @@
#include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/DomTreeUpdater.h"
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/BasicBlock.h"
@@ -113,61 +114,6 @@ static unsigned adjustForEndian(const DataLayout &DL, unsigned VectorWidth,
return DL.isBigEndian() ? VectorWidth - 1 - Idx : Idx;
}
-static void copyMetadataForMemoryAccess(
- Instruction &Dest, const Instruction &Source, const DataLayout &DL,
- unsigned SourcePtrOperand, unsigned DestPtrOperand, Type *AccessType,
- bool IsWholeAccess, std::optional<size_t> ByteOffset) {
- // Only propagate metadata that is valid on each constituent memory access.
- // In particular, do not copy metadata whose meaning is tied to the call,
- // such as !prof or !callsite.
- Dest.copyMetadata(Source,
- {LLVMContext::MD_nontemporal,
- LLVMContext::MD_mem_parallel_loop_access,
- LLVMContext::MD_access_group, LLVMContext::MD_annotation,
- LLVMContext::MD_nosanitize, LLVMContext::MD_mmra});
-
- AAMDNodes AANodes = Source.getAAMetadata();
- if (IsWholeAccess)
- Dest.setAAMetadata(AANodes);
- else if (ByteOffset)
- Dest.setAAMetadata(AANodes.adjustForAccess(*ByteOffset, AccessType, DL));
- else {
- // The packed address is runtime-dependent. The other AA metadata remains
- // applicable, but !tbaa.struct cannot be adjusted to a known byte range.
- AANodes.TBAAStruct = nullptr;
- Dest.setAAMetadata(AANodes);
- }
- setMemCacheHint(Dest, DestPtrOperand,
- getMemCacheHint(Source, SourcePtrOperand));
-}
-
-static void copyMetadataForScalarizedLoad(LoadInst &Dest,
- const Instruction &Source,
- const DataLayout &DL,
- unsigned SourcePtrOperand,
- std::optional<size_t> ByteOffset) {
- copyMetadataForMemoryAccess(Dest, Source, DL, SourcePtrOperand,
- Dest.getPointerOperandIndex(), Dest.getType(),
- Dest.getType() == Source.getType(), ByteOffset);
-
- // !range applies element-wise to vectors, so the same range describes each
- // scalar result. The other metadata here also describes the loaded result.
- Dest.copyMetadata(Source, {LLVMContext::MD_fpmath, LLVMContext::MD_range,
- LLVMContext::MD_invariant_load});
-}
-
-static void copyMetadataForScalarizedStore(StoreInst &Dest,
- const Instruction &Source,
- const DataLayout &DL,
- unsigned SourcePtrOperand,
- std::optional<size_t> ByteOffset) {
- copyMetadataForMemoryAccess(
- Dest, Source, DL, SourcePtrOperand, Dest.getPointerOperandIndex(),
- Dest.getValueOperand()->getType(),
- Dest.getValueOperand()->getType() == Source.getOperand(0)->getType(),
- ByteOffset);
-}
-
// Translate a masked load intrinsic like
// <16 x i32 > @llvm.masked.load( <16 x i32>* %addr,
// <16 x i1> %mask, <16 x i32> %passthru)
@@ -220,8 +166,7 @@ static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence,
// Short-cut if the mask is all-true.
if (isa<Constant>(Mask) && cast<Constant>(Mask)->isAllOnesValue()) {
LoadInst *NewI = Builder.CreateAlignedLoad(VecType, Ptr, AlignVal);
- copyMetadataForScalarizedLoad(*NewI, *CI, DL, /*SourcePtrOperand=*/0,
- std::nullopt);
+ copyMetadataForMemoryAccess(*NewI, *CI);
NewI->takeName(CI);
CI->replaceAllUsesWith(NewI);
CI->eraseFromParent();
@@ -242,9 +187,8 @@ static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence,
continue;
Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Gep, AdjustedAlignVal);
- copyMetadataForScalarizedLoad(
- *Load, *CI, DL, /*SourcePtrOperand=*/0,
- Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
+ copyMetadataForMemoryAccess(
+ *Load, *CI, Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
VResult = Builder.CreateInsertElement(VResult, Load, Idx);
}
CI->replaceAllUsesWith(VResult);
@@ -271,8 +215,7 @@ static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence,
Builder.SetInsertPoint(CondBlock->getTerminator());
LoadInst *Load = Builder.CreateAlignedLoad(VecType, Ptr, AlignVal,
CI->getName() + ".cond.load");
- copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
- std::nullopt);
+ copyMetadataForMemoryAccess(*Load, *CI);
BasicBlock *PostLoad = ThenTerm->getSuccessor(0);
Builder.SetInsertPoint(PostLoad->begin());
@@ -335,9 +278,8 @@ static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence,
Builder.SetInsertPoint(CondBlock->getTerminator());
Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Gep, AdjustedAlignVal);
- copyMetadataForScalarizedLoad(
- *Load, *CI, DL, /*SourcePtrOperand=*/0,
- Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
+ copyMetadataForMemoryAccess(
+ *Load, *CI, Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
Value *NewVResult = Builder.CreateInsertElement(VResult, Load, Idx);
// Create "else" block, fill it in the next iteration
@@ -406,8 +348,7 @@ static void scalarizeMaskedStore(const DataLayout &DL, bool HasBranchDivergence,
if (isa<Constant>(Mask) && cast<Constant>(Mask)->isAllOnesValue()) {
StoreInst *Store = Builder.CreateAlignedStore(Src, Ptr, AlignVal);
Store->takeName(CI);
- copyMetadataForScalarizedStore(*Store, *CI, DL, /*SourcePtrOperand=*/1,
- std::nullopt);
+ copyMetadataForMemoryAccess(*Store, *CI);
// This is a one-to-one replacement, so the assignment link remains valid.
Store->copyMetadata(*CI, LLVMContext::MD_DIAssignID);
CI->eraseFromParent();
@@ -427,9 +368,8 @@ static void scalarizeMaskedStore(const DataLayout &DL, bool HasBranchDivergence,
Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
StoreInst *Store =
Builder.CreateAlignedStore(OneElt, Gep, AdjustedAlignVal);
- copyMetadataForScalarizedStore(
- *Store, *CI, DL, /*SourcePtrOperand=*/1,
- Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
+ copyMetadataForMemoryAccess(
+ *Store, *CI, Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
}
CI->eraseFromParent();
return;
@@ -454,8 +394,7 @@ static void scalarizeMaskedStore(const DataLayout &DL, bool HasBranchDivergence,
StoreInst *Store = Builder.CreateAlignedStore(Src, Ptr, AlignVal);
Store->takeName(CI);
- copyMetadataForScalarizedStore(*Store, *CI, DL, /*SourcePtrOperand=*/1,
- std::nullopt);
+ copyMetadataForMemoryAccess(*Store, *CI);
// This is a one-to-one replacement, so the assignment link remains valid.
Store->copyMetadata(*CI, LLVMContext::MD_DIAssignID);
@@ -515,9 +454,8 @@ static void scalarizeMaskedStore(const DataLayout &DL, bool HasBranchDivergence,
Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
StoreInst *Store =
Builder.CreateAlignedStore(OneElt, Gep, AdjustedAlignVal);
- copyMetadataForScalarizedStore(
- *Store, *CI, DL, /*SourcePtrOperand=*/1,
- Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
+ copyMetadataForMemoryAccess(
+ *Store, *CI, Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
// Create "else" block, fill it in the next iteration
BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
@@ -586,8 +524,7 @@ static void scalarizeMaskedGather(const DataLayout &DL,
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
LoadInst *Load =
Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx));
- copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
- /*ByteOffset=*/0);
+ copyMetadataForMemoryAccess(*Load, *CI, /*DestByteOffset=*/0);
VResult =
Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx));
}
@@ -647,8 +584,7 @@ static void scalarizeMaskedGather(const DataLayout &DL,
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
LoadInst *Load =
Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx));
- copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
- /*ByteOffset=*/0);
+ copyMetadataForMemoryAccess(*Load, *CI, /*DestByteOffset=*/0);
Value *NewVResult =
Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx));
@@ -727,9 +663,7 @@ static void scalarizeMaskedScatter(const DataLayout &DL,
Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
StoreInst *Store = Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
- copyMetadataForScalarizedStore(*Store, *CI, DL,
- /*SourcePtrOperand=*/1,
- /*ByteOffset=*/0);
+ copyMetadataForMemoryAccess(*Store, *CI, /*DestByteOffset=*/0);
}
CI->eraseFromParent();
return;
@@ -784,9 +718,7 @@ static void scalarizeMaskedScatter(const DataLayout &DL,
Value *OneElt = Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
StoreInst *Store = Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
- copyMetadataForScalarizedStore(*Store, *CI, DL,
- /*SourcePtrOperand=*/1,
- /*ByteOffset=*/0);
+ copyMetadataForMemoryAccess(*Store, *CI, /*DestByteOffset=*/0);
// Create "else" block, fill it in the next iteration
BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
@@ -842,9 +774,8 @@ static void scalarizeMaskedExpandLoad(const DataLayout &DL,
Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, MemIndex);
LoadInst *Load = Builder.CreateAlignedLoad(
EltTy, NewPtr, AdjustedAlignment, "Load" + Twine(Idx));
- copyMetadataForScalarizedLoad(
- *Load, *CI, DL, /*SourcePtrOperand=*/0,
- MemIndex * DL.getTypeAllocSize(EltTy).getFixedValue());
+ copyMetadataForMemoryAccess(
+ *Load, *CI, MemIndex * DL.getTypeAllocSize(EltTy).getFixedValue());
InsertElt = Load;
ShuffleMask[Idx] = Idx;
++MemIndex;
@@ -907,8 +838,7 @@ static void scalarizeMaskedExpandLoad(const DataLayout &DL,
Builder.SetInsertPoint(CondBlock->getTerminator());
LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Ptr, AdjustedAlignment);
- copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
- std::nullopt);
+ copyMetadataForMemoryAccess(*Load, *CI, std::nullopt);
Value *NewVResult = Builder.CreateInsertElement(VResult, Load, Idx);
// Move the pointer if there are more blocks to come.
@@ -979,9 +909,8 @@ static void scalarizeMaskedCompressStore(const DataLayout &DL,
Value *NewPtr = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, MemIndex);
StoreInst *Store =
Builder.CreateAlignedStore(OneElt, NewPtr, AdjustedAlignment);
- copyMetadataForScalarizedStore(
- *Store, *CI, DL, /*SourcePtrOperand=*/1,
- MemIndex * DL.getTypeAllocSize(EltTy).getFixedValue());
+ copyMetadataForMemoryAccess(
+ *Store, *CI, MemIndex * DL.getTypeAllocSize(EltTy).getFixedValue());
++MemIndex;
}
CI->eraseFromParent();
@@ -1037,8 +966,7 @@ static void scalarizeMaskedCompressStore(const DataLayout &DL,
Value *OneElt = Builder.CreateExtractElement(Src, Idx);
StoreInst *Store =
Builder.CreateAlignedStore(OneElt, Ptr, AdjustedAlignment);
- copyMetadataForScalarizedStore(*Store, *CI, DL, /*SourcePtrOperand=*/1,
- std::nullopt);
+ copyMetadataForMemoryAccess(*Store, *CI, std::nullopt);
// Move the pointer if there are more blocks to come.
Value *NewPtr;
@@ -1115,14 +1043,11 @@ static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI,
continue;
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
LoadInst *Load = Builder.CreateLoad(EltTy, Ptr, "Load" + Twine(Idx));
- copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
- /*ByteOffset=*/0);
+ copyMetadataForMemoryAccess(*Load, *CI, /*DestByteOffset=*/0);
Value *Update =
CreateHistogramUpdateValue(cast<IntrinsicInst>(CI), Load, Inc);
StoreInst *Store = Builder.CreateStore(Update, Ptr);
- copyMetadataForScalarizedStore(*Store, *CI, DL,
- /*SourcePtrOperand=*/0,
- /*ByteOffset=*/0);
+ copyMetadataForMemoryAccess(*Store, *CI, /*DestByteOffset=*/0);
}
CI->eraseFromParent();
return;
@@ -1146,14 +1071,11 @@ static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI,
Builder.SetInsertPoint(CondBlock->getTerminator());
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
LoadInst *Load = Builder.CreateLoad(EltTy, Ptr, "Load" + Twine(Idx));
- copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
- /*ByteOffset=*/0);
+ copyMetadataForMemoryAccess(*Load, *CI, /*DestByteOffset=*/0);
Value *UpdateOp =
CreateHistogramUpdateValue(cast<IntrinsicInst>(CI), Load, Inc);
StoreInst *Store = Builder.CreateStore(UpdateOp, Ptr);
- copyMetadataForScalarizedStore(*Store, *CI, DL,
- /*SourcePtrOperand=*/0,
- /*ByteOffset=*/0);
+ copyMetadataForMemoryAccess(*Store, *CI, /*DestByteOffset=*/0);
// Create "else" block, fill it in the next iteration
BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/metadata.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/metadata.ll
index 40e69c79960e4d..c975819d1668fd 100644
--- a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/metadata.ll
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/metadata.ll
@@ -9,7 +9,7 @@ define <2 x i64> @masked_load_variable(ptr %p, <2 x i1> %mask, <2 x i64> %passth
; CHECK-NEXT: br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
; CHECK: cond.load:
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 0
-; CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 8, !tbaa [[TBAA4:![0-9]+]], !range [[RNG7:![0-9]+]], !invariant.load [[META8:![0-9]+]], !alias.scope [[META9:![0-9]+]], !noalias [[META9]], !nontemporal [[META12:![0-9]+]], !llvm.mem.parallel_loop_access [[META13:![0-9]+]], !llvm.access.group [[META13]], !annotation [[META14:![0-9]+]], !nosanitize [[META8]], !mmra [[META15:![0-9]+]], !noalias.addrspace [[META16:![0-9]+]], !mem.cache_hint [[META17:![0-9]+]]
+; CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 8, !tbaa [[TBAA4:![0-9]+]], !range [[RNG7:![0-9]+]], !invariant.load [[META8:![0-9]+]], !alias.scope [[META9:![0-9]+]], !noalias [[META9]], !nontemporal [[META12:![0-9]+]], !llvm.mem.parallel_loop_access [[META13:![0-9]+]], !llvm.access.group [[META13]], !annotation [[META14:![0-9]+]], !nosanitize [[META8]], !pcsections [[META15:![0-9]+]], !mmra [[META16:![0-9]+]], !noalias.addrspace [[META17:![0-9]+]], !mem.cache_hint [[META18:![0-9]+]]
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP4]], i64 0
; CHECK-NEXT: br label [[ELSE]]
; CHECK: else:
@@ -19,7 +19,7 @@ define <2 x i64> @masked_load_variable(ptr %p, <2 x i1> %mask, <2 x i64> %passth
; CHECK-NEXT: br i1 [[TMP7]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
; CHECK: cond.load1:
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[P]], i32 1
-; CHECK-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP8]], align 8, !tbaa [[TBAA4]], !range [[RNG7]], !invariant.load [[META8]], !alias.scope [[META9]], !noalias [[META9]], !nontemporal [[META12]], !llvm.mem.parallel_loop_access [[META13]], !llvm.access.group [[META13]], !annotation [[META14]], !nosanitize [[META8]], !mmra [[META15]], !noalias.addrspace [[META16]], !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP8]], align 8, !tbaa [[TBAA4]], !range [[RNG7]], !invariant.load [[META8]], !alias.scope [[META9]], !noalias [[META9]], !nontemporal [[META12]], !llvm.mem.parallel_loop_access [[META13]], !llvm.access.group [[META13]], !annotation [[META14]], !nosanitize [[META8]], !pcsections [[META15]], !mmra [[META16]], !noalias.addrspace [[META17]], !mem.cache_hint [[META18]]
; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP9]], i64 1
; CHECK-NEXT: br label [[ELSE2]]
; CHECK: else2:
@@ -33,7 +33,7 @@ define <2 x i64> @masked_load_variable(ptr %p, <2 x i1> %mask, <2 x i64> %passth
define <2 x i64> @masked_load_constant(ptr %p, <2 x i64> %passthru) {
; CHECK-LABEL: @masked_load_constant(
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 1
-; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 8, !tbaa [[TBAA19:![0-9]+]], !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 8, !tbaa [[TBAA20:![0-9]+]], !mem.cache_hint [[META18]]
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP2]], i64 1
; CHECK-NEXT: ret <2 x i64> [[TMP3]]
;
@@ -43,7 +43,7 @@ define <2 x i64> @masked_load_constant(ptr %p, <2 x i64> %passthru) {
define <2 x i64> @masked_load_all_true(ptr %p) {
; CHECK-LABEL: @masked_load_all_true(
-; CHECK-NEXT: [[RET:%.*]] = load <2 x i64>, ptr [[P:%.*]], align 8, !range [[RNG7]], !tbaa.struct [[TBAA_STRUCT21:![0-9]+]], !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[RET:%.*]] = load <2 x i64>, ptr [[P:%.*]], align 8, !range [[RNG7]], !tbaa.struct [[TBAA_STRUCT22:![0-9]+]], !mem.cache_hint [[META18]]
; CHECK-NEXT: ret <2 x i64> [[RET]]
;
%ret = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr %p, i32 8, <2 x i1> <i1 true, i1 true>, <2 x i64> poison), !tbaa.struct !27, !prof !16, !range !12, !mem.cache_hint !0
@@ -59,7 +59,7 @@ define void @masked_store_variable(ptr %p, <2 x i1> %mask, <2 x i64> %value) {
; CHECK: cond.store:
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[VALUE:%.*]], i64 0
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 0
-; CHECK-NEXT: store i64 [[TMP3]], ptr [[TMP4]], align 8, !tbaa [[TBAA4]], !alias.scope [[META9]], !noalias [[META9]], !nontemporal [[META12]], !llvm.mem.parallel_loop_access [[META13]], !llvm.access.group [[META13]], !annotation [[META14]], !nosanitize [[META8]], !mmra [[META15]], !noalias.addrspace [[META16]], !mem.cache_hint [[META22:![0-9]+]]
+; CHECK-NEXT: store i64 [[TMP3]], ptr [[TMP4]], align 8, !tbaa [[TBAA4]], !alias.scope [[META9]], !noalias [[META9]], !nontemporal [[META12]], !llvm.mem.parallel_loop_access [[META13]], !llvm.access.group [[META13]], !annotation [[META14]], !nosanitize [[META8]], !mmra [[META16]], !noalias.addrspace [[META17]], !mem.cache_hint [[META23:![0-9]+]]
; CHECK-NEXT: br label [[ELSE]]
; CHECK: else:
; CHECK-NEXT: [[TMP5:%.*]] = and i2 [[SCALAR_MASK]], -2
@@ -68,7 +68,7 @@ define void @masked_store_variable(ptr %p, <2 x i1> %mask, <2 x i64> %value) {
; CHECK: cond.store1:
; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i64> [[VALUE]], i64 1
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[P]], i32 1
-; CHECK-NEXT: store i64 [[TMP7]], ptr [[TMP8]], align 8, !tbaa [[TBAA4]], !alias.scope [[META9]], !noalias [[META9]], !nontemporal [[META12]], !llvm.mem.parallel_loop_access [[META13]], !llvm.access.group [[META13]], !annotation [[META14]], !nosanitize [[META8]], !mmra [[META15]], !noalias.addrspace [[META16]], !mem.cache_hint [[META22]]
+; CHECK-NEXT: store i64 [[TMP7]], ptr [[TMP8]], align 8, !tbaa [[TBAA4]], !alias.scope [[META9]], !noalias [[META9]], !nontemporal [[META12]], !llvm.mem.parallel_loop_access [[META13]], !llvm.access.group [[META13]], !annotation [[META14]], !nosanitize [[META8]], !mmra [[META16]], !noalias.addrspace [[META17]], !mem.cache_hint [[META23]]
; CHECK-NEXT: br label [[ELSE2]]
; CHECK: else2:
; CHECK-NEXT: ret void
@@ -81,7 +81,7 @@ define void @masked_store_constant(ptr %p, <2 x i64> %value) {
; CHECK-LABEL: @masked_store_constant(
; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i64> [[VALUE:%.*]], i64 1
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 1
-; CHECK-NEXT: store i64 [[TMP1]], ptr [[TMP2]], align 8, !tbaa [[TBAA19]], !mem.cache_hint [[META22]]
+; CHECK-NEXT: store i64 [[TMP1]], ptr [[TMP2]], align 8, !tbaa [[TBAA20]], !mem.cache_hint [[META23]]
; CHECK-NEXT: ret void
;
call void @llvm.masked.store.v2i64.p0(<2 x i64> %value, ptr %p, i32 8, <2 x i1> <i1 false, i1 true>), !tbaa.struct !27, !mem.cache_hint !1
@@ -90,7 +90,7 @@ define void @masked_store_constant(ptr %p, <2 x i64> %value) {
define void @masked_store_all_true(ptr %p, <2 x i64> %value) {
; CHECK-LABEL: @masked_store_all_true(
-; CHECK-NEXT: store <2 x i64> [[VALUE:%.*]], ptr [[P:%.*]], align 8, !tbaa.struct [[TBAA_STRUCT21]], !DIAssignID [[DIASSIGNID23:![0-9]+]], !mem.cache_hint [[META22]]
+; CHECK-NEXT: store <2 x i64> [[VALUE:%.*]], ptr [[P:%.*]], align 8, !tbaa.struct [[TBAA_STRUCT22]], !DIAssignID [[DIASSIGNID24:![0-9]+]], !mem.cache_hint [[META23]]
; CHECK-NEXT: ret void
;
call void @llvm.masked.store.v2i64.p0(<2 x i64> %value, ptr %p, i32 8, <2 x i1> <i1 true, i1 true>), !tbaa.struct !27, !DIAssignID !26, !mem.cache_hint !1
@@ -105,7 +105,7 @@ define <2 x i64> @masked_gather_variable(<2 x ptr> %p, <2 x i1> %mask, <2 x i64>
; CHECK-NEXT: br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
; CHECK: cond.load:
; CHECK-NEXT: [[PTR0:%.*]] = extractelement <2 x ptr> [[P:%.*]], i64 0
-; CHECK-NEXT: [[LOAD0:%.*]] = load i64, ptr [[PTR0]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[LOAD0:%.*]] = load i64, ptr [[PTR0]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META18]]
; CHECK-NEXT: [[RES0:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[LOAD0]], i64 0
; CHECK-NEXT: br label [[ELSE]]
; CHECK: else:
@@ -115,7 +115,7 @@ define <2 x i64> @masked_gather_variable(<2 x ptr> %p, <2 x i1> %mask, <2 x i64>
; CHECK-NEXT: br i1 [[TMP4]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
; CHECK: cond.load1:
; CHECK-NEXT: [[PTR1:%.*]] = extractelement <2 x ptr> [[P]], i64 1
-; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[PTR1]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[PTR1]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META18]]
; CHECK-NEXT: [[RES1:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[LOAD1]], i64 1
; CHECK-NEXT: br label [[ELSE2]]
; CHECK: else2:
@@ -129,7 +129,7 @@ define <2 x i64> @masked_gather_variable(<2 x ptr> %p, <2 x i1> %mask, <2 x i64>
define <2 x i64> @masked_gather_constant(<2 x ptr> %p, <2 x i64> %passthru) {
; CHECK-LABEL: @masked_gather_constant(
; CHECK-NEXT: [[PTR1:%.*]] = extractelement <2 x ptr> [[P:%.*]], i64 1
-; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[PTR1]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[PTR1]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META18]]
; CHECK-NEXT: [[RES1:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[LOAD1]], i64 1
; CHECK-NEXT: ret <2 x i64> [[RES1]]
;
@@ -146,7 +146,7 @@ define void @masked_scatter_variable(<2 x ptr> %p, <2 x i1> %mask, <2 x i64> %va
; CHECK: cond.store:
; CHECK-NEXT: [[ELT0:%.*]] = extractelement <2 x i64> [[VALUE:%.*]], i64 0
; CHECK-NEXT: [[PTR0:%.*]] = extractelement <2 x ptr> [[P:%.*]], i64 0
-; CHECK-NEXT: store i64 [[ELT0]], ptr [[PTR0]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META22]]
+; CHECK-NEXT: store i64 [[ELT0]], ptr [[PTR0]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META23]]
; CHECK-NEXT: br label [[ELSE]]
; CHECK: else:
; CHECK-NEXT: [[TMP3:%.*]] = and i2 [[SCALAR_MASK]], -2
@@ -155,7 +155,7 @@ define void @masked_scatter_variable(<2 x ptr> %p, <2 x i1> %mask, <2 x i64> %va
; CHECK: cond.store1:
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <2 x i64> [[VALUE]], i64 1
; CHECK-NEXT: [[PTR1:%.*]] = extractelement <2 x ptr> [[P]], i64 1
-; CHECK-NEXT: store i64 [[ELT1]], ptr [[PTR1]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META22]]
+; CHECK-NEXT: store i64 [[ELT1]], ptr [[PTR1]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META23]]
; CHECK-NEXT: br label [[ELSE2]]
; CHECK: else2:
; CHECK-NEXT: ret void
@@ -168,7 +168,7 @@ define void @masked_scatter_constant(<2 x ptr> %p, <2 x i64> %value) {
; CHECK-LABEL: @masked_scatter_constant(
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <2 x i64> [[VALUE:%.*]], i64 1
; CHECK-NEXT: [[PTR1:%.*]] = extractelement <2 x ptr> [[P:%.*]], i64 1
-; CHECK-NEXT: store i64 [[ELT1]], ptr [[PTR1]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META22]]
+; CHECK-NEXT: store i64 [[ELT1]], ptr [[PTR1]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META23]]
; CHECK-NEXT: ret void
;
call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> %value, <2 x ptr> %p, i32 8, <2 x i1> <i1 false, i1 true>), !tbaa.struct !27, !mem.cache_hint !1
@@ -182,7 +182,7 @@ define <2 x i64> @masked_expandload_variable(ptr %p, <2 x i1> %mask, <2 x i64> %
; CHECK-NEXT: [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
; CHECK-NEXT: br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
; CHECK: cond.load:
-; CHECK-NEXT: [[TMP3:%.*]] = load i64, ptr [[P:%.*]], align 1, !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[TMP3:%.*]] = load i64, ptr [[P:%.*]], align 1, !mem.cache_hint [[META18]]
; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP3]], i64 0
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[P]], i32 1
; CHECK-NEXT: br label [[ELSE]]
@@ -193,7 +193,7 @@ define <2 x i64> @masked_expandload_variable(ptr %p, <2 x i1> %mask, <2 x i64> %
; CHECK-NEXT: [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0
; CHECK-NEXT: br i1 [[TMP7]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
; CHECK: cond.load1:
-; CHECK-NEXT: [[TMP8:%.*]] = load i64, ptr [[PTR_PHI_ELSE]], align 1, !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i64, ptr [[PTR_PHI_ELSE]], align 1, !mem.cache_hint [[META18]]
; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP8]], i64 1
; CHECK-NEXT: br label [[ELSE2]]
; CHECK: else2:
@@ -207,7 +207,7 @@ define <2 x i64> @masked_expandload_variable(ptr %p, <2 x i1> %mask, <2 x i64> %
define <2 x i64> @masked_expandload_constant(ptr %p, <2 x i64> %passthru) {
; CHECK-LABEL: @masked_expandload_constant(
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 0
-; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[TMP1]], align 1, !tbaa [[TBAA4]], !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[TMP1]], align 1, !tbaa [[TBAA4]], !mem.cache_hint [[META18]]
; CHECK-NEXT: [[RES1:%.*]] = insertelement <2 x i64> poison, i64 [[LOAD1]], i64 1
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i64> [[RES1]], <2 x i64> [[PASSTHRU:%.*]], <2 x i32> <i32 2, i32 1>
; CHECK-NEXT: ret <2 x i64> [[TMP2]]
@@ -224,7 +224,7 @@ define void @masked_compressstore_variable(ptr %p, <2 x i1> %mask, <2 x i64> %va
; CHECK-NEXT: br i1 [[TMP2]], label [[COND_STORE:%.*]], label [[ELSE:%.*]]
; CHECK: cond.store:
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[VALUE:%.*]], i64 0
-; CHECK-NEXT: store i64 [[TMP3]], ptr [[P:%.*]], align 1, !mem.cache_hint [[META22]]
+; CHECK-NEXT: store i64 [[TMP3]], ptr [[P:%.*]], align 1, !mem.cache_hint [[META23]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[P]], i32 1
; CHECK-NEXT: br label [[ELSE]]
; CHECK: else:
@@ -234,7 +234,7 @@ define void @masked_compressstore_variable(ptr %p, <2 x i1> %mask, <2 x i64> %va
; CHECK-NEXT: br i1 [[TMP6]], label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
; CHECK: cond.store1:
; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i64> [[VALUE]], i64 1
-; CHECK-NEXT: store i64 [[TMP7]], ptr [[PTR_PHI_ELSE]], align 1, !mem.cache_hint [[META22]]
+; CHECK-NEXT: store i64 [[TMP7]], ptr [[PTR_PHI_ELSE]], align 1, !mem.cache_hint [[META23]]
; CHECK-NEXT: br label [[ELSE2]]
; CHECK: else2:
; CHECK-NEXT: ret void
@@ -247,7 +247,7 @@ define void @masked_compressstore_constant(ptr %p, <2 x i64> %value) {
; CHECK-LABEL: @masked_compressstore_constant(
; CHECK-NEXT: [[ELT1:%.*]] = extractelement <2 x i64> [[VALUE:%.*]], i64 1
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 0
-; CHECK-NEXT: store i64 [[ELT1]], ptr [[TMP1]], align 1, !tbaa [[TBAA4]], !mem.cache_hint [[META22]]
+; CHECK-NEXT: store i64 [[ELT1]], ptr [[TMP1]], align 1, !tbaa [[TBAA4]], !mem.cache_hint [[META23]]
; CHECK-NEXT: ret void
;
call void @llvm.masked.compressstore.v2i64.p0(<2 x i64> %value, ptr %p, <2 x i1> <i1 false, i1 true>), !tbaa.struct !27, !mem.cache_hint !1
@@ -260,18 +260,18 @@ define void @masked_histogram_variable(<2 x ptr> %p, i64 %inc, <2 x i1> %mask) {
; CHECK-NEXT: br i1 [[MASK0]], label [[COND_HISTOGRAM_UPDATE:%.*]], label [[ELSE:%.*]]
; CHECK: cond.histogram.update:
; CHECK-NEXT: [[PTR0:%.*]] = extractelement <2 x ptr> [[P:%.*]], i64 0
-; CHECK-NEXT: [[LOAD0:%.*]] = load i64, ptr [[PTR0]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[LOAD0:%.*]] = load i64, ptr [[PTR0]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META18]]
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[LOAD0]], [[INC:%.*]]
-; CHECK-NEXT: store i64 [[TMP1]], ptr [[PTR0]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META22]]
+; CHECK-NEXT: store i64 [[TMP1]], ptr [[PTR0]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META23]]
; CHECK-NEXT: br label [[ELSE]]
; CHECK: else:
; CHECK-NEXT: [[MASK1:%.*]] = extractelement <2 x i1> [[MASK]], i64 1
; CHECK-NEXT: br i1 [[MASK1]], label [[COND_HISTOGRAM_UPDATE1:%.*]], label [[ELSE2:%.*]]
; CHECK: cond.histogram.update1:
; CHECK-NEXT: [[PTR1:%.*]] = extractelement <2 x ptr> [[P]], i64 1
-; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[PTR1]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[PTR1]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META18]]
; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[LOAD1]], [[INC]]
-; CHECK-NEXT: store i64 [[TMP2]], ptr [[PTR1]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META22]]
+; CHECK-NEXT: store i64 [[TMP2]], ptr [[PTR1]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META23]]
; CHECK-NEXT: br label [[ELSE2]]
; CHECK: else2:
; CHECK-NEXT: ret void
@@ -283,9 +283,9 @@ define void @masked_histogram_variable(<2 x ptr> %p, i64 %inc, <2 x i1> %mask) {
define void @masked_histogram_constant(<2 x ptr> %p, i64 %inc) {
; CHECK-LABEL: @masked_histogram_constant(
; CHECK-NEXT: [[PTR1:%.*]] = extractelement <2 x ptr> [[P:%.*]], i64 1
-; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[PTR1]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META17]]
+; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[PTR1]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META18]]
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[LOAD1]], [[INC:%.*]]
-; CHECK-NEXT: store i64 [[TMP1]], ptr [[PTR1]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META22]]
+; CHECK-NEXT: store i64 [[TMP1]], ptr [[PTR1]], align 8, !tbaa [[TBAA4]], !mem.cache_hint [[META23]]
; CHECK-NEXT: ret void
;
call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %p, i64 %inc, <2 x i1> <i1 false, i1 true>), !tbaa.struct !27, !mem.cache_hint !0
@@ -300,7 +300,7 @@ define <2 x float> @masked_load_fpmath(ptr %p, <2 x i1> %mask, <2 x float> %pass
; CHECK-NEXT: br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
; CHECK: cond.load:
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[P:%.*]], i32 0
-; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[TMP3]], align 4, !fpmath [[META24:![0-9]+]]
+; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[TMP3]], align 4, !fpmath [[META25:![0-9]+]]
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x float> [[PASSTHRU:%.*]], float [[TMP4]], i64 0
; CHECK-NEXT: br label [[ELSE]]
; CHECK: else:
@@ -310,7 +310,7 @@ define <2 x float> @masked_load_fpmath(ptr %p, <2 x i1> %mask, <2 x float> %pass
; CHECK-NEXT: br i1 [[TMP7]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
; CHECK: cond.load1:
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[P]], i32 1
-; CHECK-NEXT: [[TMP9:%.*]] = load float, ptr [[TMP8]], align 4, !fpmath [[META24]]
+; CHECK-NEXT: [[TMP9:%.*]] = load float, ptr [[TMP8]], align 4, !fpmath [[META25]]
; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x float> [[RES_PHI_ELSE]], float [[TMP9]], i64 1
; CHECK-NEXT: br label [[ELSE2]]
; CHECK: else2:
@@ -323,28 +323,28 @@ define <2 x float> @masked_load_fpmath(ptr %p, <2 x i1> %mask, <2 x float> %pass
define <2 x i64> @masked_load_debug(ptr %p, <2 x i1> %mask, <2 x i64> %passthru) !dbg !23 {
; CHECK-LABEL: @masked_load_debug(
-; CHECK-NEXT: [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2, !dbg [[DBG27:![0-9]+]]
-; CHECK-NEXT: [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1, !dbg [[DBG27]]
-; CHECK-NEXT: [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0, !dbg [[DBG27]]
-; CHECK-NEXT: br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]], !dbg [[DBG27]]
+; CHECK-NEXT: [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2, !dbg [[DBG28:![0-9]+]]
+; CHECK-NEXT: [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1, !dbg [[DBG28]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0, !dbg [[DBG28]]
+; CHECK-NEXT: br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]], !dbg [[DBG28]]
; CHECK: cond.load:
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 0, !dbg [[DBG27]]
-; CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 8, !dbg [[DBG27]]
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP4]], i64 0, !dbg [[DBG27]]
-; CHECK-NEXT: br label [[ELSE]], !dbg [[DBG27]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 0, !dbg [[DBG28]]
+; CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 8, !dbg [[DBG28]]
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP4]], i64 0, !dbg [[DBG28]]
+; CHECK-NEXT: br label [[ELSE]], !dbg [[DBG28]]
; CHECK: else:
-; CHECK-NEXT: [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP5]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ], !dbg [[DBG27]]
-; CHECK-NEXT: [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2, !dbg [[DBG27]]
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0, !dbg [[DBG27]]
-; CHECK-NEXT: br i1 [[TMP7]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]], !dbg [[DBG27]]
+; CHECK-NEXT: [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP5]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ], !dbg [[DBG28]]
+; CHECK-NEXT: [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2, !dbg [[DBG28]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0, !dbg [[DBG28]]
+; CHECK-NEXT: br i1 [[TMP7]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]], !dbg [[DBG28]]
; CHECK: cond.load1:
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[P]], i32 1, !dbg [[DBG27]]
-; CHECK-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP8]], align 8, !dbg [[DBG27]]
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP9]], i64 1, !dbg [[DBG27]]
-; CHECK-NEXT: br label [[ELSE2]], !dbg [[DBG27]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[P]], i32 1, !dbg [[DBG28]]
+; CHECK-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP8]], align 8, !dbg [[DBG28]]
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP9]], i64 1, !dbg [[DBG28]]
+; CHECK-NEXT: br label [[ELSE2]], !dbg [[DBG28]]
; CHECK: else2:
-; CHECK-NEXT: [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP10]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ], !dbg [[DBG27]]
-; CHECK-NEXT: ret <2 x i64> [[RES_PHI_ELSE3]], !dbg [[DBG28:![0-9]+]]
+; CHECK-NEXT: [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP10]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ], !dbg [[DBG28]]
+; CHECK-NEXT: ret <2 x i64> [[RES_PHI_ELSE3]], !dbg [[DBG29:![0-9]+]]
;
%ret = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr %p, i32 8, <2 x i1> %mask, <2 x i64> %passthru), !dbg !24
ret <2 x i64> %ret, !dbg !25
@@ -416,18 +416,19 @@ declare <2 x float> @llvm.masked.load.v2f32.p0(ptr, i32, <2 x i1>, <2 x float>)
; CHECK: [[META12]] = !{i32 1}
; CHECK: [[META13]] = distinct !{}
; CHECK: [[META14]] = !{!"test.annotation"}
-; CHECK: [[META15]] = !{!"test", !"relaxed"}
-; CHECK: [[META16]] = !{i32 1, i32 2}
-; CHECK: [[META17]] = !{i32 0, [[META18:![0-9]+]]}
-; CHECK: [[META18]] = !{!"test.cache.hint", !"value"}
-; CHECK: [[TBAA19]] = !{[[META20:![0-9]+]], [[META20]], i64 0}
-; CHECK: [[META20]] = !{!"second long", [[META6]], i64 0}
-; CHECK: [[TBAA_STRUCT21]] = !{i64 0, i64 8, [[TBAA4]], i64 8, i64 8, [[TBAA19]]}
-; CHECK: [[META22]] = !{i32 1, [[META18]]}
-; CHECK: [[DIASSIGNID23]] = distinct !DIAssignID()
-; CHECK: [[META24]] = !{float 2.500000e+00}
-; CHECK: [[META25:![0-9]+]] = distinct !DISubprogram(name: "masked_load_debug", scope: [[META1]], file: [[META1]], line: 1, type: [[META26:![0-9]+]], scopeLine: 1, spFlags: DISPFlagDefinition, unit: [[META0]])
-; CHECK: [[META26]] = !DISubroutineType(types: [[META8]])
-; CHECK: [[DBG27]] = !DILocation(line: 2, column: 1, scope: [[META25]])
-; CHECK: [[DBG28]] = !DILocation(line: 3, column: 1, scope: [[META25]])
+; CHECK: [[META15]] = !{!"metadata_section"}
+; CHECK: [[META16]] = !{!"test", !"relaxed"}
+; CHECK: [[META17]] = !{i32 1, i32 2}
+; CHECK: [[META18]] = !{i32 0, [[META19:![0-9]+]]}
+; CHECK: [[META19]] = !{!"test.cache.hint", !"value"}
+; CHECK: [[TBAA20]] = !{[[META21:![0-9]+]], [[META21]], i64 0}
+; CHECK: [[META21]] = !{!"second long", [[META6]], i64 0}
+; CHECK: [[TBAA_STRUCT22]] = !{i64 0, i64 8, [[TBAA4]], i64 8, i64 8, [[TBAA20]]}
+; CHECK: [[META23]] = !{i32 1, [[META19]]}
+; CHECK: [[DIASSIGNID24]] = distinct !DIAssignID()
+; CHECK: [[META25]] = !{float 2.500000e+00}
+; CHECK: [[META26:![0-9]+]] = distinct !DISubprogram(name: "masked_load_debug", scope: [[META1]], file: [[META1]], line: 1, type: [[META27:![0-9]+]], scopeLine: 1, spFlags: DISPFlagDefinition, unit: [[META0]])
+; CHECK: [[META27]] = !DISubroutineType(types: [[META8]])
+; CHECK: [[DBG28]] = !DILocation(line: 2, column: 1, scope: [[META26]])
+; CHECK: [[DBG29]] = !DILocation(line: 3, column: 1, scope: [[META26]])
;.
More information about the llvm-branch-commits
mailing list