[llvm-branch-commits] [llvm] [Analysis] Add metadata propagation for replacement memory accesses (PR #242249)
Yonah Goldberg via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Nov 18 01:13:45 PST 2026
https://github.com/YonahGoldberg updated https://github.com/llvm/llvm-project/pull/242249
>From 6b95f61b9b54b0bd67d1d927646ae3c6a993ffc5 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Fri, 13 Nov 2026 10:35:37 +0000
Subject: [PATCH 01/14] [Transforms] Add memory metadata tests
---
.../masked_intrinsics_keep_metadata.ll | 19 ++
.../InstCombine/memory-cache-hint.ll | 59 +++++
.../Transforms/InstCombine/memory-metadata.ll | 185 +++++++++++++++
.../InstCombine/ptr-replace-alloca.ll | 4 +-
.../InstCombine/storemerge-metadata.ll | 213 +++++++++++++++++
llvm/test/Transforms/SROA/memory-metadata.ll | 215 ++++++++++++++++++
.../Scalarizer/basic-inseltpoison.ll | 6 +-
llvm/test/Transforms/Scalarizer/basic.ll | 6 +-
.../Scalarizer/memory-cache-hint.ll | 35 +++
.../Transforms/Scalarizer/memory-metadata.ll | 150 ++++++++++++
.../VectorCombine/RISCV/memory-metadata.ll | 47 ++++
.../VectorCombine/X86/memory-metadata.ll | 72 ++++++
.../VectorCombine/memory-metadata.ll | 195 ++++++++++++++++
13 files changed, 1197 insertions(+), 9 deletions(-)
create mode 100644 llvm/test/Transforms/InstCombine/memory-cache-hint.ll
create mode 100644 llvm/test/Transforms/InstCombine/memory-metadata.ll
create mode 100644 llvm/test/Transforms/InstCombine/storemerge-metadata.ll
create mode 100644 llvm/test/Transforms/SROA/memory-metadata.ll
create mode 100644 llvm/test/Transforms/Scalarizer/memory-cache-hint.ll
create mode 100644 llvm/test/Transforms/Scalarizer/memory-metadata.ll
create mode 100644 llvm/test/Transforms/VectorCombine/RISCV/memory-metadata.ll
create mode 100644 llvm/test/Transforms/VectorCombine/X86/memory-metadata.ll
create mode 100644 llvm/test/Transforms/VectorCombine/memory-metadata.ll
diff --git a/llvm/test/Transforms/InstCombine/masked_intrinsics_keep_metadata.ll b/llvm/test/Transforms/InstCombine/masked_intrinsics_keep_metadata.ll
index 0832561e2b02b5..e6029c7b58e73b 100644
--- a/llvm/test/Transforms/InstCombine/masked_intrinsics_keep_metadata.ll
+++ b/llvm/test/Transforms/InstCombine/masked_intrinsics_keep_metadata.ll
@@ -40,6 +40,20 @@ b0:
ret void
}
+; For an { i32, float } object, only the integer field is initially read.
+; TODO: Check whether the integer TBAA tag remains valid on the unmasked load,
+; which also reads the inactive float field. Record the existing behavior.
+define <2 x i32> @unmask_integer_field_tbaa(ptr dereferenceable(8) %p) {
+; CHECK-LABEL: define <2 x i32> @unmask_integer_field_tbaa(
+; CHECK-SAME: ptr dereferenceable(8) [[P:%.*]]) {
+; CHECK-NEXT: [[UNMASKEDLOAD:%.*]] = load <2 x i32>, ptr [[P]], align 1, !tbaa [[INT_TBAA3:![0-9]+]]
+; CHECK-NEXT: [[V1:%.*]] = insertelement <2 x i32> [[UNMASKEDLOAD]], i32 0, i64 1
+; CHECK-NEXT: ret <2 x i32> [[V1]]
+;
+ %v = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr %p, <2 x i1> <i1 true, i1 false>, <2 x i32> zeroinitializer), !tbaa !3
+ ret <2 x i32> %v
+}
+
attributes #0 = { norecurse nounwind }
declare <4 x i32> @llvm.masked.load.v4i1.p0(ptr, i32, <4 x i1>, <4 x i32>)
@@ -49,8 +63,13 @@ declare void @llvm.masked.store.v4i1.p0(<4 x i32>, ptr, i32, <4 x i1>)
!1 = !{!"omnipotent char", !2, i64 0}
!2 = !{!"Simple C/C++ TBAA"}
+!3 = !{!4, !4, i64 0}
+!4 = !{!"int", !1, i64 0}
+
;.
; CHECK: [[CHAR_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
; CHECK: [[META1]] = !{!"omnipotent char", [[META2:![0-9]+]], i64 0}
; CHECK: [[META2]] = !{!"Simple C/C++ TBAA"}
+; CHECK: [[INT_TBAA3]] = !{[[META4:![0-9]+]], [[META4]], i64 0}
+; CHECK: [[META4]] = !{!"int", [[META1]], i64 0}
;.
diff --git a/llvm/test/Transforms/InstCombine/memory-cache-hint.ll b/llvm/test/Transforms/InstCombine/memory-cache-hint.ll
new file mode 100644
index 00000000000000..653fc45056db27
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memory-cache-hint.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=instcombine < %s | FileCheck %s
+
+define void @store_bitcast(ptr %p, float %x) {
+; CHECK-LABEL: define void @store_bitcast(
+; CHECK-SAME: ptr [[P:%.*]], float [[X:%.*]]) {
+; CHECK-NEXT: store float [[X]], ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
+ %bits = bitcast float %x to i32
+ store i32 %bits, ptr %p, align 4, !mem.cache_hint !0
+ ret void
+}
+
+define void @copy_both_hints(ptr %dst, ptr %src) {
+; CHECK-LABEL: define void @copy_both_hints(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-NEXT: store i32 [[TMP1]], ptr [[DST]], align 4
+; CHECK-NEXT: ret void
+;
+ call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dst, ptr align 4 %src, i64 4, i1 false), !mem.cache_hint !2
+ ret void
+}
+
+define void @set_destination_hint(ptr %dst) {
+; CHECK-LABEL: define void @set_destination_hint(
+; CHECK-SAME: ptr [[DST:%.*]]) {
+; CHECK-NEXT: store i32 0, ptr [[DST]], align 4
+; CHECK-NEXT: ret void
+;
+ call void @llvm.memset.p0.i64(ptr align 4 %dst, i8 0, i64 4, i1 false), !mem.cache_hint !3
+ ret void
+}
+
+; TODO: Check whether invariant.load can be kept after exposing inactive lanes.
+; This records the existing behavior while preserving the cache policy.
+define <2 x i32> @speculated_masked_load(ptr dereferenceable(8) %p, <2 x i1> %mask) {
+; CHECK-LABEL: define <2 x i32> @speculated_masked_load(
+; CHECK-SAME: ptr dereferenceable(8) [[P:%.*]], <2 x i1> [[MASK:%.*]]) {
+; CHECK-NEXT: [[UNMASKEDLOAD:%.*]] = load <2 x i32>, ptr [[P]], align 1, !invariant.load [[META0:![0-9]+]], !mem.cache_hint [[META1:![0-9]+]]
+; CHECK-NEXT: [[V:%.*]] = select <2 x i1> [[MASK]], <2 x i32> [[UNMASKEDLOAD]], <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x i32> [[V]]
+;
+ %v = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr %p, <2 x i1> %mask, <2 x i32> zeroinitializer), !mem.cache_hint !3, !invariant.load !6
+ ret <2 x i32> %v
+}
+
+!0 = !{i32 1, !1}
+!1 = !{!"nvvm.l1_eviction", !"last"}
+!2 = !{i32 0, !1, i32 1, !4}
+!3 = !{i32 0, !1}
+!4 = !{!"nvvm.l1_eviction", !"first"}
+!6 = !{}
+;.
+; CHECK: [[META0]] = !{}
+; CHECK: [[META1]] = !{i32 0, [[META2:![0-9]+]]}
+; CHECK: [[META2]] = !{!"nvvm.l1_eviction", !"last"}
+;.
diff --git a/llvm/test/Transforms/InstCombine/memory-metadata.ll b/llvm/test/Transforms/InstCombine/memory-metadata.ll
new file mode 100644
index 00000000000000..8ec8483485d263
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memory-metadata.ll
@@ -0,0 +1,185 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=instcombine < %s | FileCheck %s --implicit-check-not=!test.unknown --implicit-check-not=!prof
+
+target datalayout = "e-p:64:64"
+
+define float @bitcast_load(ptr %p) {
+; CHECK-LABEL: define float @bitcast_load(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[V1:%.*]] = load float, ptr [[P]], align 4, !tbaa [[INT_TBAA0:![0-9]+]], !invariant.load [[META4:![0-9]+]], !alias.scope [[META5:![0-9]+]], !noalias [[META5]], !nontemporal [[META8:![0-9]+]], !llvm.mem.parallel_loop_access [[META9:![0-9]+]], !invariant.group [[META4]], !llvm.access.group [[META9]], !noundef [[META4]], !annotation [[META10:![0-9]+]], !noalias.addrspace [[META11:![0-9]+]], !mem.cache_hint [[META12:![0-9]+]]
+; CHECK-NEXT: ret float [[V1]]
+;
+ %v = load i32, ptr %p, align 4, !tbaa !10, !range !9, !noundef !0, !invariant.load !0, !invariant.group !0, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+ %r = bitcast i32 %v to float
+ ret float %r
+}
+
+define void @bitcast_store(ptr %p, float %v) {
+; CHECK-LABEL: define void @bitcast_store(
+; CHECK-SAME: ptr [[P:%.*]], float [[V:%.*]]) {
+; CHECK-NEXT: store float [[V]], ptr [[P]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]]
+; CHECK-NEXT: ret void
+;
+ %bits = bitcast float %v to i32
+ store i32 %bits, ptr %p, align 4, !tbaa !10, !invariant.group !0, !mem.cache_hint !18, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+ ret void
+}
+
+define i32 @unpack_struct_load(ptr %p) {
+; CHECK-LABEL: define i32 @unpack_struct_load(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[V_UNPACK:%.*]] = load i32, ptr [[P]], align 8, !tbaa.struct [[TBAA_STRUCT14:![0-9]+]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT: [[V_ELT1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4, !annotation [[META10]]
+; CHECK-NEXT: [[V_UNPACK2:%.*]] = load i32, ptr [[V_ELT1]], align 4, !tbaa.struct [[TBAA_STRUCT14]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT: [[R:%.*]] = add i32 [[V_UNPACK]], [[V_UNPACK2]]
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %v = load {i32, i32}, ptr %p, align 8, !tbaa.struct !19, !invariant.load !0, !invariant.group !0, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+ %a = extractvalue {i32, i32} %v, 0
+ %b = extractvalue {i32, i32} %v, 1
+ %r = add i32 %a, %b
+ ret i32 %r
+}
+
+define i32 @unpack_array_load(ptr %p) {
+; CHECK-LABEL: define i32 @unpack_array_load(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[V_UNPACK:%.*]] = load i32, ptr [[P]], align 8, !tbaa.struct [[TBAA_STRUCT14]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT: [[V_ELT1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4, !annotation [[META10]]
+; CHECK-NEXT: [[V_UNPACK2:%.*]] = load i32, ptr [[V_ELT1]], align 4, !tbaa.struct [[TBAA_STRUCT14]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT: [[R:%.*]] = add i32 [[V_UNPACK]], [[V_UNPACK2]]
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %v = load [2 x i32], ptr %p, align 8, !tbaa.struct !19, !invariant.load !0, !invariant.group !0, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+ %a = extractvalue [2 x i32] %v, 0
+ %b = extractvalue [2 x i32] %v, 1
+ %r = add i32 %a, %b
+ ret i32 %r
+}
+
+define void @unpack_struct_store(ptr %p, {i32, i32} %v) {
+; CHECK-LABEL: define void @unpack_struct_store(
+; CHECK-SAME: ptr [[P:%.*]], { i32, i32 } [[V:%.*]]) {
+; CHECK-NEXT: [[V_ELT:%.*]] = extractvalue { i32, i32 } [[V]], 0, !annotation [[META10]]
+; CHECK-NEXT: store i32 [[V_ELT]], ptr [[P]], align 8, !tbaa.struct [[TBAA_STRUCT14]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT: [[P_REPACK1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4, !annotation [[META10]]
+; CHECK-NEXT: [[V_ELT2:%.*]] = extractvalue { i32, i32 } [[V]], 1, !annotation [[META10]]
+; CHECK-NEXT: store i32 [[V_ELT2]], ptr [[P_REPACK1]], align 4, !tbaa.struct [[TBAA_STRUCT14]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT: ret void
+;
+ store {i32, i32} %v, ptr %p, align 8, !tbaa.struct !19, !invariant.group !0, !mem.cache_hint !18, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+ ret void
+}
+
+define void @unpack_array_store(ptr %p, [2 x i32] %v) {
+; CHECK-LABEL: define void @unpack_array_store(
+; CHECK-SAME: ptr [[P:%.*]], [2 x i32] [[V:%.*]]) {
+; CHECK-NEXT: [[V_ELT:%.*]] = extractvalue [2 x i32] [[V]], 0, !annotation [[META10]]
+; CHECK-NEXT: store i32 [[V_ELT]], ptr [[P]], align 8, !tbaa.struct [[TBAA_STRUCT14]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT: [[P_REPACK1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4, !annotation [[META10]]
+; CHECK-NEXT: [[V_ELT2:%.*]] = extractvalue [2 x i32] [[V]], 1, !annotation [[META10]]
+; CHECK-NEXT: store i32 [[V_ELT2]], ptr [[P_REPACK1]], align 4, !tbaa.struct [[TBAA_STRUCT14]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT: ret void
+;
+ store [2 x i32] %v, ptr %p, align 8, !tbaa.struct !19, !invariant.group !0, !mem.cache_hint !18, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+ ret void
+}
+
+; The two pointer operands carry different policies. Call profiling must not
+; be attached to the replacement native memory operations.
+define void @copy_memory(ptr %dst, ptr %src) {
+; CHECK-LABEL: define void @copy_memory(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load i64, ptr [[SRC]], align 8, !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT: store i64 [[TMP1]], ptr [[DST]], align 8, !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT: ret void
+;
+ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %src, i64 8, i1 false), !tbaa.struct !19, !prof !24, !mem.cache_hint !22, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+ ret void
+}
+
+define void @set_memory(ptr %dst) {
+; CHECK-LABEL: define void @set_memory(
+; CHECK-SAME: ptr [[DST:%.*]]) {
+; CHECK-NEXT: store i32 0, ptr [[DST]], align 4, !annotation [[META10]]
+; CHECK-NEXT: ret void
+;
+ call void @llvm.memset.p0.i64(ptr align 4 %dst, i8 0, i64 4, i1 false), !tbaa !10, !prof !24, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+ ret void
+}
+
+define <2 x i32> @splat_gather(ptr %p) {
+; CHECK-LABEL: define <2 x i32> @splat_gather(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[LOAD_SCALAR:%.*]] = load i32, ptr [[P]], align 1, !annotation [[META10]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[LOAD_SCALAR]], i64 0, !annotation [[META10]]
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer, !annotation [[META10]]
+; CHECK-NEXT: ret <2 x i32> [[BROADCAST_SPLAT]]
+;
+ %ptr0 = insertelement <2 x ptr> poison, ptr %p, i32 0
+ %ptrs = shufflevector <2 x ptr> %ptr0, <2 x ptr> poison, <2 x i32> zeroinitializer
+ %v = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> %ptrs, <2 x i1> <i1 true, i1 true>, <2 x i32> poison), !tbaa !10, !range !9, !invariant.load !0, !prof !24, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+ ret <2 x i32> %v
+}
+
+define void @splat_scatter(ptr %p, i32 %v) {
+; CHECK-LABEL: define void @splat_scatter(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[V:%.*]]) {
+; CHECK-NEXT: store i32 [[V]], ptr [[P]], align 1, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META17:![0-9]+]], !noalias.addrspace [[META11]], !mem.cache_hint [[META18:![0-9]+]]
+; CHECK-NEXT: ret void
+;
+ %ptr0 = insertelement <2 x ptr> poison, ptr %p, i32 0
+ %ptrs = shufflevector <2 x ptr> %ptr0, <2 x ptr> poison, <2 x i32> zeroinitializer
+ %val0 = insertelement <2 x i32> poison, i32 %v, i32 0
+ %vals = shufflevector <2 x i32> %val0, <2 x i32> poison, <2 x i32> zeroinitializer
+ call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> %vals, <2 x ptr> %ptrs, <2 x i1> <i1 true, i1 true>), !tbaa !10, !mem.cache_hint !18, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4
+ ret void
+}
+
+!0 = !{}
+!1 = !{i32 1}
+!2 = !{!"preserved memory metadata"}
+!3 = !{!"test", !"relaxed"}
+!4 = !{i32 5, i32 6}
+!5 = distinct !{}
+!6 = !{!7}
+!7 = distinct !{!7, !8, !"scope"}
+!8 = distinct !{!8, i1 false, !"domain"}
+!9 = !{i32 0, i32 100}
+!10 = !{!11, !11, i64 0}
+!11 = !{!"int", !12, i64 0}
+!12 = !{!"omnipotent char", !13, i64 0}
+!13 = !{!"root"}
+!14 = !{float 2.5}
+!15 = !{i32 3}
+!16 = !{i32 0, !17}
+!17 = !{!"nvvm.l1_eviction", !"last"}
+!18 = !{i32 1, !17}
+!19 = !{i64 0, i64 4, !10, i64 4, i64 4, !20}
+!20 = !{!21, !21, i64 0}
+!21 = !{!"float", !12, i64 0}
+!22 = !{i32 0, !17, i32 1, !23}
+!23 = !{!"nvvm.l1_eviction", !"first"}
+!24 = !{!"branch_weights", i32 10}
+!25 = !{i64 8}
+;.
+; CHECK: [[INT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; CHECK: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
+; CHECK: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
+; CHECK: [[META3]] = !{!"root"}
+; CHECK: [[META4]] = !{}
+; CHECK: [[META5]] = !{[[META6:![0-9]+]]}
+; CHECK: [[META6]] = distinct !{[[META6]], [[META7:![0-9]+]], !"scope"}
+; CHECK: [[META7]] = distinct !{[[META7]], i1 false, !"domain"}
+; CHECK: [[META8]] = !{i32 1}
+; CHECK: [[META9]] = distinct !{}
+; CHECK: [[META10]] = !{!"preserved memory metadata"}
+; CHECK: [[META11]] = !{i32 5, i32 6}
+; CHECK: [[META12]] = !{i32 0, [[META13:![0-9]+]]}
+; CHECK: [[META13]] = !{!"nvvm.l1_eviction", !"last"}
+; CHECK: [[TBAA_STRUCT14]] = !{i64 0, i64 4, [[INT_TBAA0]], i64 4, i64 4, [[META15:![0-9]+]]}
+; CHECK: [[META15]] = !{[[META16:![0-9]+]], [[META16]], i64 0}
+; CHECK: [[META16]] = !{!"float", [[META2]], i64 0}
+; CHECK: [[META17]] = !{!"test", !"relaxed"}
+; CHECK: [[META18]] = !{i32 1, [[META13]]}
+;.
diff --git a/llvm/test/Transforms/InstCombine/ptr-replace-alloca.ll b/llvm/test/Transforms/InstCombine/ptr-replace-alloca.ll
index a3bab9d5403bac..d0043e58537f82 100644
--- a/llvm/test/Transforms/InstCombine/ptr-replace-alloca.ll
+++ b/llvm/test/Transforms/InstCombine/ptr-replace-alloca.ll
@@ -514,9 +514,7 @@ define i8 @call_readonly_keep_alloca2() {
ret i8 %v
}
-; The pointer changes but the load type doesn't, so all metadata (here
-; !annotation, which the type-changing copyMetadataForLoad would drop) must be
-; preserved.
+; Only the pointer changes, so preserve all metadata on the load.
@g3 = external constant [32 x i8], align 16
define i8 @preserve_load_metadata(i64 %idx) {
diff --git a/llvm/test/Transforms/InstCombine/storemerge-metadata.ll b/llvm/test/Transforms/InstCombine/storemerge-metadata.ll
new file mode 100644
index 00000000000000..dc5fdee99c333f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/storemerge-metadata.ll
@@ -0,0 +1,213 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=instcombine < %s | FileCheck %s
+
+define void @common_metadata(ptr %p, i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: define void @common_metadata(
+; CHECK-SAME: ptr [[P:%.*]], i1 [[C:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br i1 [[C]], label %[[LEFT:.*]], label %[[RIGHT:.*]]
+; CHECK: [[LEFT]]:
+; CHECK-NEXT: br label %[[JOIN:.*]]
+; CHECK: [[RIGHT]]:
+; CHECK-NEXT: br label %[[JOIN]]
+; CHECK: [[JOIN]]:
+; CHECK-NEXT: [[STOREMERGE:%.*]] = phi i32 [ [[B]], %[[RIGHT]] ], [ [[A]], %[[LEFT]] ]
+; CHECK-NEXT: store i32 [[STOREMERGE]], ptr [[P]], align 4, !tbaa [[INT_TBAA0:![0-9]+]], !alias.scope [[META4:![0-9]+]], !noalias [[META4]], !llvm.access.group [[ACC_GRP7:![0-9]+]]
+; CHECK-NEXT: ret void
+;
+entry:
+ br i1 %c, label %left, label %right
+left:
+ store i32 %a, ptr %p, align 4, !tbaa !10, !alias.scope !14, !noalias !14, !nontemporal !4, !llvm.access.group !8, !mem.cache_hint !0
+ br label %join
+right:
+ store i32 %b, ptr %p, align 4, !tbaa !10, !alias.scope !14, !noalias !14, !nontemporal !4, !llvm.access.group !8, !mem.cache_hint !0
+ br label %join
+join:
+ ret void
+}
+
+define void @intersect_access_groups(ptr %p, i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: define void @intersect_access_groups(
+; CHECK-SAME: ptr [[P:%.*]], i1 [[C:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br i1 [[C]], label %[[LEFT:.*]], label %[[RIGHT:.*]]
+; CHECK: [[LEFT]]:
+; CHECK-NEXT: br label %[[JOIN:.*]]
+; CHECK: [[RIGHT]]:
+; CHECK-NEXT: br label %[[JOIN]]
+; CHECK: [[JOIN]]:
+; CHECK-NEXT: [[STOREMERGE:%.*]] = phi i32 [ [[B]], %[[RIGHT]] ], [ [[A]], %[[LEFT]] ]
+; CHECK-NEXT: store i32 [[STOREMERGE]], ptr [[P]], align 4, !llvm.access.group [[ACC_GRP9:![0-9]+]]
+; CHECK-NEXT: ret void
+;
+entry:
+ br i1 %c, label %left, label %right
+left:
+ store i32 %a, ptr %p, align 4, !llvm.access.group !8
+ br label %join
+right:
+ store i32 %b, ptr %p, align 4, !llvm.access.group !9
+ br label %join
+join:
+ ret void
+}
+
+define void @conflicting_cache_hints(ptr %p, i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: define void @conflicting_cache_hints(
+; CHECK-SAME: ptr [[P:%.*]], i1 [[C:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br i1 [[C]], label %[[LEFT:.*]], label %[[RIGHT:.*]]
+; CHECK: [[LEFT]]:
+; CHECK-NEXT: br label %[[JOIN:.*]]
+; CHECK: [[RIGHT]]:
+; CHECK-NEXT: br label %[[JOIN]]
+; CHECK: [[JOIN]]:
+; CHECK-NEXT: [[STOREMERGE:%.*]] = phi i32 [ [[B]], %[[RIGHT]] ], [ [[A]], %[[LEFT]] ]
+; CHECK-NEXT: store i32 [[STOREMERGE]], ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ br i1 %c, label %left, label %right
+left:
+ store i32 %a, ptr %p, align 4, !mem.cache_hint !0
+ br label %join
+right:
+ store i32 %b, ptr %p, align 4, !mem.cache_hint !2
+ br label %join
+join:
+ ret void
+}
+
+define void @metadata_only_left(ptr %p, i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: define void @metadata_only_left(
+; CHECK-SAME: ptr [[P:%.*]], i1 [[C:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br i1 [[C]], label %[[LEFT:.*]], label %[[RIGHT:.*]]
+; CHECK: [[LEFT]]:
+; CHECK-NEXT: br label %[[JOIN:.*]]
+; CHECK: [[RIGHT]]:
+; CHECK-NEXT: br label %[[JOIN]]
+; CHECK: [[JOIN]]:
+; CHECK-NEXT: [[STOREMERGE:%.*]] = phi i32 [ [[B]], %[[RIGHT]] ], [ [[A]], %[[LEFT]] ]
+; CHECK-NEXT: store i32 [[STOREMERGE]], ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ br i1 %c, label %left, label %right
+left:
+ store i32 %a, ptr %p, align 4, !tbaa !10, !alias.scope !14, !noalias !14, !nontemporal !4, !llvm.access.group !8, !mem.cache_hint !0
+ br label %join
+right:
+ store i32 %b, ptr %p, align 4
+ br label %join
+join:
+ ret void
+}
+
+define void @metadata_only_right(ptr %p, i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: define void @metadata_only_right(
+; CHECK-SAME: ptr [[P:%.*]], i1 [[C:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br i1 [[C]], label %[[LEFT:.*]], label %[[RIGHT:.*]]
+; CHECK: [[LEFT]]:
+; CHECK-NEXT: br label %[[JOIN:.*]]
+; CHECK: [[RIGHT]]:
+; CHECK-NEXT: br label %[[JOIN]]
+; CHECK: [[JOIN]]:
+; CHECK-NEXT: [[STOREMERGE:%.*]] = phi i32 [ [[B]], %[[RIGHT]] ], [ [[A]], %[[LEFT]] ]
+; CHECK-NEXT: store i32 [[STOREMERGE]], ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ br i1 %c, label %left, label %right
+left:
+ store i32 %a, ptr %p, align 4
+ br label %join
+right:
+ store i32 %b, ptr %p, align 4, !tbaa !10, !alias.scope !14, !noalias !14, !nontemporal !4, !llvm.access.group !8, !mem.cache_hint !0
+ br label %join
+join:
+ ret void
+}
+
+define void @triangle_common_metadata(ptr %p, i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: define void @triangle_common_metadata(
+; CHECK-SAME: ptr [[P:%.*]], i1 [[C:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br i1 [[C]], label %[[THEN:.*]], label %[[JOIN:.*]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: br label %[[JOIN]]
+; CHECK: [[JOIN]]:
+; CHECK-NEXT: [[STOREMERGE:%.*]] = phi i32 [ [[B]], %[[THEN]] ], [ [[A]], %[[ENTRY]] ]
+; CHECK-NEXT: store i32 [[STOREMERGE]], ptr [[P]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ store i32 %a, ptr %p, align 4, !nontemporal !4, !mem.cache_hint !0
+ br i1 %c, label %then, label %join
+then:
+ store i32 %b, ptr %p, align 4, !nontemporal !4, !mem.cache_hint !0
+ br label %join
+join:
+ ret void
+}
+; Capture components must be combined across both incoming pointer values.
+define void @merge_pointer_captures(ptr %p, i1 %c, ptr %a, ptr %b) {
+; CHECK-LABEL: define void @merge_pointer_captures(
+; CHECK-SAME: ptr [[P:%.*]], i1 [[C:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br i1 [[C]], label %[[LEFT:.*]], label %[[RIGHT:.*]]
+; CHECK: [[LEFT]]:
+; CHECK-NEXT: br label %[[JOIN:.*]]
+; CHECK: [[RIGHT]]:
+; CHECK-NEXT: br label %[[JOIN]]
+; CHECK: [[JOIN]]:
+; CHECK-NEXT: [[STOREMERGE:%.*]] = phi ptr [ [[B]], %[[RIGHT]] ], [ [[A]], %[[LEFT]] ]
+; CHECK-NEXT: store ptr [[STOREMERGE]], ptr [[P]], align 8
+; CHECK-NEXT: ret void
+;
+entry:
+ br i1 %c, label %left, label %right
+left:
+ store ptr %a, ptr %p, align 8, !captures !17
+ br label %join
+right:
+ store ptr %b, ptr %p, align 8, !captures !18
+ br label %join
+join:
+ ret void
+}
+
+!0 = !{i32 1, !1}
+!1 = !{!"nvvm.l1_eviction", !"last"}
+!2 = !{i32 1, !3}
+!3 = !{!"nvvm.l1_eviction", !"first"}
+!4 = !{i32 1}
+!5 = distinct !{}
+!6 = distinct !{}
+!7 = distinct !{}
+!8 = !{!5, !6}
+!9 = !{!6, !7}
+!10 = !{!11, !11, i64 0}
+!11 = !{!"int", !12}
+!12 = !{!"char", !13}
+!13 = !{!"root"}
+!14 = !{!15}
+!15 = distinct !{!15, !16}
+!16 = distinct !{!16, i1 false, !"domain"}
+!17 = !{!"address"}
+!18 = !{!"read_provenance"}
+
+;.
+; CHECK: [[INT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; CHECK: [[META1]] = !{!"int", [[META2:![0-9]+]]}
+; CHECK: [[META2]] = !{!"char", [[META3:![0-9]+]]}
+; CHECK: [[META3]] = !{!"root"}
+; CHECK: [[META4]] = !{[[META5:![0-9]+]]}
+; CHECK: [[META5]] = distinct !{[[META5]], [[META6:![0-9]+]]}
+; CHECK: [[META6]] = distinct !{[[META6]], i1 false, !"domain"}
+; CHECK: [[ACC_GRP7]] = !{[[META8:![0-9]+]], [[ACC_GRP9]]}
+; CHECK: [[META8]] = distinct !{}
+; CHECK: [[ACC_GRP9]] = distinct !{}
+;.
diff --git a/llvm/test/Transforms/SROA/memory-metadata.ll b/llvm/test/Transforms/SROA/memory-metadata.ll
new file mode 100644
index 00000000000000..a1d2f5687a4597
--- /dev/null
+++ b/llvm/test/Transforms/SROA/memory-metadata.ll
@@ -0,0 +1,215 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=sroa < %s | FileCheck %s --implicit-check-not=!test.unknown --implicit-check-not=!prof
+
+target datalayout = "e-p:64:64-n8:16:32:64"
+
+define i32 @split_copy(ptr %src) {
+; CHECK-LABEL: define i32 @split_copy(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT: [[A_SROA_0:%.*]] = alloca i32, align 8
+; CHECK-NEXT: [[A_SROA_2:%.*]] = alloca i32, align 4
+; CHECK-NEXT: [[A_SROA_0_0_COPYLOAD:%.*]] = load i32, ptr [[SRC]], align 8, !tbaa [[INT_TBAA0:![0-9]+]], !alias.scope [[META4:![0-9]+]], !noalias [[META4]], !llvm.mem.parallel_loop_access [[META7:![0-9]+]], !llvm.access.group [[META7]], !noalias.addrspace [[META8:![0-9]+]]
+; CHECK-NEXT: store i32 [[A_SROA_0_0_COPYLOAD]], ptr [[A_SROA_0]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META4]], !noalias [[META4]], !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]], !noalias.addrspace [[META8]]
+; CHECK-NEXT: [[A_SROA_2_0_SRC_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 4
+; CHECK-NEXT: [[A_SROA_2_0_COPYLOAD:%.*]] = load i32, ptr [[A_SROA_2_0_SRC_SROA_IDX]], align 4, !tbaa [[FLOAT_TBAA9:![0-9]+]], !alias.scope [[META4]], !noalias [[META4]], !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]], !noalias.addrspace [[META8]]
+; CHECK-NEXT: store i32 [[A_SROA_2_0_COPYLOAD]], ptr [[A_SROA_2]], align 4, !tbaa [[FLOAT_TBAA9]], !alias.scope [[META4]], !noalias [[META4]], !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]], !noalias.addrspace [[META8]]
+; CHECK-NEXT: [[A_SROA_0_0_A_SROA_0_0_X:%.*]] = load volatile i32, ptr [[A_SROA_0]], align 8
+; CHECK-NEXT: [[A_SROA_2_0_A_SROA_2_4_Y:%.*]] = load volatile i32, ptr [[A_SROA_2]], align 4
+; CHECK-NEXT: [[R:%.*]] = add i32 [[A_SROA_0_0_A_SROA_0_0_X]], [[A_SROA_2_0_A_SROA_2_4_Y]]
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %a = alloca {i32, i32}, align 8
+ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %a, ptr align 8 %src, i64 8, i1 false), !tbaa.struct !19, !prof !24, !mem.cache_hint !22, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+ %p1 = getelementptr {i32, i32}, ptr %a, i32 0, i32 1
+ %x = load volatile i32, ptr %a, align 8
+ %y = load volatile i32, ptr %p1, align 4
+ %r = add i32 %x, %y
+ ret i32 %r
+}
+
+define i32 @rewrite_set(i32 %v) {
+; CHECK-LABEL: define i32 @rewrite_set(
+; CHECK-SAME: i32 [[V:%.*]]) {
+; CHECK-NEXT: [[A_SROA_0:%.*]] = alloca i32, align 8
+; CHECK-NEXT: [[A_SROA_2:%.*]] = alloca i32, align 4
+; CHECK-NEXT: store i32 [[V]], ptr [[A_SROA_0]], align 8
+; CHECK-NEXT: store i32 0, ptr [[A_SROA_2]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META4]], !noalias [[META4]], !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]], !noalias.addrspace [[META8]]
+; CHECK-NEXT: [[A_SROA_0_0_A_SROA_0_0_X:%.*]] = load volatile i32, ptr [[A_SROA_0]], align 8
+; CHECK-NEXT: [[A_SROA_2_0_A_SROA_2_4_Y:%.*]] = load volatile i32, ptr [[A_SROA_2]], align 4
+; CHECK-NEXT: [[R:%.*]] = add i32 [[A_SROA_0_0_A_SROA_0_0_X]], [[A_SROA_2_0_A_SROA_2_4_Y]]
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %a = alloca {i32, i32}, align 8
+ %p1 = getelementptr {i32, i32}, ptr %a, i32 0, i32 1
+ store i32 %v, ptr %a, align 8
+ call void @llvm.memset.p0.i64(ptr align 4 %p1, i8 0, i64 4, i1 false), !tbaa !10, !prof !24, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+ %x = load volatile i32, ptr %a, align 8
+ %y = load volatile i32, ptr %p1, align 4
+ %r = add i32 %x, %y
+ ret i32 %r
+}
+
+define i32 @split_aggregate_store({i32, i32} %v) {
+; CHECK-LABEL: define i32 @split_aggregate_store(
+; CHECK-SAME: { i32, i32 } [[V:%.*]]) {
+; CHECK-NEXT: [[A_SROA_0:%.*]] = alloca i32, align 8
+; CHECK-NEXT: [[A_SROA_2:%.*]] = alloca i32, align 4
+; CHECK-NEXT: [[V_FCA_0_EXTRACT:%.*]] = extractvalue { i32, i32 } [[V]], 0
+; CHECK-NEXT: store i32 [[V_FCA_0_EXTRACT]], ptr [[A_SROA_0]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META4]], !noalias [[META4]], !noalias.addrspace [[META8]]
+; CHECK-NEXT: [[V_FCA_1_EXTRACT:%.*]] = extractvalue { i32, i32 } [[V]], 1
+; CHECK-NEXT: store i32 [[V_FCA_1_EXTRACT]], ptr [[A_SROA_2]], align 4, !tbaa [[FLOAT_TBAA9]], !alias.scope [[META4]], !noalias [[META4]], !noalias.addrspace [[META8]]
+; CHECK-NEXT: [[A_SROA_0_0_A_SROA_0_0_X:%.*]] = load volatile i32, ptr [[A_SROA_0]], align 8
+; CHECK-NEXT: [[A_SROA_2_0_A_SROA_2_4_Y:%.*]] = load volatile i32, ptr [[A_SROA_2]], align 4
+; CHECK-NEXT: [[R:%.*]] = add i32 [[A_SROA_0_0_A_SROA_0_0_X]], [[A_SROA_2_0_A_SROA_2_4_Y]]
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %a = alloca {i32, i32}, align 8
+ store {i32, i32} %v, ptr %a, align 8, !tbaa.struct !19, !invariant.group !0, !mem.cache_hint !18, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+ %p1 = getelementptr {i32, i32}, ptr %a, i32 0, i32 1
+ %x = load volatile i32, ptr %a, align 8
+ %y = load volatile i32, ptr %p1, align 4
+ %r = add i32 %x, %y
+ ret i32 %r
+}
+
+define {i32, i32} @split_aggregate_load(ptr %src, ptr %dst) {
+; CHECK-LABEL: define { i32, i32 } @split_aggregate_load(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]]) {
+; CHECK-NEXT: [[A_SROA_0:%.*]] = alloca i32, align 8
+; CHECK-NEXT: [[A_SROA_3:%.*]] = alloca i32, align 4
+; CHECK-NEXT: [[A_SROA_0_0_COPYLOAD:%.*]] = load i32, ptr [[SRC]], align 1
+; CHECK-NEXT: store i32 [[A_SROA_0_0_COPYLOAD]], ptr [[A_SROA_0]], align 8
+; CHECK-NEXT: [[A_SROA_3_0_SRC_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 4
+; CHECK-NEXT: [[A_SROA_3_0_COPYLOAD:%.*]] = load i32, ptr [[A_SROA_3_0_SRC_SROA_IDX]], align 1
+; CHECK-NEXT: store i32 [[A_SROA_3_0_COPYLOAD]], ptr [[A_SROA_3]], align 4
+; CHECK-NEXT: [[A_SROA_0_0_A_SROA_0_0_V_FCA_0_LOAD:%.*]] = load i32, ptr [[A_SROA_0]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META4]], !noalias [[META4]], !noalias.addrspace [[META8]]
+; CHECK-NEXT: [[V_FCA_0_INSERT:%.*]] = insertvalue { i32, i32 } poison, i32 [[A_SROA_0_0_A_SROA_0_0_V_FCA_0_LOAD]], 0
+; CHECK-NEXT: [[A_SROA_3_0_A_SROA_3_4_V_FCA_1_LOAD:%.*]] = load i32, ptr [[A_SROA_3]], align 4, !tbaa [[FLOAT_TBAA9]], !alias.scope [[META4]], !noalias [[META4]], !noalias.addrspace [[META8]]
+; CHECK-NEXT: [[V_FCA_1_INSERT:%.*]] = insertvalue { i32, i32 } [[V_FCA_0_INSERT]], i32 [[A_SROA_3_0_A_SROA_3_4_V_FCA_1_LOAD]], 1
+; CHECK-NEXT: [[A_SROA_0_0_A_SROA_0_0_COPYLOAD1:%.*]] = load volatile i32, ptr [[A_SROA_0]], align 8
+; CHECK-NEXT: store volatile i32 [[A_SROA_0_0_A_SROA_0_0_COPYLOAD1]], ptr [[DST]], align 1
+; CHECK-NEXT: [[A_SROA_3_0_DST_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 4
+; CHECK-NEXT: [[A_SROA_3_0_A_SROA_3_0_COPYLOAD2:%.*]] = load volatile i32, ptr [[A_SROA_3]], align 4
+; CHECK-NEXT: store volatile i32 [[A_SROA_3_0_A_SROA_3_0_COPYLOAD2]], ptr [[A_SROA_3_0_DST_SROA_IDX]], align 1
+; CHECK-NEXT: ret { i32, i32 } [[V_FCA_1_INSERT]]
+;
+ %a = alloca {i32, i32}, align 8
+ call void @llvm.memcpy.p0.p0.i64(ptr %a, ptr %src, i64 8, i1 false)
+ %v = load {i32, i32}, ptr %a, align 8, !tbaa.struct !19, !invariant.load !0, !invariant.group !0, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+ call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %a, i64 8, i1 true)
+ ret {i32, i32} %v
+}
+
+define double @rewrite_load(ptr %src, ptr %dst) {
+; CHECK-LABEL: define double @rewrite_load(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]]) {
+; CHECK-NEXT: [[A_SROA_0:%.*]] = alloca double, align 8
+; CHECK-NEXT: [[A_SROA_3:%.*]] = alloca i32, align 8
+; CHECK-NEXT: [[A_SROA_0_0_COPYLOAD:%.*]] = load double, ptr [[SRC]], align 1
+; CHECK-NEXT: store double [[A_SROA_0_0_COPYLOAD]], ptr [[A_SROA_0]], align 8
+; CHECK-NEXT: [[A_SROA_3_0_SRC_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 8
+; CHECK-NEXT: [[A_SROA_3_0_COPYLOAD:%.*]] = load i32, ptr [[A_SROA_3_0_SRC_SROA_IDX]], align 1
+; CHECK-NEXT: store i32 [[A_SROA_3_0_COPYLOAD]], ptr [[A_SROA_3]], align 8
+; CHECK-NEXT: [[A_SROA_0_0_A_SROA_0_0_V:%.*]] = load double, ptr [[A_SROA_0]], align 8, !fpmath [[META11:![0-9]+]], !invariant.load [[META12:![0-9]+]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META13:![0-9]+]], !llvm.mem.parallel_loop_access [[META7]], !invariant.group [[META12]], !llvm.access.group [[META7]], !noundef [[META12]], !noalias.addrspace [[META8]], !nofpclass [[META14:![0-9]+]], !mem.cache_hint [[META15:![0-9]+]]
+; CHECK-NEXT: [[A_SROA_0_0_A_SROA_0_0_COPYLOAD1:%.*]] = load volatile double, ptr [[A_SROA_0]], align 8
+; CHECK-NEXT: store volatile double [[A_SROA_0_0_A_SROA_0_0_COPYLOAD1]], ptr [[DST]], align 1
+; CHECK-NEXT: [[A_SROA_3_0_DST_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 8
+; CHECK-NEXT: [[A_SROA_3_0_A_SROA_3_0_COPYLOAD2:%.*]] = load volatile i32, ptr [[A_SROA_3]], align 8
+; CHECK-NEXT: store volatile i32 [[A_SROA_3_0_A_SROA_3_0_COPYLOAD2]], ptr [[A_SROA_3_0_DST_SROA_IDX]], align 1
+; CHECK-NEXT: ret double [[A_SROA_0_0_A_SROA_0_0_V]]
+;
+ %a = alloca {double, i32}, align 8
+ call void @llvm.memcpy.p0.p0.i64(ptr %a, ptr %src, i64 12, i1 false)
+ %v = load double, ptr %a, align 8, !noundef !0, !nofpclass !15, !fpmath !14, !invariant.load !0, !invariant.group !0, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+ call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %a, i64 12, i1 true)
+ ret double %v
+}
+
+define ptr @rewrite_pointer_load(ptr %v) {
+; CHECK-LABEL: define ptr @rewrite_pointer_load(
+; CHECK-SAME: ptr [[V:%.*]]) {
+; CHECK-NEXT: [[A_SROA_1:%.*]] = alloca ptr, align 8
+; CHECK-NEXT: store ptr [[V]], ptr [[A_SROA_1]], align 8
+; CHECK-NEXT: [[A_SROA_1_0_A_SROA_1_8_R:%.*]] = load volatile ptr, ptr [[A_SROA_1]], align 8, !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META13]], !llvm.mem.parallel_loop_access [[META7]], !nonnull [[META12]], !dereferenceable [[META17:![0-9]+]], !dereferenceable_or_null [[META17]], !invariant.group [[META12]], !align [[META17]], !llvm.access.group [[META7]], !noundef [[META12]], !noalias.addrspace [[META8]], !mem.cache_hint [[META15]]
+; CHECK-NEXT: ret ptr [[A_SROA_1_0_A_SROA_1_8_R]]
+;
+ %a = alloca [2 x ptr], align 8
+ %p1 = getelementptr [2 x ptr], ptr %a, i32 0, i32 1
+ store ptr null, ptr %a, align 8
+ store ptr %v, ptr %p1, align 8
+ %r = load volatile ptr, ptr %p1, align 8, !nonnull !0, !noundef !0, !align !25, !dereferenceable !25, !dereferenceable_or_null !25, !invariant.group !0, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+ ret ptr %r
+}
+
+define i32 @presplit_load(ptr %src) {
+; CHECK-LABEL: define i32 @presplit_load(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT: [[A_SROA_0:%.*]] = alloca i32, align 8
+; CHECK-NEXT: [[A_SROA_2:%.*]] = alloca i32, align 4
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[SRC]], align 8, !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]]
+; CHECK-NEXT: [[SRC_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 4
+; CHECK-NEXT: [[V2:%.*]] = load i32, ptr [[SRC_SROA_IDX]], align 4, !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]]
+; CHECK-NEXT: store i32 [[V1]], ptr [[A_SROA_0]], align 8, !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]]
+; CHECK-NEXT: store i32 [[V2]], ptr [[A_SROA_2]], align 4, !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]]
+; CHECK-NEXT: [[A_SROA_0_0_A_SROA_0_0_X:%.*]] = load volatile i32, ptr [[A_SROA_0]], align 8
+; CHECK-NEXT: [[A_SROA_2_0_A_SROA_2_4_Y:%.*]] = load volatile i32, ptr [[A_SROA_2]], align 4
+; CHECK-NEXT: [[R:%.*]] = add i32 [[A_SROA_0_0_A_SROA_0_0_X]], [[A_SROA_2_0_A_SROA_2_4_Y]]
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %a = alloca {i32, i32}, align 8
+ %v = load i64, ptr %src, align 8, !range !26, !noundef !0, !invariant.load !0, !invariant.group !0, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+ store i64 %v, ptr %a, align 8, !mem.cache_hint !18, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+ %p1 = getelementptr {i32, i32}, ptr %a, i32 0, i32 1
+ %x = load volatile i32, ptr %a, align 8
+ %y = load volatile i32, ptr %p1, align 4
+ %r = add i32 %x, %y
+ ret i32 %r
+}
+
+!0 = !{}
+!1 = !{i32 1}
+!2 = !{!"preserved memory metadata"}
+!3 = !{!"test", !"relaxed"}
+!4 = !{i32 5, i32 6}
+!5 = distinct !{}
+!6 = !{!7}
+!7 = distinct !{!7, !8, !"scope"}
+!8 = distinct !{!8, i1 false, !"domain"}
+!9 = !{i32 0, i32 100}
+!10 = !{!11, !11, i64 0}
+!11 = !{!"int", !12, i64 0}
+!12 = !{!"omnipotent char", !13, i64 0}
+!13 = !{!"root"}
+!14 = !{float 2.5}
+!15 = !{i32 3}
+!16 = !{i32 0, !17}
+!17 = !{!"nvvm.l1_eviction", !"last"}
+!18 = !{i32 1, !17}
+!19 = !{i64 0, i64 4, !10, i64 4, i64 4, !20}
+!20 = !{!21, !21, i64 0}
+!21 = !{!"float", !12, i64 0}
+!22 = !{i32 0, !17, i32 1, !23}
+!23 = !{!"nvvm.l1_eviction", !"first"}
+!24 = !{!"branch_weights", i32 10}
+!25 = !{i64 8}
+!26 = !{i64 0, i64 100}
+;.
+; CHECK: [[INT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; CHECK: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
+; CHECK: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
+; CHECK: [[META3]] = !{!"root"}
+; CHECK: [[META4]] = !{[[META5:![0-9]+]]}
+; CHECK: [[META5]] = distinct !{[[META5]], [[META6:![0-9]+]], !"scope"}
+; CHECK: [[META6]] = distinct !{[[META6]], i1 false, !"domain"}
+; CHECK: [[META7]] = distinct !{}
+; CHECK: [[META8]] = !{i32 5, i32 6}
+; CHECK: [[FLOAT_TBAA9]] = !{[[META10:![0-9]+]], [[META10]], i64 0}
+; CHECK: [[META10]] = !{!"float", [[META2]], i64 0}
+; CHECK: [[META11]] = !{float 2.500000e+00}
+; CHECK: [[META12]] = !{}
+; CHECK: [[META13]] = !{i32 1}
+; CHECK: [[META14]] = !{i32 3}
+; CHECK: [[META15]] = !{i32 0, [[META16:![0-9]+]]}
+; CHECK: [[META16]] = !{!"nvvm.l1_eviction", !"last"}
+; CHECK: [[META17]] = !{i64 8}
+;.
diff --git a/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll b/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
index 0a3516c82ad10b..b7db48d9b9cbc7 100644
--- a/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
+++ b/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
@@ -202,7 +202,7 @@ define void @f3(ptr %src, ptr %dst) {
ret void
}
-; Check that !tbaa.struct information is preserved.
+; Check that !tbaa.struct fields become the correct per-lane !tbaa tags.
define void @f4(ptr %src, ptr %dst) {
; CHECK-LABEL: define void @f4(
; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]]) {
@@ -858,7 +858,7 @@ define <2 x i32> @f23_crash(<2 x i32> %srcvec, i32 %v1) {
!2 = !{!15, !15, i64 0}
!3 = !{ !3, !{!"llvm.loop.parallel_accesses", !13} }
!4 = !{ float 4.0 }
-!5 = !{ i64 0, i64 8, null }
+!5 = !{i64 0, i64 4, !1, i64 4, i64 4, !2, i64 8, i64 4, !1, i64 12, i64 4, !2}
!13 = distinct !{}
!14 = !{!"set1", !0}
!15 = !{!"set2", !0}
@@ -868,7 +868,7 @@ define <2 x i32> @f23_crash(<2 x i32> %srcvec, i32 %v1) {
; CHECK: [[META2]] = !{!"root"}
; CHECK: [[SET2_TBAA3]] = !{[[META4:![0-9]+]], [[META4]], i64 0}
; CHECK: [[META4]] = !{!"set2", [[META2]]}
-; CHECK: [[TBAA_STRUCT5]] = !{i64 0, i64 8, null}
+; CHECK: [[TBAA_STRUCT5]] = !{i64 0, i64 4, [[SET1_TBAA0]], i64 4, i64 4, [[SET2_TBAA3]], i64 8, i64 4, [[SET1_TBAA0]], i64 12, i64 4, [[SET2_TBAA3]]}
; CHECK: [[ACC_GRP6]] = distinct !{}
; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META8:![0-9]+]]}
; CHECK: [[META8]] = !{!"llvm.loop.parallel_accesses", [[ACC_GRP6]]}
diff --git a/llvm/test/Transforms/Scalarizer/basic.ll b/llvm/test/Transforms/Scalarizer/basic.ll
index ffde5600bbc9a7..f620445612f6dd 100644
--- a/llvm/test/Transforms/Scalarizer/basic.ll
+++ b/llvm/test/Transforms/Scalarizer/basic.ll
@@ -202,7 +202,7 @@ define void @f3(ptr %src, ptr %dst) {
ret void
}
-; Check that !tbaa.struct information is preserved.
+; Check that !tbaa.struct fields become the correct per-lane !tbaa tags.
define void @f4(ptr %src, ptr %dst) {
; CHECK-LABEL: define void @f4(
; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]]) {
@@ -909,7 +909,7 @@ define <2 x i8> @test_copy_trunc_flags(<2 x i32> %src) {
!2 = !{!15, !15, i64 0}
!3 = !{ !3, !{!"llvm.loop.parallel_accesses", !13} }
!4 = !{ float 4.0 }
-!5 = !{ i64 0, i64 8, null }
+!5 = !{i64 0, i64 4, !1, i64 4, i64 4, !2, i64 8, i64 4, !1, i64 12, i64 4, !2}
!13 = distinct !{}
!14 = !{!"set1", !0}
!15 = !{!"set2", !0}
@@ -919,7 +919,7 @@ define <2 x i8> @test_copy_trunc_flags(<2 x i32> %src) {
; CHECK: [[META2]] = !{!"root"}
; CHECK: [[SET2_TBAA3]] = !{[[META4:![0-9]+]], [[META4]], i64 0}
; CHECK: [[META4]] = !{!"set2", [[META2]]}
-; CHECK: [[TBAA_STRUCT5]] = !{i64 0, i64 8, null}
+; CHECK: [[TBAA_STRUCT5]] = !{i64 0, i64 4, [[SET1_TBAA0]], i64 4, i64 4, [[SET2_TBAA3]], i64 8, i64 4, [[SET1_TBAA0]], i64 12, i64 4, [[SET2_TBAA3]]}
; CHECK: [[ACC_GRP6]] = distinct !{}
; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META8:![0-9]+]]}
; CHECK: [[META8]] = !{!"llvm.loop.parallel_accesses", [[ACC_GRP6]]}
diff --git a/llvm/test/Transforms/Scalarizer/memory-cache-hint.ll b/llvm/test/Transforms/Scalarizer/memory-cache-hint.ll
new file mode 100644
index 00000000000000..270b264e65961b
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/memory-cache-hint.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes='scalarizer<load-store>,dce' < %s | FileCheck %s
+
+define <2 x i32> @split_load(ptr %p) {
+; CHECK-LABEL: define <2 x i32> @split_load(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[V_I0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: [[P_I1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: [[V_I1:%.*]] = load i32, ptr [[P_I1]], align 4
+; CHECK-NEXT: [[V_UPTO0:%.*]] = insertelement <2 x i32> poison, i32 [[V_I0]], i64 0
+; CHECK-NEXT: [[V:%.*]] = insertelement <2 x i32> [[V_UPTO0]], i32 [[V_I1]], i64 1
+; CHECK-NEXT: ret <2 x i32> [[V]]
+;
+ %v = load <2 x i32>, ptr %p, align 8, !mem.cache_hint !0, !range !2
+ ret <2 x i32> %v
+}
+
+define void @split_store(ptr %p, <2 x i32> %v) {
+; CHECK-LABEL: define void @split_store(
+; CHECK-SAME: ptr [[P:%.*]], <2 x i32> [[V:%.*]]) {
+; CHECK-NEXT: [[V_I0:%.*]] = extractelement <2 x i32> [[V]], i64 0
+; CHECK-NEXT: store i32 [[V_I0]], ptr [[P]], align 8
+; CHECK-NEXT: [[V_I1:%.*]] = extractelement <2 x i32> [[V]], i64 1
+; CHECK-NEXT: [[P_I1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 [[V_I1]], ptr [[P_I1]], align 4
+; CHECK-NEXT: ret void
+;
+ store <2 x i32> %v, ptr %p, align 8, !mem.cache_hint !3
+ ret void
+}
+
+!0 = !{i32 0, !1}
+!1 = !{!"nvvm.l1_eviction", !"last"}
+!2 = !{i32 0, i32 100}
+!3 = !{i32 1, !1}
diff --git a/llvm/test/Transforms/Scalarizer/memory-metadata.ll b/llvm/test/Transforms/Scalarizer/memory-metadata.ll
new file mode 100644
index 00000000000000..cb6197983ec51c
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/memory-metadata.ll
@@ -0,0 +1,150 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes='scalarizer<load-store>,dce' < %s | FileCheck %s --implicit-check-not=!test.unknown
+; RUN: opt -S -passes='scalarizer<load-store;min-bits=64>,dce' < %s | FileCheck %s --check-prefix=PACKED --implicit-check-not=!test.unknown
+
+target datalayout = "e-p:64:64"
+
+; Exercise result facts and access annotations when splitting into scalars and
+; smaller vectors. Only the first fragment retains pointer identity.
+define <4 x i32> @integer_load(ptr %p) {
+; CHECK-LABEL: define <4 x i32> @integer_load(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[V_I0:%.*]] = load i32, ptr [[P]], align 16, !tbaa [[INT_TBAA0:![0-9]+]], !invariant.load [[META4:![0-9]+]], !alias.scope [[META5:![0-9]+]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8:![0-9]+]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[P_I1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: [[V_I1:%.*]] = load i32, ptr [[P_I1]], align 4, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[P_I2:%.*]] = getelementptr i32, ptr [[P]], i32 2
+; CHECK-NEXT: [[V_I2:%.*]] = load i32, ptr [[P_I2]], align 8, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[P_I3:%.*]] = getelementptr i32, ptr [[P]], i32 3
+; CHECK-NEXT: [[V_I3:%.*]] = load i32, ptr [[P_I3]], align 4, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_UPTO0:%.*]] = insertelement <4 x i32> poison, i32 [[V_I0]], i64 0
+; CHECK-NEXT: [[V_UPTO1:%.*]] = insertelement <4 x i32> [[V_UPTO0]], i32 [[V_I1]], i64 1
+; CHECK-NEXT: [[V_UPTO2:%.*]] = insertelement <4 x i32> [[V_UPTO1]], i32 [[V_I2]], i64 2
+; CHECK-NEXT: [[V:%.*]] = insertelement <4 x i32> [[V_UPTO2]], i32 [[V_I3]], i64 3
+; CHECK-NEXT: ret <4 x i32> [[V]]
+;
+; PACKED-LABEL: define <4 x i32> @integer_load(
+; PACKED-SAME: ptr [[P:%.*]]) {
+; PACKED-NEXT: [[V_I0:%.*]] = load <2 x i32>, ptr [[P]], align 16, !tbaa [[INT_TBAA0:![0-9]+]], !invariant.load [[META4:![0-9]+]], !alias.scope [[META5:![0-9]+]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8:![0-9]+]], !llvm.access.group [[META8]]
+; PACKED-NEXT: [[P_I1:%.*]] = getelementptr <2 x i32>, ptr [[P]], i32 1
+; PACKED-NEXT: [[V_I1:%.*]] = load <2 x i32>, ptr [[P_I1]], align 8, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[V_I0]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; PACKED-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[V_I1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; PACKED-NEXT: [[V:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; PACKED-NEXT: ret <4 x i32> [[V]]
+;
+ %v = load <4 x i32>, ptr %p, align 16, !tbaa !10, !range !9, !noundef !0, !invariant.load !0, !invariant.group !0, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+ ret <4 x i32> %v
+}
+
+define <4 x float> @floating_load(ptr %p) {
+; CHECK-LABEL: define <4 x float> @floating_load(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[V_I0:%.*]] = load float, ptr [[P]], align 16, !tbaa [[FLOAT_TBAA9:![0-9]+]], !fpmath [[META11:![0-9]+]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[P_I1:%.*]] = getelementptr float, ptr [[P]], i32 1
+; CHECK-NEXT: [[V_I1:%.*]] = load float, ptr [[P_I1]], align 4, !tbaa [[FLOAT_TBAA9]], !fpmath [[META11]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[P_I2:%.*]] = getelementptr float, ptr [[P]], i32 2
+; CHECK-NEXT: [[V_I2:%.*]] = load float, ptr [[P_I2]], align 8, !tbaa [[FLOAT_TBAA9]], !fpmath [[META11]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[P_I3:%.*]] = getelementptr float, ptr [[P]], i32 3
+; CHECK-NEXT: [[V_I3:%.*]] = load float, ptr [[P_I3]], align 4, !tbaa [[FLOAT_TBAA9]], !fpmath [[META11]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_UPTO0:%.*]] = insertelement <4 x float> poison, float [[V_I0]], i64 0
+; CHECK-NEXT: [[V_UPTO1:%.*]] = insertelement <4 x float> [[V_UPTO0]], float [[V_I1]], i64 1
+; CHECK-NEXT: [[V_UPTO2:%.*]] = insertelement <4 x float> [[V_UPTO1]], float [[V_I2]], i64 2
+; CHECK-NEXT: [[V:%.*]] = insertelement <4 x float> [[V_UPTO2]], float [[V_I3]], i64 3
+; CHECK-NEXT: ret <4 x float> [[V]]
+;
+; PACKED-LABEL: define <4 x float> @floating_load(
+; PACKED-SAME: ptr [[P:%.*]]) {
+; PACKED-NEXT: [[V_I0:%.*]] = load <2 x float>, ptr [[P]], align 16, !tbaa [[FLOAT_TBAA9:![0-9]+]], !fpmath [[META11:![0-9]+]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT: [[P_I1:%.*]] = getelementptr <2 x float>, ptr [[P]], i32 1
+; PACKED-NEXT: [[V_I1:%.*]] = load <2 x float>, ptr [[P_I1]], align 8, !tbaa [[FLOAT_TBAA9]], !fpmath [[META11]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT: [[TMP1:%.*]] = shufflevector <2 x float> [[V_I0]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; PACKED-NEXT: [[TMP2:%.*]] = shufflevector <2 x float> [[V_I1]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; PACKED-NEXT: [[V:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; PACKED-NEXT: ret <4 x float> [[V]]
+;
+ %v = load <4 x float>, ptr %p, align 16, !tbaa !20, !nofpclass !15, !fpmath !14, !noundef !0, !invariant.load !0, !invariant.group !0, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+ ret <4 x float> %v
+}
+
+define void @integer_store(ptr %p, <4 x i32> %v) {
+; CHECK-LABEL: define void @integer_store(
+; CHECK-SAME: ptr [[P:%.*]], <4 x i32> [[V:%.*]]) {
+; CHECK-NEXT: [[V_I0:%.*]] = extractelement <4 x i32> [[V]], i64 0
+; CHECK-NEXT: store i32 [[V_I0]], ptr [[P]], align 16, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_I1:%.*]] = extractelement <4 x i32> [[V]], i64 1
+; CHECK-NEXT: [[P_I1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 [[V_I1]], ptr [[P_I1]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_I2:%.*]] = extractelement <4 x i32> [[V]], i64 2
+; CHECK-NEXT: [[P_I2:%.*]] = getelementptr i32, ptr [[P]], i32 2
+; CHECK-NEXT: store i32 [[V_I2]], ptr [[P_I2]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_I3:%.*]] = extractelement <4 x i32> [[V]], i64 3
+; CHECK-NEXT: [[P_I3:%.*]] = getelementptr i32, ptr [[P]], i32 3
+; CHECK-NEXT: store i32 [[V_I3]], ptr [[P_I3]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: ret void
+;
+; PACKED-LABEL: define void @integer_store(
+; PACKED-SAME: ptr [[P:%.*]], <4 x i32> [[V:%.*]]) {
+; PACKED-NEXT: [[V_I0:%.*]] = shufflevector <4 x i32> [[V]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>
+; PACKED-NEXT: store <2 x i32> [[V_I0]], ptr [[P]], align 16, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT: [[V_I1:%.*]] = shufflevector <4 x i32> [[V]], <4 x i32> poison, <2 x i32> <i32 2, i32 3>
+; PACKED-NEXT: [[P_I1:%.*]] = getelementptr <2 x i32>, ptr [[P]], i32 1
+; PACKED-NEXT: store <2 x i32> [[V_I1]], ptr [[P_I1]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT: ret void
+;
+ store <4 x i32> %v, ptr %p, align 16, !tbaa !10, !invariant.group !0, !mem.cache_hint !18, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+ ret void
+}
+
+!0 = !{}
+!1 = !{i32 1}
+!2 = !{!"preserved memory metadata"}
+!3 = !{!"test", !"relaxed"}
+!4 = !{i32 5, i32 6}
+!5 = distinct !{}
+!6 = !{!7}
+!7 = distinct !{!7, !8, !"scope"}
+!8 = distinct !{!8, i1 false, !"domain"}
+!9 = !{i32 0, i32 100}
+!10 = !{!11, !11, i64 0}
+!11 = !{!"int", !12, i64 0}
+!12 = !{!"omnipotent char", !13, i64 0}
+!13 = !{!"root"}
+!14 = !{float 2.5}
+!15 = !{i32 3}
+!16 = !{i32 0, !17}
+!17 = !{!"nvvm.l1_eviction", !"last"}
+!18 = !{i32 1, !17}
+!19 = !{i64 0, i64 4, !10, i64 4, i64 4, !20}
+!20 = !{!21, !21, i64 0}
+!21 = !{!"float", !12, i64 0}
+!22 = !{i32 0, !17, i32 1, !23}
+!23 = !{!"nvvm.l1_eviction", !"first"}
+!24 = !{!"branch_weights", i32 10}
+!25 = !{i64 8}
+;.
+; CHECK: [[INT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; CHECK: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
+; CHECK: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
+; CHECK: [[META3]] = !{!"root"}
+; CHECK: [[META4]] = !{}
+; CHECK: [[META5]] = !{[[META6:![0-9]+]]}
+; CHECK: [[META6]] = distinct !{[[META6]], [[META7:![0-9]+]], !"scope"}
+; CHECK: [[META7]] = distinct !{[[META7]], i1 false, !"domain"}
+; CHECK: [[META8]] = distinct !{}
+; CHECK: [[FLOAT_TBAA9]] = !{[[META10:![0-9]+]], [[META10]], i64 0}
+; CHECK: [[META10]] = !{!"float", [[META2]], i64 0}
+; CHECK: [[META11]] = !{float 2.500000e+00}
+;.
+; PACKED: [[INT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; PACKED: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
+; PACKED: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
+; PACKED: [[META3]] = !{!"root"}
+; PACKED: [[META4]] = !{}
+; PACKED: [[META5]] = !{[[META6:![0-9]+]]}
+; PACKED: [[META6]] = distinct !{[[META6]], [[META7:![0-9]+]], !"scope"}
+; PACKED: [[META7]] = distinct !{[[META7]], i1 false, !"domain"}
+; PACKED: [[META8]] = distinct !{}
+; PACKED: [[FLOAT_TBAA9]] = !{[[META10:![0-9]+]], [[META10]], i64 0}
+; PACKED: [[META10]] = !{!"float", [[META2]], i64 0}
+; PACKED: [[META11]] = !{float 2.500000e+00}
+;.
diff --git a/llvm/test/Transforms/VectorCombine/RISCV/memory-metadata.ll b/llvm/test/Transforms/VectorCombine/RISCV/memory-metadata.ll
new file mode 100644
index 00000000000000..4c29c342143cd8
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/RISCV/memory-metadata.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=vector-combine -mtriple=riscv64 -mattr=+v < %s | FileCheck %s
+
+define <8 x i16> @vp_bitcast_fixed(ptr %p, i32 %evl) {
+; CHECK-LABEL: define <8 x i16> @vp_bitcast_fixed(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i32 [[EVL]], 2
+; CHECK-NEXT: [[R:%.*]] = call <8 x i16> @llvm.vp.load.v8i16.p0(ptr align 4 [[P]], <8 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: ret <8 x i16> [[R]]
+;
+ %v = call <4 x i32> @llvm.vp.load(ptr align 4 %p, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, i32 %evl), !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !range !11, !invariant.load !3
+ %r = bitcast <4 x i32> %v to <8 x i16>
+ ret <8 x i16> %r
+}
+
+define <vscale x 8 x i16> @vp_bitcast_scalable(ptr %p, i32 %evl) {
+; CHECK-LABEL: define <vscale x 8 x i16> @vp_bitcast_scalable(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i32 [[EVL]], 2
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 4 [[P]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
+;
+ %v = call <vscale x 4 x i32> @llvm.vp.load(ptr align 4 %p, <vscale x 4 x i1> splat (i1 true), i32 %evl), !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !range !11, !invariant.load !3
+ %r = bitcast <vscale x 4 x i32> %v to <vscale x 8 x i16>
+ ret <vscale x 8 x i16> %r
+}
+
+
+!0 = !{i32 1}
+!1 = !{i32 0, !2}
+!2 = !{!"nvvm.l1_eviction", !"last"}
+!3 = !{}
+!4 = !{!"original access"}
+!5 = !{!"test", !"tag"}
+!6 = !{i32 5, i32 6}
+!7 = !{!8}
+!8 = distinct !{!8, !9, !"scope"}
+!9 = distinct !{!9, i1 false, !"domain"}
+!10 = distinct !{}
+!11 = !{i32 0, i32 100}
+!12 = !{i32 3}
+!13 = !{float 2.5}
+!14 = !{i64 0, i64 4, !15, i64 4, i64 4, !15, i64 8, i64 4, !15, i64 12, i64 4, !15}
+!15 = !{!16, !16, i64 0}
+!16 = !{!"omnipotent char", !17, i64 0}
+!17 = !{!"root"}
+!18 = !{i32 1, !2}
diff --git a/llvm/test/Transforms/VectorCombine/X86/memory-metadata.ll b/llvm/test/Transforms/VectorCombine/X86/memory-metadata.ll
new file mode 100644
index 00000000000000..e9f73bbdff4e2a
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/X86/memory-metadata.ll
@@ -0,0 +1,72 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=vector-combine -mtriple=x86_64-unknown-linux-gnu -mcpu=x86-64 < %s | FileCheck %s
+
+define <4 x float> @widen_scalar(ptr align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: define <4 x float> @widen_scalar(
+; CHECK-SAME: ptr align 16 dereferenceable(16) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[R:%.*]] = load <4 x float>, ptr [[P]], align 16
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %v = load float, ptr %p, align 4, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !nofpclass !12, !fpmath !13, !noundef !3, !invariant.load !3, !invariant.group !3, !tbaa !15
+ %r = insertelement <4 x float> poison, float %v, i32 0
+ ret <4 x float> %r
+}
+
+define <4 x float> @widen_shifted_scalar(ptr align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: define <4 x float> @widen_shifted_scalar(
+; CHECK-SAME: ptr align 16 dereferenceable(16) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[P]], align 16
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 3, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %q = getelementptr inbounds float, ptr %p, i64 3
+ %v = load float, ptr %q, align 4, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !nofpclass !12, !fpmath !13, !noundef !3, !invariant.load !3, !invariant.group !3, !tbaa !15
+ %r = insertelement <4 x float> poison, float %v, i32 0
+ ret <4 x float> %r
+}
+
+define <4 x i32> @widen_subvector(ptr align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: define <4 x i32> @widen_subvector(
+; CHECK-SAME: ptr align 16 dereferenceable(16) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[R:%.*]] = load <4 x i32>, ptr [[P]], align 16
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %v = load <2 x i32>, ptr %p, align 4, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !range !11, !noundef !3, !invariant.load !3, !invariant.group !3, !tbaa !15
+ %r = shufflevector <2 x i32> %v, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ ret <4 x i32> %r
+}
+
+
+; The load-insert fold can also shrink a load. Facts about all original
+; elements remain valid for this subset.
+define <4 x i32> @shrink_load_insert(ptr align 16 dereferenceable(32) %p) {
+; CHECK-LABEL: define <4 x i32> @shrink_load_insert(
+; CHECK-SAME: ptr align 16 dereferenceable(32) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[R:%.*]] = load <4 x i32>, ptr [[P]], align 16
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %v = load <8 x i32>, ptr %p, align 16, !nontemporal !0, !mem.cache_hint !1, !range !11, !noundef !3, !invariant.load !3, !alias.scope !7
+ %e = extractelement <8 x i32> %v, i32 0
+ %r = insertelement <4 x i32> poison, i32 %e, i32 0
+ ret <4 x i32> %r
+}
+
+!0 = !{i32 1}
+!1 = !{i32 0, !2}
+!2 = !{!"nvvm.l1_eviction", !"last"}
+!3 = !{}
+!4 = !{!"original access"}
+!5 = !{!"test", !"tag"}
+!6 = !{i32 5, i32 6}
+!7 = !{!8}
+!8 = distinct !{!8, !9, !"scope"}
+!9 = distinct !{!9, i1 false, !"domain"}
+!10 = distinct !{}
+!11 = !{i32 0, i32 100}
+!12 = !{i32 3}
+!13 = !{float 2.5}
+!14 = !{i64 0, i64 4, !15, i64 4, i64 4, !15, i64 8, i64 4, !15, i64 12, i64 4, !15}
+!15 = !{!16, !16, i64 0}
+!16 = !{!"omnipotent char", !17, i64 0}
+!17 = !{!"root"}
+!18 = !{i32 1, !2}
diff --git a/llvm/test/Transforms/VectorCombine/memory-metadata.ll b/llvm/test/Transforms/VectorCombine/memory-metadata.ll
new file mode 100644
index 00000000000000..d01dffceeb940a
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/memory-metadata.ll
@@ -0,0 +1,195 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=vector-combine -data-layout=e-p:64:64 < %s | FileCheck %s
+; RUN: opt -S -passes=vector-combine -data-layout=E-p:64:64 < %s | FileCheck %s
+
+define i32 @extract_zero(ptr %p) {
+; CHECK-LABEL: define i32 @extract_zero(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[P]], align 16, !tbaa [[CHAR_TBAA0:![0-9]+]], !alias.scope [[META3:![0-9]+]], !noalias [[META3]], !noalias.addrspace [[META6:![0-9]+]]
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %v = load <4 x i32>, ptr %p, align 16, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !range !11, !noundef !3, !invariant.load !3, !invariant.group !3, !tbaa.struct !14
+ %r = extractelement <4 x i32> %v, i32 0
+ ret i32 %r
+}
+
+define i32 @extract_nonzero(ptr %p) {
+; CHECK-LABEL: define i32 @extract_nonzero(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 2
+; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP1]], align 8, !tbaa [[CHAR_TBAA0]], !alias.scope [[META3]], !noalias [[META3]], !noalias.addrspace [[META6]]
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %v = load <4 x i32>, ptr %p, align 16, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !range !11, !noundef !3, !invariant.load !3, !invariant.group !3, !tbaa.struct !14
+ %r = extractelement <4 x i32> %v, i32 2
+ ret i32 %r
+}
+
+define i32 @extract_dynamic(ptr %p, i32 %index) {
+; CHECK-LABEL: define i32 @extract_dynamic(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[INDEX:%.*]]) {
+; CHECK-NEXT: [[INDEX_FROZEN:%.*]] = freeze i32 [[INDEX]]
+; CHECK-NEXT: [[IDX:%.*]] = and i32 [[INDEX_FROZEN]], 3
+; CHECK-NEXT: [[IDX_GEPIDX:%.*]] = zext i32 [[IDX]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 [[IDX_GEPIDX]]
+; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %idx = and i32 %index, 3
+ %v = load <4 x i32>, ptr %p, align 16, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !range !11, !noundef !3, !invariant.load !3, !invariant.group !3, !tbaa.struct !14, !tbaa !15
+ %r = extractelement <4 x i32> %v, i32 %idx
+ ret i32 %r
+}
+
+define float @extract_float(ptr %p) {
+; CHECK-LABEL: define float @extract_float(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x float>, ptr [[P]], i64 0, i64 1
+; CHECK-NEXT: [[R:%.*]] = load float, ptr [[TMP1]], align 4, !alias.scope [[META3]], !noalias [[META3]], !noalias.addrspace [[META6]]
+; CHECK-NEXT: ret float [[R]]
+;
+ %v = load <4 x float>, ptr %p, align 16, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !nofpclass !12, !fpmath !13, !noundef !3, !invariant.load !3, !invariant.group !3
+ %r = extractelement <4 x float> %v, i32 1
+ ret float %r
+}
+
+define <4 x i32> @shrink_integer(ptr %p) {
+; CHECK-LABEL: define <4 x i32> @shrink_integer(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 32, !tbaa [[CHAR_TBAA0]], !range [[RNG7:![0-9]+]], !invariant.load [[META8:![0-9]+]], !alias.scope [[META3]], !noalias [[META3]], !nontemporal [[META9:![0-9]+]], !invariant.group [[META8]], !llvm.access.group [[ACC_GRP10:![0-9]+]], !noundef [[META8]], !annotation [[META11:![0-9]+]], !nosanitize [[META8]], !mmra [[META12:![0-9]+]], !noalias.addrspace [[META6]], !mem.cache_hint [[META13:![0-9]+]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %v = load <8 x i32>, ptr %p, align 32, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !range !11, !noundef !3, !invariant.load !3, !invariant.group !3, !tbaa !15
+ %r = shufflevector <8 x i32> %v, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ ret <4 x i32> %r
+}
+
+define <4 x float> @shrink_float(ptr %p) {
+; CHECK-LABEL: define <4 x float> @shrink_float(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[P]], align 32, !fpmath [[META15:![0-9]+]], !invariant.load [[META8]], !alias.scope [[META3]], !noalias [[META3]], !nontemporal [[META9]], !invariant.group [[META8]], !llvm.access.group [[ACC_GRP10]], !noundef [[META8]], !annotation [[META11]], !nosanitize [[META8]], !mmra [[META12]], !noalias.addrspace [[META6]], !nofpclass [[META16:![0-9]+]], !mem.cache_hint [[META13]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %v = load <8 x float>, ptr %p, align 32, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !nofpclass !12, !fpmath !13, !noundef !3, !invariant.load !3, !invariant.group !3
+ %r = shufflevector <8 x float> %v, <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ ret <4 x float> %r
+}
+
+define void @store_zero(ptr %p, i8 %x, i32 %index) {
+; CHECK-LABEL: define void @store_zero(
+; CHECK-SAME: ptr [[P:%.*]], i8 [[X:%.*]], i32 [[INDEX:%.*]]) {
+; CHECK-NEXT: store i8 [[X]], ptr [[P]], align 16, !tbaa.struct [[TBAA_STRUCT17:![0-9]+]], !alias.scope [[META3]], !noalias [[META3]], !nontemporal [[META9]], !llvm.access.group [[ACC_GRP10]], !annotation [[META11]], !nosanitize [[META8]], !mmra [[META12]], !noalias.addrspace [[META6]], !mem.cache_hint [[META18:![0-9]+]]
+; CHECK-NEXT: ret void
+;
+ %v = load <16 x i8>, ptr %p, align 16
+ %i = insertelement <16 x i8> %v, i8 %x, i32 0
+ store <16 x i8> %i, ptr %p, align 16, !nontemporal !0, !mem.cache_hint !18, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !invariant.group !3, !tbaa.struct !14
+ ret void
+}
+
+define void @store_nonzero(ptr %p, i8 %x, i32 %index) {
+; CHECK-LABEL: define void @store_nonzero(
+; CHECK-SAME: ptr [[P:%.*]], i8 [[X:%.*]], i32 [[INDEX:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 3
+; CHECK-NEXT: store i8 [[X]], ptr [[TMP1]], align 1, !tbaa.struct [[TBAA_STRUCT17]], !alias.scope [[META3]], !noalias [[META3]], !nontemporal [[META9]], !llvm.access.group [[ACC_GRP10]], !annotation [[META11]], !nosanitize [[META8]], !mmra [[META12]], !noalias.addrspace [[META6]], !mem.cache_hint [[META18]]
+; CHECK-NEXT: ret void
+;
+ %v = load <16 x i8>, ptr %p, align 16
+ %i = insertelement <16 x i8> %v, i8 %x, i32 3
+ store <16 x i8> %i, ptr %p, align 16, !nontemporal !0, !mem.cache_hint !18, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !invariant.group !3, !tbaa.struct !14
+ ret void
+}
+
+define void @store_dynamic(ptr %p, i8 %x, i32 %index) {
+; CHECK-LABEL: define void @store_dynamic(
+; CHECK-SAME: ptr [[P:%.*]], i8 [[X:%.*]], i32 [[INDEX:%.*]]) {
+; CHECK-NEXT: [[INDEX_FROZEN:%.*]] = freeze i32 [[INDEX]]
+; CHECK-NEXT: [[IDX:%.*]] = and i32 [[INDEX_FROZEN]], 15
+; CHECK-NEXT: [[IDX_GEPIDX:%.*]] = zext i32 [[IDX]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 [[IDX_GEPIDX]]
+; CHECK-NEXT: store i8 [[X]], ptr [[TMP1]], align 1, !tbaa.struct [[TBAA_STRUCT17]], !alias.scope [[META3]], !noalias [[META3]], !nontemporal [[META9]], !llvm.access.group [[ACC_GRP10]], !annotation [[META11]], !nosanitize [[META8]], !mmra [[META12]], !noalias.addrspace [[META6]], !mem.cache_hint [[META18]]
+; CHECK-NEXT: ret void
+;
+ %idx = and i32 %index, 15
+ %v = load <16 x i8>, ptr %p, align 16
+ %i = insertelement <16 x i8> %v, i8 %x, i32 %idx
+ store <16 x i8> %i, ptr %p, align 16, !nontemporal !0, !mem.cache_hint !18, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !invariant.group !3, !tbaa.struct !14
+ ret void
+}
+
+
+; A known offset selects the corresponding field's alias tag.
+define i32 @extract_struct_field(ptr %p) {
+; CHECK-LABEL: define i32 @extract_struct_field(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 1
+; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[INT_TBAA19:![0-9]+]]
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %v = load <4 x i32>, ptr %p, align 16, !tbaa.struct !21
+ %r = extractelement <4 x i32> %v, i32 1
+ ret i32 %r
+}
+
+define void @store_struct_field(ptr %p, i32 %x) {
+; CHECK-LABEL: define void @store_struct_field(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[X:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 1
+; CHECK-NEXT: store i32 [[X]], ptr [[TMP1]], align 4, !tbaa.struct [[TBAA_STRUCT21:![0-9]+]]
+; CHECK-NEXT: ret void
+;
+ %v = load <4 x i32>, ptr %p, align 16
+ %i = insertelement <4 x i32> %v, i32 %x, i32 1
+ store <4 x i32> %i, ptr %p, align 16, !tbaa.struct !21
+ ret void
+}
+
+!0 = !{i32 1}
+!1 = !{i32 0, !2}
+!2 = !{!"nvvm.l1_eviction", !"last"}
+!3 = !{}
+!4 = !{!"original access"}
+!5 = !{!"test", !"tag"}
+!6 = !{i32 5, i32 6}
+!7 = !{!8}
+!8 = distinct !{!8, !9, !"scope"}
+!9 = distinct !{!9, i1 false, !"domain"}
+!10 = distinct !{}
+!11 = !{i32 0, i32 100}
+!12 = !{i32 3}
+!13 = !{float 2.5}
+!14 = !{i64 0, i64 4, !15, i64 4, i64 4, !15, i64 8, i64 4, !15, i64 12, i64 4, !15}
+!15 = !{!16, !16, i64 0}
+!16 = !{!"omnipotent char", !17, i64 0}
+!17 = !{!"root"}
+!18 = !{i32 1, !2}
+
+!19 = !{!"int", !16, i64 0}
+!20 = !{!19, !19, i64 0}
+!21 = !{i64 0, i64 4, !15, i64 4, i64 4, !20, i64 8, i64 4, !15, i64 12, i64 4, !15}
+;.
+; CHECK: [[CHAR_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; CHECK: [[META1]] = !{!"omnipotent char", [[META2:![0-9]+]], i64 0}
+; CHECK: [[META2]] = !{!"root"}
+; CHECK: [[META3]] = !{[[META4:![0-9]+]]}
+; CHECK: [[META4]] = distinct !{[[META4]], [[META5:![0-9]+]], !"scope"}
+; CHECK: [[META5]] = distinct !{[[META5]], i1 false, !"domain"}
+; CHECK: [[META6]] = !{i32 5, i32 6}
+; CHECK: [[RNG7]] = !{i32 0, i32 100}
+; CHECK: [[META8]] = !{}
+; CHECK: [[META9]] = !{i32 1}
+; CHECK: [[ACC_GRP10]] = distinct !{}
+; CHECK: [[META11]] = !{!"original access"}
+; CHECK: [[META12]] = !{!"test", !"tag"}
+; CHECK: [[META13]] = !{i32 0, [[META14:![0-9]+]]}
+; CHECK: [[META14]] = !{!"nvvm.l1_eviction", !"last"}
+; CHECK: [[META15]] = !{float 2.500000e+00}
+; CHECK: [[META16]] = !{i32 3}
+; CHECK: [[TBAA_STRUCT17]] = !{i64 0, i64 4, [[CHAR_TBAA0]], i64 4, i64 4, [[CHAR_TBAA0]], i64 8, i64 4, [[CHAR_TBAA0]], i64 12, i64 4, [[CHAR_TBAA0]]}
+; CHECK: [[META18]] = !{i32 1, [[META14]]}
+; CHECK: [[INT_TBAA19]] = !{[[META20:![0-9]+]], [[META20]], i64 0}
+; CHECK: [[META20]] = !{!"int", [[META1]], i64 0}
+; CHECK: [[TBAA_STRUCT21]] = !{i64 0, i64 4, [[CHAR_TBAA0]], i64 4, i64 4, [[INT_TBAA19]], i64 8, i64 4, [[CHAR_TBAA0]], i64 12, i64 4, [[CHAR_TBAA0]]}
+;.
>From 46ee822a3976e11535e7150116ea49352984a692 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Fri, 13 Nov 2026 10:35:37 +0000
Subject: [PATCH 02/14] [Analysis] Centralize memory metadata utilities
---
llvm/include/llvm/Analysis/MetadataUtils.h | 86 +++
llvm/include/llvm/Analysis/VectorUtils.h | 29 +-
llvm/include/llvm/Transforms/Utils/Local.h | 33 +-
llvm/lib/Analysis/CMakeLists.txt | 1 +
llvm/lib/Analysis/MetadataUtils.cpp | 644 ++++++++++++++++++
llvm/lib/Analysis/VectorUtils.cpp | 163 +----
.../AMDGPU/AMDGPULowerBufferFatPointers.cpp | 5 +-
.../InstCombine/InstCombineCalls.cpp | 34 +-
.../InstCombineLoadStoreAlloca.cpp | 74 +-
llvm/lib/Transforms/Scalar/SROA.cpp | 115 +---
.../Scalar/ScalarizeMaskedMemIntrin.cpp | 132 +---
llvm/lib/Transforms/Scalar/Scalarizer.cpp | 31 +-
llvm/lib/Transforms/Utils/Local.cpp | 306 +--------
.../Vectorize/LoadStoreVectorizer.cpp | 1 +
.../Transforms/Vectorize/SLPVectorizer.cpp | 1 +
llvm/lib/Transforms/Vectorize/VPlan.h | 1 +
.../Transforms/Vectorize/VectorCombine.cpp | 36 +-
.../InstCombine/memory-cache-hint.ll | 19 +-
.../Transforms/InstCombine/memory-metadata.ll | 49 +-
.../InstCombine/storemerge-metadata.ll | 18 +-
llvm/test/Transforms/SROA/memory-metadata.ll | 55 +-
.../Scalarizer/basic-inseltpoison.ll | 51 +-
llvm/test/Transforms/Scalarizer/basic.ll | 51 +-
.../Scalarizer/memory-cache-hint.ll | 14 +-
.../Transforms/Scalarizer/memory-metadata.ll | 86 ++-
.../VectorCombine/RISCV/memory-metadata.ll | 17 +-
.../VectorCombine/X86/memory-metadata.ll | 17 +-
.../VectorCombine/load-bitcast-metadata.ll | 85 +++
.../VectorCombine/memory-metadata.ll | 50 +-
llvm/unittests/Analysis/CMakeLists.txt | 1 +
llvm/unittests/Analysis/MetadataUtilsTest.cpp | 390 +++++++++++
31 files changed, 1572 insertions(+), 1023 deletions(-)
create mode 100644 llvm/include/llvm/Analysis/MetadataUtils.h
create mode 100644 llvm/lib/Analysis/MetadataUtils.cpp
create mode 100644 llvm/test/Transforms/VectorCombine/load-bitcast-metadata.ll
create mode 100644 llvm/unittests/Analysis/MetadataUtilsTest.cpp
diff --git a/llvm/include/llvm/Analysis/MetadataUtils.h b/llvm/include/llvm/Analysis/MetadataUtils.h
new file mode 100644
index 00000000000000..bfaf8bd6912e8f
--- /dev/null
+++ b/llvm/include/llvm/Analysis/MetadataUtils.h
@@ -0,0 +1,86 @@
+//===- MetadataUtils.h - Metadata utilities ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ANALYSIS_METADATAUTILS_H
+#define LLVM_ANALYSIS_METADATAUTILS_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/Compiler.h"
+#include <cstdint>
+#include <optional>
+#include <utility>
+
+namespace llvm {
+class Instruction;
+class MDNode;
+class Value;
+
+/// Copy metadata valid for a replacement memory access. The destination must
+/// access the same bytes, a subset, or an expanded range in the same object.
+/// ByteOffset is the destination's offset within the source access. A missing
+/// offset requires the caller to establish that the new access is a subset.
+/// Access types and pointer operands are inferred for loads, stores, and
+/// masked/VP memory intrinsics. For a source memory intrinsic, the source or
+/// destination pointer is selected to match the new read or write. The caller
+/// must establish range containment when the intrinsic's length is unknown.
+/// Pointer and value facts are retained only when valid for the destination;
+/// alias tags are adjusted for its range. Masks and execution conditions are
+/// not inspected.
+LLVM_ABI void
+copyMetadataForMemoryAccess(Instruction &Dest, const Instruction &Source,
+ std::optional<int64_t> ByteOffset = 0);
+
+/// Copy metadata valid after splitting an instruction into smaller elements.
+/// ByteOffset describes the selected memory fragment, if applicable.
+LLVM_ABI void copyMetadataForScalarizedInstruction(
+ Instruction &Dest, const Instruction &Source,
+ std::optional<int64_t> ByteOffset = std::nullopt);
+
+/// Combine the metadata of two instructions so that K can replace J. This
+/// specifically handles the case of CSE-like transformations. Some
+/// metadata can only be kept if K dominates J. For this to be correct,
+/// K cannot be hoisted.
+///
+/// Unknown metadata is removed.
+LLVM_ABI void combineMetadataForCSE(Instruction *K, const Instruction *J,
+ bool DoesKMove);
+
+/// Combine metadata of two instructions, where instruction J is a memory
+/// access that has been merged into K. This will intersect alias-analysis
+/// metadata, while preserving other known metadata.
+LLVM_ABI void combineAAMetadata(Instruction *K, const Instruction *J);
+
+/// Compute the union of two access-group lists.
+///
+/// If the list contains just one access group, it is returned directly. If the
+/// list is empty, returns nullptr.
+LLVM_ABI MDNode *uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2);
+
+/// Compute the access-group list of access groups that @p Inst1 and @p Inst2
+/// are both in. If either instruction does not access memory at all, it is
+/// considered to be in every list.
+///
+/// If the list contains just one access group, it is returned directly. If the
+/// list is empty, returns nullptr.
+LLVM_ABI MDNode *intersectAccessGroups(const Instruction *Inst1,
+ const Instruction *Inst2);
+
+/// Collect metadata that can be preserved when vectorizing an instruction.
+LLVM_ABI void getMetadataToPropagate(
+ Instruction *Inst,
+ SmallVectorImpl<std::pair<unsigned, MDNode *>> &Metadata);
+
+/// Propagate metadata that can be preserved after vectorization from \p VL to
+/// \p I, retaining only information valid for all sources and the destination.
+/// Return \p I.
+LLVM_ABI Instruction *propagateMetadata(Instruction *I, ArrayRef<Value *> VL);
+
+} // namespace llvm
+
+#endif // LLVM_ANALYSIS_METADATAUTILS_H
diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h
index 3b0642b1ffbffe..6a48b007a7f4f3 100644
--- a/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/llvm/include/llvm/Analysis/VectorUtils.h
@@ -18,6 +18,7 @@
#include "llvm/ADT/Sequence.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/LoopAccessAnalysis.h"
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/VFABIDemangler.h"
#include "llvm/IR/VectorTypeUtils.h"
@@ -388,34 +389,6 @@ LLVM_ABI MapVector<Instruction *, uint64_t>
computeMinimumValueSizes(ArrayRef<BasicBlock *> Blocks, DemandedBits &DB,
const TargetTransformInfo *TTI = nullptr);
-/// Compute the union of two access-group lists.
-///
-/// If the list contains just one access group, it is returned directly. If the
-/// list is empty, returns nullptr.
-LLVM_ABI MDNode *uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2);
-
-/// Compute the access-group list of access groups that @p Inst1 and @p Inst2
-/// are both in. If either instruction does not access memory at all, it is
-/// considered to be in every list.
-///
-/// If the list contains just one access group, it is returned directly. If the
-/// list is empty, returns nullptr.
-LLVM_ABI MDNode *intersectAccessGroups(const Instruction *Inst1,
- const Instruction *Inst2);
-
-/// Add metadata from \p Inst to \p Metadata, if it can be preserved after
-/// vectorization. It can be preserved after vectorization if the kind is one of
-/// [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath, MD_nontemporal,
-/// MD_access_group, MD_mmra].
-LLVM_ABI void getMetadataToPropagate(
- Instruction *Inst,
- SmallVectorImpl<std::pair<unsigned, MDNode *>> &Metadata);
-
-/// Propagate metadata that can be preserved after vectorization from \p VL to
-/// \p I, retaining only information valid for all sources and the destination.
-/// Return \p I.
-LLVM_ABI Instruction *propagateMetadata(Instruction *I, ArrayRef<Value *> VL);
-
/// Create a mask that filters the members of an interleave group where there
/// are gaps.
///
diff --git a/llvm/include/llvm/Transforms/Utils/Local.h b/llvm/include/llvm/Transforms/Utils/Local.h
index 98b8ddd992282e..4991332881debd 100644
--- a/llvm/include/llvm/Transforms/Utils/Local.h
+++ b/llvm/include/llvm/Transforms/Utils/Local.h
@@ -15,6 +15,7 @@
#define LLVM_TRANSFORMS_UTILS_LOCAL_H
#include "llvm/ADT/ArrayRef.h"
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/IR/Dominators.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
@@ -413,24 +414,6 @@ LLVM_ABI bool removeUnreachableBlocks(Function &F,
MemorySSAUpdater *MSSAU = nullptr,
bool FoldInstsToUnreachable = true);
-/// Combine the metadata of two instructions so that K can replace J. This
-/// specifically handles the case of CSE-like transformations. Some
-/// metadata can only be kept if K dominates J. For this to be correct,
-/// K cannot be hoisted.
-///
-/// Unknown metadata is removed.
-LLVM_ABI void combineMetadataForCSE(Instruction *K, const Instruction *J,
- bool DoesKMove);
-
-/// Combine metadata of two instructions, where instruction J is a memory
-/// access that has been merged into K. This will intersect alias-analysis
-/// metadata, while preserving other known metadata.
-LLVM_ABI void combineAAMetadata(Instruction *K, const Instruction *J);
-
-/// Copy the metadata from the source instruction to the destination (the
-/// replacement for the source instruction).
-LLVM_ABI void copyMetadataForLoad(LoadInst &Dest, const LoadInst &Source);
-
/// Patch the replacement so that it is not more restrictive than the value
/// being replaced. Unless DoesKMove is set, it assumes that the replacement
/// does not get moved from its original position.
@@ -486,20 +469,6 @@ LLVM_ABI unsigned replaceDominatedUsesWithIf(
LLVM_ABI bool callsGCLeafFunction(const CallBase *Call,
const TargetLibraryInfo &TLI);
-/// Copy a nonnull metadata node to a new load instruction.
-///
-/// This handles mapping it to range metadata if the new load is an integer
-/// load instead of a pointer load.
-LLVM_ABI void copyNonnullMetadata(const LoadInst &OldLI, MDNode *N,
- LoadInst &NewLI);
-
-/// Copy a range metadata node to a new load instruction.
-///
-/// This handles mapping it to nonnull metadata if the new load is a pointer
-/// load instead of an integer load and the range doesn't cover null.
-LLVM_ABI void copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI,
- MDNode *N, LoadInst &NewLI);
-
/// Remove the debug intrinsic instructions for the given instruction.
LLVM_ABI void dropDebugUsers(Instruction &I);
diff --git a/llvm/lib/Analysis/CMakeLists.txt b/llvm/lib/Analysis/CMakeLists.txt
index 9c89cb4887d71f..301f517588a796 100644
--- a/llvm/lib/Analysis/CMakeLists.txt
+++ b/llvm/lib/Analysis/CMakeLists.txt
@@ -139,6 +139,7 @@ add_llvm_component_library(LLVMAnalysis
MemoryProfileInfo.cpp
MemorySSA.cpp
MemorySSAUpdater.cpp
+ MetadataUtils.cpp
ModelUnderTrainingRunner.cpp
ModuleDebugInfoPrinter.cpp
ModuleSummaryAnalysis.cpp
diff --git a/llvm/lib/Analysis/MetadataUtils.cpp b/llvm/lib/Analysis/MetadataUtils.cpp
new file mode 100644
index 00000000000000..936469fb58a149
--- /dev/null
+++ b/llvm/lib/Analysis/MetadataUtils.cpp
@@ -0,0 +1,644 @@
+//===- MetadataUtils.cpp - Metadata utilities --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/MetadataUtils.h"
+#include "llvm/ADT/APInt.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/IR/ConstantRange.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/MDBuilder.h"
+#include "llvm/IR/MemCacheHint.h"
+#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
+#include "llvm/IR/Metadata.h"
+#include "llvm/IR/Operator.h"
+#include "llvm/IR/ProfDataUtils.h"
+
+using namespace llvm;
+
+namespace {
+struct MemoryAccessInfo {
+ Type *Ty;
+ unsigned PointerOperand;
+ bool IsRead;
+};
+
+std::optional<MemoryAccessInfo>
+getMemoryAccessInfo(const Instruction &I,
+ std::optional<bool> IsRead = std::nullopt) {
+ if (const auto *LI = dyn_cast<LoadInst>(&I))
+ return MemoryAccessInfo{LI->getType(), 0, true};
+ if (const auto *SI = dyn_cast<StoreInst>(&I))
+ return MemoryAccessInfo{SI->getValueOperand()->getType(), 1, false};
+ if (const auto *MI = dyn_cast<AnyMemIntrinsic>(&I)) {
+ if (!IsRead || (*IsRead && !isa<AnyMemTransferInst>(MI)))
+ return std::nullopt;
+ return MemoryAccessInfo{nullptr, *IsRead ? 1u : 0u, *IsRead};
+ }
+ const auto *II = dyn_cast<IntrinsicInst>(&I);
+ if (!II)
+ return std::nullopt;
+ if (const auto *VP = dyn_cast<VPIntrinsic>(II)) {
+ auto Ptr = VPIntrinsic::getMemoryPointerParamPos(VP->getIntrinsicID());
+ if (!Ptr)
+ return std::nullopt;
+ if (auto Data = VPIntrinsic::getMemoryDataParamPos(VP->getIntrinsicID()))
+ return MemoryAccessInfo{VP->getArgOperand(*Data)->getType(), *Ptr, false};
+ Type *Ty = VP->getType();
+ if (VP->getIntrinsicID() == Intrinsic::vp_load_ff)
+ Ty = Ty->getStructElementType(0);
+ return MemoryAccessInfo{Ty, *Ptr, true};
+ }
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::experimental_vector_histogram_add:
+ case Intrinsic::experimental_vector_histogram_uadd_sat:
+ case Intrinsic::experimental_vector_histogram_umin:
+ case Intrinsic::experimental_vector_histogram_umax:
+ if (IsRead)
+ return MemoryAccessInfo{II->getArgOperand(1)->getType(), 0, *IsRead};
+ return std::nullopt;
+ case Intrinsic::masked_load:
+ case Intrinsic::masked_gather:
+ case Intrinsic::masked_expandload:
+ return MemoryAccessInfo{II->getType(), 0, true};
+ case Intrinsic::masked_store:
+ case Intrinsic::masked_scatter:
+ case Intrinsic::masked_compressstore:
+ return MemoryAccessInfo{II->getArgOperand(0)->getType(), 1, false};
+ default:
+ return std::nullopt;
+ }
+}
+
+const Value *getMemoryPointer(const Instruction &I, unsigned Index) {
+ if (const auto *CB = dyn_cast<CallBase>(&I))
+ return CB->getArgOperand(Index);
+ return I.getOperand(Index);
+}
+// invariant.group ignores bitcasts and zero-offset GEPs, not address-space
+// casts.
+const Value *getInvariantGroupPointer(const Value *V) {
+ while (true) {
+ if (const auto *Cast = dyn_cast<BitCastOperator>(V))
+ V = Cast->getOperand(0);
+ else if (const auto *GEP = dyn_cast<GEPOperator>(V);
+ GEP && GEP->hasAllZeroIndices())
+ V = GEP->getPointerOperand();
+ else
+ return V;
+ }
+}
+
+void copyNonnullMetadata(const LoadInst &OldLI, MDNode *N, LoadInst &NewLI) {
+ auto *NewTy = NewLI.getType();
+
+ // This only directly applies if the new type is also a pointer.
+ if (NewTy->isPointerTy()) {
+ NewLI.setMetadata(LLVMContext::MD_nonnull, N);
+ return;
+ }
+
+ // The only other translation we can do is to integral loads with !range
+ // metadata.
+ if (!NewTy->isIntegerTy())
+ return;
+
+ MDBuilder MDB(NewLI.getContext());
+ const Value *Ptr = OldLI.getPointerOperand();
+ auto *ITy = cast<IntegerType>(NewTy);
+ auto *NullInt = ConstantExpr::getPtrToInt(
+ ConstantPointerNull::get(cast<PointerType>(Ptr->getType())), ITy);
+ auto *NonNullInt = ConstantExpr::getAdd(NullInt, ConstantInt::get(ITy, 1));
+ NewLI.setMetadata(LLVMContext::MD_range,
+ MDB.createRange(NonNullInt, NullInt));
+}
+
+void copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI, MDNode *N,
+ LoadInst &NewLI) {
+ auto *NewTy = NewLI.getType();
+ // Simply copy the metadata if the type did not change.
+ if (NewTy == OldLI.getType()) {
+ NewLI.setMetadata(LLVMContext::MD_range, N);
+ return;
+ }
+
+ // Give up unless it is converted to a pointer where there is a single very
+ // valuable mapping we can do reliably.
+ // FIXME: It would be nice to propagate this in more ways, but the type
+ // conversions make it hard.
+ if (!NewTy->isPointerTy())
+ return;
+
+ unsigned BitWidth = DL.getPointerTypeSizeInBits(NewTy);
+ if (BitWidth == OldLI.getType()->getScalarSizeInBits() &&
+ !getConstantRangeFromMetadata(*N).contains(APInt(BitWidth, 0))) {
+ MDNode *NN = MDNode::get(OldLI.getContext(), {});
+ NewLI.setMetadata(LLVMContext::MD_nonnull, NN);
+ }
+}
+} // namespace
+
+void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
+ const Instruction &Source,
+ std::optional<int64_t> ByteOffset) {
+ auto Dst = getMemoryAccessInfo(Dest);
+ auto Src = getMemoryAccessInfo(Source, Dst ? std::optional<bool>(Dst->IsRead)
+ : std::nullopt);
+ assert(Src && Dst && Src->IsRead == Dst->IsRead &&
+ "expected corresponding memory reads or writes");
+ const DataLayout &DL = Source.getDataLayout();
+ std::optional<TypeSize> OldSize;
+ if (Src->Ty)
+ OldSize = DL.getTypeStoreSize(Src->Ty);
+ else if (auto *Length =
+ dyn_cast<ConstantInt>(cast<AnyMemIntrinsic>(Source).getLength()))
+ if (auto Bytes = Length->getValue().tryZExtValue())
+ OldSize = TypeSize::getFixed(*Bytes);
+ TypeSize NewSize = DL.getTypeStoreSize(Dst->Ty);
+ bool IsSubset = !OldSize || TypeSize::isKnownLE(NewSize, *OldSize);
+ if (ByteOffset)
+ IsSubset =
+ IsSubset && *ByteOffset >= 0 &&
+ (!OldSize || uint64_t(*ByteOffset) <= OldSize->getKnownMinValue() -
+ NewSize.getKnownMinValue());
+ bool IsWholeAccess = IsSubset && ByteOffset == 0 && OldSize == NewSize;
+ bool SameElementType =
+ Src->Ty && Src->Ty->getScalarType() == Dst->Ty->getScalarType();
+ bool SameValueBits =
+ IsWholeAccess && Src->Ty && !Src->Ty->isAggregateType() &&
+ !Dst->Ty->isAggregateType() && DL.typeSizeEqualsStoreSize(Src->Ty) &&
+ DL.typeSizeEqualsStoreSize(Dst->Ty);
+ bool SamePointer =
+ getInvariantGroupPointer(getMemoryPointer(Source, Src->PointerOperand)) ==
+ getInvariantGroupPointer(getMemoryPointer(Dest, Dst->PointerOperand));
+
+ SmallVector<std::pair<unsigned, MDNode *>, 8> MDs;
+ Source.getAllMetadata(MDs);
+ for (auto [Kind, MD] : MDs) {
+ bool Keep = false;
+ switch (Kind) {
+ case LLVMContext::MD_dbg:
+ case LLVMContext::MD_nontemporal:
+ case LLVMContext::MD_annotation:
+ case LLVMContext::MD_nosanitize:
+ Keep = true;
+ break;
+ case LLVMContext::MD_mmra:
+ Keep = canInstructionHaveMMRAs(Dest);
+ break;
+ case LLVMContext::MD_access_group:
+ case LLVMContext::MD_mem_parallel_loop_access:
+ Keep = IsSubset;
+ break;
+ case LLVMContext::MD_invariant_load:
+ Keep = IsSubset && Dst->IsRead;
+ break;
+ case LLVMContext::MD_invariant_group:
+ Keep = IsSubset && SamePointer && !isa<CallBase>(Dest);
+ break;
+ case LLVMContext::MD_noundef:
+ Keep =
+ IsSubset && (SameElementType || SameValueBits) && isa<LoadInst>(Dest);
+ break;
+ case LLVMContext::MD_fpmath:
+ case LLVMContext::MD_nofpclass:
+ Keep =
+ IsSubset && SameElementType &&
+ (Kind == LLVMContext::MD_fpmath ? Dest.getType()->isFPOrFPVectorTy()
+ : isa<LoadInst>(Dest));
+ break;
+ case LLVMContext::MD_nonnull:
+ if (SameValueBits && isa<LoadInst>(Source) && isa<LoadInst>(Dest))
+ copyNonnullMetadata(cast<LoadInst>(Source), MD, cast<LoadInst>(Dest));
+ break;
+ case LLVMContext::MD_range:
+ if (IsSubset && SameElementType && Dst->Ty->isIntOrIntVectorTy())
+ Keep = true;
+ else if (SameValueBits && isa<LoadInst>(Source) && isa<LoadInst>(Dest))
+ copyRangeMetadata(DL, cast<LoadInst>(Source), MD, cast<LoadInst>(Dest));
+ break;
+ case LLVMContext::MD_align:
+ case LLVMContext::MD_dereferenceable:
+ case LLVMContext::MD_dereferenceable_or_null:
+ Keep = SameValueBits && Dst->Ty->isPointerTy() && isa<LoadInst>(Dest);
+ break;
+ case LLVMContext::MD_DIAssignID:
+ Keep = IsWholeAccess && isa<StoreInst>(Source) && isa<StoreInst>(Dest);
+ break;
+ case LLVMContext::MD_prof:
+ Keep =
+ IsWholeAccess && ((isa<LoadInst>(Source) && isa<LoadInst>(Dest)) ||
+ (isa<StoreInst>(Source) && isa<StoreInst>(Dest)));
+ break;
+ case LLVMContext::MD_captures:
+ Keep = IsWholeAccess && isa<StoreInst>(Dest) && Dst->Ty->isPointerTy();
+ break;
+ default:
+ break;
+ }
+ if (Keep)
+ Dest.setMetadata(Kind, MD);
+ }
+
+ AAMDNodes AA = Source.getAAMetadata();
+ if (!IsSubset) {
+ AA.TBAA = nullptr;
+ AA.TBAAStruct = nullptr;
+ AA.Scope = nullptr;
+ AA.NoAlias = nullptr;
+ } else if (!IsWholeAccess) {
+ if (ByteOffset)
+ AA = AA.adjustForAccess(*ByteOffset, Dst->Ty, DL);
+ else
+ AA.TBAAStruct = nullptr;
+ } else {
+ // Prefer a single alias tag for a whole replacement, retaining the full
+ // field layout when it cannot be represented that way.
+ AAMDNodes Adjusted = AA.adjustForAccess(0, Dst->Ty, DL);
+ if (!AA.TBAA && Adjusted.TBAA)
+ AA = Adjusted;
+ }
+ Dest.setAAMetadata(AA);
+ setMemCacheHint(Dest, Dst->PointerOperand,
+ getMemCacheHint(Source, Src->PointerOperand));
+}
+
+void llvm::copyMetadataForScalarizedInstruction(
+ Instruction &Dest, const Instruction &Source,
+ std::optional<int64_t> ByteOffset) {
+ if (auto Dst = getMemoryAccessInfo(Dest))
+ if (getMemoryAccessInfo(Source, Dst->IsRead)) {
+ copyMetadataForMemoryAccess(Dest, Source, ByteOffset);
+ return;
+ }
+ if (Dest.getType()->getScalarType() == Source.getType()->getScalarType() &&
+ Dest.getType()->isFPOrFPVectorTy())
+ Dest.copyMetadata(Source, {LLVMContext::MD_fpmath});
+ if (!Dest.getDebugLoc())
+ Dest.setDebugLoc(Source.getDebugLoc());
+}
+
+/// Add all access groups in @p AccGroups to @p List.
+template <typename ListT>
+static void addToAccessGroupList(ListT &List, MDNode *AccGroups) {
+ // Interpret an access group as a list containing itself.
+ if (AccGroups->getNumOperands() == 0) {
+ assert(isValidAsAccessGroup(AccGroups) && "Node must be an access group");
+ List.insert(AccGroups);
+ return;
+ }
+
+ for (const auto &AccGroupListOp : AccGroups->operands()) {
+ auto *Item = cast<MDNode>(AccGroupListOp.get());
+ assert(isValidAsAccessGroup(Item) && "List item must be an access group");
+ List.insert(Item);
+ }
+}
+
+MDNode *llvm::uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2) {
+ if (!AccGroups1)
+ return AccGroups2;
+ if (!AccGroups2)
+ return AccGroups1;
+ if (AccGroups1 == AccGroups2)
+ return AccGroups1;
+
+ SmallSetVector<Metadata *, 4> Union;
+ addToAccessGroupList(Union, AccGroups1);
+ addToAccessGroupList(Union, AccGroups2);
+
+ if (Union.size() == 0)
+ return nullptr;
+ if (Union.size() == 1)
+ return cast<MDNode>(Union.front());
+
+ LLVMContext &Ctx = AccGroups1->getContext();
+ return MDNode::get(Ctx, Union.getArrayRef());
+}
+
+MDNode *llvm::intersectAccessGroups(const Instruction *Inst1,
+ const Instruction *Inst2) {
+ bool MayAccessMem1 = Inst1->mayReadOrWriteMemory();
+ bool MayAccessMem2 = Inst2->mayReadOrWriteMemory();
+
+ if (!MayAccessMem1 && !MayAccessMem2)
+ return nullptr;
+ if (!MayAccessMem1)
+ return Inst2->getMetadata(LLVMContext::MD_access_group);
+ if (!MayAccessMem2)
+ return Inst1->getMetadata(LLVMContext::MD_access_group);
+
+ MDNode *MD1 = Inst1->getMetadata(LLVMContext::MD_access_group);
+ MDNode *MD2 = Inst2->getMetadata(LLVMContext::MD_access_group);
+ if (!MD1 || !MD2)
+ return nullptr;
+ if (MD1 == MD2)
+ return MD1;
+
+ // Use set for scalable 'contains' check.
+ SmallPtrSet<Metadata *, 4> AccGroupSet2;
+ addToAccessGroupList(AccGroupSet2, MD2);
+
+ SmallVector<Metadata *, 4> Intersection;
+ if (MD1->getNumOperands() == 0) {
+ assert(isValidAsAccessGroup(MD1) && "Node must be an access group");
+ if (AccGroupSet2.count(MD1))
+ Intersection.push_back(MD1);
+ } else {
+ for (const MDOperand &Node : MD1->operands()) {
+ auto *Item = cast<MDNode>(Node.get());
+ assert(isValidAsAccessGroup(Item) && "List item must be an access group");
+ if (AccGroupSet2.count(Item))
+ Intersection.push_back(Item);
+ }
+ }
+
+ if (Intersection.size() == 0)
+ return nullptr;
+ if (Intersection.size() == 1)
+ return cast<MDNode>(Intersection.front());
+
+ LLVMContext &Ctx = Inst1->getContext();
+ return MDNode::get(Ctx, Intersection);
+}
+
+/// Add metadata from \p Inst to \p Metadata, if it can be preserved after
+/// vectorization.
+void llvm::getMetadataToPropagate(
+ Instruction *Inst,
+ SmallVectorImpl<std::pair<unsigned, MDNode *>> &Metadata) {
+ Inst->getAllMetadataOtherThanDebugLoc(Metadata);
+ static const unsigned SupportedIDs[] = {
+ LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope,
+ LLVMContext::MD_noalias, LLVMContext::MD_fpmath,
+ LLVMContext::MD_nontemporal, LLVMContext::MD_invariant_load,
+ LLVMContext::MD_access_group, LLVMContext::MD_mmra};
+
+ // Remove any unsupported metadata kinds from Metadata.
+ for (unsigned Idx = 0; Idx != Metadata.size();) {
+ if (is_contained(SupportedIDs, Metadata[Idx].first)) {
+ ++Idx;
+ } else {
+ // Swap element to end and remove it.
+ std::swap(Metadata[Idx], Metadata.back());
+ Metadata.pop_back();
+ }
+ }
+}
+
+/// \returns \p I after propagating metadata from \p VL.
+Instruction *llvm::propagateMetadata(Instruction *Inst, ArrayRef<Value *> VL) {
+ if (VL.empty())
+ return Inst;
+
+ // Keep cache hints out of getMetadataToPropagate(): VPlan changes operand
+ // layouts without remapping cache-hint indices.
+ propagateMemCacheHint(*Inst, VL);
+
+ SmallVector<std::pair<unsigned, MDNode *>> Metadata;
+ getMetadataToPropagate(cast<Instruction>(VL[0]), Metadata);
+
+ for (auto &[Kind, MD] : Metadata) {
+ // Skip MMRA metadata if the instruction cannot have it.
+ if (Kind == LLVMContext::MD_mmra && !canInstructionHaveMMRAs(*Inst))
+ continue;
+
+ for (int J = 1, E = VL.size(); MD && J != E; ++J) {
+ const Instruction *IJ = cast<Instruction>(VL[J]);
+ MDNode *IMD = IJ->getMetadata(Kind);
+
+ switch (Kind) {
+ case LLVMContext::MD_mmra: {
+ MD = MMRAMetadata::combine(Inst->getContext(), MD, IMD);
+ break;
+ }
+ case LLVMContext::MD_tbaa:
+ MD = MDNode::getMostGenericTBAA(MD, IMD);
+ break;
+ case LLVMContext::MD_alias_scope:
+ MD = MDNode::getMostGenericAliasScope(MD, IMD);
+ break;
+ case LLVMContext::MD_fpmath:
+ MD = MDNode::getMostGenericFPMath(MD, IMD);
+ break;
+ case LLVMContext::MD_noalias:
+ case LLVMContext::MD_nontemporal:
+ case LLVMContext::MD_invariant_load:
+ MD = MDNode::intersect(MD, IMD);
+ break;
+ case LLVMContext::MD_access_group:
+ MD = intersectAccessGroups(Inst, IJ);
+ break;
+ default:
+ llvm_unreachable("unhandled metadata");
+ }
+ }
+
+ Inst->setMetadata(Kind, MD);
+ }
+
+ return Inst;
+}
+
+/// If AAOnly is set, only intersect alias analysis metadata and preserve other
+/// known metadata. Unknown metadata is always dropped.
+static void combineMetadata(Instruction *K, const Instruction *J,
+ bool DoesKMove, bool AAOnly = false) {
+ SmallVector<std::pair<unsigned, MDNode *>, 4> Metadata;
+ K->getAllMetadataOtherThanDebugLoc(Metadata);
+ for (const auto &MD : Metadata) {
+ unsigned Kind = MD.first;
+ MDNode *JMD = J->getMetadata(Kind);
+ MDNode *KMD = MD.second;
+
+ // TODO: Assert that this switch is exhaustive for fixed MD kinds.
+ switch (Kind) {
+ default:
+ K->setMetadata(Kind, nullptr); // Remove unknown metadata
+ break;
+ case LLVMContext::MD_dbg:
+ llvm_unreachable("getAllMetadataOtherThanDebugLoc returned a MD_dbg");
+ case LLVMContext::MD_DIAssignID:
+ if (!AAOnly)
+ K->mergeDIAssignID(J);
+ break;
+ case LLVMContext::MD_tbaa:
+ if (DoesKMove)
+ K->setMetadata(Kind, MDNode::getMostGenericTBAA(JMD, KMD));
+ break;
+ case LLVMContext::MD_alias_scope:
+ if (DoesKMove)
+ K->setMetadata(Kind, MDNode::getMostGenericAliasScope(JMD, KMD));
+ break;
+ case LLVMContext::MD_noalias:
+ case LLVMContext::MD_mem_parallel_loop_access:
+ if (DoesKMove)
+ K->setMetadata(Kind, MDNode::intersect(JMD, KMD));
+ break;
+ case LLVMContext::MD_access_group:
+ if (DoesKMove)
+ K->setMetadata(LLVMContext::MD_access_group,
+ intersectAccessGroups(K, J));
+ break;
+ case LLVMContext::MD_range:
+ if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
+ K->setMetadata(Kind, MDNode::getMostGenericRange(JMD, KMD));
+ break;
+ case LLVMContext::MD_nofpclass:
+ if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
+ K->setMetadata(Kind, MDNode::getMostGenericNoFPClass(JMD, KMD));
+ break;
+ case LLVMContext::MD_fpmath:
+ if (!AAOnly)
+ K->setMetadata(Kind, MDNode::getMostGenericFPMath(JMD, KMD));
+ break;
+ case LLVMContext::MD_invariant_load:
+ case LLVMContext::MD_invariant_group:
+ // If K moves, only keep the invariant metadata if it is present on
+ // both instructions; otherwise the invariant would be asserted on a
+ // path (J's) that never promised it. If K does not move, K stays on
+ // its original path, so its existing metadata remains valid.
+ if (DoesKMove)
+ K->setMetadata(Kind, JMD);
+ break;
+ case LLVMContext::MD_nonnull:
+ if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
+ K->setMetadata(Kind, JMD);
+ break;
+ // Keep empty cases for prof, mmra, memprof, and callsite to prevent them
+ // from being removed as unknown metadata. The actual merging is handled
+ // separately below.
+ case LLVMContext::MD_prof:
+ case LLVMContext::MD_mmra:
+ case LLVMContext::MD_memprof:
+ case LLVMContext::MD_callsite:
+ break;
+ case LLVMContext::MD_callee_type:
+ if (!AAOnly) {
+ K->setMetadata(LLVMContext::MD_callee_type,
+ MDNode::getMergedCalleeTypeMetadata(KMD, JMD));
+ }
+ break;
+ case LLVMContext::MD_callees:
+ // If K moves, it replaces J on J's path and must allow J's callees as
+ // well. If K does not move, its callees remain valid.
+ if (!AAOnly && DoesKMove)
+ K->setMetadata(Kind, MDNode::getMergedCalleesMetadata(KMD, JMD));
+ break;
+ case LLVMContext::MD_align:
+ if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
+ K->setMetadata(
+ Kind, MDNode::getMostGenericAlignmentOrDereferenceable(JMD, KMD));
+ break;
+ case LLVMContext::MD_dereferenceable:
+ case LLVMContext::MD_dereferenceable_or_null:
+ if (!AAOnly && DoesKMove)
+ K->setMetadata(Kind,
+ MDNode::getMostGenericAlignmentOrDereferenceable(JMD, KMD));
+ break;
+ case LLVMContext::MD_preserve_access_index:
+ // Preserve !preserve.access.index in K.
+ break;
+ case LLVMContext::MD_noundef:
+ // If K does move, keep noundef if it is present in both instructions.
+ if (!AAOnly && DoesKMove)
+ K->setMetadata(Kind, JMD);
+ break;
+ case LLVMContext::MD_nontemporal:
+ // Preserve !nontemporal if it is present on both instructions.
+ if (!AAOnly)
+ K->setMetadata(Kind, JMD);
+ break;
+ case LLVMContext::MD_mem_cache_hint:
+ // Preserve !mem.cache_hint only if it is present and equivalent on both
+ // instructions.
+ if (!AAOnly && KMD != JMD)
+ K->setMetadata(Kind, nullptr);
+ break;
+ case LLVMContext::MD_noalias_addrspace:
+ if (DoesKMove)
+ K->setMetadata(Kind,
+ MDNode::getMostGenericNoaliasAddrspace(JMD, KMD));
+ break;
+ case LLVMContext::MD_nosanitize:
+ // Preserve !nosanitize if both K and J have it.
+ K->setMetadata(Kind, JMD);
+ break;
+ case LLVMContext::MD_captures:
+ K->setMetadata(
+ Kind, MDNode::fromCaptureComponents(
+ K->getContext(), MDNode::toCaptureComponents(JMD) |
+ MDNode::toCaptureComponents(KMD)));
+ break;
+ case LLVMContext::MD_alloc_token:
+ if (!AAOnly && KMD != JMD)
+ K->setMetadata(Kind, MDNode::getMergedAllocTokenMetadata(KMD, JMD));
+ break;
+ }
+ }
+
+ // Merge MMRAs.
+ // This is handled separately because we also want to handle cases where K
+ // doesn't have tags but J does.
+ auto JMMRA = J->getMetadata(LLVMContext::MD_mmra);
+ auto KMMRA = K->getMetadata(LLVMContext::MD_mmra);
+ if (JMMRA || KMMRA) {
+ K->setMetadata(LLVMContext::MD_mmra,
+ MMRAMetadata::combine(K->getContext(), JMMRA, KMMRA));
+ }
+
+ // Merge memprof metadata.
+ // Handle separately to support cases where only one instruction has the
+ // metadata.
+ auto *JMemProf = J->getMetadata(LLVMContext::MD_memprof);
+ auto *KMemProf = K->getMetadata(LLVMContext::MD_memprof);
+ if (!AAOnly && (JMemProf || KMemProf)) {
+ K->setMetadata(LLVMContext::MD_memprof,
+ MDNode::getMergedMemProfMetadata(KMemProf, JMemProf));
+ }
+
+ // Merge callsite metadata.
+ // Handle separately to support cases where only one instruction has the
+ // metadata.
+ auto *JCallSite = J->getMetadata(LLVMContext::MD_callsite);
+ auto *KCallSite = K->getMetadata(LLVMContext::MD_callsite);
+ if (!AAOnly && (JCallSite || KCallSite)) {
+ K->setMetadata(LLVMContext::MD_callsite,
+ MDNode::getMergedCallsiteMetadata(KCallSite, JCallSite));
+ }
+
+ // Merge prof metadata.
+ // Handle separately to support cases where only one instruction has the
+ // metadata.
+ auto *JProf = J->getMetadata(LLVMContext::MD_prof);
+ auto *KProf = K->getMetadata(LLVMContext::MD_prof);
+ if (!AAOnly && (JProf || KProf)) {
+ // We do not need to merge metadata if we have select instructions and the
+ // one we're keeping (K) does not move (explicitly merging would drop the
+ // information in this case). Otherwise, explicitly handle the merging.
+ if (DoesKMove || (!isa<SelectInst>(J) || !isa<SelectInst>(K)) ||
+ ProfcheckDisableMetadataFixes)
+ K->setMetadata(LLVMContext::MD_prof,
+ MDNode::getMergedProfMetadata(KProf, JProf, K, J));
+ }
+}
+
+void llvm::combineMetadataForCSE(Instruction *K, const Instruction *J,
+ bool DoesKMove) {
+ combineMetadata(K, J, DoesKMove);
+}
+
+void llvm::combineAAMetadata(Instruction *K, const Instruction *J) {
+ combineMetadata(K, J, /*DoesKMove=*/true, /*AAOnly=*/true);
+}
+
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index b5ccb8574b6026..8b479de7037b9e 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -18,6 +18,7 @@
#include "llvm/Analysis/DemandedBits.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopIterator.h"
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
@@ -1030,168 +1031,6 @@ llvm::computeMinimumValueSizes(ArrayRef<BasicBlock *> Blocks, DemandedBits &DB,
return MinBWs;
}
-/// Add all access groups in @p AccGroups to @p List.
-template <typename ListT>
-static void addToAccessGroupList(ListT &List, MDNode *AccGroups) {
- // Interpret an access group as a list containing itself.
- if (AccGroups->getNumOperands() == 0) {
- assert(isValidAsAccessGroup(AccGroups) && "Node must be an access group");
- List.insert(AccGroups);
- return;
- }
-
- for (const auto &AccGroupListOp : AccGroups->operands()) {
- auto *Item = cast<MDNode>(AccGroupListOp.get());
- assert(isValidAsAccessGroup(Item) && "List item must be an access group");
- List.insert(Item);
- }
-}
-
-MDNode *llvm::uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2) {
- if (!AccGroups1)
- return AccGroups2;
- if (!AccGroups2)
- return AccGroups1;
- if (AccGroups1 == AccGroups2)
- return AccGroups1;
-
- SmallSetVector<Metadata *, 4> Union;
- addToAccessGroupList(Union, AccGroups1);
- addToAccessGroupList(Union, AccGroups2);
-
- if (Union.size() == 0)
- return nullptr;
- if (Union.size() == 1)
- return cast<MDNode>(Union.front());
-
- LLVMContext &Ctx = AccGroups1->getContext();
- return MDNode::get(Ctx, Union.getArrayRef());
-}
-
-MDNode *llvm::intersectAccessGroups(const Instruction *Inst1,
- const Instruction *Inst2) {
- bool MayAccessMem1 = Inst1->mayReadOrWriteMemory();
- bool MayAccessMem2 = Inst2->mayReadOrWriteMemory();
-
- if (!MayAccessMem1 && !MayAccessMem2)
- return nullptr;
- if (!MayAccessMem1)
- return Inst2->getMetadata(LLVMContext::MD_access_group);
- if (!MayAccessMem2)
- return Inst1->getMetadata(LLVMContext::MD_access_group);
-
- MDNode *MD1 = Inst1->getMetadata(LLVMContext::MD_access_group);
- MDNode *MD2 = Inst2->getMetadata(LLVMContext::MD_access_group);
- if (!MD1 || !MD2)
- return nullptr;
- if (MD1 == MD2)
- return MD1;
-
- // Use set for scalable 'contains' check.
- SmallPtrSet<Metadata *, 4> AccGroupSet2;
- addToAccessGroupList(AccGroupSet2, MD2);
-
- SmallVector<Metadata *, 4> Intersection;
- if (MD1->getNumOperands() == 0) {
- assert(isValidAsAccessGroup(MD1) && "Node must be an access group");
- if (AccGroupSet2.count(MD1))
- Intersection.push_back(MD1);
- } else {
- for (const MDOperand &Node : MD1->operands()) {
- auto *Item = cast<MDNode>(Node.get());
- assert(isValidAsAccessGroup(Item) && "List item must be an access group");
- if (AccGroupSet2.count(Item))
- Intersection.push_back(Item);
- }
- }
-
- if (Intersection.size() == 0)
- return nullptr;
- if (Intersection.size() == 1)
- return cast<MDNode>(Intersection.front());
-
- LLVMContext &Ctx = Inst1->getContext();
- return MDNode::get(Ctx, Intersection);
-}
-
-/// Add metadata from \p Inst to \p Metadata, if it can be preserved after
-/// vectorization.
-void llvm::getMetadataToPropagate(
- Instruction *Inst,
- SmallVectorImpl<std::pair<unsigned, MDNode *>> &Metadata) {
- Inst->getAllMetadataOtherThanDebugLoc(Metadata);
- static const unsigned SupportedIDs[] = {
- LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope,
- LLVMContext::MD_noalias, LLVMContext::MD_fpmath,
- LLVMContext::MD_nontemporal, LLVMContext::MD_invariant_load,
- LLVMContext::MD_access_group, LLVMContext::MD_mmra};
-
- // Remove any unsupported metadata kinds from Metadata.
- for (unsigned Idx = 0; Idx != Metadata.size();) {
- if (is_contained(SupportedIDs, Metadata[Idx].first)) {
- ++Idx;
- } else {
- // Swap element to end and remove it.
- std::swap(Metadata[Idx], Metadata.back());
- Metadata.pop_back();
- }
- }
-}
-
-/// \returns \p I after propagating metadata from \p VL.
-Instruction *llvm::propagateMetadata(Instruction *Inst, ArrayRef<Value *> VL) {
- if (VL.empty())
- return Inst;
-
- // Keep cache hints out of getMetadataToPropagate(): VPlan changes operand
- // layouts without remapping cache-hint indices.
- propagateMemCacheHint(*Inst, VL);
-
- SmallVector<std::pair<unsigned, MDNode *>> Metadata;
- getMetadataToPropagate(cast<Instruction>(VL[0]), Metadata);
-
- for (auto &[Kind, MD] : Metadata) {
- // Skip MMRA metadata if the instruction cannot have it.
- if (Kind == LLVMContext::MD_mmra && !canInstructionHaveMMRAs(*Inst))
- continue;
-
- for (int J = 1, E = VL.size(); MD && J != E; ++J) {
- const Instruction *IJ = cast<Instruction>(VL[J]);
- MDNode *IMD = IJ->getMetadata(Kind);
-
- switch (Kind) {
- case LLVMContext::MD_mmra: {
- MD = MMRAMetadata::combine(Inst->getContext(), MD, IMD);
- break;
- }
- case LLVMContext::MD_tbaa:
- MD = MDNode::getMostGenericTBAA(MD, IMD);
- break;
- case LLVMContext::MD_alias_scope:
- MD = MDNode::getMostGenericAliasScope(MD, IMD);
- break;
- case LLVMContext::MD_fpmath:
- MD = MDNode::getMostGenericFPMath(MD, IMD);
- break;
- case LLVMContext::MD_noalias:
- case LLVMContext::MD_nontemporal:
- case LLVMContext::MD_invariant_load:
- MD = MDNode::intersect(MD, IMD);
- break;
- case LLVMContext::MD_access_group:
- MD = intersectAccessGroups(Inst, IJ);
- break;
- default:
- llvm_unreachable("unhandled metadata");
- }
- }
-
- Inst->setMetadata(Kind, MD);
- }
-
- return Inst;
-}
-
Constant *
llvm::createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF,
const InterleaveGroup<Instruction> &Group) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
index 89f024ca8a5dc9..1e1a01be30e948 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
@@ -225,6 +225,7 @@
#include "llvm/ADT/SetOperations.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/InstSimplifyFolder.h"
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetTransformInfo.h"
@@ -586,7 +587,7 @@ bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::visitLoadInst(LoadInst &LI) {
LoadInst *NewLI = IRB.CreateAlignedLoad(
IntLeafTy, Ptr, commonAlignment(LI.getAlign(), Off), Name);
NewLI->setVolatile(LI.isVolatile());
- copyMetadataForLoad(*NewLI, LI);
+ copyMetadataForMemoryAccess(*NewLI, LI);
NewLI->setAAMetadata(AATags.adjustForAccess(Off, IntLeafTy, DL));
Value *V = NewLI;
if (LeafTy != IntLeafTy)
@@ -1192,7 +1193,7 @@ bool LegalizeBufferContentTypesVisitor::visitLoadImpl(
LoadInst *NewLI = IRB.CreateAlignedLoad(
LoadableType, NewPtr, commonAlignment(OrigLI.getAlign(), ByteOffset),
Name + ".off." + Twine(ByteOffset));
- copyMetadataForLoad(*NewLI, OrigLI);
+ copyMetadataForMemoryAccess(*NewLI, OrigLI);
NewLI->setAAMetadata(
AANodes.adjustForAccess(ByteOffset, LoadableType, DL));
NewLI->setAtomic(OrigLI.getOrdering(), OrigLI.getSyncScopeID());
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 5d9e01ac505a0c..e98c6bb616e955 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -27,6 +27,7 @@
#include "llvm/Analysis/InstructionSimplify.h"
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/MemoryBuiltins.h"
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/AttributeMask.h"
@@ -169,32 +170,17 @@ Instruction *InstCombinerImpl::SimplifyAnyMemTransfer(AnyMemTransferInst *MI) {
// Use an integer load+store unless we can find something better.
IntegerType* IntType = IntegerType::get(MI->getContext(), Size<<3);
- // If the memcpy has metadata describing the members, see if we can get the
- // TBAA, scope and noalias tags describing our copy.
- AAMDNodes AACopyMD = MI->getAAMetadata().adjustForAccess(Size);
-
Value *Src = MI->getArgOperand(1);
Value *Dest = MI->getArgOperand(0);
LoadInst *L = Builder.CreateLoad(IntType, Src);
// Alignment from the mem intrinsic will be better, so use it.
L->setAlignment(*CopySrcAlign);
- L->setAAMetadata(AACopyMD);
- MDNode *LoopMemParallelMD =
- MI->getMetadata(LLVMContext::MD_mem_parallel_loop_access);
- if (LoopMemParallelMD)
- L->setMetadata(LLVMContext::MD_mem_parallel_loop_access, LoopMemParallelMD);
- MDNode *AccessGroupMD = MI->getMetadata(LLVMContext::MD_access_group);
- if (AccessGroupMD)
- L->setMetadata(LLVMContext::MD_access_group, AccessGroupMD);
+ copyMetadataForMemoryAccess(*L, *MI);
StoreInst *S = Builder.CreateStore(L, Dest);
// Alignment from the mem intrinsic will be better, so use it.
S->setAlignment(*CopyDstAlign);
- S->setAAMetadata(AACopyMD);
- if (LoopMemParallelMD)
- S->setMetadata(LLVMContext::MD_mem_parallel_loop_access, LoopMemParallelMD);
- if (AccessGroupMD)
- S->setMetadata(LLVMContext::MD_access_group, AccessGroupMD);
+ copyMetadataForMemoryAccess(*S, *MI);
S->copyMetadata(*MI, LLVMContext::MD_DIAssignID);
if (auto *MT = dyn_cast<MemTransferInst>(MI)) {
@@ -272,6 +258,7 @@ Instruction *InstCombinerImpl::SimplifyAnyMemSet(AnyMemSetInst *MI) {
return nullptr;
StoreInst *S = Builder.CreateStore(FillVal, Dest, MI->isVolatile());
+ copyMetadataForMemoryAccess(*S, *MI);
S->copyMetadata(*MI, LLVMContext::MD_DIAssignID);
for (DbgVariableRecord *DbgAssign : at::getDVRAssignmentMarkers(S)) {
if (llvm::is_contained(DbgAssign->location_ops(), Fill))
@@ -302,7 +289,7 @@ Value *InstCombinerImpl::simplifyMaskedLoad(IntrinsicInst &II) {
if (match(Mask, m_AllOnesOrPoison())) {
LoadInst *L = Builder.CreateAlignedLoad(II.getType(), LoadPtr, Alignment,
"unmaskedload");
- L->copyMetadata(II);
+ copyMetadataForMemoryAccess(*L, II);
return L;
}
@@ -312,7 +299,9 @@ Value *InstCombinerImpl::simplifyMaskedLoad(IntrinsicInst &II) {
SQ.getWithInstruction(&II))) {
LoadInst *LI = Builder.CreateAlignedLoad(II.getType(), LoadPtr, Alignment,
"unmaskedload");
- LI->copyMetadata(II);
+ // TODO: Check whether metadata valid only for active lanes can be retained
+ // on the unmasked load.
+ copyMetadataForMemoryAccess(*LI, II);
return Builder.CreateSelect(II.getArgOperand(1), LI, II.getArgOperand(2));
}
@@ -338,7 +327,7 @@ Instruction *InstCombinerImpl::simplifyMaskedStore(IntrinsicInst &II) {
if (match(ConstMask, m_AllOnesOrPoison())) {
StoreInst *S =
new StoreInst(II.getArgOperand(0), StorePtr, false, Alignment);
- S->copyMetadata(II);
+ copyMetadataForMemoryAccess(*S, II);
return S;
}
@@ -375,6 +364,7 @@ Instruction *InstCombinerImpl::simplifyMaskedGather(IntrinsicInst &II) {
const Align Alignment = II.getParamAlign(0).valueOrOne();
LoadInst *L = Builder.CreateAlignedLoad(VecTy->getElementType(), SplatPtr,
Alignment, "load.scalar");
+ copyMetadataForMemoryAccess(*L, II);
Value *Shuf =
Builder.CreateVectorSplat(VecTy->getElementCount(), L, "broadcast");
return replaceInstUsesWith(II, cast<Instruction>(Shuf));
@@ -405,7 +395,7 @@ Instruction *InstCombinerImpl::simplifyMaskedScatter(IntrinsicInst &II) {
Align Alignment = II.getParamAlign(1).valueOrOne();
StoreInst *S = new StoreInst(SplatValue, SplatPtr, /*IsVolatile=*/false,
Alignment);
- S->copyMetadata(II);
+ copyMetadataForMemoryAccess(*S, II);
return S;
}
}
@@ -421,7 +411,7 @@ Instruction *InstCombinerImpl::simplifyMaskedScatter(IntrinsicInst &II) {
Builder.CreateExtractElement(II.getArgOperand(0), LastLane);
StoreInst *S =
new StoreInst(Extract, SplatPtr, /*IsVolatile=*/false, Alignment);
- S->copyMetadata(II);
+ copyMetadataForMemoryAccess(*S, II);
return S;
}
}
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
index 60331a3d37b05b..a281e93aa6b977 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
@@ -16,6 +16,7 @@
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/Loads.h"
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
@@ -611,7 +612,7 @@ LoadInst *InstCombinerImpl::combineLoadToNewType(LoadInst &LI, Type *NewTy,
LoadInst *NewLoad = Builder.CreateLoad(
NewTy, LI.getPointerOperand(), LI.getProperties(), LI.getName() + Suffix);
- copyMetadataForLoad(*NewLoad, LI);
+ copyMetadataForMemoryAccess(*NewLoad, LI);
return NewLoad;
}
@@ -624,47 +625,8 @@ static StoreInst *combineStoreToNewValue(InstCombinerImpl &IC, StoreInst &SI,
"can't fold an atomic store of requested type");
Value *Ptr = SI.getPointerOperand();
- SmallVector<std::pair<unsigned, MDNode *>, 8> MD;
- SI.getAllMetadata(MD);
-
StoreInst *NewStore = IC.Builder.CreateStore(V, Ptr, SI.getProperties());
- for (const auto &MDPair : MD) {
- unsigned ID = MDPair.first;
- MDNode *N = MDPair.second;
- // Note, essentially every kind of metadata should be preserved here! This
- // routine is supposed to clone a store instruction changing *only its
- // type*. The only metadata it makes sense to drop is metadata which is
- // invalidated when the pointer type changes. This should essentially
- // never be the case in LLVM, but we explicitly switch over only known
- // metadata to be conservatively correct. If you are adding metadata to
- // LLVM which pertains to stores, you almost certainly want to add it
- // here.
- switch (ID) {
- case LLVMContext::MD_dbg:
- case LLVMContext::MD_DIAssignID:
- case LLVMContext::MD_tbaa:
- case LLVMContext::MD_prof:
- case LLVMContext::MD_fpmath:
- case LLVMContext::MD_tbaa_struct:
- case LLVMContext::MD_alias_scope:
- case LLVMContext::MD_noalias:
- case LLVMContext::MD_nontemporal:
- case LLVMContext::MD_mem_parallel_loop_access:
- case LLVMContext::MD_access_group:
- // All of these directly apply.
- NewStore->setMetadata(ID, N);
- break;
- case LLVMContext::MD_invariant_load:
- case LLVMContext::MD_nonnull:
- case LLVMContext::MD_noundef:
- case LLVMContext::MD_range:
- case LLVMContext::MD_align:
- case LLVMContext::MD_dereferenceable:
- case LLVMContext::MD_dereferenceable_or_null:
- // These don't apply for stores.
- break;
- }
- }
+ copyMetadataForMemoryAccess(*NewStore, SI);
return NewStore;
}
@@ -752,9 +714,6 @@ static Instruction *unpackLoadToAggregate(InstCombinerImpl &IC, LoadInst &LI) {
if (NumElements == 1) {
LoadInst *NewLoad = IC.combineLoadToNewType(LI, ST->getTypeAtIndex(0U),
".unpack");
- NewLoad->setAAMetadata(LI.getAAMetadata());
- // Copy invariant metadata from parent load.
- NewLoad->copyMetadata(LI, LLVMContext::MD_invariant_load);
return IC.replaceInstUsesWith(LI, IC.Builder.CreateInsertValue(
PoisonValue::get(T), NewLoad, 0, Name));
}
@@ -780,10 +739,8 @@ static Instruction *unpackLoadToAggregate(InstCombinerImpl &IC, LoadInst &LI) {
ST->getElementType(i), Ptr,
commonAlignment(Align, SL->getElementOffset(i).getKnownMinValue()),
Name + ".unpack");
- // Propagate AA metadata. It'll still be valid on the narrowed load.
- L->setAAMetadata(LI.getAAMetadata());
- // Copy invariant metadata from parent load.
- L->copyMetadata(LI, LLVMContext::MD_invariant_load);
+ copyMetadataForMemoryAccess(*L, LI,
+ SL->getElementOffset(i).getKnownMinValue());
V = IC.Builder.CreateInsertValue(V, L, i);
}
@@ -796,7 +753,6 @@ static Instruction *unpackLoadToAggregate(InstCombinerImpl &IC, LoadInst &LI) {
auto NumElements = AT->getNumElements();
if (NumElements == 1) {
LoadInst *NewLoad = IC.combineLoadToNewType(LI, ET, ".unpack");
- NewLoad->setAAMetadata(LI.getAAMetadata());
return IC.replaceInstUsesWith(LI, IC.Builder.CreateInsertValue(
PoisonValue::get(T), NewLoad, 0, Name));
}
@@ -828,7 +784,7 @@ static Instruction *unpackLoadToAggregate(InstCombinerImpl &IC, LoadInst &LI) {
auto EltAlign = commonAlignment(Align, Offset.getKnownMinValue());
auto *L = IC.Builder.CreateAlignedLoad(AT->getElementType(), Ptr,
EltAlign, Name + ".unpack");
- L->setAAMetadata(LI.getAAMetadata());
+ copyMetadataForMemoryAccess(*L, LI, Offset.getKnownMinValue());
V = IC.Builder.CreateInsertValue(V, L, i);
Offset += EltSize;
}
@@ -1435,7 +1391,8 @@ static bool unpackStoreToAggregate(InstCombinerImpl &IC, StoreInst &SI) {
auto EltAlign =
commonAlignment(Align, SL->getElementOffset(i).getKnownMinValue());
llvm::Instruction *NS = IC.Builder.CreateAlignedStore(Val, Ptr, EltAlign);
- NS->setAAMetadata(SI.getAAMetadata());
+ copyMetadataForMemoryAccess(*NS, SI,
+ SL->getElementOffset(i).getKnownMinValue());
}
return true;
@@ -1481,7 +1438,7 @@ static bool unpackStoreToAggregate(InstCombinerImpl &IC, StoreInst &SI) {
auto *Val = IC.Builder.CreateExtractValue(V, i, EltName);
auto EltAlign = commonAlignment(Align, Offset.getKnownMinValue());
Instruction *NS = IC.Builder.CreateAlignedStore(Val, Ptr, EltAlign);
- NS->setAAMetadata(SI.getAAMetadata());
+ copyMetadataForMemoryAccess(*NS, SI, Offset.getKnownMinValue());
Offset += EltSize;
}
@@ -1802,17 +1759,10 @@ bool InstCombinerImpl::mergeStoreIntoSuccessor(StoreInst &SI) {
StoreInst *NewSI =
new StoreInst(MergedVal, SI.getOperand(1), SI.getProperties());
InsertNewInstBefore(NewSI, BBI);
- NewSI->setDebugLoc(MergedLoc);
+ copyMetadataForMemoryAccess(*NewSI, SI);
NewSI->mergeDIAssignID({&SI, OtherStore});
-
- // If the two stores had AA tags, merge them.
- AAMDNodes AATags = SI.getAAMetadata();
- if (AATags)
- NewSI->setAAMetadata(AATags.merge(OtherStore->getAAMetadata()));
-
- // If the two stores had access groups, intersect them.
- NewSI->setMetadata(LLVMContext::MD_access_group,
- intersectAccessGroups(&SI, OtherStore));
+ combineMetadataForCSE(NewSI, OtherStore, /*DoesKMove=*/true);
+ NewSI->setDebugLoc(MergedLoc);
// Nuke the old stores.
eraseInstFromFunction(SI);
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 075ac5818082f0..51ee1f6fa95087 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -43,6 +43,7 @@
#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/Loads.h"
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/Analysis/PtrUseVisitor.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
@@ -3503,8 +3504,8 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
LoadInst *Load =
IRB.CreateAlignedLoad(NewAllocaTy, &NewAI, NewAI.getAlign(), "load");
- Load->copyMetadata(LI, {LLVMContext::MD_mem_parallel_loop_access,
- LLVMContext::MD_access_group});
+ copyMetadataForMemoryAccess(
+ *Load, LI, int64_t(NewAllocaBeginOffset) - int64_t(BeginOffset));
return extractVector(IRB, Load, BeginIndex, EndIndex, "vec");
}
@@ -3537,8 +3538,6 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
Value *OldOp = LI.getOperand(0);
assert(OldOp == OldPtr);
- AAMDNodes AATags = LI.getAAMetadata();
-
unsigned AS = LI.getPointerAddressSpace();
Type *TargetTy = IsSplit ? Type::getIntNTy(LI.getContext(), SliceSize * 8)
@@ -3564,15 +3563,7 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
if (NewLI->isAtomic())
NewLI->setAlignment(LI.getAlign());
- // Copy any metadata that is valid for the new load. This may require
- // conversion to a different kind of metadata, e.g. !nonnull might change
- // to !range or vice versa.
- copyMetadataForLoad(*NewLI, LI);
-
- // Do this after copyMetadataForLoad() to preserve the TBAA shift.
- if (AATags)
- NewLI->setAAMetadata(AATags.adjustForAccess(
- NewBeginOffset - BeginOffset, NewLI->getType(), DL));
+ copyMetadataForMemoryAccess(*NewLI, LI, NewBeginOffset - BeginOffset);
// Try to preserve nonnull metadata
V = NewLI;
@@ -3594,15 +3585,9 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
IRB.CreateAlignedLoad(TargetTy, getNewAllocaSlicePtr(IRB, LTy),
getSliceAlign(), LI.isVolatile(), LI.getName());
- if (AATags)
- NewLI->setAAMetadata(AATags.adjustForAccess(
- NewBeginOffset - BeginOffset, NewLI->getType(), DL));
-
if (LI.isVolatile())
NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID());
- NewLI->copyMetadata(LI, {LLVMContext::MD_mem_parallel_loop_access,
- LLVMContext::MD_access_group});
-
+ copyMetadataForMemoryAccess(*NewLI, LI, NewBeginOffset - BeginOffset);
V = NewLI;
IsPtrAdjusted = true;
}
@@ -3645,8 +3630,7 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
return !LI.isVolatile() && !IsPtrAdjusted;
}
- bool rewriteVectorizedStoreInst(Value *V, StoreInst &SI, Value *OldOp,
- AAMDNodes AATags) {
+ bool rewriteVectorizedStoreInst(Value *V, StoreInst &SI, Value *OldOp) {
// Capture V for the purpose of debug-info accounting once it's converted
// to a vector store.
Value *OrigV = V;
@@ -3669,11 +3653,7 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
V = insertVector(IRB, Old, V, BeginIndex, "vec");
}
StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlign());
- Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
- LLVMContext::MD_access_group});
- if (AATags)
- Store->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
- V->getType(), DL));
+ copyMetadataForMemoryAccess(*Store, SI, NewBeginOffset - BeginOffset);
Pass.DeadInsts.push_back(&SI);
// NOTE: Careful to use OrigV rather than V.
@@ -3683,7 +3663,7 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
return true;
}
- bool rewriteIntegerStore(Value *V, StoreInst &SI, AAMDNodes AATags) {
+ bool rewriteIntegerStore(Value *V, StoreInst &SI) {
assert(IntTy && "We cannot extract an integer from the alloca");
assert(!SI.isVolatile());
if (DL.getTypeSizeInBits(V->getType()).getFixedValue() !=
@@ -3697,11 +3677,7 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
}
V = IRB.CreateBitPreservingCastChain(DL, V, NewAllocaTy);
StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlign());
- Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
- LLVMContext::MD_access_group});
- if (AATags)
- Store->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
- V->getType(), DL));
+ copyMetadataForMemoryAccess(*Store, SI, NewBeginOffset - BeginOffset);
migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &SI,
Store, Store->getPointerOperand(),
@@ -3717,7 +3693,6 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
Value *OldOp = SI.getOperand(1);
assert(OldOp == OldPtr);
- AAMDNodes AATags = SI.getAAMetadata();
Value *V = SI.getValueOperand();
// Strip all inbounds GEPs and pointer casts to try to dig out any root
@@ -3739,9 +3714,9 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
}
if (VecTy)
- return rewriteVectorizedStoreInst(V, SI, OldOp, AATags);
+ return rewriteVectorizedStoreInst(V, SI, OldOp);
if (IntTy && V->getType()->isIntegerTy())
- return rewriteIntegerStore(V, SI, AATags);
+ return rewriteIntegerStore(V, SI);
StoreInst *NewSI;
if (NewBeginOffset == NewAllocaBeginOffset &&
@@ -3759,11 +3734,7 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
NewSI =
IRB.CreateAlignedStore(V, NewPtr, getSliceAlign(), SI.isVolatile());
}
- NewSI->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
- LLVMContext::MD_access_group});
- if (AATags)
- NewSI->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
- V->getType(), DL));
+ copyMetadataForMemoryAccess(*NewSI, SI, NewBeginOffset - BeginOffset);
if (SI.isVolatile())
NewSI->setAtomic(SI.getOrdering(), SI.getSyncScopeID());
if (NewSI->isAtomic())
@@ -3942,11 +3913,7 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
Value *NewPtr = getPtrToNewAI(II.getDestAddressSpace(), II.isVolatile());
StoreInst *New =
IRB.CreateAlignedStore(V, NewPtr, NewAI.getAlign(), II.isVolatile());
- New->copyMetadata(II, {LLVMContext::MD_mem_parallel_loop_access,
- LLVMContext::MD_access_group});
- if (AATags)
- New->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
- V->getType(), DL));
+ copyMetadataForMemoryAccess(*New, II, NewBeginOffset - BeginOffset);
migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &II,
New, New->getPointerOperand(), V, DL);
@@ -4144,11 +4111,7 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
} else {
LoadInst *Load = IRB.CreateAlignedLoad(OtherTy, SrcPtr, SrcAlign,
II.isVolatile(), "copyload");
- Load->copyMetadata(II, {LLVMContext::MD_mem_parallel_loop_access,
- LLVMContext::MD_access_group});
- if (AATags)
- Load->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
- Load->getType(), DL));
+ copyMetadataForMemoryAccess(*Load, II, NewBeginOffset - BeginOffset);
Src = Load;
}
@@ -4167,11 +4130,7 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
StoreInst *Store = cast<StoreInst>(
IRB.CreateAlignedStore(Src, DstPtr, DstAlign, II.isVolatile()));
- Store->copyMetadata(II, {LLVMContext::MD_mem_parallel_loop_access,
- LLVMContext::MD_access_group});
- if (AATags)
- Store->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
- Src->getType(), DL));
+ copyMetadataForMemoryAccess(*Store, II, NewBeginOffset - BeginOffset);
APInt Offset(DL.getIndexTypeSizeInBits(DstPtr->getType()), 0);
if (IsDest) {
@@ -4455,7 +4414,7 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
};
struct LoadOpSplitter : public OpSplitter<LoadOpSplitter> {
- AAMDNodes AATags;
+ LoadInst *AggLoad;
// A vector to hold the split components that we want to emit
// separate fake uses for.
SmallVector<Value *, 4> Components;
@@ -4464,11 +4423,11 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
SmallVector<Instruction *, 1> FakeUses;
LoadOpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
- AAMDNodes AATags, Align BaseAlign, const DataLayout &DL,
+ LoadInst *AggLoad, Align BaseAlign, const DataLayout &DL,
IRBuilderTy &IRB)
: OpSplitter<LoadOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign, DL,
IRB),
- AATags(AATags) {}
+ AggLoad(AggLoad) {}
/// Emit a leaf load of a single value. This is called at the leaves of the
/// recursive emission to actually load values.
@@ -4482,10 +4441,12 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
APInt Offset(
DL.getIndexSizeInBits(Ptr->getType()->getPointerAddressSpace()), 0);
- if (AATags &&
- GEPOperator::accumulateConstantOffset(BaseTy, GEPIndices, DL, Offset))
- Load->setAAMetadata(
- AATags.adjustForAccess(Offset.getZExtValue(), Load->getType(), DL));
+ bool KnownOffset =
+ GEPOperator::accumulateConstantOffset(BaseTy, GEPIndices, DL, Offset);
+ copyMetadataForMemoryAccess(
+ *Load, *AggLoad,
+ KnownOffset ? std::optional<int64_t>(Offset.getZExtValue())
+ : std::nullopt);
// Record the load so we can generate a fake use for this aggregate
// component.
Components.push_back(Load);
@@ -4521,7 +4482,7 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
// We have an aggregate being loaded, split it apart.
LLVM_DEBUG(dbgs() << " original: " << LI << "\n");
- LoadOpSplitter Splitter(&LI, *U, LI.getType(), LI.getAAMetadata(),
+ LoadOpSplitter Splitter(&LI, *U, LI.getType(), &LI,
getAdjustedAlignment(&LI, 0), DL, IRB);
Splitter.recordFakeUses(LI);
Value *V = PoisonValue::get(LI.getType());
@@ -4535,12 +4496,11 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
struct StoreOpSplitter : public OpSplitter<StoreOpSplitter> {
StoreOpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
- AAMDNodes AATags, StoreInst *AggStore, Align BaseAlign,
- const DataLayout &DL, IRBuilderTy &IRB)
+ StoreInst *AggStore, Align BaseAlign, const DataLayout &DL,
+ IRBuilderTy &IRB)
: OpSplitter<StoreOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign,
DL, IRB),
- AATags(AATags), AggStore(AggStore) {}
- AAMDNodes AATags;
+ AggStore(AggStore) {}
StoreInst *AggStore;
/// Emit a leaf store of a single value. This is called at the leaves of the
/// recursive emission to actually produce stores.
@@ -4560,10 +4520,7 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
APInt Offset(
DL.getIndexSizeInBits(Ptr->getType()->getPointerAddressSpace()), 0);
GEPOperator::accumulateConstantOffset(BaseTy, GEPIndices, DL, Offset);
- if (AATags) {
- Store->setAAMetadata(AATags.adjustForAccess(
- Offset.getZExtValue(), ExtractValue->getType(), DL));
- }
+ copyMetadataForMemoryAccess(*Store, *AggStore, Offset.getZExtValue());
// migrateDebugInfo requires the base Alloca. Walk to it from this gep.
// If we cannot (because there's an intervening non-const or unbounded
@@ -4595,7 +4552,7 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
// We have an aggregate being stored, split it apart.
LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
- StoreOpSplitter Splitter(&SI, *U, V->getType(), SI.getAAMetadata(), &SI,
+ StoreOpSplitter Splitter(&SI, *U, V->getType(), &SI,
getAdjustedAlignment(&SI, 0), DL, IRB);
Splitter.emitSplitOps(V->getType(), V, V->getName() + ".fca");
Visited.erase(&SI);
@@ -5043,8 +5000,7 @@ class SliceSplitter {
PartPtrTy, BasePtr->getName() + "."),
getAdjustedAlignment(SplitI, It.getOffset()),
/*IsVolatile*/ false, SplitI->getName());
- PLoad->copyMetadata(*SplitI, {LLVMContext::MD_mem_parallel_loop_access,
- LLVMContext::MD_access_group});
+ copyMetadataForMemoryAccess(*PLoad, *SplitI, It.getOffset());
return PLoad;
}
@@ -5059,13 +5015,8 @@ class SliceSplitter {
PartPtrTy, BasePtr->getName() + "."),
getAdjustedAlignment(SplitI, It.getOffset()),
/*IsVolatile*/ false);
- PStore->copyMetadata(*SplitI, {LLVMContext::MD_mem_parallel_loop_access,
- LLVMContext::MD_access_group,
- LLVMContext::MD_DIAssignID});
- AAMDNodes AATags = SplitI->getAAMetadata();
- if (AATags)
- PStore->setAAMetadata(
- AATags.adjustForAccess(It.getOffset(), StoreVal->getType(), DL));
+ copyMetadataForMemoryAccess(*PStore, *SplitI, It.getOffset());
+ PStore->copyMetadata(*SplitI, LLVMContext::MD_DIAssignID);
return PStore;
}
diff --git a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
index d8fd2603dd1f3f..d8d07d7b09f83c 100644
--- a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
+++ b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
@@ -16,6 +16,7 @@
#include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/DomTreeUpdater.h"
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/BasicBlock.h"
@@ -113,61 +114,6 @@ static unsigned adjustForEndian(const DataLayout &DL, unsigned VectorWidth,
return DL.isBigEndian() ? VectorWidth - 1 - Idx : Idx;
}
-static void copyMetadataForMemoryAccess(
- Instruction &Dest, const Instruction &Source, const DataLayout &DL,
- unsigned SourcePtrOperand, unsigned DestPtrOperand, Type *AccessType,
- bool IsWholeAccess, std::optional<size_t> ByteOffset) {
- // Only propagate metadata that is valid on each constituent memory access.
- // In particular, do not copy metadata whose meaning is tied to the call,
- // such as !prof or !callsite.
- Dest.copyMetadata(Source,
- {LLVMContext::MD_nontemporal,
- LLVMContext::MD_mem_parallel_loop_access,
- LLVMContext::MD_access_group, LLVMContext::MD_annotation,
- LLVMContext::MD_nosanitize, LLVMContext::MD_mmra});
-
- AAMDNodes AANodes = Source.getAAMetadata();
- if (IsWholeAccess)
- Dest.setAAMetadata(AANodes);
- else if (ByteOffset)
- Dest.setAAMetadata(AANodes.adjustForAccess(*ByteOffset, AccessType, DL));
- else {
- // The packed address is runtime-dependent. The other AA metadata remains
- // applicable, but !tbaa.struct cannot be adjusted to a known byte range.
- AANodes.TBAAStruct = nullptr;
- Dest.setAAMetadata(AANodes);
- }
- setMemCacheHint(Dest, DestPtrOperand,
- getMemCacheHint(Source, SourcePtrOperand));
-}
-
-static void copyMetadataForScalarizedLoad(LoadInst &Dest,
- const Instruction &Source,
- const DataLayout &DL,
- unsigned SourcePtrOperand,
- std::optional<size_t> ByteOffset) {
- copyMetadataForMemoryAccess(Dest, Source, DL, SourcePtrOperand,
- Dest.getPointerOperandIndex(), Dest.getType(),
- Dest.getType() == Source.getType(), ByteOffset);
-
- // !range applies element-wise to vectors, so the same range describes each
- // scalar result. The other metadata here also describes the loaded result.
- Dest.copyMetadata(Source, {LLVMContext::MD_fpmath, LLVMContext::MD_range,
- LLVMContext::MD_invariant_load});
-}
-
-static void copyMetadataForScalarizedStore(StoreInst &Dest,
- const Instruction &Source,
- const DataLayout &DL,
- unsigned SourcePtrOperand,
- std::optional<size_t> ByteOffset) {
- copyMetadataForMemoryAccess(
- Dest, Source, DL, SourcePtrOperand, Dest.getPointerOperandIndex(),
- Dest.getValueOperand()->getType(),
- Dest.getValueOperand()->getType() == Source.getOperand(0)->getType(),
- ByteOffset);
-}
-
// Translate a masked load intrinsic like
// <16 x i32 > @llvm.masked.load( <16 x i32>* %addr,
// <16 x i1> %mask, <16 x i32> %passthru)
@@ -220,8 +166,7 @@ static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence,
// Short-cut if the mask is all-true.
if (isa<Constant>(Mask) && cast<Constant>(Mask)->isAllOnesValue()) {
LoadInst *NewI = Builder.CreateAlignedLoad(VecType, Ptr, AlignVal);
- copyMetadataForScalarizedLoad(*NewI, *CI, DL, /*SourcePtrOperand=*/0,
- std::nullopt);
+ copyMetadataForMemoryAccess(*NewI, *CI);
NewI->takeName(CI);
CI->replaceAllUsesWith(NewI);
CI->eraseFromParent();
@@ -242,9 +187,8 @@ static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence,
continue;
Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Gep, AdjustedAlignVal);
- copyMetadataForScalarizedLoad(
- *Load, *CI, DL, /*SourcePtrOperand=*/0,
- Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
+ copyMetadataForMemoryAccess(
+ *Load, *CI, Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
VResult = Builder.CreateInsertElement(VResult, Load, Idx);
}
CI->replaceAllUsesWith(VResult);
@@ -271,8 +215,7 @@ static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence,
Builder.SetInsertPoint(CondBlock->getTerminator());
LoadInst *Load = Builder.CreateAlignedLoad(VecType, Ptr, AlignVal,
CI->getName() + ".cond.load");
- copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
- std::nullopt);
+ copyMetadataForMemoryAccess(*Load, *CI);
BasicBlock *PostLoad = ThenTerm->getSuccessor(0);
Builder.SetInsertPoint(PostLoad->begin());
@@ -335,9 +278,8 @@ static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence,
Builder.SetInsertPoint(CondBlock->getTerminator());
Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Gep, AdjustedAlignVal);
- copyMetadataForScalarizedLoad(
- *Load, *CI, DL, /*SourcePtrOperand=*/0,
- Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
+ copyMetadataForMemoryAccess(
+ *Load, *CI, Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
Value *NewVResult = Builder.CreateInsertElement(VResult, Load, Idx);
// Create "else" block, fill it in the next iteration
@@ -406,8 +348,7 @@ static void scalarizeMaskedStore(const DataLayout &DL, bool HasBranchDivergence,
if (isa<Constant>(Mask) && cast<Constant>(Mask)->isAllOnesValue()) {
StoreInst *Store = Builder.CreateAlignedStore(Src, Ptr, AlignVal);
Store->takeName(CI);
- copyMetadataForScalarizedStore(*Store, *CI, DL, /*SourcePtrOperand=*/1,
- std::nullopt);
+ copyMetadataForMemoryAccess(*Store, *CI);
// This is a one-to-one replacement, so the assignment link remains valid.
Store->copyMetadata(*CI, LLVMContext::MD_DIAssignID);
CI->eraseFromParent();
@@ -427,9 +368,8 @@ static void scalarizeMaskedStore(const DataLayout &DL, bool HasBranchDivergence,
Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
StoreInst *Store =
Builder.CreateAlignedStore(OneElt, Gep, AdjustedAlignVal);
- copyMetadataForScalarizedStore(
- *Store, *CI, DL, /*SourcePtrOperand=*/1,
- Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
+ copyMetadataForMemoryAccess(
+ *Store, *CI, Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
}
CI->eraseFromParent();
return;
@@ -454,8 +394,7 @@ static void scalarizeMaskedStore(const DataLayout &DL, bool HasBranchDivergence,
StoreInst *Store = Builder.CreateAlignedStore(Src, Ptr, AlignVal);
Store->takeName(CI);
- copyMetadataForScalarizedStore(*Store, *CI, DL, /*SourcePtrOperand=*/1,
- std::nullopt);
+ copyMetadataForMemoryAccess(*Store, *CI);
// This is a one-to-one replacement, so the assignment link remains valid.
Store->copyMetadata(*CI, LLVMContext::MD_DIAssignID);
@@ -515,9 +454,8 @@ static void scalarizeMaskedStore(const DataLayout &DL, bool HasBranchDivergence,
Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
StoreInst *Store =
Builder.CreateAlignedStore(OneElt, Gep, AdjustedAlignVal);
- copyMetadataForScalarizedStore(
- *Store, *CI, DL, /*SourcePtrOperand=*/1,
- Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
+ copyMetadataForMemoryAccess(
+ *Store, *CI, Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
// Create "else" block, fill it in the next iteration
BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
@@ -586,8 +524,7 @@ static void scalarizeMaskedGather(const DataLayout &DL,
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
LoadInst *Load =
Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx));
- copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
- /*ByteOffset=*/0);
+ copyMetadataForMemoryAccess(*Load, *CI, /*ByteOffset=*/0);
VResult =
Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx));
}
@@ -647,8 +584,7 @@ static void scalarizeMaskedGather(const DataLayout &DL,
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
LoadInst *Load =
Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx));
- copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
- /*ByteOffset=*/0);
+ copyMetadataForMemoryAccess(*Load, *CI, /*ByteOffset=*/0);
Value *NewVResult =
Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx));
@@ -727,9 +663,7 @@ static void scalarizeMaskedScatter(const DataLayout &DL,
Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
StoreInst *Store = Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
- copyMetadataForScalarizedStore(*Store, *CI, DL,
- /*SourcePtrOperand=*/1,
- /*ByteOffset=*/0);
+ copyMetadataForMemoryAccess(*Store, *CI, /*ByteOffset=*/0);
}
CI->eraseFromParent();
return;
@@ -784,9 +718,7 @@ static void scalarizeMaskedScatter(const DataLayout &DL,
Value *OneElt = Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
StoreInst *Store = Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
- copyMetadataForScalarizedStore(*Store, *CI, DL,
- /*SourcePtrOperand=*/1,
- /*ByteOffset=*/0);
+ copyMetadataForMemoryAccess(*Store, *CI, /*ByteOffset=*/0);
// Create "else" block, fill it in the next iteration
BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
@@ -842,9 +774,8 @@ static void scalarizeMaskedExpandLoad(const DataLayout &DL,
Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, MemIndex);
LoadInst *Load = Builder.CreateAlignedLoad(
EltTy, NewPtr, AdjustedAlignment, "Load" + Twine(Idx));
- copyMetadataForScalarizedLoad(
- *Load, *CI, DL, /*SourcePtrOperand=*/0,
- MemIndex * DL.getTypeAllocSize(EltTy).getFixedValue());
+ copyMetadataForMemoryAccess(
+ *Load, *CI, MemIndex * DL.getTypeAllocSize(EltTy).getFixedValue());
InsertElt = Load;
ShuffleMask[Idx] = Idx;
++MemIndex;
@@ -907,8 +838,7 @@ static void scalarizeMaskedExpandLoad(const DataLayout &DL,
Builder.SetInsertPoint(CondBlock->getTerminator());
LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Ptr, AdjustedAlignment);
- copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
- std::nullopt);
+ copyMetadataForMemoryAccess(*Load, *CI, std::nullopt);
Value *NewVResult = Builder.CreateInsertElement(VResult, Load, Idx);
// Move the pointer if there are more blocks to come.
@@ -979,9 +909,8 @@ static void scalarizeMaskedCompressStore(const DataLayout &DL,
Value *NewPtr = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, MemIndex);
StoreInst *Store =
Builder.CreateAlignedStore(OneElt, NewPtr, AdjustedAlignment);
- copyMetadataForScalarizedStore(
- *Store, *CI, DL, /*SourcePtrOperand=*/1,
- MemIndex * DL.getTypeAllocSize(EltTy).getFixedValue());
+ copyMetadataForMemoryAccess(
+ *Store, *CI, MemIndex * DL.getTypeAllocSize(EltTy).getFixedValue());
++MemIndex;
}
CI->eraseFromParent();
@@ -1037,8 +966,7 @@ static void scalarizeMaskedCompressStore(const DataLayout &DL,
Value *OneElt = Builder.CreateExtractElement(Src, Idx);
StoreInst *Store =
Builder.CreateAlignedStore(OneElt, Ptr, AdjustedAlignment);
- copyMetadataForScalarizedStore(*Store, *CI, DL, /*SourcePtrOperand=*/1,
- std::nullopt);
+ copyMetadataForMemoryAccess(*Store, *CI, std::nullopt);
// Move the pointer if there are more blocks to come.
Value *NewPtr;
@@ -1115,14 +1043,11 @@ static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI,
continue;
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
LoadInst *Load = Builder.CreateLoad(EltTy, Ptr, "Load" + Twine(Idx));
- copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
- /*ByteOffset=*/0);
+ copyMetadataForMemoryAccess(*Load, *CI, /*ByteOffset=*/0);
Value *Update =
CreateHistogramUpdateValue(cast<IntrinsicInst>(CI), Load, Inc);
StoreInst *Store = Builder.CreateStore(Update, Ptr);
- copyMetadataForScalarizedStore(*Store, *CI, DL,
- /*SourcePtrOperand=*/0,
- /*ByteOffset=*/0);
+ copyMetadataForMemoryAccess(*Store, *CI, /*ByteOffset=*/0);
}
CI->eraseFromParent();
return;
@@ -1146,14 +1071,11 @@ static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI,
Builder.SetInsertPoint(CondBlock->getTerminator());
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
LoadInst *Load = Builder.CreateLoad(EltTy, Ptr, "Load" + Twine(Idx));
- copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
- /*ByteOffset=*/0);
+ copyMetadataForMemoryAccess(*Load, *CI, /*ByteOffset=*/0);
Value *UpdateOp =
CreateHistogramUpdateValue(cast<IntrinsicInst>(CI), Load, Inc);
StoreInst *Store = Builder.CreateStore(UpdateOp, Ptr);
- copyMetadataForScalarizedStore(*Store, *CI, DL,
- /*SourcePtrOperand=*/0,
- /*ByteOffset=*/0);
+ copyMetadataForMemoryAccess(*Store, *CI, /*ByteOffset=*/0);
// Create "else" block, fill it in the next iteration
BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
diff --git a/llvm/lib/Transforms/Scalar/Scalarizer.cpp b/llvm/lib/Transforms/Scalar/Scalarizer.cpp
index 5af07a4400aa98..0468786e635aef 100644
--- a/llvm/lib/Transforms/Scalar/Scalarizer.cpp
+++ b/llvm/lib/Transforms/Scalar/Scalarizer.cpp
@@ -18,6 +18,7 @@
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Twine.h"
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Argument.h"
@@ -310,7 +311,6 @@ class ScalarizerVisitor : public InstVisitor<ScalarizerVisitor, bool> {
Scatterer scatter(Instruction *Point, Value *V, const VectorSplit &VS);
void gather(Instruction *Op, const ValueVector &CV, const VectorSplit &VS);
void replaceUses(Instruction *Op, Value *CV);
- bool canTransferMetadata(unsigned Kind);
void transferMetadataAndIRFlags(Instruction *Op, const ValueVector &CV);
std::optional<VectorSplit> getVectorSplit(Type *Ty);
std::optional<VectorLayout> getVectorLayout(Type *Ty, Align Alignment,
@@ -543,33 +543,16 @@ void ScalarizerVisitor::replaceUses(Instruction *Op, Value *CV) {
}
}
-// Return true if it is safe to transfer the given metadata tag from
-// vector to scalar instructions.
-bool ScalarizerVisitor::canTransferMetadata(unsigned Tag) {
- return (Tag == LLVMContext::MD_tbaa
- || Tag == LLVMContext::MD_fpmath
- || Tag == LLVMContext::MD_tbaa_struct
- || Tag == LLVMContext::MD_invariant_load
- || Tag == LLVMContext::MD_alias_scope
- || Tag == LLVMContext::MD_noalias
- || Tag == LLVMContext::MD_mem_parallel_loop_access
- || Tag == LLVMContext::MD_access_group);
-}
-
-// Transfer metadata from Op to the instructions in CV if it is known
-// to be safe to do so.
+// Transfer metadata and flags to each scalarized fragment.
void ScalarizerVisitor::transferMetadataAndIRFlags(Instruction *Op,
const ValueVector &CV) {
- SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
- Op->getAllMetadataOtherThanDebugLoc(MDs);
+ uint64_t Offset = 0;
for (Value *V : CV) {
- if (Instruction *New = dyn_cast<Instruction>(V)) {
- for (const auto &MD : MDs)
- if (canTransferMetadata(MD.first))
- New->setMetadata(MD.first, MD.second);
+ if (auto *New = dyn_cast<Instruction>(V)) {
+ copyMetadataForScalarizedInstruction(*New, *Op, Offset);
New->copyIRFlags(Op);
- if (Op->getDebugLoc() && !New->getDebugLoc())
- New->setDebugLoc(Op->getDebugLoc());
+ if (isa<LoadInst, StoreInst>(New))
+ Offset += Op->getDataLayout().getTypeStoreSize(getLoadStoreType(New));
}
}
}
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index 8b6d53483dca04..b98f7dcdf1b8e5 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -27,6 +27,7 @@
#include "llvm/Analysis/InstructionSimplify.h"
#include "llvm/Analysis/MemoryBuiltins.h"
#include "llvm/Analysis/MemorySSAUpdater.h"
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
@@ -2990,262 +2991,6 @@ bool llvm::removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU,
return Changed;
}
-/// If AAOnly is set, only intersect alias analysis metadata and preserve other
-/// known metadata. Unknown metadata is always dropped.
-static void combineMetadata(Instruction *K, const Instruction *J,
- bool DoesKMove, bool AAOnly = false) {
- SmallVector<std::pair<unsigned, MDNode *>, 4> Metadata;
- K->getAllMetadataOtherThanDebugLoc(Metadata);
- for (const auto &MD : Metadata) {
- unsigned Kind = MD.first;
- MDNode *JMD = J->getMetadata(Kind);
- MDNode *KMD = MD.second;
-
- // TODO: Assert that this switch is exhaustive for fixed MD kinds.
- switch (Kind) {
- default:
- K->setMetadata(Kind, nullptr); // Remove unknown metadata
- break;
- case LLVMContext::MD_dbg:
- llvm_unreachable("getAllMetadataOtherThanDebugLoc returned a MD_dbg");
- case LLVMContext::MD_DIAssignID:
- if (!AAOnly)
- K->mergeDIAssignID(J);
- break;
- case LLVMContext::MD_tbaa:
- if (DoesKMove)
- K->setMetadata(Kind, MDNode::getMostGenericTBAA(JMD, KMD));
- break;
- case LLVMContext::MD_alias_scope:
- if (DoesKMove)
- K->setMetadata(Kind, MDNode::getMostGenericAliasScope(JMD, KMD));
- break;
- case LLVMContext::MD_noalias:
- case LLVMContext::MD_mem_parallel_loop_access:
- if (DoesKMove)
- K->setMetadata(Kind, MDNode::intersect(JMD, KMD));
- break;
- case LLVMContext::MD_access_group:
- if (DoesKMove)
- K->setMetadata(LLVMContext::MD_access_group,
- intersectAccessGroups(K, J));
- break;
- case LLVMContext::MD_range:
- if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
- K->setMetadata(Kind, MDNode::getMostGenericRange(JMD, KMD));
- break;
- case LLVMContext::MD_nofpclass:
- if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
- K->setMetadata(Kind, MDNode::getMostGenericNoFPClass(JMD, KMD));
- break;
- case LLVMContext::MD_fpmath:
- if (!AAOnly)
- K->setMetadata(Kind, MDNode::getMostGenericFPMath(JMD, KMD));
- break;
- case LLVMContext::MD_invariant_load:
- case LLVMContext::MD_invariant_group:
- // If K moves, only keep the invariant metadata if it is present on
- // both instructions; otherwise the invariant would be asserted on a
- // path (J's) that never promised it. If K does not move, K stays on
- // its original path, so its existing metadata remains valid.
- if (DoesKMove)
- K->setMetadata(Kind, JMD);
- break;
- case LLVMContext::MD_nonnull:
- if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
- K->setMetadata(Kind, JMD);
- break;
- // Keep empty cases for prof, mmra, memprof, and callsite to prevent them
- // from being removed as unknown metadata. The actual merging is handled
- // separately below.
- case LLVMContext::MD_prof:
- case LLVMContext::MD_mmra:
- case LLVMContext::MD_memprof:
- case LLVMContext::MD_callsite:
- break;
- case LLVMContext::MD_callee_type:
- if (!AAOnly) {
- K->setMetadata(LLVMContext::MD_callee_type,
- MDNode::getMergedCalleeTypeMetadata(KMD, JMD));
- }
- break;
- case LLVMContext::MD_callees:
- // If K moves, it replaces J on J's path and must allow J's callees as
- // well. If K does not move, its callees remain valid.
- if (!AAOnly && DoesKMove)
- K->setMetadata(Kind, MDNode::getMergedCalleesMetadata(KMD, JMD));
- break;
- case LLVMContext::MD_align:
- if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
- K->setMetadata(
- Kind, MDNode::getMostGenericAlignmentOrDereferenceable(JMD, KMD));
- break;
- case LLVMContext::MD_dereferenceable:
- case LLVMContext::MD_dereferenceable_or_null:
- if (!AAOnly && DoesKMove)
- K->setMetadata(Kind,
- MDNode::getMostGenericAlignmentOrDereferenceable(JMD, KMD));
- break;
- case LLVMContext::MD_preserve_access_index:
- // Preserve !preserve.access.index in K.
- break;
- case LLVMContext::MD_noundef:
- // If K does move, keep noundef if it is present in both instructions.
- if (!AAOnly && DoesKMove)
- K->setMetadata(Kind, JMD);
- break;
- case LLVMContext::MD_nontemporal:
- // Preserve !nontemporal if it is present on both instructions.
- if (!AAOnly)
- K->setMetadata(Kind, JMD);
- break;
- case LLVMContext::MD_mem_cache_hint:
- // Preserve !mem.cache_hint only if it is present and equivalent on both
- // instructions.
- if (!AAOnly && KMD != JMD)
- K->setMetadata(Kind, nullptr);
- break;
- case LLVMContext::MD_noalias_addrspace:
- if (DoesKMove)
- K->setMetadata(Kind,
- MDNode::getMostGenericNoaliasAddrspace(JMD, KMD));
- break;
- case LLVMContext::MD_nosanitize:
- // Preserve !nosanitize if both K and J have it.
- K->setMetadata(Kind, JMD);
- break;
- case LLVMContext::MD_captures:
- K->setMetadata(
- Kind, MDNode::fromCaptureComponents(
- K->getContext(), MDNode::toCaptureComponents(JMD) |
- MDNode::toCaptureComponents(KMD)));
- break;
- case LLVMContext::MD_alloc_token:
- if (!AAOnly && KMD != JMD)
- K->setMetadata(Kind, MDNode::getMergedAllocTokenMetadata(KMD, JMD));
- break;
- }
- }
-
- // Merge MMRAs.
- // This is handled separately because we also want to handle cases where K
- // doesn't have tags but J does.
- auto JMMRA = J->getMetadata(LLVMContext::MD_mmra);
- auto KMMRA = K->getMetadata(LLVMContext::MD_mmra);
- if (JMMRA || KMMRA) {
- K->setMetadata(LLVMContext::MD_mmra,
- MMRAMetadata::combine(K->getContext(), JMMRA, KMMRA));
- }
-
- // Merge memprof metadata.
- // Handle separately to support cases where only one instruction has the
- // metadata.
- auto *JMemProf = J->getMetadata(LLVMContext::MD_memprof);
- auto *KMemProf = K->getMetadata(LLVMContext::MD_memprof);
- if (!AAOnly && (JMemProf || KMemProf)) {
- K->setMetadata(LLVMContext::MD_memprof,
- MDNode::getMergedMemProfMetadata(KMemProf, JMemProf));
- }
-
- // Merge callsite metadata.
- // Handle separately to support cases where only one instruction has the
- // metadata.
- auto *JCallSite = J->getMetadata(LLVMContext::MD_callsite);
- auto *KCallSite = K->getMetadata(LLVMContext::MD_callsite);
- if (!AAOnly && (JCallSite || KCallSite)) {
- K->setMetadata(LLVMContext::MD_callsite,
- MDNode::getMergedCallsiteMetadata(KCallSite, JCallSite));
- }
-
- // Merge prof metadata.
- // Handle separately to support cases where only one instruction has the
- // metadata.
- auto *JProf = J->getMetadata(LLVMContext::MD_prof);
- auto *KProf = K->getMetadata(LLVMContext::MD_prof);
- if (!AAOnly && (JProf || KProf)) {
- // We do not need to merge metadata if we have select instructions and the
- // one we're keeping (K) does not move (explicitly merging would drop the
- // information in this case). Otherwise, explicitly handle the merging.
- if (DoesKMove || (!isa<SelectInst>(J) || !isa<SelectInst>(K)) ||
- ProfcheckDisableMetadataFixes)
- K->setMetadata(LLVMContext::MD_prof,
- MDNode::getMergedProfMetadata(KProf, JProf, K, J));
- }
-}
-
-void llvm::combineMetadataForCSE(Instruction *K, const Instruction *J,
- bool DoesKMove) {
- combineMetadata(K, J, DoesKMove);
-}
-
-void llvm::combineAAMetadata(Instruction *K, const Instruction *J) {
- combineMetadata(K, J, /*DoesKMove=*/true, /*AAOnly=*/true);
-}
-
-void llvm::copyMetadataForLoad(LoadInst &Dest, const LoadInst &Source) {
- SmallVector<std::pair<unsigned, MDNode *>, 8> MD;
- Source.getAllMetadata(MD);
- MDBuilder MDB(Dest.getContext());
- Type *NewType = Dest.getType();
- const DataLayout &DL = Source.getDataLayout();
- for (const auto &MDPair : MD) {
- unsigned ID = MDPair.first;
- MDNode *N = MDPair.second;
- // Note, essentially every kind of metadata should be preserved here! This
- // routine is supposed to clone a load instruction changing *only its type*.
- // The only metadata it makes sense to drop is metadata which is invalidated
- // when the pointer type changes. This should essentially never be the case
- // in LLVM, but we explicitly switch over only known metadata to be
- // conservatively correct. If you are adding metadata to LLVM which pertains
- // to loads, you almost certainly want to add it here.
- switch (ID) {
- case LLVMContext::MD_dbg:
- case LLVMContext::MD_tbaa:
- case LLVMContext::MD_prof:
- case LLVMContext::MD_fpmath:
- case LLVMContext::MD_tbaa_struct:
- case LLVMContext::MD_invariant_load:
- case LLVMContext::MD_alias_scope:
- case LLVMContext::MD_noalias:
- case LLVMContext::MD_nontemporal:
- case LLVMContext::MD_mem_cache_hint:
- case LLVMContext::MD_mem_parallel_loop_access:
- case LLVMContext::MD_access_group:
- case LLVMContext::MD_noundef:
- case LLVMContext::MD_noalias_addrspace:
- case LLVMContext::MD_invariant_group:
- case LLVMContext::MD_mmra:
- // All of these directly apply.
- Dest.setMetadata(ID, N);
- break;
-
- case LLVMContext::MD_nonnull:
- copyNonnullMetadata(Source, N, Dest);
- break;
-
- case LLVMContext::MD_align:
- case LLVMContext::MD_dereferenceable:
- case LLVMContext::MD_dereferenceable_or_null:
- // These only directly apply if the new type is also a pointer.
- if (NewType->isPointerTy())
- Dest.setMetadata(ID, N);
- break;
-
- case LLVMContext::MD_range:
- copyRangeMetadata(DL, Source, N, Dest);
- break;
-
- case LLVMContext::MD_nofpclass:
- // This only applies if the floating-point type interpretation. This
- // should handle degenerate cases like casting between a scalar and single
- // element vector.
- if (NewType->getScalarType() == Source.getType()->getScalarType())
- Dest.setMetadata(ID, N);
- break;
- }
- }
-}
-
void llvm::patchReplacementInstruction(Instruction *I, Value *Repl,
bool DoesKMove) {
auto *ReplInst = dyn_cast<Instruction>(Repl);
@@ -3398,55 +3143,6 @@ bool llvm::callsGCLeafFunction(const CallBase *Call,
return TLI.has(TLI.getLibFunc(*Call));
}
-void llvm::copyNonnullMetadata(const LoadInst &OldLI, MDNode *N,
- LoadInst &NewLI) {
- auto *NewTy = NewLI.getType();
-
- // This only directly applies if the new type is also a pointer.
- if (NewTy->isPointerTy()) {
- NewLI.setMetadata(LLVMContext::MD_nonnull, N);
- return;
- }
-
- // The only other translation we can do is to integral loads with !range
- // metadata.
- if (!NewTy->isIntegerTy())
- return;
-
- MDBuilder MDB(NewLI.getContext());
- const Value *Ptr = OldLI.getPointerOperand();
- auto *ITy = cast<IntegerType>(NewTy);
- auto *NullInt = ConstantExpr::getPtrToInt(
- ConstantPointerNull::get(cast<PointerType>(Ptr->getType())), ITy);
- auto *NonNullInt = ConstantExpr::getAdd(NullInt, ConstantInt::get(ITy, 1));
- NewLI.setMetadata(LLVMContext::MD_range,
- MDB.createRange(NonNullInt, NullInt));
-}
-
-void llvm::copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI,
- MDNode *N, LoadInst &NewLI) {
- auto *NewTy = NewLI.getType();
- // Simply copy the metadata if the type did not change.
- if (NewTy == OldLI.getType()) {
- NewLI.setMetadata(LLVMContext::MD_range, N);
- return;
- }
-
- // Give up unless it is converted to a pointer where there is a single very
- // valuable mapping we can do reliably.
- // FIXME: It would be nice to propagate this in more ways, but the type
- // conversions make it hard.
- if (!NewTy->isPointerTy())
- return;
-
- unsigned BitWidth = DL.getPointerTypeSizeInBits(NewTy);
- if (BitWidth == OldLI.getType()->getScalarSizeInBits() &&
- !getConstantRangeFromMetadata(*N).contains(APInt(BitWidth, 0))) {
- MDNode *NN = MDNode::get(OldLI.getContext(), {});
- NewLI.setMetadata(LLVMContext::MD_nonnull, NN);
- }
-}
-
void llvm::dropDebugUsers(Instruction &I) {
SmallVector<DbgVariableRecord *, 1> DPUsers;
findDbgUsers(&I, DPUsers);
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index f301b931d675aa..dff031f71773f6 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -72,6 +72,7 @@
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/MemoryLocation.h"
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 50992fc3f89f44..bce9eed740fd69 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -50,6 +50,7 @@
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryLocation.h"
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 5d89141cde6f2e..4a185cbcf2f0c3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -35,6 +35,7 @@
#include "llvm/ADT/ilist_node.h"
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/MemoryLocation.h"
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/DebugLoc.h"
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index ffdef6fbb1e521..bb2d24138d0965 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -24,6 +24,7 @@
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/InstSimplifyFolder.h"
#include "llvm/Analysis/Loads.h"
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/Analysis/TargetFolder.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
@@ -359,7 +360,13 @@ bool VectorCombine::vectorizeLoadInsert(Instruction &I) {
Builder.SetInsertPoint(Load);
Value *CastedPtr =
Builder.CreatePointerBitCastOrAddrSpaceCast(SrcPtr, Builder.getPtrTy(AS));
- Value *VecLd = Builder.CreateAlignedLoad(MinVecTy, CastedPtr, Alignment);
+ auto *NewLoad = Builder.CreateAlignedLoad(MinVecTy, CastedPtr, Alignment);
+ copyMetadataForMemoryAccess(
+ *NewLoad, *Load,
+ SrcPtr == Load->getPointerOperand()->stripPointerCasts()
+ ? std::optional<int64_t>(0)
+ : -int64_t(OffsetEltIndex * (ScalarSize / 8)));
+ Value *VecLd = NewLoad;
VecLd = Builder.CreateShuffleVector(VecLd, Mask);
replaceValue(I, *VecLd);
@@ -424,7 +431,8 @@ bool VectorCombine::widenSubvectorLoad(Instruction &I) {
Builder.SetInsertPoint(Load);
Value *CastedPtr =
Builder.CreatePointerBitCastOrAddrSpaceCast(SrcPtr, Builder.getPtrTy(AS));
- Value *VecLd = Builder.CreateAlignedLoad(Ty, CastedPtr, Alignment);
+ auto *VecLd = Builder.CreateAlignedLoad(Ty, CastedPtr, Alignment);
+ copyMetadataForMemoryAccess(*VecLd, *Load);
replaceValue(I, *VecLd);
++NumVecLoad;
return true;
@@ -2136,11 +2144,11 @@ bool VectorCombine::foldInsertElementsToStores(Instruction &I) {
{ConstantInt::get(GEPIdx->getType(), 0), GEPIdx});
LastStore = Builder.CreateStore(InsertVal, GEP);
- LastStore->copyMetadata(*SI);
-
- // The new GEP may change the pointer operand, so !invariant.group cannot
- // be transferred to the scalar store.
- LastStore->setMetadata(LLVMContext::MD_invariant_group, nullptr);
+ std::optional<size_t> Offset;
+ if (auto *ConstIdx = dyn_cast<ConstantInt>(Idx))
+ Offset =
+ ConstIdx->getZExtValue() * DL->getTypeStoreSize(InsertVal->getType());
+ copyMetadataForMemoryAccess(*LastStore, *SI, Offset);
Align ScalarOpAlignment = computeAlignmentAfterScalarization(
std::max(SI->getAlign(), Load->getAlign()), InsertVal->getType(), Idx,
*DL);
@@ -2307,11 +2315,10 @@ bool VectorCombine::scalarizeLoadExtract(LoadInst *LI, VectorType *VecTy,
computeAlignmentAfterScalarization(LI->getAlign(), ElemType, Idx, *DL);
NewLoad->setAlignment(ScalarOpAlignment);
- if (auto *ConstIdx = dyn_cast<ConstantInt>(Idx)) {
- size_t Offset = ConstIdx->getZExtValue() * DL->getTypeStoreSize(ElemType);
- AAMDNodes OldAAMD = LI->getAAMetadata();
- NewLoad->setAAMetadata(OldAAMD.adjustForAccess(Offset, ElemType, *DL));
- }
+ std::optional<size_t> Offset;
+ if (auto *ConstIdx = dyn_cast<ConstantInt>(Idx))
+ Offset = ConstIdx->getZExtValue() * DL->getTypeStoreSize(ElemType);
+ copyMetadataForMemoryAccess(*NewLoad, *LI, Offset);
replaceValue(*EI, *NewLoad, false);
}
@@ -2378,7 +2385,7 @@ bool VectorCombine::scalarizeLoadBitcast(LoadInst *LI, VectorType *VecTy,
auto *ScalarLoad =
Builder.CreateLoad(TargetScalarType, Ptr, LI->getName() + ".scalar");
ScalarLoad->setAlignment(LI->getAlign());
- ScalarLoad->copyMetadata(*LI);
+ copyMetadataForMemoryAccess(*ScalarLoad, *LI);
// Replace all bitcast users with the scalar load.
for (User *U : LI->users()) {
@@ -6635,6 +6642,7 @@ bool VectorCombine::foldBitcastOfVPLoad(Instruction &I) {
CallInst *NewVP = Builder.CreateIntrinsicWithoutFolding(
NewVecTy, Intrinsic::vp_load,
{II->getMemoryPointerParam(), NewMask, NewEVL});
+ copyMetadataForMemoryAccess(*NewVP, *II);
// Preserve the original alignment.
NewVP->addParamAttrs(
0, AttrBuilder(II->getContext()).addAlignmentAttr(OrigAlign));
@@ -6891,7 +6899,7 @@ bool VectorCombine::shrinkLoadForShuffles(Instruction &I) {
// Create new load of smaller vector.
auto *NewLoad = cast<LoadInst>(
Builder.CreateAlignedLoad(NewLoadTy, PtrOp, OldLoad->getAlign()));
- NewLoad->copyMetadata(I);
+ copyMetadataForMemoryAccess(*NewLoad, *OldLoad);
// Replace all uses.
for (UseEntry &Use : NewUses) {
diff --git a/llvm/test/Transforms/InstCombine/memory-cache-hint.ll b/llvm/test/Transforms/InstCombine/memory-cache-hint.ll
index 653fc45056db27..a3645fd93d546c 100644
--- a/llvm/test/Transforms/InstCombine/memory-cache-hint.ll
+++ b/llvm/test/Transforms/InstCombine/memory-cache-hint.ll
@@ -4,7 +4,7 @@
define void @store_bitcast(ptr %p, float %x) {
; CHECK-LABEL: define void @store_bitcast(
; CHECK-SAME: ptr [[P:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT: store float [[X]], ptr [[P]], align 4
+; CHECK-NEXT: store float [[X]], ptr [[P]], align 4, !mem.cache_hint [[META0:![0-9]+]]
; CHECK-NEXT: ret void
;
%bits = bitcast float %x to i32
@@ -15,8 +15,8 @@ define void @store_bitcast(ptr %p, float %x) {
define void @copy_both_hints(ptr %dst, ptr %src) {
; CHECK-LABEL: define void @copy_both_hints(
; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[SRC]], align 4
-; CHECK-NEXT: store i32 [[TMP1]], ptr [[DST]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[SRC]], align 4, !mem.cache_hint [[META2:![0-9]+]]
+; CHECK-NEXT: store i32 [[TMP1]], ptr [[DST]], align 4, !mem.cache_hint [[META0]]
; CHECK-NEXT: ret void
;
call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dst, ptr align 4 %src, i64 4, i1 false), !mem.cache_hint !2
@@ -26,7 +26,7 @@ define void @copy_both_hints(ptr %dst, ptr %src) {
define void @set_destination_hint(ptr %dst) {
; CHECK-LABEL: define void @set_destination_hint(
; CHECK-SAME: ptr [[DST:%.*]]) {
-; CHECK-NEXT: store i32 0, ptr [[DST]], align 4
+; CHECK-NEXT: store i32 0, ptr [[DST]], align 4, !mem.cache_hint [[META0]]
; CHECK-NEXT: ret void
;
call void @llvm.memset.p0.i64(ptr align 4 %dst, i8 0, i64 4, i1 false), !mem.cache_hint !3
@@ -38,7 +38,7 @@ define void @set_destination_hint(ptr %dst) {
define <2 x i32> @speculated_masked_load(ptr dereferenceable(8) %p, <2 x i1> %mask) {
; CHECK-LABEL: define <2 x i32> @speculated_masked_load(
; CHECK-SAME: ptr dereferenceable(8) [[P:%.*]], <2 x i1> [[MASK:%.*]]) {
-; CHECK-NEXT: [[UNMASKEDLOAD:%.*]] = load <2 x i32>, ptr [[P]], align 1, !invariant.load [[META0:![0-9]+]], !mem.cache_hint [[META1:![0-9]+]]
+; CHECK-NEXT: [[UNMASKEDLOAD:%.*]] = load <2 x i32>, ptr [[P]], align 1, !invariant.load [[META4:![0-9]+]], !mem.cache_hint [[META5:![0-9]+]]
; CHECK-NEXT: [[V:%.*]] = select <2 x i1> [[MASK]], <2 x i32> [[UNMASKEDLOAD]], <2 x i32> zeroinitializer
; CHECK-NEXT: ret <2 x i32> [[V]]
;
@@ -53,7 +53,10 @@ define <2 x i32> @speculated_masked_load(ptr dereferenceable(8) %p, <2 x i1> %ma
!4 = !{!"nvvm.l1_eviction", !"first"}
!6 = !{}
;.
-; CHECK: [[META0]] = !{}
-; CHECK: [[META1]] = !{i32 0, [[META2:![0-9]+]]}
-; CHECK: [[META2]] = !{!"nvvm.l1_eviction", !"last"}
+; CHECK: [[META0]] = !{i32 1, [[META1:![0-9]+]]}
+; CHECK: [[META1]] = !{!"nvvm.l1_eviction", !"last"}
+; CHECK: [[META2]] = !{i32 0, [[META3:![0-9]+]]}
+; CHECK: [[META3]] = !{!"nvvm.l1_eviction", !"first"}
+; CHECK: [[META4]] = !{}
+; CHECK: [[META5]] = !{i32 0, [[META1]]}
;.
diff --git a/llvm/test/Transforms/InstCombine/memory-metadata.ll b/llvm/test/Transforms/InstCombine/memory-metadata.ll
index 8ec8483485d263..b87598fb0098ed 100644
--- a/llvm/test/Transforms/InstCombine/memory-metadata.ll
+++ b/llvm/test/Transforms/InstCombine/memory-metadata.ll
@@ -6,7 +6,7 @@ target datalayout = "e-p:64:64"
define float @bitcast_load(ptr %p) {
; CHECK-LABEL: define float @bitcast_load(
; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[V1:%.*]] = load float, ptr [[P]], align 4, !tbaa [[INT_TBAA0:![0-9]+]], !invariant.load [[META4:![0-9]+]], !alias.scope [[META5:![0-9]+]], !noalias [[META5]], !nontemporal [[META8:![0-9]+]], !llvm.mem.parallel_loop_access [[META9:![0-9]+]], !invariant.group [[META4]], !llvm.access.group [[META9]], !noundef [[META4]], !annotation [[META10:![0-9]+]], !noalias.addrspace [[META11:![0-9]+]], !mem.cache_hint [[META12:![0-9]+]]
+; CHECK-NEXT: [[V1:%.*]] = load float, ptr [[P]], align 4, !tbaa [[INT_TBAA0:![0-9]+]], !invariant.load [[META4:![0-9]+]], !alias.scope [[META5:![0-9]+]], !noalias [[META5]], !nontemporal [[META8:![0-9]+]], !llvm.mem.parallel_loop_access [[META9:![0-9]+]], !invariant.group [[META4]], !llvm.access.group [[META9]], !noundef [[META4]], !annotation [[META10:![0-9]+]], !nosanitize [[META4]], !mmra [[META11:![0-9]+]], !noalias.addrspace [[META12:![0-9]+]], !mem.cache_hint [[META13:![0-9]+]]
; CHECK-NEXT: ret float [[V1]]
;
%v = load i32, ptr %p, align 4, !tbaa !10, !range !9, !noundef !0, !invariant.load !0, !invariant.group !0, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
@@ -17,7 +17,7 @@ define float @bitcast_load(ptr %p) {
define void @bitcast_store(ptr %p, float %v) {
; CHECK-LABEL: define void @bitcast_store(
; CHECK-SAME: ptr [[P:%.*]], float [[V:%.*]]) {
-; CHECK-NEXT: store float [[V]], ptr [[P]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]]
+; CHECK-NEXT: store float [[V]], ptr [[P]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !invariant.group [[META4]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15:![0-9]+]]
; CHECK-NEXT: ret void
;
%bits = bitcast float %v to i32
@@ -28,9 +28,9 @@ define void @bitcast_store(ptr %p, float %v) {
define i32 @unpack_struct_load(ptr %p) {
; CHECK-LABEL: define i32 @unpack_struct_load(
; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[V_UNPACK:%.*]] = load i32, ptr [[P]], align 8, !tbaa.struct [[TBAA_STRUCT14:![0-9]+]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT: [[V_UNPACK:%.*]] = load i32, ptr [[P]], align 8, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !invariant.group [[META4]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META13]]
; CHECK-NEXT: [[V_ELT1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4, !annotation [[META10]]
-; CHECK-NEXT: [[V_UNPACK2:%.*]] = load i32, ptr [[V_ELT1]], align 4, !tbaa.struct [[TBAA_STRUCT14]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT: [[V_UNPACK2:%.*]] = load i32, ptr [[V_ELT1]], align 4, !tbaa [[FLOAT_TBAA16:![0-9]+]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META13]]
; CHECK-NEXT: [[R:%.*]] = add i32 [[V_UNPACK]], [[V_UNPACK2]]
; CHECK-NEXT: ret i32 [[R]]
;
@@ -44,9 +44,9 @@ define i32 @unpack_struct_load(ptr %p) {
define i32 @unpack_array_load(ptr %p) {
; CHECK-LABEL: define i32 @unpack_array_load(
; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[V_UNPACK:%.*]] = load i32, ptr [[P]], align 8, !tbaa.struct [[TBAA_STRUCT14]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT: [[V_UNPACK:%.*]] = load i32, ptr [[P]], align 8, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !invariant.group [[META4]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META13]]
; CHECK-NEXT: [[V_ELT1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4, !annotation [[META10]]
-; CHECK-NEXT: [[V_UNPACK2:%.*]] = load i32, ptr [[V_ELT1]], align 4, !tbaa.struct [[TBAA_STRUCT14]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT: [[V_UNPACK2:%.*]] = load i32, ptr [[V_ELT1]], align 4, !tbaa [[FLOAT_TBAA16]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META13]]
; CHECK-NEXT: [[R:%.*]] = add i32 [[V_UNPACK]], [[V_UNPACK2]]
; CHECK-NEXT: ret i32 [[R]]
;
@@ -61,10 +61,10 @@ define void @unpack_struct_store(ptr %p, {i32, i32} %v) {
; CHECK-LABEL: define void @unpack_struct_store(
; CHECK-SAME: ptr [[P:%.*]], { i32, i32 } [[V:%.*]]) {
; CHECK-NEXT: [[V_ELT:%.*]] = extractvalue { i32, i32 } [[V]], 0, !annotation [[META10]]
-; CHECK-NEXT: store i32 [[V_ELT]], ptr [[P]], align 8, !tbaa.struct [[TBAA_STRUCT14]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT: store i32 [[V_ELT]], ptr [[P]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !invariant.group [[META4]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
; CHECK-NEXT: [[P_REPACK1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4, !annotation [[META10]]
; CHECK-NEXT: [[V_ELT2:%.*]] = extractvalue { i32, i32 } [[V]], 1, !annotation [[META10]]
-; CHECK-NEXT: store i32 [[V_ELT2]], ptr [[P_REPACK1]], align 4, !tbaa.struct [[TBAA_STRUCT14]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT: store i32 [[V_ELT2]], ptr [[P_REPACK1]], align 4, !tbaa [[FLOAT_TBAA16]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
; CHECK-NEXT: ret void
;
store {i32, i32} %v, ptr %p, align 8, !tbaa.struct !19, !invariant.group !0, !mem.cache_hint !18, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
@@ -75,10 +75,10 @@ define void @unpack_array_store(ptr %p, [2 x i32] %v) {
; CHECK-LABEL: define void @unpack_array_store(
; CHECK-SAME: ptr [[P:%.*]], [2 x i32] [[V:%.*]]) {
; CHECK-NEXT: [[V_ELT:%.*]] = extractvalue [2 x i32] [[V]], 0, !annotation [[META10]]
-; CHECK-NEXT: store i32 [[V_ELT]], ptr [[P]], align 8, !tbaa.struct [[TBAA_STRUCT14]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT: store i32 [[V_ELT]], ptr [[P]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !invariant.group [[META4]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
; CHECK-NEXT: [[P_REPACK1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4, !annotation [[META10]]
; CHECK-NEXT: [[V_ELT2:%.*]] = extractvalue [2 x i32] [[V]], 1, !annotation [[META10]]
-; CHECK-NEXT: store i32 [[V_ELT2]], ptr [[P_REPACK1]], align 4, !tbaa.struct [[TBAA_STRUCT14]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT: store i32 [[V_ELT2]], ptr [[P_REPACK1]], align 4, !tbaa [[FLOAT_TBAA16]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
; CHECK-NEXT: ret void
;
store [2 x i32] %v, ptr %p, align 8, !tbaa.struct !19, !invariant.group !0, !mem.cache_hint !18, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
@@ -90,8 +90,8 @@ define void @unpack_array_store(ptr %p, [2 x i32] %v) {
define void @copy_memory(ptr %dst, ptr %src) {
; CHECK-LABEL: define void @copy_memory(
; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = load i64, ptr [[SRC]], align 8, !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !noalias.addrspace [[META11]]
-; CHECK-NEXT: store i64 [[TMP1]], ptr [[DST]], align 8, !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i64, ptr [[SRC]], align 8, !tbaa.struct [[TBAA_STRUCT18:![0-9]+]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META19:![0-9]+]]
+; CHECK-NEXT: store i64 [[TMP1]], ptr [[DST]], align 8, !tbaa.struct [[TBAA_STRUCT18]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
; CHECK-NEXT: ret void
;
call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %src, i64 8, i1 false), !tbaa.struct !19, !prof !24, !mem.cache_hint !22, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
@@ -101,7 +101,7 @@ define void @copy_memory(ptr %dst, ptr %src) {
define void @set_memory(ptr %dst) {
; CHECK-LABEL: define void @set_memory(
; CHECK-SAME: ptr [[DST:%.*]]) {
-; CHECK-NEXT: store i32 0, ptr [[DST]], align 4, !annotation [[META10]]
+; CHECK-NEXT: store i32 0, ptr [[DST]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
; CHECK-NEXT: ret void
;
call void @llvm.memset.p0.i64(ptr align 4 %dst, i8 0, i64 4, i1 false), !tbaa !10, !prof !24, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
@@ -111,7 +111,7 @@ define void @set_memory(ptr %dst) {
define <2 x i32> @splat_gather(ptr %p) {
; CHECK-LABEL: define <2 x i32> @splat_gather(
; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[LOAD_SCALAR:%.*]] = load i32, ptr [[P]], align 1, !annotation [[META10]]
+; CHECK-NEXT: [[LOAD_SCALAR:%.*]] = load i32, ptr [[P]], align 1, !tbaa [[INT_TBAA0]], !range [[RNG21:![0-9]+]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META13]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[LOAD_SCALAR]], i64 0, !annotation [[META10]]
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer, !annotation [[META10]]
; CHECK-NEXT: ret <2 x i32> [[BROADCAST_SPLAT]]
@@ -125,7 +125,7 @@ define <2 x i32> @splat_gather(ptr %p) {
define void @splat_scatter(ptr %p, i32 %v) {
; CHECK-LABEL: define void @splat_scatter(
; CHECK-SAME: ptr [[P:%.*]], i32 [[V:%.*]]) {
-; CHECK-NEXT: store i32 [[V]], ptr [[P]], align 1, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META17:![0-9]+]], !noalias.addrspace [[META11]], !mem.cache_hint [[META18:![0-9]+]]
+; CHECK-NEXT: store i32 [[V]], ptr [[P]], align 1, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
; CHECK-NEXT: ret void
;
%ptr0 = insertelement <2 x ptr> poison, ptr %p, i32 0
@@ -174,12 +174,15 @@ define void @splat_scatter(ptr %p, i32 %v) {
; CHECK: [[META8]] = !{i32 1}
; CHECK: [[META9]] = distinct !{}
; CHECK: [[META10]] = !{!"preserved memory metadata"}
-; CHECK: [[META11]] = !{i32 5, i32 6}
-; CHECK: [[META12]] = !{i32 0, [[META13:![0-9]+]]}
-; CHECK: [[META13]] = !{!"nvvm.l1_eviction", !"last"}
-; CHECK: [[TBAA_STRUCT14]] = !{i64 0, i64 4, [[INT_TBAA0]], i64 4, i64 4, [[META15:![0-9]+]]}
-; CHECK: [[META15]] = !{[[META16:![0-9]+]], [[META16]], i64 0}
-; CHECK: [[META16]] = !{!"float", [[META2]], i64 0}
-; CHECK: [[META17]] = !{!"test", !"relaxed"}
-; CHECK: [[META18]] = !{i32 1, [[META13]]}
+; CHECK: [[META11]] = !{!"test", !"relaxed"}
+; CHECK: [[META12]] = !{i32 5, i32 6}
+; CHECK: [[META13]] = !{i32 0, [[META14:![0-9]+]]}
+; CHECK: [[META14]] = !{!"nvvm.l1_eviction", !"last"}
+; CHECK: [[META15]] = !{i32 1, [[META14]]}
+; CHECK: [[FLOAT_TBAA16]] = !{[[META17:![0-9]+]], [[META17]], i64 0}
+; CHECK: [[META17]] = !{!"float", [[META2]], i64 0}
+; CHECK: [[TBAA_STRUCT18]] = !{i64 0, i64 4, [[INT_TBAA0]], i64 4, i64 4, [[FLOAT_TBAA16]]}
+; CHECK: [[META19]] = !{i32 0, [[META20:![0-9]+]]}
+; CHECK: [[META20]] = !{!"nvvm.l1_eviction", !"first"}
+; CHECK: [[RNG21]] = !{i32 0, i32 100}
;.
diff --git a/llvm/test/Transforms/InstCombine/storemerge-metadata.ll b/llvm/test/Transforms/InstCombine/storemerge-metadata.ll
index dc5fdee99c333f..dfe2f81950217d 100644
--- a/llvm/test/Transforms/InstCombine/storemerge-metadata.ll
+++ b/llvm/test/Transforms/InstCombine/storemerge-metadata.ll
@@ -12,7 +12,7 @@ define void @common_metadata(ptr %p, i1 %c, i32 %a, i32 %b) {
; CHECK-NEXT: br label %[[JOIN]]
; CHECK: [[JOIN]]:
; CHECK-NEXT: [[STOREMERGE:%.*]] = phi i32 [ [[B]], %[[RIGHT]] ], [ [[A]], %[[LEFT]] ]
-; CHECK-NEXT: store i32 [[STOREMERGE]], ptr [[P]], align 4, !tbaa [[INT_TBAA0:![0-9]+]], !alias.scope [[META4:![0-9]+]], !noalias [[META4]], !llvm.access.group [[ACC_GRP7:![0-9]+]]
+; CHECK-NEXT: store i32 [[STOREMERGE]], ptr [[P]], align 4, !tbaa [[INT_TBAA0:![0-9]+]], !alias.scope [[META4:![0-9]+]], !noalias [[META4]], !nontemporal [[META7:![0-9]+]], !llvm.access.group [[ACC_GRP8:![0-9]+]], !mem.cache_hint [[META11:![0-9]+]]
; CHECK-NEXT: ret void
;
entry:
@@ -38,7 +38,7 @@ define void @intersect_access_groups(ptr %p, i1 %c, i32 %a, i32 %b) {
; CHECK-NEXT: br label %[[JOIN]]
; CHECK: [[JOIN]]:
; CHECK-NEXT: [[STOREMERGE:%.*]] = phi i32 [ [[B]], %[[RIGHT]] ], [ [[A]], %[[LEFT]] ]
-; CHECK-NEXT: store i32 [[STOREMERGE]], ptr [[P]], align 4, !llvm.access.group [[ACC_GRP9:![0-9]+]]
+; CHECK-NEXT: store i32 [[STOREMERGE]], ptr [[P]], align 4, !llvm.access.group [[ACC_GRP10:![0-9]+]]
; CHECK-NEXT: ret void
;
entry:
@@ -140,7 +140,7 @@ define void @triangle_common_metadata(ptr %p, i1 %c, i32 %a, i32 %b) {
; CHECK-NEXT: br label %[[JOIN]]
; CHECK: [[JOIN]]:
; CHECK-NEXT: [[STOREMERGE:%.*]] = phi i32 [ [[B]], %[[THEN]] ], [ [[A]], %[[ENTRY]] ]
-; CHECK-NEXT: store i32 [[STOREMERGE]], ptr [[P]], align 4
+; CHECK-NEXT: store i32 [[STOREMERGE]], ptr [[P]], align 4, !nontemporal [[META7]], !mem.cache_hint [[META11]]
; CHECK-NEXT: ret void
;
entry:
@@ -164,7 +164,7 @@ define void @merge_pointer_captures(ptr %p, i1 %c, ptr %a, ptr %b) {
; CHECK-NEXT: br label %[[JOIN]]
; CHECK: [[JOIN]]:
; CHECK-NEXT: [[STOREMERGE:%.*]] = phi ptr [ [[B]], %[[RIGHT]] ], [ [[A]], %[[LEFT]] ]
-; CHECK-NEXT: store ptr [[STOREMERGE]], ptr [[P]], align 8
+; CHECK-NEXT: store ptr [[STOREMERGE]], ptr [[P]], align 8, !captures [[META13:![0-9]+]]
; CHECK-NEXT: ret void
;
entry:
@@ -207,7 +207,11 @@ join:
; CHECK: [[META4]] = !{[[META5:![0-9]+]]}
; CHECK: [[META5]] = distinct !{[[META5]], [[META6:![0-9]+]]}
; CHECK: [[META6]] = distinct !{[[META6]], i1 false, !"domain"}
-; CHECK: [[ACC_GRP7]] = !{[[META8:![0-9]+]], [[ACC_GRP9]]}
-; CHECK: [[META8]] = distinct !{}
-; CHECK: [[ACC_GRP9]] = distinct !{}
+; CHECK: [[META7]] = !{i32 1}
+; CHECK: [[ACC_GRP8]] = !{[[META9:![0-9]+]], [[ACC_GRP10]]}
+; CHECK: [[META9]] = distinct !{}
+; CHECK: [[ACC_GRP10]] = distinct !{}
+; CHECK: [[META11]] = !{i32 1, [[META12:![0-9]+]]}
+; CHECK: [[META12]] = !{!"nvvm.l1_eviction", !"last"}
+; CHECK: [[META13]] = !{!"address", !"read_provenance"}
;.
diff --git a/llvm/test/Transforms/SROA/memory-metadata.ll b/llvm/test/Transforms/SROA/memory-metadata.ll
index a1d2f5687a4597..f4d272e8465ef3 100644
--- a/llvm/test/Transforms/SROA/memory-metadata.ll
+++ b/llvm/test/Transforms/SROA/memory-metadata.ll
@@ -8,11 +8,11 @@ define i32 @split_copy(ptr %src) {
; CHECK-SAME: ptr [[SRC:%.*]]) {
; CHECK-NEXT: [[A_SROA_0:%.*]] = alloca i32, align 8
; CHECK-NEXT: [[A_SROA_2:%.*]] = alloca i32, align 4
-; CHECK-NEXT: [[A_SROA_0_0_COPYLOAD:%.*]] = load i32, ptr [[SRC]], align 8, !tbaa [[INT_TBAA0:![0-9]+]], !alias.scope [[META4:![0-9]+]], !noalias [[META4]], !llvm.mem.parallel_loop_access [[META7:![0-9]+]], !llvm.access.group [[META7]], !noalias.addrspace [[META8:![0-9]+]]
-; CHECK-NEXT: store i32 [[A_SROA_0_0_COPYLOAD]], ptr [[A_SROA_0]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META4]], !noalias [[META4]], !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]], !noalias.addrspace [[META8]]
+; CHECK-NEXT: [[A_SROA_0_0_COPYLOAD:%.*]] = load i32, ptr [[SRC]], align 8, !tbaa [[INT_TBAA0:![0-9]+]], !alias.scope [[META4:![0-9]+]], !noalias [[META4]], !nontemporal [[META7:![0-9]+]], !llvm.mem.parallel_loop_access [[META8:![0-9]+]], !llvm.access.group [[META8]], !annotation [[META9:![0-9]+]], !nosanitize [[META10:![0-9]+]], !mmra [[META11:![0-9]+]], !noalias.addrspace [[META12:![0-9]+]], !mem.cache_hint [[META13:![0-9]+]]
+; CHECK-NEXT: store i32 [[A_SROA_0_0_COPYLOAD]], ptr [[A_SROA_0]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15:![0-9]+]]
; CHECK-NEXT: [[A_SROA_2_0_SRC_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 4
-; CHECK-NEXT: [[A_SROA_2_0_COPYLOAD:%.*]] = load i32, ptr [[A_SROA_2_0_SRC_SROA_IDX]], align 4, !tbaa [[FLOAT_TBAA9:![0-9]+]], !alias.scope [[META4]], !noalias [[META4]], !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]], !noalias.addrspace [[META8]]
-; CHECK-NEXT: store i32 [[A_SROA_2_0_COPYLOAD]], ptr [[A_SROA_2]], align 4, !tbaa [[FLOAT_TBAA9]], !alias.scope [[META4]], !noalias [[META4]], !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]], !noalias.addrspace [[META8]]
+; CHECK-NEXT: [[A_SROA_2_0_COPYLOAD:%.*]] = load i32, ptr [[A_SROA_2_0_SRC_SROA_IDX]], align 4, !tbaa [[FLOAT_TBAA17:![0-9]+]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META13]]
+; CHECK-NEXT: store i32 [[A_SROA_2_0_COPYLOAD]], ptr [[A_SROA_2]], align 4, !tbaa [[FLOAT_TBAA17]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
; CHECK-NEXT: [[A_SROA_0_0_A_SROA_0_0_X:%.*]] = load volatile i32, ptr [[A_SROA_0]], align 8
; CHECK-NEXT: [[A_SROA_2_0_A_SROA_2_4_Y:%.*]] = load volatile i32, ptr [[A_SROA_2]], align 4
; CHECK-NEXT: [[R:%.*]] = add i32 [[A_SROA_0_0_A_SROA_0_0_X]], [[A_SROA_2_0_A_SROA_2_4_Y]]
@@ -33,7 +33,7 @@ define i32 @rewrite_set(i32 %v) {
; CHECK-NEXT: [[A_SROA_0:%.*]] = alloca i32, align 8
; CHECK-NEXT: [[A_SROA_2:%.*]] = alloca i32, align 4
; CHECK-NEXT: store i32 [[V]], ptr [[A_SROA_0]], align 8
-; CHECK-NEXT: store i32 0, ptr [[A_SROA_2]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META4]], !noalias [[META4]], !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]], !noalias.addrspace [[META8]]
+; CHECK-NEXT: store i32 0, ptr [[A_SROA_2]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
; CHECK-NEXT: [[A_SROA_0_0_A_SROA_0_0_X:%.*]] = load volatile i32, ptr [[A_SROA_0]], align 8
; CHECK-NEXT: [[A_SROA_2_0_A_SROA_2_4_Y:%.*]] = load volatile i32, ptr [[A_SROA_2]], align 4
; CHECK-NEXT: [[R:%.*]] = add i32 [[A_SROA_0_0_A_SROA_0_0_X]], [[A_SROA_2_0_A_SROA_2_4_Y]]
@@ -55,9 +55,9 @@ define i32 @split_aggregate_store({i32, i32} %v) {
; CHECK-NEXT: [[A_SROA_0:%.*]] = alloca i32, align 8
; CHECK-NEXT: [[A_SROA_2:%.*]] = alloca i32, align 4
; CHECK-NEXT: [[V_FCA_0_EXTRACT:%.*]] = extractvalue { i32, i32 } [[V]], 0
-; CHECK-NEXT: store i32 [[V_FCA_0_EXTRACT]], ptr [[A_SROA_0]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META4]], !noalias [[META4]], !noalias.addrspace [[META8]]
+; CHECK-NEXT: store i32 [[V_FCA_0_EXTRACT]], ptr [[A_SROA_0]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
; CHECK-NEXT: [[V_FCA_1_EXTRACT:%.*]] = extractvalue { i32, i32 } [[V]], 1
-; CHECK-NEXT: store i32 [[V_FCA_1_EXTRACT]], ptr [[A_SROA_2]], align 4, !tbaa [[FLOAT_TBAA9]], !alias.scope [[META4]], !noalias [[META4]], !noalias.addrspace [[META8]]
+; CHECK-NEXT: store i32 [[V_FCA_1_EXTRACT]], ptr [[A_SROA_2]], align 4, !tbaa [[FLOAT_TBAA17]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
; CHECK-NEXT: [[A_SROA_0_0_A_SROA_0_0_X:%.*]] = load volatile i32, ptr [[A_SROA_0]], align 8
; CHECK-NEXT: [[A_SROA_2_0_A_SROA_2_4_Y:%.*]] = load volatile i32, ptr [[A_SROA_2]], align 4
; CHECK-NEXT: [[R:%.*]] = add i32 [[A_SROA_0_0_A_SROA_0_0_X]], [[A_SROA_2_0_A_SROA_2_4_Y]]
@@ -82,9 +82,9 @@ define {i32, i32} @split_aggregate_load(ptr %src, ptr %dst) {
; CHECK-NEXT: [[A_SROA_3_0_SRC_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 4
; CHECK-NEXT: [[A_SROA_3_0_COPYLOAD:%.*]] = load i32, ptr [[A_SROA_3_0_SRC_SROA_IDX]], align 1
; CHECK-NEXT: store i32 [[A_SROA_3_0_COPYLOAD]], ptr [[A_SROA_3]], align 4
-; CHECK-NEXT: [[A_SROA_0_0_A_SROA_0_0_V_FCA_0_LOAD:%.*]] = load i32, ptr [[A_SROA_0]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META4]], !noalias [[META4]], !noalias.addrspace [[META8]]
+; CHECK-NEXT: [[A_SROA_0_0_A_SROA_0_0_V_FCA_0_LOAD:%.*]] = load i32, ptr [[A_SROA_0]], align 8, !tbaa [[INT_TBAA0]], !invariant.load [[META10]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META19:![0-9]+]]
; CHECK-NEXT: [[V_FCA_0_INSERT:%.*]] = insertvalue { i32, i32 } poison, i32 [[A_SROA_0_0_A_SROA_0_0_V_FCA_0_LOAD]], 0
-; CHECK-NEXT: [[A_SROA_3_0_A_SROA_3_4_V_FCA_1_LOAD:%.*]] = load i32, ptr [[A_SROA_3]], align 4, !tbaa [[FLOAT_TBAA9]], !alias.scope [[META4]], !noalias [[META4]], !noalias.addrspace [[META8]]
+; CHECK-NEXT: [[A_SROA_3_0_A_SROA_3_4_V_FCA_1_LOAD:%.*]] = load i32, ptr [[A_SROA_3]], align 4, !tbaa [[FLOAT_TBAA17]], !invariant.load [[META10]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META19]]
; CHECK-NEXT: [[V_FCA_1_INSERT:%.*]] = insertvalue { i32, i32 } [[V_FCA_0_INSERT]], i32 [[A_SROA_3_0_A_SROA_3_4_V_FCA_1_LOAD]], 1
; CHECK-NEXT: [[A_SROA_0_0_A_SROA_0_0_COPYLOAD1:%.*]] = load volatile i32, ptr [[A_SROA_0]], align 8
; CHECK-NEXT: store volatile i32 [[A_SROA_0_0_A_SROA_0_0_COPYLOAD1]], ptr [[DST]], align 1
@@ -110,7 +110,7 @@ define double @rewrite_load(ptr %src, ptr %dst) {
; CHECK-NEXT: [[A_SROA_3_0_SRC_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 8
; CHECK-NEXT: [[A_SROA_3_0_COPYLOAD:%.*]] = load i32, ptr [[A_SROA_3_0_SRC_SROA_IDX]], align 1
; CHECK-NEXT: store i32 [[A_SROA_3_0_COPYLOAD]], ptr [[A_SROA_3]], align 8
-; CHECK-NEXT: [[A_SROA_0_0_A_SROA_0_0_V:%.*]] = load double, ptr [[A_SROA_0]], align 8, !fpmath [[META11:![0-9]+]], !invariant.load [[META12:![0-9]+]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META13:![0-9]+]], !llvm.mem.parallel_loop_access [[META7]], !invariant.group [[META12]], !llvm.access.group [[META7]], !noundef [[META12]], !noalias.addrspace [[META8]], !nofpclass [[META14:![0-9]+]], !mem.cache_hint [[META15:![0-9]+]]
+; CHECK-NEXT: [[A_SROA_0_0_A_SROA_0_0_V:%.*]] = load double, ptr [[A_SROA_0]], align 8, !fpmath [[META20:![0-9]+]], !invariant.load [[META10]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !noundef [[META10]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !nofpclass [[META21:![0-9]+]], !mem.cache_hint [[META19]]
; CHECK-NEXT: [[A_SROA_0_0_A_SROA_0_0_COPYLOAD1:%.*]] = load volatile double, ptr [[A_SROA_0]], align 8
; CHECK-NEXT: store volatile double [[A_SROA_0_0_A_SROA_0_0_COPYLOAD1]], ptr [[DST]], align 1
; CHECK-NEXT: [[A_SROA_3_0_DST_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 8
@@ -130,7 +130,7 @@ define ptr @rewrite_pointer_load(ptr %v) {
; CHECK-SAME: ptr [[V:%.*]]) {
; CHECK-NEXT: [[A_SROA_1:%.*]] = alloca ptr, align 8
; CHECK-NEXT: store ptr [[V]], ptr [[A_SROA_1]], align 8
-; CHECK-NEXT: [[A_SROA_1_0_A_SROA_1_8_R:%.*]] = load volatile ptr, ptr [[A_SROA_1]], align 8, !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META13]], !llvm.mem.parallel_loop_access [[META7]], !nonnull [[META12]], !dereferenceable [[META17:![0-9]+]], !dereferenceable_or_null [[META17]], !invariant.group [[META12]], !align [[META17]], !llvm.access.group [[META7]], !noundef [[META12]], !noalias.addrspace [[META8]], !mem.cache_hint [[META15]]
+; CHECK-NEXT: [[A_SROA_1_0_A_SROA_1_8_R:%.*]] = load volatile ptr, ptr [[A_SROA_1]], align 8, !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !nonnull [[META10]], !dereferenceable [[META22:![0-9]+]], !dereferenceable_or_null [[META22]], !align [[META22]], !llvm.access.group [[META8]], !noundef [[META10]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META19]]
; CHECK-NEXT: ret ptr [[A_SROA_1_0_A_SROA_1_8_R]]
;
%a = alloca [2 x ptr], align 8
@@ -146,11 +146,11 @@ define i32 @presplit_load(ptr %src) {
; CHECK-SAME: ptr [[SRC:%.*]]) {
; CHECK-NEXT: [[A_SROA_0:%.*]] = alloca i32, align 8
; CHECK-NEXT: [[A_SROA_2:%.*]] = alloca i32, align 4
-; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[SRC]], align 8, !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]]
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[SRC]], align 8, !invariant.load [[META10]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !invariant.group [[META10]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META19]]
; CHECK-NEXT: [[SRC_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 4
-; CHECK-NEXT: [[V2:%.*]] = load i32, ptr [[SRC_SROA_IDX]], align 4, !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]]
-; CHECK-NEXT: store i32 [[V1]], ptr [[A_SROA_0]], align 8, !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]]
-; CHECK-NEXT: store i32 [[V2]], ptr [[A_SROA_2]], align 4, !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]]
+; CHECK-NEXT: [[V2:%.*]] = load i32, ptr [[SRC_SROA_IDX]], align 4, !invariant.load [[META10]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META19]]
+; CHECK-NEXT: store i32 [[V1]], ptr [[A_SROA_0]], align 8, !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
+; CHECK-NEXT: store i32 [[V2]], ptr [[A_SROA_2]], align 4, !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
; CHECK-NEXT: [[A_SROA_0_0_A_SROA_0_0_X:%.*]] = load volatile i32, ptr [[A_SROA_0]], align 8
; CHECK-NEXT: [[A_SROA_2_0_A_SROA_2_4_Y:%.*]] = load volatile i32, ptr [[A_SROA_2]], align 4
; CHECK-NEXT: [[R:%.*]] = add i32 [[A_SROA_0_0_A_SROA_0_0_X]], [[A_SROA_2_0_A_SROA_2_4_Y]]
@@ -201,15 +201,20 @@ define i32 @presplit_load(ptr %src) {
; CHECK: [[META4]] = !{[[META5:![0-9]+]]}
; CHECK: [[META5]] = distinct !{[[META5]], [[META6:![0-9]+]], !"scope"}
; CHECK: [[META6]] = distinct !{[[META6]], i1 false, !"domain"}
-; CHECK: [[META7]] = distinct !{}
-; CHECK: [[META8]] = !{i32 5, i32 6}
-; CHECK: [[FLOAT_TBAA9]] = !{[[META10:![0-9]+]], [[META10]], i64 0}
-; CHECK: [[META10]] = !{!"float", [[META2]], i64 0}
-; CHECK: [[META11]] = !{float 2.500000e+00}
-; CHECK: [[META12]] = !{}
-; CHECK: [[META13]] = !{i32 1}
-; CHECK: [[META14]] = !{i32 3}
-; CHECK: [[META15]] = !{i32 0, [[META16:![0-9]+]]}
+; CHECK: [[META7]] = !{i32 1}
+; CHECK: [[META8]] = distinct !{}
+; CHECK: [[META9]] = !{!"preserved memory metadata"}
+; CHECK: [[META10]] = !{}
+; CHECK: [[META11]] = !{!"test", !"relaxed"}
+; CHECK: [[META12]] = !{i32 5, i32 6}
+; CHECK: [[META13]] = !{i32 0, [[META14:![0-9]+]]}
+; CHECK: [[META14]] = !{!"nvvm.l1_eviction", !"first"}
+; CHECK: [[META15]] = !{i32 1, [[META16:![0-9]+]]}
; CHECK: [[META16]] = !{!"nvvm.l1_eviction", !"last"}
-; CHECK: [[META17]] = !{i64 8}
+; CHECK: [[FLOAT_TBAA17]] = !{[[META18:![0-9]+]], [[META18]], i64 0}
+; CHECK: [[META18]] = !{!"float", [[META2]], i64 0}
+; CHECK: [[META19]] = !{i32 0, [[META16]]}
+; CHECK: [[META20]] = !{float 2.500000e+00}
+; CHECK: [[META21]] = !{i32 3}
+; CHECK: [[META22]] = !{i64 8}
;.
diff --git a/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll b/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
index b7db48d9b9cbc7..96ac70bdb37e39 100644
--- a/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
+++ b/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
@@ -209,21 +209,21 @@ define void @f4(ptr %src, ptr %dst) {
; CHECK-NEXT: [[DST_I1:%.*]] = getelementptr i32, ptr [[DST]], i32 1
; CHECK-NEXT: [[DST_I2:%.*]] = getelementptr i32, ptr [[DST]], i32 2
; CHECK-NEXT: [[DST_I3:%.*]] = getelementptr i32, ptr [[DST]], i32 3
-; CHECK-NEXT: [[VAL_I0:%.*]] = load i32, ptr [[SRC]], align 16, !tbaa.struct [[TBAA_STRUCT5:![0-9]+]]
+; CHECK-NEXT: [[VAL_I0:%.*]] = load i32, ptr [[SRC]], align 16, !tbaa [[SET1_TBAA0]]
; CHECK-NEXT: [[SRC_I1:%.*]] = getelementptr i32, ptr [[SRC]], i32 1
-; CHECK-NEXT: [[VAL_I1:%.*]] = load i32, ptr [[SRC_I1]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
+; CHECK-NEXT: [[VAL_I1:%.*]] = load i32, ptr [[SRC_I1]], align 4, !tbaa [[SET2_TBAA3]]
; CHECK-NEXT: [[SRC_I2:%.*]] = getelementptr i32, ptr [[SRC]], i32 2
-; CHECK-NEXT: [[VAL_I2:%.*]] = load i32, ptr [[SRC_I2]], align 8, !tbaa.struct [[TBAA_STRUCT5]]
+; CHECK-NEXT: [[VAL_I2:%.*]] = load i32, ptr [[SRC_I2]], align 8, !tbaa [[SET1_TBAA0]]
; CHECK-NEXT: [[SRC_I3:%.*]] = getelementptr i32, ptr [[SRC]], i32 3
-; CHECK-NEXT: [[VAL_I3:%.*]] = load i32, ptr [[SRC_I3]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
+; CHECK-NEXT: [[VAL_I3:%.*]] = load i32, ptr [[SRC_I3]], align 4, !tbaa [[SET2_TBAA3]]
; CHECK-NEXT: [[ADD_I0:%.*]] = add i32 [[VAL_I0]], [[VAL_I0]]
; CHECK-NEXT: [[ADD_I1:%.*]] = add i32 [[VAL_I1]], [[VAL_I1]]
; CHECK-NEXT: [[ADD_I2:%.*]] = add i32 [[VAL_I2]], [[VAL_I2]]
; CHECK-NEXT: [[ADD_I3:%.*]] = add i32 [[VAL_I3]], [[VAL_I3]]
-; CHECK-NEXT: store i32 [[ADD_I0]], ptr [[DST]], align 16, !tbaa.struct [[TBAA_STRUCT5]]
-; CHECK-NEXT: store i32 [[ADD_I1]], ptr [[DST_I1]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
-; CHECK-NEXT: store i32 [[ADD_I2]], ptr [[DST_I2]], align 8, !tbaa.struct [[TBAA_STRUCT5]]
-; CHECK-NEXT: store i32 [[ADD_I3]], ptr [[DST_I3]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
+; CHECK-NEXT: store i32 [[ADD_I0]], ptr [[DST]], align 16, !tbaa [[SET1_TBAA0]]
+; CHECK-NEXT: store i32 [[ADD_I1]], ptr [[DST_I1]], align 4, !tbaa [[SET2_TBAA3]]
+; CHECK-NEXT: store i32 [[ADD_I2]], ptr [[DST_I2]], align 8, !tbaa [[SET1_TBAA0]]
+; CHECK-NEXT: store i32 [[ADD_I3]], ptr [[DST_I3]], align 4, !tbaa [[SET2_TBAA3]]
; CHECK-NEXT: ret void
;
%val = load <4 x i32> , ptr %src, !tbaa.struct !5
@@ -248,21 +248,21 @@ define void @f5(i32 %count, ptr %src, ptr %dst) {
; CHECK-NEXT: [[THIS_DST_I1:%.*]] = getelementptr i32, ptr [[THIS_DST]], i32 1
; CHECK-NEXT: [[THIS_DST_I2:%.*]] = getelementptr i32, ptr [[THIS_DST]], i32 2
; CHECK-NEXT: [[THIS_DST_I3:%.*]] = getelementptr i32, ptr [[THIS_DST]], i32 3
-; CHECK-NEXT: [[VAL_I0:%.*]] = load i32, ptr [[THIS_SRC]], align 16, !llvm.access.group [[ACC_GRP6:![0-9]+]]
-; CHECK-NEXT: [[VAL_I1:%.*]] = load i32, ptr [[THIS_SRC_I1]], align 4, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT: [[VAL_I2:%.*]] = load i32, ptr [[THIS_SRC_I2]], align 8, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT: [[VAL_I3:%.*]] = load i32, ptr [[THIS_SRC_I3]], align 4, !llvm.access.group [[ACC_GRP6]]
+; CHECK-NEXT: [[VAL_I0:%.*]] = load i32, ptr [[THIS_SRC]], align 16, !llvm.access.group [[ACC_GRP5:![0-9]+]]
+; CHECK-NEXT: [[VAL_I1:%.*]] = load i32, ptr [[THIS_SRC_I1]], align 4, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT: [[VAL_I2:%.*]] = load i32, ptr [[THIS_SRC_I2]], align 8, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT: [[VAL_I3:%.*]] = load i32, ptr [[THIS_SRC_I3]], align 4, !llvm.access.group [[ACC_GRP5]]
; CHECK-NEXT: [[ADD_I0:%.*]] = add i32 [[VAL_I0]], [[VAL_I0]]
; CHECK-NEXT: [[ADD_I1:%.*]] = add i32 [[VAL_I1]], [[VAL_I1]]
; CHECK-NEXT: [[ADD_I2:%.*]] = add i32 [[VAL_I2]], [[VAL_I2]]
; CHECK-NEXT: [[ADD_I3:%.*]] = add i32 [[VAL_I3]], [[VAL_I3]]
-; CHECK-NEXT: store i32 [[ADD_I0]], ptr [[THIS_DST]], align 16, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT: store i32 [[ADD_I1]], ptr [[THIS_DST_I1]], align 4, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT: store i32 [[ADD_I2]], ptr [[THIS_DST_I2]], align 8, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT: store i32 [[ADD_I3]], ptr [[THIS_DST_I3]], align 4, !llvm.access.group [[ACC_GRP6]]
+; CHECK-NEXT: store i32 [[ADD_I0]], ptr [[THIS_DST]], align 16, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT: store i32 [[ADD_I1]], ptr [[THIS_DST_I1]], align 4, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT: store i32 [[ADD_I2]], ptr [[THIS_DST_I2]], align 8, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT: store i32 [[ADD_I3]], ptr [[THIS_DST_I3]], align 4, !llvm.access.group [[ACC_GRP5]]
; CHECK-NEXT: [[NEXT_INDEX]] = add i32 [[INDEX]], -1
; CHECK-NEXT: [[CONTINUE:%.*]] = icmp ne i32 [[NEXT_INDEX]], [[COUNT]]
-; CHECK-NEXT: br i1 [[CONTINUE]], label %[[LOOP]], label %[[END:.*]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[CONTINUE]], label %[[LOOP]], label %[[END:.*]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[END]]:
; CHECK-NEXT: ret void
;
@@ -289,13 +289,13 @@ define <4 x float> @f6(<4 x float> %x) {
; CHECK-LABEL: define <4 x float> @f6(
; CHECK-SAME: <4 x float> [[X:%.*]]) {
; CHECK-NEXT: [[X_I0:%.*]] = extractelement <4 x float> [[X]], i64 0
-; CHECK-NEXT: [[RES_I0:%.*]] = fadd float [[X_I0]], 1.000000e+00, !fpmath [[META9:![0-9]+]]
+; CHECK-NEXT: [[RES_I0:%.*]] = fadd float [[X_I0]], 1.000000e+00, !fpmath [[META8:![0-9]+]]
; CHECK-NEXT: [[X_I1:%.*]] = extractelement <4 x float> [[X]], i64 1
-; CHECK-NEXT: [[RES_I1:%.*]] = fadd float [[X_I1]], 2.000000e+00, !fpmath [[META9]]
+; CHECK-NEXT: [[RES_I1:%.*]] = fadd float [[X_I1]], 2.000000e+00, !fpmath [[META8]]
; CHECK-NEXT: [[X_I2:%.*]] = extractelement <4 x float> [[X]], i64 2
-; CHECK-NEXT: [[RES_I2:%.*]] = fadd float [[X_I2]], 3.000000e+00, !fpmath [[META9]]
+; CHECK-NEXT: [[RES_I2:%.*]] = fadd float [[X_I2]], 3.000000e+00, !fpmath [[META8]]
; CHECK-NEXT: [[X_I3:%.*]] = extractelement <4 x float> [[X]], i64 3
-; CHECK-NEXT: [[RES_I3:%.*]] = fadd float [[X_I3]], 4.000000e+00, !fpmath [[META9]]
+; CHECK-NEXT: [[RES_I3:%.*]] = fadd float [[X_I3]], 4.000000e+00, !fpmath [[META8]]
; CHECK-NEXT: [[RES_UPTO0:%.*]] = insertelement <4 x float> poison, float [[RES_I0]], i64 0
; CHECK-NEXT: [[RES_UPTO1:%.*]] = insertelement <4 x float> [[RES_UPTO0]], float [[RES_I1]], i64 1
; CHECK-NEXT: [[RES_UPTO2:%.*]] = insertelement <4 x float> [[RES_UPTO1]], float [[RES_I2]], i64 2
@@ -868,9 +868,8 @@ define <2 x i32> @f23_crash(<2 x i32> %srcvec, i32 %v1) {
; CHECK: [[META2]] = !{!"root"}
; CHECK: [[SET2_TBAA3]] = !{[[META4:![0-9]+]], [[META4]], i64 0}
; CHECK: [[META4]] = !{!"set2", [[META2]]}
-; CHECK: [[TBAA_STRUCT5]] = !{i64 0, i64 4, [[SET1_TBAA0]], i64 4, i64 4, [[SET2_TBAA3]], i64 8, i64 4, [[SET1_TBAA0]], i64 12, i64 4, [[SET2_TBAA3]]}
-; CHECK: [[ACC_GRP6]] = distinct !{}
-; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META8:![0-9]+]]}
-; CHECK: [[META8]] = !{!"llvm.loop.parallel_accesses", [[ACC_GRP6]]}
-; CHECK: [[META9]] = !{float 4.000000e+00}
+; CHECK: [[ACC_GRP5]] = distinct !{}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META7:![0-9]+]]}
+; CHECK: [[META7]] = !{!"llvm.loop.parallel_accesses", [[ACC_GRP5]]}
+; CHECK: [[META8]] = !{float 4.000000e+00}
;.
diff --git a/llvm/test/Transforms/Scalarizer/basic.ll b/llvm/test/Transforms/Scalarizer/basic.ll
index f620445612f6dd..0d55e3edbedae6 100644
--- a/llvm/test/Transforms/Scalarizer/basic.ll
+++ b/llvm/test/Transforms/Scalarizer/basic.ll
@@ -209,21 +209,21 @@ define void @f4(ptr %src, ptr %dst) {
; CHECK-NEXT: [[DST_I1:%.*]] = getelementptr i32, ptr [[DST]], i32 1
; CHECK-NEXT: [[DST_I2:%.*]] = getelementptr i32, ptr [[DST]], i32 2
; CHECK-NEXT: [[DST_I3:%.*]] = getelementptr i32, ptr [[DST]], i32 3
-; CHECK-NEXT: [[VAL_I0:%.*]] = load i32, ptr [[SRC]], align 16, !tbaa.struct [[TBAA_STRUCT5:![0-9]+]]
+; CHECK-NEXT: [[VAL_I0:%.*]] = load i32, ptr [[SRC]], align 16, !tbaa [[SET1_TBAA0]]
; CHECK-NEXT: [[SRC_I1:%.*]] = getelementptr i32, ptr [[SRC]], i32 1
-; CHECK-NEXT: [[VAL_I1:%.*]] = load i32, ptr [[SRC_I1]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
+; CHECK-NEXT: [[VAL_I1:%.*]] = load i32, ptr [[SRC_I1]], align 4, !tbaa [[SET2_TBAA3]]
; CHECK-NEXT: [[SRC_I2:%.*]] = getelementptr i32, ptr [[SRC]], i32 2
-; CHECK-NEXT: [[VAL_I2:%.*]] = load i32, ptr [[SRC_I2]], align 8, !tbaa.struct [[TBAA_STRUCT5]]
+; CHECK-NEXT: [[VAL_I2:%.*]] = load i32, ptr [[SRC_I2]], align 8, !tbaa [[SET1_TBAA0]]
; CHECK-NEXT: [[SRC_I3:%.*]] = getelementptr i32, ptr [[SRC]], i32 3
-; CHECK-NEXT: [[VAL_I3:%.*]] = load i32, ptr [[SRC_I3]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
+; CHECK-NEXT: [[VAL_I3:%.*]] = load i32, ptr [[SRC_I3]], align 4, !tbaa [[SET2_TBAA3]]
; CHECK-NEXT: [[ADD_I0:%.*]] = add i32 [[VAL_I0]], [[VAL_I0]]
; CHECK-NEXT: [[ADD_I1:%.*]] = add i32 [[VAL_I1]], [[VAL_I1]]
; CHECK-NEXT: [[ADD_I2:%.*]] = add i32 [[VAL_I2]], [[VAL_I2]]
; CHECK-NEXT: [[ADD_I3:%.*]] = add i32 [[VAL_I3]], [[VAL_I3]]
-; CHECK-NEXT: store i32 [[ADD_I0]], ptr [[DST]], align 16, !tbaa.struct [[TBAA_STRUCT5]]
-; CHECK-NEXT: store i32 [[ADD_I1]], ptr [[DST_I1]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
-; CHECK-NEXT: store i32 [[ADD_I2]], ptr [[DST_I2]], align 8, !tbaa.struct [[TBAA_STRUCT5]]
-; CHECK-NEXT: store i32 [[ADD_I3]], ptr [[DST_I3]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
+; CHECK-NEXT: store i32 [[ADD_I0]], ptr [[DST]], align 16, !tbaa [[SET1_TBAA0]]
+; CHECK-NEXT: store i32 [[ADD_I1]], ptr [[DST_I1]], align 4, !tbaa [[SET2_TBAA3]]
+; CHECK-NEXT: store i32 [[ADD_I2]], ptr [[DST_I2]], align 8, !tbaa [[SET1_TBAA0]]
+; CHECK-NEXT: store i32 [[ADD_I3]], ptr [[DST_I3]], align 4, !tbaa [[SET2_TBAA3]]
; CHECK-NEXT: ret void
;
%val = load <4 x i32> , ptr %src, !tbaa.struct !5
@@ -248,21 +248,21 @@ define void @f5(i32 %count, ptr %src, ptr %dst) {
; CHECK-NEXT: [[THIS_DST_I1:%.*]] = getelementptr i32, ptr [[THIS_DST]], i32 1
; CHECK-NEXT: [[THIS_DST_I2:%.*]] = getelementptr i32, ptr [[THIS_DST]], i32 2
; CHECK-NEXT: [[THIS_DST_I3:%.*]] = getelementptr i32, ptr [[THIS_DST]], i32 3
-; CHECK-NEXT: [[VAL_I0:%.*]] = load i32, ptr [[THIS_SRC]], align 16, !llvm.access.group [[ACC_GRP6:![0-9]+]]
-; CHECK-NEXT: [[VAL_I1:%.*]] = load i32, ptr [[THIS_SRC_I1]], align 4, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT: [[VAL_I2:%.*]] = load i32, ptr [[THIS_SRC_I2]], align 8, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT: [[VAL_I3:%.*]] = load i32, ptr [[THIS_SRC_I3]], align 4, !llvm.access.group [[ACC_GRP6]]
+; CHECK-NEXT: [[VAL_I0:%.*]] = load i32, ptr [[THIS_SRC]], align 16, !llvm.access.group [[ACC_GRP5:![0-9]+]]
+; CHECK-NEXT: [[VAL_I1:%.*]] = load i32, ptr [[THIS_SRC_I1]], align 4, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT: [[VAL_I2:%.*]] = load i32, ptr [[THIS_SRC_I2]], align 8, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT: [[VAL_I3:%.*]] = load i32, ptr [[THIS_SRC_I3]], align 4, !llvm.access.group [[ACC_GRP5]]
; CHECK-NEXT: [[ADD_I0:%.*]] = add i32 [[VAL_I0]], [[VAL_I0]]
; CHECK-NEXT: [[ADD_I1:%.*]] = add i32 [[VAL_I1]], [[VAL_I1]]
; CHECK-NEXT: [[ADD_I2:%.*]] = add i32 [[VAL_I2]], [[VAL_I2]]
; CHECK-NEXT: [[ADD_I3:%.*]] = add i32 [[VAL_I3]], [[VAL_I3]]
-; CHECK-NEXT: store i32 [[ADD_I0]], ptr [[THIS_DST]], align 16, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT: store i32 [[ADD_I1]], ptr [[THIS_DST_I1]], align 4, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT: store i32 [[ADD_I2]], ptr [[THIS_DST_I2]], align 8, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT: store i32 [[ADD_I3]], ptr [[THIS_DST_I3]], align 4, !llvm.access.group [[ACC_GRP6]]
+; CHECK-NEXT: store i32 [[ADD_I0]], ptr [[THIS_DST]], align 16, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT: store i32 [[ADD_I1]], ptr [[THIS_DST_I1]], align 4, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT: store i32 [[ADD_I2]], ptr [[THIS_DST_I2]], align 8, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT: store i32 [[ADD_I3]], ptr [[THIS_DST_I3]], align 4, !llvm.access.group [[ACC_GRP5]]
; CHECK-NEXT: [[NEXT_INDEX]] = add i32 [[INDEX]], -1
; CHECK-NEXT: [[CONTINUE:%.*]] = icmp ne i32 [[NEXT_INDEX]], [[COUNT]]
-; CHECK-NEXT: br i1 [[CONTINUE]], label %[[LOOP]], label %[[END:.*]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[CONTINUE]], label %[[LOOP]], label %[[END:.*]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[END]]:
; CHECK-NEXT: ret void
;
@@ -289,13 +289,13 @@ define <4 x float> @f6(<4 x float> %x) {
; CHECK-LABEL: define <4 x float> @f6(
; CHECK-SAME: <4 x float> [[X:%.*]]) {
; CHECK-NEXT: [[X_I0:%.*]] = extractelement <4 x float> [[X]], i64 0
-; CHECK-NEXT: [[RES_I0:%.*]] = fadd float [[X_I0]], 1.000000e+00, !fpmath [[META9:![0-9]+]]
+; CHECK-NEXT: [[RES_I0:%.*]] = fadd float [[X_I0]], 1.000000e+00, !fpmath [[META8:![0-9]+]]
; CHECK-NEXT: [[X_I1:%.*]] = extractelement <4 x float> [[X]], i64 1
-; CHECK-NEXT: [[RES_I1:%.*]] = fadd float [[X_I1]], 2.000000e+00, !fpmath [[META9]]
+; CHECK-NEXT: [[RES_I1:%.*]] = fadd float [[X_I1]], 2.000000e+00, !fpmath [[META8]]
; CHECK-NEXT: [[X_I2:%.*]] = extractelement <4 x float> [[X]], i64 2
-; CHECK-NEXT: [[RES_I2:%.*]] = fadd float [[X_I2]], 3.000000e+00, !fpmath [[META9]]
+; CHECK-NEXT: [[RES_I2:%.*]] = fadd float [[X_I2]], 3.000000e+00, !fpmath [[META8]]
; CHECK-NEXT: [[X_I3:%.*]] = extractelement <4 x float> [[X]], i64 3
-; CHECK-NEXT: [[RES_I3:%.*]] = fadd float [[X_I3]], 4.000000e+00, !fpmath [[META9]]
+; CHECK-NEXT: [[RES_I3:%.*]] = fadd float [[X_I3]], 4.000000e+00, !fpmath [[META8]]
; CHECK-NEXT: [[RES_UPTO0:%.*]] = insertelement <4 x float> poison, float [[RES_I0]], i64 0
; CHECK-NEXT: [[RES_UPTO1:%.*]] = insertelement <4 x float> [[RES_UPTO0]], float [[RES_I1]], i64 1
; CHECK-NEXT: [[RES_UPTO2:%.*]] = insertelement <4 x float> [[RES_UPTO1]], float [[RES_I2]], i64 2
@@ -919,9 +919,8 @@ define <2 x i8> @test_copy_trunc_flags(<2 x i32> %src) {
; CHECK: [[META2]] = !{!"root"}
; CHECK: [[SET2_TBAA3]] = !{[[META4:![0-9]+]], [[META4]], i64 0}
; CHECK: [[META4]] = !{!"set2", [[META2]]}
-; CHECK: [[TBAA_STRUCT5]] = !{i64 0, i64 4, [[SET1_TBAA0]], i64 4, i64 4, [[SET2_TBAA3]], i64 8, i64 4, [[SET1_TBAA0]], i64 12, i64 4, [[SET2_TBAA3]]}
-; CHECK: [[ACC_GRP6]] = distinct !{}
-; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META8:![0-9]+]]}
-; CHECK: [[META8]] = !{!"llvm.loop.parallel_accesses", [[ACC_GRP6]]}
-; CHECK: [[META9]] = !{float 4.000000e+00}
+; CHECK: [[ACC_GRP5]] = distinct !{}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META7:![0-9]+]]}
+; CHECK: [[META7]] = !{!"llvm.loop.parallel_accesses", [[ACC_GRP5]]}
+; CHECK: [[META8]] = !{float 4.000000e+00}
;.
diff --git a/llvm/test/Transforms/Scalarizer/memory-cache-hint.ll b/llvm/test/Transforms/Scalarizer/memory-cache-hint.ll
index 270b264e65961b..7275c377eab72b 100644
--- a/llvm/test/Transforms/Scalarizer/memory-cache-hint.ll
+++ b/llvm/test/Transforms/Scalarizer/memory-cache-hint.ll
@@ -4,9 +4,9 @@
define <2 x i32> @split_load(ptr %p) {
; CHECK-LABEL: define <2 x i32> @split_load(
; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[V_I0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: [[V_I0:%.*]] = load i32, ptr [[P]], align 8, !range [[RNG0:![0-9]+]], !mem.cache_hint [[META1:![0-9]+]]
; CHECK-NEXT: [[P_I1:%.*]] = getelementptr i32, ptr [[P]], i32 1
-; CHECK-NEXT: [[V_I1:%.*]] = load i32, ptr [[P_I1]], align 4
+; CHECK-NEXT: [[V_I1:%.*]] = load i32, ptr [[P_I1]], align 4, !range [[RNG0]], !mem.cache_hint [[META1]]
; CHECK-NEXT: [[V_UPTO0:%.*]] = insertelement <2 x i32> poison, i32 [[V_I0]], i64 0
; CHECK-NEXT: [[V:%.*]] = insertelement <2 x i32> [[V_UPTO0]], i32 [[V_I1]], i64 1
; CHECK-NEXT: ret <2 x i32> [[V]]
@@ -19,10 +19,10 @@ define void @split_store(ptr %p, <2 x i32> %v) {
; CHECK-LABEL: define void @split_store(
; CHECK-SAME: ptr [[P:%.*]], <2 x i32> [[V:%.*]]) {
; CHECK-NEXT: [[V_I0:%.*]] = extractelement <2 x i32> [[V]], i64 0
-; CHECK-NEXT: store i32 [[V_I0]], ptr [[P]], align 8
+; CHECK-NEXT: store i32 [[V_I0]], ptr [[P]], align 8, !mem.cache_hint [[META3:![0-9]+]]
; CHECK-NEXT: [[V_I1:%.*]] = extractelement <2 x i32> [[V]], i64 1
; CHECK-NEXT: [[P_I1:%.*]] = getelementptr i32, ptr [[P]], i32 1
-; CHECK-NEXT: store i32 [[V_I1]], ptr [[P_I1]], align 4
+; CHECK-NEXT: store i32 [[V_I1]], ptr [[P_I1]], align 4, !mem.cache_hint [[META3]]
; CHECK-NEXT: ret void
;
store <2 x i32> %v, ptr %p, align 8, !mem.cache_hint !3
@@ -33,3 +33,9 @@ define void @split_store(ptr %p, <2 x i32> %v) {
!1 = !{!"nvvm.l1_eviction", !"last"}
!2 = !{i32 0, i32 100}
!3 = !{i32 1, !1}
+;.
+; CHECK: [[RNG0]] = !{i32 0, i32 100}
+; CHECK: [[META1]] = !{i32 0, [[META2:![0-9]+]]}
+; CHECK: [[META2]] = !{!"nvvm.l1_eviction", !"last"}
+; CHECK: [[META3]] = !{i32 1, [[META2]]}
+;.
diff --git a/llvm/test/Transforms/Scalarizer/memory-metadata.ll b/llvm/test/Transforms/Scalarizer/memory-metadata.ll
index cb6197983ec51c..ef58aecaf55a61 100644
--- a/llvm/test/Transforms/Scalarizer/memory-metadata.ll
+++ b/llvm/test/Transforms/Scalarizer/memory-metadata.ll
@@ -9,13 +9,13 @@ target datalayout = "e-p:64:64"
define <4 x i32> @integer_load(ptr %p) {
; CHECK-LABEL: define <4 x i32> @integer_load(
; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[V_I0:%.*]] = load i32, ptr [[P]], align 16, !tbaa [[INT_TBAA0:![0-9]+]], !invariant.load [[META4:![0-9]+]], !alias.scope [[META5:![0-9]+]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8:![0-9]+]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_I0:%.*]] = load i32, ptr [[P]], align 16, !tbaa [[INT_TBAA0:![0-9]+]], !range [[RNG4:![0-9]+]], !invariant.load [[META5:![0-9]+]], !alias.scope [[META6:![0-9]+]], !noalias [[META6]], !nontemporal [[META9:![0-9]+]], !llvm.mem.parallel_loop_access [[META10:![0-9]+]], !invariant.group [[META5]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11:![0-9]+]], !nosanitize [[META5]], !mmra [[META12:![0-9]+]], !noalias.addrspace [[META13:![0-9]+]], !mem.cache_hint [[META14:![0-9]+]]
; CHECK-NEXT: [[P_I1:%.*]] = getelementptr i32, ptr [[P]], i32 1
-; CHECK-NEXT: [[V_I1:%.*]] = load i32, ptr [[P_I1]], align 4, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_I1:%.*]] = load i32, ptr [[P_I1]], align 4, !tbaa [[INT_TBAA0]], !range [[RNG4]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META14]]
; CHECK-NEXT: [[P_I2:%.*]] = getelementptr i32, ptr [[P]], i32 2
-; CHECK-NEXT: [[V_I2:%.*]] = load i32, ptr [[P_I2]], align 8, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_I2:%.*]] = load i32, ptr [[P_I2]], align 8, !tbaa [[INT_TBAA0]], !range [[RNG4]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META14]]
; CHECK-NEXT: [[P_I3:%.*]] = getelementptr i32, ptr [[P]], i32 3
-; CHECK-NEXT: [[V_I3:%.*]] = load i32, ptr [[P_I3]], align 4, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_I3:%.*]] = load i32, ptr [[P_I3]], align 4, !tbaa [[INT_TBAA0]], !range [[RNG4]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META14]]
; CHECK-NEXT: [[V_UPTO0:%.*]] = insertelement <4 x i32> poison, i32 [[V_I0]], i64 0
; CHECK-NEXT: [[V_UPTO1:%.*]] = insertelement <4 x i32> [[V_UPTO0]], i32 [[V_I1]], i64 1
; CHECK-NEXT: [[V_UPTO2:%.*]] = insertelement <4 x i32> [[V_UPTO1]], i32 [[V_I2]], i64 2
@@ -24,9 +24,9 @@ define <4 x i32> @integer_load(ptr %p) {
;
; PACKED-LABEL: define <4 x i32> @integer_load(
; PACKED-SAME: ptr [[P:%.*]]) {
-; PACKED-NEXT: [[V_I0:%.*]] = load <2 x i32>, ptr [[P]], align 16, !tbaa [[INT_TBAA0:![0-9]+]], !invariant.load [[META4:![0-9]+]], !alias.scope [[META5:![0-9]+]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8:![0-9]+]], !llvm.access.group [[META8]]
+; PACKED-NEXT: [[V_I0:%.*]] = load <2 x i32>, ptr [[P]], align 16, !tbaa [[INT_TBAA0:![0-9]+]], !range [[RNG4:![0-9]+]], !invariant.load [[META5:![0-9]+]], !alias.scope [[META6:![0-9]+]], !noalias [[META6]], !nontemporal [[META9:![0-9]+]], !llvm.mem.parallel_loop_access [[META10:![0-9]+]], !invariant.group [[META5]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11:![0-9]+]], !nosanitize [[META5]], !mmra [[META12:![0-9]+]], !noalias.addrspace [[META13:![0-9]+]], !mem.cache_hint [[META14:![0-9]+]]
; PACKED-NEXT: [[P_I1:%.*]] = getelementptr <2 x i32>, ptr [[P]], i32 1
-; PACKED-NEXT: [[V_I1:%.*]] = load <2 x i32>, ptr [[P_I1]], align 8, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT: [[V_I1:%.*]] = load <2 x i32>, ptr [[P_I1]], align 8, !tbaa [[INT_TBAA0]], !range [[RNG4]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META14]]
; PACKED-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[V_I0]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; PACKED-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[V_I1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; PACKED-NEXT: [[V:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
@@ -39,13 +39,13 @@ define <4 x i32> @integer_load(ptr %p) {
define <4 x float> @floating_load(ptr %p) {
; CHECK-LABEL: define <4 x float> @floating_load(
; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[V_I0:%.*]] = load float, ptr [[P]], align 16, !tbaa [[FLOAT_TBAA9:![0-9]+]], !fpmath [[META11:![0-9]+]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_I0:%.*]] = load float, ptr [[P]], align 16, !tbaa [[FLOAT_TBAA16:![0-9]+]], !fpmath [[META18:![0-9]+]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !invariant.group [[META5]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !nofpclass [[META19:![0-9]+]], !mem.cache_hint [[META14]]
; CHECK-NEXT: [[P_I1:%.*]] = getelementptr float, ptr [[P]], i32 1
-; CHECK-NEXT: [[V_I1:%.*]] = load float, ptr [[P_I1]], align 4, !tbaa [[FLOAT_TBAA9]], !fpmath [[META11]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_I1:%.*]] = load float, ptr [[P_I1]], align 4, !tbaa [[FLOAT_TBAA16]], !fpmath [[META18]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !nofpclass [[META19]], !mem.cache_hint [[META14]]
; CHECK-NEXT: [[P_I2:%.*]] = getelementptr float, ptr [[P]], i32 2
-; CHECK-NEXT: [[V_I2:%.*]] = load float, ptr [[P_I2]], align 8, !tbaa [[FLOAT_TBAA9]], !fpmath [[META11]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_I2:%.*]] = load float, ptr [[P_I2]], align 8, !tbaa [[FLOAT_TBAA16]], !fpmath [[META18]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !nofpclass [[META19]], !mem.cache_hint [[META14]]
; CHECK-NEXT: [[P_I3:%.*]] = getelementptr float, ptr [[P]], i32 3
-; CHECK-NEXT: [[V_I3:%.*]] = load float, ptr [[P_I3]], align 4, !tbaa [[FLOAT_TBAA9]], !fpmath [[META11]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_I3:%.*]] = load float, ptr [[P_I3]], align 4, !tbaa [[FLOAT_TBAA16]], !fpmath [[META18]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !nofpclass [[META19]], !mem.cache_hint [[META14]]
; CHECK-NEXT: [[V_UPTO0:%.*]] = insertelement <4 x float> poison, float [[V_I0]], i64 0
; CHECK-NEXT: [[V_UPTO1:%.*]] = insertelement <4 x float> [[V_UPTO0]], float [[V_I1]], i64 1
; CHECK-NEXT: [[V_UPTO2:%.*]] = insertelement <4 x float> [[V_UPTO1]], float [[V_I2]], i64 2
@@ -54,9 +54,9 @@ define <4 x float> @floating_load(ptr %p) {
;
; PACKED-LABEL: define <4 x float> @floating_load(
; PACKED-SAME: ptr [[P:%.*]]) {
-; PACKED-NEXT: [[V_I0:%.*]] = load <2 x float>, ptr [[P]], align 16, !tbaa [[FLOAT_TBAA9:![0-9]+]], !fpmath [[META11:![0-9]+]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT: [[V_I0:%.*]] = load <2 x float>, ptr [[P]], align 16, !tbaa [[FLOAT_TBAA16:![0-9]+]], !fpmath [[META18:![0-9]+]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !invariant.group [[META5]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !nofpclass [[META19:![0-9]+]], !mem.cache_hint [[META14]]
; PACKED-NEXT: [[P_I1:%.*]] = getelementptr <2 x float>, ptr [[P]], i32 1
-; PACKED-NEXT: [[V_I1:%.*]] = load <2 x float>, ptr [[P_I1]], align 8, !tbaa [[FLOAT_TBAA9]], !fpmath [[META11]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT: [[V_I1:%.*]] = load <2 x float>, ptr [[P_I1]], align 8, !tbaa [[FLOAT_TBAA16]], !fpmath [[META18]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !nofpclass [[META19]], !mem.cache_hint [[META14]]
; PACKED-NEXT: [[TMP1:%.*]] = shufflevector <2 x float> [[V_I0]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; PACKED-NEXT: [[TMP2:%.*]] = shufflevector <2 x float> [[V_I1]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; PACKED-NEXT: [[V:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
@@ -70,25 +70,25 @@ define void @integer_store(ptr %p, <4 x i32> %v) {
; CHECK-LABEL: define void @integer_store(
; CHECK-SAME: ptr [[P:%.*]], <4 x i32> [[V:%.*]]) {
; CHECK-NEXT: [[V_I0:%.*]] = extractelement <4 x i32> [[V]], i64 0
-; CHECK-NEXT: store i32 [[V_I0]], ptr [[P]], align 16, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: store i32 [[V_I0]], ptr [[P]], align 16, !tbaa [[INT_TBAA0]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !invariant.group [[META5]], !llvm.access.group [[META10]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META20:![0-9]+]]
; CHECK-NEXT: [[V_I1:%.*]] = extractelement <4 x i32> [[V]], i64 1
; CHECK-NEXT: [[P_I1:%.*]] = getelementptr i32, ptr [[P]], i32 1
-; CHECK-NEXT: store i32 [[V_I1]], ptr [[P_I1]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: store i32 [[V_I1]], ptr [[P_I1]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META20]]
; CHECK-NEXT: [[V_I2:%.*]] = extractelement <4 x i32> [[V]], i64 2
; CHECK-NEXT: [[P_I2:%.*]] = getelementptr i32, ptr [[P]], i32 2
-; CHECK-NEXT: store i32 [[V_I2]], ptr [[P_I2]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: store i32 [[V_I2]], ptr [[P_I2]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META20]]
; CHECK-NEXT: [[V_I3:%.*]] = extractelement <4 x i32> [[V]], i64 3
; CHECK-NEXT: [[P_I3:%.*]] = getelementptr i32, ptr [[P]], i32 3
-; CHECK-NEXT: store i32 [[V_I3]], ptr [[P_I3]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: store i32 [[V_I3]], ptr [[P_I3]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META20]]
; CHECK-NEXT: ret void
;
; PACKED-LABEL: define void @integer_store(
; PACKED-SAME: ptr [[P:%.*]], <4 x i32> [[V:%.*]]) {
; PACKED-NEXT: [[V_I0:%.*]] = shufflevector <4 x i32> [[V]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>
-; PACKED-NEXT: store <2 x i32> [[V_I0]], ptr [[P]], align 16, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT: store <2 x i32> [[V_I0]], ptr [[P]], align 16, !tbaa [[INT_TBAA0]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !invariant.group [[META5]], !llvm.access.group [[META10]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META20:![0-9]+]]
; PACKED-NEXT: [[V_I1:%.*]] = shufflevector <4 x i32> [[V]], <4 x i32> poison, <2 x i32> <i32 2, i32 3>
; PACKED-NEXT: [[P_I1:%.*]] = getelementptr <2 x i32>, ptr [[P]], i32 1
-; PACKED-NEXT: store <2 x i32> [[V_I1]], ptr [[P_I1]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT: store <2 x i32> [[V_I1]], ptr [[P_I1]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META20]]
; PACKED-NEXT: ret void
;
store <4 x i32> %v, ptr %p, align 16, !tbaa !10, !invariant.group !0, !mem.cache_hint !18, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
@@ -126,25 +126,43 @@ define void @integer_store(ptr %p, <4 x i32> %v) {
; CHECK: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
; CHECK: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
; CHECK: [[META3]] = !{!"root"}
-; CHECK: [[META4]] = !{}
-; CHECK: [[META5]] = !{[[META6:![0-9]+]]}
-; CHECK: [[META6]] = distinct !{[[META6]], [[META7:![0-9]+]], !"scope"}
-; CHECK: [[META7]] = distinct !{[[META7]], i1 false, !"domain"}
-; CHECK: [[META8]] = distinct !{}
-; CHECK: [[FLOAT_TBAA9]] = !{[[META10:![0-9]+]], [[META10]], i64 0}
-; CHECK: [[META10]] = !{!"float", [[META2]], i64 0}
-; CHECK: [[META11]] = !{float 2.500000e+00}
+; CHECK: [[RNG4]] = !{i32 0, i32 100}
+; CHECK: [[META5]] = !{}
+; CHECK: [[META6]] = !{[[META7:![0-9]+]]}
+; CHECK: [[META7]] = distinct !{[[META7]], [[META8:![0-9]+]], !"scope"}
+; CHECK: [[META8]] = distinct !{[[META8]], i1 false, !"domain"}
+; CHECK: [[META9]] = !{i32 1}
+; CHECK: [[META10]] = distinct !{}
+; CHECK: [[META11]] = !{!"preserved memory metadata"}
+; CHECK: [[META12]] = !{!"test", !"relaxed"}
+; CHECK: [[META13]] = !{i32 5, i32 6}
+; CHECK: [[META14]] = !{i32 0, [[META15:![0-9]+]]}
+; CHECK: [[META15]] = !{!"nvvm.l1_eviction", !"last"}
+; CHECK: [[FLOAT_TBAA16]] = !{[[META17:![0-9]+]], [[META17]], i64 0}
+; CHECK: [[META17]] = !{!"float", [[META2]], i64 0}
+; CHECK: [[META18]] = !{float 2.500000e+00}
+; CHECK: [[META19]] = !{i32 3}
+; CHECK: [[META20]] = !{i32 1, [[META15]]}
;.
; PACKED: [[INT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
; PACKED: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
; PACKED: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
; PACKED: [[META3]] = !{!"root"}
-; PACKED: [[META4]] = !{}
-; PACKED: [[META5]] = !{[[META6:![0-9]+]]}
-; PACKED: [[META6]] = distinct !{[[META6]], [[META7:![0-9]+]], !"scope"}
-; PACKED: [[META7]] = distinct !{[[META7]], i1 false, !"domain"}
-; PACKED: [[META8]] = distinct !{}
-; PACKED: [[FLOAT_TBAA9]] = !{[[META10:![0-9]+]], [[META10]], i64 0}
-; PACKED: [[META10]] = !{!"float", [[META2]], i64 0}
-; PACKED: [[META11]] = !{float 2.500000e+00}
+; PACKED: [[RNG4]] = !{i32 0, i32 100}
+; PACKED: [[META5]] = !{}
+; PACKED: [[META6]] = !{[[META7:![0-9]+]]}
+; PACKED: [[META7]] = distinct !{[[META7]], [[META8:![0-9]+]], !"scope"}
+; PACKED: [[META8]] = distinct !{[[META8]], i1 false, !"domain"}
+; PACKED: [[META9]] = !{i32 1}
+; PACKED: [[META10]] = distinct !{}
+; PACKED: [[META11]] = !{!"preserved memory metadata"}
+; PACKED: [[META12]] = !{!"test", !"relaxed"}
+; PACKED: [[META13]] = !{i32 5, i32 6}
+; PACKED: [[META14]] = !{i32 0, [[META15:![0-9]+]]}
+; PACKED: [[META15]] = !{!"nvvm.l1_eviction", !"last"}
+; PACKED: [[FLOAT_TBAA16]] = !{[[META17:![0-9]+]], [[META17]], i64 0}
+; PACKED: [[META17]] = !{!"float", [[META2]], i64 0}
+; PACKED: [[META18]] = !{float 2.500000e+00}
+; PACKED: [[META19]] = !{i32 3}
+; PACKED: [[META20]] = !{i32 1, [[META15]]}
;.
diff --git a/llvm/test/Transforms/VectorCombine/RISCV/memory-metadata.ll b/llvm/test/Transforms/VectorCombine/RISCV/memory-metadata.ll
index 4c29c342143cd8..2bebe520189884 100644
--- a/llvm/test/Transforms/VectorCombine/RISCV/memory-metadata.ll
+++ b/llvm/test/Transforms/VectorCombine/RISCV/memory-metadata.ll
@@ -5,7 +5,7 @@ define <8 x i16> @vp_bitcast_fixed(ptr %p, i32 %evl) {
; CHECK-LABEL: define <8 x i16> @vp_bitcast_fixed(
; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i32 [[EVL]], 2
-; CHECK-NEXT: [[R:%.*]] = call <8 x i16> @llvm.vp.load.v8i16.p0(ptr align 4 [[P]], <8 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: [[R:%.*]] = call <8 x i16> @llvm.vp.load.v8i16.p0(ptr align 4 [[P]], <8 x i1> splat (i1 true), i32 [[TMP1]]), !invariant.load [[META0:![0-9]+]], !alias.scope [[META1:![0-9]+]], !noalias [[META1]], !nontemporal [[META4:![0-9]+]], !llvm.access.group [[ACC_GRP5:![0-9]+]], !annotation [[META6:![0-9]+]], !nosanitize [[META0]], !mmra [[META7:![0-9]+]], !noalias.addrspace [[META8:![0-9]+]], !mem.cache_hint [[META9:![0-9]+]]
; CHECK-NEXT: ret <8 x i16> [[R]]
;
%v = call <4 x i32> @llvm.vp.load(ptr align 4 %p, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, i32 %evl), !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !range !11, !invariant.load !3
@@ -17,7 +17,7 @@ define <vscale x 8 x i16> @vp_bitcast_scalable(ptr %p, i32 %evl) {
; CHECK-LABEL: define <vscale x 8 x i16> @vp_bitcast_scalable(
; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i32 [[EVL]], 2
-; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 4 [[P]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 4 [[P]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP1]]), !invariant.load [[META0]], !alias.scope [[META1]], !noalias [[META1]], !nontemporal [[META4]], !llvm.access.group [[ACC_GRP5]], !annotation [[META6]], !nosanitize [[META0]], !mmra [[META7]], !noalias.addrspace [[META8]], !mem.cache_hint [[META9]]
; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
;
%v = call <vscale x 4 x i32> @llvm.vp.load(ptr align 4 %p, <vscale x 4 x i1> splat (i1 true), i32 %evl), !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !range !11, !invariant.load !3
@@ -45,3 +45,16 @@ define <vscale x 8 x i16> @vp_bitcast_scalable(ptr %p, i32 %evl) {
!16 = !{!"omnipotent char", !17, i64 0}
!17 = !{!"root"}
!18 = !{i32 1, !2}
+;.
+; CHECK: [[META0]] = !{}
+; CHECK: [[META1]] = !{[[META2:![0-9]+]]}
+; CHECK: [[META2]] = distinct !{[[META2]], [[META3:![0-9]+]], !"scope"}
+; CHECK: [[META3]] = distinct !{[[META3]], i1 false, !"domain"}
+; CHECK: [[META4]] = !{i32 1}
+; CHECK: [[ACC_GRP5]] = distinct !{}
+; CHECK: [[META6]] = !{!"original access"}
+; CHECK: [[META7]] = !{!"test", !"tag"}
+; CHECK: [[META8]] = !{i32 5, i32 6}
+; CHECK: [[META9]] = !{i32 0, [[META10:![0-9]+]]}
+; CHECK: [[META10]] = !{!"nvvm.l1_eviction", !"last"}
+;.
diff --git a/llvm/test/Transforms/VectorCombine/X86/memory-metadata.ll b/llvm/test/Transforms/VectorCombine/X86/memory-metadata.ll
index e9f73bbdff4e2a..6563c3becfbef6 100644
--- a/llvm/test/Transforms/VectorCombine/X86/memory-metadata.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/memory-metadata.ll
@@ -4,7 +4,7 @@
define <4 x float> @widen_scalar(ptr align 16 dereferenceable(16) %p) {
; CHECK-LABEL: define <4 x float> @widen_scalar(
; CHECK-SAME: ptr align 16 dereferenceable(16) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[R:%.*]] = load <4 x float>, ptr [[P]], align 16
+; CHECK-NEXT: [[R:%.*]] = load <4 x float>, ptr [[P]], align 16, !nontemporal [[META0:![0-9]+]], !annotation [[META1:![0-9]+]], !nosanitize [[META2:![0-9]+]], !mmra [[META3:![0-9]+]], !noalias.addrspace [[META4:![0-9]+]], !mem.cache_hint [[META5:![0-9]+]]
; CHECK-NEXT: ret <4 x float> [[R]]
;
%v = load float, ptr %p, align 4, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !nofpclass !12, !fpmath !13, !noundef !3, !invariant.load !3, !invariant.group !3, !tbaa !15
@@ -15,7 +15,7 @@ define <4 x float> @widen_scalar(ptr align 16 dereferenceable(16) %p) {
define <4 x float> @widen_shifted_scalar(ptr align 16 dereferenceable(16) %p) {
; CHECK-LABEL: define <4 x float> @widen_shifted_scalar(
; CHECK-SAME: ptr align 16 dereferenceable(16) [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[P]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[P]], align 16, !nontemporal [[META0]], !annotation [[META1]], !nosanitize [[META2]], !mmra [[META3]], !noalias.addrspace [[META4]], !mem.cache_hint [[META5]]
; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 3, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: ret <4 x float> [[R]]
;
@@ -28,7 +28,7 @@ define <4 x float> @widen_shifted_scalar(ptr align 16 dereferenceable(16) %p) {
define <4 x i32> @widen_subvector(ptr align 16 dereferenceable(16) %p) {
; CHECK-LABEL: define <4 x i32> @widen_subvector(
; CHECK-SAME: ptr align 16 dereferenceable(16) [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[R:%.*]] = load <4 x i32>, ptr [[P]], align 16
+; CHECK-NEXT: [[R:%.*]] = load <4 x i32>, ptr [[P]], align 16, !nontemporal [[META0]], !annotation [[META1]], !nosanitize [[META2]], !mmra [[META3]], !noalias.addrspace [[META4]], !mem.cache_hint [[META5]]
; CHECK-NEXT: ret <4 x i32> [[R]]
;
%v = load <2 x i32>, ptr %p, align 4, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !range !11, !noundef !3, !invariant.load !3, !invariant.group !3, !tbaa !15
@@ -42,7 +42,7 @@ define <4 x i32> @widen_subvector(ptr align 16 dereferenceable(16) %p) {
define <4 x i32> @shrink_load_insert(ptr align 16 dereferenceable(32) %p) {
; CHECK-LABEL: define <4 x i32> @shrink_load_insert(
; CHECK-SAME: ptr align 16 dereferenceable(32) [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[R:%.*]] = load <4 x i32>, ptr [[P]], align 16
+; CHECK-NEXT: [[R:%.*]] = load <4 x i32>, ptr [[P]], align 16, !nontemporal [[META0]], !mem.cache_hint [[META5]]
; CHECK-NEXT: ret <4 x i32> [[R]]
;
%v = load <8 x i32>, ptr %p, align 16, !nontemporal !0, !mem.cache_hint !1, !range !11, !noundef !3, !invariant.load !3, !alias.scope !7
@@ -70,3 +70,12 @@ define <4 x i32> @shrink_load_insert(ptr align 16 dereferenceable(32) %p) {
!16 = !{!"omnipotent char", !17, i64 0}
!17 = !{!"root"}
!18 = !{i32 1, !2}
+;.
+; CHECK: [[META0]] = !{i32 1}
+; CHECK: [[META1]] = !{!"original access"}
+; CHECK: [[META2]] = !{}
+; CHECK: [[META3]] = !{!"test", !"tag"}
+; CHECK: [[META4]] = !{i32 5, i32 6}
+; CHECK: [[META5]] = !{i32 0, [[META6:![0-9]+]]}
+; CHECK: [[META6]] = !{!"nvvm.l1_eviction", !"last"}
+;.
diff --git a/llvm/test/Transforms/VectorCombine/load-bitcast-metadata.ll b/llvm/test/Transforms/VectorCombine/load-bitcast-metadata.ll
new file mode 100644
index 00000000000000..ea2d32527f8f6d
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/load-bitcast-metadata.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=vector-combine -data-layout=e-p:64:64 < %s | FileCheck %s
+; RUN: opt -S -passes=instcombine -data-layout=e-p:64:64 < %s | FileCheck %s --check-prefix=IC
+; RUN: opt -S -passes=vector-combine -data-layout=E-p:64:64 < %s | FileCheck %s
+
+; The old element range does not describe the reinterpreted i32 value.
+define i32 @bitcast_integer_range(ptr %p) {
+; CHECK-LABEL: define i32 @bitcast_integer_range(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[P]], align 4, !nontemporal [[META0:![0-9]+]], !noundef [[META1:![0-9]+]], !annotation [[META2:![0-9]+]], !nosanitize [[META1]], !mmra [[META3:![0-9]+]], !mem.cache_hint [[META4:![0-9]+]]
+; CHECK-NEXT: ret i32 [[R]]
+;
+; IC-LABEL: define i32 @bitcast_integer_range(
+; IC-SAME: ptr [[P:%.*]]) {
+; IC-NEXT: [[V1:%.*]] = load i32, ptr [[P]], align 4, !nontemporal [[META0:![0-9]+]], !noundef [[META1:![0-9]+]], !annotation [[META2:![0-9]+]], !nosanitize [[META1]], !mmra [[META3:![0-9]+]], !mem.cache_hint [[META4:![0-9]+]]
+; IC-NEXT: ret i32 [[V1]]
+;
+ %v = load <4 x i8>, ptr %p, align 4, !range !0, !noundef !1, !nontemporal !2, !mem.cache_hint !3, !annotation !5, !nosanitize !1, !mmra !8
+ %r = bitcast <4 x i8> %v to i32
+ ret i32 %r
+}
+
+; Floating-point value metadata must not be attached to an integer load.
+define i64 @bitcast_float_to_integer(ptr %p) {
+; CHECK-LABEL: define i64 @bitcast_float_to_integer(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = load i64, ptr [[P]], align 8, !invariant.load [[META1]], !invariant.group [[META1]], !noundef [[META1]]
+; CHECK-NEXT: ret i64 [[R]]
+;
+; IC-LABEL: define i64 @bitcast_float_to_integer(
+; IC-SAME: ptr [[P:%.*]]) {
+; IC-NEXT: [[V1:%.*]] = load i64, ptr [[P]], align 8, !invariant.load [[META1]], !invariant.group [[META1]], !noundef [[META1]]
+; IC-NEXT: ret i64 [[V1]]
+;
+ %v = load <2 x float>, ptr %p, align 8, !nofpclass !6, !fpmath !7, !noundef !1, !invariant.load !1, !invariant.group !1
+ %r = bitcast <2 x float> %v to i64
+ ret i64 %r
+}
+
+; A single-element vector and its scalar have the same element facts.
+define float @bitcast_same_float_element(ptr %p) {
+; CHECK-LABEL: define float @bitcast_same_float_element(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = load float, ptr [[P]], align 4, !fpmath [[META6:![0-9]+]], !invariant.load [[META1]], !invariant.group [[META1]], !noundef [[META1]], !nofpclass [[META7:![0-9]+]]
+; CHECK-NEXT: ret float [[R]]
+;
+; IC-LABEL: define float @bitcast_same_float_element(
+; IC-SAME: ptr [[P:%.*]]) {
+; IC-NEXT: [[V1:%.*]] = load float, ptr [[P]], align 4, !fpmath [[META6:![0-9]+]], !invariant.load [[META1]], !invariant.group [[META1]], !noundef [[META1]], !nofpclass [[META7:![0-9]+]]
+; IC-NEXT: ret float [[V1]]
+;
+ %v = load <1 x float>, ptr %p, align 4, !nofpclass !6, !fpmath !7, !noundef !1, !invariant.load !1, !invariant.group !1
+ %r = bitcast <1 x float> %v to float
+ ret float %r
+}
+
+!0 = !{i8 0, i8 100}
+!1 = !{}
+!2 = !{i32 1}
+!3 = !{i32 0, !4}
+!4 = !{!"nvvm.l1_eviction", !"last"}
+!5 = !{!"original access"}
+!6 = !{i32 3}
+!7 = !{float 2.5}
+
+!8 = !{!"test", !"tag"}
+;.
+; CHECK: [[META0]] = !{i32 1}
+; CHECK: [[META1]] = !{}
+; CHECK: [[META2]] = !{!"original access"}
+; CHECK: [[META3]] = !{!"test", !"tag"}
+; CHECK: [[META4]] = !{i32 0, [[META5:![0-9]+]]}
+; CHECK: [[META5]] = !{!"nvvm.l1_eviction", !"last"}
+; CHECK: [[META6]] = !{float 2.500000e+00}
+; CHECK: [[META7]] = !{i32 3}
+;.
+; IC: [[META0]] = !{i32 1}
+; IC: [[META1]] = !{}
+; IC: [[META2]] = !{!"original access"}
+; IC: [[META3]] = !{!"test", !"tag"}
+; IC: [[META4]] = !{i32 0, [[META5:![0-9]+]]}
+; IC: [[META5]] = !{!"nvvm.l1_eviction", !"last"}
+; IC: [[META6]] = !{float 2.500000e+00}
+; IC: [[META7]] = !{i32 3}
+;.
diff --git a/llvm/test/Transforms/VectorCombine/memory-metadata.ll b/llvm/test/Transforms/VectorCombine/memory-metadata.ll
index d01dffceeb940a..cbf704e8780f45 100644
--- a/llvm/test/Transforms/VectorCombine/memory-metadata.ll
+++ b/llvm/test/Transforms/VectorCombine/memory-metadata.ll
@@ -5,7 +5,7 @@
define i32 @extract_zero(ptr %p) {
; CHECK-LABEL: define i32 @extract_zero(
; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[P]], align 16, !tbaa [[CHAR_TBAA0:![0-9]+]], !alias.scope [[META3:![0-9]+]], !noalias [[META3]], !noalias.addrspace [[META6:![0-9]+]]
+; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[P]], align 16, !tbaa [[CHAR_TBAA0:![0-9]+]], !range [[RNG3:![0-9]+]], !invariant.load [[META4:![0-9]+]], !alias.scope [[META5:![0-9]+]], !noalias [[META5]], !nontemporal [[META8:![0-9]+]], !invariant.group [[META4]], !llvm.access.group [[ACC_GRP9:![0-9]+]], !noundef [[META4]], !annotation [[META10:![0-9]+]], !nosanitize [[META4]], !mmra [[META11:![0-9]+]], !noalias.addrspace [[META12:![0-9]+]], !mem.cache_hint [[META13:![0-9]+]]
; CHECK-NEXT: ret i32 [[R]]
;
%v = load <4 x i32>, ptr %p, align 16, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !range !11, !noundef !3, !invariant.load !3, !invariant.group !3, !tbaa.struct !14
@@ -17,7 +17,7 @@ define i32 @extract_nonzero(ptr %p) {
; CHECK-LABEL: define i32 @extract_nonzero(
; CHECK-SAME: ptr [[P:%.*]]) {
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 2
-; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP1]], align 8, !tbaa [[CHAR_TBAA0]], !alias.scope [[META3]], !noalias [[META3]], !noalias.addrspace [[META6]]
+; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP1]], align 8, !tbaa [[CHAR_TBAA0]], !range [[RNG3]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.access.group [[ACC_GRP9]], !noundef [[META4]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META13]]
; CHECK-NEXT: ret i32 [[R]]
;
%v = load <4 x i32>, ptr %p, align 16, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !range !11, !noundef !3, !invariant.load !3, !invariant.group !3, !tbaa.struct !14
@@ -32,7 +32,7 @@ define i32 @extract_dynamic(ptr %p, i32 %index) {
; CHECK-NEXT: [[IDX:%.*]] = and i32 [[INDEX_FROZEN]], 3
; CHECK-NEXT: [[IDX_GEPIDX:%.*]] = zext i32 [[IDX]] to i64
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 [[IDX_GEPIDX]]
-; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[CHAR_TBAA0]], !range [[RNG3]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.access.group [[ACC_GRP9]], !noundef [[META4]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META13]]
; CHECK-NEXT: ret i32 [[R]]
;
%idx = and i32 %index, 3
@@ -45,7 +45,7 @@ define float @extract_float(ptr %p) {
; CHECK-LABEL: define float @extract_float(
; CHECK-SAME: ptr [[P:%.*]]) {
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x float>, ptr [[P]], i64 0, i64 1
-; CHECK-NEXT: [[R:%.*]] = load float, ptr [[TMP1]], align 4, !alias.scope [[META3]], !noalias [[META3]], !noalias.addrspace [[META6]]
+; CHECK-NEXT: [[R:%.*]] = load float, ptr [[TMP1]], align 4, !fpmath [[META15:![0-9]+]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.access.group [[ACC_GRP9]], !noundef [[META4]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !nofpclass [[META16:![0-9]+]], !mem.cache_hint [[META13]]
; CHECK-NEXT: ret float [[R]]
;
%v = load <4 x float>, ptr %p, align 16, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !nofpclass !12, !fpmath !13, !noundef !3, !invariant.load !3, !invariant.group !3
@@ -56,7 +56,7 @@ define float @extract_float(ptr %p) {
define <4 x i32> @shrink_integer(ptr %p) {
; CHECK-LABEL: define <4 x i32> @shrink_integer(
; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 32, !tbaa [[CHAR_TBAA0]], !range [[RNG7:![0-9]+]], !invariant.load [[META8:![0-9]+]], !alias.scope [[META3]], !noalias [[META3]], !nontemporal [[META9:![0-9]+]], !invariant.group [[META8]], !llvm.access.group [[ACC_GRP10:![0-9]+]], !noundef [[META8]], !annotation [[META11:![0-9]+]], !nosanitize [[META8]], !mmra [[META12:![0-9]+]], !noalias.addrspace [[META6]], !mem.cache_hint [[META13:![0-9]+]]
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 32, !tbaa [[CHAR_TBAA0]], !range [[RNG3]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !invariant.group [[META4]], !llvm.access.group [[ACC_GRP9]], !noundef [[META4]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META13]]
; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
; CHECK-NEXT: ret <4 x i32> [[R]]
;
@@ -68,7 +68,7 @@ define <4 x i32> @shrink_integer(ptr %p) {
define <4 x float> @shrink_float(ptr %p) {
; CHECK-LABEL: define <4 x float> @shrink_float(
; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[P]], align 32, !fpmath [[META15:![0-9]+]], !invariant.load [[META8]], !alias.scope [[META3]], !noalias [[META3]], !nontemporal [[META9]], !invariant.group [[META8]], !llvm.access.group [[ACC_GRP10]], !noundef [[META8]], !annotation [[META11]], !nosanitize [[META8]], !mmra [[META12]], !noalias.addrspace [[META6]], !nofpclass [[META16:![0-9]+]], !mem.cache_hint [[META13]]
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[P]], align 32, !fpmath [[META15]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !invariant.group [[META4]], !llvm.access.group [[ACC_GRP9]], !noundef [[META4]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !nofpclass [[META16]], !mem.cache_hint [[META13]]
; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
; CHECK-NEXT: ret <4 x float> [[R]]
;
@@ -80,7 +80,7 @@ define <4 x float> @shrink_float(ptr %p) {
define void @store_zero(ptr %p, i8 %x, i32 %index) {
; CHECK-LABEL: define void @store_zero(
; CHECK-SAME: ptr [[P:%.*]], i8 [[X:%.*]], i32 [[INDEX:%.*]]) {
-; CHECK-NEXT: store i8 [[X]], ptr [[P]], align 16, !tbaa.struct [[TBAA_STRUCT17:![0-9]+]], !alias.scope [[META3]], !noalias [[META3]], !nontemporal [[META9]], !llvm.access.group [[ACC_GRP10]], !annotation [[META11]], !nosanitize [[META8]], !mmra [[META12]], !noalias.addrspace [[META6]], !mem.cache_hint [[META18:![0-9]+]]
+; CHECK-NEXT: store i8 [[X]], ptr [[P]], align 16, !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !invariant.group [[META4]], !llvm.access.group [[ACC_GRP9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META17:![0-9]+]]
; CHECK-NEXT: ret void
;
%v = load <16 x i8>, ptr %p, align 16
@@ -93,7 +93,7 @@ define void @store_nonzero(ptr %p, i8 %x, i32 %index) {
; CHECK-LABEL: define void @store_nonzero(
; CHECK-SAME: ptr [[P:%.*]], i8 [[X:%.*]], i32 [[INDEX:%.*]]) {
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 3
-; CHECK-NEXT: store i8 [[X]], ptr [[TMP1]], align 1, !tbaa.struct [[TBAA_STRUCT17]], !alias.scope [[META3]], !noalias [[META3]], !nontemporal [[META9]], !llvm.access.group [[ACC_GRP10]], !annotation [[META11]], !nosanitize [[META8]], !mmra [[META12]], !noalias.addrspace [[META6]], !mem.cache_hint [[META18]]
+; CHECK-NEXT: store i8 [[X]], ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.access.group [[ACC_GRP9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META17]]
; CHECK-NEXT: ret void
;
%v = load <16 x i8>, ptr %p, align 16
@@ -109,7 +109,7 @@ define void @store_dynamic(ptr %p, i8 %x, i32 %index) {
; CHECK-NEXT: [[IDX:%.*]] = and i32 [[INDEX_FROZEN]], 15
; CHECK-NEXT: [[IDX_GEPIDX:%.*]] = zext i32 [[IDX]] to i64
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 [[IDX_GEPIDX]]
-; CHECK-NEXT: store i8 [[X]], ptr [[TMP1]], align 1, !tbaa.struct [[TBAA_STRUCT17]], !alias.scope [[META3]], !noalias [[META3]], !nontemporal [[META9]], !llvm.access.group [[ACC_GRP10]], !annotation [[META11]], !nosanitize [[META8]], !mmra [[META12]], !noalias.addrspace [[META6]], !mem.cache_hint [[META18]]
+; CHECK-NEXT: store i8 [[X]], ptr [[TMP1]], align 1, !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.access.group [[ACC_GRP9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META17]]
; CHECK-NEXT: ret void
;
%idx = and i32 %index, 15
@@ -125,7 +125,7 @@ define i32 @extract_struct_field(ptr %p) {
; CHECK-LABEL: define i32 @extract_struct_field(
; CHECK-SAME: ptr [[P:%.*]]) {
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 1
-; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[INT_TBAA19:![0-9]+]]
+; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[INT_TBAA18:![0-9]+]]
; CHECK-NEXT: ret i32 [[R]]
;
%v = load <4 x i32>, ptr %p, align 16, !tbaa.struct !21
@@ -137,7 +137,7 @@ define void @store_struct_field(ptr %p, i32 %x) {
; CHECK-LABEL: define void @store_struct_field(
; CHECK-SAME: ptr [[P:%.*]], i32 [[X:%.*]]) {
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 1
-; CHECK-NEXT: store i32 [[X]], ptr [[TMP1]], align 4, !tbaa.struct [[TBAA_STRUCT21:![0-9]+]]
+; CHECK-NEXT: store i32 [[X]], ptr [[TMP1]], align 4, !tbaa [[INT_TBAA18]]
; CHECK-NEXT: ret void
;
%v = load <4 x i32>, ptr %p, align 16
@@ -173,23 +173,21 @@ define void @store_struct_field(ptr %p, i32 %x) {
; CHECK: [[CHAR_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
; CHECK: [[META1]] = !{!"omnipotent char", [[META2:![0-9]+]], i64 0}
; CHECK: [[META2]] = !{!"root"}
-; CHECK: [[META3]] = !{[[META4:![0-9]+]]}
-; CHECK: [[META4]] = distinct !{[[META4]], [[META5:![0-9]+]], !"scope"}
-; CHECK: [[META5]] = distinct !{[[META5]], i1 false, !"domain"}
-; CHECK: [[META6]] = !{i32 5, i32 6}
-; CHECK: [[RNG7]] = !{i32 0, i32 100}
-; CHECK: [[META8]] = !{}
-; CHECK: [[META9]] = !{i32 1}
-; CHECK: [[ACC_GRP10]] = distinct !{}
-; CHECK: [[META11]] = !{!"original access"}
-; CHECK: [[META12]] = !{!"test", !"tag"}
+; CHECK: [[RNG3]] = !{i32 0, i32 100}
+; CHECK: [[META4]] = !{}
+; CHECK: [[META5]] = !{[[META6:![0-9]+]]}
+; CHECK: [[META6]] = distinct !{[[META6]], [[META7:![0-9]+]], !"scope"}
+; CHECK: [[META7]] = distinct !{[[META7]], i1 false, !"domain"}
+; CHECK: [[META8]] = !{i32 1}
+; CHECK: [[ACC_GRP9]] = distinct !{}
+; CHECK: [[META10]] = !{!"original access"}
+; CHECK: [[META11]] = !{!"test", !"tag"}
+; CHECK: [[META12]] = !{i32 5, i32 6}
; CHECK: [[META13]] = !{i32 0, [[META14:![0-9]+]]}
; CHECK: [[META14]] = !{!"nvvm.l1_eviction", !"last"}
; CHECK: [[META15]] = !{float 2.500000e+00}
; CHECK: [[META16]] = !{i32 3}
-; CHECK: [[TBAA_STRUCT17]] = !{i64 0, i64 4, [[CHAR_TBAA0]], i64 4, i64 4, [[CHAR_TBAA0]], i64 8, i64 4, [[CHAR_TBAA0]], i64 12, i64 4, [[CHAR_TBAA0]]}
-; CHECK: [[META18]] = !{i32 1, [[META14]]}
-; CHECK: [[INT_TBAA19]] = !{[[META20:![0-9]+]], [[META20]], i64 0}
-; CHECK: [[META20]] = !{!"int", [[META1]], i64 0}
-; CHECK: [[TBAA_STRUCT21]] = !{i64 0, i64 4, [[CHAR_TBAA0]], i64 4, i64 4, [[INT_TBAA19]], i64 8, i64 4, [[CHAR_TBAA0]], i64 12, i64 4, [[CHAR_TBAA0]]}
+; CHECK: [[META17]] = !{i32 1, [[META14]]}
+; CHECK: [[INT_TBAA18]] = !{[[META19:![0-9]+]], [[META19]], i64 0}
+; CHECK: [[META19]] = !{!"int", [[META1]], i64 0}
;.
diff --git a/llvm/unittests/Analysis/CMakeLists.txt b/llvm/unittests/Analysis/CMakeLists.txt
index 43ad72b7bfa6dd..0732edeff7c88e 100644
--- a/llvm/unittests/Analysis/CMakeLists.txt
+++ b/llvm/unittests/Analysis/CMakeLists.txt
@@ -44,6 +44,7 @@ set(ANALYSIS_TEST_SOURCES
MemoryBuiltinsTest.cpp
MemoryProfileInfoTest.cpp
MemorySSATest.cpp
+ MetadataUtilsTest.cpp
MLGOUtilsTest.cpp
MLModelRunnerTest.cpp
PhiValuesTest.cpp
diff --git a/llvm/unittests/Analysis/MetadataUtilsTest.cpp b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
new file mode 100644
index 00000000000000..ea752404ddcfb3
--- /dev/null
+++ b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
@@ -0,0 +1,390 @@
+//===- MetadataUtilsTest.cpp - Metadata tests --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/MetadataUtils.h"
+#include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/MDBuilder.h"
+#include "llvm/IR/MemCacheHint.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Verifier.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+namespace {
+class MetadataUtilsTest : public testing::Test {
+protected:
+ LLVMContext C;
+ Module M{"test", C};
+ IRBuilder<> B{C};
+ Value *P;
+ Value *Q;
+
+ MetadataUtilsTest() {
+ M.setDataLayout("e-p:64:64");
+ auto *F = Function::Create(
+ FunctionType::get(B.getVoidTy(), {B.getPtrTy(), B.getPtrTy()}, false),
+ Function::ExternalLinkage, "test", M);
+ P = F->getArg(0);
+ Q = F->getArg(1);
+ B.SetInsertPoint(BasicBlock::Create(C, "entry", F));
+ B.SetInsertPoint(B.CreateRetVoid());
+ }
+
+ void TearDown() override { EXPECT_FALSE(verifyModule(M, &errs())); }
+
+ MDNode *empty() { return MDNode::get(C, {}); }
+ Metadata *number(uint64_t N) {
+ return ConstantAsMetadata::get(B.getInt64(N));
+ }
+ MDNode *hint() {
+ return MDNode::get(
+ C, {MDString::get(C, "nvvm.l1_eviction"), MDString::get(C, "last")});
+ }
+ MDNode *range() {
+ return MDNode::get(C, {ConstantAsMetadata::get(B.getInt32(0)),
+ ConstantAsMetadata::get(B.getInt32(100))});
+ }
+ MDNode *scope() {
+ MDBuilder MDB(C);
+ return MDNode::get(C, {MDB.createAnonymousAliasScope(
+ MDB.createAnonymousAliasScopeDomain())});
+ }
+};
+
+TEST_F(MetadataUtilsTest, NarrowedLoadPreservesElementFacts) {
+ auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), P);
+ auto *Ptr = B.CreateInBoundsGEP(B.getInt32Ty(), P, B.getInt64(2));
+ auto *New = B.CreateLoad(B.getInt32Ty(), Ptr);
+ auto *Range = range();
+ auto *Hint = hint();
+ auto *Scope = scope();
+ Old->setMetadata(LLVMContext::MD_range, Range);
+ Old->setMetadata(LLVMContext::MD_noundef, empty());
+ Old->setMetadata(LLVMContext::MD_invariant_load, empty());
+ Old->setMetadata(LLVMContext::MD_invariant_group, empty());
+ Old->setMetadata(LLVMContext::MD_alias_scope, Scope);
+ setMemCacheHint(*Old, 0, Hint);
+ copyMetadataForMemoryAccess(*New, *Old, 8);
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_range), Range);
+ EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_noundef));
+ EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_invariant_load));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_invariant_group));
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_alias_scope), Scope);
+ EXPECT_EQ(getMemCacheHint(*New), Hint);
+}
+
+TEST_F(MetadataUtilsTest, InvariantGroupRequiresPointerIdentity) {
+ auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), P);
+ auto *Same = B.CreateLoad(B.getInt32Ty(), P);
+ auto *Different = B.CreateLoad(B.getInt32Ty(), Q);
+ Old->setMetadata(LLVMContext::MD_invariant_group, empty());
+ copyMetadataForMemoryAccess(*Same, *Old);
+ copyMetadataForMemoryAccess(*Different, *Old);
+ EXPECT_TRUE(Same->hasMetadata(LLVMContext::MD_invariant_group));
+ EXPECT_FALSE(Different->hasMetadata(LLVMContext::MD_invariant_group));
+}
+
+TEST_F(MetadataUtilsTest, WideningDropsFactsAboutOriginalBytes) {
+ auto *Old = B.CreateLoad(B.getInt32Ty(), P);
+ auto *New = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), P);
+ auto *Hint = hint();
+ auto *Annotation = MDNode::get(C, {MDString::get(C, "original")});
+ Old->setMetadata(LLVMContext::MD_range, range());
+ Old->setMetadata(LLVMContext::MD_noundef, empty());
+ Old->setMetadata(LLVMContext::MD_invariant_load, empty());
+ Old->setMetadata(LLVMContext::MD_invariant_group, empty());
+ Old->setMetadata(LLVMContext::MD_alias_scope, scope());
+ Old->setMetadata(LLVMContext::MD_annotation, Annotation);
+ Old->setMetadata("test.unknown", empty());
+ setMemCacheHint(*Old, 0, Hint);
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_noundef));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_invariant_load));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_invariant_group));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_alias_scope));
+ EXPECT_FALSE(New->getMetadata("test.unknown"));
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_annotation), Annotation);
+ EXPECT_EQ(getMemCacheHint(*New), Hint);
+}
+
+TEST_F(MetadataUtilsTest, EarlierAddressIsNotASubset) {
+ auto *Ptr = B.CreateInBoundsGEP(B.getInt32Ty(), P, B.getInt64(2));
+ auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), Ptr);
+ auto *New = B.CreateLoad(B.getInt32Ty(), P);
+ Old->setMetadata(LLVMContext::MD_range, range());
+ copyMetadataForMemoryAccess(*New, *Old, -8);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+}
+
+TEST_F(MetadataUtilsTest, BitcastDropsDifferentIntegerElementRange) {
+ auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 2), P);
+ auto *New = B.CreateLoad(B.getInt64Ty(), P);
+ Old->setMetadata(LLVMContext::MD_range, range());
+ Old->setMetadata(LLVMContext::MD_noundef, empty());
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+ EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_noundef));
+}
+
+TEST_F(MetadataUtilsTest, BitcastDropsFloatingPointValueFacts) {
+ auto *Old = B.CreateLoad(FixedVectorType::get(B.getFloatTy(), 2), P);
+ auto *New = B.CreateLoad(B.getInt64Ty(), P);
+ Old->setMetadata(LLVMContext::MD_nofpclass,
+ MDNode::get(C, {ConstantAsMetadata::get(B.getInt32(3))}));
+ Old->setMetadata(LLVMContext::MD_fpmath,
+ MDNode::get(C, {ConstantAsMetadata::get(
+ ConstantFP::get(B.getFloatTy(), 2.5))}));
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_nofpclass));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_fpmath));
+}
+
+TEST_F(MetadataUtilsTest, PointerNonnullBecomesIntegerRange) {
+ auto *Old = B.CreateLoad(B.getPtrTy(), P);
+ auto *New = B.CreateLoad(B.getInt64Ty(), P);
+ Old->setMetadata(LLVMContext::MD_nonnull, empty());
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_nonnull));
+ ASSERT_TRUE(New->hasMetadata(LLVMContext::MD_range));
+ auto *Range = New->getMetadata(LLVMContext::MD_range);
+ EXPECT_EQ(mdconst::extract<ConstantInt>(Range->getOperand(0))->getZExtValue(),
+ 1u);
+ EXPECT_EQ(mdconst::extract<ConstantInt>(Range->getOperand(1))->getZExtValue(),
+ 0u);
+}
+
+TEST_F(MetadataUtilsTest, UnknownOffsetKeepsAccessTagsButDropsFieldLayout) {
+ auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), P);
+ auto *Index = B.CreateAnd(B.CreateLoad(B.getInt64Ty(), Q), B.getInt64(3));
+ auto *Ptr = B.CreateInBoundsGEP(B.getInt32Ty(), P, Index);
+ auto *New = B.CreateLoad(B.getInt32Ty(), Ptr);
+ MDBuilder MDB(C);
+ auto *Char = MDB.createTBAAScalarTypeNode("char", MDB.createTBAARoot("root"));
+ auto *Tag = MDB.createTBAAStructTagNode(Char, Char, 0);
+ auto *Scope = scope();
+ Old->setMetadata(LLVMContext::MD_tbaa, Tag);
+ Old->setMetadata(LLVMContext::MD_tbaa_struct,
+ MDNode::get(C, {number(0), number(4), Tag}));
+ Old->setMetadata(LLVMContext::MD_alias_scope, Scope);
+ copyMetadataForMemoryAccess(*New, *Old, std::nullopt);
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_tbaa), Tag);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_tbaa_struct));
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_alias_scope), Scope);
+}
+
+TEST_F(MetadataUtilsTest, ConstantOffsetSelectsFieldTag) {
+ auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 2), P);
+ auto *New = B.CreateLoad(B.getInt32Ty(), Q);
+ MDBuilder MDB(C);
+ auto *Root = MDB.createTBAARoot("root");
+ auto *Char = MDB.createTBAAScalarTypeNode("char", Root);
+ auto *Int = MDB.createTBAAScalarTypeNode("int", Char);
+ auto *CharTag = MDB.createTBAAStructTagNode(Char, Char, 0);
+ auto *IntTag = MDB.createTBAAStructTagNode(Int, Int, 0);
+ Old->setMetadata(LLVMContext::MD_tbaa_struct,
+ MDNode::get(C, {number(0), number(4), CharTag, number(4),
+ number(4), IntTag}));
+ copyMetadataForMemoryAccess(*New, *Old, 4);
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_tbaa), IntTag);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_tbaa_struct));
+}
+
+TEST_F(MetadataUtilsTest, WholeAccessSelectsSingleFieldTag) {
+ auto *Old = B.CreateLoad(B.getInt64Ty(), P);
+ auto *New = B.CreateLoad(B.getDoubleTy(), P);
+ MDBuilder MDB(C);
+ auto *Char = MDB.createTBAAScalarTypeNode("char", MDB.createTBAARoot("root"));
+ auto *Tag = MDB.createTBAAStructTagNode(Char, Char, 0);
+ Old->setMetadata(LLVMContext::MD_tbaa_struct,
+ MDNode::get(C, {number(0), number(8), Tag}));
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_tbaa), Tag);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_tbaa_struct));
+}
+
+TEST_F(MetadataUtilsTest, WholeAccessRetainsMultipleFieldTags) {
+ auto *Ty = FixedVectorType::get(B.getInt32Ty(), 2);
+ auto *Old = B.CreateLoad(Ty, P);
+ auto *New = B.CreateLoad(Ty, P);
+ MDBuilder MDB(C);
+ auto *Char = MDB.createTBAAScalarTypeNode("char", MDB.createTBAARoot("root"));
+ auto *Int = MDB.createTBAAScalarTypeNode("int", Char);
+ auto *CharTag = MDB.createTBAAStructTagNode(Char, Char, 0);
+ auto *IntTag = MDB.createTBAAStructTagNode(Int, Int, 0);
+ auto *Fields = MDNode::get(
+ C, {number(0), number(4), CharTag, number(4), number(4), IntTag});
+ Old->setMetadata(LLVMContext::MD_tbaa_struct, Fields);
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_tbaa));
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_tbaa_struct), Fields);
+}
+
+TEST_F(MetadataUtilsTest, MemoryTransferRemapsBothOperands) {
+ auto *Copy = B.CreateMemCpy(P, Align(4), Q, Align(4), B.getInt64(4));
+ auto *Read = B.CreateLoad(B.getInt32Ty(), Q);
+ auto *Write = B.CreateStore(B.getInt32(1), P);
+ auto *ReadHint = hint();
+ auto *WriteHint = MDNode::get(
+ C, {MDString::get(C, "nvvm.l1_eviction"), MDString::get(C, "first")});
+ setMemCacheHint(*Copy, 0, WriteHint);
+ setMemCacheHint(*Copy, 1, ReadHint);
+ copyMetadataForMemoryAccess(*Read, *Copy);
+ copyMetadataForMemoryAccess(*Write, *Copy);
+ EXPECT_EQ(getMemCacheHint(*Read, 0), ReadHint);
+ EXPECT_EQ(getMemCacheHint(*Write, 1), WriteHint);
+ EXPECT_FALSE(getMemCacheHint(*Read, 1));
+ EXPECT_FALSE(getMemCacheHint(*Write, 0));
+}
+
+TEST_F(MetadataUtilsTest, RuntimeMemoryTransferLength) {
+ auto *Length = B.CreateLoad(B.getInt64Ty(), Q);
+ B.CreateAssumption(B.CreateICmpUGE(Length, B.getInt64(8)));
+ auto *Copy = B.CreateMemCpy(P, Align(4), Q, Align(4), Length);
+ auto *Ptr = B.CreateInBoundsGEP(B.getInt8Ty(), Q, B.getInt64(4));
+ auto *Read = B.CreateLoad(B.getInt32Ty(), Ptr);
+ auto *Hint = hint();
+ setMemCacheHint(*Copy, 1, Hint);
+ copyMetadataForMemoryAccess(*Read, *Copy, 4);
+ EXPECT_EQ(getMemCacheHint(*Read), Hint);
+}
+
+TEST_F(MetadataUtilsTest, StoreTypeChangePreservesHints) {
+ auto *Old = B.CreateStore(B.getInt32(0), P);
+ auto *New = B.CreateStore(ConstantFP::get(B.getFloatTy(), 0), P);
+ auto *Hint = hint();
+ setMemCacheHint(*Old, 1, Hint);
+ Old->setMetadata(LLVMContext::MD_invariant_group, empty());
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_EQ(getMemCacheHint(*New, 1), Hint);
+ EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_invariant_group));
+}
+
+TEST_F(MetadataUtilsTest, NonMemoryScalarizationPreservesFPMath) {
+ auto *Ty = FixedVectorType::get(B.getFloatTy(), 2);
+ auto *Old =
+ cast<Instruction>(B.CreateFAdd(B.CreateLoad(Ty, P), B.CreateLoad(Ty, Q)));
+ auto *New = cast<Instruction>(B.CreateFAdd(B.CreateLoad(B.getFloatTy(), P),
+ B.CreateLoad(B.getFloatTy(), Q)));
+ auto *Tag = MDNode::get(
+ C, {ConstantAsMetadata::get(ConstantFP::get(B.getFloatTy(), 2.5))});
+ Old->setMetadata(LLVMContext::MD_fpmath, Tag);
+ copyMetadataForScalarizedInstruction(*New, *Old);
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_fpmath), Tag);
+}
+
+TEST_F(MetadataUtilsTest, CSEIntersectsCacheHints) {
+ auto *A = B.CreateLoad(B.getInt32Ty(), P);
+ auto *BLoad = B.CreateLoad(B.getInt32Ty(), P);
+ auto *Hint = hint();
+ setMemCacheHint(*A, 0, Hint);
+ setMemCacheHint(*BLoad, 0, Hint);
+ combineMetadataForCSE(A, BLoad, false);
+ EXPECT_EQ(getMemCacheHint(*A), Hint);
+ BLoad->setMetadata(LLVMContext::MD_mem_cache_hint, nullptr);
+ combineMetadataForCSE(A, BLoad, false);
+ EXPECT_FALSE(getMemCacheHint(*A));
+}
+
+TEST_F(MetadataUtilsTest, AddressSpaceCastChangesInvariantGroupIdentity) {
+ auto *Old = B.CreateLoad(B.getInt32Ty(), P);
+ auto *NewPtr = B.CreateAddrSpaceCast(P, B.getPtrTy(1));
+ auto *New = B.CreateLoad(B.getInt32Ty(), NewPtr);
+ Old->setMetadata(LLVMContext::MD_invariant_group, empty());
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_invariant_group));
+}
+
+TEST_F(MetadataUtilsTest, DefinedAggregateDoesNotDefinePadding) {
+ auto *Ty = StructType::get(C, {B.getInt8Ty(), B.getInt32Ty()});
+ ASSERT_EQ(M.getDataLayout().getTypeStoreSize(Ty).getFixedValue(), 8u);
+ auto *Old = B.CreateLoad(Ty, P);
+ auto *New = B.CreateLoad(B.getInt64Ty(), P);
+ Old->setMetadata(LLVMContext::MD_noundef, empty());
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_noundef));
+}
+
+TEST_F(MetadataUtilsTest, PointerFactsRequirePointerResults) {
+ auto *Old = B.CreateLoad(B.getPtrTy(), P);
+ auto *Pointer = B.CreateLoad(B.getPtrTy(), P);
+ auto *Integer = B.CreateLoad(B.getInt64Ty(), P);
+ auto *Align = MDNode::get(C, {number(16)});
+ auto *Deref = MDNode::get(C, {number(32)});
+ Old->setMetadata(LLVMContext::MD_nonnull, empty());
+ Old->setMetadata(LLVMContext::MD_align, Align);
+ Old->setMetadata(LLVMContext::MD_dereferenceable, Deref);
+ Old->setMetadata(LLVMContext::MD_dereferenceable_or_null, Deref);
+ copyMetadataForMemoryAccess(*Pointer, *Old);
+ copyMetadataForMemoryAccess(*Integer, *Old);
+ EXPECT_TRUE(Pointer->hasMetadata(LLVMContext::MD_nonnull));
+ EXPECT_EQ(Pointer->getMetadata(LLVMContext::MD_align), Align);
+ EXPECT_EQ(Pointer->getMetadata(LLVMContext::MD_dereferenceable), Deref);
+ EXPECT_EQ(Pointer->getMetadata(LLVMContext::MD_dereferenceable_or_null),
+ Deref);
+ EXPECT_FALSE(Integer->hasMetadata(LLVMContext::MD_nonnull));
+ EXPECT_FALSE(Integer->hasMetadata(LLVMContext::MD_align));
+ EXPECT_FALSE(Integer->hasMetadata(LLVMContext::MD_dereferenceable));
+ EXPECT_FALSE(Integer->hasMetadata(LLVMContext::MD_dereferenceable_or_null));
+ EXPECT_TRUE(Integer->hasMetadata(LLVMContext::MD_range));
+}
+
+TEST_F(MetadataUtilsTest, StoreFactsRequireWholeAccessAndPointerValue) {
+ auto *Old = B.CreateStore(Q, P);
+ auto *Pointer = B.CreateStore(Q, P);
+ auto *Integer = B.CreateStore(B.CreatePtrToInt(Q, B.getInt64Ty()), P);
+ auto *Part = B.CreateStore(B.getInt32(0), P);
+ auto *AssignID = DIAssignID::getDistinct(C);
+ auto *Captures = MDNode::get(C, {MDString::get(C, "address")});
+ Old->setMetadata(LLVMContext::MD_DIAssignID, AssignID);
+ Old->setMetadata(LLVMContext::MD_captures, Captures);
+ copyMetadataForMemoryAccess(*Pointer, *Old);
+ copyMetadataForMemoryAccess(*Integer, *Old);
+ copyMetadataForMemoryAccess(*Part, *Old);
+ EXPECT_EQ(Pointer->getMetadata(LLVMContext::MD_DIAssignID), AssignID);
+ EXPECT_EQ(Pointer->getMetadata(LLVMContext::MD_captures), Captures);
+ EXPECT_EQ(Integer->getMetadata(LLVMContext::MD_DIAssignID), AssignID);
+ EXPECT_FALSE(Integer->hasMetadata(LLVMContext::MD_captures));
+ EXPECT_FALSE(Part->hasMetadata(LLVMContext::MD_DIAssignID));
+ EXPECT_FALSE(Part->hasMetadata(LLVMContext::MD_captures));
+}
+
+TEST_F(MetadataUtilsTest, IntrinsicProfilesAndUnknownMetadataAreNotCopied) {
+ auto *Ty = FixedVectorType::get(B.getInt32Ty(), 2);
+ auto *Mask = B.CreateLoad(FixedVectorType::get(B.getInt1Ty(), 2), Q);
+ auto *Old = B.CreateMaskedLoad(Ty, P, Align(4), Mask);
+ auto *New = B.CreateLoad(Ty, P);
+ auto *Profile = MDNode::get(C, {MDString::get(C, "branch_weights"),
+ ConstantAsMetadata::get(B.getInt32(10))});
+ Old->setMetadata(LLVMContext::MD_prof, Profile);
+ Old->setMetadata("test.unknown", empty());
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_prof));
+ EXPECT_FALSE(New->getMetadata("test.unknown"));
+}
+
+TEST_F(MetadataUtilsTest, SmallerAccessOutsideSourceRangeIsNotASubset) {
+ auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), P);
+ auto *Ptr = B.CreateInBoundsGEP(B.getInt32Ty(), P, B.getInt64(4));
+ auto *New = B.CreateLoad(B.getInt32Ty(), Ptr);
+ Old->setMetadata(LLVMContext::MD_range, range());
+ Old->setMetadata(LLVMContext::MD_noundef, empty());
+ Old->setMetadata(LLVMContext::MD_invariant_load, empty());
+ Old->setMetadata(LLVMContext::MD_alias_scope, scope());
+ Old->setMetadata(LLVMContext::MD_noalias, scope());
+ copyMetadataForMemoryAccess(*New, *Old, 16);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_noundef));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_invariant_load));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_alias_scope));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_noalias));
+}
+
+} // namespace
>From a3aeb4c617c1131c264fcf321a074631db907fc5 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Sat, 14 Nov 2026 00:41:15 +0000
Subject: [PATCH 03/14] some fixes and comments to make things clearer
---
llvm/include/llvm/Analysis/MetadataUtils.h | 25 +--
llvm/lib/Analysis/MetadataUtils.cpp | 142 +++++++++++-------
.../Scalar/ScalarizeMaskedMemIntrin.cpp | 16 +-
llvm/test/Transforms/SROA/memory-metadata.ll | 4 +-
llvm/unittests/Analysis/MetadataUtilsTest.cpp | 66 ++++++++
5 files changed, 173 insertions(+), 80 deletions(-)
diff --git a/llvm/include/llvm/Analysis/MetadataUtils.h b/llvm/include/llvm/Analysis/MetadataUtils.h
index bfaf8bd6912e8f..2d7ce97f1cdd13 100644
--- a/llvm/include/llvm/Analysis/MetadataUtils.h
+++ b/llvm/include/llvm/Analysis/MetadataUtils.h
@@ -23,24 +23,25 @@ class Value;
/// Copy metadata valid for a replacement memory access. The destination must
/// access the same bytes, a subset, or an expanded range in the same object.
-/// ByteOffset is the destination's offset within the source access. A missing
-/// offset requires the caller to establish that the new access is a subset.
-/// Access types and pointer operands are inferred for loads, stores, and
-/// masked/VP memory intrinsics. For a source memory intrinsic, the source or
-/// destination pointer is selected to match the new read or write. The caller
-/// must establish range containment when the intrinsic's length is unknown.
-/// Pointer and value facts are retained only when valid for the destination;
-/// alias tags are adjusted for its range. Masks and execution conditions are
-/// not inspected.
+/// The destination must be a typed memory access; memory-transfer and
+/// memory-set intrinsics are supported only as sources. DestByteOffset is the
+/// destination's offset within the source access. A missing offset requires the
+/// caller to establish that the new access is a subset. Access types and
+/// pointer operands are inferred for loads, stores, and masked/VP memory
+/// intrinsics. For a source memory intrinsic, the source or destination pointer
+/// is selected to match the new read or write. The caller must establish range
+/// containment when the intrinsic's length is unknown. Pointer and value facts
+/// are retained only when valid for the destination; alias tags are adjusted
+/// for its range. Masks and execution conditions are not inspected.
LLVM_ABI void
copyMetadataForMemoryAccess(Instruction &Dest, const Instruction &Source,
- std::optional<int64_t> ByteOffset = 0);
+ std::optional<int64_t> DestByteOffset = 0);
/// Copy metadata valid after splitting an instruction into smaller elements.
-/// ByteOffset describes the selected memory fragment, if applicable.
+/// DestByteOffset describes the selected memory fragment, if applicable.
LLVM_ABI void copyMetadataForScalarizedInstruction(
Instruction &Dest, const Instruction &Source,
- std::optional<int64_t> ByteOffset = std::nullopt);
+ std::optional<int64_t> DestByteOffset = std::nullopt);
/// Combine the metadata of two instructions so that K can replace J. This
/// specifically handles the case of CSE-like transformations. Some
diff --git a/llvm/lib/Analysis/MetadataUtils.cpp b/llvm/lib/Analysis/MetadataUtils.cpp
index 936469fb58a149..646982f4118fe8 100644
--- a/llvm/lib/Analysis/MetadataUtils.cpp
+++ b/llvm/lib/Analysis/MetadataUtils.cpp
@@ -32,8 +32,16 @@ struct MemoryAccessInfo {
unsigned PointerOperand;
bool IsRead;
};
+} // namespace
-std::optional<MemoryAccessInfo>
+/// Get information about a memory instruction required to propagate metadata.
+/// Returns std::nullopt if the instruction is not supported and we will not
+/// propagate any metadata.
+///
+/// If IsRead is set and this is a instruction that reads and writes, mark it
+/// as reading or writing. This is used to make the source instruction in
+/// metadata propagation correspond with the dest.
+static std::optional<MemoryAccessInfo>
getMemoryAccessInfo(const Instruction &I,
std::optional<bool> IsRead = std::nullopt) {
if (const auto *LI = dyn_cast<LoadInst>(&I))
@@ -80,14 +88,16 @@ getMemoryAccessInfo(const Instruction &I,
}
}
-const Value *getMemoryPointer(const Instruction &I, unsigned Index) {
+/// Get the pointer operand of `I` at operand index `Index`.
+static const Value *getMemoryPointer(const Instruction &I, unsigned Index) {
if (const auto *CB = dyn_cast<CallBase>(&I))
return CB->getArgOperand(Index);
return I.getOperand(Index);
}
+
// invariant.group ignores bitcasts and zero-offset GEPs, not address-space
// casts.
-const Value *getInvariantGroupPointer(const Value *V) {
+static const Value *getInvariantGroupPointer(const Value *V) {
while (true) {
if (const auto *Cast = dyn_cast<BitCastOperator>(V))
V = Cast->getOperand(0);
@@ -99,32 +109,34 @@ const Value *getInvariantGroupPointer(const Value *V) {
}
}
-void copyNonnullMetadata(const LoadInst &OldLI, MDNode *N, LoadInst &NewLI) {
+static void copyNonnullMetadata(const LoadInst &OldLI, MDNode *N,
+ LoadInst &NewLI) {
+ auto *OldTy = cast<PointerType>(OldLI.getType());
auto *NewTy = NewLI.getType();
- // This only directly applies if the new type is also a pointer.
- if (NewTy->isPointerTy()) {
+ // Preserve the pointer fact when the pointer type is unchanged.
+ if (NewTy == OldTy) {
NewLI.setMetadata(LLVMContext::MD_nonnull, N);
return;
}
- // The only other translation we can do is to integral loads with !range
- // metadata.
- if (!NewTy->isIntegerTy())
+ // An integer range must describe the complete pointer representation.
+ if (!NewTy->isIntegerTy() ||
+ NewTy->getIntegerBitWidth() !=
+ OldLI.getDataLayout().getPointerTypeSizeInBits(OldTy))
return;
MDBuilder MDB(NewLI.getContext());
- const Value *Ptr = OldLI.getPointerOperand();
auto *ITy = cast<IntegerType>(NewTy);
- auto *NullInt = ConstantExpr::getPtrToInt(
- ConstantPointerNull::get(cast<PointerType>(Ptr->getType())), ITy);
+ auto *NullInt =
+ ConstantExpr::getPtrToInt(ConstantPointerNull::get(OldTy), ITy);
auto *NonNullInt = ConstantExpr::getAdd(NullInt, ConstantInt::get(ITy, 1));
NewLI.setMetadata(LLVMContext::MD_range,
MDB.createRange(NonNullInt, NullInt));
}
-void copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI, MDNode *N,
- LoadInst &NewLI) {
+static void copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI,
+ MDNode *N, LoadInst &NewLI) {
auto *NewTy = NewLI.getType();
// Simply copy the metadata if the type did not change.
if (NewTy == OldLI.getType()) {
@@ -146,42 +158,46 @@ void copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI, MDNode *N,
NewLI.setMetadata(LLVMContext::MD_nonnull, NN);
}
}
-} // namespace
void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
const Instruction &Source,
- std::optional<int64_t> ByteOffset) {
- auto Dst = getMemoryAccessInfo(Dest);
- auto Src = getMemoryAccessInfo(Source, Dst ? std::optional<bool>(Dst->IsRead)
- : std::nullopt);
- assert(Src && Dst && Src->IsRead == Dst->IsRead &&
+ std::optional<int64_t> DestByteOffset) {
+ auto DestInfo = getMemoryAccessInfo(Dest);
+ // Untyped memory intrinsics are supported only as sources.
+ assert(DestInfo && DestInfo->Ty &&
+ "expected a typed destination memory access");
+ auto SourceInfo = getMemoryAccessInfo(Source, DestInfo->IsRead);
+ assert(SourceInfo && SourceInfo->IsRead == DestInfo->IsRead &&
"expected corresponding memory reads or writes");
const DataLayout &DL = Source.getDataLayout();
+ // Collect the size of the old memory operation if available.
std::optional<TypeSize> OldSize;
- if (Src->Ty)
- OldSize = DL.getTypeStoreSize(Src->Ty);
+ if (SourceInfo->Ty)
+ OldSize = DL.getTypeStoreSize(SourceInfo->Ty);
else if (auto *Length =
dyn_cast<ConstantInt>(cast<AnyMemIntrinsic>(Source).getLength()))
+ // If Source is not a typed memory operation it must be an intrinsic.
if (auto Bytes = Length->getValue().tryZExtValue())
OldSize = TypeSize::getFixed(*Bytes);
- TypeSize NewSize = DL.getTypeStoreSize(Dst->Ty);
+ TypeSize NewSize = DL.getTypeStoreSize(DestInfo->Ty);
+ // Is the memory `Dest` accesses a subset of the memory `Source` acceses?
+ // The caller guarentees this is true if we don't know the size of `Source`
+ // or the offset of the `Dest` access in `Source`.
bool IsSubset = !OldSize || TypeSize::isKnownLE(NewSize, *OldSize);
- if (ByteOffset)
- IsSubset =
- IsSubset && *ByteOffset >= 0 &&
- (!OldSize || uint64_t(*ByteOffset) <= OldSize->getKnownMinValue() -
- NewSize.getKnownMinValue());
- bool IsWholeAccess = IsSubset && ByteOffset == 0 && OldSize == NewSize;
- bool SameElementType =
- Src->Ty && Src->Ty->getScalarType() == Dst->Ty->getScalarType();
- bool SameValueBits =
- IsWholeAccess && Src->Ty && !Src->Ty->isAggregateType() &&
- !Dst->Ty->isAggregateType() && DL.typeSizeEqualsStoreSize(Src->Ty) &&
- DL.typeSizeEqualsStoreSize(Dst->Ty);
- bool SamePointer =
- getInvariantGroupPointer(getMemoryPointer(Source, Src->PointerOperand)) ==
- getInvariantGroupPointer(getMemoryPointer(Dest, Dst->PointerOperand));
-
+ if (DestByteOffset)
+ IsSubset = IsSubset && *DestByteOffset >= 0 &&
+ (!OldSize ||
+ uint64_t(*DestByteOffset) <=
+ OldSize->getKnownMinValue() - NewSize.getKnownMinValue());
+ bool IsWholeAccess = IsSubset && DestByteOffset == 0 && OldSize == NewSize;
+ bool SameElementType = SourceInfo->Ty && SourceInfo->Ty->getScalarType() ==
+ DestInfo->Ty->getScalarType();
+ bool SamePointer = getInvariantGroupPointer(getMemoryPointer(
+ Source, SourceInfo->PointerOperand)) ==
+ getInvariantGroupPointer(
+ getMemoryPointer(Dest, DestInfo->PointerOperand));
+
+ // Now loop over all `Source` metadata, propagating what we can.
SmallVector<std::pair<unsigned, MDNode *>, 8> MDs;
Source.getAllMetadata(MDs);
for (auto [Kind, MD] : MDs) {
@@ -201,14 +217,19 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
Keep = IsSubset;
break;
case LLVMContext::MD_invariant_load:
- Keep = IsSubset && Dst->IsRead;
+ Keep = IsSubset && DestInfo->IsRead;
break;
case LLVMContext::MD_invariant_group:
Keep = IsSubset && SamePointer && !isa<CallBase>(Dest);
break;
case LLVMContext::MD_noundef:
- Keep =
- IsSubset && (SameElementType || SameValueBits) && isa<LoadInst>(Dest);
+ // The value loaded remains defined if its element type is unchanged,
+ // or every source byte consists of defined value bits (i.e. the source
+ // doesn't have padding).
+ Keep = IsSubset && isa<LoadInst>(Dest) &&
+ (SameElementType ||
+ (SourceInfo->Ty && !SourceInfo->Ty->isAggregateType() &&
+ DL.typeSizeEqualsStoreSize(SourceInfo->Ty)));
break;
case LLVMContext::MD_fpmath:
case LLVMContext::MD_nofpclass:
@@ -218,19 +239,23 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
: isa<LoadInst>(Dest));
break;
case LLVMContext::MD_nonnull:
- if (SameValueBits && isa<LoadInst>(Source) && isa<LoadInst>(Dest))
+ if (IsWholeAccess && isa<LoadInst>(Source) && isa<LoadInst>(Dest))
copyNonnullMetadata(cast<LoadInst>(Source), MD, cast<LoadInst>(Dest));
break;
case LLVMContext::MD_range:
- if (IsSubset && SameElementType && Dst->Ty->isIntOrIntVectorTy())
+ // Ranges apply per integer element. Other type changes require the
+ // helper's explicit conversion from an integer range to nonnull.
+ if (IsSubset && SameElementType && DestInfo->Ty->isIntOrIntVectorTy())
Keep = true;
- else if (SameValueBits && isa<LoadInst>(Source) && isa<LoadInst>(Dest))
+ else if (IsWholeAccess && isa<LoadInst>(Source) && isa<LoadInst>(Dest))
copyRangeMetadata(DL, cast<LoadInst>(Source), MD, cast<LoadInst>(Dest));
break;
case LLVMContext::MD_align:
case LLVMContext::MD_dereferenceable:
case LLVMContext::MD_dereferenceable_or_null:
- Keep = SameValueBits && Dst->Ty->isPointerTy() && isa<LoadInst>(Dest);
+ // These describe the loaded pointer value.
+ Keep = IsWholeAccess && SourceInfo->Ty == DestInfo->Ty &&
+ DestInfo->Ty->isPointerTy() && isa<LoadInst>(Dest);
break;
case LLVMContext::MD_DIAssignID:
Keep = IsWholeAccess && isa<StoreInst>(Source) && isa<StoreInst>(Dest);
@@ -241,7 +266,8 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
(isa<StoreInst>(Source) && isa<StoreInst>(Dest)));
break;
case LLVMContext::MD_captures:
- Keep = IsWholeAccess && isa<StoreInst>(Dest) && Dst->Ty->isPointerTy();
+ Keep =
+ IsWholeAccess && isa<StoreInst>(Dest) && DestInfo->Ty->isPointerTy();
break;
default:
break;
@@ -250,6 +276,7 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
Dest.setMetadata(Kind, MD);
}
+ // Adjust aliasing metadata.
AAMDNodes AA = Source.getAAMetadata();
if (!IsSubset) {
AA.TBAA = nullptr;
@@ -257,28 +284,27 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
AA.Scope = nullptr;
AA.NoAlias = nullptr;
} else if (!IsWholeAccess) {
- if (ByteOffset)
- AA = AA.adjustForAccess(*ByteOffset, Dst->Ty, DL);
+ if (DestByteOffset)
+ AA = AA.adjustForAccess(*DestByteOffset, DestInfo->Ty, DL);
else
AA.TBAAStruct = nullptr;
} else {
- // Prefer a single alias tag for a whole replacement, retaining the full
- // field layout when it cannot be represented that way.
- AAMDNodes Adjusted = AA.adjustForAccess(0, Dst->Ty, DL);
+ // Prefer !tbaa over !tbaa.struct.
+ AAMDNodes Adjusted = AA.adjustForAccess(0, DestInfo->Ty, DL);
if (!AA.TBAA && Adjusted.TBAA)
AA = Adjusted;
}
Dest.setAAMetadata(AA);
- setMemCacheHint(Dest, Dst->PointerOperand,
- getMemCacheHint(Source, Src->PointerOperand));
+ setMemCacheHint(Dest, DestInfo->PointerOperand,
+ getMemCacheHint(Source, SourceInfo->PointerOperand));
}
void llvm::copyMetadataForScalarizedInstruction(
Instruction &Dest, const Instruction &Source,
- std::optional<int64_t> ByteOffset) {
- if (auto Dst = getMemoryAccessInfo(Dest))
- if (getMemoryAccessInfo(Source, Dst->IsRead)) {
- copyMetadataForMemoryAccess(Dest, Source, ByteOffset);
+ std::optional<int64_t> DestByteOffset) {
+ if (auto DestInfo = getMemoryAccessInfo(Dest))
+ if (getMemoryAccessInfo(Source, DestInfo->IsRead)) {
+ copyMetadataForMemoryAccess(Dest, Source, DestByteOffset);
return;
}
if (Dest.getType()->getScalarType() == Source.getType()->getScalarType() &&
diff --git a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
index d8d07d7b09f83c..431037b3a0cc33 100644
--- a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
+++ b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
@@ -524,7 +524,7 @@ static void scalarizeMaskedGather(const DataLayout &DL,
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
LoadInst *Load =
Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx));
- copyMetadataForMemoryAccess(*Load, *CI, /*ByteOffset=*/0);
+ copyMetadataForMemoryAccess(*Load, *CI, /*DestByteOffset=*/0);
VResult =
Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx));
}
@@ -584,7 +584,7 @@ static void scalarizeMaskedGather(const DataLayout &DL,
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
LoadInst *Load =
Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx));
- copyMetadataForMemoryAccess(*Load, *CI, /*ByteOffset=*/0);
+ copyMetadataForMemoryAccess(*Load, *CI, /*DestByteOffset=*/0);
Value *NewVResult =
Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx));
@@ -663,7 +663,7 @@ static void scalarizeMaskedScatter(const DataLayout &DL,
Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
StoreInst *Store = Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
- copyMetadataForMemoryAccess(*Store, *CI, /*ByteOffset=*/0);
+ copyMetadataForMemoryAccess(*Store, *CI, /*DestByteOffset=*/0);
}
CI->eraseFromParent();
return;
@@ -718,7 +718,7 @@ static void scalarizeMaskedScatter(const DataLayout &DL,
Value *OneElt = Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
StoreInst *Store = Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
- copyMetadataForMemoryAccess(*Store, *CI, /*ByteOffset=*/0);
+ copyMetadataForMemoryAccess(*Store, *CI, /*DestByteOffset=*/0);
// Create "else" block, fill it in the next iteration
BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
@@ -1043,11 +1043,11 @@ static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI,
continue;
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
LoadInst *Load = Builder.CreateLoad(EltTy, Ptr, "Load" + Twine(Idx));
- copyMetadataForMemoryAccess(*Load, *CI, /*ByteOffset=*/0);
+ copyMetadataForMemoryAccess(*Load, *CI, /*DestByteOffset=*/0);
Value *Update =
CreateHistogramUpdateValue(cast<IntrinsicInst>(CI), Load, Inc);
StoreInst *Store = Builder.CreateStore(Update, Ptr);
- copyMetadataForMemoryAccess(*Store, *CI, /*ByteOffset=*/0);
+ copyMetadataForMemoryAccess(*Store, *CI, /*DestByteOffset=*/0);
}
CI->eraseFromParent();
return;
@@ -1071,11 +1071,11 @@ static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI,
Builder.SetInsertPoint(CondBlock->getTerminator());
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
LoadInst *Load = Builder.CreateLoad(EltTy, Ptr, "Load" + Twine(Idx));
- copyMetadataForMemoryAccess(*Load, *CI, /*ByteOffset=*/0);
+ copyMetadataForMemoryAccess(*Load, *CI, /*DestByteOffset=*/0);
Value *UpdateOp =
CreateHistogramUpdateValue(cast<IntrinsicInst>(CI), Load, Inc);
StoreInst *Store = Builder.CreateStore(UpdateOp, Ptr);
- copyMetadataForMemoryAccess(*Store, *CI, /*ByteOffset=*/0);
+ copyMetadataForMemoryAccess(*Store, *CI, /*DestByteOffset=*/0);
// Create "else" block, fill it in the next iteration
BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
diff --git a/llvm/test/Transforms/SROA/memory-metadata.ll b/llvm/test/Transforms/SROA/memory-metadata.ll
index f4d272e8465ef3..88be04f0d01fbd 100644
--- a/llvm/test/Transforms/SROA/memory-metadata.ll
+++ b/llvm/test/Transforms/SROA/memory-metadata.ll
@@ -146,9 +146,9 @@ define i32 @presplit_load(ptr %src) {
; CHECK-SAME: ptr [[SRC:%.*]]) {
; CHECK-NEXT: [[A_SROA_0:%.*]] = alloca i32, align 8
; CHECK-NEXT: [[A_SROA_2:%.*]] = alloca i32, align 4
-; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[SRC]], align 8, !invariant.load [[META10]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !invariant.group [[META10]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META19]]
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[SRC]], align 8, !invariant.load [[META10]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !invariant.group [[META10]], !llvm.access.group [[META8]], !noundef [[META10]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META19]]
; CHECK-NEXT: [[SRC_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 4
-; CHECK-NEXT: [[V2:%.*]] = load i32, ptr [[SRC_SROA_IDX]], align 4, !invariant.load [[META10]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META19]]
+; CHECK-NEXT: [[V2:%.*]] = load i32, ptr [[SRC_SROA_IDX]], align 4, !invariant.load [[META10]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !noundef [[META10]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META19]]
; CHECK-NEXT: store i32 [[V1]], ptr [[A_SROA_0]], align 8, !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
; CHECK-NEXT: store i32 [[V2]], ptr [[A_SROA_2]], align 4, !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
; CHECK-NEXT: [[A_SROA_0_0_A_SROA_0_0_X:%.*]] = load volatile i32, ptr [[A_SROA_0]], align 8
diff --git a/llvm/unittests/Analysis/MetadataUtilsTest.cpp b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
index ea752404ddcfb3..7066da1a69177a 100644
--- a/llvm/unittests/Analysis/MetadataUtilsTest.cpp
+++ b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
@@ -91,6 +91,21 @@ TEST_F(MetadataUtilsTest, InvariantGroupRequiresPointerIdentity) {
EXPECT_FALSE(Different->hasMetadata(LLVMContext::MD_invariant_group));
}
+TEST_F(MetadataUtilsTest, IntegerFragmentPreservesDefinednessButNotRange) {
+ auto *Old = B.CreateLoad(B.getInt64Ty(), P);
+ MDBuilder MDB(C);
+ Old->setMetadata(LLVMContext::MD_noundef, empty());
+ Old->setMetadata(LLVMContext::MD_range,
+ MDB.createRange(B.getInt64(0), B.getInt64(100)));
+ for (int64_t Offset : {0, 4}) {
+ auto *Ptr = B.CreateInBoundsGEP(B.getInt8Ty(), P, B.getInt64(Offset));
+ auto *New = B.CreateLoad(B.getInt32Ty(), Ptr);
+ copyMetadataForMemoryAccess(*New, *Old, Offset);
+ EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_noundef));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+ }
+}
+
TEST_F(MetadataUtilsTest, WideningDropsFactsAboutOriginalBytes) {
auto *Old = B.CreateLoad(B.getInt32Ty(), P);
auto *New = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), P);
@@ -312,6 +327,57 @@ TEST_F(MetadataUtilsTest, DefinedAggregateDoesNotDefinePadding) {
EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_noundef));
}
+TEST_F(MetadataUtilsTest, DefinedIntegerDoesNotDefineSourcePadding) {
+ auto *Old = B.CreateLoad(B.getInt1Ty(), P);
+ auto *New = B.CreateLoad(B.getInt8Ty(), P);
+ Old->setMetadata(LLVMContext::MD_noundef, empty());
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_noundef));
+}
+
+TEST_F(MetadataUtilsTest, DestinationPaddingDoesNotInvalidateDefinedness) {
+ auto *Old = B.CreateLoad(B.getInt8Ty(), P);
+ auto *New = B.CreateLoad(B.getInt1Ty(), P);
+ Old->setMetadata(LLVMContext::MD_noundef, empty());
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_noundef));
+}
+
+TEST_F(MetadataUtilsTest, NonnullDoesNotConstrainTruncatedPointerBits) {
+ auto *Old = B.CreateLoad(B.getPtrTy(), P);
+ auto *New = B.CreateLoad(B.getIntNTy(63), P);
+ Old->setMetadata(LLVMContext::MD_nonnull, empty());
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+}
+
+TEST_F(MetadataUtilsTest, IntegerRangeBecomesPointerNonnull) {
+ auto *Old = B.CreateLoad(B.getInt64Ty(), P);
+ auto *New = B.CreateLoad(B.getPtrTy(), P);
+ MDBuilder MDB(C);
+ Old->setMetadata(LLVMContext::MD_range,
+ MDB.createRange(B.getInt64(1), B.getInt64(100)));
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_nonnull));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+}
+
+TEST_F(MetadataUtilsTest, PointerFactsRequireSamePointerType) {
+ auto *Old = B.CreateLoad(B.getPtrTy(), P);
+ auto *New = B.CreateLoad(B.getPtrTy(1), P);
+ Old->setMetadata(LLVMContext::MD_nonnull, empty());
+ Old->setMetadata(LLVMContext::MD_align, MDNode::get(C, {number(16)}));
+ Old->setMetadata(LLVMContext::MD_dereferenceable,
+ MDNode::get(C, {number(32)}));
+ Old->setMetadata(LLVMContext::MD_dereferenceable_or_null,
+ MDNode::get(C, {number(32)}));
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_nonnull));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_align));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_dereferenceable));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_dereferenceable_or_null));
+}
+
TEST_F(MetadataUtilsTest, PointerFactsRequirePointerResults) {
auto *Old = B.CreateLoad(B.getPtrTy(), P);
auto *Pointer = B.CreateLoad(B.getPtrTy(), P);
>From 4d78157968d004891a67d06d2a94fe55d331957b Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 17 Nov 2026 20:45:03 +0000
Subject: [PATCH 04/14] additional fixes
---
llvm/lib/Analysis/MetadataUtils.cpp | 47 ++++++++----
llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp | 5 +-
llvm/lib/Transforms/Scalar/SROA.cpp | 16 +++--
.../Transforms/Vectorize/VectorCombine.cpp | 3 +
.../Transforms/InstCombine/memory-metadata.ll | 60 ++++++++++++++++
.../VectorCombine/memory-metadata.ll | 47 ++++++++++--
llvm/unittests/Analysis/MetadataUtilsTest.cpp | 72 +++++++++++++++++++
7 files changed, 227 insertions(+), 23 deletions(-)
diff --git a/llvm/lib/Analysis/MetadataUtils.cpp b/llvm/lib/Analysis/MetadataUtils.cpp
index 646982f4118fe8..f93bdad0fc0183 100644
--- a/llvm/lib/Analysis/MetadataUtils.cpp
+++ b/llvm/lib/Analysis/MetadataUtils.cpp
@@ -190,13 +190,25 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
uint64_t(*DestByteOffset) <=
OldSize->getKnownMinValue() - NewSize.getKnownMinValue());
bool IsWholeAccess = IsSubset && DestByteOffset == 0 && OldSize == NewSize;
- bool SameElementType = SourceInfo->Ty && SourceInfo->Ty->getScalarType() ==
- DestInfo->Ty->getScalarType();
+ Type *SourceValueTy = SourceInfo->Ty;
+ // Allow unpacking a single-element aggregates.
+ if (IsWholeAccess) {
+ if (auto *ST = dyn_cast_or_null<StructType>(SourceValueTy);
+ ST && ST->getNumElements() == 1 &&
+ ST->getElementType(0) == DestInfo->Ty)
+ SourceValueTy = DestInfo->Ty;
+ else if (auto *AT = dyn_cast_or_null<ArrayType>(SourceValueTy);
+ AT && AT->getNumElements() == 1 &&
+ AT->getElementType() == DestInfo->Ty)
+ SourceValueTy = DestInfo->Ty;
+ }
+ bool SameElementType = SourceValueTy && SourceValueTy->getScalarType() ==
+ DestInfo->Ty->getScalarType();
bool SamePointer = getInvariantGroupPointer(getMemoryPointer(
Source, SourceInfo->PointerOperand)) ==
getInvariantGroupPointer(
getMemoryPointer(Dest, DestInfo->PointerOperand));
-
+
// Now loop over all `Source` metadata, propagating what we can.
SmallVector<std::pair<unsigned, MDNode *>, 8> MDs;
Source.getAllMetadata(MDs);
@@ -207,6 +219,7 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
case LLVMContext::MD_nontemporal:
case LLVMContext::MD_annotation:
case LLVMContext::MD_nosanitize:
+ case LLVMContext::MD_pcsections:
Keep = true;
break;
case LLVMContext::MD_mmra:
@@ -220,6 +233,7 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
Keep = IsSubset && DestInfo->IsRead;
break;
case LLVMContext::MD_invariant_group:
+ // TODO: Requiring `SamePointer` is too conservative.
Keep = IsSubset && SamePointer && !isa<CallBase>(Dest);
break;
case LLVMContext::MD_noundef:
@@ -228,15 +242,15 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
// doesn't have padding).
Keep = IsSubset && isa<LoadInst>(Dest) &&
(SameElementType ||
- (SourceInfo->Ty && !SourceInfo->Ty->isAggregateType() &&
- DL.typeSizeEqualsStoreSize(SourceInfo->Ty)));
+ (SourceValueTy && !SourceValueTy->isAggregateType() &&
+ DL.typeSizeEqualsStoreSize(SourceValueTy)));
break;
case LLVMContext::MD_fpmath:
case LLVMContext::MD_nofpclass:
- Keep =
- IsSubset && SameElementType &&
- (Kind == LLVMContext::MD_fpmath ? Dest.getType()->isFPOrFPVectorTy()
- : isa<LoadInst>(Dest));
+ Keep = IsSubset && SameElementType &&
+ (Kind == LLVMContext::MD_fpmath
+ ? FPMathOperator::isSupportedFloatingPointType(Dest.getType())
+ : isa<LoadInst>(Dest));
break;
case LLVMContext::MD_nonnull:
if (IsWholeAccess && isa<LoadInst>(Source) && isa<LoadInst>(Dest))
@@ -257,9 +271,14 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
Keep = IsWholeAccess && SourceInfo->Ty == DestInfo->Ty &&
DestInfo->Ty->isPointerTy() && isa<LoadInst>(Dest);
break;
- case LLVMContext::MD_DIAssignID:
- Keep = IsWholeAccess && isa<StoreInst>(Source) && isa<StoreInst>(Dest);
+ case LLVMContext::MD_DIAssignID: {
+ const auto *OldSI = dyn_cast<StoreInst>(&Source);
+ const auto *NewSI = dyn_cast<StoreInst>(&Dest);
+ // A new address requires updating the linked dbg.assign records.
+ Keep = IsWholeAccess && OldSI && NewSI &&
+ OldSI->getPointerOperand() == NewSI->getPointerOperand();
break;
+ }
case LLVMContext::MD_prof:
Keep =
IsWholeAccess && ((isa<LoadInst>(Source) && isa<LoadInst>(Dest)) ||
@@ -286,8 +305,12 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
} else if (!IsWholeAccess) {
if (DestByteOffset)
AA = AA.adjustForAccess(*DestByteOffset, DestInfo->Ty, DL);
- else
+ else {
+ // A common tag for the whole source also describes any selected part.
+ if (!AA.TBAA && SourceInfo->Ty)
+ AA.TBAA = AA.adjustForAccess(0, SourceInfo->Ty, DL).TBAA;
AA.TBAAStruct = nullptr;
+ }
} else {
// Prefer !tbaa over !tbaa.struct.
AAMDNodes Adjusted = AA.adjustForAccess(0, DestInfo->Ty, DL);
diff --git a/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp b/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp
index cfabc3f1595047..4b91390c2d8b0f 100644
--- a/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp
+++ b/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp
@@ -806,7 +806,7 @@ AAMDNodes AAMDNodes::adjustForAccess(unsigned AccessSize) {
MDNode *M = New.TBAAStruct;
// The access may cover several !tbaa.struct fields (e.g. a {int, int} copy
- // widened to an i64 load/store). If those fields share a single tag and tile
+ // widened to an i64 load/store). If those fields share a single tag and cover
// [0, AccessSize) with no gaps, that tag still describes the whole access.
New.TBAAStruct = nullptr;
if (New.TBAA || !M)
@@ -825,7 +825,8 @@ AAMDNodes AAMDNodes::adjustForAccess(unsigned AccessSize) {
(CommonTag && FieldTag != CommonTag))
break;
CommonTag = FieldTag;
- Offset += *FieldSize;
+ // The final field may extend beyond the access.
+ Offset += std::min(*FieldSize, uint64_t(AccessSize) - Offset);
}
if (Offset == AccessSize)
New.TBAA = CommonTag;
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 51ee1f6fa95087..d4caaaa22da546 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -408,10 +408,16 @@ static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit,
BaseFragments[getAggregateVariable(DVR)] =
DVR->getExpression()->getFragmentInfo();
- // The new inst needs a DIAssignID unique metadata tag (if OldInst has
- // one). It shouldn't already have one: assert this assumption.
- assert(!Inst->getMetadata(LLVMContext::MD_DIAssignID));
- DIAssignID *NewID = nullptr;
+ // Reuse an assignment ID copied for an unsplit store at the same address.
+ auto *NewID =
+ cast_or_null<DIAssignID>(Inst->getMetadata(LLVMContext::MD_DIAssignID));
+ if (NewID) {
+ assert(!IsSplit &&
+ NewID == OldInst->getMetadata(LLVMContext::MD_DIAssignID) &&
+ "unexpected assignment ID on replacement instruction");
+ // Deleting OldInst must not delete the records now linked to Inst.
+ OldInst->setMetadata(LLVMContext::MD_DIAssignID, nullptr);
+ }
auto &Ctx = Inst->getContext();
DIBuilder DIB(*OldInst->getModule(), /*AllowUnresolved*/ false);
assert(OldAlloca->isStaticAlloca());
@@ -478,7 +484,7 @@ static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit,
} else {
// The store is not split, simply steal the existing dbg_assign.
NewAssign = DbgAssign;
- NewAssign->setAssignId(NewID); // FIXME: Can we avoid generating new IDs?
+ NewAssign->setAssignId(NewID);
NewAssign->setAddress(Dest);
if (Value)
NewAssign->replaceVariableLocationOp(0u, Value);
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index bb2d24138d0965..01e3988562c8ff 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -2149,6 +2149,9 @@ bool VectorCombine::foldInsertElementsToStores(Instruction &I) {
Offset =
ConstIdx->getZExtValue() * DL->getTypeStoreSize(InsertVal->getType());
copyMetadataForMemoryAccess(*LastStore, *SI, Offset);
+ // Unchanged lanes remain in memory, so the original assignment still
+ // describes the whole vector after the replacement stores.
+ LastStore->copyMetadata(*SI, LLVMContext::MD_DIAssignID);
Align ScalarOpAlignment = computeAlignmentAfterScalarization(
std::max(SI->getAlign(), Load->getAlign()), InsertVal->getType(), Idx,
*DL);
diff --git a/llvm/test/Transforms/InstCombine/memory-metadata.ll b/llvm/test/Transforms/InstCombine/memory-metadata.ll
index b87598fb0098ed..845591936d0b61 100644
--- a/llvm/test/Transforms/InstCombine/memory-metadata.ll
+++ b/llvm/test/Transforms/InstCombine/memory-metadata.ll
@@ -136,6 +136,60 @@ define void @splat_scatter(ptr %p, i32 %v) {
ret void
}
+define i32 @single_struct_noundef(ptr %p) {
+; CHECK-LABEL: define i32 @single_struct_noundef(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[V_UNPACK:%.*]] = load i32, ptr [[P]], align 4, !noundef [[META4]]
+; CHECK-NEXT: ret i32 [[V_UNPACK]]
+;
+ %v = load {i32}, ptr %p, !noundef !0
+ %x = extractvalue {i32} %v, 0
+ ret i32 %x
+}
+
+define i32 @single_array_noundef(ptr %p) {
+; CHECK-LABEL: define i32 @single_array_noundef(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[V_UNPACK:%.*]] = load i32, ptr [[P]], align 4, !noundef [[META4]]
+; CHECK-NEXT: ret i32 [[V_UNPACK]]
+;
+ %v = load [1 x i32], ptr %p, !noundef !0
+ %x = extractvalue [1 x i32] %v, 0
+ ret i32 %x
+}
+
+define float @single_struct_fpmath(ptr %p) {
+; CHECK-LABEL: define float @single_struct_fpmath(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[V_UNPACK:%.*]] = load float, ptr [[P]], align 4, !fpmath [[META22:![0-9]+]], !noundef [[META4]], !nofpclass [[META23:![0-9]+]]
+; CHECK-NEXT: ret float [[V_UNPACK]]
+;
+ %v = load {float}, ptr %p, !fpmath !14, !nofpclass !15, !noundef !0
+ %x = extractvalue {float} %v, 0
+ ret float %x
+}
+
+define float @single_array_fpmath(ptr %p) {
+; CHECK-LABEL: define float @single_array_fpmath(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[V_UNPACK:%.*]] = load float, ptr [[P]], align 4, !fpmath [[META22]], !noundef [[META4]], !nofpclass [[META23]]
+; CHECK-NEXT: ret float [[V_UNPACK]]
+;
+ %v = load [1 x float], ptr %p, !fpmath !14, !nofpclass !15, !noundef !0
+ %x = extractvalue [1 x float] %v, 0
+ ret float %x
+}
+
+define <2 x i32> @unmask_pcsections(ptr %p) {
+; CHECK-LABEL: define <2 x i32> @unmask_pcsections(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[UNMASKEDLOAD:%.*]] = load <2 x i32>, ptr [[P]], align 1, !pcsections [[META24:![0-9]+]]
+; CHECK-NEXT: ret <2 x i32> [[UNMASKEDLOAD]]
+;
+ %v = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr %p, <2 x i1> <i1 true, i1 true>, <2 x i32> poison), !pcsections !26
+ ret <2 x i32> %v
+}
+
!0 = !{}
!1 = !{i32 1}
!2 = !{!"preserved memory metadata"}
@@ -162,6 +216,8 @@ define void @splat_scatter(ptr %p, i32 %v) {
!23 = !{!"nvvm.l1_eviction", !"first"}
!24 = !{!"branch_weights", i32 10}
!25 = !{i64 8}
+!26 = !{!"test_section", !27}
+!27 = !{i32 42}
;.
; CHECK: [[INT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
; CHECK: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
@@ -185,4 +241,8 @@ define void @splat_scatter(ptr %p, i32 %v) {
; CHECK: [[META19]] = !{i32 0, [[META20:![0-9]+]]}
; CHECK: [[META20]] = !{!"nvvm.l1_eviction", !"first"}
; CHECK: [[RNG21]] = !{i32 0, i32 100}
+; CHECK: [[META22]] = !{float 2.500000e+00}
+; CHECK: [[META23]] = !{i32 3}
+; CHECK: [[META24]] = !{!"test_section", [[META25:![0-9]+]]}
+; CHECK: [[META25]] = !{i32 42}
;.
diff --git a/llvm/test/Transforms/VectorCombine/memory-metadata.ll b/llvm/test/Transforms/VectorCombine/memory-metadata.ll
index cbf704e8780f45..ef1ddffd3f6688 100644
--- a/llvm/test/Transforms/VectorCombine/memory-metadata.ll
+++ b/llvm/test/Transforms/VectorCombine/memory-metadata.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -S -passes=vector-combine -data-layout=e-p:64:64 < %s | FileCheck %s
-; RUN: opt -S -passes=vector-combine -data-layout=E-p:64:64 < %s | FileCheck %s
+; RUN: opt -S -passes=vector-combine -data-layout=e-p:64:64 < %s | FileCheck %s --implicit-check-not=!test.unknown
+; RUN: opt -S -passes=vector-combine -data-layout=E-p:64:64 < %s | FileCheck %s --implicit-check-not=!test.unknown
define i32 @extract_zero(ptr %p) {
; CHECK-LABEL: define i32 @extract_zero(
@@ -80,7 +80,7 @@ define <4 x float> @shrink_float(ptr %p) {
define void @store_zero(ptr %p, i8 %x, i32 %index) {
; CHECK-LABEL: define void @store_zero(
; CHECK-SAME: ptr [[P:%.*]], i8 [[X:%.*]], i32 [[INDEX:%.*]]) {
-; CHECK-NEXT: store i8 [[X]], ptr [[P]], align 16, !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !invariant.group [[META4]], !llvm.access.group [[ACC_GRP9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META17:![0-9]+]]
+; CHECK-NEXT: store i8 [[X]], ptr [[P]], align 16, !tbaa [[CHAR_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !invariant.group [[META4]], !llvm.access.group [[ACC_GRP9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META17:![0-9]+]]
; CHECK-NEXT: ret void
;
%v = load <16 x i8>, ptr %p, align 16
@@ -109,7 +109,7 @@ define void @store_dynamic(ptr %p, i8 %x, i32 %index) {
; CHECK-NEXT: [[IDX:%.*]] = and i32 [[INDEX_FROZEN]], 15
; CHECK-NEXT: [[IDX_GEPIDX:%.*]] = zext i32 [[IDX]] to i64
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 [[IDX_GEPIDX]]
-; CHECK-NEXT: store i8 [[X]], ptr [[TMP1]], align 1, !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.access.group [[ACC_GRP9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META17]]
+; CHECK-NEXT: store i8 [[X]], ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.access.group [[ACC_GRP9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META17]]
; CHECK-NEXT: ret void
;
%idx = and i32 %index, 15
@@ -146,6 +146,42 @@ define void @store_struct_field(ptr %p, i32 %x) {
ret void
}
+define void @scalar_store(ptr %p, i8 %x) {
+; CHECK-LABEL: define void @scalar_store(
+; CHECK-SAME: ptr [[P:%.*]], i8 [[X:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 3
+; CHECK-NEXT: store i8 [[X]], ptr [[TMP1]], align 1, !pcsections [[META20:![0-9]+]]
+; CHECK-NEXT: ret void
+;
+ %v = load <16 x i8>, ptr %p, align 16
+ %i = insertelement <16 x i8> %v, i8 %x, i32 3
+ store <16 x i8> %i, ptr %p, align 16, !pcsections !22, !test.unknown !23
+ ret void
+}
+
+define i32 @whole_bitcast(ptr %p) {
+; CHECK-LABEL: define i32 @whole_bitcast(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[P]], align 4, !pcsections [[META20]]
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %v = load <4 x i8>, ptr %p, align 4, !pcsections !22, !test.unknown !23
+ %r = bitcast <4 x i8> %v to i32
+ ret i32 %r
+}
+
+define <4 x i32> @shrink_load(ptr %p) {
+; CHECK-LABEL: define <4 x i32> @shrink_load(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 32, !pcsections [[META20]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %v = load <8 x i32>, ptr %p, align 32, !pcsections !22, !test.unknown !23
+ %r = shufflevector <8 x i32> %v, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ ret <4 x i32> %r
+}
+
!0 = !{i32 1}
!1 = !{i32 0, !2}
!2 = !{!"nvvm.l1_eviction", !"last"}
@@ -169,6 +205,8 @@ define void @store_struct_field(ptr %p, i32 %x) {
!19 = !{!"int", !16, i64 0}
!20 = !{!19, !19, i64 0}
!21 = !{i64 0, i64 4, !15, i64 4, i64 4, !20, i64 8, i64 4, !15, i64 12, i64 4, !15}
+!22 = !{!"test_section"}
+!23 = !{!"custom annotation"}
;.
; CHECK: [[CHAR_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
; CHECK: [[META1]] = !{!"omnipotent char", [[META2:![0-9]+]], i64 0}
@@ -190,4 +228,5 @@ define void @store_struct_field(ptr %p, i32 %x) {
; CHECK: [[META17]] = !{i32 1, [[META14]]}
; CHECK: [[INT_TBAA18]] = !{[[META19:![0-9]+]], [[META19]], i64 0}
; CHECK: [[META19]] = !{!"int", [[META1]], i64 0}
+; CHECK: [[META20]] = !{!"test_section"}
;.
diff --git a/llvm/unittests/Analysis/MetadataUtilsTest.cpp b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
index 7066da1a69177a..b8e0a51fb608ef 100644
--- a/llvm/unittests/Analysis/MetadataUtilsTest.cpp
+++ b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
@@ -317,6 +317,66 @@ TEST_F(MetadataUtilsTest, AddressSpaceCastChangesInvariantGroupIdentity) {
EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_invariant_group));
}
+TEST_F(MetadataUtilsTest, SingleElementAggregatePreservesDefinedness) {
+ Type *Types[] = {StructType::get(C, {B.getInt32Ty()}, /*isPacked=*/false),
+ ArrayType::get(B.getInt32Ty(), 1)};
+ for (Type *Ty : Types) {
+ auto *Old = B.CreateLoad(Ty, P);
+ auto *New = B.CreateLoad(B.getInt32Ty(), P);
+ Old->setMetadata(LLVMContext::MD_noundef, empty());
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_noundef));
+ }
+}
+
+TEST_F(MetadataUtilsTest, SingleElementAggregatePreservesFloatingPointFacts) {
+ auto *FPMath = MDNode::get(
+ C, {ConstantAsMetadata::get(ConstantFP::get(B.getFloatTy(), 2.5))});
+ auto *NoFPClass = MDNode::get(C, {ConstantAsMetadata::get(B.getInt32(3))});
+ Type *Types[] = {StructType::get(C, {B.getFloatTy()}, /*isPacked=*/false),
+ ArrayType::get(B.getFloatTy(), 1)};
+ for (Type *Ty : Types) {
+ auto *Old = B.CreateLoad(Ty, P);
+ auto *New = B.CreateLoad(B.getFloatTy(), P);
+ Old->setMetadata(LLVMContext::MD_noundef, empty());
+ Old->setMetadata(LLVMContext::MD_fpmath, FPMath);
+ Old->setMetadata(LLVMContext::MD_nofpclass, NoFPClass);
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_noundef));
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_fpmath), FPMath);
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_nofpclass), NoFPClass);
+ }
+}
+
+TEST_F(MetadataUtilsTest, ShortAccessSelectsCoveringFieldTag) {
+ auto *Old = B.CreateLoad(B.getInt64Ty(), P);
+ auto *New = B.CreateLoad(B.getInt32Ty(), P);
+ MDBuilder MDB(C);
+ auto *Int = MDB.createTBAAScalarTypeNode("int", MDB.createTBAARoot("root"));
+ auto *Tag = MDB.createTBAAStructTagNode(Int, Int, 0);
+ Old->setMetadata(LLVMContext::MD_tbaa_struct,
+ MDNode::get(C, {number(0), number(8), Tag}));
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_tbaa), Tag);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_tbaa_struct));
+}
+
+TEST_F(MetadataUtilsTest, UnknownOffsetSelectsCommonFieldTag) {
+ auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 2), P);
+ auto *Index = B.CreateAnd(B.CreateLoad(B.getInt64Ty(), Q), B.getInt64(1));
+ auto *Ptr = B.CreateInBoundsGEP(B.getInt32Ty(), P, Index);
+ auto *New = B.CreateLoad(B.getInt32Ty(), Ptr);
+ MDBuilder MDB(C);
+ auto *Int = MDB.createTBAAScalarTypeNode("int", MDB.createTBAARoot("root"));
+ auto *Tag = MDB.createTBAAStructTagNode(Int, Int, 0);
+ Old->setMetadata(
+ LLVMContext::MD_tbaa_struct,
+ MDNode::get(C, {number(0), number(4), Tag, number(4), number(4), Tag}));
+ copyMetadataForMemoryAccess(*New, *Old, std::nullopt);
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_tbaa), Tag);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_tbaa_struct));
+}
+
TEST_F(MetadataUtilsTest, DefinedAggregateDoesNotDefinePadding) {
auto *Ty = StructType::get(C, {B.getInt8Ty(), B.getInt32Ty()});
ASSERT_EQ(M.getDataLayout().getTypeStoreSize(Ty).getFixedValue(), 8u);
@@ -422,6 +482,18 @@ TEST_F(MetadataUtilsTest, StoreFactsRequireWholeAccessAndPointerValue) {
EXPECT_FALSE(Part->hasMetadata(LLVMContext::MD_captures));
}
+TEST_F(MetadataUtilsTest, AssignmentIDRequiresUnchangedStoreAddress) {
+ auto *Old = B.CreateStore(B.getInt32(1), P);
+ auto *Same = B.CreateStore(B.getInt32(1), P);
+ auto *Moved = B.CreateStore(B.getInt32(1), Q);
+ auto *AssignID = DIAssignID::getDistinct(C);
+ Old->setMetadata(LLVMContext::MD_DIAssignID, AssignID);
+ copyMetadataForMemoryAccess(*Same, *Old);
+ copyMetadataForMemoryAccess(*Moved, *Old);
+ EXPECT_EQ(Same->getMetadata(LLVMContext::MD_DIAssignID), AssignID);
+ EXPECT_FALSE(Moved->hasMetadata(LLVMContext::MD_DIAssignID));
+}
+
TEST_F(MetadataUtilsTest, IntrinsicProfilesAndUnknownMetadataAreNotCopied) {
auto *Ty = FixedVectorType::get(B.getInt32Ty(), 2);
auto *Mask = B.CreateLoad(FixedVectorType::get(B.getInt1Ty(), 2), Q);
>From ab95e66696c4964ecbd9ca88680e956198f82d75 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 17 Nov 2026 21:31:01 +0000
Subject: [PATCH 05/14] require fixed length mem intrinsics
---
llvm/include/llvm/Analysis/MetadataUtils.h | 23 ++++++++-------
llvm/lib/Analysis/MetadataUtils.cpp | 28 ++++++++-----------
llvm/unittests/Analysis/MetadataUtilsTest.cpp | 12 --------
3 files changed, 23 insertions(+), 40 deletions(-)
diff --git a/llvm/include/llvm/Analysis/MetadataUtils.h b/llvm/include/llvm/Analysis/MetadataUtils.h
index 2d7ce97f1cdd13..25bac66bdc5d24 100644
--- a/llvm/include/llvm/Analysis/MetadataUtils.h
+++ b/llvm/include/llvm/Analysis/MetadataUtils.h
@@ -21,18 +21,17 @@ class Instruction;
class MDNode;
class Value;
-/// Copy metadata valid for a replacement memory access. The destination must
-/// access the same bytes, a subset, or an expanded range in the same object.
-/// The destination must be a typed memory access; memory-transfer and
-/// memory-set intrinsics are supported only as sources. DestByteOffset is the
-/// destination's offset within the source access. A missing offset requires the
-/// caller to establish that the new access is a subset. Access types and
-/// pointer operands are inferred for loads, stores, and masked/VP memory
-/// intrinsics. For a source memory intrinsic, the source or destination pointer
-/// is selected to match the new read or write. The caller must establish range
-/// containment when the intrinsic's length is unknown. Pointer and value facts
-/// are retained only when valid for the destination; alias tags are adjusted
-/// for its range. Masks and execution conditions are not inspected.
+/// Copy metadata valid for a replacement memory access.
+///
+/// The destination must access the same bytes, a subset, or a wider byte range
+/// in the same object as the source. The destination must be a typed memory
+/// access. Memory-transfer and memory-set intrinsics are supported only as
+/// sources and must have a constant length.
+///
+/// DestByteOffset is the destination's offset within the source access. If the
+/// destination is no larger than the source, a missing offset means the offset
+/// is dynamic, but the caller guarentees it accesses a subset of the source
+/// access.
LLVM_ABI void
copyMetadataForMemoryAccess(Instruction &Dest, const Instruction &Source,
std::optional<int64_t> DestByteOffset = 0);
diff --git a/llvm/lib/Analysis/MetadataUtils.cpp b/llvm/lib/Analysis/MetadataUtils.cpp
index f93bdad0fc0183..9892612889396b 100644
--- a/llvm/lib/Analysis/MetadataUtils.cpp
+++ b/llvm/lib/Analysis/MetadataUtils.cpp
@@ -170,25 +170,21 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
assert(SourceInfo && SourceInfo->IsRead == DestInfo->IsRead &&
"expected corresponding memory reads or writes");
const DataLayout &DL = Source.getDataLayout();
- // Collect the size of the old memory operation if available.
- std::optional<TypeSize> OldSize;
- if (SourceInfo->Ty)
- OldSize = DL.getTypeStoreSize(SourceInfo->Ty);
- else if (auto *Length =
- dyn_cast<ConstantInt>(cast<AnyMemIntrinsic>(Source).getLength()))
- // If Source is not a typed memory operation it must be an intrinsic.
- if (auto Bytes = Length->getValue().tryZExtValue())
- OldSize = TypeSize::getFixed(*Bytes);
+ TypeSize OldSize =
+ SourceInfo->Ty
+ ? DL.getTypeStoreSize(SourceInfo->Ty)
+ : TypeSize::getFixed(
+ cast<ConstantInt>(cast<AnyMemIntrinsic>(Source).getLength())
+ ->getZExtValue());
TypeSize NewSize = DL.getTypeStoreSize(DestInfo->Ty);
- // Is the memory `Dest` accesses a subset of the memory `Source` acceses?
- // The caller guarentees this is true if we don't know the size of `Source`
- // or the offset of the `Dest` access in `Source`.
- bool IsSubset = !OldSize || TypeSize::isKnownLE(NewSize, *OldSize);
+ // Is the destination access contained in the source access? If the size
+ // check succeeds but the offset is unknown, containment is a caller
+ // guarantee.
+ bool IsSubset = TypeSize::isKnownLE(NewSize, OldSize);
if (DestByteOffset)
IsSubset = IsSubset && *DestByteOffset >= 0 &&
- (!OldSize ||
- uint64_t(*DestByteOffset) <=
- OldSize->getKnownMinValue() - NewSize.getKnownMinValue());
+ uint64_t(*DestByteOffset) <=
+ OldSize.getKnownMinValue() - NewSize.getKnownMinValue();
bool IsWholeAccess = IsSubset && DestByteOffset == 0 && OldSize == NewSize;
Type *SourceValueTy = SourceInfo->Ty;
// Allow unpacking a single-element aggregates.
diff --git a/llvm/unittests/Analysis/MetadataUtilsTest.cpp b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
index b8e0a51fb608ef..b55922ebd0893c 100644
--- a/llvm/unittests/Analysis/MetadataUtilsTest.cpp
+++ b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
@@ -259,18 +259,6 @@ TEST_F(MetadataUtilsTest, MemoryTransferRemapsBothOperands) {
EXPECT_FALSE(getMemCacheHint(*Write, 0));
}
-TEST_F(MetadataUtilsTest, RuntimeMemoryTransferLength) {
- auto *Length = B.CreateLoad(B.getInt64Ty(), Q);
- B.CreateAssumption(B.CreateICmpUGE(Length, B.getInt64(8)));
- auto *Copy = B.CreateMemCpy(P, Align(4), Q, Align(4), Length);
- auto *Ptr = B.CreateInBoundsGEP(B.getInt8Ty(), Q, B.getInt64(4));
- auto *Read = B.CreateLoad(B.getInt32Ty(), Ptr);
- auto *Hint = hint();
- setMemCacheHint(*Copy, 1, Hint);
- copyMetadataForMemoryAccess(*Read, *Copy, 4);
- EXPECT_EQ(getMemCacheHint(*Read), Hint);
-}
-
TEST_F(MetadataUtilsTest, StoreTypeChangePreservesHints) {
auto *Old = B.CreateStore(B.getInt32(0), P);
auto *New = B.CreateStore(ConstantFP::get(B.getFloatTy(), 0), P);
>From 90af05a88a26c0dd2ecc363170543be9414112ec Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 17 Nov 2026 21:34:54 +0000
Subject: [PATCH 06/14] get erid of general vp
---
llvm/lib/Analysis/MetadataUtils.cpp | 12 +-----------
1 file changed, 1 insertion(+), 11 deletions(-)
diff --git a/llvm/lib/Analysis/MetadataUtils.cpp b/llvm/lib/Analysis/MetadataUtils.cpp
index 9892612889396b..4c51eda0e9de93 100644
--- a/llvm/lib/Analysis/MetadataUtils.cpp
+++ b/llvm/lib/Analysis/MetadataUtils.cpp
@@ -56,17 +56,6 @@ getMemoryAccessInfo(const Instruction &I,
const auto *II = dyn_cast<IntrinsicInst>(&I);
if (!II)
return std::nullopt;
- if (const auto *VP = dyn_cast<VPIntrinsic>(II)) {
- auto Ptr = VPIntrinsic::getMemoryPointerParamPos(VP->getIntrinsicID());
- if (!Ptr)
- return std::nullopt;
- if (auto Data = VPIntrinsic::getMemoryDataParamPos(VP->getIntrinsicID()))
- return MemoryAccessInfo{VP->getArgOperand(*Data)->getType(), *Ptr, false};
- Type *Ty = VP->getType();
- if (VP->getIntrinsicID() == Intrinsic::vp_load_ff)
- Ty = Ty->getStructElementType(0);
- return MemoryAccessInfo{Ty, *Ptr, true};
- }
switch (II->getIntrinsicID()) {
case Intrinsic::experimental_vector_histogram_add:
case Intrinsic::experimental_vector_histogram_uadd_sat:
@@ -78,6 +67,7 @@ getMemoryAccessInfo(const Instruction &I,
case Intrinsic::masked_load:
case Intrinsic::masked_gather:
case Intrinsic::masked_expandload:
+ case Intrinsic::vp_load:
return MemoryAccessInfo{II->getType(), 0, true};
case Intrinsic::masked_store:
case Intrinsic::masked_scatter:
>From 346ad6a17bdd3b173cc7df6a6b35df7f10b23b12 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 17 Nov 2026 21:40:42 +0000
Subject: [PATCH 07/14] comment update
---
llvm/include/llvm/Analysis/MetadataUtils.h | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/Analysis/MetadataUtils.h b/llvm/include/llvm/Analysis/MetadataUtils.h
index 25bac66bdc5d24..2952bfca8402f5 100644
--- a/llvm/include/llvm/Analysis/MetadataUtils.h
+++ b/llvm/include/llvm/Analysis/MetadataUtils.h
@@ -36,8 +36,10 @@ LLVM_ABI void
copyMetadataForMemoryAccess(Instruction &Dest, const Instruction &Source,
std::optional<int64_t> DestByteOffset = 0);
-/// Copy metadata valid after splitting an instruction into smaller elements.
-/// DestByteOffset describes the selected memory fragment, if applicable.
+/// Copy metadata valid after splitting a vector instruction into scalar or
+/// smaller vector parts.
+/// For memory accesses, DestByteOffset is the destination's byte offset within
+/// the source access.
LLVM_ABI void copyMetadataForScalarizedInstruction(
Instruction &Dest, const Instruction &Source,
std::optional<int64_t> DestByteOffset = std::nullopt);
>From fce4e45c363e6de3df227a9987a2c8dc7053b745 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 17 Nov 2026 23:24:27 +0000
Subject: [PATCH 08/14] new test
---
.../vector-combine/store.ll | 91 +++++++++++++++++++
1 file changed, 91 insertions(+)
create mode 100644 llvm/test/DebugInfo/Generic/assignment-tracking/vector-combine/store.ll
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/vector-combine/store.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/vector-combine/store.ll
new file mode 100644
index 00000000000000..924fc7194174f7
--- /dev/null
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/vector-combine/store.ll
@@ -0,0 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=vector-combine < %s | FileCheck %s
+
+declare void @init(ptr)
+declare void @observe(ptr)
+
+; The replacement scalar store stays linked to the original dbg.assign.
+define void @scalar_store(i8 %x) !dbg !6 {
+; CHECK-LABEL: define void @scalar_store(
+; CHECK-SAME: i8 [[X:%.*]]) !dbg [[DBG4:![0-9]+]] {
+; CHECK-NEXT: [[P:%.*]] = alloca <16 x i8>, align 16
+; CHECK-NEXT: call void @init(ptr [[P]]), !dbg [[DBG7:![0-9]+]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 3, !dbg [[DBG7]]
+; CHECK-NEXT: store i8 [[X]], ptr [[TMP1]], align 1, !dbg [[DBG7]], !DIAssignID [[DIASSIGNID8:![0-9]+]]
+; CHECK-NEXT: #dbg_assign(<16 x i8> poison, [[META9:![0-9]+]], !DIExpression(), [[DIASSIGNID8]], ptr [[P]], !DIExpression(), [[DBG7]])
+; CHECK-NEXT: call void @observe(ptr [[P]]), !dbg [[DBG7]]
+; CHECK-NEXT: ret void, !dbg [[DBG7]]
+;
+ %p = alloca <16 x i8>, align 16
+ call void @init(ptr %p), !dbg !10
+ %v = load <16 x i8>, ptr %p, align 16, !dbg !10
+ %i = insertelement <16 x i8> %v, i8 %x, i32 3, !dbg !10
+ store <16 x i8> %i, ptr %p, align 16, !DIAssignID !0, !dbg !10
+ #dbg_assign(<16 x i8> %i, !11, !DIExpression(), !0, ptr %p, !DIExpression(), !10)
+ call void @observe(ptr %p), !dbg !10
+ ret void, !dbg !10
+}
+
+; Both replacement scalar stores share the original dbg.assign's assignment ID.
+define void @two_updates(i8 %x, i8 %y) !dbg !17 {
+; CHECK-LABEL: define void @two_updates(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) !dbg [[DBG14:![0-9]+]] {
+; CHECK-NEXT: [[P:%.*]] = alloca <16 x i8>, align 16
+; CHECK-NEXT: call void @init(ptr [[P]]), !dbg [[DBG15:![0-9]+]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 3, !dbg [[DBG15]]
+; CHECK-NEXT: store i8 [[X]], ptr [[TMP1]], align 1, !dbg [[DBG15]], !DIAssignID [[DIASSIGNID16:![0-9]+]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 7, !dbg [[DBG15]]
+; CHECK-NEXT: store i8 [[Y]], ptr [[TMP2]], align 1, !dbg [[DBG15]], !DIAssignID [[DIASSIGNID16]]
+; CHECK-NEXT: #dbg_assign(<16 x i8> poison, [[META17:![0-9]+]], !DIExpression(), [[DIASSIGNID16]], ptr [[P]], !DIExpression(), [[DBG15]])
+; CHECK-NEXT: call void @observe(ptr [[P]]), !dbg [[DBG15]]
+; CHECK-NEXT: ret void, !dbg [[DBG15]]
+;
+ %p = alloca <16 x i8>, align 16
+ call void @init(ptr %p), !dbg !18
+ %v = load <16 x i8>, ptr %p, align 16, !dbg !18
+ %i = insertelement <16 x i8> %v, i8 %x, i32 3, !dbg !18
+ %j = insertelement <16 x i8> %i, i8 %y, i32 7, !dbg !18
+ store <16 x i8> %j, ptr %p, align 16, !DIAssignID !20, !dbg !18
+ #dbg_assign(<16 x i8> %j, !19, !DIExpression(), !20, ptr %p, !DIExpression(), !18)
+ call void @observe(ptr %p), !dbg !18
+ ret void, !dbg !18
+}
+!llvm.module.flags = !{!4, !5}
+!llvm.dbg.cu = !{!7}
+!0 = distinct !DIAssignID()
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 7, !"debug-info-assignment-tracking", i1 true}
+!6 = distinct !DISubprogram(name: "scalar_store", scope: !8, file: !8, line: 1, type: !9, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !7)
+!7 = distinct !DICompileUnit(language: DW_LANG_C, file: !8, producer: "audit", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!8 = !DIFile(filename: "audit.c", directory: "/")
+!9 = !DISubroutineType(types: !13)
+!10 = !DILocation(line: 2, column: 1, scope: !6)
+!11 = !DILocalVariable(name: "buffer", scope: !6, file: !8, line: 1, type: !12)
+!12 = !DICompositeType(tag: DW_TAG_array_type, baseType: !14, size: 128, elements: !15)
+!13 = !{}
+!14 = !DIBasicType(name: "byte", size: 8, encoding: DW_ATE_unsigned_char)
+!15 = !{!16}
+!16 = !DISubrange(count: 16)
+
+!17 = distinct !DISubprogram(name: "two_updates", scope: !8, file: !8, line: 1, type: !9, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !7)
+!18 = !DILocation(line: 2, column: 1, scope: !17)
+!19 = !DILocalVariable(name: "buffer2", scope: !17, file: !8, line: 1, type: !12)
+!20 = distinct !DIAssignID()
+;.
+; CHECK: [[META2:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C, file: [[META3:![0-9]+]], producer: "audit", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+; CHECK: [[META3]] = !DIFile(filename: "{{.*}}audit.c", directory: {{.*}})
+; CHECK: [[DBG4]] = distinct !DISubprogram(name: "scalar_store", scope: [[META3]], file: [[META3]], line: 1, type: [[META5:![0-9]+]], scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META2]])
+; CHECK: [[META5]] = !DISubroutineType(types: [[META6:![0-9]+]])
+; CHECK: [[META6]] = !{}
+; CHECK: [[DBG7]] = !DILocation(line: 2, column: 1, scope: [[DBG4]])
+; CHECK: [[DIASSIGNID8]] = distinct !DIAssignID()
+; CHECK: [[META9]] = !DILocalVariable(name: "buffer", scope: [[DBG4]], file: [[META3]], line: 1, type: [[META10:![0-9]+]])
+; CHECK: [[META10]] = !DICompositeType(tag: DW_TAG_array_type, baseType: [[META11:![0-9]+]], size: 128, elements: [[META12:![0-9]+]])
+; CHECK: [[META11]] = !DIBasicType(name: "byte", size: 8, encoding: DW_ATE_unsigned_char)
+; CHECK: [[META12]] = !{[[META13:![0-9]+]]}
+; CHECK: [[META13]] = !DISubrange(count: 16)
+; CHECK: [[DBG14]] = distinct !DISubprogram(name: "two_updates", scope: [[META3]], file: [[META3]], line: 1, type: [[META5]], scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META2]])
+; CHECK: [[DBG15]] = !DILocation(line: 2, column: 1, scope: [[DBG14]])
+; CHECK: [[DIASSIGNID16]] = distinct !DIAssignID()
+; CHECK: [[META17]] = !DILocalVariable(name: "buffer2", scope: [[DBG14]], file: [[META3]], line: 1, type: [[META10]])
+;.
>From 5021c2895e812120051a36d54cb71dc693402c50 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 17 Nov 2026 23:30:30 +0000
Subject: [PATCH 09/14] [Analysis] Format relocated metadata combining helpers
---
llvm/lib/Analysis/MetadataUtils.cpp | 240 ++++++++++++++--------------
1 file changed, 119 insertions(+), 121 deletions(-)
diff --git a/llvm/lib/Analysis/MetadataUtils.cpp b/llvm/lib/Analysis/MetadataUtils.cpp
index 4c51eda0e9de93..02f6f8acb77d41 100644
--- a/llvm/lib/Analysis/MetadataUtils.cpp
+++ b/llvm/lib/Analysis/MetadataUtils.cpp
@@ -498,128 +498,127 @@ static void combineMetadata(Instruction *K, const Instruction *J,
// TODO: Assert that this switch is exhaustive for fixed MD kinds.
switch (Kind) {
- default:
- K->setMetadata(Kind, nullptr); // Remove unknown metadata
- break;
- case LLVMContext::MD_dbg:
- llvm_unreachable("getAllMetadataOtherThanDebugLoc returned a MD_dbg");
- case LLVMContext::MD_DIAssignID:
- if (!AAOnly)
- K->mergeDIAssignID(J);
- break;
- case LLVMContext::MD_tbaa:
- if (DoesKMove)
- K->setMetadata(Kind, MDNode::getMostGenericTBAA(JMD, KMD));
- break;
- case LLVMContext::MD_alias_scope:
- if (DoesKMove)
- K->setMetadata(Kind, MDNode::getMostGenericAliasScope(JMD, KMD));
- break;
- case LLVMContext::MD_noalias:
- case LLVMContext::MD_mem_parallel_loop_access:
- if (DoesKMove)
- K->setMetadata(Kind, MDNode::intersect(JMD, KMD));
- break;
- case LLVMContext::MD_access_group:
- if (DoesKMove)
- K->setMetadata(LLVMContext::MD_access_group,
- intersectAccessGroups(K, J));
- break;
- case LLVMContext::MD_range:
- if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
- K->setMetadata(Kind, MDNode::getMostGenericRange(JMD, KMD));
- break;
- case LLVMContext::MD_nofpclass:
- if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
- K->setMetadata(Kind, MDNode::getMostGenericNoFPClass(JMD, KMD));
- break;
- case LLVMContext::MD_fpmath:
- if (!AAOnly)
- K->setMetadata(Kind, MDNode::getMostGenericFPMath(JMD, KMD));
- break;
- case LLVMContext::MD_invariant_load:
- case LLVMContext::MD_invariant_group:
- // If K moves, only keep the invariant metadata if it is present on
- // both instructions; otherwise the invariant would be asserted on a
- // path (J's) that never promised it. If K does not move, K stays on
- // its original path, so its existing metadata remains valid.
- if (DoesKMove)
- K->setMetadata(Kind, JMD);
- break;
- case LLVMContext::MD_nonnull:
- if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
- K->setMetadata(Kind, JMD);
- break;
- // Keep empty cases for prof, mmra, memprof, and callsite to prevent them
- // from being removed as unknown metadata. The actual merging is handled
- // separately below.
- case LLVMContext::MD_prof:
- case LLVMContext::MD_mmra:
- case LLVMContext::MD_memprof:
- case LLVMContext::MD_callsite:
- break;
- case LLVMContext::MD_callee_type:
- if (!AAOnly) {
- K->setMetadata(LLVMContext::MD_callee_type,
- MDNode::getMergedCalleeTypeMetadata(KMD, JMD));
- }
- break;
- case LLVMContext::MD_callees:
- // If K moves, it replaces J on J's path and must allow J's callees as
- // well. If K does not move, its callees remain valid.
- if (!AAOnly && DoesKMove)
- K->setMetadata(Kind, MDNode::getMergedCalleesMetadata(KMD, JMD));
- break;
- case LLVMContext::MD_align:
- if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
- K->setMetadata(
- Kind, MDNode::getMostGenericAlignmentOrDereferenceable(JMD, KMD));
- break;
- case LLVMContext::MD_dereferenceable:
- case LLVMContext::MD_dereferenceable_or_null:
- if (!AAOnly && DoesKMove)
- K->setMetadata(Kind,
- MDNode::getMostGenericAlignmentOrDereferenceable(JMD, KMD));
- break;
- case LLVMContext::MD_preserve_access_index:
- // Preserve !preserve.access.index in K.
- break;
- case LLVMContext::MD_noundef:
- // If K does move, keep noundef if it is present in both instructions.
- if (!AAOnly && DoesKMove)
- K->setMetadata(Kind, JMD);
- break;
- case LLVMContext::MD_nontemporal:
- // Preserve !nontemporal if it is present on both instructions.
- if (!AAOnly)
- K->setMetadata(Kind, JMD);
- break;
- case LLVMContext::MD_mem_cache_hint:
- // Preserve !mem.cache_hint only if it is present and equivalent on both
- // instructions.
- if (!AAOnly && KMD != JMD)
- K->setMetadata(Kind, nullptr);
- break;
- case LLVMContext::MD_noalias_addrspace:
- if (DoesKMove)
- K->setMetadata(Kind,
- MDNode::getMostGenericNoaliasAddrspace(JMD, KMD));
- break;
- case LLVMContext::MD_nosanitize:
- // Preserve !nosanitize if both K and J have it.
+ default:
+ K->setMetadata(Kind, nullptr); // Remove unknown metadata
+ break;
+ case LLVMContext::MD_dbg:
+ llvm_unreachable("getAllMetadataOtherThanDebugLoc returned a MD_dbg");
+ case LLVMContext::MD_DIAssignID:
+ if (!AAOnly)
+ K->mergeDIAssignID(J);
+ break;
+ case LLVMContext::MD_tbaa:
+ if (DoesKMove)
+ K->setMetadata(Kind, MDNode::getMostGenericTBAA(JMD, KMD));
+ break;
+ case LLVMContext::MD_alias_scope:
+ if (DoesKMove)
+ K->setMetadata(Kind, MDNode::getMostGenericAliasScope(JMD, KMD));
+ break;
+ case LLVMContext::MD_noalias:
+ case LLVMContext::MD_mem_parallel_loop_access:
+ if (DoesKMove)
+ K->setMetadata(Kind, MDNode::intersect(JMD, KMD));
+ break;
+ case LLVMContext::MD_access_group:
+ if (DoesKMove)
+ K->setMetadata(LLVMContext::MD_access_group,
+ intersectAccessGroups(K, J));
+ break;
+ case LLVMContext::MD_range:
+ if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
+ K->setMetadata(Kind, MDNode::getMostGenericRange(JMD, KMD));
+ break;
+ case LLVMContext::MD_nofpclass:
+ if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
+ K->setMetadata(Kind, MDNode::getMostGenericNoFPClass(JMD, KMD));
+ break;
+ case LLVMContext::MD_fpmath:
+ if (!AAOnly)
+ K->setMetadata(Kind, MDNode::getMostGenericFPMath(JMD, KMD));
+ break;
+ case LLVMContext::MD_invariant_load:
+ case LLVMContext::MD_invariant_group:
+ // If K moves, only keep the invariant metadata if it is present on
+ // both instructions; otherwise the invariant would be asserted on a
+ // path (J's) that never promised it. If K does not move, K stays on
+ // its original path, so its existing metadata remains valid.
+ if (DoesKMove)
K->setMetadata(Kind, JMD);
- break;
- case LLVMContext::MD_captures:
- K->setMetadata(
- Kind, MDNode::fromCaptureComponents(
- K->getContext(), MDNode::toCaptureComponents(JMD) |
- MDNode::toCaptureComponents(KMD)));
- break;
- case LLVMContext::MD_alloc_token:
- if (!AAOnly && KMD != JMD)
- K->setMetadata(Kind, MDNode::getMergedAllocTokenMetadata(KMD, JMD));
- break;
+ break;
+ case LLVMContext::MD_nonnull:
+ if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
+ K->setMetadata(Kind, JMD);
+ break;
+ // Keep empty cases for prof, mmra, memprof, and callsite to prevent them
+ // from being removed as unknown metadata. The actual merging is handled
+ // separately below.
+ case LLVMContext::MD_prof:
+ case LLVMContext::MD_mmra:
+ case LLVMContext::MD_memprof:
+ case LLVMContext::MD_callsite:
+ break;
+ case LLVMContext::MD_callee_type:
+ if (!AAOnly) {
+ K->setMetadata(LLVMContext::MD_callee_type,
+ MDNode::getMergedCalleeTypeMetadata(KMD, JMD));
}
+ break;
+ case LLVMContext::MD_callees:
+ // If K moves, it replaces J on J's path and must allow J's callees as
+ // well. If K does not move, its callees remain valid.
+ if (!AAOnly && DoesKMove)
+ K->setMetadata(Kind, MDNode::getMergedCalleesMetadata(KMD, JMD));
+ break;
+ case LLVMContext::MD_align:
+ if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
+ K->setMetadata(
+ Kind, MDNode::getMostGenericAlignmentOrDereferenceable(JMD, KMD));
+ break;
+ case LLVMContext::MD_dereferenceable:
+ case LLVMContext::MD_dereferenceable_or_null:
+ if (!AAOnly && DoesKMove)
+ K->setMetadata(
+ Kind, MDNode::getMostGenericAlignmentOrDereferenceable(JMD, KMD));
+ break;
+ case LLVMContext::MD_preserve_access_index:
+ // Preserve !preserve.access.index in K.
+ break;
+ case LLVMContext::MD_noundef:
+ // If K does move, keep noundef if it is present in both instructions.
+ if (!AAOnly && DoesKMove)
+ K->setMetadata(Kind, JMD);
+ break;
+ case LLVMContext::MD_nontemporal:
+ // Preserve !nontemporal if it is present on both instructions.
+ if (!AAOnly)
+ K->setMetadata(Kind, JMD);
+ break;
+ case LLVMContext::MD_mem_cache_hint:
+ // Preserve !mem.cache_hint only if it is present and equivalent on both
+ // instructions.
+ if (!AAOnly && KMD != JMD)
+ K->setMetadata(Kind, nullptr);
+ break;
+ case LLVMContext::MD_noalias_addrspace:
+ if (DoesKMove)
+ K->setMetadata(Kind, MDNode::getMostGenericNoaliasAddrspace(JMD, KMD));
+ break;
+ case LLVMContext::MD_nosanitize:
+ // Preserve !nosanitize if both K and J have it.
+ K->setMetadata(Kind, JMD);
+ break;
+ case LLVMContext::MD_captures:
+ K->setMetadata(
+ Kind, MDNode::fromCaptureComponents(
+ K->getContext(), MDNode::toCaptureComponents(JMD) |
+ MDNode::toCaptureComponents(KMD)));
+ break;
+ case LLVMContext::MD_alloc_token:
+ if (!AAOnly && KMD != JMD)
+ K->setMetadata(Kind, MDNode::getMergedAllocTokenMetadata(KMD, JMD));
+ break;
+ }
}
// Merge MMRAs.
@@ -676,4 +675,3 @@ void llvm::combineMetadataForCSE(Instruction *K, const Instruction *J,
void llvm::combineAAMetadata(Instruction *K, const Instruction *J) {
combineMetadata(K, J, /*DoesKMove=*/true, /*AAOnly=*/true);
}
-
>From fb1eba43c149ab6f94979ae53c05fa60bb038518 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 17 Nov 2026 23:34:58 +0000
Subject: [PATCH 10/14] [MetadataUtils] Update for current VectorCombine
transforms
---
llvm/lib/Transforms/Vectorize/VectorCombine.cpp | 9 ++-------
.../assignment-tracking/vector-combine/store.ll | 6 +++---
.../Transforms/VectorCombine/memory-metadata.ll | 16 ++++++++--------
3 files changed, 13 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 01e3988562c8ff..39c5f4d87780a1 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -7094,13 +7094,8 @@ bool VectorCombine::foldShuffleOfAdjacentLoads(Instruction &I) {
LoadInst *WideLoad = Builder.CreateAlignedLoad(
WideTy, LowLoad->getPointerOperand(), LowLoad->getAlign());
- // Set the metadata on the wide load. copyMetadataForLoad seeds it from
- // LowLoad, then combineMetadataForCSE intersects every known kind against
- // HighLoad (taking the most-generic value where applicable, keeping facts
- // only where both loads agree, and dropping unknown metadata), so nothing is
- // asserted over the combined load unless justified by both halves.
- copyMetadataForLoad(*WideLoad, *LowLoad);
- combineMetadataForCSE(WideLoad, HighLoad, /*DoesKMove=*/true);
+ // Merge metadata from both loads over the combined access.
+ propagateMetadata(WideLoad, {LowLoad, HighLoad});
for (ShuffleVectorInst *SV : Shuffles) {
SmallVector<int, 32> NewMask;
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/vector-combine/store.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/vector-combine/store.ll
index 924fc7194174f7..e9a752707b206c 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/vector-combine/store.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/vector-combine/store.ll
@@ -10,7 +10,7 @@ define void @scalar_store(i8 %x) !dbg !6 {
; CHECK-SAME: i8 [[X:%.*]]) !dbg [[DBG4:![0-9]+]] {
; CHECK-NEXT: [[P:%.*]] = alloca <16 x i8>, align 16
; CHECK-NEXT: call void @init(ptr [[P]]), !dbg [[DBG7:![0-9]+]]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 3, !dbg [[DBG7]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 3, !dbg [[DBG7]]
; CHECK-NEXT: store i8 [[X]], ptr [[TMP1]], align 1, !dbg [[DBG7]], !DIAssignID [[DIASSIGNID8:![0-9]+]]
; CHECK-NEXT: #dbg_assign(<16 x i8> poison, [[META9:![0-9]+]], !DIExpression(), [[DIASSIGNID8]], ptr [[P]], !DIExpression(), [[DBG7]])
; CHECK-NEXT: call void @observe(ptr [[P]]), !dbg [[DBG7]]
@@ -32,9 +32,9 @@ define void @two_updates(i8 %x, i8 %y) !dbg !17 {
; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) !dbg [[DBG14:![0-9]+]] {
; CHECK-NEXT: [[P:%.*]] = alloca <16 x i8>, align 16
; CHECK-NEXT: call void @init(ptr [[P]]), !dbg [[DBG15:![0-9]+]]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 3, !dbg [[DBG15]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 3, !dbg [[DBG15]]
; CHECK-NEXT: store i8 [[X]], ptr [[TMP1]], align 1, !dbg [[DBG15]], !DIAssignID [[DIASSIGNID16:![0-9]+]]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 7, !dbg [[DBG15]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 7, !dbg [[DBG15]]
; CHECK-NEXT: store i8 [[Y]], ptr [[TMP2]], align 1, !dbg [[DBG15]], !DIAssignID [[DIASSIGNID16]]
; CHECK-NEXT: #dbg_assign(<16 x i8> poison, [[META17:![0-9]+]], !DIExpression(), [[DIASSIGNID16]], ptr [[P]], !DIExpression(), [[DBG15]])
; CHECK-NEXT: call void @observe(ptr [[P]]), !dbg [[DBG15]]
diff --git a/llvm/test/Transforms/VectorCombine/memory-metadata.ll b/llvm/test/Transforms/VectorCombine/memory-metadata.ll
index ef1ddffd3f6688..dac87dd4638a92 100644
--- a/llvm/test/Transforms/VectorCombine/memory-metadata.ll
+++ b/llvm/test/Transforms/VectorCombine/memory-metadata.ll
@@ -16,7 +16,7 @@ define i32 @extract_zero(ptr %p) {
define i32 @extract_nonzero(ptr %p) {
; CHECK-LABEL: define i32 @extract_nonzero(
; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 2
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP1]], align 8, !tbaa [[CHAR_TBAA0]], !range [[RNG3]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.access.group [[ACC_GRP9]], !noundef [[META4]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META13]]
; CHECK-NEXT: ret i32 [[R]]
;
@@ -31,7 +31,7 @@ define i32 @extract_dynamic(ptr %p, i32 %index) {
; CHECK-NEXT: [[INDEX_FROZEN:%.*]] = freeze i32 [[INDEX]]
; CHECK-NEXT: [[IDX:%.*]] = and i32 [[INDEX_FROZEN]], 3
; CHECK-NEXT: [[IDX_GEPIDX:%.*]] = zext i32 [[IDX]] to i64
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 [[IDX_GEPIDX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [4 x i8], ptr [[P]], i64 [[IDX_GEPIDX]]
; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[CHAR_TBAA0]], !range [[RNG3]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.access.group [[ACC_GRP9]], !noundef [[META4]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META13]]
; CHECK-NEXT: ret i32 [[R]]
;
@@ -44,7 +44,7 @@ define i32 @extract_dynamic(ptr %p, i32 %index) {
define float @extract_float(ptr %p) {
; CHECK-LABEL: define float @extract_float(
; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x float>, ptr [[P]], i64 0, i64 1
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 4
; CHECK-NEXT: [[R:%.*]] = load float, ptr [[TMP1]], align 4, !fpmath [[META15:![0-9]+]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.access.group [[ACC_GRP9]], !noundef [[META4]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !nofpclass [[META16:![0-9]+]], !mem.cache_hint [[META13]]
; CHECK-NEXT: ret float [[R]]
;
@@ -92,7 +92,7 @@ define void @store_zero(ptr %p, i8 %x, i32 %index) {
define void @store_nonzero(ptr %p, i8 %x, i32 %index) {
; CHECK-LABEL: define void @store_nonzero(
; CHECK-SAME: ptr [[P:%.*]], i8 [[X:%.*]], i32 [[INDEX:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 3
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 3
; CHECK-NEXT: store i8 [[X]], ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.access.group [[ACC_GRP9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META17]]
; CHECK-NEXT: ret void
;
@@ -108,7 +108,7 @@ define void @store_dynamic(ptr %p, i8 %x, i32 %index) {
; CHECK-NEXT: [[INDEX_FROZEN:%.*]] = freeze i32 [[INDEX]]
; CHECK-NEXT: [[IDX:%.*]] = and i32 [[INDEX_FROZEN]], 15
; CHECK-NEXT: [[IDX_GEPIDX:%.*]] = zext i32 [[IDX]] to i64
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 [[IDX_GEPIDX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[IDX_GEPIDX]]
; CHECK-NEXT: store i8 [[X]], ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.access.group [[ACC_GRP9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META17]]
; CHECK-NEXT: ret void
;
@@ -124,7 +124,7 @@ define void @store_dynamic(ptr %p, i8 %x, i32 %index) {
define i32 @extract_struct_field(ptr %p) {
; CHECK-LABEL: define i32 @extract_struct_field(
; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 1
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 4
; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[INT_TBAA18:![0-9]+]]
; CHECK-NEXT: ret i32 [[R]]
;
@@ -136,7 +136,7 @@ define i32 @extract_struct_field(ptr %p) {
define void @store_struct_field(ptr %p, i32 %x) {
; CHECK-LABEL: define void @store_struct_field(
; CHECK-SAME: ptr [[P:%.*]], i32 [[X:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 1
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 4
; CHECK-NEXT: store i32 [[X]], ptr [[TMP1]], align 4, !tbaa [[INT_TBAA18]]
; CHECK-NEXT: ret void
;
@@ -149,7 +149,7 @@ define void @store_struct_field(ptr %p, i32 %x) {
define void @scalar_store(ptr %p, i8 %x) {
; CHECK-LABEL: define void @scalar_store(
; CHECK-SAME: ptr [[P:%.*]], i8 [[X:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 3
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 3
; CHECK-NEXT: store i8 [[X]], ptr [[TMP1]], align 1, !pcsections [[META20:![0-9]+]]
; CHECK-NEXT: ret void
;
>From 07a8fd01bed401d163627039962b4bae069926a2 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 18 Nov 2026 00:27:10 +0000
Subject: [PATCH 11/14] [MetadataUtils] Use nondeprecated IRBuilder constructor
in tests
---
llvm/unittests/Analysis/MetadataUtilsTest.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/unittests/Analysis/MetadataUtilsTest.cpp b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
index b55922ebd0893c..abe80a2ad9920b 100644
--- a/llvm/unittests/Analysis/MetadataUtilsTest.cpp
+++ b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
@@ -22,7 +22,7 @@ class MetadataUtilsTest : public testing::Test {
protected:
LLVMContext C;
Module M{"test", C};
- IRBuilder<> B{C};
+ IRBuilder<> B{M};
Value *P;
Value *Q;
>From 02a014803a821aef8daccdd7f6d805ecc37f95ca Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 18 Nov 2026 01:01:02 +0000
Subject: [PATCH 12/14] [Analysis] Add metadata propagation for replacement
memory accesses
---
llvm/include/llvm/Analysis/MetadataUtils.h | 44 +-
llvm/lib/Analysis/MetadataUtils.cpp | 259 +++++++++
llvm/unittests/Analysis/CMakeLists.txt | 1 +
llvm/unittests/Analysis/MetadataUtilsTest.cpp | 504 ++++++++++++++++++
4 files changed, 792 insertions(+), 16 deletions(-)
create mode 100644 llvm/unittests/Analysis/MetadataUtilsTest.cpp
diff --git a/llvm/include/llvm/Analysis/MetadataUtils.h b/llvm/include/llvm/Analysis/MetadataUtils.h
index a7afe9514e0a8e..94a0816bbc209c 100644
--- a/llvm/include/llvm/Analysis/MetadataUtils.h
+++ b/llvm/include/llvm/Analysis/MetadataUtils.h
@@ -23,19 +23,20 @@ class Instruction;
class MDNode;
class Value;
-/// Combine the metadata of two instructions so that K can replace J. This
-/// specifically handles the case of CSE-like transformations. Some
-/// metadata can only be kept if K dominates J. For this to be correct,
-/// K cannot be hoisted.
+/// Copy metadata valid for a replacement memory access.
///
-/// Unknown metadata is removed.
-LLVM_ABI void combineMetadataForCSE(Instruction *K, const Instruction *J,
- bool DoesKMove);
-
-/// Combine metadata of two instructions, where instruction J is a memory
-/// access that has been merged into K. This will intersect alias-analysis
-/// metadata, while preserving other known metadata.
-LLVM_ABI void combineAAMetadata(Instruction *K, const Instruction *J);
+/// The destination must access the same bytes, a subset, or a wider byte range
+/// in the same object as the source. The destination must be a typed memory
+/// access. Memory-transfer and memory-set intrinsics are supported only as
+/// sources and must have a constant length.
+///
+/// DestByteOffset is the destination's offset within the source access. If the
+/// destination is no larger than the source, a missing offset means the offset
+/// is dynamic, but the caller guarentees it accesses a subset of the source
+/// access.
+LLVM_ABI void
+copyMetadataForMemoryAccess(Instruction &Dest, const Instruction &Source,
+ std::optional<int64_t> DestByteOffset = 0);
/// Copy the metadata from the source instruction to the destination (the
/// replacement for the source instruction).
@@ -55,6 +56,20 @@ LLVM_ABI void copyNonnullMetadata(const LoadInst &OldLI, MDNode *N,
LLVM_ABI void copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI,
MDNode *N, LoadInst &NewLI);
+/// Combine the metadata of two instructions so that K can replace J. This
+/// specifically handles the case of CSE-like transformations. Some
+/// metadata can only be kept if K dominates J. For this to be correct,
+/// K cannot be hoisted.
+///
+/// Unknown metadata is removed.
+LLVM_ABI void combineMetadataForCSE(Instruction *K, const Instruction *J,
+ bool DoesKMove);
+
+/// Combine metadata of two instructions, where instruction J is a memory
+/// access that has been merged into K. This will intersect alias-analysis
+/// metadata, while preserving other known metadata.
+LLVM_ABI void combineAAMetadata(Instruction *K, const Instruction *J);
+
/// Compute the union of two access-group lists.
///
/// If the list contains just one access group, it is returned directly. If the
@@ -70,10 +85,7 @@ LLVM_ABI MDNode *uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2);
LLVM_ABI MDNode *intersectAccessGroups(const Instruction *Inst1,
const Instruction *Inst2);
-/// Add metadata from \p Inst to \p Metadata, if it can be preserved after
-/// vectorization. It can be preserved after vectorization if the kind is one of
-/// [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath, MD_nontemporal,
-/// MD_access_group, MD_mmra].
+/// Collect metadata that can be preserved when vectorizing an instruction.
LLVM_ABI void getMetadataToPropagate(
Instruction *Inst,
SmallVectorImpl<std::pair<unsigned, MDNode *>> &Metadata);
diff --git a/llvm/lib/Analysis/MetadataUtils.cpp b/llvm/lib/Analysis/MetadataUtils.cpp
index a4d63aa51b8120..e748344330c134 100644
--- a/llvm/lib/Analysis/MetadataUtils.cpp
+++ b/llvm/lib/Analysis/MetadataUtils.cpp
@@ -26,6 +26,265 @@
using namespace llvm;
+namespace {
+struct MemoryAccessInfo {
+ Type *Ty;
+ unsigned PointerOperand;
+ bool IsRead;
+};
+} // namespace
+
+/// Get information about a memory instruction required to propagate metadata.
+/// Returns std::nullopt if the instruction is not supported and we will not
+/// propagate any metadata.
+///
+/// If IsRead is set and this is a instruction that reads and writes, mark it
+/// as reading or writing. This is used to make the source instruction in
+/// metadata propagation correspond with the dest.
+static std::optional<MemoryAccessInfo>
+getMemoryAccessInfo(const Instruction &I,
+ std::optional<bool> IsRead = std::nullopt) {
+ if (const auto *LI = dyn_cast<LoadInst>(&I))
+ return MemoryAccessInfo{LI->getType(), 0, true};
+ if (const auto *SI = dyn_cast<StoreInst>(&I))
+ return MemoryAccessInfo{SI->getValueOperand()->getType(), 1, false};
+ if (const auto *MI = dyn_cast<AnyMemIntrinsic>(&I)) {
+ if (!IsRead || (*IsRead && !isa<AnyMemTransferInst>(MI)))
+ return std::nullopt;
+ return MemoryAccessInfo{nullptr, *IsRead ? 1u : 0u, *IsRead};
+ }
+ const auto *II = dyn_cast<IntrinsicInst>(&I);
+ if (!II)
+ return std::nullopt;
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::experimental_vector_histogram_add:
+ case Intrinsic::experimental_vector_histogram_uadd_sat:
+ case Intrinsic::experimental_vector_histogram_umin:
+ case Intrinsic::experimental_vector_histogram_umax:
+ if (IsRead)
+ return MemoryAccessInfo{II->getArgOperand(1)->getType(), 0, *IsRead};
+ return std::nullopt;
+ case Intrinsic::masked_load:
+ case Intrinsic::masked_gather:
+ case Intrinsic::masked_expandload:
+ case Intrinsic::vp_load:
+ return MemoryAccessInfo{II->getType(), 0, true};
+ case Intrinsic::masked_store:
+ case Intrinsic::masked_scatter:
+ case Intrinsic::masked_compressstore:
+ return MemoryAccessInfo{II->getArgOperand(0)->getType(), 1, false};
+ default:
+ return std::nullopt;
+ }
+}
+
+/// Get the pointer operand of `I` at operand index `Index`.
+static const Value *getMemoryPointer(const Instruction &I, unsigned Index) {
+ if (const auto *CB = dyn_cast<CallBase>(&I))
+ return CB->getArgOperand(Index);
+ return I.getOperand(Index);
+}
+
+// invariant.group ignores bitcasts and zero-offset GEPs, not address-space
+// casts.
+static const Value *getInvariantGroupPointer(const Value *V) {
+ while (true) {
+ if (const auto *Cast = dyn_cast<BitCastOperator>(V))
+ V = Cast->getOperand(0);
+ else if (const auto *GEP = dyn_cast<GEPOperator>(V);
+ GEP && GEP->hasAllZeroIndices())
+ V = GEP->getPointerOperand();
+ else
+ return V;
+ }
+}
+
+static void copyNonnullMetadataForAccess(const LoadInst &OldLI, MDNode *N,
+ LoadInst &NewLI) {
+ auto *OldTy = cast<PointerType>(OldLI.getType());
+ auto *NewTy = NewLI.getType();
+
+ // Preserve the pointer fact when the pointer type is unchanged.
+ if (NewTy == OldTy) {
+ NewLI.setMetadata(LLVMContext::MD_nonnull, N);
+ return;
+ }
+
+ // An integer range must describe the complete pointer representation.
+ if (!NewTy->isIntegerTy() ||
+ NewTy->getIntegerBitWidth() !=
+ OldLI.getDataLayout().getPointerTypeSizeInBits(OldTy))
+ return;
+
+ MDBuilder MDB(NewLI.getContext());
+ auto *ITy = cast<IntegerType>(NewTy);
+ auto *NullInt =
+ ConstantExpr::getPtrToInt(ConstantPointerNull::get(OldTy), ITy);
+ auto *NonNullInt = ConstantExpr::getAdd(NullInt, ConstantInt::get(ITy, 1));
+ NewLI.setMetadata(LLVMContext::MD_range,
+ MDB.createRange(NonNullInt, NullInt));
+}
+
+void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
+ const Instruction &Source,
+ std::optional<int64_t> DestByteOffset) {
+ auto DestInfo = getMemoryAccessInfo(Dest);
+ // Untyped memory intrinsics are supported only as sources.
+ assert(DestInfo && DestInfo->Ty &&
+ "expected a typed destination memory access");
+ auto SourceInfo = getMemoryAccessInfo(Source, DestInfo->IsRead);
+ assert(SourceInfo && SourceInfo->IsRead == DestInfo->IsRead &&
+ "expected corresponding memory reads or writes");
+ const DataLayout &DL = Source.getDataLayout();
+ TypeSize OldSize =
+ SourceInfo->Ty
+ ? DL.getTypeStoreSize(SourceInfo->Ty)
+ : TypeSize::getFixed(
+ cast<ConstantInt>(cast<AnyMemIntrinsic>(Source).getLength())
+ ->getZExtValue());
+ TypeSize NewSize = DL.getTypeStoreSize(DestInfo->Ty);
+ // Is the destination access contained in the source access? If the size
+ // check succeeds but the offset is unknown, containment is a caller
+ // guarantee.
+ bool IsSubset = TypeSize::isKnownLE(NewSize, OldSize);
+ if (DestByteOffset)
+ IsSubset = IsSubset && *DestByteOffset >= 0 &&
+ uint64_t(*DestByteOffset) <=
+ OldSize.getKnownMinValue() - NewSize.getKnownMinValue();
+ bool IsWholeAccess = IsSubset && DestByteOffset == 0 && OldSize == NewSize;
+ Type *SourceValueTy = SourceInfo->Ty;
+ // Allow unpacking a single-element aggregates.
+ if (IsWholeAccess) {
+ if (auto *ST = dyn_cast_or_null<StructType>(SourceValueTy);
+ ST && ST->getNumElements() == 1 &&
+ ST->getElementType(0) == DestInfo->Ty)
+ SourceValueTy = DestInfo->Ty;
+ else if (auto *AT = dyn_cast_or_null<ArrayType>(SourceValueTy);
+ AT && AT->getNumElements() == 1 &&
+ AT->getElementType() == DestInfo->Ty)
+ SourceValueTy = DestInfo->Ty;
+ }
+ bool SameElementType = SourceValueTy && SourceValueTy->getScalarType() ==
+ DestInfo->Ty->getScalarType();
+ bool SamePointer = getInvariantGroupPointer(getMemoryPointer(
+ Source, SourceInfo->PointerOperand)) ==
+ getInvariantGroupPointer(
+ getMemoryPointer(Dest, DestInfo->PointerOperand));
+
+ // Now loop over all `Source` metadata, propagating what we can.
+ SmallVector<std::pair<unsigned, MDNode *>, 8> MDs;
+ Source.getAllMetadata(MDs);
+ for (auto [Kind, MD] : MDs) {
+ bool Keep = false;
+ switch (Kind) {
+ case LLVMContext::MD_dbg:
+ case LLVMContext::MD_nontemporal:
+ case LLVMContext::MD_annotation:
+ case LLVMContext::MD_nosanitize:
+ case LLVMContext::MD_pcsections:
+ Keep = true;
+ break;
+ case LLVMContext::MD_mmra:
+ Keep = canInstructionHaveMMRAs(Dest);
+ break;
+ case LLVMContext::MD_access_group:
+ case LLVMContext::MD_mem_parallel_loop_access:
+ Keep = IsSubset;
+ break;
+ case LLVMContext::MD_invariant_load:
+ Keep = IsSubset && DestInfo->IsRead;
+ break;
+ case LLVMContext::MD_invariant_group:
+ // TODO: Requiring `SamePointer` is too conservative.
+ Keep = IsSubset && SamePointer && !isa<CallBase>(Dest);
+ break;
+ case LLVMContext::MD_noundef:
+ // The value loaded remains defined if its element type is unchanged,
+ // or every source byte consists of defined value bits (i.e. the source
+ // doesn't have padding).
+ Keep = IsSubset && isa<LoadInst>(Dest) &&
+ (SameElementType ||
+ (SourceValueTy && !SourceValueTy->isAggregateType() &&
+ DL.typeSizeEqualsStoreSize(SourceValueTy)));
+ break;
+ case LLVMContext::MD_fpmath:
+ case LLVMContext::MD_nofpclass:
+ Keep = IsSubset && SameElementType &&
+ (Kind == LLVMContext::MD_fpmath
+ ? FPMathOperator::isSupportedFloatingPointType(Dest.getType())
+ : isa<LoadInst>(Dest));
+ break;
+ case LLVMContext::MD_nonnull:
+ if (IsWholeAccess && isa<LoadInst>(Source) && isa<LoadInst>(Dest))
+ copyNonnullMetadataForAccess(cast<LoadInst>(Source), MD,
+ cast<LoadInst>(Dest));
+ break;
+ case LLVMContext::MD_range:
+ // Ranges apply per integer element. Other type changes require the
+ // helper's explicit conversion from an integer range to nonnull.
+ if (IsSubset && SameElementType && DestInfo->Ty->isIntOrIntVectorTy())
+ Keep = true;
+ else if (IsWholeAccess && isa<LoadInst>(Source) && isa<LoadInst>(Dest))
+ copyRangeMetadata(DL, cast<LoadInst>(Source), MD, cast<LoadInst>(Dest));
+ break;
+ case LLVMContext::MD_align:
+ case LLVMContext::MD_dereferenceable:
+ case LLVMContext::MD_dereferenceable_or_null:
+ // These describe the loaded pointer value.
+ Keep = IsWholeAccess && SourceInfo->Ty == DestInfo->Ty &&
+ DestInfo->Ty->isPointerTy() && isa<LoadInst>(Dest);
+ break;
+ case LLVMContext::MD_DIAssignID: {
+ const auto *OldSI = dyn_cast<StoreInst>(&Source);
+ const auto *NewSI = dyn_cast<StoreInst>(&Dest);
+ // A new address requires updating the linked dbg.assign records.
+ Keep = IsWholeAccess && OldSI && NewSI &&
+ OldSI->getPointerOperand() == NewSI->getPointerOperand();
+ break;
+ }
+ case LLVMContext::MD_prof:
+ Keep =
+ IsWholeAccess && ((isa<LoadInst>(Source) && isa<LoadInst>(Dest)) ||
+ (isa<StoreInst>(Source) && isa<StoreInst>(Dest)));
+ break;
+ case LLVMContext::MD_captures:
+ Keep =
+ IsWholeAccess && isa<StoreInst>(Dest) && DestInfo->Ty->isPointerTy();
+ break;
+ default:
+ break;
+ }
+ if (Keep)
+ Dest.setMetadata(Kind, MD);
+ }
+
+ // Adjust aliasing metadata.
+ AAMDNodes AA = Source.getAAMetadata();
+ if (!IsSubset) {
+ AA.TBAA = nullptr;
+ AA.TBAAStruct = nullptr;
+ AA.Scope = nullptr;
+ AA.NoAlias = nullptr;
+ } else if (!IsWholeAccess) {
+ if (DestByteOffset)
+ AA = AA.adjustForAccess(*DestByteOffset, DestInfo->Ty, DL);
+ else {
+ // A common tag for the whole source also describes any selected part.
+ if (!AA.TBAA && SourceInfo->Ty)
+ AA.TBAA = AA.adjustForAccess(0, SourceInfo->Ty, DL).TBAA;
+ AA.TBAAStruct = nullptr;
+ }
+ } else {
+ // Prefer !tbaa over !tbaa.struct.
+ AAMDNodes Adjusted = AA.adjustForAccess(0, DestInfo->Ty, DL);
+ if (!AA.TBAA && Adjusted.TBAA)
+ AA = Adjusted;
+ }
+ Dest.setAAMetadata(AA);
+ setMemCacheHint(Dest, DestInfo->PointerOperand,
+ getMemCacheHint(Source, SourceInfo->PointerOperand));
+}
+
/// Add all access groups in @p AccGroups to @p List.
template <typename ListT>
static void addToAccessGroupList(ListT &List, MDNode *AccGroups) {
diff --git a/llvm/unittests/Analysis/CMakeLists.txt b/llvm/unittests/Analysis/CMakeLists.txt
index 43ad72b7bfa6dd..0732edeff7c88e 100644
--- a/llvm/unittests/Analysis/CMakeLists.txt
+++ b/llvm/unittests/Analysis/CMakeLists.txt
@@ -44,6 +44,7 @@ set(ANALYSIS_TEST_SOURCES
MemoryBuiltinsTest.cpp
MemoryProfileInfoTest.cpp
MemorySSATest.cpp
+ MetadataUtilsTest.cpp
MLGOUtilsTest.cpp
MLModelRunnerTest.cpp
PhiValuesTest.cpp
diff --git a/llvm/unittests/Analysis/MetadataUtilsTest.cpp b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
new file mode 100644
index 00000000000000..e8ab8d7a9c7145
--- /dev/null
+++ b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
@@ -0,0 +1,504 @@
+//===- MetadataUtilsTest.cpp - Metadata tests --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/MetadataUtils.h"
+#include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/MDBuilder.h"
+#include "llvm/IR/MemCacheHint.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Verifier.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+namespace {
+class MetadataUtilsTest : public testing::Test {
+protected:
+ LLVMContext C;
+ Module M{"test", C};
+ IRBuilder<> B{M};
+ Value *P;
+ Value *Q;
+
+ MetadataUtilsTest() {
+ M.setDataLayout("e-p:64:64");
+ auto *F = Function::Create(
+ FunctionType::get(B.getVoidTy(), {B.getPtrTy(), B.getPtrTy()}, false),
+ Function::ExternalLinkage, "test", M);
+ P = F->getArg(0);
+ Q = F->getArg(1);
+ B.SetInsertPoint(BasicBlock::Create(C, "entry", F));
+ B.SetInsertPoint(B.CreateRetVoid());
+ }
+
+ void TearDown() override { EXPECT_FALSE(verifyModule(M, &errs())); }
+
+ MDNode *empty() { return MDNode::get(C, {}); }
+ Metadata *number(uint64_t N) {
+ return ConstantAsMetadata::get(B.getInt64(N));
+ }
+ MDNode *hint() {
+ return MDNode::get(
+ C, {MDString::get(C, "nvvm.l1_eviction"), MDString::get(C, "last")});
+ }
+ MDNode *range() {
+ return MDNode::get(C, {ConstantAsMetadata::get(B.getInt32(0)),
+ ConstantAsMetadata::get(B.getInt32(100))});
+ }
+ MDNode *scope() {
+ MDBuilder MDB(C);
+ return MDNode::get(C, {MDB.createAnonymousAliasScope(
+ MDB.createAnonymousAliasScopeDomain())});
+ }
+};
+
+TEST_F(MetadataUtilsTest, NarrowedLoadPreservesElementFacts) {
+ auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), P);
+ auto *Ptr = B.CreateInBoundsGEP(B.getInt32Ty(), P, B.getInt64(2));
+ auto *New = B.CreateLoad(B.getInt32Ty(), Ptr);
+ auto *Range = range();
+ auto *Hint = hint();
+ auto *Scope = scope();
+ Old->setMetadata(LLVMContext::MD_range, Range);
+ Old->setMetadata(LLVMContext::MD_noundef, empty());
+ Old->setMetadata(LLVMContext::MD_invariant_load, empty());
+ Old->setMetadata(LLVMContext::MD_invariant_group, empty());
+ Old->setMetadata(LLVMContext::MD_alias_scope, Scope);
+ setMemCacheHint(*Old, 0, Hint);
+ copyMetadataForMemoryAccess(*New, *Old, 8);
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_range), Range);
+ EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_noundef));
+ EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_invariant_load));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_invariant_group));
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_alias_scope), Scope);
+ EXPECT_EQ(getMemCacheHint(*New), Hint);
+}
+
+TEST_F(MetadataUtilsTest, InvariantGroupRequiresPointerIdentity) {
+ auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), P);
+ auto *Same = B.CreateLoad(B.getInt32Ty(), P);
+ auto *Different = B.CreateLoad(B.getInt32Ty(), Q);
+ Old->setMetadata(LLVMContext::MD_invariant_group, empty());
+ copyMetadataForMemoryAccess(*Same, *Old);
+ copyMetadataForMemoryAccess(*Different, *Old);
+ EXPECT_TRUE(Same->hasMetadata(LLVMContext::MD_invariant_group));
+ EXPECT_FALSE(Different->hasMetadata(LLVMContext::MD_invariant_group));
+}
+
+TEST_F(MetadataUtilsTest, IntegerFragmentPreservesDefinednessButNotRange) {
+ auto *Old = B.CreateLoad(B.getInt64Ty(), P);
+ MDBuilder MDB(C);
+ Old->setMetadata(LLVMContext::MD_noundef, empty());
+ Old->setMetadata(LLVMContext::MD_range,
+ MDB.createRange(B.getInt64(0), B.getInt64(100)));
+ for (int64_t Offset : {0, 4}) {
+ auto *Ptr = B.CreateInBoundsGEP(B.getInt8Ty(), P, B.getInt64(Offset));
+ auto *New = B.CreateLoad(B.getInt32Ty(), Ptr);
+ copyMetadataForMemoryAccess(*New, *Old, Offset);
+ EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_noundef));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+ }
+}
+
+TEST_F(MetadataUtilsTest, WideningDropsFactsAboutOriginalBytes) {
+ auto *Old = B.CreateLoad(B.getInt32Ty(), P);
+ auto *New = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), P);
+ auto *Hint = hint();
+ auto *Annotation = MDNode::get(C, {MDString::get(C, "original")});
+ Old->setMetadata(LLVMContext::MD_range, range());
+ Old->setMetadata(LLVMContext::MD_noundef, empty());
+ Old->setMetadata(LLVMContext::MD_invariant_load, empty());
+ Old->setMetadata(LLVMContext::MD_invariant_group, empty());
+ Old->setMetadata(LLVMContext::MD_alias_scope, scope());
+ Old->setMetadata(LLVMContext::MD_annotation, Annotation);
+ Old->setMetadata("test.unknown", empty());
+ setMemCacheHint(*Old, 0, Hint);
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_noundef));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_invariant_load));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_invariant_group));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_alias_scope));
+ EXPECT_FALSE(New->getMetadata("test.unknown"));
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_annotation), Annotation);
+ EXPECT_EQ(getMemCacheHint(*New), Hint);
+}
+
+TEST_F(MetadataUtilsTest, EarlierAddressIsNotASubset) {
+ auto *Ptr = B.CreateInBoundsGEP(B.getInt32Ty(), P, B.getInt64(2));
+ auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), Ptr);
+ auto *New = B.CreateLoad(B.getInt32Ty(), P);
+ Old->setMetadata(LLVMContext::MD_range, range());
+ copyMetadataForMemoryAccess(*New, *Old, -8);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+}
+
+TEST_F(MetadataUtilsTest, BitcastDropsDifferentIntegerElementRange) {
+ auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 2), P);
+ auto *New = B.CreateLoad(B.getInt64Ty(), P);
+ Old->setMetadata(LLVMContext::MD_range, range());
+ Old->setMetadata(LLVMContext::MD_noundef, empty());
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+ EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_noundef));
+}
+
+TEST_F(MetadataUtilsTest, BitcastDropsFloatingPointValueFacts) {
+ auto *Old = B.CreateLoad(FixedVectorType::get(B.getFloatTy(), 2), P);
+ auto *New = B.CreateLoad(B.getInt64Ty(), P);
+ Old->setMetadata(LLVMContext::MD_nofpclass,
+ MDNode::get(C, {ConstantAsMetadata::get(B.getInt32(3))}));
+ Old->setMetadata(LLVMContext::MD_fpmath,
+ MDNode::get(C, {ConstantAsMetadata::get(
+ ConstantFP::get(B.getFloatTy(), 2.5))}));
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_nofpclass));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_fpmath));
+}
+
+TEST_F(MetadataUtilsTest, PointerNonnullBecomesIntegerRange) {
+ auto *Old = B.CreateLoad(B.getPtrTy(), P);
+ auto *New = B.CreateLoad(B.getInt64Ty(), P);
+ Old->setMetadata(LLVMContext::MD_nonnull, empty());
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_nonnull));
+ ASSERT_TRUE(New->hasMetadata(LLVMContext::MD_range));
+ auto *Range = New->getMetadata(LLVMContext::MD_range);
+ EXPECT_EQ(mdconst::extract<ConstantInt>(Range->getOperand(0))->getZExtValue(),
+ 1u);
+ EXPECT_EQ(mdconst::extract<ConstantInt>(Range->getOperand(1))->getZExtValue(),
+ 0u);
+}
+
+TEST_F(MetadataUtilsTest, UnknownOffsetKeepsAccessTagsButDropsFieldLayout) {
+ auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), P);
+ auto *Index = B.CreateAnd(B.CreateLoad(B.getInt64Ty(), Q), B.getInt64(3));
+ auto *Ptr = B.CreateInBoundsGEP(B.getInt32Ty(), P, Index);
+ auto *New = B.CreateLoad(B.getInt32Ty(), Ptr);
+ MDBuilder MDB(C);
+ auto *Char = MDB.createTBAAScalarTypeNode("char", MDB.createTBAARoot("root"));
+ auto *Tag = MDB.createTBAAStructTagNode(Char, Char, 0);
+ auto *Scope = scope();
+ Old->setMetadata(LLVMContext::MD_tbaa, Tag);
+ Old->setMetadata(LLVMContext::MD_tbaa_struct,
+ MDNode::get(C, {number(0), number(4), Tag}));
+ Old->setMetadata(LLVMContext::MD_alias_scope, Scope);
+ copyMetadataForMemoryAccess(*New, *Old, std::nullopt);
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_tbaa), Tag);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_tbaa_struct));
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_alias_scope), Scope);
+}
+
+TEST_F(MetadataUtilsTest, ConstantOffsetSelectsFieldTag) {
+ auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 2), P);
+ auto *New = B.CreateLoad(B.getInt32Ty(), Q);
+ MDBuilder MDB(C);
+ auto *Root = MDB.createTBAARoot("root");
+ auto *Char = MDB.createTBAAScalarTypeNode("char", Root);
+ auto *Int = MDB.createTBAAScalarTypeNode("int", Char);
+ auto *CharTag = MDB.createTBAAStructTagNode(Char, Char, 0);
+ auto *IntTag = MDB.createTBAAStructTagNode(Int, Int, 0);
+ Old->setMetadata(LLVMContext::MD_tbaa_struct,
+ MDNode::get(C, {number(0), number(4), CharTag, number(4),
+ number(4), IntTag}));
+ copyMetadataForMemoryAccess(*New, *Old, 4);
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_tbaa), IntTag);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_tbaa_struct));
+}
+
+TEST_F(MetadataUtilsTest, WholeAccessSelectsSingleFieldTag) {
+ auto *Old = B.CreateLoad(B.getInt64Ty(), P);
+ auto *New = B.CreateLoad(B.getDoubleTy(), P);
+ MDBuilder MDB(C);
+ auto *Char = MDB.createTBAAScalarTypeNode("char", MDB.createTBAARoot("root"));
+ auto *Tag = MDB.createTBAAStructTagNode(Char, Char, 0);
+ Old->setMetadata(LLVMContext::MD_tbaa_struct,
+ MDNode::get(C, {number(0), number(8), Tag}));
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_tbaa), Tag);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_tbaa_struct));
+}
+
+TEST_F(MetadataUtilsTest, WholeAccessRetainsMultipleFieldTags) {
+ auto *Ty = FixedVectorType::get(B.getInt32Ty(), 2);
+ auto *Old = B.CreateLoad(Ty, P);
+ auto *New = B.CreateLoad(Ty, P);
+ MDBuilder MDB(C);
+ auto *Char = MDB.createTBAAScalarTypeNode("char", MDB.createTBAARoot("root"));
+ auto *Int = MDB.createTBAAScalarTypeNode("int", Char);
+ auto *CharTag = MDB.createTBAAStructTagNode(Char, Char, 0);
+ auto *IntTag = MDB.createTBAAStructTagNode(Int, Int, 0);
+ auto *Fields = MDNode::get(
+ C, {number(0), number(4), CharTag, number(4), number(4), IntTag});
+ Old->setMetadata(LLVMContext::MD_tbaa_struct, Fields);
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_tbaa));
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_tbaa_struct), Fields);
+}
+
+TEST_F(MetadataUtilsTest, MemoryTransferRemapsBothOperands) {
+ auto *Copy = B.CreateMemCpy(P, Align(4), Q, Align(4), B.getInt64(4));
+ auto *Read = B.CreateLoad(B.getInt32Ty(), Q);
+ auto *Write = B.CreateStore(B.getInt32(1), P);
+ auto *ReadHint = hint();
+ auto *WriteHint = MDNode::get(
+ C, {MDString::get(C, "nvvm.l1_eviction"), MDString::get(C, "first")});
+ setMemCacheHint(*Copy, 0, WriteHint);
+ setMemCacheHint(*Copy, 1, ReadHint);
+ copyMetadataForMemoryAccess(*Read, *Copy);
+ copyMetadataForMemoryAccess(*Write, *Copy);
+ EXPECT_EQ(getMemCacheHint(*Read, 0), ReadHint);
+ EXPECT_EQ(getMemCacheHint(*Write, 1), WriteHint);
+ EXPECT_FALSE(getMemCacheHint(*Read, 1));
+ EXPECT_FALSE(getMemCacheHint(*Write, 0));
+}
+
+TEST_F(MetadataUtilsTest, StoreTypeChangePreservesHints) {
+ auto *Old = B.CreateStore(B.getInt32(0), P);
+ auto *New = B.CreateStore(ConstantFP::get(B.getFloatTy(), 0), P);
+ auto *Hint = hint();
+ setMemCacheHint(*Old, 1, Hint);
+ Old->setMetadata(LLVMContext::MD_invariant_group, empty());
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_EQ(getMemCacheHint(*New, 1), Hint);
+ EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_invariant_group));
+}
+
+
+TEST_F(MetadataUtilsTest, CSEIntersectsCacheHints) {
+ auto *A = B.CreateLoad(B.getInt32Ty(), P);
+ auto *BLoad = B.CreateLoad(B.getInt32Ty(), P);
+ auto *Hint = hint();
+ setMemCacheHint(*A, 0, Hint);
+ setMemCacheHint(*BLoad, 0, Hint);
+ combineMetadataForCSE(A, BLoad, false);
+ EXPECT_EQ(getMemCacheHint(*A), Hint);
+ BLoad->setMetadata(LLVMContext::MD_mem_cache_hint, nullptr);
+ combineMetadataForCSE(A, BLoad, false);
+ EXPECT_FALSE(getMemCacheHint(*A));
+}
+
+TEST_F(MetadataUtilsTest, AddressSpaceCastChangesInvariantGroupIdentity) {
+ auto *Old = B.CreateLoad(B.getInt32Ty(), P);
+ auto *NewPtr = B.CreateAddrSpaceCast(P, B.getPtrTy(1));
+ auto *New = B.CreateLoad(B.getInt32Ty(), NewPtr);
+ Old->setMetadata(LLVMContext::MD_invariant_group, empty());
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_invariant_group));
+}
+
+TEST_F(MetadataUtilsTest, SingleElementAggregatePreservesDefinedness) {
+ Type *Types[] = {StructType::get(C, {B.getInt32Ty()}, /*isPacked=*/false),
+ ArrayType::get(B.getInt32Ty(), 1)};
+ for (Type *Ty : Types) {
+ auto *Old = B.CreateLoad(Ty, P);
+ auto *New = B.CreateLoad(B.getInt32Ty(), P);
+ Old->setMetadata(LLVMContext::MD_noundef, empty());
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_noundef));
+ }
+}
+
+TEST_F(MetadataUtilsTest, SingleElementAggregatePreservesFloatingPointFacts) {
+ auto *FPMath = MDNode::get(
+ C, {ConstantAsMetadata::get(ConstantFP::get(B.getFloatTy(), 2.5))});
+ auto *NoFPClass = MDNode::get(C, {ConstantAsMetadata::get(B.getInt32(3))});
+ Type *Types[] = {StructType::get(C, {B.getFloatTy()}, /*isPacked=*/false),
+ ArrayType::get(B.getFloatTy(), 1)};
+ for (Type *Ty : Types) {
+ auto *Old = B.CreateLoad(Ty, P);
+ auto *New = B.CreateLoad(B.getFloatTy(), P);
+ Old->setMetadata(LLVMContext::MD_noundef, empty());
+ Old->setMetadata(LLVMContext::MD_fpmath, FPMath);
+ Old->setMetadata(LLVMContext::MD_nofpclass, NoFPClass);
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_noundef));
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_fpmath), FPMath);
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_nofpclass), NoFPClass);
+ }
+}
+
+TEST_F(MetadataUtilsTest, ShortAccessSelectsCoveringFieldTag) {
+ auto *Old = B.CreateLoad(B.getInt64Ty(), P);
+ auto *New = B.CreateLoad(B.getInt32Ty(), P);
+ MDBuilder MDB(C);
+ auto *Int = MDB.createTBAAScalarTypeNode("int", MDB.createTBAARoot("root"));
+ auto *Tag = MDB.createTBAAStructTagNode(Int, Int, 0);
+ Old->setMetadata(LLVMContext::MD_tbaa_struct,
+ MDNode::get(C, {number(0), number(8), Tag}));
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_tbaa), Tag);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_tbaa_struct));
+}
+
+TEST_F(MetadataUtilsTest, UnknownOffsetSelectsCommonFieldTag) {
+ auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 2), P);
+ auto *Index = B.CreateAnd(B.CreateLoad(B.getInt64Ty(), Q), B.getInt64(1));
+ auto *Ptr = B.CreateInBoundsGEP(B.getInt32Ty(), P, Index);
+ auto *New = B.CreateLoad(B.getInt32Ty(), Ptr);
+ MDBuilder MDB(C);
+ auto *Int = MDB.createTBAAScalarTypeNode("int", MDB.createTBAARoot("root"));
+ auto *Tag = MDB.createTBAAStructTagNode(Int, Int, 0);
+ Old->setMetadata(
+ LLVMContext::MD_tbaa_struct,
+ MDNode::get(C, {number(0), number(4), Tag, number(4), number(4), Tag}));
+ copyMetadataForMemoryAccess(*New, *Old, std::nullopt);
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_tbaa), Tag);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_tbaa_struct));
+}
+
+TEST_F(MetadataUtilsTest, DefinedAggregateDoesNotDefinePadding) {
+ auto *Ty = StructType::get(C, {B.getInt8Ty(), B.getInt32Ty()});
+ ASSERT_EQ(M.getDataLayout().getTypeStoreSize(Ty).getFixedValue(), 8u);
+ auto *Old = B.CreateLoad(Ty, P);
+ auto *New = B.CreateLoad(B.getInt64Ty(), P);
+ Old->setMetadata(LLVMContext::MD_noundef, empty());
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_noundef));
+}
+
+TEST_F(MetadataUtilsTest, DefinedIntegerDoesNotDefineSourcePadding) {
+ auto *Old = B.CreateLoad(B.getInt1Ty(), P);
+ auto *New = B.CreateLoad(B.getInt8Ty(), P);
+ Old->setMetadata(LLVMContext::MD_noundef, empty());
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_noundef));
+}
+
+TEST_F(MetadataUtilsTest, DestinationPaddingDoesNotInvalidateDefinedness) {
+ auto *Old = B.CreateLoad(B.getInt8Ty(), P);
+ auto *New = B.CreateLoad(B.getInt1Ty(), P);
+ Old->setMetadata(LLVMContext::MD_noundef, empty());
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_noundef));
+}
+
+TEST_F(MetadataUtilsTest, NonnullDoesNotConstrainTruncatedPointerBits) {
+ auto *Old = B.CreateLoad(B.getPtrTy(), P);
+ auto *New = B.CreateLoad(B.getIntNTy(63), P);
+ Old->setMetadata(LLVMContext::MD_nonnull, empty());
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+}
+
+TEST_F(MetadataUtilsTest, IntegerRangeBecomesPointerNonnull) {
+ auto *Old = B.CreateLoad(B.getInt64Ty(), P);
+ auto *New = B.CreateLoad(B.getPtrTy(), P);
+ MDBuilder MDB(C);
+ Old->setMetadata(LLVMContext::MD_range,
+ MDB.createRange(B.getInt64(1), B.getInt64(100)));
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_nonnull));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+}
+
+TEST_F(MetadataUtilsTest, PointerFactsRequireSamePointerType) {
+ auto *Old = B.CreateLoad(B.getPtrTy(), P);
+ auto *New = B.CreateLoad(B.getPtrTy(1), P);
+ Old->setMetadata(LLVMContext::MD_nonnull, empty());
+ Old->setMetadata(LLVMContext::MD_align, MDNode::get(C, {number(16)}));
+ Old->setMetadata(LLVMContext::MD_dereferenceable,
+ MDNode::get(C, {number(32)}));
+ Old->setMetadata(LLVMContext::MD_dereferenceable_or_null,
+ MDNode::get(C, {number(32)}));
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_nonnull));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_align));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_dereferenceable));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_dereferenceable_or_null));
+}
+
+TEST_F(MetadataUtilsTest, PointerFactsRequirePointerResults) {
+ auto *Old = B.CreateLoad(B.getPtrTy(), P);
+ auto *Pointer = B.CreateLoad(B.getPtrTy(), P);
+ auto *Integer = B.CreateLoad(B.getInt64Ty(), P);
+ auto *Align = MDNode::get(C, {number(16)});
+ auto *Deref = MDNode::get(C, {number(32)});
+ Old->setMetadata(LLVMContext::MD_nonnull, empty());
+ Old->setMetadata(LLVMContext::MD_align, Align);
+ Old->setMetadata(LLVMContext::MD_dereferenceable, Deref);
+ Old->setMetadata(LLVMContext::MD_dereferenceable_or_null, Deref);
+ copyMetadataForMemoryAccess(*Pointer, *Old);
+ copyMetadataForMemoryAccess(*Integer, *Old);
+ EXPECT_TRUE(Pointer->hasMetadata(LLVMContext::MD_nonnull));
+ EXPECT_EQ(Pointer->getMetadata(LLVMContext::MD_align), Align);
+ EXPECT_EQ(Pointer->getMetadata(LLVMContext::MD_dereferenceable), Deref);
+ EXPECT_EQ(Pointer->getMetadata(LLVMContext::MD_dereferenceable_or_null),
+ Deref);
+ EXPECT_FALSE(Integer->hasMetadata(LLVMContext::MD_nonnull));
+ EXPECT_FALSE(Integer->hasMetadata(LLVMContext::MD_align));
+ EXPECT_FALSE(Integer->hasMetadata(LLVMContext::MD_dereferenceable));
+ EXPECT_FALSE(Integer->hasMetadata(LLVMContext::MD_dereferenceable_or_null));
+ EXPECT_TRUE(Integer->hasMetadata(LLVMContext::MD_range));
+}
+
+TEST_F(MetadataUtilsTest, StoreFactsRequireWholeAccessAndPointerValue) {
+ auto *Old = B.CreateStore(Q, P);
+ auto *Pointer = B.CreateStore(Q, P);
+ auto *Integer = B.CreateStore(B.CreatePtrToInt(Q, B.getInt64Ty()), P);
+ auto *Part = B.CreateStore(B.getInt32(0), P);
+ auto *AssignID = DIAssignID::getDistinct(C);
+ auto *Captures = MDNode::get(C, {MDString::get(C, "address")});
+ Old->setMetadata(LLVMContext::MD_DIAssignID, AssignID);
+ Old->setMetadata(LLVMContext::MD_captures, Captures);
+ copyMetadataForMemoryAccess(*Pointer, *Old);
+ copyMetadataForMemoryAccess(*Integer, *Old);
+ copyMetadataForMemoryAccess(*Part, *Old);
+ EXPECT_EQ(Pointer->getMetadata(LLVMContext::MD_DIAssignID), AssignID);
+ EXPECT_EQ(Pointer->getMetadata(LLVMContext::MD_captures), Captures);
+ EXPECT_EQ(Integer->getMetadata(LLVMContext::MD_DIAssignID), AssignID);
+ EXPECT_FALSE(Integer->hasMetadata(LLVMContext::MD_captures));
+ EXPECT_FALSE(Part->hasMetadata(LLVMContext::MD_DIAssignID));
+ EXPECT_FALSE(Part->hasMetadata(LLVMContext::MD_captures));
+}
+
+TEST_F(MetadataUtilsTest, AssignmentIDRequiresUnchangedStoreAddress) {
+ auto *Old = B.CreateStore(B.getInt32(1), P);
+ auto *Same = B.CreateStore(B.getInt32(1), P);
+ auto *Moved = B.CreateStore(B.getInt32(1), Q);
+ auto *AssignID = DIAssignID::getDistinct(C);
+ Old->setMetadata(LLVMContext::MD_DIAssignID, AssignID);
+ copyMetadataForMemoryAccess(*Same, *Old);
+ copyMetadataForMemoryAccess(*Moved, *Old);
+ EXPECT_EQ(Same->getMetadata(LLVMContext::MD_DIAssignID), AssignID);
+ EXPECT_FALSE(Moved->hasMetadata(LLVMContext::MD_DIAssignID));
+}
+
+TEST_F(MetadataUtilsTest, IntrinsicProfilesAndUnknownMetadataAreNotCopied) {
+ auto *Ty = FixedVectorType::get(B.getInt32Ty(), 2);
+ auto *Mask = B.CreateLoad(FixedVectorType::get(B.getInt1Ty(), 2), Q);
+ auto *Old = B.CreateMaskedLoad(Ty, P, Align(4), Mask);
+ auto *New = B.CreateLoad(Ty, P);
+ auto *Profile = MDNode::get(C, {MDString::get(C, "branch_weights"),
+ ConstantAsMetadata::get(B.getInt32(10))});
+ Old->setMetadata(LLVMContext::MD_prof, Profile);
+ Old->setMetadata("test.unknown", empty());
+ copyMetadataForMemoryAccess(*New, *Old);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_prof));
+ EXPECT_FALSE(New->getMetadata("test.unknown"));
+}
+
+TEST_F(MetadataUtilsTest, SmallerAccessOutsideSourceRangeIsNotASubset) {
+ auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), P);
+ auto *Ptr = B.CreateInBoundsGEP(B.getInt32Ty(), P, B.getInt64(4));
+ auto *New = B.CreateLoad(B.getInt32Ty(), Ptr);
+ Old->setMetadata(LLVMContext::MD_range, range());
+ Old->setMetadata(LLVMContext::MD_noundef, empty());
+ Old->setMetadata(LLVMContext::MD_invariant_load, empty());
+ Old->setMetadata(LLVMContext::MD_alias_scope, scope());
+ Old->setMetadata(LLVMContext::MD_noalias, scope());
+ copyMetadataForMemoryAccess(*New, *Old, 16);
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_noundef));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_invariant_load));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_alias_scope));
+ EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_noalias));
+}
+
+} // namespace
>From 001f0850d292aae7299a1fe3b48f04a8f1dbea0a Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 18 Nov 2026 01:14:09 +0000
Subject: [PATCH 13/14] [MetadataUtils] Apply stack formatting cleanup
---
llvm/unittests/Analysis/MetadataUtilsTest.cpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/unittests/Analysis/MetadataUtilsTest.cpp b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
index e8ab8d7a9c7145..58fbb4cbec982b 100644
--- a/llvm/unittests/Analysis/MetadataUtilsTest.cpp
+++ b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
@@ -270,7 +270,6 @@ TEST_F(MetadataUtilsTest, StoreTypeChangePreservesHints) {
EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_invariant_group));
}
-
TEST_F(MetadataUtilsTest, CSEIntersectsCacheHints) {
auto *A = B.CreateLoad(B.getInt32Ty(), P);
auto *BLoad = B.CreateLoad(B.getInt32Ty(), P);
>From 1480df3e00ccdcb2d9a321a1119391a4a849ae66 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 18 Nov 2026 09:11:54 +0000
Subject: [PATCH 14/14] [Analysis] Preserve metadata utility declaration order
---
llvm/include/llvm/Analysis/MetadataUtils.h | 28 +++++++++++-----------
1 file changed, 14 insertions(+), 14 deletions(-)
diff --git a/llvm/include/llvm/Analysis/MetadataUtils.h b/llvm/include/llvm/Analysis/MetadataUtils.h
index 94a0816bbc209c..b7b311f9b6993c 100644
--- a/llvm/include/llvm/Analysis/MetadataUtils.h
+++ b/llvm/include/llvm/Analysis/MetadataUtils.h
@@ -23,6 +23,20 @@ class Instruction;
class MDNode;
class Value;
+/// Combine the metadata of two instructions so that K can replace J. This
+/// specifically handles the case of CSE-like transformations. Some
+/// metadata can only be kept if K dominates J. For this to be correct,
+/// K cannot be hoisted.
+///
+/// Unknown metadata is removed.
+LLVM_ABI void combineMetadataForCSE(Instruction *K, const Instruction *J,
+ bool DoesKMove);
+
+/// Combine metadata of two instructions, where instruction J is a memory
+/// access that has been merged into K. This will intersect alias-analysis
+/// metadata, while preserving other known metadata.
+LLVM_ABI void combineAAMetadata(Instruction *K, const Instruction *J);
+
/// Copy metadata valid for a replacement memory access.
///
/// The destination must access the same bytes, a subset, or a wider byte range
@@ -56,20 +70,6 @@ LLVM_ABI void copyNonnullMetadata(const LoadInst &OldLI, MDNode *N,
LLVM_ABI void copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI,
MDNode *N, LoadInst &NewLI);
-/// Combine the metadata of two instructions so that K can replace J. This
-/// specifically handles the case of CSE-like transformations. Some
-/// metadata can only be kept if K dominates J. For this to be correct,
-/// K cannot be hoisted.
-///
-/// Unknown metadata is removed.
-LLVM_ABI void combineMetadataForCSE(Instruction *K, const Instruction *J,
- bool DoesKMove);
-
-/// Combine metadata of two instructions, where instruction J is a memory
-/// access that has been merged into K. This will intersect alias-analysis
-/// metadata, while preserving other known metadata.
-LLVM_ABI void combineAAMetadata(Instruction *K, const Instruction *J);
-
/// Compute the union of two access-group lists.
///
/// If the list contains just one access group, it is returned directly. If the
More information about the llvm-branch-commits
mailing list