[llvm-branch-commits] [llvm] [Analysis] Add metadata propagation for replacement memory accesses (PR #242249)

Yonah Goldberg via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Wed Nov 18 01:13:45 PST 2026


https://github.com/YonahGoldberg updated https://github.com/llvm/llvm-project/pull/242249

>From 6b95f61b9b54b0bd67d1d927646ae3c6a993ffc5 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Fri, 13 Nov 2026 10:35:37 +0000
Subject: [PATCH 01/14] [Transforms] Add memory metadata tests

---
 .../masked_intrinsics_keep_metadata.ll        |  19 ++
 .../InstCombine/memory-cache-hint.ll          |  59 +++++
 .../Transforms/InstCombine/memory-metadata.ll | 185 +++++++++++++++
 .../InstCombine/ptr-replace-alloca.ll         |   4 +-
 .../InstCombine/storemerge-metadata.ll        | 213 +++++++++++++++++
 llvm/test/Transforms/SROA/memory-metadata.ll  | 215 ++++++++++++++++++
 .../Scalarizer/basic-inseltpoison.ll          |   6 +-
 llvm/test/Transforms/Scalarizer/basic.ll      |   6 +-
 .../Scalarizer/memory-cache-hint.ll           |  35 +++
 .../Transforms/Scalarizer/memory-metadata.ll  | 150 ++++++++++++
 .../VectorCombine/RISCV/memory-metadata.ll    |  47 ++++
 .../VectorCombine/X86/memory-metadata.ll      |  72 ++++++
 .../VectorCombine/memory-metadata.ll          | 195 ++++++++++++++++
 13 files changed, 1197 insertions(+), 9 deletions(-)
 create mode 100644 llvm/test/Transforms/InstCombine/memory-cache-hint.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memory-metadata.ll
 create mode 100644 llvm/test/Transforms/InstCombine/storemerge-metadata.ll
 create mode 100644 llvm/test/Transforms/SROA/memory-metadata.ll
 create mode 100644 llvm/test/Transforms/Scalarizer/memory-cache-hint.ll
 create mode 100644 llvm/test/Transforms/Scalarizer/memory-metadata.ll
 create mode 100644 llvm/test/Transforms/VectorCombine/RISCV/memory-metadata.ll
 create mode 100644 llvm/test/Transforms/VectorCombine/X86/memory-metadata.ll
 create mode 100644 llvm/test/Transforms/VectorCombine/memory-metadata.ll

diff --git a/llvm/test/Transforms/InstCombine/masked_intrinsics_keep_metadata.ll b/llvm/test/Transforms/InstCombine/masked_intrinsics_keep_metadata.ll
index 0832561e2b02b5..e6029c7b58e73b 100644
--- a/llvm/test/Transforms/InstCombine/masked_intrinsics_keep_metadata.ll
+++ b/llvm/test/Transforms/InstCombine/masked_intrinsics_keep_metadata.ll
@@ -40,6 +40,20 @@ b0:
   ret void
 }
 
+; For an { i32, float } object, only the integer field is initially read.
+; TODO: Check whether the integer TBAA tag remains valid on the unmasked load,
+; which also reads the inactive float field. Record the existing behavior.
+define <2 x i32> @unmask_integer_field_tbaa(ptr dereferenceable(8) %p) {
+; CHECK-LABEL: define <2 x i32> @unmask_integer_field_tbaa(
+; CHECK-SAME: ptr dereferenceable(8) [[P:%.*]]) {
+; CHECK-NEXT:    [[UNMASKEDLOAD:%.*]] = load <2 x i32>, ptr [[P]], align 1, !tbaa [[INT_TBAA3:![0-9]+]]
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <2 x i32> [[UNMASKEDLOAD]], i32 0, i64 1
+; CHECK-NEXT:    ret <2 x i32> [[V1]]
+;
+  %v = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr %p, <2 x i1> <i1 true, i1 false>, <2 x i32> zeroinitializer), !tbaa !3
+  ret <2 x i32> %v
+}
+
 attributes #0 = { norecurse nounwind }
 
 declare <4 x i32> @llvm.masked.load.v4i1.p0(ptr, i32, <4 x i1>, <4 x i32>)
@@ -49,8 +63,13 @@ declare void @llvm.masked.store.v4i1.p0(<4 x i32>, ptr, i32, <4 x i1>)
 !1 = !{!"omnipotent char", !2, i64 0}
 !2 = !{!"Simple C/C++ TBAA"}
 
+!3 = !{!4, !4, i64 0}
+!4 = !{!"int", !1, i64 0}
+
 ;.
 ; CHECK: [[CHAR_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
 ; CHECK: [[META1]] = !{!"omnipotent char", [[META2:![0-9]+]], i64 0}
 ; CHECK: [[META2]] = !{!"Simple C/C++ TBAA"}
+; CHECK: [[INT_TBAA3]] = !{[[META4:![0-9]+]], [[META4]], i64 0}
+; CHECK: [[META4]] = !{!"int", [[META1]], i64 0}
 ;.
diff --git a/llvm/test/Transforms/InstCombine/memory-cache-hint.ll b/llvm/test/Transforms/InstCombine/memory-cache-hint.ll
new file mode 100644
index 00000000000000..653fc45056db27
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memory-cache-hint.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=instcombine < %s | FileCheck %s
+
+define void @store_bitcast(ptr %p, float %x) {
+; CHECK-LABEL: define void @store_bitcast(
+; CHECK-SAME: ptr [[P:%.*]], float [[X:%.*]]) {
+; CHECK-NEXT:    store float [[X]], ptr [[P]], align 4
+; CHECK-NEXT:    ret void
+;
+  %bits = bitcast float %x to i32
+  store i32 %bits, ptr %p, align 4, !mem.cache_hint !0
+  ret void
+}
+
+define void @copy_both_hints(ptr %dst, ptr %src) {
+; CHECK-LABEL: define void @copy_both_hints(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-NEXT:    store i32 [[TMP1]], ptr [[DST]], align 4
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dst, ptr align 4 %src, i64 4, i1 false), !mem.cache_hint !2
+  ret void
+}
+
+define void @set_destination_hint(ptr %dst) {
+; CHECK-LABEL: define void @set_destination_hint(
+; CHECK-SAME: ptr [[DST:%.*]]) {
+; CHECK-NEXT:    store i32 0, ptr [[DST]], align 4
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memset.p0.i64(ptr align 4 %dst, i8 0, i64 4, i1 false), !mem.cache_hint !3
+  ret void
+}
+
+; TODO: Check whether invariant.load can be kept after exposing inactive lanes.
+; This records the existing behavior while preserving the cache policy.
+define <2 x i32> @speculated_masked_load(ptr dereferenceable(8) %p, <2 x i1> %mask) {
+; CHECK-LABEL: define <2 x i32> @speculated_masked_load(
+; CHECK-SAME: ptr dereferenceable(8) [[P:%.*]], <2 x i1> [[MASK:%.*]]) {
+; CHECK-NEXT:    [[UNMASKEDLOAD:%.*]] = load <2 x i32>, ptr [[P]], align 1, !invariant.load [[META0:![0-9]+]], !mem.cache_hint [[META1:![0-9]+]]
+; CHECK-NEXT:    [[V:%.*]] = select <2 x i1> [[MASK]], <2 x i32> [[UNMASKEDLOAD]], <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[V]]
+;
+  %v = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr %p, <2 x i1> %mask, <2 x i32> zeroinitializer), !mem.cache_hint !3, !invariant.load !6
+  ret <2 x i32> %v
+}
+
+!0 = !{i32 1, !1}
+!1 = !{!"nvvm.l1_eviction", !"last"}
+!2 = !{i32 0, !1, i32 1, !4}
+!3 = !{i32 0, !1}
+!4 = !{!"nvvm.l1_eviction", !"first"}
+!6 = !{}
+;.
+; CHECK: [[META0]] = !{}
+; CHECK: [[META1]] = !{i32 0, [[META2:![0-9]+]]}
+; CHECK: [[META2]] = !{!"nvvm.l1_eviction", !"last"}
+;.
diff --git a/llvm/test/Transforms/InstCombine/memory-metadata.ll b/llvm/test/Transforms/InstCombine/memory-metadata.ll
new file mode 100644
index 00000000000000..8ec8483485d263
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memory-metadata.ll
@@ -0,0 +1,185 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=instcombine < %s | FileCheck %s --implicit-check-not=!test.unknown --implicit-check-not=!prof
+
+target datalayout = "e-p:64:64"
+
+define float @bitcast_load(ptr %p) {
+; CHECK-LABEL: define float @bitcast_load(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[V1:%.*]] = load float, ptr [[P]], align 4, !tbaa [[INT_TBAA0:![0-9]+]], !invariant.load [[META4:![0-9]+]], !alias.scope [[META5:![0-9]+]], !noalias [[META5]], !nontemporal [[META8:![0-9]+]], !llvm.mem.parallel_loop_access [[META9:![0-9]+]], !invariant.group [[META4]], !llvm.access.group [[META9]], !noundef [[META4]], !annotation [[META10:![0-9]+]], !noalias.addrspace [[META11:![0-9]+]], !mem.cache_hint [[META12:![0-9]+]]
+; CHECK-NEXT:    ret float [[V1]]
+;
+  %v = load i32, ptr %p, align 4, !tbaa !10, !range !9, !noundef !0, !invariant.load !0, !invariant.group !0, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+  %r = bitcast i32 %v to float
+  ret float %r
+}
+
+define void @bitcast_store(ptr %p, float %v) {
+; CHECK-LABEL: define void @bitcast_store(
+; CHECK-SAME: ptr [[P:%.*]], float [[V:%.*]]) {
+; CHECK-NEXT:    store float [[V]], ptr [[P]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]]
+; CHECK-NEXT:    ret void
+;
+  %bits = bitcast float %v to i32
+  store i32 %bits, ptr %p, align 4, !tbaa !10, !invariant.group !0, !mem.cache_hint !18, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+  ret void
+}
+
+define i32 @unpack_struct_load(ptr %p) {
+; CHECK-LABEL: define i32 @unpack_struct_load(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[V_UNPACK:%.*]] = load i32, ptr [[P]], align 8, !tbaa.struct [[TBAA_STRUCT14:![0-9]+]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT:    [[V_ELT1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4, !annotation [[META10]]
+; CHECK-NEXT:    [[V_UNPACK2:%.*]] = load i32, ptr [[V_ELT1]], align 4, !tbaa.struct [[TBAA_STRUCT14]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[V_UNPACK]], [[V_UNPACK2]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %v = load {i32, i32}, ptr %p, align 8, !tbaa.struct !19, !invariant.load !0, !invariant.group !0, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+  %a = extractvalue {i32, i32} %v, 0
+  %b = extractvalue {i32, i32} %v, 1
+  %r = add i32 %a, %b
+  ret i32 %r
+}
+
+define i32 @unpack_array_load(ptr %p) {
+; CHECK-LABEL: define i32 @unpack_array_load(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[V_UNPACK:%.*]] = load i32, ptr [[P]], align 8, !tbaa.struct [[TBAA_STRUCT14]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT:    [[V_ELT1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4, !annotation [[META10]]
+; CHECK-NEXT:    [[V_UNPACK2:%.*]] = load i32, ptr [[V_ELT1]], align 4, !tbaa.struct [[TBAA_STRUCT14]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[V_UNPACK]], [[V_UNPACK2]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %v = load [2 x i32], ptr %p, align 8, !tbaa.struct !19, !invariant.load !0, !invariant.group !0, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+  %a = extractvalue [2 x i32] %v, 0
+  %b = extractvalue [2 x i32] %v, 1
+  %r = add i32 %a, %b
+  ret i32 %r
+}
+
+define void @unpack_struct_store(ptr %p, {i32, i32} %v) {
+; CHECK-LABEL: define void @unpack_struct_store(
+; CHECK-SAME: ptr [[P:%.*]], { i32, i32 } [[V:%.*]]) {
+; CHECK-NEXT:    [[V_ELT:%.*]] = extractvalue { i32, i32 } [[V]], 0, !annotation [[META10]]
+; CHECK-NEXT:    store i32 [[V_ELT]], ptr [[P]], align 8, !tbaa.struct [[TBAA_STRUCT14]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT:    [[P_REPACK1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4, !annotation [[META10]]
+; CHECK-NEXT:    [[V_ELT2:%.*]] = extractvalue { i32, i32 } [[V]], 1, !annotation [[META10]]
+; CHECK-NEXT:    store i32 [[V_ELT2]], ptr [[P_REPACK1]], align 4, !tbaa.struct [[TBAA_STRUCT14]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT:    ret void
+;
+  store {i32, i32} %v, ptr %p, align 8, !tbaa.struct !19, !invariant.group !0, !mem.cache_hint !18, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+  ret void
+}
+
+define void @unpack_array_store(ptr %p, [2 x i32] %v) {
+; CHECK-LABEL: define void @unpack_array_store(
+; CHECK-SAME: ptr [[P:%.*]], [2 x i32] [[V:%.*]]) {
+; CHECK-NEXT:    [[V_ELT:%.*]] = extractvalue [2 x i32] [[V]], 0, !annotation [[META10]]
+; CHECK-NEXT:    store i32 [[V_ELT]], ptr [[P]], align 8, !tbaa.struct [[TBAA_STRUCT14]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT:    [[P_REPACK1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4, !annotation [[META10]]
+; CHECK-NEXT:    [[V_ELT2:%.*]] = extractvalue [2 x i32] [[V]], 1, !annotation [[META10]]
+; CHECK-NEXT:    store i32 [[V_ELT2]], ptr [[P_REPACK1]], align 4, !tbaa.struct [[TBAA_STRUCT14]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT:    ret void
+;
+  store [2 x i32] %v, ptr %p, align 8, !tbaa.struct !19, !invariant.group !0, !mem.cache_hint !18, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+  ret void
+}
+
+; The two pointer operands carry different policies. Call profiling must not
+; be attached to the replacement native memory operations.
+define void @copy_memory(ptr %dst, ptr %src) {
+; CHECK-LABEL: define void @copy_memory(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr [[SRC]], align 8, !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT:    store i64 [[TMP1]], ptr [[DST]], align 8, !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %src, i64 8, i1 false), !tbaa.struct !19, !prof !24, !mem.cache_hint !22, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+  ret void
+}
+
+define void @set_memory(ptr %dst) {
+; CHECK-LABEL: define void @set_memory(
+; CHECK-SAME: ptr [[DST:%.*]]) {
+; CHECK-NEXT:    store i32 0, ptr [[DST]], align 4, !annotation [[META10]]
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memset.p0.i64(ptr align 4 %dst, i8 0, i64 4, i1 false), !tbaa !10, !prof !24, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+  ret void
+}
+
+define <2 x i32> @splat_gather(ptr %p) {
+; CHECK-LABEL: define <2 x i32> @splat_gather(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[LOAD_SCALAR:%.*]] = load i32, ptr [[P]], align 1, !annotation [[META10]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[LOAD_SCALAR]], i64 0, !annotation [[META10]]
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer, !annotation [[META10]]
+; CHECK-NEXT:    ret <2 x i32> [[BROADCAST_SPLAT]]
+;
+  %ptr0 = insertelement <2 x ptr> poison, ptr %p, i32 0
+  %ptrs = shufflevector <2 x ptr> %ptr0, <2 x ptr> poison, <2 x i32> zeroinitializer
+  %v = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> %ptrs, <2 x i1> <i1 true, i1 true>, <2 x i32> poison), !tbaa !10, !range !9, !invariant.load !0, !prof !24, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+  ret <2 x i32> %v
+}
+
+define void @splat_scatter(ptr %p, i32 %v) {
+; CHECK-LABEL: define void @splat_scatter(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[V:%.*]]) {
+; CHECK-NEXT:    store i32 [[V]], ptr [[P]], align 1, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META17:![0-9]+]], !noalias.addrspace [[META11]], !mem.cache_hint [[META18:![0-9]+]]
+; CHECK-NEXT:    ret void
+;
+  %ptr0 = insertelement <2 x ptr> poison, ptr %p, i32 0
+  %ptrs = shufflevector <2 x ptr> %ptr0, <2 x ptr> poison, <2 x i32> zeroinitializer
+  %val0 = insertelement <2 x i32> poison, i32 %v, i32 0
+  %vals = shufflevector <2 x i32> %val0, <2 x i32> poison, <2 x i32> zeroinitializer
+  call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> %vals, <2 x ptr> %ptrs, <2 x i1> <i1 true, i1 true>), !tbaa !10, !mem.cache_hint !18, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4
+  ret void
+}
+
+!0 = !{}
+!1 = !{i32 1}
+!2 = !{!"preserved memory metadata"}
+!3 = !{!"test", !"relaxed"}
+!4 = !{i32 5, i32 6}
+!5 = distinct !{}
+!6 = !{!7}
+!7 = distinct !{!7, !8, !"scope"}
+!8 = distinct !{!8, i1 false, !"domain"}
+!9 = !{i32 0, i32 100}
+!10 = !{!11, !11, i64 0}
+!11 = !{!"int", !12, i64 0}
+!12 = !{!"omnipotent char", !13, i64 0}
+!13 = !{!"root"}
+!14 = !{float 2.5}
+!15 = !{i32 3}
+!16 = !{i32 0, !17}
+!17 = !{!"nvvm.l1_eviction", !"last"}
+!18 = !{i32 1, !17}
+!19 = !{i64 0, i64 4, !10, i64 4, i64 4, !20}
+!20 = !{!21, !21, i64 0}
+!21 = !{!"float", !12, i64 0}
+!22 = !{i32 0, !17, i32 1, !23}
+!23 = !{!"nvvm.l1_eviction", !"first"}
+!24 = !{!"branch_weights", i32 10}
+!25 = !{i64 8}
+;.
+; CHECK: [[INT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; CHECK: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
+; CHECK: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
+; CHECK: [[META3]] = !{!"root"}
+; CHECK: [[META4]] = !{}
+; CHECK: [[META5]] = !{[[META6:![0-9]+]]}
+; CHECK: [[META6]] = distinct !{[[META6]], [[META7:![0-9]+]], !"scope"}
+; CHECK: [[META7]] = distinct !{[[META7]], i1 false, !"domain"}
+; CHECK: [[META8]] = !{i32 1}
+; CHECK: [[META9]] = distinct !{}
+; CHECK: [[META10]] = !{!"preserved memory metadata"}
+; CHECK: [[META11]] = !{i32 5, i32 6}
+; CHECK: [[META12]] = !{i32 0, [[META13:![0-9]+]]}
+; CHECK: [[META13]] = !{!"nvvm.l1_eviction", !"last"}
+; CHECK: [[TBAA_STRUCT14]] = !{i64 0, i64 4, [[INT_TBAA0]], i64 4, i64 4, [[META15:![0-9]+]]}
+; CHECK: [[META15]] = !{[[META16:![0-9]+]], [[META16]], i64 0}
+; CHECK: [[META16]] = !{!"float", [[META2]], i64 0}
+; CHECK: [[META17]] = !{!"test", !"relaxed"}
+; CHECK: [[META18]] = !{i32 1, [[META13]]}
+;.
diff --git a/llvm/test/Transforms/InstCombine/ptr-replace-alloca.ll b/llvm/test/Transforms/InstCombine/ptr-replace-alloca.ll
index a3bab9d5403bac..d0043e58537f82 100644
--- a/llvm/test/Transforms/InstCombine/ptr-replace-alloca.ll
+++ b/llvm/test/Transforms/InstCombine/ptr-replace-alloca.ll
@@ -514,9 +514,7 @@ define i8 @call_readonly_keep_alloca2() {
   ret i8 %v
 }
 
-; The pointer changes but the load type doesn't, so all metadata (here
-; !annotation, which the type-changing copyMetadataForLoad would drop) must be
-; preserved.
+; Only the pointer changes, so preserve all metadata on the load.
 @g3 = external constant [32 x i8], align 16
 
 define i8 @preserve_load_metadata(i64 %idx) {
diff --git a/llvm/test/Transforms/InstCombine/storemerge-metadata.ll b/llvm/test/Transforms/InstCombine/storemerge-metadata.ll
new file mode 100644
index 00000000000000..dc5fdee99c333f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/storemerge-metadata.ll
@@ -0,0 +1,213 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=instcombine < %s | FileCheck %s
+
+define void @common_metadata(ptr %p, i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: define void @common_metadata(
+; CHECK-SAME: ptr [[P:%.*]], i1 [[C:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[C]], label %[[LEFT:.*]], label %[[RIGHT:.*]]
+; CHECK:       [[LEFT]]:
+; CHECK-NEXT:    br label %[[JOIN:.*]]
+; CHECK:       [[RIGHT]]:
+; CHECK-NEXT:    br label %[[JOIN]]
+; CHECK:       [[JOIN]]:
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi i32 [ [[B]], %[[RIGHT]] ], [ [[A]], %[[LEFT]] ]
+; CHECK-NEXT:    store i32 [[STOREMERGE]], ptr [[P]], align 4, !tbaa [[INT_TBAA0:![0-9]+]], !alias.scope [[META4:![0-9]+]], !noalias [[META4]], !llvm.access.group [[ACC_GRP7:![0-9]+]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %c, label %left, label %right
+left:
+  store i32 %a, ptr %p, align 4, !tbaa !10, !alias.scope !14, !noalias !14, !nontemporal !4, !llvm.access.group !8, !mem.cache_hint !0
+  br label %join
+right:
+  store i32 %b, ptr %p, align 4, !tbaa !10, !alias.scope !14, !noalias !14, !nontemporal !4, !llvm.access.group !8, !mem.cache_hint !0
+  br label %join
+join:
+  ret void
+}
+
+define void @intersect_access_groups(ptr %p, i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: define void @intersect_access_groups(
+; CHECK-SAME: ptr [[P:%.*]], i1 [[C:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[C]], label %[[LEFT:.*]], label %[[RIGHT:.*]]
+; CHECK:       [[LEFT]]:
+; CHECK-NEXT:    br label %[[JOIN:.*]]
+; CHECK:       [[RIGHT]]:
+; CHECK-NEXT:    br label %[[JOIN]]
+; CHECK:       [[JOIN]]:
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi i32 [ [[B]], %[[RIGHT]] ], [ [[A]], %[[LEFT]] ]
+; CHECK-NEXT:    store i32 [[STOREMERGE]], ptr [[P]], align 4, !llvm.access.group [[ACC_GRP9:![0-9]+]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %c, label %left, label %right
+left:
+  store i32 %a, ptr %p, align 4, !llvm.access.group !8
+  br label %join
+right:
+  store i32 %b, ptr %p, align 4, !llvm.access.group !9
+  br label %join
+join:
+  ret void
+}
+
+define void @conflicting_cache_hints(ptr %p, i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: define void @conflicting_cache_hints(
+; CHECK-SAME: ptr [[P:%.*]], i1 [[C:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[C]], label %[[LEFT:.*]], label %[[RIGHT:.*]]
+; CHECK:       [[LEFT]]:
+; CHECK-NEXT:    br label %[[JOIN:.*]]
+; CHECK:       [[RIGHT]]:
+; CHECK-NEXT:    br label %[[JOIN]]
+; CHECK:       [[JOIN]]:
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi i32 [ [[B]], %[[RIGHT]] ], [ [[A]], %[[LEFT]] ]
+; CHECK-NEXT:    store i32 [[STOREMERGE]], ptr [[P]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %c, label %left, label %right
+left:
+  store i32 %a, ptr %p, align 4, !mem.cache_hint !0
+  br label %join
+right:
+  store i32 %b, ptr %p, align 4, !mem.cache_hint !2
+  br label %join
+join:
+  ret void
+}
+
+define void @metadata_only_left(ptr %p, i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: define void @metadata_only_left(
+; CHECK-SAME: ptr [[P:%.*]], i1 [[C:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[C]], label %[[LEFT:.*]], label %[[RIGHT:.*]]
+; CHECK:       [[LEFT]]:
+; CHECK-NEXT:    br label %[[JOIN:.*]]
+; CHECK:       [[RIGHT]]:
+; CHECK-NEXT:    br label %[[JOIN]]
+; CHECK:       [[JOIN]]:
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi i32 [ [[B]], %[[RIGHT]] ], [ [[A]], %[[LEFT]] ]
+; CHECK-NEXT:    store i32 [[STOREMERGE]], ptr [[P]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %c, label %left, label %right
+left:
+  store i32 %a, ptr %p, align 4, !tbaa !10, !alias.scope !14, !noalias !14, !nontemporal !4, !llvm.access.group !8, !mem.cache_hint !0
+  br label %join
+right:
+  store i32 %b, ptr %p, align 4
+  br label %join
+join:
+  ret void
+}
+
+define void @metadata_only_right(ptr %p, i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: define void @metadata_only_right(
+; CHECK-SAME: ptr [[P:%.*]], i1 [[C:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[C]], label %[[LEFT:.*]], label %[[RIGHT:.*]]
+; CHECK:       [[LEFT]]:
+; CHECK-NEXT:    br label %[[JOIN:.*]]
+; CHECK:       [[RIGHT]]:
+; CHECK-NEXT:    br label %[[JOIN]]
+; CHECK:       [[JOIN]]:
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi i32 [ [[B]], %[[RIGHT]] ], [ [[A]], %[[LEFT]] ]
+; CHECK-NEXT:    store i32 [[STOREMERGE]], ptr [[P]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %c, label %left, label %right
+left:
+  store i32 %a, ptr %p, align 4
+  br label %join
+right:
+  store i32 %b, ptr %p, align 4, !tbaa !10, !alias.scope !14, !noalias !14, !nontemporal !4, !llvm.access.group !8, !mem.cache_hint !0
+  br label %join
+join:
+  ret void
+}
+
+define void @triangle_common_metadata(ptr %p, i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: define void @triangle_common_metadata(
+; CHECK-SAME: ptr [[P:%.*]], i1 [[C:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[JOIN:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    br label %[[JOIN]]
+; CHECK:       [[JOIN]]:
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi i32 [ [[B]], %[[THEN]] ], [ [[A]], %[[ENTRY]] ]
+; CHECK-NEXT:    store i32 [[STOREMERGE]], ptr [[P]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  store i32 %a, ptr %p, align 4, !nontemporal !4, !mem.cache_hint !0
+  br i1 %c, label %then, label %join
+then:
+  store i32 %b, ptr %p, align 4, !nontemporal !4, !mem.cache_hint !0
+  br label %join
+join:
+  ret void
+}
+; Capture components must be combined across both incoming pointer values.
+define void @merge_pointer_captures(ptr %p, i1 %c, ptr %a, ptr %b) {
+; CHECK-LABEL: define void @merge_pointer_captures(
+; CHECK-SAME: ptr [[P:%.*]], i1 [[C:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[C]], label %[[LEFT:.*]], label %[[RIGHT:.*]]
+; CHECK:       [[LEFT]]:
+; CHECK-NEXT:    br label %[[JOIN:.*]]
+; CHECK:       [[RIGHT]]:
+; CHECK-NEXT:    br label %[[JOIN]]
+; CHECK:       [[JOIN]]:
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi ptr [ [[B]], %[[RIGHT]] ], [ [[A]], %[[LEFT]] ]
+; CHECK-NEXT:    store ptr [[STOREMERGE]], ptr [[P]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %c, label %left, label %right
+left:
+  store ptr %a, ptr %p, align 8, !captures !17
+  br label %join
+right:
+  store ptr %b, ptr %p, align 8, !captures !18
+  br label %join
+join:
+  ret void
+}
+
+!0 = !{i32 1, !1}
+!1 = !{!"nvvm.l1_eviction", !"last"}
+!2 = !{i32 1, !3}
+!3 = !{!"nvvm.l1_eviction", !"first"}
+!4 = !{i32 1}
+!5 = distinct !{}
+!6 = distinct !{}
+!7 = distinct !{}
+!8 = !{!5, !6}
+!9 = !{!6, !7}
+!10 = !{!11, !11, i64 0}
+!11 = !{!"int", !12}
+!12 = !{!"char", !13}
+!13 = !{!"root"}
+!14 = !{!15}
+!15 = distinct !{!15, !16}
+!16 = distinct !{!16, i1 false, !"domain"}
+!17 = !{!"address"}
+!18 = !{!"read_provenance"}
+
+;.
+; CHECK: [[INT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; CHECK: [[META1]] = !{!"int", [[META2:![0-9]+]]}
+; CHECK: [[META2]] = !{!"char", [[META3:![0-9]+]]}
+; CHECK: [[META3]] = !{!"root"}
+; CHECK: [[META4]] = !{[[META5:![0-9]+]]}
+; CHECK: [[META5]] = distinct !{[[META5]], [[META6:![0-9]+]]}
+; CHECK: [[META6]] = distinct !{[[META6]], i1 false, !"domain"}
+; CHECK: [[ACC_GRP7]] = !{[[META8:![0-9]+]], [[ACC_GRP9]]}
+; CHECK: [[META8]] = distinct !{}
+; CHECK: [[ACC_GRP9]] = distinct !{}
+;.
diff --git a/llvm/test/Transforms/SROA/memory-metadata.ll b/llvm/test/Transforms/SROA/memory-metadata.ll
new file mode 100644
index 00000000000000..a1d2f5687a4597
--- /dev/null
+++ b/llvm/test/Transforms/SROA/memory-metadata.ll
@@ -0,0 +1,215 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=sroa < %s | FileCheck %s --implicit-check-not=!test.unknown --implicit-check-not=!prof
+
+target datalayout = "e-p:64:64-n8:16:32:64"
+
+define i32 @split_copy(ptr %src) {
+; CHECK-LABEL: define i32 @split_copy(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:    [[A_SROA_0:%.*]] = alloca i32, align 8
+; CHECK-NEXT:    [[A_SROA_2:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[A_SROA_0_0_COPYLOAD:%.*]] = load i32, ptr [[SRC]], align 8, !tbaa [[INT_TBAA0:![0-9]+]], !alias.scope [[META4:![0-9]+]], !noalias [[META4]], !llvm.mem.parallel_loop_access [[META7:![0-9]+]], !llvm.access.group [[META7]], !noalias.addrspace [[META8:![0-9]+]]
+; CHECK-NEXT:    store i32 [[A_SROA_0_0_COPYLOAD]], ptr [[A_SROA_0]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META4]], !noalias [[META4]], !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]], !noalias.addrspace [[META8]]
+; CHECK-NEXT:    [[A_SROA_2_0_SRC_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 4
+; CHECK-NEXT:    [[A_SROA_2_0_COPYLOAD:%.*]] = load i32, ptr [[A_SROA_2_0_SRC_SROA_IDX]], align 4, !tbaa [[FLOAT_TBAA9:![0-9]+]], !alias.scope [[META4]], !noalias [[META4]], !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]], !noalias.addrspace [[META8]]
+; CHECK-NEXT:    store i32 [[A_SROA_2_0_COPYLOAD]], ptr [[A_SROA_2]], align 4, !tbaa [[FLOAT_TBAA9]], !alias.scope [[META4]], !noalias [[META4]], !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]], !noalias.addrspace [[META8]]
+; CHECK-NEXT:    [[A_SROA_0_0_A_SROA_0_0_X:%.*]] = load volatile i32, ptr [[A_SROA_0]], align 8
+; CHECK-NEXT:    [[A_SROA_2_0_A_SROA_2_4_Y:%.*]] = load volatile i32, ptr [[A_SROA_2]], align 4
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[A_SROA_0_0_A_SROA_0_0_X]], [[A_SROA_2_0_A_SROA_2_4_Y]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = alloca {i32, i32}, align 8
+  call void @llvm.memcpy.p0.p0.i64(ptr align 8 %a, ptr align 8 %src, i64 8, i1 false), !tbaa.struct !19, !prof !24, !mem.cache_hint !22, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+  %p1 = getelementptr {i32, i32}, ptr %a, i32 0, i32 1
+  %x = load volatile i32, ptr %a, align 8
+  %y = load volatile i32, ptr %p1, align 4
+  %r = add i32 %x, %y
+  ret i32 %r
+}
+
+define i32 @rewrite_set(i32 %v) {
+; CHECK-LABEL: define i32 @rewrite_set(
+; CHECK-SAME: i32 [[V:%.*]]) {
+; CHECK-NEXT:    [[A_SROA_0:%.*]] = alloca i32, align 8
+; CHECK-NEXT:    [[A_SROA_2:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 [[V]], ptr [[A_SROA_0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[A_SROA_2]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META4]], !noalias [[META4]], !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]], !noalias.addrspace [[META8]]
+; CHECK-NEXT:    [[A_SROA_0_0_A_SROA_0_0_X:%.*]] = load volatile i32, ptr [[A_SROA_0]], align 8
+; CHECK-NEXT:    [[A_SROA_2_0_A_SROA_2_4_Y:%.*]] = load volatile i32, ptr [[A_SROA_2]], align 4
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[A_SROA_0_0_A_SROA_0_0_X]], [[A_SROA_2_0_A_SROA_2_4_Y]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = alloca {i32, i32}, align 8
+  %p1 = getelementptr {i32, i32}, ptr %a, i32 0, i32 1
+  store i32 %v, ptr %a, align 8
+  call void @llvm.memset.p0.i64(ptr align 4 %p1, i8 0, i64 4, i1 false), !tbaa !10, !prof !24, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+  %x = load volatile i32, ptr %a, align 8
+  %y = load volatile i32, ptr %p1, align 4
+  %r = add i32 %x, %y
+  ret i32 %r
+}
+
+define i32 @split_aggregate_store({i32, i32} %v) {
+; CHECK-LABEL: define i32 @split_aggregate_store(
+; CHECK-SAME: { i32, i32 } [[V:%.*]]) {
+; CHECK-NEXT:    [[A_SROA_0:%.*]] = alloca i32, align 8
+; CHECK-NEXT:    [[A_SROA_2:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[V_FCA_0_EXTRACT:%.*]] = extractvalue { i32, i32 } [[V]], 0
+; CHECK-NEXT:    store i32 [[V_FCA_0_EXTRACT]], ptr [[A_SROA_0]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META4]], !noalias [[META4]], !noalias.addrspace [[META8]]
+; CHECK-NEXT:    [[V_FCA_1_EXTRACT:%.*]] = extractvalue { i32, i32 } [[V]], 1
+; CHECK-NEXT:    store i32 [[V_FCA_1_EXTRACT]], ptr [[A_SROA_2]], align 4, !tbaa [[FLOAT_TBAA9]], !alias.scope [[META4]], !noalias [[META4]], !noalias.addrspace [[META8]]
+; CHECK-NEXT:    [[A_SROA_0_0_A_SROA_0_0_X:%.*]] = load volatile i32, ptr [[A_SROA_0]], align 8
+; CHECK-NEXT:    [[A_SROA_2_0_A_SROA_2_4_Y:%.*]] = load volatile i32, ptr [[A_SROA_2]], align 4
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[A_SROA_0_0_A_SROA_0_0_X]], [[A_SROA_2_0_A_SROA_2_4_Y]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = alloca {i32, i32}, align 8
+  store {i32, i32} %v, ptr %a, align 8, !tbaa.struct !19, !invariant.group !0, !mem.cache_hint !18, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+  %p1 = getelementptr {i32, i32}, ptr %a, i32 0, i32 1
+  %x = load volatile i32, ptr %a, align 8
+  %y = load volatile i32, ptr %p1, align 4
+  %r = add i32 %x, %y
+  ret i32 %r
+}
+
+define {i32, i32} @split_aggregate_load(ptr %src, ptr %dst) {
+; CHECK-LABEL: define { i32, i32 } @split_aggregate_load(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]]) {
+; CHECK-NEXT:    [[A_SROA_0:%.*]] = alloca i32, align 8
+; CHECK-NEXT:    [[A_SROA_3:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[A_SROA_0_0_COPYLOAD:%.*]] = load i32, ptr [[SRC]], align 1
+; CHECK-NEXT:    store i32 [[A_SROA_0_0_COPYLOAD]], ptr [[A_SROA_0]], align 8
+; CHECK-NEXT:    [[A_SROA_3_0_SRC_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 4
+; CHECK-NEXT:    [[A_SROA_3_0_COPYLOAD:%.*]] = load i32, ptr [[A_SROA_3_0_SRC_SROA_IDX]], align 1
+; CHECK-NEXT:    store i32 [[A_SROA_3_0_COPYLOAD]], ptr [[A_SROA_3]], align 4
+; CHECK-NEXT:    [[A_SROA_0_0_A_SROA_0_0_V_FCA_0_LOAD:%.*]] = load i32, ptr [[A_SROA_0]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META4]], !noalias [[META4]], !noalias.addrspace [[META8]]
+; CHECK-NEXT:    [[V_FCA_0_INSERT:%.*]] = insertvalue { i32, i32 } poison, i32 [[A_SROA_0_0_A_SROA_0_0_V_FCA_0_LOAD]], 0
+; CHECK-NEXT:    [[A_SROA_3_0_A_SROA_3_4_V_FCA_1_LOAD:%.*]] = load i32, ptr [[A_SROA_3]], align 4, !tbaa [[FLOAT_TBAA9]], !alias.scope [[META4]], !noalias [[META4]], !noalias.addrspace [[META8]]
+; CHECK-NEXT:    [[V_FCA_1_INSERT:%.*]] = insertvalue { i32, i32 } [[V_FCA_0_INSERT]], i32 [[A_SROA_3_0_A_SROA_3_4_V_FCA_1_LOAD]], 1
+; CHECK-NEXT:    [[A_SROA_0_0_A_SROA_0_0_COPYLOAD1:%.*]] = load volatile i32, ptr [[A_SROA_0]], align 8
+; CHECK-NEXT:    store volatile i32 [[A_SROA_0_0_A_SROA_0_0_COPYLOAD1]], ptr [[DST]], align 1
+; CHECK-NEXT:    [[A_SROA_3_0_DST_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 4
+; CHECK-NEXT:    [[A_SROA_3_0_A_SROA_3_0_COPYLOAD2:%.*]] = load volatile i32, ptr [[A_SROA_3]], align 4
+; CHECK-NEXT:    store volatile i32 [[A_SROA_3_0_A_SROA_3_0_COPYLOAD2]], ptr [[A_SROA_3_0_DST_SROA_IDX]], align 1
+; CHECK-NEXT:    ret { i32, i32 } [[V_FCA_1_INSERT]]
+;
+  %a = alloca {i32, i32}, align 8
+  call void @llvm.memcpy.p0.p0.i64(ptr %a, ptr %src, i64 8, i1 false)
+  %v = load {i32, i32}, ptr %a, align 8, !tbaa.struct !19, !invariant.load !0, !invariant.group !0, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+  call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %a, i64 8, i1 true)
+  ret {i32, i32} %v
+}
+
+define double @rewrite_load(ptr %src, ptr %dst) {
+; CHECK-LABEL: define double @rewrite_load(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]]) {
+; CHECK-NEXT:    [[A_SROA_0:%.*]] = alloca double, align 8
+; CHECK-NEXT:    [[A_SROA_3:%.*]] = alloca i32, align 8
+; CHECK-NEXT:    [[A_SROA_0_0_COPYLOAD:%.*]] = load double, ptr [[SRC]], align 1
+; CHECK-NEXT:    store double [[A_SROA_0_0_COPYLOAD]], ptr [[A_SROA_0]], align 8
+; CHECK-NEXT:    [[A_SROA_3_0_SRC_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 8
+; CHECK-NEXT:    [[A_SROA_3_0_COPYLOAD:%.*]] = load i32, ptr [[A_SROA_3_0_SRC_SROA_IDX]], align 1
+; CHECK-NEXT:    store i32 [[A_SROA_3_0_COPYLOAD]], ptr [[A_SROA_3]], align 8
+; CHECK-NEXT:    [[A_SROA_0_0_A_SROA_0_0_V:%.*]] = load double, ptr [[A_SROA_0]], align 8, !fpmath [[META11:![0-9]+]], !invariant.load [[META12:![0-9]+]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META13:![0-9]+]], !llvm.mem.parallel_loop_access [[META7]], !invariant.group [[META12]], !llvm.access.group [[META7]], !noundef [[META12]], !noalias.addrspace [[META8]], !nofpclass [[META14:![0-9]+]], !mem.cache_hint [[META15:![0-9]+]]
+; CHECK-NEXT:    [[A_SROA_0_0_A_SROA_0_0_COPYLOAD1:%.*]] = load volatile double, ptr [[A_SROA_0]], align 8
+; CHECK-NEXT:    store volatile double [[A_SROA_0_0_A_SROA_0_0_COPYLOAD1]], ptr [[DST]], align 1
+; CHECK-NEXT:    [[A_SROA_3_0_DST_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 8
+; CHECK-NEXT:    [[A_SROA_3_0_A_SROA_3_0_COPYLOAD2:%.*]] = load volatile i32, ptr [[A_SROA_3]], align 8
+; CHECK-NEXT:    store volatile i32 [[A_SROA_3_0_A_SROA_3_0_COPYLOAD2]], ptr [[A_SROA_3_0_DST_SROA_IDX]], align 1
+; CHECK-NEXT:    ret double [[A_SROA_0_0_A_SROA_0_0_V]]
+;
+  %a = alloca {double, i32}, align 8
+  call void @llvm.memcpy.p0.p0.i64(ptr %a, ptr %src, i64 12, i1 false)
+  %v = load double, ptr %a, align 8, !noundef !0, !nofpclass !15, !fpmath !14, !invariant.load !0, !invariant.group !0, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+  call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %a, i64 12, i1 true)
+  ret double %v
+}
+
+define ptr @rewrite_pointer_load(ptr %v) {
+; CHECK-LABEL: define ptr @rewrite_pointer_load(
+; CHECK-SAME: ptr [[V:%.*]]) {
+; CHECK-NEXT:    [[A_SROA_1:%.*]] = alloca ptr, align 8
+; CHECK-NEXT:    store ptr [[V]], ptr [[A_SROA_1]], align 8
+; CHECK-NEXT:    [[A_SROA_1_0_A_SROA_1_8_R:%.*]] = load volatile ptr, ptr [[A_SROA_1]], align 8, !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META13]], !llvm.mem.parallel_loop_access [[META7]], !nonnull [[META12]], !dereferenceable [[META17:![0-9]+]], !dereferenceable_or_null [[META17]], !invariant.group [[META12]], !align [[META17]], !llvm.access.group [[META7]], !noundef [[META12]], !noalias.addrspace [[META8]], !mem.cache_hint [[META15]]
+; CHECK-NEXT:    ret ptr [[A_SROA_1_0_A_SROA_1_8_R]]
+;
+  %a = alloca [2 x ptr], align 8
+  %p1 = getelementptr [2 x ptr], ptr %a, i32 0, i32 1
+  store ptr null, ptr %a, align 8
+  store ptr %v, ptr %p1, align 8
+  %r = load volatile ptr, ptr %p1, align 8, !nonnull !0, !noundef !0, !align !25, !dereferenceable !25, !dereferenceable_or_null !25, !invariant.group !0, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+  ret ptr %r
+}
+
+define i32 @presplit_load(ptr %src) {
+; CHECK-LABEL: define i32 @presplit_load(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:    [[A_SROA_0:%.*]] = alloca i32, align 8
+; CHECK-NEXT:    [[A_SROA_2:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[V1:%.*]] = load i32, ptr [[SRC]], align 8, !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]]
+; CHECK-NEXT:    [[SRC_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 4
+; CHECK-NEXT:    [[V2:%.*]] = load i32, ptr [[SRC_SROA_IDX]], align 4, !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]]
+; CHECK-NEXT:    store i32 [[V1]], ptr [[A_SROA_0]], align 8, !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]]
+; CHECK-NEXT:    store i32 [[V2]], ptr [[A_SROA_2]], align 4, !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]]
+; CHECK-NEXT:    [[A_SROA_0_0_A_SROA_0_0_X:%.*]] = load volatile i32, ptr [[A_SROA_0]], align 8
+; CHECK-NEXT:    [[A_SROA_2_0_A_SROA_2_4_Y:%.*]] = load volatile i32, ptr [[A_SROA_2]], align 4
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[A_SROA_0_0_A_SROA_0_0_X]], [[A_SROA_2_0_A_SROA_2_4_Y]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = alloca {i32, i32}, align 8
+  %v = load i64, ptr %src, align 8, !range !26, !noundef !0, !invariant.load !0, !invariant.group !0, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+  store i64 %v, ptr %a, align 8, !mem.cache_hint !18, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+  %p1 = getelementptr {i32, i32}, ptr %a, i32 0, i32 1
+  %x = load volatile i32, ptr %a, align 8
+  %y = load volatile i32, ptr %p1, align 4
+  %r = add i32 %x, %y
+  ret i32 %r
+}
+
+!0 = !{}
+!1 = !{i32 1}
+!2 = !{!"preserved memory metadata"}
+!3 = !{!"test", !"relaxed"}
+!4 = !{i32 5, i32 6}
+!5 = distinct !{}
+!6 = !{!7}
+!7 = distinct !{!7, !8, !"scope"}
+!8 = distinct !{!8, i1 false, !"domain"}
+!9 = !{i32 0, i32 100}
+!10 = !{!11, !11, i64 0}
+!11 = !{!"int", !12, i64 0}
+!12 = !{!"omnipotent char", !13, i64 0}
+!13 = !{!"root"}
+!14 = !{float 2.5}
+!15 = !{i32 3}
+!16 = !{i32 0, !17}
+!17 = !{!"nvvm.l1_eviction", !"last"}
+!18 = !{i32 1, !17}
+!19 = !{i64 0, i64 4, !10, i64 4, i64 4, !20}
+!20 = !{!21, !21, i64 0}
+!21 = !{!"float", !12, i64 0}
+!22 = !{i32 0, !17, i32 1, !23}
+!23 = !{!"nvvm.l1_eviction", !"first"}
+!24 = !{!"branch_weights", i32 10}
+!25 = !{i64 8}
+!26 = !{i64 0, i64 100}
+;.
+; CHECK: [[INT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; CHECK: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
+; CHECK: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
+; CHECK: [[META3]] = !{!"root"}
+; CHECK: [[META4]] = !{[[META5:![0-9]+]]}
+; CHECK: [[META5]] = distinct !{[[META5]], [[META6:![0-9]+]], !"scope"}
+; CHECK: [[META6]] = distinct !{[[META6]], i1 false, !"domain"}
+; CHECK: [[META7]] = distinct !{}
+; CHECK: [[META8]] = !{i32 5, i32 6}
+; CHECK: [[FLOAT_TBAA9]] = !{[[META10:![0-9]+]], [[META10]], i64 0}
+; CHECK: [[META10]] = !{!"float", [[META2]], i64 0}
+; CHECK: [[META11]] = !{float 2.500000e+00}
+; CHECK: [[META12]] = !{}
+; CHECK: [[META13]] = !{i32 1}
+; CHECK: [[META14]] = !{i32 3}
+; CHECK: [[META15]] = !{i32 0, [[META16:![0-9]+]]}
+; CHECK: [[META16]] = !{!"nvvm.l1_eviction", !"last"}
+; CHECK: [[META17]] = !{i64 8}
+;.
diff --git a/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll b/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
index 0a3516c82ad10b..b7db48d9b9cbc7 100644
--- a/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
+++ b/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
@@ -202,7 +202,7 @@ define void @f3(ptr %src, ptr %dst) {
   ret void
 }
 
-; Check that !tbaa.struct information is preserved.
+; Check that !tbaa.struct fields become the correct per-lane !tbaa tags.
 define void @f4(ptr %src, ptr %dst) {
 ; CHECK-LABEL: define void @f4(
 ; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]]) {
@@ -858,7 +858,7 @@ define <2 x i32> @f23_crash(<2 x i32> %srcvec, i32 %v1) {
 !2 = !{!15, !15, i64 0}
 !3 = !{ !3, !{!"llvm.loop.parallel_accesses", !13} }
 !4 = !{ float 4.0 }
-!5 = !{ i64 0, i64 8, null }
+!5 = !{i64 0, i64 4, !1, i64 4, i64 4, !2, i64 8, i64 4, !1, i64 12, i64 4, !2}
 !13 = distinct !{}
 !14 = !{!"set1", !0}
 !15 = !{!"set2", !0}
@@ -868,7 +868,7 @@ define <2 x i32> @f23_crash(<2 x i32> %srcvec, i32 %v1) {
 ; CHECK: [[META2]] = !{!"root"}
 ; CHECK: [[SET2_TBAA3]] = !{[[META4:![0-9]+]], [[META4]], i64 0}
 ; CHECK: [[META4]] = !{!"set2", [[META2]]}
-; CHECK: [[TBAA_STRUCT5]] = !{i64 0, i64 8, null}
+; CHECK: [[TBAA_STRUCT5]] = !{i64 0, i64 4, [[SET1_TBAA0]], i64 4, i64 4, [[SET2_TBAA3]], i64 8, i64 4, [[SET1_TBAA0]], i64 12, i64 4, [[SET2_TBAA3]]}
 ; CHECK: [[ACC_GRP6]] = distinct !{}
 ; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META8:![0-9]+]]}
 ; CHECK: [[META8]] = !{!"llvm.loop.parallel_accesses", [[ACC_GRP6]]}
diff --git a/llvm/test/Transforms/Scalarizer/basic.ll b/llvm/test/Transforms/Scalarizer/basic.ll
index ffde5600bbc9a7..f620445612f6dd 100644
--- a/llvm/test/Transforms/Scalarizer/basic.ll
+++ b/llvm/test/Transforms/Scalarizer/basic.ll
@@ -202,7 +202,7 @@ define void @f3(ptr %src, ptr %dst) {
   ret void
 }
 
-; Check that !tbaa.struct information is preserved.
+; Check that !tbaa.struct fields become the correct per-lane !tbaa tags.
 define void @f4(ptr %src, ptr %dst) {
 ; CHECK-LABEL: define void @f4(
 ; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]]) {
@@ -909,7 +909,7 @@ define <2 x i8> @test_copy_trunc_flags(<2 x i32> %src) {
 !2 = !{!15, !15, i64 0}
 !3 = !{ !3, !{!"llvm.loop.parallel_accesses", !13} }
 !4 = !{ float 4.0 }
-!5 = !{ i64 0, i64 8, null }
+!5 = !{i64 0, i64 4, !1, i64 4, i64 4, !2, i64 8, i64 4, !1, i64 12, i64 4, !2}
 !13 = distinct !{}
 !14 = !{!"set1", !0}
 !15 = !{!"set2", !0}
@@ -919,7 +919,7 @@ define <2 x i8> @test_copy_trunc_flags(<2 x i32> %src) {
 ; CHECK: [[META2]] = !{!"root"}
 ; CHECK: [[SET2_TBAA3]] = !{[[META4:![0-9]+]], [[META4]], i64 0}
 ; CHECK: [[META4]] = !{!"set2", [[META2]]}
-; CHECK: [[TBAA_STRUCT5]] = !{i64 0, i64 8, null}
+; CHECK: [[TBAA_STRUCT5]] = !{i64 0, i64 4, [[SET1_TBAA0]], i64 4, i64 4, [[SET2_TBAA3]], i64 8, i64 4, [[SET1_TBAA0]], i64 12, i64 4, [[SET2_TBAA3]]}
 ; CHECK: [[ACC_GRP6]] = distinct !{}
 ; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META8:![0-9]+]]}
 ; CHECK: [[META8]] = !{!"llvm.loop.parallel_accesses", [[ACC_GRP6]]}
diff --git a/llvm/test/Transforms/Scalarizer/memory-cache-hint.ll b/llvm/test/Transforms/Scalarizer/memory-cache-hint.ll
new file mode 100644
index 00000000000000..270b264e65961b
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/memory-cache-hint.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes='scalarizer<load-store>,dce' < %s | FileCheck %s
+
+define <2 x i32> @split_load(ptr %p) {
+; CHECK-LABEL: define <2 x i32> @split_load(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[V_I0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT:    [[P_I1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT:    [[V_I1:%.*]] = load i32, ptr [[P_I1]], align 4
+; CHECK-NEXT:    [[V_UPTO0:%.*]] = insertelement <2 x i32> poison, i32 [[V_I0]], i64 0
+; CHECK-NEXT:    [[V:%.*]] = insertelement <2 x i32> [[V_UPTO0]], i32 [[V_I1]], i64 1
+; CHECK-NEXT:    ret <2 x i32> [[V]]
+;
+  %v = load <2 x i32>, ptr %p, align 8, !mem.cache_hint !0, !range !2
+  ret <2 x i32> %v
+}
+
+define void @split_store(ptr %p, <2 x i32> %v) {
+; CHECK-LABEL: define void @split_store(
+; CHECK-SAME: ptr [[P:%.*]], <2 x i32> [[V:%.*]]) {
+; CHECK-NEXT:    [[V_I0:%.*]] = extractelement <2 x i32> [[V]], i64 0
+; CHECK-NEXT:    store i32 [[V_I0]], ptr [[P]], align 8
+; CHECK-NEXT:    [[V_I1:%.*]] = extractelement <2 x i32> [[V]], i64 1
+; CHECK-NEXT:    [[P_I1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT:    store i32 [[V_I1]], ptr [[P_I1]], align 4
+; CHECK-NEXT:    ret void
+;
+  store <2 x i32> %v, ptr %p, align 8, !mem.cache_hint !3
+  ret void
+}
+
+!0 = !{i32 0, !1}
+!1 = !{!"nvvm.l1_eviction", !"last"}
+!2 = !{i32 0, i32 100}
+!3 = !{i32 1, !1}
diff --git a/llvm/test/Transforms/Scalarizer/memory-metadata.ll b/llvm/test/Transforms/Scalarizer/memory-metadata.ll
new file mode 100644
index 00000000000000..cb6197983ec51c
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/memory-metadata.ll
@@ -0,0 +1,150 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes='scalarizer<load-store>,dce' < %s | FileCheck %s --implicit-check-not=!test.unknown
+; RUN: opt -S -passes='scalarizer<load-store;min-bits=64>,dce' < %s | FileCheck %s --check-prefix=PACKED --implicit-check-not=!test.unknown
+
+target datalayout = "e-p:64:64"
+
+; Exercise result facts and access annotations when splitting into scalars and
+; smaller vectors. Only the first fragment retains pointer identity.
+define <4 x i32> @integer_load(ptr %p) {
+; CHECK-LABEL: define <4 x i32> @integer_load(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[V_I0:%.*]] = load i32, ptr [[P]], align 16, !tbaa [[INT_TBAA0:![0-9]+]], !invariant.load [[META4:![0-9]+]], !alias.scope [[META5:![0-9]+]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8:![0-9]+]], !llvm.access.group [[META8]]
+; CHECK-NEXT:    [[P_I1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT:    [[V_I1:%.*]] = load i32, ptr [[P_I1]], align 4, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT:    [[P_I2:%.*]] = getelementptr i32, ptr [[P]], i32 2
+; CHECK-NEXT:    [[V_I2:%.*]] = load i32, ptr [[P_I2]], align 8, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT:    [[P_I3:%.*]] = getelementptr i32, ptr [[P]], i32 3
+; CHECK-NEXT:    [[V_I3:%.*]] = load i32, ptr [[P_I3]], align 4, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT:    [[V_UPTO0:%.*]] = insertelement <4 x i32> poison, i32 [[V_I0]], i64 0
+; CHECK-NEXT:    [[V_UPTO1:%.*]] = insertelement <4 x i32> [[V_UPTO0]], i32 [[V_I1]], i64 1
+; CHECK-NEXT:    [[V_UPTO2:%.*]] = insertelement <4 x i32> [[V_UPTO1]], i32 [[V_I2]], i64 2
+; CHECK-NEXT:    [[V:%.*]] = insertelement <4 x i32> [[V_UPTO2]], i32 [[V_I3]], i64 3
+; CHECK-NEXT:    ret <4 x i32> [[V]]
+;
+; PACKED-LABEL: define <4 x i32> @integer_load(
+; PACKED-SAME: ptr [[P:%.*]]) {
+; PACKED-NEXT:    [[V_I0:%.*]] = load <2 x i32>, ptr [[P]], align 16, !tbaa [[INT_TBAA0:![0-9]+]], !invariant.load [[META4:![0-9]+]], !alias.scope [[META5:![0-9]+]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8:![0-9]+]], !llvm.access.group [[META8]]
+; PACKED-NEXT:    [[P_I1:%.*]] = getelementptr <2 x i32>, ptr [[P]], i32 1
+; PACKED-NEXT:    [[V_I1:%.*]] = load <2 x i32>, ptr [[P_I1]], align 8, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i32> [[V_I0]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; PACKED-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[V_I1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; PACKED-NEXT:    [[V:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; PACKED-NEXT:    ret <4 x i32> [[V]]
+;
+  %v = load <4 x i32>, ptr %p, align 16, !tbaa !10, !range !9, !noundef !0, !invariant.load !0, !invariant.group !0, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+  ret <4 x i32> %v
+}
+
+define <4 x float> @floating_load(ptr %p) {
+; CHECK-LABEL: define <4 x float> @floating_load(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[V_I0:%.*]] = load float, ptr [[P]], align 16, !tbaa [[FLOAT_TBAA9:![0-9]+]], !fpmath [[META11:![0-9]+]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT:    [[P_I1:%.*]] = getelementptr float, ptr [[P]], i32 1
+; CHECK-NEXT:    [[V_I1:%.*]] = load float, ptr [[P_I1]], align 4, !tbaa [[FLOAT_TBAA9]], !fpmath [[META11]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT:    [[P_I2:%.*]] = getelementptr float, ptr [[P]], i32 2
+; CHECK-NEXT:    [[V_I2:%.*]] = load float, ptr [[P_I2]], align 8, !tbaa [[FLOAT_TBAA9]], !fpmath [[META11]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT:    [[P_I3:%.*]] = getelementptr float, ptr [[P]], i32 3
+; CHECK-NEXT:    [[V_I3:%.*]] = load float, ptr [[P_I3]], align 4, !tbaa [[FLOAT_TBAA9]], !fpmath [[META11]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT:    [[V_UPTO0:%.*]] = insertelement <4 x float> poison, float [[V_I0]], i64 0
+; CHECK-NEXT:    [[V_UPTO1:%.*]] = insertelement <4 x float> [[V_UPTO0]], float [[V_I1]], i64 1
+; CHECK-NEXT:    [[V_UPTO2:%.*]] = insertelement <4 x float> [[V_UPTO1]], float [[V_I2]], i64 2
+; CHECK-NEXT:    [[V:%.*]] = insertelement <4 x float> [[V_UPTO2]], float [[V_I3]], i64 3
+; CHECK-NEXT:    ret <4 x float> [[V]]
+;
+; PACKED-LABEL: define <4 x float> @floating_load(
+; PACKED-SAME: ptr [[P:%.*]]) {
+; PACKED-NEXT:    [[V_I0:%.*]] = load <2 x float>, ptr [[P]], align 16, !tbaa [[FLOAT_TBAA9:![0-9]+]], !fpmath [[META11:![0-9]+]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT:    [[P_I1:%.*]] = getelementptr <2 x float>, ptr [[P]], i32 1
+; PACKED-NEXT:    [[V_I1:%.*]] = load <2 x float>, ptr [[P_I1]], align 8, !tbaa [[FLOAT_TBAA9]], !fpmath [[META11]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[V_I0]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; PACKED-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[V_I1]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; PACKED-NEXT:    [[V:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; PACKED-NEXT:    ret <4 x float> [[V]]
+;
+  %v = load <4 x float>, ptr %p, align 16, !tbaa !20, !nofpclass !15, !fpmath !14, !noundef !0, !invariant.load !0, !invariant.group !0, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+  ret <4 x float> %v
+}
+
+define void @integer_store(ptr %p, <4 x i32> %v) {
+; CHECK-LABEL: define void @integer_store(
+; CHECK-SAME: ptr [[P:%.*]], <4 x i32> [[V:%.*]]) {
+; CHECK-NEXT:    [[V_I0:%.*]] = extractelement <4 x i32> [[V]], i64 0
+; CHECK-NEXT:    store i32 [[V_I0]], ptr [[P]], align 16, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT:    [[V_I1:%.*]] = extractelement <4 x i32> [[V]], i64 1
+; CHECK-NEXT:    [[P_I1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT:    store i32 [[V_I1]], ptr [[P_I1]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT:    [[V_I2:%.*]] = extractelement <4 x i32> [[V]], i64 2
+; CHECK-NEXT:    [[P_I2:%.*]] = getelementptr i32, ptr [[P]], i32 2
+; CHECK-NEXT:    store i32 [[V_I2]], ptr [[P_I2]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT:    [[V_I3:%.*]] = extractelement <4 x i32> [[V]], i64 3
+; CHECK-NEXT:    [[P_I3:%.*]] = getelementptr i32, ptr [[P]], i32 3
+; CHECK-NEXT:    store i32 [[V_I3]], ptr [[P_I3]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT:    ret void
+;
+; PACKED-LABEL: define void @integer_store(
+; PACKED-SAME: ptr [[P:%.*]], <4 x i32> [[V:%.*]]) {
+; PACKED-NEXT:    [[V_I0:%.*]] = shufflevector <4 x i32> [[V]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>
+; PACKED-NEXT:    store <2 x i32> [[V_I0]], ptr [[P]], align 16, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT:    [[V_I1:%.*]] = shufflevector <4 x i32> [[V]], <4 x i32> poison, <2 x i32> <i32 2, i32 3>
+; PACKED-NEXT:    [[P_I1:%.*]] = getelementptr <2 x i32>, ptr [[P]], i32 1
+; PACKED-NEXT:    store <2 x i32> [[V_I1]], ptr [[P_I1]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT:    ret void
+;
+  store <4 x i32> %v, ptr %p, align 16, !tbaa !10, !invariant.group !0, !mem.cache_hint !18, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+  ret void
+}
+
+!0 = !{}
+!1 = !{i32 1}
+!2 = !{!"preserved memory metadata"}
+!3 = !{!"test", !"relaxed"}
+!4 = !{i32 5, i32 6}
+!5 = distinct !{}
+!6 = !{!7}
+!7 = distinct !{!7, !8, !"scope"}
+!8 = distinct !{!8, i1 false, !"domain"}
+!9 = !{i32 0, i32 100}
+!10 = !{!11, !11, i64 0}
+!11 = !{!"int", !12, i64 0}
+!12 = !{!"omnipotent char", !13, i64 0}
+!13 = !{!"root"}
+!14 = !{float 2.5}
+!15 = !{i32 3}
+!16 = !{i32 0, !17}
+!17 = !{!"nvvm.l1_eviction", !"last"}
+!18 = !{i32 1, !17}
+!19 = !{i64 0, i64 4, !10, i64 4, i64 4, !20}
+!20 = !{!21, !21, i64 0}
+!21 = !{!"float", !12, i64 0}
+!22 = !{i32 0, !17, i32 1, !23}
+!23 = !{!"nvvm.l1_eviction", !"first"}
+!24 = !{!"branch_weights", i32 10}
+!25 = !{i64 8}
+;.
+; CHECK: [[INT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; CHECK: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
+; CHECK: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
+; CHECK: [[META3]] = !{!"root"}
+; CHECK: [[META4]] = !{}
+; CHECK: [[META5]] = !{[[META6:![0-9]+]]}
+; CHECK: [[META6]] = distinct !{[[META6]], [[META7:![0-9]+]], !"scope"}
+; CHECK: [[META7]] = distinct !{[[META7]], i1 false, !"domain"}
+; CHECK: [[META8]] = distinct !{}
+; CHECK: [[FLOAT_TBAA9]] = !{[[META10:![0-9]+]], [[META10]], i64 0}
+; CHECK: [[META10]] = !{!"float", [[META2]], i64 0}
+; CHECK: [[META11]] = !{float 2.500000e+00}
+;.
+; PACKED: [[INT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; PACKED: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
+; PACKED: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
+; PACKED: [[META3]] = !{!"root"}
+; PACKED: [[META4]] = !{}
+; PACKED: [[META5]] = !{[[META6:![0-9]+]]}
+; PACKED: [[META6]] = distinct !{[[META6]], [[META7:![0-9]+]], !"scope"}
+; PACKED: [[META7]] = distinct !{[[META7]], i1 false, !"domain"}
+; PACKED: [[META8]] = distinct !{}
+; PACKED: [[FLOAT_TBAA9]] = !{[[META10:![0-9]+]], [[META10]], i64 0}
+; PACKED: [[META10]] = !{!"float", [[META2]], i64 0}
+; PACKED: [[META11]] = !{float 2.500000e+00}
+;.
diff --git a/llvm/test/Transforms/VectorCombine/RISCV/memory-metadata.ll b/llvm/test/Transforms/VectorCombine/RISCV/memory-metadata.ll
new file mode 100644
index 00000000000000..4c29c342143cd8
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/RISCV/memory-metadata.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=vector-combine -mtriple=riscv64 -mattr=+v < %s | FileCheck %s
+
+define <8 x i16> @vp_bitcast_fixed(ptr %p, i32 %evl) {
+; CHECK-LABEL: define <8 x i16> @vp_bitcast_fixed(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[EVL]], 2
+; CHECK-NEXT:    [[R:%.*]] = call <8 x i16> @llvm.vp.load.v8i16.p0(ptr align 4 [[P]], <8 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT:    ret <8 x i16> [[R]]
+;
+  %v = call <4 x i32> @llvm.vp.load(ptr align 4 %p, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, i32 %evl), !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !range !11, !invariant.load !3
+  %r = bitcast <4 x i32> %v to <8 x i16>
+  ret <8 x i16> %r
+}
+
+define <vscale x 8 x i16> @vp_bitcast_scalable(ptr %p, i32 %evl) {
+; CHECK-LABEL: define <vscale x 8 x i16> @vp_bitcast_scalable(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[EVL]], 2
+; CHECK-NEXT:    [[R:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 4 [[P]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT:    ret <vscale x 8 x i16> [[R]]
+;
+  %v = call <vscale x 4 x i32> @llvm.vp.load(ptr align 4 %p, <vscale x 4 x i1> splat (i1 true), i32 %evl), !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !range !11, !invariant.load !3
+  %r = bitcast <vscale x 4 x i32> %v to <vscale x 8 x i16>
+  ret <vscale x 8 x i16> %r
+}
+
+
+!0 = !{i32 1}
+!1 = !{i32 0, !2}
+!2 = !{!"nvvm.l1_eviction", !"last"}
+!3 = !{}
+!4 = !{!"original access"}
+!5 = !{!"test", !"tag"}
+!6 = !{i32 5, i32 6}
+!7 = !{!8}
+!8 = distinct !{!8, !9, !"scope"}
+!9 = distinct !{!9, i1 false, !"domain"}
+!10 = distinct !{}
+!11 = !{i32 0, i32 100}
+!12 = !{i32 3}
+!13 = !{float 2.5}
+!14 = !{i64 0, i64 4, !15, i64 4, i64 4, !15, i64 8, i64 4, !15, i64 12, i64 4, !15}
+!15 = !{!16, !16, i64 0}
+!16 = !{!"omnipotent char", !17, i64 0}
+!17 = !{!"root"}
+!18 = !{i32 1, !2}
diff --git a/llvm/test/Transforms/VectorCombine/X86/memory-metadata.ll b/llvm/test/Transforms/VectorCombine/X86/memory-metadata.ll
new file mode 100644
index 00000000000000..e9f73bbdff4e2a
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/X86/memory-metadata.ll
@@ -0,0 +1,72 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=vector-combine -mtriple=x86_64-unknown-linux-gnu -mcpu=x86-64 < %s | FileCheck %s
+
+define <4 x float> @widen_scalar(ptr align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: define <4 x float> @widen_scalar(
+; CHECK-SAME: ptr align 16 dereferenceable(16) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[R:%.*]] = load <4 x float>, ptr [[P]], align 16
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %v = load float, ptr %p, align 4, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !nofpclass !12, !fpmath !13, !noundef !3, !invariant.load !3, !invariant.group !3, !tbaa !15
+  %r = insertelement <4 x float> poison, float %v, i32 0
+  ret <4 x float> %r
+}
+
+define <4 x float> @widen_shifted_scalar(ptr align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: define <4 x float> @widen_shifted_scalar(
+; CHECK-SAME: ptr align 16 dereferenceable(16) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[P]], align 16
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 3, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %q = getelementptr inbounds float, ptr %p, i64 3
+  %v = load float, ptr %q, align 4, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !nofpclass !12, !fpmath !13, !noundef !3, !invariant.load !3, !invariant.group !3, !tbaa !15
+  %r = insertelement <4 x float> poison, float %v, i32 0
+  ret <4 x float> %r
+}
+
+define <4 x i32> @widen_subvector(ptr align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: define <4 x i32> @widen_subvector(
+; CHECK-SAME: ptr align 16 dereferenceable(16) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[R:%.*]] = load <4 x i32>, ptr [[P]], align 16
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %v = load <2 x i32>, ptr %p, align 4, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !range !11, !noundef !3, !invariant.load !3, !invariant.group !3, !tbaa !15
+  %r = shufflevector <2 x i32> %v, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+  ret <4 x i32> %r
+}
+
+
+; The load-insert fold can also shrink a load. Facts about all original
+; elements remain valid for this subset.
+define <4 x i32> @shrink_load_insert(ptr align 16 dereferenceable(32) %p) {
+; CHECK-LABEL: define <4 x i32> @shrink_load_insert(
+; CHECK-SAME: ptr align 16 dereferenceable(32) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[R:%.*]] = load <4 x i32>, ptr [[P]], align 16
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %v = load <8 x i32>, ptr %p, align 16, !nontemporal !0, !mem.cache_hint !1, !range !11, !noundef !3, !invariant.load !3, !alias.scope !7
+  %e = extractelement <8 x i32> %v, i32 0
+  %r = insertelement <4 x i32> poison, i32 %e, i32 0
+  ret <4 x i32> %r
+}
+
+!0 = !{i32 1}
+!1 = !{i32 0, !2}
+!2 = !{!"nvvm.l1_eviction", !"last"}
+!3 = !{}
+!4 = !{!"original access"}
+!5 = !{!"test", !"tag"}
+!6 = !{i32 5, i32 6}
+!7 = !{!8}
+!8 = distinct !{!8, !9, !"scope"}
+!9 = distinct !{!9, i1 false, !"domain"}
+!10 = distinct !{}
+!11 = !{i32 0, i32 100}
+!12 = !{i32 3}
+!13 = !{float 2.5}
+!14 = !{i64 0, i64 4, !15, i64 4, i64 4, !15, i64 8, i64 4, !15, i64 12, i64 4, !15}
+!15 = !{!16, !16, i64 0}
+!16 = !{!"omnipotent char", !17, i64 0}
+!17 = !{!"root"}
+!18 = !{i32 1, !2}
diff --git a/llvm/test/Transforms/VectorCombine/memory-metadata.ll b/llvm/test/Transforms/VectorCombine/memory-metadata.ll
new file mode 100644
index 00000000000000..d01dffceeb940a
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/memory-metadata.ll
@@ -0,0 +1,195 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=vector-combine -data-layout=e-p:64:64 < %s | FileCheck %s
+; RUN: opt -S -passes=vector-combine -data-layout=E-p:64:64 < %s | FileCheck %s
+
+define i32 @extract_zero(ptr %p) {
+; CHECK-LABEL: define i32 @extract_zero(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = load i32, ptr [[P]], align 16, !tbaa [[CHAR_TBAA0:![0-9]+]], !alias.scope [[META3:![0-9]+]], !noalias [[META3]], !noalias.addrspace [[META6:![0-9]+]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %v = load <4 x i32>, ptr %p, align 16, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !range !11, !noundef !3, !invariant.load !3, !invariant.group !3, !tbaa.struct !14
+  %r = extractelement <4 x i32> %v, i32 0
+  ret i32 %r
+}
+
+define i32 @extract_nonzero(ptr %p) {
+; CHECK-LABEL: define i32 @extract_nonzero(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 2
+; CHECK-NEXT:    [[R:%.*]] = load i32, ptr [[TMP1]], align 8, !tbaa [[CHAR_TBAA0]], !alias.scope [[META3]], !noalias [[META3]], !noalias.addrspace [[META6]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %v = load <4 x i32>, ptr %p, align 16, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !range !11, !noundef !3, !invariant.load !3, !invariant.group !3, !tbaa.struct !14
+  %r = extractelement <4 x i32> %v, i32 2
+  ret i32 %r
+}
+
+define i32 @extract_dynamic(ptr %p, i32 %index) {
+; CHECK-LABEL: define i32 @extract_dynamic(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[INDEX:%.*]]) {
+; CHECK-NEXT:    [[INDEX_FROZEN:%.*]] = freeze i32 [[INDEX]]
+; CHECK-NEXT:    [[IDX:%.*]] = and i32 [[INDEX_FROZEN]], 3
+; CHECK-NEXT:    [[IDX_GEPIDX:%.*]] = zext i32 [[IDX]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 [[IDX_GEPIDX]]
+; CHECK-NEXT:    [[R:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %idx = and i32 %index, 3
+  %v = load <4 x i32>, ptr %p, align 16, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !range !11, !noundef !3, !invariant.load !3, !invariant.group !3, !tbaa.struct !14, !tbaa !15
+  %r = extractelement <4 x i32> %v, i32 %idx
+  ret i32 %r
+}
+
+define float @extract_float(ptr %p) {
+; CHECK-LABEL: define float @extract_float(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <4 x float>, ptr [[P]], i64 0, i64 1
+; CHECK-NEXT:    [[R:%.*]] = load float, ptr [[TMP1]], align 4, !alias.scope [[META3]], !noalias [[META3]], !noalias.addrspace [[META6]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %v = load <4 x float>, ptr %p, align 16, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !nofpclass !12, !fpmath !13, !noundef !3, !invariant.load !3, !invariant.group !3
+  %r = extractelement <4 x float> %v, i32 1
+  ret float %r
+}
+
+define <4 x i32> @shrink_integer(ptr %p) {
+; CHECK-LABEL: define <4 x i32> @shrink_integer(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 32, !tbaa [[CHAR_TBAA0]], !range [[RNG7:![0-9]+]], !invariant.load [[META8:![0-9]+]], !alias.scope [[META3]], !noalias [[META3]], !nontemporal [[META9:![0-9]+]], !invariant.group [[META8]], !llvm.access.group [[ACC_GRP10:![0-9]+]], !noundef [[META8]], !annotation [[META11:![0-9]+]], !nosanitize [[META8]], !mmra [[META12:![0-9]+]], !noalias.addrspace [[META6]], !mem.cache_hint [[META13:![0-9]+]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %v = load <8 x i32>, ptr %p, align 32, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !range !11, !noundef !3, !invariant.load !3, !invariant.group !3, !tbaa !15
+  %r = shufflevector <8 x i32> %v, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  ret <4 x i32> %r
+}
+
+define <4 x float> @shrink_float(ptr %p) {
+; CHECK-LABEL: define <4 x float> @shrink_float(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[P]], align 32, !fpmath [[META15:![0-9]+]], !invariant.load [[META8]], !alias.scope [[META3]], !noalias [[META3]], !nontemporal [[META9]], !invariant.group [[META8]], !llvm.access.group [[ACC_GRP10]], !noundef [[META8]], !annotation [[META11]], !nosanitize [[META8]], !mmra [[META12]], !noalias.addrspace [[META6]], !nofpclass [[META16:![0-9]+]], !mem.cache_hint [[META13]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %v = load <8 x float>, ptr %p, align 32, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !nofpclass !12, !fpmath !13, !noundef !3, !invariant.load !3, !invariant.group !3
+  %r = shufflevector <8 x float> %v, <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  ret <4 x float> %r
+}
+
+define void @store_zero(ptr %p, i8 %x, i32 %index) {
+; CHECK-LABEL: define void @store_zero(
+; CHECK-SAME: ptr [[P:%.*]], i8 [[X:%.*]], i32 [[INDEX:%.*]]) {
+; CHECK-NEXT:    store i8 [[X]], ptr [[P]], align 16, !tbaa.struct [[TBAA_STRUCT17:![0-9]+]], !alias.scope [[META3]], !noalias [[META3]], !nontemporal [[META9]], !llvm.access.group [[ACC_GRP10]], !annotation [[META11]], !nosanitize [[META8]], !mmra [[META12]], !noalias.addrspace [[META6]], !mem.cache_hint [[META18:![0-9]+]]
+; CHECK-NEXT:    ret void
+;
+  %v = load <16 x i8>, ptr %p, align 16
+  %i = insertelement <16 x i8> %v, i8 %x, i32 0
+  store <16 x i8> %i, ptr %p, align 16, !nontemporal !0, !mem.cache_hint !18, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !invariant.group !3, !tbaa.struct !14
+  ret void
+}
+
+define void @store_nonzero(ptr %p, i8 %x, i32 %index) {
+; CHECK-LABEL: define void @store_nonzero(
+; CHECK-SAME: ptr [[P:%.*]], i8 [[X:%.*]], i32 [[INDEX:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 3
+; CHECK-NEXT:    store i8 [[X]], ptr [[TMP1]], align 1, !tbaa.struct [[TBAA_STRUCT17]], !alias.scope [[META3]], !noalias [[META3]], !nontemporal [[META9]], !llvm.access.group [[ACC_GRP10]], !annotation [[META11]], !nosanitize [[META8]], !mmra [[META12]], !noalias.addrspace [[META6]], !mem.cache_hint [[META18]]
+; CHECK-NEXT:    ret void
+;
+  %v = load <16 x i8>, ptr %p, align 16
+  %i = insertelement <16 x i8> %v, i8 %x, i32 3
+  store <16 x i8> %i, ptr %p, align 16, !nontemporal !0, !mem.cache_hint !18, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !invariant.group !3, !tbaa.struct !14
+  ret void
+}
+
+define void @store_dynamic(ptr %p, i8 %x, i32 %index) {
+; CHECK-LABEL: define void @store_dynamic(
+; CHECK-SAME: ptr [[P:%.*]], i8 [[X:%.*]], i32 [[INDEX:%.*]]) {
+; CHECK-NEXT:    [[INDEX_FROZEN:%.*]] = freeze i32 [[INDEX]]
+; CHECK-NEXT:    [[IDX:%.*]] = and i32 [[INDEX_FROZEN]], 15
+; CHECK-NEXT:    [[IDX_GEPIDX:%.*]] = zext i32 [[IDX]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 [[IDX_GEPIDX]]
+; CHECK-NEXT:    store i8 [[X]], ptr [[TMP1]], align 1, !tbaa.struct [[TBAA_STRUCT17]], !alias.scope [[META3]], !noalias [[META3]], !nontemporal [[META9]], !llvm.access.group [[ACC_GRP10]], !annotation [[META11]], !nosanitize [[META8]], !mmra [[META12]], !noalias.addrspace [[META6]], !mem.cache_hint [[META18]]
+; CHECK-NEXT:    ret void
+;
+  %idx = and i32 %index, 15
+  %v = load <16 x i8>, ptr %p, align 16
+  %i = insertelement <16 x i8> %v, i8 %x, i32 %idx
+  store <16 x i8> %i, ptr %p, align 16, !nontemporal !0, !mem.cache_hint !18, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !invariant.group !3, !tbaa.struct !14
+  ret void
+}
+
+
+; A known offset selects the corresponding field's alias tag.
+define i32 @extract_struct_field(ptr %p) {
+; CHECK-LABEL: define i32 @extract_struct_field(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 1
+; CHECK-NEXT:    [[R:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[INT_TBAA19:![0-9]+]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %v = load <4 x i32>, ptr %p, align 16, !tbaa.struct !21
+  %r = extractelement <4 x i32> %v, i32 1
+  ret i32 %r
+}
+
+define void @store_struct_field(ptr %p, i32 %x) {
+; CHECK-LABEL: define void @store_struct_field(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 1
+; CHECK-NEXT:    store i32 [[X]], ptr [[TMP1]], align 4, !tbaa.struct [[TBAA_STRUCT21:![0-9]+]]
+; CHECK-NEXT:    ret void
+;
+  %v = load <4 x i32>, ptr %p, align 16
+  %i = insertelement <4 x i32> %v, i32 %x, i32 1
+  store <4 x i32> %i, ptr %p, align 16, !tbaa.struct !21
+  ret void
+}
+
+!0 = !{i32 1}
+!1 = !{i32 0, !2}
+!2 = !{!"nvvm.l1_eviction", !"last"}
+!3 = !{}
+!4 = !{!"original access"}
+!5 = !{!"test", !"tag"}
+!6 = !{i32 5, i32 6}
+!7 = !{!8}
+!8 = distinct !{!8, !9, !"scope"}
+!9 = distinct !{!9, i1 false, !"domain"}
+!10 = distinct !{}
+!11 = !{i32 0, i32 100}
+!12 = !{i32 3}
+!13 = !{float 2.5}
+!14 = !{i64 0, i64 4, !15, i64 4, i64 4, !15, i64 8, i64 4, !15, i64 12, i64 4, !15}
+!15 = !{!16, !16, i64 0}
+!16 = !{!"omnipotent char", !17, i64 0}
+!17 = !{!"root"}
+!18 = !{i32 1, !2}
+
+!19 = !{!"int", !16, i64 0}
+!20 = !{!19, !19, i64 0}
+!21 = !{i64 0, i64 4, !15, i64 4, i64 4, !20, i64 8, i64 4, !15, i64 12, i64 4, !15}
+;.
+; CHECK: [[CHAR_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; CHECK: [[META1]] = !{!"omnipotent char", [[META2:![0-9]+]], i64 0}
+; CHECK: [[META2]] = !{!"root"}
+; CHECK: [[META3]] = !{[[META4:![0-9]+]]}
+; CHECK: [[META4]] = distinct !{[[META4]], [[META5:![0-9]+]], !"scope"}
+; CHECK: [[META5]] = distinct !{[[META5]], i1 false, !"domain"}
+; CHECK: [[META6]] = !{i32 5, i32 6}
+; CHECK: [[RNG7]] = !{i32 0, i32 100}
+; CHECK: [[META8]] = !{}
+; CHECK: [[META9]] = !{i32 1}
+; CHECK: [[ACC_GRP10]] = distinct !{}
+; CHECK: [[META11]] = !{!"original access"}
+; CHECK: [[META12]] = !{!"test", !"tag"}
+; CHECK: [[META13]] = !{i32 0, [[META14:![0-9]+]]}
+; CHECK: [[META14]] = !{!"nvvm.l1_eviction", !"last"}
+; CHECK: [[META15]] = !{float 2.500000e+00}
+; CHECK: [[META16]] = !{i32 3}
+; CHECK: [[TBAA_STRUCT17]] = !{i64 0, i64 4, [[CHAR_TBAA0]], i64 4, i64 4, [[CHAR_TBAA0]], i64 8, i64 4, [[CHAR_TBAA0]], i64 12, i64 4, [[CHAR_TBAA0]]}
+; CHECK: [[META18]] = !{i32 1, [[META14]]}
+; CHECK: [[INT_TBAA19]] = !{[[META20:![0-9]+]], [[META20]], i64 0}
+; CHECK: [[META20]] = !{!"int", [[META1]], i64 0}
+; CHECK: [[TBAA_STRUCT21]] = !{i64 0, i64 4, [[CHAR_TBAA0]], i64 4, i64 4, [[INT_TBAA19]], i64 8, i64 4, [[CHAR_TBAA0]], i64 12, i64 4, [[CHAR_TBAA0]]}
+;.

>From 46ee822a3976e11535e7150116ea49352984a692 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Fri, 13 Nov 2026 10:35:37 +0000
Subject: [PATCH 02/14] [Analysis] Centralize memory metadata utilities

---
 llvm/include/llvm/Analysis/MetadataUtils.h    |  86 +++
 llvm/include/llvm/Analysis/VectorUtils.h      |  29 +-
 llvm/include/llvm/Transforms/Utils/Local.h    |  33 +-
 llvm/lib/Analysis/CMakeLists.txt              |   1 +
 llvm/lib/Analysis/MetadataUtils.cpp           | 644 ++++++++++++++++++
 llvm/lib/Analysis/VectorUtils.cpp             | 163 +----
 .../AMDGPU/AMDGPULowerBufferFatPointers.cpp   |   5 +-
 .../InstCombine/InstCombineCalls.cpp          |  34 +-
 .../InstCombineLoadStoreAlloca.cpp            |  74 +-
 llvm/lib/Transforms/Scalar/SROA.cpp           | 115 +---
 .../Scalar/ScalarizeMaskedMemIntrin.cpp       | 132 +---
 llvm/lib/Transforms/Scalar/Scalarizer.cpp     |  31 +-
 llvm/lib/Transforms/Utils/Local.cpp           | 306 +--------
 .../Vectorize/LoadStoreVectorizer.cpp         |   1 +
 .../Transforms/Vectorize/SLPVectorizer.cpp    |   1 +
 llvm/lib/Transforms/Vectorize/VPlan.h         |   1 +
 .../Transforms/Vectorize/VectorCombine.cpp    |  36 +-
 .../InstCombine/memory-cache-hint.ll          |  19 +-
 .../Transforms/InstCombine/memory-metadata.ll |  49 +-
 .../InstCombine/storemerge-metadata.ll        |  18 +-
 llvm/test/Transforms/SROA/memory-metadata.ll  |  55 +-
 .../Scalarizer/basic-inseltpoison.ll          |  51 +-
 llvm/test/Transforms/Scalarizer/basic.ll      |  51 +-
 .../Scalarizer/memory-cache-hint.ll           |  14 +-
 .../Transforms/Scalarizer/memory-metadata.ll  |  86 ++-
 .../VectorCombine/RISCV/memory-metadata.ll    |  17 +-
 .../VectorCombine/X86/memory-metadata.ll      |  17 +-
 .../VectorCombine/load-bitcast-metadata.ll    |  85 +++
 .../VectorCombine/memory-metadata.ll          |  50 +-
 llvm/unittests/Analysis/CMakeLists.txt        |   1 +
 llvm/unittests/Analysis/MetadataUtilsTest.cpp | 390 +++++++++++
 31 files changed, 1572 insertions(+), 1023 deletions(-)
 create mode 100644 llvm/include/llvm/Analysis/MetadataUtils.h
 create mode 100644 llvm/lib/Analysis/MetadataUtils.cpp
 create mode 100644 llvm/test/Transforms/VectorCombine/load-bitcast-metadata.ll
 create mode 100644 llvm/unittests/Analysis/MetadataUtilsTest.cpp

diff --git a/llvm/include/llvm/Analysis/MetadataUtils.h b/llvm/include/llvm/Analysis/MetadataUtils.h
new file mode 100644
index 00000000000000..bfaf8bd6912e8f
--- /dev/null
+++ b/llvm/include/llvm/Analysis/MetadataUtils.h
@@ -0,0 +1,86 @@
+//===- MetadataUtils.h - Metadata utilities ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ANALYSIS_METADATAUTILS_H
+#define LLVM_ANALYSIS_METADATAUTILS_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/Compiler.h"
+#include <cstdint>
+#include <optional>
+#include <utility>
+
+namespace llvm {
+class Instruction;
+class MDNode;
+class Value;
+
+/// Copy metadata valid for a replacement memory access. The destination must
+/// access the same bytes, a subset, or an expanded range in the same object.
+/// ByteOffset is the destination's offset within the source access. A missing
+/// offset requires the caller to establish that the new access is a subset.
+/// Access types and pointer operands are inferred for loads, stores, and
+/// masked/VP memory intrinsics. For a source memory intrinsic, the source or
+/// destination pointer is selected to match the new read or write. The caller
+/// must establish range containment when the intrinsic's length is unknown.
+/// Pointer and value facts are retained only when valid for the destination;
+/// alias tags are adjusted for its range. Masks and execution conditions are
+/// not inspected.
+LLVM_ABI void
+copyMetadataForMemoryAccess(Instruction &Dest, const Instruction &Source,
+                            std::optional<int64_t> ByteOffset = 0);
+
+/// Copy metadata valid after splitting an instruction into smaller elements.
+/// ByteOffset describes the selected memory fragment, if applicable.
+LLVM_ABI void copyMetadataForScalarizedInstruction(
+    Instruction &Dest, const Instruction &Source,
+    std::optional<int64_t> ByteOffset = std::nullopt);
+
+/// Combine the metadata of two instructions so that K can replace J. This
+/// specifically handles the case of CSE-like transformations. Some
+/// metadata can only be kept if K dominates J. For this to be correct,
+/// K cannot be hoisted.
+///
+/// Unknown metadata is removed.
+LLVM_ABI void combineMetadataForCSE(Instruction *K, const Instruction *J,
+                                    bool DoesKMove);
+
+/// Combine metadata of two instructions, where instruction J is a memory
+/// access that has been merged into K. This will intersect alias-analysis
+/// metadata, while preserving other known metadata.
+LLVM_ABI void combineAAMetadata(Instruction *K, const Instruction *J);
+
+/// Compute the union of two access-group lists.
+///
+/// If the list contains just one access group, it is returned directly. If the
+/// list is empty, returns nullptr.
+LLVM_ABI MDNode *uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2);
+
+/// Compute the access-group list of access groups that @p Inst1 and @p Inst2
+/// are both in. If either instruction does not access memory at all, it is
+/// considered to be in every list.
+///
+/// If the list contains just one access group, it is returned directly. If the
+/// list is empty, returns nullptr.
+LLVM_ABI MDNode *intersectAccessGroups(const Instruction *Inst1,
+                                       const Instruction *Inst2);
+
+/// Collect metadata that can be preserved when vectorizing an instruction.
+LLVM_ABI void getMetadataToPropagate(
+    Instruction *Inst,
+    SmallVectorImpl<std::pair<unsigned, MDNode *>> &Metadata);
+
+/// Propagate metadata that can be preserved after vectorization from \p VL to
+/// \p I, retaining only information valid for all sources and the destination.
+/// Return \p I.
+LLVM_ABI Instruction *propagateMetadata(Instruction *I, ArrayRef<Value *> VL);
+
+} // namespace llvm
+
+#endif // LLVM_ANALYSIS_METADATAUTILS_H
diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h
index 3b0642b1ffbffe..6a48b007a7f4f3 100644
--- a/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/llvm/include/llvm/Analysis/VectorUtils.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/LoopAccessAnalysis.h"
+#include "llvm/Analysis/MetadataUtils.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/VFABIDemangler.h"
 #include "llvm/IR/VectorTypeUtils.h"
@@ -388,34 +389,6 @@ LLVM_ABI MapVector<Instruction *, uint64_t>
 computeMinimumValueSizes(ArrayRef<BasicBlock *> Blocks, DemandedBits &DB,
                          const TargetTransformInfo *TTI = nullptr);
 
-/// Compute the union of two access-group lists.
-///
-/// If the list contains just one access group, it is returned directly. If the
-/// list is empty, returns nullptr.
-LLVM_ABI MDNode *uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2);
-
-/// Compute the access-group list of access groups that @p Inst1 and @p Inst2
-/// are both in. If either instruction does not access memory at all, it is
-/// considered to be in every list.
-///
-/// If the list contains just one access group, it is returned directly. If the
-/// list is empty, returns nullptr.
-LLVM_ABI MDNode *intersectAccessGroups(const Instruction *Inst1,
-                                       const Instruction *Inst2);
-
-/// Add metadata from \p Inst to \p Metadata, if it can be preserved after
-/// vectorization. It can be preserved after vectorization if the kind is one of
-/// [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath, MD_nontemporal,
-/// MD_access_group, MD_mmra].
-LLVM_ABI void getMetadataToPropagate(
-    Instruction *Inst,
-    SmallVectorImpl<std::pair<unsigned, MDNode *>> &Metadata);
-
-/// Propagate metadata that can be preserved after vectorization from \p VL to
-/// \p I, retaining only information valid for all sources and the destination.
-/// Return \p I.
-LLVM_ABI Instruction *propagateMetadata(Instruction *I, ArrayRef<Value *> VL);
-
 /// Create a mask that filters the members of an interleave group where there
 /// are gaps.
 ///
diff --git a/llvm/include/llvm/Transforms/Utils/Local.h b/llvm/include/llvm/Transforms/Utils/Local.h
index 98b8ddd992282e..4991332881debd 100644
--- a/llvm/include/llvm/Transforms/Utils/Local.h
+++ b/llvm/include/llvm/Transforms/Utils/Local.h
@@ -15,6 +15,7 @@
 #define LLVM_TRANSFORMS_UTILS_LOCAL_H
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/Analysis/MetadataUtils.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compiler.h"
@@ -413,24 +414,6 @@ LLVM_ABI bool removeUnreachableBlocks(Function &F,
                                       MemorySSAUpdater *MSSAU = nullptr,
                                       bool FoldInstsToUnreachable = true);
 
-/// Combine the metadata of two instructions so that K can replace J. This
-/// specifically handles the case of CSE-like transformations. Some
-/// metadata can only be kept if K dominates J. For this to be correct,
-/// K cannot be hoisted.
-///
-/// Unknown metadata is removed.
-LLVM_ABI void combineMetadataForCSE(Instruction *K, const Instruction *J,
-                                    bool DoesKMove);
-
-/// Combine metadata of two instructions, where instruction J is a memory
-/// access that has been merged into K. This will intersect alias-analysis
-/// metadata, while preserving other known metadata.
-LLVM_ABI void combineAAMetadata(Instruction *K, const Instruction *J);
-
-/// Copy the metadata from the source instruction to the destination (the
-/// replacement for the source instruction).
-LLVM_ABI void copyMetadataForLoad(LoadInst &Dest, const LoadInst &Source);
-
 /// Patch the replacement so that it is not more restrictive than the value
 /// being replaced. Unless DoesKMove is set, it assumes that the replacement
 /// does not get moved from its original position.
@@ -486,20 +469,6 @@ LLVM_ABI unsigned replaceDominatedUsesWithIf(
 LLVM_ABI bool callsGCLeafFunction(const CallBase *Call,
                                   const TargetLibraryInfo &TLI);
 
-/// Copy a nonnull metadata node to a new load instruction.
-///
-/// This handles mapping it to range metadata if the new load is an integer
-/// load instead of a pointer load.
-LLVM_ABI void copyNonnullMetadata(const LoadInst &OldLI, MDNode *N,
-                                  LoadInst &NewLI);
-
-/// Copy a range metadata node to a new load instruction.
-///
-/// This handles mapping it to nonnull metadata if the new load is a pointer
-/// load instead of an integer load and the range doesn't cover null.
-LLVM_ABI void copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI,
-                                MDNode *N, LoadInst &NewLI);
-
 /// Remove the debug intrinsic instructions for the given instruction.
 LLVM_ABI void dropDebugUsers(Instruction &I);
 
diff --git a/llvm/lib/Analysis/CMakeLists.txt b/llvm/lib/Analysis/CMakeLists.txt
index 9c89cb4887d71f..301f517588a796 100644
--- a/llvm/lib/Analysis/CMakeLists.txt
+++ b/llvm/lib/Analysis/CMakeLists.txt
@@ -139,6 +139,7 @@ add_llvm_component_library(LLVMAnalysis
   MemoryProfileInfo.cpp
   MemorySSA.cpp
   MemorySSAUpdater.cpp
+  MetadataUtils.cpp
   ModelUnderTrainingRunner.cpp
   ModuleDebugInfoPrinter.cpp
   ModuleSummaryAnalysis.cpp
diff --git a/llvm/lib/Analysis/MetadataUtils.cpp b/llvm/lib/Analysis/MetadataUtils.cpp
new file mode 100644
index 00000000000000..936469fb58a149
--- /dev/null
+++ b/llvm/lib/Analysis/MetadataUtils.cpp
@@ -0,0 +1,644 @@
+//===- MetadataUtils.cpp - Metadata utilities --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/MetadataUtils.h"
+#include "llvm/ADT/APInt.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/IR/ConstantRange.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/MDBuilder.h"
+#include "llvm/IR/MemCacheHint.h"
+#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
+#include "llvm/IR/Metadata.h"
+#include "llvm/IR/Operator.h"
+#include "llvm/IR/ProfDataUtils.h"
+
+using namespace llvm;
+
+namespace {
+struct MemoryAccessInfo {
+  Type *Ty;
+  unsigned PointerOperand;
+  bool IsRead;
+};
+
+std::optional<MemoryAccessInfo>
+getMemoryAccessInfo(const Instruction &I,
+                    std::optional<bool> IsRead = std::nullopt) {
+  if (const auto *LI = dyn_cast<LoadInst>(&I))
+    return MemoryAccessInfo{LI->getType(), 0, true};
+  if (const auto *SI = dyn_cast<StoreInst>(&I))
+    return MemoryAccessInfo{SI->getValueOperand()->getType(), 1, false};
+  if (const auto *MI = dyn_cast<AnyMemIntrinsic>(&I)) {
+    if (!IsRead || (*IsRead && !isa<AnyMemTransferInst>(MI)))
+      return std::nullopt;
+    return MemoryAccessInfo{nullptr, *IsRead ? 1u : 0u, *IsRead};
+  }
+  const auto *II = dyn_cast<IntrinsicInst>(&I);
+  if (!II)
+    return std::nullopt;
+  if (const auto *VP = dyn_cast<VPIntrinsic>(II)) {
+    auto Ptr = VPIntrinsic::getMemoryPointerParamPos(VP->getIntrinsicID());
+    if (!Ptr)
+      return std::nullopt;
+    if (auto Data = VPIntrinsic::getMemoryDataParamPos(VP->getIntrinsicID()))
+      return MemoryAccessInfo{VP->getArgOperand(*Data)->getType(), *Ptr, false};
+    Type *Ty = VP->getType();
+    if (VP->getIntrinsicID() == Intrinsic::vp_load_ff)
+      Ty = Ty->getStructElementType(0);
+    return MemoryAccessInfo{Ty, *Ptr, true};
+  }
+  switch (II->getIntrinsicID()) {
+  case Intrinsic::experimental_vector_histogram_add:
+  case Intrinsic::experimental_vector_histogram_uadd_sat:
+  case Intrinsic::experimental_vector_histogram_umin:
+  case Intrinsic::experimental_vector_histogram_umax:
+    if (IsRead)
+      return MemoryAccessInfo{II->getArgOperand(1)->getType(), 0, *IsRead};
+    return std::nullopt;
+  case Intrinsic::masked_load:
+  case Intrinsic::masked_gather:
+  case Intrinsic::masked_expandload:
+    return MemoryAccessInfo{II->getType(), 0, true};
+  case Intrinsic::masked_store:
+  case Intrinsic::masked_scatter:
+  case Intrinsic::masked_compressstore:
+    return MemoryAccessInfo{II->getArgOperand(0)->getType(), 1, false};
+  default:
+    return std::nullopt;
+  }
+}
+
+const Value *getMemoryPointer(const Instruction &I, unsigned Index) {
+  if (const auto *CB = dyn_cast<CallBase>(&I))
+    return CB->getArgOperand(Index);
+  return I.getOperand(Index);
+}
+// invariant.group ignores bitcasts and zero-offset GEPs, not address-space
+// casts.
+const Value *getInvariantGroupPointer(const Value *V) {
+  while (true) {
+    if (const auto *Cast = dyn_cast<BitCastOperator>(V))
+      V = Cast->getOperand(0);
+    else if (const auto *GEP = dyn_cast<GEPOperator>(V);
+             GEP && GEP->hasAllZeroIndices())
+      V = GEP->getPointerOperand();
+    else
+      return V;
+  }
+}
+
+void copyNonnullMetadata(const LoadInst &OldLI, MDNode *N, LoadInst &NewLI) {
+  auto *NewTy = NewLI.getType();
+
+  // This only directly applies if the new type is also a pointer.
+  if (NewTy->isPointerTy()) {
+    NewLI.setMetadata(LLVMContext::MD_nonnull, N);
+    return;
+  }
+
+  // The only other translation we can do is to integral loads with !range
+  // metadata.
+  if (!NewTy->isIntegerTy())
+    return;
+
+  MDBuilder MDB(NewLI.getContext());
+  const Value *Ptr = OldLI.getPointerOperand();
+  auto *ITy = cast<IntegerType>(NewTy);
+  auto *NullInt = ConstantExpr::getPtrToInt(
+      ConstantPointerNull::get(cast<PointerType>(Ptr->getType())), ITy);
+  auto *NonNullInt = ConstantExpr::getAdd(NullInt, ConstantInt::get(ITy, 1));
+  NewLI.setMetadata(LLVMContext::MD_range,
+                    MDB.createRange(NonNullInt, NullInt));
+}
+
+void copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI, MDNode *N,
+                       LoadInst &NewLI) {
+  auto *NewTy = NewLI.getType();
+  // Simply copy the metadata if the type did not change.
+  if (NewTy == OldLI.getType()) {
+    NewLI.setMetadata(LLVMContext::MD_range, N);
+    return;
+  }
+
+  // Give up unless it is converted to a pointer where there is a single very
+  // valuable mapping we can do reliably.
+  // FIXME: It would be nice to propagate this in more ways, but the type
+  // conversions make it hard.
+  if (!NewTy->isPointerTy())
+    return;
+
+  unsigned BitWidth = DL.getPointerTypeSizeInBits(NewTy);
+  if (BitWidth == OldLI.getType()->getScalarSizeInBits() &&
+      !getConstantRangeFromMetadata(*N).contains(APInt(BitWidth, 0))) {
+    MDNode *NN = MDNode::get(OldLI.getContext(), {});
+    NewLI.setMetadata(LLVMContext::MD_nonnull, NN);
+  }
+}
+} // namespace
+
+void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
+                                       const Instruction &Source,
+                                       std::optional<int64_t> ByteOffset) {
+  auto Dst = getMemoryAccessInfo(Dest);
+  auto Src = getMemoryAccessInfo(Source, Dst ? std::optional<bool>(Dst->IsRead)
+                                             : std::nullopt);
+  assert(Src && Dst && Src->IsRead == Dst->IsRead &&
+         "expected corresponding memory reads or writes");
+  const DataLayout &DL = Source.getDataLayout();
+  std::optional<TypeSize> OldSize;
+  if (Src->Ty)
+    OldSize = DL.getTypeStoreSize(Src->Ty);
+  else if (auto *Length =
+               dyn_cast<ConstantInt>(cast<AnyMemIntrinsic>(Source).getLength()))
+    if (auto Bytes = Length->getValue().tryZExtValue())
+      OldSize = TypeSize::getFixed(*Bytes);
+  TypeSize NewSize = DL.getTypeStoreSize(Dst->Ty);
+  bool IsSubset = !OldSize || TypeSize::isKnownLE(NewSize, *OldSize);
+  if (ByteOffset)
+    IsSubset =
+        IsSubset && *ByteOffset >= 0 &&
+        (!OldSize || uint64_t(*ByteOffset) <= OldSize->getKnownMinValue() -
+                                                  NewSize.getKnownMinValue());
+  bool IsWholeAccess = IsSubset && ByteOffset == 0 && OldSize == NewSize;
+  bool SameElementType =
+      Src->Ty && Src->Ty->getScalarType() == Dst->Ty->getScalarType();
+  bool SameValueBits =
+      IsWholeAccess && Src->Ty && !Src->Ty->isAggregateType() &&
+      !Dst->Ty->isAggregateType() && DL.typeSizeEqualsStoreSize(Src->Ty) &&
+      DL.typeSizeEqualsStoreSize(Dst->Ty);
+  bool SamePointer =
+      getInvariantGroupPointer(getMemoryPointer(Source, Src->PointerOperand)) ==
+      getInvariantGroupPointer(getMemoryPointer(Dest, Dst->PointerOperand));
+
+  SmallVector<std::pair<unsigned, MDNode *>, 8> MDs;
+  Source.getAllMetadata(MDs);
+  for (auto [Kind, MD] : MDs) {
+    bool Keep = false;
+    switch (Kind) {
+    case LLVMContext::MD_dbg:
+    case LLVMContext::MD_nontemporal:
+    case LLVMContext::MD_annotation:
+    case LLVMContext::MD_nosanitize:
+      Keep = true;
+      break;
+    case LLVMContext::MD_mmra:
+      Keep = canInstructionHaveMMRAs(Dest);
+      break;
+    case LLVMContext::MD_access_group:
+    case LLVMContext::MD_mem_parallel_loop_access:
+      Keep = IsSubset;
+      break;
+    case LLVMContext::MD_invariant_load:
+      Keep = IsSubset && Dst->IsRead;
+      break;
+    case LLVMContext::MD_invariant_group:
+      Keep = IsSubset && SamePointer && !isa<CallBase>(Dest);
+      break;
+    case LLVMContext::MD_noundef:
+      Keep =
+          IsSubset && (SameElementType || SameValueBits) && isa<LoadInst>(Dest);
+      break;
+    case LLVMContext::MD_fpmath:
+    case LLVMContext::MD_nofpclass:
+      Keep =
+          IsSubset && SameElementType &&
+          (Kind == LLVMContext::MD_fpmath ? Dest.getType()->isFPOrFPVectorTy()
+                                          : isa<LoadInst>(Dest));
+      break;
+    case LLVMContext::MD_nonnull:
+      if (SameValueBits && isa<LoadInst>(Source) && isa<LoadInst>(Dest))
+        copyNonnullMetadata(cast<LoadInst>(Source), MD, cast<LoadInst>(Dest));
+      break;
+    case LLVMContext::MD_range:
+      if (IsSubset && SameElementType && Dst->Ty->isIntOrIntVectorTy())
+        Keep = true;
+      else if (SameValueBits && isa<LoadInst>(Source) && isa<LoadInst>(Dest))
+        copyRangeMetadata(DL, cast<LoadInst>(Source), MD, cast<LoadInst>(Dest));
+      break;
+    case LLVMContext::MD_align:
+    case LLVMContext::MD_dereferenceable:
+    case LLVMContext::MD_dereferenceable_or_null:
+      Keep = SameValueBits && Dst->Ty->isPointerTy() && isa<LoadInst>(Dest);
+      break;
+    case LLVMContext::MD_DIAssignID:
+      Keep = IsWholeAccess && isa<StoreInst>(Source) && isa<StoreInst>(Dest);
+      break;
+    case LLVMContext::MD_prof:
+      Keep =
+          IsWholeAccess && ((isa<LoadInst>(Source) && isa<LoadInst>(Dest)) ||
+                            (isa<StoreInst>(Source) && isa<StoreInst>(Dest)));
+      break;
+    case LLVMContext::MD_captures:
+      Keep = IsWholeAccess && isa<StoreInst>(Dest) && Dst->Ty->isPointerTy();
+      break;
+    default:
+      break;
+    }
+    if (Keep)
+      Dest.setMetadata(Kind, MD);
+  }
+
+  AAMDNodes AA = Source.getAAMetadata();
+  if (!IsSubset) {
+    AA.TBAA = nullptr;
+    AA.TBAAStruct = nullptr;
+    AA.Scope = nullptr;
+    AA.NoAlias = nullptr;
+  } else if (!IsWholeAccess) {
+    if (ByteOffset)
+      AA = AA.adjustForAccess(*ByteOffset, Dst->Ty, DL);
+    else
+      AA.TBAAStruct = nullptr;
+  } else {
+    // Prefer a single alias tag for a whole replacement, retaining the full
+    // field layout when it cannot be represented that way.
+    AAMDNodes Adjusted = AA.adjustForAccess(0, Dst->Ty, DL);
+    if (!AA.TBAA && Adjusted.TBAA)
+      AA = Adjusted;
+  }
+  Dest.setAAMetadata(AA);
+  setMemCacheHint(Dest, Dst->PointerOperand,
+                  getMemCacheHint(Source, Src->PointerOperand));
+}
+
+void llvm::copyMetadataForScalarizedInstruction(
+    Instruction &Dest, const Instruction &Source,
+    std::optional<int64_t> ByteOffset) {
+  if (auto Dst = getMemoryAccessInfo(Dest))
+    if (getMemoryAccessInfo(Source, Dst->IsRead)) {
+      copyMetadataForMemoryAccess(Dest, Source, ByteOffset);
+      return;
+    }
+  if (Dest.getType()->getScalarType() == Source.getType()->getScalarType() &&
+      Dest.getType()->isFPOrFPVectorTy())
+    Dest.copyMetadata(Source, {LLVMContext::MD_fpmath});
+  if (!Dest.getDebugLoc())
+    Dest.setDebugLoc(Source.getDebugLoc());
+}
+
+/// Add all access groups in @p AccGroups to @p List.
+template <typename ListT>
+static void addToAccessGroupList(ListT &List, MDNode *AccGroups) {
+  // Interpret an access group as a list containing itself.
+  if (AccGroups->getNumOperands() == 0) {
+    assert(isValidAsAccessGroup(AccGroups) && "Node must be an access group");
+    List.insert(AccGroups);
+    return;
+  }
+
+  for (const auto &AccGroupListOp : AccGroups->operands()) {
+    auto *Item = cast<MDNode>(AccGroupListOp.get());
+    assert(isValidAsAccessGroup(Item) && "List item must be an access group");
+    List.insert(Item);
+  }
+}
+
+MDNode *llvm::uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2) {
+  if (!AccGroups1)
+    return AccGroups2;
+  if (!AccGroups2)
+    return AccGroups1;
+  if (AccGroups1 == AccGroups2)
+    return AccGroups1;
+
+  SmallSetVector<Metadata *, 4> Union;
+  addToAccessGroupList(Union, AccGroups1);
+  addToAccessGroupList(Union, AccGroups2);
+
+  if (Union.size() == 0)
+    return nullptr;
+  if (Union.size() == 1)
+    return cast<MDNode>(Union.front());
+
+  LLVMContext &Ctx = AccGroups1->getContext();
+  return MDNode::get(Ctx, Union.getArrayRef());
+}
+
+MDNode *llvm::intersectAccessGroups(const Instruction *Inst1,
+                                    const Instruction *Inst2) {
+  bool MayAccessMem1 = Inst1->mayReadOrWriteMemory();
+  bool MayAccessMem2 = Inst2->mayReadOrWriteMemory();
+
+  if (!MayAccessMem1 && !MayAccessMem2)
+    return nullptr;
+  if (!MayAccessMem1)
+    return Inst2->getMetadata(LLVMContext::MD_access_group);
+  if (!MayAccessMem2)
+    return Inst1->getMetadata(LLVMContext::MD_access_group);
+
+  MDNode *MD1 = Inst1->getMetadata(LLVMContext::MD_access_group);
+  MDNode *MD2 = Inst2->getMetadata(LLVMContext::MD_access_group);
+  if (!MD1 || !MD2)
+    return nullptr;
+  if (MD1 == MD2)
+    return MD1;
+
+  // Use set for scalable 'contains' check.
+  SmallPtrSet<Metadata *, 4> AccGroupSet2;
+  addToAccessGroupList(AccGroupSet2, MD2);
+
+  SmallVector<Metadata *, 4> Intersection;
+  if (MD1->getNumOperands() == 0) {
+    assert(isValidAsAccessGroup(MD1) && "Node must be an access group");
+    if (AccGroupSet2.count(MD1))
+      Intersection.push_back(MD1);
+  } else {
+    for (const MDOperand &Node : MD1->operands()) {
+      auto *Item = cast<MDNode>(Node.get());
+      assert(isValidAsAccessGroup(Item) && "List item must be an access group");
+      if (AccGroupSet2.count(Item))
+        Intersection.push_back(Item);
+    }
+  }
+
+  if (Intersection.size() == 0)
+    return nullptr;
+  if (Intersection.size() == 1)
+    return cast<MDNode>(Intersection.front());
+
+  LLVMContext &Ctx = Inst1->getContext();
+  return MDNode::get(Ctx, Intersection);
+}
+
+/// Add metadata from \p Inst to \p Metadata, if it can be preserved after
+/// vectorization.
+void llvm::getMetadataToPropagate(
+    Instruction *Inst,
+    SmallVectorImpl<std::pair<unsigned, MDNode *>> &Metadata) {
+  Inst->getAllMetadataOtherThanDebugLoc(Metadata);
+  static const unsigned SupportedIDs[] = {
+      LLVMContext::MD_tbaa,         LLVMContext::MD_alias_scope,
+      LLVMContext::MD_noalias,      LLVMContext::MD_fpmath,
+      LLVMContext::MD_nontemporal,  LLVMContext::MD_invariant_load,
+      LLVMContext::MD_access_group, LLVMContext::MD_mmra};
+
+  // Remove any unsupported metadata kinds from Metadata.
+  for (unsigned Idx = 0; Idx != Metadata.size();) {
+    if (is_contained(SupportedIDs, Metadata[Idx].first)) {
+      ++Idx;
+    } else {
+      // Swap element to end and remove it.
+      std::swap(Metadata[Idx], Metadata.back());
+      Metadata.pop_back();
+    }
+  }
+}
+
+/// \returns \p I after propagating metadata from \p VL.
+Instruction *llvm::propagateMetadata(Instruction *Inst, ArrayRef<Value *> VL) {
+  if (VL.empty())
+    return Inst;
+
+  // Keep cache hints out of getMetadataToPropagate(): VPlan changes operand
+  // layouts without remapping cache-hint indices.
+  propagateMemCacheHint(*Inst, VL);
+
+  SmallVector<std::pair<unsigned, MDNode *>> Metadata;
+  getMetadataToPropagate(cast<Instruction>(VL[0]), Metadata);
+
+  for (auto &[Kind, MD] : Metadata) {
+    // Skip MMRA metadata if the instruction cannot have it.
+    if (Kind == LLVMContext::MD_mmra && !canInstructionHaveMMRAs(*Inst))
+      continue;
+
+    for (int J = 1, E = VL.size(); MD && J != E; ++J) {
+      const Instruction *IJ = cast<Instruction>(VL[J]);
+      MDNode *IMD = IJ->getMetadata(Kind);
+
+      switch (Kind) {
+      case LLVMContext::MD_mmra: {
+        MD = MMRAMetadata::combine(Inst->getContext(), MD, IMD);
+        break;
+      }
+      case LLVMContext::MD_tbaa:
+        MD = MDNode::getMostGenericTBAA(MD, IMD);
+        break;
+      case LLVMContext::MD_alias_scope:
+        MD = MDNode::getMostGenericAliasScope(MD, IMD);
+        break;
+      case LLVMContext::MD_fpmath:
+        MD = MDNode::getMostGenericFPMath(MD, IMD);
+        break;
+      case LLVMContext::MD_noalias:
+      case LLVMContext::MD_nontemporal:
+      case LLVMContext::MD_invariant_load:
+        MD = MDNode::intersect(MD, IMD);
+        break;
+      case LLVMContext::MD_access_group:
+        MD = intersectAccessGroups(Inst, IJ);
+        break;
+      default:
+        llvm_unreachable("unhandled metadata");
+      }
+    }
+
+    Inst->setMetadata(Kind, MD);
+  }
+
+  return Inst;
+}
+
+/// If AAOnly is set, only intersect alias analysis metadata and preserve other
+/// known metadata. Unknown metadata is always dropped.
+static void combineMetadata(Instruction *K, const Instruction *J,
+                            bool DoesKMove, bool AAOnly = false) {
+  SmallVector<std::pair<unsigned, MDNode *>, 4> Metadata;
+  K->getAllMetadataOtherThanDebugLoc(Metadata);
+  for (const auto &MD : Metadata) {
+    unsigned Kind = MD.first;
+    MDNode *JMD = J->getMetadata(Kind);
+    MDNode *KMD = MD.second;
+
+    // TODO: Assert that this switch is exhaustive for fixed MD kinds.
+    switch (Kind) {
+      default:
+        K->setMetadata(Kind, nullptr); // Remove unknown metadata
+        break;
+      case LLVMContext::MD_dbg:
+        llvm_unreachable("getAllMetadataOtherThanDebugLoc returned a MD_dbg");
+      case LLVMContext::MD_DIAssignID:
+        if (!AAOnly)
+          K->mergeDIAssignID(J);
+        break;
+      case LLVMContext::MD_tbaa:
+        if (DoesKMove)
+          K->setMetadata(Kind, MDNode::getMostGenericTBAA(JMD, KMD));
+        break;
+      case LLVMContext::MD_alias_scope:
+        if (DoesKMove)
+          K->setMetadata(Kind, MDNode::getMostGenericAliasScope(JMD, KMD));
+        break;
+      case LLVMContext::MD_noalias:
+      case LLVMContext::MD_mem_parallel_loop_access:
+        if (DoesKMove)
+          K->setMetadata(Kind, MDNode::intersect(JMD, KMD));
+        break;
+      case LLVMContext::MD_access_group:
+        if (DoesKMove)
+          K->setMetadata(LLVMContext::MD_access_group,
+                         intersectAccessGroups(K, J));
+        break;
+      case LLVMContext::MD_range:
+        if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
+          K->setMetadata(Kind, MDNode::getMostGenericRange(JMD, KMD));
+        break;
+      case LLVMContext::MD_nofpclass:
+        if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
+          K->setMetadata(Kind, MDNode::getMostGenericNoFPClass(JMD, KMD));
+        break;
+      case LLVMContext::MD_fpmath:
+        if (!AAOnly)
+          K->setMetadata(Kind, MDNode::getMostGenericFPMath(JMD, KMD));
+        break;
+      case LLVMContext::MD_invariant_load:
+      case LLVMContext::MD_invariant_group:
+        // If K moves, only keep the invariant metadata if it is present on
+        // both instructions; otherwise the invariant would be asserted on a
+        // path (J's) that never promised it. If K does not move, K stays on
+        // its original path, so its existing metadata remains valid.
+        if (DoesKMove)
+          K->setMetadata(Kind, JMD);
+        break;
+      case LLVMContext::MD_nonnull:
+        if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
+          K->setMetadata(Kind, JMD);
+        break;
+      // Keep empty cases for prof, mmra, memprof, and callsite to prevent them
+      // from being removed as unknown metadata. The actual merging is handled
+      // separately below.
+      case LLVMContext::MD_prof:
+      case LLVMContext::MD_mmra:
+      case LLVMContext::MD_memprof:
+      case LLVMContext::MD_callsite:
+        break;
+      case LLVMContext::MD_callee_type:
+        if (!AAOnly) {
+          K->setMetadata(LLVMContext::MD_callee_type,
+                         MDNode::getMergedCalleeTypeMetadata(KMD, JMD));
+        }
+        break;
+      case LLVMContext::MD_callees:
+        // If K moves, it replaces J on J's path and must allow J's callees as
+        // well. If K does not move, its callees remain valid.
+        if (!AAOnly && DoesKMove)
+          K->setMetadata(Kind, MDNode::getMergedCalleesMetadata(KMD, JMD));
+        break;
+      case LLVMContext::MD_align:
+        if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
+          K->setMetadata(
+              Kind, MDNode::getMostGenericAlignmentOrDereferenceable(JMD, KMD));
+        break;
+      case LLVMContext::MD_dereferenceable:
+      case LLVMContext::MD_dereferenceable_or_null:
+        if (!AAOnly && DoesKMove)
+          K->setMetadata(Kind,
+            MDNode::getMostGenericAlignmentOrDereferenceable(JMD, KMD));
+        break;
+      case LLVMContext::MD_preserve_access_index:
+        // Preserve !preserve.access.index in K.
+        break;
+      case LLVMContext::MD_noundef:
+        // If K does move, keep noundef if it is present in both instructions.
+        if (!AAOnly && DoesKMove)
+          K->setMetadata(Kind, JMD);
+        break;
+      case LLVMContext::MD_nontemporal:
+        // Preserve !nontemporal if it is present on both instructions.
+        if (!AAOnly)
+          K->setMetadata(Kind, JMD);
+        break;
+      case LLVMContext::MD_mem_cache_hint:
+        // Preserve !mem.cache_hint only if it is present and equivalent on both
+        // instructions.
+        if (!AAOnly && KMD != JMD)
+          K->setMetadata(Kind, nullptr);
+        break;
+      case LLVMContext::MD_noalias_addrspace:
+        if (DoesKMove)
+          K->setMetadata(Kind,
+                         MDNode::getMostGenericNoaliasAddrspace(JMD, KMD));
+        break;
+      case LLVMContext::MD_nosanitize:
+        // Preserve !nosanitize if both K and J have it.
+        K->setMetadata(Kind, JMD);
+        break;
+      case LLVMContext::MD_captures:
+        K->setMetadata(
+            Kind, MDNode::fromCaptureComponents(
+                      K->getContext(), MDNode::toCaptureComponents(JMD) |
+                                           MDNode::toCaptureComponents(KMD)));
+        break;
+      case LLVMContext::MD_alloc_token:
+        if (!AAOnly && KMD != JMD)
+          K->setMetadata(Kind, MDNode::getMergedAllocTokenMetadata(KMD, JMD));
+        break;
+      }
+  }
+
+  // Merge MMRAs.
+  // This is handled separately because we also want to handle cases where K
+  // doesn't have tags but J does.
+  auto JMMRA = J->getMetadata(LLVMContext::MD_mmra);
+  auto KMMRA = K->getMetadata(LLVMContext::MD_mmra);
+  if (JMMRA || KMMRA) {
+    K->setMetadata(LLVMContext::MD_mmra,
+                   MMRAMetadata::combine(K->getContext(), JMMRA, KMMRA));
+  }
+
+  // Merge memprof metadata.
+  // Handle separately to support cases where only one instruction has the
+  // metadata.
+  auto *JMemProf = J->getMetadata(LLVMContext::MD_memprof);
+  auto *KMemProf = K->getMetadata(LLVMContext::MD_memprof);
+  if (!AAOnly && (JMemProf || KMemProf)) {
+    K->setMetadata(LLVMContext::MD_memprof,
+                   MDNode::getMergedMemProfMetadata(KMemProf, JMemProf));
+  }
+
+  // Merge callsite metadata.
+  // Handle separately to support cases where only one instruction has the
+  // metadata.
+  auto *JCallSite = J->getMetadata(LLVMContext::MD_callsite);
+  auto *KCallSite = K->getMetadata(LLVMContext::MD_callsite);
+  if (!AAOnly && (JCallSite || KCallSite)) {
+    K->setMetadata(LLVMContext::MD_callsite,
+                   MDNode::getMergedCallsiteMetadata(KCallSite, JCallSite));
+  }
+
+  // Merge prof metadata.
+  // Handle separately to support cases where only one instruction has the
+  // metadata.
+  auto *JProf = J->getMetadata(LLVMContext::MD_prof);
+  auto *KProf = K->getMetadata(LLVMContext::MD_prof);
+  if (!AAOnly && (JProf || KProf)) {
+    // We do not need to merge metadata if we have select instructions and the
+    // one we're keeping (K) does not move (explicitly merging would drop the
+    // information in this case). Otherwise, explicitly handle the merging.
+    if (DoesKMove || (!isa<SelectInst>(J) || !isa<SelectInst>(K)) ||
+        ProfcheckDisableMetadataFixes)
+      K->setMetadata(LLVMContext::MD_prof,
+                     MDNode::getMergedProfMetadata(KProf, JProf, K, J));
+  }
+}
+
+void llvm::combineMetadataForCSE(Instruction *K, const Instruction *J,
+                                 bool DoesKMove) {
+  combineMetadata(K, J, DoesKMove);
+}
+
+void llvm::combineAAMetadata(Instruction *K, const Instruction *J) {
+  combineMetadata(K, J, /*DoesKMove=*/true, /*AAOnly=*/true);
+}
+
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index b5ccb8574b6026..8b479de7037b9e 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -18,6 +18,7 @@
 #include "llvm/Analysis/DemandedBits.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/LoopIterator.h"
+#include "llvm/Analysis/MetadataUtils.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
@@ -1030,168 +1031,6 @@ llvm::computeMinimumValueSizes(ArrayRef<BasicBlock *> Blocks, DemandedBits &DB,
   return MinBWs;
 }
 
-/// Add all access groups in @p AccGroups to @p List.
-template <typename ListT>
-static void addToAccessGroupList(ListT &List, MDNode *AccGroups) {
-  // Interpret an access group as a list containing itself.
-  if (AccGroups->getNumOperands() == 0) {
-    assert(isValidAsAccessGroup(AccGroups) && "Node must be an access group");
-    List.insert(AccGroups);
-    return;
-  }
-
-  for (const auto &AccGroupListOp : AccGroups->operands()) {
-    auto *Item = cast<MDNode>(AccGroupListOp.get());
-    assert(isValidAsAccessGroup(Item) && "List item must be an access group");
-    List.insert(Item);
-  }
-}
-
-MDNode *llvm::uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2) {
-  if (!AccGroups1)
-    return AccGroups2;
-  if (!AccGroups2)
-    return AccGroups1;
-  if (AccGroups1 == AccGroups2)
-    return AccGroups1;
-
-  SmallSetVector<Metadata *, 4> Union;
-  addToAccessGroupList(Union, AccGroups1);
-  addToAccessGroupList(Union, AccGroups2);
-
-  if (Union.size() == 0)
-    return nullptr;
-  if (Union.size() == 1)
-    return cast<MDNode>(Union.front());
-
-  LLVMContext &Ctx = AccGroups1->getContext();
-  return MDNode::get(Ctx, Union.getArrayRef());
-}
-
-MDNode *llvm::intersectAccessGroups(const Instruction *Inst1,
-                                    const Instruction *Inst2) {
-  bool MayAccessMem1 = Inst1->mayReadOrWriteMemory();
-  bool MayAccessMem2 = Inst2->mayReadOrWriteMemory();
-
-  if (!MayAccessMem1 && !MayAccessMem2)
-    return nullptr;
-  if (!MayAccessMem1)
-    return Inst2->getMetadata(LLVMContext::MD_access_group);
-  if (!MayAccessMem2)
-    return Inst1->getMetadata(LLVMContext::MD_access_group);
-
-  MDNode *MD1 = Inst1->getMetadata(LLVMContext::MD_access_group);
-  MDNode *MD2 = Inst2->getMetadata(LLVMContext::MD_access_group);
-  if (!MD1 || !MD2)
-    return nullptr;
-  if (MD1 == MD2)
-    return MD1;
-
-  // Use set for scalable 'contains' check.
-  SmallPtrSet<Metadata *, 4> AccGroupSet2;
-  addToAccessGroupList(AccGroupSet2, MD2);
-
-  SmallVector<Metadata *, 4> Intersection;
-  if (MD1->getNumOperands() == 0) {
-    assert(isValidAsAccessGroup(MD1) && "Node must be an access group");
-    if (AccGroupSet2.count(MD1))
-      Intersection.push_back(MD1);
-  } else {
-    for (const MDOperand &Node : MD1->operands()) {
-      auto *Item = cast<MDNode>(Node.get());
-      assert(isValidAsAccessGroup(Item) && "List item must be an access group");
-      if (AccGroupSet2.count(Item))
-        Intersection.push_back(Item);
-    }
-  }
-
-  if (Intersection.size() == 0)
-    return nullptr;
-  if (Intersection.size() == 1)
-    return cast<MDNode>(Intersection.front());
-
-  LLVMContext &Ctx = Inst1->getContext();
-  return MDNode::get(Ctx, Intersection);
-}
-
-/// Add metadata from \p Inst to \p Metadata, if it can be preserved after
-/// vectorization.
-void llvm::getMetadataToPropagate(
-    Instruction *Inst,
-    SmallVectorImpl<std::pair<unsigned, MDNode *>> &Metadata) {
-  Inst->getAllMetadataOtherThanDebugLoc(Metadata);
-  static const unsigned SupportedIDs[] = {
-      LLVMContext::MD_tbaa,         LLVMContext::MD_alias_scope,
-      LLVMContext::MD_noalias,      LLVMContext::MD_fpmath,
-      LLVMContext::MD_nontemporal,  LLVMContext::MD_invariant_load,
-      LLVMContext::MD_access_group, LLVMContext::MD_mmra};
-
-  // Remove any unsupported metadata kinds from Metadata.
-  for (unsigned Idx = 0; Idx != Metadata.size();) {
-    if (is_contained(SupportedIDs, Metadata[Idx].first)) {
-      ++Idx;
-    } else {
-      // Swap element to end and remove it.
-      std::swap(Metadata[Idx], Metadata.back());
-      Metadata.pop_back();
-    }
-  }
-}
-
-/// \returns \p I after propagating metadata from \p VL.
-Instruction *llvm::propagateMetadata(Instruction *Inst, ArrayRef<Value *> VL) {
-  if (VL.empty())
-    return Inst;
-
-  // Keep cache hints out of getMetadataToPropagate(): VPlan changes operand
-  // layouts without remapping cache-hint indices.
-  propagateMemCacheHint(*Inst, VL);
-
-  SmallVector<std::pair<unsigned, MDNode *>> Metadata;
-  getMetadataToPropagate(cast<Instruction>(VL[0]), Metadata);
-
-  for (auto &[Kind, MD] : Metadata) {
-    // Skip MMRA metadata if the instruction cannot have it.
-    if (Kind == LLVMContext::MD_mmra && !canInstructionHaveMMRAs(*Inst))
-      continue;
-
-    for (int J = 1, E = VL.size(); MD && J != E; ++J) {
-      const Instruction *IJ = cast<Instruction>(VL[J]);
-      MDNode *IMD = IJ->getMetadata(Kind);
-
-      switch (Kind) {
-      case LLVMContext::MD_mmra: {
-        MD = MMRAMetadata::combine(Inst->getContext(), MD, IMD);
-        break;
-      }
-      case LLVMContext::MD_tbaa:
-        MD = MDNode::getMostGenericTBAA(MD, IMD);
-        break;
-      case LLVMContext::MD_alias_scope:
-        MD = MDNode::getMostGenericAliasScope(MD, IMD);
-        break;
-      case LLVMContext::MD_fpmath:
-        MD = MDNode::getMostGenericFPMath(MD, IMD);
-        break;
-      case LLVMContext::MD_noalias:
-      case LLVMContext::MD_nontemporal:
-      case LLVMContext::MD_invariant_load:
-        MD = MDNode::intersect(MD, IMD);
-        break;
-      case LLVMContext::MD_access_group:
-        MD = intersectAccessGroups(Inst, IJ);
-        break;
-      default:
-        llvm_unreachable("unhandled metadata");
-      }
-    }
-
-    Inst->setMetadata(Kind, MD);
-  }
-
-  return Inst;
-}
-
 Constant *
 llvm::createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF,
                            const InterleaveGroup<Instruction> &Group) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
index 89f024ca8a5dc9..1e1a01be30e948 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
@@ -225,6 +225,7 @@
 #include "llvm/ADT/SetOperations.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/InstSimplifyFolder.h"
+#include "llvm/Analysis/MetadataUtils.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
@@ -586,7 +587,7 @@ bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::visitLoadInst(LoadInst &LI) {
           LoadInst *NewLI = IRB.CreateAlignedLoad(
               IntLeafTy, Ptr, commonAlignment(LI.getAlign(), Off), Name);
           NewLI->setVolatile(LI.isVolatile());
-          copyMetadataForLoad(*NewLI, LI);
+          copyMetadataForMemoryAccess(*NewLI, LI);
           NewLI->setAAMetadata(AATags.adjustForAccess(Off, IntLeafTy, DL));
           Value *V = NewLI;
           if (LeafTy != IntLeafTy)
@@ -1192,7 +1193,7 @@ bool LegalizeBufferContentTypesVisitor::visitLoadImpl(
       LoadInst *NewLI = IRB.CreateAlignedLoad(
           LoadableType, NewPtr, commonAlignment(OrigLI.getAlign(), ByteOffset),
           Name + ".off." + Twine(ByteOffset));
-      copyMetadataForLoad(*NewLI, OrigLI);
+      copyMetadataForMemoryAccess(*NewLI, OrigLI);
       NewLI->setAAMetadata(
           AANodes.adjustForAccess(ByteOffset, LoadableType, DL));
       NewLI->setAtomic(OrigLI.getOrdering(), OrigLI.getSyncScopeID());
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 5d9e01ac505a0c..e98c6bb616e955 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -27,6 +27,7 @@
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/Loads.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
+#include "llvm/Analysis/MetadataUtils.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/AttributeMask.h"
@@ -169,32 +170,17 @@ Instruction *InstCombinerImpl::SimplifyAnyMemTransfer(AnyMemTransferInst *MI) {
   // Use an integer load+store unless we can find something better.
   IntegerType* IntType = IntegerType::get(MI->getContext(), Size<<3);
 
-  // If the memcpy has metadata describing the members, see if we can get the
-  // TBAA, scope and noalias tags describing our copy.
-  AAMDNodes AACopyMD = MI->getAAMetadata().adjustForAccess(Size);
-
   Value *Src = MI->getArgOperand(1);
   Value *Dest = MI->getArgOperand(0);
   LoadInst *L = Builder.CreateLoad(IntType, Src);
   // Alignment from the mem intrinsic will be better, so use it.
   L->setAlignment(*CopySrcAlign);
-  L->setAAMetadata(AACopyMD);
-  MDNode *LoopMemParallelMD =
-    MI->getMetadata(LLVMContext::MD_mem_parallel_loop_access);
-  if (LoopMemParallelMD)
-    L->setMetadata(LLVMContext::MD_mem_parallel_loop_access, LoopMemParallelMD);
-  MDNode *AccessGroupMD = MI->getMetadata(LLVMContext::MD_access_group);
-  if (AccessGroupMD)
-    L->setMetadata(LLVMContext::MD_access_group, AccessGroupMD);
+  copyMetadataForMemoryAccess(*L, *MI);
 
   StoreInst *S = Builder.CreateStore(L, Dest);
   // Alignment from the mem intrinsic will be better, so use it.
   S->setAlignment(*CopyDstAlign);
-  S->setAAMetadata(AACopyMD);
-  if (LoopMemParallelMD)
-    S->setMetadata(LLVMContext::MD_mem_parallel_loop_access, LoopMemParallelMD);
-  if (AccessGroupMD)
-    S->setMetadata(LLVMContext::MD_access_group, AccessGroupMD);
+  copyMetadataForMemoryAccess(*S, *MI);
   S->copyMetadata(*MI, LLVMContext::MD_DIAssignID);
 
   if (auto *MT = dyn_cast<MemTransferInst>(MI)) {
@@ -272,6 +258,7 @@ Instruction *InstCombinerImpl::SimplifyAnyMemSet(AnyMemSetInst *MI) {
       return nullptr;
 
     StoreInst *S = Builder.CreateStore(FillVal, Dest, MI->isVolatile());
+    copyMetadataForMemoryAccess(*S, *MI);
     S->copyMetadata(*MI, LLVMContext::MD_DIAssignID);
     for (DbgVariableRecord *DbgAssign : at::getDVRAssignmentMarkers(S)) {
       if (llvm::is_contained(DbgAssign->location_ops(), Fill))
@@ -302,7 +289,7 @@ Value *InstCombinerImpl::simplifyMaskedLoad(IntrinsicInst &II) {
   if (match(Mask, m_AllOnesOrPoison())) {
     LoadInst *L = Builder.CreateAlignedLoad(II.getType(), LoadPtr, Alignment,
                                             "unmaskedload");
-    L->copyMetadata(II);
+    copyMetadataForMemoryAccess(*L, II);
     return L;
   }
 
@@ -312,7 +299,9 @@ Value *InstCombinerImpl::simplifyMaskedLoad(IntrinsicInst &II) {
                                SQ.getWithInstruction(&II))) {
     LoadInst *LI = Builder.CreateAlignedLoad(II.getType(), LoadPtr, Alignment,
                                              "unmaskedload");
-    LI->copyMetadata(II);
+    // TODO: Check whether metadata valid only for active lanes can be retained
+    // on the unmasked load.
+    copyMetadataForMemoryAccess(*LI, II);
     return Builder.CreateSelect(II.getArgOperand(1), LI, II.getArgOperand(2));
   }
 
@@ -338,7 +327,7 @@ Instruction *InstCombinerImpl::simplifyMaskedStore(IntrinsicInst &II) {
   if (match(ConstMask, m_AllOnesOrPoison())) {
     StoreInst *S =
         new StoreInst(II.getArgOperand(0), StorePtr, false, Alignment);
-    S->copyMetadata(II);
+    copyMetadataForMemoryAccess(*S, II);
     return S;
   }
 
@@ -375,6 +364,7 @@ Instruction *InstCombinerImpl::simplifyMaskedGather(IntrinsicInst &II) {
       const Align Alignment = II.getParamAlign(0).valueOrOne();
       LoadInst *L = Builder.CreateAlignedLoad(VecTy->getElementType(), SplatPtr,
                                               Alignment, "load.scalar");
+      copyMetadataForMemoryAccess(*L, II);
       Value *Shuf =
           Builder.CreateVectorSplat(VecTy->getElementCount(), L, "broadcast");
       return replaceInstUsesWith(II, cast<Instruction>(Shuf));
@@ -405,7 +395,7 @@ Instruction *InstCombinerImpl::simplifyMaskedScatter(IntrinsicInst &II) {
         Align Alignment = II.getParamAlign(1).valueOrOne();
         StoreInst *S = new StoreInst(SplatValue, SplatPtr, /*IsVolatile=*/false,
                                      Alignment);
-        S->copyMetadata(II);
+        copyMetadataForMemoryAccess(*S, II);
         return S;
       }
     }
@@ -421,7 +411,7 @@ Instruction *InstCombinerImpl::simplifyMaskedScatter(IntrinsicInst &II) {
           Builder.CreateExtractElement(II.getArgOperand(0), LastLane);
       StoreInst *S =
           new StoreInst(Extract, SplatPtr, /*IsVolatile=*/false, Alignment);
-      S->copyMetadata(II);
+      copyMetadataForMemoryAccess(*S, II);
       return S;
     }
   }
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
index 60331a3d37b05b..a281e93aa6b977 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
@@ -16,6 +16,7 @@
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/Loads.h"
+#include "llvm/Analysis/MetadataUtils.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
@@ -611,7 +612,7 @@ LoadInst *InstCombinerImpl::combineLoadToNewType(LoadInst &LI, Type *NewTy,
 
   LoadInst *NewLoad = Builder.CreateLoad(
       NewTy, LI.getPointerOperand(), LI.getProperties(), LI.getName() + Suffix);
-  copyMetadataForLoad(*NewLoad, LI);
+  copyMetadataForMemoryAccess(*NewLoad, LI);
   return NewLoad;
 }
 
@@ -624,47 +625,8 @@ static StoreInst *combineStoreToNewValue(InstCombinerImpl &IC, StoreInst &SI,
          "can't fold an atomic store of requested type");
 
   Value *Ptr = SI.getPointerOperand();
-  SmallVector<std::pair<unsigned, MDNode *>, 8> MD;
-  SI.getAllMetadata(MD);
-
   StoreInst *NewStore = IC.Builder.CreateStore(V, Ptr, SI.getProperties());
-  for (const auto &MDPair : MD) {
-    unsigned ID = MDPair.first;
-    MDNode *N = MDPair.second;
-    // Note, essentially every kind of metadata should be preserved here! This
-    // routine is supposed to clone a store instruction changing *only its
-    // type*. The only metadata it makes sense to drop is metadata which is
-    // invalidated when the pointer type changes. This should essentially
-    // never be the case in LLVM, but we explicitly switch over only known
-    // metadata to be conservatively correct. If you are adding metadata to
-    // LLVM which pertains to stores, you almost certainly want to add it
-    // here.
-    switch (ID) {
-    case LLVMContext::MD_dbg:
-    case LLVMContext::MD_DIAssignID:
-    case LLVMContext::MD_tbaa:
-    case LLVMContext::MD_prof:
-    case LLVMContext::MD_fpmath:
-    case LLVMContext::MD_tbaa_struct:
-    case LLVMContext::MD_alias_scope:
-    case LLVMContext::MD_noalias:
-    case LLVMContext::MD_nontemporal:
-    case LLVMContext::MD_mem_parallel_loop_access:
-    case LLVMContext::MD_access_group:
-      // All of these directly apply.
-      NewStore->setMetadata(ID, N);
-      break;
-    case LLVMContext::MD_invariant_load:
-    case LLVMContext::MD_nonnull:
-    case LLVMContext::MD_noundef:
-    case LLVMContext::MD_range:
-    case LLVMContext::MD_align:
-    case LLVMContext::MD_dereferenceable:
-    case LLVMContext::MD_dereferenceable_or_null:
-      // These don't apply for stores.
-      break;
-    }
-  }
+  copyMetadataForMemoryAccess(*NewStore, SI);
 
   return NewStore;
 }
@@ -752,9 +714,6 @@ static Instruction *unpackLoadToAggregate(InstCombinerImpl &IC, LoadInst &LI) {
     if (NumElements == 1) {
       LoadInst *NewLoad = IC.combineLoadToNewType(LI, ST->getTypeAtIndex(0U),
                                                   ".unpack");
-      NewLoad->setAAMetadata(LI.getAAMetadata());
-      // Copy invariant metadata from parent load.
-      NewLoad->copyMetadata(LI, LLVMContext::MD_invariant_load);
       return IC.replaceInstUsesWith(LI, IC.Builder.CreateInsertValue(
         PoisonValue::get(T), NewLoad, 0, Name));
     }
@@ -780,10 +739,8 @@ static Instruction *unpackLoadToAggregate(InstCombinerImpl &IC, LoadInst &LI) {
           ST->getElementType(i), Ptr,
           commonAlignment(Align, SL->getElementOffset(i).getKnownMinValue()),
           Name + ".unpack");
-      // Propagate AA metadata. It'll still be valid on the narrowed load.
-      L->setAAMetadata(LI.getAAMetadata());
-      // Copy invariant metadata from parent load.
-      L->copyMetadata(LI, LLVMContext::MD_invariant_load);
+      copyMetadataForMemoryAccess(*L, LI,
+                                  SL->getElementOffset(i).getKnownMinValue());
       V = IC.Builder.CreateInsertValue(V, L, i);
     }
 
@@ -796,7 +753,6 @@ static Instruction *unpackLoadToAggregate(InstCombinerImpl &IC, LoadInst &LI) {
     auto NumElements = AT->getNumElements();
     if (NumElements == 1) {
       LoadInst *NewLoad = IC.combineLoadToNewType(LI, ET, ".unpack");
-      NewLoad->setAAMetadata(LI.getAAMetadata());
       return IC.replaceInstUsesWith(LI, IC.Builder.CreateInsertValue(
         PoisonValue::get(T), NewLoad, 0, Name));
     }
@@ -828,7 +784,7 @@ static Instruction *unpackLoadToAggregate(InstCombinerImpl &IC, LoadInst &LI) {
       auto EltAlign = commonAlignment(Align, Offset.getKnownMinValue());
       auto *L = IC.Builder.CreateAlignedLoad(AT->getElementType(), Ptr,
                                              EltAlign, Name + ".unpack");
-      L->setAAMetadata(LI.getAAMetadata());
+      copyMetadataForMemoryAccess(*L, LI, Offset.getKnownMinValue());
       V = IC.Builder.CreateInsertValue(V, L, i);
       Offset += EltSize;
     }
@@ -1435,7 +1391,8 @@ static bool unpackStoreToAggregate(InstCombinerImpl &IC, StoreInst &SI) {
       auto EltAlign =
           commonAlignment(Align, SL->getElementOffset(i).getKnownMinValue());
       llvm::Instruction *NS = IC.Builder.CreateAlignedStore(Val, Ptr, EltAlign);
-      NS->setAAMetadata(SI.getAAMetadata());
+      copyMetadataForMemoryAccess(*NS, SI,
+                                  SL->getElementOffset(i).getKnownMinValue());
     }
 
     return true;
@@ -1481,7 +1438,7 @@ static bool unpackStoreToAggregate(InstCombinerImpl &IC, StoreInst &SI) {
       auto *Val = IC.Builder.CreateExtractValue(V, i, EltName);
       auto EltAlign = commonAlignment(Align, Offset.getKnownMinValue());
       Instruction *NS = IC.Builder.CreateAlignedStore(Val, Ptr, EltAlign);
-      NS->setAAMetadata(SI.getAAMetadata());
+      copyMetadataForMemoryAccess(*NS, SI, Offset.getKnownMinValue());
       Offset += EltSize;
     }
 
@@ -1802,17 +1759,10 @@ bool InstCombinerImpl::mergeStoreIntoSuccessor(StoreInst &SI) {
   StoreInst *NewSI =
       new StoreInst(MergedVal, SI.getOperand(1), SI.getProperties());
   InsertNewInstBefore(NewSI, BBI);
-  NewSI->setDebugLoc(MergedLoc);
+  copyMetadataForMemoryAccess(*NewSI, SI);
   NewSI->mergeDIAssignID({&SI, OtherStore});
-
-  // If the two stores had AA tags, merge them.
-  AAMDNodes AATags = SI.getAAMetadata();
-  if (AATags)
-    NewSI->setAAMetadata(AATags.merge(OtherStore->getAAMetadata()));
-
-  // If the two stores had access groups, intersect them.
-  NewSI->setMetadata(LLVMContext::MD_access_group,
-                     intersectAccessGroups(&SI, OtherStore));
+  combineMetadataForCSE(NewSI, OtherStore, /*DoesKMove=*/true);
+  NewSI->setDebugLoc(MergedLoc);
 
   // Nuke the old stores.
   eraseInstFromFunction(SI);
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 075ac5818082f0..51ee1f6fa95087 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -43,6 +43,7 @@
 #include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/Loads.h"
+#include "llvm/Analysis/MetadataUtils.h"
 #include "llvm/Analysis/PtrUseVisitor.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Analysis/VectorUtils.h"
@@ -3503,8 +3504,8 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
     LoadInst *Load =
         IRB.CreateAlignedLoad(NewAllocaTy, &NewAI, NewAI.getAlign(), "load");
 
-    Load->copyMetadata(LI, {LLVMContext::MD_mem_parallel_loop_access,
-                            LLVMContext::MD_access_group});
+    copyMetadataForMemoryAccess(
+        *Load, LI, int64_t(NewAllocaBeginOffset) - int64_t(BeginOffset));
     return extractVector(IRB, Load, BeginIndex, EndIndex, "vec");
   }
 
@@ -3537,8 +3538,6 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
     Value *OldOp = LI.getOperand(0);
     assert(OldOp == OldPtr);
 
-    AAMDNodes AATags = LI.getAAMetadata();
-
     unsigned AS = LI.getPointerAddressSpace();
 
     Type *TargetTy = IsSplit ? Type::getIntNTy(LI.getContext(), SliceSize * 8)
@@ -3564,15 +3563,7 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
       if (NewLI->isAtomic())
         NewLI->setAlignment(LI.getAlign());
 
-      // Copy any metadata that is valid for the new load. This may require
-      // conversion to a different kind of metadata, e.g. !nonnull might change
-      // to !range or vice versa.
-      copyMetadataForLoad(*NewLI, LI);
-
-      // Do this after copyMetadataForLoad() to preserve the TBAA shift.
-      if (AATags)
-        NewLI->setAAMetadata(AATags.adjustForAccess(
-            NewBeginOffset - BeginOffset, NewLI->getType(), DL));
+      copyMetadataForMemoryAccess(*NewLI, LI, NewBeginOffset - BeginOffset);
 
       // Try to preserve nonnull metadata
       V = NewLI;
@@ -3594,15 +3585,9 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
           IRB.CreateAlignedLoad(TargetTy, getNewAllocaSlicePtr(IRB, LTy),
                                 getSliceAlign(), LI.isVolatile(), LI.getName());
 
-      if (AATags)
-        NewLI->setAAMetadata(AATags.adjustForAccess(
-            NewBeginOffset - BeginOffset, NewLI->getType(), DL));
-
       if (LI.isVolatile())
         NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID());
-      NewLI->copyMetadata(LI, {LLVMContext::MD_mem_parallel_loop_access,
-                               LLVMContext::MD_access_group});
-
+      copyMetadataForMemoryAccess(*NewLI, LI, NewBeginOffset - BeginOffset);
       V = NewLI;
       IsPtrAdjusted = true;
     }
@@ -3645,8 +3630,7 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
     return !LI.isVolatile() && !IsPtrAdjusted;
   }
 
-  bool rewriteVectorizedStoreInst(Value *V, StoreInst &SI, Value *OldOp,
-                                  AAMDNodes AATags) {
+  bool rewriteVectorizedStoreInst(Value *V, StoreInst &SI, Value *OldOp) {
     // Capture V for the purpose of debug-info accounting once it's converted
     // to a vector store.
     Value *OrigV = V;
@@ -3669,11 +3653,7 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
       V = insertVector(IRB, Old, V, BeginIndex, "vec");
     }
     StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlign());
-    Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
-                             LLVMContext::MD_access_group});
-    if (AATags)
-      Store->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
-                                                  V->getType(), DL));
+    copyMetadataForMemoryAccess(*Store, SI, NewBeginOffset - BeginOffset);
     Pass.DeadInsts.push_back(&SI);
 
     // NOTE: Careful to use OrigV rather than V.
@@ -3683,7 +3663,7 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
     return true;
   }
 
-  bool rewriteIntegerStore(Value *V, StoreInst &SI, AAMDNodes AATags) {
+  bool rewriteIntegerStore(Value *V, StoreInst &SI) {
     assert(IntTy && "We cannot extract an integer from the alloca");
     assert(!SI.isVolatile());
     if (DL.getTypeSizeInBits(V->getType()).getFixedValue() !=
@@ -3697,11 +3677,7 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
     }
     V = IRB.CreateBitPreservingCastChain(DL, V, NewAllocaTy);
     StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlign());
-    Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
-                             LLVMContext::MD_access_group});
-    if (AATags)
-      Store->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
-                                                  V->getType(), DL));
+    copyMetadataForMemoryAccess(*Store, SI, NewBeginOffset - BeginOffset);
 
     migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &SI,
                      Store, Store->getPointerOperand(),
@@ -3717,7 +3693,6 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
     Value *OldOp = SI.getOperand(1);
     assert(OldOp == OldPtr);
 
-    AAMDNodes AATags = SI.getAAMetadata();
     Value *V = SI.getValueOperand();
 
     // Strip all inbounds GEPs and pointer casts to try to dig out any root
@@ -3739,9 +3714,9 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
     }
 
     if (VecTy)
-      return rewriteVectorizedStoreInst(V, SI, OldOp, AATags);
+      return rewriteVectorizedStoreInst(V, SI, OldOp);
     if (IntTy && V->getType()->isIntegerTy())
-      return rewriteIntegerStore(V, SI, AATags);
+      return rewriteIntegerStore(V, SI);
 
     StoreInst *NewSI;
     if (NewBeginOffset == NewAllocaBeginOffset &&
@@ -3759,11 +3734,7 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
       NewSI =
           IRB.CreateAlignedStore(V, NewPtr, getSliceAlign(), SI.isVolatile());
     }
-    NewSI->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
-                             LLVMContext::MD_access_group});
-    if (AATags)
-      NewSI->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
-                                                  V->getType(), DL));
+    copyMetadataForMemoryAccess(*NewSI, SI, NewBeginOffset - BeginOffset);
     if (SI.isVolatile())
       NewSI->setAtomic(SI.getOrdering(), SI.getSyncScopeID());
     if (NewSI->isAtomic())
@@ -3942,11 +3913,7 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
     Value *NewPtr = getPtrToNewAI(II.getDestAddressSpace(), II.isVolatile());
     StoreInst *New =
         IRB.CreateAlignedStore(V, NewPtr, NewAI.getAlign(), II.isVolatile());
-    New->copyMetadata(II, {LLVMContext::MD_mem_parallel_loop_access,
-                           LLVMContext::MD_access_group});
-    if (AATags)
-      New->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
-                                                V->getType(), DL));
+    copyMetadataForMemoryAccess(*New, II, NewBeginOffset - BeginOffset);
 
     migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &II,
                      New, New->getPointerOperand(), V, DL);
@@ -4144,11 +4111,7 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
     } else {
       LoadInst *Load = IRB.CreateAlignedLoad(OtherTy, SrcPtr, SrcAlign,
                                              II.isVolatile(), "copyload");
-      Load->copyMetadata(II, {LLVMContext::MD_mem_parallel_loop_access,
-                              LLVMContext::MD_access_group});
-      if (AATags)
-        Load->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
-                                                   Load->getType(), DL));
+      copyMetadataForMemoryAccess(*Load, II, NewBeginOffset - BeginOffset);
       Src = Load;
     }
 
@@ -4167,11 +4130,7 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
 
     StoreInst *Store = cast<StoreInst>(
         IRB.CreateAlignedStore(Src, DstPtr, DstAlign, II.isVolatile()));
-    Store->copyMetadata(II, {LLVMContext::MD_mem_parallel_loop_access,
-                             LLVMContext::MD_access_group});
-    if (AATags)
-      Store->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
-                                                  Src->getType(), DL));
+    copyMetadataForMemoryAccess(*Store, II, NewBeginOffset - BeginOffset);
 
     APInt Offset(DL.getIndexTypeSizeInBits(DstPtr->getType()), 0);
     if (IsDest) {
@@ -4455,7 +4414,7 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
   };
 
   struct LoadOpSplitter : public OpSplitter<LoadOpSplitter> {
-    AAMDNodes AATags;
+    LoadInst *AggLoad;
     // A vector to hold the split components that we want to emit
     // separate fake uses for.
     SmallVector<Value *, 4> Components;
@@ -4464,11 +4423,11 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
     SmallVector<Instruction *, 1> FakeUses;
 
     LoadOpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
-                   AAMDNodes AATags, Align BaseAlign, const DataLayout &DL,
+                   LoadInst *AggLoad, Align BaseAlign, const DataLayout &DL,
                    IRBuilderTy &IRB)
         : OpSplitter<LoadOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign, DL,
                                      IRB),
-          AATags(AATags) {}
+          AggLoad(AggLoad) {}
 
     /// Emit a leaf load of a single value. This is called at the leaves of the
     /// recursive emission to actually load values.
@@ -4482,10 +4441,12 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
 
       APInt Offset(
           DL.getIndexSizeInBits(Ptr->getType()->getPointerAddressSpace()), 0);
-      if (AATags &&
-          GEPOperator::accumulateConstantOffset(BaseTy, GEPIndices, DL, Offset))
-        Load->setAAMetadata(
-            AATags.adjustForAccess(Offset.getZExtValue(), Load->getType(), DL));
+      bool KnownOffset =
+          GEPOperator::accumulateConstantOffset(BaseTy, GEPIndices, DL, Offset);
+      copyMetadataForMemoryAccess(
+          *Load, *AggLoad,
+          KnownOffset ? std::optional<int64_t>(Offset.getZExtValue())
+                      : std::nullopt);
       // Record the load so we can generate a fake use for this aggregate
       // component.
       Components.push_back(Load);
@@ -4521,7 +4482,7 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
 
     // We have an aggregate being loaded, split it apart.
     LLVM_DEBUG(dbgs() << "    original: " << LI << "\n");
-    LoadOpSplitter Splitter(&LI, *U, LI.getType(), LI.getAAMetadata(),
+    LoadOpSplitter Splitter(&LI, *U, LI.getType(), &LI,
                             getAdjustedAlignment(&LI, 0), DL, IRB);
     Splitter.recordFakeUses(LI);
     Value *V = PoisonValue::get(LI.getType());
@@ -4535,12 +4496,11 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
 
   struct StoreOpSplitter : public OpSplitter<StoreOpSplitter> {
     StoreOpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
-                    AAMDNodes AATags, StoreInst *AggStore, Align BaseAlign,
-                    const DataLayout &DL, IRBuilderTy &IRB)
+                    StoreInst *AggStore, Align BaseAlign, const DataLayout &DL,
+                    IRBuilderTy &IRB)
         : OpSplitter<StoreOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign,
                                       DL, IRB),
-          AATags(AATags), AggStore(AggStore) {}
-    AAMDNodes AATags;
+          AggStore(AggStore) {}
     StoreInst *AggStore;
     /// Emit a leaf store of a single value. This is called at the leaves of the
     /// recursive emission to actually produce stores.
@@ -4560,10 +4520,7 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
       APInt Offset(
           DL.getIndexSizeInBits(Ptr->getType()->getPointerAddressSpace()), 0);
       GEPOperator::accumulateConstantOffset(BaseTy, GEPIndices, DL, Offset);
-      if (AATags) {
-        Store->setAAMetadata(AATags.adjustForAccess(
-            Offset.getZExtValue(), ExtractValue->getType(), DL));
-      }
+      copyMetadataForMemoryAccess(*Store, *AggStore, Offset.getZExtValue());
 
       // migrateDebugInfo requires the base Alloca. Walk to it from this gep.
       // If we cannot (because there's an intervening non-const or unbounded
@@ -4595,7 +4552,7 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
 
     // We have an aggregate being stored, split it apart.
     LLVM_DEBUG(dbgs() << "    original: " << SI << "\n");
-    StoreOpSplitter Splitter(&SI, *U, V->getType(), SI.getAAMetadata(), &SI,
+    StoreOpSplitter Splitter(&SI, *U, V->getType(), &SI,
                              getAdjustedAlignment(&SI, 0), DL, IRB);
     Splitter.emitSplitOps(V->getType(), V, V->getName() + ".fca");
     Visited.erase(&SI);
@@ -5043,8 +5000,7 @@ class SliceSplitter {
                        PartPtrTy, BasePtr->getName() + "."),
         getAdjustedAlignment(SplitI, It.getOffset()),
         /*IsVolatile*/ false, SplitI->getName());
-    PLoad->copyMetadata(*SplitI, {LLVMContext::MD_mem_parallel_loop_access,
-                                  LLVMContext::MD_access_group});
+    copyMetadataForMemoryAccess(*PLoad, *SplitI, It.getOffset());
     return PLoad;
   }
 
@@ -5059,13 +5015,8 @@ class SliceSplitter {
                        PartPtrTy, BasePtr->getName() + "."),
         getAdjustedAlignment(SplitI, It.getOffset()),
         /*IsVolatile*/ false);
-    PStore->copyMetadata(*SplitI, {LLVMContext::MD_mem_parallel_loop_access,
-                                   LLVMContext::MD_access_group,
-                                   LLVMContext::MD_DIAssignID});
-    AAMDNodes AATags = SplitI->getAAMetadata();
-    if (AATags)
-      PStore->setAAMetadata(
-          AATags.adjustForAccess(It.getOffset(), StoreVal->getType(), DL));
+    copyMetadataForMemoryAccess(*PStore, *SplitI, It.getOffset());
+    PStore->copyMetadata(*SplitI, LLVMContext::MD_DIAssignID);
 
     return PStore;
   }
diff --git a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
index d8fd2603dd1f3f..d8d07d7b09f83c 100644
--- a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
+++ b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
@@ -16,6 +16,7 @@
 #include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
+#include "llvm/Analysis/MetadataUtils.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/BasicBlock.h"
@@ -113,61 +114,6 @@ static unsigned adjustForEndian(const DataLayout &DL, unsigned VectorWidth,
   return DL.isBigEndian() ? VectorWidth - 1 - Idx : Idx;
 }
 
-static void copyMetadataForMemoryAccess(
-    Instruction &Dest, const Instruction &Source, const DataLayout &DL,
-    unsigned SourcePtrOperand, unsigned DestPtrOperand, Type *AccessType,
-    bool IsWholeAccess, std::optional<size_t> ByteOffset) {
-  // Only propagate metadata that is valid on each constituent memory access.
-  // In particular, do not copy metadata whose meaning is tied to the call,
-  // such as !prof or !callsite.
-  Dest.copyMetadata(Source,
-                    {LLVMContext::MD_nontemporal,
-                     LLVMContext::MD_mem_parallel_loop_access,
-                     LLVMContext::MD_access_group, LLVMContext::MD_annotation,
-                     LLVMContext::MD_nosanitize, LLVMContext::MD_mmra});
-
-  AAMDNodes AANodes = Source.getAAMetadata();
-  if (IsWholeAccess)
-    Dest.setAAMetadata(AANodes);
-  else if (ByteOffset)
-    Dest.setAAMetadata(AANodes.adjustForAccess(*ByteOffset, AccessType, DL));
-  else {
-    // The packed address is runtime-dependent. The other AA metadata remains
-    // applicable, but !tbaa.struct cannot be adjusted to a known byte range.
-    AANodes.TBAAStruct = nullptr;
-    Dest.setAAMetadata(AANodes);
-  }
-  setMemCacheHint(Dest, DestPtrOperand,
-                  getMemCacheHint(Source, SourcePtrOperand));
-}
-
-static void copyMetadataForScalarizedLoad(LoadInst &Dest,
-                                          const Instruction &Source,
-                                          const DataLayout &DL,
-                                          unsigned SourcePtrOperand,
-                                          std::optional<size_t> ByteOffset) {
-  copyMetadataForMemoryAccess(Dest, Source, DL, SourcePtrOperand,
-                              Dest.getPointerOperandIndex(), Dest.getType(),
-                              Dest.getType() == Source.getType(), ByteOffset);
-
-  // !range applies element-wise to vectors, so the same range describes each
-  // scalar result. The other metadata here also describes the loaded result.
-  Dest.copyMetadata(Source, {LLVMContext::MD_fpmath, LLVMContext::MD_range,
-                             LLVMContext::MD_invariant_load});
-}
-
-static void copyMetadataForScalarizedStore(StoreInst &Dest,
-                                           const Instruction &Source,
-                                           const DataLayout &DL,
-                                           unsigned SourcePtrOperand,
-                                           std::optional<size_t> ByteOffset) {
-  copyMetadataForMemoryAccess(
-      Dest, Source, DL, SourcePtrOperand, Dest.getPointerOperandIndex(),
-      Dest.getValueOperand()->getType(),
-      Dest.getValueOperand()->getType() == Source.getOperand(0)->getType(),
-      ByteOffset);
-}
-
 // Translate a masked load intrinsic like
 // <16 x i32 > @llvm.masked.load( <16 x i32>* %addr,
 //                               <16 x i1> %mask, <16 x i32> %passthru)
@@ -220,8 +166,7 @@ static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence,
   // Short-cut if the mask is all-true.
   if (isa<Constant>(Mask) && cast<Constant>(Mask)->isAllOnesValue()) {
     LoadInst *NewI = Builder.CreateAlignedLoad(VecType, Ptr, AlignVal);
-    copyMetadataForScalarizedLoad(*NewI, *CI, DL, /*SourcePtrOperand=*/0,
-                                  std::nullopt);
+    copyMetadataForMemoryAccess(*NewI, *CI);
     NewI->takeName(CI);
     CI->replaceAllUsesWith(NewI);
     CI->eraseFromParent();
@@ -242,9 +187,8 @@ static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence,
         continue;
       Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
       LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Gep, AdjustedAlignVal);
-      copyMetadataForScalarizedLoad(
-          *Load, *CI, DL, /*SourcePtrOperand=*/0,
-          Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
+      copyMetadataForMemoryAccess(
+          *Load, *CI, Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
       VResult = Builder.CreateInsertElement(VResult, Load, Idx);
     }
     CI->replaceAllUsesWith(VResult);
@@ -271,8 +215,7 @@ static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence,
     Builder.SetInsertPoint(CondBlock->getTerminator());
     LoadInst *Load = Builder.CreateAlignedLoad(VecType, Ptr, AlignVal,
                                                CI->getName() + ".cond.load");
-    copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
-                                  std::nullopt);
+    copyMetadataForMemoryAccess(*Load, *CI);
 
     BasicBlock *PostLoad = ThenTerm->getSuccessor(0);
     Builder.SetInsertPoint(PostLoad->begin());
@@ -335,9 +278,8 @@ static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence,
     Builder.SetInsertPoint(CondBlock->getTerminator());
     Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
     LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Gep, AdjustedAlignVal);
-    copyMetadataForScalarizedLoad(
-        *Load, *CI, DL, /*SourcePtrOperand=*/0,
-        Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
+    copyMetadataForMemoryAccess(
+        *Load, *CI, Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
     Value *NewVResult = Builder.CreateInsertElement(VResult, Load, Idx);
 
     // Create "else" block, fill it in the next iteration
@@ -406,8 +348,7 @@ static void scalarizeMaskedStore(const DataLayout &DL, bool HasBranchDivergence,
   if (isa<Constant>(Mask) && cast<Constant>(Mask)->isAllOnesValue()) {
     StoreInst *Store = Builder.CreateAlignedStore(Src, Ptr, AlignVal);
     Store->takeName(CI);
-    copyMetadataForScalarizedStore(*Store, *CI, DL, /*SourcePtrOperand=*/1,
-                                   std::nullopt);
+    copyMetadataForMemoryAccess(*Store, *CI);
     // This is a one-to-one replacement, so the assignment link remains valid.
     Store->copyMetadata(*CI, LLVMContext::MD_DIAssignID);
     CI->eraseFromParent();
@@ -427,9 +368,8 @@ static void scalarizeMaskedStore(const DataLayout &DL, bool HasBranchDivergence,
       Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
       StoreInst *Store =
           Builder.CreateAlignedStore(OneElt, Gep, AdjustedAlignVal);
-      copyMetadataForScalarizedStore(
-          *Store, *CI, DL, /*SourcePtrOperand=*/1,
-          Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
+      copyMetadataForMemoryAccess(
+          *Store, *CI, Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
     }
     CI->eraseFromParent();
     return;
@@ -454,8 +394,7 @@ static void scalarizeMaskedStore(const DataLayout &DL, bool HasBranchDivergence,
 
     StoreInst *Store = Builder.CreateAlignedStore(Src, Ptr, AlignVal);
     Store->takeName(CI);
-    copyMetadataForScalarizedStore(*Store, *CI, DL, /*SourcePtrOperand=*/1,
-                                   std::nullopt);
+    copyMetadataForMemoryAccess(*Store, *CI);
     // This is a one-to-one replacement, so the assignment link remains valid.
     Store->copyMetadata(*CI, LLVMContext::MD_DIAssignID);
 
@@ -515,9 +454,8 @@ static void scalarizeMaskedStore(const DataLayout &DL, bool HasBranchDivergence,
     Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);
     StoreInst *Store =
         Builder.CreateAlignedStore(OneElt, Gep, AdjustedAlignVal);
-    copyMetadataForScalarizedStore(
-        *Store, *CI, DL, /*SourcePtrOperand=*/1,
-        Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
+    copyMetadataForMemoryAccess(
+        *Store, *CI, Idx * DL.getTypeAllocSize(EltTy).getFixedValue());
 
     // Create "else" block, fill it in the next iteration
     BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
@@ -586,8 +524,7 @@ static void scalarizeMaskedGather(const DataLayout &DL,
       Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
       LoadInst *Load =
           Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx));
-      copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
-                                    /*ByteOffset=*/0);
+      copyMetadataForMemoryAccess(*Load, *CI, /*ByteOffset=*/0);
       VResult =
           Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx));
     }
@@ -647,8 +584,7 @@ static void scalarizeMaskedGather(const DataLayout &DL,
     Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
     LoadInst *Load =
         Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx));
-    copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
-                                  /*ByteOffset=*/0);
+    copyMetadataForMemoryAccess(*Load, *CI, /*ByteOffset=*/0);
     Value *NewVResult =
         Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx));
 
@@ -727,9 +663,7 @@ static void scalarizeMaskedScatter(const DataLayout &DL,
           Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));
       Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
       StoreInst *Store = Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
-      copyMetadataForScalarizedStore(*Store, *CI, DL,
-                                     /*SourcePtrOperand=*/1,
-                                     /*ByteOffset=*/0);
+      copyMetadataForMemoryAccess(*Store, *CI, /*ByteOffset=*/0);
     }
     CI->eraseFromParent();
     return;
@@ -784,9 +718,7 @@ static void scalarizeMaskedScatter(const DataLayout &DL,
     Value *OneElt = Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));
     Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
     StoreInst *Store = Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
-    copyMetadataForScalarizedStore(*Store, *CI, DL,
-                                   /*SourcePtrOperand=*/1,
-                                   /*ByteOffset=*/0);
+    copyMetadataForMemoryAccess(*Store, *CI, /*ByteOffset=*/0);
 
     // Create "else" block, fill it in the next iteration
     BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
@@ -842,9 +774,8 @@ static void scalarizeMaskedExpandLoad(const DataLayout &DL,
             Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, MemIndex);
         LoadInst *Load = Builder.CreateAlignedLoad(
             EltTy, NewPtr, AdjustedAlignment, "Load" + Twine(Idx));
-        copyMetadataForScalarizedLoad(
-            *Load, *CI, DL, /*SourcePtrOperand=*/0,
-            MemIndex * DL.getTypeAllocSize(EltTy).getFixedValue());
+        copyMetadataForMemoryAccess(
+            *Load, *CI, MemIndex * DL.getTypeAllocSize(EltTy).getFixedValue());
         InsertElt = Load;
         ShuffleMask[Idx] = Idx;
         ++MemIndex;
@@ -907,8 +838,7 @@ static void scalarizeMaskedExpandLoad(const DataLayout &DL,
 
     Builder.SetInsertPoint(CondBlock->getTerminator());
     LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Ptr, AdjustedAlignment);
-    copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
-                                  std::nullopt);
+    copyMetadataForMemoryAccess(*Load, *CI, std::nullopt);
     Value *NewVResult = Builder.CreateInsertElement(VResult, Load, Idx);
 
     // Move the pointer if there are more blocks to come.
@@ -979,9 +909,8 @@ static void scalarizeMaskedCompressStore(const DataLayout &DL,
       Value *NewPtr = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, MemIndex);
       StoreInst *Store =
           Builder.CreateAlignedStore(OneElt, NewPtr, AdjustedAlignment);
-      copyMetadataForScalarizedStore(
-          *Store, *CI, DL, /*SourcePtrOperand=*/1,
-          MemIndex * DL.getTypeAllocSize(EltTy).getFixedValue());
+      copyMetadataForMemoryAccess(
+          *Store, *CI, MemIndex * DL.getTypeAllocSize(EltTy).getFixedValue());
       ++MemIndex;
     }
     CI->eraseFromParent();
@@ -1037,8 +966,7 @@ static void scalarizeMaskedCompressStore(const DataLayout &DL,
     Value *OneElt = Builder.CreateExtractElement(Src, Idx);
     StoreInst *Store =
         Builder.CreateAlignedStore(OneElt, Ptr, AdjustedAlignment);
-    copyMetadataForScalarizedStore(*Store, *CI, DL, /*SourcePtrOperand=*/1,
-                                   std::nullopt);
+    copyMetadataForMemoryAccess(*Store, *CI, std::nullopt);
 
     // Move the pointer if there are more blocks to come.
     Value *NewPtr;
@@ -1115,14 +1043,11 @@ static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI,
         continue;
       Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
       LoadInst *Load = Builder.CreateLoad(EltTy, Ptr, "Load" + Twine(Idx));
-      copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
-                                    /*ByteOffset=*/0);
+      copyMetadataForMemoryAccess(*Load, *CI, /*ByteOffset=*/0);
       Value *Update =
           CreateHistogramUpdateValue(cast<IntrinsicInst>(CI), Load, Inc);
       StoreInst *Store = Builder.CreateStore(Update, Ptr);
-      copyMetadataForScalarizedStore(*Store, *CI, DL,
-                                     /*SourcePtrOperand=*/0,
-                                     /*ByteOffset=*/0);
+      copyMetadataForMemoryAccess(*Store, *CI, /*ByteOffset=*/0);
     }
     CI->eraseFromParent();
     return;
@@ -1146,14 +1071,11 @@ static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI,
     Builder.SetInsertPoint(CondBlock->getTerminator());
     Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
     LoadInst *Load = Builder.CreateLoad(EltTy, Ptr, "Load" + Twine(Idx));
-    copyMetadataForScalarizedLoad(*Load, *CI, DL, /*SourcePtrOperand=*/0,
-                                  /*ByteOffset=*/0);
+    copyMetadataForMemoryAccess(*Load, *CI, /*ByteOffset=*/0);
     Value *UpdateOp =
         CreateHistogramUpdateValue(cast<IntrinsicInst>(CI), Load, Inc);
     StoreInst *Store = Builder.CreateStore(UpdateOp, Ptr);
-    copyMetadataForScalarizedStore(*Store, *CI, DL,
-                                   /*SourcePtrOperand=*/0,
-                                   /*ByteOffset=*/0);
+    copyMetadataForMemoryAccess(*Store, *CI, /*ByteOffset=*/0);
 
     // Create "else" block, fill it in the next iteration
     BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
diff --git a/llvm/lib/Transforms/Scalar/Scalarizer.cpp b/llvm/lib/Transforms/Scalar/Scalarizer.cpp
index 5af07a4400aa98..0468786e635aef 100644
--- a/llvm/lib/Transforms/Scalar/Scalarizer.cpp
+++ b/llvm/lib/Transforms/Scalar/Scalarizer.cpp
@@ -18,6 +18,7 @@
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/Analysis/MetadataUtils.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/Argument.h"
@@ -310,7 +311,6 @@ class ScalarizerVisitor : public InstVisitor<ScalarizerVisitor, bool> {
   Scatterer scatter(Instruction *Point, Value *V, const VectorSplit &VS);
   void gather(Instruction *Op, const ValueVector &CV, const VectorSplit &VS);
   void replaceUses(Instruction *Op, Value *CV);
-  bool canTransferMetadata(unsigned Kind);
   void transferMetadataAndIRFlags(Instruction *Op, const ValueVector &CV);
   std::optional<VectorSplit> getVectorSplit(Type *Ty);
   std::optional<VectorLayout> getVectorLayout(Type *Ty, Align Alignment,
@@ -543,33 +543,16 @@ void ScalarizerVisitor::replaceUses(Instruction *Op, Value *CV) {
   }
 }
 
-// Return true if it is safe to transfer the given metadata tag from
-// vector to scalar instructions.
-bool ScalarizerVisitor::canTransferMetadata(unsigned Tag) {
-  return (Tag == LLVMContext::MD_tbaa
-          || Tag == LLVMContext::MD_fpmath
-          || Tag == LLVMContext::MD_tbaa_struct
-          || Tag == LLVMContext::MD_invariant_load
-          || Tag == LLVMContext::MD_alias_scope
-          || Tag == LLVMContext::MD_noalias
-          || Tag == LLVMContext::MD_mem_parallel_loop_access
-          || Tag == LLVMContext::MD_access_group);
-}
-
-// Transfer metadata from Op to the instructions in CV if it is known
-// to be safe to do so.
+// Transfer metadata and flags to each scalarized fragment.
 void ScalarizerVisitor::transferMetadataAndIRFlags(Instruction *Op,
                                                    const ValueVector &CV) {
-  SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
-  Op->getAllMetadataOtherThanDebugLoc(MDs);
+  uint64_t Offset = 0;
   for (Value *V : CV) {
-    if (Instruction *New = dyn_cast<Instruction>(V)) {
-      for (const auto &MD : MDs)
-        if (canTransferMetadata(MD.first))
-          New->setMetadata(MD.first, MD.second);
+    if (auto *New = dyn_cast<Instruction>(V)) {
+      copyMetadataForScalarizedInstruction(*New, *Op, Offset);
       New->copyIRFlags(Op);
-      if (Op->getDebugLoc() && !New->getDebugLoc())
-        New->setDebugLoc(Op->getDebugLoc());
+      if (isa<LoadInst, StoreInst>(New))
+        Offset += Op->getDataLayout().getTypeStoreSize(getLoadStoreType(New));
     }
   }
 }
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index 8b6d53483dca04..b98f7dcdf1b8e5 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -27,6 +27,7 @@
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/Analysis/MemorySSAUpdater.h"
+#include "llvm/Analysis/MetadataUtils.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Analysis/VectorUtils.h"
@@ -2990,262 +2991,6 @@ bool llvm::removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU,
   return Changed;
 }
 
-/// If AAOnly is set, only intersect alias analysis metadata and preserve other
-/// known metadata. Unknown metadata is always dropped.
-static void combineMetadata(Instruction *K, const Instruction *J,
-                            bool DoesKMove, bool AAOnly = false) {
-  SmallVector<std::pair<unsigned, MDNode *>, 4> Metadata;
-  K->getAllMetadataOtherThanDebugLoc(Metadata);
-  for (const auto &MD : Metadata) {
-    unsigned Kind = MD.first;
-    MDNode *JMD = J->getMetadata(Kind);
-    MDNode *KMD = MD.second;
-
-    // TODO: Assert that this switch is exhaustive for fixed MD kinds.
-    switch (Kind) {
-      default:
-        K->setMetadata(Kind, nullptr); // Remove unknown metadata
-        break;
-      case LLVMContext::MD_dbg:
-        llvm_unreachable("getAllMetadataOtherThanDebugLoc returned a MD_dbg");
-      case LLVMContext::MD_DIAssignID:
-        if (!AAOnly)
-          K->mergeDIAssignID(J);
-        break;
-      case LLVMContext::MD_tbaa:
-        if (DoesKMove)
-          K->setMetadata(Kind, MDNode::getMostGenericTBAA(JMD, KMD));
-        break;
-      case LLVMContext::MD_alias_scope:
-        if (DoesKMove)
-          K->setMetadata(Kind, MDNode::getMostGenericAliasScope(JMD, KMD));
-        break;
-      case LLVMContext::MD_noalias:
-      case LLVMContext::MD_mem_parallel_loop_access:
-        if (DoesKMove)
-          K->setMetadata(Kind, MDNode::intersect(JMD, KMD));
-        break;
-      case LLVMContext::MD_access_group:
-        if (DoesKMove)
-          K->setMetadata(LLVMContext::MD_access_group,
-                         intersectAccessGroups(K, J));
-        break;
-      case LLVMContext::MD_range:
-        if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
-          K->setMetadata(Kind, MDNode::getMostGenericRange(JMD, KMD));
-        break;
-      case LLVMContext::MD_nofpclass:
-        if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
-          K->setMetadata(Kind, MDNode::getMostGenericNoFPClass(JMD, KMD));
-        break;
-      case LLVMContext::MD_fpmath:
-        if (!AAOnly)
-          K->setMetadata(Kind, MDNode::getMostGenericFPMath(JMD, KMD));
-        break;
-      case LLVMContext::MD_invariant_load:
-      case LLVMContext::MD_invariant_group:
-        // If K moves, only keep the invariant metadata if it is present on
-        // both instructions; otherwise the invariant would be asserted on a
-        // path (J's) that never promised it. If K does not move, K stays on
-        // its original path, so its existing metadata remains valid.
-        if (DoesKMove)
-          K->setMetadata(Kind, JMD);
-        break;
-      case LLVMContext::MD_nonnull:
-        if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
-          K->setMetadata(Kind, JMD);
-        break;
-      // Keep empty cases for prof, mmra, memprof, and callsite to prevent them
-      // from being removed as unknown metadata. The actual merging is handled
-      // separately below.
-      case LLVMContext::MD_prof:
-      case LLVMContext::MD_mmra:
-      case LLVMContext::MD_memprof:
-      case LLVMContext::MD_callsite:
-        break;
-      case LLVMContext::MD_callee_type:
-        if (!AAOnly) {
-          K->setMetadata(LLVMContext::MD_callee_type,
-                         MDNode::getMergedCalleeTypeMetadata(KMD, JMD));
-        }
-        break;
-      case LLVMContext::MD_callees:
-        // If K moves, it replaces J on J's path and must allow J's callees as
-        // well. If K does not move, its callees remain valid.
-        if (!AAOnly && DoesKMove)
-          K->setMetadata(Kind, MDNode::getMergedCalleesMetadata(KMD, JMD));
-        break;
-      case LLVMContext::MD_align:
-        if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
-          K->setMetadata(
-              Kind, MDNode::getMostGenericAlignmentOrDereferenceable(JMD, KMD));
-        break;
-      case LLVMContext::MD_dereferenceable:
-      case LLVMContext::MD_dereferenceable_or_null:
-        if (!AAOnly && DoesKMove)
-          K->setMetadata(Kind,
-            MDNode::getMostGenericAlignmentOrDereferenceable(JMD, KMD));
-        break;
-      case LLVMContext::MD_preserve_access_index:
-        // Preserve !preserve.access.index in K.
-        break;
-      case LLVMContext::MD_noundef:
-        // If K does move, keep noundef if it is present in both instructions.
-        if (!AAOnly && DoesKMove)
-          K->setMetadata(Kind, JMD);
-        break;
-      case LLVMContext::MD_nontemporal:
-        // Preserve !nontemporal if it is present on both instructions.
-        if (!AAOnly)
-          K->setMetadata(Kind, JMD);
-        break;
-      case LLVMContext::MD_mem_cache_hint:
-        // Preserve !mem.cache_hint only if it is present and equivalent on both
-        // instructions.
-        if (!AAOnly && KMD != JMD)
-          K->setMetadata(Kind, nullptr);
-        break;
-      case LLVMContext::MD_noalias_addrspace:
-        if (DoesKMove)
-          K->setMetadata(Kind,
-                         MDNode::getMostGenericNoaliasAddrspace(JMD, KMD));
-        break;
-      case LLVMContext::MD_nosanitize:
-        // Preserve !nosanitize if both K and J have it.
-        K->setMetadata(Kind, JMD);
-        break;
-      case LLVMContext::MD_captures:
-        K->setMetadata(
-            Kind, MDNode::fromCaptureComponents(
-                      K->getContext(), MDNode::toCaptureComponents(JMD) |
-                                           MDNode::toCaptureComponents(KMD)));
-        break;
-      case LLVMContext::MD_alloc_token:
-        if (!AAOnly && KMD != JMD)
-          K->setMetadata(Kind, MDNode::getMergedAllocTokenMetadata(KMD, JMD));
-        break;
-      }
-  }
-
-  // Merge MMRAs.
-  // This is handled separately because we also want to handle cases where K
-  // doesn't have tags but J does.
-  auto JMMRA = J->getMetadata(LLVMContext::MD_mmra);
-  auto KMMRA = K->getMetadata(LLVMContext::MD_mmra);
-  if (JMMRA || KMMRA) {
-    K->setMetadata(LLVMContext::MD_mmra,
-                   MMRAMetadata::combine(K->getContext(), JMMRA, KMMRA));
-  }
-
-  // Merge memprof metadata.
-  // Handle separately to support cases where only one instruction has the
-  // metadata.
-  auto *JMemProf = J->getMetadata(LLVMContext::MD_memprof);
-  auto *KMemProf = K->getMetadata(LLVMContext::MD_memprof);
-  if (!AAOnly && (JMemProf || KMemProf)) {
-    K->setMetadata(LLVMContext::MD_memprof,
-                   MDNode::getMergedMemProfMetadata(KMemProf, JMemProf));
-  }
-
-  // Merge callsite metadata.
-  // Handle separately to support cases where only one instruction has the
-  // metadata.
-  auto *JCallSite = J->getMetadata(LLVMContext::MD_callsite);
-  auto *KCallSite = K->getMetadata(LLVMContext::MD_callsite);
-  if (!AAOnly && (JCallSite || KCallSite)) {
-    K->setMetadata(LLVMContext::MD_callsite,
-                   MDNode::getMergedCallsiteMetadata(KCallSite, JCallSite));
-  }
-
-  // Merge prof metadata.
-  // Handle separately to support cases where only one instruction has the
-  // metadata.
-  auto *JProf = J->getMetadata(LLVMContext::MD_prof);
-  auto *KProf = K->getMetadata(LLVMContext::MD_prof);
-  if (!AAOnly && (JProf || KProf)) {
-    // We do not need to merge metadata if we have select instructions and the
-    // one we're keeping (K) does not move (explicitly merging would drop the
-    // information in this case). Otherwise, explicitly handle the merging.
-    if (DoesKMove || (!isa<SelectInst>(J) || !isa<SelectInst>(K)) ||
-        ProfcheckDisableMetadataFixes)
-      K->setMetadata(LLVMContext::MD_prof,
-                     MDNode::getMergedProfMetadata(KProf, JProf, K, J));
-  }
-}
-
-void llvm::combineMetadataForCSE(Instruction *K, const Instruction *J,
-                                 bool DoesKMove) {
-  combineMetadata(K, J, DoesKMove);
-}
-
-void llvm::combineAAMetadata(Instruction *K, const Instruction *J) {
-  combineMetadata(K, J, /*DoesKMove=*/true, /*AAOnly=*/true);
-}
-
-void llvm::copyMetadataForLoad(LoadInst &Dest, const LoadInst &Source) {
-  SmallVector<std::pair<unsigned, MDNode *>, 8> MD;
-  Source.getAllMetadata(MD);
-  MDBuilder MDB(Dest.getContext());
-  Type *NewType = Dest.getType();
-  const DataLayout &DL = Source.getDataLayout();
-  for (const auto &MDPair : MD) {
-    unsigned ID = MDPair.first;
-    MDNode *N = MDPair.second;
-    // Note, essentially every kind of metadata should be preserved here! This
-    // routine is supposed to clone a load instruction changing *only its type*.
-    // The only metadata it makes sense to drop is metadata which is invalidated
-    // when the pointer type changes. This should essentially never be the case
-    // in LLVM, but we explicitly switch over only known metadata to be
-    // conservatively correct. If you are adding metadata to LLVM which pertains
-    // to loads, you almost certainly want to add it here.
-    switch (ID) {
-    case LLVMContext::MD_dbg:
-    case LLVMContext::MD_tbaa:
-    case LLVMContext::MD_prof:
-    case LLVMContext::MD_fpmath:
-    case LLVMContext::MD_tbaa_struct:
-    case LLVMContext::MD_invariant_load:
-    case LLVMContext::MD_alias_scope:
-    case LLVMContext::MD_noalias:
-    case LLVMContext::MD_nontemporal:
-    case LLVMContext::MD_mem_cache_hint:
-    case LLVMContext::MD_mem_parallel_loop_access:
-    case LLVMContext::MD_access_group:
-    case LLVMContext::MD_noundef:
-    case LLVMContext::MD_noalias_addrspace:
-    case LLVMContext::MD_invariant_group:
-    case LLVMContext::MD_mmra:
-      // All of these directly apply.
-      Dest.setMetadata(ID, N);
-      break;
-
-    case LLVMContext::MD_nonnull:
-      copyNonnullMetadata(Source, N, Dest);
-      break;
-
-    case LLVMContext::MD_align:
-    case LLVMContext::MD_dereferenceable:
-    case LLVMContext::MD_dereferenceable_or_null:
-      // These only directly apply if the new type is also a pointer.
-      if (NewType->isPointerTy())
-        Dest.setMetadata(ID, N);
-      break;
-
-    case LLVMContext::MD_range:
-      copyRangeMetadata(DL, Source, N, Dest);
-      break;
-
-    case LLVMContext::MD_nofpclass:
-      // This only applies if the floating-point type interpretation. This
-      // should handle degenerate cases like casting between a scalar and single
-      // element vector.
-      if (NewType->getScalarType() == Source.getType()->getScalarType())
-        Dest.setMetadata(ID, N);
-      break;
-    }
-  }
-}
-
 void llvm::patchReplacementInstruction(Instruction *I, Value *Repl,
                                        bool DoesKMove) {
   auto *ReplInst = dyn_cast<Instruction>(Repl);
@@ -3398,55 +3143,6 @@ bool llvm::callsGCLeafFunction(const CallBase *Call,
   return TLI.has(TLI.getLibFunc(*Call));
 }
 
-void llvm::copyNonnullMetadata(const LoadInst &OldLI, MDNode *N,
-                               LoadInst &NewLI) {
-  auto *NewTy = NewLI.getType();
-
-  // This only directly applies if the new type is also a pointer.
-  if (NewTy->isPointerTy()) {
-    NewLI.setMetadata(LLVMContext::MD_nonnull, N);
-    return;
-  }
-
-  // The only other translation we can do is to integral loads with !range
-  // metadata.
-  if (!NewTy->isIntegerTy())
-    return;
-
-  MDBuilder MDB(NewLI.getContext());
-  const Value *Ptr = OldLI.getPointerOperand();
-  auto *ITy = cast<IntegerType>(NewTy);
-  auto *NullInt = ConstantExpr::getPtrToInt(
-      ConstantPointerNull::get(cast<PointerType>(Ptr->getType())), ITy);
-  auto *NonNullInt = ConstantExpr::getAdd(NullInt, ConstantInt::get(ITy, 1));
-  NewLI.setMetadata(LLVMContext::MD_range,
-                    MDB.createRange(NonNullInt, NullInt));
-}
-
-void llvm::copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI,
-                             MDNode *N, LoadInst &NewLI) {
-  auto *NewTy = NewLI.getType();
-  // Simply copy the metadata if the type did not change.
-  if (NewTy == OldLI.getType()) {
-    NewLI.setMetadata(LLVMContext::MD_range, N);
-    return;
-  }
-
-  // Give up unless it is converted to a pointer where there is a single very
-  // valuable mapping we can do reliably.
-  // FIXME: It would be nice to propagate this in more ways, but the type
-  // conversions make it hard.
-  if (!NewTy->isPointerTy())
-    return;
-
-  unsigned BitWidth = DL.getPointerTypeSizeInBits(NewTy);
-  if (BitWidth == OldLI.getType()->getScalarSizeInBits() &&
-      !getConstantRangeFromMetadata(*N).contains(APInt(BitWidth, 0))) {
-    MDNode *NN = MDNode::get(OldLI.getContext(), {});
-    NewLI.setMetadata(LLVMContext::MD_nonnull, NN);
-  }
-}
-
 void llvm::dropDebugUsers(Instruction &I) {
   SmallVector<DbgVariableRecord *, 1> DPUsers;
   findDbgUsers(&I, DPUsers);
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index f301b931d675aa..dff031f71773f6 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -72,6 +72,7 @@
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/MemoryLocation.h"
+#include "llvm/Analysis/MetadataUtils.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 50992fc3f89f44..bce9eed740fd69 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -50,6 +50,7 @@
 #include "llvm/Analysis/Loads.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/MemoryLocation.h"
+#include "llvm/Analysis/MetadataUtils.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 5d89141cde6f2e..4a185cbcf2f0c3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -35,6 +35,7 @@
 #include "llvm/ADT/ilist_node.h"
 #include "llvm/Analysis/IVDescriptors.h"
 #include "llvm/Analysis/MemoryLocation.h"
+#include "llvm/Analysis/MetadataUtils.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/DebugLoc.h"
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index ffdef6fbb1e521..bb2d24138d0965 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -24,6 +24,7 @@
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/InstSimplifyFolder.h"
 #include "llvm/Analysis/Loads.h"
+#include "llvm/Analysis/MetadataUtils.h"
 #include "llvm/Analysis/TargetFolder.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
@@ -359,7 +360,13 @@ bool VectorCombine::vectorizeLoadInsert(Instruction &I) {
   Builder.SetInsertPoint(Load);
   Value *CastedPtr =
       Builder.CreatePointerBitCastOrAddrSpaceCast(SrcPtr, Builder.getPtrTy(AS));
-  Value *VecLd = Builder.CreateAlignedLoad(MinVecTy, CastedPtr, Alignment);
+  auto *NewLoad = Builder.CreateAlignedLoad(MinVecTy, CastedPtr, Alignment);
+  copyMetadataForMemoryAccess(
+      *NewLoad, *Load,
+      SrcPtr == Load->getPointerOperand()->stripPointerCasts()
+          ? std::optional<int64_t>(0)
+          : -int64_t(OffsetEltIndex * (ScalarSize / 8)));
+  Value *VecLd = NewLoad;
   VecLd = Builder.CreateShuffleVector(VecLd, Mask);
 
   replaceValue(I, *VecLd);
@@ -424,7 +431,8 @@ bool VectorCombine::widenSubvectorLoad(Instruction &I) {
   Builder.SetInsertPoint(Load);
   Value *CastedPtr =
       Builder.CreatePointerBitCastOrAddrSpaceCast(SrcPtr, Builder.getPtrTy(AS));
-  Value *VecLd = Builder.CreateAlignedLoad(Ty, CastedPtr, Alignment);
+  auto *VecLd = Builder.CreateAlignedLoad(Ty, CastedPtr, Alignment);
+  copyMetadataForMemoryAccess(*VecLd, *Load);
   replaceValue(I, *VecLd);
   ++NumVecLoad;
   return true;
@@ -2136,11 +2144,11 @@ bool VectorCombine::foldInsertElementsToStores(Instruction &I) {
         {ConstantInt::get(GEPIdx->getType(), 0), GEPIdx});
 
     LastStore = Builder.CreateStore(InsertVal, GEP);
-    LastStore->copyMetadata(*SI);
-
-    // The new GEP may change the pointer operand, so !invariant.group cannot
-    // be transferred to the scalar store.
-    LastStore->setMetadata(LLVMContext::MD_invariant_group, nullptr);
+    std::optional<size_t> Offset;
+    if (auto *ConstIdx = dyn_cast<ConstantInt>(Idx))
+      Offset =
+          ConstIdx->getZExtValue() * DL->getTypeStoreSize(InsertVal->getType());
+    copyMetadataForMemoryAccess(*LastStore, *SI, Offset);
     Align ScalarOpAlignment = computeAlignmentAfterScalarization(
         std::max(SI->getAlign(), Load->getAlign()), InsertVal->getType(), Idx,
         *DL);
@@ -2307,11 +2315,10 @@ bool VectorCombine::scalarizeLoadExtract(LoadInst *LI, VectorType *VecTy,
         computeAlignmentAfterScalarization(LI->getAlign(), ElemType, Idx, *DL);
     NewLoad->setAlignment(ScalarOpAlignment);
 
-    if (auto *ConstIdx = dyn_cast<ConstantInt>(Idx)) {
-      size_t Offset = ConstIdx->getZExtValue() * DL->getTypeStoreSize(ElemType);
-      AAMDNodes OldAAMD = LI->getAAMetadata();
-      NewLoad->setAAMetadata(OldAAMD.adjustForAccess(Offset, ElemType, *DL));
-    }
+    std::optional<size_t> Offset;
+    if (auto *ConstIdx = dyn_cast<ConstantInt>(Idx))
+      Offset = ConstIdx->getZExtValue() * DL->getTypeStoreSize(ElemType);
+    copyMetadataForMemoryAccess(*NewLoad, *LI, Offset);
 
     replaceValue(*EI, *NewLoad, false);
   }
@@ -2378,7 +2385,7 @@ bool VectorCombine::scalarizeLoadBitcast(LoadInst *LI, VectorType *VecTy,
   auto *ScalarLoad =
       Builder.CreateLoad(TargetScalarType, Ptr, LI->getName() + ".scalar");
   ScalarLoad->setAlignment(LI->getAlign());
-  ScalarLoad->copyMetadata(*LI);
+  copyMetadataForMemoryAccess(*ScalarLoad, *LI);
 
   // Replace all bitcast users with the scalar load.
   for (User *U : LI->users()) {
@@ -6635,6 +6642,7 @@ bool VectorCombine::foldBitcastOfVPLoad(Instruction &I) {
   CallInst *NewVP = Builder.CreateIntrinsicWithoutFolding(
       NewVecTy, Intrinsic::vp_load,
       {II->getMemoryPointerParam(), NewMask, NewEVL});
+  copyMetadataForMemoryAccess(*NewVP, *II);
   // Preserve the original alignment.
   NewVP->addParamAttrs(
       0, AttrBuilder(II->getContext()).addAlignmentAttr(OrigAlign));
@@ -6891,7 +6899,7 @@ bool VectorCombine::shrinkLoadForShuffles(Instruction &I) {
       // Create new load of smaller vector.
       auto *NewLoad = cast<LoadInst>(
           Builder.CreateAlignedLoad(NewLoadTy, PtrOp, OldLoad->getAlign()));
-      NewLoad->copyMetadata(I);
+      copyMetadataForMemoryAccess(*NewLoad, *OldLoad);
 
       // Replace all uses.
       for (UseEntry &Use : NewUses) {
diff --git a/llvm/test/Transforms/InstCombine/memory-cache-hint.ll b/llvm/test/Transforms/InstCombine/memory-cache-hint.ll
index 653fc45056db27..a3645fd93d546c 100644
--- a/llvm/test/Transforms/InstCombine/memory-cache-hint.ll
+++ b/llvm/test/Transforms/InstCombine/memory-cache-hint.ll
@@ -4,7 +4,7 @@
 define void @store_bitcast(ptr %p, float %x) {
 ; CHECK-LABEL: define void @store_bitcast(
 ; CHECK-SAME: ptr [[P:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    store float [[X]], ptr [[P]], align 4
+; CHECK-NEXT:    store float [[X]], ptr [[P]], align 4, !mem.cache_hint [[META0:![0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
   %bits = bitcast float %x to i32
@@ -15,8 +15,8 @@ define void @store_bitcast(ptr %p, float %x) {
 define void @copy_both_hints(ptr %dst, ptr %src) {
 ; CHECK-LABEL: define void @copy_both_hints(
 ; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[SRC]], align 4
-; CHECK-NEXT:    store i32 [[TMP1]], ptr [[DST]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[SRC]], align 4, !mem.cache_hint [[META2:![0-9]+]]
+; CHECK-NEXT:    store i32 [[TMP1]], ptr [[DST]], align 4, !mem.cache_hint [[META0]]
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dst, ptr align 4 %src, i64 4, i1 false), !mem.cache_hint !2
@@ -26,7 +26,7 @@ define void @copy_both_hints(ptr %dst, ptr %src) {
 define void @set_destination_hint(ptr %dst) {
 ; CHECK-LABEL: define void @set_destination_hint(
 ; CHECK-SAME: ptr [[DST:%.*]]) {
-; CHECK-NEXT:    store i32 0, ptr [[DST]], align 4
+; CHECK-NEXT:    store i32 0, ptr [[DST]], align 4, !mem.cache_hint [[META0]]
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.memset.p0.i64(ptr align 4 %dst, i8 0, i64 4, i1 false), !mem.cache_hint !3
@@ -38,7 +38,7 @@ define void @set_destination_hint(ptr %dst) {
 define <2 x i32> @speculated_masked_load(ptr dereferenceable(8) %p, <2 x i1> %mask) {
 ; CHECK-LABEL: define <2 x i32> @speculated_masked_load(
 ; CHECK-SAME: ptr dereferenceable(8) [[P:%.*]], <2 x i1> [[MASK:%.*]]) {
-; CHECK-NEXT:    [[UNMASKEDLOAD:%.*]] = load <2 x i32>, ptr [[P]], align 1, !invariant.load [[META0:![0-9]+]], !mem.cache_hint [[META1:![0-9]+]]
+; CHECK-NEXT:    [[UNMASKEDLOAD:%.*]] = load <2 x i32>, ptr [[P]], align 1, !invariant.load [[META4:![0-9]+]], !mem.cache_hint [[META5:![0-9]+]]
 ; CHECK-NEXT:    [[V:%.*]] = select <2 x i1> [[MASK]], <2 x i32> [[UNMASKEDLOAD]], <2 x i32> zeroinitializer
 ; CHECK-NEXT:    ret <2 x i32> [[V]]
 ;
@@ -53,7 +53,10 @@ define <2 x i32> @speculated_masked_load(ptr dereferenceable(8) %p, <2 x i1> %ma
 !4 = !{!"nvvm.l1_eviction", !"first"}
 !6 = !{}
 ;.
-; CHECK: [[META0]] = !{}
-; CHECK: [[META1]] = !{i32 0, [[META2:![0-9]+]]}
-; CHECK: [[META2]] = !{!"nvvm.l1_eviction", !"last"}
+; CHECK: [[META0]] = !{i32 1, [[META1:![0-9]+]]}
+; CHECK: [[META1]] = !{!"nvvm.l1_eviction", !"last"}
+; CHECK: [[META2]] = !{i32 0, [[META3:![0-9]+]]}
+; CHECK: [[META3]] = !{!"nvvm.l1_eviction", !"first"}
+; CHECK: [[META4]] = !{}
+; CHECK: [[META5]] = !{i32 0, [[META1]]}
 ;.
diff --git a/llvm/test/Transforms/InstCombine/memory-metadata.ll b/llvm/test/Transforms/InstCombine/memory-metadata.ll
index 8ec8483485d263..b87598fb0098ed 100644
--- a/llvm/test/Transforms/InstCombine/memory-metadata.ll
+++ b/llvm/test/Transforms/InstCombine/memory-metadata.ll
@@ -6,7 +6,7 @@ target datalayout = "e-p:64:64"
 define float @bitcast_load(ptr %p) {
 ; CHECK-LABEL: define float @bitcast_load(
 ; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:    [[V1:%.*]] = load float, ptr [[P]], align 4, !tbaa [[INT_TBAA0:![0-9]+]], !invariant.load [[META4:![0-9]+]], !alias.scope [[META5:![0-9]+]], !noalias [[META5]], !nontemporal [[META8:![0-9]+]], !llvm.mem.parallel_loop_access [[META9:![0-9]+]], !invariant.group [[META4]], !llvm.access.group [[META9]], !noundef [[META4]], !annotation [[META10:![0-9]+]], !noalias.addrspace [[META11:![0-9]+]], !mem.cache_hint [[META12:![0-9]+]]
+; CHECK-NEXT:    [[V1:%.*]] = load float, ptr [[P]], align 4, !tbaa [[INT_TBAA0:![0-9]+]], !invariant.load [[META4:![0-9]+]], !alias.scope [[META5:![0-9]+]], !noalias [[META5]], !nontemporal [[META8:![0-9]+]], !llvm.mem.parallel_loop_access [[META9:![0-9]+]], !invariant.group [[META4]], !llvm.access.group [[META9]], !noundef [[META4]], !annotation [[META10:![0-9]+]], !nosanitize [[META4]], !mmra [[META11:![0-9]+]], !noalias.addrspace [[META12:![0-9]+]], !mem.cache_hint [[META13:![0-9]+]]
 ; CHECK-NEXT:    ret float [[V1]]
 ;
   %v = load i32, ptr %p, align 4, !tbaa !10, !range !9, !noundef !0, !invariant.load !0, !invariant.group !0, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
@@ -17,7 +17,7 @@ define float @bitcast_load(ptr %p) {
 define void @bitcast_store(ptr %p, float %v) {
 ; CHECK-LABEL: define void @bitcast_store(
 ; CHECK-SAME: ptr [[P:%.*]], float [[V:%.*]]) {
-; CHECK-NEXT:    store float [[V]], ptr [[P]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]]
+; CHECK-NEXT:    store float [[V]], ptr [[P]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !invariant.group [[META4]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15:![0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
   %bits = bitcast float %v to i32
@@ -28,9 +28,9 @@ define void @bitcast_store(ptr %p, float %v) {
 define i32 @unpack_struct_load(ptr %p) {
 ; CHECK-LABEL: define i32 @unpack_struct_load(
 ; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:    [[V_UNPACK:%.*]] = load i32, ptr [[P]], align 8, !tbaa.struct [[TBAA_STRUCT14:![0-9]+]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT:    [[V_UNPACK:%.*]] = load i32, ptr [[P]], align 8, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !invariant.group [[META4]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META13]]
 ; CHECK-NEXT:    [[V_ELT1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4, !annotation [[META10]]
-; CHECK-NEXT:    [[V_UNPACK2:%.*]] = load i32, ptr [[V_ELT1]], align 4, !tbaa.struct [[TBAA_STRUCT14]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT:    [[V_UNPACK2:%.*]] = load i32, ptr [[V_ELT1]], align 4, !tbaa [[FLOAT_TBAA16:![0-9]+]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META13]]
 ; CHECK-NEXT:    [[R:%.*]] = add i32 [[V_UNPACK]], [[V_UNPACK2]]
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
@@ -44,9 +44,9 @@ define i32 @unpack_struct_load(ptr %p) {
 define i32 @unpack_array_load(ptr %p) {
 ; CHECK-LABEL: define i32 @unpack_array_load(
 ; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:    [[V_UNPACK:%.*]] = load i32, ptr [[P]], align 8, !tbaa.struct [[TBAA_STRUCT14]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT:    [[V_UNPACK:%.*]] = load i32, ptr [[P]], align 8, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !invariant.group [[META4]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META13]]
 ; CHECK-NEXT:    [[V_ELT1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4, !annotation [[META10]]
-; CHECK-NEXT:    [[V_UNPACK2:%.*]] = load i32, ptr [[V_ELT1]], align 4, !tbaa.struct [[TBAA_STRUCT14]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT:    [[V_UNPACK2:%.*]] = load i32, ptr [[V_ELT1]], align 4, !tbaa [[FLOAT_TBAA16]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META13]]
 ; CHECK-NEXT:    [[R:%.*]] = add i32 [[V_UNPACK]], [[V_UNPACK2]]
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
@@ -61,10 +61,10 @@ define void @unpack_struct_store(ptr %p, {i32, i32} %v) {
 ; CHECK-LABEL: define void @unpack_struct_store(
 ; CHECK-SAME: ptr [[P:%.*]], { i32, i32 } [[V:%.*]]) {
 ; CHECK-NEXT:    [[V_ELT:%.*]] = extractvalue { i32, i32 } [[V]], 0, !annotation [[META10]]
-; CHECK-NEXT:    store i32 [[V_ELT]], ptr [[P]], align 8, !tbaa.struct [[TBAA_STRUCT14]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT:    store i32 [[V_ELT]], ptr [[P]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !invariant.group [[META4]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
 ; CHECK-NEXT:    [[P_REPACK1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4, !annotation [[META10]]
 ; CHECK-NEXT:    [[V_ELT2:%.*]] = extractvalue { i32, i32 } [[V]], 1, !annotation [[META10]]
-; CHECK-NEXT:    store i32 [[V_ELT2]], ptr [[P_REPACK1]], align 4, !tbaa.struct [[TBAA_STRUCT14]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT:    store i32 [[V_ELT2]], ptr [[P_REPACK1]], align 4, !tbaa [[FLOAT_TBAA16]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
 ; CHECK-NEXT:    ret void
 ;
   store {i32, i32} %v, ptr %p, align 8, !tbaa.struct !19, !invariant.group !0, !mem.cache_hint !18, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
@@ -75,10 +75,10 @@ define void @unpack_array_store(ptr %p, [2 x i32] %v) {
 ; CHECK-LABEL: define void @unpack_array_store(
 ; CHECK-SAME: ptr [[P:%.*]], [2 x i32] [[V:%.*]]) {
 ; CHECK-NEXT:    [[V_ELT:%.*]] = extractvalue [2 x i32] [[V]], 0, !annotation [[META10]]
-; CHECK-NEXT:    store i32 [[V_ELT]], ptr [[P]], align 8, !tbaa.struct [[TBAA_STRUCT14]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT:    store i32 [[V_ELT]], ptr [[P]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !invariant.group [[META4]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
 ; CHECK-NEXT:    [[P_REPACK1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4, !annotation [[META10]]
 ; CHECK-NEXT:    [[V_ELT2:%.*]] = extractvalue [2 x i32] [[V]], 1, !annotation [[META10]]
-; CHECK-NEXT:    store i32 [[V_ELT2]], ptr [[P_REPACK1]], align 4, !tbaa.struct [[TBAA_STRUCT14]], !alias.scope [[META5]], !noalias [[META5]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT:    store i32 [[V_ELT2]], ptr [[P_REPACK1]], align 4, !tbaa [[FLOAT_TBAA16]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
 ; CHECK-NEXT:    ret void
 ;
   store [2 x i32] %v, ptr %p, align 8, !tbaa.struct !19, !invariant.group !0, !mem.cache_hint !18, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
@@ -90,8 +90,8 @@ define void @unpack_array_store(ptr %p, [2 x i32] %v) {
 define void @copy_memory(ptr %dst, ptr %src) {
 ; CHECK-LABEL: define void @copy_memory(
 ; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr [[SRC]], align 8, !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !noalias.addrspace [[META11]]
-; CHECK-NEXT:    store i64 [[TMP1]], ptr [[DST]], align 8, !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !noalias.addrspace [[META11]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr [[SRC]], align 8, !tbaa.struct [[TBAA_STRUCT18:![0-9]+]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META19:![0-9]+]]
+; CHECK-NEXT:    store i64 [[TMP1]], ptr [[DST]], align 8, !tbaa.struct [[TBAA_STRUCT18]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %src, i64 8, i1 false), !tbaa.struct !19, !prof !24, !mem.cache_hint !22, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
@@ -101,7 +101,7 @@ define void @copy_memory(ptr %dst, ptr %src) {
 define void @set_memory(ptr %dst) {
 ; CHECK-LABEL: define void @set_memory(
 ; CHECK-SAME: ptr [[DST:%.*]]) {
-; CHECK-NEXT:    store i32 0, ptr [[DST]], align 4, !annotation [[META10]]
+; CHECK-NEXT:    store i32 0, ptr [[DST]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
 ; CHECK-NEXT:    ret void
 ;
   call void @llvm.memset.p0.i64(ptr align 4 %dst, i8 0, i64 4, i1 false), !tbaa !10, !prof !24, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
@@ -111,7 +111,7 @@ define void @set_memory(ptr %dst) {
 define <2 x i32> @splat_gather(ptr %p) {
 ; CHECK-LABEL: define <2 x i32> @splat_gather(
 ; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:    [[LOAD_SCALAR:%.*]] = load i32, ptr [[P]], align 1, !annotation [[META10]]
+; CHECK-NEXT:    [[LOAD_SCALAR:%.*]] = load i32, ptr [[P]], align 1, !tbaa [[INT_TBAA0]], !range [[RNG21:![0-9]+]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META13]]
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[LOAD_SCALAR]], i64 0, !annotation [[META10]]
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer, !annotation [[META10]]
 ; CHECK-NEXT:    ret <2 x i32> [[BROADCAST_SPLAT]]
@@ -125,7 +125,7 @@ define <2 x i32> @splat_gather(ptr %p) {
 define void @splat_scatter(ptr %p, i32 %v) {
 ; CHECK-LABEL: define void @splat_scatter(
 ; CHECK-SAME: ptr [[P:%.*]], i32 [[V:%.*]]) {
-; CHECK-NEXT:    store i32 [[V]], ptr [[P]], align 1, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META17:![0-9]+]], !noalias.addrspace [[META11]], !mem.cache_hint [[META18:![0-9]+]]
+; CHECK-NEXT:    store i32 [[V]], ptr [[P]], align 1, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.mem.parallel_loop_access [[META9]], !llvm.access.group [[META9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
 ; CHECK-NEXT:    ret void
 ;
   %ptr0 = insertelement <2 x ptr> poison, ptr %p, i32 0
@@ -174,12 +174,15 @@ define void @splat_scatter(ptr %p, i32 %v) {
 ; CHECK: [[META8]] = !{i32 1}
 ; CHECK: [[META9]] = distinct !{}
 ; CHECK: [[META10]] = !{!"preserved memory metadata"}
-; CHECK: [[META11]] = !{i32 5, i32 6}
-; CHECK: [[META12]] = !{i32 0, [[META13:![0-9]+]]}
-; CHECK: [[META13]] = !{!"nvvm.l1_eviction", !"last"}
-; CHECK: [[TBAA_STRUCT14]] = !{i64 0, i64 4, [[INT_TBAA0]], i64 4, i64 4, [[META15:![0-9]+]]}
-; CHECK: [[META15]] = !{[[META16:![0-9]+]], [[META16]], i64 0}
-; CHECK: [[META16]] = !{!"float", [[META2]], i64 0}
-; CHECK: [[META17]] = !{!"test", !"relaxed"}
-; CHECK: [[META18]] = !{i32 1, [[META13]]}
+; CHECK: [[META11]] = !{!"test", !"relaxed"}
+; CHECK: [[META12]] = !{i32 5, i32 6}
+; CHECK: [[META13]] = !{i32 0, [[META14:![0-9]+]]}
+; CHECK: [[META14]] = !{!"nvvm.l1_eviction", !"last"}
+; CHECK: [[META15]] = !{i32 1, [[META14]]}
+; CHECK: [[FLOAT_TBAA16]] = !{[[META17:![0-9]+]], [[META17]], i64 0}
+; CHECK: [[META17]] = !{!"float", [[META2]], i64 0}
+; CHECK: [[TBAA_STRUCT18]] = !{i64 0, i64 4, [[INT_TBAA0]], i64 4, i64 4, [[FLOAT_TBAA16]]}
+; CHECK: [[META19]] = !{i32 0, [[META20:![0-9]+]]}
+; CHECK: [[META20]] = !{!"nvvm.l1_eviction", !"first"}
+; CHECK: [[RNG21]] = !{i32 0, i32 100}
 ;.
diff --git a/llvm/test/Transforms/InstCombine/storemerge-metadata.ll b/llvm/test/Transforms/InstCombine/storemerge-metadata.ll
index dc5fdee99c333f..dfe2f81950217d 100644
--- a/llvm/test/Transforms/InstCombine/storemerge-metadata.ll
+++ b/llvm/test/Transforms/InstCombine/storemerge-metadata.ll
@@ -12,7 +12,7 @@ define void @common_metadata(ptr %p, i1 %c, i32 %a, i32 %b) {
 ; CHECK-NEXT:    br label %[[JOIN]]
 ; CHECK:       [[JOIN]]:
 ; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi i32 [ [[B]], %[[RIGHT]] ], [ [[A]], %[[LEFT]] ]
-; CHECK-NEXT:    store i32 [[STOREMERGE]], ptr [[P]], align 4, !tbaa [[INT_TBAA0:![0-9]+]], !alias.scope [[META4:![0-9]+]], !noalias [[META4]], !llvm.access.group [[ACC_GRP7:![0-9]+]]
+; CHECK-NEXT:    store i32 [[STOREMERGE]], ptr [[P]], align 4, !tbaa [[INT_TBAA0:![0-9]+]], !alias.scope [[META4:![0-9]+]], !noalias [[META4]], !nontemporal [[META7:![0-9]+]], !llvm.access.group [[ACC_GRP8:![0-9]+]], !mem.cache_hint [[META11:![0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -38,7 +38,7 @@ define void @intersect_access_groups(ptr %p, i1 %c, i32 %a, i32 %b) {
 ; CHECK-NEXT:    br label %[[JOIN]]
 ; CHECK:       [[JOIN]]:
 ; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi i32 [ [[B]], %[[RIGHT]] ], [ [[A]], %[[LEFT]] ]
-; CHECK-NEXT:    store i32 [[STOREMERGE]], ptr [[P]], align 4, !llvm.access.group [[ACC_GRP9:![0-9]+]]
+; CHECK-NEXT:    store i32 [[STOREMERGE]], ptr [[P]], align 4, !llvm.access.group [[ACC_GRP10:![0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -140,7 +140,7 @@ define void @triangle_common_metadata(ptr %p, i1 %c, i32 %a, i32 %b) {
 ; CHECK-NEXT:    br label %[[JOIN]]
 ; CHECK:       [[JOIN]]:
 ; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi i32 [ [[B]], %[[THEN]] ], [ [[A]], %[[ENTRY]] ]
-; CHECK-NEXT:    store i32 [[STOREMERGE]], ptr [[P]], align 4
+; CHECK-NEXT:    store i32 [[STOREMERGE]], ptr [[P]], align 4, !nontemporal [[META7]], !mem.cache_hint [[META11]]
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -164,7 +164,7 @@ define void @merge_pointer_captures(ptr %p, i1 %c, ptr %a, ptr %b) {
 ; CHECK-NEXT:    br label %[[JOIN]]
 ; CHECK:       [[JOIN]]:
 ; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi ptr [ [[B]], %[[RIGHT]] ], [ [[A]], %[[LEFT]] ]
-; CHECK-NEXT:    store ptr [[STOREMERGE]], ptr [[P]], align 8
+; CHECK-NEXT:    store ptr [[STOREMERGE]], ptr [[P]], align 8, !captures [[META13:![0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -207,7 +207,11 @@ join:
 ; CHECK: [[META4]] = !{[[META5:![0-9]+]]}
 ; CHECK: [[META5]] = distinct !{[[META5]], [[META6:![0-9]+]]}
 ; CHECK: [[META6]] = distinct !{[[META6]], i1 false, !"domain"}
-; CHECK: [[ACC_GRP7]] = !{[[META8:![0-9]+]], [[ACC_GRP9]]}
-; CHECK: [[META8]] = distinct !{}
-; CHECK: [[ACC_GRP9]] = distinct !{}
+; CHECK: [[META7]] = !{i32 1}
+; CHECK: [[ACC_GRP8]] = !{[[META9:![0-9]+]], [[ACC_GRP10]]}
+; CHECK: [[META9]] = distinct !{}
+; CHECK: [[ACC_GRP10]] = distinct !{}
+; CHECK: [[META11]] = !{i32 1, [[META12:![0-9]+]]}
+; CHECK: [[META12]] = !{!"nvvm.l1_eviction", !"last"}
+; CHECK: [[META13]] = !{!"address", !"read_provenance"}
 ;.
diff --git a/llvm/test/Transforms/SROA/memory-metadata.ll b/llvm/test/Transforms/SROA/memory-metadata.ll
index a1d2f5687a4597..f4d272e8465ef3 100644
--- a/llvm/test/Transforms/SROA/memory-metadata.ll
+++ b/llvm/test/Transforms/SROA/memory-metadata.ll
@@ -8,11 +8,11 @@ define i32 @split_copy(ptr %src) {
 ; CHECK-SAME: ptr [[SRC:%.*]]) {
 ; CHECK-NEXT:    [[A_SROA_0:%.*]] = alloca i32, align 8
 ; CHECK-NEXT:    [[A_SROA_2:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    [[A_SROA_0_0_COPYLOAD:%.*]] = load i32, ptr [[SRC]], align 8, !tbaa [[INT_TBAA0:![0-9]+]], !alias.scope [[META4:![0-9]+]], !noalias [[META4]], !llvm.mem.parallel_loop_access [[META7:![0-9]+]], !llvm.access.group [[META7]], !noalias.addrspace [[META8:![0-9]+]]
-; CHECK-NEXT:    store i32 [[A_SROA_0_0_COPYLOAD]], ptr [[A_SROA_0]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META4]], !noalias [[META4]], !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]], !noalias.addrspace [[META8]]
+; CHECK-NEXT:    [[A_SROA_0_0_COPYLOAD:%.*]] = load i32, ptr [[SRC]], align 8, !tbaa [[INT_TBAA0:![0-9]+]], !alias.scope [[META4:![0-9]+]], !noalias [[META4]], !nontemporal [[META7:![0-9]+]], !llvm.mem.parallel_loop_access [[META8:![0-9]+]], !llvm.access.group [[META8]], !annotation [[META9:![0-9]+]], !nosanitize [[META10:![0-9]+]], !mmra [[META11:![0-9]+]], !noalias.addrspace [[META12:![0-9]+]], !mem.cache_hint [[META13:![0-9]+]]
+; CHECK-NEXT:    store i32 [[A_SROA_0_0_COPYLOAD]], ptr [[A_SROA_0]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15:![0-9]+]]
 ; CHECK-NEXT:    [[A_SROA_2_0_SRC_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 4
-; CHECK-NEXT:    [[A_SROA_2_0_COPYLOAD:%.*]] = load i32, ptr [[A_SROA_2_0_SRC_SROA_IDX]], align 4, !tbaa [[FLOAT_TBAA9:![0-9]+]], !alias.scope [[META4]], !noalias [[META4]], !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]], !noalias.addrspace [[META8]]
-; CHECK-NEXT:    store i32 [[A_SROA_2_0_COPYLOAD]], ptr [[A_SROA_2]], align 4, !tbaa [[FLOAT_TBAA9]], !alias.scope [[META4]], !noalias [[META4]], !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]], !noalias.addrspace [[META8]]
+; CHECK-NEXT:    [[A_SROA_2_0_COPYLOAD:%.*]] = load i32, ptr [[A_SROA_2_0_SRC_SROA_IDX]], align 4, !tbaa [[FLOAT_TBAA17:![0-9]+]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META13]]
+; CHECK-NEXT:    store i32 [[A_SROA_2_0_COPYLOAD]], ptr [[A_SROA_2]], align 4, !tbaa [[FLOAT_TBAA17]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
 ; CHECK-NEXT:    [[A_SROA_0_0_A_SROA_0_0_X:%.*]] = load volatile i32, ptr [[A_SROA_0]], align 8
 ; CHECK-NEXT:    [[A_SROA_2_0_A_SROA_2_4_Y:%.*]] = load volatile i32, ptr [[A_SROA_2]], align 4
 ; CHECK-NEXT:    [[R:%.*]] = add i32 [[A_SROA_0_0_A_SROA_0_0_X]], [[A_SROA_2_0_A_SROA_2_4_Y]]
@@ -33,7 +33,7 @@ define i32 @rewrite_set(i32 %v) {
 ; CHECK-NEXT:    [[A_SROA_0:%.*]] = alloca i32, align 8
 ; CHECK-NEXT:    [[A_SROA_2:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    store i32 [[V]], ptr [[A_SROA_0]], align 8
-; CHECK-NEXT:    store i32 0, ptr [[A_SROA_2]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META4]], !noalias [[META4]], !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]], !noalias.addrspace [[META8]]
+; CHECK-NEXT:    store i32 0, ptr [[A_SROA_2]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
 ; CHECK-NEXT:    [[A_SROA_0_0_A_SROA_0_0_X:%.*]] = load volatile i32, ptr [[A_SROA_0]], align 8
 ; CHECK-NEXT:    [[A_SROA_2_0_A_SROA_2_4_Y:%.*]] = load volatile i32, ptr [[A_SROA_2]], align 4
 ; CHECK-NEXT:    [[R:%.*]] = add i32 [[A_SROA_0_0_A_SROA_0_0_X]], [[A_SROA_2_0_A_SROA_2_4_Y]]
@@ -55,9 +55,9 @@ define i32 @split_aggregate_store({i32, i32} %v) {
 ; CHECK-NEXT:    [[A_SROA_0:%.*]] = alloca i32, align 8
 ; CHECK-NEXT:    [[A_SROA_2:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[V_FCA_0_EXTRACT:%.*]] = extractvalue { i32, i32 } [[V]], 0
-; CHECK-NEXT:    store i32 [[V_FCA_0_EXTRACT]], ptr [[A_SROA_0]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META4]], !noalias [[META4]], !noalias.addrspace [[META8]]
+; CHECK-NEXT:    store i32 [[V_FCA_0_EXTRACT]], ptr [[A_SROA_0]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
 ; CHECK-NEXT:    [[V_FCA_1_EXTRACT:%.*]] = extractvalue { i32, i32 } [[V]], 1
-; CHECK-NEXT:    store i32 [[V_FCA_1_EXTRACT]], ptr [[A_SROA_2]], align 4, !tbaa [[FLOAT_TBAA9]], !alias.scope [[META4]], !noalias [[META4]], !noalias.addrspace [[META8]]
+; CHECK-NEXT:    store i32 [[V_FCA_1_EXTRACT]], ptr [[A_SROA_2]], align 4, !tbaa [[FLOAT_TBAA17]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
 ; CHECK-NEXT:    [[A_SROA_0_0_A_SROA_0_0_X:%.*]] = load volatile i32, ptr [[A_SROA_0]], align 8
 ; CHECK-NEXT:    [[A_SROA_2_0_A_SROA_2_4_Y:%.*]] = load volatile i32, ptr [[A_SROA_2]], align 4
 ; CHECK-NEXT:    [[R:%.*]] = add i32 [[A_SROA_0_0_A_SROA_0_0_X]], [[A_SROA_2_0_A_SROA_2_4_Y]]
@@ -82,9 +82,9 @@ define {i32, i32} @split_aggregate_load(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    [[A_SROA_3_0_SRC_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 4
 ; CHECK-NEXT:    [[A_SROA_3_0_COPYLOAD:%.*]] = load i32, ptr [[A_SROA_3_0_SRC_SROA_IDX]], align 1
 ; CHECK-NEXT:    store i32 [[A_SROA_3_0_COPYLOAD]], ptr [[A_SROA_3]], align 4
-; CHECK-NEXT:    [[A_SROA_0_0_A_SROA_0_0_V_FCA_0_LOAD:%.*]] = load i32, ptr [[A_SROA_0]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META4]], !noalias [[META4]], !noalias.addrspace [[META8]]
+; CHECK-NEXT:    [[A_SROA_0_0_A_SROA_0_0_V_FCA_0_LOAD:%.*]] = load i32, ptr [[A_SROA_0]], align 8, !tbaa [[INT_TBAA0]], !invariant.load [[META10]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META19:![0-9]+]]
 ; CHECK-NEXT:    [[V_FCA_0_INSERT:%.*]] = insertvalue { i32, i32 } poison, i32 [[A_SROA_0_0_A_SROA_0_0_V_FCA_0_LOAD]], 0
-; CHECK-NEXT:    [[A_SROA_3_0_A_SROA_3_4_V_FCA_1_LOAD:%.*]] = load i32, ptr [[A_SROA_3]], align 4, !tbaa [[FLOAT_TBAA9]], !alias.scope [[META4]], !noalias [[META4]], !noalias.addrspace [[META8]]
+; CHECK-NEXT:    [[A_SROA_3_0_A_SROA_3_4_V_FCA_1_LOAD:%.*]] = load i32, ptr [[A_SROA_3]], align 4, !tbaa [[FLOAT_TBAA17]], !invariant.load [[META10]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META19]]
 ; CHECK-NEXT:    [[V_FCA_1_INSERT:%.*]] = insertvalue { i32, i32 } [[V_FCA_0_INSERT]], i32 [[A_SROA_3_0_A_SROA_3_4_V_FCA_1_LOAD]], 1
 ; CHECK-NEXT:    [[A_SROA_0_0_A_SROA_0_0_COPYLOAD1:%.*]] = load volatile i32, ptr [[A_SROA_0]], align 8
 ; CHECK-NEXT:    store volatile i32 [[A_SROA_0_0_A_SROA_0_0_COPYLOAD1]], ptr [[DST]], align 1
@@ -110,7 +110,7 @@ define double @rewrite_load(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    [[A_SROA_3_0_SRC_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 8
 ; CHECK-NEXT:    [[A_SROA_3_0_COPYLOAD:%.*]] = load i32, ptr [[A_SROA_3_0_SRC_SROA_IDX]], align 1
 ; CHECK-NEXT:    store i32 [[A_SROA_3_0_COPYLOAD]], ptr [[A_SROA_3]], align 8
-; CHECK-NEXT:    [[A_SROA_0_0_A_SROA_0_0_V:%.*]] = load double, ptr [[A_SROA_0]], align 8, !fpmath [[META11:![0-9]+]], !invariant.load [[META12:![0-9]+]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META13:![0-9]+]], !llvm.mem.parallel_loop_access [[META7]], !invariant.group [[META12]], !llvm.access.group [[META7]], !noundef [[META12]], !noalias.addrspace [[META8]], !nofpclass [[META14:![0-9]+]], !mem.cache_hint [[META15:![0-9]+]]
+; CHECK-NEXT:    [[A_SROA_0_0_A_SROA_0_0_V:%.*]] = load double, ptr [[A_SROA_0]], align 8, !fpmath [[META20:![0-9]+]], !invariant.load [[META10]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !noundef [[META10]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !nofpclass [[META21:![0-9]+]], !mem.cache_hint [[META19]]
 ; CHECK-NEXT:    [[A_SROA_0_0_A_SROA_0_0_COPYLOAD1:%.*]] = load volatile double, ptr [[A_SROA_0]], align 8
 ; CHECK-NEXT:    store volatile double [[A_SROA_0_0_A_SROA_0_0_COPYLOAD1]], ptr [[DST]], align 1
 ; CHECK-NEXT:    [[A_SROA_3_0_DST_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 8
@@ -130,7 +130,7 @@ define ptr @rewrite_pointer_load(ptr %v) {
 ; CHECK-SAME: ptr [[V:%.*]]) {
 ; CHECK-NEXT:    [[A_SROA_1:%.*]] = alloca ptr, align 8
 ; CHECK-NEXT:    store ptr [[V]], ptr [[A_SROA_1]], align 8
-; CHECK-NEXT:    [[A_SROA_1_0_A_SROA_1_8_R:%.*]] = load volatile ptr, ptr [[A_SROA_1]], align 8, !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META13]], !llvm.mem.parallel_loop_access [[META7]], !nonnull [[META12]], !dereferenceable [[META17:![0-9]+]], !dereferenceable_or_null [[META17]], !invariant.group [[META12]], !align [[META17]], !llvm.access.group [[META7]], !noundef [[META12]], !noalias.addrspace [[META8]], !mem.cache_hint [[META15]]
+; CHECK-NEXT:    [[A_SROA_1_0_A_SROA_1_8_R:%.*]] = load volatile ptr, ptr [[A_SROA_1]], align 8, !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !nonnull [[META10]], !dereferenceable [[META22:![0-9]+]], !dereferenceable_or_null [[META22]], !align [[META22]], !llvm.access.group [[META8]], !noundef [[META10]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META19]]
 ; CHECK-NEXT:    ret ptr [[A_SROA_1_0_A_SROA_1_8_R]]
 ;
   %a = alloca [2 x ptr], align 8
@@ -146,11 +146,11 @@ define i32 @presplit_load(ptr %src) {
 ; CHECK-SAME: ptr [[SRC:%.*]]) {
 ; CHECK-NEXT:    [[A_SROA_0:%.*]] = alloca i32, align 8
 ; CHECK-NEXT:    [[A_SROA_2:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    [[V1:%.*]] = load i32, ptr [[SRC]], align 8, !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]]
+; CHECK-NEXT:    [[V1:%.*]] = load i32, ptr [[SRC]], align 8, !invariant.load [[META10]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !invariant.group [[META10]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META19]]
 ; CHECK-NEXT:    [[SRC_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 4
-; CHECK-NEXT:    [[V2:%.*]] = load i32, ptr [[SRC_SROA_IDX]], align 4, !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]]
-; CHECK-NEXT:    store i32 [[V1]], ptr [[A_SROA_0]], align 8, !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]]
-; CHECK-NEXT:    store i32 [[V2]], ptr [[A_SROA_2]], align 4, !llvm.mem.parallel_loop_access [[META7]], !llvm.access.group [[META7]]
+; CHECK-NEXT:    [[V2:%.*]] = load i32, ptr [[SRC_SROA_IDX]], align 4, !invariant.load [[META10]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META19]]
+; CHECK-NEXT:    store i32 [[V1]], ptr [[A_SROA_0]], align 8, !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
+; CHECK-NEXT:    store i32 [[V2]], ptr [[A_SROA_2]], align 4, !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
 ; CHECK-NEXT:    [[A_SROA_0_0_A_SROA_0_0_X:%.*]] = load volatile i32, ptr [[A_SROA_0]], align 8
 ; CHECK-NEXT:    [[A_SROA_2_0_A_SROA_2_4_Y:%.*]] = load volatile i32, ptr [[A_SROA_2]], align 4
 ; CHECK-NEXT:    [[R:%.*]] = add i32 [[A_SROA_0_0_A_SROA_0_0_X]], [[A_SROA_2_0_A_SROA_2_4_Y]]
@@ -201,15 +201,20 @@ define i32 @presplit_load(ptr %src) {
 ; CHECK: [[META4]] = !{[[META5:![0-9]+]]}
 ; CHECK: [[META5]] = distinct !{[[META5]], [[META6:![0-9]+]], !"scope"}
 ; CHECK: [[META6]] = distinct !{[[META6]], i1 false, !"domain"}
-; CHECK: [[META7]] = distinct !{}
-; CHECK: [[META8]] = !{i32 5, i32 6}
-; CHECK: [[FLOAT_TBAA9]] = !{[[META10:![0-9]+]], [[META10]], i64 0}
-; CHECK: [[META10]] = !{!"float", [[META2]], i64 0}
-; CHECK: [[META11]] = !{float 2.500000e+00}
-; CHECK: [[META12]] = !{}
-; CHECK: [[META13]] = !{i32 1}
-; CHECK: [[META14]] = !{i32 3}
-; CHECK: [[META15]] = !{i32 0, [[META16:![0-9]+]]}
+; CHECK: [[META7]] = !{i32 1}
+; CHECK: [[META8]] = distinct !{}
+; CHECK: [[META9]] = !{!"preserved memory metadata"}
+; CHECK: [[META10]] = !{}
+; CHECK: [[META11]] = !{!"test", !"relaxed"}
+; CHECK: [[META12]] = !{i32 5, i32 6}
+; CHECK: [[META13]] = !{i32 0, [[META14:![0-9]+]]}
+; CHECK: [[META14]] = !{!"nvvm.l1_eviction", !"first"}
+; CHECK: [[META15]] = !{i32 1, [[META16:![0-9]+]]}
 ; CHECK: [[META16]] = !{!"nvvm.l1_eviction", !"last"}
-; CHECK: [[META17]] = !{i64 8}
+; CHECK: [[FLOAT_TBAA17]] = !{[[META18:![0-9]+]], [[META18]], i64 0}
+; CHECK: [[META18]] = !{!"float", [[META2]], i64 0}
+; CHECK: [[META19]] = !{i32 0, [[META16]]}
+; CHECK: [[META20]] = !{float 2.500000e+00}
+; CHECK: [[META21]] = !{i32 3}
+; CHECK: [[META22]] = !{i64 8}
 ;.
diff --git a/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll b/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
index b7db48d9b9cbc7..96ac70bdb37e39 100644
--- a/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
+++ b/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
@@ -209,21 +209,21 @@ define void @f4(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    [[DST_I1:%.*]] = getelementptr i32, ptr [[DST]], i32 1
 ; CHECK-NEXT:    [[DST_I2:%.*]] = getelementptr i32, ptr [[DST]], i32 2
 ; CHECK-NEXT:    [[DST_I3:%.*]] = getelementptr i32, ptr [[DST]], i32 3
-; CHECK-NEXT:    [[VAL_I0:%.*]] = load i32, ptr [[SRC]], align 16, !tbaa.struct [[TBAA_STRUCT5:![0-9]+]]
+; CHECK-NEXT:    [[VAL_I0:%.*]] = load i32, ptr [[SRC]], align 16, !tbaa [[SET1_TBAA0]]
 ; CHECK-NEXT:    [[SRC_I1:%.*]] = getelementptr i32, ptr [[SRC]], i32 1
-; CHECK-NEXT:    [[VAL_I1:%.*]] = load i32, ptr [[SRC_I1]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
+; CHECK-NEXT:    [[VAL_I1:%.*]] = load i32, ptr [[SRC_I1]], align 4, !tbaa [[SET2_TBAA3]]
 ; CHECK-NEXT:    [[SRC_I2:%.*]] = getelementptr i32, ptr [[SRC]], i32 2
-; CHECK-NEXT:    [[VAL_I2:%.*]] = load i32, ptr [[SRC_I2]], align 8, !tbaa.struct [[TBAA_STRUCT5]]
+; CHECK-NEXT:    [[VAL_I2:%.*]] = load i32, ptr [[SRC_I2]], align 8, !tbaa [[SET1_TBAA0]]
 ; CHECK-NEXT:    [[SRC_I3:%.*]] = getelementptr i32, ptr [[SRC]], i32 3
-; CHECK-NEXT:    [[VAL_I3:%.*]] = load i32, ptr [[SRC_I3]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
+; CHECK-NEXT:    [[VAL_I3:%.*]] = load i32, ptr [[SRC_I3]], align 4, !tbaa [[SET2_TBAA3]]
 ; CHECK-NEXT:    [[ADD_I0:%.*]] = add i32 [[VAL_I0]], [[VAL_I0]]
 ; CHECK-NEXT:    [[ADD_I1:%.*]] = add i32 [[VAL_I1]], [[VAL_I1]]
 ; CHECK-NEXT:    [[ADD_I2:%.*]] = add i32 [[VAL_I2]], [[VAL_I2]]
 ; CHECK-NEXT:    [[ADD_I3:%.*]] = add i32 [[VAL_I3]], [[VAL_I3]]
-; CHECK-NEXT:    store i32 [[ADD_I0]], ptr [[DST]], align 16, !tbaa.struct [[TBAA_STRUCT5]]
-; CHECK-NEXT:    store i32 [[ADD_I1]], ptr [[DST_I1]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
-; CHECK-NEXT:    store i32 [[ADD_I2]], ptr [[DST_I2]], align 8, !tbaa.struct [[TBAA_STRUCT5]]
-; CHECK-NEXT:    store i32 [[ADD_I3]], ptr [[DST_I3]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
+; CHECK-NEXT:    store i32 [[ADD_I0]], ptr [[DST]], align 16, !tbaa [[SET1_TBAA0]]
+; CHECK-NEXT:    store i32 [[ADD_I1]], ptr [[DST_I1]], align 4, !tbaa [[SET2_TBAA3]]
+; CHECK-NEXT:    store i32 [[ADD_I2]], ptr [[DST_I2]], align 8, !tbaa [[SET1_TBAA0]]
+; CHECK-NEXT:    store i32 [[ADD_I3]], ptr [[DST_I3]], align 4, !tbaa [[SET2_TBAA3]]
 ; CHECK-NEXT:    ret void
 ;
   %val = load <4 x i32> , ptr %src, !tbaa.struct !5
@@ -248,21 +248,21 @@ define void @f5(i32 %count, ptr %src, ptr %dst) {
 ; CHECK-NEXT:    [[THIS_DST_I1:%.*]] = getelementptr i32, ptr [[THIS_DST]], i32 1
 ; CHECK-NEXT:    [[THIS_DST_I2:%.*]] = getelementptr i32, ptr [[THIS_DST]], i32 2
 ; CHECK-NEXT:    [[THIS_DST_I3:%.*]] = getelementptr i32, ptr [[THIS_DST]], i32 3
-; CHECK-NEXT:    [[VAL_I0:%.*]] = load i32, ptr [[THIS_SRC]], align 16, !llvm.access.group [[ACC_GRP6:![0-9]+]]
-; CHECK-NEXT:    [[VAL_I1:%.*]] = load i32, ptr [[THIS_SRC_I1]], align 4, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT:    [[VAL_I2:%.*]] = load i32, ptr [[THIS_SRC_I2]], align 8, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT:    [[VAL_I3:%.*]] = load i32, ptr [[THIS_SRC_I3]], align 4, !llvm.access.group [[ACC_GRP6]]
+; CHECK-NEXT:    [[VAL_I0:%.*]] = load i32, ptr [[THIS_SRC]], align 16, !llvm.access.group [[ACC_GRP5:![0-9]+]]
+; CHECK-NEXT:    [[VAL_I1:%.*]] = load i32, ptr [[THIS_SRC_I1]], align 4, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT:    [[VAL_I2:%.*]] = load i32, ptr [[THIS_SRC_I2]], align 8, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT:    [[VAL_I3:%.*]] = load i32, ptr [[THIS_SRC_I3]], align 4, !llvm.access.group [[ACC_GRP5]]
 ; CHECK-NEXT:    [[ADD_I0:%.*]] = add i32 [[VAL_I0]], [[VAL_I0]]
 ; CHECK-NEXT:    [[ADD_I1:%.*]] = add i32 [[VAL_I1]], [[VAL_I1]]
 ; CHECK-NEXT:    [[ADD_I2:%.*]] = add i32 [[VAL_I2]], [[VAL_I2]]
 ; CHECK-NEXT:    [[ADD_I3:%.*]] = add i32 [[VAL_I3]], [[VAL_I3]]
-; CHECK-NEXT:    store i32 [[ADD_I0]], ptr [[THIS_DST]], align 16, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT:    store i32 [[ADD_I1]], ptr [[THIS_DST_I1]], align 4, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT:    store i32 [[ADD_I2]], ptr [[THIS_DST_I2]], align 8, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT:    store i32 [[ADD_I3]], ptr [[THIS_DST_I3]], align 4, !llvm.access.group [[ACC_GRP6]]
+; CHECK-NEXT:    store i32 [[ADD_I0]], ptr [[THIS_DST]], align 16, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT:    store i32 [[ADD_I1]], ptr [[THIS_DST_I1]], align 4, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT:    store i32 [[ADD_I2]], ptr [[THIS_DST_I2]], align 8, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT:    store i32 [[ADD_I3]], ptr [[THIS_DST_I3]], align 4, !llvm.access.group [[ACC_GRP5]]
 ; CHECK-NEXT:    [[NEXT_INDEX]] = add i32 [[INDEX]], -1
 ; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ne i32 [[NEXT_INDEX]], [[COUNT]]
-; CHECK-NEXT:    br i1 [[CONTINUE]], label %[[LOOP]], label %[[END:.*]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label %[[LOOP]], label %[[END:.*]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       [[END]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -289,13 +289,13 @@ define <4 x float> @f6(<4 x float> %x) {
 ; CHECK-LABEL: define <4 x float> @f6(
 ; CHECK-SAME: <4 x float> [[X:%.*]]) {
 ; CHECK-NEXT:    [[X_I0:%.*]] = extractelement <4 x float> [[X]], i64 0
-; CHECK-NEXT:    [[RES_I0:%.*]] = fadd float [[X_I0]], 1.000000e+00, !fpmath [[META9:![0-9]+]]
+; CHECK-NEXT:    [[RES_I0:%.*]] = fadd float [[X_I0]], 1.000000e+00, !fpmath [[META8:![0-9]+]]
 ; CHECK-NEXT:    [[X_I1:%.*]] = extractelement <4 x float> [[X]], i64 1
-; CHECK-NEXT:    [[RES_I1:%.*]] = fadd float [[X_I1]], 2.000000e+00, !fpmath [[META9]]
+; CHECK-NEXT:    [[RES_I1:%.*]] = fadd float [[X_I1]], 2.000000e+00, !fpmath [[META8]]
 ; CHECK-NEXT:    [[X_I2:%.*]] = extractelement <4 x float> [[X]], i64 2
-; CHECK-NEXT:    [[RES_I2:%.*]] = fadd float [[X_I2]], 3.000000e+00, !fpmath [[META9]]
+; CHECK-NEXT:    [[RES_I2:%.*]] = fadd float [[X_I2]], 3.000000e+00, !fpmath [[META8]]
 ; CHECK-NEXT:    [[X_I3:%.*]] = extractelement <4 x float> [[X]], i64 3
-; CHECK-NEXT:    [[RES_I3:%.*]] = fadd float [[X_I3]], 4.000000e+00, !fpmath [[META9]]
+; CHECK-NEXT:    [[RES_I3:%.*]] = fadd float [[X_I3]], 4.000000e+00, !fpmath [[META8]]
 ; CHECK-NEXT:    [[RES_UPTO0:%.*]] = insertelement <4 x float> poison, float [[RES_I0]], i64 0
 ; CHECK-NEXT:    [[RES_UPTO1:%.*]] = insertelement <4 x float> [[RES_UPTO0]], float [[RES_I1]], i64 1
 ; CHECK-NEXT:    [[RES_UPTO2:%.*]] = insertelement <4 x float> [[RES_UPTO1]], float [[RES_I2]], i64 2
@@ -868,9 +868,8 @@ define <2 x i32> @f23_crash(<2 x i32> %srcvec, i32 %v1) {
 ; CHECK: [[META2]] = !{!"root"}
 ; CHECK: [[SET2_TBAA3]] = !{[[META4:![0-9]+]], [[META4]], i64 0}
 ; CHECK: [[META4]] = !{!"set2", [[META2]]}
-; CHECK: [[TBAA_STRUCT5]] = !{i64 0, i64 4, [[SET1_TBAA0]], i64 4, i64 4, [[SET2_TBAA3]], i64 8, i64 4, [[SET1_TBAA0]], i64 12, i64 4, [[SET2_TBAA3]]}
-; CHECK: [[ACC_GRP6]] = distinct !{}
-; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META8:![0-9]+]]}
-; CHECK: [[META8]] = !{!"llvm.loop.parallel_accesses", [[ACC_GRP6]]}
-; CHECK: [[META9]] = !{float 4.000000e+00}
+; CHECK: [[ACC_GRP5]] = distinct !{}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META7:![0-9]+]]}
+; CHECK: [[META7]] = !{!"llvm.loop.parallel_accesses", [[ACC_GRP5]]}
+; CHECK: [[META8]] = !{float 4.000000e+00}
 ;.
diff --git a/llvm/test/Transforms/Scalarizer/basic.ll b/llvm/test/Transforms/Scalarizer/basic.ll
index f620445612f6dd..0d55e3edbedae6 100644
--- a/llvm/test/Transforms/Scalarizer/basic.ll
+++ b/llvm/test/Transforms/Scalarizer/basic.ll
@@ -209,21 +209,21 @@ define void @f4(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    [[DST_I1:%.*]] = getelementptr i32, ptr [[DST]], i32 1
 ; CHECK-NEXT:    [[DST_I2:%.*]] = getelementptr i32, ptr [[DST]], i32 2
 ; CHECK-NEXT:    [[DST_I3:%.*]] = getelementptr i32, ptr [[DST]], i32 3
-; CHECK-NEXT:    [[VAL_I0:%.*]] = load i32, ptr [[SRC]], align 16, !tbaa.struct [[TBAA_STRUCT5:![0-9]+]]
+; CHECK-NEXT:    [[VAL_I0:%.*]] = load i32, ptr [[SRC]], align 16, !tbaa [[SET1_TBAA0]]
 ; CHECK-NEXT:    [[SRC_I1:%.*]] = getelementptr i32, ptr [[SRC]], i32 1
-; CHECK-NEXT:    [[VAL_I1:%.*]] = load i32, ptr [[SRC_I1]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
+; CHECK-NEXT:    [[VAL_I1:%.*]] = load i32, ptr [[SRC_I1]], align 4, !tbaa [[SET2_TBAA3]]
 ; CHECK-NEXT:    [[SRC_I2:%.*]] = getelementptr i32, ptr [[SRC]], i32 2
-; CHECK-NEXT:    [[VAL_I2:%.*]] = load i32, ptr [[SRC_I2]], align 8, !tbaa.struct [[TBAA_STRUCT5]]
+; CHECK-NEXT:    [[VAL_I2:%.*]] = load i32, ptr [[SRC_I2]], align 8, !tbaa [[SET1_TBAA0]]
 ; CHECK-NEXT:    [[SRC_I3:%.*]] = getelementptr i32, ptr [[SRC]], i32 3
-; CHECK-NEXT:    [[VAL_I3:%.*]] = load i32, ptr [[SRC_I3]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
+; CHECK-NEXT:    [[VAL_I3:%.*]] = load i32, ptr [[SRC_I3]], align 4, !tbaa [[SET2_TBAA3]]
 ; CHECK-NEXT:    [[ADD_I0:%.*]] = add i32 [[VAL_I0]], [[VAL_I0]]
 ; CHECK-NEXT:    [[ADD_I1:%.*]] = add i32 [[VAL_I1]], [[VAL_I1]]
 ; CHECK-NEXT:    [[ADD_I2:%.*]] = add i32 [[VAL_I2]], [[VAL_I2]]
 ; CHECK-NEXT:    [[ADD_I3:%.*]] = add i32 [[VAL_I3]], [[VAL_I3]]
-; CHECK-NEXT:    store i32 [[ADD_I0]], ptr [[DST]], align 16, !tbaa.struct [[TBAA_STRUCT5]]
-; CHECK-NEXT:    store i32 [[ADD_I1]], ptr [[DST_I1]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
-; CHECK-NEXT:    store i32 [[ADD_I2]], ptr [[DST_I2]], align 8, !tbaa.struct [[TBAA_STRUCT5]]
-; CHECK-NEXT:    store i32 [[ADD_I3]], ptr [[DST_I3]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
+; CHECK-NEXT:    store i32 [[ADD_I0]], ptr [[DST]], align 16, !tbaa [[SET1_TBAA0]]
+; CHECK-NEXT:    store i32 [[ADD_I1]], ptr [[DST_I1]], align 4, !tbaa [[SET2_TBAA3]]
+; CHECK-NEXT:    store i32 [[ADD_I2]], ptr [[DST_I2]], align 8, !tbaa [[SET1_TBAA0]]
+; CHECK-NEXT:    store i32 [[ADD_I3]], ptr [[DST_I3]], align 4, !tbaa [[SET2_TBAA3]]
 ; CHECK-NEXT:    ret void
 ;
   %val = load <4 x i32> , ptr %src, !tbaa.struct !5
@@ -248,21 +248,21 @@ define void @f5(i32 %count, ptr %src, ptr %dst) {
 ; CHECK-NEXT:    [[THIS_DST_I1:%.*]] = getelementptr i32, ptr [[THIS_DST]], i32 1
 ; CHECK-NEXT:    [[THIS_DST_I2:%.*]] = getelementptr i32, ptr [[THIS_DST]], i32 2
 ; CHECK-NEXT:    [[THIS_DST_I3:%.*]] = getelementptr i32, ptr [[THIS_DST]], i32 3
-; CHECK-NEXT:    [[VAL_I0:%.*]] = load i32, ptr [[THIS_SRC]], align 16, !llvm.access.group [[ACC_GRP6:![0-9]+]]
-; CHECK-NEXT:    [[VAL_I1:%.*]] = load i32, ptr [[THIS_SRC_I1]], align 4, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT:    [[VAL_I2:%.*]] = load i32, ptr [[THIS_SRC_I2]], align 8, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT:    [[VAL_I3:%.*]] = load i32, ptr [[THIS_SRC_I3]], align 4, !llvm.access.group [[ACC_GRP6]]
+; CHECK-NEXT:    [[VAL_I0:%.*]] = load i32, ptr [[THIS_SRC]], align 16, !llvm.access.group [[ACC_GRP5:![0-9]+]]
+; CHECK-NEXT:    [[VAL_I1:%.*]] = load i32, ptr [[THIS_SRC_I1]], align 4, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT:    [[VAL_I2:%.*]] = load i32, ptr [[THIS_SRC_I2]], align 8, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT:    [[VAL_I3:%.*]] = load i32, ptr [[THIS_SRC_I3]], align 4, !llvm.access.group [[ACC_GRP5]]
 ; CHECK-NEXT:    [[ADD_I0:%.*]] = add i32 [[VAL_I0]], [[VAL_I0]]
 ; CHECK-NEXT:    [[ADD_I1:%.*]] = add i32 [[VAL_I1]], [[VAL_I1]]
 ; CHECK-NEXT:    [[ADD_I2:%.*]] = add i32 [[VAL_I2]], [[VAL_I2]]
 ; CHECK-NEXT:    [[ADD_I3:%.*]] = add i32 [[VAL_I3]], [[VAL_I3]]
-; CHECK-NEXT:    store i32 [[ADD_I0]], ptr [[THIS_DST]], align 16, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT:    store i32 [[ADD_I1]], ptr [[THIS_DST_I1]], align 4, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT:    store i32 [[ADD_I2]], ptr [[THIS_DST_I2]], align 8, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT:    store i32 [[ADD_I3]], ptr [[THIS_DST_I3]], align 4, !llvm.access.group [[ACC_GRP6]]
+; CHECK-NEXT:    store i32 [[ADD_I0]], ptr [[THIS_DST]], align 16, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT:    store i32 [[ADD_I1]], ptr [[THIS_DST_I1]], align 4, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT:    store i32 [[ADD_I2]], ptr [[THIS_DST_I2]], align 8, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT:    store i32 [[ADD_I3]], ptr [[THIS_DST_I3]], align 4, !llvm.access.group [[ACC_GRP5]]
 ; CHECK-NEXT:    [[NEXT_INDEX]] = add i32 [[INDEX]], -1
 ; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ne i32 [[NEXT_INDEX]], [[COUNT]]
-; CHECK-NEXT:    br i1 [[CONTINUE]], label %[[LOOP]], label %[[END:.*]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label %[[LOOP]], label %[[END:.*]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       [[END]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -289,13 +289,13 @@ define <4 x float> @f6(<4 x float> %x) {
 ; CHECK-LABEL: define <4 x float> @f6(
 ; CHECK-SAME: <4 x float> [[X:%.*]]) {
 ; CHECK-NEXT:    [[X_I0:%.*]] = extractelement <4 x float> [[X]], i64 0
-; CHECK-NEXT:    [[RES_I0:%.*]] = fadd float [[X_I0]], 1.000000e+00, !fpmath [[META9:![0-9]+]]
+; CHECK-NEXT:    [[RES_I0:%.*]] = fadd float [[X_I0]], 1.000000e+00, !fpmath [[META8:![0-9]+]]
 ; CHECK-NEXT:    [[X_I1:%.*]] = extractelement <4 x float> [[X]], i64 1
-; CHECK-NEXT:    [[RES_I1:%.*]] = fadd float [[X_I1]], 2.000000e+00, !fpmath [[META9]]
+; CHECK-NEXT:    [[RES_I1:%.*]] = fadd float [[X_I1]], 2.000000e+00, !fpmath [[META8]]
 ; CHECK-NEXT:    [[X_I2:%.*]] = extractelement <4 x float> [[X]], i64 2
-; CHECK-NEXT:    [[RES_I2:%.*]] = fadd float [[X_I2]], 3.000000e+00, !fpmath [[META9]]
+; CHECK-NEXT:    [[RES_I2:%.*]] = fadd float [[X_I2]], 3.000000e+00, !fpmath [[META8]]
 ; CHECK-NEXT:    [[X_I3:%.*]] = extractelement <4 x float> [[X]], i64 3
-; CHECK-NEXT:    [[RES_I3:%.*]] = fadd float [[X_I3]], 4.000000e+00, !fpmath [[META9]]
+; CHECK-NEXT:    [[RES_I3:%.*]] = fadd float [[X_I3]], 4.000000e+00, !fpmath [[META8]]
 ; CHECK-NEXT:    [[RES_UPTO0:%.*]] = insertelement <4 x float> poison, float [[RES_I0]], i64 0
 ; CHECK-NEXT:    [[RES_UPTO1:%.*]] = insertelement <4 x float> [[RES_UPTO0]], float [[RES_I1]], i64 1
 ; CHECK-NEXT:    [[RES_UPTO2:%.*]] = insertelement <4 x float> [[RES_UPTO1]], float [[RES_I2]], i64 2
@@ -919,9 +919,8 @@ define <2 x i8> @test_copy_trunc_flags(<2 x i32> %src) {
 ; CHECK: [[META2]] = !{!"root"}
 ; CHECK: [[SET2_TBAA3]] = !{[[META4:![0-9]+]], [[META4]], i64 0}
 ; CHECK: [[META4]] = !{!"set2", [[META2]]}
-; CHECK: [[TBAA_STRUCT5]] = !{i64 0, i64 4, [[SET1_TBAA0]], i64 4, i64 4, [[SET2_TBAA3]], i64 8, i64 4, [[SET1_TBAA0]], i64 12, i64 4, [[SET2_TBAA3]]}
-; CHECK: [[ACC_GRP6]] = distinct !{}
-; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META8:![0-9]+]]}
-; CHECK: [[META8]] = !{!"llvm.loop.parallel_accesses", [[ACC_GRP6]]}
-; CHECK: [[META9]] = !{float 4.000000e+00}
+; CHECK: [[ACC_GRP5]] = distinct !{}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META7:![0-9]+]]}
+; CHECK: [[META7]] = !{!"llvm.loop.parallel_accesses", [[ACC_GRP5]]}
+; CHECK: [[META8]] = !{float 4.000000e+00}
 ;.
diff --git a/llvm/test/Transforms/Scalarizer/memory-cache-hint.ll b/llvm/test/Transforms/Scalarizer/memory-cache-hint.ll
index 270b264e65961b..7275c377eab72b 100644
--- a/llvm/test/Transforms/Scalarizer/memory-cache-hint.ll
+++ b/llvm/test/Transforms/Scalarizer/memory-cache-hint.ll
@@ -4,9 +4,9 @@
 define <2 x i32> @split_load(ptr %p) {
 ; CHECK-LABEL: define <2 x i32> @split_load(
 ; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:    [[V_I0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT:    [[V_I0:%.*]] = load i32, ptr [[P]], align 8, !range [[RNG0:![0-9]+]], !mem.cache_hint [[META1:![0-9]+]]
 ; CHECK-NEXT:    [[P_I1:%.*]] = getelementptr i32, ptr [[P]], i32 1
-; CHECK-NEXT:    [[V_I1:%.*]] = load i32, ptr [[P_I1]], align 4
+; CHECK-NEXT:    [[V_I1:%.*]] = load i32, ptr [[P_I1]], align 4, !range [[RNG0]], !mem.cache_hint [[META1]]
 ; CHECK-NEXT:    [[V_UPTO0:%.*]] = insertelement <2 x i32> poison, i32 [[V_I0]], i64 0
 ; CHECK-NEXT:    [[V:%.*]] = insertelement <2 x i32> [[V_UPTO0]], i32 [[V_I1]], i64 1
 ; CHECK-NEXT:    ret <2 x i32> [[V]]
@@ -19,10 +19,10 @@ define void @split_store(ptr %p, <2 x i32> %v) {
 ; CHECK-LABEL: define void @split_store(
 ; CHECK-SAME: ptr [[P:%.*]], <2 x i32> [[V:%.*]]) {
 ; CHECK-NEXT:    [[V_I0:%.*]] = extractelement <2 x i32> [[V]], i64 0
-; CHECK-NEXT:    store i32 [[V_I0]], ptr [[P]], align 8
+; CHECK-NEXT:    store i32 [[V_I0]], ptr [[P]], align 8, !mem.cache_hint [[META3:![0-9]+]]
 ; CHECK-NEXT:    [[V_I1:%.*]] = extractelement <2 x i32> [[V]], i64 1
 ; CHECK-NEXT:    [[P_I1:%.*]] = getelementptr i32, ptr [[P]], i32 1
-; CHECK-NEXT:    store i32 [[V_I1]], ptr [[P_I1]], align 4
+; CHECK-NEXT:    store i32 [[V_I1]], ptr [[P_I1]], align 4, !mem.cache_hint [[META3]]
 ; CHECK-NEXT:    ret void
 ;
   store <2 x i32> %v, ptr %p, align 8, !mem.cache_hint !3
@@ -33,3 +33,9 @@ define void @split_store(ptr %p, <2 x i32> %v) {
 !1 = !{!"nvvm.l1_eviction", !"last"}
 !2 = !{i32 0, i32 100}
 !3 = !{i32 1, !1}
+;.
+; CHECK: [[RNG0]] = !{i32 0, i32 100}
+; CHECK: [[META1]] = !{i32 0, [[META2:![0-9]+]]}
+; CHECK: [[META2]] = !{!"nvvm.l1_eviction", !"last"}
+; CHECK: [[META3]] = !{i32 1, [[META2]]}
+;.
diff --git a/llvm/test/Transforms/Scalarizer/memory-metadata.ll b/llvm/test/Transforms/Scalarizer/memory-metadata.ll
index cb6197983ec51c..ef58aecaf55a61 100644
--- a/llvm/test/Transforms/Scalarizer/memory-metadata.ll
+++ b/llvm/test/Transforms/Scalarizer/memory-metadata.ll
@@ -9,13 +9,13 @@ target datalayout = "e-p:64:64"
 define <4 x i32> @integer_load(ptr %p) {
 ; CHECK-LABEL: define <4 x i32> @integer_load(
 ; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:    [[V_I0:%.*]] = load i32, ptr [[P]], align 16, !tbaa [[INT_TBAA0:![0-9]+]], !invariant.load [[META4:![0-9]+]], !alias.scope [[META5:![0-9]+]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8:![0-9]+]], !llvm.access.group [[META8]]
+; CHECK-NEXT:    [[V_I0:%.*]] = load i32, ptr [[P]], align 16, !tbaa [[INT_TBAA0:![0-9]+]], !range [[RNG4:![0-9]+]], !invariant.load [[META5:![0-9]+]], !alias.scope [[META6:![0-9]+]], !noalias [[META6]], !nontemporal [[META9:![0-9]+]], !llvm.mem.parallel_loop_access [[META10:![0-9]+]], !invariant.group [[META5]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11:![0-9]+]], !nosanitize [[META5]], !mmra [[META12:![0-9]+]], !noalias.addrspace [[META13:![0-9]+]], !mem.cache_hint [[META14:![0-9]+]]
 ; CHECK-NEXT:    [[P_I1:%.*]] = getelementptr i32, ptr [[P]], i32 1
-; CHECK-NEXT:    [[V_I1:%.*]] = load i32, ptr [[P_I1]], align 4, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT:    [[V_I1:%.*]] = load i32, ptr [[P_I1]], align 4, !tbaa [[INT_TBAA0]], !range [[RNG4]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META14]]
 ; CHECK-NEXT:    [[P_I2:%.*]] = getelementptr i32, ptr [[P]], i32 2
-; CHECK-NEXT:    [[V_I2:%.*]] = load i32, ptr [[P_I2]], align 8, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT:    [[V_I2:%.*]] = load i32, ptr [[P_I2]], align 8, !tbaa [[INT_TBAA0]], !range [[RNG4]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META14]]
 ; CHECK-NEXT:    [[P_I3:%.*]] = getelementptr i32, ptr [[P]], i32 3
-; CHECK-NEXT:    [[V_I3:%.*]] = load i32, ptr [[P_I3]], align 4, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT:    [[V_I3:%.*]] = load i32, ptr [[P_I3]], align 4, !tbaa [[INT_TBAA0]], !range [[RNG4]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META14]]
 ; CHECK-NEXT:    [[V_UPTO0:%.*]] = insertelement <4 x i32> poison, i32 [[V_I0]], i64 0
 ; CHECK-NEXT:    [[V_UPTO1:%.*]] = insertelement <4 x i32> [[V_UPTO0]], i32 [[V_I1]], i64 1
 ; CHECK-NEXT:    [[V_UPTO2:%.*]] = insertelement <4 x i32> [[V_UPTO1]], i32 [[V_I2]], i64 2
@@ -24,9 +24,9 @@ define <4 x i32> @integer_load(ptr %p) {
 ;
 ; PACKED-LABEL: define <4 x i32> @integer_load(
 ; PACKED-SAME: ptr [[P:%.*]]) {
-; PACKED-NEXT:    [[V_I0:%.*]] = load <2 x i32>, ptr [[P]], align 16, !tbaa [[INT_TBAA0:![0-9]+]], !invariant.load [[META4:![0-9]+]], !alias.scope [[META5:![0-9]+]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8:![0-9]+]], !llvm.access.group [[META8]]
+; PACKED-NEXT:    [[V_I0:%.*]] = load <2 x i32>, ptr [[P]], align 16, !tbaa [[INT_TBAA0:![0-9]+]], !range [[RNG4:![0-9]+]], !invariant.load [[META5:![0-9]+]], !alias.scope [[META6:![0-9]+]], !noalias [[META6]], !nontemporal [[META9:![0-9]+]], !llvm.mem.parallel_loop_access [[META10:![0-9]+]], !invariant.group [[META5]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11:![0-9]+]], !nosanitize [[META5]], !mmra [[META12:![0-9]+]], !noalias.addrspace [[META13:![0-9]+]], !mem.cache_hint [[META14:![0-9]+]]
 ; PACKED-NEXT:    [[P_I1:%.*]] = getelementptr <2 x i32>, ptr [[P]], i32 1
-; PACKED-NEXT:    [[V_I1:%.*]] = load <2 x i32>, ptr [[P_I1]], align 8, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT:    [[V_I1:%.*]] = load <2 x i32>, ptr [[P_I1]], align 8, !tbaa [[INT_TBAA0]], !range [[RNG4]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META14]]
 ; PACKED-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i32> [[V_I0]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; PACKED-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[V_I1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; PACKED-NEXT:    [[V:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
@@ -39,13 +39,13 @@ define <4 x i32> @integer_load(ptr %p) {
 define <4 x float> @floating_load(ptr %p) {
 ; CHECK-LABEL: define <4 x float> @floating_load(
 ; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:    [[V_I0:%.*]] = load float, ptr [[P]], align 16, !tbaa [[FLOAT_TBAA9:![0-9]+]], !fpmath [[META11:![0-9]+]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT:    [[V_I0:%.*]] = load float, ptr [[P]], align 16, !tbaa [[FLOAT_TBAA16:![0-9]+]], !fpmath [[META18:![0-9]+]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !invariant.group [[META5]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !nofpclass [[META19:![0-9]+]], !mem.cache_hint [[META14]]
 ; CHECK-NEXT:    [[P_I1:%.*]] = getelementptr float, ptr [[P]], i32 1
-; CHECK-NEXT:    [[V_I1:%.*]] = load float, ptr [[P_I1]], align 4, !tbaa [[FLOAT_TBAA9]], !fpmath [[META11]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT:    [[V_I1:%.*]] = load float, ptr [[P_I1]], align 4, !tbaa [[FLOAT_TBAA16]], !fpmath [[META18]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !nofpclass [[META19]], !mem.cache_hint [[META14]]
 ; CHECK-NEXT:    [[P_I2:%.*]] = getelementptr float, ptr [[P]], i32 2
-; CHECK-NEXT:    [[V_I2:%.*]] = load float, ptr [[P_I2]], align 8, !tbaa [[FLOAT_TBAA9]], !fpmath [[META11]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT:    [[V_I2:%.*]] = load float, ptr [[P_I2]], align 8, !tbaa [[FLOAT_TBAA16]], !fpmath [[META18]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !nofpclass [[META19]], !mem.cache_hint [[META14]]
 ; CHECK-NEXT:    [[P_I3:%.*]] = getelementptr float, ptr [[P]], i32 3
-; CHECK-NEXT:    [[V_I3:%.*]] = load float, ptr [[P_I3]], align 4, !tbaa [[FLOAT_TBAA9]], !fpmath [[META11]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT:    [[V_I3:%.*]] = load float, ptr [[P_I3]], align 4, !tbaa [[FLOAT_TBAA16]], !fpmath [[META18]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !nofpclass [[META19]], !mem.cache_hint [[META14]]
 ; CHECK-NEXT:    [[V_UPTO0:%.*]] = insertelement <4 x float> poison, float [[V_I0]], i64 0
 ; CHECK-NEXT:    [[V_UPTO1:%.*]] = insertelement <4 x float> [[V_UPTO0]], float [[V_I1]], i64 1
 ; CHECK-NEXT:    [[V_UPTO2:%.*]] = insertelement <4 x float> [[V_UPTO1]], float [[V_I2]], i64 2
@@ -54,9 +54,9 @@ define <4 x float> @floating_load(ptr %p) {
 ;
 ; PACKED-LABEL: define <4 x float> @floating_load(
 ; PACKED-SAME: ptr [[P:%.*]]) {
-; PACKED-NEXT:    [[V_I0:%.*]] = load <2 x float>, ptr [[P]], align 16, !tbaa [[FLOAT_TBAA9:![0-9]+]], !fpmath [[META11:![0-9]+]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT:    [[V_I0:%.*]] = load <2 x float>, ptr [[P]], align 16, !tbaa [[FLOAT_TBAA16:![0-9]+]], !fpmath [[META18:![0-9]+]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !invariant.group [[META5]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !nofpclass [[META19:![0-9]+]], !mem.cache_hint [[META14]]
 ; PACKED-NEXT:    [[P_I1:%.*]] = getelementptr <2 x float>, ptr [[P]], i32 1
-; PACKED-NEXT:    [[V_I1:%.*]] = load <2 x float>, ptr [[P_I1]], align 8, !tbaa [[FLOAT_TBAA9]], !fpmath [[META11]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT:    [[V_I1:%.*]] = load <2 x float>, ptr [[P_I1]], align 8, !tbaa [[FLOAT_TBAA16]], !fpmath [[META18]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !nofpclass [[META19]], !mem.cache_hint [[META14]]
 ; PACKED-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[V_I0]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; PACKED-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[V_I1]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; PACKED-NEXT:    [[V:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
@@ -70,25 +70,25 @@ define void @integer_store(ptr %p, <4 x i32> %v) {
 ; CHECK-LABEL: define void @integer_store(
 ; CHECK-SAME: ptr [[P:%.*]], <4 x i32> [[V:%.*]]) {
 ; CHECK-NEXT:    [[V_I0:%.*]] = extractelement <4 x i32> [[V]], i64 0
-; CHECK-NEXT:    store i32 [[V_I0]], ptr [[P]], align 16, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT:    store i32 [[V_I0]], ptr [[P]], align 16, !tbaa [[INT_TBAA0]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !invariant.group [[META5]], !llvm.access.group [[META10]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META20:![0-9]+]]
 ; CHECK-NEXT:    [[V_I1:%.*]] = extractelement <4 x i32> [[V]], i64 1
 ; CHECK-NEXT:    [[P_I1:%.*]] = getelementptr i32, ptr [[P]], i32 1
-; CHECK-NEXT:    store i32 [[V_I1]], ptr [[P_I1]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT:    store i32 [[V_I1]], ptr [[P_I1]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META20]]
 ; CHECK-NEXT:    [[V_I2:%.*]] = extractelement <4 x i32> [[V]], i64 2
 ; CHECK-NEXT:    [[P_I2:%.*]] = getelementptr i32, ptr [[P]], i32 2
-; CHECK-NEXT:    store i32 [[V_I2]], ptr [[P_I2]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT:    store i32 [[V_I2]], ptr [[P_I2]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META20]]
 ; CHECK-NEXT:    [[V_I3:%.*]] = extractelement <4 x i32> [[V]], i64 3
 ; CHECK-NEXT:    [[P_I3:%.*]] = getelementptr i32, ptr [[P]], i32 3
-; CHECK-NEXT:    store i32 [[V_I3]], ptr [[P_I3]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT:    store i32 [[V_I3]], ptr [[P_I3]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META20]]
 ; CHECK-NEXT:    ret void
 ;
 ; PACKED-LABEL: define void @integer_store(
 ; PACKED-SAME: ptr [[P:%.*]], <4 x i32> [[V:%.*]]) {
 ; PACKED-NEXT:    [[V_I0:%.*]] = shufflevector <4 x i32> [[V]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>
-; PACKED-NEXT:    store <2 x i32> [[V_I0]], ptr [[P]], align 16, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT:    store <2 x i32> [[V_I0]], ptr [[P]], align 16, !tbaa [[INT_TBAA0]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !invariant.group [[META5]], !llvm.access.group [[META10]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META20:![0-9]+]]
 ; PACKED-NEXT:    [[V_I1:%.*]] = shufflevector <4 x i32> [[V]], <4 x i32> poison, <2 x i32> <i32 2, i32 3>
 ; PACKED-NEXT:    [[P_I1:%.*]] = getelementptr <2 x i32>, ptr [[P]], i32 1
-; PACKED-NEXT:    store <2 x i32> [[V_I1]], ptr [[P_I1]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT:    store <2 x i32> [[V_I1]], ptr [[P_I1]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META20]]
 ; PACKED-NEXT:    ret void
 ;
   store <4 x i32> %v, ptr %p, align 16, !tbaa !10, !invariant.group !0, !mem.cache_hint !18, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
@@ -126,25 +126,43 @@ define void @integer_store(ptr %p, <4 x i32> %v) {
 ; CHECK: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
 ; CHECK: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
 ; CHECK: [[META3]] = !{!"root"}
-; CHECK: [[META4]] = !{}
-; CHECK: [[META5]] = !{[[META6:![0-9]+]]}
-; CHECK: [[META6]] = distinct !{[[META6]], [[META7:![0-9]+]], !"scope"}
-; CHECK: [[META7]] = distinct !{[[META7]], i1 false, !"domain"}
-; CHECK: [[META8]] = distinct !{}
-; CHECK: [[FLOAT_TBAA9]] = !{[[META10:![0-9]+]], [[META10]], i64 0}
-; CHECK: [[META10]] = !{!"float", [[META2]], i64 0}
-; CHECK: [[META11]] = !{float 2.500000e+00}
+; CHECK: [[RNG4]] = !{i32 0, i32 100}
+; CHECK: [[META5]] = !{}
+; CHECK: [[META6]] = !{[[META7:![0-9]+]]}
+; CHECK: [[META7]] = distinct !{[[META7]], [[META8:![0-9]+]], !"scope"}
+; CHECK: [[META8]] = distinct !{[[META8]], i1 false, !"domain"}
+; CHECK: [[META9]] = !{i32 1}
+; CHECK: [[META10]] = distinct !{}
+; CHECK: [[META11]] = !{!"preserved memory metadata"}
+; CHECK: [[META12]] = !{!"test", !"relaxed"}
+; CHECK: [[META13]] = !{i32 5, i32 6}
+; CHECK: [[META14]] = !{i32 0, [[META15:![0-9]+]]}
+; CHECK: [[META15]] = !{!"nvvm.l1_eviction", !"last"}
+; CHECK: [[FLOAT_TBAA16]] = !{[[META17:![0-9]+]], [[META17]], i64 0}
+; CHECK: [[META17]] = !{!"float", [[META2]], i64 0}
+; CHECK: [[META18]] = !{float 2.500000e+00}
+; CHECK: [[META19]] = !{i32 3}
+; CHECK: [[META20]] = !{i32 1, [[META15]]}
 ;.
 ; PACKED: [[INT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
 ; PACKED: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
 ; PACKED: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
 ; PACKED: [[META3]] = !{!"root"}
-; PACKED: [[META4]] = !{}
-; PACKED: [[META5]] = !{[[META6:![0-9]+]]}
-; PACKED: [[META6]] = distinct !{[[META6]], [[META7:![0-9]+]], !"scope"}
-; PACKED: [[META7]] = distinct !{[[META7]], i1 false, !"domain"}
-; PACKED: [[META8]] = distinct !{}
-; PACKED: [[FLOAT_TBAA9]] = !{[[META10:![0-9]+]], [[META10]], i64 0}
-; PACKED: [[META10]] = !{!"float", [[META2]], i64 0}
-; PACKED: [[META11]] = !{float 2.500000e+00}
+; PACKED: [[RNG4]] = !{i32 0, i32 100}
+; PACKED: [[META5]] = !{}
+; PACKED: [[META6]] = !{[[META7:![0-9]+]]}
+; PACKED: [[META7]] = distinct !{[[META7]], [[META8:![0-9]+]], !"scope"}
+; PACKED: [[META8]] = distinct !{[[META8]], i1 false, !"domain"}
+; PACKED: [[META9]] = !{i32 1}
+; PACKED: [[META10]] = distinct !{}
+; PACKED: [[META11]] = !{!"preserved memory metadata"}
+; PACKED: [[META12]] = !{!"test", !"relaxed"}
+; PACKED: [[META13]] = !{i32 5, i32 6}
+; PACKED: [[META14]] = !{i32 0, [[META15:![0-9]+]]}
+; PACKED: [[META15]] = !{!"nvvm.l1_eviction", !"last"}
+; PACKED: [[FLOAT_TBAA16]] = !{[[META17:![0-9]+]], [[META17]], i64 0}
+; PACKED: [[META17]] = !{!"float", [[META2]], i64 0}
+; PACKED: [[META18]] = !{float 2.500000e+00}
+; PACKED: [[META19]] = !{i32 3}
+; PACKED: [[META20]] = !{i32 1, [[META15]]}
 ;.
diff --git a/llvm/test/Transforms/VectorCombine/RISCV/memory-metadata.ll b/llvm/test/Transforms/VectorCombine/RISCV/memory-metadata.ll
index 4c29c342143cd8..2bebe520189884 100644
--- a/llvm/test/Transforms/VectorCombine/RISCV/memory-metadata.ll
+++ b/llvm/test/Transforms/VectorCombine/RISCV/memory-metadata.ll
@@ -5,7 +5,7 @@ define <8 x i16> @vp_bitcast_fixed(ptr %p, i32 %evl) {
 ; CHECK-LABEL: define <8 x i16> @vp_bitcast_fixed(
 ; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[EVL]], 2
-; CHECK-NEXT:    [[R:%.*]] = call <8 x i16> @llvm.vp.load.v8i16.p0(ptr align 4 [[P]], <8 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT:    [[R:%.*]] = call <8 x i16> @llvm.vp.load.v8i16.p0(ptr align 4 [[P]], <8 x i1> splat (i1 true), i32 [[TMP1]]), !invariant.load [[META0:![0-9]+]], !alias.scope [[META1:![0-9]+]], !noalias [[META1]], !nontemporal [[META4:![0-9]+]], !llvm.access.group [[ACC_GRP5:![0-9]+]], !annotation [[META6:![0-9]+]], !nosanitize [[META0]], !mmra [[META7:![0-9]+]], !noalias.addrspace [[META8:![0-9]+]], !mem.cache_hint [[META9:![0-9]+]]
 ; CHECK-NEXT:    ret <8 x i16> [[R]]
 ;
   %v = call <4 x i32> @llvm.vp.load(ptr align 4 %p, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, i32 %evl), !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !range !11, !invariant.load !3
@@ -17,7 +17,7 @@ define <vscale x 8 x i16> @vp_bitcast_scalable(ptr %p, i32 %evl) {
 ; CHECK-LABEL: define <vscale x 8 x i16> @vp_bitcast_scalable(
 ; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[EVL]], 2
-; CHECK-NEXT:    [[R:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 4 [[P]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT:    [[R:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 4 [[P]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP1]]), !invariant.load [[META0]], !alias.scope [[META1]], !noalias [[META1]], !nontemporal [[META4]], !llvm.access.group [[ACC_GRP5]], !annotation [[META6]], !nosanitize [[META0]], !mmra [[META7]], !noalias.addrspace [[META8]], !mem.cache_hint [[META9]]
 ; CHECK-NEXT:    ret <vscale x 8 x i16> [[R]]
 ;
   %v = call <vscale x 4 x i32> @llvm.vp.load(ptr align 4 %p, <vscale x 4 x i1> splat (i1 true), i32 %evl), !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !range !11, !invariant.load !3
@@ -45,3 +45,16 @@ define <vscale x 8 x i16> @vp_bitcast_scalable(ptr %p, i32 %evl) {
 !16 = !{!"omnipotent char", !17, i64 0}
 !17 = !{!"root"}
 !18 = !{i32 1, !2}
+;.
+; CHECK: [[META0]] = !{}
+; CHECK: [[META1]] = !{[[META2:![0-9]+]]}
+; CHECK: [[META2]] = distinct !{[[META2]], [[META3:![0-9]+]], !"scope"}
+; CHECK: [[META3]] = distinct !{[[META3]], i1 false, !"domain"}
+; CHECK: [[META4]] = !{i32 1}
+; CHECK: [[ACC_GRP5]] = distinct !{}
+; CHECK: [[META6]] = !{!"original access"}
+; CHECK: [[META7]] = !{!"test", !"tag"}
+; CHECK: [[META8]] = !{i32 5, i32 6}
+; CHECK: [[META9]] = !{i32 0, [[META10:![0-9]+]]}
+; CHECK: [[META10]] = !{!"nvvm.l1_eviction", !"last"}
+;.
diff --git a/llvm/test/Transforms/VectorCombine/X86/memory-metadata.ll b/llvm/test/Transforms/VectorCombine/X86/memory-metadata.ll
index e9f73bbdff4e2a..6563c3becfbef6 100644
--- a/llvm/test/Transforms/VectorCombine/X86/memory-metadata.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/memory-metadata.ll
@@ -4,7 +4,7 @@
 define <4 x float> @widen_scalar(ptr align 16 dereferenceable(16) %p) {
 ; CHECK-LABEL: define <4 x float> @widen_scalar(
 ; CHECK-SAME: ptr align 16 dereferenceable(16) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:    [[R:%.*]] = load <4 x float>, ptr [[P]], align 16
+; CHECK-NEXT:    [[R:%.*]] = load <4 x float>, ptr [[P]], align 16, !nontemporal [[META0:![0-9]+]], !annotation [[META1:![0-9]+]], !nosanitize [[META2:![0-9]+]], !mmra [[META3:![0-9]+]], !noalias.addrspace [[META4:![0-9]+]], !mem.cache_hint [[META5:![0-9]+]]
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %v = load float, ptr %p, align 4, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !nofpclass !12, !fpmath !13, !noundef !3, !invariant.load !3, !invariant.group !3, !tbaa !15
@@ -15,7 +15,7 @@ define <4 x float> @widen_scalar(ptr align 16 dereferenceable(16) %p) {
 define <4 x float> @widen_shifted_scalar(ptr align 16 dereferenceable(16) %p) {
 ; CHECK-LABEL: define <4 x float> @widen_shifted_scalar(
 ; CHECK-SAME: ptr align 16 dereferenceable(16) [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[P]], align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[P]], align 16, !nontemporal [[META0]], !annotation [[META1]], !nosanitize [[META2]], !mmra [[META3]], !noalias.addrspace [[META4]], !mem.cache_hint [[META5]]
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 3, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
@@ -28,7 +28,7 @@ define <4 x float> @widen_shifted_scalar(ptr align 16 dereferenceable(16) %p) {
 define <4 x i32> @widen_subvector(ptr align 16 dereferenceable(16) %p) {
 ; CHECK-LABEL: define <4 x i32> @widen_subvector(
 ; CHECK-SAME: ptr align 16 dereferenceable(16) [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[R:%.*]] = load <4 x i32>, ptr [[P]], align 16
+; CHECK-NEXT:    [[R:%.*]] = load <4 x i32>, ptr [[P]], align 16, !nontemporal [[META0]], !annotation [[META1]], !nosanitize [[META2]], !mmra [[META3]], !noalias.addrspace [[META4]], !mem.cache_hint [[META5]]
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
 ;
   %v = load <2 x i32>, ptr %p, align 4, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !range !11, !noundef !3, !invariant.load !3, !invariant.group !3, !tbaa !15
@@ -42,7 +42,7 @@ define <4 x i32> @widen_subvector(ptr align 16 dereferenceable(16) %p) {
 define <4 x i32> @shrink_load_insert(ptr align 16 dereferenceable(32) %p) {
 ; CHECK-LABEL: define <4 x i32> @shrink_load_insert(
 ; CHECK-SAME: ptr align 16 dereferenceable(32) [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[R:%.*]] = load <4 x i32>, ptr [[P]], align 16
+; CHECK-NEXT:    [[R:%.*]] = load <4 x i32>, ptr [[P]], align 16, !nontemporal [[META0]], !mem.cache_hint [[META5]]
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
 ;
   %v = load <8 x i32>, ptr %p, align 16, !nontemporal !0, !mem.cache_hint !1, !range !11, !noundef !3, !invariant.load !3, !alias.scope !7
@@ -70,3 +70,12 @@ define <4 x i32> @shrink_load_insert(ptr align 16 dereferenceable(32) %p) {
 !16 = !{!"omnipotent char", !17, i64 0}
 !17 = !{!"root"}
 !18 = !{i32 1, !2}
+;.
+; CHECK: [[META0]] = !{i32 1}
+; CHECK: [[META1]] = !{!"original access"}
+; CHECK: [[META2]] = !{}
+; CHECK: [[META3]] = !{!"test", !"tag"}
+; CHECK: [[META4]] = !{i32 5, i32 6}
+; CHECK: [[META5]] = !{i32 0, [[META6:![0-9]+]]}
+; CHECK: [[META6]] = !{!"nvvm.l1_eviction", !"last"}
+;.
diff --git a/llvm/test/Transforms/VectorCombine/load-bitcast-metadata.ll b/llvm/test/Transforms/VectorCombine/load-bitcast-metadata.ll
new file mode 100644
index 00000000000000..ea2d32527f8f6d
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/load-bitcast-metadata.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=vector-combine -data-layout=e-p:64:64 < %s | FileCheck %s
+; RUN: opt -S -passes=instcombine -data-layout=e-p:64:64 < %s | FileCheck %s --check-prefix=IC
+; RUN: opt -S -passes=vector-combine -data-layout=E-p:64:64 < %s | FileCheck %s
+
+; The old element range does not describe the reinterpreted i32 value.
+define i32 @bitcast_integer_range(ptr %p) {
+; CHECK-LABEL: define i32 @bitcast_integer_range(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = load i32, ptr [[P]], align 4, !nontemporal [[META0:![0-9]+]], !noundef [[META1:![0-9]+]], !annotation [[META2:![0-9]+]], !nosanitize [[META1]], !mmra [[META3:![0-9]+]], !mem.cache_hint [[META4:![0-9]+]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+; IC-LABEL: define i32 @bitcast_integer_range(
+; IC-SAME: ptr [[P:%.*]]) {
+; IC-NEXT:    [[V1:%.*]] = load i32, ptr [[P]], align 4, !nontemporal [[META0:![0-9]+]], !noundef [[META1:![0-9]+]], !annotation [[META2:![0-9]+]], !nosanitize [[META1]], !mmra [[META3:![0-9]+]], !mem.cache_hint [[META4:![0-9]+]]
+; IC-NEXT:    ret i32 [[V1]]
+;
+  %v = load <4 x i8>, ptr %p, align 4, !range !0, !noundef !1, !nontemporal !2, !mem.cache_hint !3, !annotation !5, !nosanitize !1, !mmra !8
+  %r = bitcast <4 x i8> %v to i32
+  ret i32 %r
+}
+
+; Floating-point value metadata must not be attached to an integer load.
+define i64 @bitcast_float_to_integer(ptr %p) {
+; CHECK-LABEL: define i64 @bitcast_float_to_integer(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = load i64, ptr [[P]], align 8, !invariant.load [[META1]], !invariant.group [[META1]], !noundef [[META1]]
+; CHECK-NEXT:    ret i64 [[R]]
+;
+; IC-LABEL: define i64 @bitcast_float_to_integer(
+; IC-SAME: ptr [[P:%.*]]) {
+; IC-NEXT:    [[V1:%.*]] = load i64, ptr [[P]], align 8, !invariant.load [[META1]], !invariant.group [[META1]], !noundef [[META1]]
+; IC-NEXT:    ret i64 [[V1]]
+;
+  %v = load <2 x float>, ptr %p, align 8, !nofpclass !6, !fpmath !7, !noundef !1, !invariant.load !1, !invariant.group !1
+  %r = bitcast <2 x float> %v to i64
+  ret i64 %r
+}
+
+; A single-element vector and its scalar have the same element facts.
+define float @bitcast_same_float_element(ptr %p) {
+; CHECK-LABEL: define float @bitcast_same_float_element(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = load float, ptr [[P]], align 4, !fpmath [[META6:![0-9]+]], !invariant.load [[META1]], !invariant.group [[META1]], !noundef [[META1]], !nofpclass [[META7:![0-9]+]]
+; CHECK-NEXT:    ret float [[R]]
+;
+; IC-LABEL: define float @bitcast_same_float_element(
+; IC-SAME: ptr [[P:%.*]]) {
+; IC-NEXT:    [[V1:%.*]] = load float, ptr [[P]], align 4, !fpmath [[META6:![0-9]+]], !invariant.load [[META1]], !invariant.group [[META1]], !noundef [[META1]], !nofpclass [[META7:![0-9]+]]
+; IC-NEXT:    ret float [[V1]]
+;
+  %v = load <1 x float>, ptr %p, align 4, !nofpclass !6, !fpmath !7, !noundef !1, !invariant.load !1, !invariant.group !1
+  %r = bitcast <1 x float> %v to float
+  ret float %r
+}
+
+!0 = !{i8 0, i8 100}
+!1 = !{}
+!2 = !{i32 1}
+!3 = !{i32 0, !4}
+!4 = !{!"nvvm.l1_eviction", !"last"}
+!5 = !{!"original access"}
+!6 = !{i32 3}
+!7 = !{float 2.5}
+
+!8 = !{!"test", !"tag"}
+;.
+; CHECK: [[META0]] = !{i32 1}
+; CHECK: [[META1]] = !{}
+; CHECK: [[META2]] = !{!"original access"}
+; CHECK: [[META3]] = !{!"test", !"tag"}
+; CHECK: [[META4]] = !{i32 0, [[META5:![0-9]+]]}
+; CHECK: [[META5]] = !{!"nvvm.l1_eviction", !"last"}
+; CHECK: [[META6]] = !{float 2.500000e+00}
+; CHECK: [[META7]] = !{i32 3}
+;.
+; IC: [[META0]] = !{i32 1}
+; IC: [[META1]] = !{}
+; IC: [[META2]] = !{!"original access"}
+; IC: [[META3]] = !{!"test", !"tag"}
+; IC: [[META4]] = !{i32 0, [[META5:![0-9]+]]}
+; IC: [[META5]] = !{!"nvvm.l1_eviction", !"last"}
+; IC: [[META6]] = !{float 2.500000e+00}
+; IC: [[META7]] = !{i32 3}
+;.
diff --git a/llvm/test/Transforms/VectorCombine/memory-metadata.ll b/llvm/test/Transforms/VectorCombine/memory-metadata.ll
index d01dffceeb940a..cbf704e8780f45 100644
--- a/llvm/test/Transforms/VectorCombine/memory-metadata.ll
+++ b/llvm/test/Transforms/VectorCombine/memory-metadata.ll
@@ -5,7 +5,7 @@
 define i32 @extract_zero(ptr %p) {
 ; CHECK-LABEL: define i32 @extract_zero(
 ; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:    [[R:%.*]] = load i32, ptr [[P]], align 16, !tbaa [[CHAR_TBAA0:![0-9]+]], !alias.scope [[META3:![0-9]+]], !noalias [[META3]], !noalias.addrspace [[META6:![0-9]+]]
+; CHECK-NEXT:    [[R:%.*]] = load i32, ptr [[P]], align 16, !tbaa [[CHAR_TBAA0:![0-9]+]], !range [[RNG3:![0-9]+]], !invariant.load [[META4:![0-9]+]], !alias.scope [[META5:![0-9]+]], !noalias [[META5]], !nontemporal [[META8:![0-9]+]], !invariant.group [[META4]], !llvm.access.group [[ACC_GRP9:![0-9]+]], !noundef [[META4]], !annotation [[META10:![0-9]+]], !nosanitize [[META4]], !mmra [[META11:![0-9]+]], !noalias.addrspace [[META12:![0-9]+]], !mem.cache_hint [[META13:![0-9]+]]
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
   %v = load <4 x i32>, ptr %p, align 16, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !range !11, !noundef !3, !invariant.load !3, !invariant.group !3, !tbaa.struct !14
@@ -17,7 +17,7 @@ define i32 @extract_nonzero(ptr %p) {
 ; CHECK-LABEL: define i32 @extract_nonzero(
 ; CHECK-SAME: ptr [[P:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 2
-; CHECK-NEXT:    [[R:%.*]] = load i32, ptr [[TMP1]], align 8, !tbaa [[CHAR_TBAA0]], !alias.scope [[META3]], !noalias [[META3]], !noalias.addrspace [[META6]]
+; CHECK-NEXT:    [[R:%.*]] = load i32, ptr [[TMP1]], align 8, !tbaa [[CHAR_TBAA0]], !range [[RNG3]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.access.group [[ACC_GRP9]], !noundef [[META4]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META13]]
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
   %v = load <4 x i32>, ptr %p, align 16, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !range !11, !noundef !3, !invariant.load !3, !invariant.group !3, !tbaa.struct !14
@@ -32,7 +32,7 @@ define i32 @extract_dynamic(ptr %p, i32 %index) {
 ; CHECK-NEXT:    [[IDX:%.*]] = and i32 [[INDEX_FROZEN]], 3
 ; CHECK-NEXT:    [[IDX_GEPIDX:%.*]] = zext i32 [[IDX]] to i64
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 [[IDX_GEPIDX]]
-; CHECK-NEXT:    [[R:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[R:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[CHAR_TBAA0]], !range [[RNG3]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.access.group [[ACC_GRP9]], !noundef [[META4]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META13]]
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
   %idx = and i32 %index, 3
@@ -45,7 +45,7 @@ define float @extract_float(ptr %p) {
 ; CHECK-LABEL: define float @extract_float(
 ; CHECK-SAME: ptr [[P:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <4 x float>, ptr [[P]], i64 0, i64 1
-; CHECK-NEXT:    [[R:%.*]] = load float, ptr [[TMP1]], align 4, !alias.scope [[META3]], !noalias [[META3]], !noalias.addrspace [[META6]]
+; CHECK-NEXT:    [[R:%.*]] = load float, ptr [[TMP1]], align 4, !fpmath [[META15:![0-9]+]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.access.group [[ACC_GRP9]], !noundef [[META4]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !nofpclass [[META16:![0-9]+]], !mem.cache_hint [[META13]]
 ; CHECK-NEXT:    ret float [[R]]
 ;
   %v = load <4 x float>, ptr %p, align 16, !nontemporal !0, !mem.cache_hint !1, !annotation !4, !nosanitize !3, !mmra !5, !noalias.addrspace !6, !alias.scope !7, !noalias !7, !llvm.access.group !10, !nofpclass !12, !fpmath !13, !noundef !3, !invariant.load !3, !invariant.group !3
@@ -56,7 +56,7 @@ define float @extract_float(ptr %p) {
 define <4 x i32> @shrink_integer(ptr %p) {
 ; CHECK-LABEL: define <4 x i32> @shrink_integer(
 ; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 32, !tbaa [[CHAR_TBAA0]], !range [[RNG7:![0-9]+]], !invariant.load [[META8:![0-9]+]], !alias.scope [[META3]], !noalias [[META3]], !nontemporal [[META9:![0-9]+]], !invariant.group [[META8]], !llvm.access.group [[ACC_GRP10:![0-9]+]], !noundef [[META8]], !annotation [[META11:![0-9]+]], !nosanitize [[META8]], !mmra [[META12:![0-9]+]], !noalias.addrspace [[META6]], !mem.cache_hint [[META13:![0-9]+]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 32, !tbaa [[CHAR_TBAA0]], !range [[RNG3]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !invariant.group [[META4]], !llvm.access.group [[ACC_GRP9]], !noundef [[META4]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META13]]
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
 ;
@@ -68,7 +68,7 @@ define <4 x i32> @shrink_integer(ptr %p) {
 define <4 x float> @shrink_float(ptr %p) {
 ; CHECK-LABEL: define <4 x float> @shrink_float(
 ; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[P]], align 32, !fpmath [[META15:![0-9]+]], !invariant.load [[META8]], !alias.scope [[META3]], !noalias [[META3]], !nontemporal [[META9]], !invariant.group [[META8]], !llvm.access.group [[ACC_GRP10]], !noundef [[META8]], !annotation [[META11]], !nosanitize [[META8]], !mmra [[META12]], !noalias.addrspace [[META6]], !nofpclass [[META16:![0-9]+]], !mem.cache_hint [[META13]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[P]], align 32, !fpmath [[META15]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !invariant.group [[META4]], !llvm.access.group [[ACC_GRP9]], !noundef [[META4]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !nofpclass [[META16]], !mem.cache_hint [[META13]]
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
@@ -80,7 +80,7 @@ define <4 x float> @shrink_float(ptr %p) {
 define void @store_zero(ptr %p, i8 %x, i32 %index) {
 ; CHECK-LABEL: define void @store_zero(
 ; CHECK-SAME: ptr [[P:%.*]], i8 [[X:%.*]], i32 [[INDEX:%.*]]) {
-; CHECK-NEXT:    store i8 [[X]], ptr [[P]], align 16, !tbaa.struct [[TBAA_STRUCT17:![0-9]+]], !alias.scope [[META3]], !noalias [[META3]], !nontemporal [[META9]], !llvm.access.group [[ACC_GRP10]], !annotation [[META11]], !nosanitize [[META8]], !mmra [[META12]], !noalias.addrspace [[META6]], !mem.cache_hint [[META18:![0-9]+]]
+; CHECK-NEXT:    store i8 [[X]], ptr [[P]], align 16, !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !invariant.group [[META4]], !llvm.access.group [[ACC_GRP9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META17:![0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
   %v = load <16 x i8>, ptr %p, align 16
@@ -93,7 +93,7 @@ define void @store_nonzero(ptr %p, i8 %x, i32 %index) {
 ; CHECK-LABEL: define void @store_nonzero(
 ; CHECK-SAME: ptr [[P:%.*]], i8 [[X:%.*]], i32 [[INDEX:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 3
-; CHECK-NEXT:    store i8 [[X]], ptr [[TMP1]], align 1, !tbaa.struct [[TBAA_STRUCT17]], !alias.scope [[META3]], !noalias [[META3]], !nontemporal [[META9]], !llvm.access.group [[ACC_GRP10]], !annotation [[META11]], !nosanitize [[META8]], !mmra [[META12]], !noalias.addrspace [[META6]], !mem.cache_hint [[META18]]
+; CHECK-NEXT:    store i8 [[X]], ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.access.group [[ACC_GRP9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META17]]
 ; CHECK-NEXT:    ret void
 ;
   %v = load <16 x i8>, ptr %p, align 16
@@ -109,7 +109,7 @@ define void @store_dynamic(ptr %p, i8 %x, i32 %index) {
 ; CHECK-NEXT:    [[IDX:%.*]] = and i32 [[INDEX_FROZEN]], 15
 ; CHECK-NEXT:    [[IDX_GEPIDX:%.*]] = zext i32 [[IDX]] to i64
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 [[IDX_GEPIDX]]
-; CHECK-NEXT:    store i8 [[X]], ptr [[TMP1]], align 1, !tbaa.struct [[TBAA_STRUCT17]], !alias.scope [[META3]], !noalias [[META3]], !nontemporal [[META9]], !llvm.access.group [[ACC_GRP10]], !annotation [[META11]], !nosanitize [[META8]], !mmra [[META12]], !noalias.addrspace [[META6]], !mem.cache_hint [[META18]]
+; CHECK-NEXT:    store i8 [[X]], ptr [[TMP1]], align 1, !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.access.group [[ACC_GRP9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META17]]
 ; CHECK-NEXT:    ret void
 ;
   %idx = and i32 %index, 15
@@ -125,7 +125,7 @@ define i32 @extract_struct_field(ptr %p) {
 ; CHECK-LABEL: define i32 @extract_struct_field(
 ; CHECK-SAME: ptr [[P:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 1
-; CHECK-NEXT:    [[R:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[INT_TBAA19:![0-9]+]]
+; CHECK-NEXT:    [[R:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[INT_TBAA18:![0-9]+]]
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
   %v = load <4 x i32>, ptr %p, align 16, !tbaa.struct !21
@@ -137,7 +137,7 @@ define void @store_struct_field(ptr %p, i32 %x) {
 ; CHECK-LABEL: define void @store_struct_field(
 ; CHECK-SAME: ptr [[P:%.*]], i32 [[X:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 1
-; CHECK-NEXT:    store i32 [[X]], ptr [[TMP1]], align 4, !tbaa.struct [[TBAA_STRUCT21:![0-9]+]]
+; CHECK-NEXT:    store i32 [[X]], ptr [[TMP1]], align 4, !tbaa [[INT_TBAA18]]
 ; CHECK-NEXT:    ret void
 ;
   %v = load <4 x i32>, ptr %p, align 16
@@ -173,23 +173,21 @@ define void @store_struct_field(ptr %p, i32 %x) {
 ; CHECK: [[CHAR_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
 ; CHECK: [[META1]] = !{!"omnipotent char", [[META2:![0-9]+]], i64 0}
 ; CHECK: [[META2]] = !{!"root"}
-; CHECK: [[META3]] = !{[[META4:![0-9]+]]}
-; CHECK: [[META4]] = distinct !{[[META4]], [[META5:![0-9]+]], !"scope"}
-; CHECK: [[META5]] = distinct !{[[META5]], i1 false, !"domain"}
-; CHECK: [[META6]] = !{i32 5, i32 6}
-; CHECK: [[RNG7]] = !{i32 0, i32 100}
-; CHECK: [[META8]] = !{}
-; CHECK: [[META9]] = !{i32 1}
-; CHECK: [[ACC_GRP10]] = distinct !{}
-; CHECK: [[META11]] = !{!"original access"}
-; CHECK: [[META12]] = !{!"test", !"tag"}
+; CHECK: [[RNG3]] = !{i32 0, i32 100}
+; CHECK: [[META4]] = !{}
+; CHECK: [[META5]] = !{[[META6:![0-9]+]]}
+; CHECK: [[META6]] = distinct !{[[META6]], [[META7:![0-9]+]], !"scope"}
+; CHECK: [[META7]] = distinct !{[[META7]], i1 false, !"domain"}
+; CHECK: [[META8]] = !{i32 1}
+; CHECK: [[ACC_GRP9]] = distinct !{}
+; CHECK: [[META10]] = !{!"original access"}
+; CHECK: [[META11]] = !{!"test", !"tag"}
+; CHECK: [[META12]] = !{i32 5, i32 6}
 ; CHECK: [[META13]] = !{i32 0, [[META14:![0-9]+]]}
 ; CHECK: [[META14]] = !{!"nvvm.l1_eviction", !"last"}
 ; CHECK: [[META15]] = !{float 2.500000e+00}
 ; CHECK: [[META16]] = !{i32 3}
-; CHECK: [[TBAA_STRUCT17]] = !{i64 0, i64 4, [[CHAR_TBAA0]], i64 4, i64 4, [[CHAR_TBAA0]], i64 8, i64 4, [[CHAR_TBAA0]], i64 12, i64 4, [[CHAR_TBAA0]]}
-; CHECK: [[META18]] = !{i32 1, [[META14]]}
-; CHECK: [[INT_TBAA19]] = !{[[META20:![0-9]+]], [[META20]], i64 0}
-; CHECK: [[META20]] = !{!"int", [[META1]], i64 0}
-; CHECK: [[TBAA_STRUCT21]] = !{i64 0, i64 4, [[CHAR_TBAA0]], i64 4, i64 4, [[INT_TBAA19]], i64 8, i64 4, [[CHAR_TBAA0]], i64 12, i64 4, [[CHAR_TBAA0]]}
+; CHECK: [[META17]] = !{i32 1, [[META14]]}
+; CHECK: [[INT_TBAA18]] = !{[[META19:![0-9]+]], [[META19]], i64 0}
+; CHECK: [[META19]] = !{!"int", [[META1]], i64 0}
 ;.
diff --git a/llvm/unittests/Analysis/CMakeLists.txt b/llvm/unittests/Analysis/CMakeLists.txt
index 43ad72b7bfa6dd..0732edeff7c88e 100644
--- a/llvm/unittests/Analysis/CMakeLists.txt
+++ b/llvm/unittests/Analysis/CMakeLists.txt
@@ -44,6 +44,7 @@ set(ANALYSIS_TEST_SOURCES
   MemoryBuiltinsTest.cpp
   MemoryProfileInfoTest.cpp
   MemorySSATest.cpp
+  MetadataUtilsTest.cpp
   MLGOUtilsTest.cpp
   MLModelRunnerTest.cpp
   PhiValuesTest.cpp
diff --git a/llvm/unittests/Analysis/MetadataUtilsTest.cpp b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
new file mode 100644
index 00000000000000..ea752404ddcfb3
--- /dev/null
+++ b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
@@ -0,0 +1,390 @@
+//===- MetadataUtilsTest.cpp - Metadata tests --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/MetadataUtils.h"
+#include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/MDBuilder.h"
+#include "llvm/IR/MemCacheHint.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Verifier.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+namespace {
+class MetadataUtilsTest : public testing::Test {
+protected:
+  LLVMContext C;
+  Module M{"test", C};
+  IRBuilder<> B{C};
+  Value *P;
+  Value *Q;
+
+  MetadataUtilsTest() {
+    M.setDataLayout("e-p:64:64");
+    auto *F = Function::Create(
+        FunctionType::get(B.getVoidTy(), {B.getPtrTy(), B.getPtrTy()}, false),
+        Function::ExternalLinkage, "test", M);
+    P = F->getArg(0);
+    Q = F->getArg(1);
+    B.SetInsertPoint(BasicBlock::Create(C, "entry", F));
+    B.SetInsertPoint(B.CreateRetVoid());
+  }
+
+  void TearDown() override { EXPECT_FALSE(verifyModule(M, &errs())); }
+
+  MDNode *empty() { return MDNode::get(C, {}); }
+  Metadata *number(uint64_t N) {
+    return ConstantAsMetadata::get(B.getInt64(N));
+  }
+  MDNode *hint() {
+    return MDNode::get(
+        C, {MDString::get(C, "nvvm.l1_eviction"), MDString::get(C, "last")});
+  }
+  MDNode *range() {
+    return MDNode::get(C, {ConstantAsMetadata::get(B.getInt32(0)),
+                           ConstantAsMetadata::get(B.getInt32(100))});
+  }
+  MDNode *scope() {
+    MDBuilder MDB(C);
+    return MDNode::get(C, {MDB.createAnonymousAliasScope(
+                              MDB.createAnonymousAliasScopeDomain())});
+  }
+};
+
+TEST_F(MetadataUtilsTest, NarrowedLoadPreservesElementFacts) {
+  auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), P);
+  auto *Ptr = B.CreateInBoundsGEP(B.getInt32Ty(), P, B.getInt64(2));
+  auto *New = B.CreateLoad(B.getInt32Ty(), Ptr);
+  auto *Range = range();
+  auto *Hint = hint();
+  auto *Scope = scope();
+  Old->setMetadata(LLVMContext::MD_range, Range);
+  Old->setMetadata(LLVMContext::MD_noundef, empty());
+  Old->setMetadata(LLVMContext::MD_invariant_load, empty());
+  Old->setMetadata(LLVMContext::MD_invariant_group, empty());
+  Old->setMetadata(LLVMContext::MD_alias_scope, Scope);
+  setMemCacheHint(*Old, 0, Hint);
+  copyMetadataForMemoryAccess(*New, *Old, 8);
+  EXPECT_EQ(New->getMetadata(LLVMContext::MD_range), Range);
+  EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_noundef));
+  EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_invariant_load));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_invariant_group));
+  EXPECT_EQ(New->getMetadata(LLVMContext::MD_alias_scope), Scope);
+  EXPECT_EQ(getMemCacheHint(*New), Hint);
+}
+
+TEST_F(MetadataUtilsTest, InvariantGroupRequiresPointerIdentity) {
+  auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), P);
+  auto *Same = B.CreateLoad(B.getInt32Ty(), P);
+  auto *Different = B.CreateLoad(B.getInt32Ty(), Q);
+  Old->setMetadata(LLVMContext::MD_invariant_group, empty());
+  copyMetadataForMemoryAccess(*Same, *Old);
+  copyMetadataForMemoryAccess(*Different, *Old);
+  EXPECT_TRUE(Same->hasMetadata(LLVMContext::MD_invariant_group));
+  EXPECT_FALSE(Different->hasMetadata(LLVMContext::MD_invariant_group));
+}
+
+TEST_F(MetadataUtilsTest, WideningDropsFactsAboutOriginalBytes) {
+  auto *Old = B.CreateLoad(B.getInt32Ty(), P);
+  auto *New = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), P);
+  auto *Hint = hint();
+  auto *Annotation = MDNode::get(C, {MDString::get(C, "original")});
+  Old->setMetadata(LLVMContext::MD_range, range());
+  Old->setMetadata(LLVMContext::MD_noundef, empty());
+  Old->setMetadata(LLVMContext::MD_invariant_load, empty());
+  Old->setMetadata(LLVMContext::MD_invariant_group, empty());
+  Old->setMetadata(LLVMContext::MD_alias_scope, scope());
+  Old->setMetadata(LLVMContext::MD_annotation, Annotation);
+  Old->setMetadata("test.unknown", empty());
+  setMemCacheHint(*Old, 0, Hint);
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_noundef));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_invariant_load));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_invariant_group));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_alias_scope));
+  EXPECT_FALSE(New->getMetadata("test.unknown"));
+  EXPECT_EQ(New->getMetadata(LLVMContext::MD_annotation), Annotation);
+  EXPECT_EQ(getMemCacheHint(*New), Hint);
+}
+
+TEST_F(MetadataUtilsTest, EarlierAddressIsNotASubset) {
+  auto *Ptr = B.CreateInBoundsGEP(B.getInt32Ty(), P, B.getInt64(2));
+  auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), Ptr);
+  auto *New = B.CreateLoad(B.getInt32Ty(), P);
+  Old->setMetadata(LLVMContext::MD_range, range());
+  copyMetadataForMemoryAccess(*New, *Old, -8);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+}
+
+TEST_F(MetadataUtilsTest, BitcastDropsDifferentIntegerElementRange) {
+  auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 2), P);
+  auto *New = B.CreateLoad(B.getInt64Ty(), P);
+  Old->setMetadata(LLVMContext::MD_range, range());
+  Old->setMetadata(LLVMContext::MD_noundef, empty());
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+  EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_noundef));
+}
+
+TEST_F(MetadataUtilsTest, BitcastDropsFloatingPointValueFacts) {
+  auto *Old = B.CreateLoad(FixedVectorType::get(B.getFloatTy(), 2), P);
+  auto *New = B.CreateLoad(B.getInt64Ty(), P);
+  Old->setMetadata(LLVMContext::MD_nofpclass,
+                   MDNode::get(C, {ConstantAsMetadata::get(B.getInt32(3))}));
+  Old->setMetadata(LLVMContext::MD_fpmath,
+                   MDNode::get(C, {ConstantAsMetadata::get(
+                                      ConstantFP::get(B.getFloatTy(), 2.5))}));
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_nofpclass));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_fpmath));
+}
+
+TEST_F(MetadataUtilsTest, PointerNonnullBecomesIntegerRange) {
+  auto *Old = B.CreateLoad(B.getPtrTy(), P);
+  auto *New = B.CreateLoad(B.getInt64Ty(), P);
+  Old->setMetadata(LLVMContext::MD_nonnull, empty());
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_nonnull));
+  ASSERT_TRUE(New->hasMetadata(LLVMContext::MD_range));
+  auto *Range = New->getMetadata(LLVMContext::MD_range);
+  EXPECT_EQ(mdconst::extract<ConstantInt>(Range->getOperand(0))->getZExtValue(),
+            1u);
+  EXPECT_EQ(mdconst::extract<ConstantInt>(Range->getOperand(1))->getZExtValue(),
+            0u);
+}
+
+TEST_F(MetadataUtilsTest, UnknownOffsetKeepsAccessTagsButDropsFieldLayout) {
+  auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), P);
+  auto *Index = B.CreateAnd(B.CreateLoad(B.getInt64Ty(), Q), B.getInt64(3));
+  auto *Ptr = B.CreateInBoundsGEP(B.getInt32Ty(), P, Index);
+  auto *New = B.CreateLoad(B.getInt32Ty(), Ptr);
+  MDBuilder MDB(C);
+  auto *Char = MDB.createTBAAScalarTypeNode("char", MDB.createTBAARoot("root"));
+  auto *Tag = MDB.createTBAAStructTagNode(Char, Char, 0);
+  auto *Scope = scope();
+  Old->setMetadata(LLVMContext::MD_tbaa, Tag);
+  Old->setMetadata(LLVMContext::MD_tbaa_struct,
+                   MDNode::get(C, {number(0), number(4), Tag}));
+  Old->setMetadata(LLVMContext::MD_alias_scope, Scope);
+  copyMetadataForMemoryAccess(*New, *Old, std::nullopt);
+  EXPECT_EQ(New->getMetadata(LLVMContext::MD_tbaa), Tag);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_tbaa_struct));
+  EXPECT_EQ(New->getMetadata(LLVMContext::MD_alias_scope), Scope);
+}
+
+TEST_F(MetadataUtilsTest, ConstantOffsetSelectsFieldTag) {
+  auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 2), P);
+  auto *New = B.CreateLoad(B.getInt32Ty(), Q);
+  MDBuilder MDB(C);
+  auto *Root = MDB.createTBAARoot("root");
+  auto *Char = MDB.createTBAAScalarTypeNode("char", Root);
+  auto *Int = MDB.createTBAAScalarTypeNode("int", Char);
+  auto *CharTag = MDB.createTBAAStructTagNode(Char, Char, 0);
+  auto *IntTag = MDB.createTBAAStructTagNode(Int, Int, 0);
+  Old->setMetadata(LLVMContext::MD_tbaa_struct,
+                   MDNode::get(C, {number(0), number(4), CharTag, number(4),
+                                   number(4), IntTag}));
+  copyMetadataForMemoryAccess(*New, *Old, 4);
+  EXPECT_EQ(New->getMetadata(LLVMContext::MD_tbaa), IntTag);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_tbaa_struct));
+}
+
+TEST_F(MetadataUtilsTest, WholeAccessSelectsSingleFieldTag) {
+  auto *Old = B.CreateLoad(B.getInt64Ty(), P);
+  auto *New = B.CreateLoad(B.getDoubleTy(), P);
+  MDBuilder MDB(C);
+  auto *Char = MDB.createTBAAScalarTypeNode("char", MDB.createTBAARoot("root"));
+  auto *Tag = MDB.createTBAAStructTagNode(Char, Char, 0);
+  Old->setMetadata(LLVMContext::MD_tbaa_struct,
+                   MDNode::get(C, {number(0), number(8), Tag}));
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_EQ(New->getMetadata(LLVMContext::MD_tbaa), Tag);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_tbaa_struct));
+}
+
+TEST_F(MetadataUtilsTest, WholeAccessRetainsMultipleFieldTags) {
+  auto *Ty = FixedVectorType::get(B.getInt32Ty(), 2);
+  auto *Old = B.CreateLoad(Ty, P);
+  auto *New = B.CreateLoad(Ty, P);
+  MDBuilder MDB(C);
+  auto *Char = MDB.createTBAAScalarTypeNode("char", MDB.createTBAARoot("root"));
+  auto *Int = MDB.createTBAAScalarTypeNode("int", Char);
+  auto *CharTag = MDB.createTBAAStructTagNode(Char, Char, 0);
+  auto *IntTag = MDB.createTBAAStructTagNode(Int, Int, 0);
+  auto *Fields = MDNode::get(
+      C, {number(0), number(4), CharTag, number(4), number(4), IntTag});
+  Old->setMetadata(LLVMContext::MD_tbaa_struct, Fields);
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_tbaa));
+  EXPECT_EQ(New->getMetadata(LLVMContext::MD_tbaa_struct), Fields);
+}
+
+TEST_F(MetadataUtilsTest, MemoryTransferRemapsBothOperands) {
+  auto *Copy = B.CreateMemCpy(P, Align(4), Q, Align(4), B.getInt64(4));
+  auto *Read = B.CreateLoad(B.getInt32Ty(), Q);
+  auto *Write = B.CreateStore(B.getInt32(1), P);
+  auto *ReadHint = hint();
+  auto *WriteHint = MDNode::get(
+      C, {MDString::get(C, "nvvm.l1_eviction"), MDString::get(C, "first")});
+  setMemCacheHint(*Copy, 0, WriteHint);
+  setMemCacheHint(*Copy, 1, ReadHint);
+  copyMetadataForMemoryAccess(*Read, *Copy);
+  copyMetadataForMemoryAccess(*Write, *Copy);
+  EXPECT_EQ(getMemCacheHint(*Read, 0), ReadHint);
+  EXPECT_EQ(getMemCacheHint(*Write, 1), WriteHint);
+  EXPECT_FALSE(getMemCacheHint(*Read, 1));
+  EXPECT_FALSE(getMemCacheHint(*Write, 0));
+}
+
+TEST_F(MetadataUtilsTest, RuntimeMemoryTransferLength) {
+  auto *Length = B.CreateLoad(B.getInt64Ty(), Q);
+  B.CreateAssumption(B.CreateICmpUGE(Length, B.getInt64(8)));
+  auto *Copy = B.CreateMemCpy(P, Align(4), Q, Align(4), Length);
+  auto *Ptr = B.CreateInBoundsGEP(B.getInt8Ty(), Q, B.getInt64(4));
+  auto *Read = B.CreateLoad(B.getInt32Ty(), Ptr);
+  auto *Hint = hint();
+  setMemCacheHint(*Copy, 1, Hint);
+  copyMetadataForMemoryAccess(*Read, *Copy, 4);
+  EXPECT_EQ(getMemCacheHint(*Read), Hint);
+}
+
+TEST_F(MetadataUtilsTest, StoreTypeChangePreservesHints) {
+  auto *Old = B.CreateStore(B.getInt32(0), P);
+  auto *New = B.CreateStore(ConstantFP::get(B.getFloatTy(), 0), P);
+  auto *Hint = hint();
+  setMemCacheHint(*Old, 1, Hint);
+  Old->setMetadata(LLVMContext::MD_invariant_group, empty());
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_EQ(getMemCacheHint(*New, 1), Hint);
+  EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_invariant_group));
+}
+
+TEST_F(MetadataUtilsTest, NonMemoryScalarizationPreservesFPMath) {
+  auto *Ty = FixedVectorType::get(B.getFloatTy(), 2);
+  auto *Old =
+      cast<Instruction>(B.CreateFAdd(B.CreateLoad(Ty, P), B.CreateLoad(Ty, Q)));
+  auto *New = cast<Instruction>(B.CreateFAdd(B.CreateLoad(B.getFloatTy(), P),
+                                             B.CreateLoad(B.getFloatTy(), Q)));
+  auto *Tag = MDNode::get(
+      C, {ConstantAsMetadata::get(ConstantFP::get(B.getFloatTy(), 2.5))});
+  Old->setMetadata(LLVMContext::MD_fpmath, Tag);
+  copyMetadataForScalarizedInstruction(*New, *Old);
+  EXPECT_EQ(New->getMetadata(LLVMContext::MD_fpmath), Tag);
+}
+
+TEST_F(MetadataUtilsTest, CSEIntersectsCacheHints) {
+  auto *A = B.CreateLoad(B.getInt32Ty(), P);
+  auto *BLoad = B.CreateLoad(B.getInt32Ty(), P);
+  auto *Hint = hint();
+  setMemCacheHint(*A, 0, Hint);
+  setMemCacheHint(*BLoad, 0, Hint);
+  combineMetadataForCSE(A, BLoad, false);
+  EXPECT_EQ(getMemCacheHint(*A), Hint);
+  BLoad->setMetadata(LLVMContext::MD_mem_cache_hint, nullptr);
+  combineMetadataForCSE(A, BLoad, false);
+  EXPECT_FALSE(getMemCacheHint(*A));
+}
+
+TEST_F(MetadataUtilsTest, AddressSpaceCastChangesInvariantGroupIdentity) {
+  auto *Old = B.CreateLoad(B.getInt32Ty(), P);
+  auto *NewPtr = B.CreateAddrSpaceCast(P, B.getPtrTy(1));
+  auto *New = B.CreateLoad(B.getInt32Ty(), NewPtr);
+  Old->setMetadata(LLVMContext::MD_invariant_group, empty());
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_invariant_group));
+}
+
+TEST_F(MetadataUtilsTest, DefinedAggregateDoesNotDefinePadding) {
+  auto *Ty = StructType::get(C, {B.getInt8Ty(), B.getInt32Ty()});
+  ASSERT_EQ(M.getDataLayout().getTypeStoreSize(Ty).getFixedValue(), 8u);
+  auto *Old = B.CreateLoad(Ty, P);
+  auto *New = B.CreateLoad(B.getInt64Ty(), P);
+  Old->setMetadata(LLVMContext::MD_noundef, empty());
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_noundef));
+}
+
+TEST_F(MetadataUtilsTest, PointerFactsRequirePointerResults) {
+  auto *Old = B.CreateLoad(B.getPtrTy(), P);
+  auto *Pointer = B.CreateLoad(B.getPtrTy(), P);
+  auto *Integer = B.CreateLoad(B.getInt64Ty(), P);
+  auto *Align = MDNode::get(C, {number(16)});
+  auto *Deref = MDNode::get(C, {number(32)});
+  Old->setMetadata(LLVMContext::MD_nonnull, empty());
+  Old->setMetadata(LLVMContext::MD_align, Align);
+  Old->setMetadata(LLVMContext::MD_dereferenceable, Deref);
+  Old->setMetadata(LLVMContext::MD_dereferenceable_or_null, Deref);
+  copyMetadataForMemoryAccess(*Pointer, *Old);
+  copyMetadataForMemoryAccess(*Integer, *Old);
+  EXPECT_TRUE(Pointer->hasMetadata(LLVMContext::MD_nonnull));
+  EXPECT_EQ(Pointer->getMetadata(LLVMContext::MD_align), Align);
+  EXPECT_EQ(Pointer->getMetadata(LLVMContext::MD_dereferenceable), Deref);
+  EXPECT_EQ(Pointer->getMetadata(LLVMContext::MD_dereferenceable_or_null),
+            Deref);
+  EXPECT_FALSE(Integer->hasMetadata(LLVMContext::MD_nonnull));
+  EXPECT_FALSE(Integer->hasMetadata(LLVMContext::MD_align));
+  EXPECT_FALSE(Integer->hasMetadata(LLVMContext::MD_dereferenceable));
+  EXPECT_FALSE(Integer->hasMetadata(LLVMContext::MD_dereferenceable_or_null));
+  EXPECT_TRUE(Integer->hasMetadata(LLVMContext::MD_range));
+}
+
+TEST_F(MetadataUtilsTest, StoreFactsRequireWholeAccessAndPointerValue) {
+  auto *Old = B.CreateStore(Q, P);
+  auto *Pointer = B.CreateStore(Q, P);
+  auto *Integer = B.CreateStore(B.CreatePtrToInt(Q, B.getInt64Ty()), P);
+  auto *Part = B.CreateStore(B.getInt32(0), P);
+  auto *AssignID = DIAssignID::getDistinct(C);
+  auto *Captures = MDNode::get(C, {MDString::get(C, "address")});
+  Old->setMetadata(LLVMContext::MD_DIAssignID, AssignID);
+  Old->setMetadata(LLVMContext::MD_captures, Captures);
+  copyMetadataForMemoryAccess(*Pointer, *Old);
+  copyMetadataForMemoryAccess(*Integer, *Old);
+  copyMetadataForMemoryAccess(*Part, *Old);
+  EXPECT_EQ(Pointer->getMetadata(LLVMContext::MD_DIAssignID), AssignID);
+  EXPECT_EQ(Pointer->getMetadata(LLVMContext::MD_captures), Captures);
+  EXPECT_EQ(Integer->getMetadata(LLVMContext::MD_DIAssignID), AssignID);
+  EXPECT_FALSE(Integer->hasMetadata(LLVMContext::MD_captures));
+  EXPECT_FALSE(Part->hasMetadata(LLVMContext::MD_DIAssignID));
+  EXPECT_FALSE(Part->hasMetadata(LLVMContext::MD_captures));
+}
+
+TEST_F(MetadataUtilsTest, IntrinsicProfilesAndUnknownMetadataAreNotCopied) {
+  auto *Ty = FixedVectorType::get(B.getInt32Ty(), 2);
+  auto *Mask = B.CreateLoad(FixedVectorType::get(B.getInt1Ty(), 2), Q);
+  auto *Old = B.CreateMaskedLoad(Ty, P, Align(4), Mask);
+  auto *New = B.CreateLoad(Ty, P);
+  auto *Profile = MDNode::get(C, {MDString::get(C, "branch_weights"),
+                                  ConstantAsMetadata::get(B.getInt32(10))});
+  Old->setMetadata(LLVMContext::MD_prof, Profile);
+  Old->setMetadata("test.unknown", empty());
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_prof));
+  EXPECT_FALSE(New->getMetadata("test.unknown"));
+}
+
+TEST_F(MetadataUtilsTest, SmallerAccessOutsideSourceRangeIsNotASubset) {
+  auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), P);
+  auto *Ptr = B.CreateInBoundsGEP(B.getInt32Ty(), P, B.getInt64(4));
+  auto *New = B.CreateLoad(B.getInt32Ty(), Ptr);
+  Old->setMetadata(LLVMContext::MD_range, range());
+  Old->setMetadata(LLVMContext::MD_noundef, empty());
+  Old->setMetadata(LLVMContext::MD_invariant_load, empty());
+  Old->setMetadata(LLVMContext::MD_alias_scope, scope());
+  Old->setMetadata(LLVMContext::MD_noalias, scope());
+  copyMetadataForMemoryAccess(*New, *Old, 16);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_noundef));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_invariant_load));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_alias_scope));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_noalias));
+}
+
+} // namespace

>From a3aeb4c617c1131c264fcf321a074631db907fc5 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Sat, 14 Nov 2026 00:41:15 +0000
Subject: [PATCH 03/14] some fixes and comments to make things clearer

---
 llvm/include/llvm/Analysis/MetadataUtils.h    |  25 +--
 llvm/lib/Analysis/MetadataUtils.cpp           | 142 +++++++++++-------
 .../Scalar/ScalarizeMaskedMemIntrin.cpp       |  16 +-
 llvm/test/Transforms/SROA/memory-metadata.ll  |   4 +-
 llvm/unittests/Analysis/MetadataUtilsTest.cpp |  66 ++++++++
 5 files changed, 173 insertions(+), 80 deletions(-)

diff --git a/llvm/include/llvm/Analysis/MetadataUtils.h b/llvm/include/llvm/Analysis/MetadataUtils.h
index bfaf8bd6912e8f..2d7ce97f1cdd13 100644
--- a/llvm/include/llvm/Analysis/MetadataUtils.h
+++ b/llvm/include/llvm/Analysis/MetadataUtils.h
@@ -23,24 +23,25 @@ class Value;
 
 /// Copy metadata valid for a replacement memory access. The destination must
 /// access the same bytes, a subset, or an expanded range in the same object.
-/// ByteOffset is the destination's offset within the source access. A missing
-/// offset requires the caller to establish that the new access is a subset.
-/// Access types and pointer operands are inferred for loads, stores, and
-/// masked/VP memory intrinsics. For a source memory intrinsic, the source or
-/// destination pointer is selected to match the new read or write. The caller
-/// must establish range containment when the intrinsic's length is unknown.
-/// Pointer and value facts are retained only when valid for the destination;
-/// alias tags are adjusted for its range. Masks and execution conditions are
-/// not inspected.
+/// The destination must be a typed memory access; memory-transfer and
+/// memory-set intrinsics are supported only as sources. DestByteOffset is the
+/// destination's offset within the source access. A missing offset requires the
+/// caller to establish that the new access is a subset. Access types and
+/// pointer operands are inferred for loads, stores, and masked/VP memory
+/// intrinsics. For a source memory intrinsic, the source or destination pointer
+/// is selected to match the new read or write. The caller must establish range
+/// containment when the intrinsic's length is unknown. Pointer and value facts
+/// are retained only when valid for the destination; alias tags are adjusted
+/// for its range. Masks and execution conditions are not inspected.
 LLVM_ABI void
 copyMetadataForMemoryAccess(Instruction &Dest, const Instruction &Source,
-                            std::optional<int64_t> ByteOffset = 0);
+                            std::optional<int64_t> DestByteOffset = 0);
 
 /// Copy metadata valid after splitting an instruction into smaller elements.
-/// ByteOffset describes the selected memory fragment, if applicable.
+/// DestByteOffset describes the selected memory fragment, if applicable.
 LLVM_ABI void copyMetadataForScalarizedInstruction(
     Instruction &Dest, const Instruction &Source,
-    std::optional<int64_t> ByteOffset = std::nullopt);
+    std::optional<int64_t> DestByteOffset = std::nullopt);
 
 /// Combine the metadata of two instructions so that K can replace J. This
 /// specifically handles the case of CSE-like transformations. Some
diff --git a/llvm/lib/Analysis/MetadataUtils.cpp b/llvm/lib/Analysis/MetadataUtils.cpp
index 936469fb58a149..646982f4118fe8 100644
--- a/llvm/lib/Analysis/MetadataUtils.cpp
+++ b/llvm/lib/Analysis/MetadataUtils.cpp
@@ -32,8 +32,16 @@ struct MemoryAccessInfo {
   unsigned PointerOperand;
   bool IsRead;
 };
+} // namespace
 
-std::optional<MemoryAccessInfo>
+/// Get information about a memory instruction required to propagate metadata.
+/// Returns std::nullopt if the instruction is not supported and we will not
+/// propagate any metadata.
+///
+/// If IsRead is set and this is a instruction that reads and writes, mark it
+/// as reading or writing. This is used to make the source instruction in
+/// metadata propagation correspond with the dest.
+static std::optional<MemoryAccessInfo>
 getMemoryAccessInfo(const Instruction &I,
                     std::optional<bool> IsRead = std::nullopt) {
   if (const auto *LI = dyn_cast<LoadInst>(&I))
@@ -80,14 +88,16 @@ getMemoryAccessInfo(const Instruction &I,
   }
 }
 
-const Value *getMemoryPointer(const Instruction &I, unsigned Index) {
+/// Get the pointer operand of `I` at operand index `Index`.
+static const Value *getMemoryPointer(const Instruction &I, unsigned Index) {
   if (const auto *CB = dyn_cast<CallBase>(&I))
     return CB->getArgOperand(Index);
   return I.getOperand(Index);
 }
+
 // invariant.group ignores bitcasts and zero-offset GEPs, not address-space
 // casts.
-const Value *getInvariantGroupPointer(const Value *V) {
+static const Value *getInvariantGroupPointer(const Value *V) {
   while (true) {
     if (const auto *Cast = dyn_cast<BitCastOperator>(V))
       V = Cast->getOperand(0);
@@ -99,32 +109,34 @@ const Value *getInvariantGroupPointer(const Value *V) {
   }
 }
 
-void copyNonnullMetadata(const LoadInst &OldLI, MDNode *N, LoadInst &NewLI) {
+static void copyNonnullMetadata(const LoadInst &OldLI, MDNode *N,
+                                LoadInst &NewLI) {
+  auto *OldTy = cast<PointerType>(OldLI.getType());
   auto *NewTy = NewLI.getType();
 
-  // This only directly applies if the new type is also a pointer.
-  if (NewTy->isPointerTy()) {
+  // Preserve the pointer fact when the pointer type is unchanged.
+  if (NewTy == OldTy) {
     NewLI.setMetadata(LLVMContext::MD_nonnull, N);
     return;
   }
 
-  // The only other translation we can do is to integral loads with !range
-  // metadata.
-  if (!NewTy->isIntegerTy())
+  // An integer range must describe the complete pointer representation.
+  if (!NewTy->isIntegerTy() ||
+      NewTy->getIntegerBitWidth() !=
+          OldLI.getDataLayout().getPointerTypeSizeInBits(OldTy))
     return;
 
   MDBuilder MDB(NewLI.getContext());
-  const Value *Ptr = OldLI.getPointerOperand();
   auto *ITy = cast<IntegerType>(NewTy);
-  auto *NullInt = ConstantExpr::getPtrToInt(
-      ConstantPointerNull::get(cast<PointerType>(Ptr->getType())), ITy);
+  auto *NullInt =
+      ConstantExpr::getPtrToInt(ConstantPointerNull::get(OldTy), ITy);
   auto *NonNullInt = ConstantExpr::getAdd(NullInt, ConstantInt::get(ITy, 1));
   NewLI.setMetadata(LLVMContext::MD_range,
                     MDB.createRange(NonNullInt, NullInt));
 }
 
-void copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI, MDNode *N,
-                       LoadInst &NewLI) {
+static void copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI,
+                              MDNode *N, LoadInst &NewLI) {
   auto *NewTy = NewLI.getType();
   // Simply copy the metadata if the type did not change.
   if (NewTy == OldLI.getType()) {
@@ -146,42 +158,46 @@ void copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI, MDNode *N,
     NewLI.setMetadata(LLVMContext::MD_nonnull, NN);
   }
 }
-} // namespace
 
 void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
                                        const Instruction &Source,
-                                       std::optional<int64_t> ByteOffset) {
-  auto Dst = getMemoryAccessInfo(Dest);
-  auto Src = getMemoryAccessInfo(Source, Dst ? std::optional<bool>(Dst->IsRead)
-                                             : std::nullopt);
-  assert(Src && Dst && Src->IsRead == Dst->IsRead &&
+                                       std::optional<int64_t> DestByteOffset) {
+  auto DestInfo = getMemoryAccessInfo(Dest);
+  // Untyped memory intrinsics are supported only as sources.
+  assert(DestInfo && DestInfo->Ty &&
+         "expected a typed destination memory access");
+  auto SourceInfo = getMemoryAccessInfo(Source, DestInfo->IsRead);
+  assert(SourceInfo && SourceInfo->IsRead == DestInfo->IsRead &&
          "expected corresponding memory reads or writes");
   const DataLayout &DL = Source.getDataLayout();
+  // Collect the size of the old memory operation if available.
   std::optional<TypeSize> OldSize;
-  if (Src->Ty)
-    OldSize = DL.getTypeStoreSize(Src->Ty);
+  if (SourceInfo->Ty)
+    OldSize = DL.getTypeStoreSize(SourceInfo->Ty);
   else if (auto *Length =
                dyn_cast<ConstantInt>(cast<AnyMemIntrinsic>(Source).getLength()))
+    // If Source is not a typed memory operation it must be an intrinsic.
     if (auto Bytes = Length->getValue().tryZExtValue())
       OldSize = TypeSize::getFixed(*Bytes);
-  TypeSize NewSize = DL.getTypeStoreSize(Dst->Ty);
+  TypeSize NewSize = DL.getTypeStoreSize(DestInfo->Ty);
+  // Is the memory `Dest` accesses a subset of the memory `Source` acceses?
+  // The caller guarentees this is true if we don't know the size of `Source`
+  // or the offset of the `Dest` access in `Source`.
   bool IsSubset = !OldSize || TypeSize::isKnownLE(NewSize, *OldSize);
-  if (ByteOffset)
-    IsSubset =
-        IsSubset && *ByteOffset >= 0 &&
-        (!OldSize || uint64_t(*ByteOffset) <= OldSize->getKnownMinValue() -
-                                                  NewSize.getKnownMinValue());
-  bool IsWholeAccess = IsSubset && ByteOffset == 0 && OldSize == NewSize;
-  bool SameElementType =
-      Src->Ty && Src->Ty->getScalarType() == Dst->Ty->getScalarType();
-  bool SameValueBits =
-      IsWholeAccess && Src->Ty && !Src->Ty->isAggregateType() &&
-      !Dst->Ty->isAggregateType() && DL.typeSizeEqualsStoreSize(Src->Ty) &&
-      DL.typeSizeEqualsStoreSize(Dst->Ty);
-  bool SamePointer =
-      getInvariantGroupPointer(getMemoryPointer(Source, Src->PointerOperand)) ==
-      getInvariantGroupPointer(getMemoryPointer(Dest, Dst->PointerOperand));
-
+  if (DestByteOffset)
+    IsSubset = IsSubset && *DestByteOffset >= 0 &&
+               (!OldSize ||
+                uint64_t(*DestByteOffset) <=
+                    OldSize->getKnownMinValue() - NewSize.getKnownMinValue());
+  bool IsWholeAccess = IsSubset && DestByteOffset == 0 && OldSize == NewSize;
+  bool SameElementType = SourceInfo->Ty && SourceInfo->Ty->getScalarType() ==
+                                               DestInfo->Ty->getScalarType();
+  bool SamePointer = getInvariantGroupPointer(getMemoryPointer(
+                         Source, SourceInfo->PointerOperand)) ==
+                     getInvariantGroupPointer(
+                         getMemoryPointer(Dest, DestInfo->PointerOperand));
+  
+  // Now loop over all `Source` metadata, propagating what we can.
   SmallVector<std::pair<unsigned, MDNode *>, 8> MDs;
   Source.getAllMetadata(MDs);
   for (auto [Kind, MD] : MDs) {
@@ -201,14 +217,19 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
       Keep = IsSubset;
       break;
     case LLVMContext::MD_invariant_load:
-      Keep = IsSubset && Dst->IsRead;
+      Keep = IsSubset && DestInfo->IsRead;
       break;
     case LLVMContext::MD_invariant_group:
       Keep = IsSubset && SamePointer && !isa<CallBase>(Dest);
       break;
     case LLVMContext::MD_noundef:
-      Keep =
-          IsSubset && (SameElementType || SameValueBits) && isa<LoadInst>(Dest);
+      // The value loaded remains defined if its element type is unchanged,
+      // or every source byte consists of defined value bits (i.e. the source
+      // doesn't have padding).
+      Keep = IsSubset && isa<LoadInst>(Dest) &&
+             (SameElementType ||
+              (SourceInfo->Ty && !SourceInfo->Ty->isAggregateType() &&
+               DL.typeSizeEqualsStoreSize(SourceInfo->Ty)));
       break;
     case LLVMContext::MD_fpmath:
     case LLVMContext::MD_nofpclass:
@@ -218,19 +239,23 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
                                           : isa<LoadInst>(Dest));
       break;
     case LLVMContext::MD_nonnull:
-      if (SameValueBits && isa<LoadInst>(Source) && isa<LoadInst>(Dest))
+      if (IsWholeAccess && isa<LoadInst>(Source) && isa<LoadInst>(Dest))
         copyNonnullMetadata(cast<LoadInst>(Source), MD, cast<LoadInst>(Dest));
       break;
     case LLVMContext::MD_range:
-      if (IsSubset && SameElementType && Dst->Ty->isIntOrIntVectorTy())
+      // Ranges apply per integer element. Other type changes require the
+      // helper's explicit conversion from an integer range to nonnull.
+      if (IsSubset && SameElementType && DestInfo->Ty->isIntOrIntVectorTy())
         Keep = true;
-      else if (SameValueBits && isa<LoadInst>(Source) && isa<LoadInst>(Dest))
+      else if (IsWholeAccess && isa<LoadInst>(Source) && isa<LoadInst>(Dest))
         copyRangeMetadata(DL, cast<LoadInst>(Source), MD, cast<LoadInst>(Dest));
       break;
     case LLVMContext::MD_align:
     case LLVMContext::MD_dereferenceable:
     case LLVMContext::MD_dereferenceable_or_null:
-      Keep = SameValueBits && Dst->Ty->isPointerTy() && isa<LoadInst>(Dest);
+      // These describe the loaded pointer value.
+      Keep = IsWholeAccess && SourceInfo->Ty == DestInfo->Ty &&
+             DestInfo->Ty->isPointerTy() && isa<LoadInst>(Dest);
       break;
     case LLVMContext::MD_DIAssignID:
       Keep = IsWholeAccess && isa<StoreInst>(Source) && isa<StoreInst>(Dest);
@@ -241,7 +266,8 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
                             (isa<StoreInst>(Source) && isa<StoreInst>(Dest)));
       break;
     case LLVMContext::MD_captures:
-      Keep = IsWholeAccess && isa<StoreInst>(Dest) && Dst->Ty->isPointerTy();
+      Keep =
+          IsWholeAccess && isa<StoreInst>(Dest) && DestInfo->Ty->isPointerTy();
       break;
     default:
       break;
@@ -250,6 +276,7 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
       Dest.setMetadata(Kind, MD);
   }
 
+  // Adjust aliasing metadata.
   AAMDNodes AA = Source.getAAMetadata();
   if (!IsSubset) {
     AA.TBAA = nullptr;
@@ -257,28 +284,27 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
     AA.Scope = nullptr;
     AA.NoAlias = nullptr;
   } else if (!IsWholeAccess) {
-    if (ByteOffset)
-      AA = AA.adjustForAccess(*ByteOffset, Dst->Ty, DL);
+    if (DestByteOffset)
+      AA = AA.adjustForAccess(*DestByteOffset, DestInfo->Ty, DL);
     else
       AA.TBAAStruct = nullptr;
   } else {
-    // Prefer a single alias tag for a whole replacement, retaining the full
-    // field layout when it cannot be represented that way.
-    AAMDNodes Adjusted = AA.adjustForAccess(0, Dst->Ty, DL);
+    // Prefer !tbaa over !tbaa.struct.
+    AAMDNodes Adjusted = AA.adjustForAccess(0, DestInfo->Ty, DL);
     if (!AA.TBAA && Adjusted.TBAA)
       AA = Adjusted;
   }
   Dest.setAAMetadata(AA);
-  setMemCacheHint(Dest, Dst->PointerOperand,
-                  getMemCacheHint(Source, Src->PointerOperand));
+  setMemCacheHint(Dest, DestInfo->PointerOperand,
+                  getMemCacheHint(Source, SourceInfo->PointerOperand));
 }
 
 void llvm::copyMetadataForScalarizedInstruction(
     Instruction &Dest, const Instruction &Source,
-    std::optional<int64_t> ByteOffset) {
-  if (auto Dst = getMemoryAccessInfo(Dest))
-    if (getMemoryAccessInfo(Source, Dst->IsRead)) {
-      copyMetadataForMemoryAccess(Dest, Source, ByteOffset);
+    std::optional<int64_t> DestByteOffset) {
+  if (auto DestInfo = getMemoryAccessInfo(Dest))
+    if (getMemoryAccessInfo(Source, DestInfo->IsRead)) {
+      copyMetadataForMemoryAccess(Dest, Source, DestByteOffset);
       return;
     }
   if (Dest.getType()->getScalarType() == Source.getType()->getScalarType() &&
diff --git a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
index d8d07d7b09f83c..431037b3a0cc33 100644
--- a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
+++ b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
@@ -524,7 +524,7 @@ static void scalarizeMaskedGather(const DataLayout &DL,
       Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
       LoadInst *Load =
           Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx));
-      copyMetadataForMemoryAccess(*Load, *CI, /*ByteOffset=*/0);
+      copyMetadataForMemoryAccess(*Load, *CI, /*DestByteOffset=*/0);
       VResult =
           Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx));
     }
@@ -584,7 +584,7 @@ static void scalarizeMaskedGather(const DataLayout &DL,
     Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
     LoadInst *Load =
         Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx));
-    copyMetadataForMemoryAccess(*Load, *CI, /*ByteOffset=*/0);
+    copyMetadataForMemoryAccess(*Load, *CI, /*DestByteOffset=*/0);
     Value *NewVResult =
         Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx));
 
@@ -663,7 +663,7 @@ static void scalarizeMaskedScatter(const DataLayout &DL,
           Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));
       Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
       StoreInst *Store = Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
-      copyMetadataForMemoryAccess(*Store, *CI, /*ByteOffset=*/0);
+      copyMetadataForMemoryAccess(*Store, *CI, /*DestByteOffset=*/0);
     }
     CI->eraseFromParent();
     return;
@@ -718,7 +718,7 @@ static void scalarizeMaskedScatter(const DataLayout &DL,
     Value *OneElt = Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));
     Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
     StoreInst *Store = Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
-    copyMetadataForMemoryAccess(*Store, *CI, /*ByteOffset=*/0);
+    copyMetadataForMemoryAccess(*Store, *CI, /*DestByteOffset=*/0);
 
     // Create "else" block, fill it in the next iteration
     BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
@@ -1043,11 +1043,11 @@ static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI,
         continue;
       Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
       LoadInst *Load = Builder.CreateLoad(EltTy, Ptr, "Load" + Twine(Idx));
-      copyMetadataForMemoryAccess(*Load, *CI, /*ByteOffset=*/0);
+      copyMetadataForMemoryAccess(*Load, *CI, /*DestByteOffset=*/0);
       Value *Update =
           CreateHistogramUpdateValue(cast<IntrinsicInst>(CI), Load, Inc);
       StoreInst *Store = Builder.CreateStore(Update, Ptr);
-      copyMetadataForMemoryAccess(*Store, *CI, /*ByteOffset=*/0);
+      copyMetadataForMemoryAccess(*Store, *CI, /*DestByteOffset=*/0);
     }
     CI->eraseFromParent();
     return;
@@ -1071,11 +1071,11 @@ static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI,
     Builder.SetInsertPoint(CondBlock->getTerminator());
     Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
     LoadInst *Load = Builder.CreateLoad(EltTy, Ptr, "Load" + Twine(Idx));
-    copyMetadataForMemoryAccess(*Load, *CI, /*ByteOffset=*/0);
+    copyMetadataForMemoryAccess(*Load, *CI, /*DestByteOffset=*/0);
     Value *UpdateOp =
         CreateHistogramUpdateValue(cast<IntrinsicInst>(CI), Load, Inc);
     StoreInst *Store = Builder.CreateStore(UpdateOp, Ptr);
-    copyMetadataForMemoryAccess(*Store, *CI, /*ByteOffset=*/0);
+    copyMetadataForMemoryAccess(*Store, *CI, /*DestByteOffset=*/0);
 
     // Create "else" block, fill it in the next iteration
     BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
diff --git a/llvm/test/Transforms/SROA/memory-metadata.ll b/llvm/test/Transforms/SROA/memory-metadata.ll
index f4d272e8465ef3..88be04f0d01fbd 100644
--- a/llvm/test/Transforms/SROA/memory-metadata.ll
+++ b/llvm/test/Transforms/SROA/memory-metadata.ll
@@ -146,9 +146,9 @@ define i32 @presplit_load(ptr %src) {
 ; CHECK-SAME: ptr [[SRC:%.*]]) {
 ; CHECK-NEXT:    [[A_SROA_0:%.*]] = alloca i32, align 8
 ; CHECK-NEXT:    [[A_SROA_2:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    [[V1:%.*]] = load i32, ptr [[SRC]], align 8, !invariant.load [[META10]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !invariant.group [[META10]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META19]]
+; CHECK-NEXT:    [[V1:%.*]] = load i32, ptr [[SRC]], align 8, !invariant.load [[META10]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !invariant.group [[META10]], !llvm.access.group [[META8]], !noundef [[META10]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META19]]
 ; CHECK-NEXT:    [[SRC_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 4
-; CHECK-NEXT:    [[V2:%.*]] = load i32, ptr [[SRC_SROA_IDX]], align 4, !invariant.load [[META10]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META19]]
+; CHECK-NEXT:    [[V2:%.*]] = load i32, ptr [[SRC_SROA_IDX]], align 4, !invariant.load [[META10]], !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !noundef [[META10]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META19]]
 ; CHECK-NEXT:    store i32 [[V1]], ptr [[A_SROA_0]], align 8, !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
 ; CHECK-NEXT:    store i32 [[V2]], ptr [[A_SROA_2]], align 4, !alias.scope [[META4]], !noalias [[META4]], !nontemporal [[META7]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]], !annotation [[META9]], !nosanitize [[META10]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META15]]
 ; CHECK-NEXT:    [[A_SROA_0_0_A_SROA_0_0_X:%.*]] = load volatile i32, ptr [[A_SROA_0]], align 8
diff --git a/llvm/unittests/Analysis/MetadataUtilsTest.cpp b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
index ea752404ddcfb3..7066da1a69177a 100644
--- a/llvm/unittests/Analysis/MetadataUtilsTest.cpp
+++ b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
@@ -91,6 +91,21 @@ TEST_F(MetadataUtilsTest, InvariantGroupRequiresPointerIdentity) {
   EXPECT_FALSE(Different->hasMetadata(LLVMContext::MD_invariant_group));
 }
 
+TEST_F(MetadataUtilsTest, IntegerFragmentPreservesDefinednessButNotRange) {
+  auto *Old = B.CreateLoad(B.getInt64Ty(), P);
+  MDBuilder MDB(C);
+  Old->setMetadata(LLVMContext::MD_noundef, empty());
+  Old->setMetadata(LLVMContext::MD_range,
+                   MDB.createRange(B.getInt64(0), B.getInt64(100)));
+  for (int64_t Offset : {0, 4}) {
+    auto *Ptr = B.CreateInBoundsGEP(B.getInt8Ty(), P, B.getInt64(Offset));
+    auto *New = B.CreateLoad(B.getInt32Ty(), Ptr);
+    copyMetadataForMemoryAccess(*New, *Old, Offset);
+    EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_noundef));
+    EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+  }
+}
+
 TEST_F(MetadataUtilsTest, WideningDropsFactsAboutOriginalBytes) {
   auto *Old = B.CreateLoad(B.getInt32Ty(), P);
   auto *New = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), P);
@@ -312,6 +327,57 @@ TEST_F(MetadataUtilsTest, DefinedAggregateDoesNotDefinePadding) {
   EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_noundef));
 }
 
+TEST_F(MetadataUtilsTest, DefinedIntegerDoesNotDefineSourcePadding) {
+  auto *Old = B.CreateLoad(B.getInt1Ty(), P);
+  auto *New = B.CreateLoad(B.getInt8Ty(), P);
+  Old->setMetadata(LLVMContext::MD_noundef, empty());
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_noundef));
+}
+
+TEST_F(MetadataUtilsTest, DestinationPaddingDoesNotInvalidateDefinedness) {
+  auto *Old = B.CreateLoad(B.getInt8Ty(), P);
+  auto *New = B.CreateLoad(B.getInt1Ty(), P);
+  Old->setMetadata(LLVMContext::MD_noundef, empty());
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_noundef));
+}
+
+TEST_F(MetadataUtilsTest, NonnullDoesNotConstrainTruncatedPointerBits) {
+  auto *Old = B.CreateLoad(B.getPtrTy(), P);
+  auto *New = B.CreateLoad(B.getIntNTy(63), P);
+  Old->setMetadata(LLVMContext::MD_nonnull, empty());
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+}
+
+TEST_F(MetadataUtilsTest, IntegerRangeBecomesPointerNonnull) {
+  auto *Old = B.CreateLoad(B.getInt64Ty(), P);
+  auto *New = B.CreateLoad(B.getPtrTy(), P);
+  MDBuilder MDB(C);
+  Old->setMetadata(LLVMContext::MD_range,
+                   MDB.createRange(B.getInt64(1), B.getInt64(100)));
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_nonnull));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+}
+
+TEST_F(MetadataUtilsTest, PointerFactsRequireSamePointerType) {
+  auto *Old = B.CreateLoad(B.getPtrTy(), P);
+  auto *New = B.CreateLoad(B.getPtrTy(1), P);
+  Old->setMetadata(LLVMContext::MD_nonnull, empty());
+  Old->setMetadata(LLVMContext::MD_align, MDNode::get(C, {number(16)}));
+  Old->setMetadata(LLVMContext::MD_dereferenceable,
+                   MDNode::get(C, {number(32)}));
+  Old->setMetadata(LLVMContext::MD_dereferenceable_or_null,
+                   MDNode::get(C, {number(32)}));
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_nonnull));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_align));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_dereferenceable));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_dereferenceable_or_null));
+}
+
 TEST_F(MetadataUtilsTest, PointerFactsRequirePointerResults) {
   auto *Old = B.CreateLoad(B.getPtrTy(), P);
   auto *Pointer = B.CreateLoad(B.getPtrTy(), P);

>From 4d78157968d004891a67d06d2a94fe55d331957b Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 17 Nov 2026 20:45:03 +0000
Subject: [PATCH 04/14] additional fixes

---
 llvm/lib/Analysis/MetadataUtils.cpp           | 47 ++++++++----
 llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp  |  5 +-
 llvm/lib/Transforms/Scalar/SROA.cpp           | 16 +++--
 .../Transforms/Vectorize/VectorCombine.cpp    |  3 +
 .../Transforms/InstCombine/memory-metadata.ll | 60 ++++++++++++++++
 .../VectorCombine/memory-metadata.ll          | 47 ++++++++++--
 llvm/unittests/Analysis/MetadataUtilsTest.cpp | 72 +++++++++++++++++++
 7 files changed, 227 insertions(+), 23 deletions(-)

diff --git a/llvm/lib/Analysis/MetadataUtils.cpp b/llvm/lib/Analysis/MetadataUtils.cpp
index 646982f4118fe8..f93bdad0fc0183 100644
--- a/llvm/lib/Analysis/MetadataUtils.cpp
+++ b/llvm/lib/Analysis/MetadataUtils.cpp
@@ -190,13 +190,25 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
                 uint64_t(*DestByteOffset) <=
                     OldSize->getKnownMinValue() - NewSize.getKnownMinValue());
   bool IsWholeAccess = IsSubset && DestByteOffset == 0 && OldSize == NewSize;
-  bool SameElementType = SourceInfo->Ty && SourceInfo->Ty->getScalarType() ==
-                                               DestInfo->Ty->getScalarType();
+  Type *SourceValueTy = SourceInfo->Ty;
+  // Allow unpacking a single-element aggregates.
+  if (IsWholeAccess) {
+    if (auto *ST = dyn_cast_or_null<StructType>(SourceValueTy);
+        ST && ST->getNumElements() == 1 &&
+        ST->getElementType(0) == DestInfo->Ty)
+      SourceValueTy = DestInfo->Ty;
+    else if (auto *AT = dyn_cast_or_null<ArrayType>(SourceValueTy);
+             AT && AT->getNumElements() == 1 &&
+             AT->getElementType() == DestInfo->Ty)
+      SourceValueTy = DestInfo->Ty;
+  }
+  bool SameElementType = SourceValueTy && SourceValueTy->getScalarType() ==
+                                              DestInfo->Ty->getScalarType();
   bool SamePointer = getInvariantGroupPointer(getMemoryPointer(
                          Source, SourceInfo->PointerOperand)) ==
                      getInvariantGroupPointer(
                          getMemoryPointer(Dest, DestInfo->PointerOperand));
-  
+
   // Now loop over all `Source` metadata, propagating what we can.
   SmallVector<std::pair<unsigned, MDNode *>, 8> MDs;
   Source.getAllMetadata(MDs);
@@ -207,6 +219,7 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
     case LLVMContext::MD_nontemporal:
     case LLVMContext::MD_annotation:
     case LLVMContext::MD_nosanitize:
+    case LLVMContext::MD_pcsections:
       Keep = true;
       break;
     case LLVMContext::MD_mmra:
@@ -220,6 +233,7 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
       Keep = IsSubset && DestInfo->IsRead;
       break;
     case LLVMContext::MD_invariant_group:
+      // TODO: Requiring `SamePointer` is too conservative.
       Keep = IsSubset && SamePointer && !isa<CallBase>(Dest);
       break;
     case LLVMContext::MD_noundef:
@@ -228,15 +242,15 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
       // doesn't have padding).
       Keep = IsSubset && isa<LoadInst>(Dest) &&
              (SameElementType ||
-              (SourceInfo->Ty && !SourceInfo->Ty->isAggregateType() &&
-               DL.typeSizeEqualsStoreSize(SourceInfo->Ty)));
+              (SourceValueTy && !SourceValueTy->isAggregateType() &&
+               DL.typeSizeEqualsStoreSize(SourceValueTy)));
       break;
     case LLVMContext::MD_fpmath:
     case LLVMContext::MD_nofpclass:
-      Keep =
-          IsSubset && SameElementType &&
-          (Kind == LLVMContext::MD_fpmath ? Dest.getType()->isFPOrFPVectorTy()
-                                          : isa<LoadInst>(Dest));
+      Keep = IsSubset && SameElementType &&
+             (Kind == LLVMContext::MD_fpmath
+                  ? FPMathOperator::isSupportedFloatingPointType(Dest.getType())
+                  : isa<LoadInst>(Dest));
       break;
     case LLVMContext::MD_nonnull:
       if (IsWholeAccess && isa<LoadInst>(Source) && isa<LoadInst>(Dest))
@@ -257,9 +271,14 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
       Keep = IsWholeAccess && SourceInfo->Ty == DestInfo->Ty &&
              DestInfo->Ty->isPointerTy() && isa<LoadInst>(Dest);
       break;
-    case LLVMContext::MD_DIAssignID:
-      Keep = IsWholeAccess && isa<StoreInst>(Source) && isa<StoreInst>(Dest);
+    case LLVMContext::MD_DIAssignID: {
+      const auto *OldSI = dyn_cast<StoreInst>(&Source);
+      const auto *NewSI = dyn_cast<StoreInst>(&Dest);
+      // A new address requires updating the linked dbg.assign records.
+      Keep = IsWholeAccess && OldSI && NewSI &&
+             OldSI->getPointerOperand() == NewSI->getPointerOperand();
       break;
+    }
     case LLVMContext::MD_prof:
       Keep =
           IsWholeAccess && ((isa<LoadInst>(Source) && isa<LoadInst>(Dest)) ||
@@ -286,8 +305,12 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
   } else if (!IsWholeAccess) {
     if (DestByteOffset)
       AA = AA.adjustForAccess(*DestByteOffset, DestInfo->Ty, DL);
-    else
+    else {
+      // A common tag for the whole source also describes any selected part.
+      if (!AA.TBAA && SourceInfo->Ty)
+        AA.TBAA = AA.adjustForAccess(0, SourceInfo->Ty, DL).TBAA;
       AA.TBAAStruct = nullptr;
+    }
   } else {
     // Prefer !tbaa over !tbaa.struct.
     AAMDNodes Adjusted = AA.adjustForAccess(0, DestInfo->Ty, DL);
diff --git a/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp b/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp
index cfabc3f1595047..4b91390c2d8b0f 100644
--- a/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp
+++ b/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp
@@ -806,7 +806,7 @@ AAMDNodes AAMDNodes::adjustForAccess(unsigned AccessSize) {
   MDNode *M = New.TBAAStruct;
 
   // The access may cover several !tbaa.struct fields (e.g. a {int, int} copy
-  // widened to an i64 load/store). If those fields share a single tag and tile
+  // widened to an i64 load/store). If those fields share a single tag and cover
   // [0, AccessSize) with no gaps, that tag still describes the whole access.
   New.TBAAStruct = nullptr;
   if (New.TBAA || !M)
@@ -825,7 +825,8 @@ AAMDNodes AAMDNodes::adjustForAccess(unsigned AccessSize) {
         (CommonTag && FieldTag != CommonTag))
       break;
     CommonTag = FieldTag;
-    Offset += *FieldSize;
+    // The final field may extend beyond the access.
+    Offset += std::min(*FieldSize, uint64_t(AccessSize) - Offset);
   }
   if (Offset == AccessSize)
     New.TBAA = CommonTag;
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 51ee1f6fa95087..d4caaaa22da546 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -408,10 +408,16 @@ static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit,
     BaseFragments[getAggregateVariable(DVR)] =
         DVR->getExpression()->getFragmentInfo();
 
-  // The new inst needs a DIAssignID unique metadata tag (if OldInst has
-  // one). It shouldn't already have one: assert this assumption.
-  assert(!Inst->getMetadata(LLVMContext::MD_DIAssignID));
-  DIAssignID *NewID = nullptr;
+  // Reuse an assignment ID copied for an unsplit store at the same address.
+  auto *NewID =
+      cast_or_null<DIAssignID>(Inst->getMetadata(LLVMContext::MD_DIAssignID));
+  if (NewID) {
+    assert(!IsSplit &&
+           NewID == OldInst->getMetadata(LLVMContext::MD_DIAssignID) &&
+           "unexpected assignment ID on replacement instruction");
+    // Deleting OldInst must not delete the records now linked to Inst.
+    OldInst->setMetadata(LLVMContext::MD_DIAssignID, nullptr);
+  }
   auto &Ctx = Inst->getContext();
   DIBuilder DIB(*OldInst->getModule(), /*AllowUnresolved*/ false);
   assert(OldAlloca->isStaticAlloca());
@@ -478,7 +484,7 @@ static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit,
     } else {
       // The store is not split, simply steal the existing dbg_assign.
       NewAssign = DbgAssign;
-      NewAssign->setAssignId(NewID); // FIXME: Can we avoid generating new IDs?
+      NewAssign->setAssignId(NewID);
       NewAssign->setAddress(Dest);
       if (Value)
         NewAssign->replaceVariableLocationOp(0u, Value);
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index bb2d24138d0965..01e3988562c8ff 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -2149,6 +2149,9 @@ bool VectorCombine::foldInsertElementsToStores(Instruction &I) {
       Offset =
           ConstIdx->getZExtValue() * DL->getTypeStoreSize(InsertVal->getType());
     copyMetadataForMemoryAccess(*LastStore, *SI, Offset);
+    // Unchanged lanes remain in memory, so the original assignment still
+    // describes the whole vector after the replacement stores.
+    LastStore->copyMetadata(*SI, LLVMContext::MD_DIAssignID);
     Align ScalarOpAlignment = computeAlignmentAfterScalarization(
         std::max(SI->getAlign(), Load->getAlign()), InsertVal->getType(), Idx,
         *DL);
diff --git a/llvm/test/Transforms/InstCombine/memory-metadata.ll b/llvm/test/Transforms/InstCombine/memory-metadata.ll
index b87598fb0098ed..845591936d0b61 100644
--- a/llvm/test/Transforms/InstCombine/memory-metadata.ll
+++ b/llvm/test/Transforms/InstCombine/memory-metadata.ll
@@ -136,6 +136,60 @@ define void @splat_scatter(ptr %p, i32 %v) {
   ret void
 }
 
+define i32 @single_struct_noundef(ptr %p) {
+; CHECK-LABEL: define i32 @single_struct_noundef(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[V_UNPACK:%.*]] = load i32, ptr [[P]], align 4, !noundef [[META4]]
+; CHECK-NEXT:    ret i32 [[V_UNPACK]]
+;
+  %v = load {i32}, ptr %p, !noundef !0
+  %x = extractvalue {i32} %v, 0
+  ret i32 %x
+}
+
+define i32 @single_array_noundef(ptr %p) {
+; CHECK-LABEL: define i32 @single_array_noundef(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[V_UNPACK:%.*]] = load i32, ptr [[P]], align 4, !noundef [[META4]]
+; CHECK-NEXT:    ret i32 [[V_UNPACK]]
+;
+  %v = load [1 x i32], ptr %p, !noundef !0
+  %x = extractvalue [1 x i32] %v, 0
+  ret i32 %x
+}
+
+define float @single_struct_fpmath(ptr %p) {
+; CHECK-LABEL: define float @single_struct_fpmath(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[V_UNPACK:%.*]] = load float, ptr [[P]], align 4, !fpmath [[META22:![0-9]+]], !noundef [[META4]], !nofpclass [[META23:![0-9]+]]
+; CHECK-NEXT:    ret float [[V_UNPACK]]
+;
+  %v = load {float}, ptr %p, !fpmath !14, !nofpclass !15, !noundef !0
+  %x = extractvalue {float} %v, 0
+  ret float %x
+}
+
+define float @single_array_fpmath(ptr %p) {
+; CHECK-LABEL: define float @single_array_fpmath(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[V_UNPACK:%.*]] = load float, ptr [[P]], align 4, !fpmath [[META22]], !noundef [[META4]], !nofpclass [[META23]]
+; CHECK-NEXT:    ret float [[V_UNPACK]]
+;
+  %v = load [1 x float], ptr %p, !fpmath !14, !nofpclass !15, !noundef !0
+  %x = extractvalue [1 x float] %v, 0
+  ret float %x
+}
+
+define <2 x i32> @unmask_pcsections(ptr %p) {
+; CHECK-LABEL: define <2 x i32> @unmask_pcsections(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[UNMASKEDLOAD:%.*]] = load <2 x i32>, ptr [[P]], align 1, !pcsections [[META24:![0-9]+]]
+; CHECK-NEXT:    ret <2 x i32> [[UNMASKEDLOAD]]
+;
+  %v = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr %p, <2 x i1> <i1 true, i1 true>, <2 x i32> poison), !pcsections !26
+  ret <2 x i32> %v
+}
+
 !0 = !{}
 !1 = !{i32 1}
 !2 = !{!"preserved memory metadata"}
@@ -162,6 +216,8 @@ define void @splat_scatter(ptr %p, i32 %v) {
 !23 = !{!"nvvm.l1_eviction", !"first"}
 !24 = !{!"branch_weights", i32 10}
 !25 = !{i64 8}
+!26 = !{!"test_section", !27}
+!27 = !{i32 42}
 ;.
 ; CHECK: [[INT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
 ; CHECK: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
@@ -185,4 +241,8 @@ define void @splat_scatter(ptr %p, i32 %v) {
 ; CHECK: [[META19]] = !{i32 0, [[META20:![0-9]+]]}
 ; CHECK: [[META20]] = !{!"nvvm.l1_eviction", !"first"}
 ; CHECK: [[RNG21]] = !{i32 0, i32 100}
+; CHECK: [[META22]] = !{float 2.500000e+00}
+; CHECK: [[META23]] = !{i32 3}
+; CHECK: [[META24]] = !{!"test_section", [[META25:![0-9]+]]}
+; CHECK: [[META25]] = !{i32 42}
 ;.
diff --git a/llvm/test/Transforms/VectorCombine/memory-metadata.ll b/llvm/test/Transforms/VectorCombine/memory-metadata.ll
index cbf704e8780f45..ef1ddffd3f6688 100644
--- a/llvm/test/Transforms/VectorCombine/memory-metadata.ll
+++ b/llvm/test/Transforms/VectorCombine/memory-metadata.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -S -passes=vector-combine -data-layout=e-p:64:64 < %s | FileCheck %s
-; RUN: opt -S -passes=vector-combine -data-layout=E-p:64:64 < %s | FileCheck %s
+; RUN: opt -S -passes=vector-combine -data-layout=e-p:64:64 < %s | FileCheck %s --implicit-check-not=!test.unknown
+; RUN: opt -S -passes=vector-combine -data-layout=E-p:64:64 < %s | FileCheck %s --implicit-check-not=!test.unknown
 
 define i32 @extract_zero(ptr %p) {
 ; CHECK-LABEL: define i32 @extract_zero(
@@ -80,7 +80,7 @@ define <4 x float> @shrink_float(ptr %p) {
 define void @store_zero(ptr %p, i8 %x, i32 %index) {
 ; CHECK-LABEL: define void @store_zero(
 ; CHECK-SAME: ptr [[P:%.*]], i8 [[X:%.*]], i32 [[INDEX:%.*]]) {
-; CHECK-NEXT:    store i8 [[X]], ptr [[P]], align 16, !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !invariant.group [[META4]], !llvm.access.group [[ACC_GRP9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META17:![0-9]+]]
+; CHECK-NEXT:    store i8 [[X]], ptr [[P]], align 16, !tbaa [[CHAR_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !invariant.group [[META4]], !llvm.access.group [[ACC_GRP9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META17:![0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
   %v = load <16 x i8>, ptr %p, align 16
@@ -109,7 +109,7 @@ define void @store_dynamic(ptr %p, i8 %x, i32 %index) {
 ; CHECK-NEXT:    [[IDX:%.*]] = and i32 [[INDEX_FROZEN]], 15
 ; CHECK-NEXT:    [[IDX_GEPIDX:%.*]] = zext i32 [[IDX]] to i64
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 [[IDX_GEPIDX]]
-; CHECK-NEXT:    store i8 [[X]], ptr [[TMP1]], align 1, !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.access.group [[ACC_GRP9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META17]]
+; CHECK-NEXT:    store i8 [[X]], ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.access.group [[ACC_GRP9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META17]]
 ; CHECK-NEXT:    ret void
 ;
   %idx = and i32 %index, 15
@@ -146,6 +146,42 @@ define void @store_struct_field(ptr %p, i32 %x) {
   ret void
 }
 
+define void @scalar_store(ptr %p, i8 %x) {
+; CHECK-LABEL: define void @scalar_store(
+; CHECK-SAME: ptr [[P:%.*]], i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 3
+; CHECK-NEXT:    store i8 [[X]], ptr [[TMP1]], align 1, !pcsections [[META20:![0-9]+]]
+; CHECK-NEXT:    ret void
+;
+  %v = load <16 x i8>, ptr %p, align 16
+  %i = insertelement <16 x i8> %v, i8 %x, i32 3
+  store <16 x i8> %i, ptr %p, align 16, !pcsections !22, !test.unknown !23
+  ret void
+}
+
+define i32 @whole_bitcast(ptr %p) {
+; CHECK-LABEL: define i32 @whole_bitcast(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = load i32, ptr [[P]], align 4, !pcsections [[META20]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %v = load <4 x i8>, ptr %p, align 4, !pcsections !22, !test.unknown !23
+  %r = bitcast <4 x i8> %v to i32
+  ret i32 %r
+}
+
+define <4 x i32> @shrink_load(ptr %p) {
+; CHECK-LABEL: define <4 x i32> @shrink_load(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[P]], align 32, !pcsections [[META20]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %v = load <8 x i32>, ptr %p, align 32, !pcsections !22, !test.unknown !23
+  %r = shufflevector <8 x i32> %v, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  ret <4 x i32> %r
+}
+
 !0 = !{i32 1}
 !1 = !{i32 0, !2}
 !2 = !{!"nvvm.l1_eviction", !"last"}
@@ -169,6 +205,8 @@ define void @store_struct_field(ptr %p, i32 %x) {
 !19 = !{!"int", !16, i64 0}
 !20 = !{!19, !19, i64 0}
 !21 = !{i64 0, i64 4, !15, i64 4, i64 4, !20, i64 8, i64 4, !15, i64 12, i64 4, !15}
+!22 = !{!"test_section"}
+!23 = !{!"custom annotation"}
 ;.
 ; CHECK: [[CHAR_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
 ; CHECK: [[META1]] = !{!"omnipotent char", [[META2:![0-9]+]], i64 0}
@@ -190,4 +228,5 @@ define void @store_struct_field(ptr %p, i32 %x) {
 ; CHECK: [[META17]] = !{i32 1, [[META14]]}
 ; CHECK: [[INT_TBAA18]] = !{[[META19:![0-9]+]], [[META19]], i64 0}
 ; CHECK: [[META19]] = !{!"int", [[META1]], i64 0}
+; CHECK: [[META20]] = !{!"test_section"}
 ;.
diff --git a/llvm/unittests/Analysis/MetadataUtilsTest.cpp b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
index 7066da1a69177a..b8e0a51fb608ef 100644
--- a/llvm/unittests/Analysis/MetadataUtilsTest.cpp
+++ b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
@@ -317,6 +317,66 @@ TEST_F(MetadataUtilsTest, AddressSpaceCastChangesInvariantGroupIdentity) {
   EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_invariant_group));
 }
 
+TEST_F(MetadataUtilsTest, SingleElementAggregatePreservesDefinedness) {
+  Type *Types[] = {StructType::get(C, {B.getInt32Ty()}, /*isPacked=*/false),
+                   ArrayType::get(B.getInt32Ty(), 1)};
+  for (Type *Ty : Types) {
+    auto *Old = B.CreateLoad(Ty, P);
+    auto *New = B.CreateLoad(B.getInt32Ty(), P);
+    Old->setMetadata(LLVMContext::MD_noundef, empty());
+    copyMetadataForMemoryAccess(*New, *Old);
+    EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_noundef));
+  }
+}
+
+TEST_F(MetadataUtilsTest, SingleElementAggregatePreservesFloatingPointFacts) {
+  auto *FPMath = MDNode::get(
+      C, {ConstantAsMetadata::get(ConstantFP::get(B.getFloatTy(), 2.5))});
+  auto *NoFPClass = MDNode::get(C, {ConstantAsMetadata::get(B.getInt32(3))});
+  Type *Types[] = {StructType::get(C, {B.getFloatTy()}, /*isPacked=*/false),
+                   ArrayType::get(B.getFloatTy(), 1)};
+  for (Type *Ty : Types) {
+    auto *Old = B.CreateLoad(Ty, P);
+    auto *New = B.CreateLoad(B.getFloatTy(), P);
+    Old->setMetadata(LLVMContext::MD_noundef, empty());
+    Old->setMetadata(LLVMContext::MD_fpmath, FPMath);
+    Old->setMetadata(LLVMContext::MD_nofpclass, NoFPClass);
+    copyMetadataForMemoryAccess(*New, *Old);
+    EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_noundef));
+    EXPECT_EQ(New->getMetadata(LLVMContext::MD_fpmath), FPMath);
+    EXPECT_EQ(New->getMetadata(LLVMContext::MD_nofpclass), NoFPClass);
+  }
+}
+
+TEST_F(MetadataUtilsTest, ShortAccessSelectsCoveringFieldTag) {
+  auto *Old = B.CreateLoad(B.getInt64Ty(), P);
+  auto *New = B.CreateLoad(B.getInt32Ty(), P);
+  MDBuilder MDB(C);
+  auto *Int = MDB.createTBAAScalarTypeNode("int", MDB.createTBAARoot("root"));
+  auto *Tag = MDB.createTBAAStructTagNode(Int, Int, 0);
+  Old->setMetadata(LLVMContext::MD_tbaa_struct,
+                   MDNode::get(C, {number(0), number(8), Tag}));
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_EQ(New->getMetadata(LLVMContext::MD_tbaa), Tag);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_tbaa_struct));
+}
+
+TEST_F(MetadataUtilsTest, UnknownOffsetSelectsCommonFieldTag) {
+  auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 2), P);
+  auto *Index = B.CreateAnd(B.CreateLoad(B.getInt64Ty(), Q), B.getInt64(1));
+  auto *Ptr = B.CreateInBoundsGEP(B.getInt32Ty(), P, Index);
+  auto *New = B.CreateLoad(B.getInt32Ty(), Ptr);
+  MDBuilder MDB(C);
+  auto *Int = MDB.createTBAAScalarTypeNode("int", MDB.createTBAARoot("root"));
+  auto *Tag = MDB.createTBAAStructTagNode(Int, Int, 0);
+  Old->setMetadata(
+      LLVMContext::MD_tbaa_struct,
+      MDNode::get(C, {number(0), number(4), Tag, number(4), number(4), Tag}));
+  copyMetadataForMemoryAccess(*New, *Old, std::nullopt);
+  EXPECT_EQ(New->getMetadata(LLVMContext::MD_tbaa), Tag);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_tbaa_struct));
+}
+
 TEST_F(MetadataUtilsTest, DefinedAggregateDoesNotDefinePadding) {
   auto *Ty = StructType::get(C, {B.getInt8Ty(), B.getInt32Ty()});
   ASSERT_EQ(M.getDataLayout().getTypeStoreSize(Ty).getFixedValue(), 8u);
@@ -422,6 +482,18 @@ TEST_F(MetadataUtilsTest, StoreFactsRequireWholeAccessAndPointerValue) {
   EXPECT_FALSE(Part->hasMetadata(LLVMContext::MD_captures));
 }
 
+TEST_F(MetadataUtilsTest, AssignmentIDRequiresUnchangedStoreAddress) {
+  auto *Old = B.CreateStore(B.getInt32(1), P);
+  auto *Same = B.CreateStore(B.getInt32(1), P);
+  auto *Moved = B.CreateStore(B.getInt32(1), Q);
+  auto *AssignID = DIAssignID::getDistinct(C);
+  Old->setMetadata(LLVMContext::MD_DIAssignID, AssignID);
+  copyMetadataForMemoryAccess(*Same, *Old);
+  copyMetadataForMemoryAccess(*Moved, *Old);
+  EXPECT_EQ(Same->getMetadata(LLVMContext::MD_DIAssignID), AssignID);
+  EXPECT_FALSE(Moved->hasMetadata(LLVMContext::MD_DIAssignID));
+}
+
 TEST_F(MetadataUtilsTest, IntrinsicProfilesAndUnknownMetadataAreNotCopied) {
   auto *Ty = FixedVectorType::get(B.getInt32Ty(), 2);
   auto *Mask = B.CreateLoad(FixedVectorType::get(B.getInt1Ty(), 2), Q);

>From ab95e66696c4964ecbd9ca88680e956198f82d75 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 17 Nov 2026 21:31:01 +0000
Subject: [PATCH 05/14] require fixed length mem intrinsics

---
 llvm/include/llvm/Analysis/MetadataUtils.h    | 23 ++++++++-------
 llvm/lib/Analysis/MetadataUtils.cpp           | 28 ++++++++-----------
 llvm/unittests/Analysis/MetadataUtilsTest.cpp | 12 --------
 3 files changed, 23 insertions(+), 40 deletions(-)

diff --git a/llvm/include/llvm/Analysis/MetadataUtils.h b/llvm/include/llvm/Analysis/MetadataUtils.h
index 2d7ce97f1cdd13..25bac66bdc5d24 100644
--- a/llvm/include/llvm/Analysis/MetadataUtils.h
+++ b/llvm/include/llvm/Analysis/MetadataUtils.h
@@ -21,18 +21,17 @@ class Instruction;
 class MDNode;
 class Value;
 
-/// Copy metadata valid for a replacement memory access. The destination must
-/// access the same bytes, a subset, or an expanded range in the same object.
-/// The destination must be a typed memory access; memory-transfer and
-/// memory-set intrinsics are supported only as sources. DestByteOffset is the
-/// destination's offset within the source access. A missing offset requires the
-/// caller to establish that the new access is a subset. Access types and
-/// pointer operands are inferred for loads, stores, and masked/VP memory
-/// intrinsics. For a source memory intrinsic, the source or destination pointer
-/// is selected to match the new read or write. The caller must establish range
-/// containment when the intrinsic's length is unknown. Pointer and value facts
-/// are retained only when valid for the destination; alias tags are adjusted
-/// for its range. Masks and execution conditions are not inspected.
+/// Copy metadata valid for a replacement memory access.
+///
+/// The destination must access the same bytes, a subset, or a wider byte range
+/// in the same object as the source. The destination must be a typed memory
+/// access. Memory-transfer and memory-set intrinsics are supported only as
+/// sources and must have a constant length.
+///
+/// DestByteOffset is the destination's offset within the source access. If the
+/// destination is no larger than the source, a missing offset means the offset
+/// is dynamic, but the caller guarentees it accesses a subset of the source
+/// access.
 LLVM_ABI void
 copyMetadataForMemoryAccess(Instruction &Dest, const Instruction &Source,
                             std::optional<int64_t> DestByteOffset = 0);
diff --git a/llvm/lib/Analysis/MetadataUtils.cpp b/llvm/lib/Analysis/MetadataUtils.cpp
index f93bdad0fc0183..9892612889396b 100644
--- a/llvm/lib/Analysis/MetadataUtils.cpp
+++ b/llvm/lib/Analysis/MetadataUtils.cpp
@@ -170,25 +170,21 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
   assert(SourceInfo && SourceInfo->IsRead == DestInfo->IsRead &&
          "expected corresponding memory reads or writes");
   const DataLayout &DL = Source.getDataLayout();
-  // Collect the size of the old memory operation if available.
-  std::optional<TypeSize> OldSize;
-  if (SourceInfo->Ty)
-    OldSize = DL.getTypeStoreSize(SourceInfo->Ty);
-  else if (auto *Length =
-               dyn_cast<ConstantInt>(cast<AnyMemIntrinsic>(Source).getLength()))
-    // If Source is not a typed memory operation it must be an intrinsic.
-    if (auto Bytes = Length->getValue().tryZExtValue())
-      OldSize = TypeSize::getFixed(*Bytes);
+  TypeSize OldSize =
+      SourceInfo->Ty
+          ? DL.getTypeStoreSize(SourceInfo->Ty)
+          : TypeSize::getFixed(
+                cast<ConstantInt>(cast<AnyMemIntrinsic>(Source).getLength())
+                    ->getZExtValue());
   TypeSize NewSize = DL.getTypeStoreSize(DestInfo->Ty);
-  // Is the memory `Dest` accesses a subset of the memory `Source` acceses?
-  // The caller guarentees this is true if we don't know the size of `Source`
-  // or the offset of the `Dest` access in `Source`.
-  bool IsSubset = !OldSize || TypeSize::isKnownLE(NewSize, *OldSize);
+  // Is the destination access contained in the source access? If the size
+  // check succeeds but the offset is unknown, containment is a caller
+  // guarantee.
+  bool IsSubset = TypeSize::isKnownLE(NewSize, OldSize);
   if (DestByteOffset)
     IsSubset = IsSubset && *DestByteOffset >= 0 &&
-               (!OldSize ||
-                uint64_t(*DestByteOffset) <=
-                    OldSize->getKnownMinValue() - NewSize.getKnownMinValue());
+               uint64_t(*DestByteOffset) <=
+                   OldSize.getKnownMinValue() - NewSize.getKnownMinValue();
   bool IsWholeAccess = IsSubset && DestByteOffset == 0 && OldSize == NewSize;
   Type *SourceValueTy = SourceInfo->Ty;
   // Allow unpacking a single-element aggregates.
diff --git a/llvm/unittests/Analysis/MetadataUtilsTest.cpp b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
index b8e0a51fb608ef..b55922ebd0893c 100644
--- a/llvm/unittests/Analysis/MetadataUtilsTest.cpp
+++ b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
@@ -259,18 +259,6 @@ TEST_F(MetadataUtilsTest, MemoryTransferRemapsBothOperands) {
   EXPECT_FALSE(getMemCacheHint(*Write, 0));
 }
 
-TEST_F(MetadataUtilsTest, RuntimeMemoryTransferLength) {
-  auto *Length = B.CreateLoad(B.getInt64Ty(), Q);
-  B.CreateAssumption(B.CreateICmpUGE(Length, B.getInt64(8)));
-  auto *Copy = B.CreateMemCpy(P, Align(4), Q, Align(4), Length);
-  auto *Ptr = B.CreateInBoundsGEP(B.getInt8Ty(), Q, B.getInt64(4));
-  auto *Read = B.CreateLoad(B.getInt32Ty(), Ptr);
-  auto *Hint = hint();
-  setMemCacheHint(*Copy, 1, Hint);
-  copyMetadataForMemoryAccess(*Read, *Copy, 4);
-  EXPECT_EQ(getMemCacheHint(*Read), Hint);
-}
-
 TEST_F(MetadataUtilsTest, StoreTypeChangePreservesHints) {
   auto *Old = B.CreateStore(B.getInt32(0), P);
   auto *New = B.CreateStore(ConstantFP::get(B.getFloatTy(), 0), P);

>From 90af05a88a26c0dd2ecc363170543be9414112ec Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 17 Nov 2026 21:34:54 +0000
Subject: [PATCH 06/14] get erid of general vp

---
 llvm/lib/Analysis/MetadataUtils.cpp | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/llvm/lib/Analysis/MetadataUtils.cpp b/llvm/lib/Analysis/MetadataUtils.cpp
index 9892612889396b..4c51eda0e9de93 100644
--- a/llvm/lib/Analysis/MetadataUtils.cpp
+++ b/llvm/lib/Analysis/MetadataUtils.cpp
@@ -56,17 +56,6 @@ getMemoryAccessInfo(const Instruction &I,
   const auto *II = dyn_cast<IntrinsicInst>(&I);
   if (!II)
     return std::nullopt;
-  if (const auto *VP = dyn_cast<VPIntrinsic>(II)) {
-    auto Ptr = VPIntrinsic::getMemoryPointerParamPos(VP->getIntrinsicID());
-    if (!Ptr)
-      return std::nullopt;
-    if (auto Data = VPIntrinsic::getMemoryDataParamPos(VP->getIntrinsicID()))
-      return MemoryAccessInfo{VP->getArgOperand(*Data)->getType(), *Ptr, false};
-    Type *Ty = VP->getType();
-    if (VP->getIntrinsicID() == Intrinsic::vp_load_ff)
-      Ty = Ty->getStructElementType(0);
-    return MemoryAccessInfo{Ty, *Ptr, true};
-  }
   switch (II->getIntrinsicID()) {
   case Intrinsic::experimental_vector_histogram_add:
   case Intrinsic::experimental_vector_histogram_uadd_sat:
@@ -78,6 +67,7 @@ getMemoryAccessInfo(const Instruction &I,
   case Intrinsic::masked_load:
   case Intrinsic::masked_gather:
   case Intrinsic::masked_expandload:
+  case Intrinsic::vp_load:
     return MemoryAccessInfo{II->getType(), 0, true};
   case Intrinsic::masked_store:
   case Intrinsic::masked_scatter:

>From 346ad6a17bdd3b173cc7df6a6b35df7f10b23b12 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 17 Nov 2026 21:40:42 +0000
Subject: [PATCH 07/14] comment update

---
 llvm/include/llvm/Analysis/MetadataUtils.h | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/Analysis/MetadataUtils.h b/llvm/include/llvm/Analysis/MetadataUtils.h
index 25bac66bdc5d24..2952bfca8402f5 100644
--- a/llvm/include/llvm/Analysis/MetadataUtils.h
+++ b/llvm/include/llvm/Analysis/MetadataUtils.h
@@ -36,8 +36,10 @@ LLVM_ABI void
 copyMetadataForMemoryAccess(Instruction &Dest, const Instruction &Source,
                             std::optional<int64_t> DestByteOffset = 0);
 
-/// Copy metadata valid after splitting an instruction into smaller elements.
-/// DestByteOffset describes the selected memory fragment, if applicable.
+/// Copy metadata valid after splitting a vector instruction into scalar or
+/// smaller vector parts.
+/// For memory accesses, DestByteOffset is the destination's byte offset within
+/// the source access.
 LLVM_ABI void copyMetadataForScalarizedInstruction(
     Instruction &Dest, const Instruction &Source,
     std::optional<int64_t> DestByteOffset = std::nullopt);

>From fce4e45c363e6de3df227a9987a2c8dc7053b745 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 17 Nov 2026 23:24:27 +0000
Subject: [PATCH 08/14] new test

---
 .../vector-combine/store.ll                   | 91 +++++++++++++++++++
 1 file changed, 91 insertions(+)
 create mode 100644 llvm/test/DebugInfo/Generic/assignment-tracking/vector-combine/store.ll

diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/vector-combine/store.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/vector-combine/store.ll
new file mode 100644
index 00000000000000..924fc7194174f7
--- /dev/null
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/vector-combine/store.ll
@@ -0,0 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=vector-combine < %s | FileCheck %s
+
+declare void @init(ptr)
+declare void @observe(ptr)
+
+; The replacement scalar store stays linked to the original dbg.assign.
+define void @scalar_store(i8 %x) !dbg !6 {
+; CHECK-LABEL: define void @scalar_store(
+; CHECK-SAME: i8 [[X:%.*]]) !dbg [[DBG4:![0-9]+]] {
+; CHECK-NEXT:    [[P:%.*]] = alloca <16 x i8>, align 16
+; CHECK-NEXT:    call void @init(ptr [[P]]), !dbg [[DBG7:![0-9]+]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 3, !dbg [[DBG7]]
+; CHECK-NEXT:    store i8 [[X]], ptr [[TMP1]], align 1, !dbg [[DBG7]], !DIAssignID [[DIASSIGNID8:![0-9]+]]
+; CHECK-NEXT:      #dbg_assign(<16 x i8> poison, [[META9:![0-9]+]], !DIExpression(), [[DIASSIGNID8]], ptr [[P]], !DIExpression(), [[DBG7]])
+; CHECK-NEXT:    call void @observe(ptr [[P]]), !dbg [[DBG7]]
+; CHECK-NEXT:    ret void, !dbg [[DBG7]]
+;
+  %p = alloca <16 x i8>, align 16
+  call void @init(ptr %p), !dbg !10
+  %v = load <16 x i8>, ptr %p, align 16, !dbg !10
+  %i = insertelement <16 x i8> %v, i8 %x, i32 3, !dbg !10
+  store <16 x i8> %i, ptr %p, align 16, !DIAssignID !0, !dbg !10
+    #dbg_assign(<16 x i8> %i, !11, !DIExpression(), !0, ptr %p, !DIExpression(), !10)
+  call void @observe(ptr %p), !dbg !10
+  ret void, !dbg !10
+}
+
+; Both replacement scalar stores share the original dbg.assign's assignment ID.
+define void @two_updates(i8 %x, i8 %y) !dbg !17 {
+; CHECK-LABEL: define void @two_updates(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) !dbg [[DBG14:![0-9]+]] {
+; CHECK-NEXT:    [[P:%.*]] = alloca <16 x i8>, align 16
+; CHECK-NEXT:    call void @init(ptr [[P]]), !dbg [[DBG15:![0-9]+]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 3, !dbg [[DBG15]]
+; CHECK-NEXT:    store i8 [[X]], ptr [[TMP1]], align 1, !dbg [[DBG15]], !DIAssignID [[DIASSIGNID16:![0-9]+]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 7, !dbg [[DBG15]]
+; CHECK-NEXT:    store i8 [[Y]], ptr [[TMP2]], align 1, !dbg [[DBG15]], !DIAssignID [[DIASSIGNID16]]
+; CHECK-NEXT:      #dbg_assign(<16 x i8> poison, [[META17:![0-9]+]], !DIExpression(), [[DIASSIGNID16]], ptr [[P]], !DIExpression(), [[DBG15]])
+; CHECK-NEXT:    call void @observe(ptr [[P]]), !dbg [[DBG15]]
+; CHECK-NEXT:    ret void, !dbg [[DBG15]]
+;
+  %p = alloca <16 x i8>, align 16
+  call void @init(ptr %p), !dbg !18
+  %v = load <16 x i8>, ptr %p, align 16, !dbg !18
+  %i = insertelement <16 x i8> %v, i8 %x, i32 3, !dbg !18
+  %j = insertelement <16 x i8> %i, i8 %y, i32 7, !dbg !18
+  store <16 x i8> %j, ptr %p, align 16, !DIAssignID !20, !dbg !18
+    #dbg_assign(<16 x i8> %j, !19, !DIExpression(), !20, ptr %p, !DIExpression(), !18)
+  call void @observe(ptr %p), !dbg !18
+  ret void, !dbg !18
+}
+!llvm.module.flags = !{!4, !5}
+!llvm.dbg.cu = !{!7}
+!0 = distinct !DIAssignID()
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 7, !"debug-info-assignment-tracking", i1 true}
+!6 = distinct !DISubprogram(name: "scalar_store", scope: !8, file: !8, line: 1, type: !9, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !7)
+!7 = distinct !DICompileUnit(language: DW_LANG_C, file: !8, producer: "audit", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!8 = !DIFile(filename: "audit.c", directory: "/")
+!9 = !DISubroutineType(types: !13)
+!10 = !DILocation(line: 2, column: 1, scope: !6)
+!11 = !DILocalVariable(name: "buffer", scope: !6, file: !8, line: 1, type: !12)
+!12 = !DICompositeType(tag: DW_TAG_array_type, baseType: !14, size: 128, elements: !15)
+!13 = !{}
+!14 = !DIBasicType(name: "byte", size: 8, encoding: DW_ATE_unsigned_char)
+!15 = !{!16}
+!16 = !DISubrange(count: 16)
+
+!17 = distinct !DISubprogram(name: "two_updates", scope: !8, file: !8, line: 1, type: !9, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !7)
+!18 = !DILocation(line: 2, column: 1, scope: !17)
+!19 = !DILocalVariable(name: "buffer2", scope: !17, file: !8, line: 1, type: !12)
+!20 = distinct !DIAssignID()
+;.
+; CHECK: [[META2:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C, file: [[META3:![0-9]+]], producer: "audit", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+; CHECK: [[META3]] = !DIFile(filename: "{{.*}}audit.c", directory: {{.*}})
+; CHECK: [[DBG4]] = distinct !DISubprogram(name: "scalar_store", scope: [[META3]], file: [[META3]], line: 1, type: [[META5:![0-9]+]], scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META2]])
+; CHECK: [[META5]] = !DISubroutineType(types: [[META6:![0-9]+]])
+; CHECK: [[META6]] = !{}
+; CHECK: [[DBG7]] = !DILocation(line: 2, column: 1, scope: [[DBG4]])
+; CHECK: [[DIASSIGNID8]] = distinct !DIAssignID()
+; CHECK: [[META9]] = !DILocalVariable(name: "buffer", scope: [[DBG4]], file: [[META3]], line: 1, type: [[META10:![0-9]+]])
+; CHECK: [[META10]] = !DICompositeType(tag: DW_TAG_array_type, baseType: [[META11:![0-9]+]], size: 128, elements: [[META12:![0-9]+]])
+; CHECK: [[META11]] = !DIBasicType(name: "byte", size: 8, encoding: DW_ATE_unsigned_char)
+; CHECK: [[META12]] = !{[[META13:![0-9]+]]}
+; CHECK: [[META13]] = !DISubrange(count: 16)
+; CHECK: [[DBG14]] = distinct !DISubprogram(name: "two_updates", scope: [[META3]], file: [[META3]], line: 1, type: [[META5]], scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META2]])
+; CHECK: [[DBG15]] = !DILocation(line: 2, column: 1, scope: [[DBG14]])
+; CHECK: [[DIASSIGNID16]] = distinct !DIAssignID()
+; CHECK: [[META17]] = !DILocalVariable(name: "buffer2", scope: [[DBG14]], file: [[META3]], line: 1, type: [[META10]])
+;.

>From 5021c2895e812120051a36d54cb71dc693402c50 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 17 Nov 2026 23:30:30 +0000
Subject: [PATCH 09/14] [Analysis] Format relocated metadata combining helpers

---
 llvm/lib/Analysis/MetadataUtils.cpp | 240 ++++++++++++++--------------
 1 file changed, 119 insertions(+), 121 deletions(-)

diff --git a/llvm/lib/Analysis/MetadataUtils.cpp b/llvm/lib/Analysis/MetadataUtils.cpp
index 4c51eda0e9de93..02f6f8acb77d41 100644
--- a/llvm/lib/Analysis/MetadataUtils.cpp
+++ b/llvm/lib/Analysis/MetadataUtils.cpp
@@ -498,128 +498,127 @@ static void combineMetadata(Instruction *K, const Instruction *J,
 
     // TODO: Assert that this switch is exhaustive for fixed MD kinds.
     switch (Kind) {
-      default:
-        K->setMetadata(Kind, nullptr); // Remove unknown metadata
-        break;
-      case LLVMContext::MD_dbg:
-        llvm_unreachable("getAllMetadataOtherThanDebugLoc returned a MD_dbg");
-      case LLVMContext::MD_DIAssignID:
-        if (!AAOnly)
-          K->mergeDIAssignID(J);
-        break;
-      case LLVMContext::MD_tbaa:
-        if (DoesKMove)
-          K->setMetadata(Kind, MDNode::getMostGenericTBAA(JMD, KMD));
-        break;
-      case LLVMContext::MD_alias_scope:
-        if (DoesKMove)
-          K->setMetadata(Kind, MDNode::getMostGenericAliasScope(JMD, KMD));
-        break;
-      case LLVMContext::MD_noalias:
-      case LLVMContext::MD_mem_parallel_loop_access:
-        if (DoesKMove)
-          K->setMetadata(Kind, MDNode::intersect(JMD, KMD));
-        break;
-      case LLVMContext::MD_access_group:
-        if (DoesKMove)
-          K->setMetadata(LLVMContext::MD_access_group,
-                         intersectAccessGroups(K, J));
-        break;
-      case LLVMContext::MD_range:
-        if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
-          K->setMetadata(Kind, MDNode::getMostGenericRange(JMD, KMD));
-        break;
-      case LLVMContext::MD_nofpclass:
-        if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
-          K->setMetadata(Kind, MDNode::getMostGenericNoFPClass(JMD, KMD));
-        break;
-      case LLVMContext::MD_fpmath:
-        if (!AAOnly)
-          K->setMetadata(Kind, MDNode::getMostGenericFPMath(JMD, KMD));
-        break;
-      case LLVMContext::MD_invariant_load:
-      case LLVMContext::MD_invariant_group:
-        // If K moves, only keep the invariant metadata if it is present on
-        // both instructions; otherwise the invariant would be asserted on a
-        // path (J's) that never promised it. If K does not move, K stays on
-        // its original path, so its existing metadata remains valid.
-        if (DoesKMove)
-          K->setMetadata(Kind, JMD);
-        break;
-      case LLVMContext::MD_nonnull:
-        if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
-          K->setMetadata(Kind, JMD);
-        break;
-      // Keep empty cases for prof, mmra, memprof, and callsite to prevent them
-      // from being removed as unknown metadata. The actual merging is handled
-      // separately below.
-      case LLVMContext::MD_prof:
-      case LLVMContext::MD_mmra:
-      case LLVMContext::MD_memprof:
-      case LLVMContext::MD_callsite:
-        break;
-      case LLVMContext::MD_callee_type:
-        if (!AAOnly) {
-          K->setMetadata(LLVMContext::MD_callee_type,
-                         MDNode::getMergedCalleeTypeMetadata(KMD, JMD));
-        }
-        break;
-      case LLVMContext::MD_callees:
-        // If K moves, it replaces J on J's path and must allow J's callees as
-        // well. If K does not move, its callees remain valid.
-        if (!AAOnly && DoesKMove)
-          K->setMetadata(Kind, MDNode::getMergedCalleesMetadata(KMD, JMD));
-        break;
-      case LLVMContext::MD_align:
-        if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
-          K->setMetadata(
-              Kind, MDNode::getMostGenericAlignmentOrDereferenceable(JMD, KMD));
-        break;
-      case LLVMContext::MD_dereferenceable:
-      case LLVMContext::MD_dereferenceable_or_null:
-        if (!AAOnly && DoesKMove)
-          K->setMetadata(Kind,
-            MDNode::getMostGenericAlignmentOrDereferenceable(JMD, KMD));
-        break;
-      case LLVMContext::MD_preserve_access_index:
-        // Preserve !preserve.access.index in K.
-        break;
-      case LLVMContext::MD_noundef:
-        // If K does move, keep noundef if it is present in both instructions.
-        if (!AAOnly && DoesKMove)
-          K->setMetadata(Kind, JMD);
-        break;
-      case LLVMContext::MD_nontemporal:
-        // Preserve !nontemporal if it is present on both instructions.
-        if (!AAOnly)
-          K->setMetadata(Kind, JMD);
-        break;
-      case LLVMContext::MD_mem_cache_hint:
-        // Preserve !mem.cache_hint only if it is present and equivalent on both
-        // instructions.
-        if (!AAOnly && KMD != JMD)
-          K->setMetadata(Kind, nullptr);
-        break;
-      case LLVMContext::MD_noalias_addrspace:
-        if (DoesKMove)
-          K->setMetadata(Kind,
-                         MDNode::getMostGenericNoaliasAddrspace(JMD, KMD));
-        break;
-      case LLVMContext::MD_nosanitize:
-        // Preserve !nosanitize if both K and J have it.
+    default:
+      K->setMetadata(Kind, nullptr); // Remove unknown metadata
+      break;
+    case LLVMContext::MD_dbg:
+      llvm_unreachable("getAllMetadataOtherThanDebugLoc returned a MD_dbg");
+    case LLVMContext::MD_DIAssignID:
+      if (!AAOnly)
+        K->mergeDIAssignID(J);
+      break;
+    case LLVMContext::MD_tbaa:
+      if (DoesKMove)
+        K->setMetadata(Kind, MDNode::getMostGenericTBAA(JMD, KMD));
+      break;
+    case LLVMContext::MD_alias_scope:
+      if (DoesKMove)
+        K->setMetadata(Kind, MDNode::getMostGenericAliasScope(JMD, KMD));
+      break;
+    case LLVMContext::MD_noalias:
+    case LLVMContext::MD_mem_parallel_loop_access:
+      if (DoesKMove)
+        K->setMetadata(Kind, MDNode::intersect(JMD, KMD));
+      break;
+    case LLVMContext::MD_access_group:
+      if (DoesKMove)
+        K->setMetadata(LLVMContext::MD_access_group,
+                       intersectAccessGroups(K, J));
+      break;
+    case LLVMContext::MD_range:
+      if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
+        K->setMetadata(Kind, MDNode::getMostGenericRange(JMD, KMD));
+      break;
+    case LLVMContext::MD_nofpclass:
+      if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
+        K->setMetadata(Kind, MDNode::getMostGenericNoFPClass(JMD, KMD));
+      break;
+    case LLVMContext::MD_fpmath:
+      if (!AAOnly)
+        K->setMetadata(Kind, MDNode::getMostGenericFPMath(JMD, KMD));
+      break;
+    case LLVMContext::MD_invariant_load:
+    case LLVMContext::MD_invariant_group:
+      // If K moves, only keep the invariant metadata if it is present on
+      // both instructions; otherwise the invariant would be asserted on a
+      // path (J's) that never promised it. If K does not move, K stays on
+      // its original path, so its existing metadata remains valid.
+      if (DoesKMove)
         K->setMetadata(Kind, JMD);
-        break;
-      case LLVMContext::MD_captures:
-        K->setMetadata(
-            Kind, MDNode::fromCaptureComponents(
-                      K->getContext(), MDNode::toCaptureComponents(JMD) |
-                                           MDNode::toCaptureComponents(KMD)));
-        break;
-      case LLVMContext::MD_alloc_token:
-        if (!AAOnly && KMD != JMD)
-          K->setMetadata(Kind, MDNode::getMergedAllocTokenMetadata(KMD, JMD));
-        break;
+      break;
+    case LLVMContext::MD_nonnull:
+      if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
+        K->setMetadata(Kind, JMD);
+      break;
+    // Keep empty cases for prof, mmra, memprof, and callsite to prevent them
+    // from being removed as unknown metadata. The actual merging is handled
+    // separately below.
+    case LLVMContext::MD_prof:
+    case LLVMContext::MD_mmra:
+    case LLVMContext::MD_memprof:
+    case LLVMContext::MD_callsite:
+      break;
+    case LLVMContext::MD_callee_type:
+      if (!AAOnly) {
+        K->setMetadata(LLVMContext::MD_callee_type,
+                       MDNode::getMergedCalleeTypeMetadata(KMD, JMD));
       }
+      break;
+    case LLVMContext::MD_callees:
+      // If K moves, it replaces J on J's path and must allow J's callees as
+      // well. If K does not move, its callees remain valid.
+      if (!AAOnly && DoesKMove)
+        K->setMetadata(Kind, MDNode::getMergedCalleesMetadata(KMD, JMD));
+      break;
+    case LLVMContext::MD_align:
+      if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
+        K->setMetadata(
+            Kind, MDNode::getMostGenericAlignmentOrDereferenceable(JMD, KMD));
+      break;
+    case LLVMContext::MD_dereferenceable:
+    case LLVMContext::MD_dereferenceable_or_null:
+      if (!AAOnly && DoesKMove)
+        K->setMetadata(
+            Kind, MDNode::getMostGenericAlignmentOrDereferenceable(JMD, KMD));
+      break;
+    case LLVMContext::MD_preserve_access_index:
+      // Preserve !preserve.access.index in K.
+      break;
+    case LLVMContext::MD_noundef:
+      // If K does move, keep noundef if it is present in both instructions.
+      if (!AAOnly && DoesKMove)
+        K->setMetadata(Kind, JMD);
+      break;
+    case LLVMContext::MD_nontemporal:
+      // Preserve !nontemporal if it is present on both instructions.
+      if (!AAOnly)
+        K->setMetadata(Kind, JMD);
+      break;
+    case LLVMContext::MD_mem_cache_hint:
+      // Preserve !mem.cache_hint only if it is present and equivalent on both
+      // instructions.
+      if (!AAOnly && KMD != JMD)
+        K->setMetadata(Kind, nullptr);
+      break;
+    case LLVMContext::MD_noalias_addrspace:
+      if (DoesKMove)
+        K->setMetadata(Kind, MDNode::getMostGenericNoaliasAddrspace(JMD, KMD));
+      break;
+    case LLVMContext::MD_nosanitize:
+      // Preserve !nosanitize if both K and J have it.
+      K->setMetadata(Kind, JMD);
+      break;
+    case LLVMContext::MD_captures:
+      K->setMetadata(
+          Kind, MDNode::fromCaptureComponents(
+                    K->getContext(), MDNode::toCaptureComponents(JMD) |
+                                         MDNode::toCaptureComponents(KMD)));
+      break;
+    case LLVMContext::MD_alloc_token:
+      if (!AAOnly && KMD != JMD)
+        K->setMetadata(Kind, MDNode::getMergedAllocTokenMetadata(KMD, JMD));
+      break;
+    }
   }
 
   // Merge MMRAs.
@@ -676,4 +675,3 @@ void llvm::combineMetadataForCSE(Instruction *K, const Instruction *J,
 void llvm::combineAAMetadata(Instruction *K, const Instruction *J) {
   combineMetadata(K, J, /*DoesKMove=*/true, /*AAOnly=*/true);
 }
-

>From fb1eba43c149ab6f94979ae53c05fa60bb038518 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 17 Nov 2026 23:34:58 +0000
Subject: [PATCH 10/14] [MetadataUtils] Update for current VectorCombine
 transforms

---
 llvm/lib/Transforms/Vectorize/VectorCombine.cpp  |  9 ++-------
 .../assignment-tracking/vector-combine/store.ll  |  6 +++---
 .../Transforms/VectorCombine/memory-metadata.ll  | 16 ++++++++--------
 3 files changed, 13 insertions(+), 18 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 01e3988562c8ff..39c5f4d87780a1 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -7094,13 +7094,8 @@ bool VectorCombine::foldShuffleOfAdjacentLoads(Instruction &I) {
   LoadInst *WideLoad = Builder.CreateAlignedLoad(
       WideTy, LowLoad->getPointerOperand(), LowLoad->getAlign());
 
-  // Set the metadata on the wide load. copyMetadataForLoad seeds it from
-  // LowLoad, then combineMetadataForCSE intersects every known kind against
-  // HighLoad (taking the most-generic value where applicable, keeping facts
-  // only where both loads agree, and dropping unknown metadata), so nothing is
-  // asserted over the combined load unless justified by both halves.
-  copyMetadataForLoad(*WideLoad, *LowLoad);
-  combineMetadataForCSE(WideLoad, HighLoad, /*DoesKMove=*/true);
+  // Merge metadata from both loads over the combined access.
+  propagateMetadata(WideLoad, {LowLoad, HighLoad});
 
   for (ShuffleVectorInst *SV : Shuffles) {
     SmallVector<int, 32> NewMask;
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/vector-combine/store.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/vector-combine/store.ll
index 924fc7194174f7..e9a752707b206c 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/vector-combine/store.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/vector-combine/store.ll
@@ -10,7 +10,7 @@ define void @scalar_store(i8 %x) !dbg !6 {
 ; CHECK-SAME: i8 [[X:%.*]]) !dbg [[DBG4:![0-9]+]] {
 ; CHECK-NEXT:    [[P:%.*]] = alloca <16 x i8>, align 16
 ; CHECK-NEXT:    call void @init(ptr [[P]]), !dbg [[DBG7:![0-9]+]]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 3, !dbg [[DBG7]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 3, !dbg [[DBG7]]
 ; CHECK-NEXT:    store i8 [[X]], ptr [[TMP1]], align 1, !dbg [[DBG7]], !DIAssignID [[DIASSIGNID8:![0-9]+]]
 ; CHECK-NEXT:      #dbg_assign(<16 x i8> poison, [[META9:![0-9]+]], !DIExpression(), [[DIASSIGNID8]], ptr [[P]], !DIExpression(), [[DBG7]])
 ; CHECK-NEXT:    call void @observe(ptr [[P]]), !dbg [[DBG7]]
@@ -32,9 +32,9 @@ define void @two_updates(i8 %x, i8 %y) !dbg !17 {
 ; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) !dbg [[DBG14:![0-9]+]] {
 ; CHECK-NEXT:    [[P:%.*]] = alloca <16 x i8>, align 16
 ; CHECK-NEXT:    call void @init(ptr [[P]]), !dbg [[DBG15:![0-9]+]]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 3, !dbg [[DBG15]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 3, !dbg [[DBG15]]
 ; CHECK-NEXT:    store i8 [[X]], ptr [[TMP1]], align 1, !dbg [[DBG15]], !DIAssignID [[DIASSIGNID16:![0-9]+]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 7, !dbg [[DBG15]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 7, !dbg [[DBG15]]
 ; CHECK-NEXT:    store i8 [[Y]], ptr [[TMP2]], align 1, !dbg [[DBG15]], !DIAssignID [[DIASSIGNID16]]
 ; CHECK-NEXT:      #dbg_assign(<16 x i8> poison, [[META17:![0-9]+]], !DIExpression(), [[DIASSIGNID16]], ptr [[P]], !DIExpression(), [[DBG15]])
 ; CHECK-NEXT:    call void @observe(ptr [[P]]), !dbg [[DBG15]]
diff --git a/llvm/test/Transforms/VectorCombine/memory-metadata.ll b/llvm/test/Transforms/VectorCombine/memory-metadata.ll
index ef1ddffd3f6688..dac87dd4638a92 100644
--- a/llvm/test/Transforms/VectorCombine/memory-metadata.ll
+++ b/llvm/test/Transforms/VectorCombine/memory-metadata.ll
@@ -16,7 +16,7 @@ define i32 @extract_zero(ptr %p) {
 define i32 @extract_nonzero(ptr %p) {
 ; CHECK-LABEL: define i32 @extract_nonzero(
 ; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 2
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
 ; CHECK-NEXT:    [[R:%.*]] = load i32, ptr [[TMP1]], align 8, !tbaa [[CHAR_TBAA0]], !range [[RNG3]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.access.group [[ACC_GRP9]], !noundef [[META4]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META13]]
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
@@ -31,7 +31,7 @@ define i32 @extract_dynamic(ptr %p, i32 %index) {
 ; CHECK-NEXT:    [[INDEX_FROZEN:%.*]] = freeze i32 [[INDEX]]
 ; CHECK-NEXT:    [[IDX:%.*]] = and i32 [[INDEX_FROZEN]], 3
 ; CHECK-NEXT:    [[IDX_GEPIDX:%.*]] = zext i32 [[IDX]] to i64
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 [[IDX_GEPIDX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [4 x i8], ptr [[P]], i64 [[IDX_GEPIDX]]
 ; CHECK-NEXT:    [[R:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[CHAR_TBAA0]], !range [[RNG3]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.access.group [[ACC_GRP9]], !noundef [[META4]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META13]]
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
@@ -44,7 +44,7 @@ define i32 @extract_dynamic(ptr %p, i32 %index) {
 define float @extract_float(ptr %p) {
 ; CHECK-LABEL: define float @extract_float(
 ; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <4 x float>, ptr [[P]], i64 0, i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 4
 ; CHECK-NEXT:    [[R:%.*]] = load float, ptr [[TMP1]], align 4, !fpmath [[META15:![0-9]+]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.access.group [[ACC_GRP9]], !noundef [[META4]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !nofpclass [[META16:![0-9]+]], !mem.cache_hint [[META13]]
 ; CHECK-NEXT:    ret float [[R]]
 ;
@@ -92,7 +92,7 @@ define void @store_zero(ptr %p, i8 %x, i32 %index) {
 define void @store_nonzero(ptr %p, i8 %x, i32 %index) {
 ; CHECK-LABEL: define void @store_nonzero(
 ; CHECK-SAME: ptr [[P:%.*]], i8 [[X:%.*]], i32 [[INDEX:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 3
 ; CHECK-NEXT:    store i8 [[X]], ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.access.group [[ACC_GRP9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META17]]
 ; CHECK-NEXT:    ret void
 ;
@@ -108,7 +108,7 @@ define void @store_dynamic(ptr %p, i8 %x, i32 %index) {
 ; CHECK-NEXT:    [[INDEX_FROZEN:%.*]] = freeze i32 [[INDEX]]
 ; CHECK-NEXT:    [[IDX:%.*]] = and i32 [[INDEX_FROZEN]], 15
 ; CHECK-NEXT:    [[IDX_GEPIDX:%.*]] = zext i32 [[IDX]] to i64
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 [[IDX_GEPIDX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[IDX_GEPIDX]]
 ; CHECK-NEXT:    store i8 [[X]], ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !nontemporal [[META8]], !llvm.access.group [[ACC_GRP9]], !annotation [[META10]], !nosanitize [[META4]], !mmra [[META11]], !noalias.addrspace [[META12]], !mem.cache_hint [[META17]]
 ; CHECK-NEXT:    ret void
 ;
@@ -124,7 +124,7 @@ define void @store_dynamic(ptr %p, i8 %x, i32 %index) {
 define i32 @extract_struct_field(ptr %p) {
 ; CHECK-LABEL: define i32 @extract_struct_field(
 ; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 4
 ; CHECK-NEXT:    [[R:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[INT_TBAA18:![0-9]+]]
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
@@ -136,7 +136,7 @@ define i32 @extract_struct_field(ptr %p) {
 define void @store_struct_field(ptr %p, i32 %x) {
 ; CHECK-LABEL: define void @store_struct_field(
 ; CHECK-SAME: ptr [[P:%.*]], i32 [[X:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P]], i64 0, i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 4
 ; CHECK-NEXT:    store i32 [[X]], ptr [[TMP1]], align 4, !tbaa [[INT_TBAA18]]
 ; CHECK-NEXT:    ret void
 ;
@@ -149,7 +149,7 @@ define void @store_struct_field(ptr %p, i32 %x) {
 define void @scalar_store(ptr %p, i8 %x) {
 ; CHECK-LABEL: define void @scalar_store(
 ; CHECK-SAME: ptr [[P:%.*]], i8 [[X:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[P]], i64 0, i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 3
 ; CHECK-NEXT:    store i8 [[X]], ptr [[TMP1]], align 1, !pcsections [[META20:![0-9]+]]
 ; CHECK-NEXT:    ret void
 ;

>From 07a8fd01bed401d163627039962b4bae069926a2 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 18 Nov 2026 00:27:10 +0000
Subject: [PATCH 11/14] [MetadataUtils] Use nondeprecated IRBuilder constructor
 in tests

---
 llvm/unittests/Analysis/MetadataUtilsTest.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/unittests/Analysis/MetadataUtilsTest.cpp b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
index b55922ebd0893c..abe80a2ad9920b 100644
--- a/llvm/unittests/Analysis/MetadataUtilsTest.cpp
+++ b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
@@ -22,7 +22,7 @@ class MetadataUtilsTest : public testing::Test {
 protected:
   LLVMContext C;
   Module M{"test", C};
-  IRBuilder<> B{C};
+  IRBuilder<> B{M};
   Value *P;
   Value *Q;
 

>From 02a014803a821aef8daccdd7f6d805ecc37f95ca Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 18 Nov 2026 01:01:02 +0000
Subject: [PATCH 12/14] [Analysis] Add metadata propagation for replacement
 memory accesses

---
 llvm/include/llvm/Analysis/MetadataUtils.h    |  44 +-
 llvm/lib/Analysis/MetadataUtils.cpp           | 259 +++++++++
 llvm/unittests/Analysis/CMakeLists.txt        |   1 +
 llvm/unittests/Analysis/MetadataUtilsTest.cpp | 504 ++++++++++++++++++
 4 files changed, 792 insertions(+), 16 deletions(-)
 create mode 100644 llvm/unittests/Analysis/MetadataUtilsTest.cpp

diff --git a/llvm/include/llvm/Analysis/MetadataUtils.h b/llvm/include/llvm/Analysis/MetadataUtils.h
index a7afe9514e0a8e..94a0816bbc209c 100644
--- a/llvm/include/llvm/Analysis/MetadataUtils.h
+++ b/llvm/include/llvm/Analysis/MetadataUtils.h
@@ -23,19 +23,20 @@ class Instruction;
 class MDNode;
 class Value;
 
-/// Combine the metadata of two instructions so that K can replace J. This
-/// specifically handles the case of CSE-like transformations. Some
-/// metadata can only be kept if K dominates J. For this to be correct,
-/// K cannot be hoisted.
+/// Copy metadata valid for a replacement memory access.
 ///
-/// Unknown metadata is removed.
-LLVM_ABI void combineMetadataForCSE(Instruction *K, const Instruction *J,
-                                    bool DoesKMove);
-
-/// Combine metadata of two instructions, where instruction J is a memory
-/// access that has been merged into K. This will intersect alias-analysis
-/// metadata, while preserving other known metadata.
-LLVM_ABI void combineAAMetadata(Instruction *K, const Instruction *J);
+/// The destination must access the same bytes, a subset, or a wider byte range
+/// in the same object as the source. The destination must be a typed memory
+/// access. Memory-transfer and memory-set intrinsics are supported only as
+/// sources and must have a constant length.
+///
+/// DestByteOffset is the destination's offset within the source access. If the
+/// destination is no larger than the source, a missing offset means the offset
+/// is dynamic, but the caller guarentees it accesses a subset of the source
+/// access.
+LLVM_ABI void
+copyMetadataForMemoryAccess(Instruction &Dest, const Instruction &Source,
+                            std::optional<int64_t> DestByteOffset = 0);
 
 /// Copy the metadata from the source instruction to the destination (the
 /// replacement for the source instruction).
@@ -55,6 +56,20 @@ LLVM_ABI void copyNonnullMetadata(const LoadInst &OldLI, MDNode *N,
 LLVM_ABI void copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI,
                                 MDNode *N, LoadInst &NewLI);
 
+/// Combine the metadata of two instructions so that K can replace J. This
+/// specifically handles the case of CSE-like transformations. Some
+/// metadata can only be kept if K dominates J. For this to be correct,
+/// K cannot be hoisted.
+///
+/// Unknown metadata is removed.
+LLVM_ABI void combineMetadataForCSE(Instruction *K, const Instruction *J,
+                                    bool DoesKMove);
+
+/// Combine metadata of two instructions, where instruction J is a memory
+/// access that has been merged into K. This will intersect alias-analysis
+/// metadata, while preserving other known metadata.
+LLVM_ABI void combineAAMetadata(Instruction *K, const Instruction *J);
+
 /// Compute the union of two access-group lists.
 ///
 /// If the list contains just one access group, it is returned directly. If the
@@ -70,10 +85,7 @@ LLVM_ABI MDNode *uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2);
 LLVM_ABI MDNode *intersectAccessGroups(const Instruction *Inst1,
                                        const Instruction *Inst2);
 
-/// Add metadata from \p Inst to \p Metadata, if it can be preserved after
-/// vectorization. It can be preserved after vectorization if the kind is one of
-/// [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath, MD_nontemporal,
-/// MD_access_group, MD_mmra].
+/// Collect metadata that can be preserved when vectorizing an instruction.
 LLVM_ABI void getMetadataToPropagate(
     Instruction *Inst,
     SmallVectorImpl<std::pair<unsigned, MDNode *>> &Metadata);
diff --git a/llvm/lib/Analysis/MetadataUtils.cpp b/llvm/lib/Analysis/MetadataUtils.cpp
index a4d63aa51b8120..e748344330c134 100644
--- a/llvm/lib/Analysis/MetadataUtils.cpp
+++ b/llvm/lib/Analysis/MetadataUtils.cpp
@@ -26,6 +26,265 @@
 
 using namespace llvm;
 
+namespace {
+struct MemoryAccessInfo {
+  Type *Ty;
+  unsigned PointerOperand;
+  bool IsRead;
+};
+} // namespace
+
+/// Get information about a memory instruction required to propagate metadata.
+/// Returns std::nullopt if the instruction is not supported and we will not
+/// propagate any metadata.
+///
+/// If IsRead is set and this is a instruction that reads and writes, mark it
+/// as reading or writing. This is used to make the source instruction in
+/// metadata propagation correspond with the dest.
+static std::optional<MemoryAccessInfo>
+getMemoryAccessInfo(const Instruction &I,
+                    std::optional<bool> IsRead = std::nullopt) {
+  if (const auto *LI = dyn_cast<LoadInst>(&I))
+    return MemoryAccessInfo{LI->getType(), 0, true};
+  if (const auto *SI = dyn_cast<StoreInst>(&I))
+    return MemoryAccessInfo{SI->getValueOperand()->getType(), 1, false};
+  if (const auto *MI = dyn_cast<AnyMemIntrinsic>(&I)) {
+    if (!IsRead || (*IsRead && !isa<AnyMemTransferInst>(MI)))
+      return std::nullopt;
+    return MemoryAccessInfo{nullptr, *IsRead ? 1u : 0u, *IsRead};
+  }
+  const auto *II = dyn_cast<IntrinsicInst>(&I);
+  if (!II)
+    return std::nullopt;
+  switch (II->getIntrinsicID()) {
+  case Intrinsic::experimental_vector_histogram_add:
+  case Intrinsic::experimental_vector_histogram_uadd_sat:
+  case Intrinsic::experimental_vector_histogram_umin:
+  case Intrinsic::experimental_vector_histogram_umax:
+    if (IsRead)
+      return MemoryAccessInfo{II->getArgOperand(1)->getType(), 0, *IsRead};
+    return std::nullopt;
+  case Intrinsic::masked_load:
+  case Intrinsic::masked_gather:
+  case Intrinsic::masked_expandload:
+  case Intrinsic::vp_load:
+    return MemoryAccessInfo{II->getType(), 0, true};
+  case Intrinsic::masked_store:
+  case Intrinsic::masked_scatter:
+  case Intrinsic::masked_compressstore:
+    return MemoryAccessInfo{II->getArgOperand(0)->getType(), 1, false};
+  default:
+    return std::nullopt;
+  }
+}
+
+/// Get the pointer operand of `I` at operand index `Index`.
+static const Value *getMemoryPointer(const Instruction &I, unsigned Index) {
+  if (const auto *CB = dyn_cast<CallBase>(&I))
+    return CB->getArgOperand(Index);
+  return I.getOperand(Index);
+}
+
+// invariant.group ignores bitcasts and zero-offset GEPs, not address-space
+// casts.
+static const Value *getInvariantGroupPointer(const Value *V) {
+  while (true) {
+    if (const auto *Cast = dyn_cast<BitCastOperator>(V))
+      V = Cast->getOperand(0);
+    else if (const auto *GEP = dyn_cast<GEPOperator>(V);
+             GEP && GEP->hasAllZeroIndices())
+      V = GEP->getPointerOperand();
+    else
+      return V;
+  }
+}
+
+static void copyNonnullMetadataForAccess(const LoadInst &OldLI, MDNode *N,
+                                         LoadInst &NewLI) {
+  auto *OldTy = cast<PointerType>(OldLI.getType());
+  auto *NewTy = NewLI.getType();
+
+  // Preserve the pointer fact when the pointer type is unchanged.
+  if (NewTy == OldTy) {
+    NewLI.setMetadata(LLVMContext::MD_nonnull, N);
+    return;
+  }
+
+  // An integer range must describe the complete pointer representation.
+  if (!NewTy->isIntegerTy() ||
+      NewTy->getIntegerBitWidth() !=
+          OldLI.getDataLayout().getPointerTypeSizeInBits(OldTy))
+    return;
+
+  MDBuilder MDB(NewLI.getContext());
+  auto *ITy = cast<IntegerType>(NewTy);
+  auto *NullInt =
+      ConstantExpr::getPtrToInt(ConstantPointerNull::get(OldTy), ITy);
+  auto *NonNullInt = ConstantExpr::getAdd(NullInt, ConstantInt::get(ITy, 1));
+  NewLI.setMetadata(LLVMContext::MD_range,
+                    MDB.createRange(NonNullInt, NullInt));
+}
+
+void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
+                                       const Instruction &Source,
+                                       std::optional<int64_t> DestByteOffset) {
+  auto DestInfo = getMemoryAccessInfo(Dest);
+  // Untyped memory intrinsics are supported only as sources.
+  assert(DestInfo && DestInfo->Ty &&
+         "expected a typed destination memory access");
+  auto SourceInfo = getMemoryAccessInfo(Source, DestInfo->IsRead);
+  assert(SourceInfo && SourceInfo->IsRead == DestInfo->IsRead &&
+         "expected corresponding memory reads or writes");
+  const DataLayout &DL = Source.getDataLayout();
+  TypeSize OldSize =
+      SourceInfo->Ty
+          ? DL.getTypeStoreSize(SourceInfo->Ty)
+          : TypeSize::getFixed(
+                cast<ConstantInt>(cast<AnyMemIntrinsic>(Source).getLength())
+                    ->getZExtValue());
+  TypeSize NewSize = DL.getTypeStoreSize(DestInfo->Ty);
+  // Is the destination access contained in the source access? If the size
+  // check succeeds but the offset is unknown, containment is a caller
+  // guarantee.
+  bool IsSubset = TypeSize::isKnownLE(NewSize, OldSize);
+  if (DestByteOffset)
+    IsSubset = IsSubset && *DestByteOffset >= 0 &&
+               uint64_t(*DestByteOffset) <=
+                   OldSize.getKnownMinValue() - NewSize.getKnownMinValue();
+  bool IsWholeAccess = IsSubset && DestByteOffset == 0 && OldSize == NewSize;
+  Type *SourceValueTy = SourceInfo->Ty;
+  // Allow unpacking a single-element aggregates.
+  if (IsWholeAccess) {
+    if (auto *ST = dyn_cast_or_null<StructType>(SourceValueTy);
+        ST && ST->getNumElements() == 1 &&
+        ST->getElementType(0) == DestInfo->Ty)
+      SourceValueTy = DestInfo->Ty;
+    else if (auto *AT = dyn_cast_or_null<ArrayType>(SourceValueTy);
+             AT && AT->getNumElements() == 1 &&
+             AT->getElementType() == DestInfo->Ty)
+      SourceValueTy = DestInfo->Ty;
+  }
+  bool SameElementType = SourceValueTy && SourceValueTy->getScalarType() ==
+                                              DestInfo->Ty->getScalarType();
+  bool SamePointer = getInvariantGroupPointer(getMemoryPointer(
+                         Source, SourceInfo->PointerOperand)) ==
+                     getInvariantGroupPointer(
+                         getMemoryPointer(Dest, DestInfo->PointerOperand));
+
+  // Now loop over all `Source` metadata, propagating what we can.
+  SmallVector<std::pair<unsigned, MDNode *>, 8> MDs;
+  Source.getAllMetadata(MDs);
+  for (auto [Kind, MD] : MDs) {
+    bool Keep = false;
+    switch (Kind) {
+    case LLVMContext::MD_dbg:
+    case LLVMContext::MD_nontemporal:
+    case LLVMContext::MD_annotation:
+    case LLVMContext::MD_nosanitize:
+    case LLVMContext::MD_pcsections:
+      Keep = true;
+      break;
+    case LLVMContext::MD_mmra:
+      Keep = canInstructionHaveMMRAs(Dest);
+      break;
+    case LLVMContext::MD_access_group:
+    case LLVMContext::MD_mem_parallel_loop_access:
+      Keep = IsSubset;
+      break;
+    case LLVMContext::MD_invariant_load:
+      Keep = IsSubset && DestInfo->IsRead;
+      break;
+    case LLVMContext::MD_invariant_group:
+      // TODO: Requiring `SamePointer` is too conservative.
+      Keep = IsSubset && SamePointer && !isa<CallBase>(Dest);
+      break;
+    case LLVMContext::MD_noundef:
+      // The value loaded remains defined if its element type is unchanged,
+      // or every source byte consists of defined value bits (i.e. the source
+      // doesn't have padding).
+      Keep = IsSubset && isa<LoadInst>(Dest) &&
+             (SameElementType ||
+              (SourceValueTy && !SourceValueTy->isAggregateType() &&
+               DL.typeSizeEqualsStoreSize(SourceValueTy)));
+      break;
+    case LLVMContext::MD_fpmath:
+    case LLVMContext::MD_nofpclass:
+      Keep = IsSubset && SameElementType &&
+             (Kind == LLVMContext::MD_fpmath
+                  ? FPMathOperator::isSupportedFloatingPointType(Dest.getType())
+                  : isa<LoadInst>(Dest));
+      break;
+    case LLVMContext::MD_nonnull:
+      if (IsWholeAccess && isa<LoadInst>(Source) && isa<LoadInst>(Dest))
+        copyNonnullMetadataForAccess(cast<LoadInst>(Source), MD,
+                                     cast<LoadInst>(Dest));
+      break;
+    case LLVMContext::MD_range:
+      // Ranges apply per integer element. Other type changes require the
+      // helper's explicit conversion from an integer range to nonnull.
+      if (IsSubset && SameElementType && DestInfo->Ty->isIntOrIntVectorTy())
+        Keep = true;
+      else if (IsWholeAccess && isa<LoadInst>(Source) && isa<LoadInst>(Dest))
+        copyRangeMetadata(DL, cast<LoadInst>(Source), MD, cast<LoadInst>(Dest));
+      break;
+    case LLVMContext::MD_align:
+    case LLVMContext::MD_dereferenceable:
+    case LLVMContext::MD_dereferenceable_or_null:
+      // These describe the loaded pointer value.
+      Keep = IsWholeAccess && SourceInfo->Ty == DestInfo->Ty &&
+             DestInfo->Ty->isPointerTy() && isa<LoadInst>(Dest);
+      break;
+    case LLVMContext::MD_DIAssignID: {
+      const auto *OldSI = dyn_cast<StoreInst>(&Source);
+      const auto *NewSI = dyn_cast<StoreInst>(&Dest);
+      // A new address requires updating the linked dbg.assign records.
+      Keep = IsWholeAccess && OldSI && NewSI &&
+             OldSI->getPointerOperand() == NewSI->getPointerOperand();
+      break;
+    }
+    case LLVMContext::MD_prof:
+      Keep =
+          IsWholeAccess && ((isa<LoadInst>(Source) && isa<LoadInst>(Dest)) ||
+                            (isa<StoreInst>(Source) && isa<StoreInst>(Dest)));
+      break;
+    case LLVMContext::MD_captures:
+      Keep =
+          IsWholeAccess && isa<StoreInst>(Dest) && DestInfo->Ty->isPointerTy();
+      break;
+    default:
+      break;
+    }
+    if (Keep)
+      Dest.setMetadata(Kind, MD);
+  }
+
+  // Adjust aliasing metadata.
+  AAMDNodes AA = Source.getAAMetadata();
+  if (!IsSubset) {
+    AA.TBAA = nullptr;
+    AA.TBAAStruct = nullptr;
+    AA.Scope = nullptr;
+    AA.NoAlias = nullptr;
+  } else if (!IsWholeAccess) {
+    if (DestByteOffset)
+      AA = AA.adjustForAccess(*DestByteOffset, DestInfo->Ty, DL);
+    else {
+      // A common tag for the whole source also describes any selected part.
+      if (!AA.TBAA && SourceInfo->Ty)
+        AA.TBAA = AA.adjustForAccess(0, SourceInfo->Ty, DL).TBAA;
+      AA.TBAAStruct = nullptr;
+    }
+  } else {
+    // Prefer !tbaa over !tbaa.struct.
+    AAMDNodes Adjusted = AA.adjustForAccess(0, DestInfo->Ty, DL);
+    if (!AA.TBAA && Adjusted.TBAA)
+      AA = Adjusted;
+  }
+  Dest.setAAMetadata(AA);
+  setMemCacheHint(Dest, DestInfo->PointerOperand,
+                  getMemCacheHint(Source, SourceInfo->PointerOperand));
+}
+
 /// Add all access groups in @p AccGroups to @p List.
 template <typename ListT>
 static void addToAccessGroupList(ListT &List, MDNode *AccGroups) {
diff --git a/llvm/unittests/Analysis/CMakeLists.txt b/llvm/unittests/Analysis/CMakeLists.txt
index 43ad72b7bfa6dd..0732edeff7c88e 100644
--- a/llvm/unittests/Analysis/CMakeLists.txt
+++ b/llvm/unittests/Analysis/CMakeLists.txt
@@ -44,6 +44,7 @@ set(ANALYSIS_TEST_SOURCES
   MemoryBuiltinsTest.cpp
   MemoryProfileInfoTest.cpp
   MemorySSATest.cpp
+  MetadataUtilsTest.cpp
   MLGOUtilsTest.cpp
   MLModelRunnerTest.cpp
   PhiValuesTest.cpp
diff --git a/llvm/unittests/Analysis/MetadataUtilsTest.cpp b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
new file mode 100644
index 00000000000000..e8ab8d7a9c7145
--- /dev/null
+++ b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
@@ -0,0 +1,504 @@
+//===- MetadataUtilsTest.cpp - Metadata tests --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/MetadataUtils.h"
+#include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/MDBuilder.h"
+#include "llvm/IR/MemCacheHint.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Verifier.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+namespace {
+class MetadataUtilsTest : public testing::Test {
+protected:
+  LLVMContext C;
+  Module M{"test", C};
+  IRBuilder<> B{M};
+  Value *P;
+  Value *Q;
+
+  MetadataUtilsTest() {
+    M.setDataLayout("e-p:64:64");
+    auto *F = Function::Create(
+        FunctionType::get(B.getVoidTy(), {B.getPtrTy(), B.getPtrTy()}, false),
+        Function::ExternalLinkage, "test", M);
+    P = F->getArg(0);
+    Q = F->getArg(1);
+    B.SetInsertPoint(BasicBlock::Create(C, "entry", F));
+    B.SetInsertPoint(B.CreateRetVoid());
+  }
+
+  void TearDown() override { EXPECT_FALSE(verifyModule(M, &errs())); }
+
+  MDNode *empty() { return MDNode::get(C, {}); }
+  Metadata *number(uint64_t N) {
+    return ConstantAsMetadata::get(B.getInt64(N));
+  }
+  MDNode *hint() {
+    return MDNode::get(
+        C, {MDString::get(C, "nvvm.l1_eviction"), MDString::get(C, "last")});
+  }
+  MDNode *range() {
+    return MDNode::get(C, {ConstantAsMetadata::get(B.getInt32(0)),
+                           ConstantAsMetadata::get(B.getInt32(100))});
+  }
+  MDNode *scope() {
+    MDBuilder MDB(C);
+    return MDNode::get(C, {MDB.createAnonymousAliasScope(
+                              MDB.createAnonymousAliasScopeDomain())});
+  }
+};
+
+TEST_F(MetadataUtilsTest, NarrowedLoadPreservesElementFacts) {
+  auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), P);
+  auto *Ptr = B.CreateInBoundsGEP(B.getInt32Ty(), P, B.getInt64(2));
+  auto *New = B.CreateLoad(B.getInt32Ty(), Ptr);
+  auto *Range = range();
+  auto *Hint = hint();
+  auto *Scope = scope();
+  Old->setMetadata(LLVMContext::MD_range, Range);
+  Old->setMetadata(LLVMContext::MD_noundef, empty());
+  Old->setMetadata(LLVMContext::MD_invariant_load, empty());
+  Old->setMetadata(LLVMContext::MD_invariant_group, empty());
+  Old->setMetadata(LLVMContext::MD_alias_scope, Scope);
+  setMemCacheHint(*Old, 0, Hint);
+  copyMetadataForMemoryAccess(*New, *Old, 8);
+  EXPECT_EQ(New->getMetadata(LLVMContext::MD_range), Range);
+  EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_noundef));
+  EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_invariant_load));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_invariant_group));
+  EXPECT_EQ(New->getMetadata(LLVMContext::MD_alias_scope), Scope);
+  EXPECT_EQ(getMemCacheHint(*New), Hint);
+}
+
+TEST_F(MetadataUtilsTest, InvariantGroupRequiresPointerIdentity) {
+  auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), P);
+  auto *Same = B.CreateLoad(B.getInt32Ty(), P);
+  auto *Different = B.CreateLoad(B.getInt32Ty(), Q);
+  Old->setMetadata(LLVMContext::MD_invariant_group, empty());
+  copyMetadataForMemoryAccess(*Same, *Old);
+  copyMetadataForMemoryAccess(*Different, *Old);
+  EXPECT_TRUE(Same->hasMetadata(LLVMContext::MD_invariant_group));
+  EXPECT_FALSE(Different->hasMetadata(LLVMContext::MD_invariant_group));
+}
+
+TEST_F(MetadataUtilsTest, IntegerFragmentPreservesDefinednessButNotRange) {
+  auto *Old = B.CreateLoad(B.getInt64Ty(), P);
+  MDBuilder MDB(C);
+  Old->setMetadata(LLVMContext::MD_noundef, empty());
+  Old->setMetadata(LLVMContext::MD_range,
+                   MDB.createRange(B.getInt64(0), B.getInt64(100)));
+  for (int64_t Offset : {0, 4}) {
+    auto *Ptr = B.CreateInBoundsGEP(B.getInt8Ty(), P, B.getInt64(Offset));
+    auto *New = B.CreateLoad(B.getInt32Ty(), Ptr);
+    copyMetadataForMemoryAccess(*New, *Old, Offset);
+    EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_noundef));
+    EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+  }
+}
+
+TEST_F(MetadataUtilsTest, WideningDropsFactsAboutOriginalBytes) {
+  auto *Old = B.CreateLoad(B.getInt32Ty(), P);
+  auto *New = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), P);
+  auto *Hint = hint();
+  auto *Annotation = MDNode::get(C, {MDString::get(C, "original")});
+  Old->setMetadata(LLVMContext::MD_range, range());
+  Old->setMetadata(LLVMContext::MD_noundef, empty());
+  Old->setMetadata(LLVMContext::MD_invariant_load, empty());
+  Old->setMetadata(LLVMContext::MD_invariant_group, empty());
+  Old->setMetadata(LLVMContext::MD_alias_scope, scope());
+  Old->setMetadata(LLVMContext::MD_annotation, Annotation);
+  Old->setMetadata("test.unknown", empty());
+  setMemCacheHint(*Old, 0, Hint);
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_noundef));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_invariant_load));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_invariant_group));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_alias_scope));
+  EXPECT_FALSE(New->getMetadata("test.unknown"));
+  EXPECT_EQ(New->getMetadata(LLVMContext::MD_annotation), Annotation);
+  EXPECT_EQ(getMemCacheHint(*New), Hint);
+}
+
+TEST_F(MetadataUtilsTest, EarlierAddressIsNotASubset) {
+  auto *Ptr = B.CreateInBoundsGEP(B.getInt32Ty(), P, B.getInt64(2));
+  auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), Ptr);
+  auto *New = B.CreateLoad(B.getInt32Ty(), P);
+  Old->setMetadata(LLVMContext::MD_range, range());
+  copyMetadataForMemoryAccess(*New, *Old, -8);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+}
+
+TEST_F(MetadataUtilsTest, BitcastDropsDifferentIntegerElementRange) {
+  auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 2), P);
+  auto *New = B.CreateLoad(B.getInt64Ty(), P);
+  Old->setMetadata(LLVMContext::MD_range, range());
+  Old->setMetadata(LLVMContext::MD_noundef, empty());
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+  EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_noundef));
+}
+
+TEST_F(MetadataUtilsTest, BitcastDropsFloatingPointValueFacts) {
+  auto *Old = B.CreateLoad(FixedVectorType::get(B.getFloatTy(), 2), P);
+  auto *New = B.CreateLoad(B.getInt64Ty(), P);
+  Old->setMetadata(LLVMContext::MD_nofpclass,
+                   MDNode::get(C, {ConstantAsMetadata::get(B.getInt32(3))}));
+  Old->setMetadata(LLVMContext::MD_fpmath,
+                   MDNode::get(C, {ConstantAsMetadata::get(
+                                      ConstantFP::get(B.getFloatTy(), 2.5))}));
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_nofpclass));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_fpmath));
+}
+
+TEST_F(MetadataUtilsTest, PointerNonnullBecomesIntegerRange) {
+  auto *Old = B.CreateLoad(B.getPtrTy(), P);
+  auto *New = B.CreateLoad(B.getInt64Ty(), P);
+  Old->setMetadata(LLVMContext::MD_nonnull, empty());
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_nonnull));
+  ASSERT_TRUE(New->hasMetadata(LLVMContext::MD_range));
+  auto *Range = New->getMetadata(LLVMContext::MD_range);
+  EXPECT_EQ(mdconst::extract<ConstantInt>(Range->getOperand(0))->getZExtValue(),
+            1u);
+  EXPECT_EQ(mdconst::extract<ConstantInt>(Range->getOperand(1))->getZExtValue(),
+            0u);
+}
+
+TEST_F(MetadataUtilsTest, UnknownOffsetKeepsAccessTagsButDropsFieldLayout) {
+  auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), P);
+  auto *Index = B.CreateAnd(B.CreateLoad(B.getInt64Ty(), Q), B.getInt64(3));
+  auto *Ptr = B.CreateInBoundsGEP(B.getInt32Ty(), P, Index);
+  auto *New = B.CreateLoad(B.getInt32Ty(), Ptr);
+  MDBuilder MDB(C);
+  auto *Char = MDB.createTBAAScalarTypeNode("char", MDB.createTBAARoot("root"));
+  auto *Tag = MDB.createTBAAStructTagNode(Char, Char, 0);
+  auto *Scope = scope();
+  Old->setMetadata(LLVMContext::MD_tbaa, Tag);
+  Old->setMetadata(LLVMContext::MD_tbaa_struct,
+                   MDNode::get(C, {number(0), number(4), Tag}));
+  Old->setMetadata(LLVMContext::MD_alias_scope, Scope);
+  copyMetadataForMemoryAccess(*New, *Old, std::nullopt);
+  EXPECT_EQ(New->getMetadata(LLVMContext::MD_tbaa), Tag);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_tbaa_struct));
+  EXPECT_EQ(New->getMetadata(LLVMContext::MD_alias_scope), Scope);
+}
+
+TEST_F(MetadataUtilsTest, ConstantOffsetSelectsFieldTag) {
+  auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 2), P);
+  auto *New = B.CreateLoad(B.getInt32Ty(), Q);
+  MDBuilder MDB(C);
+  auto *Root = MDB.createTBAARoot("root");
+  auto *Char = MDB.createTBAAScalarTypeNode("char", Root);
+  auto *Int = MDB.createTBAAScalarTypeNode("int", Char);
+  auto *CharTag = MDB.createTBAAStructTagNode(Char, Char, 0);
+  auto *IntTag = MDB.createTBAAStructTagNode(Int, Int, 0);
+  Old->setMetadata(LLVMContext::MD_tbaa_struct,
+                   MDNode::get(C, {number(0), number(4), CharTag, number(4),
+                                   number(4), IntTag}));
+  copyMetadataForMemoryAccess(*New, *Old, 4);
+  EXPECT_EQ(New->getMetadata(LLVMContext::MD_tbaa), IntTag);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_tbaa_struct));
+}
+
+TEST_F(MetadataUtilsTest, WholeAccessSelectsSingleFieldTag) {
+  auto *Old = B.CreateLoad(B.getInt64Ty(), P);
+  auto *New = B.CreateLoad(B.getDoubleTy(), P);
+  MDBuilder MDB(C);
+  auto *Char = MDB.createTBAAScalarTypeNode("char", MDB.createTBAARoot("root"));
+  auto *Tag = MDB.createTBAAStructTagNode(Char, Char, 0);
+  Old->setMetadata(LLVMContext::MD_tbaa_struct,
+                   MDNode::get(C, {number(0), number(8), Tag}));
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_EQ(New->getMetadata(LLVMContext::MD_tbaa), Tag);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_tbaa_struct));
+}
+
+TEST_F(MetadataUtilsTest, WholeAccessRetainsMultipleFieldTags) {
+  auto *Ty = FixedVectorType::get(B.getInt32Ty(), 2);
+  auto *Old = B.CreateLoad(Ty, P);
+  auto *New = B.CreateLoad(Ty, P);
+  MDBuilder MDB(C);
+  auto *Char = MDB.createTBAAScalarTypeNode("char", MDB.createTBAARoot("root"));
+  auto *Int = MDB.createTBAAScalarTypeNode("int", Char);
+  auto *CharTag = MDB.createTBAAStructTagNode(Char, Char, 0);
+  auto *IntTag = MDB.createTBAAStructTagNode(Int, Int, 0);
+  auto *Fields = MDNode::get(
+      C, {number(0), number(4), CharTag, number(4), number(4), IntTag});
+  Old->setMetadata(LLVMContext::MD_tbaa_struct, Fields);
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_tbaa));
+  EXPECT_EQ(New->getMetadata(LLVMContext::MD_tbaa_struct), Fields);
+}
+
+TEST_F(MetadataUtilsTest, MemoryTransferRemapsBothOperands) {
+  auto *Copy = B.CreateMemCpy(P, Align(4), Q, Align(4), B.getInt64(4));
+  auto *Read = B.CreateLoad(B.getInt32Ty(), Q);
+  auto *Write = B.CreateStore(B.getInt32(1), P);
+  auto *ReadHint = hint();
+  auto *WriteHint = MDNode::get(
+      C, {MDString::get(C, "nvvm.l1_eviction"), MDString::get(C, "first")});
+  setMemCacheHint(*Copy, 0, WriteHint);
+  setMemCacheHint(*Copy, 1, ReadHint);
+  copyMetadataForMemoryAccess(*Read, *Copy);
+  copyMetadataForMemoryAccess(*Write, *Copy);
+  EXPECT_EQ(getMemCacheHint(*Read, 0), ReadHint);
+  EXPECT_EQ(getMemCacheHint(*Write, 1), WriteHint);
+  EXPECT_FALSE(getMemCacheHint(*Read, 1));
+  EXPECT_FALSE(getMemCacheHint(*Write, 0));
+}
+
+TEST_F(MetadataUtilsTest, StoreTypeChangePreservesHints) {
+  auto *Old = B.CreateStore(B.getInt32(0), P);
+  auto *New = B.CreateStore(ConstantFP::get(B.getFloatTy(), 0), P);
+  auto *Hint = hint();
+  setMemCacheHint(*Old, 1, Hint);
+  Old->setMetadata(LLVMContext::MD_invariant_group, empty());
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_EQ(getMemCacheHint(*New, 1), Hint);
+  EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_invariant_group));
+}
+
+
+TEST_F(MetadataUtilsTest, CSEIntersectsCacheHints) {
+  auto *A = B.CreateLoad(B.getInt32Ty(), P);
+  auto *BLoad = B.CreateLoad(B.getInt32Ty(), P);
+  auto *Hint = hint();
+  setMemCacheHint(*A, 0, Hint);
+  setMemCacheHint(*BLoad, 0, Hint);
+  combineMetadataForCSE(A, BLoad, false);
+  EXPECT_EQ(getMemCacheHint(*A), Hint);
+  BLoad->setMetadata(LLVMContext::MD_mem_cache_hint, nullptr);
+  combineMetadataForCSE(A, BLoad, false);
+  EXPECT_FALSE(getMemCacheHint(*A));
+}
+
+TEST_F(MetadataUtilsTest, AddressSpaceCastChangesInvariantGroupIdentity) {
+  auto *Old = B.CreateLoad(B.getInt32Ty(), P);
+  auto *NewPtr = B.CreateAddrSpaceCast(P, B.getPtrTy(1));
+  auto *New = B.CreateLoad(B.getInt32Ty(), NewPtr);
+  Old->setMetadata(LLVMContext::MD_invariant_group, empty());
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_invariant_group));
+}
+
+TEST_F(MetadataUtilsTest, SingleElementAggregatePreservesDefinedness) {
+  Type *Types[] = {StructType::get(C, {B.getInt32Ty()}, /*isPacked=*/false),
+                   ArrayType::get(B.getInt32Ty(), 1)};
+  for (Type *Ty : Types) {
+    auto *Old = B.CreateLoad(Ty, P);
+    auto *New = B.CreateLoad(B.getInt32Ty(), P);
+    Old->setMetadata(LLVMContext::MD_noundef, empty());
+    copyMetadataForMemoryAccess(*New, *Old);
+    EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_noundef));
+  }
+}
+
+TEST_F(MetadataUtilsTest, SingleElementAggregatePreservesFloatingPointFacts) {
+  auto *FPMath = MDNode::get(
+      C, {ConstantAsMetadata::get(ConstantFP::get(B.getFloatTy(), 2.5))});
+  auto *NoFPClass = MDNode::get(C, {ConstantAsMetadata::get(B.getInt32(3))});
+  Type *Types[] = {StructType::get(C, {B.getFloatTy()}, /*isPacked=*/false),
+                   ArrayType::get(B.getFloatTy(), 1)};
+  for (Type *Ty : Types) {
+    auto *Old = B.CreateLoad(Ty, P);
+    auto *New = B.CreateLoad(B.getFloatTy(), P);
+    Old->setMetadata(LLVMContext::MD_noundef, empty());
+    Old->setMetadata(LLVMContext::MD_fpmath, FPMath);
+    Old->setMetadata(LLVMContext::MD_nofpclass, NoFPClass);
+    copyMetadataForMemoryAccess(*New, *Old);
+    EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_noundef));
+    EXPECT_EQ(New->getMetadata(LLVMContext::MD_fpmath), FPMath);
+    EXPECT_EQ(New->getMetadata(LLVMContext::MD_nofpclass), NoFPClass);
+  }
+}
+
+TEST_F(MetadataUtilsTest, ShortAccessSelectsCoveringFieldTag) {
+  auto *Old = B.CreateLoad(B.getInt64Ty(), P);
+  auto *New = B.CreateLoad(B.getInt32Ty(), P);
+  MDBuilder MDB(C);
+  auto *Int = MDB.createTBAAScalarTypeNode("int", MDB.createTBAARoot("root"));
+  auto *Tag = MDB.createTBAAStructTagNode(Int, Int, 0);
+  Old->setMetadata(LLVMContext::MD_tbaa_struct,
+                   MDNode::get(C, {number(0), number(8), Tag}));
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_EQ(New->getMetadata(LLVMContext::MD_tbaa), Tag);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_tbaa_struct));
+}
+
+TEST_F(MetadataUtilsTest, UnknownOffsetSelectsCommonFieldTag) {
+  auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 2), P);
+  auto *Index = B.CreateAnd(B.CreateLoad(B.getInt64Ty(), Q), B.getInt64(1));
+  auto *Ptr = B.CreateInBoundsGEP(B.getInt32Ty(), P, Index);
+  auto *New = B.CreateLoad(B.getInt32Ty(), Ptr);
+  MDBuilder MDB(C);
+  auto *Int = MDB.createTBAAScalarTypeNode("int", MDB.createTBAARoot("root"));
+  auto *Tag = MDB.createTBAAStructTagNode(Int, Int, 0);
+  Old->setMetadata(
+      LLVMContext::MD_tbaa_struct,
+      MDNode::get(C, {number(0), number(4), Tag, number(4), number(4), Tag}));
+  copyMetadataForMemoryAccess(*New, *Old, std::nullopt);
+  EXPECT_EQ(New->getMetadata(LLVMContext::MD_tbaa), Tag);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_tbaa_struct));
+}
+
+TEST_F(MetadataUtilsTest, DefinedAggregateDoesNotDefinePadding) {
+  auto *Ty = StructType::get(C, {B.getInt8Ty(), B.getInt32Ty()});
+  ASSERT_EQ(M.getDataLayout().getTypeStoreSize(Ty).getFixedValue(), 8u);
+  auto *Old = B.CreateLoad(Ty, P);
+  auto *New = B.CreateLoad(B.getInt64Ty(), P);
+  Old->setMetadata(LLVMContext::MD_noundef, empty());
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_noundef));
+}
+
+TEST_F(MetadataUtilsTest, DefinedIntegerDoesNotDefineSourcePadding) {
+  auto *Old = B.CreateLoad(B.getInt1Ty(), P);
+  auto *New = B.CreateLoad(B.getInt8Ty(), P);
+  Old->setMetadata(LLVMContext::MD_noundef, empty());
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_noundef));
+}
+
+TEST_F(MetadataUtilsTest, DestinationPaddingDoesNotInvalidateDefinedness) {
+  auto *Old = B.CreateLoad(B.getInt8Ty(), P);
+  auto *New = B.CreateLoad(B.getInt1Ty(), P);
+  Old->setMetadata(LLVMContext::MD_noundef, empty());
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_noundef));
+}
+
+TEST_F(MetadataUtilsTest, NonnullDoesNotConstrainTruncatedPointerBits) {
+  auto *Old = B.CreateLoad(B.getPtrTy(), P);
+  auto *New = B.CreateLoad(B.getIntNTy(63), P);
+  Old->setMetadata(LLVMContext::MD_nonnull, empty());
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+}
+
+TEST_F(MetadataUtilsTest, IntegerRangeBecomesPointerNonnull) {
+  auto *Old = B.CreateLoad(B.getInt64Ty(), P);
+  auto *New = B.CreateLoad(B.getPtrTy(), P);
+  MDBuilder MDB(C);
+  Old->setMetadata(LLVMContext::MD_range,
+                   MDB.createRange(B.getInt64(1), B.getInt64(100)));
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_nonnull));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+}
+
+TEST_F(MetadataUtilsTest, PointerFactsRequireSamePointerType) {
+  auto *Old = B.CreateLoad(B.getPtrTy(), P);
+  auto *New = B.CreateLoad(B.getPtrTy(1), P);
+  Old->setMetadata(LLVMContext::MD_nonnull, empty());
+  Old->setMetadata(LLVMContext::MD_align, MDNode::get(C, {number(16)}));
+  Old->setMetadata(LLVMContext::MD_dereferenceable,
+                   MDNode::get(C, {number(32)}));
+  Old->setMetadata(LLVMContext::MD_dereferenceable_or_null,
+                   MDNode::get(C, {number(32)}));
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_nonnull));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_align));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_dereferenceable));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_dereferenceable_or_null));
+}
+
+TEST_F(MetadataUtilsTest, PointerFactsRequirePointerResults) {
+  auto *Old = B.CreateLoad(B.getPtrTy(), P);
+  auto *Pointer = B.CreateLoad(B.getPtrTy(), P);
+  auto *Integer = B.CreateLoad(B.getInt64Ty(), P);
+  auto *Align = MDNode::get(C, {number(16)});
+  auto *Deref = MDNode::get(C, {number(32)});
+  Old->setMetadata(LLVMContext::MD_nonnull, empty());
+  Old->setMetadata(LLVMContext::MD_align, Align);
+  Old->setMetadata(LLVMContext::MD_dereferenceable, Deref);
+  Old->setMetadata(LLVMContext::MD_dereferenceable_or_null, Deref);
+  copyMetadataForMemoryAccess(*Pointer, *Old);
+  copyMetadataForMemoryAccess(*Integer, *Old);
+  EXPECT_TRUE(Pointer->hasMetadata(LLVMContext::MD_nonnull));
+  EXPECT_EQ(Pointer->getMetadata(LLVMContext::MD_align), Align);
+  EXPECT_EQ(Pointer->getMetadata(LLVMContext::MD_dereferenceable), Deref);
+  EXPECT_EQ(Pointer->getMetadata(LLVMContext::MD_dereferenceable_or_null),
+            Deref);
+  EXPECT_FALSE(Integer->hasMetadata(LLVMContext::MD_nonnull));
+  EXPECT_FALSE(Integer->hasMetadata(LLVMContext::MD_align));
+  EXPECT_FALSE(Integer->hasMetadata(LLVMContext::MD_dereferenceable));
+  EXPECT_FALSE(Integer->hasMetadata(LLVMContext::MD_dereferenceable_or_null));
+  EXPECT_TRUE(Integer->hasMetadata(LLVMContext::MD_range));
+}
+
+TEST_F(MetadataUtilsTest, StoreFactsRequireWholeAccessAndPointerValue) {
+  auto *Old = B.CreateStore(Q, P);
+  auto *Pointer = B.CreateStore(Q, P);
+  auto *Integer = B.CreateStore(B.CreatePtrToInt(Q, B.getInt64Ty()), P);
+  auto *Part = B.CreateStore(B.getInt32(0), P);
+  auto *AssignID = DIAssignID::getDistinct(C);
+  auto *Captures = MDNode::get(C, {MDString::get(C, "address")});
+  Old->setMetadata(LLVMContext::MD_DIAssignID, AssignID);
+  Old->setMetadata(LLVMContext::MD_captures, Captures);
+  copyMetadataForMemoryAccess(*Pointer, *Old);
+  copyMetadataForMemoryAccess(*Integer, *Old);
+  copyMetadataForMemoryAccess(*Part, *Old);
+  EXPECT_EQ(Pointer->getMetadata(LLVMContext::MD_DIAssignID), AssignID);
+  EXPECT_EQ(Pointer->getMetadata(LLVMContext::MD_captures), Captures);
+  EXPECT_EQ(Integer->getMetadata(LLVMContext::MD_DIAssignID), AssignID);
+  EXPECT_FALSE(Integer->hasMetadata(LLVMContext::MD_captures));
+  EXPECT_FALSE(Part->hasMetadata(LLVMContext::MD_DIAssignID));
+  EXPECT_FALSE(Part->hasMetadata(LLVMContext::MD_captures));
+}
+
+TEST_F(MetadataUtilsTest, AssignmentIDRequiresUnchangedStoreAddress) {
+  auto *Old = B.CreateStore(B.getInt32(1), P);
+  auto *Same = B.CreateStore(B.getInt32(1), P);
+  auto *Moved = B.CreateStore(B.getInt32(1), Q);
+  auto *AssignID = DIAssignID::getDistinct(C);
+  Old->setMetadata(LLVMContext::MD_DIAssignID, AssignID);
+  copyMetadataForMemoryAccess(*Same, *Old);
+  copyMetadataForMemoryAccess(*Moved, *Old);
+  EXPECT_EQ(Same->getMetadata(LLVMContext::MD_DIAssignID), AssignID);
+  EXPECT_FALSE(Moved->hasMetadata(LLVMContext::MD_DIAssignID));
+}
+
+TEST_F(MetadataUtilsTest, IntrinsicProfilesAndUnknownMetadataAreNotCopied) {
+  auto *Ty = FixedVectorType::get(B.getInt32Ty(), 2);
+  auto *Mask = B.CreateLoad(FixedVectorType::get(B.getInt1Ty(), 2), Q);
+  auto *Old = B.CreateMaskedLoad(Ty, P, Align(4), Mask);
+  auto *New = B.CreateLoad(Ty, P);
+  auto *Profile = MDNode::get(C, {MDString::get(C, "branch_weights"),
+                                  ConstantAsMetadata::get(B.getInt32(10))});
+  Old->setMetadata(LLVMContext::MD_prof, Profile);
+  Old->setMetadata("test.unknown", empty());
+  copyMetadataForMemoryAccess(*New, *Old);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_prof));
+  EXPECT_FALSE(New->getMetadata("test.unknown"));
+}
+
+TEST_F(MetadataUtilsTest, SmallerAccessOutsideSourceRangeIsNotASubset) {
+  auto *Old = B.CreateLoad(FixedVectorType::get(B.getInt32Ty(), 4), P);
+  auto *Ptr = B.CreateInBoundsGEP(B.getInt32Ty(), P, B.getInt64(4));
+  auto *New = B.CreateLoad(B.getInt32Ty(), Ptr);
+  Old->setMetadata(LLVMContext::MD_range, range());
+  Old->setMetadata(LLVMContext::MD_noundef, empty());
+  Old->setMetadata(LLVMContext::MD_invariant_load, empty());
+  Old->setMetadata(LLVMContext::MD_alias_scope, scope());
+  Old->setMetadata(LLVMContext::MD_noalias, scope());
+  copyMetadataForMemoryAccess(*New, *Old, 16);
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_range));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_noundef));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_invariant_load));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_alias_scope));
+  EXPECT_FALSE(New->hasMetadata(LLVMContext::MD_noalias));
+}
+
+} // namespace

>From 001f0850d292aae7299a1fe3b48f04a8f1dbea0a Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 18 Nov 2026 01:14:09 +0000
Subject: [PATCH 13/14] [MetadataUtils] Apply stack formatting cleanup

---
 llvm/unittests/Analysis/MetadataUtilsTest.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/llvm/unittests/Analysis/MetadataUtilsTest.cpp b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
index e8ab8d7a9c7145..58fbb4cbec982b 100644
--- a/llvm/unittests/Analysis/MetadataUtilsTest.cpp
+++ b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
@@ -270,7 +270,6 @@ TEST_F(MetadataUtilsTest, StoreTypeChangePreservesHints) {
   EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_invariant_group));
 }
 
-
 TEST_F(MetadataUtilsTest, CSEIntersectsCacheHints) {
   auto *A = B.CreateLoad(B.getInt32Ty(), P);
   auto *BLoad = B.CreateLoad(B.getInt32Ty(), P);

>From 1480df3e00ccdcb2d9a321a1119391a4a849ae66 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 18 Nov 2026 09:11:54 +0000
Subject: [PATCH 14/14] [Analysis] Preserve metadata utility declaration order

---
 llvm/include/llvm/Analysis/MetadataUtils.h | 28 +++++++++++-----------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/llvm/include/llvm/Analysis/MetadataUtils.h b/llvm/include/llvm/Analysis/MetadataUtils.h
index 94a0816bbc209c..b7b311f9b6993c 100644
--- a/llvm/include/llvm/Analysis/MetadataUtils.h
+++ b/llvm/include/llvm/Analysis/MetadataUtils.h
@@ -23,6 +23,20 @@ class Instruction;
 class MDNode;
 class Value;
 
+/// Combine the metadata of two instructions so that K can replace J. This
+/// specifically handles the case of CSE-like transformations. Some
+/// metadata can only be kept if K dominates J. For this to be correct,
+/// K cannot be hoisted.
+///
+/// Unknown metadata is removed.
+LLVM_ABI void combineMetadataForCSE(Instruction *K, const Instruction *J,
+                                    bool DoesKMove);
+
+/// Combine metadata of two instructions, where instruction J is a memory
+/// access that has been merged into K. This will intersect alias-analysis
+/// metadata, while preserving other known metadata.
+LLVM_ABI void combineAAMetadata(Instruction *K, const Instruction *J);
+
 /// Copy metadata valid for a replacement memory access.
 ///
 /// The destination must access the same bytes, a subset, or a wider byte range
@@ -56,20 +70,6 @@ LLVM_ABI void copyNonnullMetadata(const LoadInst &OldLI, MDNode *N,
 LLVM_ABI void copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI,
                                 MDNode *N, LoadInst &NewLI);
 
-/// Combine the metadata of two instructions so that K can replace J. This
-/// specifically handles the case of CSE-like transformations. Some
-/// metadata can only be kept if K dominates J. For this to be correct,
-/// K cannot be hoisted.
-///
-/// Unknown metadata is removed.
-LLVM_ABI void combineMetadataForCSE(Instruction *K, const Instruction *J,
-                                    bool DoesKMove);
-
-/// Combine metadata of two instructions, where instruction J is a memory
-/// access that has been merged into K. This will intersect alias-analysis
-/// metadata, while preserving other known metadata.
-LLVM_ABI void combineAAMetadata(Instruction *K, const Instruction *J);
-
 /// Compute the union of two access-group lists.
 ///
 /// If the list contains just one access group, it is returned directly. If the



More information about the llvm-branch-commits mailing list