[llvm-branch-commits] [llvm] [Scalarizer] Share metadata propagation for scalarized instructions (PR #243677)
Yonah Goldberg via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Nov 18 01:13:37 PST 2026
https://github.com/YonahGoldberg updated https://github.com/llvm/llvm-project/pull/243677
>From 7e641e2e63000e14aebe988dce99599acf015be9 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 18 Nov 2026 01:05:42 +0000
Subject: [PATCH 1/2] [Scalarizer] Add memory metadata tests
---
.../Scalarizer/basic-inseltpoison.ll | 6 +-
llvm/test/Transforms/Scalarizer/basic.ll | 6 +-
.../Scalarizer/memory-cache-hint.ll | 37 +++++
.../Transforms/Scalarizer/memory-metadata.ll | 156 ++++++++++++++++++
4 files changed, 199 insertions(+), 6 deletions(-)
create mode 100644 llvm/test/Transforms/Scalarizer/memory-cache-hint.ll
create mode 100644 llvm/test/Transforms/Scalarizer/memory-metadata.ll
diff --git a/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll b/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
index 0a3516c82ad10b..b7db48d9b9cbc7 100644
--- a/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
+++ b/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
@@ -202,7 +202,7 @@ define void @f3(ptr %src, ptr %dst) {
ret void
}
-; Check that !tbaa.struct information is preserved.
+; Check that !tbaa.struct fields become the correct per-lane !tbaa tags.
define void @f4(ptr %src, ptr %dst) {
; CHECK-LABEL: define void @f4(
; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]]) {
@@ -858,7 +858,7 @@ define <2 x i32> @f23_crash(<2 x i32> %srcvec, i32 %v1) {
!2 = !{!15, !15, i64 0}
!3 = !{ !3, !{!"llvm.loop.parallel_accesses", !13} }
!4 = !{ float 4.0 }
-!5 = !{ i64 0, i64 8, null }
+!5 = !{i64 0, i64 4, !1, i64 4, i64 4, !2, i64 8, i64 4, !1, i64 12, i64 4, !2}
!13 = distinct !{}
!14 = !{!"set1", !0}
!15 = !{!"set2", !0}
@@ -868,7 +868,7 @@ define <2 x i32> @f23_crash(<2 x i32> %srcvec, i32 %v1) {
; CHECK: [[META2]] = !{!"root"}
; CHECK: [[SET2_TBAA3]] = !{[[META4:![0-9]+]], [[META4]], i64 0}
; CHECK: [[META4]] = !{!"set2", [[META2]]}
-; CHECK: [[TBAA_STRUCT5]] = !{i64 0, i64 8, null}
+; CHECK: [[TBAA_STRUCT5]] = !{i64 0, i64 4, [[SET1_TBAA0]], i64 4, i64 4, [[SET2_TBAA3]], i64 8, i64 4, [[SET1_TBAA0]], i64 12, i64 4, [[SET2_TBAA3]]}
; CHECK: [[ACC_GRP6]] = distinct !{}
; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META8:![0-9]+]]}
; CHECK: [[META8]] = !{!"llvm.loop.parallel_accesses", [[ACC_GRP6]]}
diff --git a/llvm/test/Transforms/Scalarizer/basic.ll b/llvm/test/Transforms/Scalarizer/basic.ll
index ffde5600bbc9a7..f620445612f6dd 100644
--- a/llvm/test/Transforms/Scalarizer/basic.ll
+++ b/llvm/test/Transforms/Scalarizer/basic.ll
@@ -202,7 +202,7 @@ define void @f3(ptr %src, ptr %dst) {
ret void
}
-; Check that !tbaa.struct information is preserved.
+; Check that !tbaa.struct fields become the correct per-lane !tbaa tags.
define void @f4(ptr %src, ptr %dst) {
; CHECK-LABEL: define void @f4(
; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]]) {
@@ -909,7 +909,7 @@ define <2 x i8> @test_copy_trunc_flags(<2 x i32> %src) {
!2 = !{!15, !15, i64 0}
!3 = !{ !3, !{!"llvm.loop.parallel_accesses", !13} }
!4 = !{ float 4.0 }
-!5 = !{ i64 0, i64 8, null }
+!5 = !{i64 0, i64 4, !1, i64 4, i64 4, !2, i64 8, i64 4, !1, i64 12, i64 4, !2}
!13 = distinct !{}
!14 = !{!"set1", !0}
!15 = !{!"set2", !0}
@@ -919,7 +919,7 @@ define <2 x i8> @test_copy_trunc_flags(<2 x i32> %src) {
; CHECK: [[META2]] = !{!"root"}
; CHECK: [[SET2_TBAA3]] = !{[[META4:![0-9]+]], [[META4]], i64 0}
; CHECK: [[META4]] = !{!"set2", [[META2]]}
-; CHECK: [[TBAA_STRUCT5]] = !{i64 0, i64 8, null}
+; CHECK: [[TBAA_STRUCT5]] = !{i64 0, i64 4, [[SET1_TBAA0]], i64 4, i64 4, [[SET2_TBAA3]], i64 8, i64 4, [[SET1_TBAA0]], i64 12, i64 4, [[SET2_TBAA3]]}
; CHECK: [[ACC_GRP6]] = distinct !{}
; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META8:![0-9]+]]}
; CHECK: [[META8]] = !{!"llvm.loop.parallel_accesses", [[ACC_GRP6]]}
diff --git a/llvm/test/Transforms/Scalarizer/memory-cache-hint.ll b/llvm/test/Transforms/Scalarizer/memory-cache-hint.ll
new file mode 100644
index 00000000000000..63d6e6f63916b1
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/memory-cache-hint.ll
@@ -0,0 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes='scalarizer<load-store>,dce' < %s | FileCheck %s
+
+define <2 x i32> @split_load(ptr %p) {
+;
+; CHECK-LABEL: define <2 x i32> @split_load(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[V_I0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: [[P_I1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: [[V_I1:%.*]] = load i32, ptr [[P_I1]], align 4
+; CHECK-NEXT: [[V_UPTO0:%.*]] = insertelement <2 x i32> poison, i32 [[V_I0]], i64 0
+; CHECK-NEXT: [[V:%.*]] = insertelement <2 x i32> [[V_UPTO0]], i32 [[V_I1]], i64 1
+; CHECK-NEXT: ret <2 x i32> [[V]]
+;
+ %v = load <2 x i32>, ptr %p, align 8, !mem.cache_hint !0, !range !2
+ ret <2 x i32> %v
+}
+
+define void @split_store(ptr %p, <2 x i32> %v) {
+;
+; CHECK-LABEL: define void @split_store(
+; CHECK-SAME: ptr [[P:%.*]], <2 x i32> [[V:%.*]]) {
+; CHECK-NEXT: [[V_I0:%.*]] = extractelement <2 x i32> [[V]], i64 0
+; CHECK-NEXT: store i32 [[V_I0]], ptr [[P]], align 8
+; CHECK-NEXT: [[V_I1:%.*]] = extractelement <2 x i32> [[V]], i64 1
+; CHECK-NEXT: [[P_I1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 [[V_I1]], ptr [[P_I1]], align 4
+; CHECK-NEXT: ret void
+;
+ store <2 x i32> %v, ptr %p, align 8, !mem.cache_hint !3
+ ret void
+}
+
+!0 = !{i32 0, !1}
+!1 = !{!"nvvm.l1_eviction", !"last"}
+!2 = !{i32 0, i32 100}
+!3 = !{i32 1, !1}
diff --git a/llvm/test/Transforms/Scalarizer/memory-metadata.ll b/llvm/test/Transforms/Scalarizer/memory-metadata.ll
new file mode 100644
index 00000000000000..ba380395a7b647
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/memory-metadata.ll
@@ -0,0 +1,156 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes='scalarizer<load-store>,dce' < %s | FileCheck %s --implicit-check-not=!test.unknown
+; RUN: opt -S -passes='scalarizer<load-store;min-bits=64>,dce' < %s | FileCheck %s --check-prefix=PACKED --implicit-check-not=!test.unknown
+
+target datalayout = "e-p:64:64"
+
+; Exercise result facts and access annotations when splitting into scalars and
+; smaller vectors. Only the first fragment retains pointer identity.
+define <4 x i32> @integer_load(ptr %p) {
+;
+;
+; CHECK-LABEL: define <4 x i32> @integer_load(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[V_I0:%.*]] = load i32, ptr [[P]], align 16, !tbaa [[INT_TBAA0:![0-9]+]], !invariant.load [[META4:![0-9]+]], !alias.scope [[META5:![0-9]+]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8:![0-9]+]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[P_I1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: [[V_I1:%.*]] = load i32, ptr [[P_I1]], align 4, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[P_I2:%.*]] = getelementptr i32, ptr [[P]], i32 2
+; CHECK-NEXT: [[V_I2:%.*]] = load i32, ptr [[P_I2]], align 8, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[P_I3:%.*]] = getelementptr i32, ptr [[P]], i32 3
+; CHECK-NEXT: [[V_I3:%.*]] = load i32, ptr [[P_I3]], align 4, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_UPTO0:%.*]] = insertelement <4 x i32> poison, i32 [[V_I0]], i64 0
+; CHECK-NEXT: [[V_UPTO1:%.*]] = insertelement <4 x i32> [[V_UPTO0]], i32 [[V_I1]], i64 1
+; CHECK-NEXT: [[V_UPTO2:%.*]] = insertelement <4 x i32> [[V_UPTO1]], i32 [[V_I2]], i64 2
+; CHECK-NEXT: [[V:%.*]] = insertelement <4 x i32> [[V_UPTO2]], i32 [[V_I3]], i64 3
+; CHECK-NEXT: ret <4 x i32> [[V]]
+;
+; PACKED-LABEL: define <4 x i32> @integer_load(
+; PACKED-SAME: ptr [[P:%.*]]) {
+; PACKED-NEXT: [[V_I0:%.*]] = load <2 x i32>, ptr [[P]], align 16, !tbaa [[INT_TBAA0:![0-9]+]], !invariant.load [[META4:![0-9]+]], !alias.scope [[META5:![0-9]+]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8:![0-9]+]], !llvm.access.group [[META8]]
+; PACKED-NEXT: [[P_I1:%.*]] = getelementptr <2 x i32>, ptr [[P]], i32 1
+; PACKED-NEXT: [[V_I1:%.*]] = load <2 x i32>, ptr [[P_I1]], align 8, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[V_I0]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; PACKED-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[V_I1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; PACKED-NEXT: [[V:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; PACKED-NEXT: ret <4 x i32> [[V]]
+;
+ %v = load <4 x i32>, ptr %p, align 16, !tbaa !10, !range !9, !noundef !0, !invariant.load !0, !invariant.group !0, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+ ret <4 x i32> %v
+}
+
+define <4 x float> @floating_load(ptr %p) {
+;
+;
+; CHECK-LABEL: define <4 x float> @floating_load(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[V_I0:%.*]] = load float, ptr [[P]], align 16, !tbaa [[FLOAT_TBAA9:![0-9]+]], !fpmath [[META11:![0-9]+]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[P_I1:%.*]] = getelementptr float, ptr [[P]], i32 1
+; CHECK-NEXT: [[V_I1:%.*]] = load float, ptr [[P_I1]], align 4, !tbaa [[FLOAT_TBAA9]], !fpmath [[META11]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[P_I2:%.*]] = getelementptr float, ptr [[P]], i32 2
+; CHECK-NEXT: [[V_I2:%.*]] = load float, ptr [[P_I2]], align 8, !tbaa [[FLOAT_TBAA9]], !fpmath [[META11]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[P_I3:%.*]] = getelementptr float, ptr [[P]], i32 3
+; CHECK-NEXT: [[V_I3:%.*]] = load float, ptr [[P_I3]], align 4, !tbaa [[FLOAT_TBAA9]], !fpmath [[META11]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_UPTO0:%.*]] = insertelement <4 x float> poison, float [[V_I0]], i64 0
+; CHECK-NEXT: [[V_UPTO1:%.*]] = insertelement <4 x float> [[V_UPTO0]], float [[V_I1]], i64 1
+; CHECK-NEXT: [[V_UPTO2:%.*]] = insertelement <4 x float> [[V_UPTO1]], float [[V_I2]], i64 2
+; CHECK-NEXT: [[V:%.*]] = insertelement <4 x float> [[V_UPTO2]], float [[V_I3]], i64 3
+; CHECK-NEXT: ret <4 x float> [[V]]
+;
+; PACKED-LABEL: define <4 x float> @floating_load(
+; PACKED-SAME: ptr [[P:%.*]]) {
+; PACKED-NEXT: [[V_I0:%.*]] = load <2 x float>, ptr [[P]], align 16, !tbaa [[FLOAT_TBAA9:![0-9]+]], !fpmath [[META11:![0-9]+]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT: [[P_I1:%.*]] = getelementptr <2 x float>, ptr [[P]], i32 1
+; PACKED-NEXT: [[V_I1:%.*]] = load <2 x float>, ptr [[P_I1]], align 8, !tbaa [[FLOAT_TBAA9]], !fpmath [[META11]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT: [[TMP1:%.*]] = shufflevector <2 x float> [[V_I0]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; PACKED-NEXT: [[TMP2:%.*]] = shufflevector <2 x float> [[V_I1]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; PACKED-NEXT: [[V:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; PACKED-NEXT: ret <4 x float> [[V]]
+;
+ %v = load <4 x float>, ptr %p, align 16, !tbaa !20, !nofpclass !15, !fpmath !14, !noundef !0, !invariant.load !0, !invariant.group !0, !mem.cache_hint !16, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+ ret <4 x float> %v
+}
+
+define void @integer_store(ptr %p, <4 x i32> %v) {
+;
+;
+; CHECK-LABEL: define void @integer_store(
+; CHECK-SAME: ptr [[P:%.*]], <4 x i32> [[V:%.*]]) {
+; CHECK-NEXT: [[V_I0:%.*]] = extractelement <4 x i32> [[V]], i64 0
+; CHECK-NEXT: store i32 [[V_I0]], ptr [[P]], align 16, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_I1:%.*]] = extractelement <4 x i32> [[V]], i64 1
+; CHECK-NEXT: [[P_I1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 [[V_I1]], ptr [[P_I1]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_I2:%.*]] = extractelement <4 x i32> [[V]], i64 2
+; CHECK-NEXT: [[P_I2:%.*]] = getelementptr i32, ptr [[P]], i32 2
+; CHECK-NEXT: store i32 [[V_I2]], ptr [[P_I2]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_I3:%.*]] = extractelement <4 x i32> [[V]], i64 3
+; CHECK-NEXT: [[P_I3:%.*]] = getelementptr i32, ptr [[P]], i32 3
+; CHECK-NEXT: store i32 [[V_I3]], ptr [[P_I3]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: ret void
+;
+; PACKED-LABEL: define void @integer_store(
+; PACKED-SAME: ptr [[P:%.*]], <4 x i32> [[V:%.*]]) {
+; PACKED-NEXT: [[V_I0:%.*]] = shufflevector <4 x i32> [[V]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>
+; PACKED-NEXT: store <2 x i32> [[V_I0]], ptr [[P]], align 16, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT: [[V_I1:%.*]] = shufflevector <4 x i32> [[V]], <4 x i32> poison, <2 x i32> <i32 2, i32 3>
+; PACKED-NEXT: [[P_I1:%.*]] = getelementptr <2 x i32>, ptr [[P]], i32 1
+; PACKED-NEXT: store <2 x i32> [[V_I1]], ptr [[P_I1]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT: ret void
+;
+ store <4 x i32> %v, ptr %p, align 16, !tbaa !10, !invariant.group !0, !mem.cache_hint !18, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
+ ret void
+}
+
+!0 = !{}
+!1 = !{i32 1}
+!2 = !{!"preserved memory metadata"}
+!3 = !{!"test", !"relaxed"}
+!4 = !{i32 5, i32 6}
+!5 = distinct !{}
+!6 = !{!7}
+!7 = distinct !{!7, !8, !"scope"}
+!8 = distinct !{!8, i1 false, !"domain"}
+!9 = !{i32 0, i32 100}
+!10 = !{!11, !11, i64 0}
+!11 = !{!"int", !12, i64 0}
+!12 = !{!"omnipotent char", !13, i64 0}
+!13 = !{!"root"}
+!14 = !{float 2.5}
+!15 = !{i32 3}
+!16 = !{i32 0, !17}
+!17 = !{!"nvvm.l1_eviction", !"last"}
+!18 = !{i32 1, !17}
+!19 = !{i64 0, i64 4, !10, i64 4, i64 4, !20}
+!20 = !{!21, !21, i64 0}
+!21 = !{!"float", !12, i64 0}
+!22 = !{i32 0, !17, i32 1, !23}
+!23 = !{!"nvvm.l1_eviction", !"first"}
+!24 = !{!"branch_weights", i32 10}
+!25 = !{i64 8}
+;.
+; CHECK: [[INT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; CHECK: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
+; CHECK: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
+; CHECK: [[META3]] = !{!"root"}
+; CHECK: [[META4]] = !{}
+; CHECK: [[META5]] = !{[[META6:![0-9]+]]}
+; CHECK: [[META6]] = distinct !{[[META6]], [[META7:![0-9]+]], !"scope"}
+; CHECK: [[META7]] = distinct !{[[META7]], i1 false, !"domain"}
+; CHECK: [[META8]] = distinct !{}
+; CHECK: [[FLOAT_TBAA9]] = !{[[META10:![0-9]+]], [[META10]], i64 0}
+; CHECK: [[META10]] = !{!"float", [[META2]], i64 0}
+; CHECK: [[META11]] = !{float 2.500000e+00}
+;.
+; PACKED: [[INT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; PACKED: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
+; PACKED: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
+; PACKED: [[META3]] = !{!"root"}
+; PACKED: [[META4]] = !{}
+; PACKED: [[META5]] = !{[[META6:![0-9]+]]}
+; PACKED: [[META6]] = distinct !{[[META6]], [[META7:![0-9]+]], !"scope"}
+; PACKED: [[META7]] = distinct !{[[META7]], i1 false, !"domain"}
+; PACKED: [[META8]] = distinct !{}
+; PACKED: [[FLOAT_TBAA9]] = !{[[META10:![0-9]+]], [[META10]], i64 0}
+; PACKED: [[META10]] = !{!"float", [[META2]], i64 0}
+; PACKED: [[META11]] = !{float 2.500000e+00}
+;.
>From a03fb6c8312d1a288e6757a3a9376c480c85f0d4 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 18 Nov 2026 01:07:05 +0000
Subject: [PATCH 2/2] [Scalarizer] Share metadata propagation for scalarized
instructions
---
llvm/include/llvm/Analysis/MetadataUtils.h | 8 ++
llvm/lib/Analysis/MetadataUtils.cpp | 15 ++++
llvm/lib/Transforms/Scalar/Scalarizer.cpp | 31 ++-----
.../Scalarizer/basic-inseltpoison.ll | 51 ++++++-----
llvm/test/Transforms/Scalarizer/basic.ll | 51 ++++++-----
.../Scalarizer/memory-cache-hint.ll | 14 ++-
.../Transforms/Scalarizer/memory-metadata.ll | 86 +++++++++++--------
llvm/unittests/Analysis/MetadataUtilsTest.cpp | 12 +++
8 files changed, 154 insertions(+), 114 deletions(-)
diff --git a/llvm/include/llvm/Analysis/MetadataUtils.h b/llvm/include/llvm/Analysis/MetadataUtils.h
index 94a0816bbc209c..9c474d16a0b048 100644
--- a/llvm/include/llvm/Analysis/MetadataUtils.h
+++ b/llvm/include/llvm/Analysis/MetadataUtils.h
@@ -38,6 +38,14 @@ LLVM_ABI void
copyMetadataForMemoryAccess(Instruction &Dest, const Instruction &Source,
std::optional<int64_t> DestByteOffset = 0);
+/// Copy metadata valid after splitting a vector instruction into scalar or
+/// smaller vector parts.
+/// For memory accesses, DestByteOffset is the destination's byte offset within
+/// the source access.
+LLVM_ABI void copyMetadataForScalarizedInstruction(
+ Instruction &Dest, const Instruction &Source,
+ std::optional<int64_t> DestByteOffset = std::nullopt);
+
/// Copy the metadata from the source instruction to the destination (the
/// replacement for the source instruction).
LLVM_ABI void copyMetadataForLoad(LoadInst &Dest, const LoadInst &Source);
diff --git a/llvm/lib/Analysis/MetadataUtils.cpp b/llvm/lib/Analysis/MetadataUtils.cpp
index e748344330c134..4309bd65b01a98 100644
--- a/llvm/lib/Analysis/MetadataUtils.cpp
+++ b/llvm/lib/Analysis/MetadataUtils.cpp
@@ -285,6 +285,21 @@ void llvm::copyMetadataForMemoryAccess(Instruction &Dest,
getMemCacheHint(Source, SourceInfo->PointerOperand));
}
+void llvm::copyMetadataForScalarizedInstruction(
+ Instruction &Dest, const Instruction &Source,
+ std::optional<int64_t> DestByteOffset) {
+ if (auto DestInfo = getMemoryAccessInfo(Dest))
+ if (getMemoryAccessInfo(Source, DestInfo->IsRead)) {
+ copyMetadataForMemoryAccess(Dest, Source, DestByteOffset);
+ return;
+ }
+ if (Dest.getType()->getScalarType() == Source.getType()->getScalarType() &&
+ Dest.getType()->isFPOrFPVectorTy())
+ Dest.copyMetadata(Source, {LLVMContext::MD_fpmath});
+ if (!Dest.getDebugLoc())
+ Dest.setDebugLoc(Source.getDebugLoc());
+}
+
/// Add all access groups in @p AccGroups to @p List.
template <typename ListT>
static void addToAccessGroupList(ListT &List, MDNode *AccGroups) {
diff --git a/llvm/lib/Transforms/Scalar/Scalarizer.cpp b/llvm/lib/Transforms/Scalar/Scalarizer.cpp
index 5af07a4400aa98..0468786e635aef 100644
--- a/llvm/lib/Transforms/Scalar/Scalarizer.cpp
+++ b/llvm/lib/Transforms/Scalar/Scalarizer.cpp
@@ -18,6 +18,7 @@
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Twine.h"
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Argument.h"
@@ -310,7 +311,6 @@ class ScalarizerVisitor : public InstVisitor<ScalarizerVisitor, bool> {
Scatterer scatter(Instruction *Point, Value *V, const VectorSplit &VS);
void gather(Instruction *Op, const ValueVector &CV, const VectorSplit &VS);
void replaceUses(Instruction *Op, Value *CV);
- bool canTransferMetadata(unsigned Kind);
void transferMetadataAndIRFlags(Instruction *Op, const ValueVector &CV);
std::optional<VectorSplit> getVectorSplit(Type *Ty);
std::optional<VectorLayout> getVectorLayout(Type *Ty, Align Alignment,
@@ -543,33 +543,16 @@ void ScalarizerVisitor::replaceUses(Instruction *Op, Value *CV) {
}
}
-// Return true if it is safe to transfer the given metadata tag from
-// vector to scalar instructions.
-bool ScalarizerVisitor::canTransferMetadata(unsigned Tag) {
- return (Tag == LLVMContext::MD_tbaa
- || Tag == LLVMContext::MD_fpmath
- || Tag == LLVMContext::MD_tbaa_struct
- || Tag == LLVMContext::MD_invariant_load
- || Tag == LLVMContext::MD_alias_scope
- || Tag == LLVMContext::MD_noalias
- || Tag == LLVMContext::MD_mem_parallel_loop_access
- || Tag == LLVMContext::MD_access_group);
-}
-
-// Transfer metadata from Op to the instructions in CV if it is known
-// to be safe to do so.
+// Transfer metadata and flags to each scalarized fragment.
void ScalarizerVisitor::transferMetadataAndIRFlags(Instruction *Op,
const ValueVector &CV) {
- SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
- Op->getAllMetadataOtherThanDebugLoc(MDs);
+ uint64_t Offset = 0;
for (Value *V : CV) {
- if (Instruction *New = dyn_cast<Instruction>(V)) {
- for (const auto &MD : MDs)
- if (canTransferMetadata(MD.first))
- New->setMetadata(MD.first, MD.second);
+ if (auto *New = dyn_cast<Instruction>(V)) {
+ copyMetadataForScalarizedInstruction(*New, *Op, Offset);
New->copyIRFlags(Op);
- if (Op->getDebugLoc() && !New->getDebugLoc())
- New->setDebugLoc(Op->getDebugLoc());
+ if (isa<LoadInst, StoreInst>(New))
+ Offset += Op->getDataLayout().getTypeStoreSize(getLoadStoreType(New));
}
}
}
diff --git a/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll b/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
index b7db48d9b9cbc7..96ac70bdb37e39 100644
--- a/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
+++ b/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
@@ -209,21 +209,21 @@ define void @f4(ptr %src, ptr %dst) {
; CHECK-NEXT: [[DST_I1:%.*]] = getelementptr i32, ptr [[DST]], i32 1
; CHECK-NEXT: [[DST_I2:%.*]] = getelementptr i32, ptr [[DST]], i32 2
; CHECK-NEXT: [[DST_I3:%.*]] = getelementptr i32, ptr [[DST]], i32 3
-; CHECK-NEXT: [[VAL_I0:%.*]] = load i32, ptr [[SRC]], align 16, !tbaa.struct [[TBAA_STRUCT5:![0-9]+]]
+; CHECK-NEXT: [[VAL_I0:%.*]] = load i32, ptr [[SRC]], align 16, !tbaa [[SET1_TBAA0]]
; CHECK-NEXT: [[SRC_I1:%.*]] = getelementptr i32, ptr [[SRC]], i32 1
-; CHECK-NEXT: [[VAL_I1:%.*]] = load i32, ptr [[SRC_I1]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
+; CHECK-NEXT: [[VAL_I1:%.*]] = load i32, ptr [[SRC_I1]], align 4, !tbaa [[SET2_TBAA3]]
; CHECK-NEXT: [[SRC_I2:%.*]] = getelementptr i32, ptr [[SRC]], i32 2
-; CHECK-NEXT: [[VAL_I2:%.*]] = load i32, ptr [[SRC_I2]], align 8, !tbaa.struct [[TBAA_STRUCT5]]
+; CHECK-NEXT: [[VAL_I2:%.*]] = load i32, ptr [[SRC_I2]], align 8, !tbaa [[SET1_TBAA0]]
; CHECK-NEXT: [[SRC_I3:%.*]] = getelementptr i32, ptr [[SRC]], i32 3
-; CHECK-NEXT: [[VAL_I3:%.*]] = load i32, ptr [[SRC_I3]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
+; CHECK-NEXT: [[VAL_I3:%.*]] = load i32, ptr [[SRC_I3]], align 4, !tbaa [[SET2_TBAA3]]
; CHECK-NEXT: [[ADD_I0:%.*]] = add i32 [[VAL_I0]], [[VAL_I0]]
; CHECK-NEXT: [[ADD_I1:%.*]] = add i32 [[VAL_I1]], [[VAL_I1]]
; CHECK-NEXT: [[ADD_I2:%.*]] = add i32 [[VAL_I2]], [[VAL_I2]]
; CHECK-NEXT: [[ADD_I3:%.*]] = add i32 [[VAL_I3]], [[VAL_I3]]
-; CHECK-NEXT: store i32 [[ADD_I0]], ptr [[DST]], align 16, !tbaa.struct [[TBAA_STRUCT5]]
-; CHECK-NEXT: store i32 [[ADD_I1]], ptr [[DST_I1]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
-; CHECK-NEXT: store i32 [[ADD_I2]], ptr [[DST_I2]], align 8, !tbaa.struct [[TBAA_STRUCT5]]
-; CHECK-NEXT: store i32 [[ADD_I3]], ptr [[DST_I3]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
+; CHECK-NEXT: store i32 [[ADD_I0]], ptr [[DST]], align 16, !tbaa [[SET1_TBAA0]]
+; CHECK-NEXT: store i32 [[ADD_I1]], ptr [[DST_I1]], align 4, !tbaa [[SET2_TBAA3]]
+; CHECK-NEXT: store i32 [[ADD_I2]], ptr [[DST_I2]], align 8, !tbaa [[SET1_TBAA0]]
+; CHECK-NEXT: store i32 [[ADD_I3]], ptr [[DST_I3]], align 4, !tbaa [[SET2_TBAA3]]
; CHECK-NEXT: ret void
;
%val = load <4 x i32> , ptr %src, !tbaa.struct !5
@@ -248,21 +248,21 @@ define void @f5(i32 %count, ptr %src, ptr %dst) {
; CHECK-NEXT: [[THIS_DST_I1:%.*]] = getelementptr i32, ptr [[THIS_DST]], i32 1
; CHECK-NEXT: [[THIS_DST_I2:%.*]] = getelementptr i32, ptr [[THIS_DST]], i32 2
; CHECK-NEXT: [[THIS_DST_I3:%.*]] = getelementptr i32, ptr [[THIS_DST]], i32 3
-; CHECK-NEXT: [[VAL_I0:%.*]] = load i32, ptr [[THIS_SRC]], align 16, !llvm.access.group [[ACC_GRP6:![0-9]+]]
-; CHECK-NEXT: [[VAL_I1:%.*]] = load i32, ptr [[THIS_SRC_I1]], align 4, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT: [[VAL_I2:%.*]] = load i32, ptr [[THIS_SRC_I2]], align 8, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT: [[VAL_I3:%.*]] = load i32, ptr [[THIS_SRC_I3]], align 4, !llvm.access.group [[ACC_GRP6]]
+; CHECK-NEXT: [[VAL_I0:%.*]] = load i32, ptr [[THIS_SRC]], align 16, !llvm.access.group [[ACC_GRP5:![0-9]+]]
+; CHECK-NEXT: [[VAL_I1:%.*]] = load i32, ptr [[THIS_SRC_I1]], align 4, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT: [[VAL_I2:%.*]] = load i32, ptr [[THIS_SRC_I2]], align 8, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT: [[VAL_I3:%.*]] = load i32, ptr [[THIS_SRC_I3]], align 4, !llvm.access.group [[ACC_GRP5]]
; CHECK-NEXT: [[ADD_I0:%.*]] = add i32 [[VAL_I0]], [[VAL_I0]]
; CHECK-NEXT: [[ADD_I1:%.*]] = add i32 [[VAL_I1]], [[VAL_I1]]
; CHECK-NEXT: [[ADD_I2:%.*]] = add i32 [[VAL_I2]], [[VAL_I2]]
; CHECK-NEXT: [[ADD_I3:%.*]] = add i32 [[VAL_I3]], [[VAL_I3]]
-; CHECK-NEXT: store i32 [[ADD_I0]], ptr [[THIS_DST]], align 16, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT: store i32 [[ADD_I1]], ptr [[THIS_DST_I1]], align 4, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT: store i32 [[ADD_I2]], ptr [[THIS_DST_I2]], align 8, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT: store i32 [[ADD_I3]], ptr [[THIS_DST_I3]], align 4, !llvm.access.group [[ACC_GRP6]]
+; CHECK-NEXT: store i32 [[ADD_I0]], ptr [[THIS_DST]], align 16, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT: store i32 [[ADD_I1]], ptr [[THIS_DST_I1]], align 4, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT: store i32 [[ADD_I2]], ptr [[THIS_DST_I2]], align 8, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT: store i32 [[ADD_I3]], ptr [[THIS_DST_I3]], align 4, !llvm.access.group [[ACC_GRP5]]
; CHECK-NEXT: [[NEXT_INDEX]] = add i32 [[INDEX]], -1
; CHECK-NEXT: [[CONTINUE:%.*]] = icmp ne i32 [[NEXT_INDEX]], [[COUNT]]
-; CHECK-NEXT: br i1 [[CONTINUE]], label %[[LOOP]], label %[[END:.*]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[CONTINUE]], label %[[LOOP]], label %[[END:.*]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[END]]:
; CHECK-NEXT: ret void
;
@@ -289,13 +289,13 @@ define <4 x float> @f6(<4 x float> %x) {
; CHECK-LABEL: define <4 x float> @f6(
; CHECK-SAME: <4 x float> [[X:%.*]]) {
; CHECK-NEXT: [[X_I0:%.*]] = extractelement <4 x float> [[X]], i64 0
-; CHECK-NEXT: [[RES_I0:%.*]] = fadd float [[X_I0]], 1.000000e+00, !fpmath [[META9:![0-9]+]]
+; CHECK-NEXT: [[RES_I0:%.*]] = fadd float [[X_I0]], 1.000000e+00, !fpmath [[META8:![0-9]+]]
; CHECK-NEXT: [[X_I1:%.*]] = extractelement <4 x float> [[X]], i64 1
-; CHECK-NEXT: [[RES_I1:%.*]] = fadd float [[X_I1]], 2.000000e+00, !fpmath [[META9]]
+; CHECK-NEXT: [[RES_I1:%.*]] = fadd float [[X_I1]], 2.000000e+00, !fpmath [[META8]]
; CHECK-NEXT: [[X_I2:%.*]] = extractelement <4 x float> [[X]], i64 2
-; CHECK-NEXT: [[RES_I2:%.*]] = fadd float [[X_I2]], 3.000000e+00, !fpmath [[META9]]
+; CHECK-NEXT: [[RES_I2:%.*]] = fadd float [[X_I2]], 3.000000e+00, !fpmath [[META8]]
; CHECK-NEXT: [[X_I3:%.*]] = extractelement <4 x float> [[X]], i64 3
-; CHECK-NEXT: [[RES_I3:%.*]] = fadd float [[X_I3]], 4.000000e+00, !fpmath [[META9]]
+; CHECK-NEXT: [[RES_I3:%.*]] = fadd float [[X_I3]], 4.000000e+00, !fpmath [[META8]]
; CHECK-NEXT: [[RES_UPTO0:%.*]] = insertelement <4 x float> poison, float [[RES_I0]], i64 0
; CHECK-NEXT: [[RES_UPTO1:%.*]] = insertelement <4 x float> [[RES_UPTO0]], float [[RES_I1]], i64 1
; CHECK-NEXT: [[RES_UPTO2:%.*]] = insertelement <4 x float> [[RES_UPTO1]], float [[RES_I2]], i64 2
@@ -868,9 +868,8 @@ define <2 x i32> @f23_crash(<2 x i32> %srcvec, i32 %v1) {
; CHECK: [[META2]] = !{!"root"}
; CHECK: [[SET2_TBAA3]] = !{[[META4:![0-9]+]], [[META4]], i64 0}
; CHECK: [[META4]] = !{!"set2", [[META2]]}
-; CHECK: [[TBAA_STRUCT5]] = !{i64 0, i64 4, [[SET1_TBAA0]], i64 4, i64 4, [[SET2_TBAA3]], i64 8, i64 4, [[SET1_TBAA0]], i64 12, i64 4, [[SET2_TBAA3]]}
-; CHECK: [[ACC_GRP6]] = distinct !{}
-; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META8:![0-9]+]]}
-; CHECK: [[META8]] = !{!"llvm.loop.parallel_accesses", [[ACC_GRP6]]}
-; CHECK: [[META9]] = !{float 4.000000e+00}
+; CHECK: [[ACC_GRP5]] = distinct !{}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META7:![0-9]+]]}
+; CHECK: [[META7]] = !{!"llvm.loop.parallel_accesses", [[ACC_GRP5]]}
+; CHECK: [[META8]] = !{float 4.000000e+00}
;.
diff --git a/llvm/test/Transforms/Scalarizer/basic.ll b/llvm/test/Transforms/Scalarizer/basic.ll
index f620445612f6dd..0d55e3edbedae6 100644
--- a/llvm/test/Transforms/Scalarizer/basic.ll
+++ b/llvm/test/Transforms/Scalarizer/basic.ll
@@ -209,21 +209,21 @@ define void @f4(ptr %src, ptr %dst) {
; CHECK-NEXT: [[DST_I1:%.*]] = getelementptr i32, ptr [[DST]], i32 1
; CHECK-NEXT: [[DST_I2:%.*]] = getelementptr i32, ptr [[DST]], i32 2
; CHECK-NEXT: [[DST_I3:%.*]] = getelementptr i32, ptr [[DST]], i32 3
-; CHECK-NEXT: [[VAL_I0:%.*]] = load i32, ptr [[SRC]], align 16, !tbaa.struct [[TBAA_STRUCT5:![0-9]+]]
+; CHECK-NEXT: [[VAL_I0:%.*]] = load i32, ptr [[SRC]], align 16, !tbaa [[SET1_TBAA0]]
; CHECK-NEXT: [[SRC_I1:%.*]] = getelementptr i32, ptr [[SRC]], i32 1
-; CHECK-NEXT: [[VAL_I1:%.*]] = load i32, ptr [[SRC_I1]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
+; CHECK-NEXT: [[VAL_I1:%.*]] = load i32, ptr [[SRC_I1]], align 4, !tbaa [[SET2_TBAA3]]
; CHECK-NEXT: [[SRC_I2:%.*]] = getelementptr i32, ptr [[SRC]], i32 2
-; CHECK-NEXT: [[VAL_I2:%.*]] = load i32, ptr [[SRC_I2]], align 8, !tbaa.struct [[TBAA_STRUCT5]]
+; CHECK-NEXT: [[VAL_I2:%.*]] = load i32, ptr [[SRC_I2]], align 8, !tbaa [[SET1_TBAA0]]
; CHECK-NEXT: [[SRC_I3:%.*]] = getelementptr i32, ptr [[SRC]], i32 3
-; CHECK-NEXT: [[VAL_I3:%.*]] = load i32, ptr [[SRC_I3]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
+; CHECK-NEXT: [[VAL_I3:%.*]] = load i32, ptr [[SRC_I3]], align 4, !tbaa [[SET2_TBAA3]]
; CHECK-NEXT: [[ADD_I0:%.*]] = add i32 [[VAL_I0]], [[VAL_I0]]
; CHECK-NEXT: [[ADD_I1:%.*]] = add i32 [[VAL_I1]], [[VAL_I1]]
; CHECK-NEXT: [[ADD_I2:%.*]] = add i32 [[VAL_I2]], [[VAL_I2]]
; CHECK-NEXT: [[ADD_I3:%.*]] = add i32 [[VAL_I3]], [[VAL_I3]]
-; CHECK-NEXT: store i32 [[ADD_I0]], ptr [[DST]], align 16, !tbaa.struct [[TBAA_STRUCT5]]
-; CHECK-NEXT: store i32 [[ADD_I1]], ptr [[DST_I1]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
-; CHECK-NEXT: store i32 [[ADD_I2]], ptr [[DST_I2]], align 8, !tbaa.struct [[TBAA_STRUCT5]]
-; CHECK-NEXT: store i32 [[ADD_I3]], ptr [[DST_I3]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
+; CHECK-NEXT: store i32 [[ADD_I0]], ptr [[DST]], align 16, !tbaa [[SET1_TBAA0]]
+; CHECK-NEXT: store i32 [[ADD_I1]], ptr [[DST_I1]], align 4, !tbaa [[SET2_TBAA3]]
+; CHECK-NEXT: store i32 [[ADD_I2]], ptr [[DST_I2]], align 8, !tbaa [[SET1_TBAA0]]
+; CHECK-NEXT: store i32 [[ADD_I3]], ptr [[DST_I3]], align 4, !tbaa [[SET2_TBAA3]]
; CHECK-NEXT: ret void
;
%val = load <4 x i32> , ptr %src, !tbaa.struct !5
@@ -248,21 +248,21 @@ define void @f5(i32 %count, ptr %src, ptr %dst) {
; CHECK-NEXT: [[THIS_DST_I1:%.*]] = getelementptr i32, ptr [[THIS_DST]], i32 1
; CHECK-NEXT: [[THIS_DST_I2:%.*]] = getelementptr i32, ptr [[THIS_DST]], i32 2
; CHECK-NEXT: [[THIS_DST_I3:%.*]] = getelementptr i32, ptr [[THIS_DST]], i32 3
-; CHECK-NEXT: [[VAL_I0:%.*]] = load i32, ptr [[THIS_SRC]], align 16, !llvm.access.group [[ACC_GRP6:![0-9]+]]
-; CHECK-NEXT: [[VAL_I1:%.*]] = load i32, ptr [[THIS_SRC_I1]], align 4, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT: [[VAL_I2:%.*]] = load i32, ptr [[THIS_SRC_I2]], align 8, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT: [[VAL_I3:%.*]] = load i32, ptr [[THIS_SRC_I3]], align 4, !llvm.access.group [[ACC_GRP6]]
+; CHECK-NEXT: [[VAL_I0:%.*]] = load i32, ptr [[THIS_SRC]], align 16, !llvm.access.group [[ACC_GRP5:![0-9]+]]
+; CHECK-NEXT: [[VAL_I1:%.*]] = load i32, ptr [[THIS_SRC_I1]], align 4, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT: [[VAL_I2:%.*]] = load i32, ptr [[THIS_SRC_I2]], align 8, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT: [[VAL_I3:%.*]] = load i32, ptr [[THIS_SRC_I3]], align 4, !llvm.access.group [[ACC_GRP5]]
; CHECK-NEXT: [[ADD_I0:%.*]] = add i32 [[VAL_I0]], [[VAL_I0]]
; CHECK-NEXT: [[ADD_I1:%.*]] = add i32 [[VAL_I1]], [[VAL_I1]]
; CHECK-NEXT: [[ADD_I2:%.*]] = add i32 [[VAL_I2]], [[VAL_I2]]
; CHECK-NEXT: [[ADD_I3:%.*]] = add i32 [[VAL_I3]], [[VAL_I3]]
-; CHECK-NEXT: store i32 [[ADD_I0]], ptr [[THIS_DST]], align 16, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT: store i32 [[ADD_I1]], ptr [[THIS_DST_I1]], align 4, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT: store i32 [[ADD_I2]], ptr [[THIS_DST_I2]], align 8, !llvm.access.group [[ACC_GRP6]]
-; CHECK-NEXT: store i32 [[ADD_I3]], ptr [[THIS_DST_I3]], align 4, !llvm.access.group [[ACC_GRP6]]
+; CHECK-NEXT: store i32 [[ADD_I0]], ptr [[THIS_DST]], align 16, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT: store i32 [[ADD_I1]], ptr [[THIS_DST_I1]], align 4, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT: store i32 [[ADD_I2]], ptr [[THIS_DST_I2]], align 8, !llvm.access.group [[ACC_GRP5]]
+; CHECK-NEXT: store i32 [[ADD_I3]], ptr [[THIS_DST_I3]], align 4, !llvm.access.group [[ACC_GRP5]]
; CHECK-NEXT: [[NEXT_INDEX]] = add i32 [[INDEX]], -1
; CHECK-NEXT: [[CONTINUE:%.*]] = icmp ne i32 [[NEXT_INDEX]], [[COUNT]]
-; CHECK-NEXT: br i1 [[CONTINUE]], label %[[LOOP]], label %[[END:.*]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[CONTINUE]], label %[[LOOP]], label %[[END:.*]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[END]]:
; CHECK-NEXT: ret void
;
@@ -289,13 +289,13 @@ define <4 x float> @f6(<4 x float> %x) {
; CHECK-LABEL: define <4 x float> @f6(
; CHECK-SAME: <4 x float> [[X:%.*]]) {
; CHECK-NEXT: [[X_I0:%.*]] = extractelement <4 x float> [[X]], i64 0
-; CHECK-NEXT: [[RES_I0:%.*]] = fadd float [[X_I0]], 1.000000e+00, !fpmath [[META9:![0-9]+]]
+; CHECK-NEXT: [[RES_I0:%.*]] = fadd float [[X_I0]], 1.000000e+00, !fpmath [[META8:![0-9]+]]
; CHECK-NEXT: [[X_I1:%.*]] = extractelement <4 x float> [[X]], i64 1
-; CHECK-NEXT: [[RES_I1:%.*]] = fadd float [[X_I1]], 2.000000e+00, !fpmath [[META9]]
+; CHECK-NEXT: [[RES_I1:%.*]] = fadd float [[X_I1]], 2.000000e+00, !fpmath [[META8]]
; CHECK-NEXT: [[X_I2:%.*]] = extractelement <4 x float> [[X]], i64 2
-; CHECK-NEXT: [[RES_I2:%.*]] = fadd float [[X_I2]], 3.000000e+00, !fpmath [[META9]]
+; CHECK-NEXT: [[RES_I2:%.*]] = fadd float [[X_I2]], 3.000000e+00, !fpmath [[META8]]
; CHECK-NEXT: [[X_I3:%.*]] = extractelement <4 x float> [[X]], i64 3
-; CHECK-NEXT: [[RES_I3:%.*]] = fadd float [[X_I3]], 4.000000e+00, !fpmath [[META9]]
+; CHECK-NEXT: [[RES_I3:%.*]] = fadd float [[X_I3]], 4.000000e+00, !fpmath [[META8]]
; CHECK-NEXT: [[RES_UPTO0:%.*]] = insertelement <4 x float> poison, float [[RES_I0]], i64 0
; CHECK-NEXT: [[RES_UPTO1:%.*]] = insertelement <4 x float> [[RES_UPTO0]], float [[RES_I1]], i64 1
; CHECK-NEXT: [[RES_UPTO2:%.*]] = insertelement <4 x float> [[RES_UPTO1]], float [[RES_I2]], i64 2
@@ -919,9 +919,8 @@ define <2 x i8> @test_copy_trunc_flags(<2 x i32> %src) {
; CHECK: [[META2]] = !{!"root"}
; CHECK: [[SET2_TBAA3]] = !{[[META4:![0-9]+]], [[META4]], i64 0}
; CHECK: [[META4]] = !{!"set2", [[META2]]}
-; CHECK: [[TBAA_STRUCT5]] = !{i64 0, i64 4, [[SET1_TBAA0]], i64 4, i64 4, [[SET2_TBAA3]], i64 8, i64 4, [[SET1_TBAA0]], i64 12, i64 4, [[SET2_TBAA3]]}
-; CHECK: [[ACC_GRP6]] = distinct !{}
-; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META8:![0-9]+]]}
-; CHECK: [[META8]] = !{!"llvm.loop.parallel_accesses", [[ACC_GRP6]]}
-; CHECK: [[META9]] = !{float 4.000000e+00}
+; CHECK: [[ACC_GRP5]] = distinct !{}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META7:![0-9]+]]}
+; CHECK: [[META7]] = !{!"llvm.loop.parallel_accesses", [[ACC_GRP5]]}
+; CHECK: [[META8]] = !{float 4.000000e+00}
;.
diff --git a/llvm/test/Transforms/Scalarizer/memory-cache-hint.ll b/llvm/test/Transforms/Scalarizer/memory-cache-hint.ll
index 63d6e6f63916b1..22efbd8e83bdb1 100644
--- a/llvm/test/Transforms/Scalarizer/memory-cache-hint.ll
+++ b/llvm/test/Transforms/Scalarizer/memory-cache-hint.ll
@@ -5,9 +5,9 @@ define <2 x i32> @split_load(ptr %p) {
;
; CHECK-LABEL: define <2 x i32> @split_load(
; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[V_I0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: [[V_I0:%.*]] = load i32, ptr [[P]], align 8, !range [[RNG0:![0-9]+]], !mem.cache_hint [[META1:![0-9]+]]
; CHECK-NEXT: [[P_I1:%.*]] = getelementptr i32, ptr [[P]], i32 1
-; CHECK-NEXT: [[V_I1:%.*]] = load i32, ptr [[P_I1]], align 4
+; CHECK-NEXT: [[V_I1:%.*]] = load i32, ptr [[P_I1]], align 4, !range [[RNG0]], !mem.cache_hint [[META1]]
; CHECK-NEXT: [[V_UPTO0:%.*]] = insertelement <2 x i32> poison, i32 [[V_I0]], i64 0
; CHECK-NEXT: [[V:%.*]] = insertelement <2 x i32> [[V_UPTO0]], i32 [[V_I1]], i64 1
; CHECK-NEXT: ret <2 x i32> [[V]]
@@ -21,10 +21,10 @@ define void @split_store(ptr %p, <2 x i32> %v) {
; CHECK-LABEL: define void @split_store(
; CHECK-SAME: ptr [[P:%.*]], <2 x i32> [[V:%.*]]) {
; CHECK-NEXT: [[V_I0:%.*]] = extractelement <2 x i32> [[V]], i64 0
-; CHECK-NEXT: store i32 [[V_I0]], ptr [[P]], align 8
+; CHECK-NEXT: store i32 [[V_I0]], ptr [[P]], align 8, !mem.cache_hint [[META3:![0-9]+]]
; CHECK-NEXT: [[V_I1:%.*]] = extractelement <2 x i32> [[V]], i64 1
; CHECK-NEXT: [[P_I1:%.*]] = getelementptr i32, ptr [[P]], i32 1
-; CHECK-NEXT: store i32 [[V_I1]], ptr [[P_I1]], align 4
+; CHECK-NEXT: store i32 [[V_I1]], ptr [[P_I1]], align 4, !mem.cache_hint [[META3]]
; CHECK-NEXT: ret void
;
store <2 x i32> %v, ptr %p, align 8, !mem.cache_hint !3
@@ -35,3 +35,9 @@ define void @split_store(ptr %p, <2 x i32> %v) {
!1 = !{!"nvvm.l1_eviction", !"last"}
!2 = !{i32 0, i32 100}
!3 = !{i32 1, !1}
+;.
+; CHECK: [[RNG0]] = !{i32 0, i32 100}
+; CHECK: [[META1]] = !{i32 0, [[META2:![0-9]+]]}
+; CHECK: [[META2]] = !{!"nvvm.l1_eviction", !"last"}
+; CHECK: [[META3]] = !{i32 1, [[META2]]}
+;.
diff --git a/llvm/test/Transforms/Scalarizer/memory-metadata.ll b/llvm/test/Transforms/Scalarizer/memory-metadata.ll
index ba380395a7b647..b15e8420ba6fc7 100644
--- a/llvm/test/Transforms/Scalarizer/memory-metadata.ll
+++ b/llvm/test/Transforms/Scalarizer/memory-metadata.ll
@@ -11,13 +11,13 @@ define <4 x i32> @integer_load(ptr %p) {
;
; CHECK-LABEL: define <4 x i32> @integer_load(
; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[V_I0:%.*]] = load i32, ptr [[P]], align 16, !tbaa [[INT_TBAA0:![0-9]+]], !invariant.load [[META4:![0-9]+]], !alias.scope [[META5:![0-9]+]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8:![0-9]+]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_I0:%.*]] = load i32, ptr [[P]], align 16, !tbaa [[INT_TBAA0:![0-9]+]], !range [[RNG4:![0-9]+]], !invariant.load [[META5:![0-9]+]], !alias.scope [[META6:![0-9]+]], !noalias [[META6]], !nontemporal [[META9:![0-9]+]], !llvm.mem.parallel_loop_access [[META10:![0-9]+]], !invariant.group [[META5]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11:![0-9]+]], !nosanitize [[META5]], !mmra [[META12:![0-9]+]], !noalias.addrspace [[META13:![0-9]+]], !mem.cache_hint [[META14:![0-9]+]]
; CHECK-NEXT: [[P_I1:%.*]] = getelementptr i32, ptr [[P]], i32 1
-; CHECK-NEXT: [[V_I1:%.*]] = load i32, ptr [[P_I1]], align 4, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_I1:%.*]] = load i32, ptr [[P_I1]], align 4, !tbaa [[INT_TBAA0]], !range [[RNG4]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META14]]
; CHECK-NEXT: [[P_I2:%.*]] = getelementptr i32, ptr [[P]], i32 2
-; CHECK-NEXT: [[V_I2:%.*]] = load i32, ptr [[P_I2]], align 8, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_I2:%.*]] = load i32, ptr [[P_I2]], align 8, !tbaa [[INT_TBAA0]], !range [[RNG4]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META14]]
; CHECK-NEXT: [[P_I3:%.*]] = getelementptr i32, ptr [[P]], i32 3
-; CHECK-NEXT: [[V_I3:%.*]] = load i32, ptr [[P_I3]], align 4, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_I3:%.*]] = load i32, ptr [[P_I3]], align 4, !tbaa [[INT_TBAA0]], !range [[RNG4]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META14]]
; CHECK-NEXT: [[V_UPTO0:%.*]] = insertelement <4 x i32> poison, i32 [[V_I0]], i64 0
; CHECK-NEXT: [[V_UPTO1:%.*]] = insertelement <4 x i32> [[V_UPTO0]], i32 [[V_I1]], i64 1
; CHECK-NEXT: [[V_UPTO2:%.*]] = insertelement <4 x i32> [[V_UPTO1]], i32 [[V_I2]], i64 2
@@ -26,9 +26,9 @@ define <4 x i32> @integer_load(ptr %p) {
;
; PACKED-LABEL: define <4 x i32> @integer_load(
; PACKED-SAME: ptr [[P:%.*]]) {
-; PACKED-NEXT: [[V_I0:%.*]] = load <2 x i32>, ptr [[P]], align 16, !tbaa [[INT_TBAA0:![0-9]+]], !invariant.load [[META4:![0-9]+]], !alias.scope [[META5:![0-9]+]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8:![0-9]+]], !llvm.access.group [[META8]]
+; PACKED-NEXT: [[V_I0:%.*]] = load <2 x i32>, ptr [[P]], align 16, !tbaa [[INT_TBAA0:![0-9]+]], !range [[RNG4:![0-9]+]], !invariant.load [[META5:![0-9]+]], !alias.scope [[META6:![0-9]+]], !noalias [[META6]], !nontemporal [[META9:![0-9]+]], !llvm.mem.parallel_loop_access [[META10:![0-9]+]], !invariant.group [[META5]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11:![0-9]+]], !nosanitize [[META5]], !mmra [[META12:![0-9]+]], !noalias.addrspace [[META13:![0-9]+]], !mem.cache_hint [[META14:![0-9]+]]
; PACKED-NEXT: [[P_I1:%.*]] = getelementptr <2 x i32>, ptr [[P]], i32 1
-; PACKED-NEXT: [[V_I1:%.*]] = load <2 x i32>, ptr [[P_I1]], align 8, !tbaa [[INT_TBAA0]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT: [[V_I1:%.*]] = load <2 x i32>, ptr [[P_I1]], align 8, !tbaa [[INT_TBAA0]], !range [[RNG4]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META14]]
; PACKED-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[V_I0]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; PACKED-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[V_I1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; PACKED-NEXT: [[V:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
@@ -43,13 +43,13 @@ define <4 x float> @floating_load(ptr %p) {
;
; CHECK-LABEL: define <4 x float> @floating_load(
; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[V_I0:%.*]] = load float, ptr [[P]], align 16, !tbaa [[FLOAT_TBAA9:![0-9]+]], !fpmath [[META11:![0-9]+]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_I0:%.*]] = load float, ptr [[P]], align 16, !tbaa [[FLOAT_TBAA16:![0-9]+]], !fpmath [[META18:![0-9]+]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !invariant.group [[META5]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !nofpclass [[META19:![0-9]+]], !mem.cache_hint [[META14]]
; CHECK-NEXT: [[P_I1:%.*]] = getelementptr float, ptr [[P]], i32 1
-; CHECK-NEXT: [[V_I1:%.*]] = load float, ptr [[P_I1]], align 4, !tbaa [[FLOAT_TBAA9]], !fpmath [[META11]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_I1:%.*]] = load float, ptr [[P_I1]], align 4, !tbaa [[FLOAT_TBAA16]], !fpmath [[META18]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !nofpclass [[META19]], !mem.cache_hint [[META14]]
; CHECK-NEXT: [[P_I2:%.*]] = getelementptr float, ptr [[P]], i32 2
-; CHECK-NEXT: [[V_I2:%.*]] = load float, ptr [[P_I2]], align 8, !tbaa [[FLOAT_TBAA9]], !fpmath [[META11]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_I2:%.*]] = load float, ptr [[P_I2]], align 8, !tbaa [[FLOAT_TBAA16]], !fpmath [[META18]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !nofpclass [[META19]], !mem.cache_hint [[META14]]
; CHECK-NEXT: [[P_I3:%.*]] = getelementptr float, ptr [[P]], i32 3
-; CHECK-NEXT: [[V_I3:%.*]] = load float, ptr [[P_I3]], align 4, !tbaa [[FLOAT_TBAA9]], !fpmath [[META11]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: [[V_I3:%.*]] = load float, ptr [[P_I3]], align 4, !tbaa [[FLOAT_TBAA16]], !fpmath [[META18]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !nofpclass [[META19]], !mem.cache_hint [[META14]]
; CHECK-NEXT: [[V_UPTO0:%.*]] = insertelement <4 x float> poison, float [[V_I0]], i64 0
; CHECK-NEXT: [[V_UPTO1:%.*]] = insertelement <4 x float> [[V_UPTO0]], float [[V_I1]], i64 1
; CHECK-NEXT: [[V_UPTO2:%.*]] = insertelement <4 x float> [[V_UPTO1]], float [[V_I2]], i64 2
@@ -58,9 +58,9 @@ define <4 x float> @floating_load(ptr %p) {
;
; PACKED-LABEL: define <4 x float> @floating_load(
; PACKED-SAME: ptr [[P:%.*]]) {
-; PACKED-NEXT: [[V_I0:%.*]] = load <2 x float>, ptr [[P]], align 16, !tbaa [[FLOAT_TBAA9:![0-9]+]], !fpmath [[META11:![0-9]+]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT: [[V_I0:%.*]] = load <2 x float>, ptr [[P]], align 16, !tbaa [[FLOAT_TBAA16:![0-9]+]], !fpmath [[META18:![0-9]+]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !invariant.group [[META5]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !nofpclass [[META19:![0-9]+]], !mem.cache_hint [[META14]]
; PACKED-NEXT: [[P_I1:%.*]] = getelementptr <2 x float>, ptr [[P]], i32 1
-; PACKED-NEXT: [[V_I1:%.*]] = load <2 x float>, ptr [[P_I1]], align 8, !tbaa [[FLOAT_TBAA9]], !fpmath [[META11]], !invariant.load [[META4]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT: [[V_I1:%.*]] = load <2 x float>, ptr [[P_I1]], align 8, !tbaa [[FLOAT_TBAA16]], !fpmath [[META18]], !invariant.load [[META5]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !noundef [[META5]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !nofpclass [[META19]], !mem.cache_hint [[META14]]
; PACKED-NEXT: [[TMP1:%.*]] = shufflevector <2 x float> [[V_I0]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; PACKED-NEXT: [[TMP2:%.*]] = shufflevector <2 x float> [[V_I1]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; PACKED-NEXT: [[V:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
@@ -76,25 +76,25 @@ define void @integer_store(ptr %p, <4 x i32> %v) {
; CHECK-LABEL: define void @integer_store(
; CHECK-SAME: ptr [[P:%.*]], <4 x i32> [[V:%.*]]) {
; CHECK-NEXT: [[V_I0:%.*]] = extractelement <4 x i32> [[V]], i64 0
-; CHECK-NEXT: store i32 [[V_I0]], ptr [[P]], align 16, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: store i32 [[V_I0]], ptr [[P]], align 16, !tbaa [[INT_TBAA0]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !invariant.group [[META5]], !llvm.access.group [[META10]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META20:![0-9]+]]
; CHECK-NEXT: [[V_I1:%.*]] = extractelement <4 x i32> [[V]], i64 1
; CHECK-NEXT: [[P_I1:%.*]] = getelementptr i32, ptr [[P]], i32 1
-; CHECK-NEXT: store i32 [[V_I1]], ptr [[P_I1]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: store i32 [[V_I1]], ptr [[P_I1]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META20]]
; CHECK-NEXT: [[V_I2:%.*]] = extractelement <4 x i32> [[V]], i64 2
; CHECK-NEXT: [[P_I2:%.*]] = getelementptr i32, ptr [[P]], i32 2
-; CHECK-NEXT: store i32 [[V_I2]], ptr [[P_I2]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: store i32 [[V_I2]], ptr [[P_I2]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META20]]
; CHECK-NEXT: [[V_I3:%.*]] = extractelement <4 x i32> [[V]], i64 3
; CHECK-NEXT: [[P_I3:%.*]] = getelementptr i32, ptr [[P]], i32 3
-; CHECK-NEXT: store i32 [[V_I3]], ptr [[P_I3]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; CHECK-NEXT: store i32 [[V_I3]], ptr [[P_I3]], align 4, !tbaa [[INT_TBAA0]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META20]]
; CHECK-NEXT: ret void
;
; PACKED-LABEL: define void @integer_store(
; PACKED-SAME: ptr [[P:%.*]], <4 x i32> [[V:%.*]]) {
; PACKED-NEXT: [[V_I0:%.*]] = shufflevector <4 x i32> [[V]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>
-; PACKED-NEXT: store <2 x i32> [[V_I0]], ptr [[P]], align 16, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT: store <2 x i32> [[V_I0]], ptr [[P]], align 16, !tbaa [[INT_TBAA0]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !invariant.group [[META5]], !llvm.access.group [[META10]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META20:![0-9]+]]
; PACKED-NEXT: [[V_I1:%.*]] = shufflevector <4 x i32> [[V]], <4 x i32> poison, <2 x i32> <i32 2, i32 3>
; PACKED-NEXT: [[P_I1:%.*]] = getelementptr <2 x i32>, ptr [[P]], i32 1
-; PACKED-NEXT: store <2 x i32> [[V_I1]], ptr [[P_I1]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META5]], !noalias [[META5]], !llvm.mem.parallel_loop_access [[META8]], !llvm.access.group [[META8]]
+; PACKED-NEXT: store <2 x i32> [[V_I1]], ptr [[P_I1]], align 8, !tbaa [[INT_TBAA0]], !alias.scope [[META6]], !noalias [[META6]], !nontemporal [[META9]], !llvm.mem.parallel_loop_access [[META10]], !llvm.access.group [[META10]], !annotation [[META11]], !nosanitize [[META5]], !mmra [[META12]], !noalias.addrspace [[META13]], !mem.cache_hint [[META20]]
; PACKED-NEXT: ret void
;
store <4 x i32> %v, ptr %p, align 16, !tbaa !10, !invariant.group !0, !mem.cache_hint !18, !alias.scope !6, !noalias !6, !nontemporal !1, !llvm.mem.parallel_loop_access !5, !llvm.access.group !5, !annotation !2, !nosanitize !0, !mmra !3, !noalias.addrspace !4, !test.unknown !0
@@ -132,25 +132,43 @@ define void @integer_store(ptr %p, <4 x i32> %v) {
; CHECK: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
; CHECK: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
; CHECK: [[META3]] = !{!"root"}
-; CHECK: [[META4]] = !{}
-; CHECK: [[META5]] = !{[[META6:![0-9]+]]}
-; CHECK: [[META6]] = distinct !{[[META6]], [[META7:![0-9]+]], !"scope"}
-; CHECK: [[META7]] = distinct !{[[META7]], i1 false, !"domain"}
-; CHECK: [[META8]] = distinct !{}
-; CHECK: [[FLOAT_TBAA9]] = !{[[META10:![0-9]+]], [[META10]], i64 0}
-; CHECK: [[META10]] = !{!"float", [[META2]], i64 0}
-; CHECK: [[META11]] = !{float 2.500000e+00}
+; CHECK: [[RNG4]] = !{i32 0, i32 100}
+; CHECK: [[META5]] = !{}
+; CHECK: [[META6]] = !{[[META7:![0-9]+]]}
+; CHECK: [[META7]] = distinct !{[[META7]], [[META8:![0-9]+]], !"scope"}
+; CHECK: [[META8]] = distinct !{[[META8]], i1 false, !"domain"}
+; CHECK: [[META9]] = !{i32 1}
+; CHECK: [[META10]] = distinct !{}
+; CHECK: [[META11]] = !{!"preserved memory metadata"}
+; CHECK: [[META12]] = !{!"test", !"relaxed"}
+; CHECK: [[META13]] = !{i32 5, i32 6}
+; CHECK: [[META14]] = !{i32 0, [[META15:![0-9]+]]}
+; CHECK: [[META15]] = !{!"nvvm.l1_eviction", !"last"}
+; CHECK: [[FLOAT_TBAA16]] = !{[[META17:![0-9]+]], [[META17]], i64 0}
+; CHECK: [[META17]] = !{!"float", [[META2]], i64 0}
+; CHECK: [[META18]] = !{float 2.500000e+00}
+; CHECK: [[META19]] = !{i32 3}
+; CHECK: [[META20]] = !{i32 1, [[META15]]}
;.
; PACKED: [[INT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
; PACKED: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
; PACKED: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
; PACKED: [[META3]] = !{!"root"}
-; PACKED: [[META4]] = !{}
-; PACKED: [[META5]] = !{[[META6:![0-9]+]]}
-; PACKED: [[META6]] = distinct !{[[META6]], [[META7:![0-9]+]], !"scope"}
-; PACKED: [[META7]] = distinct !{[[META7]], i1 false, !"domain"}
-; PACKED: [[META8]] = distinct !{}
-; PACKED: [[FLOAT_TBAA9]] = !{[[META10:![0-9]+]], [[META10]], i64 0}
-; PACKED: [[META10]] = !{!"float", [[META2]], i64 0}
-; PACKED: [[META11]] = !{float 2.500000e+00}
+; PACKED: [[RNG4]] = !{i32 0, i32 100}
+; PACKED: [[META5]] = !{}
+; PACKED: [[META6]] = !{[[META7:![0-9]+]]}
+; PACKED: [[META7]] = distinct !{[[META7]], [[META8:![0-9]+]], !"scope"}
+; PACKED: [[META8]] = distinct !{[[META8]], i1 false, !"domain"}
+; PACKED: [[META9]] = !{i32 1}
+; PACKED: [[META10]] = distinct !{}
+; PACKED: [[META11]] = !{!"preserved memory metadata"}
+; PACKED: [[META12]] = !{!"test", !"relaxed"}
+; PACKED: [[META13]] = !{i32 5, i32 6}
+; PACKED: [[META14]] = !{i32 0, [[META15:![0-9]+]]}
+; PACKED: [[META15]] = !{!"nvvm.l1_eviction", !"last"}
+; PACKED: [[FLOAT_TBAA16]] = !{[[META17:![0-9]+]], [[META17]], i64 0}
+; PACKED: [[META17]] = !{!"float", [[META2]], i64 0}
+; PACKED: [[META18]] = !{float 2.500000e+00}
+; PACKED: [[META19]] = !{i32 3}
+; PACKED: [[META20]] = !{i32 1, [[META15]]}
;.
diff --git a/llvm/unittests/Analysis/MetadataUtilsTest.cpp b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
index e8ab8d7a9c7145..abe80a2ad9920b 100644
--- a/llvm/unittests/Analysis/MetadataUtilsTest.cpp
+++ b/llvm/unittests/Analysis/MetadataUtilsTest.cpp
@@ -270,6 +270,18 @@ TEST_F(MetadataUtilsTest, StoreTypeChangePreservesHints) {
EXPECT_TRUE(New->hasMetadata(LLVMContext::MD_invariant_group));
}
+TEST_F(MetadataUtilsTest, NonMemoryScalarizationPreservesFPMath) {
+ auto *Ty = FixedVectorType::get(B.getFloatTy(), 2);
+ auto *Old =
+ cast<Instruction>(B.CreateFAdd(B.CreateLoad(Ty, P), B.CreateLoad(Ty, Q)));
+ auto *New = cast<Instruction>(B.CreateFAdd(B.CreateLoad(B.getFloatTy(), P),
+ B.CreateLoad(B.getFloatTy(), Q)));
+ auto *Tag = MDNode::get(
+ C, {ConstantAsMetadata::get(ConstantFP::get(B.getFloatTy(), 2.5))});
+ Old->setMetadata(LLVMContext::MD_fpmath, Tag);
+ copyMetadataForScalarizedInstruction(*New, *Old);
+ EXPECT_EQ(New->getMetadata(LLVMContext::MD_fpmath), Tag);
+}
TEST_F(MetadataUtilsTest, CSEIntersectsCacheHints) {
auto *A = B.CreateLoad(B.getInt32Ty(), P);
More information about the llvm-branch-commits
mailing list