[llvm-branch-commits] [llvm] [VectorUtils] Propagate cache hint metadata (PR #241114)

Yonah Goldberg via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Fri Nov 13 16:46:38 PST 2026


https://github.com/YonahGoldberg updated https://github.com/llvm/llvm-project/pull/241114

>From ffc51f2873cfcf013a65022f4b12383ab66a1292 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Fri, 13 Nov 2026 03:42:17 +0000
Subject: [PATCH 1/3] [VectorUtils] Add cache-hint tests

---
 .../LoadStoreVectorizer/NVPTX/cache-hint.ll   | 313 ++++++++++++++++++
 .../SLPVectorizer/X86/cache-hint.ll           | 239 +++++++++++++
 2 files changed, 552 insertions(+)
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/NVPTX/cache-hint.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/cache-hint.ll

diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/cache-hint.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/cache-hint.ll
new file mode 100644
index 0000000000000..9408dd43e02d5
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/cache-hint.ll
@@ -0,0 +1,313 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=load-store-vectorizer -S -mtriple=nvptx64-nvidia-cuda -mcpu=sm_100 -mattr=+ptx88 | FileCheck %s --implicit-check-not=mem.cache_hint
+
+define i32 @load_identical_hints(ptr addrspace(1) %p) {
+;
+; CHECK-LABEL: define i32 @load_identical_hints(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[P]], align 8
+; CHECK-NEXT:    [[X1:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[Y2:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+; CHECK-NEXT:    [[SUM:%.*]] = add i32 [[X1]], [[Y2]]
+; CHECK-NEXT:    ret i32 [[SUM]]
+;
+  %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
+  %x = load i32, ptr addrspace(1) %p, align 8, !mem.cache_hint !0
+  %y = load i32, ptr addrspace(1) %p1, align 4, !mem.cache_hint !0
+  %sum = add i32 %x, %y
+  ret i32 %sum
+}
+
+define i32 @load_missing_hint(ptr addrspace(1) %p) {
+;
+; CHECK-LABEL: define i32 @load_missing_hint(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[P]], align 8
+; CHECK-NEXT:    [[X1:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[Y2:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+; CHECK-NEXT:    [[SUM:%.*]] = add i32 [[X1]], [[Y2]]
+; CHECK-NEXT:    ret i32 [[SUM]]
+;
+  %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
+  %x = load i32, ptr addrspace(1) %p, align 8, !mem.cache_hint !0
+  %y = load i32, ptr addrspace(1) %p1, align 4
+  %sum = add i32 %x, %y
+  ret i32 %sum
+}
+
+define i32 @load_conflicting_hints(ptr addrspace(1) %p) {
+;
+; CHECK-LABEL: define i32 @load_conflicting_hints(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[P]], align 8
+; CHECK-NEXT:    [[X1:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[Y2:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+; CHECK-NEXT:    [[SUM:%.*]] = add i32 [[X1]], [[Y2]]
+; CHECK-NEXT:    ret i32 [[SUM]]
+;
+  %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
+  %x = load i32, ptr addrspace(1) %p, align 8, !mem.cache_hint !0
+  %y = load i32, ptr addrspace(1) %p1, align 4, !mem.cache_hint !1
+  %sum = add i32 %x, %y
+  ret i32 %sum
+}
+
+define void @store_identical_hints(ptr addrspace(1) %p, i32 %x, i32 %y) {
+;
+; CHECK-LABEL: define void @store_identical_hints(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[Y]], i64 1
+; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr addrspace(1) [[P]], align 8
+; CHECK-NEXT:    ret void
+;
+  %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
+  store i32 %x, ptr addrspace(1) %p, align 8, !mem.cache_hint !2
+  store i32 %y, ptr addrspace(1) %p1, align 4, !mem.cache_hint !2
+  ret void
+}
+
+define void @store_missing_hint(ptr addrspace(1) %p, i32 %x, i32 %y) {
+;
+; CHECK-LABEL: define void @store_missing_hint(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[Y]], i64 1
+; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr addrspace(1) [[P]], align 8
+; CHECK-NEXT:    ret void
+;
+  %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
+  store i32 %x, ptr addrspace(1) %p, align 8, !mem.cache_hint !2
+  store i32 %y, ptr addrspace(1) %p1, align 4
+  ret void
+}
+
+define void @store_conflicting_hints(ptr addrspace(1) %p, i32 %x, i32 %y) {
+;
+; CHECK-LABEL: define void @store_conflicting_hints(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[Y]], i64 1
+; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr addrspace(1) [[P]], align 8
+; CHECK-NEXT:    ret void
+;
+  %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
+  store i32 %x, ptr addrspace(1) %p, align 8, !mem.cache_hint !2
+  store i32 %y, ptr addrspace(1) %p1, align 4, !mem.cache_hint !3
+  ret void
+}
+
+define i32 @load_missing_first_hint(ptr addrspace(1) %p) {
+;
+; CHECK-LABEL: define i32 @load_missing_first_hint(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[P]], align 8
+; CHECK-NEXT:    [[X1:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[Y2:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+; CHECK-NEXT:    [[SUM:%.*]] = add i32 [[X1]], [[Y2]]
+; CHECK-NEXT:    ret i32 [[SUM]]
+;
+  %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
+  %x = load i32, ptr addrspace(1) %p, align 8
+  %y = load i32, ptr addrspace(1) %p1, align 4, !mem.cache_hint !0
+  %sum = add i32 %x, %y
+  ret i32 %sum
+}
+
+define void @store_missing_first_hint(ptr addrspace(1) %p, i32 %x, i32 %y) {
+;
+; CHECK-LABEL: define void @store_missing_first_hint(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[Y]], i64 1
+; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr addrspace(1) [[P]], align 8
+; CHECK-NEXT:    ret void
+;
+  %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
+  store i32 %x, ptr addrspace(1) %p, align 8
+  store i32 %y, ptr addrspace(1) %p1, align 4, !mem.cache_hint !2
+  ret void
+}
+
+; Leave element 2 inactive in the resulting masked access.
+define i32 @masked_load_identical_hints(ptr addrspace(1) %p) {
+;
+; CHECK-LABEL: define i32 @masked_load_identical_hints(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison)
+; CHECK-NEXT:    [[X1:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[Y2:%.*]] = extractelement <4 x i32> [[TMP1]], i64 1
+; CHECK-NEXT:    [[GAPFILL3:%.*]] = extractelement <4 x i32> [[TMP1]], i64 2
+; CHECK-NEXT:    [[Z4:%.*]] = extractelement <4 x i32> [[TMP1]], i64 3
+; CHECK-NEXT:    [[XY:%.*]] = add i32 [[X1]], [[Y2]]
+; CHECK-NEXT:    [[SUM:%.*]] = add i32 [[XY]], [[Z4]]
+; CHECK-NEXT:    ret i32 [[SUM]]
+;
+  %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
+  %p3 = getelementptr i32, ptr addrspace(1) %p, i64 3
+  %x = load i32, ptr addrspace(1) %p, align 16, !mem.cache_hint !0
+  %y = load i32, ptr addrspace(1) %p1, align 4, !mem.cache_hint !0
+  %z = load i32, ptr addrspace(1) %p3, align 4, !mem.cache_hint !0
+  %xy = add i32 %x, %y
+  %sum = add i32 %xy, %z
+  ret i32 %sum
+}
+
+; Leave element 2 inactive in the resulting masked access.
+define i32 @masked_load_missing_first_hints(ptr addrspace(1) %p) {
+;
+; CHECK-LABEL: define i32 @masked_load_missing_first_hints(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison)
+; CHECK-NEXT:    [[X1:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[Y2:%.*]] = extractelement <4 x i32> [[TMP1]], i64 1
+; CHECK-NEXT:    [[GAPFILL3:%.*]] = extractelement <4 x i32> [[TMP1]], i64 2
+; CHECK-NEXT:    [[Z4:%.*]] = extractelement <4 x i32> [[TMP1]], i64 3
+; CHECK-NEXT:    [[XY:%.*]] = add i32 [[X1]], [[Y2]]
+; CHECK-NEXT:    [[SUM:%.*]] = add i32 [[XY]], [[Z4]]
+; CHECK-NEXT:    ret i32 [[SUM]]
+;
+  %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
+  %p3 = getelementptr i32, ptr addrspace(1) %p, i64 3
+  %x = load i32, ptr addrspace(1) %p, align 16
+  %y = load i32, ptr addrspace(1) %p1, align 4, !mem.cache_hint !0
+  %z = load i32, ptr addrspace(1) %p3, align 4, !mem.cache_hint !0
+  %xy = add i32 %x, %y
+  %sum = add i32 %xy, %z
+  ret i32 %sum
+}
+
+; Leave element 2 inactive in the resulting masked access.
+define i32 @masked_load_missing_last_hints(ptr addrspace(1) %p) {
+;
+; CHECK-LABEL: define i32 @masked_load_missing_last_hints(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison)
+; CHECK-NEXT:    [[X1:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[Y2:%.*]] = extractelement <4 x i32> [[TMP1]], i64 1
+; CHECK-NEXT:    [[GAPFILL3:%.*]] = extractelement <4 x i32> [[TMP1]], i64 2
+; CHECK-NEXT:    [[Z4:%.*]] = extractelement <4 x i32> [[TMP1]], i64 3
+; CHECK-NEXT:    [[XY:%.*]] = add i32 [[X1]], [[Y2]]
+; CHECK-NEXT:    [[SUM:%.*]] = add i32 [[XY]], [[Z4]]
+; CHECK-NEXT:    ret i32 [[SUM]]
+;
+  %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
+  %p3 = getelementptr i32, ptr addrspace(1) %p, i64 3
+  %x = load i32, ptr addrspace(1) %p, align 16, !mem.cache_hint !0
+  %y = load i32, ptr addrspace(1) %p1, align 4, !mem.cache_hint !0
+  %z = load i32, ptr addrspace(1) %p3, align 4
+  %xy = add i32 %x, %y
+  %sum = add i32 %xy, %z
+  ret i32 %sum
+}
+
+; Leave element 2 inactive in the resulting masked access.
+define i32 @masked_load_conflicting_hints(ptr addrspace(1) %p) {
+;
+; CHECK-LABEL: define i32 @masked_load_conflicting_hints(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison)
+; CHECK-NEXT:    [[X1:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    [[Y2:%.*]] = extractelement <4 x i32> [[TMP1]], i64 1
+; CHECK-NEXT:    [[GAPFILL3:%.*]] = extractelement <4 x i32> [[TMP1]], i64 2
+; CHECK-NEXT:    [[Z4:%.*]] = extractelement <4 x i32> [[TMP1]], i64 3
+; CHECK-NEXT:    [[XY:%.*]] = add i32 [[X1]], [[Y2]]
+; CHECK-NEXT:    [[SUM:%.*]] = add i32 [[XY]], [[Z4]]
+; CHECK-NEXT:    ret i32 [[SUM]]
+;
+  %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
+  %p3 = getelementptr i32, ptr addrspace(1) %p, i64 3
+  %x = load i32, ptr addrspace(1) %p, align 16, !mem.cache_hint !0
+  %y = load i32, ptr addrspace(1) %p1, align 4, !mem.cache_hint !0
+  %z = load i32, ptr addrspace(1) %p3, align 4, !mem.cache_hint !1
+  %xy = add i32 %x, %y
+  %sum = add i32 %xy, %z
+  ret i32 %sum
+}
+
+; Leave element 2 inactive in the resulting masked access.
+define void @masked_store_identical_hints(ptr addrspace(1) %p, i64 %x, i64 %y, i64 %z) {
+;
+; CHECK-LABEL: define void @masked_store_identical_hints(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i64 [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i64> [[TMP1]], i64 [[Y]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i64> [[TMP2]], i64 poison, i64 2
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i64> [[TMP3]], i64 [[Z]], i64 3
+; CHECK-NEXT:    call void @llvm.masked.store.v4i64.p1(<4 x i64> [[TMP4]], ptr addrspace(1) align 32 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>)
+; CHECK-NEXT:    ret void
+;
+  %p1 = getelementptr i64, ptr addrspace(1) %p, i64 1
+  %p3 = getelementptr i64, ptr addrspace(1) %p, i64 3
+  store i64 %x, ptr addrspace(1) %p, align 32, !mem.cache_hint !2
+  store i64 %y, ptr addrspace(1) %p1, align 8, !mem.cache_hint !2
+  store i64 %z, ptr addrspace(1) %p3, align 8, !mem.cache_hint !2
+  ret void
+}
+
+; Leave element 2 inactive in the resulting masked access.
+define void @masked_store_missing_first_hints(ptr addrspace(1) %p, i64 %x, i64 %y, i64 %z) {
+;
+; CHECK-LABEL: define void @masked_store_missing_first_hints(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i64 [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i64> [[TMP1]], i64 [[Y]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i64> [[TMP2]], i64 poison, i64 2
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i64> [[TMP3]], i64 [[Z]], i64 3
+; CHECK-NEXT:    call void @llvm.masked.store.v4i64.p1(<4 x i64> [[TMP4]], ptr addrspace(1) align 32 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>)
+; CHECK-NEXT:    ret void
+;
+  %p1 = getelementptr i64, ptr addrspace(1) %p, i64 1
+  %p3 = getelementptr i64, ptr addrspace(1) %p, i64 3
+  store i64 %x, ptr addrspace(1) %p, align 32
+  store i64 %y, ptr addrspace(1) %p1, align 8, !mem.cache_hint !2
+  store i64 %z, ptr addrspace(1) %p3, align 8, !mem.cache_hint !2
+  ret void
+}
+
+; Leave element 2 inactive in the resulting masked access.
+define void @masked_store_missing_last_hints(ptr addrspace(1) %p, i64 %x, i64 %y, i64 %z) {
+;
+; CHECK-LABEL: define void @masked_store_missing_last_hints(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i64 [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i64> [[TMP1]], i64 [[Y]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i64> [[TMP2]], i64 poison, i64 2
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i64> [[TMP3]], i64 [[Z]], i64 3
+; CHECK-NEXT:    call void @llvm.masked.store.v4i64.p1(<4 x i64> [[TMP4]], ptr addrspace(1) align 32 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>)
+; CHECK-NEXT:    ret void
+;
+  %p1 = getelementptr i64, ptr addrspace(1) %p, i64 1
+  %p3 = getelementptr i64, ptr addrspace(1) %p, i64 3
+  store i64 %x, ptr addrspace(1) %p, align 32, !mem.cache_hint !2
+  store i64 %y, ptr addrspace(1) %p1, align 8, !mem.cache_hint !2
+  store i64 %z, ptr addrspace(1) %p3, align 8
+  ret void
+}
+
+; Leave element 2 inactive in the resulting masked access.
+define void @masked_store_conflicting_hints(ptr addrspace(1) %p, i64 %x, i64 %y, i64 %z) {
+;
+; CHECK-LABEL: define void @masked_store_conflicting_hints(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i64 [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i64> [[TMP1]], i64 [[Y]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i64> [[TMP2]], i64 poison, i64 2
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i64> [[TMP3]], i64 [[Z]], i64 3
+; CHECK-NEXT:    call void @llvm.masked.store.v4i64.p1(<4 x i64> [[TMP4]], ptr addrspace(1) align 32 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>)
+; CHECK-NEXT:    ret void
+;
+  %p1 = getelementptr i64, ptr addrspace(1) %p, i64 1
+  %p3 = getelementptr i64, ptr addrspace(1) %p, i64 3
+  store i64 %x, ptr addrspace(1) %p, align 32, !mem.cache_hint !2
+  store i64 %y, ptr addrspace(1) %p1, align 8, !mem.cache_hint !2
+  store i64 %z, ptr addrspace(1) %p3, align 8, !mem.cache_hint !3
+  ret void
+}
+
+!0 = !{i32 0, !4}
+!1 = !{i32 0, !5}
+!2 = !{i32 1, !4}
+!3 = !{i32 1, !5}
+!4 = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_cache_hint", i64 1508705875169116160}
+!5 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_cache_hint", i64 1364590687093260288}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cache-hint.ll b/llvm/test/Transforms/SLPVectorizer/X86/cache-hint.ll
new file mode 100644
index 0000000000000..124136398b0d6
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cache-hint.ll
@@ -0,0 +1,239 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=znver4 < %s | FileCheck %s --implicit-check-not=mem.cache_hint
+
+define void @load_identical_hints(ptr noalias %src, ptr noalias %dst) {
+;
+; CHECK-LABEL: define void @load_identical_hints(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP1]], ptr [[DST]], align 8
+; CHECK-NEXT:    ret void
+;
+  %src1 = getelementptr inbounds double, ptr %src, i64 1
+  %dst1 = getelementptr inbounds double, ptr %dst, i64 1
+  %v0 = load double, ptr %src, align 8, !mem.cache_hint !0
+  %v1 = load double, ptr %src1, align 8, !mem.cache_hint !0
+  store double %v0, ptr %dst, align 8
+  store double %v1, ptr %dst1, align 8
+  ret void
+}
+
+define void @load_missing_first_hint(ptr noalias %src, ptr noalias %dst) {
+;
+; CHECK-LABEL: define void @load_missing_first_hint(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP1]], ptr [[DST]], align 8
+; CHECK-NEXT:    ret void
+;
+  %src1 = getelementptr inbounds double, ptr %src, i64 1
+  %dst1 = getelementptr inbounds double, ptr %dst, i64 1
+  %v0 = load double, ptr %src, align 8
+  %v1 = load double, ptr %src1, align 8, !mem.cache_hint !0
+  store double %v0, ptr %dst, align 8
+  store double %v1, ptr %dst1, align 8
+  ret void
+}
+
+define void @load_missing_last_hint(ptr noalias %src, ptr noalias %dst) {
+;
+; CHECK-LABEL: define void @load_missing_last_hint(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP1]], ptr [[DST]], align 8
+; CHECK-NEXT:    ret void
+;
+  %src1 = getelementptr inbounds double, ptr %src, i64 1
+  %dst1 = getelementptr inbounds double, ptr %dst, i64 1
+  %v0 = load double, ptr %src, align 8, !mem.cache_hint !0
+  %v1 = load double, ptr %src1, align 8
+  store double %v0, ptr %dst, align 8
+  store double %v1, ptr %dst1, align 8
+  ret void
+}
+
+define void @load_conflicting_hints(ptr noalias %src, ptr noalias %dst) {
+;
+; CHECK-LABEL: define void @load_conflicting_hints(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP1]], ptr [[DST]], align 8
+; CHECK-NEXT:    ret void
+;
+  %src1 = getelementptr inbounds double, ptr %src, i64 1
+  %dst1 = getelementptr inbounds double, ptr %dst, i64 1
+  %v0 = load double, ptr %src, align 8, !mem.cache_hint !0
+  %v1 = load double, ptr %src1, align 8, !mem.cache_hint !1
+  store double %v0, ptr %dst, align 8
+  store double %v1, ptr %dst1, align 8
+  ret void
+}
+
+define void @store_identical_hints(ptr noalias %src, ptr noalias %dst) {
+;
+; CHECK-LABEL: define void @store_identical_hints(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP1]], ptr [[DST]], align 8
+; CHECK-NEXT:    ret void
+;
+  %src1 = getelementptr inbounds double, ptr %src, i64 1
+  %dst1 = getelementptr inbounds double, ptr %dst, i64 1
+  %v0 = load double, ptr %src, align 8
+  %v1 = load double, ptr %src1, align 8
+  store double %v0, ptr %dst, align 8, !mem.cache_hint !2
+  store double %v1, ptr %dst1, align 8, !mem.cache_hint !2
+  ret void
+}
+
+define void @store_missing_first_hint(ptr noalias %src, ptr noalias %dst) {
+;
+; CHECK-LABEL: define void @store_missing_first_hint(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP1]], ptr [[DST]], align 8
+; CHECK-NEXT:    ret void
+;
+  %src1 = getelementptr inbounds double, ptr %src, i64 1
+  %dst1 = getelementptr inbounds double, ptr %dst, i64 1
+  %v0 = load double, ptr %src, align 8
+  %v1 = load double, ptr %src1, align 8
+  store double %v0, ptr %dst, align 8
+  store double %v1, ptr %dst1, align 8, !mem.cache_hint !2
+  ret void
+}
+
+define void @store_missing_last_hint(ptr noalias %src, ptr noalias %dst) {
+;
+; CHECK-LABEL: define void @store_missing_last_hint(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP1]], ptr [[DST]], align 8
+; CHECK-NEXT:    ret void
+;
+  %src1 = getelementptr inbounds double, ptr %src, i64 1
+  %dst1 = getelementptr inbounds double, ptr %dst, i64 1
+  %v0 = load double, ptr %src, align 8
+  %v1 = load double, ptr %src1, align 8
+  store double %v0, ptr %dst, align 8, !mem.cache_hint !2
+  store double %v1, ptr %dst1, align 8
+  ret void
+}
+
+define void @store_conflicting_hints(ptr noalias %src, ptr noalias %dst) {
+;
+; CHECK-LABEL: define void @store_conflicting_hints(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP1]], ptr [[DST]], align 8
+; CHECK-NEXT:    ret void
+;
+  %src1 = getelementptr inbounds double, ptr %src, i64 1
+  %dst1 = getelementptr inbounds double, ptr %dst, i64 1
+  %v0 = load double, ptr %src, align 8
+  %v1 = load double, ptr %src1, align 8
+  store double %v0, ptr %dst, align 8, !mem.cache_hint !2
+  store double %v1, ptr %dst1, align 8, !mem.cache_hint !3
+  ret void
+}
+
+define void @gather_identical_hints(ptr noalias %src, ptr noalias %dst) {
+;
+; CHECK-LABEL: define void @gather_identical_hints(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x ptr> poison, ptr [[SRC]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x ptr> [[TMP1]], <8 x ptr> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds double, <8 x ptr> [[TMP2]], <8 x i64> <i64 0, i64 3, i64 7, i64 11, i64 18, i64 21, i64 27, i64 32>
+; CHECK-NEXT:    [[DST0:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> align 8 [[TMP3]], <8 x i1> splat (i1 true), <8 x double> poison)
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd <8 x double> [[TMP4]], splat (double 1.000000e+00)
+; CHECK-NEXT:    store <8 x double> [[TMP5]], ptr [[DST0]], align 8
+; CHECK-NEXT:    ret void
+;
+  %src0 = getelementptr inbounds double, ptr %src, i64 0
+  %v0 = load double, ptr %src0, align 8, !mem.cache_hint !0
+  %src1 = getelementptr inbounds double, ptr %src, i64 3
+  %v1 = load double, ptr %src1, align 8, !mem.cache_hint !0
+  %src2 = getelementptr inbounds double, ptr %src, i64 7
+  %v2 = load double, ptr %src2, align 8, !mem.cache_hint !0
+  %src3 = getelementptr inbounds double, ptr %src, i64 11
+  %v3 = load double, ptr %src3, align 8, !mem.cache_hint !0
+  %src4 = getelementptr inbounds double, ptr %src, i64 18
+  %v4 = load double, ptr %src4, align 8, !mem.cache_hint !0
+  %src5 = getelementptr inbounds double, ptr %src, i64 21
+  %v5 = load double, ptr %src5, align 8, !mem.cache_hint !0
+  %src6 = getelementptr inbounds double, ptr %src, i64 27
+  %v6 = load double, ptr %src6, align 8, !mem.cache_hint !0
+  %src7 = getelementptr inbounds double, ptr %src, i64 32
+  %v7 = load double, ptr %src7, align 8, !mem.cache_hint !0
+  %a0 = fadd double %v0, 1.0
+  %a1 = fadd double %v1, 1.0
+  %a2 = fadd double %v2, 1.0
+  %a3 = fadd double %v3, 1.0
+  %a4 = fadd double %v4, 1.0
+  %a5 = fadd double %v5, 1.0
+  %a6 = fadd double %v6, 1.0
+  %a7 = fadd double %v7, 1.0
+  %dst0 = getelementptr inbounds double, ptr %dst, i64 0
+  store double %a0, ptr %dst0, align 8
+  %dst1 = getelementptr inbounds double, ptr %dst, i64 1
+  store double %a1, ptr %dst1, align 8
+  %dst2 = getelementptr inbounds double, ptr %dst, i64 2
+  store double %a2, ptr %dst2, align 8
+  %dst3 = getelementptr inbounds double, ptr %dst, i64 3
+  store double %a3, ptr %dst3, align 8
+  %dst4 = getelementptr inbounds double, ptr %dst, i64 4
+  store double %a4, ptr %dst4, align 8
+  %dst5 = getelementptr inbounds double, ptr %dst, i64 5
+  store double %a5, ptr %dst5, align 8
+  %dst6 = getelementptr inbounds double, ptr %dst, i64 6
+  store double %a6, ptr %dst6, align 8
+  %dst7 = getelementptr inbounds double, ptr %dst, i64 7
+  store double %a7, ptr %dst7, align 8
+  ret void
+}
+
+define void @masked_store_identical_hints(ptr noalias %src, ptr noalias %dst) {
+;
+; CHECK-LABEL: define void @masked_store_identical_hints(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[V:%.*]] = load <8 x i32>, ptr [[SRC]], align 4
+; CHECK-NEXT:    [[DST0:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[V]], <8 x i32> poison, <15 x i32> <i32 0, i32 poison, i32 1, i32 poison, i32 2, i32 poison, i32 3, i32 poison, i32 4, i32 poison, i32 5, i32 poison, i32 6, i32 poison, i32 7>
+; CHECK-NEXT:    call void @llvm.masked.store.v15i32.p0(<15 x i32> [[TMP1]], ptr align 4 [[DST0]], <15 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true>)
+; CHECK-NEXT:    ret void
+;
+  %v = load <8 x i32>, ptr %src, align 4
+  %e0 = extractelement <8 x i32> %v, i32 0
+  %dst0 = getelementptr inbounds i32, ptr %dst, i64 0
+  %e1 = extractelement <8 x i32> %v, i32 1
+  %dst1 = getelementptr inbounds i32, ptr %dst, i64 2
+  %e2 = extractelement <8 x i32> %v, i32 2
+  %dst2 = getelementptr inbounds i32, ptr %dst, i64 4
+  %e3 = extractelement <8 x i32> %v, i32 3
+  %dst3 = getelementptr inbounds i32, ptr %dst, i64 6
+  %e4 = extractelement <8 x i32> %v, i32 4
+  %dst4 = getelementptr inbounds i32, ptr %dst, i64 8
+  %e5 = extractelement <8 x i32> %v, i32 5
+  %dst5 = getelementptr inbounds i32, ptr %dst, i64 10
+  %e6 = extractelement <8 x i32> %v, i32 6
+  %dst6 = getelementptr inbounds i32, ptr %dst, i64 12
+  %e7 = extractelement <8 x i32> %v, i32 7
+  %dst7 = getelementptr inbounds i32, ptr %dst, i64 14
+  store i32 %e0, ptr %dst0, align 4, !mem.cache_hint !2
+  store i32 %e1, ptr %dst1, align 4, !mem.cache_hint !2
+  store i32 %e2, ptr %dst2, align 4, !mem.cache_hint !2
+  store i32 %e3, ptr %dst3, align 4, !mem.cache_hint !2
+  store i32 %e4, ptr %dst4, align 4, !mem.cache_hint !2
+  store i32 %e5, ptr %dst5, align 4, !mem.cache_hint !2
+  store i32 %e6, ptr %dst6, align 4, !mem.cache_hint !2
+  store i32 %e7, ptr %dst7, align 4, !mem.cache_hint !2
+  ret void
+}
+
+!0 = !{i32 0, !4}
+!1 = !{i32 0, !5}
+!2 = !{i32 1, !4}
+!3 = !{i32 1, !5}
+!4 = !{!"nvvm.l1_eviction", !"last"}
+!5 = !{!"nvvm.l1_eviction", !"first"}

>From 6c38f308c6bdcbead353cd1b18d4aead42b9db31 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Fri, 13 Nov 2026 03:43:12 +0000
Subject: [PATCH 2/3] [VectorUtils] Preserve cache-hint metadata

---
 llvm/include/llvm/Analysis/VectorUtils.h      |  11 +-
 llvm/include/llvm/IR/MemCacheHint.h           |   9 ++
 llvm/lib/Analysis/VectorUtils.cpp             |   7 ++
 llvm/lib/IR/MemCacheHint.cpp                  |  53 ++++++++-
 .../LoadStoreVectorizer/NVPTX/cache-hint.ll   |  29 ++---
 .../SLPVectorizer/X86/cache-hint.ll           |  23 ++--
 llvm/unittests/Analysis/VectorUtilsTest.cpp   |  86 ++++++++++++++-
 llvm/unittests/IR/MemCacheHintTest.cpp        | 103 ++++++++++++++++++
 8 files changed, 277 insertions(+), 44 deletions(-)

diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h
index 9e31ca33ce95f..dd41af9f84086 100644
--- a/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/llvm/include/llvm/Analysis/VectorUtils.h
@@ -403,14 +403,9 @@ LLVM_ABI void getMetadataToPropagate(
     Instruction *Inst,
     SmallVectorImpl<std::pair<unsigned, MDNode *>> &Metadata);
 
-/// Specifically, let Kinds = [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath,
-/// MD_nontemporal, MD_access_group, MD_mmra].
-/// For K in Kinds, we get the MDNode for K from each of the
-/// elements of VL, compute their "intersection" (i.e., the most generic
-/// metadata value that covers all of the individual values), and set I's
-/// metadata for M equal to the intersection value.
-///
-/// This function always sets a (possibly null) value for each K in Kinds.
+/// Propagate metadata that can be preserved after vectorization from \p VL to
+/// \p I, retaining only information valid for all sources and the destination.
+/// Return \p I.
 LLVM_ABI Instruction *propagateMetadata(Instruction *I, ArrayRef<Value *> VL);
 
 /// Create a mask that filters the members of an interleave group where there
diff --git a/llvm/include/llvm/IR/MemCacheHint.h b/llvm/include/llvm/IR/MemCacheHint.h
index cb697d5cc9066..65b5f70b371c7 100644
--- a/llvm/include/llvm/IR/MemCacheHint.h
+++ b/llvm/include/llvm/IR/MemCacheHint.h
@@ -9,12 +9,14 @@
 #ifndef LLVM_IR_MEMCACHEHINT_H
 #define LLVM_IR_MEMCACHEHINT_H
 
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
 class Instruction;
 class MDNode;
+class Value;
 
 /// Return the hint payload for operand \p OperandNo of \p I, or nullptr if
 /// absent. Operand numbers are argument indices for intrinsic calls and
@@ -27,6 +29,13 @@ LLVM_ABI MDNode *getMemCacheHint(const Instruction &I, unsigned OperandNo = 0);
 /// attachment. Operand numbering follows getMemCacheHint().
 LLVM_ABI void setMemCacheHint(Instruction &I, unsigned OperandNo, MDNode *Hint);
 
+/// Copy identical source cache-hint attachments to \p Dest, or clear its
+/// attachment if propagation fails. All accesses must be supported reads (loads
+/// or masked loads/gathers) or writes (stores or masked stores/scatters), with
+/// matching pointer operand positions.
+LLVM_ABI void propagateMemCacheHint(Instruction &Dest,
+                                    ArrayRef<Value *> Sources);
+
 } // namespace llvm
 
 #endif // LLVM_IR_MEMCACHEHINT_H
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index d9065485a3eb6..9821c3daa280b 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -23,6 +23,8 @@
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/MemCacheHint.h"
 #include "llvm/IR/MemoryModelRelaxationAnnotations.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/IR/Value.h"
@@ -1123,6 +1125,11 @@ void llvm::getMetadataToPropagate(
 Instruction *llvm::propagateMetadata(Instruction *Inst, ArrayRef<Value *> VL) {
   if (VL.empty())
     return Inst;
+
+  // Keep cache hints out of getMetadataToPropagate(): VPlan changes operand
+  // layouts without remapping cache-hint indices.
+  propagateMemCacheHint(*Inst, VL);
+
   SmallVector<std::pair<unsigned, MDNode *>> Metadata;
   getMetadataToPropagate(cast<Instruction>(VL[0]), Metadata);
 
diff --git a/llvm/lib/IR/MemCacheHint.cpp b/llvm/lib/IR/MemCacheHint.cpp
index 87e1c6e6d9557..a2e9018630ad4 100644
--- a/llvm/lib/IR/MemCacheHint.cpp
+++ b/llvm/lib/IR/MemCacheHint.cpp
@@ -10,9 +10,11 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/IR/Constants.h"
-#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Metadata.h"
+#include <optional>
 #include <utility>
 
 using namespace llvm;
@@ -62,3 +64,52 @@ void llvm::setMemCacheHint(Instruction &I, unsigned OperandNo, MDNode *Hint) {
                 Operands.empty() ? nullptr
                                  : MDNode::get(I.getContext(), Operands));
 }
+
+// The supported accesses have one memory operand. Its position also
+// distinguishes reads (0) from writes (1).
+static std::optional<unsigned> getPointerOperandIndex(const Instruction &I) {
+  if (isa<LoadInst>(I))
+    return LoadInst::getPointerOperandIndex();
+  if (isa<StoreInst>(I))
+    return StoreInst::getPointerOperandIndex();
+  const auto *II = dyn_cast<IntrinsicInst>(&I);
+  if (!II)
+    return std::nullopt;
+  switch (II->getIntrinsicID()) {
+  case Intrinsic::masked_load:
+  case Intrinsic::masked_gather:
+    return 0;
+  case Intrinsic::masked_store:
+  case Intrinsic::masked_scatter:
+    return 1;
+  default:
+    return std::nullopt;
+  }
+}
+
+static MDNode *getCommonCacheHint(const Instruction &Dest,
+                                  ArrayRef<Value *> Sources) {
+  std::optional<unsigned> PointerIndex = getPointerOperandIndex(Dest);
+  if (!PointerIndex || Sources.empty())
+    return nullptr;
+
+  const auto *First = dyn_cast<Instruction>(Sources.front());
+  if (!First)
+    return nullptr;
+  MDNode *MD = First->getMetadata(LLVMContext::MD_mem_cache_hint);
+  if (!MD)
+    return nullptr;
+
+  for (Value *V : Sources) {
+    const auto *Source = dyn_cast<Instruction>(V);
+    if (!Source || getPointerOperandIndex(*Source) != PointerIndex ||
+        Source->getMetadata(LLVMContext::MD_mem_cache_hint) != MD)
+      return nullptr;
+  }
+  return MD;
+}
+
+void llvm::propagateMemCacheHint(Instruction &Dest, ArrayRef<Value *> Sources) {
+  Dest.setMetadata(LLVMContext::MD_mem_cache_hint,
+                   getCommonCacheHint(Dest, Sources));
+}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/cache-hint.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/cache-hint.ll
index 9408dd43e02d5..72cda0a087bc4 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/cache-hint.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/cache-hint.ll
@@ -2,10 +2,9 @@
 ; RUN: opt < %s -passes=load-store-vectorizer -S -mtriple=nvptx64-nvidia-cuda -mcpu=sm_100 -mattr=+ptx88 | FileCheck %s --implicit-check-not=mem.cache_hint
 
 define i32 @load_identical_hints(ptr addrspace(1) %p) {
-;
 ; CHECK-LABEL: define i32 @load_identical_hints(
 ; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[P]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[P]], align 8, !mem.cache_hint [[META0:![0-9]+]]
 ; CHECK-NEXT:    [[X1:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
 ; CHECK-NEXT:    [[Y2:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
 ; CHECK-NEXT:    [[SUM:%.*]] = add i32 [[X1]], [[Y2]]
@@ -19,7 +18,6 @@ define i32 @load_identical_hints(ptr addrspace(1) %p) {
 }
 
 define i32 @load_missing_hint(ptr addrspace(1) %p) {
-;
 ; CHECK-LABEL: define i32 @load_missing_hint(
 ; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[P]], align 8
@@ -36,7 +34,6 @@ define i32 @load_missing_hint(ptr addrspace(1) %p) {
 }
 
 define i32 @load_conflicting_hints(ptr addrspace(1) %p) {
-;
 ; CHECK-LABEL: define i32 @load_conflicting_hints(
 ; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[P]], align 8
@@ -53,12 +50,11 @@ define i32 @load_conflicting_hints(ptr addrspace(1) %p) {
 }
 
 define void @store_identical_hints(ptr addrspace(1) %p, i32 %x, i32 %y) {
-;
 ; CHECK-LABEL: define void @store_identical_hints(
 ; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i64 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[Y]], i64 1
-; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr addrspace(1) [[P]], align 8
+; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr addrspace(1) [[P]], align 8, !mem.cache_hint [[META2:![0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
   %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
@@ -68,7 +64,6 @@ define void @store_identical_hints(ptr addrspace(1) %p, i32 %x, i32 %y) {
 }
 
 define void @store_missing_hint(ptr addrspace(1) %p, i32 %x, i32 %y) {
-;
 ; CHECK-LABEL: define void @store_missing_hint(
 ; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i64 0
@@ -83,7 +78,6 @@ define void @store_missing_hint(ptr addrspace(1) %p, i32 %x, i32 %y) {
 }
 
 define void @store_conflicting_hints(ptr addrspace(1) %p, i32 %x, i32 %y) {
-;
 ; CHECK-LABEL: define void @store_conflicting_hints(
 ; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i64 0
@@ -98,7 +92,6 @@ define void @store_conflicting_hints(ptr addrspace(1) %p, i32 %x, i32 %y) {
 }
 
 define i32 @load_missing_first_hint(ptr addrspace(1) %p) {
-;
 ; CHECK-LABEL: define i32 @load_missing_first_hint(
 ; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[P]], align 8
@@ -115,7 +108,6 @@ define i32 @load_missing_first_hint(ptr addrspace(1) %p) {
 }
 
 define void @store_missing_first_hint(ptr addrspace(1) %p, i32 %x, i32 %y) {
-;
 ; CHECK-LABEL: define void @store_missing_first_hint(
 ; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i64 0
@@ -131,10 +123,9 @@ define void @store_missing_first_hint(ptr addrspace(1) %p, i32 %x, i32 %y) {
 
 ; Leave element 2 inactive in the resulting masked access.
 define i32 @masked_load_identical_hints(ptr addrspace(1) %p) {
-;
 ; CHECK-LABEL: define i32 @masked_load_identical_hints(
 ; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison)
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison), !mem.cache_hint [[META0]]
 ; CHECK-NEXT:    [[X1:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
 ; CHECK-NEXT:    [[Y2:%.*]] = extractelement <4 x i32> [[TMP1]], i64 1
 ; CHECK-NEXT:    [[GAPFILL3:%.*]] = extractelement <4 x i32> [[TMP1]], i64 2
@@ -155,7 +146,6 @@ define i32 @masked_load_identical_hints(ptr addrspace(1) %p) {
 
 ; Leave element 2 inactive in the resulting masked access.
 define i32 @masked_load_missing_first_hints(ptr addrspace(1) %p) {
-;
 ; CHECK-LABEL: define i32 @masked_load_missing_first_hints(
 ; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison)
@@ -179,7 +169,6 @@ define i32 @masked_load_missing_first_hints(ptr addrspace(1) %p) {
 
 ; Leave element 2 inactive in the resulting masked access.
 define i32 @masked_load_missing_last_hints(ptr addrspace(1) %p) {
-;
 ; CHECK-LABEL: define i32 @masked_load_missing_last_hints(
 ; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison)
@@ -203,7 +192,6 @@ define i32 @masked_load_missing_last_hints(ptr addrspace(1) %p) {
 
 ; Leave element 2 inactive in the resulting masked access.
 define i32 @masked_load_conflicting_hints(ptr addrspace(1) %p) {
-;
 ; CHECK-LABEL: define i32 @masked_load_conflicting_hints(
 ; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison)
@@ -227,14 +215,13 @@ define i32 @masked_load_conflicting_hints(ptr addrspace(1) %p) {
 
 ; Leave element 2 inactive in the resulting masked access.
 define void @masked_store_identical_hints(ptr addrspace(1) %p, i64 %x, i64 %y, i64 %z) {
-;
 ; CHECK-LABEL: define void @masked_store_identical_hints(
 ; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i64 [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i64> [[TMP1]], i64 [[Y]], i64 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i64> [[TMP2]], i64 poison, i64 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i64> [[TMP3]], i64 [[Z]], i64 3
-; CHECK-NEXT:    call void @llvm.masked.store.v4i64.p1(<4 x i64> [[TMP4]], ptr addrspace(1) align 32 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>)
+; CHECK-NEXT:    call void @llvm.masked.store.v4i64.p1(<4 x i64> [[TMP4]], ptr addrspace(1) align 32 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>), !mem.cache_hint [[META2]]
 ; CHECK-NEXT:    ret void
 ;
   %p1 = getelementptr i64, ptr addrspace(1) %p, i64 1
@@ -247,7 +234,6 @@ define void @masked_store_identical_hints(ptr addrspace(1) %p, i64 %x, i64 %y, i
 
 ; Leave element 2 inactive in the resulting masked access.
 define void @masked_store_missing_first_hints(ptr addrspace(1) %p, i64 %x, i64 %y, i64 %z) {
-;
 ; CHECK-LABEL: define void @masked_store_missing_first_hints(
 ; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i64 [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
@@ -267,7 +253,6 @@ define void @masked_store_missing_first_hints(ptr addrspace(1) %p, i64 %x, i64 %
 
 ; Leave element 2 inactive in the resulting masked access.
 define void @masked_store_missing_last_hints(ptr addrspace(1) %p, i64 %x, i64 %y, i64 %z) {
-;
 ; CHECK-LABEL: define void @masked_store_missing_last_hints(
 ; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i64 [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
@@ -287,7 +272,6 @@ define void @masked_store_missing_last_hints(ptr addrspace(1) %p, i64 %x, i64 %y
 
 ; Leave element 2 inactive in the resulting masked access.
 define void @masked_store_conflicting_hints(ptr addrspace(1) %p, i64 %x, i64 %y, i64 %z) {
-;
 ; CHECK-LABEL: define void @masked_store_conflicting_hints(
 ; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i64 [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
@@ -311,3 +295,8 @@ define void @masked_store_conflicting_hints(ptr addrspace(1) %p, i64 %x, i64 %y,
 !3 = !{i32 1, !5}
 !4 = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_cache_hint", i64 1508705875169116160}
 !5 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_cache_hint", i64 1364590687093260288}
+;.
+; CHECK: [[META0]] = !{i32 0, [[META1:![0-9]+]]}
+; CHECK: [[META1]] = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_cache_hint", i64 1508705875169116160}
+; CHECK: [[META2]] = !{i32 1, [[META1]]}
+;.
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cache-hint.ll b/llvm/test/Transforms/SLPVectorizer/X86/cache-hint.ll
index 124136398b0d6..a810d88390927 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/cache-hint.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cache-hint.ll
@@ -2,10 +2,9 @@
 ; RUN: opt -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=znver4 < %s | FileCheck %s --implicit-check-not=mem.cache_hint
 
 define void @load_identical_hints(ptr noalias %src, ptr noalias %dst) {
-;
 ; CHECK-LABEL: define void @load_identical_hints(
 ; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8, !mem.cache_hint [[META0:![0-9]+]]
 ; CHECK-NEXT:    store <2 x double> [[TMP1]], ptr [[DST]], align 8
 ; CHECK-NEXT:    ret void
 ;
@@ -19,7 +18,6 @@ define void @load_identical_hints(ptr noalias %src, ptr noalias %dst) {
 }
 
 define void @load_missing_first_hint(ptr noalias %src, ptr noalias %dst) {
-;
 ; CHECK-LABEL: define void @load_missing_first_hint(
 ; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
@@ -36,7 +34,6 @@ define void @load_missing_first_hint(ptr noalias %src, ptr noalias %dst) {
 }
 
 define void @load_missing_last_hint(ptr noalias %src, ptr noalias %dst) {
-;
 ; CHECK-LABEL: define void @load_missing_last_hint(
 ; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
@@ -53,7 +50,6 @@ define void @load_missing_last_hint(ptr noalias %src, ptr noalias %dst) {
 }
 
 define void @load_conflicting_hints(ptr noalias %src, ptr noalias %dst) {
-;
 ; CHECK-LABEL: define void @load_conflicting_hints(
 ; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
@@ -70,11 +66,10 @@ define void @load_conflicting_hints(ptr noalias %src, ptr noalias %dst) {
 }
 
 define void @store_identical_hints(ptr noalias %src, ptr noalias %dst) {
-;
 ; CHECK-LABEL: define void @store_identical_hints(
 ; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
-; CHECK-NEXT:    store <2 x double> [[TMP1]], ptr [[DST]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP1]], ptr [[DST]], align 8, !mem.cache_hint [[META2:![0-9]+]]
 ; CHECK-NEXT:    ret void
 ;
   %src1 = getelementptr inbounds double, ptr %src, i64 1
@@ -87,7 +82,6 @@ define void @store_identical_hints(ptr noalias %src, ptr noalias %dst) {
 }
 
 define void @store_missing_first_hint(ptr noalias %src, ptr noalias %dst) {
-;
 ; CHECK-LABEL: define void @store_missing_first_hint(
 ; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
@@ -104,7 +98,6 @@ define void @store_missing_first_hint(ptr noalias %src, ptr noalias %dst) {
 }
 
 define void @store_missing_last_hint(ptr noalias %src, ptr noalias %dst) {
-;
 ; CHECK-LABEL: define void @store_missing_last_hint(
 ; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
@@ -121,7 +114,6 @@ define void @store_missing_last_hint(ptr noalias %src, ptr noalias %dst) {
 }
 
 define void @store_conflicting_hints(ptr noalias %src, ptr noalias %dst) {
-;
 ; CHECK-LABEL: define void @store_conflicting_hints(
 ; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
@@ -138,14 +130,13 @@ define void @store_conflicting_hints(ptr noalias %src, ptr noalias %dst) {
 }
 
 define void @gather_identical_hints(ptr noalias %src, ptr noalias %dst) {
-;
 ; CHECK-LABEL: define void @gather_identical_hints(
 ; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x ptr> poison, ptr [[SRC]], i64 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x ptr> [[TMP1]], <8 x ptr> poison, <8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds double, <8 x ptr> [[TMP2]], <8 x i64> <i64 0, i64 3, i64 7, i64 11, i64 18, i64 21, i64 27, i64 32>
 ; CHECK-NEXT:    [[DST0:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 0
-; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> align 8 [[TMP3]], <8 x i1> splat (i1 true), <8 x double> poison)
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> align 8 [[TMP3]], <8 x i1> splat (i1 true), <8 x double> poison), !mem.cache_hint [[META0]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = fadd <8 x double> [[TMP4]], splat (double 1.000000e+00)
 ; CHECK-NEXT:    store <8 x double> [[TMP5]], ptr [[DST0]], align 8
 ; CHECK-NEXT:    ret void
@@ -194,13 +185,12 @@ define void @gather_identical_hints(ptr noalias %src, ptr noalias %dst) {
 }
 
 define void @masked_store_identical_hints(ptr noalias %src, ptr noalias %dst) {
-;
 ; CHECK-LABEL: define void @masked_store_identical_hints(
 ; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[V:%.*]] = load <8 x i32>, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[DST0:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[V]], <8 x i32> poison, <15 x i32> <i32 0, i32 poison, i32 1, i32 poison, i32 2, i32 poison, i32 3, i32 poison, i32 4, i32 poison, i32 5, i32 poison, i32 6, i32 poison, i32 7>
-; CHECK-NEXT:    call void @llvm.masked.store.v15i32.p0(<15 x i32> [[TMP1]], ptr align 4 [[DST0]], <15 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true>)
+; CHECK-NEXT:    call void @llvm.masked.store.v15i32.p0(<15 x i32> [[TMP1]], ptr align 4 [[DST0]], <15 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true>), !mem.cache_hint [[META2]]
 ; CHECK-NEXT:    ret void
 ;
   %v = load <8 x i32>, ptr %src, align 4
@@ -237,3 +227,8 @@ define void @masked_store_identical_hints(ptr noalias %src, ptr noalias %dst) {
 !3 = !{i32 1, !5}
 !4 = !{!"nvvm.l1_eviction", !"last"}
 !5 = !{!"nvvm.l1_eviction", !"first"}
+;.
+; CHECK: [[META0]] = !{i32 0, [[META1:![0-9]+]]}
+; CHECK: [[META1]] = !{!"nvvm.l1_eviction", !"last"}
+; CHECK: [[META2]] = !{i32 1, [[META1]]}
+;.
diff --git a/llvm/unittests/Analysis/VectorUtilsTest.cpp b/llvm/unittests/Analysis/VectorUtilsTest.cpp
index b4d74f715b2db..a9fe68f8a0aca 100644
--- a/llvm/unittests/Analysis/VectorUtilsTest.cpp
+++ b/llvm/unittests/Analysis/VectorUtilsTest.cpp
@@ -10,11 +10,12 @@
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/IR/Function.h"
-#include "llvm/IR/InstIterator.h"
 #include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstIterator.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/NoFolder.h"
+#include "llvm/IR/Verifier.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/SourceMgr.h"
 #include "gtest/gtest.h"
@@ -75,6 +76,89 @@ struct BasicTest : public testing::Test {
 
 } // namespace
 
+TEST_F(VectorUtilsTest, PropagateCacheHintToMaskedLoad) {
+  parseAssembly(R"(
+    define <4 x i32> @test(ptr %p, <4 x i1> %mask) {
+      %A = load <4 x i32>, ptr %p, !mem.cache_hint !0
+      %masked = call <4 x i32> @llvm.masked.load.v4i32.p0(
+          ptr %p, <4 x i1> %mask, <4 x i32> poison)
+      ret <4 x i32> %masked
+    }
+    declare <4 x i32> @llvm.masked.load.v4i32.p0(
+        ptr, <4 x i1>, <4 x i32>)
+    !0 = !{i32 0, !1}
+    !1 = !{!"test.cache", i32 1}
+  )");
+
+  Instruction *Masked = A->getNextNode();
+  propagateMetadata(Masked, {A});
+  EXPECT_EQ(Masked->getMetadata(LLVMContext::MD_mem_cache_hint),
+            A->getMetadata(LLVMContext::MD_mem_cache_hint));
+  EXPECT_FALSE(verifyModule(*M, &errs()));
+}
+
+TEST_F(VectorUtilsTest, PropagateCacheHintToMaskedScatter) {
+  parseAssembly(R"(
+    define void @test(ptr %p, <4 x ptr> %ptrs, <4 x i32> %value,
+                      <4 x i1> %mask) {
+      %A = getelementptr i32, ptr %p, i32 0
+      store <4 x i32> %value, ptr %A, !mem.cache_hint !0
+      call void @llvm.masked.scatter.v4i32.v4p0(
+          <4 x i32> %value, <4 x ptr> %ptrs, <4 x i1> %mask)
+      ret void
+    }
+    declare void @llvm.masked.scatter.v4i32.v4p0(
+        <4 x i32>, <4 x ptr>, <4 x i1>)
+    !0 = !{i32 1, !1}
+    !1 = !{!"test.cache", i32 1}
+  )");
+
+  Instruction *Store = A->getNextNode();
+  Instruction *Scatter = Store->getNextNode();
+  propagateMetadata(Scatter, {Store});
+  EXPECT_EQ(Scatter->getMetadata(LLVMContext::MD_mem_cache_hint),
+            Store->getMetadata(LLVMContext::MD_mem_cache_hint));
+  EXPECT_FALSE(verifyModule(*M, &errs()));
+}
+
+TEST_F(VectorUtilsTest, DoNotPropagateCacheHintToTargetIntrinsic) {
+  parseAssembly(R"(
+    define <32 x i32> @test(ptr %p, i1 %pred) {
+      %A = load <32 x i32>, ptr %p, !mem.cache_hint !0
+      %predicated = call <32 x i32> @llvm.hexagon.V6.vL32b.pred.ai.128B(
+          i1 %pred, ptr %p, i32 0)
+      ret <32 x i32> %predicated
+    }
+    declare <32 x i32> @llvm.hexagon.V6.vL32b.pred.ai.128B(i1, ptr, i32)
+    !0 = !{i32 0, !1}
+    !1 = !{!"test.cache", i32 1}
+  )");
+
+  Instruction *Predicated = A->getNextNode();
+  // Hexagon vector alignment creates this intrinsic from ordinary loads, but
+  // its pointer is operand 1. Copying the source hint would name the predicate.
+  propagateMetadata(Predicated, {A});
+  EXPECT_EQ(Predicated->getMetadata(LLVMContext::MD_mem_cache_hint), nullptr);
+  EXPECT_FALSE(verifyModule(*M, &errs()));
+}
+
+TEST_F(VectorUtilsTest, DoNotPropagateStoreCacheHintToLoad) {
+  parseAssembly(R"(
+    define <4 x i32> @test(ptr %p, <4 x i32> %value) {
+      store <4 x i32> %value, ptr %p, !mem.cache_hint !0
+      %A = load <4 x i32>, ptr %p
+      ret <4 x i32> %A
+    }
+    !0 = !{i32 1, !1}
+    !1 = !{!"test.cache", i32 1}
+  )");
+
+  // A load synthesized while expanding a store must not inherit its policy.
+  propagateMetadata(A, {A->getPrevNode()});
+  EXPECT_EQ(A->getMetadata(LLVMContext::MD_mem_cache_hint), nullptr);
+  EXPECT_FALSE(verifyModule(*M, &errs()));
+}
+
 TEST_F(BasicTest, isSplat) {
   Value *UndefVec = UndefValue::get(FixedVectorType::get(IRB.getInt8Ty(), 4));
   EXPECT_TRUE(isSplatValue(UndefVec));
diff --git a/llvm/unittests/IR/MemCacheHintTest.cpp b/llvm/unittests/IR/MemCacheHintTest.cpp
index 37ba7f4d37f5c..549c2ac69fb3a 100644
--- a/llvm/unittests/IR/MemCacheHintTest.cpp
+++ b/llvm/unittests/IR/MemCacheHintTest.cpp
@@ -142,4 +142,107 @@ TEST_F(MemCacheHintTest, RemoveAbsentHint) {
   EXPECT_EQ(Copy->getMetadata(LLVMContext::MD_mem_cache_hint), MD);
 }
 
+TEST_F(MemCacheHintTest, CommonReadHints) {
+  auto *Ty = FixedVectorType::get(Builder.getInt32Ty(), 4);
+  Value *Mask = Builder.CreateVectorSplat(4, Builder.getTrue());
+  Value *Pointers = Builder.CreateVectorSplat(4, Source);
+  auto *Load = Builder.CreateLoad(Ty, Source);
+  auto *MaskedLoad = Builder.CreateMaskedLoad(Ty, Source, Align(4), Mask);
+  auto *Gather = Builder.CreateMaskedGather(Ty, Pointers, Align(4), Mask);
+  Value *Sources[] = {Load, MaskedLoad, Gather};
+  MDNode *Hint = hint("read");
+  for (Value *V : Sources)
+    setMemCacheHint(*cast<Instruction>(V), 0, Hint);
+  auto *Target = Builder.CreateLoad(Ty, Dest);
+  propagateMemCacheHint(*Target, Sources);
+  EXPECT_EQ(Target->getMetadata(LLVMContext::MD_mem_cache_hint),
+            attachment(0, Hint));
+}
+
+TEST_F(MemCacheHintTest, CommonWriteHints) {
+  Value *Data = Builder.CreateVectorSplat(4, Builder.getInt32(1));
+  Value *Mask = Builder.CreateVectorSplat(4, Builder.getTrue());
+  Value *Pointers = Builder.CreateVectorSplat(4, Dest);
+  auto *Store = Builder.CreateStore(Data, Dest);
+  auto *MaskedStore = Builder.CreateMaskedStore(Data, Dest, Align(4), Mask);
+  auto *Scatter = Builder.CreateMaskedScatter(Data, Pointers, Align(4), Mask);
+  Value *Sources[] = {Store, MaskedStore, Scatter};
+  MDNode *Hint = hint("write");
+  for (Value *V : Sources)
+    setMemCacheHint(*cast<Instruction>(V), 1, Hint);
+  auto *Target = Builder.CreateStore(Data, Dest);
+  setMemCacheHint(*Target, 1, hint("old"));
+  propagateMemCacheHint(*Target, Sources);
+  EXPECT_EQ(Target->getMetadata(LLVMContext::MD_mem_cache_hint),
+            attachment(1, Hint));
+}
+
+TEST_F(MemCacheHintTest, MissingOrConflictingHints) {
+  auto *A = Builder.CreateLoad(Builder.getInt32Ty(), Source);
+  auto *B = Builder.CreateLoad(Builder.getInt32Ty(), Source);
+  auto *Target = Builder.CreateLoad(Builder.getInt32Ty(), Source);
+  Value *Sources[] = {A, B};
+  MDNode *Hint = hint("read");
+  setMemCacheHint(*A, 0, Hint);
+  setMemCacheHint(*Target, 0, hint("old"));
+  propagateMemCacheHint(*Target, Sources);
+  EXPECT_EQ(Target->getMetadata(LLVMContext::MD_mem_cache_hint), nullptr);
+  setMemCacheHint(*B, 0, hint("different"));
+  setMemCacheHint(*Target, 0, hint("old"));
+  propagateMemCacheHint(*Target, Sources);
+  EXPECT_EQ(Target->getMetadata(LLVMContext::MD_mem_cache_hint), nullptr);
+  setMemCacheHint(*B, 0, Hint);
+  propagateMemCacheHint(*Target, Sources);
+  EXPECT_EQ(Target->getMetadata(LLVMContext::MD_mem_cache_hint),
+            attachment(0, Hint));
+  setMemCacheHint(*A, 0, nullptr);
+  propagateMemCacheHint(*Target, Sources);
+  EXPECT_EQ(Target->getMetadata(LLVMContext::MD_mem_cache_hint), nullptr);
+}
+
+TEST_F(MemCacheHintTest, ReadWriteMismatch) {
+  auto *Load = Builder.CreateLoad(Builder.getInt32Ty(), Source);
+  auto *Store = Builder.CreateStore(Builder.getInt32(1), Dest);
+  auto *TargetLoad = Builder.CreateLoad(Builder.getInt32Ty(), Source);
+  auto *TargetStore = Builder.CreateStore(Builder.getInt32(1), Dest);
+  MDNode *Read = hint("read"), *Write = hint("write");
+  setMemCacheHint(*Load, 0, Read);
+  setMemCacheHint(*Store, 1, Write);
+  setMemCacheHint(*TargetLoad, 0, Read);
+  setMemCacheHint(*TargetStore, 1, Write);
+  propagateMemCacheHint(*TargetLoad, {Store});
+  propagateMemCacheHint(*TargetStore, {Load});
+  EXPECT_EQ(TargetLoad->getMetadata(LLVMContext::MD_mem_cache_hint), nullptr);
+  EXPECT_EQ(TargetStore->getMetadata(LLVMContext::MD_mem_cache_hint), nullptr);
+}
+
+TEST_F(MemCacheHintTest, UnsupportedAccess) {
+  auto *Load = Builder.CreateLoad(Builder.getInt32Ty(), Source);
+  auto *Copy = createMemCpy();
+  auto *Target = Builder.CreateLoad(Builder.getInt32Ty(), Source);
+  MDNode *Hint = hint("read");
+  setMemCacheHint(*Load, 0, Hint);
+  setMemCacheHint(*Copy, 0, Hint);
+  propagateMemCacheHint(*Copy, {Load});
+  EXPECT_EQ(Copy->getMetadata(LLVMContext::MD_mem_cache_hint), nullptr);
+  setMemCacheHint(*Copy, 0, Hint);
+  setMemCacheHint(*Target, 0, Hint);
+  propagateMemCacheHint(*Target, {Load, Copy});
+  EXPECT_EQ(Target->getMetadata(LLVMContext::MD_mem_cache_hint), nullptr);
+}
+
+TEST_F(MemCacheHintTest, EmptyOrNonInstructionSources) {
+  auto *Load = Builder.CreateLoad(Builder.getInt32Ty(), Source);
+  MDNode *Hint = hint("read");
+  setMemCacheHint(*Load, 0, Hint);
+  propagateMemCacheHint(*Load, {});
+  EXPECT_EQ(Load->getMetadata(LLVMContext::MD_mem_cache_hint), nullptr);
+  setMemCacheHint(*Load, 0, Hint);
+  propagateMemCacheHint(*Load, {Builder.getInt32(0)});
+  EXPECT_EQ(Load->getMetadata(LLVMContext::MD_mem_cache_hint), nullptr);
+  setMemCacheHint(*Load, 0, Hint);
+  propagateMemCacheHint(*Load, {Load, Builder.getInt32(0)});
+  EXPECT_EQ(Load->getMetadata(LLVMContext::MD_mem_cache_hint), nullptr);
+}
+
 } // namespace

>From 362d151c3af21be0675d02bc98e71e2035f8b2c7 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Sat, 14 Nov 2026 00:45:30 +0000
Subject: [PATCH 3/3] [VectorUtils] Stabilize cache-hint vectorizer tests

Use the byte-array GEP element type emitted on main so the gather checks also pass in premerge CI. Document that sm_100 and ptx88 are necessary to exercise 256-bit masked stores.
---
 .../LoadStoreVectorizer/NVPTX/cache-hint.ll    |  1 +
 .../Transforms/SLPVectorizer/X86/cache-hint.ll | 18 +++++++++---------
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/cache-hint.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/cache-hint.ll
index 72cda0a087bc4..e2d6c58a80cc3 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/cache-hint.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/cache-hint.ll
@@ -1,4 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; sm_100 and ptx88 are required for the 256-bit masked stores below.
 ; RUN: opt < %s -passes=load-store-vectorizer -S -mtriple=nvptx64-nvidia-cuda -mcpu=sm_100 -mattr=+ptx88 | FileCheck %s --implicit-check-not=mem.cache_hint
 
 define i32 @load_identical_hints(ptr addrspace(1) %p) {
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cache-hint.ll b/llvm/test/Transforms/SLPVectorizer/X86/cache-hint.ll
index a810d88390927..e2dc02df39362 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/cache-hint.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cache-hint.ll
@@ -134,28 +134,28 @@ define void @gather_identical_hints(ptr noalias %src, ptr noalias %dst) {
 ; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x ptr> poison, ptr [[SRC]], i64 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x ptr> [[TMP1]], <8 x ptr> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds double, <8 x ptr> [[TMP2]], <8 x i64> <i64 0, i64 3, i64 7, i64 11, i64 18, i64 21, i64 27, i64 32>
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [8 x i8], <8 x ptr> [[TMP2]], <8 x i64> <i64 0, i64 3, i64 7, i64 11, i64 18, i64 21, i64 27, i64 32>
 ; CHECK-NEXT:    [[DST0:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> align 8 [[TMP3]], <8 x i1> splat (i1 true), <8 x double> poison), !mem.cache_hint [[META0]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = fadd <8 x double> [[TMP4]], splat (double 1.000000e+00)
 ; CHECK-NEXT:    store <8 x double> [[TMP5]], ptr [[DST0]], align 8
 ; CHECK-NEXT:    ret void
 ;
-  %src0 = getelementptr inbounds double, ptr %src, i64 0
+  %src0 = getelementptr inbounds [8 x i8], ptr %src, i64 0
   %v0 = load double, ptr %src0, align 8, !mem.cache_hint !0
-  %src1 = getelementptr inbounds double, ptr %src, i64 3
+  %src1 = getelementptr inbounds [8 x i8], ptr %src, i64 3
   %v1 = load double, ptr %src1, align 8, !mem.cache_hint !0
-  %src2 = getelementptr inbounds double, ptr %src, i64 7
+  %src2 = getelementptr inbounds [8 x i8], ptr %src, i64 7
   %v2 = load double, ptr %src2, align 8, !mem.cache_hint !0
-  %src3 = getelementptr inbounds double, ptr %src, i64 11
+  %src3 = getelementptr inbounds [8 x i8], ptr %src, i64 11
   %v3 = load double, ptr %src3, align 8, !mem.cache_hint !0
-  %src4 = getelementptr inbounds double, ptr %src, i64 18
+  %src4 = getelementptr inbounds [8 x i8], ptr %src, i64 18
   %v4 = load double, ptr %src4, align 8, !mem.cache_hint !0
-  %src5 = getelementptr inbounds double, ptr %src, i64 21
+  %src5 = getelementptr inbounds [8 x i8], ptr %src, i64 21
   %v5 = load double, ptr %src5, align 8, !mem.cache_hint !0
-  %src6 = getelementptr inbounds double, ptr %src, i64 27
+  %src6 = getelementptr inbounds [8 x i8], ptr %src, i64 27
   %v6 = load double, ptr %src6, align 8, !mem.cache_hint !0
-  %src7 = getelementptr inbounds double, ptr %src, i64 32
+  %src7 = getelementptr inbounds [8 x i8], ptr %src, i64 32
   %v7 = load double, ptr %src7, align 8, !mem.cache_hint !0
   %a0 = fadd double %v0, 1.0
   %a1 = fadd double %v1, 1.0



More information about the llvm-branch-commits mailing list