[llvm-branch-commits] [llvm] [VectorUtils] Propagate cache hint metadata (PR #241114)
Yonah Goldberg via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Nov 12 22:49:45 PST 2026
https://github.com/YonahGoldberg updated https://github.com/llvm/llvm-project/pull/241114
>From ffc51f2873cfcf013a65022f4b12383ab66a1292 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Fri, 13 Nov 2026 03:42:17 +0000
Subject: [PATCH 1/2] [VectorUtils] Add cache-hint tests
---
.../LoadStoreVectorizer/NVPTX/cache-hint.ll | 313 ++++++++++++++++++
.../SLPVectorizer/X86/cache-hint.ll | 239 +++++++++++++
2 files changed, 552 insertions(+)
create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/NVPTX/cache-hint.ll
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/cache-hint.ll
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/cache-hint.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/cache-hint.ll
new file mode 100644
index 0000000000000..9408dd43e02d5
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/cache-hint.ll
@@ -0,0 +1,313 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=load-store-vectorizer -S -mtriple=nvptx64-nvidia-cuda -mcpu=sm_100 -mattr=+ptx88 | FileCheck %s --implicit-check-not=mem.cache_hint
+
+define i32 @load_identical_hints(ptr addrspace(1) %p) {
+;
+; CHECK-LABEL: define i32 @load_identical_hints(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[P]], align 8
+; CHECK-NEXT: [[X1:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[Y2:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[X1]], [[Y2]]
+; CHECK-NEXT: ret i32 [[SUM]]
+;
+ %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
+ %x = load i32, ptr addrspace(1) %p, align 8, !mem.cache_hint !0
+ %y = load i32, ptr addrspace(1) %p1, align 4, !mem.cache_hint !0
+ %sum = add i32 %x, %y
+ ret i32 %sum
+}
+
+define i32 @load_missing_hint(ptr addrspace(1) %p) {
+;
+; CHECK-LABEL: define i32 @load_missing_hint(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[P]], align 8
+; CHECK-NEXT: [[X1:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[Y2:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[X1]], [[Y2]]
+; CHECK-NEXT: ret i32 [[SUM]]
+;
+ %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
+ %x = load i32, ptr addrspace(1) %p, align 8, !mem.cache_hint !0
+ %y = load i32, ptr addrspace(1) %p1, align 4
+ %sum = add i32 %x, %y
+ ret i32 %sum
+}
+
+define i32 @load_conflicting_hints(ptr addrspace(1) %p) {
+;
+; CHECK-LABEL: define i32 @load_conflicting_hints(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[P]], align 8
+; CHECK-NEXT: [[X1:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[Y2:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[X1]], [[Y2]]
+; CHECK-NEXT: ret i32 [[SUM]]
+;
+ %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
+ %x = load i32, ptr addrspace(1) %p, align 8, !mem.cache_hint !0
+ %y = load i32, ptr addrspace(1) %p1, align 4, !mem.cache_hint !1
+ %sum = add i32 %x, %y
+ ret i32 %sum
+}
+
+define void @store_identical_hints(ptr addrspace(1) %p, i32 %x, i32 %y) {
+;
+; CHECK-LABEL: define void @store_identical_hints(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[Y]], i64 1
+; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr addrspace(1) [[P]], align 8
+; CHECK-NEXT: ret void
+;
+ %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
+ store i32 %x, ptr addrspace(1) %p, align 8, !mem.cache_hint !2
+ store i32 %y, ptr addrspace(1) %p1, align 4, !mem.cache_hint !2
+ ret void
+}
+
+define void @store_missing_hint(ptr addrspace(1) %p, i32 %x, i32 %y) {
+;
+; CHECK-LABEL: define void @store_missing_hint(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[Y]], i64 1
+; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr addrspace(1) [[P]], align 8
+; CHECK-NEXT: ret void
+;
+ %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
+ store i32 %x, ptr addrspace(1) %p, align 8, !mem.cache_hint !2
+ store i32 %y, ptr addrspace(1) %p1, align 4
+ ret void
+}
+
+define void @store_conflicting_hints(ptr addrspace(1) %p, i32 %x, i32 %y) {
+;
+; CHECK-LABEL: define void @store_conflicting_hints(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[Y]], i64 1
+; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr addrspace(1) [[P]], align 8
+; CHECK-NEXT: ret void
+;
+ %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
+ store i32 %x, ptr addrspace(1) %p, align 8, !mem.cache_hint !2
+ store i32 %y, ptr addrspace(1) %p1, align 4, !mem.cache_hint !3
+ ret void
+}
+
+define i32 @load_missing_first_hint(ptr addrspace(1) %p) {
+;
+; CHECK-LABEL: define i32 @load_missing_first_hint(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[P]], align 8
+; CHECK-NEXT: [[X1:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[Y2:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[X1]], [[Y2]]
+; CHECK-NEXT: ret i32 [[SUM]]
+;
+ %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
+ %x = load i32, ptr addrspace(1) %p, align 8
+ %y = load i32, ptr addrspace(1) %p1, align 4, !mem.cache_hint !0
+ %sum = add i32 %x, %y
+ ret i32 %sum
+}
+
+define void @store_missing_first_hint(ptr addrspace(1) %p, i32 %x, i32 %y) {
+;
+; CHECK-LABEL: define void @store_missing_first_hint(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[Y]], i64 1
+; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr addrspace(1) [[P]], align 8
+; CHECK-NEXT: ret void
+;
+ %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
+ store i32 %x, ptr addrspace(1) %p, align 8
+ store i32 %y, ptr addrspace(1) %p1, align 4, !mem.cache_hint !2
+ ret void
+}
+
+; Leave element 2 inactive in the resulting masked access.
+define i32 @masked_load_identical_hints(ptr addrspace(1) %p) {
+;
+; CHECK-LABEL: define i32 @masked_load_identical_hints(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison)
+; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: [[GAPFILL3:%.*]] = extractelement <4 x i32> [[TMP1]], i64 2
+; CHECK-NEXT: [[Z4:%.*]] = extractelement <4 x i32> [[TMP1]], i64 3
+; CHECK-NEXT: [[XY:%.*]] = add i32 [[X1]], [[Y2]]
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[XY]], [[Z4]]
+; CHECK-NEXT: ret i32 [[SUM]]
+;
+ %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
+ %p3 = getelementptr i32, ptr addrspace(1) %p, i64 3
+ %x = load i32, ptr addrspace(1) %p, align 16, !mem.cache_hint !0
+ %y = load i32, ptr addrspace(1) %p1, align 4, !mem.cache_hint !0
+ %z = load i32, ptr addrspace(1) %p3, align 4, !mem.cache_hint !0
+ %xy = add i32 %x, %y
+ %sum = add i32 %xy, %z
+ ret i32 %sum
+}
+
+; Leave element 2 inactive in the resulting masked access.
+define i32 @masked_load_missing_first_hints(ptr addrspace(1) %p) {
+;
+; CHECK-LABEL: define i32 @masked_load_missing_first_hints(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison)
+; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: [[GAPFILL3:%.*]] = extractelement <4 x i32> [[TMP1]], i64 2
+; CHECK-NEXT: [[Z4:%.*]] = extractelement <4 x i32> [[TMP1]], i64 3
+; CHECK-NEXT: [[XY:%.*]] = add i32 [[X1]], [[Y2]]
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[XY]], [[Z4]]
+; CHECK-NEXT: ret i32 [[SUM]]
+;
+ %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
+ %p3 = getelementptr i32, ptr addrspace(1) %p, i64 3
+ %x = load i32, ptr addrspace(1) %p, align 16
+ %y = load i32, ptr addrspace(1) %p1, align 4, !mem.cache_hint !0
+ %z = load i32, ptr addrspace(1) %p3, align 4, !mem.cache_hint !0
+ %xy = add i32 %x, %y
+ %sum = add i32 %xy, %z
+ ret i32 %sum
+}
+
+; Leave element 2 inactive in the resulting masked access.
+define i32 @masked_load_missing_last_hints(ptr addrspace(1) %p) {
+;
+; CHECK-LABEL: define i32 @masked_load_missing_last_hints(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison)
+; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: [[GAPFILL3:%.*]] = extractelement <4 x i32> [[TMP1]], i64 2
+; CHECK-NEXT: [[Z4:%.*]] = extractelement <4 x i32> [[TMP1]], i64 3
+; CHECK-NEXT: [[XY:%.*]] = add i32 [[X1]], [[Y2]]
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[XY]], [[Z4]]
+; CHECK-NEXT: ret i32 [[SUM]]
+;
+ %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
+ %p3 = getelementptr i32, ptr addrspace(1) %p, i64 3
+ %x = load i32, ptr addrspace(1) %p, align 16, !mem.cache_hint !0
+ %y = load i32, ptr addrspace(1) %p1, align 4, !mem.cache_hint !0
+ %z = load i32, ptr addrspace(1) %p3, align 4
+ %xy = add i32 %x, %y
+ %sum = add i32 %xy, %z
+ ret i32 %sum
+}
+
+; Leave element 2 inactive in the resulting masked access.
+define i32 @masked_load_conflicting_hints(ptr addrspace(1) %p) {
+;
+; CHECK-LABEL: define i32 @masked_load_conflicting_hints(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison)
+; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x i32> [[TMP1]], i64 1
+; CHECK-NEXT: [[GAPFILL3:%.*]] = extractelement <4 x i32> [[TMP1]], i64 2
+; CHECK-NEXT: [[Z4:%.*]] = extractelement <4 x i32> [[TMP1]], i64 3
+; CHECK-NEXT: [[XY:%.*]] = add i32 [[X1]], [[Y2]]
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[XY]], [[Z4]]
+; CHECK-NEXT: ret i32 [[SUM]]
+;
+ %p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
+ %p3 = getelementptr i32, ptr addrspace(1) %p, i64 3
+ %x = load i32, ptr addrspace(1) %p, align 16, !mem.cache_hint !0
+ %y = load i32, ptr addrspace(1) %p1, align 4, !mem.cache_hint !0
+ %z = load i32, ptr addrspace(1) %p3, align 4, !mem.cache_hint !1
+ %xy = add i32 %x, %y
+ %sum = add i32 %xy, %z
+ ret i32 %sum
+}
+
+; Leave element 2 inactive in the resulting masked access.
+define void @masked_store_identical_hints(ptr addrspace(1) %p, i64 %x, i64 %y, i64 %z) {
+;
+; CHECK-LABEL: define void @masked_store_identical_hints(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i64 [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> [[TMP1]], i64 [[Y]], i64 1
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i64> [[TMP2]], i64 poison, i64 2
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> [[TMP3]], i64 [[Z]], i64 3
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p1(<4 x i64> [[TMP4]], ptr addrspace(1) align 32 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>)
+; CHECK-NEXT: ret void
+;
+ %p1 = getelementptr i64, ptr addrspace(1) %p, i64 1
+ %p3 = getelementptr i64, ptr addrspace(1) %p, i64 3
+ store i64 %x, ptr addrspace(1) %p, align 32, !mem.cache_hint !2
+ store i64 %y, ptr addrspace(1) %p1, align 8, !mem.cache_hint !2
+ store i64 %z, ptr addrspace(1) %p3, align 8, !mem.cache_hint !2
+ ret void
+}
+
+; Leave element 2 inactive in the resulting masked access.
+define void @masked_store_missing_first_hints(ptr addrspace(1) %p, i64 %x, i64 %y, i64 %z) {
+;
+; CHECK-LABEL: define void @masked_store_missing_first_hints(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i64 [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> [[TMP1]], i64 [[Y]], i64 1
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i64> [[TMP2]], i64 poison, i64 2
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> [[TMP3]], i64 [[Z]], i64 3
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p1(<4 x i64> [[TMP4]], ptr addrspace(1) align 32 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>)
+; CHECK-NEXT: ret void
+;
+ %p1 = getelementptr i64, ptr addrspace(1) %p, i64 1
+ %p3 = getelementptr i64, ptr addrspace(1) %p, i64 3
+ store i64 %x, ptr addrspace(1) %p, align 32
+ store i64 %y, ptr addrspace(1) %p1, align 8, !mem.cache_hint !2
+ store i64 %z, ptr addrspace(1) %p3, align 8, !mem.cache_hint !2
+ ret void
+}
+
+; Leave element 2 inactive in the resulting masked access.
+define void @masked_store_missing_last_hints(ptr addrspace(1) %p, i64 %x, i64 %y, i64 %z) {
+;
+; CHECK-LABEL: define void @masked_store_missing_last_hints(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i64 [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> [[TMP1]], i64 [[Y]], i64 1
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i64> [[TMP2]], i64 poison, i64 2
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> [[TMP3]], i64 [[Z]], i64 3
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p1(<4 x i64> [[TMP4]], ptr addrspace(1) align 32 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>)
+; CHECK-NEXT: ret void
+;
+ %p1 = getelementptr i64, ptr addrspace(1) %p, i64 1
+ %p3 = getelementptr i64, ptr addrspace(1) %p, i64 3
+ store i64 %x, ptr addrspace(1) %p, align 32, !mem.cache_hint !2
+ store i64 %y, ptr addrspace(1) %p1, align 8, !mem.cache_hint !2
+ store i64 %z, ptr addrspace(1) %p3, align 8
+ ret void
+}
+
+; Leave element 2 inactive in the resulting masked access.
+define void @masked_store_conflicting_hints(ptr addrspace(1) %p, i64 %x, i64 %y, i64 %z) {
+;
+; CHECK-LABEL: define void @masked_store_conflicting_hints(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i64 [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> [[TMP1]], i64 [[Y]], i64 1
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i64> [[TMP2]], i64 poison, i64 2
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> [[TMP3]], i64 [[Z]], i64 3
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p1(<4 x i64> [[TMP4]], ptr addrspace(1) align 32 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>)
+; CHECK-NEXT: ret void
+;
+ %p1 = getelementptr i64, ptr addrspace(1) %p, i64 1
+ %p3 = getelementptr i64, ptr addrspace(1) %p, i64 3
+ store i64 %x, ptr addrspace(1) %p, align 32, !mem.cache_hint !2
+ store i64 %y, ptr addrspace(1) %p1, align 8, !mem.cache_hint !2
+ store i64 %z, ptr addrspace(1) %p3, align 8, !mem.cache_hint !3
+ ret void
+}
+
+!0 = !{i32 0, !4}
+!1 = !{i32 0, !5}
+!2 = !{i32 1, !4}
+!3 = !{i32 1, !5}
+!4 = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_cache_hint", i64 1508705875169116160}
+!5 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_cache_hint", i64 1364590687093260288}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cache-hint.ll b/llvm/test/Transforms/SLPVectorizer/X86/cache-hint.ll
new file mode 100644
index 0000000000000..124136398b0d6
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cache-hint.ll
@@ -0,0 +1,239 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=znver4 < %s | FileCheck %s --implicit-check-not=mem.cache_hint
+
+define void @load_identical_hints(ptr noalias %src, ptr noalias %dst) {
+;
+; CHECK-LABEL: define void @load_identical_hints(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
+; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[DST]], align 8
+; CHECK-NEXT: ret void
+;
+ %src1 = getelementptr inbounds double, ptr %src, i64 1
+ %dst1 = getelementptr inbounds double, ptr %dst, i64 1
+ %v0 = load double, ptr %src, align 8, !mem.cache_hint !0
+ %v1 = load double, ptr %src1, align 8, !mem.cache_hint !0
+ store double %v0, ptr %dst, align 8
+ store double %v1, ptr %dst1, align 8
+ ret void
+}
+
+define void @load_missing_first_hint(ptr noalias %src, ptr noalias %dst) {
+;
+; CHECK-LABEL: define void @load_missing_first_hint(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
+; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[DST]], align 8
+; CHECK-NEXT: ret void
+;
+ %src1 = getelementptr inbounds double, ptr %src, i64 1
+ %dst1 = getelementptr inbounds double, ptr %dst, i64 1
+ %v0 = load double, ptr %src, align 8
+ %v1 = load double, ptr %src1, align 8, !mem.cache_hint !0
+ store double %v0, ptr %dst, align 8
+ store double %v1, ptr %dst1, align 8
+ ret void
+}
+
+define void @load_missing_last_hint(ptr noalias %src, ptr noalias %dst) {
+;
+; CHECK-LABEL: define void @load_missing_last_hint(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
+; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[DST]], align 8
+; CHECK-NEXT: ret void
+;
+ %src1 = getelementptr inbounds double, ptr %src, i64 1
+ %dst1 = getelementptr inbounds double, ptr %dst, i64 1
+ %v0 = load double, ptr %src, align 8, !mem.cache_hint !0
+ %v1 = load double, ptr %src1, align 8
+ store double %v0, ptr %dst, align 8
+ store double %v1, ptr %dst1, align 8
+ ret void
+}
+
+define void @load_conflicting_hints(ptr noalias %src, ptr noalias %dst) {
+;
+; CHECK-LABEL: define void @load_conflicting_hints(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
+; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[DST]], align 8
+; CHECK-NEXT: ret void
+;
+ %src1 = getelementptr inbounds double, ptr %src, i64 1
+ %dst1 = getelementptr inbounds double, ptr %dst, i64 1
+ %v0 = load double, ptr %src, align 8, !mem.cache_hint !0
+ %v1 = load double, ptr %src1, align 8, !mem.cache_hint !1
+ store double %v0, ptr %dst, align 8
+ store double %v1, ptr %dst1, align 8
+ ret void
+}
+
+define void @store_identical_hints(ptr noalias %src, ptr noalias %dst) {
+;
+; CHECK-LABEL: define void @store_identical_hints(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
+; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[DST]], align 8
+; CHECK-NEXT: ret void
+;
+ %src1 = getelementptr inbounds double, ptr %src, i64 1
+ %dst1 = getelementptr inbounds double, ptr %dst, i64 1
+ %v0 = load double, ptr %src, align 8
+ %v1 = load double, ptr %src1, align 8
+ store double %v0, ptr %dst, align 8, !mem.cache_hint !2
+ store double %v1, ptr %dst1, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @store_missing_first_hint(ptr noalias %src, ptr noalias %dst) {
+;
+; CHECK-LABEL: define void @store_missing_first_hint(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
+; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[DST]], align 8
+; CHECK-NEXT: ret void
+;
+ %src1 = getelementptr inbounds double, ptr %src, i64 1
+ %dst1 = getelementptr inbounds double, ptr %dst, i64 1
+ %v0 = load double, ptr %src, align 8
+ %v1 = load double, ptr %src1, align 8
+ store double %v0, ptr %dst, align 8
+ store double %v1, ptr %dst1, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @store_missing_last_hint(ptr noalias %src, ptr noalias %dst) {
+;
+; CHECK-LABEL: define void @store_missing_last_hint(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
+; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[DST]], align 8
+; CHECK-NEXT: ret void
+;
+ %src1 = getelementptr inbounds double, ptr %src, i64 1
+ %dst1 = getelementptr inbounds double, ptr %dst, i64 1
+ %v0 = load double, ptr %src, align 8
+ %v1 = load double, ptr %src1, align 8
+ store double %v0, ptr %dst, align 8, !mem.cache_hint !2
+ store double %v1, ptr %dst1, align 8
+ ret void
+}
+
+define void @store_conflicting_hints(ptr noalias %src, ptr noalias %dst) {
+;
+; CHECK-LABEL: define void @store_conflicting_hints(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
+; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[DST]], align 8
+; CHECK-NEXT: ret void
+;
+ %src1 = getelementptr inbounds double, ptr %src, i64 1
+ %dst1 = getelementptr inbounds double, ptr %dst, i64 1
+ %v0 = load double, ptr %src, align 8
+ %v1 = load double, ptr %src1, align 8
+ store double %v0, ptr %dst, align 8, !mem.cache_hint !2
+ store double %v1, ptr %dst1, align 8, !mem.cache_hint !3
+ ret void
+}
+
+define void @gather_identical_hints(ptr noalias %src, ptr noalias %dst) {
+;
+; CHECK-LABEL: define void @gather_identical_hints(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <8 x ptr> poison, ptr [[SRC]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x ptr> [[TMP1]], <8 x ptr> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, <8 x ptr> [[TMP2]], <8 x i64> <i64 0, i64 3, i64 7, i64 11, i64 18, i64 21, i64 27, i64 32>
+; CHECK-NEXT: [[DST0:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> align 8 [[TMP3]], <8 x i1> splat (i1 true), <8 x double> poison)
+; CHECK-NEXT: [[TMP5:%.*]] = fadd <8 x double> [[TMP4]], splat (double 1.000000e+00)
+; CHECK-NEXT: store <8 x double> [[TMP5]], ptr [[DST0]], align 8
+; CHECK-NEXT: ret void
+;
+ %src0 = getelementptr inbounds double, ptr %src, i64 0
+ %v0 = load double, ptr %src0, align 8, !mem.cache_hint !0
+ %src1 = getelementptr inbounds double, ptr %src, i64 3
+ %v1 = load double, ptr %src1, align 8, !mem.cache_hint !0
+ %src2 = getelementptr inbounds double, ptr %src, i64 7
+ %v2 = load double, ptr %src2, align 8, !mem.cache_hint !0
+ %src3 = getelementptr inbounds double, ptr %src, i64 11
+ %v3 = load double, ptr %src3, align 8, !mem.cache_hint !0
+ %src4 = getelementptr inbounds double, ptr %src, i64 18
+ %v4 = load double, ptr %src4, align 8, !mem.cache_hint !0
+ %src5 = getelementptr inbounds double, ptr %src, i64 21
+ %v5 = load double, ptr %src5, align 8, !mem.cache_hint !0
+ %src6 = getelementptr inbounds double, ptr %src, i64 27
+ %v6 = load double, ptr %src6, align 8, !mem.cache_hint !0
+ %src7 = getelementptr inbounds double, ptr %src, i64 32
+ %v7 = load double, ptr %src7, align 8, !mem.cache_hint !0
+ %a0 = fadd double %v0, 1.0
+ %a1 = fadd double %v1, 1.0
+ %a2 = fadd double %v2, 1.0
+ %a3 = fadd double %v3, 1.0
+ %a4 = fadd double %v4, 1.0
+ %a5 = fadd double %v5, 1.0
+ %a6 = fadd double %v6, 1.0
+ %a7 = fadd double %v7, 1.0
+ %dst0 = getelementptr inbounds double, ptr %dst, i64 0
+ store double %a0, ptr %dst0, align 8
+ %dst1 = getelementptr inbounds double, ptr %dst, i64 1
+ store double %a1, ptr %dst1, align 8
+ %dst2 = getelementptr inbounds double, ptr %dst, i64 2
+ store double %a2, ptr %dst2, align 8
+ %dst3 = getelementptr inbounds double, ptr %dst, i64 3
+ store double %a3, ptr %dst3, align 8
+ %dst4 = getelementptr inbounds double, ptr %dst, i64 4
+ store double %a4, ptr %dst4, align 8
+ %dst5 = getelementptr inbounds double, ptr %dst, i64 5
+ store double %a5, ptr %dst5, align 8
+ %dst6 = getelementptr inbounds double, ptr %dst, i64 6
+ store double %a6, ptr %dst6, align 8
+ %dst7 = getelementptr inbounds double, ptr %dst, i64 7
+ store double %a7, ptr %dst7, align 8
+ ret void
+}
+
+define void @masked_store_identical_hints(ptr noalias %src, ptr noalias %dst) {
+;
+; CHECK-LABEL: define void @masked_store_identical_hints(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[V:%.*]] = load <8 x i32>, ptr [[SRC]], align 4
+; CHECK-NEXT: [[DST0:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[V]], <8 x i32> poison, <15 x i32> <i32 0, i32 poison, i32 1, i32 poison, i32 2, i32 poison, i32 3, i32 poison, i32 4, i32 poison, i32 5, i32 poison, i32 6, i32 poison, i32 7>
+; CHECK-NEXT: call void @llvm.masked.store.v15i32.p0(<15 x i32> [[TMP1]], ptr align 4 [[DST0]], <15 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true>)
+; CHECK-NEXT: ret void
+;
+ %v = load <8 x i32>, ptr %src, align 4
+ %e0 = extractelement <8 x i32> %v, i32 0
+ %dst0 = getelementptr inbounds i32, ptr %dst, i64 0
+ %e1 = extractelement <8 x i32> %v, i32 1
+ %dst1 = getelementptr inbounds i32, ptr %dst, i64 2
+ %e2 = extractelement <8 x i32> %v, i32 2
+ %dst2 = getelementptr inbounds i32, ptr %dst, i64 4
+ %e3 = extractelement <8 x i32> %v, i32 3
+ %dst3 = getelementptr inbounds i32, ptr %dst, i64 6
+ %e4 = extractelement <8 x i32> %v, i32 4
+ %dst4 = getelementptr inbounds i32, ptr %dst, i64 8
+ %e5 = extractelement <8 x i32> %v, i32 5
+ %dst5 = getelementptr inbounds i32, ptr %dst, i64 10
+ %e6 = extractelement <8 x i32> %v, i32 6
+ %dst6 = getelementptr inbounds i32, ptr %dst, i64 12
+ %e7 = extractelement <8 x i32> %v, i32 7
+ %dst7 = getelementptr inbounds i32, ptr %dst, i64 14
+ store i32 %e0, ptr %dst0, align 4, !mem.cache_hint !2
+ store i32 %e1, ptr %dst1, align 4, !mem.cache_hint !2
+ store i32 %e2, ptr %dst2, align 4, !mem.cache_hint !2
+ store i32 %e3, ptr %dst3, align 4, !mem.cache_hint !2
+ store i32 %e4, ptr %dst4, align 4, !mem.cache_hint !2
+ store i32 %e5, ptr %dst5, align 4, !mem.cache_hint !2
+ store i32 %e6, ptr %dst6, align 4, !mem.cache_hint !2
+ store i32 %e7, ptr %dst7, align 4, !mem.cache_hint !2
+ ret void
+}
+
+!0 = !{i32 0, !4}
+!1 = !{i32 0, !5}
+!2 = !{i32 1, !4}
+!3 = !{i32 1, !5}
+!4 = !{!"nvvm.l1_eviction", !"last"}
+!5 = !{!"nvvm.l1_eviction", !"first"}
>From 6c38f308c6bdcbead353cd1b18d4aead42b9db31 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Fri, 13 Nov 2026 03:43:12 +0000
Subject: [PATCH 2/2] [VectorUtils] Preserve cache-hint metadata
---
llvm/include/llvm/Analysis/VectorUtils.h | 11 +-
llvm/include/llvm/IR/MemCacheHint.h | 9 ++
llvm/lib/Analysis/VectorUtils.cpp | 7 ++
llvm/lib/IR/MemCacheHint.cpp | 53 ++++++++-
.../LoadStoreVectorizer/NVPTX/cache-hint.ll | 29 ++---
.../SLPVectorizer/X86/cache-hint.ll | 23 ++--
llvm/unittests/Analysis/VectorUtilsTest.cpp | 86 ++++++++++++++-
llvm/unittests/IR/MemCacheHintTest.cpp | 103 ++++++++++++++++++
8 files changed, 277 insertions(+), 44 deletions(-)
diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h
index 9e31ca33ce95f..dd41af9f84086 100644
--- a/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/llvm/include/llvm/Analysis/VectorUtils.h
@@ -403,14 +403,9 @@ LLVM_ABI void getMetadataToPropagate(
Instruction *Inst,
SmallVectorImpl<std::pair<unsigned, MDNode *>> &Metadata);
-/// Specifically, let Kinds = [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath,
-/// MD_nontemporal, MD_access_group, MD_mmra].
-/// For K in Kinds, we get the MDNode for K from each of the
-/// elements of VL, compute their "intersection" (i.e., the most generic
-/// metadata value that covers all of the individual values), and set I's
-/// metadata for M equal to the intersection value.
-///
-/// This function always sets a (possibly null) value for each K in Kinds.
+/// Propagate metadata that can be preserved after vectorization from \p VL to
+/// \p I, retaining only information valid for all sources and the destination.
+/// Return \p I.
LLVM_ABI Instruction *propagateMetadata(Instruction *I, ArrayRef<Value *> VL);
/// Create a mask that filters the members of an interleave group where there
diff --git a/llvm/include/llvm/IR/MemCacheHint.h b/llvm/include/llvm/IR/MemCacheHint.h
index cb697d5cc9066..65b5f70b371c7 100644
--- a/llvm/include/llvm/IR/MemCacheHint.h
+++ b/llvm/include/llvm/IR/MemCacheHint.h
@@ -9,12 +9,14 @@
#ifndef LLVM_IR_MEMCACHEHINT_H
#define LLVM_IR_MEMCACHEHINT_H
+#include "llvm/ADT/ArrayRef.h"
#include "llvm/Support/Compiler.h"
namespace llvm {
class Instruction;
class MDNode;
+class Value;
/// Return the hint payload for operand \p OperandNo of \p I, or nullptr if
/// absent. Operand numbers are argument indices for intrinsic calls and
@@ -27,6 +29,13 @@ LLVM_ABI MDNode *getMemCacheHint(const Instruction &I, unsigned OperandNo = 0);
/// attachment. Operand numbering follows getMemCacheHint().
LLVM_ABI void setMemCacheHint(Instruction &I, unsigned OperandNo, MDNode *Hint);
+/// Copy identical source cache-hint attachments to \p Dest, or clear its
+/// attachment if propagation fails. All accesses must be supported reads (loads
+/// or masked loads/gathers) or writes (stores or masked stores/scatters), with
+/// matching pointer operand positions.
+LLVM_ABI void propagateMemCacheHint(Instruction &Dest,
+ ArrayRef<Value *> Sources);
+
} // namespace llvm
#endif // LLVM_IR_MEMCACHEHINT_H
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index d9065485a3eb6..9821c3daa280b 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -23,6 +23,8 @@
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/MemCacheHint.h"
#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/Value.h"
@@ -1123,6 +1125,11 @@ void llvm::getMetadataToPropagate(
Instruction *llvm::propagateMetadata(Instruction *Inst, ArrayRef<Value *> VL) {
if (VL.empty())
return Inst;
+
+ // Keep cache hints out of getMetadataToPropagate(): VPlan changes operand
+ // layouts without remapping cache-hint indices.
+ propagateMemCacheHint(*Inst, VL);
+
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
getMetadataToPropagate(cast<Instruction>(VL[0]), Metadata);
diff --git a/llvm/lib/IR/MemCacheHint.cpp b/llvm/lib/IR/MemCacheHint.cpp
index 87e1c6e6d9557..a2e9018630ad4 100644
--- a/llvm/lib/IR/MemCacheHint.cpp
+++ b/llvm/lib/IR/MemCacheHint.cpp
@@ -10,9 +10,11 @@
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/IR/Constants.h"
-#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Metadata.h"
+#include <optional>
#include <utility>
using namespace llvm;
@@ -62,3 +64,52 @@ void llvm::setMemCacheHint(Instruction &I, unsigned OperandNo, MDNode *Hint) {
Operands.empty() ? nullptr
: MDNode::get(I.getContext(), Operands));
}
+
+// The supported accesses have one memory operand. Its position also
+// distinguishes reads (0) from writes (1).
+static std::optional<unsigned> getPointerOperandIndex(const Instruction &I) {
+ if (isa<LoadInst>(I))
+ return LoadInst::getPointerOperandIndex();
+ if (isa<StoreInst>(I))
+ return StoreInst::getPointerOperandIndex();
+ const auto *II = dyn_cast<IntrinsicInst>(&I);
+ if (!II)
+ return std::nullopt;
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::masked_load:
+ case Intrinsic::masked_gather:
+ return 0;
+ case Intrinsic::masked_store:
+ case Intrinsic::masked_scatter:
+ return 1;
+ default:
+ return std::nullopt;
+ }
+}
+
+static MDNode *getCommonCacheHint(const Instruction &Dest,
+ ArrayRef<Value *> Sources) {
+ std::optional<unsigned> PointerIndex = getPointerOperandIndex(Dest);
+ if (!PointerIndex || Sources.empty())
+ return nullptr;
+
+ const auto *First = dyn_cast<Instruction>(Sources.front());
+ if (!First)
+ return nullptr;
+ MDNode *MD = First->getMetadata(LLVMContext::MD_mem_cache_hint);
+ if (!MD)
+ return nullptr;
+
+ for (Value *V : Sources) {
+ const auto *Source = dyn_cast<Instruction>(V);
+ if (!Source || getPointerOperandIndex(*Source) != PointerIndex ||
+ Source->getMetadata(LLVMContext::MD_mem_cache_hint) != MD)
+ return nullptr;
+ }
+ return MD;
+}
+
+void llvm::propagateMemCacheHint(Instruction &Dest, ArrayRef<Value *> Sources) {
+ Dest.setMetadata(LLVMContext::MD_mem_cache_hint,
+ getCommonCacheHint(Dest, Sources));
+}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/cache-hint.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/cache-hint.ll
index 9408dd43e02d5..72cda0a087bc4 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/cache-hint.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/cache-hint.ll
@@ -2,10 +2,9 @@
; RUN: opt < %s -passes=load-store-vectorizer -S -mtriple=nvptx64-nvidia-cuda -mcpu=sm_100 -mattr=+ptx88 | FileCheck %s --implicit-check-not=mem.cache_hint
define i32 @load_identical_hints(ptr addrspace(1) %p) {
-;
; CHECK-LABEL: define i32 @load_identical_hints(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[P]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[P]], align 8, !mem.cache_hint [[META0:![0-9]+]]
; CHECK-NEXT: [[X1:%.*]] = extractelement <2 x i32> [[TMP1]], i64 0
; CHECK-NEXT: [[Y2:%.*]] = extractelement <2 x i32> [[TMP1]], i64 1
; CHECK-NEXT: [[SUM:%.*]] = add i32 [[X1]], [[Y2]]
@@ -19,7 +18,6 @@ define i32 @load_identical_hints(ptr addrspace(1) %p) {
}
define i32 @load_missing_hint(ptr addrspace(1) %p) {
-;
; CHECK-LABEL: define i32 @load_missing_hint(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[P]], align 8
@@ -36,7 +34,6 @@ define i32 @load_missing_hint(ptr addrspace(1) %p) {
}
define i32 @load_conflicting_hints(ptr addrspace(1) %p) {
-;
; CHECK-LABEL: define i32 @load_conflicting_hints(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[P]], align 8
@@ -53,12 +50,11 @@ define i32 @load_conflicting_hints(ptr addrspace(1) %p) {
}
define void @store_identical_hints(ptr addrspace(1) %p, i32 %x, i32 %y) {
-;
; CHECK-LABEL: define void @store_identical_hints(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[Y]], i64 1
-; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr addrspace(1) [[P]], align 8
+; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr addrspace(1) [[P]], align 8, !mem.cache_hint [[META2:![0-9]+]]
; CHECK-NEXT: ret void
;
%p1 = getelementptr i32, ptr addrspace(1) %p, i64 1
@@ -68,7 +64,6 @@ define void @store_identical_hints(ptr addrspace(1) %p, i32 %x, i32 %y) {
}
define void @store_missing_hint(ptr addrspace(1) %p, i32 %x, i32 %y) {
-;
; CHECK-LABEL: define void @store_missing_hint(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i64 0
@@ -83,7 +78,6 @@ define void @store_missing_hint(ptr addrspace(1) %p, i32 %x, i32 %y) {
}
define void @store_conflicting_hints(ptr addrspace(1) %p, i32 %x, i32 %y) {
-;
; CHECK-LABEL: define void @store_conflicting_hints(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i64 0
@@ -98,7 +92,6 @@ define void @store_conflicting_hints(ptr addrspace(1) %p, i32 %x, i32 %y) {
}
define i32 @load_missing_first_hint(ptr addrspace(1) %p) {
-;
; CHECK-LABEL: define i32 @load_missing_first_hint(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[P]], align 8
@@ -115,7 +108,6 @@ define i32 @load_missing_first_hint(ptr addrspace(1) %p) {
}
define void @store_missing_first_hint(ptr addrspace(1) %p, i32 %x, i32 %y) {
-;
; CHECK-LABEL: define void @store_missing_first_hint(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i64 0
@@ -131,10 +123,9 @@ define void @store_missing_first_hint(ptr addrspace(1) %p, i32 %x, i32 %y) {
; Leave element 2 inactive in the resulting masked access.
define i32 @masked_load_identical_hints(ptr addrspace(1) %p) {
-;
; CHECK-LABEL: define i32 @masked_load_identical_hints(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison)
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison), !mem.cache_hint [[META0]]
; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x i32> [[TMP1]], i64 1
; CHECK-NEXT: [[GAPFILL3:%.*]] = extractelement <4 x i32> [[TMP1]], i64 2
@@ -155,7 +146,6 @@ define i32 @masked_load_identical_hints(ptr addrspace(1) %p) {
; Leave element 2 inactive in the resulting masked access.
define i32 @masked_load_missing_first_hints(ptr addrspace(1) %p) {
-;
; CHECK-LABEL: define i32 @masked_load_missing_first_hints(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison)
@@ -179,7 +169,6 @@ define i32 @masked_load_missing_first_hints(ptr addrspace(1) %p) {
; Leave element 2 inactive in the resulting masked access.
define i32 @masked_load_missing_last_hints(ptr addrspace(1) %p) {
-;
; CHECK-LABEL: define i32 @masked_load_missing_last_hints(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison)
@@ -203,7 +192,6 @@ define i32 @masked_load_missing_last_hints(ptr addrspace(1) %p) {
; Leave element 2 inactive in the resulting masked access.
define i32 @masked_load_conflicting_hints(ptr addrspace(1) %p) {
-;
; CHECK-LABEL: define i32 @masked_load_conflicting_hints(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison)
@@ -227,14 +215,13 @@ define i32 @masked_load_conflicting_hints(ptr addrspace(1) %p) {
; Leave element 2 inactive in the resulting masked access.
define void @masked_store_identical_hints(ptr addrspace(1) %p, i64 %x, i64 %y, i64 %z) {
-;
; CHECK-LABEL: define void @masked_store_identical_hints(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i64 [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> [[TMP1]], i64 [[Y]], i64 1
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i64> [[TMP2]], i64 poison, i64 2
; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> [[TMP3]], i64 [[Z]], i64 3
-; CHECK-NEXT: call void @llvm.masked.store.v4i64.p1(<4 x i64> [[TMP4]], ptr addrspace(1) align 32 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p1(<4 x i64> [[TMP4]], ptr addrspace(1) align 32 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>), !mem.cache_hint [[META2]]
; CHECK-NEXT: ret void
;
%p1 = getelementptr i64, ptr addrspace(1) %p, i64 1
@@ -247,7 +234,6 @@ define void @masked_store_identical_hints(ptr addrspace(1) %p, i64 %x, i64 %y, i
; Leave element 2 inactive in the resulting masked access.
define void @masked_store_missing_first_hints(ptr addrspace(1) %p, i64 %x, i64 %y, i64 %z) {
-;
; CHECK-LABEL: define void @masked_store_missing_first_hints(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i64 [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
@@ -267,7 +253,6 @@ define void @masked_store_missing_first_hints(ptr addrspace(1) %p, i64 %x, i64 %
; Leave element 2 inactive in the resulting masked access.
define void @masked_store_missing_last_hints(ptr addrspace(1) %p, i64 %x, i64 %y, i64 %z) {
-;
; CHECK-LABEL: define void @masked_store_missing_last_hints(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i64 [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
@@ -287,7 +272,6 @@ define void @masked_store_missing_last_hints(ptr addrspace(1) %p, i64 %x, i64 %y
; Leave element 2 inactive in the resulting masked access.
define void @masked_store_conflicting_hints(ptr addrspace(1) %p, i64 %x, i64 %y, i64 %z) {
-;
; CHECK-LABEL: define void @masked_store_conflicting_hints(
; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i64 [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
@@ -311,3 +295,8 @@ define void @masked_store_conflicting_hints(ptr addrspace(1) %p, i64 %x, i64 %y,
!3 = !{i32 1, !5}
!4 = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_cache_hint", i64 1508705875169116160}
!5 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_cache_hint", i64 1364590687093260288}
+;.
+; CHECK: [[META0]] = !{i32 0, [[META1:![0-9]+]]}
+; CHECK: [[META1]] = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_cache_hint", i64 1508705875169116160}
+; CHECK: [[META2]] = !{i32 1, [[META1]]}
+;.
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cache-hint.ll b/llvm/test/Transforms/SLPVectorizer/X86/cache-hint.ll
index 124136398b0d6..a810d88390927 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/cache-hint.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cache-hint.ll
@@ -2,10 +2,9 @@
; RUN: opt -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=znver4 < %s | FileCheck %s --implicit-check-not=mem.cache_hint
define void @load_identical_hints(ptr noalias %src, ptr noalias %dst) {
-;
; CHECK-LABEL: define void @load_identical_hints(
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8, !mem.cache_hint [[META0:![0-9]+]]
; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[DST]], align 8
; CHECK-NEXT: ret void
;
@@ -19,7 +18,6 @@ define void @load_identical_hints(ptr noalias %src, ptr noalias %dst) {
}
define void @load_missing_first_hint(ptr noalias %src, ptr noalias %dst) {
-;
; CHECK-LABEL: define void @load_missing_first_hint(
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
@@ -36,7 +34,6 @@ define void @load_missing_first_hint(ptr noalias %src, ptr noalias %dst) {
}
define void @load_missing_last_hint(ptr noalias %src, ptr noalias %dst) {
-;
; CHECK-LABEL: define void @load_missing_last_hint(
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
@@ -53,7 +50,6 @@ define void @load_missing_last_hint(ptr noalias %src, ptr noalias %dst) {
}
define void @load_conflicting_hints(ptr noalias %src, ptr noalias %dst) {
-;
; CHECK-LABEL: define void @load_conflicting_hints(
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
@@ -70,11 +66,10 @@ define void @load_conflicting_hints(ptr noalias %src, ptr noalias %dst) {
}
define void @store_identical_hints(ptr noalias %src, ptr noalias %dst) {
-;
; CHECK-LABEL: define void @store_identical_hints(
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
-; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[DST]], align 8
+; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[DST]], align 8, !mem.cache_hint [[META2:![0-9]+]]
; CHECK-NEXT: ret void
;
%src1 = getelementptr inbounds double, ptr %src, i64 1
@@ -87,7 +82,6 @@ define void @store_identical_hints(ptr noalias %src, ptr noalias %dst) {
}
define void @store_missing_first_hint(ptr noalias %src, ptr noalias %dst) {
-;
; CHECK-LABEL: define void @store_missing_first_hint(
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
@@ -104,7 +98,6 @@ define void @store_missing_first_hint(ptr noalias %src, ptr noalias %dst) {
}
define void @store_missing_last_hint(ptr noalias %src, ptr noalias %dst) {
-;
; CHECK-LABEL: define void @store_missing_last_hint(
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
@@ -121,7 +114,6 @@ define void @store_missing_last_hint(ptr noalias %src, ptr noalias %dst) {
}
define void @store_conflicting_hints(ptr noalias %src, ptr noalias %dst) {
-;
; CHECK-LABEL: define void @store_conflicting_hints(
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[SRC]], align 8
@@ -138,14 +130,13 @@ define void @store_conflicting_hints(ptr noalias %src, ptr noalias %dst) {
}
define void @gather_identical_hints(ptr noalias %src, ptr noalias %dst) {
-;
; CHECK-LABEL: define void @gather_identical_hints(
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <8 x ptr> poison, ptr [[SRC]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x ptr> [[TMP1]], <8 x ptr> poison, <8 x i32> zeroinitializer
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, <8 x ptr> [[TMP2]], <8 x i64> <i64 0, i64 3, i64 7, i64 11, i64 18, i64 21, i64 27, i64 32>
; CHECK-NEXT: [[DST0:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 0
-; CHECK-NEXT: [[TMP4:%.*]] = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> align 8 [[TMP3]], <8 x i1> splat (i1 true), <8 x double> poison)
+; CHECK-NEXT: [[TMP4:%.*]] = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> align 8 [[TMP3]], <8 x i1> splat (i1 true), <8 x double> poison), !mem.cache_hint [[META0]]
; CHECK-NEXT: [[TMP5:%.*]] = fadd <8 x double> [[TMP4]], splat (double 1.000000e+00)
; CHECK-NEXT: store <8 x double> [[TMP5]], ptr [[DST0]], align 8
; CHECK-NEXT: ret void
@@ -194,13 +185,12 @@ define void @gather_identical_hints(ptr noalias %src, ptr noalias %dst) {
}
define void @masked_store_identical_hints(ptr noalias %src, ptr noalias %dst) {
-;
; CHECK-LABEL: define void @masked_store_identical_hints(
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[V:%.*]] = load <8 x i32>, ptr [[SRC]], align 4
; CHECK-NEXT: [[DST0:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 0
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[V]], <8 x i32> poison, <15 x i32> <i32 0, i32 poison, i32 1, i32 poison, i32 2, i32 poison, i32 3, i32 poison, i32 4, i32 poison, i32 5, i32 poison, i32 6, i32 poison, i32 7>
-; CHECK-NEXT: call void @llvm.masked.store.v15i32.p0(<15 x i32> [[TMP1]], ptr align 4 [[DST0]], <15 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true>)
+; CHECK-NEXT: call void @llvm.masked.store.v15i32.p0(<15 x i32> [[TMP1]], ptr align 4 [[DST0]], <15 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true>), !mem.cache_hint [[META2]]
; CHECK-NEXT: ret void
;
%v = load <8 x i32>, ptr %src, align 4
@@ -237,3 +227,8 @@ define void @masked_store_identical_hints(ptr noalias %src, ptr noalias %dst) {
!3 = !{i32 1, !5}
!4 = !{!"nvvm.l1_eviction", !"last"}
!5 = !{!"nvvm.l1_eviction", !"first"}
+;.
+; CHECK: [[META0]] = !{i32 0, [[META1:![0-9]+]]}
+; CHECK: [[META1]] = !{!"nvvm.l1_eviction", !"last"}
+; CHECK: [[META2]] = !{i32 1, [[META1]]}
+;.
diff --git a/llvm/unittests/Analysis/VectorUtilsTest.cpp b/llvm/unittests/Analysis/VectorUtilsTest.cpp
index b4d74f715b2db..a9fe68f8a0aca 100644
--- a/llvm/unittests/Analysis/VectorUtilsTest.cpp
+++ b/llvm/unittests/Analysis/VectorUtilsTest.cpp
@@ -10,11 +10,12 @@
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/AsmParser/Parser.h"
#include "llvm/IR/Function.h"
-#include "llvm/IR/InstIterator.h"
#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstIterator.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/NoFolder.h"
+#include "llvm/IR/Verifier.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/SourceMgr.h"
#include "gtest/gtest.h"
@@ -75,6 +76,89 @@ struct BasicTest : public testing::Test {
} // namespace
+TEST_F(VectorUtilsTest, PropagateCacheHintToMaskedLoad) {
+ parseAssembly(R"(
+ define <4 x i32> @test(ptr %p, <4 x i1> %mask) {
+ %A = load <4 x i32>, ptr %p, !mem.cache_hint !0
+ %masked = call <4 x i32> @llvm.masked.load.v4i32.p0(
+ ptr %p, <4 x i1> %mask, <4 x i32> poison)
+ ret <4 x i32> %masked
+ }
+ declare <4 x i32> @llvm.masked.load.v4i32.p0(
+ ptr, <4 x i1>, <4 x i32>)
+ !0 = !{i32 0, !1}
+ !1 = !{!"test.cache", i32 1}
+ )");
+
+ Instruction *Masked = A->getNextNode();
+ propagateMetadata(Masked, {A});
+ EXPECT_EQ(Masked->getMetadata(LLVMContext::MD_mem_cache_hint),
+ A->getMetadata(LLVMContext::MD_mem_cache_hint));
+ EXPECT_FALSE(verifyModule(*M, &errs()));
+}
+
+TEST_F(VectorUtilsTest, PropagateCacheHintToMaskedScatter) {
+ parseAssembly(R"(
+ define void @test(ptr %p, <4 x ptr> %ptrs, <4 x i32> %value,
+ <4 x i1> %mask) {
+ %A = getelementptr i32, ptr %p, i32 0
+ store <4 x i32> %value, ptr %A, !mem.cache_hint !0
+ call void @llvm.masked.scatter.v4i32.v4p0(
+ <4 x i32> %value, <4 x ptr> %ptrs, <4 x i1> %mask)
+ ret void
+ }
+ declare void @llvm.masked.scatter.v4i32.v4p0(
+ <4 x i32>, <4 x ptr>, <4 x i1>)
+ !0 = !{i32 1, !1}
+ !1 = !{!"test.cache", i32 1}
+ )");
+
+ Instruction *Store = A->getNextNode();
+ Instruction *Scatter = Store->getNextNode();
+ propagateMetadata(Scatter, {Store});
+ EXPECT_EQ(Scatter->getMetadata(LLVMContext::MD_mem_cache_hint),
+ Store->getMetadata(LLVMContext::MD_mem_cache_hint));
+ EXPECT_FALSE(verifyModule(*M, &errs()));
+}
+
+TEST_F(VectorUtilsTest, DoNotPropagateCacheHintToTargetIntrinsic) {
+ parseAssembly(R"(
+ define <32 x i32> @test(ptr %p, i1 %pred) {
+ %A = load <32 x i32>, ptr %p, !mem.cache_hint !0
+ %predicated = call <32 x i32> @llvm.hexagon.V6.vL32b.pred.ai.128B(
+ i1 %pred, ptr %p, i32 0)
+ ret <32 x i32> %predicated
+ }
+ declare <32 x i32> @llvm.hexagon.V6.vL32b.pred.ai.128B(i1, ptr, i32)
+ !0 = !{i32 0, !1}
+ !1 = !{!"test.cache", i32 1}
+ )");
+
+ Instruction *Predicated = A->getNextNode();
+ // Hexagon vector alignment creates this intrinsic from ordinary loads, but
+ // its pointer is operand 1. Copying the source hint would name the predicate.
+ propagateMetadata(Predicated, {A});
+ EXPECT_EQ(Predicated->getMetadata(LLVMContext::MD_mem_cache_hint), nullptr);
+ EXPECT_FALSE(verifyModule(*M, &errs()));
+}
+
+TEST_F(VectorUtilsTest, DoNotPropagateStoreCacheHintToLoad) {
+ parseAssembly(R"(
+ define <4 x i32> @test(ptr %p, <4 x i32> %value) {
+ store <4 x i32> %value, ptr %p, !mem.cache_hint !0
+ %A = load <4 x i32>, ptr %p
+ ret <4 x i32> %A
+ }
+ !0 = !{i32 1, !1}
+ !1 = !{!"test.cache", i32 1}
+ )");
+
+ // A load synthesized while expanding a store must not inherit its policy.
+ propagateMetadata(A, {A->getPrevNode()});
+ EXPECT_EQ(A->getMetadata(LLVMContext::MD_mem_cache_hint), nullptr);
+ EXPECT_FALSE(verifyModule(*M, &errs()));
+}
+
TEST_F(BasicTest, isSplat) {
Value *UndefVec = UndefValue::get(FixedVectorType::get(IRB.getInt8Ty(), 4));
EXPECT_TRUE(isSplatValue(UndefVec));
diff --git a/llvm/unittests/IR/MemCacheHintTest.cpp b/llvm/unittests/IR/MemCacheHintTest.cpp
index 37ba7f4d37f5c..549c2ac69fb3a 100644
--- a/llvm/unittests/IR/MemCacheHintTest.cpp
+++ b/llvm/unittests/IR/MemCacheHintTest.cpp
@@ -142,4 +142,107 @@ TEST_F(MemCacheHintTest, RemoveAbsentHint) {
EXPECT_EQ(Copy->getMetadata(LLVMContext::MD_mem_cache_hint), MD);
}
+TEST_F(MemCacheHintTest, CommonReadHints) {
+ auto *Ty = FixedVectorType::get(Builder.getInt32Ty(), 4);
+ Value *Mask = Builder.CreateVectorSplat(4, Builder.getTrue());
+ Value *Pointers = Builder.CreateVectorSplat(4, Source);
+ auto *Load = Builder.CreateLoad(Ty, Source);
+ auto *MaskedLoad = Builder.CreateMaskedLoad(Ty, Source, Align(4), Mask);
+ auto *Gather = Builder.CreateMaskedGather(Ty, Pointers, Align(4), Mask);
+ Value *Sources[] = {Load, MaskedLoad, Gather};
+ MDNode *Hint = hint("read");
+ for (Value *V : Sources)
+ setMemCacheHint(*cast<Instruction>(V), 0, Hint);
+ auto *Target = Builder.CreateLoad(Ty, Dest);
+ propagateMemCacheHint(*Target, Sources);
+ EXPECT_EQ(Target->getMetadata(LLVMContext::MD_mem_cache_hint),
+ attachment(0, Hint));
+}
+
+TEST_F(MemCacheHintTest, CommonWriteHints) {
+ Value *Data = Builder.CreateVectorSplat(4, Builder.getInt32(1));
+ Value *Mask = Builder.CreateVectorSplat(4, Builder.getTrue());
+ Value *Pointers = Builder.CreateVectorSplat(4, Dest);
+ auto *Store = Builder.CreateStore(Data, Dest);
+ auto *MaskedStore = Builder.CreateMaskedStore(Data, Dest, Align(4), Mask);
+ auto *Scatter = Builder.CreateMaskedScatter(Data, Pointers, Align(4), Mask);
+ Value *Sources[] = {Store, MaskedStore, Scatter};
+ MDNode *Hint = hint("write");
+ for (Value *V : Sources)
+ setMemCacheHint(*cast<Instruction>(V), 1, Hint);
+ auto *Target = Builder.CreateStore(Data, Dest);
+ setMemCacheHint(*Target, 1, hint("old"));
+ propagateMemCacheHint(*Target, Sources);
+ EXPECT_EQ(Target->getMetadata(LLVMContext::MD_mem_cache_hint),
+ attachment(1, Hint));
+}
+
+TEST_F(MemCacheHintTest, MissingOrConflictingHints) {
+ auto *A = Builder.CreateLoad(Builder.getInt32Ty(), Source);
+ auto *B = Builder.CreateLoad(Builder.getInt32Ty(), Source);
+ auto *Target = Builder.CreateLoad(Builder.getInt32Ty(), Source);
+ Value *Sources[] = {A, B};
+ MDNode *Hint = hint("read");
+ setMemCacheHint(*A, 0, Hint);
+ setMemCacheHint(*Target, 0, hint("old"));
+ propagateMemCacheHint(*Target, Sources);
+ EXPECT_EQ(Target->getMetadata(LLVMContext::MD_mem_cache_hint), nullptr);
+ setMemCacheHint(*B, 0, hint("different"));
+ setMemCacheHint(*Target, 0, hint("old"));
+ propagateMemCacheHint(*Target, Sources);
+ EXPECT_EQ(Target->getMetadata(LLVMContext::MD_mem_cache_hint), nullptr);
+ setMemCacheHint(*B, 0, Hint);
+ propagateMemCacheHint(*Target, Sources);
+ EXPECT_EQ(Target->getMetadata(LLVMContext::MD_mem_cache_hint),
+ attachment(0, Hint));
+ setMemCacheHint(*A, 0, nullptr);
+ propagateMemCacheHint(*Target, Sources);
+ EXPECT_EQ(Target->getMetadata(LLVMContext::MD_mem_cache_hint), nullptr);
+}
+
+TEST_F(MemCacheHintTest, ReadWriteMismatch) {
+ auto *Load = Builder.CreateLoad(Builder.getInt32Ty(), Source);
+ auto *Store = Builder.CreateStore(Builder.getInt32(1), Dest);
+ auto *TargetLoad = Builder.CreateLoad(Builder.getInt32Ty(), Source);
+ auto *TargetStore = Builder.CreateStore(Builder.getInt32(1), Dest);
+ MDNode *Read = hint("read"), *Write = hint("write");
+ setMemCacheHint(*Load, 0, Read);
+ setMemCacheHint(*Store, 1, Write);
+ setMemCacheHint(*TargetLoad, 0, Read);
+ setMemCacheHint(*TargetStore, 1, Write);
+ propagateMemCacheHint(*TargetLoad, {Store});
+ propagateMemCacheHint(*TargetStore, {Load});
+ EXPECT_EQ(TargetLoad->getMetadata(LLVMContext::MD_mem_cache_hint), nullptr);
+ EXPECT_EQ(TargetStore->getMetadata(LLVMContext::MD_mem_cache_hint), nullptr);
+}
+
+TEST_F(MemCacheHintTest, UnsupportedAccess) {
+ auto *Load = Builder.CreateLoad(Builder.getInt32Ty(), Source);
+ auto *Copy = createMemCpy();
+ auto *Target = Builder.CreateLoad(Builder.getInt32Ty(), Source);
+ MDNode *Hint = hint("read");
+ setMemCacheHint(*Load, 0, Hint);
+ setMemCacheHint(*Copy, 0, Hint);
+ propagateMemCacheHint(*Copy, {Load});
+ EXPECT_EQ(Copy->getMetadata(LLVMContext::MD_mem_cache_hint), nullptr);
+ setMemCacheHint(*Copy, 0, Hint);
+ setMemCacheHint(*Target, 0, Hint);
+ propagateMemCacheHint(*Target, {Load, Copy});
+ EXPECT_EQ(Target->getMetadata(LLVMContext::MD_mem_cache_hint), nullptr);
+}
+
+TEST_F(MemCacheHintTest, EmptyOrNonInstructionSources) {
+ auto *Load = Builder.CreateLoad(Builder.getInt32Ty(), Source);
+ MDNode *Hint = hint("read");
+ setMemCacheHint(*Load, 0, Hint);
+ propagateMemCacheHint(*Load, {});
+ EXPECT_EQ(Load->getMetadata(LLVMContext::MD_mem_cache_hint), nullptr);
+ setMemCacheHint(*Load, 0, Hint);
+ propagateMemCacheHint(*Load, {Builder.getInt32(0)});
+ EXPECT_EQ(Load->getMetadata(LLVMContext::MD_mem_cache_hint), nullptr);
+ setMemCacheHint(*Load, 0, Hint);
+ propagateMemCacheHint(*Load, {Load, Builder.getInt32(0)});
+ EXPECT_EQ(Load->getMetadata(LLVMContext::MD_mem_cache_hint), nullptr);
+}
+
} // namespace
More information about the llvm-branch-commits
mailing list