[llvm] [SelectionDAG] Preserve cache hint metadata through DAGCombiner (PR #225954)
Yonah Goldberg via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 23 15:46:39 PDT 2026
https://github.com/YonahGoldberg updated https://github.com/llvm/llvm-project/pull/225954
>From 2f2f4e54280145321d4c2341be4e90a239c7f9d4 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 22 Sep 2026 21:42:21 +0000
Subject: [PATCH 1/5] [SelectionDAG] Add baseline cache hint DAG combine tests
---
.../AArch64/mem-cache-hint-dag-combine.ll | 41 ++
.../CodeGen/NVPTX/cache-hint-transforms.ll | 11 +-
.../X86/mem-cache-hint-dag-combine-i386.ll | 53 ++
.../CodeGen/X86/mem-cache-hint-dag-combine.ll | 461 ++++++++++++++++++
4 files changed, 565 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/AArch64/mem-cache-hint-dag-combine.ll
create mode 100644 llvm/test/CodeGen/X86/mem-cache-hint-dag-combine-i386.ll
create mode 100644 llvm/test/CodeGen/X86/mem-cache-hint-dag-combine.ll
diff --git a/llvm/test/CodeGen/AArch64/mem-cache-hint-dag-combine.ll b/llvm/test/CodeGen/AArch64/mem-cache-hint-dag-combine.ll
new file mode 100644
index 0000000000000..d4e508fc8e8fa
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/mem-cache-hint-dag-combine.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64 -stop-after=finalize-isel %s -o - | FileCheck %s
+
+define <8 x i8> @combine_insert_load_same(ptr %p) {
+ ; CHECK-LABEL: name: combine_insert_load_same
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-NEXT: [[LDRDui:%[0-9]+]]:fpr64 = LDRDui [[COPY]], 0 :: (load (s64) from %ir.p, align 1)
+ ; CHECK-NEXT: $d0 = COPY [[LDRDui]]
+ ; CHECK-NEXT: RET_ReallyLR implicit $d0
+ %p1 = getelementptr i8, ptr %p, i64 1
+ %vec = load <8 x i8>, ptr %p1, align 1, !mem.cache_hint !0
+ %scalar = load i8, ptr %p, align 1, !mem.cache_hint !0
+ %shuf = shufflevector <8 x i8> %vec, <8 x i8> poison, <8 x i32> <i32 poison, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6>
+ %r = insertelement <8 x i8> %shuf, i8 %scalar, i32 0
+ ret <8 x i8> %r
+}
+
+define <8 x i8> @combine_insert_load_different(ptr %p) {
+ ; CHECK-LABEL: name: combine_insert_load_different
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-NEXT: [[LDRDui:%[0-9]+]]:fpr64 = LDRDui [[COPY]], 0 :: (load (s64) from %ir.p, align 1)
+ ; CHECK-NEXT: $d0 = COPY [[LDRDui]]
+ ; CHECK-NEXT: RET_ReallyLR implicit $d0
+ %p1 = getelementptr i8, ptr %p, i64 1
+ %vec = load <8 x i8>, ptr %p1, align 1, !mem.cache_hint !0
+ %scalar = load i8, ptr %p, align 1, !mem.cache_hint !1
+ %shuf = shufflevector <8 x i8> %vec, <8 x i8> poison, <8 x i32> <i32 poison, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6>
+ %r = insertelement <8 x i8> %shuf, i8 %scalar, i32 0
+ ret <8 x i8> %r
+}
+
+!0 = !{i32 0, !10}
+!1 = !{i32 0, !11}
+!10 = !{!"test.cache_hint", !"first"}
+!11 = !{!"test.cache_hint", !"last"}
diff --git a/llvm/test/CodeGen/NVPTX/cache-hint-transforms.ll b/llvm/test/CodeGen/NVPTX/cache-hint-transforms.ll
index afa2342d025c7..94a0532ae1816 100644
--- a/llvm/test/CodeGen/NVPTX/cache-hint-transforms.ll
+++ b/llvm/test/CodeGen/NVPTX/cache-hint-transforms.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "^\s*(?:mov\.b64|ld(?:\.[A-Za-z0-9_:]+)*\.global|st(?:\.[A-Za-z0-9_:]+)*\.global|atom(?:\.[A-Za-z0-9_:]+)*\.global|ld\.param\.b32)" --version 6
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "^\s*(?:mov\.b64|ld(?:\.[A-Za-z0-9_:]+)*\.global|st(?:\.[A-Za-z0-9_:]+)*\.global|atom(?:\.[A-Za-z0-9_:]+)*\.global|ld\.param\.b32)" --no-generate-body-for-unused-prefixes --version 6
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | FileCheck %s --check-prefixes=CHECK,O2
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 -O0 | FileCheck %s --check-prefixes=CHECK,O0
; RUN: %if ptxas-sm_80 && ptxas-isa-7.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | %ptxas-verify -arch=sm_80 %}
@@ -103,6 +103,15 @@ define void @test_dagcombine_store_concat_trunc_v8i32(ptr addrspace(1) %p, <4 x
ret void
}
+define i1 @test_dagcombine_reduce_load_width(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_dagcombine_reduce_load_width(
+; CHECK: ld.global.b8 %rs1, [%rd1];
+ %v = load i64, ptr addrspace(1) %p, align 8, !mem.cache_hint !2
+ %masked = and i64 %v, 255
+ %cmp = icmp eq i64 %masked, 0
+ ret i1 %cmp
+}
+
;-----------------------------------------------------------------------------
; TODO: Preserve cache hints across one-to-N DAG memory rewrites.
; These tests document the current split/scalarized behavior: newly-created
diff --git a/llvm/test/CodeGen/X86/mem-cache-hint-dag-combine-i386.ll b/llvm/test/CodeGen/X86/mem-cache-hint-dag-combine-i386.ll
new file mode 100644
index 0000000000000..68d956502f95f
--- /dev/null
+++ b/llvm/test/CodeGen/X86/mem-cache-hint-dag-combine-i386.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=i386 -mattr=+sse2 -stop-after=finalize-isel %s -o - | FileCheck %s
+
+define void @replace_store_of_fp_constant(ptr %p) {
+ ; CHECK-LABEL: name: replace_store_of_fp_constant
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s32) from %fixed-stack.0)
+ ; CHECK-NEXT: MOV32mi [[MOV32rm]], 1, $noreg, 4, $noreg, 1072939139 :: (store (s32) into %ir.p + 4, basealign 8)
+ ; CHECK-NEXT: MOV32mi [[MOV32rm]], 1, $noreg, 0, $noreg, 309237645 :: (store (s32) into %ir.p, align 8)
+ ; CHECK-NEXT: RET 0
+ store double 0x3FF3C083126E978D, ptr %p, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define double @combine_consecutive_loads_as_double(ptr %p) {
+ ; CHECK-LABEL: name: combine_consecutive_loads_as_double
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s32) from %fixed-stack.0)
+ ; CHECK-NEXT: [[LD_Fp64m80_:%[0-9]+]]:rfp80 = nofpexcept LD_Fp64m80 killed [[MOV32rm]], 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (load (s64) from %ir.p)
+ ; CHECK-NEXT: RET 0, killed [[LD_Fp64m80_]]
+ %lo = load i32, ptr %p, align 8, !mem.cache_hint !1
+ %p4 = getelementptr i8, ptr %p, i32 4
+ %hi = load i32, ptr %p4, align 4, !mem.cache_hint !1
+ %lo.ext = zext i32 %lo to i64
+ %hi.ext = zext i32 %hi to i64
+ %hi.shl = shl i64 %hi.ext, 32
+ %bits = or i64 %lo.ext, %hi.shl
+ %r = bitcast i64 %bits to double
+ ret double %r
+}
+
+define double @combine_consecutive_loads_as_double_different(ptr %p) {
+ ; CHECK-LABEL: name: combine_consecutive_loads_as_double_different
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s32) from %fixed-stack.0)
+ ; CHECK-NEXT: [[LD_Fp64m80_:%[0-9]+]]:rfp80 = nofpexcept LD_Fp64m80 killed [[MOV32rm]], 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (load (s64) from %ir.p)
+ ; CHECK-NEXT: RET 0, killed [[LD_Fp64m80_]]
+ %lo = load i32, ptr %p, align 8, !mem.cache_hint !1
+ %p4 = getelementptr i8, ptr %p, i32 4
+ %hi = load i32, ptr %p4, align 4, !mem.cache_hint !2
+ %lo.ext = zext i32 %lo to i64
+ %hi.ext = zext i32 %hi to i64
+ %hi.shl = shl i64 %hi.ext, 32
+ %bits = or i64 %lo.ext, %hi.shl
+ %r = bitcast i64 %bits to double
+ ret double %r
+}
+
+!0 = !{i32 1, !10}
+!1 = !{i32 0, !10}
+!2 = !{i32 0, !11}
+!10 = !{!"test.cache_hint", !"first"}
+!11 = !{!"test.cache_hint", !"last"}
diff --git a/llvm/test/CodeGen/X86/mem-cache-hint-dag-combine.ll b/llvm/test/CodeGen/X86/mem-cache-hint-dag-combine.ll
new file mode 100644
index 0000000000000..d16fde08476ce
--- /dev/null
+++ b/llvm/test/CodeGen/X86/mem-cache-hint-dag-combine.ll
@@ -0,0 +1,461 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=x86_64 -mattr=+avx2 -stop-after=finalize-isel -combiner-reduce-load-op-store-width-force-narrowing-profitable=1 %s -o - | FileCheck %s
+
+declare i32 @llvm.fshl.i32(i32, i32, i32)
+declare <4 x i32> @llvm.masked.load.v4i32.p0(ptr, i32 immarg, <4 x i1>, <4 x i32>)
+declare void @llvm.masked.store.v4i32.p0(<4 x i32>, ptr, i32 immarg, <4 x i1>)
+declare void @use(ptr)
+
+define i32 @funnel_shift_same(ptr %p) {
+ ; CHECK-LABEL: name: funnel_shift_same
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $rdi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm [[COPY]], 1, $noreg, 3, $noreg :: (load (s32) from %ir.p + 3, align 1)
+ ; CHECK-NEXT: $eax = COPY [[MOV32rm]]
+ ; CHECK-NEXT: RET 0, $eax
+ %p1 = getelementptr i8, ptr %p, i64 4
+ %a = load i32, ptr %p, align 4, !mem.cache_hint !0
+ %b = load i32, ptr %p1, align 4, !mem.cache_hint !0
+ %r = call i32 @llvm.fshl.i32(i32 %b, i32 %a, i32 8)
+ ret i32 %r
+}
+
+define i32 @funnel_shift_different(ptr %p) {
+ ; CHECK-LABEL: name: funnel_shift_different
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $rdi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm [[COPY]], 1, $noreg, 3, $noreg :: (load (s32) from %ir.p + 3, align 1)
+ ; CHECK-NEXT: $eax = COPY [[MOV32rm]]
+ ; CHECK-NEXT: RET 0, $eax
+ %p1 = getelementptr i8, ptr %p, i64 4
+ %a = load i32, ptr %p, align 4, !mem.cache_hint !0
+ %b = load i32, ptr %p1, align 4, !mem.cache_hint !1
+ %r = call i32 @llvm.fshl.i32(i32 %b, i32 %a, i32 8)
+ ret i32 %r
+}
+
+define <4 x i32> @all_ones_masked_load(ptr %p) {
+ ; CHECK-LABEL: name: all_ones_masked_load
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $rdi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: [[VMOVDQArm:%[0-9]+]]:vr128 = VMOVDQArm [[COPY]], 1, $noreg, 0, $noreg :: (load (s128) from %ir.p)
+ ; CHECK-NEXT: $xmm0 = COPY [[VMOVDQArm]]
+ ; CHECK-NEXT: RET 0, $xmm0
+ %v = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 16 %p, i32 16, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> poison), !mem.cache_hint !0
+ ret <4 x i32> %v
+}
+
+define void @all_ones_masked_store(ptr %p, <4 x i32> %v) {
+ ; CHECK-LABEL: name: all_ones_masked_store
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $rdi, $xmm0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vr128 = COPY $xmm0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: VMOVDQAmr [[COPY1]], 1, $noreg, 0, $noreg, [[COPY]] :: (store (s128) into %ir.p)
+ ; CHECK-NEXT: RET 0
+ call void @llvm.masked.store.v4i32.p0(<4 x i32> %v, ptr align 16 %p, i32 16, <4 x i1> <i1 true, i1 true, i1 true, i1 true>), !mem.cache_hint !2
+ ret void
+}
+
+define <16 x i32> @combine_ext_load(ptr %p) {
+ ; CHECK-LABEL: name: combine_ext_load
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $rdi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: [[VPMOVSXWDYrm:%[0-9]+]]:vr256 = VPMOVSXWDYrm [[COPY]], 1, $noreg, 0, $noreg :: (load (s128) from %ir.p, align 32)
+ ; CHECK-NEXT: [[VPMOVSXWDYrm1:%[0-9]+]]:vr256 = VPMOVSXWDYrm [[COPY]], 1, $noreg, 16, $noreg :: (load (s128) from %ir.p + 16, basealign 32)
+ ; CHECK-NEXT: $ymm0 = COPY [[VPMOVSXWDYrm]]
+ ; CHECK-NEXT: $ymm1 = COPY [[VPMOVSXWDYrm1]]
+ ; CHECK-NEXT: RET 0, $ymm0, $ymm1
+ %v = load <16 x i16>, ptr %p, align 32, !mem.cache_hint !0
+ %ext = sext <16 x i16> %v to <16 x i32>
+ ret <16 x i32> %ext
+}
+
+define i8 @reduce_load_width(ptr %p) {
+ ; CHECK-LABEL: name: reduce_load_width
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $rdi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: [[MOV8rm:%[0-9]+]]:gr8 = MOV8rm [[COPY]], 1, $noreg, 0, $noreg :: (load (s8) from %ir.p, align 8)
+ ; CHECK-NEXT: $al = COPY [[MOV8rm]]
+ ; CHECK-NEXT: RET 0, $al
+ %v = load i64, ptr %p, align 8, !mem.cache_hint !0
+ %r = trunc i64 %v to i8
+ ret i8 %r
+}
+
+define i32 @refine_alignment(i32 %v) {
+ ; CHECK-LABEL: name: refine_alignment
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $edi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $edi
+ ; CHECK-NEXT: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+ ; CHECK-NEXT: [[LEA64r:%[0-9]+]]:gr64 = LEA64r %stack.0.p, 1, $noreg, 0, $noreg
+ ; CHECK-NEXT: $rdi = COPY [[LEA64r]]
+ ; CHECK-NEXT: CALL64pcrel32 target-flags(x86-plt) @use, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+ ; CHECK-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+ ; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %stack.0.p, 1, $noreg, 0, $noreg :: (dereferenceable load (s32) from %ir.p)
+ ; CHECK-NEXT: MOV32mr %stack.0.p, 1, $noreg, 0, $noreg, [[COPY]] :: (store (s32) into %ir.p)
+ ; CHECK-NEXT: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+ ; CHECK-NEXT: $rdi = COPY [[LEA64r]]
+ ; CHECK-NEXT: CALL64pcrel32 target-flags(x86-plt) @use, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
+ ; CHECK-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+ ; CHECK-NEXT: $eax = COPY [[MOV32rm]]
+ ; CHECK-NEXT: RET 0, $eax
+ %p = alloca i32, align 4
+ call void @use(ptr %p)
+ %l = load i32, ptr %p, align 1, !range !30, !mem.cache_hint !0
+ store i32 %v, ptr %p, align 1, !mem.cache_hint !2
+ call void @use(ptr %p)
+ ret i32 %l
+}
+
+define void @shrink_load_replace_store(ptr %p, i8 %v) {
+ ; CHECK-LABEL: name: shrink_load_replace_store
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $rdi, $esi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $esi
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr8 = COPY [[COPY]].sub_8bit
+ ; CHECK-NEXT: MOV8mr [[COPY1]], 1, $noreg, 0, $noreg, killed [[COPY2]] :: (store (s8) into %ir.p, align 4)
+ ; CHECK-NEXT: RET 0
+ %old = load i32, ptr %p, align 4
+ %keep = and i32 %old, -256
+ %ext = zext i8 %v to i32
+ %new = or i32 %keep, %ext
+ store i32 %new, ptr %p, align 4, !alias.scope !20, !noalias !23, !mem.cache_hint !2
+ ret void
+}
+
+define void @reduce_load_op_store(ptr %p) {
+ ; CHECK-LABEL: name: reduce_load_op_store
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $rdi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: NOT8m [[COPY]], 1, $noreg, 0, $noreg :: (store (s8) into %ir.p, align 8), (load (s8) from %ir.p, align 8, !alias.scope !6, !noalias !9)
+ ; CHECK-NEXT: RET 0
+ %old = load i64, ptr %p, align 8, !alias.scope !20, !noalias !23, !mem.cache_hint !0
+ %new = xor i64 %old, 255
+ store i64 %new, ptr %p, align 8, !alias.scope !20, !noalias !23, !mem.cache_hint !2
+ ret void
+}
+
+define void @merge_trunc_stores_same(ptr %p, i16 %v) {
+ ; CHECK-LABEL: name: merge_trunc_stores_same
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $rdi, $esi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $esi
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr16 = COPY [[COPY]].sub_16bit
+ ; CHECK-NEXT: MOV16mr [[COPY1]], 1, $noreg, 0, $noreg, killed [[COPY2]] :: (store (s16) into %ir.p)
+ ; CHECK-NEXT: RET 0
+ %lo = trunc i16 %v to i8
+ store i8 %lo, ptr %p, align 2, !mem.cache_hint !2
+ %hiword = lshr i16 %v, 8
+ %hi = trunc i16 %hiword to i8
+ %p1 = getelementptr i8, ptr %p, i64 1
+ store i8 %hi, ptr %p1, align 1, !mem.cache_hint !2
+ ret void
+}
+
+define void @merge_trunc_stores_different(ptr %p, i16 %v) {
+ ; CHECK-LABEL: name: merge_trunc_stores_different
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $rdi, $esi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $esi
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr16 = COPY [[COPY]].sub_16bit
+ ; CHECK-NEXT: MOV16mr [[COPY1]], 1, $noreg, 0, $noreg, killed [[COPY2]] :: (store (s16) into %ir.p)
+ ; CHECK-NEXT: RET 0
+ %lo = trunc i16 %v to i8
+ store i8 %lo, ptr %p, align 2, !mem.cache_hint !2
+ %hiword = lshr i16 %v, 8
+ %hi = trunc i16 %hiword to i8
+ %p1 = getelementptr i8, ptr %p, i64 1
+ store i8 %hi, ptr %p1, align 1, !mem.cache_hint !3
+ ret void
+}
+
+define i32 @match_load_combine_same(ptr %p) {
+ ; CHECK-LABEL: name: match_load_combine_same
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $rdi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm [[COPY]], 1, $noreg, 0, $noreg :: (load (s32) from %ir.p)
+ ; CHECK-NEXT: $eax = COPY [[MOV32rm]]
+ ; CHECK-NEXT: RET 0, $eax
+ %b0 = load i8, ptr %p, align 4, !mem.cache_hint !0
+ %p1 = getelementptr i8, ptr %p, i64 1
+ %b1 = load i8, ptr %p1, align 1, !mem.cache_hint !0
+ %p2 = getelementptr i8, ptr %p, i64 2
+ %b2 = load i8, ptr %p2, align 2, !mem.cache_hint !0
+ %p3 = getelementptr i8, ptr %p, i64 3
+ %b3 = load i8, ptr %p3, align 1, !mem.cache_hint !0
+ %e0 = zext i8 %b0 to i32
+ %e1 = zext i8 %b1 to i32
+ %e2 = zext i8 %b2 to i32
+ %e3 = zext i8 %b3 to i32
+ %s1 = shl i32 %e1, 8
+ %s2 = shl i32 %e2, 16
+ %s3 = shl i32 %e3, 24
+ %o1 = or i32 %e0, %s1
+ %o2 = or i32 %s2, %s3
+ %r = or i32 %o1, %o2
+ ret i32 %r
+}
+
+define i32 @match_load_combine_different(ptr %p) {
+ ; CHECK-LABEL: name: match_load_combine_different
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $rdi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm [[COPY]], 1, $noreg, 0, $noreg :: (load (s32) from %ir.p)
+ ; CHECK-NEXT: $eax = COPY [[MOV32rm]]
+ ; CHECK-NEXT: RET 0, $eax
+ %b0 = load i8, ptr %p, align 4, !mem.cache_hint !0
+ %p1 = getelementptr i8, ptr %p, i64 1
+ %b1 = load i8, ptr %p1, align 1, !mem.cache_hint !0
+ %p2 = getelementptr i8, ptr %p, i64 2
+ %b2 = load i8, ptr %p2, align 2, !mem.cache_hint !1
+ %p3 = getelementptr i8, ptr %p, i64 3
+ %b3 = load i8, ptr %p3, align 1, !mem.cache_hint !0
+ %e0 = zext i8 %b0 to i32
+ %e1 = zext i8 %b1 to i32
+ %e2 = zext i8 %b2 to i32
+ %e3 = zext i8 %b3 to i32
+ %s1 = shl i32 %e1, 8
+ %s2 = shl i32 %e2, 16
+ %s3 = shl i32 %e3, 24
+ %o1 = or i32 %e0, %s1
+ %o2 = or i32 %s2, %s3
+ %r = or i32 %o1, %o2
+ ret i32 %r
+}
+
+define void @merge_constant_stores_same(ptr %p) {
+ ; CHECK-LABEL: name: merge_constant_stores_same
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $rdi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: MOV32mi [[COPY]], 1, $noreg, 0, $noreg, 67305985 :: (store (s32) into %ir.p)
+ ; CHECK-NEXT: RET 0
+ store i8 1, ptr %p, align 4, !mem.cache_hint !2
+ %p1 = getelementptr i8, ptr %p, i64 1
+ store i8 2, ptr %p1, align 1, !mem.cache_hint !2
+ %p2 = getelementptr i8, ptr %p, i64 2
+ store i8 3, ptr %p2, align 2, !mem.cache_hint !2
+ %p3 = getelementptr i8, ptr %p, i64 3
+ store i8 4, ptr %p3, align 1, !mem.cache_hint !2
+ ret void
+}
+
+define void @merge_constant_stores_different(ptr %p) {
+ ; CHECK-LABEL: name: merge_constant_stores_different
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $rdi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: MOV32mi [[COPY]], 1, $noreg, 0, $noreg, 67305985 :: (store (s32) into %ir.p)
+ ; CHECK-NEXT: RET 0
+ store i8 1, ptr %p, align 4, !mem.cache_hint !2
+ %p1 = getelementptr i8, ptr %p, i64 1
+ store i8 2, ptr %p1, align 1, !mem.cache_hint !2
+ %p2 = getelementptr i8, ptr %p, i64 2
+ store i8 3, ptr %p2, align 2, !mem.cache_hint !3
+ %p3 = getelementptr i8, ptr %p, i64 3
+ store i8 4, ptr %p3, align 1, !mem.cache_hint !2
+ ret void
+}
+
+define void @merge_store_of_loads_same(ptr noalias %dst, ptr noalias %src) {
+ ; CHECK-LABEL: name: merge_store_of_loads_same
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $rdi, $rsi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rsi
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm [[COPY]], 1, $noreg, 0, $noreg :: (load (s32) from %ir.src)
+ ; CHECK-NEXT: MOV32mr [[COPY1]], 1, $noreg, 0, $noreg, killed [[MOV32rm]] :: (store (s32) into %ir.dst)
+ ; CHECK-NEXT: RET 0
+ %a = load i8, ptr %src, align 4, !mem.cache_hint !0
+ %s1 = getelementptr i8, ptr %src, i64 1
+ %b = load i8, ptr %s1, align 1, !mem.cache_hint !0
+ %s2 = getelementptr i8, ptr %src, i64 2
+ %c = load i8, ptr %s2, align 2, !mem.cache_hint !0
+ %s3 = getelementptr i8, ptr %src, i64 3
+ %d = load i8, ptr %s3, align 1, !mem.cache_hint !0
+ store i8 %a, ptr %dst, align 4, !mem.cache_hint !2
+ %d1 = getelementptr i8, ptr %dst, i64 1
+ store i8 %b, ptr %d1, align 1, !mem.cache_hint !2
+ %d2 = getelementptr i8, ptr %dst, i64 2
+ store i8 %c, ptr %d2, align 2, !mem.cache_hint !2
+ %d3 = getelementptr i8, ptr %dst, i64 3
+ store i8 %d, ptr %d3, align 1, !mem.cache_hint !2
+ ret void
+}
+
+define void @merge_store_of_loads_different(ptr noalias %dst, ptr noalias %src) {
+ ; CHECK-LABEL: name: merge_store_of_loads_different
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $rdi, $rsi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rsi
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm [[COPY]], 1, $noreg, 0, $noreg :: (load (s32) from %ir.src)
+ ; CHECK-NEXT: MOV32mr [[COPY1]], 1, $noreg, 0, $noreg, killed [[MOV32rm]] :: (store (s32) into %ir.dst)
+ ; CHECK-NEXT: RET 0
+ %a = load i8, ptr %src, align 4, !mem.cache_hint !0
+ %s1 = getelementptr i8, ptr %src, i64 1
+ %b = load i8, ptr %s1, align 1, !mem.cache_hint !0
+ %s2 = getelementptr i8, ptr %src, i64 2
+ %c = load i8, ptr %s2, align 2, !mem.cache_hint !1
+ %s3 = getelementptr i8, ptr %src, i64 3
+ %d = load i8, ptr %s3, align 1, !mem.cache_hint !0
+ store i8 %a, ptr %dst, align 4, !mem.cache_hint !2
+ %d1 = getelementptr i8, ptr %dst, i64 1
+ store i8 %b, ptr %d1, align 1, !mem.cache_hint !2
+ %d2 = getelementptr i8, ptr %dst, i64 2
+ store i8 %c, ptr %d2, align 2, !mem.cache_hint !3
+ %d3 = getelementptr i8, ptr %dst, i64 3
+ store i8 %d, ptr %d3, align 1, !mem.cache_hint !2
+ ret void
+}
+
+define void @replace_store_of_insert_load(ptr %p, i32 %v) {
+ ; CHECK-LABEL: name: replace_store_of_insert_load
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $rdi, $esi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $esi
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: MOV32mr [[COPY1]], 1, $noreg, 8, $noreg, [[COPY]] :: (store (s32) into %ir.p + 8, align 8, basealign 16)
+ ; CHECK-NEXT: RET 0
+ %old = load <4 x i32>, ptr %p, align 16
+ %new = insertelement <4 x i32> %old, i32 %v, i32 2
+ store <4 x i32> %new, ptr %p, align 16, !alias.scope !20, !noalias !23, !mem.cache_hint !2
+ ret void
+}
+
+define <4 x i32> @concat_shuffles_same(ptr %p) {
+ ; CHECK-LABEL: name: concat_shuffles_same
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $rdi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: [[VPSHUFDmi:%[0-9]+]]:vr128 = VPSHUFDmi [[COPY]], 1, $noreg, 0, $noreg, -40 :: (load (s128) from %ir.p, !mem.cache_hint !1)
+ ; CHECK-NEXT: $xmm0 = COPY [[VPSHUFDmi]]
+ ; CHECK-NEXT: RET 0, $xmm0
+ %a = load <2 x i32>, ptr %p, align 16, !mem.cache_hint !0
+ %p2 = getelementptr i8, ptr %p, i64 8
+ %b = load <2 x i32>, ptr %p2, align 8, !mem.cache_hint !0
+ %s0 = shufflevector <2 x i32> %a, <2 x i32> %b, <2 x i32> <i32 0, i32 2>
+ %s1 = shufflevector <2 x i32> %a, <2 x i32> %b, <2 x i32> <i32 1, i32 3>
+ %r = shufflevector <2 x i32> %s0, <2 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ ret <4 x i32> %r
+}
+
+define <4 x i32> @concat_shuffles_different(ptr %p) {
+ ; CHECK-LABEL: name: concat_shuffles_different
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $rdi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: [[VPSHUFDmi:%[0-9]+]]:vr128 = VPSHUFDmi [[COPY]], 1, $noreg, 0, $noreg, -40 :: (load (s128) from %ir.p, !mem.cache_hint !1)
+ ; CHECK-NEXT: $xmm0 = COPY [[VPSHUFDmi]]
+ ; CHECK-NEXT: RET 0, $xmm0
+ %a = load <2 x i32>, ptr %p, align 16, !mem.cache_hint !0
+ %p2 = getelementptr i8, ptr %p, i64 8
+ %b = load <2 x i32>, ptr %p2, align 8, !mem.cache_hint !1
+ %s0 = shufflevector <2 x i32> %a, <2 x i32> %b, <2 x i32> <i32 0, i32 2>
+ %s1 = shufflevector <2 x i32> %a, <2 x i32> %b, <2 x i32> <i32 1, i32 3>
+ %r = shufflevector <2 x i32> %s0, <2 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ ret <4 x i32> %r
+}
+
+define i32 @select_loads_same(i1 %c, ptr %p, ptr %q) {
+ ; CHECK-LABEL: name: select_loads_same
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $edi, $rsi, $rdx
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdx
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rsi
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr32 = COPY $edi
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr8 = COPY [[COPY2]].sub_8bit
+ ; CHECK-NEXT: TEST8ri killed [[COPY3]], 1, implicit-def $eflags
+ ; CHECK-NEXT: [[CMOV64rr:%[0-9]+]]:gr64 = CMOV64rr [[COPY]], [[COPY1]], 5, implicit $eflags
+ ; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm killed [[CMOV64rr]], 1, $noreg, 0, $noreg :: (load (s32))
+ ; CHECK-NEXT: $eax = COPY [[MOV32rm]]
+ ; CHECK-NEXT: RET 0, $eax
+ %a = load i32, ptr %p, align 4, !mem.cache_hint !0
+ %b = load i32, ptr %q, align 4, !mem.cache_hint !0
+ %r = select i1 %c, i32 %a, i32 %b
+ ret i32 %r
+}
+
+define i32 @select_loads_different(i1 %c, ptr %p, ptr %q) {
+ ; CHECK-LABEL: name: select_loads_different
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $edi, $rsi, $rdx
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdx
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rsi
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr32 = COPY $edi
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr8 = COPY [[COPY2]].sub_8bit
+ ; CHECK-NEXT: TEST8ri killed [[COPY3]], 1, implicit-def $eflags
+ ; CHECK-NEXT: [[CMOV64rr:%[0-9]+]]:gr64 = CMOV64rr [[COPY]], [[COPY1]], 5, implicit $eflags
+ ; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm killed [[CMOV64rr]], 1, $noreg, 0, $noreg :: (load (s32))
+ ; CHECK-NEXT: $eax = COPY [[MOV32rm]]
+ ; CHECK-NEXT: RET 0, $eax
+ %a = load i32, ptr %p, align 4, !mem.cache_hint !0
+ %b = load i32, ptr %q, align 4, !mem.cache_hint !1
+ %r = select i1 %c, i32 %a, i32 %b
+ ret i32 %r
+}
+
+define i1 @simplify_setcc(ptr %p) {
+ ; CHECK-LABEL: name: simplify_setcc
+ ; CHECK: bb.0 (%ir-block.0):
+ ; CHECK-NEXT: liveins: $rdi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: CMP8mi [[COPY]], 1, $noreg, 0, $noreg, 0, implicit-def $eflags :: (load (s8) from %ir.p, align 8, !alias.scope !6, !noalias !9)
+ ; CHECK-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 4, implicit $eflags
+ ; CHECK-NEXT: $al = COPY [[SETCCr]]
+ ; CHECK-NEXT: RET 0, $al
+ %v = load i64, ptr %p, align 8, !alias.scope !20, !noalias !23, !mem.cache_hint !0
+ %masked = and i64 %v, 255
+ %r = icmp eq i64 %masked, 0
+ ret i1 %r
+}
+
+!0 = !{i32 0, !10}
+!1 = !{i32 0, !11}
+!2 = !{i32 1, !10}
+!3 = !{i32 1, !11}
+!10 = !{!"test.cache_hint", !"first"}
+!11 = !{!"test.cache_hint", !"last"}
+
+!20 = !{!21}
+!21 = distinct !{!21, !22, !"scope"}
+!22 = distinct !{!22, !"domain"}
+!23 = !{!24}
+!24 = distinct !{!24, !22, !"other"}
+!30 = !{i32 0, i32 100}
>From ce26ed2b7506699fa63d576b338a1253e025a741 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 22 Sep 2026 21:42:52 +0000
Subject: [PATCH 2/5] [SelectionDAG] Preserve cache hints during DAG combines
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 181 ++++++++++++------
.../CodeGen/SelectionDAG/TargetLowering.cpp | 9 +-
.../AArch64/mem-cache-hint-dag-combine.ll | 2 +-
.../CodeGen/NVPTX/cache-hint-transforms.ll | 2 +-
.../X86/mem-cache-hint-dag-combine-i386.ll | 6 +-
.../CodeGen/X86/mem-cache-hint-dag-combine.ll | 38 ++--
6 files changed, 155 insertions(+), 83 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index d4a7403a6b25f..748d5bdec9c04 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -10149,9 +10149,18 @@ SDValue DAGCombiner::mergeTruncStores(StoreSDNode *N) {
SourceValue = DAG.getNode(ISD::ROTR, DL, WideVT, SourceValue, RotAmt);
}
+ const MDNode *MemCacheHint = Stores.front()->getMemCacheHint();
+ for (StoreSDNode *Store : drop_begin(Stores))
+ if (Store->getMemCacheHint() != MemCacheHint) {
+ MemCacheHint = nullptr;
+ break;
+ }
+
SDValue NewStore =
DAG.getStore(Chain, DL, SourceValue, FirstStore->getBasePtr(),
- FirstStore->getPointerInfo(), FirstStore->getAlign());
+ FirstStore->getPointerInfo(), FirstStore->getAlign(),
+ MachineMemOperand::MONone,
+ MMOMetadata(AAMDNodes(), nullptr, MemCacheHint));
// Rely on other DAG combine rules to remove the other individual stores.
DAG.ReplaceAllUsesWith(N, NewStore.getNode());
@@ -10354,10 +10363,18 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
if (!Allowed || !Fast)
return SDValue();
- SDValue NewLoad =
- DAG.getExtLoad(NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, SDLoc(N), VT,
- Chain, FirstLoad->getBasePtr(),
- FirstLoad->getPointerInfo(), MemVT, FirstLoad->getAlign());
+ const MDNode *MemCacheHint = FirstLoad->getMemCacheHint();
+ for (LoadSDNode *Load : Loads)
+ if (Load->getMemCacheHint() != MemCacheHint) {
+ MemCacheHint = nullptr;
+ break;
+ }
+
+ SDValue NewLoad = DAG.getExtLoad(
+ NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, SDLoc(N), VT, Chain,
+ FirstLoad->getBasePtr(), FirstLoad->getPointerInfo(), MemVT,
+ FirstLoad->getAlign(), MachineMemOperand::MONone,
+ MMOMetadata(AAMDNodes(), nullptr, MemCacheHint));
// Transfer chain users from old loads to the new load.
for (LoadSDNode *L : Loads)
@@ -12179,10 +12196,15 @@ SDValue DAGCombiner::visitFunnelShift(SDNode *N) {
SDValue NewPtr = DAG.getMemBasePlusOffset(
RHS->getBasePtr(), TypeSize::getFixed(PtrOff), DL);
AddToWorklist(NewPtr.getNode());
+ const MDNode *MemCacheHint =
+ LHS->getMemCacheHint() == RHS->getMemCacheHint()
+ ? RHS->getMemCacheHint()
+ : nullptr;
SDValue Load = DAG.getLoad(
VT, DL, RHS->getChain(), NewPtr,
RHS->getPointerInfo().getWithOffset(PtrOff), NewAlign,
- RHS->getMemOperand()->getFlags(), RHS->getAAInfo());
+ RHS->getMemOperand()->getFlags(),
+ MMOMetadata(RHS->getAAInfo(), nullptr, MemCacheHint));
DAG.makeEquivalentMemoryOrdering(LHS, Load.getValue(1));
DAG.makeEquivalentMemoryOrdering(RHS, Load.getValue(1));
return Load;
@@ -13915,10 +13937,11 @@ SDValue DAGCombiner::visitMSTORE(SDNode *N) {
// FIXME: Can we do this for indexed, compressing, or truncating stores?
if (ISD::isConstantSplatVectorAllOnes(Mask.getNode()) && MST->isUnindexed() &&
!MST->isCompressingStore() && !MST->isTruncatingStore())
- return DAG.getStore(MST->getChain(), SDLoc(N), MST->getValue(),
- MST->getBasePtr(), MST->getPointerInfo(),
- MST->getBaseAlign(), MST->getMemOperand()->getFlags(),
- MST->getAAInfo());
+ return DAG.getStore(
+ MST->getChain(), SDLoc(N), MST->getValue(), MST->getBasePtr(),
+ MST->getPointerInfo(), MST->getBaseAlign(),
+ MST->getMemOperand()->getFlags(),
+ MMOMetadata(MST->getAAInfo(), nullptr, MST->getMemCacheHint()));
// Try transforming N to an indexed store.
if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
@@ -14105,7 +14128,8 @@ SDValue DAGCombiner::visitMLOAD(SDNode *N) {
DAG.getLoad(N->getValueType(0), SDLoc(N), MLD->getChain(),
MLD->getBasePtr(), MLD->getPointerInfo(),
MLD->getBaseAlign(), MLD->getMemOperand()->getFlags(),
- MMOMetadata(MLD->getAAInfo(), MLD->getRanges()));
+ MMOMetadata(MLD->getAAInfo(), MLD->getRanges(),
+ MLD->getMemCacheHint()));
return CombineTo(N, NewLd, NewLd.getValue(1));
}
@@ -15433,11 +15457,11 @@ SDValue DAGCombiner::CombineExtLoad(SDNode *N) {
for (unsigned Idx = 0; Idx < NumSplits; Idx++) {
const unsigned Offset = Idx * Stride;
- SDValue SplitLoad =
- DAG.getExtLoad(ExtType, SDLoc(LN0), SplitDstVT, LN0->getChain(),
- BasePtr, LN0->getPointerInfo().getWithOffset(Offset),
- SplitSrcVT, LN0->getBaseAlign(),
- LN0->getMemOperand()->getFlags(), LN0->getAAInfo());
+ SDValue SplitLoad = DAG.getExtLoad(
+ ExtType, SDLoc(LN0), SplitDstVT, LN0->getChain(), BasePtr,
+ LN0->getPointerInfo().getWithOffset(Offset), SplitSrcVT,
+ LN0->getBaseAlign(), LN0->getMemOperand()->getFlags(),
+ MMOMetadata(LN0->getAAInfo(), nullptr, LN0->getMemCacheHint()));
BasePtr = DAG.getMemBasePlusOffset(BasePtr, TypeSize::getFixed(Stride), DL);
@@ -17251,15 +17275,17 @@ SDValue DAGCombiner::reduceLoadWidth(SDNode *N) {
} else if (CR.getBitWidth() == BitSize)
NewRanges = OldRanges;
}
- Load = DAG.getLoad(VT, DL, LN0->getChain(), NewPtr,
- LN0->getPointerInfo().getWithOffset(PtrOff),
- LN0->getBaseAlign(), LN0->getMemOperand()->getFlags(),
- MMOMetadata(LN0->getAAInfo(), NewRanges));
+ Load = DAG.getLoad(
+ VT, DL, LN0->getChain(), NewPtr,
+ LN0->getPointerInfo().getWithOffset(PtrOff), LN0->getBaseAlign(),
+ LN0->getMemOperand()->getFlags(),
+ MMOMetadata(LN0->getAAInfo(), NewRanges, LN0->getMemCacheHint()));
} else
- Load = DAG.getExtLoad(ExtType, DL, VT, LN0->getChain(), NewPtr,
- LN0->getPointerInfo().getWithOffset(PtrOff), ExtVT,
- LN0->getBaseAlign(), LN0->getMemOperand()->getFlags(),
- LN0->getAAInfo());
+ Load = DAG.getExtLoad(
+ ExtType, DL, VT, LN0->getChain(), NewPtr,
+ LN0->getPointerInfo().getWithOffset(PtrOff), ExtVT, LN0->getBaseAlign(),
+ LN0->getMemOperand()->getFlags(),
+ MMOMetadata(LN0->getAAInfo(), nullptr, LN0->getMemCacheHint()));
// Replace the old load's chain with the new load's chain.
WorklistRemover DeadNodes(*this);
@@ -18144,8 +18170,13 @@ SDValue DAGCombiner::CombineConsecutiveLoads(SDNode *N, EVT VT) {
DAG.areNonVolatileConsecutiveLoads(LD2, LD1, LD1Bytes, 1) &&
TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT,
*LD1->getMemOperand(), &LD1Fast) && LD1Fast)
- return DAG.getLoad(VT, SDLoc(N), LD1->getChain(), LD1->getBasePtr(),
- LD1->getPointerInfo(), LD1->getAlign());
+ return DAG.getLoad(
+ VT, SDLoc(N), LD1->getChain(), LD1->getBasePtr(), LD1->getPointerInfo(),
+ LD1->getAlign(), MachineMemOperand::MONone,
+ MMOMetadata(AAMDNodes(), nullptr,
+ LD1->getMemCacheHint() == LD2->getMemCacheHint()
+ ? LD1->getMemCacheHint()
+ : nullptr));
return SDValue();
}
@@ -22097,7 +22128,9 @@ SDValue DAGCombiner::visitLOAD(SDNode *N) {
LD->getAddressingMode(), LD->getExtensionType(),
LD->getValueType(0), SDLoc(N), Chain, Ptr, LD->getOffset(),
LD->getPointerInfo(), LD->getMemoryVT(), *Alignment,
- LD->getMemOperand()->getFlags(), LD->getAAInfo());
+ LD->getMemOperand()->getFlags(),
+ MMOMetadata(LD->getAAInfo(), LD->getRanges(),
+ LD->getMemCacheHint()));
// NewLoad will always be N as we are only refining the alignment
assert(NewLoad.getNode() == N);
(void)NewLoad;
@@ -22829,17 +22862,19 @@ ShrinkLoadReplaceStoreWithStore(const std::pair<unsigned, unsigned> &MaskInfo,
}
++OpsNarrowed;
+ MMOMetadata Metadata(St->getAAInfo(), nullptr, St->getMemCacheHint());
if (UseTruncStore)
return DAG.getTruncStore(St->getChain(), SDLoc(St), IVal, Ptr,
St->getPointerInfo().getWithOffset(StOffset), VT,
- St->getBaseAlign());
+ St->getBaseAlign(), MachineMemOperand::MONone,
+ Metadata);
// Truncate down to the new size.
IVal = DAG.getNode(ISD::TRUNCATE, SDLoc(IVal), VT, IVal);
return DAG.getStore(St->getChain(), SDLoc(St), IVal, Ptr,
St->getPointerInfo().getWithOffset(StOffset),
- St->getBaseAlign());
+ St->getBaseAlign(), MachineMemOperand::MONone, Metadata);
}
/// Look for sequence of load / op / store where op is one of 'or', 'xor', and
@@ -22976,15 +23011,18 @@ SDValue DAGCombiner::ReduceLoadOpStoreWidth(SDNode *N) {
Align NewAlign = commonAlignment(LD->getAlign(), PtrOff);
SDValue NewPtr =
DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(PtrOff), SDLoc(LD));
- SDValue NewLD =
- DAG.getLoad(NewVT, SDLoc(N0), LD->getChain(), NewPtr,
- LD->getPointerInfo().getWithOffset(PtrOff), NewAlign,
- LD->getMemOperand()->getFlags(), LD->getAAInfo());
+ SDValue NewLD = DAG.getLoad(
+ NewVT, SDLoc(N0), LD->getChain(), NewPtr,
+ LD->getPointerInfo().getWithOffset(PtrOff), NewAlign,
+ LD->getMemOperand()->getFlags(),
+ MMOMetadata(LD->getAAInfo(), nullptr, LD->getMemCacheHint()));
SDValue NewVal = DAG.getNode(Opc, SDLoc(Value), NewVT, NewLD,
DAG.getConstant(NewImm, SDLoc(Value), NewVT));
- SDValue NewST =
- DAG.getStore(Chain, SDLoc(N), NewVal, NewPtr,
- ST->getPointerInfo().getWithOffset(PtrOff), NewAlign);
+ SDValue NewST = DAG.getStore(
+ Chain, SDLoc(N), NewVal, NewPtr,
+ ST->getPointerInfo().getWithOffset(PtrOff), NewAlign,
+ MachineMemOperand::MONone,
+ MMOMetadata(ST->getAAInfo(), nullptr, ST->getMemCacheHint()));
AddToWorklist(NewPtr.getNode());
AddToWorklist(NewLD.getNode());
@@ -23187,11 +23225,13 @@ bool DAGCombiner::mergeStoresOfConstantsOrVecElts(
std::optional<MachineMemOperand::Flags> Flags;
AAMDNodes AAInfo;
+ const MDNode *MemCacheHint = nullptr;
for (unsigned I = 0; I != NumStores; ++I) {
StoreSDNode *St = cast<StoreSDNode>(StoreNodes[I].MemNode);
if (!Flags) {
Flags = St->getMemOperand()->getFlags();
AAInfo = St->getAAInfo();
+ MemCacheHint = St->getMemCacheHint();
continue;
}
// Skip merging if there's an inconsistent flag.
@@ -23199,6 +23239,8 @@ bool DAGCombiner::mergeStoresOfConstantsOrVecElts(
return false;
// Concatenate AA metadata.
AAInfo = AAInfo.concat(St->getAAInfo());
+ if (MemCacheHint != St->getMemCacheHint())
+ MemCacheHint = nullptr;
}
EVT StoreTy;
@@ -23338,7 +23380,8 @@ bool DAGCombiner::mergeStoresOfConstantsOrVecElts(
CanReusePtrInfo
? FirstInChain->getPointerInfo()
: MachinePointerInfo(FirstInChain->getPointerInfo().getAddrSpace()),
- FirstInChain->getAlign(), *Flags, AAInfo);
+ FirstInChain->getAlign(), *Flags,
+ MMOMetadata(AAInfo, nullptr, MemCacheHint));
} else { // Must be realized as a trunc store
EVT LegalizedStoredValTy =
TLI.getTypeToTransformTo(*DAG.getContext(), StoredVal.getValueType());
@@ -23353,7 +23396,7 @@ bool DAGCombiner::mergeStoresOfConstantsOrVecElts(
? FirstInChain->getPointerInfo()
: MachinePointerInfo(FirstInChain->getPointerInfo().getAddrSpace()),
StoredVal.getValueType() /*TVT*/, FirstInChain->getAlign(), *Flags,
- AAInfo);
+ MMOMetadata(AAInfo, nullptr, MemCacheHint));
}
// Replace all merged stores with the new store.
@@ -24127,11 +24170,21 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
StMMOFlags |= TLI.getTargetMMOFlags(*StoreNodes[0].MemNode);
+ const MDNode *LoadMemCacheHint = FirstLoad->getMemCacheHint();
+ const MDNode *StoreMemCacheHint = FirstInChain->getMemCacheHint();
+ for (unsigned I = 1; I != NumElem; ++I) {
+ if (LoadNodes[I].MemNode->getMemCacheHint() != LoadMemCacheHint)
+ LoadMemCacheHint = nullptr;
+ if (StoreNodes[I].MemNode->getMemCacheHint() != StoreMemCacheHint)
+ StoreMemCacheHint = nullptr;
+ }
+
SDValue NewLoad, NewStore;
if (UseVectorTy || !DoIntegerTruncate) {
NewLoad = DAG.getLoad(
JointMemOpVT, LoadDL, FirstLoad->getChain(), FirstLoad->getBasePtr(),
- FirstLoad->getPointerInfo(), FirstLoadAlign, LdMMOFlags);
+ FirstLoad->getPointerInfo(), FirstLoadAlign, LdMMOFlags,
+ MMOMetadata(AAMDNodes(), nullptr, LoadMemCacheHint));
SDValue StoreOp = NewLoad;
if (NeedRotate) {
unsigned LoadWidth = ElementSizeBytes * 8 * 2;
@@ -24146,20 +24199,23 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
NewStoreChain, StoreDL, StoreOp, FirstInChain->getBasePtr(),
CanReusePtrInfo ? FirstInChain->getPointerInfo()
: MachinePointerInfo(FirstStoreAS),
- FirstStoreAlign, StMMOFlags);
+ FirstStoreAlign, StMMOFlags,
+ MMOMetadata(AAMDNodes(), nullptr, StoreMemCacheHint));
} else { // This must be the truncstore/extload case
EVT ExtendedTy =
TLI.getTypeToTransformTo(*DAG.getContext(), JointMemOpVT);
- NewLoad = DAG.getExtLoad(ISD::EXTLOAD, LoadDL, ExtendedTy,
- FirstLoad->getChain(), FirstLoad->getBasePtr(),
- FirstLoad->getPointerInfo(), JointMemOpVT,
- FirstLoadAlign, LdMMOFlags);
+ NewLoad = DAG.getExtLoad(
+ ISD::EXTLOAD, LoadDL, ExtendedTy, FirstLoad->getChain(),
+ FirstLoad->getBasePtr(), FirstLoad->getPointerInfo(), JointMemOpVT,
+ FirstLoadAlign, LdMMOFlags,
+ MMOMetadata(AAMDNodes(), nullptr, LoadMemCacheHint));
NewStore = DAG.getTruncStore(
NewStoreChain, StoreDL, NewLoad, FirstInChain->getBasePtr(),
CanReusePtrInfo ? FirstInChain->getPointerInfo()
: MachinePointerInfo(FirstStoreAS),
JointMemOpVT, FirstInChain->getAlign(),
- FirstInChain->getMemOperand()->getFlags());
+ FirstInChain->getMemOperand()->getFlags(),
+ MMOMetadata(AAMDNodes(), nullptr, StoreMemCacheHint));
}
// Transfer chain users from old loads to the new load.
@@ -24366,14 +24422,14 @@ SDValue DAGCombiner::replaceStoreOfFPConstant(StoreSDNode *ST) {
std::swap(Lo, Hi);
MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
- AAMDNodes AAInfo = ST->getAAInfo();
+ MMOMetadata Metadata(ST->getAAInfo(), nullptr, ST->getMemCacheHint());
SDValue St0 = DAG.getStore(Chain, DL, Lo, Ptr, ST->getPointerInfo(),
- ST->getBaseAlign(), MMOFlags, AAInfo);
+ ST->getBaseAlign(), MMOFlags, Metadata);
Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(4), DL);
SDValue St1 = DAG.getStore(Chain, DL, Hi, Ptr,
ST->getPointerInfo().getWithOffset(4),
- ST->getBaseAlign(), MMOFlags, AAInfo);
+ ST->getBaseAlign(), MMOFlags, Metadata);
return DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
St0, St1);
}
@@ -24442,8 +24498,10 @@ SDValue DAGCombiner::replaceStoreOfInsertLoad(StoreSDNode *ST) {
NewAlign = commonAlignment(ST->getAlign(), EltVT.getFixedSizeInBits() / 8);
}
- return DAG.getStore(Chain, DL, Elt, NewPtr, PointerInfo, NewAlign,
- ST->getMemOperand()->getFlags());
+ return DAG.getStore(
+ Chain, DL, Elt, NewPtr, PointerInfo, NewAlign,
+ ST->getMemOperand()->getFlags(),
+ MMOMetadata(ST->getAAInfo(), nullptr, ST->getMemCacheHint()));
}
SDValue DAGCombiner::visitATOMIC_STORE(SDNode *N) {
@@ -24637,7 +24695,7 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) {
SDValue NewStore = DAG.getTruncStore(
Chain, SDLoc(N), Value, Ptr, ST->getOffset(), ST->getPointerInfo(),
ST->getMemoryVT(), *Alignment, ST->getMemOperand()->getFlags(),
- ST->getAAInfo());
+ MMOMetadata(ST->getAAInfo(), nullptr, ST->getMemCacheHint()));
// NewStore will always be N as we are only refining the alignment
assert(NewStore.getNode() == N);
(void)NewStore;
@@ -25251,8 +25309,14 @@ SDValue DAGCombiner::combineInsertEltToLoad(SDNode *N, unsigned InsIndex) {
InsIndex == 0 ? ScalarLoad->getPointerInfo()
: VecLoad->getPointerInfo().getWithOffset(EltSize / 8);
- SDValue Load = DAG.getLoad(VecLoad->getValueType(0), DL,
- ScalarLoad->getChain(), Ptr, PtrInfo, NewAlign);
+ const MDNode *MemCacheHint =
+ ScalarLoad->getMemCacheHint() == VecLoad->getMemCacheHint()
+ ? ScalarLoad->getMemCacheHint()
+ : nullptr;
+ SDValue Load =
+ DAG.getLoad(VecLoad->getValueType(0), DL, ScalarLoad->getChain(), Ptr,
+ PtrInfo, NewAlign, MachineMemOperand::MONone,
+ MMOMetadata(AAMDNodes(), nullptr, MemCacheHint));
DAG.makeEquivalentMemoryOrdering(ScalarLoad, Load.getValue(1));
DAG.makeEquivalentMemoryOrdering(VecLoad, Load.getValue(1));
return Extend ? DAG.getNode(Extend, DL, VT, Load) : Load;
@@ -27581,6 +27645,8 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *WideMMO = MF.getMachineMemOperand(
Base->getMemOperand(), /*Offset=*/0, WideVT.getStoreSize());
+ if (LoadA->getMemCacheHint() != LoadB->getMemCacheHint())
+ WideMMO->clearMemCacheHint();
SDValue WideLoad = DAG.getLoad(WideVT, SDLoc(N), Base->getChain(),
Base->getBasePtr(), WideMMO);
// Redirect old chain users to the new chain.
@@ -31013,11 +31079,16 @@ bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDValue LHS,
MMOFlags &= ~MachineMemOperand::MOInvariant;
if (!RLD->isDereferenceable())
MMOFlags &= ~MachineMemOperand::MODereferenceable;
+ const MDNode *MemCacheHint =
+ LLD->getMemCacheHint() == RLD->getMemCacheHint()
+ ? LLD->getMemCacheHint()
+ : nullptr;
if (LLD->getExtensionType() == ISD::NON_EXTLOAD) {
// FIXME: Discards pointer and AA info.
Load = DAG.getLoad(TheSelect->getValueType(0), SDLoc(TheSelect),
LLD->getChain(), Addr, MachinePointerInfo(AddrSpace),
- Alignment, MMOFlags);
+ Alignment, MMOFlags,
+ MMOMetadata(AAMDNodes(), nullptr, MemCacheHint));
} else {
// FIXME: Discards pointer and AA info.
Load = DAG.getExtLoad(
@@ -31025,7 +31096,7 @@ bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDValue LHS,
: LLD->getExtensionType(),
SDLoc(TheSelect), TheSelect->getValueType(0), LLD->getChain(), Addr,
MachinePointerInfo(AddrSpace), LLD->getMemoryVT(), Alignment,
- MMOFlags);
+ MMOFlags, MMOMetadata(AAMDNodes(), nullptr, MemCacheHint));
}
// Users of the select now use the result of the load.
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 6e559d91952f2..abbb13de5ce37 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -5082,10 +5082,11 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
SDValue Ptr = Lod->getBasePtr();
if (bestOffset != 0)
Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(bestOffset));
- SDValue NewLoad =
- DAG.getLoad(newVT, dl, Lod->getChain(), Ptr,
- Lod->getPointerInfo().getWithOffset(bestOffset),
- Lod->getBaseAlign());
+ SDValue NewLoad = DAG.getLoad(
+ newVT, dl, Lod->getChain(), Ptr,
+ Lod->getPointerInfo().getWithOffset(bestOffset),
+ Lod->getBaseAlign(), MachineMemOperand::MONone,
+ MMOMetadata(Lod->getAAInfo(), nullptr, Lod->getMemCacheHint()));
SDValue And =
DAG.getNode(ISD::AND, dl, newVT, NewLoad,
DAG.getConstant(bestMask.trunc(bestWidth), dl, newVT));
diff --git a/llvm/test/CodeGen/AArch64/mem-cache-hint-dag-combine.ll b/llvm/test/CodeGen/AArch64/mem-cache-hint-dag-combine.ll
index d4e508fc8e8fa..ca2c69ebc8b1c 100644
--- a/llvm/test/CodeGen/AArch64/mem-cache-hint-dag-combine.ll
+++ b/llvm/test/CodeGen/AArch64/mem-cache-hint-dag-combine.ll
@@ -7,7 +7,7 @@ define <8 x i8> @combine_insert_load_same(ptr %p) {
; CHECK-NEXT: liveins: $x0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
- ; CHECK-NEXT: [[LDRDui:%[0-9]+]]:fpr64 = LDRDui [[COPY]], 0 :: (load (s64) from %ir.p, align 1)
+ ; CHECK-NEXT: [[LDRDui:%[0-9]+]]:fpr64 = LDRDui [[COPY]], 0 :: (load (s64) from %ir.p, align 1, !mem.cache_hint !1)
; CHECK-NEXT: $d0 = COPY [[LDRDui]]
; CHECK-NEXT: RET_ReallyLR implicit $d0
%p1 = getelementptr i8, ptr %p, i64 1
diff --git a/llvm/test/CodeGen/NVPTX/cache-hint-transforms.ll b/llvm/test/CodeGen/NVPTX/cache-hint-transforms.ll
index 94a0532ae1816..995230f8b4770 100644
--- a/llvm/test/CodeGen/NVPTX/cache-hint-transforms.ll
+++ b/llvm/test/CodeGen/NVPTX/cache-hint-transforms.ll
@@ -105,7 +105,7 @@ define void @test_dagcombine_store_concat_trunc_v8i32(ptr addrspace(1) %p, <4 x
define i1 @test_dagcombine_reduce_load_width(ptr addrspace(1) %p) {
; CHECK-LABEL: test_dagcombine_reduce_load_width(
-; CHECK: ld.global.b8 %rs1, [%rd1];
+; CHECK: ld.global.L1::evict_first.b8 %rs1, [%rd1];
%v = load i64, ptr addrspace(1) %p, align 8, !mem.cache_hint !2
%masked = and i64 %v, 255
%cmp = icmp eq i64 %masked, 0
diff --git a/llvm/test/CodeGen/X86/mem-cache-hint-dag-combine-i386.ll b/llvm/test/CodeGen/X86/mem-cache-hint-dag-combine-i386.ll
index 68d956502f95f..eefcccec8a6c7 100644
--- a/llvm/test/CodeGen/X86/mem-cache-hint-dag-combine-i386.ll
+++ b/llvm/test/CodeGen/X86/mem-cache-hint-dag-combine-i386.ll
@@ -5,8 +5,8 @@ define void @replace_store_of_fp_constant(ptr %p) {
; CHECK-LABEL: name: replace_store_of_fp_constant
; CHECK: bb.0 (%ir-block.0):
; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s32) from %fixed-stack.0)
- ; CHECK-NEXT: MOV32mi [[MOV32rm]], 1, $noreg, 4, $noreg, 1072939139 :: (store (s32) into %ir.p + 4, basealign 8)
- ; CHECK-NEXT: MOV32mi [[MOV32rm]], 1, $noreg, 0, $noreg, 309237645 :: (store (s32) into %ir.p, align 8)
+ ; CHECK-NEXT: MOV32mi [[MOV32rm]], 1, $noreg, 4, $noreg, 1072939139 :: (store (s32) into %ir.p + 4, basealign 8, !mem.cache_hint !1)
+ ; CHECK-NEXT: MOV32mi [[MOV32rm]], 1, $noreg, 0, $noreg, 309237645 :: (store (s32) into %ir.p, align 8, !mem.cache_hint !1)
; CHECK-NEXT: RET 0
store double 0x3FF3C083126E978D, ptr %p, align 8, !mem.cache_hint !0
ret void
@@ -16,7 +16,7 @@ define double @combine_consecutive_loads_as_double(ptr %p) {
; CHECK-LABEL: name: combine_consecutive_loads_as_double
; CHECK: bb.0 (%ir-block.0):
; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s32) from %fixed-stack.0)
- ; CHECK-NEXT: [[LD_Fp64m80_:%[0-9]+]]:rfp80 = nofpexcept LD_Fp64m80 killed [[MOV32rm]], 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (load (s64) from %ir.p)
+ ; CHECK-NEXT: [[LD_Fp64m80_:%[0-9]+]]:rfp80 = nofpexcept LD_Fp64m80 killed [[MOV32rm]], 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (load (s64) from %ir.p, !mem.cache_hint !1)
; CHECK-NEXT: RET 0, killed [[LD_Fp64m80_]]
%lo = load i32, ptr %p, align 8, !mem.cache_hint !1
%p4 = getelementptr i8, ptr %p, i32 4
diff --git a/llvm/test/CodeGen/X86/mem-cache-hint-dag-combine.ll b/llvm/test/CodeGen/X86/mem-cache-hint-dag-combine.ll
index d16fde08476ce..2c57e4f59797f 100644
--- a/llvm/test/CodeGen/X86/mem-cache-hint-dag-combine.ll
+++ b/llvm/test/CodeGen/X86/mem-cache-hint-dag-combine.ll
@@ -12,7 +12,7 @@ define i32 @funnel_shift_same(ptr %p) {
; CHECK-NEXT: liveins: $rdi
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
- ; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm [[COPY]], 1, $noreg, 3, $noreg :: (load (s32) from %ir.p + 3, align 1)
+ ; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm [[COPY]], 1, $noreg, 3, $noreg :: (load (s32) from %ir.p + 3, align 1, !mem.cache_hint !1)
; CHECK-NEXT: $eax = COPY [[MOV32rm]]
; CHECK-NEXT: RET 0, $eax
%p1 = getelementptr i8, ptr %p, i64 4
@@ -44,7 +44,7 @@ define <4 x i32> @all_ones_masked_load(ptr %p) {
; CHECK-NEXT: liveins: $rdi
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
- ; CHECK-NEXT: [[VMOVDQArm:%[0-9]+]]:vr128 = VMOVDQArm [[COPY]], 1, $noreg, 0, $noreg :: (load (s128) from %ir.p)
+ ; CHECK-NEXT: [[VMOVDQArm:%[0-9]+]]:vr128 = VMOVDQArm [[COPY]], 1, $noreg, 0, $noreg :: (load (s128) from %ir.p, !mem.cache_hint !1)
; CHECK-NEXT: $xmm0 = COPY [[VMOVDQArm]]
; CHECK-NEXT: RET 0, $xmm0
%v = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 16 %p, i32 16, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> poison), !mem.cache_hint !0
@@ -58,7 +58,7 @@ define void @all_ones_masked_store(ptr %p, <4 x i32> %v) {
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vr128 = COPY $xmm0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rdi
- ; CHECK-NEXT: VMOVDQAmr [[COPY1]], 1, $noreg, 0, $noreg, [[COPY]] :: (store (s128) into %ir.p)
+ ; CHECK-NEXT: VMOVDQAmr [[COPY1]], 1, $noreg, 0, $noreg, [[COPY]] :: (store (s128) into %ir.p, !mem.cache_hint !1)
; CHECK-NEXT: RET 0
call void @llvm.masked.store.v4i32.p0(<4 x i32> %v, ptr align 16 %p, i32 16, <4 x i1> <i1 true, i1 true, i1 true, i1 true>), !mem.cache_hint !2
ret void
@@ -70,8 +70,8 @@ define <16 x i32> @combine_ext_load(ptr %p) {
; CHECK-NEXT: liveins: $rdi
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
- ; CHECK-NEXT: [[VPMOVSXWDYrm:%[0-9]+]]:vr256 = VPMOVSXWDYrm [[COPY]], 1, $noreg, 0, $noreg :: (load (s128) from %ir.p, align 32)
- ; CHECK-NEXT: [[VPMOVSXWDYrm1:%[0-9]+]]:vr256 = VPMOVSXWDYrm [[COPY]], 1, $noreg, 16, $noreg :: (load (s128) from %ir.p + 16, basealign 32)
+ ; CHECK-NEXT: [[VPMOVSXWDYrm:%[0-9]+]]:vr256 = VPMOVSXWDYrm [[COPY]], 1, $noreg, 0, $noreg :: (load (s128) from %ir.p, align 32, !mem.cache_hint !1)
+ ; CHECK-NEXT: [[VPMOVSXWDYrm1:%[0-9]+]]:vr256 = VPMOVSXWDYrm [[COPY]], 1, $noreg, 16, $noreg :: (load (s128) from %ir.p + 16, basealign 32, !mem.cache_hint !1)
; CHECK-NEXT: $ymm0 = COPY [[VPMOVSXWDYrm]]
; CHECK-NEXT: $ymm1 = COPY [[VPMOVSXWDYrm1]]
; CHECK-NEXT: RET 0, $ymm0, $ymm1
@@ -86,7 +86,7 @@ define i8 @reduce_load_width(ptr %p) {
; CHECK-NEXT: liveins: $rdi
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
- ; CHECK-NEXT: [[MOV8rm:%[0-9]+]]:gr8 = MOV8rm [[COPY]], 1, $noreg, 0, $noreg :: (load (s8) from %ir.p, align 8)
+ ; CHECK-NEXT: [[MOV8rm:%[0-9]+]]:gr8 = MOV8rm [[COPY]], 1, $noreg, 0, $noreg :: (load (s8) from %ir.p, align 8, !mem.cache_hint !1)
; CHECK-NEXT: $al = COPY [[MOV8rm]]
; CHECK-NEXT: RET 0, $al
%v = load i64, ptr %p, align 8, !mem.cache_hint !0
@@ -105,8 +105,8 @@ define i32 @refine_alignment(i32 %v) {
; CHECK-NEXT: $rdi = COPY [[LEA64r]]
; CHECK-NEXT: CALL64pcrel32 target-flags(x86-plt) @use, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
; CHECK-NEXT: ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
- ; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %stack.0.p, 1, $noreg, 0, $noreg :: (dereferenceable load (s32) from %ir.p)
- ; CHECK-NEXT: MOV32mr %stack.0.p, 1, $noreg, 0, $noreg, [[COPY]] :: (store (s32) into %ir.p)
+ ; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %stack.0.p, 1, $noreg, 0, $noreg :: (dereferenceable load (s32) from %ir.p, !range !5, !mem.cache_hint !1)
+ ; CHECK-NEXT: MOV32mr %stack.0.p, 1, $noreg, 0, $noreg, [[COPY]] :: (store (s32) into %ir.p, !mem.cache_hint !1)
; CHECK-NEXT: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
; CHECK-NEXT: $rdi = COPY [[LEA64r]]
; CHECK-NEXT: CALL64pcrel32 target-flags(x86-plt) @use, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp
@@ -129,7 +129,7 @@ define void @shrink_load_replace_store(ptr %p, i8 %v) {
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $esi
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rdi
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr8 = COPY [[COPY]].sub_8bit
- ; CHECK-NEXT: MOV8mr [[COPY1]], 1, $noreg, 0, $noreg, killed [[COPY2]] :: (store (s8) into %ir.p, align 4)
+ ; CHECK-NEXT: MOV8mr [[COPY1]], 1, $noreg, 0, $noreg, killed [[COPY2]] :: (store (s8) into %ir.p, align 4, !alias.scope !6, !noalias !9, !mem.cache_hint !1)
; CHECK-NEXT: RET 0
%old = load i32, ptr %p, align 4
%keep = and i32 %old, -256
@@ -145,7 +145,7 @@ define void @reduce_load_op_store(ptr %p) {
; CHECK-NEXT: liveins: $rdi
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
- ; CHECK-NEXT: NOT8m [[COPY]], 1, $noreg, 0, $noreg :: (store (s8) into %ir.p, align 8), (load (s8) from %ir.p, align 8, !alias.scope !6, !noalias !9)
+ ; CHECK-NEXT: NOT8m [[COPY]], 1, $noreg, 0, $noreg :: (store (s8) into %ir.p, align 8, !alias.scope !6, !noalias !9, !mem.cache_hint !1), (load (s8) from %ir.p, align 8, !alias.scope !6, !noalias !9, !mem.cache_hint !1)
; CHECK-NEXT: RET 0
%old = load i64, ptr %p, align 8, !alias.scope !20, !noalias !23, !mem.cache_hint !0
%new = xor i64 %old, 255
@@ -161,7 +161,7 @@ define void @merge_trunc_stores_same(ptr %p, i16 %v) {
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $esi
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rdi
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr16 = COPY [[COPY]].sub_16bit
- ; CHECK-NEXT: MOV16mr [[COPY1]], 1, $noreg, 0, $noreg, killed [[COPY2]] :: (store (s16) into %ir.p)
+ ; CHECK-NEXT: MOV16mr [[COPY1]], 1, $noreg, 0, $noreg, killed [[COPY2]] :: (store (s16) into %ir.p, !mem.cache_hint !1)
; CHECK-NEXT: RET 0
%lo = trunc i16 %v to i8
store i8 %lo, ptr %p, align 2, !mem.cache_hint !2
@@ -197,7 +197,7 @@ define i32 @match_load_combine_same(ptr %p) {
; CHECK-NEXT: liveins: $rdi
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
- ; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm [[COPY]], 1, $noreg, 0, $noreg :: (load (s32) from %ir.p)
+ ; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm [[COPY]], 1, $noreg, 0, $noreg :: (load (s32) from %ir.p, !mem.cache_hint !1)
; CHECK-NEXT: $eax = COPY [[MOV32rm]]
; CHECK-NEXT: RET 0, $eax
%b0 = load i8, ptr %p, align 4, !mem.cache_hint !0
@@ -255,7 +255,7 @@ define void @merge_constant_stores_same(ptr %p) {
; CHECK-NEXT: liveins: $rdi
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
- ; CHECK-NEXT: MOV32mi [[COPY]], 1, $noreg, 0, $noreg, 67305985 :: (store (s32) into %ir.p)
+ ; CHECK-NEXT: MOV32mi [[COPY]], 1, $noreg, 0, $noreg, 67305985 :: (store (s32) into %ir.p, !mem.cache_hint !1)
; CHECK-NEXT: RET 0
store i8 1, ptr %p, align 4, !mem.cache_hint !2
%p1 = getelementptr i8, ptr %p, i64 1
@@ -292,8 +292,8 @@ define void @merge_store_of_loads_same(ptr noalias %dst, ptr noalias %src) {
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rsi
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rdi
- ; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm [[COPY]], 1, $noreg, 0, $noreg :: (load (s32) from %ir.src)
- ; CHECK-NEXT: MOV32mr [[COPY1]], 1, $noreg, 0, $noreg, killed [[MOV32rm]] :: (store (s32) into %ir.dst)
+ ; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm [[COPY]], 1, $noreg, 0, $noreg :: (load (s32) from %ir.src, !mem.cache_hint !1)
+ ; CHECK-NEXT: MOV32mr [[COPY1]], 1, $noreg, 0, $noreg, killed [[MOV32rm]] :: (store (s32) into %ir.dst, !mem.cache_hint !1)
; CHECK-NEXT: RET 0
%a = load i8, ptr %src, align 4, !mem.cache_hint !0
%s1 = getelementptr i8, ptr %src, i64 1
@@ -346,7 +346,7 @@ define void @replace_store_of_insert_load(ptr %p, i32 %v) {
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $esi
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rdi
- ; CHECK-NEXT: MOV32mr [[COPY1]], 1, $noreg, 8, $noreg, [[COPY]] :: (store (s32) into %ir.p + 8, align 8, basealign 16)
+ ; CHECK-NEXT: MOV32mr [[COPY1]], 1, $noreg, 8, $noreg, [[COPY]] :: (store (s32) into %ir.p + 8, align 8, basealign 16, !alias.scope !6, !noalias !9, !mem.cache_hint !1)
; CHECK-NEXT: RET 0
%old = load <4 x i32>, ptr %p, align 16
%new = insertelement <4 x i32> %old, i32 %v, i32 2
@@ -378,7 +378,7 @@ define <4 x i32> @concat_shuffles_different(ptr %p) {
; CHECK-NEXT: liveins: $rdi
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
- ; CHECK-NEXT: [[VPSHUFDmi:%[0-9]+]]:vr128 = VPSHUFDmi [[COPY]], 1, $noreg, 0, $noreg, -40 :: (load (s128) from %ir.p, !mem.cache_hint !1)
+ ; CHECK-NEXT: [[VPSHUFDmi:%[0-9]+]]:vr128 = VPSHUFDmi [[COPY]], 1, $noreg, 0, $noreg, -40 :: (load (s128) from %ir.p)
; CHECK-NEXT: $xmm0 = COPY [[VPSHUFDmi]]
; CHECK-NEXT: RET 0, $xmm0
%a = load <2 x i32>, ptr %p, align 16, !mem.cache_hint !0
@@ -401,7 +401,7 @@ define i32 @select_loads_same(i1 %c, ptr %p, ptr %q) {
; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr8 = COPY [[COPY2]].sub_8bit
; CHECK-NEXT: TEST8ri killed [[COPY3]], 1, implicit-def $eflags
; CHECK-NEXT: [[CMOV64rr:%[0-9]+]]:gr64 = CMOV64rr [[COPY]], [[COPY1]], 5, implicit $eflags
- ; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm killed [[CMOV64rr]], 1, $noreg, 0, $noreg :: (load (s32))
+ ; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm killed [[CMOV64rr]], 1, $noreg, 0, $noreg :: (load (s32), !mem.cache_hint !1)
; CHECK-NEXT: $eax = COPY [[MOV32rm]]
; CHECK-NEXT: RET 0, $eax
%a = load i32, ptr %p, align 4, !mem.cache_hint !0
@@ -436,7 +436,7 @@ define i1 @simplify_setcc(ptr %p) {
; CHECK-NEXT: liveins: $rdi
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
- ; CHECK-NEXT: CMP8mi [[COPY]], 1, $noreg, 0, $noreg, 0, implicit-def $eflags :: (load (s8) from %ir.p, align 8, !alias.scope !6, !noalias !9)
+ ; CHECK-NEXT: CMP8mi [[COPY]], 1, $noreg, 0, $noreg, 0, implicit-def $eflags :: (load (s8) from %ir.p, align 8, !alias.scope !6, !noalias !9, !mem.cache_hint !1)
; CHECK-NEXT: [[SETCCr:%[0-9]+]]:gr8 = SETCCr 4, implicit $eflags
; CHECK-NEXT: $al = COPY [[SETCCr]]
; CHECK-NEXT: RET 0, $al
>From 39610ba15733d953a67745f72ca7bbc85155444b Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 23 Sep 2026 21:29:59 +0000
Subject: [PATCH 3/5] [SelectionDAG] Use getNonRangeMMOMetadata in DAG combines
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 23 ++++++++-----------
.../CodeGen/SelectionDAG/TargetLowering.cpp | 2 +-
2 files changed, 10 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 748d5bdec9c04..841312090dc65 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -13940,8 +13940,7 @@ SDValue DAGCombiner::visitMSTORE(SDNode *N) {
return DAG.getStore(
MST->getChain(), SDLoc(N), MST->getValue(), MST->getBasePtr(),
MST->getPointerInfo(), MST->getBaseAlign(),
- MST->getMemOperand()->getFlags(),
- MMOMetadata(MST->getAAInfo(), nullptr, MST->getMemCacheHint()));
+ MST->getMemOperand()->getFlags(), MST->getNonRangeMMOMetadata());
// Try transforming N to an indexed store.
if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
@@ -15461,7 +15460,7 @@ SDValue DAGCombiner::CombineExtLoad(SDNode *N) {
ExtType, SDLoc(LN0), SplitDstVT, LN0->getChain(), BasePtr,
LN0->getPointerInfo().getWithOffset(Offset), SplitSrcVT,
LN0->getBaseAlign(), LN0->getMemOperand()->getFlags(),
- MMOMetadata(LN0->getAAInfo(), nullptr, LN0->getMemCacheHint()));
+ LN0->getNonRangeMMOMetadata());
BasePtr = DAG.getMemBasePlusOffset(BasePtr, TypeSize::getFixed(Stride), DL);
@@ -17284,8 +17283,7 @@ SDValue DAGCombiner::reduceLoadWidth(SDNode *N) {
Load = DAG.getExtLoad(
ExtType, DL, VT, LN0->getChain(), NewPtr,
LN0->getPointerInfo().getWithOffset(PtrOff), ExtVT, LN0->getBaseAlign(),
- LN0->getMemOperand()->getFlags(),
- MMOMetadata(LN0->getAAInfo(), nullptr, LN0->getMemCacheHint()));
+ LN0->getMemOperand()->getFlags(), LN0->getNonRangeMMOMetadata());
// Replace the old load's chain with the new load's chain.
WorklistRemover DeadNodes(*this);
@@ -22862,7 +22860,7 @@ ShrinkLoadReplaceStoreWithStore(const std::pair<unsigned, unsigned> &MaskInfo,
}
++OpsNarrowed;
- MMOMetadata Metadata(St->getAAInfo(), nullptr, St->getMemCacheHint());
+ MMOMetadata Metadata = St->getNonRangeMMOMetadata();
if (UseTruncStore)
return DAG.getTruncStore(St->getChain(), SDLoc(St), IVal, Ptr,
St->getPointerInfo().getWithOffset(StOffset), VT,
@@ -23014,15 +23012,13 @@ SDValue DAGCombiner::ReduceLoadOpStoreWidth(SDNode *N) {
SDValue NewLD = DAG.getLoad(
NewVT, SDLoc(N0), LD->getChain(), NewPtr,
LD->getPointerInfo().getWithOffset(PtrOff), NewAlign,
- LD->getMemOperand()->getFlags(),
- MMOMetadata(LD->getAAInfo(), nullptr, LD->getMemCacheHint()));
+ LD->getMemOperand()->getFlags(), LD->getNonRangeMMOMetadata());
SDValue NewVal = DAG.getNode(Opc, SDLoc(Value), NewVT, NewLD,
DAG.getConstant(NewImm, SDLoc(Value), NewVT));
SDValue NewST = DAG.getStore(
Chain, SDLoc(N), NewVal, NewPtr,
ST->getPointerInfo().getWithOffset(PtrOff), NewAlign,
- MachineMemOperand::MONone,
- MMOMetadata(ST->getAAInfo(), nullptr, ST->getMemCacheHint()));
+ MachineMemOperand::MONone, ST->getNonRangeMMOMetadata());
AddToWorklist(NewPtr.getNode());
AddToWorklist(NewLD.getNode());
@@ -24422,7 +24418,7 @@ SDValue DAGCombiner::replaceStoreOfFPConstant(StoreSDNode *ST) {
std::swap(Lo, Hi);
MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
- MMOMetadata Metadata(ST->getAAInfo(), nullptr, ST->getMemCacheHint());
+ MMOMetadata Metadata = ST->getNonRangeMMOMetadata();
SDValue St0 = DAG.getStore(Chain, DL, Lo, Ptr, ST->getPointerInfo(),
ST->getBaseAlign(), MMOFlags, Metadata);
@@ -24500,8 +24496,7 @@ SDValue DAGCombiner::replaceStoreOfInsertLoad(StoreSDNode *ST) {
return DAG.getStore(
Chain, DL, Elt, NewPtr, PointerInfo, NewAlign,
- ST->getMemOperand()->getFlags(),
- MMOMetadata(ST->getAAInfo(), nullptr, ST->getMemCacheHint()));
+ ST->getMemOperand()->getFlags(), ST->getNonRangeMMOMetadata());
}
SDValue DAGCombiner::visitATOMIC_STORE(SDNode *N) {
@@ -24695,7 +24690,7 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) {
SDValue NewStore = DAG.getTruncStore(
Chain, SDLoc(N), Value, Ptr, ST->getOffset(), ST->getPointerInfo(),
ST->getMemoryVT(), *Alignment, ST->getMemOperand()->getFlags(),
- MMOMetadata(ST->getAAInfo(), nullptr, ST->getMemCacheHint()));
+ ST->getNonRangeMMOMetadata());
// NewStore will always be N as we are only refining the alignment
assert(NewStore.getNode() == N);
(void)NewStore;
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index abbb13de5ce37..a3e056dcf25d6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -5086,7 +5086,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
newVT, dl, Lod->getChain(), Ptr,
Lod->getPointerInfo().getWithOffset(bestOffset),
Lod->getBaseAlign(), MachineMemOperand::MONone,
- MMOMetadata(Lod->getAAInfo(), nullptr, Lod->getMemCacheHint()));
+ Lod->getNonRangeMMOMetadata());
SDValue And =
DAG.getNode(ISD::AND, dl, newVT, NewLoad,
DAG.getConstant(bestMask.trunc(bestWidth), dl, newVT));
>From 61c6a3f15c51f739bbf9271a87bf27d84d0cab90 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 23 Sep 2026 22:36:48 +0000
Subject: [PATCH 4/5] [SelectionDAG] Use all_equal for common cache hints
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 40 +++++++++----------
1 file changed, 20 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 841312090dc65..e138a2c8077bb 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -10149,12 +10149,10 @@ SDValue DAGCombiner::mergeTruncStores(StoreSDNode *N) {
SourceValue = DAG.getNode(ISD::ROTR, DL, WideVT, SourceValue, RotAmt);
}
- const MDNode *MemCacheHint = Stores.front()->getMemCacheHint();
- for (StoreSDNode *Store : drop_begin(Stores))
- if (Store->getMemCacheHint() != MemCacheHint) {
- MemCacheHint = nullptr;
- break;
- }
+ auto MemCacheHints = map_range(
+ Stores, [](StoreSDNode *Store) { return Store->getMemCacheHint(); });
+ const MDNode *MemCacheHint =
+ all_equal(MemCacheHints) ? Stores.front()->getMemCacheHint() : nullptr;
SDValue NewStore =
DAG.getStore(Chain, DL, SourceValue, FirstStore->getBasePtr(),
@@ -10363,12 +10361,10 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
if (!Allowed || !Fast)
return SDValue();
- const MDNode *MemCacheHint = FirstLoad->getMemCacheHint();
- for (LoadSDNode *Load : Loads)
- if (Load->getMemCacheHint() != MemCacheHint) {
- MemCacheHint = nullptr;
- break;
- }
+ auto MemCacheHints = map_range(
+ Loads, [](LoadSDNode *Load) { return Load->getMemCacheHint(); });
+ const MDNode *MemCacheHint =
+ all_equal(MemCacheHints) ? FirstLoad->getMemCacheHint() : nullptr;
SDValue NewLoad = DAG.getExtLoad(
NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, SDLoc(N), VT, Chain,
@@ -24166,14 +24162,18 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
StMMOFlags |= TLI.getTargetMMOFlags(*StoreNodes[0].MemNode);
- const MDNode *LoadMemCacheHint = FirstLoad->getMemCacheHint();
- const MDNode *StoreMemCacheHint = FirstInChain->getMemCacheHint();
- for (unsigned I = 1; I != NumElem; ++I) {
- if (LoadNodes[I].MemNode->getMemCacheHint() != LoadMemCacheHint)
- LoadMemCacheHint = nullptr;
- if (StoreNodes[I].MemNode->getMemCacheHint() != StoreMemCacheHint)
- StoreMemCacheHint = nullptr;
- }
+ auto GetMemCacheHint = [](const MemOpLink &MemOp) {
+ return MemOp.MemNode->getMemCacheHint();
+ };
+ auto LoadMemCacheHints =
+ map_range(ArrayRef(LoadNodes).take_front(NumElem), GetMemCacheHint);
+ const MDNode *LoadMemCacheHint =
+ all_equal(LoadMemCacheHints) ? FirstLoad->getMemCacheHint() : nullptr;
+ auto StoreMemCacheHints =
+ map_range(ArrayRef(StoreNodes).take_front(NumElem), GetMemCacheHint);
+ const MDNode *StoreMemCacheHint = all_equal(StoreMemCacheHints)
+ ? FirstInChain->getMemCacheHint()
+ : nullptr;
SDValue NewLoad, NewStore;
if (UseVectorTy || !DoIntegerTruncate) {
>From b59ddca97f11e948b7bc85d8d135b0f3657b3895 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 23 Sep 2026 22:46:12 +0000
Subject: [PATCH 5/5] format
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 38 +++++++++----------
.../CodeGen/SelectionDAG/TargetLowering.cpp | 10 ++---
2 files changed, 24 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index e138a2c8077bb..fc61b9a189163 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -13933,10 +13933,10 @@ SDValue DAGCombiner::visitMSTORE(SDNode *N) {
// FIXME: Can we do this for indexed, compressing, or truncating stores?
if (ISD::isConstantSplatVectorAllOnes(Mask.getNode()) && MST->isUnindexed() &&
!MST->isCompressingStore() && !MST->isTruncatingStore())
- return DAG.getStore(
- MST->getChain(), SDLoc(N), MST->getValue(), MST->getBasePtr(),
- MST->getPointerInfo(), MST->getBaseAlign(),
- MST->getMemOperand()->getFlags(), MST->getNonRangeMMOMetadata());
+ return DAG.getStore(MST->getChain(), SDLoc(N), MST->getValue(),
+ MST->getBasePtr(), MST->getPointerInfo(),
+ MST->getBaseAlign(), MST->getMemOperand()->getFlags(),
+ MST->getNonRangeMMOMetadata());
// Try transforming N to an indexed store.
if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
@@ -17276,10 +17276,10 @@ SDValue DAGCombiner::reduceLoadWidth(SDNode *N) {
LN0->getMemOperand()->getFlags(),
MMOMetadata(LN0->getAAInfo(), NewRanges, LN0->getMemCacheHint()));
} else
- Load = DAG.getExtLoad(
- ExtType, DL, VT, LN0->getChain(), NewPtr,
- LN0->getPointerInfo().getWithOffset(PtrOff), ExtVT, LN0->getBaseAlign(),
- LN0->getMemOperand()->getFlags(), LN0->getNonRangeMMOMetadata());
+ Load = DAG.getExtLoad(ExtType, DL, VT, LN0->getChain(), NewPtr,
+ LN0->getPointerInfo().getWithOffset(PtrOff), ExtVT,
+ LN0->getBaseAlign(), LN0->getMemOperand()->getFlags(),
+ LN0->getNonRangeMMOMetadata());
// Replace the old load's chain with the new load's chain.
WorklistRemover DeadNodes(*this);
@@ -23005,16 +23005,16 @@ SDValue DAGCombiner::ReduceLoadOpStoreWidth(SDNode *N) {
Align NewAlign = commonAlignment(LD->getAlign(), PtrOff);
SDValue NewPtr =
DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(PtrOff), SDLoc(LD));
- SDValue NewLD = DAG.getLoad(
- NewVT, SDLoc(N0), LD->getChain(), NewPtr,
- LD->getPointerInfo().getWithOffset(PtrOff), NewAlign,
- LD->getMemOperand()->getFlags(), LD->getNonRangeMMOMetadata());
+ SDValue NewLD = DAG.getLoad(NewVT, SDLoc(N0), LD->getChain(), NewPtr,
+ LD->getPointerInfo().getWithOffset(PtrOff),
+ NewAlign, LD->getMemOperand()->getFlags(),
+ LD->getNonRangeMMOMetadata());
SDValue NewVal = DAG.getNode(Opc, SDLoc(Value), NewVT, NewLD,
DAG.getConstant(NewImm, SDLoc(Value), NewVT));
- SDValue NewST = DAG.getStore(
- Chain, SDLoc(N), NewVal, NewPtr,
- ST->getPointerInfo().getWithOffset(PtrOff), NewAlign,
- MachineMemOperand::MONone, ST->getNonRangeMMOMetadata());
+ SDValue NewST =
+ DAG.getStore(Chain, SDLoc(N), NewVal, NewPtr,
+ ST->getPointerInfo().getWithOffset(PtrOff), NewAlign,
+ MachineMemOperand::MONone, ST->getNonRangeMMOMetadata());
AddToWorklist(NewPtr.getNode());
AddToWorklist(NewLD.getNode());
@@ -24494,9 +24494,9 @@ SDValue DAGCombiner::replaceStoreOfInsertLoad(StoreSDNode *ST) {
NewAlign = commonAlignment(ST->getAlign(), EltVT.getFixedSizeInBits() / 8);
}
- return DAG.getStore(
- Chain, DL, Elt, NewPtr, PointerInfo, NewAlign,
- ST->getMemOperand()->getFlags(), ST->getNonRangeMMOMetadata());
+ return DAG.getStore(Chain, DL, Elt, NewPtr, PointerInfo, NewAlign,
+ ST->getMemOperand()->getFlags(),
+ ST->getNonRangeMMOMetadata());
}
SDValue DAGCombiner::visitATOMIC_STORE(SDNode *N) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index a3e056dcf25d6..a0aef1e9f287b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -5082,11 +5082,11 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
SDValue Ptr = Lod->getBasePtr();
if (bestOffset != 0)
Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(bestOffset));
- SDValue NewLoad = DAG.getLoad(
- newVT, dl, Lod->getChain(), Ptr,
- Lod->getPointerInfo().getWithOffset(bestOffset),
- Lod->getBaseAlign(), MachineMemOperand::MONone,
- Lod->getNonRangeMMOMetadata());
+ SDValue NewLoad =
+ DAG.getLoad(newVT, dl, Lod->getChain(), Ptr,
+ Lod->getPointerInfo().getWithOffset(bestOffset),
+ Lod->getBaseAlign(), MachineMemOperand::MONone,
+ Lod->getNonRangeMMOMetadata());
SDValue And =
DAG.getNode(ISD::AND, dl, newVT, NewLoad,
DAG.getConstant(bestMask.trunc(bestWidth), dl, newVT));
More information about the llvm-commits
mailing list