[llvm] [SelectionDAG] Apply target MMO flags to masked load/store (PR #212228)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 6 05:17:50 PDT 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/212228
>From efb86cc638ec8aaad9a1050394e0553215c9c49d Mon Sep 17 00:00:00 2001
From: Stefan Pejic <stefan.pejic at nextsilicon.com>
Date: Fri, 24 Jul 2026 11:52:37 +0200
Subject: [PATCH] [SelectionDAG] Apply target MMO flags to masked load/store
visitMaskedLoad and visitMaskedStore ignore the target MMO flags. For
RISC-V, this manifests itself by LLVM generating ntl.p1 instruction
regardless of the metadata attached to the masked load/store IR
instruction.
This change makes sure that visitMaskedLoad/Store properly take into
account the attached metadata and generate corresponding instruction.
Co-authored-by: Claude (Opus 4.8) <noreply at anthropic.com>
---
.../SelectionDAG/SelectionDAGBuilder.cpp | 10 +-
.../CodeGen/RISCV/nontemporal-scalable.ll | 190 ++++++++++++++++++
2 files changed, 196 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index effa4a8d5f1b9..3caed0b46699b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -5005,7 +5005,10 @@ void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
EVT VT = Src0.getValueType();
+ const auto &TLI = DAG.getTargetLoweringInfo();
+
auto MMOFlags = MachineMemOperand::MOStore;
+ MMOFlags |= TLI.getTargetMMOFlags(I);
if (I.hasMetadata(LLVMContext::MD_nontemporal))
MMOFlags |= MachineMemOperand::MONonTemporal;
@@ -5014,8 +5017,6 @@ void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
LocationSize::upperBound(VT.getStoreSize()), Alignment,
I.getAAMetadata());
- const auto &TLI = DAG.getTargetLoweringInfo();
-
SDValue StoreNode =
!IsCompressing && TTI->hasConditionalLoadStoreForType(
I.getArgOperand(0)->getType(), /*IsStore=*/true)
@@ -5163,7 +5164,10 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
+ const auto &TLI = DAG.getTargetLoweringInfo();
+
auto MMOFlags = MachineMemOperand::MOLoad;
+ MMOFlags |= TLI.getTargetMMOFlags(I);
if (I.hasMetadata(LLVMContext::MD_nontemporal))
MMOFlags |= MachineMemOperand::MONonTemporal;
if (I.hasMetadata(LLVMContext::MD_invariant_load))
@@ -5173,8 +5177,6 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
MachinePointerInfo(PtrOperand), MMOFlags,
LocationSize::upperBound(VT.getStoreSize()), Alignment, AAInfo, Ranges);
- const auto &TLI = DAG.getTargetLoweringInfo();
-
// The Load/Res may point to different values and both of them are output
// variables.
SDValue Load;
diff --git a/llvm/test/CodeGen/RISCV/nontemporal-scalable.ll b/llvm/test/CodeGen/RISCV/nontemporal-scalable.ll
index 460a1d94b6871..2e189029b9d1c 100644
--- a/llvm/test/CodeGen/RISCV/nontemporal-scalable.ll
+++ b/llvm/test/CodeGen/RISCV/nontemporal-scalable.ll
@@ -642,6 +642,196 @@ define void @test_nontemporal_ALL_store_nxv16i8(ptr %p, <vscale x 16 x i8> %v) {
ret void
}
+define <vscale x 2 x i64> @test_nontemporal_masked_load_nxv2i64(ptr %p, <vscale x 2 x i1> %m) {
+; CHECK-RV64V-LABEL: test_nontemporal_masked_load_nxv2i64:
+; CHECK-RV64V: # %bb.0:
+; CHECK-RV64V-NEXT: vsetvli a1, zero, e64, m2, ta, mu
+; CHECK-RV64V-NEXT: vmv.v.i v8, 0
+; CHECK-RV64V-NEXT: ntl.all
+; CHECK-RV64V-NEXT: vle64.v v8, (a0), v0.t
+; CHECK-RV64V-NEXT: ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_masked_load_nxv2i64:
+; CHECK-RV32V: # %bb.0:
+; CHECK-RV32V-NEXT: vsetvli a1, zero, e64, m2, ta, mu
+; CHECK-RV32V-NEXT: vmv.v.i v8, 0
+; CHECK-RV32V-NEXT: ntl.all
+; CHECK-RV32V-NEXT: vle64.v v8, (a0), v0.t
+; CHECK-RV32V-NEXT: ret
+ %1 = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr %p, i32 8, <vscale x 2 x i1> %m, <vscale x 2 x i64> zeroinitializer), !nontemporal !0
+ ret <vscale x 2 x i64> %1
+}
+
+define <vscale x 2 x i64> @test_nontemporal_P1_masked_load_nxv2i64(ptr %p, <vscale x 2 x i1> %m) {
+; CHECK-RV64V-LABEL: test_nontemporal_P1_masked_load_nxv2i64:
+; CHECK-RV64V: # %bb.0:
+; CHECK-RV64V-NEXT: vsetvli a1, zero, e64, m2, ta, mu
+; CHECK-RV64V-NEXT: vmv.v.i v8, 0
+; CHECK-RV64V-NEXT: ntl.p1
+; CHECK-RV64V-NEXT: vle64.v v8, (a0), v0.t
+; CHECK-RV64V-NEXT: ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_masked_load_nxv2i64:
+; CHECK-RV32V: # %bb.0:
+; CHECK-RV32V-NEXT: vsetvli a1, zero, e64, m2, ta, mu
+; CHECK-RV32V-NEXT: vmv.v.i v8, 0
+; CHECK-RV32V-NEXT: ntl.p1
+; CHECK-RV32V-NEXT: vle64.v v8, (a0), v0.t
+; CHECK-RV32V-NEXT: ret
+ %1 = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr %p, i32 8, <vscale x 2 x i1> %m, <vscale x 2 x i64> zeroinitializer), !nontemporal !0, !riscv-nontemporal-domain !1
+ ret <vscale x 2 x i64> %1
+}
+
+define <vscale x 2 x i64> @test_nontemporal_PALL_masked_load_nxv2i64(ptr %p, <vscale x 2 x i1> %m) {
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_masked_load_nxv2i64:
+; CHECK-RV64V: # %bb.0:
+; CHECK-RV64V-NEXT: vsetvli a1, zero, e64, m2, ta, mu
+; CHECK-RV64V-NEXT: vmv.v.i v8, 0
+; CHECK-RV64V-NEXT: ntl.pall
+; CHECK-RV64V-NEXT: vle64.v v8, (a0), v0.t
+; CHECK-RV64V-NEXT: ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_masked_load_nxv2i64:
+; CHECK-RV32V: # %bb.0:
+; CHECK-RV32V-NEXT: vsetvli a1, zero, e64, m2, ta, mu
+; CHECK-RV32V-NEXT: vmv.v.i v8, 0
+; CHECK-RV32V-NEXT: ntl.pall
+; CHECK-RV32V-NEXT: vle64.v v8, (a0), v0.t
+; CHECK-RV32V-NEXT: ret
+ %1 = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr %p, i32 8, <vscale x 2 x i1> %m, <vscale x 2 x i64> zeroinitializer), !nontemporal !0, !riscv-nontemporal-domain !2
+ ret <vscale x 2 x i64> %1
+}
+
+define <vscale x 2 x i64> @test_nontemporal_S1_masked_load_nxv2i64(ptr %p, <vscale x 2 x i1> %m) {
+; CHECK-RV64V-LABEL: test_nontemporal_S1_masked_load_nxv2i64:
+; CHECK-RV64V: # %bb.0:
+; CHECK-RV64V-NEXT: vsetvli a1, zero, e64, m2, ta, mu
+; CHECK-RV64V-NEXT: vmv.v.i v8, 0
+; CHECK-RV64V-NEXT: ntl.s1
+; CHECK-RV64V-NEXT: vle64.v v8, (a0), v0.t
+; CHECK-RV64V-NEXT: ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_masked_load_nxv2i64:
+; CHECK-RV32V: # %bb.0:
+; CHECK-RV32V-NEXT: vsetvli a1, zero, e64, m2, ta, mu
+; CHECK-RV32V-NEXT: vmv.v.i v8, 0
+; CHECK-RV32V-NEXT: ntl.s1
+; CHECK-RV32V-NEXT: vle64.v v8, (a0), v0.t
+; CHECK-RV32V-NEXT: ret
+ %1 = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr %p, i32 8, <vscale x 2 x i1> %m, <vscale x 2 x i64> zeroinitializer), !nontemporal !0, !riscv-nontemporal-domain !3
+ ret <vscale x 2 x i64> %1
+}
+
+define <vscale x 2 x i64> @test_nontemporal_ALL_masked_load_nxv2i64(ptr %p, <vscale x 2 x i1> %m) {
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_masked_load_nxv2i64:
+; CHECK-RV64V: # %bb.0:
+; CHECK-RV64V-NEXT: vsetvli a1, zero, e64, m2, ta, mu
+; CHECK-RV64V-NEXT: vmv.v.i v8, 0
+; CHECK-RV64V-NEXT: ntl.all
+; CHECK-RV64V-NEXT: vle64.v v8, (a0), v0.t
+; CHECK-RV64V-NEXT: ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_masked_load_nxv2i64:
+; CHECK-RV32V: # %bb.0:
+; CHECK-RV32V-NEXT: vsetvli a1, zero, e64, m2, ta, mu
+; CHECK-RV32V-NEXT: vmv.v.i v8, 0
+; CHECK-RV32V-NEXT: ntl.all
+; CHECK-RV32V-NEXT: vle64.v v8, (a0), v0.t
+; CHECK-RV32V-NEXT: ret
+ %1 = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr %p, i32 8, <vscale x 2 x i1> %m, <vscale x 2 x i64> zeroinitializer), !nontemporal !0, !riscv-nontemporal-domain !4
+ ret <vscale x 2 x i64> %1
+}
+
+define void @test_nontemporal_masked_store_nxv2i64(ptr %p, <vscale x 2 x i64> %v, <vscale x 2 x i1> %m) {
+; CHECK-RV64V-LABEL: test_nontemporal_masked_store_nxv2i64:
+; CHECK-RV64V: # %bb.0:
+; CHECK-RV64V-NEXT: vsetvli a1, zero, e64, m2, ta, ma
+; CHECK-RV64V-NEXT: ntl.all
+; CHECK-RV64V-NEXT: vse64.v v8, (a0), v0.t
+; CHECK-RV64V-NEXT: ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_masked_store_nxv2i64:
+; CHECK-RV32V: # %bb.0:
+; CHECK-RV32V-NEXT: vsetvli a1, zero, e64, m2, ta, ma
+; CHECK-RV32V-NEXT: ntl.all
+; CHECK-RV32V-NEXT: vse64.v v8, (a0), v0.t
+; CHECK-RV32V-NEXT: ret
+ call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> %v, ptr %p, i32 8, <vscale x 2 x i1> %m), !nontemporal !0
+ ret void
+}
+
+define void @test_nontemporal_P1_masked_store_nxv2i64(ptr %p, <vscale x 2 x i64> %v, <vscale x 2 x i1> %m) {
+; CHECK-RV64V-LABEL: test_nontemporal_P1_masked_store_nxv2i64:
+; CHECK-RV64V: # %bb.0:
+; CHECK-RV64V-NEXT: vsetvli a1, zero, e64, m2, ta, ma
+; CHECK-RV64V-NEXT: ntl.p1
+; CHECK-RV64V-NEXT: vse64.v v8, (a0), v0.t
+; CHECK-RV64V-NEXT: ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_masked_store_nxv2i64:
+; CHECK-RV32V: # %bb.0:
+; CHECK-RV32V-NEXT: vsetvli a1, zero, e64, m2, ta, ma
+; CHECK-RV32V-NEXT: ntl.p1
+; CHECK-RV32V-NEXT: vse64.v v8, (a0), v0.t
+; CHECK-RV32V-NEXT: ret
+ call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> %v, ptr %p, i32 8, <vscale x 2 x i1> %m), !nontemporal !0, !riscv-nontemporal-domain !1
+ ret void
+}
+
+define void @test_nontemporal_PALL_masked_store_nxv2i64(ptr %p, <vscale x 2 x i64> %v, <vscale x 2 x i1> %m) {
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_masked_store_nxv2i64:
+; CHECK-RV64V: # %bb.0:
+; CHECK-RV64V-NEXT: vsetvli a1, zero, e64, m2, ta, ma
+; CHECK-RV64V-NEXT: ntl.pall
+; CHECK-RV64V-NEXT: vse64.v v8, (a0), v0.t
+; CHECK-RV64V-NEXT: ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_masked_store_nxv2i64:
+; CHECK-RV32V: # %bb.0:
+; CHECK-RV32V-NEXT: vsetvli a1, zero, e64, m2, ta, ma
+; CHECK-RV32V-NEXT: ntl.pall
+; CHECK-RV32V-NEXT: vse64.v v8, (a0), v0.t
+; CHECK-RV32V-NEXT: ret
+ call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> %v, ptr %p, i32 8, <vscale x 2 x i1> %m), !nontemporal !0, !riscv-nontemporal-domain !2
+ ret void
+}
+
+define void @test_nontemporal_S1_masked_store_nxv2i64(ptr %p, <vscale x 2 x i64> %v, <vscale x 2 x i1> %m) {
+; CHECK-RV64V-LABEL: test_nontemporal_S1_masked_store_nxv2i64:
+; CHECK-RV64V: # %bb.0:
+; CHECK-RV64V-NEXT: vsetvli a1, zero, e64, m2, ta, ma
+; CHECK-RV64V-NEXT: ntl.s1
+; CHECK-RV64V-NEXT: vse64.v v8, (a0), v0.t
+; CHECK-RV64V-NEXT: ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_masked_store_nxv2i64:
+; CHECK-RV32V: # %bb.0:
+; CHECK-RV32V-NEXT: vsetvli a1, zero, e64, m2, ta, ma
+; CHECK-RV32V-NEXT: ntl.s1
+; CHECK-RV32V-NEXT: vse64.v v8, (a0), v0.t
+; CHECK-RV32V-NEXT: ret
+ call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> %v, ptr %p, i32 8, <vscale x 2 x i1> %m), !nontemporal !0, !riscv-nontemporal-domain !3
+ ret void
+}
+
+define void @test_nontemporal_ALL_masked_store_nxv2i64(ptr %p, <vscale x 2 x i64> %v, <vscale x 2 x i1> %m) {
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_masked_store_nxv2i64:
+; CHECK-RV64V: # %bb.0:
+; CHECK-RV64V-NEXT: vsetvli a1, zero, e64, m2, ta, ma
+; CHECK-RV64V-NEXT: ntl.all
+; CHECK-RV64V-NEXT: vse64.v v8, (a0), v0.t
+; CHECK-RV64V-NEXT: ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_masked_store_nxv2i64:
+; CHECK-RV32V: # %bb.0:
+; CHECK-RV32V-NEXT: vsetvli a1, zero, e64, m2, ta, ma
+; CHECK-RV32V-NEXT: ntl.all
+; CHECK-RV32V-NEXT: vse64.v v8, (a0), v0.t
+; CHECK-RV32V-NEXT: ret
+ call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> %v, ptr %p, i32 8, <vscale x 2 x i1> %m), !nontemporal !0, !riscv-nontemporal-domain !4
+ ret void
+}
+
!0 = !{i32 1}
!1 = !{i32 2}
!2 = !{i32 3}
More information about the llvm-commits
mailing list