[llvm] [X86] Add llvm.x86.movnt, an opaque non-temporal store intrinsic (PR #228875)
Jarrod Petz via llvm-commits
llvm-commits at lists.llvm.org
Sun Oct 4 05:25:39 PDT 2026
https://github.com/jlpetz updated https://github.com/llvm/llvm-project/pull/228875
>From 0d8ae8ad39358b0b2f7dffbd3422f67100479004 Mon Sep 17 00:00:00 2001
From: Jarrod Petz <petz at nisshoko.net>
Date: Sun, 4 Oct 2026 23:25:26 +1100
Subject: [PATCH] [X86] Add llvm.x86.movnt, an opaque non-temporal store
intrinsic
llvm.x86.movnt is a non-temporal store that the optimizer treats as an
opaque memory operation rather than an ordinary store: it is overloaded on
the stored type and has no memory attributes, like the MOVNTQ, MOVDIRI and
MOVDIR64B intrinsics. The X86 backend lowers it to a volatile non-temporal
store, which selects the same MOVNTI/MOVNTPS/MOVNTPD/MOVNTDQ instructions
as a !nontemporal store while keeping the backend from merging,
eliminating or reordering it.
How !nontemporal stores interact with LLVM's memory model is unspecified
(#64521), so frontends that need a defined operation, such as Rust's
std::arch, use inline assembly today. This gives them an alternative with
the same semantics as that asm and better code generation.
Assisted-by: Claude Opus 5.5
---
llvm/docs/ReleaseNotes.md | 3 +
llvm/include/llvm/IR/IntrinsicsX86.td | 13 ++
llvm/lib/Target/X86/X86ISelLowering.cpp | 27 +++
llvm/test/CodeGen/X86/movnt-intrinsic.ll | 220 ++++++++++++++++++
.../x86-movnt-intrinsic.ll | 27 +++
5 files changed, 290 insertions(+)
create mode 100644 llvm/test/CodeGen/X86/movnt-intrinsic.ll
create mode 100644 llvm/test/Transforms/DeadStoreElimination/x86-movnt-intrinsic.ll
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 9a5dbbcdb6609..53d53e72699fa 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -302,6 +302,9 @@ Makes programs 10x faster by doing Special New Thing.
* Added assembler and code generation support for the `AVX10_V2_AUX`
instruction set.
+* Added the `llvm.x86.movnt` intrinsic: a non-temporal store (`MOVNTI`,
+ `MOVNTPS`, `MOVNTPD` or `MOVNTDQ`) that is not optimized as an ordinary
+ store, like the existing `MOVNTQ`, `MOVDIRI` and `MOVDIR64B` intrinsics.
### Changes to the OCaml bindings
diff --git a/llvm/include/llvm/IR/IntrinsicsX86.td b/llvm/include/llvm/IR/IntrinsicsX86.td
index 3617008d2ee8e..ce4ebefc58222 100644
--- a/llvm/include/llvm/IR/IntrinsicsX86.td
+++ b/llvm/include/llvm/IR/IntrinsicsX86.td
@@ -235,6 +235,19 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.".
let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.".
def int_x86_sse_sfence : ClangBuiltin<"__builtin_ia32_sfence">,
Intrinsic<[], [], []>;
+
+ // Non-temporal store of the second operand to the pointer, as an opaque
+ // operation. It selects the same instructions (MOVNTI, MOVNTPS, MOVNTPD,
+ // MOVNTDQ) as a store with !nontemporal metadata, but like the MOVNTQ,
+ // MOVDIRI and MOVDIR64B intrinsics it has no memory attributes: the
+ // optimizer treats it as a call that may access any memory, not as an
+ // ordinary store. Non-temporal stores are weakly ordered, and how
+ // !nontemporal stores interact with LLVM's memory model and optimizations
+ // is not specified (see #64521), so this gives frontends a defined
+ // alternative to inline assembly. Ordering it with other threads still
+ // needs an SFENCE. A vector store requires the pointer to be aligned to the
+ // vector's size.
+ def int_x86_movnt : Intrinsic<[], [llvm_ptr_ty, llvm_any_ty], []>;
}
// Control register.
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index c29b09ae846bb..4196f92db224f 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -28426,6 +28426,25 @@ bool X86::isExtendedSwiftAsyncFrameSupported(const X86Subtarget &Subtarget,
return !MF.getTarget().getMCAsmInfo().usesWindowsCFI();
}
+// Lower llvm.x86.movnt to a non-temporal store, which selects the same MOVNT
+// instructions as a !nontemporal store. The store is also volatile, so the
+// backend doesn't merge, eliminate or reorder it with other memory accesses,
+// matching the intrinsic's opaque IR semantics.
+static SDValue lowerX86MOVNT(SDNode *N, SelectionDAG &DAG) {
+ SDLoc DL(N);
+ SDValue Chain = N->getOperand(0);
+ SDValue Ptr = N->getOperand(2);
+ SDValue Val = N->getOperand(3);
+ EVT VT = Val.getValueType();
+ // MOVNTI has no alignment requirement; the vector forms require the pointer
+ // to be aligned to the vector's size, as documented on the intrinsic.
+ Align Alignment =
+ VT.isVector() ? Align(VT.getStoreSize().getFixedValue()) : Align(1);
+ return DAG.getStore(Chain, DL, Val, Ptr, MachinePointerInfo(), Alignment,
+ MachineMemOperand::MOVolatile |
+ MachineMemOperand::MONonTemporal);
+}
+
static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
SelectionDAG &DAG) {
unsigned IntNo = Op.getConstantOperandVal(1);
@@ -28433,6 +28452,9 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
if (!IntrData) {
switch (IntNo) {
+ case Intrinsic::x86_movnt:
+ return lowerX86MOVNT(Op.getNode(), DAG);
+
case Intrinsic::swift_async_context_addr: {
SDLoc dl(Op);
auto &MF = DAG.getMachineFunction();
@@ -64005,6 +64027,11 @@ static SDValue combineINTRINSIC_VOID(SDNode *N, SelectionDAG &DAG,
if (IntrData && IntrData->Type == INTR_TYPE_CAST_MMX)
return FixupMMXIntrinsicTypes(N, DAG);
+ // Before type legalization, so that a vector wider than the target's
+ // registers is split like any other store.
+ if (IntNo == Intrinsic::x86_movnt)
+ return lowerX86MOVNT(N, DAG);
+
return SDValue();
}
diff --git a/llvm/test/CodeGen/X86/movnt-intrinsic.ll b/llvm/test/CodeGen/X86/movnt-intrinsic.ll
new file mode 100644
index 0000000000000..ee86a0f66a789
--- /dev/null
+++ b/llvm/test/CodeGen/X86/movnt-intrinsic.ll
@@ -0,0 +1,220 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX,AVX512
+
+; llvm.x86.movnt selects the same non-temporal stores as !nontemporal stores.
+
+define void @movnt_i32(ptr %p, i32 %v) {
+; SSE-LABEL: movnt_i32:
+; SSE: # %bb.0:
+; SSE-NEXT: movntil %esi, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX-LABEL: movnt_i32:
+; AVX: # %bb.0:
+; AVX-NEXT: movntil %esi, (%rdi)
+; AVX-NEXT: retq
+ call void @llvm.x86.movnt.i32(ptr %p, i32 %v)
+ ret void
+}
+
+define void @movnt_i64(ptr %p, i64 %v) {
+; SSE-LABEL: movnt_i64:
+; SSE: # %bb.0:
+; SSE-NEXT: movntiq %rsi, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX-LABEL: movnt_i64:
+; AVX: # %bb.0:
+; AVX-NEXT: movntiq %rsi, (%rdi)
+; AVX-NEXT: retq
+ call void @llvm.x86.movnt.i64(ptr %p, i64 %v)
+ ret void
+}
+
+define void @movnt_v4f32(ptr %p, <4 x float> %v) {
+; SSE-LABEL: movnt_v4f32:
+; SSE: # %bb.0:
+; SSE-NEXT: movntps %xmm0, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX-LABEL: movnt_v4f32:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovntps %xmm0, (%rdi)
+; AVX-NEXT: retq
+ call void @llvm.x86.movnt.v4f32(ptr %p, <4 x float> %v)
+ ret void
+}
+
+define void @movnt_v2f64(ptr %p, <2 x double> %v) {
+; SSE-LABEL: movnt_v2f64:
+; SSE: # %bb.0:
+; SSE-NEXT: movntps %xmm0, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX-LABEL: movnt_v2f64:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovntps %xmm0, (%rdi)
+; AVX-NEXT: retq
+ call void @llvm.x86.movnt.v2f64(ptr %p, <2 x double> %v)
+ ret void
+}
+
+define void @movnt_v2i64(ptr %p, <2 x i64> %v) {
+; SSE-LABEL: movnt_v2i64:
+; SSE: # %bb.0:
+; SSE-NEXT: movntps %xmm0, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX-LABEL: movnt_v2i64:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovntps %xmm0, (%rdi)
+; AVX-NEXT: retq
+ call void @llvm.x86.movnt.v2i64(ptr %p, <2 x i64> %v)
+ ret void
+}
+
+define void @movnt_v8f32(ptr %p, <8 x float> %v) {
+; SSE-LABEL: movnt_v8f32:
+; SSE: # %bb.0:
+; SSE-NEXT: movntps %xmm1, 16(%rdi)
+; SSE-NEXT: movntps %xmm0, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX-LABEL: movnt_v8f32:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovntps %ymm0, (%rdi)
+; AVX-NEXT: vzeroupper
+; AVX-NEXT: retq
+ call void @llvm.x86.movnt.v8f32(ptr %p, <8 x float> %v)
+ ret void
+}
+
+define void @movnt_v4f64(ptr %p, <4 x double> %v) {
+; SSE-LABEL: movnt_v4f64:
+; SSE: # %bb.0:
+; SSE-NEXT: movntps %xmm1, 16(%rdi)
+; SSE-NEXT: movntps %xmm0, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX-LABEL: movnt_v4f64:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovntps %ymm0, (%rdi)
+; AVX-NEXT: vzeroupper
+; AVX-NEXT: retq
+ call void @llvm.x86.movnt.v4f64(ptr %p, <4 x double> %v)
+ ret void
+}
+
+define void @movnt_v4i64(ptr %p, <4 x i64> %v) {
+; SSE-LABEL: movnt_v4i64:
+; SSE: # %bb.0:
+; SSE-NEXT: movntps %xmm1, 16(%rdi)
+; SSE-NEXT: movntps %xmm0, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX-LABEL: movnt_v4i64:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovntps %ymm0, (%rdi)
+; AVX-NEXT: vzeroupper
+; AVX-NEXT: retq
+ call void @llvm.x86.movnt.v4i64(ptr %p, <4 x i64> %v)
+ ret void
+}
+
+define void @movnt_v16f32(ptr %p, <16 x float> %v) {
+; SSE-LABEL: movnt_v16f32:
+; SSE: # %bb.0:
+; SSE-NEXT: movntps %xmm3, 48(%rdi)
+; SSE-NEXT: movntps %xmm2, 32(%rdi)
+; SSE-NEXT: movntps %xmm1, 16(%rdi)
+; SSE-NEXT: movntps %xmm0, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX1-LABEL: movnt_v16f32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vmovntps %ymm1, 32(%rdi)
+; AVX1-NEXT: vmovntps %ymm0, (%rdi)
+; AVX1-NEXT: vzeroupper
+; AVX1-NEXT: retq
+;
+; AVX512-LABEL: movnt_v16f32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovntps %zmm0, (%rdi)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ call void @llvm.x86.movnt.v16f32(ptr %p, <16 x float> %v)
+ ret void
+}
+
+define void @movnt_v8f64(ptr %p, <8 x double> %v) {
+; SSE-LABEL: movnt_v8f64:
+; SSE: # %bb.0:
+; SSE-NEXT: movntps %xmm3, 48(%rdi)
+; SSE-NEXT: movntps %xmm2, 32(%rdi)
+; SSE-NEXT: movntps %xmm1, 16(%rdi)
+; SSE-NEXT: movntps %xmm0, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX1-LABEL: movnt_v8f64:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vmovntps %ymm1, 32(%rdi)
+; AVX1-NEXT: vmovntps %ymm0, (%rdi)
+; AVX1-NEXT: vzeroupper
+; AVX1-NEXT: retq
+;
+; AVX512-LABEL: movnt_v8f64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovntps %zmm0, (%rdi)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ call void @llvm.x86.movnt.v8f64(ptr %p, <8 x double> %v)
+ ret void
+}
+
+define void @movnt_v8i64(ptr %p, <8 x i64> %v) {
+; SSE-LABEL: movnt_v8i64:
+; SSE: # %bb.0:
+; SSE-NEXT: movntps %xmm3, 48(%rdi)
+; SSE-NEXT: movntps %xmm2, 32(%rdi)
+; SSE-NEXT: movntps %xmm1, 16(%rdi)
+; SSE-NEXT: movntps %xmm0, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX1-LABEL: movnt_v8i64:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vmovntps %ymm1, 32(%rdi)
+; AVX1-NEXT: vmovntps %ymm0, (%rdi)
+; AVX1-NEXT: vzeroupper
+; AVX1-NEXT: retq
+;
+; AVX512-LABEL: movnt_v8i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovntps %zmm0, (%rdi)
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ call void @llvm.x86.movnt.v8i64(ptr %p, <8 x i64> %v)
+ ret void
+}
+
+; The address is folded into the store, as for the MOVDIRI/MOVDIR64B intrinsics.
+define void @movnt_v2i64_offsets(ptr %p, <2 x i64> %a, <2 x i64> %b) {
+; SSE-LABEL: movnt_v2i64_offsets:
+; SSE: # %bb.0:
+; SSE-NEXT: movntps %xmm0, 16(%rdi)
+; SSE-NEXT: movntps %xmm1, 48(%rdi)
+; SSE-NEXT: retq
+;
+; AVX-LABEL: movnt_v2i64_offsets:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovntps %xmm0, 16(%rdi)
+; AVX-NEXT: vmovntps %xmm1, 48(%rdi)
+; AVX-NEXT: retq
+ %p1 = getelementptr inbounds <2 x i64>, ptr %p, i64 1
+ %p3 = getelementptr inbounds <2 x i64>, ptr %p, i64 3
+ call void @llvm.x86.movnt.v2i64(ptr %p1, <2 x i64> %a)
+ call void @llvm.x86.movnt.v2i64(ptr %p3, <2 x i64> %b)
+ ret void
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/x86-movnt-intrinsic.ll b/llvm/test/Transforms/DeadStoreElimination/x86-movnt-intrinsic.ll
new file mode 100644
index 0000000000000..a036daf22ce5a
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/x86-movnt-intrinsic.ll
@@ -0,0 +1,27 @@
+; RUN: opt -S -passes=dse,gvn < %s | FileCheck %s
+
+; llvm.x86.movnt has no memory attributes, so it is not optimized as an
+; ordinary store: an overwritten call is not removed, and its value is not
+; forwarded to a later load.
+
+define void @overwritten(ptr %p, <2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: @overwritten(
+; CHECK-NEXT: call void @llvm.x86.movnt.v2i64(ptr [[P:%.*]], <2 x i64> [[A:%.*]])
+; CHECK-NEXT: store <2 x i64> [[B:%.*]], ptr [[P]], align 16
+; CHECK-NEXT: ret void
+;
+ call void @llvm.x86.movnt.v2i64(ptr %p, <2 x i64> %a)
+ store <2 x i64> %b, ptr %p, align 16
+ ret void
+}
+
+define i64 @not_forwarded(ptr %p) {
+; CHECK-LABEL: @not_forwarded(
+; CHECK-NEXT: call void @llvm.x86.movnt.i64(ptr [[P:%.*]], i64 5)
+; CHECK-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; CHECK-NEXT: ret i64 [[V]]
+;
+ call void @llvm.x86.movnt.i64(ptr %p, i64 5)
+ %v = load i64, ptr %p, align 8
+ ret i64 %v
+}
More information about the llvm-commits
mailing list