[llvm] [X86] Fold extensions into atomic loads (PR #217886)
Jan Ječmen via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 21 04:54:06 PDT 2026
https://github.com/JanJecmen created https://github.com/llvm/llvm-project/pull/217886
Mark extending atomic loads legal for 32 and 64 bit results and add the matching patterns, so a sign or zero extension of an atomic load becomes part of the load instead of a separate instruction. The memory access keeps its original width and ordering, so it stays atomic.
A 16 bit result is left out: MOVZX16rm8 writes a partial register.
Assisted-by: Claude Opus 5 <noreply at anthropic.com>
>From 70ec0b891d245f413eaa2deeae14a20e545bf824 Mon Sep 17 00:00:00 2001
From: Jan Jecmen <jjecmen at azul.com>
Date: Thu, 20 Aug 2026 13:38:22 +0000
Subject: [PATCH] [X86] Fold extensions into atomic loads
Mark extending atomic loads legal for 32 and 64 bit results and add the
matching patterns, so a sign or zero extension of an atomic load becomes
part of the load instead of a separate instruction. The memory access
keeps its original width and ordering, so it stays atomic.
A 16 bit result is left out: MOVZX16rm8 writes a partial register.
Assisted-by: Claude Opus 5 <noreply at anthropic.com>
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 9 +
llvm/lib/Target/X86/X86InstrCompiler.td | 22 ++
llvm/test/CodeGen/X86/atomic-load-extend.ll | 262 ++++++++++++++++++++
llvm/test/CodeGen/X86/atomic-load-store.ll | 30 +--
4 files changed, 301 insertions(+), 22 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/atomic-load-extend.ll
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index eed9416741c17..95ae686cbef21 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -186,6 +186,15 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
for (MVT VT : MVT::integer_valuetypes())
setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
+ // Allow folding extensions into atomic loads. This is legal as the memory
+ // access keeps the original width and ordering. Deliberately leave out
+ // the 16 bit result to avoid writing a partial register.
+ setAtomicLoadExtAction({ISD::SEXTLOAD, ISD::ZEXTLOAD}, MVT::i32,
+ {MVT::i8, MVT::i16}, Legal);
+ if (Subtarget.is64Bit())
+ setAtomicLoadExtAction({ISD::SEXTLOAD, ISD::ZEXTLOAD}, MVT::i64,
+ {MVT::i8, MVT::i16, MVT::i32}, Legal);
+
// We don't accept any truncstore of integer registers.
setTruncStoreAction(MVT::i64, MVT::i32, Expand);
setTruncStoreAction(MVT::i64, MVT::i16, Expand);
diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td
index d3e8fdbdcd53c..ed75f4f125031 100644
--- a/llvm/lib/Target/X86/X86InstrCompiler.td
+++ b/llvm/lib/Target/X86/X86InstrCompiler.td
@@ -1216,6 +1216,28 @@ def : Pat<(i16 (atomic_load_nonext_16 addr:$src)), (MOV16rm addr:$src)>;
def : Pat<(i32 (atomic_load_nonext_32 addr:$src)), (MOV32rm addr:$src)>;
def : Pat<(i64 (atomic_load_nonext_64 addr:$src)), (MOV64rm addr:$src)>;
+// The extending moves load the same number of bytes as the plain move, so
+// they can be used for an atomic load that zero or sign extends its result.
+def : Pat<(i32 (atomic_load_azext_8 addr:$src)), (MOVZX32rm8 addr:$src)>;
+def : Pat<(i32 (atomic_load_azext_16 addr:$src)), (MOVZX32rm16 addr:$src)>;
+def : Pat<(i32 (atomic_load_asext_8 addr:$src)), (MOVSX32rm8 addr:$src)>;
+def : Pat<(i32 (atomic_load_asext_16 addr:$src)), (MOVSX32rm16 addr:$src)>;
+
+let Predicates = [In64BitMode] in {
+ def : Pat<(i64 (atomic_load_azext_8 addr:$src)),
+ (SUBREG_TO_REG (MOVZX32rm8 addr:$src), sub_32bit)>;
+ def : Pat<(i64 (atomic_load_azext_16 addr:$src)),
+ (SUBREG_TO_REG (MOVZX32rm16 addr:$src), sub_32bit)>;
+ def : Pat<(i64 (atomic_load_azext_32 addr:$src)),
+ (SUBREG_TO_REG (MOV32rm addr:$src), sub_32bit)>;
+ def : Pat<(i64 (atomic_load_asext_8 addr:$src)),
+ (MOVSX64rm8 addr:$src)>;
+ def : Pat<(i64 (atomic_load_asext_16 addr:$src)),
+ (MOVSX64rm16 addr:$src)>;
+ def : Pat<(i64 (atomic_load_asext_32 addr:$src)),
+ (MOVSX64rm32 addr:$src)>;
+}
+
// load atomic <2 x i16>
def : Pat<(v4i32 (scalar_to_vector (i32 (atomic_load_32 addr:$src)))),
(MOVDI2PDIrm addr:$src)>, Requires<[UseSSE2]>;
diff --git a/llvm/test/CodeGen/X86/atomic-load-extend.ll b/llvm/test/CodeGen/X86/atomic-load-extend.ll
new file mode 100644
index 0000000000000..0ab9a33754dd9
--- /dev/null
+++ b/llvm/test/CodeGen/X86/atomic-load-extend.ll
@@ -0,0 +1,262 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs | FileCheck %s --check-prefixes=X64,X64-O3
+; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -O0 | FileCheck %s --check-prefixes=X64,X64-O0
+; RUN: llc < %s -mtriple=i686-- -verify-machineinstrs | FileCheck %s --check-prefix=X86
+
+; An extending load reads the same number of bytes as the plain load it
+; replaces, so the extension can be folded into an atomic load.
+
+define i32 @zext_i8_i32(ptr %p) {
+; X64-LABEL: zext_i8_i32:
+; X64: # %bb.0:
+; X64-NEXT: movzbl (%rdi), %eax
+; X64-NEXT: retq
+;
+; X86-LABEL: zext_i8_i32:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movzbl (%eax), %eax
+; X86-NEXT: retl
+ %v = load atomic i8, ptr %p unordered, align 1
+ %e = zext i8 %v to i32
+ ret i32 %e
+}
+
+define i32 @zext_i16_i32(ptr %p) {
+; X64-LABEL: zext_i16_i32:
+; X64: # %bb.0:
+; X64-NEXT: movzwl (%rdi), %eax
+; X64-NEXT: retq
+;
+; X86-LABEL: zext_i16_i32:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movzwl (%eax), %eax
+; X86-NEXT: retl
+ %v = load atomic i16, ptr %p unordered, align 2
+ %e = zext i16 %v to i32
+ ret i32 %e
+}
+
+define i32 @sext_i8_i32(ptr %p) {
+; X64-LABEL: sext_i8_i32:
+; X64: # %bb.0:
+; X64-NEXT: movsbl (%rdi), %eax
+; X64-NEXT: retq
+;
+; X86-LABEL: sext_i8_i32:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movsbl (%eax), %eax
+; X86-NEXT: retl
+ %v = load atomic i8, ptr %p unordered, align 1
+ %e = sext i8 %v to i32
+ ret i32 %e
+}
+
+define i32 @sext_i16_i32(ptr %p) {
+; X64-LABEL: sext_i16_i32:
+; X64: # %bb.0:
+; X64-NEXT: movswl (%rdi), %eax
+; X64-NEXT: retq
+;
+; X86-LABEL: sext_i16_i32:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movswl (%eax), %eax
+; X86-NEXT: retl
+ %v = load atomic i16, ptr %p unordered, align 2
+ %e = sext i16 %v to i32
+ ret i32 %e
+}
+
+define i64 @zext_i8_i64(ptr %p) {
+; X64-O3-LABEL: zext_i8_i64:
+; X64-O3: # %bb.0:
+; X64-O3-NEXT: movzbl (%rdi), %eax
+; X64-O3-NEXT: retq
+;
+; X64-O0-LABEL: zext_i8_i64:
+; X64-O0: # %bb.0:
+; X64-O0-NEXT: movzbl (%rdi), %eax
+; X64-O0-NEXT: # kill: def $rax killed $eax
+; X64-O0-NEXT: retq
+;
+; X86-LABEL: zext_i8_i64:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movzbl (%eax), %eax
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: retl
+ %v = load atomic i8, ptr %p unordered, align 1
+ %e = zext i8 %v to i64
+ ret i64 %e
+}
+
+define i64 @zext_i16_i64(ptr %p) {
+; X64-O3-LABEL: zext_i16_i64:
+; X64-O3: # %bb.0:
+; X64-O3-NEXT: movzwl (%rdi), %eax
+; X64-O3-NEXT: retq
+;
+; X64-O0-LABEL: zext_i16_i64:
+; X64-O0: # %bb.0:
+; X64-O0-NEXT: movzwl (%rdi), %eax
+; X64-O0-NEXT: # kill: def $rax killed $eax
+; X64-O0-NEXT: retq
+;
+; X86-LABEL: zext_i16_i64:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movzwl (%eax), %eax
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: retl
+ %v = load atomic i16, ptr %p unordered, align 2
+ %e = zext i16 %v to i64
+ ret i64 %e
+}
+
+define i64 @zext_i32_i64(ptr %p) {
+; X64-O3-LABEL: zext_i32_i64:
+; X64-O3: # %bb.0:
+; X64-O3-NEXT: movl (%rdi), %eax
+; X64-O3-NEXT: retq
+;
+; X64-O0-LABEL: zext_i32_i64:
+; X64-O0: # %bb.0:
+; X64-O0-NEXT: movl (%rdi), %eax
+; X64-O0-NEXT: # kill: def $rax killed $eax
+; X64-O0-NEXT: retq
+;
+; X86-LABEL: zext_i32_i64:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl (%eax), %eax
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: retl
+ %v = load atomic i32, ptr %p unordered, align 4
+ %e = zext i32 %v to i64
+ ret i64 %e
+}
+
+define i64 @sext_i8_i64(ptr %p) {
+; X64-LABEL: sext_i8_i64:
+; X64: # %bb.0:
+; X64-NEXT: movsbq (%rdi), %rax
+; X64-NEXT: retq
+;
+; X86-LABEL: sext_i8_i64:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movsbl (%eax), %eax
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: sarl $31, %edx
+; X86-NEXT: retl
+ %v = load atomic i8, ptr %p unordered, align 1
+ %e = sext i8 %v to i64
+ ret i64 %e
+}
+
+define i64 @sext_i16_i64(ptr %p) {
+; X64-LABEL: sext_i16_i64:
+; X64: # %bb.0:
+; X64-NEXT: movswq (%rdi), %rax
+; X64-NEXT: retq
+;
+; X86-LABEL: sext_i16_i64:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movswl (%eax), %eax
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: sarl $31, %edx
+; X86-NEXT: retl
+ %v = load atomic i16, ptr %p unordered, align 2
+ %e = sext i16 %v to i64
+ ret i64 %e
+}
+
+define i64 @sext_i32_i64(ptr %p) {
+; X64-LABEL: sext_i32_i64:
+; X64: # %bb.0:
+; X64-NEXT: movslq (%rdi), %rax
+; X64-NEXT: retq
+;
+; X86-LABEL: sext_i32_i64:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl (%eax), %eax
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: sarl $31, %edx
+; X86-NEXT: retl
+ %v = load atomic i32, ptr %p unordered, align 4
+ %e = sext i32 %v to i64
+ ret i64 %e
+}
+
+; A stronger ordering does not change how the load is encoded on x86, so the
+; extension can still be folded in.
+
+define i32 @zext_i8_i32_acquire(ptr %p) {
+; X64-O3-LABEL: zext_i8_i32_acquire:
+; X64-O3: # %bb.0:
+; X64-O3-NEXT: movzbl (%rdi), %eax
+; X64-O3-NEXT: retq
+;
+; X64-O0-LABEL: zext_i8_i32_acquire:
+; X64-O0: # %bb.0:
+; X64-O0-NEXT: movb (%rdi), %al
+; X64-O0-NEXT: movzbl %al, %eax
+; X64-O0-NEXT: retq
+;
+; X86-LABEL: zext_i8_i32_acquire:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movzbl (%eax), %eax
+; X86-NEXT: retl
+ %v = load atomic i8, ptr %p acquire, align 1
+ %e = zext i8 %v to i32
+ ret i32 %e
+}
+
+define i32 @zext_i8_i32_seq_cst(ptr %p) {
+; X64-O3-LABEL: zext_i8_i32_seq_cst:
+; X64-O3: # %bb.0:
+; X64-O3-NEXT: movzbl (%rdi), %eax
+; X64-O3-NEXT: retq
+;
+; X64-O0-LABEL: zext_i8_i32_seq_cst:
+; X64-O0: # %bb.0:
+; X64-O0-NEXT: movb (%rdi), %al
+; X64-O0-NEXT: movzbl %al, %eax
+; X64-O0-NEXT: retq
+;
+; X86-LABEL: zext_i8_i32_seq_cst:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movzbl (%eax), %eax
+; X86-NEXT: retl
+ %v = load atomic i8, ptr %p seq_cst, align 1
+ %e = zext i8 %v to i32
+ ret i32 %e
+}
+
+; A 16 bit destination is deliberately not folded into: movzbw writes a partial
+; register. The extension is widened to 32 bits instead.
+
+define i16 @zext_i8_i16(ptr %p) {
+; X64-LABEL: zext_i8_i16:
+; X64: # %bb.0:
+; X64-NEXT: movzbl (%rdi), %eax
+; X64-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-NEXT: retq
+;
+; X86-LABEL: zext_i8_i16:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movzbl (%eax), %eax
+; X86-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-NEXT: retl
+ %v = load atomic i8, ptr %p unordered, align 1
+ %e = zext i8 %v to i16
+ ret i16 %e
+}
diff --git a/llvm/test/CodeGen/X86/atomic-load-store.ll b/llvm/test/CodeGen/X86/atomic-load-store.ll
index 7cfe7af47748a..f8c9893e4f817 100644
--- a/llvm/test/CodeGen/X86/atomic-load-store.ll
+++ b/llvm/test/CodeGen/X86/atomic-load-store.ll
@@ -73,34 +73,20 @@ define <1 x i16> @atomic_vec1_i16(ptr %x) {
}
define <1 x i32> @atomic_vec1_i8_zext(ptr %x) {
-; CHECK-O3-LABEL: atomic_vec1_i8_zext:
-; CHECK-O3: # %bb.0:
-; CHECK-O3-NEXT: movzbl (%rdi), %eax
-; CHECK-O3-NEXT: movzbl %al, %eax
-; CHECK-O3-NEXT: retq
-;
-; CHECK-O0-LABEL: atomic_vec1_i8_zext:
-; CHECK-O0: # %bb.0:
-; CHECK-O0-NEXT: movb (%rdi), %al
-; CHECK-O0-NEXT: movzbl %al, %eax
-; CHECK-O0-NEXT: retq
+; CHECK-LABEL: atomic_vec1_i8_zext:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movzbl (%rdi), %eax
+; CHECK-NEXT: retq
%ret = load atomic <1 x i8>, ptr %x acquire, align 1
%zret = zext <1 x i8> %ret to <1 x i32>
ret <1 x i32> %zret
}
define <1 x i64> @atomic_vec1_i16_sext(ptr %x) {
-; CHECK-O3-LABEL: atomic_vec1_i16_sext:
-; CHECK-O3: # %bb.0:
-; CHECK-O3-NEXT: movzwl (%rdi), %eax
-; CHECK-O3-NEXT: movswq %ax, %rax
-; CHECK-O3-NEXT: retq
-;
-; CHECK-O0-LABEL: atomic_vec1_i16_sext:
-; CHECK-O0: # %bb.0:
-; CHECK-O0-NEXT: movw (%rdi), %ax
-; CHECK-O0-NEXT: movswq %ax, %rax
-; CHECK-O0-NEXT: retq
+; CHECK-LABEL: atomic_vec1_i16_sext:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movswq (%rdi), %rax
+; CHECK-NEXT: retq
%ret = load atomic <1 x i16>, ptr %x acquire, align 2
%sret = sext <1 x i16> %ret to <1 x i64>
ret <1 x i64> %sret
More information about the llvm-commits
mailing list