[llvm] [X86] Fold atomic loads into compares (PR #221290)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 7 22:55:52 PDT 2026
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/221290
>From f5d566c2f7e6d8fad78024c1cd1219a29fc83879 Mon Sep 17 00:00:00 2001
From: Andrew Gaul <andrew at gaul.org>
Date: Fri, 4 Sep 2026 09:24:13 -0700
Subject: [PATCH 1/2] [X86] Add test coverage for compares of atomic loads
Compares of atomic loads currently keep the load in a register and
compare there, e.g. mov + test, where compares of regular loads fold
the load into the compare's memory operand. Record the current
codegen; a follow-up will fold the load.
Co-Authored-By: Claude Fable 5 <noreply at anthropic.com>
---
llvm/test/CodeGen/X86/atomic-cmp-fold.ll | 389 +++++++++++++++++++++++
1 file changed, 389 insertions(+)
create mode 100644 llvm/test/CodeGen/X86/atomic-cmp-fold.ll
diff --git a/llvm/test/CodeGen/X86/atomic-cmp-fold.ll b/llvm/test/CodeGen/X86/atomic-cmp-fold.ll
new file mode 100644
index 0000000000000..54532b4d8ffda
--- /dev/null
+++ b/llvm/test/CodeGen/X86/atomic-cmp-fold.ll
@@ -0,0 +1,389 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu | FileCheck %s --check-prefix=X86
+
+; Compares of atomic loads should fold the load into the compare's memory
+; operand, like compares of regular loads do: the folded form still performs
+; a single full-width load, and on x86 an ordinary load already provides
+; every ordering up to seq_cst.
+
+define i1 @cmp8_eq0_monotonic(ptr %p) {
+; X64-LABEL: cmp8_eq0_monotonic:
+; X64: # %bb.0:
+; X64-NEXT: movzbl (%rdi), %eax
+; X64-NEXT: testb %al, %al
+; X64-NEXT: sete %al
+; X64-NEXT: retq
+;
+; X86-LABEL: cmp8_eq0_monotonic:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movzbl (%eax), %eax
+; X86-NEXT: testb %al, %al
+; X86-NEXT: sete %al
+; X86-NEXT: retl
+ %v = load atomic i8, ptr %p monotonic, align 1
+ %c = icmp eq i8 %v, 0
+ ret i1 %c
+}
+
+define i1 @cmp16_eq0_acquire(ptr %p) {
+; X64-LABEL: cmp16_eq0_acquire:
+; X64: # %bb.0:
+; X64-NEXT: movzwl (%rdi), %eax
+; X64-NEXT: testw %ax, %ax
+; X64-NEXT: sete %al
+; X64-NEXT: retq
+;
+; X86-LABEL: cmp16_eq0_acquire:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movzwl (%eax), %eax
+; X86-NEXT: testw %ax, %ax
+; X86-NEXT: sete %al
+; X86-NEXT: retl
+ %v = load atomic i16, ptr %p acquire, align 2
+ %c = icmp eq i16 %v, 0
+ ret i1 %c
+}
+
+define i1 @cmp32_eq0_seq_cst(ptr %p) {
+; X64-LABEL: cmp32_eq0_seq_cst:
+; X64: # %bb.0:
+; X64-NEXT: movl (%rdi), %eax
+; X64-NEXT: testl %eax, %eax
+; X64-NEXT: sete %al
+; X64-NEXT: retq
+;
+; X86-LABEL: cmp32_eq0_seq_cst:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl (%eax), %eax
+; X86-NEXT: testl %eax, %eax
+; X86-NEXT: sete %al
+; X86-NEXT: retl
+ %v = load atomic i32, ptr %p seq_cst, align 4
+ %c = icmp eq i32 %v, 0
+ ret i1 %c
+}
+
+define i1 @cmp64_eq0_unordered(ptr %p) {
+; X64-LABEL: cmp64_eq0_unordered:
+; X64: # %bb.0:
+; X64-NEXT: cmpq $0, (%rdi)
+; X64-NEXT: sete %al
+; X64-NEXT: retq
+;
+; X86-LABEL: cmp64_eq0_unordered:
+; X86: # %bb.0:
+; X86-NEXT: subl $12, %esp
+; X86-NEXT: .cfi_def_cfa_offset 16
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: fildll (%eax)
+; X86-NEXT: fistpll (%esp)
+; X86-NEXT: movl (%esp), %eax
+; X86-NEXT: orl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: sete %al
+; X86-NEXT: addl $12, %esp
+; X86-NEXT: .cfi_def_cfa_offset 4
+; X86-NEXT: retl
+ %v = load atomic i64, ptr %p unordered, align 8
+ %c = icmp eq i64 %v, 0
+ ret i1 %c
+}
+
+define i1 @cmp32_imm_acquire(ptr %p) {
+; X64-LABEL: cmp32_imm_acquire:
+; X64: # %bb.0:
+; X64-NEXT: movl (%rdi), %eax
+; X64-NEXT: cmpl $42, %eax
+; X64-NEXT: sete %al
+; X64-NEXT: retq
+;
+; X86-LABEL: cmp32_imm_acquire:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl (%eax), %eax
+; X86-NEXT: cmpl $42, %eax
+; X86-NEXT: sete %al
+; X86-NEXT: retl
+ %v = load atomic i32, ptr %p acquire, align 4
+ %c = icmp eq i32 %v, 42
+ ret i1 %c
+}
+
+define i1 @cmp32_imm_large(ptr %p) {
+; X64-LABEL: cmp32_imm_large:
+; X64: # %bb.0:
+; X64-NEXT: movl (%rdi), %eax
+; X64-NEXT: cmpl $305419896, %eax # imm = 0x12345678
+; X64-NEXT: sete %al
+; X64-NEXT: retq
+;
+; X86-LABEL: cmp32_imm_large:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl (%eax), %eax
+; X86-NEXT: cmpl $305419896, %eax # imm = 0x12345678
+; X86-NEXT: sete %al
+; X86-NEXT: retl
+ %v = load atomic i32, ptr %p monotonic, align 4
+ %c = icmp eq i32 %v, 305419896 ; 0x12345678
+ ret i1 %c
+}
+
+define i1 @cmp64_imm_sext32(ptr %p) {
+; X64-LABEL: cmp64_imm_sext32:
+; X64: # %bb.0:
+; X64-NEXT: movq (%rdi), %rax
+; X64-NEXT: cmpq $-100, %rax
+; X64-NEXT: sete %al
+; X64-NEXT: retq
+;
+; X86-LABEL: cmp64_imm_sext32:
+; X86: # %bb.0:
+; X86-NEXT: subl $12, %esp
+; X86-NEXT: .cfi_def_cfa_offset 16
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: fildll (%eax)
+; X86-NEXT: fistpll (%esp)
+; X86-NEXT: movl (%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: notl %ecx
+; X86-NEXT: xorl $-100, %eax
+; X86-NEXT: orl %ecx, %eax
+; X86-NEXT: sete %al
+; X86-NEXT: addl $12, %esp
+; X86-NEXT: .cfi_def_cfa_offset 4
+; X86-NEXT: retl
+ %v = load atomic i64, ptr %p monotonic, align 8
+ %c = icmp eq i64 %v, -100
+ ret i1 %c
+}
+
+; The immediate does not fit CMP64mi32, so the load stays in a register.
+define i1 @cmp64_imm_too_wide(ptr %p) {
+; X64-LABEL: cmp64_imm_too_wide:
+; X64: # %bb.0:
+; X64-NEXT: movq (%rdi), %rax
+; X64-NEXT: movabsq $4886718345, %rcx # imm = 0x123456789
+; X64-NEXT: cmpq %rcx, %rax
+; X64-NEXT: sete %al
+; X64-NEXT: retq
+;
+; X86-LABEL: cmp64_imm_too_wide:
+; X86: # %bb.0:
+; X86-NEXT: subl $12, %esp
+; X86-NEXT: .cfi_def_cfa_offset 16
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: fildll (%eax)
+; X86-NEXT: fistpll (%esp)
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl $591751049, %ecx # imm = 0x23456789
+; X86-NEXT: xorl (%esp), %ecx
+; X86-NEXT: xorl $1, %eax
+; X86-NEXT: orl %ecx, %eax
+; X86-NEXT: sete %al
+; X86-NEXT: addl $12, %esp
+; X86-NEXT: .cfi_def_cfa_offset 4
+; X86-NEXT: retl
+ %v = load atomic i64, ptr %p monotonic, align 8
+ %c = icmp eq i64 %v, 4886718345 ; 0x123456789
+ ret i1 %c
+}
+
+define i1 @cmp32_reg(ptr %p, i32 %x) {
+; X64-LABEL: cmp32_reg:
+; X64: # %bb.0:
+; X64-NEXT: movl (%rdi), %eax
+; X64-NEXT: cmpl %esi, %eax
+; X64-NEXT: sete %al
+; X64-NEXT: retq
+;
+; X86-LABEL: cmp32_reg:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl (%eax), %eax
+; X86-NEXT: cmpl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: sete %al
+; X86-NEXT: retl
+ %v = load atomic i32, ptr %p acquire, align 4
+ %c = icmp eq i32 %v, %x
+ ret i1 %c
+}
+
+; The loaded value sits on the compare's RHS.
+define i1 @cmp32_reg_rhs(ptr %p, i32 %x) {
+; X64-LABEL: cmp32_reg_rhs:
+; X64: # %bb.0:
+; X64-NEXT: movl (%rdi), %eax
+; X64-NEXT: cmpl %eax, %esi
+; X64-NEXT: setl %al
+; X64-NEXT: retq
+;
+; X86-LABEL: cmp32_reg_rhs:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl (%eax), %eax
+; X86-NEXT: cmpl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: setl %al
+; X86-NEXT: retl
+ %v = load atomic i32, ptr %p acquire, align 4
+ %c = icmp slt i32 %x, %v
+ ret i1 %c
+}
+
+; Relaxed load of a level, compare, branch (the log-level-check shape).
+define i32 @cmp32_sge_branch(ptr %p, i32 %lvl) {
+; X64-LABEL: cmp32_sge_branch:
+; X64: # %bb.0:
+; X64-NEXT: movl (%rdi), %eax
+; X64-NEXT: cmpl %esi, %eax
+; X64-NEXT: jl .LBB10_2
+; X64-NEXT: # %bb.1: # %log
+; X64-NEXT: movl $1, %eax
+; X64-NEXT: retq
+; X64-NEXT: .LBB10_2: # %skip
+; X64-NEXT: xorl %eax, %eax
+; X64-NEXT: retq
+;
+; X86-LABEL: cmp32_sge_branch:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl (%eax), %eax
+; X86-NEXT: cmpl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: jl .LBB10_2
+; X86-NEXT: # %bb.1: # %log
+; X86-NEXT: movl $1, %eax
+; X86-NEXT: retl
+; X86-NEXT: .LBB10_2: # %skip
+; X86-NEXT: xorl %eax, %eax
+; X86-NEXT: retl
+ %v = load atomic i32, ptr %p monotonic, align 4
+ %c = icmp sge i32 %v, %lvl
+ br i1 %c, label %log, label %skip
+log:
+ ret i32 1
+skip:
+ ret i32 0
+}
+
+define i1 @test32_mask_bit(ptr %p) {
+; X64-LABEL: test32_mask_bit:
+; X64: # %bb.0:
+; X64-NEXT: movl (%rdi), %eax
+; X64-NEXT: andl $8, %eax
+; X64-NEXT: shrl $3, %eax
+; X64-NEXT: # kill: def $al killed $al killed $eax
+; X64-NEXT: retq
+;
+; X86-LABEL: test32_mask_bit:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl (%eax), %eax
+; X86-NEXT: andl $8, %eax
+; X86-NEXT: shrl $3, %eax
+; X86-NEXT: # kill: def $al killed $al killed $eax
+; X86-NEXT: retl
+ %v = load atomic i32, ptr %p monotonic, align 4
+ %a = and i32 %v, 8
+ %c = icmp ne i32 %a, 0
+ ret i1 %c
+}
+
+; The mask fits neither i8 nor i16 and is not a shifted mask, so the
+; masked compare cannot be narrowed and keeps its full width.
+define i1 @test32_mask_wide(ptr %p) {
+; X64-LABEL: test32_mask_wide:
+; X64: # %bb.0:
+; X64-NEXT: movl (%rdi), %eax
+; X64-NEXT: testl $65792, %eax # imm = 0x10100
+; X64-NEXT: sete %al
+; X64-NEXT: retq
+;
+; X86-LABEL: test32_mask_wide:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl (%eax), %eax
+; X86-NEXT: testl $65792, %eax # imm = 0x10100
+; X86-NEXT: sete %al
+; X86-NEXT: retl
+ %v = load atomic i32, ptr %p monotonic, align 4
+ %a = and i32 %v, 65792 ; 0x10100
+ %c = icmp eq i32 %a, 0
+ ret i1 %c
+}
+
+; The loaded value has a second use: the load must stay.
+define i32 @no_fold_multiuse(ptr %p) {
+; X64-LABEL: no_fold_multiuse:
+; X64: # %bb.0:
+; X64-NEXT: movl (%rdi), %ecx
+; X64-NEXT: cmpl $7, %ecx
+; X64-NEXT: movl $100, %eax
+; X64-NEXT: cmovnel %ecx, %eax
+; X64-NEXT: retq
+;
+; X86-LABEL: no_fold_multiuse:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl (%eax), %ecx
+; X86-NEXT: cmpl $7, %ecx
+; X86-NEXT: movl $100, %eax
+; X86-NEXT: je .LBB13_2
+; X86-NEXT: # %bb.1:
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: .LBB13_2:
+; X86-NEXT: retl
+ %v = load atomic i32, ptr %p acquire, align 4
+ %c = icmp eq i32 %v, 7
+ %r = select i1 %c, i32 100, i32 %v
+ ret i32 %r
+}
+
+; Two atomic loads: at most one side can fold.
+define i1 @cmp_two_loads(ptr %p, ptr %q) {
+; X64-LABEL: cmp_two_loads:
+; X64: # %bb.0:
+; X64-NEXT: movl (%rdi), %eax
+; X64-NEXT: movl (%rsi), %ecx
+; X64-NEXT: cmpl %ecx, %eax
+; X64-NEXT: sete %al
+; X64-NEXT: retq
+;
+; X86-LABEL: cmp_two_loads:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl (%ecx), %ecx
+; X86-NEXT: movl (%eax), %eax
+; X86-NEXT: cmpl %eax, %ecx
+; X86-NEXT: sete %al
+; X86-NEXT: retl
+ %a = load atomic i32, ptr %p acquire, align 4
+ %b = load atomic i32, ptr %q acquire, align 4
+ %c = icmp eq i32 %a, %b
+ ret i1 %c
+}
+
+; Volatile atomic load: folding keeps the single access, matching how
+; compares of volatile non-atomic loads are handled.
+define i1 @cmp32_volatile(ptr %p) {
+; X64-LABEL: cmp32_volatile:
+; X64: # %bb.0:
+; X64-NEXT: movl (%rdi), %eax
+; X64-NEXT: testl %eax, %eax
+; X64-NEXT: sete %al
+; X64-NEXT: retq
+;
+; X86-LABEL: cmp32_volatile:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl (%eax), %eax
+; X86-NEXT: testl %eax, %eax
+; X86-NEXT: sete %al
+; X86-NEXT: retl
+ %v = load atomic volatile i32, ptr %p acquire, align 4
+ %c = icmp eq i32 %v, 0
+ ret i1 %c
+}
>From b162dfdca99931cd2c956061452640ca37789214 Mon Sep 17 00:00:00 2001
From: Andrew Gaul <andrew at gaul.org>
Date: Fri, 4 Sep 2026 09:56:06 -0700
Subject: [PATCH 2/2] [X86] Fold atomic loads into compares
A compare of an atomic load kept the load in a register (mov + cmp/test)
where a compare of a regular load folds the load into the compare's
memory operand. The folded form performs a single full-width load, and
on x86 an ordinary load already provides every ordering up to seq_cst,
so folding is legal for any non-extending atomic load.
Add isel patterns folding an atomic load into CMP (mi/mr/rm forms across
i8/i16/i32/i64). Compare-to-zero already reaches these via EmitTest;
nonzero and register compares are emitted as a flag-only X86ISD::SUB (to
enable CSE with a real subtract) whose load the peephole then folds. The
peephole cannot move an ordered access, so those never folded for atomic
loads. Emit X86ISD::CMP instead when an operand is a foldable atomic
load: it has no non-atomic sibling to CSE with anyway.
x86lint reports 5854 "load foldable into compare" sites in a release
build of Firefox's libxul.so; sampling shows the majority are atomic-load
compares of the kind this folds -- mozilla::Atomic and std::atomic loads,
MOZ_LOG level checks (an Atomic<LogLevel>), and Rust core::sync::atomic
and log-crate level checks.
Co-Authored-By: Claude Fable 5 <noreply at anthropic.com>
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 12 +++++
llvm/lib/Target/X86/X86InstrCompiler.td | 31 ++++++++++++
llvm/test/CodeGen/X86/PR40322.ll | 3 +-
llvm/test/CodeGen/X86/atomic-cmp-fold.ll | 59 ++++++++---------------
llvm/test/CodeGen/X86/atomic-unordered.ll | 10 ++--
5 files changed, 69 insertions(+), 46 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index cf5ea221be0e2..09575a0423dec 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -24043,6 +24043,18 @@ static SDValue EmitCmp(SDValue Op0, SDValue Op1, X86::CondCode X86CC,
assert((CmpVT == MVT::i8 || CmpVT == MVT::i16 ||
CmpVT == MVT::i32 || CmpVT == MVT::i64) && "Unexpected VT!");
+ // If one operand is a non-extending atomic load, compare with CMP so the
+ // load folds into the compare's memory operand during isel. The SUB form
+ // chosen below for CSE would leave the load in a register: the peephole that
+ // folds a load into a following compare cannot move an ordered access, and
+ // an atomic load has no non-atomic sibling to be CSE'd with anyway.
+ auto IsFoldableAtomicLoad = [](SDValue Op) {
+ return Op.getOpcode() == ISD::ATOMIC_LOAD && Op.hasOneUse() &&
+ cast<AtomicSDNode>(Op)->getExtensionType() == ISD::NON_EXTLOAD;
+ };
+ if (IsFoldableAtomicLoad(Op0) || IsFoldableAtomicLoad(Op1))
+ return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op0, Op1);
+
// Only promote the compare up to I32 if it is a 16 bit operation
// with an immediate. 16 bit immediates are to be avoided unless the target
// isn't slowed down by length changing prefixes, we're optimizing for
diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td
index 8470eec896f13..45619663c45cb 100644
--- a/llvm/lib/Target/X86/X86InstrCompiler.td
+++ b/llvm/lib/Target/X86/X86InstrCompiler.td
@@ -1207,6 +1207,37 @@ def : Pat<(i16 (atomic_load_nonext_16 addr:$src)), (MOV16rm addr:$src)>;
def : Pat<(i32 (atomic_load_nonext_32 addr:$src)), (MOV32rm addr:$src)>;
def : Pat<(i64 (atomic_load_nonext_64 addr:$src)), (MOV64rm addr:$src)>;
+// Flag-setting compares of an atomic load can fold the load into the
+// compare's memory operand like compares of a regular load: the folded form
+// still performs a single full-width load, which on x86 already provides
+// every ordering up to seq_cst.
+def : Pat<(X86cmp (i8 (atomic_load_nonext_8 addr:$src1)), imm:$src2),
+ (CMP8mi addr:$src1, imm:$src2)>;
+def : Pat<(X86cmp (i16 (atomic_load_nonext_16 addr:$src1)), imm:$src2),
+ (CMP16mi addr:$src1, imm:$src2)>;
+def : Pat<(X86cmp (i32 (atomic_load_nonext_32 addr:$src1)), imm:$src2),
+ (CMP32mi addr:$src1, imm:$src2)>;
+def : Pat<(X86cmp (i64 (atomic_load_nonext_64 addr:$src1)), i64immSExt32:$src2),
+ (CMP64mi32 addr:$src1, i64immSExt32:$src2)>;
+
+def : Pat<(X86cmp (i8 (atomic_load_nonext_8 addr:$src1)), GR8:$src2),
+ (CMP8mr addr:$src1, GR8:$src2)>;
+def : Pat<(X86cmp (i16 (atomic_load_nonext_16 addr:$src1)), GR16:$src2),
+ (CMP16mr addr:$src1, GR16:$src2)>;
+def : Pat<(X86cmp (i32 (atomic_load_nonext_32 addr:$src1)), GR32:$src2),
+ (CMP32mr addr:$src1, GR32:$src2)>;
+def : Pat<(X86cmp (i64 (atomic_load_nonext_64 addr:$src1)), GR64:$src2),
+ (CMP64mr addr:$src1, GR64:$src2)>;
+
+def : Pat<(X86cmp GR8:$src1, (i8 (atomic_load_nonext_8 addr:$src2))),
+ (CMP8rm GR8:$src1, addr:$src2)>;
+def : Pat<(X86cmp GR16:$src1, (i16 (atomic_load_nonext_16 addr:$src2))),
+ (CMP16rm GR16:$src1, addr:$src2)>;
+def : Pat<(X86cmp GR32:$src1, (i32 (atomic_load_nonext_32 addr:$src2))),
+ (CMP32rm GR32:$src1, addr:$src2)>;
+def : Pat<(X86cmp GR64:$src1, (i64 (atomic_load_nonext_64 addr:$src2))),
+ (CMP64rm GR64:$src1, addr:$src2)>;
+
// load atomic <2 x i16>
def : Pat<(v4i32 (scalar_to_vector (i32 (atomic_load_32 addr:$src)))),
(MOVDI2PDIrm addr:$src)>, Requires<[UseSSE2]>;
diff --git a/llvm/test/CodeGen/X86/PR40322.ll b/llvm/test/CodeGen/X86/PR40322.ll
index 49709cb9b88f8..5f02b4f572526 100644
--- a/llvm/test/CodeGen/X86/PR40322.ll
+++ b/llvm/test/CodeGen/X86/PR40322.ll
@@ -15,8 +15,7 @@ define void @_Z2ami(i32) #0 personality ptr @__gxx_personality_v0 {
; CHECK-MINGW-X86-NEXT: .cfi_def_cfa_offset 12
; CHECK-MINGW-X86-NEXT: .cfi_offset %esi, -12
; CHECK-MINGW-X86-NEXT: .cfi_offset %edi, -8
-; CHECK-MINGW-X86-NEXT: movzbl __ZGVZ2amiE2au, %eax
-; CHECK-MINGW-X86-NEXT: testb %al, %al
+; CHECK-MINGW-X86-NEXT: cmpb $0, __ZGVZ2amiE2au
; CHECK-MINGW-X86-NEXT: jne LBB0_4
; CHECK-MINGW-X86-NEXT: # %bb.1: # %init.check
; CHECK-MINGW-X86-NEXT: .cfi_escape 0x2e, 0x04
diff --git a/llvm/test/CodeGen/X86/atomic-cmp-fold.ll b/llvm/test/CodeGen/X86/atomic-cmp-fold.ll
index 54532b4d8ffda..c3c1af48422b7 100644
--- a/llvm/test/CodeGen/X86/atomic-cmp-fold.ll
+++ b/llvm/test/CodeGen/X86/atomic-cmp-fold.ll
@@ -10,16 +10,14 @@
define i1 @cmp8_eq0_monotonic(ptr %p) {
; X64-LABEL: cmp8_eq0_monotonic:
; X64: # %bb.0:
-; X64-NEXT: movzbl (%rdi), %eax
-; X64-NEXT: testb %al, %al
+; X64-NEXT: cmpb $0, (%rdi)
; X64-NEXT: sete %al
; X64-NEXT: retq
;
; X86-LABEL: cmp8_eq0_monotonic:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movzbl (%eax), %eax
-; X86-NEXT: testb %al, %al
+; X86-NEXT: cmpb $0, (%eax)
; X86-NEXT: sete %al
; X86-NEXT: retl
%v = load atomic i8, ptr %p monotonic, align 1
@@ -30,16 +28,14 @@ define i1 @cmp8_eq0_monotonic(ptr %p) {
define i1 @cmp16_eq0_acquire(ptr %p) {
; X64-LABEL: cmp16_eq0_acquire:
; X64: # %bb.0:
-; X64-NEXT: movzwl (%rdi), %eax
-; X64-NEXT: testw %ax, %ax
+; X64-NEXT: cmpw $0, (%rdi)
; X64-NEXT: sete %al
; X64-NEXT: retq
;
; X86-LABEL: cmp16_eq0_acquire:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movzwl (%eax), %eax
-; X86-NEXT: testw %ax, %ax
+; X86-NEXT: cmpw $0, (%eax)
; X86-NEXT: sete %al
; X86-NEXT: retl
%v = load atomic i16, ptr %p acquire, align 2
@@ -50,16 +46,14 @@ define i1 @cmp16_eq0_acquire(ptr %p) {
define i1 @cmp32_eq0_seq_cst(ptr %p) {
; X64-LABEL: cmp32_eq0_seq_cst:
; X64: # %bb.0:
-; X64-NEXT: movl (%rdi), %eax
-; X64-NEXT: testl %eax, %eax
+; X64-NEXT: cmpl $0, (%rdi)
; X64-NEXT: sete %al
; X64-NEXT: retq
;
; X86-LABEL: cmp32_eq0_seq_cst:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl (%eax), %eax
-; X86-NEXT: testl %eax, %eax
+; X86-NEXT: cmpl $0, (%eax)
; X86-NEXT: sete %al
; X86-NEXT: retl
%v = load atomic i32, ptr %p seq_cst, align 4
@@ -95,16 +89,14 @@ define i1 @cmp64_eq0_unordered(ptr %p) {
define i1 @cmp32_imm_acquire(ptr %p) {
; X64-LABEL: cmp32_imm_acquire:
; X64: # %bb.0:
-; X64-NEXT: movl (%rdi), %eax
-; X64-NEXT: cmpl $42, %eax
+; X64-NEXT: cmpl $42, (%rdi)
; X64-NEXT: sete %al
; X64-NEXT: retq
;
; X86-LABEL: cmp32_imm_acquire:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl (%eax), %eax
-; X86-NEXT: cmpl $42, %eax
+; X86-NEXT: cmpl $42, (%eax)
; X86-NEXT: sete %al
; X86-NEXT: retl
%v = load atomic i32, ptr %p acquire, align 4
@@ -115,16 +107,14 @@ define i1 @cmp32_imm_acquire(ptr %p) {
define i1 @cmp32_imm_large(ptr %p) {
; X64-LABEL: cmp32_imm_large:
; X64: # %bb.0:
-; X64-NEXT: movl (%rdi), %eax
-; X64-NEXT: cmpl $305419896, %eax # imm = 0x12345678
+; X64-NEXT: cmpl $305419896, (%rdi) # imm = 0x12345678
; X64-NEXT: sete %al
; X64-NEXT: retq
;
; X86-LABEL: cmp32_imm_large:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl (%eax), %eax
-; X86-NEXT: cmpl $305419896, %eax # imm = 0x12345678
+; X86-NEXT: cmpl $305419896, (%eax) # imm = 0x12345678
; X86-NEXT: sete %al
; X86-NEXT: retl
%v = load atomic i32, ptr %p monotonic, align 4
@@ -135,8 +125,7 @@ define i1 @cmp32_imm_large(ptr %p) {
define i1 @cmp64_imm_sext32(ptr %p) {
; X64-LABEL: cmp64_imm_sext32:
; X64: # %bb.0:
-; X64-NEXT: movq (%rdi), %rax
-; X64-NEXT: cmpq $-100, %rax
+; X64-NEXT: cmpq $-100, (%rdi)
; X64-NEXT: sete %al
; X64-NEXT: retq
;
@@ -165,9 +154,8 @@ define i1 @cmp64_imm_sext32(ptr %p) {
define i1 @cmp64_imm_too_wide(ptr %p) {
; X64-LABEL: cmp64_imm_too_wide:
; X64: # %bb.0:
-; X64-NEXT: movq (%rdi), %rax
-; X64-NEXT: movabsq $4886718345, %rcx # imm = 0x123456789
-; X64-NEXT: cmpq %rcx, %rax
+; X64-NEXT: movabsq $4886718345, %rax # imm = 0x123456789
+; X64-NEXT: cmpq %rax, (%rdi)
; X64-NEXT: sete %al
; X64-NEXT: retq
;
@@ -195,8 +183,7 @@ define i1 @cmp64_imm_too_wide(ptr %p) {
define i1 @cmp32_reg(ptr %p, i32 %x) {
; X64-LABEL: cmp32_reg:
; X64: # %bb.0:
-; X64-NEXT: movl (%rdi), %eax
-; X64-NEXT: cmpl %esi, %eax
+; X64-NEXT: cmpl %esi, (%rdi)
; X64-NEXT: sete %al
; X64-NEXT: retq
;
@@ -216,8 +203,7 @@ define i1 @cmp32_reg(ptr %p, i32 %x) {
define i1 @cmp32_reg_rhs(ptr %p, i32 %x) {
; X64-LABEL: cmp32_reg_rhs:
; X64: # %bb.0:
-; X64-NEXT: movl (%rdi), %eax
-; X64-NEXT: cmpl %eax, %esi
+; X64-NEXT: cmpl (%rdi), %esi
; X64-NEXT: setl %al
; X64-NEXT: retq
;
@@ -237,8 +223,7 @@ define i1 @cmp32_reg_rhs(ptr %p, i32 %x) {
define i32 @cmp32_sge_branch(ptr %p, i32 %lvl) {
; X64-LABEL: cmp32_sge_branch:
; X64: # %bb.0:
-; X64-NEXT: movl (%rdi), %eax
-; X64-NEXT: cmpl %esi, %eax
+; X64-NEXT: cmpl %esi, (%rdi)
; X64-NEXT: jl .LBB10_2
; X64-NEXT: # %bb.1: # %log
; X64-NEXT: movl $1, %eax
@@ -346,8 +331,7 @@ define i1 @cmp_two_loads(ptr %p, ptr %q) {
; X64-LABEL: cmp_two_loads:
; X64: # %bb.0:
; X64-NEXT: movl (%rdi), %eax
-; X64-NEXT: movl (%rsi), %ecx
-; X64-NEXT: cmpl %ecx, %eax
+; X64-NEXT: cmpl (%rsi), %eax
; X64-NEXT: sete %al
; X64-NEXT: retq
;
@@ -356,8 +340,7 @@ define i1 @cmp_two_loads(ptr %p, ptr %q) {
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl (%ecx), %ecx
-; X86-NEXT: movl (%eax), %eax
-; X86-NEXT: cmpl %eax, %ecx
+; X86-NEXT: cmpl (%eax), %ecx
; X86-NEXT: sete %al
; X86-NEXT: retl
%a = load atomic i32, ptr %p acquire, align 4
@@ -371,16 +354,14 @@ define i1 @cmp_two_loads(ptr %p, ptr %q) {
define i1 @cmp32_volatile(ptr %p) {
; X64-LABEL: cmp32_volatile:
; X64: # %bb.0:
-; X64-NEXT: movl (%rdi), %eax
-; X64-NEXT: testl %eax, %eax
+; X64-NEXT: cmpl $0, (%rdi)
; X64-NEXT: sete %al
; X64-NEXT: retq
;
; X86-LABEL: cmp32_volatile:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl (%eax), %eax
-; X86-NEXT: testl %eax, %eax
+; X86-NEXT: cmpl $0, (%eax)
; X86-NEXT: sete %al
; X86-NEXT: retl
%v = load atomic volatile i32, ptr %p acquire, align 4
diff --git a/llvm/test/CodeGen/X86/atomic-unordered.ll b/llvm/test/CodeGen/X86/atomic-unordered.ll
index edfdec37150c2..d309cdf149bc9 100644
--- a/llvm/test/CodeGen/X86/atomic-unordered.ll
+++ b/llvm/test/CodeGen/X86/atomic-unordered.ll
@@ -1248,7 +1248,7 @@ define i1 @load_fold_icmp1(ptr %p) {
; CHECK-O0-LABEL: load_fold_icmp1:
; CHECK-O0: # %bb.0:
; CHECK-O0-NEXT: movq (%rdi), %rax
-; CHECK-O0-NEXT: subq $15, %rax
+; CHECK-O0-NEXT: cmpq $15, %rax
; CHECK-O0-NEXT: sete %al
; CHECK-O0-NEXT: retq
;
@@ -1266,7 +1266,7 @@ define i1 @load_fold_icmp2(ptr %p, i64 %v2) {
; CHECK-O0-LABEL: load_fold_icmp2:
; CHECK-O0: # %bb.0:
; CHECK-O0-NEXT: movq (%rdi), %rax
-; CHECK-O0-NEXT: subq %rsi, %rax
+; CHECK-O0-NEXT: cmpq %rsi, %rax
; CHECK-O0-NEXT: sete %al
; CHECK-O0-NEXT: retq
;
@@ -1285,14 +1285,14 @@ define i1 @load_fold_icmp3(ptr %p1, ptr %p2) {
; CHECK-O0: # %bb.0:
; CHECK-O0-NEXT: movq (%rdi), %rax
; CHECK-O0-NEXT: movq (%rsi), %rcx
-; CHECK-O0-NEXT: subq %rcx, %rax
+; CHECK-O0-NEXT: cmpq %rcx, %rax
; CHECK-O0-NEXT: sete %al
; CHECK-O0-NEXT: retq
;
; CHECK-O3-LABEL: load_fold_icmp3:
; CHECK-O3: # %bb.0:
-; CHECK-O3-NEXT: movq (%rsi), %rax
-; CHECK-O3-NEXT: cmpq %rax, (%rdi)
+; CHECK-O3-NEXT: movq (%rdi), %rax
+; CHECK-O3-NEXT: cmpq (%rsi), %rax
; CHECK-O3-NEXT: sete %al
; CHECK-O3-NEXT: retq
%v = load atomic i64, ptr %p1 unordered, align 8
More information about the llvm-commits
mailing list