[llvm] [X86] Prefer SIMD min/max/abs for scalars when staying in XMM domain (PR #210654)
Shaurya Srivastava via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 20 00:40:47 PDT 2026
https://github.com/Shaurya2k06 updated https://github.com/llvm/llvm-project/pull/210654
>From 84997fba4c6d02d1d306c4760caed2bf77825484 Mon Sep 17 00:00:00 2001
From: shaurya2k06 <shaurya2k06 at gmail.com>
Date: Mon, 20 Jul 2026 12:06:36 +0530
Subject: [PATCH] [X86] Prefer SIMD min/max/abs for scalars when staying in XMM
domain
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Fold scalar i32 smin/smax/umin/umax/abs whose operands are loads and
whose result is only stored (or insert_vector_elt) into a vector op on
v4i32 via SCALAR_TO_VECTOR, which ISel folds to movd + PMIN/PMAX/PABS.
This matches GCC for memory-bound sort2/abs and avoids introducing
GPR↔XMM domain crosses for values that already live in GPRs.
Implemented as DAGCombine (PerformDAGCombine) rather than custom
LowerOperation so paired smin+smax of the same loads are visible before
legalization, letting CSE share the SCALAR_TO_VECTOR nodes. Only plain
loads are accepted as sources: EXTRACT_VECTOR_ELT and constants
typically require a GPR path, which would reintroduce a domain cross.
Only i32 is handled: narrower integers typically load via GPR zero-
extends. Min/max requires SSE4.1; abs requires SSSE3. Bails under
OptForSize, NoImplicitFloat, and useSoftFloat. Combines are gated at
the call site for scalar nodes only.
Fixes #210569
Assisted-by: Cursor
Signed-off-by: shaurya2k06 <shaurya2k06 at gmail.com>
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 156 +++++
llvm/test/CodeGen/X86/scalar-minmax-simd.ll | 686 ++++++++++++++++++++
2 files changed, 842 insertions(+)
create mode 100644 llvm/test/CodeGen/X86/scalar-minmax-simd.ll
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 791e04deb9583..696b9dc47c728 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2802,6 +2802,11 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
ISD::ADD,
ISD::SADDSAT,
ISD::SSUBSAT,
+ ISD::ABS,
+ ISD::SMIN,
+ ISD::SMAX,
+ ISD::UMIN,
+ ISD::UMAX,
ISD::FADD,
ISD::FSUB,
ISD::FNEG,
@@ -62973,6 +62978,144 @@ static SDValue combineINTRINSIC_VOID(SDNode *N, SelectionDAG &DAG,
return SDValue();
}
+/// True if V is a plain (non-extending) load of VT. Don't peek through
+/// bitcasts: that could pull a float-domain value into an integer vector.
+static bool isPlainScalarLoad(SDValue V, EVT VT) {
+ auto *Ld = dyn_cast<LoadSDNode>(V);
+ if (!Ld || Ld->getExtensionType() != ISD::NON_EXTLOAD)
+ return false;
+ return Ld->getMemoryVT() == VT;
+}
+
+/// True if V has a single use that is an XMM-domain sink (store or
+/// insert_vector_elt). Otherwise widening would force a GPR↔XMM cross.
+static bool isOnlyUsedByXMMDomainSink(SDValue V) {
+ if (!V->hasNUsesOfValue(1, V.getResNo()))
+ return false;
+ for (SDUse &Use : V->uses()) {
+ if (Use.getResNo() != V.getResNo())
+ continue;
+ unsigned O = Use.getUser()->getOpcode();
+ if (O != ISD::STORE && O != ISD::INSERT_VECTOR_ELT)
+ return false;
+ }
+ return true;
+}
+
+/// Combine scalar i32 min/max whose operands are loads and whose result is only
+/// stored (or insert_vector_elt) into a vector min/max on v4i32, which ISel
+/// folds to movd + PMIN/PMAX. This keeps memory-bound sort2 in the XMM domain,
+/// matching GCC, without introducing GPR↔XMM domain crosses for values that
+/// live in GPRs.
+///
+/// Only plain i32 loads are accepted as sources: EXTRACT_VECTOR_ELT typically
+/// materializes in a GPR (movd/pextr), so re-widening would create a domain
+/// cross. Constants and GPR operands are likewise rejected. Relies on scalar
+/// i32 min/max remaining Expand so shouldScalarizeBinop does not undo the
+/// extract(vector_binop) form we create.
+static SDValue combineScalarMinMaxToSIMD(SDNode *N, SelectionDAG &DAG,
+ const X86Subtarget &Subtarget) {
+ EVT VT = N->getValueType(0);
+ // Caller must only pass scalar nodes; still guard for safety.
+ if (!VT.isSimple() || VT.getSimpleVT() != MVT::i32 || !Subtarget.hasSSE41())
+ return SDValue();
+
+ const Function &F = DAG.getMachineFunction().getFunction();
+ if (F.hasFnAttribute(Attribute::NoImplicitFloat) || Subtarget.useSoftFloat())
+ return SDValue();
+ if (F.hasOptSize())
+ return SDValue();
+
+ SDValue Op0 = N->getOperand(0);
+ SDValue Op1 = N->getOperand(1);
+
+ // Both operands must be plain i32 loads — the only XMM-native source that
+ // doesn't already require a GPR↔XMM move.
+ if (!isPlainScalarLoad(Op0, VT) || !isPlainScalarLoad(Op1, VT))
+ return SDValue();
+
+ // Each load must only feed scalar min/max nodes, or a SCALAR_TO_VECTOR from
+ // a sibling min/max already combined this turn (CSE shares the S2V). This
+ // lets paired smin+smax of the same loads both convert. A GPR consumer
+ // would force the load into a GPR and block memory folding.
+ auto isScalarMinMax = [](SDNode *U) {
+ unsigned O = U->getOpcode();
+ return (O == ISD::SMIN || O == ISD::SMAX || O == ISD::UMIN ||
+ O == ISD::UMAX) &&
+ !U->getValueType(0).isVector();
+ };
+ auto checkLoadUses = [&](SDValue Ld) {
+ for (SDUse &Use : Ld->uses()) {
+ if (Use.getResNo() != Ld.getResNo())
+ continue;
+ SDNode *User = Use.getUser();
+ if (User->getOpcode() == ISD::SCALAR_TO_VECTOR)
+ continue;
+ if (!isScalarMinMax(User))
+ return false;
+ }
+ return true;
+ };
+ if (!checkLoadUses(Op0) || !checkLoadUses(Op1))
+ return SDValue();
+
+ if (!isOnlyUsedByXMMDomainSink(SDValue(N, 0)))
+ return SDValue();
+
+ SDLoc DL(N);
+ MVT VecVT = MVT::v4i32;
+ SDValue V0 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op0);
+ SDValue V1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op1);
+ SDValue Vec = DAG.getNode(N->getOpcode(), DL, VecVT, V0, V1);
+ return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Vec,
+ DAG.getVectorIdxConstant(0, DL));
+}
+
+/// Combine scalar i32 abs of a plain load whose result is only stored (or
+/// insert_vector_elt) into a vector ABS on v4i32, which ISel folds to
+/// movd + PABSD. Requires SSSE3. Same domain-cross rationale as
+/// combineScalarMinMaxToSIMD.
+static SDValue combineScalarAbsToSIMD(SDNode *N, SelectionDAG &DAG,
+ const X86Subtarget &Subtarget) {
+ EVT VT = N->getValueType(0);
+ // Caller must only pass scalar nodes; still guard for safety.
+ if (!VT.isSimple() || VT.getSimpleVT() != MVT::i32 || !Subtarget.hasSSSE3())
+ return SDValue();
+
+ const Function &F = DAG.getMachineFunction().getFunction();
+ if (F.hasFnAttribute(Attribute::NoImplicitFloat) || Subtarget.useSoftFloat())
+ return SDValue();
+ if (F.hasOptSize())
+ return SDValue();
+
+ SDValue Op0 = N->getOperand(0);
+ if (!isPlainScalarLoad(Op0, VT))
+ return SDValue();
+
+ // The load must only feed this abs (or a SCALAR_TO_VECTOR from a sibling
+ // already combined this turn). A GPR consumer would force the load into a
+ // GPR and block memory folding.
+ for (SDUse &Use : Op0->uses()) {
+ if (Use.getResNo() != Op0.getResNo())
+ continue;
+ SDNode *User = Use.getUser();
+ if (User->getOpcode() == ISD::SCALAR_TO_VECTOR)
+ continue;
+ if (User->getOpcode() != ISD::ABS || User->getValueType(0).isVector())
+ return SDValue();
+ }
+
+ if (!isOnlyUsedByXMMDomainSink(SDValue(N, 0)))
+ return SDValue();
+
+ SDLoc DL(N);
+ MVT VecVT = MVT::v4i32;
+ SDValue V0 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op0);
+ SDValue Vec = DAG.getNode(ISD::ABS, DL, VecVT, V0);
+ return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Vec,
+ DAG.getVectorIdxConstant(0, DL));
+}
+
SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
SelectionDAG &DAG = DCI.DAG;
@@ -63003,6 +63146,19 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
case X86ISD::SUB: return combineX86AddSub(N, DAG, DCI, Subtarget);
case ISD::SADDSAT:
case ISD::SSUBSAT: return combineToHorizontalAddSub(N, DAG, Subtarget);
+ case ISD::ABS:
+ // Vector abs is already Legal/Custom; only try the scalar→SIMD widen.
+ if (!N->getValueType(0).isVector())
+ return combineScalarAbsToSIMD(N, DAG, Subtarget);
+ break;
+ case ISD::SMIN:
+ case ISD::SMAX:
+ case ISD::UMIN:
+ case ISD::UMAX:
+ // Vector min/max are already Legal; only try the scalar→SIMD widen.
+ if (!N->getValueType(0).isVector())
+ return combineScalarMinMaxToSIMD(N, DAG, Subtarget);
+ break;
case X86ISD::CLOAD:
case X86ISD::CSTORE: return combineX86CloadCstore(N, DAG);
case X86ISD::SBB: return combineSBB(N, DAG);
diff --git a/llvm/test/CodeGen/X86/scalar-minmax-simd.ll b/llvm/test/CodeGen/X86/scalar-minmax-simd.ll
new file mode 100644
index 0000000000000..8fc29706c613c
--- /dev/null
+++ b/llvm/test/CodeGen/X86/scalar-minmax-simd.ll
@@ -0,0 +1,686 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=X86-SSE41
+
+declare i32 @llvm.smin.i32(i32, i32)
+declare i32 @llvm.smax.i32(i32, i32)
+declare i32 @llvm.umin.i32(i32, i32)
+declare i32 @llvm.umax.i32(i32, i32)
+declare i32 @llvm.abs.i32(i32, i1)
+
+; Memory-bound sort2 (#210569): prefer vpmin/vpmax over cmp+cmov.
+define void @sort2_i32(ptr %a) nounwind {
+; SSE2-LABEL: sort2_i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %eax
+; SSE2-NEXT: movl 4(%rdi), %ecx
+; SSE2-NEXT: cmpl %ecx, %eax
+; SSE2-NEXT: movl %ecx, %edx
+; SSE2-NEXT: cmovll %eax, %edx
+; SSE2-NEXT: cmovgl %eax, %ecx
+; SSE2-NEXT: movl %edx, (%rdi)
+; SSE2-NEXT: movl %ecx, 4(%rdi)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: sort2_i32:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE41-NEXT: movdqa %xmm1, %xmm2
+; SSE41-NEXT: pminsd %xmm0, %xmm2
+; SSE41-NEXT: pmaxsd %xmm0, %xmm1
+; SSE41-NEXT: movd %xmm2, (%rdi)
+; SSE41-NEXT: movd %xmm1, 4(%rdi)
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: sort2_i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT: vpminsd %xmm0, %xmm1, %xmm2
+; AVX-NEXT: vpmaxsd %xmm0, %xmm1, %xmm0
+; AVX-NEXT: vmovd %xmm2, (%rdi)
+; AVX-NEXT: vmovd %xmm0, 4(%rdi)
+; AVX-NEXT: retq
+;
+; X86-SSE41-LABEL: sort2_i32:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT: movdqa %xmm1, %xmm2
+; X86-SSE41-NEXT: pminsd %xmm0, %xmm2
+; X86-SSE41-NEXT: pmaxsd %xmm0, %xmm1
+; X86-SSE41-NEXT: movd %xmm2, (%eax)
+; X86-SSE41-NEXT: movd %xmm1, 4(%eax)
+; X86-SSE41-NEXT: retl
+ %p0 = getelementptr inbounds i32, ptr %a, i64 0
+ %p1 = getelementptr inbounds i32, ptr %a, i64 1
+ %x = load i32, ptr %p0, align 4
+ %y = load i32, ptr %p1, align 4
+ %lo = call i32 @llvm.smin.i32(i32 %x, i32 %y)
+ %hi = call i32 @llvm.smax.i32(i32 %x, i32 %y)
+ store i32 %lo, ptr %p0, align 4
+ store i32 %hi, ptr %p1, align 4
+ ret void
+}
+
+define void @store_smin_i32(ptr %p, ptr %q, ptr %r) nounwind {
+; SSE2-LABEL: store_smin_i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %eax
+; SSE2-NEXT: movl (%rsi), %ecx
+; SSE2-NEXT: cmpl %ecx, %eax
+; SSE2-NEXT: cmovll %eax, %ecx
+; SSE2-NEXT: movl %ecx, (%rdx)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: store_smin_i32:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE41-NEXT: pminsd %xmm0, %xmm1
+; SSE41-NEXT: movd %xmm1, (%rdx)
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: store_smin_i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT: vpminsd %xmm0, %xmm1, %xmm0
+; AVX-NEXT: vmovd %xmm0, (%rdx)
+; AVX-NEXT: retq
+;
+; X86-SSE41-LABEL: store_smin_i32:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT: pminsd %xmm0, %xmm1
+; X86-SSE41-NEXT: movd %xmm1, (%eax)
+; X86-SSE41-NEXT: retl
+ %x = load i32, ptr %p, align 4
+ %y = load i32, ptr %q, align 4
+ %m = call i32 @llvm.smin.i32(i32 %x, i32 %y)
+ store i32 %m, ptr %r, align 4
+ ret void
+}
+
+define void @store_smax_i32(ptr %p, ptr %q, ptr %r) nounwind {
+; SSE2-LABEL: store_smax_i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %eax
+; SSE2-NEXT: movl (%rsi), %ecx
+; SSE2-NEXT: cmpl %ecx, %eax
+; SSE2-NEXT: cmovgl %eax, %ecx
+; SSE2-NEXT: movl %ecx, (%rdx)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: store_smax_i32:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE41-NEXT: pmaxsd %xmm0, %xmm1
+; SSE41-NEXT: movd %xmm1, (%rdx)
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: store_smax_i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT: vpmaxsd %xmm0, %xmm1, %xmm0
+; AVX-NEXT: vmovd %xmm0, (%rdx)
+; AVX-NEXT: retq
+;
+; X86-SSE41-LABEL: store_smax_i32:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT: pmaxsd %xmm0, %xmm1
+; X86-SSE41-NEXT: movd %xmm1, (%eax)
+; X86-SSE41-NEXT: retl
+ %x = load i32, ptr %p, align 4
+ %y = load i32, ptr %q, align 4
+ %m = call i32 @llvm.smax.i32(i32 %x, i32 %y)
+ store i32 %m, ptr %r, align 4
+ ret void
+}
+
+define void @store_umin_i32(ptr %p, ptr %q, ptr %r) nounwind {
+; SSE2-LABEL: store_umin_i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %eax
+; SSE2-NEXT: movl (%rsi), %ecx
+; SSE2-NEXT: cmpl %ecx, %eax
+; SSE2-NEXT: cmovbl %eax, %ecx
+; SSE2-NEXT: movl %ecx, (%rdx)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: store_umin_i32:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE41-NEXT: pminud %xmm0, %xmm1
+; SSE41-NEXT: movd %xmm1, (%rdx)
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: store_umin_i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT: vpminud %xmm0, %xmm1, %xmm0
+; AVX-NEXT: vmovd %xmm0, (%rdx)
+; AVX-NEXT: retq
+;
+; X86-SSE41-LABEL: store_umin_i32:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT: pminud %xmm0, %xmm1
+; X86-SSE41-NEXT: movd %xmm1, (%eax)
+; X86-SSE41-NEXT: retl
+ %x = load i32, ptr %p, align 4
+ %y = load i32, ptr %q, align 4
+ %m = call i32 @llvm.umin.i32(i32 %x, i32 %y)
+ store i32 %m, ptr %r, align 4
+ ret void
+}
+
+define void @store_umax_i32(ptr %p, ptr %q, ptr %r) nounwind {
+; SSE2-LABEL: store_umax_i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %eax
+; SSE2-NEXT: movl (%rsi), %ecx
+; SSE2-NEXT: cmpl %ecx, %eax
+; SSE2-NEXT: cmoval %eax, %ecx
+; SSE2-NEXT: movl %ecx, (%rdx)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: store_umax_i32:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE41-NEXT: pmaxud %xmm0, %xmm1
+; SSE41-NEXT: movd %xmm1, (%rdx)
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: store_umax_i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT: vpmaxud %xmm0, %xmm1, %xmm0
+; AVX-NEXT: vmovd %xmm0, (%rdx)
+; AVX-NEXT: retq
+;
+; X86-SSE41-LABEL: store_umax_i32:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT: pmaxud %xmm0, %xmm1
+; X86-SSE41-NEXT: movd %xmm1, (%eax)
+; X86-SSE41-NEXT: retl
+ %x = load i32, ptr %p, align 4
+ %y = load i32, ptr %q, align 4
+ %m = call i32 @llvm.umax.i32(i32 %x, i32 %y)
+ store i32 %m, ptr %r, align 4
+ ret void
+}
+
+; Negative: register operands / GPR result should stay on CMOV.
+define i32 @smin_gpr(i32 %x, i32 %y) nounwind {
+; SSE2-LABEL: smin_gpr:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl %esi, %eax
+; SSE2-NEXT: cmpl %esi, %edi
+; SSE2-NEXT: cmovll %edi, %eax
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: smin_gpr:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movl %esi, %eax
+; SSE41-NEXT: cmpl %esi, %edi
+; SSE41-NEXT: cmovll %edi, %eax
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: smin_gpr:
+; AVX: # %bb.0:
+; AVX-NEXT: movl %esi, %eax
+; AVX-NEXT: cmpl %esi, %edi
+; AVX-NEXT: cmovll %edi, %eax
+; AVX-NEXT: retq
+;
+; X86-SSE41-LABEL: smin_gpr:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: cmpl %eax, %ecx
+; X86-SSE41-NEXT: cmovll %ecx, %eax
+; X86-SSE41-NEXT: retl
+ %m = call i32 @llvm.smin.i32(i32 %x, i32 %y)
+ ret i32 %m
+}
+
+define i32 @smin_load_to_gpr(ptr %p, ptr %q) nounwind {
+; SSE2-LABEL: smin_load_to_gpr:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %ecx
+; SSE2-NEXT: movl (%rsi), %eax
+; SSE2-NEXT: cmpl %eax, %ecx
+; SSE2-NEXT: cmovll %ecx, %eax
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: smin_load_to_gpr:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movl (%rdi), %ecx
+; SSE41-NEXT: movl (%rsi), %eax
+; SSE41-NEXT: cmpl %eax, %ecx
+; SSE41-NEXT: cmovll %ecx, %eax
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: smin_load_to_gpr:
+; AVX: # %bb.0:
+; AVX-NEXT: movl (%rdi), %ecx
+; AVX-NEXT: movl (%rsi), %eax
+; AVX-NEXT: cmpl %eax, %ecx
+; AVX-NEXT: cmovll %ecx, %eax
+; AVX-NEXT: retq
+;
+; X86-SSE41-LABEL: smin_load_to_gpr:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl (%ecx), %ecx
+; X86-SSE41-NEXT: movl (%eax), %eax
+; X86-SSE41-NEXT: cmpl %eax, %ecx
+; X86-SSE41-NEXT: cmovll %ecx, %eax
+; X86-SSE41-NEXT: retl
+ %x = load i32, ptr %p, align 4
+ %y = load i32, ptr %q, align 4
+ %m = call i32 @llvm.smin.i32(i32 %x, i32 %y)
+ ret i32 %m
+}
+
+; Negative: noimplicitfloat must not introduce XMM ops.
+define void @sort2_i32_noimplicitfloat(ptr %a) nounwind noimplicitfloat {
+; SSE2-LABEL: sort2_i32_noimplicitfloat:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %eax
+; SSE2-NEXT: movl 4(%rdi), %ecx
+; SSE2-NEXT: cmpl %ecx, %eax
+; SSE2-NEXT: movl %ecx, %edx
+; SSE2-NEXT: cmovll %eax, %edx
+; SSE2-NEXT: cmovgl %eax, %ecx
+; SSE2-NEXT: movl %edx, (%rdi)
+; SSE2-NEXT: movl %ecx, 4(%rdi)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: sort2_i32_noimplicitfloat:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movl (%rdi), %eax
+; SSE41-NEXT: movl 4(%rdi), %ecx
+; SSE41-NEXT: cmpl %ecx, %eax
+; SSE41-NEXT: movl %ecx, %edx
+; SSE41-NEXT: cmovll %eax, %edx
+; SSE41-NEXT: cmovgl %eax, %ecx
+; SSE41-NEXT: movl %edx, (%rdi)
+; SSE41-NEXT: movl %ecx, 4(%rdi)
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: sort2_i32_noimplicitfloat:
+; AVX: # %bb.0:
+; AVX-NEXT: movl (%rdi), %eax
+; AVX-NEXT: movl 4(%rdi), %ecx
+; AVX-NEXT: cmpl %ecx, %eax
+; AVX-NEXT: movl %ecx, %edx
+; AVX-NEXT: cmovll %eax, %edx
+; AVX-NEXT: cmovgl %eax, %ecx
+; AVX-NEXT: movl %edx, (%rdi)
+; AVX-NEXT: movl %ecx, 4(%rdi)
+; AVX-NEXT: retq
+;
+; X86-SSE41-LABEL: sort2_i32_noimplicitfloat:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: pushl %esi
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl (%eax), %ecx
+; X86-SSE41-NEXT: movl 4(%eax), %edx
+; X86-SSE41-NEXT: cmpl %edx, %ecx
+; X86-SSE41-NEXT: movl %edx, %esi
+; X86-SSE41-NEXT: cmovll %ecx, %esi
+; X86-SSE41-NEXT: cmovgl %ecx, %edx
+; X86-SSE41-NEXT: movl %esi, (%eax)
+; X86-SSE41-NEXT: movl %edx, 4(%eax)
+; X86-SSE41-NEXT: popl %esi
+; X86-SSE41-NEXT: retl
+ %p0 = getelementptr inbounds i32, ptr %a, i64 0
+ %p1 = getelementptr inbounds i32, ptr %a, i64 1
+ %x = load i32, ptr %p0, align 4
+ %y = load i32, ptr %p1, align 4
+ %lo = call i32 @llvm.smin.i32(i32 %x, i32 %y)
+ %hi = call i32 @llvm.smax.i32(i32 %x, i32 %y)
+ store i32 %lo, ptr %p0, align 4
+ store i32 %hi, ptr %p1, align 4
+ ret void
+}
+
+; Negative: optsize prefers compact cmp+cmov over movd+pmin/pmax.
+define void @sort2_i32_optsize(ptr %a) nounwind optsize {
+; SSE2-LABEL: sort2_i32_optsize:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %eax
+; SSE2-NEXT: movl 4(%rdi), %ecx
+; SSE2-NEXT: cmpl %ecx, %eax
+; SSE2-NEXT: movl %ecx, %edx
+; SSE2-NEXT: cmovll %eax, %edx
+; SSE2-NEXT: cmovgl %eax, %ecx
+; SSE2-NEXT: movl %edx, (%rdi)
+; SSE2-NEXT: movl %ecx, 4(%rdi)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: sort2_i32_optsize:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movl (%rdi), %eax
+; SSE41-NEXT: movl 4(%rdi), %ecx
+; SSE41-NEXT: cmpl %ecx, %eax
+; SSE41-NEXT: movl %ecx, %edx
+; SSE41-NEXT: cmovll %eax, %edx
+; SSE41-NEXT: cmovgl %eax, %ecx
+; SSE41-NEXT: movl %edx, (%rdi)
+; SSE41-NEXT: movl %ecx, 4(%rdi)
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: sort2_i32_optsize:
+; AVX: # %bb.0:
+; AVX-NEXT: movl (%rdi), %eax
+; AVX-NEXT: movl 4(%rdi), %ecx
+; AVX-NEXT: cmpl %ecx, %eax
+; AVX-NEXT: movl %ecx, %edx
+; AVX-NEXT: cmovll %eax, %edx
+; AVX-NEXT: cmovgl %eax, %ecx
+; AVX-NEXT: movl %edx, (%rdi)
+; AVX-NEXT: movl %ecx, 4(%rdi)
+; AVX-NEXT: retq
+;
+; X86-SSE41-LABEL: sort2_i32_optsize:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: pushl %esi
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl (%eax), %ecx
+; X86-SSE41-NEXT: movl 4(%eax), %edx
+; X86-SSE41-NEXT: cmpl %edx, %ecx
+; X86-SSE41-NEXT: movl %edx, %esi
+; X86-SSE41-NEXT: cmovll %ecx, %esi
+; X86-SSE41-NEXT: cmovgl %ecx, %edx
+; X86-SSE41-NEXT: movl %esi, (%eax)
+; X86-SSE41-NEXT: movl %edx, 4(%eax)
+; X86-SSE41-NEXT: popl %esi
+; X86-SSE41-NEXT: retl
+ %p0 = getelementptr inbounds i32, ptr %a, i64 0
+ %p1 = getelementptr inbounds i32, ptr %a, i64 1
+ %x = load i32, ptr %p0, align 4
+ %y = load i32, ptr %p1, align 4
+ %lo = call i32 @llvm.smin.i32(i32 %x, i32 %y)
+ %hi = call i32 @llvm.smax.i32(i32 %x, i32 %y)
+ store i32 %lo, ptr %p0, align 4
+ store i32 %hi, ptr %p1, align 4
+ ret void
+}
+
+; Negative: constant operands are not XMM-native sources; stay on CMOV.
+define void @store_smin_load_const(ptr %p, ptr %r) nounwind {
+; SSE2-LABEL: store_smin_load_const:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %eax
+; SSE2-NEXT: cmpl $5, %eax
+; SSE2-NEXT: movl $5, %ecx
+; SSE2-NEXT: cmovll %eax, %ecx
+; SSE2-NEXT: movl %ecx, (%rsi)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: store_smin_load_const:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movl (%rdi), %eax
+; SSE41-NEXT: cmpl $5, %eax
+; SSE41-NEXT: movl $5, %ecx
+; SSE41-NEXT: cmovll %eax, %ecx
+; SSE41-NEXT: movl %ecx, (%rsi)
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: store_smin_load_const:
+; AVX: # %bb.0:
+; AVX-NEXT: movl (%rdi), %eax
+; AVX-NEXT: cmpl $5, %eax
+; AVX-NEXT: movl $5, %ecx
+; AVX-NEXT: cmovll %eax, %ecx
+; AVX-NEXT: movl %ecx, (%rsi)
+; AVX-NEXT: retq
+;
+; X86-SSE41-LABEL: store_smin_load_const:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl (%ecx), %ecx
+; X86-SSE41-NEXT: cmpl $5, %ecx
+; X86-SSE41-NEXT: movl $5, %edx
+; X86-SSE41-NEXT: cmovll %ecx, %edx
+; X86-SSE41-NEXT: movl %edx, (%eax)
+; X86-SSE41-NEXT: retl
+ %x = load i32, ptr %p, align 4
+ %m = call i32 @llvm.smin.i32(i32 %x, i32 5)
+ store i32 %m, ptr %r, align 4
+ ret void
+}
+
+; Memory-bound abs (#210569): prefer pabsd over neg+cmov.
+define void @store_abs_i32(ptr %p, ptr %r) nounwind {
+; SSE2-LABEL: store_abs_i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: negl %ecx
+; SSE2-NEXT: cmovsl %eax, %ecx
+; SSE2-NEXT: movl %ecx, (%rsi)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: store_abs_i32:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-NEXT: pabsd %xmm0, %xmm0
+; SSE41-NEXT: movd %xmm0, (%rsi)
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: store_abs_i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT: vpabsd %xmm0, %xmm0
+; AVX-NEXT: vmovd %xmm0, (%rsi)
+; AVX-NEXT: retq
+;
+; X86-SSE41-LABEL: store_abs_i32:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT: pabsd %xmm0, %xmm0
+; X86-SSE41-NEXT: movd %xmm0, (%eax)
+; X86-SSE41-NEXT: retl
+ %x = load i32, ptr %p, align 4
+ %a = call i32 @llvm.abs.i32(i32 %x, i1 false)
+ store i32 %a, ptr %r, align 4
+ ret void
+}
+
+; Negative: register operand / GPR result should stay on neg+cmov.
+define i32 @abs_gpr(i32 %x) nounwind {
+; SSE2-LABEL: abs_gpr:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl %edi, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: cmovsl %edi, %eax
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: abs_gpr:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movl %edi, %eax
+; SSE41-NEXT: negl %eax
+; SSE41-NEXT: cmovsl %edi, %eax
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: abs_gpr:
+; AVX: # %bb.0:
+; AVX-NEXT: movl %edi, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: cmovsl %edi, %eax
+; AVX-NEXT: retq
+;
+; X86-SSE41-LABEL: abs_gpr:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl %ecx, %eax
+; X86-SSE41-NEXT: negl %eax
+; X86-SSE41-NEXT: cmovsl %ecx, %eax
+; X86-SSE41-NEXT: retl
+ %a = call i32 @llvm.abs.i32(i32 %x, i1 false)
+ ret i32 %a
+}
+
+define i32 @abs_load_to_gpr(ptr %p) nounwind {
+; SSE2-LABEL: abs_load_to_gpr:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %ecx
+; SSE2-NEXT: movl %ecx, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: cmovsl %ecx, %eax
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: abs_load_to_gpr:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movl (%rdi), %ecx
+; SSE41-NEXT: movl %ecx, %eax
+; SSE41-NEXT: negl %eax
+; SSE41-NEXT: cmovsl %ecx, %eax
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: abs_load_to_gpr:
+; AVX: # %bb.0:
+; AVX-NEXT: movl (%rdi), %ecx
+; AVX-NEXT: movl %ecx, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: cmovsl %ecx, %eax
+; AVX-NEXT: retq
+;
+; X86-SSE41-LABEL: abs_load_to_gpr:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl (%eax), %ecx
+; X86-SSE41-NEXT: movl %ecx, %eax
+; X86-SSE41-NEXT: negl %eax
+; X86-SSE41-NEXT: cmovsl %ecx, %eax
+; X86-SSE41-NEXT: retl
+ %x = load i32, ptr %p, align 4
+ %a = call i32 @llvm.abs.i32(i32 %x, i1 false)
+ ret i32 %a
+}
+
+; Negative: noimplicitfloat must not introduce XMM ops for abs.
+define void @store_abs_i32_noimplicitfloat(ptr %p, ptr %r) nounwind noimplicitfloat {
+; SSE2-LABEL: store_abs_i32_noimplicitfloat:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: negl %ecx
+; SSE2-NEXT: cmovsl %eax, %ecx
+; SSE2-NEXT: movl %ecx, (%rsi)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: store_abs_i32_noimplicitfloat:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movl (%rdi), %eax
+; SSE41-NEXT: movl %eax, %ecx
+; SSE41-NEXT: negl %ecx
+; SSE41-NEXT: cmovsl %eax, %ecx
+; SSE41-NEXT: movl %ecx, (%rsi)
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: store_abs_i32_noimplicitfloat:
+; AVX: # %bb.0:
+; AVX-NEXT: movl (%rdi), %eax
+; AVX-NEXT: movl %eax, %ecx
+; AVX-NEXT: negl %ecx
+; AVX-NEXT: cmovsl %eax, %ecx
+; AVX-NEXT: movl %ecx, (%rsi)
+; AVX-NEXT: retq
+;
+; X86-SSE41-LABEL: store_abs_i32_noimplicitfloat:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl (%ecx), %ecx
+; X86-SSE41-NEXT: movl %ecx, %edx
+; X86-SSE41-NEXT: negl %edx
+; X86-SSE41-NEXT: cmovsl %ecx, %edx
+; X86-SSE41-NEXT: movl %edx, (%eax)
+; X86-SSE41-NEXT: retl
+ %x = load i32, ptr %p, align 4
+ %a = call i32 @llvm.abs.i32(i32 %x, i1 false)
+ store i32 %a, ptr %r, align 4
+ ret void
+}
+
+; Negative: optsize prefers compact neg+cmov over movd+pabsd.
+define void @store_abs_i32_optsize(ptr %p, ptr %r) nounwind optsize {
+; SSE2-LABEL: store_abs_i32_optsize:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: negl %ecx
+; SSE2-NEXT: cmovsl %eax, %ecx
+; SSE2-NEXT: movl %ecx, (%rsi)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: store_abs_i32_optsize:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movl (%rdi), %eax
+; SSE41-NEXT: movl %eax, %ecx
+; SSE41-NEXT: negl %ecx
+; SSE41-NEXT: cmovsl %eax, %ecx
+; SSE41-NEXT: movl %ecx, (%rsi)
+; SSE41-NEXT: retq
+;
+; AVX-LABEL: store_abs_i32_optsize:
+; AVX: # %bb.0:
+; AVX-NEXT: movl (%rdi), %eax
+; AVX-NEXT: movl %eax, %ecx
+; AVX-NEXT: negl %ecx
+; AVX-NEXT: cmovsl %eax, %ecx
+; AVX-NEXT: movl %ecx, (%rsi)
+; AVX-NEXT: retq
+;
+; X86-SSE41-LABEL: store_abs_i32_optsize:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl (%ecx), %ecx
+; X86-SSE41-NEXT: movl %ecx, %edx
+; X86-SSE41-NEXT: negl %edx
+; X86-SSE41-NEXT: cmovsl %ecx, %edx
+; X86-SSE41-NEXT: movl %edx, (%eax)
+; X86-SSE41-NEXT: retl
+ %x = load i32, ptr %p, align 4
+ %a = call i32 @llvm.abs.i32(i32 %x, i1 false)
+ store i32 %a, ptr %r, align 4
+ ret void
+}
More information about the llvm-commits
mailing list