[llvm] 6f406b7 - [X86] Prefer SIMD min/max/abs for scalars when staying in XMM domain (#210654)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 14 01:13:52 PDT 2026
Author: Shaurya Srivastava
Date: 2026-08-14T08:13:46Z
New Revision: 6f406b7e3bff397e20603c78b15c892bf2ff6bb1
URL: https://github.com/llvm/llvm-project/commit/6f406b7e3bff397e20603c78b15c892bf2ff6bb1
DIFF: https://github.com/llvm/llvm-project/commit/6f406b7e3bff397e20603c78b15c892bf2ff6bb1.diff
LOG: [X86] Prefer SIMD min/max/abs for scalars when staying in XMM domain (#210654)
For memory-bound scalar `smin`/`smax`/`umin`/`umax`/`abs` of
`i16`/`i32`/`i64` loads whose result is stored, prefer SSE `PMIN`/`PMAX`/`PABS` over
`cmp`+`cmov` / `neg`+`cmov`, matching GCC on cases like `sort2` and
scalar abs-to-memory.
Fixes #210569
Added:
llvm/test/CodeGen/X86/scalar-minmax-simd.ll
Modified:
llvm/include/llvm/CodeGen/ISDOpcodes.h
llvm/lib/Target/X86/X86ISelLowering.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 43eba643bc110..0ce6805ae8f5a 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -1688,6 +1688,12 @@ inline bool isAbsOpcode(unsigned Opcode) {
return Opcode == ISD::ABS || Opcode == ISD::ABS_MIN_POISON;
}
+/// Whether this is an integer min/max opcode (ISD::(U|S)MIN or ISD::(U|S)MAX).
+inline bool isMinMaxOpcode(unsigned Opcode) {
+ return Opcode == ISD::SMIN || Opcode == ISD::SMAX || Opcode == ISD::UMIN ||
+ Opcode == ISD::UMAX;
+}
+
/// Given a \p MinMaxOpc of ISD::(U|S)MIN or ISD::(U|S)MAX, returns
/// ISD::(U|S)MAX and ISD::(U|S)MIN, respectively.
LLVM_ABI NodeType getInverseMinMaxOpcode(unsigned MinMaxOpc);
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index c6e1051f0bd8e..86e67626cdadb 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -54936,6 +54936,90 @@ static SDValue narrowBitOpRMW(StoreSDNode *St, const SDLoc &DL,
return NewStore;
}
+/// Fold store(abs/min/max(load…)) of scalar i16/i32/i64 into SIMD
+/// PABS/PMIN/PMAX to keep memory-bound sort2/abs in the XMM domain.
+static SDValue combineScalarMinMaxAbsStore(StoreSDNode *St, const SDLoc &DL,
+ SelectionDAG &DAG,
+ const X86Subtarget &Subtarget) {
+ if (!ISD::isNormalStore(St))
+ return SDValue();
+
+ SDValue StoredVal = St->getValue();
+ unsigned Opc = StoredVal.getOpcode();
+ bool IsAbs = Opc == ISD::ABS;
+ if ((!IsAbs && !ISD::isMinMaxOpcode(Opc)) || !StoredVal.hasOneUse())
+ return SDValue();
+
+ EVT VT = StoredVal.getValueType();
+ const Function &F = DAG.getMachineFunction().getFunction();
+ if (F.hasFnAttribute(Attribute::NoImplicitFloat) ||
+ Subtarget.useSoftFloat() || F.hasOptSize())
+ return SDValue();
+
+ // Pick the widened vector type; require a legal vector op and that
+ // SCALAR_TO_VECTOR can fold the scalar load into XMM (i32/i64: SSE2
+ // movd/movq; i16: FP16 vmovw).
+ auto getVecVT = [&]() -> std::optional<MVT> {
+ MVT VecVT;
+ if (VT == MVT::i32 && Subtarget.hasSSE2())
+ VecVT = MVT::v4i32;
+ else if (VT == MVT::i64 && Subtarget.hasSSE2())
+ VecVT = MVT::v2i64;
+ else if (VT == MVT::i16 && Subtarget.hasFP16())
+ VecVT = MVT::v8i16;
+ else
+ return std::nullopt;
+
+ if (!DAG.getTargetLoweringInfo().isOperationLegal(Opc, VecVT))
+ return std::nullopt;
+ return VecVT;
+ };
+ std::optional<MVT> VecVTOpt = getVecVT();
+ if (!VecVTOpt)
+ return SDValue();
+ MVT VecVT = *VecVTOpt;
+
+ SDValue Op0 = StoredVal.getOperand(0);
+ if (!ISD::isNormalLoad(Op0.getNode()))
+ return SDValue();
+
+ SDValue Vec;
+ if (IsAbs) {
+ if (!Op0.hasOneUse())
+ return SDValue();
+ Vec = DAG.getNode(ISD::ABS, DL, VecVT,
+ DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op0));
+ } else {
+ SDValue Op1 = StoredVal.getOperand(1);
+ if (!ISD::isNormalLoad(Op1.getNode()))
+ return SDValue();
+
+ // Loads may feed paired smin+smax (sort2); allow SCALAR_TO_VECTOR from a
+ // sibling already combined this turn so CSE can share them.
+ auto checkLoadUses = [](SDValue Ld) {
+ return all_of(Ld->uses(), [&](const SDUse &Use) {
+ if (Use.getResNo() != Ld.getResNo())
+ return true;
+ const SDNode *User = Use.getUser();
+ return User->getOpcode() == ISD::SCALAR_TO_VECTOR ||
+ (ISD::isMinMaxOpcode(User->getOpcode()) &&
+ !User->getValueType(0).isVector());
+ });
+ };
+ if (!checkLoadUses(Op0) || !checkLoadUses(Op1))
+ return SDValue();
+
+ Vec = DAG.getNode(Opc, DL, VecVT,
+ DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op0),
+ DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op1));
+ }
+
+ return DAG.getStore(St->getChain(), DL,
+ DAG.getExtractVectorElt(DL, VT, Vec, 0), St->getBasePtr(),
+ St->getPointerInfo(), St->getBaseAlign(),
+ St->getMemOperand()->getFlags());
+}
+
static SDValue combineStore(SDNode *N, SelectionDAG &DAG,
TargetLowering::DAGCombinerInfo &DCI,
const X86Subtarget &Subtarget) {
@@ -55061,6 +55145,11 @@ static SDValue combineStore(SDNode *N, SelectionDAG &DAG,
}
}
+ // Convert scalar abs/min/max of loads stored back to memory into SIMD
+ // PABS/PMIN/PMAX, keeping the values in the XMM domain.
+ if (SDValue R = combineScalarMinMaxAbsStore(St, dl, DAG, Subtarget))
+ return R;
+
// If we are saving a 32-byte vector and 32-byte stores are slow, such as on
// Sandy Bridge, perform two 16-byte stores.
unsigned Fast;
diff --git a/llvm/test/CodeGen/X86/scalar-minmax-simd.ll b/llvm/test/CodeGen/X86/scalar-minmax-simd.ll
new file mode 100644
index 0000000000000..f49c6bb34e781
--- /dev/null
+++ b/llvm/test/CodeGen/X86/scalar-minmax-simd.ll
@@ -0,0 +1,1725 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE41
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=X86-SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512VL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16 | FileCheck %s --check-prefixes=AVX512FP16
+
+; Memory-bound sort2 (#210569): prefer vpmin/vpmax over cmp+cmov.
+define void @sort2_i32(ptr %a) nounwind {
+; SSE2-LABEL: sort2_i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %eax
+; SSE2-NEXT: movl 4(%rdi), %ecx
+; SSE2-NEXT: cmpl %ecx, %eax
+; SSE2-NEXT: movl %ecx, %edx
+; SSE2-NEXT: cmovll %eax, %edx
+; SSE2-NEXT: cmovgl %eax, %ecx
+; SSE2-NEXT: movl %edx, (%rdi)
+; SSE2-NEXT: movl %ecx, 4(%rdi)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: sort2_i32:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE41-NEXT: movdqa %xmm1, %xmm2
+; SSE41-NEXT: pminsd %xmm0, %xmm2
+; SSE41-NEXT: movd %xmm2, (%rdi)
+; SSE41-NEXT: pmaxsd %xmm0, %xmm1
+; SSE41-NEXT: movd %xmm1, 4(%rdi)
+; SSE41-NEXT: retq
+;
+; X86-SSE41-LABEL: sort2_i32:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT: movdqa %xmm1, %xmm2
+; X86-SSE41-NEXT: pminsd %xmm0, %xmm2
+; X86-SSE41-NEXT: movd %xmm2, (%eax)
+; X86-SSE41-NEXT: pmaxsd %xmm0, %xmm1
+; X86-SSE41-NEXT: movd %xmm1, 4(%eax)
+; X86-SSE41-NEXT: retl
+;
+; AVX-LABEL: sort2_i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT: vpminsd %xmm0, %xmm1, %xmm2
+; AVX-NEXT: vmovd %xmm2, (%rdi)
+; AVX-NEXT: vpmaxsd %xmm0, %xmm1, %xmm0
+; AVX-NEXT: vmovd %xmm0, 4(%rdi)
+; AVX-NEXT: retq
+;
+; AVX512F-LABEL: sort2_i32:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512F-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512F-NEXT: vpminsd %xmm0, %xmm1, %xmm2
+; AVX512F-NEXT: vmovd %xmm2, (%rdi)
+; AVX512F-NEXT: vpmaxsd %xmm0, %xmm1, %xmm0
+; AVX512F-NEXT: vmovd %xmm0, 4(%rdi)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: sort2_i32:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512VL-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512VL-NEXT: vpminsd %xmm0, %xmm1, %xmm2
+; AVX512VL-NEXT: vmovd %xmm2, (%rdi)
+; AVX512VL-NEXT: vpmaxsd %xmm0, %xmm1, %xmm0
+; AVX512VL-NEXT: vmovd %xmm0, 4(%rdi)
+; AVX512VL-NEXT: retq
+;
+; AVX512FP16-LABEL: sort2_i32:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512FP16-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512FP16-NEXT: vpminsd %xmm0, %xmm1, %xmm2
+; AVX512FP16-NEXT: vmovd %xmm2, (%rdi)
+; AVX512FP16-NEXT: vpmaxsd %xmm0, %xmm1, %xmm0
+; AVX512FP16-NEXT: vmovd %xmm0, 4(%rdi)
+; AVX512FP16-NEXT: retq
+ %p0 = getelementptr inbounds i32, ptr %a, i64 0
+ %p1 = getelementptr inbounds i32, ptr %a, i64 1
+ %x = load i32, ptr %p0, align 4
+ %y = load i32, ptr %p1, align 4
+ %lo = call i32 @llvm.smin.i32(i32 %x, i32 %y)
+ %hi = call i32 @llvm.smax.i32(i32 %x, i32 %y)
+ store i32 %lo, ptr %p0, align 4
+ store i32 %hi, ptr %p1, align 4
+ ret void
+}
+
+define void @store_smin_i32(ptr %p, ptr %q, ptr %r) nounwind {
+; SSE2-LABEL: store_smin_i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %eax
+; SSE2-NEXT: movl (%rsi), %ecx
+; SSE2-NEXT: cmpl %ecx, %eax
+; SSE2-NEXT: cmovll %eax, %ecx
+; SSE2-NEXT: movl %ecx, (%rdx)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: store_smin_i32:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE41-NEXT: pminsd %xmm0, %xmm1
+; SSE41-NEXT: movd %xmm1, (%rdx)
+; SSE41-NEXT: retq
+;
+; X86-SSE41-LABEL: store_smin_i32:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT: pminsd %xmm0, %xmm1
+; X86-SSE41-NEXT: movd %xmm1, (%eax)
+; X86-SSE41-NEXT: retl
+;
+; AVX-LABEL: store_smin_i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT: vpminsd %xmm0, %xmm1, %xmm0
+; AVX-NEXT: vmovd %xmm0, (%rdx)
+; AVX-NEXT: retq
+;
+; AVX512F-LABEL: store_smin_i32:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512F-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512F-NEXT: vpminsd %xmm0, %xmm1, %xmm0
+; AVX512F-NEXT: vmovd %xmm0, (%rdx)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: store_smin_i32:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512VL-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512VL-NEXT: vpminsd %xmm0, %xmm1, %xmm0
+; AVX512VL-NEXT: vmovd %xmm0, (%rdx)
+; AVX512VL-NEXT: retq
+;
+; AVX512FP16-LABEL: store_smin_i32:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512FP16-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512FP16-NEXT: vpminsd %xmm0, %xmm1, %xmm0
+; AVX512FP16-NEXT: vmovd %xmm0, (%rdx)
+; AVX512FP16-NEXT: retq
+ %x = load i32, ptr %p, align 4
+ %y = load i32, ptr %q, align 4
+ %m = call i32 @llvm.smin.i32(i32 %x, i32 %y)
+ store i32 %m, ptr %r, align 4
+ ret void
+}
+
+define void @store_smax_i32(ptr %p, ptr %q, ptr %r) nounwind {
+; SSE2-LABEL: store_smax_i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %eax
+; SSE2-NEXT: movl (%rsi), %ecx
+; SSE2-NEXT: cmpl %ecx, %eax
+; SSE2-NEXT: cmovgl %eax, %ecx
+; SSE2-NEXT: movl %ecx, (%rdx)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: store_smax_i32:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE41-NEXT: pmaxsd %xmm0, %xmm1
+; SSE41-NEXT: movd %xmm1, (%rdx)
+; SSE41-NEXT: retq
+;
+; X86-SSE41-LABEL: store_smax_i32:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT: pmaxsd %xmm0, %xmm1
+; X86-SSE41-NEXT: movd %xmm1, (%eax)
+; X86-SSE41-NEXT: retl
+;
+; AVX-LABEL: store_smax_i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT: vpmaxsd %xmm0, %xmm1, %xmm0
+; AVX-NEXT: vmovd %xmm0, (%rdx)
+; AVX-NEXT: retq
+;
+; AVX512F-LABEL: store_smax_i32:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512F-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512F-NEXT: vpmaxsd %xmm0, %xmm1, %xmm0
+; AVX512F-NEXT: vmovd %xmm0, (%rdx)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: store_smax_i32:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512VL-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512VL-NEXT: vpmaxsd %xmm0, %xmm1, %xmm0
+; AVX512VL-NEXT: vmovd %xmm0, (%rdx)
+; AVX512VL-NEXT: retq
+;
+; AVX512FP16-LABEL: store_smax_i32:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512FP16-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512FP16-NEXT: vpmaxsd %xmm0, %xmm1, %xmm0
+; AVX512FP16-NEXT: vmovd %xmm0, (%rdx)
+; AVX512FP16-NEXT: retq
+ %x = load i32, ptr %p, align 4
+ %y = load i32, ptr %q, align 4
+ %m = call i32 @llvm.smax.i32(i32 %x, i32 %y)
+ store i32 %m, ptr %r, align 4
+ ret void
+}
+
+define void @store_umin_i32(ptr %p, ptr %q, ptr %r) nounwind {
+; SSE2-LABEL: store_umin_i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %eax
+; SSE2-NEXT: movl (%rsi), %ecx
+; SSE2-NEXT: cmpl %ecx, %eax
+; SSE2-NEXT: cmovbl %eax, %ecx
+; SSE2-NEXT: movl %ecx, (%rdx)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: store_umin_i32:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE41-NEXT: pminud %xmm0, %xmm1
+; SSE41-NEXT: movd %xmm1, (%rdx)
+; SSE41-NEXT: retq
+;
+; X86-SSE41-LABEL: store_umin_i32:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT: pminud %xmm0, %xmm1
+; X86-SSE41-NEXT: movd %xmm1, (%eax)
+; X86-SSE41-NEXT: retl
+;
+; AVX-LABEL: store_umin_i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT: vpminud %xmm0, %xmm1, %xmm0
+; AVX-NEXT: vmovd %xmm0, (%rdx)
+; AVX-NEXT: retq
+;
+; AVX512F-LABEL: store_umin_i32:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512F-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512F-NEXT: vpminud %xmm0, %xmm1, %xmm0
+; AVX512F-NEXT: vmovd %xmm0, (%rdx)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: store_umin_i32:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512VL-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512VL-NEXT: vpminud %xmm0, %xmm1, %xmm0
+; AVX512VL-NEXT: vmovd %xmm0, (%rdx)
+; AVX512VL-NEXT: retq
+;
+; AVX512FP16-LABEL: store_umin_i32:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512FP16-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512FP16-NEXT: vpminud %xmm0, %xmm1, %xmm0
+; AVX512FP16-NEXT: vmovd %xmm0, (%rdx)
+; AVX512FP16-NEXT: retq
+ %x = load i32, ptr %p, align 4
+ %y = load i32, ptr %q, align 4
+ %m = call i32 @llvm.umin.i32(i32 %x, i32 %y)
+ store i32 %m, ptr %r, align 4
+ ret void
+}
+
+define void @store_umax_i32(ptr %p, ptr %q, ptr %r) nounwind {
+; SSE2-LABEL: store_umax_i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %eax
+; SSE2-NEXT: movl (%rsi), %ecx
+; SSE2-NEXT: cmpl %ecx, %eax
+; SSE2-NEXT: cmoval %eax, %ecx
+; SSE2-NEXT: movl %ecx, (%rdx)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: store_umax_i32:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE41-NEXT: pmaxud %xmm0, %xmm1
+; SSE41-NEXT: movd %xmm1, (%rdx)
+; SSE41-NEXT: retq
+;
+; X86-SSE41-LABEL: store_umax_i32:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT: pmaxud %xmm0, %xmm1
+; X86-SSE41-NEXT: movd %xmm1, (%eax)
+; X86-SSE41-NEXT: retl
+;
+; AVX-LABEL: store_umax_i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT: vpmaxud %xmm0, %xmm1, %xmm0
+; AVX-NEXT: vmovd %xmm0, (%rdx)
+; AVX-NEXT: retq
+;
+; AVX512F-LABEL: store_umax_i32:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512F-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512F-NEXT: vpmaxud %xmm0, %xmm1, %xmm0
+; AVX512F-NEXT: vmovd %xmm0, (%rdx)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: store_umax_i32:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512VL-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512VL-NEXT: vpmaxud %xmm0, %xmm1, %xmm0
+; AVX512VL-NEXT: vmovd %xmm0, (%rdx)
+; AVX512VL-NEXT: retq
+;
+; AVX512FP16-LABEL: store_umax_i32:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512FP16-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX512FP16-NEXT: vpmaxud %xmm0, %xmm1, %xmm0
+; AVX512FP16-NEXT: vmovd %xmm0, (%rdx)
+; AVX512FP16-NEXT: retq
+ %x = load i32, ptr %p, align 4
+ %y = load i32, ptr %q, align 4
+ %m = call i32 @llvm.umax.i32(i32 %x, i32 %y)
+ store i32 %m, ptr %r, align 4
+ ret void
+}
+
+; Negative: register operands / GPR result should stay on CMOV.
+define i32 @smin_gpr(i32 %x, i32 %y) nounwind {
+; SSE2-LABEL: smin_gpr:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl %esi, %eax
+; SSE2-NEXT: cmpl %esi, %edi
+; SSE2-NEXT: cmovll %edi, %eax
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: smin_gpr:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movl %esi, %eax
+; SSE41-NEXT: cmpl %esi, %edi
+; SSE41-NEXT: cmovll %edi, %eax
+; SSE41-NEXT: retq
+;
+; X86-SSE41-LABEL: smin_gpr:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: cmpl %eax, %ecx
+; X86-SSE41-NEXT: cmovll %ecx, %eax
+; X86-SSE41-NEXT: retl
+;
+; AVX-LABEL: smin_gpr:
+; AVX: # %bb.0:
+; AVX-NEXT: movl %esi, %eax
+; AVX-NEXT: cmpl %esi, %edi
+; AVX-NEXT: cmovll %edi, %eax
+; AVX-NEXT: retq
+;
+; AVX512F-LABEL: smin_gpr:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movl %esi, %eax
+; AVX512F-NEXT: cmpl %esi, %edi
+; AVX512F-NEXT: cmovll %edi, %eax
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: smin_gpr:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movl %esi, %eax
+; AVX512VL-NEXT: cmpl %esi, %edi
+; AVX512VL-NEXT: cmovll %edi, %eax
+; AVX512VL-NEXT: retq
+;
+; AVX512FP16-LABEL: smin_gpr:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: movl %esi, %eax
+; AVX512FP16-NEXT: cmpl %esi, %edi
+; AVX512FP16-NEXT: cmovll %edi, %eax
+; AVX512FP16-NEXT: retq
+ %m = call i32 @llvm.smin.i32(i32 %x, i32 %y)
+ ret i32 %m
+}
+
+define i32 @smin_load_to_gpr(ptr %p, ptr %q) nounwind {
+; SSE2-LABEL: smin_load_to_gpr:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %ecx
+; SSE2-NEXT: movl (%rsi), %eax
+; SSE2-NEXT: cmpl %eax, %ecx
+; SSE2-NEXT: cmovll %ecx, %eax
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: smin_load_to_gpr:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movl (%rdi), %ecx
+; SSE41-NEXT: movl (%rsi), %eax
+; SSE41-NEXT: cmpl %eax, %ecx
+; SSE41-NEXT: cmovll %ecx, %eax
+; SSE41-NEXT: retq
+;
+; X86-SSE41-LABEL: smin_load_to_gpr:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl (%ecx), %ecx
+; X86-SSE41-NEXT: movl (%eax), %eax
+; X86-SSE41-NEXT: cmpl %eax, %ecx
+; X86-SSE41-NEXT: cmovll %ecx, %eax
+; X86-SSE41-NEXT: retl
+;
+; AVX-LABEL: smin_load_to_gpr:
+; AVX: # %bb.0:
+; AVX-NEXT: movl (%rdi), %ecx
+; AVX-NEXT: movl (%rsi), %eax
+; AVX-NEXT: cmpl %eax, %ecx
+; AVX-NEXT: cmovll %ecx, %eax
+; AVX-NEXT: retq
+;
+; AVX512F-LABEL: smin_load_to_gpr:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movl (%rdi), %ecx
+; AVX512F-NEXT: movl (%rsi), %eax
+; AVX512F-NEXT: cmpl %eax, %ecx
+; AVX512F-NEXT: cmovll %ecx, %eax
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: smin_load_to_gpr:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movl (%rdi), %ecx
+; AVX512VL-NEXT: movl (%rsi), %eax
+; AVX512VL-NEXT: cmpl %eax, %ecx
+; AVX512VL-NEXT: cmovll %ecx, %eax
+; AVX512VL-NEXT: retq
+;
+; AVX512FP16-LABEL: smin_load_to_gpr:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: movl (%rdi), %ecx
+; AVX512FP16-NEXT: movl (%rsi), %eax
+; AVX512FP16-NEXT: cmpl %eax, %ecx
+; AVX512FP16-NEXT: cmovll %ecx, %eax
+; AVX512FP16-NEXT: retq
+ %x = load i32, ptr %p, align 4
+ %y = load i32, ptr %q, align 4
+ %m = call i32 @llvm.smin.i32(i32 %x, i32 %y)
+ ret i32 %m
+}
+
+; Negative: noimplicitfloat must not introduce XMM ops.
+define void @sort2_i32_noimplicitfloat(ptr %a) nounwind noimplicitfloat {
+; SSE2-LABEL: sort2_i32_noimplicitfloat:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %eax
+; SSE2-NEXT: movl 4(%rdi), %ecx
+; SSE2-NEXT: cmpl %ecx, %eax
+; SSE2-NEXT: movl %ecx, %edx
+; SSE2-NEXT: cmovll %eax, %edx
+; SSE2-NEXT: cmovgl %eax, %ecx
+; SSE2-NEXT: movl %edx, (%rdi)
+; SSE2-NEXT: movl %ecx, 4(%rdi)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: sort2_i32_noimplicitfloat:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movl (%rdi), %eax
+; SSE41-NEXT: movl 4(%rdi), %ecx
+; SSE41-NEXT: cmpl %ecx, %eax
+; SSE41-NEXT: movl %ecx, %edx
+; SSE41-NEXT: cmovll %eax, %edx
+; SSE41-NEXT: cmovgl %eax, %ecx
+; SSE41-NEXT: movl %edx, (%rdi)
+; SSE41-NEXT: movl %ecx, 4(%rdi)
+; SSE41-NEXT: retq
+;
+; X86-SSE41-LABEL: sort2_i32_noimplicitfloat:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: pushl %esi
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl (%eax), %ecx
+; X86-SSE41-NEXT: movl 4(%eax), %edx
+; X86-SSE41-NEXT: cmpl %edx, %ecx
+; X86-SSE41-NEXT: movl %edx, %esi
+; X86-SSE41-NEXT: cmovll %ecx, %esi
+; X86-SSE41-NEXT: cmovgl %ecx, %edx
+; X86-SSE41-NEXT: movl %esi, (%eax)
+; X86-SSE41-NEXT: movl %edx, 4(%eax)
+; X86-SSE41-NEXT: popl %esi
+; X86-SSE41-NEXT: retl
+;
+; AVX-LABEL: sort2_i32_noimplicitfloat:
+; AVX: # %bb.0:
+; AVX-NEXT: movl (%rdi), %eax
+; AVX-NEXT: movl 4(%rdi), %ecx
+; AVX-NEXT: cmpl %ecx, %eax
+; AVX-NEXT: movl %ecx, %edx
+; AVX-NEXT: cmovll %eax, %edx
+; AVX-NEXT: cmovgl %eax, %ecx
+; AVX-NEXT: movl %edx, (%rdi)
+; AVX-NEXT: movl %ecx, 4(%rdi)
+; AVX-NEXT: retq
+;
+; AVX512F-LABEL: sort2_i32_noimplicitfloat:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movl (%rdi), %eax
+; AVX512F-NEXT: movl 4(%rdi), %ecx
+; AVX512F-NEXT: cmpl %ecx, %eax
+; AVX512F-NEXT: movl %ecx, %edx
+; AVX512F-NEXT: cmovll %eax, %edx
+; AVX512F-NEXT: cmovgl %eax, %ecx
+; AVX512F-NEXT: movl %edx, (%rdi)
+; AVX512F-NEXT: movl %ecx, 4(%rdi)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: sort2_i32_noimplicitfloat:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movl (%rdi), %eax
+; AVX512VL-NEXT: movl 4(%rdi), %ecx
+; AVX512VL-NEXT: cmpl %ecx, %eax
+; AVX512VL-NEXT: movl %ecx, %edx
+; AVX512VL-NEXT: cmovll %eax, %edx
+; AVX512VL-NEXT: cmovgl %eax, %ecx
+; AVX512VL-NEXT: movl %edx, (%rdi)
+; AVX512VL-NEXT: movl %ecx, 4(%rdi)
+; AVX512VL-NEXT: retq
+;
+; AVX512FP16-LABEL: sort2_i32_noimplicitfloat:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: movl (%rdi), %eax
+; AVX512FP16-NEXT: movl 4(%rdi), %ecx
+; AVX512FP16-NEXT: cmpl %ecx, %eax
+; AVX512FP16-NEXT: movl %ecx, %edx
+; AVX512FP16-NEXT: cmovll %eax, %edx
+; AVX512FP16-NEXT: cmovgl %eax, %ecx
+; AVX512FP16-NEXT: movl %edx, (%rdi)
+; AVX512FP16-NEXT: movl %ecx, 4(%rdi)
+; AVX512FP16-NEXT: retq
+ %p0 = getelementptr inbounds i32, ptr %a, i64 0
+ %p1 = getelementptr inbounds i32, ptr %a, i64 1
+ %x = load i32, ptr %p0, align 4
+ %y = load i32, ptr %p1, align 4
+ %lo = call i32 @llvm.smin.i32(i32 %x, i32 %y)
+ %hi = call i32 @llvm.smax.i32(i32 %x, i32 %y)
+ store i32 %lo, ptr %p0, align 4
+ store i32 %hi, ptr %p1, align 4
+ ret void
+}
+
+; Negative: optsize prefers compact cmp+cmov over movd+pmin/pmax.
+define void @sort2_i32_optsize(ptr %a) nounwind optsize {
+; SSE2-LABEL: sort2_i32_optsize:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %eax
+; SSE2-NEXT: movl 4(%rdi), %ecx
+; SSE2-NEXT: cmpl %ecx, %eax
+; SSE2-NEXT: movl %ecx, %edx
+; SSE2-NEXT: cmovll %eax, %edx
+; SSE2-NEXT: cmovgl %eax, %ecx
+; SSE2-NEXT: movl %edx, (%rdi)
+; SSE2-NEXT: movl %ecx, 4(%rdi)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: sort2_i32_optsize:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movl (%rdi), %eax
+; SSE41-NEXT: movl 4(%rdi), %ecx
+; SSE41-NEXT: cmpl %ecx, %eax
+; SSE41-NEXT: movl %ecx, %edx
+; SSE41-NEXT: cmovll %eax, %edx
+; SSE41-NEXT: cmovgl %eax, %ecx
+; SSE41-NEXT: movl %edx, (%rdi)
+; SSE41-NEXT: movl %ecx, 4(%rdi)
+; SSE41-NEXT: retq
+;
+; X86-SSE41-LABEL: sort2_i32_optsize:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: pushl %esi
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl (%eax), %ecx
+; X86-SSE41-NEXT: movl 4(%eax), %edx
+; X86-SSE41-NEXT: cmpl %edx, %ecx
+; X86-SSE41-NEXT: movl %edx, %esi
+; X86-SSE41-NEXT: cmovll %ecx, %esi
+; X86-SSE41-NEXT: cmovgl %ecx, %edx
+; X86-SSE41-NEXT: movl %esi, (%eax)
+; X86-SSE41-NEXT: movl %edx, 4(%eax)
+; X86-SSE41-NEXT: popl %esi
+; X86-SSE41-NEXT: retl
+;
+; AVX-LABEL: sort2_i32_optsize:
+; AVX: # %bb.0:
+; AVX-NEXT: movl (%rdi), %eax
+; AVX-NEXT: movl 4(%rdi), %ecx
+; AVX-NEXT: cmpl %ecx, %eax
+; AVX-NEXT: movl %ecx, %edx
+; AVX-NEXT: cmovll %eax, %edx
+; AVX-NEXT: cmovgl %eax, %ecx
+; AVX-NEXT: movl %edx, (%rdi)
+; AVX-NEXT: movl %ecx, 4(%rdi)
+; AVX-NEXT: retq
+;
+; AVX512F-LABEL: sort2_i32_optsize:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movl (%rdi), %eax
+; AVX512F-NEXT: movl 4(%rdi), %ecx
+; AVX512F-NEXT: cmpl %ecx, %eax
+; AVX512F-NEXT: movl %ecx, %edx
+; AVX512F-NEXT: cmovll %eax, %edx
+; AVX512F-NEXT: cmovgl %eax, %ecx
+; AVX512F-NEXT: movl %edx, (%rdi)
+; AVX512F-NEXT: movl %ecx, 4(%rdi)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: sort2_i32_optsize:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movl (%rdi), %eax
+; AVX512VL-NEXT: movl 4(%rdi), %ecx
+; AVX512VL-NEXT: cmpl %ecx, %eax
+; AVX512VL-NEXT: movl %ecx, %edx
+; AVX512VL-NEXT: cmovll %eax, %edx
+; AVX512VL-NEXT: cmovgl %eax, %ecx
+; AVX512VL-NEXT: movl %edx, (%rdi)
+; AVX512VL-NEXT: movl %ecx, 4(%rdi)
+; AVX512VL-NEXT: retq
+;
+; AVX512FP16-LABEL: sort2_i32_optsize:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: movl (%rdi), %eax
+; AVX512FP16-NEXT: movl 4(%rdi), %ecx
+; AVX512FP16-NEXT: cmpl %ecx, %eax
+; AVX512FP16-NEXT: movl %ecx, %edx
+; AVX512FP16-NEXT: cmovll %eax, %edx
+; AVX512FP16-NEXT: cmovgl %eax, %ecx
+; AVX512FP16-NEXT: movl %edx, (%rdi)
+; AVX512FP16-NEXT: movl %ecx, 4(%rdi)
+; AVX512FP16-NEXT: retq
+ %p0 = getelementptr inbounds i32, ptr %a, i64 0
+ %p1 = getelementptr inbounds i32, ptr %a, i64 1
+ %x = load i32, ptr %p0, align 4
+ %y = load i32, ptr %p1, align 4
+ %lo = call i32 @llvm.smin.i32(i32 %x, i32 %y)
+ %hi = call i32 @llvm.smax.i32(i32 %x, i32 %y)
+ store i32 %lo, ptr %p0, align 4
+ store i32 %hi, ptr %p1, align 4
+ ret void
+}
+
+; Negative: constant operands are not XMM-native sources; stay on CMOV.
+define void @store_smin_load_const(ptr %p, ptr %r) nounwind {
+; SSE2-LABEL: store_smin_load_const:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %eax
+; SSE2-NEXT: cmpl $5, %eax
+; SSE2-NEXT: movl $5, %ecx
+; SSE2-NEXT: cmovll %eax, %ecx
+; SSE2-NEXT: movl %ecx, (%rsi)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: store_smin_load_const:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movl (%rdi), %eax
+; SSE41-NEXT: cmpl $5, %eax
+; SSE41-NEXT: movl $5, %ecx
+; SSE41-NEXT: cmovll %eax, %ecx
+; SSE41-NEXT: movl %ecx, (%rsi)
+; SSE41-NEXT: retq
+;
+; X86-SSE41-LABEL: store_smin_load_const:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl (%ecx), %ecx
+; X86-SSE41-NEXT: cmpl $5, %ecx
+; X86-SSE41-NEXT: movl $5, %edx
+; X86-SSE41-NEXT: cmovll %ecx, %edx
+; X86-SSE41-NEXT: movl %edx, (%eax)
+; X86-SSE41-NEXT: retl
+;
+; AVX-LABEL: store_smin_load_const:
+; AVX: # %bb.0:
+; AVX-NEXT: movl (%rdi), %eax
+; AVX-NEXT: cmpl $5, %eax
+; AVX-NEXT: movl $5, %ecx
+; AVX-NEXT: cmovll %eax, %ecx
+; AVX-NEXT: movl %ecx, (%rsi)
+; AVX-NEXT: retq
+;
+; AVX512F-LABEL: store_smin_load_const:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movl (%rdi), %eax
+; AVX512F-NEXT: cmpl $5, %eax
+; AVX512F-NEXT: movl $5, %ecx
+; AVX512F-NEXT: cmovll %eax, %ecx
+; AVX512F-NEXT: movl %ecx, (%rsi)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: store_smin_load_const:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movl (%rdi), %eax
+; AVX512VL-NEXT: cmpl $5, %eax
+; AVX512VL-NEXT: movl $5, %ecx
+; AVX512VL-NEXT: cmovll %eax, %ecx
+; AVX512VL-NEXT: movl %ecx, (%rsi)
+; AVX512VL-NEXT: retq
+;
+; AVX512FP16-LABEL: store_smin_load_const:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: movl (%rdi), %eax
+; AVX512FP16-NEXT: cmpl $5, %eax
+; AVX512FP16-NEXT: movl $5, %ecx
+; AVX512FP16-NEXT: cmovll %eax, %ecx
+; AVX512FP16-NEXT: movl %ecx, (%rsi)
+; AVX512FP16-NEXT: retq
+ %x = load i32, ptr %p, align 4
+ %m = call i32 @llvm.smin.i32(i32 %x, i32 5)
+ store i32 %m, ptr %r, align 4
+ ret void
+}
+
+; Memory-bound abs (#210569): prefer pabsd over neg+cmov.
+define void @store_abs_i32(ptr %p, ptr %r) nounwind {
+; SSE2-LABEL: store_abs_i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: negl %ecx
+; SSE2-NEXT: cmovsl %eax, %ecx
+; SSE2-NEXT: movl %ecx, (%rsi)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: store_abs_i32:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-NEXT: pabsd %xmm0, %xmm0
+; SSE41-NEXT: movd %xmm0, (%rsi)
+; SSE41-NEXT: retq
+;
+; X86-SSE41-LABEL: store_abs_i32:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT: pabsd %xmm0, %xmm0
+; X86-SSE41-NEXT: movd %xmm0, (%eax)
+; X86-SSE41-NEXT: retl
+;
+; AVX-LABEL: store_abs_i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT: vpabsd %xmm0, %xmm0
+; AVX-NEXT: vmovd %xmm0, (%rsi)
+; AVX-NEXT: retq
+;
+; AVX512F-LABEL: store_abs_i32:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512F-NEXT: vpabsd %xmm0, %xmm0
+; AVX512F-NEXT: vmovd %xmm0, (%rsi)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: store_abs_i32:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512VL-NEXT: vpabsd %xmm0, %xmm0
+; AVX512VL-NEXT: vmovd %xmm0, (%rsi)
+; AVX512VL-NEXT: retq
+;
+; AVX512FP16-LABEL: store_abs_i32:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512FP16-NEXT: vpabsd %xmm0, %xmm0
+; AVX512FP16-NEXT: vmovd %xmm0, (%rsi)
+; AVX512FP16-NEXT: retq
+ %x = load i32, ptr %p, align 4
+ %a = call i32 @llvm.abs.i32(i32 %x, i1 false)
+ store i32 %a, ptr %r, align 4
+ ret void
+}
+
+; Negative: register operand / GPR result should stay on neg+cmov.
+define i32 @abs_gpr(i32 %x) nounwind {
+; SSE2-LABEL: abs_gpr:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl %edi, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: cmovsl %edi, %eax
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: abs_gpr:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movl %edi, %eax
+; SSE41-NEXT: negl %eax
+; SSE41-NEXT: cmovsl %edi, %eax
+; SSE41-NEXT: retq
+;
+; X86-SSE41-LABEL: abs_gpr:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl %ecx, %eax
+; X86-SSE41-NEXT: negl %eax
+; X86-SSE41-NEXT: cmovsl %ecx, %eax
+; X86-SSE41-NEXT: retl
+;
+; AVX-LABEL: abs_gpr:
+; AVX: # %bb.0:
+; AVX-NEXT: movl %edi, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: cmovsl %edi, %eax
+; AVX-NEXT: retq
+;
+; AVX512F-LABEL: abs_gpr:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movl %edi, %eax
+; AVX512F-NEXT: negl %eax
+; AVX512F-NEXT: cmovsl %edi, %eax
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: abs_gpr:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movl %edi, %eax
+; AVX512VL-NEXT: negl %eax
+; AVX512VL-NEXT: cmovsl %edi, %eax
+; AVX512VL-NEXT: retq
+;
+; AVX512FP16-LABEL: abs_gpr:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: movl %edi, %eax
+; AVX512FP16-NEXT: negl %eax
+; AVX512FP16-NEXT: cmovsl %edi, %eax
+; AVX512FP16-NEXT: retq
+ %a = call i32 @llvm.abs.i32(i32 %x, i1 false)
+ ret i32 %a
+}
+
+define i32 @abs_load_to_gpr(ptr %p) nounwind {
+; SSE2-LABEL: abs_load_to_gpr:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %ecx
+; SSE2-NEXT: movl %ecx, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: cmovsl %ecx, %eax
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: abs_load_to_gpr:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movl (%rdi), %ecx
+; SSE41-NEXT: movl %ecx, %eax
+; SSE41-NEXT: negl %eax
+; SSE41-NEXT: cmovsl %ecx, %eax
+; SSE41-NEXT: retq
+;
+; X86-SSE41-LABEL: abs_load_to_gpr:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl (%eax), %ecx
+; X86-SSE41-NEXT: movl %ecx, %eax
+; X86-SSE41-NEXT: negl %eax
+; X86-SSE41-NEXT: cmovsl %ecx, %eax
+; X86-SSE41-NEXT: retl
+;
+; AVX-LABEL: abs_load_to_gpr:
+; AVX: # %bb.0:
+; AVX-NEXT: movl (%rdi), %ecx
+; AVX-NEXT: movl %ecx, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: cmovsl %ecx, %eax
+; AVX-NEXT: retq
+;
+; AVX512F-LABEL: abs_load_to_gpr:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movl (%rdi), %ecx
+; AVX512F-NEXT: movl %ecx, %eax
+; AVX512F-NEXT: negl %eax
+; AVX512F-NEXT: cmovsl %ecx, %eax
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: abs_load_to_gpr:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movl (%rdi), %ecx
+; AVX512VL-NEXT: movl %ecx, %eax
+; AVX512VL-NEXT: negl %eax
+; AVX512VL-NEXT: cmovsl %ecx, %eax
+; AVX512VL-NEXT: retq
+;
+; AVX512FP16-LABEL: abs_load_to_gpr:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: movl (%rdi), %ecx
+; AVX512FP16-NEXT: movl %ecx, %eax
+; AVX512FP16-NEXT: negl %eax
+; AVX512FP16-NEXT: cmovsl %ecx, %eax
+; AVX512FP16-NEXT: retq
+ %x = load i32, ptr %p, align 4
+ %a = call i32 @llvm.abs.i32(i32 %x, i1 false)
+ ret i32 %a
+}
+
+; Negative: noimplicitfloat must not introduce XMM ops for abs.
+define void @store_abs_i32_noimplicitfloat(ptr %p, ptr %r) nounwind noimplicitfloat {
+; SSE2-LABEL: store_abs_i32_noimplicitfloat:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: negl %ecx
+; SSE2-NEXT: cmovsl %eax, %ecx
+; SSE2-NEXT: movl %ecx, (%rsi)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: store_abs_i32_noimplicitfloat:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movl (%rdi), %eax
+; SSE41-NEXT: movl %eax, %ecx
+; SSE41-NEXT: negl %ecx
+; SSE41-NEXT: cmovsl %eax, %ecx
+; SSE41-NEXT: movl %ecx, (%rsi)
+; SSE41-NEXT: retq
+;
+; X86-SSE41-LABEL: store_abs_i32_noimplicitfloat:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl (%ecx), %ecx
+; X86-SSE41-NEXT: movl %ecx, %edx
+; X86-SSE41-NEXT: negl %edx
+; X86-SSE41-NEXT: cmovsl %ecx, %edx
+; X86-SSE41-NEXT: movl %edx, (%eax)
+; X86-SSE41-NEXT: retl
+;
+; AVX-LABEL: store_abs_i32_noimplicitfloat:
+; AVX: # %bb.0:
+; AVX-NEXT: movl (%rdi), %eax
+; AVX-NEXT: movl %eax, %ecx
+; AVX-NEXT: negl %ecx
+; AVX-NEXT: cmovsl %eax, %ecx
+; AVX-NEXT: movl %ecx, (%rsi)
+; AVX-NEXT: retq
+;
+; AVX512F-LABEL: store_abs_i32_noimplicitfloat:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movl (%rdi), %eax
+; AVX512F-NEXT: movl %eax, %ecx
+; AVX512F-NEXT: negl %ecx
+; AVX512F-NEXT: cmovsl %eax, %ecx
+; AVX512F-NEXT: movl %ecx, (%rsi)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: store_abs_i32_noimplicitfloat:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movl (%rdi), %eax
+; AVX512VL-NEXT: movl %eax, %ecx
+; AVX512VL-NEXT: negl %ecx
+; AVX512VL-NEXT: cmovsl %eax, %ecx
+; AVX512VL-NEXT: movl %ecx, (%rsi)
+; AVX512VL-NEXT: retq
+;
+; AVX512FP16-LABEL: store_abs_i32_noimplicitfloat:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: movl (%rdi), %eax
+; AVX512FP16-NEXT: movl %eax, %ecx
+; AVX512FP16-NEXT: negl %ecx
+; AVX512FP16-NEXT: cmovsl %eax, %ecx
+; AVX512FP16-NEXT: movl %ecx, (%rsi)
+; AVX512FP16-NEXT: retq
+ %x = load i32, ptr %p, align 4
+ %a = call i32 @llvm.abs.i32(i32 %x, i1 false)
+ store i32 %a, ptr %r, align 4
+ ret void
+}
+
+; Negative: optsize prefers compact neg+cmov over movd+pabsd.
+define void @store_abs_i32_optsize(ptr %p, ptr %r) nounwind optsize {
+; SSE2-LABEL: store_abs_i32_optsize:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movl (%rdi), %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: negl %ecx
+; SSE2-NEXT: cmovsl %eax, %ecx
+; SSE2-NEXT: movl %ecx, (%rsi)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: store_abs_i32_optsize:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movl (%rdi), %eax
+; SSE41-NEXT: movl %eax, %ecx
+; SSE41-NEXT: negl %ecx
+; SSE41-NEXT: cmovsl %eax, %ecx
+; SSE41-NEXT: movl %ecx, (%rsi)
+; SSE41-NEXT: retq
+;
+; X86-SSE41-LABEL: store_abs_i32_optsize:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl (%ecx), %ecx
+; X86-SSE41-NEXT: movl %ecx, %edx
+; X86-SSE41-NEXT: negl %edx
+; X86-SSE41-NEXT: cmovsl %ecx, %edx
+; X86-SSE41-NEXT: movl %edx, (%eax)
+; X86-SSE41-NEXT: retl
+;
+; AVX-LABEL: store_abs_i32_optsize:
+; AVX: # %bb.0:
+; AVX-NEXT: movl (%rdi), %eax
+; AVX-NEXT: movl %eax, %ecx
+; AVX-NEXT: negl %ecx
+; AVX-NEXT: cmovsl %eax, %ecx
+; AVX-NEXT: movl %ecx, (%rsi)
+; AVX-NEXT: retq
+;
+; AVX512F-LABEL: store_abs_i32_optsize:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movl (%rdi), %eax
+; AVX512F-NEXT: movl %eax, %ecx
+; AVX512F-NEXT: negl %ecx
+; AVX512F-NEXT: cmovsl %eax, %ecx
+; AVX512F-NEXT: movl %ecx, (%rsi)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: store_abs_i32_optsize:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movl (%rdi), %eax
+; AVX512VL-NEXT: movl %eax, %ecx
+; AVX512VL-NEXT: negl %ecx
+; AVX512VL-NEXT: cmovsl %eax, %ecx
+; AVX512VL-NEXT: movl %ecx, (%rsi)
+; AVX512VL-NEXT: retq
+;
+; AVX512FP16-LABEL: store_abs_i32_optsize:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: movl (%rdi), %eax
+; AVX512FP16-NEXT: movl %eax, %ecx
+; AVX512FP16-NEXT: negl %ecx
+; AVX512FP16-NEXT: cmovsl %eax, %ecx
+; AVX512FP16-NEXT: movl %ecx, (%rsi)
+; AVX512FP16-NEXT: retq
+ %x = load i32, ptr %p, align 4
+ %a = call i32 @llvm.abs.i32(i32 %x, i1 false)
+ store i32 %a, ptr %r, align 4
+ ret void
+}
+
+; i64 needs AVX512F+VLX (movq + vpminsq/vpmaxsq/vpabsq).
+define void @sort2_i64(ptr %a) nounwind {
+; SSE2-LABEL: sort2_i64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movq (%rdi), %rax
+; SSE2-NEXT: movq 8(%rdi), %rcx
+; SSE2-NEXT: cmpq %rcx, %rax
+; SSE2-NEXT: movq %rcx, %rdx
+; SSE2-NEXT: cmovlq %rax, %rdx
+; SSE2-NEXT: cmovgq %rax, %rcx
+; SSE2-NEXT: movq %rdx, (%rdi)
+; SSE2-NEXT: movq %rcx, 8(%rdi)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: sort2_i64:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movq (%rdi), %rax
+; SSE41-NEXT: movq 8(%rdi), %rcx
+; SSE41-NEXT: cmpq %rcx, %rax
+; SSE41-NEXT: movq %rcx, %rdx
+; SSE41-NEXT: cmovlq %rax, %rdx
+; SSE41-NEXT: cmovgq %rax, %rcx
+; SSE41-NEXT: movq %rdx, (%rdi)
+; SSE41-NEXT: movq %rcx, 8(%rdi)
+; SSE41-NEXT: retq
+;
+; X86-SSE41-LABEL: sort2_i64:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: pushl %ebp
+; X86-SSE41-NEXT: pushl %ebx
+; X86-SSE41-NEXT: pushl %edi
+; X86-SSE41-NEXT: pushl %esi
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl (%ecx), %edx
+; X86-SSE41-NEXT: movl 4(%ecx), %edi
+; X86-SSE41-NEXT: movl 12(%ecx), %eax
+; X86-SSE41-NEXT: movl 8(%ecx), %esi
+; X86-SSE41-NEXT: cmpl %esi, %edx
+; X86-SSE41-NEXT: movl %edi, %ebx
+; X86-SSE41-NEXT: sbbl %eax, %ebx
+; X86-SSE41-NEXT: movl %eax, %ebx
+; X86-SSE41-NEXT: cmovll %edi, %ebx
+; X86-SSE41-NEXT: movl %esi, %ebp
+; X86-SSE41-NEXT: cmovll %edx, %ebp
+; X86-SSE41-NEXT: cmpl %edx, %esi
+; X86-SSE41-NEXT: movl %eax, %ecx
+; X86-SSE41-NEXT: sbbl %edi, %ecx
+; X86-SSE41-NEXT: cmovll %edi, %eax
+; X86-SSE41-NEXT: cmovll %edx, %esi
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl %ebp, (%ecx)
+; X86-SSE41-NEXT: movl %ebx, 4(%ecx)
+; X86-SSE41-NEXT: movl %esi, 8(%ecx)
+; X86-SSE41-NEXT: movl %eax, 12(%ecx)
+; X86-SSE41-NEXT: popl %esi
+; X86-SSE41-NEXT: popl %edi
+; X86-SSE41-NEXT: popl %ebx
+; X86-SSE41-NEXT: popl %ebp
+; X86-SSE41-NEXT: retl
+;
+; AVX-LABEL: sort2_i64:
+; AVX: # %bb.0:
+; AVX-NEXT: movq (%rdi), %rax
+; AVX-NEXT: movq 8(%rdi), %rcx
+; AVX-NEXT: cmpq %rcx, %rax
+; AVX-NEXT: movq %rcx, %rdx
+; AVX-NEXT: cmovlq %rax, %rdx
+; AVX-NEXT: cmovgq %rax, %rcx
+; AVX-NEXT: movq %rdx, (%rdi)
+; AVX-NEXT: movq %rcx, 8(%rdi)
+; AVX-NEXT: retq
+;
+; AVX512F-LABEL: sort2_i64:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX512F-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX512F-NEXT: vpminsq %zmm0, %zmm1, %zmm2
+; AVX512F-NEXT: vpmaxsq %zmm0, %zmm1, %zmm0
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX512F-NEXT: vmovdqu %xmm0, (%rdi)
+; AVX512F-NEXT: vzeroupper
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: sort2_i64:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX512VL-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX512VL-NEXT: vpminsq %xmm0, %xmm1, %xmm2
+; AVX512VL-NEXT: vpmaxsq %xmm0, %xmm1, %xmm0
+; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX512VL-NEXT: vmovdqu %xmm0, (%rdi)
+; AVX512VL-NEXT: retq
+;
+; AVX512FP16-LABEL: sort2_i64:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX512FP16-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX512FP16-NEXT: vpminsq %zmm0, %zmm1, %zmm2
+; AVX512FP16-NEXT: vpmaxsq %zmm0, %zmm1, %zmm0
+; AVX512FP16-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX512FP16-NEXT: vmovdqu %xmm0, (%rdi)
+; AVX512FP16-NEXT: vzeroupper
+; AVX512FP16-NEXT: retq
+ %p0 = getelementptr inbounds i64, ptr %a, i64 0
+ %p1 = getelementptr inbounds i64, ptr %a, i64 1
+ %x = load i64, ptr %p0, align 8
+ %y = load i64, ptr %p1, align 8
+ %lo = call i64 @llvm.smin.i64(i64 %x, i64 %y)
+ %hi = call i64 @llvm.smax.i64(i64 %x, i64 %y)
+ store i64 %lo, ptr %p0, align 8
+ store i64 %hi, ptr %p1, align 8
+ ret void
+}
+
+define void @store_abs_i64(ptr %p, ptr %r) nounwind {
+; SSE2-LABEL: store_abs_i64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movq (%rdi), %rax
+; SSE2-NEXT: movq %rax, %rcx
+; SSE2-NEXT: negq %rcx
+; SSE2-NEXT: cmovsq %rax, %rcx
+; SSE2-NEXT: movq %rcx, (%rsi)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: store_abs_i64:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movq (%rdi), %rax
+; SSE41-NEXT: movq %rax, %rcx
+; SSE41-NEXT: negq %rcx
+; SSE41-NEXT: cmovsq %rax, %rcx
+; SSE41-NEXT: movq %rcx, (%rsi)
+; SSE41-NEXT: retq
+;
+; X86-SSE41-LABEL: store_abs_i64:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: pushl %esi
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl 4(%ecx), %edx
+; X86-SSE41-NEXT: movl %edx, %esi
+; X86-SSE41-NEXT: sarl $31, %esi
+; X86-SSE41-NEXT: xorl %esi, %edx
+; X86-SSE41-NEXT: movl (%ecx), %ecx
+; X86-SSE41-NEXT: xorl %esi, %ecx
+; X86-SSE41-NEXT: subl %esi, %ecx
+; X86-SSE41-NEXT: sbbl %esi, %edx
+; X86-SSE41-NEXT: movl %ecx, (%eax)
+; X86-SSE41-NEXT: movl %edx, 4(%eax)
+; X86-SSE41-NEXT: popl %esi
+; X86-SSE41-NEXT: retl
+;
+; AVX-LABEL: store_abs_i64:
+; AVX: # %bb.0:
+; AVX-NEXT: movq (%rdi), %rax
+; AVX-NEXT: movq %rax, %rcx
+; AVX-NEXT: negq %rcx
+; AVX-NEXT: cmovsq %rax, %rcx
+; AVX-NEXT: movq %rcx, (%rsi)
+; AVX-NEXT: retq
+;
+; AVX512F-LABEL: store_abs_i64:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX512F-NEXT: vpabsq %zmm0, %zmm0
+; AVX512F-NEXT: vmovq %xmm0, (%rsi)
+; AVX512F-NEXT: vzeroupper
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: store_abs_i64:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX512VL-NEXT: vpabsq %xmm0, %xmm0
+; AVX512VL-NEXT: vmovq %xmm0, (%rsi)
+; AVX512VL-NEXT: retq
+;
+; AVX512FP16-LABEL: store_abs_i64:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX512FP16-NEXT: vpabsq %zmm0, %zmm0
+; AVX512FP16-NEXT: vmovq %xmm0, (%rsi)
+; AVX512FP16-NEXT: vzeroupper
+; AVX512FP16-NEXT: retq
+ %x = load i64, ptr %p, align 8
+ %a = call i64 @llvm.abs.i64(i64 %x, i1 false)
+ store i64 %a, ptr %r, align 8
+ ret void
+}
+
+; Negative: register / GPR-result i64 stays on cmp+cmov.
+define i64 @smin_gpr_i64(i64 %x, i64 %y) nounwind {
+; SSE2-LABEL: smin_gpr_i64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movq %rsi, %rax
+; SSE2-NEXT: cmpq %rsi, %rdi
+; SSE2-NEXT: cmovlq %rdi, %rax
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: smin_gpr_i64:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movq %rsi, %rax
+; SSE41-NEXT: cmpq %rsi, %rdi
+; SSE41-NEXT: cmovlq %rdi, %rax
+; SSE41-NEXT: retq
+;
+; X86-SSE41-LABEL: smin_gpr_i64:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: pushl %edi
+; X86-SSE41-NEXT: pushl %esi
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-SSE41-NEXT: cmpl %eax, %ecx
+; X86-SSE41-NEXT: movl %esi, %edi
+; X86-SSE41-NEXT: sbbl %edx, %edi
+; X86-SSE41-NEXT: cmovll %ecx, %eax
+; X86-SSE41-NEXT: cmovll %esi, %edx
+; X86-SSE41-NEXT: popl %esi
+; X86-SSE41-NEXT: popl %edi
+; X86-SSE41-NEXT: retl
+;
+; AVX-LABEL: smin_gpr_i64:
+; AVX: # %bb.0:
+; AVX-NEXT: movq %rsi, %rax
+; AVX-NEXT: cmpq %rsi, %rdi
+; AVX-NEXT: cmovlq %rdi, %rax
+; AVX-NEXT: retq
+;
+; AVX512F-LABEL: smin_gpr_i64:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movq %rsi, %rax
+; AVX512F-NEXT: cmpq %rsi, %rdi
+; AVX512F-NEXT: cmovlq %rdi, %rax
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: smin_gpr_i64:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq %rsi, %rax
+; AVX512VL-NEXT: cmpq %rsi, %rdi
+; AVX512VL-NEXT: cmovlq %rdi, %rax
+; AVX512VL-NEXT: retq
+;
+; AVX512FP16-LABEL: smin_gpr_i64:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: movq %rsi, %rax
+; AVX512FP16-NEXT: cmpq %rsi, %rdi
+; AVX512FP16-NEXT: cmovlq %rdi, %rax
+; AVX512FP16-NEXT: retq
+ %m = call i64 @llvm.smin.i64(i64 %x, i64 %y)
+ ret i64 %m
+}
+
+define void @sort2_i64_optsize(ptr %a) nounwind optsize {
+; SSE2-LABEL: sort2_i64_optsize:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movq (%rdi), %rax
+; SSE2-NEXT: movq 8(%rdi), %rcx
+; SSE2-NEXT: cmpq %rcx, %rax
+; SSE2-NEXT: movq %rcx, %rdx
+; SSE2-NEXT: cmovlq %rax, %rdx
+; SSE2-NEXT: cmovgq %rax, %rcx
+; SSE2-NEXT: movq %rdx, (%rdi)
+; SSE2-NEXT: movq %rcx, 8(%rdi)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: sort2_i64_optsize:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movq (%rdi), %rax
+; SSE41-NEXT: movq 8(%rdi), %rcx
+; SSE41-NEXT: cmpq %rcx, %rax
+; SSE41-NEXT: movq %rcx, %rdx
+; SSE41-NEXT: cmovlq %rax, %rdx
+; SSE41-NEXT: cmovgq %rax, %rcx
+; SSE41-NEXT: movq %rdx, (%rdi)
+; SSE41-NEXT: movq %rcx, 8(%rdi)
+; SSE41-NEXT: retq
+;
+; X86-SSE41-LABEL: sort2_i64_optsize:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: pushl %ebp
+; X86-SSE41-NEXT: pushl %ebx
+; X86-SSE41-NEXT: pushl %edi
+; X86-SSE41-NEXT: pushl %esi
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl (%ecx), %edx
+; X86-SSE41-NEXT: movl 4(%ecx), %edi
+; X86-SSE41-NEXT: movl 12(%ecx), %eax
+; X86-SSE41-NEXT: movl 8(%ecx), %esi
+; X86-SSE41-NEXT: cmpl %esi, %edx
+; X86-SSE41-NEXT: movl %edi, %ebx
+; X86-SSE41-NEXT: sbbl %eax, %ebx
+; X86-SSE41-NEXT: movl %eax, %ebx
+; X86-SSE41-NEXT: cmovll %edi, %ebx
+; X86-SSE41-NEXT: movl %esi, %ebp
+; X86-SSE41-NEXT: cmovll %edx, %ebp
+; X86-SSE41-NEXT: cmpl %edx, %esi
+; X86-SSE41-NEXT: movl %eax, %ecx
+; X86-SSE41-NEXT: sbbl %edi, %ecx
+; X86-SSE41-NEXT: cmovll %edi, %eax
+; X86-SSE41-NEXT: cmovll %edx, %esi
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl %ebp, (%ecx)
+; X86-SSE41-NEXT: movl %ebx, 4(%ecx)
+; X86-SSE41-NEXT: movl %esi, 8(%ecx)
+; X86-SSE41-NEXT: movl %eax, 12(%ecx)
+; X86-SSE41-NEXT: popl %esi
+; X86-SSE41-NEXT: popl %edi
+; X86-SSE41-NEXT: popl %ebx
+; X86-SSE41-NEXT: popl %ebp
+; X86-SSE41-NEXT: retl
+;
+; AVX-LABEL: sort2_i64_optsize:
+; AVX: # %bb.0:
+; AVX-NEXT: movq (%rdi), %rax
+; AVX-NEXT: movq 8(%rdi), %rcx
+; AVX-NEXT: cmpq %rcx, %rax
+; AVX-NEXT: movq %rcx, %rdx
+; AVX-NEXT: cmovlq %rax, %rdx
+; AVX-NEXT: cmovgq %rax, %rcx
+; AVX-NEXT: movq %rdx, (%rdi)
+; AVX-NEXT: movq %rcx, 8(%rdi)
+; AVX-NEXT: retq
+;
+; AVX512F-LABEL: sort2_i64_optsize:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movq (%rdi), %rax
+; AVX512F-NEXT: movq 8(%rdi), %rcx
+; AVX512F-NEXT: cmpq %rcx, %rax
+; AVX512F-NEXT: movq %rcx, %rdx
+; AVX512F-NEXT: cmovlq %rax, %rdx
+; AVX512F-NEXT: cmovgq %rax, %rcx
+; AVX512F-NEXT: movq %rdx, (%rdi)
+; AVX512F-NEXT: movq %rcx, 8(%rdi)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: sort2_i64_optsize:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq (%rdi), %rax
+; AVX512VL-NEXT: movq 8(%rdi), %rcx
+; AVX512VL-NEXT: cmpq %rcx, %rax
+; AVX512VL-NEXT: movq %rcx, %rdx
+; AVX512VL-NEXT: cmovlq %rax, %rdx
+; AVX512VL-NEXT: cmovgq %rax, %rcx
+; AVX512VL-NEXT: movq %rdx, (%rdi)
+; AVX512VL-NEXT: movq %rcx, 8(%rdi)
+; AVX512VL-NEXT: retq
+;
+; AVX512FP16-LABEL: sort2_i64_optsize:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: movq (%rdi), %rax
+; AVX512FP16-NEXT: movq 8(%rdi), %rcx
+; AVX512FP16-NEXT: cmpq %rcx, %rax
+; AVX512FP16-NEXT: movq %rcx, %rdx
+; AVX512FP16-NEXT: cmovlq %rax, %rdx
+; AVX512FP16-NEXT: cmovgq %rax, %rcx
+; AVX512FP16-NEXT: movq %rdx, (%rdi)
+; AVX512FP16-NEXT: movq %rcx, 8(%rdi)
+; AVX512FP16-NEXT: retq
+ %p0 = getelementptr inbounds i64, ptr %a, i64 0
+ %p1 = getelementptr inbounds i64, ptr %a, i64 1
+ %x = load i64, ptr %p0, align 8
+ %y = load i64, ptr %p1, align 8
+ %lo = call i64 @llvm.smin.i64(i64 %x, i64 %y)
+ %hi = call i64 @llvm.smax.i64(i64 %x, i64 %y)
+ store i64 %lo, ptr %p0, align 8
+ store i64 %hi, ptr %p1, align 8
+ ret void
+}
+
+define void @sort2_i64_noimplicitfloat(ptr %a) nounwind noimplicitfloat {
+; SSE2-LABEL: sort2_i64_noimplicitfloat:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movq (%rdi), %rax
+; SSE2-NEXT: movq 8(%rdi), %rcx
+; SSE2-NEXT: cmpq %rcx, %rax
+; SSE2-NEXT: movq %rcx, %rdx
+; SSE2-NEXT: cmovlq %rax, %rdx
+; SSE2-NEXT: cmovgq %rax, %rcx
+; SSE2-NEXT: movq %rdx, (%rdi)
+; SSE2-NEXT: movq %rcx, 8(%rdi)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: sort2_i64_noimplicitfloat:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movq (%rdi), %rax
+; SSE41-NEXT: movq 8(%rdi), %rcx
+; SSE41-NEXT: cmpq %rcx, %rax
+; SSE41-NEXT: movq %rcx, %rdx
+; SSE41-NEXT: cmovlq %rax, %rdx
+; SSE41-NEXT: cmovgq %rax, %rcx
+; SSE41-NEXT: movq %rdx, (%rdi)
+; SSE41-NEXT: movq %rcx, 8(%rdi)
+; SSE41-NEXT: retq
+;
+; X86-SSE41-LABEL: sort2_i64_noimplicitfloat:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: pushl %ebp
+; X86-SSE41-NEXT: pushl %ebx
+; X86-SSE41-NEXT: pushl %edi
+; X86-SSE41-NEXT: pushl %esi
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl (%ecx), %edx
+; X86-SSE41-NEXT: movl 4(%ecx), %edi
+; X86-SSE41-NEXT: movl 12(%ecx), %eax
+; X86-SSE41-NEXT: movl 8(%ecx), %esi
+; X86-SSE41-NEXT: cmpl %esi, %edx
+; X86-SSE41-NEXT: movl %edi, %ebx
+; X86-SSE41-NEXT: sbbl %eax, %ebx
+; X86-SSE41-NEXT: movl %eax, %ebx
+; X86-SSE41-NEXT: cmovll %edi, %ebx
+; X86-SSE41-NEXT: movl %esi, %ebp
+; X86-SSE41-NEXT: cmovll %edx, %ebp
+; X86-SSE41-NEXT: cmpl %edx, %esi
+; X86-SSE41-NEXT: movl %eax, %ecx
+; X86-SSE41-NEXT: sbbl %edi, %ecx
+; X86-SSE41-NEXT: cmovll %edi, %eax
+; X86-SSE41-NEXT: cmovll %edx, %esi
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movl %ebp, (%ecx)
+; X86-SSE41-NEXT: movl %ebx, 4(%ecx)
+; X86-SSE41-NEXT: movl %esi, 8(%ecx)
+; X86-SSE41-NEXT: movl %eax, 12(%ecx)
+; X86-SSE41-NEXT: popl %esi
+; X86-SSE41-NEXT: popl %edi
+; X86-SSE41-NEXT: popl %ebx
+; X86-SSE41-NEXT: popl %ebp
+; X86-SSE41-NEXT: retl
+;
+; AVX-LABEL: sort2_i64_noimplicitfloat:
+; AVX: # %bb.0:
+; AVX-NEXT: movq (%rdi), %rax
+; AVX-NEXT: movq 8(%rdi), %rcx
+; AVX-NEXT: cmpq %rcx, %rax
+; AVX-NEXT: movq %rcx, %rdx
+; AVX-NEXT: cmovlq %rax, %rdx
+; AVX-NEXT: cmovgq %rax, %rcx
+; AVX-NEXT: movq %rdx, (%rdi)
+; AVX-NEXT: movq %rcx, 8(%rdi)
+; AVX-NEXT: retq
+;
+; AVX512F-LABEL: sort2_i64_noimplicitfloat:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movq (%rdi), %rax
+; AVX512F-NEXT: movq 8(%rdi), %rcx
+; AVX512F-NEXT: cmpq %rcx, %rax
+; AVX512F-NEXT: movq %rcx, %rdx
+; AVX512F-NEXT: cmovlq %rax, %rdx
+; AVX512F-NEXT: cmovgq %rax, %rcx
+; AVX512F-NEXT: movq %rdx, (%rdi)
+; AVX512F-NEXT: movq %rcx, 8(%rdi)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: sort2_i64_noimplicitfloat:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq (%rdi), %rax
+; AVX512VL-NEXT: movq 8(%rdi), %rcx
+; AVX512VL-NEXT: cmpq %rcx, %rax
+; AVX512VL-NEXT: movq %rcx, %rdx
+; AVX512VL-NEXT: cmovlq %rax, %rdx
+; AVX512VL-NEXT: cmovgq %rax, %rcx
+; AVX512VL-NEXT: movq %rdx, (%rdi)
+; AVX512VL-NEXT: movq %rcx, 8(%rdi)
+; AVX512VL-NEXT: retq
+;
+; AVX512FP16-LABEL: sort2_i64_noimplicitfloat:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: movq (%rdi), %rax
+; AVX512FP16-NEXT: movq 8(%rdi), %rcx
+; AVX512FP16-NEXT: cmpq %rcx, %rax
+; AVX512FP16-NEXT: movq %rcx, %rdx
+; AVX512FP16-NEXT: cmovlq %rax, %rdx
+; AVX512FP16-NEXT: cmovgq %rax, %rcx
+; AVX512FP16-NEXT: movq %rdx, (%rdi)
+; AVX512FP16-NEXT: movq %rcx, 8(%rdi)
+; AVX512FP16-NEXT: retq
+ %p0 = getelementptr inbounds i64, ptr %a, i64 0
+ %p1 = getelementptr inbounds i64, ptr %a, i64 1
+ %x = load i64, ptr %p0, align 8
+ %y = load i64, ptr %p1, align 8
+ %lo = call i64 @llvm.smin.i64(i64 %x, i64 %y)
+ %hi = call i64 @llvm.smax.i64(i64 %x, i64 %y)
+ store i64 %lo, ptr %p0, align 8
+ store i64 %hi, ptr %p1, align 8
+ ret void
+}
+
+; i16 needs AVX512FP16 (vmovw + vpminsw/vpmaxsw/vpabsw).
+define void @sort2_i16(ptr %a) nounwind {
+; SSE2-LABEL: sort2_i16:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movzwl (%rdi), %eax
+; SSE2-NEXT: movzwl 2(%rdi), %ecx
+; SSE2-NEXT: cmpw %cx, %ax
+; SSE2-NEXT: movl %ecx, %edx
+; SSE2-NEXT: cmovll %eax, %edx
+; SSE2-NEXT: cmovgl %eax, %ecx
+; SSE2-NEXT: movw %dx, (%rdi)
+; SSE2-NEXT: movw %cx, 2(%rdi)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: sort2_i16:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movzwl (%rdi), %eax
+; SSE41-NEXT: movzwl 2(%rdi), %ecx
+; SSE41-NEXT: cmpw %cx, %ax
+; SSE41-NEXT: movl %ecx, %edx
+; SSE41-NEXT: cmovll %eax, %edx
+; SSE41-NEXT: cmovgl %eax, %ecx
+; SSE41-NEXT: movw %dx, (%rdi)
+; SSE41-NEXT: movw %cx, 2(%rdi)
+; SSE41-NEXT: retq
+;
+; X86-SSE41-LABEL: sort2_i16:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: pushl %esi
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movzwl (%eax), %ecx
+; X86-SSE41-NEXT: movzwl 2(%eax), %edx
+; X86-SSE41-NEXT: cmpw %dx, %cx
+; X86-SSE41-NEXT: movl %edx, %esi
+; X86-SSE41-NEXT: cmovll %ecx, %esi
+; X86-SSE41-NEXT: cmovgl %ecx, %edx
+; X86-SSE41-NEXT: movw %si, (%eax)
+; X86-SSE41-NEXT: movw %dx, 2(%eax)
+; X86-SSE41-NEXT: popl %esi
+; X86-SSE41-NEXT: retl
+;
+; AVX-LABEL: sort2_i16:
+; AVX: # %bb.0:
+; AVX-NEXT: movzwl (%rdi), %eax
+; AVX-NEXT: movzwl 2(%rdi), %ecx
+; AVX-NEXT: cmpw %cx, %ax
+; AVX-NEXT: movl %ecx, %edx
+; AVX-NEXT: cmovll %eax, %edx
+; AVX-NEXT: cmovgl %eax, %ecx
+; AVX-NEXT: movw %dx, (%rdi)
+; AVX-NEXT: movw %cx, 2(%rdi)
+; AVX-NEXT: retq
+;
+; AVX512F-LABEL: sort2_i16:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movzwl (%rdi), %eax
+; AVX512F-NEXT: movzwl 2(%rdi), %ecx
+; AVX512F-NEXT: cmpw %cx, %ax
+; AVX512F-NEXT: movl %ecx, %edx
+; AVX512F-NEXT: cmovll %eax, %edx
+; AVX512F-NEXT: cmovgl %eax, %ecx
+; AVX512F-NEXT: movw %dx, (%rdi)
+; AVX512F-NEXT: movw %cx, 2(%rdi)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: sort2_i16:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movzwl (%rdi), %eax
+; AVX512VL-NEXT: movzwl 2(%rdi), %ecx
+; AVX512VL-NEXT: cmpw %cx, %ax
+; AVX512VL-NEXT: movl %ecx, %edx
+; AVX512VL-NEXT: cmovll %eax, %edx
+; AVX512VL-NEXT: cmovgl %eax, %ecx
+; AVX512VL-NEXT: movw %dx, (%rdi)
+; AVX512VL-NEXT: movw %cx, 2(%rdi)
+; AVX512VL-NEXT: retq
+;
+; AVX512FP16-LABEL: sort2_i16:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: vmovw 2(%rdi), %xmm0
+; AVX512FP16-NEXT: vmovw (%rdi), %xmm1
+; AVX512FP16-NEXT: vpminsw %xmm0, %xmm1, %xmm2
+; AVX512FP16-NEXT: vpextrw $0, %xmm2, (%rdi)
+; AVX512FP16-NEXT: vpmaxsw %xmm0, %xmm1, %xmm0
+; AVX512FP16-NEXT: vpextrw $0, %xmm0, 2(%rdi)
+; AVX512FP16-NEXT: retq
+ %p0 = getelementptr inbounds i16, ptr %a, i64 0
+ %p1 = getelementptr inbounds i16, ptr %a, i64 1
+ %x = load i16, ptr %p0, align 2
+ %y = load i16, ptr %p1, align 2
+ %lo = call i16 @llvm.smin.i16(i16 %x, i16 %y)
+ %hi = call i16 @llvm.smax.i16(i16 %x, i16 %y)
+ store i16 %lo, ptr %p0, align 2
+ store i16 %hi, ptr %p1, align 2
+ ret void
+}
+
+define void @store_abs_i16(ptr %p, ptr %r) nounwind {
+; SSE2-LABEL: store_abs_i16:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movzwl (%rdi), %eax
+; SSE2-NEXT: movl %eax, %ecx
+; SSE2-NEXT: negw %cx
+; SSE2-NEXT: cmovsw %ax, %cx
+; SSE2-NEXT: movw %cx, (%rsi)
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: store_abs_i16:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movzwl (%rdi), %eax
+; SSE41-NEXT: movl %eax, %ecx
+; SSE41-NEXT: negw %cx
+; SSE41-NEXT: cmovsw %ax, %cx
+; SSE41-NEXT: movw %cx, (%rsi)
+; SSE41-NEXT: retq
+;
+; X86-SSE41-LABEL: store_abs_i16:
+; X86-SSE41: # %bb.0:
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT: movzwl (%ecx), %ecx
+; X86-SSE41-NEXT: movl %ecx, %edx
+; X86-SSE41-NEXT: negw %dx
+; X86-SSE41-NEXT: cmovsw %cx, %dx
+; X86-SSE41-NEXT: movw %dx, (%eax)
+; X86-SSE41-NEXT: retl
+;
+; AVX-LABEL: store_abs_i16:
+; AVX: # %bb.0:
+; AVX-NEXT: movzwl (%rdi), %eax
+; AVX-NEXT: movl %eax, %ecx
+; AVX-NEXT: negw %cx
+; AVX-NEXT: cmovsw %ax, %cx
+; AVX-NEXT: movw %cx, (%rsi)
+; AVX-NEXT: retq
+;
+; AVX512F-LABEL: store_abs_i16:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movzwl (%rdi), %eax
+; AVX512F-NEXT: movl %eax, %ecx
+; AVX512F-NEXT: negw %cx
+; AVX512F-NEXT: cmovsw %ax, %cx
+; AVX512F-NEXT: movw %cx, (%rsi)
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: store_abs_i16:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movzwl (%rdi), %eax
+; AVX512VL-NEXT: movl %eax, %ecx
+; AVX512VL-NEXT: negw %cx
+; AVX512VL-NEXT: cmovsw %ax, %cx
+; AVX512VL-NEXT: movw %cx, (%rsi)
+; AVX512VL-NEXT: retq
+;
+; AVX512FP16-LABEL: store_abs_i16:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: vmovw (%rdi), %xmm0
+; AVX512FP16-NEXT: vpabsw %xmm0, %xmm0
+; AVX512FP16-NEXT: vpextrw $0, %xmm0, (%rsi)
+; AVX512FP16-NEXT: retq
+ %x = load i16, ptr %p, align 2
+ %a = call i16 @llvm.abs.i16(i16 %x, i1 false)
+ store i16 %a, ptr %r, align 2
+ ret void
+}
More information about the llvm-commits
mailing list