[llvm] [X86] Prefer SIMD min/max/abs for scalars when staying in XMM domain (PR #210654)

[Simon Pilgrim via llvm-commits]

================
@@ -54649,6 +54649,143 @@ static SDValue narrowBitOpRMW(StoreSDNode *St, const SDLoc &DL,
   return NewStore;
 }
 
+/// True if V is a plain (non-extending) load of VT. Don't peek through
+/// bitcasts: that could pull a float-domain value into an integer vector.
+static bool isPlainScalarLoad(SDValue V, EVT VT) {
+  auto *Ld = dyn_cast<LoadSDNode>(V);
+  if (!Ld || Ld->getExtensionType() != ISD::NON_EXTLOAD)
+    return false;
+  return Ld->getMemoryVT() == VT;
+}
+
+/// Return the widened vector type to use for a scalar min/max/abs SIMD
+/// combine of VT, or std::nullopt if VT/subtarget isn't supported.
+/// i32 needs SSE4.1 (min/max) / SSSE3 (abs); i64 needs AVX512F+VLX (movq
+/// covers the mem->XMM move; VLX avoids zmm widening + vzeroupper); i16 needs
+/// AVX512FP16, since VMOVW is the only instruction that folds a 16-bit scalar
+/// load directly into XMM without a GPR round trip.
+static std::optional<MVT>
+getScalarSIMDVecVT(EVT VT, const X86Subtarget &Subtarget, bool IsAbs) {
+  if (!VT.isSimple())
+    return std::nullopt;
+  switch (VT.getSimpleVT().SimpleTy) {
+  case MVT::i32:
+    if (IsAbs ? Subtarget.hasSSSE3() : Subtarget.hasSSE41())
+      return MVT::v4i32;
+    break;
+  case MVT::i64:
+    // Without VLX, isel widens v2i64 min/max/abs to zmm and emits
+    // vzeroupper — worse than cmp+cmov for a scalar memory op.
+    if (Subtarget.hasAVX512() && Subtarget.hasVLX())
+      return MVT::v2i64;
+    break;
+  case MVT::i16:
+    if (Subtarget.hasFP16())
+      return MVT::v8i16;
+    break;
+  default:
+    break;
+  }
+  return std::nullopt;
+}
+
+/// Fold store(abs/min/max(load…)) of scalar i16/i32/i64 into a vector
+/// abs/min/max, which ISel folds to movd/movq/vmovw + PABS/PMIN/PMAX. This
+/// keeps memory-bound sort2 in the XMM domain, matching GCC, without
+/// introducing GPR↔XMM domain crosses for values that live in GPRs.
+///
+/// Only plain loads are accepted as sources: EXTRACT_VECTOR_ELT typically
+/// materializes in a GPR (movd/pextr), so re-widening would create a domain
+/// cross. Constants and GPR operands are likewise rejected. Relies on scalar
+/// min/max remaining Expand so shouldScalarizeBinop does not undo the
+/// extract(vector_binop) form we create. Store-only by design — other sinks
+/// (e.g. insert_vector_elt) belong in mid-end vectorization, not the backend.
+static SDValue combineScalarMinMaxAbsStore(StoreSDNode *St, const SDLoc &DL,
+                                           SelectionDAG &DAG,
+                                           const X86Subtarget &Subtarget) {
+  if (!ISD::isNormalStore(St) || St->isTruncatingStore())
+    return SDValue();
+
+  SDValue StoredVal = St->getValue();
+  if (!StoredVal.hasOneUse())
+    return SDValue();
+
+  unsigned Opc = StoredVal.getOpcode();
+  bool IsAbs = Opc == ISD::ABS;
+  bool IsMinMax = Opc == ISD::SMIN || Opc == ISD::SMAX || Opc == ISD::UMIN ||
+                  Opc == ISD::UMAX;
+  if (!IsAbs && !IsMinMax)
+    return SDValue();
+
+  EVT VT = StoredVal.getValueType();
+  std::optional<MVT> VecVTOpt = getScalarSIMDVecVT(VT, Subtarget, IsAbs);
+  if (!VecVTOpt)
+    return SDValue();
+  MVT VecVT = *VecVTOpt;
+
+  const Function &F = DAG.getMachineFunction().getFunction();
+  if (F.hasFnAttribute(Attribute::NoImplicitFloat) || Subtarget.useSoftFloat())
+    return SDValue();
+  if (F.hasOptSize())
+    return SDValue();
+
+  if (IsAbs) {
+    SDValue Op0 = StoredVal.getOperand(0);
+    if (!isPlainScalarLoad(Op0, VT) || !Op0.hasOneUse())
+      return SDValue();
+
+    SDValue V0 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op0);
+    SDValue Vec = DAG.getNode(ISD::ABS, DL, VecVT, V0);
+    SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Vec,
+                                 DAG.getVectorIdxConstant(0, DL));
+    return DAG.getStore(St->getChain(), DL, NewVal, St->getBasePtr(),
+                        St->getPointerInfo(), St->getBaseAlign(),
+                        St->getMemOperand()->getFlags());
+  }
+
+  SDValue Op0 = StoredVal.getOperand(0);
+  SDValue Op1 = StoredVal.getOperand(1);
+
+  // Both operands must be plain loads — the only XMM-native source that
+  // doesn't already require a GPR↔XMM move.
+  if (!isPlainScalarLoad(Op0, VT) || !isPlainScalarLoad(Op1, VT))
+    return SDValue();
+
+  // Each load must only feed scalar min/max nodes, or a SCALAR_TO_VECTOR from
+  // a sibling min/max already combined this turn (CSE shares the S2V). This
+  // lets paired smin+smax of the same loads both convert. A GPR consumer
+  // would force the load into a GPR and block memory folding.
+  auto isScalarMinMax = [](SDNode *U) {
+    unsigned O = U->getOpcode();
+    return (O == ISD::SMIN || O == ISD::SMAX || O == ISD::UMIN ||
+            O == ISD::UMAX) &&
+           !U->getValueType(0).isVector();
+  };
+  auto checkLoadUses = [&](SDValue Ld) {
+    for (SDUse &Use : Ld->uses()) {
+      if (Use.getResNo() != Ld.getResNo())
+        continue;
+      SDNode *User = Use.getUser();
+      if (User->getOpcode() == ISD::SCALAR_TO_VECTOR)
+        continue;
+      if (!isScalarMinMax(User))
+        return false;
+    }
+    return true;
+  };
+  if (!checkLoadUses(Op0) || !checkLoadUses(Op1))
+    return SDValue();
+
+  SDValue V0 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op0);
+  SDValue V1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op1);
+  SDValue Vec = DAG.getNode(Opc, DL, VecVT, V0, V1);
+  SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Vec,
----------------
RKSimon wrote:

getExtractVectorElt

https://github.com/llvm/llvm-project/pull/210654