[llvm] [IR] Add `llvm.dynamicshuffle` intrinsic for vector shuffles with runtime masks (PR #208537)

[Luke Lau via llvm-commits]

================
@@ -14126,6 +14132,79 @@ SDValue RISCVTargetLowering::lowerVectorCompress(SDValue Op,
   return Res;
 }
 
+SDValue RISCVTargetLowering::lowerVECTOR_SHUFFLE_VAR(SDValue Op,
+                                                     SelectionDAG &DAG) const {
+  SDLoc DL(Op);
+  SDValue V1 = Op.getOperand(0);
+  SDValue V2 = Op.getOperand(1);
+  SDValue Mask = Op.getOperand(2);
+  MVT VT = Op.getSimpleValueType();
+  MVT XLenVT = Subtarget.getXLenVT();
+
+  // Only handle shuffles that preserve the input type; anything else takes
+  // the generic expansion.
+  if (V1.getSimpleValueType() != VT)
+    return SDValue();
+
+  // Gather with indices of the data SEW, or i16 (vrgatherei16.vv) when SEW
+  // can't represent the whole two-source index space (i8) or is wider than
+  // XLEN (i64 on RV32).
+  unsigned GatherOpc = RISCVISD::VRGATHER_VV_VL;
+  MVT IndexVT = VT.changeTypeToInteger();
+  if (IndexVT.getScalarType().bitsGT(XLenVT) ||
+      (VT.getScalarSizeInBits() == 8 &&
+       (VT.isScalableVector() || 2 * VT.getVectorNumElements() > 256))) {
+    GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
+    IndexVT = IndexVT.changeVectorElementType(MVT::i16);
+  }
+
+  MVT ContainerVT = VT;
+  MVT IndexContainerVT = IndexVT;
+  if (VT.isFixedLengthVector()) {
+    ContainerVT = getContainerForFixedLengthVector(VT);
+    IndexContainerVT =
+        ContainerVT.changeVectorElementType(IndexVT.getScalarType());
+  }
+  // vrgatherei16 at SEW=8 needs an index vector of twice the data LMUL; bail
+  // if that (or an RV64 index for RV32 i64 data) isn't a legal type.
+  if (!isTypeLegal(IndexContainerVT))
+    return SDValue();
+
+  // Indices are unsigned; truncation can only wrap out-of-range (poison)
+  // lanes back into range, which poison permits.
+  SDValue Idx = DAG.getZExtOrTrunc(Mask, DL, IndexVT);
+  auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
+
+  auto Gather = [&](SDValue Src, SDValue Indices) {
+    if (VT.isFixedLengthVector()) {
+      Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
+      Indices =
+          convertToScalableVector(IndexContainerVT, Indices, DAG, Subtarget);
+    }
+    SDValue G = DAG.getNode(GatherOpc, DL, ContainerVT, Src, Indices,
+                            DAG.getUNDEF(ContainerVT), TrueMask, VL);
+    if (VT.isFixedLengthVector())
+      G = convertFromScalableVector(VT, G, DAG, Subtarget);
+    return G;
+  };
+
+  // vrgather zeroes out-of-range lanes; those are poison here, so a single
+  // gather covers the one-source case and two gathers plus a select on
+  // Idx < NumElts cover the two-source case with no other range checks.
+  SDValue Res = Gather(V1, Idx);
+  if (!V2.isUndef()) {
+    SDValue NumElts = DAG.getSplat(
+        IndexVT, DL,
+        DAG.getElementCount(DL, XLenVT, VT.getVectorElementCount()));
+    SDValue Res2 = Gather(V2, DAG.getNode(ISD::SUB, DL, IndexVT, Idx, NumElts));
+    EVT CCVT =
+        getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), IndexVT);
+    SDValue FromV1 = DAG.getSetCC(DL, CCVT, Idx, NumElts, ISD::SETULT);
+    Res = DAG.getSelect(DL, VT, FromV1, Res, Res2);
+  }
+  return Res;
+}
----------------
lukel97 wrote:

> There's interest from the AArch64 SVE side of things to use a dynaicshuffle-like intrinsics to more directly target TBL instructions from the loop vectorizer (which is where https://github.com/llvm/llvm-project/pull/208688 came from). Currently, lowering to TBL requires quite a bit of post vectorization pattern matching. 

What pattern in the loop vectorizer does this match? Do we ever dynamically need to shuffle across VF lanes there?

https://github.com/llvm/llvm-project/pull/208537