[llvm] 3cf97d8 - [X86] Make ISD::ROTL/ROTR vector rotates legal on XOP+AVX512 targets (#184587)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Mar 5 01:11:29 PST 2026
Author: Simon Pilgrim
Date: 2026-03-05T09:11:23Z
New Revision: 3cf97d8d48ccb309a5add656e7b94e27561a7dfa
URL: https://github.com/llvm/llvm-project/commit/3cf97d8d48ccb309a5add656e7b94e27561a7dfa
DIFF: https://github.com/llvm/llvm-project/commit/3cf97d8d48ccb309a5add656e7b94e27561a7dfa.diff
LOG: [X86] Make ISD::ROTL/ROTR vector rotates legal on XOP+AVX512 targets (#184587)
Similar to what we did for funnel shifts on #166949 - set vector rotates
as legal on XOP (128-bit ROTL) and AVX512 (vXi32/vXi64 ROTL/ROTR)
targets, and custom fold to X86ISD::VROTLI/VROTRI as a later fixup.
128/256-bit vector widening to 512-bit instructions is already fully
supported + tested on AVX512F-only targets
First part of #184002
Added:
Modified:
llvm/lib/Target/X86/X86ISelLowering.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 6384c4d58a480..2612befd71b15 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -1451,9 +1451,9 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
}
if (!Subtarget.useSoftFloat() && Subtarget.hasXOP()) {
- for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64,
- MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
- setOperationAction(ISD::ROTL, VT, Custom);
+ for (MVT VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64,
+ MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
+ setOperationAction(ISD::ROTL, VT, VT.is128BitVector() ? Legal : Custom);
setOperationAction(ISD::ROTR, VT, Custom);
}
@@ -2035,6 +2035,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::UMAX, VT, Legal);
setOperationAction(ISD::SMIN, VT, Legal);
setOperationAction(ISD::UMIN, VT, Legal);
+ setOperationAction(ISD::ROTL, VT, Legal);
+ setOperationAction(ISD::ROTR, VT, Legal);
setOperationAction(ISD::ABS, VT, Legal);
setOperationAction(ISD::CTPOP, VT, Custom);
}
@@ -2199,8 +2201,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
}
for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {
- setOperationAction(ISD::ROTL, VT, Custom);
- setOperationAction(ISD::ROTR, VT, Custom);
+ setOperationAction(ISD::ROTL, VT, Legal);
+ setOperationAction(ISD::ROTR, VT, Legal);
}
// Custom legalize 2x32 to get a little better code.
@@ -2778,6 +2780,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
ISD::STRICT_FP_EXTEND,
ISD::FP_ROUND,
ISD::STRICT_FP_ROUND,
+ ISD::ROTL,
+ ISD::ROTR,
ISD::FSHL,
ISD::FSHR,
ISD::INTRINSIC_VOID,
@@ -31938,20 +31942,6 @@ static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
if (IsCstSplat && CstSplatValue.urem(EltSizeInBits) == 0)
return R;
- // AVX512 implicitly uses modulo rotation amounts.
- if ((Subtarget.hasVLX() || Subtarget.hasAVX512()) && 32 <= EltSizeInBits) {
- // Attempt to rotate by immediate.
- if (IsCstSplat) {
- unsigned RotOpc = IsROTL ? X86ISD::VROTLI : X86ISD::VROTRI;
- uint64_t RotAmt = CstSplatValue.urem(EltSizeInBits);
- return DAG.getNode(RotOpc, DL, VT, R,
- DAG.getTargetConstant(RotAmt, DL, MVT::i8));
- }
-
- // Else, fall-back on VPROLV/VPRORV.
- return Op;
- }
-
// AVX512 VBMI2 vXi16 - lower to funnel shifts.
if (Subtarget.hasVBMI2() && 16 == EltSizeInBits) {
unsigned FunnelOpc = IsROTL ? ISD::FSHL : ISD::FSHR;
@@ -31985,24 +31975,6 @@ static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
if (VT.is256BitVector() && (Subtarget.hasXOP() || !Subtarget.hasAVX2()))
return splitVectorIntBinary(Op, DAG, DL);
- // XOP has 128-bit vector variable + immediate rotates.
- // +ve/-ve Amt = rotate left/right - just need to handle ISD::ROTL.
- // XOP implicitly uses modulo rotation amounts.
- if (Subtarget.hasXOP()) {
- assert(IsROTL && "Only ROTL expected");
- assert(VT.is128BitVector() && "Only rotate 128-bit vectors!");
-
- // Attempt to rotate by immediate.
- if (IsCstSplat) {
- uint64_t RotAmt = CstSplatValue.urem(EltSizeInBits);
- return DAG.getNode(X86ISD::VROTLI, DL, VT, R,
- DAG.getTargetConstant(RotAmt, DL, MVT::i8));
- }
-
- // Use general rotate by variable (per-element).
- return Op;
- }
-
// Rotate by an uniform constant - expand back to shifts.
// TODO: Can't use generic expansion as UNDEF amt elements can be converted
// to other values when folded to shift amounts, losing the splat.
@@ -58629,6 +58601,33 @@ static SDValue combineFP_TO_xINT_SAT(SDNode *N, SelectionDAG &DAG,
return SDValue();
}
+// Turn uniform-constant splat rotates into VROTLI/VROTRI
+static SDValue combineRotate(SDNode *N, SelectionDAG &DAG,
+ TargetLowering::DAGCombinerInfo &DCI) {
+ SDLoc DL(N);
+ SDValue Src = N->getOperand(0);
+ SDValue Amt = N->getOperand(1);
+ EVT VT = Src.getValueType();
+
+ // Only combine if the operation is legal for this vector type.
+ // This ensures we don't try to convert types that need to be
+ // widened/promoted or prematurely convert to VROTLI/VROTRI before generic
+ // folds have a chance.
+ if (!VT.isVector() || !DCI.isAfterLegalizeDAG() ||
+ !DAG.getTargetLoweringInfo().isOperationLegal(N->getOpcode(), VT))
+ return SDValue();
+
+ APInt RotateVal;
+ if (!X86::isConstantSplat(Amt, RotateVal))
+ return SDValue();
+
+ bool IsROTR = N->getOpcode() == ISD::ROTR;
+ unsigned Opcode = IsROTR ? X86ISD::VROTRI : X86ISD::VROTLI;
+ uint64_t ModAmt = RotateVal.urem(VT.getScalarSizeInBits());
+ SDValue Imm = DAG.getTargetConstant(ModAmt, DL, MVT::i8);
+ return DAG.getNode(Opcode, DL, VT, {Src, Imm});
+}
+
// Combiner: turn uniform-constant splat funnel shifts into VSHLD/VSHRD
static SDValue combineFunnelShift(SDNode *N, SelectionDAG &DAG,
TargetLowering::DAGCombinerInfo &DCI,
@@ -62380,6 +62379,8 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
case ISD::OR: return combineOr(N, DAG, DCI, Subtarget);
case ISD::XOR: return combineXor(N, DAG, DCI, Subtarget);
case ISD::BITREVERSE: return combineBITREVERSE(N, DAG, DCI, Subtarget);
+ case ISD::ROTL:
+ case ISD::ROTR: return combineRotate(N, DAG, DCI);
case ISD::AVGCEILS:
case ISD::AVGCEILU:
case ISD::AVGFLOORS:
More information about the llvm-commits
mailing list