[llvm] 182eb9d - [X86] Move getTargetVShift helpers earlier in the source file. NFC. (#180972)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Feb 11 09:19:37 PST 2026
Author: Simon Pilgrim
Date: 2026-02-11T17:19:32Z
New Revision: 182eb9d21a7a3cdf24c206d71b15bb847f41ecb3
URL: https://github.com/llvm/llvm-project/commit/182eb9d21a7a3cdf24c206d71b15bb847f41ecb3
DIFF: https://github.com/llvm/llvm-project/commit/182eb9d21a7a3cdf24c206d71b15bb847f41ecb3.diff
LOG: [X86] Move getTargetVShift helpers earlier in the source file. NFC. (#180972)
Avoid having to add forward declarations for earlier functions to use them.
Added:
Modified:
llvm/lib/Target/X86/X86ISelLowering.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 9c6cc95cc5eac..88616e0f2a0c9 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -4886,6 +4886,176 @@ static SDValue getOnesVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) {
return DAG.getBitcast(VT, Vec);
}
+// Helper to get immediate/variable SSE shift opcode from other shift opcodes.
+static unsigned getTargetVShiftUniformOpcode(unsigned Opc, bool IsVariable) {
+ switch (Opc) {
+ case ISD::SHL:
+ case X86ISD::VSHL:
+ case X86ISD::VSHLI:
+ return IsVariable ? X86ISD::VSHL : X86ISD::VSHLI;
+ case ISD::SRL:
+ case X86ISD::VSRL:
+ case X86ISD::VSRLI:
+ return IsVariable ? X86ISD::VSRL : X86ISD::VSRLI;
+ case ISD::SRA:
+ case X86ISD::VSRA:
+ case X86ISD::VSRAI:
+ return IsVariable ? X86ISD::VSRA : X86ISD::VSRAI;
+ }
+ llvm_unreachable("Unknown target vector shift node");
+}
+
+/// Handle vector element shifts where the shift amount is a constant.
+/// Takes immediate version of shift as input.
+static SDValue getTargetVShiftByConstNode(unsigned Opc, const SDLoc &dl, MVT VT,
+ SDValue SrcOp, uint64_t ShiftAmt,
+ SelectionDAG &DAG) {
+ MVT ElementType = VT.getVectorElementType();
+
+ // Bitcast the source vector to the output type, this is mainly necessary for
+ // vXi8/vXi64 shifts.
+ if (VT != SrcOp.getSimpleValueType())
+ SrcOp = DAG.getBitcast(VT, SrcOp);
+
+ // Fold this packed shift into its first operand if ShiftAmt is 0.
+ if (ShiftAmt == 0)
+ return SrcOp;
+
+ // Check for ShiftAmt >= element width
+ if (ShiftAmt >= ElementType.getSizeInBits()) {
+ if (Opc == X86ISD::VSRAI)
+ ShiftAmt = ElementType.getSizeInBits() - 1;
+ else
+ return DAG.getConstant(0, dl, VT);
+ }
+
+ assert(
+ (Opc == X86ISD::VSHLI || Opc == X86ISD::VSRLI || Opc == X86ISD::VSRAI) &&
+ "Unknown target vector shift-by-constant node");
+
+ // Fold this packed vector shift into a build vector if SrcOp is a
+ // vector of Constants or UNDEFs.
+ if (ISD::isBuildVectorOfConstantSDNodes(SrcOp.getNode())) {
+ unsigned ShiftOpc;
+ switch (Opc) {
+ default:
+ llvm_unreachable("Unknown opcode!");
+ case X86ISD::VSHLI:
+ ShiftOpc = ISD::SHL;
+ break;
+ case X86ISD::VSRLI:
+ ShiftOpc = ISD::SRL;
+ break;
+ case X86ISD::VSRAI:
+ ShiftOpc = ISD::SRA;
+ break;
+ }
+
+ SDValue Amt = DAG.getConstant(ShiftAmt, dl, VT);
+ if (SDValue C = DAG.FoldConstantArithmetic(ShiftOpc, dl, VT, {SrcOp, Amt}))
+ return C;
+ }
+
+ return DAG.getNode(Opc, dl, VT, SrcOp,
+ DAG.getTargetConstant(ShiftAmt, dl, MVT::i8));
+}
+
+/// Handle vector element shifts by a splat shift amount
+static SDValue getTargetVShiftNode(unsigned Opc, const SDLoc &dl, MVT VT,
+ SDValue SrcOp, SDValue ShAmt, int ShAmtIdx,
+ const X86Subtarget &Subtarget,
+ SelectionDAG &DAG) {
+ MVT AmtVT = ShAmt.getSimpleValueType();
+ assert(AmtVT.isVector() && "Vector shift type mismatch");
+ assert(0 <= ShAmtIdx && ShAmtIdx < (int)AmtVT.getVectorNumElements() &&
+ "Illegal vector splat index");
+
+ // Move the splat element to the bottom element.
+ if (ShAmtIdx != 0) {
+ SmallVector<int> Mask(AmtVT.getVectorNumElements(), -1);
+ Mask[0] = ShAmtIdx;
+ ShAmt = DAG.getVectorShuffle(AmtVT, dl, ShAmt, DAG.getUNDEF(AmtVT), Mask);
+ }
+
+ // Peek through any zext node if we can get back to a 128-bit source.
+ if (AmtVT.getScalarSizeInBits() == 64 &&
+ (ShAmt.getOpcode() == ISD::ZERO_EXTEND ||
+ ShAmt.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) &&
+ ShAmt.getOperand(0).getValueType().isSimple() &&
+ ShAmt.getOperand(0).getValueType().is128BitVector()) {
+ ShAmt = ShAmt.getOperand(0);
+ AmtVT = ShAmt.getSimpleValueType();
+ }
+
+ // See if we can mask off the upper elements using the existing source node.
+ // The shift uses the entire lower 64-bits of the amount vector, so no need to
+ // do this for vXi64 types.
+ bool IsMasked = false;
+ if (AmtVT.getScalarSizeInBits() < 64) {
+ if (ShAmt.getOpcode() == ISD::BUILD_VECTOR ||
+ ShAmt.getOpcode() == ISD::SCALAR_TO_VECTOR) {
+ // If the shift amount has come from a scalar, then zero-extend the scalar
+ // before moving to the vector.
+ ShAmt = DAG.getZExtOrTrunc(ShAmt.getOperand(0), dl, MVT::i32);
+ ShAmt = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, ShAmt);
+ ShAmt = DAG.getNode(X86ISD::VZEXT_MOVL, dl, MVT::v4i32, ShAmt);
+ AmtVT = MVT::v4i32;
+ IsMasked = true;
+ } else if (ShAmt.getOpcode() == ISD::AND) {
+ // See if the shift amount is already masked (e.g. for rotation modulo),
+ // then we can zero-extend it by setting all the other mask elements to
+ // zero.
+ SmallVector<SDValue> MaskElts(
+ AmtVT.getVectorNumElements(),
+ DAG.getConstant(0, dl, AmtVT.getScalarType()));
+ MaskElts[0] = DAG.getAllOnesConstant(dl, AmtVT.getScalarType());
+ SDValue Mask = DAG.getBuildVector(AmtVT, dl, MaskElts);
+ if ((Mask = DAG.FoldConstantArithmetic(ISD::AND, dl, AmtVT,
+ {ShAmt.getOperand(1), Mask}))) {
+ ShAmt = DAG.getNode(ISD::AND, dl, AmtVT, ShAmt.getOperand(0), Mask);
+ IsMasked = true;
+ }
+ }
+ }
+
+ // Extract if the shift amount vector is larger than 128-bits.
+ if (AmtVT.getSizeInBits() > 128) {
+ ShAmt = extract128BitVector(ShAmt, 0, DAG, dl);
+ AmtVT = ShAmt.getSimpleValueType();
+ }
+
+ // Zero-extend bottom element to v2i64 vector type, either by extension or
+ // shuffle masking.
+ if (!IsMasked && AmtVT.getScalarSizeInBits() < 64) {
+ if (AmtVT == MVT::v4i32 && (ShAmt.getOpcode() == X86ISD::VBROADCAST ||
+ ShAmt.getOpcode() == X86ISD::VBROADCAST_LOAD)) {
+ ShAmt = DAG.getNode(X86ISD::VZEXT_MOVL, SDLoc(ShAmt), MVT::v4i32, ShAmt);
+ } else if (Subtarget.hasSSE41()) {
+ ShAmt = DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(ShAmt),
+ MVT::v2i64, ShAmt);
+ } else {
+ SDValue ByteShift = DAG.getTargetConstant(
+ (128 - AmtVT.getScalarSizeInBits()) / 8, SDLoc(ShAmt), MVT::i8);
+ ShAmt = DAG.getBitcast(MVT::v16i8, ShAmt);
+ ShAmt = DAG.getNode(X86ISD::VSHLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt,
+ ByteShift);
+ ShAmt = DAG.getNode(X86ISD::VSRLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt,
+ ByteShift);
+ }
+ }
+
+ // Change opcode to non-immediate version.
+ Opc = getTargetVShiftUniformOpcode(Opc, true);
+
+ // The return type has to be a 128-bit type with the same element
+ // type as the input type.
+ MVT EltVT = VT.getVectorElementType();
+ MVT ShVT = MVT::getVectorVT(EltVT, 128 / EltVT.getSizeInBits());
+
+ ShAmt = DAG.getBitcast(ShVT, ShAmt);
+ return DAG.getNode(Opc, dl, VT, SrcOp, ShAmt);
+}
+
static SDValue getEXTEND_VECTOR_INREG(unsigned Opcode, const SDLoc &DL, EVT VT,
SDValue In, SelectionDAG &DAG) {
EVT InVT = In.getValueType();
@@ -26486,174 +26656,6 @@ static SDValue LowerVACOPY(SDValue Op, const X86Subtarget &Subtarget,
MachinePointerInfo(SrcSV));
}
-// Helper to get immediate/variable SSE shift opcode from other shift opcodes.
-static unsigned getTargetVShiftUniformOpcode(unsigned Opc, bool IsVariable) {
- switch (Opc) {
- case ISD::SHL:
- case X86ISD::VSHL:
- case X86ISD::VSHLI:
- return IsVariable ? X86ISD::VSHL : X86ISD::VSHLI;
- case ISD::SRL:
- case X86ISD::VSRL:
- case X86ISD::VSRLI:
- return IsVariable ? X86ISD::VSRL : X86ISD::VSRLI;
- case ISD::SRA:
- case X86ISD::VSRA:
- case X86ISD::VSRAI:
- return IsVariable ? X86ISD::VSRA : X86ISD::VSRAI;
- }
- llvm_unreachable("Unknown target vector shift node");
-}
-
-/// Handle vector element shifts where the shift amount is a constant.
-/// Takes immediate version of shift as input.
-static SDValue getTargetVShiftByConstNode(unsigned Opc, const SDLoc &dl, MVT VT,
- SDValue SrcOp, uint64_t ShiftAmt,
- SelectionDAG &DAG) {
- MVT ElementType = VT.getVectorElementType();
-
- // Bitcast the source vector to the output type, this is mainly necessary for
- // vXi8/vXi64 shifts.
- if (VT != SrcOp.getSimpleValueType())
- SrcOp = DAG.getBitcast(VT, SrcOp);
-
- // Fold this packed shift into its first operand if ShiftAmt is 0.
- if (ShiftAmt == 0)
- return SrcOp;
-
- // Check for ShiftAmt >= element width
- if (ShiftAmt >= ElementType.getSizeInBits()) {
- if (Opc == X86ISD::VSRAI)
- ShiftAmt = ElementType.getSizeInBits() - 1;
- else
- return DAG.getConstant(0, dl, VT);
- }
-
- assert((Opc == X86ISD::VSHLI || Opc == X86ISD::VSRLI || Opc == X86ISD::VSRAI)
- && "Unknown target vector shift-by-constant node");
-
- // Fold this packed vector shift into a build vector if SrcOp is a
- // vector of Constants or UNDEFs.
- if (ISD::isBuildVectorOfConstantSDNodes(SrcOp.getNode())) {
- unsigned ShiftOpc;
- switch (Opc) {
- default: llvm_unreachable("Unknown opcode!");
- case X86ISD::VSHLI:
- ShiftOpc = ISD::SHL;
- break;
- case X86ISD::VSRLI:
- ShiftOpc = ISD::SRL;
- break;
- case X86ISD::VSRAI:
- ShiftOpc = ISD::SRA;
- break;
- }
-
- SDValue Amt = DAG.getConstant(ShiftAmt, dl, VT);
- if (SDValue C = DAG.FoldConstantArithmetic(ShiftOpc, dl, VT, {SrcOp, Amt}))
- return C;
- }
-
- return DAG.getNode(Opc, dl, VT, SrcOp,
- DAG.getTargetConstant(ShiftAmt, dl, MVT::i8));
-}
-
-/// Handle vector element shifts by a splat shift amount
-static SDValue getTargetVShiftNode(unsigned Opc, const SDLoc &dl, MVT VT,
- SDValue SrcOp, SDValue ShAmt, int ShAmtIdx,
- const X86Subtarget &Subtarget,
- SelectionDAG &DAG) {
- MVT AmtVT = ShAmt.getSimpleValueType();
- assert(AmtVT.isVector() && "Vector shift type mismatch");
- assert(0 <= ShAmtIdx && ShAmtIdx < (int)AmtVT.getVectorNumElements() &&
- "Illegal vector splat index");
-
- // Move the splat element to the bottom element.
- if (ShAmtIdx != 0) {
- SmallVector<int> Mask(AmtVT.getVectorNumElements(), -1);
- Mask[0] = ShAmtIdx;
- ShAmt = DAG.getVectorShuffle(AmtVT, dl, ShAmt, DAG.getUNDEF(AmtVT), Mask);
- }
-
- // Peek through any zext node if we can get back to a 128-bit source.
- if (AmtVT.getScalarSizeInBits() == 64 &&
- (ShAmt.getOpcode() == ISD::ZERO_EXTEND ||
- ShAmt.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) &&
- ShAmt.getOperand(0).getValueType().isSimple() &&
- ShAmt.getOperand(0).getValueType().is128BitVector()) {
- ShAmt = ShAmt.getOperand(0);
- AmtVT = ShAmt.getSimpleValueType();
- }
-
- // See if we can mask off the upper elements using the existing source node.
- // The shift uses the entire lower 64-bits of the amount vector, so no need to
- // do this for vXi64 types.
- bool IsMasked = false;
- if (AmtVT.getScalarSizeInBits() < 64) {
- if (ShAmt.getOpcode() == ISD::BUILD_VECTOR ||
- ShAmt.getOpcode() == ISD::SCALAR_TO_VECTOR) {
- // If the shift amount has come from a scalar, then zero-extend the scalar
- // before moving to the vector.
- ShAmt = DAG.getZExtOrTrunc(ShAmt.getOperand(0), dl, MVT::i32);
- ShAmt = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, ShAmt);
- ShAmt = DAG.getNode(X86ISD::VZEXT_MOVL, dl, MVT::v4i32, ShAmt);
- AmtVT = MVT::v4i32;
- IsMasked = true;
- } else if (ShAmt.getOpcode() == ISD::AND) {
- // See if the shift amount is already masked (e.g. for rotation modulo),
- // then we can zero-extend it by setting all the other mask elements to
- // zero.
- SmallVector<SDValue> MaskElts(
- AmtVT.getVectorNumElements(),
- DAG.getConstant(0, dl, AmtVT.getScalarType()));
- MaskElts[0] = DAG.getAllOnesConstant(dl, AmtVT.getScalarType());
- SDValue Mask = DAG.getBuildVector(AmtVT, dl, MaskElts);
- if ((Mask = DAG.FoldConstantArithmetic(ISD::AND, dl, AmtVT,
- {ShAmt.getOperand(1), Mask}))) {
- ShAmt = DAG.getNode(ISD::AND, dl, AmtVT, ShAmt.getOperand(0), Mask);
- IsMasked = true;
- }
- }
- }
-
- // Extract if the shift amount vector is larger than 128-bits.
- if (AmtVT.getSizeInBits() > 128) {
- ShAmt = extract128BitVector(ShAmt, 0, DAG, dl);
- AmtVT = ShAmt.getSimpleValueType();
- }
-
- // Zero-extend bottom element to v2i64 vector type, either by extension or
- // shuffle masking.
- if (!IsMasked && AmtVT.getScalarSizeInBits() < 64) {
- if (AmtVT == MVT::v4i32 && (ShAmt.getOpcode() == X86ISD::VBROADCAST ||
- ShAmt.getOpcode() == X86ISD::VBROADCAST_LOAD)) {
- ShAmt = DAG.getNode(X86ISD::VZEXT_MOVL, SDLoc(ShAmt), MVT::v4i32, ShAmt);
- } else if (Subtarget.hasSSE41()) {
- ShAmt = DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(ShAmt),
- MVT::v2i64, ShAmt);
- } else {
- SDValue ByteShift = DAG.getTargetConstant(
- (128 - AmtVT.getScalarSizeInBits()) / 8, SDLoc(ShAmt), MVT::i8);
- ShAmt = DAG.getBitcast(MVT::v16i8, ShAmt);
- ShAmt = DAG.getNode(X86ISD::VSHLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt,
- ByteShift);
- ShAmt = DAG.getNode(X86ISD::VSRLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt,
- ByteShift);
- }
- }
-
- // Change opcode to non-immediate version.
- Opc = getTargetVShiftUniformOpcode(Opc, true);
-
- // The return type has to be a 128-bit type with the same element
- // type as the input type.
- MVT EltVT = VT.getVectorElementType();
- MVT ShVT = MVT::getVectorVT(EltVT, 128 / EltVT.getSizeInBits());
-
- ShAmt = DAG.getBitcast(ShVT, ShAmt);
- return DAG.getNode(Opc, dl, VT, SrcOp, ShAmt);
-}
-
/// Return Mask with the necessary casting or extending
/// for \p Mask according to \p MaskVT when lowering masking intrinsics
static SDValue getMaskNode(SDValue Mask, MVT MaskVT,
More information about the llvm-commits
mailing list