[llvm] [X86] combineX86ShufflesRecursively - replace Root node argument with opcode/valuetype/ismaskedshuffle data. NFC. (PR #132437)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Fri Mar 21 10:52:58 PDT 2025
https://github.com/RKSimon created https://github.com/llvm/llvm-project/pull/132437
Preparatory cleanup up patch to makes it easier for combineX86ShufflesRecursively/combineX86ShuffleChain to handle length changing shuffles up the shuffle chain than what combineX86ShuffleChainWithExtract can manage.
Instead of passing the original Root node, pass the root opcode and the current effective value type (which may have widened as we recurse through EXTRACT_SUBVECTOR/TRUNCATE nodes etc.).
>From a8b4fa4f6d2289497338dc2618756849b2c8937e Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Wed, 12 Mar 2025 15:15:38 +0000
Subject: [PATCH 1/3] [X86] Remove Root node from
combineX86ShuffleChain/combineX86ShuffleChainWithExtract
Just provide the (original) root opcode and value type instead of using the Root node itself
This will be necessary to allow us to combine shuffle across different vector widths in a more general manner than what combineX86ShuffleChainWithExtract allows
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 56 +++++++++++++------------
1 file changed, 29 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 02398923ebc90..1c6fb1a505ea8 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -39642,9 +39642,10 @@ static bool matchBinaryPermuteShuffle(
}
static SDValue combineX86ShuffleChainWithExtract(
- ArrayRef<SDValue> Inputs, SDValue Root, ArrayRef<int> BaseMask, int Depth,
- ArrayRef<const SDNode *> SrcNodes, bool AllowVariableCrossLaneMask,
- bool AllowVariablePerLaneMask, bool IsMaskedShuffle, SelectionDAG &DAG,
+ ArrayRef<SDValue> Inputs, unsigned RootOpcode, MVT RootVT,
+ ArrayRef<int> BaseMask, int Depth, ArrayRef<const SDNode *> SrcNodes,
+ bool AllowVariableCrossLaneMask, bool AllowVariablePerLaneMask,
+ bool IsMaskedShuffle, SelectionDAG &DAG, const SDLoc &DL,
const X86Subtarget &Subtarget);
/// Combine an arbitrary chain of shuffles into a single instruction if
@@ -39657,16 +39658,14 @@ static SDValue combineX86ShuffleChainWithExtract(
/// for this operation, or into a PSHUFB instruction which is a fully general
/// instruction but should only be used to replace chains over a certain depth.
static SDValue combineX86ShuffleChain(
- ArrayRef<SDValue> Inputs, SDValue Root, ArrayRef<int> BaseMask, int Depth,
- ArrayRef<const SDNode *> SrcNodes, bool AllowVariableCrossLaneMask,
- bool AllowVariablePerLaneMask, bool IsMaskedShuffle, SelectionDAG &DAG,
- const SDLoc &DL, const X86Subtarget &Subtarget) {
+ ArrayRef<SDValue> Inputs, unsigned RootOpc, MVT RootVT,
+ ArrayRef<int> BaseMask, int Depth, ArrayRef<const SDNode *> SrcNodes,
+ bool AllowVariableCrossLaneMask, bool AllowVariablePerLaneMask,
+ bool IsMaskedShuffle, SelectionDAG &DAG, const SDLoc &DL,
+ const X86Subtarget &Subtarget) {
assert(!BaseMask.empty() && "Cannot combine an empty shuffle mask!");
assert((Inputs.size() == 1 || Inputs.size() == 2) &&
"Unexpected number of shuffle inputs!");
-
- unsigned RootOpc = Root.getOpcode();
- MVT RootVT = Root.getSimpleValueType();
unsigned RootSizeInBits = RootVT.getSizeInBits();
unsigned NumRootElts = RootVT.getVectorNumElements();
@@ -40194,8 +40193,9 @@ static SDValue combineX86ShuffleChain(
// If that failed and either input is extracted then try to combine as a
// shuffle with the larger type.
if (SDValue WideShuffle = combineX86ShuffleChainWithExtract(
- Inputs, Root, BaseMask, Depth, SrcNodes, AllowVariableCrossLaneMask,
- AllowVariablePerLaneMask, IsMaskedShuffle, DAG, Subtarget))
+ Inputs, RootOpc, RootVT, BaseMask, Depth, SrcNodes,
+ AllowVariableCrossLaneMask, AllowVariablePerLaneMask,
+ IsMaskedShuffle, DAG, DL, Subtarget))
return WideShuffle;
// If we have a dual input lane-crossing shuffle then lower to VPERMV3,
@@ -40366,8 +40366,9 @@ static SDValue combineX86ShuffleChain(
// If that failed and either input is extracted then try to combine as a
// shuffle with the larger type.
if (SDValue WideShuffle = combineX86ShuffleChainWithExtract(
- Inputs, Root, BaseMask, Depth, SrcNodes, AllowVariableCrossLaneMask,
- AllowVariablePerLaneMask, IsMaskedShuffle, DAG, Subtarget))
+ Inputs, RootOpc, RootVT, BaseMask, Depth, SrcNodes,
+ AllowVariableCrossLaneMask, AllowVariablePerLaneMask, IsMaskedShuffle,
+ DAG, DL, Subtarget))
return WideShuffle;
// If we have a dual input shuffle then lower to VPERMV3,
@@ -40404,16 +40405,16 @@ static SDValue combineX86ShuffleChain(
// -->
// extract_subvector(shuffle(x,y,m2),0)
static SDValue combineX86ShuffleChainWithExtract(
- ArrayRef<SDValue> Inputs, SDValue Root, ArrayRef<int> BaseMask, int Depth,
- ArrayRef<const SDNode *> SrcNodes, bool AllowVariableCrossLaneMask,
- bool AllowVariablePerLaneMask, bool IsMaskedShuffle, SelectionDAG &DAG,
+ ArrayRef<SDValue> Inputs, unsigned RootOpcode, MVT RootVT,
+ ArrayRef<int> BaseMask, int Depth, ArrayRef<const SDNode *> SrcNodes,
+ bool AllowVariableCrossLaneMask, bool AllowVariablePerLaneMask,
+ bool IsMaskedShuffle, SelectionDAG &DAG, const SDLoc &DL,
const X86Subtarget &Subtarget) {
unsigned NumMaskElts = BaseMask.size();
unsigned NumInputs = Inputs.size();
if (NumInputs == 0)
return SDValue();
- EVT RootVT = Root.getValueType();
unsigned RootSizeInBits = RootVT.getSizeInBits();
unsigned RootEltSizeInBits = RootSizeInBits / NumMaskElts;
assert((RootSizeInBits % NumMaskElts) == 0 && "Unexpected root shuffle mask");
@@ -40533,11 +40534,10 @@ static SDValue combineX86ShuffleChainWithExtract(
"WideRootSize mismatch");
if (SDValue WideShuffle = combineX86ShuffleChain(
- WideInputs, WideRoot, WideMask, Depth, SrcNodes,
- AllowVariableCrossLaneMask, AllowVariablePerLaneMask, IsMaskedShuffle,
- DAG, SDLoc(WideRoot), Subtarget)) {
- WideShuffle =
- extractSubVector(WideShuffle, 0, DAG, SDLoc(Root), RootSizeInBits);
+ WideInputs, RootOpcode, WideRoot.getSimpleValueType(), WideMask,
+ Depth, SrcNodes, AllowVariableCrossLaneMask, AllowVariablePerLaneMask,
+ IsMaskedShuffle, DAG, SDLoc(WideRoot), Subtarget)) {
+ WideShuffle = extractSubVector(WideShuffle, 0, DAG, DL, RootSizeInBits);
return DAG.getBitcast(RootVT, WideShuffle);
}
@@ -41298,8 +41298,9 @@ static SDValue combineX86ShufflesRecursively(
// Try to combine into a single shuffle instruction.
if (SDValue Shuffle = combineX86ShuffleChain(
- Ops, Root, Mask, Depth, CombinedNodes, AllowVariableCrossLaneMask,
- AllowVariablePerLaneMask, IsMaskedShuffle, DAG, DL, Subtarget))
+ Ops, Root.getOpcode(), RootVT, Mask, Depth, CombinedNodes,
+ AllowVariableCrossLaneMask, AllowVariablePerLaneMask,
+ IsMaskedShuffle, DAG, DL, Subtarget))
return Shuffle;
// If all the operands come from the same larger vector, fallthrough and try
@@ -41317,8 +41318,9 @@ static SDValue combineX86ShufflesRecursively(
// If that failed and any input is extracted then try to combine as a
// shuffle with the larger type.
return combineX86ShuffleChainWithExtract(
- Ops, Root, Mask, Depth, CombinedNodes, AllowVariableCrossLaneMask,
- AllowVariablePerLaneMask, IsMaskedShuffle, DAG, Subtarget);
+ Ops, Root.getOpcode(), RootVT, Mask, Depth, CombinedNodes,
+ AllowVariableCrossLaneMask, AllowVariablePerLaneMask, IsMaskedShuffle,
+ DAG, DL, Subtarget);
}
/// Helper entry wrapper to combineX86ShufflesRecursively.
>From 07352352730cfa70cad3f9ca8f50a313f9a6d95c Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Wed, 12 Mar 2025 18:15:51 +0000
Subject: [PATCH 2/3] [X86] combineX86ShufflesRecursively - pull out repeated
Root.getOpcode() calls. NFC.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 1c6fb1a505ea8..96bafc89ae863 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -40889,6 +40889,7 @@ static SDValue combineX86ShufflesRecursively(
assert(!RootMask.empty() &&
(RootMask.size() > 1 || (RootMask[0] == 0 && SrcOpIndex == 0)) &&
"Illegal shuffle root mask");
+ unsigned RootOpc = Root.getOpcode();
MVT RootVT = Root.getSimpleValueType();
assert(RootVT.isVector() && "Shuffles operate on vector types!");
unsigned RootSizeInBits = RootVT.getSizeInBits();
@@ -41298,7 +41299,7 @@ static SDValue combineX86ShufflesRecursively(
// Try to combine into a single shuffle instruction.
if (SDValue Shuffle = combineX86ShuffleChain(
- Ops, Root.getOpcode(), RootVT, Mask, Depth, CombinedNodes,
+ Ops, RootOpc, RootVT, Mask, Depth, CombinedNodes,
AllowVariableCrossLaneMask, AllowVariablePerLaneMask,
IsMaskedShuffle, DAG, DL, Subtarget))
return Shuffle;
@@ -41318,7 +41319,7 @@ static SDValue combineX86ShufflesRecursively(
// If that failed and any input is extracted then try to combine as a
// shuffle with the larger type.
return combineX86ShuffleChainWithExtract(
- Ops, Root.getOpcode(), RootVT, Mask, Depth, CombinedNodes,
+ Ops, RootOpc, RootVT, Mask, Depth, CombinedNodes,
AllowVariableCrossLaneMask, AllowVariablePerLaneMask, IsMaskedShuffle,
DAG, DL, Subtarget);
}
>From 33d4cb33230178bb686dd35af9d08c7aaf03117a Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Wed, 12 Mar 2025 18:54:32 +0000
Subject: [PATCH 3/3] [X86] combineX86ShufflesRecursively - replace Root node
argument with opcode/valuetype/ismaskedshuffle data. NFC.
Makes it easier for combineX86ShufflesRecursively to handle length changing shuffles up the shuffle chain.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 43 ++++++++++++-------------
1 file changed, 20 insertions(+), 23 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 96bafc89ae863..9e0c73607ce71 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -40881,16 +40881,14 @@ namespace llvm {
/// combine-ordering. To fix this, we should do the redundant instruction
/// combining in this recursive walk.
static SDValue combineX86ShufflesRecursively(
- ArrayRef<SDValue> SrcOps, int SrcOpIndex, SDValue Root,
+ ArrayRef<SDValue> SrcOps, int SrcOpIndex, unsigned RootOpc, MVT RootVT,
ArrayRef<int> RootMask, ArrayRef<const SDNode *> SrcNodes, unsigned Depth,
unsigned MaxDepth, bool AllowVariableCrossLaneMask,
- bool AllowVariablePerLaneMask, SelectionDAG &DAG, const SDLoc &DL,
- const X86Subtarget &Subtarget) {
+ bool AllowVariablePerLaneMask, bool IsMaskedShuffle, SelectionDAG &DAG,
+ const SDLoc &DL, const X86Subtarget &Subtarget) {
assert(!RootMask.empty() &&
(RootMask.size() > 1 || (RootMask[0] == 0 && SrcOpIndex == 0)) &&
"Illegal shuffle root mask");
- unsigned RootOpc = Root.getOpcode();
- MVT RootVT = Root.getSimpleValueType();
assert(RootVT.isVector() && "Shuffles operate on vector types!");
unsigned RootSizeInBits = RootVT.getSizeInBits();
@@ -41185,8 +41183,9 @@ static SDValue combineX86ShufflesRecursively(
AllowPerLaneVar = AllowVariablePerLaneMask;
}
if (SDValue Res = combineX86ShufflesRecursively(
- Ops, i, Root, ResolvedMask, CombinedNodes, Depth + 1, MaxDepth,
- AllowCrossLaneVar, AllowPerLaneVar, DAG, DL, Subtarget))
+ Ops, i, RootOpc, RootVT, ResolvedMask, CombinedNodes, Depth + 1,
+ MaxDepth, AllowCrossLaneVar, AllowPerLaneVar, IsMaskedShuffle,
+ DAG, DL, Subtarget))
return Res;
}
}
@@ -41272,10 +41271,6 @@ static SDValue combineX86ShufflesRecursively(
resolveTargetShuffleInputsAndMask(Ops, Mask);
}
- // If we are a AVX512/EVEX target the mask element size should match the root
- // element size to allow writemasks to be reused.
- bool IsMaskedShuffle = isMaskableNode(Root, Subtarget);
-
// We can only combine unary and binary shuffle mask cases.
if (Ops.size() <= 2) {
// Minor canonicalization of the accumulated shuffle mask to make it easier
@@ -41328,8 +41323,9 @@ static SDValue combineX86ShufflesRecursively(
static SDValue combineX86ShufflesRecursively(SDValue Op, SelectionDAG &DAG,
const X86Subtarget &Subtarget) {
return combineX86ShufflesRecursively(
- {Op}, 0, Op, {0}, {}, /*Depth*/ 0, X86::MaxShuffleCombineDepth,
- /*AllowCrossLaneVarMask*/ true, /*AllowPerLaneVarMask*/ true, DAG,
+ {Op}, 0, Op.getOpcode(), Op.getSimpleValueType(), {0}, {}, /*Depth*/ 0,
+ X86::MaxShuffleCombineDepth, /*AllowCrossLaneVarMask*/ true,
+ /*AllowPerLaneVarMask*/ true, isMaskableNode(Op, Subtarget), DAG,
SDLoc(Op), Subtarget);
}
@@ -41980,10 +41976,10 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
for (unsigned i = 0; i != Scale; ++i)
DemandedMask[i] = i;
if (SDValue Res = combineX86ShufflesRecursively(
- {BC}, 0, BC, DemandedMask, {}, /*Depth*/ 0,
- X86::MaxShuffleCombineDepth,
- /*AllowCrossLaneVarMask*/ true,
- /*AllowPerLaneVarMask*/ true, DAG, DL, Subtarget))
+ {BC}, 0, BC.getOpcode(), BC.getSimpleValueType(), DemandedMask,
+ {}, /*Depth*/ 0, X86::MaxShuffleCombineDepth,
+ /*AllowCrossLaneVarMask*/ true, /*AllowPerLaneVarMask*/ true,
+ /*IsMaskedShuffle*/ false, DAG, DL, Subtarget))
return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
DAG.getBitcast(SrcVT, Res));
}
@@ -43984,8 +43980,9 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
DemandedMask[i] = i;
SDValue NewShuffle = combineX86ShufflesRecursively(
- {Op}, 0, Op, DemandedMask, {}, 0, X86::MaxShuffleCombineDepth - Depth,
- /*AllowCrossLaneVarMask*/ true, /*AllowPerLaneVarMask*/ true, TLO.DAG,
+ {Op}, 0, Op.getOpcode(), Op.getSimpleValueType(), DemandedMask, {}, 0,
+ X86::MaxShuffleCombineDepth - Depth, /*AllowCrossLaneVarMask*/ true,
+ /*AllowPerLaneVarMask*/ true, isMaskableNode(Op, Subtarget), TLO.DAG,
SDLoc(Op), Subtarget);
if (NewShuffle)
return TLO.CombineTo(Op, NewShuffle);
@@ -51620,10 +51617,10 @@ static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
}
if (SDValue Shuffle = combineX86ShufflesRecursively(
- {SrcVec}, 0, SrcVec, ShuffleMask, {}, /*Depth*/ 1,
- X86::MaxShuffleCombineDepth,
- /*AllowVarCrossLaneMask*/ true,
- /*AllowVarPerLaneMask*/ true, DAG, SDLoc(SrcVec), Subtarget))
+ {SrcVec}, 0, SrcVec.getOpcode(), SrcVec.getSimpleValueType(),
+ ShuffleMask, {}, /*Depth*/ 1, X86::MaxShuffleCombineDepth,
+ /*AllowVarCrossLaneMask*/ true, /*AllowVarPerLaneMask*/ true,
+ /*IsMaskedShuffle*/ false, DAG, SDLoc(SrcVec), Subtarget))
return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Shuffle,
N0.getOperand(1));
}
More information about the llvm-commits
mailing list