[llvm] 8f29e44 - [X86][SSE] combineX86ShufflesRecursively - don't bother merging shuffles with empty roots. NFCI.
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Sun Nov 3 09:47:59 PST 2019
Author: Simon Pilgrim
Date: 2019-11-03T17:46:00Z
New Revision: 8f29e4407cc609764eeb450dc432297952ec6f49
URL: https://github.com/llvm/llvm-project/commit/8f29e4407cc609764eeb450dc432297952ec6f49
DIFF: https://github.com/llvm/llvm-project/commit/8f29e4407cc609764eeb450dc432297952ec6f49.diff
LOG: [X86][SSE] combineX86ShufflesRecursively - don't bother merging shuffles with empty roots. NFCI.
This doesn't affect actual codegen, but is a minor refactor toward fixing PR43024 where we need to avoid excess changes (folding zeroables etc.) to the shuffle mask at Depth == 0.
Added:
Modified:
llvm/lib/Target/X86/X86ISelLowering.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 8bd8bff1d5ed..9b59b8a91e05 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -33037,7 +33037,8 @@ static SDValue combineX86ShufflesRecursively(
ArrayRef<int> RootMask, ArrayRef<const SDNode *> SrcNodes, unsigned Depth,
bool HasVariableMask, bool AllowVariableMask, SelectionDAG &DAG,
const X86Subtarget &Subtarget) {
- assert(RootMask.size() > 0 && (RootMask.size() > 1 || RootMask[0] == 0) &&
+ assert(RootMask.size() > 0 &&
+ (RootMask.size() > 1 || (RootMask[0] == 0 && SrcOpIndex == 0)) &&
"Illegal shuffle root mask");
// Bound the depth of our recursive combine because this is ultimately
@@ -33072,104 +33073,116 @@ static SDValue combineX86ShufflesRecursively(
resolveTargetShuffleFromZeroables(OpMask, OpUndef, OpZero);
- // Add the inputs to the Ops list, avoiding duplicates.
- SmallVector<SDValue, 16> Ops(SrcOps.begin(), SrcOps.end());
-
- auto AddOp = [&Ops](SDValue Input, int InsertionPoint) -> int {
- // Attempt to find an existing match.
- SDValue InputBC = peekThroughBitcasts(Input);
- for (int i = 0, e = Ops.size(); i < e; ++i)
- if (InputBC == peekThroughBitcasts(Ops[i]))
- return i;
- // Match failed - should we replace an existing Op?
- if (InsertionPoint >= 0) {
- Ops[InsertionPoint] = Input;
- return InsertionPoint;
- }
- // Add to the end of the Ops list.
- Ops.push_back(Input);
- return Ops.size() - 1;
- };
+ SmallVector<int, 64> Mask;
+ SmallVector<SDValue, 16> Ops;
- SmallVector<int, 2> OpInputIdx;
- for (SDValue OpInput : OpInputs)
- OpInputIdx.push_back(AddOp(OpInput, OpInputIdx.empty() ? SrcOpIndex : -1));
-
- assert(((RootMask.size() > OpMask.size() &&
- RootMask.size() % OpMask.size() == 0) ||
- (OpMask.size() > RootMask.size() &&
- OpMask.size() % RootMask.size() == 0) ||
- OpMask.size() == RootMask.size()) &&
- "The smaller number of elements must divide the larger.");
-
- // This function can be performance-critical, so we rely on the power-of-2
- // knowledge that we have about the mask sizes to replace div/rem ops with
- // bit-masks and shifts.
- assert(isPowerOf2_32(RootMask.size()) && "Non-power-of-2 shuffle mask sizes");
- assert(isPowerOf2_32(OpMask.size()) && "Non-power-of-2 shuffle mask sizes");
- unsigned RootMaskSizeLog2 = countTrailingZeros(RootMask.size());
- unsigned OpMaskSizeLog2 = countTrailingZeros(OpMask.size());
-
- unsigned MaskWidth = std::max<unsigned>(OpMask.size(), RootMask.size());
- unsigned RootRatio = std::max<unsigned>(1, OpMask.size() >> RootMaskSizeLog2);
- unsigned OpRatio = std::max<unsigned>(1, RootMask.size() >> OpMaskSizeLog2);
- assert((RootRatio == 1 || OpRatio == 1) &&
- "Must not have a ratio for both incoming and op masks!");
-
- assert(isPowerOf2_32(MaskWidth) && "Non-power-of-2 shuffle mask sizes");
- assert(isPowerOf2_32(RootRatio) && "Non-power-of-2 shuffle mask sizes");
- assert(isPowerOf2_32(OpRatio) && "Non-power-of-2 shuffle mask sizes");
- unsigned RootRatioLog2 = countTrailingZeros(RootRatio);
- unsigned OpRatioLog2 = countTrailingZeros(OpRatio);
-
- SmallVector<int, 64> Mask(MaskWidth, SM_SentinelUndef);
-
- // Merge this shuffle operation's mask into our accumulated mask. Note that
- // this shuffle's mask will be the first applied to the input, followed by the
- // root mask to get us all the way to the root value arrangement. The reason
- // for this order is that we are recursing up the operation chain.
- for (unsigned i = 0; i < MaskWidth; ++i) {
- unsigned RootIdx = i >> RootRatioLog2;
- if (RootMask[RootIdx] < 0) {
- // This is a zero or undef lane, we're done.
- Mask[i] = RootMask[RootIdx];
- continue;
- }
+ // We don't need to merge masks if the root is empty.
+ bool EmptyRoot = (Depth == 0) && (RootMask.size() == 1);
+ if (EmptyRoot) {
+ Mask = OpMask;
+ Ops.append(OpInputs.begin(), OpInputs.end());
+ } else {
+ // Add the inputs to the Ops list, avoiding duplicates.
+ Ops.append(SrcOps.begin(), SrcOps.end());
+
+ auto AddOp = [&Ops](SDValue Input, int InsertionPoint) -> int {
+ // Attempt to find an existing match.
+ SDValue InputBC = peekThroughBitcasts(Input);
+ for (int i = 0, e = Ops.size(); i < e; ++i)
+ if (InputBC == peekThroughBitcasts(Ops[i]))
+ return i;
+ // Match failed - should we replace an existing Op?
+ if (InsertionPoint >= 0) {
+ Ops[InsertionPoint] = Input;
+ return InsertionPoint;
+ }
+ // Add to the end of the Ops list.
+ Ops.push_back(Input);
+ return Ops.size() - 1;
+ };
- unsigned RootMaskedIdx =
- RootRatio == 1
- ? RootMask[RootIdx]
- : (RootMask[RootIdx] << RootRatioLog2) + (i & (RootRatio - 1));
+ SmallVector<int, 2> OpInputIdx;
+ for (SDValue OpInput : OpInputs)
+ OpInputIdx.push_back(
+ AddOp(OpInput, OpInputIdx.empty() ? SrcOpIndex : -1));
+
+ assert(((RootMask.size() > OpMask.size() &&
+ RootMask.size() % OpMask.size() == 0) ||
+ (OpMask.size() > RootMask.size() &&
+ OpMask.size() % RootMask.size() == 0) ||
+ OpMask.size() == RootMask.size()) &&
+ "The smaller number of elements must divide the larger.");
+
+ // This function can be performance-critical, so we rely on the power-of-2
+ // knowledge that we have about the mask sizes to replace div/rem ops with
+ // bit-masks and shifts.
+ assert(isPowerOf2_32(RootMask.size()) &&
+ "Non-power-of-2 shuffle mask sizes");
+ assert(isPowerOf2_32(OpMask.size()) && "Non-power-of-2 shuffle mask sizes");
+ unsigned RootMaskSizeLog2 = countTrailingZeros(RootMask.size());
+ unsigned OpMaskSizeLog2 = countTrailingZeros(OpMask.size());
+
+ unsigned MaskWidth = std::max<unsigned>(OpMask.size(), RootMask.size());
+ unsigned RootRatio =
+ std::max<unsigned>(1, OpMask.size() >> RootMaskSizeLog2);
+ unsigned OpRatio = std::max<unsigned>(1, RootMask.size() >> OpMaskSizeLog2);
+ assert((RootRatio == 1 || OpRatio == 1) &&
+ "Must not have a ratio for both incoming and op masks!");
+
+ assert(isPowerOf2_32(MaskWidth) && "Non-power-of-2 shuffle mask sizes");
+ assert(isPowerOf2_32(RootRatio) && "Non-power-of-2 shuffle mask sizes");
+ assert(isPowerOf2_32(OpRatio) && "Non-power-of-2 shuffle mask sizes");
+ unsigned RootRatioLog2 = countTrailingZeros(RootRatio);
+ unsigned OpRatioLog2 = countTrailingZeros(OpRatio);
+
+ Mask.resize(MaskWidth, SM_SentinelUndef);
+
+ // Merge this shuffle operation's mask into our accumulated mask. Note that
+ // this shuffle's mask will be the first applied to the input, followed by
+ // the root mask to get us all the way to the root value arrangement. The
+ // reason for this order is that we are recursing up the operation chain.
+ for (unsigned i = 0; i < MaskWidth; ++i) {
+ unsigned RootIdx = i >> RootRatioLog2;
+ if (RootMask[RootIdx] < 0) {
+ // This is a zero or undef lane, we're done.
+ Mask[i] = RootMask[RootIdx];
+ continue;
+ }
- // Just insert the scaled root mask value if it references an input other
- // than the SrcOp we're currently inserting.
- if ((RootMaskedIdx < (SrcOpIndex * MaskWidth)) ||
- (((SrcOpIndex + 1) * MaskWidth) <= RootMaskedIdx)) {
- Mask[i] = RootMaskedIdx;
- continue;
- }
+ unsigned RootMaskedIdx =
+ RootRatio == 1
+ ? RootMask[RootIdx]
+ : (RootMask[RootIdx] << RootRatioLog2) + (i & (RootRatio - 1));
- RootMaskedIdx = RootMaskedIdx & (MaskWidth - 1);
- unsigned OpIdx = RootMaskedIdx >> OpRatioLog2;
- if (OpMask[OpIdx] < 0) {
- // The incoming lanes are zero or undef, it doesn't matter which ones we
- // are using.
- Mask[i] = OpMask[OpIdx];
- continue;
- }
+ // Just insert the scaled root mask value if it references an input other
+ // than the SrcOp we're currently inserting.
+ if ((RootMaskedIdx < (SrcOpIndex * MaskWidth)) ||
+ (((SrcOpIndex + 1) * MaskWidth) <= RootMaskedIdx)) {
+ Mask[i] = RootMaskedIdx;
+ continue;
+ }
- // Ok, we have non-zero lanes, map them through to one of the Op's inputs.
- unsigned OpMaskedIdx =
- OpRatio == 1
- ? OpMask[OpIdx]
- : (OpMask[OpIdx] << OpRatioLog2) + (RootMaskedIdx & (OpRatio - 1));
+ RootMaskedIdx = RootMaskedIdx & (MaskWidth - 1);
+ unsigned OpIdx = RootMaskedIdx >> OpRatioLog2;
+ if (OpMask[OpIdx] < 0) {
+ // The incoming lanes are zero or undef, it doesn't matter which ones we
+ // are using.
+ Mask[i] = OpMask[OpIdx];
+ continue;
+ }
- OpMaskedIdx = OpMaskedIdx & (MaskWidth - 1);
- int InputIdx = OpMask[OpIdx] / (int)OpMask.size();
- assert(0 <= OpInputIdx[InputIdx] && "Unknown target shuffle input");
- OpMaskedIdx += OpInputIdx[InputIdx] * MaskWidth;
+ // Ok, we have non-zero lanes, map them through to one of the Op's inputs.
+ unsigned OpMaskedIdx = OpRatio == 1 ? OpMask[OpIdx]
+ : (OpMask[OpIdx] << OpRatioLog2) +
+ (RootMaskedIdx & (OpRatio - 1));
- Mask[i] = OpMaskedIdx;
+ OpMaskedIdx = OpMaskedIdx & (MaskWidth - 1);
+ int InputIdx = OpMask[OpIdx] / (int)OpMask.size();
+ assert(0 <= OpInputIdx[InputIdx] && "Unknown target shuffle input");
+ OpMaskedIdx += OpInputIdx[InputIdx] * MaskWidth;
+
+ Mask[i] = OpMaskedIdx;
+ }
}
// Remove unused/repeated shuffle source ops.
More information about the llvm-commits
mailing list