[llvm] 55f15f9 - [AggressiveInstCombine] foldGuardedRotateToFunnelShift - generalize rotation to funnel shift matcher.
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Mon Nov 2 09:09:33 PST 2020
Author: Simon Pilgrim
Date: 2020-11-02T17:09:17Z
New Revision: 55f15f99cbb33ce39fa44b6d2c90eef7ad9a376d
URL: https://github.com/llvm/llvm-project/commit/55f15f99cbb33ce39fa44b6d2c90eef7ad9a376d
DIFF: https://github.com/llvm/llvm-project/commit/55f15f99cbb33ce39fa44b6d2c90eef7ad9a376d.diff
LOG: [AggressiveInstCombine] foldGuardedRotateToFunnelShift - generalize rotation to funnel shift matcher.
Replace matchRotate with a more general matchFunnelShift - at the moment this is still just used for rotation patterns.
Added:
Modified:
llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
index 77c794037d6e..e7fb699d9fda 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
+++ b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
@@ -81,29 +81,31 @@ static bool foldGuardedRotateToFunnelShift(Instruction &I) {
if (!isPowerOf2_32(I.getType()->getScalarSizeInBits()))
return false;
- // Match V to funnel shift left/right and capture the source operand and
- // shift amount in X and Y.
- auto matchRotate = [](Value *V, Value *&X, Value *&Y) {
- Value *L0, *L1, *R0, *R1;
+ // Match V to funnel shift left/right and capture the source operands and
+ // shift amount.
+ auto matchFunnelShift = [](Value *V, Value *&ShVal0, Value *&ShVal1,
+ Value *&ShAmt) {
+ Value *SubAmt;
unsigned Width = V->getType()->getScalarSizeInBits();
- auto Sub = m_Sub(m_SpecificInt(Width), m_Value(R1));
-
- // rotate_left(X, Y) == (X << Y) | (X >> (Width - Y))
- auto RotL = m_OneUse(
- m_c_Or(m_Shl(m_Value(L0), m_Value(L1)), m_LShr(m_Value(R0), Sub)));
- if (RotL.match(V) && L0 == R0 && L1 == R1) {
- X = L0;
- Y = L1;
- return Intrinsic::fshl;
+
+ // fshl(ShVal0, ShVal1, ShAmt)
+ // == (ShVal0 << ShAmt) | (ShVal1 >> (Width -ShAmt))
+ if (match(V, m_OneUse(m_c_Or(
+ m_Shl(m_Value(ShVal0), m_Value(ShAmt)),
+ m_LShr(m_Value(ShVal1),
+ m_Sub(m_SpecificInt(Width), m_Value(SubAmt))))))) {
+ if (ShAmt == SubAmt) // TODO: Use m_Specific
+ return Intrinsic::fshl;
}
- // rotate_right(X, Y) == (X >> Y) | (X << (Width - Y))
- auto RotR = m_OneUse(
- m_c_Or(m_LShr(m_Value(L0), m_Value(L1)), m_Shl(m_Value(R0), Sub)));
- if (RotR.match(V) && L0 == R0 && L1 == R1) {
- X = L0;
- Y = L1;
- return Intrinsic::fshr;
+ // fshr(ShVal0, ShVal1, ShAmt)
+ // == (ShVal0 >> ShAmt) | (ShVal1 << (Width - ShAmt))
+ if (match(V,
+ m_OneUse(m_c_Or(m_Shl(m_Value(ShVal0), m_Sub(m_SpecificInt(Width),
+ m_Value(SubAmt))),
+ m_LShr(m_Value(ShVal1), m_Value(ShAmt)))))) {
+ if (ShAmt == SubAmt) // TODO: Use m_Specific
+ return Intrinsic::fshr;
}
return Intrinsic::not_intrinsic;
@@ -111,29 +113,32 @@ static bool foldGuardedRotateToFunnelShift(Instruction &I) {
// One phi operand must be a rotate operation, and the other phi operand must
// be the source value of that rotate operation:
- // phi [ rotate(RotSrc, RotAmt), RotBB ], [ RotSrc, GuardBB ]
+ // phi [ rotate(RotSrc, ShAmt), FunnelBB ], [ RotSrc, GuardBB ]
PHINode &Phi = cast<PHINode>(I);
+ unsigned FunnelOp = 0, GuardOp = 1;
Value *P0 = Phi.getOperand(0), *P1 = Phi.getOperand(1);
- Value *RotSrc, *RotAmt;
- Intrinsic::ID IID = matchRotate(P0, RotSrc, RotAmt);
- if (IID == Intrinsic::not_intrinsic || RotSrc != P1) {
- IID = matchRotate(P1, RotSrc, RotAmt);
- if (IID == Intrinsic::not_intrinsic || RotSrc != P0)
+ Value *ShVal0, *ShVal1, *ShAmt;
+ Intrinsic::ID IID = matchFunnelShift(P0, ShVal0, ShVal1, ShAmt);
+ if (IID == Intrinsic::not_intrinsic || ShVal0 != ShVal1 || ShVal0 != P1) {
+ IID = matchFunnelShift(P1, ShVal0, ShVal1, ShAmt);
+ if (IID == Intrinsic::not_intrinsic || ShVal0 != ShVal1 || ShVal0 != P0)
return false;
assert((IID == Intrinsic::fshl || IID == Intrinsic::fshr) &&
"Pattern must match funnel shift left or right");
+ std::swap(FunnelOp, GuardOp);
}
+ assert(ShVal0 == ShVal1 && "Rotation funnel shift pattern expected");
// The incoming block with our source operand must be the "guard" block.
// That must contain a cmp+branch to avoid the rotate when the shift amount
// is equal to 0. The other incoming block is the block with the rotate.
- BasicBlock *GuardBB = Phi.getIncomingBlock(RotSrc == P1);
- BasicBlock *RotBB = Phi.getIncomingBlock(RotSrc != P1);
+ BasicBlock *GuardBB = Phi.getIncomingBlock(GuardOp);
+ BasicBlock *FunnelBB = Phi.getIncomingBlock(FunnelOp);
Instruction *TermI = GuardBB->getTerminator();
ICmpInst::Predicate Pred;
BasicBlock *PhiBB = Phi.getParent();
- if (!match(TermI, m_Br(m_ICmp(Pred, m_Specific(RotAmt), m_ZeroInt()),
- m_SpecificBB(PhiBB), m_SpecificBB(RotBB))))
+ if (!match(TermI, m_Br(m_ICmp(Pred, m_Specific(ShAmt), m_ZeroInt()),
+ m_SpecificBB(PhiBB), m_SpecificBB(FunnelBB))))
return false;
if (Pred != CmpInst::ICMP_EQ)
@@ -141,21 +146,21 @@ static bool foldGuardedRotateToFunnelShift(Instruction &I) {
// We matched a variation of this IR pattern:
// GuardBB:
- // %cmp = icmp eq i32 %RotAmt, 0
- // br i1 %cmp, label %PhiBB, label %RotBB
- // RotBB:
- // %sub = sub i32 32, %RotAmt
- // %shr = lshr i32 %X, %sub
- // %shl = shl i32 %X, %RotAmt
+ // %cmp = icmp eq i32 %ShAmt, 0
+ // br i1 %cmp, label %PhiBB, label %FunnelBB
+ // FunnelBB:
+ // %sub = sub i32 32, %ShAmt
+ // %shr = lshr i32 %RotSrc, %sub
+ // %shl = shl i32 %RotSrc, %ShAmt
// %rot = or i32 %shr, %shl
// br label %PhiBB
// PhiBB:
- // %cond = phi i32 [ %rot, %RotBB ], [ %X, %GuardBB ]
+ // %cond = phi i32 [ %RotSrc, %FunnelBB ], [ %RotSrc, %GuardBB ]
// -->
- // llvm.fshl.i32(i32 %X, i32 %RotAmt)
+ // llvm.fshl.i32(i32 %RotSrc, i32 %RotSrc, i32 %ShAmt)
IRBuilder<> Builder(PhiBB, PhiBB->getFirstInsertionPt());
Function *F = Intrinsic::getDeclaration(Phi.getModule(), IID, Phi.getType());
- Phi.replaceAllUsesWith(Builder.CreateCall(F, {RotSrc, RotSrc, RotAmt}));
+ Phi.replaceAllUsesWith(Builder.CreateCall(F, {ShVal0, ShVal1, ShAmt}));
++NumGuardedRotates;
return true;
}
More information about the llvm-commits
mailing list