[llvm] r331937 - [AggressiveInstCombine] convert a chain of 'and-shift' bits into masked compare
Sanjay Patel via llvm-commits
llvm-commits at lists.llvm.org
Wed May 9 16:08:15 PDT 2018
Author: spatel
Date: Wed May 9 16:08:15 2018
New Revision: 331937
URL: http://llvm.org/viewvc/llvm-project?rev=331937&view=rev
Log:
[AggressiveInstCombine] convert a chain of 'and-shift' bits into masked compare
This is a follow-up to D45986. As suggested there, we should match the "all-bits-set"
pattern in addition to "any-bits-set".
This was a little more complicated than I thought it would be initially because the
"and 1" instruction can be anywhere in the chain. Hopefully, the code comments make
that logic understandable, but if you see a way to simplify or improve that, it's
most appreciated.
This transforms patterns that emerge from bitfield tests as seen in PR37098:
https://bugs.llvm.org/show_bug.cgi?id=37098
I think it would also help reduce the large test from:
D46336
D46595
but we need something to reassociate that case to the forms we're expecting here first.
Differential Revision: https://reviews.llvm.org/D46649
Modified:
llvm/trunk/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
llvm/trunk/test/Transforms/AggressiveInstCombine/masked-cmp.ll
llvm/trunk/test/Transforms/PhaseOrdering/bitfield-bittests.ll
Modified: llvm/trunk/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp?rev=331937&r1=331936&r2=331937&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp (original)
+++ llvm/trunk/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp Wed May 9 16:08:15 2018
@@ -57,18 +57,45 @@ public:
};
} // namespace
-/// This is a recursive helper for 'and X, 1' that walks through a chain of 'or'
-/// instructions looking for shift ops of a common source value (first member of
-/// the pair). The second member of the pair is a mask constant for all of the
-/// bits that are being compared. So this:
-/// or (or (or X, (X >> 3)), (X >> 5)), (X >> 8)
-/// returns {X, 0x129} and those are the operands of an 'and' that is compared
-/// to zero.
-static bool matchMaskedCmpOp(Value *V, std::pair<Value *, APInt> &Result) {
- // Recurse through a chain of 'or' operands.
+/// This is used by foldAnyOrAllBitsSet() to capture a source value (Root) and
+/// the bit indexes (Mask) needed by a masked compare. If we're matching a chain
+/// of 'and' ops, then we also need to capture the fact that we saw an
+/// "and X, 1", so that's an extra return value for that case.
+struct MaskOps {
+ Value *Root;
+ APInt Mask;
+ bool MatchAndChain;
+ bool FoundAnd1;
+
+ MaskOps(unsigned BitWidth, bool MatchAnds) :
+ Root(nullptr), Mask(APInt::getNullValue(BitWidth)),
+ MatchAndChain(MatchAnds), FoundAnd1(false) {}
+};
+
+/// This is a recursive helper for foldAnyOrAllBitsSet() that walks through a
+/// chain of 'and' or 'or' instructions looking for shift ops of a common source
+/// value. Examples:
+/// or (or (or X, (X >> 3)), (X >> 5)), (X >> 8)
+/// returns { X, 0x129 }
+/// and (and (X >> 1), 1), (X >> 4)
+/// returns { X, 0x12 }
+static bool matchAndOrChain(Value *V, MaskOps &MOps) {
Value *Op0, *Op1;
- if (match(V, m_Or(m_Value(Op0), m_Value(Op1))))
- return matchMaskedCmpOp(Op0, Result) && matchMaskedCmpOp(Op1, Result);
+ if (MOps.MatchAndChain) {
+ // Recurse through a chain of 'and' operands. This requires an extra check
+ // vs. the 'or' matcher: we must find an "and X, 1" instruction somewhere
+ // in the chain to know that all of the high bits are cleared.
+ if (match(V, m_And(m_Value(Op0), m_One()))) {
+ MOps.FoundAnd1 = true;
+ return matchAndOrChain(Op0, MOps);
+ }
+ if (match(V, m_And(m_Value(Op0), m_Value(Op1))))
+ return matchAndOrChain(Op0, MOps) && matchAndOrChain(Op1, MOps);
+ } else {
+ // Recurse through a chain of 'or' operands.
+ if (match(V, m_Or(m_Value(Op0), m_Value(Op1))))
+ return matchAndOrChain(Op0, MOps) && matchAndOrChain(Op1, MOps);
+ }
// We need a shift-right or a bare value representing a compare of bit 0 of
// the original source operand.
@@ -78,33 +105,50 @@ static bool matchMaskedCmpOp(Value *V, s
Candidate = V;
// Initialize result source operand.
- if (!Result.first)
- Result.first = Candidate;
+ if (!MOps.Root)
+ MOps.Root = Candidate;
// Fill in the mask bit derived from the shift constant.
- Result.second.setBit(BitIndex);
- return Result.first == Candidate;
+ MOps.Mask.setBit(BitIndex);
+ return MOps.Root == Candidate;
}
-/// Match an 'and' of a chain of or-shifted bits from a common source value into
-/// a masked compare:
-/// and (or (lshr X, C), ...), 1 --> (X & C') != 0
-static bool foldToMaskedCmp(Instruction &I) {
- // TODO: This is only looking for 'any-bits-set' and 'all-bits-clear'.
- // We should also match 'all-bits-set' and 'any-bits-clear' by looking for a
- // a chain of 'and'.
- if (!match(&I, m_And(m_OneUse(m_Or(m_Value(), m_Value())), m_One())))
+/// Match patterns that correspond to "any-bits-set" and "all-bits-set".
+/// These will include a chain of 'or' or 'and'-shifted bits from a
+/// common source value:
+/// and (or (lshr X, C), ...), 1 --> (X & CMask) != 0
+/// and (and (lshr X, C), ...), 1 --> (X & CMask) == CMask
+/// Note: "any-bits-clear" and "all-bits-clear" are variations of these patterns
+/// that differ only with a final 'not' of the result. We expect that final
+/// 'not' to be folded with the compare that we create here (invert predicate).
+static bool foldAnyOrAllBitsSet(Instruction &I) {
+ // The 'any-bits-set' ('or' chain) pattern is simpler to match because the
+ // final "and X, 1" instruction must be the final op in the sequence.
+ bool MatchAllBitsSet;
+ if (match(&I, m_c_And(m_OneUse(m_And(m_Value(), m_Value())), m_Value())))
+ MatchAllBitsSet = true;
+ else if (match(&I, m_And(m_OneUse(m_Or(m_Value(), m_Value())), m_One())))
+ MatchAllBitsSet = false;
+ else
return false;
- std::pair<Value *, APInt>
- MaskOps(nullptr, APInt::getNullValue(I.getType()->getScalarSizeInBits()));
- if (!matchMaskedCmpOp(cast<BinaryOperator>(&I)->getOperand(0), MaskOps))
- return false;
+ MaskOps MOps(I.getType()->getScalarSizeInBits(), MatchAllBitsSet);
+ if (MatchAllBitsSet) {
+ if (!matchAndOrChain(cast<BinaryOperator>(&I), MOps) || !MOps.FoundAnd1)
+ return false;
+ } else {
+ if (!matchAndOrChain(cast<BinaryOperator>(&I)->getOperand(0), MOps))
+ return false;
+ }
+ // The pattern was found. Create a masked compare that replaces all of the
+ // shift and logic ops.
IRBuilder<> Builder(&I);
- Value *Mask = Builder.CreateAnd(MaskOps.first, MaskOps.second);
- Value *CmpZero = Builder.CreateIsNotNull(Mask);
- Value *Zext = Builder.CreateZExt(CmpZero, I.getType());
+ Constant *Mask = ConstantInt::get(I.getType(), MOps.Mask);
+ Value *And = Builder.CreateAnd(MOps.Root, Mask);
+ Value *Cmp = MatchAllBitsSet ? Builder.CreateICmpEQ(And, Mask) :
+ Builder.CreateIsNotNull(And);
+ Value *Zext = Builder.CreateZExt(Cmp, I.getType());
I.replaceAllUsesWith(Zext);
return true;
}
@@ -119,8 +163,13 @@ static bool foldUnusualPatterns(Function
if (!DT.isReachableFromEntry(&BB))
continue;
// Do not delete instructions under here and invalidate the iterator.
- for (Instruction &I : BB)
- MadeChange |= foldToMaskedCmp(I);
+ // Walk the block backwards for efficiency. We're matching a chain of
+ // use->defs, so we're more likely to succeed by starting from the bottom.
+ // Also, we want to avoid matching partial patterns.
+ // TODO: It would be more efficient if we removed dead instructions
+ // iteratively in this loop rather than waiting until the end.
+ for (Instruction &I : make_range(BB.rbegin(), BB.rend()))
+ MadeChange |= foldAnyOrAllBitsSet(I);
}
// We're done with transforms, so remove dead instructions.
Modified: llvm/trunk/test/Transforms/AggressiveInstCombine/masked-cmp.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/AggressiveInstCombine/masked-cmp.ll?rev=331937&r1=331936&r2=331937&view=diff
==============================================================================
--- llvm/trunk/test/Transforms/AggressiveInstCombine/masked-cmp.ll (original)
+++ llvm/trunk/test/Transforms/AggressiveInstCombine/masked-cmp.ll Wed May 9 16:08:15 2018
@@ -51,19 +51,27 @@ define i32 @anyset_three_bit_mask_all_sh
ret i32 %r
}
-; TODO: Recognize the 'and' sibling pattern. The 'and 1' may not be at the end.
+; Recognize the 'and' sibling pattern (all-bits-set). The 'and 1' may not be at the end.
+
+define i32 @allset_two_bit_mask(i32 %x) {
+; CHECK-LABEL: @allset_two_bit_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = and i32 [[X:%.*]], 129
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 129
+; CHECK-NEXT: [[TMP3:%.*]] = zext i1 [[TMP2]] to i32
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %s = lshr i32 %x, 7
+ %o = and i32 %s, %x
+ %r = and i32 %o, 1
+ ret i32 %r
+}
define i64 @allset_four_bit_mask(i64 %x) {
; CHECK-LABEL: @allset_four_bit_mask(
-; CHECK-NEXT: [[T1:%.*]] = lshr i64 [[X:%.*]], 1
-; CHECK-NEXT: [[T2:%.*]] = lshr i64 [[X]], 2
-; CHECK-NEXT: [[T3:%.*]] = lshr i64 [[X]], 3
-; CHECK-NEXT: [[T4:%.*]] = lshr i64 [[X]], 4
-; CHECK-NEXT: [[A1:%.*]] = and i64 [[T4]], 1
-; CHECK-NEXT: [[A2:%.*]] = and i64 [[T2]], [[A1]]
-; CHECK-NEXT: [[A3:%.*]] = and i64 [[A2]], [[T1]]
-; CHECK-NEXT: [[R:%.*]] = and i64 [[A3]], [[T3]]
-; CHECK-NEXT: ret i64 [[R]]
+; CHECK-NEXT: [[TMP1:%.*]] = and i64 [[X:%.*]], 30
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 30
+; CHECK-NEXT: [[TMP3:%.*]] = zext i1 [[TMP2]] to i64
+; CHECK-NEXT: ret i64 [[TMP3]]
;
%t1 = lshr i64 %x, 1
%t2 = lshr i64 %x, 2
@@ -76,3 +84,136 @@ define i64 @allset_four_bit_mask(i64 %x)
ret i64 %r
}
+declare void @use(i32)
+
+; negative test - extra use means the transform would increase instruction count
+
+define i32 @allset_two_bit_mask_multiuse(i32 %x) {
+; CHECK-LABEL: @allset_two_bit_mask_multiuse(
+; CHECK-NEXT: [[S:%.*]] = lshr i32 [[X:%.*]], 7
+; CHECK-NEXT: [[O:%.*]] = and i32 [[S]], [[X]]
+; CHECK-NEXT: [[R:%.*]] = and i32 [[O]], 1
+; CHECK-NEXT: call void @use(i32 [[O]])
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %s = lshr i32 %x, 7
+ %o = and i32 %s, %x
+ %r = and i32 %o, 1
+ call void @use(i32 %o)
+ ret i32 %r
+}
+
+; negative test - missing 'and 1' mask, so more than the low bit is used here
+
+define i8 @allset_three_bit_mask_no_and1(i8 %x) {
+; CHECK-LABEL: @allset_three_bit_mask_no_and1(
+; CHECK-NEXT: [[T1:%.*]] = lshr i8 [[X:%.*]], 1
+; CHECK-NEXT: [[T2:%.*]] = lshr i8 [[X]], 2
+; CHECK-NEXT: [[T3:%.*]] = lshr i8 [[X]], 3
+; CHECK-NEXT: [[A2:%.*]] = and i8 [[T1]], [[T2]]
+; CHECK-NEXT: [[R:%.*]] = and i8 [[A2]], [[T3]]
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %t1 = lshr i8 %x, 1
+ %t2 = lshr i8 %x, 2
+ %t3 = lshr i8 %x, 3
+ %a2 = and i8 %t1, %t2
+ %r = and i8 %a2, %t3
+ ret i8 %r
+}
+
+; This test demonstrates that the transform can be large. If the implementation
+; is slow or explosive (stack overflow due to recursion), it should be made efficient.
+
+define i64 @allset_40_bit_mask(i64 %x) {
+; CHECK-LABEL: @allset_40_bit_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = and i64 [[X:%.*]], 2199023255550
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 2199023255550
+; CHECK-NEXT: [[TMP3:%.*]] = zext i1 [[TMP2]] to i64
+; CHECK-NEXT: ret i64 [[TMP3]]
+;
+ %t1 = lshr i64 %x, 1
+ %t2 = lshr i64 %x, 2
+ %t3 = lshr i64 %x, 3
+ %t4 = lshr i64 %x, 4
+ %t5 = lshr i64 %x, 5
+ %t6 = lshr i64 %x, 6
+ %t7 = lshr i64 %x, 7
+ %t8 = lshr i64 %x, 8
+ %t9 = lshr i64 %x, 9
+ %t10 = lshr i64 %x, 10
+ %t11 = lshr i64 %x, 11
+ %t12 = lshr i64 %x, 12
+ %t13 = lshr i64 %x, 13
+ %t14 = lshr i64 %x, 14
+ %t15 = lshr i64 %x, 15
+ %t16 = lshr i64 %x, 16
+ %t17 = lshr i64 %x, 17
+ %t18 = lshr i64 %x, 18
+ %t19 = lshr i64 %x, 19
+ %t20 = lshr i64 %x, 20
+ %t21 = lshr i64 %x, 21
+ %t22 = lshr i64 %x, 22
+ %t23 = lshr i64 %x, 23
+ %t24 = lshr i64 %x, 24
+ %t25 = lshr i64 %x, 25
+ %t26 = lshr i64 %x, 26
+ %t27 = lshr i64 %x, 27
+ %t28 = lshr i64 %x, 28
+ %t29 = lshr i64 %x, 29
+ %t30 = lshr i64 %x, 30
+ %t31 = lshr i64 %x, 31
+ %t32 = lshr i64 %x, 32
+ %t33 = lshr i64 %x, 33
+ %t34 = lshr i64 %x, 34
+ %t35 = lshr i64 %x, 35
+ %t36 = lshr i64 %x, 36
+ %t37 = lshr i64 %x, 37
+ %t38 = lshr i64 %x, 38
+ %t39 = lshr i64 %x, 39
+ %t40 = lshr i64 %x, 40
+
+ %a1 = and i64 %t1, 1
+ %a2 = and i64 %t2, %a1
+ %a3 = and i64 %t3, %a2
+ %a4 = and i64 %t4, %a3
+ %a5 = and i64 %t5, %a4
+ %a6 = and i64 %t6, %a5
+ %a7 = and i64 %t7, %a6
+ %a8 = and i64 %t8, %a7
+ %a9 = and i64 %t9, %a8
+ %a10 = and i64 %t10, %a9
+ %a11 = and i64 %t11, %a10
+ %a12 = and i64 %t12, %a11
+ %a13 = and i64 %t13, %a12
+ %a14 = and i64 %t14, %a13
+ %a15 = and i64 %t15, %a14
+ %a16 = and i64 %t16, %a15
+ %a17 = and i64 %t17, %a16
+ %a18 = and i64 %t18, %a17
+ %a19 = and i64 %t19, %a18
+ %a20 = and i64 %t20, %a19
+ %a21 = and i64 %t21, %a20
+ %a22 = and i64 %t22, %a21
+ %a23 = and i64 %t23, %a22
+ %a24 = and i64 %t24, %a23
+ %a25 = and i64 %t25, %a24
+ %a26 = and i64 %t26, %a25
+ %a27 = and i64 %t27, %a26
+ %a28 = and i64 %t28, %a27
+ %a29 = and i64 %t29, %a28
+ %a30 = and i64 %t30, %a29
+ %a31 = and i64 %t31, %a30
+ %a32 = and i64 %t32, %a31
+ %a33 = and i64 %t33, %a32
+ %a34 = and i64 %t34, %a33
+ %a35 = and i64 %t35, %a34
+ %a36 = and i64 %t36, %a35
+ %a37 = and i64 %t37, %a36
+ %a38 = and i64 %t38, %a37
+ %a39 = and i64 %t39, %a38
+ %a40 = and i64 %t40, %a39
+
+ ret i64 %a40
+}
+
Modified: llvm/trunk/test/Transforms/PhaseOrdering/bitfield-bittests.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/PhaseOrdering/bitfield-bittests.ll?rev=331937&r1=331936&r2=331937&view=diff
==============================================================================
--- llvm/trunk/test/Transforms/PhaseOrdering/bitfield-bittests.ll (original)
+++ llvm/trunk/test/Transforms/PhaseOrdering/bitfield-bittests.ll Wed May 9 16:08:15 2018
@@ -76,14 +76,10 @@ define i32 @anyset(i32 %a) {
define i32 @allset(i32 %a) {
; CHECK-LABEL: @allset(
-; CHECK-NEXT: [[BF_LSHR:%.*]] = lshr i32 [[A:%.*]], 1
-; CHECK-NEXT: [[BF_LSHR5:%.*]] = lshr i32 [[A]], 2
-; CHECK-NEXT: [[BF_LSHR10:%.*]] = lshr i32 [[A]], 3
-; CHECK-NEXT: [[BF_CLEAR2:%.*]] = and i32 [[A]], 1
-; CHECK-NEXT: [[AND:%.*]] = and i32 [[BF_CLEAR2]], [[BF_LSHR]]
-; CHECK-NEXT: [[AND8:%.*]] = and i32 [[AND]], [[BF_LSHR5]]
-; CHECK-NEXT: [[AND13:%.*]] = and i32 [[AND8]], [[BF_LSHR10]]
-; CHECK-NEXT: ret i32 [[AND13]]
+; CHECK-NEXT: [[TMP1:%.*]] = and i32 [[A:%.*]], 15
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 15
+; CHECK-NEXT: [[TMP3:%.*]] = zext i1 [[TMP2]] to i32
+; CHECK-NEXT: ret i32 [[TMP3]]
;
%a.sroa.0.0.trunc = trunc i32 %a to i8
%a.sroa.5.0.shift = lshr i32 %a, 8
@@ -110,15 +106,10 @@ define i32 @allset(i32 %a) {
define i32 @anyclear(i32 %a) {
; CHECK-LABEL: @anyclear(
-; CHECK-NEXT: [[BF_LSHR:%.*]] = lshr i32 [[A:%.*]], 1
-; CHECK-NEXT: [[BF_LSHR5:%.*]] = lshr i32 [[A]], 2
-; CHECK-NEXT: [[BF_LSHR10:%.*]] = lshr i32 [[A]], 3
-; CHECK-NEXT: [[BF_CLEAR2:%.*]] = and i32 [[A]], 1
-; CHECK-NEXT: [[AND:%.*]] = and i32 [[BF_CLEAR2]], [[BF_LSHR]]
-; CHECK-NEXT: [[AND8:%.*]] = and i32 [[AND]], [[BF_LSHR5]]
-; CHECK-NEXT: [[AND13:%.*]] = and i32 [[AND8]], [[BF_LSHR10]]
-; CHECK-NEXT: [[TMP1:%.*]] = xor i32 [[AND13]], 1
-; CHECK-NEXT: ret i32 [[TMP1]]
+; CHECK-NEXT: [[TMP1:%.*]] = and i32 [[A:%.*]], 15
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 15
+; CHECK-NEXT: [[TMP3:%.*]] = zext i1 [[TMP2]] to i32
+; CHECK-NEXT: ret i32 [[TMP3]]
;
%a.sroa.0.0.trunc = trunc i32 %a to i8
%a.sroa.5.0.shift = lshr i32 %a, 8
More information about the llvm-commits
mailing list