[llvm] d1426cd - [DAG] visitAnd - fold (and (ext (and V, c1)), c2) -> (and (ext V), (and c1, (ext c2)))
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Mon Jan 23 06:28:55 PST 2023
Author: Simon Pilgrim
Date: 2023-01-23T14:28:37Z
New Revision: d1426cd4848b42c4a97fac3558404f9117100394
URL: https://github.com/llvm/llvm-project/commit/d1426cd4848b42c4a97fac3558404f9117100394
DIFF: https://github.com/llvm/llvm-project/commit/d1426cd4848b42c4a97fac3558404f9117100394.diff
LOG: [DAG] visitAnd - fold (and (ext (and V, c1)), c2) -> (and (ext V), (and c1, (ext c2)))
Also, move the XformToShuffleWithZero and combineCarryDiamond folds later after some of the more basic canonicalizations/combines (such as this) have had a chance to occur
Fixes the v8i1-masks.ll regression from D127115
Added:
Modified:
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
llvm/test/CodeGen/AArch64/sve-fixed-length-fp-select.ll
llvm/test/CodeGen/AArch64/sve-fixed-length-int-select.ll
llvm/test/CodeGen/X86/combine-and.ll
Removed:
################################################################################
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 73172bb5c1de1..0a3ebd73d2722 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -6337,14 +6337,6 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
if (SDValue RAND = reassociateOps(ISD::AND, SDLoc(N), N0, N1, N->getFlags()))
return RAND;
- // Try to convert a constant mask AND into a shuffle clear mask.
- if (VT.isVector())
- if (SDValue Shuffle = XformToShuffleWithZero(N))
- return Shuffle;
-
- if (SDValue Combined = combineCarryDiamond(DAG, TLI, N0, N1, N))
- return Combined;
-
// fold (and (or x, C), D) -> D if (C & D) == D
auto MatchSubset = [](ConstantSDNode *LHS, ConstantSDNode *RHS) {
return RHS->getAPIntValue().isSubsetOf(LHS->getAPIntValue());
@@ -6352,6 +6344,7 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
if (N0.getOpcode() == ISD::OR &&
ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchSubset))
return N1;
+
// fold (and (any_ext V), c) -> (zero_ext V) if 'and' only clears top bits.
if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
SDValue N0Op0 = N0.getOperand(0);
@@ -6361,6 +6354,25 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), N0.getValueType(), N0Op0);
}
+ // fold (and (ext (and V, c1)), c2) -> (and (ext V), (and c1, (ext c2)))
+ if (ISD::isExtOpcode(N0.getOpcode())) {
+ unsigned ExtOpc = N0.getOpcode();
+ SDValue N0Op0 = N0.getOperand(0);
+ if (N0Op0.getOpcode() == ISD::AND &&
+ (ExtOpc != ISD::ZERO_EXTEND || !TLI.isZExtFree(N0Op0, VT)) &&
+ DAG.isConstantIntBuildVectorOrConstantInt(N1) &&
+ DAG.isConstantIntBuildVectorOrConstantInt(N0Op0.getOperand(1)) &&
+ N0->hasOneUse() && N0Op0->hasOneUse()) {
+ SDLoc DL(N);
+ SDValue NewMask =
+ DAG.getNode(ISD::AND, DL, VT, N1,
+ DAG.getNode(ExtOpc, DL, VT, N0Op0.getOperand(1)));
+ return DAG.getNode(ISD::AND, DL, VT,
+ DAG.getNode(ExtOpc, DL, VT, N0Op0.getOperand(0)),
+ NewMask);
+ }
+ }
+
// similarly fold (and (X (load ([non_ext|any_ext|zero_ext] V))), c) ->
// (X (load ([non_ext|zero_ext] V))) if 'and' only clears top bits which must
// already be zero by virtue of the width of the base type of the load.
@@ -6466,6 +6478,14 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
}
}
+ // Try to convert a constant mask AND into a shuffle clear mask.
+ if (VT.isVector())
+ if (SDValue Shuffle = XformToShuffleWithZero(N))
+ return Shuffle;
+
+ if (SDValue Combined = combineCarryDiamond(DAG, TLI, N0, N1, N))
+ return Combined;
+
if (N0.getOpcode() == ISD::EXTRACT_SUBVECTOR && N0.hasOneUse() && N1C &&
ISD::isExtOpcode(N0.getOperand(0).getOpcode())) {
SDValue Ext = N0.getOperand(0);
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-select.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-select.ll
index 553acc7930982..91c16d08b8fdf 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-select.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-select.ll
@@ -282,8 +282,9 @@ define <2 x double> @select_v2f64(<2 x double> %op1, <2 x double> %op2, i1 %mask
define void @select_v4f64(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 {
; CHECK-LABEL: select_v4f64:
; CHECK: // %bb.0:
-; CHECK-NEXT: and w8, w2, #0x1
+; CHECK-NEXT: // kill: def $w2 killed $w2 def $x2
; CHECK-NEXT: ptrue p0.d, vl4
+; CHECK-NEXT: and x8, x2, #0x1
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
; CHECK-NEXT: ptrue p1.d
@@ -303,8 +304,9 @@ define void @select_v8f64(ptr %a, ptr %b, i1 %mask) #0 {
; VBITS_GE_256-LABEL: select_v8f64:
; VBITS_GE_256: // %bb.0:
; VBITS_GE_256-NEXT: mov x8, #4
+; VBITS_GE_256-NEXT: // kill: def $w2 killed $w2 def $x2
; VBITS_GE_256-NEXT: ptrue p0.d, vl4
-; VBITS_GE_256-NEXT: and w9, w2, #0x1
+; VBITS_GE_256-NEXT: and x9, x2, #0x1
; VBITS_GE_256-NEXT: ptrue p1.d
; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]
@@ -320,8 +322,9 @@ define void @select_v8f64(ptr %a, ptr %b, i1 %mask) #0 {
;
; VBITS_GE_512-LABEL: select_v8f64:
; VBITS_GE_512: // %bb.0:
-; VBITS_GE_512-NEXT: and w8, w2, #0x1
+; VBITS_GE_512-NEXT: // kill: def $w2 killed $w2 def $x2
; VBITS_GE_512-NEXT: ptrue p0.d, vl8
+; VBITS_GE_512-NEXT: and x8, x2, #0x1
; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]
; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]
; VBITS_GE_512-NEXT: ptrue p1.d
@@ -340,8 +343,9 @@ define void @select_v8f64(ptr %a, ptr %b, i1 %mask) #0 {
define void @select_v16f64(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 {
; CHECK-LABEL: select_v16f64:
; CHECK: // %bb.0:
-; CHECK-NEXT: and w8, w2, #0x1
+; CHECK-NEXT: // kill: def $w2 killed $w2 def $x2
; CHECK-NEXT: ptrue p0.d, vl16
+; CHECK-NEXT: and x8, x2, #0x1
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
; CHECK-NEXT: ptrue p1.d
@@ -360,8 +364,9 @@ define void @select_v16f64(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 {
define void @select_v32f64(ptr %a, ptr %b, i1 %mask) vscale_range(16,0) #0 {
; CHECK-LABEL: select_v32f64:
; CHECK: // %bb.0:
-; CHECK-NEXT: and w8, w2, #0x1
+; CHECK-NEXT: // kill: def $w2 killed $w2 def $x2
; CHECK-NEXT: ptrue p0.d, vl32
+; CHECK-NEXT: and x8, x2, #0x1
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
; CHECK-NEXT: ptrue p1.d
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-select.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-select.ll
index 19d7c4212842f..e62dc5900e843 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-select.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-select.ll
@@ -406,8 +406,9 @@ define <2 x i64> @select_v2i64(<2 x i64> %op1, <2 x i64> %op2, i1 %mask) vscale_
define void @select_v4i64(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 {
; CHECK-LABEL: select_v4i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: and w8, w2, #0x1
+; CHECK-NEXT: // kill: def $w2 killed $w2 def $x2
; CHECK-NEXT: ptrue p0.d, vl4
+; CHECK-NEXT: and x8, x2, #0x1
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
; CHECK-NEXT: ptrue p1.d
@@ -427,8 +428,9 @@ define void @select_v8i64(ptr %a, ptr %b, i1 %mask) #0 {
; VBITS_GE_256-LABEL: select_v8i64:
; VBITS_GE_256: // %bb.0:
; VBITS_GE_256-NEXT: mov x8, #4
+; VBITS_GE_256-NEXT: // kill: def $w2 killed $w2 def $x2
; VBITS_GE_256-NEXT: ptrue p0.d, vl4
-; VBITS_GE_256-NEXT: and w9, w2, #0x1
+; VBITS_GE_256-NEXT: and x9, x2, #0x1
; VBITS_GE_256-NEXT: ptrue p1.d
; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]
@@ -444,8 +446,9 @@ define void @select_v8i64(ptr %a, ptr %b, i1 %mask) #0 {
;
; VBITS_GE_512-LABEL: select_v8i64:
; VBITS_GE_512: // %bb.0:
-; VBITS_GE_512-NEXT: and w8, w2, #0x1
+; VBITS_GE_512-NEXT: // kill: def $w2 killed $w2 def $x2
; VBITS_GE_512-NEXT: ptrue p0.d, vl8
+; VBITS_GE_512-NEXT: and x8, x2, #0x1
; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]
; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]
; VBITS_GE_512-NEXT: ptrue p1.d
@@ -464,8 +467,9 @@ define void @select_v8i64(ptr %a, ptr %b, i1 %mask) #0 {
define void @select_v16i64(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 {
; CHECK-LABEL: select_v16i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: and w8, w2, #0x1
+; CHECK-NEXT: // kill: def $w2 killed $w2 def $x2
; CHECK-NEXT: ptrue p0.d, vl16
+; CHECK-NEXT: and x8, x2, #0x1
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
; CHECK-NEXT: ptrue p1.d
@@ -484,8 +488,9 @@ define void @select_v16i64(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 {
define void @select_v32i64(ptr %a, ptr %b, i1 %mask) vscale_range(16,0) #0 {
; CHECK-LABEL: select_v32i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: and w8, w2, #0x1
+; CHECK-NEXT: // kill: def $w2 killed $w2 def $x2
; CHECK-NEXT: ptrue p0.d, vl32
+; CHECK-NEXT: and x8, x2, #0x1
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]
; CHECK-NEXT: ptrue p1.d
diff --git a/llvm/test/CodeGen/X86/combine-and.ll b/llvm/test/CodeGen/X86/combine-and.ll
index 4891af4f6e586..3c556a42f0d8d 100644
--- a/llvm/test/CodeGen/X86/combine-and.ll
+++ b/llvm/test/CodeGen/X86/combine-and.ll
@@ -386,46 +386,39 @@ define <8 x i16> @and_or_v8i16(<8 x i16> %a0) {
}
;
-; TODO: Failure to merge and(ext(and(x,c1)),c2) if one gets folded to a shuffle clear mask
+; Check we merge and(ext(and(x,c1)),c2) before an and gets folded to a shuffle clear mask
;
define <8 x i32> @clear_sext_and(<8 x i16> %x) {
; SSE-LABEL: clear_sext_and:
; SSE: # %bb.0:
-; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; SSE-NEXT: pmovsxwd %xmm1, %xmm1
-; SSE-NEXT: pmovsxwd %xmm0, %xmm0
-; SSE-NEXT: pxor %xmm2, %xmm2
-; SSE-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; SSE-NEXT: pblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3,4,5,6,7]
+; SSE-NEXT: pmovsxwd %xmm0, %xmm2
+; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE-NEXT: pmovsxwd %xmm0, %xmm1
+; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE-NEXT: movdqa %xmm2, %xmm0
; SSE-NEXT: retq
;
; AVX1-LABEL: clear_sext_and:
; AVX1: # %bb.0:
-; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
; AVX1-NEXT: vpmovsxwd %xmm0, %xmm1
; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; AVX1-NEXT: vpmovsxwd %xmm0, %xmm0
; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3,4],ymm0[5,6,7]
+; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: clear_sext_and:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
; AVX2-NEXT: vpmovsxwd %xmm0, %ymm0
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3,4],ymm0[5,6,7]
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: clear_sext_and:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
; AVX512-NEXT: vpmovsxwd %xmm0, %ymm0
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3,4],ymm0[5,6,7]
+; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX512-NEXT: retq
%1 = and <8 x i16> %x, <i16 -1, i16 3, i16 7, i16 15, i16 31, i16 63, i16 127, i16 -1>
%2 = sext <8 x i16> %1 to <8 x i32>
@@ -436,40 +429,31 @@ define <8 x i32> @clear_sext_and(<8 x i16> %x) {
define <8 x i32> @clear_zext_and(<8 x i16> %x) {
; SSE-LABEL: clear_zext_and:
; SSE: # %bb.0:
+; SSE-NEXT: movdqa %xmm0, %xmm1
+; SSE-NEXT: pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; SSE-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
+; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE-NEXT: pxor %xmm1, %xmm1
-; SSE-NEXT: pmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; SSE-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; SSE-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; SSE-NEXT: pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,3,4,5,6,7]
-; SSE-NEXT: movdqa %xmm2, %xmm0
; SSE-NEXT: retq
;
; AVX1-LABEL: clear_zext_and:
; AVX1: # %bb.0:
-; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm0[4,4,5,5,6,6,7,7]
; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3,4],ymm0[5,6,7]
+; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: clear_zext_and:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3,4],ymm0[5,6,7]
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: clear_zext_and:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3,4],ymm0[5,6,7]
+; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX512-NEXT: retq
%1 = and <8 x i16> %x, <i16 -1, i16 3, i16 7, i16 15, i16 31, i16 63, i16 127, i16 -1>
%2 = zext <8 x i16> %1 to <8 x i32>
More information about the llvm-commits
mailing list