[llvm] 3d8149c - [X86][SSE] Fold BITOP(MOVMSK(X),MOVMSK(Y)) -> MOVMSK(BITOP(X,Y))
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Sun Jun 14 13:44:48 PDT 2020
Author: Simon Pilgrim
Date: 2020-06-14T21:37:58+01:00
New Revision: 3d8149c2a1228609fd7d7c91a04681304a2f0ca9
URL: https://github.com/llvm/llvm-project/commit/3d8149c2a1228609fd7d7c91a04681304a2f0ca9
DIFF: https://github.com/llvm/llvm-project/commit/3d8149c2a1228609fd7d7c91a04681304a2f0ca9.diff
LOG: [X86][SSE] Fold BITOP(MOVMSK(X),MOVMSK(Y)) -> MOVMSK(BITOP(X,Y))
Reduce XMM->GPR traffic by performing bitops on the vectors, and using a single MOVMSK call.
This requires us to use vectors of the same size and element width, but we can mix fp/int type equivalents with suitable bitcasting.
Added:
Modified:
llvm/lib/Target/X86/X86ISelLowering.cpp
llvm/test/CodeGen/X86/combine-movmsk.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index ddea6fa4df95..a0d9d218982a 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -41966,6 +41966,17 @@ static SDValue PromoteMaskArithmetic(SDNode *N, SelectionDAG &DAG,
}
}
+unsigned convertIntLogicToFPLogicOpcode(unsigned Opcode) {
+ unsigned FPOpcode;
+ switch (Opcode) {
+ default: llvm_unreachable("Unexpected input node for FP logic conversion");
+ case ISD::AND: FPOpcode = X86ISD::FAND; break;
+ case ISD::OR: FPOpcode = X86ISD::FOR; break;
+ case ISD::XOR: FPOpcode = X86ISD::FXOR; break;
+ }
+ return FPOpcode;
+}
+
/// If both input operands of a logic op are being cast from floating point
/// types, try to convert this into a floating point logic node to avoid
/// unnecessary moves from SSE to integer registers.
@@ -41990,18 +42001,45 @@ static SDValue convertIntLogicToFPLogic(SDNode *N, SelectionDAG &DAG,
(Subtarget.hasSSE2() && N00Type == MVT::f64)))
return SDValue();
- unsigned FPOpcode;
- switch (N->getOpcode()) {
- default: llvm_unreachable("Unexpected input node for FP logic conversion");
- case ISD::AND: FPOpcode = X86ISD::FAND; break;
- case ISD::OR: FPOpcode = X86ISD::FOR; break;
- case ISD::XOR: FPOpcode = X86ISD::FXOR; break;
- }
-
+ unsigned FPOpcode = convertIntLogicToFPLogicOpcode(N->getOpcode());
SDValue FPLogic = DAG.getNode(FPOpcode, DL, N00Type, N00, N10);
return DAG.getBitcast(VT, FPLogic);
}
+// Attempt to fold BITOP(MOVMSK(X),MOVMSK(Y)) -> MOVMSK(BITOP(X,Y))
+// to reduce XMM->GPR traffic.
+static SDValue combineBitOpWithMOVMSK(SDNode *N, SelectionDAG &DAG) {
+ unsigned Opc = N->getOpcode();
+ assert((Opc == ISD::OR || Opc == ISD::AND || Opc == ISD::XOR) &&
+ "Unexpected bit opcode");
+
+ SDValue N0 = N->getOperand(0);
+ SDValue N1 = N->getOperand(1);
+
+ // Both operands must be single use MOVMSK.
+ if (N0.getOpcode() != X86ISD::MOVMSK || !N0.hasOneUse() ||
+ N1.getOpcode() != X86ISD::MOVMSK || !N1.hasOneUse())
+ return SDValue();
+
+ SDValue Vec0 = N0.getOperand(0);
+ SDValue Vec1 = N1.getOperand(0);
+ EVT VecVT0 = Vec0.getValueType();
+ EVT VecVT1 = Vec1.getValueType();
+
+ // Both MOVMSK operands must be from vectors of the same size and same element
+ // size, but its OK for a fp/int
diff .
+ if (VecVT0.getSizeInBits() != VecVT1.getSizeInBits() ||
+ VecVT0.getScalarSizeInBits() != VecVT1.getScalarSizeInBits())
+ return SDValue();
+
+ SDLoc DL(N);
+ unsigned VecOpc =
+ VecVT0.isFloatingPoint() ? convertIntLogicToFPLogicOpcode(Opc) : Opc;
+ SDValue Result =
+ DAG.getNode(VecOpc, DL, VecVT0, Vec0, DAG.getBitcast(VecVT0, Vec1));
+ return DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
+}
+
/// If this is a zero/all-bits result that is bitwise-anded with a low bits
/// mask. (Mask == 1 for the x86 lowering of a SETCC + ZEXT), replace the 'and'
/// with a shift-right to eliminate loading the vector constant mask value.
@@ -42347,6 +42385,9 @@ static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
if (SDValue V = combineScalarAndWithMaskSetcc(N, DAG, Subtarget))
return V;
+ if (SDValue R = combineBitOpWithMOVMSK(N, DAG))
+ return R;
+
if (DCI.isBeforeLegalizeOps())
return SDValue();
@@ -42694,6 +42735,9 @@ static SDValue combineOr(SDNode *N, SelectionDAG &DAG,
}
}
+ if (SDValue R = combineBitOpWithMOVMSK(N, DAG))
+ return R;
+
if (DCI.isBeforeLegalizeOps())
return SDValue();
@@ -44707,6 +44751,9 @@ static SDValue combineXor(SDNode *N, SelectionDAG &DAG,
if (SDValue Cmp = foldVectorXorShiftIntoCmp(N, DAG, Subtarget))
return Cmp;
+ if (SDValue R = combineBitOpWithMOVMSK(N, DAG))
+ return R;
+
if (DCI.isBeforeLegalizeOps())
return SDValue();
diff --git a/llvm/test/CodeGen/X86/combine-movmsk.ll b/llvm/test/CodeGen/X86/combine-movmsk.ll
index df5571aeccf3..7c2041a975c6 100644
--- a/llvm/test/CodeGen/X86/combine-movmsk.ll
+++ b/llvm/test/CodeGen/X86/combine-movmsk.ll
@@ -188,18 +188,16 @@ define i32 @and_movmskpd_movmskpd(<2 x double> %a0, <2 x i64> %a1) {
; SSE: # %bb.0:
; SSE-NEXT: xorpd %xmm2, %xmm2
; SSE-NEXT: cmpeqpd %xmm0, %xmm2
-; SSE-NEXT: movmskpd %xmm2, %ecx
-; SSE-NEXT: movmskpd %xmm1, %eax
-; SSE-NEXT: andl %ecx, %eax
+; SSE-NEXT: andpd %xmm1, %xmm2
+; SSE-NEXT: movmskpd %xmm2, %eax
; SSE-NEXT: retq
;
; AVX-LABEL: and_movmskpd_movmskpd:
; AVX: # %bb.0:
; AVX-NEXT: vxorpd %xmm2, %xmm2, %xmm2
; AVX-NEXT: vcmpeqpd %xmm0, %xmm2, %xmm0
-; AVX-NEXT: vmovmskpd %xmm0, %ecx
-; AVX-NEXT: vmovmskpd %xmm1, %eax
-; AVX-NEXT: andl %ecx, %eax
+; AVX-NEXT: vandpd %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vmovmskpd %xmm0, %eax
; AVX-NEXT: retq
%1 = fcmp oeq <2 x double> zeroinitializer, %a0
%2 = sext <2 x i1> %1 to <2 x i64>
@@ -217,18 +215,16 @@ define i32 @xor_movmskps_movmskps(<4 x float> %a0, <4 x i32> %a1) {
; SSE: # %bb.0:
; SSE-NEXT: xorps %xmm2, %xmm2
; SSE-NEXT: cmpeqps %xmm0, %xmm2
-; SSE-NEXT: movmskps %xmm2, %ecx
-; SSE-NEXT: movmskps %xmm1, %eax
-; SSE-NEXT: xorl %ecx, %eax
+; SSE-NEXT: xorps %xmm1, %xmm2
+; SSE-NEXT: movmskps %xmm2, %eax
; SSE-NEXT: retq
;
; AVX-LABEL: xor_movmskps_movmskps:
; AVX: # %bb.0:
; AVX-NEXT: vxorps %xmm2, %xmm2, %xmm2
; AVX-NEXT: vcmpeqps %xmm0, %xmm2, %xmm0
-; AVX-NEXT: vmovmskps %xmm0, %ecx
-; AVX-NEXT: vmovmskps %xmm1, %eax
-; AVX-NEXT: xorl %ecx, %eax
+; AVX-NEXT: vxorps %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vmovmskps %xmm0, %eax
; AVX-NEXT: retq
%1 = fcmp oeq <4 x float> zeroinitializer, %a0
%2 = sext <4 x i1> %1 to <4 x i32>
@@ -246,20 +242,18 @@ define i32 @or_pmovmskb_pmovmskb(<16 x i8> %a0, <8 x i16> %a1) {
; SSE: # %bb.0:
; SSE-NEXT: pxor %xmm2, %xmm2
; SSE-NEXT: pcmpeqb %xmm0, %xmm2
-; SSE-NEXT: pmovmskb %xmm2, %ecx
; SSE-NEXT: psraw $15, %xmm1
+; SSE-NEXT: por %xmm2, %xmm1
; SSE-NEXT: pmovmskb %xmm1, %eax
-; SSE-NEXT: orl %ecx, %eax
; SSE-NEXT: retq
;
; AVX-LABEL: or_pmovmskb_pmovmskb:
; AVX: # %bb.0:
; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX-NEXT: vpcmpeqb %xmm2, %xmm0, %xmm0
-; AVX-NEXT: vpmovmskb %xmm0, %ecx
-; AVX-NEXT: vpsraw $15, %xmm1, %xmm0
+; AVX-NEXT: vpsraw $15, %xmm1, %xmm1
+; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0
; AVX-NEXT: vpmovmskb %xmm0, %eax
-; AVX-NEXT: orl %ecx, %eax
; AVX-NEXT: retq
%1 = icmp eq <16 x i8> zeroinitializer, %a0
%2 = sext <16 x i1> %1 to <16 x i8>
More information about the llvm-commits
mailing list