[llvm] r315460 - [DAGCombiner] convert insertelement of bitcasted vector into shuffle
Sanjay Patel via llvm-commits
llvm-commits at lists.llvm.org
Wed Oct 11 07:12:16 PDT 2017
Author: spatel
Date: Wed Oct 11 07:12:16 2017
New Revision: 315460
URL: http://llvm.org/viewvc/llvm-project?rev=315460&view=rev
Log:
[DAGCombiner] convert insertelement of bitcasted vector into shuffle
Eg:
insert v4i32 V, (v2i16 X), 2 --> shuffle v8i16 V', X', {0,1,2,3,8,9,6,7}
This is a generalization of the IR fold in D38316 to handle insertion into a non-undef vector.
We may want to abandon that one if we can't find value in squashing the more specific pattern sooner.
We're using the existing legal shuffle target hook to avoid AVX512 horror with vXi1 shuffles.
There may be room for improvement in the shuffle lowering here, but that would be follow-up work.
Differential Revision: https://reviews.llvm.org/D38388
Modified:
llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
llvm/trunk/test/CodeGen/AArch64/arm64-neon-copy.ll
llvm/trunk/test/CodeGen/X86/insertelement-shuffle.ll
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp?rev=315460&r1=315459&r2=315460&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/DAGCombiner.cpp Wed Oct 11 07:12:16 2017
@@ -415,6 +415,7 @@ namespace {
SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
SDValue CombineExtLoad(SDNode *N);
SDValue combineRepeatedFPDivisors(SDNode *N);
+ SDValue combineInsertEltToShuffle(SDNode *N, unsigned InsIndex);
SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT);
SDValue BuildSDIV(SDNode *N);
SDValue BuildSDIVPow2(SDNode *N);
@@ -13747,6 +13748,60 @@ SDValue DAGCombiner::splitMergedValStore
return St1;
}
+/// Convert a disguised subvector insertion into a shuffle:
+/// insert_vector_elt V, (bitcast X from vector type), IdxC -->
+/// bitcast(shuffle (bitcast V), (extended X), Mask)
+/// Note: We do not use an insert_subvector node because that requires a legal
+/// subvector type.
+SDValue DAGCombiner::combineInsertEltToShuffle(SDNode *N, unsigned InsIndex) {
+ SDValue InsertVal = N->getOperand(1);
+ if (InsertVal.getOpcode() != ISD::BITCAST || !InsertVal.hasOneUse() ||
+ !InsertVal.getOperand(0).getValueType().isVector())
+ return SDValue();
+
+ SDValue SubVec = InsertVal.getOperand(0);
+ SDValue DestVec = N->getOperand(0);
+ EVT SubVecVT = SubVec.getValueType();
+ EVT VT = DestVec.getValueType();
+ unsigned NumSrcElts = SubVecVT.getVectorNumElements();
+ unsigned ExtendRatio = VT.getSizeInBits() / SubVecVT.getSizeInBits();
+ unsigned NumMaskVals = ExtendRatio * NumSrcElts;
+
+ // Step 1: Create a shuffle mask that implements this insert operation. The
+ // vector that we are inserting into will be operand 0 of the shuffle, so
+ // those elements are just 'i'. The inserted subvector is in the first
+ // positions of operand 1 of the shuffle. Example:
+ // insert v4i32 V, (v2i16 X), 2 --> shuffle v8i16 V', X', {0,1,2,3,8,9,6,7}
+ SmallVector<int, 16> Mask(NumMaskVals);
+ for (unsigned i = 0; i != NumMaskVals; ++i) {
+ if (i / NumSrcElts == InsIndex)
+ Mask[i] = (i % NumSrcElts) + NumMaskVals;
+ else
+ Mask[i] = i;
+ }
+
+ // Bail out if the target can not handle the shuffle we want to create.
+ EVT SubVecEltVT = SubVecVT.getVectorElementType();
+ EVT ShufVT = EVT::getVectorVT(*DAG.getContext(), SubVecEltVT, NumMaskVals);
+ if (!TLI.isShuffleMaskLegal(Mask, ShufVT))
+ return SDValue();
+
+ // Step 2: Create a wide vector from the inserted source vector by appending
+ // undefined elements. This is the same size as our destination vector.
+ SDLoc DL(N);
+ SmallVector<SDValue, 8> ConcatOps(ExtendRatio, DAG.getUNDEF(SubVecVT));
+ ConcatOps[0] = SubVec;
+ SDValue PaddedSubV = DAG.getNode(ISD::CONCAT_VECTORS, DL, ShufVT, ConcatOps);
+
+ // Step 3: Shuffle in the padded subvector.
+ SDValue DestVecBC = DAG.getBitcast(ShufVT, DestVec);
+ SDValue Shuf = DAG.getVectorShuffle(ShufVT, DL, DestVecBC, PaddedSubV, Mask);
+ AddToWorklist(PaddedSubV.getNode());
+ AddToWorklist(DestVecBC.getNode());
+ AddToWorklist(Shuf.getNode());
+ return DAG.getBitcast(VT, Shuf);
+}
+
SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) {
SDValue InVec = N->getOperand(0);
SDValue InVal = N->getOperand(1);
@@ -13765,10 +13820,14 @@ SDValue DAGCombiner::visitINSERT_VECTOR_
InVec == InVal.getOperand(0) && EltNo == InVal.getOperand(1))
return InVec;
- // Check that we know which element is being inserted
- if (!isa<ConstantSDNode>(EltNo))
+ // We must know which element is being inserted for folds below here.
+ auto *IndexC = dyn_cast<ConstantSDNode>(EltNo);
+ if (!IndexC)
return SDValue();
- unsigned Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
+ unsigned Elt = IndexC->getZExtValue();
+
+ if (SDValue Shuf = combineInsertEltToShuffle(N, Elt))
+ return Shuf;
// Canonicalize insert_vector_elt dag nodes.
// Example:
Modified: llvm/trunk/test/CodeGen/AArch64/arm64-neon-copy.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/AArch64/arm64-neon-copy.ll?rev=315460&r1=315459&r2=315460&view=diff
==============================================================================
--- llvm/trunk/test/CodeGen/AArch64/arm64-neon-copy.ll (original)
+++ llvm/trunk/test/CodeGen/AArch64/arm64-neon-copy.ll Wed Oct 11 07:12:16 2017
@@ -140,7 +140,7 @@ define <4 x float> @ins2f4(<2 x float> %
define <2 x double> @ins1f2(<1 x double> %tmp1, <2 x double> %tmp2) {
; CHECK-LABEL: ins1f2:
-; CHECK: ins {{v[0-9]+}}.d[1], {{v[0-9]+}}.d[0]
+; CHECK: zip1 {{v[0-9]+}}.2d, {{v[0-9]+}}.2d
%tmp3 = extractelement <1 x double> %tmp1, i32 0
%tmp4 = insertelement <2 x double> %tmp2, double %tmp3, i32 1
ret <2 x double> %tmp4
Modified: llvm/trunk/test/CodeGen/X86/insertelement-shuffle.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/X86/insertelement-shuffle.ll?rev=315460&r1=315459&r2=315460&view=diff
==============================================================================
--- llvm/trunk/test/CodeGen/X86/insertelement-shuffle.ll (original)
+++ llvm/trunk/test/CodeGen/X86/insertelement-shuffle.ll Wed Oct 11 07:12:16 2017
@@ -7,42 +7,34 @@
define <8 x float> @insert_subvector_256(i16 %x0, i16 %x1, <8 x float> %v) nounwind {
; X32_AVX256-LABEL: insert_subvector_256:
; X32_AVX256: # BB#0:
-; X32_AVX256-NEXT: pushl %eax
; X32_AVX256-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
; X32_AVX256-NEXT: vpinsrw $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; X32_AVX256-NEXT: vmovd %xmm1, (%esp)
-; X32_AVX256-NEXT: vinsertps {{.*#+}} xmm1 = xmm0[0],mem[0],xmm0[2,3]
-; X32_AVX256-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; X32_AVX256-NEXT: popl %eax
+; X32_AVX256-NEXT: vpbroadcastd %xmm1, %xmm1
+; X32_AVX256-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6,7]
; X32_AVX256-NEXT: retl
;
; X64_AVX256-LABEL: insert_subvector_256:
; X64_AVX256: # BB#0:
; X64_AVX256-NEXT: vmovd %edi, %xmm1
; X64_AVX256-NEXT: vpinsrw $1, %esi, %xmm1, %xmm1
-; X64_AVX256-NEXT: vmovd %xmm1, -{{[0-9]+}}(%rsp)
-; X64_AVX256-NEXT: vinsertps {{.*#+}} xmm1 = xmm0[0],mem[0],xmm0[2,3]
-; X64_AVX256-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; X64_AVX256-NEXT: vpbroadcastd %xmm1, %xmm1
+; X64_AVX256-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6,7]
; X64_AVX256-NEXT: retq
;
; X32_AVX512-LABEL: insert_subvector_256:
; X32_AVX512: # BB#0:
-; X32_AVX512-NEXT: pushl %eax
; X32_AVX512-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
; X32_AVX512-NEXT: vpinsrw $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; X32_AVX512-NEXT: vmovd %xmm1, (%esp)
-; X32_AVX512-NEXT: vinsertps {{.*#+}} xmm1 = xmm0[0],mem[0],xmm0[2,3]
-; X32_AVX512-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; X32_AVX512-NEXT: popl %eax
+; X32_AVX512-NEXT: vpbroadcastd %xmm1, %xmm1
+; X32_AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6,7]
; X32_AVX512-NEXT: retl
;
; X64_AVX512-LABEL: insert_subvector_256:
; X64_AVX512: # BB#0:
; X64_AVX512-NEXT: vmovd %edi, %xmm1
; X64_AVX512-NEXT: vpinsrw $1, %esi, %xmm1, %xmm1
-; X64_AVX512-NEXT: vmovd %xmm1, -{{[0-9]+}}(%rsp)
-; X64_AVX512-NEXT: vinsertps {{.*#+}} xmm1 = xmm0[0],mem[0],xmm0[2,3]
-; X64_AVX512-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; X64_AVX512-NEXT: vpbroadcastd %xmm1, %xmm1
+; X64_AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6,7]
; X64_AVX512-NEXT: retq
%ins1 = insertelement <2 x i16> undef, i16 %x0, i32 0
%ins2 = insertelement <2 x i16> %ins1, i16 %x1, i32 1
@@ -80,28 +72,17 @@ define <8 x i64> @insert_subvector_512(i
;
; X32_AVX512-LABEL: insert_subvector_512:
; X32_AVX512: # BB#0:
-; X32_AVX512-NEXT: pushl %ebp
-; X32_AVX512-NEXT: movl %esp, %ebp
-; X32_AVX512-NEXT: andl $-8, %esp
-; X32_AVX512-NEXT: subl $8, %esp
-; X32_AVX512-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero
-; X32_AVX512-NEXT: vmovlps %xmm1, (%esp)
-; X32_AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
-; X32_AVX512-NEXT: vpinsrd $0, (%esp), %xmm1, %xmm1
-; X32_AVX512-NEXT: vpinsrd $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; X32_AVX512-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
-; X32_AVX512-NEXT: movl %ebp, %esp
-; X32_AVX512-NEXT: popl %ebp
+; X32_AVX512-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
+; X32_AVX512-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,0,1,0,8,0,3,0,4,0,5,0,6,0,7,0]
+; X32_AVX512-NEXT: vpermt2q %zmm1, %zmm2, %zmm0
; X32_AVX512-NEXT: retl
;
; X64_AVX512-LABEL: insert_subvector_512:
; X64_AVX512: # BB#0:
; X64_AVX512-NEXT: vmovd %edi, %xmm1
; X64_AVX512-NEXT: vpinsrd $1, %esi, %xmm1, %xmm1
-; X64_AVX512-NEXT: vmovq %xmm1, %rax
-; X64_AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
-; X64_AVX512-NEXT: vpinsrq $0, %rax, %xmm1, %xmm1
-; X64_AVX512-NEXT: vinserti32x4 $1, %xmm1, %zmm0, %zmm0
+; X64_AVX512-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,1,8,3,4,5,6,7]
+; X64_AVX512-NEXT: vpermt2q %zmm1, %zmm2, %zmm0
; X64_AVX512-NEXT: retq
%ins1 = insertelement <2 x i32> undef, i32 %x0, i32 0
%ins2 = insertelement <2 x i32> %ins1, i32 %x1, i32 1
@@ -144,22 +125,8 @@ define <8 x i64> @insert_subvector_into_
;
; X32_AVX512-LABEL: insert_subvector_into_undef:
; X32_AVX512: # BB#0:
-; X32_AVX512-NEXT: pushl %ebp
-; X32_AVX512-NEXT: movl %esp, %ebp
-; X32_AVX512-NEXT: andl $-8, %esp
-; X32_AVX512-NEXT: subl $8, %esp
; X32_AVX512-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32_AVX512-NEXT: vmovlps %xmm0, (%esp)
-; X32_AVX512-NEXT: movl (%esp), %eax
-; X32_AVX512-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32_AVX512-NEXT: vmovd %eax, %xmm0
-; X32_AVX512-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0
-; X32_AVX512-NEXT: vpinsrd $2, %eax, %xmm0, %xmm0
-; X32_AVX512-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
-; X32_AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
-; X32_AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0
-; X32_AVX512-NEXT: movl %ebp, %esp
-; X32_AVX512-NEXT: popl %ebp
+; X32_AVX512-NEXT: vbroadcastsd %xmm0, %zmm0
; X32_AVX512-NEXT: retl
;
; X64_AVX512-LABEL: insert_subvector_into_undef:
More information about the llvm-commits
mailing list