[llvm] 401cbe3 - [X86][AVX] Attempt to scale masked shuffles to match the root type
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 13 06:57:40 PDT 2020
Author: Simon Pilgrim
Date: 2020-04-13T14:57:25+01:00
New Revision: 401cbe373b078ff3325df3b41847713369d2132c
URL: https://github.com/llvm/llvm-project/commit/401cbe373b078ff3325df3b41847713369d2132c
DIFF: https://github.com/llvm/llvm-project/commit/401cbe373b078ff3325df3b41847713369d2132c.diff
LOG: [X86][AVX] Attempt to scale masked shuffles to match the root type
Improve the chances of folding the writemask into the combined shuffle by scaling a wider shuffle mask to match the root's original type.
This creates a few minor issues with variable shuffles, preventing combines of shuffles because of the more limited support binary shuffle types. In most cases we're probably better off combining the shuffles and losing the writemask fold, but this isn't always going to be true.
Added:
Modified:
llvm/lib/Target/X86/X86ISelLowering.cpp
llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bw.ll
llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll
llvm/test/CodeGen/X86/vector-shuffle-combining-avx512vbmi.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 34a794cae386..50abd75e87b9 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -34075,8 +34075,6 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
// Don't combine if we are a AVX512/EVEX target and the mask element size
// is
diff erent from the root element size - this would prevent writemasks
// from being reused.
- // TODO - this currently prevents all lane shuffles from occurring.
- // TODO - attempt to narrow Mask back to writemask size.
bool IsMaskedShuffle = false;
if (RootSizeInBits == 512 || (Subtarget.hasVLX() && RootSizeInBits >= 128)) {
if (Root.hasOneUse() && Root->use_begin()->getOpcode() == ISD::VSELECT &&
@@ -34251,6 +34249,17 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
Mask.assign(BaseMask.begin(), BaseMask.end());
}
+ // For masked shuffles, we're trying to match the root width for better
+ // writemask folding, attempt to scale the mask.
+ // TODO - variable shuffles might need this to be widened again.
+ if (IsMaskedShuffle && NumRootElts > Mask.size()) {
+ assert((NumRootElts % Mask.size()) == 0 && "Illegal mask size");
+ int MaskScale = NumRootElts / Mask.size();
+ SmallVector<int, 64> ScaledMask;
+ narrowShuffleMaskElts(MaskScale, Mask, ScaledMask);
+ Mask = std::move(ScaledMask);
+ }
+
unsigned NumMaskElts = Mask.size();
unsigned MaskEltSizeInBits = RootSizeInBits / NumMaskElts;
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bw.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bw.ll
index 2adceb2f2cab..e984d7e779c1 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bw.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bw.ll
@@ -156,18 +156,18 @@ define <64 x i8> @combine_permi2q_pshufb_as_permi2d(<8 x i64> %a0, <8 x i64> %a1
define <64 x i8> @combine_permi2q_pshufb_as_permi2d_mask(<8 x i64> %a0, <8 x i64> %a1, i64 %m) {
; X86-LABEL: combine_permi2q_pshufb_as_permi2d_mask:
; X86: # %bb.0:
-; X86-NEXT: vmovdqa64 {{.*#+}} zmm2 = [14,14,14,14,11,11,11,11,24,24,24,24,29,29,29,29]
-; X86-NEXT: vpermi2d %zmm0, %zmm1, %zmm2
+; X86-NEXT: vmovdqa64 {{.*#+}} zmm2 = [7,0,8,0,5,0,10,0,3,0,12,0,1,0,14,0]
+; X86-NEXT: vpermi2q %zmm0, %zmm1, %zmm2
; X86-NEXT: kmovq {{[0-9]+}}(%esp), %k1
-; X86-NEXT: vmovdqu8 %zmm2, %zmm0 {%k1} {z}
+; X86-NEXT: vpshufb {{.*#+}} zmm0 {%k1} {z} = zmm2[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,20,21,22,23,20,21,22,23,20,21,22,23,20,21,22,23,40,41,42,43,40,41,42,43,40,41,42,43,40,41,42,43,60,61,62,63,60,61,62,63,60,61,62,63,60,61,62,63]
; X86-NEXT: retl
;
; X64-LABEL: combine_permi2q_pshufb_as_permi2d_mask:
; X64: # %bb.0:
-; X64-NEXT: vmovdqa64 {{.*#+}} zmm2 = [14,14,14,14,11,11,11,11,24,24,24,24,29,29,29,29]
-; X64-NEXT: vpermi2d %zmm0, %zmm1, %zmm2
+; X64-NEXT: vmovdqa64 {{.*#+}} zmm2 = [7,8,5,10,3,12,1,14]
+; X64-NEXT: vpermi2q %zmm0, %zmm1, %zmm2
; X64-NEXT: kmovq %rdi, %k1
-; X64-NEXT: vmovdqu8 %zmm2, %zmm0 {%k1} {z}
+; X64-NEXT: vpshufb {{.*#+}} zmm0 {%k1} {z} = zmm2[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3,20,21,22,23,20,21,22,23,20,21,22,23,20,21,22,23,40,41,42,43,40,41,42,43,40,41,42,43,40,41,42,43,60,61,62,63,60,61,62,63,60,61,62,63,60,61,62,63]
; X64-NEXT: retq
%res0 = shufflevector <8 x i64> %a0, <8 x i64> %a1, <8 x i32> <i32 15, i32 0, i32 13, i32 2, i32 11, i32 4, i32 9, i32 6>
%res1 = bitcast <8 x i64> %res0 to <64 x i8>
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll
index a91327096a1e..257bd03dee8e 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll
@@ -361,23 +361,20 @@ define <16 x float> @combine_vpermt2var_16f32_vmovddup_load(<16 x float> *%p0, <
define <16 x float> @combine_vpermt2var_16f32_vmovddup_mask(<16 x float> %x0, <16 x float> %x1, i16 %m) {
; X86-LABEL: combine_vpermt2var_16f32_vmovddup_mask:
; X86: # %bb.0:
-; X86-NEXT: vmovaps {{.*#+}} zmm1 = [0,1,0,1,4,5,4,5,8,9,8,9,12,13,12,13]
; X86-NEXT: kmovw {{[0-9]+}}(%esp), %k1
-; X86-NEXT: vpermt2ps %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X86-NEXT: vpermilps {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,0,1,4,5,4,5,8,9,8,9,12,13,12,13]
; X86-NEXT: retl
;
; X64-AVX512F-LABEL: combine_vpermt2var_16f32_vmovddup_mask:
; X64-AVX512F: # %bb.0:
-; X64-AVX512F-NEXT: vmovaps {{.*#+}} zmm1 = [0,1,0,1,4,5,4,5,8,9,8,9,12,13,12,13]
; X64-AVX512F-NEXT: kmovw %edi, %k1
-; X64-AVX512F-NEXT: vpermt2ps %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X64-AVX512F-NEXT: vpermilps {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,0,1,4,5,4,5,8,9,8,9,12,13,12,13]
; X64-AVX512F-NEXT: retq
;
; X64-AVX512BW-LABEL: combine_vpermt2var_16f32_vmovddup_mask:
; X64-AVX512BW: # %bb.0:
-; X64-AVX512BW-NEXT: vmovaps {{.*#+}} zmm1 = [0,1,0,1,4,5,4,5,8,9,8,9,12,13,12,13]
; X64-AVX512BW-NEXT: kmovd %edi, %k1
-; X64-AVX512BW-NEXT: vpermt2ps %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X64-AVX512BW-NEXT: vpermilps {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,0,1,4,5,4,5,8,9,8,9,12,13,12,13]
; X64-AVX512BW-NEXT: retq
%res0 = call <16 x float> @llvm.x86.avx512.maskz.vpermt2var.ps.512(<16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 4, i32 5, i32 4, i32 5, i32 8, i32 9, i32 8, i32 9, i32 12, i32 13, i32 12, i32 13>, <16 x float> %x0, <16 x float> %x1, i16 %m)
ret <16 x float> %res0
@@ -386,26 +383,20 @@ define <16 x float> @combine_vpermt2var_16f32_vmovddup_mask_load(<16 x float> *%
; X86-LABEL: combine_vpermt2var_16f32_vmovddup_mask_load:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: vmovaps (%eax), %zmm1
-; X86-NEXT: vmovaps {{.*#+}} zmm0 = [0,1,0,1,4,5,4,5,8,9,8,9,12,13,12,13]
; X86-NEXT: kmovw {{[0-9]+}}(%esp), %k1
-; X86-NEXT: vpermi2ps %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X86-NEXT: vpermilps {{.*#+}} zmm0 {%k1} {z} = mem[0,1,0,1,4,5,4,5,8,9,8,9,12,13,12,13]
; X86-NEXT: retl
;
; X64-AVX512F-LABEL: combine_vpermt2var_16f32_vmovddup_mask_load:
; X64-AVX512F: # %bb.0:
-; X64-AVX512F-NEXT: vmovaps (%rdi), %zmm1
-; X64-AVX512F-NEXT: vmovaps {{.*#+}} zmm0 = [0,1,0,1,4,5,4,5,8,9,8,9,12,13,12,13]
; X64-AVX512F-NEXT: kmovw %esi, %k1
-; X64-AVX512F-NEXT: vpermi2ps %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X64-AVX512F-NEXT: vpermilps {{.*#+}} zmm0 {%k1} {z} = mem[0,1,0,1,4,5,4,5,8,9,8,9,12,13,12,13]
; X64-AVX512F-NEXT: retq
;
; X64-AVX512BW-LABEL: combine_vpermt2var_16f32_vmovddup_mask_load:
; X64-AVX512BW: # %bb.0:
-; X64-AVX512BW-NEXT: vmovaps (%rdi), %zmm1
-; X64-AVX512BW-NEXT: vmovaps {{.*#+}} zmm0 = [0,1,0,1,4,5,4,5,8,9,8,9,12,13,12,13]
; X64-AVX512BW-NEXT: kmovd %esi, %k1
-; X64-AVX512BW-NEXT: vpermi2ps %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X64-AVX512BW-NEXT: vpermilps {{.*#+}} zmm0 {%k1} {z} = mem[0,1,0,1,4,5,4,5,8,9,8,9,12,13,12,13]
; X64-AVX512BW-NEXT: retq
%x0 = load <16 x float>, <16 x float> *%p0
%res0 = call <16 x float> @llvm.x86.avx512.maskz.vpermt2var.ps.512(<16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 4, i32 5, i32 4, i32 5, i32 8, i32 9, i32 8, i32 9, i32 12, i32 13, i32 12, i32 13>, <16 x float> %x0, <16 x float> %x1, i16 %m)
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512vbmi.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512vbmi.ll
index bca3fb892ebe..ef7d8fc1b79c 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512vbmi.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512vbmi.ll
@@ -128,18 +128,18 @@ define <64 x i8> @combine_permi2q_pshufb_as_permi2d(<8 x i64> %a0, <8 x i64> %a1
define <64 x i8> @combine_permi2q_pshufb_as_permi2d_mask(<8 x i64> %a0, <8 x i64> %a1, i64 %m) {
; X86-LABEL: combine_permi2q_pshufb_as_permi2d_mask:
; X86: # %bb.0:
-; X86-NEXT: vmovdqa64 {{.*#+}} zmm2 = [14,14,14,14,11,11,11,11,24,24,24,24,29,29,29,29]
-; X86-NEXT: vpermi2d %zmm0, %zmm1, %zmm2
+; X86-NEXT: vmovdqa64 {{.*#+}} zmm2 = [56,57,58,59,56,57,58,59,56,57,58,59,56,57,58,59,44,45,46,47,44,45,46,47,44,45,46,47,44,45,46,47,96,97,98,99,96,97,98,99,96,97,98,99,96,97,98,99,116,117,118,119,116,117,118,119,116,117,118,119,116,117,118,119]
; X86-NEXT: kmovq {{[0-9]+}}(%esp), %k1
-; X86-NEXT: vmovdqu8 %zmm2, %zmm0 {%k1} {z}
+; X86-NEXT: vpermi2b %zmm0, %zmm1, %zmm2 {%k1} {z}
+; X86-NEXT: vmovdqa64 %zmm2, %zmm0
; X86-NEXT: retl
;
; X64-LABEL: combine_permi2q_pshufb_as_permi2d_mask:
; X64: # %bb.0:
-; X64-NEXT: vmovdqa64 {{.*#+}} zmm2 = [14,14,14,14,11,11,11,11,24,24,24,24,29,29,29,29]
-; X64-NEXT: vpermi2d %zmm0, %zmm1, %zmm2
+; X64-NEXT: vmovdqa64 {{.*#+}} zmm2 = [56,57,58,59,56,57,58,59,56,57,58,59,56,57,58,59,44,45,46,47,44,45,46,47,44,45,46,47,44,45,46,47,96,97,98,99,96,97,98,99,96,97,98,99,96,97,98,99,116,117,118,119,116,117,118,119,116,117,118,119,116,117,118,119]
; X64-NEXT: kmovq %rdi, %k1
-; X64-NEXT: vmovdqu8 %zmm2, %zmm0 {%k1} {z}
+; X64-NEXT: vpermi2b %zmm0, %zmm1, %zmm2 {%k1} {z}
+; X64-NEXT: vmovdqa64 %zmm2, %zmm0
; X64-NEXT: retq
%res0 = shufflevector <8 x i64> %a0, <8 x i64> %a1, <8 x i32> <i32 15, i32 0, i32 13, i32 2, i32 11, i32 4, i32 9, i32 6>
%res1 = bitcast <8 x i64> %res0 to <64 x i8>
More information about the llvm-commits
mailing list