[llvm] r204770 - [X86] Add broadcast instructions to the table used by ExeDepsFix pass.
Quentin Colombet
qcolombet at apple.com
Tue Mar 25 17:10:23 PDT 2014
Author: qcolombet
Date: Tue Mar 25 19:10:22 2014
New Revision: 204770
URL: http://llvm.org/viewvc/llvm-project?rev=204770&view=rev
Log:
[X86] Add broadcast instructions to the table used by ExeDepsFix pass.
Adds the different broadcast instructions to the ReplaceableInstrsAVX2 table.
That way the ExeDepsFix pass can take better decisions when AVX2 broadcasts are
across domain (int <-> float).
In particular, prior to this patch we were generating:
vpbroadcastd LCPI1_0(%rip), %ymm2
vpand %ymm2, %ymm0, %ymm0
vmaxps %ymm1, %ymm0, %ymm0 ## <- domain change penalty
Now, we generate the following nice sequence where everything is in the float
domain:
vbroadcastss LCPI1_0(%rip), %ymm2
vandps %ymm2, %ymm0, %ymm0
vmaxps %ymm1, %ymm0, %ymm0
<rdar://problem/16354675>
Added:
llvm/trunk/test/CodeGen/X86/exedepsfix-broadcast.ll
Modified:
llvm/trunk/lib/Target/X86/X86InstrInfo.cpp
llvm/trunk/test/CodeGen/X86/avx2-intrinsics-x86.ll
llvm/trunk/test/CodeGen/X86/avx2-vbroadcast.ll
Modified: llvm/trunk/lib/Target/X86/X86InstrInfo.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86InstrInfo.cpp?rev=204770&r1=204769&r2=204770&view=diff
==============================================================================
--- llvm/trunk/lib/Target/X86/X86InstrInfo.cpp (original)
+++ llvm/trunk/lib/Target/X86/X86InstrInfo.cpp Tue Mar 25 19:10:22 2014
@@ -5125,7 +5125,13 @@ static const uint16_t ReplaceableInstrsA
{ X86::VINSERTF128rm, X86::VINSERTF128rm, X86::VINSERTI128rm },
{ X86::VINSERTF128rr, X86::VINSERTF128rr, X86::VINSERTI128rr },
{ X86::VPERM2F128rm, X86::VPERM2F128rm, X86::VPERM2I128rm },
- { X86::VPERM2F128rr, X86::VPERM2F128rr, X86::VPERM2I128rr }
+ { X86::VPERM2F128rr, X86::VPERM2F128rr, X86::VPERM2I128rr },
+ { X86::VBROADCASTSSrm, X86::VBROADCASTSSrm, X86::VPBROADCASTDrm},
+ { X86::VBROADCASTSSrr, X86::VBROADCASTSSrr, X86::VPBROADCASTDrr},
+ { X86::VBROADCASTSSYrr, X86::VBROADCASTSSYrr, X86::VPBROADCASTDYrr},
+ { X86::VBROADCASTSSYrm, X86::VBROADCASTSSYrm, X86::VPBROADCASTDYrm},
+ { X86::VBROADCASTSDYrr, X86::VBROADCASTSDYrr, X86::VPBROADCASTQYrr},
+ { X86::VBROADCASTSDYrm, X86::VBROADCASTSDYrm, X86::VPBROADCASTQYrm}
};
// FIXME: Some shuffle and unpack instructions have equivalents in different
Modified: llvm/trunk/test/CodeGen/X86/avx2-intrinsics-x86.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/X86/avx2-intrinsics-x86.ll?rev=204770&r1=204769&r2=204770&view=diff
==============================================================================
--- llvm/trunk/test/CodeGen/X86/avx2-intrinsics-x86.ll (original)
+++ llvm/trunk/test/CodeGen/X86/avx2-intrinsics-x86.ll Tue Mar 25 19:10:22 2014
@@ -753,7 +753,7 @@ declare <16 x i16> @llvm.x86.avx2.pbroad
define <4 x i32> @test_x86_avx2_pbroadcastd_128(<4 x i32> %a0) {
- ; CHECK: vpbroadcastd
+ ; CHECK: vbroadcastss
%res = call <4 x i32> @llvm.x86.avx2.pbroadcastd.128(<4 x i32> %a0) ; <<4 x i32>> [#uses=1]
ret <4 x i32> %res
}
@@ -761,7 +761,7 @@ declare <4 x i32> @llvm.x86.avx2.pbroadc
define <8 x i32> @test_x86_avx2_pbroadcastd_256(<4 x i32> %a0) {
- ; CHECK: vpbroadcastd
+ ; CHECK: vbroadcastss {{[^,]+}}, %ymm{{[0-9]+}}
%res = call <8 x i32> @llvm.x86.avx2.pbroadcastd.256(<4 x i32> %a0) ; <<8 x i32>> [#uses=1]
ret <8 x i32> %res
}
@@ -777,7 +777,7 @@ declare <2 x i64> @llvm.x86.avx2.pbroadc
define <4 x i64> @test_x86_avx2_pbroadcastq_256(<2 x i64> %a0) {
- ; CHECK: vpbroadcastq
+ ; CHECK: vbroadcastsd {{[^,]+}}, %ymm{{[0-9]+}}
%res = call <4 x i64> @llvm.x86.avx2.pbroadcastq.256(<2 x i64> %a0) ; <<4 x i64>> [#uses=1]
ret <4 x i64> %res
}
Modified: llvm/trunk/test/CodeGen/X86/avx2-vbroadcast.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/X86/avx2-vbroadcast.ll?rev=204770&r1=204769&r2=204770&view=diff
==============================================================================
--- llvm/trunk/test/CodeGen/X86/avx2-vbroadcast.ll (original)
+++ llvm/trunk/test/CodeGen/X86/avx2-vbroadcast.ll Tue Mar 25 19:10:22 2014
@@ -98,7 +98,7 @@ entry:
%qf = insertelement <16 x i16> %qe, i16 %q, i32 15
ret <16 x i16> %qf
}
-; CHECK: vpbroadcastd (%
+; CHECK: vbroadcastss (%
define <4 x i32> @D32(i32* %ptr) nounwind uwtable readnone ssp {
entry:
%q = load i32* %ptr, align 4
@@ -108,7 +108,7 @@ entry:
%q3 = insertelement <4 x i32> %q2, i32 %q, i32 3
ret <4 x i32> %q3
}
-; CHECK: vpbroadcastd (%
+; CHECK: vbroadcastss (%
define <8 x i32> @DD32(i32* %ptr) nounwind uwtable readnone ssp {
entry:
%q = load i32* %ptr, align 4
@@ -130,7 +130,7 @@ entry:
%q1 = insertelement <2 x i64> %q0, i64 %q, i32 1
ret <2 x i64> %q1
}
-; CHECK: vpbroadcastq (%
+; CHECK: vbroadcastsd (%
define <4 x i64> @QQ64(i64* %ptr) nounwind uwtable readnone ssp {
entry:
%q = load i64* %ptr, align 4
@@ -293,7 +293,7 @@ define <8 x i16> @_inreg8xi16(<8 x i16
;CHECK-LABEL: _inreg4xi64:
-;CHECK: vpbroadcastq
+;CHECK: vbroadcastsd
;CHECK: ret
define <4 x i64> @_inreg4xi64(<4 x i64> %a) {
%b = shufflevector <4 x i64> %a, <4 x i64> undef, <4 x i32> zeroinitializer
@@ -325,7 +325,7 @@ define <2 x double> @_inreg2xdouble(<2
}
;CHECK-LABEL: _inreg8xi32:
-;CHECK: vpbroadcastd
+;CHECK: vbroadcastss
;CHECK: ret
define <8 x i32> @_inreg8xi32(<8 x i32> %a) {
%b = shufflevector <8 x i32> %a, <8 x i32> undef, <8 x i32> zeroinitializer
@@ -333,7 +333,7 @@ define <8 x i32> @_inreg8xi32(<8 x i32
}
;CHECK-LABEL: _inreg4xi32:
-;CHECK: vpbroadcastd
+;CHECK: vbroadcastss
;CHECK: ret
define <4 x i32> @_inreg4xi32(<4 x i32> %a) {
%b = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> zeroinitializer
Added: llvm/trunk/test/CodeGen/X86/exedepsfix-broadcast.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/X86/exedepsfix-broadcast.ll?rev=204770&view=auto
==============================================================================
--- llvm/trunk/test/CodeGen/X86/exedepsfix-broadcast.ll (added)
+++ llvm/trunk/test/CodeGen/X86/exedepsfix-broadcast.ll Tue Mar 25 19:10:22 2014
@@ -0,0 +1,128 @@
+; RUN: llc -O3 -mtriple=x86_64-apple-macosx -o - < %s -mattr=+avx2 -enable-unsafe-fp-math -mcpu=core2 | FileCheck %s
+; Check that the ExeDepsFix pass correctly fixes the domain for broadcast instructions.
+; <rdar://problem/16354675>
+
+; CHECK-LABEL: ExeDepsFix_broadcastss
+; CHECK: broadcastss
+; CHECK: vandps
+; CHECK: vmaxps
+; CHECK: ret
+define <4 x float> @ExeDepsFix_broadcastss(<4 x float> %arg, <4 x float> %arg2) {
+ %bitcast = bitcast <4 x float> %arg to <4 x i32>
+ %and = and <4 x i32> %bitcast, <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+ %floatcast = bitcast <4 x i32> %and to <4 x float>
+ %max_is_x = fcmp oge <4 x float> %floatcast, %arg2
+ %max = select <4 x i1> %max_is_x, <4 x float> %floatcast, <4 x float> %arg2
+ ret <4 x float> %max
+}
+
+; CHECK-LABEL: ExeDepsFix_broadcastss256
+; CHECK: broadcastss
+; CHECK: vandps
+; CHECK: vmaxps
+; CHECK: ret
+define <8 x float> @ExeDepsFix_broadcastss256(<8 x float> %arg, <8 x float> %arg2) {
+ %bitcast = bitcast <8 x float> %arg to <8 x i32>
+ %and = and <8 x i32> %bitcast, <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+ %floatcast = bitcast <8 x i32> %and to <8 x float>
+ %max_is_x = fcmp oge <8 x float> %floatcast, %arg2
+ %max = select <8 x i1> %max_is_x, <8 x float> %floatcast, <8 x float> %arg2
+ ret <8 x float> %max
+}
+
+
+; CHECK-LABEL: ExeDepsFix_broadcastss_inreg
+; CHECK: broadcastss
+; CHECK: vandps
+; CHECK: vmaxps
+; CHECK: ret
+define <4 x float> @ExeDepsFix_broadcastss_inreg(<4 x float> %arg, <4 x float> %arg2, i32 %broadcastvalue) {
+ %bitcast = bitcast <4 x float> %arg to <4 x i32>
+ %in = insertelement <4 x i32> undef, i32 %broadcastvalue, i32 0
+ %mask = shufflevector <4 x i32> %in, <4 x i32> undef, <4 x i32> zeroinitializer
+ %and = and <4 x i32> %bitcast, %mask
+ %floatcast = bitcast <4 x i32> %and to <4 x float>
+ %max_is_x = fcmp oge <4 x float> %floatcast, %arg2
+ %max = select <4 x i1> %max_is_x, <4 x float> %floatcast, <4 x float> %arg2
+ ret <4 x float> %max
+}
+
+; CHECK-LABEL: ExeDepsFix_broadcastss256_inreg
+; CHECK: broadcastss
+; CHECK: vandps
+; CHECK: vmaxps
+; CHECK: ret
+define <8 x float> @ExeDepsFix_broadcastss256_inreg(<8 x float> %arg, <8 x float> %arg2, i32 %broadcastvalue) {
+ %bitcast = bitcast <8 x float> %arg to <8 x i32>
+ %in = insertelement <8 x i32> undef, i32 %broadcastvalue, i32 0
+ %mask = shufflevector <8 x i32> %in, <8 x i32> undef, <8 x i32> zeroinitializer
+ %and = and <8 x i32> %bitcast, %mask
+ %floatcast = bitcast <8 x i32> %and to <8 x float>
+ %max_is_x = fcmp oge <8 x float> %floatcast, %arg2
+ %max = select <8 x i1> %max_is_x, <8 x float> %floatcast, <8 x float> %arg2
+ ret <8 x float> %max
+}
+
+; CHECK-LABEL: ExeDepsFix_broadcastsd
+; In that case the broadcast is directly folded into vandpd.
+; CHECK: vandpd
+; CHECK: vmaxpd
+; CHECK:ret
+define <2 x double> @ExeDepsFix_broadcastsd(<2 x double> %arg, <2 x double> %arg2) {
+ %bitcast = bitcast <2 x double> %arg to <2 x i64>
+ %and = and <2 x i64> %bitcast, <i64 2147483647, i64 2147483647>
+ %floatcast = bitcast <2 x i64> %and to <2 x double>
+ %max_is_x = fcmp oge <2 x double> %floatcast, %arg2
+ %max = select <2 x i1> %max_is_x, <2 x double> %floatcast, <2 x double> %arg2
+ ret <2 x double> %max
+}
+
+; CHECK-LABEL: ExeDepsFix_broadcastsd256
+; CHECK: broadcastsd
+; CHECK: vandpd
+; CHECK: vmaxpd
+; CHECK: ret
+define <4 x double> @ExeDepsFix_broadcastsd256(<4 x double> %arg, <4 x double> %arg2) {
+ %bitcast = bitcast <4 x double> %arg to <4 x i64>
+ %and = and <4 x i64> %bitcast, <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>
+ %floatcast = bitcast <4 x i64> %and to <4 x double>
+ %max_is_x = fcmp oge <4 x double> %floatcast, %arg2
+ %max = select <4 x i1> %max_is_x, <4 x double> %floatcast, <4 x double> %arg2
+ ret <4 x double> %max
+}
+
+
+; CHECK-LABEL: ExeDepsFix_broadcastsd_inreg
+; ExeDepsFix works top down, thus it coalesces vmovlhps domain with
+; vandps and there is nothing more you can do to match vmaxpd.
+; CHECK: vmovlhps
+; CHECK: vandps
+; CHECK: vmaxpd
+; CHECK: ret
+define <2 x double> @ExeDepsFix_broadcastsd_inreg(<2 x double> %arg, <2 x double> %arg2, i64 %broadcastvalue) {
+ %bitcast = bitcast <2 x double> %arg to <2 x i64>
+ %in = insertelement <2 x i64> undef, i64 %broadcastvalue, i32 0
+ %mask = shufflevector <2 x i64> %in, <2 x i64> undef, <2 x i32> zeroinitializer
+ %and = and <2 x i64> %bitcast, %mask
+ %floatcast = bitcast <2 x i64> %and to <2 x double>
+ %max_is_x = fcmp oge <2 x double> %floatcast, %arg2
+ %max = select <2 x i1> %max_is_x, <2 x double> %floatcast, <2 x double> %arg2
+ ret <2 x double> %max
+}
+
+; CHECK-LABEL: ExeDepsFix_broadcastsd256_inreg
+; CHECK: broadcastsd
+; CHECK: vandpd
+; CHECK: vmaxpd
+; CHECK: ret
+define <4 x double> @ExeDepsFix_broadcastsd256_inreg(<4 x double> %arg, <4 x double> %arg2, i64 %broadcastvalue) {
+ %bitcast = bitcast <4 x double> %arg to <4 x i64>
+ %in = insertelement <4 x i64> undef, i64 %broadcastvalue, i32 0
+ %mask = shufflevector <4 x i64> %in, <4 x i64> undef, <4 x i32> zeroinitializer
+ %and = and <4 x i64> %bitcast, %mask
+ %floatcast = bitcast <4 x i64> %and to <4 x double>
+ %max_is_x = fcmp oge <4 x double> %floatcast, %arg2
+ %max = select <4 x i1> %max_is_x, <4 x double> %floatcast, <4 x double> %arg2
+ ret <4 x double> %max
+}
+
More information about the llvm-commits
mailing list