[X86] Add broadcast instructions to the table used by ExeDepsFix pass.

Adds the different broadcast instructions to the ReplaceableInstrsAVX2 table.
That way the ExeDepsFix pass can take better decisions when AVX2 broadcasts are
across domain (int <-> float).

In particular, prior to this patch we were generating:
  vpbroadcastd  LCPI1_0(%rip), %ymm2
  vpand %ymm2, %ymm0, %ymm0
  vmaxps  %ymm1, %ymm0, %ymm0 ## <- domain change penalty

Now, we generate the following nice sequence where everything is in the float
domain:
  vbroadcastss  LCPI1_0(%rip), %ymm2
  vandps  %ymm2, %ymm0, %ymm0
  vmaxps  %ymm1, %ymm0, %ymm0

<rdar://problem/16354675>

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@204770 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/lib/Target/X86/X86InstrInfo.cpp b/lib/Target/X86/X86InstrInfo.cpp
index 8eb4a17269b6ca259cded65f27afbaadfaddb82f..d131791dfd77187dbec8320a5e937d233aee7309 100644 (file)
--- a/lib/Target/X86/X86InstrInfo.cpp
+++ b/lib/Target/X86/X86InstrInfo.cpp
@@ -5125,7 +5125,13 @@ static const uint16_t ReplaceableInstrsAVX2[][3] = {
   { X86::VINSERTF128rm,  X86::VINSERTF128rm,  X86::VINSERTI128rm },
   { X86::VINSERTF128rr,  X86::VINSERTF128rr,  X86::VINSERTI128rr },
   { X86::VPERM2F128rm,   X86::VPERM2F128rm,   X86::VPERM2I128rm },
-  { X86::VPERM2F128rr,   X86::VPERM2F128rr,   X86::VPERM2I128rr }
+  { X86::VPERM2F128rr,   X86::VPERM2F128rr,   X86::VPERM2I128rr },
+  { X86::VBROADCASTSSrm, X86::VBROADCASTSSrm, X86::VPBROADCASTDrm},
+  { X86::VBROADCASTSSrr, X86::VBROADCASTSSrr, X86::VPBROADCASTDrr},
+  { X86::VBROADCASTSSYrr, X86::VBROADCASTSSYrr, X86::VPBROADCASTDYrr},
+  { X86::VBROADCASTSSYrm, X86::VBROADCASTSSYrm, X86::VPBROADCASTDYrm},
+  { X86::VBROADCASTSDYrr, X86::VBROADCASTSDYrr, X86::VPBROADCASTQYrr},
+  { X86::VBROADCASTSDYrm, X86::VBROADCASTSDYrm, X86::VPBROADCASTQYrm}
 };
 
 // FIXME: Some shuffle and unpack instructions have equivalents in different

diff --git a/test/CodeGen/X86/avx2-intrinsics-x86.ll b/test/CodeGen/X86/avx2-intrinsics-x86.ll
index 7ee944b7c9271407d269837de93edd008be5e2ed..ab3d591e1d9fbbdf9233e83f9c21316ef6f79816 100644 (file)
--- a/test/CodeGen/X86/avx2-intrinsics-x86.ll
+++ b/test/CodeGen/X86/avx2-intrinsics-x86.ll
@@ -753,7 +753,7 @@ declare <16 x i16> @llvm.x86.avx2.pbroadcastw.256(<8 x i16>) nounwind readonly
 
 
 define <4 x i32> @test_x86_avx2_pbroadcastd_128(<4 x i32> %a0) {
-  ; CHECK: vpbroadcastd
+  ; CHECK: vbroadcastss
   %res = call <4 x i32> @llvm.x86.avx2.pbroadcastd.128(<4 x i32> %a0) ; <<4 x i32>> [#uses=1]
   ret <4 x i32> %res
 }
@@ -761,7 +761,7 @@ declare <4 x i32> @llvm.x86.avx2.pbroadcastd.128(<4 x i32>) nounwind readonly
 
 
 define <8 x i32> @test_x86_avx2_pbroadcastd_256(<4 x i32> %a0) {
-  ; CHECK: vpbroadcastd
+  ; CHECK: vbroadcastss {{[^,]+}}, %ymm{{[0-9]+}}
   %res = call <8 x i32> @llvm.x86.avx2.pbroadcastd.256(<4 x i32> %a0) ; <<8 x i32>> [#uses=1]
   ret <8 x i32> %res
 }
@@ -777,7 +777,7 @@ declare <2 x i64> @llvm.x86.avx2.pbroadcastq.128(<2 x i64>) nounwind readonly
 
 
 define <4 x i64> @test_x86_avx2_pbroadcastq_256(<2 x i64> %a0) {
-  ; CHECK: vpbroadcastq
+  ; CHECK: vbroadcastsd {{[^,]+}}, %ymm{{[0-9]+}}
   %res = call <4 x i64> @llvm.x86.avx2.pbroadcastq.256(<2 x i64> %a0) ; <<4 x i64>> [#uses=1]
   ret <4 x i64> %res
 }

diff --git a/test/CodeGen/X86/avx2-vbroadcast.ll b/test/CodeGen/X86/avx2-vbroadcast.ll
index bac9c66c3aed34511183ca2c32659d9e9b66bab0..66f586d23d14c55d22100ddc2b2315c9404b87f2 100644 (file)
--- a/test/CodeGen/X86/avx2-vbroadcast.ll
+++ b/test/CodeGen/X86/avx2-vbroadcast.ll
@@ -98,7 +98,7 @@ entry:
   %qf = insertelement <16 x i16> %qe, i16 %q, i32 15
   ret <16 x i16> %qf
 }
-; CHECK: vpbroadcastd (%
+; CHECK: vbroadcastss (%
 define <4 x i32> @D32(i32* %ptr) nounwind uwtable readnone ssp {
 entry:
   %q = load i32* %ptr, align 4
@@ -108,7 +108,7 @@ entry:
   %q3 = insertelement <4 x i32> %q2, i32 %q, i32 3
   ret <4 x i32> %q3
 }
-; CHECK: vpbroadcastd (%
+; CHECK: vbroadcastss (%
 define <8 x i32> @DD32(i32* %ptr) nounwind uwtable readnone ssp {
 entry:
   %q = load i32* %ptr, align 4
@@ -130,7 +130,7 @@ entry:
   %q1 = insertelement <2 x i64> %q0, i64 %q, i32 1
   ret <2 x i64> %q1
 }
-; CHECK: vpbroadcastq (%
+; CHECK: vbroadcastsd (%
 define <4 x i64> @QQ64(i64* %ptr) nounwind uwtable readnone ssp {
 entry:
   %q = load i64* %ptr, align 4
@@ -293,7 +293,7 @@ define   <8 x i16> @_inreg8xi16(<8 x i16> %a) {
 
 
 ;CHECK-LABEL: _inreg4xi64:
-;CHECK: vpbroadcastq
+;CHECK: vbroadcastsd
 ;CHECK: ret
 define   <4 x i64> @_inreg4xi64(<4 x i64> %a) {
   %b = shufflevector <4 x i64> %a, <4 x i64> undef, <4 x i32> zeroinitializer
@@ -325,7 +325,7 @@ define   <2 x double> @_inreg2xdouble(<2 x double> %a) {
 }
 
 ;CHECK-LABEL: _inreg8xi32:
-;CHECK: vpbroadcastd
+;CHECK: vbroadcastss
 ;CHECK: ret
 define   <8 x i32> @_inreg8xi32(<8 x i32> %a) {
   %b = shufflevector <8 x i32> %a, <8 x i32> undef, <8 x i32> zeroinitializer
@@ -333,7 +333,7 @@ define   <8 x i32> @_inreg8xi32(<8 x i32> %a) {
 }
 
 ;CHECK-LABEL: _inreg4xi32:
-;CHECK: vpbroadcastd
+;CHECK: vbroadcastss
 ;CHECK: ret
 define   <4 x i32> @_inreg4xi32(<4 x i32> %a) {
   %b = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> zeroinitializer

diff --git a/test/CodeGen/X86/exedepsfix-broadcast.ll b/test/CodeGen/X86/exedepsfix-broadcast.ll
new file mode 100644 (file)
index 0000000..a18f751
--- /dev/null
+++ b/test/CodeGen/X86/exedepsfix-broadcast.ll
@@ -0,0 +1,128 @@
+; RUN: llc -O3 -mtriple=x86_64-apple-macosx -o - < %s -mattr=+avx2 -enable-unsafe-fp-math -mcpu=core2 | FileCheck %s
+; Check that the ExeDepsFix pass correctly fixes the domain for broadcast instructions.
+; <rdar://problem/16354675>
+
+; CHECK-LABEL: ExeDepsFix_broadcastss
+; CHECK: broadcastss
+; CHECK: vandps
+; CHECK: vmaxps
+; CHECK: ret
+define <4 x float> @ExeDepsFix_broadcastss(<4 x float> %arg, <4 x float> %arg2) {
+  %bitcast = bitcast <4 x float> %arg to <4 x i32>
+  %and = and <4 x i32> %bitcast, <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+  %floatcast = bitcast <4 x i32> %and to <4 x float>
+  %max_is_x = fcmp oge <4 x float> %floatcast, %arg2
+  %max = select <4 x i1> %max_is_x, <4 x float> %floatcast, <4 x float> %arg2
+  ret <4 x float> %max
+}
+
+; CHECK-LABEL: ExeDepsFix_broadcastss256
+; CHECK: broadcastss
+; CHECK: vandps
+; CHECK: vmaxps
+; CHECK: ret
+define <8 x float> @ExeDepsFix_broadcastss256(<8 x float> %arg, <8 x float> %arg2) {
+  %bitcast = bitcast <8 x float> %arg to <8 x i32>
+  %and = and <8 x i32> %bitcast, <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+  %floatcast = bitcast <8 x i32> %and to <8 x float>
+  %max_is_x = fcmp oge <8 x float> %floatcast, %arg2
+  %max = select <8 x i1> %max_is_x, <8 x float> %floatcast, <8 x float> %arg2
+  ret <8 x float> %max
+}
+
+
+; CHECK-LABEL: ExeDepsFix_broadcastss_inreg
+; CHECK: broadcastss
+; CHECK: vandps
+; CHECK: vmaxps
+; CHECK: ret
+define <4 x float> @ExeDepsFix_broadcastss_inreg(<4 x float> %arg, <4 x float> %arg2, i32 %broadcastvalue) {
+  %bitcast = bitcast <4 x float> %arg to <4 x i32>
+  %in = insertelement <4 x i32> undef, i32 %broadcastvalue, i32 0
+  %mask = shufflevector <4 x i32> %in, <4 x i32> undef, <4 x i32> zeroinitializer
+  %and = and <4 x i32> %bitcast, %mask
+  %floatcast = bitcast <4 x i32> %and to <4 x float>
+  %max_is_x = fcmp oge <4 x float> %floatcast, %arg2
+  %max = select <4 x i1> %max_is_x, <4 x float> %floatcast, <4 x float> %arg2
+  ret <4 x float> %max
+}
+
+; CHECK-LABEL: ExeDepsFix_broadcastss256_inreg
+; CHECK: broadcastss
+; CHECK: vandps
+; CHECK: vmaxps
+; CHECK: ret
+define <8 x float> @ExeDepsFix_broadcastss256_inreg(<8 x float> %arg, <8 x float> %arg2, i32 %broadcastvalue) {
+  %bitcast = bitcast <8 x float> %arg to <8 x i32>
+  %in = insertelement <8 x i32> undef, i32 %broadcastvalue, i32 0
+  %mask = shufflevector <8 x i32> %in, <8 x i32> undef, <8 x i32> zeroinitializer
+  %and = and <8 x i32> %bitcast, %mask
+  %floatcast = bitcast <8 x i32> %and to <8 x float>
+  %max_is_x = fcmp oge <8 x float> %floatcast, %arg2
+  %max = select <8 x i1> %max_is_x, <8 x float> %floatcast, <8 x float> %arg2
+  ret <8 x float> %max
+}
+
+; CHECK-LABEL: ExeDepsFix_broadcastsd
+; In that case the broadcast is directly folded into vandpd.
+; CHECK: vandpd
+; CHECK: vmaxpd
+; CHECK:ret
+define <2 x double> @ExeDepsFix_broadcastsd(<2 x double> %arg, <2 x double> %arg2) {
+  %bitcast = bitcast <2 x double> %arg to <2 x i64>
+  %and = and <2 x i64> %bitcast, <i64 2147483647, i64 2147483647>
+  %floatcast = bitcast <2 x i64> %and to <2 x double>
+  %max_is_x = fcmp oge <2 x double> %floatcast, %arg2
+  %max = select <2 x i1> %max_is_x, <2 x double> %floatcast, <2 x double> %arg2
+  ret <2 x double> %max
+}
+
+; CHECK-LABEL: ExeDepsFix_broadcastsd256
+; CHECK: broadcastsd
+; CHECK: vandpd
+; CHECK: vmaxpd
+; CHECK: ret
+define <4 x double> @ExeDepsFix_broadcastsd256(<4 x double> %arg, <4 x double> %arg2) {
+  %bitcast = bitcast <4 x double> %arg to <4 x i64>
+  %and = and <4 x i64> %bitcast, <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>
+  %floatcast = bitcast <4 x i64> %and to <4 x double>
+  %max_is_x = fcmp oge <4 x double> %floatcast, %arg2
+  %max = select <4 x i1> %max_is_x, <4 x double> %floatcast, <4 x double> %arg2
+  ret <4 x double> %max
+}
+
+
+; CHECK-LABEL: ExeDepsFix_broadcastsd_inreg
+; ExeDepsFix works top down, thus it coalesces vmovlhps domain with
+; vandps and there is nothing more you can do to match vmaxpd.
+; CHECK: vmovlhps
+; CHECK: vandps
+; CHECK: vmaxpd
+; CHECK: ret
+define <2 x double> @ExeDepsFix_broadcastsd_inreg(<2 x double> %arg, <2 x double> %arg2, i64 %broadcastvalue) {
+  %bitcast = bitcast <2 x double> %arg to <2 x i64>
+  %in = insertelement <2 x i64> undef, i64 %broadcastvalue, i32 0
+  %mask = shufflevector <2 x i64> %in, <2 x i64> undef, <2 x i32> zeroinitializer
+  %and = and <2 x i64> %bitcast, %mask
+  %floatcast = bitcast <2 x i64> %and to <2 x double>
+  %max_is_x = fcmp oge <2 x double> %floatcast, %arg2
+  %max = select <2 x i1> %max_is_x, <2 x double> %floatcast, <2 x double> %arg2
+  ret <2 x double> %max
+}
+
+; CHECK-LABEL: ExeDepsFix_broadcastsd256_inreg
+; CHECK: broadcastsd
+; CHECK: vandpd
+; CHECK: vmaxpd
+; CHECK: ret
+define <4 x double> @ExeDepsFix_broadcastsd256_inreg(<4 x double> %arg, <4 x double> %arg2, i64 %broadcastvalue) {
+  %bitcast = bitcast <4 x double> %arg to <4 x i64>
+  %in = insertelement <4 x i64> undef, i64 %broadcastvalue, i32 0
+  %mask = shufflevector <4 x i64> %in, <4 x i64> undef, <4 x i32> zeroinitializer
+  %and = and <4 x i64> %bitcast, %mask
+  %floatcast = bitcast <4 x i64> %and to <4 x double>
+  %max_is_x = fcmp oge <4 x double> %floatcast, %arg2
+  %max = select <4 x i1> %max_is_x, <4 x double> %floatcast, <4 x double> %arg2
+  ret <4 x double> %max
+}
+