1 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512F
2 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512BW
4 define <16 x i32> @_inreg16xi32(i32 %a) {
5 ; ALL-LABEL: _inreg16xi32:
7 ; ALL-NEXT: vpbroadcastd %edi, %zmm0
9 %b = insertelement <16 x i32> undef, i32 %a, i32 0
10 %c = shufflevector <16 x i32> %b, <16 x i32> undef, <16 x i32> zeroinitializer
14 define <8 x i64> @_inreg8xi64(i64 %a) {
15 ; ALL-LABEL: _inreg8xi64:
17 ; ALL-NEXT: vpbroadcastq %rdi, %zmm0
19 %b = insertelement <8 x i64> undef, i64 %a, i32 0
20 %c = shufflevector <8 x i64> %b, <8 x i64> undef, <8 x i32> zeroinitializer
24 define <16 x float> @_ss16xfloat_v4(<4 x float> %a) {
25 ; ALL-LABEL: _ss16xfloat_v4:
27 ; ALL-NEXT: vbroadcastss %xmm0, %zmm0
29 %b = shufflevector <4 x float> %a, <4 x float> undef, <16 x i32> zeroinitializer
33 define <16 x float> @_inreg16xfloat(float %a) {
34 ; ALL-LABEL: _inreg16xfloat:
36 ; ALL-NEXT: vbroadcastss %xmm0, %zmm0
38 %b = insertelement <16 x float> undef, float %a, i32 0
39 %c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer
43 define <16 x float> @_ss16xfloat_mask(float %a, <16 x float> %i, <16 x i32> %mask1) {
44 ; ALL-LABEL: _ss16xfloat_mask:
46 ; ALL-NEXT: vpxord %zmm3, %zmm3, %zmm3
47 ; ALL-NEXT: vpcmpneqd %zmm3, %zmm2, %k1
48 ; ALL-NEXT: vbroadcastss %xmm0, %zmm1 {%k1}
49 ; ALL-NEXT: vmovaps %zmm1, %zmm0
51 %mask = icmp ne <16 x i32> %mask1, zeroinitializer
52 %b = insertelement <16 x float> undef, float %a, i32 0
53 %c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer
54 %r = select <16 x i1> %mask, <16 x float> %c, <16 x float> %i
58 define <16 x float> @_ss16xfloat_maskz(float %a, <16 x i32> %mask1) {
59 ; ALL-LABEL: _ss16xfloat_maskz:
61 ; ALL-NEXT: vpxord %zmm2, %zmm2, %zmm2
62 ; ALL-NEXT: vpcmpneqd %zmm2, %zmm1, %k1
63 ; ALL-NEXT: vbroadcastss %xmm0, %zmm0 {%k1} {z}
65 %mask = icmp ne <16 x i32> %mask1, zeroinitializer
66 %b = insertelement <16 x float> undef, float %a, i32 0
67 %c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer
68 %r = select <16 x i1> %mask, <16 x float> %c, <16 x float> zeroinitializer
72 define <16 x float> @_ss16xfloat_load(float* %a.ptr) {
73 ; ALL-LABEL: _ss16xfloat_load:
75 ; ALL-NEXT: vbroadcastss (%rdi), %zmm0
77 %a = load float, float* %a.ptr
78 %b = insertelement <16 x float> undef, float %a, i32 0
79 %c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer
83 define <16 x float> @_ss16xfloat_mask_load(float* %a.ptr, <16 x float> %i, <16 x i32> %mask1) {
84 ; ALL-LABEL: _ss16xfloat_mask_load:
86 ; ALL-NEXT: vpxord %zmm2, %zmm2, %zmm2
87 ; ALL-NEXT: vpcmpneqd %zmm2, %zmm1, %k1
88 ; ALL-NEXT: vbroadcastss (%rdi), %zmm0 {%k1}
90 %a = load float, float* %a.ptr
91 %mask = icmp ne <16 x i32> %mask1, zeroinitializer
92 %b = insertelement <16 x float> undef, float %a, i32 0
93 %c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer
94 %r = select <16 x i1> %mask, <16 x float> %c, <16 x float> %i
98 define <16 x float> @_ss16xfloat_maskz_load(float* %a.ptr, <16 x i32> %mask1) {
99 ; ALL-LABEL: _ss16xfloat_maskz_load:
101 ; ALL-NEXT: vpxord %zmm1, %zmm1, %zmm1
102 ; ALL-NEXT: vpcmpneqd %zmm1, %zmm0, %k1
103 ; ALL-NEXT: vbroadcastss (%rdi), %zmm0 {%k1} {z}
105 %a = load float, float* %a.ptr
106 %mask = icmp ne <16 x i32> %mask1, zeroinitializer
107 %b = insertelement <16 x float> undef, float %a, i32 0
108 %c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer
109 %r = select <16 x i1> %mask, <16 x float> %c, <16 x float> zeroinitializer
113 define <8 x double> @_inreg8xdouble(double %a) {
114 ; ALL-LABEL: _inreg8xdouble:
116 ; ALL-NEXT: vbroadcastsd %xmm0, %zmm0
118 %b = insertelement <8 x double> undef, double %a, i32 0
119 %c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer
123 define <8 x double> @_sd8xdouble_mask(double %a, <8 x double> %i, <8 x i32> %mask1) {
124 ; ALL-LABEL: _sd8xdouble_mask:
126 ; ALL-NEXT: vpxor %ymm3, %ymm3, %ymm3
127 ; ALL-NEXT: vpcmpneqd %zmm3, %zmm2, %k1
128 ; ALL-NEXT: vbroadcastsd %xmm0, %zmm1 {%k1}
129 ; ALL-NEXT: vmovaps %zmm1, %zmm0
131 %mask = icmp ne <8 x i32> %mask1, zeroinitializer
132 %b = insertelement <8 x double> undef, double %a, i32 0
133 %c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer
134 %r = select <8 x i1> %mask, <8 x double> %c, <8 x double> %i
138 define <8 x double> @_sd8xdouble_maskz(double %a, <8 x i32> %mask1) {
139 ; ALL-LABEL: _sd8xdouble_maskz:
141 ; ALL-NEXT: vpxor %ymm2, %ymm2, %ymm2
142 ; ALL-NEXT: vpcmpneqd %zmm2, %zmm1, %k1
143 ; ALL-NEXT: vbroadcastsd %xmm0, %zmm0 {%k1} {z}
145 %mask = icmp ne <8 x i32> %mask1, zeroinitializer
146 %b = insertelement <8 x double> undef, double %a, i32 0
147 %c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer
148 %r = select <8 x i1> %mask, <8 x double> %c, <8 x double> zeroinitializer
152 define <8 x double> @_sd8xdouble_load(double* %a.ptr) {
153 ; ALL-LABEL: _sd8xdouble_load:
155 ; ALL-NEXT: vbroadcastsd (%rdi), %zmm0
157 %a = load double, double* %a.ptr
158 %b = insertelement <8 x double> undef, double %a, i32 0
159 %c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer
163 define <8 x double> @_sd8xdouble_mask_load(double* %a.ptr, <8 x double> %i, <8 x i32> %mask1) {
164 ; ALL-LABEL: _sd8xdouble_mask_load:
166 ; ALL-NEXT: vpxor %ymm2, %ymm2, %ymm2
167 ; ALL-NEXT: vpcmpneqd %zmm2, %zmm1, %k1
168 ; ALL-NEXT: vbroadcastsd (%rdi), %zmm0 {%k1}
170 %a = load double, double* %a.ptr
171 %mask = icmp ne <8 x i32> %mask1, zeroinitializer
172 %b = insertelement <8 x double> undef, double %a, i32 0
173 %c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer
174 %r = select <8 x i1> %mask, <8 x double> %c, <8 x double> %i
178 define <8 x double> @_sd8xdouble_maskz_load(double* %a.ptr, <8 x i32> %mask1) {
179 ; ALL-LABEL: _sd8xdouble_maskz_load:
181 ; ALL-NEXT: vpxor %ymm1, %ymm1, %ymm1
182 ; ALL-NEXT: vpcmpneqd %zmm1, %zmm0, %k1
183 ; ALL-NEXT: vbroadcastsd (%rdi), %zmm0 {%k1} {z}
185 %a = load double, double* %a.ptr
186 %mask = icmp ne <8 x i32> %mask1, zeroinitializer
187 %b = insertelement <8 x double> undef, double %a, i32 0
188 %c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer
189 %r = select <8 x i1> %mask, <8 x double> %c, <8 x double> zeroinitializer
193 define <16 x i32> @_xmm16xi32(<16 x i32> %a) {
194 ; ALL-LABEL: _xmm16xi32:
196 ; ALL-NEXT: vpbroadcastd %xmm0, %zmm0
198 %b = shufflevector <16 x i32> %a, <16 x i32> undef, <16 x i32> zeroinitializer
202 define <16 x float> @_xmm16xfloat(<16 x float> %a) {
203 ; ALL-LABEL: _xmm16xfloat:
205 ; ALL-NEXT: vbroadcastss %xmm0, %zmm0
207 %b = shufflevector <16 x float> %a, <16 x float> undef, <16 x i32> zeroinitializer
211 define <16 x i32> @test_vbroadcast() {
212 ; ALL-LABEL: test_vbroadcast:
213 ; ALL: # BB#0: # %entry
214 ; ALL-NEXT: vpxord %zmm0, %zmm0, %zmm0
215 ; ALL-NEXT: vcmpunordps %zmm0, %zmm0, %k1
216 ; ALL-NEXT: vpbroadcastd {{.*}}(%rip), %zmm0 {%k1} {z}
217 ; ALL-NEXT: knotw %k1, %k1
218 ; ALL-NEXT: vmovdqu32 %zmm0, %zmm0 {%k1} {z}
221 %0 = sext <16 x i1> zeroinitializer to <16 x i32>
222 %1 = fcmp uno <16 x float> undef, zeroinitializer
223 %2 = sext <16 x i1> %1 to <16 x i32>
224 %3 = select <16 x i1> %1, <16 x i32> %0, <16 x i32> %2
228 ; We implement the set1 intrinsics with vector initializers. Verify that the
229 ; IR generated will produce broadcasts at the end.
230 define <8 x double> @test_set1_pd(double %d) #2 {
231 ; ALL-LABEL: test_set1_pd:
232 ; ALL: # BB#0: # %entry
233 ; ALL-NEXT: vbroadcastsd %xmm0, %zmm0
236 %vecinit.i = insertelement <8 x double> undef, double %d, i32 0
237 %vecinit1.i = insertelement <8 x double> %vecinit.i, double %d, i32 1
238 %vecinit2.i = insertelement <8 x double> %vecinit1.i, double %d, i32 2
239 %vecinit3.i = insertelement <8 x double> %vecinit2.i, double %d, i32 3
240 %vecinit4.i = insertelement <8 x double> %vecinit3.i, double %d, i32 4
241 %vecinit5.i = insertelement <8 x double> %vecinit4.i, double %d, i32 5
242 %vecinit6.i = insertelement <8 x double> %vecinit5.i, double %d, i32 6
243 %vecinit7.i = insertelement <8 x double> %vecinit6.i, double %d, i32 7
244 ret <8 x double> %vecinit7.i
247 define <8 x i64> @test_set1_epi64(i64 %d) #2 {
248 ; ALL-LABEL: test_set1_epi64:
249 ; ALL: # BB#0: # %entry
250 ; ALL-NEXT: vpbroadcastq %rdi, %zmm0
253 %vecinit.i = insertelement <8 x i64> undef, i64 %d, i32 0
254 %vecinit1.i = insertelement <8 x i64> %vecinit.i, i64 %d, i32 1
255 %vecinit2.i = insertelement <8 x i64> %vecinit1.i, i64 %d, i32 2
256 %vecinit3.i = insertelement <8 x i64> %vecinit2.i, i64 %d, i32 3
257 %vecinit4.i = insertelement <8 x i64> %vecinit3.i, i64 %d, i32 4
258 %vecinit5.i = insertelement <8 x i64> %vecinit4.i, i64 %d, i32 5
259 %vecinit6.i = insertelement <8 x i64> %vecinit5.i, i64 %d, i32 6
260 %vecinit7.i = insertelement <8 x i64> %vecinit6.i, i64 %d, i32 7
261 ret <8 x i64> %vecinit7.i
264 define <16 x float> @test_set1_ps(float %f) #2 {
265 ; ALL-LABEL: test_set1_ps:
266 ; ALL: # BB#0: # %entry
267 ; ALL-NEXT: vbroadcastss %xmm0, %zmm0
270 %vecinit.i = insertelement <16 x float> undef, float %f, i32 0
271 %vecinit1.i = insertelement <16 x float> %vecinit.i, float %f, i32 1
272 %vecinit2.i = insertelement <16 x float> %vecinit1.i, float %f, i32 2
273 %vecinit3.i = insertelement <16 x float> %vecinit2.i, float %f, i32 3
274 %vecinit4.i = insertelement <16 x float> %vecinit3.i, float %f, i32 4
275 %vecinit5.i = insertelement <16 x float> %vecinit4.i, float %f, i32 5
276 %vecinit6.i = insertelement <16 x float> %vecinit5.i, float %f, i32 6
277 %vecinit7.i = insertelement <16 x float> %vecinit6.i, float %f, i32 7
278 %vecinit8.i = insertelement <16 x float> %vecinit7.i, float %f, i32 8
279 %vecinit9.i = insertelement <16 x float> %vecinit8.i, float %f, i32 9
280 %vecinit10.i = insertelement <16 x float> %vecinit9.i, float %f, i32 10
281 %vecinit11.i = insertelement <16 x float> %vecinit10.i, float %f, i32 11
282 %vecinit12.i = insertelement <16 x float> %vecinit11.i, float %f, i32 12
283 %vecinit13.i = insertelement <16 x float> %vecinit12.i, float %f, i32 13
284 %vecinit14.i = insertelement <16 x float> %vecinit13.i, float %f, i32 14
285 %vecinit15.i = insertelement <16 x float> %vecinit14.i, float %f, i32 15
286 ret <16 x float> %vecinit15.i
289 define <16 x i32> @test_set1_epi32(i32 %f) #2 {
290 ; ALL-LABEL: test_set1_epi32:
291 ; ALL: # BB#0: # %entry
292 ; ALL-NEXT: vpbroadcastd %edi, %zmm0
295 %vecinit.i = insertelement <16 x i32> undef, i32 %f, i32 0
296 %vecinit1.i = insertelement <16 x i32> %vecinit.i, i32 %f, i32 1
297 %vecinit2.i = insertelement <16 x i32> %vecinit1.i, i32 %f, i32 2
298 %vecinit3.i = insertelement <16 x i32> %vecinit2.i, i32 %f, i32 3
299 %vecinit4.i = insertelement <16 x i32> %vecinit3.i, i32 %f, i32 4
300 %vecinit5.i = insertelement <16 x i32> %vecinit4.i, i32 %f, i32 5
301 %vecinit6.i = insertelement <16 x i32> %vecinit5.i, i32 %f, i32 6
302 %vecinit7.i = insertelement <16 x i32> %vecinit6.i, i32 %f, i32 7
303 %vecinit8.i = insertelement <16 x i32> %vecinit7.i, i32 %f, i32 8
304 %vecinit9.i = insertelement <16 x i32> %vecinit8.i, i32 %f, i32 9
305 %vecinit10.i = insertelement <16 x i32> %vecinit9.i, i32 %f, i32 10
306 %vecinit11.i = insertelement <16 x i32> %vecinit10.i, i32 %f, i32 11
307 %vecinit12.i = insertelement <16 x i32> %vecinit11.i, i32 %f, i32 12
308 %vecinit13.i = insertelement <16 x i32> %vecinit12.i, i32 %f, i32 13
309 %vecinit14.i = insertelement <16 x i32> %vecinit13.i, i32 %f, i32 14
310 %vecinit15.i = insertelement <16 x i32> %vecinit14.i, i32 %f, i32 15
311 ret <16 x i32> %vecinit15.i
314 ; We implement the scalar broadcast intrinsics with vector initializers.
315 ; Verify that the IR generated will produce the broadcast at the end.
316 define <8 x double> @test_mm512_broadcastsd_pd(<2 x double> %a) {
317 ; ALL-LABEL: test_mm512_broadcastsd_pd:
318 ; ALL: # BB#0: # %entry
319 ; ALL-NEXT: vbroadcastsd %xmm0, %zmm0
322 %0 = extractelement <2 x double> %a, i32 0
323 %vecinit.i = insertelement <8 x double> undef, double %0, i32 0
324 %vecinit1.i = insertelement <8 x double> %vecinit.i, double %0, i32 1
325 %vecinit2.i = insertelement <8 x double> %vecinit1.i, double %0, i32 2
326 %vecinit3.i = insertelement <8 x double> %vecinit2.i, double %0, i32 3
327 %vecinit4.i = insertelement <8 x double> %vecinit3.i, double %0, i32 4
328 %vecinit5.i = insertelement <8 x double> %vecinit4.i, double %0, i32 5
329 %vecinit6.i = insertelement <8 x double> %vecinit5.i, double %0, i32 6
330 %vecinit7.i = insertelement <8 x double> %vecinit6.i, double %0, i32 7
331 ret <8 x double> %vecinit7.i
334 define <16 x float> @test1(<8 x float>%a) {
337 ; ALL-NEXT: vbroadcastss %xmm0, %zmm0
339 %res = shufflevector <8 x float> %a, <8 x float> undef, <16 x i32> zeroinitializer
343 define <8 x double> @test2(<4 x double>%a) {
346 ; ALL-NEXT: vbroadcastsd %xmm0, %zmm0
348 %res = shufflevector <4 x double> %a, <4 x double> undef, <8 x i32> zeroinitializer
352 define <64 x i8> @_invec32xi8(<32 x i8>%a) {
353 ; AVX512F-LABEL: _invec32xi8:
355 ; AVX512F-NEXT: vpbroadcastb %xmm0, %ymm0
356 ; AVX512F-NEXT: vmovaps %zmm0, %zmm1
359 ; AVX512BW-LABEL: _invec32xi8:
361 ; AVX512BW-NEXT: vpbroadcastb %xmm0, %zmm0
362 ; AVX512BW-NEXT: retq
363 %res = shufflevector <32 x i8> %a, <32 x i8> undef, <64 x i32> zeroinitializer
367 define <32 x i16> @_invec16xi16(<16 x i16>%a) {
368 ; AVX512F-LABEL: _invec16xi16:
370 ; AVX512F-NEXT: vpbroadcastw %xmm0, %ymm0
371 ; AVX512F-NEXT: vmovaps %zmm0, %zmm1
374 ; AVX512BW-LABEL: _invec16xi16:
376 ; AVX512BW-NEXT: vpbroadcastw %xmm0, %zmm0
377 ; AVX512BW-NEXT: retq
378 %res = shufflevector <16 x i16> %a, <16 x i16> undef, <32 x i32> zeroinitializer
382 define <16 x i32> @_invec8xi32(<8 x i32>%a) {
383 ; ALL-LABEL: _invec8xi32:
385 ; ALL-NEXT: vpbroadcastd %xmm0, %zmm0
387 %res = shufflevector <8 x i32> %a, <8 x i32> undef, <16 x i32> zeroinitializer
391 define <8 x i64> @_invec4xi64(<4 x i64>%a) {
392 ; ALL-LABEL: _invec4xi64:
394 ; ALL-NEXT: vpbroadcastq %xmm0, %zmm0
396 %res = shufflevector <4 x i64> %a, <4 x i64> undef, <8 x i32> zeroinitializer