1 ; Make sure we still form mad even when unsafe math or fp-contract is allowed instead of fma.
3 ; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=SI-STD -check-prefix=FUNC %s
4 ; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs -fp-contract=fast < %s | FileCheck -check-prefix=SI -check-prefix=SI-STD -check-prefix=FUNC %s
5 ; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=SI -check-prefix=SI-STD -check-prefix=FUNC %s
7 ; Make sure we don't form mad with denormals
8 ; RUN: llc -march=amdgcn -mcpu=tahiti -mattr=+fp32-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=SI-DENORM -check-prefix=FUNC %s
9 ; RUN: llc -march=amdgcn -mcpu=verde -mattr=+fp32-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=SI-DENORM-SLOWFMAF -check-prefix=FUNC %s
11 declare i32 @llvm.r600.read.tidig.x() #0
12 declare float @llvm.fabs.f32(float) #0
13 declare float @llvm.fma.f32(float, float, float) #0
14 declare float @llvm.fmuladd.f32(float, float, float) #0
16 ; (fadd (fmul x, y), z) -> (fma x, y, z)
17 ; FUNC-LABEL: {{^}}combine_to_mad_f32_0:
18 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
19 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
20 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
22 ; SI-STD: v_mad_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[C]]
24 ; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[C]]
26 ; SI-DENORM-SLOWFMAF-NOT: v_fma
27 ; SI-DENORM-SLOWFMAF-NOT: v_mad
29 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
30 ; SI-DENORM-SLOWFMAF: v_add_f32_e32 [[RESULT:v[0-9]+]], [[C]], [[TMP]]
32 ; SI: buffer_store_dword [[RESULT]]
33 define void @combine_to_mad_f32_0(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
34 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
35 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
36 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
37 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
38 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
40 %a = load float, float addrspace(1)* %gep.0
41 %b = load float, float addrspace(1)* %gep.1
42 %c = load float, float addrspace(1)* %gep.2
44 %mul = fmul float %a, %b
45 %fma = fadd float %mul, %c
46 store float %fma, float addrspace(1)* %gep.out
50 ; (fadd (fmul x, y), z) -> (fma x, y, z)
51 ; FUNC-LABEL: {{^}}combine_to_mad_f32_0_2use:
52 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
53 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
54 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
55 ; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}}
57 ; SI-STD-DAG: v_mad_f32 [[RESULT0:v[0-9]+]], [[A]], [[B]], [[C]]
58 ; SI-STD-DAG: v_mad_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], [[D]]
60 ; SI-DENORM-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], [[A]], [[B]], [[C]]
61 ; SI-DENORM-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], [[D]]
63 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
64 ; SI-DENORM-SLOWFMAF-DAG: v_add_f32_e32 [[RESULT0:v[0-9]+]], [[C]], [[TMP]]
65 ; SI-DENORM-SLOWFMAF-DAG: v_add_f32_e32 [[RESULT1:v[0-9]+]], [[D]], [[TMP]]
67 ; SI-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
68 ; SI-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
70 define void @combine_to_mad_f32_0_2use(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
71 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
72 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
73 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
74 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
75 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
76 %gep.out.0 = getelementptr float, float addrspace(1)* %out, i32 %tid
77 %gep.out.1 = getelementptr float, float addrspace(1)* %gep.out.0, i32 1
79 %a = load float, float addrspace(1)* %gep.0
80 %b = load float, float addrspace(1)* %gep.1
81 %c = load float, float addrspace(1)* %gep.2
82 %d = load float, float addrspace(1)* %gep.3
84 %mul = fmul float %a, %b
85 %fma0 = fadd float %mul, %c
86 %fma1 = fadd float %mul, %d
88 store float %fma0, float addrspace(1)* %gep.out.0
89 store float %fma1, float addrspace(1)* %gep.out.1
93 ; (fadd x, (fmul y, z)) -> (fma y, z, x)
94 ; FUNC-LABEL: {{^}}combine_to_mad_f32_1:
95 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
96 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
97 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
99 ; SI-STD: v_mad_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[C]]
100 ; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[C]]
102 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
103 ; SI-DENORM-SLOWFMAF: v_add_f32_e32 [[RESULT:v[0-9]+]], [[TMP]], [[C]]
105 ; SI: buffer_store_dword [[RESULT]]
106 define void @combine_to_mad_f32_1(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
107 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
108 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
109 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
110 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
111 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
113 %a = load float, float addrspace(1)* %gep.0
114 %b = load float, float addrspace(1)* %gep.1
115 %c = load float, float addrspace(1)* %gep.2
117 %mul = fmul float %a, %b
118 %fma = fadd float %c, %mul
119 store float %fma, float addrspace(1)* %gep.out
123 ; (fsub (fmul x, y), z) -> (fma x, y, (fneg z))
124 ; FUNC-LABEL: {{^}}combine_to_mad_fsub_0_f32:
125 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
126 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
127 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
129 ; SI-STD: v_mad_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], -[[C]]
130 ; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], -[[C]]
132 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
133 ; SI-DENORM-SLOWFMAF: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[C]], [[TMP]]
135 ; SI: buffer_store_dword [[RESULT]]
136 define void @combine_to_mad_fsub_0_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
137 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
138 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
139 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
140 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
141 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
143 %a = load float, float addrspace(1)* %gep.0
144 %b = load float, float addrspace(1)* %gep.1
145 %c = load float, float addrspace(1)* %gep.2
147 %mul = fmul float %a, %b
148 %fma = fsub float %mul, %c
149 store float %fma, float addrspace(1)* %gep.out
153 ; (fsub (fmul x, y), z) -> (fma x, y, (fneg z))
154 ; FUNC-LABEL: {{^}}combine_to_mad_fsub_0_f32_2use:
155 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
156 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
157 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
158 ; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}}
160 ; SI-STD-DAG: v_mad_f32 [[RESULT0:v[0-9]+]], [[A]], [[B]], -[[C]]
161 ; SI-STD-DAG: v_mad_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], -[[D]]
163 ; SI-DENORM-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], [[A]], [[B]], -[[C]]
164 ; SI-DENORM-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], -[[D]]
166 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
167 ; SI-DENORM-SLOWFMAF-DAG: v_subrev_f32_e32 [[RESULT0:v[0-9]+]], [[C]], [[TMP]]
168 ; SI-DENORM-SLOWFMAF-DAG: v_subrev_f32_e32 [[RESULT1:v[0-9]+]], [[D]], [[TMP]]
170 ; SI-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
171 ; SI-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
173 define void @combine_to_mad_fsub_0_f32_2use(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
174 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
175 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
176 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
177 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
178 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
179 %gep.out.0 = getelementptr float, float addrspace(1)* %out, i32 %tid
180 %gep.out.1 = getelementptr float, float addrspace(1)* %gep.out.0, i32 1
182 %a = load float, float addrspace(1)* %gep.0
183 %b = load float, float addrspace(1)* %gep.1
184 %c = load float, float addrspace(1)* %gep.2
185 %d = load float, float addrspace(1)* %gep.3
187 %mul = fmul float %a, %b
188 %fma0 = fsub float %mul, %c
189 %fma1 = fsub float %mul, %d
190 store float %fma0, float addrspace(1)* %gep.out.0
191 store float %fma1, float addrspace(1)* %gep.out.1
195 ; (fsub x, (fmul y, z)) -> (fma (fneg y), z, x)
196 ; FUNC-LABEL: {{^}}combine_to_mad_fsub_1_f32:
197 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
198 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
199 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
201 ; SI-STD: v_mad_f32 [[RESULT:v[0-9]+]], -[[A]], [[B]], [[C]]
202 ; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], -[[A]], [[B]], [[C]]
204 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
205 ; SI-DENORM-SLOWFMAF: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[TMP]], [[C]]
207 ; SI: buffer_store_dword [[RESULT]]
208 define void @combine_to_mad_fsub_1_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
209 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
210 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
211 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
212 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
213 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
215 %a = load float, float addrspace(1)* %gep.0
216 %b = load float, float addrspace(1)* %gep.1
217 %c = load float, float addrspace(1)* %gep.2
219 %mul = fmul float %a, %b
220 %fma = fsub float %c, %mul
221 store float %fma, float addrspace(1)* %gep.out
225 ; (fsub x, (fmul y, z)) -> (fma (fneg y), z, x)
226 ; FUNC-LABEL: {{^}}combine_to_mad_fsub_1_f32_2use:
227 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
228 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
229 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
231 ; SI-STD-DAG: v_mad_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], [[C]]
232 ; SI-STD-DAG: v_mad_f32 [[RESULT1:v[0-9]+]], -[[A]], [[B]], [[D]]
234 ; SI-DENORM-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], [[C]]
235 ; SI-DENORM-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], -[[A]], [[B]], [[D]]
237 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
238 ; SI-DENORM-SLOWFMAF-DAG: v_subrev_f32_e32 [[RESULT0:v[0-9]+]], [[TMP]], [[C]]
239 ; SI-DENORM-SLOWFMAF-DAG: v_subrev_f32_e32 [[RESULT1:v[0-9]+]], [[TMP]], [[D]]
241 ; SI-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
242 ; SI-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
244 define void @combine_to_mad_fsub_1_f32_2use(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
245 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
246 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
247 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
248 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
249 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
250 %gep.out.0 = getelementptr float, float addrspace(1)* %out, i32 %tid
251 %gep.out.1 = getelementptr float, float addrspace(1)* %gep.out.0, i32 1
253 %a = load float, float addrspace(1)* %gep.0
254 %b = load float, float addrspace(1)* %gep.1
255 %c = load float, float addrspace(1)* %gep.2
256 %d = load float, float addrspace(1)* %gep.3
258 %mul = fmul float %a, %b
259 %fma0 = fsub float %c, %mul
260 %fma1 = fsub float %d, %mul
261 store float %fma0, float addrspace(1)* %gep.out.0
262 store float %fma1, float addrspace(1)* %gep.out.1
266 ; (fsub (fneg (fmul x, y)), z) -> (fma (fneg x), y, (fneg z))
267 ; FUNC-LABEL: {{^}}combine_to_mad_fsub_2_f32:
268 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
269 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
270 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
272 ; SI-STD: v_mad_f32 [[RESULT:v[0-9]+]], -[[A]], [[B]], -[[C]]
274 ; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], -[[A]], [[B]], -[[C]]
276 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
277 ; SI-DENORM-SLOWFMAF: v_sub_f32_e64 [[RESULT:v[0-9]+]], -[[TMP]], [[C]]
279 ; SI: buffer_store_dword [[RESULT]]
280 define void @combine_to_mad_fsub_2_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
281 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
282 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
283 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
284 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
285 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
287 %a = load float, float addrspace(1)* %gep.0
288 %b = load float, float addrspace(1)* %gep.1
289 %c = load float, float addrspace(1)* %gep.2
291 %mul = fmul float %a, %b
292 %mul.neg = fsub float -0.0, %mul
293 %fma = fsub float %mul.neg, %c
295 store float %fma, float addrspace(1)* %gep.out
299 ; (fsub (fneg (fmul x, y)), z) -> (fma (fneg x), y, (fneg z))
300 ; FUNC-LABEL: {{^}}combine_to_mad_fsub_2_f32_2uses_neg:
301 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
302 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
303 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
305 ; SI-STD-DAG: v_mad_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], -[[C]]
306 ; SI-STD-DAG: v_mad_f32 [[RESULT1:v[0-9]+]], -[[A]], [[B]], -[[D]]
308 ; SI-DENORM-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], -[[C]]
309 ; SI-DENORM-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], -[[A]], [[B]], -[[D]]
311 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
312 ; SI-DENORM-SLOWFMAF-DAG: v_sub_f32_e64 [[RESULT0:v[0-9]+]], -[[TMP]], [[C]]
313 ; SI-DENORM-SLOWFMAF-DAG: v_sub_f32_e64 [[RESULT1:v[0-9]+]], -[[TMP]], [[D]]
315 ; SI-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
316 ; SI-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
318 define void @combine_to_mad_fsub_2_f32_2uses_neg(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
319 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
320 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
321 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
322 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
323 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
324 %gep.out.0 = getelementptr float, float addrspace(1)* %out, i32 %tid
325 %gep.out.1 = getelementptr float, float addrspace(1)* %gep.out.0, i32 1
327 %a = load float, float addrspace(1)* %gep.0
328 %b = load float, float addrspace(1)* %gep.1
329 %c = load float, float addrspace(1)* %gep.2
330 %d = load float, float addrspace(1)* %gep.3
332 %mul = fmul float %a, %b
333 %mul.neg = fsub float -0.0, %mul
334 %fma0 = fsub float %mul.neg, %c
335 %fma1 = fsub float %mul.neg, %d
337 store float %fma0, float addrspace(1)* %gep.out.0
338 store float %fma1, float addrspace(1)* %gep.out.1
342 ; (fsub (fneg (fmul x, y)), z) -> (fma (fneg x), y, (fneg z))
343 ; FUNC-LABEL: {{^}}combine_to_mad_fsub_2_f32_2uses_mul:
344 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
345 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
346 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
348 ; SI-STD-DAG: v_mad_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], -[[C]]
349 ; SI-STD-DAG: v_mad_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], -[[D]]
351 ; SI-DENORM-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], -[[C]]
352 ; SI-DENORM-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], -[[D]]
354 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
355 ; SI-DENORM-SLOWFMAF-DAG: v_sub_f32_e64 [[RESULT0:v[0-9]+]], -[[TMP]], [[C]]
356 ; SI-DENORM-SLOWFMAF-DAG: v_subrev_f32_e32 [[RESULT1:v[0-9]+]], [[D]], [[TMP]]
358 ; SI-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
359 ; SI-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
361 define void @combine_to_mad_fsub_2_f32_2uses_mul(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
362 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
363 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
364 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
365 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
366 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
367 %gep.out.0 = getelementptr float, float addrspace(1)* %out, i32 %tid
368 %gep.out.1 = getelementptr float, float addrspace(1)* %gep.out.0, i32 1
370 %a = load float, float addrspace(1)* %gep.0
371 %b = load float, float addrspace(1)* %gep.1
372 %c = load float, float addrspace(1)* %gep.2
373 %d = load float, float addrspace(1)* %gep.3
375 %mul = fmul float %a, %b
376 %mul.neg = fsub float -0.0, %mul
377 %fma0 = fsub float %mul.neg, %c
378 %fma1 = fsub float %mul, %d
380 store float %fma0, float addrspace(1)* %gep.out.0
381 store float %fma1, float addrspace(1)* %gep.out.1
385 ; fold (fsub (fma x, y, (fmul u, v)), z) -> (fma x, y (fma u, v, (fneg z)))
387 ; FUNC-LABEL: {{^}}aggressive_combine_to_mad_fsub_0_f32:
388 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
389 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
390 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
391 ; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}}
392 ; SI-DAG: buffer_load_dword [[E:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16{{$}}
394 ; SI-STD: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[E]], [[D]]
395 ; SI-STD: v_fma_f32 [[TMP1:v[0-9]+]], [[A]], [[B]], [[TMP0]]
396 ; SI-STD: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[C]], [[TMP1]]
398 ; SI-DENORM: v_fma_f32 [[TMP0:v[0-9]+]], [[D]], [[E]], -[[C]]
399 ; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[TMP0]]
401 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[E]], [[D]]
402 ; SI-DENORM-SLOWFMAF: v_fma_f32 [[TMP1:v[0-9]+]], [[A]], [[B]], [[TMP0]]
403 ; SI-DENORM-SLOWFMAF: v_subrev_f32_e32 [[RESULT1:v[0-9]+]], [[C]], [[TMP1]]
405 ; SI: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
406 define void @aggressive_combine_to_mad_fsub_0_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
407 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
408 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
409 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
410 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
411 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
412 %gep.4 = getelementptr float, float addrspace(1)* %gep.0, i32 4
413 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
415 %x = load float, float addrspace(1)* %gep.0
416 %y = load float, float addrspace(1)* %gep.1
417 %z = load float, float addrspace(1)* %gep.2
418 %u = load float, float addrspace(1)* %gep.3
419 %v = load float, float addrspace(1)* %gep.4
421 %tmp0 = fmul float %u, %v
422 %tmp1 = call float @llvm.fma.f32(float %x, float %y, float %tmp0) #0
423 %tmp2 = fsub float %tmp1, %z
425 store float %tmp2, float addrspace(1)* %gep.out
429 ; fold (fsub x, (fma y, z, (fmul u, v)))
430 ; -> (fma (fneg y), z, (fma (fneg u), v, x))
432 ; FUNC-LABEL: {{^}}aggressive_combine_to_mad_fsub_1_f32:
433 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
434 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
435 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
436 ; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}}
437 ; SI-DAG: buffer_load_dword [[E:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16{{$}}
439 ; SI-STD: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[E]], [[D]]
440 ; SI-STD: v_fma_f32 [[TMP1:v[0-9]+]], [[B]], [[C]], [[TMP0]]
441 ; SI-STD: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[TMP1]], [[A]]
443 ; SI-DENORM: v_fma_f32 [[TMP0:v[0-9]+]], -[[D]], [[E]], [[A]]
444 ; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], -[[B]], [[C]], [[TMP0]]
446 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[E]], [[D]]
447 ; SI-DENORM-SLOWFMAF: v_fma_f32 [[TMP1:v[0-9]+]], [[B]], [[C]], [[TMP0]]
448 ; SI-DENORM-SLOWFMAF: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[TMP1]], [[A]]
450 ; SI: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
452 define void @aggressive_combine_to_mad_fsub_1_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
453 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
454 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
455 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
456 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
457 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
458 %gep.4 = getelementptr float, float addrspace(1)* %gep.0, i32 4
459 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
461 %x = load float, float addrspace(1)* %gep.0
462 %y = load float, float addrspace(1)* %gep.1
463 %z = load float, float addrspace(1)* %gep.2
464 %u = load float, float addrspace(1)* %gep.3
465 %v = load float, float addrspace(1)* %gep.4
467 %tmp0 = fmul float %u, %v
468 %tmp1 = call float @llvm.fma.f32(float %y, float %z, float %tmp0) #0
469 %tmp2 = fsub float %x, %tmp1
471 store float %tmp2, float addrspace(1)* %gep.out
475 ; fold (fsub (fma x, y, (fmul u, v)), z) -> (fma x, y (fma u, v, (fneg z)))
477 ; FUNC-LABEL: {{^}}aggressive_combine_to_mad_fsub_2_f32:
478 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
479 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
480 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
481 ; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}}
482 ; SI-DAG: buffer_load_dword [[E:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16{{$}}
484 ; SI-STD: v_mad_f32 [[TMP:v[0-9]+]], [[D]], [[E]], -[[C]]
485 ; SI-STD: v_mad_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[TMP]]
487 ; SI-DENORM: v_fma_f32 [[TMP:v[0-9]+]], [[D]], [[E]], -[[C]]
488 ; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[TMP]]
490 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[E]], [[D]]
491 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP1:v[0-9]+]], [[B]], [[A]]
492 ; SI-DENORM-SLOWFMAF: v_add_f32_e32 [[TMP2:v[0-9]+]], [[TMP0]], [[TMP1]]
493 ; SI-DENORM-SLOWFMAF: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[C]], [[TMP2]]
495 ; SI: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
497 define void @aggressive_combine_to_mad_fsub_2_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
498 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
499 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
500 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
501 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
502 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
503 %gep.4 = getelementptr float, float addrspace(1)* %gep.0, i32 4
504 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
506 %x = load float, float addrspace(1)* %gep.0
507 %y = load float, float addrspace(1)* %gep.1
508 %z = load float, float addrspace(1)* %gep.2
509 %u = load float, float addrspace(1)* %gep.3
510 %v = load float, float addrspace(1)* %gep.4
512 %tmp0 = fmul float %u, %v
513 %tmp1 = call float @llvm.fmuladd.f32(float %x, float %y, float %tmp0) #0
514 %tmp2 = fsub float %tmp1, %z
516 store float %tmp2, float addrspace(1)* %gep.out
520 ; fold (fsub x, (fmuladd y, z, (fmul u, v)))
521 ; -> (fmuladd (fneg y), z, (fmuladd (fneg u), v, x))
523 ; FUNC-LABEL: {{^}}aggressive_combine_to_mad_fsub_3_f32:
524 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
525 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
526 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
527 ; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}}
528 ; SI-DAG: buffer_load_dword [[E:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16{{$}}
530 ; SI-STD: v_mad_f32 [[TMP:v[0-9]+]], -[[D]], [[E]], [[A]]
531 ; SI-STD: v_mad_f32 [[RESULT:v[0-9]+]], -[[B]], [[C]], [[TMP]]
533 ; SI-DENORM: v_fma_f32 [[TMP:v[0-9]+]], -[[D]], [[E]], [[A]]
534 ; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], -[[B]], [[C]], [[TMP]]
536 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[E]], [[D]]
537 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP1:v[0-9]+]], [[C]], [[B]]
538 ; SI-DENORM-SLOWFMAF: v_add_f32_e32 [[TMP2:v[0-9]+]], [[TMP0]], [[TMP1]]
539 ; SI-DENORM-SLOWFMAF: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[TMP2]], [[A]]
541 ; SI: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
543 define void @aggressive_combine_to_mad_fsub_3_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
544 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
545 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
546 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
547 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
548 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
549 %gep.4 = getelementptr float, float addrspace(1)* %gep.0, i32 4
550 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
552 %x = load float, float addrspace(1)* %gep.0
553 %y = load float, float addrspace(1)* %gep.1
554 %z = load float, float addrspace(1)* %gep.2
555 %u = load float, float addrspace(1)* %gep.3
556 %v = load float, float addrspace(1)* %gep.4
558 %tmp0 = fmul float %u, %v
559 %tmp1 = call float @llvm.fmuladd.f32(float %y, float %z, float %tmp0) #0
560 %tmp2 = fsub float %x, %tmp1
562 store float %tmp2, float addrspace(1)* %gep.out
566 attributes #0 = { nounwind readnone }
567 attributes #1 = { nounwind }