lib/Target/X86/X86InstrFMA.td

   1 //====- X86InstrFMA.td - Describe the X86 Instruction Set --*- tablegen -*-===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // This file describes FMA (Fused Multiply-Add) instructions.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 //===----------------------------------------------------------------------===//
  15 // FMA3 - Intel 3 operand Fused Multiply-Add instructions
  16 //===----------------------------------------------------------------------===//
  17
  18 multiclass fma_rm<bits<8> opc, string OpcodeStr> {
  19   def r : FMA3<opc, MRMSrcReg, (outs VR128:$dst),
  20            (ins VR128:$src1, VR128:$src2),
  21            !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
  22            []>;
  23   def m : FMA3<opc, MRMSrcMem, (outs VR128:$dst),
  24            (ins VR128:$src1, f128mem:$src2),
  25            !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
  26            []>;
  27   def rY : FMA3<opc, MRMSrcReg, (outs VR256:$dst),
  28            (ins VR256:$src1, VR256:$src2),
  29            !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
  30            []>;
  31   def mY : FMA3<opc, MRMSrcMem, (outs VR256:$dst),
  32            (ins VR256:$src1, f256mem:$src2),
  33            !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
  34            []>;
  35 }
  36
  37 multiclass fma_forms<bits<8> opc132, bits<8> opc213, bits<8> opc231,
  38                      string OpcodeStr, string PackTy> {
  39   defm r132 : fma_rm<opc132, !strconcat(OpcodeStr, !strconcat("132", PackTy))>;
  40   defm r213 : fma_rm<opc213, !strconcat(OpcodeStr, !strconcat("213", PackTy))>;
  41   defm r231 : fma_rm<opc231, !strconcat(OpcodeStr, !strconcat("231", PackTy))>;
  42 }
  43
  44 // Fused Multiply-Add
  45 defm VFMADDPS    : fma_forms<0x98, 0xA8, 0xB8, "vfmadd", "ps">;
  46 defm VFMADDPD    : fma_forms<0x98, 0xA8, 0xB8, "vfmadd", "pd">, VEX_W;
  47 defm VFMADDSUBPS : fma_forms<0x96, 0xA6, 0xB6, "vfmaddsub", "ps">;
  48 defm VFMADDSUBPD : fma_forms<0x96, 0xA6, 0xB6, "vfmaddsub", "pd">, VEX_W;
  49 defm VFMSUBADDPS : fma_forms<0x97, 0xA7, 0xB7, "vfmsubadd", "ps">;
  50 defm VFMSUBADDPD : fma_forms<0x97, 0xA7, 0xB7, "vfmsubadd", "pd">, VEX_W;
  51 defm VFMSUBPS    : fma_forms<0x9A, 0xAA, 0xBA, "vfmsub", "ps">;
  52 defm VFMSUBPD    : fma_forms<0x9A, 0xAA, 0xBA, "vfmsub", "pd">, VEX_W;
  53
  54 // Fused Negative Multiply-Add
  55 defm VFNMADDPS : fma_forms<0x9C, 0xAC, 0xBC, "vfnmadd", "ps">;
  56 defm VFNMADDPD : fma_forms<0x9C, 0xAC, 0xBC, "vfnmadd", "pd">, VEX_W;
  57 defm VFNMSUBPS : fma_forms<0x9E, 0xAE, 0xBE, "vfnmsub", "ps">;
  58 defm VFNMSUBPD : fma_forms<0x9E, 0xAE, 0xBE, "vfnmsub", "pd">, VEX_W;
  59
  60 //===----------------------------------------------------------------------===//
  61 // FMA4 - AMD 4 operand Fused Multiply-Add instructions
  62 //===----------------------------------------------------------------------===//
  63
  64
  65 multiclass fma4s<bits<8> opc, string OpcodeStr> {
  66   def rr : FMA4<opc, MRMSrcReg, (outs VR128:$dst),
  67            (ins VR128:$src1, VR128:$src2, VR128:$src3),
  68            !strconcat(OpcodeStr,
  69            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
  70            []>, XOP_W;
  71   def rm : FMA4<opc, MRMSrcMem, (outs VR128:$dst),
  72            (ins VR128:$src1, VR128:$src2, f128mem:$src3),
  73            !strconcat(OpcodeStr,
  74            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
  75            []>, XOP_W;
  76   def mr : FMA4<opc, MRMSrcMem, (outs VR128:$dst),
  77            (ins VR128:$src1, f128mem:$src2, VR128:$src3),
  78            !strconcat(OpcodeStr,
  79            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
  80            []>;
  81
  82 }
  83
  84 multiclass fma4p<bits<8> opc, string OpcodeStr> {
  85   def rr : FMA4<opc, MRMSrcReg, (outs VR128:$dst),
  86            (ins VR128:$src1, VR128:$src2, VR128:$src3),
  87            !strconcat(OpcodeStr,
  88            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
  89            []>, XOP_W;
  90   def rm : FMA4<opc, MRMSrcMem, (outs VR128:$dst),
  91            (ins VR128:$src1, VR128:$src2, f128mem:$src3),
  92            !strconcat(OpcodeStr,
  93            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
  94            []>, XOP_W;
  95   def mr : FMA4<opc, MRMSrcMem, (outs VR128:$dst),
  96            (ins VR128:$src1, f128mem:$src2, VR128:$src3),
  97            !strconcat(OpcodeStr,
  98            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
  99            []>;
 100   def rrY : FMA4<opc, MRMSrcReg, (outs VR256:$dst),
 101            (ins VR256:$src1, VR256:$src2, VR256:$src3),
 102            !strconcat(OpcodeStr,
 103            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
 104            []>, XOP_W;
 105   def rmY : FMA4<opc, MRMSrcMem, (outs VR256:$dst),
 106            (ins VR256:$src1, VR256:$src2, f256mem:$src3),
 107            !strconcat(OpcodeStr,
 108            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
 109            []>, XOP_W;
 110   def mrY : FMA4<opc, MRMSrcMem, (outs VR256:$dst),
 111            (ins VR256:$src1, f256mem:$src2, VR256:$src3),
 112            !strconcat(OpcodeStr,
 113            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
 114            []>;
 115 }
 116
 117 let isAsmParserOnly = 1 in {
 118   defm VFMADDSS4    : fma4s<0x6A, "vfmaddss">;
 119   defm VFMADDSD4    : fma4s<0x6B, "vfmaddsd">;
 120   defm VFMADDPS4    : fma4p<0x68, "vfmaddps">;
 121   defm VFMADDPD4    : fma4p<0x69, "vfmaddpd">;
 122   defm VFMSUBSS4    : fma4s<0x6E, "vfmsubss">;
 123   defm VFMSUBSD4    : fma4s<0x6F, "vfmsubsd">;
 124   defm VFMSUBPS4    : fma4p<0x6C, "vfmsubps">;
 125   defm VFMSUBPD4    : fma4p<0x6D, "vfmsubpd">;
 126   defm VFNMADDSS4   : fma4s<0x7A, "vfnmaddss">;
 127   defm VFNMADDSD4   : fma4s<0x7B, "vfnmaddsd">;
 128   defm VFNMADDPS4   : fma4p<0x78, "vfnmaddps">;
 129   defm VFNMADDPD4   : fma4p<0x79, "vfnmaddpd">;
 130   defm VFNMSUBSS4   : fma4s<0x7E, "vfnmsubss">;
 131   defm VFNMSUBSD4   : fma4s<0x7F, "vfnmsubsd">;
 132   defm VFNMSUBPS4   : fma4p<0x7C, "vfnmsubps">;
 133   defm VFNMSUBPD4   : fma4p<0x7D, "vfnmsubpd">;
 134   defm VFMADDSUBPS4 : fma4p<0x5C, "vfmaddsubps">;
 135   defm VFMADDSUBPD4 : fma4p<0x5D, "vfmaddsubpd">;
 136   defm VFMSUBADDPS4 : fma4p<0x5E, "vfmsubaddps">;
 137   defm VFMSUBADDPD4 : fma4p<0x5F, "vfmsubaddpd">;
 138 }
 139
 140 // FMA4 Intrinsics patterns
 141
 142 // VFMADD
 143 def : Pat<(int_x86_fma4_vfmadd_ss VR128:$src1, VR128:$src2, VR128:$src3),
 144           (VFMADDSS4rr VR128:$src1, VR128:$src2, VR128:$src3)>;
 145 def : Pat<(int_x86_fma4_vfmadd_ss VR128:$src1, VR128:$src2,
 146                                   (alignedloadv4f32 addr:$src3)),
 147           (VFMADDSS4rm VR128:$src1, VR128:$src2, addr:$src3)>;
 148 def : Pat<(int_x86_fma4_vfmadd_ss VR128:$src1, (alignedloadv4f32 addr:$src2),
 149                                   VR128:$src3),
 150           (VFMADDSS4mr VR128:$src1, addr:$src2, VR128:$src3)>;
 151
 152 def : Pat<(int_x86_fma4_vfmadd_sd VR128:$src1, VR128:$src2, VR128:$src3),
 153           (VFMADDSD4rr VR128:$src1, VR128:$src2, VR128:$src3)>;
 154 def : Pat<(int_x86_fma4_vfmadd_sd VR128:$src1, VR128:$src2,
 155                                   (alignedloadv2f64 addr:$src3)),
 156           (VFMADDSD4rm VR128:$src1, VR128:$src2, addr:$src3)>;
 157 def : Pat<(int_x86_fma4_vfmadd_sd VR128:$src1, (alignedloadv2f64 addr:$src2),
 158                                   VR128:$src3),
 159           (VFMADDSD4mr VR128:$src1, addr:$src2, VR128:$src3)>;
 160
 161 def : Pat<(int_x86_fma4_vfmadd_ps VR128:$src1, VR128:$src2, VR128:$src3),
 162           (VFMADDPS4rr VR128:$src1, VR128:$src2, VR128:$src3)>;
 163 def : Pat<(int_x86_fma4_vfmadd_ps VR128:$src1, VR128:$src2,
 164                                   (alignedloadv4f32 addr:$src3)),
 165           (VFMADDPS4rm VR128:$src1, VR128:$src2, addr:$src3)>;
 166 def : Pat<(int_x86_fma4_vfmadd_ps VR128:$src1, (alignedloadv4f32 addr:$src2),
 167                                   VR128:$src3),
 168           (VFMADDPS4mr VR128:$src1, addr:$src2, VR128:$src3)>;
 169
 170 def : Pat<(int_x86_fma4_vfmadd_pd VR128:$src1, VR128:$src2, VR128:$src3),
 171           (VFMADDPD4rr VR128:$src1, VR128:$src2, VR128:$src3)>;
 172 def : Pat<(int_x86_fma4_vfmadd_pd VR128:$src1, VR128:$src2,
 173                                   (alignedloadv2f64 addr:$src3)),
 174           (VFMADDPD4rm VR128:$src1, VR128:$src2, addr:$src3)>;
 175 def : Pat<(int_x86_fma4_vfmadd_pd VR128:$src1, (alignedloadv2f64 addr:$src2),
 176                                   VR128:$src3),
 177           (VFMADDPD4mr VR128:$src1, addr:$src2, VR128:$src3)>;
 178
 179 def : Pat<(int_x86_fma4_vfmadd_ps_256 VR256:$src1, VR256:$src2, VR256:$src3),
 180           (VFMADDPS4rrY VR256:$src1, VR256:$src2, VR256:$src3)>;
 181 def : Pat<(int_x86_fma4_vfmadd_ps_256 VR256:$src1, VR256:$src2,
 182                                   (alignedloadv8f32 addr:$src3)),
 183           (VFMADDPS4rmY VR256:$src1, VR256:$src2, addr:$src3)>;
 184 def : Pat<(int_x86_fma4_vfmadd_ps_256 VR256:$src1,
 185                                       (alignedloadv8f32 addr:$src2),
 186                                       VR256:$src3),
 187           (VFMADDPS4mrY VR256:$src1, addr:$src2, VR256:$src3)>;
 188
 189 def : Pat<(int_x86_fma4_vfmadd_pd_256 VR256:$src1, VR256:$src2, VR256:$src3),
 190           (VFMADDPD4rrY VR256:$src1, VR256:$src2, VR256:$src3)>;
 191 def : Pat<(int_x86_fma4_vfmadd_pd_256 VR256:$src1, VR256:$src2,
 192                                   (alignedloadv4f64 addr:$src3)),
 193           (VFMADDPD4rmY VR256:$src1, VR256:$src2, addr:$src3)>;
 194 def : Pat<(int_x86_fma4_vfmadd_pd_256 VR256:$src1,
 195                                       (alignedloadv4f64 addr:$src2),
 196                                       VR256:$src3),
 197           (VFMADDPD4mrY VR256:$src1, addr:$src2, VR256:$src3)>;
 198
 199 // VFMSUB
 200 def : Pat<(int_x86_fma4_vfmsub_ss VR128:$src1, VR128:$src2, VR128:$src3),
 201           (VFMSUBSS4rr VR128:$src1, VR128:$src2, VR128:$src3)>;
 202 def : Pat<(int_x86_fma4_vfmsub_ss VR128:$src1, VR128:$src2,
 203                                   (alignedloadv4f32 addr:$src3)),
 204           (VFMSUBSS4rm VR128:$src1, VR128:$src2, addr:$src3)>;
 205 def : Pat<(int_x86_fma4_vfmsub_ss VR128:$src1, (alignedloadv4f32 addr:$src2),
 206                                   VR128:$src3),
 207           (VFMSUBSS4mr VR128:$src1, addr:$src2, VR128:$src3)>;
 208
 209 def : Pat<(int_x86_fma4_vfmsub_sd VR128:$src1, VR128:$src2, VR128:$src3),
 210           (VFMSUBSD4rr VR128:$src1, VR128:$src2, VR128:$src3)>;
 211 def : Pat<(int_x86_fma4_vfmsub_sd VR128:$src1, VR128:$src2,
 212                                   (alignedloadv2f64 addr:$src3)),
 213           (VFMSUBSD4rm VR128:$src1, VR128:$src2, addr:$src3)>;
 214 def : Pat<(int_x86_fma4_vfmsub_sd VR128:$src1, (alignedloadv2f64 addr:$src2),
 215                                   VR128:$src3),
 216           (VFMSUBSD4mr VR128:$src1, addr:$src2, VR128:$src3)>;
 217
 218 def : Pat<(int_x86_fma4_vfmsub_ps VR128:$src1, VR128:$src2, VR128:$src3),
 219           (VFMSUBPS4rr VR128:$src1, VR128:$src2, VR128:$src3)>;
 220 def : Pat<(int_x86_fma4_vfmsub_ps VR128:$src1, VR128:$src2,
 221                                   (alignedloadv4f32 addr:$src3)),
 222           (VFMSUBPS4rm VR128:$src1, VR128:$src2, addr:$src3)>;
 223 def : Pat<(int_x86_fma4_vfmsub_ps VR128:$src1, (alignedloadv4f32 addr:$src2),
 224                                   VR128:$src3),
 225           (VFMSUBPS4mr VR128:$src1, addr:$src2, VR128:$src3)>;
 226
 227 def : Pat<(int_x86_fma4_vfmsub_pd VR128:$src1, VR128:$src2, VR128:$src3),
 228           (VFMSUBPD4rr VR128:$src1, VR128:$src2, VR128:$src3)>;
 229 def : Pat<(int_x86_fma4_vfmsub_pd VR128:$src1, VR128:$src2,
 230                                   (alignedloadv2f64 addr:$src3)),
 231           (VFMSUBPD4rm VR128:$src1, VR128:$src2, addr:$src3)>;
 232 def : Pat<(int_x86_fma4_vfmsub_pd VR128:$src1, (alignedloadv2f64 addr:$src2),
 233                                   VR128:$src3),
 234           (VFMSUBPD4mr VR128:$src1, addr:$src2, VR128:$src3)>;
 235
 236 def : Pat<(int_x86_fma4_vfmsub_ps_256 VR256:$src1, VR256:$src2, VR256:$src3),
 237           (VFMSUBPS4rrY VR256:$src1, VR256:$src2, VR256:$src3)>;
 238 def : Pat<(int_x86_fma4_vfmsub_ps_256 VR256:$src1, VR256:$src2,
 239                                   (alignedloadv8f32 addr:$src3)),
 240           (VFMSUBPS4rmY VR256:$src1, VR256:$src2, addr:$src3)>;
 241 def : Pat<(int_x86_fma4_vfmsub_ps_256 VR256:$src1,
 242                                       (alignedloadv8f32 addr:$src2),
 243                                       VR256:$src3),
 244           (VFMSUBPS4mrY VR256:$src1, addr:$src2, VR256:$src3)>;
 245
 246 def : Pat<(int_x86_fma4_vfmsub_pd_256 VR256:$src1, VR256:$src2, VR256:$src3),
 247           (VFMSUBPD4rrY VR256:$src1, VR256:$src2, VR256:$src3)>;
 248 def : Pat<(int_x86_fma4_vfmsub_pd_256 VR256:$src1, VR256:$src2,
 249                                   (alignedloadv4f64 addr:$src3)),
 250           (VFMSUBPD4rmY VR256:$src1, VR256:$src2, addr:$src3)>;
 251 def : Pat<(int_x86_fma4_vfmsub_pd_256 VR256:$src1,
 252                                       (alignedloadv4f64 addr:$src2),
 253                                       VR256:$src3),
 254           (VFMSUBPD4mrY VR256:$src1, addr:$src2, VR256:$src3)>;
 255
 256 // VFNMADD
 257 def : Pat<(int_x86_fma4_vfnmadd_ss VR128:$src1, VR128:$src2, VR128:$src3),
 258           (VFNMADDSS4rr VR128:$src1, VR128:$src2, VR128:$src3)>;
 259 def : Pat<(int_x86_fma4_vfnmadd_ss VR128:$src1, VR128:$src2,
 260                                   (alignedloadv4f32 addr:$src3)),
 261           (VFNMADDSS4rm VR128:$src1, VR128:$src2, addr:$src3)>;
 262 def : Pat<(int_x86_fma4_vfnmadd_ss VR128:$src1, (alignedloadv4f32 addr:$src2),
 263                                   VR128:$src3),
 264           (VFNMADDSS4mr VR128:$src1, addr:$src2, VR128:$src3)>;
 265
 266 def : Pat<(int_x86_fma4_vfnmadd_sd VR128:$src1, VR128:$src2, VR128:$src3),
 267           (VFNMADDSD4rr VR128:$src1, VR128:$src2, VR128:$src3)>;
 268 def : Pat<(int_x86_fma4_vfnmadd_sd VR128:$src1, VR128:$src2,
 269                                   (alignedloadv2f64 addr:$src3)),
 270           (VFNMADDSD4rm VR128:$src1, VR128:$src2, addr:$src3)>;
 271 def : Pat<(int_x86_fma4_vfnmadd_sd VR128:$src1, (alignedloadv2f64 addr:$src2),
 272                                   VR128:$src3),
 273           (VFNMADDSD4mr VR128:$src1, addr:$src2, VR128:$src3)>;
 274
 275 def : Pat<(int_x86_fma4_vfnmadd_ps VR128:$src1, VR128:$src2, VR128:$src3),
 276           (VFNMADDPS4rr VR128:$src1, VR128:$src2, VR128:$src3)>;
 277 def : Pat<(int_x86_fma4_vfnmadd_ps VR128:$src1, VR128:$src2,
 278                                   (alignedloadv4f32 addr:$src3)),
 279           (VFNMADDPS4rm VR128:$src1, VR128:$src2, addr:$src3)>;
 280 def : Pat<(int_x86_fma4_vfnmadd_ps VR128:$src1, (alignedloadv4f32 addr:$src2),
 281                                   VR128:$src3),
 282           (VFNMADDPS4mr VR128:$src1, addr:$src2, VR128:$src3)>;
 283
 284 def : Pat<(int_x86_fma4_vfnmadd_pd VR128:$src1, VR128:$src2, VR128:$src3),
 285           (VFNMADDPD4rr VR128:$src1, VR128:$src2, VR128:$src3)>;
 286 def : Pat<(int_x86_fma4_vfnmadd_pd VR128:$src1, VR128:$src2,
 287                                   (alignedloadv2f64 addr:$src3)),
 288           (VFNMADDPD4rm VR128:$src1, VR128:$src2, addr:$src3)>;
 289 def : Pat<(int_x86_fma4_vfnmadd_pd VR128:$src1, (alignedloadv2f64 addr:$src2),
 290                                   VR128:$src3),
 291           (VFNMADDPD4mr VR128:$src1, addr:$src2, VR128:$src3)>;
 292
 293 def : Pat<(int_x86_fma4_vfnmadd_ps_256 VR256:$src1, VR256:$src2, VR256:$src3),
 294           (VFNMADDPS4rrY VR256:$src1, VR256:$src2, VR256:$src3)>;
 295 def : Pat<(int_x86_fma4_vfnmadd_ps_256 VR256:$src1, VR256:$src2,
 296                                   (alignedloadv8f32 addr:$src3)),
 297           (VFNMADDPS4rmY VR256:$src1, VR256:$src2, addr:$src3)>;
 298 def : Pat<(int_x86_fma4_vfnmadd_ps_256 VR256:$src1,
 299                                       (alignedloadv8f32 addr:$src2),
 300                                       VR256:$src3),
 301           (VFNMADDPS4mrY VR256:$src1, addr:$src2, VR256:$src3)>;
 302
 303 def : Pat<(int_x86_fma4_vfnmadd_pd_256 VR256:$src1, VR256:$src2, VR256:$src3),
 304           (VFNMADDPD4rrY VR256:$src1, VR256:$src2, VR256:$src3)>;
 305 def : Pat<(int_x86_fma4_vfnmadd_pd_256 VR256:$src1, VR256:$src2,
 306                                   (alignedloadv4f64 addr:$src3)),
 307           (VFNMADDPD4rmY VR256:$src1, VR256:$src2, addr:$src3)>;
 308 def : Pat<(int_x86_fma4_vfnmadd_pd_256 VR256:$src1,
 309                                       (alignedloadv4f64 addr:$src2),
 310                                       VR256:$src3),
 311           (VFNMADDPD4mrY VR256:$src1, addr:$src2, VR256:$src3)>;
 312
 313 // VFNMSUB
 314 def : Pat<(int_x86_fma4_vfnmsub_ss VR128:$src1, VR128:$src2, VR128:$src3),
 315           (VFNMSUBSS4rr VR128:$src1, VR128:$src2, VR128:$src3)>;
 316 def : Pat<(int_x86_fma4_vfnmsub_ss VR128:$src1, VR128:$src2,
 317                                   (alignedloadv4f32 addr:$src3)),
 318           (VFNMSUBSS4rm VR128:$src1, VR128:$src2, addr:$src3)>;
 319 def : Pat<(int_x86_fma4_vfnmsub_ss VR128:$src1, (alignedloadv4f32 addr:$src2),
 320                                   VR128:$src3),
 321           (VFNMSUBSS4mr VR128:$src1, addr:$src2, VR128:$src3)>;
 322
 323 def : Pat<(int_x86_fma4_vfnmsub_sd VR128:$src1, VR128:$src2, VR128:$src3),
 324           (VFNMSUBSD4rr VR128:$src1, VR128:$src2, VR128:$src3)>;
 325 def : Pat<(int_x86_fma4_vfnmsub_sd VR128:$src1, VR128:$src2,
 326                                   (alignedloadv2f64 addr:$src3)),
 327           (VFNMSUBSD4rm VR128:$src1, VR128:$src2, addr:$src3)>;
 328 def : Pat<(int_x86_fma4_vfnmsub_sd VR128:$src1, (alignedloadv2f64 addr:$src2),
 329                                   VR128:$src3),
 330           (VFNMSUBSD4mr VR128:$src1, addr:$src2, VR128:$src3)>;
 331
 332 def : Pat<(int_x86_fma4_vfnmsub_ps VR128:$src1, VR128:$src2, VR128:$src3),
 333           (VFNMSUBPS4rr VR128:$src1, VR128:$src2, VR128:$src3)>;
 334 def : Pat<(int_x86_fma4_vfnmsub_ps VR128:$src1, VR128:$src2,
 335                                   (alignedloadv4f32 addr:$src3)),
 336           (VFNMSUBPS4rm VR128:$src1, VR128:$src2, addr:$src3)>;
 337 def : Pat<(int_x86_fma4_vfnmsub_ps VR128:$src1, (alignedloadv4f32 addr:$src2),
 338                                   VR128:$src3),
 339           (VFNMSUBPS4mr VR128:$src1, addr:$src2, VR128:$src3)>;
 340
 341 def : Pat<(int_x86_fma4_vfnmsub_pd VR128:$src1, VR128:$src2, VR128:$src3),
 342           (VFNMSUBPD4rr VR128:$src1, VR128:$src2, VR128:$src3)>;
 343 def : Pat<(int_x86_fma4_vfnmsub_pd VR128:$src1, VR128:$src2,
 344                                   (alignedloadv2f64 addr:$src3)),
 345           (VFNMSUBPD4rm VR128:$src1, VR128:$src2, addr:$src3)>;
 346 def : Pat<(int_x86_fma4_vfnmsub_pd VR128:$src1, (alignedloadv2f64 addr:$src2),
 347                                   VR128:$src3),
 348           (VFNMSUBPD4mr VR128:$src1, addr:$src2, VR128:$src3)>;
 349
 350 def : Pat<(int_x86_fma4_vfnmsub_ps_256 VR256:$src1, VR256:$src2, VR256:$src3),
 351           (VFNMSUBPS4rrY VR256:$src1, VR256:$src2, VR256:$src3)>;
 352 def : Pat<(int_x86_fma4_vfnmsub_ps_256 VR256:$src1, VR256:$src2,
 353                                   (alignedloadv8f32 addr:$src3)),
 354           (VFNMSUBPS4rmY VR256:$src1, VR256:$src2, addr:$src3)>;
 355 def : Pat<(int_x86_fma4_vfnmsub_ps_256 VR256:$src1,
 356                                       (alignedloadv8f32 addr:$src2),
 357                                       VR256:$src3),
 358           (VFNMSUBPS4mrY VR256:$src1, addr:$src2, VR256:$src3)>;
 359
 360 def : Pat<(int_x86_fma4_vfnmsub_pd_256 VR256:$src1, VR256:$src2, VR256:$src3),
 361           (VFNMSUBPD4rrY VR256:$src1, VR256:$src2, VR256:$src3)>;
 362 def : Pat<(int_x86_fma4_vfnmsub_pd_256 VR256:$src1, VR256:$src2,
 363                                   (alignedloadv4f64 addr:$src3)),
 364           (VFNMSUBPD4rmY VR256:$src1, VR256:$src2, addr:$src3)>;
 365 def : Pat<(int_x86_fma4_vfnmsub_pd_256 VR256:$src1,
 366                                       (alignedloadv4f64 addr:$src2),
 367                                       VR256:$src3),
 368           (VFNMSUBPD4mrY VR256:$src1, addr:$src2, VR256:$src3)>;
 369
 370 // VFMADDSUB
 371 def : Pat<(int_x86_fma4_vfmaddsub_ps VR128:$src1, VR128:$src2, VR128:$src3),
 372           (VFMADDSUBPS4rr VR128:$src1, VR128:$src2, VR128:$src3)>;
 373 def : Pat<(int_x86_fma4_vfmaddsub_ps VR128:$src1, VR128:$src2,
 374                                   (alignedloadv4f32 addr:$src3)),
 375           (VFMADDSUBPS4rm VR128:$src1, VR128:$src2, addr:$src3)>;
 376 def : Pat<(int_x86_fma4_vfmaddsub_ps VR128:$src1, (alignedloadv4f32 addr:$src2),
 377                                   VR128:$src3),
 378           (VFMADDSUBPS4mr VR128:$src1, addr:$src2, VR128:$src3)>;
 379
 380 def : Pat<(int_x86_fma4_vfmaddsub_pd VR128:$src1, VR128:$src2, VR128:$src3),
 381           (VFMADDSUBPD4rr VR128:$src1, VR128:$src2, VR128:$src3)>;
 382 def : Pat<(int_x86_fma4_vfmaddsub_pd VR128:$src1, VR128:$src2,
 383                                   (alignedloadv2f64 addr:$src3)),
 384           (VFMADDSUBPD4rm VR128:$src1, VR128:$src2, addr:$src3)>;
 385 def : Pat<(int_x86_fma4_vfmaddsub_pd VR128:$src1, (alignedloadv2f64 addr:$src2),
 386                                   VR128:$src3),
 387           (VFMADDSUBPD4mr VR128:$src1, addr:$src2, VR128:$src3)>;
 388
 389 def : Pat<(int_x86_fma4_vfmaddsub_ps_256 VR256:$src1, VR256:$src2, VR256:$src3),
 390           (VFMADDSUBPS4rrY VR256:$src1, VR256:$src2, VR256:$src3)>;
 391 def : Pat<(int_x86_fma4_vfmaddsub_ps_256 VR256:$src1, VR256:$src2,
 392                                   (alignedloadv8f32 addr:$src3)),
 393           (VFMADDSUBPS4rmY VR256:$src1, VR256:$src2, addr:$src3)>;
 394 def : Pat<(int_x86_fma4_vfmaddsub_ps_256 VR256:$src1,
 395                                       (alignedloadv8f32 addr:$src2),
 396                                       VR256:$src3),
 397           (VFMADDSUBPS4mrY VR256:$src1, addr:$src2, VR256:$src3)>;
 398
 399 def : Pat<(int_x86_fma4_vfmaddsub_pd_256 VR256:$src1, VR256:$src2, VR256:$src3),
 400           (VFMADDSUBPD4rrY VR256:$src1, VR256:$src2, VR256:$src3)>;
 401 def : Pat<(int_x86_fma4_vfmaddsub_pd_256 VR256:$src1, VR256:$src2,
 402                                   (alignedloadv4f64 addr:$src3)),
 403           (VFMADDSUBPD4rmY VR256:$src1, VR256:$src2, addr:$src3)>;
 404 def : Pat<(int_x86_fma4_vfmaddsub_pd_256 VR256:$src1,
 405                                       (alignedloadv4f64 addr:$src2),
 406                                       VR256:$src3),
 407           (VFMADDSUBPD4mrY VR256:$src1, addr:$src2, VR256:$src3)>;
 408
 409 // VFMSUBADD
 410 def : Pat<(int_x86_fma4_vfmsubadd_ps VR128:$src1, VR128:$src2, VR128:$src3),
 411           (VFMSUBADDPS4rr VR128:$src1, VR128:$src2, VR128:$src3)>;
 412 def : Pat<(int_x86_fma4_vfmsubadd_ps VR128:$src1, VR128:$src2,
 413                                   (alignedloadv4f32 addr:$src3)),
 414           (VFMSUBADDPS4rm VR128:$src1, VR128:$src2, addr:$src3)>;
 415 def : Pat<(int_x86_fma4_vfmsubadd_ps VR128:$src1, (alignedloadv4f32 addr:$src2),
 416                                   VR128:$src3),
 417           (VFMSUBADDPS4mr VR128:$src1, addr:$src2, VR128:$src3)>;
 418
 419 def : Pat<(int_x86_fma4_vfmsubadd_pd VR128:$src1, VR128:$src2, VR128:$src3),
 420           (VFMSUBADDPD4rr VR128:$src1, VR128:$src2, VR128:$src3)>;
 421 def : Pat<(int_x86_fma4_vfmsubadd_pd VR128:$src1, VR128:$src2,
 422                                   (alignedloadv2f64 addr:$src3)),
 423           (VFMSUBADDPD4rm VR128:$src1, VR128:$src2, addr:$src3)>;
 424 def : Pat<(int_x86_fma4_vfmsubadd_pd VR128:$src1, (alignedloadv2f64 addr:$src2),
 425                                   VR128:$src3),
 426           (VFMSUBADDPD4mr VR128:$src1, addr:$src2, VR128:$src3)>;
 427
 428 def : Pat<(int_x86_fma4_vfmsubadd_ps_256 VR256:$src1, VR256:$src2, VR256:$src3),
 429           (VFMSUBADDPS4rrY VR256:$src1, VR256:$src2, VR256:$src3)>;
 430 def : Pat<(int_x86_fma4_vfmsubadd_ps_256 VR256:$src1, VR256:$src2,
 431                                   (alignedloadv8f32 addr:$src3)),
 432           (VFMSUBADDPS4rmY VR256:$src1, VR256:$src2, addr:$src3)>;
 433 def : Pat<(int_x86_fma4_vfmsubadd_ps_256 VR256:$src1,
 434                                       (alignedloadv8f32 addr:$src2),
 435                                       VR256:$src3),
 436           (VFMSUBADDPS4mrY VR256:$src1, addr:$src2, VR256:$src3)>;
 437
 438 def : Pat<(int_x86_fma4_vfmsubadd_pd_256 VR256:$src1, VR256:$src2, VR256:$src3),
 439           (VFMSUBADDPD4rrY VR256:$src1, VR256:$src2, VR256:$src3)>;
 440 def : Pat<(int_x86_fma4_vfmsubadd_pd_256 VR256:$src1, VR256:$src2,
 441                                   (alignedloadv4f64 addr:$src3)),
 442           (VFMSUBADDPD4rmY VR256:$src1, VR256:$src2, addr:$src3)>;
 443 def : Pat<(int_x86_fma4_vfmsubadd_pd_256 VR256:$src1,
 444                                       (alignedloadv4f64 addr:$src2),
 445                                       VR256:$src3),
 446           (VFMSUBADDPD4mrY VR256:$src1, addr:$src2, VR256:$src3)>;