lib/Target/AArch64/AArch64InstrNEON.td

   1 //===-- AArch64InstrNEON.td - NEON support for AArch64 -----*- tablegen -*-===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // This file describes the AArch64 NEON instruction set.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 //===----------------------------------------------------------------------===//
  15 // NEON-specific DAG Nodes.
  16 //===----------------------------------------------------------------------===//
  17 def Neon_bsl       : SDNode<"AArch64ISD::NEON_BSL", SDTypeProfile<1, 3,
  18                       [SDTCisVec<0>, SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>,
  19                       SDTCisSameAs<0, 3>]>>;
  20
  21 // (outs Result), (ins Imm, OpCmode)
  22 def SDT_Neon_movi : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVT<1, i32>]>;
  23
  24 def Neon_movi     : SDNode<"AArch64ISD::NEON_MOVIMM", SDT_Neon_movi>;
  25
  26 def Neon_mvni     : SDNode<"AArch64ISD::NEON_MVNIMM", SDT_Neon_movi>;
  27
  28 // (outs Result), (ins Imm)
  29 def Neon_fmovi : SDNode<"AArch64ISD::NEON_FMOVIMM", SDTypeProfile<1, 1,
  30                         [SDTCisVec<0>, SDTCisVT<1, i32>]>>;
  31
  32 // (outs Result), (ins LHS, RHS, CondCode)
  33 def Neon_cmp : SDNode<"AArch64ISD::NEON_CMP", SDTypeProfile<1, 3,
  34                  [SDTCisVec<0>,  SDTCisSameAs<1, 2>]>>;
  35
  36 // (outs Result), (ins LHS, 0/0.0 constant, CondCode)
  37 def Neon_cmpz : SDNode<"AArch64ISD::NEON_CMPZ", SDTypeProfile<1, 3,
  38                  [SDTCisVec<0>,  SDTCisVec<1>]>>;
  39
  40 // (outs Result), (ins LHS, RHS)
  41 def Neon_tst : SDNode<"AArch64ISD::NEON_TST", SDTypeProfile<1, 2,
  42                  [SDTCisVec<0>,  SDTCisSameAs<1, 2>]>>;
  43
  44 def SDTARMVSH : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisSameAs<0, 1>,
  45                                      SDTCisVT<2, i32>]>;
  46 def Neon_sqrshlImm   : SDNode<"AArch64ISD::NEON_QSHLs", SDTARMVSH>;
  47 def Neon_uqrshlImm   : SDNode<"AArch64ISD::NEON_QSHLu", SDTARMVSH>;
  48
  49 def SDTPERMUTE : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisSameAs<0, 1>,
  50                                SDTCisSameAs<0, 2>]>;
  51 def Neon_uzp1    : SDNode<"AArch64ISD::NEON_UZP1", SDTPERMUTE>;
  52 def Neon_uzp2    : SDNode<"AArch64ISD::NEON_UZP2", SDTPERMUTE>;
  53 def Neon_zip1    : SDNode<"AArch64ISD::NEON_ZIP1", SDTPERMUTE>;
  54 def Neon_zip2    : SDNode<"AArch64ISD::NEON_ZIP2", SDTPERMUTE>;
  55 def Neon_trn1    : SDNode<"AArch64ISD::NEON_TRN1", SDTPERMUTE>;
  56 def Neon_trn2    : SDNode<"AArch64ISD::NEON_TRN2", SDTPERMUTE>;
  57
  58 def SDTVSHUF : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisSameAs<0, 1>]>;
  59 def Neon_rev64    : SDNode<"AArch64ISD::NEON_REV64", SDTVSHUF>;
  60 def Neon_rev32    : SDNode<"AArch64ISD::NEON_REV32", SDTVSHUF>;
  61 def Neon_rev16    : SDNode<"AArch64ISD::NEON_REV16", SDTVSHUF>;
  62 def Neon_vdup : SDNode<"AArch64ISD::NEON_VDUP", SDTypeProfile<1, 1,
  63                        [SDTCisVec<0>]>>;
  64 def Neon_vduplane : SDNode<"AArch64ISD::NEON_VDUPLANE", SDTypeProfile<1, 2,
  65                            [SDTCisVec<0>, SDTCisVec<1>, SDTCisVT<2, i64>]>>;
  66 def Neon_vextract : SDNode<"AArch64ISD::NEON_VEXTRACT", SDTypeProfile<1, 3,
  67                            [SDTCisVec<0>,  SDTCisSameAs<0, 1>,
  68                            SDTCisSameAs<0, 2>, SDTCisVT<3, i64>]>>;
  69
  70 def SDT_assertext : SDTypeProfile<1, 1,
  71   [SDTCisInt<0>, SDTCisInt<1>, SDTCisSameAs<1, 0>]>;
  72 def assertsext : SDNode<"ISD::AssertSext", SDT_assertext>;
  73 def assertzext : SDNode<"ISD::AssertZext", SDT_assertext>;
  74
  75 //===----------------------------------------------------------------------===//
  76 // Multiclasses
  77 //===----------------------------------------------------------------------===//
  78
  79 multiclass NeonI_3VSame_B_sizes<bit u, bits<2> size,  bits<5> opcode,
  80                                 string asmop, SDPatternOperator opnode8B,
  81                                 SDPatternOperator opnode16B,
  82                                 bit Commutable = 0> {
  83   let isCommutable = Commutable in {
  84     def _8B :  NeonI_3VSame<0b0, u, size, opcode,
  85                (outs VPR64:$Rd), (ins VPR64:$Rn, VPR64:$Rm),
  86                asmop # "\t$Rd.8b, $Rn.8b, $Rm.8b",
  87                [(set (v8i8 VPR64:$Rd),
  88                   (v8i8 (opnode8B (v8i8 VPR64:$Rn), (v8i8 VPR64:$Rm))))],
  89                NoItinerary>;
  90
  91     def _16B : NeonI_3VSame<0b1, u, size, opcode,
  92                (outs VPR128:$Rd), (ins VPR128:$Rn, VPR128:$Rm),
  93                asmop # "\t$Rd.16b, $Rn.16b, $Rm.16b",
  94                [(set (v16i8 VPR128:$Rd),
  95                   (v16i8 (opnode16B (v16i8 VPR128:$Rn), (v16i8 VPR128:$Rm))))],
  96                NoItinerary>;
  97   }
  98
  99 }
 100
 101 multiclass NeonI_3VSame_HS_sizes<bit u, bits<5> opcode,
 102                                   string asmop, SDPatternOperator opnode,
 103                                   bit Commutable = 0> {
 104   let isCommutable = Commutable in {
 105     def _4H : NeonI_3VSame<0b0, u, 0b01, opcode,
 106               (outs VPR64:$Rd), (ins VPR64:$Rn, VPR64:$Rm),
 107               asmop # "\t$Rd.4h, $Rn.4h, $Rm.4h",
 108               [(set (v4i16 VPR64:$Rd),
 109                  (v4i16 (opnode (v4i16 VPR64:$Rn), (v4i16 VPR64:$Rm))))],
 110               NoItinerary>;
 111
 112     def _8H : NeonI_3VSame<0b1, u, 0b01, opcode,
 113               (outs VPR128:$Rd), (ins VPR128:$Rn, VPR128:$Rm),
 114               asmop # "\t$Rd.8h, $Rn.8h, $Rm.8h",
 115               [(set (v8i16 VPR128:$Rd),
 116                  (v8i16 (opnode (v8i16 VPR128:$Rn), (v8i16 VPR128:$Rm))))],
 117               NoItinerary>;
 118
 119     def _2S : NeonI_3VSame<0b0, u, 0b10, opcode,
 120               (outs VPR64:$Rd), (ins VPR64:$Rn, VPR64:$Rm),
 121               asmop # "\t$Rd.2s, $Rn.2s, $Rm.2s",
 122               [(set (v2i32 VPR64:$Rd),
 123                  (v2i32 (opnode (v2i32 VPR64:$Rn), (v2i32 VPR64:$Rm))))],
 124               NoItinerary>;
 125
 126     def _4S : NeonI_3VSame<0b1, u, 0b10, opcode,
 127               (outs VPR128:$Rd), (ins VPR128:$Rn, VPR128:$Rm),
 128               asmop # "\t$Rd.4s, $Rn.4s, $Rm.4s",
 129               [(set (v4i32 VPR128:$Rd),
 130                  (v4i32 (opnode (v4i32 VPR128:$Rn), (v4i32 VPR128:$Rm))))],
 131               NoItinerary>;
 132   }
 133 }
 134 multiclass NeonI_3VSame_BHS_sizes<bit u, bits<5> opcode,
 135                                   string asmop, SDPatternOperator opnode,
 136                                   bit Commutable = 0>
 137    : NeonI_3VSame_HS_sizes<u, opcode,  asmop, opnode, Commutable> {
 138   let isCommutable = Commutable in {
 139     def _8B :  NeonI_3VSame<0b0, u, 0b00, opcode,
 140                (outs VPR64:$Rd), (ins VPR64:$Rn, VPR64:$Rm),
 141                asmop # "\t$Rd.8b, $Rn.8b, $Rm.8b",
 142                [(set (v8i8 VPR64:$Rd),
 143                   (v8i8 (opnode (v8i8 VPR64:$Rn), (v8i8 VPR64:$Rm))))],
 144                NoItinerary>;
 145
 146     def _16B : NeonI_3VSame<0b1, u, 0b00, opcode,
 147                (outs VPR128:$Rd), (ins VPR128:$Rn, VPR128:$Rm),
 148                asmop # "\t$Rd.16b, $Rn.16b, $Rm.16b",
 149                [(set (v16i8 VPR128:$Rd),
 150                   (v16i8 (opnode (v16i8 VPR128:$Rn), (v16i8 VPR128:$Rm))))],
 151                NoItinerary>;
 152   }
 153 }
 154
 155 multiclass NeonI_3VSame_BHSD_sizes<bit u, bits<5> opcode,
 156                                    string asmop, SDPatternOperator opnode,
 157                                    bit Commutable = 0>
 158    : NeonI_3VSame_BHS_sizes<u, opcode,  asmop, opnode, Commutable> {
 159   let isCommutable = Commutable in {
 160     def _2D : NeonI_3VSame<0b1, u, 0b11, opcode,
 161               (outs VPR128:$Rd), (ins VPR128:$Rn, VPR128:$Rm),
 162               asmop # "\t$Rd.2d, $Rn.2d, $Rm.2d",
 163               [(set (v2i64 VPR128:$Rd),
 164                  (v2i64 (opnode (v2i64 VPR128:$Rn), (v2i64 VPR128:$Rm))))],
 165               NoItinerary>;
 166   }
 167 }
 168
 169 // Multiclass NeonI_3VSame_SD_sizes: Operand types are floating point types,
 170 // but Result types can be integer or floating point types.
 171 multiclass NeonI_3VSame_SD_sizes<bit u, bit size, bits<5> opcode,
 172                                  string asmop, SDPatternOperator opnode2S,
 173                                  SDPatternOperator opnode4S,
 174                                  SDPatternOperator opnode2D,
 175                                  ValueType ResTy2S, ValueType ResTy4S,
 176                                  ValueType ResTy2D, bit Commutable = 0> {
 177   let isCommutable = Commutable in {
 178     def _2S : NeonI_3VSame<0b0, u, {size, 0b0}, opcode,
 179               (outs VPR64:$Rd), (ins VPR64:$Rn, VPR64:$Rm),
 180               asmop # "\t$Rd.2s, $Rn.2s, $Rm.2s",
 181               [(set (ResTy2S VPR64:$Rd),
 182                  (ResTy2S (opnode2S (v2f32 VPR64:$Rn), (v2f32 VPR64:$Rm))))],
 183               NoItinerary>;
 184
 185     def _4S : NeonI_3VSame<0b1, u, {size, 0b0}, opcode,
 186               (outs VPR128:$Rd), (ins VPR128:$Rn, VPR128:$Rm),
 187               asmop # "\t$Rd.4s, $Rn.4s, $Rm.4s",
 188               [(set (ResTy4S VPR128:$Rd),
 189                  (ResTy4S (opnode4S (v4f32 VPR128:$Rn), (v4f32 VPR128:$Rm))))],
 190               NoItinerary>;
 191
 192     def _2D : NeonI_3VSame<0b1, u, {size, 0b1}, opcode,
 193               (outs VPR128:$Rd), (ins VPR128:$Rn, VPR128:$Rm),
 194               asmop # "\t$Rd.2d, $Rn.2d, $Rm.2d",
 195               [(set (ResTy2D VPR128:$Rd),
 196                  (ResTy2D (opnode2D (v2f64 VPR128:$Rn), (v2f64 VPR128:$Rm))))],
 197                NoItinerary>;
 198   }
 199 }
 200
 201 //===----------------------------------------------------------------------===//
 202 // Instruction Definitions
 203 //===----------------------------------------------------------------------===//
 204
 205 // Vector Arithmetic Instructions
 206
 207 // Vector Add (Integer and Floating-Point)
 208
 209 defm ADDvvv :  NeonI_3VSame_BHSD_sizes<0b0, 0b10000, "add", add, 1>;
 210 defm FADDvvv : NeonI_3VSame_SD_sizes<0b0, 0b0, 0b11010, "fadd", fadd, fadd, fadd,
 211                                      v2f32, v4f32, v2f64, 1>;
 212
 213 // Vector Sub (Integer and Floating-Point)
 214
 215 defm SUBvvv :  NeonI_3VSame_BHSD_sizes<0b1, 0b10000, "sub", sub, 0>;
 216 defm FSUBvvv : NeonI_3VSame_SD_sizes<0b0, 0b1, 0b11010, "fsub", fsub, fsub, fsub,
 217                                      v2f32, v4f32, v2f64, 0>;
 218
 219 // Vector Multiply (Integer and Floating-Point)
 220
 221 defm MULvvv :  NeonI_3VSame_BHS_sizes<0b0, 0b10011, "mul", mul, 1>;
 222 defm FMULvvv : NeonI_3VSame_SD_sizes<0b1, 0b0, 0b11011, "fmul", fmul, fmul, fmul,
 223                                      v2f32, v4f32, v2f64, 1>;
 224
 225 // Vector Multiply (Polynomial)
 226
 227 defm PMULvvv : NeonI_3VSame_B_sizes<0b1, 0b00, 0b10011, "pmul",
 228                                     int_arm_neon_vmulp, int_arm_neon_vmulp, 1>;
 229
 230 // Vector Multiply-accumulate and Multiply-subtract (Integer)
 231
 232 // class NeonI_3VSame_Constraint_impl: NeonI_3VSame with no data type and
 233 // two operands constraints.
 234 class NeonI_3VSame_Constraint_impl<string asmop, string asmlane,
 235   RegisterOperand VPRC, ValueType OpTy, bit q, bit u, bits<2> size,
 236   bits<5> opcode, SDPatternOperator opnode>
 237   : NeonI_3VSame<q, u, size, opcode,
 238     (outs VPRC:$Rd), (ins VPRC:$src, VPRC:$Rn, VPRC:$Rm),
 239     asmop # "\t$Rd" # asmlane # ", $Rn" # asmlane # ", $Rm" # asmlane,
 240     [(set (OpTy VPRC:$Rd),
 241        (OpTy (opnode (OpTy VPRC:$src), (OpTy VPRC:$Rn), (OpTy VPRC:$Rm))))],
 242     NoItinerary> {
 243   let Constraints = "$src = $Rd";
 244 }
 245
 246 def Neon_mla : PatFrag<(ops node:$Ra, node:$Rn, node:$Rm),
 247                        (add node:$Ra, (mul node:$Rn, node:$Rm))>;
 248
 249 def Neon_mls : PatFrag<(ops node:$Ra, node:$Rn, node:$Rm),
 250                        (sub node:$Ra, (mul node:$Rn, node:$Rm))>;
 251
 252
 253 def MLAvvv_8B:  NeonI_3VSame_Constraint_impl<"mla", ".8b",  VPR64,  v8i8,
 254                                              0b0, 0b0, 0b00, 0b10010, Neon_mla>;
 255 def MLAvvv_16B: NeonI_3VSame_Constraint_impl<"mla", ".16b", VPR128, v16i8,
 256                                              0b1, 0b0, 0b00, 0b10010, Neon_mla>;
 257 def MLAvvv_4H:  NeonI_3VSame_Constraint_impl<"mla", ".4h",  VPR64,  v4i16,
 258                                              0b0, 0b0, 0b01, 0b10010, Neon_mla>;
 259 def MLAvvv_8H:  NeonI_3VSame_Constraint_impl<"mla", ".8h",  VPR128, v8i16,
 260                                              0b1, 0b0, 0b01, 0b10010, Neon_mla>;
 261 def MLAvvv_2S:  NeonI_3VSame_Constraint_impl<"mla", ".2s",  VPR64,  v2i32,
 262                                              0b0, 0b0, 0b10, 0b10010, Neon_mla>;
 263 def MLAvvv_4S:  NeonI_3VSame_Constraint_impl<"mla", ".4s",  VPR128, v4i32,
 264                                              0b1, 0b0, 0b10, 0b10010, Neon_mla>;
 265
 266 def MLSvvv_8B:  NeonI_3VSame_Constraint_impl<"mls", ".8b",  VPR64,  v8i8,
 267                                              0b0, 0b1, 0b00, 0b10010, Neon_mls>;
 268 def MLSvvv_16B: NeonI_3VSame_Constraint_impl<"mls", ".16b", VPR128, v16i8,
 269                                              0b1, 0b1, 0b00, 0b10010, Neon_mls>;
 270 def MLSvvv_4H:  NeonI_3VSame_Constraint_impl<"mls", ".4h",  VPR64,  v4i16,
 271                                              0b0, 0b1, 0b01, 0b10010, Neon_mls>;
 272 def MLSvvv_8H:  NeonI_3VSame_Constraint_impl<"mls", ".8h",  VPR128, v8i16,
 273                                              0b1, 0b1, 0b01, 0b10010, Neon_mls>;
 274 def MLSvvv_2S:  NeonI_3VSame_Constraint_impl<"mls", ".2s",  VPR64,  v2i32,
 275                                              0b0, 0b1, 0b10, 0b10010, Neon_mls>;
 276 def MLSvvv_4S:  NeonI_3VSame_Constraint_impl<"mls", ".4s",  VPR128, v4i32,
 277                                              0b1, 0b1, 0b10, 0b10010, Neon_mls>;
 278
 279 // Vector Multiply-accumulate and Multiply-subtract (Floating Point)
 280
 281 def Neon_fmla : PatFrag<(ops node:$Ra, node:$Rn, node:$Rm),
 282                         (fadd node:$Ra, (fmul node:$Rn, node:$Rm))>;
 283
 284 def Neon_fmls : PatFrag<(ops node:$Ra, node:$Rn, node:$Rm),
 285                         (fsub node:$Ra, (fmul node:$Rn, node:$Rm))>;
 286
 287 let Predicates = [HasNEON, UseFusedMAC] in {
 288 def FMLAvvv_2S: NeonI_3VSame_Constraint_impl<"fmla", ".2s",  VPR64,  v2f32,
 289                                              0b0, 0b0, 0b00, 0b11001, Neon_fmla>;
 290 def FMLAvvv_4S: NeonI_3VSame_Constraint_impl<"fmla", ".4s",  VPR128, v4f32,
 291                                              0b1, 0b0, 0b00, 0b11001, Neon_fmla>;
 292 def FMLAvvv_2D: NeonI_3VSame_Constraint_impl<"fmla", ".2d",  VPR128, v2f64,
 293                                              0b1, 0b0, 0b01, 0b11001, Neon_fmla>;
 294
 295 def FMLSvvv_2S: NeonI_3VSame_Constraint_impl<"fmls", ".2s",  VPR64,  v2f32,
 296                                               0b0, 0b0, 0b10, 0b11001, Neon_fmls>;
 297 def FMLSvvv_4S: NeonI_3VSame_Constraint_impl<"fmls", ".4s",  VPR128, v4f32,
 298                                              0b1, 0b0, 0b10, 0b11001, Neon_fmls>;
 299 def FMLSvvv_2D: NeonI_3VSame_Constraint_impl<"fmls", ".2d",  VPR128, v2f64,
 300                                              0b1, 0b0, 0b11, 0b11001, Neon_fmls>;
 301 }
 302
 303 // We're also allowed to match the fma instruction regardless of compile
 304 // options.
 305 def : Pat<(v2f32 (fma VPR64:$Rn, VPR64:$Rm, VPR64:$Ra)),
 306           (FMLAvvv_2S VPR64:$Ra, VPR64:$Rn, VPR64:$Rm)>;
 307 def : Pat<(v4f32 (fma VPR128:$Rn, VPR128:$Rm, VPR128:$Ra)),
 308           (FMLAvvv_4S VPR128:$Ra, VPR128:$Rn, VPR128:$Rm)>;
 309 def : Pat<(v2f64 (fma VPR128:$Rn, VPR128:$Rm, VPR128:$Ra)),
 310           (FMLAvvv_2D VPR128:$Ra, VPR128:$Rn, VPR128:$Rm)>;
 311
 312 def : Pat<(v2f32 (fma (fneg VPR64:$Rn), VPR64:$Rm, VPR64:$Ra)),
 313           (FMLSvvv_2S VPR64:$Ra, VPR64:$Rn, VPR64:$Rm)>;
 314 def : Pat<(v4f32 (fma (fneg VPR128:$Rn), VPR128:$Rm, VPR128:$Ra)),
 315           (FMLSvvv_4S VPR128:$Ra, VPR128:$Rn, VPR128:$Rm)>;
 316 def : Pat<(v2f64 (fma (fneg VPR128:$Rn), VPR128:$Rm, VPR128:$Ra)),
 317           (FMLSvvv_2D VPR128:$Ra, VPR128:$Rn, VPR128:$Rm)>;
 318
 319 // Vector Divide (Floating-Point)
 320
 321 defm FDIVvvv : NeonI_3VSame_SD_sizes<0b1, 0b0, 0b11111, "fdiv", fdiv, fdiv, fdiv,
 322                                      v2f32, v4f32, v2f64, 0>;
 323
 324 // Vector Bitwise Operations
 325
 326 // Vector Bitwise AND
 327
 328 defm ANDvvv : NeonI_3VSame_B_sizes<0b0, 0b00, 0b00011, "and", and, and, 1>;
 329
 330 // Vector Bitwise Exclusive OR
 331
 332 defm EORvvv : NeonI_3VSame_B_sizes<0b1, 0b00, 0b00011, "eor", xor, xor, 1>;
 333
 334 // Vector Bitwise OR
 335
 336 defm ORRvvv : NeonI_3VSame_B_sizes<0b0, 0b10, 0b00011, "orr", or, or, 1>;
 337
 338 // ORR disassembled as MOV if Vn==Vm
 339
 340 // Vector Move - register
 341 // Alias for ORR if Vn=Vm.
 342 // FIXME: This is actually the preferred syntax but TableGen can't deal with
 343 // custom printing of aliases.
 344 def : NeonInstAlias<"mov $Rd.8b, $Rn.8b",
 345                     (ORRvvv_8B VPR64:$Rd, VPR64:$Rn, VPR64:$Rn), 0>;
 346 def : NeonInstAlias<"mov $Rd.16b, $Rn.16b",
 347                     (ORRvvv_16B VPR128:$Rd, VPR128:$Rn, VPR128:$Rn), 0>;
 348
 349 // The MOVI instruction takes two immediate operands.  The first is the
 350 // immediate encoding, while the second is the cmode.  A cmode of 14, or
 351 // 0b1110, produces a MOVI operation, rather than a MVNI, ORR, or BIC.
 352 def Neon_AllZero : PatFrag<(ops), (Neon_movi (i32 0), (i32 14))>;
 353 def Neon_AllOne : PatFrag<(ops), (Neon_movi (i32 255), (i32 14))>;
 354
 355 def Neon_not8B  : PatFrag<(ops node:$in),
 356                           (xor node:$in, (bitconvert (v8i8 Neon_AllOne)))>;
 357 def Neon_not16B : PatFrag<(ops node:$in),
 358                           (xor node:$in, (bitconvert (v16i8 Neon_AllOne)))>;
 359
 360 def Neon_orn8B : PatFrag<(ops node:$Rn, node:$Rm),
 361                          (or node:$Rn, (Neon_not8B node:$Rm))>;
 362
 363 def Neon_orn16B : PatFrag<(ops node:$Rn, node:$Rm),
 364                           (or node:$Rn, (Neon_not16B node:$Rm))>;
 365
 366 def Neon_bic8B : PatFrag<(ops node:$Rn, node:$Rm),
 367                          (and node:$Rn, (Neon_not8B node:$Rm))>;
 368
 369 def Neon_bic16B : PatFrag<(ops node:$Rn, node:$Rm),
 370                           (and node:$Rn, (Neon_not16B node:$Rm))>;
 371
 372
 373 // Vector Bitwise OR NOT - register
 374
 375 defm ORNvvv : NeonI_3VSame_B_sizes<0b0, 0b11, 0b00011, "orn",
 376                                    Neon_orn8B, Neon_orn16B, 0>;
 377
 378 // Vector Bitwise Bit Clear (AND NOT) - register
 379
 380 defm BICvvv : NeonI_3VSame_B_sizes<0b0, 0b01, 0b00011, "bic",
 381                                    Neon_bic8B, Neon_bic16B, 0>;
 382
 383 multiclass Neon_bitwise2V_patterns<SDPatternOperator opnode8B,
 384                                    SDPatternOperator opnode16B,
 385                                    Instruction INST8B,
 386                                    Instruction INST16B> {
 387   def : Pat<(v2i32 (opnode8B VPR64:$Rn, VPR64:$Rm)),
 388             (INST8B VPR64:$Rn, VPR64:$Rm)>;
 389   def : Pat<(v4i16 (opnode8B VPR64:$Rn, VPR64:$Rm)),
 390             (INST8B VPR64:$Rn, VPR64:$Rm)>;
 391   def : Pat<(v1i64 (opnode8B VPR64:$Rn, VPR64:$Rm)),
 392             (INST8B VPR64:$Rn, VPR64:$Rm)>;
 393   def : Pat<(v4i32 (opnode16B VPR128:$Rn, VPR128:$Rm)),
 394             (INST16B VPR128:$Rn, VPR128:$Rm)>;
 395   def : Pat<(v8i16 (opnode16B VPR128:$Rn, VPR128:$Rm)),
 396             (INST16B VPR128:$Rn, VPR128:$Rm)>;
 397   def : Pat<(v2i64 (opnode16B VPR128:$Rn, VPR128:$Rm)),
 398             (INST16B VPR128:$Rn, VPR128:$Rm)>;
 399 }
 400
 401 // Additional patterns for bitwise instructions AND, EOR, ORR, BIC, ORN
 402 defm : Neon_bitwise2V_patterns<and, and, ANDvvv_8B, ANDvvv_16B>;
 403 defm : Neon_bitwise2V_patterns<or,  or,  ORRvvv_8B, ORRvvv_16B>;
 404 defm : Neon_bitwise2V_patterns<xor, xor, EORvvv_8B, EORvvv_16B>;
 405 defm : Neon_bitwise2V_patterns<Neon_bic8B, Neon_bic16B, BICvvv_8B, BICvvv_16B>;
 406 defm : Neon_bitwise2V_patterns<Neon_orn8B, Neon_orn16B, ORNvvv_8B, ORNvvv_16B>;
 407
 408 //   Vector Bitwise Select
 409 def BSLvvv_8B  : NeonI_3VSame_Constraint_impl<"bsl", ".8b",  VPR64, v8i8,
 410                                               0b0, 0b1, 0b01, 0b00011, Neon_bsl>;
 411
 412 def BSLvvv_16B : NeonI_3VSame_Constraint_impl<"bsl", ".16b", VPR128, v16i8,
 413                                               0b1, 0b1, 0b01, 0b00011, Neon_bsl>;
 414
 415 multiclass Neon_bitwise3V_patterns<SDPatternOperator opnode,
 416                                    Instruction INST8B,
 417                                    Instruction INST16B> {
 418   // Disassociate type from instruction definition
 419   def : Pat<(v2i32 (opnode VPR64:$src,VPR64:$Rn, VPR64:$Rm)),
 420             (INST8B VPR64:$src, VPR64:$Rn, VPR64:$Rm)>;
 421   def : Pat<(v4i16 (opnode VPR64:$src, VPR64:$Rn, VPR64:$Rm)),
 422             (INST8B VPR64:$src, VPR64:$Rn, VPR64:$Rm)>;
 423   def : Pat<(v1i64 (opnode VPR64:$src, VPR64:$Rn, VPR64:$Rm)),
 424             (INST8B VPR64:$src, VPR64:$Rn, VPR64:$Rm)>;
 425   def : Pat<(v4i32 (opnode VPR128:$src, VPR128:$Rn, VPR128:$Rm)),
 426             (INST16B VPR128:$src, VPR128:$Rn, VPR128:$Rm)>;
 427   def : Pat<(v8i16 (opnode VPR128:$src, VPR128:$Rn, VPR128:$Rm)),
 428             (INST16B VPR128:$src, VPR128:$Rn, VPR128:$Rm)>;
 429   def : Pat<(v2i64 (opnode VPR128:$src, VPR128:$Rn, VPR128:$Rm)),
 430             (INST16B VPR128:$src, VPR128:$Rn, VPR128:$Rm)>;
 431
 432   // Allow to match BSL instruction pattern with non-constant operand
 433   def : Pat<(v8i8 (or (and VPR64:$Rn, VPR64:$Rd),
 434                     (and VPR64:$Rm, (Neon_not8B VPR64:$Rd)))),
 435           (INST8B VPR64:$Rd, VPR64:$Rn, VPR64:$Rm)>;
 436   def : Pat<(v4i16 (or (and VPR64:$Rn, VPR64:$Rd),
 437                      (and VPR64:$Rm, (Neon_not8B VPR64:$Rd)))),
 438           (INST8B VPR64:$Rd, VPR64:$Rn, VPR64:$Rm)>;
 439   def : Pat<(v2i32 (or (and VPR64:$Rn, VPR64:$Rd),
 440                      (and VPR64:$Rm, (Neon_not8B VPR64:$Rd)))),
 441           (INST8B VPR64:$Rd, VPR64:$Rn, VPR64:$Rm)>;
 442   def : Pat<(v1i64 (or (and VPR64:$Rn, VPR64:$Rd),
 443                      (and VPR64:$Rm, (Neon_not8B VPR64:$Rd)))),
 444           (INST8B VPR64:$Rd, VPR64:$Rn, VPR64:$Rm)>;
 445   def : Pat<(v16i8 (or (and VPR128:$Rn, VPR128:$Rd),
 446                      (and VPR128:$Rm, (Neon_not16B VPR128:$Rd)))),
 447           (INST16B VPR128:$Rd, VPR128:$Rn, VPR128:$Rm)>;
 448   def : Pat<(v8i16 (or (and VPR128:$Rn, VPR128:$Rd),
 449                      (and VPR128:$Rm, (Neon_not16B VPR128:$Rd)))),
 450           (INST16B VPR128:$Rd, VPR128:$Rn, VPR128:$Rm)>;
 451   def : Pat<(v4i32 (or (and VPR128:$Rn, VPR128:$Rd),
 452                      (and VPR128:$Rm, (Neon_not16B VPR128:$Rd)))),
 453           (INST16B VPR128:$Rd, VPR128:$Rn, VPR128:$Rm)>;
 454   def : Pat<(v2i64 (or (and VPR128:$Rn, VPR128:$Rd),
 455                      (and VPR128:$Rm, (Neon_not16B VPR128:$Rd)))),
 456           (INST16B VPR128:$Rd, VPR128:$Rn, VPR128:$Rm)>;
 457
 458   // Allow to match llvm.arm.* intrinsics.
 459   def : Pat<(v8i8 (int_arm_neon_vbsl (v8i8 VPR64:$src),
 460                     (v8i8 VPR64:$Rn), (v8i8 VPR64:$Rm))),
 461             (INST8B VPR64:$src, VPR64:$Rn, VPR64:$Rm)>;
 462   def : Pat<(v4i16 (int_arm_neon_vbsl (v4i16 VPR64:$src),
 463                     (v4i16 VPR64:$Rn), (v4i16 VPR64:$Rm))),
 464             (INST8B VPR64:$src, VPR64:$Rn, VPR64:$Rm)>;
 465   def : Pat<(v2i32 (int_arm_neon_vbsl (v2i32 VPR64:$src),
 466                     (v2i32 VPR64:$Rn), (v2i32 VPR64:$Rm))),
 467             (INST8B VPR64:$src, VPR64:$Rn, VPR64:$Rm)>;
 468   def : Pat<(v1i64 (int_arm_neon_vbsl (v1i64 VPR64:$src),
 469                     (v1i64 VPR64:$Rn), (v1i64 VPR64:$Rm))),
 470             (INST8B VPR64:$src, VPR64:$Rn, VPR64:$Rm)>;
 471   def : Pat<(v2f32 (int_arm_neon_vbsl (v2f32 VPR64:$src),
 472                     (v2f32 VPR64:$Rn), (v2f32 VPR64:$Rm))),
 473             (INST8B VPR64:$src, VPR64:$Rn, VPR64:$Rm)>;
 474   def : Pat<(v1f64 (int_arm_neon_vbsl (v1f64 VPR64:$src),
 475                     (v1f64 VPR64:$Rn), (v1f64 VPR64:$Rm))),
 476             (INST8B VPR64:$src, VPR64:$Rn, VPR64:$Rm)>;
 477   def : Pat<(v16i8 (int_arm_neon_vbsl (v16i8 VPR128:$src),
 478                     (v16i8 VPR128:$Rn), (v16i8 VPR128:$Rm))),
 479             (INST16B VPR128:$src, VPR128:$Rn, VPR128:$Rm)>;
 480   def : Pat<(v8i16 (int_arm_neon_vbsl (v8i16 VPR128:$src),
 481                     (v8i16 VPR128:$Rn), (v8i16 VPR128:$Rm))),
 482             (INST16B VPR128:$src, VPR128:$Rn, VPR128:$Rm)>;
 483   def : Pat<(v4i32 (int_arm_neon_vbsl (v4i32 VPR128:$src),
 484                     (v4i32 VPR128:$Rn), (v4i32 VPR128:$Rm))),
 485             (INST16B VPR128:$src, VPR128:$Rn, VPR128:$Rm)>;
 486   def : Pat<(v2i64 (int_arm_neon_vbsl (v2i64 VPR128:$src),
 487                     (v2i64 VPR128:$Rn), (v2i64 VPR128:$Rm))),
 488             (INST16B VPR128:$src, VPR128:$Rn, VPR128:$Rm)>;
 489   def : Pat<(v4f32 (int_arm_neon_vbsl (v4f32 VPR128:$src),
 490                     (v4f32 VPR128:$Rn), (v4f32 VPR128:$Rm))),
 491             (INST16B VPR128:$src, VPR128:$Rn, VPR128:$Rm)>;
 492   def : Pat<(v2f64 (int_arm_neon_vbsl (v2f64 VPR128:$src),
 493                     (v2f64 VPR128:$Rn), (v2f64 VPR128:$Rm))),
 494             (INST16B VPR128:$src, VPR128:$Rn, VPR128:$Rm)>;
 495 }
 496
 497 // Additional patterns for bitwise instruction BSL
 498 defm: Neon_bitwise3V_patterns<Neon_bsl, BSLvvv_8B, BSLvvv_16B>;
 499
 500 def Neon_NoBSLop : PatFrag<(ops node:$src, node:$Rn, node:$Rm),
 501                            (Neon_bsl node:$src, node:$Rn, node:$Rm),
 502                            [{ (void)N; return false; }]>;
 503
 504 // Vector Bitwise Insert if True
 505
 506 def BITvvv_8B  : NeonI_3VSame_Constraint_impl<"bit", ".8b", VPR64,   v8i8,
 507                    0b0, 0b1, 0b10, 0b00011, Neon_NoBSLop>;
 508 def BITvvv_16B : NeonI_3VSame_Constraint_impl<"bit", ".16b", VPR128, v16i8,
 509                    0b1, 0b1, 0b10, 0b00011, Neon_NoBSLop>;
 510
 511 // Vector Bitwise Insert if False
 512
 513 def BIFvvv_8B  : NeonI_3VSame_Constraint_impl<"bif", ".8b", VPR64,  v8i8,
 514                                 0b0, 0b1, 0b11, 0b00011, Neon_NoBSLop>;
 515 def BIFvvv_16B : NeonI_3VSame_Constraint_impl<"bif", ".16b", VPR128, v16i8,
 516                                 0b1, 0b1, 0b11, 0b00011, Neon_NoBSLop>;
 517
 518 // Vector Absolute Difference and Accumulate (Signed, Unsigned)
 519
 520 def Neon_uaba : PatFrag<(ops node:$Ra, node:$Rn, node:$Rm),
 521                        (add node:$Ra, (int_arm_neon_vabdu node:$Rn, node:$Rm))>;
 522 def Neon_saba : PatFrag<(ops node:$Ra, node:$Rn, node:$Rm),
 523                        (add node:$Ra, (int_arm_neon_vabds node:$Rn, node:$Rm))>;
 524
 525 // Vector Absolute Difference and Accumulate (Unsigned)
 526 def UABAvvv_8B :  NeonI_3VSame_Constraint_impl<"uaba", ".8b",  VPR64,  v8i8,
 527                     0b0, 0b1, 0b00, 0b01111, Neon_uaba>;
 528 def UABAvvv_16B : NeonI_3VSame_Constraint_impl<"uaba", ".16b", VPR128, v16i8,
 529                     0b1, 0b1, 0b00, 0b01111, Neon_uaba>;
 530 def UABAvvv_4H :  NeonI_3VSame_Constraint_impl<"uaba", ".4h",  VPR64,  v4i16,
 531                     0b0, 0b1, 0b01, 0b01111, Neon_uaba>;
 532 def UABAvvv_8H :  NeonI_3VSame_Constraint_impl<"uaba", ".8h",  VPR128, v8i16,
 533                     0b1, 0b1, 0b01, 0b01111, Neon_uaba>;
 534 def UABAvvv_2S :  NeonI_3VSame_Constraint_impl<"uaba", ".2s",  VPR64,  v2i32,
 535                     0b0, 0b1, 0b10, 0b01111, Neon_uaba>;
 536 def UABAvvv_4S :  NeonI_3VSame_Constraint_impl<"uaba", ".4s",  VPR128, v4i32,
 537                     0b1, 0b1, 0b10, 0b01111, Neon_uaba>;
 538
 539 // Vector Absolute Difference and Accumulate (Signed)
 540 def SABAvvv_8B :  NeonI_3VSame_Constraint_impl<"saba", ".8b",  VPR64,  v8i8,
 541                     0b0, 0b0, 0b00, 0b01111, Neon_saba>;
 542 def SABAvvv_16B : NeonI_3VSame_Constraint_impl<"saba", ".16b", VPR128, v16i8,
 543                     0b1, 0b0, 0b00, 0b01111, Neon_saba>;
 544 def SABAvvv_4H :  NeonI_3VSame_Constraint_impl<"saba", ".4h",  VPR64,  v4i16,
 545                     0b0, 0b0, 0b01, 0b01111, Neon_saba>;
 546 def SABAvvv_8H :  NeonI_3VSame_Constraint_impl<"saba", ".8h",  VPR128, v8i16,
 547                     0b1, 0b0, 0b01, 0b01111, Neon_saba>;
 548 def SABAvvv_2S :  NeonI_3VSame_Constraint_impl<"saba", ".2s",  VPR64,  v2i32,
 549                     0b0, 0b0, 0b10, 0b01111, Neon_saba>;
 550 def SABAvvv_4S :  NeonI_3VSame_Constraint_impl<"saba", ".4s",  VPR128, v4i32,
 551                     0b1, 0b0, 0b10, 0b01111, Neon_saba>;
 552
 553
 554 // Vector Absolute Difference (Signed, Unsigned)
 555 defm UABDvvv : NeonI_3VSame_BHS_sizes<0b1, 0b01110, "uabd", int_arm_neon_vabdu, 0>;
 556 defm SABDvvv : NeonI_3VSame_BHS_sizes<0b0, 0b01110, "sabd", int_arm_neon_vabds, 0>;
 557
 558 // Vector Absolute Difference (Floating Point)
 559 defm FABDvvv: NeonI_3VSame_SD_sizes<0b1, 0b1, 0b11010, "fabd",
 560                                     int_arm_neon_vabds, int_arm_neon_vabds,
 561                                     int_arm_neon_vabds, v2f32, v4f32, v2f64, 0>;
 562
 563 // Vector Reciprocal Step (Floating Point)
 564 defm FRECPSvvv : NeonI_3VSame_SD_sizes<0b0, 0b0, 0b11111, "frecps",
 565                                        int_arm_neon_vrecps, int_arm_neon_vrecps,
 566                                        int_arm_neon_vrecps,
 567                                        v2f32, v4f32, v2f64, 0>;
 568
 569 // Vector Reciprocal Square Root Step (Floating Point)
 570 defm FRSQRTSvvv : NeonI_3VSame_SD_sizes<0b0, 0b1, 0b11111, "frsqrts",
 571                                         int_arm_neon_vrsqrts,
 572                                         int_arm_neon_vrsqrts,
 573                                         int_arm_neon_vrsqrts,
 574                                         v2f32, v4f32, v2f64, 0>;
 575
 576 // Vector Comparisons
 577
 578 def Neon_cmeq : PatFrag<(ops node:$lhs, node:$rhs),
 579                         (Neon_cmp node:$lhs, node:$rhs, SETEQ)>;
 580 def Neon_cmphs : PatFrag<(ops node:$lhs, node:$rhs),
 581                          (Neon_cmp node:$lhs, node:$rhs, SETUGE)>;
 582 def Neon_cmge : PatFrag<(ops node:$lhs, node:$rhs),
 583                         (Neon_cmp node:$lhs, node:$rhs, SETGE)>;
 584 def Neon_cmhi : PatFrag<(ops node:$lhs, node:$rhs),
 585                         (Neon_cmp node:$lhs, node:$rhs, SETUGT)>;
 586 def Neon_cmgt : PatFrag<(ops node:$lhs, node:$rhs),
 587                         (Neon_cmp node:$lhs, node:$rhs, SETGT)>;
 588
 589 // NeonI_compare_aliases class: swaps register operands to implement
 590 // comparison aliases, e.g., CMLE is alias for CMGE with operands reversed.
 591 class NeonI_compare_aliases<string asmop, string asmlane,
 592                             Instruction inst, RegisterOperand VPRC>
 593   : NeonInstAlias<asmop # "\t$Rd" # asmlane #", $Rn" # asmlane #
 594                     ", $Rm" # asmlane,
 595                   (inst VPRC:$Rd, VPRC:$Rm, VPRC:$Rn), 0b0>;
 596
 597 // Vector Comparisons (Integer)
 598
 599 // Vector Compare Mask Equal (Integer)
 600 let isCommutable =1 in {
 601 defm CMEQvvv : NeonI_3VSame_BHSD_sizes<0b1, 0b10001, "cmeq", Neon_cmeq, 0>;
 602 }
 603
 604 // Vector Compare Mask Higher or Same (Unsigned Integer)
 605 defm CMHSvvv : NeonI_3VSame_BHSD_sizes<0b1, 0b00111, "cmhs", Neon_cmphs, 0>;
 606
 607 // Vector Compare Mask Greater Than or Equal (Integer)
 608 defm CMGEvvv : NeonI_3VSame_BHSD_sizes<0b0, 0b00111, "cmge", Neon_cmge, 0>;
 609
 610 // Vector Compare Mask Higher (Unsigned Integer)
 611 defm CMHIvvv : NeonI_3VSame_BHSD_sizes<0b1, 0b00110, "cmhi", Neon_cmhi, 0>;
 612
 613 // Vector Compare Mask Greater Than (Integer)
 614 defm CMGTvvv : NeonI_3VSame_BHSD_sizes<0b0, 0b00110, "cmgt", Neon_cmgt, 0>;
 615
 616 // Vector Compare Mask Bitwise Test (Integer)
 617 defm CMTSTvvv:  NeonI_3VSame_BHSD_sizes<0b0, 0b10001, "cmtst", Neon_tst, 0>;
 618
 619 // Vector Compare Mask Less or Same (Unsigned Integer)
 620 // CMLS is alias for CMHS with operands reversed.
 621 def CMLSvvv_8B  : NeonI_compare_aliases<"cmls", ".8b",  CMHSvvv_8B,  VPR64>;
 622 def CMLSvvv_16B : NeonI_compare_aliases<"cmls", ".16b", CMHSvvv_16B, VPR128>;
 623 def CMLSvvv_4H  : NeonI_compare_aliases<"cmls", ".4h",  CMHSvvv_4H,  VPR64>;
 624 def CMLSvvv_8H  : NeonI_compare_aliases<"cmls", ".8h",  CMHSvvv_8H,  VPR128>;
 625 def CMLSvvv_2S  : NeonI_compare_aliases<"cmls", ".2s",  CMHSvvv_2S,  VPR64>;
 626 def CMLSvvv_4S  : NeonI_compare_aliases<"cmls", ".4s",  CMHSvvv_4S,  VPR128>;
 627 def CMLSvvv_2D  : NeonI_compare_aliases<"cmls", ".2d",  CMHSvvv_2D,  VPR128>;
 628
 629 // Vector Compare Mask Less Than or Equal (Integer)
 630 // CMLE is alias for CMGE with operands reversed.
 631 def CMLEvvv_8B  : NeonI_compare_aliases<"cmle", ".8b",  CMGEvvv_8B,  VPR64>;
 632 def CMLEvvv_16B : NeonI_compare_aliases<"cmle", ".16b", CMGEvvv_16B, VPR128>;
 633 def CMLEvvv_4H  : NeonI_compare_aliases<"cmle", ".4h",  CMGEvvv_4H,  VPR64>;
 634 def CMLEvvv_8H  : NeonI_compare_aliases<"cmle", ".8h",  CMGEvvv_8H,  VPR128>;
 635 def CMLEvvv_2S  : NeonI_compare_aliases<"cmle", ".2s",  CMGEvvv_2S,  VPR64>;
 636 def CMLEvvv_4S  : NeonI_compare_aliases<"cmle", ".4s",  CMGEvvv_4S,  VPR128>;
 637 def CMLEvvv_2D  : NeonI_compare_aliases<"cmle", ".2d",  CMGEvvv_2D,  VPR128>;
 638
 639 // Vector Compare Mask Lower (Unsigned Integer)
 640 // CMLO is alias for CMHI with operands reversed.
 641 def CMLOvvv_8B  : NeonI_compare_aliases<"cmlo", ".8b",  CMHIvvv_8B,  VPR64>;
 642 def CMLOvvv_16B : NeonI_compare_aliases<"cmlo", ".16b", CMHIvvv_16B, VPR128>;
 643 def CMLOvvv_4H  : NeonI_compare_aliases<"cmlo", ".4h",  CMHIvvv_4H,  VPR64>;
 644 def CMLOvvv_8H  : NeonI_compare_aliases<"cmlo", ".8h",  CMHIvvv_8H,  VPR128>;
 645 def CMLOvvv_2S  : NeonI_compare_aliases<"cmlo", ".2s",  CMHIvvv_2S,  VPR64>;
 646 def CMLOvvv_4S  : NeonI_compare_aliases<"cmlo", ".4s",  CMHIvvv_4S,  VPR128>;
 647 def CMLOvvv_2D  : NeonI_compare_aliases<"cmlo", ".2d",  CMHIvvv_2D,  VPR128>;
 648
 649 // Vector Compare Mask Less Than (Integer)
 650 // CMLT is alias for CMGT with operands reversed.
 651 def CMLTvvv_8B  : NeonI_compare_aliases<"cmlt", ".8b",  CMGTvvv_8B,  VPR64>;
 652 def CMLTvvv_16B : NeonI_compare_aliases<"cmlt", ".16b", CMGTvvv_16B, VPR128>;
 653 def CMLTvvv_4H  : NeonI_compare_aliases<"cmlt", ".4h",  CMGTvvv_4H,  VPR64>;
 654 def CMLTvvv_8H  : NeonI_compare_aliases<"cmlt", ".8h",  CMGTvvv_8H,  VPR128>;
 655 def CMLTvvv_2S  : NeonI_compare_aliases<"cmlt", ".2s",  CMGTvvv_2S,  VPR64>;
 656 def CMLTvvv_4S  : NeonI_compare_aliases<"cmlt", ".4s",  CMGTvvv_4S,  VPR128>;
 657 def CMLTvvv_2D  : NeonI_compare_aliases<"cmlt", ".2d",  CMGTvvv_2D,  VPR128>;
 658
 659
 660 def neon_uimm0_asmoperand : AsmOperandClass
 661 {
 662   let Name = "UImm0";
 663   let PredicateMethod = "isUImm<0>";
 664   let RenderMethod = "addImmOperands";
 665 }
 666
 667 def neon_uimm0 : Operand<i32>, ImmLeaf<i32, [{return Imm == 0;}]> {
 668   let ParserMatchClass = neon_uimm0_asmoperand;
 669   let PrintMethod = "printNeonUImm0Operand";
 670
 671 }
 672
 673 multiclass NeonI_cmpz_sizes<bit u, bits<5> opcode, string asmop, CondCode CC>
 674 {
 675   def _8B :  NeonI_2VMisc<0b0, u, 0b00, opcode,
 676              (outs VPR64:$Rd), (ins VPR64:$Rn, neon_uimm0:$Imm),
 677              asmop # "\t$Rd.8b, $Rn.8b, $Imm",
 678              [(set (v8i8 VPR64:$Rd),
 679                 (v8i8 (Neon_cmpz (v8i8 VPR64:$Rn), (i32 imm:$Imm), CC)))],
 680              NoItinerary>;
 681
 682   def _16B : NeonI_2VMisc<0b1, u, 0b00, opcode,
 683              (outs VPR128:$Rd), (ins VPR128:$Rn, neon_uimm0:$Imm),
 684              asmop # "\t$Rd.16b, $Rn.16b, $Imm",
 685              [(set (v16i8 VPR128:$Rd),
 686                 (v16i8 (Neon_cmpz (v16i8 VPR128:$Rn), (i32 imm:$Imm), CC)))],
 687              NoItinerary>;
 688
 689   def _4H : NeonI_2VMisc<0b0, u, 0b01, opcode,
 690             (outs VPR64:$Rd), (ins VPR64:$Rn, neon_uimm0:$Imm),
 691             asmop # "\t$Rd.4h, $Rn.4h, $Imm",
 692             [(set (v4i16 VPR64:$Rd),
 693                (v4i16 (Neon_cmpz (v4i16 VPR64:$Rn), (i32 imm:$Imm), CC)))],
 694             NoItinerary>;
 695
 696   def _8H : NeonI_2VMisc<0b1, u, 0b01, opcode,
 697             (outs VPR128:$Rd), (ins VPR128:$Rn, neon_uimm0:$Imm),
 698             asmop # "\t$Rd.8h, $Rn.8h, $Imm",
 699             [(set (v8i16 VPR128:$Rd),
 700                (v8i16 (Neon_cmpz (v8i16 VPR128:$Rn), (i32 imm:$Imm), CC)))],
 701             NoItinerary>;
 702
 703   def _2S : NeonI_2VMisc<0b0, u, 0b10, opcode,
 704             (outs VPR64:$Rd), (ins VPR64:$Rn, neon_uimm0:$Imm),
 705             asmop # "\t$Rd.2s, $Rn.2s, $Imm",
 706             [(set (v2i32 VPR64:$Rd),
 707                (v2i32 (Neon_cmpz (v2i32 VPR64:$Rn), (i32 imm:$Imm), CC)))],
 708             NoItinerary>;
 709
 710   def _4S : NeonI_2VMisc<0b1, u, 0b10, opcode,
 711             (outs VPR128:$Rd), (ins VPR128:$Rn, neon_uimm0:$Imm),
 712             asmop # "\t$Rd.4s, $Rn.4s, $Imm",
 713             [(set (v4i32 VPR128:$Rd),
 714                (v4i32 (Neon_cmpz (v4i32 VPR128:$Rn), (i32 imm:$Imm), CC)))],
 715             NoItinerary>;
 716
 717   def _2D : NeonI_2VMisc<0b1, u, 0b11, opcode,
 718             (outs VPR128:$Rd), (ins VPR128:$Rn, neon_uimm0:$Imm),
 719             asmop # "\t$Rd.2d, $Rn.2d, $Imm",
 720             [(set (v2i64 VPR128:$Rd),
 721                (v2i64 (Neon_cmpz (v2i64 VPR128:$Rn), (i32 imm:$Imm), CC)))],
 722             NoItinerary>;
 723 }
 724
 725 // Vector Compare Mask Equal to Zero (Integer)
 726 defm CMEQvvi : NeonI_cmpz_sizes<0b0, 0b01001, "cmeq", SETEQ>;
 727
 728 // Vector Compare Mask Greater Than or Equal to Zero (Signed Integer)
 729 defm CMGEvvi : NeonI_cmpz_sizes<0b1, 0b01000, "cmge", SETGE>;
 730
 731 // Vector Compare Mask Greater Than Zero (Signed Integer)
 732 defm CMGTvvi : NeonI_cmpz_sizes<0b0, 0b01000, "cmgt", SETGT>;
 733
 734 // Vector Compare Mask Less Than or Equal To Zero (Signed Integer)
 735 defm CMLEvvi : NeonI_cmpz_sizes<0b1, 0b01001, "cmle", SETLE>;
 736
 737 // Vector Compare Mask Less Than Zero (Signed Integer)
 738 defm CMLTvvi : NeonI_cmpz_sizes<0b0, 0b01010, "cmlt", SETLT>;
 739
 740 // Vector Comparisons (Floating Point)
 741
 742 // Vector Compare Mask Equal (Floating Point)
 743 let isCommutable =1 in {
 744 defm FCMEQvvv : NeonI_3VSame_SD_sizes<0b0, 0b0, 0b11100, "fcmeq", Neon_cmeq,
 745                                       Neon_cmeq, Neon_cmeq,
 746                                       v2i32, v4i32, v2i64, 0>;
 747 }
 748
 749 // Vector Compare Mask Greater Than Or Equal (Floating Point)
 750 defm FCMGEvvv : NeonI_3VSame_SD_sizes<0b1, 0b0, 0b11100, "fcmge", Neon_cmge,
 751                                       Neon_cmge, Neon_cmge,
 752                                       v2i32, v4i32, v2i64, 0>;
 753
 754 // Vector Compare Mask Greater Than (Floating Point)
 755 defm FCMGTvvv : NeonI_3VSame_SD_sizes<0b1, 0b1, 0b11100, "fcmgt", Neon_cmgt,
 756                                       Neon_cmgt, Neon_cmgt,
 757                                       v2i32, v4i32, v2i64, 0>;
 758
 759 // Vector Compare Mask Less Than Or Equal (Floating Point)
 760 // FCMLE is alias for FCMGE with operands reversed.
 761 def FCMLEvvv_2S  : NeonI_compare_aliases<"fcmle", ".2s",  FCMGEvvv_2S,  VPR64>;
 762 def FCMLEvvv_4S  : NeonI_compare_aliases<"fcmle", ".4s",  FCMGEvvv_4S,  VPR128>;
 763 def FCMLEvvv_2D  : NeonI_compare_aliases<"fcmle", ".2d",  FCMGEvvv_2D,  VPR128>;
 764
 765 // Vector Compare Mask Less Than (Floating Point)
 766 // FCMLT is alias for FCMGT with operands reversed.
 767 def FCMLTvvv_2S  : NeonI_compare_aliases<"fcmlt", ".2s",  FCMGTvvv_2S,  VPR64>;
 768 def FCMLTvvv_4S  : NeonI_compare_aliases<"fcmlt", ".4s",  FCMGTvvv_4S,  VPR128>;
 769 def FCMLTvvv_2D  : NeonI_compare_aliases<"fcmlt", ".2d",  FCMGTvvv_2D,  VPR128>;
 770
 771
 772 multiclass NeonI_fpcmpz_sizes<bit u, bit size, bits<5> opcode,
 773                               string asmop, CondCode CC>
 774 {
 775   def _2S : NeonI_2VMisc<0b0, u, {size, 0b0}, opcode,
 776             (outs VPR64:$Rd), (ins VPR64:$Rn, fpz32:$FPImm),
 777             asmop # "\t$Rd.2s, $Rn.2s, $FPImm",
 778             [(set (v2i32 VPR64:$Rd),
 779                (v2i32 (Neon_cmpz (v2f32 VPR64:$Rn), (f32 fpimm:$FPImm), CC)))],
 780             NoItinerary>;
 781
 782   def _4S : NeonI_2VMisc<0b1, u, {size, 0b0}, opcode,
 783             (outs VPR128:$Rd), (ins VPR128:$Rn, fpz32:$FPImm),
 784             asmop # "\t$Rd.4s, $Rn.4s, $FPImm",
 785             [(set (v4i32 VPR128:$Rd),
 786                (v4i32 (Neon_cmpz (v4f32 VPR128:$Rn), (f32 fpimm:$FPImm), CC)))],
 787             NoItinerary>;
 788
 789   def _2D : NeonI_2VMisc<0b1, u, {size, 0b1}, opcode,
 790             (outs VPR128:$Rd), (ins VPR128:$Rn, fpz32:$FPImm),
 791             asmop # "\t$Rd.2d, $Rn.2d, $FPImm",
 792             [(set (v2i64 VPR128:$Rd),
 793                (v2i64 (Neon_cmpz (v2f64 VPR128:$Rn), (f32 fpimm:$FPImm), CC)))],
 794             NoItinerary>;
 795 }
 796
 797 // Vector Compare Mask Equal to Zero (Floating Point)
 798 defm FCMEQvvi : NeonI_fpcmpz_sizes<0b0, 0b1, 0b01101, "fcmeq", SETEQ>;
 799
 800 // Vector Compare Mask Greater Than or Equal to Zero (Floating Point)
 801 defm FCMGEvvi : NeonI_fpcmpz_sizes<0b1, 0b1, 0b01100, "fcmge", SETGE>;
 802
 803 // Vector Compare Mask Greater Than Zero (Floating Point)
 804 defm FCMGTvvi : NeonI_fpcmpz_sizes<0b0, 0b1, 0b01100, "fcmgt", SETGT>;
 805
 806 // Vector Compare Mask Less Than or Equal To Zero (Floating Point)
 807 defm FCMLEvvi : NeonI_fpcmpz_sizes<0b1, 0b1, 0b01101, "fcmle", SETLE>;
 808
 809 // Vector Compare Mask Less Than Zero (Floating Point)
 810 defm FCMLTvvi : NeonI_fpcmpz_sizes<0b0, 0b1, 0b01110, "fcmlt", SETLT>;
 811
 812 // Vector Absolute Comparisons (Floating Point)
 813
 814 // Vector Absolute Compare Mask Greater Than Or Equal (Floating Point)
 815 defm FACGEvvv : NeonI_3VSame_SD_sizes<0b1, 0b0, 0b11101, "facge",
 816                                       int_arm_neon_vacged, int_arm_neon_vacgeq,
 817                                       int_aarch64_neon_vacgeq,
 818                                       v2i32, v4i32, v2i64, 0>;
 819
 820 // Vector Absolute Compare Mask Greater Than (Floating Point)
 821 defm FACGTvvv : NeonI_3VSame_SD_sizes<0b1, 0b1, 0b11101, "facgt",
 822                                       int_arm_neon_vacgtd, int_arm_neon_vacgtq,
 823                                       int_aarch64_neon_vacgtq,
 824                                       v2i32, v4i32, v2i64, 0>;
 825
 826 // Vector Absolute Compare Mask Less Than Or Equal (Floating Point)
 827 // FACLE is alias for FACGE with operands reversed.
 828 def FACLEvvv_2S  : NeonI_compare_aliases<"facle", ".2s",  FACGEvvv_2S,  VPR64>;
 829 def FACLEvvv_4S  : NeonI_compare_aliases<"facle", ".4s",  FACGEvvv_4S,  VPR128>;
 830 def FACLEvvv_2D  : NeonI_compare_aliases<"facle", ".2d",  FACGEvvv_2D,  VPR128>;
 831
 832 // Vector Absolute Compare Mask Less Than (Floating Point)
 833 // FACLT is alias for FACGT with operands reversed.
 834 def FACLTvvv_2S  : NeonI_compare_aliases<"faclt", ".2s",  FACGTvvv_2S,  VPR64>;
 835 def FACLTvvv_4S  : NeonI_compare_aliases<"faclt", ".4s",  FACGTvvv_4S,  VPR128>;
 836 def FACLTvvv_2D  : NeonI_compare_aliases<"faclt", ".2d",  FACGTvvv_2D,  VPR128>;
 837
 838 // Vector halving add (Integer Signed, Unsigned)
 839 defm SHADDvvv :  NeonI_3VSame_BHS_sizes<0b0, 0b00000, "shadd",
 840                                         int_arm_neon_vhadds, 1>;
 841 defm UHADDvvv :  NeonI_3VSame_BHS_sizes<0b1, 0b00000, "uhadd",
 842                                         int_arm_neon_vhaddu, 1>;
 843
 844 // Vector halving sub (Integer Signed, Unsigned)
 845 defm SHSUBvvv :  NeonI_3VSame_BHS_sizes<0b0, 0b00100, "shsub",
 846                                         int_arm_neon_vhsubs, 0>;
 847 defm UHSUBvvv :  NeonI_3VSame_BHS_sizes<0b1, 0b00100, "uhsub",
 848                                         int_arm_neon_vhsubu, 0>;
 849
 850 // Vector rouding halving add (Integer Signed, Unsigned)
 851 defm SRHADDvvv :  NeonI_3VSame_BHS_sizes<0b0, 0b00010, "srhadd",
 852                                          int_arm_neon_vrhadds, 1>;
 853 defm URHADDvvv :  NeonI_3VSame_BHS_sizes<0b1, 0b00010, "urhadd",
 854                                          int_arm_neon_vrhaddu, 1>;
 855
 856 // Vector Saturating add (Integer Signed, Unsigned)
 857 defm SQADDvvv :  NeonI_3VSame_BHSD_sizes<0b0, 0b00001, "sqadd",
 858                    int_arm_neon_vqadds, 1>;
 859 defm UQADDvvv :  NeonI_3VSame_BHSD_sizes<0b1, 0b00001, "uqadd",
 860                    int_arm_neon_vqaddu, 1>;
 861
 862 // Vector Saturating sub (Integer Signed, Unsigned)
 863 defm SQSUBvvv :  NeonI_3VSame_BHSD_sizes<0b0, 0b00101, "sqsub",
 864                    int_arm_neon_vqsubs, 1>;
 865 defm UQSUBvvv :  NeonI_3VSame_BHSD_sizes<0b1, 0b00101, "uqsub",
 866                    int_arm_neon_vqsubu, 1>;
 867
 868 // Vector Shift Left (Signed and Unsigned Integer)
 869 defm SSHLvvv : NeonI_3VSame_BHSD_sizes<0b0, 0b01000, "sshl",
 870                  int_arm_neon_vshifts, 1>;
 871 defm USHLvvv : NeonI_3VSame_BHSD_sizes<0b1, 0b01000, "ushl",
 872                  int_arm_neon_vshiftu, 1>;
 873
 874 // Vector Saturating Shift Left (Signed and Unsigned Integer)
 875 defm SQSHLvvv : NeonI_3VSame_BHSD_sizes<0b0, 0b01001, "sqshl",
 876                   int_arm_neon_vqshifts, 1>;
 877 defm UQSHLvvv : NeonI_3VSame_BHSD_sizes<0b1, 0b01001, "uqshl",
 878                   int_arm_neon_vqshiftu, 1>;
 879
 880 // Vector Rouding Shift Left (Signed and Unsigned Integer)
 881 defm SRSHLvvv : NeonI_3VSame_BHSD_sizes<0b0, 0b01010, "srshl",
 882                   int_arm_neon_vrshifts, 1>;
 883 defm URSHLvvv : NeonI_3VSame_BHSD_sizes<0b1, 0b01010, "urshl",
 884                   int_arm_neon_vrshiftu, 1>;
 885
 886 // Vector Saturating Rouding Shift Left (Signed and Unsigned Integer)
 887 defm SQRSHLvvv : NeonI_3VSame_BHSD_sizes<0b0, 0b01011, "sqrshl",
 888                    int_arm_neon_vqrshifts, 1>;
 889 defm UQRSHLvvv : NeonI_3VSame_BHSD_sizes<0b1, 0b01011, "uqrshl",
 890                    int_arm_neon_vqrshiftu, 1>;
 891
 892 // Vector Maximum (Signed and Unsigned Integer)
 893 defm SMAXvvv : NeonI_3VSame_BHS_sizes<0b0, 0b01100, "smax", int_arm_neon_vmaxs, 1>;
 894 defm UMAXvvv : NeonI_3VSame_BHS_sizes<0b1, 0b01100, "umax", int_arm_neon_vmaxu, 1>;
 895
 896 // Vector Minimum (Signed and Unsigned Integer)
 897 defm SMINvvv : NeonI_3VSame_BHS_sizes<0b0, 0b01101, "smin", int_arm_neon_vmins, 1>;
 898 defm UMINvvv : NeonI_3VSame_BHS_sizes<0b1, 0b01101, "umin", int_arm_neon_vminu, 1>;
 899
 900 // Vector Maximum (Floating Point)
 901 defm FMAXvvv : NeonI_3VSame_SD_sizes<0b0, 0b0, 0b11110, "fmax",
 902                                      int_arm_neon_vmaxs, int_arm_neon_vmaxs,
 903                                      int_arm_neon_vmaxs, v2f32, v4f32, v2f64, 1>;
 904
 905 // Vector Minimum (Floating Point)
 906 defm FMINvvv : NeonI_3VSame_SD_sizes<0b0, 0b1, 0b11110, "fmin",
 907                                      int_arm_neon_vmins, int_arm_neon_vmins,
 908                                      int_arm_neon_vmins, v2f32, v4f32, v2f64, 1>;
 909
 910 // Vector maxNum (Floating Point) -  prefer a number over a quiet NaN)
 911 defm FMAXNMvvv : NeonI_3VSame_SD_sizes<0b0, 0b0, 0b11000, "fmaxnm",
 912                                        int_aarch64_neon_vmaxnm,
 913                                        int_aarch64_neon_vmaxnm,
 914                                        int_aarch64_neon_vmaxnm,
 915                                        v2f32, v4f32, v2f64, 1>;
 916
 917 // Vector minNum (Floating Point) - prefer a number over a quiet NaN)
 918 defm FMINNMvvv : NeonI_3VSame_SD_sizes<0b0, 0b1, 0b11000, "fminnm",
 919                                        int_aarch64_neon_vminnm,
 920                                        int_aarch64_neon_vminnm,
 921                                        int_aarch64_neon_vminnm,
 922                                        v2f32, v4f32, v2f64, 1>;
 923
 924 // Vector Maximum Pairwise (Signed and Unsigned Integer)
 925 defm SMAXPvvv : NeonI_3VSame_BHS_sizes<0b0, 0b10100, "smaxp", int_arm_neon_vpmaxs, 1>;
 926 defm UMAXPvvv : NeonI_3VSame_BHS_sizes<0b1, 0b10100, "umaxp", int_arm_neon_vpmaxu, 1>;
 927
 928 // Vector Minimum Pairwise (Signed and Unsigned Integer)
 929 defm SMINPvvv : NeonI_3VSame_BHS_sizes<0b0, 0b10101, "sminp", int_arm_neon_vpmins, 1>;
 930 defm UMINPvvv : NeonI_3VSame_BHS_sizes<0b1, 0b10101, "uminp", int_arm_neon_vpminu, 1>;
 931
 932 // Vector Maximum Pairwise (Floating Point)
 933 defm FMAXPvvv : NeonI_3VSame_SD_sizes<0b1, 0b0, 0b11110, "fmaxp",
 934                                      int_arm_neon_vpmaxs, int_arm_neon_vpmaxs,
 935                                      int_arm_neon_vpmaxs, v2f32, v4f32, v2f64, 1>;
 936
 937 // Vector Minimum Pairwise (Floating Point)
 938 defm FMINPvvv : NeonI_3VSame_SD_sizes<0b1, 0b1, 0b11110, "fminp",
 939                                      int_arm_neon_vpmins, int_arm_neon_vpmins,
 940                                      int_arm_neon_vpmins, v2f32, v4f32, v2f64, 1>;
 941
 942 // Vector maxNum Pairwise (Floating Point) -  prefer a number over a quiet NaN)
 943 defm FMAXNMPvvv : NeonI_3VSame_SD_sizes<0b1, 0b0, 0b11000, "fmaxnmp",
 944                                        int_aarch64_neon_vpmaxnm,
 945                                        int_aarch64_neon_vpmaxnm,
 946                                        int_aarch64_neon_vpmaxnm,
 947                                        v2f32, v4f32, v2f64, 1>;
 948
 949 // Vector minNum Pairwise (Floating Point) -  prefer a number over a quiet NaN)
 950 defm FMINNMPvvv : NeonI_3VSame_SD_sizes<0b1, 0b1, 0b11000, "fminnmp",
 951                                        int_aarch64_neon_vpminnm,
 952                                        int_aarch64_neon_vpminnm,
 953                                        int_aarch64_neon_vpminnm,
 954                                        v2f32, v4f32, v2f64, 1>;
 955
 956 // Vector Addition Pairwise (Integer)
 957 defm ADDP : NeonI_3VSame_BHSD_sizes<0b0, 0b10111, "addp", int_arm_neon_vpadd, 1>;
 958
 959 // Vector Addition Pairwise (Floating Point)
 960 defm FADDP : NeonI_3VSame_SD_sizes<0b1, 0b0, 0b11010, "faddp",
 961                                        int_arm_neon_vpadd,
 962                                        int_arm_neon_vpadd,
 963                                        int_arm_neon_vpadd,
 964                                        v2f32, v4f32, v2f64, 1>;
 965
 966 // Vector Saturating Doubling Multiply High
 967 defm SQDMULHvvv : NeonI_3VSame_HS_sizes<0b0, 0b10110, "sqdmulh",
 968                     int_arm_neon_vqdmulh, 1>;
 969
 970 // Vector Saturating Rouding Doubling Multiply High
 971 defm SQRDMULHvvv : NeonI_3VSame_HS_sizes<0b1, 0b10110, "sqrdmulh",
 972                      int_arm_neon_vqrdmulh, 1>;
 973
 974 // Vector Multiply Extended (Floating Point)
 975 defm FMULXvvv : NeonI_3VSame_SD_sizes<0b0, 0b0, 0b11011, "fmulx",
 976                                       int_aarch64_neon_vmulx,
 977                                       int_aarch64_neon_vmulx,
 978                                       int_aarch64_neon_vmulx,
 979                                       v2f32, v4f32, v2f64, 1>;
 980
 981 // Vector Immediate Instructions
 982
 983 multiclass neon_mov_imm_shift_asmoperands<string PREFIX>
 984 {
 985   def _asmoperand : AsmOperandClass
 986     {
 987       let Name = "NeonMovImmShift" # PREFIX;
 988       let RenderMethod = "addNeonMovImmShift" # PREFIX # "Operands";
 989       let PredicateMethod = "isNeonMovImmShift" # PREFIX;
 990     }
 991 }
 992
 993 // Definition of vector immediates shift operands
 994
 995 // The selectable use-cases extract the shift operation
 996 // information from the OpCmode fields encoded in the immediate.
 997 def neon_mod_shift_imm_XFORM : SDNodeXForm<imm, [{
 998   uint64_t OpCmode = N->getZExtValue();
 999   unsigned ShiftImm;
1000   unsigned ShiftOnesIn;
1001   unsigned HasShift =
1002     A64Imms::decodeNeonModShiftImm(OpCmode, ShiftImm, ShiftOnesIn);
1003   if (!HasShift) return SDValue();
1004   return CurDAG->getTargetConstant(ShiftImm, MVT::i32);
1005 }]>;
1006
1007 // Vector immediates shift operands which accept LSL and MSL
1008 // shift operators with shift value in the range of 0, 8, 16, 24 (LSL),
1009 // or 0, 8 (LSLH) or 8, 16 (MSL).
1010 defm neon_mov_imm_LSL : neon_mov_imm_shift_asmoperands<"LSL">;
1011 defm neon_mov_imm_MSL : neon_mov_imm_shift_asmoperands<"MSL">;
1012 // LSLH restricts shift amount to  0, 8 out of 0, 8, 16, 24
1013 defm neon_mov_imm_LSLH : neon_mov_imm_shift_asmoperands<"LSLH">;
1014
1015 multiclass neon_mov_imm_shift_operands<string PREFIX,
1016                                        string HALF, string ISHALF, code pred>
1017 {
1018    def _operand : Operand<i32>, ImmLeaf<i32, pred, neon_mod_shift_imm_XFORM>
1019     {
1020       let PrintMethod =
1021         "printNeonMovImmShiftOperand<A64SE::" # PREFIX # ", " # ISHALF # ">";
1022       let DecoderMethod =
1023         "DecodeNeonMovImmShiftOperand<A64SE::" # PREFIX # ", " # ISHALF # ">";
1024       let ParserMatchClass =
1025         !cast<AsmOperandClass>("neon_mov_imm_" # PREFIX # HALF # "_asmoperand");
1026     }
1027 }
1028
1029 defm neon_mov_imm_LSL  : neon_mov_imm_shift_operands<"LSL", "", "false", [{
1030   unsigned ShiftImm;
1031   unsigned ShiftOnesIn;
1032   unsigned HasShift =
1033     A64Imms::decodeNeonModShiftImm(Imm, ShiftImm, ShiftOnesIn);
1034   return (HasShift && !ShiftOnesIn);
1035 }]>;
1036
1037 defm neon_mov_imm_MSL  : neon_mov_imm_shift_operands<"MSL", "", "false", [{
1038   unsigned ShiftImm;
1039   unsigned ShiftOnesIn;
1040   unsigned HasShift =
1041     A64Imms::decodeNeonModShiftImm(Imm, ShiftImm, ShiftOnesIn);
1042   return (HasShift && ShiftOnesIn);
1043 }]>;
1044
1045 defm neon_mov_imm_LSLH  : neon_mov_imm_shift_operands<"LSL", "H", "true", [{
1046   unsigned ShiftImm;
1047   unsigned ShiftOnesIn;
1048   unsigned HasShift =
1049     A64Imms::decodeNeonModShiftImm(Imm, ShiftImm, ShiftOnesIn);
1050   return (HasShift && !ShiftOnesIn);
1051 }]>;
1052
1053 def neon_uimm1_asmoperand : AsmOperandClass
1054 {
1055   let Name = "UImm1";
1056   let PredicateMethod = "isUImm<1>";
1057   let RenderMethod = "addImmOperands";
1058 }
1059
1060 def neon_uimm2_asmoperand : AsmOperandClass
1061 {
1062   let Name = "UImm2";
1063   let PredicateMethod = "isUImm<2>";
1064   let RenderMethod = "addImmOperands";
1065 }
1066
1067 def neon_uimm8_asmoperand : AsmOperandClass
1068 {
1069   let Name = "UImm8";
1070   let PredicateMethod = "isUImm<8>";
1071   let RenderMethod = "addImmOperands";
1072 }
1073
1074 def neon_uimm8 : Operand<i32>, ImmLeaf<i32, [{(void)Imm; return true;}]> {
1075   let ParserMatchClass = neon_uimm8_asmoperand;
1076   let PrintMethod = "printUImmHexOperand";
1077 }
1078
1079 def neon_uimm64_mask_asmoperand : AsmOperandClass
1080 {
1081   let Name = "NeonUImm64Mask";
1082   let PredicateMethod = "isNeonUImm64Mask";
1083   let RenderMethod = "addNeonUImm64MaskOperands";
1084 }
1085
1086 // MCOperand for 64-bit bytemask with each byte having only the
1087 // value 0x00 and 0xff is encoded as an unsigned 8-bit value
1088 def neon_uimm64_mask : Operand<i32>, ImmLeaf<i32, [{(void)Imm; return true;}]> {
1089   let ParserMatchClass = neon_uimm64_mask_asmoperand;
1090   let PrintMethod = "printNeonUImm64MaskOperand";
1091 }
1092
1093 multiclass NeonI_mov_imm_lsl_sizes<string asmop, bit op,
1094                                    SDPatternOperator opnode>
1095 {
1096     // shift zeros, per word
1097     def _2S  : NeonI_1VModImm<0b0, op,
1098                               (outs VPR64:$Rd),
1099                               (ins neon_uimm8:$Imm,
1100                                 neon_mov_imm_LSL_operand:$Simm),
1101                               !strconcat(asmop, "\t$Rd.2s, $Imm$Simm"),
1102                               [(set (v2i32 VPR64:$Rd),
1103                                  (v2i32 (opnode (timm:$Imm),
1104                                    (neon_mov_imm_LSL_operand:$Simm))))],
1105                               NoItinerary> {
1106        bits<2> Simm;
1107        let cmode = {0b0, Simm{1}, Simm{0}, 0b0};
1108      }
1109
1110     def _4S  : NeonI_1VModImm<0b1, op,
1111                               (outs VPR128:$Rd),
1112                               (ins neon_uimm8:$Imm,
1113                                 neon_mov_imm_LSL_operand:$Simm),
1114                               !strconcat(asmop, "\t$Rd.4s, $Imm$Simm"),
1115                               [(set (v4i32 VPR128:$Rd),
1116                                  (v4i32 (opnode (timm:$Imm),
1117                                    (neon_mov_imm_LSL_operand:$Simm))))],
1118                               NoItinerary> {
1119       bits<2> Simm;
1120       let cmode = {0b0, Simm{1}, Simm{0}, 0b0};
1121     }
1122
1123     // shift zeros, per halfword
1124     def _4H  : NeonI_1VModImm<0b0, op,
1125                               (outs VPR64:$Rd),
1126                               (ins neon_uimm8:$Imm,
1127                                 neon_mov_imm_LSLH_operand:$Simm),
1128                               !strconcat(asmop, "\t$Rd.4h, $Imm$Simm"),
1129                               [(set (v4i16 VPR64:$Rd),
1130                                  (v4i16 (opnode (timm:$Imm),
1131                                    (neon_mov_imm_LSLH_operand:$Simm))))],
1132                               NoItinerary> {
1133       bit  Simm;
1134       let cmode = {0b1, 0b0, Simm, 0b0};
1135     }
1136
1137     def _8H  : NeonI_1VModImm<0b1, op,
1138                               (outs VPR128:$Rd),
1139                               (ins neon_uimm8:$Imm,
1140                                 neon_mov_imm_LSLH_operand:$Simm),
1141                               !strconcat(asmop, "\t$Rd.8h, $Imm$Simm"),
1142                               [(set (v8i16 VPR128:$Rd),
1143                                  (v8i16 (opnode (timm:$Imm),
1144                                    (neon_mov_imm_LSLH_operand:$Simm))))],
1145                               NoItinerary> {
1146       bit Simm;
1147       let cmode = {0b1, 0b0, Simm, 0b0};
1148      }
1149 }
1150
1151 multiclass NeonI_mov_imm_with_constraint_lsl_sizes<string asmop, bit op,
1152                                                    SDPatternOperator opnode,
1153                                                    SDPatternOperator neonopnode>
1154 {
1155   let Constraints = "$src = $Rd" in {
1156     // shift zeros, per word
1157     def _2S  : NeonI_1VModImm<0b0, op,
1158                  (outs VPR64:$Rd),
1159                  (ins VPR64:$src, neon_uimm8:$Imm,
1160                    neon_mov_imm_LSL_operand:$Simm),
1161                  !strconcat(asmop, "\t$Rd.2s, $Imm$Simm"),
1162                  [(set (v2i32 VPR64:$Rd),
1163                     (v2i32 (opnode (v2i32 VPR64:$src),
1164                       (v2i32 (bitconvert (v2i32 (neonopnode timm:$Imm,
1165                         neon_mov_imm_LSL_operand:$Simm)))))))],
1166                  NoItinerary> {
1167       bits<2> Simm;
1168       let cmode = {0b0, Simm{1}, Simm{0}, 0b1};
1169     }
1170
1171     def _4S  : NeonI_1VModImm<0b1, op,
1172                  (outs VPR128:$Rd),
1173                  (ins VPR128:$src, neon_uimm8:$Imm,
1174                    neon_mov_imm_LSL_operand:$Simm),
1175                  !strconcat(asmop, "\t$Rd.4s, $Imm$Simm"),
1176                  [(set (v4i32 VPR128:$Rd),
1177                     (v4i32 (opnode (v4i32 VPR128:$src),
1178                       (v4i32 (bitconvert (v4i32 (neonopnode timm:$Imm,
1179                         neon_mov_imm_LSL_operand:$Simm)))))))],
1180                  NoItinerary> {
1181       bits<2> Simm;
1182       let cmode = {0b0, Simm{1}, Simm{0}, 0b1};
1183     }
1184
1185     // shift zeros, per halfword
1186     def _4H  : NeonI_1VModImm<0b0, op,
1187                  (outs VPR64:$Rd),
1188                  (ins VPR64:$src, neon_uimm8:$Imm,
1189                    neon_mov_imm_LSLH_operand:$Simm),
1190                  !strconcat(asmop, "\t$Rd.4h, $Imm$Simm"),
1191                  [(set (v4i16 VPR64:$Rd),
1192                     (v4i16 (opnode (v4i16 VPR64:$src),
1193                        (v4i16 (bitconvert (v4i16 (neonopnode timm:$Imm,
1194                           neon_mov_imm_LSL_operand:$Simm)))))))],
1195                  NoItinerary> {
1196       bit  Simm;
1197       let cmode = {0b1, 0b0, Simm, 0b1};
1198     }
1199
1200     def _8H  : NeonI_1VModImm<0b1, op,
1201                  (outs VPR128:$Rd),
1202                  (ins VPR128:$src, neon_uimm8:$Imm,
1203                    neon_mov_imm_LSLH_operand:$Simm),
1204                  !strconcat(asmop, "\t$Rd.8h, $Imm$Simm"),
1205                  [(set (v8i16 VPR128:$Rd),
1206                     (v8i16 (opnode (v8i16 VPR128:$src),
1207                       (v8i16 (bitconvert (v8i16 (neonopnode timm:$Imm,
1208                         neon_mov_imm_LSL_operand:$Simm)))))))],
1209                  NoItinerary> {
1210       bit Simm;
1211       let cmode = {0b1, 0b0, Simm, 0b1};
1212     }
1213   }
1214 }
1215
1216 multiclass NeonI_mov_imm_msl_sizes<string asmop, bit op,
1217                                    SDPatternOperator opnode>
1218 {
1219     // shift ones, per word
1220     def _2S  : NeonI_1VModImm<0b0, op,
1221                              (outs VPR64:$Rd),
1222                              (ins neon_uimm8:$Imm,
1223                                neon_mov_imm_MSL_operand:$Simm),
1224                              !strconcat(asmop, "\t$Rd.2s, $Imm$Simm"),
1225                               [(set (v2i32 VPR64:$Rd),
1226                                  (v2i32 (opnode (timm:$Imm),
1227                                    (neon_mov_imm_MSL_operand:$Simm))))],
1228                              NoItinerary> {
1229        bit Simm;
1230        let cmode = {0b1, 0b1, 0b0, Simm};
1231      }
1232
1233    def _4S  : NeonI_1VModImm<0b1, op,
1234                               (outs VPR128:$Rd),
1235                               (ins neon_uimm8:$Imm,
1236                                 neon_mov_imm_MSL_operand:$Simm),
1237                               !strconcat(asmop, "\t$Rd.4s, $Imm$Simm"),
1238                               [(set (v4i32 VPR128:$Rd),
1239                                  (v4i32 (opnode (timm:$Imm),
1240                                    (neon_mov_imm_MSL_operand:$Simm))))],
1241                               NoItinerary> {
1242      bit Simm;
1243      let cmode = {0b1, 0b1, 0b0, Simm};
1244    }
1245 }
1246
1247 // Vector Move Immediate Shifted
1248 let isReMaterializable = 1 in {
1249 defm MOVIvi_lsl : NeonI_mov_imm_lsl_sizes<"movi", 0b0, Neon_movi>;
1250 }
1251
1252 // Vector Move Inverted Immediate Shifted
1253 let isReMaterializable = 1 in {
1254 defm MVNIvi_lsl : NeonI_mov_imm_lsl_sizes<"mvni", 0b1, Neon_mvni>;
1255 }
1256
1257 // Vector Bitwise Bit Clear (AND NOT) - immediate
1258 let isReMaterializable = 1 in {
1259 defm BICvi_lsl : NeonI_mov_imm_with_constraint_lsl_sizes<"bic", 0b1,
1260                                                          and, Neon_mvni>;
1261 }
1262
1263 // Vector Bitwise OR - immedidate
1264
1265 let isReMaterializable = 1 in {
1266 defm ORRvi_lsl   : NeonI_mov_imm_with_constraint_lsl_sizes<"orr", 0b0,
1267                                                            or, Neon_movi>;
1268 }
1269
1270 // Additional patterns for Vector Bitwise Bit Clear (AND NOT) - immedidate
1271 // LowerBUILD_VECTOR favors lowering MOVI over MVNI.
1272 // BIC immediate instructions selection requires additional patterns to
1273 // transform Neon_movi operands into BIC immediate operands
1274
1275 def neon_mov_imm_LSLH_transform_XFORM : SDNodeXForm<imm, [{
1276   uint64_t OpCmode = N->getZExtValue();
1277   unsigned ShiftImm;
1278   unsigned ShiftOnesIn;
1279   (void)A64Imms::decodeNeonModShiftImm(OpCmode, ShiftImm, ShiftOnesIn);
1280   // LSLH restricts shift amount to  0, 8 which are encoded as 0 and 1
1281   // Transform encoded shift amount 0 to 1 and 1 to 0.
1282   return CurDAG->getTargetConstant(!ShiftImm, MVT::i32);
1283 }]>;
1284
1285 def neon_mov_imm_LSLH_transform_operand
1286   : ImmLeaf<i32, [{
1287     unsigned ShiftImm;
1288     unsigned ShiftOnesIn;
1289     unsigned HasShift =
1290       A64Imms::decodeNeonModShiftImm(Imm, ShiftImm, ShiftOnesIn);
1291     return (HasShift && !ShiftOnesIn); }],
1292   neon_mov_imm_LSLH_transform_XFORM>;
1293
1294 // Transform (and A, (4h Neon_movi 0xff)) -> BIC 4h (A, 0x00, LSL 8)
1295 // Transform (and A, (4h Neon_movi 0xff LSL #8)) -> BIC 4h (A, 0x00)
1296 def : Pat<(v4i16 (and VPR64:$src,
1297             (v4i16 (Neon_movi 255, neon_mov_imm_LSLH_transform_operand:$Simm)))),
1298           (BICvi_lsl_4H VPR64:$src, 0,
1299             neon_mov_imm_LSLH_transform_operand:$Simm)>;
1300
1301 // Transform (and A, (8h Neon_movi 8h 0xff)) -> BIC 8h (A, 0x00, LSL 8)
1302 // Transform (and A, (8h Neon_movi 0xff LSL #8)) -> BIC 8h (A, 0x00)
1303 def : Pat<(v8i16 (and VPR128:$src,
1304             (v8i16 (Neon_movi 255, neon_mov_imm_LSLH_transform_operand:$Simm)))),
1305           (BICvi_lsl_8H VPR128:$src, 0,
1306             neon_mov_imm_LSLH_transform_operand:$Simm)>;
1307
1308
1309 multiclass Neon_bitwiseVi_patterns<SDPatternOperator opnode,
1310                                    SDPatternOperator neonopnode,
1311                                    Instruction INST4H,
1312                                    Instruction INST8H> {
1313   def : Pat<(v8i8 (opnode VPR64:$src,
1314                     (bitconvert(v4i16 (neonopnode timm:$Imm,
1315                       neon_mov_imm_LSLH_operand:$Simm))))),
1316             (INST4H VPR64:$src, neon_uimm8:$Imm,
1317               neon_mov_imm_LSLH_operand:$Simm)>;
1318   def : Pat<(v1i64 (opnode VPR64:$src,
1319                   (bitconvert(v4i16 (neonopnode timm:$Imm,
1320                     neon_mov_imm_LSLH_operand:$Simm))))),
1321           (INST4H VPR64:$src, neon_uimm8:$Imm,
1322             neon_mov_imm_LSLH_operand:$Simm)>;
1323
1324   def : Pat<(v16i8 (opnode VPR128:$src,
1325                    (bitconvert(v8i16 (neonopnode timm:$Imm,
1326                      neon_mov_imm_LSLH_operand:$Simm))))),
1327           (INST8H VPR128:$src, neon_uimm8:$Imm,
1328             neon_mov_imm_LSLH_operand:$Simm)>;
1329   def : Pat<(v4i32 (opnode VPR128:$src,
1330                    (bitconvert(v8i16 (neonopnode timm:$Imm,
1331                      neon_mov_imm_LSLH_operand:$Simm))))),
1332           (INST8H VPR128:$src, neon_uimm8:$Imm,
1333             neon_mov_imm_LSLH_operand:$Simm)>;
1334   def : Pat<(v2i64 (opnode VPR128:$src,
1335                    (bitconvert(v8i16 (neonopnode timm:$Imm,
1336                      neon_mov_imm_LSLH_operand:$Simm))))),
1337           (INST8H VPR128:$src, neon_uimm8:$Imm,
1338             neon_mov_imm_LSLH_operand:$Simm)>;
1339 }
1340
1341 // Additional patterns for Vector Vector Bitwise Bit Clear (AND NOT) - immediate
1342 defm : Neon_bitwiseVi_patterns<or, Neon_mvni, BICvi_lsl_4H, BICvi_lsl_8H>;
1343
1344 // Additional patterns for Vector Bitwise OR - immedidate
1345 defm : Neon_bitwiseVi_patterns<or, Neon_movi, ORRvi_lsl_4H, ORRvi_lsl_8H>;
1346
1347
1348 // Vector Move Immediate Masked
1349 let isReMaterializable = 1 in {
1350 defm MOVIvi_msl : NeonI_mov_imm_msl_sizes<"movi", 0b0, Neon_movi>;
1351 }
1352
1353 // Vector Move Inverted Immediate Masked
1354 let isReMaterializable = 1 in {
1355 defm MVNIvi_msl : NeonI_mov_imm_msl_sizes<"mvni", 0b1, Neon_mvni>;
1356 }
1357
1358 class NeonI_mov_imm_lsl_aliases<string asmop, string asmlane,
1359                                 Instruction inst, RegisterOperand VPRC>
1360   : NeonInstAlias<!strconcat(asmop, "\t$Rd," # asmlane # ", $Imm"),
1361                         (inst VPRC:$Rd, neon_uimm8:$Imm,  0), 0b0>;
1362
1363 // Aliases for Vector Move Immediate Shifted
1364 def : NeonI_mov_imm_lsl_aliases<"movi", ".2s", MOVIvi_lsl_2S, VPR64>;
1365 def : NeonI_mov_imm_lsl_aliases<"movi", ".4s", MOVIvi_lsl_4S, VPR128>;
1366 def : NeonI_mov_imm_lsl_aliases<"movi", ".4h", MOVIvi_lsl_4H, VPR64>;
1367 def : NeonI_mov_imm_lsl_aliases<"movi", ".8h", MOVIvi_lsl_8H, VPR128>;
1368
1369 // Aliases for Vector Move Inverted Immediate Shifted
1370 def : NeonI_mov_imm_lsl_aliases<"mvni", ".2s", MVNIvi_lsl_2S, VPR64>;
1371 def : NeonI_mov_imm_lsl_aliases<"mvni", ".4s", MVNIvi_lsl_4S, VPR128>;
1372 def : NeonI_mov_imm_lsl_aliases<"mvni", ".4h", MVNIvi_lsl_4H, VPR64>;
1373 def : NeonI_mov_imm_lsl_aliases<"mvni", ".8h", MVNIvi_lsl_8H, VPR128>;
1374
1375 // Aliases for Vector Bitwise Bit Clear (AND NOT) - immediate
1376 def : NeonI_mov_imm_lsl_aliases<"bic", ".2s", BICvi_lsl_2S, VPR64>;
1377 def : NeonI_mov_imm_lsl_aliases<"bic", ".4s", BICvi_lsl_4S, VPR128>;
1378 def : NeonI_mov_imm_lsl_aliases<"bic", ".4h", BICvi_lsl_4H, VPR64>;
1379 def : NeonI_mov_imm_lsl_aliases<"bic", ".8h", BICvi_lsl_8H, VPR128>;
1380
1381 // Aliases for Vector Bitwise OR - immedidate
1382 def : NeonI_mov_imm_lsl_aliases<"orr", ".2s", ORRvi_lsl_2S, VPR64>;
1383 def : NeonI_mov_imm_lsl_aliases<"orr", ".4s", ORRvi_lsl_4S, VPR128>;
1384 def : NeonI_mov_imm_lsl_aliases<"orr", ".4h", ORRvi_lsl_4H, VPR64>;
1385 def : NeonI_mov_imm_lsl_aliases<"orr", ".8h", ORRvi_lsl_8H, VPR128>;
1386
1387 //  Vector Move Immediate - per byte
1388 let isReMaterializable = 1 in {
1389 def MOVIvi_8B : NeonI_1VModImm<0b0, 0b0,
1390                                (outs VPR64:$Rd), (ins neon_uimm8:$Imm),
1391                                "movi\t$Rd.8b, $Imm",
1392                                [(set (v8i8 VPR64:$Rd),
1393                                   (v8i8 (Neon_movi (timm:$Imm), (i32 imm))))],
1394                                 NoItinerary> {
1395   let cmode = 0b1110;
1396 }
1397
1398 def MOVIvi_16B : NeonI_1VModImm<0b1, 0b0,
1399                                 (outs VPR128:$Rd), (ins neon_uimm8:$Imm),
1400                                 "movi\t$Rd.16b, $Imm",
1401                                 [(set (v16i8 VPR128:$Rd),
1402                                    (v16i8 (Neon_movi (timm:$Imm), (i32 imm))))],
1403                                  NoItinerary> {
1404   let cmode = 0b1110;
1405 }
1406 }
1407
1408 // Vector Move Immediate - bytemask, per double word
1409 let isReMaterializable = 1 in {
1410 def MOVIvi_2D : NeonI_1VModImm<0b1, 0b1,
1411                                (outs VPR128:$Rd), (ins neon_uimm64_mask:$Imm),
1412                                "movi\t $Rd.2d, $Imm",
1413                                [(set (v2i64 VPR128:$Rd),
1414                                   (v2i64 (Neon_movi (timm:$Imm), (i32 imm))))],
1415                                NoItinerary> {
1416   let cmode = 0b1110;
1417 }
1418 }
1419
1420 // Vector Move Immediate - bytemask, one doubleword
1421
1422 let isReMaterializable = 1 in {
1423 def MOVIdi : NeonI_1VModImm<0b0, 0b1,
1424                            (outs FPR64:$Rd), (ins neon_uimm64_mask:$Imm),
1425                            "movi\t $Rd, $Imm",
1426                            [(set (f64 FPR64:$Rd),
1427                               (f64 (bitconvert
1428                                 (v1i64 (Neon_movi (timm:$Imm), (i32 imm))))))],
1429                            NoItinerary> {
1430   let cmode = 0b1110;
1431 }
1432 }
1433
1434 // Vector Floating Point Move Immediate
1435
1436 class NeonI_FMOV_impl<string asmlane, RegisterOperand VPRC, ValueType OpTy,
1437                       Operand immOpType, bit q, bit op>
1438   : NeonI_1VModImm<q, op,
1439                    (outs VPRC:$Rd), (ins immOpType:$Imm),
1440                    "fmov\t$Rd" # asmlane # ", $Imm",
1441                    [(set (OpTy VPRC:$Rd),
1442                       (OpTy (Neon_fmovi (timm:$Imm))))],
1443                    NoItinerary> {
1444      let cmode = 0b1111;
1445    }
1446
1447 let isReMaterializable = 1 in {
1448 def FMOVvi_2S : NeonI_FMOV_impl<".2s", VPR64,  v2f32, fmov32_operand, 0b0, 0b0>;
1449 def FMOVvi_4S : NeonI_FMOV_impl<".4s", VPR128, v4f32, fmov32_operand, 0b1, 0b0>;
1450 def FMOVvi_2D : NeonI_FMOV_impl<".2d", VPR128, v2f64, fmov64_operand, 0b1, 0b1>;
1451 }
1452
1453 // Vector Shift (Immediate)
1454 // Immediate in [0, 63]
1455 def imm0_63 : Operand<i32> {
1456   let ParserMatchClass = uimm6_asmoperand;
1457 }
1458
1459 // Shift Right/Left Immediate - The immh:immb field of these shifts are encoded
1460 // as follows:
1461 //
1462 //    Offset    Encoding
1463 //     8        immh:immb<6:3> = '0001xxx', <imm> is encoded in immh:immb<2:0>
1464 //     16       immh:immb<6:4> = '001xxxx', <imm> is encoded in immh:immb<3:0>
1465 //     32       immh:immb<6:5> = '01xxxxx', <imm> is encoded in immh:immb<4:0>
1466 //     64       immh:immb<6>   = '1xxxxxx', <imm> is encoded in immh:immb<5:0>
1467 //
1468 // The shift right immediate amount, in the range 1 to element bits, is computed
1469 // as Offset - UInt(immh:immb).  The shift left immediate amount, in the range 0
1470 // to element bits - 1, is computed as UInt(immh:immb) - Offset.
1471
1472 class shr_imm_asmoperands<string OFFSET> : AsmOperandClass {
1473   let Name = "ShrImm" # OFFSET;
1474   let RenderMethod = "addImmOperands";
1475   let DiagnosticType = "ShrImm" # OFFSET;
1476 }
1477
1478 class shr_imm<string OFFSET> : Operand<i32> {
1479   let EncoderMethod = "getShiftRightImm" # OFFSET;
1480   let DecoderMethod = "DecodeShiftRightImm" # OFFSET;
1481   let ParserMatchClass =
1482     !cast<AsmOperandClass>("shr_imm" # OFFSET # "_asmoperand");
1483 }
1484
1485 def shr_imm8_asmoperand : shr_imm_asmoperands<"8">;
1486 def shr_imm16_asmoperand : shr_imm_asmoperands<"16">;
1487 def shr_imm32_asmoperand : shr_imm_asmoperands<"32">;
1488 def shr_imm64_asmoperand : shr_imm_asmoperands<"64">;
1489
1490 def shr_imm8 : shr_imm<"8">, ImmLeaf<i32, [{return Imm > 0 && Imm <= 8;}]>;
1491 def shr_imm16 : shr_imm<"16">, ImmLeaf<i32, [{return Imm > 0 && Imm <= 16;}]>;
1492 def shr_imm32 : shr_imm<"32">, ImmLeaf<i32, [{return Imm > 0 && Imm <= 32;}]>;
1493 def shr_imm64 : shr_imm<"64">, ImmLeaf<i32, [{return Imm > 0 && Imm <= 64;}]>;
1494
1495 class shl_imm_asmoperands<string OFFSET> : AsmOperandClass {
1496   let Name = "ShlImm" # OFFSET;
1497   let RenderMethod = "addImmOperands";
1498   let DiagnosticType = "ShlImm" # OFFSET;
1499 }
1500
1501 class shl_imm<string OFFSET> : Operand<i32> {
1502   let EncoderMethod = "getShiftLeftImm" # OFFSET;
1503   let DecoderMethod = "DecodeShiftLeftImm" # OFFSET;
1504   let ParserMatchClass =
1505     !cast<AsmOperandClass>("shl_imm" # OFFSET # "_asmoperand");
1506 }
1507
1508 def shl_imm8_asmoperand : shl_imm_asmoperands<"8">;
1509 def shl_imm16_asmoperand : shl_imm_asmoperands<"16">;
1510 def shl_imm32_asmoperand : shl_imm_asmoperands<"32">;
1511 def shl_imm64_asmoperand : shl_imm_asmoperands<"64">;
1512
1513 def shl_imm8 : shl_imm<"8">, ImmLeaf<i32, [{return Imm >= 0 && Imm < 8;}]>;
1514 def shl_imm16 : shl_imm<"16">, ImmLeaf<i32, [{return Imm >= 0 && Imm < 16;}]>;
1515 def shl_imm32 : shl_imm<"32">, ImmLeaf<i32, [{return Imm >= 0 && Imm < 32;}]>;
1516 def shl_imm64 : shl_imm<"64">, ImmLeaf<i32, [{return Imm >= 0 && Imm < 64;}]>;
1517
1518 class N2VShift<bit q, bit u, bits<5> opcode, string asmop, string T,
1519                RegisterOperand VPRC, ValueType Ty, Operand ImmTy, SDNode OpNode>
1520   : NeonI_2VShiftImm<q, u, opcode,
1521                      (outs VPRC:$Rd), (ins VPRC:$Rn, ImmTy:$Imm),
1522                      asmop # "\t$Rd." # T # ", $Rn." # T # ", $Imm",
1523                      [(set (Ty VPRC:$Rd),
1524                         (Ty (OpNode (Ty VPRC:$Rn),
1525                           (Ty (Neon_vdup (i32 ImmTy:$Imm))))))],
1526                      NoItinerary>;
1527
1528 multiclass NeonI_N2VShL<bit u, bits<5> opcode, string asmop> {
1529   // 64-bit vector types.
1530   def _8B : N2VShift<0b0, u, opcode, asmop, "8b", VPR64, v8i8, shl_imm8, shl> {
1531     let Inst{22-19} = 0b0001;  // immh:immb = 0001xxx
1532   }
1533
1534   def _4H : N2VShift<0b0, u, opcode, asmop, "4h", VPR64, v4i16, shl_imm16, shl> {
1535     let Inst{22-20} = 0b001;   // immh:immb = 001xxxx
1536   }
1537
1538   def _2S : N2VShift<0b0, u, opcode, asmop, "2s", VPR64, v2i32, shl_imm32, shl> {
1539     let Inst{22-21} = 0b01;    // immh:immb = 01xxxxx
1540   }
1541
1542   // 128-bit vector types.
1543   def _16B : N2VShift<0b1, u, opcode, asmop, "16b", VPR128, v16i8, shl_imm8, shl> {
1544     let Inst{22-19} = 0b0001;  // immh:immb = 0001xxx
1545   }
1546
1547   def _8H : N2VShift<0b1, u, opcode, asmop, "8h", VPR128, v8i16, shl_imm16, shl> {
1548     let Inst{22-20} = 0b001;   // immh:immb = 001xxxx
1549   }
1550
1551   def _4S : N2VShift<0b1, u, opcode, asmop, "4s", VPR128, v4i32, shl_imm32, shl> {
1552     let Inst{22-21} = 0b01;    // immh:immb = 01xxxxx
1553   }
1554
1555   def _2D : N2VShift<0b1, u, opcode, asmop, "2d", VPR128, v2i64, shl_imm64, shl> {
1556     let Inst{22} = 0b1;        // immh:immb = 1xxxxxx
1557   }
1558 }
1559
1560 multiclass NeonI_N2VShR<bit u, bits<5> opcode, string asmop, SDNode OpNode> {
1561   def _8B : N2VShift<0b0, u, opcode, asmop, "8b", VPR64, v8i8, shr_imm8,
1562                      OpNode> {
1563     let Inst{22-19} = 0b0001;
1564   }
1565
1566   def _4H : N2VShift<0b0, u, opcode, asmop, "4h", VPR64, v4i16, shr_imm16,
1567                      OpNode> {
1568     let Inst{22-20} = 0b001;
1569   }
1570
1571   def _2S : N2VShift<0b0, u, opcode, asmop, "2s", VPR64, v2i32, shr_imm32,
1572                      OpNode> {
1573      let Inst{22-21} = 0b01;
1574   }
1575
1576   def _16B : N2VShift<0b1, u, opcode, asmop, "16b", VPR128, v16i8, shr_imm8,
1577                       OpNode> {
1578                       let Inst{22-19} = 0b0001;
1579                     }
1580
1581   def _8H : N2VShift<0b1, u, opcode, asmop, "8h", VPR128, v8i16, shr_imm16,
1582                      OpNode> {
1583                      let Inst{22-20} = 0b001;
1584                     }
1585
1586   def _4S : N2VShift<0b1, u, opcode, asmop, "4s", VPR128, v4i32, shr_imm32,
1587                      OpNode> {
1588                       let Inst{22-21} = 0b01;
1589                     }
1590
1591   def _2D : N2VShift<0b1, u, opcode, asmop, "2d", VPR128, v2i64, shr_imm64,
1592                      OpNode> {
1593                       let Inst{22} = 0b1;
1594                     }
1595 }
1596
1597 // Shift left
1598 defm SHLvvi : NeonI_N2VShL<0b0, 0b01010, "shl">;
1599
1600 // Shift right
1601 defm SSHRvvi : NeonI_N2VShR<0b0, 0b00000, "sshr", sra>;
1602 defm USHRvvi : NeonI_N2VShR<0b1, 0b00000, "ushr", srl>;
1603
1604 def Neon_High16B : PatFrag<(ops node:$in),
1605                            (extract_subvector (v16i8 node:$in), (iPTR 8))>;
1606 def Neon_High8H  : PatFrag<(ops node:$in),
1607                            (extract_subvector (v8i16 node:$in), (iPTR 4))>;
1608 def Neon_High4S  : PatFrag<(ops node:$in),
1609                            (extract_subvector (v4i32 node:$in), (iPTR 2))>;
1610 def Neon_High2D  : PatFrag<(ops node:$in),
1611                            (extract_subvector (v2i64 node:$in), (iPTR 1))>;
1612 def Neon_High4float : PatFrag<(ops node:$in),
1613                                (extract_subvector (v4f32 node:$in), (iPTR 2))>;
1614 def Neon_High2double : PatFrag<(ops node:$in),
1615                                (extract_subvector (v2f64 node:$in), (iPTR 1))>;
1616
1617 def Neon_Low16B : PatFrag<(ops node:$in),
1618                           (v8i8 (extract_subvector (v16i8 node:$in),
1619                                                    (iPTR 0)))>;
1620 def Neon_Low8H : PatFrag<(ops node:$in),
1621                          (v4i16 (extract_subvector (v8i16 node:$in),
1622                                                    (iPTR 0)))>;
1623 def Neon_Low4S : PatFrag<(ops node:$in),
1624                          (v2i32 (extract_subvector (v4i32 node:$in),
1625                                                    (iPTR 0)))>;
1626 def Neon_Low2D : PatFrag<(ops node:$in),
1627                          (v1i64 (extract_subvector (v2i64 node:$in),
1628                                                    (iPTR 0)))>;
1629 def Neon_Low4float : PatFrag<(ops node:$in),
1630                              (v2f32 (extract_subvector (v4f32 node:$in),
1631                                                        (iPTR 0)))>;
1632 def Neon_Low2double : PatFrag<(ops node:$in),
1633                               (v1f64 (extract_subvector (v2f64 node:$in),
1634                                                         (iPTR 0)))>;
1635
1636 class N2VShiftLong<bit q, bit u, bits<5> opcode, string asmop, string DestT,
1637                    string SrcT, ValueType DestTy, ValueType SrcTy,
1638                    Operand ImmTy, SDPatternOperator ExtOp>
1639   : NeonI_2VShiftImm<q, u, opcode, (outs VPR128:$Rd),
1640                      (ins VPR64:$Rn, ImmTy:$Imm),
1641                      asmop # "\t$Rd." # DestT # ", $Rn." # SrcT # ", $Imm",
1642                      [(set (DestTy VPR128:$Rd),
1643                         (DestTy (shl
1644                           (DestTy (ExtOp (SrcTy VPR64:$Rn))),
1645                             (DestTy (Neon_vdup (i32 ImmTy:$Imm))))))],
1646                      NoItinerary>;
1647
1648 class N2VShiftLongHigh<bit q, bit u, bits<5> opcode, string asmop, string DestT,
1649                        string SrcT, ValueType DestTy, ValueType SrcTy,
1650                        int StartIndex, Operand ImmTy,
1651                        SDPatternOperator ExtOp, PatFrag getTop>
1652   : NeonI_2VShiftImm<q, u, opcode, (outs VPR128:$Rd),
1653                      (ins VPR128:$Rn, ImmTy:$Imm),
1654                      asmop # "2\t$Rd." # DestT # ", $Rn." # SrcT # ", $Imm",
1655                      [(set (DestTy VPR128:$Rd),
1656                         (DestTy (shl
1657                           (DestTy (ExtOp
1658                             (SrcTy (getTop VPR128:$Rn)))),
1659                               (DestTy (Neon_vdup (i32 ImmTy:$Imm))))))],
1660                      NoItinerary>;
1661
1662 multiclass NeonI_N2VShLL<string prefix, bit u, bits<5> opcode, string asmop,
1663                          SDNode ExtOp> {
1664   // 64-bit vector types.
1665   def _8B : N2VShiftLong<0b0, u, opcode, asmop, "8h", "8b", v8i16, v8i8,
1666                          shl_imm8, ExtOp> {
1667     let Inst{22-19} = 0b0001;  // immh:immb = 0001xxx
1668   }
1669
1670   def _4H : N2VShiftLong<0b0, u, opcode, asmop, "4s", "4h", v4i32, v4i16,
1671                          shl_imm16, ExtOp> {
1672     let Inst{22-20} = 0b001;   // immh:immb = 001xxxx
1673   }
1674
1675   def _2S : N2VShiftLong<0b0, u, opcode, asmop, "2d", "2s", v2i64, v2i32,
1676                          shl_imm32, ExtOp> {
1677     let Inst{22-21} = 0b01;    // immh:immb = 01xxxxx
1678   }
1679
1680   // 128-bit vector types
1681   def _16B : N2VShiftLongHigh<0b1, u, opcode, asmop, "8h", "16b", v8i16, v8i8,
1682                               8, shl_imm8, ExtOp, Neon_High16B> {
1683     let Inst{22-19} = 0b0001;  // immh:immb = 0001xxx
1684   }
1685
1686   def _8H : N2VShiftLongHigh<0b1, u, opcode, asmop, "4s", "8h", v4i32, v4i16,
1687                              4, shl_imm16, ExtOp, Neon_High8H> {
1688     let Inst{22-20} = 0b001;   // immh:immb = 001xxxx
1689   }
1690
1691   def _4S : N2VShiftLongHigh<0b1, u, opcode, asmop, "2d", "4s", v2i64, v2i32,
1692                              2, shl_imm32, ExtOp, Neon_High4S> {
1693     let Inst{22-21} = 0b01;    // immh:immb = 01xxxxx
1694   }
1695
1696   // Use other patterns to match when the immediate is 0.
1697   def : Pat<(v8i16 (ExtOp (v8i8 VPR64:$Rn))),
1698             (!cast<Instruction>(prefix # "_8B") VPR64:$Rn, 0)>;
1699
1700   def : Pat<(v4i32 (ExtOp (v4i16 VPR64:$Rn))),
1701             (!cast<Instruction>(prefix # "_4H") VPR64:$Rn, 0)>;
1702
1703   def : Pat<(v2i64 (ExtOp (v2i32 VPR64:$Rn))),
1704             (!cast<Instruction>(prefix # "_2S") VPR64:$Rn, 0)>;
1705
1706   def : Pat<(v8i16 (ExtOp (v8i8 (Neon_High16B VPR128:$Rn)))),
1707             (!cast<Instruction>(prefix # "_16B") VPR128:$Rn, 0)>;
1708
1709   def : Pat<(v4i32 (ExtOp (v4i16 (Neon_High8H VPR128:$Rn)))),
1710             (!cast<Instruction>(prefix # "_8H") VPR128:$Rn, 0)>;
1711
1712   def : Pat<(v2i64 (ExtOp (v2i32 (Neon_High4S VPR128:$Rn)))),
1713             (!cast<Instruction>(prefix # "_4S") VPR128:$Rn, 0)>;
1714 }
1715
1716 // Shift left long
1717 defm SSHLLvvi : NeonI_N2VShLL<"SSHLLvvi", 0b0, 0b10100, "sshll", sext>;
1718 defm USHLLvvi : NeonI_N2VShLL<"USHLLvvi", 0b1, 0b10100, "ushll", zext>;
1719
1720 // Rounding/Saturating shift
1721 class N2VShift_RQ<bit q, bit u, bits<5> opcode, string asmop, string T,
1722                   RegisterOperand VPRC, ValueType Ty, Operand ImmTy,
1723                   SDPatternOperator OpNode>
1724   : NeonI_2VShiftImm<q, u, opcode,
1725                      (outs VPRC:$Rd), (ins VPRC:$Rn, ImmTy:$Imm),
1726                      asmop # "\t$Rd." # T # ", $Rn." # T # ", $Imm",
1727                      [(set (Ty VPRC:$Rd), (Ty (OpNode (Ty VPRC:$Rn),
1728                         (i32 ImmTy:$Imm))))],
1729                      NoItinerary>;
1730
1731 // shift right (vector by immediate)
1732 multiclass NeonI_N2VShR_RQ<bit u, bits<5> opcode, string asmop,
1733                            SDPatternOperator OpNode> {
1734   def _8B  : N2VShift_RQ<0b0, u, opcode, asmop, "8b", VPR64, v8i8, shr_imm8,
1735                          OpNode> {
1736     let Inst{22-19} = 0b0001;
1737   }
1738
1739   def _4H  : N2VShift_RQ<0b0, u, opcode, asmop, "4h", VPR64, v4i16, shr_imm16,
1740                          OpNode> {
1741     let Inst{22-20} = 0b001;
1742   }
1743
1744   def _2S  : N2VShift_RQ<0b0, u, opcode, asmop, "2s", VPR64, v2i32, shr_imm32,
1745                          OpNode> {
1746     let Inst{22-21} = 0b01;
1747   }
1748
1749   def _16B : N2VShift_RQ<0b1, u, opcode, asmop, "16b", VPR128, v16i8, shr_imm8,
1750                          OpNode> {
1751     let Inst{22-19} = 0b0001;
1752   }
1753
1754   def _8H : N2VShift_RQ<0b1, u, opcode, asmop, "8h", VPR128, v8i16, shr_imm16,
1755                         OpNode> {
1756     let Inst{22-20} = 0b001;
1757   }
1758
1759   def _4S : N2VShift_RQ<0b1, u, opcode, asmop, "4s", VPR128, v4i32, shr_imm32,
1760                         OpNode> {
1761     let Inst{22-21} = 0b01;
1762   }
1763
1764   def _2D : N2VShift_RQ<0b1, u, opcode, asmop, "2d", VPR128, v2i64, shr_imm64,
1765                         OpNode> {
1766     let Inst{22} = 0b1;
1767   }
1768 }
1769
1770 multiclass NeonI_N2VShL_Q<bit u, bits<5> opcode, string asmop,
1771                           SDPatternOperator OpNode> {
1772   // 64-bit vector types.
1773   def _8B : N2VShift_RQ<0b0, u, opcode, asmop, "8b", VPR64, v8i8, shl_imm8,
1774                         OpNode> {
1775     let Inst{22-19} = 0b0001;
1776   }
1777
1778   def _4H : N2VShift_RQ<0b0, u, opcode, asmop, "4h", VPR64, v4i16, shl_imm16,
1779                         OpNode> {
1780     let Inst{22-20} = 0b001;
1781   }
1782
1783   def _2S : N2VShift_RQ<0b0, u, opcode, asmop, "2s", VPR64, v2i32, shl_imm32,
1784                         OpNode> {
1785     let Inst{22-21} = 0b01;
1786   }
1787
1788   // 128-bit vector types.
1789   def _16B : N2VShift_RQ<0b1, u, opcode, asmop, "16b", VPR128, v16i8, shl_imm8,
1790                          OpNode> {
1791     let Inst{22-19} = 0b0001;
1792   }
1793
1794   def _8H : N2VShift_RQ<0b1, u, opcode, asmop, "8h", VPR128, v8i16, shl_imm16,
1795                         OpNode> {
1796     let Inst{22-20} = 0b001;
1797   }
1798
1799   def _4S : N2VShift_RQ<0b1, u, opcode, asmop, "4s", VPR128, v4i32, shl_imm32,
1800                         OpNode> {
1801     let Inst{22-21} = 0b01;
1802   }
1803
1804   def _2D : N2VShift_RQ<0b1, u, opcode, asmop, "2d", VPR128, v2i64, shl_imm64,
1805                         OpNode> {
1806     let Inst{22} = 0b1;
1807   }
1808 }
1809
1810 // Rounding shift right
1811 defm SRSHRvvi : NeonI_N2VShR_RQ<0b0, 0b00100, "srshr",
1812                                 int_aarch64_neon_vsrshr>;
1813 defm URSHRvvi : NeonI_N2VShR_RQ<0b1, 0b00100, "urshr",
1814                                 int_aarch64_neon_vurshr>;
1815
1816 // Saturating shift left unsigned
1817 defm SQSHLUvvi : NeonI_N2VShL_Q<0b1, 0b01100, "sqshlu", int_aarch64_neon_vsqshlu>;
1818
1819 // Saturating shift left
1820 defm SQSHLvvi : NeonI_N2VShL_Q<0b0, 0b01110, "sqshl", Neon_sqrshlImm>;
1821 defm UQSHLvvi : NeonI_N2VShL_Q<0b1, 0b01110, "uqshl", Neon_uqrshlImm>;
1822
1823 class N2VShiftAdd<bit q, bit u, bits<5> opcode, string asmop, string T,
1824                   RegisterOperand VPRC, ValueType Ty, Operand ImmTy,
1825                   SDNode OpNode>
1826   : NeonI_2VShiftImm<q, u, opcode,
1827            (outs VPRC:$Rd), (ins VPRC:$src, VPRC:$Rn, ImmTy:$Imm),
1828            asmop # "\t$Rd." # T # ", $Rn." # T # ", $Imm",
1829            [(set (Ty VPRC:$Rd), (Ty (add (Ty VPRC:$src),
1830               (Ty (OpNode (Ty VPRC:$Rn),
1831                 (Ty (Neon_vdup (i32 ImmTy:$Imm))))))))],
1832            NoItinerary> {
1833   let Constraints = "$src = $Rd";
1834 }
1835
1836 // Shift Right accumulate
1837 multiclass NeonI_N2VShRAdd<bit u, bits<5> opcode, string asmop, SDNode OpNode> {
1838   def _8B : N2VShiftAdd<0b0, u, opcode, asmop, "8b", VPR64, v8i8, shr_imm8,
1839                         OpNode> {
1840     let Inst{22-19} = 0b0001;
1841   }
1842
1843   def _4H : N2VShiftAdd<0b0, u, opcode, asmop, "4h", VPR64, v4i16, shr_imm16,
1844                         OpNode> {
1845     let Inst{22-20} = 0b001;
1846   }
1847
1848   def _2S : N2VShiftAdd<0b0, u, opcode, asmop, "2s", VPR64, v2i32, shr_imm32,
1849                         OpNode> {
1850     let Inst{22-21} = 0b01;
1851   }
1852
1853   def _16B : N2VShiftAdd<0b1, u, opcode, asmop, "16b", VPR128, v16i8, shr_imm8,
1854                          OpNode> {
1855     let Inst{22-19} = 0b0001;
1856   }
1857
1858   def _8H : N2VShiftAdd<0b1, u, opcode, asmop, "8h", VPR128, v8i16, shr_imm16,
1859                         OpNode> {
1860     let Inst{22-20} = 0b001;
1861   }
1862
1863   def _4S : N2VShiftAdd<0b1, u, opcode, asmop, "4s", VPR128, v4i32, shr_imm32,
1864                         OpNode> {
1865     let Inst{22-21} = 0b01;
1866   }
1867
1868   def _2D : N2VShiftAdd<0b1, u, opcode, asmop, "2d", VPR128, v2i64, shr_imm64,
1869                         OpNode> {
1870     let Inst{22} = 0b1;
1871   }
1872 }
1873
1874 // Shift right and accumulate
1875 defm SSRAvvi    : NeonI_N2VShRAdd<0, 0b00010, "ssra", sra>;
1876 defm USRAvvi    : NeonI_N2VShRAdd<1, 0b00010, "usra", srl>;
1877
1878 // Rounding shift accumulate
1879 class N2VShiftAdd_R<bit q, bit u, bits<5> opcode, string asmop, string T,
1880                     RegisterOperand VPRC, ValueType Ty, Operand ImmTy,
1881                     SDPatternOperator OpNode>
1882   : NeonI_2VShiftImm<q, u, opcode,
1883                      (outs VPRC:$Rd), (ins VPRC:$src, VPRC:$Rn, ImmTy:$Imm),
1884                      asmop # "\t$Rd." # T # ", $Rn." # T # ", $Imm",
1885                      [(set (Ty VPRC:$Rd), (Ty (add (Ty VPRC:$src),
1886                         (Ty (OpNode (Ty VPRC:$Rn), (i32 ImmTy:$Imm))))))],
1887                      NoItinerary> {
1888   let Constraints = "$src = $Rd";
1889 }
1890
1891 multiclass NeonI_N2VShRAdd_R<bit u, bits<5> opcode, string asmop,
1892                              SDPatternOperator OpNode> {
1893   def _8B : N2VShiftAdd_R<0b0, u, opcode, asmop, "8b", VPR64, v8i8, shr_imm8,
1894                           OpNode> {
1895     let Inst{22-19} = 0b0001;
1896   }
1897
1898   def _4H : N2VShiftAdd_R<0b0, u, opcode, asmop, "4h", VPR64, v4i16, shr_imm16,
1899                           OpNode> {
1900     let Inst{22-20} = 0b001;
1901   }
1902
1903   def _2S : N2VShiftAdd_R<0b0, u, opcode, asmop, "2s", VPR64, v2i32, shr_imm32,
1904                           OpNode> {
1905     let Inst{22-21} = 0b01;
1906   }
1907
1908   def _16B : N2VShiftAdd_R<0b1, u, opcode, asmop, "16b", VPR128, v16i8, shr_imm8,
1909                            OpNode> {
1910     let Inst{22-19} = 0b0001;
1911   }
1912
1913   def _8H : N2VShiftAdd_R<0b1, u, opcode, asmop, "8h", VPR128, v8i16, shr_imm16,
1914                           OpNode> {
1915     let Inst{22-20} = 0b001;
1916   }
1917
1918   def _4S : N2VShiftAdd_R<0b1, u, opcode, asmop, "4s", VPR128, v4i32, shr_imm32,
1919                           OpNode> {
1920     let Inst{22-21} = 0b01;
1921   }
1922
1923   def _2D : N2VShiftAdd_R<0b1, u, opcode, asmop, "2d", VPR128, v2i64, shr_imm64,
1924                           OpNode> {
1925     let Inst{22} = 0b1;
1926   }
1927 }
1928
1929 // Rounding shift right and accumulate
1930 defm SRSRAvvi : NeonI_N2VShRAdd_R<0, 0b00110, "srsra", int_aarch64_neon_vsrshr>;
1931 defm URSRAvvi : NeonI_N2VShRAdd_R<1, 0b00110, "ursra", int_aarch64_neon_vurshr>;
1932
1933 // Shift insert by immediate
1934 class N2VShiftIns<bit q, bit u, bits<5> opcode, string asmop, string T,
1935                   RegisterOperand VPRC, ValueType Ty, Operand ImmTy,
1936                   SDPatternOperator OpNode>
1937     : NeonI_2VShiftImm<q, u, opcode,
1938            (outs VPRC:$Rd), (ins VPRC:$src, VPRC:$Rn, ImmTy:$Imm),
1939            asmop # "\t$Rd." # T # ", $Rn." # T # ", $Imm",
1940            [(set (Ty VPRC:$Rd), (Ty (OpNode (Ty VPRC:$src), (Ty VPRC:$Rn),
1941              (i32 ImmTy:$Imm))))],
1942            NoItinerary> {
1943   let Constraints = "$src = $Rd";
1944 }
1945
1946 // shift left insert (vector by immediate)
1947 multiclass NeonI_N2VShLIns<bit u, bits<5> opcode, string asmop> {
1948   def _8B : N2VShiftIns<0b0, u, opcode, asmop, "8b", VPR64, v8i8, shl_imm8,
1949                         int_aarch64_neon_vsli> {
1950     let Inst{22-19} = 0b0001;
1951   }
1952
1953   def _4H : N2VShiftIns<0b0, u, opcode, asmop, "4h", VPR64, v4i16, shl_imm16,
1954                         int_aarch64_neon_vsli> {
1955     let Inst{22-20} = 0b001;
1956   }
1957
1958   def _2S : N2VShiftIns<0b0, u, opcode, asmop, "2s", VPR64, v2i32, shl_imm32,
1959                         int_aarch64_neon_vsli> {
1960     let Inst{22-21} = 0b01;
1961   }
1962
1963     // 128-bit vector types
1964   def _16B : N2VShiftIns<0b1, u, opcode, asmop, "16b", VPR128, v16i8, shl_imm8,
1965                          int_aarch64_neon_vsli> {
1966     let Inst{22-19} = 0b0001;
1967   }
1968
1969   def _8H : N2VShiftIns<0b1, u, opcode, asmop, "8h", VPR128, v8i16, shl_imm16,
1970                         int_aarch64_neon_vsli> {
1971     let Inst{22-20} = 0b001;
1972   }
1973
1974   def _4S : N2VShiftIns<0b1, u, opcode, asmop, "4s", VPR128, v4i32, shl_imm32,
1975                         int_aarch64_neon_vsli> {
1976     let Inst{22-21} = 0b01;
1977   }
1978
1979   def _2D : N2VShiftIns<0b1, u, opcode, asmop, "2d", VPR128, v2i64, shl_imm64,
1980                         int_aarch64_neon_vsli> {
1981     let Inst{22} = 0b1;
1982   }
1983 }
1984
1985 // shift right insert (vector by immediate)
1986 multiclass NeonI_N2VShRIns<bit u, bits<5> opcode, string asmop> {
1987     // 64-bit vector types.
1988   def _8B : N2VShiftIns<0b0, u, opcode, asmop, "8b", VPR64, v8i8, shr_imm8,
1989                         int_aarch64_neon_vsri> {
1990     let Inst{22-19} = 0b0001;
1991   }
1992
1993   def _4H : N2VShiftIns<0b0, u, opcode, asmop, "4h", VPR64, v4i16, shr_imm16,
1994                         int_aarch64_neon_vsri> {
1995     let Inst{22-20} = 0b001;
1996   }
1997
1998   def _2S : N2VShiftIns<0b0, u, opcode, asmop, "2s", VPR64, v2i32, shr_imm32,
1999                         int_aarch64_neon_vsri> {
2000     let Inst{22-21} = 0b01;
2001   }
2002
2003     // 128-bit vector types
2004   def _16B : N2VShiftIns<0b1, u, opcode, asmop, "16b", VPR128, v16i8, shr_imm8,
2005                          int_aarch64_neon_vsri> {
2006     let Inst{22-19} = 0b0001;
2007   }
2008
2009   def _8H : N2VShiftIns<0b1, u, opcode, asmop, "8h", VPR128, v8i16, shr_imm16,
2010                         int_aarch64_neon_vsri> {
2011     let Inst{22-20} = 0b001;
2012   }
2013
2014   def _4S : N2VShiftIns<0b1, u, opcode, asmop, "4s", VPR128, v4i32, shr_imm32,
2015                         int_aarch64_neon_vsri> {
2016     let Inst{22-21} = 0b01;
2017   }
2018
2019   def _2D : N2VShiftIns<0b1, u, opcode, asmop, "2d", VPR128, v2i64, shr_imm64,
2020                         int_aarch64_neon_vsri> {
2021     let Inst{22} = 0b1;
2022   }
2023 }
2024
2025 // Shift left and insert
2026 defm SLIvvi   : NeonI_N2VShLIns<0b1, 0b01010, "sli">;
2027
2028 // Shift right and insert
2029 defm SRIvvi   : NeonI_N2VShRIns<0b1, 0b01000, "sri">;
2030
2031 class N2VShR_Narrow<bit q, bit u, bits<5> opcode, string asmop, string DestT,
2032                     string SrcT, Operand ImmTy>
2033   : NeonI_2VShiftImm<q, u, opcode,
2034                      (outs VPR64:$Rd), (ins VPR128:$Rn, ImmTy:$Imm),
2035                      asmop # "\t$Rd." # DestT # ", $Rn." # SrcT # ", $Imm",
2036                      [], NoItinerary>;
2037
2038 class N2VShR_Narrow_Hi<bit q, bit u, bits<5> opcode, string asmop, string DestT,
2039                        string SrcT, Operand ImmTy>
2040   : NeonI_2VShiftImm<q, u, opcode, (outs VPR128:$Rd),
2041                      (ins VPR128:$src, VPR128:$Rn, ImmTy:$Imm),
2042                      asmop # "\t$Rd." # DestT # ", $Rn." # SrcT # ", $Imm",
2043                      [], NoItinerary> {
2044   let Constraints = "$src = $Rd";
2045 }
2046
2047 // left long shift by immediate
2048 multiclass NeonI_N2VShR_Narrow<bit u, bits<5> opcode, string asmop> {
2049   def _8B : N2VShR_Narrow<0b0, u, opcode, asmop, "8b", "8h", shr_imm8> {
2050     let Inst{22-19} = 0b0001;
2051   }
2052
2053   def _4H : N2VShR_Narrow<0b0, u, opcode, asmop, "4h", "4s", shr_imm16> {
2054     let Inst{22-20} = 0b001;
2055   }
2056
2057   def _2S : N2VShR_Narrow<0b0, u, opcode, asmop, "2s", "2d", shr_imm32> {
2058     let Inst{22-21} = 0b01;
2059   }
2060
2061   // Shift Narrow High
2062   def _16B : N2VShR_Narrow_Hi<0b1, u, opcode, asmop # "2", "16b", "8h",
2063                               shr_imm8> {
2064     let Inst{22-19} = 0b0001;
2065   }
2066
2067   def _8H : N2VShR_Narrow_Hi<0b1, u, opcode, asmop # "2", "8h", "4s",
2068                              shr_imm16> {
2069     let Inst{22-20} = 0b001;
2070   }
2071
2072   def _4S : N2VShR_Narrow_Hi<0b1, u, opcode, asmop # "2", "4s", "2d",
2073                              shr_imm32> {
2074     let Inst{22-21} = 0b01;
2075   }
2076 }
2077
2078 // Shift right narrow
2079 defm SHRNvvi : NeonI_N2VShR_Narrow<0b0, 0b10000, "shrn">;
2080
2081 // Shift right narrow (prefix Q is saturating, prefix R is rounding)
2082 defm QSHRUNvvi :NeonI_N2VShR_Narrow<0b1, 0b10000, "sqshrun">;
2083 defm RSHRNvvi : NeonI_N2VShR_Narrow<0b0, 0b10001, "rshrn">;
2084 defm QRSHRUNvvi : NeonI_N2VShR_Narrow<0b1, 0b10001, "sqrshrun">;
2085 defm SQSHRNvvi : NeonI_N2VShR_Narrow<0b0, 0b10010, "sqshrn">;
2086 defm UQSHRNvvi : NeonI_N2VShR_Narrow<0b1, 0b10010, "uqshrn">;
2087 defm SQRSHRNvvi : NeonI_N2VShR_Narrow<0b0, 0b10011, "sqrshrn">;
2088 defm UQRSHRNvvi : NeonI_N2VShR_Narrow<0b1, 0b10011, "uqrshrn">;
2089
2090 def Neon_combine_2D : PatFrag<(ops node:$Rm, node:$Rn),
2091                               (v2i64 (concat_vectors (v1i64 node:$Rm),
2092                                                      (v1i64 node:$Rn)))>;
2093 def Neon_combine_8H : PatFrag<(ops node:$Rm, node:$Rn),
2094                               (v8i16 (concat_vectors (v4i16 node:$Rm),
2095                                                      (v4i16 node:$Rn)))>;
2096 def Neon_combine_4S : PatFrag<(ops node:$Rm, node:$Rn),
2097                               (v4i32 (concat_vectors (v2i32 node:$Rm),
2098                                                      (v2i32 node:$Rn)))>;
2099 def Neon_combine_4f : PatFrag<(ops node:$Rm, node:$Rn),
2100                               (v4f32 (concat_vectors (v2f32 node:$Rm),
2101                                                      (v2f32 node:$Rn)))>;
2102 def Neon_combine_2d : PatFrag<(ops node:$Rm, node:$Rn),
2103                               (v2f64 (concat_vectors (v1f64 node:$Rm),
2104                                                      (v1f64 node:$Rn)))>;
2105
2106 def Neon_lshrImm8H : PatFrag<(ops node:$lhs, node:$rhs),
2107                              (v8i16 (srl (v8i16 node:$lhs),
2108                                (v8i16 (Neon_vdup (i32 node:$rhs)))))>;
2109 def Neon_lshrImm4S : PatFrag<(ops node:$lhs, node:$rhs),
2110                              (v4i32 (srl (v4i32 node:$lhs),
2111                                (v4i32 (Neon_vdup (i32 node:$rhs)))))>;
2112 def Neon_lshrImm2D : PatFrag<(ops node:$lhs, node:$rhs),
2113                              (v2i64 (srl (v2i64 node:$lhs),
2114                                (v2i64 (Neon_vdup (i32 node:$rhs)))))>;
2115 def Neon_ashrImm8H : PatFrag<(ops node:$lhs, node:$rhs),
2116                              (v8i16 (sra (v8i16 node:$lhs),
2117                                (v8i16 (Neon_vdup (i32 node:$rhs)))))>;
2118 def Neon_ashrImm4S : PatFrag<(ops node:$lhs, node:$rhs),
2119                              (v4i32 (sra (v4i32 node:$lhs),
2120                                (v4i32 (Neon_vdup (i32 node:$rhs)))))>;
2121 def Neon_ashrImm2D : PatFrag<(ops node:$lhs, node:$rhs),
2122                              (v2i64 (sra (v2i64 node:$lhs),
2123                                (v2i64 (Neon_vdup (i32 node:$rhs)))))>;
2124
2125 // Normal shift right narrow is matched by IR (srl/sra, trunc, concat_vectors)
2126 multiclass Neon_shiftNarrow_patterns<string shr> {
2127   def : Pat<(v8i8 (trunc (!cast<PatFrag>("Neon_" # shr # "Imm8H") VPR128:$Rn,
2128               (i32 shr_imm8:$Imm)))),
2129             (SHRNvvi_8B VPR128:$Rn, imm:$Imm)>;
2130   def : Pat<(v4i16 (trunc (!cast<PatFrag>("Neon_" # shr # "Imm4S") VPR128:$Rn,
2131               (i32 shr_imm16:$Imm)))),
2132             (SHRNvvi_4H VPR128:$Rn, imm:$Imm)>;
2133   def : Pat<(v2i32 (trunc (!cast<PatFrag>("Neon_" # shr # "Imm2D") VPR128:$Rn,
2134               (i32 shr_imm32:$Imm)))),
2135             (SHRNvvi_2S VPR128:$Rn, imm:$Imm)>;
2136
2137   def : Pat<(Neon_combine_2D (v1i64 VPR64:$src), (v1i64 (bitconvert
2138               (v8i8 (trunc (!cast<PatFrag>("Neon_" # shr # "Imm8H")
2139                 VPR128:$Rn, (i32 shr_imm8:$Imm))))))),
2140             (SHRNvvi_16B (v2i64 (SUBREG_TO_REG (i64 0), VPR64:$src, sub_64)),
2141                          VPR128:$Rn, imm:$Imm)>;
2142   def : Pat<(Neon_combine_2D (v1i64 VPR64:$src), (v1i64 (bitconvert
2143               (v4i16 (trunc (!cast<PatFrag>("Neon_" # shr # "Imm4S")
2144                 VPR128:$Rn, (i32 shr_imm16:$Imm))))))),
2145             (SHRNvvi_8H (SUBREG_TO_REG (i64 0), VPR64:$src, sub_64),
2146                         VPR128:$Rn, imm:$Imm)>;
2147   def : Pat<(Neon_combine_2D (v1i64 VPR64:$src), (v1i64 (bitconvert
2148               (v2i32 (trunc (!cast<PatFrag>("Neon_" # shr # "Imm2D")
2149                 VPR128:$Rn, (i32 shr_imm32:$Imm))))))),
2150             (SHRNvvi_4S (SUBREG_TO_REG (i64 0), VPR64:$src, sub_64),
2151                         VPR128:$Rn, imm:$Imm)>;
2152 }
2153
2154 multiclass Neon_shiftNarrow_QR_patterns<SDPatternOperator op, string prefix> {
2155   def : Pat<(v8i8 (op (v8i16 VPR128:$Rn), shr_imm8:$Imm)),
2156             (!cast<Instruction>(prefix # "_8B") VPR128:$Rn, imm:$Imm)>;
2157   def : Pat<(v4i16 (op (v4i32 VPR128:$Rn), shr_imm16:$Imm)),
2158             (!cast<Instruction>(prefix # "_4H") VPR128:$Rn, imm:$Imm)>;
2159   def : Pat<(v2i32 (op (v2i64 VPR128:$Rn), shr_imm32:$Imm)),
2160             (!cast<Instruction>(prefix # "_2S") VPR128:$Rn, imm:$Imm)>;
2161
2162   def : Pat<(Neon_combine_2D (v1i64 VPR64:$src),
2163                 (v1i64 (bitconvert (v8i8
2164                     (op (v8i16 VPR128:$Rn), shr_imm8:$Imm))))),
2165             (!cast<Instruction>(prefix # "_16B")
2166                 (SUBREG_TO_REG (i64 0), VPR64:$src, sub_64),
2167                 VPR128:$Rn, imm:$Imm)>;
2168   def : Pat<(Neon_combine_2D (v1i64 VPR64:$src),
2169                 (v1i64 (bitconvert (v4i16
2170                     (op (v4i32 VPR128:$Rn), shr_imm16:$Imm))))),
2171             (!cast<Instruction>(prefix # "_8H")
2172                 (SUBREG_TO_REG (i64 0), VPR64:$src, sub_64),
2173                 VPR128:$Rn, imm:$Imm)>;
2174   def : Pat<(Neon_combine_2D (v1i64 VPR64:$src),
2175                 (v1i64 (bitconvert (v2i32
2176                     (op (v2i64 VPR128:$Rn), shr_imm32:$Imm))))),
2177             (!cast<Instruction>(prefix # "_4S")
2178                   (SUBREG_TO_REG (i64 0), VPR64:$src, sub_64),
2179                   VPR128:$Rn, imm:$Imm)>;
2180 }
2181
2182 defm : Neon_shiftNarrow_patterns<"lshr">;
2183 defm : Neon_shiftNarrow_patterns<"ashr">;
2184
2185 defm : Neon_shiftNarrow_QR_patterns<int_aarch64_neon_vsqshrun, "QSHRUNvvi">;
2186 defm : Neon_shiftNarrow_QR_patterns<int_aarch64_neon_vrshrn, "RSHRNvvi">;
2187 defm : Neon_shiftNarrow_QR_patterns<int_aarch64_neon_vsqrshrun, "QRSHRUNvvi">;
2188 defm : Neon_shiftNarrow_QR_patterns<int_aarch64_neon_vsqshrn, "SQSHRNvvi">;
2189 defm : Neon_shiftNarrow_QR_patterns<int_aarch64_neon_vuqshrn, "UQSHRNvvi">;
2190 defm : Neon_shiftNarrow_QR_patterns<int_aarch64_neon_vsqrshrn, "SQRSHRNvvi">;
2191 defm : Neon_shiftNarrow_QR_patterns<int_aarch64_neon_vuqrshrn, "UQRSHRNvvi">;
2192
2193 // Convert fix-point and float-pointing
2194 class N2VCvt_Fx<bit q, bit u, bits<5> opcode, string asmop, string T,
2195                 RegisterOperand VPRC, ValueType DestTy, ValueType SrcTy,
2196                 Operand ImmTy, SDPatternOperator IntOp>
2197   : NeonI_2VShiftImm<q, u, opcode,
2198                      (outs VPRC:$Rd), (ins VPRC:$Rn, ImmTy:$Imm),
2199                      asmop # "\t$Rd." # T # ", $Rn." # T # ", $Imm",
2200                      [(set (DestTy VPRC:$Rd), (DestTy (IntOp (SrcTy VPRC:$Rn),
2201                        (i32 ImmTy:$Imm))))],
2202                      NoItinerary>;
2203
2204 multiclass NeonI_N2VCvt_Fx2fp<bit u, bits<5> opcode, string asmop,
2205                               SDPatternOperator IntOp> {
2206   def _2S : N2VCvt_Fx<0, u, opcode, asmop, "2s", VPR64, v2f32, v2i32,
2207                       shr_imm32, IntOp> {
2208     let Inst{22-21} = 0b01;
2209   }
2210
2211   def _4S : N2VCvt_Fx<1, u, opcode, asmop, "4s", VPR128, v4f32, v4i32,
2212                       shr_imm32, IntOp> {
2213     let Inst{22-21} = 0b01;
2214   }
2215
2216   def _2D : N2VCvt_Fx<1, u, opcode, asmop, "2d", VPR128, v2f64, v2i64,
2217                       shr_imm64, IntOp> {
2218     let Inst{22} = 0b1;
2219   }
2220 }
2221
2222 multiclass NeonI_N2VCvt_Fp2fx<bit u, bits<5> opcode, string asmop,
2223                               SDPatternOperator IntOp> {
2224   def _2S : N2VCvt_Fx<0, u, opcode, asmop, "2s", VPR64, v2i32, v2f32,
2225                       shr_imm32, IntOp> {
2226     let Inst{22-21} = 0b01;
2227   }
2228
2229   def _4S : N2VCvt_Fx<1, u, opcode, asmop, "4s", VPR128, v4i32, v4f32,
2230                       shr_imm32, IntOp> {
2231     let Inst{22-21} = 0b01;
2232   }
2233
2234   def _2D : N2VCvt_Fx<1, u, opcode, asmop, "2d", VPR128, v2i64, v2f64,
2235                       shr_imm64, IntOp> {
2236     let Inst{22} = 0b1;
2237   }
2238 }
2239
2240 // Convert fixed-point to floating-point
2241 defm VCVTxs2f : NeonI_N2VCvt_Fx2fp<0, 0b11100, "scvtf",
2242                                    int_arm_neon_vcvtfxs2fp>;
2243 defm VCVTxu2f : NeonI_N2VCvt_Fx2fp<1, 0b11100, "ucvtf",
2244                                    int_arm_neon_vcvtfxu2fp>;
2245
2246 // Convert floating-point to fixed-point
2247 defm VCVTf2xs : NeonI_N2VCvt_Fp2fx<0, 0b11111, "fcvtzs",
2248                                    int_arm_neon_vcvtfp2fxs>;
2249 defm VCVTf2xu : NeonI_N2VCvt_Fp2fx<1, 0b11111, "fcvtzu",
2250                                    int_arm_neon_vcvtfp2fxu>;
2251
2252 multiclass Neon_sshll2_0<SDNode ext>
2253 {
2254   def _v8i8  : PatFrag<(ops node:$Rn),
2255                        (v8i16 (ext (v8i8 (Neon_High16B node:$Rn))))>;
2256   def _v4i16 : PatFrag<(ops node:$Rn),
2257                        (v4i32 (ext (v4i16 (Neon_High8H node:$Rn))))>;
2258   def _v2i32 : PatFrag<(ops node:$Rn),
2259                        (v2i64 (ext (v2i32 (Neon_High4S node:$Rn))))>;
2260 }
2261
2262 defm NI_sext_high : Neon_sshll2_0<sext>;
2263 defm NI_zext_high : Neon_sshll2_0<zext>;
2264
2265
2266 //===----------------------------------------------------------------------===//
2267 // Multiclasses for NeonI_Across
2268 //===----------------------------------------------------------------------===//
2269
2270 // Variant 1
2271
2272 multiclass NeonI_2VAcross_1<bit u, bits<5> opcode,
2273                             string asmop, SDPatternOperator opnode>
2274 {
2275     def _1h8b:  NeonI_2VAcross<0b0, u, 0b00, opcode,
2276                 (outs FPR16:$Rd), (ins VPR64:$Rn),
2277                 asmop # "\t$Rd, $Rn.8b",
2278                 [(set (v1i16 FPR16:$Rd),
2279                     (v1i16 (opnode (v8i8 VPR64:$Rn))))],
2280                 NoItinerary>;
2281
2282     def _1h16b: NeonI_2VAcross<0b1, u, 0b00, opcode,
2283                 (outs FPR16:$Rd), (ins VPR128:$Rn),
2284                 asmop # "\t$Rd, $Rn.16b",
2285                 [(set (v1i16 FPR16:$Rd),
2286                     (v1i16 (opnode (v16i8 VPR128:$Rn))))],
2287                 NoItinerary>;
2288
2289     def _1s4h:  NeonI_2VAcross<0b0, u, 0b01, opcode,
2290                 (outs FPR32:$Rd), (ins VPR64:$Rn),
2291                 asmop # "\t$Rd, $Rn.4h",
2292                 [(set (v1i32 FPR32:$Rd),
2293                     (v1i32 (opnode (v4i16 VPR64:$Rn))))],
2294                 NoItinerary>;
2295
2296     def _1s8h:  NeonI_2VAcross<0b1, u, 0b01, opcode,
2297                 (outs FPR32:$Rd), (ins VPR128:$Rn),
2298                 asmop # "\t$Rd, $Rn.8h",
2299                 [(set (v1i32 FPR32:$Rd),
2300                     (v1i32 (opnode (v8i16 VPR128:$Rn))))],
2301                 NoItinerary>;
2302
2303     // _1d2s doesn't exist!
2304
2305     def _1d4s:  NeonI_2VAcross<0b1, u, 0b10, opcode,
2306                 (outs FPR64:$Rd), (ins VPR128:$Rn),
2307                 asmop # "\t$Rd, $Rn.4s",
2308                 [(set (v1i64 FPR64:$Rd),
2309                     (v1i64 (opnode (v4i32 VPR128:$Rn))))],
2310                 NoItinerary>;
2311 }
2312
2313 defm SADDLV : NeonI_2VAcross_1<0b0, 0b00011, "saddlv", int_aarch64_neon_saddlv>;
2314 defm UADDLV : NeonI_2VAcross_1<0b1, 0b00011, "uaddlv", int_aarch64_neon_uaddlv>;
2315
2316 // Variant 2
2317
2318 multiclass NeonI_2VAcross_2<bit u, bits<5> opcode,
2319                             string asmop, SDPatternOperator opnode>
2320 {
2321     def _1b8b:  NeonI_2VAcross<0b0, u, 0b00, opcode,
2322                 (outs FPR8:$Rd), (ins VPR64:$Rn),
2323                 asmop # "\t$Rd, $Rn.8b",
2324                 [(set (v1i8 FPR8:$Rd),
2325                     (v1i8 (opnode (v8i8 VPR64:$Rn))))],
2326                 NoItinerary>;
2327
2328     def _1b16b: NeonI_2VAcross<0b1, u, 0b00, opcode,
2329                 (outs FPR8:$Rd), (ins VPR128:$Rn),
2330                 asmop # "\t$Rd, $Rn.16b",
2331                 [(set (v1i8 FPR8:$Rd),
2332                     (v1i8 (opnode (v16i8 VPR128:$Rn))))],
2333                 NoItinerary>;
2334
2335     def _1h4h:  NeonI_2VAcross<0b0, u, 0b01, opcode,
2336                 (outs FPR16:$Rd), (ins VPR64:$Rn),
2337                 asmop # "\t$Rd, $Rn.4h",
2338                 [(set (v1i16 FPR16:$Rd),
2339                     (v1i16 (opnode (v4i16 VPR64:$Rn))))],
2340                 NoItinerary>;
2341
2342     def _1h8h:  NeonI_2VAcross<0b1, u, 0b01, opcode,
2343                 (outs FPR16:$Rd), (ins VPR128:$Rn),
2344                 asmop # "\t$Rd, $Rn.8h",
2345                 [(set (v1i16 FPR16:$Rd),
2346                     (v1i16 (opnode (v8i16 VPR128:$Rn))))],
2347                 NoItinerary>;
2348
2349     // _1s2s doesn't exist!
2350
2351     def _1s4s:  NeonI_2VAcross<0b1, u, 0b10, opcode,
2352                 (outs FPR32:$Rd), (ins VPR128:$Rn),
2353                 asmop # "\t$Rd, $Rn.4s",
2354                 [(set (v1i32 FPR32:$Rd),
2355                     (v1i32 (opnode (v4i32 VPR128:$Rn))))],
2356                 NoItinerary>;
2357 }
2358
2359 defm SMAXV : NeonI_2VAcross_2<0b0, 0b01010, "smaxv", int_aarch64_neon_smaxv>;
2360 defm UMAXV : NeonI_2VAcross_2<0b1, 0b01010, "umaxv", int_aarch64_neon_umaxv>;
2361
2362 defm SMINV : NeonI_2VAcross_2<0b0, 0b11010, "sminv", int_aarch64_neon_sminv>;
2363 defm UMINV : NeonI_2VAcross_2<0b1, 0b11010, "uminv", int_aarch64_neon_uminv>;
2364
2365 defm ADDV : NeonI_2VAcross_2<0b0, 0b11011, "addv", int_aarch64_neon_vaddv>;
2366
2367 // Variant 3
2368
2369 multiclass NeonI_2VAcross_3<bit u, bits<5> opcode, bits<2> size,
2370                             string asmop, SDPatternOperator opnode> {
2371     def _1s4s:  NeonI_2VAcross<0b1, u, size, opcode,
2372                 (outs FPR32:$Rd), (ins VPR128:$Rn),
2373                 asmop # "\t$Rd, $Rn.4s",
2374                 [(set (v1f32 FPR32:$Rd),
2375                     (v1f32 (opnode (v4f32 VPR128:$Rn))))],
2376                 NoItinerary>;
2377 }
2378
2379 defm FMAXNMV : NeonI_2VAcross_3<0b1, 0b01100, 0b00, "fmaxnmv",
2380                                 int_aarch64_neon_vmaxnmv>;
2381 defm FMINNMV : NeonI_2VAcross_3<0b1, 0b01100, 0b10, "fminnmv",
2382                                 int_aarch64_neon_vminnmv>;
2383
2384 defm FMAXV : NeonI_2VAcross_3<0b1, 0b01111, 0b00, "fmaxv",
2385                               int_aarch64_neon_vmaxv>;
2386 defm FMINV : NeonI_2VAcross_3<0b1, 0b01111, 0b10, "fminv",
2387                               int_aarch64_neon_vminv>;
2388
2389 // The followings are for instruction class (Perm)
2390
2391 class NeonI_Permute<bit q, bits<2> size, bits<3> opcode,
2392                     string asmop, RegisterOperand OpVPR, string OpS,
2393                     SDPatternOperator opnode, ValueType Ty>
2394   : NeonI_Perm<q, size, opcode,
2395                (outs OpVPR:$Rd), (ins OpVPR:$Rn, OpVPR:$Rm),
2396                asmop # "\t$Rd." # OpS # ", $Rn." # OpS # ", $Rm." # OpS,
2397                [(set (Ty OpVPR:$Rd),
2398                   (Ty (opnode (Ty OpVPR:$Rn), (Ty OpVPR:$Rm))))],
2399                NoItinerary>;
2400
2401 multiclass NeonI_Perm_pat<bits<3> opcode, string asmop,
2402                           SDPatternOperator opnode> {
2403   def _8b  : NeonI_Permute<0b0, 0b00, opcode, asmop,
2404                            VPR64, "8b", opnode, v8i8>;
2405   def _16b : NeonI_Permute<0b1, 0b00, opcode, asmop,
2406                            VPR128, "16b",opnode, v16i8>;
2407   def _4h  : NeonI_Permute<0b0, 0b01, opcode, asmop,
2408                            VPR64, "4h", opnode, v4i16>;
2409   def _8h  : NeonI_Permute<0b1, 0b01, opcode, asmop,
2410                            VPR128, "8h", opnode, v8i16>;
2411   def _2s  : NeonI_Permute<0b0, 0b10, opcode, asmop,
2412                            VPR64, "2s", opnode, v2i32>;
2413   def _4s  : NeonI_Permute<0b1, 0b10, opcode, asmop,
2414                            VPR128, "4s", opnode, v4i32>;
2415   def _2d  : NeonI_Permute<0b1, 0b11, opcode, asmop,
2416                            VPR128, "2d", opnode, v2i64>;
2417 }
2418
2419 defm UZP1vvv : NeonI_Perm_pat<0b001, "uzp1", Neon_uzp1>;
2420 defm TRN1vvv : NeonI_Perm_pat<0b010, "trn1", Neon_trn1>;
2421 defm ZIP1vvv : NeonI_Perm_pat<0b011, "zip1", Neon_zip1>;
2422 defm UZP2vvv : NeonI_Perm_pat<0b101, "uzp2", Neon_uzp2>;
2423 defm TRN2vvv : NeonI_Perm_pat<0b110, "trn2", Neon_trn2>;
2424 defm ZIP2vvv : NeonI_Perm_pat<0b111, "zip2", Neon_zip2>;
2425
2426 multiclass NeonI_Perm_float_pat<string INS, SDPatternOperator opnode> {
2427   def : Pat<(v2f32 (opnode (v2f32 VPR64:$Rn), (v2f32 VPR64:$Rm))),
2428             (!cast<Instruction>(INS # "_2s") VPR64:$Rn, VPR64:$Rm)>;
2429
2430   def : Pat<(v4f32 (opnode (v4f32 VPR128:$Rn), (v4f32 VPR128:$Rm))),
2431             (!cast<Instruction>(INS # "_4s") VPR128:$Rn, VPR128:$Rm)>;
2432
2433   def : Pat<(v2f64 (opnode (v2f64 VPR128:$Rn), (v2f64 VPR128:$Rm))),
2434             (!cast<Instruction>(INS # "_2d") VPR128:$Rn, VPR128:$Rm)>;
2435 }
2436
2437 defm : NeonI_Perm_float_pat<"UZP1vvv", Neon_uzp1>;
2438 defm : NeonI_Perm_float_pat<"UZP2vvv", Neon_uzp2>;
2439 defm : NeonI_Perm_float_pat<"ZIP1vvv", Neon_zip1>;
2440 defm : NeonI_Perm_float_pat<"ZIP2vvv", Neon_zip2>;
2441 defm : NeonI_Perm_float_pat<"TRN1vvv", Neon_trn1>;
2442 defm : NeonI_Perm_float_pat<"TRN2vvv", Neon_trn2>;
2443
2444 // The followings are for instruction class (3V Diff)
2445
2446 // normal long/long2 pattern
2447 class NeonI_3VDL<bit q, bit u, bits<2> size, bits<4> opcode,
2448                  string asmop, string ResS, string OpS,
2449                  SDPatternOperator opnode, SDPatternOperator ext,
2450                  RegisterOperand OpVPR,
2451                  ValueType ResTy, ValueType OpTy>
2452   : NeonI_3VDiff<q, u, size, opcode,
2453                  (outs VPR128:$Rd), (ins OpVPR:$Rn, OpVPR:$Rm),
2454                  asmop # "\t$Rd." # ResS # ", $Rn." # OpS # ", $Rm." # OpS,
2455                  [(set (ResTy VPR128:$Rd),
2456                     (ResTy (opnode (ResTy (ext (OpTy OpVPR:$Rn))),
2457                                    (ResTy (ext (OpTy OpVPR:$Rm))))))],
2458                  NoItinerary>;
2459
2460 multiclass NeonI_3VDL_s<bit u, bits<4> opcode,
2461                         string asmop, SDPatternOperator opnode,
2462                         bit Commutable = 0> {
2463   let isCommutable = Commutable in {
2464     def _8h8b : NeonI_3VDL<0b0, u, 0b00, opcode, asmop, "8h", "8b",
2465                            opnode, sext, VPR64, v8i16, v8i8>;
2466     def _4s4h : NeonI_3VDL<0b0, u, 0b01, opcode, asmop, "4s", "4h",
2467                            opnode, sext, VPR64, v4i32, v4i16>;
2468     def _2d2s : NeonI_3VDL<0b0, u, 0b10, opcode, asmop, "2d", "2s",
2469                            opnode, sext, VPR64, v2i64, v2i32>;
2470   }
2471 }
2472
2473 multiclass NeonI_3VDL2_s<bit u, bits<4> opcode, string asmop,
2474                          SDPatternOperator opnode, bit Commutable = 0> {
2475   let isCommutable = Commutable in {
2476     def _8h16b : NeonI_3VDL<0b1, u, 0b00, opcode, asmop, "8h", "16b",
2477                             opnode, NI_sext_high_v8i8, VPR128, v8i16, v16i8>;
2478     def _4s8h  : NeonI_3VDL<0b1, u, 0b01, opcode, asmop, "4s", "8h",
2479                             opnode, NI_sext_high_v4i16, VPR128, v4i32, v8i16>;
2480     def _2d4s  : NeonI_3VDL<0b1, u, 0b10, opcode, asmop, "2d", "4s",
2481                             opnode, NI_sext_high_v2i32, VPR128, v2i64, v4i32>;
2482   }
2483 }
2484
2485 multiclass NeonI_3VDL_u<bit u, bits<4> opcode, string asmop,
2486                         SDPatternOperator opnode, bit Commutable = 0> {
2487   let isCommutable = Commutable in {
2488     def _8h8b : NeonI_3VDL<0b0, u, 0b00, opcode, asmop, "8h", "8b",
2489                            opnode, zext, VPR64, v8i16, v8i8>;
2490     def _4s4h : NeonI_3VDL<0b0, u, 0b01, opcode, asmop, "4s", "4h",
2491                            opnode, zext, VPR64, v4i32, v4i16>;
2492     def _2d2s : NeonI_3VDL<0b0, u, 0b10, opcode, asmop, "2d", "2s",
2493                            opnode, zext, VPR64, v2i64, v2i32>;
2494   }
2495 }
2496
2497 multiclass NeonI_3VDL2_u<bit u, bits<4> opcode, string asmop,
2498                          SDPatternOperator opnode, bit Commutable = 0> {
2499   let isCommutable = Commutable in {
2500     def _8h16b : NeonI_3VDL<0b1, u, 0b00, opcode, asmop, "8h", "16b",
2501                             opnode, NI_zext_high_v8i8, VPR128, v8i16, v16i8>;
2502     def _4s8h : NeonI_3VDL<0b1, u, 0b01, opcode, asmop, "4s", "8h",
2503                            opnode, NI_zext_high_v4i16, VPR128, v4i32, v8i16>;
2504     def _2d4s : NeonI_3VDL<0b1, u, 0b10, opcode, asmop, "2d", "4s",
2505                            opnode, NI_zext_high_v2i32, VPR128, v2i64, v4i32>;
2506   }
2507 }
2508
2509 defm SADDLvvv :  NeonI_3VDL_s<0b0, 0b0000, "saddl", add, 1>;
2510 defm UADDLvvv :  NeonI_3VDL_u<0b1, 0b0000, "uaddl", add, 1>;
2511
2512 defm SADDL2vvv :  NeonI_3VDL2_s<0b0, 0b0000, "saddl2", add, 1>;
2513 defm UADDL2vvv :  NeonI_3VDL2_u<0b1, 0b0000, "uaddl2", add, 1>;
2514
2515 defm SSUBLvvv :  NeonI_3VDL_s<0b0, 0b0010, "ssubl", sub, 0>;
2516 defm USUBLvvv :  NeonI_3VDL_u<0b1, 0b0010, "usubl", sub, 0>;
2517
2518 defm SSUBL2vvv :  NeonI_3VDL2_s<0b0, 0b0010, "ssubl2", sub, 0>;
2519 defm USUBL2vvv :  NeonI_3VDL2_u<0b1, 0b0010, "usubl2", sub, 0>;
2520
2521 // normal wide/wide2 pattern
2522 class NeonI_3VDW<bit q, bit u, bits<2> size, bits<4> opcode,
2523                  string asmop, string ResS, string OpS,
2524                  SDPatternOperator opnode, SDPatternOperator ext,
2525                  RegisterOperand OpVPR,
2526                  ValueType ResTy, ValueType OpTy>
2527   : NeonI_3VDiff<q, u, size, opcode,
2528                  (outs VPR128:$Rd), (ins VPR128:$Rn, OpVPR:$Rm),
2529                  asmop # "\t$Rd." # ResS # ", $Rn." # ResS # ", $Rm." # OpS,
2530                  [(set (ResTy VPR128:$Rd),
2531                     (ResTy (opnode (ResTy VPR128:$Rn),
2532                                    (ResTy (ext (OpTy OpVPR:$Rm))))))],
2533                  NoItinerary>;
2534
2535 multiclass NeonI_3VDW_s<bit u, bits<4> opcode, string asmop,
2536                         SDPatternOperator opnode> {
2537   def _8h8b : NeonI_3VDW<0b0, u, 0b00, opcode, asmop, "8h", "8b",
2538                          opnode, sext, VPR64, v8i16, v8i8>;
2539   def _4s4h : NeonI_3VDW<0b0, u, 0b01, opcode, asmop, "4s", "4h",
2540                          opnode, sext, VPR64, v4i32, v4i16>;
2541   def _2d2s : NeonI_3VDW<0b0, u, 0b10, opcode, asmop, "2d", "2s",
2542                          opnode, sext, VPR64, v2i64, v2i32>;
2543 }
2544
2545 defm SADDWvvv :  NeonI_3VDW_s<0b0, 0b0001, "saddw", add>;
2546 defm SSUBWvvv :  NeonI_3VDW_s<0b0, 0b0011, "ssubw", sub>;
2547
2548 multiclass NeonI_3VDW2_s<bit u, bits<4> opcode, string asmop,
2549                          SDPatternOperator opnode> {
2550   def _8h16b : NeonI_3VDW<0b1, u, 0b00, opcode, asmop, "8h", "16b",
2551                           opnode, NI_sext_high_v8i8, VPR128, v8i16, v16i8>;
2552   def _4s8h  : NeonI_3VDW<0b1, u, 0b01, opcode, asmop, "4s", "8h",
2553                           opnode, NI_sext_high_v4i16, VPR128, v4i32, v8i16>;
2554   def _2d4s  : NeonI_3VDW<0b1, u, 0b10, opcode, asmop, "2d", "4s",
2555                           opnode, NI_sext_high_v2i32, VPR128, v2i64, v4i32>;
2556 }
2557
2558 defm SADDW2vvv :  NeonI_3VDW2_s<0b0, 0b0001, "saddw2", add>;
2559 defm SSUBW2vvv :  NeonI_3VDW2_s<0b0, 0b0011, "ssubw2", sub>;
2560
2561 multiclass NeonI_3VDW_u<bit u, bits<4> opcode, string asmop,
2562                         SDPatternOperator opnode> {
2563   def _8h8b : NeonI_3VDW<0b0, u, 0b00, opcode, asmop, "8h", "8b",
2564                          opnode, zext, VPR64, v8i16, v8i8>;
2565   def _4s4h : NeonI_3VDW<0b0, u, 0b01, opcode, asmop, "4s", "4h",
2566                          opnode, zext, VPR64, v4i32, v4i16>;
2567   def _2d2s : NeonI_3VDW<0b0, u, 0b10, opcode, asmop, "2d", "2s",
2568                          opnode, zext, VPR64, v2i64, v2i32>;
2569 }
2570
2571 defm UADDWvvv :  NeonI_3VDW_u<0b1, 0b0001, "uaddw", add>;
2572 defm USUBWvvv :  NeonI_3VDW_u<0b1, 0b0011, "usubw", sub>;
2573
2574 multiclass NeonI_3VDW2_u<bit u, bits<4> opcode, string asmop,
2575                          SDPatternOperator opnode> {
2576   def _8h16b : NeonI_3VDW<0b1, u, 0b00, opcode, asmop, "8h", "16b",
2577                           opnode, NI_zext_high_v8i8, VPR128, v8i16, v16i8>;
2578   def _4s8h : NeonI_3VDW<0b1, u, 0b01, opcode, asmop, "4s", "8h",
2579                          opnode, NI_zext_high_v4i16, VPR128, v4i32, v8i16>;
2580   def _2d4s : NeonI_3VDW<0b1, u, 0b10, opcode, asmop, "2d", "4s",
2581                          opnode, NI_zext_high_v2i32, VPR128, v2i64, v4i32>;
2582 }
2583
2584 defm UADDW2vvv :  NeonI_3VDW2_u<0b1, 0b0001, "uaddw2", add>;
2585 defm USUBW2vvv :  NeonI_3VDW2_u<0b1, 0b0011, "usubw2", sub>;
2586
2587 // Get the high half part of the vector element.
2588 multiclass NeonI_get_high {
2589   def _8h : PatFrag<(ops node:$Rn),
2590                     (v8i8 (trunc (v8i16 (srl (v8i16 node:$Rn),
2591                                              (v8i16 (Neon_vdup (i32 8)))))))>;
2592   def _4s : PatFrag<(ops node:$Rn),
2593                     (v4i16 (trunc (v4i32 (srl (v4i32 node:$Rn),
2594                                               (v4i32 (Neon_vdup (i32 16)))))))>;
2595   def _2d : PatFrag<(ops node:$Rn),
2596                     (v2i32 (trunc (v2i64 (srl (v2i64 node:$Rn),
2597                                               (v2i64 (Neon_vdup (i32 32)))))))>;
2598 }
2599
2600 defm NI_get_hi : NeonI_get_high;
2601
2602 // pattern for addhn/subhn with 2 operands
2603 class NeonI_3VDN_addhn_2Op<bit q, bit u, bits<2> size, bits<4> opcode,
2604                            string asmop, string ResS, string OpS,
2605                            SDPatternOperator opnode, SDPatternOperator get_hi,
2606                            ValueType ResTy, ValueType OpTy>
2607   : NeonI_3VDiff<q, u, size, opcode,
2608                  (outs VPR64:$Rd), (ins VPR128:$Rn, VPR128:$Rm),
2609                  asmop # "\t$Rd." # ResS # ", $Rn." # OpS # ", $Rm." # OpS,
2610                  [(set (ResTy VPR64:$Rd),
2611                     (ResTy (get_hi
2612                       (OpTy (opnode (OpTy VPR128:$Rn),
2613                                     (OpTy VPR128:$Rm))))))],
2614                  NoItinerary>;
2615
2616 multiclass NeonI_3VDN_addhn_2Op<bit u, bits<4> opcode, string asmop,
2617                                 SDPatternOperator opnode, bit Commutable = 0> {
2618   let isCommutable = Commutable in {
2619     def _8b8h : NeonI_3VDN_addhn_2Op<0b0, u, 0b00, opcode, asmop, "8b", "8h",
2620                                      opnode, NI_get_hi_8h, v8i8, v8i16>;
2621     def _4h4s : NeonI_3VDN_addhn_2Op<0b0, u, 0b01, opcode, asmop, "4h", "4s",
2622                                      opnode, NI_get_hi_4s, v4i16, v4i32>;
2623     def _2s2d : NeonI_3VDN_addhn_2Op<0b0, u, 0b10, opcode, asmop, "2s", "2d",
2624                                      opnode, NI_get_hi_2d, v2i32, v2i64>;
2625   }
2626 }
2627
2628 defm ADDHNvvv  : NeonI_3VDN_addhn_2Op<0b0, 0b0100, "addhn", add, 1>;
2629 defm SUBHNvvv  : NeonI_3VDN_addhn_2Op<0b0, 0b0110, "subhn", sub, 0>;
2630
2631 // pattern for operation with 2 operands
2632 class NeonI_3VD_2Op<bit q, bit u, bits<2> size, bits<4> opcode,
2633                     string asmop, string ResS, string OpS,
2634                     SDPatternOperator opnode,
2635                     RegisterOperand ResVPR, RegisterOperand OpVPR,
2636                     ValueType ResTy, ValueType OpTy>
2637   : NeonI_3VDiff<q, u, size, opcode,
2638                  (outs ResVPR:$Rd), (ins OpVPR:$Rn, OpVPR:$Rm),
2639                  asmop # "\t$Rd." # ResS # ", $Rn." # OpS # ", $Rm." # OpS,
2640                  [(set (ResTy ResVPR:$Rd),
2641                     (ResTy (opnode (OpTy OpVPR:$Rn), (OpTy OpVPR:$Rm))))],
2642                  NoItinerary>;
2643
2644 // normal narrow pattern
2645 multiclass NeonI_3VDN_2Op<bit u, bits<4> opcode, string asmop,
2646                           SDPatternOperator opnode, bit Commutable = 0> {
2647   let isCommutable = Commutable in {
2648     def _8b8h : NeonI_3VD_2Op<0b0, u, 0b00, opcode, asmop, "8b", "8h",
2649                               opnode, VPR64, VPR128, v8i8, v8i16>;
2650     def _4h4s : NeonI_3VD_2Op<0b0, u, 0b01, opcode, asmop, "4h", "4s",
2651                               opnode, VPR64, VPR128, v4i16, v4i32>;
2652     def _2s2d : NeonI_3VD_2Op<0b0, u, 0b10, opcode, asmop, "2s", "2d",
2653                               opnode, VPR64, VPR128, v2i32, v2i64>;
2654   }
2655 }
2656
2657 defm RADDHNvvv : NeonI_3VDN_2Op<0b1, 0b0100, "raddhn", int_arm_neon_vraddhn, 1>;
2658 defm RSUBHNvvv : NeonI_3VDN_2Op<0b1, 0b0110, "rsubhn", int_arm_neon_vrsubhn, 0>;
2659
2660 // pattern for acle intrinsic with 3 operands
2661 class NeonI_3VDN_3Op<bit q, bit u, bits<2> size, bits<4> opcode,
2662                      string asmop, string ResS, string OpS>
2663   : NeonI_3VDiff<q, u, size, opcode,
2664                  (outs VPR128:$Rd), (ins VPR128:$src, VPR128:$Rn, VPR128:$Rm),
2665                  asmop # "\t$Rd." # ResS # ", $Rn." # OpS # ", $Rm." # OpS,
2666                  [], NoItinerary> {
2667   let Constraints = "$src = $Rd";
2668   let neverHasSideEffects = 1;
2669 }
2670
2671 multiclass NeonI_3VDN_3Op_v1<bit u, bits<4> opcode, string asmop> {
2672   def _16b8h : NeonI_3VDN_3Op<0b1, u, 0b00, opcode, asmop, "16b", "8h">;
2673   def _8h4s : NeonI_3VDN_3Op<0b1, u, 0b01, opcode, asmop, "8h", "4s">;
2674   def _4s2d : NeonI_3VDN_3Op<0b1, u, 0b10, opcode, asmop, "4s", "2d">;
2675 }
2676
2677 defm ADDHN2vvv  : NeonI_3VDN_3Op_v1<0b0, 0b0100, "addhn2">;
2678 defm SUBHN2vvv  : NeonI_3VDN_3Op_v1<0b0, 0b0110, "subhn2">;
2679
2680 defm RADDHN2vvv : NeonI_3VDN_3Op_v1<0b1, 0b0100, "raddhn2">;
2681 defm RSUBHN2vvv : NeonI_3VDN_3Op_v1<0b1, 0b0110, "rsubhn2">;
2682
2683 // Patterns have to be separate because there's a SUBREG_TO_REG in the output
2684 // part.
2685 class NarrowHighHalfPat<Instruction INST, ValueType DstTy, ValueType SrcTy,
2686                         SDPatternOperator coreop>
2687   : Pat<(Neon_combine_2D (v1i64 VPR64:$src),
2688                       (v1i64 (bitconvert (DstTy (coreop (SrcTy VPR128:$Rn),
2689                                                         (SrcTy VPR128:$Rm)))))),
2690         (INST (SUBREG_TO_REG (i64 0), VPR64:$src, sub_64),
2691               VPR128:$Rn, VPR128:$Rm)>;
2692
2693 // addhn2 patterns
2694 def : NarrowHighHalfPat<ADDHN2vvv_16b8h, v8i8,  v8i16,
2695           BinOpFrag<(NI_get_hi_8h (add node:$LHS, node:$RHS))>>;
2696 def : NarrowHighHalfPat<ADDHN2vvv_8h4s,  v4i16, v4i32,
2697           BinOpFrag<(NI_get_hi_4s (add node:$LHS, node:$RHS))>>;
2698 def : NarrowHighHalfPat<ADDHN2vvv_4s2d,  v2i32, v2i64,
2699           BinOpFrag<(NI_get_hi_2d (add node:$LHS, node:$RHS))>>;
2700
2701 // subhn2 patterns
2702 def : NarrowHighHalfPat<SUBHN2vvv_16b8h, v8i8,  v8i16,
2703           BinOpFrag<(NI_get_hi_8h (sub node:$LHS, node:$RHS))>>;
2704 def : NarrowHighHalfPat<SUBHN2vvv_8h4s,  v4i16, v4i32,
2705           BinOpFrag<(NI_get_hi_4s (sub node:$LHS, node:$RHS))>>;
2706 def : NarrowHighHalfPat<SUBHN2vvv_4s2d,  v2i32, v2i64,
2707           BinOpFrag<(NI_get_hi_2d (sub node:$LHS, node:$RHS))>>;
2708
2709 // raddhn2 patterns
2710 def : NarrowHighHalfPat<RADDHN2vvv_16b8h, v8i8,  v8i16, int_arm_neon_vraddhn>;
2711 def : NarrowHighHalfPat<RADDHN2vvv_8h4s,  v4i16, v4i32, int_arm_neon_vraddhn>;
2712 def : NarrowHighHalfPat<RADDHN2vvv_4s2d,  v2i32, v2i64, int_arm_neon_vraddhn>;
2713
2714 // rsubhn2 patterns
2715 def : NarrowHighHalfPat<RSUBHN2vvv_16b8h, v8i8,  v8i16, int_arm_neon_vrsubhn>;
2716 def : NarrowHighHalfPat<RSUBHN2vvv_8h4s,  v4i16, v4i32, int_arm_neon_vrsubhn>;
2717 def : NarrowHighHalfPat<RSUBHN2vvv_4s2d,  v2i32, v2i64, int_arm_neon_vrsubhn>;
2718
2719 // pattern that need to extend result
2720 class NeonI_3VDL_Ext<bit q, bit u, bits<2> size, bits<4> opcode,
2721                      string asmop, string ResS, string OpS,
2722                      SDPatternOperator opnode,
2723                      RegisterOperand OpVPR,
2724                      ValueType ResTy, ValueType OpTy, ValueType OpSTy>
2725   : NeonI_3VDiff<q, u, size, opcode,
2726                  (outs VPR128:$Rd), (ins OpVPR:$Rn, OpVPR:$Rm),
2727                  asmop # "\t$Rd." # ResS # ", $Rn." # OpS # ", $Rm." # OpS,
2728                  [(set (ResTy VPR128:$Rd),
2729                     (ResTy (zext (OpSTy (opnode (OpTy OpVPR:$Rn),
2730                                                 (OpTy OpVPR:$Rm))))))],
2731                  NoItinerary>;
2732
2733 multiclass NeonI_3VDL_zext<bit u, bits<4> opcode, string asmop,
2734                            SDPatternOperator opnode, bit Commutable = 0> {
2735   let isCommutable = Commutable in {
2736     def _8h8b : NeonI_3VDL_Ext<0b0, u, 0b00, opcode, asmop, "8h", "8b",
2737                                opnode, VPR64, v8i16, v8i8, v8i8>;
2738     def _4s4h : NeonI_3VDL_Ext<0b0, u, 0b01, opcode, asmop, "4s", "4h",
2739                                opnode, VPR64, v4i32, v4i16, v4i16>;
2740     def _2d2s : NeonI_3VDL_Ext<0b0, u, 0b10, opcode, asmop, "2d", "2s",
2741                                opnode, VPR64, v2i64, v2i32, v2i32>;
2742   }
2743 }
2744
2745 defm SABDLvvv : NeonI_3VDL_zext<0b0, 0b0111, "sabdl", int_arm_neon_vabds, 1>;
2746 defm UABDLvvv : NeonI_3VDL_zext<0b1, 0b0111, "uabdl", int_arm_neon_vabdu, 1>;
2747
2748 multiclass NeonI_Op_High<SDPatternOperator op> {
2749   def _16B : PatFrag<(ops node:$Rn, node:$Rm),
2750                      (op (v8i8 (Neon_High16B node:$Rn)),
2751                          (v8i8 (Neon_High16B node:$Rm)))>;
2752   def _8H  : PatFrag<(ops node:$Rn, node:$Rm),
2753                      (op (v4i16 (Neon_High8H node:$Rn)),
2754                          (v4i16 (Neon_High8H node:$Rm)))>;
2755   def _4S  : PatFrag<(ops node:$Rn, node:$Rm),
2756                      (op (v2i32 (Neon_High4S node:$Rn)),
2757                          (v2i32 (Neon_High4S node:$Rm)))>;
2758 }
2759
2760 defm NI_sabdl_hi : NeonI_Op_High<int_arm_neon_vabds>;
2761 defm NI_uabdl_hi : NeonI_Op_High<int_arm_neon_vabdu>;
2762 defm NI_smull_hi : NeonI_Op_High<int_arm_neon_vmulls>;
2763 defm NI_umull_hi : NeonI_Op_High<int_arm_neon_vmullu>;
2764 defm NI_qdmull_hi : NeonI_Op_High<int_arm_neon_vqdmull>;
2765 defm NI_pmull_hi : NeonI_Op_High<int_arm_neon_vmullp>;
2766
2767 multiclass NeonI_3VDL_Abd_u<bit u, bits<4> opcode, string asmop, string opnode,
2768                             bit Commutable = 0> {
2769   let isCommutable = Commutable in {
2770     def _8h8b  : NeonI_3VDL_Ext<0b1, u, 0b00, opcode, asmop, "8h", "16b",
2771                                 !cast<PatFrag>(opnode # "_16B"),
2772                                 VPR128, v8i16, v16i8, v8i8>;
2773     def _4s4h  : NeonI_3VDL_Ext<0b1, u, 0b01, opcode, asmop, "4s", "8h",
2774                                 !cast<PatFrag>(opnode # "_8H"),
2775                                 VPR128, v4i32, v8i16, v4i16>;
2776     def _2d2s  : NeonI_3VDL_Ext<0b1, u, 0b10, opcode, asmop, "2d", "4s",
2777                                 !cast<PatFrag>(opnode # "_4S"),
2778                                 VPR128, v2i64, v4i32, v2i32>;
2779   }
2780 }
2781
2782 defm SABDL2vvv : NeonI_3VDL_Abd_u<0b0, 0b0111, "sabdl2", "NI_sabdl_hi", 1>;
2783 defm UABDL2vvv : NeonI_3VDL_Abd_u<0b1, 0b0111, "uabdl2", "NI_uabdl_hi", 1>;
2784
2785 // For pattern that need two operators being chained.
2786 class NeonI_3VDL_Aba<bit q, bit u, bits<2> size, bits<4> opcode,
2787                      string asmop, string ResS, string OpS,
2788                      SDPatternOperator opnode, SDPatternOperator subop,
2789                      RegisterOperand OpVPR,
2790                      ValueType ResTy, ValueType OpTy, ValueType OpSTy>
2791   : NeonI_3VDiff<q, u, size, opcode,
2792                  (outs VPR128:$Rd), (ins VPR128:$src, OpVPR:$Rn, OpVPR:$Rm),
2793                  asmop # "\t$Rd." # ResS # ", $Rn." # OpS # ", $Rm." # OpS,
2794                  [(set (ResTy VPR128:$Rd),
2795                     (ResTy (opnode
2796                       (ResTy VPR128:$src),
2797                       (ResTy (zext (OpSTy (subop (OpTy OpVPR:$Rn),
2798                                                  (OpTy OpVPR:$Rm))))))))],
2799                  NoItinerary> {
2800   let Constraints = "$src = $Rd";
2801 }
2802
2803 multiclass NeonI_3VDL_Aba_v1<bit u, bits<4> opcode, string asmop,
2804                              SDPatternOperator opnode, SDPatternOperator subop>{
2805   def _8h8b : NeonI_3VDL_Aba<0b0, u, 0b00, opcode, asmop, "8h", "8b",
2806                              opnode, subop, VPR64, v8i16, v8i8, v8i8>;
2807   def _4s4h : NeonI_3VDL_Aba<0b0, u, 0b01, opcode, asmop, "4s", "4h",
2808                              opnode, subop, VPR64, v4i32, v4i16, v4i16>;
2809   def _2d2s : NeonI_3VDL_Aba<0b0, u, 0b10, opcode, asmop, "2d", "2s",
2810                              opnode, subop, VPR64, v2i64, v2i32, v2i32>;
2811 }
2812
2813 defm SABALvvv :  NeonI_3VDL_Aba_v1<0b0, 0b0101, "sabal",
2814                                    add, int_arm_neon_vabds>;
2815 defm UABALvvv :  NeonI_3VDL_Aba_v1<0b1, 0b0101, "uabal",
2816                                    add, int_arm_neon_vabdu>;
2817
2818 multiclass NeonI_3VDL2_Aba_v1<bit u, bits<4> opcode, string asmop,
2819                               SDPatternOperator opnode, string subop> {
2820   def _8h8b : NeonI_3VDL_Aba<0b1, u, 0b00, opcode, asmop, "8h", "16b",
2821                              opnode, !cast<PatFrag>(subop # "_16B"),
2822                              VPR128, v8i16, v16i8, v8i8>;
2823   def _4s4h : NeonI_3VDL_Aba<0b1, u, 0b01, opcode, asmop, "4s", "8h",
2824                              opnode, !cast<PatFrag>(subop # "_8H"),
2825                              VPR128, v4i32, v8i16, v4i16>;
2826   def _2d2s : NeonI_3VDL_Aba<0b1, u, 0b10, opcode, asmop, "2d", "4s",
2827                              opnode, !cast<PatFrag>(subop # "_4S"),
2828                              VPR128, v2i64, v4i32, v2i32>;
2829 }
2830
2831 defm SABAL2vvv :  NeonI_3VDL2_Aba_v1<0b0, 0b0101, "sabal2", add,
2832                                      "NI_sabdl_hi">;
2833 defm UABAL2vvv :  NeonI_3VDL2_Aba_v1<0b1, 0b0101, "uabal2", add,
2834                                      "NI_uabdl_hi">;
2835
2836 // Long pattern with 2 operands
2837 multiclass NeonI_3VDL_2Op<bit u, bits<4> opcode, string asmop,
2838                           SDPatternOperator opnode, bit Commutable = 0> {
2839   let isCommutable = Commutable in {
2840     def _8h8b : NeonI_3VD_2Op<0b0, u, 0b00, opcode, asmop, "8h", "8b",
2841                               opnode, VPR128, VPR64, v8i16, v8i8>;
2842     def _4s4h : NeonI_3VD_2Op<0b0, u, 0b01, opcode, asmop, "4s", "4h",
2843                               opnode, VPR128, VPR64, v4i32, v4i16>;
2844     def _2d2s : NeonI_3VD_2Op<0b0, u, 0b10, opcode, asmop, "2d", "2s",
2845                               opnode, VPR128, VPR64, v2i64, v2i32>;
2846   }
2847 }
2848
2849 defm SMULLvvv :  NeonI_3VDL_2Op<0b0, 0b1100, "smull", int_arm_neon_vmulls, 1>;
2850 defm UMULLvvv :  NeonI_3VDL_2Op<0b1, 0b1100, "umull", int_arm_neon_vmullu, 1>;
2851
2852 class NeonI_3VDL2_2Op_mull<bit q, bit u, bits<2> size, bits<4> opcode,
2853                            string asmop, string ResS, string OpS,
2854                            SDPatternOperator opnode,
2855                            ValueType ResTy, ValueType OpTy>
2856   : NeonI_3VDiff<q, u, size, opcode,
2857                  (outs VPR128:$Rd), (ins VPR128:$Rn, VPR128:$Rm),
2858                  asmop # "\t$Rd." # ResS # ", $Rn." # OpS # ", $Rm." # OpS,
2859                  [(set (ResTy VPR128:$Rd),
2860                     (ResTy (opnode (OpTy VPR128:$Rn), (OpTy VPR128:$Rm))))],
2861                  NoItinerary>;
2862
2863 multiclass NeonI_3VDL2_2Op_mull_v1<bit u, bits<4> opcode, string asmop,
2864                                    string opnode, bit Commutable = 0> {
2865   let isCommutable = Commutable in {
2866     def _8h16b : NeonI_3VDL2_2Op_mull<0b1, u, 0b00, opcode, asmop, "8h", "16b",
2867                                       !cast<PatFrag>(opnode # "_16B"),
2868                                       v8i16, v16i8>;
2869     def _4s8h : NeonI_3VDL2_2Op_mull<0b1, u, 0b01, opcode, asmop, "4s", "8h",
2870                                      !cast<PatFrag>(opnode # "_8H"),
2871                                      v4i32, v8i16>;
2872     def _2d4s : NeonI_3VDL2_2Op_mull<0b1, u, 0b10, opcode, asmop, "2d", "4s",
2873                                      !cast<PatFrag>(opnode # "_4S"),
2874                                      v2i64, v4i32>;
2875   }
2876 }
2877
2878 defm SMULL2vvv : NeonI_3VDL2_2Op_mull_v1<0b0, 0b1100, "smull2",
2879                                          "NI_smull_hi", 1>;
2880 defm UMULL2vvv : NeonI_3VDL2_2Op_mull_v1<0b1, 0b1100, "umull2",
2881                                          "NI_umull_hi", 1>;
2882
2883 // Long pattern with 3 operands
2884 class NeonI_3VDL_3Op<bit q, bit u, bits<2> size, bits<4> opcode,
2885                      string asmop, string ResS, string OpS,
2886                      SDPatternOperator opnode,
2887                      ValueType ResTy, ValueType OpTy>
2888   : NeonI_3VDiff<q, u, size, opcode,
2889                  (outs VPR128:$Rd), (ins VPR128:$src, VPR64:$Rn, VPR64:$Rm),
2890                  asmop # "\t$Rd." # ResS # ", $Rn." # OpS # ", $Rm." # OpS,
2891                  [(set (ResTy VPR128:$Rd),
2892                     (ResTy (opnode
2893                       (ResTy VPR128:$src),
2894                       (OpTy VPR64:$Rn), (OpTy VPR64:$Rm))))],
2895                NoItinerary> {
2896   let Constraints = "$src = $Rd";
2897 }
2898
2899 multiclass NeonI_3VDL_3Op_v1<bit u, bits<4> opcode, string asmop,
2900                              SDPatternOperator opnode> {
2901   def _8h8b : NeonI_3VDL_3Op<0b0, u, 0b00, opcode, asmop, "8h", "8b",
2902                              opnode, v8i16, v8i8>;
2903   def _4s4h : NeonI_3VDL_3Op<0b0, u, 0b01, opcode, asmop, "4s", "4h",
2904                              opnode, v4i32, v4i16>;
2905   def _2d2s : NeonI_3VDL_3Op<0b0, u, 0b10, opcode, asmop, "2d", "2s",
2906                              opnode, v2i64, v2i32>;
2907 }
2908
2909 def Neon_smlal : PatFrag<(ops node:$Rd, node:$Rn, node:$Rm),
2910                          (add node:$Rd,
2911                             (int_arm_neon_vmulls node:$Rn, node:$Rm))>;
2912
2913 def Neon_umlal : PatFrag<(ops node:$Rd, node:$Rn, node:$Rm),
2914                          (add node:$Rd,
2915                             (int_arm_neon_vmullu node:$Rn, node:$Rm))>;
2916
2917 def Neon_smlsl : PatFrag<(ops node:$Rd, node:$Rn, node:$Rm),
2918                          (sub node:$Rd,
2919                             (int_arm_neon_vmulls node:$Rn, node:$Rm))>;
2920
2921 def Neon_umlsl : PatFrag<(ops node:$Rd, node:$Rn, node:$Rm),
2922                          (sub node:$Rd,
2923                             (int_arm_neon_vmullu node:$Rn, node:$Rm))>;
2924
2925 defm SMLALvvv :  NeonI_3VDL_3Op_v1<0b0, 0b1000, "smlal", Neon_smlal>;
2926 defm UMLALvvv :  NeonI_3VDL_3Op_v1<0b1, 0b1000, "umlal", Neon_umlal>;
2927
2928 defm SMLSLvvv :  NeonI_3VDL_3Op_v1<0b0, 0b1010, "smlsl", Neon_smlsl>;
2929 defm UMLSLvvv :  NeonI_3VDL_3Op_v1<0b1, 0b1010, "umlsl", Neon_umlsl>;
2930
2931 class NeonI_3VDL2_3Op_mlas<bit q, bit u, bits<2> size, bits<4> opcode,
2932                            string asmop, string ResS, string OpS,
2933                            SDPatternOperator subop, SDPatternOperator opnode,
2934                            RegisterOperand OpVPR,
2935                            ValueType ResTy, ValueType OpTy>
2936   : NeonI_3VDiff<q, u, size, opcode,
2937                (outs VPR128:$Rd), (ins VPR128:$src, OpVPR:$Rn, OpVPR:$Rm),
2938                asmop # "\t$Rd." # ResS # ", $Rn." # OpS # ", $Rm." # OpS,
2939                [(set (ResTy VPR128:$Rd),
2940                   (ResTy (subop
2941                     (ResTy VPR128:$src),
2942                     (ResTy (opnode (OpTy OpVPR:$Rn), (OpTy OpVPR:$Rm))))))],
2943                NoItinerary> {
2944   let Constraints = "$src = $Rd";
2945 }
2946
2947 multiclass NeonI_3VDL2_3Op_mlas_v1<bit u, bits<4> opcode, string asmop,
2948                                    SDPatternOperator subop, string opnode> {
2949   def _8h16b : NeonI_3VDL2_3Op_mlas<0b1, u, 0b00, opcode, asmop, "8h", "16b",
2950                                     subop, !cast<PatFrag>(opnode # "_16B"),
2951                                     VPR128, v8i16, v16i8>;
2952   def _4s8h : NeonI_3VDL2_3Op_mlas<0b1, u, 0b01, opcode, asmop, "4s", "8h",
2953                                    subop, !cast<PatFrag>(opnode # "_8H"),
2954                                    VPR128, v4i32, v8i16>;
2955   def _2d4s : NeonI_3VDL2_3Op_mlas<0b1, u, 0b10, opcode, asmop, "2d", "4s",
2956                                    subop, !cast<PatFrag>(opnode # "_4S"),
2957                                    VPR128, v2i64, v4i32>;
2958 }
2959
2960 defm SMLAL2vvv :  NeonI_3VDL2_3Op_mlas_v1<0b0, 0b1000, "smlal2",
2961                                           add, "NI_smull_hi">;
2962 defm UMLAL2vvv :  NeonI_3VDL2_3Op_mlas_v1<0b1, 0b1000, "umlal2",
2963                                           add, "NI_umull_hi">;
2964
2965 defm SMLSL2vvv :  NeonI_3VDL2_3Op_mlas_v1<0b0, 0b1010, "smlsl2",
2966                                           sub, "NI_smull_hi">;
2967 defm UMLSL2vvv :  NeonI_3VDL2_3Op_mlas_v1<0b1, 0b1010, "umlsl2",
2968                                           sub, "NI_umull_hi">;
2969
2970 multiclass NeonI_3VDL_qdmlal_3Op_v2<bit u, bits<4> opcode, string asmop,
2971                                     SDPatternOperator opnode> {
2972   def _4s4h : NeonI_3VDL2_3Op_mlas<0b0, u, 0b01, opcode, asmop, "4s", "4h",
2973                                    opnode, int_arm_neon_vqdmull,
2974                                    VPR64, v4i32, v4i16>;
2975   def _2d2s : NeonI_3VDL2_3Op_mlas<0b0, u, 0b10, opcode, asmop, "2d", "2s",
2976                                    opnode, int_arm_neon_vqdmull,
2977                                    VPR64, v2i64, v2i32>;
2978 }
2979
2980 defm SQDMLALvvv : NeonI_3VDL_qdmlal_3Op_v2<0b0, 0b1001, "sqdmlal",
2981                                            int_arm_neon_vqadds>;
2982 defm SQDMLSLvvv : NeonI_3VDL_qdmlal_3Op_v2<0b0, 0b1011, "sqdmlsl",
2983                                            int_arm_neon_vqsubs>;
2984
2985 multiclass NeonI_3VDL_v2<bit u, bits<4> opcode, string asmop,
2986                          SDPatternOperator opnode, bit Commutable = 0> {
2987   let isCommutable = Commutable in {
2988     def _4s4h : NeonI_3VD_2Op<0b0, u, 0b01, opcode, asmop, "4s", "4h",
2989                               opnode, VPR128, VPR64, v4i32, v4i16>;
2990     def _2d2s : NeonI_3VD_2Op<0b0, u, 0b10, opcode, asmop, "2d", "2s",
2991                               opnode, VPR128, VPR64, v2i64, v2i32>;
2992   }
2993 }
2994
2995 defm SQDMULLvvv : NeonI_3VDL_v2<0b0, 0b1101, "sqdmull",
2996                                 int_arm_neon_vqdmull, 1>;
2997
2998 multiclass NeonI_3VDL2_2Op_mull_v2<bit u, bits<4> opcode, string asmop,
2999                                    string opnode, bit Commutable = 0> {
3000   let isCommutable = Commutable in {
3001     def _4s8h : NeonI_3VDL2_2Op_mull<0b1, u, 0b01, opcode, asmop, "4s", "8h",
3002                                      !cast<PatFrag>(opnode # "_8H"),
3003                                      v4i32, v8i16>;
3004     def _2d4s : NeonI_3VDL2_2Op_mull<0b1, u, 0b10, opcode, asmop, "2d", "4s",
3005                                      !cast<PatFrag>(opnode # "_4S"),
3006                                      v2i64, v4i32>;
3007   }
3008 }
3009
3010 defm SQDMULL2vvv : NeonI_3VDL2_2Op_mull_v2<0b0, 0b1101, "sqdmull2",
3011                                            "NI_qdmull_hi", 1>;
3012
3013 multiclass NeonI_3VDL2_3Op_qdmlal_v2<bit u, bits<4> opcode, string asmop,
3014                                      SDPatternOperator opnode> {
3015   def _4s8h : NeonI_3VDL2_3Op_mlas<0b1, u, 0b01, opcode, asmop, "4s", "8h",
3016                                    opnode, NI_qdmull_hi_8H,
3017                                    VPR128, v4i32, v8i16>;
3018   def _2d4s : NeonI_3VDL2_3Op_mlas<0b1, u, 0b10, opcode, asmop, "2d", "4s",
3019                                    opnode, NI_qdmull_hi_4S,
3020                                    VPR128, v2i64, v4i32>;
3021 }
3022
3023 defm SQDMLAL2vvv : NeonI_3VDL2_3Op_qdmlal_v2<0b0, 0b1001, "sqdmlal2",
3024                                              int_arm_neon_vqadds>;
3025 defm SQDMLSL2vvv : NeonI_3VDL2_3Op_qdmlal_v2<0b0, 0b1011, "sqdmlsl2",
3026                                              int_arm_neon_vqsubs>;
3027
3028 multiclass NeonI_3VDL_v3<bit u, bits<4> opcode, string asmop,
3029                          SDPatternOperator opnode, bit Commutable = 0> {
3030   let isCommutable = Commutable in {
3031     def _8h8b : NeonI_3VD_2Op<0b0, u, 0b00, opcode, asmop, "8h", "8b",
3032                               opnode, VPR128, VPR64, v8i16, v8i8>;
3033
3034     def _1q1d : NeonI_3VDiff<0b0, u, 0b11, opcode,
3035                              (outs VPR128:$Rd), (ins VPR64:$Rn, VPR64:$Rm),
3036                              asmop # "\t$Rd.1q, $Rn.1d, $Rm.1d",
3037                              [], NoItinerary>;
3038   }
3039 }
3040
3041 defm PMULLvvv : NeonI_3VDL_v3<0b0, 0b1110, "pmull", int_arm_neon_vmullp, 1>;
3042
3043 multiclass NeonI_3VDL2_2Op_mull_v3<bit u, bits<4> opcode, string asmop,
3044                                    string opnode, bit Commutable = 0> {
3045   let isCommutable = Commutable in {
3046     def _8h16b : NeonI_3VDL2_2Op_mull<0b1, u, 0b00, opcode, asmop, "8h", "16b",
3047                                       !cast<PatFrag>(opnode # "_16B"),
3048                                       v8i16, v16i8>;
3049
3050     def _1q2d : NeonI_3VDiff<0b1, u, 0b11, opcode,
3051                              (outs VPR128:$Rd), (ins VPR128:$Rn, VPR128:$Rm),
3052                              asmop # "\t$Rd.1q, $Rn.2d, $Rm.2d",
3053                              [], NoItinerary>;
3054   }
3055 }
3056
3057 defm PMULL2vvv : NeonI_3VDL2_2Op_mull_v3<0b0, 0b1110, "pmull2", "NI_pmull_hi",
3058                                          1>;
3059
3060 // End of implementation for instruction class (3V Diff)
3061
3062 // The followings are vector load/store multiple N-element structure
3063 // (class SIMD lselem).
3064
3065 // ld1:         load multiple 1-element structure to 1/2/3/4 registers.
3066 // ld2/ld3/ld4: load multiple N-element structure to N registers (N = 2, 3, 4).
3067 //              The structure consists of a sequence of sets of N values.
3068 //              The first element of the structure is placed in the first lane
3069 //              of the first first vector, the second element in the first lane
3070 //              of the second vector, and so on.
3071 // E.g. LD1_3V_2S will load 32-bit elements {A, B, C, D, E, F} sequentially into
3072 // the three 64-bit vectors list {BA, DC, FE}.
3073 // E.g. LD3_2S will load 32-bit elements {A, B, C, D, E, F} into the three
3074 // 64-bit vectors list {DA, EB, FC}.
3075 // Store instructions store multiple structure to N registers like load.
3076
3077
3078 class NeonI_LDVList<bit q, bits<4> opcode, bits<2> size,
3079                     RegisterOperand VecList, string asmop>
3080   : NeonI_LdStMult<q, 1, opcode, size,
3081                  (outs VecList:$Rt), (ins GPR64xsp:$Rn),
3082                  asmop # "\t$Rt, [$Rn]",
3083                  [],
3084                  NoItinerary> {
3085   let mayLoad = 1;
3086   let neverHasSideEffects = 1;
3087 }
3088
3089 multiclass LDVList_BHSD<bits<4> opcode, string List, string asmop> {
3090   def _8B : NeonI_LDVList<0, opcode, 0b00,
3091                           !cast<RegisterOperand>(List # "8B_operand"), asmop>;
3092
3093   def _4H : NeonI_LDVList<0, opcode, 0b01,
3094                           !cast<RegisterOperand>(List # "4H_operand"), asmop>;
3095
3096   def _2S : NeonI_LDVList<0, opcode, 0b10,
3097                           !cast<RegisterOperand>(List # "2S_operand"), asmop>;
3098
3099   def _16B : NeonI_LDVList<1, opcode, 0b00,
3100                            !cast<RegisterOperand>(List # "16B_operand"), asmop>;
3101
3102   def _8H : NeonI_LDVList<1, opcode, 0b01,
3103                           !cast<RegisterOperand>(List # "8H_operand"), asmop>;
3104
3105   def _4S : NeonI_LDVList<1, opcode, 0b10,
3106                           !cast<RegisterOperand>(List # "4S_operand"), asmop>;
3107
3108   def _2D : NeonI_LDVList<1, opcode, 0b11,
3109                           !cast<RegisterOperand>(List # "2D_operand"), asmop>;
3110 }
3111
3112 // Load multiple N-element structure to N consecutive registers (N = 1,2,3,4)
3113 defm LD1 : LDVList_BHSD<0b0111, "VOne", "ld1">;
3114 def LD1_1D : NeonI_LDVList<0, 0b0111, 0b11, VOne1D_operand, "ld1">;
3115
3116 defm LD2 : LDVList_BHSD<0b1000, "VPair", "ld2">;
3117
3118 defm LD3 : LDVList_BHSD<0b0100, "VTriple", "ld3">;
3119
3120 defm LD4 : LDVList_BHSD<0b0000, "VQuad", "ld4">;
3121
3122 // Load multiple 1-element structure to N consecutive registers (N = 2,3,4)
3123 defm LD1x2 : LDVList_BHSD<0b1010, "VPair", "ld1">;
3124 def LD1x2_1D : NeonI_LDVList<0, 0b1010, 0b11, VPair1D_operand, "ld1">;
3125
3126 defm LD1x3 : LDVList_BHSD<0b0110, "VTriple", "ld1">;
3127 def LD1x3_1D : NeonI_LDVList<0, 0b0110, 0b11, VTriple1D_operand, "ld1">;
3128
3129 defm LD1x4 : LDVList_BHSD<0b0010, "VQuad", "ld1">;
3130 def LD1x4_1D : NeonI_LDVList<0, 0b0010, 0b11, VQuad1D_operand, "ld1">;
3131
3132 class NeonI_STVList<bit q, bits<4> opcode, bits<2> size,
3133                     RegisterOperand VecList, string asmop>
3134   : NeonI_LdStMult<q, 0, opcode, size,
3135                  (outs), (ins GPR64xsp:$Rn, VecList:$Rt),
3136                  asmop # "\t$Rt, [$Rn]",
3137                  [],
3138                  NoItinerary> {
3139   let mayStore = 1;
3140   let neverHasSideEffects = 1;
3141 }
3142
3143 multiclass STVList_BHSD<bits<4> opcode, string List, string asmop> {
3144   def _8B : NeonI_STVList<0, opcode, 0b00,
3145                           !cast<RegisterOperand>(List # "8B_operand"), asmop>;
3146
3147   def _4H : NeonI_STVList<0, opcode, 0b01,
3148                           !cast<RegisterOperand>(List # "4H_operand"), asmop>;
3149
3150   def _2S : NeonI_STVList<0, opcode, 0b10,
3151                           !cast<RegisterOperand>(List # "2S_operand"), asmop>;
3152
3153   def _16B : NeonI_STVList<1, opcode, 0b00,
3154                            !cast<RegisterOperand>(List # "16B_operand"), asmop>;
3155
3156   def _8H : NeonI_STVList<1, opcode, 0b01,
3157                           !cast<RegisterOperand>(List # "8H_operand"), asmop>;
3158
3159   def _4S : NeonI_STVList<1, opcode, 0b10,
3160                           !cast<RegisterOperand>(List # "4S_operand"), asmop>;
3161
3162   def _2D : NeonI_STVList<1, opcode, 0b11,
3163                           !cast<RegisterOperand>(List # "2D_operand"), asmop>;
3164 }
3165
3166 // Store multiple N-element structures from N registers (N = 1,2,3,4)
3167 defm ST1 : STVList_BHSD<0b0111, "VOne", "st1">;
3168 def ST1_1D : NeonI_STVList<0, 0b0111, 0b11, VOne1D_operand, "st1">;
3169
3170 defm ST2 : STVList_BHSD<0b1000, "VPair", "st2">;
3171
3172 defm ST3 : STVList_BHSD<0b0100, "VTriple", "st3">;
3173
3174 defm ST4 : STVList_BHSD<0b0000, "VQuad", "st4">;
3175
3176 // Store multiple 1-element structures from N consecutive registers (N = 2,3,4)
3177 defm ST1x2 : STVList_BHSD<0b1010, "VPair", "st1">;
3178 def ST1x2_1D : NeonI_STVList<0, 0b1010, 0b11, VPair1D_operand, "st1">;
3179
3180 defm ST1x3 : STVList_BHSD<0b0110, "VTriple", "st1">;
3181 def ST1x3_1D : NeonI_STVList<0, 0b0110, 0b11, VTriple1D_operand, "st1">;
3182
3183 defm ST1x4 : STVList_BHSD<0b0010, "VQuad", "st1">;
3184 def ST1x4_1D : NeonI_STVList<0, 0b0010, 0b11, VQuad1D_operand, "st1">;
3185
3186 def : Pat<(v2f64 (load GPR64xsp:$addr)), (LD1_2D GPR64xsp:$addr)>;
3187 def : Pat<(v2i64 (load GPR64xsp:$addr)), (LD1_2D GPR64xsp:$addr)>;
3188
3189 def : Pat<(v4f32 (load GPR64xsp:$addr)), (LD1_4S GPR64xsp:$addr)>;
3190 def : Pat<(v4i32 (load GPR64xsp:$addr)), (LD1_4S GPR64xsp:$addr)>;
3191
3192 def : Pat<(v8i16 (load GPR64xsp:$addr)), (LD1_8H GPR64xsp:$addr)>;
3193 def : Pat<(v16i8 (load GPR64xsp:$addr)), (LD1_16B GPR64xsp:$addr)>;
3194
3195 def : Pat<(v1f64 (load GPR64xsp:$addr)), (LD1_1D GPR64xsp:$addr)>;
3196 def : Pat<(v1i64 (load GPR64xsp:$addr)), (LD1_1D GPR64xsp:$addr)>;
3197
3198 def : Pat<(v2f32 (load GPR64xsp:$addr)), (LD1_2S GPR64xsp:$addr)>;
3199 def : Pat<(v2i32 (load GPR64xsp:$addr)), (LD1_2S GPR64xsp:$addr)>;
3200
3201 def : Pat<(v4i16 (load GPR64xsp:$addr)), (LD1_4H GPR64xsp:$addr)>;
3202 def : Pat<(v8i8 (load GPR64xsp:$addr)), (LD1_8B GPR64xsp:$addr)>;
3203
3204 def : Pat<(store (v2i64 VPR128:$value), GPR64xsp:$addr),
3205           (ST1_2D GPR64xsp:$addr, VPR128:$value)>;
3206 def : Pat<(store (v2f64 VPR128:$value), GPR64xsp:$addr),
3207           (ST1_2D GPR64xsp:$addr, VPR128:$value)>;
3208
3209 def : Pat<(store (v4i32 VPR128:$value), GPR64xsp:$addr),
3210           (ST1_4S GPR64xsp:$addr, VPR128:$value)>;
3211 def : Pat<(store (v4f32 VPR128:$value), GPR64xsp:$addr),
3212           (ST1_4S GPR64xsp:$addr, VPR128:$value)>;
3213
3214 def : Pat<(store (v8i16 VPR128:$value), GPR64xsp:$addr),
3215           (ST1_8H GPR64xsp:$addr, VPR128:$value)>;
3216 def : Pat<(store (v16i8 VPR128:$value), GPR64xsp:$addr),
3217           (ST1_16B GPR64xsp:$addr, VPR128:$value)>;
3218
3219 def : Pat<(store (v1i64 VPR64:$value), GPR64xsp:$addr),
3220           (ST1_1D GPR64xsp:$addr, VPR64:$value)>;
3221 def : Pat<(store (v1f64 VPR64:$value), GPR64xsp:$addr),
3222           (ST1_1D GPR64xsp:$addr, VPR64:$value)>;
3223
3224 def : Pat<(store (v2i32 VPR64:$value), GPR64xsp:$addr),
3225           (ST1_2S GPR64xsp:$addr, VPR64:$value)>;
3226 def : Pat<(store (v2f32 VPR64:$value), GPR64xsp:$addr),
3227           (ST1_2S GPR64xsp:$addr, VPR64:$value)>;
3228
3229 def : Pat<(store (v4i16 VPR64:$value), GPR64xsp:$addr),
3230           (ST1_4H GPR64xsp:$addr, VPR64:$value)>;
3231 def : Pat<(store (v8i8 VPR64:$value), GPR64xsp:$addr),
3232           (ST1_8B GPR64xsp:$addr, VPR64:$value)>;
3233
3234 // End of vector load/store multiple N-element structure(class SIMD lselem)
3235
3236 // The followings are post-index vector load/store multiple N-element
3237 // structure(class SIMD lselem-post)
3238 def exact1_asmoperand : AsmOperandClass {
3239   let Name = "Exact1";
3240   let PredicateMethod = "isExactImm<1>";
3241   let RenderMethod = "addImmOperands";
3242 }
3243 def uimm_exact1 : Operand<i32>, ImmLeaf<i32, [{return Imm == 1;}]> {
3244   let ParserMatchClass = exact1_asmoperand;
3245 }
3246
3247 def exact2_asmoperand : AsmOperandClass {
3248   let Name = "Exact2";
3249   let PredicateMethod = "isExactImm<2>";
3250   let RenderMethod = "addImmOperands";
3251 }
3252 def uimm_exact2 : Operand<i32>, ImmLeaf<i32, [{return Imm == 2;}]> {
3253   let ParserMatchClass = exact2_asmoperand;
3254 }
3255
3256 def exact3_asmoperand : AsmOperandClass {
3257   let Name = "Exact3";
3258   let PredicateMethod = "isExactImm<3>";
3259   let RenderMethod = "addImmOperands";
3260 }
3261 def uimm_exact3 : Operand<i32>, ImmLeaf<i32, [{return Imm == 3;}]> {
3262   let ParserMatchClass = exact3_asmoperand;
3263 }
3264
3265 def exact4_asmoperand : AsmOperandClass {
3266   let Name = "Exact4";
3267   let PredicateMethod = "isExactImm<4>";
3268   let RenderMethod = "addImmOperands";
3269 }
3270 def uimm_exact4 : Operand<i32>, ImmLeaf<i32, [{return Imm == 4;}]> {
3271   let ParserMatchClass = exact4_asmoperand;
3272 }
3273
3274 def exact6_asmoperand : AsmOperandClass {
3275   let Name = "Exact6";
3276   let PredicateMethod = "isExactImm<6>";
3277   let RenderMethod = "addImmOperands";
3278 }
3279 def uimm_exact6 : Operand<i32>, ImmLeaf<i32, [{return Imm == 6;}]> {
3280   let ParserMatchClass = exact6_asmoperand;
3281 }
3282
3283 def exact8_asmoperand : AsmOperandClass {
3284   let Name = "Exact8";
3285   let PredicateMethod = "isExactImm<8>";
3286   let RenderMethod = "addImmOperands";
3287 }
3288 def uimm_exact8 : Operand<i32>, ImmLeaf<i32, [{return Imm == 8;}]> {
3289   let ParserMatchClass = exact8_asmoperand;
3290 }
3291
3292 def exact12_asmoperand : AsmOperandClass {
3293   let Name = "Exact12";
3294   let PredicateMethod = "isExactImm<12>";
3295   let RenderMethod = "addImmOperands";
3296 }
3297 def uimm_exact12 : Operand<i32>, ImmLeaf<i32, [{return Imm == 12;}]> {
3298   let ParserMatchClass = exact12_asmoperand;
3299 }
3300
3301 def exact16_asmoperand : AsmOperandClass {
3302   let Name = "Exact16";
3303   let PredicateMethod = "isExactImm<16>";
3304   let RenderMethod = "addImmOperands";
3305 }
3306 def uimm_exact16 : Operand<i32>, ImmLeaf<i32, [{return Imm == 16;}]> {
3307   let ParserMatchClass = exact16_asmoperand;
3308 }
3309
3310 def exact24_asmoperand : AsmOperandClass {
3311   let Name = "Exact24";
3312   let PredicateMethod = "isExactImm<24>";
3313   let RenderMethod = "addImmOperands";
3314 }
3315 def uimm_exact24 : Operand<i32>, ImmLeaf<i32, [{return Imm == 24;}]> {
3316   let ParserMatchClass = exact24_asmoperand;
3317 }
3318
3319 def exact32_asmoperand : AsmOperandClass {
3320   let Name = "Exact32";
3321   let PredicateMethod = "isExactImm<32>";
3322   let RenderMethod = "addImmOperands";
3323 }
3324 def uimm_exact32 : Operand<i32>, ImmLeaf<i32, [{return Imm == 32;}]> {
3325   let ParserMatchClass = exact32_asmoperand;
3326 }
3327
3328 def exact48_asmoperand : AsmOperandClass {
3329   let Name = "Exact48";
3330   let PredicateMethod = "isExactImm<48>";
3331   let RenderMethod = "addImmOperands";
3332 }
3333 def uimm_exact48 : Operand<i32>, ImmLeaf<i32, [{return Imm == 48;}]> {
3334   let ParserMatchClass = exact48_asmoperand;
3335 }
3336
3337 def exact64_asmoperand : AsmOperandClass {
3338   let Name = "Exact64";
3339   let PredicateMethod = "isExactImm<64>";
3340   let RenderMethod = "addImmOperands";
3341 }
3342 def uimm_exact64 : Operand<i32>, ImmLeaf<i32, [{return Imm == 64;}]> {
3343   let ParserMatchClass = exact64_asmoperand;
3344 }
3345
3346 multiclass NeonI_LDWB_VList<bit q, bits<4> opcode, bits<2> size,
3347                            RegisterOperand VecList, Operand ImmTy,
3348                            string asmop> {
3349   let Constraints = "$Rn = $wb", mayLoad = 1, neverHasSideEffects = 1,
3350       DecoderMethod = "DecodeVLDSTPostInstruction" in {
3351     def _fixed : NeonI_LdStMult_Post<q, 1, opcode, size,
3352                      (outs VecList:$Rt, GPR64xsp:$wb),
3353                      (ins GPR64xsp:$Rn, ImmTy:$amt),
3354                      asmop # "\t$Rt, [$Rn], $amt",
3355                      [],
3356                      NoItinerary> {
3357       let Rm = 0b11111;
3358     }
3359
3360     def _register : NeonI_LdStMult_Post<q, 1, opcode, size,
3361                         (outs VecList:$Rt, GPR64xsp:$wb),
3362                         (ins GPR64xsp:$Rn, GPR64noxzr:$Rm),
3363                         asmop # "\t$Rt, [$Rn], $Rm",
3364                         [],
3365                         NoItinerary>;
3366   }
3367 }
3368
3369 multiclass LDWB_VList_BHSD<bits<4> opcode, string List, Operand ImmTy,
3370     Operand ImmTy2, string asmop> {
3371   defm _8B : NeonI_LDWB_VList<0, opcode, 0b00,
3372                               !cast<RegisterOperand>(List # "8B_operand"),
3373                               ImmTy, asmop>;
3374
3375   defm _4H : NeonI_LDWB_VList<0, opcode, 0b01,
3376                               !cast<RegisterOperand>(List # "4H_operand"),
3377                               ImmTy, asmop>;
3378
3379   defm _2S : NeonI_LDWB_VList<0, opcode, 0b10,
3380                               !cast<RegisterOperand>(List # "2S_operand"),
3381                               ImmTy, asmop>;
3382
3383   defm _16B : NeonI_LDWB_VList<1, opcode, 0b00,
3384                                !cast<RegisterOperand>(List # "16B_operand"),
3385                                ImmTy2, asmop>;
3386
3387   defm _8H : NeonI_LDWB_VList<1, opcode, 0b01,
3388                               !cast<RegisterOperand>(List # "8H_operand"),
3389                               ImmTy2, asmop>;
3390
3391   defm _4S : NeonI_LDWB_VList<1, opcode, 0b10,
3392                               !cast<RegisterOperand>(List # "4S_operand"),
3393                               ImmTy2, asmop>;
3394
3395   defm _2D : NeonI_LDWB_VList<1, opcode, 0b11,
3396                               !cast<RegisterOperand>(List # "2D_operand"),
3397                               ImmTy2, asmop>;
3398 }
3399
3400 // Post-index load multiple N-element structures from N registers (N = 1,2,3,4)
3401 defm LD1WB : LDWB_VList_BHSD<0b0111, "VOne", uimm_exact8, uimm_exact16, "ld1">;
3402 defm LD1WB_1D : NeonI_LDWB_VList<0, 0b0111, 0b11, VOne1D_operand, uimm_exact8,
3403                                  "ld1">;
3404
3405 defm LD2WB : LDWB_VList_BHSD<0b1000, "VPair", uimm_exact16, uimm_exact32, "ld2">;
3406
3407 defm LD3WB : LDWB_VList_BHSD<0b0100, "VTriple", uimm_exact24, uimm_exact48,
3408                              "ld3">;
3409
3410 defm LD4WB : LDWB_VList_BHSD<0b0000, "VQuad", uimm_exact32, uimm_exact64, "ld4">;
3411
3412 // Post-index load multiple 1-element structures from N consecutive registers
3413 // (N = 2,3,4)
3414 defm LD1x2WB : LDWB_VList_BHSD<0b1010, "VPair", uimm_exact16, uimm_exact32,
3415                                "ld1">;
3416 defm LD1x2WB_1D : NeonI_LDWB_VList<0, 0b1010, 0b11, VPair1D_operand,
3417                                    uimm_exact16, "ld1">;
3418
3419 defm LD1x3WB : LDWB_VList_BHSD<0b0110, "VTriple", uimm_exact24, uimm_exact48,
3420                                "ld1">;
3421 defm LD1x3WB_1D : NeonI_LDWB_VList<0, 0b0110, 0b11, VTriple1D_operand,
3422                                    uimm_exact24, "ld1">;
3423
3424 defm LD1x4WB : LDWB_VList_BHSD<0b0010, "VQuad", uimm_exact32, uimm_exact64,
3425                                 "ld1">;
3426 defm LD1x4WB_1D : NeonI_LDWB_VList<0, 0b0010, 0b11, VQuad1D_operand,
3427                                    uimm_exact32, "ld1">;
3428
3429 multiclass NeonI_STWB_VList<bit q, bits<4> opcode, bits<2> size,
3430                             RegisterOperand VecList, Operand ImmTy,
3431                             string asmop> {
3432   let Constraints = "$Rn = $wb", mayStore = 1, neverHasSideEffects = 1,
3433       DecoderMethod = "DecodeVLDSTPostInstruction" in {
3434     def _fixed : NeonI_LdStMult_Post<q, 0, opcode, size,
3435                      (outs GPR64xsp:$wb),
3436                      (ins GPR64xsp:$Rn, ImmTy:$amt, VecList:$Rt),
3437                      asmop # "\t$Rt, [$Rn], $amt",
3438                      [],
3439                      NoItinerary> {
3440       let Rm = 0b11111;
3441     }
3442
3443     def _register : NeonI_LdStMult_Post<q, 0, opcode, size,
3444                       (outs GPR64xsp:$wb),
3445                       (ins GPR64xsp:$Rn, GPR64noxzr:$Rm, VecList:$Rt),
3446                       asmop # "\t$Rt, [$Rn], $Rm",
3447                       [],
3448                       NoItinerary>;
3449   }
3450 }
3451
3452 multiclass STWB_VList_BHSD<bits<4> opcode, string List, Operand ImmTy,
3453                            Operand ImmTy2, string asmop> {
3454   defm _8B : NeonI_STWB_VList<0, opcode, 0b00,
3455                  !cast<RegisterOperand>(List # "8B_operand"), ImmTy, asmop>;
3456
3457   defm _4H : NeonI_STWB_VList<0, opcode, 0b01,
3458                               !cast<RegisterOperand>(List # "4H_operand"),
3459                               ImmTy, asmop>;
3460
3461   defm _2S : NeonI_STWB_VList<0, opcode, 0b10,
3462                               !cast<RegisterOperand>(List # "2S_operand"),
3463                               ImmTy, asmop>;
3464
3465   defm _16B : NeonI_STWB_VList<1, opcode, 0b00,
3466                                !cast<RegisterOperand>(List # "16B_operand"),
3467                                ImmTy2, asmop>;
3468
3469   defm _8H : NeonI_STWB_VList<1, opcode, 0b01,
3470                               !cast<RegisterOperand>(List # "8H_operand"),
3471                               ImmTy2, asmop>;
3472
3473   defm _4S : NeonI_STWB_VList<1, opcode, 0b10,
3474                               !cast<RegisterOperand>(List # "4S_operand"),
3475                               ImmTy2, asmop>;
3476
3477   defm _2D : NeonI_STWB_VList<1, opcode, 0b11,
3478                               !cast<RegisterOperand>(List # "2D_operand"),
3479                               ImmTy2, asmop>;
3480 }
3481
3482 // Post-index load multiple N-element structures from N registers (N = 1,2,3,4)
3483 defm ST1WB : STWB_VList_BHSD<0b0111, "VOne", uimm_exact8, uimm_exact16, "st1">;
3484 defm ST1WB_1D : NeonI_STWB_VList<0, 0b0111, 0b11, VOne1D_operand, uimm_exact8,
3485                                  "st1">;
3486
3487 defm ST2WB : STWB_VList_BHSD<0b1000, "VPair", uimm_exact16, uimm_exact32, "st2">;
3488
3489 defm ST3WB : STWB_VList_BHSD<0b0100, "VTriple", uimm_exact24, uimm_exact48,
3490                              "st3">;
3491
3492 defm ST4WB : STWB_VList_BHSD<0b0000, "VQuad", uimm_exact32, uimm_exact64, "st4">;
3493
3494 // Post-index load multiple 1-element structures from N consecutive registers
3495 // (N = 2,3,4)
3496 defm ST1x2WB : STWB_VList_BHSD<0b1010, "VPair", uimm_exact16, uimm_exact32,
3497                                "st1">;
3498 defm ST1x2WB_1D : NeonI_STWB_VList<0, 0b1010, 0b11, VPair1D_operand,
3499                                    uimm_exact16, "st1">;
3500
3501 defm ST1x3WB : STWB_VList_BHSD<0b0110, "VTriple", uimm_exact24, uimm_exact48,
3502                                "st1">;
3503 defm ST1x3WB_1D : NeonI_STWB_VList<0, 0b0110, 0b11, VTriple1D_operand,
3504                                    uimm_exact24, "st1">;
3505
3506 defm ST1x4WB : STWB_VList_BHSD<0b0010, "VQuad", uimm_exact32, uimm_exact64,
3507                                "st1">;
3508 defm ST1x4WB_1D : NeonI_STWB_VList<0, 0b0010, 0b11, VQuad1D_operand,
3509                                    uimm_exact32, "st1">;
3510
3511 // End of post-index vector load/store multiple N-element structure
3512 // (class SIMD lselem-post)
3513
3514 // The followings are vector load/store single N-element structure
3515 // (class SIMD lsone).
3516 def neon_uimm0_bare : Operand<i64>,
3517                         ImmLeaf<i64, [{return Imm == 0;}]> {
3518   let ParserMatchClass = neon_uimm0_asmoperand;
3519   let PrintMethod = "printUImmBareOperand";
3520 }
3521
3522 def neon_uimm1_bare : Operand<i64>,
3523                         ImmLeaf<i64, [{return Imm < 2;}]> {
3524   let ParserMatchClass = neon_uimm1_asmoperand;
3525   let PrintMethod = "printUImmBareOperand";
3526 }
3527
3528 def neon_uimm2_bare : Operand<i64>,
3529                         ImmLeaf<i64, [{return Imm < 4;}]> {
3530   let ParserMatchClass = neon_uimm2_asmoperand;
3531   let PrintMethod = "printUImmBareOperand";
3532 }
3533
3534 def neon_uimm3_bare : Operand<i64>,
3535                         ImmLeaf<i64, [{return Imm < 8;}]> {
3536   let ParserMatchClass = uimm3_asmoperand;
3537   let PrintMethod = "printUImmBareOperand";
3538 }
3539
3540 def neon_uimm4_bare : Operand<i64>,
3541                         ImmLeaf<i64, [{return Imm < 16;}]> {
3542   let ParserMatchClass = uimm4_asmoperand;
3543   let PrintMethod = "printUImmBareOperand";
3544 }
3545
3546 class NeonI_LDN_Dup<bit q, bit r, bits<3> opcode, bits<2> size,
3547                     RegisterOperand VecList, string asmop>
3548     : NeonI_LdOne_Dup<q, r, opcode, size,
3549                       (outs VecList:$Rt), (ins GPR64xsp:$Rn),
3550                       asmop # "\t$Rt, [$Rn]",
3551                       [],
3552                       NoItinerary> {
3553   let mayLoad = 1;
3554   let neverHasSideEffects = 1;
3555 }
3556
3557 multiclass LDN_Dup_BHSD<bit r, bits<3> opcode, string List, string asmop> {
3558   def _8B : NeonI_LDN_Dup<0, r, opcode, 0b00,
3559                           !cast<RegisterOperand>(List # "8B_operand"), asmop>;
3560
3561   def _4H : NeonI_LDN_Dup<0, r, opcode, 0b01,
3562                           !cast<RegisterOperand>(List # "4H_operand"), asmop>;
3563
3564   def _2S : NeonI_LDN_Dup<0, r, opcode, 0b10,
3565                           !cast<RegisterOperand>(List # "2S_operand"), asmop>;
3566
3567   def _1D : NeonI_LDN_Dup<0, r, opcode, 0b11,
3568                           !cast<RegisterOperand>(List # "1D_operand"), asmop>;
3569
3570   def _16B : NeonI_LDN_Dup<1, r, opcode, 0b00,
3571                            !cast<RegisterOperand>(List # "16B_operand"), asmop>;
3572
3573   def _8H : NeonI_LDN_Dup<1, r, opcode, 0b01,
3574                           !cast<RegisterOperand>(List # "8H_operand"), asmop>;
3575
3576   def _4S : NeonI_LDN_Dup<1, r, opcode, 0b10,
3577                           !cast<RegisterOperand>(List # "4S_operand"), asmop>;
3578
3579   def _2D : NeonI_LDN_Dup<1, r, opcode, 0b11,
3580                           !cast<RegisterOperand>(List # "2D_operand"), asmop>;
3581 }
3582
3583 // Load single 1-element structure to all lanes of 1 register
3584 defm LD1R : LDN_Dup_BHSD<0b0, 0b110, "VOne", "ld1r">;
3585
3586 // Load single N-element structure to all lanes of N consecutive
3587 // registers (N = 2,3,4)
3588 defm LD2R : LDN_Dup_BHSD<0b1, 0b110, "VPair", "ld2r">;
3589 defm LD3R : LDN_Dup_BHSD<0b0, 0b111, "VTriple", "ld3r">;
3590 defm LD4R : LDN_Dup_BHSD<0b1, 0b111, "VQuad", "ld4r">;
3591
3592
3593 class LD1R_pattern <ValueType VTy, ValueType DTy, PatFrag LoadOp,
3594                     Instruction INST>
3595     : Pat<(VTy (Neon_vdup (DTy (LoadOp GPR64xsp:$Rn)))),
3596           (VTy (INST GPR64xsp:$Rn))>;
3597
3598 // Match all LD1R instructions
3599 def : LD1R_pattern<v8i8, i32, extloadi8, LD1R_8B>;
3600
3601 def : LD1R_pattern<v16i8, i32, extloadi8, LD1R_16B>;
3602
3603 def : LD1R_pattern<v4i16, i32, extloadi16, LD1R_4H>;
3604
3605 def : LD1R_pattern<v8i16, i32, extloadi16, LD1R_8H>;
3606
3607 def : LD1R_pattern<v2i32, i32, load, LD1R_2S>;
3608 def : LD1R_pattern<v2f32, f32, load, LD1R_2S>;
3609
3610 def : LD1R_pattern<v4i32, i32, load, LD1R_4S>;
3611 def : LD1R_pattern<v4f32, f32, load, LD1R_4S>;
3612
3613 def : LD1R_pattern<v1i64, i64, load, LD1R_1D>;
3614 def : LD1R_pattern<v1f64, f64, load, LD1R_1D>;
3615
3616 def : LD1R_pattern<v2i64, i64, load, LD1R_2D>;
3617 def : LD1R_pattern<v2f64, f64, load, LD1R_2D>;
3618
3619
3620 multiclass VectorList_Bare_BHSD<string PREFIX, int Count,
3621                                 RegisterClass RegList> {
3622   defm B : VectorList_operands<PREFIX, "B", Count, RegList>;
3623   defm H : VectorList_operands<PREFIX, "H", Count, RegList>;
3624   defm S : VectorList_operands<PREFIX, "S", Count, RegList>;
3625   defm D : VectorList_operands<PREFIX, "D", Count, RegList>;
3626 }
3627
3628 // Special vector list operand of 128-bit vectors with bare layout.
3629 // i.e. only show ".b", ".h", ".s", ".d"
3630 defm VOne : VectorList_Bare_BHSD<"VOne", 1, FPR128>;
3631 defm VPair : VectorList_Bare_BHSD<"VPair", 2, QPair>;
3632 defm VTriple : VectorList_Bare_BHSD<"VTriple", 3, QTriple>;
3633 defm VQuad : VectorList_Bare_BHSD<"VQuad", 4, QQuad>;
3634
3635 class NeonI_LDN_Lane<bit r, bits<2> op2_1, bit op0, RegisterOperand VList,
3636                      Operand ImmOp, string asmop>
3637     : NeonI_LdStOne_Lane<1, r, op2_1, op0,
3638                          (outs VList:$Rt),
3639                          (ins GPR64xsp:$Rn, VList:$src, ImmOp:$lane),
3640                          asmop # "\t$Rt[$lane], [$Rn]",
3641                          [],
3642                          NoItinerary> {
3643   let mayLoad = 1;
3644   let neverHasSideEffects = 1;
3645   let hasExtraDefRegAllocReq = 1;
3646   let Constraints = "$src = $Rt";
3647 }
3648
3649 multiclass LDN_Lane_BHSD<bit r, bit op0, string List, string asmop> {
3650   def _B : NeonI_LDN_Lane<r, 0b00, op0,
3651                           !cast<RegisterOperand>(List # "B_operand"),
3652                           neon_uimm4_bare, asmop> {
3653     let Inst{12-10} = lane{2-0};
3654     let Inst{30} = lane{3};
3655   }
3656
3657   def _H : NeonI_LDN_Lane<r, 0b01, op0,
3658                           !cast<RegisterOperand>(List # "H_operand"),
3659                           neon_uimm3_bare, asmop> {
3660     let Inst{12-10} = {lane{1}, lane{0}, 0b0};
3661     let Inst{30} = lane{2};
3662   }
3663
3664   def _S : NeonI_LDN_Lane<r, 0b10, op0,
3665                           !cast<RegisterOperand>(List # "S_operand"),
3666                           neon_uimm2_bare, asmop> {
3667     let Inst{12-10} = {lane{0}, 0b0, 0b0};
3668     let Inst{30} = lane{1};
3669   }
3670
3671   def _D : NeonI_LDN_Lane<r, 0b10, op0,
3672                           !cast<RegisterOperand>(List # "D_operand"),
3673                           neon_uimm1_bare, asmop> {
3674     let Inst{12-10} = 0b001;
3675     let Inst{30} = lane{0};
3676   }
3677 }
3678
3679 // Load single 1-element structure to one lane of 1 register.
3680 defm LD1LN : LDN_Lane_BHSD<0b0, 0b0, "VOne", "ld1">;
3681
3682 // Load single N-element structure to one lane of N consecutive registers
3683 // (N = 2,3,4)
3684 defm LD2LN : LDN_Lane_BHSD<0b1, 0b0, "VPair", "ld2">;
3685 defm LD3LN : LDN_Lane_BHSD<0b0, 0b1, "VTriple", "ld3">;
3686 defm LD4LN : LDN_Lane_BHSD<0b1, 0b1, "VQuad", "ld4">;
3687
3688 multiclass LD1LN_patterns<ValueType VTy, ValueType VTy2, ValueType DTy,
3689                           Operand ImmOp, Operand ImmOp2, PatFrag LoadOp,
3690                           Instruction INST> {
3691   def : Pat<(VTy (vector_insert (VTy VPR64:$src),
3692                      (DTy (LoadOp GPR64xsp:$Rn)), (ImmOp:$lane))),
3693             (VTy (EXTRACT_SUBREG
3694                      (INST GPR64xsp:$Rn,
3695                            (SUBREG_TO_REG (i64 0), VPR64:$src, sub_64),
3696                            ImmOp:$lane),
3697                      sub_64))>;
3698
3699   def : Pat<(VTy2 (vector_insert (VTy2 VPR128:$src),
3700                       (DTy (LoadOp GPR64xsp:$Rn)), (ImmOp2:$lane))),
3701             (VTy2 (INST GPR64xsp:$Rn, VPR128:$src, ImmOp2:$lane))>;
3702 }
3703
3704 // Match all LD1LN instructions
3705 defm : LD1LN_patterns<v8i8, v16i8, i32, neon_uimm3_bare, neon_uimm4_bare,
3706                       extloadi8, LD1LN_B>;
3707
3708 defm : LD1LN_patterns<v4i16, v8i16, i32, neon_uimm2_bare, neon_uimm3_bare,
3709                       extloadi16, LD1LN_H>;
3710
3711 defm : LD1LN_patterns<v2i32, v4i32, i32, neon_uimm1_bare, neon_uimm2_bare,
3712                       load, LD1LN_S>;
3713 defm : LD1LN_patterns<v2f32, v4f32, f32, neon_uimm1_bare, neon_uimm2_bare,
3714                       load, LD1LN_S>;
3715
3716 defm : LD1LN_patterns<v1i64, v2i64, i64, neon_uimm0_bare, neon_uimm1_bare,
3717                       load, LD1LN_D>;
3718 defm : LD1LN_patterns<v1f64, v2f64, f64, neon_uimm0_bare, neon_uimm1_bare,
3719                       load, LD1LN_D>;
3720
3721 class NeonI_STN_Lane<bit r, bits<2> op2_1, bit op0, RegisterOperand VList,
3722                      Operand ImmOp, string asmop>
3723     : NeonI_LdStOne_Lane<0, r, op2_1, op0,
3724                          (outs), (ins GPR64xsp:$Rn, VList:$Rt, ImmOp:$lane),
3725                          asmop # "\t$Rt[$lane], [$Rn]",
3726                          [],
3727                          NoItinerary> {
3728   let mayStore = 1;
3729   let neverHasSideEffects = 1;
3730   let hasExtraDefRegAllocReq = 1;
3731 }
3732
3733 multiclass STN_Lane_BHSD<bit r, bit op0, string List, string asmop> {
3734   def _B : NeonI_STN_Lane<r, 0b00, op0,
3735                           !cast<RegisterOperand>(List # "B_operand"),
3736                           neon_uimm4_bare, asmop> {
3737     let Inst{12-10} = lane{2-0};
3738     let Inst{30} = lane{3};
3739   }
3740
3741   def _H : NeonI_STN_Lane<r, 0b01, op0,
3742                           !cast<RegisterOperand>(List # "H_operand"),
3743                           neon_uimm3_bare, asmop> {
3744     let Inst{12-10} = {lane{1}, lane{0}, 0b0};
3745     let Inst{30} = lane{2};
3746   }
3747
3748   def _S : NeonI_STN_Lane<r, 0b10, op0,
3749                           !cast<RegisterOperand>(List # "S_operand"),
3750                            neon_uimm2_bare, asmop> {
3751     let Inst{12-10} = {lane{0}, 0b0, 0b0};
3752     let Inst{30} = lane{1};
3753   }
3754
3755   def _D : NeonI_STN_Lane<r, 0b10, op0,
3756                           !cast<RegisterOperand>(List # "D_operand"),
3757                           neon_uimm1_bare, asmop>{
3758     let Inst{12-10} = 0b001;
3759     let Inst{30} = lane{0};
3760   }
3761 }
3762
3763 // Store single 1-element structure from one lane of 1 register.
3764 defm ST1LN : STN_Lane_BHSD<0b0, 0b0, "VOne", "st1">;
3765
3766 // Store single N-element structure from one lane of N consecutive registers
3767 // (N = 2,3,4)
3768 defm ST2LN : STN_Lane_BHSD<0b1, 0b0, "VPair", "st2">;
3769 defm ST3LN : STN_Lane_BHSD<0b0, 0b1, "VTriple", "st3">;
3770 defm ST4LN : STN_Lane_BHSD<0b1, 0b1, "VQuad", "st4">;
3771
3772 multiclass ST1LN_patterns<ValueType VTy, ValueType VTy2, ValueType DTy,
3773                           Operand ImmOp, Operand ImmOp2, PatFrag StoreOp,
3774                           Instruction INST> {
3775   def : Pat<(StoreOp (DTy (vector_extract (VTy VPR64:$Rt), ImmOp:$lane)),
3776                      GPR64xsp:$Rn),
3777             (INST GPR64xsp:$Rn,
3778                   (SUBREG_TO_REG (i64 0), VPR64:$Rt, sub_64),
3779                   ImmOp:$lane)>;
3780
3781   def : Pat<(StoreOp (DTy (vector_extract (VTy2 VPR128:$Rt), ImmOp2:$lane)),
3782                      GPR64xsp:$Rn),
3783             (INST GPR64xsp:$Rn, VPR128:$Rt, ImmOp2:$lane)>;
3784 }
3785
3786 // Match all ST1LN instructions
3787 defm : ST1LN_patterns<v8i8, v16i8, i32, neon_uimm3_bare, neon_uimm4_bare,
3788                       truncstorei8, ST1LN_B>;
3789
3790 defm : ST1LN_patterns<v4i16, v8i16, i32, neon_uimm2_bare, neon_uimm3_bare,
3791                       truncstorei16, ST1LN_H>;
3792
3793 defm : ST1LN_patterns<v2i32, v4i32, i32, neon_uimm1_bare, neon_uimm2_bare,
3794                       store, ST1LN_S>;
3795 defm : ST1LN_patterns<v2f32, v4f32, f32, neon_uimm1_bare, neon_uimm2_bare,
3796                       store, ST1LN_S>;
3797
3798 defm : ST1LN_patterns<v1i64, v2i64, i64, neon_uimm0_bare, neon_uimm1_bare,
3799                       store, ST1LN_D>;
3800 defm : ST1LN_patterns<v1f64, v2f64, f64, neon_uimm0_bare, neon_uimm1_bare,
3801                       store, ST1LN_D>;
3802
3803 // End of vector load/store single N-element structure (class SIMD lsone).
3804
3805
3806 // The following are post-index load/store single N-element instructions
3807 // (class SIMD lsone-post)
3808
3809 multiclass NeonI_LDN_WB_Dup<bit q, bit r, bits<3> opcode, bits<2> size,
3810                             RegisterOperand VecList, Operand ImmTy,
3811                             string asmop> {
3812   let mayLoad = 1, neverHasSideEffects = 1, Constraints = "$wb = $Rn",
3813   DecoderMethod = "DecodeVLDSTLanePostInstruction" in {
3814     def _fixed : NeonI_LdOne_Dup_Post<q, r, opcode, size,
3815                       (outs VecList:$Rt, GPR64xsp:$wb),
3816                       (ins GPR64xsp:$Rn, ImmTy:$amt),
3817                       asmop # "\t$Rt, [$Rn], $amt",
3818                       [],
3819                       NoItinerary> {
3820                         let Rm = 0b11111;
3821                       }
3822
3823     def _register : NeonI_LdOne_Dup_Post<q, r, opcode, size,
3824                       (outs VecList:$Rt, GPR64xsp:$wb),
3825                       (ins GPR64xsp:$Rn, GPR64noxzr:$Rm),
3826                       asmop # "\t$Rt, [$Rn], $Rm",
3827                       [],
3828                       NoItinerary>;
3829   }
3830 }
3831
3832 multiclass LDWB_Dup_BHSD<bit r, bits<3> opcode, string List, string asmop,
3833                          Operand uimm_b, Operand uimm_h,
3834                          Operand uimm_s, Operand uimm_d> {
3835   defm _8B : NeonI_LDN_WB_Dup<0, r, opcode, 0b00,
3836                               !cast<RegisterOperand>(List # "8B_operand"),
3837                               uimm_b, asmop>;
3838
3839   defm _4H : NeonI_LDN_WB_Dup<0, r, opcode, 0b01,
3840                               !cast<RegisterOperand>(List # "4H_operand"),
3841                               uimm_h, asmop>;
3842
3843   defm _2S : NeonI_LDN_WB_Dup<0, r, opcode, 0b10,
3844                               !cast<RegisterOperand>(List # "2S_operand"),
3845                               uimm_s, asmop>;
3846
3847   defm _1D : NeonI_LDN_WB_Dup<0, r, opcode, 0b11,
3848                               !cast<RegisterOperand>(List # "1D_operand"),
3849                               uimm_d, asmop>;
3850
3851   defm _16B : NeonI_LDN_WB_Dup<1, r, opcode, 0b00,
3852                                !cast<RegisterOperand>(List # "16B_operand"),
3853                                uimm_b, asmop>;
3854
3855   defm _8H : NeonI_LDN_WB_Dup<1, r, opcode, 0b01,
3856                               !cast<RegisterOperand>(List # "8H_operand"),
3857                               uimm_h, asmop>;
3858
3859   defm _4S : NeonI_LDN_WB_Dup<1, r, opcode, 0b10,
3860                               !cast<RegisterOperand>(List # "4S_operand"),
3861                               uimm_s, asmop>;
3862
3863   defm _2D : NeonI_LDN_WB_Dup<1, r, opcode, 0b11,
3864                               !cast<RegisterOperand>(List # "2D_operand"),
3865                               uimm_d, asmop>;
3866 }
3867
3868 // Post-index load single 1-element structure to all lanes of 1 register
3869 defm LD1R_WB : LDWB_Dup_BHSD<0b0, 0b110, "VOne", "ld1r", uimm_exact1,
3870                              uimm_exact2, uimm_exact4, uimm_exact8>;
3871
3872 // Post-index load single N-element structure to all lanes of N consecutive
3873 // registers (N = 2,3,4)
3874 defm LD2R_WB : LDWB_Dup_BHSD<0b1, 0b110, "VPair", "ld2r", uimm_exact2,
3875                              uimm_exact4, uimm_exact8, uimm_exact16>;
3876 defm LD3R_WB : LDWB_Dup_BHSD<0b0, 0b111, "VTriple", "ld3r", uimm_exact3,
3877                              uimm_exact6, uimm_exact12, uimm_exact24>;
3878 defm LD4R_WB : LDWB_Dup_BHSD<0b1, 0b111, "VQuad", "ld4r", uimm_exact4,
3879                              uimm_exact8, uimm_exact16, uimm_exact32>;
3880
3881 let mayLoad = 1, neverHasSideEffects = 1, hasExtraDefRegAllocReq = 1,
3882     Constraints = "$Rn = $wb, $Rt = $src",
3883     DecoderMethod = "DecodeVLDSTLanePostInstruction" in {
3884   class LDN_WBFx_Lane<bit r, bits<2> op2_1, bit op0, RegisterOperand VList,
3885                                 Operand ImmTy, Operand ImmOp, string asmop>
3886       : NeonI_LdStOne_Lane_Post<1, r, op2_1, op0,
3887                                 (outs VList:$Rt, GPR64xsp:$wb),
3888                                 (ins GPR64xsp:$Rn, ImmTy:$amt,
3889                                     VList:$src, ImmOp:$lane),
3890                                 asmop # "\t$Rt[$lane], [$Rn], $amt",
3891                                 [],
3892                                 NoItinerary> {
3893     let Rm = 0b11111;
3894   }
3895
3896   class LDN_WBReg_Lane<bit r, bits<2> op2_1, bit op0, RegisterOperand VList,
3897                                  Operand ImmTy, Operand ImmOp, string asmop>
3898       : NeonI_LdStOne_Lane_Post<1, r, op2_1, op0,
3899                                 (outs VList:$Rt, GPR64xsp:$wb),
3900                                 (ins GPR64xsp:$Rn, GPR64noxzr:$Rm,
3901                                     VList:$src, ImmOp:$lane),
3902                                 asmop # "\t$Rt[$lane], [$Rn], $Rm",
3903                                 [],
3904                                 NoItinerary>;
3905 }
3906
3907 multiclass LD_Lane_WB_BHSD<bit r, bit op0, string List, string asmop,
3908                            Operand uimm_b, Operand uimm_h,
3909                            Operand uimm_s, Operand uimm_d> {
3910   def _B_fixed : LDN_WBFx_Lane<r, 0b00, op0,
3911                                !cast<RegisterOperand>(List # "B_operand"),
3912                                uimm_b, neon_uimm4_bare, asmop> {
3913     let Inst{12-10} = lane{2-0};
3914     let Inst{30} = lane{3};
3915   }
3916
3917   def _B_register : LDN_WBReg_Lane<r, 0b00, op0,
3918                                    !cast<RegisterOperand>(List # "B_operand"),
3919                                    uimm_b, neon_uimm4_bare, asmop> {
3920     let Inst{12-10} = lane{2-0};
3921     let Inst{30} = lane{3};
3922   }
3923
3924   def _H_fixed : LDN_WBFx_Lane<r, 0b01, op0,
3925                                !cast<RegisterOperand>(List # "H_operand"),
3926                                uimm_h, neon_uimm3_bare, asmop> {
3927     let Inst{12-10} = {lane{1}, lane{0}, 0b0};
3928     let Inst{30} = lane{2};
3929   }
3930
3931   def _H_register : LDN_WBReg_Lane<r, 0b01, op0,
3932                                    !cast<RegisterOperand>(List # "H_operand"),
3933                                    uimm_h, neon_uimm3_bare, asmop> {
3934     let Inst{12-10} = {lane{1}, lane{0}, 0b0};
3935     let Inst{30} = lane{2};
3936   }
3937
3938   def _S_fixed : LDN_WBFx_Lane<r, 0b10, op0,
3939                                !cast<RegisterOperand>(List # "S_operand"),
3940                                uimm_s, neon_uimm2_bare, asmop> {
3941     let Inst{12-10} = {lane{0}, 0b0, 0b0};
3942     let Inst{30} = lane{1};
3943   }
3944
3945   def _S_register : LDN_WBReg_Lane<r, 0b10, op0,
3946                                    !cast<RegisterOperand>(List # "S_operand"),
3947                                    uimm_s, neon_uimm2_bare, asmop> {
3948     let Inst{12-10} = {lane{0}, 0b0, 0b0};
3949     let Inst{30} = lane{1};
3950   }
3951
3952   def _D_fixed : LDN_WBFx_Lane<r, 0b10, op0,
3953                                !cast<RegisterOperand>(List # "D_operand"),
3954                                uimm_d, neon_uimm1_bare, asmop> {
3955     let Inst{12-10} = 0b001;
3956     let Inst{30} = lane{0};
3957   }
3958
3959   def _D_register : LDN_WBReg_Lane<r, 0b10, op0,
3960                                    !cast<RegisterOperand>(List # "D_operand"),
3961                                    uimm_d, neon_uimm1_bare, asmop> {
3962     let Inst{12-10} = 0b001;
3963     let Inst{30} = lane{0};
3964   }
3965 }
3966
3967 // Post-index load single 1-element structure to one lane of 1 register.
3968 defm LD1LN_WB : LD_Lane_WB_BHSD<0b0, 0b0, "VOne", "ld1", uimm_exact1,
3969                                 uimm_exact2, uimm_exact4, uimm_exact8>;
3970
3971 // Post-index load single N-element structure to one lane of N consecutive
3972 // registers
3973 // (N = 2,3,4)
3974 defm LD2LN_WB : LD_Lane_WB_BHSD<0b1, 0b0, "VPair", "ld2", uimm_exact2,
3975                                 uimm_exact4, uimm_exact8, uimm_exact16>;
3976 defm LD3LN_WB : LD_Lane_WB_BHSD<0b0, 0b1, "VTriple", "ld3", uimm_exact3,
3977                                 uimm_exact6, uimm_exact12, uimm_exact24>;
3978 defm LD4LN_WB : LD_Lane_WB_BHSD<0b1, 0b1, "VQuad", "ld4", uimm_exact4,
3979                                 uimm_exact8, uimm_exact16, uimm_exact32>;
3980
3981 let mayStore = 1, neverHasSideEffects = 1,
3982     hasExtraDefRegAllocReq = 1, Constraints = "$Rn = $wb",
3983     DecoderMethod = "DecodeVLDSTLanePostInstruction" in {
3984   class STN_WBFx_Lane<bit r, bits<2> op2_1, bit op0, RegisterOperand VList,
3985                       Operand ImmTy, Operand ImmOp, string asmop>
3986       : NeonI_LdStOne_Lane_Post<0, r, op2_1, op0,
3987                                 (outs GPR64xsp:$wb),
3988                                 (ins GPR64xsp:$Rn, ImmTy:$amt,
3989                                     VList:$Rt, ImmOp:$lane),
3990                                 asmop # "\t$Rt[$lane], [$Rn], $amt",
3991                                 [],
3992                                 NoItinerary> {
3993     let Rm = 0b11111;
3994   }
3995
3996   class STN_WBReg_Lane<bit r, bits<2> op2_1, bit op0, RegisterOperand VList,
3997                        Operand ImmTy, Operand ImmOp, string asmop>
3998       : NeonI_LdStOne_Lane_Post<0, r, op2_1, op0,
3999                                 (outs GPR64xsp:$wb),
4000                                 (ins GPR64xsp:$Rn, GPR64noxzr:$Rm, VList:$Rt,
4001                                     ImmOp:$lane),
4002                                 asmop # "\t$Rt[$lane], [$Rn], $Rm",
4003                                 [],
4004                                 NoItinerary>;
4005 }
4006
4007 multiclass ST_Lane_WB_BHSD<bit r, bit op0, string List, string asmop,
4008                            Operand uimm_b, Operand uimm_h,
4009                            Operand uimm_s, Operand uimm_d> {
4010   def _B_fixed : STN_WBFx_Lane<r, 0b00, op0,
4011                                !cast<RegisterOperand>(List # "B_operand"),
4012                                uimm_b, neon_uimm4_bare, asmop> {
4013     let Inst{12-10} = lane{2-0};
4014     let Inst{30} = lane{3};
4015   }
4016
4017   def _B_register : STN_WBReg_Lane<r, 0b00, op0,
4018                                    !cast<RegisterOperand>(List # "B_operand"),
4019                                    uimm_b, neon_uimm4_bare, asmop> {
4020     let Inst{12-10} = lane{2-0};
4021     let Inst{30} = lane{3};
4022   }
4023
4024   def _H_fixed : STN_WBFx_Lane<r, 0b01, op0,
4025                                !cast<RegisterOperand>(List # "H_operand"),
4026                                uimm_h, neon_uimm3_bare, asmop> {
4027     let Inst{12-10} = {lane{1}, lane{0}, 0b0};
4028     let Inst{30} = lane{2};
4029   }
4030
4031   def _H_register : STN_WBReg_Lane<r, 0b01, op0,
4032                                    !cast<RegisterOperand>(List # "H_operand"),
4033                                    uimm_h, neon_uimm3_bare, asmop> {
4034     let Inst{12-10} = {lane{1}, lane{0}, 0b0};
4035     let Inst{30} = lane{2};
4036   }
4037
4038   def _S_fixed : STN_WBFx_Lane<r, 0b10, op0,
4039                                !cast<RegisterOperand>(List # "S_operand"),
4040                                uimm_s, neon_uimm2_bare, asmop> {
4041     let Inst{12-10} = {lane{0}, 0b0, 0b0};
4042     let Inst{30} = lane{1};
4043   }
4044
4045   def _S_register : STN_WBReg_Lane<r, 0b10, op0,
4046                                    !cast<RegisterOperand>(List # "S_operand"),
4047                                    uimm_s, neon_uimm2_bare, asmop> {
4048     let Inst{12-10} = {lane{0}, 0b0, 0b0};
4049     let Inst{30} = lane{1};
4050   }
4051
4052   def _D_fixed : STN_WBFx_Lane<r, 0b10, op0,
4053                                !cast<RegisterOperand>(List # "D_operand"),
4054                                uimm_d, neon_uimm1_bare, asmop> {
4055     let Inst{12-10} = 0b001;
4056     let Inst{30} = lane{0};
4057   }
4058
4059   def _D_register : STN_WBReg_Lane<r, 0b10, op0,
4060                                    !cast<RegisterOperand>(List # "D_operand"),
4061                                    uimm_d, neon_uimm1_bare, asmop> {
4062     let Inst{12-10} = 0b001;
4063     let Inst{30} = lane{0};
4064   }
4065 }
4066
4067 // Post-index store single 1-element structure from one lane of 1 register.
4068 defm ST1LN_WB : ST_Lane_WB_BHSD<0b0, 0b0, "VOne", "st1", uimm_exact1,
4069                                 uimm_exact2, uimm_exact4, uimm_exact8>;
4070
4071 // Post-index store single N-element structure from one lane of N consecutive
4072 // registers (N = 2,3,4)
4073 defm ST2LN_WB : ST_Lane_WB_BHSD<0b1, 0b0, "VPair", "st2", uimm_exact2,
4074                                 uimm_exact4, uimm_exact8, uimm_exact16>;
4075 defm ST3LN_WB : ST_Lane_WB_BHSD<0b0, 0b1, "VTriple", "st3", uimm_exact3,
4076                                 uimm_exact6, uimm_exact12, uimm_exact24>;
4077 defm ST4LN_WB : ST_Lane_WB_BHSD<0b1, 0b1, "VQuad", "st4", uimm_exact4,
4078                                 uimm_exact8, uimm_exact16, uimm_exact32>;
4079
4080 // End of post-index load/store single N-element instructions
4081 // (class SIMD lsone-post)
4082
4083 // Neon Scalar instructions implementation
4084 // Scalar Three Same
4085
4086 class NeonI_Scalar3Same_size<bit u, bits<2> size, bits<5> opcode, string asmop,
4087                              RegisterClass FPRC>
4088   : NeonI_Scalar3Same<u, size, opcode,
4089                       (outs FPRC:$Rd), (ins FPRC:$Rn, FPRC:$Rm),
4090                       !strconcat(asmop, "\t$Rd, $Rn, $Rm"),
4091                       [],
4092                       NoItinerary>;
4093
4094 class NeonI_Scalar3Same_D_size<bit u, bits<5> opcode, string asmop>
4095   : NeonI_Scalar3Same_size<u, 0b11, opcode, asmop, FPR64>;
4096
4097 multiclass NeonI_Scalar3Same_HS_sizes<bit u, bits<5> opcode, string asmop,
4098                                       bit Commutable = 0> {
4099   let isCommutable = Commutable in {
4100     def hhh : NeonI_Scalar3Same_size<u, 0b01, opcode, asmop, FPR16>;
4101     def sss : NeonI_Scalar3Same_size<u, 0b10, opcode, asmop, FPR32>;
4102   }
4103 }
4104
4105 multiclass NeonI_Scalar3Same_SD_sizes<bit u, bit size_high, bits<5> opcode,
4106                                       string asmop, bit Commutable = 0> {
4107   let isCommutable = Commutable in {
4108     def sss : NeonI_Scalar3Same_size<u, {size_high, 0b0}, opcode, asmop, FPR32>;
4109     def ddd : NeonI_Scalar3Same_size<u, {size_high, 0b1}, opcode, asmop, FPR64>;
4110   }
4111 }
4112
4113 multiclass NeonI_Scalar3Same_BHSD_sizes<bit u, bits<5> opcode,
4114                                         string asmop, bit Commutable = 0> {
4115   let isCommutable = Commutable in {
4116     def bbb : NeonI_Scalar3Same_size<u, 0b00, opcode, asmop, FPR8>;
4117     def hhh : NeonI_Scalar3Same_size<u, 0b01, opcode, asmop, FPR16>;
4118     def sss : NeonI_Scalar3Same_size<u, 0b10, opcode, asmop, FPR32>;
4119     def ddd : NeonI_Scalar3Same_size<u, 0b11, opcode, asmop, FPR64>;
4120   }
4121 }
4122
4123 multiclass Neon_Scalar3Same_D_size_patterns<SDPatternOperator opnode,
4124                                             Instruction INSTD> {
4125   def : Pat<(v1i64 (opnode (v1i64 FPR64:$Rn), (v1i64 FPR64:$Rm))),
4126             (INSTD FPR64:$Rn, FPR64:$Rm)>;
4127 }
4128
4129 multiclass Neon_Scalar3Same_BHSD_size_patterns<SDPatternOperator opnode,
4130                                                Instruction INSTB,
4131                                                Instruction INSTH,
4132                                                Instruction INSTS,
4133                                                Instruction INSTD>
4134   : Neon_Scalar3Same_D_size_patterns<opnode, INSTD> {
4135   def: Pat<(v1i8 (opnode (v1i8 FPR8:$Rn), (v1i8 FPR8:$Rm))),
4136            (INSTB FPR8:$Rn, FPR8:$Rm)>;
4137
4138   def: Pat<(v1i16 (opnode (v1i16 FPR16:$Rn), (v1i16 FPR16:$Rm))),
4139            (INSTH FPR16:$Rn, FPR16:$Rm)>;
4140
4141   def: Pat<(v1i32 (opnode (v1i32 FPR32:$Rn), (v1i32 FPR32:$Rm))),
4142            (INSTS FPR32:$Rn, FPR32:$Rm)>;
4143 }
4144
4145 class Neon_Scalar3Same_cmp_D_size_patterns<SDPatternOperator opnode,
4146                                            Instruction INSTD>
4147   : Pat<(v1i64 (opnode (v1i64 FPR64:$Rn), (v1i64 FPR64:$Rm))),
4148         (INSTD FPR64:$Rn, FPR64:$Rm)>;
4149
4150 multiclass Neon_Scalar3Same_HS_size_patterns<SDPatternOperator opnode,
4151                                              Instruction INSTH,
4152                                              Instruction INSTS> {
4153   def : Pat<(v1i16 (opnode (v1i16 FPR16:$Rn), (v1i16 FPR16:$Rm))),
4154             (INSTH FPR16:$Rn, FPR16:$Rm)>;
4155   def : Pat<(v1i32 (opnode (v1i32 FPR32:$Rn), (v1i32 FPR32:$Rm))),
4156             (INSTS FPR32:$Rn, FPR32:$Rm)>;
4157 }
4158
4159 multiclass Neon_Scalar3Same_SD_size_patterns<SDPatternOperator opnode,
4160                                              Instruction INSTS,
4161                                              Instruction INSTD> {
4162   def : Pat<(v1f32 (opnode (v1f32 FPR32:$Rn), (v1f32 FPR32:$Rm))),
4163             (INSTS FPR32:$Rn, FPR32:$Rm)>;
4164   def : Pat<(v1f64 (opnode (v1f64 FPR64:$Rn), (v1f64 FPR64:$Rm))),
4165             (INSTD FPR64:$Rn, FPR64:$Rm)>;
4166 }
4167
4168 multiclass Neon_Scalar3Same_cmp_SD_size_patterns<SDPatternOperator opnode,
4169                                                  Instruction INSTS,
4170                                                  Instruction INSTD> {
4171   def : Pat<(v1i32 (opnode (v1f32 FPR32:$Rn), (v1f32 FPR32:$Rm))),
4172             (INSTS FPR32:$Rn, FPR32:$Rm)>;
4173   def : Pat<(v1i64 (opnode (v1f64 FPR64:$Rn), (v1f64 FPR64:$Rm))),
4174             (INSTD FPR64:$Rn, FPR64:$Rm)>;
4175 }
4176
4177 class Neon_Scalar3Same_cmp_V1_D_size_patterns<CondCode CC,
4178                                               Instruction INSTD>
4179   : Pat<(v1i64 (Neon_cmp (v1f64 FPR64:$Rn), (v1f64 FPR64:$Rm), CC)),
4180         (INSTD FPR64:$Rn, FPR64:$Rm)>;
4181
4182 // Scalar Three Different
4183
4184 class NeonI_Scalar3Diff_size<bit u, bits<2> size, bits<4> opcode, string asmop,
4185                              RegisterClass FPRCD, RegisterClass FPRCS>
4186   : NeonI_Scalar3Diff<u, size, opcode,
4187                       (outs FPRCD:$Rd), (ins FPRCS:$Rn, FPRCS:$Rm),
4188                       !strconcat(asmop, "\t$Rd, $Rn, $Rm"),
4189                       [],
4190                       NoItinerary>;
4191
4192 multiclass NeonI_Scalar3Diff_HS_size<bit u, bits<4> opcode, string asmop> {
4193   def shh : NeonI_Scalar3Diff_size<u, 0b01, opcode, asmop, FPR32, FPR16>;
4194   def dss : NeonI_Scalar3Diff_size<u, 0b10, opcode, asmop, FPR64, FPR32>;
4195 }
4196
4197 multiclass NeonI_Scalar3Diff_ml_HS_size<bit u, bits<4> opcode, string asmop> {
4198   let Constraints = "$Src = $Rd" in {
4199     def shh : NeonI_Scalar3Diff<u, 0b01, opcode,
4200                        (outs FPR32:$Rd), (ins FPR32:$Src, FPR16:$Rn, FPR16:$Rm),
4201                        !strconcat(asmop, "\t$Rd, $Rn, $Rm"),
4202                        [],
4203                        NoItinerary>;
4204     def dss : NeonI_Scalar3Diff<u, 0b10, opcode,
4205                        (outs FPR64:$Rd), (ins FPR64:$Src, FPR32:$Rn, FPR32:$Rm),
4206                        !strconcat(asmop, "\t$Rd, $Rn, $Rm"),
4207                        [],
4208                        NoItinerary>;
4209   }
4210 }
4211
4212 multiclass Neon_Scalar3Diff_HS_size_patterns<SDPatternOperator opnode,
4213                                              Instruction INSTH,
4214                                              Instruction INSTS> {
4215   def : Pat<(v1i32 (opnode (v1i16 FPR16:$Rn), (v1i16 FPR16:$Rm))),
4216             (INSTH FPR16:$Rn, FPR16:$Rm)>;
4217   def : Pat<(v1i64 (opnode (v1i32 FPR32:$Rn), (v1i32 FPR32:$Rm))),
4218             (INSTS FPR32:$Rn, FPR32:$Rm)>;
4219 }
4220
4221 multiclass Neon_Scalar3Diff_ml_HS_size_patterns<SDPatternOperator opnode,
4222                                              Instruction INSTH,
4223                                              Instruction INSTS> {
4224   def : Pat<(v1i32 (opnode (v1i32 FPR32:$Src), (v1i16 FPR16:$Rn), (v1i16 FPR16:$Rm))),
4225             (INSTH FPR32:$Src, FPR16:$Rn, FPR16:$Rm)>;
4226   def : Pat<(v1i64 (opnode (v1i64 FPR64:$Src), (v1i32 FPR32:$Rn), (v1i32 FPR32:$Rm))),
4227             (INSTS FPR64:$Src, FPR32:$Rn, FPR32:$Rm)>;
4228 }
4229
4230 // Scalar Two Registers Miscellaneous
4231
4232 class NeonI_Scalar2SameMisc_size<bit u, bits<2> size, bits<5> opcode, string asmop,
4233                              RegisterClass FPRCD, RegisterClass FPRCS>
4234   : NeonI_Scalar2SameMisc<u, size, opcode,
4235                           (outs FPRCD:$Rd), (ins FPRCS:$Rn),
4236                           !strconcat(asmop, "\t$Rd, $Rn"),
4237                           [],
4238                           NoItinerary>;
4239
4240 multiclass NeonI_Scalar2SameMisc_SD_size<bit u, bit size_high, bits<5> opcode,
4241                                          string asmop> {
4242   def ss : NeonI_Scalar2SameMisc_size<u, {size_high, 0b0}, opcode, asmop, FPR32,
4243                                       FPR32>;
4244   def dd : NeonI_Scalar2SameMisc_size<u, {size_high, 0b1}, opcode, asmop, FPR64,
4245                                       FPR64>;
4246 }
4247
4248 multiclass NeonI_Scalar2SameMisc_D_size<bit u, bits<5> opcode, string asmop> {
4249   def dd : NeonI_Scalar2SameMisc_size<u, 0b11, opcode, asmop, FPR64, FPR64>;
4250 }
4251
4252 multiclass NeonI_Scalar2SameMisc_BHSD_size<bit u, bits<5> opcode, string asmop>
4253   : NeonI_Scalar2SameMisc_D_size<u, opcode, asmop> {
4254   def bb : NeonI_Scalar2SameMisc_size<u, 0b00, opcode, asmop, FPR8, FPR8>;
4255   def hh : NeonI_Scalar2SameMisc_size<u, 0b01, opcode, asmop, FPR16, FPR16>;
4256   def ss : NeonI_Scalar2SameMisc_size<u, 0b10, opcode, asmop, FPR32, FPR32>;
4257 }
4258
4259 class NeonI_Scalar2SameMisc_fcvtxn_D_size<bit u, bits<5> opcode, string asmop>
4260   : NeonI_Scalar2SameMisc_size<u, 0b01, opcode, asmop, FPR32, FPR64>;
4261
4262 multiclass NeonI_Scalar2SameMisc_narrow_HSD_size<bit u, bits<5> opcode,
4263                                                  string asmop> {
4264   def bh : NeonI_Scalar2SameMisc_size<u, 0b00, opcode, asmop, FPR8, FPR16>;
4265   def hs : NeonI_Scalar2SameMisc_size<u, 0b01, opcode, asmop, FPR16, FPR32>;
4266   def sd : NeonI_Scalar2SameMisc_size<u, 0b10, opcode, asmop, FPR32, FPR64>;
4267 }
4268
4269 class NeonI_Scalar2SameMisc_accum_size<bit u, bits<2> size, bits<5> opcode,
4270                                        string asmop, RegisterClass FPRC>
4271   : NeonI_Scalar2SameMisc<u, size, opcode,
4272                           (outs FPRC:$Rd), (ins FPRC:$Src, FPRC:$Rn),
4273                           !strconcat(asmop, "\t$Rd, $Rn"),
4274                           [],
4275                           NoItinerary>;
4276
4277 multiclass NeonI_Scalar2SameMisc_accum_BHSD_size<bit u, bits<5> opcode,
4278                                                  string asmop> {
4279
4280   let Constraints = "$Src = $Rd" in {
4281     def bb : NeonI_Scalar2SameMisc_accum_size<u, 0b00, opcode, asmop, FPR8>;
4282     def hh : NeonI_Scalar2SameMisc_accum_size<u, 0b01, opcode, asmop, FPR16>;
4283     def ss : NeonI_Scalar2SameMisc_accum_size<u, 0b10, opcode, asmop, FPR32>;
4284     def dd : NeonI_Scalar2SameMisc_accum_size<u, 0b11, opcode, asmop, FPR64>;
4285   }
4286 }
4287
4288 class Neon_Scalar2SameMisc_fcvtxn_D_size_patterns<SDPatternOperator opnode,
4289                                                   Instruction INSTD>
4290   : Pat<(v1f32 (opnode (v1f64 FPR64:$Rn))),
4291         (INSTD FPR64:$Rn)>;
4292
4293 multiclass Neon_Scalar2SameMisc_fcvt_SD_size_patterns<SDPatternOperator opnode,
4294                                                       Instruction INSTS,
4295                                                       Instruction INSTD> {
4296   def : Pat<(v1i32 (opnode (v1f32 FPR32:$Rn))),
4297             (INSTS FPR32:$Rn)>;
4298   def : Pat<(v1i64 (opnode (v1f64 FPR64:$Rn))),
4299             (INSTD FPR64:$Rn)>;
4300 }
4301
4302 multiclass Neon_Scalar2SameMisc_cvt_SD_size_patterns<SDPatternOperator Sopnode,
4303                                                      SDPatternOperator Dopnode,
4304                                                      Instruction INSTS,
4305                                                      Instruction INSTD> {
4306   def : Pat<(f32 (Sopnode (v1i32 FPR32:$Rn))),
4307             (INSTS FPR32:$Rn)>;
4308   def : Pat<(f64 (Dopnode (v1i64 FPR64:$Rn))),
4309             (INSTD FPR64:$Rn)>;
4310 }
4311
4312 multiclass Neon_Scalar2SameMisc_SD_size_patterns<SDPatternOperator opnode,
4313                                                  Instruction INSTS,
4314                                                  Instruction INSTD> {
4315   def : Pat<(v1f32 (opnode (v1f32 FPR32:$Rn))),
4316             (INSTS FPR32:$Rn)>;
4317   def : Pat<(v1f64 (opnode (v1f64 FPR64:$Rn))),
4318             (INSTD FPR64:$Rn)>;
4319 }
4320
4321 class NeonI_Scalar2SameMisc_cmpz_D_size<bit u, bits<5> opcode, string asmop>
4322   : NeonI_Scalar2SameMisc<u, 0b11, opcode,
4323                           (outs FPR64:$Rd), (ins FPR64:$Rn, neon_uimm0:$Imm),
4324                           !strconcat(asmop, "\t$Rd, $Rn, $Imm"),
4325                           [],
4326                           NoItinerary>;
4327
4328 multiclass NeonI_Scalar2SameMisc_cmpz_SD_size<bit u, bits<5> opcode,
4329                                               string asmop> {
4330   def ssi : NeonI_Scalar2SameMisc<u, 0b10, opcode,
4331                            (outs FPR32:$Rd), (ins FPR32:$Rn, fpz32:$FPImm),
4332                            !strconcat(asmop, "\t$Rd, $Rn, $FPImm"),
4333                            [],
4334                            NoItinerary>;
4335   def ddi : NeonI_Scalar2SameMisc<u, 0b11, opcode,
4336                            (outs FPR64:$Rd), (ins FPR64:$Rn, fpz32:$FPImm),
4337                            !strconcat(asmop, "\t$Rd, $Rn, $FPImm"),
4338                            [],
4339                            NoItinerary>;
4340 }
4341
4342 class Neon_Scalar2SameMisc_cmpz_D_size_patterns<SDPatternOperator opnode,
4343                                                 Instruction INSTD>
4344   : Pat<(v1i64 (opnode (v1i64 FPR64:$Rn),
4345                        (v1i64 (bitconvert (v8i8 Neon_AllZero))))),
4346         (INSTD FPR64:$Rn, 0)>;
4347
4348 class Neon_Scalar2SameMisc_cmpz_D_V1_size_patterns<CondCode CC,
4349                                                    Instruction INSTD>
4350   : Pat<(v1i64 (Neon_cmpz (v1i64 FPR64:$Rn),
4351                           (i32 neon_uimm0:$Imm), CC)),
4352         (INSTD FPR64:$Rn, neon_uimm0:$Imm)>;
4353
4354 multiclass Neon_Scalar2SameMisc_cmpz_SD_size_patterns<SDPatternOperator opnode,
4355                                                       Instruction INSTS,
4356                                                       Instruction INSTD> {
4357   def : Pat<(v1i32 (opnode (v1f32 FPR32:$Rn),
4358                            (v1f32 (scalar_to_vector (f32 fpz32:$FPImm))))),
4359             (INSTS FPR32:$Rn, fpz32:$FPImm)>;
4360   def : Pat<(v1i64 (opnode (v1f64 FPR64:$Rn),
4361                            (v1f32 (scalar_to_vector (f32 fpz32:$FPImm))))),
4362             (INSTD FPR64:$Rn, fpz32:$FPImm)>;
4363 }
4364
4365 multiclass Neon_Scalar2SameMisc_D_size_patterns<SDPatternOperator opnode,
4366                                                 Instruction INSTD> {
4367   def : Pat<(v1i64 (opnode (v1i64 FPR64:$Rn))),
4368             (INSTD FPR64:$Rn)>;
4369 }
4370
4371 multiclass Neon_Scalar2SameMisc_BHSD_size_patterns<SDPatternOperator opnode,
4372                                                    Instruction INSTB,
4373                                                    Instruction INSTH,
4374                                                    Instruction INSTS,
4375                                                    Instruction INSTD>
4376   : Neon_Scalar2SameMisc_D_size_patterns<opnode, INSTD> {
4377   def : Pat<(v1i8 (opnode (v1i8 FPR8:$Rn))),
4378             (INSTB FPR8:$Rn)>;
4379   def : Pat<(v1i16 (opnode (v1i16 FPR16:$Rn))),
4380             (INSTH FPR16:$Rn)>;
4381   def : Pat<(v1i32 (opnode (v1i32 FPR32:$Rn))),
4382             (INSTS FPR32:$Rn)>;
4383 }
4384
4385 multiclass Neon_Scalar2SameMisc_narrow_HSD_size_patterns<
4386                                                        SDPatternOperator opnode,
4387                                                        Instruction INSTH,
4388                                                        Instruction INSTS,
4389                                                        Instruction INSTD> {
4390   def : Pat<(v1i8 (opnode (v1i16 FPR16:$Rn))),
4391             (INSTH FPR16:$Rn)>;
4392   def : Pat<(v1i16 (opnode (v1i32 FPR32:$Rn))),
4393             (INSTS FPR32:$Rn)>;
4394   def : Pat<(v1i32 (opnode (v1i64 FPR64:$Rn))),
4395             (INSTD FPR64:$Rn)>;
4396
4397 }
4398
4399 multiclass Neon_Scalar2SameMisc_accum_BHSD_size_patterns<
4400                                                        SDPatternOperator opnode,
4401                                                        Instruction INSTB,
4402                                                        Instruction INSTH,
4403                                                        Instruction INSTS,
4404                                                        Instruction INSTD> {
4405   def : Pat<(v1i8 (opnode (v1i8 FPR8:$Src), (v1i8 FPR8:$Rn))),
4406             (INSTB FPR8:$Src, FPR8:$Rn)>;
4407   def : Pat<(v1i16 (opnode (v1i16 FPR16:$Src), (v1i16 FPR16:$Rn))),
4408             (INSTH FPR16:$Src, FPR16:$Rn)>;
4409   def : Pat<(v1i32 (opnode (v1i32 FPR32:$Src), (v1i32 FPR32:$Rn))),
4410             (INSTS FPR32:$Src, FPR32:$Rn)>;
4411   def : Pat<(v1i64 (opnode (v1i64 FPR64:$Src), (v1i64 FPR64:$Rn))),
4412             (INSTD FPR64:$Src, FPR64:$Rn)>;
4413 }
4414
4415 // Scalar Shift By Immediate
4416
4417 class NeonI_ScalarShiftImm_size<bit u, bits<5> opcode, string asmop,
4418                                 RegisterClass FPRC, Operand ImmTy>
4419   : NeonI_ScalarShiftImm<u, opcode,
4420                          (outs FPRC:$Rd), (ins FPRC:$Rn, ImmTy:$Imm),
4421                          !strconcat(asmop, "\t$Rd, $Rn, $Imm"),
4422                          [], NoItinerary>;
4423
4424 multiclass NeonI_ScalarShiftRightImm_D_size<bit u, bits<5> opcode,
4425                                             string asmop> {
4426   def ddi : NeonI_ScalarShiftImm_size<u, opcode, asmop, FPR64, shr_imm64> {
4427     bits<6> Imm;
4428     let Inst{22} = 0b1; // immh:immb = 1xxxxxx
4429     let Inst{21-16} = Imm;
4430   }
4431 }
4432
4433 multiclass NeonI_ScalarShiftRightImm_BHSD_size<bit u, bits<5> opcode,
4434                                                string asmop>
4435   : NeonI_ScalarShiftRightImm_D_size<u, opcode, asmop> {
4436   def bbi : NeonI_ScalarShiftImm_size<u, opcode, asmop, FPR8, shr_imm8> {
4437     bits<3> Imm;
4438     let Inst{22-19} = 0b0001; // immh:immb = 0001xxx
4439     let Inst{18-16} = Imm;
4440   }
4441   def hhi : NeonI_ScalarShiftImm_size<u, opcode, asmop, FPR16, shr_imm16> {
4442     bits<4> Imm;
4443     let Inst{22-20} = 0b001; // immh:immb = 001xxxx
4444     let Inst{19-16} = Imm;
4445   }
4446   def ssi : NeonI_ScalarShiftImm_size<u, opcode, asmop, FPR32, shr_imm32> {
4447     bits<5> Imm;
4448     let Inst{22-21} = 0b01; // immh:immb = 01xxxxx
4449     let Inst{20-16} = Imm;
4450   }
4451 }
4452
4453 multiclass NeonI_ScalarShiftLeftImm_D_size<bit u, bits<5> opcode,
4454                                             string asmop> {
4455   def ddi : NeonI_ScalarShiftImm_size<u, opcode, asmop, FPR64, shl_imm64> {
4456     bits<6> Imm;
4457     let Inst{22} = 0b1; // immh:immb = 1xxxxxx
4458     let Inst{21-16} = Imm;
4459   }
4460 }
4461
4462 multiclass NeonI_ScalarShiftLeftImm_BHSD_size<bit u, bits<5> opcode,
4463                                               string asmop>
4464   : NeonI_ScalarShiftLeftImm_D_size<u, opcode, asmop> {
4465   def bbi : NeonI_ScalarShiftImm_size<u, opcode, asmop, FPR8, shl_imm8> {
4466     bits<3> Imm;
4467     let Inst{22-19} = 0b0001; // immh:immb = 0001xxx
4468     let Inst{18-16} = Imm;
4469   }
4470   def hhi : NeonI_ScalarShiftImm_size<u, opcode, asmop, FPR16, shl_imm16> {
4471     bits<4> Imm;
4472     let Inst{22-20} = 0b001; // immh:immb = 001xxxx
4473     let Inst{19-16} = Imm;
4474   }
4475   def ssi : NeonI_ScalarShiftImm_size<u, opcode, asmop, FPR32, shl_imm32> {
4476     bits<5> Imm;
4477     let Inst{22-21} = 0b01; // immh:immb = 01xxxxx
4478     let Inst{20-16} = Imm;
4479   }
4480 }
4481
4482 class NeonI_ScalarShiftRightImm_accum_D_size<bit u, bits<5> opcode, string asmop>
4483   : NeonI_ScalarShiftImm<u, opcode,
4484                          (outs FPR64:$Rd),
4485                          (ins FPR64:$Src, FPR64:$Rn, shr_imm64:$Imm),
4486                          !strconcat(asmop, "\t$Rd, $Rn, $Imm"),
4487                          [], NoItinerary> {
4488     bits<6> Imm;
4489     let Inst{22} = 0b1; // immh:immb = 1xxxxxx
4490     let Inst{21-16} = Imm;
4491     let Constraints = "$Src = $Rd";
4492 }
4493
4494 class NeonI_ScalarShiftLeftImm_accum_D_size<bit u, bits<5> opcode, string asmop>
4495   : NeonI_ScalarShiftImm<u, opcode,
4496                          (outs FPR64:$Rd),
4497                          (ins FPR64:$Src, FPR64:$Rn, shl_imm64:$Imm),
4498                          !strconcat(asmop, "\t$Rd, $Rn, $Imm"),
4499                          [], NoItinerary> {
4500     bits<6> Imm;
4501     let Inst{22} = 0b1; // immh:immb = 1xxxxxx
4502     let Inst{21-16} = Imm;
4503     let Constraints = "$Src = $Rd";
4504 }
4505
4506 class NeonI_ScalarShiftImm_narrow_size<bit u, bits<5> opcode, string asmop,
4507                                        RegisterClass FPRCD, RegisterClass FPRCS,
4508                                        Operand ImmTy>
4509   : NeonI_ScalarShiftImm<u, opcode,
4510                          (outs FPRCD:$Rd), (ins FPRCS:$Rn, ImmTy:$Imm),
4511                          !strconcat(asmop, "\t$Rd, $Rn, $Imm"),
4512                          [], NoItinerary>;
4513
4514 multiclass NeonI_ScalarShiftImm_narrow_HSD_size<bit u, bits<5> opcode,
4515                                                 string asmop> {
4516   def bhi : NeonI_ScalarShiftImm_narrow_size<u, opcode, asmop, FPR8, FPR16,
4517                                              shr_imm8> {
4518     bits<3> Imm;
4519     let Inst{22-19} = 0b0001; // immh:immb = 0001xxx
4520     let Inst{18-16} = Imm;
4521   }
4522   def hsi : NeonI_ScalarShiftImm_narrow_size<u, opcode, asmop, FPR16, FPR32,
4523                                              shr_imm16> {
4524     bits<4> Imm;
4525     let Inst{22-20} = 0b001; // immh:immb = 001xxxx
4526     let Inst{19-16} = Imm;
4527   }
4528   def sdi : NeonI_ScalarShiftImm_narrow_size<u, opcode, asmop, FPR32, FPR64,
4529                                              shr_imm32> {
4530     bits<5> Imm;
4531     let Inst{22-21} = 0b01; // immh:immb = 01xxxxx
4532     let Inst{20-16} = Imm;
4533   }
4534 }
4535
4536 multiclass NeonI_ScalarShiftImm_cvt_SD_size<bit u, bits<5> opcode, string asmop> {
4537   def ssi : NeonI_ScalarShiftImm_size<u, opcode, asmop, FPR32, shr_imm32> {
4538     bits<5> Imm;
4539     let Inst{22-21} = 0b01; // immh:immb = 01xxxxx
4540     let Inst{20-16} = Imm;
4541   }
4542   def ddi : NeonI_ScalarShiftImm_size<u, opcode, asmop, FPR64, shr_imm64> {
4543     bits<6> Imm;
4544     let Inst{22} = 0b1; // immh:immb = 1xxxxxx
4545     let Inst{21-16} = Imm;
4546   }
4547 }
4548
4549 multiclass Neon_ScalarShiftRImm_D_size_patterns<SDPatternOperator opnode,
4550                                                Instruction INSTD> {
4551   def ddi : Pat<(v1i64 (opnode (v1i64 FPR64:$Rn), (i32 shr_imm64:$Imm))),
4552                 (INSTD FPR64:$Rn, imm:$Imm)>;
4553 }
4554
4555 multiclass Neon_ScalarShiftLImm_D_size_patterns<SDPatternOperator opnode,
4556                                                Instruction INSTD> {
4557   def ddi : Pat<(v1i64 (opnode (v1i64 FPR64:$Rn), (i32 shl_imm64:$Imm))),
4558                 (INSTD FPR64:$Rn, imm:$Imm)>;
4559 }
4560
4561 class Neon_ScalarShiftImm_arm_D_size_patterns<SDPatternOperator opnode,
4562                                               Instruction INSTD>
4563   : Pat<(v1i64 (opnode (v1i64 FPR64:$Rn),
4564             (v1i64 (Neon_vdup (i32 shr_imm64:$Imm))))),
4565         (INSTD FPR64:$Rn, imm:$Imm)>;
4566
4567 multiclass Neon_ScalarShiftLImm_BHSD_size_patterns<SDPatternOperator opnode,
4568                                                    Instruction INSTB,
4569                                                    Instruction INSTH,
4570                                                    Instruction INSTS,
4571                                                    Instruction INSTD>
4572   : Neon_ScalarShiftLImm_D_size_patterns<opnode, INSTD> {
4573   def bbi : Pat<(v1i8 (opnode (v1i8 FPR8:$Rn), (i32 shl_imm8:$Imm))),
4574                 (INSTB FPR8:$Rn, imm:$Imm)>;
4575   def hhi : Pat<(v1i16 (opnode (v1i16 FPR16:$Rn), (i32 shl_imm16:$Imm))),
4576                 (INSTH FPR16:$Rn, imm:$Imm)>;
4577   def ssi : Pat<(v1i32 (opnode (v1i32 FPR32:$Rn), (i32 shl_imm32:$Imm))),
4578                 (INSTS FPR32:$Rn, imm:$Imm)>;
4579 }
4580
4581 class Neon_ScalarShiftLImm_accum_D_size_patterns<SDPatternOperator opnode,
4582                                                 Instruction INSTD>
4583   : Pat<(v1i64 (opnode (v1i64 FPR64:$Src), (v1i64 FPR64:$Rn),
4584             (i32 shl_imm64:$Imm))),
4585         (INSTD FPR64:$Src, FPR64:$Rn, imm:$Imm)>;
4586
4587 class Neon_ScalarShiftRImm_accum_D_size_patterns<SDPatternOperator opnode,
4588                                                 Instruction INSTD>
4589   : Pat<(v1i64 (opnode (v1i64 FPR64:$Src), (v1i64 FPR64:$Rn),
4590             (i32 shr_imm64:$Imm))),
4591         (INSTD FPR64:$Src, FPR64:$Rn, imm:$Imm)>;
4592
4593 multiclass Neon_ScalarShiftImm_narrow_HSD_size_patterns<
4594                                                        SDPatternOperator opnode,
4595                                                        Instruction INSTH,
4596                                                        Instruction INSTS,
4597                                                        Instruction INSTD> {
4598   def bhi : Pat<(v1i8 (opnode (v1i16 FPR16:$Rn), (i32 shr_imm16:$Imm))),
4599                 (INSTH FPR16:$Rn, imm:$Imm)>;
4600   def hsi : Pat<(v1i16 (opnode (v1i32 FPR32:$Rn), (i32 shr_imm32:$Imm))),
4601                 (INSTS FPR32:$Rn, imm:$Imm)>;
4602   def sdi : Pat<(v1i32 (opnode (v1i64 FPR64:$Rn), (i32 shr_imm64:$Imm))),
4603                 (INSTD FPR64:$Rn, imm:$Imm)>;
4604 }
4605
4606 multiclass Neon_ScalarShiftImm_scvtf_SD_size_patterns<SDPatternOperator Sopnode,
4607                                                       SDPatternOperator Dopnode,
4608                                                       Instruction INSTS,
4609                                                       Instruction INSTD> {
4610   def ssi : Pat<(f32 (Sopnode (v1i32 FPR32:$Rn), (i32 shr_imm32:$Imm))),
4611                 (INSTS FPR32:$Rn, imm:$Imm)>;
4612   def ddi : Pat<(f64 (Dopnode (v1i64 FPR64:$Rn), (i32 shr_imm64:$Imm))),
4613                 (INSTD FPR64:$Rn, imm:$Imm)>;
4614 }
4615
4616 multiclass Neon_ScalarShiftImm_fcvts_SD_size_patterns<SDPatternOperator Sopnode,
4617                                                       SDPatternOperator Dopnode,
4618                                                       Instruction INSTS,
4619                                                       Instruction INSTD> {
4620   def ssi : Pat<(v1i32 (Sopnode (v1f32 FPR32:$Rn), (i32 shr_imm32:$Imm))),
4621                 (INSTS FPR32:$Rn, imm:$Imm)>;
4622   def ddi : Pat<(v1i64 (Dopnode (v1f64 FPR64:$Rn), (i32 shr_imm64:$Imm))),
4623                 (INSTD FPR64:$Rn, imm:$Imm)>;
4624 }
4625
4626 // Scalar Signed Shift Right (Immediate)
4627 defm SSHR : NeonI_ScalarShiftRightImm_D_size<0b0, 0b00000, "sshr">;
4628 defm : Neon_ScalarShiftRImm_D_size_patterns<int_aarch64_neon_vshrds_n, SSHRddi>;
4629 // Pattern to match llvm.arm.* intrinsic.
4630 def : Neon_ScalarShiftImm_arm_D_size_patterns<sra, SSHRddi>;
4631
4632 // Scalar Unsigned Shift Right (Immediate)
4633 defm USHR : NeonI_ScalarShiftRightImm_D_size<0b1, 0b00000, "ushr">;
4634 defm : Neon_ScalarShiftRImm_D_size_patterns<int_aarch64_neon_vshrdu_n, USHRddi>;
4635 // Pattern to match llvm.arm.* intrinsic.
4636 def : Neon_ScalarShiftImm_arm_D_size_patterns<srl, USHRddi>;
4637
4638 // Scalar Signed Rounding Shift Right (Immediate)
4639 defm SRSHR : NeonI_ScalarShiftRightImm_D_size<0b0, 0b00100, "srshr">;
4640 defm : Neon_ScalarShiftRImm_D_size_patterns<int_aarch64_neon_vsrshr, SRSHRddi>;
4641
4642 // Scalar Unigned Rounding Shift Right (Immediate)
4643 defm URSHR : NeonI_ScalarShiftRightImm_D_size<0b1, 0b00100, "urshr">;
4644 defm : Neon_ScalarShiftRImm_D_size_patterns<int_aarch64_neon_vurshr, URSHRddi>;
4645
4646 // Scalar Signed Shift Right and Accumulate (Immediate)
4647 def SSRA : NeonI_ScalarShiftRightImm_accum_D_size<0b0, 0b00010, "ssra">;
4648 def : Neon_ScalarShiftRImm_accum_D_size_patterns
4649           <int_aarch64_neon_vsrads_n, SSRA>;
4650
4651 // Scalar Unsigned Shift Right and Accumulate (Immediate)
4652 def USRA : NeonI_ScalarShiftRightImm_accum_D_size<0b1, 0b00010, "usra">;
4653 def : Neon_ScalarShiftRImm_accum_D_size_patterns
4654           <int_aarch64_neon_vsradu_n, USRA>;
4655
4656 // Scalar Signed Rounding Shift Right and Accumulate (Immediate)
4657 def SRSRA : NeonI_ScalarShiftRightImm_accum_D_size<0b0, 0b00110, "srsra">;
4658 def : Neon_ScalarShiftRImm_accum_D_size_patterns
4659           <int_aarch64_neon_vrsrads_n, SRSRA>;
4660
4661 // Scalar Unsigned Rounding Shift Right and Accumulate (Immediate)
4662 def URSRA : NeonI_ScalarShiftRightImm_accum_D_size<0b1, 0b00110, "ursra">;
4663 def : Neon_ScalarShiftRImm_accum_D_size_patterns
4664           <int_aarch64_neon_vrsradu_n, URSRA>;
4665
4666 // Scalar Shift Left (Immediate)
4667 defm SHL : NeonI_ScalarShiftLeftImm_D_size<0b0, 0b01010, "shl">;
4668 defm : Neon_ScalarShiftLImm_D_size_patterns<int_aarch64_neon_vshld_n, SHLddi>;
4669 // Pattern to match llvm.arm.* intrinsic.
4670 def : Neon_ScalarShiftImm_arm_D_size_patterns<shl, SHLddi>;
4671
4672 // Signed Saturating Shift Left (Immediate)
4673 defm SQSHL : NeonI_ScalarShiftLeftImm_BHSD_size<0b0, 0b01110, "sqshl">;
4674 defm : Neon_ScalarShiftLImm_BHSD_size_patterns<int_aarch64_neon_vqshls_n,
4675                                                SQSHLbbi, SQSHLhhi,
4676                                                SQSHLssi, SQSHLddi>;
4677 // Pattern to match llvm.arm.* intrinsic.
4678 defm : Neon_ScalarShiftLImm_D_size_patterns<Neon_sqrshlImm, SQSHLddi>;
4679
4680 // Unsigned Saturating Shift Left (Immediate)
4681 defm UQSHL : NeonI_ScalarShiftLeftImm_BHSD_size<0b1, 0b01110, "uqshl">;
4682 defm : Neon_ScalarShiftLImm_BHSD_size_patterns<int_aarch64_neon_vqshlu_n,
4683                                                UQSHLbbi, UQSHLhhi,
4684                                                UQSHLssi, UQSHLddi>;
4685 // Pattern to match llvm.arm.* intrinsic.
4686 defm : Neon_ScalarShiftLImm_D_size_patterns<Neon_uqrshlImm, UQSHLddi>;
4687
4688 // Signed Saturating Shift Left Unsigned (Immediate)
4689 defm SQSHLU : NeonI_ScalarShiftLeftImm_BHSD_size<0b1, 0b01100, "sqshlu">;
4690 defm : Neon_ScalarShiftLImm_BHSD_size_patterns<int_aarch64_neon_vsqshlu,
4691                                                SQSHLUbbi, SQSHLUhhi,
4692                                                SQSHLUssi, SQSHLUddi>;
4693
4694 // Shift Right And Insert (Immediate)
4695 def SRI : NeonI_ScalarShiftRightImm_accum_D_size<0b1, 0b01000, "sri">;
4696 def : Neon_ScalarShiftRImm_accum_D_size_patterns
4697           <int_aarch64_neon_vsri, SRI>;
4698
4699 // Shift Left And Insert (Immediate)
4700 def SLI : NeonI_ScalarShiftLeftImm_accum_D_size<0b1, 0b01010, "sli">;
4701 def : Neon_ScalarShiftLImm_accum_D_size_patterns
4702           <int_aarch64_neon_vsli, SLI>;
4703
4704 // Signed Saturating Shift Right Narrow (Immediate)
4705 defm SQSHRN : NeonI_ScalarShiftImm_narrow_HSD_size<0b0, 0b10010, "sqshrn">;
4706 defm : Neon_ScalarShiftImm_narrow_HSD_size_patterns<int_aarch64_neon_vsqshrn,
4707                                                     SQSHRNbhi, SQSHRNhsi,
4708                                                     SQSHRNsdi>;
4709
4710 // Unsigned Saturating Shift Right Narrow (Immediate)
4711 defm UQSHRN : NeonI_ScalarShiftImm_narrow_HSD_size<0b1, 0b10010, "uqshrn">;
4712 defm : Neon_ScalarShiftImm_narrow_HSD_size_patterns<int_aarch64_neon_vuqshrn,
4713                                                     UQSHRNbhi, UQSHRNhsi,
4714                                                     UQSHRNsdi>;
4715
4716 // Signed Saturating Rounded Shift Right Narrow (Immediate)
4717 defm SQRSHRN : NeonI_ScalarShiftImm_narrow_HSD_size<0b0, 0b10011, "sqrshrn">;
4718 defm : Neon_ScalarShiftImm_narrow_HSD_size_patterns<int_aarch64_neon_vsqrshrn,
4719                                                     SQRSHRNbhi, SQRSHRNhsi,
4720                                                     SQRSHRNsdi>;
4721
4722 // Unsigned Saturating Rounded Shift Right Narrow (Immediate)
4723 defm UQRSHRN : NeonI_ScalarShiftImm_narrow_HSD_size<0b1, 0b10011, "uqrshrn">;
4724 defm : Neon_ScalarShiftImm_narrow_HSD_size_patterns<int_aarch64_neon_vuqrshrn,
4725                                                     UQRSHRNbhi, UQRSHRNhsi,
4726                                                     UQRSHRNsdi>;
4727
4728 // Signed Saturating Shift Right Unsigned Narrow (Immediate)
4729 defm SQSHRUN : NeonI_ScalarShiftImm_narrow_HSD_size<0b1, 0b10000, "sqshrun">;
4730 defm : Neon_ScalarShiftImm_narrow_HSD_size_patterns<int_aarch64_neon_vsqshrun,
4731                                                     SQSHRUNbhi, SQSHRUNhsi,
4732                                                     SQSHRUNsdi>;
4733
4734 // Signed Saturating Rounded Shift Right Unsigned Narrow (Immediate)
4735 defm SQRSHRUN : NeonI_ScalarShiftImm_narrow_HSD_size<0b1, 0b10001, "sqrshrun">;
4736 defm : Neon_ScalarShiftImm_narrow_HSD_size_patterns<int_aarch64_neon_vsqrshrun,
4737                                                     SQRSHRUNbhi, SQRSHRUNhsi,
4738                                                     SQRSHRUNsdi>;
4739
4740 // Scalar Signed Fixed-point Convert To Floating-Point (Immediate)
4741 defm SCVTF_N : NeonI_ScalarShiftImm_cvt_SD_size<0b0, 0b11100, "scvtf">;
4742 defm : Neon_ScalarShiftImm_scvtf_SD_size_patterns<int_aarch64_neon_vcvtf32_n_s32,
4743                                                   int_aarch64_neon_vcvtf64_n_s64,
4744                                                   SCVTF_Nssi, SCVTF_Nddi>;
4745
4746 // Scalar Unsigned Fixed-point Convert To Floating-Point (Immediate)
4747 defm UCVTF_N : NeonI_ScalarShiftImm_cvt_SD_size<0b1, 0b11100, "ucvtf">;
4748 defm : Neon_ScalarShiftImm_scvtf_SD_size_patterns<int_aarch64_neon_vcvtf32_n_u32,
4749                                                   int_aarch64_neon_vcvtf64_n_u64,
4750                                                   UCVTF_Nssi, UCVTF_Nddi>;
4751
4752 // Scalar Floating-point Convert To Signed Fixed-point (Immediate)
4753 defm FCVTZS_N : NeonI_ScalarShiftImm_cvt_SD_size<0b0, 0b11111, "fcvtzs">;
4754 defm : Neon_ScalarShiftImm_fcvts_SD_size_patterns<int_aarch64_neon_vcvts_n_s32_f32,
4755                                                   int_aarch64_neon_vcvtd_n_s64_f64,
4756                                                   FCVTZS_Nssi, FCVTZS_Nddi>;
4757
4758 // Scalar Floating-point Convert To Unsigned Fixed-point (Immediate)
4759 defm FCVTZU_N : NeonI_ScalarShiftImm_cvt_SD_size<0b1, 0b11111, "fcvtzu">;
4760 defm : Neon_ScalarShiftImm_fcvts_SD_size_patterns<int_aarch64_neon_vcvts_n_u32_f32,
4761                                                   int_aarch64_neon_vcvtd_n_u64_f64,
4762                                                   FCVTZU_Nssi, FCVTZU_Nddi>;
4763
4764 // Patterns For Convert Instructions Between v1f64 and v1i64
4765 class Neon_ScalarShiftImm_cvtf_v1f64_pattern<SDPatternOperator opnode,
4766                                              Instruction INST>
4767     : Pat<(v1f64 (opnode (v1i64 FPR64:$Rn), (i32 shr_imm64:$Imm))),
4768           (INST FPR64:$Rn, imm:$Imm)>;
4769
4770 class Neon_ScalarShiftImm_fcvt_v1f64_pattern<SDPatternOperator opnode,
4771                                              Instruction INST>
4772     : Pat<(v1i64 (opnode (v1f64 FPR64:$Rn), (i32 shr_imm64:$Imm))),
4773           (INST FPR64:$Rn, imm:$Imm)>;
4774
4775 def : Neon_ScalarShiftImm_cvtf_v1f64_pattern<int_arm_neon_vcvtfxs2fp,
4776                                              SCVTF_Nddi>;
4777
4778 def : Neon_ScalarShiftImm_cvtf_v1f64_pattern<int_arm_neon_vcvtfxu2fp,
4779                                              UCVTF_Nddi>;
4780
4781 def : Neon_ScalarShiftImm_fcvt_v1f64_pattern<int_arm_neon_vcvtfp2fxs,
4782                                              FCVTZS_Nddi>;
4783
4784 def : Neon_ScalarShiftImm_fcvt_v1f64_pattern<int_arm_neon_vcvtfp2fxu,
4785                                              FCVTZU_Nddi>;
4786
4787 // Scalar Integer Add
4788 let isCommutable = 1 in {
4789 def ADDddd : NeonI_Scalar3Same_D_size<0b0, 0b10000, "add">;
4790 }
4791
4792 // Scalar Integer Sub
4793 def SUBddd : NeonI_Scalar3Same_D_size<0b1, 0b10000, "sub">;
4794
4795 // Pattern for Scalar Integer Add and Sub with D register only
4796 defm : Neon_Scalar3Same_D_size_patterns<add, ADDddd>;
4797 defm : Neon_Scalar3Same_D_size_patterns<sub, SUBddd>;
4798
4799 // Patterns to match llvm.aarch64.* intrinsic for Scalar Add, Sub
4800 defm : Neon_Scalar3Same_D_size_patterns<int_aarch64_neon_vaddds, ADDddd>;
4801 defm : Neon_Scalar3Same_D_size_patterns<int_aarch64_neon_vadddu, ADDddd>;
4802 defm : Neon_Scalar3Same_D_size_patterns<int_aarch64_neon_vsubds, SUBddd>;
4803 defm : Neon_Scalar3Same_D_size_patterns<int_aarch64_neon_vsubdu, SUBddd>;
4804
4805 // Scalar Integer Saturating Add (Signed, Unsigned)
4806 defm SQADD : NeonI_Scalar3Same_BHSD_sizes<0b0, 0b00001, "sqadd", 1>;
4807 defm UQADD : NeonI_Scalar3Same_BHSD_sizes<0b1, 0b00001, "uqadd", 1>;
4808
4809 // Scalar Integer Saturating Sub (Signed, Unsigned)
4810 defm SQSUB : NeonI_Scalar3Same_BHSD_sizes<0b0, 0b00101, "sqsub", 0>;
4811 defm UQSUB : NeonI_Scalar3Same_BHSD_sizes<0b1, 0b00101, "uqsub", 0>;
4812
4813
4814 // Patterns to match llvm.aarch64.* intrinsic for
4815 // Scalar Integer Saturating Add, Sub  (Signed, Unsigned)
4816 defm : Neon_Scalar3Same_BHSD_size_patterns<int_arm_neon_vqadds, SQADDbbb,
4817                                            SQADDhhh, SQADDsss, SQADDddd>;
4818 defm : Neon_Scalar3Same_BHSD_size_patterns<int_arm_neon_vqaddu, UQADDbbb,
4819                                            UQADDhhh, UQADDsss, UQADDddd>;
4820 defm : Neon_Scalar3Same_BHSD_size_patterns<int_arm_neon_vqsubs, SQSUBbbb,
4821                                            SQSUBhhh, SQSUBsss, SQSUBddd>;
4822 defm : Neon_Scalar3Same_BHSD_size_patterns<int_arm_neon_vqsubu, UQSUBbbb,
4823                                            UQSUBhhh, UQSUBsss, UQSUBddd>;
4824
4825 // Scalar Integer Saturating Doubling Multiply Half High
4826 defm SQDMULH : NeonI_Scalar3Same_HS_sizes<0b0, 0b10110, "sqdmulh", 1>;
4827
4828 // Scalar Integer Saturating Rounding Doubling Multiply Half High
4829 defm SQRDMULH : NeonI_Scalar3Same_HS_sizes<0b1, 0b10110, "sqrdmulh", 1>;
4830
4831 // Patterns to match llvm.arm.* intrinsic for
4832 // Scalar Integer Saturating Doubling Multiply Half High and
4833 // Scalar Integer Saturating Rounding Doubling Multiply Half High
4834 defm : Neon_Scalar3Same_HS_size_patterns<int_arm_neon_vqdmulh, SQDMULHhhh,
4835                                                                SQDMULHsss>;
4836 defm : Neon_Scalar3Same_HS_size_patterns<int_arm_neon_vqrdmulh, SQRDMULHhhh,
4837                                                                 SQRDMULHsss>;
4838
4839 // Scalar Floating-point Multiply Extended
4840 defm FMULX : NeonI_Scalar3Same_SD_sizes<0b0, 0b0, 0b11011, "fmulx", 1>;
4841
4842 // Scalar Floating-point Reciprocal Step
4843 defm FRECPS : NeonI_Scalar3Same_SD_sizes<0b0, 0b0, 0b11111, "frecps", 0>;
4844
4845 // Scalar Floating-point Reciprocal Square Root Step
4846 defm FRSQRTS : NeonI_Scalar3Same_SD_sizes<0b0, 0b1, 0b11111, "frsqrts", 0>;
4847
4848 // Patterns to match llvm.arm.* intrinsic for
4849 // Scalar Floating-point Reciprocal Step and
4850 // Scalar Floating-point Reciprocal Square Root Step
4851 defm : Neon_Scalar3Same_SD_size_patterns<int_arm_neon_vrecps, FRECPSsss,
4852                                                               FRECPSddd>;
4853 defm : Neon_Scalar3Same_SD_size_patterns<int_arm_neon_vrsqrts, FRSQRTSsss,
4854                                                                FRSQRTSddd>;
4855
4856 def : Pat<(v1f64 (fsqrt (v1f64 FPR64:$Rn))), (FSQRTdd FPR64:$Rn)>;
4857
4858 // Patterns to match llvm.aarch64.* intrinsic for
4859 // Scalar Floating-point Multiply Extended,
4860 multiclass Neon_Scalar3Same_MULX_SD_size_patterns<SDPatternOperator opnode,
4861                                                   Instruction INSTS,
4862                                                   Instruction INSTD> {
4863   def : Pat<(f32 (opnode (f32 FPR32:$Rn), (f32 FPR32:$Rm))),
4864             (INSTS FPR32:$Rn, FPR32:$Rm)>;
4865   def : Pat<(f64 (opnode (f64 FPR64:$Rn), (f64 FPR64:$Rm))),
4866             (INSTD FPR64:$Rn, FPR64:$Rm)>;
4867 }
4868
4869 defm : Neon_Scalar3Same_MULX_SD_size_patterns<int_aarch64_neon_vmulx,
4870                                               FMULXsss,FMULXddd>;
4871
4872 // Scalar Integer Shift Left (Signed, Unsigned)
4873 def SSHLddd : NeonI_Scalar3Same_D_size<0b0, 0b01000, "sshl">;
4874 def USHLddd : NeonI_Scalar3Same_D_size<0b1, 0b01000, "ushl">;
4875
4876 // Patterns to match llvm.arm.* intrinsic for
4877 // Scalar Integer Shift Left (Signed, Unsigned)
4878 defm : Neon_Scalar3Same_D_size_patterns<int_arm_neon_vshifts, SSHLddd>;
4879 defm : Neon_Scalar3Same_D_size_patterns<int_arm_neon_vshiftu, USHLddd>;
4880
4881 // Patterns to match llvm.aarch64.* intrinsic for
4882 // Scalar Integer Shift Left (Signed, Unsigned)
4883 defm : Neon_Scalar3Same_D_size_patterns<int_aarch64_neon_vshlds, SSHLddd>;
4884 defm : Neon_Scalar3Same_D_size_patterns<int_aarch64_neon_vshldu, USHLddd>;
4885
4886 // Scalar Integer Saturating Shift Left (Signed, Unsigned)
4887 defm SQSHL: NeonI_Scalar3Same_BHSD_sizes<0b0, 0b01001, "sqshl", 0>;
4888 defm UQSHL: NeonI_Scalar3Same_BHSD_sizes<0b1, 0b01001, "uqshl", 0>;
4889
4890 // Patterns to match llvm.aarch64.* intrinsic for
4891 // Scalar  Integer Saturating Shift Letf (Signed, Unsigned)
4892 defm : Neon_Scalar3Same_BHSD_size_patterns<int_aarch64_neon_vqshls, SQSHLbbb,
4893                                            SQSHLhhh, SQSHLsss, SQSHLddd>;
4894 defm : Neon_Scalar3Same_BHSD_size_patterns<int_aarch64_neon_vqshlu, UQSHLbbb,
4895                                            UQSHLhhh, UQSHLsss, UQSHLddd>;
4896
4897 // Patterns to match llvm.arm.* intrinsic for
4898 // Scalar  Integer Saturating Shift Letf (Signed, Unsigned)
4899 defm : Neon_Scalar3Same_D_size_patterns<int_arm_neon_vqshifts, SQSHLddd>;
4900 defm : Neon_Scalar3Same_D_size_patterns<int_arm_neon_vqshiftu, UQSHLddd>;
4901
4902 // Scalar Integer Rounding Shift Left (Signed, Unsigned)
4903 def SRSHLddd: NeonI_Scalar3Same_D_size<0b0, 0b01010, "srshl">;
4904 def URSHLddd: NeonI_Scalar3Same_D_size<0b1, 0b01010, "urshl">;
4905
4906 // Patterns to match llvm.aarch64.* intrinsic for
4907 // Scalar Integer Rounding Shift Left (Signed, Unsigned)
4908 defm : Neon_Scalar3Same_D_size_patterns<int_aarch64_neon_vrshlds, SRSHLddd>;
4909 defm : Neon_Scalar3Same_D_size_patterns<int_aarch64_neon_vrshldu, URSHLddd>;
4910
4911 // Patterns to match llvm.arm.* intrinsic for
4912 // Scalar Integer Rounding Shift Left (Signed, Unsigned)
4913 defm : Neon_Scalar3Same_D_size_patterns<int_arm_neon_vrshifts, SRSHLddd>;
4914 defm : Neon_Scalar3Same_D_size_patterns<int_arm_neon_vrshiftu, URSHLddd>;
4915
4916 // Scalar Integer Saturating Rounding Shift Left (Signed, Unsigned)
4917 defm SQRSHL: NeonI_Scalar3Same_BHSD_sizes<0b0, 0b01011, "sqrshl", 0>;
4918 defm UQRSHL: NeonI_Scalar3Same_BHSD_sizes<0b1, 0b01011, "uqrshl", 0>;
4919
4920 // Patterns to match llvm.aarch64.* intrinsic for
4921 // Scalar Integer Saturating Rounding Shift Left (Signed, Unsigned)
4922 defm : Neon_Scalar3Same_BHSD_size_patterns<int_aarch64_neon_vqrshls, SQRSHLbbb,
4923                                            SQRSHLhhh, SQRSHLsss, SQRSHLddd>;
4924 defm : Neon_Scalar3Same_BHSD_size_patterns<int_aarch64_neon_vqrshlu, UQRSHLbbb,
4925                                            UQRSHLhhh, UQRSHLsss, UQRSHLddd>;
4926
4927 // Patterns to match llvm.arm.* intrinsic for
4928 // Scalar Integer Saturating Rounding Shift Left (Signed, Unsigned)
4929 defm : Neon_Scalar3Same_D_size_patterns<int_arm_neon_vqrshifts, SQRSHLddd>;
4930 defm : Neon_Scalar3Same_D_size_patterns<int_arm_neon_vqrshiftu, UQRSHLddd>;
4931
4932 // Signed Saturating Doubling Multiply-Add Long
4933 defm SQDMLAL : NeonI_Scalar3Diff_ml_HS_size<0b0, 0b1001, "sqdmlal">;
4934 defm : Neon_Scalar3Diff_ml_HS_size_patterns<int_aarch64_neon_vqdmlal,
4935                                             SQDMLALshh, SQDMLALdss>;
4936
4937 // Signed Saturating Doubling Multiply-Subtract Long
4938 defm SQDMLSL : NeonI_Scalar3Diff_ml_HS_size<0b0, 0b1011, "sqdmlsl">;
4939 defm : Neon_Scalar3Diff_ml_HS_size_patterns<int_aarch64_neon_vqdmlsl,
4940                                             SQDMLSLshh, SQDMLSLdss>;
4941
4942 // Signed Saturating Doubling Multiply Long
4943 defm SQDMULL : NeonI_Scalar3Diff_HS_size<0b0, 0b1101, "sqdmull">;
4944 defm : Neon_Scalar3Diff_HS_size_patterns<int_arm_neon_vqdmull,
4945                                          SQDMULLshh, SQDMULLdss>;
4946
4947 // Scalar Signed Integer Convert To Floating-point
4948 defm SCVTF  : NeonI_Scalar2SameMisc_SD_size<0b0, 0b0, 0b11101, "scvtf">;
4949 defm : Neon_Scalar2SameMisc_cvt_SD_size_patterns<int_aarch64_neon_vcvtf32_s32,
4950                                                  int_aarch64_neon_vcvtf64_s64,
4951                                                  SCVTFss, SCVTFdd>;
4952
4953 // Scalar Unsigned Integer Convert To Floating-point
4954 defm UCVTF  : NeonI_Scalar2SameMisc_SD_size<0b1, 0b0, 0b11101, "ucvtf">;
4955 defm : Neon_Scalar2SameMisc_cvt_SD_size_patterns<int_aarch64_neon_vcvtf32_u32,
4956                                                  int_aarch64_neon_vcvtf64_u64,
4957                                                  UCVTFss, UCVTFdd>;
4958
4959 // Scalar Floating-point Converts
4960 def FCVTXN : NeonI_Scalar2SameMisc_fcvtxn_D_size<0b1, 0b10110, "fcvtxn">;
4961 def : Neon_Scalar2SameMisc_fcvtxn_D_size_patterns<int_aarch64_neon_fcvtxn,
4962                                                   FCVTXN>;
4963
4964 defm FCVTNS : NeonI_Scalar2SameMisc_SD_size<0b0, 0b0, 0b11010, "fcvtns">;
4965 defm : Neon_Scalar2SameMisc_fcvt_SD_size_patterns<int_aarch64_neon_fcvtns,
4966                                                   FCVTNSss, FCVTNSdd>;
4967
4968 defm FCVTNU : NeonI_Scalar2SameMisc_SD_size<0b1, 0b0, 0b11010, "fcvtnu">;
4969 defm : Neon_Scalar2SameMisc_fcvt_SD_size_patterns<int_aarch64_neon_fcvtnu,
4970                                                   FCVTNUss, FCVTNUdd>;
4971
4972 defm FCVTMS : NeonI_Scalar2SameMisc_SD_size<0b0, 0b0, 0b11011, "fcvtms">;
4973 defm : Neon_Scalar2SameMisc_fcvt_SD_size_patterns<int_aarch64_neon_fcvtms,
4974                                                   FCVTMSss, FCVTMSdd>;
4975
4976 defm FCVTMU : NeonI_Scalar2SameMisc_SD_size<0b1, 0b0, 0b11011, "fcvtmu">;
4977 defm : Neon_Scalar2SameMisc_fcvt_SD_size_patterns<int_aarch64_neon_fcvtmu,
4978                                                   FCVTMUss, FCVTMUdd>;
4979
4980 defm FCVTAS : NeonI_Scalar2SameMisc_SD_size<0b0, 0b0, 0b11100, "fcvtas">;
4981 defm : Neon_Scalar2SameMisc_fcvt_SD_size_patterns<int_aarch64_neon_fcvtas,
4982                                                   FCVTASss, FCVTASdd>;
4983
4984 defm FCVTAU : NeonI_Scalar2SameMisc_SD_size<0b1, 0b0, 0b11100, "fcvtau">;
4985 defm : Neon_Scalar2SameMisc_fcvt_SD_size_patterns<int_aarch64_neon_fcvtau,
4986                                                   FCVTAUss, FCVTAUdd>;
4987
4988 defm FCVTPS : NeonI_Scalar2SameMisc_SD_size<0b0, 0b1, 0b11010, "fcvtps">;
4989 defm : Neon_Scalar2SameMisc_fcvt_SD_size_patterns<int_aarch64_neon_fcvtps,
4990                                                   FCVTPSss, FCVTPSdd>;
4991
4992 defm FCVTPU : NeonI_Scalar2SameMisc_SD_size<0b1, 0b1, 0b11010, "fcvtpu">;
4993 defm : Neon_Scalar2SameMisc_fcvt_SD_size_patterns<int_aarch64_neon_fcvtpu,
4994                                                   FCVTPUss, FCVTPUdd>;
4995
4996 defm FCVTZS : NeonI_Scalar2SameMisc_SD_size<0b0, 0b1, 0b11011, "fcvtzs">;
4997 defm : Neon_Scalar2SameMisc_fcvt_SD_size_patterns<int_aarch64_neon_fcvtzs,
4998                                                   FCVTZSss, FCVTZSdd>;
4999
5000 defm FCVTZU : NeonI_Scalar2SameMisc_SD_size<0b1, 0b1, 0b11011, "fcvtzu">;
5001 defm : Neon_Scalar2SameMisc_fcvt_SD_size_patterns<int_aarch64_neon_fcvtzu,
5002                                                   FCVTZUss, FCVTZUdd>;
5003
5004 // Patterns For Convert Instructions Between v1f64 and v1i64
5005 class Neon_Scalar2SameMisc_cvtf_v1f64_pattern<SDPatternOperator opnode,
5006                                               Instruction INST>
5007     : Pat<(v1f64 (opnode (v1i64 FPR64:$Rn))), (INST FPR64:$Rn)>;
5008
5009 class Neon_Scalar2SameMisc_fcvt_v1f64_pattern<SDPatternOperator opnode,
5010                                               Instruction INST>
5011     : Pat<(v1i64 (opnode (v1f64 FPR64:$Rn))), (INST FPR64:$Rn)>;
5012
5013 def : Neon_Scalar2SameMisc_cvtf_v1f64_pattern<sint_to_fp, SCVTFdd>;
5014 def : Neon_Scalar2SameMisc_cvtf_v1f64_pattern<uint_to_fp, UCVTFdd>;
5015
5016 def : Neon_Scalar2SameMisc_fcvt_v1f64_pattern<fp_to_sint, FCVTZSdd>;
5017 def : Neon_Scalar2SameMisc_fcvt_v1f64_pattern<fp_to_uint, FCVTZUdd>;
5018
5019 // Scalar Floating-point Reciprocal Estimate
5020 defm FRECPE : NeonI_Scalar2SameMisc_SD_size<0b0, 0b1, 0b11101, "frecpe">;
5021 defm : Neon_Scalar2SameMisc_SD_size_patterns<int_arm_neon_vrecpe,
5022                                              FRECPEss, FRECPEdd>;
5023
5024 // Scalar Floating-point Reciprocal Exponent
5025 defm FRECPX : NeonI_Scalar2SameMisc_SD_size<0b0, 0b1, 0b11111, "frecpx">;
5026 defm : Neon_Scalar2SameMisc_SD_size_patterns<int_aarch64_neon_vrecpx,
5027                                              FRECPXss, FRECPXdd>;
5028
5029 // Scalar Floating-point Reciprocal Square Root Estimate
5030 defm FRSQRTE: NeonI_Scalar2SameMisc_SD_size<0b1, 0b1, 0b11101, "frsqrte">;
5031 defm : Neon_Scalar2SameMisc_SD_size_patterns<int_arm_neon_vrsqrte,
5032                                              FRSQRTEss, FRSQRTEdd>;
5033
5034 // Scalar Floating-point Round
5035 class Neon_ScalarFloatRound_pattern<SDPatternOperator opnode, Instruction INST>
5036     : Pat<(v1f64 (opnode (v1f64 FPR64:$Rn))), (INST FPR64:$Rn)>;
5037
5038 def : Neon_ScalarFloatRound_pattern<fceil, FRINTPdd>;
5039 def : Neon_ScalarFloatRound_pattern<ffloor, FRINTMdd>;
5040 def : Neon_ScalarFloatRound_pattern<ftrunc, FRINTZdd>;
5041 def : Neon_ScalarFloatRound_pattern<frint, FRINTXdd>;
5042 def : Neon_ScalarFloatRound_pattern<fnearbyint, FRINTIdd>;
5043 def : Neon_ScalarFloatRound_pattern<frnd, FRINTAdd>;
5044 def : Neon_ScalarFloatRound_pattern<int_aarch64_neon_frintn, FRINTNdd>;
5045
5046 // Scalar Integer Compare
5047
5048 // Scalar Compare Bitwise Equal
5049 def CMEQddd: NeonI_Scalar3Same_D_size<0b1, 0b10001, "cmeq">;
5050 def : Neon_Scalar3Same_cmp_D_size_patterns<int_aarch64_neon_vceq, CMEQddd>;
5051
5052 class Neon_Scalar3Same_cmp_D_size_v1_patterns<SDPatternOperator opnode,
5053                                               Instruction INSTD,
5054                                               CondCode CC>
5055   : Pat<(v1i64 (opnode (v1i64 FPR64:$Rn), (v1i64 FPR64:$Rm), CC)),
5056         (INSTD FPR64:$Rn, FPR64:$Rm)>;
5057
5058 def : Neon_Scalar3Same_cmp_D_size_v1_patterns<Neon_cmp, CMEQddd, SETEQ>;
5059
5060 // Scalar Compare Signed Greather Than Or Equal
5061 def CMGEddd: NeonI_Scalar3Same_D_size<0b0, 0b00111, "cmge">;
5062 def : Neon_Scalar3Same_cmp_D_size_patterns<int_aarch64_neon_vcge, CMGEddd>;
5063 def : Neon_Scalar3Same_cmp_D_size_v1_patterns<Neon_cmp, CMGEddd, SETGE>;
5064
5065 // Scalar Compare Unsigned Higher Or Same
5066 def CMHSddd: NeonI_Scalar3Same_D_size<0b1, 0b00111, "cmhs">;
5067 def : Neon_Scalar3Same_cmp_D_size_patterns<int_aarch64_neon_vchs, CMHSddd>;
5068 def : Neon_Scalar3Same_cmp_D_size_v1_patterns<Neon_cmp, CMHSddd, SETUGE>;
5069
5070 // Scalar Compare Unsigned Higher
5071 def CMHIddd: NeonI_Scalar3Same_D_size<0b1, 0b00110, "cmhi">;
5072 def : Neon_Scalar3Same_cmp_D_size_patterns<int_aarch64_neon_vchi, CMHIddd>;
5073 def : Neon_Scalar3Same_cmp_D_size_v1_patterns<Neon_cmp, CMHIddd, SETUGT>;
5074
5075 // Scalar Compare Signed Greater Than
5076 def CMGTddd: NeonI_Scalar3Same_D_size<0b0, 0b00110, "cmgt">;
5077 def : Neon_Scalar3Same_cmp_D_size_patterns<int_aarch64_neon_vcgt, CMGTddd>;
5078 def : Neon_Scalar3Same_cmp_D_size_v1_patterns<Neon_cmp, CMGTddd, SETGT>;
5079
5080 // Scalar Compare Bitwise Test Bits
5081 def CMTSTddd: NeonI_Scalar3Same_D_size<0b0, 0b10001, "cmtst">;
5082 def : Neon_Scalar3Same_cmp_D_size_patterns<int_aarch64_neon_vtstd, CMTSTddd>;
5083 def : Neon_Scalar3Same_cmp_D_size_patterns<Neon_tst, CMTSTddd>;
5084
5085 // Scalar Compare Bitwise Equal To Zero
5086 def CMEQddi: NeonI_Scalar2SameMisc_cmpz_D_size<0b0, 0b01001, "cmeq">;
5087 def : Neon_Scalar2SameMisc_cmpz_D_size_patterns<int_aarch64_neon_vceq,
5088                                                 CMEQddi>;
5089 def : Neon_Scalar2SameMisc_cmpz_D_V1_size_patterns<SETEQ, CMEQddi>;
5090
5091 // Scalar Compare Signed Greather Than Or Equal To Zero
5092 def CMGEddi: NeonI_Scalar2SameMisc_cmpz_D_size<0b1, 0b01000, "cmge">;
5093 def : Neon_Scalar2SameMisc_cmpz_D_size_patterns<int_aarch64_neon_vcge,
5094                                                 CMGEddi>;
5095 def : Neon_Scalar2SameMisc_cmpz_D_V1_size_patterns<SETGE, CMGEddi>;
5096
5097 // Scalar Compare Signed Greater Than Zero
5098 def CMGTddi: NeonI_Scalar2SameMisc_cmpz_D_size<0b0, 0b01000, "cmgt">;
5099 def : Neon_Scalar2SameMisc_cmpz_D_size_patterns<int_aarch64_neon_vcgt,
5100                                                 CMGTddi>;
5101 def : Neon_Scalar2SameMisc_cmpz_D_V1_size_patterns<SETGT, CMGTddi>;
5102
5103 // Scalar Compare Signed Less Than Or Equal To Zero
5104 def CMLEddi: NeonI_Scalar2SameMisc_cmpz_D_size<0b1, 0b01001, "cmle">;
5105 def : Neon_Scalar2SameMisc_cmpz_D_size_patterns<int_aarch64_neon_vclez,
5106                                                 CMLEddi>;
5107 def : Neon_Scalar2SameMisc_cmpz_D_V1_size_patterns<SETLE, CMLEddi>;
5108
5109 // Scalar Compare Less Than Zero
5110 def CMLTddi: NeonI_Scalar2SameMisc_cmpz_D_size<0b0, 0b01010, "cmlt">;
5111 def : Neon_Scalar2SameMisc_cmpz_D_size_patterns<int_aarch64_neon_vcltz,
5112                                                 CMLTddi>;
5113 def : Neon_Scalar2SameMisc_cmpz_D_V1_size_patterns<SETLT, CMLTddi>;
5114
5115 // Scalar Floating-point Compare
5116
5117 // Scalar Floating-point Compare Mask Equal
5118 defm FCMEQ: NeonI_Scalar3Same_SD_sizes<0b0, 0b0, 0b11100, "fcmeq">;
5119 defm : Neon_Scalar3Same_cmp_SD_size_patterns<int_aarch64_neon_vceq,
5120                                              FCMEQsss, FCMEQddd>;
5121 def : Neon_Scalar3Same_cmp_V1_D_size_patterns<SETEQ, FCMEQddd>;
5122
5123 // Scalar Floating-point Compare Mask Equal To Zero
5124 defm FCMEQZ: NeonI_Scalar2SameMisc_cmpz_SD_size<0b0, 0b01101, "fcmeq">;
5125 defm : Neon_Scalar2SameMisc_cmpz_SD_size_patterns<int_aarch64_neon_vceq,
5126                                                   FCMEQZssi, FCMEQZddi>;
5127 def : Pat<(v1i64 (Neon_cmpz (v1f64 FPR64:$Rn), (f32 fpz32:$FPImm), SETEQ)),
5128           (FCMEQZddi FPR64:$Rn, fpz32:$FPImm)>;
5129
5130 // Scalar Floating-point Compare Mask Greater Than Or Equal
5131 defm FCMGE: NeonI_Scalar3Same_SD_sizes<0b1, 0b0, 0b11100, "fcmge">;
5132 defm : Neon_Scalar3Same_cmp_SD_size_patterns<int_aarch64_neon_vcge,
5133                                              FCMGEsss, FCMGEddd>;
5134 def : Neon_Scalar3Same_cmp_V1_D_size_patterns<SETGE, FCMGEddd>;
5135
5136 // Scalar Floating-point Compare Mask Greater Than Or Equal To Zero
5137 defm FCMGEZ: NeonI_Scalar2SameMisc_cmpz_SD_size<0b1, 0b01100, "fcmge">;
5138 defm : Neon_Scalar2SameMisc_cmpz_SD_size_patterns<int_aarch64_neon_vcge,
5139                                                   FCMGEZssi, FCMGEZddi>;
5140
5141 // Scalar Floating-point Compare Mask Greather Than
5142 defm FCMGT: NeonI_Scalar3Same_SD_sizes<0b1, 0b1, 0b11100, "fcmgt">;
5143 defm : Neon_Scalar3Same_cmp_SD_size_patterns<int_aarch64_neon_vcgt,
5144                                              FCMGTsss, FCMGTddd>;
5145 def : Neon_Scalar3Same_cmp_V1_D_size_patterns<SETGT, FCMGTddd>;
5146
5147 // Scalar Floating-point Compare Mask Greather Than Zero
5148 defm FCMGTZ: NeonI_Scalar2SameMisc_cmpz_SD_size<0b0, 0b01100, "fcmgt">;
5149 defm : Neon_Scalar2SameMisc_cmpz_SD_size_patterns<int_aarch64_neon_vcgt,
5150                                                   FCMGTZssi, FCMGTZddi>;
5151
5152 // Scalar Floating-point Compare Mask Less Than Or Equal To Zero
5153 defm FCMLEZ: NeonI_Scalar2SameMisc_cmpz_SD_size<0b1, 0b01101, "fcmle">;
5154 defm : Neon_Scalar2SameMisc_cmpz_SD_size_patterns<int_aarch64_neon_vclez,
5155                                                   FCMLEZssi, FCMLEZddi>;
5156
5157 // Scalar Floating-point Compare Mask Less Than Zero
5158 defm FCMLTZ: NeonI_Scalar2SameMisc_cmpz_SD_size<0b0, 0b01110, "fcmlt">;
5159 defm : Neon_Scalar2SameMisc_cmpz_SD_size_patterns<int_aarch64_neon_vcltz,
5160                                                   FCMLTZssi, FCMLTZddi>;
5161
5162 // Scalar Floating-point Absolute Compare Mask Greater Than Or Equal
5163 defm FACGE: NeonI_Scalar3Same_SD_sizes<0b1, 0b0, 0b11101, "facge">;
5164 defm : Neon_Scalar3Same_cmp_SD_size_patterns<int_aarch64_neon_vcage,
5165                                              FACGEsss, FACGEddd>;
5166
5167 // Scalar Floating-point Absolute Compare Mask Greater Than
5168 defm FACGT: NeonI_Scalar3Same_SD_sizes<0b1, 0b1, 0b11101, "facgt">;
5169 defm : Neon_Scalar3Same_cmp_SD_size_patterns<int_aarch64_neon_vcagt,
5170                                              FACGTsss, FACGTddd>;
5171
5172 // Scakar Floating-point Absolute Difference
5173 defm FABD: NeonI_Scalar3Same_SD_sizes<0b1, 0b1, 0b11010, "fabd">;
5174 defm : Neon_Scalar3Same_SD_size_patterns<int_aarch64_neon_vabd,
5175                                          FABDsss, FABDddd>;
5176
5177 // Scalar Absolute Value
5178 defm ABS : NeonI_Scalar2SameMisc_D_size<0b0, 0b01011, "abs">;
5179 defm : Neon_Scalar2SameMisc_D_size_patterns<int_aarch64_neon_vabs, ABSdd>;
5180
5181 // Scalar Signed Saturating Absolute Value
5182 defm SQABS : NeonI_Scalar2SameMisc_BHSD_size<0b0, 0b00111, "sqabs">;
5183 defm : Neon_Scalar2SameMisc_BHSD_size_patterns<int_arm_neon_vqabs,
5184                                                SQABSbb, SQABShh, SQABSss, SQABSdd>;
5185
5186 // Scalar Negate
5187 defm NEG : NeonI_Scalar2SameMisc_D_size<0b1, 0b01011, "neg">;
5188 defm : Neon_Scalar2SameMisc_D_size_patterns<int_aarch64_neon_vneg, NEGdd>;
5189
5190 // Scalar Signed Saturating Negate
5191 defm SQNEG : NeonI_Scalar2SameMisc_BHSD_size<0b1, 0b00111, "sqneg">;
5192 defm : Neon_Scalar2SameMisc_BHSD_size_patterns<int_arm_neon_vqneg,
5193                                                SQNEGbb, SQNEGhh, SQNEGss, SQNEGdd>;
5194
5195 // Scalar Signed Saturating Accumulated of Unsigned Value
5196 defm SUQADD : NeonI_Scalar2SameMisc_accum_BHSD_size<0b0, 0b00011, "suqadd">;
5197 defm : Neon_Scalar2SameMisc_accum_BHSD_size_patterns<int_aarch64_neon_vuqadd,
5198                                                      SUQADDbb, SUQADDhh,
5199                                                      SUQADDss, SUQADDdd>;
5200
5201 // Scalar Unsigned Saturating Accumulated of Signed Value
5202 defm USQADD : NeonI_Scalar2SameMisc_accum_BHSD_size<0b1, 0b00011, "usqadd">;
5203 defm : Neon_Scalar2SameMisc_accum_BHSD_size_patterns<int_aarch64_neon_vsqadd,
5204                                                      USQADDbb, USQADDhh,
5205                                                      USQADDss, USQADDdd>;
5206
5207 def : Pat<(v1i64 (int_aarch64_neon_suqadd (v1i64 FPR64:$Src),
5208                                           (v1i64 FPR64:$Rn))),
5209           (SUQADDdd FPR64:$Src, FPR64:$Rn)>;
5210
5211 def : Pat<(v1i64 (int_aarch64_neon_usqadd (v1i64 FPR64:$Src),
5212                                           (v1i64 FPR64:$Rn))),
5213           (USQADDdd FPR64:$Src, FPR64:$Rn)>;
5214
5215 def : Pat<(v1i64 (int_arm_neon_vabs (v1i64 FPR64:$Rn))),
5216           (ABSdd FPR64:$Rn)>;
5217
5218 def : Pat<(v1i64 (int_arm_neon_vqabs (v1i64 FPR64:$Rn))),
5219           (SQABSdd FPR64:$Rn)>;
5220
5221 def : Pat<(v1i64 (int_arm_neon_vqneg (v1i64 FPR64:$Rn))),
5222           (SQNEGdd FPR64:$Rn)>;
5223
5224 def : Pat<(v1i64 (sub (v1i64 (bitconvert (v8i8 Neon_AllZero))),
5225                       (v1i64 FPR64:$Rn))),
5226           (NEGdd FPR64:$Rn)>;
5227
5228 // Scalar Signed Saturating Extract Unsigned Narrow
5229 defm SQXTUN : NeonI_Scalar2SameMisc_narrow_HSD_size<0b1, 0b10010, "sqxtun">;
5230 defm : Neon_Scalar2SameMisc_narrow_HSD_size_patterns<int_arm_neon_vqmovnsu,
5231                                                      SQXTUNbh, SQXTUNhs,
5232                                                      SQXTUNsd>;
5233
5234 // Scalar Signed Saturating Extract Narrow
5235 defm SQXTN  : NeonI_Scalar2SameMisc_narrow_HSD_size<0b0, 0b10100, "sqxtn">;
5236 defm : Neon_Scalar2SameMisc_narrow_HSD_size_patterns<int_arm_neon_vqmovns,
5237                                                      SQXTNbh, SQXTNhs,
5238                                                      SQXTNsd>;
5239
5240 // Scalar Unsigned Saturating Extract Narrow
5241 defm UQXTN  : NeonI_Scalar2SameMisc_narrow_HSD_size<0b1, 0b10100, "uqxtn">;
5242 defm : Neon_Scalar2SameMisc_narrow_HSD_size_patterns<int_arm_neon_vqmovnu,
5243                                                      UQXTNbh, UQXTNhs,
5244                                                      UQXTNsd>;
5245
5246 // Scalar Reduce Pairwise
5247
5248 multiclass NeonI_ScalarPair_D_sizes<bit u, bit size, bits<5> opcode,
5249                                      string asmop, bit Commutable = 0> {
5250   let isCommutable = Commutable in {
5251     def _D_2D : NeonI_ScalarPair<u, {size, 0b1}, opcode,
5252                                 (outs FPR64:$Rd), (ins VPR128:$Rn),
5253                                 !strconcat(asmop, "\t$Rd, $Rn.2d"),
5254                                 [],
5255                                 NoItinerary>;
5256   }
5257 }
5258
5259 multiclass NeonI_ScalarPair_SD_sizes<bit u, bit size, bits<5> opcode,
5260                                      string asmop, bit Commutable = 0>
5261   : NeonI_ScalarPair_D_sizes<u, size, opcode, asmop, Commutable> {
5262   let isCommutable = Commutable in {
5263     def _S_2S : NeonI_ScalarPair<u, {size, 0b0}, opcode,
5264                                 (outs FPR32:$Rd), (ins VPR64:$Rn),
5265                                 !strconcat(asmop, "\t$Rd, $Rn.2s"),
5266                                 [],
5267                                 NoItinerary>;
5268   }
5269 }
5270
5271 // Scalar Reduce Addition Pairwise (Integer) with
5272 // Pattern to match llvm.arm.* intrinsic
5273 defm ADDPvv : NeonI_ScalarPair_D_sizes<0b0, 0b1, 0b11011, "addp", 0>;
5274
5275 // Pattern to match llvm.aarch64.* intrinsic for
5276 // Scalar Reduce Addition Pairwise (Integer)
5277 def : Pat<(v1i64 (int_aarch64_neon_vpadd (v2i64 VPR128:$Rn))),
5278           (ADDPvv_D_2D VPR128:$Rn)>;
5279 def : Pat<(v1i64 (int_aarch64_neon_vaddv (v2i64 VPR128:$Rn))),
5280           (ADDPvv_D_2D VPR128:$Rn)>;
5281
5282 // Scalar Reduce Addition Pairwise (Floating Point)
5283 defm FADDPvv : NeonI_ScalarPair_SD_sizes<0b1, 0b0, 0b01101, "faddp", 0>;
5284
5285 // Scalar Reduce Maximum Pairwise (Floating Point)
5286 defm FMAXPvv : NeonI_ScalarPair_SD_sizes<0b1, 0b0, 0b01111, "fmaxp", 0>;
5287
5288 // Scalar Reduce Minimum Pairwise (Floating Point)
5289 defm FMINPvv : NeonI_ScalarPair_SD_sizes<0b1, 0b1, 0b01111, "fminp", 0>;
5290
5291 // Scalar Reduce maxNum Pairwise (Floating Point)
5292 defm FMAXNMPvv : NeonI_ScalarPair_SD_sizes<0b1, 0b0, 0b01100, "fmaxnmp", 0>;
5293
5294 // Scalar Reduce minNum Pairwise (Floating Point)
5295 defm FMINNMPvv : NeonI_ScalarPair_SD_sizes<0b1, 0b1, 0b01100, "fminnmp", 0>;
5296
5297 multiclass Neon_ScalarPair_SD_size_patterns<SDPatternOperator opnodeS,
5298                                             SDPatternOperator opnodeD,
5299                                             Instruction INSTS,
5300                                             Instruction INSTD> {
5301   def : Pat<(v1f32 (opnodeS (v2f32 VPR64:$Rn))),
5302             (INSTS VPR64:$Rn)>;
5303   def : Pat<(v1f64 (opnodeD (v2f64 VPR128:$Rn))),
5304             (INSTD VPR128:$Rn)>;
5305 }
5306
5307 // Patterns to match llvm.aarch64.* intrinsic for
5308 // Scalar Reduce Add, Max, Min, MaxiNum, MinNum Pairwise (Floating Point)
5309 defm : Neon_ScalarPair_SD_size_patterns<int_aarch64_neon_vpfadd,
5310   int_aarch64_neon_vpfaddq, FADDPvv_S_2S, FADDPvv_D_2D>;
5311
5312 defm : Neon_ScalarPair_SD_size_patterns<int_aarch64_neon_vpmax,
5313   int_aarch64_neon_vpmaxq, FMAXPvv_S_2S, FMAXPvv_D_2D>;
5314
5315 defm : Neon_ScalarPair_SD_size_patterns<int_aarch64_neon_vpmin,
5316   int_aarch64_neon_vpminq, FMINPvv_S_2S, FMINPvv_D_2D>;
5317
5318 defm : Neon_ScalarPair_SD_size_patterns<int_aarch64_neon_vpfmaxnm,
5319   int_aarch64_neon_vpfmaxnmq, FMAXNMPvv_S_2S, FMAXNMPvv_D_2D>;
5320
5321 defm : Neon_ScalarPair_SD_size_patterns<int_aarch64_neon_vpfminnm,
5322   int_aarch64_neon_vpfminnmq, FMINNMPvv_S_2S, FMINNMPvv_D_2D>;
5323
5324 defm : Neon_ScalarPair_SD_size_patterns<int_aarch64_neon_vaddv,
5325     int_aarch64_neon_vaddv, FADDPvv_S_2S, FADDPvv_D_2D>;
5326
5327 def : Pat<(v1f32 (int_aarch64_neon_vaddv (v4f32 VPR128:$Rn))),
5328           (FADDPvv_S_2S (v2f32
5329                (EXTRACT_SUBREG
5330                    (v4f32 (FADDP_4S (v4f32 VPR128:$Rn), (v4f32 VPR128:$Rn))),
5331                    sub_64)))>;
5332
5333 defm : Neon_ScalarPair_SD_size_patterns<int_aarch64_neon_vmaxv,
5334     int_aarch64_neon_vmaxv, FMAXPvv_S_2S, FMAXPvv_D_2D>;
5335
5336 defm : Neon_ScalarPair_SD_size_patterns<int_aarch64_neon_vminv,
5337     int_aarch64_neon_vminv, FMINPvv_S_2S, FMINPvv_D_2D>;
5338
5339 defm : Neon_ScalarPair_SD_size_patterns<int_aarch64_neon_vmaxnmv,
5340     int_aarch64_neon_vmaxnmv, FMAXNMPvv_S_2S, FMAXNMPvv_D_2D>;
5341
5342 defm : Neon_ScalarPair_SD_size_patterns<int_aarch64_neon_vminnmv,
5343     int_aarch64_neon_vminnmv, FMINNMPvv_S_2S, FMINNMPvv_D_2D>;
5344
5345 // Scalar by element Arithmetic
5346
5347 class NeonI_ScalarXIndexedElemArith<string asmop, bits<4> opcode,
5348                                     string rmlane, bit u, bit szhi, bit szlo,
5349                                     RegisterClass ResFPR, RegisterClass OpFPR,
5350                                     RegisterOperand OpVPR, Operand OpImm>
5351   : NeonI_ScalarXIndexedElem<u, szhi, szlo, opcode,
5352                              (outs ResFPR:$Rd),
5353                              (ins OpFPR:$Rn, OpVPR:$MRm, OpImm:$Imm),
5354                              asmop # "\t$Rd, $Rn, $MRm" # rmlane # "[$Imm]",
5355                              [],
5356                              NoItinerary> {
5357   bits<3> Imm;
5358   bits<5> MRm;
5359 }
5360
5361 class NeonI_ScalarXIndexedElemArith_Constraint_Impl<string asmop, bits<4> opcode,
5362                                                     string rmlane,
5363                                                     bit u, bit szhi, bit szlo,
5364                                                     RegisterClass ResFPR,
5365                                                     RegisterClass OpFPR,
5366                                                     RegisterOperand OpVPR,
5367                                                     Operand OpImm>
5368   : NeonI_ScalarXIndexedElem<u, szhi, szlo, opcode,
5369                              (outs ResFPR:$Rd),
5370                              (ins ResFPR:$src, OpFPR:$Rn, OpVPR:$MRm, OpImm:$Imm),
5371                              asmop # "\t$Rd, $Rn, $MRm" # rmlane # "[$Imm]",
5372                              [],
5373                              NoItinerary> {
5374   let Constraints = "$src = $Rd";
5375   bits<3> Imm;
5376   bits<5> MRm;
5377 }
5378
5379 // Scalar Floating Point  multiply (scalar, by element)
5380 def FMULssv_4S : NeonI_ScalarXIndexedElemArith<"fmul",
5381   0b1001, ".s", 0b0, 0b1, 0b0, FPR32, FPR32, VPR128, neon_uimm2_bare> {
5382   let Inst{11} = Imm{1}; // h
5383   let Inst{21} = Imm{0}; // l
5384   let Inst{20-16} = MRm;
5385 }
5386 def FMULddv_2D : NeonI_ScalarXIndexedElemArith<"fmul",
5387   0b1001, ".d", 0b0, 0b1, 0b1, FPR64, FPR64, VPR128, neon_uimm1_bare> {
5388   let Inst{11} = Imm{0}; // h
5389   let Inst{21} = 0b0;    // l
5390   let Inst{20-16} = MRm;
5391 }
5392
5393 // Scalar Floating Point  multiply extended (scalar, by element)
5394 def FMULXssv_4S : NeonI_ScalarXIndexedElemArith<"fmulx",
5395   0b1001, ".s", 0b1, 0b1, 0b0, FPR32, FPR32, VPR128, neon_uimm2_bare> {
5396   let Inst{11} = Imm{1}; // h
5397   let Inst{21} = Imm{0}; // l
5398   let Inst{20-16} = MRm;
5399 }
5400 def FMULXddv_2D : NeonI_ScalarXIndexedElemArith<"fmulx",
5401   0b1001, ".d", 0b1, 0b1, 0b1, FPR64, FPR64, VPR128, neon_uimm1_bare> {
5402   let Inst{11} = Imm{0}; // h
5403   let Inst{21} = 0b0;    // l
5404   let Inst{20-16} = MRm;
5405 }
5406
5407 multiclass Neon_ScalarXIndexedElem_MUL_MULX_Patterns<
5408   SDPatternOperator opnode,
5409   Instruction INST,
5410   ValueType ResTy, RegisterClass FPRC, ValueType OpTy, Operand OpImm,
5411   ValueType OpNTy, ValueType ExTy, Operand OpNImm> {
5412
5413   def  : Pat<(ResTy (opnode (ResTy FPRC:$Rn),
5414                (ResTy (vector_extract (OpTy VPR128:$MRm), OpImm:$Imm)))),
5415              (ResTy (INST (ResTy FPRC:$Rn), (OpTy VPR128:$MRm), OpImm:$Imm))>;
5416
5417   def  : Pat<(ResTy (opnode (ResTy FPRC:$Rn),
5418                (ResTy (vector_extract (OpNTy VPR64:$MRm), OpNImm:$Imm)))),
5419              (ResTy (INST (ResTy FPRC:$Rn),
5420                (ExTy (SUBREG_TO_REG (i64 0), VPR64:$MRm, sub_64)),
5421                OpNImm:$Imm))>;
5422
5423   // swapped operands
5424   def  : Pat<(ResTy (opnode
5425                (ResTy (vector_extract (OpTy VPR128:$MRm), OpImm:$Imm)),
5426                (ResTy FPRC:$Rn))),
5427              (ResTy (INST (ResTy FPRC:$Rn), (OpTy VPR128:$MRm), OpImm:$Imm))>;
5428
5429   def  : Pat<(ResTy (opnode
5430                (ResTy (vector_extract (OpNTy VPR64:$MRm), OpNImm:$Imm)),
5431                (ResTy FPRC:$Rn))),
5432              (ResTy (INST (ResTy FPRC:$Rn),
5433                (ExTy (SUBREG_TO_REG (i64 0), VPR64:$MRm, sub_64)),
5434                OpNImm:$Imm))>;
5435 }
5436
5437 // Patterns for Scalar Floating Point  multiply (scalar, by element)
5438 defm : Neon_ScalarXIndexedElem_MUL_MULX_Patterns<fmul, FMULssv_4S,
5439   f32, FPR32, v4f32, neon_uimm2_bare, v2f32, v4f32, neon_uimm1_bare>;
5440 defm : Neon_ScalarXIndexedElem_MUL_MULX_Patterns<fmul, FMULddv_2D,
5441   f64, FPR64, v2f64, neon_uimm1_bare, v1f64, v2f64, neon_uimm0_bare>;
5442
5443 // Patterns for Scalar Floating Point  multiply extended (scalar, by element)
5444 defm : Neon_ScalarXIndexedElem_MUL_MULX_Patterns<int_aarch64_neon_vmulx,
5445   FMULXssv_4S, f32, FPR32, v4f32, neon_uimm2_bare,
5446   v2f32, v4f32, neon_uimm1_bare>;
5447 defm : Neon_ScalarXIndexedElem_MUL_MULX_Patterns<int_aarch64_neon_vmulx,
5448   FMULXddv_2D, f64, FPR64, v2f64, neon_uimm1_bare,
5449   v1f64, v2f64, neon_uimm0_bare>;
5450
5451
5452 // Scalar Floating Point fused multiply-add (scalar, by element)
5453 def FMLAssv_4S : NeonI_ScalarXIndexedElemArith_Constraint_Impl<"fmla",
5454   0b0001, ".s", 0b0, 0b1, 0b0, FPR32, FPR32, VPR128, neon_uimm2_bare> {
5455   let Inst{11} = Imm{1}; // h
5456   let Inst{21} = Imm{0}; // l
5457   let Inst{20-16} = MRm;
5458 }
5459 def FMLAddv_2D : NeonI_ScalarXIndexedElemArith_Constraint_Impl<"fmla",
5460   0b0001, ".d", 0b0, 0b1, 0b1, FPR64, FPR64, VPR128, neon_uimm1_bare> {
5461   let Inst{11} = Imm{0}; // h
5462   let Inst{21} = 0b0;    // l
5463   let Inst{20-16} = MRm;
5464 }
5465
5466 // Scalar Floating Point fused multiply-subtract (scalar, by element)
5467 def FMLSssv_4S : NeonI_ScalarXIndexedElemArith_Constraint_Impl<"fmls",
5468   0b0101, ".s", 0b0, 0b1, 0b0, FPR32, FPR32, VPR128, neon_uimm2_bare> {
5469   let Inst{11} = Imm{1}; // h
5470   let Inst{21} = Imm{0}; // l
5471   let Inst{20-16} = MRm;
5472 }
5473 def FMLSddv_2D : NeonI_ScalarXIndexedElemArith_Constraint_Impl<"fmls",
5474   0b0101, ".d", 0b0, 0b1, 0b1, FPR64, FPR64, VPR128, neon_uimm1_bare> {
5475   let Inst{11} = Imm{0}; // h
5476   let Inst{21} = 0b0;    // l
5477   let Inst{20-16} = MRm;
5478 }
5479 // We are allowed to match the fma instruction regardless of compile options.
5480 multiclass Neon_ScalarXIndexedElem_FMA_Patterns<
5481   Instruction FMLAI, Instruction FMLSI,
5482   ValueType ResTy, RegisterClass FPRC, ValueType OpTy, Operand OpImm,
5483   ValueType OpNTy, ValueType ExTy, Operand OpNImm> {
5484   // fmla
5485   def  : Pat<(ResTy (fma (ResTy FPRC:$Rn),
5486                (ResTy (vector_extract (OpTy VPR128:$MRm), OpImm:$Imm)),
5487                (ResTy FPRC:$Ra))),
5488              (ResTy (FMLAI (ResTy FPRC:$Ra),
5489                (ResTy FPRC:$Rn), (OpTy VPR128:$MRm), OpImm:$Imm))>;
5490
5491   def  : Pat<(ResTy (fma (ResTy FPRC:$Rn),
5492                (ResTy (vector_extract (OpNTy VPR64:$MRm), OpNImm:$Imm)),
5493                (ResTy FPRC:$Ra))),
5494              (ResTy (FMLAI (ResTy FPRC:$Ra),
5495                (ResTy FPRC:$Rn),
5496                (ExTy (SUBREG_TO_REG (i64 0), VPR64:$MRm, sub_64)),
5497                OpNImm:$Imm))>;
5498
5499   // swapped fmla operands
5500   def  : Pat<(ResTy (fma
5501                (ResTy (vector_extract (OpTy VPR128:$MRm), OpImm:$Imm)),
5502                (ResTy FPRC:$Rn),
5503                (ResTy FPRC:$Ra))),
5504              (ResTy (FMLAI (ResTy FPRC:$Ra),
5505                (ResTy FPRC:$Rn), (OpTy VPR128:$MRm), OpImm:$Imm))>;
5506
5507   def  : Pat<(ResTy (fma
5508                (ResTy (vector_extract (OpNTy VPR64:$MRm), OpNImm:$Imm)),
5509                (ResTy FPRC:$Rn),
5510                (ResTy FPRC:$Ra))),
5511              (ResTy (FMLAI (ResTy FPRC:$Ra),
5512                (ResTy FPRC:$Rn),
5513                (ExTy (SUBREG_TO_REG (i64 0), VPR64:$MRm, sub_64)),
5514                OpNImm:$Imm))>;
5515
5516   // fmls
5517   def  : Pat<(ResTy (fma (ResTy FPRC:$Rn),
5518                (fneg (ResTy (vector_extract (OpTy VPR128:$MRm), OpImm:$Imm))),
5519                (ResTy FPRC:$Ra))),
5520              (ResTy (FMLSI (ResTy FPRC:$Ra),
5521                (ResTy FPRC:$Rn), (OpTy VPR128:$MRm), OpImm:$Imm))>;
5522
5523   def  : Pat<(ResTy (fma (ResTy FPRC:$Rn),
5524                (fneg (ResTy (vector_extract (OpNTy VPR64:$MRm), OpNImm:$Imm))),
5525                (ResTy FPRC:$Ra))),
5526              (ResTy (FMLSI (ResTy FPRC:$Ra),
5527                (ResTy FPRC:$Rn),
5528                (ExTy (SUBREG_TO_REG (i64 0), VPR64:$MRm, sub_64)),
5529                OpNImm:$Imm))>;
5530
5531   // swapped fmls operands
5532   def  : Pat<(ResTy (fma
5533                (fneg (ResTy (vector_extract (OpTy VPR128:$MRm), OpImm:$Imm))),
5534                (ResTy FPRC:$Rn),
5535                (ResTy FPRC:$Ra))),
5536              (ResTy (FMLSI (ResTy FPRC:$Ra),
5537                (ResTy FPRC:$Rn), (OpTy VPR128:$MRm), OpImm:$Imm))>;
5538
5539   def  : Pat<(ResTy (fma
5540                (fneg (ResTy (vector_extract (OpNTy VPR64:$MRm), OpNImm:$Imm))),
5541                (ResTy FPRC:$Rn),
5542                (ResTy FPRC:$Ra))),
5543              (ResTy (FMLSI (ResTy FPRC:$Ra),
5544                (ResTy FPRC:$Rn),
5545                (ExTy (SUBREG_TO_REG (i64 0), VPR64:$MRm, sub_64)),
5546                OpNImm:$Imm))>;
5547 }
5548
5549 // Scalar Floating Point fused multiply-add and
5550 // multiply-subtract (scalar, by element)
5551 defm : Neon_ScalarXIndexedElem_FMA_Patterns<FMLAssv_4S, FMLSssv_4S,
5552   f32, FPR32, v4f32, neon_uimm2_bare, v2f32, v4f32, neon_uimm1_bare>;
5553 defm : Neon_ScalarXIndexedElem_FMA_Patterns<FMLAddv_2D, FMLSddv_2D,
5554   f64, FPR64, v2f64, neon_uimm1_bare, v1f64, v2f64, neon_uimm0_bare>;
5555 defm : Neon_ScalarXIndexedElem_FMA_Patterns<FMLAddv_2D, FMLSddv_2D,
5556   f64, FPR64, v2f64, neon_uimm1_bare, v1f64, v2f64, neon_uimm0_bare>;
5557
5558 // Scalar Signed saturating doubling multiply long (scalar, by element)
5559 def SQDMULLshv_4H : NeonI_ScalarXIndexedElemArith<"sqdmull",
5560   0b1011, ".h", 0b0, 0b0, 0b1, FPR32, FPR16, VPR64Lo, neon_uimm2_bare> {
5561   let Inst{11} = 0b0; // h
5562   let Inst{21} = Imm{1}; // l
5563   let Inst{20} = Imm{0}; // m
5564   let Inst{19-16} = MRm{3-0};
5565 }
5566 def SQDMULLshv_8H : NeonI_ScalarXIndexedElemArith<"sqdmull",
5567   0b1011, ".h", 0b0, 0b0, 0b1, FPR32, FPR16, VPR128Lo, neon_uimm3_bare> {
5568   let Inst{11} = Imm{2}; // h
5569   let Inst{21} = Imm{1}; // l
5570   let Inst{20} = Imm{0}; // m
5571   let Inst{19-16} = MRm{3-0};
5572 }
5573 def SQDMULLdsv_2S : NeonI_ScalarXIndexedElemArith<"sqdmull",
5574   0b1011, ".s", 0b0, 0b1, 0b0, FPR64, FPR32, VPR64, neon_uimm1_bare> {
5575   let Inst{11} = 0b0;    // h
5576   let Inst{21} = Imm{0}; // l
5577   let Inst{20-16} = MRm;
5578 }
5579 def SQDMULLdsv_4S : NeonI_ScalarXIndexedElemArith<"sqdmull",
5580   0b1011, ".s", 0b0, 0b1, 0b0, FPR64, FPR32, VPR128, neon_uimm2_bare> {
5581   let Inst{11} = Imm{1};    // h
5582   let Inst{21} = Imm{0};    // l
5583   let Inst{20-16} = MRm;
5584 }
5585
5586 multiclass Neon_ScalarXIndexedElem_MUL_Patterns<
5587   SDPatternOperator opnode,
5588   Instruction INST,
5589   ValueType ResTy, RegisterClass FPRC,
5590   ValueType OpVTy, ValueType OpTy,
5591   ValueType VecOpTy, ValueType ExTy, RegisterOperand VPRC, Operand OpImm> {
5592
5593   def  : Pat<(ResTy (opnode (OpVTy FPRC:$Rn),
5594                (OpVTy (scalar_to_vector
5595                  (ExTy (vector_extract (VecOpTy VPRC:$MRm), OpImm:$Imm)))))),
5596              (ResTy (INST (OpVTy FPRC:$Rn), (VecOpTy VPRC:$MRm), OpImm:$Imm))>;
5597
5598   //swapped operands
5599   def  : Pat<(ResTy (opnode
5600                (OpVTy (scalar_to_vector
5601                  (ExTy (vector_extract (VecOpTy VPRC:$MRm), OpImm:$Imm)))),
5602                  (OpVTy FPRC:$Rn))),
5603              (ResTy (INST (OpVTy FPRC:$Rn), (VecOpTy VPRC:$MRm), OpImm:$Imm))>;
5604 }
5605
5606
5607 // Patterns for Scalar Signed saturating doubling
5608 // multiply long (scalar, by element)
5609 defm : Neon_ScalarXIndexedElem_MUL_Patterns<int_arm_neon_vqdmull,
5610   SQDMULLshv_4H, v1i32, FPR16, v1i16, i16, v4i16,
5611   i32, VPR64Lo, neon_uimm2_bare>;
5612 defm : Neon_ScalarXIndexedElem_MUL_Patterns<int_arm_neon_vqdmull,
5613   SQDMULLshv_8H, v1i32, FPR16, v1i16, i16, v8i16,
5614   i32, VPR128Lo, neon_uimm3_bare>;
5615 defm : Neon_ScalarXIndexedElem_MUL_Patterns<int_arm_neon_vqdmull,
5616   SQDMULLdsv_2S, v1i64, FPR32, v1i32, i32, v2i32,
5617   i32, VPR64Lo, neon_uimm1_bare>;
5618 defm : Neon_ScalarXIndexedElem_MUL_Patterns<int_arm_neon_vqdmull,
5619   SQDMULLdsv_4S, v1i64, FPR32, v1i32, i32, v4i32,
5620   i32, VPR128Lo, neon_uimm2_bare>;
5621
5622 // Scalar Signed saturating doubling multiply-add long (scalar, by element)
5623 def SQDMLALshv_4H : NeonI_ScalarXIndexedElemArith_Constraint_Impl<"sqdmlal",
5624   0b0011, ".h", 0b0, 0b0, 0b1, FPR32, FPR16, VPR64Lo, neon_uimm2_bare> {
5625   let Inst{11} = 0b0; // h
5626   let Inst{21} = Imm{1}; // l
5627   let Inst{20} = Imm{0}; // m
5628   let Inst{19-16} = MRm{3-0};
5629 }
5630 def SQDMLALshv_8H : NeonI_ScalarXIndexedElemArith_Constraint_Impl<"sqdmlal",
5631   0b0011, ".h", 0b0, 0b0, 0b1, FPR32, FPR16, VPR128Lo, neon_uimm3_bare> {
5632   let Inst{11} = Imm{2}; // h
5633   let Inst{21} = Imm{1}; // l
5634   let Inst{20} = Imm{0}; // m
5635   let Inst{19-16} = MRm{3-0};
5636 }
5637 def SQDMLALdsv_2S : NeonI_ScalarXIndexedElemArith_Constraint_Impl<"sqdmlal",
5638   0b0011, ".s", 0b0, 0b1, 0b0, FPR64, FPR32, VPR64, neon_uimm1_bare> {
5639   let Inst{11} = 0b0;    // h
5640   let Inst{21} = Imm{0}; // l
5641   let Inst{20-16} = MRm;
5642 }
5643 def SQDMLALdsv_4S : NeonI_ScalarXIndexedElemArith_Constraint_Impl<"sqdmlal",
5644   0b0011, ".s", 0b0, 0b1, 0b0, FPR64, FPR32, VPR128, neon_uimm2_bare> {
5645   let Inst{11} = Imm{1};    // h
5646   let Inst{21} = Imm{0};    // l
5647   let Inst{20-16} = MRm;
5648 }
5649
5650 // Scalar Signed saturating doubling
5651 // multiply-subtract long (scalar, by element)
5652 def SQDMLSLshv_4H : NeonI_ScalarXIndexedElemArith_Constraint_Impl<"sqdmlsl",
5653   0b0111, ".h", 0b0, 0b0, 0b1, FPR32, FPR16, VPR64Lo, neon_uimm2_bare> {
5654   let Inst{11} = 0b0; // h
5655   let Inst{21} = Imm{1}; // l
5656   let Inst{20} = Imm{0}; // m
5657   let Inst{19-16} = MRm{3-0};
5658 }
5659 def SQDMLSLshv_8H : NeonI_ScalarXIndexedElemArith_Constraint_Impl<"sqdmlsl",
5660   0b0111, ".h", 0b0, 0b0, 0b1, FPR32, FPR16, VPR128Lo, neon_uimm3_bare> {
5661   let Inst{11} = Imm{2}; // h
5662   let Inst{21} = Imm{1}; // l
5663   let Inst{20} = Imm{0}; // m
5664   let Inst{19-16} = MRm{3-0};
5665 }
5666 def SQDMLSLdsv_2S : NeonI_ScalarXIndexedElemArith_Constraint_Impl<"sqdmlsl",
5667   0b0111, ".s", 0b0, 0b1, 0b0, FPR64, FPR32, VPR64, neon_uimm1_bare> {
5668   let Inst{11} = 0b0;    // h
5669   let Inst{21} = Imm{0}; // l
5670   let Inst{20-16} = MRm;
5671 }
5672 def SQDMLSLdsv_4S : NeonI_ScalarXIndexedElemArith_Constraint_Impl<"sqdmlsl",
5673   0b0111, ".s", 0b0, 0b1, 0b0, FPR64, FPR32, VPR128, neon_uimm2_bare> {
5674   let Inst{11} = Imm{1};    // h
5675   let Inst{21} = Imm{0};    // l
5676   let Inst{20-16} = MRm;
5677 }
5678
5679 multiclass Neon_ScalarXIndexedElem_MLAL_Patterns<
5680   SDPatternOperator opnode,
5681   SDPatternOperator coreopnode,
5682   Instruction INST,
5683   ValueType ResTy, RegisterClass ResFPRC, RegisterClass FPRC,
5684   ValueType OpTy,
5685   ValueType OpVTy, ValueType ExTy, RegisterOperand VPRC, Operand OpImm> {
5686
5687   def  : Pat<(ResTy (opnode
5688                (ResTy ResFPRC:$Ra),
5689                (ResTy (coreopnode (OpTy FPRC:$Rn),
5690                  (OpTy (scalar_to_vector
5691                    (ExTy (vector_extract (OpVTy VPRC:$MRm), OpImm:$Imm)))))))),
5692              (ResTy (INST (ResTy ResFPRC:$Ra),
5693                (OpTy FPRC:$Rn), (OpVTy VPRC:$MRm), OpImm:$Imm))>;
5694
5695   // swapped operands
5696   def  : Pat<(ResTy (opnode
5697                (ResTy ResFPRC:$Ra),
5698                (ResTy (coreopnode
5699                  (OpTy (scalar_to_vector
5700                    (ExTy (vector_extract (OpVTy VPRC:$MRm), OpImm:$Imm)))),
5701                  (OpTy FPRC:$Rn))))),
5702              (ResTy (INST (ResTy ResFPRC:$Ra),
5703                (OpTy FPRC:$Rn), (OpVTy VPRC:$MRm), OpImm:$Imm))>;
5704 }
5705
5706 // Patterns for Scalar Signed saturating
5707 // doubling multiply-add long (scalar, by element)
5708 defm : Neon_ScalarXIndexedElem_MLAL_Patterns<int_arm_neon_vqadds,
5709   int_arm_neon_vqdmull, SQDMLALshv_4H, v1i32, FPR32, FPR16, v1i16, v4i16,
5710   i32, VPR64Lo, neon_uimm2_bare>;
5711 defm : Neon_ScalarXIndexedElem_MLAL_Patterns<int_arm_neon_vqadds,
5712   int_arm_neon_vqdmull, SQDMLALshv_8H, v1i32, FPR32, FPR16, v1i16, v8i16,
5713   i32, VPR128Lo, neon_uimm3_bare>;
5714 defm : Neon_ScalarXIndexedElem_MLAL_Patterns<int_arm_neon_vqadds,
5715   int_arm_neon_vqdmull, SQDMLALdsv_2S, v1i64, FPR64, FPR32, v1i32, v2i32,
5716   i32, VPR64Lo, neon_uimm1_bare>;
5717 defm : Neon_ScalarXIndexedElem_MLAL_Patterns<int_arm_neon_vqadds,
5718   int_arm_neon_vqdmull, SQDMLALdsv_4S, v1i64, FPR64, FPR32, v1i32, v4i32,
5719   i32, VPR128Lo, neon_uimm2_bare>;
5720
5721 // Patterns for Scalar Signed saturating
5722 // doubling multiply-sub long (scalar, by element)
5723 defm : Neon_ScalarXIndexedElem_MLAL_Patterns<int_arm_neon_vqsubs,
5724   int_arm_neon_vqdmull, SQDMLSLshv_4H, v1i32, FPR32, FPR16, v1i16, v4i16,
5725   i32, VPR64Lo, neon_uimm2_bare>;
5726 defm : Neon_ScalarXIndexedElem_MLAL_Patterns<int_arm_neon_vqsubs,
5727   int_arm_neon_vqdmull, SQDMLSLshv_8H, v1i32, FPR32, FPR16, v1i16, v8i16,
5728   i32, VPR128Lo, neon_uimm3_bare>;
5729 defm : Neon_ScalarXIndexedElem_MLAL_Patterns<int_arm_neon_vqsubs,
5730   int_arm_neon_vqdmull, SQDMLSLdsv_2S, v1i64, FPR64, FPR32, v1i32, v2i32,
5731   i32, VPR64Lo, neon_uimm1_bare>;
5732 defm : Neon_ScalarXIndexedElem_MLAL_Patterns<int_arm_neon_vqsubs,
5733   int_arm_neon_vqdmull, SQDMLSLdsv_4S, v1i64, FPR64, FPR32, v1i32, v4i32,
5734   i32, VPR128Lo, neon_uimm2_bare>;
5735
5736 // Scalar general arithmetic operation
5737 class Neon_Scalar_GeneralMath2D_pattern<SDPatternOperator opnode,
5738                                         Instruction INST>
5739     : Pat<(v1f64 (opnode (v1f64 FPR64:$Rn))), (INST FPR64:$Rn)>;
5740
5741 class Neon_Scalar_GeneralMath3D_pattern<SDPatternOperator opnode,
5742                                         Instruction INST>
5743     : Pat<(v1f64 (opnode (v1f64 FPR64:$Rn), (v1f64 FPR64:$Rm))),
5744           (INST FPR64:$Rn, FPR64:$Rm)>;
5745
5746 class Neon_Scalar_GeneralMath4D_pattern<SDPatternOperator opnode,
5747                                         Instruction INST>
5748     : Pat<(v1f64 (opnode (v1f64 FPR64:$Rn), (v1f64 FPR64:$Rm),
5749               (v1f64 FPR64:$Ra))),
5750           (INST FPR64:$Rn, FPR64:$Rm, FPR64:$Ra)>;
5751
5752 def : Neon_Scalar_GeneralMath3D_pattern<fadd, FADDddd>;
5753 def : Neon_Scalar_GeneralMath3D_pattern<fmul, FMULddd>;
5754 def : Neon_Scalar_GeneralMath3D_pattern<fsub, FSUBddd>;
5755 def : Neon_Scalar_GeneralMath3D_pattern<fdiv, FDIVddd>;
5756 def : Neon_Scalar_GeneralMath3D_pattern<int_arm_neon_vabds, FABDddd>;
5757 def : Neon_Scalar_GeneralMath3D_pattern<int_arm_neon_vmaxs, FMAXddd>;
5758 def : Neon_Scalar_GeneralMath3D_pattern<int_arm_neon_vmins, FMINddd>;
5759 def : Neon_Scalar_GeneralMath3D_pattern<int_aarch64_neon_vmaxnm, FMAXNMddd>;
5760 def : Neon_Scalar_GeneralMath3D_pattern<int_aarch64_neon_vminnm, FMINNMddd>;
5761
5762 def : Neon_Scalar_GeneralMath2D_pattern<fabs, FABSdd>;
5763 def : Neon_Scalar_GeneralMath2D_pattern<fneg, FNEGdd>;
5764
5765 def : Neon_Scalar_GeneralMath4D_pattern<fma, FMADDdddd>;
5766 def : Neon_Scalar_GeneralMath4D_pattern<fmsub, FMSUBdddd>;
5767
5768 // Scalar Signed saturating doubling multiply returning
5769 // high half (scalar, by element)
5770 def SQDMULHhhv_4H : NeonI_ScalarXIndexedElemArith<"sqdmulh",
5771   0b1100, ".h", 0b0, 0b0, 0b1, FPR16, FPR16, VPR64Lo, neon_uimm2_bare> {
5772   let Inst{11} = 0b0; // h
5773   let Inst{21} = Imm{1}; // l
5774   let Inst{20} = Imm{0}; // m
5775   let Inst{19-16} = MRm{3-0};
5776 }
5777 def SQDMULHhhv_8H : NeonI_ScalarXIndexedElemArith<"sqdmulh",
5778   0b1100, ".h", 0b0, 0b0, 0b1, FPR16, FPR16, VPR128Lo, neon_uimm3_bare> {
5779   let Inst{11} = Imm{2}; // h
5780   let Inst{21} = Imm{1}; // l
5781   let Inst{20} = Imm{0}; // m
5782   let Inst{19-16} = MRm{3-0};
5783 }
5784 def SQDMULHssv_2S : NeonI_ScalarXIndexedElemArith<"sqdmulh",
5785   0b1100, ".s", 0b0, 0b1, 0b0, FPR32, FPR32, VPR64, neon_uimm1_bare> {
5786   let Inst{11} = 0b0;    // h
5787   let Inst{21} = Imm{0}; // l
5788   let Inst{20-16} = MRm;
5789 }
5790 def SQDMULHssv_4S : NeonI_ScalarXIndexedElemArith<"sqdmulh",
5791   0b1100, ".s", 0b0, 0b1, 0b0, FPR32, FPR32, VPR128, neon_uimm2_bare> {
5792   let Inst{11} = Imm{1};    // h
5793   let Inst{21} = Imm{0};    // l
5794   let Inst{20-16} = MRm;
5795 }
5796
5797 // Patterns for Scalar Signed saturating doubling multiply returning
5798 // high half (scalar, by element)
5799 defm : Neon_ScalarXIndexedElem_MUL_Patterns<int_arm_neon_vqdmulh,
5800   SQDMULHhhv_4H, v1i16, FPR16, v1i16, i16, v4i16,
5801   i32, VPR64Lo, neon_uimm2_bare>;
5802 defm : Neon_ScalarXIndexedElem_MUL_Patterns<int_arm_neon_vqdmulh,
5803   SQDMULHhhv_8H, v1i16, FPR16, v1i16, i16, v8i16,
5804   i32, VPR128Lo, neon_uimm3_bare>;
5805 defm : Neon_ScalarXIndexedElem_MUL_Patterns<int_arm_neon_vqdmulh,
5806   SQDMULHssv_2S, v1i32, FPR32, v1i32, i32, v2i32,
5807   i32, VPR64Lo, neon_uimm1_bare>;
5808 defm : Neon_ScalarXIndexedElem_MUL_Patterns<int_arm_neon_vqdmulh,
5809   SQDMULHssv_4S, v1i32, FPR32, v1i32, i32, v4i32,
5810   i32, VPR128Lo, neon_uimm2_bare>;
5811
5812 // Scalar Signed saturating rounding doubling multiply
5813 // returning high half (scalar, by element)
5814 def SQRDMULHhhv_4H : NeonI_ScalarXIndexedElemArith<"sqrdmulh",
5815   0b1101, ".h", 0b0, 0b0, 0b1, FPR16, FPR16, VPR64Lo, neon_uimm2_bare> {
5816   let Inst{11} = 0b0; // h
5817   let Inst{21} = Imm{1}; // l
5818   let Inst{20} = Imm{0}; // m
5819   let Inst{19-16} = MRm{3-0};
5820 }
5821 def SQRDMULHhhv_8H : NeonI_ScalarXIndexedElemArith<"sqrdmulh",
5822   0b1101, ".h", 0b0, 0b0, 0b1, FPR16, FPR16, VPR128Lo, neon_uimm3_bare> {
5823   let Inst{11} = Imm{2}; // h
5824   let Inst{21} = Imm{1}; // l
5825   let Inst{20} = Imm{0}; // m
5826   let Inst{19-16} = MRm{3-0};
5827 }
5828 def SQRDMULHssv_2S : NeonI_ScalarXIndexedElemArith<"sqrdmulh",
5829   0b1101, ".s", 0b0, 0b1, 0b0, FPR32, FPR32, VPR64, neon_uimm1_bare> {
5830   let Inst{11} = 0b0;    // h
5831   let Inst{21} = Imm{0}; // l
5832   let Inst{20-16} = MRm;
5833 }
5834 def SQRDMULHssv_4S : NeonI_ScalarXIndexedElemArith<"sqrdmulh",
5835   0b1101, ".s", 0b0, 0b1, 0b0, FPR32, FPR32, VPR128, neon_uimm2_bare> {
5836   let Inst{11} = Imm{1};    // h
5837   let Inst{21} = Imm{0};    // l
5838   let Inst{20-16} = MRm;
5839 }
5840
5841 defm : Neon_ScalarXIndexedElem_MUL_Patterns<int_arm_neon_vqrdmulh,
5842   SQRDMULHhhv_4H, v1i16, FPR16, v1i16, i16, v4i16, i32,
5843   VPR64Lo, neon_uimm2_bare>;
5844 defm : Neon_ScalarXIndexedElem_MUL_Patterns<int_arm_neon_vqrdmulh,
5845   SQRDMULHhhv_8H, v1i16, FPR16, v1i16, i16, v8i16, i32,
5846   VPR128Lo, neon_uimm3_bare>;
5847 defm : Neon_ScalarXIndexedElem_MUL_Patterns<int_arm_neon_vqrdmulh,
5848   SQRDMULHssv_2S, v1i32, FPR32, v1i32, i32, v2i32, i32,
5849   VPR64Lo, neon_uimm1_bare>;
5850 defm : Neon_ScalarXIndexedElem_MUL_Patterns<int_arm_neon_vqrdmulh,
5851   SQRDMULHssv_4S, v1i32, FPR32, v1i32, i32, v4i32, i32,
5852   VPR128Lo, neon_uimm2_bare>;
5853
5854 // Scalar Copy - DUP element to scalar
5855 class NeonI_Scalar_DUP<string asmop, string asmlane,
5856                        RegisterClass ResRC, RegisterOperand VPRC,
5857                        Operand OpImm>
5858   : NeonI_ScalarCopy<(outs ResRC:$Rd), (ins VPRC:$Rn, OpImm:$Imm),
5859                      asmop # "\t$Rd, $Rn." # asmlane # "[$Imm]",
5860                      [],
5861                      NoItinerary> {
5862   bits<4> Imm;
5863 }
5864
5865 def DUPbv_B : NeonI_Scalar_DUP<"dup", "b", FPR8, VPR128, neon_uimm4_bare> {
5866   let Inst{20-16} = {Imm{3}, Imm{2}, Imm{1}, Imm{0}, 0b1};
5867 }
5868 def DUPhv_H : NeonI_Scalar_DUP<"dup", "h", FPR16, VPR128, neon_uimm3_bare> {
5869   let Inst{20-16} = {Imm{2}, Imm{1}, Imm{0}, 0b1, 0b0};
5870 }
5871 def DUPsv_S : NeonI_Scalar_DUP<"dup", "s", FPR32, VPR128, neon_uimm2_bare> {
5872   let Inst{20-16} = {Imm{1}, Imm{0}, 0b1, 0b0, 0b0};
5873 }
5874 def DUPdv_D : NeonI_Scalar_DUP<"dup", "d", FPR64, VPR128, neon_uimm1_bare> {
5875   let Inst{20-16} = {Imm, 0b1, 0b0, 0b0, 0b0};
5876 }
5877
5878 multiclass NeonI_Scalar_DUP_Elt_pattern<Instruction DUPI, ValueType ResTy,
5879   ValueType OpTy, Operand OpImm,
5880   ValueType OpNTy, ValueType ExTy, Operand OpNImm> {
5881   def : Pat<(ResTy (vector_extract (OpTy VPR128:$Rn), OpImm:$Imm)),
5882             (ResTy (DUPI (OpTy VPR128:$Rn), OpImm:$Imm))>;
5883
5884   def : Pat<(ResTy (vector_extract (OpNTy VPR64:$Rn), OpNImm:$Imm)),
5885             (ResTy (DUPI
5886               (ExTy (SUBREG_TO_REG (i64 0), VPR64:$Rn, sub_64)),
5887                 OpNImm:$Imm))>;
5888 }
5889
5890 // Patterns for vector extract of FP data using scalar DUP instructions
5891 defm : NeonI_Scalar_DUP_Elt_pattern<DUPsv_S, f32,
5892   v4f32, neon_uimm2_bare, v2f32, v4f32, neon_uimm1_bare>;
5893 defm : NeonI_Scalar_DUP_Elt_pattern<DUPdv_D, f64,
5894   v2f64, neon_uimm1_bare, v1f64, v2f64, neon_uimm0_bare>;
5895
5896 multiclass NeonI_Scalar_DUP_Ext_Vec_pattern<Instruction DUPI,
5897   ValueType ResTy, ValueType OpTy,Operand OpLImm,
5898   ValueType NOpTy, ValueType ExTy, Operand OpNImm> {
5899
5900   def : Pat<(ResTy (extract_subvector (OpTy VPR128:$Rn), OpLImm:$Imm)),
5901             (ResTy (DUPI VPR128:$Rn, OpLImm:$Imm))>;
5902
5903   def : Pat<(ResTy (extract_subvector (NOpTy VPR64:$Rn), OpNImm:$Imm)),
5904             (ResTy (DUPI
5905               (ExTy (SUBREG_TO_REG (i64 0), VPR64:$Rn, sub_64)),
5906                 OpNImm:$Imm))>;
5907 }
5908
5909 // Patterns for extract subvectors of v1ix data using scalar DUP instructions.
5910 defm : NeonI_Scalar_DUP_Ext_Vec_pattern<DUPbv_B, v1i8, v16i8, neon_uimm4_bare,
5911                                         v8i8, v16i8, neon_uimm3_bare>;
5912 defm : NeonI_Scalar_DUP_Ext_Vec_pattern<DUPhv_H, v1i16, v8i16, neon_uimm3_bare,
5913                                         v4i16, v8i16, neon_uimm2_bare>;
5914 defm : NeonI_Scalar_DUP_Ext_Vec_pattern<DUPsv_S, v1i32, v4i32, neon_uimm2_bare,
5915                                         v2i32, v4i32, neon_uimm1_bare>;
5916
5917 multiclass NeonI_Scalar_DUP_Copy_pattern1<Instruction DUPI, ValueType ResTy,
5918                                           ValueType OpTy, ValueType ElemTy,
5919                                           Operand OpImm, ValueType OpNTy,
5920                                           ValueType ExTy, Operand OpNImm> {
5921
5922   def : Pat<(ResTy (vector_insert (ResTy undef),
5923               (ElemTy (vector_extract (OpTy VPR128:$Rn), OpImm:$Imm)),
5924               (neon_uimm0_bare:$Imm))),
5925             (ResTy (DUPI (OpTy VPR128:$Rn), OpImm:$Imm))>;
5926
5927   def : Pat<(ResTy (vector_insert (ResTy undef),
5928               (ElemTy (vector_extract (OpNTy VPR64:$Rn), OpNImm:$Imm)),
5929               (OpNImm:$Imm))),
5930             (ResTy (DUPI
5931               (ExTy (SUBREG_TO_REG (i64 0), VPR64:$Rn, sub_64)),
5932               OpNImm:$Imm))>;
5933 }
5934
5935 multiclass NeonI_Scalar_DUP_Copy_pattern2<Instruction DUPI, ValueType ResTy,
5936                                           ValueType OpTy, ValueType ElemTy,
5937                                           Operand OpImm, ValueType OpNTy,
5938                                           ValueType ExTy, Operand OpNImm> {
5939
5940   def : Pat<(ResTy (scalar_to_vector
5941               (ElemTy (vector_extract (OpTy VPR128:$Rn), OpImm:$Imm)))),
5942             (ResTy (DUPI (OpTy VPR128:$Rn), OpImm:$Imm))>;
5943
5944   def : Pat<(ResTy (scalar_to_vector
5945               (ElemTy (vector_extract (OpNTy VPR64:$Rn), OpNImm:$Imm)))),
5946             (ResTy (DUPI
5947               (ExTy (SUBREG_TO_REG (i64 0), VPR64:$Rn, sub_64)),
5948               OpNImm:$Imm))>;
5949 }
5950
5951 // Patterns for vector copy to v1ix and v1fx vectors using scalar DUP
5952 // instructions.
5953 defm : NeonI_Scalar_DUP_Copy_pattern1<DUPdv_D,
5954   v1i64, v2i64, i64, neon_uimm1_bare,
5955   v1i64, v2i64, neon_uimm0_bare>;
5956 defm : NeonI_Scalar_DUP_Copy_pattern1<DUPsv_S,
5957   v1i32, v4i32, i32, neon_uimm2_bare,
5958   v2i32, v4i32, neon_uimm1_bare>;
5959 defm : NeonI_Scalar_DUP_Copy_pattern1<DUPhv_H,
5960   v1i16, v8i16, i32, neon_uimm3_bare,
5961   v4i16, v8i16, neon_uimm2_bare>;
5962 defm : NeonI_Scalar_DUP_Copy_pattern1<DUPbv_B,
5963   v1i8, v16i8, i32, neon_uimm4_bare,
5964   v8i8, v16i8, neon_uimm3_bare>;
5965 defm : NeonI_Scalar_DUP_Copy_pattern1<DUPdv_D,
5966   v1f64, v2f64, f64, neon_uimm1_bare,
5967   v1f64, v2f64, neon_uimm0_bare>;
5968 defm : NeonI_Scalar_DUP_Copy_pattern1<DUPsv_S,
5969   v1f32, v4f32, f32, neon_uimm2_bare,
5970   v2f32, v4f32, neon_uimm1_bare>;
5971 defm : NeonI_Scalar_DUP_Copy_pattern2<DUPdv_D,
5972   v1i64, v2i64, i64, neon_uimm1_bare,
5973   v1i64, v2i64, neon_uimm0_bare>;
5974 defm : NeonI_Scalar_DUP_Copy_pattern2<DUPsv_S,
5975   v1i32, v4i32, i32, neon_uimm2_bare,
5976   v2i32, v4i32, neon_uimm1_bare>;
5977 defm : NeonI_Scalar_DUP_Copy_pattern2<DUPhv_H,
5978   v1i16, v8i16, i32, neon_uimm3_bare,
5979   v4i16, v8i16, neon_uimm2_bare>;
5980 defm : NeonI_Scalar_DUP_Copy_pattern2<DUPbv_B,
5981   v1i8, v16i8, i32, neon_uimm4_bare,
5982   v8i8, v16i8, neon_uimm3_bare>;
5983 defm : NeonI_Scalar_DUP_Copy_pattern2<DUPdv_D,
5984   v1f64, v2f64, f64, neon_uimm1_bare,
5985   v1f64, v2f64, neon_uimm0_bare>;
5986 defm : NeonI_Scalar_DUP_Copy_pattern2<DUPsv_S,
5987   v1f32, v4f32, f32, neon_uimm2_bare,
5988   v2f32, v4f32, neon_uimm1_bare>;
5989
5990 multiclass NeonI_Scalar_DUP_alias<string asmop, string asmlane,
5991                                   Instruction DUPI, Operand OpImm,
5992                                   RegisterClass ResRC> {
5993   def : NeonInstAlias<!strconcat(asmop, "$Rd, $Rn" # asmlane # "[$Imm]"),
5994           (DUPI ResRC:$Rd, VPR128:$Rn, OpImm:$Imm), 0b0>;
5995 }
5996
5997 // Aliases for Scalar copy - DUP element (scalar)
5998 // FIXME: This is actually the preferred syntax but TableGen can't deal with
5999 // custom printing of aliases.
6000 defm : NeonI_Scalar_DUP_alias<"mov", ".b", DUPbv_B, neon_uimm4_bare, FPR8>;
6001 defm : NeonI_Scalar_DUP_alias<"mov", ".h", DUPhv_H, neon_uimm3_bare, FPR16>;
6002 defm : NeonI_Scalar_DUP_alias<"mov", ".s", DUPsv_S, neon_uimm2_bare, FPR32>;
6003 defm : NeonI_Scalar_DUP_alias<"mov", ".d", DUPdv_D, neon_uimm1_bare, FPR64>;
6004
6005 multiclass NeonI_SDUP<PatFrag GetLow, PatFrag GetHigh, ValueType ResTy,
6006                       ValueType OpTy> {
6007   def : Pat<(ResTy (GetLow VPR128:$Rn)),
6008             (ResTy (EXTRACT_SUBREG (OpTy VPR128:$Rn), sub_64))>;
6009   def : Pat<(ResTy (GetHigh VPR128:$Rn)),
6010             (ResTy (DUPdv_D (OpTy VPR128:$Rn), 1))>;
6011 }
6012
6013 defm : NeonI_SDUP<Neon_Low16B, Neon_High16B, v8i8, v16i8>;
6014 defm : NeonI_SDUP<Neon_Low8H, Neon_High8H, v4i16, v8i16>;
6015 defm : NeonI_SDUP<Neon_Low4S, Neon_High4S, v2i32, v4i32>;
6016 defm : NeonI_SDUP<Neon_Low2D, Neon_High2D, v1i64, v2i64>;
6017 defm : NeonI_SDUP<Neon_Low4float, Neon_High4float, v2f32, v4f32>;
6018 defm : NeonI_SDUP<Neon_Low2double, Neon_High2double, v1f64, v2f64>;
6019
6020 //===----------------------------------------------------------------------===//
6021 // Non-Instruction Patterns
6022 //===----------------------------------------------------------------------===//
6023
6024 // 64-bit vector bitcasts...
6025
6026 def : Pat<(v1i64 (bitconvert (v8i8  VPR64:$src))), (v1i64 VPR64:$src)>;
6027 def : Pat<(v2f32 (bitconvert (v8i8  VPR64:$src))), (v2f32 VPR64:$src)>;
6028 def : Pat<(v2i32 (bitconvert (v8i8  VPR64:$src))), (v2i32 VPR64:$src)>;
6029 def : Pat<(v4i16 (bitconvert (v8i8  VPR64:$src))), (v4i16 VPR64:$src)>;
6030
6031 def : Pat<(v1i64 (bitconvert (v4i16  VPR64:$src))), (v1i64 VPR64:$src)>;
6032 def : Pat<(v2i32 (bitconvert (v4i16  VPR64:$src))), (v2i32 VPR64:$src)>;
6033 def : Pat<(v2f32 (bitconvert (v4i16  VPR64:$src))), (v2f32 VPR64:$src)>;
6034 def : Pat<(v8i8  (bitconvert (v4i16  VPR64:$src))), (v8i8 VPR64:$src)>;
6035
6036 def : Pat<(v1i64 (bitconvert (v2i32  VPR64:$src))), (v1i64 VPR64:$src)>;
6037 def : Pat<(v2f32 (bitconvert (v2i32  VPR64:$src))), (v2f32 VPR64:$src)>;
6038 def : Pat<(v4i16 (bitconvert (v2i32  VPR64:$src))), (v4i16 VPR64:$src)>;
6039 def : Pat<(v8i8  (bitconvert (v2i32  VPR64:$src))), (v8i8 VPR64:$src)>;
6040
6041 def : Pat<(v1i64 (bitconvert (v2f32  VPR64:$src))), (v1i64 VPR64:$src)>;
6042 def : Pat<(v2i32 (bitconvert (v2f32  VPR64:$src))), (v2i32 VPR64:$src)>;
6043 def : Pat<(v4i16 (bitconvert (v2f32  VPR64:$src))), (v4i16 VPR64:$src)>;
6044 def : Pat<(v8i8  (bitconvert (v2f32  VPR64:$src))), (v8i8 VPR64:$src)>;
6045
6046 def : Pat<(v2f32 (bitconvert (v1i64  VPR64:$src))), (v2f32 VPR64:$src)>;
6047 def : Pat<(v2i32 (bitconvert (v1i64  VPR64:$src))), (v2i32 VPR64:$src)>;
6048 def : Pat<(v4i16 (bitconvert (v1i64  VPR64:$src))), (v4i16 VPR64:$src)>;
6049 def : Pat<(v8i8  (bitconvert (v1i64  VPR64:$src))), (v8i8 VPR64:$src)>;
6050
6051 // ..and 128-bit vector bitcasts...
6052
6053 def : Pat<(v2f64 (bitconvert (v16i8  VPR128:$src))), (v2f64 VPR128:$src)>;
6054 def : Pat<(v2i64 (bitconvert (v16i8  VPR128:$src))), (v2i64 VPR128:$src)>;
6055 def : Pat<(v4f32 (bitconvert (v16i8  VPR128:$src))), (v4f32 VPR128:$src)>;
6056 def : Pat<(v4i32 (bitconvert (v16i8  VPR128:$src))), (v4i32 VPR128:$src)>;
6057 def : Pat<(v8i16 (bitconvert (v16i8  VPR128:$src))), (v8i16 VPR128:$src)>;
6058
6059 def : Pat<(v2f64 (bitconvert (v8i16  VPR128:$src))), (v2f64 VPR128:$src)>;
6060 def : Pat<(v2i64 (bitconvert (v8i16  VPR128:$src))), (v2i64 VPR128:$src)>;
6061 def : Pat<(v4i32 (bitconvert (v8i16  VPR128:$src))), (v4i32 VPR128:$src)>;
6062 def : Pat<(v4f32 (bitconvert (v8i16  VPR128:$src))), (v4f32 VPR128:$src)>;
6063 def : Pat<(v16i8 (bitconvert (v8i16  VPR128:$src))), (v16i8 VPR128:$src)>;
6064
6065 def : Pat<(v2f64 (bitconvert (v4i32  VPR128:$src))), (v2f64 VPR128:$src)>;
6066 def : Pat<(v2i64 (bitconvert (v4i32  VPR128:$src))), (v2i64 VPR128:$src)>;
6067 def : Pat<(v4f32 (bitconvert (v4i32  VPR128:$src))), (v4f32 VPR128:$src)>;
6068 def : Pat<(v8i16 (bitconvert (v4i32  VPR128:$src))), (v8i16 VPR128:$src)>;
6069 def : Pat<(v16i8 (bitconvert (v4i32  VPR128:$src))), (v16i8 VPR128:$src)>;
6070
6071 def : Pat<(v2f64 (bitconvert (v4f32  VPR128:$src))), (v2f64 VPR128:$src)>;
6072 def : Pat<(v2i64 (bitconvert (v4f32  VPR128:$src))), (v2i64 VPR128:$src)>;
6073 def : Pat<(v4i32 (bitconvert (v4f32  VPR128:$src))), (v4i32 VPR128:$src)>;
6074 def : Pat<(v8i16 (bitconvert (v4f32  VPR128:$src))), (v8i16 VPR128:$src)>;
6075 def : Pat<(v16i8 (bitconvert (v4f32  VPR128:$src))), (v16i8 VPR128:$src)>;
6076
6077 def : Pat<(v2f64 (bitconvert (v2i64  VPR128:$src))), (v2f64 VPR128:$src)>;
6078 def : Pat<(v4f32 (bitconvert (v2i64  VPR128:$src))), (v4f32 VPR128:$src)>;
6079 def : Pat<(v4i32 (bitconvert (v2i64  VPR128:$src))), (v4i32 VPR128:$src)>;
6080 def : Pat<(v8i16 (bitconvert (v2i64  VPR128:$src))), (v8i16 VPR128:$src)>;
6081 def : Pat<(v16i8 (bitconvert (v2i64  VPR128:$src))), (v16i8 VPR128:$src)>;
6082
6083 def : Pat<(v2i64 (bitconvert (v2f64  VPR128:$src))), (v2i64 VPR128:$src)>;
6084 def : Pat<(v4f32 (bitconvert (v2f64  VPR128:$src))), (v4f32 VPR128:$src)>;
6085 def : Pat<(v4i32 (bitconvert (v2f64  VPR128:$src))), (v4i32 VPR128:$src)>;
6086 def : Pat<(v8i16 (bitconvert (v2f64  VPR128:$src))), (v8i16 VPR128:$src)>;
6087 def : Pat<(v16i8 (bitconvert (v2f64  VPR128:$src))), (v16i8 VPR128:$src)>;
6088
6089 // ...and scalar bitcasts...
6090 def : Pat<(f16 (bitconvert (v1i16  FPR16:$src))), (f16 FPR16:$src)>;
6091 def : Pat<(f32 (bitconvert (v1i32  FPR32:$src))), (f32 FPR32:$src)>;
6092 def : Pat<(f64 (bitconvert (v1i64  FPR64:$src))), (f64 FPR64:$src)>;
6093 def : Pat<(f32 (bitconvert (v1f32  FPR32:$src))), (f32 FPR32:$src)>;
6094 def : Pat<(f64 (bitconvert (v1f64  FPR64:$src))), (f64 FPR64:$src)>;
6095
6096 def : Pat<(i64 (bitconvert (v1i64  FPR64:$src))), (FMOVxd $src)>;
6097 def : Pat<(i64 (bitconvert (v1f64  FPR64:$src))), (FMOVxd $src)>;
6098 def : Pat<(i64 (bitconvert (v2i32  FPR64:$src))), (FMOVxd $src)>;
6099 def : Pat<(i64 (bitconvert (v2f32  FPR64:$src))), (FMOVxd $src)>;
6100 def : Pat<(i64 (bitconvert (v4i16  FPR64:$src))), (FMOVxd $src)>;
6101 def : Pat<(i64 (bitconvert (v8i8  FPR64:$src))), (FMOVxd $src)>;
6102
6103 def : Pat<(i32 (bitconvert (v1i32  FPR32:$src))), (FMOVws $src)>;
6104
6105 def : Pat<(v8i8  (bitconvert (v1i64  VPR64:$src))), (v8i8 VPR64:$src)>;
6106 def : Pat<(v4i16 (bitconvert (v1i64  VPR64:$src))), (v4i16 VPR64:$src)>;
6107 def : Pat<(v2i32 (bitconvert (v1i64  VPR64:$src))), (v2i32 VPR64:$src)>;
6108
6109 def : Pat<(f64   (bitconvert (v8i8  VPR64:$src))), (f64 VPR64:$src)>;
6110 def : Pat<(f64   (bitconvert (v4i16  VPR64:$src))), (f64 VPR64:$src)>;
6111 def : Pat<(f64   (bitconvert (v2i32  VPR64:$src))), (f64 VPR64:$src)>;
6112 def : Pat<(f64   (bitconvert (v2f32  VPR64:$src))), (f64 VPR64:$src)>;
6113 def : Pat<(f64   (bitconvert (v1i64  VPR64:$src))), (f64 VPR64:$src)>;
6114
6115 def : Pat<(f128  (bitconvert (v16i8  VPR128:$src))), (f128 VPR128:$src)>;
6116 def : Pat<(f128  (bitconvert (v8i16  VPR128:$src))), (f128 VPR128:$src)>;
6117 def : Pat<(f128  (bitconvert (v4i32  VPR128:$src))), (f128 VPR128:$src)>;
6118 def : Pat<(f128  (bitconvert (v2i64  VPR128:$src))), (f128 VPR128:$src)>;
6119 def : Pat<(f128  (bitconvert (v4f32  VPR128:$src))), (f128 VPR128:$src)>;
6120 def : Pat<(f128  (bitconvert (v2f64  VPR128:$src))), (f128 VPR128:$src)>;
6121
6122 def : Pat<(v1i16 (bitconvert (f16  FPR16:$src))), (v1i16 FPR16:$src)>;
6123 def : Pat<(v1i32 (bitconvert (f32  FPR32:$src))), (v1i32 FPR32:$src)>;
6124 def : Pat<(v1i64 (bitconvert (f64  FPR64:$src))), (v1i64 FPR64:$src)>;
6125 def : Pat<(v1f32 (bitconvert (f32  FPR32:$src))), (v1f32 FPR32:$src)>;
6126 def : Pat<(v1f64 (bitconvert (f64  FPR64:$src))), (v1f64 FPR64:$src)>;
6127
6128 def : Pat<(v1i64 (bitconvert (i64  GPR64:$src))), (FMOVdx $src)>;
6129 def : Pat<(v1f64 (bitconvert (i64  GPR64:$src))), (FMOVdx $src)>;
6130 def : Pat<(v2i32 (bitconvert (i64  GPR64:$src))), (FMOVdx $src)>;
6131 def : Pat<(v2f32 (bitconvert (i64  GPR64:$src))), (FMOVdx $src)>;
6132 def : Pat<(v4i16 (bitconvert (i64  GPR64:$src))), (FMOVdx $src)>;
6133 def : Pat<(v8i8 (bitconvert (i64  GPR64:$src))), (FMOVdx $src)>;
6134
6135 def : Pat<(v1i32 (bitconvert (i32  GPR32:$src))), (FMOVsw $src)>;
6136
6137 def : Pat<(v8i8   (bitconvert (f64   FPR64:$src))), (v8i8 FPR64:$src)>;
6138 def : Pat<(v4i16  (bitconvert (f64   FPR64:$src))), (v4i16 FPR64:$src)>;
6139 def : Pat<(v2i32  (bitconvert (f64   FPR64:$src))), (v2i32 FPR64:$src)>;
6140 def : Pat<(v2f32  (bitconvert (f64   FPR64:$src))), (v2f32 FPR64:$src)>;
6141 def : Pat<(v1i64  (bitconvert (f64   FPR64:$src))), (v1i64 FPR64:$src)>;
6142
6143 def : Pat<(v16i8  (bitconvert (f128   FPR128:$src))), (v16i8 FPR128:$src)>;
6144 def : Pat<(v8i16  (bitconvert (f128   FPR128:$src))), (v8i16 FPR128:$src)>;
6145 def : Pat<(v4i32  (bitconvert (f128   FPR128:$src))), (v4i32 FPR128:$src)>;
6146 def : Pat<(v2i64  (bitconvert (f128   FPR128:$src))), (v2i64 FPR128:$src)>;
6147 def : Pat<(v4f32  (bitconvert (f128   FPR128:$src))), (v4f32 FPR128:$src)>;
6148 def : Pat<(v2f64  (bitconvert (f128   FPR128:$src))), (v2f64 FPR128:$src)>;
6149
6150 // Scalar Three Same
6151
6152 def neon_uimm3 : Operand<i64>,
6153                    ImmLeaf<i64, [{return Imm < 8;}]> {
6154   let ParserMatchClass = uimm3_asmoperand;
6155   let PrintMethod = "printUImmHexOperand";
6156 }
6157
6158 def neon_uimm4 : Operand<i64>,
6159                    ImmLeaf<i64, [{return Imm < 16;}]> {
6160   let ParserMatchClass = uimm4_asmoperand;
6161   let PrintMethod = "printUImmHexOperand";
6162 }
6163
6164 // Bitwise Extract
6165 class NeonI_Extract<bit q, bits<2> op2, string asmop,
6166                     string OpS, RegisterOperand OpVPR, Operand OpImm>
6167   : NeonI_BitExtract<q, op2, (outs OpVPR:$Rd),
6168                      (ins OpVPR:$Rn, OpVPR:$Rm, OpImm:$Index),
6169                      asmop # "\t$Rd." # OpS # ", $Rn." # OpS #
6170                      ", $Rm." # OpS # ", $Index",
6171                      [],
6172                      NoItinerary>{
6173   bits<4> Index;
6174 }
6175
6176 def EXTvvvi_8b : NeonI_Extract<0b0, 0b00, "ext", "8b",
6177                                VPR64, neon_uimm3> {
6178   let Inst{14-11} = {0b0, Index{2}, Index{1}, Index{0}};
6179 }
6180
6181 def EXTvvvi_16b: NeonI_Extract<0b1, 0b00, "ext", "16b",
6182                                VPR128, neon_uimm4> {
6183   let Inst{14-11} = Index;
6184 }
6185
6186 class NI_Extract<ValueType OpTy, RegisterOperand OpVPR, Instruction INST,
6187                  Operand OpImm>
6188   : Pat<(OpTy (Neon_vextract (OpTy OpVPR:$Rn), (OpTy OpVPR:$Rm),
6189                                  (i64 OpImm:$Imm))),
6190               (INST OpVPR:$Rn, OpVPR:$Rm, OpImm:$Imm)>;
6191
6192 def : NI_Extract<v8i8,  VPR64,  EXTvvvi_8b,  neon_uimm3>;
6193 def : NI_Extract<v4i16, VPR64,  EXTvvvi_8b,  neon_uimm3>;
6194 def : NI_Extract<v2i32, VPR64,  EXTvvvi_8b,  neon_uimm3>;
6195 def : NI_Extract<v1i64, VPR64,  EXTvvvi_8b,  neon_uimm3>;
6196 def : NI_Extract<v2f32, VPR64,  EXTvvvi_8b,  neon_uimm3>;
6197 def : NI_Extract<v1f64, VPR64,  EXTvvvi_8b,  neon_uimm3>;
6198 def : NI_Extract<v16i8, VPR128, EXTvvvi_16b, neon_uimm4>;
6199 def : NI_Extract<v8i16, VPR128, EXTvvvi_16b, neon_uimm4>;
6200 def : NI_Extract<v4i32, VPR128, EXTvvvi_16b, neon_uimm4>;
6201 def : NI_Extract<v2i64, VPR128, EXTvvvi_16b, neon_uimm4>;
6202 def : NI_Extract<v4f32, VPR128, EXTvvvi_16b, neon_uimm4>;
6203 def : NI_Extract<v2f64, VPR128, EXTvvvi_16b, neon_uimm4>;
6204
6205 // Table lookup
6206 class NI_TBL<bit q, bits<2> op2, bits<2> len, bit op,
6207              string asmop, string OpS, RegisterOperand OpVPR,
6208              RegisterOperand VecList>
6209   : NeonI_TBL<q, op2, len, op,
6210               (outs OpVPR:$Rd), (ins VecList:$Rn, OpVPR:$Rm),
6211               asmop # "\t$Rd." # OpS # ", $Rn, $Rm." # OpS,
6212               [],
6213               NoItinerary>;
6214
6215 // The vectors in look up table are always 16b
6216 multiclass NI_TBL_pat<bits<2> len, bit op, string asmop, string List> {
6217   def _8b  : NI_TBL<0, 0b00, len, op, asmop, "8b", VPR64,
6218                     !cast<RegisterOperand>(List # "16B_operand")>;
6219
6220   def _16b : NI_TBL<1, 0b00, len, op, asmop, "16b", VPR128,
6221                     !cast<RegisterOperand>(List # "16B_operand")>;
6222 }
6223
6224 defm TBL1 : NI_TBL_pat<0b00, 0b0, "tbl", "VOne">;
6225 defm TBL2 : NI_TBL_pat<0b01, 0b0, "tbl", "VPair">;
6226 defm TBL3 : NI_TBL_pat<0b10, 0b0, "tbl", "VTriple">;
6227 defm TBL4 : NI_TBL_pat<0b11, 0b0, "tbl", "VQuad">;
6228
6229 // Table lookup extention
6230 class NI_TBX<bit q, bits<2> op2, bits<2> len, bit op,
6231              string asmop, string OpS, RegisterOperand OpVPR,
6232              RegisterOperand VecList>
6233   : NeonI_TBL<q, op2, len, op,
6234               (outs OpVPR:$Rd), (ins OpVPR:$src, VecList:$Rn, OpVPR:$Rm),
6235               asmop # "\t$Rd." # OpS # ", $Rn, $Rm." # OpS,
6236               [],
6237               NoItinerary> {
6238   let Constraints = "$src = $Rd";
6239 }
6240
6241 // The vectors in look up table are always 16b
6242 multiclass NI_TBX_pat<bits<2> len, bit op, string asmop, string List> {
6243   def _8b  : NI_TBX<0, 0b00, len, op, asmop, "8b", VPR64,
6244                     !cast<RegisterOperand>(List # "16B_operand")>;
6245
6246   def _16b : NI_TBX<1, 0b00, len, op, asmop, "16b", VPR128,
6247                     !cast<RegisterOperand>(List # "16B_operand")>;
6248 }
6249
6250 defm TBX1 : NI_TBX_pat<0b00, 0b1, "tbx", "VOne">;
6251 defm TBX2 : NI_TBX_pat<0b01, 0b1, "tbx", "VPair">;
6252 defm TBX3 : NI_TBX_pat<0b10, 0b1, "tbx", "VTriple">;
6253 defm TBX4 : NI_TBX_pat<0b11, 0b1, "tbx", "VQuad">;
6254
6255 class NeonI_INS_main<string asmop, string Res, ValueType ResTy,
6256                      RegisterClass OpGPR, ValueType OpTy, Operand OpImm>
6257   : NeonI_copy<0b1, 0b0, 0b0011,
6258                (outs VPR128:$Rd), (ins VPR128:$src, OpGPR:$Rn, OpImm:$Imm),
6259                asmop # "\t$Rd." # Res # "[$Imm], $Rn",
6260                [(set (ResTy VPR128:$Rd),
6261                  (ResTy (vector_insert
6262                    (ResTy VPR128:$src),
6263                    (OpTy OpGPR:$Rn),
6264                    (OpImm:$Imm))))],
6265                NoItinerary> {
6266   bits<4> Imm;
6267   let Constraints = "$src = $Rd";
6268 }
6269
6270 //Insert element (vector, from main)
6271 def INSbw : NeonI_INS_main<"ins", "b", v16i8, GPR32, i32,
6272                            neon_uimm4_bare> {
6273   let Inst{20-16} = {Imm{3}, Imm{2}, Imm{1}, Imm{0}, 0b1};
6274 }
6275 def INShw : NeonI_INS_main<"ins", "h", v8i16, GPR32, i32,
6276                            neon_uimm3_bare> {
6277   let Inst{20-16} = {Imm{2}, Imm{1}, Imm{0}, 0b1, 0b0};
6278 }
6279 def INSsw : NeonI_INS_main<"ins", "s", v4i32, GPR32, i32,
6280                            neon_uimm2_bare> {
6281   let Inst{20-16} = {Imm{1}, Imm{0}, 0b1, 0b0, 0b0};
6282 }
6283 def INSdx : NeonI_INS_main<"ins", "d", v2i64, GPR64, i64,
6284                            neon_uimm1_bare> {
6285   let Inst{20-16} = {Imm, 0b1, 0b0, 0b0, 0b0};
6286 }
6287
6288 def : NeonInstAlias<"mov $Rd.b[$Imm], $Rn",
6289                     (INSbw VPR128:$Rd, GPR32:$Rn, neon_uimm4_bare:$Imm), 0>;
6290 def : NeonInstAlias<"mov $Rd.h[$Imm], $Rn",
6291                     (INShw VPR128:$Rd, GPR32:$Rn, neon_uimm3_bare:$Imm), 0>;
6292 def : NeonInstAlias<"mov $Rd.s[$Imm], $Rn",
6293                     (INSsw VPR128:$Rd, GPR32:$Rn, neon_uimm2_bare:$Imm), 0>;
6294 def : NeonInstAlias<"mov $Rd.d[$Imm], $Rn",
6295                     (INSdx VPR128:$Rd, GPR64:$Rn, neon_uimm1_bare:$Imm), 0>;
6296
6297 class Neon_INS_main_pattern <ValueType ResTy,ValueType ExtResTy,
6298                              RegisterClass OpGPR, ValueType OpTy,
6299                              Operand OpImm, Instruction INS>
6300   : Pat<(ResTy (vector_insert
6301               (ResTy VPR64:$src),
6302               (OpTy OpGPR:$Rn),
6303               (OpImm:$Imm))),
6304         (ResTy (EXTRACT_SUBREG
6305           (ExtResTy (INS (ExtResTy (SUBREG_TO_REG (i64 0), VPR64:$src, sub_64)),
6306             OpGPR:$Rn, OpImm:$Imm)), sub_64))>;
6307
6308 def INSbw_pattern : Neon_INS_main_pattern<v8i8, v16i8, GPR32, i32,
6309                                           neon_uimm3_bare, INSbw>;
6310 def INShw_pattern : Neon_INS_main_pattern<v4i16, v8i16, GPR32, i32,
6311                                           neon_uimm2_bare, INShw>;
6312 def INSsw_pattern : Neon_INS_main_pattern<v2i32, v4i32, GPR32, i32,
6313                                           neon_uimm1_bare, INSsw>;
6314 def INSdx_pattern : Neon_INS_main_pattern<v1i64, v2i64, GPR64, i64,
6315                                           neon_uimm0_bare, INSdx>;
6316
6317 class NeonI_INS_element<string asmop, string Res, Operand ResImm>
6318   : NeonI_insert<0b1, 0b1,
6319                  (outs VPR128:$Rd), (ins VPR128:$src, VPR128:$Rn,
6320                  ResImm:$Immd, ResImm:$Immn),
6321                  asmop # "\t$Rd." # Res # "[$Immd], $Rn." # Res # "[$Immn]",
6322                  [],
6323                  NoItinerary> {
6324   let Constraints = "$src = $Rd";
6325   bits<4> Immd;
6326   bits<4> Immn;
6327 }
6328
6329 //Insert element (vector, from element)
6330 def INSELb : NeonI_INS_element<"ins", "b", neon_uimm4_bare> {
6331   let Inst{20-16} = {Immd{3}, Immd{2}, Immd{1}, Immd{0}, 0b1};
6332   let Inst{14-11} = {Immn{3}, Immn{2}, Immn{1}, Immn{0}};
6333 }
6334 def INSELh : NeonI_INS_element<"ins", "h", neon_uimm3_bare> {
6335   let Inst{20-16} = {Immd{2}, Immd{1}, Immd{0}, 0b1, 0b0};
6336   let Inst{14-11} = {Immn{2}, Immn{1}, Immn{0}, 0b0};
6337   // bit 11 is unspecified, but should be set to zero.
6338 }
6339 def INSELs : NeonI_INS_element<"ins", "s", neon_uimm2_bare> {
6340   let Inst{20-16} = {Immd{1}, Immd{0}, 0b1, 0b0, 0b0};
6341   let Inst{14-11} = {Immn{1}, Immn{0}, 0b0, 0b0};
6342   // bits 11-12 are unspecified, but should be set to zero.
6343 }
6344 def INSELd : NeonI_INS_element<"ins", "d", neon_uimm1_bare> {
6345   let Inst{20-16} = {Immd, 0b1, 0b0, 0b0, 0b0};
6346   let Inst{14-11} = {Immn{0}, 0b0, 0b0, 0b0};
6347   // bits 11-13 are unspecified, but should be set to zero.
6348 }
6349
6350 def : NeonInstAlias<"mov $Rd.b[$Immd], $Rn.b[$Immn]",
6351                     (INSELb VPR128:$Rd, VPR128:$Rn,
6352                       neon_uimm4_bare:$Immd, neon_uimm4_bare:$Immn), 0>;
6353 def : NeonInstAlias<"mov $Rd.h[$Immd], $Rn.h[$Immn]",
6354                     (INSELh VPR128:$Rd, VPR128:$Rn,
6355                       neon_uimm3_bare:$Immd, neon_uimm3_bare:$Immn), 0>;
6356 def : NeonInstAlias<"mov $Rd.s[$Immd], $Rn.s[$Immn]",
6357                     (INSELs VPR128:$Rd, VPR128:$Rn,
6358                       neon_uimm2_bare:$Immd, neon_uimm2_bare:$Immn), 0>;
6359 def : NeonInstAlias<"mov $Rd.d[$Immd], $Rn.d[$Immn]",
6360                     (INSELd VPR128:$Rd, VPR128:$Rn,
6361                       neon_uimm1_bare:$Immd, neon_uimm1_bare:$Immn), 0>;
6362
6363 multiclass Neon_INS_elt_pattern<ValueType ResTy, ValueType NaTy,
6364                                 ValueType MidTy, Operand StImm, Operand NaImm,
6365                                 Instruction INS> {
6366 def : Pat<(ResTy (vector_insert
6367             (ResTy VPR128:$src),
6368             (MidTy (vector_extract
6369               (ResTy VPR128:$Rn),
6370               (StImm:$Immn))),
6371             (StImm:$Immd))),
6372           (INS (ResTy VPR128:$src), (ResTy VPR128:$Rn),
6373               StImm:$Immd, StImm:$Immn)>;
6374
6375 def : Pat <(ResTy (vector_insert
6376              (ResTy VPR128:$src),
6377              (MidTy (vector_extract
6378                (NaTy VPR64:$Rn),
6379                (NaImm:$Immn))),
6380              (StImm:$Immd))),
6381            (INS (ResTy VPR128:$src),
6382              (ResTy (SUBREG_TO_REG (i64 0), (NaTy VPR64:$Rn), sub_64)),
6383              StImm:$Immd, NaImm:$Immn)>;
6384
6385 def : Pat <(NaTy (vector_insert
6386              (NaTy VPR64:$src),
6387              (MidTy (vector_extract
6388                (ResTy VPR128:$Rn),
6389                (StImm:$Immn))),
6390              (NaImm:$Immd))),
6391            (NaTy (EXTRACT_SUBREG
6392              (ResTy (INS
6393                (ResTy (SUBREG_TO_REG (i64 0), (NaTy VPR64:$src), sub_64)),
6394                (ResTy VPR128:$Rn),
6395                NaImm:$Immd, StImm:$Immn)),
6396              sub_64))>;
6397
6398 def : Pat <(NaTy (vector_insert
6399              (NaTy VPR64:$src),
6400              (MidTy (vector_extract
6401                (NaTy VPR64:$Rn),
6402                (NaImm:$Immn))),
6403              (NaImm:$Immd))),
6404            (NaTy (EXTRACT_SUBREG
6405              (ResTy (INS
6406                (ResTy (SUBREG_TO_REG (i64 0), (NaTy VPR64:$src), sub_64)),
6407                (ResTy (SUBREG_TO_REG (i64 0), (NaTy VPR64:$Rn), sub_64)),
6408                NaImm:$Immd, NaImm:$Immn)),
6409              sub_64))>;
6410 }
6411
6412 defm : Neon_INS_elt_pattern<v4f32, v2f32, f32, neon_uimm2_bare,
6413                             neon_uimm1_bare, INSELs>;
6414 defm : Neon_INS_elt_pattern<v2f64, v1f64, f64, neon_uimm1_bare,
6415                             neon_uimm0_bare, INSELd>;
6416 defm : Neon_INS_elt_pattern<v16i8, v8i8, i32, neon_uimm4_bare,
6417                             neon_uimm3_bare, INSELb>;
6418 defm : Neon_INS_elt_pattern<v8i16, v4i16, i32, neon_uimm3_bare,
6419                             neon_uimm2_bare, INSELh>;
6420 defm : Neon_INS_elt_pattern<v4i32, v2i32, i32, neon_uimm2_bare,
6421                             neon_uimm1_bare, INSELs>;
6422 defm : Neon_INS_elt_pattern<v2i64, v1i64, i64, neon_uimm1_bare,
6423                             neon_uimm0_bare, INSELd>;
6424
6425 multiclass Neon_INS_elt_float_pattern<ValueType ResTy, ValueType NaTy,
6426                                       ValueType MidTy,
6427                                       RegisterClass OpFPR, Operand ResImm,
6428                                       SubRegIndex SubIndex, Instruction INS> {
6429 def : Pat <(ResTy (vector_insert
6430              (ResTy VPR128:$src),
6431              (MidTy OpFPR:$Rn),
6432              (ResImm:$Imm))),
6433            (INS (ResTy VPR128:$src),
6434              (ResTy (SUBREG_TO_REG (i64 0), OpFPR:$Rn, SubIndex)),
6435              ResImm:$Imm,
6436              (i64 0))>;
6437
6438 def : Pat <(NaTy (vector_insert
6439              (NaTy VPR64:$src),
6440              (MidTy OpFPR:$Rn),
6441              (ResImm:$Imm))),
6442            (NaTy (EXTRACT_SUBREG
6443              (ResTy (INS
6444                (ResTy (SUBREG_TO_REG (i64 0), (NaTy VPR64:$src), sub_64)),
6445                (ResTy (SUBREG_TO_REG (i64 0), (MidTy OpFPR:$Rn), SubIndex)),
6446                ResImm:$Imm,
6447                (i64 0))),
6448              sub_64))>;
6449 }
6450
6451 defm : Neon_INS_elt_float_pattern<v4f32, v2f32, f32, FPR32, neon_uimm2_bare,
6452                                   sub_32, INSELs>;
6453 defm : Neon_INS_elt_float_pattern<v2f64, v1f64, f64, FPR64, neon_uimm1_bare,
6454                                   sub_64, INSELd>;
6455
6456 class NeonI_SMOV<string asmop, string Res, bit Q,
6457                  ValueType OpTy, ValueType eleTy,
6458                  Operand OpImm, RegisterClass ResGPR, ValueType ResTy>
6459   : NeonI_copy<Q, 0b0, 0b0101,
6460                (outs ResGPR:$Rd), (ins VPR128:$Rn, OpImm:$Imm),
6461                asmop # "\t$Rd, $Rn." # Res # "[$Imm]",
6462                [(set (ResTy ResGPR:$Rd),
6463                  (ResTy (sext_inreg
6464                    (ResTy (vector_extract
6465                      (OpTy VPR128:$Rn), (OpImm:$Imm))),
6466                    eleTy)))],
6467                NoItinerary> {
6468   bits<4> Imm;
6469 }
6470
6471 //Signed integer move (main, from element)
6472 def SMOVwb : NeonI_SMOV<"smov", "b", 0b0, v16i8, i8, neon_uimm4_bare,
6473                         GPR32, i32> {
6474   let Inst{20-16} = {Imm{3}, Imm{2}, Imm{1}, Imm{0}, 0b1};
6475 }
6476 def SMOVwh : NeonI_SMOV<"smov", "h", 0b0, v8i16, i16, neon_uimm3_bare,
6477                         GPR32, i32> {
6478   let Inst{20-16} = {Imm{2}, Imm{1}, Imm{0}, 0b1, 0b0};
6479 }
6480 def SMOVxb : NeonI_SMOV<"smov", "b", 0b1, v16i8, i8, neon_uimm4_bare,
6481                         GPR64, i64> {
6482   let Inst{20-16} = {Imm{3}, Imm{2}, Imm{1}, Imm{0}, 0b1};
6483 }
6484 def SMOVxh : NeonI_SMOV<"smov", "h", 0b1, v8i16, i16, neon_uimm3_bare,
6485                         GPR64, i64> {
6486   let Inst{20-16} = {Imm{2}, Imm{1}, Imm{0}, 0b1, 0b0};
6487 }
6488 def SMOVxs : NeonI_SMOV<"smov", "s", 0b1, v4i32, i32, neon_uimm2_bare,
6489                         GPR64, i64> {
6490   let Inst{20-16} = {Imm{1}, Imm{0}, 0b1, 0b0, 0b0};
6491 }
6492
6493 multiclass Neon_SMOVx_pattern <ValueType StTy, ValueType NaTy,
6494                                ValueType eleTy, Operand StImm,  Operand NaImm,
6495                                Instruction SMOVI> {
6496   def : Pat<(i64 (sext_inreg
6497               (i64 (anyext
6498                 (i32 (vector_extract
6499                   (StTy VPR128:$Rn), (StImm:$Imm))))),
6500               eleTy)),
6501             (SMOVI VPR128:$Rn, StImm:$Imm)>;
6502
6503   def : Pat<(i64 (sext
6504               (i32 (vector_extract
6505                 (StTy VPR128:$Rn), (StImm:$Imm))))),
6506             (SMOVI VPR128:$Rn, StImm:$Imm)>;
6507
6508   def : Pat<(i64 (sext_inreg
6509               (i64 (vector_extract
6510                 (NaTy VPR64:$Rn), (NaImm:$Imm))),
6511               eleTy)),
6512             (SMOVI (StTy (SUBREG_TO_REG (i64 0), VPR64:$Rn, sub_64)),
6513               NaImm:$Imm)>;
6514
6515   def : Pat<(i64 (sext_inreg
6516               (i64 (anyext
6517                 (i32 (vector_extract
6518                   (NaTy VPR64:$Rn), (NaImm:$Imm))))),
6519               eleTy)),
6520             (SMOVI (StTy (SUBREG_TO_REG (i64 0), VPR64:$Rn, sub_64)),
6521               NaImm:$Imm)>;
6522
6523   def : Pat<(i64 (sext
6524               (i32 (vector_extract
6525                 (NaTy VPR64:$Rn), (NaImm:$Imm))))),
6526             (SMOVI (StTy (SUBREG_TO_REG (i64 0), VPR64:$Rn, sub_64)),
6527               NaImm:$Imm)>;
6528 }
6529
6530 defm : Neon_SMOVx_pattern<v16i8, v8i8, i8, neon_uimm4_bare,
6531                           neon_uimm3_bare, SMOVxb>;
6532 defm : Neon_SMOVx_pattern<v8i16, v4i16, i16, neon_uimm3_bare,
6533                           neon_uimm2_bare, SMOVxh>;
6534 defm : Neon_SMOVx_pattern<v4i32, v2i32, i32, neon_uimm2_bare,
6535                           neon_uimm1_bare, SMOVxs>;
6536
6537 class Neon_SMOVw_pattern <ValueType StTy, ValueType NaTy,
6538                           ValueType eleTy, Operand StImm,  Operand NaImm,
6539                           Instruction SMOVI>
6540   : Pat<(i32 (sext_inreg
6541           (i32 (vector_extract
6542             (NaTy VPR64:$Rn), (NaImm:$Imm))),
6543           eleTy)),
6544         (SMOVI (StTy (SUBREG_TO_REG (i64 0), VPR64:$Rn, sub_64)),
6545           NaImm:$Imm)>;
6546
6547 def : Neon_SMOVw_pattern<v16i8, v8i8, i8, neon_uimm4_bare,
6548                          neon_uimm3_bare, SMOVwb>;
6549 def : Neon_SMOVw_pattern<v8i16, v4i16, i16, neon_uimm3_bare,
6550                          neon_uimm2_bare, SMOVwh>;
6551
6552 class NeonI_UMOV<string asmop, string Res, bit Q,
6553                  ValueType OpTy, Operand OpImm,
6554                  RegisterClass ResGPR, ValueType ResTy>
6555   : NeonI_copy<Q, 0b0, 0b0111,
6556                (outs ResGPR:$Rd), (ins VPR128:$Rn, OpImm:$Imm),
6557                asmop # "\t$Rd, $Rn." # Res # "[$Imm]",
6558                [(set (ResTy ResGPR:$Rd),
6559                   (ResTy (vector_extract
6560                     (OpTy VPR128:$Rn), (OpImm:$Imm))))],
6561                NoItinerary> {
6562   bits<4> Imm;
6563 }
6564
6565 //Unsigned integer move (main, from element)
6566 def UMOVwb : NeonI_UMOV<"umov", "b", 0b0, v16i8, neon_uimm4_bare,
6567                          GPR32, i32> {
6568   let Inst{20-16} = {Imm{3}, Imm{2}, Imm{1}, Imm{0}, 0b1};
6569 }
6570 def UMOVwh : NeonI_UMOV<"umov", "h", 0b0, v8i16, neon_uimm3_bare,
6571                          GPR32, i32> {
6572   let Inst{20-16} = {Imm{2}, Imm{1}, Imm{0}, 0b1, 0b0};
6573 }
6574 def UMOVws : NeonI_UMOV<"umov", "s", 0b0, v4i32, neon_uimm2_bare,
6575                          GPR32, i32> {
6576   let Inst{20-16} = {Imm{1}, Imm{0}, 0b1, 0b0, 0b0};
6577 }
6578 def UMOVxd : NeonI_UMOV<"umov", "d", 0b1, v2i64, neon_uimm1_bare,
6579                          GPR64, i64> {
6580   let Inst{20-16} = {Imm, 0b1, 0b0, 0b0, 0b0};
6581 }
6582
6583 def : NeonInstAlias<"mov $Rd, $Rn.s[$Imm]",
6584                     (UMOVws GPR32:$Rd, VPR128:$Rn, neon_uimm2_bare:$Imm), 0>;
6585 def : NeonInstAlias<"mov $Rd, $Rn.d[$Imm]",
6586                     (UMOVxd GPR64:$Rd, VPR128:$Rn, neon_uimm1_bare:$Imm), 0>;
6587
6588 class Neon_UMOV_pattern <ValueType StTy, ValueType NaTy, ValueType ResTy,
6589                          Operand StImm,  Operand NaImm,
6590                          Instruction SMOVI>
6591   : Pat<(ResTy (vector_extract
6592           (NaTy VPR64:$Rn), NaImm:$Imm)),
6593         (SMOVI (StTy (SUBREG_TO_REG (i64 0), VPR64:$Rn, sub_64)),
6594           NaImm:$Imm)>;
6595
6596 def : Neon_UMOV_pattern<v16i8, v8i8, i32, neon_uimm4_bare,
6597                         neon_uimm3_bare, UMOVwb>;
6598 def : Neon_UMOV_pattern<v8i16, v4i16, i32, neon_uimm3_bare,
6599                         neon_uimm2_bare, UMOVwh>;
6600 def : Neon_UMOV_pattern<v4i32, v2i32, i32, neon_uimm2_bare,
6601                         neon_uimm1_bare, UMOVws>;
6602
6603 def : Pat<(i32 (and
6604             (i32 (vector_extract
6605               (v16i8 VPR128:$Rn), (neon_uimm4_bare:$Imm))),
6606             255)),
6607           (UMOVwb VPR128:$Rn, neon_uimm4_bare:$Imm)>;
6608
6609 def : Pat<(i32 (and
6610             (i32 (vector_extract
6611               (v8i16 VPR128:$Rn), (neon_uimm3_bare:$Imm))),
6612             65535)),
6613           (UMOVwh VPR128:$Rn, neon_uimm3_bare:$Imm)>;
6614
6615 def : Pat<(i64 (zext
6616             (i32 (vector_extract
6617               (v2i64 VPR128:$Rn), (neon_uimm1_bare:$Imm))))),
6618           (UMOVxd VPR128:$Rn, neon_uimm1_bare:$Imm)>;
6619
6620 def : Pat<(i32 (and
6621             (i32 (vector_extract
6622               (v8i8 VPR64:$Rn), (neon_uimm3_bare:$Imm))),
6623             255)),
6624           (UMOVwb (SUBREG_TO_REG (i64 0), VPR64:$Rn, sub_64),
6625             neon_uimm3_bare:$Imm)>;
6626
6627 def : Pat<(i32 (and
6628             (i32 (vector_extract
6629               (v4i16 VPR64:$Rn), (neon_uimm2_bare:$Imm))),
6630             65535)),
6631           (UMOVwh (SUBREG_TO_REG (i64 0), VPR64:$Rn, sub_64),
6632             neon_uimm2_bare:$Imm)>;
6633
6634 def : Pat<(i64 (zext
6635             (i32 (vector_extract
6636               (v1i64 VPR64:$Rn), (neon_uimm0_bare:$Imm))))),
6637           (UMOVxd (SUBREG_TO_REG (i64 0), VPR64:$Rn, sub_64),
6638             neon_uimm0_bare:$Imm)>;
6639
6640 // Additional copy patterns for scalar types
6641 def : Pat<(i32 (vector_extract (v1i8 FPR8:$Rn), (i64 0))),
6642           (UMOVwb (v16i8
6643             (SUBREG_TO_REG (i64 0), FPR8:$Rn, sub_8)), (i64 0))>;
6644
6645 def : Pat<(i32 (vector_extract (v1i16 FPR16:$Rn), (i64 0))),
6646           (UMOVwh (v8i16
6647             (SUBREG_TO_REG (i64 0), FPR16:$Rn, sub_16)), (i64 0))>;
6648
6649 def : Pat<(i32 (vector_extract (v1i32 FPR32:$Rn), (i64 0))),
6650           (FMOVws FPR32:$Rn)>;
6651
6652 def : Pat<(i64 (vector_extract (v1i64 FPR64:$Rn), (i64 0))),
6653           (FMOVxd FPR64:$Rn)>;
6654
6655 def : Pat<(f64 (vector_extract (v1f64 FPR64:$Rn), (i64 0))),
6656           (f64 FPR64:$Rn)>;
6657
6658 def : Pat<(f32 (vector_extract (v1f32 FPR32:$Rn), (i64 0))),
6659           (f32 FPR32:$Rn)>;
6660
6661 def : Pat<(v1i8 (scalar_to_vector GPR32:$Rn)),
6662           (v1i8 (EXTRACT_SUBREG (v16i8
6663             (INSbw (v16i8 (IMPLICIT_DEF)), $Rn, (i64 0))),
6664             sub_8))>;
6665
6666 def : Pat<(v1i16 (scalar_to_vector GPR32:$Rn)),
6667           (v1i16 (EXTRACT_SUBREG (v8i16
6668             (INShw (v8i16 (IMPLICIT_DEF)), $Rn, (i64 0))),
6669             sub_16))>;
6670
6671 def : Pat<(v1i32 (scalar_to_vector GPR32:$src)),
6672           (FMOVsw $src)>;
6673
6674 def : Pat<(v1i64 (scalar_to_vector GPR64:$src)),
6675           (FMOVdx $src)>;
6676
6677 def : Pat<(v1f32 (scalar_to_vector (f32 FPR32:$Rn))),
6678           (v1f32 FPR32:$Rn)>;
6679 def : Pat<(v1f64 (scalar_to_vector (f64 FPR64:$Rn))),
6680           (v1f64 FPR64:$Rn)>;
6681
6682 def : Pat<(v1f64 (scalar_to_vector (f64 FPR64:$src))),
6683           (FMOVdd $src)>;
6684
6685 def : Pat<(v2f64 (scalar_to_vector (f64 FPR64:$src))),
6686           (INSERT_SUBREG (v2f64 (IMPLICIT_DEF)),
6687                          (f64 FPR64:$src), sub_64)>;
6688
6689 class NeonI_DUP_Elt<bit Q, string asmop, string rdlane,  string rnlane,
6690                     RegisterOperand ResVPR, Operand OpImm>
6691   : NeonI_copy<Q, 0b0, 0b0000, (outs ResVPR:$Rd),
6692                (ins VPR128:$Rn, OpImm:$Imm),
6693                asmop # "\t$Rd" # rdlane # ", $Rn" # rnlane # "[$Imm]",
6694                [],
6695                NoItinerary> {
6696   bits<4> Imm;
6697 }
6698
6699 def DUPELT16b : NeonI_DUP_Elt<0b1, "dup", ".16b", ".b", VPR128,
6700                               neon_uimm4_bare> {
6701   let Inst{20-16} = {Imm{3}, Imm{2}, Imm{1}, Imm{0}, 0b1};
6702 }
6703
6704 def DUPELT8h : NeonI_DUP_Elt<0b1, "dup", ".8h", ".h", VPR128,
6705                               neon_uimm3_bare> {
6706   let Inst{20-16} = {Imm{2}, Imm{1}, Imm{0}, 0b1, 0b0};
6707 }
6708
6709 def DUPELT4s : NeonI_DUP_Elt<0b1, "dup", ".4s", ".s", VPR128,
6710                               neon_uimm2_bare> {
6711   let Inst{20-16} = {Imm{1}, Imm{0}, 0b1, 0b0, 0b0};
6712 }
6713
6714 def DUPELT2d : NeonI_DUP_Elt<0b1, "dup", ".2d", ".d", VPR128,
6715                               neon_uimm1_bare> {
6716   let Inst{20-16} = {Imm, 0b1, 0b0, 0b0, 0b0};
6717 }
6718
6719 def DUPELT8b : NeonI_DUP_Elt<0b0, "dup", ".8b", ".b", VPR64,
6720                               neon_uimm4_bare> {
6721   let Inst{20-16} = {Imm{3}, Imm{2}, Imm{1}, Imm{0}, 0b1};
6722 }
6723
6724 def DUPELT4h : NeonI_DUP_Elt<0b0, "dup", ".4h", ".h", VPR64,
6725                               neon_uimm3_bare> {
6726   let Inst{20-16} = {Imm{2}, Imm{1}, Imm{0}, 0b1, 0b0};
6727 }
6728
6729 def DUPELT2s : NeonI_DUP_Elt<0b0, "dup", ".2s", ".s", VPR64,
6730                               neon_uimm2_bare> {
6731   let Inst{20-16} = {Imm{1}, Imm{0}, 0b1, 0b0, 0b0};
6732 }
6733
6734 multiclass NeonI_DUP_Elt_pattern<Instruction DUPELT, ValueType ResTy,
6735                                        ValueType OpTy,ValueType NaTy,
6736                                        ValueType ExTy, Operand OpLImm,
6737                                        Operand OpNImm> {
6738 def  : Pat<(ResTy (Neon_vduplane (OpTy VPR128:$Rn), OpLImm:$Imm)),
6739         (ResTy (DUPELT (OpTy VPR128:$Rn), OpLImm:$Imm))>;
6740
6741 def : Pat<(ResTy (Neon_vduplane
6742             (NaTy VPR64:$Rn), OpNImm:$Imm)),
6743           (ResTy (DUPELT
6744             (ExTy (SUBREG_TO_REG (i64 0), VPR64:$Rn, sub_64)), OpNImm:$Imm))>;
6745 }
6746 defm : NeonI_DUP_Elt_pattern<DUPELT16b, v16i8, v16i8, v8i8, v16i8,
6747                              neon_uimm4_bare, neon_uimm3_bare>;
6748 defm : NeonI_DUP_Elt_pattern<DUPELT8b, v8i8, v16i8, v8i8, v16i8,
6749                              neon_uimm4_bare, neon_uimm3_bare>;
6750 defm : NeonI_DUP_Elt_pattern<DUPELT8h, v8i16, v8i16, v4i16, v8i16,
6751                              neon_uimm3_bare, neon_uimm2_bare>;
6752 defm : NeonI_DUP_Elt_pattern<DUPELT4h, v4i16, v8i16, v4i16, v8i16,
6753                              neon_uimm3_bare, neon_uimm2_bare>;
6754 defm : NeonI_DUP_Elt_pattern<DUPELT4s, v4i32, v4i32, v2i32, v4i32,
6755                              neon_uimm2_bare, neon_uimm1_bare>;
6756 defm : NeonI_DUP_Elt_pattern<DUPELT2s, v2i32, v4i32, v2i32, v4i32,
6757                              neon_uimm2_bare, neon_uimm1_bare>;
6758 defm : NeonI_DUP_Elt_pattern<DUPELT2d, v2i64, v2i64, v1i64, v2i64,
6759                              neon_uimm1_bare, neon_uimm0_bare>;
6760 defm : NeonI_DUP_Elt_pattern<DUPELT4s, v4f32, v4f32, v2f32, v4f32,
6761                              neon_uimm2_bare, neon_uimm1_bare>;
6762 defm : NeonI_DUP_Elt_pattern<DUPELT2s, v2f32, v4f32, v2f32, v4f32,
6763                              neon_uimm2_bare, neon_uimm1_bare>;
6764 defm : NeonI_DUP_Elt_pattern<DUPELT2d, v2f64, v2f64, v1f64, v2f64,
6765                              neon_uimm1_bare, neon_uimm0_bare>;
6766
6767 def : Pat<(v2f32 (Neon_vdup (f32 FPR32:$Rn))),
6768           (v2f32 (DUPELT2s
6769             (SUBREG_TO_REG (i64 0), FPR32:$Rn, sub_32),
6770             (i64 0)))>;
6771 def : Pat<(v4f32 (Neon_vdup (f32 FPR32:$Rn))),
6772           (v4f32 (DUPELT4s
6773             (SUBREG_TO_REG (i64 0), FPR32:$Rn, sub_32),
6774             (i64 0)))>;
6775 def : Pat<(v2f64 (Neon_vdup (f64 FPR64:$Rn))),
6776           (v2f64 (DUPELT2d
6777             (SUBREG_TO_REG (i64 0), FPR64:$Rn, sub_64),
6778             (i64 0)))>;
6779
6780 class NeonI_DUP<bit Q, string asmop, string rdlane,
6781                 RegisterOperand ResVPR, ValueType ResTy,
6782                 RegisterClass OpGPR, ValueType OpTy>
6783   : NeonI_copy<Q, 0b0, 0b0001, (outs ResVPR:$Rd), (ins OpGPR:$Rn),
6784                asmop # "\t$Rd" # rdlane # ", $Rn",
6785                [(set (ResTy ResVPR:$Rd),
6786                  (ResTy (Neon_vdup (OpTy OpGPR:$Rn))))],
6787                NoItinerary>;
6788
6789 def DUP16b : NeonI_DUP<0b1, "dup", ".16b", VPR128, v16i8, GPR32, i32> {
6790   let Inst{20-16} = 0b00001;
6791   // bits 17-20 are unspecified, but should be set to zero.
6792 }
6793
6794 def DUP8h : NeonI_DUP<0b1, "dup", ".8h", VPR128, v8i16, GPR32, i32> {
6795   let Inst{20-16} = 0b00010;
6796   // bits 18-20 are unspecified, but should be set to zero.
6797 }
6798
6799 def DUP4s : NeonI_DUP<0b1, "dup", ".4s", VPR128, v4i32, GPR32, i32> {
6800   let Inst{20-16} = 0b00100;
6801   // bits 19-20 are unspecified, but should be set to zero.
6802 }
6803
6804 def DUP2d : NeonI_DUP<0b1, "dup", ".2d", VPR128, v2i64, GPR64, i64> {
6805   let Inst{20-16} = 0b01000;
6806   // bit 20 is unspecified, but should be set to zero.
6807 }
6808
6809 def DUP8b : NeonI_DUP<0b0, "dup", ".8b", VPR64, v8i8, GPR32, i32> {
6810   let Inst{20-16} = 0b00001;
6811   // bits 17-20 are unspecified, but should be set to zero.
6812 }
6813
6814 def DUP4h : NeonI_DUP<0b0, "dup", ".4h", VPR64, v4i16, GPR32, i32> {
6815   let Inst{20-16} = 0b00010;
6816   // bits 18-20 are unspecified, but should be set to zero.
6817 }
6818
6819 def DUP2s : NeonI_DUP<0b0, "dup", ".2s", VPR64, v2i32, GPR32, i32> {
6820   let Inst{20-16} = 0b00100;
6821   // bits 19-20 are unspecified, but should be set to zero.
6822 }
6823
6824 // patterns for CONCAT_VECTORS
6825 multiclass Concat_Vector_Pattern<ValueType ResTy, ValueType OpTy> {
6826 def : Pat<(ResTy (concat_vectors (OpTy VPR64:$Rn), undef)),
6827           (SUBREG_TO_REG (i64 0), VPR64:$Rn, sub_64)>;
6828 def : Pat<(ResTy (concat_vectors (OpTy VPR64:$Rn), (OpTy VPR64:$Rm))),
6829           (INSELd
6830             (v2i64 (SUBREG_TO_REG (i64 0), VPR64:$Rn, sub_64)),
6831             (v2i64 (SUBREG_TO_REG (i64 0), VPR64:$Rm, sub_64)),
6832             (i64 1),
6833             (i64 0))>;
6834 def : Pat<(ResTy (concat_vectors (OpTy VPR64:$Rn), (OpTy VPR64:$Rn))),
6835           (DUPELT2d
6836             (v2i64 (SUBREG_TO_REG (i64 0), VPR64:$Rn, sub_64)),
6837             (i64 0))> ;
6838 }
6839
6840 defm : Concat_Vector_Pattern<v16i8, v8i8>;
6841 defm : Concat_Vector_Pattern<v8i16, v4i16>;
6842 defm : Concat_Vector_Pattern<v4i32, v2i32>;
6843 defm : Concat_Vector_Pattern<v2i64, v1i64>;
6844 defm : Concat_Vector_Pattern<v4f32, v2f32>;
6845 defm : Concat_Vector_Pattern<v2f64, v1f64>;
6846
6847 //patterns for EXTRACT_SUBVECTOR
6848 def : Pat<(v8i8 (extract_subvector (v16i8 VPR128:$Rn), (i64 0))),
6849           (v8i8 (EXTRACT_SUBREG VPR128:$Rn, sub_64))>;
6850 def : Pat<(v4i16 (extract_subvector (v8i16 VPR128:$Rn), (i64 0))),
6851           (v4i16 (EXTRACT_SUBREG VPR128:$Rn, sub_64))>;
6852 def : Pat<(v2i32 (extract_subvector (v4i32 VPR128:$Rn), (i64 0))),
6853           (v2i32 (EXTRACT_SUBREG VPR128:$Rn, sub_64))>;
6854 def : Pat<(v1i64 (extract_subvector (v2i64 VPR128:$Rn), (i64 0))),
6855           (v1i64 (EXTRACT_SUBREG VPR128:$Rn, sub_64))>;
6856 def : Pat<(v2f32 (extract_subvector (v4f32 VPR128:$Rn), (i64 0))),
6857           (v2f32 (EXTRACT_SUBREG VPR128:$Rn, sub_64))>;
6858 def : Pat<(v1f64 (extract_subvector (v2f64 VPR128:$Rn), (i64 0))),
6859           (v1f64 (EXTRACT_SUBREG VPR128:$Rn, sub_64))>;
6860
6861 // The followings are for instruction class (3V Elem)
6862
6863 // Variant 1
6864
6865 class NI_2VE<bit q, bit u, bits<2> size, bits<4> opcode,
6866              string asmop, string ResS, string OpS, string EleOpS,
6867              Operand OpImm, RegisterOperand ResVPR,
6868              RegisterOperand OpVPR, RegisterOperand EleOpVPR>
6869   : NeonI_2VElem<q, u, size, opcode,
6870                  (outs ResVPR:$Rd), (ins ResVPR:$src, OpVPR:$Rn,
6871                                          EleOpVPR:$Re, OpImm:$Index),
6872                  asmop # "\t$Rd." # ResS # ", $Rn." # OpS #
6873                  ", $Re." # EleOpS # "[$Index]",
6874                  [],
6875                  NoItinerary> {
6876   bits<3> Index;
6877   bits<5> Re;
6878
6879   let Constraints = "$src = $Rd";
6880 }
6881
6882 multiclass NI_2VE_v1<bit u, bits<4> opcode, string asmop> {
6883   // vector register class for element is always 128-bit to cover the max index
6884   def _2s4s : NI_2VE<0b0, u, 0b10, opcode, asmop, "2s", "2s", "s",
6885                      neon_uimm2_bare, VPR64, VPR64, VPR128> {
6886     let Inst{11} = {Index{1}};
6887     let Inst{21} = {Index{0}};
6888     let Inst{20-16} = Re;
6889   }
6890
6891   def _4s4s : NI_2VE<0b1, u, 0b10, opcode, asmop, "4s", "4s", "s",
6892                      neon_uimm2_bare, VPR128, VPR128, VPR128> {
6893     let Inst{11} = {Index{1}};
6894     let Inst{21} = {Index{0}};
6895     let Inst{20-16} = Re;
6896   }
6897
6898   // Index operations on 16-bit(H) elements are restricted to using v0-v15.
6899   def _4h8h : NI_2VE<0b0, u, 0b01, opcode, asmop, "4h", "4h", "h",
6900                      neon_uimm3_bare, VPR64, VPR64, VPR128Lo> {
6901     let Inst{11} = {Index{2}};
6902     let Inst{21} = {Index{1}};
6903     let Inst{20} = {Index{0}};
6904     let Inst{19-16} = Re{3-0};
6905   }
6906
6907   def _8h8h : NI_2VE<0b1, u, 0b01, opcode, asmop, "8h", "8h", "h",
6908                      neon_uimm3_bare, VPR128, VPR128, VPR128Lo> {
6909     let Inst{11} = {Index{2}};
6910     let Inst{21} = {Index{1}};
6911     let Inst{20} = {Index{0}};
6912     let Inst{19-16} = Re{3-0};
6913   }
6914 }
6915
6916 defm MLAvve : NI_2VE_v1<0b1, 0b0000, "mla">;
6917 defm MLSvve : NI_2VE_v1<0b1, 0b0100, "mls">;
6918
6919 // Pattern for lane in 128-bit vector
6920 class NI_2VE_laneq<Instruction INST, Operand OpImm, SDPatternOperator op,
6921                    RegisterOperand ResVPR, RegisterOperand OpVPR,
6922                    RegisterOperand EleOpVPR, ValueType ResTy, ValueType OpTy,
6923                    ValueType EleOpTy>
6924   : Pat<(ResTy (op (ResTy ResVPR:$src), (OpTy OpVPR:$Rn),
6925           (OpTy (Neon_vduplane (EleOpTy EleOpVPR:$Re), (i64 OpImm:$Index))))),
6926         (INST ResVPR:$src, OpVPR:$Rn, EleOpVPR:$Re, OpImm:$Index)>;
6927
6928 // Pattern for lane in 64-bit vector
6929 class NI_2VE_lane<Instruction INST, Operand OpImm, SDPatternOperator op,
6930                   RegisterOperand ResVPR, RegisterOperand OpVPR,
6931                   RegisterOperand EleOpVPR, ValueType ResTy, ValueType OpTy,
6932                   ValueType EleOpTy>
6933   : Pat<(ResTy (op (ResTy ResVPR:$src), (OpTy OpVPR:$Rn),
6934           (OpTy (Neon_vduplane (EleOpTy EleOpVPR:$Re), (i64 OpImm:$Index))))),
6935         (INST ResVPR:$src, OpVPR:$Rn,
6936           (SUBREG_TO_REG (i64 0), EleOpVPR:$Re, sub_64), OpImm:$Index)>;
6937
6938 multiclass NI_2VE_v1_pat<string subop, SDPatternOperator op>
6939 {
6940   def : NI_2VE_laneq<!cast<Instruction>(subop # "_2s4s"), neon_uimm2_bare,
6941                      op, VPR64, VPR64, VPR128, v2i32, v2i32, v4i32>;
6942
6943   def : NI_2VE_laneq<!cast<Instruction>(subop # "_4s4s"), neon_uimm2_bare,
6944                      op, VPR128, VPR128, VPR128, v4i32, v4i32, v4i32>;
6945
6946   def : NI_2VE_laneq<!cast<Instruction>(subop # "_4h8h"), neon_uimm3_bare,
6947                      op, VPR64, VPR64, VPR128Lo, v4i16, v4i16, v8i16>;
6948
6949   def : NI_2VE_laneq<!cast<Instruction>(subop # "_8h8h"), neon_uimm3_bare,
6950                      op, VPR128, VPR128, VPR128Lo, v8i16, v8i16, v8i16>;
6951
6952   // Index can only be half of the max value for lane in 64-bit vector
6953
6954   def : NI_2VE_lane<!cast<Instruction>(subop # "_2s4s"), neon_uimm1_bare,
6955                     op, VPR64, VPR64, VPR64, v2i32, v2i32, v2i32>;
6956
6957   def : NI_2VE_lane<!cast<Instruction>(subop # "_4h8h"), neon_uimm2_bare,
6958                     op, VPR64, VPR64, VPR64Lo, v4i16, v4i16, v4i16>;
6959 }
6960
6961 defm MLA_lane_v1 : NI_2VE_v1_pat<"MLAvve", Neon_mla>;
6962 defm MLS_lane_v1 : NI_2VE_v1_pat<"MLSvve", Neon_mls>;
6963
6964 class NI_2VE_2op<bit q, bit u, bits<2> size, bits<4> opcode,
6965                  string asmop, string ResS, string OpS, string EleOpS,
6966                  Operand OpImm, RegisterOperand ResVPR,
6967                  RegisterOperand OpVPR, RegisterOperand EleOpVPR>
6968   : NeonI_2VElem<q, u, size, opcode,
6969                  (outs ResVPR:$Rd), (ins OpVPR:$Rn,
6970                                          EleOpVPR:$Re, OpImm:$Index),
6971                  asmop # "\t$Rd." # ResS # ", $Rn." # OpS #
6972                  ", $Re." # EleOpS # "[$Index]",
6973                  [],
6974                  NoItinerary> {
6975   bits<3> Index;
6976   bits<5> Re;
6977 }
6978
6979 multiclass NI_2VE_v1_2op<bit u, bits<4> opcode, string asmop> {
6980   // vector register class for element is always 128-bit to cover the max index
6981   def _2s4s : NI_2VE_2op<0b0, u, 0b10, opcode, asmop, "2s", "2s", "s",
6982                          neon_uimm2_bare, VPR64, VPR64, VPR128> {
6983     let Inst{11} = {Index{1}};
6984     let Inst{21} = {Index{0}};
6985     let Inst{20-16} = Re;
6986   }
6987
6988   def _4s4s : NI_2VE_2op<0b1, u, 0b10, opcode, asmop, "4s", "4s", "s",
6989                          neon_uimm2_bare, VPR128, VPR128, VPR128> {
6990     let Inst{11} = {Index{1}};
6991     let Inst{21} = {Index{0}};
6992     let Inst{20-16} = Re;
6993   }
6994
6995   // Index operations on 16-bit(H) elements are restricted to using v0-v15.
6996   def _4h8h : NI_2VE_2op<0b0, u, 0b01, opcode, asmop, "4h", "4h", "h",
6997                          neon_uimm3_bare, VPR64, VPR64, VPR128Lo> {
6998     let Inst{11} = {Index{2}};
6999     let Inst{21} = {Index{1}};
7000     let Inst{20} = {Index{0}};
7001     let Inst{19-16} = Re{3-0};
7002   }
7003
7004   def _8h8h : NI_2VE_2op<0b1, u, 0b01, opcode, asmop, "8h", "8h", "h",
7005                          neon_uimm3_bare, VPR128, VPR128, VPR128Lo> {
7006     let Inst{11} = {Index{2}};
7007     let Inst{21} = {Index{1}};
7008     let Inst{20} = {Index{0}};
7009     let Inst{19-16} = Re{3-0};
7010   }
7011 }
7012
7013 defm MULve : NI_2VE_v1_2op<0b0, 0b1000, "mul">;
7014 defm SQDMULHve : NI_2VE_v1_2op<0b0, 0b1100, "sqdmulh">;
7015 defm SQRDMULHve : NI_2VE_v1_2op<0b0, 0b1101, "sqrdmulh">;
7016
7017 // Pattern for lane in 128-bit vector
7018 class NI_2VE_mul_laneq<Instruction INST, Operand OpImm, SDPatternOperator op,
7019                        RegisterOperand OpVPR, RegisterOperand EleOpVPR,
7020                        ValueType ResTy, ValueType OpTy, ValueType EleOpTy>
7021   : Pat<(ResTy (op (OpTy OpVPR:$Rn),
7022           (OpTy (Neon_vduplane (EleOpTy EleOpVPR:$Re), (i64 OpImm:$Index))))),
7023         (INST OpVPR:$Rn, EleOpVPR:$Re, OpImm:$Index)>;
7024
7025 // Pattern for lane in 64-bit vector
7026 class NI_2VE_mul_lane<Instruction INST, Operand OpImm, SDPatternOperator op,
7027                       RegisterOperand OpVPR, RegisterOperand EleOpVPR,
7028                       ValueType ResTy, ValueType OpTy, ValueType EleOpTy>
7029   : Pat<(ResTy (op (OpTy OpVPR:$Rn),
7030           (OpTy (Neon_vduplane (EleOpTy EleOpVPR:$Re), (i64 OpImm:$Index))))),
7031         (INST OpVPR:$Rn,
7032           (SUBREG_TO_REG (i64 0), EleOpVPR:$Re, sub_64), OpImm:$Index)>;
7033
7034 multiclass NI_2VE_mul_v1_pat<string subop, SDPatternOperator op> {
7035   def : NI_2VE_mul_laneq<!cast<Instruction>(subop # "_2s4s"), neon_uimm2_bare,
7036                          op, VPR64, VPR128, v2i32, v2i32, v4i32>;
7037
7038   def : NI_2VE_mul_laneq<!cast<Instruction>(subop # "_4s4s"), neon_uimm2_bare,
7039                          op, VPR128, VPR128, v4i32, v4i32, v4i32>;
7040
7041   def : NI_2VE_mul_laneq<!cast<Instruction>(subop # "_4h8h"), neon_uimm3_bare,
7042                          op, VPR64, VPR128Lo, v4i16, v4i16, v8i16>;
7043
7044   def : NI_2VE_mul_laneq<!cast<Instruction>(subop # "_8h8h"), neon_uimm3_bare,
7045                          op, VPR128, VPR128Lo, v8i16, v8i16, v8i16>;
7046
7047   // Index can only be half of the max value for lane in 64-bit vector
7048
7049   def : NI_2VE_mul_lane<!cast<Instruction>(subop # "_2s4s"), neon_uimm1_bare,
7050                         op, VPR64, VPR64, v2i32, v2i32, v2i32>;
7051
7052   def : NI_2VE_mul_lane<!cast<Instruction>(subop # "_4h8h"), neon_uimm2_bare,
7053                         op, VPR64, VPR64Lo, v4i16, v4i16, v4i16>;
7054 }
7055
7056 defm MUL_lane_v1 : NI_2VE_mul_v1_pat<"MULve", mul>;
7057 defm SQDMULH_lane_v1 : NI_2VE_mul_v1_pat<"SQDMULHve", int_arm_neon_vqdmulh>;
7058 defm SQRDMULH_lane_v1 : NI_2VE_mul_v1_pat<"SQRDMULHve", int_arm_neon_vqrdmulh>;
7059
7060 // Variant 2
7061
7062 multiclass NI_2VE_v2_2op<bit u, bits<4> opcode, string asmop> {
7063   // vector register class for element is always 128-bit to cover the max index
7064   def _2s4s : NI_2VE_2op<0b0, u, 0b10, opcode, asmop, "2s", "2s", "s",
7065                          neon_uimm2_bare, VPR64, VPR64, VPR128> {
7066     let Inst{11} = {Index{1}};
7067     let Inst{21} = {Index{0}};
7068     let Inst{20-16} = Re;
7069   }
7070
7071   def _4s4s : NI_2VE_2op<0b1, u, 0b10, opcode, asmop, "4s", "4s", "s",
7072                          neon_uimm2_bare, VPR128, VPR128, VPR128> {
7073     let Inst{11} = {Index{1}};
7074     let Inst{21} = {Index{0}};
7075     let Inst{20-16} = Re;
7076   }
7077
7078   // _1d2d doesn't exist!
7079
7080   def _2d2d : NI_2VE_2op<0b1, u, 0b11, opcode, asmop, "2d", "2d", "d",
7081                          neon_uimm1_bare, VPR128, VPR128, VPR128> {
7082     let Inst{11} = {Index{0}};
7083     let Inst{21} = 0b0;
7084     let Inst{20-16} = Re;
7085   }
7086 }
7087
7088 defm FMULve : NI_2VE_v2_2op<0b0, 0b1001, "fmul">;
7089 defm FMULXve : NI_2VE_v2_2op<0b1, 0b1001, "fmulx">;
7090
7091 class NI_2VE_mul_lane_2d<Instruction INST, Operand OpImm, SDPatternOperator op,
7092                          RegisterOperand OpVPR, RegisterOperand EleOpVPR,
7093                          ValueType ResTy, ValueType OpTy, ValueType EleOpTy,
7094                          SDPatternOperator coreop>
7095   : Pat<(ResTy (op (OpTy OpVPR:$Rn),
7096           (OpTy (coreop (EleOpTy EleOpVPR:$Re), (EleOpTy EleOpVPR:$Re))))),
7097         (INST OpVPR:$Rn,
7098           (SUBREG_TO_REG (i64 0), EleOpVPR:$Re, sub_64), 0)>;
7099
7100 multiclass NI_2VE_mul_v2_pat<string subop, SDPatternOperator op> {
7101   def : NI_2VE_mul_laneq<!cast<Instruction>(subop # "_2s4s"), neon_uimm2_bare,
7102                          op, VPR64, VPR128, v2f32, v2f32, v4f32>;
7103
7104   def : NI_2VE_mul_laneq<!cast<Instruction>(subop # "_4s4s"), neon_uimm2_bare,
7105                          op, VPR128, VPR128, v4f32, v4f32, v4f32>;
7106
7107   def : NI_2VE_mul_laneq<!cast<Instruction>(subop # "_2d2d"), neon_uimm1_bare,
7108                          op, VPR128, VPR128, v2f64, v2f64, v2f64>;
7109
7110   // Index can only be half of the max value for lane in 64-bit vector
7111
7112   def : NI_2VE_mul_lane<!cast<Instruction>(subop # "_2s4s"), neon_uimm1_bare,
7113                         op, VPR64, VPR64, v2f32, v2f32, v2f32>;
7114
7115   def : NI_2VE_mul_lane_2d<!cast<Instruction>(subop # "_2d2d"), neon_uimm1_bare,
7116                            op, VPR128, VPR64, v2f64, v2f64, v1f64,
7117                            BinOpFrag<(Neon_combine_2d node:$LHS, node:$RHS)>>;
7118 }
7119
7120 defm FMUL_lane_v2 : NI_2VE_mul_v2_pat<"FMULve", fmul>;
7121 defm FMULX_lane_v2 : NI_2VE_mul_v2_pat<"FMULXve", int_aarch64_neon_vmulx>;
7122
7123 def : Pat<(v2f32 (fmul (v2f32 (Neon_vdup (f32 FPR32:$Re))),
7124                        (v2f32 VPR64:$Rn))),
7125           (FMULve_2s4s VPR64:$Rn, (SUBREG_TO_REG (i32 0), $Re, sub_32), 0)>;
7126
7127 def : Pat<(v4f32 (fmul (v4f32 (Neon_vdup (f32 FPR32:$Re))),
7128                        (v4f32 VPR128:$Rn))),
7129           (FMULve_4s4s VPR128:$Rn, (SUBREG_TO_REG (i32 0), $Re, sub_32), 0)>;
7130
7131 def : Pat<(v2f64 (fmul (v2f64 (Neon_vdup (f64 FPR64:$Re))),
7132                        (v2f64 VPR128:$Rn))),
7133           (FMULve_2d2d VPR128:$Rn, (SUBREG_TO_REG (i64 0), $Re, sub_64), 0)>;
7134
7135 // The followings are patterns using fma
7136 // -ffp-contract=fast generates fma
7137
7138 multiclass NI_2VE_v2<bit u, bits<4> opcode, string asmop> {
7139   // vector register class for element is always 128-bit to cover the max index
7140   def _2s4s : NI_2VE<0b0, u, 0b10, opcode, asmop, "2s", "2s", "s",
7141                      neon_uimm2_bare, VPR64, VPR64, VPR128> {
7142     let Inst{11} = {Index{1}};
7143     let Inst{21} = {Index{0}};
7144     let Inst{20-16} = Re;
7145   }
7146
7147   def _4s4s : NI_2VE<0b1, u, 0b10, opcode, asmop, "4s", "4s", "s",
7148                      neon_uimm2_bare, VPR128, VPR128, VPR128> {
7149     let Inst{11} = {Index{1}};
7150     let Inst{21} = {Index{0}};
7151     let Inst{20-16} = Re;
7152   }
7153
7154   // _1d2d doesn't exist!
7155
7156   def _2d2d : NI_2VE<0b1, u, 0b11, opcode, asmop, "2d", "2d", "d",
7157                      neon_uimm1_bare, VPR128, VPR128, VPR128> {
7158     let Inst{11} = {Index{0}};
7159     let Inst{21} = 0b0;
7160     let Inst{20-16} = Re;
7161   }
7162 }
7163
7164 defm FMLAvve : NI_2VE_v2<0b0, 0b0001, "fmla">;
7165 defm FMLSvve : NI_2VE_v2<0b0, 0b0101, "fmls">;
7166
7167 // Pattern for lane in 128-bit vector
7168 class NI_2VEswap_laneq<Instruction INST, Operand OpImm, SDPatternOperator op,
7169                        RegisterOperand ResVPR, RegisterOperand OpVPR,
7170                        ValueType ResTy, ValueType OpTy,
7171                        SDPatternOperator coreop>
7172   : Pat<(ResTy (op (ResTy (coreop (OpTy OpVPR:$Re), (i64 OpImm:$Index))),
7173                    (ResTy ResVPR:$src), (ResTy ResVPR:$Rn))),
7174         (INST ResVPR:$src, ResVPR:$Rn, OpVPR:$Re, OpImm:$Index)>;
7175
7176 // Pattern for lane 0
7177 class NI_2VEfma_lane0<Instruction INST, SDPatternOperator op,
7178                       RegisterOperand ResVPR, ValueType ResTy>
7179   : Pat<(ResTy (op (ResTy ResVPR:$Rn),
7180                    (ResTy (Neon_vdup (f32 FPR32:$Re))),
7181                    (ResTy ResVPR:$src))),
7182         (INST ResVPR:$src, ResVPR:$Rn,
7183               (SUBREG_TO_REG (i32 0), $Re, sub_32), 0)>;
7184
7185 // Pattern for lane in 64-bit vector
7186 class NI_2VEswap_lane<Instruction INST, Operand OpImm, SDPatternOperator op,
7187                       RegisterOperand ResVPR, RegisterOperand OpVPR,
7188                       ValueType ResTy, ValueType OpTy,
7189                       SDPatternOperator coreop>
7190   : Pat<(ResTy (op (ResTy (coreop (OpTy OpVPR:$Re), (i64 OpImm:$Index))),
7191                    (ResTy ResVPR:$Rn), (ResTy ResVPR:$src))),
7192         (INST ResVPR:$src, ResVPR:$Rn,
7193           (SUBREG_TO_REG (i64 0), OpVPR:$Re, sub_64), OpImm:$Index)>;
7194
7195 // Pattern for lane in 64-bit vector
7196 class NI_2VEswap_lane_2d2d<Instruction INST, Operand OpImm,
7197                            SDPatternOperator op,
7198                            RegisterOperand ResVPR, RegisterOperand OpVPR,
7199                            ValueType ResTy, ValueType OpTy,
7200                            SDPatternOperator coreop>
7201   : Pat<(ResTy (op (ResTy (coreop (OpTy OpVPR:$Re), (OpTy OpVPR:$Re))),
7202                    (ResTy ResVPR:$Rn), (ResTy ResVPR:$src))),
7203         (INST ResVPR:$src, ResVPR:$Rn,
7204           (SUBREG_TO_REG (i64 0), OpVPR:$Re, sub_64), 0)>;
7205
7206
7207 multiclass NI_2VE_fma_v2_pat<string subop, SDPatternOperator op> {
7208   def : NI_2VEswap_laneq<!cast<Instruction>(subop # "_2s4s"),
7209                          neon_uimm2_bare, op, VPR64, VPR128, v2f32, v4f32,
7210                          BinOpFrag<(Neon_vduplane node:$LHS, node:$RHS)>>;
7211
7212   def : NI_2VEfma_lane0<!cast<Instruction>(subop # "_2s4s"),
7213                         op, VPR64, v2f32>;
7214
7215   def : NI_2VEswap_laneq<!cast<Instruction>(subop # "_4s4s"),
7216                          neon_uimm2_bare, op, VPR128, VPR128, v4f32, v4f32,
7217                          BinOpFrag<(Neon_vduplane node:$LHS, node:$RHS)>>;
7218
7219   def : NI_2VEfma_lane0<!cast<Instruction>(subop # "_4s4s"),
7220                         op, VPR128, v4f32>;
7221
7222   def : NI_2VEswap_laneq<!cast<Instruction>(subop # "_2d2d"),
7223                          neon_uimm1_bare, op, VPR128, VPR128, v2f64, v2f64,
7224                          BinOpFrag<(Neon_vduplane node:$LHS, node:$RHS)>>;
7225
7226   // Index can only be half of the max value for lane in 64-bit vector
7227
7228   def : NI_2VEswap_lane<!cast<Instruction>(subop # "_2s4s"),
7229                         neon_uimm1_bare, op, VPR64, VPR64, v2f32, v2f32,
7230                         BinOpFrag<(Neon_vduplane node:$LHS, node:$RHS)>>;
7231
7232   def : NI_2VEswap_lane_2d2d<!cast<Instruction>(subop # "_2d2d"),
7233                              neon_uimm1_bare, op, VPR128, VPR64, v2f64, v1f64,
7234                              BinOpFrag<(Neon_combine_2d node:$LHS, node:$RHS)>>;
7235 }
7236
7237 defm FMLA_lane_v2_s : NI_2VE_fma_v2_pat<"FMLAvve", fma>;
7238
7239 // Pattern for lane 0
7240 class NI_2VEfms_lane0<Instruction INST, SDPatternOperator op,
7241                       RegisterOperand ResVPR, ValueType ResTy>
7242   : Pat<(ResTy (op (ResTy (fneg ResVPR:$Rn)),
7243                    (ResTy (Neon_vdup (f32 FPR32:$Re))),
7244                    (ResTy ResVPR:$src))),
7245         (INST ResVPR:$src, ResVPR:$Rn,
7246               (SUBREG_TO_REG (i32 0), $Re, sub_32), 0)>;
7247
7248 multiclass NI_2VE_fms_v2_pat<string subop, SDPatternOperator op>
7249 {
7250   def : NI_2VEswap_laneq<!cast<Instruction>(subop # "_2s4s"),
7251                          neon_uimm2_bare, op, VPR64, VPR128, v2f32, v4f32,
7252                          BinOpFrag<(fneg (Neon_vduplane node:$LHS, node:$RHS))>>;
7253
7254   def : NI_2VEswap_laneq<!cast<Instruction>(subop # "_2s4s"),
7255                          neon_uimm2_bare, op, VPR64, VPR128, v2f32, v4f32,
7256                          BinOpFrag<(Neon_vduplane
7257                                      (fneg node:$LHS), node:$RHS)>>;
7258
7259   def : NI_2VEfms_lane0<!cast<Instruction>(subop # "_2s4s"),
7260                         op, VPR64, v2f32>;
7261
7262   def : NI_2VEswap_laneq<!cast<Instruction>(subop # "_4s4s"),
7263                          neon_uimm2_bare, op, VPR128, VPR128, v4f32, v4f32,
7264                          BinOpFrag<(fneg (Neon_vduplane
7265                                      node:$LHS, node:$RHS))>>;
7266
7267   def : NI_2VEswap_laneq<!cast<Instruction>(subop # "_4s4s"),
7268                          neon_uimm2_bare, op, VPR128, VPR128, v4f32, v4f32,
7269                          BinOpFrag<(Neon_vduplane
7270                                      (fneg node:$LHS), node:$RHS)>>;
7271
7272   def : NI_2VEfms_lane0<!cast<Instruction>(subop # "_4s4s"),
7273                         op, VPR128, v4f32>;
7274
7275   def : NI_2VEswap_laneq<!cast<Instruction>(subop # "_2d2d"),
7276                          neon_uimm1_bare, op, VPR128, VPR128, v2f64, v2f64,
7277                          BinOpFrag<(fneg (Neon_vduplane
7278                                      node:$LHS, node:$RHS))>>;
7279
7280   def : NI_2VEswap_laneq<!cast<Instruction>(subop # "_2d2d"),
7281                          neon_uimm1_bare, op, VPR128, VPR128, v2f64, v2f64,
7282                          BinOpFrag<(Neon_vduplane
7283                                      (fneg node:$LHS), node:$RHS)>>;
7284
7285   // Index can only be half of the max value for lane in 64-bit vector
7286
7287   def : NI_2VEswap_lane<!cast<Instruction>(subop # "_2s4s"),
7288                         neon_uimm1_bare, op, VPR64, VPR64, v2f32, v2f32,
7289                         BinOpFrag<(fneg (Neon_vduplane
7290                                     node:$LHS, node:$RHS))>>;
7291
7292   def : NI_2VEswap_lane<!cast<Instruction>(subop # "_2s4s"),
7293                         neon_uimm1_bare, op, VPR64, VPR64, v2f32, v2f32,
7294                         BinOpFrag<(Neon_vduplane
7295                                     (fneg node:$LHS), node:$RHS)>>;
7296
7297   def : NI_2VEswap_lane<!cast<Instruction>(subop # "_4s4s"),
7298                         neon_uimm1_bare, op, VPR128, VPR64, v4f32, v2f32,
7299                         BinOpFrag<(fneg (Neon_vduplane node:$LHS, node:$RHS))>>;
7300
7301   def : NI_2VEswap_lane<!cast<Instruction>(subop # "_4s4s"),
7302                         neon_uimm1_bare, op, VPR128, VPR64, v4f32, v2f32,
7303                         BinOpFrag<(Neon_vduplane (fneg node:$LHS), node:$RHS)>>;
7304
7305   def : NI_2VEswap_lane_2d2d<!cast<Instruction>(subop # "_2d2d"),
7306                              neon_uimm1_bare, op, VPR128, VPR64, v2f64, v1f64,
7307                              BinOpFrag<(fneg (Neon_combine_2d
7308                                          node:$LHS, node:$RHS))>>;
7309
7310   def : NI_2VEswap_lane_2d2d<!cast<Instruction>(subop # "_2d2d"),
7311                              neon_uimm1_bare, op, VPR128, VPR64, v2f64, v1f64,
7312                              BinOpFrag<(Neon_combine_2d
7313                                          (fneg node:$LHS), (fneg node:$RHS))>>;
7314 }
7315
7316 defm FMLS_lane_v2_s : NI_2VE_fms_v2_pat<"FMLSvve", fma>;
7317
7318 // Variant 3: Long type
7319 // E.g. SMLAL : 4S/4H/H (v0-v15), 2D/2S/S
7320 //      SMLAL2: 4S/8H/H (v0-v15), 2D/4S/S
7321
7322 multiclass NI_2VE_v3<bit u, bits<4> opcode, string asmop> {
7323   // vector register class for element is always 128-bit to cover the max index
7324   def _2d2s : NI_2VE<0b0, u, 0b10, opcode, asmop, "2d", "2s", "s",
7325                      neon_uimm2_bare, VPR128, VPR64, VPR128> {
7326     let Inst{11} = {Index{1}};
7327     let Inst{21} = {Index{0}};
7328     let Inst{20-16} = Re;
7329   }
7330
7331   def _2d4s : NI_2VE<0b1, u, 0b10, opcode, asmop # "2", "2d", "4s", "s",
7332                      neon_uimm2_bare, VPR128, VPR128, VPR128> {
7333     let Inst{11} = {Index{1}};
7334     let Inst{21} = {Index{0}};
7335     let Inst{20-16} = Re;
7336   }
7337
7338   // Index operations on 16-bit(H) elements are restricted to using v0-v15.
7339   def _4s8h : NI_2VE<0b1, u, 0b01, opcode, asmop # "2", "4s", "8h", "h",
7340                      neon_uimm3_bare, VPR128, VPR128, VPR128Lo> {
7341     let Inst{11} = {Index{2}};
7342     let Inst{21} = {Index{1}};
7343     let Inst{20} = {Index{0}};
7344     let Inst{19-16} = Re{3-0};
7345   }
7346
7347   def _4s4h : NI_2VE<0b0, u, 0b01, opcode, asmop, "4s", "4h", "h",
7348                      neon_uimm3_bare, VPR128, VPR64, VPR128Lo> {
7349     let Inst{11} = {Index{2}};
7350     let Inst{21} = {Index{1}};
7351     let Inst{20} = {Index{0}};
7352     let Inst{19-16} = Re{3-0};
7353   }
7354 }
7355
7356 defm SMLALvve : NI_2VE_v3<0b0, 0b0010, "smlal">;
7357 defm UMLALvve : NI_2VE_v3<0b1, 0b0010, "umlal">;
7358 defm SMLSLvve : NI_2VE_v3<0b0, 0b0110, "smlsl">;
7359 defm UMLSLvve : NI_2VE_v3<0b1, 0b0110, "umlsl">;
7360 defm SQDMLALvve : NI_2VE_v3<0b0, 0b0011, "sqdmlal">;
7361 defm SQDMLSLvve : NI_2VE_v3<0b0, 0b0111, "sqdmlsl">;
7362
7363 multiclass NI_2VE_v3_2op<bit u, bits<4> opcode, string asmop> {
7364   // vector register class for element is always 128-bit to cover the max index
7365   def _2d2s : NI_2VE_2op<0b0, u, 0b10, opcode, asmop, "2d", "2s", "s",
7366                          neon_uimm2_bare, VPR128, VPR64, VPR128> {
7367     let Inst{11} = {Index{1}};
7368     let Inst{21} = {Index{0}};
7369     let Inst{20-16} = Re;
7370   }
7371
7372   def _2d4s : NI_2VE_2op<0b1, u, 0b10, opcode, asmop # "2", "2d", "4s", "s",
7373                          neon_uimm2_bare, VPR128, VPR128, VPR128> {
7374     let Inst{11} = {Index{1}};
7375     let Inst{21} = {Index{0}};
7376     let Inst{20-16} = Re;
7377   }
7378
7379   // Index operations on 16-bit(H) elements are restricted to using v0-v15.
7380   def _4s8h : NI_2VE_2op<0b1, u, 0b01, opcode, asmop # "2", "4s", "8h", "h",
7381                          neon_uimm3_bare, VPR128, VPR128, VPR128Lo> {
7382     let Inst{11} = {Index{2}};
7383     let Inst{21} = {Index{1}};
7384     let Inst{20} = {Index{0}};
7385     let Inst{19-16} = Re{3-0};
7386   }
7387
7388   def _4s4h : NI_2VE_2op<0b0, u, 0b01, opcode, asmop, "4s", "4h", "h",
7389                          neon_uimm3_bare, VPR128, VPR64, VPR128Lo> {
7390     let Inst{11} = {Index{2}};
7391     let Inst{21} = {Index{1}};
7392     let Inst{20} = {Index{0}};
7393     let Inst{19-16} = Re{3-0};
7394   }
7395 }
7396
7397 defm SMULLve : NI_2VE_v3_2op<0b0, 0b1010, "smull">;
7398 defm UMULLve : NI_2VE_v3_2op<0b1, 0b1010, "umull">;
7399 defm SQDMULLve : NI_2VE_v3_2op<0b0, 0b1011, "sqdmull">;
7400
7401 def : Pat<(v1f64 (scalar_to_vector (f64 FPR64:$src))),
7402           (FMOVdd $src)>;
7403 def : Pat<(v1f32 (scalar_to_vector (f32 FPR32:$src))),
7404           (FMOVss $src)>;
7405
7406 // Pattern for lane in 128-bit vector
7407 class NI_2VEL2_laneq<Instruction INST, Operand OpImm, SDPatternOperator op,
7408                      RegisterOperand EleOpVPR, ValueType ResTy,
7409                      ValueType OpTy, ValueType EleOpTy, ValueType HalfOpTy,
7410                      SDPatternOperator hiop>
7411   : Pat<(ResTy (op (ResTy VPR128:$src),
7412           (HalfOpTy (hiop (OpTy VPR128:$Rn))),
7413           (HalfOpTy (Neon_vduplane
7414                       (EleOpTy EleOpVPR:$Re), (i64 OpImm:$Index))))),
7415         (INST VPR128:$src, VPR128:$Rn, EleOpVPR:$Re, OpImm:$Index)>;
7416
7417 // Pattern for lane in 64-bit vector
7418 class NI_2VEL2_lane<Instruction INST, Operand OpImm, SDPatternOperator op,
7419                     RegisterOperand EleOpVPR, ValueType ResTy,
7420                     ValueType OpTy, ValueType EleOpTy, ValueType HalfOpTy,
7421                     SDPatternOperator hiop>
7422   : Pat<(ResTy (op (ResTy VPR128:$src),
7423           (HalfOpTy (hiop (OpTy VPR128:$Rn))),
7424           (HalfOpTy (Neon_vduplane
7425                       (EleOpTy EleOpVPR:$Re), (i64 OpImm:$Index))))),
7426         (INST VPR128:$src, VPR128:$Rn,
7427           (SUBREG_TO_REG (i64 0), EleOpVPR:$Re, sub_64), OpImm:$Index)>;
7428
7429 class NI_2VEL2_lane0<Instruction INST, SDPatternOperator op,
7430                      ValueType ResTy, ValueType OpTy, ValueType HalfOpTy,
7431                      SDPatternOperator hiop, Instruction DupInst>
7432   : Pat<(ResTy (op (ResTy VPR128:$src),
7433           (HalfOpTy (hiop (OpTy VPR128:$Rn))),
7434           (HalfOpTy (Neon_vdup (i32 GPR32:$Re))))),
7435         (INST VPR128:$src, VPR128:$Rn, (DupInst $Re), 0)>;
7436
7437 multiclass NI_2VEL_v3_pat<string subop, SDPatternOperator op> {
7438   def : NI_2VE_laneq<!cast<Instruction>(subop # "_4s4h"), neon_uimm3_bare,
7439                      op, VPR128, VPR64, VPR128Lo, v4i32, v4i16, v8i16>;
7440
7441   def : NI_2VE_laneq<!cast<Instruction>(subop # "_2d2s"), neon_uimm2_bare,
7442                      op, VPR128, VPR64, VPR128, v2i64, v2i32, v4i32>;
7443
7444   def : NI_2VEL2_laneq<!cast<Instruction>(subop # "_4s8h"), neon_uimm3_bare,
7445                        op, VPR128Lo, v4i32, v8i16, v8i16, v4i16, Neon_High8H>;
7446
7447   def : NI_2VEL2_laneq<!cast<Instruction>(subop # "_2d4s"), neon_uimm2_bare,
7448                        op, VPR128, v2i64, v4i32, v4i32, v2i32, Neon_High4S>;
7449
7450   def : NI_2VEL2_lane0<!cast<Instruction>(subop # "_4s8h"),
7451                        op, v4i32, v8i16, v4i16, Neon_High8H, DUP8h>;
7452
7453   def : NI_2VEL2_lane0<!cast<Instruction>(subop # "_2d4s"),
7454                        op, v2i64, v4i32, v2i32, Neon_High4S, DUP4s>;
7455
7456   // Index can only be half of the max value for lane in 64-bit vector
7457
7458   def : NI_2VE_lane<!cast<Instruction>(subop # "_4s4h"), neon_uimm2_bare,
7459                     op, VPR128, VPR64, VPR64Lo, v4i32, v4i16, v4i16>;
7460
7461   def : NI_2VE_lane<!cast<Instruction>(subop # "_2d2s"), neon_uimm1_bare,
7462                     op, VPR128, VPR64, VPR64, v2i64, v2i32, v2i32>;
7463
7464   def : NI_2VEL2_lane<!cast<Instruction>(subop # "_4s8h"), neon_uimm2_bare,
7465                       op, VPR64Lo, v4i32, v8i16, v4i16, v4i16, Neon_High8H>;
7466
7467   def : NI_2VEL2_lane<!cast<Instruction>(subop # "_2d4s"), neon_uimm1_bare,
7468                       op, VPR64, v2i64, v4i32, v2i32, v2i32, Neon_High4S>;
7469 }
7470
7471 defm SMLAL_lane_v3 : NI_2VEL_v3_pat<"SMLALvve", Neon_smlal>;
7472 defm UMLAL_lane_v3 : NI_2VEL_v3_pat<"UMLALvve", Neon_umlal>;
7473 defm SMLSL_lane_v3 : NI_2VEL_v3_pat<"SMLSLvve", Neon_smlsl>;
7474 defm UMLSL_lane_v3 : NI_2VEL_v3_pat<"UMLSLvve", Neon_umlsl>;
7475
7476 // Pattern for lane in 128-bit vector
7477 class NI_2VEL2_mul_laneq<Instruction INST, Operand OpImm, SDPatternOperator op,
7478                          RegisterOperand EleOpVPR, ValueType ResTy,
7479                          ValueType OpTy, ValueType EleOpTy, ValueType HalfOpTy,
7480                          SDPatternOperator hiop>
7481   : Pat<(ResTy (op
7482           (HalfOpTy (hiop (OpTy VPR128:$Rn))),
7483           (HalfOpTy (Neon_vduplane
7484                       (EleOpTy EleOpVPR:$Re), (i64 OpImm:$Index))))),
7485         (INST VPR128:$Rn, EleOpVPR:$Re, OpImm:$Index)>;
7486
7487 // Pattern for lane in 64-bit vector
7488 class NI_2VEL2_mul_lane<Instruction INST, Operand OpImm, SDPatternOperator op,
7489                         RegisterOperand EleOpVPR, ValueType ResTy,
7490                         ValueType OpTy, ValueType EleOpTy, ValueType HalfOpTy,
7491                         SDPatternOperator hiop>
7492   : Pat<(ResTy (op
7493           (HalfOpTy (hiop (OpTy VPR128:$Rn))),
7494           (HalfOpTy (Neon_vduplane
7495                       (EleOpTy EleOpVPR:$Re), (i64 OpImm:$Index))))),
7496         (INST VPR128:$Rn,
7497           (SUBREG_TO_REG (i64 0), EleOpVPR:$Re, sub_64), OpImm:$Index)>;
7498
7499 // Pattern for fixed lane 0
7500 class NI_2VEL2_mul_lane0<Instruction INST, SDPatternOperator op,
7501                          ValueType ResTy, ValueType OpTy, ValueType HalfOpTy,
7502                          SDPatternOperator hiop, Instruction DupInst>
7503   : Pat<(ResTy (op
7504           (HalfOpTy (hiop (OpTy VPR128:$Rn))),
7505           (HalfOpTy (Neon_vdup (i32 GPR32:$Re))))),
7506         (INST VPR128:$Rn, (DupInst $Re), 0)>;
7507
7508 multiclass NI_2VEL_mul_v3_pat<string subop, SDPatternOperator op> {
7509   def : NI_2VE_mul_laneq<!cast<Instruction>(subop # "_4s4h"), neon_uimm3_bare,
7510                          op, VPR64, VPR128Lo, v4i32, v4i16, v8i16>;
7511
7512   def : NI_2VE_mul_laneq<!cast<Instruction>(subop # "_2d2s"), neon_uimm2_bare,
7513                          op, VPR64, VPR128, v2i64, v2i32, v4i32>;
7514
7515   def : NI_2VEL2_mul_laneq<!cast<Instruction>(subop # "_4s8h"), neon_uimm3_bare,
7516                          op, VPR128Lo, v4i32, v8i16, v8i16, v4i16, Neon_High8H>;
7517
7518   def : NI_2VEL2_mul_laneq<!cast<Instruction>(subop # "_2d4s"), neon_uimm2_bare,
7519                            op, VPR128, v2i64, v4i32, v4i32, v2i32, Neon_High4S>;
7520
7521   def : NI_2VEL2_mul_lane0<!cast<Instruction>(subop # "_4s8h"),
7522                            op, v4i32, v8i16, v4i16, Neon_High8H, DUP8h>;
7523
7524   def : NI_2VEL2_mul_lane0<!cast<Instruction>(subop # "_2d4s"),
7525                            op, v2i64, v4i32, v2i32, Neon_High4S, DUP4s>;
7526
7527   // Index can only be half of the max value for lane in 64-bit vector
7528
7529   def : NI_2VE_mul_lane<!cast<Instruction>(subop # "_4s4h"), neon_uimm2_bare,
7530                         op, VPR64, VPR64Lo, v4i32, v4i16, v4i16>;
7531
7532   def : NI_2VE_mul_lane<!cast<Instruction>(subop # "_2d2s"), neon_uimm1_bare,
7533                         op, VPR64, VPR64, v2i64, v2i32, v2i32>;
7534
7535   def : NI_2VEL2_mul_lane<!cast<Instruction>(subop # "_4s8h"), neon_uimm2_bare,
7536                           op, VPR64Lo, v4i32, v8i16, v4i16, v4i16, Neon_High8H>;
7537
7538   def : NI_2VEL2_mul_lane<!cast<Instruction>(subop # "_2d4s"), neon_uimm1_bare,
7539                           op, VPR64, v2i64, v4i32, v2i32, v2i32, Neon_High4S>;
7540 }
7541
7542 defm SMULL_lane_v3 : NI_2VEL_mul_v3_pat<"SMULLve", int_arm_neon_vmulls>;
7543 defm UMULL_lane_v3 : NI_2VEL_mul_v3_pat<"UMULLve", int_arm_neon_vmullu>;
7544 defm SQDMULL_lane_v3 : NI_2VEL_mul_v3_pat<"SQDMULLve", int_arm_neon_vqdmull>;
7545
7546 multiclass NI_qdma<SDPatternOperator op> {
7547   def _4s : PatFrag<(ops node:$Ra, node:$Rn, node:$Rm),
7548                     (op node:$Ra,
7549                       (v4i32 (int_arm_neon_vqdmull node:$Rn, node:$Rm)))>;
7550
7551   def _2d : PatFrag<(ops node:$Ra, node:$Rn, node:$Rm),
7552                     (op node:$Ra,
7553                       (v2i64 (int_arm_neon_vqdmull node:$Rn, node:$Rm)))>;
7554 }
7555
7556 defm Neon_qdmlal : NI_qdma<int_arm_neon_vqadds>;
7557 defm Neon_qdmlsl : NI_qdma<int_arm_neon_vqsubs>;
7558
7559 multiclass NI_2VEL_v3_qdma_pat<string subop, string op> {
7560   def : NI_2VE_laneq<!cast<Instruction>(subop # "_4s4h"), neon_uimm3_bare,
7561                      !cast<PatFrag>(op # "_4s"), VPR128, VPR64, VPR128Lo,
7562                      v4i32, v4i16, v8i16>;
7563
7564   def : NI_2VE_laneq<!cast<Instruction>(subop # "_2d2s"), neon_uimm2_bare,
7565                      !cast<PatFrag>(op # "_2d"), VPR128, VPR64, VPR128,
7566                      v2i64, v2i32, v4i32>;
7567
7568   def : NI_2VEL2_laneq<!cast<Instruction>(subop # "_4s8h"), neon_uimm3_bare,
7569                        !cast<PatFrag>(op # "_4s"), VPR128Lo,
7570                        v4i32, v8i16, v8i16, v4i16, Neon_High8H>;
7571
7572   def : NI_2VEL2_laneq<!cast<Instruction>(subop # "_2d4s"), neon_uimm2_bare,
7573                        !cast<PatFrag>(op # "_2d"), VPR128,
7574                        v2i64, v4i32, v4i32, v2i32, Neon_High4S>;
7575
7576   def : NI_2VEL2_lane0<!cast<Instruction>(subop # "_4s8h"),
7577                        !cast<PatFrag>(op # "_4s"),
7578                        v4i32, v8i16, v4i16, Neon_High8H, DUP8h>;
7579
7580   def : NI_2VEL2_lane0<!cast<Instruction>(subop # "_2d4s"),
7581                        !cast<PatFrag>(op # "_2d"),
7582                        v2i64, v4i32, v2i32, Neon_High4S, DUP4s>;
7583
7584   // Index can only be half of the max value for lane in 64-bit vector
7585
7586   def : NI_2VE_lane<!cast<Instruction>(subop # "_4s4h"), neon_uimm2_bare,
7587                     !cast<PatFrag>(op # "_4s"), VPR128, VPR64, VPR64Lo,
7588                     v4i32, v4i16, v4i16>;
7589
7590   def : NI_2VE_lane<!cast<Instruction>(subop # "_2d2s"), neon_uimm1_bare,
7591                     !cast<PatFrag>(op # "_2d"), VPR128, VPR64, VPR64,
7592                     v2i64, v2i32, v2i32>;
7593
7594   def : NI_2VEL2_lane<!cast<Instruction>(subop # "_4s8h"), neon_uimm2_bare,
7595                       !cast<PatFrag>(op # "_4s"), VPR64Lo,
7596                       v4i32, v8i16, v4i16, v4i16, Neon_High8H>;
7597
7598   def : NI_2VEL2_lane<!cast<Instruction>(subop # "_2d4s"), neon_uimm1_bare,
7599                       !cast<PatFrag>(op # "_2d"), VPR64,
7600                       v2i64, v4i32, v2i32, v2i32, Neon_High4S>;
7601 }
7602
7603 defm SQDMLAL_lane_v3 : NI_2VEL_v3_qdma_pat<"SQDMLALvve", "Neon_qdmlal">;
7604 defm SQDMLSL_lane_v3 : NI_2VEL_v3_qdma_pat<"SQDMLSLvve", "Neon_qdmlsl">;
7605
7606 // End of implementation for instruction class (3V Elem)
7607
7608 class NeonI_REV<string asmop, string Res, bits<2> size, bit Q, bit U,
7609                 bits<5> opcode, RegisterOperand ResVPR, ValueType ResTy,
7610                 SDPatternOperator Neon_Rev>
7611   : NeonI_2VMisc<Q, U, size, opcode,
7612                (outs ResVPR:$Rd), (ins ResVPR:$Rn),
7613                asmop # "\t$Rd." # Res # ", $Rn." # Res,
7614                [(set (ResTy ResVPR:$Rd),
7615                   (ResTy (Neon_Rev (ResTy ResVPR:$Rn))))],
7616                NoItinerary> ;
7617
7618 def REV64_16b : NeonI_REV<"rev64", "16b", 0b00, 0b1, 0b0, 0b00000, VPR128,
7619                           v16i8, Neon_rev64>;
7620 def REV64_8h : NeonI_REV<"rev64", "8h", 0b01, 0b1, 0b0, 0b00000, VPR128,
7621                          v8i16, Neon_rev64>;
7622 def REV64_4s : NeonI_REV<"rev64", "4s", 0b10, 0b1, 0b0, 0b00000, VPR128,
7623                          v4i32, Neon_rev64>;
7624 def REV64_8b : NeonI_REV<"rev64", "8b", 0b00, 0b0, 0b0, 0b00000, VPR64,
7625                          v8i8, Neon_rev64>;
7626 def REV64_4h : NeonI_REV<"rev64", "4h", 0b01, 0b0, 0b0, 0b00000, VPR64,
7627                          v4i16, Neon_rev64>;
7628 def REV64_2s : NeonI_REV<"rev64", "2s", 0b10, 0b0, 0b0, 0b00000, VPR64,
7629                          v2i32, Neon_rev64>;
7630
7631 def : Pat<(v4f32 (Neon_rev64 (v4f32 VPR128:$Rn))), (REV64_4s VPR128:$Rn)>;
7632 def : Pat<(v2f32 (Neon_rev64 (v2f32 VPR64:$Rn))), (REV64_2s VPR64:$Rn)>;
7633
7634 def REV32_16b : NeonI_REV<"rev32", "16b", 0b00, 0b1, 0b1, 0b00000, VPR128,
7635                           v16i8, Neon_rev32>;
7636 def REV32_8h : NeonI_REV<"rev32", "8h", 0b01, 0b1, 0b1, 0b00000, VPR128,
7637                           v8i16, Neon_rev32>;
7638 def REV32_8b : NeonI_REV<"rev32", "8b", 0b00, 0b0, 0b1, 0b00000, VPR64,
7639                          v8i8, Neon_rev32>;
7640 def REV32_4h : NeonI_REV<"rev32", "4h", 0b01, 0b0, 0b1, 0b00000, VPR64,
7641                          v4i16, Neon_rev32>;
7642
7643 def REV16_16b : NeonI_REV<"rev16", "16b", 0b00, 0b1, 0b0, 0b00001, VPR128,
7644                           v16i8, Neon_rev16>;
7645 def REV16_8b : NeonI_REV<"rev16", "8b", 0b00, 0b0, 0b0, 0b00001, VPR64,
7646                          v8i8, Neon_rev16>;
7647
7648 multiclass NeonI_PairwiseAdd<string asmop, bit U, bits<5> opcode,
7649                              SDPatternOperator Neon_Padd> {
7650   def 16b8h : NeonI_2VMisc<0b1, U, 0b00, opcode,
7651                            (outs VPR128:$Rd), (ins VPR128:$Rn),
7652                            asmop # "\t$Rd.8h, $Rn.16b",
7653                            [(set (v8i16 VPR128:$Rd),
7654                               (v8i16 (Neon_Padd (v16i8 VPR128:$Rn))))],
7655                            NoItinerary>;
7656
7657   def 8b4h : NeonI_2VMisc<0b0, U, 0b00, opcode,
7658                           (outs VPR64:$Rd), (ins VPR64:$Rn),
7659                           asmop # "\t$Rd.4h, $Rn.8b",
7660                           [(set (v4i16 VPR64:$Rd),
7661                              (v4i16 (Neon_Padd (v8i8 VPR64:$Rn))))],
7662                           NoItinerary>;
7663
7664   def 8h4s : NeonI_2VMisc<0b1, U, 0b01, opcode,
7665                            (outs VPR128:$Rd), (ins VPR128:$Rn),
7666                            asmop # "\t$Rd.4s, $Rn.8h",
7667                            [(set (v4i32 VPR128:$Rd),
7668                               (v4i32 (Neon_Padd (v8i16 VPR128:$Rn))))],
7669                            NoItinerary>;
7670
7671   def 4h2s : NeonI_2VMisc<0b0, U, 0b01, opcode,
7672                           (outs VPR64:$Rd), (ins VPR64:$Rn),
7673                           asmop # "\t$Rd.2s, $Rn.4h",
7674                           [(set (v2i32 VPR64:$Rd),
7675                              (v2i32 (Neon_Padd (v4i16 VPR64:$Rn))))],
7676                           NoItinerary>;
7677
7678   def 4s2d : NeonI_2VMisc<0b1, U, 0b10, opcode,
7679                            (outs VPR128:$Rd), (ins VPR128:$Rn),
7680                            asmop # "\t$Rd.2d, $Rn.4s",
7681                            [(set (v2i64 VPR128:$Rd),
7682                               (v2i64 (Neon_Padd (v4i32 VPR128:$Rn))))],
7683                            NoItinerary>;
7684
7685   def 2s1d : NeonI_2VMisc<0b0, U, 0b10, opcode,
7686                           (outs VPR64:$Rd), (ins VPR64:$Rn),
7687                           asmop # "\t$Rd.1d, $Rn.2s",
7688                           [(set (v1i64 VPR64:$Rd),
7689                              (v1i64 (Neon_Padd (v2i32 VPR64:$Rn))))],
7690                           NoItinerary>;
7691 }
7692
7693 defm SADDLP : NeonI_PairwiseAdd<"saddlp", 0b0, 0b00010,
7694                                 int_arm_neon_vpaddls>;
7695 defm UADDLP : NeonI_PairwiseAdd<"uaddlp", 0b1, 0b00010,
7696                                 int_arm_neon_vpaddlu>;
7697
7698 multiclass NeonI_PairwiseAddAcc<string asmop, bit U, bits<5> opcode,
7699                              SDPatternOperator Neon_Padd> {
7700   let Constraints = "$src = $Rd" in {
7701     def 16b8h : NeonI_2VMisc<0b1, U, 0b00, opcode,
7702                              (outs VPR128:$Rd), (ins VPR128:$src, VPR128:$Rn),
7703                              asmop # "\t$Rd.8h, $Rn.16b",
7704                              [(set (v8i16 VPR128:$Rd),
7705                                 (v8i16 (Neon_Padd
7706                                   (v8i16 VPR128:$src), (v16i8 VPR128:$Rn))))],
7707                              NoItinerary>;
7708
7709     def 8b4h : NeonI_2VMisc<0b0, U, 0b00, opcode,
7710                             (outs VPR64:$Rd), (ins VPR64:$src, VPR64:$Rn),
7711                             asmop # "\t$Rd.4h, $Rn.8b",
7712                             [(set (v4i16 VPR64:$Rd),
7713                                (v4i16 (Neon_Padd
7714                                  (v4i16 VPR64:$src), (v8i8 VPR64:$Rn))))],
7715                             NoItinerary>;
7716
7717     def 8h4s : NeonI_2VMisc<0b1, U, 0b01, opcode,
7718                             (outs VPR128:$Rd), (ins VPR128:$src, VPR128:$Rn),
7719                             asmop # "\t$Rd.4s, $Rn.8h",
7720                             [(set (v4i32 VPR128:$Rd),
7721                                (v4i32 (Neon_Padd
7722                                  (v4i32 VPR128:$src), (v8i16 VPR128:$Rn))))],
7723                             NoItinerary>;
7724
7725     def 4h2s : NeonI_2VMisc<0b0, U, 0b01, opcode,
7726                             (outs VPR64:$Rd), (ins VPR64:$src, VPR64:$Rn),
7727                             asmop # "\t$Rd.2s, $Rn.4h",
7728                             [(set (v2i32 VPR64:$Rd),
7729                                (v2i32 (Neon_Padd
7730                                  (v2i32 VPR64:$src), (v4i16 VPR64:$Rn))))],
7731                             NoItinerary>;
7732
7733     def 4s2d : NeonI_2VMisc<0b1, U, 0b10, opcode,
7734                             (outs VPR128:$Rd), (ins VPR128:$src, VPR128:$Rn),
7735                             asmop # "\t$Rd.2d, $Rn.4s",
7736                             [(set (v2i64 VPR128:$Rd),
7737                                (v2i64 (Neon_Padd
7738                                  (v2i64 VPR128:$src), (v4i32 VPR128:$Rn))))],
7739                             NoItinerary>;
7740
7741     def 2s1d : NeonI_2VMisc<0b0, U, 0b10, opcode,
7742                             (outs VPR64:$Rd), (ins VPR64:$src, VPR64:$Rn),
7743                             asmop # "\t$Rd.1d, $Rn.2s",
7744                             [(set (v1i64 VPR64:$Rd),
7745                                (v1i64 (Neon_Padd
7746                                  (v1i64 VPR64:$src), (v2i32 VPR64:$Rn))))],
7747                             NoItinerary>;
7748   }
7749 }
7750
7751 defm SADALP : NeonI_PairwiseAddAcc<"sadalp", 0b0, 0b00110,
7752                                    int_arm_neon_vpadals>;
7753 defm UADALP : NeonI_PairwiseAddAcc<"uadalp", 0b1, 0b00110,
7754                                    int_arm_neon_vpadalu>;
7755
7756 multiclass NeonI_2VMisc_BHSDsize_1Arg<string asmop, bit U, bits<5> opcode> {
7757   def 16b : NeonI_2VMisc<0b1, U, 0b00, opcode,
7758                          (outs VPR128:$Rd), (ins VPR128:$Rn),
7759                          asmop # "\t$Rd.16b, $Rn.16b",
7760                          [], NoItinerary>;
7761
7762   def 8h : NeonI_2VMisc<0b1, U, 0b01, opcode,
7763                         (outs VPR128:$Rd), (ins VPR128:$Rn),
7764                         asmop # "\t$Rd.8h, $Rn.8h",
7765                         [], NoItinerary>;
7766
7767   def 4s : NeonI_2VMisc<0b1, U, 0b10, opcode,
7768                         (outs VPR128:$Rd), (ins VPR128:$Rn),
7769                         asmop # "\t$Rd.4s, $Rn.4s",
7770                         [], NoItinerary>;
7771
7772   def 2d : NeonI_2VMisc<0b1, U, 0b11, opcode,
7773                         (outs VPR128:$Rd), (ins VPR128:$Rn),
7774                         asmop # "\t$Rd.2d, $Rn.2d",
7775                         [], NoItinerary>;
7776
7777   def 8b : NeonI_2VMisc<0b0, U, 0b00, opcode,
7778                          (outs VPR64:$Rd), (ins VPR64:$Rn),
7779                          asmop # "\t$Rd.8b, $Rn.8b",
7780                          [], NoItinerary>;
7781
7782   def 4h : NeonI_2VMisc<0b0, U, 0b01, opcode,
7783                         (outs VPR64:$Rd), (ins VPR64:$Rn),
7784                         asmop # "\t$Rd.4h, $Rn.4h",
7785                         [], NoItinerary>;
7786
7787   def 2s : NeonI_2VMisc<0b0, U, 0b10, opcode,
7788                         (outs VPR64:$Rd), (ins VPR64:$Rn),
7789                         asmop # "\t$Rd.2s, $Rn.2s",
7790                         [], NoItinerary>;
7791 }
7792
7793 defm SQABS : NeonI_2VMisc_BHSDsize_1Arg<"sqabs", 0b0, 0b00111>;
7794 defm SQNEG : NeonI_2VMisc_BHSDsize_1Arg<"sqneg", 0b1, 0b00111>;
7795 defm ABS : NeonI_2VMisc_BHSDsize_1Arg<"abs", 0b0, 0b01011>;
7796 defm NEG : NeonI_2VMisc_BHSDsize_1Arg<"neg", 0b1, 0b01011>;
7797
7798 multiclass NeonI_2VMisc_BHSD_1Arg_Pattern<string Prefix,
7799                                           SDPatternOperator Neon_Op> {
7800   def : Pat<(v16i8 (Neon_Op (v16i8 VPR128:$Rn))),
7801             (v16i8 (!cast<Instruction>(Prefix # 16b) (v16i8 VPR128:$Rn)))>;
7802
7803   def : Pat<(v8i16 (Neon_Op (v8i16 VPR128:$Rn))),
7804             (v8i16 (!cast<Instruction>(Prefix # 8h) (v8i16 VPR128:$Rn)))>;
7805
7806   def : Pat<(v4i32 (Neon_Op (v4i32 VPR128:$Rn))),
7807             (v4i32 (!cast<Instruction>(Prefix # 4s) (v4i32 VPR128:$Rn)))>;
7808
7809   def : Pat<(v2i64 (Neon_Op (v2i64 VPR128:$Rn))),
7810             (v2i64 (!cast<Instruction>(Prefix # 2d) (v2i64 VPR128:$Rn)))>;
7811
7812   def : Pat<(v8i8 (Neon_Op (v8i8 VPR64:$Rn))),
7813             (v8i8 (!cast<Instruction>(Prefix # 8b) (v8i8 VPR64:$Rn)))>;
7814
7815   def : Pat<(v4i16 (Neon_Op (v4i16 VPR64:$Rn))),
7816             (v4i16 (!cast<Instruction>(Prefix # 4h) (v4i16 VPR64:$Rn)))>;
7817
7818   def : Pat<(v2i32 (Neon_Op (v2i32 VPR64:$Rn))),
7819             (v2i32 (!cast<Instruction>(Prefix # 2s) (v2i32 VPR64:$Rn)))>;
7820 }
7821
7822 defm : NeonI_2VMisc_BHSD_1Arg_Pattern<"SQABS", int_arm_neon_vqabs>;
7823 defm : NeonI_2VMisc_BHSD_1Arg_Pattern<"SQNEG", int_arm_neon_vqneg>;
7824 defm : NeonI_2VMisc_BHSD_1Arg_Pattern<"ABS", int_arm_neon_vabs>;
7825
7826 def : Pat<(v16i8 (sub
7827             (v16i8 Neon_AllZero),
7828             (v16i8 VPR128:$Rn))),
7829           (v16i8 (NEG16b (v16i8 VPR128:$Rn)))>;
7830 def : Pat<(v8i8 (sub
7831             (v8i8 Neon_AllZero),
7832             (v8i8 VPR64:$Rn))),
7833           (v8i8 (NEG8b (v8i8 VPR64:$Rn)))>;
7834 def : Pat<(v8i16 (sub
7835             (v8i16 (bitconvert (v16i8 Neon_AllZero))),
7836             (v8i16 VPR128:$Rn))),
7837           (v8i16 (NEG8h (v8i16 VPR128:$Rn)))>;
7838 def : Pat<(v4i16 (sub
7839             (v4i16 (bitconvert (v8i8 Neon_AllZero))),
7840             (v4i16 VPR64:$Rn))),
7841           (v4i16 (NEG4h (v4i16 VPR64:$Rn)))>;
7842 def : Pat<(v4i32 (sub
7843             (v4i32 (bitconvert (v16i8 Neon_AllZero))),
7844             (v4i32 VPR128:$Rn))),
7845           (v4i32 (NEG4s (v4i32 VPR128:$Rn)))>;
7846 def : Pat<(v2i32 (sub
7847             (v2i32 (bitconvert (v8i8 Neon_AllZero))),
7848             (v2i32 VPR64:$Rn))),
7849           (v2i32 (NEG2s (v2i32 VPR64:$Rn)))>;
7850 def : Pat<(v2i64 (sub
7851             (v2i64 (bitconvert (v16i8 Neon_AllZero))),
7852             (v2i64 VPR128:$Rn))),
7853           (v2i64 (NEG2d (v2i64 VPR128:$Rn)))>;
7854
7855 multiclass NeonI_2VMisc_BHSDsize_2Args<string asmop, bit U, bits<5> opcode> {
7856   let Constraints = "$src = $Rd" in {
7857     def 16b : NeonI_2VMisc<0b1, U, 0b00, opcode,
7858                            (outs VPR128:$Rd), (ins VPR128:$src, VPR128:$Rn),
7859                            asmop # "\t$Rd.16b, $Rn.16b",
7860                            [], NoItinerary>;
7861
7862     def 8h : NeonI_2VMisc<0b1, U, 0b01, opcode,
7863                           (outs VPR128:$Rd), (ins VPR128:$src, VPR128:$Rn),
7864                           asmop # "\t$Rd.8h, $Rn.8h",
7865                           [], NoItinerary>;
7866
7867     def 4s : NeonI_2VMisc<0b1, U, 0b10, opcode,
7868                           (outs VPR128:$Rd), (ins VPR128:$src, VPR128:$Rn),
7869                           asmop # "\t$Rd.4s, $Rn.4s",
7870                           [], NoItinerary>;
7871
7872     def 2d : NeonI_2VMisc<0b1, U, 0b11, opcode,
7873                           (outs VPR128:$Rd), (ins VPR128:$src, VPR128:$Rn),
7874                           asmop # "\t$Rd.2d, $Rn.2d",
7875                           [], NoItinerary>;
7876
7877     def 8b : NeonI_2VMisc<0b0, U, 0b00, opcode,
7878                           (outs VPR64:$Rd), (ins VPR64:$src, VPR64:$Rn),
7879                           asmop # "\t$Rd.8b, $Rn.8b",
7880                           [], NoItinerary>;
7881
7882     def 4h : NeonI_2VMisc<0b0, U, 0b01, opcode,
7883                           (outs VPR64:$Rd), (ins VPR64:$src, VPR64:$Rn),
7884                           asmop # "\t$Rd.4h, $Rn.4h",
7885                           [], NoItinerary>;
7886
7887     def 2s : NeonI_2VMisc<0b0, U, 0b10, opcode,
7888                           (outs VPR64:$Rd), (ins VPR64:$src, VPR64:$Rn),
7889                           asmop # "\t$Rd.2s, $Rn.2s",
7890                           [], NoItinerary>;
7891   }
7892 }
7893
7894 defm SUQADD : NeonI_2VMisc_BHSDsize_2Args<"suqadd", 0b0, 0b00011>;
7895 defm USQADD : NeonI_2VMisc_BHSDsize_2Args<"usqadd", 0b1, 0b00011>;
7896
7897 multiclass NeonI_2VMisc_BHSD_2Args_Pattern<string Prefix,
7898                                            SDPatternOperator Neon_Op> {
7899   def : Pat<(v16i8 (Neon_Op (v16i8 VPR128:$src), (v16i8 VPR128:$Rn))),
7900             (v16i8 (!cast<Instruction>(Prefix # 16b)
7901               (v16i8 VPR128:$src), (v16i8 VPR128:$Rn)))>;
7902
7903   def : Pat<(v8i16 (Neon_Op (v8i16 VPR128:$src), (v8i16 VPR128:$Rn))),
7904             (v8i16 (!cast<Instruction>(Prefix # 8h)
7905               (v8i16 VPR128:$src), (v8i16 VPR128:$Rn)))>;
7906
7907   def : Pat<(v4i32 (Neon_Op (v4i32 VPR128:$src), (v4i32 VPR128:$Rn))),
7908             (v4i32 (!cast<Instruction>(Prefix # 4s)
7909               (v4i32 VPR128:$src), (v4i32 VPR128:$Rn)))>;
7910
7911   def : Pat<(v2i64 (Neon_Op (v2i64 VPR128:$src), (v2i64 VPR128:$Rn))),
7912             (v2i64 (!cast<Instruction>(Prefix # 2d)
7913               (v2i64 VPR128:$src), (v2i64 VPR128:$Rn)))>;
7914
7915   def : Pat<(v8i8 (Neon_Op (v8i8 VPR64:$src), (v8i8 VPR64:$Rn))),
7916             (v8i8 (!cast<Instruction>(Prefix # 8b)
7917               (v8i8 VPR64:$src), (v8i8 VPR64:$Rn)))>;
7918
7919   def : Pat<(v4i16 (Neon_Op (v4i16 VPR64:$src), (v4i16 VPR64:$Rn))),
7920             (v4i16 (!cast<Instruction>(Prefix # 4h)
7921               (v4i16 VPR64:$src), (v4i16 VPR64:$Rn)))>;
7922
7923   def : Pat<(v2i32 (Neon_Op (v2i32 VPR64:$src), (v2i32 VPR64:$Rn))),
7924             (v2i32 (!cast<Instruction>(Prefix # 2s)
7925               (v2i32 VPR64:$src), (v2i32 VPR64:$Rn)))>;
7926 }
7927
7928 defm : NeonI_2VMisc_BHSD_2Args_Pattern<"SUQADD", int_aarch64_neon_suqadd>;
7929 defm : NeonI_2VMisc_BHSD_2Args_Pattern<"USQADD", int_aarch64_neon_usqadd>;
7930
7931 multiclass NeonI_2VMisc_BHSsizes<string asmop, bit U,
7932                           SDPatternOperator Neon_Op> {
7933   def 16b : NeonI_2VMisc<0b1, U, 0b00, 0b00100,
7934                          (outs VPR128:$Rd), (ins VPR128:$Rn),
7935                          asmop # "\t$Rd.16b, $Rn.16b",
7936                          [(set (v16i8 VPR128:$Rd),
7937                             (v16i8 (Neon_Op (v16i8 VPR128:$Rn))))],
7938                          NoItinerary>;
7939
7940   def 8h : NeonI_2VMisc<0b1, U, 0b01, 0b00100,
7941                         (outs VPR128:$Rd), (ins VPR128:$Rn),
7942                         asmop # "\t$Rd.8h, $Rn.8h",
7943                         [(set (v8i16 VPR128:$Rd),
7944                            (v8i16 (Neon_Op (v8i16 VPR128:$Rn))))],
7945                         NoItinerary>;
7946
7947   def 4s : NeonI_2VMisc<0b1, U, 0b10, 0b00100,
7948                         (outs VPR128:$Rd), (ins VPR128:$Rn),
7949                         asmop # "\t$Rd.4s, $Rn.4s",
7950                         [(set (v4i32 VPR128:$Rd),
7951                            (v4i32 (Neon_Op (v4i32 VPR128:$Rn))))],
7952                         NoItinerary>;
7953
7954   def 8b : NeonI_2VMisc<0b0, U, 0b00, 0b00100,
7955                         (outs VPR64:$Rd), (ins VPR64:$Rn),
7956                         asmop # "\t$Rd.8b, $Rn.8b",
7957                         [(set (v8i8 VPR64:$Rd),
7958                            (v8i8 (Neon_Op (v8i8 VPR64:$Rn))))],
7959                         NoItinerary>;
7960
7961   def 4h : NeonI_2VMisc<0b0, U, 0b01, 0b00100,
7962                         (outs VPR64:$Rd), (ins VPR64:$Rn),
7963                         asmop # "\t$Rd.4h, $Rn.4h",
7964                         [(set (v4i16 VPR64:$Rd),
7965                            (v4i16 (Neon_Op (v4i16 VPR64:$Rn))))],
7966                         NoItinerary>;
7967
7968   def 2s : NeonI_2VMisc<0b0, U, 0b10, 0b00100,
7969                         (outs VPR64:$Rd), (ins VPR64:$Rn),
7970                         asmop # "\t$Rd.2s, $Rn.2s",
7971                         [(set (v2i32 VPR64:$Rd),
7972                            (v2i32 (Neon_Op (v2i32 VPR64:$Rn))))],
7973                         NoItinerary>;
7974 }
7975
7976 defm CLS : NeonI_2VMisc_BHSsizes<"cls", 0b0, int_arm_neon_vcls>;
7977 defm CLZ : NeonI_2VMisc_BHSsizes<"clz", 0b1, ctlz>;
7978
7979 multiclass NeonI_2VMisc_Bsize<string asmop, bit U, bits<2> size,
7980                               bits<5> Opcode> {
7981   def 16b : NeonI_2VMisc<0b1, U, size, Opcode,
7982                          (outs VPR128:$Rd), (ins VPR128:$Rn),
7983                          asmop # "\t$Rd.16b, $Rn.16b",
7984                          [], NoItinerary>;
7985
7986   def 8b : NeonI_2VMisc<0b0, U, size, Opcode,
7987                         (outs VPR64:$Rd), (ins VPR64:$Rn),
7988                         asmop # "\t$Rd.8b, $Rn.8b",
7989                         [], NoItinerary>;
7990 }
7991
7992 defm CNT : NeonI_2VMisc_Bsize<"cnt", 0b0, 0b00, 0b00101>;
7993 defm NOT : NeonI_2VMisc_Bsize<"not", 0b1, 0b00, 0b00101>;
7994 defm RBIT : NeonI_2VMisc_Bsize<"rbit", 0b1, 0b01, 0b00101>;
7995
7996 def : NeonInstAlias<"mvn $Rd.16b, $Rn.16b",
7997                     (NOT16b VPR128:$Rd, VPR128:$Rn), 0>;
7998 def : NeonInstAlias<"mvn $Rd.8b, $Rn.8b",
7999                     (NOT8b VPR64:$Rd, VPR64:$Rn), 0>;
8000
8001 def : Pat<(v16i8 (ctpop (v16i8 VPR128:$Rn))),
8002           (v16i8 (CNT16b (v16i8 VPR128:$Rn)))>;
8003 def : Pat<(v8i8 (ctpop (v8i8 VPR64:$Rn))),
8004           (v8i8 (CNT8b (v8i8 VPR64:$Rn)))>;
8005
8006 def : Pat<(v16i8 (xor
8007             (v16i8 VPR128:$Rn),
8008             (v16i8 Neon_AllOne))),
8009           (v16i8 (NOT16b (v16i8 VPR128:$Rn)))>;
8010 def : Pat<(v8i8 (xor
8011             (v8i8 VPR64:$Rn),
8012             (v8i8 Neon_AllOne))),
8013           (v8i8 (NOT8b (v8i8 VPR64:$Rn)))>;
8014 def : Pat<(v8i16 (xor
8015             (v8i16 VPR128:$Rn),
8016             (v8i16 (bitconvert (v16i8 Neon_AllOne))))),
8017           (NOT16b VPR128:$Rn)>;
8018 def : Pat<(v4i16 (xor
8019             (v4i16 VPR64:$Rn),
8020             (v4i16 (bitconvert (v8i8 Neon_AllOne))))),
8021           (NOT8b VPR64:$Rn)>;
8022 def : Pat<(v4i32 (xor
8023             (v4i32 VPR128:$Rn),
8024             (v4i32 (bitconvert (v16i8 Neon_AllOne))))),
8025           (NOT16b VPR128:$Rn)>;
8026 def : Pat<(v2i32 (xor
8027             (v2i32 VPR64:$Rn),
8028             (v2i32 (bitconvert (v8i8 Neon_AllOne))))),
8029           (NOT8b VPR64:$Rn)>;
8030 def : Pat<(v2i64 (xor
8031             (v2i64 VPR128:$Rn),
8032             (v2i64 (bitconvert (v16i8 Neon_AllOne))))),
8033           (NOT16b VPR128:$Rn)>;
8034
8035 def : Pat<(v16i8 (int_aarch64_neon_rbit (v16i8 VPR128:$Rn))),
8036           (v16i8 (RBIT16b (v16i8 VPR128:$Rn)))>;
8037 def : Pat<(v8i8 (int_aarch64_neon_rbit (v8i8 VPR64:$Rn))),
8038           (v8i8 (RBIT8b (v8i8 VPR64:$Rn)))>;
8039
8040 multiclass NeonI_2VMisc_SDsizes<string asmop, bit U, bits<5> opcode,
8041                                 SDPatternOperator Neon_Op> {
8042   def 4s : NeonI_2VMisc<0b1, U, 0b10, opcode,
8043                         (outs VPR128:$Rd), (ins VPR128:$Rn),
8044                         asmop # "\t$Rd.4s, $Rn.4s",
8045                         [(set (v4f32 VPR128:$Rd),
8046                            (v4f32 (Neon_Op (v4f32 VPR128:$Rn))))],
8047                         NoItinerary>;
8048
8049   def 2d : NeonI_2VMisc<0b1, U, 0b11, opcode,
8050                         (outs VPR128:$Rd), (ins VPR128:$Rn),
8051                         asmop # "\t$Rd.2d, $Rn.2d",
8052                         [(set (v2f64 VPR128:$Rd),
8053                            (v2f64 (Neon_Op (v2f64 VPR128:$Rn))))],
8054                         NoItinerary>;
8055
8056   def 2s : NeonI_2VMisc<0b0, U, 0b10, opcode,
8057                         (outs VPR64:$Rd), (ins VPR64:$Rn),
8058                         asmop # "\t$Rd.2s, $Rn.2s",
8059                         [(set (v2f32 VPR64:$Rd),
8060                            (v2f32 (Neon_Op (v2f32 VPR64:$Rn))))],
8061                         NoItinerary>;
8062 }
8063
8064 defm FABS : NeonI_2VMisc_SDsizes<"fabs", 0b0, 0b01111, fabs>;
8065 defm FNEG : NeonI_2VMisc_SDsizes<"fneg", 0b1, 0b01111, fneg>;
8066
8067 multiclass NeonI_2VMisc_HSD_Narrow<string asmop, bit U, bits<5> opcode> {
8068   def 8h8b : NeonI_2VMisc<0b0, U, 0b00, opcode,
8069                           (outs VPR64:$Rd), (ins VPR128:$Rn),
8070                           asmop # "\t$Rd.8b, $Rn.8h",
8071                           [], NoItinerary>;
8072
8073   def 4s4h : NeonI_2VMisc<0b0, U, 0b01, opcode,
8074                           (outs VPR64:$Rd), (ins VPR128:$Rn),
8075                           asmop # "\t$Rd.4h, $Rn.4s",
8076                           [], NoItinerary>;
8077
8078   def 2d2s : NeonI_2VMisc<0b0, U, 0b10, opcode,
8079                           (outs VPR64:$Rd), (ins VPR128:$Rn),
8080                           asmop # "\t$Rd.2s, $Rn.2d",
8081                           [], NoItinerary>;
8082
8083   let Constraints = "$Rd = $src" in {
8084     def 8h16b : NeonI_2VMisc<0b1, U, 0b00, opcode,
8085                              (outs VPR128:$Rd), (ins VPR128:$src, VPR128:$Rn),
8086                              asmop # "2\t$Rd.16b, $Rn.8h",
8087                              [], NoItinerary>;
8088
8089     def 4s8h : NeonI_2VMisc<0b1, U, 0b01, opcode,
8090                             (outs VPR128:$Rd), (ins VPR128:$src, VPR128:$Rn),
8091                             asmop # "2\t$Rd.8h, $Rn.4s",
8092                             [], NoItinerary>;
8093
8094     def 2d4s : NeonI_2VMisc<0b1, U, 0b10, opcode,
8095                             (outs VPR128:$Rd), (ins VPR128:$src, VPR128:$Rn),
8096                             asmop # "2\t$Rd.4s, $Rn.2d",
8097                             [], NoItinerary>;
8098   }
8099 }
8100
8101 defm XTN : NeonI_2VMisc_HSD_Narrow<"xtn", 0b0, 0b10010>;
8102 defm SQXTUN : NeonI_2VMisc_HSD_Narrow<"sqxtun", 0b1, 0b10010>;
8103 defm SQXTN : NeonI_2VMisc_HSD_Narrow<"sqxtn", 0b0, 0b10100>;
8104 defm UQXTN : NeonI_2VMisc_HSD_Narrow<"uqxtn", 0b1, 0b10100>;
8105
8106 multiclass NeonI_2VMisc_Narrow_Patterns<string Prefix,
8107                                         SDPatternOperator Neon_Op> {
8108   def : Pat<(v8i8 (Neon_Op (v8i16 VPR128:$Rn))),
8109             (v8i8 (!cast<Instruction>(Prefix # 8h8b) (v8i16 VPR128:$Rn)))>;
8110
8111   def : Pat<(v4i16 (Neon_Op (v4i32 VPR128:$Rn))),
8112             (v4i16 (!cast<Instruction>(Prefix # 4s4h) (v4i32 VPR128:$Rn)))>;
8113
8114   def : Pat<(v2i32 (Neon_Op (v2i64 VPR128:$Rn))),
8115             (v2i32 (!cast<Instruction>(Prefix # 2d2s) (v2i64 VPR128:$Rn)))>;
8116
8117   def : Pat<(v16i8 (concat_vectors
8118               (v8i8 VPR64:$src),
8119               (v8i8 (Neon_Op (v8i16 VPR128:$Rn))))),
8120             (!cast<Instruction>(Prefix # 8h16b)
8121               (SUBREG_TO_REG (i32 0), VPR64:$src, sub_64),
8122               VPR128:$Rn)>;
8123
8124   def : Pat<(v8i16 (concat_vectors
8125               (v4i16 VPR64:$src),
8126               (v4i16 (Neon_Op (v4i32 VPR128:$Rn))))),
8127             (!cast<Instruction>(Prefix # 4s8h)
8128               (SUBREG_TO_REG (i32 0), VPR64:$src, sub_64),
8129               VPR128:$Rn)>;
8130
8131   def : Pat<(v4i32 (concat_vectors
8132               (v2i32 VPR64:$src),
8133               (v2i32 (Neon_Op (v2i64 VPR128:$Rn))))),
8134             (!cast<Instruction>(Prefix # 2d4s)
8135               (SUBREG_TO_REG (i32 0), VPR64:$src, sub_64),
8136               VPR128:$Rn)>;
8137 }
8138
8139 defm : NeonI_2VMisc_Narrow_Patterns<"XTN", trunc>;
8140 defm : NeonI_2VMisc_Narrow_Patterns<"SQXTUN", int_arm_neon_vqmovnsu>;
8141 defm : NeonI_2VMisc_Narrow_Patterns<"SQXTN", int_arm_neon_vqmovns>;
8142 defm : NeonI_2VMisc_Narrow_Patterns<"UQXTN", int_arm_neon_vqmovnu>;
8143
8144 multiclass NeonI_2VMisc_SHIFT<string asmop, bit U, bits<5> opcode> {
8145   let DecoderMethod = "DecodeSHLLInstruction" in {
8146     def 8b8h : NeonI_2VMisc<0b0, U, 0b00, opcode,
8147                             (outs VPR128:$Rd),
8148                             (ins VPR64:$Rn, uimm_exact8:$Imm),
8149                             asmop # "\t$Rd.8h, $Rn.8b, $Imm",
8150                             [], NoItinerary>;
8151
8152     def 4h4s : NeonI_2VMisc<0b0, U, 0b01, opcode,
8153                             (outs VPR128:$Rd),
8154                             (ins VPR64:$Rn, uimm_exact16:$Imm),
8155                             asmop # "\t$Rd.4s, $Rn.4h, $Imm",
8156                             [], NoItinerary>;
8157
8158     def 2s2d : NeonI_2VMisc<0b0, U, 0b10, opcode,
8159                             (outs VPR128:$Rd),
8160                             (ins VPR64:$Rn, uimm_exact32:$Imm),
8161                             asmop # "\t$Rd.2d, $Rn.2s, $Imm",
8162                             [], NoItinerary>;
8163
8164     def 16b8h : NeonI_2VMisc<0b1, U, 0b00, opcode,
8165                             (outs VPR128:$Rd),
8166                             (ins VPR128:$Rn, uimm_exact8:$Imm),
8167                             asmop # "2\t$Rd.8h, $Rn.16b, $Imm",
8168                             [], NoItinerary>;
8169
8170     def 8h4s : NeonI_2VMisc<0b1, U, 0b01, opcode,
8171                             (outs VPR128:$Rd),
8172                             (ins VPR128:$Rn, uimm_exact16:$Imm),
8173                             asmop # "2\t$Rd.4s, $Rn.8h, $Imm",
8174                             [], NoItinerary>;
8175
8176     def 4s2d : NeonI_2VMisc<0b1, U, 0b10, opcode,
8177                             (outs VPR128:$Rd),
8178                             (ins VPR128:$Rn, uimm_exact32:$Imm),
8179                             asmop # "2\t$Rd.2d, $Rn.4s, $Imm",
8180                             [], NoItinerary>;
8181   }
8182 }
8183
8184 defm SHLL : NeonI_2VMisc_SHIFT<"shll", 0b1, 0b10011>;
8185
8186 class NeonI_SHLL_Patterns<ValueType OpTy, ValueType DesTy,
8187                           SDPatternOperator ExtOp, Operand Neon_Imm,
8188                           string suffix>
8189   : Pat<(DesTy (shl
8190           (DesTy (ExtOp (OpTy VPR64:$Rn))),
8191             (DesTy (Neon_vdup
8192               (i32 Neon_Imm:$Imm))))),
8193         (!cast<Instruction>("SHLL" # suffix) VPR64:$Rn, Neon_Imm:$Imm)>;
8194
8195 class NeonI_SHLL_High_Patterns<ValueType OpTy, ValueType DesTy,
8196                                SDPatternOperator ExtOp, Operand Neon_Imm,
8197                                string suffix, PatFrag GetHigh>
8198   : Pat<(DesTy (shl
8199           (DesTy (ExtOp
8200             (OpTy (GetHigh VPR128:$Rn)))),
8201               (DesTy (Neon_vdup
8202                 (i32 Neon_Imm:$Imm))))),
8203         (!cast<Instruction>("SHLL" # suffix) VPR128:$Rn, Neon_Imm:$Imm)>;
8204
8205 def : NeonI_SHLL_Patterns<v8i8, v8i16, zext, uimm_exact8, "8b8h">;
8206 def : NeonI_SHLL_Patterns<v8i8, v8i16, sext, uimm_exact8, "8b8h">;
8207 def : NeonI_SHLL_Patterns<v4i16, v4i32, zext, uimm_exact16, "4h4s">;
8208 def : NeonI_SHLL_Patterns<v4i16, v4i32, sext, uimm_exact16, "4h4s">;
8209 def : NeonI_SHLL_Patterns<v2i32, v2i64, zext, uimm_exact32, "2s2d">;
8210 def : NeonI_SHLL_Patterns<v2i32, v2i64, sext, uimm_exact32, "2s2d">;
8211 def : NeonI_SHLL_High_Patterns<v8i8, v8i16, zext, uimm_exact8, "16b8h",
8212                                Neon_High16B>;
8213 def : NeonI_SHLL_High_Patterns<v8i8, v8i16, sext, uimm_exact8, "16b8h",
8214                                Neon_High16B>;
8215 def : NeonI_SHLL_High_Patterns<v4i16, v4i32, zext, uimm_exact16, "8h4s",
8216                                Neon_High8H>;
8217 def : NeonI_SHLL_High_Patterns<v4i16, v4i32, sext, uimm_exact16, "8h4s",
8218                                Neon_High8H>;
8219 def : NeonI_SHLL_High_Patterns<v2i32, v2i64, zext, uimm_exact32, "4s2d",
8220                                Neon_High4S>;
8221 def : NeonI_SHLL_High_Patterns<v2i32, v2i64, sext, uimm_exact32, "4s2d",
8222                                Neon_High4S>;
8223
8224 multiclass NeonI_2VMisc_SD_Narrow<string asmop, bit U, bits<5> opcode> {
8225   def 4s4h : NeonI_2VMisc<0b0, U, 0b00, opcode,
8226                           (outs VPR64:$Rd), (ins VPR128:$Rn),
8227                           asmop # "\t$Rd.4h, $Rn.4s",
8228                           [], NoItinerary>;
8229
8230   def 2d2s : NeonI_2VMisc<0b0, U, 0b01, opcode,
8231                           (outs VPR64:$Rd), (ins VPR128:$Rn),
8232                           asmop # "\t$Rd.2s, $Rn.2d",
8233                           [], NoItinerary>;
8234
8235   let Constraints = "$src = $Rd" in {
8236     def 4s8h : NeonI_2VMisc<0b1, U, 0b00, opcode,
8237                             (outs VPR128:$Rd), (ins VPR128:$src, VPR128:$Rn),
8238                             asmop # "2\t$Rd.8h, $Rn.4s",
8239                             [], NoItinerary>;
8240
8241     def 2d4s : NeonI_2VMisc<0b1, U, 0b01, opcode,
8242                             (outs VPR128:$Rd), (ins VPR128:$src, VPR128:$Rn),
8243                             asmop # "2\t$Rd.4s, $Rn.2d",
8244                             [], NoItinerary>;
8245   }
8246 }
8247
8248 defm FCVTN : NeonI_2VMisc_SD_Narrow<"fcvtn", 0b0, 0b10110>;
8249
8250 multiclass NeonI_2VMisc_Narrow_Pattern<string prefix,
8251                                        SDPatternOperator f32_to_f16_Op,
8252                                        SDPatternOperator f64_to_f32_Op> {
8253
8254   def : Pat<(v4i16 (f32_to_f16_Op (v4f32 VPR128:$Rn))),
8255               (!cast<Instruction>(prefix # "4s4h") (v4f32 VPR128:$Rn))>;
8256
8257   def : Pat<(v8i16 (concat_vectors
8258                 (v4i16 VPR64:$src),
8259                 (v4i16 (f32_to_f16_Op (v4f32 VPR128:$Rn))))),
8260                   (!cast<Instruction>(prefix # "4s8h")
8261                     (v4f32 (SUBREG_TO_REG (i32 0), VPR64:$src, sub_64)),
8262                     (v4f32 VPR128:$Rn))>;
8263
8264   def : Pat<(v2f32 (f64_to_f32_Op (v2f64 VPR128:$Rn))),
8265             (!cast<Instruction>(prefix # "2d2s") (v2f64 VPR128:$Rn))>;
8266
8267   def : Pat<(v4f32 (concat_vectors
8268               (v2f32 VPR64:$src),
8269               (v2f32 (f64_to_f32_Op (v2f64 VPR128:$Rn))))),
8270                 (!cast<Instruction>(prefix # "2d4s")
8271                   (v4f32 (SUBREG_TO_REG (i32 0), VPR64:$src, sub_64)),
8272                   (v2f64 VPR128:$Rn))>;
8273 }
8274
8275 defm : NeonI_2VMisc_Narrow_Pattern<"FCVTN", int_arm_neon_vcvtfp2hf, fround>;
8276
8277 multiclass NeonI_2VMisc_D_Narrow<string asmop, string prefix, bit U,
8278                                  bits<5> opcode> {
8279   def 2d2s : NeonI_2VMisc<0b0, U, 0b01, opcode,
8280                           (outs VPR64:$Rd), (ins VPR128:$Rn),
8281                           asmop # "\t$Rd.2s, $Rn.2d",
8282                           [], NoItinerary>;
8283
8284   def 2d4s : NeonI_2VMisc<0b1, U, 0b01, opcode,
8285                           (outs VPR128:$Rd), (ins VPR128:$src, VPR128:$Rn),
8286                           asmop # "2\t$Rd.4s, $Rn.2d",
8287                           [], NoItinerary> {
8288     let Constraints = "$src = $Rd";
8289   }
8290
8291   def : Pat<(v2f32 (int_aarch64_neon_fcvtxn (v2f64 VPR128:$Rn))),
8292             (!cast<Instruction>(prefix # "2d2s") VPR128:$Rn)>;
8293
8294   def : Pat<(v4f32 (concat_vectors
8295               (v2f32 VPR64:$src),
8296               (v2f32 (int_aarch64_neon_fcvtxn (v2f64 VPR128:$Rn))))),
8297             (!cast<Instruction>(prefix # "2d4s")
8298                (v4f32 (SUBREG_TO_REG (i32 0), VPR64:$src, sub_64)),
8299                VPR128:$Rn)>;
8300 }
8301
8302 defm FCVTXN : NeonI_2VMisc_D_Narrow<"fcvtxn","FCVTXN", 0b1, 0b10110>;
8303
8304 def Neon_High4Float : PatFrag<(ops node:$in),
8305                               (extract_subvector (v4f32 node:$in), (iPTR 2))>;
8306
8307 multiclass NeonI_2VMisc_HS_Extend<string asmop, bit U, bits<5> opcode> {
8308   def 4h4s : NeonI_2VMisc<0b0, U, 0b00, opcode,
8309                           (outs VPR128:$Rd), (ins VPR64:$Rn),
8310                           asmop # "\t$Rd.4s, $Rn.4h",
8311                           [], NoItinerary>;
8312
8313   def 2s2d : NeonI_2VMisc<0b0, U, 0b01, opcode,
8314                           (outs VPR128:$Rd), (ins VPR64:$Rn),
8315                           asmop # "\t$Rd.2d, $Rn.2s",
8316                           [], NoItinerary>;
8317
8318   def 8h4s : NeonI_2VMisc<0b1, U, 0b00, opcode,
8319                           (outs VPR128:$Rd), (ins VPR128:$Rn),
8320                           asmop # "2\t$Rd.4s, $Rn.8h",
8321                           [], NoItinerary>;
8322
8323   def 4s2d : NeonI_2VMisc<0b1, U, 0b01, opcode,
8324                           (outs VPR128:$Rd), (ins VPR128:$Rn),
8325                           asmop # "2\t$Rd.2d, $Rn.4s",
8326                           [], NoItinerary>;
8327 }
8328
8329 defm FCVTL : NeonI_2VMisc_HS_Extend<"fcvtl", 0b0, 0b10111>;
8330
8331 multiclass NeonI_2VMisc_Extend_Pattern<string prefix> {
8332   def : Pat<(v4f32 (int_arm_neon_vcvthf2fp (v4i16 VPR64:$Rn))),
8333             (!cast<Instruction>(prefix # "4h4s") VPR64:$Rn)>;
8334
8335   def : Pat<(v4f32 (int_arm_neon_vcvthf2fp
8336               (v4i16 (Neon_High8H
8337                 (v8i16 VPR128:$Rn))))),
8338             (!cast<Instruction>(prefix # "8h4s") VPR128:$Rn)>;
8339
8340   def : Pat<(v2f64 (fextend (v2f32 VPR64:$Rn))),
8341             (!cast<Instruction>(prefix # "2s2d") VPR64:$Rn)>;
8342
8343   def : Pat<(v2f64 (fextend
8344               (v2f32 (Neon_High4Float
8345                 (v4f32 VPR128:$Rn))))),
8346             (!cast<Instruction>(prefix # "4s2d") VPR128:$Rn)>;
8347 }
8348
8349 defm : NeonI_2VMisc_Extend_Pattern<"FCVTL">;
8350
8351 multiclass NeonI_2VMisc_SD_Conv<string asmop, bit Size, bit U, bits<5> opcode,
8352                                 ValueType ResTy4s, ValueType OpTy4s,
8353                                 ValueType ResTy2d, ValueType OpTy2d,
8354                                 ValueType ResTy2s, ValueType OpTy2s,
8355                                 SDPatternOperator Neon_Op> {
8356
8357   def 4s : NeonI_2VMisc<0b1, U, {Size, 0b0}, opcode,
8358                         (outs VPR128:$Rd), (ins VPR128:$Rn),
8359                         asmop # "\t$Rd.4s, $Rn.4s",
8360                         [(set (ResTy4s VPR128:$Rd),
8361                            (ResTy4s (Neon_Op (OpTy4s VPR128:$Rn))))],
8362                         NoItinerary>;
8363
8364   def 2d : NeonI_2VMisc<0b1, U, {Size, 0b1}, opcode,
8365                         (outs VPR128:$Rd), (ins VPR128:$Rn),
8366                         asmop # "\t$Rd.2d, $Rn.2d",
8367                         [(set (ResTy2d VPR128:$Rd),
8368                            (ResTy2d (Neon_Op (OpTy2d VPR128:$Rn))))],
8369                         NoItinerary>;
8370
8371   def 2s : NeonI_2VMisc<0b0, U, {Size, 0b0}, opcode,
8372                         (outs VPR64:$Rd), (ins VPR64:$Rn),
8373                         asmop # "\t$Rd.2s, $Rn.2s",
8374                         [(set (ResTy2s VPR64:$Rd),
8375                            (ResTy2s (Neon_Op (OpTy2s VPR64:$Rn))))],
8376                         NoItinerary>;
8377 }
8378
8379 multiclass NeonI_2VMisc_fp_to_int<string asmop, bit Size, bit U,
8380                                   bits<5> opcode, SDPatternOperator Neon_Op> {
8381   defm _ : NeonI_2VMisc_SD_Conv<asmop, Size, U, opcode, v4i32, v4f32, v2i64,
8382                                 v2f64, v2i32, v2f32, Neon_Op>;
8383 }
8384
8385 defm FCVTNS : NeonI_2VMisc_fp_to_int<"fcvtns", 0b0, 0b0, 0b11010,
8386                                      int_aarch64_neon_fcvtns>;
8387 defm FCVTNU : NeonI_2VMisc_fp_to_int<"fcvtnu", 0b0, 0b1, 0b11010,
8388                                      int_aarch64_neon_fcvtnu>;
8389 defm FCVTPS : NeonI_2VMisc_fp_to_int<"fcvtps", 0b1, 0b0, 0b11010,
8390                                      int_aarch64_neon_fcvtps>;
8391 defm FCVTPU : NeonI_2VMisc_fp_to_int<"fcvtpu", 0b1, 0b1, 0b11010,
8392                                      int_aarch64_neon_fcvtpu>;
8393 defm FCVTMS : NeonI_2VMisc_fp_to_int<"fcvtms", 0b0, 0b0, 0b11011,
8394                                      int_aarch64_neon_fcvtms>;
8395 defm FCVTMU : NeonI_2VMisc_fp_to_int<"fcvtmu", 0b0, 0b1, 0b11011,
8396                                      int_aarch64_neon_fcvtmu>;
8397 defm FCVTZS : NeonI_2VMisc_fp_to_int<"fcvtzs", 0b1, 0b0, 0b11011, fp_to_sint>;
8398 defm FCVTZU : NeonI_2VMisc_fp_to_int<"fcvtzu", 0b1, 0b1, 0b11011, fp_to_uint>;
8399 defm FCVTAS : NeonI_2VMisc_fp_to_int<"fcvtas", 0b0, 0b0, 0b11100,
8400                                      int_aarch64_neon_fcvtas>;
8401 defm FCVTAU : NeonI_2VMisc_fp_to_int<"fcvtau", 0b0, 0b1, 0b11100,
8402                                      int_aarch64_neon_fcvtau>;
8403
8404 multiclass NeonI_2VMisc_int_to_fp<string asmop, bit Size, bit U,
8405                                   bits<5> opcode, SDPatternOperator Neon_Op> {
8406   defm _ : NeonI_2VMisc_SD_Conv<asmop, Size, U, opcode, v4f32, v4i32, v2f64,
8407                                 v2i64, v2f32, v2i32, Neon_Op>;
8408 }
8409
8410 defm SCVTF : NeonI_2VMisc_int_to_fp<"scvtf", 0b0, 0b0, 0b11101, sint_to_fp>;
8411 defm UCVTF : NeonI_2VMisc_int_to_fp<"ucvtf", 0b0, 0b1, 0b11101, uint_to_fp>;
8412
8413 multiclass NeonI_2VMisc_fp_to_fp<string asmop, bit Size, bit U,
8414                                  bits<5> opcode, SDPatternOperator Neon_Op> {
8415   defm _ : NeonI_2VMisc_SD_Conv<asmop, Size, U, opcode, v4f32, v4f32, v2f64,
8416                                 v2f64, v2f32, v2f32, Neon_Op>;
8417 }
8418
8419 defm FRINTN : NeonI_2VMisc_fp_to_fp<"frintn", 0b0, 0b0, 0b11000,
8420                                      int_aarch64_neon_frintn>;
8421 defm FRINTA : NeonI_2VMisc_fp_to_fp<"frinta", 0b0, 0b1, 0b11000, frnd>;
8422 defm FRINTP : NeonI_2VMisc_fp_to_fp<"frintp", 0b1, 0b0, 0b11000, fceil>;
8423 defm FRINTM : NeonI_2VMisc_fp_to_fp<"frintm", 0b0, 0b0, 0b11001, ffloor>;
8424 defm FRINTX : NeonI_2VMisc_fp_to_fp<"frintx", 0b0, 0b1, 0b11001, frint>;
8425 defm FRINTZ : NeonI_2VMisc_fp_to_fp<"frintz", 0b1, 0b0, 0b11001, ftrunc>;
8426 defm FRINTI : NeonI_2VMisc_fp_to_fp<"frinti", 0b1, 0b1, 0b11001, fnearbyint>;
8427 defm FRECPE : NeonI_2VMisc_fp_to_fp<"frecpe", 0b1, 0b0, 0b11101,
8428                                     int_arm_neon_vrecpe>;
8429 defm FRSQRTE : NeonI_2VMisc_fp_to_fp<"frsqrte", 0b1, 0b1, 0b11101,
8430                                      int_arm_neon_vrsqrte>;
8431 defm FSQRT : NeonI_2VMisc_fp_to_fp<"fsqrt", 0b1, 0b1, 0b11111, fsqrt>;
8432
8433 multiclass NeonI_2VMisc_S_Conv<string asmop, bit Size, bit U,
8434                                bits<5> opcode, SDPatternOperator Neon_Op> {
8435   def 4s : NeonI_2VMisc<0b1, U, {Size, 0b0}, opcode,
8436                         (outs VPR128:$Rd), (ins VPR128:$Rn),
8437                         asmop # "\t$Rd.4s, $Rn.4s",
8438                         [(set (v4i32 VPR128:$Rd),
8439                            (v4i32 (Neon_Op (v4i32 VPR128:$Rn))))],
8440                         NoItinerary>;
8441
8442   def 2s : NeonI_2VMisc<0b0, U, {Size, 0b0}, opcode,
8443                         (outs VPR64:$Rd), (ins VPR64:$Rn),
8444                         asmop # "\t$Rd.2s, $Rn.2s",
8445                         [(set (v2i32 VPR64:$Rd),
8446                            (v2i32 (Neon_Op (v2i32 VPR64:$Rn))))],
8447                         NoItinerary>;
8448 }
8449
8450 defm URECPE : NeonI_2VMisc_S_Conv<"urecpe", 0b1, 0b0, 0b11100,
8451                                   int_arm_neon_vrecpe>;
8452 defm URSQRTE : NeonI_2VMisc_S_Conv<"ursqrte", 0b1, 0b1, 0b11100,
8453                                    int_arm_neon_vrsqrte>;
8454
8455 // Crypto Class
8456 class NeonI_Cryptoaes_2v<bits<2> size, bits<5> opcode,
8457                          string asmop, SDPatternOperator opnode>
8458   : NeonI_Crypto_AES<size, opcode,
8459                      (outs VPR128:$Rd), (ins VPR128:$src, VPR128:$Rn),
8460                      asmop # "\t$Rd.16b, $Rn.16b",
8461                      [(set (v16i8 VPR128:$Rd),
8462                         (v16i8 (opnode (v16i8 VPR128:$src),
8463                                        (v16i8 VPR128:$Rn))))],
8464                      NoItinerary>{
8465   let Constraints = "$src = $Rd";
8466   let Predicates = [HasNEON, HasCrypto];
8467 }
8468
8469 def AESE : NeonI_Cryptoaes_2v<0b00, 0b00100, "aese", int_arm_neon_aese>;
8470 def AESD : NeonI_Cryptoaes_2v<0b00, 0b00101, "aesd", int_arm_neon_aesd>;
8471
8472 class NeonI_Cryptoaes<bits<2> size, bits<5> opcode,
8473                       string asmop, SDPatternOperator opnode>
8474   : NeonI_Crypto_AES<size, opcode,
8475                      (outs VPR128:$Rd), (ins VPR128:$Rn),
8476                      asmop # "\t$Rd.16b, $Rn.16b",
8477                      [(set (v16i8 VPR128:$Rd),
8478                         (v16i8 (opnode (v16i8 VPR128:$Rn))))],
8479                      NoItinerary>;
8480
8481 def AESMC : NeonI_Cryptoaes<0b00, 0b00110, "aesmc", int_arm_neon_aesmc>;
8482 def AESIMC : NeonI_Cryptoaes<0b00, 0b00111, "aesimc", int_arm_neon_aesimc>;
8483
8484 class NeonI_Cryptosha_vv<bits<2> size, bits<5> opcode,
8485                          string asmop, SDPatternOperator opnode>
8486   : NeonI_Crypto_SHA<size, opcode,
8487                      (outs VPR128:$Rd), (ins VPR128:$src, VPR128:$Rn),
8488                      asmop # "\t$Rd.4s, $Rn.4s",
8489                      [(set (v4i32 VPR128:$Rd),
8490                         (v4i32 (opnode (v4i32 VPR128:$src),
8491                                        (v4i32 VPR128:$Rn))))],
8492                      NoItinerary> {
8493   let Constraints = "$src = $Rd";
8494   let Predicates = [HasNEON, HasCrypto];
8495 }
8496
8497 def SHA1SU1 : NeonI_Cryptosha_vv<0b00, 0b00001, "sha1su1",
8498                                  int_arm_neon_sha1su1>;
8499 def SHA256SU0 : NeonI_Cryptosha_vv<0b00, 0b00010, "sha256su0",
8500                                    int_arm_neon_sha256su0>;
8501
8502 class NeonI_Cryptosha_ss<bits<2> size, bits<5> opcode,
8503                          string asmop, SDPatternOperator opnode>
8504   : NeonI_Crypto_SHA<size, opcode,
8505                      (outs FPR32:$Rd), (ins FPR32:$Rn),
8506                      asmop # "\t$Rd, $Rn",
8507                      [(set (v1i32 FPR32:$Rd),
8508                         (v1i32 (opnode (v1i32 FPR32:$Rn))))],
8509                      NoItinerary> {
8510   let Predicates = [HasNEON, HasCrypto];
8511 }
8512
8513 def SHA1H : NeonI_Cryptosha_ss<0b00, 0b00000, "sha1h", int_arm_neon_sha1h>;
8514
8515 class NeonI_Cryptosha3_vvv<bits<2> size, bits<3> opcode, string asmop,
8516                            SDPatternOperator opnode>
8517   : NeonI_Crypto_3VSHA<size, opcode,
8518                        (outs VPR128:$Rd),
8519                        (ins VPR128:$src, VPR128:$Rn, VPR128:$Rm),
8520                        asmop # "\t$Rd.4s, $Rn.4s, $Rm.4s",
8521                        [(set (v4i32 VPR128:$Rd),
8522                           (v4i32 (opnode (v4i32 VPR128:$src),
8523                                          (v4i32 VPR128:$Rn),
8524                                          (v4i32 VPR128:$Rm))))],
8525                        NoItinerary> {
8526   let Constraints = "$src = $Rd";
8527   let Predicates = [HasNEON, HasCrypto];
8528 }
8529
8530 def SHA1SU0 : NeonI_Cryptosha3_vvv<0b00, 0b011, "sha1su0",
8531                                    int_arm_neon_sha1su0>;
8532 def SHA256SU1 : NeonI_Cryptosha3_vvv<0b00, 0b110, "sha256su1",
8533                                      int_arm_neon_sha256su1>;
8534
8535 class NeonI_Cryptosha3_qqv<bits<2> size, bits<3> opcode, string asmop,
8536                            SDPatternOperator opnode>
8537   : NeonI_Crypto_3VSHA<size, opcode,
8538                        (outs FPR128:$Rd),
8539                        (ins FPR128:$src, FPR128:$Rn, VPR128:$Rm),
8540                        asmop # "\t$Rd, $Rn, $Rm.4s",
8541                        [(set (v4i32 FPR128:$Rd),
8542                           (v4i32 (opnode (v4i32 FPR128:$src),
8543                                          (v4i32 FPR128:$Rn),
8544                                          (v4i32 VPR128:$Rm))))],
8545                        NoItinerary> {
8546   let Constraints = "$src = $Rd";
8547   let Predicates = [HasNEON, HasCrypto];
8548 }
8549
8550 def SHA256H : NeonI_Cryptosha3_qqv<0b00, 0b100, "sha256h",
8551                                    int_arm_neon_sha256h>;
8552 def SHA256H2 : NeonI_Cryptosha3_qqv<0b00, 0b101, "sha256h2",
8553                                     int_arm_neon_sha256h2>;
8554
8555 class NeonI_Cryptosha3_qsv<bits<2> size, bits<3> opcode, string asmop,
8556                            SDPatternOperator opnode>
8557   : NeonI_Crypto_3VSHA<size, opcode,
8558                        (outs FPR128:$Rd),
8559                        (ins FPR128:$src, FPR32:$Rn, VPR128:$Rm),
8560                        asmop # "\t$Rd, $Rn, $Rm.4s",
8561                        [(set (v4i32 FPR128:$Rd),
8562                           (v4i32 (opnode (v4i32 FPR128:$src),
8563                                          (v1i32 FPR32:$Rn),
8564                                          (v4i32 VPR128:$Rm))))],
8565                        NoItinerary> {
8566   let Constraints = "$src = $Rd";
8567   let Predicates = [HasNEON, HasCrypto];
8568 }
8569
8570 def SHA1C : NeonI_Cryptosha3_qsv<0b00, 0b000, "sha1c", int_aarch64_neon_sha1c>;
8571 def SHA1P : NeonI_Cryptosha3_qsv<0b00, 0b001, "sha1p", int_aarch64_neon_sha1p>;
8572 def SHA1M : NeonI_Cryptosha3_qsv<0b00, 0b010, "sha1m", int_aarch64_neon_sha1m>;
8573
8574 //
8575 // Patterns for handling half-precision values
8576 //
8577
8578 // Convert f16 value coming in as i16 value to f32
8579 def : Pat<(f32 (f16_to_f32 (i32 (and (i32 GPR32:$Rn), 65535)))),
8580           (FCVTsh (EXTRACT_SUBREG (FMOVsw GPR32:$Rn), sub_16))>;
8581 def : Pat<(f32 (f16_to_f32 (i32 (assertzext GPR32:$Rn)))),
8582           (FCVTsh (EXTRACT_SUBREG (FMOVsw GPR32:$Rn), sub_16))>;
8583
8584 def : Pat<(f32 (f16_to_f32 (i32 (assertzext (i32 (
8585             f32_to_f16 (f32 FPR32:$Rn))))))),
8586           (f32 FPR32:$Rn)>;
8587
8588 // Patterns for vector extract of half-precision FP value in i16 storage type
8589 def : Pat<(f32 (f16_to_f32 ( i32 (and (i32 (vector_extract
8590             (v4i16 VPR64:$Rn), neon_uimm2_bare:$Imm)), 65535)))),
8591           (FCVTsh (f16 (DUPhv_H
8592             (v8i16 (SUBREG_TO_REG (i64 0), VPR64:$Rn, sub_64)),
8593             neon_uimm2_bare:$Imm)))>;
8594
8595 def : Pat<(f32 (f16_to_f32 ( i32 (and (i32 (vector_extract
8596             (v8i16 VPR128:$Rn), neon_uimm3_bare:$Imm)), 65535)))),
8597           (FCVTsh (f16 (DUPhv_H (v8i16 VPR128:$Rn), neon_uimm3_bare:$Imm)))>;
8598
8599 // Patterns for vector insert of half-precision FP value 0 in i16 storage type
8600 def : Pat<(v8i16 (vector_insert (v8i16 VPR128:$Rn),
8601             (i32 (assertsext (i32 (fp_to_sint(f32 (f16_to_f32 (i32 0))))))),
8602             (neon_uimm3_bare:$Imm))),
8603           (v8i16 (INSELh (v8i16 VPR128:$Rn),
8604             (v8i16 (SUBREG_TO_REG (i64 0),
8605               (f16 (EXTRACT_SUBREG (f32 (FMOVsw (i32 WZR))), sub_16)),
8606               sub_16)),
8607             neon_uimm3_bare:$Imm, 0))>;
8608
8609 def : Pat<(v4i16 (vector_insert (v4i16 VPR64:$Rn),
8610             (i32 (assertsext (i32 (fp_to_sint(f32 (f16_to_f32 (i32 0))))))),
8611             (neon_uimm2_bare:$Imm))),
8612           (v4i16 (EXTRACT_SUBREG
8613             (v8i16 (INSELh
8614               (v8i16 (SUBREG_TO_REG (i64 0), VPR64:$Rn, sub_64)),
8615               (v8i16 (SUBREG_TO_REG (i64 0),
8616                 (f16 (EXTRACT_SUBREG (f32 (FMOVsw (i32 WZR))), sub_16)),
8617                 sub_16)),
8618               neon_uimm2_bare:$Imm, 0)),
8619             sub_64))>;
8620
8621 // Patterns for vector insert of half-precision FP value in i16 storage type
8622 def : Pat<(v8i16 (vector_insert (v8i16 VPR128:$Rn),
8623             (i32 (assertsext (i32 (fp_to_sint
8624               (f32 (f16_to_f32 (i32 (and (i32 GPR32:$src), 65535)))))))),
8625             (neon_uimm3_bare:$Imm))),
8626           (v8i16 (INSELh (v8i16 VPR128:$Rn),
8627             (v8i16 (SUBREG_TO_REG (i64 0),
8628               (f16 (EXTRACT_SUBREG (f32 (FMOVsw (i32 GPR32:$src))), sub_16)),
8629               sub_16)),
8630             neon_uimm3_bare:$Imm, 0))>;
8631
8632 def : Pat<(v4i16 (vector_insert (v4i16 VPR64:$Rn),
8633             (i32 (assertsext (i32 (fp_to_sint
8634               (f32 (f16_to_f32 (i32 (and (i32 GPR32:$src), 65535)))))))),
8635             (neon_uimm2_bare:$Imm))),
8636           (v4i16 (EXTRACT_SUBREG
8637             (v8i16 (INSELh
8638               (v8i16 (SUBREG_TO_REG (i64 0), VPR64:$Rn, sub_64)),
8639               (v8i16 (SUBREG_TO_REG (i64 0),
8640                 (f16 (EXTRACT_SUBREG (f32 (FMOVsw (i32 GPR32:$src))), sub_16)),
8641                 sub_16)),
8642               neon_uimm2_bare:$Imm, 0)),
8643             sub_64))>;
8644
8645 def : Pat<(v8i16 (vector_insert (v8i16 VPR128:$Rn),
8646             (i32 (vector_extract (v8i16 VPR128:$src), neon_uimm3_bare:$Imm2)),
8647               (neon_uimm3_bare:$Imm1))),
8648           (v8i16 (INSELh (v8i16 VPR128:$Rn), (v8i16 VPR128:$src),
8649             neon_uimm3_bare:$Imm1, neon_uimm3_bare:$Imm2))>;
8650
8651 // Patterns for vector copy of half-precision FP value in i16 storage type
8652 def : Pat<(v8i16 (vector_insert (v8i16 VPR128:$Rn),
8653             (i32 (assertsext (i32 (fp_to_sint(f32 (f16_to_f32 (i32 (and (i32
8654               (vector_extract (v8i16 VPR128:$src), neon_uimm3_bare:$Imm2)),
8655               65535)))))))),
8656             (neon_uimm3_bare:$Imm1))),
8657           (v8i16 (INSELh (v8i16 VPR128:$Rn), (v8i16 VPR128:$src),
8658             neon_uimm3_bare:$Imm1, neon_uimm3_bare:$Imm2))>;
8659
8660 def : Pat<(v4i16 (vector_insert (v4i16 VPR64:$Rn),
8661             (i32 (assertsext (i32 (fp_to_sint(f32 (f16_to_f32 (i32 (and (i32
8662               (vector_extract (v4i16 VPR64:$src), neon_uimm3_bare:$Imm2)),
8663               65535)))))))),
8664             (neon_uimm3_bare:$Imm1))),
8665           (v4i16 (EXTRACT_SUBREG
8666             (v8i16 (INSELh
8667               (v8i16 (SUBREG_TO_REG (i64 0), VPR64:$Rn, sub_64)),
8668               (v8i16 (SUBREG_TO_REG (i64 0), VPR64:$src, sub_64)),
8669               neon_uimm3_bare:$Imm1, neon_uimm3_bare:$Imm2)),
8670             sub_64))>;
8671
8672