lib/Target/AArch64/AArch64InstrNEON.td

   1 //===-- AArch64InstrNEON.td - NEON support for AArch64 -----*- tablegen -*-===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // This file describes the AArch64 NEON instruction set.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 //===----------------------------------------------------------------------===//
  15 // NEON-specific DAG Nodes.
  16 //===----------------------------------------------------------------------===//
  17 def Neon_bsl       : SDNode<"AArch64ISD::NEON_BSL", SDTypeProfile<1, 3,
  18                       [SDTCisVec<0>, SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>,
  19                       SDTCisSameAs<0, 3>]>>;
  20
  21 // (outs Result), (ins Imm, OpCmode)
  22 def SDT_Neon_movi : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVT<1, i32>]>;
  23
  24 def Neon_movi     : SDNode<"AArch64ISD::NEON_MOVIMM", SDT_Neon_movi>;
  25
  26 def Neon_mvni     : SDNode<"AArch64ISD::NEON_MVNIMM", SDT_Neon_movi>;
  27
  28 // (outs Result), (ins Imm)
  29 def Neon_fmovi : SDNode<"AArch64ISD::NEON_FMOVIMM", SDTypeProfile<1, 1,
  30                         [SDTCisVec<0>, SDTCisVT<1, i32>]>>;
  31
  32 // (outs Result), (ins LHS, RHS, CondCode)
  33 def Neon_cmp : SDNode<"AArch64ISD::NEON_CMP", SDTypeProfile<1, 3,
  34                  [SDTCisVec<0>,  SDTCisSameAs<1, 2>]>>;
  35
  36 // (outs Result), (ins LHS, 0/0.0 constant, CondCode)
  37 def Neon_cmpz : SDNode<"AArch64ISD::NEON_CMPZ", SDTypeProfile<1, 3,
  38                  [SDTCisVec<0>,  SDTCisVec<1>]>>;
  39
  40 // (outs Result), (ins LHS, RHS)
  41 def Neon_tst : SDNode<"AArch64ISD::NEON_TST", SDTypeProfile<1, 2,
  42                  [SDTCisVec<0>,  SDTCisSameAs<1, 2>]>>;
  43
  44 //===----------------------------------------------------------------------===//
  45 // Multiclasses
  46 //===----------------------------------------------------------------------===//
  47
  48 multiclass NeonI_3VSame_B_sizes<bit u, bits<2> size,  bits<5> opcode,
  49                                 string asmop, SDPatternOperator opnode8B,
  50                                 SDPatternOperator opnode16B,
  51                                 bit Commutable = 0>
  52 {
  53   let isCommutable = Commutable in {
  54     def _8B :  NeonI_3VSame<0b0, u, size, opcode,
  55                (outs VPR64:$Rd), (ins VPR64:$Rn, VPR64:$Rm),
  56                asmop # "\t$Rd.8b, $Rn.8b, $Rm.8b",
  57                [(set (v8i8 VPR64:$Rd),
  58                   (v8i8 (opnode8B (v8i8 VPR64:$Rn), (v8i8 VPR64:$Rm))))],
  59                NoItinerary>;
  60
  61     def _16B : NeonI_3VSame<0b1, u, size, opcode,
  62                (outs VPR128:$Rd), (ins VPR128:$Rn, VPR128:$Rm),
  63                asmop # "\t$Rd.16b, $Rn.16b, $Rm.16b",
  64                [(set (v16i8 VPR128:$Rd),
  65                   (v16i8 (opnode16B (v16i8 VPR128:$Rn), (v16i8 VPR128:$Rm))))],
  66                NoItinerary>;
  67   }
  68
  69 }
  70
  71 multiclass NeonI_3VSame_HS_sizes<bit u, bits<5> opcode,
  72                                   string asmop, SDPatternOperator opnode,
  73                                   bit Commutable = 0>
  74 {
  75   let isCommutable = Commutable in {
  76     def _4H : NeonI_3VSame<0b0, u, 0b01, opcode,
  77               (outs VPR64:$Rd), (ins VPR64:$Rn, VPR64:$Rm),
  78               asmop # "\t$Rd.4h, $Rn.4h, $Rm.4h",
  79               [(set (v4i16 VPR64:$Rd),
  80                  (v4i16 (opnode (v4i16 VPR64:$Rn), (v4i16 VPR64:$Rm))))],
  81               NoItinerary>;
  82
  83     def _8H : NeonI_3VSame<0b1, u, 0b01, opcode,
  84               (outs VPR128:$Rd), (ins VPR128:$Rn, VPR128:$Rm),
  85               asmop # "\t$Rd.8h, $Rn.8h, $Rm.8h",
  86               [(set (v8i16 VPR128:$Rd),
  87                  (v8i16 (opnode (v8i16 VPR128:$Rn), (v8i16 VPR128:$Rm))))],
  88               NoItinerary>;
  89
  90     def _2S : NeonI_3VSame<0b0, u, 0b10, opcode,
  91               (outs VPR64:$Rd), (ins VPR64:$Rn, VPR64:$Rm),
  92               asmop # "\t$Rd.2s, $Rn.2s, $Rm.2s",
  93               [(set (v2i32 VPR64:$Rd),
  94                  (v2i32 (opnode (v2i32 VPR64:$Rn), (v2i32 VPR64:$Rm))))],
  95               NoItinerary>;
  96
  97     def _4S : NeonI_3VSame<0b1, u, 0b10, opcode,
  98               (outs VPR128:$Rd), (ins VPR128:$Rn, VPR128:$Rm),
  99               asmop # "\t$Rd.4s, $Rn.4s, $Rm.4s",
 100               [(set (v4i32 VPR128:$Rd),
 101                  (v4i32 (opnode (v4i32 VPR128:$Rn), (v4i32 VPR128:$Rm))))],
 102               NoItinerary>;
 103   }
 104 }
 105 multiclass NeonI_3VSame_BHS_sizes<bit u, bits<5> opcode,
 106                                   string asmop, SDPatternOperator opnode,
 107                                   bit Commutable = 0>
 108    : NeonI_3VSame_HS_sizes<u, opcode,  asmop, opnode, Commutable>
 109 {
 110   let isCommutable = Commutable in {
 111     def _8B :  NeonI_3VSame<0b0, u, 0b00, opcode,
 112                (outs VPR64:$Rd), (ins VPR64:$Rn, VPR64:$Rm),
 113                asmop # "\t$Rd.8b, $Rn.8b, $Rm.8b",
 114                [(set (v8i8 VPR64:$Rd),
 115                   (v8i8 (opnode (v8i8 VPR64:$Rn), (v8i8 VPR64:$Rm))))],
 116                NoItinerary>;
 117
 118     def _16B : NeonI_3VSame<0b1, u, 0b00, opcode,
 119                (outs VPR128:$Rd), (ins VPR128:$Rn, VPR128:$Rm),
 120                asmop # "\t$Rd.16b, $Rn.16b, $Rm.16b",
 121                [(set (v16i8 VPR128:$Rd),
 122                   (v16i8 (opnode (v16i8 VPR128:$Rn), (v16i8 VPR128:$Rm))))],
 123                NoItinerary>;
 124   }
 125 }
 126
 127 multiclass NeonI_3VSame_BHSD_sizes<bit u, bits<5> opcode,
 128                                    string asmop, SDPatternOperator opnode,
 129                                    bit Commutable = 0>
 130    : NeonI_3VSame_BHS_sizes<u, opcode,  asmop, opnode, Commutable>
 131 {
 132   let isCommutable = Commutable in {
 133     def _2D : NeonI_3VSame<0b1, u, 0b11, opcode,
 134               (outs VPR128:$Rd), (ins VPR128:$Rn, VPR128:$Rm),
 135               asmop # "\t$Rd.2d, $Rn.2d, $Rm.2d",
 136               [(set (v2i64 VPR128:$Rd),
 137                  (v2i64 (opnode (v2i64 VPR128:$Rn), (v2i64 VPR128:$Rm))))],
 138               NoItinerary>;
 139   }
 140 }
 141
 142 // Multiclass NeonI_3VSame_SD_sizes: Operand types are floating point types,
 143 // but Result types can be integer or floating point types.
 144 multiclass NeonI_3VSame_SD_sizes<bit u, bit size, bits<5> opcode,
 145                                  string asmop, SDPatternOperator opnode2S,
 146                                  SDPatternOperator opnode4S,
 147                                  SDPatternOperator opnode2D,
 148                                  ValueType ResTy2S, ValueType ResTy4S,
 149                                  ValueType ResTy2D, bit Commutable = 0>
 150 {
 151   let isCommutable = Commutable in {
 152     def _2S : NeonI_3VSame<0b0, u, {size, 0b0}, opcode,
 153               (outs VPR64:$Rd), (ins VPR64:$Rn, VPR64:$Rm),
 154               asmop # "\t$Rd.2s, $Rn.2s, $Rm.2s",
 155               [(set (ResTy2S VPR64:$Rd),
 156                  (ResTy2S (opnode2S (v2f32 VPR64:$Rn), (v2f32 VPR64:$Rm))))],
 157               NoItinerary>;
 158
 159     def _4S : NeonI_3VSame<0b1, u, {size, 0b0}, opcode,
 160               (outs VPR128:$Rd), (ins VPR128:$Rn, VPR128:$Rm),
 161               asmop # "\t$Rd.4s, $Rn.4s, $Rm.4s",
 162               [(set (ResTy4S VPR128:$Rd),
 163                  (ResTy4S (opnode4S (v4f32 VPR128:$Rn), (v4f32 VPR128:$Rm))))],
 164               NoItinerary>;
 165
 166     def _2D : NeonI_3VSame<0b1, u, {size, 0b1}, opcode,
 167               (outs VPR128:$Rd), (ins VPR128:$Rn, VPR128:$Rm),
 168               asmop # "\t$Rd.2d, $Rn.2d, $Rm.2d",
 169               [(set (ResTy2D VPR128:$Rd),
 170                  (ResTy2D (opnode2D (v2f64 VPR128:$Rn), (v2f64 VPR128:$Rm))))],
 171                NoItinerary>;
 172   }
 173 }
 174
 175 //===----------------------------------------------------------------------===//
 176 // Instruction Definitions
 177 //===----------------------------------------------------------------------===//
 178
 179 // Vector Arithmetic Instructions
 180
 181 // Vector Add (Integer and Floating-Point)
 182
 183 defm ADDvvv :  NeonI_3VSame_BHSD_sizes<0b0, 0b10000, "add", add, 1>;
 184 defm FADDvvv : NeonI_3VSame_SD_sizes<0b0, 0b0, 0b11010, "fadd", fadd, fadd, fadd,
 185                                      v2f32, v4f32, v2f64, 1>;
 186
 187 // Vector Sub (Integer and Floating-Point)
 188
 189 defm SUBvvv :  NeonI_3VSame_BHSD_sizes<0b1, 0b10000, "sub", sub, 0>;
 190 defm FSUBvvv : NeonI_3VSame_SD_sizes<0b0, 0b1, 0b11010, "fsub", fsub, fsub, fsub,
 191                                      v2f32, v4f32, v2f64, 0>;
 192
 193 // Vector Multiply (Integer and Floating-Point)
 194
 195 defm MULvvv :  NeonI_3VSame_BHS_sizes<0b0, 0b10011, "mul", mul, 1>;
 196 defm FMULvvv : NeonI_3VSame_SD_sizes<0b1, 0b0, 0b11011, "fmul", fmul, fmul, fmul,
 197                                      v2f32, v4f32, v2f64, 1>;
 198
 199 // Vector Multiply (Polynomial)
 200
 201 defm PMULvvv : NeonI_3VSame_B_sizes<0b1, 0b00, 0b10011, "pmul",
 202                                     int_arm_neon_vmulp, int_arm_neon_vmulp, 1>;
 203
 204 // Vector Multiply-accumulate and Multiply-subtract (Integer)
 205
 206 // class NeonI_3VSame_Constraint_impl: NeonI_3VSame with no data type and
 207 // two operands constraints.
 208 class NeonI_3VSame_Constraint_impl<string asmop, string asmlane,
 209   RegisterClass VPRC, ValueType OpTy, bit q, bit u, bits<2> size, bits<5> opcode,
 210   SDPatternOperator opnode>
 211   : NeonI_3VSame<q, u, size, opcode,
 212     (outs VPRC:$Rd), (ins VPRC:$src, VPRC:$Rn, VPRC:$Rm),
 213     asmop # "\t$Rd" # asmlane # ", $Rn" # asmlane # ", $Rm" # asmlane,
 214     [(set (OpTy VPRC:$Rd),
 215        (OpTy (opnode (OpTy VPRC:$src), (OpTy VPRC:$Rn), (OpTy VPRC:$Rm))))],
 216     NoItinerary> {
 217   let Constraints = "$src = $Rd";
 218 }
 219
 220 def Neon_mla : PatFrag<(ops node:$Ra, node:$Rn, node:$Rm),
 221                        (add node:$Ra, (mul node:$Rn, node:$Rm))>;
 222
 223 def Neon_mls : PatFrag<(ops node:$Ra, node:$Rn, node:$Rm),
 224                        (sub node:$Ra, (mul node:$Rn, node:$Rm))>;
 225
 226
 227 def MLAvvv_8B:  NeonI_3VSame_Constraint_impl<"mla", ".8b",  VPR64,  v8i8,
 228                                              0b0, 0b0, 0b00, 0b10010, Neon_mla>;
 229 def MLAvvv_16B: NeonI_3VSame_Constraint_impl<"mla", ".16b", VPR128, v16i8,
 230                                              0b1, 0b0, 0b00, 0b10010, Neon_mla>;
 231 def MLAvvv_4H:  NeonI_3VSame_Constraint_impl<"mla", ".4h",  VPR64,  v4i16,
 232                                              0b0, 0b0, 0b01, 0b10010, Neon_mla>;
 233 def MLAvvv_8H:  NeonI_3VSame_Constraint_impl<"mla", ".8h",  VPR128, v8i16,
 234                                              0b1, 0b0, 0b01, 0b10010, Neon_mla>;
 235 def MLAvvv_2S:  NeonI_3VSame_Constraint_impl<"mla", ".2s",  VPR64,  v2i32,
 236                                              0b0, 0b0, 0b10, 0b10010, Neon_mla>;
 237 def MLAvvv_4S:  NeonI_3VSame_Constraint_impl<"mla", ".4s",  VPR128, v4i32,
 238                                              0b1, 0b0, 0b10, 0b10010, Neon_mla>;
 239
 240 def MLSvvv_8B:  NeonI_3VSame_Constraint_impl<"mls", ".8b",  VPR64,  v8i8,
 241                                              0b0, 0b1, 0b00, 0b10010, Neon_mls>;
 242 def MLSvvv_16B: NeonI_3VSame_Constraint_impl<"mls", ".16b", VPR128, v16i8,
 243                                              0b1, 0b1, 0b00, 0b10010, Neon_mls>;
 244 def MLSvvv_4H:  NeonI_3VSame_Constraint_impl<"mls", ".4h",  VPR64,  v4i16,
 245                                              0b0, 0b1, 0b01, 0b10010, Neon_mls>;
 246 def MLSvvv_8H:  NeonI_3VSame_Constraint_impl<"mls", ".8h",  VPR128, v8i16,
 247                                              0b1, 0b1, 0b01, 0b10010, Neon_mls>;
 248 def MLSvvv_2S:  NeonI_3VSame_Constraint_impl<"mls", ".2s",  VPR64,  v2i32,
 249                                              0b0, 0b1, 0b10, 0b10010, Neon_mls>;
 250 def MLSvvv_4S:  NeonI_3VSame_Constraint_impl<"mls", ".4s",  VPR128, v4i32,
 251                                              0b1, 0b1, 0b10, 0b10010, Neon_mls>;
 252
 253 // Vector Multiply-accumulate and Multiply-subtract (Floating Point)
 254
 255 def Neon_fmla : PatFrag<(ops node:$Ra, node:$Rn, node:$Rm),
 256                         (fadd node:$Ra, (fmul node:$Rn, node:$Rm))>;
 257
 258 def Neon_fmls : PatFrag<(ops node:$Ra, node:$Rn, node:$Rm),
 259                         (fsub node:$Ra, (fmul node:$Rn, node:$Rm))>;
 260
 261 let Predicates = [HasNEON, UseFusedMAC] in {
 262 def FMLAvvv_2S: NeonI_3VSame_Constraint_impl<"fmla", ".2s",  VPR64,  v2f32,
 263                                              0b0, 0b0, 0b00, 0b11001, Neon_fmla>;
 264 def FMLAvvv_4S: NeonI_3VSame_Constraint_impl<"fmla", ".4s",  VPR128, v4f32,
 265                                              0b1, 0b0, 0b00, 0b11001, Neon_fmla>;
 266 def FMLAvvv_2D: NeonI_3VSame_Constraint_impl<"fmla", ".2d",  VPR128, v2f64,
 267                                              0b1, 0b0, 0b01, 0b11001, Neon_fmla>;
 268
 269 def FMLSvvv_2S: NeonI_3VSame_Constraint_impl<"fmls", ".2s",  VPR64,  v2f32,
 270                                               0b0, 0b0, 0b10, 0b11001, Neon_fmls>;
 271 def FMLSvvv_4S: NeonI_3VSame_Constraint_impl<"fmls", ".4s",  VPR128, v4f32,
 272                                              0b1, 0b0, 0b10, 0b11001, Neon_fmls>;
 273 def FMLSvvv_2D: NeonI_3VSame_Constraint_impl<"fmls", ".2d",  VPR128, v2f64,
 274                                              0b1, 0b0, 0b11, 0b11001, Neon_fmls>;
 275 }
 276
 277 // We're also allowed to match the fma instruction regardless of compile
 278 // options.
 279 def : Pat<(v2f32 (fma VPR64:$Rn, VPR64:$Rm, VPR64:$Ra)),
 280           (FMLAvvv_2S VPR64:$Ra, VPR64:$Rn, VPR64:$Rm)>;
 281 def : Pat<(v4f32 (fma VPR128:$Rn, VPR128:$Rm, VPR128:$Ra)),
 282           (FMLAvvv_4S VPR128:$Ra, VPR128:$Rn, VPR128:$Rm)>;
 283 def : Pat<(v2f64 (fma VPR128:$Rn, VPR128:$Rm, VPR128:$Ra)),
 284           (FMLAvvv_2D VPR128:$Ra, VPR128:$Rn, VPR128:$Rm)>;
 285
 286 def : Pat<(v2f32 (fma (fneg VPR64:$Rn), VPR64:$Rm, VPR64:$Ra)),
 287           (FMLSvvv_2S VPR64:$Ra, VPR64:$Rn, VPR64:$Rm)>;
 288 def : Pat<(v4f32 (fma (fneg VPR128:$Rn), VPR128:$Rm, VPR128:$Ra)),
 289           (FMLSvvv_4S VPR128:$Ra, VPR128:$Rn, VPR128:$Rm)>;
 290 def : Pat<(v2f64 (fma (fneg VPR128:$Rn), VPR128:$Rm, VPR128:$Ra)),
 291           (FMLSvvv_2D VPR128:$Ra, VPR128:$Rn, VPR128:$Rm)>;
 292
 293 // Vector Divide (Floating-Point)
 294
 295 defm FDIVvvv : NeonI_3VSame_SD_sizes<0b1, 0b0, 0b11111, "fdiv", fdiv, fdiv, fdiv,
 296                                      v2f32, v4f32, v2f64, 0>;
 297
 298 // Vector Bitwise Operations
 299
 300 // Vector Bitwise AND
 301
 302 defm ANDvvv : NeonI_3VSame_B_sizes<0b0, 0b00, 0b00011, "and", and, and, 1>;
 303
 304 // Vector Bitwise Exclusive OR
 305
 306 defm EORvvv : NeonI_3VSame_B_sizes<0b1, 0b00, 0b00011, "eor", xor, xor, 1>;
 307
 308 // Vector Bitwise OR
 309
 310 defm ORRvvv : NeonI_3VSame_B_sizes<0b0, 0b10, 0b00011, "orr", or, or, 1>;
 311
 312 // ORR disassembled as MOV if Vn==Vm
 313
 314 // Vector Move - register
 315 // Alias for ORR if Vn=Vm and it is the preferred syntax
 316 def : NeonInstAlias<"mov $Rd.8b, $Rn.8b",
 317                     (ORRvvv_8B VPR64:$Rd, VPR64:$Rn, VPR64:$Rn)>;
 318 def : NeonInstAlias<"mov $Rd.16b, $Rn.16b",
 319                     (ORRvvv_16B VPR128:$Rd, VPR128:$Rn, VPR128:$Rn)>;
 320
 321 def Neon_immAllOnes: PatLeaf<(Neon_movi (i32 timm), (i32 imm)), [{
 322   ConstantSDNode *ImmConstVal = cast<ConstantSDNode>(N->getOperand(0));
 323   ConstantSDNode *OpCmodeConstVal = cast<ConstantSDNode>(N->getOperand(1));
 324   unsigned EltBits;
 325   uint64_t EltVal = A64Imms::decodeNeonModImm(ImmConstVal->getZExtValue(),
 326     OpCmodeConstVal->getZExtValue(), EltBits);
 327   return (EltBits == 8 && EltVal == 0xff);
 328 }]>;
 329
 330
 331 def Neon_not8B  : PatFrag<(ops node:$in),
 332                           (xor node:$in, (bitconvert (v8i8 Neon_immAllOnes)))>;
 333 def Neon_not16B : PatFrag<(ops node:$in),
 334                           (xor node:$in, (bitconvert (v16i8 Neon_immAllOnes)))>;
 335
 336 def Neon_orn8B : PatFrag<(ops node:$Rn, node:$Rm),
 337                          (or node:$Rn, (Neon_not8B node:$Rm))>;
 338
 339 def Neon_orn16B : PatFrag<(ops node:$Rn, node:$Rm),
 340                           (or node:$Rn, (Neon_not16B node:$Rm))>;
 341
 342 def Neon_bic8B : PatFrag<(ops node:$Rn, node:$Rm),
 343                          (and node:$Rn, (Neon_not8B node:$Rm))>;
 344
 345 def Neon_bic16B : PatFrag<(ops node:$Rn, node:$Rm),
 346                           (and node:$Rn, (Neon_not16B node:$Rm))>;
 347
 348
 349 // Vector Bitwise OR NOT - register
 350
 351 defm ORNvvv : NeonI_3VSame_B_sizes<0b0, 0b11, 0b00011, "orn",
 352                                    Neon_orn8B, Neon_orn16B, 0>;
 353
 354 // Vector Bitwise Bit Clear (AND NOT) - register
 355
 356 defm BICvvv : NeonI_3VSame_B_sizes<0b0, 0b01, 0b00011, "bic",
 357                                    Neon_bic8B, Neon_bic16B, 0>;
 358
 359 multiclass Neon_bitwise2V_patterns<SDPatternOperator opnode8B,
 360                                    SDPatternOperator opnode16B,
 361                                    Instruction INST8B,
 362                                    Instruction INST16B> {
 363   def : Pat<(v2i32 (opnode8B VPR64:$Rn, VPR64:$Rm)),
 364             (INST8B VPR64:$Rn, VPR64:$Rm)>;
 365   def : Pat<(v4i16 (opnode8B VPR64:$Rn, VPR64:$Rm)),
 366             (INST8B VPR64:$Rn, VPR64:$Rm)>;
 367   def : Pat<(v1i64 (opnode8B VPR64:$Rn, VPR64:$Rm)),
 368             (INST8B VPR64:$Rn, VPR64:$Rm)>;
 369   def : Pat<(v4i32 (opnode16B VPR128:$Rn, VPR128:$Rm)),
 370             (INST16B VPR128:$Rn, VPR128:$Rm)>;
 371   def : Pat<(v8i16 (opnode16B VPR128:$Rn, VPR128:$Rm)),
 372             (INST16B VPR128:$Rn, VPR128:$Rm)>;
 373   def : Pat<(v2i64 (opnode16B VPR128:$Rn, VPR128:$Rm)),
 374             (INST16B VPR128:$Rn, VPR128:$Rm)>;
 375 }
 376
 377 // Additional patterns for bitwise instructions AND, EOR, ORR, BIC, ORN
 378 defm : Neon_bitwise2V_patterns<and, and, ANDvvv_8B, ANDvvv_16B>;
 379 defm : Neon_bitwise2V_patterns<or,  or,  ORRvvv_8B, ORRvvv_16B>;
 380 defm : Neon_bitwise2V_patterns<xor, xor, EORvvv_8B, EORvvv_16B>;
 381 defm : Neon_bitwise2V_patterns<Neon_bic8B, Neon_bic16B, BICvvv_8B, BICvvv_16B>;
 382 defm : Neon_bitwise2V_patterns<Neon_orn8B, Neon_orn16B, ORNvvv_8B, ORNvvv_16B>;
 383
 384 //   Vector Bitwise Select
 385 def BSLvvv_8B  : NeonI_3VSame_Constraint_impl<"bsl", ".8b",  VPR64, v8i8,
 386                                               0b0, 0b1, 0b01, 0b00011, Neon_bsl>;
 387
 388 def BSLvvv_16B : NeonI_3VSame_Constraint_impl<"bsl", ".16b", VPR128, v16i8,
 389                                               0b1, 0b1, 0b01, 0b00011, Neon_bsl>;
 390
 391 multiclass Neon_bitwise3V_patterns<SDPatternOperator opnode,
 392                                    Instruction INST8B,
 393                                    Instruction INST16B> {
 394   // Disassociate type from instruction definition
 395   def : Pat<(v2i32 (opnode VPR64:$src,VPR64:$Rn, VPR64:$Rm)),
 396             (INST8B VPR64:$src, VPR64:$Rn, VPR64:$Rm)>;
 397   def : Pat<(v4i16 (opnode VPR64:$src, VPR64:$Rn, VPR64:$Rm)),
 398             (INST8B VPR64:$src, VPR64:$Rn, VPR64:$Rm)>;
 399   def : Pat<(v1i64 (opnode VPR64:$src, VPR64:$Rn, VPR64:$Rm)),
 400             (INST8B VPR64:$src, VPR64:$Rn, VPR64:$Rm)>;
 401   def : Pat<(v4i32 (opnode VPR128:$src, VPR128:$Rn, VPR128:$Rm)),
 402             (INST16B VPR128:$src, VPR128:$Rn, VPR128:$Rm)>;
 403   def : Pat<(v8i16 (opnode VPR128:$src, VPR128:$Rn, VPR128:$Rm)),
 404             (INST16B VPR128:$src, VPR128:$Rn, VPR128:$Rm)>;
 405   def : Pat<(v2i64 (opnode VPR128:$src, VPR128:$Rn, VPR128:$Rm)),
 406             (INST16B VPR128:$src, VPR128:$Rn, VPR128:$Rm)>;
 407
 408   // Allow to match BSL instruction pattern with non-constant operand
 409   def : Pat<(v8i8 (or (and VPR64:$Rn, VPR64:$Rd),
 410                     (and VPR64:$Rm, (Neon_not8B VPR64:$Rd)))),
 411           (INST8B VPR64:$Rd, VPR64:$Rn, VPR64:$Rm)>;
 412   def : Pat<(v4i16 (or (and VPR64:$Rn, VPR64:$Rd),
 413                      (and VPR64:$Rm, (Neon_not8B VPR64:$Rd)))),
 414           (INST8B VPR64:$Rd, VPR64:$Rn, VPR64:$Rm)>;
 415   def : Pat<(v2i32 (or (and VPR64:$Rn, VPR64:$Rd),
 416                      (and VPR64:$Rm, (Neon_not8B VPR64:$Rd)))),
 417           (INST8B VPR64:$Rd, VPR64:$Rn, VPR64:$Rm)>;
 418   def : Pat<(v1i64 (or (and VPR64:$Rn, VPR64:$Rd),
 419                      (and VPR64:$Rm, (Neon_not8B VPR64:$Rd)))),
 420           (INST8B VPR64:$Rd, VPR64:$Rn, VPR64:$Rm)>;
 421   def : Pat<(v16i8 (or (and VPR128:$Rn, VPR128:$Rd),
 422                      (and VPR128:$Rm, (Neon_not16B VPR128:$Rd)))),
 423           (INST16B VPR128:$Rd, VPR128:$Rn, VPR128:$Rm)>;
 424   def : Pat<(v8i16 (or (and VPR128:$Rn, VPR128:$Rd),
 425                      (and VPR128:$Rm, (Neon_not16B VPR128:$Rd)))),
 426           (INST16B VPR128:$Rd, VPR128:$Rn, VPR128:$Rm)>;
 427   def : Pat<(v4i32 (or (and VPR128:$Rn, VPR128:$Rd),
 428                      (and VPR128:$Rm, (Neon_not16B VPR128:$Rd)))),
 429           (INST16B VPR128:$Rd, VPR128:$Rn, VPR128:$Rm)>;
 430   def : Pat<(v2i64 (or (and VPR128:$Rn, VPR128:$Rd),
 431                      (and VPR128:$Rm, (Neon_not16B VPR128:$Rd)))),
 432           (INST16B VPR128:$Rd, VPR128:$Rn, VPR128:$Rm)>;
 433
 434   // Allow to match llvm.arm.* intrinsics.
 435   def : Pat<(v8i8 (int_arm_neon_vbsl (v8i8 VPR64:$src),
 436                     (v8i8 VPR64:$Rn), (v8i8 VPR64:$Rm))),
 437             (INST8B VPR64:$src, VPR64:$Rn, VPR64:$Rm)>;
 438   def : Pat<(v4i16 (int_arm_neon_vbsl (v4i16 VPR64:$src),
 439                     (v4i16 VPR64:$Rn), (v4i16 VPR64:$Rm))),
 440             (INST8B VPR64:$src, VPR64:$Rn, VPR64:$Rm)>;
 441   def : Pat<(v2i32 (int_arm_neon_vbsl (v2i32 VPR64:$src),
 442                     (v2i32 VPR64:$Rn), (v2i32 VPR64:$Rm))),
 443             (INST8B VPR64:$src, VPR64:$Rn, VPR64:$Rm)>;
 444   def : Pat<(v1i64 (int_arm_neon_vbsl (v1i64 VPR64:$src),
 445                     (v1i64 VPR64:$Rn), (v1i64 VPR64:$Rm))),
 446             (INST8B VPR64:$src, VPR64:$Rn, VPR64:$Rm)>;
 447   def : Pat<(v2f32 (int_arm_neon_vbsl (v2f32 VPR64:$src),
 448                     (v2f32 VPR64:$Rn), (v2f32 VPR64:$Rm))),
 449             (INST8B VPR64:$src, VPR64:$Rn, VPR64:$Rm)>;
 450   def : Pat<(v16i8 (int_arm_neon_vbsl (v16i8 VPR128:$src),
 451                     (v16i8 VPR128:$Rn), (v16i8 VPR128:$Rm))),
 452             (INST16B VPR128:$src, VPR128:$Rn, VPR128:$Rm)>;
 453   def : Pat<(v8i16 (int_arm_neon_vbsl (v8i16 VPR128:$src),
 454                     (v8i16 VPR128:$Rn), (v8i16 VPR128:$Rm))),
 455             (INST16B VPR128:$src, VPR128:$Rn, VPR128:$Rm)>;
 456   def : Pat<(v4i32 (int_arm_neon_vbsl (v4i32 VPR128:$src),
 457                     (v4i32 VPR128:$Rn), (v4i32 VPR128:$Rm))),
 458             (INST16B VPR128:$src, VPR128:$Rn, VPR128:$Rm)>;
 459   def : Pat<(v2i64 (int_arm_neon_vbsl (v2i64 VPR128:$src),
 460                     (v2i64 VPR128:$Rn), (v2i64 VPR128:$Rm))),
 461             (INST16B VPR128:$src, VPR128:$Rn, VPR128:$Rm)>;
 462   def : Pat<(v4f32 (int_arm_neon_vbsl (v4f32 VPR128:$src),
 463                     (v4f32 VPR128:$Rn), (v4f32 VPR128:$Rm))),
 464             (INST16B VPR128:$src, VPR128:$Rn, VPR128:$Rm)>;
 465   def : Pat<(v2f64 (int_arm_neon_vbsl (v2f64 VPR128:$src),
 466                     (v2f64 VPR128:$Rn), (v2f64 VPR128:$Rm))),
 467             (INST16B VPR128:$src, VPR128:$Rn, VPR128:$Rm)>;
 468 }
 469
 470 // Additional patterns for bitwise instruction BSL
 471 defm: Neon_bitwise3V_patterns<Neon_bsl, BSLvvv_8B, BSLvvv_16B>;
 472
 473 def Neon_NoBSLop : PatFrag<(ops node:$src, node:$Rn, node:$Rm),
 474                            (Neon_bsl node:$src, node:$Rn, node:$Rm),
 475                            [{ (void)N; return false; }]>;
 476
 477 // Vector Bitwise Insert if True
 478
 479 def BITvvv_8B  : NeonI_3VSame_Constraint_impl<"bit", ".8b", VPR64,   v8i8,
 480                    0b0, 0b1, 0b10, 0b00011, Neon_NoBSLop>;
 481 def BITvvv_16B : NeonI_3VSame_Constraint_impl<"bit", ".16b", VPR128, v16i8,
 482                    0b1, 0b1, 0b10, 0b00011, Neon_NoBSLop>;
 483
 484 // Vector Bitwise Insert if False
 485
 486 def BIFvvv_8B  : NeonI_3VSame_Constraint_impl<"bif", ".8b", VPR64,  v8i8,
 487                                 0b0, 0b1, 0b11, 0b00011, Neon_NoBSLop>;
 488 def BIFvvv_16B : NeonI_3VSame_Constraint_impl<"bif", ".16b", VPR128, v16i8,
 489                                 0b1, 0b1, 0b11, 0b00011, Neon_NoBSLop>;
 490
 491 // Vector Absolute Difference and Accumulate (Signed, Unsigned)
 492
 493 def Neon_uaba : PatFrag<(ops node:$Ra, node:$Rn, node:$Rm),
 494                        (add node:$Ra, (int_arm_neon_vabdu node:$Rn, node:$Rm))>;
 495 def Neon_saba : PatFrag<(ops node:$Ra, node:$Rn, node:$Rm),
 496                        (add node:$Ra, (int_arm_neon_vabds node:$Rn, node:$Rm))>;
 497
 498 // Vector Absolute Difference and Accumulate (Unsigned)
 499 def UABAvvv_8B :  NeonI_3VSame_Constraint_impl<"uaba", ".8b",  VPR64,  v8i8,
 500                     0b0, 0b1, 0b00, 0b01111, Neon_uaba>;
 501 def UABAvvv_16B : NeonI_3VSame_Constraint_impl<"uaba", ".16b", VPR128, v16i8,
 502                     0b1, 0b1, 0b00, 0b01111, Neon_uaba>;
 503 def UABAvvv_4H :  NeonI_3VSame_Constraint_impl<"uaba", ".4h",  VPR64,  v4i16,
 504                     0b0, 0b1, 0b01, 0b01111, Neon_uaba>;
 505 def UABAvvv_8H :  NeonI_3VSame_Constraint_impl<"uaba", ".8h",  VPR128, v8i16,
 506                     0b1, 0b1, 0b01, 0b01111, Neon_uaba>;
 507 def UABAvvv_2S :  NeonI_3VSame_Constraint_impl<"uaba", ".2s",  VPR64,  v2i32,
 508                     0b0, 0b1, 0b10, 0b01111, Neon_uaba>;
 509 def UABAvvv_4S :  NeonI_3VSame_Constraint_impl<"uaba", ".4s",  VPR128, v4i32,
 510                     0b1, 0b1, 0b10, 0b01111, Neon_uaba>;
 511
 512 // Vector Absolute Difference and Accumulate (Signed)
 513 def SABAvvv_8B :  NeonI_3VSame_Constraint_impl<"saba", ".8b",  VPR64,  v8i8,
 514                     0b0, 0b0, 0b00, 0b01111, Neon_saba>;
 515 def SABAvvv_16B : NeonI_3VSame_Constraint_impl<"saba", ".16b", VPR128, v16i8,
 516                     0b1, 0b0, 0b00, 0b01111, Neon_saba>;
 517 def SABAvvv_4H :  NeonI_3VSame_Constraint_impl<"saba", ".4h",  VPR64,  v4i16,
 518                     0b0, 0b0, 0b01, 0b01111, Neon_saba>;
 519 def SABAvvv_8H :  NeonI_3VSame_Constraint_impl<"saba", ".8h",  VPR128, v8i16,
 520                     0b1, 0b0, 0b01, 0b01111, Neon_saba>;
 521 def SABAvvv_2S :  NeonI_3VSame_Constraint_impl<"saba", ".2s",  VPR64,  v2i32,
 522                     0b0, 0b0, 0b10, 0b01111, Neon_saba>;
 523 def SABAvvv_4S :  NeonI_3VSame_Constraint_impl<"saba", ".4s",  VPR128, v4i32,
 524                     0b1, 0b0, 0b10, 0b01111, Neon_saba>;
 525
 526
 527 // Vector Absolute Difference (Signed, Unsigned)
 528 defm UABDvvv : NeonI_3VSame_BHS_sizes<0b1, 0b01110, "uabd", int_arm_neon_vabdu, 0>;
 529 defm SABDvvv : NeonI_3VSame_BHS_sizes<0b0, 0b01110, "sabd", int_arm_neon_vabds, 0>;
 530
 531 // Vector Absolute Difference (Floating Point)
 532 defm FABDvvv: NeonI_3VSame_SD_sizes<0b1, 0b1, 0b11010, "fabd",
 533                                     int_arm_neon_vabds, int_arm_neon_vabds,
 534                                     int_arm_neon_vabds, v2f32, v4f32, v2f64, 0>;
 535
 536 // Vector Reciprocal Step (Floating Point)
 537 defm FRECPSvvv : NeonI_3VSame_SD_sizes<0b0, 0b0, 0b11111, "frecps",
 538                                        int_arm_neon_vrecps, int_arm_neon_vrecps,
 539                                        int_arm_neon_vrecps,
 540                                        v2f32, v4f32, v2f64, 0>;
 541
 542 // Vector Reciprocal Square Root Step (Floating Point)
 543 defm FRSQRTSvvv : NeonI_3VSame_SD_sizes<0b0, 0b1, 0b11111, "frsqrts",
 544                                         int_arm_neon_vrsqrts,
 545                                         int_arm_neon_vrsqrts,
 546                                         int_arm_neon_vrsqrts,
 547                                         v2f32, v4f32, v2f64, 0>;
 548
 549 // Vector Comparisons
 550
 551 def Neon_cmeq : PatFrag<(ops node:$lhs, node:$rhs),
 552                         (Neon_cmp node:$lhs, node:$rhs, SETEQ)>;
 553 def Neon_cmphs : PatFrag<(ops node:$lhs, node:$rhs),
 554                          (Neon_cmp node:$lhs, node:$rhs, SETUGE)>;
 555 def Neon_cmge : PatFrag<(ops node:$lhs, node:$rhs),
 556                         (Neon_cmp node:$lhs, node:$rhs, SETGE)>;
 557 def Neon_cmhi : PatFrag<(ops node:$lhs, node:$rhs),
 558                         (Neon_cmp node:$lhs, node:$rhs, SETUGT)>;
 559 def Neon_cmgt : PatFrag<(ops node:$lhs, node:$rhs),
 560                         (Neon_cmp node:$lhs, node:$rhs, SETGT)>;
 561
 562 // NeonI_compare_aliases class: swaps register operands to implement
 563 // comparison aliases, e.g., CMLE is alias for CMGE with operands reversed.
 564 class NeonI_compare_aliases<string asmop, string asmlane,
 565                             Instruction inst, RegisterClass VPRC>
 566   : NeonInstAlias<asmop # "\t$Rd" # asmlane #", $Rn" # asmlane #
 567                     ", $Rm" # asmlane,
 568                   (inst VPRC:$Rd, VPRC:$Rm, VPRC:$Rn), 0b0>;
 569
 570 // Vector Comparisons (Integer)
 571
 572 // Vector Compare Mask Equal (Integer)
 573 let isCommutable =1 in {
 574 defm CMEQvvv : NeonI_3VSame_BHSD_sizes<0b1, 0b10001, "cmeq", Neon_cmeq, 0>;
 575 }
 576
 577 // Vector Compare Mask Higher or Same (Unsigned Integer)
 578 defm CMHSvvv : NeonI_3VSame_BHSD_sizes<0b1, 0b00111, "cmhs", Neon_cmphs, 0>;
 579
 580 // Vector Compare Mask Greater Than or Equal (Integer)
 581 defm CMGEvvv : NeonI_3VSame_BHSD_sizes<0b0, 0b00111, "cmge", Neon_cmge, 0>;
 582
 583 // Vector Compare Mask Higher (Unsigned Integer)
 584 defm CMHIvvv : NeonI_3VSame_BHSD_sizes<0b1, 0b00110, "cmhi", Neon_cmhi, 0>;
 585
 586 // Vector Compare Mask Greater Than (Integer)
 587 defm CMGTvvv : NeonI_3VSame_BHSD_sizes<0b0, 0b00110, "cmgt", Neon_cmgt, 0>;
 588
 589 // Vector Compare Mask Bitwise Test (Integer)
 590 defm CMTSTvvv:  NeonI_3VSame_BHSD_sizes<0b0, 0b10001, "cmtst", Neon_tst, 0>;
 591
 592 // Vector Compare Mask Less or Same (Unsigned Integer)
 593 // CMLS is alias for CMHS with operands reversed.
 594 def CMLSvvv_8B  : NeonI_compare_aliases<"cmls", ".8b",  CMHSvvv_8B,  VPR64>;
 595 def CMLSvvv_16B : NeonI_compare_aliases<"cmls", ".16b", CMHSvvv_16B, VPR128>;
 596 def CMLSvvv_4H  : NeonI_compare_aliases<"cmls", ".4h",  CMHSvvv_4H,  VPR64>;
 597 def CMLSvvv_8H  : NeonI_compare_aliases<"cmls", ".8h",  CMHSvvv_8H,  VPR128>;
 598 def CMLSvvv_2S  : NeonI_compare_aliases<"cmls", ".2s",  CMHSvvv_2S,  VPR64>;
 599 def CMLSvvv_4S  : NeonI_compare_aliases<"cmls", ".4s",  CMHSvvv_4S,  VPR128>;
 600 def CMLSvvv_2D  : NeonI_compare_aliases<"cmls", ".2d",  CMHSvvv_2D,  VPR128>;
 601
 602 // Vector Compare Mask Less Than or Equal (Integer)
 603 // CMLE is alias for CMGE with operands reversed.
 604 def CMLEvvv_8B  : NeonI_compare_aliases<"cmle", ".8b",  CMGEvvv_8B,  VPR64>;
 605 def CMLEvvv_16B : NeonI_compare_aliases<"cmle", ".16b", CMGEvvv_16B, VPR128>;
 606 def CMLEvvv_4H  : NeonI_compare_aliases<"cmle", ".4h",  CMGEvvv_4H,  VPR64>;
 607 def CMLEvvv_8H  : NeonI_compare_aliases<"cmle", ".8h",  CMGEvvv_8H,  VPR128>;
 608 def CMLEvvv_2S  : NeonI_compare_aliases<"cmle", ".2s",  CMGEvvv_2S,  VPR64>;
 609 def CMLEvvv_4S  : NeonI_compare_aliases<"cmle", ".4s",  CMGEvvv_4S,  VPR128>;
 610 def CMLEvvv_2D  : NeonI_compare_aliases<"cmle", ".2d",  CMGEvvv_2D,  VPR128>;
 611
 612 // Vector Compare Mask Lower (Unsigned Integer)
 613 // CMLO is alias for CMHI with operands reversed.
 614 def CMLOvvv_8B  : NeonI_compare_aliases<"cmlo", ".8b",  CMHIvvv_8B,  VPR64>;
 615 def CMLOvvv_16B : NeonI_compare_aliases<"cmlo", ".16b", CMHIvvv_16B, VPR128>;
 616 def CMLOvvv_4H  : NeonI_compare_aliases<"cmlo", ".4h",  CMHIvvv_4H,  VPR64>;
 617 def CMLOvvv_8H  : NeonI_compare_aliases<"cmlo", ".8h",  CMHIvvv_8H,  VPR128>;
 618 def CMLOvvv_2S  : NeonI_compare_aliases<"cmlo", ".2s",  CMHIvvv_2S,  VPR64>;
 619 def CMLOvvv_4S  : NeonI_compare_aliases<"cmlo", ".4s",  CMHIvvv_4S,  VPR128>;
 620 def CMLOvvv_2D  : NeonI_compare_aliases<"cmlo", ".2d",  CMHIvvv_2D,  VPR128>;
 621
 622 // Vector Compare Mask Less Than (Integer)
 623 // CMLT is alias for CMGT with operands reversed.
 624 def CMLTvvv_8B  : NeonI_compare_aliases<"cmlt", ".8b",  CMGTvvv_8B,  VPR64>;
 625 def CMLTvvv_16B : NeonI_compare_aliases<"cmlt", ".16b", CMGTvvv_16B, VPR128>;
 626 def CMLTvvv_4H  : NeonI_compare_aliases<"cmlt", ".4h",  CMGTvvv_4H,  VPR64>;
 627 def CMLTvvv_8H  : NeonI_compare_aliases<"cmlt", ".8h",  CMGTvvv_8H,  VPR128>;
 628 def CMLTvvv_2S  : NeonI_compare_aliases<"cmlt", ".2s",  CMGTvvv_2S,  VPR64>;
 629 def CMLTvvv_4S  : NeonI_compare_aliases<"cmlt", ".4s",  CMGTvvv_4S,  VPR128>;
 630 def CMLTvvv_2D  : NeonI_compare_aliases<"cmlt", ".2d",  CMGTvvv_2D,  VPR128>;
 631
 632
 633 def neon_uimm0_asmoperand : AsmOperandClass
 634 {
 635   let Name = "UImm0";
 636   let PredicateMethod = "isUImm<0>";
 637   let RenderMethod = "addImmOperands";
 638 }
 639
 640 def neon_uimm0 : Operand<i32>, ImmLeaf<i32, [{return Imm == 0;}]> {
 641   let ParserMatchClass = neon_uimm0_asmoperand;
 642   let PrintMethod = "printNeonUImm0Operand";
 643
 644 }
 645
 646 multiclass NeonI_cmpz_sizes<bit u, bits<5> opcode, string asmop, CondCode CC>
 647 {
 648   def _8B :  NeonI_2VMisc<0b0, u, 0b00, opcode,
 649              (outs VPR64:$Rd), (ins VPR64:$Rn, neon_uimm0:$Imm),
 650              asmop # "\t$Rd.8b, $Rn.8b, $Imm",
 651              [(set (v8i8 VPR64:$Rd),
 652                 (v8i8 (Neon_cmpz (v8i8 VPR64:$Rn), (i32 imm:$Imm), CC)))],
 653              NoItinerary>;
 654
 655   def _16B : NeonI_2VMisc<0b1, u, 0b00, opcode,
 656              (outs VPR128:$Rd), (ins VPR128:$Rn, neon_uimm0:$Imm),
 657              asmop # "\t$Rd.16b, $Rn.16b, $Imm",
 658              [(set (v16i8 VPR128:$Rd),
 659                 (v16i8 (Neon_cmpz (v16i8 VPR128:$Rn), (i32 imm:$Imm), CC)))],
 660              NoItinerary>;
 661
 662   def _4H : NeonI_2VMisc<0b0, u, 0b01, opcode,
 663             (outs VPR64:$Rd), (ins VPR64:$Rn, neon_uimm0:$Imm),
 664             asmop # "\t$Rd.4h, $Rn.4h, $Imm",
 665             [(set (v4i16 VPR64:$Rd),
 666                (v4i16 (Neon_cmpz (v4i16 VPR64:$Rn), (i32 imm:$Imm), CC)))],
 667             NoItinerary>;
 668
 669   def _8H : NeonI_2VMisc<0b1, u, 0b01, opcode,
 670             (outs VPR128:$Rd), (ins VPR128:$Rn, neon_uimm0:$Imm),
 671             asmop # "\t$Rd.8h, $Rn.8h, $Imm",
 672             [(set (v8i16 VPR128:$Rd),
 673                (v8i16 (Neon_cmpz (v8i16 VPR128:$Rn), (i32 imm:$Imm), CC)))],
 674             NoItinerary>;
 675
 676   def _2S : NeonI_2VMisc<0b0, u, 0b10, opcode,
 677             (outs VPR64:$Rd), (ins VPR64:$Rn, neon_uimm0:$Imm),
 678             asmop # "\t$Rd.2s, $Rn.2s, $Imm",
 679             [(set (v2i32 VPR64:$Rd),
 680                (v2i32 (Neon_cmpz (v2i32 VPR64:$Rn), (i32 imm:$Imm), CC)))],
 681             NoItinerary>;
 682
 683   def _4S : NeonI_2VMisc<0b1, u, 0b10, opcode,
 684             (outs VPR128:$Rd), (ins VPR128:$Rn, neon_uimm0:$Imm),
 685             asmop # "\t$Rd.4s, $Rn.4s, $Imm",
 686             [(set (v4i32 VPR128:$Rd),
 687                (v4i32 (Neon_cmpz (v4i32 VPR128:$Rn), (i32 imm:$Imm), CC)))],
 688             NoItinerary>;
 689
 690   def _2D : NeonI_2VMisc<0b1, u, 0b11, opcode,
 691             (outs VPR128:$Rd), (ins VPR128:$Rn, neon_uimm0:$Imm),
 692             asmop # "\t$Rd.2d, $Rn.2d, $Imm",
 693             [(set (v2i64 VPR128:$Rd),
 694                (v2i64 (Neon_cmpz (v2i64 VPR128:$Rn), (i32 imm:$Imm), CC)))],
 695             NoItinerary>;
 696 }
 697
 698 // Vector Compare Mask Equal to Zero (Integer)
 699 defm CMEQvvi : NeonI_cmpz_sizes<0b0, 0b01001, "cmeq", SETEQ>;
 700
 701 // Vector Compare Mask Greater Than or Equal to Zero (Signed Integer)
 702 defm CMGEvvi : NeonI_cmpz_sizes<0b1, 0b01000, "cmge", SETGE>;
 703
 704 // Vector Compare Mask Greater Than Zero (Signed Integer)
 705 defm CMGTvvi : NeonI_cmpz_sizes<0b0, 0b01000, "cmgt", SETGT>;
 706
 707 // Vector Compare Mask Less Than or Equal To Zero (Signed Integer)
 708 defm CMLEvvi : NeonI_cmpz_sizes<0b1, 0b01001, "cmle", SETLE>;
 709
 710 // Vector Compare Mask Less Than Zero (Signed Integer)
 711 defm CMLTvvi : NeonI_cmpz_sizes<0b0, 0b01010, "cmlt", SETLT>;
 712
 713 // Vector Comparisons (Floating Point)
 714
 715 // Vector Compare Mask Equal (Floating Point)
 716 let isCommutable =1 in {
 717 defm FCMEQvvv : NeonI_3VSame_SD_sizes<0b0, 0b0, 0b11100, "fcmeq", Neon_cmeq,
 718                                       Neon_cmeq, Neon_cmeq,
 719                                       v2i32, v4i32, v2i64, 0>;
 720 }
 721
 722 // Vector Compare Mask Greater Than Or Equal (Floating Point)
 723 defm FCMGEvvv : NeonI_3VSame_SD_sizes<0b1, 0b0, 0b11100, "fcmge", Neon_cmge,
 724                                       Neon_cmge, Neon_cmge,
 725                                       v2i32, v4i32, v2i64, 0>;
 726
 727 // Vector Compare Mask Greater Than (Floating Point)
 728 defm FCMGTvvv : NeonI_3VSame_SD_sizes<0b1, 0b1, 0b11100, "fcmgt", Neon_cmgt,
 729                                       Neon_cmgt, Neon_cmgt,
 730                                       v2i32, v4i32, v2i64, 0>;
 731
 732 // Vector Compare Mask Less Than Or Equal (Floating Point)
 733 // FCMLE is alias for FCMGE with operands reversed.
 734 def FCMLEvvv_2S  : NeonI_compare_aliases<"fcmle", ".2s",  FCMGEvvv_2S,  VPR64>;
 735 def FCMLEvvv_4S  : NeonI_compare_aliases<"fcmle", ".4s",  FCMGEvvv_4S,  VPR128>;
 736 def FCMLEvvv_2D  : NeonI_compare_aliases<"fcmle", ".2d",  FCMGEvvv_2D,  VPR128>;
 737
 738 // Vector Compare Mask Less Than (Floating Point)
 739 // FCMLT is alias for FCMGT with operands reversed.
 740 def FCMLTvvv_2S  : NeonI_compare_aliases<"fcmlt", ".2s",  FCMGTvvv_2S,  VPR64>;
 741 def FCMLTvvv_4S  : NeonI_compare_aliases<"fcmlt", ".4s",  FCMGTvvv_4S,  VPR128>;
 742 def FCMLTvvv_2D  : NeonI_compare_aliases<"fcmlt", ".2d",  FCMGTvvv_2D,  VPR128>;
 743
 744
 745 multiclass NeonI_fpcmpz_sizes<bit u, bit size, bits<5> opcode,
 746                               string asmop, CondCode CC>
 747 {
 748   def _2S : NeonI_2VMisc<0b0, u, {size, 0b0}, opcode,
 749             (outs VPR64:$Rd), (ins VPR64:$Rn, fpz32:$FPImm),
 750             asmop # "\t$Rd.2s, $Rn.2s, $FPImm",
 751             [(set (v2i32 VPR64:$Rd),
 752                (v2i32 (Neon_cmpz (v2f32 VPR64:$Rn), (f32 fpimm:$FPImm), CC)))],
 753             NoItinerary>;
 754
 755   def _4S : NeonI_2VMisc<0b1, u, {size, 0b0}, opcode,
 756             (outs VPR128:$Rd), (ins VPR128:$Rn, fpz32:$FPImm),
 757             asmop # "\t$Rd.4s, $Rn.4s, $FPImm",
 758             [(set (v4i32 VPR128:$Rd),
 759                (v4i32 (Neon_cmpz (v4f32 VPR128:$Rn), (f32 fpimm:$FPImm), CC)))],
 760             NoItinerary>;
 761
 762   def _2D : NeonI_2VMisc<0b1, u, {size, 0b1}, opcode,
 763             (outs VPR128:$Rd), (ins VPR128:$Rn, fpz32:$FPImm),
 764             asmop # "\t$Rd.2d, $Rn.2d, $FPImm",
 765             [(set (v2i64 VPR128:$Rd),
 766                (v2i64 (Neon_cmpz (v2f64 VPR128:$Rn), (f32 fpimm:$FPImm), CC)))],
 767             NoItinerary>;
 768 }
 769
 770 // Vector Compare Mask Equal to Zero (Floating Point)
 771 defm FCMEQvvi : NeonI_fpcmpz_sizes<0b0, 0b1, 0b01101, "fcmeq", SETEQ>;
 772
 773 // Vector Compare Mask Greater Than or Equal to Zero (Floating Point)
 774 defm FCMGEvvi : NeonI_fpcmpz_sizes<0b1, 0b1, 0b01100, "fcmge", SETGE>;
 775
 776 // Vector Compare Mask Greater Than Zero (Floating Point)
 777 defm FCMGTvvi : NeonI_fpcmpz_sizes<0b0, 0b1, 0b01100, "fcmgt", SETGT>;
 778
 779 // Vector Compare Mask Less Than or Equal To Zero (Floating Point)
 780 defm FCMLEvvi : NeonI_fpcmpz_sizes<0b1, 0b1, 0b01101, "fcmle", SETLE>;
 781
 782 // Vector Compare Mask Less Than Zero (Floating Point)
 783 defm FCMLTvvi : NeonI_fpcmpz_sizes<0b0, 0b1, 0b01110, "fcmlt", SETLT>;
 784
 785 // Vector Absolute Comparisons (Floating Point)
 786
 787 // Vector Absolute Compare Mask Greater Than Or Equal (Floating Point)
 788 defm FACGEvvv : NeonI_3VSame_SD_sizes<0b1, 0b0, 0b11101, "facge",
 789                                       int_arm_neon_vacged, int_arm_neon_vacgeq,
 790                                       int_aarch64_neon_vacgeq,
 791                                       v2i32, v4i32, v2i64, 0>;
 792
 793 // Vector Absolute Compare Mask Greater Than (Floating Point)
 794 defm FACGTvvv : NeonI_3VSame_SD_sizes<0b1, 0b1, 0b11101, "facgt",
 795                                       int_arm_neon_vacgtd, int_arm_neon_vacgtq,
 796                                       int_aarch64_neon_vacgtq,
 797                                       v2i32, v4i32, v2i64, 0>;
 798
 799 // Vector Absolute Compare Mask Less Than Or Equal (Floating Point)
 800 // FACLE is alias for FACGE with operands reversed.
 801 def FACLEvvv_2S  : NeonI_compare_aliases<"facle", ".2s",  FACGEvvv_2S,  VPR64>;
 802 def FACLEvvv_4S  : NeonI_compare_aliases<"facle", ".4s",  FACGEvvv_4S,  VPR128>;
 803 def FACLEvvv_2D  : NeonI_compare_aliases<"facle", ".2d",  FACGEvvv_2D,  VPR128>;
 804
 805 // Vector Absolute Compare Mask Less Than (Floating Point)
 806 // FACLT is alias for FACGT with operands reversed.
 807 def FACLTvvv_2S  : NeonI_compare_aliases<"faclt", ".2s",  FACGTvvv_2S,  VPR64>;
 808 def FACLTvvv_4S  : NeonI_compare_aliases<"faclt", ".4s",  FACGTvvv_4S,  VPR128>;
 809 def FACLTvvv_2D  : NeonI_compare_aliases<"faclt", ".2d",  FACGTvvv_2D,  VPR128>;
 810
 811 // Vector halving add (Integer Signed, Unsigned)
 812 defm SHADDvvv :  NeonI_3VSame_BHS_sizes<0b0, 0b00000, "shadd",
 813                                         int_arm_neon_vhadds, 1>;
 814 defm UHADDvvv :  NeonI_3VSame_BHS_sizes<0b1, 0b00000, "uhadd",
 815                                         int_arm_neon_vhaddu, 1>;
 816
 817 // Vector halving sub (Integer Signed, Unsigned)
 818 defm SHSUBvvv :  NeonI_3VSame_BHS_sizes<0b0, 0b00100, "shsub",
 819                                         int_arm_neon_vhsubs, 0>;
 820 defm UHSUBvvv :  NeonI_3VSame_BHS_sizes<0b1, 0b00100, "uhsub",
 821                                         int_arm_neon_vhsubu, 0>;
 822
 823 // Vector rouding halving add (Integer Signed, Unsigned)
 824 defm SRHADDvvv :  NeonI_3VSame_BHS_sizes<0b0, 0b00010, "srhadd",
 825                                          int_arm_neon_vrhadds, 1>;
 826 defm URHADDvvv :  NeonI_3VSame_BHS_sizes<0b1, 0b00010, "urhadd",
 827                                          int_arm_neon_vrhaddu, 1>;
 828
 829 // Vector Saturating add (Integer Signed, Unsigned)
 830 defm SQADDvvv :  NeonI_3VSame_BHSD_sizes<0b0, 0b00001, "sqadd",
 831                    int_arm_neon_vqadds, 1>;
 832 defm UQADDvvv :  NeonI_3VSame_BHSD_sizes<0b1, 0b00001, "uqadd",
 833                    int_arm_neon_vqaddu, 1>;
 834
 835 // Vector Saturating sub (Integer Signed, Unsigned)
 836 defm SQSUBvvv :  NeonI_3VSame_BHSD_sizes<0b0, 0b00101, "sqsub",
 837                    int_arm_neon_vqsubs, 1>;
 838 defm UQSUBvvv :  NeonI_3VSame_BHSD_sizes<0b1, 0b00101, "uqsub",
 839                    int_arm_neon_vqsubu, 1>;
 840
 841 // Vector Shift Left (Signed and Unsigned Integer)
 842 defm SSHLvvv : NeonI_3VSame_BHSD_sizes<0b0, 0b01000, "sshl",
 843                  int_arm_neon_vshifts, 1>;
 844 defm USHLvvv : NeonI_3VSame_BHSD_sizes<0b1, 0b01000, "ushl",
 845                  int_arm_neon_vshiftu, 1>;
 846
 847 // Vector Saturating Shift Left (Signed and Unsigned Integer)
 848 defm SQSHLvvv : NeonI_3VSame_BHSD_sizes<0b0, 0b01001, "sqshl",
 849                   int_arm_neon_vqshifts, 1>;
 850 defm UQSHLvvv : NeonI_3VSame_BHSD_sizes<0b1, 0b01001, "uqshl",
 851                   int_arm_neon_vqshiftu, 1>;
 852
 853 // Vector Rouding Shift Left (Signed and Unsigned Integer)
 854 defm SRSHLvvv : NeonI_3VSame_BHSD_sizes<0b0, 0b01010, "srshl",
 855                   int_arm_neon_vrshifts, 1>;
 856 defm URSHLvvv : NeonI_3VSame_BHSD_sizes<0b1, 0b01010, "urshl",
 857                   int_arm_neon_vrshiftu, 1>;
 858
 859 // Vector Saturating Rouding Shift Left (Signed and Unsigned Integer)
 860 defm SQRSHLvvv : NeonI_3VSame_BHSD_sizes<0b0, 0b01011, "sqrshl",
 861                    int_arm_neon_vqrshifts, 1>;
 862 defm UQRSHLvvv : NeonI_3VSame_BHSD_sizes<0b1, 0b01011, "uqrshl",
 863                    int_arm_neon_vqrshiftu, 1>;
 864
 865 // Vector Maximum (Signed and Unsigned Integer)
 866 defm SMAXvvv : NeonI_3VSame_BHS_sizes<0b0, 0b01100, "smax", int_arm_neon_vmaxs, 1>;
 867 defm UMAXvvv : NeonI_3VSame_BHS_sizes<0b1, 0b01100, "umax", int_arm_neon_vmaxu, 1>;
 868
 869 // Vector Minimum (Signed and Unsigned Integer)
 870 defm SMINvvv : NeonI_3VSame_BHS_sizes<0b0, 0b01101, "smin", int_arm_neon_vmins, 1>;
 871 defm UMINvvv : NeonI_3VSame_BHS_sizes<0b1, 0b01101, "umin", int_arm_neon_vminu, 1>;
 872
 873 // Vector Maximum (Floating Point)
 874 defm FMAXvvv : NeonI_3VSame_SD_sizes<0b0, 0b0, 0b11110, "fmax",
 875                                      int_arm_neon_vmaxs, int_arm_neon_vmaxs,
 876                                      int_arm_neon_vmaxs, v2f32, v4f32, v2f64, 1>;
 877
 878 // Vector Minimum (Floating Point)
 879 defm FMINvvv : NeonI_3VSame_SD_sizes<0b0, 0b1, 0b11110, "fmin",
 880                                      int_arm_neon_vmins, int_arm_neon_vmins,
 881                                      int_arm_neon_vmins, v2f32, v4f32, v2f64, 1>;
 882
 883 // Vector maxNum (Floating Point) -  prefer a number over a quiet NaN)
 884 defm FMAXNMvvv : NeonI_3VSame_SD_sizes<0b0, 0b0, 0b11000, "fmaxnm",
 885                                        int_aarch64_neon_vmaxnm,
 886                                        int_aarch64_neon_vmaxnm,
 887                                        int_aarch64_neon_vmaxnm,
 888                                        v2f32, v4f32, v2f64, 1>;
 889
 890 // Vector minNum (Floating Point) - prefer a number over a quiet NaN)
 891 defm FMINNMvvv : NeonI_3VSame_SD_sizes<0b0, 0b1, 0b11000, "fminnm",
 892                                        int_aarch64_neon_vminnm,
 893                                        int_aarch64_neon_vminnm,
 894                                        int_aarch64_neon_vminnm,
 895                                        v2f32, v4f32, v2f64, 1>;
 896
 897 // Vector Maximum Pairwise (Signed and Unsigned Integer)
 898 defm SMAXPvvv : NeonI_3VSame_BHS_sizes<0b0, 0b10100, "smaxp", int_arm_neon_vpmaxs, 1>;
 899 defm UMAXPvvv : NeonI_3VSame_BHS_sizes<0b1, 0b10100, "umaxp", int_arm_neon_vpmaxu, 1>;
 900
 901 // Vector Minimum Pairwise (Signed and Unsigned Integer)
 902 defm SMINPvvv : NeonI_3VSame_BHS_sizes<0b0, 0b10101, "sminp", int_arm_neon_vpmins, 1>;
 903 defm UMINPvvv : NeonI_3VSame_BHS_sizes<0b1, 0b10101, "uminp", int_arm_neon_vpminu, 1>;
 904
 905 // Vector Maximum Pairwise (Floating Point)
 906 defm FMAXPvvv : NeonI_3VSame_SD_sizes<0b1, 0b0, 0b11110, "fmaxp",
 907                                      int_arm_neon_vpmaxs, int_arm_neon_vpmaxs,
 908                                      int_arm_neon_vpmaxs, v2f32, v4f32, v2f64, 1>;
 909
 910 // Vector Minimum Pairwise (Floating Point)
 911 defm FMINPvvv : NeonI_3VSame_SD_sizes<0b1, 0b1, 0b11110, "fminp",
 912                                      int_arm_neon_vpmins, int_arm_neon_vpmins,
 913                                      int_arm_neon_vpmins, v2f32, v4f32, v2f64, 1>;
 914
 915 // Vector maxNum Pairwise (Floating Point) -  prefer a number over a quiet NaN)
 916 defm FMAXNMPvvv : NeonI_3VSame_SD_sizes<0b1, 0b0, 0b11000, "fmaxnmp",
 917                                        int_aarch64_neon_vpmaxnm,
 918                                        int_aarch64_neon_vpmaxnm,
 919                                        int_aarch64_neon_vpmaxnm,
 920                                        v2f32, v4f32, v2f64, 1>;
 921
 922 // Vector minNum Pairwise (Floating Point) -  prefer a number over a quiet NaN)
 923 defm FMINNMPvvv : NeonI_3VSame_SD_sizes<0b1, 0b1, 0b11000, "fminnmp",
 924                                        int_aarch64_neon_vpminnm,
 925                                        int_aarch64_neon_vpminnm,
 926                                        int_aarch64_neon_vpminnm,
 927                                        v2f32, v4f32, v2f64, 1>;
 928
 929 // Vector Addition Pairwise (Integer)
 930 defm ADDP : NeonI_3VSame_BHSD_sizes<0b0, 0b10111, "addp", int_arm_neon_vpadd, 1>;
 931
 932 // Vector Addition Pairwise (Floating Point)
 933 defm FADDP : NeonI_3VSame_SD_sizes<0b1, 0b0, 0b11010, "faddp",
 934                                        int_arm_neon_vpadd,
 935                                        int_arm_neon_vpadd,
 936                                        int_arm_neon_vpadd,
 937                                        v2f32, v4f32, v2f64, 1>;
 938
 939 // Vector Saturating Doubling Multiply High
 940 defm SQDMULHvvv : NeonI_3VSame_HS_sizes<0b0, 0b10110, "sqdmulh",
 941                     int_arm_neon_vqdmulh, 1>;
 942
 943 // Vector Saturating Rouding Doubling Multiply High
 944 defm SQRDMULHvvv : NeonI_3VSame_HS_sizes<0b1, 0b10110, "sqrdmulh",
 945                      int_arm_neon_vqrdmulh, 1>;
 946
 947 // Vector Multiply Extended (Floating Point)
 948 defm FMULXvvv : NeonI_3VSame_SD_sizes<0b0, 0b0, 0b11011, "fmulx",
 949                                       int_aarch64_neon_vmulx,
 950                                       int_aarch64_neon_vmulx,
 951                                       int_aarch64_neon_vmulx,
 952                                       v2f32, v4f32, v2f64, 1>;
 953
 954 // Vector Immediate Instructions
 955
 956 multiclass neon_mov_imm_shift_asmoperands<string PREFIX>
 957 {
 958   def _asmoperand : AsmOperandClass
 959     {
 960       let Name = "NeonMovImmShift" # PREFIX;
 961       let RenderMethod = "addNeonMovImmShift" # PREFIX # "Operands";
 962       let PredicateMethod = "isNeonMovImmShift" # PREFIX;
 963     }
 964 }
 965
 966 // Definition of vector immediates shift operands
 967
 968 // The selectable use-cases extract the shift operation
 969 // information from the OpCmode fields encoded in the immediate.
 970 def neon_mod_shift_imm_XFORM : SDNodeXForm<imm, [{
 971   uint64_t OpCmode = N->getZExtValue();
 972   unsigned ShiftImm;
 973   unsigned ShiftOnesIn;
 974   unsigned HasShift =
 975     A64Imms::decodeNeonModShiftImm(OpCmode, ShiftImm, ShiftOnesIn);
 976   if (!HasShift) return SDValue();
 977   return CurDAG->getTargetConstant(ShiftImm, MVT::i32);
 978 }]>;
 979
 980 // Vector immediates shift operands which accept LSL and MSL
 981 // shift operators with shift value in the range of 0, 8, 16, 24 (LSL),
 982 // or 0, 8 (LSLH) or 8, 16 (MSL).
 983 defm neon_mov_imm_LSL : neon_mov_imm_shift_asmoperands<"LSL">;
 984 defm neon_mov_imm_MSL : neon_mov_imm_shift_asmoperands<"MSL">;
 985 // LSLH restricts shift amount to  0, 8 out of 0, 8, 16, 24
 986 defm neon_mov_imm_LSLH : neon_mov_imm_shift_asmoperands<"LSLH">;
 987
 988 multiclass neon_mov_imm_shift_operands<string PREFIX,
 989                                        string HALF, string ISHALF, code pred>
 990 {
 991    def _operand : Operand<i32>, ImmLeaf<i32, pred, neon_mod_shift_imm_XFORM>
 992     {
 993       let PrintMethod =
 994         "printNeonMovImmShiftOperand<A64SE::" # PREFIX # ", " # ISHALF # ">";
 995       let DecoderMethod =
 996         "DecodeNeonMovImmShiftOperand<A64SE::" # PREFIX # ", " # ISHALF # ">";
 997       let ParserMatchClass =
 998         !cast<AsmOperandClass>("neon_mov_imm_" # PREFIX # HALF # "_asmoperand");
 999     }
1000 }
1001
1002 defm neon_mov_imm_LSL  : neon_mov_imm_shift_operands<"LSL", "", "false", [{
1003   unsigned ShiftImm;
1004   unsigned ShiftOnesIn;
1005   unsigned HasShift =
1006     A64Imms::decodeNeonModShiftImm(Imm, ShiftImm, ShiftOnesIn);
1007   return (HasShift && !ShiftOnesIn);
1008 }]>;
1009
1010 defm neon_mov_imm_MSL  : neon_mov_imm_shift_operands<"MSL", "", "false", [{
1011   unsigned ShiftImm;
1012   unsigned ShiftOnesIn;
1013   unsigned HasShift =
1014     A64Imms::decodeNeonModShiftImm(Imm, ShiftImm, ShiftOnesIn);
1015   return (HasShift && ShiftOnesIn);
1016 }]>;
1017
1018 defm neon_mov_imm_LSLH  : neon_mov_imm_shift_operands<"LSL", "H", "true", [{
1019   unsigned ShiftImm;
1020   unsigned ShiftOnesIn;
1021   unsigned HasShift =
1022     A64Imms::decodeNeonModShiftImm(Imm, ShiftImm, ShiftOnesIn);
1023   return (HasShift && !ShiftOnesIn);
1024 }]>;
1025
1026 def neon_uimm8_asmoperand : AsmOperandClass
1027 {
1028   let Name = "UImm8";
1029   let PredicateMethod = "isUImm<8>";
1030   let RenderMethod = "addImmOperands";
1031 }
1032
1033 def neon_uimm8 : Operand<i32>, ImmLeaf<i32, [{(void)Imm; return true;}]> {
1034   let ParserMatchClass = neon_uimm8_asmoperand;
1035   let PrintMethod = "printNeonUImm8Operand";
1036 }
1037
1038 def neon_uimm64_mask_asmoperand : AsmOperandClass
1039 {
1040   let Name = "NeonUImm64Mask";
1041   let PredicateMethod = "isNeonUImm64Mask";
1042   let RenderMethod = "addNeonUImm64MaskOperands";
1043 }
1044
1045 // MCOperand for 64-bit bytemask with each byte having only the
1046 // value 0x00 and 0xff is encoded as an unsigned 8-bit value
1047 def neon_uimm64_mask : Operand<i32>, ImmLeaf<i32, [{(void)Imm; return true;}]> {
1048   let ParserMatchClass = neon_uimm64_mask_asmoperand;
1049   let PrintMethod = "printNeonUImm64MaskOperand";
1050 }
1051
1052 multiclass NeonI_mov_imm_lsl_sizes<string asmop, bit op,
1053                                    SDPatternOperator opnode>
1054 {
1055     // shift zeros, per word
1056     def _2S  : NeonI_1VModImm<0b0, op,
1057                               (outs VPR64:$Rd),
1058                               (ins neon_uimm8:$Imm,
1059                                 neon_mov_imm_LSL_operand:$Simm),
1060                               !strconcat(asmop, " $Rd.2s, $Imm$Simm"),
1061                               [(set (v2i32 VPR64:$Rd),
1062                                  (v2i32 (opnode (timm:$Imm),
1063                                    (neon_mov_imm_LSL_operand:$Simm))))],
1064                               NoItinerary> {
1065        bits<2> Simm;
1066        let cmode = {0b0, Simm{1}, Simm{0}, 0b0};
1067      }
1068
1069     def _4S  : NeonI_1VModImm<0b1, op,
1070                               (outs VPR128:$Rd),
1071                               (ins neon_uimm8:$Imm,
1072                                 neon_mov_imm_LSL_operand:$Simm),
1073                               !strconcat(asmop, " $Rd.4s, $Imm$Simm"),
1074                               [(set (v4i32 VPR128:$Rd),
1075                                  (v4i32 (opnode (timm:$Imm),
1076                                    (neon_mov_imm_LSL_operand:$Simm))))],
1077                               NoItinerary> {
1078       bits<2> Simm;
1079       let cmode = {0b0, Simm{1}, Simm{0}, 0b0};
1080     }
1081
1082     // shift zeros, per halfword
1083     def _4H  : NeonI_1VModImm<0b0, op,
1084                               (outs VPR64:$Rd),
1085                               (ins neon_uimm8:$Imm,
1086                                 neon_mov_imm_LSLH_operand:$Simm),
1087                               !strconcat(asmop, " $Rd.4h, $Imm$Simm"),
1088                               [(set (v4i16 VPR64:$Rd),
1089                                  (v4i16 (opnode (timm:$Imm),
1090                                    (neon_mov_imm_LSLH_operand:$Simm))))],
1091                               NoItinerary> {
1092       bit  Simm;
1093       let cmode = {0b1, 0b0, Simm, 0b0};
1094     }
1095
1096     def _8H  : NeonI_1VModImm<0b1, op,
1097                               (outs VPR128:$Rd),
1098                               (ins neon_uimm8:$Imm,
1099                                 neon_mov_imm_LSLH_operand:$Simm),
1100                               !strconcat(asmop, " $Rd.8h, $Imm$Simm"),
1101                               [(set (v8i16 VPR128:$Rd),
1102                                  (v8i16 (opnode (timm:$Imm),
1103                                    (neon_mov_imm_LSLH_operand:$Simm))))],
1104                               NoItinerary> {
1105       bit Simm;
1106       let cmode = {0b1, 0b0, Simm, 0b0};
1107      }
1108 }
1109
1110 multiclass NeonI_mov_imm_with_constraint_lsl_sizes<string asmop, bit op,
1111                                                    SDPatternOperator opnode,
1112                                                    SDPatternOperator neonopnode>
1113 {
1114   let Constraints = "$src = $Rd" in {
1115     // shift zeros, per word
1116     def _2S  : NeonI_1VModImm<0b0, op,
1117                  (outs VPR64:$Rd),
1118                  (ins VPR64:$src, neon_uimm8:$Imm,
1119                    neon_mov_imm_LSL_operand:$Simm),
1120                  !strconcat(asmop, " $Rd.2s, $Imm$Simm"),
1121                  [(set (v2i32 VPR64:$Rd),
1122                     (v2i32 (opnode (v2i32 VPR64:$src),
1123                       (v2i32 (bitconvert (v2i32 (neonopnode timm:$Imm,
1124                         neon_mov_imm_LSL_operand:$Simm)))))))],
1125                  NoItinerary> {
1126       bits<2> Simm;
1127       let cmode = {0b0, Simm{1}, Simm{0}, 0b1};
1128     }
1129
1130     def _4S  : NeonI_1VModImm<0b1, op,
1131                  (outs VPR128:$Rd),
1132                  (ins VPR128:$src, neon_uimm8:$Imm,
1133                    neon_mov_imm_LSL_operand:$Simm),
1134                  !strconcat(asmop, " $Rd.4s, $Imm$Simm"),
1135                  [(set (v4i32 VPR128:$Rd),
1136                     (v4i32 (opnode (v4i32 VPR128:$src),
1137                       (v4i32 (bitconvert (v4i32 (neonopnode timm:$Imm,
1138                         neon_mov_imm_LSL_operand:$Simm)))))))],
1139                  NoItinerary> {
1140       bits<2> Simm;
1141       let cmode = {0b0, Simm{1}, Simm{0}, 0b1};
1142     }
1143
1144     // shift zeros, per halfword
1145     def _4H  : NeonI_1VModImm<0b0, op,
1146                  (outs VPR64:$Rd),
1147                  (ins VPR64:$src, neon_uimm8:$Imm,
1148                    neon_mov_imm_LSLH_operand:$Simm),
1149                  !strconcat(asmop, " $Rd.4h, $Imm$Simm"),
1150                  [(set (v4i16 VPR64:$Rd),
1151                     (v4i16 (opnode (v4i16 VPR64:$src),
1152                        (v4i16 (bitconvert (v4i16 (neonopnode timm:$Imm,
1153                           neon_mov_imm_LSL_operand:$Simm)))))))],
1154                  NoItinerary> {
1155       bit  Simm;
1156       let cmode = {0b1, 0b0, Simm, 0b1};
1157     }
1158
1159     def _8H  : NeonI_1VModImm<0b1, op,
1160                  (outs VPR128:$Rd),
1161                  (ins VPR128:$src, neon_uimm8:$Imm,
1162                    neon_mov_imm_LSLH_operand:$Simm),
1163                  !strconcat(asmop, " $Rd.8h, $Imm$Simm"),
1164                  [(set (v8i16 VPR128:$Rd),
1165                     (v8i16 (opnode (v8i16 VPR128:$src),
1166                       (v8i16 (bitconvert (v8i16 (neonopnode timm:$Imm,
1167                         neon_mov_imm_LSL_operand:$Simm)))))))],
1168                  NoItinerary> {
1169       bit Simm;
1170       let cmode = {0b1, 0b0, Simm, 0b1};
1171     }
1172   }
1173 }
1174
1175 multiclass NeonI_mov_imm_msl_sizes<string asmop, bit op,
1176                                    SDPatternOperator opnode>
1177 {
1178     // shift ones, per word
1179     def _2S  : NeonI_1VModImm<0b0, op,
1180                              (outs VPR64:$Rd),
1181                              (ins neon_uimm8:$Imm,
1182                                neon_mov_imm_MSL_operand:$Simm),
1183                              !strconcat(asmop, " $Rd.2s, $Imm$Simm"),
1184                               [(set (v2i32 VPR64:$Rd),
1185                                  (v2i32 (opnode (timm:$Imm),
1186                                    (neon_mov_imm_MSL_operand:$Simm))))],
1187                              NoItinerary> {
1188        bit Simm;
1189        let cmode = {0b1, 0b1, 0b0, Simm};
1190      }
1191
1192    def _4S  : NeonI_1VModImm<0b1, op,
1193                               (outs VPR128:$Rd),
1194                               (ins neon_uimm8:$Imm,
1195                                 neon_mov_imm_MSL_operand:$Simm),
1196                               !strconcat(asmop, " $Rd.4s, $Imm$Simm"),
1197                               [(set (v4i32 VPR128:$Rd),
1198                                  (v4i32 (opnode (timm:$Imm),
1199                                    (neon_mov_imm_MSL_operand:$Simm))))],
1200                               NoItinerary> {
1201      bit Simm;
1202      let cmode = {0b1, 0b1, 0b0, Simm};
1203    }
1204 }
1205
1206 // Vector Move Immediate Shifted
1207 let isReMaterializable = 1 in {
1208 defm MOVIvi_lsl : NeonI_mov_imm_lsl_sizes<"movi", 0b0, Neon_movi>;
1209 }
1210
1211 // Vector Move Inverted Immediate Shifted
1212 let isReMaterializable = 1 in {
1213 defm MVNIvi_lsl : NeonI_mov_imm_lsl_sizes<"mvni", 0b1, Neon_mvni>;
1214 }
1215
1216 // Vector Bitwise Bit Clear (AND NOT) - immediate
1217 let isReMaterializable = 1 in {
1218 defm BICvi_lsl : NeonI_mov_imm_with_constraint_lsl_sizes<"bic", 0b1,
1219                                                          and, Neon_mvni>;
1220 }
1221
1222 // Vector Bitwise OR - immedidate
1223
1224 let isReMaterializable = 1 in {
1225 defm ORRvi_lsl   : NeonI_mov_imm_with_constraint_lsl_sizes<"orr", 0b0,
1226                                                            or, Neon_movi>;
1227 }
1228
1229 // Additional patterns for Vector Bitwise Bit Clear (AND NOT) - immedidate
1230 // LowerBUILD_VECTOR favors lowering MOVI over MVNI.
1231 // BIC immediate instructions selection requires additional patterns to
1232 // transform Neon_movi operands into BIC immediate operands
1233
1234 def neon_mov_imm_LSLH_transform_XFORM : SDNodeXForm<imm, [{
1235   uint64_t OpCmode = N->getZExtValue();
1236   unsigned ShiftImm;
1237   unsigned ShiftOnesIn;
1238   (void)A64Imms::decodeNeonModShiftImm(OpCmode, ShiftImm, ShiftOnesIn);
1239   // LSLH restricts shift amount to  0, 8 which are encoded as 0 and 1
1240   // Transform encoded shift amount 0 to 1 and 1 to 0.
1241   return CurDAG->getTargetConstant(!ShiftImm, MVT::i32);
1242 }]>;
1243
1244 def neon_mov_imm_LSLH_transform_operand
1245   : ImmLeaf<i32, [{
1246     unsigned ShiftImm;
1247     unsigned ShiftOnesIn;
1248     unsigned HasShift =
1249       A64Imms::decodeNeonModShiftImm(Imm, ShiftImm, ShiftOnesIn);
1250     return (HasShift && !ShiftOnesIn); }],
1251   neon_mov_imm_LSLH_transform_XFORM>;
1252
1253 // Transform (and A, (4h Neon_movi 0xff)) -> BIC 4h (A, 0x00, LSL 8)
1254 // Transform (and A, (4h Neon_movi 0xff LSL #8)) -> BIC 4h (A, 0x00)
1255 def : Pat<(v4i16 (and VPR64:$src,
1256             (v4i16 (Neon_movi 255, neon_mov_imm_LSLH_transform_operand:$Simm)))),
1257           (BICvi_lsl_4H VPR64:$src, 0,
1258             neon_mov_imm_LSLH_transform_operand:$Simm)>;
1259
1260 // Transform (and A, (8h Neon_movi 8h 0xff)) -> BIC 8h (A, 0x00, LSL 8)
1261 // Transform (and A, (8h Neon_movi 0xff LSL #8)) -> BIC 8h (A, 0x00)
1262 def : Pat<(v8i16 (and VPR128:$src,
1263             (v8i16 (Neon_movi 255, neon_mov_imm_LSLH_transform_operand:$Simm)))),
1264           (BICvi_lsl_8H VPR128:$src, 0,
1265             neon_mov_imm_LSLH_transform_operand:$Simm)>;
1266
1267
1268 multiclass Neon_bitwiseVi_patterns<SDPatternOperator opnode,
1269                                    SDPatternOperator neonopnode,
1270                                    Instruction INST4H,
1271                                    Instruction INST8H> {
1272   def : Pat<(v8i8 (opnode VPR64:$src,
1273                     (bitconvert(v4i16 (neonopnode timm:$Imm,
1274                       neon_mov_imm_LSLH_operand:$Simm))))),
1275             (INST4H VPR64:$src, neon_uimm8:$Imm,
1276               neon_mov_imm_LSLH_operand:$Simm)>;
1277   def : Pat<(v1i64 (opnode VPR64:$src,
1278                   (bitconvert(v4i16 (neonopnode timm:$Imm,
1279                     neon_mov_imm_LSLH_operand:$Simm))))),
1280           (INST4H VPR64:$src, neon_uimm8:$Imm,
1281             neon_mov_imm_LSLH_operand:$Simm)>;
1282
1283   def : Pat<(v16i8 (opnode VPR128:$src,
1284                    (bitconvert(v8i16 (neonopnode timm:$Imm,
1285                      neon_mov_imm_LSLH_operand:$Simm))))),
1286           (INST8H VPR128:$src, neon_uimm8:$Imm,
1287             neon_mov_imm_LSLH_operand:$Simm)>;
1288   def : Pat<(v4i32 (opnode VPR128:$src,
1289                    (bitconvert(v8i16 (neonopnode timm:$Imm,
1290                      neon_mov_imm_LSLH_operand:$Simm))))),
1291           (INST8H VPR128:$src, neon_uimm8:$Imm,
1292             neon_mov_imm_LSLH_operand:$Simm)>;
1293   def : Pat<(v2i64 (opnode VPR128:$src,
1294                    (bitconvert(v8i16 (neonopnode timm:$Imm,
1295                      neon_mov_imm_LSLH_operand:$Simm))))),
1296           (INST8H VPR128:$src, neon_uimm8:$Imm,
1297             neon_mov_imm_LSLH_operand:$Simm)>;
1298 }
1299
1300 // Additional patterns for Vector Vector Bitwise Bit Clear (AND NOT) - immediate
1301 defm : Neon_bitwiseVi_patterns<or, Neon_mvni, BICvi_lsl_4H, BICvi_lsl_8H>;
1302
1303 // Additional patterns for Vector Bitwise OR - immedidate
1304 defm : Neon_bitwiseVi_patterns<or, Neon_movi, ORRvi_lsl_4H, ORRvi_lsl_8H>;
1305
1306
1307 // Vector Move Immediate Masked
1308 let isReMaterializable = 1 in {
1309 defm MOVIvi_msl : NeonI_mov_imm_msl_sizes<"movi", 0b0, Neon_movi>;
1310 }
1311
1312 // Vector Move Inverted Immediate Masked
1313 let isReMaterializable = 1 in {
1314 defm MVNIvi_msl : NeonI_mov_imm_msl_sizes<"mvni", 0b1, Neon_mvni>;
1315 }
1316
1317 class NeonI_mov_imm_lsl_aliases<string asmop, string asmlane,
1318                                 Instruction inst, RegisterClass VPRC>
1319   : NeonInstAlias<!strconcat(asmop, " $Rd," # asmlane # ", $Imm"),
1320                         (inst VPRC:$Rd, neon_uimm8:$Imm,  0), 0b0>;
1321
1322 // Aliases for Vector Move Immediate Shifted
1323 def : NeonI_mov_imm_lsl_aliases<"movi", ".2s", MOVIvi_lsl_2S, VPR64>;
1324 def : NeonI_mov_imm_lsl_aliases<"movi", ".4s", MOVIvi_lsl_4S, VPR128>;
1325 def : NeonI_mov_imm_lsl_aliases<"movi", ".4h", MOVIvi_lsl_4H, VPR64>;
1326 def : NeonI_mov_imm_lsl_aliases<"movi", ".8h", MOVIvi_lsl_8H, VPR128>;
1327
1328 // Aliases for Vector Move Inverted Immediate Shifted
1329 def : NeonI_mov_imm_lsl_aliases<"mvni", ".2s", MVNIvi_lsl_2S, VPR64>;
1330 def : NeonI_mov_imm_lsl_aliases<"mvni", ".4s", MVNIvi_lsl_4S, VPR128>;
1331 def : NeonI_mov_imm_lsl_aliases<"mvni", ".4h", MVNIvi_lsl_4H, VPR64>;
1332 def : NeonI_mov_imm_lsl_aliases<"mvni", ".8h", MVNIvi_lsl_8H, VPR128>;
1333
1334 // Aliases for Vector Bitwise Bit Clear (AND NOT) - immediate
1335 def : NeonI_mov_imm_lsl_aliases<"bic", ".2s", BICvi_lsl_2S, VPR64>;
1336 def : NeonI_mov_imm_lsl_aliases<"bic", ".4s", BICvi_lsl_4S, VPR128>;
1337 def : NeonI_mov_imm_lsl_aliases<"bic", ".4h", BICvi_lsl_4H, VPR64>;
1338 def : NeonI_mov_imm_lsl_aliases<"bic", ".8h", BICvi_lsl_8H, VPR128>;
1339
1340 // Aliases for Vector Bitwise OR - immedidate
1341 def : NeonI_mov_imm_lsl_aliases<"orr", ".2s", ORRvi_lsl_2S, VPR64>;
1342 def : NeonI_mov_imm_lsl_aliases<"orr", ".4s", ORRvi_lsl_4S, VPR128>;
1343 def : NeonI_mov_imm_lsl_aliases<"orr", ".4h", ORRvi_lsl_4H, VPR64>;
1344 def : NeonI_mov_imm_lsl_aliases<"orr", ".8h", ORRvi_lsl_8H, VPR128>;
1345
1346 //  Vector Move Immediate - per byte
1347 let isReMaterializable = 1 in {
1348 def MOVIvi_8B : NeonI_1VModImm<0b0, 0b0,
1349                                (outs VPR64:$Rd), (ins neon_uimm8:$Imm),
1350                                "movi\t$Rd.8b, $Imm",
1351                                [(set (v8i8 VPR64:$Rd),
1352                                   (v8i8 (Neon_movi (timm:$Imm), (i32 imm))))],
1353                                 NoItinerary> {
1354   let cmode = 0b1110;
1355 }
1356
1357 def MOVIvi_16B : NeonI_1VModImm<0b1, 0b0,
1358                                 (outs VPR128:$Rd), (ins neon_uimm8:$Imm),
1359                                 "movi\t$Rd.16b, $Imm",
1360                                 [(set (v16i8 VPR128:$Rd),
1361                                    (v16i8 (Neon_movi (timm:$Imm), (i32 imm))))],
1362                                  NoItinerary> {
1363   let cmode = 0b1110;
1364 }
1365 }
1366
1367 // Vector Move Immediate - bytemask, per double word
1368 let isReMaterializable = 1 in {
1369 def MOVIvi_2D : NeonI_1VModImm<0b1, 0b1,
1370                                (outs VPR128:$Rd), (ins neon_uimm64_mask:$Imm),
1371                                "movi\t $Rd.2d, $Imm",
1372                                [(set (v2i64 VPR128:$Rd),
1373                                   (v2i64 (Neon_movi (timm:$Imm), (i32 imm))))],
1374                                NoItinerary> {
1375   let cmode = 0b1110;
1376 }
1377 }
1378
1379 // Vector Move Immediate - bytemask, one doubleword
1380
1381 let isReMaterializable = 1 in {
1382 def MOVIdi : NeonI_1VModImm<0b0, 0b1,
1383                            (outs FPR64:$Rd), (ins neon_uimm64_mask:$Imm),
1384                            "movi\t $Rd, $Imm",
1385                            [(set (f64 FPR64:$Rd),
1386                               (f64 (bitconvert
1387                                 (v1i64 (Neon_movi (timm:$Imm), (i32 imm))))))],
1388                            NoItinerary> {
1389   let cmode = 0b1110;
1390 }
1391 }
1392
1393 // Vector Floating Point Move Immediate
1394
1395 class NeonI_FMOV_impl<string asmlane, RegisterClass VPRC, ValueType OpTy,
1396                       Operand immOpType, bit q, bit op>
1397   : NeonI_1VModImm<q, op,
1398                    (outs VPRC:$Rd), (ins immOpType:$Imm),
1399                    "fmov\t$Rd" # asmlane # ", $Imm",
1400                    [(set (OpTy VPRC:$Rd),
1401                       (OpTy (Neon_fmovi (timm:$Imm))))],
1402                    NoItinerary> {
1403      let cmode = 0b1111;
1404    }
1405
1406 let isReMaterializable = 1 in {
1407 def FMOVvi_2S : NeonI_FMOV_impl<".2s", VPR64,  v2f32, fmov32_operand, 0b0, 0b0>;
1408 def FMOVvi_4S : NeonI_FMOV_impl<".4s", VPR128, v4f32, fmov32_operand, 0b1, 0b0>;
1409 def FMOVvi_2D : NeonI_FMOV_impl<".2d", VPR128, v2f64, fmov64_operand, 0b1, 0b1>;
1410 }
1411
1412 // Scalar Arithmetic
1413
1414 class NeonI_Scalar3Same_D_size<bit u, bits<5> opcode, string asmop>
1415   : NeonI_Scalar3Same<u, 0b11, opcode,
1416                 (outs FPR64:$Rd), (ins FPR64:$Rn, FPR64:$Rm),
1417                 !strconcat(asmop, " $Rd, $Rn, $Rm"),
1418                 [],
1419                 NoItinerary>;
1420
1421 multiclass NeonI_Scalar3Same_BHSD_sizes<bit u, bits<5> opcode,
1422                                         string asmop, bit Commutable = 0>
1423 {
1424   let isCommutable = Commutable in {
1425     def bbb : NeonI_Scalar3Same<u, 0b00, opcode,
1426                                 (outs FPR8:$Rd), (ins FPR8:$Rn, FPR8:$Rm),
1427                                 !strconcat(asmop, " $Rd, $Rn, $Rm"),
1428                                 [],
1429                                 NoItinerary>;
1430     def hhh : NeonI_Scalar3Same<u, 0b01, opcode,
1431                                 (outs FPR16:$Rd), (ins FPR16:$Rn, FPR16:$Rm),
1432                                 !strconcat(asmop, " $Rd, $Rn, $Rm"),
1433                                 [],
1434                                 NoItinerary>;
1435     def sss : NeonI_Scalar3Same<u, 0b10, opcode,
1436                                 (outs FPR32:$Rd), (ins FPR32:$Rn, FPR32:$Rm),
1437                                 !strconcat(asmop, " $Rd, $Rn, $Rm"),
1438                                 [],
1439                                 NoItinerary>;
1440     def ddd : NeonI_Scalar3Same<u, 0b11, opcode,
1441                                (outs FPR64:$Rd), (ins FPR64:$Rn, FPR64:$Rm),
1442                                !strconcat(asmop, " $Rd, $Rn, $Rm"),
1443                                [],
1444                                NoItinerary>;
1445   }
1446 }
1447
1448 class Neon_Scalar_D_size_patterns<SDPatternOperator opnode, Instruction INSTD>
1449   : Pat<(v1i64 (opnode (v1i64 VPR64:$Rn), (v1i64 VPR64:$Rm))),
1450         (SUBREG_TO_REG (i64 0),
1451               (INSTD (EXTRACT_SUBREG VPR64:$Rn, sub_64),
1452              (EXTRACT_SUBREG VPR64:$Rm, sub_64)),
1453           sub_64)>;
1454
1455
1456 // Scalar Integer Add
1457 let isCommutable = 1 in {
1458 def ADDddd : NeonI_Scalar3Same_D_size<0b0, 0b10000, "add">;
1459 }
1460
1461 // Scalar Integer Sub
1462 def SUBddd : NeonI_Scalar3Same_D_size<0b1, 0b10000, "sub">;
1463
1464 // Pattern for Scalar Integer Add and Sub with D register
1465 def : Neon_Scalar_D_size_patterns<add, ADDddd>;
1466 def : Neon_Scalar_D_size_patterns<sub, SUBddd>;
1467
1468 // Scalar Integer Saturating Add (Signed, Unsigned)
1469 defm SQADD : NeonI_Scalar3Same_BHSD_sizes<0b0, 0b00001, "sqadd", 1>;
1470 defm UQADD : NeonI_Scalar3Same_BHSD_sizes<0b1, 0b00001, "uqadd", 1>;
1471
1472 // Scalar Integer Saturating Sub (Signed, Unsigned)
1473 defm SQSUB : NeonI_Scalar3Same_BHSD_sizes<0b0, 0b00101, "sqsub", 0>;
1474 defm UQSUB : NeonI_Scalar3Same_BHSD_sizes<0b1, 0b00101, "uqsub", 0>;
1475
1476 // Patterns for Scalar Integer Saturating Add, Sub with D register only
1477 def : Neon_Scalar_D_size_patterns<int_arm_neon_vqadds, SQADDddd>;
1478 def : Neon_Scalar_D_size_patterns<int_arm_neon_vqaddu, UQADDddd>;
1479 def : Neon_Scalar_D_size_patterns<int_arm_neon_vqsubs, SQSUBddd>;
1480 def : Neon_Scalar_D_size_patterns<int_arm_neon_vqsubu, UQSUBddd>;
1481
1482 // Scalar Integer Shift Left (Signed, Unsigned)
1483 def SSHLddd : NeonI_Scalar3Same_D_size<0b0, 0b01000, "sshl">;
1484 def USHLddd : NeonI_Scalar3Same_D_size<0b1, 0b01000, "ushl">;
1485
1486 // Scalar Integer Saturating Shift Left (Signed, Unsigned)
1487 defm SQSHL: NeonI_Scalar3Same_BHSD_sizes<0b0, 0b01001, "sqshl", 0>;
1488 defm UQSHL: NeonI_Scalar3Same_BHSD_sizes<0b1, 0b01001, "uqshl", 0>;
1489
1490 // Scalar Integer Rouding Shift Left (Signed, Unsigned)
1491 def SRSHLddd: NeonI_Scalar3Same_D_size<0b0, 0b01010, "srshl">;
1492 def URSHLddd: NeonI_Scalar3Same_D_size<0b1, 0b01010, "urshl">;
1493
1494 // Scalar Integer Saturating Rounding Shift Left (Signed, Unsigned)
1495 defm SQRSHL: NeonI_Scalar3Same_BHSD_sizes<0b0, 0b01011, "sqrshl", 0>;
1496 defm UQRSHL: NeonI_Scalar3Same_BHSD_sizes<0b1, 0b01011, "uqrshl", 0>;
1497
1498 // Patterns for Scalar Integer Shift Lef, Saturating Shift Left,
1499 // Rounding Shift Left, Rounding Saturating Shift Left with D register only
1500 def : Neon_Scalar_D_size_patterns<int_arm_neon_vshifts, SSHLddd>;
1501 def : Neon_Scalar_D_size_patterns<int_arm_neon_vshiftu, USHLddd>;
1502 def : Neon_Scalar_D_size_patterns<shl, SSHLddd>;
1503 def : Neon_Scalar_D_size_patterns<shl, USHLddd>;
1504 def : Neon_Scalar_D_size_patterns<int_arm_neon_vqshifts, SQSHLddd>;
1505 def : Neon_Scalar_D_size_patterns<int_arm_neon_vqshiftu, UQSHLddd>;
1506 def : Neon_Scalar_D_size_patterns<int_arm_neon_vrshifts, SRSHLddd>;
1507 def : Neon_Scalar_D_size_patterns<int_arm_neon_vrshiftu, URSHLddd>;
1508 def : Neon_Scalar_D_size_patterns<int_arm_neon_vqrshifts, SQRSHLddd>;
1509 def : Neon_Scalar_D_size_patterns<int_arm_neon_vqrshiftu, UQRSHLddd>;
1510
1511
1512 //===----------------------------------------------------------------------===//
1513 // Non-Instruction Patterns
1514 //===----------------------------------------------------------------------===//
1515
1516 // 64-bit vector bitcasts...
1517
1518 def : Pat<(v1i64 (bitconvert (v8i8  VPR64:$src))), (v1i64 VPR64:$src)>;
1519 def : Pat<(v2f32 (bitconvert (v8i8  VPR64:$src))), (v2f32 VPR64:$src)>;
1520 def : Pat<(v2i32 (bitconvert (v8i8  VPR64:$src))), (v2i32 VPR64:$src)>;
1521 def : Pat<(v4i16 (bitconvert (v8i8  VPR64:$src))), (v4i16 VPR64:$src)>;
1522
1523 def : Pat<(v1i64 (bitconvert (v4i16  VPR64:$src))), (v1i64 VPR64:$src)>;
1524 def : Pat<(v2i32 (bitconvert (v4i16  VPR64:$src))), (v2i32 VPR64:$src)>;
1525 def : Pat<(v2f32 (bitconvert (v4i16  VPR64:$src))), (v2f32 VPR64:$src)>;
1526 def : Pat<(v8i8  (bitconvert (v4i16  VPR64:$src))), (v8i8 VPR64:$src)>;
1527
1528 def : Pat<(v1i64 (bitconvert (v2i32  VPR64:$src))), (v1i64 VPR64:$src)>;
1529 def : Pat<(v2f32 (bitconvert (v2i32  VPR64:$src))), (v2f32 VPR64:$src)>;
1530 def : Pat<(v4i16 (bitconvert (v2i32  VPR64:$src))), (v4i16 VPR64:$src)>;
1531 def : Pat<(v8i8  (bitconvert (v2i32  VPR64:$src))), (v8i8 VPR64:$src)>;
1532
1533 def : Pat<(v1i64 (bitconvert (v2f32  VPR64:$src))), (v1i64 VPR64:$src)>;
1534 def : Pat<(v2i32 (bitconvert (v2f32  VPR64:$src))), (v2i32 VPR64:$src)>;
1535 def : Pat<(v4i16 (bitconvert (v2f32  VPR64:$src))), (v4i16 VPR64:$src)>;
1536 def : Pat<(v8i8  (bitconvert (v2f32  VPR64:$src))), (v8i8 VPR64:$src)>;
1537
1538 def : Pat<(v2f32 (bitconvert (v1i64  VPR64:$src))), (v2f32 VPR64:$src)>;
1539 def : Pat<(v2i32 (bitconvert (v1i64  VPR64:$src))), (v2i32 VPR64:$src)>;
1540 def : Pat<(v4i16 (bitconvert (v1i64  VPR64:$src))), (v4i16 VPR64:$src)>;
1541 def : Pat<(v8i8  (bitconvert (v1i64  VPR64:$src))), (v8i8 VPR64:$src)>;
1542
1543 // ..and 128-bit vector bitcasts...
1544
1545 def : Pat<(v2f64 (bitconvert (v16i8  VPR128:$src))), (v2f64 VPR128:$src)>;
1546 def : Pat<(v2i64 (bitconvert (v16i8  VPR128:$src))), (v2i64 VPR128:$src)>;
1547 def : Pat<(v4f32 (bitconvert (v16i8  VPR128:$src))), (v4f32 VPR128:$src)>;
1548 def : Pat<(v4i32 (bitconvert (v16i8  VPR128:$src))), (v4i32 VPR128:$src)>;
1549 def : Pat<(v8i16 (bitconvert (v16i8  VPR128:$src))), (v8i16 VPR128:$src)>;
1550
1551 def : Pat<(v2f64 (bitconvert (v8i16  VPR128:$src))), (v2f64 VPR128:$src)>;
1552 def : Pat<(v2i64 (bitconvert (v8i16  VPR128:$src))), (v2i64 VPR128:$src)>;
1553 def : Pat<(v4i32 (bitconvert (v8i16  VPR128:$src))), (v4i32 VPR128:$src)>;
1554 def : Pat<(v4f32 (bitconvert (v8i16  VPR128:$src))), (v4f32 VPR128:$src)>;
1555 def : Pat<(v16i8 (bitconvert (v8i16  VPR128:$src))), (v16i8 VPR128:$src)>;
1556
1557 def : Pat<(v2f64 (bitconvert (v4i32  VPR128:$src))), (v2f64 VPR128:$src)>;
1558 def : Pat<(v2i64 (bitconvert (v4i32  VPR128:$src))), (v2i64 VPR128:$src)>;
1559 def : Pat<(v4f32 (bitconvert (v4i32  VPR128:$src))), (v4f32 VPR128:$src)>;
1560 def : Pat<(v8i16 (bitconvert (v4i32  VPR128:$src))), (v8i16 VPR128:$src)>;
1561 def : Pat<(v16i8 (bitconvert (v4i32  VPR128:$src))), (v16i8 VPR128:$src)>;
1562
1563 def : Pat<(v2f64 (bitconvert (v4f32  VPR128:$src))), (v2f64 VPR128:$src)>;
1564 def : Pat<(v2i64 (bitconvert (v4f32  VPR128:$src))), (v2i64 VPR128:$src)>;
1565 def : Pat<(v4i32 (bitconvert (v4f32  VPR128:$src))), (v4i32 VPR128:$src)>;
1566 def : Pat<(v8i16 (bitconvert (v4f32  VPR128:$src))), (v8i16 VPR128:$src)>;
1567 def : Pat<(v16i8 (bitconvert (v4f32  VPR128:$src))), (v16i8 VPR128:$src)>;
1568
1569 def : Pat<(v2f64 (bitconvert (v2i64  VPR128:$src))), (v2f64 VPR128:$src)>;
1570 def : Pat<(v4f32 (bitconvert (v2i64  VPR128:$src))), (v4f32 VPR128:$src)>;
1571 def : Pat<(v4i32 (bitconvert (v2i64  VPR128:$src))), (v4i32 VPR128:$src)>;
1572 def : Pat<(v8i16 (bitconvert (v2i64  VPR128:$src))), (v8i16 VPR128:$src)>;
1573 def : Pat<(v16i8 (bitconvert (v2i64  VPR128:$src))), (v16i8 VPR128:$src)>;
1574
1575 def : Pat<(v2i64 (bitconvert (v2f64  VPR128:$src))), (v2i64 VPR128:$src)>;
1576 def : Pat<(v4f32 (bitconvert (v2f64  VPR128:$src))), (v4f32 VPR128:$src)>;
1577 def : Pat<(v4i32 (bitconvert (v2f64  VPR128:$src))), (v4i32 VPR128:$src)>;
1578 def : Pat<(v8i16 (bitconvert (v2f64  VPR128:$src))), (v8i16 VPR128:$src)>;
1579 def : Pat<(v16i8 (bitconvert (v2f64  VPR128:$src))), (v16i8 VPR128:$src)>;
1580
1581
1582 // ...and scalar bitcasts...
1583
1584 def : Pat<(f64   (bitconvert (v8i8  VPR64:$src))),
1585                  (f64 (EXTRACT_SUBREG (v8i8  VPR64:$src), sub_64))>;
1586 def : Pat<(f64   (bitconvert (v4i16  VPR64:$src))),
1587                  (f64 (EXTRACT_SUBREG (v4i16  VPR64:$src), sub_64))>;
1588 def : Pat<(f64   (bitconvert (v2i32  VPR64:$src))),
1589                  (f64 (EXTRACT_SUBREG (v2i32  VPR64:$src), sub_64))>;
1590 def : Pat<(f64   (bitconvert (v2f32  VPR64:$src))),
1591                  (f64 (EXTRACT_SUBREG (v2f32  VPR64:$src), sub_64))>;
1592 def : Pat<(f64   (bitconvert (v1i64  VPR64:$src))),
1593                  (f64 (EXTRACT_SUBREG (v1i64  VPR64:$src), sub_64))>;
1594 def : Pat<(f128  (bitconvert (v16i8  VPR128:$src))),
1595                  (f128 (EXTRACT_SUBREG (v16i8  VPR128:$src), sub_alias))>;
1596 def : Pat<(f128  (bitconvert (v8i16  VPR128:$src))),
1597                  (f128 (EXTRACT_SUBREG (v8i16  VPR128:$src), sub_alias))>;
1598 def : Pat<(f128  (bitconvert (v4i32  VPR128:$src))),
1599                  (f128 (EXTRACT_SUBREG (v4i32  VPR128:$src), sub_alias))>;
1600 def : Pat<(f128  (bitconvert (v2i64  VPR128:$src))),
1601                  (f128 (EXTRACT_SUBREG (v2i64  VPR128:$src), sub_alias))>;
1602 def : Pat<(f128  (bitconvert (v4f32  VPR128:$src))),
1603                  (f128 (EXTRACT_SUBREG (v4f32  VPR128:$src), sub_alias))>;
1604 def : Pat<(f128  (bitconvert (v2f64  VPR128:$src))),
1605                  (f128 (EXTRACT_SUBREG (v2f64  VPR128:$src), sub_alias))>;
1606
1607 def : Pat<(v8i8   (bitconvert (f64   FPR64:$src))),
1608                   (v8i8 (SUBREG_TO_REG (i64 0), (f64  FPR64:$src), sub_64))>;
1609 def : Pat<(v4i16  (bitconvert (f64   FPR64:$src))),
1610                   (v4i16 (SUBREG_TO_REG (i64 0), (f64  FPR64:$src), sub_64))>;
1611 def : Pat<(v2i32  (bitconvert (f64   FPR64:$src))),
1612                   (v2i32 (SUBREG_TO_REG (i64 0), (f64  FPR64:$src), sub_64))>;
1613 def : Pat<(v2f32  (bitconvert (f64   FPR64:$src))),
1614                   (v2f32 (SUBREG_TO_REG (i64 0), (f64  FPR64:$src), sub_64))>;
1615 def : Pat<(v1i64  (bitconvert (f64   FPR64:$src))),
1616                   (v1i64 (SUBREG_TO_REG (i64 0), (f64  FPR64:$src), sub_64))>;
1617 def : Pat<(v16i8  (bitconvert (f128   FPR128:$src))),
1618                   (v16i8 (SUBREG_TO_REG (i128 0), (f128  FPR128:$src),
1619                   sub_alias))>;
1620 def : Pat<(v8i16  (bitconvert (f128   FPR128:$src))),
1621                   (v8i16 (SUBREG_TO_REG (i128 0), (f128  FPR128:$src),
1622                   sub_alias))>;
1623 def : Pat<(v4i32  (bitconvert (f128   FPR128:$src))),
1624                   (v4i32 (SUBREG_TO_REG (i128 0), (f128  FPR128:$src),
1625                   sub_alias))>;
1626 def : Pat<(v2i64  (bitconvert (f128   FPR128:$src))),
1627                   (v2i64 (SUBREG_TO_REG (i128 0), (f128  FPR128:$src),
1628                   sub_alias))>;
1629 def : Pat<(v4f32  (bitconvert (f128   FPR128:$src))),
1630                   (v4f32 (SUBREG_TO_REG (i128 0), (f128  FPR128:$src),
1631                   sub_alias))>;
1632 def : Pat<(v2f64  (bitconvert (f128   FPR128:$src))),
1633                   (v2f64 (SUBREG_TO_REG (i128 0), (f128  FPR128:$src),
1634                   sub_alias))>;