lib/Target/NVPTX/NVPTXIntrinsics.td

   1 //===- NVPTXIntrinsics.td - PTX Intrinsics Instructions -------*- tblgen -*-==//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9
  10 def immFloat0 : PatLeaf<(fpimm), [{
  11     float f = (float)N->getValueAPF().convertToFloat();
  12     return (f==0.0f);
  13 }]>;
  14
  15 def immFloat1 : PatLeaf<(fpimm), [{
  16     float f = (float)N->getValueAPF().convertToFloat();
  17     return (f==1.0f);
  18 }]>;
  19
  20 def immDouble0 : PatLeaf<(fpimm), [{
  21     double d = (double)N->getValueAPF().convertToDouble();
  22     return (d==0.0);
  23 }]>;
  24
  25 def immDouble1 : PatLeaf<(fpimm), [{
  26     double d = (double)N->getValueAPF().convertToDouble();
  27     return (d==1.0);
  28 }]>;
  29
  30
  31
  32 //-----------------------------------
  33 // Synchronization Functions
  34 //-----------------------------------
  35 def INT_CUDA_SYNCTHREADS : NVPTXInst<(outs), (ins),
  36                   "bar.sync \t0;",
  37       [(int_cuda_syncthreads)]>;
  38 def INT_BARRIER0 : NVPTXInst<(outs), (ins),
  39                   "bar.sync \t0;",
  40       [(int_nvvm_barrier0)]>;
  41 def INT_BARRIER0_POPC : NVPTXInst<(outs Int32Regs:$dst), (ins Int32Regs:$pred),
  42   !strconcat("{{ \n\t",
  43       !strconcat(".reg .pred \t%p1; \n\t",
  44       !strconcat("setp.ne.u32 \t%p1, $pred, 0; \n\t",
  45       !strconcat("bar.red.popc.u32 \t$dst, 0, %p1; \n\t",
  46         !strconcat("}}", ""))))),
  47       [(set Int32Regs:$dst, (int_nvvm_barrier0_popc Int32Regs:$pred))]>;
  48 def INT_BARRIER0_AND : NVPTXInst<(outs Int32Regs:$dst), (ins Int32Regs:$pred),
  49   !strconcat("{{ \n\t",
  50       !strconcat(".reg .pred \t%p1; \n\t",
  51       !strconcat(".reg .pred \t%p2; \n\t",
  52       !strconcat("setp.ne.u32 \t%p1, $pred, 0; \n\t",
  53       !strconcat("bar.red.and.pred \t%p2, 0, %p1; \n\t",
  54       !strconcat("selp.u32 \t$dst, 1, 0, %p2; \n\t",
  55         !strconcat("}}", ""))))))),
  56       [(set Int32Regs:$dst, (int_nvvm_barrier0_and Int32Regs:$pred))]>;
  57 def INT_BARRIER0_OR : NVPTXInst<(outs Int32Regs:$dst), (ins Int32Regs:$pred),
  58   !strconcat("{{ \n\t",
  59       !strconcat(".reg .pred \t%p1; \n\t",
  60       !strconcat(".reg .pred \t%p2; \n\t",
  61       !strconcat("setp.ne.u32 \t%p1, $pred, 0; \n\t",
  62       !strconcat("bar.red.or.pred \t%p2, 0, %p1; \n\t",
  63       !strconcat("selp.u32 \t$dst, 1, 0, %p2; \n\t",
  64         !strconcat("}}", ""))))))),
  65       [(set Int32Regs:$dst, (int_nvvm_barrier0_or Int32Regs:$pred))]>;
  66
  67
  68 //-----------------------------------
  69 // Explicit Memory Fence Functions
  70 //-----------------------------------
  71 class MEMBAR<string StrOp, Intrinsic IntOP> :
  72               NVPTXInst<(outs), (ins),
  73             StrOp, [(IntOP)]>;
  74
  75 def INT_MEMBAR_CTA : MEMBAR<"membar.cta;", int_nvvm_membar_cta>;
  76 def INT_MEMBAR_GL  : MEMBAR<"membar.gl;",  int_nvvm_membar_gl>;
  77 def INT_MEMBAR_SYS : MEMBAR<"membar.sys;", int_nvvm_membar_sys>;
  78
  79
  80 //-----------------------------------
  81 // Math Functions
  82 //-----------------------------------
  83
  84 // Map min(1.0, max(0.0, x)) to sat(x)
  85 // Note that max(0.0, min(x, 1.0)) cannot be mapped to sat(x) because when x is
  86 // NaN
  87 // max(0.0, min(x, 1.0)) is 1.0 while sat(x) is 0.
  88 // Same story for fmax, fmin.
  89
  90 def : Pat<(int_nvvm_fmin_f immFloat1,
  91             (int_nvvm_fmax_f immFloat0, Float32Regs:$a)),
  92           (CVT_f32_f32 Float32Regs:$a, CvtSAT)>;
  93 def : Pat<(int_nvvm_fmin_f immFloat1,
  94             (int_nvvm_fmax_f Float32Regs:$a, immFloat0)),
  95           (CVT_f32_f32 Float32Regs:$a, CvtSAT)>;
  96 def : Pat<(int_nvvm_fmin_f
  97             (int_nvvm_fmax_f immFloat0, Float32Regs:$a), immFloat1),
  98           (CVT_f32_f32 Float32Regs:$a, CvtSAT)>;
  99 def : Pat<(int_nvvm_fmin_f
 100             (int_nvvm_fmax_f Float32Regs:$a, immFloat0), immFloat1),
 101           (CVT_f32_f32 Float32Regs:$a, CvtSAT)>;
 102
 103 def : Pat<(int_nvvm_fmin_d immDouble1,
 104             (int_nvvm_fmax_d immDouble0, Float64Regs:$a)),
 105           (CVT_f64_f64 Float64Regs:$a, CvtSAT)>;
 106 def : Pat<(int_nvvm_fmin_d immDouble1,
 107             (int_nvvm_fmax_d Float64Regs:$a, immDouble0)),
 108           (CVT_f64_f64 Float64Regs:$a, CvtSAT)>;
 109 def : Pat<(int_nvvm_fmin_d
 110             (int_nvvm_fmax_d immDouble0, Float64Regs:$a), immDouble1),
 111           (CVT_f64_f64 Float64Regs:$a, CvtSAT)>;
 112 def : Pat<(int_nvvm_fmin_d
 113             (int_nvvm_fmax_d Float64Regs:$a, immDouble0), immDouble1),
 114           (CVT_f64_f64 Float64Regs:$a, CvtSAT)>;
 115
 116
 117 // We need a full string for OpcStr here because we need to deal with case like
 118 // INT_PTX_RECIP.
 119 class F_MATH_1<string OpcStr, NVPTXRegClass target_regclass,
 120   NVPTXRegClass src_regclass, Intrinsic IntOP>
 121             : NVPTXInst<(outs target_regclass:$dst), (ins src_regclass:$src0),
 122             OpcStr,
 123         [(set target_regclass:$dst, (IntOP src_regclass:$src0))]>;
 124
 125 // We need a full string for OpcStr here because we need to deal with the case
 126 // like INT_PTX_NATIVE_POWR_F.
 127 class F_MATH_2<string OpcStr, NVPTXRegClass t_regclass,
 128   NVPTXRegClass s0_regclass, NVPTXRegClass s1_regclass, Intrinsic IntOP>
 129             : NVPTXInst<(outs t_regclass:$dst),
 130               (ins s0_regclass:$src0, s1_regclass:$src1),
 131             OpcStr,
 132         [(set t_regclass:$dst, (IntOP s0_regclass:$src0, s1_regclass:$src1))]>;
 133
 134 class F_MATH_3<string OpcStr, NVPTXRegClass t_regclass,
 135   NVPTXRegClass s0_regclass, NVPTXRegClass s1_regclass,
 136   NVPTXRegClass s2_regclass, Intrinsic IntOP>
 137             : NVPTXInst<(outs t_regclass:$dst),
 138               (ins s0_regclass:$src0, s1_regclass:$src1, s2_regclass:$src2),
 139             OpcStr,
 140         [(set t_regclass:$dst,
 141           (IntOP s0_regclass:$src0, s1_regclass:$src1, s2_regclass:$src2))]>;
 142
 143 //
 144 // MISC
 145 //
 146
 147 def INT_NVVM_CLZ_I : F_MATH_1<"clz.b32 \t$dst, $src0;", Int32Regs, Int32Regs,
 148   int_nvvm_clz_i>;
 149 def INT_NVVM_CLZ_LL : F_MATH_1<"clz.b64 \t$dst, $src0;", Int32Regs, Int64Regs,
 150   int_nvvm_clz_ll>;
 151
 152 def INT_NVVM_POPC_I : F_MATH_1<"popc.b32 \t$dst, $src0;", Int32Regs, Int32Regs,
 153   int_nvvm_popc_i>;
 154 def INT_NVVM_POPC_LL : F_MATH_1<"popc.b64 \t$dst, $src0;", Int32Regs, Int64Regs,
 155   int_nvvm_popc_ll>;
 156
 157 def INT_NVVM_PRMT : F_MATH_3<"prmt.b32 \t$dst, $src0, $src1, $src2;", Int32Regs,
 158   Int32Regs, Int32Regs, Int32Regs, int_nvvm_prmt>;
 159
 160 //
 161 // Min Max
 162 //
 163
 164 def INT_NVVM_MIN_I : F_MATH_2<"min.s32 \t$dst, $src0, $src1;", Int32Regs,
 165   Int32Regs, Int32Regs, int_nvvm_min_i>;
 166 def INT_NVVM_MIN_UI : F_MATH_2<"min.u32 \t$dst, $src0, $src1;", Int32Regs,
 167   Int32Regs, Int32Regs, int_nvvm_min_ui>;
 168
 169 def INT_NVVM_MIN_LL : F_MATH_2<"min.s64 \t$dst, $src0, $src1;", Int64Regs,
 170   Int64Regs, Int64Regs, int_nvvm_min_ll>;
 171 def INT_NVVM_MIN_ULL : F_MATH_2<"min.u64 \t$dst, $src0, $src1;", Int64Regs,
 172   Int64Regs, Int64Regs, int_nvvm_min_ull>;
 173
 174 def INT_NVVM_MAX_I : F_MATH_2<"max.s32 \t$dst, $src0, $src1;", Int32Regs,
 175   Int32Regs, Int32Regs, int_nvvm_max_i>;
 176 def INT_NVVM_MAX_UI : F_MATH_2<"max.u32 \t$dst, $src0, $src1;", Int32Regs,
 177   Int32Regs, Int32Regs, int_nvvm_max_ui>;
 178
 179 def INT_NVVM_MAX_LL : F_MATH_2<"max.s64 \t$dst, $src0, $src1;", Int64Regs,
 180   Int64Regs, Int64Regs, int_nvvm_max_ll>;
 181 def INT_NVVM_MAX_ULL : F_MATH_2<"max.u64 \t$dst, $src0, $src1;", Int64Regs,
 182   Int64Regs, Int64Regs, int_nvvm_max_ull>;
 183
 184 def INT_NVVM_FMIN_F : F_MATH_2<"min.f32 \t$dst, $src0, $src1;", Float32Regs,
 185   Float32Regs, Float32Regs, int_nvvm_fmin_f>;
 186 def INT_NVVM_FMIN_FTZ_F : F_MATH_2<"min.ftz.f32 \t$dst, $src0, $src1;",
 187   Float32Regs, Float32Regs, Float32Regs, int_nvvm_fmin_ftz_f>;
 188
 189 def INT_NVVM_FMAX_F : F_MATH_2<"max.f32 \t$dst, $src0, $src1;", Float32Regs,
 190   Float32Regs, Float32Regs, int_nvvm_fmax_f>;
 191 def INT_NVVM_FMAX_FTZ_F : F_MATH_2<"max.ftz.f32 \t$dst, $src0, $src1;",
 192   Float32Regs, Float32Regs, Float32Regs, int_nvvm_fmax_ftz_f>;
 193
 194 def INT_NVVM_FMIN_D : F_MATH_2<"min.f64 \t$dst, $src0, $src1;", Float64Regs,
 195   Float64Regs, Float64Regs, int_nvvm_fmin_d>;
 196 def INT_NVVM_FMAX_D : F_MATH_2<"max.f64 \t$dst, $src0, $src1;", Float64Regs,
 197   Float64Regs, Float64Regs, int_nvvm_fmax_d>;
 198
 199 //
 200 // Multiplication
 201 //
 202
 203 def INT_NVVM_MULHI_I : F_MATH_2<"mul.hi.s32 \t$dst, $src0, $src1;", Int32Regs,
 204   Int32Regs, Int32Regs, int_nvvm_mulhi_i>;
 205 def INT_NVVM_MULHI_UI : F_MATH_2<"mul.hi.u32 \t$dst, $src0, $src1;", Int32Regs,
 206   Int32Regs, Int32Regs, int_nvvm_mulhi_ui>;
 207
 208 def INT_NVVM_MULHI_LL : F_MATH_2<"mul.hi.s64 \t$dst, $src0, $src1;", Int64Regs,
 209   Int64Regs, Int64Regs, int_nvvm_mulhi_ll>;
 210 def INT_NVVM_MULHI_ULL : F_MATH_2<"mul.hi.u64 \t$dst, $src0, $src1;", Int64Regs,
 211   Int64Regs, Int64Regs, int_nvvm_mulhi_ull>;
 212
 213 def INT_NVVM_MUL_RN_FTZ_F : F_MATH_2<"mul.rn.ftz.f32 \t$dst, $src0, $src1;",
 214   Float32Regs, Float32Regs, Float32Regs, int_nvvm_mul_rn_ftz_f>;
 215 def INT_NVVM_MUL_RN_F : F_MATH_2<"mul.rn.f32 \t$dst, $src0, $src1;",
 216   Float32Regs, Float32Regs, Float32Regs, int_nvvm_mul_rn_f>;
 217 def INT_NVVM_MUL_RZ_FTZ_F : F_MATH_2<"mul.rz.ftz.f32 \t$dst, $src0, $src1;",
 218   Float32Regs, Float32Regs, Float32Regs, int_nvvm_mul_rz_ftz_f>;
 219 def INT_NVVM_MUL_RZ_F : F_MATH_2<"mul.rz.f32 \t$dst, $src0, $src1;",
 220   Float32Regs, Float32Regs, Float32Regs, int_nvvm_mul_rz_f>;
 221 def INT_NVVM_MUL_RM_FTZ_F : F_MATH_2<"mul.rm.ftz.f32 \t$dst, $src0, $src1;",
 222   Float32Regs, Float32Regs, Float32Regs, int_nvvm_mul_rm_ftz_f>;
 223 def INT_NVVM_MUL_RM_F : F_MATH_2<"mul.rm.f32 \t$dst, $src0, $src1;",
 224   Float32Regs, Float32Regs, Float32Regs, int_nvvm_mul_rm_f>;
 225 def INT_NVVM_MUL_RP_FTZ_F : F_MATH_2<"mul.rp.ftz.f32 \t$dst, $src0, $src1;",
 226   Float32Regs, Float32Regs, Float32Regs, int_nvvm_mul_rp_ftz_f>;
 227 def INT_NVVM_MUL_RP_F : F_MATH_2<"mul.rp.f32 \t$dst, $src0, $src1;",
 228   Float32Regs, Float32Regs, Float32Regs, int_nvvm_mul_rp_f>;
 229
 230 def INT_NVVM_MUL_RN_D : F_MATH_2<"mul.rn.f64 \t$dst, $src0, $src1;",
 231   Float64Regs, Float64Regs, Float64Regs, int_nvvm_mul_rn_d>;
 232 def INT_NVVM_MUL_RZ_D : F_MATH_2<"mul.rz.f64 \t$dst, $src0, $src1;",
 233   Float64Regs, Float64Regs, Float64Regs, int_nvvm_mul_rz_d>;
 234 def INT_NVVM_MUL_RM_D : F_MATH_2<"mul.rm.f64 \t$dst, $src0, $src1;",
 235   Float64Regs, Float64Regs, Float64Regs, int_nvvm_mul_rm_d>;
 236 def INT_NVVM_MUL_RP_D : F_MATH_2<"mul.rp.f64 \t$dst, $src0, $src1;",
 237   Float64Regs, Float64Regs, Float64Regs, int_nvvm_mul_rp_d>;
 238
 239 def INT_NVVM_MUL24_I : F_MATH_2<"mul24.lo.s32 \t$dst, $src0, $src1;",
 240   Int32Regs, Int32Regs, Int32Regs, int_nvvm_mul24_i>;
 241 def INT_NVVM_MUL24_UI : F_MATH_2<"mul24.lo.u32 \t$dst, $src0, $src1;",
 242   Int32Regs, Int32Regs, Int32Regs, int_nvvm_mul24_ui>;
 243
 244 //
 245 // Div
 246 //
 247
 248 def INT_NVVM_DIV_APPROX_FTZ_F
 249   : F_MATH_2<"div.approx.ftz.f32 \t$dst, $src0, $src1;", Float32Regs,
 250     Float32Regs, Float32Regs, int_nvvm_div_approx_ftz_f>;
 251 def INT_NVVM_DIV_APPROX_F : F_MATH_2<"div.approx.f32 \t$dst, $src0, $src1;",
 252   Float32Regs, Float32Regs, Float32Regs, int_nvvm_div_approx_f>;
 253
 254 def INT_NVVM_DIV_RN_FTZ_F : F_MATH_2<"div.rn.ftz.f32 \t$dst, $src0, $src1;",
 255   Float32Regs, Float32Regs, Float32Regs, int_nvvm_div_rn_ftz_f>;
 256 def INT_NVVM_DIV_RN_F     : F_MATH_2<"div.rn.f32 \t$dst, $src0, $src1;",
 257   Float32Regs, Float32Regs, Float32Regs, int_nvvm_div_rn_f>;
 258 def INT_NVVM_DIV_RZ_FTZ_F : F_MATH_2<"div.rz.ftz.f32 \t$dst, $src0, $src1;",
 259   Float32Regs, Float32Regs, Float32Regs, int_nvvm_div_rz_ftz_f>;
 260 def INT_NVVM_DIV_RZ_F     : F_MATH_2<"div.rz.f32 \t$dst, $src0, $src1;",
 261   Float32Regs, Float32Regs, Float32Regs, int_nvvm_div_rz_f>;
 262 def INT_NVVM_DIV_RM_FTZ_F : F_MATH_2<"div.rm.ftz.f32 \t$dst, $src0, $src1;",
 263   Float32Regs, Float32Regs, Float32Regs, int_nvvm_div_rm_ftz_f>;
 264 def INT_NVVM_DIV_RM_F     : F_MATH_2<"div.rm.f32 \t$dst, $src0, $src1;",
 265   Float32Regs, Float32Regs, Float32Regs, int_nvvm_div_rm_f>;
 266 def INT_NVVM_DIV_RP_FTZ_F : F_MATH_2<"div.rp.ftz.f32 \t$dst, $src0, $src1;",
 267   Float32Regs, Float32Regs, Float32Regs, int_nvvm_div_rp_ftz_f>;
 268 def INT_NVVM_DIV_RP_F     : F_MATH_2<"div.rp.f32 \t$dst, $src0, $src1;",
 269   Float32Regs, Float32Regs, Float32Regs, int_nvvm_div_rp_f>;
 270
 271 def INT_NVVM_DIV_RN_D : F_MATH_2<"div.rn.f64 \t$dst, $src0, $src1;",
 272   Float64Regs, Float64Regs, Float64Regs, int_nvvm_div_rn_d>;
 273 def INT_NVVM_DIV_RZ_D : F_MATH_2<"div.rz.f64 \t$dst, $src0, $src1;",
 274   Float64Regs, Float64Regs, Float64Regs, int_nvvm_div_rz_d>;
 275 def INT_NVVM_DIV_RM_D : F_MATH_2<"div.rm.f64 \t$dst, $src0, $src1;",
 276   Float64Regs, Float64Regs, Float64Regs, int_nvvm_div_rm_d>;
 277 def INT_NVVM_DIV_RP_D : F_MATH_2<"div.rp.f64 \t$dst, $src0, $src1;",
 278   Float64Regs, Float64Regs, Float64Regs, int_nvvm_div_rp_d>;
 279
 280 //
 281 // Brev
 282 //
 283
 284 def INT_NVVM_BREV32 : F_MATH_1<"brev.b32 \t$dst, $src0;", Int32Regs, Int32Regs,
 285   int_nvvm_brev32>;
 286 def INT_NVVM_BREV64 : F_MATH_1<"brev.b64 \t$dst, $src0;", Int64Regs, Int64Regs,
 287   int_nvvm_brev64>;
 288
 289 //
 290 // Sad
 291 //
 292
 293 def INT_NVVM_SAD_I : F_MATH_3<"sad.s32 \t$dst, $src0, $src1, $src2;",
 294   Int32Regs, Int32Regs, Int32Regs, Int32Regs, int_nvvm_sad_i>;
 295 def INT_NVVM_SAD_UI : F_MATH_3<"sad.u32 \t$dst, $src0, $src1, $src2;",
 296   Int32Regs, Int32Regs, Int32Regs, Int32Regs, int_nvvm_sad_ui>;
 297
 298 //
 299 // Floor  Ceil
 300 //
 301
 302 def : Pat<(int_nvvm_floor_ftz_f Float32Regs:$a),
 303           (CVT_f32_f32 Float32Regs:$a, CvtRMI_FTZ)>;
 304 def : Pat<(int_nvvm_floor_f Float32Regs:$a),
 305           (CVT_f32_f32 Float32Regs:$a, CvtRMI)>;
 306 def : Pat<(int_nvvm_floor_d Float64Regs:$a),
 307           (CVT_f64_f64 Float64Regs:$a, CvtRMI)>;
 308
 309 def : Pat<(int_nvvm_ceil_ftz_f Float32Regs:$a),
 310           (CVT_f32_f32 Float32Regs:$a, CvtRPI_FTZ)>;
 311 def : Pat<(int_nvvm_ceil_f Float32Regs:$a),
 312           (CVT_f32_f32 Float32Regs:$a, CvtRPI)>;
 313 def : Pat<(int_nvvm_ceil_d Float64Regs:$a),
 314           (CVT_f64_f64 Float64Regs:$a, CvtRPI)>;
 315
 316 //
 317 // Abs
 318 //
 319
 320 def INT_NVVM_ABS_I : F_MATH_1<"abs.s32 \t$dst, $src0;", Int32Regs, Int32Regs,
 321   int_nvvm_abs_i>;
 322 def INT_NVVM_ABS_LL : F_MATH_1<"abs.s64 \t$dst, $src0;", Int64Regs, Int64Regs,
 323   int_nvvm_abs_ll>;
 324
 325 def INT_NVVM_FABS_FTZ_F : F_MATH_1<"abs.ftz.f32 \t$dst, $src0;", Float32Regs,
 326   Float32Regs, int_nvvm_fabs_ftz_f>;
 327 def INT_NVVM_FABS_F : F_MATH_1<"abs.f32 \t$dst, $src0;", Float32Regs,
 328   Float32Regs, int_nvvm_fabs_f>;
 329
 330 def INT_NVVM_FABS_D : F_MATH_1<"abs.f64 \t$dst, $src0;", Float64Regs,
 331   Float64Regs, int_nvvm_fabs_d>;
 332
 333 //
 334 // Round
 335 //
 336
 337 def : Pat<(int_nvvm_round_ftz_f Float32Regs:$a),
 338           (CVT_f32_f32 Float32Regs:$a, CvtRNI_FTZ)>;
 339 def : Pat<(int_nvvm_round_f Float32Regs:$a),
 340           (CVT_f32_f32 Float32Regs:$a, CvtRNI)>;
 341 def : Pat<(int_nvvm_round_d Float64Regs:$a),
 342           (CVT_f64_f64 Float64Regs:$a, CvtRNI)>;
 343
 344 //
 345 // Trunc
 346 //
 347
 348 def : Pat<(int_nvvm_trunc_ftz_f Float32Regs:$a),
 349           (CVT_f32_f32 Float32Regs:$a, CvtRZI_FTZ)>;
 350 def : Pat<(int_nvvm_trunc_f Float32Regs:$a),
 351           (CVT_f32_f32 Float32Regs:$a, CvtRZI)>;
 352 def : Pat<(int_nvvm_trunc_d Float64Regs:$a),
 353           (CVT_f64_f64 Float64Regs:$a, CvtRZI)>;
 354
 355 //
 356 // Saturate
 357 //
 358
 359 def : Pat<(int_nvvm_saturate_ftz_f Float32Regs:$a),
 360           (CVT_f32_f32 Float32Regs:$a, CvtSAT_FTZ)>;
 361 def : Pat<(int_nvvm_saturate_f Float32Regs:$a),
 362           (CVT_f32_f32 Float32Regs:$a, CvtSAT)>;
 363 def : Pat<(int_nvvm_saturate_d Float64Regs:$a),
 364           (CVT_f64_f64 Float64Regs:$a, CvtSAT)>;
 365
 366 //
 367 // Exp2  Log2
 368 //
 369
 370 def INT_NVVM_EX2_APPROX_FTZ_F : F_MATH_1<"ex2.approx.ftz.f32 \t$dst, $src0;",
 371   Float32Regs, Float32Regs, int_nvvm_ex2_approx_ftz_f>;
 372 def INT_NVVM_EX2_APPROX_F : F_MATH_1<"ex2.approx.f32 \t$dst, $src0;",
 373   Float32Regs, Float32Regs, int_nvvm_ex2_approx_f>;
 374 def INT_NVVM_EX2_APPROX_D : F_MATH_1<"ex2.approx.f64 \t$dst, $src0;",
 375   Float64Regs, Float64Regs, int_nvvm_ex2_approx_d>;
 376
 377 def INT_NVVM_LG2_APPROX_FTZ_F : F_MATH_1<"lg2.approx.ftz.f32 \t$dst, $src0;",
 378   Float32Regs, Float32Regs, int_nvvm_lg2_approx_ftz_f>;
 379 def INT_NVVM_LG2_APPROX_F : F_MATH_1<"lg2.approx.f32 \t$dst, $src0;",
 380   Float32Regs, Float32Regs, int_nvvm_lg2_approx_f>;
 381 def INT_NVVM_LG2_APPROX_D : F_MATH_1<"lg2.approx.f64 \t$dst, $src0;",
 382   Float64Regs, Float64Regs, int_nvvm_lg2_approx_d>;
 383
 384 //
 385 // Sin  Cos
 386 //
 387
 388 def INT_NVVM_SIN_APPROX_FTZ_F : F_MATH_1<"sin.approx.ftz.f32 \t$dst, $src0;",
 389   Float32Regs, Float32Regs, int_nvvm_sin_approx_ftz_f>;
 390 def INT_NVVM_SIN_APPROX_F : F_MATH_1<"sin.approx.f32 \t$dst, $src0;",
 391   Float32Regs, Float32Regs, int_nvvm_sin_approx_f>;
 392
 393 def INT_NVVM_COS_APPROX_FTZ_F : F_MATH_1<"cos.approx.ftz.f32 \t$dst, $src0;",
 394   Float32Regs, Float32Regs, int_nvvm_cos_approx_ftz_f>;
 395 def INT_NVVM_COS_APPROX_F : F_MATH_1<"cos.approx.f32 \t$dst, $src0;",
 396   Float32Regs, Float32Regs, int_nvvm_cos_approx_f>;
 397
 398 //
 399 // Fma
 400 //
 401
 402 def INT_NVVM_FMA_RN_FTZ_F
 403   : F_MATH_3<"fma.rn.ftz.f32 \t$dst, $src0, $src1, $src2;", Float32Regs,
 404     Float32Regs, Float32Regs, Float32Regs, int_nvvm_fma_rn_ftz_f>;
 405 def INT_NVVM_FMA_RN_F : F_MATH_3<"fma.rn.f32 \t$dst, $src0, $src1, $src2;",
 406   Float32Regs, Float32Regs, Float32Regs, Float32Regs, int_nvvm_fma_rn_f>;
 407 def INT_NVVM_FMA_RZ_FTZ_F
 408   : F_MATH_3<"fma.rz.ftz.f32 \t$dst, $src0, $src1, $src2;", Float32Regs,
 409     Float32Regs, Float32Regs, Float32Regs, int_nvvm_fma_rz_ftz_f>;
 410 def INT_NVVM_FMA_RZ_F : F_MATH_3<"fma.rz.f32 \t$dst, $src0, $src1, $src2;",
 411   Float32Regs, Float32Regs, Float32Regs, Float32Regs, int_nvvm_fma_rz_f>;
 412 def INT_NVVM_FMA_RM_FTZ_F
 413   : F_MATH_3<"fma.rm.ftz.f32 \t$dst, $src0, $src1, $src2;", Float32Regs,
 414     Float32Regs, Float32Regs, Float32Regs, int_nvvm_fma_rm_ftz_f>;
 415 def INT_NVVM_FMA_RM_F : F_MATH_3<"fma.rm.f32 \t$dst, $src0, $src1, $src2;",
 416   Float32Regs, Float32Regs, Float32Regs, Float32Regs, int_nvvm_fma_rm_f>;
 417 def INT_NVVM_FMA_RP_FTZ_F
 418   : F_MATH_3<"fma.rp.ftz.f32 \t$dst, $src0, $src1, $src2;", Float32Regs,
 419     Float32Regs, Float32Regs, Float32Regs, int_nvvm_fma_rp_ftz_f>;
 420 def INT_NVVM_FMA_RP_F : F_MATH_3<"fma.rp.f32 \t$dst, $src0, $src1, $src2;",
 421   Float32Regs, Float32Regs, Float32Regs, Float32Regs, int_nvvm_fma_rp_f>;
 422
 423 def INT_NVVM_FMA_RN_D : F_MATH_3<"fma.rn.f64 \t$dst, $src0, $src1, $src2;",
 424   Float64Regs, Float64Regs, Float64Regs, Float64Regs, int_nvvm_fma_rn_d>;
 425 def INT_NVVM_FMA_RZ_D : F_MATH_3<"fma.rz.f64 \t$dst, $src0, $src1, $src2;",
 426   Float64Regs, Float64Regs, Float64Regs, Float64Regs, int_nvvm_fma_rz_d>;
 427 def INT_NVVM_FMA_RM_D : F_MATH_3<"fma.rm.f64 \t$dst, $src0, $src1, $src2;",
 428   Float64Regs, Float64Regs, Float64Regs, Float64Regs, int_nvvm_fma_rm_d>;
 429 def INT_NVVM_FMA_RP_D : F_MATH_3<"fma.rp.f64 \t$dst, $src0, $src1, $src2;",
 430   Float64Regs, Float64Regs, Float64Regs, Float64Regs, int_nvvm_fma_rp_d>;
 431
 432 //
 433 // Rcp
 434 //
 435
 436 def INT_NVVM_RCP_RN_FTZ_F : F_MATH_1<"rcp.rn.ftz.f32 \t$dst, $src0;",
 437   Float32Regs, Float32Regs, int_nvvm_rcp_rn_ftz_f>;
 438 def INT_NVVM_RCP_RN_F : F_MATH_1<"rcp.rn.f32 \t$dst, $src0;",
 439   Float32Regs, Float32Regs, int_nvvm_rcp_rn_f>;
 440 def INT_NVVM_RCP_RZ_FTZ_F : F_MATH_1<"rcp.rz.ftz.f32 \t$dst, $src0;",
 441   Float32Regs, Float32Regs, int_nvvm_rcp_rz_ftz_f>;
 442 def INT_NVVM_RCP_RZ_F : F_MATH_1<"rcp.rz.f32 \t$dst, $src0;",
 443   Float32Regs, Float32Regs, int_nvvm_rcp_rz_f>;
 444 def INT_NVVM_RCP_RM_FTZ_F : F_MATH_1<"rcp.rm.ftz.f32 \t$dst, $src0;",
 445   Float32Regs, Float32Regs, int_nvvm_rcp_rm_ftz_f>;
 446 def INT_NVVM_RCP_RM_F : F_MATH_1<"rcp.rm.f32 \t$dst, $src0;",
 447   Float32Regs, Float32Regs, int_nvvm_rcp_rm_f>;
 448 def INT_NVVM_RCP_RP_FTZ_F : F_MATH_1<"rcp.rp.ftz.f32 \t$dst, $src0;",
 449   Float32Regs, Float32Regs, int_nvvm_rcp_rp_ftz_f>;
 450 def INT_NVVM_RCP_RP_F : F_MATH_1<"rcp.rp.f32 \t$dst, $src0;",
 451   Float32Regs, Float32Regs, int_nvvm_rcp_rp_f>;
 452
 453 def INT_NVVM_RCP_RN_D : F_MATH_1<"rcp.rn.f64 \t$dst, $src0;", Float64Regs,
 454   Float64Regs, int_nvvm_rcp_rn_d>;
 455 def INT_NVVM_RCP_RZ_D : F_MATH_1<"rcp.rz.f64 \t$dst, $src0;", Float64Regs,
 456   Float64Regs, int_nvvm_rcp_rz_d>;
 457 def INT_NVVM_RCP_RM_D : F_MATH_1<"rcp.rm.f64 \t$dst, $src0;", Float64Regs,
 458   Float64Regs, int_nvvm_rcp_rm_d>;
 459 def INT_NVVM_RCP_RP_D : F_MATH_1<"rcp.rp.f64 \t$dst, $src0;", Float64Regs,
 460   Float64Regs, int_nvvm_rcp_rp_d>;
 461
 462 def INT_NVVM_RCP_APPROX_FTZ_D : F_MATH_1<"rcp.approx.ftz.f64 \t$dst, $src0;",
 463   Float64Regs, Float64Regs, int_nvvm_rcp_approx_ftz_d>;
 464
 465 //
 466 // Sqrt
 467 //
 468
 469 def INT_NVVM_SQRT_RN_FTZ_F : F_MATH_1<"sqrt.rn.ftz.f32 \t$dst, $src0;",
 470   Float32Regs, Float32Regs, int_nvvm_sqrt_rn_ftz_f>;
 471 def INT_NVVM_SQRT_RN_F : F_MATH_1<"sqrt.rn.f32 \t$dst, $src0;", Float32Regs,
 472   Float32Regs, int_nvvm_sqrt_rn_f>;
 473 def INT_NVVM_SQRT_RZ_FTZ_F : F_MATH_1<"sqrt.rz.ftz.f32 \t$dst, $src0;",
 474   Float32Regs, Float32Regs, int_nvvm_sqrt_rz_ftz_f>;
 475 def INT_NVVM_SQRT_RZ_F : F_MATH_1<"sqrt.rz.f32 \t$dst, $src0;", Float32Regs,
 476   Float32Regs, int_nvvm_sqrt_rz_f>;
 477 def INT_NVVM_SQRT_RM_FTZ_F : F_MATH_1<"sqrt.rm.ftz.f32 \t$dst, $src0;",
 478   Float32Regs, Float32Regs, int_nvvm_sqrt_rm_ftz_f>;
 479 def INT_NVVM_SQRT_RM_F : F_MATH_1<"sqrt.rm.f32 \t$dst, $src0;", Float32Regs,
 480   Float32Regs, int_nvvm_sqrt_rm_f>;
 481 def INT_NVVM_SQRT_RP_FTZ_F : F_MATH_1<"sqrt.rp.ftz.f32 \t$dst, $src0;",
 482   Float32Regs, Float32Regs, int_nvvm_sqrt_rp_ftz_f>;
 483 def INT_NVVM_SQRT_RP_F : F_MATH_1<"sqrt.rp.f32 \t$dst, $src0;", Float32Regs,
 484   Float32Regs, int_nvvm_sqrt_rp_f>;
 485 def INT_NVVM_SQRT_APPROX_FTZ_F : F_MATH_1<"sqrt.approx.ftz.f32 \t$dst, $src0;",
 486   Float32Regs, Float32Regs, int_nvvm_sqrt_approx_ftz_f>;
 487 def INT_NVVM_SQRT_APPROX_F : F_MATH_1<"sqrt.approx.f32 \t$dst, $src0;",
 488   Float32Regs, Float32Regs, int_nvvm_sqrt_approx_f>;
 489
 490 def INT_NVVM_SQRT_RN_D : F_MATH_1<"sqrt.rn.f64 \t$dst, $src0;", Float64Regs,
 491   Float64Regs, int_nvvm_sqrt_rn_d>;
 492 def INT_NVVM_SQRT_RZ_D : F_MATH_1<"sqrt.rz.f64 \t$dst, $src0;", Float64Regs,
 493   Float64Regs, int_nvvm_sqrt_rz_d>;
 494 def INT_NVVM_SQRT_RM_D : F_MATH_1<"sqrt.rm.f64 \t$dst, $src0;", Float64Regs,
 495   Float64Regs, int_nvvm_sqrt_rm_d>;
 496 def INT_NVVM_SQRT_RP_D : F_MATH_1<"sqrt.rp.f64 \t$dst, $src0;", Float64Regs,
 497   Float64Regs, int_nvvm_sqrt_rp_d>;
 498
 499 // nvvm_sqrt intrinsic
 500 def : Pat<(int_nvvm_sqrt_f Float32Regs:$a),
 501           (INT_NVVM_SQRT_RN_FTZ_F Float32Regs:$a)>, Requires<[doF32FTZ, do_SQRTF32_RN]>;
 502 def : Pat<(int_nvvm_sqrt_f Float32Regs:$a),
 503           (INT_NVVM_SQRT_RN_F Float32Regs:$a)>, Requires<[do_SQRTF32_RN]>;
 504 def : Pat<(int_nvvm_sqrt_f Float32Regs:$a),
 505           (INT_NVVM_SQRT_APPROX_FTZ_F Float32Regs:$a)>, Requires<[doF32FTZ]>;
 506 def : Pat<(int_nvvm_sqrt_f Float32Regs:$a),
 507           (INT_NVVM_SQRT_APPROX_F Float32Regs:$a)>;
 508
 509 //
 510 // Rsqrt
 511 //
 512
 513 def INT_NVVM_RSQRT_APPROX_FTZ_F
 514   : F_MATH_1<"rsqrt.approx.ftz.f32 \t$dst, $src0;", Float32Regs, Float32Regs,
 515     int_nvvm_rsqrt_approx_ftz_f>;
 516 def INT_NVVM_RSQRT_APPROX_F : F_MATH_1<"rsqrt.approx.f32 \t$dst, $src0;",
 517   Float32Regs, Float32Regs, int_nvvm_rsqrt_approx_f>;
 518 def INT_NVVM_RSQRT_APPROX_D : F_MATH_1<"rsqrt.approx.f64 \t$dst, $src0;",
 519   Float64Regs, Float64Regs, int_nvvm_rsqrt_approx_d>;
 520
 521 //
 522 // Add
 523 //
 524
 525 def INT_NVVM_ADD_RN_FTZ_F : F_MATH_2<"add.rn.ftz.f32 \t$dst, $src0, $src1;",
 526   Float32Regs, Float32Regs, Float32Regs, int_nvvm_add_rn_ftz_f>;
 527 def INT_NVVM_ADD_RN_F : F_MATH_2<"add.rn.f32 \t$dst, $src0, $src1;",
 528   Float32Regs, Float32Regs, Float32Regs, int_nvvm_add_rn_f>;
 529 def INT_NVVM_ADD_RZ_FTZ_F : F_MATH_2<"add.rz.ftz.f32 \t$dst, $src0, $src1;",
 530   Float32Regs, Float32Regs, Float32Regs, int_nvvm_add_rz_ftz_f>;
 531 def INT_NVVM_ADD_RZ_F : F_MATH_2<"add.rz.f32 \t$dst, $src0, $src1;",
 532   Float32Regs, Float32Regs, Float32Regs, int_nvvm_add_rz_f>;
 533 def INT_NVVM_ADD_RM_FTZ_F : F_MATH_2<"add.rm.ftz.f32 \t$dst, $src0, $src1;",
 534   Float32Regs, Float32Regs, Float32Regs, int_nvvm_add_rm_ftz_f>;
 535 def INT_NVVM_ADD_RM_F : F_MATH_2<"add.rm.f32 \t$dst, $src0, $src1;",
 536   Float32Regs, Float32Regs, Float32Regs, int_nvvm_add_rm_f>;
 537 def INT_NVVM_ADD_RP_FTZ_F : F_MATH_2<"add.rp.ftz.f32 \t$dst, $src0, $src1;",
 538   Float32Regs, Float32Regs, Float32Regs, int_nvvm_add_rp_ftz_f>;
 539 def INT_NVVM_ADD_RP_F : F_MATH_2<"add.rp.f32 \t$dst, $src0, $src1;",
 540   Float32Regs, Float32Regs, Float32Regs, int_nvvm_add_rp_f>;
 541
 542 def INT_NVVM_ADD_RN_D : F_MATH_2<"add.rn.f64 \t$dst, $src0, $src1;",
 543   Float64Regs, Float64Regs, Float64Regs, int_nvvm_add_rn_d>;
 544 def INT_NVVM_ADD_RZ_D : F_MATH_2<"add.rz.f64 \t$dst, $src0, $src1;",
 545   Float64Regs, Float64Regs, Float64Regs, int_nvvm_add_rz_d>;
 546 def INT_NVVM_ADD_RM_D : F_MATH_2<"add.rm.f64 \t$dst, $src0, $src1;",
 547   Float64Regs, Float64Regs, Float64Regs, int_nvvm_add_rm_d>;
 548 def INT_NVVM_ADD_RP_D : F_MATH_2<"add.rp.f64 \t$dst, $src0, $src1;",
 549   Float64Regs, Float64Regs, Float64Regs, int_nvvm_add_rp_d>;
 550
 551 //
 552 // Convert
 553 //
 554
 555 def : Pat<(int_nvvm_d2f_rn_ftz Float64Regs:$a),
 556           (CVT_f32_f64 Float64Regs:$a, CvtRN_FTZ)>;
 557 def : Pat<(int_nvvm_d2f_rn Float64Regs:$a),
 558           (CVT_f32_f64 Float64Regs:$a, CvtRN)>;
 559 def : Pat<(int_nvvm_d2f_rz_ftz Float64Regs:$a),
 560           (CVT_f32_f64 Float64Regs:$a, CvtRZ_FTZ)>;
 561 def : Pat<(int_nvvm_d2f_rz Float64Regs:$a),
 562           (CVT_f32_f64 Float64Regs:$a, CvtRZ)>;
 563 def : Pat<(int_nvvm_d2f_rm_ftz Float64Regs:$a),
 564           (CVT_f32_f64 Float64Regs:$a, CvtRM_FTZ)>;
 565 def : Pat<(int_nvvm_d2f_rm Float64Regs:$a),
 566           (CVT_f32_f64 Float64Regs:$a, CvtRM)>;
 567 def : Pat<(int_nvvm_d2f_rp_ftz Float64Regs:$a),
 568           (CVT_f32_f64 Float64Regs:$a, CvtRP_FTZ)>;
 569 def : Pat<(int_nvvm_d2f_rp Float64Regs:$a),
 570           (CVT_f32_f64 Float64Regs:$a, CvtRP)>;
 571
 572 def : Pat<(int_nvvm_d2i_rn Float64Regs:$a),
 573           (CVT_s32_f64 Float64Regs:$a, CvtRNI)>;
 574 def : Pat<(int_nvvm_d2i_rz Float64Regs:$a),
 575           (CVT_s32_f64 Float64Regs:$a, CvtRZI)>;
 576 def : Pat<(int_nvvm_d2i_rm Float64Regs:$a),
 577           (CVT_s32_f64 Float64Regs:$a, CvtRMI)>;
 578 def : Pat<(int_nvvm_d2i_rp Float64Regs:$a),
 579           (CVT_s32_f64 Float64Regs:$a, CvtRPI)>;
 580
 581 def : Pat<(int_nvvm_d2ui_rn Float64Regs:$a),
 582           (CVT_u32_f64 Float64Regs:$a, CvtRNI)>;
 583 def : Pat<(int_nvvm_d2ui_rz Float64Regs:$a),
 584           (CVT_u32_f64 Float64Regs:$a, CvtRZI)>;
 585 def : Pat<(int_nvvm_d2ui_rm Float64Regs:$a),
 586           (CVT_u32_f64 Float64Regs:$a, CvtRMI)>;
 587 def : Pat<(int_nvvm_d2ui_rp Float64Regs:$a),
 588           (CVT_u32_f64 Float64Regs:$a, CvtRPI)>;
 589
 590 def : Pat<(int_nvvm_i2d_rn Int32Regs:$a),
 591           (CVT_f64_s32 Int32Regs:$a, CvtRN)>;
 592 def : Pat<(int_nvvm_i2d_rz Int32Regs:$a),
 593           (CVT_f64_s32 Int32Regs:$a, CvtRZ)>;
 594 def : Pat<(int_nvvm_i2d_rm Int32Regs:$a),
 595           (CVT_f64_s32 Int32Regs:$a, CvtRM)>;
 596 def : Pat<(int_nvvm_i2d_rp Int32Regs:$a),
 597           (CVT_f64_s32 Int32Regs:$a, CvtRP)>;
 598
 599 def : Pat<(int_nvvm_ui2d_rn Int32Regs:$a),
 600           (CVT_f64_u32 Int32Regs:$a, CvtRN)>;
 601 def : Pat<(int_nvvm_ui2d_rz Int32Regs:$a),
 602           (CVT_f64_u32 Int32Regs:$a, CvtRZ)>;
 603 def : Pat<(int_nvvm_ui2d_rm Int32Regs:$a),
 604           (CVT_f64_u32 Int32Regs:$a, CvtRM)>;
 605 def : Pat<(int_nvvm_ui2d_rp Int32Regs:$a),
 606           (CVT_f64_u32 Int32Regs:$a, CvtRP)>;
 607
 608 def : Pat<(int_nvvm_f2i_rn_ftz Float32Regs:$a),
 609           (CVT_s32_f32 Float32Regs:$a, CvtRNI_FTZ)>;
 610 def : Pat<(int_nvvm_f2i_rn Float32Regs:$a),
 611           (CVT_s32_f32 Float32Regs:$a, CvtRNI)>;
 612 def : Pat<(int_nvvm_f2i_rz_ftz Float32Regs:$a),
 613           (CVT_s32_f32 Float32Regs:$a, CvtRZI_FTZ)>;
 614 def : Pat<(int_nvvm_f2i_rz Float32Regs:$a),
 615           (CVT_s32_f32 Float32Regs:$a, CvtRZI)>;
 616 def : Pat<(int_nvvm_f2i_rm_ftz Float32Regs:$a),
 617           (CVT_s32_f32 Float32Regs:$a, CvtRMI_FTZ)>;
 618 def : Pat<(int_nvvm_f2i_rm Float32Regs:$a),
 619           (CVT_s32_f32 Float32Regs:$a, CvtRMI)>;
 620 def : Pat<(int_nvvm_f2i_rp_ftz Float32Regs:$a),
 621           (CVT_s32_f32 Float32Regs:$a, CvtRPI_FTZ)>;
 622 def : Pat<(int_nvvm_f2i_rp Float32Regs:$a),
 623           (CVT_s32_f32 Float32Regs:$a, CvtRPI)>;
 624
 625 def : Pat<(int_nvvm_f2ui_rn_ftz Float32Regs:$a),
 626           (CVT_u32_f32 Float32Regs:$a, CvtRNI_FTZ)>;
 627 def : Pat<(int_nvvm_f2ui_rn Float32Regs:$a),
 628           (CVT_u32_f32 Float32Regs:$a, CvtRNI)>;
 629 def : Pat<(int_nvvm_f2ui_rz_ftz Float32Regs:$a),
 630           (CVT_u32_f32 Float32Regs:$a, CvtRZI_FTZ)>;
 631 def : Pat<(int_nvvm_f2ui_rz Float32Regs:$a),
 632           (CVT_u32_f32 Float32Regs:$a, CvtRZI)>;
 633 def : Pat<(int_nvvm_f2ui_rm_ftz Float32Regs:$a),
 634           (CVT_u32_f32 Float32Regs:$a, CvtRMI_FTZ)>;
 635 def : Pat<(int_nvvm_f2ui_rm Float32Regs:$a),
 636           (CVT_u32_f32 Float32Regs:$a, CvtRMI)>;
 637 def : Pat<(int_nvvm_f2ui_rp_ftz Float32Regs:$a),
 638           (CVT_u32_f32 Float32Regs:$a, CvtRPI_FTZ)>;
 639 def : Pat<(int_nvvm_f2ui_rp Float32Regs:$a),
 640           (CVT_u32_f32 Float32Regs:$a, CvtRPI)>;
 641
 642 def : Pat<(int_nvvm_i2f_rn Int32Regs:$a),
 643           (CVT_f32_s32 Int32Regs:$a, CvtRN)>;
 644 def : Pat<(int_nvvm_i2f_rz Int32Regs:$a),
 645           (CVT_f32_s32 Int32Regs:$a, CvtRZ)>;
 646 def : Pat<(int_nvvm_i2f_rm Int32Regs:$a),
 647           (CVT_f32_s32 Int32Regs:$a, CvtRM)>;
 648 def : Pat<(int_nvvm_i2f_rp Int32Regs:$a),
 649           (CVT_f32_s32 Int32Regs:$a, CvtRP)>;
 650
 651 def : Pat<(int_nvvm_ui2f_rn Int32Regs:$a),
 652           (CVT_f32_u32 Int32Regs:$a, CvtRN)>;
 653 def : Pat<(int_nvvm_ui2f_rz Int32Regs:$a),
 654           (CVT_f32_u32 Int32Regs:$a, CvtRZ)>;
 655 def : Pat<(int_nvvm_ui2f_rm Int32Regs:$a),
 656           (CVT_f32_u32 Int32Regs:$a, CvtRM)>;
 657 def : Pat<(int_nvvm_ui2f_rp Int32Regs:$a),
 658           (CVT_f32_u32 Int32Regs:$a, CvtRP)>;
 659
 660 def INT_NVVM_LOHI_I2D : F_MATH_2<"mov.b64 \t$dst, {{$src0, $src1}};",
 661   Float64Regs, Int32Regs, Int32Regs, int_nvvm_lohi_i2d>;
 662
 663 def INT_NVVM_D2I_LO : F_MATH_1<!strconcat("{{\n\t",
 664                        !strconcat(".reg .b32 %temp; \n\t",
 665              !strconcat("mov.b64 \t{$dst, %temp}, $src0;\n\t",
 666                "}}"))),
 667              Int32Regs, Float64Regs, int_nvvm_d2i_lo>;
 668 def INT_NVVM_D2I_HI : F_MATH_1<!strconcat("{{\n\t",
 669                        !strconcat(".reg .b32 %temp; \n\t",
 670                          !strconcat("mov.b64 \t{%temp, $dst}, $src0;\n\t",
 671                            "}}"))),
 672              Int32Regs, Float64Regs, int_nvvm_d2i_hi>;
 673
 674 def : Pat<(int_nvvm_f2ll_rn_ftz Float32Regs:$a),
 675           (CVT_s64_f32 Float32Regs:$a, CvtRNI_FTZ)>;
 676 def : Pat<(int_nvvm_f2ll_rn Float32Regs:$a),
 677           (CVT_s64_f32 Float32Regs:$a, CvtRNI)>;
 678 def : Pat<(int_nvvm_f2ll_rz_ftz Float32Regs:$a),
 679           (CVT_s64_f32 Float32Regs:$a, CvtRZI_FTZ)>;
 680 def : Pat<(int_nvvm_f2ll_rz Float32Regs:$a),
 681           (CVT_s64_f32 Float32Regs:$a, CvtRZI)>;
 682 def : Pat<(int_nvvm_f2ll_rm_ftz Float32Regs:$a),
 683           (CVT_s64_f32 Float32Regs:$a, CvtRMI_FTZ)>;
 684 def : Pat<(int_nvvm_f2ll_rm Float32Regs:$a),
 685           (CVT_s64_f32 Float32Regs:$a, CvtRMI)>;
 686 def : Pat<(int_nvvm_f2ll_rp_ftz Float32Regs:$a),
 687           (CVT_s64_f32 Float32Regs:$a, CvtRPI_FTZ)>;
 688 def : Pat<(int_nvvm_f2ll_rp Float32Regs:$a),
 689           (CVT_s64_f32 Float32Regs:$a, CvtRPI)>;
 690
 691 def : Pat<(int_nvvm_f2ull_rn_ftz Float32Regs:$a),
 692           (CVT_u64_f32 Float32Regs:$a, CvtRNI_FTZ)>;
 693 def : Pat<(int_nvvm_f2ull_rn Float32Regs:$a),
 694           (CVT_u64_f32 Float32Regs:$a, CvtRNI)>;
 695 def : Pat<(int_nvvm_f2ull_rz_ftz Float32Regs:$a),
 696           (CVT_u64_f32 Float32Regs:$a, CvtRZI_FTZ)>;
 697 def : Pat<(int_nvvm_f2ull_rz Float32Regs:$a),
 698           (CVT_u64_f32 Float32Regs:$a, CvtRZI)>;
 699 def : Pat<(int_nvvm_f2ull_rm_ftz Float32Regs:$a),
 700           (CVT_u64_f32 Float32Regs:$a, CvtRMI_FTZ)>;
 701 def : Pat<(int_nvvm_f2ull_rm Float32Regs:$a),
 702           (CVT_u64_f32 Float32Regs:$a, CvtRMI)>;
 703 def : Pat<(int_nvvm_f2ull_rp_ftz Float32Regs:$a),
 704           (CVT_u64_f32 Float32Regs:$a, CvtRPI_FTZ)>;
 705 def : Pat<(int_nvvm_f2ull_rp Float32Regs:$a),
 706           (CVT_u64_f32 Float32Regs:$a, CvtRPI)>;
 707
 708 def : Pat<(int_nvvm_d2ll_rn Float64Regs:$a),
 709           (CVT_s64_f64 Float64Regs:$a, CvtRNI)>;
 710 def : Pat<(int_nvvm_d2ll_rz Float64Regs:$a),
 711           (CVT_s64_f64 Float64Regs:$a, CvtRZI)>;
 712 def : Pat<(int_nvvm_d2ll_rm Float64Regs:$a),
 713           (CVT_s64_f64 Float64Regs:$a, CvtRMI)>;
 714 def : Pat<(int_nvvm_d2ll_rp Float64Regs:$a),
 715           (CVT_s64_f64 Float64Regs:$a, CvtRPI)>;
 716
 717 def : Pat<(int_nvvm_d2ull_rn Float64Regs:$a),
 718           (CVT_u64_f64 Float64Regs:$a, CvtRNI)>;
 719 def : Pat<(int_nvvm_d2ull_rz Float64Regs:$a),
 720           (CVT_u64_f64 Float64Regs:$a, CvtRZI)>;
 721 def : Pat<(int_nvvm_d2ull_rm Float64Regs:$a),
 722           (CVT_u64_f64 Float64Regs:$a, CvtRMI)>;
 723 def : Pat<(int_nvvm_d2ull_rp Float64Regs:$a),
 724           (CVT_u64_f64 Float64Regs:$a, CvtRPI)>;
 725
 726 def : Pat<(int_nvvm_ll2f_rn Int64Regs:$a),
 727           (CVT_f32_s64 Int64Regs:$a, CvtRN)>;
 728 def : Pat<(int_nvvm_ll2f_rz Int64Regs:$a),
 729           (CVT_f32_s64 Int64Regs:$a, CvtRZ)>;
 730 def : Pat<(int_nvvm_ll2f_rm Int64Regs:$a),
 731           (CVT_f32_s64 Int64Regs:$a, CvtRM)>;
 732 def : Pat<(int_nvvm_ll2f_rp Int64Regs:$a),
 733           (CVT_f32_s64 Int64Regs:$a, CvtRP)>;
 734
 735 def : Pat<(int_nvvm_ull2f_rn Int64Regs:$a),
 736           (CVT_f32_u64 Int64Regs:$a, CvtRN)>;
 737 def : Pat<(int_nvvm_ull2f_rz Int64Regs:$a),
 738           (CVT_f32_u64 Int64Regs:$a, CvtRZ)>;
 739 def : Pat<(int_nvvm_ull2f_rm Int64Regs:$a),
 740           (CVT_f32_u64 Int64Regs:$a, CvtRM)>;
 741 def : Pat<(int_nvvm_ull2f_rp Int64Regs:$a),
 742           (CVT_f32_u64 Int64Regs:$a, CvtRP)>;
 743
 744 def : Pat<(int_nvvm_ll2d_rn Int64Regs:$a),
 745           (CVT_f64_s64 Int64Regs:$a, CvtRN)>;
 746 def : Pat<(int_nvvm_ll2d_rz Int64Regs:$a),
 747           (CVT_f64_s64 Int64Regs:$a, CvtRZ)>;
 748 def : Pat<(int_nvvm_ll2d_rm Int64Regs:$a),
 749           (CVT_f64_s64 Int64Regs:$a, CvtRM)>;
 750 def : Pat<(int_nvvm_ll2d_rp Int64Regs:$a),
 751           (CVT_f64_s64 Int64Regs:$a, CvtRP)>;
 752
 753 def : Pat<(int_nvvm_ull2d_rn Int64Regs:$a),
 754           (CVT_f64_u64 Int64Regs:$a, CvtRN)>;
 755 def : Pat<(int_nvvm_ull2d_rz Int64Regs:$a),
 756           (CVT_f64_u64 Int64Regs:$a, CvtRZ)>;
 757 def : Pat<(int_nvvm_ull2d_rm Int64Regs:$a),
 758           (CVT_f64_u64 Int64Regs:$a, CvtRM)>;
 759 def : Pat<(int_nvvm_ull2d_rp Int64Regs:$a),
 760           (CVT_f64_u64 Int64Regs:$a, CvtRP)>;
 761
 762
 763 // FIXME: Ideally, we could use these patterns instead of the scope-creating
 764 // patterns, but ptxas does not like these since .s16 is not compatible with
 765 // .f16.  The solution is to use .bXX for all integer register types, but we
 766 // are not there yet.
 767 //def : Pat<(int_nvvm_f2h_rn_ftz Float32Regs:$a),
 768 //          (CVT_f16_f32 Float32Regs:$a, CvtRN_FTZ)>;
 769 //def : Pat<(int_nvvm_f2h_rn Float32Regs:$a),
 770 //          (CVT_f16_f32 Float32Regs:$a, CvtRN)>;
 771 //
 772 //def : Pat<(int_nvvm_h2f Int16Regs:$a),
 773 //          (CVT_f32_f16 Int16Regs:$a, CvtNONE)>;
 774
 775 def INT_NVVM_F2H_RN_FTZ : F_MATH_1<!strconcat("{{\n\t",
 776                                    !strconcat(".reg .b16 %temp;\n\t",
 777            !strconcat("cvt.rn.ftz.f16.f32 \t%temp, $src0;\n\t",
 778            !strconcat("mov.b16 \t$dst, %temp;\n",
 779              "}}")))),
 780                                    Int16Regs, Float32Regs, int_nvvm_f2h_rn_ftz>;
 781 def INT_NVVM_F2H_RN : F_MATH_1<!strconcat("{{\n\t",
 782                                    !strconcat(".reg .b16 %temp;\n\t",
 783            !strconcat("cvt.rn.f16.f32 \t%temp, $src0;\n\t",
 784            !strconcat("mov.b16 \t$dst, %temp;\n",
 785              "}}")))),
 786            Int16Regs, Float32Regs, int_nvvm_f2h_rn>;
 787
 788 def INT_NVVM_H2F : F_MATH_1<!strconcat("{{\n\t",
 789                             !strconcat(".reg .b16 %temp;\n\t",
 790           !strconcat("mov.b16 \t%temp, $src0;\n\t",
 791           !strconcat("cvt.f32.f16 \t$dst, %temp;\n\t",
 792             "}}")))),
 793           Float32Regs, Int16Regs, int_nvvm_h2f>;
 794
 795 def : Pat<(f32 (f16_to_fp Int16Regs:$a)),
 796           (CVT_f32_f16 Int16Regs:$a, CvtNONE)>;
 797 def : Pat<(i16 (fp_to_f16 Float32Regs:$a)),
 798           (CVT_f16_f32 Float32Regs:$a, CvtRN_FTZ)>, Requires<[doF32FTZ]>;
 799 def : Pat<(i16 (fp_to_f16 Float32Regs:$a)),
 800           (CVT_f16_f32 Float32Regs:$a, CvtRN)>;
 801
 802 //
 803 // Bitcast
 804 //
 805
 806 def INT_NVVM_BITCAST_F2I : F_MATH_1<"mov.b32 \t$dst, $src0;", Int32Regs,
 807   Float32Regs, int_nvvm_bitcast_f2i>;
 808 def INT_NVVM_BITCAST_I2F : F_MATH_1<"mov.b32 \t$dst, $src0;", Float32Regs,
 809   Int32Regs, int_nvvm_bitcast_i2f>;
 810
 811 def INT_NVVM_BITCAST_LL2D : F_MATH_1<"mov.b64 \t$dst, $src0;", Float64Regs,
 812   Int64Regs, int_nvvm_bitcast_ll2d>;
 813 def INT_NVVM_BITCAST_D2LL : F_MATH_1<"mov.b64 \t$dst, $src0;", Int64Regs,
 814   Float64Regs, int_nvvm_bitcast_d2ll>;
 815
 816 //-----------------------------------
 817 // Atomic Functions
 818 //-----------------------------------
 819
 820 class ATOMIC_GLOBAL_CHK <dag ops, dag frag>
 821  : PatFrag<ops, frag, [{
 822    return ChkMemSDNodeAddressSpace(N, llvm::ADDRESS_SPACE_GLOBAL);
 823 }]>;
 824 class ATOMIC_SHARED_CHK <dag ops, dag frag>
 825  : PatFrag<ops, frag, [{
 826    return ChkMemSDNodeAddressSpace(N, llvm::ADDRESS_SPACE_SHARED);
 827 }]>;
 828 class ATOMIC_GENERIC_CHK <dag ops, dag frag>
 829  : PatFrag<ops, frag, [{
 830    return ChkMemSDNodeAddressSpace(N, llvm::ADDRESS_SPACE_GENERIC);
 831 }]>;
 832
 833 multiclass F_ATOMIC_2_imp<NVPTXRegClass ptrclass, NVPTXRegClass regclass,
 834   string SpaceStr, string TypeStr, string OpcStr, PatFrag IntOp,
 835   Operand IMMType, SDNode IMM, Predicate Pred> {
 836   def reg : NVPTXInst<(outs regclass:$dst), (ins ptrclass:$addr, regclass:$b),
 837                !strconcat("atom",
 838          !strconcat(SpaceStr,
 839          !strconcat(OpcStr,
 840          !strconcat(TypeStr,
 841          !strconcat(" \t$dst, [$addr], $b;", ""))))),
 842          [(set regclass:$dst, (IntOp ptrclass:$addr, regclass:$b))]>,
 843   Requires<[Pred]>;
 844   def imm : NVPTXInst<(outs regclass:$dst), (ins ptrclass:$addr, IMMType:$b),
 845                !strconcat("atom",
 846          !strconcat(SpaceStr,
 847          !strconcat(OpcStr,
 848          !strconcat(TypeStr,
 849          !strconcat(" \t$dst, [$addr], $b;", ""))))),
 850          [(set regclass:$dst, (IntOp ptrclass:$addr, IMM:$b))]>,
 851   Requires<[Pred]>;
 852 }
 853 multiclass F_ATOMIC_2<NVPTXRegClass regclass, string SpaceStr, string TypeStr,
 854   string OpcStr, PatFrag IntOp, Operand IMMType, SDNode IMM, Predicate Pred> {
 855   defm p32 : F_ATOMIC_2_imp<Int32Regs, regclass, SpaceStr, TypeStr, OpcStr,
 856     IntOp, IMMType, IMM, Pred>;
 857   defm p64 : F_ATOMIC_2_imp<Int64Regs, regclass, SpaceStr, TypeStr, OpcStr,
 858     IntOp, IMMType, IMM, Pred>;
 859 }
 860
 861 // has 2 operands, neg the second one
 862 multiclass F_ATOMIC_2_NEG_imp<NVPTXRegClass ptrclass, NVPTXRegClass regclass,
 863   string SpaceStr, string TypeStr, string OpcStr, PatFrag IntOp,
 864   Operand IMMType, Predicate Pred> {
 865   def reg : NVPTXInst<(outs regclass:$dst), (ins ptrclass:$addr, regclass:$b),
 866     !strconcat("{{ \n\t",
 867          !strconcat(".reg \t.s",
 868          !strconcat(TypeStr,
 869          !strconcat(" temp; \n\t",
 870          !strconcat("neg.s",
 871          !strconcat(TypeStr,
 872          !strconcat(" \ttemp, $b; \n\t",
 873                !strconcat("atom",
 874          !strconcat(SpaceStr,
 875          !strconcat(OpcStr,
 876          !strconcat(".u",
 877          !strconcat(TypeStr,
 878          !strconcat(" \t$dst, [$addr], temp; \n\t",
 879            !strconcat("}}", "")))))))))))))),
 880          [(set regclass:$dst, (IntOp ptrclass:$addr, regclass:$b))]>,
 881   Requires<[Pred]>;
 882 }
 883 multiclass F_ATOMIC_2_NEG<NVPTXRegClass regclass, string SpaceStr,
 884   string TypeStr, string OpcStr, PatFrag IntOp, Operand IMMType,
 885   Predicate Pred> {
 886  defm p32: F_ATOMIC_2_NEG_imp<Int32Regs, regclass, SpaceStr, TypeStr, OpcStr,
 887    IntOp, IMMType, Pred> ;
 888  defm p64: F_ATOMIC_2_NEG_imp<Int64Regs, regclass, SpaceStr, TypeStr, OpcStr,
 889    IntOp, IMMType, Pred> ;
 890 }
 891
 892 // has 3 operands
 893 multiclass F_ATOMIC_3_imp<NVPTXRegClass ptrclass, NVPTXRegClass regclass,
 894   string SpaceStr, string TypeStr, string OpcStr, PatFrag IntOp,
 895   Operand IMMType, Predicate Pred> {
 896   def reg : NVPTXInst<(outs regclass:$dst),
 897     (ins ptrclass:$addr, regclass:$b, regclass:$c),
 898                !strconcat("atom",
 899          !strconcat(SpaceStr,
 900          !strconcat(OpcStr,
 901          !strconcat(TypeStr,
 902          !strconcat(" \t$dst, [$addr], $b, $c;", ""))))),
 903          [(set regclass:$dst,
 904            (IntOp ptrclass:$addr, regclass:$b, regclass:$c))]>,
 905          Requires<[Pred]>;
 906   def imm1 : NVPTXInst<(outs regclass:$dst),
 907     (ins ptrclass:$addr, IMMType:$b, regclass:$c),
 908                !strconcat("atom",
 909          !strconcat(SpaceStr,
 910          !strconcat(OpcStr,
 911          !strconcat(TypeStr,
 912          !strconcat(" \t$dst, [$addr], $b, $c;", ""))))),
 913          [(set regclass:$dst, (IntOp ptrclass:$addr, imm:$b, regclass:$c))]>,
 914   Requires<[Pred]>;
 915   def imm2 : NVPTXInst<(outs regclass:$dst),
 916     (ins ptrclass:$addr, regclass:$b, IMMType:$c),
 917                !strconcat("atom",
 918          !strconcat(SpaceStr,
 919          !strconcat(OpcStr,
 920          !strconcat(TypeStr,
 921          !strconcat(" \t$dst, [$addr], $b, $c;", ""))))),
 922          [(set regclass:$dst, (IntOp ptrclass:$addr, regclass:$b, imm:$c))]>,
 923   Requires<[Pred]>;
 924   def imm3 : NVPTXInst<(outs regclass:$dst),
 925     (ins ptrclass:$addr, IMMType:$b, IMMType:$c),
 926                !strconcat("atom",
 927          !strconcat(SpaceStr,
 928          !strconcat(OpcStr,
 929          !strconcat(TypeStr,
 930          !strconcat(" \t$dst, [$addr], $b, $c;", ""))))),
 931          [(set regclass:$dst, (IntOp ptrclass:$addr, imm:$b, imm:$c))]>,
 932   Requires<[Pred]>;
 933 }
 934 multiclass F_ATOMIC_3<NVPTXRegClass regclass, string SpaceStr, string TypeStr,
 935   string OpcStr, PatFrag IntOp, Operand IMMType, Predicate Pred> {
 936   defm p32 : F_ATOMIC_3_imp<Int32Regs, regclass, SpaceStr, TypeStr, OpcStr,
 937     IntOp, IMMType, Pred>;
 938   defm p64 : F_ATOMIC_3_imp<Int64Regs, regclass, SpaceStr, TypeStr, OpcStr,
 939     IntOp, IMMType, Pred>;
 940 }
 941
 942 // atom_add
 943
 944 def atomic_load_add_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
 945   (atomic_load_add_32 node:$a, node:$b)>;
 946 def atomic_load_add_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
 947   (atomic_load_add_32 node:$a, node:$b)>;
 948 def atomic_load_add_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
 949   (atomic_load_add_32 node:$a, node:$b)>;
 950 def atomic_load_add_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
 951   (atomic_load_add_64 node:$a, node:$b)>;
 952 def atomic_load_add_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
 953   (atomic_load_add_64 node:$a, node:$b)>;
 954 def atomic_load_add_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
 955   (atomic_load_add_64 node:$a, node:$b)>;
 956 def atomic_load_add_f32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
 957   (int_nvvm_atomic_load_add_f32 node:$a, node:$b)>;
 958 def atomic_load_add_f32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
 959   (int_nvvm_atomic_load_add_f32 node:$a, node:$b)>;
 960 def atomic_load_add_f32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
 961   (int_nvvm_atomic_load_add_f32 node:$a, node:$b)>;
 962
 963 defm INT_PTX_ATOM_ADD_G_32 : F_ATOMIC_2<Int32Regs, ".global", ".u32", ".add",
 964   atomic_load_add_32_g, i32imm, imm, hasAtomRedG32>;
 965 defm INT_PTX_ATOM_ADD_S_32 : F_ATOMIC_2<Int32Regs, ".shared", ".u32", ".add",
 966   atomic_load_add_32_s, i32imm, imm, hasAtomRedS32>;
 967 defm INT_PTX_ATOM_ADD_GEN_32 : F_ATOMIC_2<Int32Regs, "", ".u32", ".add",
 968   atomic_load_add_32_gen, i32imm, imm, hasAtomRedGen32>;
 969 defm INT_PTX_ATOM_ADD_GEN_32_USE_G : F_ATOMIC_2<Int32Regs, ".global", ".u32",
 970   ".add", atomic_load_add_32_gen, i32imm, imm, useAtomRedG32forGen32>;
 971
 972 defm INT_PTX_ATOM_ADD_G_64 : F_ATOMIC_2<Int64Regs, ".global", ".u64", ".add",
 973   atomic_load_add_64_g, i64imm, imm, hasAtomRedG64>;
 974 defm INT_PTX_ATOM_ADD_S_64 : F_ATOMIC_2<Int64Regs, ".shared", ".u64", ".add",
 975   atomic_load_add_64_s, i64imm, imm, hasAtomRedS64>;
 976 defm INT_PTX_ATOM_ADD_GEN_64 : F_ATOMIC_2<Int64Regs, "", ".u64", ".add",
 977   atomic_load_add_64_gen, i64imm, imm, hasAtomRedGen64>;
 978 defm INT_PTX_ATOM_ADD_GEN_64_USE_G : F_ATOMIC_2<Int64Regs, ".global", ".u64",
 979   ".add", atomic_load_add_64_gen, i64imm, imm, useAtomRedG64forGen64>;
 980
 981 defm INT_PTX_ATOM_ADD_G_F32 : F_ATOMIC_2<Float32Regs, ".global", ".f32", ".add",
 982   atomic_load_add_f32_g, f32imm, fpimm, hasAtomAddF32>;
 983 defm INT_PTX_ATOM_ADD_S_F32 : F_ATOMIC_2<Float32Regs, ".shared", ".f32", ".add",
 984   atomic_load_add_f32_s, f32imm, fpimm, hasAtomAddF32>;
 985 defm INT_PTX_ATOM_ADD_GEN_F32 : F_ATOMIC_2<Float32Regs, "", ".f32", ".add",
 986   atomic_load_add_f32_gen, f32imm, fpimm, hasAtomAddF32>;
 987
 988 // atom_sub
 989
 990 def atomic_load_sub_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
 991   (atomic_load_sub_32 node:$a, node:$b)>;
 992 def atomic_load_sub_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
 993   (atomic_load_sub_32 node:$a, node:$b)>;
 994 def atomic_load_sub_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
 995   (atomic_load_sub_32 node:$a, node:$b)>;
 996 def atomic_load_sub_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
 997   (atomic_load_sub_64 node:$a, node:$b)>;
 998 def atomic_load_sub_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
 999   (atomic_load_sub_64 node:$a, node:$b)>;
1000 def atomic_load_sub_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1001   (atomic_load_sub_64 node:$a, node:$b)>;
1002
1003 defm INT_PTX_ATOM_SUB_G_32 : F_ATOMIC_2_NEG<Int32Regs, ".global", "32", ".add",
1004   atomic_load_sub_32_g, i32imm, hasAtomRedG32>;
1005 defm INT_PTX_ATOM_SUB_G_64 : F_ATOMIC_2_NEG<Int64Regs, ".global", "64", ".add",
1006   atomic_load_sub_64_g, i64imm, hasAtomRedG64>;
1007 defm INT_PTX_ATOM_SUB_GEN_32 : F_ATOMIC_2_NEG<Int32Regs, "", "32", ".add",
1008   atomic_load_sub_32_gen, i32imm, hasAtomRedGen32>;
1009 defm INT_PTX_ATOM_SUB_GEN_32_USE_G : F_ATOMIC_2_NEG<Int32Regs, ".global", "32",
1010   ".add", atomic_load_sub_32_gen, i32imm, useAtomRedG32forGen32>;
1011 defm INT_PTX_ATOM_SUB_S_32 : F_ATOMIC_2_NEG<Int32Regs, ".shared", "32", ".add",
1012   atomic_load_sub_32_s, i32imm, hasAtomRedS32>;
1013 defm INT_PTX_ATOM_SUB_S_64 : F_ATOMIC_2_NEG<Int64Regs, ".shared", "64", ".add",
1014   atomic_load_sub_64_s, i64imm, hasAtomRedS64>;
1015 defm INT_PTX_ATOM_SUB_GEN_64 : F_ATOMIC_2_NEG<Int64Regs, "", "64", ".add",
1016   atomic_load_sub_64_gen, i64imm, hasAtomRedGen64>;
1017 defm INT_PTX_ATOM_SUB_GEN_64_USE_G : F_ATOMIC_2_NEG<Int64Regs, ".global", "64",
1018   ".add", atomic_load_sub_64_gen, i64imm, useAtomRedG64forGen64>;
1019
1020 // atom_swap
1021
1022 def atomic_swap_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1023   (atomic_swap_32 node:$a, node:$b)>;
1024 def atomic_swap_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1025   (atomic_swap_32 node:$a, node:$b)>;
1026 def atomic_swap_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1027   (atomic_swap_32 node:$a, node:$b)>;
1028 def atomic_swap_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1029   (atomic_swap_64 node:$a, node:$b)>;
1030 def atomic_swap_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1031   (atomic_swap_64 node:$a, node:$b)>;
1032 def atomic_swap_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1033   (atomic_swap_64 node:$a, node:$b)>;
1034
1035 defm INT_PTX_ATOM_SWAP_G_32 : F_ATOMIC_2<Int32Regs, ".global", ".b32", ".exch",
1036   atomic_swap_32_g, i32imm, imm, hasAtomRedG32>;
1037 defm INT_PTX_ATOM_SWAP_S_32 : F_ATOMIC_2<Int32Regs, ".shared", ".b32", ".exch",
1038   atomic_swap_32_s, i32imm, imm, hasAtomRedS32>;
1039 defm INT_PTX_ATOM_SWAP_GEN_32 : F_ATOMIC_2<Int32Regs, "", ".b32", ".exch",
1040   atomic_swap_32_gen, i32imm, imm, hasAtomRedGen32>;
1041 defm INT_PTX_ATOM_SWAP_GEN_32_USE_G : F_ATOMIC_2<Int32Regs, ".global", ".b32",
1042   ".exch", atomic_swap_32_gen, i32imm, imm, useAtomRedG32forGen32>;
1043 defm INT_PTX_ATOM_SWAP_G_64 : F_ATOMIC_2<Int64Regs, ".global", ".b64", ".exch",
1044   atomic_swap_64_g, i64imm, imm, hasAtomRedG64>;
1045 defm INT_PTX_ATOM_SWAP_S_64 : F_ATOMIC_2<Int64Regs, ".shared", ".b64", ".exch",
1046   atomic_swap_64_s, i64imm, imm, hasAtomRedS64>;
1047 defm INT_PTX_ATOM_SWAP_GEN_64 : F_ATOMIC_2<Int64Regs, "", ".b64", ".exch",
1048   atomic_swap_64_gen, i64imm, imm, hasAtomRedGen64>;
1049 defm INT_PTX_ATOM_SWAP_GEN_64_USE_G : F_ATOMIC_2<Int64Regs, ".global", ".b64",
1050   ".exch", atomic_swap_64_gen, i64imm, imm, useAtomRedG64forGen64>;
1051
1052 // atom_max
1053
1054 def atomic_load_max_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b)
1055   , (atomic_load_max_32 node:$a, node:$b)>;
1056 def atomic_load_max_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1057   (atomic_load_max_32 node:$a, node:$b)>;
1058 def atomic_load_max_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1059   (atomic_load_max_32 node:$a, node:$b)>;
1060 def atomic_load_max_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b)
1061   , (atomic_load_max_64 node:$a, node:$b)>;
1062 def atomic_load_max_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1063   (atomic_load_max_64 node:$a, node:$b)>;
1064 def atomic_load_max_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1065   (atomic_load_max_64 node:$a, node:$b)>;
1066 def atomic_load_umax_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1067   (atomic_load_umax_32 node:$a, node:$b)>;
1068 def atomic_load_umax_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1069   (atomic_load_umax_32 node:$a, node:$b)>;
1070 def atomic_load_umax_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1071   (atomic_load_umax_32 node:$a, node:$b)>;
1072 def atomic_load_umax_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1073   (atomic_load_umax_64 node:$a, node:$b)>;
1074 def atomic_load_umax_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1075   (atomic_load_umax_64 node:$a, node:$b)>;
1076 def atomic_load_umax_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1077   (atomic_load_umax_64 node:$a, node:$b)>;
1078
1079 defm INT_PTX_ATOM_LOAD_MAX_G_32 : F_ATOMIC_2<Int32Regs, ".global", ".s32",
1080   ".max", atomic_load_max_32_g, i32imm, imm, hasAtomRedG32>;
1081 defm INT_PTX_ATOM_LOAD_MAX_S_32 : F_ATOMIC_2<Int32Regs, ".shared", ".s32",
1082   ".max", atomic_load_max_32_s, i32imm, imm, hasAtomRedS32>;
1083 defm INT_PTX_ATOM_LOAD_MAX_GEN_32 : F_ATOMIC_2<Int32Regs, "", ".s32", ".max",
1084   atomic_load_max_32_gen, i32imm, imm, hasAtomRedGen32>;
1085 defm INT_PTX_ATOM_LOAD_MAX_GEN_32_USE_G : F_ATOMIC_2<Int32Regs, ".global",
1086   ".s32", ".max", atomic_load_max_32_gen, i32imm, imm, useAtomRedG32forGen32>;
1087 defm INT_PTX_ATOM_LOAD_MAX_G_64 : F_ATOMIC_2<Int64Regs, ".global", ".s64",
1088   ".max", atomic_load_max_64_g, i64imm, imm, hasAtomRedG64>;
1089 defm INT_PTX_ATOM_LOAD_MAX_S_64 : F_ATOMIC_2<Int64Regs, ".shared", ".s64",
1090   ".max", atomic_load_max_64_s, i64imm, imm, hasAtomRedS64>;
1091 defm INT_PTX_ATOM_LOAD_MAX_GEN_64 : F_ATOMIC_2<Int64Regs, "", ".s64", ".max",
1092   atomic_load_max_64_gen, i64imm, imm, hasAtomRedGen64>;
1093 defm INT_PTX_ATOM_LOAD_MAX_GEN_64_USE_G : F_ATOMIC_2<Int64Regs, ".global",
1094   ".s64", ".max", atomic_load_max_64_gen, i64imm, imm, useAtomRedG64forGen64>;
1095 defm INT_PTX_ATOM_LOAD_UMAX_G_32 : F_ATOMIC_2<Int32Regs, ".global", ".u32",
1096   ".max", atomic_load_umax_32_g, i32imm, imm, hasAtomRedG32>;
1097 defm INT_PTX_ATOM_LOAD_UMAX_S_32 : F_ATOMIC_2<Int32Regs, ".shared", ".u32",
1098   ".max", atomic_load_umax_32_s, i32imm, imm, hasAtomRedS32>;
1099 defm INT_PTX_ATOM_LOAD_UMAX_GEN_32 : F_ATOMIC_2<Int32Regs, "", ".u32", ".max",
1100   atomic_load_umax_32_gen, i32imm, imm, hasAtomRedGen32>;
1101 defm INT_PTX_ATOM_LOAD_UMAX_GEN_32_USE_G : F_ATOMIC_2<Int32Regs, ".global",
1102   ".u32", ".max", atomic_load_umax_32_gen, i32imm, imm, useAtomRedG32forGen32>;
1103 defm INT_PTX_ATOM_LOAD_UMAX_G_64 : F_ATOMIC_2<Int64Regs, ".global", ".u64",
1104   ".max", atomic_load_umax_64_g, i64imm, imm, hasAtomRedG64>;
1105 defm INT_PTX_ATOM_LOAD_UMAX_S_64 : F_ATOMIC_2<Int64Regs, ".shared", ".u64",
1106   ".max", atomic_load_umax_64_s, i64imm, imm, hasAtomRedS64>;
1107 defm INT_PTX_ATOM_LOAD_UMAX_GEN_64 : F_ATOMIC_2<Int64Regs, "", ".u64", ".max",
1108   atomic_load_umax_64_gen, i64imm, imm, hasAtomRedGen64>;
1109 defm INT_PTX_ATOM_LOAD_UMAX_GEN_64_USE_G : F_ATOMIC_2<Int64Regs, ".global",
1110   ".u64", ".max", atomic_load_umax_64_gen, i64imm, imm, useAtomRedG64forGen64>;
1111
1112 // atom_min
1113
1114 def atomic_load_min_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1115   (atomic_load_min_32 node:$a, node:$b)>;
1116 def atomic_load_min_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1117   (atomic_load_min_32 node:$a, node:$b)>;
1118 def atomic_load_min_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1119   (atomic_load_min_32 node:$a, node:$b)>;
1120 def atomic_load_min_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1121   (atomic_load_min_64 node:$a, node:$b)>;
1122 def atomic_load_min_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1123   (atomic_load_min_64 node:$a, node:$b)>;
1124 def atomic_load_min_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1125   (atomic_load_min_64 node:$a, node:$b)>;
1126 def atomic_load_umin_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1127   (atomic_load_umin_32 node:$a, node:$b)>;
1128 def atomic_load_umin_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1129   (atomic_load_umin_32 node:$a, node:$b)>;
1130 def atomic_load_umin_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1131   (atomic_load_umin_32 node:$a, node:$b)>;
1132 def atomic_load_umin_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1133   (atomic_load_umin_64 node:$a, node:$b)>;
1134 def atomic_load_umin_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1135   (atomic_load_umin_64 node:$a, node:$b)>;
1136 def atomic_load_umin_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1137   (atomic_load_umin_64 node:$a, node:$b)>;
1138
1139 defm INT_PTX_ATOM_LOAD_MIN_G_32 : F_ATOMIC_2<Int32Regs, ".global", ".s32",
1140   ".min", atomic_load_min_32_g, i32imm, imm, hasAtomRedG32>;
1141 defm INT_PTX_ATOM_LOAD_MIN_S_32 : F_ATOMIC_2<Int32Regs, ".shared", ".s32",
1142   ".min", atomic_load_min_32_s, i32imm, imm, hasAtomRedS32>;
1143 defm INT_PTX_ATOM_LOAD_MIN_GEN_32 : F_ATOMIC_2<Int32Regs, "", ".s32", ".min",
1144   atomic_load_min_32_gen, i32imm, imm, hasAtomRedGen32>;
1145 defm INT_PTX_ATOM_LOAD_MIN_GEN_32_USE_G : F_ATOMIC_2<Int32Regs, ".global",
1146   ".s32", ".min", atomic_load_min_32_gen, i32imm, imm, useAtomRedG32forGen32>;
1147 defm INT_PTX_ATOM_LOAD_MIN_G_64 : F_ATOMIC_2<Int64Regs, ".global", ".s64",
1148   ".min", atomic_load_min_64_g, i64imm, imm, hasAtomRedG64>;
1149 defm INT_PTX_ATOM_LOAD_MIN_S_64 : F_ATOMIC_2<Int64Regs, ".shared", ".s64",
1150   ".min", atomic_load_min_64_s, i64imm, imm, hasAtomRedS64>;
1151 defm INT_PTX_ATOM_LOAD_MIN_GEN_64 : F_ATOMIC_2<Int64Regs, "", ".s64", ".min",
1152   atomic_load_min_64_gen, i64imm, imm, hasAtomRedGen64>;
1153 defm INT_PTX_ATOM_LOAD_MIN_GEN_64_USE_G : F_ATOMIC_2<Int64Regs, ".global",
1154   ".s64", ".min", atomic_load_min_64_gen, i64imm, imm, useAtomRedG64forGen64>;
1155 defm INT_PTX_ATOM_LOAD_UMIN_G_32 : F_ATOMIC_2<Int32Regs, ".global", ".u32",
1156   ".min", atomic_load_umin_32_g, i32imm, imm, hasAtomRedG32>;
1157 defm INT_PTX_ATOM_LOAD_UMIN_S_32 : F_ATOMIC_2<Int32Regs, ".shared", ".u32",
1158   ".min", atomic_load_umin_32_s, i32imm, imm, hasAtomRedS32>;
1159 defm INT_PTX_ATOM_LOAD_UMIN_GEN_32 : F_ATOMIC_2<Int32Regs, "", ".u32", ".min",
1160   atomic_load_umin_32_gen, i32imm, imm, hasAtomRedGen32>;
1161 defm INT_PTX_ATOM_LOAD_UMIN_GEN_32_USE_G : F_ATOMIC_2<Int32Regs, ".global",
1162   ".u32", ".min", atomic_load_umin_32_gen, i32imm, imm, useAtomRedG32forGen32>;
1163 defm INT_PTX_ATOM_LOAD_UMIN_G_64 : F_ATOMIC_2<Int64Regs, ".global", ".u64",
1164   ".min", atomic_load_umin_64_g, i64imm, imm, hasAtomRedG64>;
1165 defm INT_PTX_ATOM_LOAD_UMIN_S_64 : F_ATOMIC_2<Int64Regs, ".shared", ".u64",
1166   ".min", atomic_load_umin_64_s, i64imm, imm, hasAtomRedS64>;
1167 defm INT_PTX_ATOM_LOAD_UMIN_GEN_64 : F_ATOMIC_2<Int64Regs, "", ".u64", ".min",
1168   atomic_load_umin_64_gen, i64imm, imm, hasAtomRedGen64>;
1169 defm INT_PTX_ATOM_LOAD_UMIN_GEN_64_USE_G : F_ATOMIC_2<Int64Regs, ".global",
1170   ".u64", ".min", atomic_load_umin_64_gen, i64imm, imm, useAtomRedG64forGen64>;
1171
1172 // atom_inc  atom_dec
1173
1174 def atomic_load_inc_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1175   (int_nvvm_atomic_load_inc_32 node:$a, node:$b)>;
1176 def atomic_load_inc_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1177   (int_nvvm_atomic_load_inc_32 node:$a, node:$b)>;
1178 def atomic_load_inc_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1179   (int_nvvm_atomic_load_inc_32 node:$a, node:$b)>;
1180 def atomic_load_dec_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1181   (int_nvvm_atomic_load_dec_32 node:$a, node:$b)>;
1182 def atomic_load_dec_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1183   (int_nvvm_atomic_load_dec_32 node:$a, node:$b)>;
1184 def atomic_load_dec_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1185   (int_nvvm_atomic_load_dec_32 node:$a, node:$b)>;
1186
1187 defm INT_PTX_ATOM_INC_G_32 : F_ATOMIC_2<Int32Regs, ".global", ".u32", ".inc",
1188   atomic_load_inc_32_g, i32imm, imm, hasAtomRedG32>;
1189 defm INT_PTX_ATOM_INC_S_32 : F_ATOMIC_2<Int32Regs, ".shared", ".u32", ".inc",
1190   atomic_load_inc_32_s, i32imm, imm, hasAtomRedS32>;
1191 defm INT_PTX_ATOM_INC_GEN_32 : F_ATOMIC_2<Int32Regs, "", ".u32", ".inc",
1192   atomic_load_inc_32_gen, i32imm, imm, hasAtomRedGen32>;
1193 defm INT_PTX_ATOM_INC_GEN_32_USE_G : F_ATOMIC_2<Int32Regs, ".global", ".u32",
1194   ".inc", atomic_load_inc_32_gen, i32imm, imm, useAtomRedG32forGen32>;
1195 defm INT_PTX_ATOM_DEC_G_32 : F_ATOMIC_2<Int32Regs, ".global", ".u32", ".dec",
1196   atomic_load_dec_32_g, i32imm, imm, hasAtomRedG32>;
1197 defm INT_PTX_ATOM_DEC_S_32 : F_ATOMIC_2<Int32Regs, ".shared", ".u32", ".dec",
1198   atomic_load_dec_32_s, i32imm, imm, hasAtomRedS32>;
1199 defm INT_PTX_ATOM_DEC_GEN_32 : F_ATOMIC_2<Int32Regs, "", ".u32", ".dec",
1200   atomic_load_dec_32_gen, i32imm, imm, hasAtomRedGen32>;
1201 defm INT_PTX_ATOM_DEC_GEN_32_USE_G : F_ATOMIC_2<Int32Regs, ".global", ".u32",
1202   ".dec", atomic_load_dec_32_gen, i32imm, imm, useAtomRedG32forGen32>;
1203
1204 // atom_and
1205
1206 def atomic_load_and_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1207   (atomic_load_and_32 node:$a, node:$b)>;
1208 def atomic_load_and_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1209   (atomic_load_and_32 node:$a, node:$b)>;
1210 def atomic_load_and_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1211   (atomic_load_and_32 node:$a, node:$b)>;
1212 def atomic_load_and_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1213   (atomic_load_and_64 node:$a, node:$b)>;
1214 def atomic_load_and_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1215   (atomic_load_and_64 node:$a, node:$b)>;
1216 def atomic_load_and_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1217   (atomic_load_and_64 node:$a, node:$b)>;
1218
1219 defm INT_PTX_ATOM_AND_G_32 : F_ATOMIC_2<Int32Regs, ".global", ".b32", ".and",
1220   atomic_load_and_32_g, i32imm, imm, hasAtomRedG32>;
1221 defm INT_PTX_ATOM_AND_S_32 : F_ATOMIC_2<Int32Regs, ".shared", ".b32", ".and",
1222   atomic_load_and_32_s, i32imm, imm, hasAtomRedS32>;
1223 defm INT_PTX_ATOM_AND_GEN_32 : F_ATOMIC_2<Int32Regs, "", ".b32", ".and",
1224   atomic_load_and_32_gen, i32imm, imm, hasAtomRedGen32>;
1225 defm INT_PTX_ATOM_AND_GEN_32_USE_G : F_ATOMIC_2<Int32Regs, ".global", ".b32",
1226   ".and", atomic_load_and_32_gen, i32imm, imm, useAtomRedG32forGen32>;
1227 defm INT_PTX_ATOM_AND_G_64 : F_ATOMIC_2<Int64Regs, ".global", ".b64", ".and",
1228   atomic_load_and_64_g, i64imm, imm, hasAtomRedG64>;
1229 defm INT_PTX_ATOM_AND_S_64 : F_ATOMIC_2<Int64Regs, ".shared", ".b64", ".and",
1230   atomic_load_and_64_s, i64imm, imm, hasAtomRedS64>;
1231 defm INT_PTX_ATOM_AND_GEN_64 : F_ATOMIC_2<Int64Regs, "", ".b64", ".and",
1232   atomic_load_and_64_gen, i64imm, imm, hasAtomRedGen64>;
1233 defm INT_PTX_ATOM_AND_GEN_64_USE_G : F_ATOMIC_2<Int64Regs, ".global", ".b64",
1234   ".and", atomic_load_and_64_gen, i64imm, imm, useAtomRedG64forGen64>;
1235
1236 // atom_or
1237
1238 def atomic_load_or_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1239   (atomic_load_or_32 node:$a, node:$b)>;
1240 def atomic_load_or_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1241   (atomic_load_or_32 node:$a, node:$b)>;
1242 def atomic_load_or_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1243   (atomic_load_or_32 node:$a, node:$b)>;
1244 def atomic_load_or_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1245   (atomic_load_or_64 node:$a, node:$b)>;
1246 def atomic_load_or_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1247   (atomic_load_or_64 node:$a, node:$b)>;
1248 def atomic_load_or_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1249   (atomic_load_or_64 node:$a, node:$b)>;
1250
1251 defm INT_PTX_ATOM_OR_G_32 : F_ATOMIC_2<Int32Regs, ".global", ".b32", ".or",
1252   atomic_load_or_32_g, i32imm, imm, hasAtomRedG32>;
1253 defm INT_PTX_ATOM_OR_GEN_32 : F_ATOMIC_2<Int32Regs, "", ".b32", ".or",
1254   atomic_load_or_32_gen, i32imm, imm, hasAtomRedGen32>;
1255 defm INT_PTX_ATOM_OR_GEN_32_USE_G : F_ATOMIC_2<Int32Regs, ".global", ".b32",
1256   ".or", atomic_load_or_32_gen, i32imm, imm, useAtomRedG32forGen32>;
1257 defm INT_PTX_ATOM_OR_S_32 : F_ATOMIC_2<Int32Regs, ".shared", ".b32", ".or",
1258   atomic_load_or_32_s, i32imm, imm, hasAtomRedS32>;
1259 defm INT_PTX_ATOM_OR_G_64 : F_ATOMIC_2<Int64Regs, ".global", ".b64", ".or",
1260   atomic_load_or_64_g, i64imm, imm, hasAtomRedG64>;
1261 defm INT_PTX_ATOM_OR_GEN_64 : F_ATOMIC_2<Int64Regs, "", ".b64", ".or",
1262   atomic_load_or_64_gen, i64imm, imm, hasAtomRedGen64>;
1263 defm INT_PTX_ATOM_OR_GEN_64_USE_G : F_ATOMIC_2<Int64Regs, ".global", ".b64",
1264   ".or", atomic_load_or_64_gen, i64imm, imm, useAtomRedG64forGen64>;
1265 defm INT_PTX_ATOM_OR_S_64 : F_ATOMIC_2<Int64Regs, ".shared", ".b64", ".or",
1266   atomic_load_or_64_s, i64imm, imm, hasAtomRedS64>;
1267
1268 // atom_xor
1269
1270 def atomic_load_xor_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1271   (atomic_load_xor_32 node:$a, node:$b)>;
1272 def atomic_load_xor_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1273   (atomic_load_xor_32 node:$a, node:$b)>;
1274 def atomic_load_xor_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1275   (atomic_load_xor_32 node:$a, node:$b)>;
1276 def atomic_load_xor_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1277   (atomic_load_xor_64 node:$a, node:$b)>;
1278 def atomic_load_xor_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1279   (atomic_load_xor_64 node:$a, node:$b)>;
1280 def atomic_load_xor_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1281   (atomic_load_xor_64 node:$a, node:$b)>;
1282
1283 defm INT_PTX_ATOM_XOR_G_32 : F_ATOMIC_2<Int32Regs, ".global", ".b32", ".xor",
1284   atomic_load_xor_32_g, i32imm, imm, hasAtomRedG32>;
1285 defm INT_PTX_ATOM_XOR_S_32 : F_ATOMIC_2<Int32Regs, ".shared", ".b32", ".xor",
1286   atomic_load_xor_32_s, i32imm, imm, hasAtomRedS32>;
1287 defm INT_PTX_ATOM_XOR_GEN_32 : F_ATOMIC_2<Int32Regs, "", ".b32", ".xor",
1288   atomic_load_xor_32_gen, i32imm, imm, hasAtomRedGen32>;
1289 defm INT_PTX_ATOM_XOR_GEN_32_USE_G : F_ATOMIC_2<Int32Regs, ".global", ".b32",
1290   ".xor", atomic_load_xor_32_gen, i32imm, imm, useAtomRedG32forGen32>;
1291 defm INT_PTX_ATOM_XOR_G_64 : F_ATOMIC_2<Int64Regs, ".global", ".b64", ".xor",
1292   atomic_load_xor_64_g, i64imm, imm, hasAtomRedG64>;
1293 defm INT_PTX_ATOM_XOR_S_64 : F_ATOMIC_2<Int64Regs, ".shared", ".b64", ".xor",
1294   atomic_load_xor_64_s, i64imm, imm, hasAtomRedS64>;
1295 defm INT_PTX_ATOM_XOR_GEN_64 : F_ATOMIC_2<Int64Regs, "", ".b64", ".xor",
1296   atomic_load_xor_64_gen, i64imm, imm, hasAtomRedGen64>;
1297 defm INT_PTX_ATOM_XOR_GEN_64_USE_G : F_ATOMIC_2<Int64Regs, ".global", ".b64",
1298   ".xor", atomic_load_xor_64_gen, i64imm, imm, useAtomRedG64forGen64>;
1299
1300 // atom_cas
1301
1302 def atomic_cmp_swap_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b, node:$c),
1303   (atomic_cmp_swap_32 node:$a, node:$b, node:$c)>;
1304 def atomic_cmp_swap_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b, node:$c),
1305   (atomic_cmp_swap_32 node:$a, node:$b, node:$c)>;
1306 def atomic_cmp_swap_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b, node:$c),
1307   (atomic_cmp_swap_32 node:$a, node:$b, node:$c)>;
1308 def atomic_cmp_swap_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b, node:$c),
1309   (atomic_cmp_swap_64 node:$a, node:$b, node:$c)>;
1310 def atomic_cmp_swap_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b, node:$c),
1311   (atomic_cmp_swap_64 node:$a, node:$b, node:$c)>;
1312 def atomic_cmp_swap_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b, node:$c),
1313   (atomic_cmp_swap_64 node:$a, node:$b, node:$c)>;
1314
1315 defm INT_PTX_ATOM_CAS_G_32 : F_ATOMIC_3<Int32Regs, ".global", ".b32", ".cas",
1316   atomic_cmp_swap_32_g, i32imm, hasAtomRedG32>;
1317 defm INT_PTX_ATOM_CAS_S_32 : F_ATOMIC_3<Int32Regs, ".shared", ".b32", ".cas",
1318   atomic_cmp_swap_32_s, i32imm, hasAtomRedS32>;
1319 defm INT_PTX_ATOM_CAS_GEN_32 : F_ATOMIC_3<Int32Regs, "", ".b32", ".cas",
1320   atomic_cmp_swap_32_gen, i32imm, hasAtomRedGen32>;
1321 defm INT_PTX_ATOM_CAS_GEN_32_USE_G : F_ATOMIC_3<Int32Regs, ".global", ".b32",
1322   ".cas", atomic_cmp_swap_32_gen, i32imm, useAtomRedG32forGen32>;
1323 defm INT_PTX_ATOM_CAS_G_64 : F_ATOMIC_3<Int64Regs, ".global", ".b64", ".cas",
1324   atomic_cmp_swap_64_g, i64imm, hasAtomRedG64>;
1325 defm INT_PTX_ATOM_CAS_S_64 : F_ATOMIC_3<Int64Regs, ".shared", ".b64", ".cas",
1326   atomic_cmp_swap_64_s, i64imm, hasAtomRedS64>;
1327 defm INT_PTX_ATOM_CAS_GEN_64 : F_ATOMIC_3<Int64Regs, "", ".b64", ".cas",
1328   atomic_cmp_swap_64_gen, i64imm, hasAtomRedGen64>;
1329 defm INT_PTX_ATOM_CAS_GEN_64_USE_G : F_ATOMIC_3<Int64Regs, ".global", ".b64",
1330   ".cas", atomic_cmp_swap_64_gen, i64imm, useAtomRedG64forGen64>;
1331
1332
1333 //-----------------------------------
1334 // Read Special Registers
1335 //-----------------------------------
1336 class F_SREG<string OpStr, NVPTXRegClass regclassOut, Intrinsic IntOp> :
1337       NVPTXInst<(outs regclassOut:$dst), (ins),
1338                OpStr,
1339          [(set regclassOut:$dst, (IntOp))]>;
1340
1341 def INT_PTX_SREG_TID_X : F_SREG<"mov.u32 \t$dst, %tid.x;", Int32Regs,
1342   int_nvvm_read_ptx_sreg_tid_x>;
1343 def INT_PTX_SREG_TID_Y : F_SREG<"mov.u32 \t$dst, %tid.y;", Int32Regs,
1344   int_nvvm_read_ptx_sreg_tid_y>;
1345 def INT_PTX_SREG_TID_Z : F_SREG<"mov.u32 \t$dst, %tid.z;", Int32Regs,
1346   int_nvvm_read_ptx_sreg_tid_z>;
1347
1348 def INT_PTX_SREG_NTID_X : F_SREG<"mov.u32 \t$dst, %ntid.x;", Int32Regs,
1349   int_nvvm_read_ptx_sreg_ntid_x>;
1350 def INT_PTX_SREG_NTID_Y : F_SREG<"mov.u32 \t$dst, %ntid.y;", Int32Regs,
1351   int_nvvm_read_ptx_sreg_ntid_y>;
1352 def INT_PTX_SREG_NTID_Z : F_SREG<"mov.u32 \t$dst, %ntid.z;", Int32Regs,
1353   int_nvvm_read_ptx_sreg_ntid_z>;
1354
1355 def INT_PTX_SREG_CTAID_X : F_SREG<"mov.u32 \t$dst, %ctaid.x;", Int32Regs,
1356   int_nvvm_read_ptx_sreg_ctaid_x>;
1357 def INT_PTX_SREG_CTAID_Y : F_SREG<"mov.u32 \t$dst, %ctaid.y;", Int32Regs,
1358   int_nvvm_read_ptx_sreg_ctaid_y>;
1359 def INT_PTX_SREG_CTAID_Z : F_SREG<"mov.u32 \t$dst, %ctaid.z;", Int32Regs,
1360   int_nvvm_read_ptx_sreg_ctaid_z>;
1361
1362 def INT_PTX_SREG_NCTAID_X : F_SREG<"mov.u32 \t$dst, %nctaid.x;", Int32Regs,
1363   int_nvvm_read_ptx_sreg_nctaid_x>;
1364 def INT_PTX_SREG_NCTAID_Y : F_SREG<"mov.u32 \t$dst, %nctaid.y;", Int32Regs,
1365   int_nvvm_read_ptx_sreg_nctaid_y>;
1366 def INT_PTX_SREG_NCTAID_Z : F_SREG<"mov.u32 \t$dst, %nctaid.z;", Int32Regs,
1367   int_nvvm_read_ptx_sreg_nctaid_z>;
1368
1369 def INT_PTX_SREG_WARPSIZE : F_SREG<"mov.u32 \t$dst, WARP_SZ;", Int32Regs,
1370   int_nvvm_read_ptx_sreg_warpsize>;
1371
1372
1373 //-----------------------------------
1374 // Support for ldu on sm_20 or later
1375 //-----------------------------------
1376
1377 // Scalar
1378 multiclass LDU_G<string TyStr, NVPTXRegClass regclass> {
1379   def areg: NVPTXInst<(outs regclass:$result), (ins Int32Regs:$src),
1380                !strconcat("ldu.global.", TyStr),
1381                       []>, Requires<[hasLDU]>;
1382   def areg64: NVPTXInst<(outs regclass:$result), (ins Int64Regs:$src),
1383                !strconcat("ldu.global.", TyStr),
1384                         []>, Requires<[hasLDU]>;
1385  def avar:  NVPTXInst<(outs regclass:$result), (ins imemAny:$src),
1386                !strconcat("ldu.global.", TyStr),
1387                       []>, Requires<[hasLDU]>;
1388  def ari :  NVPTXInst<(outs regclass:$result), (ins MEMri:$src),
1389                !strconcat("ldu.global.", TyStr),
1390                       []>, Requires<[hasLDU]>;
1391  def ari64 :  NVPTXInst<(outs regclass:$result), (ins MEMri64:$src),
1392                !strconcat("ldu.global.", TyStr),
1393                         []>, Requires<[hasLDU]>;
1394 }
1395
1396 defm INT_PTX_LDU_GLOBAL_i8  : LDU_G<"u8 \t$result, [$src];", Int16Regs>;
1397 defm INT_PTX_LDU_GLOBAL_i16 : LDU_G<"u16 \t$result, [$src];", Int16Regs>;
1398 defm INT_PTX_LDU_GLOBAL_i32 : LDU_G<"u32 \t$result, [$src];", Int32Regs>;
1399 defm INT_PTX_LDU_GLOBAL_i64 : LDU_G<"u64 \t$result, [$src];", Int64Regs>;
1400 defm INT_PTX_LDU_GLOBAL_f32 : LDU_G<"f32 \t$result, [$src];", Float32Regs>;
1401 defm INT_PTX_LDU_GLOBAL_f64 : LDU_G<"f64 \t$result, [$src];", Float64Regs>;
1402 defm INT_PTX_LDU_GLOBAL_p32 : LDU_G<"u32 \t$result, [$src];", Int32Regs>;
1403 defm INT_PTX_LDU_GLOBAL_p64 : LDU_G<"u64 \t$result, [$src];", Int64Regs>;
1404
1405 // vector
1406
1407 // Elementized vector ldu
1408 multiclass VLDU_G_ELE_V2<string TyStr, NVPTXRegClass regclass> {
1409  def _areg32: NVPTXInst<(outs regclass:$dst1, regclass:$dst2),
1410                      (ins Int32Regs:$src),
1411                      !strconcat("ldu.global.", TyStr), []>;
1412  def _areg64: NVPTXInst<(outs regclass:$dst1, regclass:$dst2),
1413                      (ins Int64Regs:$src),
1414                      !strconcat("ldu.global.", TyStr), []>;
1415  def _ari32: NVPTXInst<(outs regclass:$dst1, regclass:$dst2),
1416                      (ins MEMri:$src),
1417                      !strconcat("ldu.global.", TyStr), []>;
1418  def _ari64: NVPTXInst<(outs regclass:$dst1, regclass:$dst2),
1419                      (ins MEMri64:$src),
1420                      !strconcat("ldu.global.", TyStr), []>;
1421  def _avar: NVPTXInst<(outs regclass:$dst1, regclass:$dst2),
1422                      (ins imemAny:$src),
1423                      !strconcat("ldu.global.", TyStr), []>;
1424 }
1425
1426 multiclass VLDU_G_ELE_V4<string TyStr, NVPTXRegClass regclass> {
1427  def _areg32: NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
1428                             regclass:$dst4), (ins Int32Regs:$src),
1429                !strconcat("ldu.global.", TyStr), []>;
1430  def _areg64: NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
1431                             regclass:$dst4), (ins Int64Regs:$src),
1432                !strconcat("ldu.global.", TyStr), []>;
1433  def _ari32: NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
1434                             regclass:$dst4), (ins MEMri:$src),
1435                !strconcat("ldu.global.", TyStr), []>;
1436  def _ari64: NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
1437                             regclass:$dst4), (ins MEMri64:$src),
1438                !strconcat("ldu.global.", TyStr), []>;
1439  def _avar: NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
1440                             regclass:$dst4), (ins imemAny:$src),
1441                !strconcat("ldu.global.", TyStr), []>;
1442 }
1443
1444 defm INT_PTX_LDU_G_v2i8_ELE
1445   : VLDU_G_ELE_V2<"v2.u8 \t{{$dst1, $dst2}}, [$src];",  Int16Regs>;
1446 defm INT_PTX_LDU_G_v2i16_ELE
1447   : VLDU_G_ELE_V2<"v2.u16 \t{{$dst1, $dst2}}, [$src];", Int16Regs>;
1448 defm INT_PTX_LDU_G_v2i32_ELE
1449   : VLDU_G_ELE_V2<"v2.u32 \t{{$dst1, $dst2}}, [$src];", Int32Regs>;
1450 defm INT_PTX_LDU_G_v2f32_ELE
1451   : VLDU_G_ELE_V2<"v2.f32 \t{{$dst1, $dst2}}, [$src];", Float32Regs>;
1452 defm INT_PTX_LDU_G_v2i64_ELE
1453   : VLDU_G_ELE_V2<"v2.u64 \t{{$dst1, $dst2}}, [$src];", Int64Regs>;
1454 defm INT_PTX_LDU_G_v2f64_ELE
1455   : VLDU_G_ELE_V2<"v2.f64 \t{{$dst1, $dst2}}, [$src];", Float64Regs>;
1456 defm INT_PTX_LDU_G_v4i8_ELE
1457   : VLDU_G_ELE_V4<"v4.u8 \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];", Int16Regs>;
1458 defm INT_PTX_LDU_G_v4i16_ELE
1459   : VLDU_G_ELE_V4<"v4.u16 \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];",
1460     Int16Regs>;
1461 defm INT_PTX_LDU_G_v4i32_ELE
1462   : VLDU_G_ELE_V4<"v4.u32 \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];",
1463     Int32Regs>;
1464 defm INT_PTX_LDU_G_v4f32_ELE
1465   : VLDU_G_ELE_V4<"v4.f32 \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];",
1466     Float32Regs>;
1467
1468
1469 //-----------------------------------
1470 // Support for ldg on sm_35 or later
1471 //-----------------------------------
1472
1473 multiclass LDG_G<string TyStr, NVPTXRegClass regclass> {
1474   def areg: NVPTXInst<(outs regclass:$result), (ins Int32Regs:$src),
1475                !strconcat("ld.global.nc.", TyStr),
1476                       []>, Requires<[hasLDG]>;
1477   def areg64: NVPTXInst<(outs regclass:$result), (ins Int64Regs:$src),
1478                !strconcat("ld.global.nc.", TyStr),
1479                         []>, Requires<[hasLDG]>;
1480  def avar:  NVPTXInst<(outs regclass:$result), (ins imemAny:$src),
1481                !strconcat("ld.global.nc.", TyStr),
1482                       []>, Requires<[hasLDG]>;
1483  def ari :  NVPTXInst<(outs regclass:$result), (ins MEMri:$src),
1484                !strconcat("ld.global.nc.", TyStr),
1485                       []>, Requires<[hasLDG]>;
1486  def ari64 :  NVPTXInst<(outs regclass:$result), (ins MEMri64:$src),
1487                !strconcat("ld.global.nc.", TyStr),
1488                         []>, Requires<[hasLDG]>;
1489 }
1490
1491 defm INT_PTX_LDG_GLOBAL_i8
1492   : LDG_G<"u8 \t$result, [$src];", Int16Regs>;
1493 defm INT_PTX_LDG_GLOBAL_i16
1494   : LDG_G<"u16 \t$result, [$src];", Int16Regs>;
1495 defm INT_PTX_LDG_GLOBAL_i32
1496   : LDG_G<"u32 \t$result, [$src];", Int32Regs>;
1497 defm INT_PTX_LDG_GLOBAL_i64
1498   : LDG_G<"u64 \t$result, [$src];", Int64Regs>;
1499 defm INT_PTX_LDG_GLOBAL_f32
1500   : LDG_G<"f32 \t$result, [$src];", Float32Regs>;
1501 defm INT_PTX_LDG_GLOBAL_f64
1502   : LDG_G<"f64 \t$result, [$src];", Float64Regs>;
1503 defm INT_PTX_LDG_GLOBAL_p32
1504   : LDG_G<"u32 \t$result, [$src];", Int32Regs>;
1505 defm INT_PTX_LDG_GLOBAL_p64
1506   : LDG_G<"u64 \t$result, [$src];", Int64Regs>;
1507
1508 // vector
1509
1510 // Elementized vector ldg
1511 multiclass VLDG_G_ELE_V2<string TyStr, NVPTXRegClass regclass> {
1512  def _areg32: NVPTXInst<(outs regclass:$dst1, regclass:$dst2),
1513                      (ins Int32Regs:$src),
1514                      !strconcat("ld.global.nc.", TyStr), []>;
1515  def _areg64: NVPTXInst<(outs regclass:$dst1, regclass:$dst2),
1516                      (ins Int64Regs:$src),
1517                      !strconcat("ld.global.nc.", TyStr), []>;
1518  def _ari32: NVPTXInst<(outs regclass:$dst1, regclass:$dst2),
1519                      (ins MEMri:$src),
1520                      !strconcat("ld.global.nc.", TyStr), []>;
1521  def _ari64: NVPTXInst<(outs regclass:$dst1, regclass:$dst2),
1522                      (ins MEMri64:$src),
1523                      !strconcat("ld.global.nc.", TyStr), []>;
1524  def _avar: NVPTXInst<(outs regclass:$dst1, regclass:$dst2),
1525                      (ins imemAny:$src),
1526                      !strconcat("ld.global.nc.", TyStr), []>;
1527 }
1528
1529 multiclass VLDG_G_ELE_V4<string TyStr, NVPTXRegClass regclass> {
1530   def _areg32: NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
1531                               regclass:$dst4), (ins Int32Regs:$src),
1532                !strconcat("ld.global.nc.", TyStr), []>;
1533   def _areg64: NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
1534                                regclass:$dst4), (ins Int64Regs:$src),
1535                !strconcat("ld.global.nc.", TyStr), []>;
1536   def _ari32: NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
1537                               regclass:$dst4), (ins MEMri:$src),
1538                !strconcat("ld.global.nc.", TyStr), []>;
1539   def _ari64: NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
1540                               regclass:$dst4), (ins MEMri64:$src),
1541                !strconcat("ld.global.nc.", TyStr), []>;
1542   def _avar: NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
1543                              regclass:$dst4), (ins imemAny:$src),
1544                !strconcat("ld.global.nc.", TyStr), []>;
1545 }
1546
1547 // FIXME: 8-bit LDG should be fixed once LDG/LDU nodes are made into proper loads.
1548 defm INT_PTX_LDG_G_v2i8_ELE
1549   : VLDG_G_ELE_V2<"v2.u8 \t{{$dst1, $dst2}}, [$src];",  Int16Regs>;
1550 defm INT_PTX_LDG_G_v2i16_ELE
1551   : VLDG_G_ELE_V2<"v2.u16 \t{{$dst1, $dst2}}, [$src];", Int16Regs>;
1552 defm INT_PTX_LDG_G_v2i32_ELE
1553   : VLDG_G_ELE_V2<"v2.u32 \t{{$dst1, $dst2}}, [$src];", Int32Regs>;
1554 defm INT_PTX_LDG_G_v2f32_ELE
1555   : VLDG_G_ELE_V2<"v2.f32 \t{{$dst1, $dst2}}, [$src];", Float32Regs>;
1556 defm INT_PTX_LDG_G_v2i64_ELE
1557   : VLDG_G_ELE_V2<"v2.u64 \t{{$dst1, $dst2}}, [$src];", Int64Regs>;
1558 defm INT_PTX_LDG_G_v2f64_ELE
1559   : VLDG_G_ELE_V2<"v2.f64 \t{{$dst1, $dst2}}, [$src];", Float64Regs>;
1560 defm INT_PTX_LDG_G_v4i8_ELE
1561   : VLDG_G_ELE_V4<"v4.u8 \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];", Int16Regs>;
1562 defm INT_PTX_LDG_G_v4i16_ELE
1563   : VLDG_G_ELE_V4<"v4.u16 \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];", Int16Regs>;
1564 defm INT_PTX_LDG_G_v4i32_ELE
1565   : VLDG_G_ELE_V4<"v4.u32 \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];", Int32Regs>;
1566 defm INT_PTX_LDG_G_v4f32_ELE
1567   : VLDG_G_ELE_V4<"v4.f32 \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];", Float32Regs>;
1568
1569
1570 multiclass NG_TO_G<string Str, Intrinsic Intrin> {
1571    def _yes : NVPTXInst<(outs Int32Regs:$result), (ins Int32Regs:$src),
1572           !strconcat("cvta.", !strconcat(Str, ".u32 \t$result, $src;")),
1573       [(set Int32Regs:$result, (Intrin Int32Regs:$src))]>,
1574    Requires<[hasGenericLdSt]>;
1575    def _yes_64 : NVPTXInst<(outs Int64Regs:$result), (ins Int64Regs:$src),
1576           !strconcat("cvta.", !strconcat(Str, ".u64 \t$result, $src;")),
1577       [(set Int64Regs:$result, (Intrin Int64Regs:$src))]>,
1578    Requires<[hasGenericLdSt]>;
1579
1580 // @TODO: Are these actually needed?  I believe global addresses will be copied
1581 // to register values anyway.
1582    /*def __addr_yes : NVPTXInst<(outs Int32Regs:$result), (ins imemAny:$src),
1583           !strconcat("cvta.", !strconcat(Str, ".u32 \t$result, $src;")),
1584       [(set Int32Regs:$result, (Intrin (Wrapper tglobaladdr:$src)))]>,
1585       Requires<[hasGenericLdSt]>;
1586    def __addr_yes_64 : NVPTXInst<(outs Int64Regs:$result), (ins imemAny:$src),
1587           !strconcat("cvta.", !strconcat(Str, ".u64 \t$result, $src;")),
1588       [(set Int64Regs:$result, (Intrin (Wrapper tglobaladdr:$src)))]>,
1589       Requires<[hasGenericLdSt]>;*/
1590
1591    def _no : NVPTXInst<(outs Int32Regs:$result), (ins Int32Regs:$src),
1592           "mov.u32 \t$result, $src;",
1593       [(set Int32Regs:$result, (Intrin Int32Regs:$src))]>;
1594    def _no_64 : NVPTXInst<(outs Int64Regs:$result), (ins Int64Regs:$src),
1595           "mov.u64 \t$result, $src;",
1596       [(set Int64Regs:$result, (Intrin Int64Regs:$src))]>;
1597
1598 // @TODO: Are these actually needed?  I believe global addresses will be copied
1599 // to register values anyway.
1600    /*def _addr_no : NVPTXInst<(outs Int32Regs:$result), (ins imem:$src),
1601           "mov.u32 \t$result, $src;",
1602       [(set Int32Regs:$result, (Intrin (Wrapper tglobaladdr:$src)))]>;
1603    def _addr_no_64 : NVPTXInst<(outs Int64Regs:$result), (ins imem:$src),
1604           "mov.u64 \t$result, $src;",
1605       [(set Int64Regs:$result, (Intrin (Wrapper tglobaladdr:$src)))]>;*/
1606 }
1607
1608 multiclass G_TO_NG<string Str, Intrinsic Intrin> {
1609    def _yes : NVPTXInst<(outs Int32Regs:$result), (ins Int32Regs:$src),
1610           !strconcat("cvta.to.", !strconcat(Str, ".u32 \t$result, $src;")),
1611       [(set Int32Regs:$result, (Intrin Int32Regs:$src))]>,
1612    Requires<[hasGenericLdSt]>;
1613    def _yes_64 : NVPTXInst<(outs Int64Regs:$result), (ins Int64Regs:$src),
1614           !strconcat("cvta.to.", !strconcat(Str, ".u64 \t$result, $src;")),
1615       [(set Int64Regs:$result, (Intrin Int64Regs:$src))]>,
1616    Requires<[hasGenericLdSt]>;
1617    def _no : NVPTXInst<(outs Int32Regs:$result), (ins Int32Regs:$src),
1618           "mov.u32 \t$result, $src;",
1619       [(set Int32Regs:$result, (Intrin Int32Regs:$src))]>;
1620    def _no_64 : NVPTXInst<(outs Int64Regs:$result), (ins Int64Regs:$src),
1621           "mov.u64 \t$result, $src;",
1622       [(set Int64Regs:$result, (Intrin Int64Regs:$src))]>;
1623 }
1624
1625 defm cvta_local  : NG_TO_G<"local", int_nvvm_ptr_local_to_gen>;
1626 defm cvta_shared : NG_TO_G<"shared", int_nvvm_ptr_shared_to_gen>;
1627 defm cvta_global : NG_TO_G<"global", int_nvvm_ptr_global_to_gen>;
1628 defm cvta_const  : NG_TO_G<"const", int_nvvm_ptr_constant_to_gen>;
1629
1630 defm cvta_to_local   : G_TO_NG<"local", int_nvvm_ptr_gen_to_local>;
1631 defm cvta_to_shared : G_TO_NG<"shared", int_nvvm_ptr_gen_to_shared>;
1632 defm cvta_to_global : G_TO_NG<"global", int_nvvm_ptr_gen_to_global>;
1633 defm cvta_to_const  : G_TO_NG<"const", int_nvvm_ptr_gen_to_constant>;
1634
1635
1636 // nvvm.ptr.gen.to.param
1637 def nvvm_ptr_gen_to_param : NVPTXInst<(outs Int32Regs:$result),
1638   (ins Int32Regs:$src),
1639                         "mov.u32 \t$result, $src;",
1640                               [(set Int32Regs:$result,
1641                                 (int_nvvm_ptr_gen_to_param Int32Regs:$src))]>;
1642 def nvvm_ptr_gen_to_param_64 : NVPTXInst<(outs Int64Regs:$result),
1643   (ins Int64Regs:$src),
1644                         "mov.u64 \t$result, $src;",
1645                               [(set Int64Regs:$result,
1646                                 (int_nvvm_ptr_gen_to_param Int64Regs:$src))]>;
1647
1648
1649 // nvvm.move intrinsicc
1650 def nvvm_move_i16 : NVPTXInst<(outs Int16Regs:$r), (ins Int16Regs:$s),
1651                              "mov.b16 \t$r, $s;",
1652                              [(set Int16Regs:$r,
1653                                (int_nvvm_move_i16 Int16Regs:$s))]>;
1654 def nvvm_move_i32 : NVPTXInst<(outs Int32Regs:$r), (ins Int32Regs:$s),
1655                              "mov.b32 \t$r, $s;",
1656                              [(set Int32Regs:$r,
1657                                (int_nvvm_move_i32 Int32Regs:$s))]>;
1658 def nvvm_move_i64 : NVPTXInst<(outs Int64Regs:$r), (ins Int64Regs:$s),
1659                              "mov.b64 \t$r, $s;",
1660                              [(set Int64Regs:$r,
1661                                (int_nvvm_move_i64 Int64Regs:$s))]>;
1662 def nvvm_move_float : NVPTXInst<(outs Float32Regs:$r), (ins Float32Regs:$s),
1663                              "mov.f32 \t$r, $s;",
1664                              [(set Float32Regs:$r,
1665                                (int_nvvm_move_float Float32Regs:$s))]>;
1666 def nvvm_move_double : NVPTXInst<(outs Float64Regs:$r), (ins Float64Regs:$s),
1667                              "mov.f64 \t$r, $s;",
1668                              [(set Float64Regs:$r,
1669                                (int_nvvm_move_double Float64Regs:$s))]>;
1670 def nvvm_move_ptr32 : NVPTXInst<(outs Int32Regs:$r), (ins Int32Regs:$s),
1671                              "mov.u32 \t$r, $s;",
1672                              [(set Int32Regs:$r,
1673                                (int_nvvm_move_ptr Int32Regs:$s))]>;
1674 def nvvm_move_ptr64 : NVPTXInst<(outs Int64Regs:$r), (ins Int64Regs:$s),
1675                              "mov.u64 \t$r, $s;",
1676                              [(set Int64Regs:$r,
1677                                (int_nvvm_move_ptr Int64Regs:$s))]>;
1678
1679 // @TODO: Are these actually needed, or will we always just see symbols
1680 // copied to registers first?
1681 /*def nvvm_move_sym32 : NVPTXInst<(outs Int32Regs:$r), (ins imem:$s),
1682                              "mov.u32 \t$r, $s;",
1683                              [(set Int32Regs:$r,
1684                              (int_nvvm_move_ptr texternalsym:$s))]>;
1685 def nvvm_move_sym64 : NVPTXInst<(outs Int64Regs:$r), (ins imem:$s),
1686                              "mov.u64 \t$r, $s;",
1687                              [(set Int64Regs:$r,
1688                              (int_nvvm_move_ptr texternalsym:$s))]>;*/
1689
1690
1691 // MoveParam        %r1, param
1692 // ptr_local_to_gen %r2, %r1
1693 // ptr_gen_to_local %r3, %r2
1694 // ->
1695 // mov %r1, param
1696
1697 // @TODO: Revisit this.  There is a type
1698 // contradiction between iPTRAny and iPTR for the addr defs, so the move_sym
1699 // instructions are not currently defined. However, we can use the ptr
1700 // variants and the asm printer will do the right thing.
1701 def : Pat<(i64 (int_nvvm_ptr_gen_to_local (int_nvvm_ptr_local_to_gen
1702                 (MoveParam texternalsym:$src)))),
1703                (nvvm_move_ptr64  texternalsym:$src)>;
1704 def : Pat<(i32 (int_nvvm_ptr_gen_to_local (int_nvvm_ptr_local_to_gen
1705                 (MoveParam texternalsym:$src)))),
1706                (nvvm_move_ptr32  texternalsym:$src)>;
1707
1708 def texsurf_handles
1709   : NVPTXInst<(outs Int64Regs:$result), (ins imem:$src),
1710               "mov.u64 \t$result, $src;", []>;
1711
1712 //-----------------------------------
1713 // Compiler Error Warn
1714 // - Just ignore them in codegen
1715 //-----------------------------------
1716
1717 def INT_NVVM_COMPILER_WARN_32 : NVPTXInst<(outs), (ins Int32Regs:$a),
1718                 "// llvm.nvvm.compiler.warn()",
1719                 [(int_nvvm_compiler_warn Int32Regs:$a)]>;
1720 def INT_NVVM_COMPILER_WARN_64 : NVPTXInst<(outs), (ins Int64Regs:$a),
1721                 "// llvm.nvvm.compiler.warn()",
1722                 [(int_nvvm_compiler_warn Int64Regs:$a)]>;
1723 def INT_NVVM_COMPILER_ERROR_32 : NVPTXInst<(outs), (ins Int32Regs:$a),
1724                 "// llvm.nvvm.compiler.error()",
1725                 [(int_nvvm_compiler_error Int32Regs:$a)]>;
1726 def INT_NVVM_COMPILER_ERROR_64 : NVPTXInst<(outs), (ins Int64Regs:$a),
1727                 "// llvm.nvvm.compiler.error()",
1728                 [(int_nvvm_compiler_error Int64Regs:$a)]>;
1729
1730
1731 // isspacep
1732
1733 def ISSPACEP_CONST_32
1734   : NVPTXInst<(outs Int1Regs:$d), (ins Int32Regs:$a),
1735               "isspacep.const \t$d, $a;",
1736               [(set Int1Regs:$d, (int_nvvm_isspacep_const Int32Regs:$a))]>,
1737     Requires<[hasPTX31]>;
1738 def ISSPACEP_CONST_64
1739   : NVPTXInst<(outs Int1Regs:$d), (ins Int64Regs:$a),
1740               "isspacep.const \t$d, $a;",
1741               [(set Int1Regs:$d, (int_nvvm_isspacep_const Int64Regs:$a))]>,
1742     Requires<[hasPTX31]>;
1743 def ISSPACEP_GLOBAL_32
1744   : NVPTXInst<(outs Int1Regs:$d), (ins Int32Regs:$a),
1745               "isspacep.global \t$d, $a;",
1746               [(set Int1Regs:$d, (int_nvvm_isspacep_global Int32Regs:$a))]>;
1747 def ISSPACEP_GLOBAL_64
1748   : NVPTXInst<(outs Int1Regs:$d), (ins Int64Regs:$a),
1749               "isspacep.global \t$d, $a;",
1750               [(set Int1Regs:$d, (int_nvvm_isspacep_global Int64Regs:$a))]>;
1751 def ISSPACEP_LOCAL_32
1752   : NVPTXInst<(outs Int1Regs:$d), (ins Int32Regs:$a),
1753               "isspacep.local \t$d, $a;",
1754               [(set Int1Regs:$d, (int_nvvm_isspacep_local Int32Regs:$a))]>;
1755 def ISSPACEP_LOCAL_64
1756   : NVPTXInst<(outs Int1Regs:$d), (ins Int64Regs:$a),
1757               "isspacep.local \t$d, $a;",
1758               [(set Int1Regs:$d, (int_nvvm_isspacep_local Int64Regs:$a))]>;
1759 def ISSPACEP_SHARED_32
1760   : NVPTXInst<(outs Int1Regs:$d), (ins Int32Regs:$a),
1761               "isspacep.shared \t$d, $a;",
1762               [(set Int1Regs:$d, (int_nvvm_isspacep_shared Int32Regs:$a))]>;
1763 def ISSPACEP_SHARED_64
1764   : NVPTXInst<(outs Int1Regs:$d), (ins Int64Regs:$a),
1765               "isspacep.shared \t$d, $a;",
1766               [(set Int1Regs:$d, (int_nvvm_isspacep_shared Int64Regs:$a))]>;
1767
1768
1769 // Special register reads
1770 def MOV_SPECIAL : NVPTXInst<(outs Int32Regs:$d),
1771                             (ins SpecialRegs:$r),
1772                             "mov.b32\t$d, $r;", []>;
1773
1774 def : Pat<(int_nvvm_read_ptx_sreg_envreg0), (MOV_SPECIAL ENVREG0)>;
1775 def : Pat<(int_nvvm_read_ptx_sreg_envreg1), (MOV_SPECIAL ENVREG1)>;
1776 def : Pat<(int_nvvm_read_ptx_sreg_envreg2), (MOV_SPECIAL ENVREG2)>;
1777 def : Pat<(int_nvvm_read_ptx_sreg_envreg3), (MOV_SPECIAL ENVREG3)>;
1778 def : Pat<(int_nvvm_read_ptx_sreg_envreg4), (MOV_SPECIAL ENVREG4)>;
1779 def : Pat<(int_nvvm_read_ptx_sreg_envreg5), (MOV_SPECIAL ENVREG5)>;
1780 def : Pat<(int_nvvm_read_ptx_sreg_envreg6), (MOV_SPECIAL ENVREG6)>;
1781 def : Pat<(int_nvvm_read_ptx_sreg_envreg7), (MOV_SPECIAL ENVREG7)>;
1782 def : Pat<(int_nvvm_read_ptx_sreg_envreg8), (MOV_SPECIAL ENVREG8)>;
1783 def : Pat<(int_nvvm_read_ptx_sreg_envreg9), (MOV_SPECIAL ENVREG9)>;
1784 def : Pat<(int_nvvm_read_ptx_sreg_envreg10), (MOV_SPECIAL ENVREG10)>;
1785 def : Pat<(int_nvvm_read_ptx_sreg_envreg11), (MOV_SPECIAL ENVREG11)>;
1786 def : Pat<(int_nvvm_read_ptx_sreg_envreg12), (MOV_SPECIAL ENVREG12)>;
1787 def : Pat<(int_nvvm_read_ptx_sreg_envreg13), (MOV_SPECIAL ENVREG13)>;
1788 def : Pat<(int_nvvm_read_ptx_sreg_envreg14), (MOV_SPECIAL ENVREG14)>;
1789 def : Pat<(int_nvvm_read_ptx_sreg_envreg15), (MOV_SPECIAL ENVREG15)>;
1790 def : Pat<(int_nvvm_read_ptx_sreg_envreg16), (MOV_SPECIAL ENVREG16)>;
1791 def : Pat<(int_nvvm_read_ptx_sreg_envreg17), (MOV_SPECIAL ENVREG17)>;
1792 def : Pat<(int_nvvm_read_ptx_sreg_envreg18), (MOV_SPECIAL ENVREG18)>;
1793 def : Pat<(int_nvvm_read_ptx_sreg_envreg19), (MOV_SPECIAL ENVREG19)>;
1794 def : Pat<(int_nvvm_read_ptx_sreg_envreg20), (MOV_SPECIAL ENVREG20)>;
1795 def : Pat<(int_nvvm_read_ptx_sreg_envreg21), (MOV_SPECIAL ENVREG21)>;
1796 def : Pat<(int_nvvm_read_ptx_sreg_envreg22), (MOV_SPECIAL ENVREG22)>;
1797 def : Pat<(int_nvvm_read_ptx_sreg_envreg23), (MOV_SPECIAL ENVREG23)>;
1798 def : Pat<(int_nvvm_read_ptx_sreg_envreg24), (MOV_SPECIAL ENVREG24)>;
1799 def : Pat<(int_nvvm_read_ptx_sreg_envreg25), (MOV_SPECIAL ENVREG25)>;
1800 def : Pat<(int_nvvm_read_ptx_sreg_envreg26), (MOV_SPECIAL ENVREG26)>;
1801 def : Pat<(int_nvvm_read_ptx_sreg_envreg27), (MOV_SPECIAL ENVREG27)>;
1802 def : Pat<(int_nvvm_read_ptx_sreg_envreg28), (MOV_SPECIAL ENVREG28)>;
1803 def : Pat<(int_nvvm_read_ptx_sreg_envreg29), (MOV_SPECIAL ENVREG29)>;
1804 def : Pat<(int_nvvm_read_ptx_sreg_envreg30), (MOV_SPECIAL ENVREG30)>;
1805 def : Pat<(int_nvvm_read_ptx_sreg_envreg31), (MOV_SPECIAL ENVREG31)>;
1806
1807
1808 // rotate builtin support
1809
1810 def ROTATE_B32_HW_IMM
1811   : NVPTXInst<(outs Int32Regs:$dst),
1812               (ins  Int32Regs:$src, i32imm:$amt),
1813               "shf.l.wrap.b32 \t$dst, $src, $src, $amt;",
1814               [(set Int32Regs:$dst,
1815                  (int_nvvm_rotate_b32 Int32Regs:$src, (i32 imm:$amt)))]>,
1816               Requires<[hasHWROT32]> ;
1817
1818 def ROTATE_B32_HW_REG
1819   : NVPTXInst<(outs Int32Regs:$dst),
1820               (ins  Int32Regs:$src, Int32Regs:$amt),
1821               "shf.l.wrap.b32 \t$dst, $src, $src, $amt;",
1822               [(set Int32Regs:$dst,
1823                  (int_nvvm_rotate_b32 Int32Regs:$src, Int32Regs:$amt))]>,
1824               Requires<[hasHWROT32]> ;
1825
1826 def : Pat<(int_nvvm_rotate_b32 Int32Regs:$src, (i32 imm:$amt)),
1827           (ROT32imm_sw Int32Regs:$src, imm:$amt, (SUB_FRM_32 node:$amt))>,
1828       Requires<[noHWROT32]> ;
1829
1830 def : Pat<(int_nvvm_rotate_b32 Int32Regs:$src, Int32Regs:$amt),
1831           (ROTL32reg_sw Int32Regs:$src, Int32Regs:$amt)>,
1832       Requires<[noHWROT32]> ;
1833
1834 def GET_LO_INT64
1835   : NVPTXInst<(outs Int32Regs:$dst), (ins Int64Regs:$src),
1836               !strconcat("{{\n\t",
1837               !strconcat(".reg .b32 %dummy;\n\t",
1838               !strconcat("mov.b64 \t{$dst,%dummy}, $src;\n\t",
1839         !strconcat("}}", "")))),
1840         []> ;
1841
1842 def GET_HI_INT64
1843   : NVPTXInst<(outs Int32Regs:$dst), (ins Int64Regs:$src),
1844               !strconcat("{{\n\t",
1845               !strconcat(".reg .b32 %dummy;\n\t",
1846               !strconcat("mov.b64 \t{%dummy,$dst}, $src;\n\t",
1847         !strconcat("}}", "")))),
1848         []> ;
1849
1850 def PACK_TWO_INT32
1851   : NVPTXInst<(outs Int64Regs:$dst), (ins Int32Regs:$lo, Int32Regs:$hi),
1852               "mov.b64 \t$dst, {{$lo, $hi}};", []> ;
1853
1854 def : Pat<(int_nvvm_swap_lo_hi_b64 Int64Regs:$src),
1855           (PACK_TWO_INT32 (GET_HI_INT64 Int64Regs:$src),
1856                           (GET_LO_INT64 Int64Regs:$src))> ;
1857
1858 // funnel shift, requires >= sm_32
1859 def SHF_L_WRAP_B32_IMM
1860   : NVPTXInst<(outs Int32Regs:$dst),
1861               (ins  Int32Regs:$lo, Int32Regs:$hi, i32imm:$amt),
1862               "shf.l.wrap.b32 \t$dst, $lo, $hi, $amt;",[]>,
1863     Requires<[hasHWROT32]>;
1864
1865 def SHF_L_WRAP_B32_REG
1866   : NVPTXInst<(outs Int32Regs:$dst),
1867               (ins  Int32Regs:$lo, Int32Regs:$hi, Int32Regs:$amt),
1868               "shf.l.wrap.b32 \t$dst, $lo, $hi, $amt;",[]>,
1869     Requires<[hasHWROT32]>;
1870
1871 def SHF_R_WRAP_B32_IMM
1872   : NVPTXInst<(outs Int32Regs:$dst),
1873               (ins  Int32Regs:$lo, Int32Regs:$hi, i32imm:$amt),
1874               "shf.r.wrap.b32 \t$dst, $lo, $hi, $amt;",[]>,
1875     Requires<[hasHWROT32]>;
1876
1877 def SHF_R_WRAP_B32_REG
1878   : NVPTXInst<(outs Int32Regs:$dst),
1879               (ins  Int32Regs:$lo, Int32Regs:$hi, Int32Regs:$amt),
1880               "shf.r.wrap.b32 \t$dst, $lo, $hi, $amt;",[]>,
1881     Requires<[hasHWROT32]>;
1882
1883 // HW version of rotate 64
1884 def : Pat<(int_nvvm_rotate_b64 Int64Regs:$src, (i32 imm:$amt)),
1885           (PACK_TWO_INT32
1886             (SHF_L_WRAP_B32_IMM (GET_HI_INT64 Int64Regs:$src),
1887                                 (GET_LO_INT64 Int64Regs:$src), imm:$amt),
1888             (SHF_L_WRAP_B32_IMM (GET_LO_INT64 Int64Regs:$src),
1889                                 (GET_HI_INT64 Int64Regs:$src), imm:$amt))>,
1890       Requires<[hasHWROT32]>;
1891
1892 def : Pat<(int_nvvm_rotate_b64 Int64Regs:$src, Int32Regs:$amt),
1893           (PACK_TWO_INT32
1894             (SHF_L_WRAP_B32_REG (GET_HI_INT64 Int64Regs:$src),
1895                                 (GET_LO_INT64 Int64Regs:$src), Int32Regs:$amt),
1896             (SHF_L_WRAP_B32_REG (GET_LO_INT64 Int64Regs:$src),
1897                                (GET_HI_INT64 Int64Regs:$src), Int32Regs:$amt))>,
1898       Requires<[hasHWROT32]>;
1899
1900
1901 def : Pat<(int_nvvm_rotate_right_b64 Int64Regs:$src, (i32 imm:$amt)),
1902           (PACK_TWO_INT32
1903             (SHF_R_WRAP_B32_IMM (GET_LO_INT64 Int64Regs:$src),
1904                                 (GET_HI_INT64 Int64Regs:$src), imm:$amt),
1905             (SHF_R_WRAP_B32_IMM (GET_HI_INT64 Int64Regs:$src),
1906                                 (GET_LO_INT64 Int64Regs:$src), imm:$amt))>,
1907       Requires<[hasHWROT32]>;
1908
1909 def : Pat<(int_nvvm_rotate_right_b64 Int64Regs:$src, Int32Regs:$amt),
1910           (PACK_TWO_INT32
1911             (SHF_R_WRAP_B32_REG (GET_LO_INT64 Int64Regs:$src),
1912                                 (GET_HI_INT64 Int64Regs:$src), Int32Regs:$amt),
1913             (SHF_R_WRAP_B32_REG (GET_HI_INT64 Int64Regs:$src),
1914                                (GET_LO_INT64 Int64Regs:$src), Int32Regs:$amt))>,
1915       Requires<[hasHWROT32]>;
1916
1917 // SW version of rotate 64
1918 def : Pat<(int_nvvm_rotate_b64 Int64Regs:$src, (i32 imm:$amt)),
1919           (ROT64imm_sw Int64Regs:$src, imm:$amt, (SUB_FRM_32 node:$amt))>,
1920       Requires<[noHWROT32]>;
1921 def : Pat<(int_nvvm_rotate_b64 Int64Regs:$src, Int32Regs:$amt),
1922           (ROTL64reg_sw Int64Regs:$src, Int32Regs:$amt)>,
1923       Requires<[noHWROT32]>;
1924 def : Pat<(int_nvvm_rotate_right_b64 Int64Regs:$src, (i32 imm:$amt)),
1925           (ROT64imm_sw Int64Regs:$src, (SUB_FRM_64 node:$amt), imm:$amt)>,
1926       Requires<[noHWROT32]>;
1927 def : Pat<(int_nvvm_rotate_right_b64 Int64Regs:$src, Int32Regs:$amt),
1928           (ROTR64reg_sw Int64Regs:$src, Int32Regs:$amt)>,
1929       Requires<[noHWROT32]>;
1930
1931
1932 //-----------------------------------
1933 // Texture Intrinsics
1934 //-----------------------------------
1935
1936 // NOTE: For Fermi support, any new texture/surface/sampler intrinsics must be
1937 // also defined in NVPTXReplaceImageHandles.cpp
1938
1939 // texmode_independent
1940 let IsTex = 1, IsTexModeUnified = 0 in {
1941 // Texture fetch instructions using handles
1942 def TEX_1D_F32_S32
1943   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
1944                     Float32Regs:$b, Float32Regs:$a),
1945               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$x),
1946               "tex.1d.v4.f32.s32\t\\{$r, $g, $b, $a\\}, [$t, $s, \\{$x\\}];",
1947               []>;
1948 def TEX_1D_F32_F32
1949   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
1950                     Float32Regs:$b, Float32Regs:$a),
1951               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x),
1952               "tex.1d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, [$t, $s, \\{$x\\}];",
1953               []>;
1954 def TEX_1D_F32_F32_LEVEL
1955   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
1956                     Float32Regs:$b, Float32Regs:$a),
1957               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$lod),
1958               "tex.level.1d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
1959               "[$t, $s, \\{$x\\}], $lod;",
1960               []>;
1961 def TEX_1D_F32_F32_GRAD
1962   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
1963                     Float32Regs:$b, Float32Regs:$a),
1964               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x,
1965                    Float32Regs:$gradx, Float32Regs:$grady),
1966               "tex.grad.1d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
1967               "[$t, $s, \\{$x\\}], \\{$gradx\\}, \\{$grady\\};",
1968               []>;
1969 def TEX_1D_S32_S32
1970   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
1971                     Int32Regs:$b, Int32Regs:$a),
1972               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$x),
1973               "tex.1d.v4.s32.s32\t\\{$r, $g, $b, $a\\}, [$t, $s, \\{$x\\}];",
1974               []>;
1975 def TEX_1D_S32_F32
1976   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
1977                     Int32Regs:$b, Int32Regs:$a),
1978               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x),
1979               "tex.1d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, [$t, $s, \\{$x\\}];",
1980               []>;
1981 def TEX_1D_S32_F32_LEVEL
1982   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
1983                     Int32Regs:$b, Int32Regs:$a),
1984               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x,
1985                    Float32Regs:$lod),
1986               "tex.level.1d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
1987               "[$t, $s, \\{$x\\}], $lod;",
1988               []>;
1989 def TEX_1D_S32_F32_GRAD
1990   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
1991                     Int32Regs:$b, Int32Regs:$a),
1992               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x,
1993                    Float32Regs:$gradx, Float32Regs:$grady),
1994               "tex.grad.1d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
1995               "[$t, $s, \\{$x\\}], \\{$gradx\\}, \\{$grady\\};",
1996               []>;
1997 def TEX_1D_U32_S32
1998   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
1999                     Int32Regs:$b, Int32Regs:$a),
2000               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$x),
2001               "tex.1d.v4.u32.s32\t\\{$r, $g, $b, $a\\}, [$t, $s, \\{$x\\}];",
2002               []>;
2003 def TEX_1D_U32_F32
2004   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2005                     Int32Regs:$b, Int32Regs:$a),
2006               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x),
2007               "tex.1d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, [$t, $s, \\{$x\\}];",
2008               []>;
2009 def TEX_1D_U32_F32_LEVEL
2010   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2011                     Int32Regs:$b, Int32Regs:$a),
2012               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x,
2013                    Float32Regs:$lod),
2014               "tex.level.1d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2015               "[$t, $s, \\{$x\\}], $lod;",
2016               []>;
2017 def TEX_1D_U32_F32_GRAD
2018   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2019                     Int32Regs:$b, Int32Regs:$a),
2020               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x,
2021                    Float32Regs:$gradx, Float32Regs:$grady),
2022               "tex.grad.1d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2023               "[$t, $s, \\{$x\\}], \\{$gradx\\}, \\{$grady\\};",
2024               []>;
2025
2026 def TEX_1D_ARRAY_F32_S32
2027   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2028                     Float32Regs:$b, Float32Regs:$a),
2029               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
2030               "tex.a1d.v4.f32.s32\t\\{$r, $g, $b, $a\\}, "
2031               "[$t, $s, \\{$l, $x\\}];",
2032               []>;
2033 def TEX_1D_ARRAY_F32_F32
2034   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2035                     Float32Regs:$b, Float32Regs:$a),
2036               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x),
2037               "tex.a1d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2038               "[$t, $s, \\{$l, $x\\}];",
2039               []>;
2040 def TEX_1D_ARRAY_F32_F32_LEVEL
2041   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2042                     Float32Regs:$b, Float32Regs:$a),
2043               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2044                    Float32Regs:$lod),
2045               "tex.level.a1d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2046               "[$t, $s, \\{$l, $x\\}], $lod;",
2047               []>;
2048 def TEX_1D_ARRAY_F32_F32_GRAD
2049   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2050                     Float32Regs:$b, Float32Regs:$a),
2051               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2052                    Float32Regs:$gradx, Float32Regs:$grady),
2053               "tex.grad.a1d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2054               "[$t, $s, \\{$l, $x\\}], \\{$gradx\\}, \\{$grady\\};",
2055               []>;
2056 def TEX_1D_ARRAY_S32_S32
2057   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2058                     Int32Regs:$b, Int32Regs:$a),
2059               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
2060               "tex.a1d.v4.s32.s32\t\\{$r, $g, $b, $a\\}, "
2061               "[$t, $s, \\{$l, $x\\}];",
2062               []>;
2063 def TEX_1D_ARRAY_S32_F32
2064   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2065                     Int32Regs:$b, Int32Regs:$a),
2066               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x),
2067               "tex.a1d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2068               "[$t, $s, \\{$l, $x\\}];",
2069               []>;
2070 def TEX_1D_ARRAY_S32_F32_LEVEL
2071   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2072                     Int32Regs:$b, Int32Regs:$a),
2073               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2074                    Float32Regs:$lod),
2075               "tex.level.a1d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2076               "[$t, $s, \\{$l, $x\\}], $lod;",
2077               []>;
2078 def TEX_1D_ARRAY_S32_F32_GRAD
2079   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2080                     Int32Regs:$b, Int32Regs:$a),
2081               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2082                    Float32Regs:$gradx, Float32Regs:$grady),
2083               "tex.grad.a1d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2084               "[$t, $s, \\{$l, $x\\}], \\{$gradx\\}, \\{$grady\\};",
2085               []>;
2086 def TEX_1D_ARRAY_U32_S32
2087   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2088                     Int32Regs:$b, Int32Regs:$a),
2089               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
2090               "tex.a1d.v4.u32.s32\t\\{$r, $g, $b, $a\\}, "
2091               "[$t, $s, \\{$l, $x\\}];",
2092               []>;
2093 def TEX_1D_ARRAY_U32_F32
2094   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2095                     Int32Regs:$b, Int32Regs:$a),
2096               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x),
2097               "tex.a1d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2098               "[$t, $s, \\{$l, $x\\}];",
2099               []>;
2100 def TEX_1D_ARRAY_U32_F32_LEVEL
2101   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2102                     Int32Regs:$b, Int32Regs:$a),
2103               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2104                    Float32Regs:$lod),
2105               "tex.level.a1d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2106               "[$t, $s, \\{$l, $x\\}], $lod;",
2107               []>;
2108 def TEX_1D_ARRAY_U32_F32_GRAD
2109   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2110                     Int32Regs:$b, Int32Regs:$a),
2111               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2112                    Float32Regs:$gradx, Float32Regs:$grady),
2113               "tex.grad.a1d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2114               "[$t, $s, \\{$l, $x\\}], \\{$gradx\\}, \\{$grady\\};",
2115               []>;
2116
2117 def TEX_2D_F32_S32
2118   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2119                     Float32Regs:$b, Float32Regs:$a),
2120               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
2121               "tex.2d.v4.f32.s32\t\\{$r, $g, $b, $a\\}, "
2122               "[$t, $s, \\{$x, $y\\}];",
2123               []>;
2124 def TEX_2D_F32_F32
2125   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2126                     Float32Regs:$b, Float32Regs:$a),
2127               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2128               "tex.2d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2129               "[$t, $s, \\{$x, $y\\}];",
2130               []>;
2131 def TEX_2D_F32_F32_LEVEL
2132   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2133                     Float32Regs:$b, Float32Regs:$a),
2134               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2135                    Float32Regs:$lod),
2136               "tex.level.2d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2137               "[$t, $s, \\{$x, $y\\}], $lod;",
2138               []>;
2139 def TEX_2D_F32_F32_GRAD
2140   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2141                     Float32Regs:$b, Float32Regs:$a),
2142               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2143                    Float32Regs:$gradx0, Float32Regs:$gradx1,
2144                    Float32Regs:$grady0, Float32Regs:$grady1),
2145               "tex.grad.2d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2146               "[$t, $s, \\{$x, $y\\}], \\{$gradx0, $gradx1\\}, "
2147               "\\{$grady0, $grady1\\};",
2148               []>;
2149 def TEX_2D_S32_S32
2150   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2151                     Int32Regs:$b, Int32Regs:$a),
2152               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
2153               "tex.2d.v4.s32.s32\t\\{$r, $g, $b, $a\\}, "
2154               "[$t, $s, \\{$x, $y\\}];",
2155               []>;
2156 def TEX_2D_S32_F32
2157   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2158                     Int32Regs:$b, Int32Regs:$a),
2159               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2160               "tex.2d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2161               "[$t, $s, \\{$x, $y\\}];",
2162               []>;
2163 def TEX_2D_S32_F32_LEVEL
2164   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2165                     Int32Regs:$b, Int32Regs:$a),
2166               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2167                    Float32Regs:$lod),
2168               "tex.level.2d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2169               "[$t, $s, \\{$x, $y\\}], $lod;",
2170               []>;
2171 def TEX_2D_S32_F32_GRAD
2172   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2173                     Int32Regs:$b, Int32Regs:$a),
2174               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2175                    Float32Regs:$gradx0, Float32Regs:$gradx1,
2176                    Float32Regs:$grady0, Float32Regs:$grady1),
2177               "tex.grad.2d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2178               "[$t, $s, \\{$x, $y\\}], \\{$gradx0, $gradx1\\}, "
2179               "\\{$grady0, $grady1\\};",
2180               []>;
2181 def TEX_2D_U32_S32
2182   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2183                     Int32Regs:$b, Int32Regs:$a),
2184               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
2185               "tex.2d.v4.u32.s32\t\\{$r, $g, $b, $a\\}, "
2186               "[$t, $s, \\{$x, $y\\}];",
2187               []>;
2188 def TEX_2D_U32_F32
2189   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2190                     Int32Regs:$b, Int32Regs:$a),
2191               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2192               "tex.2d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2193               "[$t, $s, \\{$x, $y\\}];",
2194               []>;
2195 def TEX_2D_U32_F32_LEVEL
2196   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2197                     Int32Regs:$b, Int32Regs:$a),
2198               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2199                    Float32Regs:$lod),
2200               "tex.level.2d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2201               "[$t, $s, \\{$x, $y\\}], $lod;",
2202               []>;
2203 def TEX_2D_U32_F32_GRAD
2204   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2205                     Int32Regs:$b, Int32Regs:$a),
2206               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2207                    Float32Regs:$gradx0, Float32Regs:$gradx1,
2208                    Float32Regs:$grady0, Float32Regs:$grady1),
2209               "tex.grad.2d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2210               "[$t, $s, \\{$x, $y\\}], \\{$gradx0, $gradx1\\}, "
2211               "\\{$grady0, $grady1\\};",
2212               []>;
2213
2214 def TEX_2D_ARRAY_F32_S32
2215   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2216                     Float32Regs:$b, Float32Regs:$a),
2217               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
2218                    Int32Regs:$y),
2219               "tex.a2d.v4.f32.s32\t\\{$r, $g, $b, $a\\}, "
2220               "[$t, $s, \\{$l, $x, $y, $y\\}];",
2221               []>;
2222 def TEX_2D_ARRAY_F32_F32
2223   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2224                     Float32Regs:$b, Float32Regs:$a),
2225               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2226                    Float32Regs:$y),
2227               "tex.a2d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2228               "[$t, $s, \\{$l, $x, $y, $y\\}];",
2229               []>;
2230 def TEX_2D_ARRAY_F32_F32_LEVEL
2231   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2232                     Float32Regs:$b, Float32Regs:$a),
2233               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2234                    Float32Regs:$y, Float32Regs:$lod),
2235               "tex.level.a2d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2236               "[$t, $s, \\{$l, $x, $y, $y\\}], $lod;",
2237               []>;
2238 def TEX_2D_ARRAY_F32_F32_GRAD
2239   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2240                     Float32Regs:$b, Float32Regs:$a),
2241               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2242                    Float32Regs:$y, Float32Regs:$gradx0, Float32Regs:$gradx1,
2243                    Float32Regs:$grady0, Float32Regs:$grady1),
2244               "tex.grad.a2d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2245               "[$t, $s, \\{$l, $x, $y, $y\\}], \\{$gradx0, $gradx1\\}, "
2246               "\\{$grady0, $grady1\\};",
2247               []>;
2248 def TEX_2D_ARRAY_S32_S32
2249   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2250                     Int32Regs:$b, Int32Regs:$a),
2251               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
2252                    Int32Regs:$y),
2253               "tex.a2d.v4.s32.s32\t\\{$r, $g, $b, $a\\}, "
2254               "[$t, $s, \\{$l, $x, $y, $y\\}];",
2255               []>;
2256 def TEX_2D_ARRAY_S32_F32
2257   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2258                     Int32Regs:$b, Int32Regs:$a),
2259               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2260                    Float32Regs:$y),
2261               "tex.a2d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2262               "[$t, $s, \\{$l, $x, $y, $y\\}];",
2263               []>;
2264 def TEX_2D_ARRAY_S32_F32_LEVEL
2265   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2266                     Int32Regs:$b, Int32Regs:$a),
2267               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2268                    Float32Regs:$y, Float32Regs:$lod),
2269               "tex.level.a2d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2270               "[$t, $s, \\{$l, $x, $y, $y\\}], $lod;",
2271               []>;
2272 def TEX_2D_ARRAY_S32_F32_GRAD
2273   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2274                     Int32Regs:$b, Int32Regs:$a),
2275               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2276                    Float32Regs:$y,
2277                    Float32Regs:$gradx0, Float32Regs:$gradx1,
2278                    Float32Regs:$grady0, Float32Regs:$grady1),
2279               "tex.grad.a2d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2280               "[$t, $s, \\{$l, $x, $y, $y\\}], \\{$gradx0, $gradx1\\}, "
2281               "\\{$grady0, $grady1\\};",
2282               []>;
2283 def TEX_2D_ARRAY_U32_S32
2284   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2285                     Int32Regs:$b, Int32Regs:$a),
2286               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
2287                    Int32Regs:$y),
2288               "tex.a2d.v4.u32.s32\t\\{$r, $g, $b, $a\\}, "
2289               "[$t, $s, \\{$l, $x, $y, $y\\}];",
2290               []>;
2291 def TEX_2D_ARRAY_U32_F32
2292   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2293                     Int32Regs:$b, Int32Regs:$a),
2294               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2295                    Float32Regs:$y),
2296               "tex.a2d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2297               "[$t, $s, \\{$l, $x, $y, $y\\}];",
2298               []>;
2299 def TEX_2D_ARRAY_U32_F32_LEVEL
2300   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2301                     Int32Regs:$b, Int32Regs:$a),
2302               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2303                    Float32Regs:$y, Float32Regs:$lod),
2304               "tex.level.a2d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2305               "[$t, $s, \\{$l, $x, $y, $y\\}], $lod;",
2306               []>;
2307 def TEX_2D_ARRAY_U32_F32_GRAD
2308   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2309                     Int32Regs:$b, Int32Regs:$a),
2310               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2311                    Float32Regs:$y,
2312                    Float32Regs:$gradx0, Float32Regs:$gradx1,
2313                    Float32Regs:$grady0, Float32Regs:$grady1),
2314               "tex.grad.a2d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2315               "[$t, $s, \\{$l, $x, $y, $y\\}], \\{$gradx0, $gradx1\\}, "
2316               "\\{$grady0, $grady1\\};",
2317               []>;
2318
2319 def TEX_3D_F32_S32
2320   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2321                     Float32Regs:$b, Float32Regs:$a),
2322               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
2323                    Int32Regs:$z),
2324               "tex.3d.v4.f32.s32\t\\{$r, $g, $b, $a\\}, "
2325               "[$t, $s, \\{$x, $y, $z, $z\\}];",
2326               []>;
2327 def TEX_3D_F32_F32
2328   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2329                     Float32Regs:$b, Float32Regs:$a),
2330               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2331                    Float32Regs:$z),
2332               "tex.3d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2333               "[$t, $s, \\{$x, $y, $z, $z\\}];",
2334               []>;
2335 def TEX_3D_F32_F32_LEVEL
2336   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2337                     Float32Regs:$b, Float32Regs:$a),
2338               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2339                    Float32Regs:$z, Float32Regs:$lod),
2340               "tex.level.3d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2341               "[$t, $s, \\{$x, $y, $z, $z\\}], $lod;",
2342               []>;
2343 def TEX_3D_F32_F32_GRAD
2344   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2345                     Float32Regs:$b, Float32Regs:$a),
2346               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2347                    Float32Regs:$z,
2348                    Float32Regs:$gradx0, Float32Regs:$gradx1,
2349                    Float32Regs:$gradx2, Float32Regs:$grady0,
2350                    Float32Regs:$grady1, Float32Regs:$grady2),
2351               "tex.grad.3d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2352               "[$t, $s, \\{$x, $y, $z, $z\\}], "
2353               "\\{$gradx0, $gradx1, $gradx2, $gradx2\\}, "
2354               "\\{$grady0, $grady1, $grady2, $grady2\\};",
2355               []>;
2356 def TEX_3D_S32_S32
2357   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2358                     Int32Regs:$b, Int32Regs:$a),
2359               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
2360                    Int32Regs:$z),
2361               "tex.3d.v4.s32.s32\t\\{$r, $g, $b, $a\\}, "
2362               "[$t, $s, \\{$x, $y, $z, $z\\}];",
2363               []>;
2364 def TEX_3D_S32_F32
2365   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2366                     Int32Regs:$b, Int32Regs:$a),
2367               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2368                    Float32Regs:$z),
2369               "tex.3d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2370               "[$t, $s, \\{$x, $y, $z, $z\\}];",
2371               []>;
2372 def TEX_3D_S32_F32_LEVEL
2373   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2374                     Int32Regs:$b, Int32Regs:$a),
2375               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2376                    Float32Regs:$z, Float32Regs:$lod),
2377               "tex.level.3d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2378               "[$t, $s, \\{$x, $y, $z, $z\\}], $lod;",
2379               []>;
2380 def TEX_3D_S32_F32_GRAD
2381   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2382                     Int32Regs:$b, Int32Regs:$a),
2383               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2384                    Float32Regs:$z,
2385                    Float32Regs:$gradx0, Float32Regs:$gradx1,
2386                    Float32Regs:$gradx2, Float32Regs:$grady0,
2387                    Float32Regs:$grady1, Float32Regs:$grady2),
2388               "tex.grad.3d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2389               "[$t, $s, \\{$x, $y, $z, $z\\}], "
2390               "\\{$gradx0, $gradx1, $gradx2, $gradx2\\}, "
2391               "\\{$grady0, $grady1, $grady2, $grady2\\};",
2392               []>;
2393 def TEX_3D_U32_S32
2394   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2395                     Int32Regs:$b, Int32Regs:$a),
2396               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
2397                    Int32Regs:$z),
2398               "tex.3d.v4.u32.s32\t\\{$r, $g, $b, $a\\}, "
2399               "[$t, $s, \\{$x, $y, $z, $z\\}];",
2400               []>;
2401 def TEX_3D_U32_F32
2402   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2403                     Int32Regs:$b, Int32Regs:$a),
2404               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2405                    Float32Regs:$z),
2406               "tex.3d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2407               "[$t, $s, \\{$x, $y, $z, $z\\}];",
2408               []>;
2409 def TEX_3D_U32_F32_LEVEL
2410   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2411                     Int32Regs:$b, Int32Regs:$a),
2412               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2413                    Float32Regs:$z, Float32Regs:$lod),
2414               "tex.level.3d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2415               "[$t, $s, \\{$x, $y, $z, $z\\}], $lod;",
2416               []>;
2417 def TEX_3D_U32_F32_GRAD
2418   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2419                     Int32Regs:$b, Int32Regs:$a),
2420               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2421                    Float32Regs:$z,
2422                    Float32Regs:$gradx0, Float32Regs:$gradx1,
2423                    Float32Regs:$gradx2, Float32Regs:$grady0,
2424                    Float32Regs:$grady1, Float32Regs:$grady2),
2425               "tex.grad.3d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2426               "[$t, $s, \\{$x, $y, $z, $z\\}], "
2427               "\\{$gradx0, $gradx1, $gradx2, $gradx2\\}, "
2428               "\\{$grady0, $grady1, $grady2, $grady2\\};",
2429               []>;
2430
2431 def TEX_CUBE_F32_F32
2432   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2433                     Float32Regs:$b, Float32Regs:$a),
2434               (ins Int64Regs:$t, Int64Regs:$s,
2435                Float32Regs:$x, Float32Regs:$y, Float32Regs:$z),
2436               "tex.cube.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2437               "[$t, $s, \\{$x, $y, $z, $z\\}];",
2438               []>;
2439 def TEX_CUBE_F32_F32_LEVEL
2440   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2441                     Float32Regs:$b, Float32Regs:$a),
2442               (ins Int64Regs:$t, Int64Regs:$s,
2443                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z,
2444                    Float32Regs:$lod),
2445               "tex.level.cube.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2446               "[$t, $s, \\{$x, $y, $z, $z\\}], $lod;",
2447               []>;
2448 def TEX_CUBE_S32_F32
2449   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2450                     Int32Regs:$b, Int32Regs:$a),
2451               (ins Int64Regs:$t, Int64Regs:$s,
2452                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z),
2453               "tex.cube.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2454               "[$t, $s, \\{$x, $y, $z, $z\\}];",
2455               []>;
2456 def TEX_CUBE_S32_F32_LEVEL
2457   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2458                     Int32Regs:$b, Int32Regs:$a),
2459               (ins Int64Regs:$t, Int64Regs:$s,
2460                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z,
2461                    Float32Regs:$lod),
2462               "tex.level.cube.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2463               "[$t, $s, \\{$x, $y, $z, $z\\}], $lod;",
2464               []>;
2465 def TEX_CUBE_U32_F32
2466   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2467                     Int32Regs:$b, Int32Regs:$a),
2468               (ins Int64Regs:$t, Int64Regs:$s,
2469                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z),
2470               "tex.cube.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2471               "[$t, $s, \\{$x, $y, $z, $z\\}];",
2472               []>;
2473 def TEX_CUBE_U32_F32_LEVEL
2474   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2475                     Int32Regs:$b, Int32Regs:$a),
2476               (ins Int64Regs:$t, Int64Regs:$s,
2477                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z,
2478                    Float32Regs:$lod),
2479               "tex.level.cube.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2480               "[$t, $s, \\{$x, $y, $z, $z\\}], $lod;",
2481               []>;
2482
2483 def TEX_CUBE_ARRAY_F32_F32
2484   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2485                     Float32Regs:$b, Float32Regs:$a),
2486               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l,
2487                Float32Regs:$x, Float32Regs:$y, Float32Regs:$z),
2488               "tex.acube.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2489               "[$t, $s, \\{$l, $x, $y, $z\\}];",
2490               []>;
2491 def TEX_CUBE_ARRAY_F32_F32_LEVEL
2492   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2493                     Float32Regs:$b, Float32Regs:$a),
2494               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l,
2495                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z,
2496                    Float32Regs:$lod),
2497               "tex.level.acube.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2498               "[$t, $s, \\{$l, $x, $y, $z\\}], $lod;",
2499               []>;
2500 def TEX_CUBE_ARRAY_S32_F32
2501   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2502                     Int32Regs:$b, Int32Regs:$a),
2503               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l,
2504                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z),
2505               "tex.acube.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2506               "[$t, $s, \\{$l, $x, $y, $z\\}];",
2507               []>;
2508 def TEX_CUBE_ARRAY_S32_F32_LEVEL
2509   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2510                     Int32Regs:$b, Int32Regs:$a),
2511               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l,
2512                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z,
2513                    Float32Regs:$lod),
2514               "tex.level.acube.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2515               "[$t, $s, \\{$l, $x, $y, $z\\}], $lod;",
2516               []>;
2517 def TEX_CUBE_ARRAY_U32_F32
2518   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2519                     Int32Regs:$b, Int32Regs:$a),
2520               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l,
2521                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z),
2522               "tex.acube.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2523               "[$t, $s, \\{$l, $x, $y, $z\\}];",
2524               []>;
2525 def TEX_CUBE_ARRAY_U32_F32_LEVEL
2526   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2527                     Int32Regs:$b, Int32Regs:$a),
2528               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l,
2529                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z,
2530                    Float32Regs:$lod),
2531               "tex.level.acube.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2532               "[$t, $s, \\{$l, $x, $y, $z\\}], $lod;",
2533               []>;
2534
2535 def TLD4_R_2D_F32_F32
2536   : NVPTXInst<(outs Float32Regs:$v0, Float32Regs:$v1,
2537                     Float32Regs:$v2, Float32Regs:$v3),
2538               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2539               "tld4.r.2d.v4.f32.f32\t\\{$v0, $v1, $v2, $v3\\}, "
2540               "[$t, $s, \\{$x, $y\\}];",
2541               []>;
2542 def TLD4_G_2D_F32_F32
2543   : NVPTXInst<(outs Float32Regs:$v0, Float32Regs:$v1,
2544                     Float32Regs:$v2, Float32Regs:$v3),
2545               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2546               "tld4.g.2d.v4.f32.f32\t\\{$v0, $v1, $v2, $v3\\}, "
2547               "[$t, $s, \\{$x, $y\\}];",
2548               []>;
2549 def TLD4_B_2D_F32_F32
2550   : NVPTXInst<(outs Float32Regs:$v0, Float32Regs:$v1,
2551                     Float32Regs:$v2, Float32Regs:$v3),
2552               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2553               "tld4.b.2d.v4.f32.f32\t\\{$v0, $v1, $v2, $v3\\}, "
2554               "[$t, $s, \\{$x, $y\\}];",
2555               []>;
2556 def TLD4_A_2D_F32_F32
2557   : NVPTXInst<(outs Float32Regs:$v0, Float32Regs:$v1,
2558                     Float32Regs:$v2, Float32Regs:$v3),
2559               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2560               "tld4.a.2d.v4.f32.f32\t\\{$v0, $v1, $v2, $v3\\}, "
2561               "[$t, $s, \\{$x, $y\\}];",
2562               []>;
2563 def TLD4_R_2D_S32_F32
2564   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
2565                     Int32Regs:$v2, Int32Regs:$v3),
2566               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2567               "tld4.r.2d.v4.s32.f32\t\\{$v0, $v1, $v2, $v3\\}, "
2568               "[$t, $s, \\{$x, $y\\}];",
2569               []>;
2570 def TLD4_G_2D_S32_F32
2571   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
2572                     Int32Regs:$v2, Int32Regs:$v3),
2573               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2574               "tld4.g.2d.v4.s32.f32\t\\{$v0, $v1, $v2, $v3\\}, "
2575               "[$t, $s, \\{$x, $y\\}];",
2576               []>;
2577 def TLD4_B_2D_S32_F32
2578   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
2579                     Int32Regs:$v2, Int32Regs:$v3),
2580               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2581               "tld4.b.2d.v4.s32.f32\t\\{$v0, $v1, $v2, $v3\\}, "
2582               "[$t, $s, \\{$x, $y\\}];",
2583               []>;
2584 def TLD4_A_2D_S32_F32
2585   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
2586                     Int32Regs:$v2, Int32Regs:$v3),
2587               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2588               "tld4.a.2d.v4.s32.f32\t\\{$v0, $v1, $v2, $v3\\}, "
2589               "[$t, $s, \\{$x, $y\\}];",
2590               []>;
2591 def TLD4_R_2D_U32_F32
2592   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
2593                     Int32Regs:$v2, Int32Regs:$v3),
2594               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2595               "tld4.r.2d.v4.u32.f32\t\\{$v0, $v1, $v2, $v3\\}, "
2596               "[$t, $s, \\{$x, $y\\}];",
2597               []>;
2598 def TLD4_G_2D_U32_F32
2599   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
2600                     Int32Regs:$v2, Int32Regs:$v3),
2601               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2602               "tld4.g.2d.v4.u32.f32\t\\{$v0, $v1, $v2, $v3\\}, "
2603               "[$t, $s, \\{$x, $y\\}];",
2604               []>;
2605 def TLD4_B_2D_U32_F32
2606   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
2607                     Int32Regs:$v2, Int32Regs:$v3),
2608               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2609               "tld4.b.2d.v4.u32.f32\t\\{$v0, $v1, $v2, $v3\\}, "
2610               "[$t, $s, \\{$x, $y\\}];",
2611               []>;
2612 def TLD4_A_2D_U32_F32
2613   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
2614                     Int32Regs:$v2, Int32Regs:$v3),
2615               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2616               "tld4.a.2d.v4.u32.f32\t\\{$v0, $v1, $v2, $v3\\}, "
2617               "[$t, $s, \\{$x, $y\\}];",
2618               []>;
2619 }
2620
2621
2622 // texmode_unified
2623 let IsTex = 1, IsTexModeUnified = 1 in {
2624 // Texture fetch instructions using handles
2625 def TEX_UNIFIED_1D_F32_S32
2626   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2627                     Float32Regs:$b, Float32Regs:$a),
2628               (ins Int64Regs:$t, Int32Regs:$x),
2629               "tex.1d.v4.f32.s32\t\\{$r, $g, $b, $a\\}, [$t, \\{$x\\}];",
2630               []>;
2631 def TEX_UNIFIED_1D_F32_F32
2632   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2633                     Float32Regs:$b, Float32Regs:$a),
2634               (ins Int64Regs:$t, Float32Regs:$x),
2635               "tex.1d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, [$t, \\{$x\\}];",
2636               []>;
2637 def TEX_UNIFIED_1D_F32_F32_LEVEL
2638   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2639                     Float32Regs:$b, Float32Regs:$a),
2640               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$lod),
2641               "tex.level.1d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2642               "[$t, \\{$x\\}], $lod;",
2643               []>;
2644 def TEX_UNIFIED_1D_F32_F32_GRAD
2645   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2646                     Float32Regs:$b, Float32Regs:$a),
2647               (ins Int64Regs:$t, Float32Regs:$x,
2648                    Float32Regs:$gradx, Float32Regs:$grady),
2649               "tex.grad.1d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2650               "[$t, \\{$x\\}], \\{$gradx\\}, \\{$grady\\};",
2651               []>;
2652 def TEX_UNIFIED_1D_S32_S32
2653   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2654                     Int32Regs:$b, Int32Regs:$a),
2655               (ins Int64Regs:$t, Int32Regs:$x),
2656               "tex.1d.v4.s32.s32\t\\{$r, $g, $b, $a\\}, [$t, \\{$x\\}];",
2657               []>;
2658 def TEX_UNIFIED_1D_S32_F32
2659   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2660                     Int32Regs:$b, Int32Regs:$a),
2661               (ins Int64Regs:$t, Float32Regs:$x),
2662               "tex.1d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, [$t, \\{$x\\}];",
2663               []>;
2664 def TEX_UNIFIED_1D_S32_F32_LEVEL
2665   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2666                     Int32Regs:$b, Int32Regs:$a),
2667               (ins Int64Regs:$t, Float32Regs:$x,
2668                    Float32Regs:$lod),
2669               "tex.level.1d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2670               "[$t, \\{$x\\}], $lod;",
2671               []>;
2672 def TEX_UNIFIED_1D_S32_F32_GRAD
2673   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2674                     Int32Regs:$b, Int32Regs:$a),
2675               (ins Int64Regs:$t, Float32Regs:$x,
2676                    Float32Regs:$gradx, Float32Regs:$grady),
2677               "tex.grad.1d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2678               "[$t, \\{$x\\}], \\{$gradx\\}, \\{$grady\\};",
2679               []>;
2680 def TEX_UNIFIED_1D_U32_S32
2681   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2682                     Int32Regs:$b, Int32Regs:$a),
2683               (ins Int64Regs:$t, Int32Regs:$x),
2684               "tex.1d.v4.u32.s32\t\\{$r, $g, $b, $a\\}, [$t, \\{$x\\}];",
2685               []>;
2686 def TEX_UNIFIED_1D_U32_F32
2687   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2688                     Int32Regs:$b, Int32Regs:$a),
2689               (ins Int64Regs:$t, Float32Regs:$x),
2690               "tex.1d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, [$t, \\{$x\\}];",
2691               []>;
2692 def TEX_UNIFIED_1D_U32_F32_LEVEL
2693   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2694                     Int32Regs:$b, Int32Regs:$a),
2695               (ins Int64Regs:$t, Float32Regs:$x,
2696                    Float32Regs:$lod),
2697               "tex.level.1d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2698               "[$t, \\{$x\\}], $lod;",
2699               []>;
2700 def TEX_UNIFIED_1D_U32_F32_GRAD
2701   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2702                     Int32Regs:$b, Int32Regs:$a),
2703               (ins Int64Regs:$t, Float32Regs:$x,
2704                    Float32Regs:$gradx, Float32Regs:$grady),
2705               "tex.grad.1d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2706               "[$t, \\{$x\\}], \\{$gradx\\}, \\{$grady\\};",
2707               []>;
2708
2709 def TEX_UNIFIED_1D_ARRAY_F32_S32
2710   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2711                     Float32Regs:$b, Float32Regs:$a),
2712               (ins Int64Regs:$t, Int32Regs:$l, Int32Regs:$x),
2713               "tex.a1d.v4.f32.s32\t\\{$r, $g, $b, $a\\}, "
2714               "[$t, \\{$l, $x\\}];",
2715               []>;
2716 def TEX_UNIFIED_1D_ARRAY_F32_F32
2717   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2718                     Float32Regs:$b, Float32Regs:$a),
2719               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x),
2720               "tex.a1d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2721               "[$t, \\{$l, $x\\}];",
2722               []>;
2723 def TEX_UNIFIED_1D_ARRAY_F32_F32_LEVEL
2724   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2725                     Float32Regs:$b, Float32Regs:$a),
2726               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
2727                    Float32Regs:$lod),
2728               "tex.level.a1d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2729               "[$t, \\{$l, $x\\}], $lod;",
2730               []>;
2731 def TEX_UNIFIED_1D_ARRAY_F32_F32_GRAD
2732   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2733                     Float32Regs:$b, Float32Regs:$a),
2734               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
2735                    Float32Regs:$gradx, Float32Regs:$grady),
2736               "tex.grad.a1d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2737               "[$t, \\{$l, $x\\}], \\{$gradx\\}, \\{$grady\\};",
2738               []>;
2739 def TEX_UNIFIED_1D_ARRAY_S32_S32
2740   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2741                     Int32Regs:$b, Int32Regs:$a),
2742               (ins Int64Regs:$t, Int32Regs:$l, Int32Regs:$x),
2743               "tex.a1d.v4.s32.s32\t\\{$r, $g, $b, $a\\}, "
2744               "[$t, \\{$l, $x\\}];",
2745               []>;
2746 def TEX_UNIFIED_1D_ARRAY_S32_F32
2747   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2748                     Int32Regs:$b, Int32Regs:$a),
2749               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x),
2750               "tex.a1d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2751               "[$t, \\{$l, $x\\}];",
2752               []>;
2753 def TEX_UNIFIED_1D_ARRAY_S32_F32_LEVEL
2754   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2755                     Int32Regs:$b, Int32Regs:$a),
2756               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
2757                    Float32Regs:$lod),
2758               "tex.level.a1d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2759               "[$t, \\{$l, $x\\}], $lod;",
2760               []>;
2761 def TEX_UNIFIED_1D_ARRAY_S32_F32_GRAD
2762   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2763                     Int32Regs:$b, Int32Regs:$a),
2764               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
2765                    Float32Regs:$gradx, Float32Regs:$grady),
2766               "tex.grad.a1d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2767               "[$t, \\{$l, $x\\}], \\{$gradx\\}, \\{$grady\\};",
2768               []>;
2769 def TEX_UNIFIED_1D_ARRAY_U32_S32
2770   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2771                     Int32Regs:$b, Int32Regs:$a),
2772               (ins Int64Regs:$t, Int32Regs:$l, Int32Regs:$x),
2773               "tex.a1d.v4.u32.s32\t\\{$r, $g, $b, $a\\}, "
2774               "[$t, \\{$l, $x\\}];",
2775               []>;
2776 def TEX_UNIFIED_1D_ARRAY_U32_F32
2777   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2778                     Int32Regs:$b, Int32Regs:$a),
2779               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x),
2780               "tex.a1d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2781               "[$t, \\{$l, $x\\}];",
2782               []>;
2783 def TEX_UNIFIED_1D_ARRAY_U32_F32_LEVEL
2784   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2785                     Int32Regs:$b, Int32Regs:$a),
2786               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
2787                    Float32Regs:$lod),
2788               "tex.level.a1d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2789               "[$t, \\{$l, $x\\}], $lod;",
2790               []>;
2791 def TEX_UNIFIED_1D_ARRAY_U32_F32_GRAD
2792   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2793                     Int32Regs:$b, Int32Regs:$a),
2794               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
2795                    Float32Regs:$gradx, Float32Regs:$grady),
2796               "tex.grad.a1d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2797               "[$t, \\{$l, $x\\}], \\{$gradx\\}, \\{$grady\\};",
2798               []>;
2799
2800 def TEX_UNIFIED_2D_F32_S32
2801   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2802                     Float32Regs:$b, Float32Regs:$a),
2803               (ins Int64Regs:$t, Int32Regs:$x, Int32Regs:$y),
2804               "tex.2d.v4.f32.s32\t\\{$r, $g, $b, $a\\}, "
2805               "[$t, \\{$x, $y\\}];",
2806               []>;
2807 def TEX_UNIFIED_2D_F32_F32
2808   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2809                     Float32Regs:$b, Float32Regs:$a),
2810               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
2811               "tex.2d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2812               "[$t, \\{$x, $y\\}];",
2813               []>;
2814 def TEX_UNIFIED_2D_F32_F32_LEVEL
2815   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2816                     Float32Regs:$b, Float32Regs:$a),
2817               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
2818                    Float32Regs:$lod),
2819               "tex.level.2d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2820               "[$t, \\{$x, $y\\}], $lod;",
2821               []>;
2822 def TEX_UNIFIED_2D_F32_F32_GRAD
2823   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2824                     Float32Regs:$b, Float32Regs:$a),
2825               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
2826                    Float32Regs:$gradx0, Float32Regs:$gradx1,
2827                    Float32Regs:$grady0, Float32Regs:$grady1),
2828               "tex.grad.2d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2829               "[$t, \\{$x, $y\\}], \\{$gradx0, $gradx1\\}, "
2830               "\\{$grady0, $grady1\\};",
2831               []>;
2832 def TEX_UNIFIED_2D_S32_S32
2833   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2834                     Int32Regs:$b, Int32Regs:$a),
2835               (ins Int64Regs:$t, Int32Regs:$x, Int32Regs:$y),
2836               "tex.2d.v4.s32.s32\t\\{$r, $g, $b, $a\\}, "
2837               "[$t, \\{$x, $y\\}];",
2838               []>;
2839 def TEX_UNIFIED_2D_S32_F32
2840   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2841                     Int32Regs:$b, Int32Regs:$a),
2842               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
2843               "tex.2d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2844               "[$t, \\{$x, $y\\}];",
2845               []>;
2846 def TEX_UNIFIED_2D_S32_F32_LEVEL
2847   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2848                     Int32Regs:$b, Int32Regs:$a),
2849               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
2850                    Float32Regs:$lod),
2851               "tex.level.2d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2852               "[$t, \\{$x, $y\\}], $lod;",
2853               []>;
2854 def TEX_UNIFIED_2D_S32_F32_GRAD
2855   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2856                     Int32Regs:$b, Int32Regs:$a),
2857               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
2858                    Float32Regs:$gradx0, Float32Regs:$gradx1,
2859                    Float32Regs:$grady0, Float32Regs:$grady1),
2860               "tex.grad.2d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2861               "[$t, \\{$x, $y\\}], \\{$gradx0, $gradx1\\}, "
2862               "\\{$grady0, $grady1\\};",
2863               []>;
2864 def TEX_UNIFIED_2D_U32_S32
2865   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2866                     Int32Regs:$b, Int32Regs:$a),
2867               (ins Int64Regs:$t, Int32Regs:$x, Int32Regs:$y),
2868               "tex.2d.v4.u32.s32\t\\{$r, $g, $b, $a\\}, "
2869               "[$t, \\{$x, $y\\}];",
2870               []>;
2871 def TEX_UNIFIED_2D_U32_F32
2872   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2873                     Int32Regs:$b, Int32Regs:$a),
2874               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
2875               "tex.2d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2876               "[$t, \\{$x, $y\\}];",
2877               []>;
2878 def TEX_UNIFIED_2D_U32_F32_LEVEL
2879   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2880                     Int32Regs:$b, Int32Regs:$a),
2881               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
2882                    Float32Regs:$lod),
2883               "tex.level.2d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2884               "[$t, \\{$x, $y\\}], $lod;",
2885               []>;
2886 def TEX_UNIFIED_2D_U32_F32_GRAD
2887   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2888                     Int32Regs:$b, Int32Regs:$a),
2889               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
2890                    Float32Regs:$gradx0, Float32Regs:$gradx1,
2891                    Float32Regs:$grady0, Float32Regs:$grady1),
2892               "tex.grad.2d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2893               "[$t, \\{$x, $y\\}], \\{$gradx0, $gradx1\\}, "
2894               "\\{$grady0, $grady1\\};",
2895               []>;
2896
2897 def TEX_UNIFIED_2D_ARRAY_F32_S32
2898   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2899                     Float32Regs:$b, Float32Regs:$a),
2900               (ins Int64Regs:$t, Int32Regs:$l, Int32Regs:$x,
2901                    Int32Regs:$y),
2902               "tex.a2d.v4.f32.s32\t\\{$r, $g, $b, $a\\}, "
2903               "[$t, \\{$l, $x, $y, $y\\}];",
2904               []>;
2905 def TEX_UNIFIED_2D_ARRAY_F32_F32
2906   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2907                     Float32Regs:$b, Float32Regs:$a),
2908               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
2909                    Float32Regs:$y),
2910               "tex.a2d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2911               "[$t, \\{$l, $x, $y, $y\\}];",
2912               []>;
2913 def TEX_UNIFIED_2D_ARRAY_F32_F32_LEVEL
2914   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2915                     Float32Regs:$b, Float32Regs:$a),
2916               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
2917                    Float32Regs:$y, Float32Regs:$lod),
2918               "tex.level.a2d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2919               "[$t, \\{$l, $x, $y, $y\\}], $lod;",
2920               []>;
2921 def TEX_UNIFIED_2D_ARRAY_F32_F32_GRAD
2922   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2923                     Float32Regs:$b, Float32Regs:$a),
2924               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
2925                    Float32Regs:$y, Float32Regs:$gradx0, Float32Regs:$gradx1,
2926                    Float32Regs:$grady0, Float32Regs:$grady1),
2927               "tex.grad.a2d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
2928               "[$t, \\{$l, $x, $y, $y\\}], \\{$gradx0, $gradx1\\}, "
2929               "\\{$grady0, $grady1\\};",
2930               []>;
2931 def TEX_UNIFIED_2D_ARRAY_S32_S32
2932   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2933                     Int32Regs:$b, Int32Regs:$a),
2934               (ins Int64Regs:$t, Int32Regs:$l, Int32Regs:$x,
2935                    Int32Regs:$y),
2936               "tex.a2d.v4.s32.s32\t\\{$r, $g, $b, $a\\}, "
2937               "[$t, \\{$l, $x, $y, $y\\}];",
2938               []>;
2939 def TEX_UNIFIED_2D_ARRAY_S32_F32
2940   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2941                     Int32Regs:$b, Int32Regs:$a),
2942               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
2943                    Float32Regs:$y),
2944               "tex.a2d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2945               "[$t, \\{$l, $x, $y, $y\\}];",
2946               []>;
2947 def TEX_UNIFIED_2D_ARRAY_S32_F32_LEVEL
2948   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2949                     Int32Regs:$b, Int32Regs:$a),
2950               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
2951                    Float32Regs:$y, Float32Regs:$lod),
2952               "tex.level.a2d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2953               "[$t, \\{$l, $x, $y, $y\\}], $lod;",
2954               []>;
2955 def TEX_UNIFIED_2D_ARRAY_S32_F32_GRAD
2956   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2957                     Int32Regs:$b, Int32Regs:$a),
2958               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
2959                    Float32Regs:$y,
2960                    Float32Regs:$gradx0, Float32Regs:$gradx1,
2961                    Float32Regs:$grady0, Float32Regs:$grady1),
2962               "tex.grad.a2d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
2963               "[$t, \\{$l, $x, $y, $y\\}], \\{$gradx0, $gradx1\\}, "
2964               "\\{$grady0, $grady1\\};",
2965               []>;
2966 def TEX_UNIFIED_2D_ARRAY_U32_S32
2967   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2968                     Int32Regs:$b, Int32Regs:$a),
2969               (ins Int64Regs:$t, Int32Regs:$l, Int32Regs:$x,
2970                    Int32Regs:$y),
2971               "tex.a2d.v4.u32.s32\t\\{$r, $g, $b, $a\\}, "
2972               "[$t, \\{$l, $x, $y, $y\\}];",
2973               []>;
2974 def TEX_UNIFIED_2D_ARRAY_U32_F32
2975   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2976                     Int32Regs:$b, Int32Regs:$a),
2977               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
2978                    Float32Regs:$y),
2979               "tex.a2d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2980               "[$t, \\{$l, $x, $y, $y\\}];",
2981               []>;
2982 def TEX_UNIFIED_2D_ARRAY_U32_F32_LEVEL
2983   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2984                     Int32Regs:$b, Int32Regs:$a),
2985               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
2986                    Float32Regs:$y, Float32Regs:$lod),
2987               "tex.level.a2d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2988               "[$t, \\{$l, $x, $y, $y\\}], $lod;",
2989               []>;
2990 def TEX_UNIFIED_2D_ARRAY_U32_F32_GRAD
2991   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2992                     Int32Regs:$b, Int32Regs:$a),
2993               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
2994                    Float32Regs:$y,
2995                    Float32Regs:$gradx0, Float32Regs:$gradx1,
2996                    Float32Regs:$grady0, Float32Regs:$grady1),
2997               "tex.grad.a2d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
2998               "[$t, \\{$l, $x, $y, $y\\}], \\{$gradx0, $gradx1\\}, "
2999               "\\{$grady0, $grady1\\};",
3000               []>;
3001
3002 def TEX_UNIFIED_3D_F32_S32
3003   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3004                     Float32Regs:$b, Float32Regs:$a),
3005               (ins Int64Regs:$t, Int32Regs:$x, Int32Regs:$y,
3006                    Int32Regs:$z),
3007               "tex.3d.v4.f32.s32\t\\{$r, $g, $b, $a\\}, "
3008               "[$t, \\{$x, $y, $z, $z\\}];",
3009               []>;
3010 def TEX_UNIFIED_3D_F32_F32
3011   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3012                     Float32Regs:$b, Float32Regs:$a),
3013               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
3014                    Float32Regs:$z),
3015               "tex.3d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
3016               "[$t, \\{$x, $y, $z, $z\\}];",
3017               []>;
3018 def TEX_UNIFIED_3D_F32_F32_LEVEL
3019   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3020                     Float32Regs:$b, Float32Regs:$a),
3021               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
3022                    Float32Regs:$z, Float32Regs:$lod),
3023               "tex.level.3d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
3024               "[$t, \\{$x, $y, $z, $z\\}], $lod;",
3025               []>;
3026 def TEX_UNIFIED_3D_F32_F32_GRAD
3027   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3028                     Float32Regs:$b, Float32Regs:$a),
3029               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
3030                    Float32Regs:$z,
3031                    Float32Regs:$gradx0, Float32Regs:$gradx1,
3032                    Float32Regs:$gradx2, Float32Regs:$grady0,
3033                    Float32Regs:$grady1, Float32Regs:$grady2),
3034               "tex.grad.3d.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
3035               "[$t, \\{$x, $y, $z, $z\\}], "
3036               "\\{$gradx0, $gradx1, $gradx2, $gradx2\\}, "
3037               "\\{$grady0, $grady1, $grady2, $grady2\\};",
3038               []>;
3039 def TEX_UNIFIED_3D_S32_S32
3040   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3041                     Int32Regs:$b, Int32Regs:$a),
3042               (ins Int64Regs:$t, Int32Regs:$x, Int32Regs:$y,
3043                    Int32Regs:$z),
3044               "tex.3d.v4.s32.s32\t\\{$r, $g, $b, $a\\}, "
3045               "[$t, \\{$x, $y, $z, $z\\}];",
3046               []>;
3047 def TEX_UNIFIED_3D_S32_F32
3048   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3049                     Int32Regs:$b, Int32Regs:$a),
3050               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
3051                    Float32Regs:$z),
3052               "tex.3d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
3053               "[$t, \\{$x, $y, $z, $z\\}];",
3054               []>;
3055 def TEX_UNIFIED_3D_S32_F32_LEVEL
3056   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3057                     Int32Regs:$b, Int32Regs:$a),
3058               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
3059                    Float32Regs:$z, Float32Regs:$lod),
3060               "tex.level.3d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
3061               "[$t, \\{$x, $y, $z, $z\\}], $lod;",
3062               []>;
3063 def TEX_UNIFIED_3D_S32_F32_GRAD
3064   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3065                     Int32Regs:$b, Int32Regs:$a),
3066               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
3067                    Float32Regs:$z,
3068                    Float32Regs:$gradx0, Float32Regs:$gradx1,
3069                    Float32Regs:$gradx2, Float32Regs:$grady0,
3070                    Float32Regs:$grady1, Float32Regs:$grady2),
3071               "tex.grad.3d.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
3072               "[$t, \\{$x, $y, $z, $z\\}], "
3073               "\\{$gradx0, $gradx1, $gradx2, $gradx2\\}, "
3074               "\\{$grady0, $grady1, $grady2, $grady2\\};",
3075               []>;
3076 def TEX_UNIFIED_3D_U32_S32
3077   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3078                     Int32Regs:$b, Int32Regs:$a),
3079               (ins Int64Regs:$t, Int32Regs:$x, Int32Regs:$y,
3080                    Int32Regs:$z),
3081               "tex.3d.v4.u32.s32\t\\{$r, $g, $b, $a\\}, "
3082               "[$t, \\{$x, $y, $z, $z\\}];",
3083               []>;
3084 def TEX_UNIFIED_3D_U32_F32
3085   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3086                     Int32Regs:$b, Int32Regs:$a),
3087               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
3088                    Float32Regs:$z),
3089               "tex.3d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
3090               "[$t, \\{$x, $y, $z, $z\\}];",
3091               []>;
3092 def TEX_UNIFIED_3D_U32_F32_LEVEL
3093   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3094                     Int32Regs:$b, Int32Regs:$a),
3095               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
3096                    Float32Regs:$z, Float32Regs:$lod),
3097               "tex.level.3d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
3098               "[$t, \\{$x, $y, $z, $z\\}], $lod;",
3099               []>;
3100 def TEX_UNIFIED_3D_U32_F32_GRAD
3101   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3102                     Int32Regs:$b, Int32Regs:$a),
3103               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
3104                    Float32Regs:$z,
3105                    Float32Regs:$gradx0, Float32Regs:$gradx1,
3106                    Float32Regs:$gradx2, Float32Regs:$grady0,
3107                    Float32Regs:$grady1, Float32Regs:$grady2),
3108               "tex.grad.3d.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
3109               "[$t, \\{$x, $y, $z, $z\\}], "
3110               "\\{$gradx0, $gradx1, $gradx2, $gradx2\\}, "
3111               "\\{$grady0, $grady1, $grady2, $grady2\\};",
3112               []>;
3113
3114 def TEX_UNIFIED_CUBE_F32_F32
3115   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3116                     Float32Regs:$b, Float32Regs:$a),
3117               (ins Int64Regs:$t,
3118                Float32Regs:$x, Float32Regs:$y, Float32Regs:$z),
3119               "tex.cube.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
3120               "[$t, \\{$x, $y, $z, $z\\}];",
3121               []>;
3122 def TEX_UNIFIED_CUBE_F32_F32_LEVEL
3123   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3124                     Float32Regs:$b, Float32Regs:$a),
3125               (ins Int64Regs:$t,
3126                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z,
3127                    Float32Regs:$lod),
3128               "tex.level.cube.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
3129               "[$t, \\{$x, $y, $z, $z\\}], $lod;",
3130               []>;
3131 def TEX_UNIFIED_CUBE_S32_F32
3132   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3133                     Int32Regs:$b, Int32Regs:$a),
3134               (ins Int64Regs:$t,
3135                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z),
3136               "tex.cube.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
3137               "[$t, \\{$x, $y, $z, $z\\}];",
3138               []>;
3139 def TEX_UNIFIED_CUBE_S32_F32_LEVEL
3140   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3141                     Int32Regs:$b, Int32Regs:$a),
3142               (ins Int64Regs:$t,
3143                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z,
3144                    Float32Regs:$lod),
3145               "tex.level.cube.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
3146               "[$t, \\{$x, $y, $z, $z\\}], $lod;",
3147               []>;
3148 def TEX_UNIFIED_CUBE_U32_F32
3149   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3150                     Int32Regs:$b, Int32Regs:$a),
3151               (ins Int64Regs:$t,
3152                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z),
3153               "tex.cube.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
3154               "[$t, \\{$x, $y, $z, $z\\}];",
3155               []>;
3156 def TEX_UNIFIED_CUBE_U32_F32_LEVEL
3157   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3158                     Int32Regs:$b, Int32Regs:$a),
3159               (ins Int64Regs:$t,
3160                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z,
3161                    Float32Regs:$lod),
3162               "tex.level.cube.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
3163               "[$t, \\{$x, $y, $z, $z\\}], $lod;",
3164               []>;
3165
3166 def TEX_UNIFIED_CUBE_ARRAY_F32_F32
3167   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3168                     Float32Regs:$b, Float32Regs:$a),
3169               (ins Int64Regs:$t, Int32Regs:$l,
3170                Float32Regs:$x, Float32Regs:$y, Float32Regs:$z),
3171               "tex.acube.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
3172               "[$t, \\{$l, $x, $y, $z\\}];",
3173               []>;
3174 def TEX_UNIFIED_CUBE_ARRAY_F32_F32_LEVEL
3175   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3176                     Float32Regs:$b, Float32Regs:$a),
3177               (ins Int64Regs:$t, Int32Regs:$l,
3178                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z,
3179                    Float32Regs:$lod),
3180               "tex.level.acube.v4.f32.f32\t\\{$r, $g, $b, $a\\}, "
3181               "[$t, \\{$l, $x, $y, $z\\}], $lod;",
3182               []>;
3183 def TEX_UNIFIED_CUBE_ARRAY_S32_F32
3184   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3185                     Int32Regs:$b, Int32Regs:$a),
3186               (ins Int64Regs:$t, Int32Regs:$l,
3187                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z),
3188               "tex.acube.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
3189               "[$t, \\{$l, $x, $y, $z\\}];",
3190               []>;
3191 def TEX_UNIFIED_CUBE_ARRAY_S32_F32_LEVEL
3192   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3193                     Int32Regs:$b, Int32Regs:$a),
3194               (ins Int64Regs:$t, Int32Regs:$l,
3195                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z,
3196                    Float32Regs:$lod),
3197               "tex.level.acube.v4.s32.f32\t\\{$r, $g, $b, $a\\}, "
3198               "[$t, \\{$l, $x, $y, $z\\}], $lod;",
3199               []>;
3200 def TEX_UNIFIED_CUBE_ARRAY_U32_F32
3201   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3202                     Int32Regs:$b, Int32Regs:$a),
3203               (ins Int64Regs:$t, Int32Regs:$l,
3204                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z),
3205               "tex.acube.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
3206               "[$t, \\{$l, $x, $y, $z\\}];",
3207               []>;
3208 def TEX_UNIFIED_CUBE_ARRAY_U32_F32_LEVEL
3209   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3210                     Int32Regs:$b, Int32Regs:$a),
3211               (ins Int64Regs:$t, Int32Regs:$l,
3212                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z,
3213                    Float32Regs:$lod),
3214               "tex.level.acube.v4.u32.f32\t\\{$r, $g, $b, $a\\}, "
3215               "[$t, \\{$l, $x, $y, $z\\}], $lod;",
3216               []>;
3217
3218 def TLD4_UNIFIED_R_2D_F32_F32
3219   : NVPTXInst<(outs Float32Regs:$v0, Float32Regs:$v1,
3220                     Float32Regs:$v2, Float32Regs:$v3),
3221               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3222               "tld4.r.2d.v4.f32.f32\t\\{$v0, $v1, $v2, $v3\\}, "
3223               "[$t, \\{$x, $y\\}];",
3224               []>;
3225 def TLD4_UNIFIED_G_2D_F32_F32
3226   : NVPTXInst<(outs Float32Regs:$v0, Float32Regs:$v1,
3227                     Float32Regs:$v2, Float32Regs:$v3),
3228               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3229               "tld4.g.2d.v4.f32.f32\t\\{$v0, $v1, $v2, $v3\\}, "
3230               "[$t, \\{$x, $y\\}];",
3231               []>;
3232 def TLD4_UNIFIED_B_2D_F32_F32
3233   : NVPTXInst<(outs Float32Regs:$v0, Float32Regs:$v1,
3234                     Float32Regs:$v2, Float32Regs:$v3),
3235               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3236               "tld4.b.2d.v4.f32.f32\t\\{$v0, $v1, $v2, $v3\\}, "
3237               "[$t, \\{$x, $y\\}];",
3238               []>;
3239 def TLD4_UNIFIED_A_2D_F32_F32
3240   : NVPTXInst<(outs Float32Regs:$v0, Float32Regs:$v1,
3241                     Float32Regs:$v2, Float32Regs:$v3),
3242               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3243               "tld4.a.2d.v4.f32.f32\t\\{$v0, $v1, $v2, $v3\\}, "
3244               "[$t, \\{$x, $y\\}];",
3245               []>;
3246 def TLD4_UNIFIED_R_2D_S32_F32
3247   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
3248                     Int32Regs:$v2, Int32Regs:$v3),
3249               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3250               "tld4.r.2d.v4.s32.f32\t\\{$v0, $v1, $v2, $v3\\}, "
3251               "[$t, \\{$x, $y\\}];",
3252               []>;
3253 def TLD4_UNIFIED_G_2D_S32_F32
3254   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
3255                     Int32Regs:$v2, Int32Regs:$v3),
3256               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3257               "tld4.g.2d.v4.s32.f32\t\\{$v0, $v1, $v2, $v3\\}, "
3258               "[$t, \\{$x, $y\\}];",
3259               []>;
3260 def TLD4_UNIFIED_B_2D_S32_F32
3261   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
3262                     Int32Regs:$v2, Int32Regs:$v3),
3263               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3264               "tld4.b.2d.v4.s32.f32\t\\{$v0, $v1, $v2, $v3\\}, "
3265               "[$t, \\{$x, $y\\}];",
3266               []>;
3267 def TLD4_UNIFIED_A_2D_S32_F32
3268   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
3269                     Int32Regs:$v2, Int32Regs:$v3),
3270               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3271               "tld4.a.2d.v4.s32.f32\t\\{$v0, $v1, $v2, $v3\\}, "
3272               "[$t, \\{$x, $y\\}];",
3273               []>;
3274 def TLD4_UNIFIED_R_2D_U32_F32
3275   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
3276                     Int32Regs:$v2, Int32Regs:$v3),
3277               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3278               "tld4.r.2d.v4.u32.f32\t\\{$v0, $v1, $v2, $v3\\}, "
3279               "[$t, \\{$x, $y\\}];",
3280               []>;
3281 def TLD4_UNIFIED_G_2D_U32_F32
3282   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
3283                     Int32Regs:$v2, Int32Regs:$v3),
3284               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3285               "tld4.g.2d.v4.u32.f32\t\\{$v0, $v1, $v2, $v3\\}, "
3286               "[$t, \\{$x, $y\\}];",
3287               []>;
3288 def TLD4_UNIFIED_B_2D_U32_F32
3289   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
3290                     Int32Regs:$v2, Int32Regs:$v3),
3291               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3292               "tld4.b.2d.v4.u32.f32\t\\{$v0, $v1, $v2, $v3\\}, "
3293               "[$t, \\{$x, $y\\}];",
3294               []>;
3295 def TLD4_UNIFIED_A_2D_U32_F32
3296   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
3297                     Int32Regs:$v2, Int32Regs:$v3),
3298               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3299               "tld4.a.2d.v4.u32.f32\t\\{$v0, $v1, $v2, $v3\\}, "
3300               "[$t, \\{$x, $y\\}];",
3301               []>;
3302 }
3303
3304
3305
3306 //=== Surface load instructions
3307 // .clamp variant
3308 let IsSuld = 1 in {
3309 def SULD_1D_I8_CLAMP
3310   : NVPTXInst<(outs Int16Regs:$r),
3311               (ins Int64Regs:$s, Int32Regs:$x),
3312               "suld.b.1d.b8.clamp \\{$r\\}, [$s, \\{$x\\}];",
3313               []>;
3314 def SULD_1D_I16_CLAMP
3315   : NVPTXInst<(outs Int16Regs:$r),
3316               (ins Int64Regs:$s, Int32Regs:$x),
3317               "suld.b.1d.b16.clamp \\{$r\\}, [$s, \\{$x\\}];",
3318               []>;
3319 def SULD_1D_I32_CLAMP
3320   : NVPTXInst<(outs Int32Regs:$r),
3321               (ins Int64Regs:$s, Int32Regs:$x),
3322               "suld.b.1d.b32.clamp \\{$r\\}, [$s, \\{$x\\}];",
3323               []>;
3324 def SULD_1D_I64_CLAMP
3325   : NVPTXInst<(outs Int64Regs:$r),
3326               (ins Int64Regs:$s, Int32Regs:$x),
3327               "suld.b.1d.b64.clamp \\{$r\\}, [$s, \\{$x\\}];",
3328               []>;
3329
3330 def SULD_1D_ARRAY_I8_CLAMP
3331   : NVPTXInst<(outs Int16Regs:$r),
3332               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3333               "suld.b.a1d.b8.clamp \\{$r\\}, [$s, \\{$l, $x\\}];",
3334               []>;
3335 def SULD_1D_ARRAY_I16_CLAMP
3336   : NVPTXInst<(outs Int16Regs:$r),
3337               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3338               "suld.b.a1d.b16.clamp \\{$r\\}, [$s, \\{$l, $x\\}];",
3339               []>;
3340 def SULD_1D_ARRAY_I32_CLAMP
3341   : NVPTXInst<(outs Int32Regs:$r),
3342               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3343               "suld.b.a1d.b32.clamp \\{$r\\}, [$s, \\{$l, $x\\}];",
3344               []>;
3345 def SULD_1D_ARRAY_I64_CLAMP
3346   : NVPTXInst<(outs Int64Regs:$r),
3347               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3348               "suld.b.a1d.b64.clamp \\{$r\\}, [$s, \\{$l, $x\\}];",
3349               []>;
3350
3351 def SULD_2D_I8_CLAMP
3352   : NVPTXInst<(outs Int16Regs:$r),
3353               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3354               "suld.b.2d.b8.clamp \\{$r\\}, [$s, \\{$x, $y\\}];",
3355               []>;
3356 def SULD_2D_I16_CLAMP
3357   : NVPTXInst<(outs Int16Regs:$r),
3358               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3359               "suld.b.2d.b16.clamp \\{$r\\}, [$s, \\{$x, $y\\}];",
3360               []>;
3361 def SULD_2D_I32_CLAMP
3362   : NVPTXInst<(outs Int32Regs:$r),
3363               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3364               "suld.b.2d.b32.clamp \\{$r\\}, [$s, \\{$x, $y\\}];",
3365               []>;
3366 def SULD_2D_I64_CLAMP
3367   : NVPTXInst<(outs Int64Regs:$r),
3368               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3369               "suld.b.2d.b64.clamp \\{$r\\}, [$s, \\{$x, $y\\}];",
3370               []>;
3371
3372 def SULD_2D_ARRAY_I8_CLAMP
3373   : NVPTXInst<(outs Int16Regs:$r),
3374               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3375               "suld.b.a2d.b8.clamp \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",
3376               []>;
3377 def SULD_2D_ARRAY_I16_CLAMP
3378   : NVPTXInst<(outs Int16Regs:$r),
3379               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3380               "suld.b.a2d.b16.clamp \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",
3381               []>;
3382 def SULD_2D_ARRAY_I32_CLAMP
3383   : NVPTXInst<(outs Int32Regs:$r),
3384               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3385               "suld.b.a2d.b32.clamp \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",
3386               []>;
3387 def SULD_2D_ARRAY_I64_CLAMP
3388   : NVPTXInst<(outs Int64Regs:$r),
3389               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3390               "suld.b.a2d.b64.clamp \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",
3391               []>;
3392
3393 def SULD_3D_I8_CLAMP
3394   : NVPTXInst<(outs Int16Regs:$r),
3395               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3396               "suld.b.3d.b8.clamp \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",
3397               []>;
3398 def SULD_3D_I16_CLAMP
3399   : NVPTXInst<(outs Int16Regs:$r),
3400               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3401               "suld.b.3d.b16.clamp \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",
3402               []>;
3403 def SULD_3D_I32_CLAMP
3404   : NVPTXInst<(outs Int32Regs:$r),
3405               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3406               "suld.b.3d.b32.clamp \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",
3407               []>;
3408 def SULD_3D_I64_CLAMP
3409   : NVPTXInst<(outs Int64Regs:$r),
3410               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3411               "suld.b.3d.b64.clamp \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",
3412               []>;
3413 }
3414
3415 let IsSuld = 2 in {
3416 def SULD_1D_V2I8_CLAMP
3417   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3418               (ins Int64Regs:$s, Int32Regs:$x),
3419               "suld.b.1d.v2.b8.clamp \\{$r, $g\\}, [$s, \\{$x\\}];",
3420               []>;
3421 def SULD_1D_V2I16_CLAMP
3422   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3423               (ins Int64Regs:$s, Int32Regs:$x),
3424               "suld.b.1d.v2.b16.clamp \\{$r, $g\\}, [$s, \\{$x\\}];",
3425               []>;
3426 def SULD_1D_V2I32_CLAMP
3427   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
3428               (ins Int64Regs:$s, Int32Regs:$x),
3429               "suld.b.1d.v2.b32.clamp \\{$r, $g\\}, [$s, \\{$x\\}];",
3430               []>;
3431 def SULD_1D_V2I64_CLAMP
3432   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
3433               (ins Int64Regs:$s, Int32Regs:$x),
3434               "suld.b.1d.v2.b64.clamp \\{$r, $g\\}, [$s, \\{$x\\}];",
3435               []>;
3436
3437 def SULD_1D_ARRAY_V2I8_CLAMP
3438   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3439               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3440               "suld.b.a1d.v2.b8.clamp \\{$r, $g\\}, [$s, \\{$l, $x\\}];",
3441               []>;
3442 def SULD_1D_ARRAY_V2I16_CLAMP
3443   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3444               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3445               "suld.b.a1d.v2.b16.clamp \\{$r, $g\\}, [$s, \\{$l, $x\\}];",
3446               []>;
3447 def SULD_1D_ARRAY_V2I32_CLAMP
3448   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
3449               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3450               "suld.b.a1d.v2.b32.clamp \\{$r, $g\\}, [$s, \\{$l, $x\\}];",
3451               []>;
3452 def SULD_1D_ARRAY_V2I64_CLAMP
3453   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
3454               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3455               "suld.b.a1d.v2.b64.clamp \\{$r, $g\\}, [$s, \\{$l, $x\\}];",
3456               []>;
3457
3458 def SULD_2D_V2I8_CLAMP
3459   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3460               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3461               "suld.b.2d.v2.b8.clamp \\{$r, $g\\}, [$s, \\{$x, $y\\}];",
3462               []>;
3463 def SULD_2D_V2I16_CLAMP
3464   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3465               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3466               "suld.b.2d.v2.b16.clamp \\{$r, $g\\}, [$s, \\{$x, $y\\}];",
3467               []>;
3468 def SULD_2D_V2I32_CLAMP
3469   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
3470               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3471               "suld.b.2d.v2.b32.clamp \\{$r, $g\\}, [$s, \\{$x, $y\\}];",
3472               []>;
3473 def SULD_2D_V2I64_CLAMP
3474   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
3475               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3476               "suld.b.2d.v2.b64.clamp \\{$r, $g\\}, [$s, \\{$x, $y\\}];",
3477               []>;
3478
3479 def SULD_2D_ARRAY_V2I8_CLAMP
3480   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3481               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3482               "suld.b.a2d.v2.b8.clamp \\{$r, $g\\}, "
3483               "[$s, \\{$l, $x, $y, $y\\}];",
3484               []>;
3485 def SULD_2D_ARRAY_V2I16_CLAMP
3486   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3487               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3488               "suld.b.a2d.v2.b16.clamp \\{$r, $g\\}, "
3489               "[$s, \\{$l, $x, $y, $y\\}];",
3490               []>;
3491 def SULD_2D_ARRAY_V2I32_CLAMP
3492   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
3493               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3494               "suld.b.a2d.v2.b32.clamp \\{$r, $g\\}, "
3495               "[$s, \\{$l, $x, $y, $y\\}];",
3496               []>;
3497 def SULD_2D_ARRAY_V2I64_CLAMP
3498   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
3499               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3500               "suld.b.a2d.v2.b64.clamp \\{$r, $g\\}, "
3501               "[$s, \\{$l, $x, $y, $y\\}];",
3502               []>;
3503
3504 def SULD_3D_V2I8_CLAMP
3505   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3506               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3507               "suld.b.3d.v2.b8.clamp \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",
3508               []>;
3509 def SULD_3D_V2I16_CLAMP
3510   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3511               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3512               "suld.b.3d.v2.b16.clamp \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",
3513               []>;
3514 def SULD_3D_V2I32_CLAMP
3515   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
3516               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3517               "suld.b.3d.v2.b32.clamp \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",
3518               []>;
3519 def SULD_3D_V2I64_CLAMP
3520   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
3521               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3522               "suld.b.3d.v2.b64.clamp \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",
3523               []>;
3524 }
3525
3526 let IsSuld = 3 in {
3527 def SULD_1D_V4I8_CLAMP
3528   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3529               (ins Int64Regs:$s, Int32Regs:$x),
3530               "suld.b.1d.v4.b8.clamp \\{$r, $g, $b, $a\\}, [$s, \\{$x\\}];",
3531               []>;
3532 def SULD_1D_V4I16_CLAMP
3533   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3534               (ins Int64Regs:$s, Int32Regs:$x),
3535               "suld.b.1d.v4.b16.clamp \\{$r, $g, $b, $a\\}, [$s, \\{$x\\}];",
3536               []>;
3537 def SULD_1D_V4I32_CLAMP
3538   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
3539               (ins Int64Regs:$s, Int32Regs:$x),
3540               "suld.b.1d.v4.b32.clamp \\{$r, $g, $b, $a\\}, [$s, \\{$x\\}];",
3541               []>;
3542
3543 def SULD_1D_ARRAY_V4I8_CLAMP
3544   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3545               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3546               "suld.b.a1d.v4.b8.clamp \\{$r, $g, $b, $a\\}, "
3547               "[$s, \\{$l, $x\\}];",
3548               []>;
3549 def SULD_1D_ARRAY_V4I16_CLAMP
3550   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3551               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3552               "suld.b.a1d.v4.b16.clamp \\{$r, $g, $b, $a\\}, "
3553               "[$s, \\{$l, $x\\}];",
3554               []>;
3555 def SULD_1D_ARRAY_V4I32_CLAMP
3556   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
3557               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3558               "suld.b.a1d.v4.b32.clamp \\{$r, $g, $b, $a\\}, "
3559               "[$s, \\{$l, $x\\}];",
3560               []>;
3561
3562 def SULD_2D_V4I8_CLAMP
3563   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3564               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3565               "suld.b.2d.v4.b8.clamp \\{$r, $g, $b, $a\\}, [$s, \\{$x, $y\\}];",
3566               []>;
3567 def SULD_2D_V4I16_CLAMP
3568   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3569               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3570               "suld.b.2d.v4.b16.clamp \\{$r, $g, $b, $a\\}, [$s, \\{$x, $y\\}];",
3571               []>;
3572 def SULD_2D_V4I32_CLAMP
3573   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
3574               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3575               "suld.b.2d.v4.b32.clamp \\{$r, $g, $b, $a\\}, [$s, \\{$x, $y\\}];",
3576               []>;
3577
3578 def SULD_2D_ARRAY_V4I8_CLAMP
3579   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3580               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3581               "suld.b.a2d.v4.b8.clamp \\{$r, $g, $b, $a\\}, "
3582               "[$s, \\{$l, $x, $y, $y\\}];",
3583               []>;
3584 def SULD_2D_ARRAY_V4I16_CLAMP
3585   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3586               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3587               "suld.b.a2d.v4.b16.clamp \\{$r, $g, $b, $a\\}, "
3588               "[$s, \\{$l, $x, $y, $y\\}];",
3589               []>;
3590 def SULD_2D_ARRAY_V4I32_CLAMP
3591   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
3592               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3593               "suld.b.a2d.v4.b32.clamp \\{$r, $g, $b, $a\\}, "
3594               "[$s, \\{$l, $x, $y, $y\\}];",
3595               []>;
3596
3597
3598 def SULD_3D_V4I8_CLAMP
3599   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3600               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3601               "suld.b.3d.v4.b8.clamp \\{$r, $g, $b, $a\\}, "
3602               "[$s, \\{$x, $y, $z, $z\\}];",
3603               []>;
3604 def SULD_3D_V4I16_CLAMP
3605   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3606               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3607               "suld.b.3d.v4.b16.clamp \\{$r, $g, $b, $a\\}, "
3608               "[$s, \\{$x, $y, $z, $z\\}];",
3609               []>;
3610 def SULD_3D_V4I32_CLAMP
3611   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
3612               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3613               "suld.b.3d.v4.b32.clamp \\{$r, $g, $b, $a\\}, "
3614               "[$s, \\{$x, $y, $z, $z\\}];",
3615               []>;
3616 }
3617
3618
3619 // .trap variant
3620 let IsSuld = 1 in {
3621 def SULD_1D_I8_TRAP
3622   : NVPTXInst<(outs Int16Regs:$r),
3623               (ins Int64Regs:$s, Int32Regs:$x),
3624               "suld.b.1d.b8.trap \\{$r\\}, [$s, \\{$x\\}];",
3625               []>;
3626 def SULD_1D_I16_TRAP
3627   : NVPTXInst<(outs Int16Regs:$r),
3628               (ins Int64Regs:$s, Int32Regs:$x),
3629               "suld.b.1d.b16.trap \\{$r\\}, [$s, \\{$x\\}];",
3630               []>;
3631 def SULD_1D_I32_TRAP
3632   : NVPTXInst<(outs Int32Regs:$r),
3633               (ins Int64Regs:$s, Int32Regs:$x),
3634               "suld.b.1d.b32.trap \\{$r\\}, [$s, \\{$x\\}];",
3635               []>;
3636 def SULD_1D_I64_TRAP
3637   : NVPTXInst<(outs Int64Regs:$r),
3638               (ins Int64Regs:$s, Int32Regs:$x),
3639               "suld.b.1d.b64.trap \\{$r\\}, [$s, \\{$x\\}];",
3640               []>;
3641
3642 def SULD_1D_ARRAY_I8_TRAP
3643   : NVPTXInst<(outs Int16Regs:$r),
3644               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3645               "suld.b.a1d.b8.trap \\{$r\\}, [$s, \\{$l, $x\\}];",
3646               []>;
3647 def SULD_1D_ARRAY_I16_TRAP
3648   : NVPTXInst<(outs Int16Regs:$r),
3649               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3650               "suld.b.a1d.b16.trap \\{$r\\}, [$s, \\{$l, $x\\}];",
3651               []>;
3652 def SULD_1D_ARRAY_I32_TRAP
3653   : NVPTXInst<(outs Int32Regs:$r),
3654               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3655               "suld.b.a1d.b32.trap \\{$r\\}, [$s, \\{$l, $x\\}];",
3656               []>;
3657 def SULD_1D_ARRAY_I64_TRAP
3658   : NVPTXInst<(outs Int64Regs:$r),
3659               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3660               "suld.b.a1d.b64.trap \\{$r\\}, [$s, \\{$l, $x\\}];",
3661               []>;
3662
3663 def SULD_2D_I8_TRAP
3664   : NVPTXInst<(outs Int16Regs:$r),
3665               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3666               "suld.b.2d.b8.trap \\{$r\\}, [$s, \\{$x, $y\\}];",
3667               []>;
3668 def SULD_2D_I16_TRAP
3669   : NVPTXInst<(outs Int16Regs:$r),
3670               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3671               "suld.b.2d.b16.trap \\{$r\\}, [$s, \\{$x, $y\\}];",
3672               []>;
3673 def SULD_2D_I32_TRAP
3674   : NVPTXInst<(outs Int32Regs:$r),
3675               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3676               "suld.b.2d.b32.trap \\{$r\\}, [$s, \\{$x, $y\\}];",
3677               []>;
3678 def SULD_2D_I64_TRAP
3679   : NVPTXInst<(outs Int64Regs:$r),
3680               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3681               "suld.b.2d.b64.trap \\{$r\\}, [$s, \\{$x, $y\\}];",
3682               []>;
3683
3684 def SULD_2D_ARRAY_I8_TRAP
3685   : NVPTXInst<(outs Int16Regs:$r),
3686               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3687               "suld.b.a2d.b8.trap \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",
3688               []>;
3689 def SULD_2D_ARRAY_I16_TRAP
3690   : NVPTXInst<(outs Int16Regs:$r),
3691               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3692               "suld.b.a2d.b16.trap \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",
3693               []>;
3694 def SULD_2D_ARRAY_I32_TRAP
3695   : NVPTXInst<(outs Int32Regs:$r),
3696               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3697               "suld.b.a2d.b32.trap \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",
3698               []>;
3699 def SULD_2D_ARRAY_I64_TRAP
3700   : NVPTXInst<(outs Int64Regs:$r),
3701               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3702               "suld.b.a2d.b64.trap \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",
3703               []>;
3704
3705 def SULD_3D_I8_TRAP
3706   : NVPTXInst<(outs Int16Regs:$r),
3707               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3708               "suld.b.3d.b8.trap \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",
3709               []>;
3710 def SULD_3D_I16_TRAP
3711   : NVPTXInst<(outs Int16Regs:$r),
3712               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3713               "suld.b.3d.b16.trap \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",
3714               []>;
3715 def SULD_3D_I32_TRAP
3716   : NVPTXInst<(outs Int32Regs:$r),
3717               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3718               "suld.b.3d.b32.trap \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",
3719               []>;
3720 def SULD_3D_I64_TRAP
3721   : NVPTXInst<(outs Int64Regs:$r),
3722               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3723               "suld.b.3d.b64.trap \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",
3724               []>;
3725 }
3726
3727 let IsSuld = 2 in {
3728 def SULD_1D_V2I8_TRAP
3729   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3730               (ins Int64Regs:$s, Int32Regs:$x),
3731               "suld.b.1d.v2.b8.trap \\{$r, $g\\}, [$s, \\{$x\\}];",
3732               []>;
3733 def SULD_1D_V2I16_TRAP
3734   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3735               (ins Int64Regs:$s, Int32Regs:$x),
3736               "suld.b.1d.v2.b16.trap \\{$r, $g\\}, [$s, \\{$x\\}];",
3737               []>;
3738 def SULD_1D_V2I32_TRAP
3739   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
3740               (ins Int64Regs:$s, Int32Regs:$x),
3741               "suld.b.1d.v2.b32.trap \\{$r, $g\\}, [$s, \\{$x\\}];",
3742               []>;
3743 def SULD_1D_V2I64_TRAP
3744   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
3745               (ins Int64Regs:$s, Int32Regs:$x),
3746               "suld.b.1d.v2.b64.trap \\{$r, $g\\}, [$s, \\{$x\\}];",
3747               []>;
3748
3749 def SULD_1D_ARRAY_V2I8_TRAP
3750   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3751               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3752               "suld.b.a1d.v2.b8.trap \\{$r, $g\\}, [$s, \\{$l, $x\\}];",
3753               []>;
3754 def SULD_1D_ARRAY_V2I16_TRAP
3755   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3756               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3757               "suld.b.a1d.v2.b16.trap \\{$r, $g\\}, [$s, \\{$l, $x\\}];",
3758               []>;
3759 def SULD_1D_ARRAY_V2I32_TRAP
3760   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
3761               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3762               "suld.b.a1d.v2.b32.trap \\{$r, $g\\}, [$s, \\{$l, $x\\}];",
3763               []>;
3764 def SULD_1D_ARRAY_V2I64_TRAP
3765   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
3766               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3767               "suld.b.a1d.v2.b64.trap \\{$r, $g\\}, [$s, \\{$l, $x\\}];",
3768               []>;
3769
3770 def SULD_2D_V2I8_TRAP
3771   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3772               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3773               "suld.b.2d.v2.b8.trap \\{$r, $g\\}, [$s, \\{$x, $y\\}];",
3774               []>;
3775 def SULD_2D_V2I16_TRAP
3776   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3777               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3778               "suld.b.2d.v2.b16.trap \\{$r, $g\\}, [$s, \\{$x, $y\\}];",
3779               []>;
3780 def SULD_2D_V2I32_TRAP
3781   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
3782               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3783               "suld.b.2d.v2.b32.trap \\{$r, $g\\}, [$s, \\{$x, $y\\}];",
3784               []>;
3785 def SULD_2D_V2I64_TRAP
3786   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
3787               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3788               "suld.b.2d.v2.b64.trap \\{$r, $g\\}, [$s, \\{$x, $y\\}];",
3789               []>;
3790
3791 def SULD_2D_ARRAY_V2I8_TRAP
3792   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3793               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3794               "suld.b.a2d.v2.b8.trap \\{$r, $g\\}, "
3795               "[$s, \\{$l, $x, $y, $y\\}];",
3796               []>;
3797 def SULD_2D_ARRAY_V2I16_TRAP
3798   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3799               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3800               "suld.b.a2d.v2.b16.trap \\{$r, $g\\}, "
3801               "[$s, \\{$l, $x, $y, $y\\}];",
3802               []>;
3803 def SULD_2D_ARRAY_V2I32_TRAP
3804   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
3805               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3806               "suld.b.a2d.v2.b32.trap \\{$r, $g\\}, "
3807               "[$s, \\{$l, $x, $y, $y\\}];",
3808               []>;
3809 def SULD_2D_ARRAY_V2I64_TRAP
3810   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
3811               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3812               "suld.b.a2d.v2.b64.trap \\{$r, $g\\}, "
3813               "[$s, \\{$l, $x, $y, $y\\}];",
3814               []>;
3815
3816 def SULD_3D_V2I8_TRAP
3817   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3818               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3819               "suld.b.3d.v2.b8.trap \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",
3820               []>;
3821 def SULD_3D_V2I16_TRAP
3822   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3823               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3824               "suld.b.3d.v2.b16.trap \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",
3825               []>;
3826 def SULD_3D_V2I32_TRAP
3827   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
3828               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3829               "suld.b.3d.v2.b32.trap \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",
3830               []>;
3831 def SULD_3D_V2I64_TRAP
3832   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
3833               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3834               "suld.b.3d.v2.b64.trap \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",
3835               []>;
3836 }
3837
3838 let IsSuld = 3 in {
3839 def SULD_1D_V4I8_TRAP
3840   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3841               (ins Int64Regs:$s, Int32Regs:$x),
3842               "suld.b.1d.v4.b8.trap \\{$r, $g, $b, $a\\}, [$s, \\{$x\\}];",
3843               []>;
3844 def SULD_1D_V4I16_TRAP
3845   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3846               (ins Int64Regs:$s, Int32Regs:$x),
3847               "suld.b.1d.v4.b16.trap \\{$r, $g, $b, $a\\}, [$s, \\{$x\\}];",
3848               []>;
3849 def SULD_1D_V4I32_TRAP
3850   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
3851               (ins Int64Regs:$s, Int32Regs:$x),
3852               "suld.b.1d.v4.b32.trap \\{$r, $g, $b, $a\\}, [$s, \\{$x\\}];",
3853               []>;
3854
3855 def SULD_1D_ARRAY_V4I8_TRAP
3856   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3857               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3858               "suld.b.a1d.v4.b8.trap \\{$r, $g, $b, $a\\}, "
3859               "[$s, \\{$l, $x\\}];",
3860               []>;
3861 def SULD_1D_ARRAY_V4I16_TRAP
3862   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3863               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3864               "suld.b.a1d.v4.b16.trap \\{$r, $g, $b, $a\\}, "
3865               "[$s, \\{$l, $x\\}];",
3866               []>;
3867 def SULD_1D_ARRAY_V4I32_TRAP
3868   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
3869               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3870               "suld.b.a1d.v4.b32.trap \\{$r, $g, $b, $a\\}, "
3871               "[$s, \\{$l, $x\\}];",
3872               []>;
3873
3874 def SULD_2D_V4I8_TRAP
3875   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3876               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3877               "suld.b.2d.v4.b8.trap \\{$r, $g, $b, $a\\}, [$s, \\{$x, $y\\}];",
3878               []>;
3879 def SULD_2D_V4I16_TRAP
3880   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3881               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3882               "suld.b.2d.v4.b16.trap \\{$r, $g, $b, $a\\}, [$s, \\{$x, $y\\}];",
3883               []>;
3884 def SULD_2D_V4I32_TRAP
3885   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
3886               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3887               "suld.b.2d.v4.b32.trap \\{$r, $g, $b, $a\\}, [$s, \\{$x, $y\\}];",
3888               []>;
3889
3890 def SULD_2D_ARRAY_V4I8_TRAP
3891   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3892               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3893               "suld.b.a2d.v4.b8.trap \\{$r, $g, $b, $a\\}, "
3894               "[$s, \\{$l, $x, $y, $y\\}];",
3895               []>;
3896 def SULD_2D_ARRAY_V4I16_TRAP
3897   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3898               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3899               "suld.b.a2d.v4.b16.trap \\{$r, $g, $b, $a\\}, "
3900               "[$s, \\{$l, $x, $y, $y\\}];",
3901               []>;
3902 def SULD_2D_ARRAY_V4I32_TRAP
3903   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
3904               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3905               "suld.b.a2d.v4.b32.trap \\{$r, $g, $b, $a\\}, "
3906               "[$s, \\{$l, $x, $y, $y\\}];",
3907               []>;
3908
3909
3910 def SULD_3D_V4I8_TRAP
3911   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3912               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3913               "suld.b.3d.v4.b8.trap \\{$r, $g, $b, $a\\}, "
3914               "[$s, \\{$x, $y, $z, $z\\}];",
3915               []>;
3916 def SULD_3D_V4I16_TRAP
3917   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3918               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3919               "suld.b.3d.v4.b16.trap \\{$r, $g, $b, $a\\}, "
3920               "[$s, \\{$x, $y, $z, $z\\}];",
3921               []>;
3922 def SULD_3D_V4I32_TRAP
3923   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
3924               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3925               "suld.b.3d.v4.b32.trap \\{$r, $g, $b, $a\\}, "
3926               "[$s, \\{$x, $y, $z, $z\\}];",
3927               []>;
3928 }
3929
3930 // .zero variant
3931 let IsSuld = 1 in {
3932 def SULD_1D_I8_ZERO
3933   : NVPTXInst<(outs Int16Regs:$r),
3934               (ins Int64Regs:$s, Int32Regs:$x),
3935               "suld.b.1d.b8.zero \\{$r\\}, [$s, \\{$x\\}];",
3936               []>;
3937 def SULD_1D_I16_ZERO
3938   : NVPTXInst<(outs Int16Regs:$r),
3939               (ins Int64Regs:$s, Int32Regs:$x),
3940               "suld.b.1d.b16.zero \\{$r\\}, [$s, \\{$x\\}];",
3941               []>;
3942 def SULD_1D_I32_ZERO
3943   : NVPTXInst<(outs Int32Regs:$r),
3944               (ins Int64Regs:$s, Int32Regs:$x),
3945               "suld.b.1d.b32.zero \\{$r\\}, [$s, \\{$x\\}];",
3946               []>;
3947 def SULD_1D_I64_ZERO
3948   : NVPTXInst<(outs Int64Regs:$r),
3949               (ins Int64Regs:$s, Int32Regs:$x),
3950               "suld.b.1d.b64.zero \\{$r\\}, [$s, \\{$x\\}];",
3951               []>;
3952
3953 def SULD_1D_ARRAY_I8_ZERO
3954   : NVPTXInst<(outs Int16Regs:$r),
3955               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3956               "suld.b.a1d.b8.zero \\{$r\\}, [$s, \\{$l, $x\\}];",
3957               []>;
3958 def SULD_1D_ARRAY_I16_ZERO
3959   : NVPTXInst<(outs Int16Regs:$r),
3960               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3961               "suld.b.a1d.b16.zero \\{$r\\}, [$s, \\{$l, $x\\}];",
3962               []>;
3963 def SULD_1D_ARRAY_I32_ZERO
3964   : NVPTXInst<(outs Int32Regs:$r),
3965               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3966               "suld.b.a1d.b32.zero \\{$r\\}, [$s, \\{$l, $x\\}];",
3967               []>;
3968 def SULD_1D_ARRAY_I64_ZERO
3969   : NVPTXInst<(outs Int64Regs:$r),
3970               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3971               "suld.b.a1d.b64.zero \\{$r\\}, [$s, \\{$l, $x\\}];",
3972               []>;
3973
3974 def SULD_2D_I8_ZERO
3975   : NVPTXInst<(outs Int16Regs:$r),
3976               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3977               "suld.b.2d.b8.zero \\{$r\\}, [$s, \\{$x, $y\\}];",
3978               []>;
3979 def SULD_2D_I16_ZERO
3980   : NVPTXInst<(outs Int16Regs:$r),
3981               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3982               "suld.b.2d.b16.zero \\{$r\\}, [$s, \\{$x, $y\\}];",
3983               []>;
3984 def SULD_2D_I32_ZERO
3985   : NVPTXInst<(outs Int32Regs:$r),
3986               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3987               "suld.b.2d.b32.zero \\{$r\\}, [$s, \\{$x, $y\\}];",
3988               []>;
3989 def SULD_2D_I64_ZERO
3990   : NVPTXInst<(outs Int64Regs:$r),
3991               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3992               "suld.b.2d.b64.zero \\{$r\\}, [$s, \\{$x, $y\\}];",
3993               []>;
3994
3995 def SULD_2D_ARRAY_I8_ZERO
3996   : NVPTXInst<(outs Int16Regs:$r),
3997               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3998               "suld.b.a2d.b8.zero \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",
3999               []>;
4000 def SULD_2D_ARRAY_I16_ZERO
4001   : NVPTXInst<(outs Int16Regs:$r),
4002               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4003               "suld.b.a2d.b16.zero \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",
4004               []>;
4005 def SULD_2D_ARRAY_I32_ZERO
4006   : NVPTXInst<(outs Int32Regs:$r),
4007               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4008               "suld.b.a2d.b32.zero \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",
4009               []>;
4010 def SULD_2D_ARRAY_I64_ZERO
4011   : NVPTXInst<(outs Int64Regs:$r),
4012               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4013               "suld.b.a2d.b64.zero \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",
4014               []>;
4015
4016 def SULD_3D_I8_ZERO
4017   : NVPTXInst<(outs Int16Regs:$r),
4018               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4019               "suld.b.3d.b8.zero \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",
4020               []>;
4021 def SULD_3D_I16_ZERO
4022   : NVPTXInst<(outs Int16Regs:$r),
4023               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4024               "suld.b.3d.b16.zero \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",
4025               []>;
4026 def SULD_3D_I32_ZERO
4027   : NVPTXInst<(outs Int32Regs:$r),
4028               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4029               "suld.b.3d.b32.zero \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",
4030               []>;
4031 def SULD_3D_I64_ZERO
4032   : NVPTXInst<(outs Int64Regs:$r),
4033               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4034               "suld.b.3d.b64.zero \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",
4035               []>;
4036 }
4037
4038 let IsSuld = 2 in {
4039 def SULD_1D_V2I8_ZERO
4040   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4041               (ins Int64Regs:$s, Int32Regs:$x),
4042               "suld.b.1d.v2.b8.zero \\{$r, $g\\}, [$s, \\{$x\\}];",
4043               []>;
4044 def SULD_1D_V2I16_ZERO
4045   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4046               (ins Int64Regs:$s, Int32Regs:$x),
4047               "suld.b.1d.v2.b16.zero \\{$r, $g\\}, [$s, \\{$x\\}];",
4048               []>;
4049 def SULD_1D_V2I32_ZERO
4050   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
4051               (ins Int64Regs:$s, Int32Regs:$x),
4052               "suld.b.1d.v2.b32.zero \\{$r, $g\\}, [$s, \\{$x\\}];",
4053               []>;
4054 def SULD_1D_V2I64_ZERO
4055   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
4056               (ins Int64Regs:$s, Int32Regs:$x),
4057               "suld.b.1d.v2.b64.zero \\{$r, $g\\}, [$s, \\{$x\\}];",
4058               []>;
4059
4060 def SULD_1D_ARRAY_V2I8_ZERO
4061   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4062               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4063               "suld.b.a1d.v2.b8.zero \\{$r, $g\\}, [$s, \\{$l, $x\\}];",
4064               []>;
4065 def SULD_1D_ARRAY_V2I16_ZERO
4066   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4067               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4068               "suld.b.a1d.v2.b16.zero \\{$r, $g\\}, [$s, \\{$l, $x\\}];",
4069               []>;
4070 def SULD_1D_ARRAY_V2I32_ZERO
4071   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
4072               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4073               "suld.b.a1d.v2.b32.zero \\{$r, $g\\}, [$s, \\{$l, $x\\}];",
4074               []>;
4075 def SULD_1D_ARRAY_V2I64_ZERO
4076   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
4077               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4078               "suld.b.a1d.v2.b64.zero \\{$r, $g\\}, [$s, \\{$l, $x\\}];",
4079               []>;
4080
4081 def SULD_2D_V2I8_ZERO
4082   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4083               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4084               "suld.b.2d.v2.b8.zero \\{$r, $g\\}, [$s, \\{$x, $y\\}];",
4085               []>;
4086 def SULD_2D_V2I16_ZERO
4087   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4088               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4089               "suld.b.2d.v2.b16.zero \\{$r, $g\\}, [$s, \\{$x, $y\\}];",
4090               []>;
4091 def SULD_2D_V2I32_ZERO
4092   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
4093               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4094               "suld.b.2d.v2.b32.zero \\{$r, $g\\}, [$s, \\{$x, $y\\}];",
4095               []>;
4096 def SULD_2D_V2I64_ZERO
4097   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
4098               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4099               "suld.b.2d.v2.b64.zero \\{$r, $g\\}, [$s, \\{$x, $y\\}];",
4100               []>;
4101
4102 def SULD_2D_ARRAY_V2I8_ZERO
4103   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4104               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4105               "suld.b.a2d.v2.b8.zero \\{$r, $g\\}, "
4106               "[$s, \\{$l, $x, $y, $y\\}];",
4107               []>;
4108 def SULD_2D_ARRAY_V2I16_ZERO
4109   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4110               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4111               "suld.b.a2d.v2.b16.zero \\{$r, $g\\}, "
4112               "[$s, \\{$l, $x, $y, $y\\}];",
4113               []>;
4114 def SULD_2D_ARRAY_V2I32_ZERO
4115   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
4116               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4117               "suld.b.a2d.v2.b32.zero \\{$r, $g\\}, "
4118               "[$s, \\{$l, $x, $y, $y\\}];",
4119               []>;
4120 def SULD_2D_ARRAY_V2I64_ZERO
4121   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
4122               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4123               "suld.b.a2d.v2.b64.zero \\{$r, $g\\}, "
4124               "[$s, \\{$l, $x, $y, $y\\}];",
4125               []>;
4126
4127 def SULD_3D_V2I8_ZERO
4128   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4129               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4130               "suld.b.3d.v2.b8.zero \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",
4131               []>;
4132 def SULD_3D_V2I16_ZERO
4133   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4134               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4135               "suld.b.3d.v2.b16.zero \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",
4136               []>;
4137 def SULD_3D_V2I32_ZERO
4138   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
4139               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4140               "suld.b.3d.v2.b32.zero \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",
4141               []>;
4142 def SULD_3D_V2I64_ZERO
4143   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
4144               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4145               "suld.b.3d.v2.b64.zero \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",
4146               []>;
4147 }
4148
4149 let IsSuld = 3 in {
4150 def SULD_1D_V4I8_ZERO
4151   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4152               (ins Int64Regs:$s, Int32Regs:$x),
4153               "suld.b.1d.v4.b8.zero \\{$r, $g, $b, $a\\}, [$s, \\{$x\\}];",
4154               []>;
4155 def SULD_1D_V4I16_ZERO
4156   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4157               (ins Int64Regs:$s, Int32Regs:$x),
4158               "suld.b.1d.v4.b16.zero \\{$r, $g, $b, $a\\}, [$s, \\{$x\\}];",
4159               []>;
4160 def SULD_1D_V4I32_ZERO
4161   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4162               (ins Int64Regs:$s, Int32Regs:$x),
4163               "suld.b.1d.v4.b32.zero \\{$r, $g, $b, $a\\}, [$s, \\{$x\\}];",
4164               []>;
4165
4166 def SULD_1D_ARRAY_V4I8_ZERO
4167   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4168               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4169               "suld.b.a1d.v4.b8.zero \\{$r, $g, $b, $a\\}, "
4170               "[$s, \\{$l, $x\\}];",
4171               []>;
4172 def SULD_1D_ARRAY_V4I16_ZERO
4173   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4174               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4175               "suld.b.a1d.v4.b16.zero \\{$r, $g, $b, $a\\}, "
4176               "[$s, \\{$l, $x\\}];",
4177               []>;
4178 def SULD_1D_ARRAY_V4I32_ZERO
4179   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4180               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4181               "suld.b.a1d.v4.b32.zero \\{$r, $g, $b, $a\\}, "
4182               "[$s, \\{$l, $x\\}];",
4183               []>;
4184
4185 def SULD_2D_V4I8_ZERO
4186   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4187               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4188               "suld.b.2d.v4.b8.zero \\{$r, $g, $b, $a\\}, [$s, \\{$x, $y\\}];",
4189               []>;
4190 def SULD_2D_V4I16_ZERO
4191   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4192               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4193               "suld.b.2d.v4.b16.zero \\{$r, $g, $b, $a\\}, [$s, \\{$x, $y\\}];",
4194               []>;
4195 def SULD_2D_V4I32_ZERO
4196   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4197               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4198               "suld.b.2d.v4.b32.zero \\{$r, $g, $b, $a\\}, [$s, \\{$x, $y\\}];",
4199               []>;
4200
4201 def SULD_2D_ARRAY_V4I8_ZERO
4202   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4203               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4204               "suld.b.a2d.v4.b8.zero \\{$r, $g, $b, $a\\}, "
4205               "[$s, \\{$l, $x, $y, $y\\}];",
4206               []>;
4207 def SULD_2D_ARRAY_V4I16_ZERO
4208   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4209               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4210               "suld.b.a2d.v4.b16.zero \\{$r, $g, $b, $a\\}, "
4211               "[$s, \\{$l, $x, $y, $y\\}];",
4212               []>;
4213 def SULD_2D_ARRAY_V4I32_ZERO
4214   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4215               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4216               "suld.b.a2d.v4.b32.zero \\{$r, $g, $b, $a\\}, "
4217               "[$s, \\{$l, $x, $y, $y\\}];",
4218               []>;
4219
4220
4221 def SULD_3D_V4I8_ZERO
4222   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4223               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4224               "suld.b.3d.v4.b8.zero \\{$r, $g, $b, $a\\}, "
4225               "[$s, \\{$x, $y, $z, $z\\}];",
4226               []>;
4227 def SULD_3D_V4I16_ZERO
4228   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4229               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4230               "suld.b.3d.v4.b16.zero \\{$r, $g, $b, $a\\}, "
4231               "[$s, \\{$x, $y, $z, $z\\}];",
4232               []>;
4233 def SULD_3D_V4I32_ZERO
4234   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4235               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4236               "suld.b.3d.v4.b32.zero \\{$r, $g, $b, $a\\}, "
4237               "[$s, \\{$x, $y, $z, $z\\}];",
4238               []>;
4239 }
4240
4241 //-----------------------------------
4242 // Texture Query Intrinsics
4243 //-----------------------------------
4244 def TXQ_CHANNEL_ORDER
4245   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4246               "txq.channel_order.b32 \t$d, [$a];",
4247               []>;
4248 def TXQ_CHANNEL_DATA_TYPE
4249   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4250               "txq.channel_data_type.b32 \t$d, [$a];",
4251               []>;
4252 def TXQ_WIDTH
4253   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4254               "txq.width.b32 \t$d, [$a];",
4255               []>;
4256 def TXQ_HEIGHT
4257   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4258               "txq.height.b32 \t$d, [$a];",
4259               []>;
4260 def TXQ_DEPTH
4261   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4262               "txq.depth.b32 \t$d, [$a];",
4263               []>;
4264 def TXQ_ARRAY_SIZE
4265   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4266               "txq.array_size.b32 \t$d, [$a];",
4267               []>;
4268 def TXQ_NUM_SAMPLES
4269   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4270               "txq.num_samples.b32 \t$d, [$a];",
4271               []>;
4272 def TXQ_NUM_MIPMAP_LEVELS
4273   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4274               "txq.num_mipmap_levels.b32 \t$d, [$a];",
4275               []>;
4276
4277 def : Pat<(int_nvvm_txq_channel_order Int64Regs:$a),
4278           (TXQ_CHANNEL_ORDER Int64Regs:$a)>;
4279 def : Pat<(int_nvvm_txq_channel_data_type Int64Regs:$a),
4280           (TXQ_CHANNEL_DATA_TYPE Int64Regs:$a)>;
4281 def : Pat<(int_nvvm_txq_width Int64Regs:$a),
4282           (TXQ_WIDTH Int64Regs:$a)>;
4283 def : Pat<(int_nvvm_txq_height Int64Regs:$a),
4284           (TXQ_HEIGHT Int64Regs:$a)>;
4285 def : Pat<(int_nvvm_txq_depth Int64Regs:$a),
4286           (TXQ_DEPTH Int64Regs:$a)>;
4287 def : Pat<(int_nvvm_txq_array_size Int64Regs:$a),
4288           (TXQ_ARRAY_SIZE Int64Regs:$a)>;
4289 def : Pat<(int_nvvm_txq_num_samples Int64Regs:$a),
4290           (TXQ_NUM_SAMPLES Int64Regs:$a)>;
4291 def : Pat<(int_nvvm_txq_num_mipmap_levels Int64Regs:$a),
4292           (TXQ_NUM_MIPMAP_LEVELS Int64Regs:$a)>;
4293
4294
4295 //-----------------------------------
4296 // Surface Query Intrinsics
4297 //-----------------------------------
4298 def SUQ_CHANNEL_ORDER
4299   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4300               "suq.channel_order.b32 \t$d, [$a];",
4301               []>;
4302 def SUQ_CHANNEL_DATA_TYPE
4303   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4304               "suq.channel_data_type.b32 \t$d, [$a];",
4305               []>;
4306 def SUQ_WIDTH
4307   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4308               "suq.width.b32 \t$d, [$a];",
4309               []>;
4310 def SUQ_HEIGHT
4311   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4312               "suq.height.b32 \t$d, [$a];",
4313               []>;
4314 def SUQ_DEPTH
4315   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4316               "suq.depth.b32 \t$d, [$a];",
4317               []>;
4318 def SUQ_ARRAY_SIZE
4319   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4320               "suq.array_size.b32 \t$d, [$a];",
4321               []>;
4322
4323 def : Pat<(int_nvvm_suq_channel_order Int64Regs:$a),
4324           (SUQ_CHANNEL_ORDER Int64Regs:$a)>;
4325 def : Pat<(int_nvvm_suq_channel_data_type Int64Regs:$a),
4326           (SUQ_CHANNEL_DATA_TYPE Int64Regs:$a)>;
4327 def : Pat<(int_nvvm_suq_width Int64Regs:$a),
4328           (SUQ_WIDTH Int64Regs:$a)>;
4329 def : Pat<(int_nvvm_suq_height Int64Regs:$a),
4330           (SUQ_HEIGHT Int64Regs:$a)>;
4331 def : Pat<(int_nvvm_suq_depth Int64Regs:$a),
4332           (SUQ_DEPTH Int64Regs:$a)>;
4333 def : Pat<(int_nvvm_suq_array_size Int64Regs:$a),
4334           (SUQ_ARRAY_SIZE Int64Regs:$a)>;
4335
4336
4337 //===- Handle Query -------------------------------------------------------===//
4338
4339 // TODO: These intrinsics are not yet finalized, pending PTX ISA design work
4340 def ISTYPEP_SAMPLER
4341   : NVPTXInst<(outs Int1Regs:$d), (ins Int64Regs:$a),
4342               "istypep.samplerref \t$d, $a;",
4343               [(set Int1Regs:$d, (int_nvvm_istypep_sampler Int64Regs:$a))]>;
4344 def ISTYPEP_SURFACE
4345   : NVPTXInst<(outs Int1Regs:$d), (ins Int64Regs:$a),
4346               "istypep.surfref \t$d, $a;",
4347               [(set Int1Regs:$d, (int_nvvm_istypep_surface Int64Regs:$a))]>;
4348 def ISTYPEP_TEXTURE
4349   : NVPTXInst<(outs Int1Regs:$d), (ins Int64Regs:$a),
4350               "istypep.texref \t$d, $a;",
4351               [(set Int1Regs:$d, (int_nvvm_istypep_texture Int64Regs:$a))]>;
4352
4353 //===- Surface Stores -----------------------------------------------------===//
4354
4355 let IsSust = 1 in {
4356 // Unformatted
4357 // .clamp variant
4358 def SUST_B_1D_B8_CLAMP
4359   : NVPTXInst<(outs),
4360               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
4361               "sust.b.1d.b8.clamp \t[$s, \\{$x\\}], \\{$r\\};",
4362               []>;
4363 def SUST_B_1D_B16_CLAMP
4364   : NVPTXInst<(outs),
4365               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
4366               "sust.b.1d.b16.clamp \t[$s, \\{$x\\}], \\{$r\\};",
4367               []>;
4368 def SUST_B_1D_B32_CLAMP
4369   : NVPTXInst<(outs),
4370               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$r),
4371               "sust.b.1d.b32.clamp \t[$s, \\{$x\\}], \\{$r\\};",
4372               []>;
4373 def SUST_B_1D_B64_CLAMP
4374   : NVPTXInst<(outs),
4375               (ins Int64Regs:$s, Int32Regs:$x, Int64Regs:$r),
4376               "sust.b.1d.b64.clamp \t[$s, \\{$x\\}], \\{$r\\};",
4377               []>;
4378 def SUST_B_1D_V2B8_CLAMP
4379   : NVPTXInst<(outs),
4380               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
4381               "sust.b.1d.v2.b8.clamp \t[$s, \\{$x\\}], \\{$r, $g\\};",
4382               []>;
4383 def SUST_B_1D_V2B16_CLAMP
4384   : NVPTXInst<(outs),
4385               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
4386               "sust.b.1d.v2.b16.clamp \t[$s, \\{$x\\}], \\{$r, $g\\};",
4387               []>;
4388 def SUST_B_1D_V2B32_CLAMP
4389   : NVPTXInst<(outs),
4390               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
4391               "sust.b.1d.v2.b32.clamp \t[$s, \\{$x\\}], \\{$r, $g\\};",
4392               []>;
4393 def SUST_B_1D_V2B64_CLAMP
4394   : NVPTXInst<(outs),
4395               (ins Int64Regs:$s, Int32Regs:$x, Int64Regs:$r, Int64Regs:$g),
4396               "sust.b.1d.v2.b64.clamp \t[$s, \\{$x\\}], \\{$r, $g\\};",
4397               []>;
4398 def SUST_B_1D_V4B8_CLAMP
4399   : NVPTXInst<(outs),
4400               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g,
4401                    Int16Regs:$b, Int16Regs:$a),
4402               "sust.b.1d.v4.b8.clamp \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
4403               []>;
4404 def SUST_B_1D_V4B16_CLAMP
4405   : NVPTXInst<(outs),
4406               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g,
4407                    Int16Regs:$b, Int16Regs:$a),
4408               "sust.b.1d.v4.b16.clamp \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
4409               []>;
4410 def SUST_B_1D_V4B32_CLAMP
4411   : NVPTXInst<(outs),
4412               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g,
4413                    Int32Regs:$b, Int32Regs:$a),
4414               "sust.b.1d.v4.b32.clamp \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
4415               []>;
4416
4417
4418 def SUST_B_1D_ARRAY_B8_CLAMP
4419   : NVPTXInst<(outs),
4420               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r),
4421               "sust.b.a1d.b8.clamp \t[$s, \\{$idx, $x\\}], \\{$r\\};",
4422               []>;
4423 def SUST_B_1D_ARRAY_B16_CLAMP
4424   : NVPTXInst<(outs),
4425               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r),
4426               "sust.b.a1d.b16.clamp \t[$s, \\{$idx, $x\\}], \\{$r\\};",
4427               []>;
4428 def SUST_B_1D_ARRAY_B32_CLAMP
4429   : NVPTXInst<(outs),
4430               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$r),
4431               "sust.b.a1d.b32.clamp \t[$s, \\{$idx, $x\\}], \\{$r\\};",
4432               []>;
4433 def SUST_B_1D_ARRAY_B64_CLAMP
4434   : NVPTXInst<(outs),
4435               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int64Regs:$r),
4436               "sust.b.a1d.b64.clamp \t[$s, \\{$idx, $x\\}], \\{$r\\};",
4437               []>;
4438 def SUST_B_1D_ARRAY_V2B8_CLAMP
4439   : NVPTXInst<(outs),
4440               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
4441                    Int16Regs:$g),
4442               "sust.b.a1d.v2.b8.clamp \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
4443               []>;
4444 def SUST_B_1D_ARRAY_V2B16_CLAMP
4445   : NVPTXInst<(outs),
4446               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
4447                    Int16Regs:$g),
4448               "sust.b.a1d.v2.b16.clamp \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
4449               []>;
4450 def SUST_B_1D_ARRAY_V2B32_CLAMP
4451   : NVPTXInst<(outs),
4452               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$r,
4453                    Int32Regs:$g),
4454               "sust.b.a1d.v2.b32.clamp \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
4455               []>;
4456 def SUST_B_1D_ARRAY_V2B64_CLAMP
4457   : NVPTXInst<(outs),
4458               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int64Regs:$r,
4459                    Int64Regs:$g),
4460               "sust.b.a1d.v2.b64.clamp \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
4461               []>;
4462 def SUST_B_1D_ARRAY_V4B8_CLAMP
4463   : NVPTXInst<(outs),
4464               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
4465                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4466               "sust.b.a1d.v4.b8.clamp \t[$s, \\{$idx, $x\\}], "
4467               "\\{$r, $g, $b, $a\\};",
4468               []>;
4469 def SUST_B_1D_ARRAY_V4B16_CLAMP
4470   : NVPTXInst<(outs),
4471               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
4472                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4473              "sust.b.a1d.v4.b16.clamp \t[$s, \\{$idx, $x\\}], "
4474              "\\{$r, $g, $b, $a\\};",
4475               []>;
4476 def SUST_B_1D_ARRAY_V4B32_CLAMP
4477   : NVPTXInst<(outs),
4478               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$r,
4479                    Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4480              "sust.b.a1d.v4.b32.clamp \t[$s, \\{$idx, $x\\}], "
4481              "\\{$r, $g, $b, $a\\};",
4482               []>;
4483
4484
4485 def SUST_B_2D_B8_CLAMP
4486   : NVPTXInst<(outs),
4487               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
4488               "sust.b.2d.b8.clamp \t[$s, \\{$x, $y\\}], \\{$r\\};",
4489               []>;
4490 def SUST_B_2D_B16_CLAMP
4491   : NVPTXInst<(outs),
4492               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
4493               "sust.b.2d.b16.clamp \t[$s, \\{$x, $y\\}], \\{$r\\};",
4494               []>;
4495 def SUST_B_2D_B32_CLAMP
4496   : NVPTXInst<(outs),
4497               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
4498               "sust.b.2d.b32.clamp \t[$s, \\{$x, $y\\}], \\{$r\\};",
4499               []>;
4500 def SUST_B_2D_B64_CLAMP
4501   : NVPTXInst<(outs),
4502               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r),
4503               "sust.b.2d.b64.clamp \t[$s, \\{$x, $y\\}], \\{$r\\};",
4504               []>;
4505 def SUST_B_2D_V2B8_CLAMP
4506   : NVPTXInst<(outs),
4507               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
4508                    Int16Regs:$g),
4509               "sust.b.2d.v2.b8.clamp \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
4510               []>;
4511 def SUST_B_2D_V2B16_CLAMP
4512   : NVPTXInst<(outs),
4513               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
4514                    Int16Regs:$g),
4515               "sust.b.2d.v2.b16.clamp \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
4516               []>;
4517 def SUST_B_2D_V2B32_CLAMP
4518   : NVPTXInst<(outs),
4519               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
4520                    Int32Regs:$g),
4521               "sust.b.2d.v2.b32.clamp \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
4522               []>;
4523 def SUST_B_2D_V2B64_CLAMP
4524   : NVPTXInst<(outs),
4525               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r,
4526                    Int64Regs:$g),
4527               "sust.b.2d.v2.b64.clamp \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
4528               []>;
4529 def SUST_B_2D_V4B8_CLAMP
4530   : NVPTXInst<(outs),
4531               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
4532                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4533               "sust.b.2d.v4.b8.clamp \t[$s, \\{$x, $y\\}], "
4534               "\\{$r, $g, $b, $a\\};",
4535               []>;
4536 def SUST_B_2D_V4B16_CLAMP
4537   : NVPTXInst<(outs),
4538               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
4539                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4540              "sust.b.2d.v4.b16.clamp \t[$s, \\{$x, $y\\}], "
4541              "\\{$r, $g, $b, $a\\};",
4542               []>;
4543 def SUST_B_2D_V4B32_CLAMP
4544   : NVPTXInst<(outs),
4545               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
4546                    Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4547              "sust.b.2d.v4.b32.clamp \t[$s, \\{$x, $y\\}], "
4548              "\\{$r, $g, $b, $a\\};",
4549               []>;
4550
4551
4552 def SUST_B_2D_ARRAY_B8_CLAMP
4553   : NVPTXInst<(outs),
4554               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4555                    Int16Regs:$r),
4556               "sust.b.a2d.b8.clamp \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
4557               []>;
4558 def SUST_B_2D_ARRAY_B16_CLAMP
4559   : NVPTXInst<(outs),
4560               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4561                    Int16Regs:$r),
4562               "sust.b.a2d.b16.clamp \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
4563               []>;
4564 def SUST_B_2D_ARRAY_B32_CLAMP
4565   : NVPTXInst<(outs),
4566               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4567                    Int32Regs:$r),
4568               "sust.b.a2d.b32.clamp \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
4569               []>;
4570 def SUST_B_2D_ARRAY_B64_CLAMP
4571   : NVPTXInst<(outs),
4572               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4573                    Int64Regs:$r),
4574               "sust.b.a2d.b64.clamp \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
4575               []>;
4576 def SUST_B_2D_ARRAY_V2B8_CLAMP
4577   : NVPTXInst<(outs),
4578               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4579                    Int16Regs:$r, Int16Regs:$g),
4580               "sust.b.a2d.v2.b8.clamp \t[$s, \\{$idx, $x, $y, $y\\}], "
4581               "\\{$r, $g\\};",
4582               []>;
4583 def SUST_B_2D_ARRAY_V2B16_CLAMP
4584   : NVPTXInst<(outs),
4585               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4586                    Int16Regs:$r, Int16Regs:$g),
4587              "sust.b.a2d.v2.b16.clamp \t[$s, \\{$idx, $x, $y, $y\\}], "
4588              "\\{$r, $g\\};",
4589               []>;
4590 def SUST_B_2D_ARRAY_V2B32_CLAMP
4591   : NVPTXInst<(outs),
4592               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4593                    Int32Regs:$r, Int32Regs:$g),
4594              "sust.b.a2d.v2.b32.clamp \t[$s, \\{$idx, $x, $y, $y\\}], "
4595              "\\{$r, $g\\};",
4596               []>;
4597 def SUST_B_2D_ARRAY_V2B64_CLAMP
4598   : NVPTXInst<(outs),
4599               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4600                    Int64Regs:$r, Int64Regs:$g),
4601              "sust.b.a2d.v2.b64.clamp \t[$s, \\{$idx, $x, $y, $y\\}], "
4602              "\\{$r, $g\\};",
4603               []>;
4604 def SUST_B_2D_ARRAY_V4B8_CLAMP
4605   : NVPTXInst<(outs),
4606               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4607                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4608       "sust.b.a2d.v4.b8.clamp \t[$s, \\{$idx, $x, $y, $y\\}], "
4609       "\\{$r, $g, $b, $a\\};",
4610               []>;
4611 def SUST_B_2D_ARRAY_V4B16_CLAMP
4612   : NVPTXInst<(outs),
4613               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4614                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4615      "sust.b.a2d.v4.b16.clamp \t[$s, \\{$idx, $x, $y, $y\\}], "
4616      "\\{$r, $g, $b, $a\\};",
4617               []>;
4618 def SUST_B_2D_ARRAY_V4B32_CLAMP
4619   : NVPTXInst<(outs),
4620               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4621                    Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4622      "sust.b.a2d.v4.b32.clamp \t[$s, \\{$idx, $x, $y, $y\\}], "
4623      "\\{$r, $g, $b, $a\\};",
4624               []>;
4625
4626
4627 def SUST_B_3D_B8_CLAMP
4628   : NVPTXInst<(outs),
4629               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
4630                    Int16Regs:$r),
4631               "sust.b.3d.b8.clamp \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
4632               []>;
4633 def SUST_B_3D_B16_CLAMP
4634   : NVPTXInst<(outs),
4635               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
4636                    Int16Regs:$r),
4637               "sust.b.3d.b16.clamp \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
4638               []>;
4639 def SUST_B_3D_B32_CLAMP
4640   : NVPTXInst<(outs),
4641               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
4642                    Int32Regs:$r),
4643               "sust.b.3d.b32.clamp \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
4644               []>;
4645 def SUST_B_3D_B64_CLAMP
4646   : NVPTXInst<(outs),
4647               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
4648                    Int64Regs:$r),
4649               "sust.b.3d.b64.clamp \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
4650               []>;
4651 def SUST_B_3D_V2B8_CLAMP
4652   : NVPTXInst<(outs),
4653               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
4654                    Int16Regs:$r, Int16Regs:$g),
4655               "sust.b.3d.v2.b8.clamp \t[$s, \\{$x, $y, $z, $z\\}], "
4656               "\\{$r, $g\\};",
4657               []>;
4658 def SUST_B_3D_V2B16_CLAMP
4659   : NVPTXInst<(outs),
4660               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
4661                    Int16Regs:$r, Int16Regs:$g),
4662               "sust.b.3d.v2.b16.clamp \t[$s, \\{$x, $y, $z, $z\\}], "
4663               "\\{$r, $g\\};",
4664               []>;
4665 def SUST_B_3D_V2B32_CLAMP
4666   : NVPTXInst<(outs),
4667               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
4668                    Int32Regs:$r, Int32Regs:$g),
4669               "sust.b.3d.v2.b32.clamp \t[$s, \\{$x, $y, $z, $z\\}], "
4670               "\\{$r, $g\\};",
4671               []>;
4672 def SUST_B_3D_V2B64_CLAMP
4673   : NVPTXInst<(outs),
4674               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
4675                    Int64Regs:$r, Int64Regs:$g),
4676               "sust.b.3d.v2.b64.clamp \t[$s, \\{$x, $y, $z, $z\\}], "
4677               "\\{$r, $g\\};",
4678               []>;
4679 def SUST_B_3D_V4B8_CLAMP
4680   : NVPTXInst<(outs),
4681               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
4682                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4683          "sust.b.3d.v4.b8.clamp \t[$s, \\{$x, $y, $z, $z\\}], "
4684          "\\{$r, $g, $b, $a\\};",
4685               []>;
4686 def SUST_B_3D_V4B16_CLAMP
4687   : NVPTXInst<(outs),
4688               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
4689                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4690         "sust.b.3d.v4.b16.clamp \t[$s, \\{$x, $y, $z, $z\\}], "
4691         "\\{$r, $g, $b, $a\\};",
4692               []>;
4693 def SUST_B_3D_V4B32_CLAMP
4694   : NVPTXInst<(outs),
4695               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
4696                    Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4697         "sust.b.3d.v4.b32.clamp \t[$s, \\{$x, $y, $z, $z\\}], "
4698         "\\{$r, $g, $b, $a\\};",
4699               []>;
4700
4701
4702 // .trap variant
4703 def SUST_B_1D_B8_TRAP
4704   : NVPTXInst<(outs),
4705               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
4706               "sust.b.1d.b8.trap \t[$s, \\{$x\\}], \\{$r\\};",
4707               []>;
4708 def SUST_B_1D_B16_TRAP
4709   : NVPTXInst<(outs),
4710               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
4711               "sust.b.1d.b16.trap \t[$s, \\{$x\\}], \\{$r\\};",
4712               []>;
4713 def SUST_B_1D_B32_TRAP
4714   : NVPTXInst<(outs),
4715               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$r),
4716               "sust.b.1d.b32.trap \t[$s, \\{$x\\}], \\{$r\\};",
4717               []>;
4718 def SUST_B_1D_B64_TRAP
4719   : NVPTXInst<(outs),
4720               (ins Int64Regs:$s, Int32Regs:$x, Int64Regs:$r),
4721               "sust.b.1d.b64.trap \t[$s, \\{$x\\}], \\{$r\\};",
4722               []>;
4723 def SUST_B_1D_V2B8_TRAP
4724   : NVPTXInst<(outs),
4725               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
4726               "sust.b.1d.v2.b8.trap \t[$s, \\{$x\\}], \\{$r, $g\\};",
4727               []>;
4728 def SUST_B_1D_V2B16_TRAP
4729   : NVPTXInst<(outs),
4730               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
4731               "sust.b.1d.v2.b16.trap \t[$s, \\{$x\\}], \\{$r, $g\\};",
4732               []>;
4733 def SUST_B_1D_V2B32_TRAP
4734   : NVPTXInst<(outs),
4735               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
4736               "sust.b.1d.v2.b32.trap \t[$s, \\{$x\\}], \\{$r, $g\\};",
4737               []>;
4738 def SUST_B_1D_V2B64_TRAP
4739   : NVPTXInst<(outs),
4740               (ins Int64Regs:$s, Int32Regs:$x, Int64Regs:$r, Int64Regs:$g),
4741               "sust.b.1d.v2.b64.trap \t[$s, \\{$x\\}], \\{$r, $g\\};",
4742               []>;
4743 def SUST_B_1D_V4B8_TRAP
4744   : NVPTXInst<(outs),
4745               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g,
4746                    Int16Regs:$b, Int16Regs:$a),
4747               "sust.b.1d.v4.b8.trap \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
4748               []>;
4749 def SUST_B_1D_V4B16_TRAP
4750   : NVPTXInst<(outs),
4751               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g,
4752                    Int16Regs:$b, Int16Regs:$a),
4753               "sust.b.1d.v4.b16.trap \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
4754               []>;
4755 def SUST_B_1D_V4B32_TRAP
4756   : NVPTXInst<(outs),
4757               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g,
4758                    Int32Regs:$b, Int32Regs:$a),
4759               "sust.b.1d.v4.b32.trap \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
4760               []>;
4761
4762
4763 def SUST_B_1D_ARRAY_B8_TRAP
4764   : NVPTXInst<(outs),
4765               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r),
4766               "sust.b.a1d.b8.trap \t[$s, \\{$idx, $x\\}], \\{$r\\};",
4767               []>;
4768 def SUST_B_1D_ARRAY_B16_TRAP
4769   : NVPTXInst<(outs),
4770               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r),
4771               "sust.b.a1d.b16.trap \t[$s, \\{$idx, $x\\}], \\{$r\\};",
4772               []>;
4773 def SUST_B_1D_ARRAY_B32_TRAP
4774   : NVPTXInst<(outs),
4775               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$r),
4776               "sust.b.a1d.b32.trap \t[$s, \\{$idx, $x\\}], \\{$r\\};",
4777               []>;
4778 def SUST_B_1D_ARRAY_B64_TRAP
4779   : NVPTXInst<(outs),
4780               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int64Regs:$r),
4781               "sust.b.a1d.b64.trap \t[$s, \\{$idx, $x\\}], \\{$r\\};",
4782               []>;
4783 def SUST_B_1D_ARRAY_V2B8_TRAP
4784   : NVPTXInst<(outs),
4785               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
4786                    Int16Regs:$g),
4787               "sust.b.a1d.v2.b8.trap \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
4788               []>;
4789 def SUST_B_1D_ARRAY_V2B16_TRAP
4790   : NVPTXInst<(outs),
4791               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
4792                    Int16Regs:$g),
4793               "sust.b.a1d.v2.b16.trap \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
4794               []>;
4795 def SUST_B_1D_ARRAY_V2B32_TRAP
4796   : NVPTXInst<(outs),
4797               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$r,
4798                    Int32Regs:$g),
4799               "sust.b.a1d.v2.b32.trap \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
4800               []>;
4801 def SUST_B_1D_ARRAY_V2B64_TRAP
4802   : NVPTXInst<(outs),
4803               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int64Regs:$r,
4804                    Int64Regs:$g),
4805               "sust.b.a1d.v2.b64.trap \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
4806               []>;
4807 def SUST_B_1D_ARRAY_V4B8_TRAP
4808   : NVPTXInst<(outs),
4809               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
4810                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4811               "sust.b.a1d.v4.b8.trap \t[$s, \\{$idx, $x\\}], "
4812               "\\{$r, $g, $b, $a\\};",
4813               []>;
4814 def SUST_B_1D_ARRAY_V4B16_TRAP
4815   : NVPTXInst<(outs),
4816               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
4817                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4818              "sust.b.a1d.v4.b16.trap \t[$s, \\{$idx, $x\\}], "
4819              "\\{$r, $g, $b, $a\\};",
4820               []>;
4821 def SUST_B_1D_ARRAY_V4B32_TRAP
4822   : NVPTXInst<(outs),
4823               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$r,
4824                    Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4825              "sust.b.a1d.v4.b32.trap \t[$s, \\{$idx, $x\\}], "
4826              "\\{$r, $g, $b, $a\\};",
4827               []>;
4828
4829
4830 def SUST_B_2D_B8_TRAP
4831   : NVPTXInst<(outs),
4832               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
4833               "sust.b.2d.b8.trap \t[$s, \\{$x, $y\\}], \\{$r\\};",
4834               []>;
4835 def SUST_B_2D_B16_TRAP
4836   : NVPTXInst<(outs),
4837               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
4838               "sust.b.2d.b16.trap \t[$s, \\{$x, $y\\}], \\{$r\\};",
4839               []>;
4840 def SUST_B_2D_B32_TRAP
4841   : NVPTXInst<(outs),
4842               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
4843               "sust.b.2d.b32.trap \t[$s, \\{$x, $y\\}], \\{$r\\};",
4844               []>;
4845 def SUST_B_2D_B64_TRAP
4846   : NVPTXInst<(outs),
4847               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r),
4848               "sust.b.2d.b64.trap \t[$s, \\{$x, $y\\}], \\{$r\\};",
4849               []>;
4850 def SUST_B_2D_V2B8_TRAP
4851   : NVPTXInst<(outs),
4852               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
4853                    Int16Regs:$g),
4854               "sust.b.2d.v2.b8.trap \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
4855               []>;
4856 def SUST_B_2D_V2B16_TRAP
4857   : NVPTXInst<(outs),
4858               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
4859                    Int16Regs:$g),
4860               "sust.b.2d.v2.b16.trap \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
4861               []>;
4862 def SUST_B_2D_V2B32_TRAP
4863   : NVPTXInst<(outs),
4864               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
4865                    Int32Regs:$g),
4866               "sust.b.2d.v2.b32.trap \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
4867               []>;
4868 def SUST_B_2D_V2B64_TRAP
4869   : NVPTXInst<(outs),
4870               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r,
4871                    Int64Regs:$g),
4872               "sust.b.2d.v2.b64.trap \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
4873               []>;
4874 def SUST_B_2D_V4B8_TRAP
4875   : NVPTXInst<(outs),
4876               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
4877                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4878               "sust.b.2d.v4.b8.trap \t[$s, \\{$x, $y\\}], "
4879               "\\{$r, $g, $b, $a\\};",
4880               []>;
4881 def SUST_B_2D_V4B16_TRAP
4882   : NVPTXInst<(outs),
4883               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
4884                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4885              "sust.b.2d.v4.b16.trap \t[$s, \\{$x, $y\\}], "
4886              "\\{$r, $g, $b, $a\\};",
4887               []>;
4888 def SUST_B_2D_V4B32_TRAP
4889   : NVPTXInst<(outs),
4890               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
4891                    Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4892              "sust.b.2d.v4.b32.trap \t[$s, \\{$x, $y\\}], "
4893              "\\{$r, $g, $b, $a\\};",
4894               []>;
4895
4896
4897 def SUST_B_2D_ARRAY_B8_TRAP
4898   : NVPTXInst<(outs),
4899               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4900                    Int16Regs:$r),
4901               "sust.b.a2d.b8.trap \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
4902               []>;
4903 def SUST_B_2D_ARRAY_B16_TRAP
4904   : NVPTXInst<(outs),
4905               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4906                    Int16Regs:$r),
4907               "sust.b.a2d.b16.trap \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
4908               []>;
4909 def SUST_B_2D_ARRAY_B32_TRAP
4910   : NVPTXInst<(outs),
4911               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4912                    Int32Regs:$r),
4913               "sust.b.a2d.b32.trap \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
4914               []>;
4915 def SUST_B_2D_ARRAY_B64_TRAP
4916   : NVPTXInst<(outs),
4917               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4918                    Int64Regs:$r),
4919               "sust.b.a2d.b64.trap \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
4920               []>;
4921 def SUST_B_2D_ARRAY_V2B8_TRAP
4922   : NVPTXInst<(outs),
4923               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4924                    Int16Regs:$r, Int16Regs:$g),
4925               "sust.b.a2d.v2.b8.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
4926               "\\{$r, $g\\};",
4927               []>;
4928 def SUST_B_2D_ARRAY_V2B16_TRAP
4929   : NVPTXInst<(outs),
4930               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4931                    Int16Regs:$r, Int16Regs:$g),
4932              "sust.b.a2d.v2.b16.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
4933              "\\{$r, $g\\};",
4934               []>;
4935 def SUST_B_2D_ARRAY_V2B32_TRAP
4936   : NVPTXInst<(outs),
4937               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4938                    Int32Regs:$r, Int32Regs:$g),
4939              "sust.b.a2d.v2.b32.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
4940              "\\{$r, $g\\};",
4941               []>;
4942 def SUST_B_2D_ARRAY_V2B64_TRAP
4943   : NVPTXInst<(outs),
4944               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4945                    Int64Regs:$r, Int64Regs:$g),
4946              "sust.b.a2d.v2.b64.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
4947              "\\{$r, $g\\};",
4948               []>;
4949 def SUST_B_2D_ARRAY_V4B8_TRAP
4950   : NVPTXInst<(outs),
4951               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4952                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4953       "sust.b.a2d.v4.b8.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
4954       "\\{$r, $g, $b, $a\\};",
4955               []>;
4956 def SUST_B_2D_ARRAY_V4B16_TRAP
4957   : NVPTXInst<(outs),
4958               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4959                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4960      "sust.b.a2d.v4.b16.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
4961      "\\{$r, $g, $b, $a\\};",
4962               []>;
4963 def SUST_B_2D_ARRAY_V4B32_TRAP
4964   : NVPTXInst<(outs),
4965               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4966                    Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4967      "sust.b.a2d.v4.b32.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
4968      "\\{$r, $g, $b, $a\\};",
4969               []>;
4970
4971
4972 def SUST_B_3D_B8_TRAP
4973   : NVPTXInst<(outs),
4974               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
4975                    Int16Regs:$r),
4976               "sust.b.3d.b8.trap \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
4977               []>;
4978 def SUST_B_3D_B16_TRAP
4979   : NVPTXInst<(outs),
4980               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
4981                    Int16Regs:$r),
4982               "sust.b.3d.b16.trap \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
4983               []>;
4984 def SUST_B_3D_B32_TRAP
4985   : NVPTXInst<(outs),
4986               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
4987                    Int32Regs:$r),
4988               "sust.b.3d.b32.trap \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
4989               []>;
4990 def SUST_B_3D_B64_TRAP
4991   : NVPTXInst<(outs),
4992               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
4993                    Int64Regs:$r),
4994               "sust.b.3d.b64.trap \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
4995               []>;
4996 def SUST_B_3D_V2B8_TRAP
4997   : NVPTXInst<(outs),
4998               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
4999                    Int16Regs:$r, Int16Regs:$g),
5000               "sust.b.3d.v2.b8.trap \t[$s, \\{$x, $y, $z, $z\\}], "
5001               "\\{$r, $g\\};",
5002               []>;
5003 def SUST_B_3D_V2B16_TRAP
5004   : NVPTXInst<(outs),
5005               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5006                    Int16Regs:$r, Int16Regs:$g),
5007               "sust.b.3d.v2.b16.trap \t[$s, \\{$x, $y, $z, $z\\}], "
5008               "\\{$r, $g\\};",
5009               []>;
5010 def SUST_B_3D_V2B32_TRAP
5011   : NVPTXInst<(outs),
5012               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5013                    Int32Regs:$r, Int32Regs:$g),
5014               "sust.b.3d.v2.b32.trap \t[$s, \\{$x, $y, $z, $z\\}], "
5015               "\\{$r, $g\\};",
5016               []>;
5017 def SUST_B_3D_V2B64_TRAP
5018   : NVPTXInst<(outs),
5019               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5020                    Int64Regs:$r, Int64Regs:$g),
5021               "sust.b.3d.v2.b64.trap \t[$s, \\{$x, $y, $z, $z\\}], "
5022               "\\{$r, $g\\};",
5023               []>;
5024 def SUST_B_3D_V4B8_TRAP
5025   : NVPTXInst<(outs),
5026               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5027                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5028          "sust.b.3d.v4.b8.trap \t[$s, \\{$x, $y, $z, $z\\}], "
5029          "\\{$r, $g, $b, $a\\};",
5030               []>;
5031 def SUST_B_3D_V4B16_TRAP
5032   : NVPTXInst<(outs),
5033               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5034                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5035         "sust.b.3d.v4.b16.trap \t[$s, \\{$x, $y, $z, $z\\}], "
5036         "\\{$r, $g, $b, $a\\};",
5037               []>;
5038 def SUST_B_3D_V4B32_TRAP
5039   : NVPTXInst<(outs),
5040               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5041                    Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5042         "sust.b.3d.v4.b32.trap \t[$s, \\{$x, $y, $z, $z\\}], "
5043         "\\{$r, $g, $b, $a\\};",
5044               []>;
5045
5046
5047 // .zero variant
5048 def SUST_B_1D_B8_ZERO
5049   : NVPTXInst<(outs),
5050               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
5051               "sust.b.1d.b8.zero \t[$s, \\{$x\\}], \\{$r\\};",
5052               []>;
5053 def SUST_B_1D_B16_ZERO
5054   : NVPTXInst<(outs),
5055               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
5056               "sust.b.1d.b16.zero \t[$s, \\{$x\\}], \\{$r\\};",
5057               []>;
5058 def SUST_B_1D_B32_ZERO
5059   : NVPTXInst<(outs),
5060               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$r),
5061               "sust.b.1d.b32.zero \t[$s, \\{$x\\}], \\{$r\\};",
5062               []>;
5063 def SUST_B_1D_B64_ZERO
5064   : NVPTXInst<(outs),
5065               (ins Int64Regs:$s, Int32Regs:$x, Int64Regs:$r),
5066               "sust.b.1d.b64.zero \t[$s, \\{$x\\}], \\{$r\\};",
5067               []>;
5068 def SUST_B_1D_V2B8_ZERO
5069   : NVPTXInst<(outs),
5070               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
5071               "sust.b.1d.v2.b8.zero \t[$s, \\{$x\\}], \\{$r, $g\\};",
5072               []>;
5073 def SUST_B_1D_V2B16_ZERO
5074   : NVPTXInst<(outs),
5075               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
5076               "sust.b.1d.v2.b16.zero \t[$s, \\{$x\\}], \\{$r, $g\\};",
5077               []>;
5078 def SUST_B_1D_V2B32_ZERO
5079   : NVPTXInst<(outs),
5080               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
5081               "sust.b.1d.v2.b32.zero \t[$s, \\{$x\\}], \\{$r, $g\\};",
5082               []>;
5083 def SUST_B_1D_V2B64_ZERO
5084   : NVPTXInst<(outs),
5085               (ins Int64Regs:$s, Int32Regs:$x, Int64Regs:$r, Int64Regs:$g),
5086               "sust.b.1d.v2.b64.zero \t[$s, \\{$x\\}], \\{$r, $g\\};",
5087               []>;
5088 def SUST_B_1D_V4B8_ZERO
5089   : NVPTXInst<(outs),
5090               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g,
5091                    Int16Regs:$b, Int16Regs:$a),
5092               "sust.b.1d.v4.b8.zero \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
5093               []>;
5094 def SUST_B_1D_V4B16_ZERO
5095   : NVPTXInst<(outs),
5096               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g,
5097                    Int16Regs:$b, Int16Regs:$a),
5098               "sust.b.1d.v4.b16.zero \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
5099               []>;
5100 def SUST_B_1D_V4B32_ZERO
5101   : NVPTXInst<(outs),
5102               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g,
5103                    Int32Regs:$b, Int32Regs:$a),
5104               "sust.b.1d.v4.b32.zero \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
5105               []>;
5106
5107
5108 def SUST_B_1D_ARRAY_B8_ZERO
5109   : NVPTXInst<(outs),
5110               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r),
5111               "sust.b.a1d.b8.zero \t[$s, \\{$idx, $x\\}], \\{$r\\};",
5112               []>;
5113 def SUST_B_1D_ARRAY_B16_ZERO
5114   : NVPTXInst<(outs),
5115               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r),
5116               "sust.b.a1d.b16.zero \t[$s, \\{$idx, $x\\}], \\{$r\\};",
5117               []>;
5118 def SUST_B_1D_ARRAY_B32_ZERO
5119   : NVPTXInst<(outs),
5120               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$r),
5121               "sust.b.a1d.b32.zero \t[$s, \\{$idx, $x\\}], \\{$r\\};",
5122               []>;
5123 def SUST_B_1D_ARRAY_B64_ZERO
5124   : NVPTXInst<(outs),
5125               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int64Regs:$r),
5126               "sust.b.a1d.b64.zero \t[$s, \\{$idx, $x\\}], \\{$r\\};",
5127               []>;
5128 def SUST_B_1D_ARRAY_V2B8_ZERO
5129   : NVPTXInst<(outs),
5130               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
5131                    Int16Regs:$g),
5132               "sust.b.a1d.v2.b8.zero \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
5133               []>;
5134 def SUST_B_1D_ARRAY_V2B16_ZERO
5135   : NVPTXInst<(outs),
5136               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
5137                    Int16Regs:$g),
5138               "sust.b.a1d.v2.b16.zero \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
5139               []>;
5140 def SUST_B_1D_ARRAY_V2B32_ZERO
5141   : NVPTXInst<(outs),
5142               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$r,
5143                    Int32Regs:$g),
5144               "sust.b.a1d.v2.b32.zero \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
5145               []>;
5146 def SUST_B_1D_ARRAY_V2B64_ZERO
5147   : NVPTXInst<(outs),
5148               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int64Regs:$r,
5149                    Int64Regs:$g),
5150               "sust.b.a1d.v2.b64.zero \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
5151               []>;
5152 def SUST_B_1D_ARRAY_V4B8_ZERO
5153   : NVPTXInst<(outs),
5154               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
5155                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5156               "sust.b.a1d.v4.b8.zero \t[$s, \\{$idx, $x\\}], "
5157               "\\{$r, $g, $b, $a\\};",
5158               []>;
5159 def SUST_B_1D_ARRAY_V4B16_ZERO
5160   : NVPTXInst<(outs),
5161               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
5162                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5163              "sust.b.a1d.v4.b16.zero \t[$s, \\{$idx, $x\\}], "
5164              "\\{$r, $g, $b, $a\\};",
5165               []>;
5166 def SUST_B_1D_ARRAY_V4B32_ZERO
5167   : NVPTXInst<(outs),
5168               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$r,
5169                    Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5170              "sust.b.a1d.v4.b32.zero \t[$s, \\{$idx, $x\\}], "
5171              "\\{$r, $g, $b, $a\\};",
5172               []>;
5173
5174
5175 def SUST_B_2D_B8_ZERO
5176   : NVPTXInst<(outs),
5177               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
5178               "sust.b.2d.b8.zero \t[$s, \\{$x, $y\\}], \\{$r\\};",
5179               []>;
5180 def SUST_B_2D_B16_ZERO
5181   : NVPTXInst<(outs),
5182               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
5183               "sust.b.2d.b16.zero \t[$s, \\{$x, $y\\}], \\{$r\\};",
5184               []>;
5185 def SUST_B_2D_B32_ZERO
5186   : NVPTXInst<(outs),
5187               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
5188               "sust.b.2d.b32.zero \t[$s, \\{$x, $y\\}], \\{$r\\};",
5189               []>;
5190 def SUST_B_2D_B64_ZERO
5191   : NVPTXInst<(outs),
5192               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r),
5193               "sust.b.2d.b64.zero \t[$s, \\{$x, $y\\}], \\{$r\\};",
5194               []>;
5195 def SUST_B_2D_V2B8_ZERO
5196   : NVPTXInst<(outs),
5197               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
5198                    Int16Regs:$g),
5199               "sust.b.2d.v2.b8.zero \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
5200               []>;
5201 def SUST_B_2D_V2B16_ZERO
5202   : NVPTXInst<(outs),
5203               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
5204                    Int16Regs:$g),
5205               "sust.b.2d.v2.b16.zero \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
5206               []>;
5207 def SUST_B_2D_V2B32_ZERO
5208   : NVPTXInst<(outs),
5209               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
5210                    Int32Regs:$g),
5211               "sust.b.2d.v2.b32.zero \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
5212               []>;
5213 def SUST_B_2D_V2B64_ZERO
5214   : NVPTXInst<(outs),
5215               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r,
5216                    Int64Regs:$g),
5217               "sust.b.2d.v2.b64.zero \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
5218               []>;
5219 def SUST_B_2D_V4B8_ZERO
5220   : NVPTXInst<(outs),
5221               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
5222                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5223               "sust.b.2d.v4.b8.zero \t[$s, \\{$x, $y\\}], "
5224               "\\{$r, $g, $b, $a\\};",
5225               []>;
5226 def SUST_B_2D_V4B16_ZERO
5227   : NVPTXInst<(outs),
5228               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
5229                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5230              "sust.b.2d.v4.b16.zero \t[$s, \\{$x, $y\\}], "
5231              "\\{$r, $g, $b, $a\\};",
5232               []>;
5233 def SUST_B_2D_V4B32_ZERO
5234   : NVPTXInst<(outs),
5235               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
5236                    Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5237              "sust.b.2d.v4.b32.zero \t[$s, \\{$x, $y\\}], "
5238              "\\{$r, $g, $b, $a\\};",
5239               []>;
5240
5241
5242 def SUST_B_2D_ARRAY_B8_ZERO
5243   : NVPTXInst<(outs),
5244               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5245                    Int16Regs:$r),
5246               "sust.b.a2d.b8.zero \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
5247               []>;
5248 def SUST_B_2D_ARRAY_B16_ZERO
5249   : NVPTXInst<(outs),
5250               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5251                    Int16Regs:$r),
5252               "sust.b.a2d.b16.zero \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
5253               []>;
5254 def SUST_B_2D_ARRAY_B32_ZERO
5255   : NVPTXInst<(outs),
5256               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5257                    Int32Regs:$r),
5258               "sust.b.a2d.b32.zero \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
5259               []>;
5260 def SUST_B_2D_ARRAY_B64_ZERO
5261   : NVPTXInst<(outs),
5262               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5263                    Int64Regs:$r),
5264               "sust.b.a2d.b64.zero \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
5265               []>;
5266 def SUST_B_2D_ARRAY_V2B8_ZERO
5267   : NVPTXInst<(outs),
5268               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5269                    Int16Regs:$r, Int16Regs:$g),
5270               "sust.b.a2d.v2.b8.zero \t[$s, \\{$idx, $x, $y, $y\\}], "
5271               "\\{$r, $g\\};",
5272               []>;
5273 def SUST_B_2D_ARRAY_V2B16_ZERO
5274   : NVPTXInst<(outs),
5275               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5276                    Int16Regs:$r, Int16Regs:$g),
5277              "sust.b.a2d.v2.b16.zero \t[$s, \\{$idx, $x, $y, $y\\}], "
5278              "\\{$r, $g\\};",
5279               []>;
5280 def SUST_B_2D_ARRAY_V2B32_ZERO
5281   : NVPTXInst<(outs),
5282               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5283                    Int32Regs:$r, Int32Regs:$g),
5284              "sust.b.a2d.v2.b32.zero \t[$s, \\{$idx, $x, $y, $y\\}], "
5285              "\\{$r, $g\\};",
5286               []>;
5287 def SUST_B_2D_ARRAY_V2B64_ZERO
5288   : NVPTXInst<(outs),
5289               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5290                    Int64Regs:$r, Int64Regs:$g),
5291              "sust.b.a2d.v2.b64.zero \t[$s, \\{$idx, $x, $y, $y\\}], "
5292              "\\{$r, $g\\};",
5293               []>;
5294 def SUST_B_2D_ARRAY_V4B8_ZERO
5295   : NVPTXInst<(outs),
5296               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5297                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5298       "sust.b.a2d.v4.b8.zero \t[$s, \\{$idx, $x, $y, $y\\}], "
5299       "\\{$r, $g, $b, $a\\};",
5300               []>;
5301 def SUST_B_2D_ARRAY_V4B16_ZERO
5302   : NVPTXInst<(outs),
5303               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5304                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5305      "sust.b.a2d.v4.b16.zero \t[$s, \\{$idx, $x, $y, $y\\}], "
5306      "\\{$r, $g, $b, $a\\};",
5307               []>;
5308 def SUST_B_2D_ARRAY_V4B32_ZERO
5309   : NVPTXInst<(outs),
5310               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5311                    Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5312      "sust.b.a2d.v4.b32.zero \t[$s, \\{$idx, $x, $y, $y\\}], "
5313      "\\{$r, $g, $b, $a\\};",
5314               []>;
5315
5316
5317 def SUST_B_3D_B8_ZERO
5318   : NVPTXInst<(outs),
5319               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5320                    Int16Regs:$r),
5321               "sust.b.3d.b8.zero \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
5322               []>;
5323 def SUST_B_3D_B16_ZERO
5324   : NVPTXInst<(outs),
5325               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5326                    Int16Regs:$r),
5327               "sust.b.3d.b16.zero \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
5328               []>;
5329 def SUST_B_3D_B32_ZERO
5330   : NVPTXInst<(outs),
5331               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5332                    Int32Regs:$r),
5333               "sust.b.3d.b32.zero \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
5334               []>;
5335 def SUST_B_3D_B64_ZERO
5336   : NVPTXInst<(outs),
5337               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5338                    Int64Regs:$r),
5339               "sust.b.3d.b64.zero \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
5340               []>;
5341 def SUST_B_3D_V2B8_ZERO
5342   : NVPTXInst<(outs),
5343               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5344                    Int16Regs:$r, Int16Regs:$g),
5345               "sust.b.3d.v2.b8.zero \t[$s, \\{$x, $y, $z, $z\\}], "
5346               "\\{$r, $g\\};",
5347               []>;
5348 def SUST_B_3D_V2B16_ZERO
5349   : NVPTXInst<(outs),
5350               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5351                    Int16Regs:$r, Int16Regs:$g),
5352               "sust.b.3d.v2.b16.zero \t[$s, \\{$x, $y, $z, $z\\}], "
5353               "\\{$r, $g\\};",
5354               []>;
5355 def SUST_B_3D_V2B32_ZERO
5356   : NVPTXInst<(outs),
5357               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5358                    Int32Regs:$r, Int32Regs:$g),
5359               "sust.b.3d.v2.b32.zero \t[$s, \\{$x, $y, $z, $z\\}], "
5360               "\\{$r, $g\\};",
5361               []>;
5362 def SUST_B_3D_V2B64_ZERO
5363   : NVPTXInst<(outs),
5364               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5365                    Int64Regs:$r, Int64Regs:$g),
5366               "sust.b.3d.v2.b64.zero \t[$s, \\{$x, $y, $z, $z\\}], "
5367               "\\{$r, $g\\};",
5368               []>;
5369 def SUST_B_3D_V4B8_ZERO
5370   : NVPTXInst<(outs),
5371               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5372                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5373          "sust.b.3d.v4.b8.zero \t[$s, \\{$x, $y, $z, $z\\}], "
5374          "\\{$r, $g, $b, $a\\};",
5375               []>;
5376 def SUST_B_3D_V4B16_ZERO
5377   : NVPTXInst<(outs),
5378               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5379                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5380         "sust.b.3d.v4.b16.zero \t[$s, \\{$x, $y, $z, $z\\}], "
5381         "\\{$r, $g, $b, $a\\};",
5382               []>;
5383 def SUST_B_3D_V4B32_ZERO
5384   : NVPTXInst<(outs),
5385               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5386                    Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5387         "sust.b.3d.v4.b32.zero \t[$s, \\{$x, $y, $z, $z\\}], "
5388         "\\{$r, $g, $b, $a\\};",
5389               []>;
5390
5391
5392
5393 // Formatted
5394
5395 def SUST_P_1D_B8_TRAP
5396   : NVPTXInst<(outs),
5397               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
5398               "sust.p.1d.b8.trap \t[$s, \\{$x\\}], \\{$r\\};",
5399               []>;
5400 def SUST_P_1D_B16_TRAP
5401   : NVPTXInst<(outs),
5402               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
5403               "sust.p.1d.b16.trap \t[$s, \\{$x\\}], \\{$r\\};",
5404               []>;
5405 def SUST_P_1D_B32_TRAP
5406   : NVPTXInst<(outs),
5407               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$r),
5408               "sust.p.1d.b32.trap \t[$s, \\{$x\\}], \\{$r\\};",
5409               []>;
5410 def SUST_P_1D_V2B8_TRAP
5411   : NVPTXInst<(outs),
5412               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
5413               "sust.p.1d.v2.b8.trap \t[$s, \\{$x\\}], \\{$r, $g\\};",
5414               []>;
5415 def SUST_P_1D_V2B16_TRAP
5416   : NVPTXInst<(outs),
5417               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
5418               "sust.p.1d.v2.b16.trap \t[$s, \\{$x\\}], \\{$r, $g\\};",
5419               []>;
5420 def SUST_P_1D_V2B32_TRAP
5421   : NVPTXInst<(outs),
5422               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
5423               "sust.p.1d.v2.b32.trap \t[$s, \\{$x\\}], \\{$r, $g\\};",
5424               []>;
5425 def SUST_P_1D_V4B8_TRAP
5426   : NVPTXInst<(outs),
5427               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g,
5428                    Int16Regs:$b, Int16Regs:$a),
5429               "sust.p.1d.v4.b8.trap \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
5430               []>;
5431 def SUST_P_1D_V4B16_TRAP
5432   : NVPTXInst<(outs),
5433               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g,
5434                    Int16Regs:$b, Int16Regs:$a),
5435               "sust.p.1d.v4.b16.trap \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
5436               []>;
5437 def SUST_P_1D_V4B32_TRAP
5438   : NVPTXInst<(outs),
5439               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g,
5440                    Int32Regs:$b, Int32Regs:$a),
5441               "sust.p.1d.v4.b32.trap \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
5442               []>;
5443
5444
5445 def SUST_P_1D_ARRAY_B8_TRAP
5446   : NVPTXInst<(outs),
5447               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r),
5448               "sust.p.a1d.b8.trap \t[$s, \\{$idx, $x\\}], \\{$r\\};",
5449               []>;
5450 def SUST_P_1D_ARRAY_B16_TRAP
5451   : NVPTXInst<(outs),
5452               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r),
5453               "sust.p.a1d.b16.trap \t[$s, \\{$idx, $x\\}], \\{$r\\};",
5454               []>;
5455 def SUST_P_1D_ARRAY_B32_TRAP
5456   : NVPTXInst<(outs),
5457               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$r),
5458               "sust.p.a1d.b32.trap \t[$s, \\{$idx, $x\\}], \\{$r\\};",
5459               []>;
5460 def SUST_P_1D_ARRAY_V2B8_TRAP
5461   : NVPTXInst<(outs),
5462               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
5463                    Int16Regs:$g),
5464               "sust.p.a1d.v2.b8.trap \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
5465               []>;
5466 def SUST_P_1D_ARRAY_V2B16_TRAP
5467   : NVPTXInst<(outs),
5468               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
5469                    Int16Regs:$g),
5470               "sust.p.a1d.v2.b16.trap \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
5471               []>;
5472 def SUST_P_1D_ARRAY_V2B32_TRAP
5473   : NVPTXInst<(outs),
5474               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$r,
5475                    Int32Regs:$g),
5476               "sust.p.a1d.v2.b32.trap \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
5477               []>;
5478 def SUST_P_1D_ARRAY_V4B8_TRAP
5479   : NVPTXInst<(outs),
5480               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
5481                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5482               "sust.p.a1d.v4.b8.trap \t[$s, \\{$idx, $x\\}], "
5483               "\\{$r, $g, $b, $a\\};",
5484               []>;
5485 def SUST_P_1D_ARRAY_V4B16_TRAP
5486   : NVPTXInst<(outs),
5487               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
5488                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5489              "sust.p.a1d.v4.b16.trap \t[$s, \\{$idx, $x\\}], "
5490              "\\{$r, $g, $b, $a\\};",
5491               []>;
5492 def SUST_P_1D_ARRAY_V4B32_TRAP
5493   : NVPTXInst<(outs),
5494               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$r,
5495                    Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5496              "sust.p.a1d.v4.b32.trap \t[$s, \\{$idx, $x\\}], "
5497              "\\{$r, $g, $b, $a\\};",
5498               []>;
5499
5500
5501 def SUST_P_2D_B8_TRAP
5502   : NVPTXInst<(outs),
5503               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
5504               "sust.p.2d.b8.trap \t[$s, \\{$x, $y\\}], \\{$r\\};",
5505               []>;
5506 def SUST_P_2D_B16_TRAP
5507   : NVPTXInst<(outs),
5508               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
5509               "sust.p.2d.b16.trap \t[$s, \\{$x, $y\\}], \\{$r\\};",
5510               []>;
5511 def SUST_P_2D_B32_TRAP
5512   : NVPTXInst<(outs),
5513               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
5514               "sust.p.2d.b32.trap \t[$s, \\{$x, $y\\}], \\{$r\\};",
5515               []>;
5516 def SUST_P_2D_V2B8_TRAP
5517   : NVPTXInst<(outs),
5518               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
5519                    Int16Regs:$g),
5520               "sust.p.2d.v2.b8.trap \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
5521               []>;
5522 def SUST_P_2D_V2B16_TRAP
5523   : NVPTXInst<(outs),
5524               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
5525                    Int16Regs:$g),
5526               "sust.p.2d.v2.b16.trap \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
5527               []>;
5528 def SUST_P_2D_V2B32_TRAP
5529   : NVPTXInst<(outs),
5530               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
5531                    Int32Regs:$g),
5532               "sust.p.2d.v2.b32.trap \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
5533               []>;
5534 def SUST_P_2D_V4B8_TRAP
5535   : NVPTXInst<(outs),
5536               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
5537                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5538               "sust.p.2d.v4.b8.trap \t[$s, \\{$x, $y\\}], "
5539               "\\{$r, $g, $b, $a\\};",
5540               []>;
5541 def SUST_P_2D_V4B16_TRAP
5542   : NVPTXInst<(outs),
5543               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
5544                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5545              "sust.p.2d.v4.b16.trap \t[$s, \\{$x, $y\\}], "
5546              "\\{$r, $g, $b, $a\\};",
5547               []>;
5548 def SUST_P_2D_V4B32_TRAP
5549   : NVPTXInst<(outs),
5550               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
5551                    Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5552              "sust.p.2d.v4.b32.trap \t[$s, \\{$x, $y\\}], "
5553              "\\{$r, $g, $b, $a\\};",
5554               []>;
5555
5556
5557 def SUST_P_2D_ARRAY_B8_TRAP
5558   : NVPTXInst<(outs),
5559               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5560                    Int16Regs:$r),
5561               "sust.p.a2d.b8.trap \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
5562               []>;
5563 def SUST_P_2D_ARRAY_B16_TRAP
5564   : NVPTXInst<(outs),
5565               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5566                    Int16Regs:$r),
5567               "sust.p.a2d.b16.trap \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
5568               []>;
5569 def SUST_P_2D_ARRAY_B32_TRAP
5570   : NVPTXInst<(outs),
5571               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5572                    Int32Regs:$r),
5573               "sust.p.a2d.b32.trap \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
5574               []>;
5575 def SUST_P_2D_ARRAY_V2B8_TRAP
5576   : NVPTXInst<(outs),
5577               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5578                    Int16Regs:$r, Int16Regs:$g),
5579               "sust.p.a2d.v2.b8.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
5580               "\\{$r, $g\\};",
5581               []>;
5582 def SUST_P_2D_ARRAY_V2B16_TRAP
5583   : NVPTXInst<(outs),
5584               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5585                    Int16Regs:$r, Int16Regs:$g),
5586              "sust.p.a2d.v2.b16.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
5587              "\\{$r, $g\\};",
5588               []>;
5589 def SUST_P_2D_ARRAY_V2B32_TRAP
5590   : NVPTXInst<(outs),
5591               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5592                    Int32Regs:$r, Int32Regs:$g),
5593              "sust.p.a2d.v2.b32.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
5594              "\\{$r, $g\\};",
5595               []>;
5596 def SUST_P_2D_ARRAY_V4B8_TRAP
5597   : NVPTXInst<(outs),
5598               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5599                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5600       "sust.p.a2d.v4.b8.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
5601       "\\{$r, $g, $b, $a\\};",
5602               []>;
5603 def SUST_P_2D_ARRAY_V4B16_TRAP
5604   : NVPTXInst<(outs),
5605               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5606                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5607      "sust.p.a2d.v4.b16.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
5608      "\\{$r, $g, $b, $a\\};",
5609               []>;
5610 def SUST_P_2D_ARRAY_V4B32_TRAP
5611   : NVPTXInst<(outs),
5612               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5613                    Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5614      "sust.p.a2d.v4.b32.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
5615      "\\{$r, $g, $b, $a\\};",
5616               []>;
5617
5618
5619 def SUST_P_3D_B8_TRAP
5620   : NVPTXInst<(outs),
5621               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5622                    Int16Regs:$r),
5623               "sust.p.3d.b8.trap \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
5624               []>;
5625 def SUST_P_3D_B16_TRAP
5626   : NVPTXInst<(outs),
5627               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5628                    Int16Regs:$r),
5629               "sust.p.3d.b16.trap \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
5630               []>;
5631 def SUST_P_3D_B32_TRAP
5632   : NVPTXInst<(outs),
5633               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5634                    Int32Regs:$r),
5635               "sust.p.3d.b32.trap \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
5636               []>;
5637 def SUST_P_3D_V2B8_TRAP
5638   : NVPTXInst<(outs),
5639               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5640                    Int16Regs:$r, Int16Regs:$g),
5641               "sust.p.3d.v2.b8.trap \t[$s, \\{$x, $y, $z, $z\\}], "
5642               "\\{$r, $g\\};",
5643               []>;
5644 def SUST_P_3D_V2B16_TRAP
5645   : NVPTXInst<(outs),
5646               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5647                    Int16Regs:$r, Int16Regs:$g),
5648               "sust.p.3d.v2.b16.trap \t[$s, \\{$x, $y, $z, $z\\}], "
5649               "\\{$r, $g\\};",
5650               []>;
5651 def SUST_P_3D_V2B32_TRAP
5652   : NVPTXInst<(outs),
5653               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5654                    Int32Regs:$r, Int32Regs:$g),
5655               "sust.p.3d.v2.b32.trap \t[$s, \\{$x, $y, $z, $z\\}], "
5656               "\\{$r, $g\\};",
5657               []>;
5658 def SUST_P_3D_V4B8_TRAP
5659   : NVPTXInst<(outs),
5660               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5661                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5662          "sust.p.3d.v4.b8.trap \t[$s, \\{$x, $y, $z, $z\\}], "
5663          "\\{$r, $g, $b, $a\\};",
5664               []>;
5665 def SUST_P_3D_V4B16_TRAP
5666   : NVPTXInst<(outs),
5667               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5668                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5669         "sust.p.3d.v4.b16.trap \t[$s, \\{$x, $y, $z, $z\\}], "
5670         "\\{$r, $g, $b, $a\\};",
5671               []>;
5672 def SUST_P_3D_V4B32_TRAP
5673   : NVPTXInst<(outs),
5674               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5675                    Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5676         "sust.p.3d.v4.b32.trap \t[$s, \\{$x, $y, $z, $z\\}], "
5677         "\\{$r, $g, $b, $a\\};",
5678               []>;
5679 }
5680
5681 // Surface store instruction patterns
5682 // I'm not sure why we can't just include these in the instruction definitions,
5683 // but TableGen complains of type errors :(
5684
5685 // .clamp variant
5686 def : Pat<(int_nvvm_sust_b_1d_i8_clamp
5687            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
5688           (SUST_B_1D_B8_CLAMP Int64Regs:$s, Int32Regs:$x, Int16Regs:$r)>;
5689
5690 def : Pat<(int_nvvm_sust_b_1d_i16_clamp
5691            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
5692           (SUST_B_1D_B16_CLAMP Int64Regs:$s, Int32Regs:$x, Int16Regs:$r)>;
5693
5694 def : Pat<(int_nvvm_sust_b_1d_i32_clamp
5695            Int64Regs:$s, Int32Regs:$x, Int32Regs:$r),
5696           (SUST_B_1D_B32_CLAMP Int64Regs:$s, Int32Regs:$x, Int32Regs:$r)>;
5697
5698 def : Pat<(int_nvvm_sust_b_1d_i64_clamp
5699            Int64Regs:$s, Int32Regs:$x, Int64Regs:$r),
5700           (SUST_B_1D_B64_CLAMP Int64Regs:$s, Int32Regs:$x, Int64Regs:$r)>;
5701
5702 def : Pat<(int_nvvm_sust_b_1d_v2i8_clamp
5703            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
5704           (SUST_B_1D_V2B8_CLAMP Int64Regs:$s, Int32Regs:$x,
5705            Int16Regs:$r, Int16Regs:$g)>;
5706
5707 def : Pat<(int_nvvm_sust_b_1d_v2i16_clamp
5708            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
5709           (SUST_B_1D_V2B16_CLAMP Int64Regs:$s, Int32Regs:$x,
5710            Int16Regs:$r, Int16Regs:$g)>;
5711
5712 def : Pat<(int_nvvm_sust_b_1d_v2i32_clamp
5713            Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
5714           (SUST_B_1D_V2B32_CLAMP Int64Regs:$s, Int32Regs:$x,
5715            Int32Regs:$r, Int32Regs:$g)>;
5716
5717 def : Pat<(int_nvvm_sust_b_1d_v2i64_clamp
5718            Int64Regs:$s, Int32Regs:$x, Int64Regs:$r, Int64Regs:$g),
5719           (SUST_B_1D_V2B64_CLAMP Int64Regs:$s, Int32Regs:$x,
5720            Int64Regs:$r, Int64Regs:$g)>;
5721
5722 def : Pat<(int_nvvm_sust_b_1d_v4i8_clamp
5723            Int64Regs:$s, Int32Regs:$x,
5724            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5725           (SUST_B_1D_V4B8_CLAMP Int64Regs:$s, Int32Regs:$x,
5726            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
5727
5728 def : Pat<(int_nvvm_sust_b_1d_v4i16_clamp
5729            Int64Regs:$s, Int32Regs:$x,
5730            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5731           (SUST_B_1D_V4B16_CLAMP Int64Regs:$s, Int32Regs:$x,
5732            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
5733
5734 def : Pat<(int_nvvm_sust_b_1d_v4i32_clamp
5735            Int64Regs:$s, Int32Regs:$x,
5736            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5737           (SUST_B_1D_V4B32_CLAMP Int64Regs:$s, Int32Regs:$x,
5738            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
5739
5740
5741
5742 def : Pat<(int_nvvm_sust_b_1d_array_i8_clamp
5743            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r),
5744           (SUST_B_1D_ARRAY_B8_CLAMP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
5745            Int16Regs:$r)>;
5746
5747 def : Pat<(int_nvvm_sust_b_1d_array_i16_clamp
5748            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r),
5749           (SUST_B_1D_ARRAY_B16_CLAMP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
5750            Int16Regs:$r)>;
5751
5752 def : Pat<(int_nvvm_sust_b_1d_array_i32_clamp
5753            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$r),
5754           (SUST_B_1D_ARRAY_B32_CLAMP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
5755            Int32Regs:$r)>;
5756
5757 def : Pat<(int_nvvm_sust_b_1d_array_i64_clamp
5758            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int64Regs:$r),
5759           (SUST_B_1D_ARRAY_B64_CLAMP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
5760            Int64Regs:$r)>;
5761
5762 def : Pat<(int_nvvm_sust_b_1d_array_v2i8_clamp
5763           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
5764           (SUST_B_1D_ARRAY_V2B8_CLAMP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
5765            Int16Regs:$r, Int16Regs:$g)>;
5766
5767 def : Pat<(int_nvvm_sust_b_1d_array_v2i16_clamp
5768           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
5769           (SUST_B_1D_ARRAY_V2B16_CLAMP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
5770            Int16Regs:$r, Int16Regs:$g)>;
5771
5772 def : Pat<(int_nvvm_sust_b_1d_array_v2i32_clamp
5773           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
5774           (SUST_B_1D_ARRAY_V2B32_CLAMP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
5775            Int32Regs:$r, Int32Regs:$g)>;
5776
5777 def : Pat<(int_nvvm_sust_b_1d_array_v2i64_clamp
5778           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int64Regs:$r, Int64Regs:$g),
5779           (SUST_B_1D_ARRAY_V2B64_CLAMP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
5780            Int64Regs:$r, Int64Regs:$g)>;
5781
5782 def : Pat<(int_nvvm_sust_b_1d_array_v4i8_clamp
5783            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
5784            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5785           (SUST_B_1D_ARRAY_V4B8_CLAMP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
5786            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
5787
5788 def : Pat<(int_nvvm_sust_b_1d_array_v4i16_clamp
5789            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
5790            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5791           (SUST_B_1D_ARRAY_V4B16_CLAMP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
5792            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
5793
5794 def : Pat<(int_nvvm_sust_b_1d_array_v4i32_clamp
5795            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
5796            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5797           (SUST_B_1D_ARRAY_V4B32_CLAMP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
5798            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
5799
5800
5801
5802 def : Pat<(int_nvvm_sust_b_2d_i8_clamp
5803            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
5804           (SUST_B_2D_B8_CLAMP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
5805            Int16Regs:$r)>;
5806
5807 def : Pat<(int_nvvm_sust_b_2d_i16_clamp
5808            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
5809           (SUST_B_2D_B16_CLAMP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
5810            Int16Regs:$r)>;
5811
5812 def : Pat<(int_nvvm_sust_b_2d_i32_clamp
5813            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
5814           (SUST_B_2D_B32_CLAMP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
5815            Int32Regs:$r)>;
5816
5817 def : Pat<(int_nvvm_sust_b_2d_i64_clamp
5818            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r),
5819           (SUST_B_2D_B64_CLAMP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
5820            Int64Regs:$r)>;
5821
5822 def : Pat<(int_nvvm_sust_b_2d_v2i8_clamp
5823           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r, Int16Regs:$g),
5824           (SUST_B_2D_V2B8_CLAMP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
5825            Int16Regs:$r, Int16Regs:$g)>;
5826
5827 def : Pat<(int_nvvm_sust_b_2d_v2i16_clamp
5828           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r, Int16Regs:$g),
5829           (SUST_B_2D_V2B16_CLAMP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
5830            Int16Regs:$r, Int16Regs:$g)>;
5831
5832 def : Pat<(int_nvvm_sust_b_2d_v2i32_clamp
5833           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r, Int32Regs:$g),
5834           (SUST_B_2D_V2B32_CLAMP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
5835            Int32Regs:$r, Int32Regs:$g)>;
5836
5837 def : Pat<(int_nvvm_sust_b_2d_v2i64_clamp
5838           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r, Int64Regs:$g),
5839           (SUST_B_2D_V2B64_CLAMP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
5840            Int64Regs:$r, Int64Regs:$g)>;
5841
5842 def : Pat<(int_nvvm_sust_b_2d_v4i8_clamp
5843            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
5844            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5845           (SUST_B_2D_V4B8_CLAMP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
5846            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
5847
5848 def : Pat<(int_nvvm_sust_b_2d_v4i16_clamp
5849            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
5850            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5851           (SUST_B_2D_V4B16_CLAMP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
5852            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
5853
5854 def : Pat<(int_nvvm_sust_b_2d_v4i32_clamp
5855            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
5856            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5857           (SUST_B_2D_V4B32_CLAMP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
5858            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
5859
5860
5861
5862 def : Pat<(int_nvvm_sust_b_2d_array_i8_clamp
5863           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
5864           (SUST_B_2D_ARRAY_B8_CLAMP Int64Regs:$s,
5865            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
5866            Int16Regs:$r)>;
5867
5868 def : Pat<(int_nvvm_sust_b_2d_array_i16_clamp
5869           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
5870           (SUST_B_2D_ARRAY_B16_CLAMP Int64Regs:$s,
5871            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
5872            Int16Regs:$r)>;
5873
5874 def : Pat<(int_nvvm_sust_b_2d_array_i32_clamp
5875           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
5876           (SUST_B_2D_ARRAY_B32_CLAMP Int64Regs:$s,
5877            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
5878            Int32Regs:$r)>;
5879
5880 def : Pat<(int_nvvm_sust_b_2d_array_i64_clamp
5881           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r),
5882           (SUST_B_2D_ARRAY_B64_CLAMP Int64Regs:$s,
5883            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
5884            Int64Regs:$r)>;
5885
5886 def : Pat<(int_nvvm_sust_b_2d_array_v2i8_clamp
5887            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
5888            Int16Regs:$r, Int16Regs:$g),
5889           (SUST_B_2D_ARRAY_V2B8_CLAMP Int64Regs:$s, Int32Regs:$l,
5890            Int32Regs:$x, Int32Regs:$y,
5891            Int16Regs:$r, Int16Regs:$g)>;
5892
5893 def : Pat<(int_nvvm_sust_b_2d_array_v2i16_clamp
5894            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
5895            Int16Regs:$r, Int16Regs:$g),
5896           (SUST_B_2D_ARRAY_V2B16_CLAMP Int64Regs:$s, Int32Regs:$l,
5897            Int32Regs:$x, Int32Regs:$y,
5898            Int16Regs:$r, Int16Regs:$g)>;
5899
5900 def : Pat<(int_nvvm_sust_b_2d_array_v2i32_clamp
5901            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
5902            Int32Regs:$g),
5903           (SUST_B_2D_ARRAY_V2B32_CLAMP Int64Regs:$s, Int32Regs:$l,
5904            Int32Regs:$x, Int32Regs:$y, Int32Regs:$r, Int32Regs:$g)>;
5905
5906 def : Pat<(int_nvvm_sust_b_2d_array_v2i64_clamp
5907            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r,
5908            Int64Regs:$g),
5909           (SUST_B_2D_ARRAY_V2B64_CLAMP Int64Regs:$s, Int32Regs:$l,
5910            Int32Regs:$x, Int32Regs:$y, Int64Regs:$r, Int64Regs:$g)>;
5911
5912 def : Pat<(int_nvvm_sust_b_2d_array_v4i8_clamp
5913            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
5914            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5915           (SUST_B_2D_ARRAY_V4B8_CLAMP Int64Regs:$s,
5916            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
5917            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
5918
5919 def : Pat<(int_nvvm_sust_b_2d_array_v4i16_clamp
5920            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
5921            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5922           (SUST_B_2D_ARRAY_V4B16_CLAMP Int64Regs:$s,
5923            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
5924            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
5925
5926 def : Pat<(int_nvvm_sust_b_2d_array_v4i32_clamp
5927            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
5928            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5929           (SUST_B_2D_ARRAY_V4B32_CLAMP Int64Regs:$s, Int32Regs:$l,
5930            Int32Regs:$x, Int32Regs:$y,
5931            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
5932
5933
5934
5935 def : Pat<(int_nvvm_sust_b_3d_i8_clamp
5936            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5937            Int16Regs:$r),
5938           (SUST_B_3D_B8_CLAMP Int64Regs:$s,
5939            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5940            Int16Regs:$r)>;
5941
5942 def : Pat<(int_nvvm_sust_b_3d_i16_clamp
5943            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5944            Int16Regs:$r),
5945           (SUST_B_3D_B16_CLAMP Int64Regs:$s,
5946            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5947            Int16Regs:$r)>;
5948
5949 def : Pat<(int_nvvm_sust_b_3d_i32_clamp
5950            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5951            Int32Regs:$r),
5952           (SUST_B_3D_B32_CLAMP Int64Regs:$s,
5953            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5954            Int32Regs:$r)>;
5955
5956 def : Pat<(int_nvvm_sust_b_3d_i64_clamp
5957            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5958            Int64Regs:$r),
5959           (SUST_B_3D_B64_CLAMP Int64Regs:$s,
5960            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5961            Int64Regs:$r)>;
5962
5963 def : Pat<(int_nvvm_sust_b_3d_v2i8_clamp
5964            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5965            Int16Regs:$r, Int16Regs:$g),
5966           (SUST_B_3D_V2B8_CLAMP Int64Regs:$s,
5967            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5968            Int16Regs:$r, Int16Regs:$g)>;
5969
5970 def : Pat<(int_nvvm_sust_b_3d_v2i16_clamp
5971            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5972            Int16Regs:$r, Int16Regs:$g),
5973           (SUST_B_3D_V2B16_CLAMP Int64Regs:$s,
5974            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5975            Int16Regs:$r, Int16Regs:$g)>;
5976
5977 def : Pat<(int_nvvm_sust_b_3d_v2i32_clamp
5978            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5979            Int32Regs:$r, Int32Regs:$g),
5980           (SUST_B_3D_V2B32_CLAMP Int64Regs:$s,
5981            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5982            Int32Regs:$r, Int32Regs:$g)>;
5983
5984 def : Pat<(int_nvvm_sust_b_3d_v2i64_clamp
5985            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5986            Int64Regs:$r, Int64Regs:$g),
5987           (SUST_B_3D_V2B64_CLAMP Int64Regs:$s,
5988            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5989            Int64Regs:$r, Int64Regs:$g)>;
5990
5991 def : Pat<(int_nvvm_sust_b_3d_v4i8_clamp
5992            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5993            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5994           (SUST_B_3D_V4B8_CLAMP Int64Regs:$s,
5995            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5996            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
5997
5998 def : Pat<(int_nvvm_sust_b_3d_v4i16_clamp
5999            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6000            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6001           (SUST_B_3D_V4B16_CLAMP Int64Regs:$s,
6002            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6003            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6004
6005 def : Pat<(int_nvvm_sust_b_3d_v4i32_clamp
6006            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6007            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6008           (SUST_B_3D_V4B32_CLAMP Int64Regs:$s,
6009            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6010            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6011
6012
6013 // .trap variant
6014 def : Pat<(int_nvvm_sust_b_1d_i8_trap
6015            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
6016           (SUST_B_1D_B8_TRAP Int64Regs:$s, Int32Regs:$x, Int16Regs:$r)>;
6017
6018 def : Pat<(int_nvvm_sust_b_1d_i16_trap
6019            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
6020           (SUST_B_1D_B16_TRAP Int64Regs:$s, Int32Regs:$x, Int16Regs:$r)>;
6021
6022 def : Pat<(int_nvvm_sust_b_1d_i32_trap
6023            Int64Regs:$s, Int32Regs:$x, Int32Regs:$r),
6024           (SUST_B_1D_B32_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$r)>;
6025
6026 def : Pat<(int_nvvm_sust_b_1d_i64_trap
6027            Int64Regs:$s, Int32Regs:$x, Int64Regs:$r),
6028           (SUST_B_1D_B64_TRAP Int64Regs:$s, Int32Regs:$x, Int64Regs:$r)>;
6029
6030 def : Pat<(int_nvvm_sust_b_1d_v2i8_trap
6031            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
6032           (SUST_B_1D_V2B8_TRAP Int64Regs:$s, Int32Regs:$x,
6033            Int16Regs:$r, Int16Regs:$g)>;
6034
6035 def : Pat<(int_nvvm_sust_b_1d_v2i16_trap
6036            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
6037           (SUST_B_1D_V2B16_TRAP Int64Regs:$s, Int32Regs:$x,
6038            Int16Regs:$r, Int16Regs:$g)>;
6039
6040 def : Pat<(int_nvvm_sust_b_1d_v2i32_trap
6041            Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
6042           (SUST_B_1D_V2B32_TRAP Int64Regs:$s, Int32Regs:$x,
6043            Int32Regs:$r, Int32Regs:$g)>;
6044
6045 def : Pat<(int_nvvm_sust_b_1d_v2i64_trap
6046            Int64Regs:$s, Int32Regs:$x, Int64Regs:$r, Int64Regs:$g),
6047           (SUST_B_1D_V2B64_TRAP Int64Regs:$s, Int32Regs:$x,
6048            Int64Regs:$r, Int64Regs:$g)>;
6049
6050 def : Pat<(int_nvvm_sust_b_1d_v4i8_trap
6051            Int64Regs:$s, Int32Regs:$x,
6052            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6053           (SUST_B_1D_V4B8_TRAP Int64Regs:$s, Int32Regs:$x,
6054            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6055
6056 def : Pat<(int_nvvm_sust_b_1d_v4i16_trap
6057            Int64Regs:$s, Int32Regs:$x,
6058            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6059           (SUST_B_1D_V4B16_TRAP Int64Regs:$s, Int32Regs:$x,
6060            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6061
6062 def : Pat<(int_nvvm_sust_b_1d_v4i32_trap
6063            Int64Regs:$s, Int32Regs:$x,
6064            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6065           (SUST_B_1D_V4B32_TRAP Int64Regs:$s, Int32Regs:$x,
6066            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6067
6068
6069
6070 def : Pat<(int_nvvm_sust_b_1d_array_i8_trap
6071            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r),
6072           (SUST_B_1D_ARRAY_B8_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6073            Int16Regs:$r)>;
6074
6075 def : Pat<(int_nvvm_sust_b_1d_array_i16_trap
6076            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r),
6077           (SUST_B_1D_ARRAY_B16_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6078            Int16Regs:$r)>;
6079
6080 def : Pat<(int_nvvm_sust_b_1d_array_i32_trap
6081            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$r),
6082           (SUST_B_1D_ARRAY_B32_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6083            Int32Regs:$r)>;
6084
6085 def : Pat<(int_nvvm_sust_b_1d_array_i64_trap
6086            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int64Regs:$r),
6087           (SUST_B_1D_ARRAY_B64_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6088            Int64Regs:$r)>;
6089
6090 def : Pat<(int_nvvm_sust_b_1d_array_v2i8_trap
6091           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
6092           (SUST_B_1D_ARRAY_V2B8_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6093            Int16Regs:$r, Int16Regs:$g)>;
6094
6095 def : Pat<(int_nvvm_sust_b_1d_array_v2i16_trap
6096           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
6097           (SUST_B_1D_ARRAY_V2B16_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6098            Int16Regs:$r, Int16Regs:$g)>;
6099
6100 def : Pat<(int_nvvm_sust_b_1d_array_v2i32_trap
6101           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
6102           (SUST_B_1D_ARRAY_V2B32_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6103            Int32Regs:$r, Int32Regs:$g)>;
6104
6105 def : Pat<(int_nvvm_sust_b_1d_array_v2i64_trap
6106           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int64Regs:$r, Int64Regs:$g),
6107           (SUST_B_1D_ARRAY_V2B64_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6108            Int64Regs:$r, Int64Regs:$g)>;
6109
6110 def : Pat<(int_nvvm_sust_b_1d_array_v4i8_trap
6111            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6112            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6113           (SUST_B_1D_ARRAY_V4B8_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6114            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6115
6116 def : Pat<(int_nvvm_sust_b_1d_array_v4i16_trap
6117            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6118            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6119           (SUST_B_1D_ARRAY_V4B16_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6120            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6121
6122 def : Pat<(int_nvvm_sust_b_1d_array_v4i32_trap
6123            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6124            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6125           (SUST_B_1D_ARRAY_V4B32_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6126            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6127
6128
6129
6130 def : Pat<(int_nvvm_sust_b_2d_i8_trap
6131            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
6132           (SUST_B_2D_B8_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6133            Int16Regs:$r)>;
6134
6135 def : Pat<(int_nvvm_sust_b_2d_i16_trap
6136            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
6137           (SUST_B_2D_B16_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6138            Int16Regs:$r)>;
6139
6140 def : Pat<(int_nvvm_sust_b_2d_i32_trap
6141            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
6142           (SUST_B_2D_B32_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6143            Int32Regs:$r)>;
6144
6145 def : Pat<(int_nvvm_sust_b_2d_i64_trap
6146            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r),
6147           (SUST_B_2D_B64_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6148            Int64Regs:$r)>;
6149
6150 def : Pat<(int_nvvm_sust_b_2d_v2i8_trap
6151           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r, Int16Regs:$g),
6152           (SUST_B_2D_V2B8_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6153            Int16Regs:$r, Int16Regs:$g)>;
6154
6155 def : Pat<(int_nvvm_sust_b_2d_v2i16_trap
6156           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r, Int16Regs:$g),
6157           (SUST_B_2D_V2B16_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6158            Int16Regs:$r, Int16Regs:$g)>;
6159
6160 def : Pat<(int_nvvm_sust_b_2d_v2i32_trap
6161           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r, Int32Regs:$g),
6162           (SUST_B_2D_V2B32_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6163            Int32Regs:$r, Int32Regs:$g)>;
6164
6165 def : Pat<(int_nvvm_sust_b_2d_v2i64_trap
6166           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r, Int64Regs:$g),
6167           (SUST_B_2D_V2B64_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6168            Int64Regs:$r, Int64Regs:$g)>;
6169
6170 def : Pat<(int_nvvm_sust_b_2d_v4i8_trap
6171            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6172            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6173           (SUST_B_2D_V4B8_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6174            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6175
6176 def : Pat<(int_nvvm_sust_b_2d_v4i16_trap
6177            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6178            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6179           (SUST_B_2D_V4B16_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6180            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6181
6182 def : Pat<(int_nvvm_sust_b_2d_v4i32_trap
6183            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6184            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6185           (SUST_B_2D_V4B32_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6186            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6187
6188
6189
6190 def : Pat<(int_nvvm_sust_b_2d_array_i8_trap
6191           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
6192           (SUST_B_2D_ARRAY_B8_TRAP Int64Regs:$s,
6193            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6194            Int16Regs:$r)>;
6195
6196 def : Pat<(int_nvvm_sust_b_2d_array_i16_trap
6197           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
6198           (SUST_B_2D_ARRAY_B16_TRAP Int64Regs:$s,
6199            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6200            Int16Regs:$r)>;
6201
6202 def : Pat<(int_nvvm_sust_b_2d_array_i32_trap
6203           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
6204           (SUST_B_2D_ARRAY_B32_TRAP Int64Regs:$s,
6205            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6206            Int32Regs:$r)>;
6207
6208 def : Pat<(int_nvvm_sust_b_2d_array_i64_trap
6209           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r),
6210           (SUST_B_2D_ARRAY_B64_TRAP Int64Regs:$s,
6211            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6212            Int64Regs:$r)>;
6213
6214 def : Pat<(int_nvvm_sust_b_2d_array_v2i8_trap
6215            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6216            Int16Regs:$r, Int16Regs:$g),
6217           (SUST_B_2D_ARRAY_V2B8_TRAP Int64Regs:$s, Int32Regs:$l,
6218            Int32Regs:$x, Int32Regs:$y,
6219            Int16Regs:$r, Int16Regs:$g)>;
6220
6221 def : Pat<(int_nvvm_sust_b_2d_array_v2i16_trap
6222            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6223            Int16Regs:$r, Int16Regs:$g),
6224           (SUST_B_2D_ARRAY_V2B16_TRAP Int64Regs:$s, Int32Regs:$l,
6225            Int32Regs:$x, Int32Regs:$y,
6226            Int16Regs:$r, Int16Regs:$g)>;
6227
6228 def : Pat<(int_nvvm_sust_b_2d_array_v2i32_trap
6229            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
6230            Int32Regs:$g),
6231           (SUST_B_2D_ARRAY_V2B32_TRAP Int64Regs:$s, Int32Regs:$l,
6232            Int32Regs:$x, Int32Regs:$y, Int32Regs:$r, Int32Regs:$g)>;
6233
6234 def : Pat<(int_nvvm_sust_b_2d_array_v2i64_trap
6235            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r,
6236            Int64Regs:$g),
6237           (SUST_B_2D_ARRAY_V2B64_TRAP Int64Regs:$s, Int32Regs:$l,
6238            Int32Regs:$x, Int32Regs:$y, Int64Regs:$r, Int64Regs:$g)>;
6239
6240 def : Pat<(int_nvvm_sust_b_2d_array_v4i8_trap
6241            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6242            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6243           (SUST_B_2D_ARRAY_V4B8_TRAP Int64Regs:$s,
6244            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6245            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6246
6247 def : Pat<(int_nvvm_sust_b_2d_array_v4i16_trap
6248            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6249            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6250           (SUST_B_2D_ARRAY_V4B16_TRAP Int64Regs:$s,
6251            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6252            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6253
6254 def : Pat<(int_nvvm_sust_b_2d_array_v4i32_trap
6255            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6256            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6257           (SUST_B_2D_ARRAY_V4B32_TRAP Int64Regs:$s, Int32Regs:$l,
6258            Int32Regs:$x, Int32Regs:$y,
6259            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6260
6261
6262
6263 def : Pat<(int_nvvm_sust_b_3d_i8_trap
6264            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6265            Int16Regs:$r),
6266           (SUST_B_3D_B8_TRAP Int64Regs:$s,
6267            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6268            Int16Regs:$r)>;
6269
6270 def : Pat<(int_nvvm_sust_b_3d_i16_trap
6271            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6272            Int16Regs:$r),
6273           (SUST_B_3D_B16_TRAP Int64Regs:$s,
6274            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6275            Int16Regs:$r)>;
6276
6277 def : Pat<(int_nvvm_sust_b_3d_i32_trap
6278            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6279            Int32Regs:$r),
6280           (SUST_B_3D_B32_TRAP Int64Regs:$s,
6281            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6282            Int32Regs:$r)>;
6283
6284 def : Pat<(int_nvvm_sust_b_3d_i64_trap
6285            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6286            Int64Regs:$r),
6287           (SUST_B_3D_B64_TRAP Int64Regs:$s,
6288            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6289            Int64Regs:$r)>;
6290
6291 def : Pat<(int_nvvm_sust_b_3d_v2i8_trap
6292            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6293            Int16Regs:$r, Int16Regs:$g),
6294           (SUST_B_3D_V2B8_TRAP Int64Regs:$s,
6295            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6296            Int16Regs:$r, Int16Regs:$g)>;
6297
6298 def : Pat<(int_nvvm_sust_b_3d_v2i16_trap
6299            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6300            Int16Regs:$r, Int16Regs:$g),
6301           (SUST_B_3D_V2B16_TRAP Int64Regs:$s,
6302            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6303            Int16Regs:$r, Int16Regs:$g)>;
6304
6305 def : Pat<(int_nvvm_sust_b_3d_v2i32_trap
6306            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6307            Int32Regs:$r, Int32Regs:$g),
6308           (SUST_B_3D_V2B32_TRAP Int64Regs:$s,
6309            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6310            Int32Regs:$r, Int32Regs:$g)>;
6311
6312 def : Pat<(int_nvvm_sust_b_3d_v2i64_trap
6313            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6314            Int64Regs:$r, Int64Regs:$g),
6315           (SUST_B_3D_V2B64_TRAP Int64Regs:$s,
6316            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6317            Int64Regs:$r, Int64Regs:$g)>;
6318
6319 def : Pat<(int_nvvm_sust_b_3d_v4i8_trap
6320            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6321            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6322           (SUST_B_3D_V4B8_TRAP Int64Regs:$s,
6323            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6324            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6325
6326 def : Pat<(int_nvvm_sust_b_3d_v4i16_trap
6327            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6328            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6329           (SUST_B_3D_V4B16_TRAP Int64Regs:$s,
6330            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6331            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6332
6333 def : Pat<(int_nvvm_sust_b_3d_v4i32_trap
6334            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6335            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6336           (SUST_B_3D_V4B32_TRAP Int64Regs:$s,
6337            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6338            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6339
6340
6341 // .zero variant
6342 def : Pat<(int_nvvm_sust_b_1d_i8_zero
6343            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
6344           (SUST_B_1D_B8_ZERO Int64Regs:$s, Int32Regs:$x, Int16Regs:$r)>;
6345
6346 def : Pat<(int_nvvm_sust_b_1d_i16_zero
6347            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
6348           (SUST_B_1D_B16_ZERO Int64Regs:$s, Int32Regs:$x, Int16Regs:$r)>;
6349
6350 def : Pat<(int_nvvm_sust_b_1d_i32_zero
6351            Int64Regs:$s, Int32Regs:$x, Int32Regs:$r),
6352           (SUST_B_1D_B32_ZERO Int64Regs:$s, Int32Regs:$x, Int32Regs:$r)>;
6353
6354 def : Pat<(int_nvvm_sust_b_1d_i64_zero
6355            Int64Regs:$s, Int32Regs:$x, Int64Regs:$r),
6356           (SUST_B_1D_B64_ZERO Int64Regs:$s, Int32Regs:$x, Int64Regs:$r)>;
6357
6358 def : Pat<(int_nvvm_sust_b_1d_v2i8_zero
6359            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
6360           (SUST_B_1D_V2B8_ZERO Int64Regs:$s, Int32Regs:$x,
6361            Int16Regs:$r, Int16Regs:$g)>;
6362
6363 def : Pat<(int_nvvm_sust_b_1d_v2i16_zero
6364            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
6365           (SUST_B_1D_V2B16_ZERO Int64Regs:$s, Int32Regs:$x,
6366            Int16Regs:$r, Int16Regs:$g)>;
6367
6368 def : Pat<(int_nvvm_sust_b_1d_v2i32_zero
6369            Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
6370           (SUST_B_1D_V2B32_ZERO Int64Regs:$s, Int32Regs:$x,
6371            Int32Regs:$r, Int32Regs:$g)>;
6372
6373 def : Pat<(int_nvvm_sust_b_1d_v2i64_zero
6374            Int64Regs:$s, Int32Regs:$x, Int64Regs:$r, Int64Regs:$g),
6375           (SUST_B_1D_V2B64_ZERO Int64Regs:$s, Int32Regs:$x,
6376            Int64Regs:$r, Int64Regs:$g)>;
6377
6378 def : Pat<(int_nvvm_sust_b_1d_v4i8_zero
6379            Int64Regs:$s, Int32Regs:$x,
6380            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6381           (SUST_B_1D_V4B8_ZERO Int64Regs:$s, Int32Regs:$x,
6382            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6383
6384 def : Pat<(int_nvvm_sust_b_1d_v4i16_zero
6385            Int64Regs:$s, Int32Regs:$x,
6386            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6387           (SUST_B_1D_V4B16_ZERO Int64Regs:$s, Int32Regs:$x,
6388            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6389
6390 def : Pat<(int_nvvm_sust_b_1d_v4i32_zero
6391            Int64Regs:$s, Int32Regs:$x,
6392            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6393           (SUST_B_1D_V4B32_ZERO Int64Regs:$s, Int32Regs:$x,
6394            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6395
6396
6397
6398 def : Pat<(int_nvvm_sust_b_1d_array_i8_zero
6399            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r),
6400           (SUST_B_1D_ARRAY_B8_ZERO Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6401            Int16Regs:$r)>;
6402
6403 def : Pat<(int_nvvm_sust_b_1d_array_i16_zero
6404            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r),
6405           (SUST_B_1D_ARRAY_B16_ZERO Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6406            Int16Regs:$r)>;
6407
6408 def : Pat<(int_nvvm_sust_b_1d_array_i32_zero
6409            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$r),
6410           (SUST_B_1D_ARRAY_B32_ZERO Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6411            Int32Regs:$r)>;
6412
6413 def : Pat<(int_nvvm_sust_b_1d_array_i64_zero
6414            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int64Regs:$r),
6415           (SUST_B_1D_ARRAY_B64_ZERO Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6416            Int64Regs:$r)>;
6417
6418 def : Pat<(int_nvvm_sust_b_1d_array_v2i8_zero
6419           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
6420           (SUST_B_1D_ARRAY_V2B8_ZERO Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6421            Int16Regs:$r, Int16Regs:$g)>;
6422
6423 def : Pat<(int_nvvm_sust_b_1d_array_v2i16_zero
6424           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
6425           (SUST_B_1D_ARRAY_V2B16_ZERO Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6426            Int16Regs:$r, Int16Regs:$g)>;
6427
6428 def : Pat<(int_nvvm_sust_b_1d_array_v2i32_zero
6429           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
6430           (SUST_B_1D_ARRAY_V2B32_ZERO Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6431            Int32Regs:$r, Int32Regs:$g)>;
6432
6433 def : Pat<(int_nvvm_sust_b_1d_array_v2i64_zero
6434           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int64Regs:$r, Int64Regs:$g),
6435           (SUST_B_1D_ARRAY_V2B64_ZERO Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6436            Int64Regs:$r, Int64Regs:$g)>;
6437
6438 def : Pat<(int_nvvm_sust_b_1d_array_v4i8_zero
6439            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6440            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6441           (SUST_B_1D_ARRAY_V4B8_ZERO Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6442            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6443
6444 def : Pat<(int_nvvm_sust_b_1d_array_v4i16_zero
6445            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6446            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6447           (SUST_B_1D_ARRAY_V4B16_ZERO Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6448            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6449
6450 def : Pat<(int_nvvm_sust_b_1d_array_v4i32_zero
6451            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6452            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6453           (SUST_B_1D_ARRAY_V4B32_ZERO Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6454            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6455
6456
6457
6458 def : Pat<(int_nvvm_sust_b_2d_i8_zero
6459            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
6460           (SUST_B_2D_B8_ZERO Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6461            Int16Regs:$r)>;
6462
6463 def : Pat<(int_nvvm_sust_b_2d_i16_zero
6464            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
6465           (SUST_B_2D_B16_ZERO Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6466            Int16Regs:$r)>;
6467
6468 def : Pat<(int_nvvm_sust_b_2d_i32_zero
6469            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
6470           (SUST_B_2D_B32_ZERO Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6471            Int32Regs:$r)>;
6472
6473 def : Pat<(int_nvvm_sust_b_2d_i64_zero
6474            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r),
6475           (SUST_B_2D_B64_ZERO Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6476            Int64Regs:$r)>;
6477
6478 def : Pat<(int_nvvm_sust_b_2d_v2i8_zero
6479           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r, Int16Regs:$g),
6480           (SUST_B_2D_V2B8_ZERO Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6481            Int16Regs:$r, Int16Regs:$g)>;
6482
6483 def : Pat<(int_nvvm_sust_b_2d_v2i16_zero
6484           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r, Int16Regs:$g),
6485           (SUST_B_2D_V2B16_ZERO Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6486            Int16Regs:$r, Int16Regs:$g)>;
6487
6488 def : Pat<(int_nvvm_sust_b_2d_v2i32_zero
6489           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r, Int32Regs:$g),
6490           (SUST_B_2D_V2B32_ZERO Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6491            Int32Regs:$r, Int32Regs:$g)>;
6492
6493 def : Pat<(int_nvvm_sust_b_2d_v2i64_zero
6494           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r, Int64Regs:$g),
6495           (SUST_B_2D_V2B64_ZERO Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6496            Int64Regs:$r, Int64Regs:$g)>;
6497
6498 def : Pat<(int_nvvm_sust_b_2d_v4i8_zero
6499            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6500            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6501           (SUST_B_2D_V4B8_ZERO Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6502            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6503
6504 def : Pat<(int_nvvm_sust_b_2d_v4i16_zero
6505            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6506            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6507           (SUST_B_2D_V4B16_ZERO Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6508            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6509
6510 def : Pat<(int_nvvm_sust_b_2d_v4i32_zero
6511            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6512            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6513           (SUST_B_2D_V4B32_ZERO Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6514            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6515
6516
6517
6518 def : Pat<(int_nvvm_sust_b_2d_array_i8_zero
6519           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
6520           (SUST_B_2D_ARRAY_B8_ZERO Int64Regs:$s,
6521            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6522            Int16Regs:$r)>;
6523
6524 def : Pat<(int_nvvm_sust_b_2d_array_i16_zero
6525           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
6526           (SUST_B_2D_ARRAY_B16_ZERO Int64Regs:$s,
6527            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6528            Int16Regs:$r)>;
6529
6530 def : Pat<(int_nvvm_sust_b_2d_array_i32_zero
6531           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
6532           (SUST_B_2D_ARRAY_B32_ZERO Int64Regs:$s,
6533            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6534            Int32Regs:$r)>;
6535
6536 def : Pat<(int_nvvm_sust_b_2d_array_i64_zero
6537           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r),
6538           (SUST_B_2D_ARRAY_B64_ZERO Int64Regs:$s,
6539            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6540            Int64Regs:$r)>;
6541
6542 def : Pat<(int_nvvm_sust_b_2d_array_v2i8_zero
6543            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6544            Int16Regs:$r, Int16Regs:$g),
6545           (SUST_B_2D_ARRAY_V2B8_ZERO Int64Regs:$s, Int32Regs:$l,
6546            Int32Regs:$x, Int32Regs:$y,
6547            Int16Regs:$r, Int16Regs:$g)>;
6548
6549 def : Pat<(int_nvvm_sust_b_2d_array_v2i16_zero
6550            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6551            Int16Regs:$r, Int16Regs:$g),
6552           (SUST_B_2D_ARRAY_V2B16_ZERO Int64Regs:$s, Int32Regs:$l,
6553            Int32Regs:$x, Int32Regs:$y,
6554            Int16Regs:$r, Int16Regs:$g)>;
6555
6556 def : Pat<(int_nvvm_sust_b_2d_array_v2i32_zero
6557            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
6558            Int32Regs:$g),
6559           (SUST_B_2D_ARRAY_V2B32_ZERO Int64Regs:$s, Int32Regs:$l,
6560            Int32Regs:$x, Int32Regs:$y, Int32Regs:$r, Int32Regs:$g)>;
6561
6562 def : Pat<(int_nvvm_sust_b_2d_array_v2i64_zero
6563            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r,
6564            Int64Regs:$g),
6565           (SUST_B_2D_ARRAY_V2B64_ZERO Int64Regs:$s, Int32Regs:$l,
6566            Int32Regs:$x, Int32Regs:$y, Int64Regs:$r, Int64Regs:$g)>;
6567
6568 def : Pat<(int_nvvm_sust_b_2d_array_v4i8_zero
6569            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6570            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6571           (SUST_B_2D_ARRAY_V4B8_ZERO Int64Regs:$s,
6572            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6573            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6574
6575 def : Pat<(int_nvvm_sust_b_2d_array_v4i16_zero
6576            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6577            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6578           (SUST_B_2D_ARRAY_V4B16_ZERO Int64Regs:$s,
6579            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6580            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6581
6582 def : Pat<(int_nvvm_sust_b_2d_array_v4i32_zero
6583            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6584            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6585           (SUST_B_2D_ARRAY_V4B32_ZERO Int64Regs:$s, Int32Regs:$l,
6586            Int32Regs:$x, Int32Regs:$y,
6587            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6588
6589
6590
6591 def : Pat<(int_nvvm_sust_b_3d_i8_zero
6592            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6593            Int16Regs:$r),
6594           (SUST_B_3D_B8_ZERO Int64Regs:$s,
6595            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6596            Int16Regs:$r)>;
6597
6598 def : Pat<(int_nvvm_sust_b_3d_i16_zero
6599            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6600            Int16Regs:$r),
6601           (SUST_B_3D_B16_ZERO Int64Regs:$s,
6602            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6603            Int16Regs:$r)>;
6604
6605 def : Pat<(int_nvvm_sust_b_3d_i32_zero
6606            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6607            Int32Regs:$r),
6608           (SUST_B_3D_B32_ZERO Int64Regs:$s,
6609            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6610            Int32Regs:$r)>;
6611
6612 def : Pat<(int_nvvm_sust_b_3d_i64_zero
6613            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6614            Int64Regs:$r),
6615           (SUST_B_3D_B64_ZERO Int64Regs:$s,
6616            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6617            Int64Regs:$r)>;
6618
6619 def : Pat<(int_nvvm_sust_b_3d_v2i8_zero
6620            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6621            Int16Regs:$r, Int16Regs:$g),
6622           (SUST_B_3D_V2B8_ZERO Int64Regs:$s,
6623            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6624            Int16Regs:$r, Int16Regs:$g)>;
6625
6626 def : Pat<(int_nvvm_sust_b_3d_v2i16_zero
6627            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6628            Int16Regs:$r, Int16Regs:$g),
6629           (SUST_B_3D_V2B16_ZERO Int64Regs:$s,
6630            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6631            Int16Regs:$r, Int16Regs:$g)>;
6632
6633 def : Pat<(int_nvvm_sust_b_3d_v2i32_zero
6634            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6635            Int32Regs:$r, Int32Regs:$g),
6636           (SUST_B_3D_V2B32_ZERO Int64Regs:$s,
6637            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6638            Int32Regs:$r, Int32Regs:$g)>;
6639
6640 def : Pat<(int_nvvm_sust_b_3d_v2i64_zero
6641            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6642            Int64Regs:$r, Int64Regs:$g),
6643           (SUST_B_3D_V2B64_ZERO Int64Regs:$s,
6644            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6645            Int64Regs:$r, Int64Regs:$g)>;
6646
6647 def : Pat<(int_nvvm_sust_b_3d_v4i8_zero
6648            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6649            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6650           (SUST_B_3D_V4B8_ZERO Int64Regs:$s,
6651            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6652            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6653
6654 def : Pat<(int_nvvm_sust_b_3d_v4i16_zero
6655            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6656            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6657           (SUST_B_3D_V4B16_ZERO Int64Regs:$s,
6658            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6659            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6660
6661 def : Pat<(int_nvvm_sust_b_3d_v4i32_zero
6662            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6663            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6664           (SUST_B_3D_V4B32_ZERO Int64Regs:$s,
6665            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6666            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6667
6668
6669
6670
6671 def : Pat<(int_nvvm_sust_p_1d_i8_trap
6672            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
6673           (SUST_P_1D_B8_TRAP Int64Regs:$s, Int32Regs:$x, Int16Regs:$r)>;
6674
6675 def : Pat<(int_nvvm_sust_p_1d_i16_trap
6676            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
6677           (SUST_P_1D_B16_TRAP Int64Regs:$s, Int32Regs:$x, Int16Regs:$r)>;
6678
6679 def : Pat<(int_nvvm_sust_p_1d_i32_trap
6680            Int64Regs:$s, Int32Regs:$x, Int32Regs:$r),
6681           (SUST_P_1D_B32_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$r)>;
6682
6683 def : Pat<(int_nvvm_sust_p_1d_v2i8_trap
6684            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
6685           (SUST_P_1D_V2B8_TRAP Int64Regs:$s, Int32Regs:$x,
6686            Int16Regs:$r, Int16Regs:$g)>;
6687
6688 def : Pat<(int_nvvm_sust_p_1d_v2i16_trap
6689            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
6690           (SUST_P_1D_V2B16_TRAP Int64Regs:$s, Int32Regs:$x,
6691            Int16Regs:$r, Int16Regs:$g)>;
6692
6693 def : Pat<(int_nvvm_sust_p_1d_v2i32_trap
6694            Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
6695           (SUST_P_1D_V2B32_TRAP Int64Regs:$s, Int32Regs:$x,
6696            Int32Regs:$r, Int32Regs:$g)>;
6697
6698 def : Pat<(int_nvvm_sust_p_1d_v4i8_trap
6699            Int64Regs:$s, Int32Regs:$x,
6700            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6701           (SUST_P_1D_V4B8_TRAP Int64Regs:$s, Int32Regs:$x,
6702            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6703
6704 def : Pat<(int_nvvm_sust_p_1d_v4i16_trap
6705            Int64Regs:$s, Int32Regs:$x,
6706            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6707           (SUST_P_1D_V4B16_TRAP Int64Regs:$s, Int32Regs:$x,
6708            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6709
6710 def : Pat<(int_nvvm_sust_p_1d_v4i32_trap
6711            Int64Regs:$s, Int32Regs:$x,
6712            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6713           (SUST_P_1D_V4B32_TRAP Int64Regs:$s, Int32Regs:$x,
6714            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6715
6716
6717
6718 def : Pat<(int_nvvm_sust_p_1d_array_i8_trap
6719            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r),
6720           (SUST_P_1D_ARRAY_B8_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6721            Int16Regs:$r)>;
6722
6723 def : Pat<(int_nvvm_sust_p_1d_array_i16_trap
6724            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r),
6725           (SUST_P_1D_ARRAY_B16_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6726            Int16Regs:$r)>;
6727
6728 def : Pat<(int_nvvm_sust_p_1d_array_i32_trap
6729            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$r),
6730           (SUST_P_1D_ARRAY_B32_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6731            Int32Regs:$r)>;
6732
6733 def : Pat<(int_nvvm_sust_p_1d_array_v2i8_trap
6734           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
6735           (SUST_P_1D_ARRAY_V2B8_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6736            Int16Regs:$r, Int16Regs:$g)>;
6737
6738 def : Pat<(int_nvvm_sust_p_1d_array_v2i16_trap
6739           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
6740           (SUST_P_1D_ARRAY_V2B16_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6741            Int16Regs:$r, Int16Regs:$g)>;
6742
6743 def : Pat<(int_nvvm_sust_p_1d_array_v2i32_trap
6744           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
6745           (SUST_P_1D_ARRAY_V2B32_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6746            Int32Regs:$r, Int32Regs:$g)>;
6747
6748 def : Pat<(int_nvvm_sust_p_1d_array_v4i8_trap
6749            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6750            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6751           (SUST_P_1D_ARRAY_V4B8_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6752            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6753
6754 def : Pat<(int_nvvm_sust_p_1d_array_v4i16_trap
6755            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6756            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6757           (SUST_P_1D_ARRAY_V4B16_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6758            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6759
6760 def : Pat<(int_nvvm_sust_p_1d_array_v4i32_trap
6761            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6762            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6763           (SUST_P_1D_ARRAY_V4B32_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6764            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6765
6766
6767
6768 def : Pat<(int_nvvm_sust_p_2d_i8_trap
6769            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
6770           (SUST_P_2D_B8_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6771            Int16Regs:$r)>;
6772
6773 def : Pat<(int_nvvm_sust_p_2d_i16_trap
6774            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
6775           (SUST_P_2D_B16_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6776            Int16Regs:$r)>;
6777
6778 def : Pat<(int_nvvm_sust_p_2d_i32_trap
6779            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
6780           (SUST_P_2D_B32_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6781            Int32Regs:$r)>;
6782
6783 def : Pat<(int_nvvm_sust_p_2d_v2i8_trap
6784           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r, Int16Regs:$g),
6785           (SUST_P_2D_V2B8_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6786            Int16Regs:$r, Int16Regs:$g)>;
6787
6788 def : Pat<(int_nvvm_sust_p_2d_v2i16_trap
6789           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r, Int16Regs:$g),
6790           (SUST_P_2D_V2B16_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6791            Int16Regs:$r, Int16Regs:$g)>;
6792
6793 def : Pat<(int_nvvm_sust_p_2d_v2i32_trap
6794           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r, Int32Regs:$g),
6795           (SUST_P_2D_V2B32_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6796            Int32Regs:$r, Int32Regs:$g)>;
6797
6798 def : Pat<(int_nvvm_sust_p_2d_v4i8_trap
6799            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6800            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6801           (SUST_P_2D_V4B8_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6802            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6803
6804 def : Pat<(int_nvvm_sust_p_2d_v4i16_trap
6805            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6806            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6807           (SUST_P_2D_V4B16_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6808            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6809
6810 def : Pat<(int_nvvm_sust_p_2d_v4i32_trap
6811            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6812            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6813           (SUST_P_2D_V4B32_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6814            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6815
6816
6817
6818 def : Pat<(int_nvvm_sust_p_2d_array_i8_trap
6819           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
6820           (SUST_P_2D_ARRAY_B8_TRAP Int64Regs:$s,
6821            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6822            Int16Regs:$r)>;
6823
6824 def : Pat<(int_nvvm_sust_p_2d_array_i16_trap
6825           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
6826           (SUST_P_2D_ARRAY_B16_TRAP Int64Regs:$s,
6827            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6828            Int16Regs:$r)>;
6829
6830 def : Pat<(int_nvvm_sust_p_2d_array_i32_trap
6831           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
6832           (SUST_P_2D_ARRAY_B32_TRAP Int64Regs:$s,
6833            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6834            Int32Regs:$r)>;
6835
6836 def : Pat<(int_nvvm_sust_p_2d_array_v2i8_trap
6837            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6838            Int16Regs:$r, Int16Regs:$g),
6839           (SUST_P_2D_ARRAY_V2B8_TRAP Int64Regs:$s, Int32Regs:$l,
6840            Int32Regs:$x, Int32Regs:$y,
6841            Int16Regs:$r, Int16Regs:$g)>;
6842
6843 def : Pat<(int_nvvm_sust_p_2d_array_v2i16_trap
6844            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6845            Int16Regs:$r, Int16Regs:$g),
6846           (SUST_P_2D_ARRAY_V2B16_TRAP Int64Regs:$s, Int32Regs:$l,
6847            Int32Regs:$x, Int32Regs:$y,
6848            Int16Regs:$r, Int16Regs:$g)>;
6849
6850 def : Pat<(int_nvvm_sust_p_2d_array_v2i32_trap
6851            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
6852            Int32Regs:$g),
6853           (SUST_P_2D_ARRAY_V2B32_TRAP Int64Regs:$s, Int32Regs:$l,
6854            Int32Regs:$x, Int32Regs:$y, Int32Regs:$r, Int32Regs:$g)>;
6855
6856 def : Pat<(int_nvvm_sust_p_2d_array_v4i8_trap
6857            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6858            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6859           (SUST_P_2D_ARRAY_V4B8_TRAP Int64Regs:$s,
6860            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6861            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6862
6863 def : Pat<(int_nvvm_sust_p_2d_array_v4i16_trap
6864            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6865            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6866           (SUST_P_2D_ARRAY_V4B16_TRAP Int64Regs:$s,
6867            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6868            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6869
6870 def : Pat<(int_nvvm_sust_p_2d_array_v4i32_trap
6871            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6872            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6873           (SUST_P_2D_ARRAY_V4B32_TRAP Int64Regs:$s, Int32Regs:$l,
6874            Int32Regs:$x, Int32Regs:$y,
6875            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6876
6877
6878
6879 def : Pat<(int_nvvm_sust_p_3d_i8_trap
6880            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6881            Int16Regs:$r),
6882           (SUST_P_3D_B8_TRAP Int64Regs:$s,
6883            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6884            Int16Regs:$r)>;
6885
6886 def : Pat<(int_nvvm_sust_p_3d_i16_trap
6887            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6888            Int16Regs:$r),
6889           (SUST_P_3D_B16_TRAP Int64Regs:$s,
6890            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6891            Int16Regs:$r)>;
6892
6893 def : Pat<(int_nvvm_sust_p_3d_i32_trap
6894            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6895            Int32Regs:$r),
6896           (SUST_P_3D_B32_TRAP Int64Regs:$s,
6897            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6898            Int32Regs:$r)>;
6899
6900 def : Pat<(int_nvvm_sust_p_3d_v2i8_trap
6901            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6902            Int16Regs:$r, Int16Regs:$g),
6903           (SUST_P_3D_V2B8_TRAP Int64Regs:$s,
6904            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6905            Int16Regs:$r, Int16Regs:$g)>;
6906
6907 def : Pat<(int_nvvm_sust_p_3d_v2i16_trap
6908            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6909            Int16Regs:$r, Int16Regs:$g),
6910           (SUST_P_3D_V2B16_TRAP Int64Regs:$s,
6911            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6912            Int16Regs:$r, Int16Regs:$g)>;
6913
6914 def : Pat<(int_nvvm_sust_p_3d_v2i32_trap
6915            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6916            Int32Regs:$r, Int32Regs:$g),
6917           (SUST_P_3D_V2B32_TRAP Int64Regs:$s,
6918            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6919            Int32Regs:$r, Int32Regs:$g)>;
6920
6921 def : Pat<(int_nvvm_sust_p_3d_v4i8_trap
6922            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6923            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6924           (SUST_P_3D_V4B8_TRAP Int64Regs:$s,
6925            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6926            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6927
6928 def : Pat<(int_nvvm_sust_p_3d_v4i16_trap
6929            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6930            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6931           (SUST_P_3D_V4B16_TRAP Int64Regs:$s,
6932            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6933            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6934
6935 def : Pat<(int_nvvm_sust_p_3d_v4i32_trap
6936            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6937            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6938           (SUST_P_3D_V4B32_TRAP Int64Regs:$s,
6939            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6940            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6941
6942
6943
6944 //===-- Old PTX Back-end Intrinsics ---------------------------------------===//
6945
6946 // These intrinsics are handled to retain compatibility with the old backend.
6947
6948 // PTX Special Purpose Register Accessor Intrinsics
6949
6950 class PTX_READ_SPECIAL_REGISTER_R64<string regname, Intrinsic intop>
6951   : NVPTXInst<(outs Int64Regs:$d), (ins),
6952               !strconcat(!strconcat("mov.u64\t$d, %", regname), ";"),
6953               [(set Int64Regs:$d, (intop))]>;
6954
6955 class PTX_READ_SPECIAL_REGISTER_R32<string regname, Intrinsic intop>
6956   : NVPTXInst<(outs Int32Regs:$d), (ins),
6957               !strconcat(!strconcat("mov.u32\t$d, %", regname), ";"),
6958               [(set Int32Regs:$d, (intop))]>;
6959
6960 // TODO Add read vector-version of special registers
6961
6962 def PTX_READ_TID_X   : PTX_READ_SPECIAL_REGISTER_R32<"tid.x",
6963                                                      int_ptx_read_tid_x>;
6964 def PTX_READ_TID_Y   : PTX_READ_SPECIAL_REGISTER_R32<"tid.y",
6965                                                      int_ptx_read_tid_y>;
6966 def PTX_READ_TID_Z   : PTX_READ_SPECIAL_REGISTER_R32<"tid.z",
6967                                                      int_ptx_read_tid_z>;
6968 def PTX_READ_TID_W   : PTX_READ_SPECIAL_REGISTER_R32<"tid.w",
6969                                                      int_ptx_read_tid_w>;
6970
6971 def PTX_READ_NTID_X   : PTX_READ_SPECIAL_REGISTER_R32<"ntid.x",
6972                                                       int_ptx_read_ntid_x>;
6973 def PTX_READ_NTID_Y   : PTX_READ_SPECIAL_REGISTER_R32<"ntid.y",
6974                                                       int_ptx_read_ntid_y>;
6975 def PTX_READ_NTID_Z   : PTX_READ_SPECIAL_REGISTER_R32<"ntid.z",
6976                                                       int_ptx_read_ntid_z>;
6977 def PTX_READ_NTID_W   : PTX_READ_SPECIAL_REGISTER_R32<"ntid.w",
6978                                                       int_ptx_read_ntid_w>;
6979
6980 def PTX_READ_LANEID  : PTX_READ_SPECIAL_REGISTER_R32<"laneid",
6981                                                      int_ptx_read_laneid>;
6982 def PTX_READ_WARPID  : PTX_READ_SPECIAL_REGISTER_R32<"warpid",
6983                                                      int_ptx_read_warpid>;
6984 def PTX_READ_NWARPID : PTX_READ_SPECIAL_REGISTER_R32<"nwarpid",
6985                                                      int_ptx_read_nwarpid>;
6986
6987 def PTX_READ_CTAID_X   : PTX_READ_SPECIAL_REGISTER_R32<"ctaid.x",
6988                                                        int_ptx_read_ctaid_x>;
6989 def PTX_READ_CTAID_Y   : PTX_READ_SPECIAL_REGISTER_R32<"ctaid.y",
6990                                                        int_ptx_read_ctaid_y>;
6991 def PTX_READ_CTAID_Z   : PTX_READ_SPECIAL_REGISTER_R32<"ctaid.z",
6992                                                        int_ptx_read_ctaid_z>;
6993 def PTX_READ_CTAID_W   : PTX_READ_SPECIAL_REGISTER_R32<"ctaid.w",
6994                                                        int_ptx_read_ctaid_w>;
6995
6996 def PTX_READ_NCTAID_X   : PTX_READ_SPECIAL_REGISTER_R32<"nctaid.x",
6997                                                         int_ptx_read_nctaid_x>;
6998 def PTX_READ_NCTAID_Y   : PTX_READ_SPECIAL_REGISTER_R32<"nctaid.y",
6999                                                         int_ptx_read_nctaid_y>;
7000 def PTX_READ_NCTAID_Z   : PTX_READ_SPECIAL_REGISTER_R32<"nctaid.z",
7001                                                         int_ptx_read_nctaid_z>;
7002 def PTX_READ_NCTAID_W   : PTX_READ_SPECIAL_REGISTER_R32<"nctaid.w",
7003                                                         int_ptx_read_nctaid_w>;
7004
7005 def PTX_READ_SMID  : PTX_READ_SPECIAL_REGISTER_R32<"smid",
7006                                                    int_ptx_read_smid>;
7007 def PTX_READ_NSMID  : PTX_READ_SPECIAL_REGISTER_R32<"nsmid",
7008                                                     int_ptx_read_nsmid>;
7009 def PTX_READ_GRIDID  : PTX_READ_SPECIAL_REGISTER_R32<"gridid",
7010                                                      int_ptx_read_gridid>;
7011
7012 def PTX_READ_LANEMASK_EQ
7013   : PTX_READ_SPECIAL_REGISTER_R32<"lanemask_eq", int_ptx_read_lanemask_eq>;
7014 def PTX_READ_LANEMASK_LE
7015   : PTX_READ_SPECIAL_REGISTER_R32<"lanemask_le", int_ptx_read_lanemask_le>;
7016 def PTX_READ_LANEMASK_LT
7017   : PTX_READ_SPECIAL_REGISTER_R32<"lanemask_lt", int_ptx_read_lanemask_lt>;
7018 def PTX_READ_LANEMASK_GE
7019   : PTX_READ_SPECIAL_REGISTER_R32<"lanemask_ge", int_ptx_read_lanemask_ge>;
7020 def PTX_READ_LANEMASK_GT
7021   : PTX_READ_SPECIAL_REGISTER_R32<"lanemask_gt", int_ptx_read_lanemask_gt>;
7022
7023 def PTX_READ_CLOCK
7024   : PTX_READ_SPECIAL_REGISTER_R32<"clock", int_ptx_read_clock>;
7025 def PTX_READ_CLOCK64
7026   : PTX_READ_SPECIAL_REGISTER_R64<"clock64", int_ptx_read_clock64>;
7027
7028 def PTX_READ_PM0 : PTX_READ_SPECIAL_REGISTER_R32<"pm0", int_ptx_read_pm0>;
7029 def PTX_READ_PM1 : PTX_READ_SPECIAL_REGISTER_R32<"pm1", int_ptx_read_pm1>;
7030 def PTX_READ_PM2 : PTX_READ_SPECIAL_REGISTER_R32<"pm2", int_ptx_read_pm2>;
7031 def PTX_READ_PM3 : PTX_READ_SPECIAL_REGISTER_R32<"pm3", int_ptx_read_pm3>;
7032
7033 // PTX Parallel Synchronization and Communication Intrinsics
7034
7035 def PTX_BAR_SYNC : NVPTXInst<(outs), (ins i32imm:$i), "bar.sync\t$i;",
7036                              [(int_ptx_bar_sync imm:$i)]>;