drivers/gpu/drm/radeon/evergreen_cs.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <drm/drmP.h>
  29 #include "radeon.h"
  30 #include "evergreend.h"
  31 #include "evergreen_reg_safe.h"
  32 #include "cayman_reg_safe.h"
  33
  34 #define MAX(a,b)                   (((a)>(b))?(a):(b))
  35 #define MIN(a,b)                   (((a)<(b))?(a):(b))
  36
  37 int r600_dma_cs_next_reloc(struct radeon_cs_parser *p,
  38                            struct radeon_cs_reloc **cs_reloc);
  39 struct evergreen_cs_track {
  40         u32                     group_size;
  41         u32                     nbanks;
  42         u32                     npipes;
  43         u32                     row_size;
  44         /* value we track */
  45         u32                     nsamples;               /* unused */
  46         struct radeon_bo        *cb_color_bo[12];
  47         u32                     cb_color_bo_offset[12];
  48         struct radeon_bo        *cb_color_fmask_bo[8];  /* unused */
  49         struct radeon_bo        *cb_color_cmask_bo[8];  /* unused */
  50         u32                     cb_color_info[12];
  51         u32                     cb_color_view[12];
  52         u32                     cb_color_pitch[12];
  53         u32                     cb_color_slice[12];
  54         u32                     cb_color_slice_idx[12];
  55         u32                     cb_color_attrib[12];
  56         u32                     cb_color_cmask_slice[8];/* unused */
  57         u32                     cb_color_fmask_slice[8];/* unused */
  58         u32                     cb_target_mask;
  59         u32                     cb_shader_mask; /* unused */
  60         u32                     vgt_strmout_config;
  61         u32                     vgt_strmout_buffer_config;
  62         struct radeon_bo        *vgt_strmout_bo[4];
  63         u32                     vgt_strmout_bo_offset[4];
  64         u32                     vgt_strmout_size[4];
  65         u32                     db_depth_control;
  66         u32                     db_depth_view;
  67         u32                     db_depth_slice;
  68         u32                     db_depth_size;
  69         u32                     db_z_info;
  70         u32                     db_z_read_offset;
  71         u32                     db_z_write_offset;
  72         struct radeon_bo        *db_z_read_bo;
  73         struct radeon_bo        *db_z_write_bo;
  74         u32                     db_s_info;
  75         u32                     db_s_read_offset;
  76         u32                     db_s_write_offset;
  77         struct radeon_bo        *db_s_read_bo;
  78         struct radeon_bo        *db_s_write_bo;
  79         bool                    sx_misc_kill_all_prims;
  80         bool                    cb_dirty;
  81         bool                    db_dirty;
  82         bool                    streamout_dirty;
  83         u32                     htile_offset;
  84         u32                     htile_surface;
  85         struct radeon_bo        *htile_bo;
  86 };
  87
  88 static u32 evergreen_cs_get_aray_mode(u32 tiling_flags)
  89 {
  90         if (tiling_flags & RADEON_TILING_MACRO)
  91                 return ARRAY_2D_TILED_THIN1;
  92         else if (tiling_flags & RADEON_TILING_MICRO)
  93                 return ARRAY_1D_TILED_THIN1;
  94         else
  95                 return ARRAY_LINEAR_GENERAL;
  96 }
  97
  98 static u32 evergreen_cs_get_num_banks(u32 nbanks)
  99 {
 100         switch (nbanks) {
 101         case 2:
 102                 return ADDR_SURF_2_BANK;
 103         case 4:
 104                 return ADDR_SURF_4_BANK;
 105         case 8:
 106         default:
 107                 return ADDR_SURF_8_BANK;
 108         case 16:
 109                 return ADDR_SURF_16_BANK;
 110         }
 111 }
 112
 113 static void evergreen_cs_track_init(struct evergreen_cs_track *track)
 114 {
 115         int i;
 116
 117         for (i = 0; i < 8; i++) {
 118                 track->cb_color_fmask_bo[i] = NULL;
 119                 track->cb_color_cmask_bo[i] = NULL;
 120                 track->cb_color_cmask_slice[i] = 0;
 121                 track->cb_color_fmask_slice[i] = 0;
 122         }
 123
 124         for (i = 0; i < 12; i++) {
 125                 track->cb_color_bo[i] = NULL;
 126                 track->cb_color_bo_offset[i] = 0xFFFFFFFF;
 127                 track->cb_color_info[i] = 0;
 128                 track->cb_color_view[i] = 0xFFFFFFFF;
 129                 track->cb_color_pitch[i] = 0;
 130                 track->cb_color_slice[i] = 0xfffffff;
 131                 track->cb_color_slice_idx[i] = 0;
 132         }
 133         track->cb_target_mask = 0xFFFFFFFF;
 134         track->cb_shader_mask = 0xFFFFFFFF;
 135         track->cb_dirty = true;
 136
 137         track->db_depth_slice = 0xffffffff;
 138         track->db_depth_view = 0xFFFFC000;
 139         track->db_depth_size = 0xFFFFFFFF;
 140         track->db_depth_control = 0xFFFFFFFF;
 141         track->db_z_info = 0xFFFFFFFF;
 142         track->db_z_read_offset = 0xFFFFFFFF;
 143         track->db_z_write_offset = 0xFFFFFFFF;
 144         track->db_z_read_bo = NULL;
 145         track->db_z_write_bo = NULL;
 146         track->db_s_info = 0xFFFFFFFF;
 147         track->db_s_read_offset = 0xFFFFFFFF;
 148         track->db_s_write_offset = 0xFFFFFFFF;
 149         track->db_s_read_bo = NULL;
 150         track->db_s_write_bo = NULL;
 151         track->db_dirty = true;
 152         track->htile_bo = NULL;
 153         track->htile_offset = 0xFFFFFFFF;
 154         track->htile_surface = 0;
 155
 156         for (i = 0; i < 4; i++) {
 157                 track->vgt_strmout_size[i] = 0;
 158                 track->vgt_strmout_bo[i] = NULL;
 159                 track->vgt_strmout_bo_offset[i] = 0xFFFFFFFF;
 160         }
 161         track->streamout_dirty = true;
 162         track->sx_misc_kill_all_prims = false;
 163 }
 164
 165 struct eg_surface {
 166         /* value gathered from cs */
 167         unsigned        nbx;
 168         unsigned        nby;
 169         unsigned        format;
 170         unsigned        mode;
 171         unsigned        nbanks;
 172         unsigned        bankw;
 173         unsigned        bankh;
 174         unsigned        tsplit;
 175         unsigned        mtilea;
 176         unsigned        nsamples;
 177         /* output value */
 178         unsigned        bpe;
 179         unsigned        layer_size;
 180         unsigned        palign;
 181         unsigned        halign;
 182         unsigned long   base_align;
 183 };
 184
 185 static int evergreen_surface_check_linear(struct radeon_cs_parser *p,
 186                                           struct eg_surface *surf,
 187                                           const char *prefix)
 188 {
 189         surf->layer_size = surf->nbx * surf->nby * surf->bpe * surf->nsamples;
 190         surf->base_align = surf->bpe;
 191         surf->palign = 1;
 192         surf->halign = 1;
 193         return 0;
 194 }
 195
 196 static int evergreen_surface_check_linear_aligned(struct radeon_cs_parser *p,
 197                                                   struct eg_surface *surf,
 198                                                   const char *prefix)
 199 {
 200         struct evergreen_cs_track *track = p->track;
 201         unsigned palign;
 202
 203         palign = MAX(64, track->group_size / surf->bpe);
 204         surf->layer_size = surf->nbx * surf->nby * surf->bpe * surf->nsamples;
 205         surf->base_align = track->group_size;
 206         surf->palign = palign;
 207         surf->halign = 1;
 208         if (surf->nbx & (palign - 1)) {
 209                 if (prefix) {
 210                         dev_warn(p->dev, "%s:%d %s pitch %d invalid must be aligned with %d\n",
 211                                  __func__, __LINE__, prefix, surf->nbx, palign);
 212                 }
 213                 return -EINVAL;
 214         }
 215         return 0;
 216 }
 217
 218 static int evergreen_surface_check_1d(struct radeon_cs_parser *p,
 219                                       struct eg_surface *surf,
 220                                       const char *prefix)
 221 {
 222         struct evergreen_cs_track *track = p->track;
 223         unsigned palign;
 224
 225         palign = track->group_size / (8 * surf->bpe * surf->nsamples);
 226         palign = MAX(8, palign);
 227         surf->layer_size = surf->nbx * surf->nby * surf->bpe;
 228         surf->base_align = track->group_size;
 229         surf->palign = palign;
 230         surf->halign = 8;
 231         if ((surf->nbx & (palign - 1))) {
 232                 if (prefix) {
 233                         dev_warn(p->dev, "%s:%d %s pitch %d invalid must be aligned with %d (%d %d %d)\n",
 234                                  __func__, __LINE__, prefix, surf->nbx, palign,
 235                                  track->group_size, surf->bpe, surf->nsamples);
 236                 }
 237                 return -EINVAL;
 238         }
 239         if ((surf->nby & (8 - 1))) {
 240                 if (prefix) {
 241                         dev_warn(p->dev, "%s:%d %s height %d invalid must be aligned with 8\n",
 242                                  __func__, __LINE__, prefix, surf->nby);
 243                 }
 244                 return -EINVAL;
 245         }
 246         return 0;
 247 }
 248
 249 static int evergreen_surface_check_2d(struct radeon_cs_parser *p,
 250                                       struct eg_surface *surf,
 251                                       const char *prefix)
 252 {
 253         struct evergreen_cs_track *track = p->track;
 254         unsigned palign, halign, tileb, slice_pt;
 255         unsigned mtile_pr, mtile_ps, mtileb;
 256
 257         tileb = 64 * surf->bpe * surf->nsamples;
 258         slice_pt = 1;
 259         if (tileb > surf->tsplit) {
 260                 slice_pt = tileb / surf->tsplit;
 261         }
 262         tileb = tileb / slice_pt;
 263         /* macro tile width & height */
 264         palign = (8 * surf->bankw * track->npipes) * surf->mtilea;
 265         halign = (8 * surf->bankh * surf->nbanks) / surf->mtilea;
 266         mtileb = (palign / 8) * (halign / 8) * tileb;
 267         mtile_pr = surf->nbx / palign;
 268         mtile_ps = (mtile_pr * surf->nby) / halign;
 269         surf->layer_size = mtile_ps * mtileb * slice_pt;
 270         surf->base_align = (palign / 8) * (halign / 8) * tileb;
 271         surf->palign = palign;
 272         surf->halign = halign;
 273
 274         if ((surf->nbx & (palign - 1))) {
 275                 if (prefix) {
 276                         dev_warn(p->dev, "%s:%d %s pitch %d invalid must be aligned with %d\n",
 277                                  __func__, __LINE__, prefix, surf->nbx, palign);
 278                 }
 279                 return -EINVAL;
 280         }
 281         if ((surf->nby & (halign - 1))) {
 282                 if (prefix) {
 283                         dev_warn(p->dev, "%s:%d %s height %d invalid must be aligned with %d\n",
 284                                  __func__, __LINE__, prefix, surf->nby, halign);
 285                 }
 286                 return -EINVAL;
 287         }
 288
 289         return 0;
 290 }
 291
 292 static int evergreen_surface_check(struct radeon_cs_parser *p,
 293                                    struct eg_surface *surf,
 294                                    const char *prefix)
 295 {
 296         /* some common value computed here */
 297         surf->bpe = r600_fmt_get_blocksize(surf->format);
 298
 299         switch (surf->mode) {
 300         case ARRAY_LINEAR_GENERAL:
 301                 return evergreen_surface_check_linear(p, surf, prefix);
 302         case ARRAY_LINEAR_ALIGNED:
 303                 return evergreen_surface_check_linear_aligned(p, surf, prefix);
 304         case ARRAY_1D_TILED_THIN1:
 305                 return evergreen_surface_check_1d(p, surf, prefix);
 306         case ARRAY_2D_TILED_THIN1:
 307                 return evergreen_surface_check_2d(p, surf, prefix);
 308         default:
 309                 dev_warn(p->dev, "%s:%d %s invalid array mode %d\n",
 310                                 __func__, __LINE__, prefix, surf->mode);
 311                 return -EINVAL;
 312         }
 313         return -EINVAL;
 314 }
 315
 316 static int evergreen_surface_value_conv_check(struct radeon_cs_parser *p,
 317                                               struct eg_surface *surf,
 318                                               const char *prefix)
 319 {
 320         switch (surf->mode) {
 321         case ARRAY_2D_TILED_THIN1:
 322                 break;
 323         case ARRAY_LINEAR_GENERAL:
 324         case ARRAY_LINEAR_ALIGNED:
 325         case ARRAY_1D_TILED_THIN1:
 326                 return 0;
 327         default:
 328                 dev_warn(p->dev, "%s:%d %s invalid array mode %d\n",
 329                                 __func__, __LINE__, prefix, surf->mode);
 330                 return -EINVAL;
 331         }
 332
 333         switch (surf->nbanks) {
 334         case 0: surf->nbanks = 2; break;
 335         case 1: surf->nbanks = 4; break;
 336         case 2: surf->nbanks = 8; break;
 337         case 3: surf->nbanks = 16; break;
 338         default:
 339                 dev_warn(p->dev, "%s:%d %s invalid number of banks %d\n",
 340                          __func__, __LINE__, prefix, surf->nbanks);
 341                 return -EINVAL;
 342         }
 343         switch (surf->bankw) {
 344         case 0: surf->bankw = 1; break;
 345         case 1: surf->bankw = 2; break;
 346         case 2: surf->bankw = 4; break;
 347         case 3: surf->bankw = 8; break;
 348         default:
 349                 dev_warn(p->dev, "%s:%d %s invalid bankw %d\n",
 350                          __func__, __LINE__, prefix, surf->bankw);
 351                 return -EINVAL;
 352         }
 353         switch (surf->bankh) {
 354         case 0: surf->bankh = 1; break;
 355         case 1: surf->bankh = 2; break;
 356         case 2: surf->bankh = 4; break;
 357         case 3: surf->bankh = 8; break;
 358         default:
 359                 dev_warn(p->dev, "%s:%d %s invalid bankh %d\n",
 360                          __func__, __LINE__, prefix, surf->bankh);
 361                 return -EINVAL;
 362         }
 363         switch (surf->mtilea) {
 364         case 0: surf->mtilea = 1; break;
 365         case 1: surf->mtilea = 2; break;
 366         case 2: surf->mtilea = 4; break;
 367         case 3: surf->mtilea = 8; break;
 368         default:
 369                 dev_warn(p->dev, "%s:%d %s invalid macro tile aspect %d\n",
 370                          __func__, __LINE__, prefix, surf->mtilea);
 371                 return -EINVAL;
 372         }
 373         switch (surf->tsplit) {
 374         case 0: surf->tsplit = 64; break;
 375         case 1: surf->tsplit = 128; break;
 376         case 2: surf->tsplit = 256; break;
 377         case 3: surf->tsplit = 512; break;
 378         case 4: surf->tsplit = 1024; break;
 379         case 5: surf->tsplit = 2048; break;
 380         case 6: surf->tsplit = 4096; break;
 381         default:
 382                 dev_warn(p->dev, "%s:%d %s invalid tile split %d\n",
 383                          __func__, __LINE__, prefix, surf->tsplit);
 384                 return -EINVAL;
 385         }
 386         return 0;
 387 }
 388
 389 static int evergreen_cs_track_validate_cb(struct radeon_cs_parser *p, unsigned id)
 390 {
 391         struct evergreen_cs_track *track = p->track;
 392         struct eg_surface surf;
 393         unsigned pitch, slice, mslice;
 394         unsigned long offset;
 395         int r;
 396
 397         mslice = G_028C6C_SLICE_MAX(track->cb_color_view[id]) + 1;
 398         pitch = track->cb_color_pitch[id];
 399         slice = track->cb_color_slice[id];
 400         surf.nbx = (pitch + 1) * 8;
 401         surf.nby = ((slice + 1) * 64) / surf.nbx;
 402         surf.mode = G_028C70_ARRAY_MODE(track->cb_color_info[id]);
 403         surf.format = G_028C70_FORMAT(track->cb_color_info[id]);
 404         surf.tsplit = G_028C74_TILE_SPLIT(track->cb_color_attrib[id]);
 405         surf.nbanks = G_028C74_NUM_BANKS(track->cb_color_attrib[id]);
 406         surf.bankw = G_028C74_BANK_WIDTH(track->cb_color_attrib[id]);
 407         surf.bankh = G_028C74_BANK_HEIGHT(track->cb_color_attrib[id]);
 408         surf.mtilea = G_028C74_MACRO_TILE_ASPECT(track->cb_color_attrib[id]);
 409         surf.nsamples = 1;
 410
 411         if (!r600_fmt_is_valid_color(surf.format)) {
 412                 dev_warn(p->dev, "%s:%d cb invalid format %d for %d (0x%08x)\n",
 413                          __func__, __LINE__, surf.format,
 414                         id, track->cb_color_info[id]);
 415                 return -EINVAL;
 416         }
 417
 418         r = evergreen_surface_value_conv_check(p, &surf, "cb");
 419         if (r) {
 420                 return r;
 421         }
 422
 423         r = evergreen_surface_check(p, &surf, "cb");
 424         if (r) {
 425                 dev_warn(p->dev, "%s:%d cb[%d] invalid (0x%08x 0x%08x 0x%08x 0x%08x)\n",
 426                          __func__, __LINE__, id, track->cb_color_pitch[id],
 427                          track->cb_color_slice[id], track->cb_color_attrib[id],
 428                          track->cb_color_info[id]);
 429                 return r;
 430         }
 431
 432         offset = track->cb_color_bo_offset[id] << 8;
 433         if (offset & (surf.base_align - 1)) {
 434                 dev_warn(p->dev, "%s:%d cb[%d] bo base %ld not aligned with %ld\n",
 435                          __func__, __LINE__, id, offset, surf.base_align);
 436                 return -EINVAL;
 437         }
 438
 439         offset += surf.layer_size * mslice;
 440         if (offset > radeon_bo_size(track->cb_color_bo[id])) {
 441                 /* old ddx are broken they allocate bo with w*h*bpp but
 442                  * program slice with ALIGN(h, 8), catch this and patch
 443                  * command stream.
 444                  */
 445                 if (!surf.mode) {
 446                         volatile u32 *ib = p->ib.ptr;
 447                         unsigned long tmp, nby, bsize, size, min = 0;
 448
 449                         /* find the height the ddx wants */
 450                         if (surf.nby > 8) {
 451                                 min = surf.nby - 8;
 452                         }
 453                         bsize = radeon_bo_size(track->cb_color_bo[id]);
 454                         tmp = track->cb_color_bo_offset[id] << 8;
 455                         for (nby = surf.nby; nby > min; nby--) {
 456                                 size = nby * surf.nbx * surf.bpe * surf.nsamples;
 457                                 if ((tmp + size * mslice) <= bsize) {
 458                                         break;
 459                                 }
 460                         }
 461                         if (nby > min) {
 462                                 surf.nby = nby;
 463                                 slice = ((nby * surf.nbx) / 64) - 1;
 464                                 if (!evergreen_surface_check(p, &surf, "cb")) {
 465                                         /* check if this one works */
 466                                         tmp += surf.layer_size * mslice;
 467                                         if (tmp <= bsize) {
 468                                                 ib[track->cb_color_slice_idx[id]] = slice;
 469                                                 goto old_ddx_ok;
 470                                         }
 471                                 }
 472                         }
 473                 }
 474                 dev_warn(p->dev, "%s:%d cb[%d] bo too small (layer size %d, "
 475                          "offset %d, max layer %d, bo size %ld, slice %d)\n",
 476                          __func__, __LINE__, id, surf.layer_size,
 477                         track->cb_color_bo_offset[id] << 8, mslice,
 478                         radeon_bo_size(track->cb_color_bo[id]), slice);
 479                 dev_warn(p->dev, "%s:%d problematic surf: (%d %d) (%d %d %d %d %d %d %d)\n",
 480                          __func__, __LINE__, surf.nbx, surf.nby,
 481                         surf.mode, surf.bpe, surf.nsamples,
 482                         surf.bankw, surf.bankh,
 483                         surf.tsplit, surf.mtilea);
 484                 return -EINVAL;
 485         }
 486 old_ddx_ok:
 487
 488         return 0;
 489 }
 490
 491 static int evergreen_cs_track_validate_htile(struct radeon_cs_parser *p,
 492                                                 unsigned nbx, unsigned nby)
 493 {
 494         struct evergreen_cs_track *track = p->track;
 495         unsigned long size;
 496
 497         if (track->htile_bo == NULL) {
 498                 dev_warn(p->dev, "%s:%d htile enabled without htile surface 0x%08x\n",
 499                                 __func__, __LINE__, track->db_z_info);
 500                 return -EINVAL;
 501         }
 502
 503         if (G_028ABC_LINEAR(track->htile_surface)) {
 504                 /* pitch must be 16 htiles aligned == 16 * 8 pixel aligned */
 505                 nbx = round_up(nbx, 16 * 8);
 506                 /* height is npipes htiles aligned == npipes * 8 pixel aligned */
 507                 nby = round_up(nby, track->npipes * 8);
 508         } else {
 509                 /* always assume 8x8 htile */
 510                 /* align is htile align * 8, htile align vary according to
 511                  * number of pipe and tile width and nby
 512                  */
 513                 switch (track->npipes) {
 514                 case 8:
 515                         /* HTILE_WIDTH = 8 & HTILE_HEIGHT = 8*/
 516                         nbx = round_up(nbx, 64 * 8);
 517                         nby = round_up(nby, 64 * 8);
 518                         break;
 519                 case 4:
 520                         /* HTILE_WIDTH = 8 & HTILE_HEIGHT = 8*/
 521                         nbx = round_up(nbx, 64 * 8);
 522                         nby = round_up(nby, 32 * 8);
 523                         break;
 524                 case 2:
 525                         /* HTILE_WIDTH = 8 & HTILE_HEIGHT = 8*/
 526                         nbx = round_up(nbx, 32 * 8);
 527                         nby = round_up(nby, 32 * 8);
 528                         break;
 529                 case 1:
 530                         /* HTILE_WIDTH = 8 & HTILE_HEIGHT = 8*/
 531                         nbx = round_up(nbx, 32 * 8);
 532                         nby = round_up(nby, 16 * 8);
 533                         break;
 534                 default:
 535                         dev_warn(p->dev, "%s:%d invalid num pipes %d\n",
 536                                         __func__, __LINE__, track->npipes);
 537                         return -EINVAL;
 538                 }
 539         }
 540         /* compute number of htile */
 541         nbx = nbx >> 3;
 542         nby = nby >> 3;
 543         /* size must be aligned on npipes * 2K boundary */
 544         size = roundup(nbx * nby * 4, track->npipes * (2 << 10));
 545         size += track->htile_offset;
 546
 547         if (size > radeon_bo_size(track->htile_bo)) {
 548                 dev_warn(p->dev, "%s:%d htile surface too small %ld for %ld (%d %d)\n",
 549                                 __func__, __LINE__, radeon_bo_size(track->htile_bo),
 550                                 size, nbx, nby);
 551                 return -EINVAL;
 552         }
 553         return 0;
 554 }
 555
 556 static int evergreen_cs_track_validate_stencil(struct radeon_cs_parser *p)
 557 {
 558         struct evergreen_cs_track *track = p->track;
 559         struct eg_surface surf;
 560         unsigned pitch, slice, mslice;
 561         unsigned long offset;
 562         int r;
 563
 564         mslice = G_028008_SLICE_MAX(track->db_depth_view) + 1;
 565         pitch = G_028058_PITCH_TILE_MAX(track->db_depth_size);
 566         slice = track->db_depth_slice;
 567         surf.nbx = (pitch + 1) * 8;
 568         surf.nby = ((slice + 1) * 64) / surf.nbx;
 569         surf.mode = G_028040_ARRAY_MODE(track->db_z_info);
 570         surf.format = G_028044_FORMAT(track->db_s_info);
 571         surf.tsplit = G_028044_TILE_SPLIT(track->db_s_info);
 572         surf.nbanks = G_028040_NUM_BANKS(track->db_z_info);
 573         surf.bankw = G_028040_BANK_WIDTH(track->db_z_info);
 574         surf.bankh = G_028040_BANK_HEIGHT(track->db_z_info);
 575         surf.mtilea = G_028040_MACRO_TILE_ASPECT(track->db_z_info);
 576         surf.nsamples = 1;
 577
 578         if (surf.format != 1) {
 579                 dev_warn(p->dev, "%s:%d stencil invalid format %d\n",
 580                          __func__, __LINE__, surf.format);
 581                 return -EINVAL;
 582         }
 583         /* replace by color format so we can use same code */
 584         surf.format = V_028C70_COLOR_8;
 585
 586         r = evergreen_surface_value_conv_check(p, &surf, "stencil");
 587         if (r) {
 588                 return r;
 589         }
 590
 591         r = evergreen_surface_check(p, &surf, NULL);
 592         if (r) {
 593                 /* old userspace doesn't compute proper depth/stencil alignment
 594                  * check that alignment against a bigger byte per elements and
 595                  * only report if that alignment is wrong too.
 596                  */
 597                 surf.format = V_028C70_COLOR_8_8_8_8;
 598                 r = evergreen_surface_check(p, &surf, "stencil");
 599                 if (r) {
 600                         dev_warn(p->dev, "%s:%d stencil invalid (0x%08x 0x%08x 0x%08x 0x%08x)\n",
 601                                  __func__, __LINE__, track->db_depth_size,
 602                                  track->db_depth_slice, track->db_s_info, track->db_z_info);
 603                 }
 604                 return r;
 605         }
 606
 607         offset = track->db_s_read_offset << 8;
 608         if (offset & (surf.base_align - 1)) {
 609                 dev_warn(p->dev, "%s:%d stencil read bo base %ld not aligned with %ld\n",
 610                          __func__, __LINE__, offset, surf.base_align);
 611                 return -EINVAL;
 612         }
 613         offset += surf.layer_size * mslice;
 614         if (offset > radeon_bo_size(track->db_s_read_bo)) {
 615                 dev_warn(p->dev, "%s:%d stencil read bo too small (layer size %d, "
 616                          "offset %ld, max layer %d, bo size %ld)\n",
 617                          __func__, __LINE__, surf.layer_size,
 618                         (unsigned long)track->db_s_read_offset << 8, mslice,
 619                         radeon_bo_size(track->db_s_read_bo));
 620                 dev_warn(p->dev, "%s:%d stencil invalid (0x%08x 0x%08x 0x%08x 0x%08x)\n",
 621                          __func__, __LINE__, track->db_depth_size,
 622                          track->db_depth_slice, track->db_s_info, track->db_z_info);
 623                 return -EINVAL;
 624         }
 625
 626         offset = track->db_s_write_offset << 8;
 627         if (offset & (surf.base_align - 1)) {
 628                 dev_warn(p->dev, "%s:%d stencil write bo base %ld not aligned with %ld\n",
 629                          __func__, __LINE__, offset, surf.base_align);
 630                 return -EINVAL;
 631         }
 632         offset += surf.layer_size * mslice;
 633         if (offset > radeon_bo_size(track->db_s_write_bo)) {
 634                 dev_warn(p->dev, "%s:%d stencil write bo too small (layer size %d, "
 635                          "offset %ld, max layer %d, bo size %ld)\n",
 636                          __func__, __LINE__, surf.layer_size,
 637                         (unsigned long)track->db_s_write_offset << 8, mslice,
 638                         radeon_bo_size(track->db_s_write_bo));
 639                 return -EINVAL;
 640         }
 641
 642         /* hyperz */
 643         if (G_028040_TILE_SURFACE_ENABLE(track->db_z_info)) {
 644                 r = evergreen_cs_track_validate_htile(p, surf.nbx, surf.nby);
 645                 if (r) {
 646                         return r;
 647                 }
 648         }
 649
 650         return 0;
 651 }
 652
 653 static int evergreen_cs_track_validate_depth(struct radeon_cs_parser *p)
 654 {
 655         struct evergreen_cs_track *track = p->track;
 656         struct eg_surface surf;
 657         unsigned pitch, slice, mslice;
 658         unsigned long offset;
 659         int r;
 660
 661         mslice = G_028008_SLICE_MAX(track->db_depth_view) + 1;
 662         pitch = G_028058_PITCH_TILE_MAX(track->db_depth_size);
 663         slice = track->db_depth_slice;
 664         surf.nbx = (pitch + 1) * 8;
 665         surf.nby = ((slice + 1) * 64) / surf.nbx;
 666         surf.mode = G_028040_ARRAY_MODE(track->db_z_info);
 667         surf.format = G_028040_FORMAT(track->db_z_info);
 668         surf.tsplit = G_028040_TILE_SPLIT(track->db_z_info);
 669         surf.nbanks = G_028040_NUM_BANKS(track->db_z_info);
 670         surf.bankw = G_028040_BANK_WIDTH(track->db_z_info);
 671         surf.bankh = G_028040_BANK_HEIGHT(track->db_z_info);
 672         surf.mtilea = G_028040_MACRO_TILE_ASPECT(track->db_z_info);
 673         surf.nsamples = 1;
 674
 675         switch (surf.format) {
 676         case V_028040_Z_16:
 677                 surf.format = V_028C70_COLOR_16;
 678                 break;
 679         case V_028040_Z_24:
 680         case V_028040_Z_32_FLOAT:
 681                 surf.format = V_028C70_COLOR_8_8_8_8;
 682                 break;
 683         default:
 684                 dev_warn(p->dev, "%s:%d depth invalid format %d\n",
 685                          __func__, __LINE__, surf.format);
 686                 return -EINVAL;
 687         }
 688
 689         r = evergreen_surface_value_conv_check(p, &surf, "depth");
 690         if (r) {
 691                 dev_warn(p->dev, "%s:%d depth invalid (0x%08x 0x%08x 0x%08x)\n",
 692                          __func__, __LINE__, track->db_depth_size,
 693                          track->db_depth_slice, track->db_z_info);
 694                 return r;
 695         }
 696
 697         r = evergreen_surface_check(p, &surf, "depth");
 698         if (r) {
 699                 dev_warn(p->dev, "%s:%d depth invalid (0x%08x 0x%08x 0x%08x)\n",
 700                          __func__, __LINE__, track->db_depth_size,
 701                          track->db_depth_slice, track->db_z_info);
 702                 return r;
 703         }
 704
 705         offset = track->db_z_read_offset << 8;
 706         if (offset & (surf.base_align - 1)) {
 707                 dev_warn(p->dev, "%s:%d stencil read bo base %ld not aligned with %ld\n",
 708                          __func__, __LINE__, offset, surf.base_align);
 709                 return -EINVAL;
 710         }
 711         offset += surf.layer_size * mslice;
 712         if (offset > radeon_bo_size(track->db_z_read_bo)) {
 713                 dev_warn(p->dev, "%s:%d depth read bo too small (layer size %d, "
 714                          "offset %ld, max layer %d, bo size %ld)\n",
 715                          __func__, __LINE__, surf.layer_size,
 716                         (unsigned long)track->db_z_read_offset << 8, mslice,
 717                         radeon_bo_size(track->db_z_read_bo));
 718                 return -EINVAL;
 719         }
 720
 721         offset = track->db_z_write_offset << 8;
 722         if (offset & (surf.base_align - 1)) {
 723                 dev_warn(p->dev, "%s:%d stencil write bo base %ld not aligned with %ld\n",
 724                          __func__, __LINE__, offset, surf.base_align);
 725                 return -EINVAL;
 726         }
 727         offset += surf.layer_size * mslice;
 728         if (offset > radeon_bo_size(track->db_z_write_bo)) {
 729                 dev_warn(p->dev, "%s:%d depth write bo too small (layer size %d, "
 730                          "offset %ld, max layer %d, bo size %ld)\n",
 731                          __func__, __LINE__, surf.layer_size,
 732                         (unsigned long)track->db_z_write_offset << 8, mslice,
 733                         radeon_bo_size(track->db_z_write_bo));
 734                 return -EINVAL;
 735         }
 736
 737         /* hyperz */
 738         if (G_028040_TILE_SURFACE_ENABLE(track->db_z_info)) {
 739                 r = evergreen_cs_track_validate_htile(p, surf.nbx, surf.nby);
 740                 if (r) {
 741                         return r;
 742                 }
 743         }
 744
 745         return 0;
 746 }
 747
 748 static int evergreen_cs_track_validate_texture(struct radeon_cs_parser *p,
 749                                                struct radeon_bo *texture,
 750                                                struct radeon_bo *mipmap,
 751                                                unsigned idx)
 752 {
 753         struct eg_surface surf;
 754         unsigned long toffset, moffset;
 755         unsigned dim, llevel, mslice, width, height, depth, i;
 756         u32 texdw[8];
 757         int r;
 758
 759         texdw[0] = radeon_get_ib_value(p, idx + 0);
 760         texdw[1] = radeon_get_ib_value(p, idx + 1);
 761         texdw[2] = radeon_get_ib_value(p, idx + 2);
 762         texdw[3] = radeon_get_ib_value(p, idx + 3);
 763         texdw[4] = radeon_get_ib_value(p, idx + 4);
 764         texdw[5] = radeon_get_ib_value(p, idx + 5);
 765         texdw[6] = radeon_get_ib_value(p, idx + 6);
 766         texdw[7] = radeon_get_ib_value(p, idx + 7);
 767         dim = G_030000_DIM(texdw[0]);
 768         llevel = G_030014_LAST_LEVEL(texdw[5]);
 769         mslice = G_030014_LAST_ARRAY(texdw[5]) + 1;
 770         width = G_030000_TEX_WIDTH(texdw[0]) + 1;
 771         height =  G_030004_TEX_HEIGHT(texdw[1]) + 1;
 772         depth = G_030004_TEX_DEPTH(texdw[1]) + 1;
 773         surf.format = G_03001C_DATA_FORMAT(texdw[7]);
 774         surf.nbx = (G_030000_PITCH(texdw[0]) + 1) * 8;
 775         surf.nbx = r600_fmt_get_nblocksx(surf.format, surf.nbx);
 776         surf.nby = r600_fmt_get_nblocksy(surf.format, height);
 777         surf.mode = G_030004_ARRAY_MODE(texdw[1]);
 778         surf.tsplit = G_030018_TILE_SPLIT(texdw[6]);
 779         surf.nbanks = G_03001C_NUM_BANKS(texdw[7]);
 780         surf.bankw = G_03001C_BANK_WIDTH(texdw[7]);
 781         surf.bankh = G_03001C_BANK_HEIGHT(texdw[7]);
 782         surf.mtilea = G_03001C_MACRO_TILE_ASPECT(texdw[7]);
 783         surf.nsamples = 1;
 784         toffset = texdw[2] << 8;
 785         moffset = texdw[3] << 8;
 786
 787         if (!r600_fmt_is_valid_texture(surf.format, p->family)) {
 788                 dev_warn(p->dev, "%s:%d texture invalid format %d\n",
 789                          __func__, __LINE__, surf.format);
 790                 return -EINVAL;
 791         }
 792         switch (dim) {
 793         case V_030000_SQ_TEX_DIM_1D:
 794         case V_030000_SQ_TEX_DIM_2D:
 795         case V_030000_SQ_TEX_DIM_CUBEMAP:
 796         case V_030000_SQ_TEX_DIM_1D_ARRAY:
 797         case V_030000_SQ_TEX_DIM_2D_ARRAY:
 798                 depth = 1;
 799                 break;
 800         case V_030000_SQ_TEX_DIM_2D_MSAA:
 801         case V_030000_SQ_TEX_DIM_2D_ARRAY_MSAA:
 802                 surf.nsamples = 1 << llevel;
 803                 llevel = 0;
 804                 depth = 1;
 805                 break;
 806         case V_030000_SQ_TEX_DIM_3D:
 807                 break;
 808         default:
 809                 dev_warn(p->dev, "%s:%d texture invalid dimension %d\n",
 810                          __func__, __LINE__, dim);
 811                 return -EINVAL;
 812         }
 813
 814         r = evergreen_surface_value_conv_check(p, &surf, "texture");
 815         if (r) {
 816                 return r;
 817         }
 818
 819         /* align height */
 820         evergreen_surface_check(p, &surf, NULL);
 821         surf.nby = ALIGN(surf.nby, surf.halign);
 822
 823         r = evergreen_surface_check(p, &surf, "texture");
 824         if (r) {
 825                 dev_warn(p->dev, "%s:%d texture invalid 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
 826                          __func__, __LINE__, texdw[0], texdw[1], texdw[4],
 827                          texdw[5], texdw[6], texdw[7]);
 828                 return r;
 829         }
 830
 831         /* check texture size */
 832         if (toffset & (surf.base_align - 1)) {
 833                 dev_warn(p->dev, "%s:%d texture bo base %ld not aligned with %ld\n",
 834                          __func__, __LINE__, toffset, surf.base_align);
 835                 return -EINVAL;
 836         }
 837         if (surf.nsamples <= 1 && moffset & (surf.base_align - 1)) {
 838                 dev_warn(p->dev, "%s:%d mipmap bo base %ld not aligned with %ld\n",
 839                          __func__, __LINE__, moffset, surf.base_align);
 840                 return -EINVAL;
 841         }
 842         if (dim == SQ_TEX_DIM_3D) {
 843                 toffset += surf.layer_size * depth;
 844         } else {
 845                 toffset += surf.layer_size * mslice;
 846         }
 847         if (toffset > radeon_bo_size(texture)) {
 848                 dev_warn(p->dev, "%s:%d texture bo too small (layer size %d, "
 849                          "offset %ld, max layer %d, depth %d, bo size %ld) (%d %d)\n",
 850                          __func__, __LINE__, surf.layer_size,
 851                         (unsigned long)texdw[2] << 8, mslice,
 852                         depth, radeon_bo_size(texture),
 853                         surf.nbx, surf.nby);
 854                 return -EINVAL;
 855         }
 856
 857         if (!mipmap) {
 858                 if (llevel) {
 859                         dev_warn(p->dev, "%s:%i got NULL MIP_ADDRESS relocation\n",
 860                                  __func__, __LINE__);
 861                         return -EINVAL;
 862                 } else {
 863                         return 0; /* everything's ok */
 864                 }
 865         }
 866
 867         /* check mipmap size */
 868         for (i = 1; i <= llevel; i++) {
 869                 unsigned w, h, d;
 870
 871                 w = r600_mip_minify(width, i);
 872                 h = r600_mip_minify(height, i);
 873                 d = r600_mip_minify(depth, i);
 874                 surf.nbx = r600_fmt_get_nblocksx(surf.format, w);
 875                 surf.nby = r600_fmt_get_nblocksy(surf.format, h);
 876
 877                 switch (surf.mode) {
 878                 case ARRAY_2D_TILED_THIN1:
 879                         if (surf.nbx < surf.palign || surf.nby < surf.halign) {
 880                                 surf.mode = ARRAY_1D_TILED_THIN1;
 881                         }
 882                         /* recompute alignment */
 883                         evergreen_surface_check(p, &surf, NULL);
 884                         break;
 885                 case ARRAY_LINEAR_GENERAL:
 886                 case ARRAY_LINEAR_ALIGNED:
 887                 case ARRAY_1D_TILED_THIN1:
 888                         break;
 889                 default:
 890                         dev_warn(p->dev, "%s:%d invalid array mode %d\n",
 891                                  __func__, __LINE__, surf.mode);
 892                         return -EINVAL;
 893                 }
 894                 surf.nbx = ALIGN(surf.nbx, surf.palign);
 895                 surf.nby = ALIGN(surf.nby, surf.halign);
 896
 897                 r = evergreen_surface_check(p, &surf, "mipmap");
 898                 if (r) {
 899                         return r;
 900                 }
 901
 902                 if (dim == SQ_TEX_DIM_3D) {
 903                         moffset += surf.layer_size * d;
 904                 } else {
 905                         moffset += surf.layer_size * mslice;
 906                 }
 907                 if (moffset > radeon_bo_size(mipmap)) {
 908                         dev_warn(p->dev, "%s:%d mipmap [%d] bo too small (layer size %d, "
 909                                         "offset %ld, coffset %ld, max layer %d, depth %d, "
 910                                         "bo size %ld) level0 (%d %d %d)\n",
 911                                         __func__, __LINE__, i, surf.layer_size,
 912                                         (unsigned long)texdw[3] << 8, moffset, mslice,
 913                                         d, radeon_bo_size(mipmap),
 914                                         width, height, depth);
 915                         dev_warn(p->dev, "%s:%d problematic surf: (%d %d) (%d %d %d %d %d %d %d)\n",
 916                                  __func__, __LINE__, surf.nbx, surf.nby,
 917                                 surf.mode, surf.bpe, surf.nsamples,
 918                                 surf.bankw, surf.bankh,
 919                                 surf.tsplit, surf.mtilea);
 920                         return -EINVAL;
 921                 }
 922         }
 923
 924         return 0;
 925 }
 926
 927 static int evergreen_cs_track_check(struct radeon_cs_parser *p)
 928 {
 929         struct evergreen_cs_track *track = p->track;
 930         unsigned tmp, i;
 931         int r;
 932         unsigned buffer_mask = 0;
 933
 934         /* check streamout */
 935         if (track->streamout_dirty && track->vgt_strmout_config) {
 936                 for (i = 0; i < 4; i++) {
 937                         if (track->vgt_strmout_config & (1 << i)) {
 938                                 buffer_mask |= (track->vgt_strmout_buffer_config >> (i * 4)) & 0xf;
 939                         }
 940                 }
 941
 942                 for (i = 0; i < 4; i++) {
 943                         if (buffer_mask & (1 << i)) {
 944                                 if (track->vgt_strmout_bo[i]) {
 945                                         u64 offset = (u64)track->vgt_strmout_bo_offset[i] +
 946                                                         (u64)track->vgt_strmout_size[i];
 947                                         if (offset > radeon_bo_size(track->vgt_strmout_bo[i])) {
 948                                                 DRM_ERROR("streamout %d bo too small: 0x%llx, 0x%lx\n",
 949                                                           i, offset,
 950                                                           radeon_bo_size(track->vgt_strmout_bo[i]));
 951                                                 return -EINVAL;
 952                                         }
 953                                 } else {
 954                                         dev_warn(p->dev, "No buffer for streamout %d\n", i);
 955                                         return -EINVAL;
 956                                 }
 957                         }
 958                 }
 959                 track->streamout_dirty = false;
 960         }
 961
 962         if (track->sx_misc_kill_all_prims)
 963                 return 0;
 964
 965         /* check that we have a cb for each enabled target
 966          */
 967         if (track->cb_dirty) {
 968                 tmp = track->cb_target_mask;
 969                 for (i = 0; i < 8; i++) {
 970                         if ((tmp >> (i * 4)) & 0xF) {
 971                                 /* at least one component is enabled */
 972                                 if (track->cb_color_bo[i] == NULL) {
 973                                         dev_warn(p->dev, "%s:%d mask 0x%08X | 0x%08X no cb for %d\n",
 974                                                 __func__, __LINE__, track->cb_target_mask, track->cb_shader_mask, i);
 975                                         return -EINVAL;
 976                                 }
 977                                 /* check cb */
 978                                 r = evergreen_cs_track_validate_cb(p, i);
 979                                 if (r) {
 980                                         return r;
 981                                 }
 982                         }
 983                 }
 984                 track->cb_dirty = false;
 985         }
 986
 987         if (track->db_dirty) {
 988                 /* Check stencil buffer */
 989                 if (G_028044_FORMAT(track->db_s_info) != V_028044_STENCIL_INVALID &&
 990                     G_028800_STENCIL_ENABLE(track->db_depth_control)) {
 991                         r = evergreen_cs_track_validate_stencil(p);
 992                         if (r)
 993                                 return r;
 994                 }
 995                 /* Check depth buffer */
 996                 if (G_028040_FORMAT(track->db_z_info) != V_028040_Z_INVALID &&
 997                     G_028800_Z_ENABLE(track->db_depth_control)) {
 998                         r = evergreen_cs_track_validate_depth(p);
 999                         if (r)
1000                                 return r;
1001                 }
1002                 track->db_dirty = false;
1003         }
1004
1005         return 0;
1006 }
1007
1008 /**
1009  * evergreen_cs_packet_parse_vline() - parse userspace VLINE packet
1010  * @parser:             parser structure holding parsing context.
1011  *
1012  * This is an Evergreen(+)-specific function for parsing VLINE packets.
1013  * Real work is done by r600_cs_common_vline_parse function.
1014  * Here we just set up ASIC-specific register table and call
1015  * the common implementation function.
1016  */
1017 static int evergreen_cs_packet_parse_vline(struct radeon_cs_parser *p)
1018 {
1019
1020         static uint32_t vline_start_end[6] = {
1021                 EVERGREEN_VLINE_START_END + EVERGREEN_CRTC0_REGISTER_OFFSET,
1022                 EVERGREEN_VLINE_START_END + EVERGREEN_CRTC1_REGISTER_OFFSET,
1023                 EVERGREEN_VLINE_START_END + EVERGREEN_CRTC2_REGISTER_OFFSET,
1024                 EVERGREEN_VLINE_START_END + EVERGREEN_CRTC3_REGISTER_OFFSET,
1025                 EVERGREEN_VLINE_START_END + EVERGREEN_CRTC4_REGISTER_OFFSET,
1026                 EVERGREEN_VLINE_START_END + EVERGREEN_CRTC5_REGISTER_OFFSET
1027         };
1028         static uint32_t vline_status[6] = {
1029                 EVERGREEN_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
1030                 EVERGREEN_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
1031                 EVERGREEN_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
1032                 EVERGREEN_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
1033                 EVERGREEN_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
1034                 EVERGREEN_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET
1035         };
1036
1037         return r600_cs_common_vline_parse(p, vline_start_end, vline_status);
1038 }
1039
1040 static int evergreen_packet0_check(struct radeon_cs_parser *p,
1041                                    struct radeon_cs_packet *pkt,
1042                                    unsigned idx, unsigned reg)
1043 {
1044         int r;
1045
1046         switch (reg) {
1047         case EVERGREEN_VLINE_START_END:
1048                 r = evergreen_cs_packet_parse_vline(p);
1049                 if (r) {
1050                         DRM_ERROR("No reloc for ib[%d]=0x%04X\n",
1051                                         idx, reg);
1052                         return r;
1053                 }
1054                 break;
1055         default:
1056                 printk(KERN_ERR "Forbidden register 0x%04X in cs at %d\n",
1057                        reg, idx);
1058                 return -EINVAL;
1059         }
1060         return 0;
1061 }
1062
1063 static int evergreen_cs_parse_packet0(struct radeon_cs_parser *p,
1064                                       struct radeon_cs_packet *pkt)
1065 {
1066         unsigned reg, i;
1067         unsigned idx;
1068         int r;
1069
1070         idx = pkt->idx + 1;
1071         reg = pkt->reg;
1072         for (i = 0; i <= pkt->count; i++, idx++, reg += 4) {
1073                 r = evergreen_packet0_check(p, pkt, idx, reg);
1074                 if (r) {
1075                         return r;
1076                 }
1077         }
1078         return 0;
1079 }
1080
1081 /**
1082  * evergreen_cs_check_reg() - check if register is authorized or not
1083  * @parser: parser structure holding parsing context
1084  * @reg: register we are testing
1085  * @idx: index into the cs buffer
1086  *
1087  * This function will test against evergreen_reg_safe_bm and return 0
1088  * if register is safe. If register is not flag as safe this function
1089  * will test it against a list of register needind special handling.
1090  */
1091 static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
1092 {
1093         struct evergreen_cs_track *track = (struct evergreen_cs_track *)p->track;
1094         struct radeon_cs_reloc *reloc;
1095         u32 last_reg;
1096         u32 m, i, tmp, *ib;
1097         int r;
1098
1099         if (p->rdev->family >= CHIP_CAYMAN)
1100                 last_reg = ARRAY_SIZE(cayman_reg_safe_bm);
1101         else
1102                 last_reg = ARRAY_SIZE(evergreen_reg_safe_bm);
1103
1104         i = (reg >> 7);
1105         if (i >= last_reg) {
1106                 dev_warn(p->dev, "forbidden register 0x%08x at %d\n", reg, idx);
1107                 return -EINVAL;
1108         }
1109         m = 1 << ((reg >> 2) & 31);
1110         if (p->rdev->family >= CHIP_CAYMAN) {
1111                 if (!(cayman_reg_safe_bm[i] & m))
1112                         return 0;
1113         } else {
1114                 if (!(evergreen_reg_safe_bm[i] & m))
1115                         return 0;
1116         }
1117         ib = p->ib.ptr;
1118         switch (reg) {
1119         /* force following reg to 0 in an attempt to disable out buffer
1120          * which will need us to better understand how it works to perform
1121          * security check on it (Jerome)
1122          */
1123         case SQ_ESGS_RING_SIZE:
1124         case SQ_GSVS_RING_SIZE:
1125         case SQ_ESTMP_RING_SIZE:
1126         case SQ_GSTMP_RING_SIZE:
1127         case SQ_HSTMP_RING_SIZE:
1128         case SQ_LSTMP_RING_SIZE:
1129         case SQ_PSTMP_RING_SIZE:
1130         case SQ_VSTMP_RING_SIZE:
1131         case SQ_ESGS_RING_ITEMSIZE:
1132         case SQ_ESTMP_RING_ITEMSIZE:
1133         case SQ_GSTMP_RING_ITEMSIZE:
1134         case SQ_GSVS_RING_ITEMSIZE:
1135         case SQ_GS_VERT_ITEMSIZE:
1136         case SQ_GS_VERT_ITEMSIZE_1:
1137         case SQ_GS_VERT_ITEMSIZE_2:
1138         case SQ_GS_VERT_ITEMSIZE_3:
1139         case SQ_GSVS_RING_OFFSET_1:
1140         case SQ_GSVS_RING_OFFSET_2:
1141         case SQ_GSVS_RING_OFFSET_3:
1142         case SQ_HSTMP_RING_ITEMSIZE:
1143         case SQ_LSTMP_RING_ITEMSIZE:
1144         case SQ_PSTMP_RING_ITEMSIZE:
1145         case SQ_VSTMP_RING_ITEMSIZE:
1146         case VGT_TF_RING_SIZE:
1147                 /* get value to populate the IB don't remove */
1148                 /*tmp =radeon_get_ib_value(p, idx);
1149                   ib[idx] = 0;*/
1150                 break;
1151         case SQ_ESGS_RING_BASE:
1152         case SQ_GSVS_RING_BASE:
1153         case SQ_ESTMP_RING_BASE:
1154         case SQ_GSTMP_RING_BASE:
1155         case SQ_HSTMP_RING_BASE:
1156         case SQ_LSTMP_RING_BASE:
1157         case SQ_PSTMP_RING_BASE:
1158         case SQ_VSTMP_RING_BASE:
1159                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1160                 if (r) {
1161                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1162                                         "0x%04X\n", reg);
1163                         return -EINVAL;
1164                 }
1165                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1166                 break;
1167         case DB_DEPTH_CONTROL:
1168                 track->db_depth_control = radeon_get_ib_value(p, idx);
1169                 track->db_dirty = true;
1170                 break;
1171         case CAYMAN_DB_EQAA:
1172                 if (p->rdev->family < CHIP_CAYMAN) {
1173                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1174                                  "0x%04X\n", reg);
1175                         return -EINVAL;
1176                 }
1177                 break;
1178         case CAYMAN_DB_DEPTH_INFO:
1179                 if (p->rdev->family < CHIP_CAYMAN) {
1180                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1181                                  "0x%04X\n", reg);
1182                         return -EINVAL;
1183                 }
1184                 break;
1185         case DB_Z_INFO:
1186                 track->db_z_info = radeon_get_ib_value(p, idx);
1187                 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) {
1188                         r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1189                         if (r) {
1190                                 dev_warn(p->dev, "bad SET_CONTEXT_REG "
1191                                                 "0x%04X\n", reg);
1192                                 return -EINVAL;
1193                         }
1194                         ib[idx] &= ~Z_ARRAY_MODE(0xf);
1195                         track->db_z_info &= ~Z_ARRAY_MODE(0xf);
1196                         ib[idx] |= Z_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags));
1197                         track->db_z_info |= Z_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags));
1198                         if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) {
1199                                 unsigned bankw, bankh, mtaspect, tile_split;
1200
1201                                 evergreen_tiling_fields(reloc->lobj.tiling_flags,
1202                                                         &bankw, &bankh, &mtaspect,
1203                                                         &tile_split);
1204                                 ib[idx] |= DB_NUM_BANKS(evergreen_cs_get_num_banks(track->nbanks));
1205                                 ib[idx] |= DB_TILE_SPLIT(tile_split) |
1206                                                 DB_BANK_WIDTH(bankw) |
1207                                                 DB_BANK_HEIGHT(bankh) |
1208                                                 DB_MACRO_TILE_ASPECT(mtaspect);
1209                         }
1210                 }
1211                 track->db_dirty = true;
1212                 break;
1213         case DB_STENCIL_INFO:
1214                 track->db_s_info = radeon_get_ib_value(p, idx);
1215                 track->db_dirty = true;
1216                 break;
1217         case DB_DEPTH_VIEW:
1218                 track->db_depth_view = radeon_get_ib_value(p, idx);
1219                 track->db_dirty = true;
1220                 break;
1221         case DB_DEPTH_SIZE:
1222                 track->db_depth_size = radeon_get_ib_value(p, idx);
1223                 track->db_dirty = true;
1224                 break;
1225         case R_02805C_DB_DEPTH_SLICE:
1226                 track->db_depth_slice = radeon_get_ib_value(p, idx);
1227                 track->db_dirty = true;
1228                 break;
1229         case DB_Z_READ_BASE:
1230                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1231                 if (r) {
1232                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1233                                         "0x%04X\n", reg);
1234                         return -EINVAL;
1235                 }
1236                 track->db_z_read_offset = radeon_get_ib_value(p, idx);
1237                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1238                 track->db_z_read_bo = reloc->robj;
1239                 track->db_dirty = true;
1240                 break;
1241         case DB_Z_WRITE_BASE:
1242                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1243                 if (r) {
1244                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1245                                         "0x%04X\n", reg);
1246                         return -EINVAL;
1247                 }
1248                 track->db_z_write_offset = radeon_get_ib_value(p, idx);
1249                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1250                 track->db_z_write_bo = reloc->robj;
1251                 track->db_dirty = true;
1252                 break;
1253         case DB_STENCIL_READ_BASE:
1254                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1255                 if (r) {
1256                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1257                                         "0x%04X\n", reg);
1258                         return -EINVAL;
1259                 }
1260                 track->db_s_read_offset = radeon_get_ib_value(p, idx);
1261                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1262                 track->db_s_read_bo = reloc->robj;
1263                 track->db_dirty = true;
1264                 break;
1265         case DB_STENCIL_WRITE_BASE:
1266                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1267                 if (r) {
1268                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1269                                         "0x%04X\n", reg);
1270                         return -EINVAL;
1271                 }
1272                 track->db_s_write_offset = radeon_get_ib_value(p, idx);
1273                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1274                 track->db_s_write_bo = reloc->robj;
1275                 track->db_dirty = true;
1276                 break;
1277         case VGT_STRMOUT_CONFIG:
1278                 track->vgt_strmout_config = radeon_get_ib_value(p, idx);
1279                 track->streamout_dirty = true;
1280                 break;
1281         case VGT_STRMOUT_BUFFER_CONFIG:
1282                 track->vgt_strmout_buffer_config = radeon_get_ib_value(p, idx);
1283                 track->streamout_dirty = true;
1284                 break;
1285         case VGT_STRMOUT_BUFFER_BASE_0:
1286         case VGT_STRMOUT_BUFFER_BASE_1:
1287         case VGT_STRMOUT_BUFFER_BASE_2:
1288         case VGT_STRMOUT_BUFFER_BASE_3:
1289                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1290                 if (r) {
1291                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1292                                         "0x%04X\n", reg);
1293                         return -EINVAL;
1294                 }
1295                 tmp = (reg - VGT_STRMOUT_BUFFER_BASE_0) / 16;
1296                 track->vgt_strmout_bo_offset[tmp] = radeon_get_ib_value(p, idx) << 8;
1297                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1298                 track->vgt_strmout_bo[tmp] = reloc->robj;
1299                 track->streamout_dirty = true;
1300                 break;
1301         case VGT_STRMOUT_BUFFER_SIZE_0:
1302         case VGT_STRMOUT_BUFFER_SIZE_1:
1303         case VGT_STRMOUT_BUFFER_SIZE_2:
1304         case VGT_STRMOUT_BUFFER_SIZE_3:
1305                 tmp = (reg - VGT_STRMOUT_BUFFER_SIZE_0) / 16;
1306                 /* size in register is DWs, convert to bytes */
1307                 track->vgt_strmout_size[tmp] = radeon_get_ib_value(p, idx) * 4;
1308                 track->streamout_dirty = true;
1309                 break;
1310         case CP_COHER_BASE:
1311                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1312                 if (r) {
1313                         dev_warn(p->dev, "missing reloc for CP_COHER_BASE "
1314                                         "0x%04X\n", reg);
1315                         return -EINVAL;
1316                 }
1317                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1318         case CB_TARGET_MASK:
1319                 track->cb_target_mask = radeon_get_ib_value(p, idx);
1320                 track->cb_dirty = true;
1321                 break;
1322         case CB_SHADER_MASK:
1323                 track->cb_shader_mask = radeon_get_ib_value(p, idx);
1324                 track->cb_dirty = true;
1325                 break;
1326         case PA_SC_AA_CONFIG:
1327                 if (p->rdev->family >= CHIP_CAYMAN) {
1328                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1329                                  "0x%04X\n", reg);
1330                         return -EINVAL;
1331                 }
1332                 tmp = radeon_get_ib_value(p, idx) & MSAA_NUM_SAMPLES_MASK;
1333                 track->nsamples = 1 << tmp;
1334                 break;
1335         case CAYMAN_PA_SC_AA_CONFIG:
1336                 if (p->rdev->family < CHIP_CAYMAN) {
1337                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1338                                  "0x%04X\n", reg);
1339                         return -EINVAL;
1340                 }
1341                 tmp = radeon_get_ib_value(p, idx) & CAYMAN_MSAA_NUM_SAMPLES_MASK;
1342                 track->nsamples = 1 << tmp;
1343                 break;
1344         case CB_COLOR0_VIEW:
1345         case CB_COLOR1_VIEW:
1346         case CB_COLOR2_VIEW:
1347         case CB_COLOR3_VIEW:
1348         case CB_COLOR4_VIEW:
1349         case CB_COLOR5_VIEW:
1350         case CB_COLOR6_VIEW:
1351         case CB_COLOR7_VIEW:
1352                 tmp = (reg - CB_COLOR0_VIEW) / 0x3c;
1353                 track->cb_color_view[tmp] = radeon_get_ib_value(p, idx);
1354                 track->cb_dirty = true;
1355                 break;
1356         case CB_COLOR8_VIEW:
1357         case CB_COLOR9_VIEW:
1358         case CB_COLOR10_VIEW:
1359         case CB_COLOR11_VIEW:
1360                 tmp = ((reg - CB_COLOR8_VIEW) / 0x1c) + 8;
1361                 track->cb_color_view[tmp] = radeon_get_ib_value(p, idx);
1362                 track->cb_dirty = true;
1363                 break;
1364         case CB_COLOR0_INFO:
1365         case CB_COLOR1_INFO:
1366         case CB_COLOR2_INFO:
1367         case CB_COLOR3_INFO:
1368         case CB_COLOR4_INFO:
1369         case CB_COLOR5_INFO:
1370         case CB_COLOR6_INFO:
1371         case CB_COLOR7_INFO:
1372                 tmp = (reg - CB_COLOR0_INFO) / 0x3c;
1373                 track->cb_color_info[tmp] = radeon_get_ib_value(p, idx);
1374                 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) {
1375                         r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1376                         if (r) {
1377                                 dev_warn(p->dev, "bad SET_CONTEXT_REG "
1378                                                 "0x%04X\n", reg);
1379                                 return -EINVAL;
1380                         }
1381                         ib[idx] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags));
1382                         track->cb_color_info[tmp] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags));
1383                 }
1384                 track->cb_dirty = true;
1385                 break;
1386         case CB_COLOR8_INFO:
1387         case CB_COLOR9_INFO:
1388         case CB_COLOR10_INFO:
1389         case CB_COLOR11_INFO:
1390                 tmp = ((reg - CB_COLOR8_INFO) / 0x1c) + 8;
1391                 track->cb_color_info[tmp] = radeon_get_ib_value(p, idx);
1392                 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) {
1393                         r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1394                         if (r) {
1395                                 dev_warn(p->dev, "bad SET_CONTEXT_REG "
1396                                                 "0x%04X\n", reg);
1397                                 return -EINVAL;
1398                         }
1399                         ib[idx] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags));
1400                         track->cb_color_info[tmp] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags));
1401                 }
1402                 track->cb_dirty = true;
1403                 break;
1404         case CB_COLOR0_PITCH:
1405         case CB_COLOR1_PITCH:
1406         case CB_COLOR2_PITCH:
1407         case CB_COLOR3_PITCH:
1408         case CB_COLOR4_PITCH:
1409         case CB_COLOR5_PITCH:
1410         case CB_COLOR6_PITCH:
1411         case CB_COLOR7_PITCH:
1412                 tmp = (reg - CB_COLOR0_PITCH) / 0x3c;
1413                 track->cb_color_pitch[tmp] = radeon_get_ib_value(p, idx);
1414                 track->cb_dirty = true;
1415                 break;
1416         case CB_COLOR8_PITCH:
1417         case CB_COLOR9_PITCH:
1418         case CB_COLOR10_PITCH:
1419         case CB_COLOR11_PITCH:
1420                 tmp = ((reg - CB_COLOR8_PITCH) / 0x1c) + 8;
1421                 track->cb_color_pitch[tmp] = radeon_get_ib_value(p, idx);
1422                 track->cb_dirty = true;
1423                 break;
1424         case CB_COLOR0_SLICE:
1425         case CB_COLOR1_SLICE:
1426         case CB_COLOR2_SLICE:
1427         case CB_COLOR3_SLICE:
1428         case CB_COLOR4_SLICE:
1429         case CB_COLOR5_SLICE:
1430         case CB_COLOR6_SLICE:
1431         case CB_COLOR7_SLICE:
1432                 tmp = (reg - CB_COLOR0_SLICE) / 0x3c;
1433                 track->cb_color_slice[tmp] = radeon_get_ib_value(p, idx);
1434                 track->cb_color_slice_idx[tmp] = idx;
1435                 track->cb_dirty = true;
1436                 break;
1437         case CB_COLOR8_SLICE:
1438         case CB_COLOR9_SLICE:
1439         case CB_COLOR10_SLICE:
1440         case CB_COLOR11_SLICE:
1441                 tmp = ((reg - CB_COLOR8_SLICE) / 0x1c) + 8;
1442                 track->cb_color_slice[tmp] = radeon_get_ib_value(p, idx);
1443                 track->cb_color_slice_idx[tmp] = idx;
1444                 track->cb_dirty = true;
1445                 break;
1446         case CB_COLOR0_ATTRIB:
1447         case CB_COLOR1_ATTRIB:
1448         case CB_COLOR2_ATTRIB:
1449         case CB_COLOR3_ATTRIB:
1450         case CB_COLOR4_ATTRIB:
1451         case CB_COLOR5_ATTRIB:
1452         case CB_COLOR6_ATTRIB:
1453         case CB_COLOR7_ATTRIB:
1454                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1455                 if (r) {
1456                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1457                                         "0x%04X\n", reg);
1458                         return -EINVAL;
1459                 }
1460                 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) {
1461                         if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) {
1462                                 unsigned bankw, bankh, mtaspect, tile_split;
1463
1464                                 evergreen_tiling_fields(reloc->lobj.tiling_flags,
1465                                                         &bankw, &bankh, &mtaspect,
1466                                                         &tile_split);
1467                                 ib[idx] |= CB_NUM_BANKS(evergreen_cs_get_num_banks(track->nbanks));
1468                                 ib[idx] |= CB_TILE_SPLIT(tile_split) |
1469                                            CB_BANK_WIDTH(bankw) |
1470                                            CB_BANK_HEIGHT(bankh) |
1471                                            CB_MACRO_TILE_ASPECT(mtaspect);
1472                         }
1473                 }
1474                 tmp = ((reg - CB_COLOR0_ATTRIB) / 0x3c);
1475                 track->cb_color_attrib[tmp] = ib[idx];
1476                 track->cb_dirty = true;
1477                 break;
1478         case CB_COLOR8_ATTRIB:
1479         case CB_COLOR9_ATTRIB:
1480         case CB_COLOR10_ATTRIB:
1481         case CB_COLOR11_ATTRIB:
1482                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1483                 if (r) {
1484                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1485                                         "0x%04X\n", reg);
1486                         return -EINVAL;
1487                 }
1488                 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) {
1489                         if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) {
1490                                 unsigned bankw, bankh, mtaspect, tile_split;
1491
1492                                 evergreen_tiling_fields(reloc->lobj.tiling_flags,
1493                                                         &bankw, &bankh, &mtaspect,
1494                                                         &tile_split);
1495                                 ib[idx] |= CB_NUM_BANKS(evergreen_cs_get_num_banks(track->nbanks));
1496                                 ib[idx] |= CB_TILE_SPLIT(tile_split) |
1497                                            CB_BANK_WIDTH(bankw) |
1498                                            CB_BANK_HEIGHT(bankh) |
1499                                            CB_MACRO_TILE_ASPECT(mtaspect);
1500                         }
1501                 }
1502                 tmp = ((reg - CB_COLOR8_ATTRIB) / 0x1c) + 8;
1503                 track->cb_color_attrib[tmp] = ib[idx];
1504                 track->cb_dirty = true;
1505                 break;
1506         case CB_COLOR0_FMASK:
1507         case CB_COLOR1_FMASK:
1508         case CB_COLOR2_FMASK:
1509         case CB_COLOR3_FMASK:
1510         case CB_COLOR4_FMASK:
1511         case CB_COLOR5_FMASK:
1512         case CB_COLOR6_FMASK:
1513         case CB_COLOR7_FMASK:
1514                 tmp = (reg - CB_COLOR0_FMASK) / 0x3c;
1515                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1516                 if (r) {
1517                         dev_err(p->dev, "bad SET_CONTEXT_REG 0x%04X\n", reg);
1518                         return -EINVAL;
1519                 }
1520                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1521                 track->cb_color_fmask_bo[tmp] = reloc->robj;
1522                 break;
1523         case CB_COLOR0_CMASK:
1524         case CB_COLOR1_CMASK:
1525         case CB_COLOR2_CMASK:
1526         case CB_COLOR3_CMASK:
1527         case CB_COLOR4_CMASK:
1528         case CB_COLOR5_CMASK:
1529         case CB_COLOR6_CMASK:
1530         case CB_COLOR7_CMASK:
1531                 tmp = (reg - CB_COLOR0_CMASK) / 0x3c;
1532                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1533                 if (r) {
1534                         dev_err(p->dev, "bad SET_CONTEXT_REG 0x%04X\n", reg);
1535                         return -EINVAL;
1536                 }
1537                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1538                 track->cb_color_cmask_bo[tmp] = reloc->robj;
1539                 break;
1540         case CB_COLOR0_FMASK_SLICE:
1541         case CB_COLOR1_FMASK_SLICE:
1542         case CB_COLOR2_FMASK_SLICE:
1543         case CB_COLOR3_FMASK_SLICE:
1544         case CB_COLOR4_FMASK_SLICE:
1545         case CB_COLOR5_FMASK_SLICE:
1546         case CB_COLOR6_FMASK_SLICE:
1547         case CB_COLOR7_FMASK_SLICE:
1548                 tmp = (reg - CB_COLOR0_FMASK_SLICE) / 0x3c;
1549                 track->cb_color_fmask_slice[tmp] = radeon_get_ib_value(p, idx);
1550                 break;
1551         case CB_COLOR0_CMASK_SLICE:
1552         case CB_COLOR1_CMASK_SLICE:
1553         case CB_COLOR2_CMASK_SLICE:
1554         case CB_COLOR3_CMASK_SLICE:
1555         case CB_COLOR4_CMASK_SLICE:
1556         case CB_COLOR5_CMASK_SLICE:
1557         case CB_COLOR6_CMASK_SLICE:
1558         case CB_COLOR7_CMASK_SLICE:
1559                 tmp = (reg - CB_COLOR0_CMASK_SLICE) / 0x3c;
1560                 track->cb_color_cmask_slice[tmp] = radeon_get_ib_value(p, idx);
1561                 break;
1562         case CB_COLOR0_BASE:
1563         case CB_COLOR1_BASE:
1564         case CB_COLOR2_BASE:
1565         case CB_COLOR3_BASE:
1566         case CB_COLOR4_BASE:
1567         case CB_COLOR5_BASE:
1568         case CB_COLOR6_BASE:
1569         case CB_COLOR7_BASE:
1570                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1571                 if (r) {
1572                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1573                                         "0x%04X\n", reg);
1574                         return -EINVAL;
1575                 }
1576                 tmp = (reg - CB_COLOR0_BASE) / 0x3c;
1577                 track->cb_color_bo_offset[tmp] = radeon_get_ib_value(p, idx);
1578                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1579                 track->cb_color_bo[tmp] = reloc->robj;
1580                 track->cb_dirty = true;
1581                 break;
1582         case CB_COLOR8_BASE:
1583         case CB_COLOR9_BASE:
1584         case CB_COLOR10_BASE:
1585         case CB_COLOR11_BASE:
1586                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1587                 if (r) {
1588                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1589                                         "0x%04X\n", reg);
1590                         return -EINVAL;
1591                 }
1592                 tmp = ((reg - CB_COLOR8_BASE) / 0x1c) + 8;
1593                 track->cb_color_bo_offset[tmp] = radeon_get_ib_value(p, idx);
1594                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1595                 track->cb_color_bo[tmp] = reloc->robj;
1596                 track->cb_dirty = true;
1597                 break;
1598         case DB_HTILE_DATA_BASE:
1599                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1600                 if (r) {
1601                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1602                                         "0x%04X\n", reg);
1603                         return -EINVAL;
1604                 }
1605                 track->htile_offset = radeon_get_ib_value(p, idx);
1606                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1607                 track->htile_bo = reloc->robj;
1608                 track->db_dirty = true;
1609                 break;
1610         case DB_HTILE_SURFACE:
1611                 /* 8x8 only */
1612                 track->htile_surface = radeon_get_ib_value(p, idx);
1613                 /* force 8x8 htile width and height */
1614                 ib[idx] |= 3;
1615                 track->db_dirty = true;
1616                 break;
1617         case CB_IMMED0_BASE:
1618         case CB_IMMED1_BASE:
1619         case CB_IMMED2_BASE:
1620         case CB_IMMED3_BASE:
1621         case CB_IMMED4_BASE:
1622         case CB_IMMED5_BASE:
1623         case CB_IMMED6_BASE:
1624         case CB_IMMED7_BASE:
1625         case CB_IMMED8_BASE:
1626         case CB_IMMED9_BASE:
1627         case CB_IMMED10_BASE:
1628         case CB_IMMED11_BASE:
1629         case SQ_PGM_START_FS:
1630         case SQ_PGM_START_ES:
1631         case SQ_PGM_START_VS:
1632         case SQ_PGM_START_GS:
1633         case SQ_PGM_START_PS:
1634         case SQ_PGM_START_HS:
1635         case SQ_PGM_START_LS:
1636         case SQ_CONST_MEM_BASE:
1637         case SQ_ALU_CONST_CACHE_GS_0:
1638         case SQ_ALU_CONST_CACHE_GS_1:
1639         case SQ_ALU_CONST_CACHE_GS_2:
1640         case SQ_ALU_CONST_CACHE_GS_3:
1641         case SQ_ALU_CONST_CACHE_GS_4:
1642         case SQ_ALU_CONST_CACHE_GS_5:
1643         case SQ_ALU_CONST_CACHE_GS_6:
1644         case SQ_ALU_CONST_CACHE_GS_7:
1645         case SQ_ALU_CONST_CACHE_GS_8:
1646         case SQ_ALU_CONST_CACHE_GS_9:
1647         case SQ_ALU_CONST_CACHE_GS_10:
1648         case SQ_ALU_CONST_CACHE_GS_11:
1649         case SQ_ALU_CONST_CACHE_GS_12:
1650         case SQ_ALU_CONST_CACHE_GS_13:
1651         case SQ_ALU_CONST_CACHE_GS_14:
1652         case SQ_ALU_CONST_CACHE_GS_15:
1653         case SQ_ALU_CONST_CACHE_PS_0:
1654         case SQ_ALU_CONST_CACHE_PS_1:
1655         case SQ_ALU_CONST_CACHE_PS_2:
1656         case SQ_ALU_CONST_CACHE_PS_3:
1657         case SQ_ALU_CONST_CACHE_PS_4:
1658         case SQ_ALU_CONST_CACHE_PS_5:
1659         case SQ_ALU_CONST_CACHE_PS_6:
1660         case SQ_ALU_CONST_CACHE_PS_7:
1661         case SQ_ALU_CONST_CACHE_PS_8:
1662         case SQ_ALU_CONST_CACHE_PS_9:
1663         case SQ_ALU_CONST_CACHE_PS_10:
1664         case SQ_ALU_CONST_CACHE_PS_11:
1665         case SQ_ALU_CONST_CACHE_PS_12:
1666         case SQ_ALU_CONST_CACHE_PS_13:
1667         case SQ_ALU_CONST_CACHE_PS_14:
1668         case SQ_ALU_CONST_CACHE_PS_15:
1669         case SQ_ALU_CONST_CACHE_VS_0:
1670         case SQ_ALU_CONST_CACHE_VS_1:
1671         case SQ_ALU_CONST_CACHE_VS_2:
1672         case SQ_ALU_CONST_CACHE_VS_3:
1673         case SQ_ALU_CONST_CACHE_VS_4:
1674         case SQ_ALU_CONST_CACHE_VS_5:
1675         case SQ_ALU_CONST_CACHE_VS_6:
1676         case SQ_ALU_CONST_CACHE_VS_7:
1677         case SQ_ALU_CONST_CACHE_VS_8:
1678         case SQ_ALU_CONST_CACHE_VS_9:
1679         case SQ_ALU_CONST_CACHE_VS_10:
1680         case SQ_ALU_CONST_CACHE_VS_11:
1681         case SQ_ALU_CONST_CACHE_VS_12:
1682         case SQ_ALU_CONST_CACHE_VS_13:
1683         case SQ_ALU_CONST_CACHE_VS_14:
1684         case SQ_ALU_CONST_CACHE_VS_15:
1685         case SQ_ALU_CONST_CACHE_HS_0:
1686         case SQ_ALU_CONST_CACHE_HS_1:
1687         case SQ_ALU_CONST_CACHE_HS_2:
1688         case SQ_ALU_CONST_CACHE_HS_3:
1689         case SQ_ALU_CONST_CACHE_HS_4:
1690         case SQ_ALU_CONST_CACHE_HS_5:
1691         case SQ_ALU_CONST_CACHE_HS_6:
1692         case SQ_ALU_CONST_CACHE_HS_7:
1693         case SQ_ALU_CONST_CACHE_HS_8:
1694         case SQ_ALU_CONST_CACHE_HS_9:
1695         case SQ_ALU_CONST_CACHE_HS_10:
1696         case SQ_ALU_CONST_CACHE_HS_11:
1697         case SQ_ALU_CONST_CACHE_HS_12:
1698         case SQ_ALU_CONST_CACHE_HS_13:
1699         case SQ_ALU_CONST_CACHE_HS_14:
1700         case SQ_ALU_CONST_CACHE_HS_15:
1701         case SQ_ALU_CONST_CACHE_LS_0:
1702         case SQ_ALU_CONST_CACHE_LS_1:
1703         case SQ_ALU_CONST_CACHE_LS_2:
1704         case SQ_ALU_CONST_CACHE_LS_3:
1705         case SQ_ALU_CONST_CACHE_LS_4:
1706         case SQ_ALU_CONST_CACHE_LS_5:
1707         case SQ_ALU_CONST_CACHE_LS_6:
1708         case SQ_ALU_CONST_CACHE_LS_7:
1709         case SQ_ALU_CONST_CACHE_LS_8:
1710         case SQ_ALU_CONST_CACHE_LS_9:
1711         case SQ_ALU_CONST_CACHE_LS_10:
1712         case SQ_ALU_CONST_CACHE_LS_11:
1713         case SQ_ALU_CONST_CACHE_LS_12:
1714         case SQ_ALU_CONST_CACHE_LS_13:
1715         case SQ_ALU_CONST_CACHE_LS_14:
1716         case SQ_ALU_CONST_CACHE_LS_15:
1717                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1718                 if (r) {
1719                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1720                                         "0x%04X\n", reg);
1721                         return -EINVAL;
1722                 }
1723                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1724                 break;
1725         case SX_MEMORY_EXPORT_BASE:
1726                 if (p->rdev->family >= CHIP_CAYMAN) {
1727                         dev_warn(p->dev, "bad SET_CONFIG_REG "
1728                                  "0x%04X\n", reg);
1729                         return -EINVAL;
1730                 }
1731                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1732                 if (r) {
1733                         dev_warn(p->dev, "bad SET_CONFIG_REG "
1734                                         "0x%04X\n", reg);
1735                         return -EINVAL;
1736                 }
1737                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1738                 break;
1739         case CAYMAN_SX_SCATTER_EXPORT_BASE:
1740                 if (p->rdev->family < CHIP_CAYMAN) {
1741                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1742                                  "0x%04X\n", reg);
1743                         return -EINVAL;
1744                 }
1745                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1746                 if (r) {
1747                         dev_warn(p->dev, "bad SET_CONTEXT_REG "
1748                                         "0x%04X\n", reg);
1749                         return -EINVAL;
1750                 }
1751                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
1752                 break;
1753         case SX_MISC:
1754                 track->sx_misc_kill_all_prims = (radeon_get_ib_value(p, idx) & 0x1) != 0;
1755                 break;
1756         default:
1757                 dev_warn(p->dev, "forbidden register 0x%08x at %d\n", reg, idx);
1758                 return -EINVAL;
1759         }
1760         return 0;
1761 }
1762
1763 static bool evergreen_is_safe_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
1764 {
1765         u32 last_reg, m, i;
1766
1767         if (p->rdev->family >= CHIP_CAYMAN)
1768                 last_reg = ARRAY_SIZE(cayman_reg_safe_bm);
1769         else
1770                 last_reg = ARRAY_SIZE(evergreen_reg_safe_bm);
1771
1772         i = (reg >> 7);
1773         if (i >= last_reg) {
1774                 dev_warn(p->dev, "forbidden register 0x%08x at %d\n", reg, idx);
1775                 return false;
1776         }
1777         m = 1 << ((reg >> 2) & 31);
1778         if (p->rdev->family >= CHIP_CAYMAN) {
1779                 if (!(cayman_reg_safe_bm[i] & m))
1780                         return true;
1781         } else {
1782                 if (!(evergreen_reg_safe_bm[i] & m))
1783                         return true;
1784         }
1785         dev_warn(p->dev, "forbidden register 0x%08x at %d\n", reg, idx);
1786         return false;
1787 }
1788
1789 static int evergreen_packet3_check(struct radeon_cs_parser *p,
1790                                    struct radeon_cs_packet *pkt)
1791 {
1792         struct radeon_cs_reloc *reloc;
1793         struct evergreen_cs_track *track;
1794         volatile u32 *ib;
1795         unsigned idx;
1796         unsigned i;
1797         unsigned start_reg, end_reg, reg;
1798         int r;
1799         u32 idx_value;
1800
1801         track = (struct evergreen_cs_track *)p->track;
1802         ib = p->ib.ptr;
1803         idx = pkt->idx + 1;
1804         idx_value = radeon_get_ib_value(p, idx);
1805
1806         switch (pkt->opcode) {
1807         case PACKET3_SET_PREDICATION:
1808         {
1809                 int pred_op;
1810                 int tmp;
1811                 uint64_t offset;
1812
1813                 if (pkt->count != 1) {
1814                         DRM_ERROR("bad SET PREDICATION\n");
1815                         return -EINVAL;
1816                 }
1817
1818                 tmp = radeon_get_ib_value(p, idx + 1);
1819                 pred_op = (tmp >> 16) & 0x7;
1820
1821                 /* for the clear predicate operation */
1822                 if (pred_op == 0)
1823                         return 0;
1824
1825                 if (pred_op > 2) {
1826                         DRM_ERROR("bad SET PREDICATION operation %d\n", pred_op);
1827                         return -EINVAL;
1828                 }
1829
1830                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1831                 if (r) {
1832                         DRM_ERROR("bad SET PREDICATION\n");
1833                         return -EINVAL;
1834                 }
1835
1836                 offset = reloc->lobj.gpu_offset +
1837                          (idx_value & 0xfffffff0) +
1838                          ((u64)(tmp & 0xff) << 32);
1839
1840                 ib[idx + 0] = offset;
1841                 ib[idx + 1] = (tmp & 0xffffff00) | (upper_32_bits(offset) & 0xff);
1842         }
1843         break;
1844         case PACKET3_CONTEXT_CONTROL:
1845                 if (pkt->count != 1) {
1846                         DRM_ERROR("bad CONTEXT_CONTROL\n");
1847                         return -EINVAL;
1848                 }
1849                 break;
1850         case PACKET3_INDEX_TYPE:
1851         case PACKET3_NUM_INSTANCES:
1852         case PACKET3_CLEAR_STATE:
1853                 if (pkt->count) {
1854                         DRM_ERROR("bad INDEX_TYPE/NUM_INSTANCES/CLEAR_STATE\n");
1855                         return -EINVAL;
1856                 }
1857                 break;
1858         case CAYMAN_PACKET3_DEALLOC_STATE:
1859                 if (p->rdev->family < CHIP_CAYMAN) {
1860                         DRM_ERROR("bad PACKET3_DEALLOC_STATE\n");
1861                         return -EINVAL;
1862                 }
1863                 if (pkt->count) {
1864                         DRM_ERROR("bad INDEX_TYPE/NUM_INSTANCES/CLEAR_STATE\n");
1865                         return -EINVAL;
1866                 }
1867                 break;
1868         case PACKET3_INDEX_BASE:
1869         {
1870                 uint64_t offset;
1871
1872                 if (pkt->count != 1) {
1873                         DRM_ERROR("bad INDEX_BASE\n");
1874                         return -EINVAL;
1875                 }
1876                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1877                 if (r) {
1878                         DRM_ERROR("bad INDEX_BASE\n");
1879                         return -EINVAL;
1880                 }
1881
1882                 offset = reloc->lobj.gpu_offset +
1883                          idx_value +
1884                          ((u64)(radeon_get_ib_value(p, idx+1) & 0xff) << 32);
1885
1886                 ib[idx+0] = offset;
1887                 ib[idx+1] = upper_32_bits(offset) & 0xff;
1888
1889                 r = evergreen_cs_track_check(p);
1890                 if (r) {
1891                         dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__);
1892                         return r;
1893                 }
1894                 break;
1895         }
1896         case PACKET3_DRAW_INDEX:
1897         {
1898                 uint64_t offset;
1899                 if (pkt->count != 3) {
1900                         DRM_ERROR("bad DRAW_INDEX\n");
1901                         return -EINVAL;
1902                 }
1903                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1904                 if (r) {
1905                         DRM_ERROR("bad DRAW_INDEX\n");
1906                         return -EINVAL;
1907                 }
1908
1909                 offset = reloc->lobj.gpu_offset +
1910                          idx_value +
1911                          ((u64)(radeon_get_ib_value(p, idx+1) & 0xff) << 32);
1912
1913                 ib[idx+0] = offset;
1914                 ib[idx+1] = upper_32_bits(offset) & 0xff;
1915
1916                 r = evergreen_cs_track_check(p);
1917                 if (r) {
1918                         dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__);
1919                         return r;
1920                 }
1921                 break;
1922         }
1923         case PACKET3_DRAW_INDEX_2:
1924         {
1925                 uint64_t offset;
1926
1927                 if (pkt->count != 4) {
1928                         DRM_ERROR("bad DRAW_INDEX_2\n");
1929                         return -EINVAL;
1930                 }
1931                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
1932                 if (r) {
1933                         DRM_ERROR("bad DRAW_INDEX_2\n");
1934                         return -EINVAL;
1935                 }
1936
1937                 offset = reloc->lobj.gpu_offset +
1938                          radeon_get_ib_value(p, idx+1) +
1939                          ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32);
1940
1941                 ib[idx+1] = offset;
1942                 ib[idx+2] = upper_32_bits(offset) & 0xff;
1943
1944                 r = evergreen_cs_track_check(p);
1945                 if (r) {
1946                         dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__);
1947                         return r;
1948                 }
1949                 break;
1950         }
1951         case PACKET3_DRAW_INDEX_AUTO:
1952                 if (pkt->count != 1) {
1953                         DRM_ERROR("bad DRAW_INDEX_AUTO\n");
1954                         return -EINVAL;
1955                 }
1956                 r = evergreen_cs_track_check(p);
1957                 if (r) {
1958                         dev_warn(p->dev, "%s:%d invalid cmd stream %d\n", __func__, __LINE__, idx);
1959                         return r;
1960                 }
1961                 break;
1962         case PACKET3_DRAW_INDEX_MULTI_AUTO:
1963                 if (pkt->count != 2) {
1964                         DRM_ERROR("bad DRAW_INDEX_MULTI_AUTO\n");
1965                         return -EINVAL;
1966                 }
1967                 r = evergreen_cs_track_check(p);
1968                 if (r) {
1969                         dev_warn(p->dev, "%s:%d invalid cmd stream %d\n", __func__, __LINE__, idx);
1970                         return r;
1971                 }
1972                 break;
1973         case PACKET3_DRAW_INDEX_IMMD:
1974                 if (pkt->count < 2) {
1975                         DRM_ERROR("bad DRAW_INDEX_IMMD\n");
1976                         return -EINVAL;
1977                 }
1978                 r = evergreen_cs_track_check(p);
1979                 if (r) {
1980                         dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__);
1981                         return r;
1982                 }
1983                 break;
1984         case PACKET3_DRAW_INDEX_OFFSET:
1985                 if (pkt->count != 2) {
1986                         DRM_ERROR("bad DRAW_INDEX_OFFSET\n");
1987                         return -EINVAL;
1988                 }
1989                 r = evergreen_cs_track_check(p);
1990                 if (r) {
1991                         dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__);
1992                         return r;
1993                 }
1994                 break;
1995         case PACKET3_DRAW_INDEX_OFFSET_2:
1996                 if (pkt->count != 3) {
1997                         DRM_ERROR("bad DRAW_INDEX_OFFSET_2\n");
1998                         return -EINVAL;
1999                 }
2000                 r = evergreen_cs_track_check(p);
2001                 if (r) {
2002                         dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__);
2003                         return r;
2004                 }
2005                 break;
2006         case PACKET3_DISPATCH_DIRECT:
2007                 if (pkt->count != 3) {
2008                         DRM_ERROR("bad DISPATCH_DIRECT\n");
2009                         return -EINVAL;
2010                 }
2011                 r = evergreen_cs_track_check(p);
2012                 if (r) {
2013                         dev_warn(p->dev, "%s:%d invalid cmd stream %d\n", __func__, __LINE__, idx);
2014                         return r;
2015                 }
2016                 break;
2017         case PACKET3_DISPATCH_INDIRECT:
2018                 if (pkt->count != 1) {
2019                         DRM_ERROR("bad DISPATCH_INDIRECT\n");
2020                         return -EINVAL;
2021                 }
2022                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2023                 if (r) {
2024                         DRM_ERROR("bad DISPATCH_INDIRECT\n");
2025                         return -EINVAL;
2026                 }
2027                 ib[idx+0] = idx_value + (u32)(reloc->lobj.gpu_offset & 0xffffffff);
2028                 r = evergreen_cs_track_check(p);
2029                 if (r) {
2030                         dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__);
2031                         return r;
2032                 }
2033                 break;
2034         case PACKET3_WAIT_REG_MEM:
2035                 if (pkt->count != 5) {
2036                         DRM_ERROR("bad WAIT_REG_MEM\n");
2037                         return -EINVAL;
2038                 }
2039                 /* bit 4 is reg (0) or mem (1) */
2040                 if (idx_value & 0x10) {
2041                         uint64_t offset;
2042
2043                         r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2044                         if (r) {
2045                                 DRM_ERROR("bad WAIT_REG_MEM\n");
2046                                 return -EINVAL;
2047                         }
2048
2049                         offset = reloc->lobj.gpu_offset +
2050                                  (radeon_get_ib_value(p, idx+1) & 0xfffffffc) +
2051                                  ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32);
2052
2053                         ib[idx+1] = (ib[idx+1] & 0x3) | (offset & 0xfffffffc);
2054                         ib[idx+2] = upper_32_bits(offset) & 0xff;
2055                 } else if (idx_value & 0x100) {
2056                         DRM_ERROR("cannot use PFP on REG wait\n");
2057                         return -EINVAL;
2058                 }
2059                 break;
2060         case PACKET3_CP_DMA:
2061         {
2062                 u32 command, size, info;
2063                 u64 offset, tmp;
2064                 if (pkt->count != 4) {
2065                         DRM_ERROR("bad CP DMA\n");
2066                         return -EINVAL;
2067                 }
2068                 command = radeon_get_ib_value(p, idx+4);
2069                 size = command & 0x1fffff;
2070                 info = radeon_get_ib_value(p, idx+1);
2071                 if ((((info & 0x60000000) >> 29) != 0) || /* src = GDS or DATA */
2072                     (((info & 0x00300000) >> 20) != 0) || /* dst = GDS */
2073                     ((((info & 0x00300000) >> 20) == 0) &&
2074                      (command & PACKET3_CP_DMA_CMD_DAS)) || /* dst = register */
2075                     ((((info & 0x60000000) >> 29) == 0) &&
2076                      (command & PACKET3_CP_DMA_CMD_SAS))) { /* src = register */
2077                         /* non mem to mem copies requires dw aligned count */
2078                         if (size % 4) {
2079                                 DRM_ERROR("CP DMA command requires dw count alignment\n");
2080                                 return -EINVAL;
2081                         }
2082                 }
2083                 if (command & PACKET3_CP_DMA_CMD_SAS) {
2084                         /* src address space is register */
2085                         /* GDS is ok */
2086                         if (((info & 0x60000000) >> 29) != 1) {
2087                                 DRM_ERROR("CP DMA SAS not supported\n");
2088                                 return -EINVAL;
2089                         }
2090                 } else {
2091                         if (command & PACKET3_CP_DMA_CMD_SAIC) {
2092                                 DRM_ERROR("CP DMA SAIC only supported for registers\n");
2093                                 return -EINVAL;
2094                         }
2095                         /* src address space is memory */
2096                         if (((info & 0x60000000) >> 29) == 0) {
2097                                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2098                                 if (r) {
2099                                         DRM_ERROR("bad CP DMA SRC\n");
2100                                         return -EINVAL;
2101                                 }
2102
2103                                 tmp = radeon_get_ib_value(p, idx) +
2104                                         ((u64)(radeon_get_ib_value(p, idx+1) & 0xff) << 32);
2105
2106                                 offset = reloc->lobj.gpu_offset + tmp;
2107
2108                                 if ((tmp + size) > radeon_bo_size(reloc->robj)) {
2109                                         dev_warn(p->dev, "CP DMA src buffer too small (%llu %lu)\n",
2110                                                  tmp + size, radeon_bo_size(reloc->robj));
2111                                         return -EINVAL;
2112                                 }
2113
2114                                 ib[idx] = offset;
2115                                 ib[idx+1] = (ib[idx+1] & 0xffffff00) | (upper_32_bits(offset) & 0xff);
2116                         } else if (((info & 0x60000000) >> 29) != 2) {
2117                                 DRM_ERROR("bad CP DMA SRC_SEL\n");
2118                                 return -EINVAL;
2119                         }
2120                 }
2121                 if (command & PACKET3_CP_DMA_CMD_DAS) {
2122                         /* dst address space is register */
2123                         /* GDS is ok */
2124                         if (((info & 0x00300000) >> 20) != 1) {
2125                                 DRM_ERROR("CP DMA DAS not supported\n");
2126                                 return -EINVAL;
2127                         }
2128                 } else {
2129                         /* dst address space is memory */
2130                         if (command & PACKET3_CP_DMA_CMD_DAIC) {
2131                                 DRM_ERROR("CP DMA DAIC only supported for registers\n");
2132                                 return -EINVAL;
2133                         }
2134                         if (((info & 0x00300000) >> 20) == 0) {
2135                                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2136                                 if (r) {
2137                                         DRM_ERROR("bad CP DMA DST\n");
2138                                         return -EINVAL;
2139                                 }
2140
2141                                 tmp = radeon_get_ib_value(p, idx+2) +
2142                                         ((u64)(radeon_get_ib_value(p, idx+3) & 0xff) << 32);
2143
2144                                 offset = reloc->lobj.gpu_offset + tmp;
2145
2146                                 if ((tmp + size) > radeon_bo_size(reloc->robj)) {
2147                                         dev_warn(p->dev, "CP DMA dst buffer too small (%llu %lu)\n",
2148                                                  tmp + size, radeon_bo_size(reloc->robj));
2149                                         return -EINVAL;
2150                                 }
2151
2152                                 ib[idx+2] = offset;
2153                                 ib[idx+3] = upper_32_bits(offset) & 0xff;
2154                         } else {
2155                                 DRM_ERROR("bad CP DMA DST_SEL\n");
2156                                 return -EINVAL;
2157                         }
2158                 }
2159                 break;
2160         }
2161         case PACKET3_SURFACE_SYNC:
2162                 if (pkt->count != 3) {
2163                         DRM_ERROR("bad SURFACE_SYNC\n");
2164                         return -EINVAL;
2165                 }
2166                 /* 0xffffffff/0x0 is flush all cache flag */
2167                 if (radeon_get_ib_value(p, idx + 1) != 0xffffffff ||
2168                     radeon_get_ib_value(p, idx + 2) != 0) {
2169                         r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2170                         if (r) {
2171                                 DRM_ERROR("bad SURFACE_SYNC\n");
2172                                 return -EINVAL;
2173                         }
2174                         ib[idx+2] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
2175                 }
2176                 break;
2177         case PACKET3_EVENT_WRITE:
2178                 if (pkt->count != 2 && pkt->count != 0) {
2179                         DRM_ERROR("bad EVENT_WRITE\n");
2180                         return -EINVAL;
2181                 }
2182                 if (pkt->count) {
2183                         uint64_t offset;
2184
2185                         r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2186                         if (r) {
2187                                 DRM_ERROR("bad EVENT_WRITE\n");
2188                                 return -EINVAL;
2189                         }
2190                         offset = reloc->lobj.gpu_offset +
2191                                  (radeon_get_ib_value(p, idx+1) & 0xfffffff8) +
2192                                  ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32);
2193
2194                         ib[idx+1] = offset & 0xfffffff8;
2195                         ib[idx+2] = upper_32_bits(offset) & 0xff;
2196                 }
2197                 break;
2198         case PACKET3_EVENT_WRITE_EOP:
2199         {
2200                 uint64_t offset;
2201
2202                 if (pkt->count != 4) {
2203                         DRM_ERROR("bad EVENT_WRITE_EOP\n");
2204                         return -EINVAL;
2205                 }
2206                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2207                 if (r) {
2208                         DRM_ERROR("bad EVENT_WRITE_EOP\n");
2209                         return -EINVAL;
2210                 }
2211
2212                 offset = reloc->lobj.gpu_offset +
2213                          (radeon_get_ib_value(p, idx+1) & 0xfffffffc) +
2214                          ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32);
2215
2216                 ib[idx+1] = offset & 0xfffffffc;
2217                 ib[idx+2] = (ib[idx+2] & 0xffffff00) | (upper_32_bits(offset) & 0xff);
2218                 break;
2219         }
2220         case PACKET3_EVENT_WRITE_EOS:
2221         {
2222                 uint64_t offset;
2223
2224                 if (pkt->count != 3) {
2225                         DRM_ERROR("bad EVENT_WRITE_EOS\n");
2226                         return -EINVAL;
2227                 }
2228                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2229                 if (r) {
2230                         DRM_ERROR("bad EVENT_WRITE_EOS\n");
2231                         return -EINVAL;
2232                 }
2233
2234                 offset = reloc->lobj.gpu_offset +
2235                          (radeon_get_ib_value(p, idx+1) & 0xfffffffc) +
2236                          ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32);
2237
2238                 ib[idx+1] = offset & 0xfffffffc;
2239                 ib[idx+2] = (ib[idx+2] & 0xffffff00) | (upper_32_bits(offset) & 0xff);
2240                 break;
2241         }
2242         case PACKET3_SET_CONFIG_REG:
2243                 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
2244                 end_reg = 4 * pkt->count + start_reg - 4;
2245                 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
2246                     (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
2247                     (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
2248                         DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
2249                         return -EINVAL;
2250                 }
2251                 for (i = 0; i < pkt->count; i++) {
2252                         reg = start_reg + (4 * i);
2253                         r = evergreen_cs_check_reg(p, reg, idx+1+i);
2254                         if (r)
2255                                 return r;
2256                 }
2257                 break;
2258         case PACKET3_SET_CONTEXT_REG:
2259                 start_reg = (idx_value << 2) + PACKET3_SET_CONTEXT_REG_START;
2260                 end_reg = 4 * pkt->count + start_reg - 4;
2261                 if ((start_reg < PACKET3_SET_CONTEXT_REG_START) ||
2262                     (start_reg >= PACKET3_SET_CONTEXT_REG_END) ||
2263                     (end_reg >= PACKET3_SET_CONTEXT_REG_END)) {
2264                         DRM_ERROR("bad PACKET3_SET_CONTEXT_REG\n");
2265                         return -EINVAL;
2266                 }
2267                 for (i = 0; i < pkt->count; i++) {
2268                         reg = start_reg + (4 * i);
2269                         r = evergreen_cs_check_reg(p, reg, idx+1+i);
2270                         if (r)
2271                                 return r;
2272                 }
2273                 break;
2274         case PACKET3_SET_RESOURCE:
2275                 if (pkt->count % 8) {
2276                         DRM_ERROR("bad SET_RESOURCE\n");
2277                         return -EINVAL;
2278                 }
2279                 start_reg = (idx_value << 2) + PACKET3_SET_RESOURCE_START;
2280                 end_reg = 4 * pkt->count + start_reg - 4;
2281                 if ((start_reg < PACKET3_SET_RESOURCE_START) ||
2282                     (start_reg >= PACKET3_SET_RESOURCE_END) ||
2283                     (end_reg >= PACKET3_SET_RESOURCE_END)) {
2284                         DRM_ERROR("bad SET_RESOURCE\n");
2285                         return -EINVAL;
2286                 }
2287                 for (i = 0; i < (pkt->count / 8); i++) {
2288                         struct radeon_bo *texture, *mipmap;
2289                         u32 toffset, moffset;
2290                         u32 size, offset, mip_address, tex_dim;
2291
2292                         switch (G__SQ_CONSTANT_TYPE(radeon_get_ib_value(p, idx+1+(i*8)+7))) {
2293                         case SQ_TEX_VTX_VALID_TEXTURE:
2294                                 /* tex base */
2295                                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2296                                 if (r) {
2297                                         DRM_ERROR("bad SET_RESOURCE (tex)\n");
2298                                         return -EINVAL;
2299                                 }
2300                                 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) {
2301                                         ib[idx+1+(i*8)+1] |=
2302                                                 TEX_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags));
2303                                         if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) {
2304                                                 unsigned bankw, bankh, mtaspect, tile_split;
2305
2306                                                 evergreen_tiling_fields(reloc->lobj.tiling_flags,
2307                                                                         &bankw, &bankh, &mtaspect,
2308                                                                         &tile_split);
2309                                                 ib[idx+1+(i*8)+6] |= TEX_TILE_SPLIT(tile_split);
2310                                                 ib[idx+1+(i*8)+7] |=
2311                                                         TEX_BANK_WIDTH(bankw) |
2312                                                         TEX_BANK_HEIGHT(bankh) |
2313                                                         MACRO_TILE_ASPECT(mtaspect) |
2314                                                         TEX_NUM_BANKS(evergreen_cs_get_num_banks(track->nbanks));
2315                                         }
2316                                 }
2317                                 texture = reloc->robj;
2318                                 toffset = (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
2319
2320                                 /* tex mip base */
2321                                 tex_dim = ib[idx+1+(i*8)+0] & 0x7;
2322                                 mip_address = ib[idx+1+(i*8)+3];
2323
2324                                 if ((tex_dim == SQ_TEX_DIM_2D_MSAA || tex_dim == SQ_TEX_DIM_2D_ARRAY_MSAA) &&
2325                                     !mip_address &&
2326                                     !radeon_cs_packet_next_is_pkt3_nop(p)) {
2327                                         /* MIP_ADDRESS should point to FMASK for an MSAA texture.
2328                                          * It should be 0 if FMASK is disabled. */
2329                                         moffset = 0;
2330                                         mipmap = NULL;
2331                                 } else {
2332                                         r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2333                                         if (r) {
2334                                                 DRM_ERROR("bad SET_RESOURCE (tex)\n");
2335                                                 return -EINVAL;
2336                                         }
2337                                         moffset = (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
2338                                         mipmap = reloc->robj;
2339                                 }
2340
2341                                 r = evergreen_cs_track_validate_texture(p, texture, mipmap, idx+1+(i*8));
2342                                 if (r)
2343                                         return r;
2344                                 ib[idx+1+(i*8)+2] += toffset;
2345                                 ib[idx+1+(i*8)+3] += moffset;
2346                                 break;
2347                         case SQ_TEX_VTX_VALID_BUFFER:
2348                         {
2349                                 uint64_t offset64;
2350                                 /* vtx base */
2351                                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2352                                 if (r) {
2353                                         DRM_ERROR("bad SET_RESOURCE (vtx)\n");
2354                                         return -EINVAL;
2355                                 }
2356                                 offset = radeon_get_ib_value(p, idx+1+(i*8)+0);
2357                                 size = radeon_get_ib_value(p, idx+1+(i*8)+1);
2358                                 if (p->rdev && (size + offset) > radeon_bo_size(reloc->robj)) {
2359                                         /* force size to size of the buffer */
2360                                         dev_warn(p->dev, "vbo resource seems too big for the bo\n");
2361                                         ib[idx+1+(i*8)+1] = radeon_bo_size(reloc->robj) - offset;
2362                                 }
2363
2364                                 offset64 = reloc->lobj.gpu_offset + offset;
2365                                 ib[idx+1+(i*8)+0] = offset64;
2366                                 ib[idx+1+(i*8)+2] = (ib[idx+1+(i*8)+2] & 0xffffff00) |
2367                                                     (upper_32_bits(offset64) & 0xff);
2368                                 break;
2369                         }
2370                         case SQ_TEX_VTX_INVALID_TEXTURE:
2371                         case SQ_TEX_VTX_INVALID_BUFFER:
2372                         default:
2373                                 DRM_ERROR("bad SET_RESOURCE\n");
2374                                 return -EINVAL;
2375                         }
2376                 }
2377                 break;
2378         case PACKET3_SET_ALU_CONST:
2379                 /* XXX fix me ALU const buffers only */
2380                 break;
2381         case PACKET3_SET_BOOL_CONST:
2382                 start_reg = (idx_value << 2) + PACKET3_SET_BOOL_CONST_START;
2383                 end_reg = 4 * pkt->count + start_reg - 4;
2384                 if ((start_reg < PACKET3_SET_BOOL_CONST_START) ||
2385                     (start_reg >= PACKET3_SET_BOOL_CONST_END) ||
2386                     (end_reg >= PACKET3_SET_BOOL_CONST_END)) {
2387                         DRM_ERROR("bad SET_BOOL_CONST\n");
2388                         return -EINVAL;
2389                 }
2390                 break;
2391         case PACKET3_SET_LOOP_CONST:
2392                 start_reg = (idx_value << 2) + PACKET3_SET_LOOP_CONST_START;
2393                 end_reg = 4 * pkt->count + start_reg - 4;
2394                 if ((start_reg < PACKET3_SET_LOOP_CONST_START) ||
2395                     (start_reg >= PACKET3_SET_LOOP_CONST_END) ||
2396                     (end_reg >= PACKET3_SET_LOOP_CONST_END)) {
2397                         DRM_ERROR("bad SET_LOOP_CONST\n");
2398                         return -EINVAL;
2399                 }
2400                 break;
2401         case PACKET3_SET_CTL_CONST:
2402                 start_reg = (idx_value << 2) + PACKET3_SET_CTL_CONST_START;
2403                 end_reg = 4 * pkt->count + start_reg - 4;
2404                 if ((start_reg < PACKET3_SET_CTL_CONST_START) ||
2405                     (start_reg >= PACKET3_SET_CTL_CONST_END) ||
2406                     (end_reg >= PACKET3_SET_CTL_CONST_END)) {
2407                         DRM_ERROR("bad SET_CTL_CONST\n");
2408                         return -EINVAL;
2409                 }
2410                 break;
2411         case PACKET3_SET_SAMPLER:
2412                 if (pkt->count % 3) {
2413                         DRM_ERROR("bad SET_SAMPLER\n");
2414                         return -EINVAL;
2415                 }
2416                 start_reg = (idx_value << 2) + PACKET3_SET_SAMPLER_START;
2417                 end_reg = 4 * pkt->count + start_reg - 4;
2418                 if ((start_reg < PACKET3_SET_SAMPLER_START) ||
2419                     (start_reg >= PACKET3_SET_SAMPLER_END) ||
2420                     (end_reg >= PACKET3_SET_SAMPLER_END)) {
2421                         DRM_ERROR("bad SET_SAMPLER\n");
2422                         return -EINVAL;
2423                 }
2424                 break;
2425         case PACKET3_STRMOUT_BUFFER_UPDATE:
2426                 if (pkt->count != 4) {
2427                         DRM_ERROR("bad STRMOUT_BUFFER_UPDATE (invalid count)\n");
2428                         return -EINVAL;
2429                 }
2430                 /* Updating memory at DST_ADDRESS. */
2431                 if (idx_value & 0x1) {
2432                         u64 offset;
2433                         r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2434                         if (r) {
2435                                 DRM_ERROR("bad STRMOUT_BUFFER_UPDATE (missing dst reloc)\n");
2436                                 return -EINVAL;
2437                         }
2438                         offset = radeon_get_ib_value(p, idx+1);
2439                         offset += ((u64)(radeon_get_ib_value(p, idx+2) & 0xff)) << 32;
2440                         if ((offset + 4) > radeon_bo_size(reloc->robj)) {
2441                                 DRM_ERROR("bad STRMOUT_BUFFER_UPDATE dst bo too small: 0x%llx, 0x%lx\n",
2442                                           offset + 4, radeon_bo_size(reloc->robj));
2443                                 return -EINVAL;
2444                         }
2445                         offset += reloc->lobj.gpu_offset;
2446                         ib[idx+1] = offset;
2447                         ib[idx+2] = upper_32_bits(offset) & 0xff;
2448                 }
2449                 /* Reading data from SRC_ADDRESS. */
2450                 if (((idx_value >> 1) & 0x3) == 2) {
2451                         u64 offset;
2452                         r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2453                         if (r) {
2454                                 DRM_ERROR("bad STRMOUT_BUFFER_UPDATE (missing src reloc)\n");
2455                                 return -EINVAL;
2456                         }
2457                         offset = radeon_get_ib_value(p, idx+3);
2458                         offset += ((u64)(radeon_get_ib_value(p, idx+4) & 0xff)) << 32;
2459                         if ((offset + 4) > radeon_bo_size(reloc->robj)) {
2460                                 DRM_ERROR("bad STRMOUT_BUFFER_UPDATE src bo too small: 0x%llx, 0x%lx\n",
2461                                           offset + 4, radeon_bo_size(reloc->robj));
2462                                 return -EINVAL;
2463                         }
2464                         offset += reloc->lobj.gpu_offset;
2465                         ib[idx+3] = offset;
2466                         ib[idx+4] = upper_32_bits(offset) & 0xff;
2467                 }
2468                 break;
2469         case PACKET3_MEM_WRITE:
2470         {
2471                 u64 offset;
2472
2473                 if (pkt->count != 3) {
2474                         DRM_ERROR("bad MEM_WRITE (invalid count)\n");
2475                         return -EINVAL;
2476                 }
2477                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2478                 if (r) {
2479                         DRM_ERROR("bad MEM_WRITE (missing reloc)\n");
2480                         return -EINVAL;
2481                 }
2482                 offset = radeon_get_ib_value(p, idx+0);
2483                 offset += ((u64)(radeon_get_ib_value(p, idx+1) & 0xff)) << 32UL;
2484                 if (offset & 0x7) {
2485                         DRM_ERROR("bad MEM_WRITE (address not qwords aligned)\n");
2486                         return -EINVAL;
2487                 }
2488                 if ((offset + 8) > radeon_bo_size(reloc->robj)) {
2489                         DRM_ERROR("bad MEM_WRITE bo too small: 0x%llx, 0x%lx\n",
2490                                   offset + 8, radeon_bo_size(reloc->robj));
2491                         return -EINVAL;
2492                 }
2493                 offset += reloc->lobj.gpu_offset;
2494                 ib[idx+0] = offset;
2495                 ib[idx+1] = upper_32_bits(offset) & 0xff;
2496                 break;
2497         }
2498         case PACKET3_COPY_DW:
2499                 if (pkt->count != 4) {
2500                         DRM_ERROR("bad COPY_DW (invalid count)\n");
2501                         return -EINVAL;
2502                 }
2503                 if (idx_value & 0x1) {
2504                         u64 offset;
2505                         /* SRC is memory. */
2506                         r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2507                         if (r) {
2508                                 DRM_ERROR("bad COPY_DW (missing src reloc)\n");
2509                                 return -EINVAL;
2510                         }
2511                         offset = radeon_get_ib_value(p, idx+1);
2512                         offset += ((u64)(radeon_get_ib_value(p, idx+2) & 0xff)) << 32;
2513                         if ((offset + 4) > radeon_bo_size(reloc->robj)) {
2514                                 DRM_ERROR("bad COPY_DW src bo too small: 0x%llx, 0x%lx\n",
2515                                           offset + 4, radeon_bo_size(reloc->robj));
2516                                 return -EINVAL;
2517                         }
2518                         offset += reloc->lobj.gpu_offset;
2519                         ib[idx+1] = offset;
2520                         ib[idx+2] = upper_32_bits(offset) & 0xff;
2521                 } else {
2522                         /* SRC is a reg. */
2523                         reg = radeon_get_ib_value(p, idx+1) << 2;
2524                         if (!evergreen_is_safe_reg(p, reg, idx+1))
2525                                 return -EINVAL;
2526                 }
2527                 if (idx_value & 0x2) {
2528                         u64 offset;
2529                         /* DST is memory. */
2530                         r = radeon_cs_packet_next_reloc(p, &reloc, 0);
2531                         if (r) {
2532                                 DRM_ERROR("bad COPY_DW (missing dst reloc)\n");
2533                                 return -EINVAL;
2534                         }
2535                         offset = radeon_get_ib_value(p, idx+3);
2536                         offset += ((u64)(radeon_get_ib_value(p, idx+4) & 0xff)) << 32;
2537                         if ((offset + 4) > radeon_bo_size(reloc->robj)) {
2538                                 DRM_ERROR("bad COPY_DW dst bo too small: 0x%llx, 0x%lx\n",
2539                                           offset + 4, radeon_bo_size(reloc->robj));
2540                                 return -EINVAL;
2541                         }
2542                         offset += reloc->lobj.gpu_offset;
2543                         ib[idx+3] = offset;
2544                         ib[idx+4] = upper_32_bits(offset) & 0xff;
2545                 } else {
2546                         /* DST is a reg. */
2547                         reg = radeon_get_ib_value(p, idx+3) << 2;
2548                         if (!evergreen_is_safe_reg(p, reg, idx+3))
2549                                 return -EINVAL;
2550                 }
2551                 break;
2552         case PACKET3_NOP:
2553                 break;
2554         default:
2555                 DRM_ERROR("Packet3 opcode %x not supported\n", pkt->opcode);
2556                 return -EINVAL;
2557         }
2558         return 0;
2559 }
2560
2561 int evergreen_cs_parse(struct radeon_cs_parser *p)
2562 {
2563         struct radeon_cs_packet pkt;
2564         struct evergreen_cs_track *track;
2565         u32 tmp;
2566         int r;
2567
2568         if (p->track == NULL) {
2569                 /* initialize tracker, we are in kms */
2570                 track = kzalloc(sizeof(*track), GFP_KERNEL);
2571                 if (track == NULL)
2572                         return -ENOMEM;
2573                 evergreen_cs_track_init(track);
2574                 if (p->rdev->family >= CHIP_CAYMAN)
2575                         tmp = p->rdev->config.cayman.tile_config;
2576                 else
2577                         tmp = p->rdev->config.evergreen.tile_config;
2578
2579                 switch (tmp & 0xf) {
2580                 case 0:
2581                         track->npipes = 1;
2582                         break;
2583                 case 1:
2584                 default:
2585                         track->npipes = 2;
2586                         break;
2587                 case 2:
2588                         track->npipes = 4;
2589                         break;
2590                 case 3:
2591                         track->npipes = 8;
2592                         break;
2593                 }
2594
2595                 switch ((tmp & 0xf0) >> 4) {
2596                 case 0:
2597                         track->nbanks = 4;
2598                         break;
2599                 case 1:
2600                 default:
2601                         track->nbanks = 8;
2602                         break;
2603                 case 2:
2604                         track->nbanks = 16;
2605                         break;
2606                 }
2607
2608                 switch ((tmp & 0xf00) >> 8) {
2609                 case 0:
2610                         track->group_size = 256;
2611                         break;
2612                 case 1:
2613                 default:
2614                         track->group_size = 512;
2615                         break;
2616                 }
2617
2618                 switch ((tmp & 0xf000) >> 12) {
2619                 case 0:
2620                         track->row_size = 1;
2621                         break;
2622                 case 1:
2623                 default:
2624                         track->row_size = 2;
2625                         break;
2626                 case 2:
2627                         track->row_size = 4;
2628                         break;
2629                 }
2630
2631                 p->track = track;
2632         }
2633         do {
2634                 r = radeon_cs_packet_parse(p, &pkt, p->idx);
2635                 if (r) {
2636                         kfree(p->track);
2637                         p->track = NULL;
2638                         return r;
2639                 }
2640                 p->idx += pkt.count + 2;
2641                 switch (pkt.type) {
2642                 case RADEON_PACKET_TYPE0:
2643                         r = evergreen_cs_parse_packet0(p, &pkt);
2644                         break;
2645                 case RADEON_PACKET_TYPE2:
2646                         break;
2647                 case RADEON_PACKET_TYPE3:
2648                         r = evergreen_packet3_check(p, &pkt);
2649                         break;
2650                 default:
2651                         DRM_ERROR("Unknown packet type %d !\n", pkt.type);
2652                         kfree(p->track);
2653                         p->track = NULL;
2654                         return -EINVAL;
2655                 }
2656                 if (r) {
2657                         kfree(p->track);
2658                         p->track = NULL;
2659                         return r;
2660                 }
2661         } while (p->idx < p->chunks[p->chunk_ib_idx].length_dw);
2662 #if 0
2663         for (r = 0; r < p->ib.length_dw; r++) {
2664                 printk(KERN_INFO "%05d  0x%08X\n", r, p->ib.ptr[r]);
2665                 mdelay(1);
2666         }
2667 #endif
2668         kfree(p->track);
2669         p->track = NULL;
2670         return 0;
2671 }
2672
2673 /**
2674  * evergreen_dma_cs_parse() - parse the DMA IB
2675  * @p:          parser structure holding parsing context.
2676  *
2677  * Parses the DMA IB from the CS ioctl and updates
2678  * the GPU addresses based on the reloc information and
2679  * checks for errors. (Evergreen-Cayman)
2680  * Returns 0 for success and an error on failure.
2681  **/
2682 int evergreen_dma_cs_parse(struct radeon_cs_parser *p)
2683 {
2684         struct radeon_cs_chunk *ib_chunk = &p->chunks[p->chunk_ib_idx];
2685         struct radeon_cs_reloc *src_reloc, *dst_reloc, *dst2_reloc;
2686         u32 header, cmd, count, sub_cmd;
2687         volatile u32 *ib = p->ib.ptr;
2688         u32 idx;
2689         u64 src_offset, dst_offset, dst2_offset;
2690         int r;
2691
2692         do {
2693                 if (p->idx >= ib_chunk->length_dw) {
2694                         DRM_ERROR("Can not parse packet at %d after CS end %d !\n",
2695                                   p->idx, ib_chunk->length_dw);
2696                         return -EINVAL;
2697                 }
2698                 idx = p->idx;
2699                 header = radeon_get_ib_value(p, idx);
2700                 cmd = GET_DMA_CMD(header);
2701                 count = GET_DMA_COUNT(header);
2702                 sub_cmd = GET_DMA_SUB_CMD(header);
2703
2704                 switch (cmd) {
2705                 case DMA_PACKET_WRITE:
2706                         r = r600_dma_cs_next_reloc(p, &dst_reloc);
2707                         if (r) {
2708                                 DRM_ERROR("bad DMA_PACKET_WRITE\n");
2709                                 return -EINVAL;
2710                         }
2711                         switch (sub_cmd) {
2712                         /* tiled */
2713                         case 8:
2714                                 dst_offset = radeon_get_ib_value(p, idx+1);
2715                                 dst_offset <<= 8;
2716
2717                                 ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
2718                                 p->idx += count + 7;
2719                                 break;
2720                         /* linear */
2721                         case 0:
2722                                 dst_offset = radeon_get_ib_value(p, idx+1);
2723                                 dst_offset |= ((u64)(radeon_get_ib_value(p, idx+2) & 0xff)) << 32;
2724
2725                                 ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
2726                                 ib[idx+2] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
2727                                 p->idx += count + 3;
2728                                 break;
2729                         default:
2730                                 DRM_ERROR("bad DMA_PACKET_WRITE [%6d] 0x%08x sub cmd is not 0 or 8\n", idx, header);
2731                                 return -EINVAL;
2732                         }
2733                         if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) {
2734                                 dev_warn(p->dev, "DMA write buffer too small (%llu %lu)\n",
2735                                          dst_offset, radeon_bo_size(dst_reloc->robj));
2736                                 return -EINVAL;
2737                         }
2738                         break;
2739                 case DMA_PACKET_COPY:
2740                         r = r600_dma_cs_next_reloc(p, &src_reloc);
2741                         if (r) {
2742                                 DRM_ERROR("bad DMA_PACKET_COPY\n");
2743                                 return -EINVAL;
2744                         }
2745                         r = r600_dma_cs_next_reloc(p, &dst_reloc);
2746                         if (r) {
2747                                 DRM_ERROR("bad DMA_PACKET_COPY\n");
2748                                 return -EINVAL;
2749                         }
2750                         switch (sub_cmd) {
2751                         /* Copy L2L, DW aligned */
2752                         case 0x00:
2753                                 /* L2L, dw */
2754                                 src_offset = radeon_get_ib_value(p, idx+2);
2755                                 src_offset |= ((u64)(radeon_get_ib_value(p, idx+4) & 0xff)) << 32;
2756                                 dst_offset = radeon_get_ib_value(p, idx+1);
2757                                 dst_offset |= ((u64)(radeon_get_ib_value(p, idx+3) & 0xff)) << 32;
2758                                 if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) {
2759                                         dev_warn(p->dev, "DMA L2L, dw src buffer too small (%llu %lu)\n",
2760                                                         src_offset + (count * 4), radeon_bo_size(src_reloc->robj));
2761                                         return -EINVAL;
2762                                 }
2763                                 if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) {
2764                                         dev_warn(p->dev, "DMA L2L, dw dst buffer too small (%llu %lu)\n",
2765                                                         dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj));
2766                                         return -EINVAL;
2767                                 }
2768                                 ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
2769                                 ib[idx+2] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
2770                                 ib[idx+3] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
2771                                 ib[idx+4] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
2772                                 p->idx += 5;
2773                                 break;
2774                         /* Copy L2T/T2L */
2775                         case 0x08:
2776                                 /* detile bit */
2777                                 if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) {
2778                                         /* tiled src, linear dst */
2779                                         src_offset = radeon_get_ib_value(p, idx+1);
2780                                         src_offset <<= 8;
2781                                         ib[idx+1] += (u32)(src_reloc->lobj.gpu_offset >> 8);
2782
2783                                         dst_offset = radeon_get_ib_value(p, idx + 7);
2784                                         dst_offset |= ((u64)(radeon_get_ib_value(p, idx+8) & 0xff)) << 32;
2785                                         ib[idx+7] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
2786                                         ib[idx+8] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
2787                                 } else {
2788                                         /* linear src, tiled dst */
2789                                         src_offset = radeon_get_ib_value(p, idx+7);
2790                                         src_offset |= ((u64)(radeon_get_ib_value(p, idx+8) & 0xff)) << 32;
2791                                         ib[idx+7] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
2792                                         ib[idx+8] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
2793
2794                                         dst_offset = radeon_get_ib_value(p, idx+1);
2795                                         dst_offset <<= 8;
2796                                         ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
2797                                 }
2798                                 if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) {
2799                                         dev_warn(p->dev, "DMA L2T, src buffer too small (%llu %lu)\n",
2800                                                         src_offset + (count * 4), radeon_bo_size(src_reloc->robj));
2801                                         return -EINVAL;
2802                                 }
2803                                 if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) {
2804                                         dev_warn(p->dev, "DMA L2T, dst buffer too small (%llu %lu)\n",
2805                                                         dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj));
2806                                         return -EINVAL;
2807                                 }
2808                                 p->idx += 9;
2809                                 break;
2810                         /* Copy L2L, byte aligned */
2811                         case 0x40:
2812                                 /* L2L, byte */
2813                                 src_offset = radeon_get_ib_value(p, idx+2);
2814                                 src_offset |= ((u64)(radeon_get_ib_value(p, idx+4) & 0xff)) << 32;
2815                                 dst_offset = radeon_get_ib_value(p, idx+1);
2816                                 dst_offset |= ((u64)(radeon_get_ib_value(p, idx+3) & 0xff)) << 32;
2817                                 if ((src_offset + count) > radeon_bo_size(src_reloc->robj)) {
2818                                         dev_warn(p->dev, "DMA L2L, byte src buffer too small (%llu %lu)\n",
2819                                                         src_offset + count, radeon_bo_size(src_reloc->robj));
2820                                         return -EINVAL;
2821                                 }
2822                                 if ((dst_offset + count) > radeon_bo_size(dst_reloc->robj)) {
2823                                         dev_warn(p->dev, "DMA L2L, byte dst buffer too small (%llu %lu)\n",
2824                                                         dst_offset + count, radeon_bo_size(dst_reloc->robj));
2825                                         return -EINVAL;
2826                                 }
2827                                 ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xffffffff);
2828                                 ib[idx+2] += (u32)(src_reloc->lobj.gpu_offset & 0xffffffff);
2829                                 ib[idx+3] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
2830                                 ib[idx+4] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
2831                                 p->idx += 5;
2832                                 break;
2833                         /* Copy L2L, partial */
2834                         case 0x41:
2835                                 /* L2L, partial */
2836                                 if (p->family < CHIP_CAYMAN) {
2837                                         DRM_ERROR("L2L Partial is cayman only !\n");
2838                                         return -EINVAL;
2839                                 }
2840                                 ib[idx+1] += (u32)(src_reloc->lobj.gpu_offset & 0xffffffff);
2841                                 ib[idx+2] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
2842                                 ib[idx+4] += (u32)(dst_reloc->lobj.gpu_offset & 0xffffffff);
2843                                 ib[idx+5] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
2844
2845                                 p->idx += 9;
2846                                 break;
2847                         /* Copy L2L, DW aligned, broadcast */
2848                         case 0x44:
2849                                 /* L2L, dw, broadcast */
2850                                 r = r600_dma_cs_next_reloc(p, &dst2_reloc);
2851                                 if (r) {
2852                                         DRM_ERROR("bad L2L, dw, broadcast DMA_PACKET_COPY\n");
2853                                         return -EINVAL;
2854                                 }
2855                                 dst_offset = radeon_get_ib_value(p, idx+1);
2856                                 dst_offset |= ((u64)(radeon_get_ib_value(p, idx+4) & 0xff)) << 32;
2857                                 dst2_offset = radeon_get_ib_value(p, idx+2);
2858                                 dst2_offset |= ((u64)(radeon_get_ib_value(p, idx+5) & 0xff)) << 32;
2859                                 src_offset = radeon_get_ib_value(p, idx+3);
2860                                 src_offset |= ((u64)(radeon_get_ib_value(p, idx+6) & 0xff)) << 32;
2861                                 if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) {
2862                                         dev_warn(p->dev, "DMA L2L, dw, broadcast src buffer too small (%llu %lu)\n",
2863                                                         src_offset + (count * 4), radeon_bo_size(src_reloc->robj));
2864                                         return -EINVAL;
2865                                 }
2866                                 if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) {
2867                                         dev_warn(p->dev, "DMA L2L, dw, broadcast dst buffer too small (%llu %lu)\n",
2868                                                         dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj));
2869                                         return -EINVAL;
2870                                 }
2871                                 if ((dst2_offset + (count * 4)) > radeon_bo_size(dst2_reloc->robj)) {
2872                                         dev_warn(p->dev, "DMA L2L, dw, broadcast dst2 buffer too small (%llu %lu)\n",
2873                                                         dst2_offset + (count * 4), radeon_bo_size(dst2_reloc->robj));
2874                                         return -EINVAL;
2875                                 }
2876                                 ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
2877                                 ib[idx+2] += (u32)(dst2_reloc->lobj.gpu_offset & 0xfffffffc);
2878                                 ib[idx+3] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
2879                                 ib[idx+4] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
2880                                 ib[idx+5] += upper_32_bits(dst2_reloc->lobj.gpu_offset) & 0xff;
2881                                 ib[idx+6] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
2882                                 p->idx += 7;
2883                                 break;
2884                         /* Copy L2T Frame to Field */
2885                         case 0x48:
2886                                 if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) {
2887                                         DRM_ERROR("bad L2T, frame to fields DMA_PACKET_COPY\n");
2888                                         return -EINVAL;
2889                                 }
2890                                 r = r600_dma_cs_next_reloc(p, &dst2_reloc);
2891                                 if (r) {
2892                                         DRM_ERROR("bad L2T, frame to fields DMA_PACKET_COPY\n");
2893                                         return -EINVAL;
2894                                 }
2895                                 dst_offset = radeon_get_ib_value(p, idx+1);
2896                                 dst_offset <<= 8;
2897                                 dst2_offset = radeon_get_ib_value(p, idx+2);
2898                                 dst2_offset <<= 8;
2899                                 src_offset = radeon_get_ib_value(p, idx+8);
2900                                 src_offset |= ((u64)(radeon_get_ib_value(p, idx+9) & 0xff)) << 32;
2901                                 if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) {
2902                                         dev_warn(p->dev, "DMA L2T, frame to fields src buffer too small (%llu %lu)\n",
2903                                                         src_offset + (count * 4), radeon_bo_size(src_reloc->robj));
2904                                         return -EINVAL;
2905                                 }
2906                                 if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) {
2907                                         dev_warn(p->dev, "DMA L2T, frame to fields buffer too small (%llu %lu)\n",
2908                                                         dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj));
2909                                         return -EINVAL;
2910                                 }
2911                                 if ((dst2_offset + (count * 4)) > radeon_bo_size(dst2_reloc->robj)) {
2912                                         dev_warn(p->dev, "DMA L2T, frame to fields buffer too small (%llu %lu)\n",
2913                                                         dst2_offset + (count * 4), radeon_bo_size(dst2_reloc->robj));
2914                                         return -EINVAL;
2915                                 }
2916                                 ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
2917                                 ib[idx+2] += (u32)(dst2_reloc->lobj.gpu_offset >> 8);
2918                                 ib[idx+8] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
2919                                 ib[idx+9] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
2920                                 p->idx += 10;
2921                                 break;
2922                         /* Copy L2T/T2L, partial */
2923                         case 0x49:
2924                                 /* L2T, T2L partial */
2925                                 if (p->family < CHIP_CAYMAN) {
2926                                         DRM_ERROR("L2T, T2L Partial is cayman only !\n");
2927                                         return -EINVAL;
2928                                 }
2929                                 /* detile bit */
2930                                 if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) {
2931                                         /* tiled src, linear dst */
2932                                         ib[idx+1] += (u32)(src_reloc->lobj.gpu_offset >> 8);
2933
2934                                         ib[idx+7] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
2935                                         ib[idx+8] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
2936                                 } else {
2937                                         /* linear src, tiled dst */
2938                                         ib[idx+7] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
2939                                         ib[idx+8] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
2940
2941                                         ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
2942                                 }
2943                                 p->idx += 12;
2944                                 break;
2945                         /* Copy L2T broadcast */
2946                         case 0x4b:
2947                                 /* L2T, broadcast */
2948                                 if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) {
2949                                         DRM_ERROR("bad L2T, broadcast DMA_PACKET_COPY\n");
2950                                         return -EINVAL;
2951                                 }
2952                                 r = r600_dma_cs_next_reloc(p, &dst2_reloc);
2953                                 if (r) {
2954                                         DRM_ERROR("bad L2T, broadcast DMA_PACKET_COPY\n");
2955                                         return -EINVAL;
2956                                 }
2957                                 dst_offset = radeon_get_ib_value(p, idx+1);
2958                                 dst_offset <<= 8;
2959                                 dst2_offset = radeon_get_ib_value(p, idx+2);
2960                                 dst2_offset <<= 8;
2961                                 src_offset = radeon_get_ib_value(p, idx+8);
2962                                 src_offset |= ((u64)(radeon_get_ib_value(p, idx+9) & 0xff)) << 32;
2963                                 if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) {
2964                                         dev_warn(p->dev, "DMA L2T, broadcast src buffer too small (%llu %lu)\n",
2965                                                         src_offset + (count * 4), radeon_bo_size(src_reloc->robj));
2966                                         return -EINVAL;
2967                                 }
2968                                 if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) {
2969                                         dev_warn(p->dev, "DMA L2T, broadcast dst buffer too small (%llu %lu)\n",
2970                                                         dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj));
2971                                         return -EINVAL;
2972                                 }
2973                                 if ((dst2_offset + (count * 4)) > radeon_bo_size(dst2_reloc->robj)) {
2974                                         dev_warn(p->dev, "DMA L2T, broadcast dst2 buffer too small (%llu %lu)\n",
2975                                                         dst2_offset + (count * 4), radeon_bo_size(dst2_reloc->robj));
2976                                         return -EINVAL;
2977                                 }
2978                                 ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
2979                                 ib[idx+2] += (u32)(dst2_reloc->lobj.gpu_offset >> 8);
2980                                 ib[idx+8] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
2981                                 ib[idx+9] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
2982                                 p->idx += 10;
2983                                 break;
2984                         /* Copy L2T/T2L (tile units) */
2985                         case 0x4c:
2986                                 /* L2T, T2L */
2987                                 /* detile bit */
2988                                 if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) {
2989                                         /* tiled src, linear dst */
2990                                         src_offset = radeon_get_ib_value(p, idx+1);
2991                                         src_offset <<= 8;
2992                                         ib[idx+1] += (u32)(src_reloc->lobj.gpu_offset >> 8);
2993
2994                                         dst_offset = radeon_get_ib_value(p, idx+7);
2995                                         dst_offset |= ((u64)(radeon_get_ib_value(p, idx+8) & 0xff)) << 32;
2996                                         ib[idx+7] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
2997                                         ib[idx+8] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
2998                                 } else {
2999                                         /* linear src, tiled dst */
3000                                         src_offset = radeon_get_ib_value(p, idx+7);
3001                                         src_offset |= ((u64)(radeon_get_ib_value(p, idx+8) & 0xff)) << 32;
3002                                         ib[idx+7] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
3003                                         ib[idx+8] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
3004
3005                                         dst_offset = radeon_get_ib_value(p, idx+1);
3006                                         dst_offset <<= 8;
3007                                         ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
3008                                 }
3009                                 if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) {
3010                                         dev_warn(p->dev, "DMA L2T, T2L src buffer too small (%llu %lu)\n",
3011                                                         src_offset + (count * 4), radeon_bo_size(src_reloc->robj));
3012                                         return -EINVAL;
3013                                 }
3014                                 if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) {
3015                                         dev_warn(p->dev, "DMA L2T, T2L dst buffer too small (%llu %lu)\n",
3016                                                         dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj));
3017                                         return -EINVAL;
3018                                 }
3019                                 p->idx += 9;
3020                                 break;
3021                         /* Copy T2T, partial (tile units) */
3022                         case 0x4d:
3023                                 /* T2T partial */
3024                                 if (p->family < CHIP_CAYMAN) {
3025                                         DRM_ERROR("L2T, T2L Partial is cayman only !\n");
3026                                         return -EINVAL;
3027                                 }
3028                                 ib[idx+1] += (u32)(src_reloc->lobj.gpu_offset >> 8);
3029                                 ib[idx+4] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
3030                                 p->idx += 13;
3031                                 break;
3032                         /* Copy L2T broadcast (tile units) */
3033                         case 0x4f:
3034                                 /* L2T, broadcast */
3035                                 if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) {
3036                                         DRM_ERROR("bad L2T, broadcast DMA_PACKET_COPY\n");
3037                                         return -EINVAL;
3038                                 }
3039                                 r = r600_dma_cs_next_reloc(p, &dst2_reloc);
3040                                 if (r) {
3041                                         DRM_ERROR("bad L2T, broadcast DMA_PACKET_COPY\n");
3042                                         return -EINVAL;
3043                                 }
3044                                 dst_offset = radeon_get_ib_value(p, idx+1);
3045                                 dst_offset <<= 8;
3046                                 dst2_offset = radeon_get_ib_value(p, idx+2);
3047                                 dst2_offset <<= 8;
3048                                 src_offset = radeon_get_ib_value(p, idx+8);
3049                                 src_offset |= ((u64)(radeon_get_ib_value(p, idx+9) & 0xff)) << 32;
3050                                 if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) {
3051                                         dev_warn(p->dev, "DMA L2T, broadcast src buffer too small (%llu %lu)\n",
3052                                                         src_offset + (count * 4), radeon_bo_size(src_reloc->robj));
3053                                         return -EINVAL;
3054                                 }
3055                                 if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) {
3056                                         dev_warn(p->dev, "DMA L2T, broadcast dst buffer too small (%llu %lu)\n",
3057                                                         dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj));
3058                                         return -EINVAL;
3059                                 }
3060                                 if ((dst2_offset + (count * 4)) > radeon_bo_size(dst2_reloc->robj)) {
3061                                         dev_warn(p->dev, "DMA L2T, broadcast dst2 buffer too small (%llu %lu)\n",
3062                                                         dst2_offset + (count * 4), radeon_bo_size(dst2_reloc->robj));
3063                                         return -EINVAL;
3064                                 }
3065                                 ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
3066                                 ib[idx+2] += (u32)(dst2_reloc->lobj.gpu_offset >> 8);
3067                                 ib[idx+8] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
3068                                 ib[idx+9] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
3069                                 p->idx += 10;
3070                                 break;
3071                         default:
3072                                 DRM_ERROR("bad DMA_PACKET_COPY [%6d] 0x%08x invalid sub cmd\n", idx, header);
3073                                 return -EINVAL;
3074                         }
3075                         break;
3076                 case DMA_PACKET_CONSTANT_FILL:
3077                         r = r600_dma_cs_next_reloc(p, &dst_reloc);
3078                         if (r) {
3079                                 DRM_ERROR("bad DMA_PACKET_CONSTANT_FILL\n");
3080                                 return -EINVAL;
3081                         }
3082                         dst_offset = radeon_get_ib_value(p, idx+1);
3083                         dst_offset |= ((u64)(radeon_get_ib_value(p, idx+3) & 0x00ff0000)) << 16;
3084                         if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) {
3085                                 dev_warn(p->dev, "DMA constant fill buffer too small (%llu %lu)\n",
3086                                          dst_offset, radeon_bo_size(dst_reloc->robj));
3087                                 return -EINVAL;
3088                         }
3089                         ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
3090                         ib[idx+3] += (upper_32_bits(dst_reloc->lobj.gpu_offset) << 16) & 0x00ff0000;
3091                         p->idx += 4;
3092                         break;
3093                 case DMA_PACKET_NOP:
3094                         p->idx += 1;
3095                         break;
3096                 default:
3097                         DRM_ERROR("Unknown packet type %d at %d !\n", cmd, idx);
3098                         return -EINVAL;
3099                 }
3100         } while (p->idx < p->chunks[p->chunk_ib_idx].length_dw);
3101 #if 0
3102         for (r = 0; r < p->ib->length_dw; r++) {
3103                 printk(KERN_INFO "%05d  0x%08X\n", r, p->ib.ptr[r]);
3104                 mdelay(1);
3105         }
3106 #endif
3107         return 0;
3108 }
3109
3110 /* vm parser */
3111 static bool evergreen_vm_reg_valid(u32 reg)
3112 {
3113         /* context regs are fine */
3114         if (reg >= 0x28000)
3115                 return true;
3116
3117         /* check config regs */
3118         switch (reg) {
3119         case WAIT_UNTIL:
3120         case GRBM_GFX_INDEX:
3121         case CP_STRMOUT_CNTL:
3122         case CP_COHER_CNTL:
3123         case CP_COHER_SIZE:
3124         case VGT_VTX_VECT_EJECT_REG:
3125         case VGT_CACHE_INVALIDATION:
3126         case VGT_GS_VERTEX_REUSE:
3127         case VGT_PRIMITIVE_TYPE:
3128         case VGT_INDEX_TYPE:
3129         case VGT_NUM_INDICES:
3130         case VGT_NUM_INSTANCES:
3131         case VGT_COMPUTE_DIM_X:
3132         case VGT_COMPUTE_DIM_Y:
3133         case VGT_COMPUTE_DIM_Z:
3134         case VGT_COMPUTE_START_X:
3135         case VGT_COMPUTE_START_Y:
3136         case VGT_COMPUTE_START_Z:
3137         case VGT_COMPUTE_INDEX:
3138         case VGT_COMPUTE_THREAD_GROUP_SIZE:
3139         case VGT_HS_OFFCHIP_PARAM:
3140         case PA_CL_ENHANCE:
3141         case PA_SU_LINE_STIPPLE_VALUE:
3142         case PA_SC_LINE_STIPPLE_STATE:
3143         case PA_SC_ENHANCE:
3144         case SQ_DYN_GPR_CNTL_PS_FLUSH_REQ:
3145         case SQ_DYN_GPR_SIMD_LOCK_EN:
3146         case SQ_CONFIG:
3147         case SQ_GPR_RESOURCE_MGMT_1:
3148         case SQ_GLOBAL_GPR_RESOURCE_MGMT_1:
3149         case SQ_GLOBAL_GPR_RESOURCE_MGMT_2:
3150         case SQ_CONST_MEM_BASE:
3151         case SQ_STATIC_THREAD_MGMT_1:
3152         case SQ_STATIC_THREAD_MGMT_2:
3153         case SQ_STATIC_THREAD_MGMT_3:
3154         case SPI_CONFIG_CNTL:
3155         case SPI_CONFIG_CNTL_1:
3156         case TA_CNTL_AUX:
3157         case DB_DEBUG:
3158         case DB_DEBUG2:
3159         case DB_DEBUG3:
3160         case DB_DEBUG4:
3161         case DB_WATERMARKS:
3162         case TD_PS_BORDER_COLOR_INDEX:
3163         case TD_PS_BORDER_COLOR_RED:
3164         case TD_PS_BORDER_COLOR_GREEN:
3165         case TD_PS_BORDER_COLOR_BLUE:
3166         case TD_PS_BORDER_COLOR_ALPHA:
3167         case TD_VS_BORDER_COLOR_INDEX:
3168         case TD_VS_BORDER_COLOR_RED:
3169         case TD_VS_BORDER_COLOR_GREEN:
3170         case TD_VS_BORDER_COLOR_BLUE:
3171         case TD_VS_BORDER_COLOR_ALPHA:
3172         case TD_GS_BORDER_COLOR_INDEX:
3173         case TD_GS_BORDER_COLOR_RED:
3174         case TD_GS_BORDER_COLOR_GREEN:
3175         case TD_GS_BORDER_COLOR_BLUE:
3176         case TD_GS_BORDER_COLOR_ALPHA:
3177         case TD_HS_BORDER_COLOR_INDEX:
3178         case TD_HS_BORDER_COLOR_RED:
3179         case TD_HS_BORDER_COLOR_GREEN:
3180         case TD_HS_BORDER_COLOR_BLUE:
3181         case TD_HS_BORDER_COLOR_ALPHA:
3182         case TD_LS_BORDER_COLOR_INDEX:
3183         case TD_LS_BORDER_COLOR_RED:
3184         case TD_LS_BORDER_COLOR_GREEN:
3185         case TD_LS_BORDER_COLOR_BLUE:
3186         case TD_LS_BORDER_COLOR_ALPHA:
3187         case TD_CS_BORDER_COLOR_INDEX:
3188         case TD_CS_BORDER_COLOR_RED:
3189         case TD_CS_BORDER_COLOR_GREEN:
3190         case TD_CS_BORDER_COLOR_BLUE:
3191         case TD_CS_BORDER_COLOR_ALPHA:
3192         case SQ_ESGS_RING_SIZE:
3193         case SQ_GSVS_RING_SIZE:
3194         case SQ_ESTMP_RING_SIZE:
3195         case SQ_GSTMP_RING_SIZE:
3196         case SQ_HSTMP_RING_SIZE:
3197         case SQ_LSTMP_RING_SIZE:
3198         case SQ_PSTMP_RING_SIZE:
3199         case SQ_VSTMP_RING_SIZE:
3200         case SQ_ESGS_RING_ITEMSIZE:
3201         case SQ_ESTMP_RING_ITEMSIZE:
3202         case SQ_GSTMP_RING_ITEMSIZE:
3203         case SQ_GSVS_RING_ITEMSIZE:
3204         case SQ_GS_VERT_ITEMSIZE:
3205         case SQ_GS_VERT_ITEMSIZE_1:
3206         case SQ_GS_VERT_ITEMSIZE_2:
3207         case SQ_GS_VERT_ITEMSIZE_3:
3208         case SQ_GSVS_RING_OFFSET_1:
3209         case SQ_GSVS_RING_OFFSET_2:
3210         case SQ_GSVS_RING_OFFSET_3:
3211         case SQ_HSTMP_RING_ITEMSIZE:
3212         case SQ_LSTMP_RING_ITEMSIZE:
3213         case SQ_PSTMP_RING_ITEMSIZE:
3214         case SQ_VSTMP_RING_ITEMSIZE:
3215         case VGT_TF_RING_SIZE:
3216         case SQ_ESGS_RING_BASE:
3217         case SQ_GSVS_RING_BASE:
3218         case SQ_ESTMP_RING_BASE:
3219         case SQ_GSTMP_RING_BASE:
3220         case SQ_HSTMP_RING_BASE:
3221         case SQ_LSTMP_RING_BASE:
3222         case SQ_PSTMP_RING_BASE:
3223         case SQ_VSTMP_RING_BASE:
3224         case CAYMAN_VGT_OFFCHIP_LDS_BASE:
3225         case CAYMAN_SQ_EX_ALLOC_TABLE_SLOTS:
3226                 return true;
3227         default:
3228                 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
3229                 return false;
3230         }
3231 }
3232
3233 static int evergreen_vm_packet3_check(struct radeon_device *rdev,
3234                                       u32 *ib, struct radeon_cs_packet *pkt)
3235 {
3236         u32 idx = pkt->idx + 1;
3237         u32 idx_value = ib[idx];
3238         u32 start_reg, end_reg, reg, i;
3239         u32 command, info;
3240
3241         switch (pkt->opcode) {
3242         case PACKET3_NOP:
3243         case PACKET3_SET_BASE:
3244         case PACKET3_CLEAR_STATE:
3245         case PACKET3_INDEX_BUFFER_SIZE:
3246         case PACKET3_DISPATCH_DIRECT:
3247         case PACKET3_DISPATCH_INDIRECT:
3248         case PACKET3_MODE_CONTROL:
3249         case PACKET3_SET_PREDICATION:
3250         case PACKET3_COND_EXEC:
3251         case PACKET3_PRED_EXEC:
3252         case PACKET3_DRAW_INDIRECT:
3253         case PACKET3_DRAW_INDEX_INDIRECT:
3254         case PACKET3_INDEX_BASE:
3255         case PACKET3_DRAW_INDEX_2:
3256         case PACKET3_CONTEXT_CONTROL:
3257         case PACKET3_DRAW_INDEX_OFFSET:
3258         case PACKET3_INDEX_TYPE:
3259         case PACKET3_DRAW_INDEX:
3260         case PACKET3_DRAW_INDEX_AUTO:
3261         case PACKET3_DRAW_INDEX_IMMD:
3262         case PACKET3_NUM_INSTANCES:
3263         case PACKET3_DRAW_INDEX_MULTI_AUTO:
3264         case PACKET3_STRMOUT_BUFFER_UPDATE:
3265         case PACKET3_DRAW_INDEX_OFFSET_2:
3266         case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
3267         case PACKET3_MPEG_INDEX:
3268         case PACKET3_WAIT_REG_MEM:
3269         case PACKET3_MEM_WRITE:
3270         case PACKET3_SURFACE_SYNC:
3271         case PACKET3_EVENT_WRITE:
3272         case PACKET3_EVENT_WRITE_EOP:
3273         case PACKET3_EVENT_WRITE_EOS:
3274         case PACKET3_SET_CONTEXT_REG:
3275         case PACKET3_SET_BOOL_CONST:
3276         case PACKET3_SET_LOOP_CONST:
3277         case PACKET3_SET_RESOURCE:
3278         case PACKET3_SET_SAMPLER:
3279         case PACKET3_SET_CTL_CONST:
3280         case PACKET3_SET_RESOURCE_OFFSET:
3281         case PACKET3_SET_CONTEXT_REG_INDIRECT:
3282         case PACKET3_SET_RESOURCE_INDIRECT:
3283         case CAYMAN_PACKET3_DEALLOC_STATE:
3284                 break;
3285         case PACKET3_COND_WRITE:
3286                 if (idx_value & 0x100) {
3287                         reg = ib[idx + 5] * 4;
3288                         if (!evergreen_vm_reg_valid(reg))
3289                                 return -EINVAL;
3290                 }
3291                 break;
3292         case PACKET3_COPY_DW:
3293                 if (idx_value & 0x2) {
3294                         reg = ib[idx + 3] * 4;
3295                         if (!evergreen_vm_reg_valid(reg))
3296                                 return -EINVAL;
3297                 }
3298                 break;
3299         case PACKET3_SET_CONFIG_REG:
3300                 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
3301                 end_reg = 4 * pkt->count + start_reg - 4;
3302                 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
3303                     (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
3304                     (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
3305                         DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
3306                         return -EINVAL;
3307                 }
3308                 for (i = 0; i < pkt->count; i++) {
3309                         reg = start_reg + (4 * i);
3310                         if (!evergreen_vm_reg_valid(reg))
3311                                 return -EINVAL;
3312                 }
3313                 break;
3314         case PACKET3_CP_DMA:
3315                 command = ib[idx + 4];
3316                 info = ib[idx + 1];
3317                 if ((((info & 0x60000000) >> 29) != 0) || /* src = GDS or DATA */
3318                     (((info & 0x00300000) >> 20) != 0) || /* dst = GDS */
3319                     ((((info & 0x00300000) >> 20) == 0) &&
3320                      (command & PACKET3_CP_DMA_CMD_DAS)) || /* dst = register */
3321                     ((((info & 0x60000000) >> 29) == 0) &&
3322                      (command & PACKET3_CP_DMA_CMD_SAS))) { /* src = register */
3323                         /* non mem to mem copies requires dw aligned count */
3324                         if ((command & 0x1fffff) % 4) {
3325                                 DRM_ERROR("CP DMA command requires dw count alignment\n");
3326                                 return -EINVAL;
3327                         }
3328                 }
3329                 if (command & PACKET3_CP_DMA_CMD_SAS) {
3330                         /* src address space is register */
3331                         if (((info & 0x60000000) >> 29) == 0) {
3332                                 start_reg = idx_value << 2;
3333                                 if (command & PACKET3_CP_DMA_CMD_SAIC) {
3334                                         reg = start_reg;
3335                                         if (!evergreen_vm_reg_valid(reg)) {
3336                                                 DRM_ERROR("CP DMA Bad SRC register\n");
3337                                                 return -EINVAL;
3338                                         }
3339                                 } else {
3340                                         for (i = 0; i < (command & 0x1fffff); i++) {
3341                                                 reg = start_reg + (4 * i);
3342                                                 if (!evergreen_vm_reg_valid(reg)) {
3343                                                         DRM_ERROR("CP DMA Bad SRC register\n");
3344                                                         return -EINVAL;
3345                                                 }
3346                                         }
3347                                 }
3348                         }
3349                 }
3350                 if (command & PACKET3_CP_DMA_CMD_DAS) {
3351                         /* dst address space is register */
3352                         if (((info & 0x00300000) >> 20) == 0) {
3353                                 start_reg = ib[idx + 2];
3354                                 if (command & PACKET3_CP_DMA_CMD_DAIC) {
3355                                         reg = start_reg;
3356                                         if (!evergreen_vm_reg_valid(reg)) {
3357                                                 DRM_ERROR("CP DMA Bad DST register\n");
3358                                                 return -EINVAL;
3359                                         }
3360                                 } else {
3361                                         for (i = 0; i < (command & 0x1fffff); i++) {
3362                                                 reg = start_reg + (4 * i);
3363                                                 if (!evergreen_vm_reg_valid(reg)) {
3364                                                         DRM_ERROR("CP DMA Bad DST register\n");
3365                                                         return -EINVAL;
3366                                                 }
3367                                         }
3368                                 }
3369                         }
3370                 }
3371                 break;
3372         default:
3373                 return -EINVAL;
3374         }
3375         return 0;
3376 }
3377
3378 int evergreen_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
3379 {
3380         int ret = 0;
3381         u32 idx = 0;
3382         struct radeon_cs_packet pkt;
3383
3384         do {
3385                 pkt.idx = idx;
3386                 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
3387                 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
3388                 pkt.one_reg_wr = 0;
3389                 switch (pkt.type) {
3390                 case RADEON_PACKET_TYPE0:
3391                         dev_err(rdev->dev, "Packet0 not allowed!\n");
3392                         ret = -EINVAL;
3393                         break;
3394                 case RADEON_PACKET_TYPE2:
3395                         idx += 1;
3396                         break;
3397                 case RADEON_PACKET_TYPE3:
3398                         pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
3399                         ret = evergreen_vm_packet3_check(rdev, ib->ptr, &pkt);
3400                         idx += pkt.count + 2;
3401                         break;
3402                 default:
3403                         dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
3404                         ret = -EINVAL;
3405                         break;
3406                 }
3407                 if (ret)
3408                         break;
3409         } while (idx < ib->length_dw);
3410
3411         return ret;
3412 }
3413
3414 /**
3415  * evergreen_dma_ib_parse() - parse the DMA IB for VM
3416  * @rdev: radeon_device pointer
3417  * @ib: radeon_ib pointer
3418  *
3419  * Parses the DMA IB from the VM CS ioctl
3420  * checks for errors. (Cayman-SI)
3421  * Returns 0 for success and an error on failure.
3422  **/
3423 int evergreen_dma_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
3424 {
3425         u32 idx = 0;
3426         u32 header, cmd, count, sub_cmd;
3427
3428         do {
3429                 header = ib->ptr[idx];
3430                 cmd = GET_DMA_CMD(header);
3431                 count = GET_DMA_COUNT(header);
3432                 sub_cmd = GET_DMA_SUB_CMD(header);
3433
3434                 switch (cmd) {
3435                 case DMA_PACKET_WRITE:
3436                         switch (sub_cmd) {
3437                         /* tiled */
3438                         case 8:
3439                                 idx += count + 7;
3440                                 break;
3441                         /* linear */
3442                         case 0:
3443                                 idx += count + 3;
3444                                 break;
3445                         default:
3446                                 DRM_ERROR("bad DMA_PACKET_WRITE [%6d] 0x%08x sub cmd is not 0 or 8\n", idx, ib->ptr[idx]);
3447                                 return -EINVAL;
3448                         }
3449                         break;
3450                 case DMA_PACKET_COPY:
3451                         switch (sub_cmd) {
3452                         /* Copy L2L, DW aligned */
3453                         case 0x00:
3454                                 idx += 5;
3455                                 break;
3456                         /* Copy L2T/T2L */
3457                         case 0x08:
3458                                 idx += 9;
3459                                 break;
3460                         /* Copy L2L, byte aligned */
3461                         case 0x40:
3462                                 idx += 5;
3463                                 break;
3464                         /* Copy L2L, partial */
3465                         case 0x41:
3466                                 idx += 9;
3467                                 break;
3468                         /* Copy L2L, DW aligned, broadcast */
3469                         case 0x44:
3470                                 idx += 7;
3471                                 break;
3472                         /* Copy L2T Frame to Field */
3473                         case 0x48:
3474                                 idx += 10;
3475                                 break;
3476                         /* Copy L2T/T2L, partial */
3477                         case 0x49:
3478                                 idx += 12;
3479                                 break;
3480                         /* Copy L2T broadcast */
3481                         case 0x4b:
3482                                 idx += 10;
3483                                 break;
3484                         /* Copy L2T/T2L (tile units) */
3485                         case 0x4c:
3486                                 idx += 9;
3487                                 break;
3488                         /* Copy T2T, partial (tile units) */
3489                         case 0x4d:
3490                                 idx += 13;
3491                                 break;
3492                         /* Copy L2T broadcast (tile units) */
3493                         case 0x4f:
3494                                 idx += 10;
3495                                 break;
3496                         default:
3497                                 DRM_ERROR("bad DMA_PACKET_COPY [%6d] 0x%08x invalid sub cmd\n", idx, ib->ptr[idx]);
3498                                 return -EINVAL;
3499                         }
3500                         break;
3501                 case DMA_PACKET_CONSTANT_FILL:
3502                         idx += 4;
3503                         break;
3504                 case DMA_PACKET_NOP:
3505                         idx += 1;
3506                         break;
3507                 default:
3508                         DRM_ERROR("Unknown packet type %d at %d !\n", cmd, idx);
3509                         return -EINVAL;
3510                 }
3511         } while (idx < ib->length_dw);
3512
3513         return 0;
3514 }