drivers/gpu/drm/nouveau/core/engine/fifo/nve0.c

   1 /*
   2  * Copyright 2012 Red Hat Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Ben Skeggs
  23  */
  24
  25 #include <core/client.h>
  26 #include <core/handle.h>
  27 #include <core/namedb.h>
  28 #include <core/gpuobj.h>
  29 #include <core/engctx.h>
  30 #include <core/event.h>
  31 #include <core/class.h>
  32 #include <core/enum.h>
  33
  34 #include <subdev/timer.h>
  35 #include <subdev/bar.h>
  36 #include <subdev/fb.h>
  37 #include <subdev/vm.h>
  38
  39 #include <engine/dmaobj.h>
  40
  41 #include "nve0.h"
  42
  43 #define _(a,b) { (a), ((1ULL << (a)) | (b)) }
  44 static const struct {
  45         u64 subdev;
  46         u64 mask;
  47 } fifo_engine[] = {
  48         _(NVDEV_ENGINE_GR      , (1ULL << NVDEV_ENGINE_SW) |
  49                                  (1ULL << NVDEV_ENGINE_COPY2)),
  50         _(NVDEV_ENGINE_VP      , 0),
  51         _(NVDEV_ENGINE_PPP     , 0),
  52         _(NVDEV_ENGINE_BSP     , 0),
  53         _(NVDEV_ENGINE_COPY0   , 0),
  54         _(NVDEV_ENGINE_COPY1   , 0),
  55         _(NVDEV_ENGINE_VENC    , 0),
  56 };
  57 #undef _
  58 #define FIFO_ENGINE_NR ARRAY_SIZE(fifo_engine)
  59
  60 struct nve0_fifo_engn {
  61         struct nouveau_gpuobj *runlist[2];
  62         int cur_runlist;
  63         wait_queue_head_t wait;
  64 };
  65
  66 struct nve0_fifo_priv {
  67         struct nouveau_fifo base;
  68
  69         struct work_struct fault;
  70         u64 mask;
  71
  72         struct nve0_fifo_engn engine[FIFO_ENGINE_NR];
  73         struct {
  74                 struct nouveau_gpuobj *mem;
  75                 struct nouveau_vma bar;
  76         } user;
  77         int spoon_nr;
  78 };
  79
  80 struct nve0_fifo_base {
  81         struct nouveau_fifo_base base;
  82         struct nouveau_gpuobj *pgd;
  83         struct nouveau_vm *vm;
  84 };
  85
  86 struct nve0_fifo_chan {
  87         struct nouveau_fifo_chan base;
  88         u32 engine;
  89         enum {
  90                 STOPPED,
  91                 RUNNING,
  92                 KILLED
  93         } state;
  94 };
  95
  96 /*******************************************************************************
  97  * FIFO channel objects
  98  ******************************************************************************/
  99
 100 static void
 101 nve0_fifo_runlist_update(struct nve0_fifo_priv *priv, u32 engine)
 102 {
 103         struct nouveau_bar *bar = nouveau_bar(priv);
 104         struct nve0_fifo_engn *engn = &priv->engine[engine];
 105         struct nouveau_gpuobj *cur;
 106         int i, p;
 107
 108         mutex_lock(&nv_subdev(priv)->mutex);
 109         cur = engn->runlist[engn->cur_runlist];
 110         engn->cur_runlist = !engn->cur_runlist;
 111
 112         for (i = 0, p = 0; i < priv->base.max; i++) {
 113                 struct nve0_fifo_chan *chan = (void *)priv->base.channel[i];
 114                 if (chan && chan->state == RUNNING && chan->engine == engine) {
 115                         nv_wo32(cur, p + 0, i);
 116                         nv_wo32(cur, p + 4, 0x00000000);
 117                         p += 8;
 118                 }
 119         }
 120         bar->flush(bar);
 121
 122         nv_wr32(priv, 0x002270, cur->addr >> 12);
 123         nv_wr32(priv, 0x002274, (engine << 20) | (p >> 3));
 124
 125         if (wait_event_timeout(engn->wait, !(nv_rd32(priv, 0x002284 +
 126                                (engine * 0x08)) & 0x00100000),
 127                                 msecs_to_jiffies(2000)) == 0)
 128                 nv_error(priv, "runlist %d update timeout\n", engine);
 129         mutex_unlock(&nv_subdev(priv)->mutex);
 130 }
 131
 132 static int
 133 nve0_fifo_context_attach(struct nouveau_object *parent,
 134                          struct nouveau_object *object)
 135 {
 136         struct nouveau_bar *bar = nouveau_bar(parent);
 137         struct nve0_fifo_base *base = (void *)parent->parent;
 138         struct nouveau_engctx *ectx = (void *)object;
 139         u32 addr;
 140         int ret;
 141
 142         switch (nv_engidx(object->engine)) {
 143         case NVDEV_ENGINE_SW   :
 144                 return 0;
 145         case NVDEV_ENGINE_COPY0:
 146         case NVDEV_ENGINE_COPY1:
 147         case NVDEV_ENGINE_COPY2:
 148                 nv_engctx(ectx)->addr = nv_gpuobj(base)->addr >> 12;
 149                 return 0;
 150         case NVDEV_ENGINE_GR   : addr = 0x0210; break;
 151         case NVDEV_ENGINE_BSP  : addr = 0x0270; break;
 152         case NVDEV_ENGINE_VP   : addr = 0x0250; break;
 153         case NVDEV_ENGINE_PPP  : addr = 0x0260; break;
 154         default:
 155                 return -EINVAL;
 156         }
 157
 158         if (!ectx->vma.node) {
 159                 ret = nouveau_gpuobj_map_vm(nv_gpuobj(ectx), base->vm,
 160                                             NV_MEM_ACCESS_RW, &ectx->vma);
 161                 if (ret)
 162                         return ret;
 163
 164                 nv_engctx(ectx)->addr = nv_gpuobj(base)->addr >> 12;
 165         }
 166
 167         nv_wo32(base, addr + 0x00, lower_32_bits(ectx->vma.offset) | 4);
 168         nv_wo32(base, addr + 0x04, upper_32_bits(ectx->vma.offset));
 169         bar->flush(bar);
 170         return 0;
 171 }
 172
 173 static int
 174 nve0_fifo_context_detach(struct nouveau_object *parent, bool suspend,
 175                          struct nouveau_object *object)
 176 {
 177         struct nouveau_bar *bar = nouveau_bar(parent);
 178         struct nve0_fifo_priv *priv = (void *)parent->engine;
 179         struct nve0_fifo_base *base = (void *)parent->parent;
 180         struct nve0_fifo_chan *chan = (void *)parent;
 181         u32 addr;
 182
 183         switch (nv_engidx(object->engine)) {
 184         case NVDEV_ENGINE_SW   : return 0;
 185         case NVDEV_ENGINE_COPY0:
 186         case NVDEV_ENGINE_COPY1:
 187         case NVDEV_ENGINE_COPY2: addr = 0x0000; break;
 188         case NVDEV_ENGINE_GR   : addr = 0x0210; break;
 189         case NVDEV_ENGINE_BSP  : addr = 0x0270; break;
 190         case NVDEV_ENGINE_VP   : addr = 0x0250; break;
 191         case NVDEV_ENGINE_PPP  : addr = 0x0260; break;
 192         default:
 193                 return -EINVAL;
 194         }
 195
 196         nv_wr32(priv, 0x002634, chan->base.chid);
 197         if (!nv_wait(priv, 0x002634, 0xffffffff, chan->base.chid)) {
 198                 nv_error(priv, "channel %d [%s] kick timeout\n",
 199                          chan->base.chid, nouveau_client_name(chan));
 200                 if (suspend)
 201                         return -EBUSY;
 202         }
 203
 204         if (addr) {
 205                 nv_wo32(base, addr + 0x00, 0x00000000);
 206                 nv_wo32(base, addr + 0x04, 0x00000000);
 207                 bar->flush(bar);
 208         }
 209
 210         return 0;
 211 }
 212
 213 static int
 214 nve0_fifo_chan_ctor(struct nouveau_object *parent,
 215                     struct nouveau_object *engine,
 216                     struct nouveau_oclass *oclass, void *data, u32 size,
 217                     struct nouveau_object **pobject)
 218 {
 219         struct nouveau_bar *bar = nouveau_bar(parent);
 220         struct nve0_fifo_priv *priv = (void *)engine;
 221         struct nve0_fifo_base *base = (void *)parent;
 222         struct nve0_fifo_chan *chan;
 223         struct nve0_channel_ind_class *args = data;
 224         u64 usermem, ioffset, ilength;
 225         int ret, i;
 226
 227         if (size < sizeof(*args))
 228                 return -EINVAL;
 229
 230         for (i = 0; i < FIFO_ENGINE_NR; i++) {
 231                 if (args->engine & (1 << i)) {
 232                         if (nouveau_engine(parent, fifo_engine[i].subdev)) {
 233                                 args->engine = (1 << i);
 234                                 break;
 235                         }
 236                 }
 237         }
 238
 239         if (i == FIFO_ENGINE_NR) {
 240                 nv_error(priv, "unsupported engines 0x%08x\n", args->engine);
 241                 return -ENODEV;
 242         }
 243
 244         ret = nouveau_fifo_channel_create(parent, engine, oclass, 1,
 245                                           priv->user.bar.offset, 0x200,
 246                                           args->pushbuf,
 247                                           fifo_engine[i].mask, &chan);
 248         *pobject = nv_object(chan);
 249         if (ret)
 250                 return ret;
 251
 252         nv_parent(chan)->context_attach = nve0_fifo_context_attach;
 253         nv_parent(chan)->context_detach = nve0_fifo_context_detach;
 254         chan->engine = i;
 255
 256         usermem = chan->base.chid * 0x200;
 257         ioffset = args->ioffset;
 258         ilength = order_base_2(args->ilength / 8);
 259
 260         for (i = 0; i < 0x200; i += 4)
 261                 nv_wo32(priv->user.mem, usermem + i, 0x00000000);
 262
 263         nv_wo32(base, 0x08, lower_32_bits(priv->user.mem->addr + usermem));
 264         nv_wo32(base, 0x0c, upper_32_bits(priv->user.mem->addr + usermem));
 265         nv_wo32(base, 0x10, 0x0000face);
 266         nv_wo32(base, 0x30, 0xfffff902);
 267         nv_wo32(base, 0x48, lower_32_bits(ioffset));
 268         nv_wo32(base, 0x4c, upper_32_bits(ioffset) | (ilength << 16));
 269         nv_wo32(base, 0x84, 0x20400000);
 270         nv_wo32(base, 0x94, 0x30000001);
 271         nv_wo32(base, 0x9c, 0x00000100);
 272         nv_wo32(base, 0xac, 0x0000001f);
 273         nv_wo32(base, 0xe8, chan->base.chid);
 274         nv_wo32(base, 0xb8, 0xf8000000);
 275         nv_wo32(base, 0xf8, 0x10003080); /* 0x002310 */
 276         nv_wo32(base, 0xfc, 0x10000010); /* 0x002350 */
 277         bar->flush(bar);
 278         return 0;
 279 }
 280
 281 static int
 282 nve0_fifo_chan_init(struct nouveau_object *object)
 283 {
 284         struct nouveau_gpuobj *base = nv_gpuobj(object->parent);
 285         struct nve0_fifo_priv *priv = (void *)object->engine;
 286         struct nve0_fifo_chan *chan = (void *)object;
 287         u32 chid = chan->base.chid;
 288         int ret;
 289
 290         ret = nouveau_fifo_channel_init(&chan->base);
 291         if (ret)
 292                 return ret;
 293
 294         nv_mask(priv, 0x800004 + (chid * 8), 0x000f0000, chan->engine << 16);
 295         nv_wr32(priv, 0x800000 + (chid * 8), 0x80000000 | base->addr >> 12);
 296
 297         if (chan->state == STOPPED && (chan->state = RUNNING) == RUNNING) {
 298                 nv_mask(priv, 0x800004 + (chid * 8), 0x00000400, 0x00000400);
 299                 nve0_fifo_runlist_update(priv, chan->engine);
 300                 nv_mask(priv, 0x800004 + (chid * 8), 0x00000400, 0x00000400);
 301         }
 302
 303         return 0;
 304 }
 305
 306 static int
 307 nve0_fifo_chan_fini(struct nouveau_object *object, bool suspend)
 308 {
 309         struct nve0_fifo_priv *priv = (void *)object->engine;
 310         struct nve0_fifo_chan *chan = (void *)object;
 311         u32 chid = chan->base.chid;
 312
 313         if (chan->state == RUNNING && (chan->state = STOPPED) == STOPPED) {
 314                 nv_mask(priv, 0x800004 + (chid * 8), 0x00000800, 0x00000800);
 315                 nve0_fifo_runlist_update(priv, chan->engine);
 316         }
 317
 318         nv_wr32(priv, 0x800000 + (chid * 8), 0x00000000);
 319         return nouveau_fifo_channel_fini(&chan->base, suspend);
 320 }
 321
 322 static struct nouveau_ofuncs
 323 nve0_fifo_ofuncs = {
 324         .ctor = nve0_fifo_chan_ctor,
 325         .dtor = _nouveau_fifo_channel_dtor,
 326         .init = nve0_fifo_chan_init,
 327         .fini = nve0_fifo_chan_fini,
 328         .rd32 = _nouveau_fifo_channel_rd32,
 329         .wr32 = _nouveau_fifo_channel_wr32,
 330 };
 331
 332 static struct nouveau_oclass
 333 nve0_fifo_sclass[] = {
 334         { NVE0_CHANNEL_IND_CLASS, &nve0_fifo_ofuncs },
 335         {}
 336 };
 337
 338 /*******************************************************************************
 339  * FIFO context - instmem heap and vm setup
 340  ******************************************************************************/
 341
 342 static int
 343 nve0_fifo_context_ctor(struct nouveau_object *parent,
 344                     struct nouveau_object *engine,
 345                     struct nouveau_oclass *oclass, void *data, u32 size,
 346                     struct nouveau_object **pobject)
 347 {
 348         struct nve0_fifo_base *base;
 349         int ret;
 350
 351         ret = nouveau_fifo_context_create(parent, engine, oclass, NULL, 0x1000,
 352                                           0x1000, NVOBJ_FLAG_ZERO_ALLOC, &base);
 353         *pobject = nv_object(base);
 354         if (ret)
 355                 return ret;
 356
 357         ret = nouveau_gpuobj_new(nv_object(base), NULL, 0x10000, 0x1000, 0,
 358                                 &base->pgd);
 359         if (ret)
 360                 return ret;
 361
 362         nv_wo32(base, 0x0200, lower_32_bits(base->pgd->addr));
 363         nv_wo32(base, 0x0204, upper_32_bits(base->pgd->addr));
 364         nv_wo32(base, 0x0208, 0xffffffff);
 365         nv_wo32(base, 0x020c, 0x000000ff);
 366
 367         ret = nouveau_vm_ref(nouveau_client(parent)->vm, &base->vm, base->pgd);
 368         if (ret)
 369                 return ret;
 370
 371         return 0;
 372 }
 373
 374 static void
 375 nve0_fifo_context_dtor(struct nouveau_object *object)
 376 {
 377         struct nve0_fifo_base *base = (void *)object;
 378         nouveau_vm_ref(NULL, &base->vm, base->pgd);
 379         nouveau_gpuobj_ref(NULL, &base->pgd);
 380         nouveau_fifo_context_destroy(&base->base);
 381 }
 382
 383 static struct nouveau_oclass
 384 nve0_fifo_cclass = {
 385         .handle = NV_ENGCTX(FIFO, 0xe0),
 386         .ofuncs = &(struct nouveau_ofuncs) {
 387                 .ctor = nve0_fifo_context_ctor,
 388                 .dtor = nve0_fifo_context_dtor,
 389                 .init = _nouveau_fifo_context_init,
 390                 .fini = _nouveau_fifo_context_fini,
 391                 .rd32 = _nouveau_fifo_context_rd32,
 392                 .wr32 = _nouveau_fifo_context_wr32,
 393         },
 394 };
 395
 396 /*******************************************************************************
 397  * PFIFO engine
 398  ******************************************************************************/
 399
 400 static inline int
 401 nve0_fifo_engidx(struct nve0_fifo_priv *priv, u32 engn)
 402 {
 403         switch (engn) {
 404         case NVDEV_ENGINE_GR   :
 405         case NVDEV_ENGINE_COPY2: engn = 0; break;
 406         case NVDEV_ENGINE_BSP  : engn = 1; break;
 407         case NVDEV_ENGINE_PPP  : engn = 2; break;
 408         case NVDEV_ENGINE_VP   : engn = 3; break;
 409         case NVDEV_ENGINE_COPY0: engn = 4; break;
 410         case NVDEV_ENGINE_COPY1: engn = 5; break;
 411         case NVDEV_ENGINE_VENC : engn = 6; break;
 412         default:
 413                 return -1;
 414         }
 415
 416         return engn;
 417 }
 418
 419 static inline struct nouveau_engine *
 420 nve0_fifo_engine(struct nve0_fifo_priv *priv, u32 engn)
 421 {
 422         if (engn >= ARRAY_SIZE(fifo_engine))
 423                 return NULL;
 424         return nouveau_engine(priv, fifo_engine[engn].subdev);
 425 }
 426
 427 static void
 428 nve0_fifo_recover_work(struct work_struct *work)
 429 {
 430         struct nve0_fifo_priv *priv = container_of(work, typeof(*priv), fault);
 431         struct nouveau_object *engine;
 432         unsigned long flags;
 433         u32 engn, engm = 0;
 434         u64 mask, todo;
 435
 436         spin_lock_irqsave(&priv->base.lock, flags);
 437         mask = priv->mask;
 438         priv->mask = 0ULL;
 439         spin_unlock_irqrestore(&priv->base.lock, flags);
 440
 441         for (todo = mask; engn = __ffs64(todo), todo; todo &= ~(1 << engn))
 442                 engm |= 1 << nve0_fifo_engidx(priv, engn);
 443         nv_mask(priv, 0x002630, engm, engm);
 444
 445         for (todo = mask; engn = __ffs64(todo), todo; todo &= ~(1 << engn)) {
 446                 if ((engine = (void *)nouveau_engine(priv, engn))) {
 447                         nv_ofuncs(engine)->fini(engine, false);
 448                         WARN_ON(nv_ofuncs(engine)->init(engine));
 449                 }
 450                 nve0_fifo_runlist_update(priv, nve0_fifo_engidx(priv, engn));
 451         }
 452
 453         nv_wr32(priv, 0x00262c, engm);
 454         nv_mask(priv, 0x002630, engm, 0x00000000);
 455 }
 456
 457 static void
 458 nve0_fifo_recover(struct nve0_fifo_priv *priv, struct nouveau_engine *engine,
 459                   struct nve0_fifo_chan *chan)
 460 {
 461         struct nouveau_object *engobj = nv_object(engine);
 462         u32 chid = chan->base.chid;
 463         unsigned long flags;
 464
 465         nv_error(priv, "%s engine fault on channel %d, recovering...\n",
 466                        nv_subdev(engine)->name, chid);
 467
 468         nv_mask(priv, 0x800004 + (chid * 0x08), 0x00000800, 0x00000800);
 469         chan->state = KILLED;
 470
 471         spin_lock_irqsave(&priv->base.lock, flags);
 472         priv->mask |= 1ULL << nv_engidx(engobj);
 473         spin_unlock_irqrestore(&priv->base.lock, flags);
 474         schedule_work(&priv->fault);
 475 }
 476
 477 static int
 478 nve0_fifo_swmthd(struct nve0_fifo_priv *priv, u32 chid, u32 mthd, u32 data)
 479 {
 480         struct nve0_fifo_chan *chan = NULL;
 481         struct nouveau_handle *bind;
 482         unsigned long flags;
 483         int ret = -EINVAL;
 484
 485         spin_lock_irqsave(&priv->base.lock, flags);
 486         if (likely(chid >= priv->base.min && chid <= priv->base.max))
 487                 chan = (void *)priv->base.channel[chid];
 488         if (unlikely(!chan))
 489                 goto out;
 490
 491         bind = nouveau_namedb_get_class(nv_namedb(chan), 0x906e);
 492         if (likely(bind)) {
 493                 if (!mthd || !nv_call(bind->object, mthd, data))
 494                         ret = 0;
 495                 nouveau_namedb_put(bind);
 496         }
 497
 498 out:
 499         spin_unlock_irqrestore(&priv->base.lock, flags);
 500         return ret;
 501 }
 502
 503 static const struct nouveau_enum
 504 nve0_fifo_bind_reason[] = {
 505         { 0x01, "BIND_NOT_UNBOUND" },
 506         { 0x02, "SNOOP_WITHOUT_BAR1" },
 507         { 0x03, "UNBIND_WHILE_RUNNING" },
 508         { 0x05, "INVALID_RUNLIST" },
 509         { 0x06, "INVALID_CTX_TGT" },
 510         { 0x0b, "UNBIND_WHILE_PARKED" },
 511         {}
 512 };
 513
 514 static void
 515 nve0_fifo_intr_bind(struct nve0_fifo_priv *priv)
 516 {
 517         u32 intr = nv_rd32(priv, 0x00252c);
 518         u32 code = intr & 0x000000ff;
 519         const struct nouveau_enum *en;
 520         char enunk[6] = "";
 521
 522         en = nouveau_enum_find(nve0_fifo_bind_reason, code);
 523         if (!en)
 524                 snprintf(enunk, sizeof(enunk), "UNK%02x", code);
 525
 526         nv_error(priv, "BIND_ERROR [ %s ]\n", en ? en->name : enunk);
 527 }
 528
 529 static const struct nouveau_enum
 530 nve0_fifo_sched_reason[] = {
 531         { 0x0a, "CTXSW_TIMEOUT" },
 532         {}
 533 };
 534
 535 static void
 536 nve0_fifo_intr_sched_ctxsw(struct nve0_fifo_priv *priv)
 537 {
 538         struct nouveau_engine *engine;
 539         struct nve0_fifo_chan *chan;
 540         u32 engn;
 541
 542         for (engn = 0; engn < ARRAY_SIZE(fifo_engine); engn++) {
 543                 u32 stat = nv_rd32(priv, 0x002640 + (engn * 0x04));
 544                 u32 busy = (stat & 0x80000000);
 545                 u32 next = (stat & 0x07ff0000) >> 16;
 546                 u32 chsw = (stat & 0x00008000);
 547                 u32 save = (stat & 0x00004000);
 548                 u32 load = (stat & 0x00002000);
 549                 u32 prev = (stat & 0x000007ff);
 550                 u32 chid = load ? next : prev;
 551                 (void)save;
 552
 553                 if (busy && chsw) {
 554                         if (!(chan = (void *)priv->base.channel[chid]))
 555                                 continue;
 556                         if (!(engine = nve0_fifo_engine(priv, engn)))
 557                                 continue;
 558                         nve0_fifo_recover(priv, engine, chan);
 559                 }
 560         }
 561 }
 562
 563 static void
 564 nve0_fifo_intr_sched(struct nve0_fifo_priv *priv)
 565 {
 566         u32 intr = nv_rd32(priv, 0x00254c);
 567         u32 code = intr & 0x000000ff;
 568         const struct nouveau_enum *en;
 569         char enunk[6] = "";
 570
 571         en = nouveau_enum_find(nve0_fifo_sched_reason, code);
 572         if (!en)
 573                 snprintf(enunk, sizeof(enunk), "UNK%02x", code);
 574
 575         nv_error(priv, "SCHED_ERROR [ %s ]\n", en ? en->name : enunk);
 576
 577         switch (code) {
 578         case 0x0a:
 579                 nve0_fifo_intr_sched_ctxsw(priv);
 580                 break;
 581         default:
 582                 break;
 583         }
 584 }
 585
 586 static void
 587 nve0_fifo_intr_chsw(struct nve0_fifo_priv *priv)
 588 {
 589         u32 stat = nv_rd32(priv, 0x00256c);
 590         nv_error(priv, "CHSW_ERROR 0x%08x\n", stat);
 591         nv_wr32(priv, 0x00256c, stat);
 592 }
 593
 594 static void
 595 nve0_fifo_intr_dropped_fault(struct nve0_fifo_priv *priv)
 596 {
 597         u32 stat = nv_rd32(priv, 0x00259c);
 598         nv_error(priv, "DROPPED_MMU_FAULT 0x%08x\n", stat);
 599 }
 600
 601 static const struct nouveau_enum
 602 nve0_fifo_fault_engine[] = {
 603         { 0x00, "GR", NULL, NVDEV_ENGINE_GR },
 604         { 0x03, "IFB", NULL, NVDEV_ENGINE_IFB },
 605         { 0x04, "BAR1", NULL, NVDEV_SUBDEV_BAR },
 606         { 0x05, "BAR3", NULL, NVDEV_SUBDEV_INSTMEM },
 607         { 0x07, "PBDMA0", NULL, NVDEV_ENGINE_FIFO },
 608         { 0x08, "PBDMA1", NULL, NVDEV_ENGINE_FIFO },
 609         { 0x09, "PBDMA2", NULL, NVDEV_ENGINE_FIFO },
 610         { 0x10, "MSVLD", NULL, NVDEV_ENGINE_BSP },
 611         { 0x11, "MSPPP", NULL, NVDEV_ENGINE_PPP },
 612         { 0x13, "PERF" },
 613         { 0x14, "MSPDEC", NULL, NVDEV_ENGINE_VP },
 614         { 0x15, "CE0", NULL, NVDEV_ENGINE_COPY0 },
 615         { 0x16, "CE1", NULL, NVDEV_ENGINE_COPY1 },
 616         { 0x17, "PMU" },
 617         { 0x19, "MSENC", NULL, NVDEV_ENGINE_VENC },
 618         { 0x1b, "CE2", NULL, NVDEV_ENGINE_COPY2 },
 619         {}
 620 };
 621
 622 static const struct nouveau_enum
 623 nve0_fifo_fault_reason[] = {
 624         { 0x00, "PDE" },
 625         { 0x01, "PDE_SIZE" },
 626         { 0x02, "PTE" },
 627         { 0x03, "VA_LIMIT_VIOLATION" },
 628         { 0x04, "UNBOUND_INST_BLOCK" },
 629         { 0x05, "PRIV_VIOLATION" },
 630         { 0x06, "RO_VIOLATION" },
 631         { 0x07, "WO_VIOLATION" },
 632         { 0x08, "PITCH_MASK_VIOLATION" },
 633         { 0x09, "WORK_CREATION" },
 634         { 0x0a, "UNSUPPORTED_APERTURE" },
 635         { 0x0b, "COMPRESSION_FAILURE" },
 636         { 0x0c, "UNSUPPORTED_KIND" },
 637         { 0x0d, "REGION_VIOLATION" },
 638         { 0x0e, "BOTH_PTES_VALID" },
 639         { 0x0f, "INFO_TYPE_POISONED" },
 640         {}
 641 };
 642
 643 static const struct nouveau_enum
 644 nve0_fifo_fault_hubclient[] = {
 645         { 0x00, "VIP" },
 646         { 0x01, "CE0" },
 647         { 0x02, "CE1" },
 648         { 0x03, "DNISO" },
 649         { 0x04, "FE" },
 650         { 0x05, "FECS" },
 651         { 0x06, "HOST" },
 652         { 0x07, "HOST_CPU" },
 653         { 0x08, "HOST_CPU_NB" },
 654         { 0x09, "ISO" },
 655         { 0x0a, "MMU" },
 656         { 0x0b, "MSPDEC" },
 657         { 0x0c, "MSPPP" },
 658         { 0x0d, "MSVLD" },
 659         { 0x0e, "NISO" },
 660         { 0x0f, "P2P" },
 661         { 0x10, "PD" },
 662         { 0x11, "PERF" },
 663         { 0x12, "PMU" },
 664         { 0x13, "RASTERTWOD" },
 665         { 0x14, "SCC" },
 666         { 0x15, "SCC_NB" },
 667         { 0x16, "SEC" },
 668         { 0x17, "SSYNC" },
 669         { 0x18, "GR_COPY" },
 670         { 0x19, "CE2" },
 671         { 0x1a, "XV" },
 672         { 0x1b, "MMU_NB" },
 673         { 0x1c, "MSENC" },
 674         { 0x1d, "DFALCON" },
 675         { 0x1e, "SKED" },
 676         { 0x1f, "AFALCON" },
 677         {}
 678 };
 679
 680 static const struct nouveau_enum
 681 nve0_fifo_fault_gpcclient[] = {
 682         { 0x00, "L1_0" }, { 0x01, "T1_0" }, { 0x02, "PE_0" },
 683         { 0x03, "L1_1" }, { 0x04, "T1_1" }, { 0x05, "PE_1" },
 684         { 0x06, "L1_2" }, { 0x07, "T1_2" }, { 0x08, "PE_2" },
 685         { 0x09, "L1_3" }, { 0x0a, "T1_3" }, { 0x0b, "PE_3" },
 686         { 0x0c, "RAST" },
 687         { 0x0d, "GCC" },
 688         { 0x0e, "GPCCS" },
 689         { 0x0f, "PROP_0" },
 690         { 0x10, "PROP_1" },
 691         { 0x11, "PROP_2" },
 692         { 0x12, "PROP_3" },
 693         { 0x13, "L1_4" }, { 0x14, "T1_4" }, { 0x15, "PE_4" },
 694         { 0x16, "L1_5" }, { 0x17, "T1_5" }, { 0x18, "PE_5" },
 695         { 0x19, "L1_6" }, { 0x1a, "T1_6" }, { 0x1b, "PE_6" },
 696         { 0x1c, "L1_7" }, { 0x1d, "T1_7" }, { 0x1e, "PE_7" },
 697         { 0x1f, "GPM" },
 698         { 0x20, "LTP_UTLB_0" },
 699         { 0x21, "LTP_UTLB_1" },
 700         { 0x22, "LTP_UTLB_2" },
 701         { 0x23, "LTP_UTLB_3" },
 702         { 0x24, "GPC_RGG_UTLB" },
 703         {}
 704 };
 705
 706 static void
 707 nve0_fifo_intr_fault(struct nve0_fifo_priv *priv, int unit)
 708 {
 709         u32 inst = nv_rd32(priv, 0x002800 + (unit * 0x10));
 710         u32 valo = nv_rd32(priv, 0x002804 + (unit * 0x10));
 711         u32 vahi = nv_rd32(priv, 0x002808 + (unit * 0x10));
 712         u32 stat = nv_rd32(priv, 0x00280c + (unit * 0x10));
 713         u32 gpc    = (stat & 0x1f000000) >> 24;
 714         u32 client = (stat & 0x00001f00) >> 8;
 715         u32 write  = (stat & 0x00000080);
 716         u32 hub    = (stat & 0x00000040);
 717         u32 reason = (stat & 0x0000000f);
 718         struct nouveau_object *engctx = NULL, *object;
 719         struct nouveau_engine *engine = NULL;
 720         const struct nouveau_enum *er, *eu, *ec;
 721         char erunk[6] = "";
 722         char euunk[6] = "";
 723         char ecunk[6] = "";
 724         char gpcid[3] = "";
 725
 726         er = nouveau_enum_find(nve0_fifo_fault_reason, reason);
 727         if (!er)
 728                 snprintf(erunk, sizeof(erunk), "UNK%02X", reason);
 729
 730         eu = nouveau_enum_find(nve0_fifo_fault_engine, unit);
 731         if (eu) {
 732                 switch (eu->data2) {
 733                 case NVDEV_SUBDEV_BAR:
 734                         nv_mask(priv, 0x001704, 0x00000000, 0x00000000);
 735                         break;
 736                 case NVDEV_SUBDEV_INSTMEM:
 737                         nv_mask(priv, 0x001714, 0x00000000, 0x00000000);
 738                         break;
 739                 case NVDEV_ENGINE_IFB:
 740                         nv_mask(priv, 0x001718, 0x00000000, 0x00000000);
 741                         break;
 742                 default:
 743                         engine = nouveau_engine(priv, eu->data2);
 744                         if (engine)
 745                                 engctx = nouveau_engctx_get(engine, inst);
 746                         break;
 747                 }
 748         } else {
 749                 snprintf(euunk, sizeof(euunk), "UNK%02x", unit);
 750         }
 751
 752         if (hub) {
 753                 ec = nouveau_enum_find(nve0_fifo_fault_hubclient, client);
 754         } else {
 755                 ec = nouveau_enum_find(nve0_fifo_fault_gpcclient, client);
 756                 snprintf(gpcid, sizeof(gpcid), "%d", gpc);
 757         }
 758
 759         if (!ec)
 760                 snprintf(ecunk, sizeof(ecunk), "UNK%02x", client);
 761
 762         nv_error(priv, "%s fault at 0x%010llx [%s] from %s/%s%s%s%s on "
 763                        "channel 0x%010llx [%s]\n", write ? "write" : "read",
 764                  (u64)vahi << 32 | valo, er ? er->name : erunk,
 765                  eu ? eu->name : euunk, hub ? "" : "GPC", gpcid, hub ? "" : "/",
 766                  ec ? ec->name : ecunk, (u64)inst << 12,
 767                  nouveau_client_name(engctx));
 768
 769         object = engctx;
 770         while (object) {
 771                 switch (nv_mclass(object)) {
 772                 case NVE0_CHANNEL_IND_CLASS:
 773                         nve0_fifo_recover(priv, engine, (void *)object);
 774                         break;
 775                 }
 776                 object = object->parent;
 777         }
 778
 779         nouveau_engctx_put(engctx);
 780 }
 781
 782 static const struct nouveau_bitfield nve0_fifo_pbdma_intr[] = {
 783         { 0x00000001, "MEMREQ" },
 784         { 0x00000002, "MEMACK_TIMEOUT" },
 785         { 0x00000004, "MEMACK_EXTRA" },
 786         { 0x00000008, "MEMDAT_TIMEOUT" },
 787         { 0x00000010, "MEMDAT_EXTRA" },
 788         { 0x00000020, "MEMFLUSH" },
 789         { 0x00000040, "MEMOP" },
 790         { 0x00000080, "LBCONNECT" },
 791         { 0x00000100, "LBREQ" },
 792         { 0x00000200, "LBACK_TIMEOUT" },
 793         { 0x00000400, "LBACK_EXTRA" },
 794         { 0x00000800, "LBDAT_TIMEOUT" },
 795         { 0x00001000, "LBDAT_EXTRA" },
 796         { 0x00002000, "GPFIFO" },
 797         { 0x00004000, "GPPTR" },
 798         { 0x00008000, "GPENTRY" },
 799         { 0x00010000, "GPCRC" },
 800         { 0x00020000, "PBPTR" },
 801         { 0x00040000, "PBENTRY" },
 802         { 0x00080000, "PBCRC" },
 803         { 0x00100000, "XBARCONNECT" },
 804         { 0x00200000, "METHOD" },
 805         { 0x00400000, "METHODCRC" },
 806         { 0x00800000, "DEVICE" },
 807         { 0x02000000, "SEMAPHORE" },
 808         { 0x04000000, "ACQUIRE" },
 809         { 0x08000000, "PRI" },
 810         { 0x20000000, "NO_CTXSW_SEG" },
 811         { 0x40000000, "PBSEG" },
 812         { 0x80000000, "SIGNATURE" },
 813         {}
 814 };
 815
 816 static void
 817 nve0_fifo_intr_pbdma(struct nve0_fifo_priv *priv, int unit)
 818 {
 819         u32 stat = nv_rd32(priv, 0x040108 + (unit * 0x2000));
 820         u32 addr = nv_rd32(priv, 0x0400c0 + (unit * 0x2000));
 821         u32 data = nv_rd32(priv, 0x0400c4 + (unit * 0x2000));
 822         u32 chid = nv_rd32(priv, 0x040120 + (unit * 0x2000)) & 0xfff;
 823         u32 subc = (addr & 0x00070000) >> 16;
 824         u32 mthd = (addr & 0x00003ffc);
 825         u32 show = stat;
 826
 827         if (stat & 0x00800000) {
 828                 if (!nve0_fifo_swmthd(priv, chid, mthd, data))
 829                         show &= ~0x00800000;
 830         }
 831
 832         if (show) {
 833                 nv_error(priv, "PBDMA%d:", unit);
 834                 nouveau_bitfield_print(nve0_fifo_pbdma_intr, show);
 835                 pr_cont("\n");
 836                 nv_error(priv,
 837                          "PBDMA%d: ch %d [%s] subc %d mthd 0x%04x data 0x%08x\n",
 838                          unit, chid,
 839                          nouveau_client_name_for_fifo_chid(&priv->base, chid),
 840                          subc, mthd, data);
 841         }
 842
 843         nv_wr32(priv, 0x0400c0 + (unit * 0x2000), 0x80600008);
 844         nv_wr32(priv, 0x040108 + (unit * 0x2000), stat);
 845 }
 846
 847 static void
 848 nve0_fifo_intr_runlist(struct nve0_fifo_priv *priv)
 849 {
 850         u32 mask = nv_rd32(priv, 0x002a00);
 851         while (mask) {
 852                 u32 engn = __ffs(mask);
 853                 wake_up(&priv->engine[engn].wait);
 854                 nv_wr32(priv, 0x002a00, 1 << engn);
 855                 mask &= ~(1 << engn);
 856         }
 857 }
 858
 859 static void
 860 nve0_fifo_intr_engine(struct nve0_fifo_priv *priv)
 861 {
 862         nvkm_event_send(&priv->base.uevent, 1, 0, NULL, 0);
 863 }
 864
 865 static void
 866 nve0_fifo_intr(struct nouveau_subdev *subdev)
 867 {
 868         struct nve0_fifo_priv *priv = (void *)subdev;
 869         u32 mask = nv_rd32(priv, 0x002140);
 870         u32 stat = nv_rd32(priv, 0x002100) & mask;
 871
 872         if (stat & 0x00000001) {
 873                 nve0_fifo_intr_bind(priv);
 874                 nv_wr32(priv, 0x002100, 0x00000001);
 875                 stat &= ~0x00000001;
 876         }
 877
 878         if (stat & 0x00000010) {
 879                 nv_error(priv, "PIO_ERROR\n");
 880                 nv_wr32(priv, 0x002100, 0x00000010);
 881                 stat &= ~0x00000010;
 882         }
 883
 884         if (stat & 0x00000100) {
 885                 nve0_fifo_intr_sched(priv);
 886                 nv_wr32(priv, 0x002100, 0x00000100);
 887                 stat &= ~0x00000100;
 888         }
 889
 890         if (stat & 0x00010000) {
 891                 nve0_fifo_intr_chsw(priv);
 892                 nv_wr32(priv, 0x002100, 0x00010000);
 893                 stat &= ~0x00010000;
 894         }
 895
 896         if (stat & 0x00800000) {
 897                 nv_error(priv, "FB_FLUSH_TIMEOUT\n");
 898                 nv_wr32(priv, 0x002100, 0x00800000);
 899                 stat &= ~0x00800000;
 900         }
 901
 902         if (stat & 0x01000000) {
 903                 nv_error(priv, "LB_ERROR\n");
 904                 nv_wr32(priv, 0x002100, 0x01000000);
 905                 stat &= ~0x01000000;
 906         }
 907
 908         if (stat & 0x08000000) {
 909                 nve0_fifo_intr_dropped_fault(priv);
 910                 nv_wr32(priv, 0x002100, 0x08000000);
 911                 stat &= ~0x08000000;
 912         }
 913
 914         if (stat & 0x10000000) {
 915                 u32 mask = nv_rd32(priv, 0x00259c);
 916                 while (mask) {
 917                         u32 unit = __ffs(mask);
 918                         nve0_fifo_intr_fault(priv, unit);
 919                         nv_wr32(priv, 0x00259c, (1 << unit));
 920                         mask &= ~(1 << unit);
 921                 }
 922                 stat &= ~0x10000000;
 923         }
 924
 925         if (stat & 0x20000000) {
 926                 u32 mask = nv_rd32(priv, 0x0025a0);
 927                 while (mask) {
 928                         u32 unit = __ffs(mask);
 929                         nve0_fifo_intr_pbdma(priv, unit);
 930                         nv_wr32(priv, 0x0025a0, (1 << unit));
 931                         mask &= ~(1 << unit);
 932                 }
 933                 stat &= ~0x20000000;
 934         }
 935
 936         if (stat & 0x40000000) {
 937                 nve0_fifo_intr_runlist(priv);
 938                 stat &= ~0x40000000;
 939         }
 940
 941         if (stat & 0x80000000) {
 942                 nve0_fifo_intr_engine(priv);
 943                 nv_wr32(priv, 0x002100, 0x80000000);
 944                 stat &= ~0x80000000;
 945         }
 946
 947         if (stat) {
 948                 nv_error(priv, "INTR 0x%08x\n", stat);
 949                 nv_mask(priv, 0x002140, stat, 0x00000000);
 950                 nv_wr32(priv, 0x002100, stat);
 951         }
 952 }
 953
 954 static void
 955 nve0_fifo_uevent_init(struct nvkm_event *event, int type, int index)
 956 {
 957         struct nouveau_fifo *fifo = container_of(event, typeof(*fifo), uevent);
 958         nv_mask(fifo, 0x002140, 0x80000000, 0x80000000);
 959 }
 960
 961 static void
 962 nve0_fifo_uevent_fini(struct nvkm_event *event, int type, int index)
 963 {
 964         struct nouveau_fifo *fifo = container_of(event, typeof(*fifo), uevent);
 965         nv_mask(fifo, 0x002140, 0x80000000, 0x00000000);
 966 }
 967
 968 static int
 969 nve0_fifo_uevent_ctor(void *data, u32 size, struct nvkm_notify *notify)
 970 {
 971         if (size == 0) {
 972                 notify->size  = 0;
 973                 notify->types = 1;
 974                 notify->index = 0;
 975                 return 0;
 976         }
 977         return -ENOSYS;
 978 }
 979
 980 static const struct nvkm_event_func
 981 nve0_fifo_uevent_func = {
 982         .ctor = nve0_fifo_uevent_ctor,
 983         .init = nve0_fifo_uevent_init,
 984         .fini = nve0_fifo_uevent_fini,
 985 };
 986
 987 int
 988 nve0_fifo_fini(struct nouveau_object *object, bool suspend)
 989 {
 990         struct nve0_fifo_priv *priv = (void *)object;
 991         int ret;
 992
 993         ret = nouveau_fifo_fini(&priv->base, suspend);
 994         if (ret)
 995                 return ret;
 996
 997         /* allow mmu fault interrupts, even when we're not using fifo */
 998         nv_mask(priv, 0x002140, 0x10000000, 0x10000000);
 999         return 0;
1000 }
1001
1002 int
1003 nve0_fifo_init(struct nouveau_object *object)
1004 {
1005         struct nve0_fifo_priv *priv = (void *)object;
1006         int ret, i;
1007
1008         ret = nouveau_fifo_init(&priv->base);
1009         if (ret)
1010                 return ret;
1011
1012         /* enable all available PBDMA units */
1013         nv_wr32(priv, 0x000204, 0xffffffff);
1014         priv->spoon_nr = hweight32(nv_rd32(priv, 0x000204));
1015         nv_debug(priv, "%d PBDMA unit(s)\n", priv->spoon_nr);
1016
1017         /* PBDMA[n] */
1018         for (i = 0; i < priv->spoon_nr; i++) {
1019                 nv_mask(priv, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000);
1020                 nv_wr32(priv, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */
1021                 nv_wr32(priv, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTREN */
1022         }
1023
1024         nv_wr32(priv, 0x002254, 0x10000000 | priv->user.bar.offset >> 12);
1025
1026         nv_wr32(priv, 0x002100, 0xffffffff);
1027         nv_wr32(priv, 0x002140, 0x7fffffff);
1028         return 0;
1029 }
1030
1031 void
1032 nve0_fifo_dtor(struct nouveau_object *object)
1033 {
1034         struct nve0_fifo_priv *priv = (void *)object;
1035         int i;
1036
1037         nouveau_gpuobj_unmap(&priv->user.bar);
1038         nouveau_gpuobj_ref(NULL, &priv->user.mem);
1039
1040         for (i = 0; i < FIFO_ENGINE_NR; i++) {
1041                 nouveau_gpuobj_ref(NULL, &priv->engine[i].runlist[1]);
1042                 nouveau_gpuobj_ref(NULL, &priv->engine[i].runlist[0]);
1043         }
1044
1045         nouveau_fifo_destroy(&priv->base);
1046 }
1047
1048 int
1049 nve0_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
1050                struct nouveau_oclass *oclass, void *data, u32 size,
1051                struct nouveau_object **pobject)
1052 {
1053         struct nve0_fifo_impl *impl = (void *)oclass;
1054         struct nve0_fifo_priv *priv;
1055         int ret, i;
1056
1057         ret = nouveau_fifo_create(parent, engine, oclass, 0,
1058                                   impl->channels - 1, &priv);
1059         *pobject = nv_object(priv);
1060         if (ret)
1061                 return ret;
1062
1063         INIT_WORK(&priv->fault, nve0_fifo_recover_work);
1064
1065         for (i = 0; i < FIFO_ENGINE_NR; i++) {
1066                 ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x8000, 0x1000,
1067                                          0, &priv->engine[i].runlist[0]);
1068                 if (ret)
1069                         return ret;
1070
1071                 ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x8000, 0x1000,
1072                                          0, &priv->engine[i].runlist[1]);
1073                 if (ret)
1074                         return ret;
1075
1076                 init_waitqueue_head(&priv->engine[i].wait);
1077         }
1078
1079         ret = nouveau_gpuobj_new(nv_object(priv), NULL, impl->channels * 0x200,
1080                                 0x1000, NVOBJ_FLAG_ZERO_ALLOC, &priv->user.mem);
1081         if (ret)
1082                 return ret;
1083
1084         ret = nouveau_gpuobj_map(priv->user.mem, NV_MEM_ACCESS_RW,
1085                                 &priv->user.bar);
1086         if (ret)
1087                 return ret;
1088
1089         ret = nvkm_event_init(&nve0_fifo_uevent_func, 1, 1, &priv->base.uevent);
1090         if (ret)
1091                 return ret;
1092
1093         nv_subdev(priv)->unit = 0x00000100;
1094         nv_subdev(priv)->intr = nve0_fifo_intr;
1095         nv_engine(priv)->cclass = &nve0_fifo_cclass;
1096         nv_engine(priv)->sclass = nve0_fifo_sclass;
1097         return 0;
1098 }
1099
1100 struct nouveau_oclass *
1101 nve0_fifo_oclass = &(struct nve0_fifo_impl) {
1102         .base.handle = NV_ENGINE(FIFO, 0xe0),
1103         .base.ofuncs = &(struct nouveau_ofuncs) {
1104                 .ctor = nve0_fifo_ctor,
1105                 .dtor = nve0_fifo_dtor,
1106                 .init = nve0_fifo_init,
1107                 .fini = nve0_fifo_fini,
1108         },
1109         .channels = 4096,
1110 }.base;