drivers/gpu/drm/i915/intel_ringbuffer.c

   1 /*
   2  * Copyright © 2008-2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Zou Nan hai <nanhai.zou@intel.com>
  26  *    Xiang Hai hao<haihao.xiang@intel.com>
  27  *
  28  */
  29
  30 #include "drmP.h"
  31 #include "drm.h"
  32 #include "i915_drv.h"
  33 #include "i915_drm.h"
  34 #include "i915_trace.h"
  35 #include "intel_drv.h"
  36
  37 /*
  38  * 965+ support PIPE_CONTROL commands, which provide finer grained control
  39  * over cache flushing.
  40  */
  41 struct pipe_control {
  42         struct drm_i915_gem_object *obj;
  43         volatile u32 *cpu_page;
  44         u32 gtt_offset;
  45 };
  46
  47 static inline int ring_space(struct intel_ring_buffer *ring)
  48 {
  49         int space = (ring->head & HEAD_ADDR) - (ring->tail + 8);
  50         if (space < 0)
  51                 space += ring->size;
  52         return space;
  53 }
  54
  55 static u32 i915_gem_get_seqno(struct drm_device *dev)
  56 {
  57         drm_i915_private_t *dev_priv = dev->dev_private;
  58         u32 seqno;
  59
  60         seqno = dev_priv->next_seqno;
  61
  62         /* reserve 0 for non-seqno */
  63         if (++dev_priv->next_seqno == 0)
  64                 dev_priv->next_seqno = 1;
  65
  66         return seqno;
  67 }
  68
  69 static int
  70 render_ring_flush(struct intel_ring_buffer *ring,
  71                   u32   invalidate_domains,
  72                   u32   flush_domains)
  73 {
  74         struct drm_device *dev = ring->dev;
  75         u32 cmd;
  76         int ret;
  77
  78         /*
  79          * read/write caches:
  80          *
  81          * I915_GEM_DOMAIN_RENDER is always invalidated, but is
  82          * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
  83          * also flushed at 2d versus 3d pipeline switches.
  84          *
  85          * read-only caches:
  86          *
  87          * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
  88          * MI_READ_FLUSH is set, and is always flushed on 965.
  89          *
  90          * I915_GEM_DOMAIN_COMMAND may not exist?
  91          *
  92          * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
  93          * invalidated when MI_EXE_FLUSH is set.
  94          *
  95          * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
  96          * invalidated with every MI_FLUSH.
  97          *
  98          * TLBs:
  99          *
 100          * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
 101          * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
 102          * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
 103          * are flushed at any MI_FLUSH.
 104          */
 105
 106         cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
 107         if ((invalidate_domains|flush_domains) &
 108             I915_GEM_DOMAIN_RENDER)
 109                 cmd &= ~MI_NO_WRITE_FLUSH;
 110         if (INTEL_INFO(dev)->gen < 4) {
 111                 /*
 112                  * On the 965, the sampler cache always gets flushed
 113                  * and this bit is reserved.
 114                  */
 115                 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
 116                         cmd |= MI_READ_FLUSH;
 117         }
 118         if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
 119                 cmd |= MI_EXE_FLUSH;
 120
 121         if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
 122             (IS_G4X(dev) || IS_GEN5(dev)))
 123                 cmd |= MI_INVALIDATE_ISP;
 124
 125         ret = intel_ring_begin(ring, 2);
 126         if (ret)
 127                 return ret;
 128
 129         intel_ring_emit(ring, cmd);
 130         intel_ring_emit(ring, MI_NOOP);
 131         intel_ring_advance(ring);
 132
 133         return 0;
 134 }
 135
 136 /**
 137  * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
 138  * implementing two workarounds on gen6.  From section 1.4.7.1
 139  * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
 140  *
 141  * [DevSNB-C+{W/A}] Before any depth stall flush (including those
 142  * produced by non-pipelined state commands), software needs to first
 143  * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
 144  * 0.
 145  *
 146  * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
 147  * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
 148  *
 149  * And the workaround for these two requires this workaround first:
 150  *
 151  * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
 152  * BEFORE the pipe-control with a post-sync op and no write-cache
 153  * flushes.
 154  *
 155  * And this last workaround is tricky because of the requirements on
 156  * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
 157  * volume 2 part 1:
 158  *
 159  *     "1 of the following must also be set:
 160  *      - Render Target Cache Flush Enable ([12] of DW1)
 161  *      - Depth Cache Flush Enable ([0] of DW1)
 162  *      - Stall at Pixel Scoreboard ([1] of DW1)
 163  *      - Depth Stall ([13] of DW1)
 164  *      - Post-Sync Operation ([13] of DW1)
 165  *      - Notify Enable ([8] of DW1)"
 166  *
 167  * The cache flushes require the workaround flush that triggered this
 168  * one, so we can't use it.  Depth stall would trigger the same.
 169  * Post-sync nonzero is what triggered this second workaround, so we
 170  * can't use that one either.  Notify enable is IRQs, which aren't
 171  * really our business.  That leaves only stall at scoreboard.
 172  */
 173 static int
 174 intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring)
 175 {
 176         struct pipe_control *pc = ring->private;
 177         u32 scratch_addr = pc->gtt_offset + 128;
 178         int ret;
 179
 180
 181         ret = intel_ring_begin(ring, 6);
 182         if (ret)
 183                 return ret;
 184
 185         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
 186         intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
 187                         PIPE_CONTROL_STALL_AT_SCOREBOARD);
 188         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
 189         intel_ring_emit(ring, 0); /* low dword */
 190         intel_ring_emit(ring, 0); /* high dword */
 191         intel_ring_emit(ring, MI_NOOP);
 192         intel_ring_advance(ring);
 193
 194         ret = intel_ring_begin(ring, 6);
 195         if (ret)
 196                 return ret;
 197
 198         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
 199         intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
 200         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
 201         intel_ring_emit(ring, 0);
 202         intel_ring_emit(ring, 0);
 203         intel_ring_emit(ring, MI_NOOP);
 204         intel_ring_advance(ring);
 205
 206         return 0;
 207 }
 208
 209 static int
 210 gen6_render_ring_flush(struct intel_ring_buffer *ring,
 211                          u32 invalidate_domains, u32 flush_domains)
 212 {
 213         u32 flags = 0;
 214         struct pipe_control *pc = ring->private;
 215         u32 scratch_addr = pc->gtt_offset + 128;
 216         int ret;
 217
 218         /* Force SNB workarounds for PIPE_CONTROL flushes */
 219         intel_emit_post_sync_nonzero_flush(ring);
 220
 221         /* Just flush everything.  Experiments have shown that reducing the
 222          * number of bits based on the write domains has little performance
 223          * impact.
 224          */
 225         flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
 226         flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
 227         flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
 228         flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
 229         flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
 230         flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
 231         flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
 232
 233         ret = intel_ring_begin(ring, 6);
 234         if (ret)
 235                 return ret;
 236
 237         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
 238         intel_ring_emit(ring, flags);
 239         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
 240         intel_ring_emit(ring, 0); /* lower dword */
 241         intel_ring_emit(ring, 0); /* uppwer dword */
 242         intel_ring_emit(ring, MI_NOOP);
 243         intel_ring_advance(ring);
 244
 245         return 0;
 246 }
 247
 248 static void ring_write_tail(struct intel_ring_buffer *ring,
 249                             u32 value)
 250 {
 251         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 252         I915_WRITE_TAIL(ring, value);
 253 }
 254
 255 u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
 256 {
 257         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 258         u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
 259                         RING_ACTHD(ring->mmio_base) : ACTHD;
 260
 261         return I915_READ(acthd_reg);
 262 }
 263
 264 static int init_ring_common(struct intel_ring_buffer *ring)
 265 {
 266         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 267         struct drm_i915_gem_object *obj = ring->obj;
 268         u32 head;
 269
 270         /* Stop the ring if it's running. */
 271         I915_WRITE_CTL(ring, 0);
 272         I915_WRITE_HEAD(ring, 0);
 273         ring->write_tail(ring, 0);
 274
 275         /* Initialize the ring. */
 276         I915_WRITE_START(ring, obj->gtt_offset);
 277         head = I915_READ_HEAD(ring) & HEAD_ADDR;
 278
 279         /* G45 ring initialization fails to reset head to zero */
 280         if (head != 0) {
 281                 DRM_DEBUG_KMS("%s head not reset to zero "
 282                               "ctl %08x head %08x tail %08x start %08x\n",
 283                               ring->name,
 284                               I915_READ_CTL(ring),
 285                               I915_READ_HEAD(ring),
 286                               I915_READ_TAIL(ring),
 287                               I915_READ_START(ring));
 288
 289                 I915_WRITE_HEAD(ring, 0);
 290
 291                 if (I915_READ_HEAD(ring) & HEAD_ADDR) {
 292                         DRM_ERROR("failed to set %s head to zero "
 293                                   "ctl %08x head %08x tail %08x start %08x\n",
 294                                   ring->name,
 295                                   I915_READ_CTL(ring),
 296                                   I915_READ_HEAD(ring),
 297                                   I915_READ_TAIL(ring),
 298                                   I915_READ_START(ring));
 299                 }
 300         }
 301
 302         I915_WRITE_CTL(ring,
 303                         ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
 304                         | RING_REPORT_64K | RING_VALID);
 305
 306         /* If the head is still not zero, the ring is dead */
 307         if ((I915_READ_CTL(ring) & RING_VALID) == 0 ||
 308             I915_READ_START(ring) != obj->gtt_offset ||
 309             (I915_READ_HEAD(ring) & HEAD_ADDR) != 0) {
 310                 DRM_ERROR("%s initialization failed "
 311                                 "ctl %08x head %08x tail %08x start %08x\n",
 312                                 ring->name,
 313                                 I915_READ_CTL(ring),
 314                                 I915_READ_HEAD(ring),
 315                                 I915_READ_TAIL(ring),
 316                                 I915_READ_START(ring));
 317                 return -EIO;
 318         }
 319
 320         if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
 321                 i915_kernel_lost_context(ring->dev);
 322         else {
 323                 ring->head = I915_READ_HEAD(ring);
 324                 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
 325                 ring->space = ring_space(ring);
 326         }
 327
 328         return 0;
 329 }
 330
 331 static int
 332 init_pipe_control(struct intel_ring_buffer *ring)
 333 {
 334         struct pipe_control *pc;
 335         struct drm_i915_gem_object *obj;
 336         int ret;
 337
 338         if (ring->private)
 339                 return 0;
 340
 341         pc = kmalloc(sizeof(*pc), GFP_KERNEL);
 342         if (!pc)
 343                 return -ENOMEM;
 344
 345         obj = i915_gem_alloc_object(ring->dev, 4096);
 346         if (obj == NULL) {
 347                 DRM_ERROR("Failed to allocate seqno page\n");
 348                 ret = -ENOMEM;
 349                 goto err;
 350         }
 351
 352         i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
 353
 354         ret = i915_gem_object_pin(obj, 4096, true);
 355         if (ret)
 356                 goto err_unref;
 357
 358         pc->gtt_offset = obj->gtt_offset;
 359         pc->cpu_page =  kmap(obj->pages[0]);
 360         if (pc->cpu_page == NULL)
 361                 goto err_unpin;
 362
 363         pc->obj = obj;
 364         ring->private = pc;
 365         return 0;
 366
 367 err_unpin:
 368         i915_gem_object_unpin(obj);
 369 err_unref:
 370         drm_gem_object_unreference(&obj->base);
 371 err:
 372         kfree(pc);
 373         return ret;
 374 }
 375
 376 static void
 377 cleanup_pipe_control(struct intel_ring_buffer *ring)
 378 {
 379         struct pipe_control *pc = ring->private;
 380         struct drm_i915_gem_object *obj;
 381
 382         if (!ring->private)
 383                 return;
 384
 385         obj = pc->obj;
 386         kunmap(obj->pages[0]);
 387         i915_gem_object_unpin(obj);
 388         drm_gem_object_unreference(&obj->base);
 389
 390         kfree(pc);
 391         ring->private = NULL;
 392 }
 393
 394 static int init_render_ring(struct intel_ring_buffer *ring)
 395 {
 396         struct drm_device *dev = ring->dev;
 397         struct drm_i915_private *dev_priv = dev->dev_private;
 398         int ret = init_ring_common(ring);
 399
 400         if (INTEL_INFO(dev)->gen > 3) {
 401                 int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
 402                 if (IS_GEN6(dev) || IS_GEN7(dev))
 403                         mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;
 404                 I915_WRITE(MI_MODE, mode);
 405                 if (IS_GEN7(dev))
 406                         I915_WRITE(GFX_MODE_GEN7,
 407                                    GFX_MODE_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) |
 408                                    GFX_MODE_ENABLE(GFX_REPLAY_MODE));
 409         }
 410
 411         if (INTEL_INFO(dev)->gen >= 5) {
 412                 ret = init_pipe_control(ring);
 413                 if (ret)
 414                         return ret;
 415         }
 416
 417         if (INTEL_INFO(dev)->gen >= 6) {
 418                 I915_WRITE(INSTPM,
 419                            INSTPM_FORCE_ORDERING << 16 | INSTPM_FORCE_ORDERING);
 420         }
 421
 422         return ret;
 423 }
 424
 425 static void render_ring_cleanup(struct intel_ring_buffer *ring)
 426 {
 427         if (!ring->private)
 428                 return;
 429
 430         cleanup_pipe_control(ring);
 431 }
 432
 433 static void
 434 update_mboxes(struct intel_ring_buffer *ring,
 435             u32 seqno,
 436             u32 mmio_offset)
 437 {
 438         intel_ring_emit(ring, MI_SEMAPHORE_MBOX |
 439                               MI_SEMAPHORE_GLOBAL_GTT |
 440                               MI_SEMAPHORE_REGISTER |
 441                               MI_SEMAPHORE_UPDATE);
 442         intel_ring_emit(ring, seqno);
 443         intel_ring_emit(ring, mmio_offset);
 444 }
 445
 446 /**
 447  * gen6_add_request - Update the semaphore mailbox registers
 448  *
 449  * @ring - ring that is adding a request
 450  * @seqno - return seqno stuck into the ring
 451  *
 452  * Update the mailbox registers in the *other* rings with the current seqno.
 453  * This acts like a signal in the canonical semaphore.
 454  */
 455 static int
 456 gen6_add_request(struct intel_ring_buffer *ring,
 457                  u32 *seqno)
 458 {
 459         u32 mbox1_reg;
 460         u32 mbox2_reg;
 461         int ret;
 462
 463         ret = intel_ring_begin(ring, 10);
 464         if (ret)
 465                 return ret;
 466
 467         mbox1_reg = ring->signal_mbox[0];
 468         mbox2_reg = ring->signal_mbox[1];
 469
 470         *seqno = i915_gem_get_seqno(ring->dev);
 471
 472         update_mboxes(ring, *seqno, mbox1_reg);
 473         update_mboxes(ring, *seqno, mbox2_reg);
 474         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
 475         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 476         intel_ring_emit(ring, *seqno);
 477         intel_ring_emit(ring, MI_USER_INTERRUPT);
 478         intel_ring_advance(ring);
 479
 480         return 0;
 481 }
 482
 483 /**
 484  * intel_ring_sync - sync the waiter to the signaller on seqno
 485  *
 486  * @waiter - ring that is waiting
 487  * @signaller - ring which has, or will signal
 488  * @seqno - seqno which the waiter will block on
 489  */
 490 static int
 491 intel_ring_sync(struct intel_ring_buffer *waiter,
 492                 struct intel_ring_buffer *signaller,
 493                 int ring,
 494                 u32 seqno)
 495 {
 496         int ret;
 497         u32 dw1 = MI_SEMAPHORE_MBOX |
 498                   MI_SEMAPHORE_COMPARE |
 499                   MI_SEMAPHORE_REGISTER;
 500
 501         ret = intel_ring_begin(waiter, 4);
 502         if (ret)
 503                 return ret;
 504
 505         intel_ring_emit(waiter, dw1 | signaller->semaphore_register[ring]);
 506         intel_ring_emit(waiter, seqno);
 507         intel_ring_emit(waiter, 0);
 508         intel_ring_emit(waiter, MI_NOOP);
 509         intel_ring_advance(waiter);
 510
 511         return 0;
 512 }
 513
 514 /* VCS->RCS (RVSYNC) or BCS->RCS (RBSYNC) */
 515 int
 516 render_ring_sync_to(struct intel_ring_buffer *waiter,
 517                     struct intel_ring_buffer *signaller,
 518                     u32 seqno)
 519 {
 520         WARN_ON(signaller->semaphore_register[RCS] == MI_SEMAPHORE_SYNC_INVALID);
 521         return intel_ring_sync(waiter,
 522                                signaller,
 523                                RCS,
 524                                seqno);
 525 }
 526
 527 /* RCS->VCS (VRSYNC) or BCS->VCS (VBSYNC) */
 528 int
 529 gen6_bsd_ring_sync_to(struct intel_ring_buffer *waiter,
 530                       struct intel_ring_buffer *signaller,
 531                       u32 seqno)
 532 {
 533         WARN_ON(signaller->semaphore_register[VCS] == MI_SEMAPHORE_SYNC_INVALID);
 534         return intel_ring_sync(waiter,
 535                                signaller,
 536                                VCS,
 537                                seqno);
 538 }
 539
 540 /* RCS->BCS (BRSYNC) or VCS->BCS (BVSYNC) */
 541 int
 542 gen6_blt_ring_sync_to(struct intel_ring_buffer *waiter,
 543                       struct intel_ring_buffer *signaller,
 544                       u32 seqno)
 545 {
 546         WARN_ON(signaller->semaphore_register[BCS] == MI_SEMAPHORE_SYNC_INVALID);
 547         return intel_ring_sync(waiter,
 548                                signaller,
 549                                BCS,
 550                                seqno);
 551 }
 552
 553
 554
 555 #define PIPE_CONTROL_FLUSH(ring__, addr__)                                      \
 556 do {                                                                    \
 557         intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |                \
 558                  PIPE_CONTROL_DEPTH_STALL);                             \
 559         intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
 560         intel_ring_emit(ring__, 0);                                                     \
 561         intel_ring_emit(ring__, 0);                                                     \
 562 } while (0)
 563
 564 static int
 565 pc_render_add_request(struct intel_ring_buffer *ring,
 566                       u32 *result)
 567 {
 568         struct drm_device *dev = ring->dev;
 569         u32 seqno = i915_gem_get_seqno(dev);
 570         struct pipe_control *pc = ring->private;
 571         u32 scratch_addr = pc->gtt_offset + 128;
 572         int ret;
 573
 574         /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
 575          * incoherent with writes to memory, i.e. completely fubar,
 576          * so we need to use PIPE_NOTIFY instead.
 577          *
 578          * However, we also need to workaround the qword write
 579          * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
 580          * memory before requesting an interrupt.
 581          */
 582         ret = intel_ring_begin(ring, 32);
 583         if (ret)
 584                 return ret;
 585
 586         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
 587                         PIPE_CONTROL_WRITE_FLUSH |
 588                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
 589         intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
 590         intel_ring_emit(ring, seqno);
 591         intel_ring_emit(ring, 0);
 592         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 593         scratch_addr += 128; /* write to separate cachelines */
 594         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 595         scratch_addr += 128;
 596         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 597         scratch_addr += 128;
 598         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 599         scratch_addr += 128;
 600         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 601         scratch_addr += 128;
 602         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 603         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
 604                         PIPE_CONTROL_WRITE_FLUSH |
 605                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
 606                         PIPE_CONTROL_NOTIFY);
 607         intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
 608         intel_ring_emit(ring, seqno);
 609         intel_ring_emit(ring, 0);
 610         intel_ring_advance(ring);
 611
 612         *result = seqno;
 613         return 0;
 614 }
 615
 616 static int
 617 render_ring_add_request(struct intel_ring_buffer *ring,
 618                         u32 *result)
 619 {
 620         struct drm_device *dev = ring->dev;
 621         u32 seqno = i915_gem_get_seqno(dev);
 622         int ret;
 623
 624         ret = intel_ring_begin(ring, 4);
 625         if (ret)
 626                 return ret;
 627
 628         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
 629         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 630         intel_ring_emit(ring, seqno);
 631         intel_ring_emit(ring, MI_USER_INTERRUPT);
 632         intel_ring_advance(ring);
 633
 634         *result = seqno;
 635         return 0;
 636 }
 637
 638 static u32
 639 gen6_ring_get_seqno(struct intel_ring_buffer *ring)
 640 {
 641         struct drm_device *dev = ring->dev;
 642
 643         /* Workaround to force correct ordering between irq and seqno writes on
 644          * ivb (and maybe also on snb) by reading from a CS register (like
 645          * ACTHD) before reading the status page. */
 646         if (IS_GEN7(dev))
 647                 intel_ring_get_active_head(ring);
 648         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 649 }
 650
 651 static u32
 652 ring_get_seqno(struct intel_ring_buffer *ring)
 653 {
 654         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 655 }
 656
 657 static u32
 658 pc_render_get_seqno(struct intel_ring_buffer *ring)
 659 {
 660         struct pipe_control *pc = ring->private;
 661         return pc->cpu_page[0];
 662 }
 663
 664 static void
 665 ironlake_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
 666 {
 667         dev_priv->gt_irq_mask &= ~mask;
 668         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 669         POSTING_READ(GTIMR);
 670 }
 671
 672 static void
 673 ironlake_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
 674 {
 675         dev_priv->gt_irq_mask |= mask;
 676         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 677         POSTING_READ(GTIMR);
 678 }
 679
 680 static void
 681 i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
 682 {
 683         dev_priv->irq_mask &= ~mask;
 684         I915_WRITE(IMR, dev_priv->irq_mask);
 685         POSTING_READ(IMR);
 686 }
 687
 688 static void
 689 i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
 690 {
 691         dev_priv->irq_mask |= mask;
 692         I915_WRITE(IMR, dev_priv->irq_mask);
 693         POSTING_READ(IMR);
 694 }
 695
 696 static bool
 697 render_ring_get_irq(struct intel_ring_buffer *ring)
 698 {
 699         struct drm_device *dev = ring->dev;
 700         drm_i915_private_t *dev_priv = dev->dev_private;
 701
 702         if (!dev->irq_enabled)
 703                 return false;
 704
 705         spin_lock(&ring->irq_lock);
 706         if (ring->irq_refcount++ == 0) {
 707                 if (HAS_PCH_SPLIT(dev))
 708                         ironlake_enable_irq(dev_priv,
 709                                             GT_PIPE_NOTIFY | GT_USER_INTERRUPT);
 710                 else
 711                         i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
 712         }
 713         spin_unlock(&ring->irq_lock);
 714
 715         return true;
 716 }
 717
 718 static void
 719 render_ring_put_irq(struct intel_ring_buffer *ring)
 720 {
 721         struct drm_device *dev = ring->dev;
 722         drm_i915_private_t *dev_priv = dev->dev_private;
 723
 724         spin_lock(&ring->irq_lock);
 725         if (--ring->irq_refcount == 0) {
 726                 if (HAS_PCH_SPLIT(dev))
 727                         ironlake_disable_irq(dev_priv,
 728                                              GT_USER_INTERRUPT |
 729                                              GT_PIPE_NOTIFY);
 730                 else
 731                         i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
 732         }
 733         spin_unlock(&ring->irq_lock);
 734 }
 735
 736 void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
 737 {
 738         struct drm_device *dev = ring->dev;
 739         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 740         u32 mmio = 0;
 741
 742         /* The ring status page addresses are no longer next to the rest of
 743          * the ring registers as of gen7.
 744          */
 745         if (IS_GEN7(dev)) {
 746                 switch (ring->id) {
 747                 case RING_RENDER:
 748                         mmio = RENDER_HWS_PGA_GEN7;
 749                         break;
 750                 case RING_BLT:
 751                         mmio = BLT_HWS_PGA_GEN7;
 752                         break;
 753                 case RING_BSD:
 754                         mmio = BSD_HWS_PGA_GEN7;
 755                         break;
 756                 }
 757         } else if (IS_GEN6(ring->dev)) {
 758                 mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
 759         } else {
 760                 mmio = RING_HWS_PGA(ring->mmio_base);
 761         }
 762
 763         I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
 764         POSTING_READ(mmio);
 765 }
 766
 767 static int
 768 bsd_ring_flush(struct intel_ring_buffer *ring,
 769                u32     invalidate_domains,
 770                u32     flush_domains)
 771 {
 772         int ret;
 773
 774         ret = intel_ring_begin(ring, 2);
 775         if (ret)
 776                 return ret;
 777
 778         intel_ring_emit(ring, MI_FLUSH);
 779         intel_ring_emit(ring, MI_NOOP);
 780         intel_ring_advance(ring);
 781         return 0;
 782 }
 783
 784 static int
 785 ring_add_request(struct intel_ring_buffer *ring,
 786                  u32 *result)
 787 {
 788         u32 seqno;
 789         int ret;
 790
 791         ret = intel_ring_begin(ring, 4);
 792         if (ret)
 793                 return ret;
 794
 795         seqno = i915_gem_get_seqno(ring->dev);
 796
 797         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
 798         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 799         intel_ring_emit(ring, seqno);
 800         intel_ring_emit(ring, MI_USER_INTERRUPT);
 801         intel_ring_advance(ring);
 802
 803         *result = seqno;
 804         return 0;
 805 }
 806
 807 static bool
 808 gen6_ring_get_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
 809 {
 810         struct drm_device *dev = ring->dev;
 811         drm_i915_private_t *dev_priv = dev->dev_private;
 812
 813         if (!dev->irq_enabled)
 814                return false;
 815
 816         /* It looks like we need to prevent the gt from suspending while waiting
 817          * for an notifiy irq, otherwise irqs seem to get lost on at least the
 818          * blt/bsd rings on ivb. */
 819         if (IS_GEN7(dev))
 820                 gen6_gt_force_wake_get(dev_priv);
 821
 822         spin_lock(&ring->irq_lock);
 823         if (ring->irq_refcount++ == 0) {
 824                 ring->irq_mask &= ~rflag;
 825                 I915_WRITE_IMR(ring, ring->irq_mask);
 826                 ironlake_enable_irq(dev_priv, gflag);
 827         }
 828         spin_unlock(&ring->irq_lock);
 829
 830         return true;
 831 }
 832
 833 static void
 834 gen6_ring_put_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
 835 {
 836         struct drm_device *dev = ring->dev;
 837         drm_i915_private_t *dev_priv = dev->dev_private;
 838
 839         spin_lock(&ring->irq_lock);
 840         if (--ring->irq_refcount == 0) {
 841                 ring->irq_mask |= rflag;
 842                 I915_WRITE_IMR(ring, ring->irq_mask);
 843                 ironlake_disable_irq(dev_priv, gflag);
 844         }
 845         spin_unlock(&ring->irq_lock);
 846
 847         if (IS_GEN7(dev))
 848                 gen6_gt_force_wake_put(dev_priv);
 849 }
 850
 851 static bool
 852 bsd_ring_get_irq(struct intel_ring_buffer *ring)
 853 {
 854         struct drm_device *dev = ring->dev;
 855         drm_i915_private_t *dev_priv = dev->dev_private;
 856
 857         if (!dev->irq_enabled)
 858                 return false;
 859
 860         spin_lock(&ring->irq_lock);
 861         if (ring->irq_refcount++ == 0) {
 862                 if (IS_G4X(dev))
 863                         i915_enable_irq(dev_priv, I915_BSD_USER_INTERRUPT);
 864                 else
 865                         ironlake_enable_irq(dev_priv, GT_BSD_USER_INTERRUPT);
 866         }
 867         spin_unlock(&ring->irq_lock);
 868
 869         return true;
 870 }
 871 static void
 872 bsd_ring_put_irq(struct intel_ring_buffer *ring)
 873 {
 874         struct drm_device *dev = ring->dev;
 875         drm_i915_private_t *dev_priv = dev->dev_private;
 876
 877         spin_lock(&ring->irq_lock);
 878         if (--ring->irq_refcount == 0) {
 879                 if (IS_G4X(dev))
 880                         i915_disable_irq(dev_priv, I915_BSD_USER_INTERRUPT);
 881                 else
 882                         ironlake_disable_irq(dev_priv, GT_BSD_USER_INTERRUPT);
 883         }
 884         spin_unlock(&ring->irq_lock);
 885 }
 886
 887 static int
 888 ring_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length)
 889 {
 890         int ret;
 891
 892         ret = intel_ring_begin(ring, 2);
 893         if (ret)
 894                 return ret;
 895
 896         intel_ring_emit(ring,
 897                         MI_BATCH_BUFFER_START | (2 << 6) |
 898                         MI_BATCH_NON_SECURE_I965);
 899         intel_ring_emit(ring, offset);
 900         intel_ring_advance(ring);
 901
 902         return 0;
 903 }
 904
 905 static int
 906 render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
 907                                 u32 offset, u32 len)
 908 {
 909         struct drm_device *dev = ring->dev;
 910         int ret;
 911
 912         if (IS_I830(dev) || IS_845G(dev)) {
 913                 ret = intel_ring_begin(ring, 4);
 914                 if (ret)
 915                         return ret;
 916
 917                 intel_ring_emit(ring, MI_BATCH_BUFFER);
 918                 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
 919                 intel_ring_emit(ring, offset + len - 8);
 920                 intel_ring_emit(ring, 0);
 921         } else {
 922                 ret = intel_ring_begin(ring, 2);
 923                 if (ret)
 924                         return ret;
 925
 926                 if (INTEL_INFO(dev)->gen >= 4) {
 927                         intel_ring_emit(ring,
 928                                         MI_BATCH_BUFFER_START | (2 << 6) |
 929                                         MI_BATCH_NON_SECURE_I965);
 930                         intel_ring_emit(ring, offset);
 931                 } else {
 932                         intel_ring_emit(ring,
 933                                         MI_BATCH_BUFFER_START | (2 << 6));
 934                         intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
 935                 }
 936         }
 937         intel_ring_advance(ring);
 938
 939         return 0;
 940 }
 941
 942 static void cleanup_status_page(struct intel_ring_buffer *ring)
 943 {
 944         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 945         struct drm_i915_gem_object *obj;
 946
 947         obj = ring->status_page.obj;
 948         if (obj == NULL)
 949                 return;
 950
 951         kunmap(obj->pages[0]);
 952         i915_gem_object_unpin(obj);
 953         drm_gem_object_unreference(&obj->base);
 954         ring->status_page.obj = NULL;
 955
 956         memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 957 }
 958
 959 static int init_status_page(struct intel_ring_buffer *ring)
 960 {
 961         struct drm_device *dev = ring->dev;
 962         drm_i915_private_t *dev_priv = dev->dev_private;
 963         struct drm_i915_gem_object *obj;
 964         int ret;
 965
 966         obj = i915_gem_alloc_object(dev, 4096);
 967         if (obj == NULL) {
 968                 DRM_ERROR("Failed to allocate status page\n");
 969                 ret = -ENOMEM;
 970                 goto err;
 971         }
 972
 973         i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
 974
 975         ret = i915_gem_object_pin(obj, 4096, true);
 976         if (ret != 0) {
 977                 goto err_unref;
 978         }
 979
 980         ring->status_page.gfx_addr = obj->gtt_offset;
 981         ring->status_page.page_addr = kmap(obj->pages[0]);
 982         if (ring->status_page.page_addr == NULL) {
 983                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 984                 goto err_unpin;
 985         }
 986         ring->status_page.obj = obj;
 987         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
 988
 989         intel_ring_setup_status_page(ring);
 990         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
 991                         ring->name, ring->status_page.gfx_addr);
 992
 993         return 0;
 994
 995 err_unpin:
 996         i915_gem_object_unpin(obj);
 997 err_unref:
 998         drm_gem_object_unreference(&obj->base);
 999 err:
1000         return ret;
1001 }
1002
1003 int intel_init_ring_buffer(struct drm_device *dev,
1004                            struct intel_ring_buffer *ring)
1005 {
1006         struct drm_i915_gem_object *obj;
1007         int ret;
1008
1009         ring->dev = dev;
1010         INIT_LIST_HEAD(&ring->active_list);
1011         INIT_LIST_HEAD(&ring->request_list);
1012         INIT_LIST_HEAD(&ring->gpu_write_list);
1013
1014         init_waitqueue_head(&ring->irq_queue);
1015         spin_lock_init(&ring->irq_lock);
1016         ring->irq_mask = ~0;
1017
1018         if (I915_NEED_GFX_HWS(dev)) {
1019                 ret = init_status_page(ring);
1020                 if (ret)
1021                         return ret;
1022         }
1023
1024         obj = i915_gem_alloc_object(dev, ring->size);
1025         if (obj == NULL) {
1026                 DRM_ERROR("Failed to allocate ringbuffer\n");
1027                 ret = -ENOMEM;
1028                 goto err_hws;
1029         }
1030
1031         ring->obj = obj;
1032
1033         ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
1034         if (ret)
1035                 goto err_unref;
1036
1037         ring->map.size = ring->size;
1038         ring->map.offset = dev->agp->base + obj->gtt_offset;
1039         ring->map.type = 0;
1040         ring->map.flags = 0;
1041         ring->map.mtrr = 0;
1042
1043         drm_core_ioremap_wc(&ring->map, dev);
1044         if (ring->map.handle == NULL) {
1045                 DRM_ERROR("Failed to map ringbuffer.\n");
1046                 ret = -EINVAL;
1047                 goto err_unpin;
1048         }
1049
1050         ring->virtual_start = ring->map.handle;
1051         ret = ring->init(ring);
1052         if (ret)
1053                 goto err_unmap;
1054
1055         /* Workaround an erratum on the i830 which causes a hang if
1056          * the TAIL pointer points to within the last 2 cachelines
1057          * of the buffer.
1058          */
1059         ring->effective_size = ring->size;
1060         if (IS_I830(ring->dev))
1061                 ring->effective_size -= 128;
1062
1063         return 0;
1064
1065 err_unmap:
1066         drm_core_ioremapfree(&ring->map, dev);
1067 err_unpin:
1068         i915_gem_object_unpin(obj);
1069 err_unref:
1070         drm_gem_object_unreference(&obj->base);
1071         ring->obj = NULL;
1072 err_hws:
1073         cleanup_status_page(ring);
1074         return ret;
1075 }
1076
1077 void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
1078 {
1079         struct drm_i915_private *dev_priv;
1080         int ret;
1081
1082         if (ring->obj == NULL)
1083                 return;
1084
1085         /* Disable the ring buffer. The ring must be idle at this point */
1086         dev_priv = ring->dev->dev_private;
1087         ret = intel_wait_ring_idle(ring);
1088         if (ret)
1089                 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
1090                           ring->name, ret);
1091
1092         I915_WRITE_CTL(ring, 0);
1093
1094         drm_core_ioremapfree(&ring->map, ring->dev);
1095
1096         i915_gem_object_unpin(ring->obj);
1097         drm_gem_object_unreference(&ring->obj->base);
1098         ring->obj = NULL;
1099
1100         if (ring->cleanup)
1101                 ring->cleanup(ring);
1102
1103         cleanup_status_page(ring);
1104 }
1105
1106 static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
1107 {
1108         unsigned int *virt;
1109         int rem = ring->size - ring->tail;
1110
1111         if (ring->space < rem) {
1112                 int ret = intel_wait_ring_buffer(ring, rem);
1113                 if (ret)
1114                         return ret;
1115         }
1116
1117         virt = (unsigned int *)(ring->virtual_start + ring->tail);
1118         rem /= 8;
1119         while (rem--) {
1120                 *virt++ = MI_NOOP;
1121                 *virt++ = MI_NOOP;
1122         }
1123
1124         ring->tail = 0;
1125         ring->space = ring_space(ring);
1126
1127         return 0;
1128 }
1129
1130 int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
1131 {
1132         struct drm_device *dev = ring->dev;
1133         struct drm_i915_private *dev_priv = dev->dev_private;
1134         unsigned long end;
1135         u32 head;
1136
1137         /* If the reported head position has wrapped or hasn't advanced,
1138          * fallback to the slow and accurate path.
1139          */
1140         head = intel_read_status_page(ring, 4);
1141         if (head > ring->head) {
1142                 ring->head = head;
1143                 ring->space = ring_space(ring);
1144                 if (ring->space >= n)
1145                         return 0;
1146         }
1147
1148         trace_i915_ring_wait_begin(ring);
1149         if (drm_core_check_feature(dev, DRIVER_GEM))
1150                 /* With GEM the hangcheck timer should kick us out of the loop,
1151                  * leaving it early runs the risk of corrupting GEM state (due
1152                  * to running on almost untested codepaths). But on resume
1153                  * timers don't work yet, so prevent a complete hang in that
1154                  * case by choosing an insanely large timeout. */
1155                 end = jiffies + 60 * HZ;
1156         else
1157                 end = jiffies + 3 * HZ;
1158
1159         do {
1160                 ring->head = I915_READ_HEAD(ring);
1161                 ring->space = ring_space(ring);
1162                 if (ring->space >= n) {
1163                         trace_i915_ring_wait_end(ring);
1164                         return 0;
1165                 }
1166
1167                 if (dev->primary->master) {
1168                         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
1169                         if (master_priv->sarea_priv)
1170                                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
1171                 }
1172
1173                 msleep(1);
1174                 if (atomic_read(&dev_priv->mm.wedged))
1175                         return -EAGAIN;
1176         } while (!time_after(jiffies, end));
1177         trace_i915_ring_wait_end(ring);
1178         return -EBUSY;
1179 }
1180
1181 int intel_ring_begin(struct intel_ring_buffer *ring,
1182                      int num_dwords)
1183 {
1184         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1185         int n = 4*num_dwords;
1186         int ret;
1187
1188         if (unlikely(atomic_read(&dev_priv->mm.wedged)))
1189                 return -EIO;
1190
1191         if (unlikely(ring->tail + n > ring->effective_size)) {
1192                 ret = intel_wrap_ring_buffer(ring);
1193                 if (unlikely(ret))
1194                         return ret;
1195         }
1196
1197         if (unlikely(ring->space < n)) {
1198                 ret = intel_wait_ring_buffer(ring, n);
1199                 if (unlikely(ret))
1200                         return ret;
1201         }
1202
1203         ring->space -= n;
1204         return 0;
1205 }
1206
1207 void intel_ring_advance(struct intel_ring_buffer *ring)
1208 {
1209         ring->tail &= ring->size - 1;
1210         ring->write_tail(ring, ring->tail);
1211 }
1212
1213 static const struct intel_ring_buffer render_ring = {
1214         .name                   = "render ring",
1215         .id                     = RING_RENDER,
1216         .mmio_base              = RENDER_RING_BASE,
1217         .size                   = 32 * PAGE_SIZE,
1218         .init                   = init_render_ring,
1219         .write_tail             = ring_write_tail,
1220         .flush                  = render_ring_flush,
1221         .add_request            = render_ring_add_request,
1222         .get_seqno              = ring_get_seqno,
1223         .irq_get                = render_ring_get_irq,
1224         .irq_put                = render_ring_put_irq,
1225         .dispatch_execbuffer    = render_ring_dispatch_execbuffer,
1226         .cleanup                = render_ring_cleanup,
1227         .sync_to                = render_ring_sync_to,
1228         .semaphore_register     = {MI_SEMAPHORE_SYNC_INVALID,
1229                                    MI_SEMAPHORE_SYNC_RV,
1230                                    MI_SEMAPHORE_SYNC_RB},
1231         .signal_mbox            = {GEN6_VRSYNC, GEN6_BRSYNC},
1232 };
1233
1234 /* ring buffer for bit-stream decoder */
1235
1236 static const struct intel_ring_buffer bsd_ring = {
1237         .name                   = "bsd ring",
1238         .id                     = RING_BSD,
1239         .mmio_base              = BSD_RING_BASE,
1240         .size                   = 32 * PAGE_SIZE,
1241         .init                   = init_ring_common,
1242         .write_tail             = ring_write_tail,
1243         .flush                  = bsd_ring_flush,
1244         .add_request            = ring_add_request,
1245         .get_seqno              = ring_get_seqno,
1246         .irq_get                = bsd_ring_get_irq,
1247         .irq_put                = bsd_ring_put_irq,
1248         .dispatch_execbuffer    = ring_dispatch_execbuffer,
1249 };
1250
1251
1252 static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
1253                                      u32 value)
1254 {
1255         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1256
1257        /* Every tail move must follow the sequence below */
1258         I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1259                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
1260                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
1261         I915_WRITE(GEN6_BSD_RNCID, 0x0);
1262
1263         if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
1264                 GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
1265                 50))
1266         DRM_ERROR("timed out waiting for IDLE Indicator\n");
1267
1268         I915_WRITE_TAIL(ring, value);
1269         I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1270                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
1271                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
1272 }
1273
1274 static int gen6_ring_flush(struct intel_ring_buffer *ring,
1275                            u32 invalidate, u32 flush)
1276 {
1277         uint32_t cmd;
1278         int ret;
1279
1280         ret = intel_ring_begin(ring, 4);
1281         if (ret)
1282                 return ret;
1283
1284         cmd = MI_FLUSH_DW;
1285         if (invalidate & I915_GEM_GPU_DOMAINS)
1286                 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD;
1287         intel_ring_emit(ring, cmd);
1288         intel_ring_emit(ring, 0);
1289         intel_ring_emit(ring, 0);
1290         intel_ring_emit(ring, MI_NOOP);
1291         intel_ring_advance(ring);
1292         return 0;
1293 }
1294
1295 static int
1296 gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
1297                               u32 offset, u32 len)
1298 {
1299         int ret;
1300
1301         ret = intel_ring_begin(ring, 2);
1302         if (ret)
1303                 return ret;
1304
1305         intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
1306         /* bit0-7 is the length on GEN6+ */
1307         intel_ring_emit(ring, offset);
1308         intel_ring_advance(ring);
1309
1310         return 0;
1311 }
1312
1313 static bool
1314 gen6_render_ring_get_irq(struct intel_ring_buffer *ring)
1315 {
1316         return gen6_ring_get_irq(ring,
1317                                  GT_USER_INTERRUPT,
1318                                  GEN6_RENDER_USER_INTERRUPT);
1319 }
1320
1321 static void
1322 gen6_render_ring_put_irq(struct intel_ring_buffer *ring)
1323 {
1324         return gen6_ring_put_irq(ring,
1325                                  GT_USER_INTERRUPT,
1326                                  GEN6_RENDER_USER_INTERRUPT);
1327 }
1328
1329 static bool
1330 gen6_bsd_ring_get_irq(struct intel_ring_buffer *ring)
1331 {
1332         return gen6_ring_get_irq(ring,
1333                                  GT_GEN6_BSD_USER_INTERRUPT,
1334                                  GEN6_BSD_USER_INTERRUPT);
1335 }
1336
1337 static void
1338 gen6_bsd_ring_put_irq(struct intel_ring_buffer *ring)
1339 {
1340         return gen6_ring_put_irq(ring,
1341                                  GT_GEN6_BSD_USER_INTERRUPT,
1342                                  GEN6_BSD_USER_INTERRUPT);
1343 }
1344
1345 /* ring buffer for Video Codec for Gen6+ */
1346 static const struct intel_ring_buffer gen6_bsd_ring = {
1347         .name                   = "gen6 bsd ring",
1348         .id                     = RING_BSD,
1349         .mmio_base              = GEN6_BSD_RING_BASE,
1350         .size                   = 32 * PAGE_SIZE,
1351         .init                   = init_ring_common,
1352         .write_tail             = gen6_bsd_ring_write_tail,
1353         .flush                  = gen6_ring_flush,
1354         .add_request            = gen6_add_request,
1355         .get_seqno              = gen6_ring_get_seqno,
1356         .irq_get                = gen6_bsd_ring_get_irq,
1357         .irq_put                = gen6_bsd_ring_put_irq,
1358         .dispatch_execbuffer    = gen6_ring_dispatch_execbuffer,
1359         .sync_to                = gen6_bsd_ring_sync_to,
1360         .semaphore_register     = {MI_SEMAPHORE_SYNC_VR,
1361                                    MI_SEMAPHORE_SYNC_INVALID,
1362                                    MI_SEMAPHORE_SYNC_VB},
1363         .signal_mbox            = {GEN6_RVSYNC, GEN6_BVSYNC},
1364 };
1365
1366 /* Blitter support (SandyBridge+) */
1367
1368 static bool
1369 blt_ring_get_irq(struct intel_ring_buffer *ring)
1370 {
1371         return gen6_ring_get_irq(ring,
1372                                  GT_BLT_USER_INTERRUPT,
1373                                  GEN6_BLITTER_USER_INTERRUPT);
1374 }
1375
1376 static void
1377 blt_ring_put_irq(struct intel_ring_buffer *ring)
1378 {
1379         gen6_ring_put_irq(ring,
1380                           GT_BLT_USER_INTERRUPT,
1381                           GEN6_BLITTER_USER_INTERRUPT);
1382 }
1383
1384
1385 /* Workaround for some stepping of SNB,
1386  * each time when BLT engine ring tail moved,
1387  * the first command in the ring to be parsed
1388  * should be MI_BATCH_BUFFER_START
1389  */
1390 #define NEED_BLT_WORKAROUND(dev) \
1391         (IS_GEN6(dev) && (dev->pdev->revision < 8))
1392
1393 static inline struct drm_i915_gem_object *
1394 to_blt_workaround(struct intel_ring_buffer *ring)
1395 {
1396         return ring->private;
1397 }
1398
1399 static int blt_ring_init(struct intel_ring_buffer *ring)
1400 {
1401         if (NEED_BLT_WORKAROUND(ring->dev)) {
1402                 struct drm_i915_gem_object *obj;
1403                 u32 *ptr;
1404                 int ret;
1405
1406                 obj = i915_gem_alloc_object(ring->dev, 4096);
1407                 if (obj == NULL)
1408                         return -ENOMEM;
1409
1410                 ret = i915_gem_object_pin(obj, 4096, true);
1411                 if (ret) {
1412                         drm_gem_object_unreference(&obj->base);
1413                         return ret;
1414                 }
1415
1416                 ptr = kmap(obj->pages[0]);
1417                 *ptr++ = MI_BATCH_BUFFER_END;
1418                 *ptr++ = MI_NOOP;
1419                 kunmap(obj->pages[0]);
1420
1421                 ret = i915_gem_object_set_to_gtt_domain(obj, false);
1422                 if (ret) {
1423                         i915_gem_object_unpin(obj);
1424                         drm_gem_object_unreference(&obj->base);
1425                         return ret;
1426                 }
1427
1428                 ring->private = obj;
1429         }
1430
1431         return init_ring_common(ring);
1432 }
1433
1434 static int blt_ring_begin(struct intel_ring_buffer *ring,
1435                           int num_dwords)
1436 {
1437         if (ring->private) {
1438                 int ret = intel_ring_begin(ring, num_dwords+2);
1439                 if (ret)
1440                         return ret;
1441
1442                 intel_ring_emit(ring, MI_BATCH_BUFFER_START);
1443                 intel_ring_emit(ring, to_blt_workaround(ring)->gtt_offset);
1444
1445                 return 0;
1446         } else
1447                 return intel_ring_begin(ring, 4);
1448 }
1449
1450 static int blt_ring_flush(struct intel_ring_buffer *ring,
1451                           u32 invalidate, u32 flush)
1452 {
1453         uint32_t cmd;
1454         int ret;
1455
1456         ret = blt_ring_begin(ring, 4);
1457         if (ret)
1458                 return ret;
1459
1460         cmd = MI_FLUSH_DW;
1461         if (invalidate & I915_GEM_DOMAIN_RENDER)
1462                 cmd |= MI_INVALIDATE_TLB;
1463         intel_ring_emit(ring, cmd);
1464         intel_ring_emit(ring, 0);
1465         intel_ring_emit(ring, 0);
1466         intel_ring_emit(ring, MI_NOOP);
1467         intel_ring_advance(ring);
1468         return 0;
1469 }
1470
1471 static void blt_ring_cleanup(struct intel_ring_buffer *ring)
1472 {
1473         if (!ring->private)
1474                 return;
1475
1476         i915_gem_object_unpin(ring->private);
1477         drm_gem_object_unreference(ring->private);
1478         ring->private = NULL;
1479 }
1480
1481 static const struct intel_ring_buffer gen6_blt_ring = {
1482         .name                   = "blt ring",
1483         .id                     = RING_BLT,
1484         .mmio_base              = BLT_RING_BASE,
1485         .size                   = 32 * PAGE_SIZE,
1486         .init                   = blt_ring_init,
1487         .write_tail             = ring_write_tail,
1488         .flush                  = blt_ring_flush,
1489         .add_request            = gen6_add_request,
1490         .get_seqno              = gen6_ring_get_seqno,
1491         .irq_get                = blt_ring_get_irq,
1492         .irq_put                = blt_ring_put_irq,
1493         .dispatch_execbuffer    = gen6_ring_dispatch_execbuffer,
1494         .cleanup                = blt_ring_cleanup,
1495         .sync_to                = gen6_blt_ring_sync_to,
1496         .semaphore_register     = {MI_SEMAPHORE_SYNC_BR,
1497                                    MI_SEMAPHORE_SYNC_BV,
1498                                    MI_SEMAPHORE_SYNC_INVALID},
1499         .signal_mbox            = {GEN6_RBSYNC, GEN6_VBSYNC},
1500 };
1501
1502 int intel_init_render_ring_buffer(struct drm_device *dev)
1503 {
1504         drm_i915_private_t *dev_priv = dev->dev_private;
1505         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1506
1507         *ring = render_ring;
1508         if (INTEL_INFO(dev)->gen >= 6) {
1509                 ring->add_request = gen6_add_request;
1510                 ring->flush = gen6_render_ring_flush;
1511                 ring->irq_get = gen6_render_ring_get_irq;
1512                 ring->irq_put = gen6_render_ring_put_irq;
1513                 ring->get_seqno = gen6_ring_get_seqno;
1514         } else if (IS_GEN5(dev)) {
1515                 ring->add_request = pc_render_add_request;
1516                 ring->get_seqno = pc_render_get_seqno;
1517         }
1518
1519         if (!I915_NEED_GFX_HWS(dev)) {
1520                 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1521                 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1522         }
1523
1524         return intel_init_ring_buffer(dev, ring);
1525 }
1526
1527 int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
1528 {
1529         drm_i915_private_t *dev_priv = dev->dev_private;
1530         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1531
1532         *ring = render_ring;
1533         if (INTEL_INFO(dev)->gen >= 6) {
1534                 ring->add_request = gen6_add_request;
1535                 ring->irq_get = gen6_render_ring_get_irq;
1536                 ring->irq_put = gen6_render_ring_put_irq;
1537         } else if (IS_GEN5(dev)) {
1538                 ring->add_request = pc_render_add_request;
1539                 ring->get_seqno = pc_render_get_seqno;
1540         }
1541
1542         if (!I915_NEED_GFX_HWS(dev))
1543                 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1544
1545         ring->dev = dev;
1546         INIT_LIST_HEAD(&ring->active_list);
1547         INIT_LIST_HEAD(&ring->request_list);
1548         INIT_LIST_HEAD(&ring->gpu_write_list);
1549
1550         ring->size = size;
1551         ring->effective_size = ring->size;
1552         if (IS_I830(ring->dev))
1553                 ring->effective_size -= 128;
1554
1555         ring->map.offset = start;
1556         ring->map.size = size;
1557         ring->map.type = 0;
1558         ring->map.flags = 0;
1559         ring->map.mtrr = 0;
1560
1561         drm_core_ioremap_wc(&ring->map, dev);
1562         if (ring->map.handle == NULL) {
1563                 DRM_ERROR("can not ioremap virtual address for"
1564                           " ring buffer\n");
1565                 return -ENOMEM;
1566         }
1567
1568         ring->virtual_start = (void __force __iomem *)ring->map.handle;
1569         return 0;
1570 }
1571
1572 int intel_init_bsd_ring_buffer(struct drm_device *dev)
1573 {
1574         drm_i915_private_t *dev_priv = dev->dev_private;
1575         struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
1576
1577         if (IS_GEN6(dev) || IS_GEN7(dev))
1578                 *ring = gen6_bsd_ring;
1579         else
1580                 *ring = bsd_ring;
1581
1582         return intel_init_ring_buffer(dev, ring);
1583 }
1584
1585 int intel_init_blt_ring_buffer(struct drm_device *dev)
1586 {
1587         drm_i915_private_t *dev_priv = dev->dev_private;
1588         struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
1589
1590         *ring = gen6_blt_ring;
1591
1592         return intel_init_ring_buffer(dev, ring);
1593 }