drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/slab.h>
  35 #include <linux/swap.h>
  36 #include <linux/pci.h>
  37 #include <linux/intel-gtt.h>
  38
  39 static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
  40
  41 static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
  42                                                   bool pipelined);
  43 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
  44 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
  45 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
  46                                              int write);
  47 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
  48                                                      uint64_t offset,
  49                                                      uint64_t size);
  50 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
  51 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj,
  52                                           bool interruptible);
  53 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
  54                                        unsigned alignment, bool mappable);
  55 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
  56 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
  57                                 struct drm_i915_gem_pwrite *args,
  58                                 struct drm_file *file_priv);
  59 static void i915_gem_free_object_tail(struct drm_gem_object *obj);
  60
  61 static int i915_gem_inactive_shrink(struct shrinker *shrinker,
  62                                     int nr_to_scan,
  63                                     gfp_t gfp_mask);
  64
  65
  66 /* some bookkeeping */
  67 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  68                                   size_t size)
  69 {
  70         dev_priv->mm.object_count++;
  71         dev_priv->mm.object_memory += size;
  72 }
  73
  74 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  75                                      size_t size)
  76 {
  77         dev_priv->mm.object_count--;
  78         dev_priv->mm.object_memory -= size;
  79 }
  80
  81 static void i915_gem_info_add_gtt(struct drm_i915_private *dev_priv,
  82                                   struct drm_gem_object *obj)
  83 {
  84         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  85         dev_priv->mm.gtt_count++;
  86         dev_priv->mm.gtt_memory += obj->size;
  87         if (obj_priv->gtt_offset < dev_priv->mm.gtt_mappable_end) {
  88                 dev_priv->mm.mappable_gtt_used +=
  89                         min_t(size_t, obj->size,
  90                               dev_priv->mm.gtt_mappable_end
  91                                         - obj_priv->gtt_offset);
  92         }
  93 }
  94
  95 static void i915_gem_info_remove_gtt(struct drm_i915_private *dev_priv,
  96                                      struct drm_gem_object *obj)
  97 {
  98         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
  99         dev_priv->mm.gtt_count--;
 100         dev_priv->mm.gtt_memory -= obj->size;
 101         if (obj_priv->gtt_offset < dev_priv->mm.gtt_mappable_end) {
 102                 dev_priv->mm.mappable_gtt_used -=
 103                         min_t(size_t, obj->size,
 104                               dev_priv->mm.gtt_mappable_end
 105                                         - obj_priv->gtt_offset);
 106         }
 107 }
 108
 109 /**
 110  * Update the mappable working set counters. Call _only_ when there is a change
 111  * in one of (pin|fault)_mappable and update *_mappable _before_ calling.
 112  * @mappable: new state the changed mappable flag (either pin_ or fault_).
 113  */
 114 static void
 115 i915_gem_info_update_mappable(struct drm_i915_private *dev_priv,
 116                               struct drm_gem_object *obj,
 117                               bool mappable)
 118 {
 119         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 120
 121         if (mappable) {
 122                 if (obj_priv->pin_mappable && obj_priv->fault_mappable)
 123                         /* Combined state was already mappable. */
 124                         return;
 125                 dev_priv->mm.gtt_mappable_count++;
 126                 dev_priv->mm.gtt_mappable_memory += obj->size;
 127         } else {
 128                 if (obj_priv->pin_mappable || obj_priv->fault_mappable)
 129                         /* Combined state still mappable. */
 130                         return;
 131                 dev_priv->mm.gtt_mappable_count--;
 132                 dev_priv->mm.gtt_mappable_memory -= obj->size;
 133         }
 134 }
 135
 136 static void i915_gem_info_add_pin(struct drm_i915_private *dev_priv,
 137                                   struct drm_gem_object *obj,
 138                                   bool mappable)
 139 {
 140         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 141         dev_priv->mm.pin_count++;
 142         dev_priv->mm.pin_memory += obj->size;
 143         if (mappable) {
 144                 obj_priv->pin_mappable = true;
 145                 i915_gem_info_update_mappable(dev_priv, obj, true);
 146         }
 147 }
 148
 149 static void i915_gem_info_remove_pin(struct drm_i915_private *dev_priv,
 150                                      struct drm_gem_object *obj)
 151 {
 152         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 153         dev_priv->mm.pin_count--;
 154         dev_priv->mm.pin_memory -= obj->size;
 155         if (obj_priv->pin_mappable) {
 156                 obj_priv->pin_mappable = false;
 157                 i915_gem_info_update_mappable(dev_priv, obj, false);
 158         }
 159 }
 160
 161 int
 162 i915_gem_check_is_wedged(struct drm_device *dev)
 163 {
 164         struct drm_i915_private *dev_priv = dev->dev_private;
 165         struct completion *x = &dev_priv->error_completion;
 166         unsigned long flags;
 167         int ret;
 168
 169         if (!atomic_read(&dev_priv->mm.wedged))
 170                 return 0;
 171
 172         ret = wait_for_completion_interruptible(x);
 173         if (ret)
 174                 return ret;
 175
 176         /* Success, we reset the GPU! */
 177         if (!atomic_read(&dev_priv->mm.wedged))
 178                 return 0;
 179
 180         /* GPU is hung, bump the completion count to account for
 181          * the token we just consumed so that we never hit zero and
 182          * end up waiting upon a subsequent completion event that
 183          * will never happen.
 184          */
 185         spin_lock_irqsave(&x->wait.lock, flags);
 186         x->done++;
 187         spin_unlock_irqrestore(&x->wait.lock, flags);
 188         return -EIO;
 189 }
 190
 191 static int i915_mutex_lock_interruptible(struct drm_device *dev)
 192 {
 193         struct drm_i915_private *dev_priv = dev->dev_private;
 194         int ret;
 195
 196         ret = i915_gem_check_is_wedged(dev);
 197         if (ret)
 198                 return ret;
 199
 200         ret = mutex_lock_interruptible(&dev->struct_mutex);
 201         if (ret)
 202                 return ret;
 203
 204         if (atomic_read(&dev_priv->mm.wedged)) {
 205                 mutex_unlock(&dev->struct_mutex);
 206                 return -EAGAIN;
 207         }
 208
 209         WARN_ON(i915_verify_lists(dev));
 210         return 0;
 211 }
 212
 213 static inline bool
 214 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
 215 {
 216         return obj_priv->gtt_space &&
 217                 !obj_priv->active &&
 218                 obj_priv->pin_count == 0;
 219 }
 220
 221 int i915_gem_do_init(struct drm_device *dev,
 222                      unsigned long start,
 223                      unsigned long mappable_end,
 224                      unsigned long end)
 225 {
 226         drm_i915_private_t *dev_priv = dev->dev_private;
 227
 228         if (start >= end ||
 229             (start & (PAGE_SIZE - 1)) != 0 ||
 230             (end & (PAGE_SIZE - 1)) != 0) {
 231                 return -EINVAL;
 232         }
 233
 234         drm_mm_init(&dev_priv->mm.gtt_space, start,
 235                     end - start);
 236
 237         dev_priv->mm.gtt_total = end - start;
 238         dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
 239         dev_priv->mm.gtt_mappable_end = mappable_end;
 240
 241         return 0;
 242 }
 243
 244 int
 245 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 246                     struct drm_file *file_priv)
 247 {
 248         struct drm_i915_gem_init *args = data;
 249         int ret;
 250
 251         mutex_lock(&dev->struct_mutex);
 252         ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
 253         mutex_unlock(&dev->struct_mutex);
 254
 255         return ret;
 256 }
 257
 258 int
 259 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 260                             struct drm_file *file_priv)
 261 {
 262         struct drm_i915_private *dev_priv = dev->dev_private;
 263         struct drm_i915_gem_get_aperture *args = data;
 264
 265         if (!(dev->driver->driver_features & DRIVER_GEM))
 266                 return -ENODEV;
 267
 268         mutex_lock(&dev->struct_mutex);
 269         args->aper_size = dev_priv->mm.gtt_total;
 270         args->aper_available_size = args->aper_size - dev_priv->mm.pin_memory;
 271         mutex_unlock(&dev->struct_mutex);
 272
 273         return 0;
 274 }
 275
 276
 277 /**
 278  * Creates a new mm object and returns a handle to it.
 279  */
 280 int
 281 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 282                       struct drm_file *file_priv)
 283 {
 284         struct drm_i915_gem_create *args = data;
 285         struct drm_gem_object *obj;
 286         int ret;
 287         u32 handle;
 288
 289         args->size = roundup(args->size, PAGE_SIZE);
 290
 291         /* Allocate the new object */
 292         obj = i915_gem_alloc_object(dev, args->size);
 293         if (obj == NULL)
 294                 return -ENOMEM;
 295
 296         ret = drm_gem_handle_create(file_priv, obj, &handle);
 297         if (ret) {
 298                 drm_gem_object_release(obj);
 299                 i915_gem_info_remove_obj(dev->dev_private, obj->size);
 300                 kfree(obj);
 301                 return ret;
 302         }
 303
 304         /* drop reference from allocate - handle holds it now */
 305         drm_gem_object_unreference(obj);
 306         trace_i915_gem_object_create(obj);
 307
 308         args->handle = handle;
 309         return 0;
 310 }
 311
 312 static bool
 313 i915_gem_object_cpu_accessible(struct drm_i915_gem_object *obj)
 314 {
 315         struct drm_device *dev = obj->base.dev;
 316         drm_i915_private_t *dev_priv = dev->dev_private;
 317
 318         return obj->gtt_space == NULL ||
 319                 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
 320 }
 321
 322 static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
 323 {
 324         drm_i915_private_t *dev_priv = obj->dev->dev_private;
 325         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 326
 327         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 328                 obj_priv->tiling_mode != I915_TILING_NONE;
 329 }
 330
 331 static inline void
 332 slow_shmem_copy(struct page *dst_page,
 333                 int dst_offset,
 334                 struct page *src_page,
 335                 int src_offset,
 336                 int length)
 337 {
 338         char *dst_vaddr, *src_vaddr;
 339
 340         dst_vaddr = kmap(dst_page);
 341         src_vaddr = kmap(src_page);
 342
 343         memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
 344
 345         kunmap(src_page);
 346         kunmap(dst_page);
 347 }
 348
 349 static inline void
 350 slow_shmem_bit17_copy(struct page *gpu_page,
 351                       int gpu_offset,
 352                       struct page *cpu_page,
 353                       int cpu_offset,
 354                       int length,
 355                       int is_read)
 356 {
 357         char *gpu_vaddr, *cpu_vaddr;
 358
 359         /* Use the unswizzled path if this page isn't affected. */
 360         if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
 361                 if (is_read)
 362                         return slow_shmem_copy(cpu_page, cpu_offset,
 363                                                gpu_page, gpu_offset, length);
 364                 else
 365                         return slow_shmem_copy(gpu_page, gpu_offset,
 366                                                cpu_page, cpu_offset, length);
 367         }
 368
 369         gpu_vaddr = kmap(gpu_page);
 370         cpu_vaddr = kmap(cpu_page);
 371
 372         /* Copy the data, XORing A6 with A17 (1). The user already knows he's
 373          * XORing with the other bits (A9 for Y, A9 and A10 for X)
 374          */
 375         while (length > 0) {
 376                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 377                 int this_length = min(cacheline_end - gpu_offset, length);
 378                 int swizzled_gpu_offset = gpu_offset ^ 64;
 379
 380                 if (is_read) {
 381                         memcpy(cpu_vaddr + cpu_offset,
 382                                gpu_vaddr + swizzled_gpu_offset,
 383                                this_length);
 384                 } else {
 385                         memcpy(gpu_vaddr + swizzled_gpu_offset,
 386                                cpu_vaddr + cpu_offset,
 387                                this_length);
 388                 }
 389                 cpu_offset += this_length;
 390                 gpu_offset += this_length;
 391                 length -= this_length;
 392         }
 393
 394         kunmap(cpu_page);
 395         kunmap(gpu_page);
 396 }
 397
 398 /**
 399  * This is the fast shmem pread path, which attempts to copy_from_user directly
 400  * from the backing pages of the object to the user's address space.  On a
 401  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
 402  */
 403 static int
 404 i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
 405                           struct drm_i915_gem_pread *args,
 406                           struct drm_file *file_priv)
 407 {
 408         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 409         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
 410         ssize_t remain;
 411         loff_t offset;
 412         char __user *user_data;
 413         int page_offset, page_length;
 414
 415         user_data = (char __user *) (uintptr_t) args->data_ptr;
 416         remain = args->size;
 417
 418         obj_priv = to_intel_bo(obj);
 419         offset = args->offset;
 420
 421         while (remain > 0) {
 422                 struct page *page;
 423                 char *vaddr;
 424                 int ret;
 425
 426                 /* Operation in this page
 427                  *
 428                  * page_offset = offset within page
 429                  * page_length = bytes to copy for this page
 430                  */
 431                 page_offset = offset & (PAGE_SIZE-1);
 432                 page_length = remain;
 433                 if ((page_offset + remain) > PAGE_SIZE)
 434                         page_length = PAGE_SIZE - page_offset;
 435
 436                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 437                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 438                 if (IS_ERR(page))
 439                         return PTR_ERR(page);
 440
 441                 vaddr = kmap_atomic(page);
 442                 ret = __copy_to_user_inatomic(user_data,
 443                                               vaddr + page_offset,
 444                                               page_length);
 445                 kunmap_atomic(vaddr);
 446
 447                 mark_page_accessed(page);
 448                 page_cache_release(page);
 449                 if (ret)
 450                         return -EFAULT;
 451
 452                 remain -= page_length;
 453                 user_data += page_length;
 454                 offset += page_length;
 455         }
 456
 457         return 0;
 458 }
 459
 460 /**
 461  * This is the fallback shmem pread path, which allocates temporary storage
 462  * in kernel space to copy_to_user into outside of the struct_mutex, so we
 463  * can copy out of the object's backing pages while holding the struct mutex
 464  * and not take page faults.
 465  */
 466 static int
 467 i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
 468                           struct drm_i915_gem_pread *args,
 469                           struct drm_file *file_priv)
 470 {
 471         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
 472         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 473         struct mm_struct *mm = current->mm;
 474         struct page **user_pages;
 475         ssize_t remain;
 476         loff_t offset, pinned_pages, i;
 477         loff_t first_data_page, last_data_page, num_pages;
 478         int shmem_page_offset;
 479         int data_page_index, data_page_offset;
 480         int page_length;
 481         int ret;
 482         uint64_t data_ptr = args->data_ptr;
 483         int do_bit17_swizzling;
 484
 485         remain = args->size;
 486
 487         /* Pin the user pages containing the data.  We can't fault while
 488          * holding the struct mutex, yet we want to hold it while
 489          * dereferencing the user data.
 490          */
 491         first_data_page = data_ptr / PAGE_SIZE;
 492         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 493         num_pages = last_data_page - first_data_page + 1;
 494
 495         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
 496         if (user_pages == NULL)
 497                 return -ENOMEM;
 498
 499         mutex_unlock(&dev->struct_mutex);
 500         down_read(&mm->mmap_sem);
 501         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 502                                       num_pages, 1, 0, user_pages, NULL);
 503         up_read(&mm->mmap_sem);
 504         mutex_lock(&dev->struct_mutex);
 505         if (pinned_pages < num_pages) {
 506                 ret = -EFAULT;
 507                 goto out;
 508         }
 509
 510         ret = i915_gem_object_set_cpu_read_domain_range(obj,
 511                                                         args->offset,
 512                                                         args->size);
 513         if (ret)
 514                 goto out;
 515
 516         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 517
 518         obj_priv = to_intel_bo(obj);
 519         offset = args->offset;
 520
 521         while (remain > 0) {
 522                 struct page *page;
 523
 524                 /* Operation in this page
 525                  *
 526                  * shmem_page_offset = offset within page in shmem file
 527                  * data_page_index = page number in get_user_pages return
 528                  * data_page_offset = offset with data_page_index page.
 529                  * page_length = bytes to copy for this page
 530                  */
 531                 shmem_page_offset = offset & ~PAGE_MASK;
 532                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 533                 data_page_offset = data_ptr & ~PAGE_MASK;
 534
 535                 page_length = remain;
 536                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 537                         page_length = PAGE_SIZE - shmem_page_offset;
 538                 if ((data_page_offset + page_length) > PAGE_SIZE)
 539                         page_length = PAGE_SIZE - data_page_offset;
 540
 541                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 542                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 543                 if (IS_ERR(page))
 544                         return PTR_ERR(page);
 545
 546                 if (do_bit17_swizzling) {
 547                         slow_shmem_bit17_copy(page,
 548                                               shmem_page_offset,
 549                                               user_pages[data_page_index],
 550                                               data_page_offset,
 551                                               page_length,
 552                                               1);
 553                 } else {
 554                         slow_shmem_copy(user_pages[data_page_index],
 555                                         data_page_offset,
 556                                         page,
 557                                         shmem_page_offset,
 558                                         page_length);
 559                 }
 560
 561                 mark_page_accessed(page);
 562                 page_cache_release(page);
 563
 564                 remain -= page_length;
 565                 data_ptr += page_length;
 566                 offset += page_length;
 567         }
 568
 569 out:
 570         for (i = 0; i < pinned_pages; i++) {
 571                 SetPageDirty(user_pages[i]);
 572                 mark_page_accessed(user_pages[i]);
 573                 page_cache_release(user_pages[i]);
 574         }
 575         drm_free_large(user_pages);
 576
 577         return ret;
 578 }
 579
 580 /**
 581  * Reads data from the object referenced by handle.
 582  *
 583  * On error, the contents of *data are undefined.
 584  */
 585 int
 586 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 587                      struct drm_file *file_priv)
 588 {
 589         struct drm_i915_gem_pread *args = data;
 590         struct drm_gem_object *obj;
 591         struct drm_i915_gem_object *obj_priv;
 592         int ret = 0;
 593
 594         ret = i915_mutex_lock_interruptible(dev);
 595         if (ret)
 596                 return ret;
 597
 598         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 599         if (obj == NULL) {
 600                 ret = -ENOENT;
 601                 goto unlock;
 602         }
 603         obj_priv = to_intel_bo(obj);
 604
 605         /* Bounds check source.  */
 606         if (args->offset > obj->size || args->size > obj->size - args->offset) {
 607                 ret = -EINVAL;
 608                 goto out;
 609         }
 610
 611         if (args->size == 0)
 612                 goto out;
 613
 614         if (!access_ok(VERIFY_WRITE,
 615                        (char __user *)(uintptr_t)args->data_ptr,
 616                        args->size)) {
 617                 ret = -EFAULT;
 618                 goto out;
 619         }
 620
 621         ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,
 622                                        args->size);
 623         if (ret) {
 624                 ret = -EFAULT;
 625                 goto out;
 626         }
 627
 628         ret = i915_gem_object_set_cpu_read_domain_range(obj,
 629                                                         args->offset,
 630                                                         args->size);
 631         if (ret)
 632                 goto out;
 633
 634         ret = -EFAULT;
 635         if (!i915_gem_object_needs_bit17_swizzle(obj))
 636                 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
 637         if (ret == -EFAULT)
 638                 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
 639
 640 out:
 641         drm_gem_object_unreference(obj);
 642 unlock:
 643         mutex_unlock(&dev->struct_mutex);
 644         return ret;
 645 }
 646
 647 /* This is the fast write path which cannot handle
 648  * page faults in the source data
 649  */
 650
 651 static inline int
 652 fast_user_write(struct io_mapping *mapping,
 653                 loff_t page_base, int page_offset,
 654                 char __user *user_data,
 655                 int length)
 656 {
 657         char *vaddr_atomic;
 658         unsigned long unwritten;
 659
 660         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 661         unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
 662                                                       user_data, length);
 663         io_mapping_unmap_atomic(vaddr_atomic);
 664         return unwritten;
 665 }
 666
 667 /* Here's the write path which can sleep for
 668  * page faults
 669  */
 670
 671 static inline void
 672 slow_kernel_write(struct io_mapping *mapping,
 673                   loff_t gtt_base, int gtt_offset,
 674                   struct page *user_page, int user_offset,
 675                   int length)
 676 {
 677         char __iomem *dst_vaddr;
 678         char *src_vaddr;
 679
 680         dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
 681         src_vaddr = kmap(user_page);
 682
 683         memcpy_toio(dst_vaddr + gtt_offset,
 684                     src_vaddr + user_offset,
 685                     length);
 686
 687         kunmap(user_page);
 688         io_mapping_unmap(dst_vaddr);
 689 }
 690
 691 /**
 692  * This is the fast pwrite path, where we copy the data directly from the
 693  * user into the GTT, uncached.
 694  */
 695 static int
 696 i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 697                          struct drm_i915_gem_pwrite *args,
 698                          struct drm_file *file_priv)
 699 {
 700         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 701         drm_i915_private_t *dev_priv = dev->dev_private;
 702         ssize_t remain;
 703         loff_t offset, page_base;
 704         char __user *user_data;
 705         int page_offset, page_length;
 706
 707         user_data = (char __user *) (uintptr_t) args->data_ptr;
 708         remain = args->size;
 709
 710         obj_priv = to_intel_bo(obj);
 711         offset = obj_priv->gtt_offset + args->offset;
 712
 713         while (remain > 0) {
 714                 /* Operation in this page
 715                  *
 716                  * page_base = page offset within aperture
 717                  * page_offset = offset within page
 718                  * page_length = bytes to copy for this page
 719                  */
 720                 page_base = (offset & ~(PAGE_SIZE-1));
 721                 page_offset = offset & (PAGE_SIZE-1);
 722                 page_length = remain;
 723                 if ((page_offset + remain) > PAGE_SIZE)
 724                         page_length = PAGE_SIZE - page_offset;
 725
 726                 /* If we get a fault while copying data, then (presumably) our
 727                  * source page isn't available.  Return the error and we'll
 728                  * retry in the slow path.
 729                  */
 730                 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
 731                                     page_offset, user_data, page_length))
 732
 733                         return -EFAULT;
 734
 735                 remain -= page_length;
 736                 user_data += page_length;
 737                 offset += page_length;
 738         }
 739
 740         return 0;
 741 }
 742
 743 /**
 744  * This is the fallback GTT pwrite path, which uses get_user_pages to pin
 745  * the memory and maps it using kmap_atomic for copying.
 746  *
 747  * This code resulted in x11perf -rgb10text consuming about 10% more CPU
 748  * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
 749  */
 750 static int
 751 i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 752                          struct drm_i915_gem_pwrite *args,
 753                          struct drm_file *file_priv)
 754 {
 755         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 756         drm_i915_private_t *dev_priv = dev->dev_private;
 757         ssize_t remain;
 758         loff_t gtt_page_base, offset;
 759         loff_t first_data_page, last_data_page, num_pages;
 760         loff_t pinned_pages, i;
 761         struct page **user_pages;
 762         struct mm_struct *mm = current->mm;
 763         int gtt_page_offset, data_page_offset, data_page_index, page_length;
 764         int ret;
 765         uint64_t data_ptr = args->data_ptr;
 766
 767         remain = args->size;
 768
 769         /* Pin the user pages containing the data.  We can't fault while
 770          * holding the struct mutex, and all of the pwrite implementations
 771          * want to hold it while dereferencing the user data.
 772          */
 773         first_data_page = data_ptr / PAGE_SIZE;
 774         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 775         num_pages = last_data_page - first_data_page + 1;
 776
 777         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
 778         if (user_pages == NULL)
 779                 return -ENOMEM;
 780
 781         mutex_unlock(&dev->struct_mutex);
 782         down_read(&mm->mmap_sem);
 783         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 784                                       num_pages, 0, 0, user_pages, NULL);
 785         up_read(&mm->mmap_sem);
 786         mutex_lock(&dev->struct_mutex);
 787         if (pinned_pages < num_pages) {
 788                 ret = -EFAULT;
 789                 goto out_unpin_pages;
 790         }
 791
 792         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 793         if (ret)
 794                 goto out_unpin_pages;
 795
 796         obj_priv = to_intel_bo(obj);
 797         offset = obj_priv->gtt_offset + args->offset;
 798
 799         while (remain > 0) {
 800                 /* Operation in this page
 801                  *
 802                  * gtt_page_base = page offset within aperture
 803                  * gtt_page_offset = offset within page in aperture
 804                  * data_page_index = page number in get_user_pages return
 805                  * data_page_offset = offset with data_page_index page.
 806                  * page_length = bytes to copy for this page
 807                  */
 808                 gtt_page_base = offset & PAGE_MASK;
 809                 gtt_page_offset = offset & ~PAGE_MASK;
 810                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 811                 data_page_offset = data_ptr & ~PAGE_MASK;
 812
 813                 page_length = remain;
 814                 if ((gtt_page_offset + page_length) > PAGE_SIZE)
 815                         page_length = PAGE_SIZE - gtt_page_offset;
 816                 if ((data_page_offset + page_length) > PAGE_SIZE)
 817                         page_length = PAGE_SIZE - data_page_offset;
 818
 819                 slow_kernel_write(dev_priv->mm.gtt_mapping,
 820                                   gtt_page_base, gtt_page_offset,
 821                                   user_pages[data_page_index],
 822                                   data_page_offset,
 823                                   page_length);
 824
 825                 remain -= page_length;
 826                 offset += page_length;
 827                 data_ptr += page_length;
 828         }
 829
 830 out_unpin_pages:
 831         for (i = 0; i < pinned_pages; i++)
 832                 page_cache_release(user_pages[i]);
 833         drm_free_large(user_pages);
 834
 835         return ret;
 836 }
 837
 838 /**
 839  * This is the fast shmem pwrite path, which attempts to directly
 840  * copy_from_user into the kmapped pages backing the object.
 841  */
 842 static int
 843 i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 844                            struct drm_i915_gem_pwrite *args,
 845                            struct drm_file *file_priv)
 846 {
 847         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
 848         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 849         ssize_t remain;
 850         loff_t offset;
 851         char __user *user_data;
 852         int page_offset, page_length;
 853
 854         user_data = (char __user *) (uintptr_t) args->data_ptr;
 855         remain = args->size;
 856
 857         obj_priv = to_intel_bo(obj);
 858         offset = args->offset;
 859         obj_priv->dirty = 1;
 860
 861         while (remain > 0) {
 862                 struct page *page;
 863                 char *vaddr;
 864                 int ret;
 865
 866                 /* Operation in this page
 867                  *
 868                  * page_offset = offset within page
 869                  * page_length = bytes to copy for this page
 870                  */
 871                 page_offset = offset & (PAGE_SIZE-1);
 872                 page_length = remain;
 873                 if ((page_offset + remain) > PAGE_SIZE)
 874                         page_length = PAGE_SIZE - page_offset;
 875
 876                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 877                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 878                 if (IS_ERR(page))
 879                         return PTR_ERR(page);
 880
 881                 vaddr = kmap_atomic(page, KM_USER0);
 882                 ret = __copy_from_user_inatomic(vaddr + page_offset,
 883                                                 user_data,
 884                                                 page_length);
 885                 kunmap_atomic(vaddr, KM_USER0);
 886
 887                 set_page_dirty(page);
 888                 mark_page_accessed(page);
 889                 page_cache_release(page);
 890
 891                 /* If we get a fault while copying data, then (presumably) our
 892                  * source page isn't available.  Return the error and we'll
 893                  * retry in the slow path.
 894                  */
 895                 if (ret)
 896                         return -EFAULT;
 897
 898                 remain -= page_length;
 899                 user_data += page_length;
 900                 offset += page_length;
 901         }
 902
 903         return 0;
 904 }
 905
 906 /**
 907  * This is the fallback shmem pwrite path, which uses get_user_pages to pin
 908  * the memory and maps it using kmap_atomic for copying.
 909  *
 910  * This avoids taking mmap_sem for faulting on the user's address while the
 911  * struct_mutex is held.
 912  */
 913 static int
 914 i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 915                            struct drm_i915_gem_pwrite *args,
 916                            struct drm_file *file_priv)
 917 {
 918         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
 919         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 920         struct mm_struct *mm = current->mm;
 921         struct page **user_pages;
 922         ssize_t remain;
 923         loff_t offset, pinned_pages, i;
 924         loff_t first_data_page, last_data_page, num_pages;
 925         int shmem_page_offset;
 926         int data_page_index,  data_page_offset;
 927         int page_length;
 928         int ret;
 929         uint64_t data_ptr = args->data_ptr;
 930         int do_bit17_swizzling;
 931
 932         remain = args->size;
 933
 934         /* Pin the user pages containing the data.  We can't fault while
 935          * holding the struct mutex, and all of the pwrite implementations
 936          * want to hold it while dereferencing the user data.
 937          */
 938         first_data_page = data_ptr / PAGE_SIZE;
 939         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 940         num_pages = last_data_page - first_data_page + 1;
 941
 942         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
 943         if (user_pages == NULL)
 944                 return -ENOMEM;
 945
 946         mutex_unlock(&dev->struct_mutex);
 947         down_read(&mm->mmap_sem);
 948         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 949                                       num_pages, 0, 0, user_pages, NULL);
 950         up_read(&mm->mmap_sem);
 951         mutex_lock(&dev->struct_mutex);
 952         if (pinned_pages < num_pages) {
 953                 ret = -EFAULT;
 954                 goto out;
 955         }
 956
 957         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 958         if (ret)
 959                 goto out;
 960
 961         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 962
 963         obj_priv = to_intel_bo(obj);
 964         offset = args->offset;
 965         obj_priv->dirty = 1;
 966
 967         while (remain > 0) {
 968                 struct page *page;
 969
 970                 /* Operation in this page
 971                  *
 972                  * shmem_page_offset = offset within page in shmem file
 973                  * data_page_index = page number in get_user_pages return
 974                  * data_page_offset = offset with data_page_index page.
 975                  * page_length = bytes to copy for this page
 976                  */
 977                 shmem_page_offset = offset & ~PAGE_MASK;
 978                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 979                 data_page_offset = data_ptr & ~PAGE_MASK;
 980
 981                 page_length = remain;
 982                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 983                         page_length = PAGE_SIZE - shmem_page_offset;
 984                 if ((data_page_offset + page_length) > PAGE_SIZE)
 985                         page_length = PAGE_SIZE - data_page_offset;
 986
 987                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 988                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 989                 if (IS_ERR(page)) {
 990                         ret = PTR_ERR(page);
 991                         goto out;
 992                 }
 993
 994                 if (do_bit17_swizzling) {
 995                         slow_shmem_bit17_copy(page,
 996                                               shmem_page_offset,
 997                                               user_pages[data_page_index],
 998                                               data_page_offset,
 999                                               page_length,
1000                                               0);
1001                 } else {
1002                         slow_shmem_copy(page,
1003                                         shmem_page_offset,
1004                                         user_pages[data_page_index],
1005                                         data_page_offset,
1006                                         page_length);
1007                 }
1008
1009                 set_page_dirty(page);
1010                 mark_page_accessed(page);
1011                 page_cache_release(page);
1012
1013                 remain -= page_length;
1014                 data_ptr += page_length;
1015                 offset += page_length;
1016         }
1017
1018 out:
1019         for (i = 0; i < pinned_pages; i++)
1020                 page_cache_release(user_pages[i]);
1021         drm_free_large(user_pages);
1022
1023         return ret;
1024 }
1025
1026 /**
1027  * Writes data to the object referenced by handle.
1028  *
1029  * On error, the contents of the buffer that were to be modified are undefined.
1030  */
1031 int
1032 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1033                       struct drm_file *file)
1034 {
1035         struct drm_i915_gem_pwrite *args = data;
1036         struct drm_gem_object *obj;
1037         struct drm_i915_gem_object *obj_priv;
1038         int ret = 0;
1039
1040         ret = i915_mutex_lock_interruptible(dev);
1041         if (ret)
1042                 return ret;
1043
1044         obj = drm_gem_object_lookup(dev, file, args->handle);
1045         if (obj == NULL) {
1046                 ret = -ENOENT;
1047                 goto unlock;
1048         }
1049         obj_priv = to_intel_bo(obj);
1050
1051
1052         /* Bounds check destination. */
1053         if (args->offset > obj->size || args->size > obj->size - args->offset) {
1054                 ret = -EINVAL;
1055                 goto out;
1056         }
1057
1058         if (args->size == 0)
1059                 goto out;
1060
1061         if (!access_ok(VERIFY_READ,
1062                        (char __user *)(uintptr_t)args->data_ptr,
1063                        args->size)) {
1064                 ret = -EFAULT;
1065                 goto out;
1066         }
1067
1068         ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
1069                                       args->size);
1070         if (ret) {
1071                 ret = -EFAULT;
1072                 goto out;
1073         }
1074
1075         /* We can only do the GTT pwrite on untiled buffers, as otherwise
1076          * it would end up going through the fenced access, and we'll get
1077          * different detiling behavior between reading and writing.
1078          * pread/pwrite currently are reading and writing from the CPU
1079          * perspective, requiring manual detiling by the client.
1080          */
1081         if (obj_priv->phys_obj)
1082                 ret = i915_gem_phys_pwrite(dev, obj, args, file);
1083         else if (obj_priv->tiling_mode == I915_TILING_NONE &&
1084                  obj_priv->gtt_space &&
1085                  obj->write_domain != I915_GEM_DOMAIN_CPU) {
1086                 ret = i915_gem_object_pin(obj, 0, true);
1087                 if (ret)
1088                         goto out;
1089
1090                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
1091                 if (ret)
1092                         goto out_unpin;
1093
1094                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1095                 if (ret == -EFAULT)
1096                         ret = i915_gem_gtt_pwrite_slow(dev, obj, args, file);
1097
1098 out_unpin:
1099                 i915_gem_object_unpin(obj);
1100         } else {
1101                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1102                 if (ret)
1103                         goto out;
1104
1105                 ret = -EFAULT;
1106                 if (!i915_gem_object_needs_bit17_swizzle(obj))
1107                         ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
1108                 if (ret == -EFAULT)
1109                         ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
1110         }
1111
1112 out:
1113         drm_gem_object_unreference(obj);
1114 unlock:
1115         mutex_unlock(&dev->struct_mutex);
1116         return ret;
1117 }
1118
1119 /**
1120  * Called when user space prepares to use an object with the CPU, either
1121  * through the mmap ioctl's mapping or a GTT mapping.
1122  */
1123 int
1124 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1125                           struct drm_file *file_priv)
1126 {
1127         struct drm_i915_private *dev_priv = dev->dev_private;
1128         struct drm_i915_gem_set_domain *args = data;
1129         struct drm_gem_object *obj;
1130         struct drm_i915_gem_object *obj_priv;
1131         uint32_t read_domains = args->read_domains;
1132         uint32_t write_domain = args->write_domain;
1133         int ret;
1134
1135         if (!(dev->driver->driver_features & DRIVER_GEM))
1136                 return -ENODEV;
1137
1138         /* Only handle setting domains to types used by the CPU. */
1139         if (write_domain & I915_GEM_GPU_DOMAINS)
1140                 return -EINVAL;
1141
1142         if (read_domains & I915_GEM_GPU_DOMAINS)
1143                 return -EINVAL;
1144
1145         /* Having something in the write domain implies it's in the read
1146          * domain, and only that read domain.  Enforce that in the request.
1147          */
1148         if (write_domain != 0 && read_domains != write_domain)
1149                 return -EINVAL;
1150
1151         ret = i915_mutex_lock_interruptible(dev);
1152         if (ret)
1153                 return ret;
1154
1155         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1156         if (obj == NULL) {
1157                 ret = -ENOENT;
1158                 goto unlock;
1159         }
1160         obj_priv = to_intel_bo(obj);
1161
1162         intel_mark_busy(dev, obj);
1163
1164         if (read_domains & I915_GEM_DOMAIN_GTT) {
1165                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1166
1167                 /* Update the LRU on the fence for the CPU access that's
1168                  * about to occur.
1169                  */
1170                 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1171                         struct drm_i915_fence_reg *reg =
1172                                 &dev_priv->fence_regs[obj_priv->fence_reg];
1173                         list_move_tail(&reg->lru_list,
1174                                        &dev_priv->mm.fence_list);
1175                 }
1176
1177                 /* Silently promote "you're not bound, there was nothing to do"
1178                  * to success, since the client was just asking us to
1179                  * make sure everything was done.
1180                  */
1181                 if (ret == -EINVAL)
1182                         ret = 0;
1183         } else {
1184                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1185         }
1186
1187         /* Maintain LRU order of "inactive" objects */
1188         if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1189                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1190
1191         drm_gem_object_unreference(obj);
1192 unlock:
1193         mutex_unlock(&dev->struct_mutex);
1194         return ret;
1195 }
1196
1197 /**
1198  * Called when user space has done writes to this buffer
1199  */
1200 int
1201 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1202                       struct drm_file *file_priv)
1203 {
1204         struct drm_i915_gem_sw_finish *args = data;
1205         struct drm_gem_object *obj;
1206         int ret = 0;
1207
1208         if (!(dev->driver->driver_features & DRIVER_GEM))
1209                 return -ENODEV;
1210
1211         ret = i915_mutex_lock_interruptible(dev);
1212         if (ret)
1213                 return ret;
1214
1215         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1216         if (obj == NULL) {
1217                 ret = -ENOENT;
1218                 goto unlock;
1219         }
1220
1221         /* Pinned buffers may be scanout, so flush the cache */
1222         if (to_intel_bo(obj)->pin_count)
1223                 i915_gem_object_flush_cpu_write_domain(obj);
1224
1225         drm_gem_object_unreference(obj);
1226 unlock:
1227         mutex_unlock(&dev->struct_mutex);
1228         return ret;
1229 }
1230
1231 /**
1232  * Maps the contents of an object, returning the address it is mapped
1233  * into.
1234  *
1235  * While the mapping holds a reference on the contents of the object, it doesn't
1236  * imply a ref on the object itself.
1237  */
1238 int
1239 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1240                    struct drm_file *file_priv)
1241 {
1242         struct drm_i915_private *dev_priv = dev->dev_private;
1243         struct drm_i915_gem_mmap *args = data;
1244         struct drm_gem_object *obj;
1245         loff_t offset;
1246         unsigned long addr;
1247
1248         if (!(dev->driver->driver_features & DRIVER_GEM))
1249                 return -ENODEV;
1250
1251         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1252         if (obj == NULL)
1253                 return -ENOENT;
1254
1255         if (obj->size > dev_priv->mm.gtt_mappable_end) {
1256                 drm_gem_object_unreference_unlocked(obj);
1257                 return -E2BIG;
1258         }
1259
1260         offset = args->offset;
1261
1262         down_write(&current->mm->mmap_sem);
1263         addr = do_mmap(obj->filp, 0, args->size,
1264                        PROT_READ | PROT_WRITE, MAP_SHARED,
1265                        args->offset);
1266         up_write(&current->mm->mmap_sem);
1267         drm_gem_object_unreference_unlocked(obj);
1268         if (IS_ERR((void *)addr))
1269                 return addr;
1270
1271         args->addr_ptr = (uint64_t) addr;
1272
1273         return 0;
1274 }
1275
1276 /**
1277  * i915_gem_fault - fault a page into the GTT
1278  * vma: VMA in question
1279  * vmf: fault info
1280  *
1281  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1282  * from userspace.  The fault handler takes care of binding the object to
1283  * the GTT (if needed), allocating and programming a fence register (again,
1284  * only if needed based on whether the old reg is still valid or the object
1285  * is tiled) and inserting a new PTE into the faulting process.
1286  *
1287  * Note that the faulting process may involve evicting existing objects
1288  * from the GTT and/or fence registers to make room.  So performance may
1289  * suffer if the GTT working set is large or there are few fence registers
1290  * left.
1291  */
1292 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1293 {
1294         struct drm_gem_object *obj = vma->vm_private_data;
1295         struct drm_device *dev = obj->dev;
1296         drm_i915_private_t *dev_priv = dev->dev_private;
1297         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1298         pgoff_t page_offset;
1299         unsigned long pfn;
1300         int ret = 0;
1301         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1302
1303         /* We don't use vmf->pgoff since that has the fake offset */
1304         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1305                 PAGE_SHIFT;
1306
1307         /* Now bind it into the GTT if needed */
1308         mutex_lock(&dev->struct_mutex);
1309         BUG_ON(obj_priv->pin_count && !obj_priv->pin_mappable);
1310         if (!i915_gem_object_cpu_accessible(obj_priv))
1311                 i915_gem_object_unbind(obj);
1312
1313         if (!obj_priv->gtt_space) {
1314                 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
1315                 if (ret)
1316                         goto unlock;
1317         }
1318
1319         ret = i915_gem_object_set_to_gtt_domain(obj, write);
1320         if (ret)
1321                 goto unlock;
1322
1323         if (!obj_priv->fault_mappable) {
1324                 obj_priv->fault_mappable = true;
1325                 i915_gem_info_update_mappable(dev_priv, obj, true);
1326         }
1327
1328         /* Need a new fence register? */
1329         if (obj_priv->tiling_mode != I915_TILING_NONE) {
1330                 ret = i915_gem_object_get_fence_reg(obj, true);
1331                 if (ret)
1332                         goto unlock;
1333         }
1334
1335         if (i915_gem_object_is_inactive(obj_priv))
1336                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1337
1338         pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1339                 page_offset;
1340
1341         /* Finally, remap it using the new GTT offset */
1342         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1343 unlock:
1344         mutex_unlock(&dev->struct_mutex);
1345
1346         switch (ret) {
1347         case 0:
1348         case -ERESTARTSYS:
1349                 return VM_FAULT_NOPAGE;
1350         case -ENOMEM:
1351         case -EAGAIN:
1352                 return VM_FAULT_OOM;
1353         default:
1354                 return VM_FAULT_SIGBUS;
1355         }
1356 }
1357
1358 /**
1359  * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1360  * @obj: obj in question
1361  *
1362  * GEM memory mapping works by handing back to userspace a fake mmap offset
1363  * it can use in a subsequent mmap(2) call.  The DRM core code then looks
1364  * up the object based on the offset and sets up the various memory mapping
1365  * structures.
1366  *
1367  * This routine allocates and attaches a fake offset for @obj.
1368  */
1369 static int
1370 i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1371 {
1372         struct drm_device *dev = obj->dev;
1373         struct drm_gem_mm *mm = dev->mm_private;
1374         struct drm_map_list *list;
1375         struct drm_local_map *map;
1376         int ret = 0;
1377
1378         /* Set the object up for mmap'ing */
1379         list = &obj->map_list;
1380         list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1381         if (!list->map)
1382                 return -ENOMEM;
1383
1384         map = list->map;
1385         map->type = _DRM_GEM;
1386         map->size = obj->size;
1387         map->handle = obj;
1388
1389         /* Get a DRM GEM mmap offset allocated... */
1390         list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1391                                                     obj->size / PAGE_SIZE, 0, 0);
1392         if (!list->file_offset_node) {
1393                 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1394                 ret = -ENOSPC;
1395                 goto out_free_list;
1396         }
1397
1398         list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1399                                                   obj->size / PAGE_SIZE, 0);
1400         if (!list->file_offset_node) {
1401                 ret = -ENOMEM;
1402                 goto out_free_list;
1403         }
1404
1405         list->hash.key = list->file_offset_node->start;
1406         ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1407         if (ret) {
1408                 DRM_ERROR("failed to add to map hash\n");
1409                 goto out_free_mm;
1410         }
1411
1412         return 0;
1413
1414 out_free_mm:
1415         drm_mm_put_block(list->file_offset_node);
1416 out_free_list:
1417         kfree(list->map);
1418         list->map = NULL;
1419
1420         return ret;
1421 }
1422
1423 /**
1424  * i915_gem_release_mmap - remove physical page mappings
1425  * @obj: obj in question
1426  *
1427  * Preserve the reservation of the mmapping with the DRM core code, but
1428  * relinquish ownership of the pages back to the system.
1429  *
1430  * It is vital that we remove the page mapping if we have mapped a tiled
1431  * object through the GTT and then lose the fence register due to
1432  * resource pressure. Similarly if the object has been moved out of the
1433  * aperture, than pages mapped into userspace must be revoked. Removing the
1434  * mapping will then trigger a page fault on the next user access, allowing
1435  * fixup by i915_gem_fault().
1436  */
1437 void
1438 i915_gem_release_mmap(struct drm_gem_object *obj)
1439 {
1440         struct drm_device *dev = obj->dev;
1441         struct drm_i915_private *dev_priv = dev->dev_private;
1442         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1443
1444         if (unlikely(obj->map_list.map && dev->dev_mapping))
1445                 unmap_mapping_range(dev->dev_mapping,
1446                                     (loff_t)obj->map_list.hash.key<<PAGE_SHIFT,
1447                                     obj->size, 1);
1448
1449         if (obj_priv->fault_mappable) {
1450                 obj_priv->fault_mappable = false;
1451                 i915_gem_info_update_mappable(dev_priv, obj, false);
1452         }
1453 }
1454
1455 static void
1456 i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1457 {
1458         struct drm_device *dev = obj->dev;
1459         struct drm_gem_mm *mm = dev->mm_private;
1460         struct drm_map_list *list = &obj->map_list;
1461
1462         drm_ht_remove_item(&mm->offset_hash, &list->hash);
1463         drm_mm_put_block(list->file_offset_node);
1464         kfree(list->map);
1465         list->map = NULL;
1466 }
1467
1468 /**
1469  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1470  * @obj: object to check
1471  *
1472  * Return the required GTT alignment for an object, taking into account
1473  * potential fence register mapping if needed.
1474  */
1475 static uint32_t
1476 i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1477 {
1478         struct drm_device *dev = obj->dev;
1479         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1480         int start, i;
1481
1482         /*
1483          * Minimum alignment is 4k (GTT page size), but might be greater
1484          * if a fence register is needed for the object.
1485          */
1486         if (INTEL_INFO(dev)->gen >= 4 || obj_priv->tiling_mode == I915_TILING_NONE)
1487                 return 4096;
1488
1489         /*
1490          * Previous chips need to be aligned to the size of the smallest
1491          * fence register that can contain the object.
1492          */
1493         if (INTEL_INFO(dev)->gen == 3)
1494                 start = 1024*1024;
1495         else
1496                 start = 512*1024;
1497
1498         for (i = start; i < obj->size; i <<= 1)
1499                 ;
1500
1501         return i;
1502 }
1503
1504 /**
1505  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1506  * @dev: DRM device
1507  * @data: GTT mapping ioctl data
1508  * @file_priv: GEM object info
1509  *
1510  * Simply returns the fake offset to userspace so it can mmap it.
1511  * The mmap call will end up in drm_gem_mmap(), which will set things
1512  * up so we can get faults in the handler above.
1513  *
1514  * The fault handler will take care of binding the object into the GTT
1515  * (since it may have been evicted to make room for something), allocating
1516  * a fence register, and mapping the appropriate aperture address into
1517  * userspace.
1518  */
1519 int
1520 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1521                         struct drm_file *file_priv)
1522 {
1523         struct drm_i915_private *dev_priv = dev->dev_private;
1524         struct drm_i915_gem_mmap_gtt *args = data;
1525         struct drm_gem_object *obj;
1526         struct drm_i915_gem_object *obj_priv;
1527         int ret;
1528
1529         if (!(dev->driver->driver_features & DRIVER_GEM))
1530                 return -ENODEV;
1531
1532         ret = i915_mutex_lock_interruptible(dev);
1533         if (ret)
1534                 return ret;
1535
1536         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1537         if (obj == NULL) {
1538                 ret = -ENOENT;
1539                 goto unlock;
1540         }
1541         obj_priv = to_intel_bo(obj);
1542
1543         if (obj->size > dev_priv->mm.gtt_mappable_end) {
1544                 ret = -E2BIG;
1545                 goto unlock;
1546         }
1547
1548         if (obj_priv->madv != I915_MADV_WILLNEED) {
1549                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1550                 ret = -EINVAL;
1551                 goto out;
1552         }
1553
1554         if (!obj->map_list.map) {
1555                 ret = i915_gem_create_mmap_offset(obj);
1556                 if (ret)
1557                         goto out;
1558         }
1559
1560         args->offset = (u64)obj->map_list.hash.key << PAGE_SHIFT;
1561
1562 out:
1563         drm_gem_object_unreference(obj);
1564 unlock:
1565         mutex_unlock(&dev->struct_mutex);
1566         return ret;
1567 }
1568
1569 static int
1570 i915_gem_object_get_pages_gtt(struct drm_gem_object *obj,
1571                               gfp_t gfpmask)
1572 {
1573         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1574         int page_count, i;
1575         struct address_space *mapping;
1576         struct inode *inode;
1577         struct page *page;
1578
1579         /* Get the list of pages out of our struct file.  They'll be pinned
1580          * at this point until we release them.
1581          */
1582         page_count = obj->size / PAGE_SIZE;
1583         BUG_ON(obj_priv->pages != NULL);
1584         obj_priv->pages = drm_malloc_ab(page_count, sizeof(struct page *));
1585         if (obj_priv->pages == NULL)
1586                 return -ENOMEM;
1587
1588         inode = obj->filp->f_path.dentry->d_inode;
1589         mapping = inode->i_mapping;
1590         for (i = 0; i < page_count; i++) {
1591                 page = read_cache_page_gfp(mapping, i,
1592                                            GFP_HIGHUSER |
1593                                            __GFP_COLD |
1594                                            __GFP_RECLAIMABLE |
1595                                            gfpmask);
1596                 if (IS_ERR(page))
1597                         goto err_pages;
1598
1599                 obj_priv->pages[i] = page;
1600         }
1601
1602         if (obj_priv->tiling_mode != I915_TILING_NONE)
1603                 i915_gem_object_do_bit_17_swizzle(obj);
1604
1605         return 0;
1606
1607 err_pages:
1608         while (i--)
1609                 page_cache_release(obj_priv->pages[i]);
1610
1611         drm_free_large(obj_priv->pages);
1612         obj_priv->pages = NULL;
1613         return PTR_ERR(page);
1614 }
1615
1616 static void
1617 i915_gem_object_put_pages_gtt(struct drm_gem_object *obj)
1618 {
1619         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1620         int page_count = obj->size / PAGE_SIZE;
1621         int i;
1622
1623         BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1624
1625         if (obj_priv->tiling_mode != I915_TILING_NONE)
1626                 i915_gem_object_save_bit_17_swizzle(obj);
1627
1628         if (obj_priv->madv == I915_MADV_DONTNEED)
1629                 obj_priv->dirty = 0;
1630
1631         for (i = 0; i < page_count; i++) {
1632                 if (obj_priv->dirty)
1633                         set_page_dirty(obj_priv->pages[i]);
1634
1635                 if (obj_priv->madv == I915_MADV_WILLNEED)
1636                         mark_page_accessed(obj_priv->pages[i]);
1637
1638                 page_cache_release(obj_priv->pages[i]);
1639         }
1640         obj_priv->dirty = 0;
1641
1642         drm_free_large(obj_priv->pages);
1643         obj_priv->pages = NULL;
1644 }
1645
1646 static uint32_t
1647 i915_gem_next_request_seqno(struct drm_device *dev,
1648                             struct intel_ring_buffer *ring)
1649 {
1650         drm_i915_private_t *dev_priv = dev->dev_private;
1651
1652         ring->outstanding_lazy_request = true;
1653         return dev_priv->next_seqno;
1654 }
1655
1656 static void
1657 i915_gem_object_move_to_active(struct drm_gem_object *obj,
1658                                struct intel_ring_buffer *ring)
1659 {
1660         struct drm_device *dev = obj->dev;
1661         struct drm_i915_private *dev_priv = dev->dev_private;
1662         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1663         uint32_t seqno = i915_gem_next_request_seqno(dev, ring);
1664
1665         BUG_ON(ring == NULL);
1666         obj_priv->ring = ring;
1667
1668         /* Add a reference if we're newly entering the active list. */
1669         if (!obj_priv->active) {
1670                 drm_gem_object_reference(obj);
1671                 obj_priv->active = 1;
1672         }
1673
1674         /* Move from whatever list we were on to the tail of execution. */
1675         list_move_tail(&obj_priv->mm_list, &dev_priv->mm.active_list);
1676         list_move_tail(&obj_priv->ring_list, &ring->active_list);
1677         obj_priv->last_rendering_seqno = seqno;
1678 }
1679
1680 static void
1681 i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1682 {
1683         struct drm_device *dev = obj->dev;
1684         drm_i915_private_t *dev_priv = dev->dev_private;
1685         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1686
1687         BUG_ON(!obj_priv->active);
1688         list_move_tail(&obj_priv->mm_list, &dev_priv->mm.flushing_list);
1689         list_del_init(&obj_priv->ring_list);
1690         obj_priv->last_rendering_seqno = 0;
1691 }
1692
1693 /* Immediately discard the backing storage */
1694 static void
1695 i915_gem_object_truncate(struct drm_gem_object *obj)
1696 {
1697         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1698         struct inode *inode;
1699
1700         /* Our goal here is to return as much of the memory as
1701          * is possible back to the system as we are called from OOM.
1702          * To do this we must instruct the shmfs to drop all of its
1703          * backing pages, *now*. Here we mirror the actions taken
1704          * when by shmem_delete_inode() to release the backing store.
1705          */
1706         inode = obj->filp->f_path.dentry->d_inode;
1707         truncate_inode_pages(inode->i_mapping, 0);
1708         if (inode->i_op->truncate_range)
1709                 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
1710
1711         obj_priv->madv = __I915_MADV_PURGED;
1712 }
1713
1714 static inline int
1715 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1716 {
1717         return obj_priv->madv == I915_MADV_DONTNEED;
1718 }
1719
1720 static void
1721 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1722 {
1723         struct drm_device *dev = obj->dev;
1724         drm_i915_private_t *dev_priv = dev->dev_private;
1725         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1726
1727         if (obj_priv->pin_count != 0)
1728                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.pinned_list);
1729         else
1730                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1731         list_del_init(&obj_priv->ring_list);
1732
1733         BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1734
1735         obj_priv->last_rendering_seqno = 0;
1736         obj_priv->ring = NULL;
1737         if (obj_priv->active) {
1738                 obj_priv->active = 0;
1739                 drm_gem_object_unreference(obj);
1740         }
1741         WARN_ON(i915_verify_lists(dev));
1742 }
1743
1744 static void
1745 i915_gem_process_flushing_list(struct drm_device *dev,
1746                                uint32_t flush_domains,
1747                                struct intel_ring_buffer *ring)
1748 {
1749         drm_i915_private_t *dev_priv = dev->dev_private;
1750         struct drm_i915_gem_object *obj_priv, *next;
1751
1752         list_for_each_entry_safe(obj_priv, next,
1753                                  &ring->gpu_write_list,
1754                                  gpu_write_list) {
1755                 struct drm_gem_object *obj = &obj_priv->base;
1756
1757                 if (obj->write_domain & flush_domains) {
1758                         uint32_t old_write_domain = obj->write_domain;
1759
1760                         obj->write_domain = 0;
1761                         list_del_init(&obj_priv->gpu_write_list);
1762                         i915_gem_object_move_to_active(obj, ring);
1763
1764                         /* update the fence lru list */
1765                         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1766                                 struct drm_i915_fence_reg *reg =
1767                                         &dev_priv->fence_regs[obj_priv->fence_reg];
1768                                 list_move_tail(&reg->lru_list,
1769                                                 &dev_priv->mm.fence_list);
1770                         }
1771
1772                         trace_i915_gem_object_change_domain(obj,
1773                                                             obj->read_domains,
1774                                                             old_write_domain);
1775                 }
1776         }
1777 }
1778
1779 int
1780 i915_add_request(struct drm_device *dev,
1781                  struct drm_file *file,
1782                  struct drm_i915_gem_request *request,
1783                  struct intel_ring_buffer *ring)
1784 {
1785         drm_i915_private_t *dev_priv = dev->dev_private;
1786         struct drm_i915_file_private *file_priv = NULL;
1787         uint32_t seqno;
1788         int was_empty;
1789         int ret;
1790
1791         BUG_ON(request == NULL);
1792
1793         if (file != NULL)
1794                 file_priv = file->driver_priv;
1795
1796         ret = ring->add_request(ring, &seqno);
1797         if (ret)
1798             return ret;
1799
1800         ring->outstanding_lazy_request = false;
1801
1802         request->seqno = seqno;
1803         request->ring = ring;
1804         request->emitted_jiffies = jiffies;
1805         was_empty = list_empty(&ring->request_list);
1806         list_add_tail(&request->list, &ring->request_list);
1807
1808         if (file_priv) {
1809                 spin_lock(&file_priv->mm.lock);
1810                 request->file_priv = file_priv;
1811                 list_add_tail(&request->client_list,
1812                               &file_priv->mm.request_list);
1813                 spin_unlock(&file_priv->mm.lock);
1814         }
1815
1816         if (!dev_priv->mm.suspended) {
1817                 mod_timer(&dev_priv->hangcheck_timer,
1818                           jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
1819                 if (was_empty)
1820                         queue_delayed_work(dev_priv->wq,
1821                                            &dev_priv->mm.retire_work, HZ);
1822         }
1823         return 0;
1824 }
1825
1826 /**
1827  * Command execution barrier
1828  *
1829  * Ensures that all commands in the ring are finished
1830  * before signalling the CPU
1831  */
1832 static void
1833 i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
1834 {
1835         uint32_t flush_domains = 0;
1836
1837         /* The sampler always gets flushed on i965 (sigh) */
1838         if (INTEL_INFO(dev)->gen >= 4)
1839                 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1840
1841         ring->flush(ring, I915_GEM_DOMAIN_COMMAND, flush_domains);
1842 }
1843
1844 static inline void
1845 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1846 {
1847         struct drm_i915_file_private *file_priv = request->file_priv;
1848
1849         if (!file_priv)
1850                 return;
1851
1852         spin_lock(&file_priv->mm.lock);
1853         list_del(&request->client_list);
1854         request->file_priv = NULL;
1855         spin_unlock(&file_priv->mm.lock);
1856 }
1857
1858 static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1859                                       struct intel_ring_buffer *ring)
1860 {
1861         while (!list_empty(&ring->request_list)) {
1862                 struct drm_i915_gem_request *request;
1863
1864                 request = list_first_entry(&ring->request_list,
1865                                            struct drm_i915_gem_request,
1866                                            list);
1867
1868                 list_del(&request->list);
1869                 i915_gem_request_remove_from_client(request);
1870                 kfree(request);
1871         }
1872
1873         while (!list_empty(&ring->active_list)) {
1874                 struct drm_i915_gem_object *obj_priv;
1875
1876                 obj_priv = list_first_entry(&ring->active_list,
1877                                             struct drm_i915_gem_object,
1878                                             ring_list);
1879
1880                 obj_priv->base.write_domain = 0;
1881                 list_del_init(&obj_priv->gpu_write_list);
1882                 i915_gem_object_move_to_inactive(&obj_priv->base);
1883         }
1884 }
1885
1886 void i915_gem_reset(struct drm_device *dev)
1887 {
1888         struct drm_i915_private *dev_priv = dev->dev_private;
1889         struct drm_i915_gem_object *obj_priv;
1890         int i;
1891
1892         i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring);
1893         i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring);
1894         i915_gem_reset_ring_lists(dev_priv, &dev_priv->blt_ring);
1895
1896         /* Remove anything from the flushing lists. The GPU cache is likely
1897          * to be lost on reset along with the data, so simply move the
1898          * lost bo to the inactive list.
1899          */
1900         while (!list_empty(&dev_priv->mm.flushing_list)) {
1901                 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1902                                             struct drm_i915_gem_object,
1903                                             mm_list);
1904
1905                 obj_priv->base.write_domain = 0;
1906                 list_del_init(&obj_priv->gpu_write_list);
1907                 i915_gem_object_move_to_inactive(&obj_priv->base);
1908         }
1909
1910         /* Move everything out of the GPU domains to ensure we do any
1911          * necessary invalidation upon reuse.
1912          */
1913         list_for_each_entry(obj_priv,
1914                             &dev_priv->mm.inactive_list,
1915                             mm_list)
1916         {
1917                 obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1918         }
1919
1920         /* The fence registers are invalidated so clear them out */
1921         for (i = 0; i < 16; i++) {
1922                 struct drm_i915_fence_reg *reg;
1923
1924                 reg = &dev_priv->fence_regs[i];
1925                 if (!reg->obj)
1926                         continue;
1927
1928                 i915_gem_clear_fence_reg(reg->obj);
1929         }
1930 }
1931
1932 /**
1933  * This function clears the request list as sequence numbers are passed.
1934  */
1935 static void
1936 i915_gem_retire_requests_ring(struct drm_device *dev,
1937                               struct intel_ring_buffer *ring)
1938 {
1939         drm_i915_private_t *dev_priv = dev->dev_private;
1940         uint32_t seqno;
1941
1942         if (!ring->status_page.page_addr ||
1943             list_empty(&ring->request_list))
1944                 return;
1945
1946         WARN_ON(i915_verify_lists(dev));
1947
1948         seqno = ring->get_seqno(ring);
1949         while (!list_empty(&ring->request_list)) {
1950                 struct drm_i915_gem_request *request;
1951
1952                 request = list_first_entry(&ring->request_list,
1953                                            struct drm_i915_gem_request,
1954                                            list);
1955
1956                 if (!i915_seqno_passed(seqno, request->seqno))
1957                         break;
1958
1959                 trace_i915_gem_request_retire(dev, request->seqno);
1960
1961                 list_del(&request->list);
1962                 i915_gem_request_remove_from_client(request);
1963                 kfree(request);
1964         }
1965
1966         /* Move any buffers on the active list that are no longer referenced
1967          * by the ringbuffer to the flushing/inactive lists as appropriate.
1968          */
1969         while (!list_empty(&ring->active_list)) {
1970                 struct drm_gem_object *obj;
1971                 struct drm_i915_gem_object *obj_priv;
1972
1973                 obj_priv = list_first_entry(&ring->active_list,
1974                                             struct drm_i915_gem_object,
1975                                             ring_list);
1976
1977                 if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno))
1978                         break;
1979
1980                 obj = &obj_priv->base;
1981                 if (obj->write_domain != 0)
1982                         i915_gem_object_move_to_flushing(obj);
1983                 else
1984                         i915_gem_object_move_to_inactive(obj);
1985         }
1986
1987         if (unlikely (dev_priv->trace_irq_seqno &&
1988                       i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
1989                 ring->user_irq_put(ring);
1990                 dev_priv->trace_irq_seqno = 0;
1991         }
1992
1993         WARN_ON(i915_verify_lists(dev));
1994 }
1995
1996 void
1997 i915_gem_retire_requests(struct drm_device *dev)
1998 {
1999         drm_i915_private_t *dev_priv = dev->dev_private;
2000
2001         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
2002             struct drm_i915_gem_object *obj_priv, *tmp;
2003
2004             /* We must be careful that during unbind() we do not
2005              * accidentally infinitely recurse into retire requests.
2006              * Currently:
2007              *   retire -> free -> unbind -> wait -> retire_ring
2008              */
2009             list_for_each_entry_safe(obj_priv, tmp,
2010                                      &dev_priv->mm.deferred_free_list,
2011                                      mm_list)
2012                     i915_gem_free_object_tail(&obj_priv->base);
2013         }
2014
2015         i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
2016         i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
2017         i915_gem_retire_requests_ring(dev, &dev_priv->blt_ring);
2018 }
2019
2020 static void
2021 i915_gem_retire_work_handler(struct work_struct *work)
2022 {
2023         drm_i915_private_t *dev_priv;
2024         struct drm_device *dev;
2025
2026         dev_priv = container_of(work, drm_i915_private_t,
2027                                 mm.retire_work.work);
2028         dev = dev_priv->dev;
2029
2030         /* Come back later if the device is busy... */
2031         if (!mutex_trylock(&dev->struct_mutex)) {
2032                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
2033                 return;
2034         }
2035
2036         i915_gem_retire_requests(dev);
2037
2038         if (!dev_priv->mm.suspended &&
2039                 (!list_empty(&dev_priv->render_ring.request_list) ||
2040                  !list_empty(&dev_priv->bsd_ring.request_list) ||
2041                  !list_empty(&dev_priv->blt_ring.request_list)))
2042                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
2043         mutex_unlock(&dev->struct_mutex);
2044 }
2045
2046 int
2047 i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
2048                      bool interruptible, struct intel_ring_buffer *ring)
2049 {
2050         drm_i915_private_t *dev_priv = dev->dev_private;
2051         u32 ier;
2052         int ret = 0;
2053
2054         BUG_ON(seqno == 0);
2055
2056         if (atomic_read(&dev_priv->mm.wedged))
2057                 return -EAGAIN;
2058
2059         if (ring->outstanding_lazy_request) {
2060                 struct drm_i915_gem_request *request;
2061
2062                 request = kzalloc(sizeof(*request), GFP_KERNEL);
2063                 if (request == NULL)
2064                         return -ENOMEM;
2065
2066                 ret = i915_add_request(dev, NULL, request, ring);
2067                 if (ret) {
2068                         kfree(request);
2069                         return ret;
2070                 }
2071
2072                 seqno = request->seqno;
2073         }
2074         BUG_ON(seqno == dev_priv->next_seqno);
2075
2076         if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
2077                 if (HAS_PCH_SPLIT(dev))
2078                         ier = I915_READ(DEIER) | I915_READ(GTIER);
2079                 else
2080                         ier = I915_READ(IER);
2081                 if (!ier) {
2082                         DRM_ERROR("something (likely vbetool) disabled "
2083                                   "interrupts, re-enabling\n");
2084                         i915_driver_irq_preinstall(dev);
2085                         i915_driver_irq_postinstall(dev);
2086                 }
2087
2088                 trace_i915_gem_request_wait_begin(dev, seqno);
2089
2090                 ring->waiting_seqno = seqno;
2091                 ring->user_irq_get(ring);
2092                 if (interruptible)
2093                         ret = wait_event_interruptible(ring->irq_queue,
2094                                 i915_seqno_passed(ring->get_seqno(ring), seqno)
2095                                 || atomic_read(&dev_priv->mm.wedged));
2096                 else
2097                         wait_event(ring->irq_queue,
2098                                 i915_seqno_passed(ring->get_seqno(ring), seqno)
2099                                 || atomic_read(&dev_priv->mm.wedged));
2100
2101                 ring->user_irq_put(ring);
2102                 ring->waiting_seqno = 0;
2103
2104                 trace_i915_gem_request_wait_end(dev, seqno);
2105         }
2106         if (atomic_read(&dev_priv->mm.wedged))
2107                 ret = -EAGAIN;
2108
2109         if (ret && ret != -ERESTARTSYS)
2110                 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
2111                           __func__, ret, seqno, ring->get_seqno(ring),
2112                           dev_priv->next_seqno);
2113
2114         /* Directly dispatch request retiring.  While we have the work queue
2115          * to handle this, the waiter on a request often wants an associated
2116          * buffer to have made it to the inactive list, and we would need
2117          * a separate wait queue to handle that.
2118          */
2119         if (ret == 0)
2120                 i915_gem_retire_requests_ring(dev, ring);
2121
2122         return ret;
2123 }
2124
2125 /**
2126  * Waits for a sequence number to be signaled, and cleans up the
2127  * request and object lists appropriately for that event.
2128  */
2129 static int
2130 i915_wait_request(struct drm_device *dev, uint32_t seqno,
2131                   struct intel_ring_buffer *ring)
2132 {
2133         return i915_do_wait_request(dev, seqno, 1, ring);
2134 }
2135
2136 static void
2137 i915_gem_flush_ring(struct drm_device *dev,
2138                     struct drm_file *file_priv,
2139                     struct intel_ring_buffer *ring,
2140                     uint32_t invalidate_domains,
2141                     uint32_t flush_domains)
2142 {
2143         ring->flush(ring, invalidate_domains, flush_domains);
2144         i915_gem_process_flushing_list(dev, flush_domains, ring);
2145 }
2146
2147 static void
2148 i915_gem_flush(struct drm_device *dev,
2149                struct drm_file *file_priv,
2150                uint32_t invalidate_domains,
2151                uint32_t flush_domains,
2152                uint32_t flush_rings)
2153 {
2154         drm_i915_private_t *dev_priv = dev->dev_private;
2155
2156         if (flush_domains & I915_GEM_DOMAIN_CPU)
2157                 drm_agp_chipset_flush(dev);
2158
2159         if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
2160                 if (flush_rings & RING_RENDER)
2161                         i915_gem_flush_ring(dev, file_priv,
2162                                             &dev_priv->render_ring,
2163                                             invalidate_domains, flush_domains);
2164                 if (flush_rings & RING_BSD)
2165                         i915_gem_flush_ring(dev, file_priv,
2166                                             &dev_priv->bsd_ring,
2167                                             invalidate_domains, flush_domains);
2168                 if (flush_rings & RING_BLT)
2169                         i915_gem_flush_ring(dev, file_priv,
2170                                             &dev_priv->blt_ring,
2171                                             invalidate_domains, flush_domains);
2172         }
2173 }
2174
2175 /**
2176  * Ensures that all rendering to the object has completed and the object is
2177  * safe to unbind from the GTT or access from the CPU.
2178  */
2179 static int
2180 i915_gem_object_wait_rendering(struct drm_gem_object *obj,
2181                                bool interruptible)
2182 {
2183         struct drm_device *dev = obj->dev;
2184         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2185         int ret;
2186
2187         /* This function only exists to support waiting for existing rendering,
2188          * not for emitting required flushes.
2189          */
2190         BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
2191
2192         /* If there is rendering queued on the buffer being evicted, wait for
2193          * it.
2194          */
2195         if (obj_priv->active) {
2196                 ret = i915_do_wait_request(dev,
2197                                            obj_priv->last_rendering_seqno,
2198                                            interruptible,
2199                                            obj_priv->ring);
2200                 if (ret)
2201                         return ret;
2202         }
2203
2204         return 0;
2205 }
2206
2207 /**
2208  * Unbinds an object from the GTT aperture.
2209  */
2210 int
2211 i915_gem_object_unbind(struct drm_gem_object *obj)
2212 {
2213         struct drm_device *dev = obj->dev;
2214         struct drm_i915_private *dev_priv = dev->dev_private;
2215         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2216         int ret = 0;
2217
2218         if (obj_priv->gtt_space == NULL)
2219                 return 0;
2220
2221         if (obj_priv->pin_count != 0) {
2222                 DRM_ERROR("Attempting to unbind pinned buffer\n");
2223                 return -EINVAL;
2224         }
2225
2226         /* blow away mappings if mapped through GTT */
2227         i915_gem_release_mmap(obj);
2228
2229         /* Move the object to the CPU domain to ensure that
2230          * any possible CPU writes while it's not in the GTT
2231          * are flushed when we go to remap it. This will
2232          * also ensure that all pending GPU writes are finished
2233          * before we unbind.
2234          */
2235         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2236         if (ret == -ERESTARTSYS)
2237                 return ret;
2238         /* Continue on if we fail due to EIO, the GPU is hung so we
2239          * should be safe and we need to cleanup or else we might
2240          * cause memory corruption through use-after-free.
2241          */
2242         if (ret) {
2243                 i915_gem_clflush_object(obj);
2244                 obj->read_domains = obj->write_domain = I915_GEM_DOMAIN_CPU;
2245         }
2246
2247         /* release the fence reg _after_ flushing */
2248         if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2249                 i915_gem_clear_fence_reg(obj);
2250
2251         drm_unbind_agp(obj_priv->agp_mem);
2252         drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2253
2254         i915_gem_object_put_pages_gtt(obj);
2255
2256         i915_gem_info_remove_gtt(dev_priv, obj);
2257         list_del_init(&obj_priv->mm_list);
2258
2259         drm_mm_put_block(obj_priv->gtt_space);
2260         obj_priv->gtt_space = NULL;
2261         obj_priv->gtt_offset = 0;
2262
2263         if (i915_gem_object_is_purgeable(obj_priv))
2264                 i915_gem_object_truncate(obj);
2265
2266         trace_i915_gem_object_unbind(obj);
2267
2268         return ret;
2269 }
2270
2271 static int i915_ring_idle(struct drm_device *dev,
2272                           struct intel_ring_buffer *ring)
2273 {
2274         if (list_empty(&ring->gpu_write_list))
2275                 return 0;
2276
2277         i915_gem_flush_ring(dev, NULL, ring,
2278                             I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2279         return i915_wait_request(dev,
2280                                  i915_gem_next_request_seqno(dev, ring),
2281                                  ring);
2282 }
2283
2284 int
2285 i915_gpu_idle(struct drm_device *dev)
2286 {
2287         drm_i915_private_t *dev_priv = dev->dev_private;
2288         bool lists_empty;
2289         int ret;
2290
2291         lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2292                        list_empty(&dev_priv->render_ring.active_list) &&
2293                        list_empty(&dev_priv->bsd_ring.active_list) &&
2294                        list_empty(&dev_priv->blt_ring.active_list));
2295         if (lists_empty)
2296                 return 0;
2297
2298         /* Flush everything onto the inactive list. */
2299         ret = i915_ring_idle(dev, &dev_priv->render_ring);
2300         if (ret)
2301                 return ret;
2302
2303         ret = i915_ring_idle(dev, &dev_priv->bsd_ring);
2304         if (ret)
2305                 return ret;
2306
2307         ret = i915_ring_idle(dev, &dev_priv->blt_ring);
2308         if (ret)
2309                 return ret;
2310
2311         return 0;
2312 }
2313
2314 static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2315 {
2316         struct drm_gem_object *obj = reg->obj;
2317         struct drm_device *dev = obj->dev;
2318         drm_i915_private_t *dev_priv = dev->dev_private;
2319         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2320         int regnum = obj_priv->fence_reg;
2321         uint64_t val;
2322
2323         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2324                     0xfffff000) << 32;
2325         val |= obj_priv->gtt_offset & 0xfffff000;
2326         val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2327                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2328
2329         if (obj_priv->tiling_mode == I915_TILING_Y)
2330                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2331         val |= I965_FENCE_REG_VALID;
2332
2333         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2334 }
2335
2336 static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2337 {
2338         struct drm_gem_object *obj = reg->obj;
2339         struct drm_device *dev = obj->dev;
2340         drm_i915_private_t *dev_priv = dev->dev_private;
2341         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2342         int regnum = obj_priv->fence_reg;
2343         uint64_t val;
2344
2345         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2346                     0xfffff000) << 32;
2347         val |= obj_priv->gtt_offset & 0xfffff000;
2348         val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2349         if (obj_priv->tiling_mode == I915_TILING_Y)
2350                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2351         val |= I965_FENCE_REG_VALID;
2352
2353         I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2354 }
2355
2356 static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2357 {
2358         struct drm_gem_object *obj = reg->obj;
2359         struct drm_device *dev = obj->dev;
2360         drm_i915_private_t *dev_priv = dev->dev_private;
2361         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2362         int regnum = obj_priv->fence_reg;
2363         int tile_width;
2364         uint32_t fence_reg, val;
2365         uint32_t pitch_val;
2366
2367         if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2368             (obj_priv->gtt_offset & (obj->size - 1))) {
2369                 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
2370                      __func__, obj_priv->gtt_offset, obj->size);
2371                 return;
2372         }
2373
2374         if (obj_priv->tiling_mode == I915_TILING_Y &&
2375             HAS_128_BYTE_Y_TILING(dev))
2376                 tile_width = 128;
2377         else
2378                 tile_width = 512;
2379
2380         /* Note: pitch better be a power of two tile widths */
2381         pitch_val = obj_priv->stride / tile_width;
2382         pitch_val = ffs(pitch_val) - 1;
2383
2384         if (obj_priv->tiling_mode == I915_TILING_Y &&
2385             HAS_128_BYTE_Y_TILING(dev))
2386                 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2387         else
2388                 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2389
2390         val = obj_priv->gtt_offset;
2391         if (obj_priv->tiling_mode == I915_TILING_Y)
2392                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2393         val |= I915_FENCE_SIZE_BITS(obj->size);
2394         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2395         val |= I830_FENCE_REG_VALID;
2396
2397         if (regnum < 8)
2398                 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2399         else
2400                 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2401         I915_WRITE(fence_reg, val);
2402 }
2403
2404 static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2405 {
2406         struct drm_gem_object *obj = reg->obj;
2407         struct drm_device *dev = obj->dev;
2408         drm_i915_private_t *dev_priv = dev->dev_private;
2409         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2410         int regnum = obj_priv->fence_reg;
2411         uint32_t val;
2412         uint32_t pitch_val;
2413         uint32_t fence_size_bits;
2414
2415         if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
2416             (obj_priv->gtt_offset & (obj->size - 1))) {
2417                 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
2418                      __func__, obj_priv->gtt_offset);
2419                 return;
2420         }
2421
2422         pitch_val = obj_priv->stride / 128;
2423         pitch_val = ffs(pitch_val) - 1;
2424         WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2425
2426         val = obj_priv->gtt_offset;
2427         if (obj_priv->tiling_mode == I915_TILING_Y)
2428                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2429         fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2430         WARN_ON(fence_size_bits & ~0x00000f00);
2431         val |= fence_size_bits;
2432         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2433         val |= I830_FENCE_REG_VALID;
2434
2435         I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2436 }
2437
2438 static int i915_find_fence_reg(struct drm_device *dev,
2439                                bool interruptible)
2440 {
2441         struct drm_i915_fence_reg *reg = NULL;
2442         struct drm_i915_gem_object *obj_priv = NULL;
2443         struct drm_i915_private *dev_priv = dev->dev_private;
2444         struct drm_gem_object *obj = NULL;
2445         int i, avail, ret;
2446
2447         /* First try to find a free reg */
2448         avail = 0;
2449         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2450                 reg = &dev_priv->fence_regs[i];
2451                 if (!reg->obj)
2452                         return i;
2453
2454                 obj_priv = to_intel_bo(reg->obj);
2455                 if (!obj_priv->pin_count)
2456                     avail++;
2457         }
2458
2459         if (avail == 0)
2460                 return -ENOSPC;
2461
2462         /* None available, try to steal one or wait for a user to finish */
2463         i = I915_FENCE_REG_NONE;
2464         list_for_each_entry(reg, &dev_priv->mm.fence_list,
2465                             lru_list) {
2466                 obj = reg->obj;
2467                 obj_priv = to_intel_bo(obj);
2468
2469                 if (obj_priv->pin_count)
2470                         continue;
2471
2472                 /* found one! */
2473                 i = obj_priv->fence_reg;
2474                 break;
2475         }
2476
2477         BUG_ON(i == I915_FENCE_REG_NONE);
2478
2479         /* We only have a reference on obj from the active list. put_fence_reg
2480          * might drop that one, causing a use-after-free in it. So hold a
2481          * private reference to obj like the other callers of put_fence_reg
2482          * (set_tiling ioctl) do. */
2483         drm_gem_object_reference(obj);
2484         ret = i915_gem_object_put_fence_reg(obj, interruptible);
2485         drm_gem_object_unreference(obj);
2486         if (ret != 0)
2487                 return ret;
2488
2489         return i;
2490 }
2491
2492 /**
2493  * i915_gem_object_get_fence_reg - set up a fence reg for an object
2494  * @obj: object to map through a fence reg
2495  *
2496  * When mapping objects through the GTT, userspace wants to be able to write
2497  * to them without having to worry about swizzling if the object is tiled.
2498  *
2499  * This function walks the fence regs looking for a free one for @obj,
2500  * stealing one if it can't find any.
2501  *
2502  * It then sets up the reg based on the object's properties: address, pitch
2503  * and tiling format.
2504  */
2505 int
2506 i915_gem_object_get_fence_reg(struct drm_gem_object *obj,
2507                               bool interruptible)
2508 {
2509         struct drm_device *dev = obj->dev;
2510         struct drm_i915_private *dev_priv = dev->dev_private;
2511         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2512         struct drm_i915_fence_reg *reg = NULL;
2513         int ret;
2514
2515         /* Just update our place in the LRU if our fence is getting used. */
2516         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2517                 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2518                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2519                 return 0;
2520         }
2521
2522         switch (obj_priv->tiling_mode) {
2523         case I915_TILING_NONE:
2524                 WARN(1, "allocating a fence for non-tiled object?\n");
2525                 break;
2526         case I915_TILING_X:
2527                 if (!obj_priv->stride)
2528                         return -EINVAL;
2529                 WARN((obj_priv->stride & (512 - 1)),
2530                      "object 0x%08x is X tiled but has non-512B pitch\n",
2531                      obj_priv->gtt_offset);
2532                 break;
2533         case I915_TILING_Y:
2534                 if (!obj_priv->stride)
2535                         return -EINVAL;
2536                 WARN((obj_priv->stride & (128 - 1)),
2537                      "object 0x%08x is Y tiled but has non-128B pitch\n",
2538                      obj_priv->gtt_offset);
2539                 break;
2540         }
2541
2542         ret = i915_find_fence_reg(dev, interruptible);
2543         if (ret < 0)
2544                 return ret;
2545
2546         obj_priv->fence_reg = ret;
2547         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2548         list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2549
2550         reg->obj = obj;
2551
2552         switch (INTEL_INFO(dev)->gen) {
2553         case 6:
2554                 sandybridge_write_fence_reg(reg);
2555                 break;
2556         case 5:
2557         case 4:
2558                 i965_write_fence_reg(reg);
2559                 break;
2560         case 3:
2561                 i915_write_fence_reg(reg);
2562                 break;
2563         case 2:
2564                 i830_write_fence_reg(reg);
2565                 break;
2566         }
2567
2568         trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
2569                         obj_priv->tiling_mode);
2570
2571         return 0;
2572 }
2573
2574 /**
2575  * i915_gem_clear_fence_reg - clear out fence register info
2576  * @obj: object to clear
2577  *
2578  * Zeroes out the fence register itself and clears out the associated
2579  * data structures in dev_priv and obj_priv.
2580  */
2581 static void
2582 i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2583 {
2584         struct drm_device *dev = obj->dev;
2585         drm_i915_private_t *dev_priv = dev->dev_private;
2586         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2587         struct drm_i915_fence_reg *reg =
2588                 &dev_priv->fence_regs[obj_priv->fence_reg];
2589         uint32_t fence_reg;
2590
2591         switch (INTEL_INFO(dev)->gen) {
2592         case 6:
2593                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2594                              (obj_priv->fence_reg * 8), 0);
2595                 break;
2596         case 5:
2597         case 4:
2598                 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
2599                 break;
2600         case 3:
2601                 if (obj_priv->fence_reg >= 8)
2602                         fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4;
2603                 else
2604         case 2:
2605                         fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2606
2607                 I915_WRITE(fence_reg, 0);
2608                 break;
2609         }
2610
2611         reg->obj = NULL;
2612         obj_priv->fence_reg = I915_FENCE_REG_NONE;
2613         list_del_init(&reg->lru_list);
2614 }
2615
2616 /**
2617  * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2618  * to the buffer to finish, and then resets the fence register.
2619  * @obj: tiled object holding a fence register.
2620  * @bool: whether the wait upon the fence is interruptible
2621  *
2622  * Zeroes out the fence register itself and clears out the associated
2623  * data structures in dev_priv and obj_priv.
2624  */
2625 int
2626 i915_gem_object_put_fence_reg(struct drm_gem_object *obj,
2627                               bool interruptible)
2628 {
2629         struct drm_device *dev = obj->dev;
2630         struct drm_i915_private *dev_priv = dev->dev_private;
2631         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2632         struct drm_i915_fence_reg *reg;
2633
2634         if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2635                 return 0;
2636
2637         /* If we've changed tiling, GTT-mappings of the object
2638          * need to re-fault to ensure that the correct fence register
2639          * setup is in place.
2640          */
2641         i915_gem_release_mmap(obj);
2642
2643         /* On the i915, GPU access to tiled buffers is via a fence,
2644          * therefore we must wait for any outstanding access to complete
2645          * before clearing the fence.
2646          */
2647         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2648         if (reg->gpu) {
2649                 int ret;
2650
2651                 ret = i915_gem_object_flush_gpu_write_domain(obj, true);
2652                 if (ret)
2653                         return ret;
2654
2655                 ret = i915_gem_object_wait_rendering(obj, interruptible);
2656                 if (ret)
2657                         return ret;
2658
2659                 reg->gpu = false;
2660         }
2661
2662         i915_gem_object_flush_gtt_write_domain(obj);
2663         i915_gem_clear_fence_reg(obj);
2664
2665         return 0;
2666 }
2667
2668 /**
2669  * Finds free space in the GTT aperture and binds the object there.
2670  */
2671 static int
2672 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
2673                             unsigned alignment,
2674                             bool mappable)
2675 {
2676         struct drm_device *dev = obj->dev;
2677         drm_i915_private_t *dev_priv = dev->dev_private;
2678         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2679         struct drm_mm_node *free_space;
2680         gfp_t gfpmask =  __GFP_NORETRY | __GFP_NOWARN;
2681         int ret;
2682
2683         if (obj_priv->madv != I915_MADV_WILLNEED) {
2684                 DRM_ERROR("Attempting to bind a purgeable object\n");
2685                 return -EINVAL;
2686         }
2687
2688         if (alignment == 0)
2689                 alignment = i915_gem_get_gtt_alignment(obj);
2690         if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
2691                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2692                 return -EINVAL;
2693         }
2694
2695         /* If the object is bigger than the entire aperture, reject it early
2696          * before evicting everything in a vain attempt to find space.
2697          */
2698         if (obj->size >
2699             (mappable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
2700                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2701                 return -E2BIG;
2702         }
2703
2704  search_free:
2705         if (mappable)
2706                 free_space =
2707                         drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
2708                                                     obj->size, alignment, 0,
2709                                                     dev_priv->mm.gtt_mappable_end,
2710                                                     0);
2711         else
2712                 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2713                                                 obj->size, alignment, 0);
2714
2715         if (free_space != NULL) {
2716                 if (mappable)
2717                         obj_priv->gtt_space =
2718                                 drm_mm_get_block_range_generic(free_space,
2719                                                                obj->size,
2720                                                                alignment, 0,
2721                                                                dev_priv->mm.gtt_mappable_end,
2722                                                                0);
2723                 else
2724                         obj_priv->gtt_space =
2725                                 drm_mm_get_block(free_space, obj->size,
2726                                                  alignment);
2727         }
2728         if (obj_priv->gtt_space == NULL) {
2729                 /* If the gtt is empty and we're still having trouble
2730                  * fitting our object in, we're out of memory.
2731                  */
2732                 ret = i915_gem_evict_something(dev, obj->size, alignment,
2733                                                mappable);
2734                 if (ret)
2735                         return ret;
2736
2737                 goto search_free;
2738         }
2739
2740         ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
2741         if (ret) {
2742                 drm_mm_put_block(obj_priv->gtt_space);
2743                 obj_priv->gtt_space = NULL;
2744
2745                 if (ret == -ENOMEM) {
2746                         /* first try to clear up some space from the GTT */
2747                         ret = i915_gem_evict_something(dev, obj->size,
2748                                                        alignment, mappable);
2749                         if (ret) {
2750                                 /* now try to shrink everyone else */
2751                                 if (gfpmask) {
2752                                         gfpmask = 0;
2753                                         goto search_free;
2754                                 }
2755
2756                                 return ret;
2757                         }
2758
2759                         goto search_free;
2760                 }
2761
2762                 return ret;
2763         }
2764
2765         /* Create an AGP memory structure pointing at our pages, and bind it
2766          * into the GTT.
2767          */
2768         obj_priv->agp_mem = drm_agp_bind_pages(dev,
2769                                                obj_priv->pages,
2770                                                obj->size >> PAGE_SHIFT,
2771                                                obj_priv->gtt_space->start,
2772                                                obj_priv->agp_type);
2773         if (obj_priv->agp_mem == NULL) {
2774                 i915_gem_object_put_pages_gtt(obj);
2775                 drm_mm_put_block(obj_priv->gtt_space);
2776                 obj_priv->gtt_space = NULL;
2777
2778                 ret = i915_gem_evict_something(dev, obj->size, alignment,
2779                                                mappable);
2780                 if (ret)
2781                         return ret;
2782
2783                 goto search_free;
2784         }
2785
2786         obj_priv->gtt_offset = obj_priv->gtt_space->start;
2787
2788         /* keep track of bounds object by adding it to the inactive list */
2789         list_add_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
2790         i915_gem_info_add_gtt(dev_priv, obj);
2791
2792         /* Assert that the object is not currently in any GPU domain. As it
2793          * wasn't in the GTT, there shouldn't be any way it could have been in
2794          * a GPU cache
2795          */
2796         BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2797         BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2798
2799         trace_i915_gem_object_bind(obj, obj_priv->gtt_offset, mappable);
2800
2801         return 0;
2802 }
2803
2804 void
2805 i915_gem_clflush_object(struct drm_gem_object *obj)
2806 {
2807         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
2808
2809         /* If we don't have a page list set up, then we're not pinned
2810          * to GPU, and we can ignore the cache flush because it'll happen
2811          * again at bind time.
2812          */
2813         if (obj_priv->pages == NULL)
2814                 return;
2815
2816         trace_i915_gem_object_clflush(obj);
2817
2818         drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
2819 }
2820
2821 /** Flushes any GPU write domain for the object if it's dirty. */
2822 static int
2823 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
2824                                        bool pipelined)
2825 {
2826         struct drm_device *dev = obj->dev;
2827         uint32_t old_write_domain;
2828
2829         if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2830                 return 0;
2831
2832         /* Queue the GPU write cache flushing we need. */
2833         old_write_domain = obj->write_domain;
2834         i915_gem_flush_ring(dev, NULL,
2835                             to_intel_bo(obj)->ring,
2836                             0, obj->write_domain);
2837         BUG_ON(obj->write_domain);
2838
2839         trace_i915_gem_object_change_domain(obj,
2840                                             obj->read_domains,
2841                                             old_write_domain);
2842
2843         if (pipelined)
2844                 return 0;
2845
2846         return i915_gem_object_wait_rendering(obj, true);
2847 }
2848
2849 /** Flushes the GTT write domain for the object if it's dirty. */
2850 static void
2851 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2852 {
2853         uint32_t old_write_domain;
2854
2855         if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2856                 return;
2857
2858         /* No actual flushing is required for the GTT write domain.   Writes
2859          * to it immediately go to main memory as far as we know, so there's
2860          * no chipset flush.  It also doesn't land in render cache.
2861          */
2862         i915_gem_release_mmap(obj);
2863
2864         old_write_domain = obj->write_domain;
2865         obj->write_domain = 0;
2866
2867         trace_i915_gem_object_change_domain(obj,
2868                                             obj->read_domains,
2869                                             old_write_domain);
2870 }
2871
2872 /** Flushes the CPU write domain for the object if it's dirty. */
2873 static void
2874 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2875 {
2876         struct drm_device *dev = obj->dev;
2877         uint32_t old_write_domain;
2878
2879         if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2880                 return;
2881
2882         i915_gem_clflush_object(obj);
2883         drm_agp_chipset_flush(dev);
2884         old_write_domain = obj->write_domain;
2885         obj->write_domain = 0;
2886
2887         trace_i915_gem_object_change_domain(obj,
2888                                             obj->read_domains,
2889                                             old_write_domain);
2890 }
2891
2892 /**
2893  * Moves a single object to the GTT read, and possibly write domain.
2894  *
2895  * This function returns when the move is complete, including waiting on
2896  * flushes to occur.
2897  */
2898 int
2899 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2900 {
2901         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2902         uint32_t old_write_domain, old_read_domains;
2903         int ret;
2904
2905         /* Not valid to be called on unbound objects. */
2906         if (obj_priv->gtt_space == NULL)
2907                 return -EINVAL;
2908
2909         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
2910         if (ret != 0)
2911                 return ret;
2912
2913         i915_gem_object_flush_cpu_write_domain(obj);
2914
2915         if (write) {
2916                 ret = i915_gem_object_wait_rendering(obj, true);
2917                 if (ret)
2918                         return ret;
2919         }
2920
2921         old_write_domain = obj->write_domain;
2922         old_read_domains = obj->read_domains;
2923
2924         /* It should now be out of any other write domains, and we can update
2925          * the domain values for our changes.
2926          */
2927         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2928         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2929         if (write) {
2930                 obj->read_domains = I915_GEM_DOMAIN_GTT;
2931                 obj->write_domain = I915_GEM_DOMAIN_GTT;
2932                 obj_priv->dirty = 1;
2933         }
2934
2935         trace_i915_gem_object_change_domain(obj,
2936                                             old_read_domains,
2937                                             old_write_domain);
2938
2939         return 0;
2940 }
2941
2942 /*
2943  * Prepare buffer for display plane. Use uninterruptible for possible flush
2944  * wait, as in modesetting process we're not supposed to be interrupted.
2945  */
2946 int
2947 i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
2948                                      bool pipelined)
2949 {
2950         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2951         uint32_t old_read_domains;
2952         int ret;
2953
2954         /* Not valid to be called on unbound objects. */
2955         if (obj_priv->gtt_space == NULL)
2956                 return -EINVAL;
2957
2958         ret = i915_gem_object_flush_gpu_write_domain(obj, true);
2959         if (ret)
2960                 return ret;
2961
2962         /* Currently, we are always called from an non-interruptible context. */
2963         if (!pipelined) {
2964                 ret = i915_gem_object_wait_rendering(obj, false);
2965                 if (ret)
2966                         return ret;
2967         }
2968
2969         i915_gem_object_flush_cpu_write_domain(obj);
2970
2971         old_read_domains = obj->read_domains;
2972         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2973
2974         trace_i915_gem_object_change_domain(obj,
2975                                             old_read_domains,
2976                                             obj->write_domain);
2977
2978         return 0;
2979 }
2980
2981 /**
2982  * Moves a single object to the CPU read, and possibly write domain.
2983  *
2984  * This function returns when the move is complete, including waiting on
2985  * flushes to occur.
2986  */
2987 static int
2988 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2989 {
2990         uint32_t old_write_domain, old_read_domains;
2991         int ret;
2992
2993         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
2994         if (ret != 0)
2995                 return ret;
2996
2997         i915_gem_object_flush_gtt_write_domain(obj);
2998
2999         /* If we have a partially-valid cache of the object in the CPU,
3000          * finish invalidating it and free the per-page flags.
3001          */
3002         i915_gem_object_set_to_full_cpu_read_domain(obj);
3003
3004         if (write) {
3005                 ret = i915_gem_object_wait_rendering(obj, true);
3006                 if (ret)
3007                         return ret;
3008         }
3009
3010         old_write_domain = obj->write_domain;
3011         old_read_domains = obj->read_domains;
3012
3013         /* Flush the CPU cache if it's still invalid. */
3014         if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3015                 i915_gem_clflush_object(obj);
3016
3017                 obj->read_domains |= I915_GEM_DOMAIN_CPU;
3018         }
3019
3020         /* It should now be out of any other write domains, and we can update
3021          * the domain values for our changes.
3022          */
3023         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3024
3025         /* If we're writing through the CPU, then the GPU read domains will
3026          * need to be invalidated at next use.
3027          */
3028         if (write) {
3029                 obj->read_domains = I915_GEM_DOMAIN_CPU;
3030                 obj->write_domain = I915_GEM_DOMAIN_CPU;
3031         }
3032
3033         trace_i915_gem_object_change_domain(obj,
3034                                             old_read_domains,
3035                                             old_write_domain);
3036
3037         return 0;
3038 }
3039
3040 /*
3041  * Set the next domain for the specified object. This
3042  * may not actually perform the necessary flushing/invaliding though,
3043  * as that may want to be batched with other set_domain operations
3044  *
3045  * This is (we hope) the only really tricky part of gem. The goal
3046  * is fairly simple -- track which caches hold bits of the object
3047  * and make sure they remain coherent. A few concrete examples may
3048  * help to explain how it works. For shorthand, we use the notation
3049  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
3050  * a pair of read and write domain masks.
3051  *
3052  * Case 1: the batch buffer
3053  *
3054  *      1. Allocated
3055  *      2. Written by CPU
3056  *      3. Mapped to GTT
3057  *      4. Read by GPU
3058  *      5. Unmapped from GTT
3059  *      6. Freed
3060  *
3061  *      Let's take these a step at a time
3062  *
3063  *      1. Allocated
3064  *              Pages allocated from the kernel may still have
3065  *              cache contents, so we set them to (CPU, CPU) always.
3066  *      2. Written by CPU (using pwrite)
3067  *              The pwrite function calls set_domain (CPU, CPU) and
3068  *              this function does nothing (as nothing changes)
3069  *      3. Mapped by GTT
3070  *              This function asserts that the object is not
3071  *              currently in any GPU-based read or write domains
3072  *      4. Read by GPU
3073  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
3074  *              As write_domain is zero, this function adds in the
3075  *              current read domains (CPU+COMMAND, 0).
3076  *              flush_domains is set to CPU.
3077  *              invalidate_domains is set to COMMAND
3078  *              clflush is run to get data out of the CPU caches
3079  *              then i915_dev_set_domain calls i915_gem_flush to
3080  *              emit an MI_FLUSH and drm_agp_chipset_flush
3081  *      5. Unmapped from GTT
3082  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
3083  *              flush_domains and invalidate_domains end up both zero
3084  *              so no flushing/invalidating happens
3085  *      6. Freed
3086  *              yay, done
3087  *
3088  * Case 2: The shared render buffer
3089  *
3090  *      1. Allocated
3091  *      2. Mapped to GTT
3092  *      3. Read/written by GPU
3093  *      4. set_domain to (CPU,CPU)
3094  *      5. Read/written by CPU
3095  *      6. Read/written by GPU
3096  *
3097  *      1. Allocated
3098  *              Same as last example, (CPU, CPU)
3099  *      2. Mapped to GTT
3100  *              Nothing changes (assertions find that it is not in the GPU)
3101  *      3. Read/written by GPU
3102  *              execbuffer calls set_domain (RENDER, RENDER)
3103  *              flush_domains gets CPU
3104  *              invalidate_domains gets GPU
3105  *              clflush (obj)
3106  *              MI_FLUSH and drm_agp_chipset_flush
3107  *      4. set_domain (CPU, CPU)
3108  *              flush_domains gets GPU
3109  *              invalidate_domains gets CPU
3110  *              wait_rendering (obj) to make sure all drawing is complete.
3111  *              This will include an MI_FLUSH to get the data from GPU
3112  *              to memory
3113  *              clflush (obj) to invalidate the CPU cache
3114  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3115  *      5. Read/written by CPU
3116  *              cache lines are loaded and dirtied
3117  *      6. Read written by GPU
3118  *              Same as last GPU access
3119  *
3120  * Case 3: The constant buffer
3121  *
3122  *      1. Allocated
3123  *      2. Written by CPU
3124  *      3. Read by GPU
3125  *      4. Updated (written) by CPU again
3126  *      5. Read by GPU
3127  *
3128  *      1. Allocated
3129  *              (CPU, CPU)
3130  *      2. Written by CPU
3131  *              (CPU, CPU)
3132  *      3. Read by GPU
3133  *              (CPU+RENDER, 0)
3134  *              flush_domains = CPU
3135  *              invalidate_domains = RENDER
3136  *              clflush (obj)
3137  *              MI_FLUSH
3138  *              drm_agp_chipset_flush
3139  *      4. Updated (written) by CPU again
3140  *              (CPU, CPU)
3141  *              flush_domains = 0 (no previous write domain)
3142  *              invalidate_domains = 0 (no new read domains)
3143  *      5. Read by GPU
3144  *              (CPU+RENDER, 0)
3145  *              flush_domains = CPU
3146  *              invalidate_domains = RENDER
3147  *              clflush (obj)
3148  *              MI_FLUSH
3149  *              drm_agp_chipset_flush
3150  */
3151 static void
3152 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
3153                                   struct intel_ring_buffer *ring)
3154 {
3155         struct drm_device               *dev = obj->dev;
3156         struct drm_i915_private         *dev_priv = dev->dev_private;
3157         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
3158         uint32_t                        invalidate_domains = 0;
3159         uint32_t                        flush_domains = 0;
3160
3161         /*
3162          * If the object isn't moving to a new write domain,
3163          * let the object stay in multiple read domains
3164          */
3165         if (obj->pending_write_domain == 0)
3166                 obj->pending_read_domains |= obj->read_domains;
3167
3168         /*
3169          * Flush the current write domain if
3170          * the new read domains don't match. Invalidate
3171          * any read domains which differ from the old
3172          * write domain
3173          */
3174         if (obj->write_domain &&
3175             obj->write_domain != obj->pending_read_domains) {
3176                 flush_domains |= obj->write_domain;
3177                 invalidate_domains |=
3178                         obj->pending_read_domains & ~obj->write_domain;
3179         }
3180         /*
3181          * Invalidate any read caches which may have
3182          * stale data. That is, any new read domains.
3183          */
3184         invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3185         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
3186                 i915_gem_clflush_object(obj);
3187
3188         /* blow away mappings if mapped through GTT */
3189         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_GTT)
3190                 i915_gem_release_mmap(obj);
3191
3192         /* The actual obj->write_domain will be updated with
3193          * pending_write_domain after we emit the accumulated flush for all
3194          * of our domain changes in execbuffers (which clears objects'
3195          * write_domains).  So if we have a current write domain that we
3196          * aren't changing, set pending_write_domain to that.
3197          */
3198         if (flush_domains == 0 && obj->pending_write_domain == 0)
3199                 obj->pending_write_domain = obj->write_domain;
3200
3201         dev->invalidate_domains |= invalidate_domains;
3202         dev->flush_domains |= flush_domains;
3203         if (flush_domains & I915_GEM_GPU_DOMAINS)
3204                 dev_priv->mm.flush_rings |= obj_priv->ring->id;
3205         if (invalidate_domains & I915_GEM_GPU_DOMAINS)
3206                 dev_priv->mm.flush_rings |= ring->id;
3207 }
3208
3209 /**
3210  * Moves the object from a partially CPU read to a full one.
3211  *
3212  * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3213  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3214  */
3215 static void
3216 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3217 {
3218         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3219
3220         if (!obj_priv->page_cpu_valid)
3221                 return;
3222
3223         /* If we're partially in the CPU read domain, finish moving it in.
3224          */
3225         if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3226                 int i;
3227
3228                 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3229                         if (obj_priv->page_cpu_valid[i])
3230                                 continue;
3231                         drm_clflush_pages(obj_priv->pages + i, 1);
3232                 }
3233         }
3234
3235         /* Free the page_cpu_valid mappings which are now stale, whether
3236          * or not we've got I915_GEM_DOMAIN_CPU.
3237          */
3238         kfree(obj_priv->page_cpu_valid);
3239         obj_priv->page_cpu_valid = NULL;
3240 }
3241
3242 /**
3243  * Set the CPU read domain on a range of the object.
3244  *
3245  * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3246  * not entirely valid.  The page_cpu_valid member of the object flags which
3247  * pages have been flushed, and will be respected by
3248  * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3249  * of the whole object.
3250  *
3251  * This function returns when the move is complete, including waiting on
3252  * flushes to occur.
3253  */
3254 static int
3255 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3256                                           uint64_t offset, uint64_t size)
3257 {
3258         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3259         uint32_t old_read_domains;
3260         int i, ret;
3261
3262         if (offset == 0 && size == obj->size)
3263                 return i915_gem_object_set_to_cpu_domain(obj, 0);
3264
3265         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
3266         if (ret != 0)
3267                 return ret;
3268         i915_gem_object_flush_gtt_write_domain(obj);
3269
3270         /* If we're already fully in the CPU read domain, we're done. */
3271         if (obj_priv->page_cpu_valid == NULL &&
3272             (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3273                 return 0;
3274
3275         /* Otherwise, create/clear the per-page CPU read domain flag if we're
3276          * newly adding I915_GEM_DOMAIN_CPU
3277          */
3278         if (obj_priv->page_cpu_valid == NULL) {
3279                 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3280                                                    GFP_KERNEL);
3281                 if (obj_priv->page_cpu_valid == NULL)
3282                         return -ENOMEM;
3283         } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3284                 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
3285
3286         /* Flush the cache on any pages that are still invalid from the CPU's
3287          * perspective.
3288          */
3289         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3290              i++) {
3291                 if (obj_priv->page_cpu_valid[i])
3292                         continue;
3293
3294                 drm_clflush_pages(obj_priv->pages + i, 1);
3295
3296                 obj_priv->page_cpu_valid[i] = 1;
3297         }
3298
3299         /* It should now be out of any other write domains, and we can update
3300          * the domain values for our changes.
3301          */
3302         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3303
3304         old_read_domains = obj->read_domains;
3305         obj->read_domains |= I915_GEM_DOMAIN_CPU;
3306
3307         trace_i915_gem_object_change_domain(obj,
3308                                             old_read_domains,
3309                                             obj->write_domain);
3310
3311         return 0;
3312 }
3313
3314 /**
3315  * Pin an object to the GTT and evaluate the relocations landing in it.
3316  */
3317 static int
3318 i915_gem_execbuffer_relocate(struct drm_i915_gem_object *obj,
3319                              struct drm_file *file_priv,
3320                              struct drm_i915_gem_exec_object2 *entry)
3321 {
3322         struct drm_device *dev = obj->base.dev;
3323         drm_i915_private_t *dev_priv = dev->dev_private;
3324         struct drm_i915_gem_relocation_entry __user *user_relocs;
3325         struct drm_gem_object *target_obj = NULL;
3326         uint32_t target_handle = 0;
3327         int i, ret = 0;
3328
3329         user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
3330         for (i = 0; i < entry->relocation_count; i++) {
3331                 struct drm_i915_gem_relocation_entry reloc;
3332                 uint32_t target_offset;
3333
3334                 if (__copy_from_user_inatomic(&reloc,
3335                                               user_relocs+i,
3336                                               sizeof(reloc))) {
3337                         ret = -EFAULT;
3338                         break;
3339                 }
3340
3341                 if (reloc.target_handle != target_handle) {
3342                         drm_gem_object_unreference(target_obj);
3343
3344                         target_obj = drm_gem_object_lookup(dev, file_priv,
3345                                                            reloc.target_handle);
3346                         if (target_obj == NULL) {
3347                                 ret = -ENOENT;
3348                                 break;
3349                         }
3350
3351                         target_handle = reloc.target_handle;
3352                 }
3353                 target_offset = to_intel_bo(target_obj)->gtt_offset;
3354
3355 #if WATCH_RELOC
3356                 DRM_INFO("%s: obj %p offset %08x target %d "
3357                          "read %08x write %08x gtt %08x "
3358                          "presumed %08x delta %08x\n",
3359                          __func__,
3360                          obj,
3361                          (int) reloc.offset,
3362                          (int) reloc.target_handle,
3363                          (int) reloc.read_domains,
3364                          (int) reloc.write_domain,
3365                          (int) target_offset,
3366                          (int) reloc.presumed_offset,
3367                          reloc.delta);
3368 #endif
3369
3370                 /* The target buffer should have appeared before us in the
3371                  * exec_object list, so it should have a GTT space bound by now.
3372                  */
3373                 if (target_offset == 0) {
3374                         DRM_ERROR("No GTT space found for object %d\n",
3375                                   reloc.target_handle);
3376                         ret = -EINVAL;
3377                         break;
3378                 }
3379
3380                 /* Validate that the target is in a valid r/w GPU domain */
3381                 if (reloc.write_domain & (reloc.write_domain - 1)) {
3382                         DRM_ERROR("reloc with multiple write domains: "
3383                                   "obj %p target %d offset %d "
3384                                   "read %08x write %08x",
3385                                   obj, reloc.target_handle,
3386                                   (int) reloc.offset,
3387                                   reloc.read_domains,
3388                                   reloc.write_domain);
3389                         ret = -EINVAL;
3390                         break;
3391                 }
3392                 if (reloc.write_domain & I915_GEM_DOMAIN_CPU ||
3393                     reloc.read_domains & I915_GEM_DOMAIN_CPU) {
3394                         DRM_ERROR("reloc with read/write CPU domains: "
3395                                   "obj %p target %d offset %d "
3396                                   "read %08x write %08x",
3397                                   obj, reloc.target_handle,
3398                                   (int) reloc.offset,
3399                                   reloc.read_domains,
3400                                   reloc.write_domain);
3401                         ret = -EINVAL;
3402                         break;
3403                 }
3404                 if (reloc.write_domain && target_obj->pending_write_domain &&
3405                     reloc.write_domain != target_obj->pending_write_domain) {
3406                         DRM_ERROR("Write domain conflict: "
3407                                   "obj %p target %d offset %d "
3408                                   "new %08x old %08x\n",
3409                                   obj, reloc.target_handle,
3410                                   (int) reloc.offset,
3411                                   reloc.write_domain,
3412                                   target_obj->pending_write_domain);
3413                         ret = -EINVAL;
3414                         break;
3415                 }
3416
3417                 target_obj->pending_read_domains |= reloc.read_domains;
3418                 target_obj->pending_write_domain |= reloc.write_domain;
3419
3420                 /* If the relocation already has the right value in it, no
3421                  * more work needs to be done.
3422                  */
3423                 if (target_offset == reloc.presumed_offset)
3424                         continue;
3425
3426                 /* Check that the relocation address is valid... */
3427                 if (reloc.offset > obj->base.size - 4) {
3428                         DRM_ERROR("Relocation beyond object bounds: "
3429                                   "obj %p target %d offset %d size %d.\n",
3430                                   obj, reloc.target_handle,
3431                                   (int) reloc.offset, (int) obj->base.size);
3432                         ret = -EINVAL;
3433                         break;
3434                 }
3435                 if (reloc.offset & 3) {
3436                         DRM_ERROR("Relocation not 4-byte aligned: "
3437                                   "obj %p target %d offset %d.\n",
3438                                   obj, reloc.target_handle,
3439                                   (int) reloc.offset);
3440                         ret = -EINVAL;
3441                         break;
3442                 }
3443
3444                 /* and points to somewhere within the target object. */
3445                 if (reloc.delta >= target_obj->size) {
3446                         DRM_ERROR("Relocation beyond target object bounds: "
3447                                   "obj %p target %d delta %d size %d.\n",
3448                                   obj, reloc.target_handle,
3449                                   (int) reloc.delta, (int) target_obj->size);
3450                         ret = -EINVAL;
3451                         break;
3452                 }
3453
3454                 reloc.delta += target_offset;
3455                 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
3456                         uint32_t page_offset = reloc.offset & ~PAGE_MASK;
3457                         char *vaddr;
3458
3459                         vaddr = kmap_atomic(obj->pages[reloc.offset >> PAGE_SHIFT]);
3460                         *(uint32_t *)(vaddr + page_offset) = reloc.delta;
3461                         kunmap_atomic(vaddr);
3462                 } else {
3463                         uint32_t __iomem *reloc_entry;
3464                         void __iomem *reloc_page;
3465
3466                         ret = i915_gem_object_set_to_gtt_domain(&obj->base, 1);
3467                         if (ret)
3468                                 break;
3469
3470                         /* Map the page containing the relocation we're going to perform.  */
3471                         reloc.offset += obj->gtt_offset;
3472                         reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3473                                                               reloc.offset & PAGE_MASK);
3474                         reloc_entry = (uint32_t __iomem *)
3475                                 (reloc_page + (reloc.offset & ~PAGE_MASK));
3476                         iowrite32(reloc.delta, reloc_entry);
3477                         io_mapping_unmap_atomic(reloc_page);
3478                 }
3479
3480                 /* and update the user's relocation entry */
3481                 reloc.presumed_offset = target_offset;
3482                 if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset,
3483                                               &reloc.presumed_offset,
3484                                               sizeof(reloc.presumed_offset))) {
3485                     ret = -EFAULT;
3486                     break;
3487                 }
3488         }
3489
3490         drm_gem_object_unreference(target_obj);
3491         return ret;
3492 }
3493
3494 static int
3495 i915_gem_execbuffer_pin(struct drm_device *dev,
3496                         struct drm_file *file,
3497                         struct drm_gem_object **object_list,
3498                         struct drm_i915_gem_exec_object2 *exec_list,
3499                         int count)
3500 {
3501         struct drm_i915_private *dev_priv = dev->dev_private;
3502         int ret, i, retry;
3503
3504         /* attempt to pin all of the buffers into the GTT */
3505         for (retry = 0; retry < 2; retry++) {
3506                 ret = 0;
3507                 for (i = 0; i < count; i++) {
3508                         struct drm_i915_gem_exec_object2 *entry = &exec_list[i];
3509                         struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3510                         bool need_fence =
3511                                 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3512                                 obj->tiling_mode != I915_TILING_NONE;
3513
3514                         /* g33/pnv can't fence buffers in the unmappable part */
3515                         bool need_mappable =
3516                                 entry->relocation_count ? true : need_fence;
3517
3518                         /* Check fence reg constraints and rebind if necessary */
3519                         if (need_fence &&
3520                             !i915_gem_object_fence_offset_ok(&obj->base,
3521                                                              obj->tiling_mode)) {
3522                                 ret = i915_gem_object_unbind(&obj->base);
3523                                 if (ret)
3524                                         break;
3525                         }
3526
3527                         ret = i915_gem_object_pin(&obj->base,
3528                                                   entry->alignment,
3529                                                   need_mappable);
3530                         if (ret)
3531                                 break;
3532
3533                         /*
3534                          * Pre-965 chips need a fence register set up in order
3535                          * to properly handle blits to/from tiled surfaces.
3536                          */
3537                         if (need_fence) {
3538                                 ret = i915_gem_object_get_fence_reg(&obj->base, true);
3539                                 if (ret) {
3540                                         i915_gem_object_unpin(&obj->base);
3541                                         break;
3542                                 }
3543
3544                                 dev_priv->fence_regs[obj->fence_reg].gpu = true;
3545                         }
3546
3547                         entry->offset = obj->gtt_offset;
3548                 }
3549
3550                 while (i--)
3551                         i915_gem_object_unpin(object_list[i]);
3552
3553                 if (ret == 0)
3554                         break;
3555
3556                 if (ret != -ENOSPC || retry)
3557                         return ret;
3558
3559                 ret = i915_gem_evict_everything(dev);
3560                 if (ret)
3561                         return ret;
3562         }
3563
3564         return 0;
3565 }
3566
3567 /* Throttle our rendering by waiting until the ring has completed our requests
3568  * emitted over 20 msec ago.
3569  *
3570  * Note that if we were to use the current jiffies each time around the loop,
3571  * we wouldn't escape the function with any frames outstanding if the time to
3572  * render a frame was over 20ms.
3573  *
3574  * This should get us reasonable parallelism between CPU and GPU but also
3575  * relatively low latency when blocking on a particular request to finish.
3576  */
3577 static int
3578 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3579 {
3580         struct drm_i915_private *dev_priv = dev->dev_private;
3581         struct drm_i915_file_private *file_priv = file->driver_priv;
3582         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3583         struct drm_i915_gem_request *request;
3584         struct intel_ring_buffer *ring = NULL;
3585         u32 seqno = 0;
3586         int ret;
3587
3588         spin_lock(&file_priv->mm.lock);
3589         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3590                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3591                         break;
3592
3593                 ring = request->ring;
3594                 seqno = request->seqno;
3595         }
3596         spin_unlock(&file_priv->mm.lock);
3597
3598         if (seqno == 0)
3599                 return 0;
3600
3601         ret = 0;
3602         if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
3603                 /* And wait for the seqno passing without holding any locks and
3604                  * causing extra latency for others. This is safe as the irq
3605                  * generation is designed to be run atomically and so is
3606                  * lockless.
3607                  */
3608                 ring->user_irq_get(ring);
3609                 ret = wait_event_interruptible(ring->irq_queue,
3610                                                i915_seqno_passed(ring->get_seqno(ring), seqno)
3611                                                || atomic_read(&dev_priv->mm.wedged));
3612                 ring->user_irq_put(ring);
3613
3614                 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3615                         ret = -EIO;
3616         }
3617
3618         if (ret == 0)
3619                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3620
3621         return ret;
3622 }
3623
3624 static int
3625 i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec,
3626                           uint64_t exec_offset)
3627 {
3628         uint32_t exec_start, exec_len;
3629
3630         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3631         exec_len = (uint32_t) exec->batch_len;
3632
3633         if ((exec_start | exec_len) & 0x7)
3634                 return -EINVAL;
3635
3636         if (!exec_start)
3637                 return -EINVAL;
3638
3639         return 0;
3640 }
3641
3642 static int
3643 validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
3644                    int count)
3645 {
3646         int i;
3647
3648         for (i = 0; i < count; i++) {
3649                 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
3650                 size_t length = exec[i].relocation_count * sizeof(struct drm_i915_gem_relocation_entry);
3651
3652                 if (!access_ok(VERIFY_READ, ptr, length))
3653                         return -EFAULT;
3654
3655                 /* we may also need to update the presumed offsets */
3656                 if (!access_ok(VERIFY_WRITE, ptr, length))
3657                         return -EFAULT;
3658
3659                 if (fault_in_pages_readable(ptr, length))
3660                         return -EFAULT;
3661         }
3662
3663         return 0;
3664 }
3665
3666 static int
3667 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3668                        struct drm_file *file,
3669                        struct drm_i915_gem_execbuffer2 *args,
3670                        struct drm_i915_gem_exec_object2 *exec_list)
3671 {
3672         drm_i915_private_t *dev_priv = dev->dev_private;
3673         struct drm_gem_object **object_list = NULL;
3674         struct drm_gem_object *batch_obj;
3675         struct drm_clip_rect *cliprects = NULL;
3676         struct drm_i915_gem_request *request = NULL;
3677         int ret, i, flips;
3678         uint64_t exec_offset;
3679
3680         struct intel_ring_buffer *ring = NULL;
3681
3682         ret = i915_gem_check_is_wedged(dev);
3683         if (ret)
3684                 return ret;
3685
3686         ret = validate_exec_list(exec_list, args->buffer_count);
3687         if (ret)
3688                 return ret;
3689
3690 #if WATCH_EXEC
3691         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3692                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3693 #endif
3694         switch (args->flags & I915_EXEC_RING_MASK) {
3695         case I915_EXEC_DEFAULT:
3696         case I915_EXEC_RENDER:
3697                 ring = &dev_priv->render_ring;
3698                 break;
3699         case I915_EXEC_BSD:
3700                 if (!HAS_BSD(dev)) {
3701                         DRM_ERROR("execbuf with invalid ring (BSD)\n");
3702                         return -EINVAL;
3703                 }
3704                 ring = &dev_priv->bsd_ring;
3705                 break;
3706         case I915_EXEC_BLT:
3707                 if (!HAS_BLT(dev)) {
3708                         DRM_ERROR("execbuf with invalid ring (BLT)\n");
3709                         return -EINVAL;
3710                 }
3711                 ring = &dev_priv->blt_ring;
3712                 break;
3713         default:
3714                 DRM_ERROR("execbuf with unknown ring: %d\n",
3715                           (int)(args->flags & I915_EXEC_RING_MASK));
3716                 return -EINVAL;
3717         }
3718
3719         if (args->buffer_count < 1) {
3720                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3721                 return -EINVAL;
3722         }
3723         object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3724         if (object_list == NULL) {
3725                 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3726                           args->buffer_count);
3727                 ret = -ENOMEM;
3728                 goto pre_mutex_err;
3729         }
3730
3731         if (args->num_cliprects != 0) {
3732                 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3733                                     GFP_KERNEL);
3734                 if (cliprects == NULL) {
3735                         ret = -ENOMEM;
3736                         goto pre_mutex_err;
3737                 }
3738
3739                 ret = copy_from_user(cliprects,
3740                                      (struct drm_clip_rect __user *)
3741                                      (uintptr_t) args->cliprects_ptr,
3742                                      sizeof(*cliprects) * args->num_cliprects);
3743                 if (ret != 0) {
3744                         DRM_ERROR("copy %d cliprects failed: %d\n",
3745                                   args->num_cliprects, ret);
3746                         ret = -EFAULT;
3747                         goto pre_mutex_err;
3748                 }
3749         }
3750
3751         request = kzalloc(sizeof(*request), GFP_KERNEL);
3752         if (request == NULL) {
3753                 ret = -ENOMEM;
3754                 goto pre_mutex_err;
3755         }
3756
3757         ret = i915_mutex_lock_interruptible(dev);
3758         if (ret)
3759                 goto pre_mutex_err;
3760
3761         if (dev_priv->mm.suspended) {
3762                 mutex_unlock(&dev->struct_mutex);
3763                 ret = -EBUSY;
3764                 goto pre_mutex_err;
3765         }
3766
3767         /* Look up object handles */
3768         for (i = 0; i < args->buffer_count; i++) {
3769                 struct drm_i915_gem_object *obj_priv;
3770
3771                 object_list[i] = drm_gem_object_lookup(dev, file,
3772                                                        exec_list[i].handle);
3773                 if (object_list[i] == NULL) {
3774                         DRM_ERROR("Invalid object handle %d at index %d\n",
3775                                    exec_list[i].handle, i);
3776                         /* prevent error path from reading uninitialized data */
3777                         args->buffer_count = i + 1;
3778                         ret = -ENOENT;
3779                         goto err;
3780                 }
3781
3782                 obj_priv = to_intel_bo(object_list[i]);
3783                 if (obj_priv->in_execbuffer) {
3784                         DRM_ERROR("Object %p appears more than once in object list\n",
3785                                    object_list[i]);
3786                         /* prevent error path from reading uninitialized data */
3787                         args->buffer_count = i + 1;
3788                         ret = -EINVAL;
3789                         goto err;
3790                 }
3791                 obj_priv->in_execbuffer = true;
3792         }
3793
3794         /* Move the objects en-masse into the GTT, evicting if necessary. */
3795         ret = i915_gem_execbuffer_pin(dev, file,
3796                                       object_list, exec_list,
3797                                       args->buffer_count);
3798         if (ret)
3799                 goto err;
3800
3801         /* The objects are in their final locations, apply the relocations. */
3802         for (i = 0; i < args->buffer_count; i++) {
3803                 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3804                 obj->base.pending_read_domains = 0;
3805                 obj->base.pending_write_domain = 0;
3806                 ret = i915_gem_execbuffer_relocate(obj, file, &exec_list[i]);
3807                 if (ret)
3808                         goto err;
3809         }
3810
3811         /* Set the pending read domains for the batch buffer to COMMAND */
3812         batch_obj = object_list[args->buffer_count-1];
3813         if (batch_obj->pending_write_domain) {
3814                 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3815                 ret = -EINVAL;
3816                 goto err;
3817         }
3818         batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3819
3820         /* Sanity check the batch buffer */
3821         exec_offset = to_intel_bo(batch_obj)->gtt_offset;
3822         ret = i915_gem_check_execbuffer(args, exec_offset);
3823         if (ret != 0) {
3824                 DRM_ERROR("execbuf with invalid offset/length\n");
3825                 goto err;
3826         }
3827
3828         /* Zero the global flush/invalidate flags. These
3829          * will be modified as new domains are computed
3830          * for each object
3831          */
3832         dev->invalidate_domains = 0;
3833         dev->flush_domains = 0;
3834         dev_priv->mm.flush_rings = 0;
3835         for (i = 0; i < args->buffer_count; i++)
3836                 i915_gem_object_set_to_gpu_domain(object_list[i], ring);
3837
3838         if (dev->invalidate_domains | dev->flush_domains) {
3839 #if WATCH_EXEC
3840                 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3841                           __func__,
3842                          dev->invalidate_domains,
3843                          dev->flush_domains);
3844 #endif
3845                 i915_gem_flush(dev, file,
3846                                dev->invalidate_domains,
3847                                dev->flush_domains,
3848                                dev_priv->mm.flush_rings);
3849         }
3850
3851 #if WATCH_COHERENCY
3852         for (i = 0; i < args->buffer_count; i++) {
3853                 i915_gem_object_check_coherency(object_list[i],
3854                                                 exec_list[i].handle);
3855         }
3856 #endif
3857
3858 #if WATCH_EXEC
3859         i915_gem_dump_object(batch_obj,
3860                               args->batch_len,
3861                               __func__,
3862                               ~0);
3863 #endif
3864
3865         /* Check for any pending flips. As we only maintain a flip queue depth
3866          * of 1, we can simply insert a WAIT for the next display flip prior
3867          * to executing the batch and avoid stalling the CPU.
3868          */
3869         flips = 0;
3870         for (i = 0; i < args->buffer_count; i++) {
3871                 if (object_list[i]->write_domain)
3872                         flips |= atomic_read(&to_intel_bo(object_list[i])->pending_flip);
3873         }
3874         if (flips) {
3875                 int plane, flip_mask;
3876
3877                 for (plane = 0; flips >> plane; plane++) {
3878                         if (((flips >> plane) & 1) == 0)
3879                                 continue;
3880
3881                         if (plane)
3882                                 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
3883                         else
3884                                 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
3885
3886                         ret = intel_ring_begin(ring, 2);
3887                         if (ret)
3888                                 goto err;
3889
3890                         intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
3891                         intel_ring_emit(ring, MI_NOOP);
3892                         intel_ring_advance(ring);
3893                 }
3894         }
3895
3896         /* Exec the batchbuffer */
3897         ret = ring->dispatch_execbuffer(ring, args, cliprects, exec_offset);
3898         if (ret) {
3899                 DRM_ERROR("dispatch failed %d\n", ret);
3900                 goto err;
3901         }
3902
3903         for (i = 0; i < args->buffer_count; i++) {
3904                 struct drm_gem_object *obj = object_list[i];
3905
3906                 obj->read_domains = obj->pending_read_domains;
3907                 obj->write_domain = obj->pending_write_domain;
3908
3909                 i915_gem_object_move_to_active(obj, ring);
3910                 if (obj->write_domain) {
3911                         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3912                         obj_priv->dirty = 1;
3913                         list_move_tail(&obj_priv->gpu_write_list,
3914                                        &ring->gpu_write_list);
3915                         intel_mark_busy(dev, obj);
3916                 }
3917
3918                 trace_i915_gem_object_change_domain(obj,
3919                                                     obj->read_domains,
3920                                                     obj->write_domain);
3921         }
3922
3923         /*
3924          * Ensure that the commands in the batch buffer are
3925          * finished before the interrupt fires
3926          */
3927         i915_retire_commands(dev, ring);
3928
3929         if (i915_add_request(dev, file, request, ring))
3930                 ring->outstanding_lazy_request = true;
3931         else
3932                 request = NULL;
3933
3934 err:
3935         for (i = 0; i < args->buffer_count; i++) {
3936                 if (object_list[i] == NULL)
3937                     break;
3938
3939                 to_intel_bo(object_list[i])->in_execbuffer = false;
3940                 drm_gem_object_unreference(object_list[i]);
3941         }
3942
3943         mutex_unlock(&dev->struct_mutex);
3944
3945 pre_mutex_err:
3946         drm_free_large(object_list);
3947         kfree(cliprects);
3948         kfree(request);
3949
3950         return ret;
3951 }
3952
3953 /*
3954  * Legacy execbuffer just creates an exec2 list from the original exec object
3955  * list array and passes it to the real function.
3956  */
3957 int
3958 i915_gem_execbuffer(struct drm_device *dev, void *data,
3959                     struct drm_file *file_priv)
3960 {
3961         struct drm_i915_gem_execbuffer *args = data;
3962         struct drm_i915_gem_execbuffer2 exec2;
3963         struct drm_i915_gem_exec_object *exec_list = NULL;
3964         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3965         int ret, i;
3966
3967 #if WATCH_EXEC
3968         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3969                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3970 #endif
3971
3972         if (args->buffer_count < 1) {
3973                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3974                 return -EINVAL;
3975         }
3976
3977         /* Copy in the exec list from userland */
3978         exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
3979         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
3980         if (exec_list == NULL || exec2_list == NULL) {
3981                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
3982                           args->buffer_count);
3983                 drm_free_large(exec_list);
3984                 drm_free_large(exec2_list);
3985                 return -ENOMEM;
3986         }
3987         ret = copy_from_user(exec_list,
3988                              (struct drm_i915_relocation_entry __user *)
3989                              (uintptr_t) args->buffers_ptr,
3990                              sizeof(*exec_list) * args->buffer_count);
3991         if (ret != 0) {
3992                 DRM_ERROR("copy %d exec entries failed %d\n",
3993                           args->buffer_count, ret);
3994                 drm_free_large(exec_list);
3995                 drm_free_large(exec2_list);
3996                 return -EFAULT;
3997         }
3998
3999         for (i = 0; i < args->buffer_count; i++) {
4000                 exec2_list[i].handle = exec_list[i].handle;
4001                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
4002                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
4003                 exec2_list[i].alignment = exec_list[i].alignment;
4004                 exec2_list[i].offset = exec_list[i].offset;
4005                 if (INTEL_INFO(dev)->gen < 4)
4006                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
4007                 else
4008                         exec2_list[i].flags = 0;
4009         }
4010
4011         exec2.buffers_ptr = args->buffers_ptr;
4012         exec2.buffer_count = args->buffer_count;
4013         exec2.batch_start_offset = args->batch_start_offset;
4014         exec2.batch_len = args->batch_len;
4015         exec2.DR1 = args->DR1;
4016         exec2.DR4 = args->DR4;
4017         exec2.num_cliprects = args->num_cliprects;
4018         exec2.cliprects_ptr = args->cliprects_ptr;
4019         exec2.flags = I915_EXEC_RENDER;
4020
4021         ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
4022         if (!ret) {
4023                 /* Copy the new buffer offsets back to the user's exec list. */
4024                 for (i = 0; i < args->buffer_count; i++)
4025                         exec_list[i].offset = exec2_list[i].offset;
4026                 /* ... and back out to userspace */
4027                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4028                                    (uintptr_t) args->buffers_ptr,
4029                                    exec_list,
4030                                    sizeof(*exec_list) * args->buffer_count);
4031                 if (ret) {
4032                         ret = -EFAULT;
4033                         DRM_ERROR("failed to copy %d exec entries "
4034                                   "back to user (%d)\n",
4035                                   args->buffer_count, ret);
4036                 }
4037         }
4038
4039         drm_free_large(exec_list);
4040         drm_free_large(exec2_list);
4041         return ret;
4042 }
4043
4044 int
4045 i915_gem_execbuffer2(struct drm_device *dev, void *data,
4046                      struct drm_file *file_priv)
4047 {
4048         struct drm_i915_gem_execbuffer2 *args = data;
4049         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4050         int ret;
4051
4052 #if WATCH_EXEC
4053         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4054                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4055 #endif
4056
4057         if (args->buffer_count < 1) {
4058                 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4059                 return -EINVAL;
4060         }
4061
4062         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4063         if (exec2_list == NULL) {
4064                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4065                           args->buffer_count);
4066                 return -ENOMEM;
4067         }
4068         ret = copy_from_user(exec2_list,
4069                              (struct drm_i915_relocation_entry __user *)
4070                              (uintptr_t) args->buffers_ptr,
4071                              sizeof(*exec2_list) * args->buffer_count);
4072         if (ret != 0) {
4073                 DRM_ERROR("copy %d exec entries failed %d\n",
4074                           args->buffer_count, ret);
4075                 drm_free_large(exec2_list);
4076                 return -EFAULT;
4077         }
4078
4079         ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4080         if (!ret) {
4081                 /* Copy the new buffer offsets back to the user's exec list. */
4082                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4083                                    (uintptr_t) args->buffers_ptr,
4084                                    exec2_list,
4085                                    sizeof(*exec2_list) * args->buffer_count);
4086                 if (ret) {
4087                         ret = -EFAULT;
4088                         DRM_ERROR("failed to copy %d exec entries "
4089                                   "back to user (%d)\n",
4090                                   args->buffer_count, ret);
4091                 }
4092         }
4093
4094         drm_free_large(exec2_list);
4095         return ret;
4096 }
4097
4098 int
4099 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment,
4100                     bool mappable)
4101 {
4102         struct drm_device *dev = obj->dev;
4103         struct drm_i915_private *dev_priv = dev->dev_private;
4104         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4105         int ret;
4106
4107         BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4108         WARN_ON(i915_verify_lists(dev));
4109
4110         if (obj_priv->gtt_space != NULL) {
4111                 if (alignment == 0)
4112                         alignment = i915_gem_get_gtt_alignment(obj);
4113                 if (obj_priv->gtt_offset & (alignment - 1) ||
4114                     (mappable && !i915_gem_object_cpu_accessible(obj_priv))) {
4115                         WARN(obj_priv->pin_count,
4116                              "bo is already pinned with incorrect alignment:"
4117                              " offset=%x, req.alignment=%x\n",
4118                              obj_priv->gtt_offset, alignment);
4119                         ret = i915_gem_object_unbind(obj);
4120                         if (ret)
4121                                 return ret;
4122                 }
4123         }
4124
4125         if (obj_priv->gtt_space == NULL) {
4126                 ret = i915_gem_object_bind_to_gtt(obj, alignment, mappable);
4127                 if (ret)
4128                         return ret;
4129         }
4130
4131         obj_priv->pin_count++;
4132
4133         /* If the object is not active and not pending a flush,
4134          * remove it from the inactive list
4135          */
4136         if (obj_priv->pin_count == 1) {
4137                 i915_gem_info_add_pin(dev_priv, obj, mappable);
4138                 if (!obj_priv->active)
4139                         list_move_tail(&obj_priv->mm_list,
4140                                        &dev_priv->mm.pinned_list);
4141         }
4142         BUG_ON(!obj_priv->pin_mappable && mappable);
4143
4144         WARN_ON(i915_verify_lists(dev));
4145         return 0;
4146 }
4147
4148 void
4149 i915_gem_object_unpin(struct drm_gem_object *obj)
4150 {
4151         struct drm_device *dev = obj->dev;
4152         drm_i915_private_t *dev_priv = dev->dev_private;
4153         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4154
4155         WARN_ON(i915_verify_lists(dev));
4156         obj_priv->pin_count--;
4157         BUG_ON(obj_priv->pin_count < 0);
4158         BUG_ON(obj_priv->gtt_space == NULL);
4159
4160         /* If the object is no longer pinned, and is
4161          * neither active nor being flushed, then stick it on
4162          * the inactive list
4163          */
4164         if (obj_priv->pin_count == 0) {
4165                 if (!obj_priv->active)
4166                         list_move_tail(&obj_priv->mm_list,
4167                                        &dev_priv->mm.inactive_list);
4168                 i915_gem_info_remove_pin(dev_priv, obj);
4169         }
4170         WARN_ON(i915_verify_lists(dev));
4171 }
4172
4173 int
4174 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4175                    struct drm_file *file_priv)
4176 {
4177         struct drm_i915_gem_pin *args = data;
4178         struct drm_gem_object *obj;
4179         struct drm_i915_gem_object *obj_priv;
4180         int ret;
4181
4182         ret = i915_mutex_lock_interruptible(dev);
4183         if (ret)
4184                 return ret;
4185
4186         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4187         if (obj == NULL) {
4188                 ret = -ENOENT;
4189                 goto unlock;
4190         }
4191         obj_priv = to_intel_bo(obj);
4192
4193         if (obj_priv->madv != I915_MADV_WILLNEED) {
4194                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4195                 ret = -EINVAL;
4196                 goto out;
4197         }
4198
4199         if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4200                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4201                           args->handle);
4202                 ret = -EINVAL;
4203                 goto out;
4204         }
4205
4206         obj_priv->user_pin_count++;
4207         obj_priv->pin_filp = file_priv;
4208         if (obj_priv->user_pin_count == 1) {
4209                 ret = i915_gem_object_pin(obj, args->alignment, true);
4210                 if (ret)
4211                         goto out;
4212         }
4213
4214         /* XXX - flush the CPU caches for pinned objects
4215          * as the X server doesn't manage domains yet
4216          */
4217         i915_gem_object_flush_cpu_write_domain(obj);
4218         args->offset = obj_priv->gtt_offset;
4219 out:
4220         drm_gem_object_unreference(obj);
4221 unlock:
4222         mutex_unlock(&dev->struct_mutex);
4223         return ret;
4224 }
4225
4226 int
4227 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4228                      struct drm_file *file_priv)
4229 {
4230         struct drm_i915_gem_pin *args = data;
4231         struct drm_gem_object *obj;
4232         struct drm_i915_gem_object *obj_priv;
4233         int ret;
4234
4235         ret = i915_mutex_lock_interruptible(dev);
4236         if (ret)
4237                 return ret;
4238
4239         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4240         if (obj == NULL) {
4241                 ret = -ENOENT;
4242                 goto unlock;
4243         }
4244         obj_priv = to_intel_bo(obj);
4245
4246         if (obj_priv->pin_filp != file_priv) {
4247                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4248                           args->handle);
4249                 ret = -EINVAL;
4250                 goto out;
4251         }
4252         obj_priv->user_pin_count--;
4253         if (obj_priv->user_pin_count == 0) {
4254                 obj_priv->pin_filp = NULL;
4255                 i915_gem_object_unpin(obj);
4256         }
4257
4258 out:
4259         drm_gem_object_unreference(obj);
4260 unlock:
4261         mutex_unlock(&dev->struct_mutex);
4262         return ret;
4263 }
4264
4265 int
4266 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4267                     struct drm_file *file_priv)
4268 {
4269         struct drm_i915_gem_busy *args = data;
4270         struct drm_gem_object *obj;
4271         struct drm_i915_gem_object *obj_priv;
4272         int ret;
4273
4274         ret = i915_mutex_lock_interruptible(dev);
4275         if (ret)
4276                 return ret;
4277
4278         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4279         if (obj == NULL) {
4280                 ret = -ENOENT;
4281                 goto unlock;
4282         }
4283         obj_priv = to_intel_bo(obj);
4284
4285         /* Count all active objects as busy, even if they are currently not used
4286          * by the gpu. Users of this interface expect objects to eventually
4287          * become non-busy without any further actions, therefore emit any
4288          * necessary flushes here.
4289          */
4290         args->busy = obj_priv->active;
4291         if (args->busy) {
4292                 /* Unconditionally flush objects, even when the gpu still uses this
4293                  * object. Userspace calling this function indicates that it wants to
4294                  * use this buffer rather sooner than later, so issuing the required
4295                  * flush earlier is beneficial.
4296                  */
4297                 if (obj->write_domain & I915_GEM_GPU_DOMAINS)
4298                         i915_gem_flush_ring(dev, file_priv,
4299                                             obj_priv->ring,
4300                                             0, obj->write_domain);
4301
4302                 /* Update the active list for the hardware's current position.
4303                  * Otherwise this only updates on a delayed timer or when irqs
4304                  * are actually unmasked, and our working set ends up being
4305                  * larger than required.
4306                  */
4307                 i915_gem_retire_requests_ring(dev, obj_priv->ring);
4308
4309                 args->busy = obj_priv->active;
4310         }
4311
4312         drm_gem_object_unreference(obj);
4313 unlock:
4314         mutex_unlock(&dev->struct_mutex);
4315         return ret;
4316 }
4317
4318 int
4319 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4320                         struct drm_file *file_priv)
4321 {
4322     return i915_gem_ring_throttle(dev, file_priv);
4323 }
4324
4325 int
4326 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4327                        struct drm_file *file_priv)
4328 {
4329         struct drm_i915_gem_madvise *args = data;
4330         struct drm_gem_object *obj;
4331         struct drm_i915_gem_object *obj_priv;
4332         int ret;
4333
4334         switch (args->madv) {
4335         case I915_MADV_DONTNEED:
4336         case I915_MADV_WILLNEED:
4337             break;
4338         default:
4339             return -EINVAL;
4340         }
4341
4342         ret = i915_mutex_lock_interruptible(dev);
4343         if (ret)
4344                 return ret;
4345
4346         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4347         if (obj == NULL) {
4348                 ret = -ENOENT;
4349                 goto unlock;
4350         }
4351         obj_priv = to_intel_bo(obj);
4352
4353         if (obj_priv->pin_count) {
4354                 ret = -EINVAL;
4355                 goto out;
4356         }
4357
4358         if (obj_priv->madv != __I915_MADV_PURGED)
4359                 obj_priv->madv = args->madv;
4360
4361         /* if the object is no longer bound, discard its backing storage */
4362         if (i915_gem_object_is_purgeable(obj_priv) &&
4363             obj_priv->gtt_space == NULL)
4364                 i915_gem_object_truncate(obj);
4365
4366         args->retained = obj_priv->madv != __I915_MADV_PURGED;
4367
4368 out:
4369         drm_gem_object_unreference(obj);
4370 unlock:
4371         mutex_unlock(&dev->struct_mutex);
4372         return ret;
4373 }
4374
4375 struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4376                                               size_t size)
4377 {
4378         struct drm_i915_private *dev_priv = dev->dev_private;
4379         struct drm_i915_gem_object *obj;
4380
4381         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4382         if (obj == NULL)
4383                 return NULL;
4384
4385         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4386                 kfree(obj);
4387                 return NULL;
4388         }
4389
4390         i915_gem_info_add_obj(dev_priv, size);
4391
4392         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4393         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4394
4395         obj->agp_type = AGP_USER_MEMORY;
4396         obj->base.driver_private = NULL;
4397         obj->fence_reg = I915_FENCE_REG_NONE;
4398         INIT_LIST_HEAD(&obj->mm_list);
4399         INIT_LIST_HEAD(&obj->ring_list);
4400         INIT_LIST_HEAD(&obj->gpu_write_list);
4401         obj->madv = I915_MADV_WILLNEED;
4402
4403         return &obj->base;
4404 }
4405
4406 int i915_gem_init_object(struct drm_gem_object *obj)
4407 {
4408         BUG();
4409
4410         return 0;
4411 }
4412
4413 static void i915_gem_free_object_tail(struct drm_gem_object *obj)
4414 {
4415         struct drm_device *dev = obj->dev;
4416         drm_i915_private_t *dev_priv = dev->dev_private;
4417         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4418         int ret;
4419
4420         ret = i915_gem_object_unbind(obj);
4421         if (ret == -ERESTARTSYS) {
4422                 list_move(&obj_priv->mm_list,
4423                           &dev_priv->mm.deferred_free_list);
4424                 return;
4425         }
4426
4427         if (obj->map_list.map)
4428                 i915_gem_free_mmap_offset(obj);
4429
4430         drm_gem_object_release(obj);
4431         i915_gem_info_remove_obj(dev_priv, obj->size);
4432
4433         kfree(obj_priv->page_cpu_valid);
4434         kfree(obj_priv->bit_17);
4435         kfree(obj_priv);
4436 }
4437
4438 void i915_gem_free_object(struct drm_gem_object *obj)
4439 {
4440         struct drm_device *dev = obj->dev;
4441         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4442
4443         trace_i915_gem_object_destroy(obj);
4444
4445         while (obj_priv->pin_count > 0)
4446                 i915_gem_object_unpin(obj);
4447
4448         if (obj_priv->phys_obj)
4449                 i915_gem_detach_phys_object(dev, obj);
4450
4451         i915_gem_free_object_tail(obj);
4452 }
4453
4454 int
4455 i915_gem_idle(struct drm_device *dev)
4456 {
4457         drm_i915_private_t *dev_priv = dev->dev_private;
4458         int ret;
4459
4460         mutex_lock(&dev->struct_mutex);
4461
4462         if (dev_priv->mm.suspended) {
4463                 mutex_unlock(&dev->struct_mutex);
4464                 return 0;
4465         }
4466
4467         ret = i915_gpu_idle(dev);
4468         if (ret) {
4469                 mutex_unlock(&dev->struct_mutex);
4470                 return ret;
4471         }
4472
4473         /* Under UMS, be paranoid and evict. */
4474         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4475                 ret = i915_gem_evict_inactive(dev);
4476                 if (ret) {
4477                         mutex_unlock(&dev->struct_mutex);
4478                         return ret;
4479                 }
4480         }
4481
4482         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
4483          * We need to replace this with a semaphore, or something.
4484          * And not confound mm.suspended!
4485          */
4486         dev_priv->mm.suspended = 1;
4487         del_timer_sync(&dev_priv->hangcheck_timer);
4488
4489         i915_kernel_lost_context(dev);
4490         i915_gem_cleanup_ringbuffer(dev);
4491
4492         mutex_unlock(&dev->struct_mutex);
4493
4494         /* Cancel the retire work handler, which should be idle now. */
4495         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4496
4497         return 0;
4498 }
4499
4500 /*
4501  * 965+ support PIPE_CONTROL commands, which provide finer grained control
4502  * over cache flushing.
4503  */
4504 static int
4505 i915_gem_init_pipe_control(struct drm_device *dev)
4506 {
4507         drm_i915_private_t *dev_priv = dev->dev_private;
4508         struct drm_gem_object *obj;
4509         struct drm_i915_gem_object *obj_priv;
4510         int ret;
4511
4512         obj = i915_gem_alloc_object(dev, 4096);
4513         if (obj == NULL) {
4514                 DRM_ERROR("Failed to allocate seqno page\n");
4515                 ret = -ENOMEM;
4516                 goto err;
4517         }
4518         obj_priv = to_intel_bo(obj);
4519         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4520
4521         ret = i915_gem_object_pin(obj, 4096, true);
4522         if (ret)
4523                 goto err_unref;
4524
4525         dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4526         dev_priv->seqno_page =  kmap(obj_priv->pages[0]);
4527         if (dev_priv->seqno_page == NULL)
4528                 goto err_unpin;
4529
4530         dev_priv->seqno_obj = obj;
4531         memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4532
4533         return 0;
4534
4535 err_unpin:
4536         i915_gem_object_unpin(obj);
4537 err_unref:
4538         drm_gem_object_unreference(obj);
4539 err:
4540         return ret;
4541 }
4542
4543
4544 static void
4545 i915_gem_cleanup_pipe_control(struct drm_device *dev)
4546 {
4547         drm_i915_private_t *dev_priv = dev->dev_private;
4548         struct drm_gem_object *obj;
4549         struct drm_i915_gem_object *obj_priv;
4550
4551         obj = dev_priv->seqno_obj;
4552         obj_priv = to_intel_bo(obj);
4553         kunmap(obj_priv->pages[0]);
4554         i915_gem_object_unpin(obj);
4555         drm_gem_object_unreference(obj);
4556         dev_priv->seqno_obj = NULL;
4557
4558         dev_priv->seqno_page = NULL;
4559 }
4560
4561 int
4562 i915_gem_init_ringbuffer(struct drm_device *dev)
4563 {
4564         drm_i915_private_t *dev_priv = dev->dev_private;
4565         int ret;
4566
4567         if (HAS_PIPE_CONTROL(dev)) {
4568                 ret = i915_gem_init_pipe_control(dev);
4569                 if (ret)
4570                         return ret;
4571         }
4572
4573         ret = intel_init_render_ring_buffer(dev);
4574         if (ret)
4575                 goto cleanup_pipe_control;
4576
4577         if (HAS_BSD(dev)) {
4578                 ret = intel_init_bsd_ring_buffer(dev);
4579                 if (ret)
4580                         goto cleanup_render_ring;
4581         }
4582
4583         if (HAS_BLT(dev)) {
4584                 ret = intel_init_blt_ring_buffer(dev);
4585                 if (ret)
4586                         goto cleanup_bsd_ring;
4587         }
4588
4589         dev_priv->next_seqno = 1;
4590
4591         return 0;
4592
4593 cleanup_bsd_ring:
4594         intel_cleanup_ring_buffer(&dev_priv->bsd_ring);
4595 cleanup_render_ring:
4596         intel_cleanup_ring_buffer(&dev_priv->render_ring);
4597 cleanup_pipe_control:
4598         if (HAS_PIPE_CONTROL(dev))
4599                 i915_gem_cleanup_pipe_control(dev);
4600         return ret;
4601 }
4602
4603 void
4604 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4605 {
4606         drm_i915_private_t *dev_priv = dev->dev_private;
4607
4608         intel_cleanup_ring_buffer(&dev_priv->render_ring);
4609         intel_cleanup_ring_buffer(&dev_priv->bsd_ring);
4610         intel_cleanup_ring_buffer(&dev_priv->blt_ring);
4611         if (HAS_PIPE_CONTROL(dev))
4612                 i915_gem_cleanup_pipe_control(dev);
4613 }
4614
4615 int
4616 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4617                        struct drm_file *file_priv)
4618 {
4619         drm_i915_private_t *dev_priv = dev->dev_private;
4620         int ret;
4621
4622         if (drm_core_check_feature(dev, DRIVER_MODESET))
4623                 return 0;
4624
4625         if (atomic_read(&dev_priv->mm.wedged)) {
4626                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4627                 atomic_set(&dev_priv->mm.wedged, 0);
4628         }
4629
4630         mutex_lock(&dev->struct_mutex);
4631         dev_priv->mm.suspended = 0;
4632
4633         ret = i915_gem_init_ringbuffer(dev);
4634         if (ret != 0) {
4635                 mutex_unlock(&dev->struct_mutex);
4636                 return ret;
4637         }
4638
4639         BUG_ON(!list_empty(&dev_priv->mm.active_list));
4640         BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
4641         BUG_ON(!list_empty(&dev_priv->bsd_ring.active_list));
4642         BUG_ON(!list_empty(&dev_priv->blt_ring.active_list));
4643         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4644         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4645         BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
4646         BUG_ON(!list_empty(&dev_priv->bsd_ring.request_list));
4647         BUG_ON(!list_empty(&dev_priv->blt_ring.request_list));
4648         mutex_unlock(&dev->struct_mutex);
4649
4650         ret = drm_irq_install(dev);
4651         if (ret)
4652                 goto cleanup_ringbuffer;
4653
4654         return 0;
4655
4656 cleanup_ringbuffer:
4657         mutex_lock(&dev->struct_mutex);
4658         i915_gem_cleanup_ringbuffer(dev);
4659         dev_priv->mm.suspended = 1;
4660         mutex_unlock(&dev->struct_mutex);
4661
4662         return ret;
4663 }
4664
4665 int
4666 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4667                        struct drm_file *file_priv)
4668 {
4669         if (drm_core_check_feature(dev, DRIVER_MODESET))
4670                 return 0;
4671
4672         drm_irq_uninstall(dev);
4673         return i915_gem_idle(dev);
4674 }
4675
4676 void
4677 i915_gem_lastclose(struct drm_device *dev)
4678 {
4679         int ret;
4680
4681         if (drm_core_check_feature(dev, DRIVER_MODESET))
4682                 return;
4683
4684         ret = i915_gem_idle(dev);
4685         if (ret)
4686                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4687 }
4688
4689 static void
4690 init_ring_lists(struct intel_ring_buffer *ring)
4691 {
4692         INIT_LIST_HEAD(&ring->active_list);
4693         INIT_LIST_HEAD(&ring->request_list);
4694         INIT_LIST_HEAD(&ring->gpu_write_list);
4695 }
4696
4697 void
4698 i915_gem_load(struct drm_device *dev)
4699 {
4700         int i;
4701         drm_i915_private_t *dev_priv = dev->dev_private;
4702
4703         INIT_LIST_HEAD(&dev_priv->mm.active_list);
4704         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4705         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4706         INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
4707         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4708         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
4709         init_ring_lists(&dev_priv->render_ring);
4710         init_ring_lists(&dev_priv->bsd_ring);
4711         init_ring_lists(&dev_priv->blt_ring);
4712         for (i = 0; i < 16; i++)
4713                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4714         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4715                           i915_gem_retire_work_handler);
4716         init_completion(&dev_priv->error_completion);
4717
4718         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4719         if (IS_GEN3(dev)) {
4720                 u32 tmp = I915_READ(MI_ARB_STATE);
4721                 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4722                         /* arb state is a masked write, so set bit + bit in mask */
4723                         tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4724                         I915_WRITE(MI_ARB_STATE, tmp);
4725                 }
4726         }
4727
4728         /* Old X drivers will take 0-2 for front, back, depth buffers */
4729         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4730                 dev_priv->fence_reg_start = 3;
4731
4732         if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4733                 dev_priv->num_fence_regs = 16;
4734         else
4735                 dev_priv->num_fence_regs = 8;
4736
4737         /* Initialize fence registers to zero */
4738         switch (INTEL_INFO(dev)->gen) {
4739         case 6:
4740                 for (i = 0; i < 16; i++)
4741                         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), 0);
4742                 break;
4743         case 5:
4744         case 4:
4745                 for (i = 0; i < 16; i++)
4746                         I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4747                 break;
4748         case 3:
4749                 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4750                         for (i = 0; i < 8; i++)
4751                                 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4752         case 2:
4753                 for (i = 0; i < 8; i++)
4754                         I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4755                 break;
4756         }
4757         i915_gem_detect_bit_6_swizzle(dev);
4758         init_waitqueue_head(&dev_priv->pending_flip_queue);
4759
4760         dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
4761         dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
4762         register_shrinker(&dev_priv->mm.inactive_shrinker);
4763 }
4764
4765 /*
4766  * Create a physically contiguous memory object for this object
4767  * e.g. for cursor + overlay regs
4768  */
4769 static int i915_gem_init_phys_object(struct drm_device *dev,
4770                                      int id, int size, int align)
4771 {
4772         drm_i915_private_t *dev_priv = dev->dev_private;
4773         struct drm_i915_gem_phys_object *phys_obj;
4774         int ret;
4775
4776         if (dev_priv->mm.phys_objs[id - 1] || !size)
4777                 return 0;
4778
4779         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4780         if (!phys_obj)
4781                 return -ENOMEM;
4782
4783         phys_obj->id = id;
4784
4785         phys_obj->handle = drm_pci_alloc(dev, size, align);
4786         if (!phys_obj->handle) {
4787                 ret = -ENOMEM;
4788                 goto kfree_obj;
4789         }
4790 #ifdef CONFIG_X86
4791         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4792 #endif
4793
4794         dev_priv->mm.phys_objs[id - 1] = phys_obj;
4795
4796         return 0;
4797 kfree_obj:
4798         kfree(phys_obj);
4799         return ret;
4800 }
4801
4802 static void i915_gem_free_phys_object(struct drm_device *dev, int id)
4803 {
4804         drm_i915_private_t *dev_priv = dev->dev_private;
4805         struct drm_i915_gem_phys_object *phys_obj;
4806
4807         if (!dev_priv->mm.phys_objs[id - 1])
4808                 return;
4809
4810         phys_obj = dev_priv->mm.phys_objs[id - 1];
4811         if (phys_obj->cur_obj) {
4812                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4813         }
4814
4815 #ifdef CONFIG_X86
4816         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4817 #endif
4818         drm_pci_free(dev, phys_obj->handle);
4819         kfree(phys_obj);
4820         dev_priv->mm.phys_objs[id - 1] = NULL;
4821 }
4822
4823 void i915_gem_free_all_phys_object(struct drm_device *dev)
4824 {
4825         int i;
4826
4827         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4828                 i915_gem_free_phys_object(dev, i);
4829 }
4830
4831 void i915_gem_detach_phys_object(struct drm_device *dev,
4832                                  struct drm_gem_object *obj)
4833 {
4834         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
4835         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4836         char *vaddr;
4837         int i;
4838         int page_count;
4839
4840         if (!obj_priv->phys_obj)
4841                 return;
4842         vaddr = obj_priv->phys_obj->handle->vaddr;
4843
4844         page_count = obj->size / PAGE_SIZE;
4845
4846         for (i = 0; i < page_count; i++) {
4847                 struct page *page = read_cache_page_gfp(mapping, i,
4848                                                         GFP_HIGHUSER | __GFP_RECLAIMABLE);
4849                 if (!IS_ERR(page)) {
4850                         char *dst = kmap_atomic(page);
4851                         memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
4852                         kunmap_atomic(dst);
4853
4854                         drm_clflush_pages(&page, 1);
4855
4856                         set_page_dirty(page);
4857                         mark_page_accessed(page);
4858                         page_cache_release(page);
4859                 }
4860         }
4861         drm_agp_chipset_flush(dev);
4862
4863         obj_priv->phys_obj->cur_obj = NULL;
4864         obj_priv->phys_obj = NULL;
4865 }
4866
4867 int
4868 i915_gem_attach_phys_object(struct drm_device *dev,
4869                             struct drm_gem_object *obj,
4870                             int id,
4871                             int align)
4872 {
4873         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
4874         drm_i915_private_t *dev_priv = dev->dev_private;
4875         struct drm_i915_gem_object *obj_priv;
4876         int ret = 0;
4877         int page_count;
4878         int i;
4879
4880         if (id > I915_MAX_PHYS_OBJECT)
4881                 return -EINVAL;
4882
4883         obj_priv = to_intel_bo(obj);
4884
4885         if (obj_priv->phys_obj) {
4886                 if (obj_priv->phys_obj->id == id)
4887                         return 0;
4888                 i915_gem_detach_phys_object(dev, obj);
4889         }
4890
4891         /* create a new object */
4892         if (!dev_priv->mm.phys_objs[id - 1]) {
4893                 ret = i915_gem_init_phys_object(dev, id,
4894                                                 obj->size, align);
4895                 if (ret) {
4896                         DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
4897                         return ret;
4898                 }
4899         }
4900
4901         /* bind to the object */
4902         obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4903         obj_priv->phys_obj->cur_obj = obj;
4904
4905         page_count = obj->size / PAGE_SIZE;
4906
4907         for (i = 0; i < page_count; i++) {
4908                 struct page *page;
4909                 char *dst, *src;
4910
4911                 page = read_cache_page_gfp(mapping, i,
4912                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
4913                 if (IS_ERR(page))
4914                         return PTR_ERR(page);
4915
4916                 src = kmap_atomic(obj_priv->pages[i]);
4917                 dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4918                 memcpy(dst, src, PAGE_SIZE);
4919                 kunmap_atomic(src);
4920
4921                 mark_page_accessed(page);
4922                 page_cache_release(page);
4923         }
4924
4925         return 0;
4926 }
4927
4928 static int
4929 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4930                      struct drm_i915_gem_pwrite *args,
4931                      struct drm_file *file_priv)
4932 {
4933         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4934         void *obj_addr;
4935         int ret;
4936         char __user *user_data;
4937
4938         user_data = (char __user *) (uintptr_t) args->data_ptr;
4939         obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4940
4941         DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
4942         ret = copy_from_user(obj_addr, user_data, args->size);
4943         if (ret)
4944                 return -EFAULT;
4945
4946         drm_agp_chipset_flush(dev);
4947         return 0;
4948 }
4949
4950 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4951 {
4952         struct drm_i915_file_private *file_priv = file->driver_priv;
4953
4954         /* Clean up our request list when the client is going away, so that
4955          * later retire_requests won't dereference our soon-to-be-gone
4956          * file_priv.
4957          */
4958         spin_lock(&file_priv->mm.lock);
4959         while (!list_empty(&file_priv->mm.request_list)) {
4960                 struct drm_i915_gem_request *request;
4961
4962                 request = list_first_entry(&file_priv->mm.request_list,
4963                                            struct drm_i915_gem_request,
4964                                            client_list);
4965                 list_del(&request->client_list);
4966                 request->file_priv = NULL;
4967         }
4968         spin_unlock(&file_priv->mm.lock);
4969 }
4970
4971 static int
4972 i915_gpu_is_active(struct drm_device *dev)
4973 {
4974         drm_i915_private_t *dev_priv = dev->dev_private;
4975         int lists_empty;
4976
4977         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
4978                       list_empty(&dev_priv->mm.active_list);
4979
4980         return !lists_empty;
4981 }
4982
4983 static int
4984 i915_gem_inactive_shrink(struct shrinker *shrinker,
4985                          int nr_to_scan,
4986                          gfp_t gfp_mask)
4987 {
4988         struct drm_i915_private *dev_priv =
4989                 container_of(shrinker,
4990                              struct drm_i915_private,
4991                              mm.inactive_shrinker);
4992         struct drm_device *dev = dev_priv->dev;
4993         struct drm_i915_gem_object *obj, *next;
4994         int cnt;
4995
4996         if (!mutex_trylock(&dev->struct_mutex))
4997                 return nr_to_scan ? 0 : -1;
4998
4999         /* "fast-path" to count number of available objects */
5000         if (nr_to_scan == 0) {
5001                 cnt = 0;
5002                 list_for_each_entry(obj,
5003                                     &dev_priv->mm.inactive_list,
5004                                     mm_list)
5005                         cnt++;
5006                 mutex_unlock(&dev->struct_mutex);
5007                 return cnt / 100 * sysctl_vfs_cache_pressure;
5008         }
5009
5010 rescan:
5011         /* first scan for clean buffers */
5012         i915_gem_retire_requests(dev);
5013
5014         list_for_each_entry_safe(obj, next,
5015                                  &dev_priv->mm.inactive_list,
5016                                  mm_list) {
5017                 if (i915_gem_object_is_purgeable(obj)) {
5018                         i915_gem_object_unbind(&obj->base);
5019                         if (--nr_to_scan == 0)
5020                                 break;
5021                 }
5022         }
5023
5024         /* second pass, evict/count anything still on the inactive list */
5025         cnt = 0;
5026         list_for_each_entry_safe(obj, next,
5027                                  &dev_priv->mm.inactive_list,
5028                                  mm_list) {
5029                 if (nr_to_scan) {
5030                         i915_gem_object_unbind(&obj->base);
5031                         nr_to_scan--;
5032                 } else
5033                         cnt++;
5034         }
5035
5036         if (nr_to_scan && i915_gpu_is_active(dev)) {
5037                 /*
5038                  * We are desperate for pages, so as a last resort, wait
5039                  * for the GPU to finish and discard whatever we can.
5040                  * This has a dramatic impact to reduce the number of
5041                  * OOM-killer events whilst running the GPU aggressively.
5042                  */
5043                 if (i915_gpu_idle(dev) == 0)
5044                         goto rescan;
5045         }
5046         mutex_unlock(&dev->struct_mutex);
5047         return cnt / 100 * sysctl_vfs_cache_pressure;
5048 }