drivers/gpu/drm/nouveau/nouveau_bo.c

   1 /*
   2  * Copyright 2007 Dave Airlied
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  */
  24 /*
  25  * Authors: Dave Airlied <airlied@linux.ie>
  26  *          Ben Skeggs   <darktama@iinet.net.au>
  27  *          Jeremy Kolb  <jkolb@brandeis.edu>
  28  */
  29
  30 #include "drmP.h"
  31 #include "ttm/ttm_page_alloc.h"
  32
  33 #include "nouveau_drm.h"
  34 #include "nouveau_drv.h"
  35 #include "nouveau_dma.h"
  36 #include "nouveau_mm.h"
  37 #include "nouveau_vm.h"
  38 #include "nouveau_fence.h"
  39 #include "nouveau_ramht.h"
  40
  41 #include <linux/log2.h>
  42 #include <linux/slab.h>
  43
  44 static void
  45 nouveau_bo_del_ttm(struct ttm_buffer_object *bo)
  46 {
  47         struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
  48         struct drm_device *dev = dev_priv->dev;
  49         struct nouveau_bo *nvbo = nouveau_bo(bo);
  50
  51         if (unlikely(nvbo->gem))
  52                 DRM_ERROR("bo %p still attached to GEM object\n", bo);
  53
  54         nv10_mem_put_tile_region(dev, nvbo->tile, NULL);
  55         kfree(nvbo);
  56 }
  57
  58 static void
  59 nouveau_bo_fixup_align(struct nouveau_bo *nvbo, u32 flags,
  60                        int *align, int *size)
  61 {
  62         struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev);
  63
  64         if (dev_priv->card_type < NV_50) {
  65                 if (nvbo->tile_mode) {
  66                         if (dev_priv->chipset >= 0x40) {
  67                                 *align = 65536;
  68                                 *size = roundup(*size, 64 * nvbo->tile_mode);
  69
  70                         } else if (dev_priv->chipset >= 0x30) {
  71                                 *align = 32768;
  72                                 *size = roundup(*size, 64 * nvbo->tile_mode);
  73
  74                         } else if (dev_priv->chipset >= 0x20) {
  75                                 *align = 16384;
  76                                 *size = roundup(*size, 64 * nvbo->tile_mode);
  77
  78                         } else if (dev_priv->chipset >= 0x10) {
  79                                 *align = 16384;
  80                                 *size = roundup(*size, 32 * nvbo->tile_mode);
  81                         }
  82                 }
  83         } else {
  84                 *size = roundup(*size, (1 << nvbo->page_shift));
  85                 *align = max((1 <<  nvbo->page_shift), *align);
  86         }
  87
  88         *size = roundup(*size, PAGE_SIZE);
  89 }
  90
  91 int
  92 nouveau_bo_new(struct drm_device *dev, int size, int align,
  93                uint32_t flags, uint32_t tile_mode, uint32_t tile_flags,
  94                struct sg_table *sg,
  95                struct nouveau_bo **pnvbo)
  96 {
  97         struct drm_nouveau_private *dev_priv = dev->dev_private;
  98         struct nouveau_bo *nvbo;
  99         size_t acc_size;
 100         int ret;
 101         int type = ttm_bo_type_device;
 102
 103         if (sg)
 104                 type = ttm_bo_type_sg;
 105
 106         nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL);
 107         if (!nvbo)
 108                 return -ENOMEM;
 109         INIT_LIST_HEAD(&nvbo->head);
 110         INIT_LIST_HEAD(&nvbo->entry);
 111         INIT_LIST_HEAD(&nvbo->vma_list);
 112         nvbo->tile_mode = tile_mode;
 113         nvbo->tile_flags = tile_flags;
 114         nvbo->bo.bdev = &dev_priv->ttm.bdev;
 115
 116         nvbo->page_shift = 12;
 117         if (dev_priv->bar1_vm) {
 118                 if (!(flags & TTM_PL_FLAG_TT) && size > 256 * 1024)
 119                         nvbo->page_shift = dev_priv->bar1_vm->lpg_shift;
 120         }
 121
 122         nouveau_bo_fixup_align(nvbo, flags, &align, &size);
 123         nvbo->bo.mem.num_pages = size >> PAGE_SHIFT;
 124         nouveau_bo_placement_set(nvbo, flags, 0);
 125
 126         acc_size = ttm_bo_dma_acc_size(&dev_priv->ttm.bdev, size,
 127                                        sizeof(struct nouveau_bo));
 128
 129         ret = ttm_bo_init(&dev_priv->ttm.bdev, &nvbo->bo, size,
 130                           type, &nvbo->placement,
 131                           align >> PAGE_SHIFT, 0, false, NULL, acc_size, sg,
 132                           nouveau_bo_del_ttm);
 133         if (ret) {
 134                 /* ttm will call nouveau_bo_del_ttm if it fails.. */
 135                 return ret;
 136         }
 137
 138         *pnvbo = nvbo;
 139         return 0;
 140 }
 141
 142 static void
 143 set_placement_list(uint32_t *pl, unsigned *n, uint32_t type, uint32_t flags)
 144 {
 145         *n = 0;
 146
 147         if (type & TTM_PL_FLAG_VRAM)
 148                 pl[(*n)++] = TTM_PL_FLAG_VRAM | flags;
 149         if (type & TTM_PL_FLAG_TT)
 150                 pl[(*n)++] = TTM_PL_FLAG_TT | flags;
 151         if (type & TTM_PL_FLAG_SYSTEM)
 152                 pl[(*n)++] = TTM_PL_FLAG_SYSTEM | flags;
 153 }
 154
 155 static void
 156 set_placement_range(struct nouveau_bo *nvbo, uint32_t type)
 157 {
 158         struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev);
 159         int vram_pages = dev_priv->vram_size >> PAGE_SHIFT;
 160
 161         if (dev_priv->card_type == NV_10 &&
 162             nvbo->tile_mode && (type & TTM_PL_FLAG_VRAM) &&
 163             nvbo->bo.mem.num_pages < vram_pages / 4) {
 164                 /*
 165                  * Make sure that the color and depth buffers are handled
 166                  * by independent memory controller units. Up to a 9x
 167                  * speed up when alpha-blending and depth-test are enabled
 168                  * at the same time.
 169                  */
 170                 if (nvbo->tile_flags & NOUVEAU_GEM_TILE_ZETA) {
 171                         nvbo->placement.fpfn = vram_pages / 2;
 172                         nvbo->placement.lpfn = ~0;
 173                 } else {
 174                         nvbo->placement.fpfn = 0;
 175                         nvbo->placement.lpfn = vram_pages / 2;
 176                 }
 177         }
 178 }
 179
 180 void
 181 nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t type, uint32_t busy)
 182 {
 183         struct ttm_placement *pl = &nvbo->placement;
 184         uint32_t flags = TTM_PL_MASK_CACHING |
 185                 (nvbo->pin_refcnt ? TTM_PL_FLAG_NO_EVICT : 0);
 186
 187         pl->placement = nvbo->placements;
 188         set_placement_list(nvbo->placements, &pl->num_placement,
 189                            type, flags);
 190
 191         pl->busy_placement = nvbo->busy_placements;
 192         set_placement_list(nvbo->busy_placements, &pl->num_busy_placement,
 193                            type | busy, flags);
 194
 195         set_placement_range(nvbo, type);
 196 }
 197
 198 int
 199 nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype)
 200 {
 201         struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev);
 202         struct ttm_buffer_object *bo = &nvbo->bo;
 203         int ret;
 204
 205         if (nvbo->pin_refcnt && !(memtype & (1 << bo->mem.mem_type))) {
 206                 NV_ERROR(nouveau_bdev(bo->bdev)->dev,
 207                          "bo %p pinned elsewhere: 0x%08x vs 0x%08x\n", bo,
 208                          1 << bo->mem.mem_type, memtype);
 209                 return -EINVAL;
 210         }
 211
 212         if (nvbo->pin_refcnt++)
 213                 return 0;
 214
 215         ret = ttm_bo_reserve(bo, false, false, false, 0);
 216         if (ret)
 217                 goto out;
 218
 219         nouveau_bo_placement_set(nvbo, memtype, 0);
 220
 221         ret = nouveau_bo_validate(nvbo, false, false, false);
 222         if (ret == 0) {
 223                 switch (bo->mem.mem_type) {
 224                 case TTM_PL_VRAM:
 225                         dev_priv->fb_aper_free -= bo->mem.size;
 226                         break;
 227                 case TTM_PL_TT:
 228                         dev_priv->gart_info.aper_free -= bo->mem.size;
 229                         break;
 230                 default:
 231                         break;
 232                 }
 233         }
 234         ttm_bo_unreserve(bo);
 235 out:
 236         if (unlikely(ret))
 237                 nvbo->pin_refcnt--;
 238         return ret;
 239 }
 240
 241 int
 242 nouveau_bo_unpin(struct nouveau_bo *nvbo)
 243 {
 244         struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev);
 245         struct ttm_buffer_object *bo = &nvbo->bo;
 246         int ret;
 247
 248         if (--nvbo->pin_refcnt)
 249                 return 0;
 250
 251         ret = ttm_bo_reserve(bo, false, false, false, 0);
 252         if (ret)
 253                 return ret;
 254
 255         nouveau_bo_placement_set(nvbo, bo->mem.placement, 0);
 256
 257         ret = nouveau_bo_validate(nvbo, false, false, false);
 258         if (ret == 0) {
 259                 switch (bo->mem.mem_type) {
 260                 case TTM_PL_VRAM:
 261                         dev_priv->fb_aper_free += bo->mem.size;
 262                         break;
 263                 case TTM_PL_TT:
 264                         dev_priv->gart_info.aper_free += bo->mem.size;
 265                         break;
 266                 default:
 267                         break;
 268                 }
 269         }
 270
 271         ttm_bo_unreserve(bo);
 272         return ret;
 273 }
 274
 275 int
 276 nouveau_bo_map(struct nouveau_bo *nvbo)
 277 {
 278         int ret;
 279
 280         ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0);
 281         if (ret)
 282                 return ret;
 283
 284         ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, &nvbo->kmap);
 285         ttm_bo_unreserve(&nvbo->bo);
 286         return ret;
 287 }
 288
 289 void
 290 nouveau_bo_unmap(struct nouveau_bo *nvbo)
 291 {
 292         if (nvbo)
 293                 ttm_bo_kunmap(&nvbo->kmap);
 294 }
 295
 296 int
 297 nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible,
 298                     bool no_wait_reserve, bool no_wait_gpu)
 299 {
 300         int ret;
 301
 302         ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement, interruptible,
 303                               no_wait_reserve, no_wait_gpu);
 304         if (ret)
 305                 return ret;
 306
 307         return 0;
 308 }
 309
 310 u16
 311 nouveau_bo_rd16(struct nouveau_bo *nvbo, unsigned index)
 312 {
 313         bool is_iomem;
 314         u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
 315         mem = &mem[index];
 316         if (is_iomem)
 317                 return ioread16_native((void __force __iomem *)mem);
 318         else
 319                 return *mem;
 320 }
 321
 322 void
 323 nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val)
 324 {
 325         bool is_iomem;
 326         u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
 327         mem = &mem[index];
 328         if (is_iomem)
 329                 iowrite16_native(val, (void __force __iomem *)mem);
 330         else
 331                 *mem = val;
 332 }
 333
 334 u32
 335 nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index)
 336 {
 337         bool is_iomem;
 338         u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
 339         mem = &mem[index];
 340         if (is_iomem)
 341                 return ioread32_native((void __force __iomem *)mem);
 342         else
 343                 return *mem;
 344 }
 345
 346 void
 347 nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val)
 348 {
 349         bool is_iomem;
 350         u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
 351         mem = &mem[index];
 352         if (is_iomem)
 353                 iowrite32_native(val, (void __force __iomem *)mem);
 354         else
 355                 *mem = val;
 356 }
 357
 358 static struct ttm_tt *
 359 nouveau_ttm_tt_create(struct ttm_bo_device *bdev,
 360                       unsigned long size, uint32_t page_flags,
 361                       struct page *dummy_read_page)
 362 {
 363         struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
 364         struct drm_device *dev = dev_priv->dev;
 365
 366         switch (dev_priv->gart_info.type) {
 367 #if __OS_HAS_AGP
 368         case NOUVEAU_GART_AGP:
 369                 return ttm_agp_tt_create(bdev, dev->agp->bridge,
 370                                          size, page_flags, dummy_read_page);
 371 #endif
 372         case NOUVEAU_GART_PDMA:
 373         case NOUVEAU_GART_HW:
 374                 return nouveau_sgdma_create_ttm(bdev, size, page_flags,
 375                                                 dummy_read_page);
 376         default:
 377                 NV_ERROR(dev, "Unknown GART type %d\n",
 378                          dev_priv->gart_info.type);
 379                 break;
 380         }
 381
 382         return NULL;
 383 }
 384
 385 static int
 386 nouveau_bo_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
 387 {
 388         /* We'll do this from user space. */
 389         return 0;
 390 }
 391
 392 static int
 393 nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
 394                          struct ttm_mem_type_manager *man)
 395 {
 396         struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
 397         struct drm_device *dev = dev_priv->dev;
 398
 399         switch (type) {
 400         case TTM_PL_SYSTEM:
 401                 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
 402                 man->available_caching = TTM_PL_MASK_CACHING;
 403                 man->default_caching = TTM_PL_FLAG_CACHED;
 404                 break;
 405         case TTM_PL_VRAM:
 406                 if (dev_priv->card_type >= NV_50) {
 407                         man->func = &nouveau_vram_manager;
 408                         man->io_reserve_fastpath = false;
 409                         man->use_io_reserve_lru = true;
 410                 } else {
 411                         man->func = &ttm_bo_manager_func;
 412                 }
 413                 man->flags = TTM_MEMTYPE_FLAG_FIXED |
 414                              TTM_MEMTYPE_FLAG_MAPPABLE;
 415                 man->available_caching = TTM_PL_FLAG_UNCACHED |
 416                                          TTM_PL_FLAG_WC;
 417                 man->default_caching = TTM_PL_FLAG_WC;
 418                 break;
 419         case TTM_PL_TT:
 420                 if (dev_priv->card_type >= NV_50)
 421                         man->func = &nouveau_gart_manager;
 422                 else
 423                         man->func = &ttm_bo_manager_func;
 424                 switch (dev_priv->gart_info.type) {
 425                 case NOUVEAU_GART_AGP:
 426                         man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
 427                         man->available_caching = TTM_PL_FLAG_UNCACHED |
 428                                 TTM_PL_FLAG_WC;
 429                         man->default_caching = TTM_PL_FLAG_WC;
 430                         break;
 431                 case NOUVEAU_GART_PDMA:
 432                 case NOUVEAU_GART_HW:
 433                         man->flags = TTM_MEMTYPE_FLAG_MAPPABLE |
 434                                      TTM_MEMTYPE_FLAG_CMA;
 435                         man->available_caching = TTM_PL_MASK_CACHING;
 436                         man->default_caching = TTM_PL_FLAG_CACHED;
 437                         break;
 438                 default:
 439                         NV_ERROR(dev, "Unknown GART type: %d\n",
 440                                  dev_priv->gart_info.type);
 441                         return -EINVAL;
 442                 }
 443                 break;
 444         default:
 445                 NV_ERROR(dev, "Unsupported memory type %u\n", (unsigned)type);
 446                 return -EINVAL;
 447         }
 448         return 0;
 449 }
 450
 451 static void
 452 nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl)
 453 {
 454         struct nouveau_bo *nvbo = nouveau_bo(bo);
 455
 456         switch (bo->mem.mem_type) {
 457         case TTM_PL_VRAM:
 458                 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT,
 459                                          TTM_PL_FLAG_SYSTEM);
 460                 break;
 461         default:
 462                 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_SYSTEM, 0);
 463                 break;
 464         }
 465
 466         *pl = nvbo->placement;
 467 }
 468
 469
 470 /* GPU-assisted copy using NV_MEMORY_TO_MEMORY_FORMAT, can access
 471  * TTM_PL_{VRAM,TT} directly.
 472  */
 473
 474 static int
 475 nouveau_bo_move_accel_cleanup(struct nouveau_channel *chan,
 476                               struct nouveau_bo *nvbo, bool evict,
 477                               bool no_wait_reserve, bool no_wait_gpu,
 478                               struct ttm_mem_reg *new_mem)
 479 {
 480         struct nouveau_fence *fence = NULL;
 481         int ret;
 482
 483         ret = nouveau_fence_new(chan, &fence);
 484         if (ret)
 485                 return ret;
 486
 487         ret = ttm_bo_move_accel_cleanup(&nvbo->bo, fence, NULL, evict,
 488                                         no_wait_reserve, no_wait_gpu, new_mem);
 489         nouveau_fence_unref(&fence);
 490         return ret;
 491 }
 492
 493 static int
 494 nve0_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
 495                   struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
 496 {
 497         struct nouveau_mem *node = old_mem->mm_node;
 498         int ret = RING_SPACE(chan, 10);
 499         if (ret == 0) {
 500                 BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8);
 501                 OUT_RING  (chan, upper_32_bits(node->vma[0].offset));
 502                 OUT_RING  (chan, lower_32_bits(node->vma[0].offset));
 503                 OUT_RING  (chan, upper_32_bits(node->vma[1].offset));
 504                 OUT_RING  (chan, lower_32_bits(node->vma[1].offset));
 505                 OUT_RING  (chan, PAGE_SIZE);
 506                 OUT_RING  (chan, PAGE_SIZE);
 507                 OUT_RING  (chan, PAGE_SIZE);
 508                 OUT_RING  (chan, new_mem->num_pages);
 509                 BEGIN_IMC0(chan, NvSubCopy, 0x0300, 0x0386);
 510         }
 511         return ret;
 512 }
 513
 514 static int
 515 nvc0_bo_move_init(struct nouveau_channel *chan, u32 handle)
 516 {
 517         int ret = RING_SPACE(chan, 2);
 518         if (ret == 0) {
 519                 BEGIN_NVC0(chan, NvSubCopy, 0x0000, 1);
 520                 OUT_RING  (chan, handle);
 521         }
 522         return ret;
 523 }
 524
 525 static int
 526 nvc0_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
 527                   struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
 528 {
 529         struct nouveau_mem *node = old_mem->mm_node;
 530         u64 src_offset = node->vma[0].offset;
 531         u64 dst_offset = node->vma[1].offset;
 532         u32 page_count = new_mem->num_pages;
 533         int ret;
 534
 535         page_count = new_mem->num_pages;
 536         while (page_count) {
 537                 int line_count = (page_count > 8191) ? 8191 : page_count;
 538
 539                 ret = RING_SPACE(chan, 11);
 540                 if (ret)
 541                         return ret;
 542
 543                 BEGIN_NVC0(chan, NvSubCopy, 0x030c, 8);
 544                 OUT_RING  (chan, upper_32_bits(src_offset));
 545                 OUT_RING  (chan, lower_32_bits(src_offset));
 546                 OUT_RING  (chan, upper_32_bits(dst_offset));
 547                 OUT_RING  (chan, lower_32_bits(dst_offset));
 548                 OUT_RING  (chan, PAGE_SIZE);
 549                 OUT_RING  (chan, PAGE_SIZE);
 550                 OUT_RING  (chan, PAGE_SIZE);
 551                 OUT_RING  (chan, line_count);
 552                 BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
 553                 OUT_RING  (chan, 0x00000110);
 554
 555                 page_count -= line_count;
 556                 src_offset += (PAGE_SIZE * line_count);
 557                 dst_offset += (PAGE_SIZE * line_count);
 558         }
 559
 560         return 0;
 561 }
 562
 563 static int
 564 nvc0_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
 565                   struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
 566 {
 567         struct nouveau_mem *node = old_mem->mm_node;
 568         u64 src_offset = node->vma[0].offset;
 569         u64 dst_offset = node->vma[1].offset;
 570         u32 page_count = new_mem->num_pages;
 571         int ret;
 572
 573         page_count = new_mem->num_pages;
 574         while (page_count) {
 575                 int line_count = (page_count > 2047) ? 2047 : page_count;
 576
 577                 ret = RING_SPACE(chan, 12);
 578                 if (ret)
 579                         return ret;
 580
 581                 BEGIN_NVC0(chan, NvSubCopy, 0x0238, 2);
 582                 OUT_RING  (chan, upper_32_bits(dst_offset));
 583                 OUT_RING  (chan, lower_32_bits(dst_offset));
 584                 BEGIN_NVC0(chan, NvSubCopy, 0x030c, 6);
 585                 OUT_RING  (chan, upper_32_bits(src_offset));
 586                 OUT_RING  (chan, lower_32_bits(src_offset));
 587                 OUT_RING  (chan, PAGE_SIZE); /* src_pitch */
 588                 OUT_RING  (chan, PAGE_SIZE); /* dst_pitch */
 589                 OUT_RING  (chan, PAGE_SIZE); /* line_length */
 590                 OUT_RING  (chan, line_count);
 591                 BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
 592                 OUT_RING  (chan, 0x00100110);
 593
 594                 page_count -= line_count;
 595                 src_offset += (PAGE_SIZE * line_count);
 596                 dst_offset += (PAGE_SIZE * line_count);
 597         }
 598
 599         return 0;
 600 }
 601
 602 static int
 603 nva3_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
 604                   struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
 605 {
 606         struct nouveau_mem *node = old_mem->mm_node;
 607         u64 src_offset = node->vma[0].offset;
 608         u64 dst_offset = node->vma[1].offset;
 609         u32 page_count = new_mem->num_pages;
 610         int ret;
 611
 612         page_count = new_mem->num_pages;
 613         while (page_count) {
 614                 int line_count = (page_count > 8191) ? 8191 : page_count;
 615
 616                 ret = RING_SPACE(chan, 11);
 617                 if (ret)
 618                         return ret;
 619
 620                 BEGIN_NV04(chan, NvSubCopy, 0x030c, 8);
 621                 OUT_RING  (chan, upper_32_bits(src_offset));
 622                 OUT_RING  (chan, lower_32_bits(src_offset));
 623                 OUT_RING  (chan, upper_32_bits(dst_offset));
 624                 OUT_RING  (chan, lower_32_bits(dst_offset));
 625                 OUT_RING  (chan, PAGE_SIZE);
 626                 OUT_RING  (chan, PAGE_SIZE);
 627                 OUT_RING  (chan, PAGE_SIZE);
 628                 OUT_RING  (chan, line_count);
 629                 BEGIN_NV04(chan, NvSubCopy, 0x0300, 1);
 630                 OUT_RING  (chan, 0x00000110);
 631
 632                 page_count -= line_count;
 633                 src_offset += (PAGE_SIZE * line_count);
 634                 dst_offset += (PAGE_SIZE * line_count);
 635         }
 636
 637         return 0;
 638 }
 639
 640 static int
 641 nv98_bo_move_exec(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
 642                   struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
 643 {
 644         struct nouveau_mem *node = old_mem->mm_node;
 645         int ret = RING_SPACE(chan, 7);
 646         if (ret == 0) {
 647                 BEGIN_NV04(chan, NvSubCopy, 0x0320, 6);
 648                 OUT_RING  (chan, upper_32_bits(node->vma[0].offset));
 649                 OUT_RING  (chan, lower_32_bits(node->vma[0].offset));
 650                 OUT_RING  (chan, upper_32_bits(node->vma[1].offset));
 651                 OUT_RING  (chan, lower_32_bits(node->vma[1].offset));
 652                 OUT_RING  (chan, 0x00000000 /* COPY */);
 653                 OUT_RING  (chan, new_mem->num_pages << PAGE_SHIFT);
 654         }
 655         return ret;
 656 }
 657
 658 static int
 659 nv84_bo_move_exec(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
 660                   struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
 661 {
 662         struct nouveau_mem *node = old_mem->mm_node;
 663         int ret = RING_SPACE(chan, 7);
 664         if (ret == 0) {
 665                 BEGIN_NV04(chan, NvSubCopy, 0x0304, 6);
 666                 OUT_RING  (chan, new_mem->num_pages << PAGE_SHIFT);
 667                 OUT_RING  (chan, upper_32_bits(node->vma[0].offset));
 668                 OUT_RING  (chan, lower_32_bits(node->vma[0].offset));
 669                 OUT_RING  (chan, upper_32_bits(node->vma[1].offset));
 670                 OUT_RING  (chan, lower_32_bits(node->vma[1].offset));
 671                 OUT_RING  (chan, 0x00000000 /* MODE_COPY, QUERY_NONE */);
 672         }
 673         return ret;
 674 }
 675
 676 static int
 677 nv50_bo_move_init(struct nouveau_channel *chan, u32 handle)
 678 {
 679         int ret = nouveau_notifier_alloc(chan, NvNotify0, 32, 0xfe0, 0x1000,
 680                                          &chan->m2mf_ntfy);
 681         if (ret == 0) {
 682                 ret = RING_SPACE(chan, 6);
 683                 if (ret == 0) {
 684                         BEGIN_NV04(chan, NvSubCopy, 0x0000, 1);
 685                         OUT_RING  (chan, handle);
 686                         BEGIN_NV04(chan, NvSubCopy, 0x0180, 3);
 687                         OUT_RING  (chan, NvNotify0);
 688                         OUT_RING  (chan, NvDmaFB);
 689                         OUT_RING  (chan, NvDmaFB);
 690                 } else {
 691                         nouveau_ramht_remove(chan, NvNotify0);
 692                 }
 693         }
 694
 695         return ret;
 696 }
 697
 698 static int
 699 nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
 700                   struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
 701 {
 702         struct nouveau_mem *node = old_mem->mm_node;
 703         struct nouveau_bo *nvbo = nouveau_bo(bo);
 704         u64 length = (new_mem->num_pages << PAGE_SHIFT);
 705         u64 src_offset = node->vma[0].offset;
 706         u64 dst_offset = node->vma[1].offset;
 707         int ret;
 708
 709         while (length) {
 710                 u32 amount, stride, height;
 711
 712                 amount  = min(length, (u64)(4 * 1024 * 1024));
 713                 stride  = 16 * 4;
 714                 height  = amount / stride;
 715
 716                 if (new_mem->mem_type == TTM_PL_VRAM &&
 717                     nouveau_bo_tile_layout(nvbo)) {
 718                         ret = RING_SPACE(chan, 8);
 719                         if (ret)
 720                                 return ret;
 721
 722                         BEGIN_NV04(chan, NvSubCopy, 0x0200, 7);
 723                         OUT_RING  (chan, 0);
 724                         OUT_RING  (chan, 0);
 725                         OUT_RING  (chan, stride);
 726                         OUT_RING  (chan, height);
 727                         OUT_RING  (chan, 1);
 728                         OUT_RING  (chan, 0);
 729                         OUT_RING  (chan, 0);
 730                 } else {
 731                         ret = RING_SPACE(chan, 2);
 732                         if (ret)
 733                                 return ret;
 734
 735                         BEGIN_NV04(chan, NvSubCopy, 0x0200, 1);
 736                         OUT_RING  (chan, 1);
 737                 }
 738                 if (old_mem->mem_type == TTM_PL_VRAM &&
 739                     nouveau_bo_tile_layout(nvbo)) {
 740                         ret = RING_SPACE(chan, 8);
 741                         if (ret)
 742                                 return ret;
 743
 744                         BEGIN_NV04(chan, NvSubCopy, 0x021c, 7);
 745                         OUT_RING  (chan, 0);
 746                         OUT_RING  (chan, 0);
 747                         OUT_RING  (chan, stride);
 748                         OUT_RING  (chan, height);
 749                         OUT_RING  (chan, 1);
 750                         OUT_RING  (chan, 0);
 751                         OUT_RING  (chan, 0);
 752                 } else {
 753                         ret = RING_SPACE(chan, 2);
 754                         if (ret)
 755                                 return ret;
 756
 757                         BEGIN_NV04(chan, NvSubCopy, 0x021c, 1);
 758                         OUT_RING  (chan, 1);
 759                 }
 760
 761                 ret = RING_SPACE(chan, 14);
 762                 if (ret)
 763                         return ret;
 764
 765                 BEGIN_NV04(chan, NvSubCopy, 0x0238, 2);
 766                 OUT_RING  (chan, upper_32_bits(src_offset));
 767                 OUT_RING  (chan, upper_32_bits(dst_offset));
 768                 BEGIN_NV04(chan, NvSubCopy, 0x030c, 8);
 769                 OUT_RING  (chan, lower_32_bits(src_offset));
 770                 OUT_RING  (chan, lower_32_bits(dst_offset));
 771                 OUT_RING  (chan, stride);
 772                 OUT_RING  (chan, stride);
 773                 OUT_RING  (chan, stride);
 774                 OUT_RING  (chan, height);
 775                 OUT_RING  (chan, 0x00000101);
 776                 OUT_RING  (chan, 0x00000000);
 777                 BEGIN_NV04(chan, NvSubCopy, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
 778                 OUT_RING  (chan, 0);
 779
 780                 length -= amount;
 781                 src_offset += amount;
 782                 dst_offset += amount;
 783         }
 784
 785         return 0;
 786 }
 787
 788 static int
 789 nv04_bo_move_init(struct nouveau_channel *chan, u32 handle)
 790 {
 791         int ret = nouveau_notifier_alloc(chan, NvNotify0, 32, 0xfe0, 0x1000,
 792                                          &chan->m2mf_ntfy);
 793         if (ret == 0) {
 794                 ret = RING_SPACE(chan, 4);
 795                 if (ret == 0) {
 796                         BEGIN_NV04(chan, NvSubCopy, 0x0000, 1);
 797                         OUT_RING  (chan, handle);
 798                         BEGIN_NV04(chan, NvSubCopy, 0x0180, 1);
 799                         OUT_RING  (chan, NvNotify0);
 800                 }
 801         }
 802
 803         return ret;
 804 }
 805
 806 static inline uint32_t
 807 nouveau_bo_mem_ctxdma(struct ttm_buffer_object *bo,
 808                       struct nouveau_channel *chan, struct ttm_mem_reg *mem)
 809 {
 810         if (mem->mem_type == TTM_PL_TT)
 811                 return chan->gart_handle;
 812         return chan->vram_handle;
 813 }
 814
 815 static int
 816 nv04_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
 817                   struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
 818 {
 819         u32 src_offset = old_mem->start << PAGE_SHIFT;
 820         u32 dst_offset = new_mem->start << PAGE_SHIFT;
 821         u32 page_count = new_mem->num_pages;
 822         int ret;
 823
 824         ret = RING_SPACE(chan, 3);
 825         if (ret)
 826                 return ret;
 827
 828         BEGIN_NV04(chan, NvSubCopy, NV_MEMORY_TO_MEMORY_FORMAT_DMA_SOURCE, 2);
 829         OUT_RING  (chan, nouveau_bo_mem_ctxdma(bo, chan, old_mem));
 830         OUT_RING  (chan, nouveau_bo_mem_ctxdma(bo, chan, new_mem));
 831
 832         page_count = new_mem->num_pages;
 833         while (page_count) {
 834                 int line_count = (page_count > 2047) ? 2047 : page_count;
 835
 836                 ret = RING_SPACE(chan, 11);
 837                 if (ret)
 838                         return ret;
 839
 840                 BEGIN_NV04(chan, NvSubCopy,
 841                                  NV_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN, 8);
 842                 OUT_RING  (chan, src_offset);
 843                 OUT_RING  (chan, dst_offset);
 844                 OUT_RING  (chan, PAGE_SIZE); /* src_pitch */
 845                 OUT_RING  (chan, PAGE_SIZE); /* dst_pitch */
 846                 OUT_RING  (chan, PAGE_SIZE); /* line_length */
 847                 OUT_RING  (chan, line_count);
 848                 OUT_RING  (chan, 0x00000101);
 849                 OUT_RING  (chan, 0x00000000);
 850                 BEGIN_NV04(chan, NvSubCopy, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
 851                 OUT_RING  (chan, 0);
 852
 853                 page_count -= line_count;
 854                 src_offset += (PAGE_SIZE * line_count);
 855                 dst_offset += (PAGE_SIZE * line_count);
 856         }
 857
 858         return 0;
 859 }
 860
 861 static int
 862 nouveau_vma_getmap(struct nouveau_channel *chan, struct nouveau_bo *nvbo,
 863                    struct ttm_mem_reg *mem, struct nouveau_vma *vma)
 864 {
 865         struct nouveau_mem *node = mem->mm_node;
 866         int ret;
 867
 868         ret = nouveau_vm_get(chan->vm, mem->num_pages << PAGE_SHIFT,
 869                              node->page_shift, NV_MEM_ACCESS_RO, vma);
 870         if (ret)
 871                 return ret;
 872
 873         if (mem->mem_type == TTM_PL_VRAM)
 874                 nouveau_vm_map(vma, node);
 875         else
 876                 nouveau_vm_map_sg(vma, 0, mem->num_pages << PAGE_SHIFT, node);
 877
 878         return 0;
 879 }
 880
 881 static int
 882 nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
 883                      bool no_wait_reserve, bool no_wait_gpu,
 884                      struct ttm_mem_reg *new_mem)
 885 {
 886         struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
 887         struct nouveau_channel *chan = chan = dev_priv->channel;
 888         struct nouveau_bo *nvbo = nouveau_bo(bo);
 889         struct ttm_mem_reg *old_mem = &bo->mem;
 890         int ret;
 891
 892         mutex_lock_nested(&chan->mutex, NOUVEAU_KCHANNEL_MUTEX);
 893
 894         /* create temporary vmas for the transfer and attach them to the
 895          * old nouveau_mem node, these will get cleaned up after ttm has
 896          * destroyed the ttm_mem_reg
 897          */
 898         if (dev_priv->card_type >= NV_50) {
 899                 struct nouveau_mem *node = old_mem->mm_node;
 900
 901                 ret = nouveau_vma_getmap(chan, nvbo, old_mem, &node->vma[0]);
 902                 if (ret)
 903                         goto out;
 904
 905                 ret = nouveau_vma_getmap(chan, nvbo, new_mem, &node->vma[1]);
 906                 if (ret)
 907                         goto out;
 908         }
 909
 910         ret = dev_priv->ttm.move(chan, bo, &bo->mem, new_mem);
 911         if (ret == 0) {
 912                 ret = nouveau_bo_move_accel_cleanup(chan, nvbo, evict,
 913                                                     no_wait_reserve,
 914                                                     no_wait_gpu, new_mem);
 915         }
 916
 917 out:
 918         mutex_unlock(&chan->mutex);
 919         return ret;
 920 }
 921
 922 void
 923 nouveau_bo_move_init(struct nouveau_channel *chan)
 924 {
 925         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
 926         static const struct {
 927                 const char *name;
 928                 int engine;
 929                 u32 oclass;
 930                 int (*exec)(struct nouveau_channel *,
 931                             struct ttm_buffer_object *,
 932                             struct ttm_mem_reg *, struct ttm_mem_reg *);
 933                 int (*init)(struct nouveau_channel *, u32 handle);
 934         } _methods[] = {
 935                 {  "COPY", 0, 0xa0b5, nve0_bo_move_copy, nvc0_bo_move_init },
 936                 { "COPY1", 5, 0x90b8, nvc0_bo_move_copy, nvc0_bo_move_init },
 937                 { "COPY0", 4, 0x90b5, nvc0_bo_move_copy, nvc0_bo_move_init },
 938                 {  "COPY", 0, 0x85b5, nva3_bo_move_copy, nv50_bo_move_init },
 939                 { "CRYPT", 0, 0x74c1, nv84_bo_move_exec, nv50_bo_move_init },
 940                 {  "M2MF", 0, 0x9039, nvc0_bo_move_m2mf, nvc0_bo_move_init },
 941                 {  "M2MF", 0, 0x5039, nv50_bo_move_m2mf, nv50_bo_move_init },
 942                 {  "M2MF", 0, 0x0039, nv04_bo_move_m2mf, nv04_bo_move_init },
 943                 {},
 944                 { "CRYPT", 0, 0x88b4, nv98_bo_move_exec, nv50_bo_move_init },
 945         }, *mthd = _methods;
 946         const char *name = "CPU";
 947         int ret;
 948
 949         do {
 950                 u32 handle = (mthd->engine << 16) | mthd->oclass;
 951                 ret = nouveau_gpuobj_gr_new(chan, handle, mthd->oclass);
 952                 if (ret == 0) {
 953                         ret = mthd->init(chan, handle);
 954                         if (ret == 0) {
 955                                 dev_priv->ttm.move = mthd->exec;
 956                                 name = mthd->name;
 957                                 break;
 958                         }
 959                 }
 960         } while ((++mthd)->exec);
 961
 962         NV_INFO(chan->dev, "MM: using %s for buffer copies\n", name);
 963 }
 964
 965 static int
 966 nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr,
 967                       bool no_wait_reserve, bool no_wait_gpu,
 968                       struct ttm_mem_reg *new_mem)
 969 {
 970         u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING;
 971         struct ttm_placement placement;
 972         struct ttm_mem_reg tmp_mem;
 973         int ret;
 974
 975         placement.fpfn = placement.lpfn = 0;
 976         placement.num_placement = placement.num_busy_placement = 1;
 977         placement.placement = placement.busy_placement = &placement_memtype;
 978
 979         tmp_mem = *new_mem;
 980         tmp_mem.mm_node = NULL;
 981         ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_reserve, no_wait_gpu);
 982         if (ret)
 983                 return ret;
 984
 985         ret = ttm_tt_bind(bo->ttm, &tmp_mem);
 986         if (ret)
 987                 goto out;
 988
 989         ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_reserve, no_wait_gpu, &tmp_mem);
 990         if (ret)
 991                 goto out;
 992
 993         ret = ttm_bo_move_ttm(bo, true, no_wait_reserve, no_wait_gpu, new_mem);
 994 out:
 995         ttm_bo_mem_put(bo, &tmp_mem);
 996         return ret;
 997 }
 998
 999 static int
1000 nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr,
1001                       bool no_wait_reserve, bool no_wait_gpu,
1002                       struct ttm_mem_reg *new_mem)
1003 {
1004         u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING;
1005         struct ttm_placement placement;
1006         struct ttm_mem_reg tmp_mem;
1007         int ret;
1008
1009         placement.fpfn = placement.lpfn = 0;
1010         placement.num_placement = placement.num_busy_placement = 1;
1011         placement.placement = placement.busy_placement = &placement_memtype;
1012
1013         tmp_mem = *new_mem;
1014         tmp_mem.mm_node = NULL;
1015         ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_reserve, no_wait_gpu);
1016         if (ret)
1017                 return ret;
1018
1019         ret = ttm_bo_move_ttm(bo, true, no_wait_reserve, no_wait_gpu, &tmp_mem);
1020         if (ret)
1021                 goto out;
1022
1023         ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_reserve, no_wait_gpu, new_mem);
1024         if (ret)
1025                 goto out;
1026
1027 out:
1028         ttm_bo_mem_put(bo, &tmp_mem);
1029         return ret;
1030 }
1031
1032 static void
1033 nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem)
1034 {
1035         struct nouveau_bo *nvbo = nouveau_bo(bo);
1036         struct nouveau_vma *vma;
1037
1038         /* ttm can now (stupidly) pass the driver bos it didn't create... */
1039         if (bo->destroy != nouveau_bo_del_ttm)
1040                 return;
1041
1042         list_for_each_entry(vma, &nvbo->vma_list, head) {
1043                 if (new_mem && new_mem->mem_type == TTM_PL_VRAM) {
1044                         nouveau_vm_map(vma, new_mem->mm_node);
1045                 } else
1046                 if (new_mem && new_mem->mem_type == TTM_PL_TT &&
1047                     nvbo->page_shift == vma->vm->spg_shift) {
1048                         if (((struct nouveau_mem *)new_mem->mm_node)->sg)
1049                                 nouveau_vm_map_sg_table(vma, 0, new_mem->
1050                                                   num_pages << PAGE_SHIFT,
1051                                                   new_mem->mm_node);
1052                         else
1053                                 nouveau_vm_map_sg(vma, 0, new_mem->
1054                                                   num_pages << PAGE_SHIFT,
1055                                                   new_mem->mm_node);
1056                 } else {
1057                         nouveau_vm_unmap(vma);
1058                 }
1059         }
1060 }
1061
1062 static int
1063 nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem,
1064                    struct nouveau_tile_reg **new_tile)
1065 {
1066         struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
1067         struct drm_device *dev = dev_priv->dev;
1068         struct nouveau_bo *nvbo = nouveau_bo(bo);
1069         u64 offset = new_mem->start << PAGE_SHIFT;
1070
1071         *new_tile = NULL;
1072         if (new_mem->mem_type != TTM_PL_VRAM)
1073                 return 0;
1074
1075         if (dev_priv->card_type >= NV_10) {
1076                 *new_tile = nv10_mem_set_tiling(dev, offset, new_mem->size,
1077                                                 nvbo->tile_mode,
1078                                                 nvbo->tile_flags);
1079         }
1080
1081         return 0;
1082 }
1083
1084 static void
1085 nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo,
1086                       struct nouveau_tile_reg *new_tile,
1087                       struct nouveau_tile_reg **old_tile)
1088 {
1089         struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
1090         struct drm_device *dev = dev_priv->dev;
1091
1092         nv10_mem_put_tile_region(dev, *old_tile, bo->sync_obj);
1093         *old_tile = new_tile;
1094 }
1095
1096 static int
1097 nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
1098                 bool no_wait_reserve, bool no_wait_gpu,
1099                 struct ttm_mem_reg *new_mem)
1100 {
1101         struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
1102         struct nouveau_bo *nvbo = nouveau_bo(bo);
1103         struct ttm_mem_reg *old_mem = &bo->mem;
1104         struct nouveau_tile_reg *new_tile = NULL;
1105         int ret = 0;
1106
1107         if (dev_priv->card_type < NV_50) {
1108                 ret = nouveau_bo_vm_bind(bo, new_mem, &new_tile);
1109                 if (ret)
1110                         return ret;
1111         }
1112
1113         /* Fake bo copy. */
1114         if (old_mem->mem_type == TTM_PL_SYSTEM && !bo->ttm) {
1115                 BUG_ON(bo->mem.mm_node != NULL);
1116                 bo->mem = *new_mem;
1117                 new_mem->mm_node = NULL;
1118                 goto out;
1119         }
1120
1121         /* CPU copy if we have no accelerated method available */
1122         if (!dev_priv->ttm.move) {
1123                 ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem);
1124                 goto out;
1125         }
1126
1127         /* Hardware assisted copy. */
1128         if (new_mem->mem_type == TTM_PL_SYSTEM)
1129                 ret = nouveau_bo_move_flipd(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem);
1130         else if (old_mem->mem_type == TTM_PL_SYSTEM)
1131                 ret = nouveau_bo_move_flips(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem);
1132         else
1133                 ret = nouveau_bo_move_m2mf(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem);
1134
1135         if (!ret)
1136                 goto out;
1137
1138         /* Fallback to software copy. */
1139         ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem);
1140
1141 out:
1142         if (dev_priv->card_type < NV_50) {
1143                 if (ret)
1144                         nouveau_bo_vm_cleanup(bo, NULL, &new_tile);
1145                 else
1146                         nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile);
1147         }
1148
1149         return ret;
1150 }
1151
1152 static int
1153 nouveau_bo_verify_access(struct ttm_buffer_object *bo, struct file *filp)
1154 {
1155         return 0;
1156 }
1157
1158 static int
1159 nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1160 {
1161         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
1162         struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
1163         struct drm_device *dev = dev_priv->dev;
1164         int ret;
1165
1166         mem->bus.addr = NULL;
1167         mem->bus.offset = 0;
1168         mem->bus.size = mem->num_pages << PAGE_SHIFT;
1169         mem->bus.base = 0;
1170         mem->bus.is_iomem = false;
1171         if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
1172                 return -EINVAL;
1173         switch (mem->mem_type) {
1174         case TTM_PL_SYSTEM:
1175                 /* System memory */
1176                 return 0;
1177         case TTM_PL_TT:
1178 #if __OS_HAS_AGP
1179                 if (dev_priv->gart_info.type == NOUVEAU_GART_AGP) {
1180                         mem->bus.offset = mem->start << PAGE_SHIFT;
1181                         mem->bus.base = dev_priv->gart_info.aper_base;
1182                         mem->bus.is_iomem = true;
1183                 }
1184 #endif
1185                 break;
1186         case TTM_PL_VRAM:
1187         {
1188                 struct nouveau_mem *node = mem->mm_node;
1189                 u8 page_shift;
1190
1191                 if (!dev_priv->bar1_vm) {
1192                         mem->bus.offset = mem->start << PAGE_SHIFT;
1193                         mem->bus.base = pci_resource_start(dev->pdev, 1);
1194                         mem->bus.is_iomem = true;
1195                         break;
1196                 }
1197
1198                 if (dev_priv->card_type >= NV_C0)
1199                         page_shift = node->page_shift;
1200                 else
1201                         page_shift = 12;
1202
1203                 ret = nouveau_vm_get(dev_priv->bar1_vm, mem->bus.size,
1204                                      page_shift, NV_MEM_ACCESS_RW,
1205                                      &node->bar_vma);
1206                 if (ret)
1207                         return ret;
1208
1209                 nouveau_vm_map(&node->bar_vma, node);
1210                 if (ret) {
1211                         nouveau_vm_put(&node->bar_vma);
1212                         return ret;
1213                 }
1214
1215                 mem->bus.offset = node->bar_vma.offset;
1216                 if (dev_priv->card_type == NV_50) /*XXX*/
1217                         mem->bus.offset -= 0x0020000000ULL;
1218                 mem->bus.base = pci_resource_start(dev->pdev, 1);
1219                 mem->bus.is_iomem = true;
1220         }
1221                 break;
1222         default:
1223                 return -EINVAL;
1224         }
1225         return 0;
1226 }
1227
1228 static void
1229 nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1230 {
1231         struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
1232         struct nouveau_mem *node = mem->mm_node;
1233
1234         if (!dev_priv->bar1_vm || mem->mem_type != TTM_PL_VRAM)
1235                 return;
1236
1237         if (!node->bar_vma.node)
1238                 return;
1239
1240         nouveau_vm_unmap(&node->bar_vma);
1241         nouveau_vm_put(&node->bar_vma);
1242 }
1243
1244 static int
1245 nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
1246 {
1247         struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
1248         struct nouveau_bo *nvbo = nouveau_bo(bo);
1249
1250         /* as long as the bo isn't in vram, and isn't tiled, we've got
1251          * nothing to do here.
1252          */
1253         if (bo->mem.mem_type != TTM_PL_VRAM) {
1254                 if (dev_priv->card_type < NV_50 ||
1255                     !nouveau_bo_tile_layout(nvbo))
1256                         return 0;
1257         }
1258
1259         /* make sure bo is in mappable vram */
1260         if (bo->mem.start + bo->mem.num_pages < dev_priv->fb_mappable_pages)
1261                 return 0;
1262
1263
1264         nvbo->placement.fpfn = 0;
1265         nvbo->placement.lpfn = dev_priv->fb_mappable_pages;
1266         nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_VRAM, 0);
1267         return nouveau_bo_validate(nvbo, false, true, false);
1268 }
1269
1270 static int
1271 nouveau_ttm_tt_populate(struct ttm_tt *ttm)
1272 {
1273         struct ttm_dma_tt *ttm_dma = (void *)ttm;
1274         struct drm_nouveau_private *dev_priv;
1275         struct drm_device *dev;
1276         unsigned i;
1277         int r;
1278         bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
1279
1280         if (ttm->state != tt_unpopulated)
1281                 return 0;
1282
1283         if (slave && ttm->sg) {
1284                 /* make userspace faulting work */
1285                 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
1286                                                  ttm_dma->dma_address, ttm->num_pages);
1287                 ttm->state = tt_unbound;
1288                 return 0;
1289         }
1290
1291         dev_priv = nouveau_bdev(ttm->bdev);
1292         dev = dev_priv->dev;
1293
1294 #if __OS_HAS_AGP
1295         if (dev_priv->gart_info.type == NOUVEAU_GART_AGP) {
1296                 return ttm_agp_tt_populate(ttm);
1297         }
1298 #endif
1299
1300 #ifdef CONFIG_SWIOTLB
1301         if (swiotlb_nr_tbl()) {
1302                 return ttm_dma_populate((void *)ttm, dev->dev);
1303         }
1304 #endif
1305
1306         r = ttm_pool_populate(ttm);
1307         if (r) {
1308                 return r;
1309         }
1310
1311         for (i = 0; i < ttm->num_pages; i++) {
1312                 ttm_dma->dma_address[i] = pci_map_page(dev->pdev, ttm->pages[i],
1313                                                    0, PAGE_SIZE,
1314                                                    PCI_DMA_BIDIRECTIONAL);
1315                 if (pci_dma_mapping_error(dev->pdev, ttm_dma->dma_address[i])) {
1316                         while (--i) {
1317                                 pci_unmap_page(dev->pdev, ttm_dma->dma_address[i],
1318                                                PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
1319                                 ttm_dma->dma_address[i] = 0;
1320                         }
1321                         ttm_pool_unpopulate(ttm);
1322                         return -EFAULT;
1323                 }
1324         }
1325         return 0;
1326 }
1327
1328 static void
1329 nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm)
1330 {
1331         struct ttm_dma_tt *ttm_dma = (void *)ttm;
1332         struct drm_nouveau_private *dev_priv;
1333         struct drm_device *dev;
1334         unsigned i;
1335         bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
1336
1337         if (slave)
1338                 return;
1339
1340         dev_priv = nouveau_bdev(ttm->bdev);
1341         dev = dev_priv->dev;
1342
1343 #if __OS_HAS_AGP
1344         if (dev_priv->gart_info.type == NOUVEAU_GART_AGP) {
1345                 ttm_agp_tt_unpopulate(ttm);
1346                 return;
1347         }
1348 #endif
1349
1350 #ifdef CONFIG_SWIOTLB
1351         if (swiotlb_nr_tbl()) {
1352                 ttm_dma_unpopulate((void *)ttm, dev->dev);
1353                 return;
1354         }
1355 #endif
1356
1357         for (i = 0; i < ttm->num_pages; i++) {
1358                 if (ttm_dma->dma_address[i]) {
1359                         pci_unmap_page(dev->pdev, ttm_dma->dma_address[i],
1360                                        PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
1361                 }
1362         }
1363
1364         ttm_pool_unpopulate(ttm);
1365 }
1366
1367 void
1368 nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence)
1369 {
1370         struct nouveau_fence *old_fence = NULL;
1371
1372         if (likely(fence))
1373                 nouveau_fence_ref(fence);
1374
1375         spin_lock(&nvbo->bo.bdev->fence_lock);
1376         old_fence = nvbo->bo.sync_obj;
1377         nvbo->bo.sync_obj = fence;
1378         spin_unlock(&nvbo->bo.bdev->fence_lock);
1379
1380         nouveau_fence_unref(&old_fence);
1381 }
1382
1383 static void
1384 nouveau_bo_fence_unref(void **sync_obj)
1385 {
1386         nouveau_fence_unref((struct nouveau_fence **)sync_obj);
1387 }
1388
1389 static void *
1390 nouveau_bo_fence_ref(void *sync_obj)
1391 {
1392         return nouveau_fence_ref(sync_obj);
1393 }
1394
1395 static bool
1396 nouveau_bo_fence_signalled(void *sync_obj, void *sync_arg)
1397 {
1398         return nouveau_fence_done(sync_obj);
1399 }
1400
1401 static int
1402 nouveau_bo_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr)
1403 {
1404         return nouveau_fence_wait(sync_obj, lazy, intr);
1405 }
1406
1407 static int
1408 nouveau_bo_fence_flush(void *sync_obj, void *sync_arg)
1409 {
1410         return 0;
1411 }
1412
1413 struct ttm_bo_driver nouveau_bo_driver = {
1414         .ttm_tt_create = &nouveau_ttm_tt_create,
1415         .ttm_tt_populate = &nouveau_ttm_tt_populate,
1416         .ttm_tt_unpopulate = &nouveau_ttm_tt_unpopulate,
1417         .invalidate_caches = nouveau_bo_invalidate_caches,
1418         .init_mem_type = nouveau_bo_init_mem_type,
1419         .evict_flags = nouveau_bo_evict_flags,
1420         .move_notify = nouveau_bo_move_ntfy,
1421         .move = nouveau_bo_move,
1422         .verify_access = nouveau_bo_verify_access,
1423         .sync_obj_signaled = nouveau_bo_fence_signalled,
1424         .sync_obj_wait = nouveau_bo_fence_wait,
1425         .sync_obj_flush = nouveau_bo_fence_flush,
1426         .sync_obj_unref = nouveau_bo_fence_unref,
1427         .sync_obj_ref = nouveau_bo_fence_ref,
1428         .fault_reserve_notify = &nouveau_ttm_fault_reserve_notify,
1429         .io_mem_reserve = &nouveau_ttm_io_mem_reserve,
1430         .io_mem_free = &nouveau_ttm_io_mem_free,
1431 };
1432
1433 struct nouveau_vma *
1434 nouveau_bo_vma_find(struct nouveau_bo *nvbo, struct nouveau_vm *vm)
1435 {
1436         struct nouveau_vma *vma;
1437         list_for_each_entry(vma, &nvbo->vma_list, head) {
1438                 if (vma->vm == vm)
1439                         return vma;
1440         }
1441
1442         return NULL;
1443 }
1444
1445 int
1446 nouveau_bo_vma_add(struct nouveau_bo *nvbo, struct nouveau_vm *vm,
1447                    struct nouveau_vma *vma)
1448 {
1449         const u32 size = nvbo->bo.mem.num_pages << PAGE_SHIFT;
1450         struct nouveau_mem *node = nvbo->bo.mem.mm_node;
1451         int ret;
1452
1453         ret = nouveau_vm_get(vm, size, nvbo->page_shift,
1454                              NV_MEM_ACCESS_RW, vma);
1455         if (ret)
1456                 return ret;
1457
1458         if (nvbo->bo.mem.mem_type == TTM_PL_VRAM)
1459                 nouveau_vm_map(vma, nvbo->bo.mem.mm_node);
1460         else if (nvbo->bo.mem.mem_type == TTM_PL_TT) {
1461                 if (node->sg)
1462                         nouveau_vm_map_sg_table(vma, 0, size, node);
1463                 else
1464                         nouveau_vm_map_sg(vma, 0, size, node);
1465         }
1466
1467         list_add_tail(&vma->head, &nvbo->vma_list);
1468         vma->refcount = 1;
1469         return 0;
1470 }
1471
1472 void
1473 nouveau_bo_vma_del(struct nouveau_bo *nvbo, struct nouveau_vma *vma)
1474 {
1475         if (vma->node) {
1476                 if (nvbo->bo.mem.mem_type != TTM_PL_SYSTEM) {
1477                         spin_lock(&nvbo->bo.bdev->fence_lock);
1478                         ttm_bo_wait(&nvbo->bo, false, false, false);
1479                         spin_unlock(&nvbo->bo.bdev->fence_lock);
1480                         nouveau_vm_unmap(vma);
1481                 }
1482
1483                 nouveau_vm_put(vma);
1484                 list_del(&vma->head);
1485         }
1486 }