drivers/video/rockchip/vcodec/vcodec_iommu_drm.c

   1 /*
   2  * Copyright (C) 2016 Fuzhou Rockchip Electronics Co., Ltd
   3  * author: Jung Zhao jung.zhao@rock-chips.com
   4  *         Randy Li, randy.li@rock-chips.com
   5  *
   6  * This software is licensed under the terms of the GNU General Public
   7  * License version 2, as published by the Free Software Foundation, and
   8  * may be copied, distributed, and modified under those terms.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  */
  16 #include <linux/dma-iommu.h>
  17
  18 #include <linux/dma-buf.h>
  19 #include <drm/drmP.h>
  20 #include <drm/drm_atomic.h>
  21 #include <drm/drm_crtc_helper.h>
  22 #include <drm/drm_fb_helper.h>
  23 #include <drm/drm_sync_helper.h>
  24 #include <drm/rockchip_drm.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/rockchip-iovmm.h>
  27 #include <linux/pm_runtime.h>
  28 #include <linux/memblock.h>
  29 #include <linux/module.h>
  30 #include <linux/of_address.h>
  31 #include <linux/of_graph.h>
  32 #include <linux/component.h>
  33 #include <linux/fence.h>
  34 #include <linux/console.h>
  35 #include <linux/kref.h>
  36 #include <linux/fdtable.h>
  37 #include <linux/ktime.h>
  38
  39 #include "vcodec_iommu_ops.h"
  40
  41 struct vcodec_drm_buffer {
  42         struct list_head list;
  43         struct dma_buf *dma_buf;
  44         union {
  45                 unsigned long iova;
  46                 unsigned long phys;
  47         };
  48         void *cpu_addr;
  49         unsigned long size;
  50         int index;
  51         struct dma_buf_attachment *attach;
  52         struct sg_table *sgt;
  53         struct page **pages;
  54         struct kref ref;
  55         struct vcodec_iommu_session_info *session_info;
  56         ktime_t last_used;
  57 };
  58
  59 struct vcodec_iommu_drm_info {
  60         struct iommu_domain *domain;
  61         bool attached;
  62 };
  63
  64 static struct vcodec_drm_buffer *
  65 vcodec_drm_get_buffer_no_lock(struct vcodec_iommu_session_info *session_info,
  66                               int idx)
  67 {
  68         struct vcodec_drm_buffer *drm_buffer = NULL, *n;
  69
  70         list_for_each_entry_safe(drm_buffer, n, &session_info->buffer_list,
  71                                  list) {
  72                 if (drm_buffer->index == idx) {
  73                         drm_buffer->last_used = ktime_get();
  74                         return drm_buffer;
  75                 }
  76         }
  77
  78         return NULL;
  79 }
  80
  81 static struct vcodec_drm_buffer *
  82 vcodec_drm_get_buffer_fd_no_lock(struct vcodec_iommu_session_info *session_info,
  83                                  int fd)
  84 {
  85         struct vcodec_drm_buffer *drm_buffer = NULL, *n;
  86         struct dma_buf *dma_buf = NULL;
  87
  88         dma_buf = dma_buf_get(fd);
  89
  90         list_for_each_entry_safe(drm_buffer, n, &session_info->buffer_list,
  91                                  list) {
  92                 if (drm_buffer->dma_buf == dma_buf) {
  93                         drm_buffer->last_used = ktime_get();
  94                         dma_buf_put(dma_buf);
  95                         return drm_buffer;
  96                 }
  97         }
  98
  99         dma_buf_put(dma_buf);
 100
 101         return NULL;
 102 }
 103
 104 static void vcodec_drm_detach(struct vcodec_iommu_info *iommu_info)
 105 {
 106         struct vcodec_iommu_drm_info *drm_info = iommu_info->private;
 107         struct device *dev = iommu_info->dev;
 108         struct iommu_domain *domain = drm_info->domain;
 109
 110         mutex_lock(&iommu_info->iommu_mutex);
 111
 112         if (!drm_info->attached) {
 113                 mutex_unlock(&iommu_info->iommu_mutex);
 114                 return;
 115         }
 116
 117         iommu_detach_device(domain, dev);
 118         drm_info->attached = false;
 119
 120         mutex_unlock(&iommu_info->iommu_mutex);
 121 }
 122
 123 static int vcodec_drm_attach_unlock(struct vcodec_iommu_info *iommu_info)
 124 {
 125         struct vcodec_iommu_drm_info *drm_info = iommu_info->private;
 126         struct device *dev = iommu_info->dev;
 127         struct iommu_domain *domain = drm_info->domain;
 128         int ret = 0;
 129
 130         ret = dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
 131         if (ret)
 132                 return ret;
 133
 134         dma_set_max_seg_size(dev, DMA_BIT_MASK(32));
 135         ret = iommu_attach_device(domain, dev);
 136         if (ret) {
 137                 dev_err(dev, "Failed to attach iommu device\n");
 138                 return ret;
 139         }
 140
 141         if (!common_iommu_setup_dma_ops(dev, 0x10000000, SZ_2G, domain->ops)) {
 142                 dev_err(dev, "Failed to set dma_ops\n");
 143                 iommu_detach_device(domain, dev);
 144                 ret = -ENODEV;
 145         }
 146
 147         return ret;
 148 }
 149
 150 static int vcodec_drm_attach(struct vcodec_iommu_info *iommu_info)
 151 {
 152         struct vcodec_iommu_drm_info *drm_info = iommu_info->private;
 153         int ret;
 154
 155         mutex_lock(&iommu_info->iommu_mutex);
 156
 157         if (drm_info->attached) {
 158                 mutex_unlock(&iommu_info->iommu_mutex);
 159                 return 0;
 160         }
 161
 162         ret = vcodec_drm_attach_unlock(iommu_info);
 163         if (ret) {
 164                 mutex_unlock(&iommu_info->iommu_mutex);
 165                 return ret;
 166         }
 167
 168         drm_info->attached = true;
 169
 170         mutex_unlock(&iommu_info->iommu_mutex);
 171
 172         return ret;
 173 }
 174
 175 static void *vcodec_drm_sgt_map_kernel(struct vcodec_drm_buffer *drm_buffer)
 176 {
 177         struct vcodec_iommu_session_info *session_info =
 178                 drm_buffer->session_info;
 179         struct device *dev = session_info->dev;
 180         struct scatterlist *sgl, *sg;
 181         int nr_pages = PAGE_ALIGN(drm_buffer->size) >> PAGE_SHIFT;
 182         int i = 0, j = 0, k = 0;
 183         struct page *page;
 184
 185         drm_buffer->pages = kmalloc_array(nr_pages, sizeof(*drm_buffer->pages),
 186                                           GFP_KERNEL);
 187         if (!(drm_buffer->pages)) {
 188                 dev_err(dev, "drm map can not alloc pages\n");
 189
 190                 return NULL;
 191         }
 192
 193         sgl = drm_buffer->sgt->sgl;
 194
 195         for_each_sg(sgl, sg, drm_buffer->sgt->nents, i) {
 196                 page = sg_page(sg);
 197                 for (j = 0; j < sg->length / PAGE_SIZE; j++)
 198                         drm_buffer->pages[k++] = page++;
 199         }
 200
 201         return vmap(drm_buffer->pages, nr_pages, VM_MAP,
 202                     pgprot_noncached(PAGE_KERNEL));
 203 }
 204
 205 static void vcodec_drm_sgt_unmap_kernel(struct vcodec_drm_buffer *drm_buffer)
 206 {
 207         vunmap(drm_buffer->cpu_addr);
 208         kfree(drm_buffer->pages);
 209 }
 210
 211 static void vcodec_drm_clear_map(struct kref *ref)
 212 {
 213         struct vcodec_drm_buffer *drm_buffer =
 214                 container_of(ref, struct vcodec_drm_buffer, ref);
 215         struct vcodec_iommu_session_info *session_info =
 216                 drm_buffer->session_info;
 217         struct vcodec_iommu_info *iommu_info = session_info->iommu_info;
 218         struct vcodec_iommu_drm_info *drm_info = iommu_info->private;
 219         struct device *dev = session_info->dev;
 220         struct iommu_domain *domain = drm_info->domain;
 221
 222         mutex_lock(&iommu_info->iommu_mutex);
 223         drm_info = session_info->iommu_info->private;
 224         if (!drm_info->attached) {
 225                 if (vcodec_drm_attach_unlock(session_info->iommu_info))
 226                         dev_err(dev, "can't clea map, attach iommu failed.\n");
 227         }
 228
 229         if (drm_buffer->cpu_addr) {
 230                 vcodec_drm_sgt_unmap_kernel(drm_buffer);
 231                 drm_buffer->cpu_addr = NULL;
 232         }
 233
 234         if (drm_buffer->attach) {
 235                 dma_buf_unmap_attachment(drm_buffer->attach, drm_buffer->sgt,
 236                                          DMA_BIDIRECTIONAL);
 237                 dma_buf_detach(drm_buffer->dma_buf, drm_buffer->attach);
 238                 dma_buf_put(drm_buffer->dma_buf);
 239                 drm_buffer->attach = NULL;
 240         }
 241
 242         if (!drm_info->attached)
 243                 iommu_detach_device(domain, dev);
 244
 245         mutex_unlock(&iommu_info->iommu_mutex);
 246 }
 247
 248 static void vcdoec_drm_dump_info(struct vcodec_iommu_session_info *session_info)
 249 {
 250         struct vcodec_drm_buffer *drm_buffer = NULL, *n;
 251
 252         vpu_iommu_debug(session_info->debug_level, DEBUG_IOMMU_OPS_DUMP,
 253                         "still there are below buffers stored in list\n");
 254         list_for_each_entry_safe(drm_buffer, n, &session_info->buffer_list,
 255                                  list) {
 256                 vpu_iommu_debug(session_info->debug_level, DEBUG_IOMMU_OPS_DUMP,
 257                                 "index %d drm_buffer dma_buf %p cpu_addr %p\n",
 258                                 drm_buffer->index,
 259                                 drm_buffer->dma_buf, drm_buffer->cpu_addr);
 260         }
 261 }
 262
 263 static int vcodec_drm_free(struct vcodec_iommu_session_info *session_info,
 264                            int idx)
 265 {
 266         struct device *dev = session_info->dev;
 267         /* please double-check all maps have been release */
 268         struct vcodec_drm_buffer *drm_buffer;
 269
 270         mutex_lock(&session_info->list_mutex);
 271         drm_buffer = vcodec_drm_get_buffer_no_lock(session_info, idx);
 272
 273         if (!drm_buffer) {
 274                 dev_err(dev, "can not find %d buffer in list\n", idx);
 275                 mutex_unlock(&session_info->list_mutex);
 276
 277                 return -EINVAL;
 278         }
 279
 280         if (atomic_read(&drm_buffer->ref.refcount) == 0) {
 281                 dma_buf_put(drm_buffer->dma_buf);
 282                 list_del_init(&drm_buffer->list);
 283                 kfree(drm_buffer);
 284                 session_info->buffer_nums--;
 285                 vpu_iommu_debug(session_info->debug_level, DEBUG_IOMMU_NORMAL,
 286                         "buffer nums %d\n", session_info->buffer_nums);
 287         }
 288         mutex_unlock(&session_info->list_mutex);
 289
 290         return 0;
 291 }
 292
 293 static int
 294 vcodec_drm_unmap_iommu(struct vcodec_iommu_session_info *session_info,
 295                        int idx)
 296 {
 297         struct device *dev = session_info->dev;
 298         struct vcodec_drm_buffer *drm_buffer;
 299
 300         /* Force to flush iommu table */
 301         if (of_machine_is_compatible("rockchip,rk3288"))
 302                 rockchip_iovmm_invalidate_tlb(session_info->mmu_dev);
 303
 304         mutex_lock(&session_info->list_mutex);
 305         drm_buffer = vcodec_drm_get_buffer_no_lock(session_info, idx);
 306         mutex_unlock(&session_info->list_mutex);
 307
 308         if (!drm_buffer) {
 309                 dev_err(dev, "can not find %d buffer in list\n", idx);
 310                 return -EINVAL;
 311         }
 312
 313         kref_put(&drm_buffer->ref, vcodec_drm_clear_map);
 314
 315         return 0;
 316 }
 317
 318 static int vcodec_drm_map_iommu(struct vcodec_iommu_session_info *session_info,
 319                                 int idx,
 320                                 unsigned long *iova,
 321                                 unsigned long *size)
 322 {
 323         struct device *dev = session_info->dev;
 324         struct vcodec_drm_buffer *drm_buffer;
 325
 326         /* Force to flush iommu table */
 327         if (of_machine_is_compatible("rockchip,rk3288"))
 328                 rockchip_iovmm_invalidate_tlb(session_info->mmu_dev);
 329
 330         mutex_lock(&session_info->list_mutex);
 331         drm_buffer = vcodec_drm_get_buffer_no_lock(session_info, idx);
 332         mutex_unlock(&session_info->list_mutex);
 333
 334         if (!drm_buffer) {
 335                 dev_err(dev, "can not find %d buffer in list\n", idx);
 336                 return -EINVAL;
 337         }
 338
 339         kref_get(&drm_buffer->ref);
 340         if (iova)
 341                 *iova = drm_buffer->iova;
 342         if (size)
 343                 *size = drm_buffer->size;
 344         return 0;
 345 }
 346
 347 static int
 348 vcodec_drm_unmap_kernel(struct vcodec_iommu_session_info *session_info, int idx)
 349 {
 350         struct device *dev = session_info->dev;
 351         struct vcodec_drm_buffer *drm_buffer;
 352
 353         mutex_lock(&session_info->list_mutex);
 354         drm_buffer = vcodec_drm_get_buffer_no_lock(session_info, idx);
 355         mutex_unlock(&session_info->list_mutex);
 356
 357         if (!drm_buffer) {
 358                 dev_err(dev, "can not find %d buffer in list\n", idx);
 359
 360                 return -EINVAL;
 361         }
 362
 363         if (drm_buffer->cpu_addr) {
 364                 vcodec_drm_sgt_unmap_kernel(drm_buffer);
 365                 drm_buffer->cpu_addr = NULL;
 366         }
 367
 368         kref_put(&drm_buffer->ref, vcodec_drm_clear_map);
 369         return 0;
 370 }
 371
 372 static int
 373 vcodec_drm_free_fd(struct vcodec_iommu_session_info *session_info, int fd)
 374 {
 375         struct device *dev = session_info->dev;
 376         /* please double-check all maps have been release */
 377         struct vcodec_drm_buffer *drm_buffer = NULL;
 378
 379         mutex_lock(&session_info->list_mutex);
 380         drm_buffer = vcodec_drm_get_buffer_fd_no_lock(session_info, fd);
 381
 382         if (!drm_buffer) {
 383                 dev_err(dev, "can not find %d buffer in list\n", fd);
 384                 mutex_unlock(&session_info->list_mutex);
 385
 386                 return -EINVAL;
 387         }
 388         mutex_unlock(&session_info->list_mutex);
 389
 390         vcodec_drm_unmap_iommu(session_info, drm_buffer->index);
 391
 392         mutex_lock(&session_info->list_mutex);
 393         if (atomic_read(&drm_buffer->ref.refcount) == 0) {
 394                 dma_buf_put(drm_buffer->dma_buf);
 395                 list_del_init(&drm_buffer->list);
 396                 kfree(drm_buffer);
 397                 session_info->buffer_nums--;
 398                 vpu_iommu_debug(session_info->debug_level, DEBUG_IOMMU_NORMAL,
 399                         "buffer nums %d\n", session_info->buffer_nums);
 400         }
 401         mutex_unlock(&session_info->list_mutex);
 402
 403         return 0;
 404 }
 405
 406 static void
 407 vcodec_drm_clear_session(struct vcodec_iommu_session_info *session_info)
 408 {
 409         struct vcodec_drm_buffer *drm_buffer = NULL, *n;
 410
 411         list_for_each_entry_safe(drm_buffer, n, &session_info->buffer_list,
 412                                  list) {
 413                 kref_put(&drm_buffer->ref, vcodec_drm_clear_map);
 414                 vcodec_drm_free(session_info, drm_buffer->index);
 415         }
 416 }
 417
 418 static void *
 419 vcodec_drm_map_kernel(struct vcodec_iommu_session_info *session_info, int idx)
 420 {
 421         struct device *dev = session_info->dev;
 422         struct vcodec_drm_buffer *drm_buffer;
 423
 424         mutex_lock(&session_info->list_mutex);
 425         drm_buffer = vcodec_drm_get_buffer_no_lock(session_info, idx);
 426         mutex_unlock(&session_info->list_mutex);
 427
 428         if (!drm_buffer) {
 429                 dev_err(dev, "can not find %d buffer in list\n", idx);
 430                 return NULL;
 431         }
 432
 433         if (!drm_buffer->cpu_addr)
 434                 drm_buffer->cpu_addr =
 435                         vcodec_drm_sgt_map_kernel(drm_buffer);
 436
 437         kref_get(&drm_buffer->ref);
 438
 439         return drm_buffer->cpu_addr;
 440 }
 441
 442 static int vcodec_drm_import(struct vcodec_iommu_session_info *session_info,
 443                              int fd)
 444 {
 445         struct vcodec_drm_buffer *drm_buffer = NULL, *n;
 446         struct vcodec_drm_buffer *oldest_buffer = NULL, *loop_buffer = NULL;
 447         struct vcodec_iommu_info *iommu_info = session_info->iommu_info;
 448         struct vcodec_iommu_drm_info *drm_info = iommu_info->private;
 449         struct iommu_domain *domain = drm_info->domain;
 450         struct device *dev = session_info->dev;
 451         struct dma_buf_attachment *attach;
 452         struct sg_table *sgt;
 453         struct dma_buf *dma_buf;
 454         ktime_t oldest_time = ktime_set(0, 0);
 455         int ret = 0;
 456
 457         dma_buf = dma_buf_get(fd);
 458         if (IS_ERR(dma_buf)) {
 459                 ret = PTR_ERR(dma_buf);
 460                 return ret;
 461         }
 462
 463         list_for_each_entry_safe(drm_buffer, n,
 464                                  &session_info->buffer_list, list) {
 465                 if (drm_buffer->dma_buf == dma_buf) {
 466                         dma_buf_put(dma_buf);
 467                         drm_buffer->last_used = ktime_get();
 468                         return drm_buffer->index;
 469                 }
 470         }
 471
 472         drm_buffer = kzalloc(sizeof(*drm_buffer), GFP_KERNEL);
 473         if (!drm_buffer) {
 474                 ret = -ENOMEM;
 475                 return ret;
 476         }
 477
 478         drm_buffer->dma_buf = dma_buf;
 479         drm_buffer->session_info = session_info;
 480         drm_buffer->last_used = ktime_get();
 481
 482         kref_init(&drm_buffer->ref);
 483
 484         mutex_lock(&iommu_info->iommu_mutex);
 485         drm_info = session_info->iommu_info->private;
 486         if (!drm_info->attached) {
 487                 ret = vcodec_drm_attach_unlock(session_info->iommu_info);
 488                 if (ret)
 489                         goto fail_out;
 490         }
 491
 492         attach = dma_buf_attach(drm_buffer->dma_buf, dev);
 493         if (IS_ERR(attach)) {
 494                 ret = PTR_ERR(attach);
 495                 goto fail_out;
 496         }
 497
 498         get_dma_buf(drm_buffer->dma_buf);
 499
 500         sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
 501         if (IS_ERR(sgt)) {
 502                 ret = PTR_ERR(sgt);
 503                 goto fail_detach;
 504         }
 505
 506         drm_buffer->iova = sg_dma_address(sgt->sgl);
 507         drm_buffer->size = drm_buffer->dma_buf->size;
 508
 509         drm_buffer->attach = attach;
 510         drm_buffer->sgt = sgt;
 511
 512         if (!drm_info->attached)
 513                 iommu_detach_device(domain, dev);
 514
 515         mutex_unlock(&iommu_info->iommu_mutex);
 516
 517         INIT_LIST_HEAD(&drm_buffer->list);
 518         mutex_lock(&session_info->list_mutex);
 519         session_info->buffer_nums++;
 520         vpu_iommu_debug(session_info->debug_level, DEBUG_IOMMU_NORMAL,
 521                         "buffer nums %d\n", session_info->buffer_nums);
 522         if (session_info->buffer_nums > BUFFER_LIST_MAX_NUMS) {
 523                 list_for_each_entry_safe(loop_buffer, n,
 524                                  &session_info->buffer_list, list) {
 525                         if (ktime_to_ns(oldest_time) == 0 ||
 526                             ktime_after(oldest_time,
 527                                         loop_buffer->last_used)) {
 528                                 oldest_time = loop_buffer->last_used;
 529                                 oldest_buffer = loop_buffer;
 530                         }
 531                 }
 532                 kref_put(&oldest_buffer->ref, vcodec_drm_clear_map);
 533                 dma_buf_put(oldest_buffer->dma_buf);
 534                 list_del_init(&oldest_buffer->list);
 535                 kfree(oldest_buffer);
 536                 session_info->buffer_nums--;
 537         }
 538         drm_buffer->index = session_info->max_idx;
 539         list_add_tail(&drm_buffer->list, &session_info->buffer_list);
 540         session_info->max_idx++;
 541         if ((session_info->max_idx & 0xfffffff) == 0)
 542                 session_info->max_idx = 0;
 543         mutex_unlock(&session_info->list_mutex);
 544
 545         return drm_buffer->index;
 546
 547 fail_detach:
 548         dev_err(dev, "dmabuf map attach failed\n");
 549         dma_buf_detach(drm_buffer->dma_buf, attach);
 550         dma_buf_put(drm_buffer->dma_buf);
 551 fail_out:
 552         kfree(drm_buffer);
 553         mutex_unlock(&iommu_info->iommu_mutex);
 554
 555         return ret;
 556 }
 557
 558 static int vcodec_drm_create(struct vcodec_iommu_info *iommu_info)
 559 {
 560         struct vcodec_iommu_drm_info *drm_info;
 561         int ret;
 562
 563         iommu_info->private = kzalloc(sizeof(*drm_info),
 564                                       GFP_KERNEL);
 565         drm_info = iommu_info->private;
 566         if (!drm_info)
 567                 return -ENOMEM;
 568
 569         drm_info->domain = iommu_domain_alloc(&platform_bus_type);
 570         drm_info->attached = false;
 571         if (!drm_info->domain)
 572                 return -ENOMEM;
 573
 574         ret = iommu_get_dma_cookie(drm_info->domain);
 575         if (ret)
 576                 goto err_free_domain;
 577
 578         vcodec_drm_attach(iommu_info);
 579
 580         return 0;
 581
 582 err_free_domain:
 583         iommu_domain_free(drm_info->domain);
 584
 585         return ret;
 586 }
 587
 588 static int vcodec_drm_destroy(struct vcodec_iommu_info *iommu_info)
 589 {
 590         struct vcodec_iommu_drm_info *drm_info = iommu_info->private;
 591
 592         vcodec_drm_detach(iommu_info);
 593         iommu_put_dma_cookie(drm_info->domain);
 594         iommu_domain_free(drm_info->domain);
 595
 596         kfree(drm_info);
 597         iommu_info->private = NULL;
 598
 599         return 0;
 600 }
 601
 602 static struct vcodec_iommu_ops drm_ops = {
 603         .create = vcodec_drm_create,
 604         .import = vcodec_drm_import,
 605         .free = vcodec_drm_free,
 606         .free_fd = vcodec_drm_free_fd,
 607         .map_kernel = vcodec_drm_map_kernel,
 608         .unmap_kernel = vcodec_drm_unmap_kernel,
 609         .map_iommu = vcodec_drm_map_iommu,
 610         .unmap_iommu = vcodec_drm_unmap_iommu,
 611         .destroy = vcodec_drm_destroy,
 612         .dump = vcdoec_drm_dump_info,
 613         .attach = vcodec_drm_attach,
 614         .detach = vcodec_drm_detach,
 615         .clear = vcodec_drm_clear_session,
 616 };
 617
 618 void vcodec_iommu_drm_set_ops(struct vcodec_iommu_info *iommu_info)
 619 {
 620         if (!iommu_info)
 621                 return;
 622         iommu_info->ops = &drm_ops;
 623 }