drivers/iommu/tegra-smmu.c

   1 /*
   2  * IOMMU API for SMMU in Tegra30
   3  *
   4  * Copyright (c) 2011-2012, NVIDIA CORPORATION.  All rights reserved.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along with
  16  * this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18  */
  19
  20 #define pr_fmt(fmt)     "%s(): " fmt, __func__
  21
  22 #include <linux/module.h>
  23 #include <linux/platform_device.h>
  24 #include <linux/spinlock.h>
  25 #include <linux/slab.h>
  26 #include <linux/vmalloc.h>
  27 #include <linux/mm.h>
  28 #include <linux/pagemap.h>
  29 #include <linux/device.h>
  30 #include <linux/sched.h>
  31 #include <linux/iommu.h>
  32 #include <linux/io.h>
  33 #include <linux/of.h>
  34 #include <linux/of_iommu.h>
  35 #include <linux/debugfs.h>
  36 #include <linux/seq_file.h>
  37
  38 #include <asm/page.h>
  39 #include <asm/cacheflush.h>
  40
  41 #include <mach/iomap.h>
  42 #include <mach/smmu.h>
  43 #include <mach/tegra-ahb.h>
  44
  45 /* bitmap of the page sizes currently supported */
  46 #define SMMU_IOMMU_PGSIZES      (SZ_4K)
  47
  48 #define SMMU_CONFIG                             0x10
  49 #define SMMU_CONFIG_DISABLE                     0
  50 #define SMMU_CONFIG_ENABLE                      1
  51
  52 /* REVISIT: To support multiple MCs */
  53 enum {
  54         _MC = 0,
  55 };
  56
  57 enum {
  58         _TLB = 0,
  59         _PTC,
  60 };
  61
  62 #define SMMU_CACHE_CONFIG_BASE                  0x14
  63 #define __SMMU_CACHE_CONFIG(mc, cache)          (SMMU_CACHE_CONFIG_BASE + 4 * cache)
  64 #define SMMU_CACHE_CONFIG(cache)                __SMMU_CACHE_CONFIG(_MC, cache)
  65
  66 #define SMMU_CACHE_CONFIG_STATS_SHIFT           31
  67 #define SMMU_CACHE_CONFIG_STATS_ENABLE          (1 << SMMU_CACHE_CONFIG_STATS_SHIFT)
  68 #define SMMU_CACHE_CONFIG_STATS_TEST_SHIFT      30
  69 #define SMMU_CACHE_CONFIG_STATS_TEST            (1 << SMMU_CACHE_CONFIG_STATS_TEST_SHIFT)
  70
  71 #define SMMU_TLB_CONFIG_HIT_UNDER_MISS__ENABLE  (1 << 29)
  72 #define SMMU_TLB_CONFIG_ACTIVE_LINES__VALUE     0x10
  73 #define SMMU_TLB_CONFIG_RESET_VAL               0x20000010
  74
  75 #define SMMU_PTC_CONFIG_CACHE__ENABLE           (1 << 29)
  76 #define SMMU_PTC_CONFIG_INDEX_MAP__PATTERN      0x3f
  77 #define SMMU_PTC_CONFIG_RESET_VAL               0x2000003f
  78
  79 #define SMMU_PTB_ASID                           0x1c
  80 #define SMMU_PTB_ASID_CURRENT_SHIFT             0
  81
  82 #define SMMU_PTB_DATA                           0x20
  83 #define SMMU_PTB_DATA_RESET_VAL                 0
  84 #define SMMU_PTB_DATA_ASID_NONSECURE_SHIFT      29
  85 #define SMMU_PTB_DATA_ASID_WRITABLE_SHIFT       30
  86 #define SMMU_PTB_DATA_ASID_READABLE_SHIFT       31
  87
  88 #define SMMU_TLB_FLUSH                          0x30
  89 #define SMMU_TLB_FLUSH_VA_MATCH_ALL             0
  90 #define SMMU_TLB_FLUSH_VA_MATCH_SECTION         2
  91 #define SMMU_TLB_FLUSH_VA_MATCH_GROUP           3
  92 #define SMMU_TLB_FLUSH_ASID_SHIFT               29
  93 #define SMMU_TLB_FLUSH_ASID_MATCH_DISABLE       0
  94 #define SMMU_TLB_FLUSH_ASID_MATCH_ENABLE        1
  95 #define SMMU_TLB_FLUSH_ASID_MATCH_SHIFT         31
  96
  97 #define SMMU_PTC_FLUSH                          0x34
  98 #define SMMU_PTC_FLUSH_TYPE_ALL                 0
  99 #define SMMU_PTC_FLUSH_TYPE_ADR                 1
 100 #define SMMU_PTC_FLUSH_ADR_SHIFT                4
 101
 102 #define SMMU_ASID_SECURITY                      0x38
 103
 104 #define SMMU_STATS_CACHE_COUNT_BASE             0x1f0
 105
 106 #define SMMU_STATS_CACHE_COUNT(mc, cache, hitmiss)              \
 107         (SMMU_STATS_CACHE_COUNT_BASE + 8 * cache + 4 * hitmiss)
 108
 109 #define SMMU_TRANSLATION_ENABLE_0               0x228
 110 #define SMMU_TRANSLATION_ENABLE_1               0x22c
 111 #define SMMU_TRANSLATION_ENABLE_2               0x230
 112
 113 #define SMMU_AFI_ASID   0x238   /* PCIE */
 114 #define SMMU_AVPC_ASID  0x23c   /* AVP */
 115 #define SMMU_DC_ASID    0x240   /* Display controller */
 116 #define SMMU_DCB_ASID   0x244   /* Display controller B */
 117 #define SMMU_EPP_ASID   0x248   /* Encoder pre-processor */
 118 #define SMMU_G2_ASID    0x24c   /* 2D engine */
 119 #define SMMU_HC_ASID    0x250   /* Host1x */
 120 #define SMMU_HDA_ASID   0x254   /* High-def audio */
 121 #define SMMU_ISP_ASID   0x258   /* Image signal processor */
 122 #define SMMU_MPE_ASID   0x264   /* MPEG encoder */
 123 #define SMMU_NV_ASID    0x268   /* (3D) */
 124 #define SMMU_NV2_ASID   0x26c   /* (3D) */
 125 #define SMMU_PPCS_ASID  0x270   /* AHB */
 126 #define SMMU_SATA_ASID  0x278   /* SATA */
 127 #define SMMU_VDE_ASID   0x27c   /* Video decoder */
 128 #define SMMU_VI_ASID    0x280   /* Video input */
 129
 130 #define SMMU_PDE_NEXT_SHIFT             28
 131
 132 #define SMMU_TLB_FLUSH_VA_SECTION__MASK         0xffc00000
 133 #define SMMU_TLB_FLUSH_VA_SECTION__SHIFT        12 /* right shift */
 134 #define SMMU_TLB_FLUSH_VA_GROUP__MASK           0xffffc000
 135 #define SMMU_TLB_FLUSH_VA_GROUP__SHIFT          12 /* right shift */
 136 #define SMMU_TLB_FLUSH_VA(iova, which)  \
 137         ((((iova) & SMMU_TLB_FLUSH_VA_##which##__MASK) >> \
 138                 SMMU_TLB_FLUSH_VA_##which##__SHIFT) |   \
 139         SMMU_TLB_FLUSH_VA_MATCH_##which)
 140 #define SMMU_PTB_ASID_CUR(n)    \
 141                 ((n) << SMMU_PTB_ASID_CURRENT_SHIFT)
 142 #define SMMU_TLB_FLUSH_ASID_MATCH_disable               \
 143                 (SMMU_TLB_FLUSH_ASID_MATCH_DISABLE <<   \
 144                         SMMU_TLB_FLUSH_ASID_MATCH_SHIFT)
 145 #define SMMU_TLB_FLUSH_ASID_MATCH__ENABLE               \
 146                 (SMMU_TLB_FLUSH_ASID_MATCH_ENABLE <<    \
 147                         SMMU_TLB_FLUSH_ASID_MATCH_SHIFT)
 148
 149 #define SMMU_PAGE_SHIFT 12
 150 #define SMMU_PAGE_SIZE  (1 << SMMU_PAGE_SHIFT)
 151 #define SMMU_PAGE_MASK  ((1 << SMMU_PAGE_SHIFT) - 1)
 152
 153 #define SMMU_PDIR_COUNT 1024
 154 #define SMMU_PDIR_SIZE  (sizeof(unsigned long) * SMMU_PDIR_COUNT)
 155 #define SMMU_PTBL_COUNT 1024
 156 #define SMMU_PTBL_SIZE  (sizeof(unsigned long) * SMMU_PTBL_COUNT)
 157 #define SMMU_PDIR_SHIFT 12
 158 #define SMMU_PDE_SHIFT  12
 159 #define SMMU_PTE_SHIFT  12
 160 #define SMMU_PFN_MASK   0x000fffff
 161
 162 #define SMMU_ADDR_TO_PFN(addr)  ((addr) >> 12)
 163 #define SMMU_ADDR_TO_PDN(addr)  ((addr) >> 22)
 164 #define SMMU_PDN_TO_ADDR(addr)  ((pdn) << 22)
 165
 166 #define _READABLE       (1 << SMMU_PTB_DATA_ASID_READABLE_SHIFT)
 167 #define _WRITABLE       (1 << SMMU_PTB_DATA_ASID_WRITABLE_SHIFT)
 168 #define _NONSECURE      (1 << SMMU_PTB_DATA_ASID_NONSECURE_SHIFT)
 169 #define _PDE_NEXT       (1 << SMMU_PDE_NEXT_SHIFT)
 170 #define _MASK_ATTR      (_READABLE | _WRITABLE | _NONSECURE)
 171
 172 #define _PDIR_ATTR      (_READABLE | _WRITABLE | _NONSECURE)
 173
 174 #define _PDE_ATTR       (_READABLE | _WRITABLE | _NONSECURE)
 175 #define _PDE_ATTR_N     (_PDE_ATTR | _PDE_NEXT)
 176 #define _PDE_VACANT(pdn)        (((pdn) << 10) | _PDE_ATTR)
 177
 178 #define _PTE_ATTR       (_READABLE | _WRITABLE | _NONSECURE)
 179 #define _PTE_VACANT(addr)       (((addr) >> SMMU_PAGE_SHIFT) | _PTE_ATTR)
 180
 181 #define SMMU_MK_PDIR(page, attr)        \
 182                 ((page_to_phys(page) >> SMMU_PDIR_SHIFT) | (attr))
 183 #define SMMU_MK_PDE(page, attr)         \
 184                 (unsigned long)((page_to_phys(page) >> SMMU_PDE_SHIFT) | (attr))
 185 #define SMMU_EX_PTBL_PAGE(pde)          \
 186                 pfn_to_page((unsigned long)(pde) & SMMU_PFN_MASK)
 187 #define SMMU_PFN_TO_PTE(pfn, attr)      (unsigned long)((pfn) | (attr))
 188
 189 #define SMMU_ASID_ENABLE(asid)  ((asid) | (1 << 31))
 190 #define SMMU_ASID_DISABLE       0
 191 #define SMMU_ASID_ASID(n)       ((n) & ~SMMU_ASID_ENABLE(0))
 192
 193 #define NUM_SMMU_REG_BANKS      3
 194
 195 #define smmu_client_enable_hwgrp(c, m)  smmu_client_set_hwgrp(c, m, 1)
 196 #define smmu_client_disable_hwgrp(c)    smmu_client_set_hwgrp(c, 0, 0)
 197 #define __smmu_client_enable_hwgrp(c, m) __smmu_client_set_hwgrp(c, m, 1)
 198 #define __smmu_client_disable_hwgrp(c)  __smmu_client_set_hwgrp(c, 0, 0)
 199
 200 #define HWGRP_INIT(client) [HWGRP_##client] = SMMU_##client##_ASID
 201
 202 static const u32 smmu_hwgrp_asid_reg[] = {
 203         HWGRP_INIT(AFI),
 204         HWGRP_INIT(AVPC),
 205         HWGRP_INIT(DC),
 206         HWGRP_INIT(DCB),
 207         HWGRP_INIT(EPP),
 208         HWGRP_INIT(G2),
 209         HWGRP_INIT(HC),
 210         HWGRP_INIT(HDA),
 211         HWGRP_INIT(ISP),
 212         HWGRP_INIT(MPE),
 213         HWGRP_INIT(NV),
 214         HWGRP_INIT(NV2),
 215         HWGRP_INIT(PPCS),
 216         HWGRP_INIT(SATA),
 217         HWGRP_INIT(VDE),
 218         HWGRP_INIT(VI),
 219 };
 220 #define HWGRP_ASID_REG(x) (smmu_hwgrp_asid_reg[x])
 221
 222 /*
 223  * Per client for address space
 224  */
 225 struct smmu_client {
 226         struct device           *dev;
 227         struct list_head        list;
 228         struct smmu_as          *as;
 229         u32                     hwgrp;
 230 };
 231
 232 /*
 233  * Per address space
 234  */
 235 struct smmu_as {
 236         struct smmu_device      *smmu;  /* back pointer to container */
 237         unsigned int            asid;
 238         spinlock_t              lock;   /* for pagetable */
 239         struct page             *pdir_page;
 240         unsigned long           pdir_attr;
 241         unsigned long           pde_attr;
 242         unsigned long           pte_attr;
 243         unsigned int            *pte_count;
 244
 245         struct list_head        client;
 246         spinlock_t              client_lock; /* for client list */
 247 };
 248
 249 struct smmu_debugfs_info {
 250         struct smmu_device *smmu;
 251         int mc;
 252         int cache;
 253 };
 254
 255 /*
 256  * Per SMMU device - IOMMU device
 257  */
 258 struct smmu_device {
 259         void __iomem    *regs[NUM_SMMU_REG_BANKS];
 260         unsigned long   iovmm_base;     /* remappable base address */
 261         unsigned long   page_count;     /* total remappable size */
 262         spinlock_t      lock;
 263         char            *name;
 264         struct device   *dev;
 265         struct page *avp_vector_page;   /* dummy page shared by all AS's */
 266
 267         /*
 268          * Register image savers for suspend/resume
 269          */
 270         unsigned long translation_enable_0;
 271         unsigned long translation_enable_1;
 272         unsigned long translation_enable_2;
 273         unsigned long asid_security;
 274
 275         struct dentry *debugfs_root;
 276         struct smmu_debugfs_info *debugfs_info;
 277
 278         struct device_node *ahb;
 279
 280         int             num_as;
 281         struct smmu_as  as[0];          /* Run-time allocated array */
 282 };
 283
 284 static struct smmu_device *smmu_handle; /* unique for a system */
 285
 286 /*
 287  *      SMMU register accessors
 288  */
 289 static inline u32 smmu_read(struct smmu_device *smmu, size_t offs)
 290 {
 291         BUG_ON(offs < 0x10);
 292         if (offs < 0x3c)
 293                 return readl(smmu->regs[0] + offs - 0x10);
 294         BUG_ON(offs < 0x1f0);
 295         if (offs < 0x200)
 296                 return readl(smmu->regs[1] + offs - 0x1f0);
 297         BUG_ON(offs < 0x228);
 298         if (offs < 0x284)
 299                 return readl(smmu->regs[2] + offs - 0x228);
 300         BUG();
 301 }
 302
 303 static inline void smmu_write(struct smmu_device *smmu, u32 val, size_t offs)
 304 {
 305         BUG_ON(offs < 0x10);
 306         if (offs < 0x3c) {
 307                 writel(val, smmu->regs[0] + offs - 0x10);
 308                 return;
 309         }
 310         BUG_ON(offs < 0x1f0);
 311         if (offs < 0x200) {
 312                 writel(val, smmu->regs[1] + offs - 0x1f0);
 313                 return;
 314         }
 315         BUG_ON(offs < 0x228);
 316         if (offs < 0x284) {
 317                 writel(val, smmu->regs[2] + offs - 0x228);
 318                 return;
 319         }
 320         BUG();
 321 }
 322
 323 #define VA_PAGE_TO_PA(va, page) \
 324         (page_to_phys(page) + ((unsigned long)(va) & ~PAGE_MASK))
 325
 326 #define FLUSH_CPU_DCACHE(va, page, size)        \
 327         do {    \
 328                 unsigned long _pa_ = VA_PAGE_TO_PA(va, page);           \
 329                 __cpuc_flush_dcache_area((void *)(va), (size_t)(size)); \
 330                 outer_flush_range(_pa_, _pa_+(size_t)(size));           \
 331         } while (0)
 332
 333 /*
 334  * Any interaction between any block on PPSB and a block on APB or AHB
 335  * must have these read-back barriers to ensure the APB/AHB bus
 336  * transaction is complete before initiating activity on the PPSB
 337  * block.
 338  */
 339 #define FLUSH_SMMU_REGS(smmu)   smmu_read(smmu, SMMU_CONFIG)
 340
 341 #define smmu_client_hwgrp(c) (u32)((c)->dev->platform_data)
 342
 343 static int __smmu_client_set_hwgrp(struct smmu_client *c,
 344                                    unsigned long map, int on)
 345 {
 346         int i;
 347         struct smmu_as *as = c->as;
 348         u32 val, offs, mask = SMMU_ASID_ENABLE(as->asid);
 349         struct smmu_device *smmu = as->smmu;
 350
 351         WARN_ON(!on && map);
 352         if (on && !map)
 353                 return -EINVAL;
 354         if (!on)
 355                 map = smmu_client_hwgrp(c);
 356
 357         for_each_set_bit(i, &map, HWGRP_COUNT) {
 358                 offs = HWGRP_ASID_REG(i);
 359                 val = smmu_read(smmu, offs);
 360                 if (on) {
 361                         if (WARN_ON(val & mask))
 362                                 goto err_hw_busy;
 363                         val |= mask;
 364                 } else {
 365                         WARN_ON((val & mask) == mask);
 366                         val &= ~mask;
 367                 }
 368                 smmu_write(smmu, val, offs);
 369         }
 370         FLUSH_SMMU_REGS(smmu);
 371         c->hwgrp = map;
 372         return 0;
 373
 374 err_hw_busy:
 375         for_each_set_bit(i, &map, HWGRP_COUNT) {
 376                 offs = HWGRP_ASID_REG(i);
 377                 val = smmu_read(smmu, offs);
 378                 val &= ~mask;
 379                 smmu_write(smmu, val, offs);
 380         }
 381         return -EBUSY;
 382 }
 383
 384 static int smmu_client_set_hwgrp(struct smmu_client *c, u32 map, int on)
 385 {
 386         u32 val;
 387         unsigned long flags;
 388         struct smmu_as *as = c->as;
 389         struct smmu_device *smmu = as->smmu;
 390
 391         spin_lock_irqsave(&smmu->lock, flags);
 392         val = __smmu_client_set_hwgrp(c, map, on);
 393         spin_unlock_irqrestore(&smmu->lock, flags);
 394         return val;
 395 }
 396
 397 /*
 398  * Flush all TLB entries and all PTC entries
 399  * Caller must lock smmu
 400  */
 401 static void smmu_flush_regs(struct smmu_device *smmu, int enable)
 402 {
 403         u32 val;
 404
 405         smmu_write(smmu, SMMU_PTC_FLUSH_TYPE_ALL, SMMU_PTC_FLUSH);
 406         FLUSH_SMMU_REGS(smmu);
 407         val = SMMU_TLB_FLUSH_VA_MATCH_ALL |
 408                 SMMU_TLB_FLUSH_ASID_MATCH_disable;
 409         smmu_write(smmu, val, SMMU_TLB_FLUSH);
 410
 411         if (enable)
 412                 smmu_write(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG);
 413         FLUSH_SMMU_REGS(smmu);
 414 }
 415
 416 static int smmu_setup_regs(struct smmu_device *smmu)
 417 {
 418         int i;
 419         u32 val;
 420
 421         for (i = 0; i < smmu->num_as; i++) {
 422                 struct smmu_as *as = &smmu->as[i];
 423                 struct smmu_client *c;
 424
 425                 smmu_write(smmu, SMMU_PTB_ASID_CUR(as->asid), SMMU_PTB_ASID);
 426                 val = as->pdir_page ?
 427                         SMMU_MK_PDIR(as->pdir_page, as->pdir_attr) :
 428                         SMMU_PTB_DATA_RESET_VAL;
 429                 smmu_write(smmu, val, SMMU_PTB_DATA);
 430
 431                 list_for_each_entry(c, &as->client, list)
 432                         __smmu_client_set_hwgrp(c, c->hwgrp, 1);
 433         }
 434
 435         smmu_write(smmu, smmu->translation_enable_0, SMMU_TRANSLATION_ENABLE_0);
 436         smmu_write(smmu, smmu->translation_enable_1, SMMU_TRANSLATION_ENABLE_1);
 437         smmu_write(smmu, smmu->translation_enable_2, SMMU_TRANSLATION_ENABLE_2);
 438         smmu_write(smmu, smmu->asid_security, SMMU_ASID_SECURITY);
 439         smmu_write(smmu, SMMU_TLB_CONFIG_RESET_VAL, SMMU_CACHE_CONFIG(_TLB));
 440         smmu_write(smmu, SMMU_PTC_CONFIG_RESET_VAL, SMMU_CACHE_CONFIG(_PTC));
 441
 442         smmu_flush_regs(smmu, 1);
 443
 444         return tegra_ahb_enable_smmu(smmu->ahb);
 445 }
 446
 447 static void flush_ptc_and_tlb(struct smmu_device *smmu,
 448                       struct smmu_as *as, dma_addr_t iova,
 449                       unsigned long *pte, struct page *page, int is_pde)
 450 {
 451         u32 val;
 452         unsigned long tlb_flush_va = is_pde
 453                 ?  SMMU_TLB_FLUSH_VA(iova, SECTION)
 454                 :  SMMU_TLB_FLUSH_VA(iova, GROUP);
 455
 456         val = SMMU_PTC_FLUSH_TYPE_ADR | VA_PAGE_TO_PA(pte, page);
 457         smmu_write(smmu, val, SMMU_PTC_FLUSH);
 458         FLUSH_SMMU_REGS(smmu);
 459         val = tlb_flush_va |
 460                 SMMU_TLB_FLUSH_ASID_MATCH__ENABLE |
 461                 (as->asid << SMMU_TLB_FLUSH_ASID_SHIFT);
 462         smmu_write(smmu, val, SMMU_TLB_FLUSH);
 463         FLUSH_SMMU_REGS(smmu);
 464 }
 465
 466 static void free_ptbl(struct smmu_as *as, dma_addr_t iova)
 467 {
 468         unsigned long pdn = SMMU_ADDR_TO_PDN(iova);
 469         unsigned long *pdir = (unsigned long *)page_address(as->pdir_page);
 470
 471         if (pdir[pdn] != _PDE_VACANT(pdn)) {
 472                 dev_dbg(as->smmu->dev, "pdn: %lx\n", pdn);
 473
 474                 ClearPageReserved(SMMU_EX_PTBL_PAGE(pdir[pdn]));
 475                 __free_page(SMMU_EX_PTBL_PAGE(pdir[pdn]));
 476                 pdir[pdn] = _PDE_VACANT(pdn);
 477                 FLUSH_CPU_DCACHE(&pdir[pdn], as->pdir_page, sizeof pdir[pdn]);
 478                 flush_ptc_and_tlb(as->smmu, as, iova, &pdir[pdn],
 479                                   as->pdir_page, 1);
 480         }
 481 }
 482
 483 static void free_pdir(struct smmu_as *as)
 484 {
 485         unsigned addr;
 486         int count;
 487         struct device *dev = as->smmu->dev;
 488
 489         if (!as->pdir_page)
 490                 return;
 491
 492         addr = as->smmu->iovmm_base;
 493         count = as->smmu->page_count;
 494         while (count-- > 0) {
 495                 free_ptbl(as, addr);
 496                 addr += SMMU_PAGE_SIZE * SMMU_PTBL_COUNT;
 497         }
 498         ClearPageReserved(as->pdir_page);
 499         __free_page(as->pdir_page);
 500         as->pdir_page = NULL;
 501         devm_kfree(dev, as->pte_count);
 502         as->pte_count = NULL;
 503 }
 504
 505 /*
 506  * Maps PTBL for given iova and returns the PTE address
 507  * Caller must unmap the mapped PTBL returned in *ptbl_page_p
 508  */
 509 static unsigned long *locate_pte(struct smmu_as *as,
 510                                  dma_addr_t iova, bool allocate,
 511                                  struct page **ptbl_page_p,
 512                                  unsigned int **count)
 513 {
 514         unsigned long ptn = SMMU_ADDR_TO_PFN(iova);
 515         unsigned long pdn = SMMU_ADDR_TO_PDN(iova);
 516         unsigned long *pdir = page_address(as->pdir_page);
 517         unsigned long *ptbl;
 518
 519         if (pdir[pdn] != _PDE_VACANT(pdn)) {
 520                 /* Mapped entry table already exists */
 521                 *ptbl_page_p = SMMU_EX_PTBL_PAGE(pdir[pdn]);
 522                 ptbl = page_address(*ptbl_page_p);
 523         } else if (!allocate) {
 524                 return NULL;
 525         } else {
 526                 int pn;
 527                 unsigned long addr = SMMU_PDN_TO_ADDR(pdn);
 528
 529                 /* Vacant - allocate a new page table */
 530                 dev_dbg(as->smmu->dev, "New PTBL pdn: %lx\n", pdn);
 531
 532                 *ptbl_page_p = alloc_page(GFP_ATOMIC);
 533                 if (!*ptbl_page_p) {
 534                         dev_err(as->smmu->dev,
 535                                 "failed to allocate smmu_device page table\n");
 536                         return NULL;
 537                 }
 538                 SetPageReserved(*ptbl_page_p);
 539                 ptbl = (unsigned long *)page_address(*ptbl_page_p);
 540                 for (pn = 0; pn < SMMU_PTBL_COUNT;
 541                      pn++, addr += SMMU_PAGE_SIZE) {
 542                         ptbl[pn] = _PTE_VACANT(addr);
 543                 }
 544                 FLUSH_CPU_DCACHE(ptbl, *ptbl_page_p, SMMU_PTBL_SIZE);
 545                 pdir[pdn] = SMMU_MK_PDE(*ptbl_page_p,
 546                                         as->pde_attr | _PDE_NEXT);
 547                 FLUSH_CPU_DCACHE(&pdir[pdn], as->pdir_page, sizeof pdir[pdn]);
 548                 flush_ptc_and_tlb(as->smmu, as, iova, &pdir[pdn],
 549                                   as->pdir_page, 1);
 550         }
 551         *count = &as->pte_count[pdn];
 552
 553         return &ptbl[ptn % SMMU_PTBL_COUNT];
 554 }
 555
 556 #ifdef CONFIG_SMMU_SIG_DEBUG
 557 static void put_signature(struct smmu_as *as,
 558                           dma_addr_t iova, unsigned long pfn)
 559 {
 560         struct page *page;
 561         unsigned long *vaddr;
 562
 563         page = pfn_to_page(pfn);
 564         vaddr = page_address(page);
 565         if (!vaddr)
 566                 return;
 567
 568         vaddr[0] = iova;
 569         vaddr[1] = pfn << PAGE_SHIFT;
 570         FLUSH_CPU_DCACHE(vaddr, page, sizeof(vaddr[0]) * 2);
 571 }
 572 #else
 573 static inline void put_signature(struct smmu_as *as,
 574                                  unsigned long addr, unsigned long pfn)
 575 {
 576 }
 577 #endif
 578
 579 /*
 580  * Caller must not hold as->lock
 581  */
 582 static int alloc_pdir(struct smmu_as *as)
 583 {
 584         unsigned long *pdir, flags;
 585         int pdn, err = 0;
 586         u32 val;
 587         struct smmu_device *smmu = as->smmu;
 588         struct page *page;
 589         unsigned int *cnt;
 590
 591         /*
 592          * do the allocation, then grab as->lock
 593          */
 594         cnt = devm_kzalloc(smmu->dev,
 595                            sizeof(cnt[0]) * SMMU_PDIR_COUNT,
 596                            GFP_KERNEL);
 597         page = alloc_page(GFP_KERNEL | __GFP_DMA);
 598
 599         spin_lock_irqsave(&as->lock, flags);
 600
 601         if (as->pdir_page) {
 602                 /* We raced, free the redundant */
 603                 err = -EAGAIN;
 604                 goto err_out;
 605         }
 606
 607         if (!page || !cnt) {
 608                 dev_err(smmu->dev, "failed to allocate at %s\n", __func__);
 609                 err = -ENOMEM;
 610                 goto err_out;
 611         }
 612
 613         as->pdir_page = page;
 614         as->pte_count = cnt;
 615
 616         SetPageReserved(as->pdir_page);
 617         pdir = page_address(as->pdir_page);
 618
 619         for (pdn = 0; pdn < SMMU_PDIR_COUNT; pdn++)
 620                 pdir[pdn] = _PDE_VACANT(pdn);
 621         FLUSH_CPU_DCACHE(pdir, as->pdir_page, SMMU_PDIR_SIZE);
 622         val = SMMU_PTC_FLUSH_TYPE_ADR | VA_PAGE_TO_PA(pdir, as->pdir_page);
 623         smmu_write(smmu, val, SMMU_PTC_FLUSH);
 624         FLUSH_SMMU_REGS(as->smmu);
 625         val = SMMU_TLB_FLUSH_VA_MATCH_ALL |
 626                 SMMU_TLB_FLUSH_ASID_MATCH__ENABLE |
 627                 (as->asid << SMMU_TLB_FLUSH_ASID_SHIFT);
 628         smmu_write(smmu, val, SMMU_TLB_FLUSH);
 629         FLUSH_SMMU_REGS(as->smmu);
 630
 631         spin_unlock_irqrestore(&as->lock, flags);
 632
 633         return 0;
 634
 635 err_out:
 636         spin_unlock_irqrestore(&as->lock, flags);
 637
 638         devm_kfree(smmu->dev, cnt);
 639         if (page)
 640                 __free_page(page);
 641         return err;
 642 }
 643
 644 static void __smmu_iommu_unmap(struct smmu_as *as, dma_addr_t iova)
 645 {
 646         unsigned long *pte;
 647         struct page *page;
 648         unsigned int *count;
 649
 650         pte = locate_pte(as, iova, false, &page, &count);
 651         if (WARN_ON(!pte))
 652                 return;
 653
 654         if (WARN_ON(*pte == _PTE_VACANT(iova)))
 655                 return;
 656
 657         *pte = _PTE_VACANT(iova);
 658         FLUSH_CPU_DCACHE(pte, page, sizeof(*pte));
 659         flush_ptc_and_tlb(as->smmu, as, iova, pte, page, 0);
 660         if (!--(*count)) {
 661                 free_ptbl(as, iova);
 662                 smmu_flush_regs(as->smmu, 0);
 663         }
 664 }
 665
 666 static void __smmu_iommu_map_pfn(struct smmu_as *as, dma_addr_t iova,
 667                                  unsigned long pfn)
 668 {
 669         struct smmu_device *smmu = as->smmu;
 670         unsigned long *pte;
 671         unsigned int *count;
 672         struct page *page;
 673
 674         pte = locate_pte(as, iova, true, &page, &count);
 675         if (WARN_ON(!pte))
 676                 return;
 677
 678         if (*pte == _PTE_VACANT(iova))
 679                 (*count)++;
 680         *pte = SMMU_PFN_TO_PTE(pfn, as->pte_attr);
 681         if (unlikely((*pte == _PTE_VACANT(iova))))
 682                 (*count)--;
 683         FLUSH_CPU_DCACHE(pte, page, sizeof(*pte));
 684         flush_ptc_and_tlb(smmu, as, iova, pte, page, 0);
 685         put_signature(as, iova, pfn);
 686 }
 687
 688 static int smmu_iommu_map(struct iommu_domain *domain, unsigned long iova,
 689                           phys_addr_t pa, size_t bytes, int prot)
 690 {
 691         struct smmu_as *as = domain->priv;
 692         unsigned long pfn = __phys_to_pfn(pa);
 693         unsigned long flags;
 694
 695         dev_dbg(as->smmu->dev, "[%d] %08lx:%08x\n", as->asid, iova, pa);
 696
 697         if (!pfn_valid(pfn))
 698                 return -ENOMEM;
 699
 700         spin_lock_irqsave(&as->lock, flags);
 701         __smmu_iommu_map_pfn(as, iova, pfn);
 702         spin_unlock_irqrestore(&as->lock, flags);
 703         return 0;
 704 }
 705
 706 static size_t smmu_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
 707                                size_t bytes)
 708 {
 709         struct smmu_as *as = domain->priv;
 710         unsigned long flags;
 711
 712         dev_dbg(as->smmu->dev, "[%d] %08lx\n", as->asid, iova);
 713
 714         spin_lock_irqsave(&as->lock, flags);
 715         __smmu_iommu_unmap(as, iova);
 716         spin_unlock_irqrestore(&as->lock, flags);
 717         return SMMU_PAGE_SIZE;
 718 }
 719
 720 static phys_addr_t smmu_iommu_iova_to_phys(struct iommu_domain *domain,
 721                                            unsigned long iova)
 722 {
 723         struct smmu_as *as = domain->priv;
 724         unsigned long *pte;
 725         unsigned int *count;
 726         struct page *page;
 727         unsigned long pfn;
 728         unsigned long flags;
 729
 730         spin_lock_irqsave(&as->lock, flags);
 731
 732         pte = locate_pte(as, iova, true, &page, &count);
 733         pfn = *pte & SMMU_PFN_MASK;
 734         WARN_ON(!pfn_valid(pfn));
 735         dev_dbg(as->smmu->dev,
 736                 "iova:%08lx pfn:%08lx asid:%d\n", iova, pfn, as->asid);
 737
 738         spin_unlock_irqrestore(&as->lock, flags);
 739         return PFN_PHYS(pfn);
 740 }
 741
 742 static int smmu_iommu_domain_has_cap(struct iommu_domain *domain,
 743                                      unsigned long cap)
 744 {
 745         return 0;
 746 }
 747
 748 static int smmu_iommu_attach_dev(struct iommu_domain *domain,
 749                                  struct device *dev)
 750 {
 751         struct smmu_as *as = domain->priv;
 752         struct smmu_device *smmu = as->smmu;
 753         struct smmu_client *client, *c;
 754         u32 map;
 755         int err;
 756
 757         client = devm_kzalloc(smmu->dev, sizeof(*c), GFP_KERNEL);
 758         if (!client)
 759                 return -ENOMEM;
 760         client->dev = dev;
 761         client->as = as;
 762         map = (unsigned long)dev->platform_data;
 763         if (!map)
 764                 return -EINVAL;
 765
 766         err = smmu_client_enable_hwgrp(client, map);
 767         if (err)
 768                 goto err_hwgrp;
 769
 770         spin_lock(&as->client_lock);
 771         list_for_each_entry(c, &as->client, list) {
 772                 if (c->dev == dev) {
 773                         dev_err(smmu->dev,
 774                                 "%s is already attached\n", dev_name(c->dev));
 775                         err = -EINVAL;
 776                         goto err_client;
 777                 }
 778         }
 779         list_add(&client->list, &as->client);
 780         spin_unlock(&as->client_lock);
 781
 782         /*
 783          * Reserve "page zero" for AVP vectors using a common dummy
 784          * page.
 785          */
 786         if (map & HWG_AVPC) {
 787                 struct page *page;
 788
 789                 page = as->smmu->avp_vector_page;
 790                 __smmu_iommu_map_pfn(as, 0, page_to_pfn(page));
 791
 792                 pr_info("Reserve \"page zero\" for AVP vectors using a common dummy\n");
 793         }
 794
 795         dev_dbg(smmu->dev, "%s is attached\n", dev_name(dev));
 796         return 0;
 797
 798 err_client:
 799         smmu_client_disable_hwgrp(client);
 800         spin_unlock(&as->client_lock);
 801 err_hwgrp:
 802         devm_kfree(smmu->dev, client);
 803         return err;
 804 }
 805
 806 static void smmu_iommu_detach_dev(struct iommu_domain *domain,
 807                                   struct device *dev)
 808 {
 809         struct smmu_as *as = domain->priv;
 810         struct smmu_device *smmu = as->smmu;
 811         struct smmu_client *c;
 812
 813         spin_lock(&as->client_lock);
 814
 815         list_for_each_entry(c, &as->client, list) {
 816                 if (c->dev == dev) {
 817                         smmu_client_disable_hwgrp(c);
 818                         list_del(&c->list);
 819                         devm_kfree(smmu->dev, c);
 820                         c->as = NULL;
 821                         dev_dbg(smmu->dev,
 822                                 "%s is detached\n", dev_name(c->dev));
 823                         goto out;
 824                 }
 825         }
 826         dev_err(smmu->dev, "Couldn't find %s\n", dev_name(c->dev));
 827 out:
 828         spin_unlock(&as->client_lock);
 829 }
 830
 831 static int smmu_iommu_domain_init(struct iommu_domain *domain)
 832 {
 833         int i, err = -ENODEV;
 834         unsigned long flags;
 835         struct smmu_as *as;
 836         struct smmu_device *smmu = smmu_handle;
 837
 838         /* Look for a free AS with lock held */
 839         for  (i = 0; i < smmu->num_as; i++) {
 840                 as = &smmu->as[i];
 841                 if (!as->pdir_page) {
 842                         err = alloc_pdir(as);
 843                         if (!err)
 844                                 goto found;
 845                 }
 846                 if (err != -EAGAIN)
 847                         break;
 848         }
 849         if (i == smmu->num_as)
 850                 dev_err(smmu->dev,  "no free AS\n");
 851         return err;
 852
 853 found:
 854         spin_lock_irqsave(&smmu->lock, flags);
 855
 856         /* Update PDIR register */
 857         smmu_write(smmu, SMMU_PTB_ASID_CUR(as->asid), SMMU_PTB_ASID);
 858         smmu_write(smmu,
 859                    SMMU_MK_PDIR(as->pdir_page, as->pdir_attr), SMMU_PTB_DATA);
 860         FLUSH_SMMU_REGS(smmu);
 861
 862         spin_unlock_irqrestore(&smmu->lock, flags);
 863
 864         domain->priv = as;
 865
 866         domain->geometry.aperture_start = smmu->iovmm_base;
 867         domain->geometry.aperture_end   = smmu->iovmm_base +
 868                 smmu->page_count * SMMU_PAGE_SIZE - 1;
 869         domain->geometry.force_aperture = true;
 870
 871         dev_dbg(smmu->dev, "smmu_as@%p\n", as);
 872
 873         return 0;
 874 }
 875
 876 static void smmu_iommu_domain_destroy(struct iommu_domain *domain)
 877 {
 878         struct smmu_as *as = domain->priv;
 879         struct smmu_device *smmu = as->smmu;
 880         unsigned long flags;
 881
 882         spin_lock_irqsave(&as->lock, flags);
 883
 884         if (as->pdir_page) {
 885                 spin_lock(&smmu->lock);
 886                 smmu_write(smmu, SMMU_PTB_ASID_CUR(as->asid), SMMU_PTB_ASID);
 887                 smmu_write(smmu, SMMU_PTB_DATA_RESET_VAL, SMMU_PTB_DATA);
 888                 FLUSH_SMMU_REGS(smmu);
 889                 spin_unlock(&smmu->lock);
 890
 891                 free_pdir(as);
 892         }
 893
 894         if (!list_empty(&as->client)) {
 895                 struct smmu_client *c;
 896
 897                 list_for_each_entry(c, &as->client, list)
 898                         smmu_iommu_detach_dev(domain, c->dev);
 899         }
 900
 901         spin_unlock_irqrestore(&as->lock, flags);
 902
 903         domain->priv = NULL;
 904         dev_dbg(smmu->dev, "smmu_as@%p\n", as);
 905 }
 906
 907 static struct iommu_ops smmu_iommu_ops = {
 908         .domain_init    = smmu_iommu_domain_init,
 909         .domain_destroy = smmu_iommu_domain_destroy,
 910         .attach_dev     = smmu_iommu_attach_dev,
 911         .detach_dev     = smmu_iommu_detach_dev,
 912         .map            = smmu_iommu_map,
 913         .unmap          = smmu_iommu_unmap,
 914         .iova_to_phys   = smmu_iommu_iova_to_phys,
 915         .domain_has_cap = smmu_iommu_domain_has_cap,
 916         .pgsize_bitmap  = SMMU_IOMMU_PGSIZES,
 917 };
 918
 919 /* Should be in the order of enum */
 920 static const char * const smmu_debugfs_mc[] = { "mc", };
 921 static const char * const smmu_debugfs_cache[] = {  "tlb", "ptc", };
 922
 923 static ssize_t smmu_debugfs_stats_write(struct file *file,
 924                                         const char __user *buffer,
 925                                         size_t count, loff_t *pos)
 926 {
 927         struct smmu_debugfs_info *info;
 928         struct smmu_device *smmu;
 929         struct dentry *dent;
 930         int i;
 931         enum {
 932                 _OFF = 0,
 933                 _ON,
 934                 _RESET,
 935         };
 936         const char * const command[] = {
 937                 [_OFF]          = "off",
 938                 [_ON]           = "on",
 939                 [_RESET]        = "reset",
 940         };
 941         char str[] = "reset";
 942         u32 val;
 943         size_t offs;
 944
 945         count = min_t(size_t, count, sizeof(str));
 946         if (copy_from_user(str, buffer, count))
 947                 return -EINVAL;
 948
 949         for (i = 0; i < ARRAY_SIZE(command); i++)
 950                 if (strncmp(str, command[i],
 951                             strlen(command[i])) == 0)
 952                         break;
 953
 954         if (i == ARRAY_SIZE(command))
 955                 return -EINVAL;
 956
 957         dent = file->f_dentry;
 958         info = dent->d_inode->i_private;
 959         smmu = info->smmu;
 960
 961         offs = SMMU_CACHE_CONFIG(info->cache);
 962         val = smmu_read(smmu, offs);
 963         switch (i) {
 964         case _OFF:
 965                 val &= ~SMMU_CACHE_CONFIG_STATS_ENABLE;
 966                 val &= ~SMMU_CACHE_CONFIG_STATS_TEST;
 967                 smmu_write(smmu, val, offs);
 968                 break;
 969         case _ON:
 970                 val |= SMMU_CACHE_CONFIG_STATS_ENABLE;
 971                 val &= ~SMMU_CACHE_CONFIG_STATS_TEST;
 972                 smmu_write(smmu, val, offs);
 973                 break;
 974         case _RESET:
 975                 val |= SMMU_CACHE_CONFIG_STATS_TEST;
 976                 smmu_write(smmu, val, offs);
 977                 val &= ~SMMU_CACHE_CONFIG_STATS_TEST;
 978                 smmu_write(smmu, val, offs);
 979                 break;
 980         default:
 981                 BUG();
 982                 break;
 983         }
 984
 985         dev_dbg(smmu->dev, "%s() %08x, %08x @%08x\n", __func__,
 986                 val, smmu_read(smmu, offs), offs);
 987
 988         return count;
 989 }
 990
 991 static int smmu_debugfs_stats_show(struct seq_file *s, void *v)
 992 {
 993         struct smmu_debugfs_info *info;
 994         struct smmu_device *smmu;
 995         struct dentry *dent;
 996         int i;
 997         const char * const stats[] = { "hit", "miss", };
 998
 999         dent = d_find_alias(s->private);
1000         info = dent->d_inode->i_private;
1001         smmu = info->smmu;
1002
1003         for (i = 0; i < ARRAY_SIZE(stats); i++) {
1004                 u32 val;
1005                 size_t offs;
1006
1007                 offs = SMMU_STATS_CACHE_COUNT(info->mc, info->cache, i);
1008                 val = smmu_read(smmu, offs);
1009                 seq_printf(s, "%s:%08x ", stats[i], val);
1010
1011                 dev_dbg(smmu->dev, "%s() %s %08x @%08x\n", __func__,
1012                         stats[i], val, offs);
1013         }
1014         seq_printf(s, "\n");
1015
1016         return 0;
1017 }
1018
1019 static int smmu_debugfs_stats_open(struct inode *inode, struct file *file)
1020 {
1021         return single_open(file, smmu_debugfs_stats_show, inode);
1022 }
1023
1024 static const struct file_operations smmu_debugfs_stats_fops = {
1025         .open           = smmu_debugfs_stats_open,
1026         .read           = seq_read,
1027         .llseek         = seq_lseek,
1028         .release        = single_release,
1029         .write          = smmu_debugfs_stats_write,
1030 };
1031
1032 static void smmu_debugfs_delete(struct smmu_device *smmu)
1033 {
1034         debugfs_remove_recursive(smmu->debugfs_root);
1035         kfree(smmu->debugfs_info);
1036 }
1037
1038 static void smmu_debugfs_create(struct smmu_device *smmu)
1039 {
1040         int i;
1041         size_t bytes;
1042         struct dentry *root;
1043
1044         bytes = ARRAY_SIZE(smmu_debugfs_mc) * ARRAY_SIZE(smmu_debugfs_cache) *
1045                 sizeof(*smmu->debugfs_info);
1046         smmu->debugfs_info = kmalloc(bytes, GFP_KERNEL);
1047         if (!smmu->debugfs_info)
1048                 return;
1049
1050         root = debugfs_create_dir(dev_name(smmu->dev), NULL);
1051         if (!root)
1052                 goto err_out;
1053         smmu->debugfs_root = root;
1054
1055         for (i = 0; i < ARRAY_SIZE(smmu_debugfs_mc); i++) {
1056                 int j;
1057                 struct dentry *mc;
1058
1059                 mc = debugfs_create_dir(smmu_debugfs_mc[i], root);
1060                 if (!mc)
1061                         goto err_out;
1062
1063                 for (j = 0; j < ARRAY_SIZE(smmu_debugfs_cache); j++) {
1064                         struct dentry *cache;
1065                         struct smmu_debugfs_info *info;
1066
1067                         info = smmu->debugfs_info;
1068                         info += i * ARRAY_SIZE(smmu_debugfs_mc) + j;
1069                         info->smmu = smmu;
1070                         info->mc = i;
1071                         info->cache = j;
1072
1073                         cache = debugfs_create_file(smmu_debugfs_cache[j],
1074                                                     S_IWUGO | S_IRUGO, mc,
1075                                                     (void *)info,
1076                                                     &smmu_debugfs_stats_fops);
1077                         if (!cache)
1078                                 goto err_out;
1079                 }
1080         }
1081
1082         return;
1083
1084 err_out:
1085         smmu_debugfs_delete(smmu);
1086 }
1087
1088 static int tegra_smmu_suspend(struct device *dev)
1089 {
1090         struct smmu_device *smmu = dev_get_drvdata(dev);
1091
1092         smmu->translation_enable_0 = smmu_read(smmu, SMMU_TRANSLATION_ENABLE_0);
1093         smmu->translation_enable_1 = smmu_read(smmu, SMMU_TRANSLATION_ENABLE_1);
1094         smmu->translation_enable_2 = smmu_read(smmu, SMMU_TRANSLATION_ENABLE_2);
1095         smmu->asid_security = smmu_read(smmu, SMMU_ASID_SECURITY);
1096         return 0;
1097 }
1098
1099 static int tegra_smmu_resume(struct device *dev)
1100 {
1101         struct smmu_device *smmu = dev_get_drvdata(dev);
1102         unsigned long flags;
1103         int err;
1104
1105         spin_lock_irqsave(&smmu->lock, flags);
1106         err = smmu_setup_regs(smmu);
1107         spin_unlock_irqrestore(&smmu->lock, flags);
1108         return err;
1109 }
1110
1111 static int tegra_smmu_probe(struct platform_device *pdev)
1112 {
1113         struct smmu_device *smmu;
1114         struct device *dev = &pdev->dev;
1115         int i, asids, err = 0;
1116         dma_addr_t uninitialized_var(base);
1117         size_t bytes, uninitialized_var(size);
1118
1119         if (smmu_handle)
1120                 return -EIO;
1121
1122         BUILD_BUG_ON(PAGE_SHIFT != SMMU_PAGE_SHIFT);
1123
1124         if (of_property_read_u32(dev->of_node, "nvidia,#asids", &asids))
1125                 return -ENODEV;
1126
1127         bytes = sizeof(*smmu) + asids * sizeof(*smmu->as);
1128         smmu = devm_kzalloc(dev, bytes, GFP_KERNEL);
1129         if (!smmu) {
1130                 dev_err(dev, "failed to allocate smmu_device\n");
1131                 return -ENOMEM;
1132         }
1133
1134         for (i = 0; i < ARRAY_SIZE(smmu->regs); i++) {
1135                 struct resource *res;
1136
1137                 res = platform_get_resource(pdev, IORESOURCE_MEM, i);
1138                 if (!res)
1139                         return -ENODEV;
1140                 smmu->regs[i] = devm_request_and_ioremap(&pdev->dev, res);
1141                 if (!smmu->regs[i])
1142                         return -EBUSY;
1143         }
1144
1145         err = of_get_dma_window(dev->of_node, NULL, 0, NULL, &base, &size);
1146         if (err)
1147                 return -ENODEV;
1148
1149         if (size & SMMU_PAGE_MASK)
1150                 return -EINVAL;
1151
1152         size >>= SMMU_PAGE_SHIFT;
1153         if (!size)
1154                 return -EINVAL;
1155
1156         smmu->ahb = of_parse_phandle(dev->of_node, "nvidia,ahb", 0);
1157         if (!smmu->ahb)
1158                 return -ENODEV;
1159
1160         smmu->dev = dev;
1161         smmu->num_as = asids;
1162         smmu->iovmm_base = base;
1163         smmu->page_count = size;
1164
1165         smmu->translation_enable_0 = ~0;
1166         smmu->translation_enable_1 = ~0;
1167         smmu->translation_enable_2 = ~0;
1168         smmu->asid_security = 0;
1169
1170         for (i = 0; i < smmu->num_as; i++) {
1171                 struct smmu_as *as = &smmu->as[i];
1172
1173                 as->smmu = smmu;
1174                 as->asid = i;
1175                 as->pdir_attr = _PDIR_ATTR;
1176                 as->pde_attr = _PDE_ATTR;
1177                 as->pte_attr = _PTE_ATTR;
1178
1179                 spin_lock_init(&as->lock);
1180                 INIT_LIST_HEAD(&as->client);
1181         }
1182         spin_lock_init(&smmu->lock);
1183         err = smmu_setup_regs(smmu);
1184         if (err)
1185                 return err;
1186         platform_set_drvdata(pdev, smmu);
1187
1188         smmu->avp_vector_page = alloc_page(GFP_KERNEL);
1189         if (!smmu->avp_vector_page)
1190                 return -ENOMEM;
1191
1192         smmu_debugfs_create(smmu);
1193         smmu_handle = smmu;
1194         return 0;
1195 }
1196
1197 static int tegra_smmu_remove(struct platform_device *pdev)
1198 {
1199         struct smmu_device *smmu = platform_get_drvdata(pdev);
1200         int i;
1201
1202         smmu_debugfs_delete(smmu);
1203
1204         smmu_write(smmu, SMMU_CONFIG_DISABLE, SMMU_CONFIG);
1205         for (i = 0; i < smmu->num_as; i++)
1206                 free_pdir(&smmu->as[i]);
1207         __free_page(smmu->avp_vector_page);
1208         smmu_handle = NULL;
1209         return 0;
1210 }
1211
1212 const struct dev_pm_ops tegra_smmu_pm_ops = {
1213         .suspend        = tegra_smmu_suspend,
1214         .resume         = tegra_smmu_resume,
1215 };
1216
1217 #ifdef CONFIG_OF
1218 static struct of_device_id tegra_smmu_of_match[] __devinitdata = {
1219         { .compatible = "nvidia,tegra30-smmu", },
1220         { },
1221 };
1222 MODULE_DEVICE_TABLE(of, tegra_smmu_of_match);
1223 #endif
1224
1225 static struct platform_driver tegra_smmu_driver = {
1226         .probe          = tegra_smmu_probe,
1227         .remove         = tegra_smmu_remove,
1228         .driver = {
1229                 .owner  = THIS_MODULE,
1230                 .name   = "tegra-smmu",
1231                 .pm     = &tegra_smmu_pm_ops,
1232                 .of_match_table = of_match_ptr(tegra_smmu_of_match),
1233         },
1234 };
1235
1236 static int __devinit tegra_smmu_init(void)
1237 {
1238         bus_set_iommu(&platform_bus_type, &smmu_iommu_ops);
1239         return platform_driver_register(&tegra_smmu_driver);
1240 }
1241
1242 static void __exit tegra_smmu_exit(void)
1243 {
1244         platform_driver_unregister(&tegra_smmu_driver);
1245 }
1246
1247 subsys_initcall(tegra_smmu_init);
1248 module_exit(tegra_smmu_exit);
1249
1250 MODULE_DESCRIPTION("IOMMU API for SMMU in Tegra30");
1251 MODULE_AUTHOR("Hiroshi DOYU <hdoyu@nvidia.com>");
1252 MODULE_ALIAS("platform:tegra-smmu");
1253 MODULE_LICENSE("GPL v2");