drivers/iommu/intel-iommu.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  * Author: Fenghua Yu <fenghua.yu@intel.com>
  22  */
  23
  24 #include <linux/init.h>
  25 #include <linux/bitmap.h>
  26 #include <linux/debugfs.h>
  27 #include <linux/export.h>
  28 #include <linux/slab.h>
  29 #include <linux/irq.h>
  30 #include <linux/interrupt.h>
  31 #include <linux/spinlock.h>
  32 #include <linux/pci.h>
  33 #include <linux/dmar.h>
  34 #include <linux/dma-mapping.h>
  35 #include <linux/mempool.h>
  36 #include <linux/timer.h>
  37 #include <linux/iova.h>
  38 #include <linux/iommu.h>
  39 #include <linux/intel-iommu.h>
  40 #include <linux/syscore_ops.h>
  41 #include <linux/tboot.h>
  42 #include <linux/dmi.h>
  43 #include <linux/pci-ats.h>
  44 #include <linux/memblock.h>
  45 #include <asm/irq_remapping.h>
  46 #include <asm/cacheflush.h>
  47 #include <asm/iommu.h>
  48
  49 #include "irq_remapping.h"
  50 #include "pci.h"
  51
  52 #define ROOT_SIZE               VTD_PAGE_SIZE
  53 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  54
  55 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  56 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  57 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
  58
  59 #define IOAPIC_RANGE_START      (0xfee00000)
  60 #define IOAPIC_RANGE_END        (0xfeefffff)
  61 #define IOVA_START_ADDR         (0x1000)
  62
  63 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
  64
  65 #define MAX_AGAW_WIDTH 64
  66
  67 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
  68 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
  69
  70 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
  71    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
  72 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
  73                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
  74 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
  75
  76 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  77 #define DMA_32BIT_PFN           IOVA_PFN(DMA_BIT_MASK(32))
  78 #define DMA_64BIT_PFN           IOVA_PFN(DMA_BIT_MASK(64))
  79
  80 /* page table handling */
  81 #define LEVEL_STRIDE            (9)
  82 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
  83
  84 /*
  85  * This bitmap is used to advertise the page sizes our hardware support
  86  * to the IOMMU core, which will then use this information to split
  87  * physically contiguous memory regions it is mapping into page sizes
  88  * that we support.
  89  *
  90  * Traditionally the IOMMU core just handed us the mappings directly,
  91  * after making sure the size is an order of a 4KiB page and that the
  92  * mapping has natural alignment.
  93  *
  94  * To retain this behavior, we currently advertise that we support
  95  * all page sizes that are an order of 4KiB.
  96  *
  97  * If at some point we'd like to utilize the IOMMU core's new behavior,
  98  * we could change this to advertise the real page sizes we support.
  99  */
 100 #define INTEL_IOMMU_PGSIZES     (~0xFFFUL)
 101
 102 static inline int agaw_to_level(int agaw)
 103 {
 104         return agaw + 2;
 105 }
 106
 107 static inline int agaw_to_width(int agaw)
 108 {
 109         return 30 + agaw * LEVEL_STRIDE;
 110 }
 111
 112 static inline int width_to_agaw(int width)
 113 {
 114         return (width - 30) / LEVEL_STRIDE;
 115 }
 116
 117 static inline unsigned int level_to_offset_bits(int level)
 118 {
 119         return (level - 1) * LEVEL_STRIDE;
 120 }
 121
 122 static inline int pfn_level_offset(unsigned long pfn, int level)
 123 {
 124         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
 125 }
 126
 127 static inline unsigned long level_mask(int level)
 128 {
 129         return -1UL << level_to_offset_bits(level);
 130 }
 131
 132 static inline unsigned long level_size(int level)
 133 {
 134         return 1UL << level_to_offset_bits(level);
 135 }
 136
 137 static inline unsigned long align_to_level(unsigned long pfn, int level)
 138 {
 139         return (pfn + level_size(level) - 1) & level_mask(level);
 140 }
 141
 142 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
 143 {
 144         return  1 << ((lvl - 1) * LEVEL_STRIDE);
 145 }
 146
 147 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
 148    are never going to work. */
 149 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
 150 {
 151         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
 152 }
 153
 154 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
 155 {
 156         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
 157 }
 158 static inline unsigned long page_to_dma_pfn(struct page *pg)
 159 {
 160         return mm_to_dma_pfn(page_to_pfn(pg));
 161 }
 162 static inline unsigned long virt_to_dma_pfn(void *p)
 163 {
 164         return page_to_dma_pfn(virt_to_page(p));
 165 }
 166
 167 /* global iommu list, set NULL for ignored DMAR units */
 168 static struct intel_iommu **g_iommus;
 169
 170 static void __init check_tylersburg_isoch(void);
 171 static int rwbf_quirk;
 172
 173 /*
 174  * set to 1 to panic kernel if can't successfully enable VT-d
 175  * (used when kernel is launched w/ TXT)
 176  */
 177 static int force_on = 0;
 178
 179 /*
 180  * 0: Present
 181  * 1-11: Reserved
 182  * 12-63: Context Ptr (12 - (haw-1))
 183  * 64-127: Reserved
 184  */
 185 struct root_entry {
 186         u64     val;
 187         u64     rsvd1;
 188 };
 189 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
 190 static inline bool root_present(struct root_entry *root)
 191 {
 192         return (root->val & 1);
 193 }
 194 static inline void set_root_present(struct root_entry *root)
 195 {
 196         root->val |= 1;
 197 }
 198 static inline void set_root_value(struct root_entry *root, unsigned long value)
 199 {
 200         root->val |= value & VTD_PAGE_MASK;
 201 }
 202
 203 static inline struct context_entry *
 204 get_context_addr_from_root(struct root_entry *root)
 205 {
 206         return (struct context_entry *)
 207                 (root_present(root)?phys_to_virt(
 208                 root->val & VTD_PAGE_MASK) :
 209                 NULL);
 210 }
 211
 212 /*
 213  * low 64 bits:
 214  * 0: present
 215  * 1: fault processing disable
 216  * 2-3: translation type
 217  * 12-63: address space root
 218  * high 64 bits:
 219  * 0-2: address width
 220  * 3-6: aval
 221  * 8-23: domain id
 222  */
 223 struct context_entry {
 224         u64 lo;
 225         u64 hi;
 226 };
 227
 228 static inline bool context_present(struct context_entry *context)
 229 {
 230         return (context->lo & 1);
 231 }
 232 static inline void context_set_present(struct context_entry *context)
 233 {
 234         context->lo |= 1;
 235 }
 236
 237 static inline void context_set_fault_enable(struct context_entry *context)
 238 {
 239         context->lo &= (((u64)-1) << 2) | 1;
 240 }
 241
 242 static inline void context_set_translation_type(struct context_entry *context,
 243                                                 unsigned long value)
 244 {
 245         context->lo &= (((u64)-1) << 4) | 3;
 246         context->lo |= (value & 3) << 2;
 247 }
 248
 249 static inline void context_set_address_root(struct context_entry *context,
 250                                             unsigned long value)
 251 {
 252         context->lo |= value & VTD_PAGE_MASK;
 253 }
 254
 255 static inline void context_set_address_width(struct context_entry *context,
 256                                              unsigned long value)
 257 {
 258         context->hi |= value & 7;
 259 }
 260
 261 static inline void context_set_domain_id(struct context_entry *context,
 262                                          unsigned long value)
 263 {
 264         context->hi |= (value & ((1 << 16) - 1)) << 8;
 265 }
 266
 267 static inline void context_clear_entry(struct context_entry *context)
 268 {
 269         context->lo = 0;
 270         context->hi = 0;
 271 }
 272
 273 /*
 274  * 0: readable
 275  * 1: writable
 276  * 2-6: reserved
 277  * 7: super page
 278  * 8-10: available
 279  * 11: snoop behavior
 280  * 12-63: Host physcial address
 281  */
 282 struct dma_pte {
 283         u64 val;
 284 };
 285
 286 static inline void dma_clear_pte(struct dma_pte *pte)
 287 {
 288         pte->val = 0;
 289 }
 290
 291 static inline void dma_set_pte_readable(struct dma_pte *pte)
 292 {
 293         pte->val |= DMA_PTE_READ;
 294 }
 295
 296 static inline void dma_set_pte_writable(struct dma_pte *pte)
 297 {
 298         pte->val |= DMA_PTE_WRITE;
 299 }
 300
 301 static inline void dma_set_pte_snp(struct dma_pte *pte)
 302 {
 303         pte->val |= DMA_PTE_SNP;
 304 }
 305
 306 static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
 307 {
 308         pte->val = (pte->val & ~3) | (prot & 3);
 309 }
 310
 311 static inline u64 dma_pte_addr(struct dma_pte *pte)
 312 {
 313 #ifdef CONFIG_64BIT
 314         return pte->val & VTD_PAGE_MASK;
 315 #else
 316         /* Must have a full atomic 64-bit read */
 317         return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
 318 #endif
 319 }
 320
 321 static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
 322 {
 323         pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
 324 }
 325
 326 static inline bool dma_pte_present(struct dma_pte *pte)
 327 {
 328         return (pte->val & 3) != 0;
 329 }
 330
 331 static inline bool dma_pte_superpage(struct dma_pte *pte)
 332 {
 333         return (pte->val & (1 << 7));
 334 }
 335
 336 static inline int first_pte_in_page(struct dma_pte *pte)
 337 {
 338         return !((unsigned long)pte & ~VTD_PAGE_MASK);
 339 }
 340
 341 /*
 342  * This domain is a statically identity mapping domain.
 343  *      1. This domain creats a static 1:1 mapping to all usable memory.
 344  *      2. It maps to each iommu if successful.
 345  *      3. Each iommu mapps to this domain if successful.
 346  */
 347 static struct dmar_domain *si_domain;
 348 static int hw_pass_through = 1;
 349
 350 /* devices under the same p2p bridge are owned in one domain */
 351 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
 352
 353 /* domain represents a virtual machine, more than one devices
 354  * across iommus may be owned in one domain, e.g. kvm guest.
 355  */
 356 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 1)
 357
 358 /* si_domain contains mulitple devices */
 359 #define DOMAIN_FLAG_STATIC_IDENTITY     (1 << 2)
 360
 361 /* define the limit of IOMMUs supported in each domain */
 362 #ifdef  CONFIG_X86
 363 # define        IOMMU_UNITS_SUPPORTED   MAX_IO_APICS
 364 #else
 365 # define        IOMMU_UNITS_SUPPORTED   64
 366 #endif
 367
 368 struct dmar_domain {
 369         int     id;                     /* domain id */
 370         int     nid;                    /* node id */
 371         DECLARE_BITMAP(iommu_bmp, IOMMU_UNITS_SUPPORTED);
 372                                         /* bitmap of iommus this domain uses*/
 373
 374         struct list_head devices;       /* all devices' list */
 375         struct iova_domain iovad;       /* iova's that belong to this domain */
 376
 377         struct dma_pte  *pgd;           /* virtual address */
 378         int             gaw;            /* max guest address width */
 379
 380         /* adjusted guest address width, 0 is level 2 30-bit */
 381         int             agaw;
 382
 383         int             flags;          /* flags to find out type of domain */
 384
 385         int             iommu_coherency;/* indicate coherency of iommu access */
 386         int             iommu_snooping; /* indicate snooping control feature*/
 387         int             iommu_count;    /* reference count of iommu */
 388         int             iommu_superpage;/* Level of superpages supported:
 389                                            0 == 4KiB (no superpages), 1 == 2MiB,
 390                                            2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
 391         spinlock_t      iommu_lock;     /* protect iommu set in domain */
 392         u64             max_addr;       /* maximum mapped address */
 393 };
 394
 395 /* PCI domain-device relationship */
 396 struct device_domain_info {
 397         struct list_head link;  /* link to domain siblings */
 398         struct list_head global; /* link to global list */
 399         int segment;            /* PCI domain */
 400         u8 bus;                 /* PCI bus number */
 401         u8 devfn;               /* PCI devfn number */
 402         struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */
 403         struct intel_iommu *iommu; /* IOMMU used by this device */
 404         struct dmar_domain *domain; /* pointer to domain */
 405 };
 406
 407 static void flush_unmaps_timeout(unsigned long data);
 408
 409 DEFINE_TIMER(unmap_timer,  flush_unmaps_timeout, 0, 0);
 410
 411 #define HIGH_WATER_MARK 250
 412 struct deferred_flush_tables {
 413         int next;
 414         struct iova *iova[HIGH_WATER_MARK];
 415         struct dmar_domain *domain[HIGH_WATER_MARK];
 416 };
 417
 418 static struct deferred_flush_tables *deferred_flush;
 419
 420 /* bitmap for indexing intel_iommus */
 421 static int g_num_of_iommus;
 422
 423 static DEFINE_SPINLOCK(async_umap_flush_lock);
 424 static LIST_HEAD(unmaps_to_do);
 425
 426 static int timer_on;
 427 static long list_size;
 428
 429 static void domain_remove_dev_info(struct dmar_domain *domain);
 430
 431 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
 432 int dmar_disabled = 0;
 433 #else
 434 int dmar_disabled = 1;
 435 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
 436
 437 int intel_iommu_enabled = 0;
 438 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
 439
 440 static int dmar_map_gfx = 1;
 441 static int dmar_forcedac;
 442 static int intel_iommu_strict;
 443 static int intel_iommu_superpage = 1;
 444
 445 int intel_iommu_gfx_mapped;
 446 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
 447
 448 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 449 static DEFINE_SPINLOCK(device_domain_lock);
 450 static LIST_HEAD(device_domain_list);
 451
 452 static struct iommu_ops intel_iommu_ops;
 453
 454 static int __init intel_iommu_setup(char *str)
 455 {
 456         if (!str)
 457                 return -EINVAL;
 458         while (*str) {
 459                 if (!strncmp(str, "on", 2)) {
 460                         dmar_disabled = 0;
 461                         printk(KERN_INFO "Intel-IOMMU: enabled\n");
 462                 } else if (!strncmp(str, "off", 3)) {
 463                         dmar_disabled = 1;
 464                         printk(KERN_INFO "Intel-IOMMU: disabled\n");
 465                 } else if (!strncmp(str, "igfx_off", 8)) {
 466                         dmar_map_gfx = 0;
 467                         printk(KERN_INFO
 468                                 "Intel-IOMMU: disable GFX device mapping\n");
 469                 } else if (!strncmp(str, "forcedac", 8)) {
 470                         printk(KERN_INFO
 471                                 "Intel-IOMMU: Forcing DAC for PCI devices\n");
 472                         dmar_forcedac = 1;
 473                 } else if (!strncmp(str, "strict", 6)) {
 474                         printk(KERN_INFO
 475                                 "Intel-IOMMU: disable batched IOTLB flush\n");
 476                         intel_iommu_strict = 1;
 477                 } else if (!strncmp(str, "sp_off", 6)) {
 478                         printk(KERN_INFO
 479                                 "Intel-IOMMU: disable supported super page\n");
 480                         intel_iommu_superpage = 0;
 481                 }
 482
 483                 str += strcspn(str, ",");
 484                 while (*str == ',')
 485                         str++;
 486         }
 487         return 0;
 488 }
 489 __setup("intel_iommu=", intel_iommu_setup);
 490
 491 static struct kmem_cache *iommu_domain_cache;
 492 static struct kmem_cache *iommu_devinfo_cache;
 493 static struct kmem_cache *iommu_iova_cache;
 494
 495 static inline void *alloc_pgtable_page(int node)
 496 {
 497         struct page *page;
 498         void *vaddr = NULL;
 499
 500         page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
 501         if (page)
 502                 vaddr = page_address(page);
 503         return vaddr;
 504 }
 505
 506 static inline void free_pgtable_page(void *vaddr)
 507 {
 508         free_page((unsigned long)vaddr);
 509 }
 510
 511 static inline void *alloc_domain_mem(void)
 512 {
 513         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
 514 }
 515
 516 static void free_domain_mem(void *vaddr)
 517 {
 518         kmem_cache_free(iommu_domain_cache, vaddr);
 519 }
 520
 521 static inline void * alloc_devinfo_mem(void)
 522 {
 523         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
 524 }
 525
 526 static inline void free_devinfo_mem(void *vaddr)
 527 {
 528         kmem_cache_free(iommu_devinfo_cache, vaddr);
 529 }
 530
 531 struct iova *alloc_iova_mem(void)
 532 {
 533         return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
 534 }
 535
 536 void free_iova_mem(struct iova *iova)
 537 {
 538         kmem_cache_free(iommu_iova_cache, iova);
 539 }
 540
 541
 542 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 543 {
 544         unsigned long sagaw;
 545         int agaw = -1;
 546
 547         sagaw = cap_sagaw(iommu->cap);
 548         for (agaw = width_to_agaw(max_gaw);
 549              agaw >= 0; agaw--) {
 550                 if (test_bit(agaw, &sagaw))
 551                         break;
 552         }
 553
 554         return agaw;
 555 }
 556
 557 /*
 558  * Calculate max SAGAW for each iommu.
 559  */
 560 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 561 {
 562         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 563 }
 564
 565 /*
 566  * calculate agaw for each iommu.
 567  * "SAGAW" may be different across iommus, use a default agaw, and
 568  * get a supported less agaw for iommus that don't support the default agaw.
 569  */
 570 int iommu_calculate_agaw(struct intel_iommu *iommu)
 571 {
 572         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 573 }
 574
 575 /* This functionin only returns single iommu in a domain */
 576 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 577 {
 578         int iommu_id;
 579
 580         /* si_domain and vm domain should not get here. */
 581         BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
 582         BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
 583
 584         iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
 585         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 586                 return NULL;
 587
 588         return g_iommus[iommu_id];
 589 }
 590
 591 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 592 {
 593         int i;
 594
 595         i = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
 596
 597         domain->iommu_coherency = i < g_num_of_iommus ? 1 : 0;
 598
 599         for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
 600                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 601                         domain->iommu_coherency = 0;
 602                         break;
 603                 }
 604         }
 605 }
 606
 607 static void domain_update_iommu_snooping(struct dmar_domain *domain)
 608 {
 609         int i;
 610
 611         domain->iommu_snooping = 1;
 612
 613         for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
 614                 if (!ecap_sc_support(g_iommus[i]->ecap)) {
 615                         domain->iommu_snooping = 0;
 616                         break;
 617                 }
 618         }
 619 }
 620
 621 static void domain_update_iommu_superpage(struct dmar_domain *domain)
 622 {
 623         struct dmar_drhd_unit *drhd;
 624         struct intel_iommu *iommu = NULL;
 625         int mask = 0xf;
 626
 627         if (!intel_iommu_superpage) {
 628                 domain->iommu_superpage = 0;
 629                 return;
 630         }
 631
 632         /* set iommu_superpage to the smallest common denominator */
 633         for_each_active_iommu(iommu, drhd) {
 634                 mask &= cap_super_page_val(iommu->cap);
 635                 if (!mask) {
 636                         break;
 637                 }
 638         }
 639         domain->iommu_superpage = fls(mask);
 640 }
 641
 642 /* Some capabilities may be different across iommus */
 643 static void domain_update_iommu_cap(struct dmar_domain *domain)
 644 {
 645         domain_update_iommu_coherency(domain);
 646         domain_update_iommu_snooping(domain);
 647         domain_update_iommu_superpage(domain);
 648 }
 649
 650 static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
 651 {
 652         struct dmar_drhd_unit *drhd = NULL;
 653         int i;
 654
 655         for_each_drhd_unit(drhd) {
 656                 if (drhd->ignored)
 657                         continue;
 658                 if (segment != drhd->segment)
 659                         continue;
 660
 661                 for (i = 0; i < drhd->devices_cnt; i++) {
 662                         if (drhd->devices[i] &&
 663                             drhd->devices[i]->bus->number == bus &&
 664                             drhd->devices[i]->devfn == devfn)
 665                                 return drhd->iommu;
 666                         if (drhd->devices[i] &&
 667                             drhd->devices[i]->subordinate &&
 668                             drhd->devices[i]->subordinate->number <= bus &&
 669                             drhd->devices[i]->subordinate->busn_res.end >= bus)
 670                                 return drhd->iommu;
 671                 }
 672
 673                 if (drhd->include_all)
 674                         return drhd->iommu;
 675         }
 676
 677         return NULL;
 678 }
 679
 680 static void domain_flush_cache(struct dmar_domain *domain,
 681                                void *addr, int size)
 682 {
 683         if (!domain->iommu_coherency)
 684                 clflush_cache_range(addr, size);
 685 }
 686
 687 /* Gets context entry for a given bus and devfn */
 688 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
 689                 u8 bus, u8 devfn)
 690 {
 691         struct root_entry *root;
 692         struct context_entry *context;
 693         unsigned long phy_addr;
 694         unsigned long flags;
 695
 696         spin_lock_irqsave(&iommu->lock, flags);
 697         root = &iommu->root_entry[bus];
 698         context = get_context_addr_from_root(root);
 699         if (!context) {
 700                 context = (struct context_entry *)
 701                                 alloc_pgtable_page(iommu->node);
 702                 if (!context) {
 703                         spin_unlock_irqrestore(&iommu->lock, flags);
 704                         return NULL;
 705                 }
 706                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 707                 phy_addr = virt_to_phys((void *)context);
 708                 set_root_value(root, phy_addr);
 709                 set_root_present(root);
 710                 __iommu_flush_cache(iommu, root, sizeof(*root));
 711         }
 712         spin_unlock_irqrestore(&iommu->lock, flags);
 713         return &context[devfn];
 714 }
 715
 716 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 717 {
 718         struct root_entry *root;
 719         struct context_entry *context;
 720         int ret;
 721         unsigned long flags;
 722
 723         spin_lock_irqsave(&iommu->lock, flags);
 724         root = &iommu->root_entry[bus];
 725         context = get_context_addr_from_root(root);
 726         if (!context) {
 727                 ret = 0;
 728                 goto out;
 729         }
 730         ret = context_present(&context[devfn]);
 731 out:
 732         spin_unlock_irqrestore(&iommu->lock, flags);
 733         return ret;
 734 }
 735
 736 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 737 {
 738         struct root_entry *root;
 739         struct context_entry *context;
 740         unsigned long flags;
 741
 742         spin_lock_irqsave(&iommu->lock, flags);
 743         root = &iommu->root_entry[bus];
 744         context = get_context_addr_from_root(root);
 745         if (context) {
 746                 context_clear_entry(&context[devfn]);
 747                 __iommu_flush_cache(iommu, &context[devfn], \
 748                         sizeof(*context));
 749         }
 750         spin_unlock_irqrestore(&iommu->lock, flags);
 751 }
 752
 753 static void free_context_table(struct intel_iommu *iommu)
 754 {
 755         struct root_entry *root;
 756         int i;
 757         unsigned long flags;
 758         struct context_entry *context;
 759
 760         spin_lock_irqsave(&iommu->lock, flags);
 761         if (!iommu->root_entry) {
 762                 goto out;
 763         }
 764         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 765                 root = &iommu->root_entry[i];
 766                 context = get_context_addr_from_root(root);
 767                 if (context)
 768                         free_pgtable_page(context);
 769         }
 770         free_pgtable_page(iommu->root_entry);
 771         iommu->root_entry = NULL;
 772 out:
 773         spin_unlock_irqrestore(&iommu->lock, flags);
 774 }
 775
 776 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
 777                                       unsigned long pfn, int target_level)
 778 {
 779         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 780         struct dma_pte *parent, *pte = NULL;
 781         int level = agaw_to_level(domain->agaw);
 782         int offset;
 783
 784         BUG_ON(!domain->pgd);
 785
 786         if (addr_width < BITS_PER_LONG && pfn >> addr_width)
 787                 /* Address beyond IOMMU's addressing capabilities. */
 788                 return NULL;
 789
 790         parent = domain->pgd;
 791
 792         while (level > 0) {
 793                 void *tmp_page;
 794
 795                 offset = pfn_level_offset(pfn, level);
 796                 pte = &parent[offset];
 797                 if (!target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
 798                         break;
 799                 if (level == target_level)
 800                         break;
 801
 802                 if (!dma_pte_present(pte)) {
 803                         uint64_t pteval;
 804
 805                         tmp_page = alloc_pgtable_page(domain->nid);
 806
 807                         if (!tmp_page)
 808                                 return NULL;
 809
 810                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
 811                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
 812                         if (cmpxchg64(&pte->val, 0ULL, pteval)) {
 813                                 /* Someone else set it while we were thinking; use theirs. */
 814                                 free_pgtable_page(tmp_page);
 815                         } else {
 816                                 dma_pte_addr(pte);
 817                                 domain_flush_cache(domain, pte, sizeof(*pte));
 818                         }
 819                 }
 820                 parent = phys_to_virt(dma_pte_addr(pte));
 821                 level--;
 822         }
 823
 824         return pte;
 825 }
 826
 827
 828 /* return address's pte at specific level */
 829 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
 830                                          unsigned long pfn,
 831                                          int level, int *large_page)
 832 {
 833         struct dma_pte *parent, *pte = NULL;
 834         int total = agaw_to_level(domain->agaw);
 835         int offset;
 836
 837         parent = domain->pgd;
 838         while (level <= total) {
 839                 offset = pfn_level_offset(pfn, total);
 840                 pte = &parent[offset];
 841                 if (level == total)
 842                         return pte;
 843
 844                 if (!dma_pte_present(pte)) {
 845                         *large_page = total;
 846                         break;
 847                 }
 848
 849                 if (pte->val & DMA_PTE_LARGE_PAGE) {
 850                         *large_page = total;
 851                         return pte;
 852                 }
 853
 854                 parent = phys_to_virt(dma_pte_addr(pte));
 855                 total--;
 856         }
 857         return NULL;
 858 }
 859
 860 /* clear last level pte, a tlb flush should be followed */
 861 static int dma_pte_clear_range(struct dmar_domain *domain,
 862                                 unsigned long start_pfn,
 863                                 unsigned long last_pfn)
 864 {
 865         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 866         unsigned int large_page = 1;
 867         struct dma_pte *first_pte, *pte;
 868         int order;
 869
 870         BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
 871         BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
 872         BUG_ON(start_pfn > last_pfn);
 873
 874         /* we don't need lock here; nobody else touches the iova range */
 875         do {
 876                 large_page = 1;
 877                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
 878                 if (!pte) {
 879                         start_pfn = align_to_level(start_pfn + 1, large_page + 1);
 880                         continue;
 881                 }
 882                 do {
 883                         dma_clear_pte(pte);
 884                         start_pfn += lvl_to_nr_pages(large_page);
 885                         pte++;
 886                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
 887
 888                 domain_flush_cache(domain, first_pte,
 889                                    (void *)pte - (void *)first_pte);
 890
 891         } while (start_pfn && start_pfn <= last_pfn);
 892
 893         order = (large_page - 1) * 9;
 894         return order;
 895 }
 896
 897 static void dma_pte_free_level(struct dmar_domain *domain, int level,
 898                                struct dma_pte *pte, unsigned long pfn,
 899                                unsigned long start_pfn, unsigned long last_pfn)
 900 {
 901         pfn = max(start_pfn, pfn);
 902         pte = &pte[pfn_level_offset(pfn, level)];
 903
 904         do {
 905                 unsigned long level_pfn;
 906                 struct dma_pte *level_pte;
 907
 908                 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
 909                         goto next;
 910
 911                 level_pfn = pfn & level_mask(level - 1);
 912                 level_pte = phys_to_virt(dma_pte_addr(pte));
 913
 914                 if (level > 2)
 915                         dma_pte_free_level(domain, level - 1, level_pte,
 916                                            level_pfn, start_pfn, last_pfn);
 917
 918                 /* If range covers entire pagetable, free it */
 919                 if (!(start_pfn > level_pfn ||
 920                       last_pfn < level_pfn + level_size(level) - 1)) {
 921                         dma_clear_pte(pte);
 922                         domain_flush_cache(domain, pte, sizeof(*pte));
 923                         free_pgtable_page(level_pte);
 924                 }
 925 next:
 926                 pfn += level_size(level);
 927         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
 928 }
 929
 930 /* free page table pages. last level pte should already be cleared */
 931 static void dma_pte_free_pagetable(struct dmar_domain *domain,
 932                                    unsigned long start_pfn,
 933                                    unsigned long last_pfn)
 934 {
 935         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 936
 937         BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
 938         BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
 939         BUG_ON(start_pfn > last_pfn);
 940
 941         /* We don't need lock here; nobody else touches the iova range */
 942         dma_pte_free_level(domain, agaw_to_level(domain->agaw),
 943                            domain->pgd, 0, start_pfn, last_pfn);
 944
 945         /* free pgd */
 946         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
 947                 free_pgtable_page(domain->pgd);
 948                 domain->pgd = NULL;
 949         }
 950 }
 951
 952 /* iommu handling */
 953 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
 954 {
 955         struct root_entry *root;
 956         unsigned long flags;
 957
 958         root = (struct root_entry *)alloc_pgtable_page(iommu->node);
 959         if (!root)
 960                 return -ENOMEM;
 961
 962         __iommu_flush_cache(iommu, root, ROOT_SIZE);
 963
 964         spin_lock_irqsave(&iommu->lock, flags);
 965         iommu->root_entry = root;
 966         spin_unlock_irqrestore(&iommu->lock, flags);
 967
 968         return 0;
 969 }
 970
 971 static void iommu_set_root_entry(struct intel_iommu *iommu)
 972 {
 973         void *addr;
 974         u32 sts;
 975         unsigned long flag;
 976
 977         addr = iommu->root_entry;
 978
 979         raw_spin_lock_irqsave(&iommu->register_lock, flag);
 980         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
 981
 982         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
 983
 984         /* Make sure hardware complete it */
 985         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 986                       readl, (sts & DMA_GSTS_RTPS), sts);
 987
 988         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
 989 }
 990
 991 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
 992 {
 993         u32 val;
 994         unsigned long flag;
 995
 996         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
 997                 return;
 998
 999         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1000         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1001
1002         /* Make sure hardware complete it */
1003         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1004                       readl, (!(val & DMA_GSTS_WBFS)), val);
1005
1006         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1007 }
1008
1009 /* return value determine if we need a write buffer flush */
1010 static void __iommu_flush_context(struct intel_iommu *iommu,
1011                                   u16 did, u16 source_id, u8 function_mask,
1012                                   u64 type)
1013 {
1014         u64 val = 0;
1015         unsigned long flag;
1016
1017         switch (type) {
1018         case DMA_CCMD_GLOBAL_INVL:
1019                 val = DMA_CCMD_GLOBAL_INVL;
1020                 break;
1021         case DMA_CCMD_DOMAIN_INVL:
1022                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1023                 break;
1024         case DMA_CCMD_DEVICE_INVL:
1025                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1026                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1027                 break;
1028         default:
1029                 BUG();
1030         }
1031         val |= DMA_CCMD_ICC;
1032
1033         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1034         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1035
1036         /* Make sure hardware complete it */
1037         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1038                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1039
1040         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1041 }
1042
1043 /* return value determine if we need a write buffer flush */
1044 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1045                                 u64 addr, unsigned int size_order, u64 type)
1046 {
1047         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1048         u64 val = 0, val_iva = 0;
1049         unsigned long flag;
1050
1051         switch (type) {
1052         case DMA_TLB_GLOBAL_FLUSH:
1053                 /* global flush doesn't need set IVA_REG */
1054                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1055                 break;
1056         case DMA_TLB_DSI_FLUSH:
1057                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1058                 break;
1059         case DMA_TLB_PSI_FLUSH:
1060                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1061                 /* Note: always flush non-leaf currently */
1062                 val_iva = size_order | addr;
1063                 break;
1064         default:
1065                 BUG();
1066         }
1067         /* Note: set drain read/write */
1068 #if 0
1069         /*
1070          * This is probably to be super secure.. Looks like we can
1071          * ignore it without any impact.
1072          */
1073         if (cap_read_drain(iommu->cap))
1074                 val |= DMA_TLB_READ_DRAIN;
1075 #endif
1076         if (cap_write_drain(iommu->cap))
1077                 val |= DMA_TLB_WRITE_DRAIN;
1078
1079         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1080         /* Note: Only uses first TLB reg currently */
1081         if (val_iva)
1082                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1083         dmar_writeq(iommu->reg + tlb_offset + 8, val);
1084
1085         /* Make sure hardware complete it */
1086         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1087                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1088
1089         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1090
1091         /* check IOTLB invalidation granularity */
1092         if (DMA_TLB_IAIG(val) == 0)
1093                 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1094         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1095                 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
1096                         (unsigned long long)DMA_TLB_IIRG(type),
1097                         (unsigned long long)DMA_TLB_IAIG(val));
1098 }
1099
1100 static struct device_domain_info *iommu_support_dev_iotlb(
1101         struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
1102 {
1103         int found = 0;
1104         unsigned long flags;
1105         struct device_domain_info *info;
1106         struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
1107
1108         if (!ecap_dev_iotlb_support(iommu->ecap))
1109                 return NULL;
1110
1111         if (!iommu->qi)
1112                 return NULL;
1113
1114         spin_lock_irqsave(&device_domain_lock, flags);
1115         list_for_each_entry(info, &domain->devices, link)
1116                 if (info->bus == bus && info->devfn == devfn) {
1117                         found = 1;
1118                         break;
1119                 }
1120         spin_unlock_irqrestore(&device_domain_lock, flags);
1121
1122         if (!found || !info->dev)
1123                 return NULL;
1124
1125         if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1126                 return NULL;
1127
1128         if (!dmar_find_matched_atsr_unit(info->dev))
1129                 return NULL;
1130
1131         info->iommu = iommu;
1132
1133         return info;
1134 }
1135
1136 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1137 {
1138         if (!info)
1139                 return;
1140
1141         pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1142 }
1143
1144 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1145 {
1146         if (!info->dev || !pci_ats_enabled(info->dev))
1147                 return;
1148
1149         pci_disable_ats(info->dev);
1150 }
1151
1152 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1153                                   u64 addr, unsigned mask)
1154 {
1155         u16 sid, qdep;
1156         unsigned long flags;
1157         struct device_domain_info *info;
1158
1159         spin_lock_irqsave(&device_domain_lock, flags);
1160         list_for_each_entry(info, &domain->devices, link) {
1161                 if (!info->dev || !pci_ats_enabled(info->dev))
1162                         continue;
1163
1164                 sid = info->bus << 8 | info->devfn;
1165                 qdep = pci_ats_queue_depth(info->dev);
1166                 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1167         }
1168         spin_unlock_irqrestore(&device_domain_lock, flags);
1169 }
1170
1171 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1172                                   unsigned long pfn, unsigned int pages, int map)
1173 {
1174         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1175         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1176
1177         BUG_ON(pages == 0);
1178
1179         /*
1180          * Fallback to domain selective flush if no PSI support or the size is
1181          * too big.
1182          * PSI requires page size to be 2 ^ x, and the base address is naturally
1183          * aligned to the size
1184          */
1185         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1186                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1187                                                 DMA_TLB_DSI_FLUSH);
1188         else
1189                 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1190                                                 DMA_TLB_PSI_FLUSH);
1191
1192         /*
1193          * In caching mode, changes of pages from non-present to present require
1194          * flush. However, device IOTLB doesn't need to be flushed in this case.
1195          */
1196         if (!cap_caching_mode(iommu->cap) || !map)
1197                 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1198 }
1199
1200 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1201 {
1202         u32 pmen;
1203         unsigned long flags;
1204
1205         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1206         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1207         pmen &= ~DMA_PMEN_EPM;
1208         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1209
1210         /* wait for the protected region status bit to clear */
1211         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1212                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1213
1214         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1215 }
1216
1217 static int iommu_enable_translation(struct intel_iommu *iommu)
1218 {
1219         u32 sts;
1220         unsigned long flags;
1221
1222         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1223         iommu->gcmd |= DMA_GCMD_TE;
1224         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1225
1226         /* Make sure hardware complete it */
1227         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1228                       readl, (sts & DMA_GSTS_TES), sts);
1229
1230         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1231         return 0;
1232 }
1233
1234 static int iommu_disable_translation(struct intel_iommu *iommu)
1235 {
1236         u32 sts;
1237         unsigned long flag;
1238
1239         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1240         iommu->gcmd &= ~DMA_GCMD_TE;
1241         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1242
1243         /* Make sure hardware complete it */
1244         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1245                       readl, (!(sts & DMA_GSTS_TES)), sts);
1246
1247         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1248         return 0;
1249 }
1250
1251
1252 static int iommu_init_domains(struct intel_iommu *iommu)
1253 {
1254         unsigned long ndomains;
1255         unsigned long nlongs;
1256
1257         ndomains = cap_ndoms(iommu->cap);
1258         pr_debug("IOMMU %d: Number of Domains supported <%ld>\n", iommu->seq_id,
1259                         ndomains);
1260         nlongs = BITS_TO_LONGS(ndomains);
1261
1262         spin_lock_init(&iommu->lock);
1263
1264         /* TBD: there might be 64K domains,
1265          * consider other allocation for future chip
1266          */
1267         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1268         if (!iommu->domain_ids) {
1269                 printk(KERN_ERR "Allocating domain id array failed\n");
1270                 return -ENOMEM;
1271         }
1272         iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1273                         GFP_KERNEL);
1274         if (!iommu->domains) {
1275                 printk(KERN_ERR "Allocating domain array failed\n");
1276                 return -ENOMEM;
1277         }
1278
1279         /*
1280          * if Caching mode is set, then invalid translations are tagged
1281          * with domainid 0. Hence we need to pre-allocate it.
1282          */
1283         if (cap_caching_mode(iommu->cap))
1284                 set_bit(0, iommu->domain_ids);
1285         return 0;
1286 }
1287
1288
1289 static void domain_exit(struct dmar_domain *domain);
1290 static void vm_domain_exit(struct dmar_domain *domain);
1291
1292 void free_dmar_iommu(struct intel_iommu *iommu)
1293 {
1294         struct dmar_domain *domain;
1295         int i;
1296         unsigned long flags;
1297
1298         if ((iommu->domains) && (iommu->domain_ids)) {
1299                 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
1300                         domain = iommu->domains[i];
1301                         clear_bit(i, iommu->domain_ids);
1302
1303                         spin_lock_irqsave(&domain->iommu_lock, flags);
1304                         if (--domain->iommu_count == 0) {
1305                                 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1306                                         vm_domain_exit(domain);
1307                                 else
1308                                         domain_exit(domain);
1309                         }
1310                         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1311                 }
1312         }
1313
1314         if (iommu->gcmd & DMA_GCMD_TE)
1315                 iommu_disable_translation(iommu);
1316
1317         if (iommu->irq) {
1318                 irq_set_handler_data(iommu->irq, NULL);
1319                 /* This will mask the irq */
1320                 free_irq(iommu->irq, iommu);
1321                 destroy_irq(iommu->irq);
1322         }
1323
1324         kfree(iommu->domains);
1325         kfree(iommu->domain_ids);
1326
1327         g_iommus[iommu->seq_id] = NULL;
1328
1329         /* if all iommus are freed, free g_iommus */
1330         for (i = 0; i < g_num_of_iommus; i++) {
1331                 if (g_iommus[i])
1332                         break;
1333         }
1334
1335         if (i == g_num_of_iommus)
1336                 kfree(g_iommus);
1337
1338         /* free context mapping */
1339         free_context_table(iommu);
1340 }
1341
1342 static struct dmar_domain *alloc_domain(void)
1343 {
1344         struct dmar_domain *domain;
1345
1346         domain = alloc_domain_mem();
1347         if (!domain)
1348                 return NULL;
1349
1350         domain->nid = -1;
1351         memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
1352         domain->flags = 0;
1353
1354         return domain;
1355 }
1356
1357 static int iommu_attach_domain(struct dmar_domain *domain,
1358                                struct intel_iommu *iommu)
1359 {
1360         int num;
1361         unsigned long ndomains;
1362         unsigned long flags;
1363
1364         ndomains = cap_ndoms(iommu->cap);
1365
1366         spin_lock_irqsave(&iommu->lock, flags);
1367
1368         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1369         if (num >= ndomains) {
1370                 spin_unlock_irqrestore(&iommu->lock, flags);
1371                 printk(KERN_ERR "IOMMU: no free domain ids\n");
1372                 return -ENOMEM;
1373         }
1374
1375         domain->id = num;
1376         set_bit(num, iommu->domain_ids);
1377         set_bit(iommu->seq_id, domain->iommu_bmp);
1378         iommu->domains[num] = domain;
1379         spin_unlock_irqrestore(&iommu->lock, flags);
1380
1381         return 0;
1382 }
1383
1384 static void iommu_detach_domain(struct dmar_domain *domain,
1385                                 struct intel_iommu *iommu)
1386 {
1387         unsigned long flags;
1388         int num, ndomains;
1389         int found = 0;
1390
1391         spin_lock_irqsave(&iommu->lock, flags);
1392         ndomains = cap_ndoms(iommu->cap);
1393         for_each_set_bit(num, iommu->domain_ids, ndomains) {
1394                 if (iommu->domains[num] == domain) {
1395                         found = 1;
1396                         break;
1397                 }
1398         }
1399
1400         if (found) {
1401                 clear_bit(num, iommu->domain_ids);
1402                 clear_bit(iommu->seq_id, domain->iommu_bmp);
1403                 iommu->domains[num] = NULL;
1404         }
1405         spin_unlock_irqrestore(&iommu->lock, flags);
1406 }
1407
1408 static struct iova_domain reserved_iova_list;
1409 static struct lock_class_key reserved_rbtree_key;
1410
1411 static int dmar_init_reserved_ranges(void)
1412 {
1413         struct pci_dev *pdev = NULL;
1414         struct iova *iova;
1415         int i;
1416
1417         init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1418
1419         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1420                 &reserved_rbtree_key);
1421
1422         /* IOAPIC ranges shouldn't be accessed by DMA */
1423         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1424                 IOVA_PFN(IOAPIC_RANGE_END));
1425         if (!iova) {
1426                 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1427                 return -ENODEV;
1428         }
1429
1430         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1431         for_each_pci_dev(pdev) {
1432                 struct resource *r;
1433
1434                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1435                         r = &pdev->resource[i];
1436                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1437                                 continue;
1438                         iova = reserve_iova(&reserved_iova_list,
1439                                             IOVA_PFN(r->start),
1440                                             IOVA_PFN(r->end));
1441                         if (!iova) {
1442                                 printk(KERN_ERR "Reserve iova failed\n");
1443                                 return -ENODEV;
1444                         }
1445                 }
1446         }
1447         return 0;
1448 }
1449
1450 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1451 {
1452         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1453 }
1454
1455 static inline int guestwidth_to_adjustwidth(int gaw)
1456 {
1457         int agaw;
1458         int r = (gaw - 12) % 9;
1459
1460         if (r == 0)
1461                 agaw = gaw;
1462         else
1463                 agaw = gaw + 9 - r;
1464         if (agaw > 64)
1465                 agaw = 64;
1466         return agaw;
1467 }
1468
1469 static int domain_init(struct dmar_domain *domain, int guest_width)
1470 {
1471         struct intel_iommu *iommu;
1472         int adjust_width, agaw;
1473         unsigned long sagaw;
1474
1475         init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1476         spin_lock_init(&domain->iommu_lock);
1477
1478         domain_reserve_special_ranges(domain);
1479
1480         /* calculate AGAW */
1481         iommu = domain_get_iommu(domain);
1482         if (guest_width > cap_mgaw(iommu->cap))
1483                 guest_width = cap_mgaw(iommu->cap);
1484         domain->gaw = guest_width;
1485         adjust_width = guestwidth_to_adjustwidth(guest_width);
1486         agaw = width_to_agaw(adjust_width);
1487         sagaw = cap_sagaw(iommu->cap);
1488         if (!test_bit(agaw, &sagaw)) {
1489                 /* hardware doesn't support it, choose a bigger one */
1490                 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1491                 agaw = find_next_bit(&sagaw, 5, agaw);
1492                 if (agaw >= 5)
1493                         return -ENODEV;
1494         }
1495         domain->agaw = agaw;
1496         INIT_LIST_HEAD(&domain->devices);
1497
1498         if (ecap_coherent(iommu->ecap))
1499                 domain->iommu_coherency = 1;
1500         else
1501                 domain->iommu_coherency = 0;
1502
1503         if (ecap_sc_support(iommu->ecap))
1504                 domain->iommu_snooping = 1;
1505         else
1506                 domain->iommu_snooping = 0;
1507
1508         domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1509         domain->iommu_count = 1;
1510         domain->nid = iommu->node;
1511
1512         /* always allocate the top pgd */
1513         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1514         if (!domain->pgd)
1515                 return -ENOMEM;
1516         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1517         return 0;
1518 }
1519
1520 static void domain_exit(struct dmar_domain *domain)
1521 {
1522         struct dmar_drhd_unit *drhd;
1523         struct intel_iommu *iommu;
1524
1525         /* Domain 0 is reserved, so dont process it */
1526         if (!domain)
1527                 return;
1528
1529         /* Flush any lazy unmaps that may reference this domain */
1530         if (!intel_iommu_strict)
1531                 flush_unmaps_timeout(0);
1532
1533         domain_remove_dev_info(domain);
1534         /* destroy iovas */
1535         put_iova_domain(&domain->iovad);
1536
1537         /* clear ptes */
1538         dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1539
1540         /* free page tables */
1541         dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1542
1543         for_each_active_iommu(iommu, drhd)
1544                 if (test_bit(iommu->seq_id, domain->iommu_bmp))
1545                         iommu_detach_domain(domain, iommu);
1546
1547         free_domain_mem(domain);
1548 }
1549
1550 static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1551                                  u8 bus, u8 devfn, int translation)
1552 {
1553         struct context_entry *context;
1554         unsigned long flags;
1555         struct intel_iommu *iommu;
1556         struct dma_pte *pgd;
1557         unsigned long num;
1558         unsigned long ndomains;
1559         int id;
1560         int agaw;
1561         struct device_domain_info *info = NULL;
1562
1563         pr_debug("Set context mapping for %02x:%02x.%d\n",
1564                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1565
1566         BUG_ON(!domain->pgd);
1567         BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1568                translation != CONTEXT_TT_MULTI_LEVEL);
1569
1570         iommu = device_to_iommu(segment, bus, devfn);
1571         if (!iommu)
1572                 return -ENODEV;
1573
1574         context = device_to_context_entry(iommu, bus, devfn);
1575         if (!context)
1576                 return -ENOMEM;
1577         spin_lock_irqsave(&iommu->lock, flags);
1578         if (context_present(context)) {
1579                 spin_unlock_irqrestore(&iommu->lock, flags);
1580                 return 0;
1581         }
1582
1583         id = domain->id;
1584         pgd = domain->pgd;
1585
1586         if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1587             domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
1588                 int found = 0;
1589
1590                 /* find an available domain id for this device in iommu */
1591                 ndomains = cap_ndoms(iommu->cap);
1592                 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1593                         if (iommu->domains[num] == domain) {
1594                                 id = num;
1595                                 found = 1;
1596                                 break;
1597                         }
1598                 }
1599
1600                 if (found == 0) {
1601                         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1602                         if (num >= ndomains) {
1603                                 spin_unlock_irqrestore(&iommu->lock, flags);
1604                                 printk(KERN_ERR "IOMMU: no free domain ids\n");
1605                                 return -EFAULT;
1606                         }
1607
1608                         set_bit(num, iommu->domain_ids);
1609                         iommu->domains[num] = domain;
1610                         id = num;
1611                 }
1612
1613                 /* Skip top levels of page tables for
1614                  * iommu which has less agaw than default.
1615                  * Unnecessary for PT mode.
1616                  */
1617                 if (translation != CONTEXT_TT_PASS_THROUGH) {
1618                         for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1619                                 pgd = phys_to_virt(dma_pte_addr(pgd));
1620                                 if (!dma_pte_present(pgd)) {
1621                                         spin_unlock_irqrestore(&iommu->lock, flags);
1622                                         return -ENOMEM;
1623                                 }
1624                         }
1625                 }
1626         }
1627
1628         context_set_domain_id(context, id);
1629
1630         if (translation != CONTEXT_TT_PASS_THROUGH) {
1631                 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1632                 translation = info ? CONTEXT_TT_DEV_IOTLB :
1633                                      CONTEXT_TT_MULTI_LEVEL;
1634         }
1635         /*
1636          * In pass through mode, AW must be programmed to indicate the largest
1637          * AGAW value supported by hardware. And ASR is ignored by hardware.
1638          */
1639         if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1640                 context_set_address_width(context, iommu->msagaw);
1641         else {
1642                 context_set_address_root(context, virt_to_phys(pgd));
1643                 context_set_address_width(context, iommu->agaw);
1644         }
1645
1646         context_set_translation_type(context, translation);
1647         context_set_fault_enable(context);
1648         context_set_present(context);
1649         domain_flush_cache(domain, context, sizeof(*context));
1650
1651         /*
1652          * It's a non-present to present mapping. If hardware doesn't cache
1653          * non-present entry we only need to flush the write-buffer. If the
1654          * _does_ cache non-present entries, then it does so in the special
1655          * domain #0, which we have to flush:
1656          */
1657         if (cap_caching_mode(iommu->cap)) {
1658                 iommu->flush.flush_context(iommu, 0,
1659                                            (((u16)bus) << 8) | devfn,
1660                                            DMA_CCMD_MASK_NOBIT,
1661                                            DMA_CCMD_DEVICE_INVL);
1662                 iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
1663         } else {
1664                 iommu_flush_write_buffer(iommu);
1665         }
1666         iommu_enable_dev_iotlb(info);
1667         spin_unlock_irqrestore(&iommu->lock, flags);
1668
1669         spin_lock_irqsave(&domain->iommu_lock, flags);
1670         if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
1671                 domain->iommu_count++;
1672                 if (domain->iommu_count == 1)
1673                         domain->nid = iommu->node;
1674                 domain_update_iommu_cap(domain);
1675         }
1676         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1677         return 0;
1678 }
1679
1680 static int
1681 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1682                         int translation)
1683 {
1684         int ret;
1685         struct pci_dev *tmp, *parent;
1686
1687         ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
1688                                          pdev->bus->number, pdev->devfn,
1689                                          translation);
1690         if (ret)
1691                 return ret;
1692
1693         /* dependent device mapping */
1694         tmp = pci_find_upstream_pcie_bridge(pdev);
1695         if (!tmp)
1696                 return 0;
1697         /* Secondary interface's bus number and devfn 0 */
1698         parent = pdev->bus->self;
1699         while (parent != tmp) {
1700                 ret = domain_context_mapping_one(domain,
1701                                                  pci_domain_nr(parent->bus),
1702                                                  parent->bus->number,
1703                                                  parent->devfn, translation);
1704                 if (ret)
1705                         return ret;
1706                 parent = parent->bus->self;
1707         }
1708         if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
1709                 return domain_context_mapping_one(domain,
1710                                         pci_domain_nr(tmp->subordinate),
1711                                         tmp->subordinate->number, 0,
1712                                         translation);
1713         else /* this is a legacy PCI bridge */
1714                 return domain_context_mapping_one(domain,
1715                                                   pci_domain_nr(tmp->bus),
1716                                                   tmp->bus->number,
1717                                                   tmp->devfn,
1718                                                   translation);
1719 }
1720
1721 static int domain_context_mapped(struct pci_dev *pdev)
1722 {
1723         int ret;
1724         struct pci_dev *tmp, *parent;
1725         struct intel_iommu *iommu;
1726
1727         iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1728                                 pdev->devfn);
1729         if (!iommu)
1730                 return -ENODEV;
1731
1732         ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
1733         if (!ret)
1734                 return ret;
1735         /* dependent device mapping */
1736         tmp = pci_find_upstream_pcie_bridge(pdev);
1737         if (!tmp)
1738                 return ret;
1739         /* Secondary interface's bus number and devfn 0 */
1740         parent = pdev->bus->self;
1741         while (parent != tmp) {
1742                 ret = device_context_mapped(iommu, parent->bus->number,
1743                                             parent->devfn);
1744                 if (!ret)
1745                         return ret;
1746                 parent = parent->bus->self;
1747         }
1748         if (pci_is_pcie(tmp))
1749                 return device_context_mapped(iommu, tmp->subordinate->number,
1750                                              0);
1751         else
1752                 return device_context_mapped(iommu, tmp->bus->number,
1753                                              tmp->devfn);
1754 }
1755
1756 /* Returns a number of VTD pages, but aligned to MM page size */
1757 static inline unsigned long aligned_nrpages(unsigned long host_addr,
1758                                             size_t size)
1759 {
1760         host_addr &= ~PAGE_MASK;
1761         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1762 }
1763
1764 /* Return largest possible superpage level for a given mapping */
1765 static inline int hardware_largepage_caps(struct dmar_domain *domain,
1766                                           unsigned long iov_pfn,
1767                                           unsigned long phy_pfn,
1768                                           unsigned long pages)
1769 {
1770         int support, level = 1;
1771         unsigned long pfnmerge;
1772
1773         support = domain->iommu_superpage;
1774
1775         /* To use a large page, the virtual *and* physical addresses
1776            must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1777            of them will mean we have to use smaller pages. So just
1778            merge them and check both at once. */
1779         pfnmerge = iov_pfn | phy_pfn;
1780
1781         while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1782                 pages >>= VTD_STRIDE_SHIFT;
1783                 if (!pages)
1784                         break;
1785                 pfnmerge >>= VTD_STRIDE_SHIFT;
1786                 level++;
1787                 support--;
1788         }
1789         return level;
1790 }
1791
1792 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1793                             struct scatterlist *sg, unsigned long phys_pfn,
1794                             unsigned long nr_pages, int prot)
1795 {
1796         struct dma_pte *first_pte = NULL, *pte = NULL;
1797         phys_addr_t uninitialized_var(pteval);
1798         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1799         unsigned long sg_res = 0;
1800         unsigned int largepage_lvl = 0;
1801         unsigned long lvl_pages = 0;
1802
1803         BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1804
1805         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1806                 return -EINVAL;
1807
1808         prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1809
1810         if (!sg) {
1811                 sg_res = nr_pages;
1812                 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1813         }
1814
1815         while (nr_pages > 0) {
1816                 uint64_t tmp;
1817
1818                 if (!sg_res) {
1819                         sg_res = aligned_nrpages(sg->offset, sg->length);
1820                         sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1821                         sg->dma_length = sg->length;
1822                         pteval = page_to_phys(sg_page(sg)) | prot;
1823                         phys_pfn = pteval >> VTD_PAGE_SHIFT;
1824                 }
1825
1826                 if (!pte) {
1827                         largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
1828
1829                         first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, largepage_lvl);
1830                         if (!pte)
1831                                 return -ENOMEM;
1832                         /* It is large page*/
1833                         if (largepage_lvl > 1) {
1834                                 pteval |= DMA_PTE_LARGE_PAGE;
1835                                 /* Ensure that old small page tables are removed to make room
1836                                    for superpage, if they exist. */
1837                                 dma_pte_clear_range(domain, iov_pfn,
1838                                                     iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
1839                                 dma_pte_free_pagetable(domain, iov_pfn,
1840                                                        iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
1841                         } else {
1842                                 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
1843                         }
1844
1845                 }
1846                 /* We don't need lock here, nobody else
1847                  * touches the iova range
1848                  */
1849                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
1850                 if (tmp) {
1851                         static int dumps = 5;
1852                         printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1853                                iov_pfn, tmp, (unsigned long long)pteval);
1854                         if (dumps) {
1855                                 dumps--;
1856                                 debug_dma_dump_mappings(NULL);
1857                         }
1858                         WARN_ON(1);
1859                 }
1860
1861                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
1862
1863                 BUG_ON(nr_pages < lvl_pages);
1864                 BUG_ON(sg_res < lvl_pages);
1865
1866                 nr_pages -= lvl_pages;
1867                 iov_pfn += lvl_pages;
1868                 phys_pfn += lvl_pages;
1869                 pteval += lvl_pages * VTD_PAGE_SIZE;
1870                 sg_res -= lvl_pages;
1871
1872                 /* If the next PTE would be the first in a new page, then we
1873                    need to flush the cache on the entries we've just written.
1874                    And then we'll need to recalculate 'pte', so clear it and
1875                    let it get set again in the if (!pte) block above.
1876
1877                    If we're done (!nr_pages) we need to flush the cache too.
1878
1879                    Also if we've been setting superpages, we may need to
1880                    recalculate 'pte' and switch back to smaller pages for the
1881                    end of the mapping, if the trailing size is not enough to
1882                    use another superpage (i.e. sg_res < lvl_pages). */
1883                 pte++;
1884                 if (!nr_pages || first_pte_in_page(pte) ||
1885                     (largepage_lvl > 1 && sg_res < lvl_pages)) {
1886                         domain_flush_cache(domain, first_pte,
1887                                            (void *)pte - (void *)first_pte);
1888                         pte = NULL;
1889                 }
1890
1891                 if (!sg_res && nr_pages)
1892                         sg = sg_next(sg);
1893         }
1894         return 0;
1895 }
1896
1897 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1898                                     struct scatterlist *sg, unsigned long nr_pages,
1899                                     int prot)
1900 {
1901         return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
1902 }
1903
1904 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1905                                      unsigned long phys_pfn, unsigned long nr_pages,
1906                                      int prot)
1907 {
1908         return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
1909 }
1910
1911 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
1912 {
1913         if (!iommu)
1914                 return;
1915
1916         clear_context_table(iommu, bus, devfn);
1917         iommu->flush.flush_context(iommu, 0, 0, 0,
1918                                            DMA_CCMD_GLOBAL_INVL);
1919         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
1920 }
1921
1922 static inline void unlink_domain_info(struct device_domain_info *info)
1923 {
1924         assert_spin_locked(&device_domain_lock);
1925         list_del(&info->link);
1926         list_del(&info->global);
1927         if (info->dev)
1928                 info->dev->dev.archdata.iommu = NULL;
1929 }
1930
1931 static void domain_remove_dev_info(struct dmar_domain *domain)
1932 {
1933         struct device_domain_info *info;
1934         unsigned long flags;
1935         struct intel_iommu *iommu;
1936
1937         spin_lock_irqsave(&device_domain_lock, flags);
1938         while (!list_empty(&domain->devices)) {
1939                 info = list_entry(domain->devices.next,
1940                         struct device_domain_info, link);
1941                 unlink_domain_info(info);
1942                 spin_unlock_irqrestore(&device_domain_lock, flags);
1943
1944                 iommu_disable_dev_iotlb(info);
1945                 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
1946                 iommu_detach_dev(iommu, info->bus, info->devfn);
1947                 free_devinfo_mem(info);
1948
1949                 spin_lock_irqsave(&device_domain_lock, flags);
1950         }
1951         spin_unlock_irqrestore(&device_domain_lock, flags);
1952 }
1953
1954 /*
1955  * find_domain
1956  * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1957  */
1958 static struct dmar_domain *
1959 find_domain(struct pci_dev *pdev)
1960 {
1961         struct device_domain_info *info;
1962
1963         /* No lock here, assumes no domain exit in normal case */
1964         info = pdev->dev.archdata.iommu;
1965         if (info)
1966                 return info->domain;
1967         return NULL;
1968 }
1969
1970 /* domain is initialized */
1971 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1972 {
1973         struct dmar_domain *domain, *found = NULL;
1974         struct intel_iommu *iommu;
1975         struct dmar_drhd_unit *drhd;
1976         struct device_domain_info *info, *tmp;
1977         struct pci_dev *dev_tmp;
1978         unsigned long flags;
1979         int bus = 0, devfn = 0;
1980         int segment;
1981         int ret;
1982
1983         domain = find_domain(pdev);
1984         if (domain)
1985                 return domain;
1986
1987         segment = pci_domain_nr(pdev->bus);
1988
1989         dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1990         if (dev_tmp) {
1991                 if (pci_is_pcie(dev_tmp)) {
1992                         bus = dev_tmp->subordinate->number;
1993                         devfn = 0;
1994                 } else {
1995                         bus = dev_tmp->bus->number;
1996                         devfn = dev_tmp->devfn;
1997                 }
1998                 spin_lock_irqsave(&device_domain_lock, flags);
1999                 list_for_each_entry(info, &device_domain_list, global) {
2000                         if (info->segment == segment &&
2001                             info->bus == bus && info->devfn == devfn) {
2002                                 found = info->domain;
2003                                 break;
2004                         }
2005                 }
2006                 spin_unlock_irqrestore(&device_domain_lock, flags);
2007                 /* pcie-pci bridge already has a domain, uses it */
2008                 if (found) {
2009                         domain = found;
2010                         goto found_domain;
2011                 }
2012         }
2013
2014         domain = alloc_domain();
2015         if (!domain)
2016                 goto error;
2017
2018         /* Allocate new domain for the device */
2019         drhd = dmar_find_matched_drhd_unit(pdev);
2020         if (!drhd) {
2021                 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
2022                         pci_name(pdev));
2023                 free_domain_mem(domain);
2024                 return NULL;
2025         }
2026         iommu = drhd->iommu;
2027
2028         ret = iommu_attach_domain(domain, iommu);
2029         if (ret) {
2030                 free_domain_mem(domain);
2031                 goto error;
2032         }
2033
2034         if (domain_init(domain, gaw)) {
2035                 domain_exit(domain);
2036                 goto error;
2037         }
2038
2039         /* register pcie-to-pci device */
2040         if (dev_tmp) {
2041                 info = alloc_devinfo_mem();
2042                 if (!info) {
2043                         domain_exit(domain);
2044                         goto error;
2045                 }
2046                 info->segment = segment;
2047                 info->bus = bus;
2048                 info->devfn = devfn;
2049                 info->dev = NULL;
2050                 info->domain = domain;
2051                 /* This domain is shared by devices under p2p bridge */
2052                 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
2053
2054                 /* pcie-to-pci bridge already has a domain, uses it */
2055                 found = NULL;
2056                 spin_lock_irqsave(&device_domain_lock, flags);
2057                 list_for_each_entry(tmp, &device_domain_list, global) {
2058                         if (tmp->segment == segment &&
2059                             tmp->bus == bus && tmp->devfn == devfn) {
2060                                 found = tmp->domain;
2061                                 break;
2062                         }
2063                 }
2064                 if (found) {
2065                         spin_unlock_irqrestore(&device_domain_lock, flags);
2066                         free_devinfo_mem(info);
2067                         domain_exit(domain);
2068                         domain = found;
2069                 } else {
2070                         list_add(&info->link, &domain->devices);
2071                         list_add(&info->global, &device_domain_list);
2072                         spin_unlock_irqrestore(&device_domain_lock, flags);
2073                 }
2074         }
2075
2076 found_domain:
2077         info = alloc_devinfo_mem();
2078         if (!info)
2079                 goto error;
2080         info->segment = segment;
2081         info->bus = pdev->bus->number;
2082         info->devfn = pdev->devfn;
2083         info->dev = pdev;
2084         info->domain = domain;
2085         spin_lock_irqsave(&device_domain_lock, flags);
2086         /* somebody is fast */
2087         found = find_domain(pdev);
2088         if (found != NULL) {
2089                 spin_unlock_irqrestore(&device_domain_lock, flags);
2090                 if (found != domain) {
2091                         domain_exit(domain);
2092                         domain = found;
2093                 }
2094                 free_devinfo_mem(info);
2095                 return domain;
2096         }
2097         list_add(&info->link, &domain->devices);
2098         list_add(&info->global, &device_domain_list);
2099         pdev->dev.archdata.iommu = info;
2100         spin_unlock_irqrestore(&device_domain_lock, flags);
2101         return domain;
2102 error:
2103         /* recheck it here, maybe others set it */
2104         return find_domain(pdev);
2105 }
2106
2107 static int iommu_identity_mapping;
2108 #define IDENTMAP_ALL            1
2109 #define IDENTMAP_GFX            2
2110 #define IDENTMAP_AZALIA         4
2111
2112 static int iommu_domain_identity_map(struct dmar_domain *domain,
2113                                      unsigned long long start,
2114                                      unsigned long long end)
2115 {
2116         unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2117         unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2118
2119         if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2120                           dma_to_mm_pfn(last_vpfn))) {
2121                 printk(KERN_ERR "IOMMU: reserve iova failed\n");
2122                 return -ENOMEM;
2123         }
2124
2125         pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2126                  start, end, domain->id);
2127         /*
2128          * RMRR range might have overlap with physical memory range,
2129          * clear it first
2130          */
2131         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2132
2133         return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2134                                   last_vpfn - first_vpfn + 1,
2135                                   DMA_PTE_READ|DMA_PTE_WRITE);
2136 }
2137
2138 static int iommu_prepare_identity_map(struct pci_dev *pdev,
2139                                       unsigned long long start,
2140                                       unsigned long long end)
2141 {
2142         struct dmar_domain *domain;
2143         int ret;
2144
2145         domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2146         if (!domain)
2147                 return -ENOMEM;
2148
2149         /* For _hardware_ passthrough, don't bother. But for software
2150            passthrough, we do it anyway -- it may indicate a memory
2151            range which is reserved in E820, so which didn't get set
2152            up to start with in si_domain */
2153         if (domain == si_domain && hw_pass_through) {
2154                 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2155                        pci_name(pdev), start, end);
2156                 return 0;
2157         }
2158
2159         printk(KERN_INFO
2160                "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2161                pci_name(pdev), start, end);
2162
2163         if (end < start) {
2164                 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2165                         "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2166                         dmi_get_system_info(DMI_BIOS_VENDOR),
2167                         dmi_get_system_info(DMI_BIOS_VERSION),
2168                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2169                 ret = -EIO;
2170                 goto error;
2171         }
2172
2173         if (end >> agaw_to_width(domain->agaw)) {
2174                 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2175                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2176                      agaw_to_width(domain->agaw),
2177                      dmi_get_system_info(DMI_BIOS_VENDOR),
2178                      dmi_get_system_info(DMI_BIOS_VERSION),
2179                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2180                 ret = -EIO;
2181                 goto error;
2182         }
2183
2184         ret = iommu_domain_identity_map(domain, start, end);
2185         if (ret)
2186                 goto error;
2187
2188         /* context entry init */
2189         ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
2190         if (ret)
2191                 goto error;
2192
2193         return 0;
2194
2195  error:
2196         domain_exit(domain);
2197         return ret;
2198 }
2199
2200 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2201         struct pci_dev *pdev)
2202 {
2203         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2204                 return 0;
2205         return iommu_prepare_identity_map(pdev, rmrr->base_address,
2206                 rmrr->end_address);
2207 }
2208
2209 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2210 static inline void iommu_prepare_isa(void)
2211 {
2212         struct pci_dev *pdev;
2213         int ret;
2214
2215         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2216         if (!pdev)
2217                 return;
2218
2219         printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
2220         ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024 - 1);
2221
2222         if (ret)
2223                 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2224                        "floppy might not work\n");
2225
2226 }
2227 #else
2228 static inline void iommu_prepare_isa(void)
2229 {
2230         return;
2231 }
2232 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2233
2234 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2235
2236 static int __init si_domain_init(int hw)
2237 {
2238         struct dmar_drhd_unit *drhd;
2239         struct intel_iommu *iommu;
2240         int nid, ret = 0;
2241
2242         si_domain = alloc_domain();
2243         if (!si_domain)
2244                 return -EFAULT;
2245
2246         pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
2247
2248         for_each_active_iommu(iommu, drhd) {
2249                 ret = iommu_attach_domain(si_domain, iommu);
2250                 if (ret) {
2251                         domain_exit(si_domain);
2252                         return -EFAULT;
2253                 }
2254         }
2255
2256         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2257                 domain_exit(si_domain);
2258                 return -EFAULT;
2259         }
2260
2261         si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2262
2263         if (hw)
2264                 return 0;
2265
2266         for_each_online_node(nid) {
2267                 unsigned long start_pfn, end_pfn;
2268                 int i;
2269
2270                 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2271                         ret = iommu_domain_identity_map(si_domain,
2272                                         PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2273                         if (ret)
2274                                 return ret;
2275                 }
2276         }
2277
2278         return 0;
2279 }
2280
2281 static void domain_remove_one_dev_info(struct dmar_domain *domain,
2282                                           struct pci_dev *pdev);
2283 static int identity_mapping(struct pci_dev *pdev)
2284 {
2285         struct device_domain_info *info;
2286
2287         if (likely(!iommu_identity_mapping))
2288                 return 0;
2289
2290         info = pdev->dev.archdata.iommu;
2291         if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2292                 return (info->domain == si_domain);
2293
2294         return 0;
2295 }
2296
2297 static int domain_add_dev_info(struct dmar_domain *domain,
2298                                struct pci_dev *pdev,
2299                                int translation)
2300 {
2301         struct device_domain_info *info;
2302         unsigned long flags;
2303         int ret;
2304
2305         info = alloc_devinfo_mem();
2306         if (!info)
2307                 return -ENOMEM;
2308
2309         info->segment = pci_domain_nr(pdev->bus);
2310         info->bus = pdev->bus->number;
2311         info->devfn = pdev->devfn;
2312         info->dev = pdev;
2313         info->domain = domain;
2314
2315         spin_lock_irqsave(&device_domain_lock, flags);
2316         list_add(&info->link, &domain->devices);
2317         list_add(&info->global, &device_domain_list);
2318         pdev->dev.archdata.iommu = info;
2319         spin_unlock_irqrestore(&device_domain_lock, flags);
2320
2321         ret = domain_context_mapping(domain, pdev, translation);
2322         if (ret) {
2323                 spin_lock_irqsave(&device_domain_lock, flags);
2324                 unlink_domain_info(info);
2325                 spin_unlock_irqrestore(&device_domain_lock, flags);
2326                 free_devinfo_mem(info);
2327                 return ret;
2328         }
2329
2330         return 0;
2331 }
2332
2333 static bool device_has_rmrr(struct pci_dev *dev)
2334 {
2335         struct dmar_rmrr_unit *rmrr;
2336         int i;
2337
2338         for_each_rmrr_units(rmrr) {
2339                 for (i = 0; i < rmrr->devices_cnt; i++) {
2340                         /*
2341                          * Return TRUE if this RMRR contains the device that
2342                          * is passed in.
2343                          */
2344                         if (rmrr->devices[i] == dev)
2345                                 return true;
2346                 }
2347         }
2348         return false;
2349 }
2350
2351 static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2352 {
2353
2354         /*
2355          * We want to prevent any device associated with an RMRR from
2356          * getting placed into the SI Domain. This is done because
2357          * problems exist when devices are moved in and out of domains
2358          * and their respective RMRR info is lost. We exempt USB devices
2359          * from this process due to their usage of RMRRs that are known
2360          * to not be needed after BIOS hand-off to OS.
2361          */
2362         if (device_has_rmrr(pdev) &&
2363             (pdev->class >> 8) != PCI_CLASS_SERIAL_USB)
2364                 return 0;
2365
2366         if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2367                 return 1;
2368
2369         if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2370                 return 1;
2371
2372         if (!(iommu_identity_mapping & IDENTMAP_ALL))
2373                 return 0;
2374
2375         /*
2376          * We want to start off with all devices in the 1:1 domain, and
2377          * take them out later if we find they can't access all of memory.
2378          *
2379          * However, we can't do this for PCI devices behind bridges,
2380          * because all PCI devices behind the same bridge will end up
2381          * with the same source-id on their transactions.
2382          *
2383          * Practically speaking, we can't change things around for these
2384          * devices at run-time, because we can't be sure there'll be no
2385          * DMA transactions in flight for any of their siblings.
2386          *
2387          * So PCI devices (unless they're on the root bus) as well as
2388          * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2389          * the 1:1 domain, just in _case_ one of their siblings turns out
2390          * not to be able to map all of memory.
2391          */
2392         if (!pci_is_pcie(pdev)) {
2393                 if (!pci_is_root_bus(pdev->bus))
2394                         return 0;
2395                 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2396                         return 0;
2397         } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2398                 return 0;
2399
2400         /*
2401          * At boot time, we don't yet know if devices will be 64-bit capable.
2402          * Assume that they will -- if they turn out not to be, then we can
2403          * take them out of the 1:1 domain later.
2404          */
2405         if (!startup) {
2406                 /*
2407                  * If the device's dma_mask is less than the system's memory
2408                  * size then this is not a candidate for identity mapping.
2409                  */
2410                 u64 dma_mask = pdev->dma_mask;
2411
2412                 if (pdev->dev.coherent_dma_mask &&
2413                     pdev->dev.coherent_dma_mask < dma_mask)
2414                         dma_mask = pdev->dev.coherent_dma_mask;
2415
2416                 return dma_mask >= dma_get_required_mask(&pdev->dev);
2417         }
2418
2419         return 1;
2420 }
2421
2422 static int __init iommu_prepare_static_identity_mapping(int hw)
2423 {
2424         struct pci_dev *pdev = NULL;
2425         int ret;
2426
2427         ret = si_domain_init(hw);
2428         if (ret)
2429                 return -EFAULT;
2430
2431         for_each_pci_dev(pdev) {
2432                 if (iommu_should_identity_map(pdev, 1)) {
2433                         ret = domain_add_dev_info(si_domain, pdev,
2434                                              hw ? CONTEXT_TT_PASS_THROUGH :
2435                                                   CONTEXT_TT_MULTI_LEVEL);
2436                         if (ret) {
2437                                 /* device not associated with an iommu */
2438                                 if (ret == -ENODEV)
2439                                         continue;
2440                                 return ret;
2441                         }
2442                         pr_info("IOMMU: %s identity mapping for device %s\n",
2443                                 hw ? "hardware" : "software", pci_name(pdev));
2444                 }
2445         }
2446
2447         return 0;
2448 }
2449
2450 static int __init init_dmars(void)
2451 {
2452         struct dmar_drhd_unit *drhd;
2453         struct dmar_rmrr_unit *rmrr;
2454         struct pci_dev *pdev;
2455         struct intel_iommu *iommu;
2456         int i, ret;
2457
2458         /*
2459          * for each drhd
2460          *    allocate root
2461          *    initialize and program root entry to not present
2462          * endfor
2463          */
2464         for_each_drhd_unit(drhd) {
2465                 /*
2466                  * lock not needed as this is only incremented in the single
2467                  * threaded kernel __init code path all other access are read
2468                  * only
2469                  */
2470                 if (g_num_of_iommus < IOMMU_UNITS_SUPPORTED) {
2471                         g_num_of_iommus++;
2472                         continue;
2473                 }
2474                 printk_once(KERN_ERR "intel-iommu: exceeded %d IOMMUs\n",
2475                           IOMMU_UNITS_SUPPORTED);
2476         }
2477
2478         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2479                         GFP_KERNEL);
2480         if (!g_iommus) {
2481                 printk(KERN_ERR "Allocating global iommu array failed\n");
2482                 ret = -ENOMEM;
2483                 goto error;
2484         }
2485
2486         deferred_flush = kzalloc(g_num_of_iommus *
2487                 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2488         if (!deferred_flush) {
2489                 ret = -ENOMEM;
2490                 goto error;
2491         }
2492
2493         for_each_drhd_unit(drhd) {
2494                 if (drhd->ignored)
2495                         continue;
2496
2497                 iommu = drhd->iommu;
2498                 g_iommus[iommu->seq_id] = iommu;
2499
2500                 ret = iommu_init_domains(iommu);
2501                 if (ret)
2502                         goto error;
2503
2504                 /*
2505                  * TBD:
2506                  * we could share the same root & context tables
2507                  * among all IOMMU's. Need to Split it later.
2508                  */
2509                 ret = iommu_alloc_root_entry(iommu);
2510                 if (ret) {
2511                         printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2512                         goto error;
2513                 }
2514                 if (!ecap_pass_through(iommu->ecap))
2515                         hw_pass_through = 0;
2516         }
2517
2518         /*
2519          * Start from the sane iommu hardware state.
2520          */
2521         for_each_drhd_unit(drhd) {
2522                 if (drhd->ignored)
2523                         continue;
2524
2525                 iommu = drhd->iommu;
2526
2527                 /*
2528                  * If the queued invalidation is already initialized by us
2529                  * (for example, while enabling interrupt-remapping) then
2530                  * we got the things already rolling from a sane state.
2531                  */
2532                 if (iommu->qi)
2533                         continue;
2534
2535                 /*
2536                  * Clear any previous faults.
2537                  */
2538                 dmar_fault(-1, iommu);
2539                 /*
2540                  * Disable queued invalidation if supported and already enabled
2541                  * before OS handover.
2542                  */
2543                 dmar_disable_qi(iommu);
2544         }
2545
2546         for_each_drhd_unit(drhd) {
2547                 if (drhd->ignored)
2548                         continue;
2549
2550                 iommu = drhd->iommu;
2551
2552                 if (dmar_enable_qi(iommu)) {
2553                         /*
2554                          * Queued Invalidate not enabled, use Register Based
2555                          * Invalidate
2556                          */
2557                         iommu->flush.flush_context = __iommu_flush_context;
2558                         iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2559                         printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
2560                                "invalidation\n",
2561                                 iommu->seq_id,
2562                                (unsigned long long)drhd->reg_base_addr);
2563                 } else {
2564                         iommu->flush.flush_context = qi_flush_context;
2565                         iommu->flush.flush_iotlb = qi_flush_iotlb;
2566                         printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
2567                                "invalidation\n",
2568                                 iommu->seq_id,
2569                                (unsigned long long)drhd->reg_base_addr);
2570                 }
2571         }
2572
2573         if (iommu_pass_through)
2574                 iommu_identity_mapping |= IDENTMAP_ALL;
2575
2576 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
2577         iommu_identity_mapping |= IDENTMAP_GFX;
2578 #endif
2579
2580         check_tylersburg_isoch();
2581
2582         /*
2583          * If pass through is not set or not enabled, setup context entries for
2584          * identity mappings for rmrr, gfx, and isa and may fall back to static
2585          * identity mapping if iommu_identity_mapping is set.
2586          */
2587         if (iommu_identity_mapping) {
2588                 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
2589                 if (ret) {
2590                         printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2591                         goto error;
2592                 }
2593         }
2594         /*
2595          * For each rmrr
2596          *   for each dev attached to rmrr
2597          *   do
2598          *     locate drhd for dev, alloc domain for dev
2599          *     allocate free domain
2600          *     allocate page table entries for rmrr
2601          *     if context not allocated for bus
2602          *           allocate and init context
2603          *           set present in root table for this bus
2604          *     init context with domain, translation etc
2605          *    endfor
2606          * endfor
2607          */
2608         printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2609         for_each_rmrr_units(rmrr) {
2610                 for (i = 0; i < rmrr->devices_cnt; i++) {
2611                         pdev = rmrr->devices[i];
2612                         /*
2613                          * some BIOS lists non-exist devices in DMAR
2614                          * table.
2615                          */
2616                         if (!pdev)
2617                                 continue;
2618                         ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2619                         if (ret)
2620                                 printk(KERN_ERR
2621                                        "IOMMU: mapping reserved region failed\n");
2622                 }
2623         }
2624
2625         iommu_prepare_isa();
2626
2627         /*
2628          * for each drhd
2629          *   enable fault log
2630          *   global invalidate context cache
2631          *   global invalidate iotlb
2632          *   enable translation
2633          */
2634         for_each_drhd_unit(drhd) {
2635                 if (drhd->ignored) {
2636                         /*
2637                          * we always have to disable PMRs or DMA may fail on
2638                          * this device
2639                          */
2640                         if (force_on)
2641                                 iommu_disable_protect_mem_regions(drhd->iommu);
2642                         continue;
2643                 }
2644                 iommu = drhd->iommu;
2645
2646                 iommu_flush_write_buffer(iommu);
2647
2648                 ret = dmar_set_interrupt(iommu);
2649                 if (ret)
2650                         goto error;
2651
2652                 iommu_set_root_entry(iommu);
2653
2654                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
2655                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2656
2657                 ret = iommu_enable_translation(iommu);
2658                 if (ret)
2659                         goto error;
2660
2661                 iommu_disable_protect_mem_regions(iommu);
2662         }
2663
2664         return 0;
2665 error:
2666         for_each_drhd_unit(drhd) {
2667                 if (drhd->ignored)
2668                         continue;
2669                 iommu = drhd->iommu;
2670                 free_iommu(iommu);
2671         }
2672         kfree(g_iommus);
2673         return ret;
2674 }
2675
2676 /* This takes a number of _MM_ pages, not VTD pages */
2677 static struct iova *intel_alloc_iova(struct device *dev,
2678                                      struct dmar_domain *domain,
2679                                      unsigned long nrpages, uint64_t dma_mask)
2680 {
2681         struct pci_dev *pdev = to_pci_dev(dev);
2682         struct iova *iova = NULL;
2683
2684         /* Restrict dma_mask to the width that the iommu can handle */
2685         dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2686
2687         if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
2688                 /*
2689                  * First try to allocate an io virtual address in
2690                  * DMA_BIT_MASK(32) and if that fails then try allocating
2691                  * from higher range
2692                  */
2693                 iova = alloc_iova(&domain->iovad, nrpages,
2694                                   IOVA_PFN(DMA_BIT_MASK(32)), 1);
2695                 if (iova)
2696                         return iova;
2697         }
2698         iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2699         if (unlikely(!iova)) {
2700                 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2701                        nrpages, pci_name(pdev));
2702                 return NULL;
2703         }
2704
2705         return iova;
2706 }
2707
2708 static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
2709 {
2710         struct dmar_domain *domain;
2711         int ret;
2712
2713         domain = get_domain_for_dev(pdev,
2714                         DEFAULT_DOMAIN_ADDRESS_WIDTH);
2715         if (!domain) {
2716                 printk(KERN_ERR
2717                         "Allocating domain for %s failed", pci_name(pdev));
2718                 return NULL;
2719         }
2720
2721         /* make sure context mapping is ok */
2722         if (unlikely(!domain_context_mapped(pdev))) {
2723                 ret = domain_context_mapping(domain, pdev,
2724                                              CONTEXT_TT_MULTI_LEVEL);
2725                 if (ret) {
2726                         printk(KERN_ERR
2727                                 "Domain context map for %s failed",
2728                                 pci_name(pdev));
2729                         return NULL;
2730                 }
2731         }
2732
2733         return domain;
2734 }
2735
2736 static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2737 {
2738         struct device_domain_info *info;
2739
2740         /* No lock here, assumes no domain exit in normal case */
2741         info = dev->dev.archdata.iommu;
2742         if (likely(info))
2743                 return info->domain;
2744
2745         return __get_valid_domain_for_dev(dev);
2746 }
2747
2748 static int iommu_dummy(struct pci_dev *pdev)
2749 {
2750         return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2751 }
2752
2753 /* Check if the pdev needs to go through non-identity map and unmap process.*/
2754 static int iommu_no_mapping(struct device *dev)
2755 {
2756         struct pci_dev *pdev;
2757         int found;
2758
2759         if (unlikely(dev->bus != &pci_bus_type))
2760                 return 1;
2761
2762         pdev = to_pci_dev(dev);
2763         if (iommu_dummy(pdev))
2764                 return 1;
2765
2766         if (!iommu_identity_mapping)
2767                 return 0;
2768
2769         found = identity_mapping(pdev);
2770         if (found) {
2771                 if (iommu_should_identity_map(pdev, 0))
2772                         return 1;
2773                 else {
2774                         /*
2775                          * 32 bit DMA is removed from si_domain and fall back
2776                          * to non-identity mapping.
2777                          */
2778                         domain_remove_one_dev_info(si_domain, pdev);
2779                         printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2780                                pci_name(pdev));
2781                         return 0;
2782                 }
2783         } else {
2784                 /*
2785                  * In case of a detached 64 bit DMA device from vm, the device
2786                  * is put into si_domain for identity mapping.
2787                  */
2788                 if (iommu_should_identity_map(pdev, 0)) {
2789                         int ret;
2790                         ret = domain_add_dev_info(si_domain, pdev,
2791                                                   hw_pass_through ?
2792                                                   CONTEXT_TT_PASS_THROUGH :
2793                                                   CONTEXT_TT_MULTI_LEVEL);
2794                         if (!ret) {
2795                                 printk(KERN_INFO "64bit %s uses identity mapping\n",
2796                                        pci_name(pdev));
2797                                 return 1;
2798                         }
2799                 }
2800         }
2801
2802         return 0;
2803 }
2804
2805 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2806                                      size_t size, int dir, u64 dma_mask)
2807 {
2808         struct pci_dev *pdev = to_pci_dev(hwdev);
2809         struct dmar_domain *domain;
2810         phys_addr_t start_paddr;
2811         struct iova *iova;
2812         int prot = 0;
2813         int ret;
2814         struct intel_iommu *iommu;
2815         unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
2816
2817         BUG_ON(dir == DMA_NONE);
2818
2819         if (iommu_no_mapping(hwdev))
2820                 return paddr;
2821
2822         domain = get_valid_domain_for_dev(pdev);
2823         if (!domain)
2824                 return 0;
2825
2826         iommu = domain_get_iommu(domain);
2827         size = aligned_nrpages(paddr, size);
2828
2829         iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size), dma_mask);
2830         if (!iova)
2831                 goto error;
2832
2833         /*
2834          * Check if DMAR supports zero-length reads on write only
2835          * mappings..
2836          */
2837         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2838                         !cap_zlr(iommu->cap))
2839                 prot |= DMA_PTE_READ;
2840         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2841                 prot |= DMA_PTE_WRITE;
2842         /*
2843          * paddr - (paddr + size) might be partial page, we should map the whole
2844          * page.  Note: if two part of one page are separately mapped, we
2845          * might have two guest_addr mapping to the same host paddr, but this
2846          * is not a big problem
2847          */
2848         ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
2849                                  mm_to_dma_pfn(paddr_pfn), size, prot);
2850         if (ret)
2851                 goto error;
2852
2853         /* it's a non-present to present mapping. Only flush if caching mode */
2854         if (cap_caching_mode(iommu->cap))
2855                 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);
2856         else
2857                 iommu_flush_write_buffer(iommu);
2858
2859         start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2860         start_paddr += paddr & ~PAGE_MASK;
2861         return start_paddr;
2862
2863 error:
2864         if (iova)
2865                 __free_iova(&domain->iovad, iova);
2866         printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
2867                 pci_name(pdev), size, (unsigned long long)paddr, dir);
2868         return 0;
2869 }
2870
2871 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2872                                  unsigned long offset, size_t size,
2873                                  enum dma_data_direction dir,
2874                                  struct dma_attrs *attrs)
2875 {
2876         return __intel_map_single(dev, page_to_phys(page) + offset, size,
2877                                   dir, to_pci_dev(dev)->dma_mask);
2878 }
2879
2880 static void flush_unmaps(void)
2881 {
2882         int i, j;
2883
2884         timer_on = 0;
2885
2886         /* just flush them all */
2887         for (i = 0; i < g_num_of_iommus; i++) {
2888                 struct intel_iommu *iommu = g_iommus[i];
2889                 if (!iommu)
2890                         continue;
2891
2892                 if (!deferred_flush[i].next)
2893                         continue;
2894
2895                 /* In caching mode, global flushes turn emulation expensive */
2896                 if (!cap_caching_mode(iommu->cap))
2897                         iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2898                                          DMA_TLB_GLOBAL_FLUSH);
2899                 for (j = 0; j < deferred_flush[i].next; j++) {
2900                         unsigned long mask;
2901                         struct iova *iova = deferred_flush[i].iova[j];
2902                         struct dmar_domain *domain = deferred_flush[i].domain[j];
2903
2904                         /* On real hardware multiple invalidations are expensive */
2905                         if (cap_caching_mode(iommu->cap))
2906                                 iommu_flush_iotlb_psi(iommu, domain->id,
2907                                 iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);
2908                         else {
2909                                 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
2910                                 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2911                                                 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
2912                         }
2913                         __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
2914                 }
2915                 deferred_flush[i].next = 0;
2916         }
2917
2918         list_size = 0;
2919 }
2920
2921 static void flush_unmaps_timeout(unsigned long data)
2922 {
2923         unsigned long flags;
2924
2925         spin_lock_irqsave(&async_umap_flush_lock, flags);
2926         flush_unmaps();
2927         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2928 }
2929
2930 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2931 {
2932         unsigned long flags;
2933         int next, iommu_id;
2934         struct intel_iommu *iommu;
2935
2936         spin_lock_irqsave(&async_umap_flush_lock, flags);
2937         if (list_size == HIGH_WATER_MARK)
2938                 flush_unmaps();
2939
2940         iommu = domain_get_iommu(dom);
2941         iommu_id = iommu->seq_id;
2942
2943         next = deferred_flush[iommu_id].next;
2944         deferred_flush[iommu_id].domain[next] = dom;
2945         deferred_flush[iommu_id].iova[next] = iova;
2946         deferred_flush[iommu_id].next++;
2947
2948         if (!timer_on) {
2949                 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2950                 timer_on = 1;
2951         }
2952         list_size++;
2953         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2954 }
2955
2956 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2957                              size_t size, enum dma_data_direction dir,
2958                              struct dma_attrs *attrs)
2959 {
2960         struct pci_dev *pdev = to_pci_dev(dev);
2961         struct dmar_domain *domain;
2962         unsigned long start_pfn, last_pfn;
2963         struct iova *iova;
2964         struct intel_iommu *iommu;
2965
2966         if (iommu_no_mapping(dev))
2967                 return;
2968
2969         domain = find_domain(pdev);
2970         BUG_ON(!domain);
2971
2972         iommu = domain_get_iommu(domain);
2973
2974         iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
2975         if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
2976                       (unsigned long long)dev_addr))
2977                 return;
2978
2979         start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2980         last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2981
2982         pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2983                  pci_name(pdev), start_pfn, last_pfn);
2984
2985         /*  clear the whole page */
2986         dma_pte_clear_range(domain, start_pfn, last_pfn);
2987
2988         /* free page tables */
2989         dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2990
2991         if (intel_iommu_strict) {
2992                 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2993                                       last_pfn - start_pfn + 1, 0);
2994                 /* free iova */
2995                 __free_iova(&domain->iovad, iova);
2996         } else {
2997                 add_unmap(domain, iova);
2998                 /*
2999                  * queue up the release of the unmap to save the 1/6th of the
3000                  * cpu used up by the iotlb flush operation...
3001                  */
3002         }
3003 }
3004
3005 static void *intel_alloc_coherent(struct device *hwdev, size_t size,
3006                                   dma_addr_t *dma_handle, gfp_t flags,
3007                                   struct dma_attrs *attrs)
3008 {
3009         void *vaddr;
3010         int order;
3011
3012         size = PAGE_ALIGN(size);
3013         order = get_order(size);
3014
3015         if (!iommu_no_mapping(hwdev))
3016                 flags &= ~(GFP_DMA | GFP_DMA32);
3017         else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
3018                 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
3019                         flags |= GFP_DMA;
3020                 else
3021                         flags |= GFP_DMA32;
3022         }
3023
3024         vaddr = (void *)__get_free_pages(flags, order);
3025         if (!vaddr)
3026                 return NULL;
3027         memset(vaddr, 0, size);
3028
3029         *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
3030                                          DMA_BIDIRECTIONAL,
3031                                          hwdev->coherent_dma_mask);
3032         if (*dma_handle)
3033                 return vaddr;
3034         free_pages((unsigned long)vaddr, order);
3035         return NULL;
3036 }
3037
3038 static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
3039                                 dma_addr_t dma_handle, struct dma_attrs *attrs)
3040 {
3041         int order;
3042
3043         size = PAGE_ALIGN(size);
3044         order = get_order(size);
3045
3046         intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
3047         free_pages((unsigned long)vaddr, order);
3048 }
3049
3050 static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
3051                            int nelems, enum dma_data_direction dir,
3052                            struct dma_attrs *attrs)
3053 {
3054         struct pci_dev *pdev = to_pci_dev(hwdev);
3055         struct dmar_domain *domain;
3056         unsigned long start_pfn, last_pfn;
3057         struct iova *iova;
3058         struct intel_iommu *iommu;
3059
3060         if (iommu_no_mapping(hwdev))
3061                 return;
3062
3063         domain = find_domain(pdev);
3064         BUG_ON(!domain);
3065
3066         iommu = domain_get_iommu(domain);
3067
3068         iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
3069         if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
3070                       (unsigned long long)sglist[0].dma_address))
3071                 return;
3072
3073         start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3074         last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
3075
3076         /*  clear the whole page */
3077         dma_pte_clear_range(domain, start_pfn, last_pfn);
3078
3079         /* free page tables */
3080         dma_pte_free_pagetable(domain, start_pfn, last_pfn);
3081
3082         if (intel_iommu_strict) {
3083                 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
3084                                       last_pfn - start_pfn + 1, 0);
3085                 /* free iova */
3086                 __free_iova(&domain->iovad, iova);
3087         } else {
3088                 add_unmap(domain, iova);
3089                 /*
3090                  * queue up the release of the unmap to save the 1/6th of the
3091                  * cpu used up by the iotlb flush operation...
3092                  */
3093         }
3094 }
3095
3096 static int intel_nontranslate_map_sg(struct device *hddev,
3097         struct scatterlist *sglist, int nelems, int dir)
3098 {
3099         int i;
3100         struct scatterlist *sg;
3101
3102         for_each_sg(sglist, sg, nelems, i) {
3103                 BUG_ON(!sg_page(sg));
3104                 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
3105                 sg->dma_length = sg->length;
3106         }
3107         return nelems;
3108 }
3109
3110 static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
3111                         enum dma_data_direction dir, struct dma_attrs *attrs)
3112 {
3113         int i;
3114         struct pci_dev *pdev = to_pci_dev(hwdev);
3115         struct dmar_domain *domain;
3116         size_t size = 0;
3117         int prot = 0;
3118         struct iova *iova = NULL;
3119         int ret;
3120         struct scatterlist *sg;
3121         unsigned long start_vpfn;
3122         struct intel_iommu *iommu;
3123
3124         BUG_ON(dir == DMA_NONE);
3125         if (iommu_no_mapping(hwdev))
3126                 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
3127
3128         domain = get_valid_domain_for_dev(pdev);
3129         if (!domain)
3130                 return 0;
3131
3132         iommu = domain_get_iommu(domain);
3133
3134         for_each_sg(sglist, sg, nelems, i)
3135                 size += aligned_nrpages(sg->offset, sg->length);
3136
3137         iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
3138                                 pdev->dma_mask);
3139         if (!iova) {
3140                 sglist->dma_length = 0;
3141                 return 0;
3142         }
3143
3144         /*
3145          * Check if DMAR supports zero-length reads on write only
3146          * mappings..
3147          */
3148         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3149                         !cap_zlr(iommu->cap))
3150                 prot |= DMA_PTE_READ;
3151         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3152                 prot |= DMA_PTE_WRITE;
3153
3154         start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
3155
3156         ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3157         if (unlikely(ret)) {
3158                 /*  clear the page */
3159                 dma_pte_clear_range(domain, start_vpfn,
3160                                     start_vpfn + size - 1);
3161                 /* free page tables */
3162                 dma_pte_free_pagetable(domain, start_vpfn,
3163                                        start_vpfn + size - 1);
3164                 /* free iova */
3165                 __free_iova(&domain->iovad, iova);
3166                 return 0;
3167         }
3168
3169         /* it's a non-present to present mapping. Only flush if caching mode */
3170         if (cap_caching_mode(iommu->cap))
3171                 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);
3172         else
3173                 iommu_flush_write_buffer(iommu);
3174
3175         return nelems;
3176 }
3177
3178 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3179 {
3180         return !dma_addr;
3181 }
3182
3183 struct dma_map_ops intel_dma_ops = {
3184         .alloc = intel_alloc_coherent,
3185         .free = intel_free_coherent,
3186         .map_sg = intel_map_sg,
3187         .unmap_sg = intel_unmap_sg,
3188         .map_page = intel_map_page,
3189         .unmap_page = intel_unmap_page,
3190         .mapping_error = intel_mapping_error,
3191 };
3192
3193 static inline int iommu_domain_cache_init(void)
3194 {
3195         int ret = 0;
3196
3197         iommu_domain_cache = kmem_cache_create("iommu_domain",
3198                                          sizeof(struct dmar_domain),
3199                                          0,
3200                                          SLAB_HWCACHE_ALIGN,
3201
3202                                          NULL);
3203         if (!iommu_domain_cache) {
3204                 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3205                 ret = -ENOMEM;
3206         }
3207
3208         return ret;
3209 }
3210
3211 static inline int iommu_devinfo_cache_init(void)
3212 {
3213         int ret = 0;
3214
3215         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3216                                          sizeof(struct device_domain_info),
3217                                          0,
3218                                          SLAB_HWCACHE_ALIGN,
3219                                          NULL);
3220         if (!iommu_devinfo_cache) {
3221                 printk(KERN_ERR "Couldn't create devinfo cache\n");
3222                 ret = -ENOMEM;
3223         }
3224
3225         return ret;
3226 }
3227
3228 static inline int iommu_iova_cache_init(void)
3229 {
3230         int ret = 0;
3231
3232         iommu_iova_cache = kmem_cache_create("iommu_iova",
3233                                          sizeof(struct iova),
3234                                          0,
3235                                          SLAB_HWCACHE_ALIGN,
3236                                          NULL);
3237         if (!iommu_iova_cache) {
3238                 printk(KERN_ERR "Couldn't create iova cache\n");
3239                 ret = -ENOMEM;
3240         }
3241
3242         return ret;
3243 }
3244
3245 static int __init iommu_init_mempool(void)
3246 {
3247         int ret;
3248         ret = iommu_iova_cache_init();
3249         if (ret)
3250                 return ret;
3251
3252         ret = iommu_domain_cache_init();
3253         if (ret)
3254                 goto domain_error;
3255
3256         ret = iommu_devinfo_cache_init();
3257         if (!ret)
3258                 return ret;
3259
3260         kmem_cache_destroy(iommu_domain_cache);
3261 domain_error:
3262         kmem_cache_destroy(iommu_iova_cache);
3263
3264         return -ENOMEM;
3265 }
3266
3267 static void __init iommu_exit_mempool(void)
3268 {
3269         kmem_cache_destroy(iommu_devinfo_cache);
3270         kmem_cache_destroy(iommu_domain_cache);
3271         kmem_cache_destroy(iommu_iova_cache);
3272
3273 }
3274
3275 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3276 {
3277         struct dmar_drhd_unit *drhd;
3278         u32 vtbar;
3279         int rc;
3280
3281         /* We know that this device on this chipset has its own IOMMU.
3282          * If we find it under a different IOMMU, then the BIOS is lying
3283          * to us. Hope that the IOMMU for this device is actually
3284          * disabled, and it needs no translation...
3285          */
3286         rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3287         if (rc) {
3288                 /* "can't" happen */
3289                 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3290                 return;
3291         }
3292         vtbar &= 0xffff0000;
3293
3294         /* we know that the this iommu should be at offset 0xa000 from vtbar */
3295         drhd = dmar_find_matched_drhd_unit(pdev);
3296         if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3297                             TAINT_FIRMWARE_WORKAROUND,
3298                             "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3299                 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3300 }
3301 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3302
3303 static void __init init_no_remapping_devices(void)
3304 {
3305         struct dmar_drhd_unit *drhd;
3306
3307         for_each_drhd_unit(drhd) {
3308                 if (!drhd->include_all) {
3309                         int i;
3310                         for (i = 0; i < drhd->devices_cnt; i++)
3311                                 if (drhd->devices[i] != NULL)
3312                                         break;
3313                         /* ignore DMAR unit if no pci devices exist */
3314                         if (i == drhd->devices_cnt)
3315                                 drhd->ignored = 1;
3316                 }
3317         }
3318
3319         for_each_drhd_unit(drhd) {
3320                 int i;
3321                 if (drhd->ignored || drhd->include_all)
3322                         continue;
3323
3324                 for (i = 0; i < drhd->devices_cnt; i++)
3325                         if (drhd->devices[i] &&
3326                             !IS_GFX_DEVICE(drhd->devices[i]))
3327                                 break;
3328
3329                 if (i < drhd->devices_cnt)
3330                         continue;
3331
3332                 /* This IOMMU has *only* gfx devices. Either bypass it or
3333                    set the gfx_mapped flag, as appropriate */
3334                 if (dmar_map_gfx) {
3335                         intel_iommu_gfx_mapped = 1;
3336                 } else {
3337                         drhd->ignored = 1;
3338                         for (i = 0; i < drhd->devices_cnt; i++) {
3339                                 if (!drhd->devices[i])
3340                                         continue;
3341                                 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3342                         }
3343                 }
3344         }
3345 }
3346
3347 #ifdef CONFIG_SUSPEND
3348 static int init_iommu_hw(void)
3349 {
3350         struct dmar_drhd_unit *drhd;
3351         struct intel_iommu *iommu = NULL;
3352
3353         for_each_active_iommu(iommu, drhd)
3354                 if (iommu->qi)
3355                         dmar_reenable_qi(iommu);
3356
3357         for_each_iommu(iommu, drhd) {
3358                 if (drhd->ignored) {
3359                         /*
3360                          * we always have to disable PMRs or DMA may fail on
3361                          * this device
3362                          */
3363                         if (force_on)
3364                                 iommu_disable_protect_mem_regions(iommu);
3365                         continue;
3366                 }
3367
3368                 iommu_flush_write_buffer(iommu);
3369
3370                 iommu_set_root_entry(iommu);
3371
3372                 iommu->flush.flush_context(iommu, 0, 0, 0,
3373                                            DMA_CCMD_GLOBAL_INVL);
3374                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3375                                          DMA_TLB_GLOBAL_FLUSH);
3376                 if (iommu_enable_translation(iommu))
3377                         return 1;
3378                 iommu_disable_protect_mem_regions(iommu);
3379         }
3380
3381         return 0;
3382 }
3383
3384 static void iommu_flush_all(void)
3385 {
3386         struct dmar_drhd_unit *drhd;
3387         struct intel_iommu *iommu;
3388
3389         for_each_active_iommu(iommu, drhd) {
3390                 iommu->flush.flush_context(iommu, 0, 0, 0,
3391                                            DMA_CCMD_GLOBAL_INVL);
3392                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3393                                          DMA_TLB_GLOBAL_FLUSH);
3394         }
3395 }
3396
3397 static int iommu_suspend(void)
3398 {
3399         struct dmar_drhd_unit *drhd;
3400         struct intel_iommu *iommu = NULL;
3401         unsigned long flag;
3402
3403         for_each_active_iommu(iommu, drhd) {
3404                 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3405                                                  GFP_ATOMIC);
3406                 if (!iommu->iommu_state)
3407                         goto nomem;
3408         }
3409
3410         iommu_flush_all();
3411
3412         for_each_active_iommu(iommu, drhd) {
3413                 iommu_disable_translation(iommu);
3414
3415                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3416
3417                 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3418                         readl(iommu->reg + DMAR_FECTL_REG);
3419                 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3420                         readl(iommu->reg + DMAR_FEDATA_REG);
3421                 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3422                         readl(iommu->reg + DMAR_FEADDR_REG);
3423                 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3424                         readl(iommu->reg + DMAR_FEUADDR_REG);
3425
3426                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3427         }
3428         return 0;
3429
3430 nomem:
3431         for_each_active_iommu(iommu, drhd)
3432                 kfree(iommu->iommu_state);
3433
3434         return -ENOMEM;
3435 }
3436
3437 static void iommu_resume(void)
3438 {
3439         struct dmar_drhd_unit *drhd;
3440         struct intel_iommu *iommu = NULL;
3441         unsigned long flag;
3442
3443         if (init_iommu_hw()) {
3444                 if (force_on)
3445                         panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3446                 else
3447                         WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3448                 return;
3449         }
3450
3451         for_each_active_iommu(iommu, drhd) {
3452
3453                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3454
3455                 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3456                         iommu->reg + DMAR_FECTL_REG);
3457                 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3458                         iommu->reg + DMAR_FEDATA_REG);
3459                 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3460                         iommu->reg + DMAR_FEADDR_REG);
3461                 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3462                         iommu->reg + DMAR_FEUADDR_REG);
3463
3464                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3465         }
3466
3467         for_each_active_iommu(iommu, drhd)
3468                 kfree(iommu->iommu_state);
3469 }
3470
3471 static struct syscore_ops iommu_syscore_ops = {
3472         .resume         = iommu_resume,
3473         .suspend        = iommu_suspend,
3474 };
3475
3476 static void __init init_iommu_pm_ops(void)
3477 {
3478         register_syscore_ops(&iommu_syscore_ops);
3479 }
3480
3481 #else
3482 static inline void init_iommu_pm_ops(void) {}
3483 #endif  /* CONFIG_PM */
3484
3485 LIST_HEAD(dmar_rmrr_units);
3486
3487 static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
3488 {
3489         list_add(&rmrr->list, &dmar_rmrr_units);
3490 }
3491
3492
3493 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header)
3494 {
3495         struct acpi_dmar_reserved_memory *rmrr;
3496         struct dmar_rmrr_unit *rmrru;
3497
3498         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3499         if (!rmrru)
3500                 return -ENOMEM;
3501
3502         rmrru->hdr = header;
3503         rmrr = (struct acpi_dmar_reserved_memory *)header;
3504         rmrru->base_address = rmrr->base_address;
3505         rmrru->end_address = rmrr->end_address;
3506
3507         dmar_register_rmrr_unit(rmrru);
3508         return 0;
3509 }
3510
3511 static int __init
3512 rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
3513 {
3514         struct acpi_dmar_reserved_memory *rmrr;
3515         int ret;
3516
3517         rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
3518         ret = dmar_parse_dev_scope((void *)(rmrr + 1),
3519                 ((void *)rmrr) + rmrr->header.length,
3520                 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
3521
3522         if (ret || (rmrru->devices_cnt == 0)) {
3523                 list_del(&rmrru->list);
3524                 kfree(rmrru);
3525         }
3526         return ret;
3527 }
3528
3529 static LIST_HEAD(dmar_atsr_units);
3530
3531 int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
3532 {
3533         struct acpi_dmar_atsr *atsr;
3534         struct dmar_atsr_unit *atsru;
3535
3536         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3537         atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
3538         if (!atsru)
3539                 return -ENOMEM;
3540
3541         atsru->hdr = hdr;
3542         atsru->include_all = atsr->flags & 0x1;
3543
3544         list_add(&atsru->list, &dmar_atsr_units);
3545
3546         return 0;
3547 }
3548
3549 static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
3550 {
3551         int rc;
3552         struct acpi_dmar_atsr *atsr;
3553
3554         if (atsru->include_all)
3555                 return 0;
3556
3557         atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3558         rc = dmar_parse_dev_scope((void *)(atsr + 1),
3559                                 (void *)atsr + atsr->header.length,
3560                                 &atsru->devices_cnt, &atsru->devices,
3561                                 atsr->segment);
3562         if (rc || !atsru->devices_cnt) {
3563                 list_del(&atsru->list);
3564                 kfree(atsru);
3565         }
3566
3567         return rc;
3568 }
3569
3570 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
3571 {
3572         int i;
3573         struct pci_bus *bus;
3574         struct acpi_dmar_atsr *atsr;
3575         struct dmar_atsr_unit *atsru;
3576
3577         dev = pci_physfn(dev);
3578
3579         list_for_each_entry(atsru, &dmar_atsr_units, list) {
3580                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3581                 if (atsr->segment == pci_domain_nr(dev->bus))
3582                         goto found;
3583         }
3584
3585         return 0;
3586
3587 found:
3588         for (bus = dev->bus; bus; bus = bus->parent) {
3589                 struct pci_dev *bridge = bus->self;
3590
3591                 if (!bridge || !pci_is_pcie(bridge) ||
3592                     pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
3593                         return 0;
3594
3595                 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT) {
3596                         for (i = 0; i < atsru->devices_cnt; i++)
3597                                 if (atsru->devices[i] == bridge)
3598                                         return 1;
3599                         break;
3600                 }
3601         }
3602
3603         if (atsru->include_all)
3604                 return 1;
3605
3606         return 0;
3607 }
3608
3609 int __init dmar_parse_rmrr_atsr_dev(void)
3610 {
3611         struct dmar_rmrr_unit *rmrr, *rmrr_n;
3612         struct dmar_atsr_unit *atsr, *atsr_n;
3613         int ret = 0;
3614
3615         list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
3616                 ret = rmrr_parse_dev(rmrr);
3617                 if (ret)
3618                         return ret;
3619         }
3620
3621         list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
3622                 ret = atsr_parse_dev(atsr);
3623                 if (ret)
3624                         return ret;
3625         }
3626
3627         return ret;
3628 }
3629
3630 /*
3631  * Here we only respond to action of unbound device from driver.
3632  *
3633  * Added device is not attached to its DMAR domain here yet. That will happen
3634  * when mapping the device to iova.
3635  */
3636 static int device_notifier(struct notifier_block *nb,
3637                                   unsigned long action, void *data)
3638 {
3639         struct device *dev = data;
3640         struct pci_dev *pdev = to_pci_dev(dev);
3641         struct dmar_domain *domain;
3642
3643         if (iommu_no_mapping(dev))
3644                 return 0;
3645
3646         domain = find_domain(pdev);
3647         if (!domain)
3648                 return 0;
3649
3650         if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through) {
3651                 domain_remove_one_dev_info(domain, pdev);
3652
3653                 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3654                     !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
3655                     list_empty(&domain->devices))
3656                         domain_exit(domain);
3657         }
3658
3659         return 0;
3660 }
3661
3662 static struct notifier_block device_nb = {
3663         .notifier_call = device_notifier,
3664 };
3665
3666 int __init intel_iommu_init(void)
3667 {
3668         int ret = 0;
3669         struct dmar_drhd_unit *drhd;
3670
3671         /* VT-d is required for a TXT/tboot launch, so enforce that */
3672         force_on = tboot_force_iommu();
3673
3674         if (dmar_table_init()) {
3675                 if (force_on)
3676                         panic("tboot: Failed to initialize DMAR table\n");
3677                 return  -ENODEV;
3678         }
3679
3680         /*
3681          * Disable translation if already enabled prior to OS handover.
3682          */
3683         for_each_drhd_unit(drhd) {
3684                 struct intel_iommu *iommu;
3685
3686                 if (drhd->ignored)
3687                         continue;
3688
3689                 iommu = drhd->iommu;
3690                 if (iommu->gcmd & DMA_GCMD_TE)
3691                         iommu_disable_translation(iommu);
3692         }
3693
3694         if (dmar_dev_scope_init() < 0) {
3695                 if (force_on)
3696                         panic("tboot: Failed to initialize DMAR device scope\n");
3697                 return  -ENODEV;
3698         }
3699
3700         if (no_iommu || dmar_disabled)
3701                 return -ENODEV;
3702
3703         if (iommu_init_mempool()) {
3704                 if (force_on)
3705                         panic("tboot: Failed to initialize iommu memory\n");
3706                 return  -ENODEV;
3707         }
3708
3709         if (list_empty(&dmar_rmrr_units))
3710                 printk(KERN_INFO "DMAR: No RMRR found\n");
3711
3712         if (list_empty(&dmar_atsr_units))
3713                 printk(KERN_INFO "DMAR: No ATSR found\n");
3714
3715         if (dmar_init_reserved_ranges()) {
3716                 if (force_on)
3717                         panic("tboot: Failed to reserve iommu ranges\n");
3718                 return  -ENODEV;
3719         }
3720
3721         init_no_remapping_devices();
3722
3723         ret = init_dmars();
3724         if (ret) {
3725                 if (force_on)
3726                         panic("tboot: Failed to initialize DMARs\n");
3727                 printk(KERN_ERR "IOMMU: dmar init failed\n");
3728                 put_iova_domain(&reserved_iova_list);
3729                 iommu_exit_mempool();
3730                 return ret;
3731         }
3732         printk(KERN_INFO
3733         "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3734
3735         init_timer(&unmap_timer);
3736 #ifdef CONFIG_SWIOTLB
3737         swiotlb = 0;
3738 #endif
3739         dma_ops = &intel_dma_ops;
3740
3741         init_iommu_pm_ops();
3742
3743         bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
3744
3745         bus_register_notifier(&pci_bus_type, &device_nb);
3746
3747         intel_iommu_enabled = 1;
3748
3749         return 0;
3750 }
3751
3752 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3753                                            struct pci_dev *pdev)
3754 {
3755         struct pci_dev *tmp, *parent;
3756
3757         if (!iommu || !pdev)
3758                 return;
3759
3760         /* dependent device detach */
3761         tmp = pci_find_upstream_pcie_bridge(pdev);
3762         /* Secondary interface's bus number and devfn 0 */
3763         if (tmp) {
3764                 parent = pdev->bus->self;
3765                 while (parent != tmp) {
3766                         iommu_detach_dev(iommu, parent->bus->number,
3767                                          parent->devfn);
3768                         parent = parent->bus->self;
3769                 }
3770                 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
3771                         iommu_detach_dev(iommu,
3772                                 tmp->subordinate->number, 0);
3773                 else /* this is a legacy PCI bridge */
3774                         iommu_detach_dev(iommu, tmp->bus->number,
3775                                          tmp->devfn);
3776         }
3777 }
3778
3779 static void domain_remove_one_dev_info(struct dmar_domain *domain,
3780                                           struct pci_dev *pdev)
3781 {
3782         struct device_domain_info *info;
3783         struct intel_iommu *iommu;
3784         unsigned long flags;
3785         int found = 0;
3786         struct list_head *entry, *tmp;
3787
3788         iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3789                                 pdev->devfn);
3790         if (!iommu)
3791                 return;
3792
3793         spin_lock_irqsave(&device_domain_lock, flags);
3794         list_for_each_safe(entry, tmp, &domain->devices) {
3795                 info = list_entry(entry, struct device_domain_info, link);
3796                 if (info->segment == pci_domain_nr(pdev->bus) &&
3797                     info->bus == pdev->bus->number &&
3798                     info->devfn == pdev->devfn) {
3799                         unlink_domain_info(info);
3800                         spin_unlock_irqrestore(&device_domain_lock, flags);
3801
3802                         iommu_disable_dev_iotlb(info);
3803                         iommu_detach_dev(iommu, info->bus, info->devfn);
3804                         iommu_detach_dependent_devices(iommu, pdev);
3805                         free_devinfo_mem(info);
3806
3807                         spin_lock_irqsave(&device_domain_lock, flags);
3808
3809                         if (found)
3810                                 break;
3811                         else
3812                                 continue;
3813                 }
3814
3815                 /* if there is no other devices under the same iommu
3816                  * owned by this domain, clear this iommu in iommu_bmp
3817                  * update iommu count and coherency
3818                  */
3819                 if (iommu == device_to_iommu(info->segment, info->bus,
3820                                             info->devfn))
3821                         found = 1;
3822         }
3823
3824         spin_unlock_irqrestore(&device_domain_lock, flags);
3825
3826         if (found == 0) {
3827                 unsigned long tmp_flags;
3828                 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
3829                 clear_bit(iommu->seq_id, domain->iommu_bmp);
3830                 domain->iommu_count--;
3831                 domain_update_iommu_cap(domain);
3832                 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
3833
3834                 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3835                     !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) {
3836                         spin_lock_irqsave(&iommu->lock, tmp_flags);
3837                         clear_bit(domain->id, iommu->domain_ids);
3838                         iommu->domains[domain->id] = NULL;
3839                         spin_unlock_irqrestore(&iommu->lock, tmp_flags);
3840                 }
3841         }
3842 }
3843
3844 static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3845 {
3846         struct device_domain_info *info;
3847         struct intel_iommu *iommu;
3848         unsigned long flags1, flags2;
3849
3850         spin_lock_irqsave(&device_domain_lock, flags1);
3851         while (!list_empty(&domain->devices)) {
3852                 info = list_entry(domain->devices.next,
3853                         struct device_domain_info, link);
3854                 unlink_domain_info(info);
3855                 spin_unlock_irqrestore(&device_domain_lock, flags1);
3856
3857                 iommu_disable_dev_iotlb(info);
3858                 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
3859                 iommu_detach_dev(iommu, info->bus, info->devfn);
3860                 iommu_detach_dependent_devices(iommu, info->dev);
3861
3862                 /* clear this iommu in iommu_bmp, update iommu count
3863                  * and capabilities
3864                  */
3865                 spin_lock_irqsave(&domain->iommu_lock, flags2);
3866                 if (test_and_clear_bit(iommu->seq_id,
3867                                        domain->iommu_bmp)) {
3868                         domain->iommu_count--;
3869                         domain_update_iommu_cap(domain);
3870                 }
3871                 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3872
3873                 free_devinfo_mem(info);
3874                 spin_lock_irqsave(&device_domain_lock, flags1);
3875         }
3876         spin_unlock_irqrestore(&device_domain_lock, flags1);
3877 }
3878
3879 /* domain id for virtual machine, it won't be set in context */
3880 static unsigned long vm_domid;
3881
3882 static struct dmar_domain *iommu_alloc_vm_domain(void)
3883 {
3884         struct dmar_domain *domain;
3885
3886         domain = alloc_domain_mem();
3887         if (!domain)
3888                 return NULL;
3889
3890         domain->id = vm_domid++;
3891         domain->nid = -1;
3892         memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
3893         domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3894
3895         return domain;
3896 }
3897
3898 static int md_domain_init(struct dmar_domain *domain, int guest_width)
3899 {
3900         int adjust_width;
3901
3902         init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
3903         spin_lock_init(&domain->iommu_lock);
3904
3905         domain_reserve_special_ranges(domain);
3906
3907         /* calculate AGAW */
3908         domain->gaw = guest_width;
3909         adjust_width = guestwidth_to_adjustwidth(guest_width);
3910         domain->agaw = width_to_agaw(adjust_width);
3911
3912         INIT_LIST_HEAD(&domain->devices);
3913
3914         domain->iommu_count = 0;
3915         domain->iommu_coherency = 0;
3916         domain->iommu_snooping = 0;
3917         domain->iommu_superpage = 0;
3918         domain->max_addr = 0;
3919         domain->nid = -1;
3920
3921         /* always allocate the top pgd */
3922         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
3923         if (!domain->pgd)
3924                 return -ENOMEM;
3925         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3926         return 0;
3927 }
3928
3929 static void iommu_free_vm_domain(struct dmar_domain *domain)
3930 {
3931         unsigned long flags;
3932         struct dmar_drhd_unit *drhd;
3933         struct intel_iommu *iommu;
3934         unsigned long i;
3935         unsigned long ndomains;
3936
3937         for_each_drhd_unit(drhd) {
3938                 if (drhd->ignored)
3939                         continue;
3940                 iommu = drhd->iommu;
3941
3942                 ndomains = cap_ndoms(iommu->cap);
3943                 for_each_set_bit(i, iommu->domain_ids, ndomains) {
3944                         if (iommu->domains[i] == domain) {
3945                                 spin_lock_irqsave(&iommu->lock, flags);
3946                                 clear_bit(i, iommu->domain_ids);
3947                                 iommu->domains[i] = NULL;
3948                                 spin_unlock_irqrestore(&iommu->lock, flags);
3949                                 break;
3950                         }
3951                 }
3952         }
3953 }
3954
3955 static void vm_domain_exit(struct dmar_domain *domain)
3956 {
3957         /* Domain 0 is reserved, so dont process it */
3958         if (!domain)
3959                 return;
3960
3961         vm_domain_remove_all_dev_info(domain);
3962         /* destroy iovas */
3963         put_iova_domain(&domain->iovad);
3964
3965         /* clear ptes */
3966         dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3967
3968         /* free page tables */
3969         dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3970
3971         iommu_free_vm_domain(domain);
3972         free_domain_mem(domain);
3973 }
3974
3975 static int intel_iommu_domain_init(struct iommu_domain *domain)
3976 {
3977         struct dmar_domain *dmar_domain;
3978
3979         dmar_domain = iommu_alloc_vm_domain();
3980         if (!dmar_domain) {
3981                 printk(KERN_ERR
3982                         "intel_iommu_domain_init: dmar_domain == NULL\n");
3983                 return -ENOMEM;
3984         }
3985         if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
3986                 printk(KERN_ERR
3987                         "intel_iommu_domain_init() failed\n");
3988                 vm_domain_exit(dmar_domain);
3989                 return -ENOMEM;
3990         }
3991         domain_update_iommu_cap(dmar_domain);
3992         domain->priv = dmar_domain;
3993
3994         domain->geometry.aperture_start = 0;
3995         domain->geometry.aperture_end   = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
3996         domain->geometry.force_aperture = true;
3997
3998         return 0;
3999 }
4000
4001 static void intel_iommu_domain_destroy(struct iommu_domain *domain)
4002 {
4003         struct dmar_domain *dmar_domain = domain->priv;
4004
4005         domain->priv = NULL;
4006         vm_domain_exit(dmar_domain);
4007 }
4008
4009 static int intel_iommu_attach_device(struct iommu_domain *domain,
4010                                      struct device *dev)
4011 {
4012         struct dmar_domain *dmar_domain = domain->priv;
4013         struct pci_dev *pdev = to_pci_dev(dev);
4014         struct intel_iommu *iommu;
4015         int addr_width;
4016
4017         /* normally pdev is not mapped */
4018         if (unlikely(domain_context_mapped(pdev))) {
4019                 struct dmar_domain *old_domain;
4020
4021                 old_domain = find_domain(pdev);
4022                 if (old_domain) {
4023                         if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
4024                             dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
4025                                 domain_remove_one_dev_info(old_domain, pdev);
4026                         else
4027                                 domain_remove_dev_info(old_domain);
4028                 }
4029         }
4030
4031         iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
4032                                 pdev->devfn);
4033         if (!iommu)
4034                 return -ENODEV;
4035
4036         /* check if this iommu agaw is sufficient for max mapped address */
4037         addr_width = agaw_to_width(iommu->agaw);
4038         if (addr_width > cap_mgaw(iommu->cap))
4039                 addr_width = cap_mgaw(iommu->cap);
4040
4041         if (dmar_domain->max_addr > (1LL << addr_width)) {
4042                 printk(KERN_ERR "%s: iommu width (%d) is not "
4043                        "sufficient for the mapped address (%llx)\n",
4044                        __func__, addr_width, dmar_domain->max_addr);
4045                 return -EFAULT;
4046         }
4047         dmar_domain->gaw = addr_width;
4048
4049         /*
4050          * Knock out extra levels of page tables if necessary
4051          */
4052         while (iommu->agaw < dmar_domain->agaw) {
4053                 struct dma_pte *pte;
4054
4055                 pte = dmar_domain->pgd;
4056                 if (dma_pte_present(pte)) {
4057                         dmar_domain->pgd = (struct dma_pte *)
4058                                 phys_to_virt(dma_pte_addr(pte));
4059                         free_pgtable_page(pte);
4060                 }
4061                 dmar_domain->agaw--;
4062         }
4063
4064         return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
4065 }
4066
4067 static void intel_iommu_detach_device(struct iommu_domain *domain,
4068                                       struct device *dev)
4069 {
4070         struct dmar_domain *dmar_domain = domain->priv;
4071         struct pci_dev *pdev = to_pci_dev(dev);
4072
4073         domain_remove_one_dev_info(dmar_domain, pdev);
4074 }
4075
4076 static int intel_iommu_map(struct iommu_domain *domain,
4077                            unsigned long iova, phys_addr_t hpa,
4078                            size_t size, int iommu_prot)
4079 {
4080         struct dmar_domain *dmar_domain = domain->priv;
4081         u64 max_addr;
4082         int prot = 0;
4083         int ret;
4084
4085         if (iommu_prot & IOMMU_READ)
4086                 prot |= DMA_PTE_READ;
4087         if (iommu_prot & IOMMU_WRITE)
4088                 prot |= DMA_PTE_WRITE;
4089         if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4090                 prot |= DMA_PTE_SNP;
4091
4092         max_addr = iova + size;
4093         if (dmar_domain->max_addr < max_addr) {
4094                 u64 end;
4095
4096                 /* check if minimum agaw is sufficient for mapped address */
4097                 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
4098                 if (end < max_addr) {
4099                         printk(KERN_ERR "%s: iommu width (%d) is not "
4100                                "sufficient for the mapped address (%llx)\n",
4101                                __func__, dmar_domain->gaw, max_addr);
4102                         return -EFAULT;
4103                 }
4104                 dmar_domain->max_addr = max_addr;
4105         }
4106         /* Round up size to next multiple of PAGE_SIZE, if it and
4107            the low bits of hpa would take us onto the next page */
4108         size = aligned_nrpages(hpa, size);
4109         ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4110                                  hpa >> VTD_PAGE_SHIFT, size, prot);
4111         return ret;
4112 }
4113
4114 static size_t intel_iommu_unmap(struct iommu_domain *domain,
4115                              unsigned long iova, size_t size)
4116 {
4117         struct dmar_domain *dmar_domain = domain->priv;
4118         int order;
4119
4120         order = dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
4121                             (iova + size - 1) >> VTD_PAGE_SHIFT);
4122
4123         if (dmar_domain->max_addr == iova + size)
4124                 dmar_domain->max_addr = iova;
4125
4126         return PAGE_SIZE << order;
4127 }
4128
4129 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
4130                                             dma_addr_t iova)
4131 {
4132         struct dmar_domain *dmar_domain = domain->priv;
4133         struct dma_pte *pte;
4134         u64 phys = 0;
4135
4136         pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, 0);
4137         if (pte)
4138                 phys = dma_pte_addr(pte);
4139
4140         return phys;
4141 }
4142
4143 static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
4144                                       unsigned long cap)
4145 {
4146         struct dmar_domain *dmar_domain = domain->priv;
4147
4148         if (cap == IOMMU_CAP_CACHE_COHERENCY)
4149                 return dmar_domain->iommu_snooping;
4150         if (cap == IOMMU_CAP_INTR_REMAP)
4151                 return irq_remapping_enabled;
4152
4153         return 0;
4154 }
4155
4156 #define REQ_ACS_FLAGS   (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
4157
4158 static int intel_iommu_add_device(struct device *dev)
4159 {
4160         struct pci_dev *pdev = to_pci_dev(dev);
4161         struct pci_dev *bridge, *dma_pdev = NULL;
4162         struct iommu_group *group;
4163         int ret;
4164
4165         if (!device_to_iommu(pci_domain_nr(pdev->bus),
4166                              pdev->bus->number, pdev->devfn))
4167                 return -ENODEV;
4168
4169         bridge = pci_find_upstream_pcie_bridge(pdev);
4170         if (bridge) {
4171                 if (pci_is_pcie(bridge))
4172                         dma_pdev = pci_get_domain_bus_and_slot(
4173                                                 pci_domain_nr(pdev->bus),
4174                                                 bridge->subordinate->number, 0);
4175                 if (!dma_pdev)
4176                         dma_pdev = pci_dev_get(bridge);
4177         } else
4178                 dma_pdev = pci_dev_get(pdev);
4179
4180         /* Account for quirked devices */
4181         swap_pci_ref(&dma_pdev, pci_get_dma_source(dma_pdev));
4182
4183         /*
4184          * If it's a multifunction device that does not support our
4185          * required ACS flags, add to the same group as function 0.
4186          */
4187         if (dma_pdev->multifunction &&
4188             !pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS))
4189                 swap_pci_ref(&dma_pdev,
4190                              pci_get_slot(dma_pdev->bus,
4191                                           PCI_DEVFN(PCI_SLOT(dma_pdev->devfn),
4192                                           0)));
4193
4194         /*
4195          * Devices on the root bus go through the iommu.  If that's not us,
4196          * find the next upstream device and test ACS up to the root bus.
4197          * Finding the next device may require skipping virtual buses.
4198          */
4199         while (!pci_is_root_bus(dma_pdev->bus)) {
4200                 struct pci_bus *bus = dma_pdev->bus;
4201
4202                 while (!bus->self) {
4203                         if (!pci_is_root_bus(bus))
4204                                 bus = bus->parent;
4205                         else
4206                                 goto root_bus;
4207                 }
4208
4209                 if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
4210                         break;
4211
4212                 swap_pci_ref(&dma_pdev, pci_dev_get(bus->self));
4213         }
4214
4215 root_bus:
4216         group = iommu_group_get(&dma_pdev->dev);
4217         pci_dev_put(dma_pdev);
4218         if (!group) {
4219                 group = iommu_group_alloc();
4220                 if (IS_ERR(group))
4221                         return PTR_ERR(group);
4222         }
4223
4224         ret = iommu_group_add_device(group, dev);
4225
4226         iommu_group_put(group);
4227         return ret;
4228 }
4229
4230 static void intel_iommu_remove_device(struct device *dev)
4231 {
4232         iommu_group_remove_device(dev);
4233 }
4234
4235 static struct iommu_ops intel_iommu_ops = {
4236         .domain_init    = intel_iommu_domain_init,
4237         .domain_destroy = intel_iommu_domain_destroy,
4238         .attach_dev     = intel_iommu_attach_device,
4239         .detach_dev     = intel_iommu_detach_device,
4240         .map            = intel_iommu_map,
4241         .unmap          = intel_iommu_unmap,
4242         .iova_to_phys   = intel_iommu_iova_to_phys,
4243         .domain_has_cap = intel_iommu_domain_has_cap,
4244         .add_device     = intel_iommu_add_device,
4245         .remove_device  = intel_iommu_remove_device,
4246         .pgsize_bitmap  = INTEL_IOMMU_PGSIZES,
4247 };
4248
4249 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4250 {
4251         /* G4x/GM45 integrated gfx dmar support is totally busted. */
4252         printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
4253         dmar_map_gfx = 0;
4254 }
4255
4256 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4257 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4258 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4259 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4260 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4261 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4262 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4263
4264 static void quirk_iommu_rwbf(struct pci_dev *dev)
4265 {
4266         /*
4267          * Mobile 4 Series Chipset neglects to set RWBF capability,
4268          * but needs it. Same seems to hold for the desktop versions.
4269          */
4270         printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
4271         rwbf_quirk = 1;
4272 }
4273
4274 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
4275 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4276 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4277 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4278 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4279 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4280 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
4281
4282 #define GGC 0x52
4283 #define GGC_MEMORY_SIZE_MASK    (0xf << 8)
4284 #define GGC_MEMORY_SIZE_NONE    (0x0 << 8)
4285 #define GGC_MEMORY_SIZE_1M      (0x1 << 8)
4286 #define GGC_MEMORY_SIZE_2M      (0x3 << 8)
4287 #define GGC_MEMORY_VT_ENABLED   (0x8 << 8)
4288 #define GGC_MEMORY_SIZE_2M_VT   (0x9 << 8)
4289 #define GGC_MEMORY_SIZE_3M_VT   (0xa << 8)
4290 #define GGC_MEMORY_SIZE_4M_VT   (0xb << 8)
4291
4292 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
4293 {
4294         unsigned short ggc;
4295
4296         if (pci_read_config_word(dev, GGC, &ggc))
4297                 return;
4298
4299         if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
4300                 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4301                 dmar_map_gfx = 0;
4302         } else if (dmar_map_gfx) {
4303                 /* we have to ensure the gfx device is idle before we flush */
4304                 printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
4305                 intel_iommu_strict = 1;
4306        }
4307 }
4308 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4309 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4310 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4311 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4312
4313 /* On Tylersburg chipsets, some BIOSes have been known to enable the
4314    ISOCH DMAR unit for the Azalia sound device, but not give it any
4315    TLB entries, which causes it to deadlock. Check for that.  We do
4316    this in a function called from init_dmars(), instead of in a PCI
4317    quirk, because we don't want to print the obnoxious "BIOS broken"
4318    message if VT-d is actually disabled.
4319 */
4320 static void __init check_tylersburg_isoch(void)
4321 {
4322         struct pci_dev *pdev;
4323         uint32_t vtisochctrl;
4324
4325         /* If there's no Azalia in the system anyway, forget it. */
4326         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4327         if (!pdev)
4328                 return;
4329         pci_dev_put(pdev);
4330
4331         /* System Management Registers. Might be hidden, in which case
4332            we can't do the sanity check. But that's OK, because the
4333            known-broken BIOSes _don't_ actually hide it, so far. */
4334         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4335         if (!pdev)
4336                 return;
4337
4338         if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4339                 pci_dev_put(pdev);
4340                 return;
4341         }
4342
4343         pci_dev_put(pdev);
4344
4345         /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4346         if (vtisochctrl & 1)
4347                 return;
4348
4349         /* Drop all bits other than the number of TLB entries */
4350         vtisochctrl &= 0x1c;
4351
4352         /* If we have the recommended number of TLB entries (16), fine. */
4353         if (vtisochctrl == 0x10)
4354                 return;
4355
4356         /* Zero TLB entries? You get to ride the short bus to school. */
4357         if (!vtisochctrl) {
4358                 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4359                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4360                      dmi_get_system_info(DMI_BIOS_VENDOR),
4361                      dmi_get_system_info(DMI_BIOS_VERSION),
4362                      dmi_get_system_info(DMI_PRODUCT_VERSION));
4363                 iommu_identity_mapping |= IDENTMAP_AZALIA;
4364                 return;
4365         }
4366
4367         printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4368                vtisochctrl);
4369 }