2 * Copyright (c) 2006, Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
22 * This file implements early detection/parsing of Remapping Devices
23 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
26 * These routines are used by both DMA-remapping and Interrupt-remapping
29 #define pr_fmt(fmt) "DMAR: " fmt
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/iova.h>
34 #include <linux/intel-iommu.h>
35 #include <linux/timer.h>
36 #include <linux/irq.h>
37 #include <linux/interrupt.h>
38 #include <linux/tboot.h>
39 #include <linux/dmi.h>
40 #include <linux/slab.h>
41 #include <linux/iommu.h>
42 #include <asm/irq_remapping.h>
43 #include <asm/iommu_table.h>
45 #include "irq_remapping.h"
47 typedef int (*dmar_res_handler_t)(struct acpi_dmar_header *, void *);
48 struct dmar_res_callback {
49 dmar_res_handler_t cb[ACPI_DMAR_TYPE_RESERVED];
50 void *arg[ACPI_DMAR_TYPE_RESERVED];
51 bool ignore_unhandled;
57 * 1) The hotplug framework guarentees that DMAR unit will be hot-added
58 * before IO devices managed by that unit.
59 * 2) The hotplug framework guarantees that DMAR unit will be hot-removed
60 * after IO devices managed by that unit.
61 * 3) Hotplug events are rare.
63 * Locking rules for DMA and interrupt remapping related global data structures:
64 * 1) Use dmar_global_lock in process context
65 * 2) Use RCU in interrupt context
67 DECLARE_RWSEM(dmar_global_lock);
68 LIST_HEAD(dmar_drhd_units);
70 struct acpi_table_header * __initdata dmar_tbl;
71 static acpi_size dmar_tbl_size;
72 static int dmar_dev_scope_status = 1;
73 static unsigned long dmar_seq_ids[BITS_TO_LONGS(DMAR_UNITS_SUPPORTED)];
75 static int alloc_iommu(struct dmar_drhd_unit *drhd);
76 static void free_iommu(struct intel_iommu *iommu);
78 static void dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
81 * add INCLUDE_ALL at the tail, so scan the list will find it at
84 if (drhd->include_all)
85 list_add_tail_rcu(&drhd->list, &dmar_drhd_units);
87 list_add_rcu(&drhd->list, &dmar_drhd_units);
90 void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
92 struct acpi_dmar_device_scope *scope;
97 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_NAMESPACE ||
98 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
99 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
101 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
102 scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
103 pr_warn("Unsupported device scope\n");
105 start += scope->length;
110 return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL);
113 void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
116 struct device *tmp_dev;
118 if (*devices && *cnt) {
119 for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
128 /* Optimize out kzalloc()/kfree() for normal cases */
129 static char dmar_pci_notify_info_buf[64];
131 static struct dmar_pci_notify_info *
132 dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
137 struct dmar_pci_notify_info *info;
139 BUG_ON(dev->is_virtfn);
141 /* Only generate path[] for device addition event */
142 if (event == BUS_NOTIFY_ADD_DEVICE)
143 for (tmp = dev; tmp; tmp = tmp->bus->self)
146 size = sizeof(*info) + level * sizeof(struct acpi_dmar_pci_path);
147 if (size <= sizeof(dmar_pci_notify_info_buf)) {
148 info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
150 info = kzalloc(size, GFP_KERNEL);
152 pr_warn("Out of memory when allocating notify_info "
153 "for %s.\n", pci_name(dev));
154 if (dmar_dev_scope_status == 0)
155 dmar_dev_scope_status = -ENOMEM;
162 info->seg = pci_domain_nr(dev->bus);
164 if (event == BUS_NOTIFY_ADD_DEVICE) {
165 for (tmp = dev; tmp; tmp = tmp->bus->self) {
167 info->path[level].bus = tmp->bus->number;
168 info->path[level].device = PCI_SLOT(tmp->devfn);
169 info->path[level].function = PCI_FUNC(tmp->devfn);
170 if (pci_is_root_bus(tmp->bus))
171 info->bus = tmp->bus->number;
178 static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
180 if ((void *)info != dmar_pci_notify_info_buf)
184 static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
185 struct acpi_dmar_pci_path *path, int count)
189 if (info->bus != bus)
191 if (info->level != count)
194 for (i = 0; i < count; i++) {
195 if (path[i].device != info->path[i].device ||
196 path[i].function != info->path[i].function)
208 if (bus == info->path[i].bus &&
209 path[0].device == info->path[i].device &&
210 path[0].function == info->path[i].function) {
211 pr_info(FW_BUG "RMRR entry for device %02x:%02x.%x is broken - applying workaround\n",
212 bus, path[0].device, path[0].function);
219 /* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
220 int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
221 void *start, void*end, u16 segment,
222 struct dmar_dev_scope *devices,
226 struct device *tmp, *dev = &info->dev->dev;
227 struct acpi_dmar_device_scope *scope;
228 struct acpi_dmar_pci_path *path;
230 if (segment != info->seg)
233 for (; start < end; start += scope->length) {
235 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
236 scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
239 path = (struct acpi_dmar_pci_path *)(scope + 1);
240 level = (scope->length - sizeof(*scope)) / sizeof(*path);
241 if (!dmar_match_pci_path(info, scope->bus, path, level))
244 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT) ^
245 (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL)) {
246 pr_warn("Device scope type does not match for %s\n",
247 pci_name(info->dev));
251 for_each_dev_scope(devices, devices_cnt, i, tmp)
253 devices[i].bus = info->dev->bus->number;
254 devices[i].devfn = info->dev->devfn;
255 rcu_assign_pointer(devices[i].dev,
259 BUG_ON(i >= devices_cnt);
265 int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
266 struct dmar_dev_scope *devices, int count)
271 if (info->seg != segment)
274 for_each_active_dev_scope(devices, count, index, tmp)
275 if (tmp == &info->dev->dev) {
276 RCU_INIT_POINTER(devices[index].dev, NULL);
285 static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
288 struct dmar_drhd_unit *dmaru;
289 struct acpi_dmar_hardware_unit *drhd;
291 for_each_drhd_unit(dmaru) {
292 if (dmaru->include_all)
295 drhd = container_of(dmaru->hdr,
296 struct acpi_dmar_hardware_unit, header);
297 ret = dmar_insert_dev_scope(info, (void *)(drhd + 1),
298 ((void *)drhd) + drhd->header.length,
300 dmaru->devices, dmaru->devices_cnt);
305 ret = dmar_iommu_notify_scope_dev(info);
306 if (ret < 0 && dmar_dev_scope_status == 0)
307 dmar_dev_scope_status = ret;
312 static void dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
314 struct dmar_drhd_unit *dmaru;
316 for_each_drhd_unit(dmaru)
317 if (dmar_remove_dev_scope(info, dmaru->segment,
318 dmaru->devices, dmaru->devices_cnt))
320 dmar_iommu_notify_scope_dev(info);
323 static int dmar_pci_bus_notifier(struct notifier_block *nb,
324 unsigned long action, void *data)
326 struct pci_dev *pdev = to_pci_dev(data);
327 struct dmar_pci_notify_info *info;
329 /* Only care about add/remove events for physical functions */
332 if (action != BUS_NOTIFY_ADD_DEVICE &&
333 action != BUS_NOTIFY_REMOVED_DEVICE)
336 info = dmar_alloc_pci_notify_info(pdev, action);
340 down_write(&dmar_global_lock);
341 if (action == BUS_NOTIFY_ADD_DEVICE)
342 dmar_pci_bus_add_dev(info);
343 else if (action == BUS_NOTIFY_REMOVED_DEVICE)
344 dmar_pci_bus_del_dev(info);
345 up_write(&dmar_global_lock);
347 dmar_free_pci_notify_info(info);
352 static struct notifier_block dmar_pci_bus_nb = {
353 .notifier_call = dmar_pci_bus_notifier,
357 static struct dmar_drhd_unit *
358 dmar_find_dmaru(struct acpi_dmar_hardware_unit *drhd)
360 struct dmar_drhd_unit *dmaru;
362 list_for_each_entry_rcu(dmaru, &dmar_drhd_units, list)
363 if (dmaru->segment == drhd->segment &&
364 dmaru->reg_base_addr == drhd->address)
371 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
372 * structure which uniquely represent one DMA remapping hardware unit
373 * present in the platform
375 static int dmar_parse_one_drhd(struct acpi_dmar_header *header, void *arg)
377 struct acpi_dmar_hardware_unit *drhd;
378 struct dmar_drhd_unit *dmaru;
381 drhd = (struct acpi_dmar_hardware_unit *)header;
382 dmaru = dmar_find_dmaru(drhd);
386 dmaru = kzalloc(sizeof(*dmaru) + header->length, GFP_KERNEL);
391 * If header is allocated from slab by ACPI _DSM method, we need to
392 * copy the content because the memory buffer will be freed on return.
394 dmaru->hdr = (void *)(dmaru + 1);
395 memcpy(dmaru->hdr, header, header->length);
396 dmaru->reg_base_addr = drhd->address;
397 dmaru->segment = drhd->segment;
398 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
399 dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
400 ((void *)drhd) + drhd->header.length,
401 &dmaru->devices_cnt);
402 if (dmaru->devices_cnt && dmaru->devices == NULL) {
407 ret = alloc_iommu(dmaru);
409 dmar_free_dev_scope(&dmaru->devices,
410 &dmaru->devices_cnt);
414 dmar_register_drhd_unit(dmaru);
423 static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
425 if (dmaru->devices && dmaru->devices_cnt)
426 dmar_free_dev_scope(&dmaru->devices, &dmaru->devices_cnt);
428 free_iommu(dmaru->iommu);
432 static int __init dmar_parse_one_andd(struct acpi_dmar_header *header,
435 struct acpi_dmar_andd *andd = (void *)header;
437 /* Check for NUL termination within the designated length */
438 if (strnlen(andd->device_name, header->length - 8) == header->length - 8) {
439 WARN_TAINT(1, TAINT_FIRMWARE_WORKAROUND,
440 "Your BIOS is broken; ANDD object name is not NUL-terminated\n"
441 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
442 dmi_get_system_info(DMI_BIOS_VENDOR),
443 dmi_get_system_info(DMI_BIOS_VERSION),
444 dmi_get_system_info(DMI_PRODUCT_VERSION));
447 pr_info("ANDD device: %x name: %s\n", andd->device_number,
453 #ifdef CONFIG_ACPI_NUMA
454 static int dmar_parse_one_rhsa(struct acpi_dmar_header *header, void *arg)
456 struct acpi_dmar_rhsa *rhsa;
457 struct dmar_drhd_unit *drhd;
459 rhsa = (struct acpi_dmar_rhsa *)header;
460 for_each_drhd_unit(drhd) {
461 if (drhd->reg_base_addr == rhsa->base_address) {
462 int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
464 if (!node_online(node))
466 drhd->iommu->node = node;
471 1, TAINT_FIRMWARE_WORKAROUND,
472 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
473 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
475 dmi_get_system_info(DMI_BIOS_VENDOR),
476 dmi_get_system_info(DMI_BIOS_VERSION),
477 dmi_get_system_info(DMI_PRODUCT_VERSION));
482 #define dmar_parse_one_rhsa dmar_res_noop
486 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
488 struct acpi_dmar_hardware_unit *drhd;
489 struct acpi_dmar_reserved_memory *rmrr;
490 struct acpi_dmar_atsr *atsr;
491 struct acpi_dmar_rhsa *rhsa;
493 switch (header->type) {
494 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
495 drhd = container_of(header, struct acpi_dmar_hardware_unit,
497 pr_info("DRHD base: %#016Lx flags: %#x\n",
498 (unsigned long long)drhd->address, drhd->flags);
500 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
501 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
503 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
504 (unsigned long long)rmrr->base_address,
505 (unsigned long long)rmrr->end_address);
507 case ACPI_DMAR_TYPE_ROOT_ATS:
508 atsr = container_of(header, struct acpi_dmar_atsr, header);
509 pr_info("ATSR flags: %#x\n", atsr->flags);
511 case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
512 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
513 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
514 (unsigned long long)rhsa->base_address,
515 rhsa->proximity_domain);
517 case ACPI_DMAR_TYPE_NAMESPACE:
518 /* We don't print this here because we need to sanity-check
519 it first. So print it in dmar_parse_one_andd() instead. */
525 * dmar_table_detect - checks to see if the platform supports DMAR devices
527 static int __init dmar_table_detect(void)
529 acpi_status status = AE_OK;
531 /* if we could find DMAR table, then there are DMAR devices */
532 status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
533 (struct acpi_table_header **)&dmar_tbl,
536 if (ACPI_SUCCESS(status) && !dmar_tbl) {
537 pr_warn("Unable to map DMAR\n");
538 status = AE_NOT_FOUND;
541 return (ACPI_SUCCESS(status) ? 1 : 0);
544 static int dmar_walk_remapping_entries(struct acpi_dmar_header *start,
545 size_t len, struct dmar_res_callback *cb)
548 struct acpi_dmar_header *iter, *next;
549 struct acpi_dmar_header *end = ((void *)start) + len;
551 for (iter = start; iter < end && ret == 0; iter = next) {
552 next = (void *)iter + iter->length;
553 if (iter->length == 0) {
554 /* Avoid looping forever on bad ACPI tables */
555 pr_debug(FW_BUG "Invalid 0-length structure\n");
557 } else if (next > end) {
558 /* Avoid passing table end */
559 pr_warn(FW_BUG "Record passes table end\n");
565 dmar_table_print_dmar_entry(iter);
567 if (iter->type >= ACPI_DMAR_TYPE_RESERVED) {
568 /* continue for forward compatibility */
569 pr_debug("Unknown DMAR structure type %d\n",
571 } else if (cb->cb[iter->type]) {
572 ret = cb->cb[iter->type](iter, cb->arg[iter->type]);
573 } else if (!cb->ignore_unhandled) {
574 pr_warn("No handler for DMAR structure type %d\n",
583 static inline int dmar_walk_dmar_table(struct acpi_table_dmar *dmar,
584 struct dmar_res_callback *cb)
586 return dmar_walk_remapping_entries((void *)(dmar + 1),
587 dmar->header.length - sizeof(*dmar), cb);
591 * parse_dmar_table - parses the DMA reporting table
594 parse_dmar_table(void)
596 struct acpi_table_dmar *dmar;
599 struct dmar_res_callback cb = {
601 .ignore_unhandled = true,
602 .arg[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &drhd_count,
603 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_parse_one_drhd,
604 .cb[ACPI_DMAR_TYPE_RESERVED_MEMORY] = &dmar_parse_one_rmrr,
605 .cb[ACPI_DMAR_TYPE_ROOT_ATS] = &dmar_parse_one_atsr,
606 .cb[ACPI_DMAR_TYPE_HARDWARE_AFFINITY] = &dmar_parse_one_rhsa,
607 .cb[ACPI_DMAR_TYPE_NAMESPACE] = &dmar_parse_one_andd,
611 * Do it again, earlier dmar_tbl mapping could be mapped with
617 * ACPI tables may not be DMA protected by tboot, so use DMAR copy
618 * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
620 dmar_tbl = tboot_get_dmar_table(dmar_tbl);
622 dmar = (struct acpi_table_dmar *)dmar_tbl;
626 if (dmar->width < PAGE_SHIFT - 1) {
627 pr_warn("Invalid DMAR haw\n");
631 pr_info("Host address width %d\n", dmar->width + 1);
632 ret = dmar_walk_dmar_table(dmar, &cb);
633 if (ret == 0 && drhd_count == 0)
634 pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
639 static int dmar_pci_device_match(struct dmar_dev_scope devices[],
640 int cnt, struct pci_dev *dev)
646 for_each_active_dev_scope(devices, cnt, index, tmp)
647 if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
650 /* Check our parent */
651 dev = dev->bus->self;
657 struct dmar_drhd_unit *
658 dmar_find_matched_drhd_unit(struct pci_dev *dev)
660 struct dmar_drhd_unit *dmaru;
661 struct acpi_dmar_hardware_unit *drhd;
663 dev = pci_physfn(dev);
666 for_each_drhd_unit(dmaru) {
667 drhd = container_of(dmaru->hdr,
668 struct acpi_dmar_hardware_unit,
671 if (dmaru->include_all &&
672 drhd->segment == pci_domain_nr(dev->bus))
675 if (dmar_pci_device_match(dmaru->devices,
676 dmaru->devices_cnt, dev))
686 static void __init dmar_acpi_insert_dev_scope(u8 device_number,
687 struct acpi_device *adev)
689 struct dmar_drhd_unit *dmaru;
690 struct acpi_dmar_hardware_unit *drhd;
691 struct acpi_dmar_device_scope *scope;
694 struct acpi_dmar_pci_path *path;
696 for_each_drhd_unit(dmaru) {
697 drhd = container_of(dmaru->hdr,
698 struct acpi_dmar_hardware_unit,
701 for (scope = (void *)(drhd + 1);
702 (unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
703 scope = ((void *)scope) + scope->length) {
704 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_NAMESPACE)
706 if (scope->enumeration_id != device_number)
709 path = (void *)(scope + 1);
710 pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
711 dev_name(&adev->dev), dmaru->reg_base_addr,
712 scope->bus, path->device, path->function);
713 for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
715 dmaru->devices[i].bus = scope->bus;
716 dmaru->devices[i].devfn = PCI_DEVFN(path->device,
718 rcu_assign_pointer(dmaru->devices[i].dev,
719 get_device(&adev->dev));
722 BUG_ON(i >= dmaru->devices_cnt);
725 pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
726 device_number, dev_name(&adev->dev));
729 static int __init dmar_acpi_dev_scope_init(void)
731 struct acpi_dmar_andd *andd;
733 if (dmar_tbl == NULL)
736 for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
737 ((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
738 andd = ((void *)andd) + andd->header.length) {
739 if (andd->header.type == ACPI_DMAR_TYPE_NAMESPACE) {
741 struct acpi_device *adev;
743 if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
746 pr_err("Failed to find handle for ACPI object %s\n",
750 if (acpi_bus_get_device(h, &adev)) {
751 pr_err("Failed to get device for ACPI object %s\n",
755 dmar_acpi_insert_dev_scope(andd->device_number, adev);
761 int __init dmar_dev_scope_init(void)
763 struct pci_dev *dev = NULL;
764 struct dmar_pci_notify_info *info;
766 if (dmar_dev_scope_status != 1)
767 return dmar_dev_scope_status;
769 if (list_empty(&dmar_drhd_units)) {
770 dmar_dev_scope_status = -ENODEV;
772 dmar_dev_scope_status = 0;
774 dmar_acpi_dev_scope_init();
776 for_each_pci_dev(dev) {
780 info = dmar_alloc_pci_notify_info(dev,
781 BUS_NOTIFY_ADD_DEVICE);
783 return dmar_dev_scope_status;
785 dmar_pci_bus_add_dev(info);
786 dmar_free_pci_notify_info(info);
790 bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb);
793 return dmar_dev_scope_status;
797 int __init dmar_table_init(void)
799 static int dmar_table_initialized;
802 if (dmar_table_initialized == 0) {
803 ret = parse_dmar_table();
806 pr_info("Parse DMAR table failure.\n");
807 } else if (list_empty(&dmar_drhd_units)) {
808 pr_info("No DMAR devices found\n");
813 dmar_table_initialized = ret;
815 dmar_table_initialized = 1;
818 return dmar_table_initialized < 0 ? dmar_table_initialized : 0;
821 static void warn_invalid_dmar(u64 addr, const char *message)
824 1, TAINT_FIRMWARE_WORKAROUND,
825 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
826 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
828 dmi_get_system_info(DMI_BIOS_VENDOR),
829 dmi_get_system_info(DMI_BIOS_VERSION),
830 dmi_get_system_info(DMI_PRODUCT_VERSION));
834 dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg)
836 struct acpi_dmar_hardware_unit *drhd;
840 drhd = (void *)entry;
841 if (!drhd->address) {
842 warn_invalid_dmar(0, "");
847 addr = ioremap(drhd->address, VTD_PAGE_SIZE);
849 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
851 pr_warn("Can't validate DRHD address: %llx\n", drhd->address);
855 cap = dmar_readq(addr + DMAR_CAP_REG);
856 ecap = dmar_readq(addr + DMAR_ECAP_REG);
861 early_iounmap(addr, VTD_PAGE_SIZE);
863 if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
864 warn_invalid_dmar(drhd->address, " returns all ones");
871 int __init detect_intel_iommu(void)
874 struct dmar_res_callback validate_drhd_cb = {
875 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_validate_one_drhd,
876 .ignore_unhandled = true,
879 down_write(&dmar_global_lock);
880 ret = dmar_table_detect();
882 ret = !dmar_walk_dmar_table((struct acpi_table_dmar *)dmar_tbl,
884 if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
886 /* Make sure ACS will be enabled */
892 x86_init.iommu.iommu_init = intel_iommu_init;
895 early_acpi_os_unmap_memory((void __iomem *)dmar_tbl, dmar_tbl_size);
897 up_write(&dmar_global_lock);
899 return ret ? 1 : -ENODEV;
903 static void unmap_iommu(struct intel_iommu *iommu)
906 release_mem_region(iommu->reg_phys, iommu->reg_size);
910 * map_iommu: map the iommu's registers
911 * @iommu: the iommu to map
912 * @phys_addr: the physical address of the base resgister
914 * Memory map the iommu's registers. Start w/ a single page, and
915 * possibly expand if that turns out to be insufficent.
917 static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
921 iommu->reg_phys = phys_addr;
922 iommu->reg_size = VTD_PAGE_SIZE;
924 if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
925 pr_err("Can't reserve memory\n");
930 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
932 pr_err("Can't map the region\n");
937 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
938 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
940 if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
942 warn_invalid_dmar(phys_addr, " returns all ones");
946 /* the registers might be more than one page */
947 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
948 cap_max_fault_reg_offset(iommu->cap));
949 map_size = VTD_PAGE_ALIGN(map_size);
950 if (map_size > iommu->reg_size) {
952 release_mem_region(iommu->reg_phys, iommu->reg_size);
953 iommu->reg_size = map_size;
954 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
956 pr_err("Can't reserve memory\n");
960 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
962 pr_err("Can't map the region\n");
973 release_mem_region(iommu->reg_phys, iommu->reg_size);
978 static int dmar_alloc_seq_id(struct intel_iommu *iommu)
980 iommu->seq_id = find_first_zero_bit(dmar_seq_ids,
981 DMAR_UNITS_SUPPORTED);
982 if (iommu->seq_id >= DMAR_UNITS_SUPPORTED) {
985 set_bit(iommu->seq_id, dmar_seq_ids);
986 sprintf(iommu->name, "dmar%d", iommu->seq_id);
989 return iommu->seq_id;
992 static void dmar_free_seq_id(struct intel_iommu *iommu)
994 if (iommu->seq_id >= 0) {
995 clear_bit(iommu->seq_id, dmar_seq_ids);
1000 static int alloc_iommu(struct dmar_drhd_unit *drhd)
1002 struct intel_iommu *iommu;
1008 if (!drhd->reg_base_addr) {
1009 warn_invalid_dmar(0, "");
1013 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
1017 if (dmar_alloc_seq_id(iommu) < 0) {
1018 pr_err("Failed to allocate seq_id\n");
1023 err = map_iommu(iommu, drhd->reg_base_addr);
1025 pr_err("Failed to map %s\n", iommu->name);
1026 goto error_free_seq_id;
1030 agaw = iommu_calculate_agaw(iommu);
1032 pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
1036 msagaw = iommu_calculate_max_sagaw(iommu);
1038 pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
1043 iommu->msagaw = msagaw;
1044 iommu->segment = drhd->segment;
1048 ver = readl(iommu->reg + DMAR_VER_REG);
1049 pr_info("%s: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
1051 (unsigned long long)drhd->reg_base_addr,
1052 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
1053 (unsigned long long)iommu->cap,
1054 (unsigned long long)iommu->ecap);
1056 /* Reflect status in gcmd */
1057 sts = readl(iommu->reg + DMAR_GSTS_REG);
1058 if (sts & DMA_GSTS_IRES)
1059 iommu->gcmd |= DMA_GCMD_IRE;
1060 if (sts & DMA_GSTS_TES)
1061 iommu->gcmd |= DMA_GCMD_TE;
1062 if (sts & DMA_GSTS_QIES)
1063 iommu->gcmd |= DMA_GCMD_QIE;
1065 raw_spin_lock_init(&iommu->register_lock);
1067 drhd->iommu = iommu;
1069 if (intel_iommu_enabled)
1070 iommu->iommu_dev = iommu_device_create(NULL, iommu,
1079 dmar_free_seq_id(iommu);
1085 static void free_iommu(struct intel_iommu *iommu)
1087 iommu_device_destroy(iommu->iommu_dev);
1090 if (iommu->pr_irq) {
1091 free_irq(iommu->pr_irq, iommu);
1092 dmar_free_hwirq(iommu->pr_irq);
1095 free_irq(iommu->irq, iommu);
1096 dmar_free_hwirq(iommu->irq);
1101 free_page((unsigned long)iommu->qi->desc);
1102 kfree(iommu->qi->desc_status);
1109 dmar_free_seq_id(iommu);
1114 * Reclaim all the submitted descriptors which have completed its work.
1116 static inline void reclaim_free_desc(struct q_inval *qi)
1118 while (qi->desc_status[qi->free_tail] == QI_DONE ||
1119 qi->desc_status[qi->free_tail] == QI_ABORT) {
1120 qi->desc_status[qi->free_tail] = QI_FREE;
1121 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
1126 static int qi_check_fault(struct intel_iommu *iommu, int index)
1130 struct q_inval *qi = iommu->qi;
1131 int wait_index = (index + 1) % QI_LENGTH;
1133 if (qi->desc_status[wait_index] == QI_ABORT)
1136 fault = readl(iommu->reg + DMAR_FSTS_REG);
1139 * If IQE happens, the head points to the descriptor associated
1140 * with the error. No new descriptors are fetched until the IQE
1143 if (fault & DMA_FSTS_IQE) {
1144 head = readl(iommu->reg + DMAR_IQH_REG);
1145 if ((head >> DMAR_IQ_SHIFT) == index) {
1146 pr_err("VT-d detected invalid descriptor: "
1147 "low=%llx, high=%llx\n",
1148 (unsigned long long)qi->desc[index].low,
1149 (unsigned long long)qi->desc[index].high);
1150 memcpy(&qi->desc[index], &qi->desc[wait_index],
1151 sizeof(struct qi_desc));
1152 __iommu_flush_cache(iommu, &qi->desc[index],
1153 sizeof(struct qi_desc));
1154 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
1160 * If ITE happens, all pending wait_desc commands are aborted.
1161 * No new descriptors are fetched until the ITE is cleared.
1163 if (fault & DMA_FSTS_ITE) {
1164 head = readl(iommu->reg + DMAR_IQH_REG);
1165 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1167 tail = readl(iommu->reg + DMAR_IQT_REG);
1168 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1170 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
1173 if (qi->desc_status[head] == QI_IN_USE)
1174 qi->desc_status[head] = QI_ABORT;
1175 head = (head - 2 + QI_LENGTH) % QI_LENGTH;
1176 } while (head != tail);
1178 if (qi->desc_status[wait_index] == QI_ABORT)
1182 if (fault & DMA_FSTS_ICE)
1183 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
1189 * Submit the queued invalidation descriptor to the remapping
1190 * hardware unit and wait for its completion.
1192 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
1195 struct q_inval *qi = iommu->qi;
1196 struct qi_desc *hw, wait_desc;
1197 int wait_index, index;
1198 unsigned long flags;
1208 raw_spin_lock_irqsave(&qi->q_lock, flags);
1209 while (qi->free_cnt < 3) {
1210 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1212 raw_spin_lock_irqsave(&qi->q_lock, flags);
1215 index = qi->free_head;
1216 wait_index = (index + 1) % QI_LENGTH;
1218 qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
1222 wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
1223 QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
1224 wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
1226 hw[wait_index] = wait_desc;
1228 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
1229 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
1231 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
1235 * update the HW tail register indicating the presence of
1238 writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
1240 while (qi->desc_status[wait_index] != QI_DONE) {
1242 * We will leave the interrupts disabled, to prevent interrupt
1243 * context to queue another cmd while a cmd is already submitted
1244 * and waiting for completion on this cpu. This is to avoid
1245 * a deadlock where the interrupt context can wait indefinitely
1246 * for free slots in the queue.
1248 rc = qi_check_fault(iommu, index);
1252 raw_spin_unlock(&qi->q_lock);
1254 raw_spin_lock(&qi->q_lock);
1257 qi->desc_status[index] = QI_DONE;
1259 reclaim_free_desc(qi);
1260 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1269 * Flush the global interrupt entry cache.
1271 void qi_global_iec(struct intel_iommu *iommu)
1273 struct qi_desc desc;
1275 desc.low = QI_IEC_TYPE;
1278 /* should never fail */
1279 qi_submit_sync(&desc, iommu);
1282 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1285 struct qi_desc desc;
1287 desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1288 | QI_CC_GRAN(type) | QI_CC_TYPE;
1291 qi_submit_sync(&desc, iommu);
1294 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1295 unsigned int size_order, u64 type)
1299 struct qi_desc desc;
1302 if (cap_write_drain(iommu->cap))
1305 if (cap_read_drain(iommu->cap))
1308 desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1309 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1310 desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1311 | QI_IOTLB_AM(size_order);
1313 qi_submit_sync(&desc, iommu);
1316 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
1317 u64 addr, unsigned mask)
1319 struct qi_desc desc;
1322 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
1323 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1324 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1326 desc.high = QI_DEV_IOTLB_ADDR(addr);
1328 if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1331 desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1334 qi_submit_sync(&desc, iommu);
1338 * Disable Queued Invalidation interface.
1340 void dmar_disable_qi(struct intel_iommu *iommu)
1342 unsigned long flags;
1344 cycles_t start_time = get_cycles();
1346 if (!ecap_qis(iommu->ecap))
1349 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1351 sts = readl(iommu->reg + DMAR_GSTS_REG);
1352 if (!(sts & DMA_GSTS_QIES))
1356 * Give a chance to HW to complete the pending invalidation requests.
1358 while ((readl(iommu->reg + DMAR_IQT_REG) !=
1359 readl(iommu->reg + DMAR_IQH_REG)) &&
1360 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1363 iommu->gcmd &= ~DMA_GCMD_QIE;
1364 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1366 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1367 !(sts & DMA_GSTS_QIES), sts);
1369 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1373 * Enable queued invalidation.
1375 static void __dmar_enable_qi(struct intel_iommu *iommu)
1378 unsigned long flags;
1379 struct q_inval *qi = iommu->qi;
1381 qi->free_head = qi->free_tail = 0;
1382 qi->free_cnt = QI_LENGTH;
1384 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1386 /* write zero to the tail reg */
1387 writel(0, iommu->reg + DMAR_IQT_REG);
1389 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1391 iommu->gcmd |= DMA_GCMD_QIE;
1392 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1394 /* Make sure hardware complete it */
1395 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1397 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1401 * Enable Queued Invalidation interface. This is a must to support
1402 * interrupt-remapping. Also used by DMA-remapping, which replaces
1403 * register based IOTLB invalidation.
1405 int dmar_enable_qi(struct intel_iommu *iommu)
1408 struct page *desc_page;
1410 if (!ecap_qis(iommu->ecap))
1414 * queued invalidation is already setup and enabled.
1419 iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1426 desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1433 qi->desc = page_address(desc_page);
1435 qi->desc_status = kzalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1436 if (!qi->desc_status) {
1437 free_page((unsigned long) qi->desc);
1443 raw_spin_lock_init(&qi->q_lock);
1445 __dmar_enable_qi(iommu);
1450 /* iommu interrupt handling. Most stuff are MSI-like. */
1458 static const char *dma_remap_fault_reasons[] =
1461 "Present bit in root entry is clear",
1462 "Present bit in context entry is clear",
1463 "Invalid context entry",
1464 "Access beyond MGAW",
1465 "PTE Write access is not set",
1466 "PTE Read access is not set",
1467 "Next page table ptr is invalid",
1468 "Root table address invalid",
1469 "Context table ptr is invalid",
1470 "non-zero reserved fields in RTP",
1471 "non-zero reserved fields in CTP",
1472 "non-zero reserved fields in PTE",
1473 "PCE for translation request specifies blocking",
1476 static const char *irq_remap_fault_reasons[] =
1478 "Detected reserved fields in the decoded interrupt-remapped request",
1479 "Interrupt index exceeded the interrupt-remapping table size",
1480 "Present field in the IRTE entry is clear",
1481 "Error accessing interrupt-remapping table pointed by IRTA_REG",
1482 "Detected reserved fields in the IRTE entry",
1483 "Blocked a compatibility format interrupt request",
1484 "Blocked an interrupt request due to source-id verification failure",
1487 static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1489 if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1490 ARRAY_SIZE(irq_remap_fault_reasons))) {
1491 *fault_type = INTR_REMAP;
1492 return irq_remap_fault_reasons[fault_reason - 0x20];
1493 } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1494 *fault_type = DMA_REMAP;
1495 return dma_remap_fault_reasons[fault_reason];
1497 *fault_type = UNKNOWN;
1503 static inline int dmar_msi_reg(struct intel_iommu *iommu, int irq)
1505 if (iommu->irq == irq)
1506 return DMAR_FECTL_REG;
1507 else if (iommu->pr_irq == irq)
1508 return DMAR_PECTL_REG;
1513 void dmar_msi_unmask(struct irq_data *data)
1515 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1516 int reg = dmar_msi_reg(iommu, data->irq);
1520 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1521 writel(0, iommu->reg + reg);
1522 /* Read a reg to force flush the post write */
1523 readl(iommu->reg + reg);
1524 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1527 void dmar_msi_mask(struct irq_data *data)
1529 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1530 int reg = dmar_msi_reg(iommu, data->irq);
1534 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1535 writel(DMA_FECTL_IM, iommu->reg + reg);
1536 /* Read a reg to force flush the post write */
1537 readl(iommu->reg + reg);
1538 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1541 void dmar_msi_write(int irq, struct msi_msg *msg)
1543 struct intel_iommu *iommu = irq_get_handler_data(irq);
1544 int reg = dmar_msi_reg(iommu, irq);
1547 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1548 writel(msg->data, iommu->reg + reg + 4);
1549 writel(msg->address_lo, iommu->reg + reg + 8);
1550 writel(msg->address_hi, iommu->reg + reg + 12);
1551 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1554 void dmar_msi_read(int irq, struct msi_msg *msg)
1556 struct intel_iommu *iommu = irq_get_handler_data(irq);
1557 int reg = dmar_msi_reg(iommu, irq);
1560 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1561 msg->data = readl(iommu->reg + reg + 4);
1562 msg->address_lo = readl(iommu->reg + reg + 8);
1563 msg->address_hi = readl(iommu->reg + reg + 12);
1564 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1567 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1568 u8 fault_reason, u16 source_id, unsigned long long addr)
1573 reason = dmar_get_fault_reason(fault_reason, &fault_type);
1575 if (fault_type == INTR_REMAP)
1576 pr_err("INTR-REMAP: Request device [[%02x:%02x.%d] "
1577 "fault index %llx\n"
1578 "INTR-REMAP:[fault reason %02d] %s\n",
1579 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1580 PCI_FUNC(source_id & 0xFF), addr >> 48,
1581 fault_reason, reason);
1583 pr_err("DMAR:[%s] Request device [%02x:%02x.%d] "
1584 "fault addr %llx \n"
1585 "DMAR:[fault reason %02d] %s\n",
1586 (type ? "DMA Read" : "DMA Write"),
1587 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1588 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1592 #define PRIMARY_FAULT_REG_LEN (16)
1593 irqreturn_t dmar_fault(int irq, void *dev_id)
1595 struct intel_iommu *iommu = dev_id;
1596 int reg, fault_index;
1600 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1601 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1603 pr_err("DRHD: handling fault status reg %x\n", fault_status);
1605 /* TBD: ignore advanced fault log currently */
1606 if (!(fault_status & DMA_FSTS_PPF))
1609 fault_index = dma_fsts_fault_record_index(fault_status);
1610 reg = cap_fault_reg_offset(iommu->cap);
1618 /* highest 32 bits */
1619 data = readl(iommu->reg + reg +
1620 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1621 if (!(data & DMA_FRCD_F))
1624 fault_reason = dma_frcd_fault_reason(data);
1625 type = dma_frcd_type(data);
1627 data = readl(iommu->reg + reg +
1628 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1629 source_id = dma_frcd_source_id(data);
1631 guest_addr = dmar_readq(iommu->reg + reg +
1632 fault_index * PRIMARY_FAULT_REG_LEN);
1633 guest_addr = dma_frcd_page_addr(guest_addr);
1634 /* clear the fault */
1635 writel(DMA_FRCD_F, iommu->reg + reg +
1636 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1638 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1640 dmar_fault_do_one(iommu, type, fault_reason,
1641 source_id, guest_addr);
1644 if (fault_index >= cap_num_fault_regs(iommu->cap))
1646 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1649 writel(DMA_FSTS_PFO | DMA_FSTS_PPF, iommu->reg + DMAR_FSTS_REG);
1652 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1656 int dmar_set_interrupt(struct intel_iommu *iommu)
1661 * Check if the fault interrupt is already initialized.
1666 irq = dmar_alloc_hwirq(iommu->seq_id, iommu->node, iommu);
1670 pr_err("No free IRQ vectors\n");
1674 ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1676 pr_err("Can't request irq\n");
1680 int __init enable_drhd_fault_handling(void)
1682 struct dmar_drhd_unit *drhd;
1683 struct intel_iommu *iommu;
1686 * Enable fault control interrupt.
1688 for_each_iommu(iommu, drhd) {
1690 int ret = dmar_set_interrupt(iommu);
1693 pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
1694 (unsigned long long)drhd->reg_base_addr, ret);
1699 * Clear any previous faults.
1701 dmar_fault(iommu->irq, iommu);
1702 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1703 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1710 * Re-enable Queued Invalidation interface.
1712 int dmar_reenable_qi(struct intel_iommu *iommu)
1714 if (!ecap_qis(iommu->ecap))
1721 * First disable queued invalidation.
1723 dmar_disable_qi(iommu);
1725 * Then enable queued invalidation again. Since there is no pending
1726 * invalidation requests now, it's safe to re-enable queued
1729 __dmar_enable_qi(iommu);
1735 * Check interrupt remapping support in DMAR table description.
1737 int __init dmar_ir_support(void)
1739 struct acpi_table_dmar *dmar;
1740 dmar = (struct acpi_table_dmar *)dmar_tbl;
1743 return dmar->flags & 0x1;
1746 /* Check whether DMAR units are in use */
1747 static inline bool dmar_in_use(void)
1749 return irq_remapping_enabled || intel_iommu_enabled;
1752 static int __init dmar_free_unused_resources(void)
1754 struct dmar_drhd_unit *dmaru, *dmaru_n;
1759 if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units))
1760 bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb);
1762 down_write(&dmar_global_lock);
1763 list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
1764 list_del(&dmaru->list);
1765 dmar_free_drhd(dmaru);
1767 up_write(&dmar_global_lock);
1772 late_initcall(dmar_free_unused_resources);
1773 IOMMU_INIT_POST(detect_intel_iommu);
1776 * DMAR Hotplug Support
1777 * For more details, please refer to Intel(R) Virtualization Technology
1778 * for Directed-IO Architecture Specifiction, Rev 2.2, Section 8.8
1779 * "Remapping Hardware Unit Hot Plug".
1781 static u8 dmar_hp_uuid[] = {
1782 /* 0000 */ 0xA6, 0xA3, 0xC1, 0xD8, 0x9B, 0xBE, 0x9B, 0x4C,
1783 /* 0008 */ 0x91, 0xBF, 0xC3, 0xCB, 0x81, 0xFC, 0x5D, 0xAF
1787 * Currently there's only one revision and BIOS will not check the revision id,
1788 * so use 0 for safety.
1790 #define DMAR_DSM_REV_ID 0
1791 #define DMAR_DSM_FUNC_DRHD 1
1792 #define DMAR_DSM_FUNC_ATSR 2
1793 #define DMAR_DSM_FUNC_RHSA 3
1795 static inline bool dmar_detect_dsm(acpi_handle handle, int func)
1797 return acpi_check_dsm(handle, dmar_hp_uuid, DMAR_DSM_REV_ID, 1 << func);
1800 static int dmar_walk_dsm_resource(acpi_handle handle, int func,
1801 dmar_res_handler_t handler, void *arg)
1804 union acpi_object *obj;
1805 struct acpi_dmar_header *start;
1806 struct dmar_res_callback callback;
1807 static int res_type[] = {
1808 [DMAR_DSM_FUNC_DRHD] = ACPI_DMAR_TYPE_HARDWARE_UNIT,
1809 [DMAR_DSM_FUNC_ATSR] = ACPI_DMAR_TYPE_ROOT_ATS,
1810 [DMAR_DSM_FUNC_RHSA] = ACPI_DMAR_TYPE_HARDWARE_AFFINITY,
1813 if (!dmar_detect_dsm(handle, func))
1816 obj = acpi_evaluate_dsm_typed(handle, dmar_hp_uuid, DMAR_DSM_REV_ID,
1817 func, NULL, ACPI_TYPE_BUFFER);
1821 memset(&callback, 0, sizeof(callback));
1822 callback.cb[res_type[func]] = handler;
1823 callback.arg[res_type[func]] = arg;
1824 start = (struct acpi_dmar_header *)obj->buffer.pointer;
1825 ret = dmar_walk_remapping_entries(start, obj->buffer.length, &callback);
1832 static int dmar_hp_add_drhd(struct acpi_dmar_header *header, void *arg)
1835 struct dmar_drhd_unit *dmaru;
1837 dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1841 ret = dmar_ir_hotplug(dmaru, true);
1843 ret = dmar_iommu_hotplug(dmaru, true);
1848 static int dmar_hp_remove_drhd(struct acpi_dmar_header *header, void *arg)
1852 struct dmar_drhd_unit *dmaru;
1854 dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1859 * All PCI devices managed by this unit should have been destroyed.
1861 if (!dmaru->include_all && dmaru->devices && dmaru->devices_cnt)
1862 for_each_active_dev_scope(dmaru->devices,
1863 dmaru->devices_cnt, i, dev)
1866 ret = dmar_ir_hotplug(dmaru, false);
1868 ret = dmar_iommu_hotplug(dmaru, false);
1873 static int dmar_hp_release_drhd(struct acpi_dmar_header *header, void *arg)
1875 struct dmar_drhd_unit *dmaru;
1877 dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1879 list_del_rcu(&dmaru->list);
1881 dmar_free_drhd(dmaru);
1887 static int dmar_hotplug_insert(acpi_handle handle)
1892 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1893 &dmar_validate_one_drhd, (void *)1);
1897 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1898 &dmar_parse_one_drhd, (void *)&drhd_count);
1899 if (ret == 0 && drhd_count == 0) {
1900 pr_warn(FW_BUG "No DRHD structures in buffer returned by _DSM method\n");
1906 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_RHSA,
1907 &dmar_parse_one_rhsa, NULL);
1911 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1912 &dmar_parse_one_atsr, NULL);
1916 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1917 &dmar_hp_add_drhd, NULL);
1921 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1922 &dmar_hp_remove_drhd, NULL);
1924 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1925 &dmar_release_one_atsr, NULL);
1927 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1928 &dmar_hp_release_drhd, NULL);
1933 static int dmar_hotplug_remove(acpi_handle handle)
1937 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1938 &dmar_check_one_atsr, NULL);
1942 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1943 &dmar_hp_remove_drhd, NULL);
1945 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1946 &dmar_release_one_atsr, NULL));
1947 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1948 &dmar_hp_release_drhd, NULL));
1950 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1951 &dmar_hp_add_drhd, NULL);
1957 static acpi_status dmar_get_dsm_handle(acpi_handle handle, u32 lvl,
1958 void *context, void **retval)
1960 acpi_handle *phdl = retval;
1962 if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
1964 return AE_CTRL_TERMINATE;
1970 static int dmar_device_hotplug(acpi_handle handle, bool insert)
1973 acpi_handle tmp = NULL;
1979 if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
1982 status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle,
1984 dmar_get_dsm_handle,
1986 if (ACPI_FAILURE(status)) {
1987 pr_warn("Failed to locate _DSM method.\n");
1994 down_write(&dmar_global_lock);
1996 ret = dmar_hotplug_insert(tmp);
1998 ret = dmar_hotplug_remove(tmp);
1999 up_write(&dmar_global_lock);
2004 int dmar_device_add(acpi_handle handle)
2006 return dmar_device_hotplug(handle, true);
2009 int dmar_device_remove(acpi_handle handle)
2011 return dmar_device_hotplug(handle, false);