*
****************************************************************************/
-- /*
-- * Writes the command to the IOMMUs command buffer and informs the
-- * hardware about the new command. Must be called with iommu->lock held.
-- */
-- static int __iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
++ static int wait_on_sem(volatile u64 *sem)
++ {
++ int i = 0;
++
++ while (*sem == 0 && i < LOOP_TIMEOUT) {
++ udelay(1);
++ i += 1;
++ }
++
++ if (i == LOOP_TIMEOUT) {
++ pr_alert("AMD-Vi: Completion-Wait loop timed out\n");
++ return -EIO;
++ }
++
++ return 0;
++ }
++
++ static void copy_cmd_to_buffer(struct amd_iommu *iommu,
++ struct iommu_cmd *cmd,
++ u32 tail)
{
-- u32 tail, head;
u8 *target;
-- WARN_ON(iommu->cmd_buf_size & CMD_BUFFER_UNINITIALIZED);
-- tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
target = iommu->cmd_buf + tail;
-- memcpy_toio(target, cmd, sizeof(*cmd));
-- tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
-- head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
-- if (tail == head)
-- return -ENOMEM;
++ tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
++
++ /* Copy command to buffer */
++ memcpy(target, cmd, sizeof(*cmd));
++
++ /* Tell the IOMMU about it */
writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
++ }
-- return 0;
++ static void build_completion_wait(struct iommu_cmd *cmd, u64 address)
++ {
++ WARN_ON(address & 0x7ULL);
++
++ memset(cmd, 0, sizeof(*cmd));
++ cmd->data[0] = lower_32_bits(__pa(address)) | CMD_COMPL_WAIT_STORE_MASK;
++ cmd->data[1] = upper_32_bits(__pa(address));
++ cmd->data[2] = 1;
++ CMD_SET_TYPE(cmd, CMD_COMPL_WAIT);
++ }
++
++ static void build_inv_dte(struct iommu_cmd *cmd, u16 devid)
++ {
++ memset(cmd, 0, sizeof(*cmd));
++ cmd->data[0] = devid;
++ CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY);
++ }
++
++ static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
++ size_t size, u16 domid, int pde)
++ {
++ u64 pages;
++ int s;
++
++ pages = iommu_num_pages(address, size, PAGE_SIZE);
++ s = 0;
++
++ if (pages > 1) {
++ /*
++ * If we have to flush more than one page, flush all
++ * TLB entries for this domain
++ */
++ address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
++ s = 1;
++ }
++
++ address &= PAGE_MASK;
++
++ memset(cmd, 0, sizeof(*cmd));
++ cmd->data[1] |= domid;
++ cmd->data[2] = lower_32_bits(address);
++ cmd->data[3] = upper_32_bits(address);
++ CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
++ if (s) /* size bit - we flush more than one 4kb page */
++ cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
++ if (pde) /* PDE bit - we wan't flush everything not only the PTEs */
++ cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
++ }
++
++ +static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,
++ + u64 address, size_t size)
++ +{
++ + u64 pages;
++ + int s;
++ +
++ + pages = iommu_num_pages(address, size, PAGE_SIZE);
++ + s = 0;
++ +
++ + if (pages > 1) {
++ + /*
++ + * If we have to flush more than one page, flush all
++ + * TLB entries for this domain
++ + */
++ + address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
++ + s = 1;
++ + }
++ +
++ + address &= PAGE_MASK;
++ +
++ + memset(cmd, 0, sizeof(*cmd));
++ + cmd->data[0] = devid;
++ + cmd->data[0] |= (qdep & 0xff) << 24;
++ + cmd->data[1] = devid;
++ + cmd->data[2] = lower_32_bits(address);
++ + cmd->data[3] = upper_32_bits(address);
++ + CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
++ + if (s)
++ + cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
++ +}
++ +
+++ static void build_inv_all(struct iommu_cmd *cmd)
+++ {
+++ memset(cmd, 0, sizeof(*cmd));
+++ CMD_SET_TYPE(cmd, CMD_INV_ALL);
+ }
+
/*
-- * General queuing function for commands. Takes iommu->lock and calls
-- * __iommu_queue_command().
++ * Writes the command to the IOMMUs command buffer and informs the
++ * hardware about the new command.
*/
static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
{
* This function queues a completion wait command into the command
* buffer of an IOMMU
*/
-- static int __iommu_completion_wait(struct amd_iommu *iommu)
++ static int iommu_completion_wait(struct amd_iommu *iommu)
{
struct iommu_cmd cmd;
++ volatile u64 sem = 0;
++ int ret;
-- memset(&cmd, 0, sizeof(cmd));
-- cmd.data[0] = CMD_COMPL_WAIT_INT_MASK;
-- CMD_SET_TYPE(&cmd, CMD_COMPL_WAIT);
++ if (!iommu->need_sync)
++ return 0;
++
++ build_completion_wait(&cmd, (u64)&sem);
++
++ ret = iommu_queue_command(iommu, &cmd);
++ if (ret)
++ return ret;
-- return __iommu_queue_command(iommu, &cmd);
++ return wait_on_sem(&sem);
}
-- /*
-- * This function is called whenever we need to ensure that the IOMMU has
-- * completed execution of all commands we sent. It sends a
-- * COMPLETION_WAIT command and waits for it to finish. The IOMMU informs
-- * us about that by writing a value to a physical address we pass with
-- * the command.
-- */
-- static int iommu_completion_wait(struct amd_iommu *iommu)
++ static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)
{
-- int ret = 0;
-- unsigned long flags;
++ struct iommu_cmd cmd;
-- spin_lock_irqsave(&iommu->lock, flags);
++ build_inv_dte(&cmd, devid);
-- if (!iommu->need_sync)
-- goto out;
++ return iommu_queue_command(iommu, &cmd);
++ }
-- ret = __iommu_completion_wait(iommu);
++ static void iommu_flush_dte_all(struct amd_iommu *iommu)
++ {
++ u32 devid;
-- iommu->need_sync = false;
++ for (devid = 0; devid <= 0xffff; ++devid)
++ iommu_flush_dte(iommu, devid);
-- if (ret)
-- goto out;
--
-- __iommu_wait_for_completion(iommu);
++ iommu_completion_wait(iommu);
++ }
-- out:
-- spin_unlock_irqrestore(&iommu->lock, flags);
++ /*
++ * This function uses heavy locking and may disable irqs for some time. But
++ * this is no issue because it is only called during resume.
++ */
++ static void iommu_flush_tlb_all(struct amd_iommu *iommu)
++ {
++ u32 dom_id;
-- if (iommu->reset_in_progress)
-- reset_iommu_command_buffer(iommu);
++ for (dom_id = 0; dom_id <= 0xffff; ++dom_id) {
++ struct iommu_cmd cmd;
++ build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
++ dom_id, 1);
++ iommu_queue_command(iommu, &cmd);
++ }
-- return 0;
++ iommu_completion_wait(iommu);
}
-- static void iommu_flush_complete(struct protection_domain *domain)
+++ static void iommu_flush_all(struct amd_iommu *iommu)
+ {
-- int i;
+++ struct iommu_cmd cmd;
+
-- for (i = 0; i < amd_iommus_present; ++i) {
-- if (!domain->dev_iommu[i])
-- continue;
+++ build_inv_all(&cmd);
+
-- /*
-- * Devices of this domain are behind this IOMMU
-- * We need to wait for completion of all commands.
-- */
-- iommu_completion_wait(amd_iommus[i]);
+++ iommu_queue_command(iommu, &cmd);
+++ iommu_completion_wait(iommu);
+++ }
+++
++ void iommu_flush_all_caches(struct amd_iommu *iommu)
++ {
- iommu_flush_dte_all(iommu);
- iommu_flush_tlb_all(iommu);
+++ if (iommu_feature(iommu, FEATURE_IA)) {
+++ iommu_flush_all(iommu);
+++ } else {
+++ iommu_flush_dte_all(iommu);
+++ iommu_flush_tlb_all(iommu);
+ }
}
-- * Command send function for invalidating a device table entry
+/*
-- static int iommu_flush_device(struct device *dev)
++ + * Command send function for flushing on-device TLB
+ */
-- /* Build command */
-- memset(&cmd, 0, sizeof(cmd));
-- CMD_SET_TYPE(&cmd, CMD_INV_DEV_ENTRY);
-- cmd.data[0] = devid;
++ +static int device_flush_iotlb(struct device *dev, u64 address, size_t size)
+{
++ + struct pci_dev *pdev = to_pci_dev(dev);
+ struct amd_iommu *iommu;
+ struct iommu_cmd cmd;
+ u16 devid;
++ + int qdep;
+
++ + qdep = pci_ats_queue_depth(pdev);
+ devid = get_device_id(dev);
+ iommu = amd_iommu_rlookup_table[devid];
+
-- static void __iommu_build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
-- u16 domid, int pde, int s)
-- {
-- memset(cmd, 0, sizeof(*cmd));
-- address &= PAGE_MASK;
-- CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
-- cmd->data[1] |= domid;
-- cmd->data[2] = lower_32_bits(address);
-- cmd->data[3] = upper_32_bits(address);
-- if (s) /* size bit - we flush more than one 4kb page */
-- cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
-- if (pde) /* PDE bit - we wan't flush everything not only the PTEs */
-- cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
-- }
--
++ + build_inv_iotlb_pages(&cmd, devid, qdep, address, size);
+
+ return iommu_queue_command(iommu, &cmd);
+}
+
/*
-- * Generic command send function for invalidaing TLB entries
++ * Command send function for invalidating a device table entry
*/
-- static int iommu_queue_inv_iommu_pages(struct amd_iommu *iommu,
-- u64 address, u16 domid, int pde, int s)
++ static int device_flush_dte(struct device *dev)
{
-- struct iommu_cmd cmd;
++ struct amd_iommu *iommu;
++ + struct pci_dev *pdev;
++ u16 devid;
+ int ret;
-- __iommu_build_inv_iommu_pages(&cmd, address, domid, pde, s);
++ + pdev = to_pci_dev(dev);
++ devid = get_device_id(dev);
++ iommu = amd_iommu_rlookup_table[devid];
-- ret = iommu_queue_command(iommu, &cmd);
- return iommu_flush_dte(iommu, devid);
++ + ret = iommu_flush_dte(iommu, devid);
++ + if (ret)
++ + return ret;
++ +
++ + if (pci_ats_enabled(pdev))
++ + ret = device_flush_iotlb(dev, 0, ~0UL);
+
+ return ret;
}
/*
* It invalidates a single PTE if the range to flush is within a single
* page. Otherwise it flushes the whole TLB of the IOMMU.
*/
-- static void __iommu_flush_pages(struct protection_domain *domain,
-- u64 address, size_t size, int pde)
++ static void __domain_flush_pages(struct protection_domain *domain,
++ u64 address, size_t size, int pde)
{
-- int s = 0, i;
-- unsigned long pages = iommu_num_pages(address, size, PAGE_SIZE);
--
-- address &= PAGE_MASK;
--
-- if (pages > 1) {
-- /*
-- * If we have to flush more than one page, flush all
-- * TLB entries for this domain
-- */
-- address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
-- s = 1;
-- }
++ + struct iommu_dev_data *dev_data;
++ struct iommu_cmd cmd;
++ int ret = 0, i;
++ build_inv_iommu_pages(&cmd, address, size, domain->id, pde);
for (i = 0; i < amd_iommus_present; ++i) {
if (!domain->dev_iommu[i])
* Devices of this domain are behind this IOMMU
* We need a TLB flush
*/
-- iommu_queue_inv_iommu_pages(amd_iommus[i], address,
-- domain->id, pde, s);
++ ret |= iommu_queue_command(amd_iommus[i], &cmd);
}
-- return;
++ + list_for_each_entry(dev_data, &domain->dev_list, list) {
++ + struct pci_dev *pdev = to_pci_dev(dev_data->dev);
++ +
++ + if (!pci_ats_enabled(pdev))
++ + continue;
++ +
++ + ret |= device_flush_iotlb(dev_data->dev, address, size);
++ + }
++ +
++ WARN_ON(ret);
}
-- static void iommu_flush_pages(struct protection_domain *domain,
-- u64 address, size_t size)
++ static void domain_flush_pages(struct protection_domain *domain,
++ u64 address, size_t size)
{
-- __iommu_flush_pages(domain, address, size, 0);
++ __domain_flush_pages(domain, address, size, 0);
}
/* Flush the whole IO/TLB for a given protection domain */
projects / firefly-linux-kernel-4.4.55.git / commitdiff