2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
7 * Copyright(c) 2012 Intel Corporation. All rights reserved.
8 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of version 2 of the GNU General Public License as
12 * published by the Free Software Foundation.
16 * Copyright(c) 2012 Intel Corporation. All rights reserved.
17 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
19 * Redistribution and use in source and binary forms, with or without
20 * modification, are permitted provided that the following conditions
23 * * Redistributions of source code must retain the above copyright
24 * notice, this list of conditions and the following disclaimer.
25 * * Redistributions in binary form must reproduce the above copy
26 * notice, this list of conditions and the following disclaimer in
27 * the documentation and/or other materials provided with the
29 * * Neither the name of Intel Corporation nor the names of its
30 * contributors may be used to endorse or promote products derived
31 * from this software without specific prior written permission.
33 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
34 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
35 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
36 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
37 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
38 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
39 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
40 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
41 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
42 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
43 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45 * Intel PCIe NTB Linux driver
47 * Contact Information:
48 * Jon Mason <jon.mason@intel.com>
51 #include <linux/debugfs.h>
52 #include <linux/delay.h>
53 #include <linux/init.h>
54 #include <linux/interrupt.h>
55 #include <linux/module.h>
56 #include <linux/pci.h>
57 #include <linux/random.h>
58 #include <linux/slab.h>
59 #include <linux/ntb.h>
61 #include "ntb_hw_intel.h"
63 #define NTB_NAME "ntb_hw_intel"
64 #define NTB_DESC "Intel(R) PCI-E Non-Transparent Bridge Driver"
67 MODULE_DESCRIPTION(NTB_DESC);
68 MODULE_VERSION(NTB_VER);
69 MODULE_LICENSE("Dual BSD/GPL");
70 MODULE_AUTHOR("Intel Corporation");
72 #define bar0_off(base, bar) ((base) + ((bar) << 2))
73 #define bar2_off(base, bar) bar0_off(base, (bar) - 2)
75 static const struct intel_ntb_reg atom_reg;
76 static const struct intel_ntb_alt_reg atom_pri_reg;
77 static const struct intel_ntb_alt_reg atom_sec_reg;
78 static const struct intel_ntb_alt_reg atom_b2b_reg;
79 static const struct intel_ntb_xlat_reg atom_pri_xlat;
80 static const struct intel_ntb_xlat_reg atom_sec_xlat;
81 static const struct intel_ntb_reg xeon_reg;
82 static const struct intel_ntb_alt_reg xeon_pri_reg;
83 static const struct intel_ntb_alt_reg xeon_sec_reg;
84 static const struct intel_ntb_alt_reg xeon_b2b_reg;
85 static const struct intel_ntb_xlat_reg xeon_pri_xlat;
86 static const struct intel_ntb_xlat_reg xeon_sec_xlat;
87 static struct intel_b2b_addr xeon_b2b_usd_addr;
88 static struct intel_b2b_addr xeon_b2b_dsd_addr;
89 static const struct ntb_dev_ops intel_ntb_ops;
91 static const struct file_operations intel_ntb_debugfs_info;
92 static struct dentry *debugfs_dir;
94 static int b2b_mw_idx = -1;
95 module_param(b2b_mw_idx, int, 0644);
96 MODULE_PARM_DESC(b2b_mw_idx, "Use this mw idx to access the peer ntb. A "
97 "value of zero or positive starts from first mw idx, and a "
98 "negative value starts from last mw idx. Both sides MUST "
99 "set the same value here!");
101 static unsigned int b2b_mw_share;
102 module_param(b2b_mw_share, uint, 0644);
103 MODULE_PARM_DESC(b2b_mw_share, "If the b2b mw is large enough, configure the "
104 "ntb so that the peer ntb only occupies the first half of "
105 "the mw, so the second half can still be used as a mw. Both "
106 "sides MUST set the same value here!");
108 module_param_named(xeon_b2b_usd_bar2_addr64,
109 xeon_b2b_usd_addr.bar2_addr64, ullong, 0644);
110 MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
111 "XEON B2B USD BAR 2 64-bit address");
113 module_param_named(xeon_b2b_usd_bar4_addr64,
114 xeon_b2b_usd_addr.bar4_addr64, ullong, 0644);
115 MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
116 "XEON B2B USD BAR 4 64-bit address");
118 module_param_named(xeon_b2b_usd_bar4_addr32,
119 xeon_b2b_usd_addr.bar4_addr32, ullong, 0644);
120 MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
121 "XEON B2B USD split-BAR 4 32-bit address");
123 module_param_named(xeon_b2b_usd_bar5_addr32,
124 xeon_b2b_usd_addr.bar5_addr32, ullong, 0644);
125 MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
126 "XEON B2B USD split-BAR 5 32-bit address");
128 module_param_named(xeon_b2b_dsd_bar2_addr64,
129 xeon_b2b_dsd_addr.bar2_addr64, ullong, 0644);
130 MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
131 "XEON B2B DSD BAR 2 64-bit address");
133 module_param_named(xeon_b2b_dsd_bar4_addr64,
134 xeon_b2b_dsd_addr.bar4_addr64, ullong, 0644);
135 MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
136 "XEON B2B DSD BAR 4 64-bit address");
138 module_param_named(xeon_b2b_dsd_bar4_addr32,
139 xeon_b2b_dsd_addr.bar4_addr32, ullong, 0644);
140 MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
141 "XEON B2B DSD split-BAR 4 32-bit address");
143 module_param_named(xeon_b2b_dsd_bar5_addr32,
144 xeon_b2b_dsd_addr.bar5_addr32, ullong, 0644);
145 MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
146 "XEON B2B DSD split-BAR 5 32-bit address");
150 #define ioread64 readq
152 #define ioread64 _ioread64
153 static inline u64 _ioread64(void __iomem *mmio)
157 low = ioread32(mmio);
158 high = ioread32(mmio + sizeof(u32));
159 return low | (high << 32);
166 #define iowrite64 writeq
168 #define iowrite64 _iowrite64
169 static inline void _iowrite64(u64 val, void __iomem *mmio)
171 iowrite32(val, mmio);
172 iowrite32(val >> 32, mmio + sizeof(u32));
177 static inline int pdev_is_atom(struct pci_dev *pdev)
179 switch (pdev->device) {
180 case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD:
186 static inline int pdev_is_xeon(struct pci_dev *pdev)
188 switch (pdev->device) {
189 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
190 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
191 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
192 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
193 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
194 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
195 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
196 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
197 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
198 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
199 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
200 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
206 static inline void ndev_reset_unsafe_flags(struct intel_ntb_dev *ndev)
208 ndev->unsafe_flags = 0;
209 ndev->unsafe_flags_ignore = 0;
211 /* Only B2B has a workaround to avoid SDOORBELL */
212 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP)
213 if (!ntb_topo_is_b2b(ndev->ntb.topo))
214 ndev->unsafe_flags |= NTB_UNSAFE_DB;
216 /* No low level workaround to avoid SB01BASE */
217 if (ndev->hwerr_flags & NTB_HWERR_SB01BASE_LOCKUP) {
218 ndev->unsafe_flags |= NTB_UNSAFE_DB;
219 ndev->unsafe_flags |= NTB_UNSAFE_SPAD;
223 static inline int ndev_is_unsafe(struct intel_ntb_dev *ndev,
226 return !!(flag & ndev->unsafe_flags & ~ndev->unsafe_flags_ignore);
229 static inline int ndev_ignore_unsafe(struct intel_ntb_dev *ndev,
232 flag &= ndev->unsafe_flags;
233 ndev->unsafe_flags_ignore |= flag;
238 static int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx)
240 if (idx < 0 || idx > ndev->mw_count)
242 return ndev->reg->mw_bar[idx];
245 static inline int ndev_db_addr(struct intel_ntb_dev *ndev,
246 phys_addr_t *db_addr, resource_size_t *db_size,
247 phys_addr_t reg_addr, unsigned long reg)
249 WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));
252 *db_addr = reg_addr + reg;
253 dev_dbg(ndev_dev(ndev), "Peer db addr %llx\n", *db_addr);
257 *db_size = ndev->reg->db_size;
258 dev_dbg(ndev_dev(ndev), "Peer db size %llx\n", *db_size);
264 static inline u64 ndev_db_read(struct intel_ntb_dev *ndev,
267 WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));
269 return ndev->reg->db_ioread(mmio);
272 static inline int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits,
275 WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));
277 if (db_bits & ~ndev->db_valid_mask)
280 ndev->reg->db_iowrite(db_bits, mmio);
285 static inline int ndev_db_set_mask(struct intel_ntb_dev *ndev, u64 db_bits,
288 unsigned long irqflags;
290 WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));
292 if (db_bits & ~ndev->db_valid_mask)
295 spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
297 ndev->db_mask |= db_bits;
298 ndev->reg->db_iowrite(ndev->db_mask, mmio);
300 spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
305 static inline int ndev_db_clear_mask(struct intel_ntb_dev *ndev, u64 db_bits,
308 unsigned long irqflags;
310 WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));
312 if (db_bits & ~ndev->db_valid_mask)
315 spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
317 ndev->db_mask &= ~db_bits;
318 ndev->reg->db_iowrite(ndev->db_mask, mmio);
320 spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
325 static inline int ndev_vec_mask(struct intel_ntb_dev *ndev, int db_vector)
329 shift = ndev->db_vec_shift;
330 mask = BIT_ULL(shift) - 1;
332 return mask << (shift * db_vector);
335 static inline int ndev_spad_addr(struct intel_ntb_dev *ndev, int idx,
336 phys_addr_t *spad_addr, phys_addr_t reg_addr,
339 WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD));
341 if (idx < 0 || idx >= ndev->spad_count)
345 *spad_addr = reg_addr + reg + (idx << 2);
346 dev_dbg(ndev_dev(ndev), "Peer spad addr %llx\n", *spad_addr);
352 static inline u32 ndev_spad_read(struct intel_ntb_dev *ndev, int idx,
355 WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD));
357 if (idx < 0 || idx >= ndev->spad_count)
360 return ioread32(mmio + (idx << 2));
363 static inline int ndev_spad_write(struct intel_ntb_dev *ndev, int idx, u32 val,
366 WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD));
368 if (idx < 0 || idx >= ndev->spad_count)
371 iowrite32(val, mmio + (idx << 2));
376 static irqreturn_t ndev_interrupt(struct intel_ntb_dev *ndev, int vec)
380 vec_mask = ndev_vec_mask(ndev, vec);
382 dev_dbg(ndev_dev(ndev), "vec %d vec_mask %llx\n", vec, vec_mask);
384 ndev->last_ts = jiffies;
386 if (vec_mask & ndev->db_link_mask) {
387 if (ndev->reg->poll_link(ndev))
388 ntb_link_event(&ndev->ntb);
391 if (vec_mask & ndev->db_valid_mask)
392 ntb_db_event(&ndev->ntb, vec);
397 static irqreturn_t ndev_vec_isr(int irq, void *dev)
399 struct intel_ntb_vec *nvec = dev;
401 return ndev_interrupt(nvec->ndev, nvec->num);
404 static irqreturn_t ndev_irq_isr(int irq, void *dev)
406 struct intel_ntb_dev *ndev = dev;
408 return ndev_interrupt(ndev, irq - ndev_pdev(ndev)->irq);
411 static int ndev_init_isr(struct intel_ntb_dev *ndev,
412 int msix_min, int msix_max,
413 int msix_shift, int total_shift)
415 struct pci_dev *pdev;
416 int rc, i, msix_count, node;
418 pdev = ndev_pdev(ndev);
420 node = dev_to_node(&pdev->dev);
422 /* Mask all doorbell interrupts */
423 ndev->db_mask = ndev->db_valid_mask;
424 ndev->reg->db_iowrite(ndev->db_mask,
426 ndev->self_reg->db_mask);
428 /* Try to set up msix irq */
430 ndev->vec = kzalloc_node(msix_max * sizeof(*ndev->vec),
433 goto err_msix_vec_alloc;
435 ndev->msix = kzalloc_node(msix_max * sizeof(*ndev->msix),
440 for (i = 0; i < msix_max; ++i)
441 ndev->msix[i].entry = i;
443 msix_count = pci_enable_msix_range(pdev, ndev->msix,
446 goto err_msix_enable;
448 for (i = 0; i < msix_count; ++i) {
449 ndev->vec[i].ndev = ndev;
450 ndev->vec[i].num = i;
451 rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
452 "ndev_vec_isr", &ndev->vec[i]);
454 goto err_msix_request;
457 dev_dbg(ndev_dev(ndev), "Using msix interrupts\n");
458 ndev->db_vec_count = msix_count;
459 ndev->db_vec_shift = msix_shift;
464 free_irq(ndev->msix[i].vector, ndev);
465 pci_disable_msix(pdev);
474 /* Try to set up msi irq */
476 rc = pci_enable_msi(pdev);
480 rc = request_irq(pdev->irq, ndev_irq_isr, 0,
481 "ndev_irq_isr", ndev);
483 goto err_msi_request;
485 dev_dbg(ndev_dev(ndev), "Using msi interrupts\n");
486 ndev->db_vec_count = 1;
487 ndev->db_vec_shift = total_shift;
491 pci_disable_msi(pdev);
494 /* Try to set up intx irq */
498 rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
499 "ndev_irq_isr", ndev);
501 goto err_intx_request;
503 dev_dbg(ndev_dev(ndev), "Using intx interrupts\n");
504 ndev->db_vec_count = 1;
505 ndev->db_vec_shift = total_shift;
512 static void ndev_deinit_isr(struct intel_ntb_dev *ndev)
514 struct pci_dev *pdev;
517 pdev = ndev_pdev(ndev);
519 /* Mask all doorbell interrupts */
520 ndev->db_mask = ndev->db_valid_mask;
521 ndev->reg->db_iowrite(ndev->db_mask,
523 ndev->self_reg->db_mask);
526 i = ndev->db_vec_count;
528 free_irq(ndev->msix[i].vector, &ndev->vec[i]);
529 pci_disable_msix(pdev);
533 free_irq(pdev->irq, ndev);
534 if (pci_dev_msi_enabled(pdev))
535 pci_disable_msi(pdev);
539 static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
540 size_t count, loff_t *offp)
542 struct intel_ntb_dev *ndev;
547 union { u64 v64; u32 v32; u16 v16; } u;
549 ndev = filp->private_data;
550 mmio = ndev->self_mmio;
552 buf_size = min(count, 0x800ul);
554 buf = kmalloc(buf_size, GFP_KERNEL);
560 off += scnprintf(buf + off, buf_size - off,
561 "NTB Device Information:\n");
563 off += scnprintf(buf + off, buf_size - off,
564 "Connection Topology -\t%s\n",
565 ntb_topo_string(ndev->ntb.topo));
567 off += scnprintf(buf + off, buf_size - off,
568 "B2B Offset -\t\t%#lx\n", ndev->b2b_off);
569 off += scnprintf(buf + off, buf_size - off,
570 "B2B MW Idx -\t\t%d\n", ndev->b2b_idx);
571 off += scnprintf(buf + off, buf_size - off,
572 "BAR4 Split -\t\t%s\n",
573 ndev->bar4_split ? "yes" : "no");
575 off += scnprintf(buf + off, buf_size - off,
576 "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl);
577 off += scnprintf(buf + off, buf_size - off,
578 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
580 if (!ndev->reg->link_is_up(ndev)) {
581 off += scnprintf(buf + off, buf_size - off,
582 "Link Status -\t\tDown\n");
584 off += scnprintf(buf + off, buf_size - off,
585 "Link Status -\t\tUp\n");
586 off += scnprintf(buf + off, buf_size - off,
587 "Link Speed -\t\tPCI-E Gen %u\n",
588 NTB_LNK_STA_SPEED(ndev->lnk_sta));
589 off += scnprintf(buf + off, buf_size - off,
590 "Link Width -\t\tx%u\n",
591 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
594 off += scnprintf(buf + off, buf_size - off,
595 "Memory Window Count -\t%u\n", ndev->mw_count);
596 off += scnprintf(buf + off, buf_size - off,
597 "Scratchpad Count -\t%u\n", ndev->spad_count);
598 off += scnprintf(buf + off, buf_size - off,
599 "Doorbell Count -\t%u\n", ndev->db_count);
600 off += scnprintf(buf + off, buf_size - off,
601 "Doorbell Vector Count -\t%u\n", ndev->db_vec_count);
602 off += scnprintf(buf + off, buf_size - off,
603 "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift);
605 off += scnprintf(buf + off, buf_size - off,
606 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
607 off += scnprintf(buf + off, buf_size - off,
608 "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask);
609 off += scnprintf(buf + off, buf_size - off,
610 "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask);
612 u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask);
613 off += scnprintf(buf + off, buf_size - off,
614 "Doorbell Mask -\t\t%#llx\n", u.v64);
616 u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell);
617 off += scnprintf(buf + off, buf_size - off,
618 "Doorbell Bell -\t\t%#llx\n", u.v64);
620 off += scnprintf(buf + off, buf_size - off,
621 "\nNTB Incoming XLAT:\n");
623 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 2));
624 off += scnprintf(buf + off, buf_size - off,
625 "XLAT23 -\t\t%#018llx\n", u.v64);
627 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
628 off += scnprintf(buf + off, buf_size - off,
629 "XLAT45 -\t\t%#018llx\n", u.v64);
631 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 2));
632 off += scnprintf(buf + off, buf_size - off,
633 "LMT23 -\t\t\t%#018llx\n", u.v64);
635 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
636 off += scnprintf(buf + off, buf_size - off,
637 "LMT45 -\t\t\t%#018llx\n", u.v64);
639 if (pdev_is_xeon(ndev->ntb.pdev)) {
640 if (ntb_topo_is_b2b(ndev->ntb.topo)) {
641 off += scnprintf(buf + off, buf_size - off,
642 "\nNTB Outgoing B2B XLAT:\n");
644 u.v64 = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
645 off += scnprintf(buf + off, buf_size - off,
646 "B2B XLAT23 -\t\t%#018llx\n", u.v64);
648 u.v64 = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
649 off += scnprintf(buf + off, buf_size - off,
650 "B2B XLAT45 -\t\t%#018llx\n", u.v64);
652 u.v64 = ioread64(mmio + XEON_PBAR23LMT_OFFSET);
653 off += scnprintf(buf + off, buf_size - off,
654 "B2B LMT23 -\t\t%#018llx\n", u.v64);
656 u.v64 = ioread64(mmio + XEON_PBAR45LMT_OFFSET);
657 off += scnprintf(buf + off, buf_size - off,
658 "B2B LMT45 -\t\t%#018llx\n", u.v64);
660 off += scnprintf(buf + off, buf_size - off,
661 "\nNTB Secondary BAR:\n");
663 u.v64 = ioread64(mmio + XEON_SBAR0BASE_OFFSET);
664 off += scnprintf(buf + off, buf_size - off,
665 "SBAR01 -\t\t%#018llx\n", u.v64);
667 u.v64 = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
668 off += scnprintf(buf + off, buf_size - off,
669 "SBAR23 -\t\t%#018llx\n", u.v64);
671 u.v64 = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
672 off += scnprintf(buf + off, buf_size - off,
673 "SBAR45 -\t\t%#018llx\n", u.v64);
676 off += scnprintf(buf + off, buf_size - off,
677 "\nXEON NTB Statistics:\n");
679 u.v16 = ioread16(mmio + XEON_USMEMMISS_OFFSET);
680 off += scnprintf(buf + off, buf_size - off,
681 "Upstream Memory Miss -\t%u\n", u.v16);
683 off += scnprintf(buf + off, buf_size - off,
684 "\nXEON NTB Hardware Errors:\n");
686 if (!pci_read_config_word(ndev->ntb.pdev,
687 XEON_DEVSTS_OFFSET, &u.v16))
688 off += scnprintf(buf + off, buf_size - off,
689 "DEVSTS -\t\t%#06x\n", u.v16);
691 if (!pci_read_config_word(ndev->ntb.pdev,
692 XEON_LINK_STATUS_OFFSET, &u.v16))
693 off += scnprintf(buf + off, buf_size - off,
694 "LNKSTS -\t\t%#06x\n", u.v16);
696 if (!pci_read_config_dword(ndev->ntb.pdev,
697 XEON_UNCERRSTS_OFFSET, &u.v32))
698 off += scnprintf(buf + off, buf_size - off,
699 "UNCERRSTS -\t\t%#06x\n", u.v32);
701 if (!pci_read_config_dword(ndev->ntb.pdev,
702 XEON_CORERRSTS_OFFSET, &u.v32))
703 off += scnprintf(buf + off, buf_size - off,
704 "CORERRSTS -\t\t%#06x\n", u.v32);
707 ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
712 static void ndev_init_debugfs(struct intel_ntb_dev *ndev)
715 ndev->debugfs_dir = NULL;
716 ndev->debugfs_info = NULL;
719 debugfs_create_dir(ndev_name(ndev), debugfs_dir);
720 if (!ndev->debugfs_dir)
721 ndev->debugfs_info = NULL;
724 debugfs_create_file("info", S_IRUSR,
725 ndev->debugfs_dir, ndev,
726 &intel_ntb_debugfs_info);
730 static void ndev_deinit_debugfs(struct intel_ntb_dev *ndev)
732 debugfs_remove_recursive(ndev->debugfs_dir);
735 static int intel_ntb_mw_count(struct ntb_dev *ntb)
737 return ntb_ndev(ntb)->mw_count;
740 static int intel_ntb_mw_get_range(struct ntb_dev *ntb, int idx,
742 resource_size_t *size,
743 resource_size_t *align,
744 resource_size_t *align_size)
746 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
749 if (idx >= ndev->b2b_idx && !ndev->b2b_off)
752 bar = ndev_mw_to_bar(ndev, idx);
757 *base = pci_resource_start(ndev->ntb.pdev, bar) +
758 (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
761 *size = pci_resource_len(ndev->ntb.pdev, bar) -
762 (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
765 *align = pci_resource_len(ndev->ntb.pdev, bar);
773 static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int idx,
774 dma_addr_t addr, resource_size_t size)
776 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
777 unsigned long base_reg, xlat_reg, limit_reg;
778 resource_size_t bar_size, mw_size;
780 u64 base, limit, reg_val;
783 if (idx >= ndev->b2b_idx && !ndev->b2b_off)
786 bar = ndev_mw_to_bar(ndev, idx);
790 bar_size = pci_resource_len(ndev->ntb.pdev, bar);
792 if (idx == ndev->b2b_idx)
793 mw_size = bar_size - ndev->b2b_off;
797 /* hardware requires that addr is aligned to bar size */
798 if (addr & (bar_size - 1))
801 /* make sure the range fits in the usable mw size */
805 mmio = ndev->self_mmio;
806 base_reg = bar0_off(ndev->xlat_reg->bar0_base, bar);
807 xlat_reg = bar2_off(ndev->xlat_reg->bar2_xlat, bar);
808 limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar);
810 if (bar < 4 || !ndev->bar4_split) {
811 base = ioread64(mmio + base_reg);
813 /* Set the limit if supported, if size is not mw_size */
814 if (limit_reg && size != mw_size)
819 /* set and verify setting the translation address */
820 iowrite64(addr, mmio + xlat_reg);
821 reg_val = ioread64(mmio + xlat_reg);
822 if (reg_val != addr) {
823 iowrite64(0, mmio + xlat_reg);
827 /* set and verify setting the limit */
828 iowrite64(limit, mmio + limit_reg);
829 reg_val = ioread64(mmio + limit_reg);
830 if (reg_val != limit) {
831 iowrite64(base, mmio + limit_reg);
832 iowrite64(0, mmio + xlat_reg);
836 /* split bar addr range must all be 32 bit */
837 if (addr & (~0ull << 32))
839 if ((addr + size) & (~0ull << 32))
842 base = ioread32(mmio + base_reg);
844 /* Set the limit if supported, if size is not mw_size */
845 if (limit_reg && size != mw_size)
850 /* set and verify setting the translation address */
851 iowrite32(addr, mmio + xlat_reg);
852 reg_val = ioread32(mmio + xlat_reg);
853 if (reg_val != addr) {
854 iowrite32(0, mmio + xlat_reg);
858 /* set and verify setting the limit */
859 iowrite32(limit, mmio + limit_reg);
860 reg_val = ioread32(mmio + limit_reg);
861 if (reg_val != limit) {
862 iowrite32(base, mmio + limit_reg);
863 iowrite32(0, mmio + xlat_reg);
871 static int intel_ntb_link_is_up(struct ntb_dev *ntb,
872 enum ntb_speed *speed,
873 enum ntb_width *width)
875 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
877 if (ndev->reg->link_is_up(ndev)) {
879 *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
881 *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
884 /* TODO MAYBE: is it possible to observe the link speed and
885 * width while link is training? */
887 *speed = NTB_SPEED_NONE;
889 *width = NTB_WIDTH_NONE;
894 static int intel_ntb_link_enable(struct ntb_dev *ntb,
895 enum ntb_speed max_speed,
896 enum ntb_width max_width)
898 struct intel_ntb_dev *ndev;
901 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
903 if (ndev->ntb.topo == NTB_TOPO_SEC)
906 dev_dbg(ndev_dev(ndev),
907 "Enabling link with max_speed %d max_width %d\n",
908 max_speed, max_width);
909 if (max_speed != NTB_SPEED_AUTO)
910 dev_dbg(ndev_dev(ndev), "ignoring max_speed %d\n", max_speed);
911 if (max_width != NTB_WIDTH_AUTO)
912 dev_dbg(ndev_dev(ndev), "ignoring max_width %d\n", max_width);
914 ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
915 ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
916 ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
917 ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
918 if (ndev->bar4_split)
919 ntb_ctl |= NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP;
920 iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
925 static int intel_ntb_link_disable(struct ntb_dev *ntb)
927 struct intel_ntb_dev *ndev;
930 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
932 if (ndev->ntb.topo == NTB_TOPO_SEC)
935 dev_dbg(ndev_dev(ndev), "Disabling link\n");
937 /* Bring NTB link down */
938 ntb_cntl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
939 ntb_cntl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP);
940 ntb_cntl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP);
941 if (ndev->bar4_split)
942 ntb_cntl &= ~(NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP);
943 ntb_cntl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK;
944 iowrite32(ntb_cntl, ndev->self_mmio + ndev->reg->ntb_ctl);
949 static int intel_ntb_db_is_unsafe(struct ntb_dev *ntb)
951 return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_DB);
954 static u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb)
956 return ntb_ndev(ntb)->db_valid_mask;
959 static int intel_ntb_db_vector_count(struct ntb_dev *ntb)
961 struct intel_ntb_dev *ndev;
963 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
965 return ndev->db_vec_count;
968 static u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
970 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
972 if (db_vector < 0 || db_vector > ndev->db_vec_count)
975 return ndev->db_valid_mask & ndev_vec_mask(ndev, db_vector);
978 static u64 intel_ntb_db_read(struct ntb_dev *ntb)
980 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
982 return ndev_db_read(ndev,
984 ndev->self_reg->db_bell);
987 static int intel_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
989 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
991 return ndev_db_write(ndev, db_bits,
993 ndev->self_reg->db_bell);
996 static int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
998 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1000 return ndev_db_set_mask(ndev, db_bits,
1002 ndev->self_reg->db_mask);
1005 static int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
1007 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1009 return ndev_db_clear_mask(ndev, db_bits,
1011 ndev->self_reg->db_mask);
1014 static int intel_ntb_peer_db_addr(struct ntb_dev *ntb,
1015 phys_addr_t *db_addr,
1016 resource_size_t *db_size)
1018 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1020 return ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr,
1021 ndev->peer_reg->db_bell);
1024 static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
1026 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1028 return ndev_db_write(ndev, db_bits,
1030 ndev->peer_reg->db_bell);
1033 static int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb)
1035 return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_SPAD);
1038 static int intel_ntb_spad_count(struct ntb_dev *ntb)
1040 struct intel_ntb_dev *ndev;
1042 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1044 return ndev->spad_count;
1047 static u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx)
1049 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1051 return ndev_spad_read(ndev, idx,
1053 ndev->self_reg->spad);
1056 static int intel_ntb_spad_write(struct ntb_dev *ntb,
1059 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1061 return ndev_spad_write(ndev, idx, val,
1063 ndev->self_reg->spad);
1066 static int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int idx,
1067 phys_addr_t *spad_addr)
1069 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1071 return ndev_spad_addr(ndev, idx, spad_addr, ndev->peer_addr,
1072 ndev->peer_reg->spad);
1075 static u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int idx)
1077 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1079 return ndev_spad_read(ndev, idx,
1081 ndev->peer_reg->spad);
1084 static int intel_ntb_peer_spad_write(struct ntb_dev *ntb,
1087 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1089 return ndev_spad_write(ndev, idx, val,
1091 ndev->peer_reg->spad);
1096 static u64 atom_db_ioread(void __iomem *mmio)
1098 return ioread64(mmio);
1101 static void atom_db_iowrite(u64 bits, void __iomem *mmio)
1103 iowrite64(bits, mmio);
1106 static int atom_poll_link(struct intel_ntb_dev *ndev)
1110 ntb_ctl = ioread32(ndev->self_mmio + ATOM_NTBCNTL_OFFSET);
1112 if (ntb_ctl == ndev->ntb_ctl)
1115 ndev->ntb_ctl = ntb_ctl;
1117 ndev->lnk_sta = ioread32(ndev->self_mmio + ATOM_LINK_STATUS_OFFSET);
1122 static int atom_link_is_up(struct intel_ntb_dev *ndev)
1124 return ATOM_NTB_CTL_ACTIVE(ndev->ntb_ctl);
1127 static int atom_link_is_err(struct intel_ntb_dev *ndev)
1129 if (ioread32(ndev->self_mmio + ATOM_LTSSMSTATEJMP_OFFSET)
1130 & ATOM_LTSSMSTATEJMP_FORCEDETECT)
1133 if (ioread32(ndev->self_mmio + ATOM_IBSTERRRCRVSTS0_OFFSET)
1134 & ATOM_IBIST_ERR_OFLOW)
1140 static inline enum ntb_topo atom_ppd_topo(struct intel_ntb_dev *ndev, u32 ppd)
1142 switch (ppd & ATOM_PPD_TOPO_MASK) {
1143 case ATOM_PPD_TOPO_B2B_USD:
1144 dev_dbg(ndev_dev(ndev), "PPD %d B2B USD\n", ppd);
1145 return NTB_TOPO_B2B_USD;
1147 case ATOM_PPD_TOPO_B2B_DSD:
1148 dev_dbg(ndev_dev(ndev), "PPD %d B2B DSD\n", ppd);
1149 return NTB_TOPO_B2B_DSD;
1151 case ATOM_PPD_TOPO_PRI_USD:
1152 case ATOM_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
1153 case ATOM_PPD_TOPO_SEC_USD:
1154 case ATOM_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
1155 dev_dbg(ndev_dev(ndev), "PPD %d non B2B disabled\n", ppd);
1156 return NTB_TOPO_NONE;
1159 dev_dbg(ndev_dev(ndev), "PPD %d invalid\n", ppd);
1160 return NTB_TOPO_NONE;
1163 static void atom_link_hb(struct work_struct *work)
1165 struct intel_ntb_dev *ndev = hb_ndev(work);
1166 unsigned long poll_ts;
1170 poll_ts = ndev->last_ts + ATOM_LINK_HB_TIMEOUT;
1172 /* Delay polling the link status if an interrupt was received,
1173 * unless the cached link status says the link is down.
1175 if (time_after(poll_ts, jiffies) && atom_link_is_up(ndev)) {
1176 schedule_delayed_work(&ndev->hb_timer, poll_ts - jiffies);
1180 if (atom_poll_link(ndev))
1181 ntb_link_event(&ndev->ntb);
1183 if (atom_link_is_up(ndev) || !atom_link_is_err(ndev)) {
1184 schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_HB_TIMEOUT);
1188 /* Link is down with error: recover the link! */
1190 mmio = ndev->self_mmio;
1192 /* Driver resets the NTB ModPhy lanes - magic! */
1193 iowrite8(0xe0, mmio + ATOM_MODPHY_PCSREG6);
1194 iowrite8(0x40, mmio + ATOM_MODPHY_PCSREG4);
1195 iowrite8(0x60, mmio + ATOM_MODPHY_PCSREG4);
1196 iowrite8(0x60, mmio + ATOM_MODPHY_PCSREG6);
1198 /* Driver waits 100ms to allow the NTB ModPhy to settle */
1201 /* Clear AER Errors, write to clear */
1202 status32 = ioread32(mmio + ATOM_ERRCORSTS_OFFSET);
1203 dev_dbg(ndev_dev(ndev), "ERRCORSTS = %x\n", status32);
1204 status32 &= PCI_ERR_COR_REP_ROLL;
1205 iowrite32(status32, mmio + ATOM_ERRCORSTS_OFFSET);
1207 /* Clear unexpected electrical idle event in LTSSM, write to clear */
1208 status32 = ioread32(mmio + ATOM_LTSSMERRSTS0_OFFSET);
1209 dev_dbg(ndev_dev(ndev), "LTSSMERRSTS0 = %x\n", status32);
1210 status32 |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI;
1211 iowrite32(status32, mmio + ATOM_LTSSMERRSTS0_OFFSET);
1213 /* Clear DeSkew Buffer error, write to clear */
1214 status32 = ioread32(mmio + ATOM_DESKEWSTS_OFFSET);
1215 dev_dbg(ndev_dev(ndev), "DESKEWSTS = %x\n", status32);
1216 status32 |= ATOM_DESKEWSTS_DBERR;
1217 iowrite32(status32, mmio + ATOM_DESKEWSTS_OFFSET);
1219 status32 = ioread32(mmio + ATOM_IBSTERRRCRVSTS0_OFFSET);
1220 dev_dbg(ndev_dev(ndev), "IBSTERRRCRVSTS0 = %x\n", status32);
1221 status32 &= ATOM_IBIST_ERR_OFLOW;
1222 iowrite32(status32, mmio + ATOM_IBSTERRRCRVSTS0_OFFSET);
1224 /* Releases the NTB state machine to allow the link to retrain */
1225 status32 = ioread32(mmio + ATOM_LTSSMSTATEJMP_OFFSET);
1226 dev_dbg(ndev_dev(ndev), "LTSSMSTATEJMP = %x\n", status32);
1227 status32 &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT;
1228 iowrite32(status32, mmio + ATOM_LTSSMSTATEJMP_OFFSET);
1230 /* There is a potential race between the 2 NTB devices recovering at the
1231 * same time. If the times are the same, the link will not recover and
1232 * the driver will be stuck in this loop forever. Add a random interval
1233 * to the recovery time to prevent this race.
1235 schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_RECOVERY_TIME
1236 + prandom_u32() % ATOM_LINK_RECOVERY_TIME);
1239 static int atom_init_isr(struct intel_ntb_dev *ndev)
1243 rc = ndev_init_isr(ndev, 1, ATOM_DB_MSIX_VECTOR_COUNT,
1244 ATOM_DB_MSIX_VECTOR_SHIFT, ATOM_DB_TOTAL_SHIFT);
1248 /* ATOM doesn't have link status interrupt, poll on that platform */
1249 ndev->last_ts = jiffies;
1250 INIT_DELAYED_WORK(&ndev->hb_timer, atom_link_hb);
1251 schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_HB_TIMEOUT);
1256 static void atom_deinit_isr(struct intel_ntb_dev *ndev)
1258 cancel_delayed_work_sync(&ndev->hb_timer);
1259 ndev_deinit_isr(ndev);
1262 static int atom_init_ntb(struct intel_ntb_dev *ndev)
1264 ndev->mw_count = ATOM_MW_COUNT;
1265 ndev->spad_count = ATOM_SPAD_COUNT;
1266 ndev->db_count = ATOM_DB_COUNT;
1268 switch (ndev->ntb.topo) {
1269 case NTB_TOPO_B2B_USD:
1270 case NTB_TOPO_B2B_DSD:
1271 ndev->self_reg = &atom_pri_reg;
1272 ndev->peer_reg = &atom_b2b_reg;
1273 ndev->xlat_reg = &atom_sec_xlat;
1275 /* Enable Bus Master and Memory Space on the secondary side */
1276 iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
1277 ndev->self_mmio + ATOM_SPCICMD_OFFSET);
1285 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1290 static int atom_init_dev(struct intel_ntb_dev *ndev)
1295 rc = pci_read_config_dword(ndev->ntb.pdev, ATOM_PPD_OFFSET, &ppd);
1299 ndev->ntb.topo = atom_ppd_topo(ndev, ppd);
1300 if (ndev->ntb.topo == NTB_TOPO_NONE)
1303 rc = atom_init_ntb(ndev);
1307 rc = atom_init_isr(ndev);
1311 if (ndev->ntb.topo != NTB_TOPO_SEC) {
1312 /* Initiate PCI-E link training */
1313 rc = pci_write_config_dword(ndev->ntb.pdev, ATOM_PPD_OFFSET,
1314 ppd | ATOM_PPD_INIT_LINK);
1322 static void atom_deinit_dev(struct intel_ntb_dev *ndev)
1324 atom_deinit_isr(ndev);
1329 static u64 xeon_db_ioread(void __iomem *mmio)
1331 return (u64)ioread16(mmio);
1334 static void xeon_db_iowrite(u64 bits, void __iomem *mmio)
1336 iowrite16((u16)bits, mmio);
1339 static int xeon_poll_link(struct intel_ntb_dev *ndev)
1344 ndev->reg->db_iowrite(ndev->db_link_mask,
1346 ndev->self_reg->db_bell);
1348 rc = pci_read_config_word(ndev->ntb.pdev,
1349 XEON_LINK_STATUS_OFFSET, ®_val);
1353 if (reg_val == ndev->lnk_sta)
1356 ndev->lnk_sta = reg_val;
1361 static int xeon_link_is_up(struct intel_ntb_dev *ndev)
1363 if (ndev->ntb.topo == NTB_TOPO_SEC)
1366 return NTB_LNK_STA_ACTIVE(ndev->lnk_sta);
1369 static inline enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd)
1371 switch (ppd & XEON_PPD_TOPO_MASK) {
1372 case XEON_PPD_TOPO_B2B_USD:
1373 return NTB_TOPO_B2B_USD;
1375 case XEON_PPD_TOPO_B2B_DSD:
1376 return NTB_TOPO_B2B_DSD;
1378 case XEON_PPD_TOPO_PRI_USD:
1379 case XEON_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
1380 return NTB_TOPO_PRI;
1382 case XEON_PPD_TOPO_SEC_USD:
1383 case XEON_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
1384 return NTB_TOPO_SEC;
1387 return NTB_TOPO_NONE;
1390 static inline int xeon_ppd_bar4_split(struct intel_ntb_dev *ndev, u8 ppd)
1392 if (ppd & XEON_PPD_SPLIT_BAR_MASK) {
1393 dev_dbg(ndev_dev(ndev), "PPD %d split bar\n", ppd);
1399 static int xeon_init_isr(struct intel_ntb_dev *ndev)
1401 return ndev_init_isr(ndev, XEON_DB_MSIX_VECTOR_COUNT,
1402 XEON_DB_MSIX_VECTOR_COUNT,
1403 XEON_DB_MSIX_VECTOR_SHIFT,
1404 XEON_DB_TOTAL_SHIFT);
1407 static void xeon_deinit_isr(struct intel_ntb_dev *ndev)
1409 ndev_deinit_isr(ndev);
1412 static int xeon_setup_b2b_mw(struct intel_ntb_dev *ndev,
1413 const struct intel_b2b_addr *addr,
1414 const struct intel_b2b_addr *peer_addr)
1416 struct pci_dev *pdev;
1418 resource_size_t bar_size;
1419 phys_addr_t bar_addr;
1423 pdev = ndev_pdev(ndev);
1424 mmio = ndev->self_mmio;
1426 if (ndev->b2b_idx >= ndev->mw_count) {
1427 dev_dbg(ndev_dev(ndev), "not using b2b mw\n");
1431 b2b_bar = ndev_mw_to_bar(ndev, ndev->b2b_idx);
1435 dev_dbg(ndev_dev(ndev), "using b2b mw bar %d\n", b2b_bar);
1437 bar_size = pci_resource_len(ndev->ntb.pdev, b2b_bar);
1439 dev_dbg(ndev_dev(ndev), "b2b bar size %#llx\n", bar_size);
1441 if (b2b_mw_share && XEON_B2B_MIN_SIZE <= bar_size >> 1) {
1442 dev_dbg(ndev_dev(ndev),
1443 "b2b using first half of bar\n");
1444 ndev->b2b_off = bar_size >> 1;
1445 } else if (XEON_B2B_MIN_SIZE <= bar_size) {
1446 dev_dbg(ndev_dev(ndev),
1447 "b2b using whole bar\n");
1451 dev_dbg(ndev_dev(ndev),
1452 "b2b bar size is too small\n");
1457 /* Reset the secondary bar sizes to match the primary bar sizes,
1458 * except disable or halve the size of the b2b secondary bar.
1460 * Note: code for each specific bar size register, because the register
1461 * offsets are not in a consistent order (bar5sz comes after ppd, odd).
1463 pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &bar_sz);
1464 dev_dbg(ndev_dev(ndev), "PBAR23SZ %#x\n", bar_sz);
1471 pci_write_config_byte(pdev, XEON_SBAR23SZ_OFFSET, bar_sz);
1472 pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &bar_sz);
1473 dev_dbg(ndev_dev(ndev), "SBAR23SZ %#x\n", bar_sz);
1475 if (!ndev->bar4_split) {
1476 pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &bar_sz);
1477 dev_dbg(ndev_dev(ndev), "PBAR45SZ %#x\n", bar_sz);
1484 pci_write_config_byte(pdev, XEON_SBAR45SZ_OFFSET, bar_sz);
1485 pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &bar_sz);
1486 dev_dbg(ndev_dev(ndev), "SBAR45SZ %#x\n", bar_sz);
1488 pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &bar_sz);
1489 dev_dbg(ndev_dev(ndev), "PBAR4SZ %#x\n", bar_sz);
1496 pci_write_config_byte(pdev, XEON_SBAR4SZ_OFFSET, bar_sz);
1497 pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &bar_sz);
1498 dev_dbg(ndev_dev(ndev), "SBAR4SZ %#x\n", bar_sz);
1500 pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &bar_sz);
1501 dev_dbg(ndev_dev(ndev), "PBAR5SZ %#x\n", bar_sz);
1508 pci_write_config_byte(pdev, XEON_SBAR5SZ_OFFSET, bar_sz);
1509 pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &bar_sz);
1510 dev_dbg(ndev_dev(ndev), "SBAR5SZ %#x\n", bar_sz);
1513 /* SBAR01 hit by first part of the b2b bar */
1515 bar_addr = addr->bar0_addr;
1516 else if (b2b_bar == 2)
1517 bar_addr = addr->bar2_addr64;
1518 else if (b2b_bar == 4 && !ndev->bar4_split)
1519 bar_addr = addr->bar4_addr64;
1520 else if (b2b_bar == 4)
1521 bar_addr = addr->bar4_addr32;
1522 else if (b2b_bar == 5)
1523 bar_addr = addr->bar5_addr32;
1527 dev_dbg(ndev_dev(ndev), "SBAR01 %#018llx\n", bar_addr);
1528 iowrite64(bar_addr, mmio + XEON_SBAR0BASE_OFFSET);
1530 /* Other SBAR are normally hit by the PBAR xlat, except for b2b bar.
1531 * The b2b bar is either disabled above, or configured half-size, and
1532 * it starts at the PBAR xlat + offset.
1535 bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
1536 iowrite64(bar_addr, mmio + XEON_SBAR23BASE_OFFSET);
1537 bar_addr = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
1538 dev_dbg(ndev_dev(ndev), "SBAR23 %#018llx\n", bar_addr);
1540 if (!ndev->bar4_split) {
1541 bar_addr = addr->bar4_addr64 +
1542 (b2b_bar == 4 ? ndev->b2b_off : 0);
1543 iowrite64(bar_addr, mmio + XEON_SBAR45BASE_OFFSET);
1544 bar_addr = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
1545 dev_dbg(ndev_dev(ndev), "SBAR45 %#018llx\n", bar_addr);
1547 bar_addr = addr->bar4_addr32 +
1548 (b2b_bar == 4 ? ndev->b2b_off : 0);
1549 iowrite32(bar_addr, mmio + XEON_SBAR4BASE_OFFSET);
1550 bar_addr = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
1551 dev_dbg(ndev_dev(ndev), "SBAR4 %#010llx\n", bar_addr);
1553 bar_addr = addr->bar5_addr32 +
1554 (b2b_bar == 5 ? ndev->b2b_off : 0);
1555 iowrite32(bar_addr, mmio + XEON_SBAR5BASE_OFFSET);
1556 bar_addr = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
1557 dev_dbg(ndev_dev(ndev), "SBAR5 %#010llx\n", bar_addr);
1560 /* setup incoming bar limits == base addrs (zero length windows) */
1562 bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
1563 iowrite64(bar_addr, mmio + XEON_SBAR23LMT_OFFSET);
1564 bar_addr = ioread64(mmio + XEON_SBAR23LMT_OFFSET);
1565 dev_dbg(ndev_dev(ndev), "SBAR23LMT %#018llx\n", bar_addr);
1567 if (!ndev->bar4_split) {
1568 bar_addr = addr->bar4_addr64 +
1569 (b2b_bar == 4 ? ndev->b2b_off : 0);
1570 iowrite64(bar_addr, mmio + XEON_SBAR45LMT_OFFSET);
1571 bar_addr = ioread64(mmio + XEON_SBAR45LMT_OFFSET);
1572 dev_dbg(ndev_dev(ndev), "SBAR45LMT %#018llx\n", bar_addr);
1574 bar_addr = addr->bar4_addr32 +
1575 (b2b_bar == 4 ? ndev->b2b_off : 0);
1576 iowrite32(bar_addr, mmio + XEON_SBAR4LMT_OFFSET);
1577 bar_addr = ioread32(mmio + XEON_SBAR4LMT_OFFSET);
1578 dev_dbg(ndev_dev(ndev), "SBAR4LMT %#010llx\n", bar_addr);
1580 bar_addr = addr->bar5_addr32 +
1581 (b2b_bar == 5 ? ndev->b2b_off : 0);
1582 iowrite32(bar_addr, mmio + XEON_SBAR5LMT_OFFSET);
1583 bar_addr = ioread32(mmio + XEON_SBAR5LMT_OFFSET);
1584 dev_dbg(ndev_dev(ndev), "SBAR5LMT %#05llx\n", bar_addr);
1587 /* zero incoming translation addrs */
1588 iowrite64(0, mmio + XEON_SBAR23XLAT_OFFSET);
1590 if (!ndev->bar4_split) {
1591 iowrite64(0, mmio + XEON_SBAR45XLAT_OFFSET);
1593 iowrite32(0, mmio + XEON_SBAR4XLAT_OFFSET);
1594 iowrite32(0, mmio + XEON_SBAR5XLAT_OFFSET);
1597 /* zero outgoing translation limits (whole bar size windows) */
1598 iowrite64(0, mmio + XEON_PBAR23LMT_OFFSET);
1599 if (!ndev->bar4_split) {
1600 iowrite64(0, mmio + XEON_PBAR45LMT_OFFSET);
1602 iowrite32(0, mmio + XEON_PBAR4LMT_OFFSET);
1603 iowrite32(0, mmio + XEON_PBAR5LMT_OFFSET);
1606 /* set outgoing translation offsets */
1607 bar_addr = peer_addr->bar2_addr64;
1608 iowrite64(bar_addr, mmio + XEON_PBAR23XLAT_OFFSET);
1609 bar_addr = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
1610 dev_dbg(ndev_dev(ndev), "PBAR23XLAT %#018llx\n", bar_addr);
1612 if (!ndev->bar4_split) {
1613 bar_addr = peer_addr->bar4_addr64;
1614 iowrite64(bar_addr, mmio + XEON_PBAR45XLAT_OFFSET);
1615 bar_addr = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
1616 dev_dbg(ndev_dev(ndev), "PBAR45XLAT %#018llx\n", bar_addr);
1618 bar_addr = peer_addr->bar4_addr32;
1619 iowrite32(bar_addr, mmio + XEON_PBAR4XLAT_OFFSET);
1620 bar_addr = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
1621 dev_dbg(ndev_dev(ndev), "PBAR4XLAT %#010llx\n", bar_addr);
1623 bar_addr = peer_addr->bar5_addr32;
1624 iowrite32(bar_addr, mmio + XEON_PBAR5XLAT_OFFSET);
1625 bar_addr = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
1626 dev_dbg(ndev_dev(ndev), "PBAR5XLAT %#010llx\n", bar_addr);
1629 /* set the translation offset for b2b registers */
1631 bar_addr = peer_addr->bar0_addr;
1632 else if (b2b_bar == 2)
1633 bar_addr = peer_addr->bar2_addr64;
1634 else if (b2b_bar == 4 && !ndev->bar4_split)
1635 bar_addr = peer_addr->bar4_addr64;
1636 else if (b2b_bar == 4)
1637 bar_addr = peer_addr->bar4_addr32;
1638 else if (b2b_bar == 5)
1639 bar_addr = peer_addr->bar5_addr32;
1643 /* B2B_XLAT_OFFSET is 64bit, but can only take 32bit writes */
1644 dev_dbg(ndev_dev(ndev), "B2BXLAT %#018llx\n", bar_addr);
1645 iowrite32(bar_addr, mmio + XEON_B2B_XLAT_OFFSETL);
1646 iowrite32(bar_addr >> 32, mmio + XEON_B2B_XLAT_OFFSETU);
1649 /* map peer ntb mmio config space registers */
1650 ndev->peer_mmio = pci_iomap(pdev, b2b_bar,
1652 if (!ndev->peer_mmio)
1659 static int xeon_init_ntb(struct intel_ntb_dev *ndev)
1664 if (ndev->bar4_split)
1665 ndev->mw_count = HSX_SPLIT_BAR_MW_COUNT;
1667 ndev->mw_count = XEON_MW_COUNT;
1669 ndev->spad_count = XEON_SPAD_COUNT;
1670 ndev->db_count = XEON_DB_COUNT;
1671 ndev->db_link_mask = XEON_DB_LINK_BIT;
1673 switch (ndev->ntb.topo) {
1675 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1676 dev_err(ndev_dev(ndev), "NTB Primary config disabled\n");
1680 /* enable link to allow secondary side device to appear */
1681 ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
1682 ntb_ctl &= ~NTB_CTL_DISABLE;
1683 iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
1685 /* use half the spads for the peer */
1686 ndev->spad_count >>= 1;
1687 ndev->self_reg = &xeon_pri_reg;
1688 ndev->peer_reg = &xeon_sec_reg;
1689 ndev->xlat_reg = &xeon_sec_xlat;
1693 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1694 dev_err(ndev_dev(ndev), "NTB Secondary config disabled\n");
1697 /* use half the spads for the peer */
1698 ndev->spad_count >>= 1;
1699 ndev->self_reg = &xeon_sec_reg;
1700 ndev->peer_reg = &xeon_pri_reg;
1701 ndev->xlat_reg = &xeon_pri_xlat;
1704 case NTB_TOPO_B2B_USD:
1705 case NTB_TOPO_B2B_DSD:
1706 ndev->self_reg = &xeon_pri_reg;
1707 ndev->peer_reg = &xeon_b2b_reg;
1708 ndev->xlat_reg = &xeon_sec_xlat;
1710 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1711 ndev->peer_reg = &xeon_pri_reg;
1714 ndev->b2b_idx = b2b_mw_idx + ndev->mw_count;
1716 ndev->b2b_idx = b2b_mw_idx;
1718 dev_dbg(ndev_dev(ndev),
1719 "setting up b2b mw idx %d means %d\n",
1720 b2b_mw_idx, ndev->b2b_idx);
1722 } else if (ndev->hwerr_flags & NTB_HWERR_B2BDOORBELL_BIT14) {
1723 dev_warn(ndev_dev(ndev), "Reduce doorbell count by 1\n");
1724 ndev->db_count -= 1;
1727 if (ndev->ntb.topo == NTB_TOPO_B2B_USD) {
1728 rc = xeon_setup_b2b_mw(ndev,
1730 &xeon_b2b_usd_addr);
1732 rc = xeon_setup_b2b_mw(ndev,
1734 &xeon_b2b_dsd_addr);
1739 /* Enable Bus Master and Memory Space on the secondary side */
1740 iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
1741 ndev->self_mmio + XEON_SPCICMD_OFFSET);
1749 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1751 ndev->reg->db_iowrite(ndev->db_valid_mask,
1753 ndev->self_reg->db_mask);
1758 static int xeon_init_dev(struct intel_ntb_dev *ndev)
1760 struct pci_dev *pdev;
1764 pdev = ndev_pdev(ndev);
1766 switch (pdev->device) {
1767 /* There is a Xeon hardware errata related to writes to SDOORBELL or
1768 * B2BDOORBELL in conjunction with inbound access to NTB MMIO Space,
1769 * which may hang the system. To workaround this use the second memory
1770 * window to access the interrupt and scratch pad registers on the
1773 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
1774 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
1775 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
1776 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
1777 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
1778 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
1779 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
1780 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
1781 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
1782 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
1783 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
1784 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
1785 ndev->hwerr_flags |= NTB_HWERR_SDOORBELL_LOCKUP;
1789 switch (pdev->device) {
1790 /* There is a hardware errata related to accessing any register in
1791 * SB01BASE in the presence of bidirectional traffic crossing the NTB.
1793 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
1794 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
1795 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
1796 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
1797 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
1798 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
1799 ndev->hwerr_flags |= NTB_HWERR_SB01BASE_LOCKUP;
1803 switch (pdev->device) {
1804 /* HW Errata on bit 14 of b2bdoorbell register. Writes will not be
1805 * mirrored to the remote system. Shrink the number of bits by one,
1806 * since bit 14 is the last bit.
1808 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
1809 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
1810 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
1811 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
1812 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
1813 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
1814 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
1815 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
1816 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
1817 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
1818 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
1819 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
1820 ndev->hwerr_flags |= NTB_HWERR_B2BDOORBELL_BIT14;
1824 ndev->reg = &xeon_reg;
1826 rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd);
1830 ndev->ntb.topo = xeon_ppd_topo(ndev, ppd);
1831 dev_dbg(ndev_dev(ndev), "ppd %#x topo %s\n", ppd,
1832 ntb_topo_string(ndev->ntb.topo));
1833 if (ndev->ntb.topo == NTB_TOPO_NONE)
1836 if (ndev->ntb.topo != NTB_TOPO_SEC) {
1837 ndev->bar4_split = xeon_ppd_bar4_split(ndev, ppd);
1838 dev_dbg(ndev_dev(ndev), "ppd %#x bar4_split %d\n",
1839 ppd, ndev->bar4_split);
1841 /* This is a way for transparent BAR to figure out if we are
1842 * doing split BAR or not. There is no way for the hw on the
1843 * transparent side to know and set the PPD.
1845 mem = pci_select_bars(pdev, IORESOURCE_MEM);
1846 ndev->bar4_split = hweight32(mem) ==
1847 HSX_SPLIT_BAR_MW_COUNT + 1;
1848 dev_dbg(ndev_dev(ndev), "mem %#x bar4_split %d\n",
1849 mem, ndev->bar4_split);
1852 rc = xeon_init_ntb(ndev);
1856 return xeon_init_isr(ndev);
1859 static void xeon_deinit_dev(struct intel_ntb_dev *ndev)
1861 xeon_deinit_isr(ndev);
1864 static int intel_ntb_init_pci(struct intel_ntb_dev *ndev, struct pci_dev *pdev)
1868 pci_set_drvdata(pdev, ndev);
1870 rc = pci_enable_device(pdev);
1872 goto err_pci_enable;
1874 rc = pci_request_regions(pdev, NTB_NAME);
1876 goto err_pci_regions;
1878 pci_set_master(pdev);
1880 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1882 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1885 dev_warn(ndev_dev(ndev), "Cannot DMA highmem\n");
1888 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1890 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1893 dev_warn(ndev_dev(ndev), "Cannot DMA consistent highmem\n");
1896 ndev->self_mmio = pci_iomap(pdev, 0, 0);
1897 if (!ndev->self_mmio) {
1901 ndev->peer_mmio = ndev->self_mmio;
1907 pci_clear_master(pdev);
1908 pci_release_regions(pdev);
1910 pci_disable_device(pdev);
1912 pci_set_drvdata(pdev, NULL);
1916 static void intel_ntb_deinit_pci(struct intel_ntb_dev *ndev)
1918 struct pci_dev *pdev = ndev_pdev(ndev);
1920 if (ndev->peer_mmio && ndev->peer_mmio != ndev->self_mmio)
1921 pci_iounmap(pdev, ndev->peer_mmio);
1922 pci_iounmap(pdev, ndev->self_mmio);
1924 pci_clear_master(pdev);
1925 pci_release_regions(pdev);
1926 pci_disable_device(pdev);
1927 pci_set_drvdata(pdev, NULL);
1930 static inline void ndev_init_struct(struct intel_ntb_dev *ndev,
1931 struct pci_dev *pdev)
1933 ndev->ntb.pdev = pdev;
1934 ndev->ntb.topo = NTB_TOPO_NONE;
1935 ndev->ntb.ops = &intel_ntb_ops;
1938 ndev->b2b_idx = INT_MAX;
1940 ndev->bar4_split = 0;
1943 ndev->spad_count = 0;
1945 ndev->db_vec_count = 0;
1946 ndev->db_vec_shift = 0;
1951 ndev->db_valid_mask = 0;
1952 ndev->db_link_mask = 0;
1955 spin_lock_init(&ndev->db_mask_lock);
1958 static int intel_ntb_pci_probe(struct pci_dev *pdev,
1959 const struct pci_device_id *id)
1961 struct intel_ntb_dev *ndev;
1964 node = dev_to_node(&pdev->dev);
1966 if (pdev_is_atom(pdev)) {
1967 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
1973 ndev_init_struct(ndev, pdev);
1975 rc = intel_ntb_init_pci(ndev, pdev);
1979 rc = atom_init_dev(ndev);
1983 } else if (pdev_is_xeon(pdev)) {
1984 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
1990 ndev_init_struct(ndev, pdev);
1992 rc = intel_ntb_init_pci(ndev, pdev);
1996 rc = xeon_init_dev(ndev);
2005 ndev_reset_unsafe_flags(ndev);
2007 ndev->reg->poll_link(ndev);
2009 ndev_init_debugfs(ndev);
2011 rc = ntb_register_device(&ndev->ntb);
2018 ndev_deinit_debugfs(ndev);
2019 if (pdev_is_atom(pdev))
2020 atom_deinit_dev(ndev);
2021 else if (pdev_is_xeon(pdev))
2022 xeon_deinit_dev(ndev);
2024 intel_ntb_deinit_pci(ndev);
2031 static void intel_ntb_pci_remove(struct pci_dev *pdev)
2033 struct intel_ntb_dev *ndev = pci_get_drvdata(pdev);
2035 ntb_unregister_device(&ndev->ntb);
2036 ndev_deinit_debugfs(ndev);
2037 if (pdev_is_atom(pdev))
2038 atom_deinit_dev(ndev);
2039 else if (pdev_is_xeon(pdev))
2040 xeon_deinit_dev(ndev);
2041 intel_ntb_deinit_pci(ndev);
2045 static const struct intel_ntb_reg atom_reg = {
2046 .poll_link = atom_poll_link,
2047 .link_is_up = atom_link_is_up,
2048 .db_ioread = atom_db_ioread,
2049 .db_iowrite = atom_db_iowrite,
2050 .db_size = sizeof(u64),
2051 .ntb_ctl = ATOM_NTBCNTL_OFFSET,
2055 static const struct intel_ntb_alt_reg atom_pri_reg = {
2056 .db_bell = ATOM_PDOORBELL_OFFSET,
2057 .db_mask = ATOM_PDBMSK_OFFSET,
2058 .spad = ATOM_SPAD_OFFSET,
2061 static const struct intel_ntb_alt_reg atom_b2b_reg = {
2062 .db_bell = ATOM_B2B_DOORBELL_OFFSET,
2063 .spad = ATOM_B2B_SPAD_OFFSET,
2066 static const struct intel_ntb_xlat_reg atom_sec_xlat = {
2067 /* FIXME : .bar0_base = ATOM_SBAR0BASE_OFFSET, */
2068 /* FIXME : .bar2_limit = ATOM_SBAR2LMT_OFFSET, */
2069 .bar2_xlat = ATOM_SBAR2XLAT_OFFSET,
2072 static const struct intel_ntb_reg xeon_reg = {
2073 .poll_link = xeon_poll_link,
2074 .link_is_up = xeon_link_is_up,
2075 .db_ioread = xeon_db_ioread,
2076 .db_iowrite = xeon_db_iowrite,
2077 .db_size = sizeof(u32),
2078 .ntb_ctl = XEON_NTBCNTL_OFFSET,
2079 .mw_bar = {2, 4, 5},
2082 static const struct intel_ntb_alt_reg xeon_pri_reg = {
2083 .db_bell = XEON_PDOORBELL_OFFSET,
2084 .db_mask = XEON_PDBMSK_OFFSET,
2085 .spad = XEON_SPAD_OFFSET,
2088 static const struct intel_ntb_alt_reg xeon_sec_reg = {
2089 .db_bell = XEON_SDOORBELL_OFFSET,
2090 .db_mask = XEON_SDBMSK_OFFSET,
2091 /* second half of the scratchpads */
2092 .spad = XEON_SPAD_OFFSET + (XEON_SPAD_COUNT << 1),
2095 static const struct intel_ntb_alt_reg xeon_b2b_reg = {
2096 .db_bell = XEON_B2B_DOORBELL_OFFSET,
2097 .spad = XEON_B2B_SPAD_OFFSET,
2100 static const struct intel_ntb_xlat_reg xeon_pri_xlat = {
2101 /* Note: no primary .bar0_base visible to the secondary side.
2103 * The secondary side cannot get the base address stored in primary
2104 * bars. The base address is necessary to set the limit register to
2105 * any value other than zero, or unlimited.
2107 * WITHOUT THE BASE ADDRESS, THE SECONDARY SIDE CANNOT DISABLE the
2108 * window by setting the limit equal to base, nor can it limit the size
2109 * of the memory window by setting the limit to base + size.
2111 .bar2_limit = XEON_PBAR23LMT_OFFSET,
2112 .bar2_xlat = XEON_PBAR23XLAT_OFFSET,
2115 static const struct intel_ntb_xlat_reg xeon_sec_xlat = {
2116 .bar0_base = XEON_SBAR0BASE_OFFSET,
2117 .bar2_limit = XEON_SBAR23LMT_OFFSET,
2118 .bar2_xlat = XEON_SBAR23XLAT_OFFSET,
2121 static struct intel_b2b_addr xeon_b2b_usd_addr = {
2122 .bar2_addr64 = XEON_B2B_BAR2_USD_ADDR64,
2123 .bar4_addr64 = XEON_B2B_BAR4_USD_ADDR64,
2124 .bar4_addr32 = XEON_B2B_BAR4_USD_ADDR32,
2125 .bar5_addr32 = XEON_B2B_BAR5_USD_ADDR32,
2128 static struct intel_b2b_addr xeon_b2b_dsd_addr = {
2129 .bar2_addr64 = XEON_B2B_BAR2_DSD_ADDR64,
2130 .bar4_addr64 = XEON_B2B_BAR4_DSD_ADDR64,
2131 .bar4_addr32 = XEON_B2B_BAR4_DSD_ADDR32,
2132 .bar5_addr32 = XEON_B2B_BAR5_DSD_ADDR32,
2135 /* operations for primary side of local ntb */
2136 static const struct ntb_dev_ops intel_ntb_ops = {
2137 .mw_count = intel_ntb_mw_count,
2138 .mw_get_range = intel_ntb_mw_get_range,
2139 .mw_set_trans = intel_ntb_mw_set_trans,
2140 .link_is_up = intel_ntb_link_is_up,
2141 .link_enable = intel_ntb_link_enable,
2142 .link_disable = intel_ntb_link_disable,
2143 .db_is_unsafe = intel_ntb_db_is_unsafe,
2144 .db_valid_mask = intel_ntb_db_valid_mask,
2145 .db_vector_count = intel_ntb_db_vector_count,
2146 .db_vector_mask = intel_ntb_db_vector_mask,
2147 .db_read = intel_ntb_db_read,
2148 .db_clear = intel_ntb_db_clear,
2149 .db_set_mask = intel_ntb_db_set_mask,
2150 .db_clear_mask = intel_ntb_db_clear_mask,
2151 .peer_db_addr = intel_ntb_peer_db_addr,
2152 .peer_db_set = intel_ntb_peer_db_set,
2153 .spad_is_unsafe = intel_ntb_spad_is_unsafe,
2154 .spad_count = intel_ntb_spad_count,
2155 .spad_read = intel_ntb_spad_read,
2156 .spad_write = intel_ntb_spad_write,
2157 .peer_spad_addr = intel_ntb_peer_spad_addr,
2158 .peer_spad_read = intel_ntb_peer_spad_read,
2159 .peer_spad_write = intel_ntb_peer_spad_write,
2162 static const struct file_operations intel_ntb_debugfs_info = {
2163 .owner = THIS_MODULE,
2164 .open = simple_open,
2165 .read = ndev_debugfs_read,
2168 static const struct pci_device_id intel_ntb_pci_tbl[] = {
2169 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)},
2170 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
2171 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
2172 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
2173 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
2174 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
2175 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
2176 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
2177 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
2178 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
2179 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
2180 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
2181 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
2184 MODULE_DEVICE_TABLE(pci, intel_ntb_pci_tbl);
2186 static struct pci_driver intel_ntb_pci_driver = {
2187 .name = KBUILD_MODNAME,
2188 .id_table = intel_ntb_pci_tbl,
2189 .probe = intel_ntb_pci_probe,
2190 .remove = intel_ntb_pci_remove,
2193 static int __init intel_ntb_pci_driver_init(void)
2195 if (debugfs_initialized())
2196 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
2198 return pci_register_driver(&intel_ntb_pci_driver);
2200 module_init(intel_ntb_pci_driver_init);
2202 static void __exit intel_ntb_pci_driver_exit(void)
2204 pci_unregister_driver(&intel_ntb_pci_driver);
2206 debugfs_remove_recursive(debugfs_dir);
2208 module_exit(intel_ntb_pci_driver_exit);