2 * Intel MIC Platform Software Stack (MPSS)
4 * Copyright(c) 2013 Intel Corporation.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License, version 2, as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
15 * The full GNU General Public License is included in this distribution in
16 * the file called "COPYING".
18 * Intel MIC Host driver.
21 #include <linux/pci.h>
22 #include <linux/interrupt.h>
24 #include "../common/mic_dev.h"
25 #include "mic_device.h"
27 static irqreturn_t mic_thread_fn(int irq, void *dev)
29 struct mic_device *mdev = dev;
30 struct mic_intr_info *intr_info = mdev->intr_info;
31 struct mic_irq_info *irq_info = &mdev->irq_info;
32 struct mic_intr_cb *intr_cb;
33 struct pci_dev *pdev = container_of(mdev->sdev->parent,
37 spin_lock(&irq_info->mic_thread_lock);
38 for (i = intr_info->intr_start_idx[MIC_INTR_DB];
39 i < intr_info->intr_len[MIC_INTR_DB]; i++)
40 if (test_and_clear_bit(i, &irq_info->mask)) {
41 list_for_each_entry(intr_cb, &irq_info->cb_list[i],
43 if (intr_cb->thread_fn)
44 intr_cb->thread_fn(pdev->irq,
47 spin_unlock(&irq_info->mic_thread_lock);
51 * mic_interrupt - Generic interrupt handler for
52 * MSI and INTx based interrupts.
54 static irqreturn_t mic_interrupt(int irq, void *dev)
56 struct mic_device *mdev = dev;
57 struct mic_intr_info *intr_info = mdev->intr_info;
58 struct mic_irq_info *irq_info = &mdev->irq_info;
59 struct mic_intr_cb *intr_cb;
60 struct pci_dev *pdev = container_of(mdev->sdev->parent,
65 mask = mdev->ops->ack_interrupt(mdev);
69 spin_lock(&irq_info->mic_intr_lock);
70 for (i = intr_info->intr_start_idx[MIC_INTR_DB];
71 i < intr_info->intr_len[MIC_INTR_DB]; i++)
73 list_for_each_entry(intr_cb, &irq_info->cb_list[i],
76 intr_cb->handler(pdev->irq,
78 set_bit(i, &irq_info->mask);
80 spin_unlock(&irq_info->mic_intr_lock);
81 return IRQ_WAKE_THREAD;
84 /* Return the interrupt offset from the index. Index is 0 based. */
85 static u16 mic_map_src_to_offset(struct mic_device *mdev,
86 int intr_src, enum mic_intr_type type)
88 if (type >= MIC_NUM_INTR_TYPES)
89 return MIC_NUM_OFFSETS;
90 if (intr_src >= mdev->intr_info->intr_len[type])
91 return MIC_NUM_OFFSETS;
93 return mdev->intr_info->intr_start_idx[type] + intr_src;
96 /* Return next available msix_entry. */
97 static struct msix_entry *mic_get_available_vector(struct mic_device *mdev)
100 struct mic_irq_info *info = &mdev->irq_info;
102 for (i = 0; i < info->num_vectors; i++)
103 if (!info->mic_msi_map[i])
104 return &info->msix_entries[i];
109 * mic_register_intr_callback - Register a callback handler for the
112 * @mdev: pointer to the mic_device instance
113 * @idx: The source id to be registered.
114 * @handler: The function to be called when the source id receives
116 * @thread_fn: thread fn. corresponding to the handler
117 * @data: Private data of the requester.
118 * Return the callback structure that was registered or an
119 * appropriate error on failure.
121 static struct mic_intr_cb *mic_register_intr_callback(struct mic_device *mdev,
122 u8 idx, irq_handler_t handler, irq_handler_t thread_fn,
125 struct mic_intr_cb *intr_cb;
128 intr_cb = kmalloc(sizeof(*intr_cb), GFP_KERNEL);
131 return ERR_PTR(-ENOMEM);
133 intr_cb->handler = handler;
134 intr_cb->thread_fn = thread_fn;
135 intr_cb->data = data;
136 intr_cb->cb_id = ida_simple_get(&mdev->irq_info.cb_ida,
138 if (intr_cb->cb_id < 0) {
143 spin_lock(&mdev->irq_info.mic_thread_lock);
144 spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);
145 list_add_tail(&intr_cb->list, &mdev->irq_info.cb_list[idx]);
146 spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);
147 spin_unlock(&mdev->irq_info.mic_thread_lock);
156 * mic_unregister_intr_callback - Unregister the callback handler
157 * identified by its callback id.
159 * @mdev: pointer to the mic_device instance
160 * @idx: The callback structure id to be unregistered.
161 * Return the source id that was unregistered or MIC_NUM_OFFSETS if no
162 * such callback handler was found.
164 static u8 mic_unregister_intr_callback(struct mic_device *mdev, u32 idx)
166 struct list_head *pos, *tmp;
167 struct mic_intr_cb *intr_cb;
171 spin_lock(&mdev->irq_info.mic_thread_lock);
172 spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);
173 for (i = 0; i < MIC_NUM_OFFSETS; i++) {
174 list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) {
175 intr_cb = list_entry(pos, struct mic_intr_cb, list);
176 if (intr_cb->cb_id == idx) {
178 ida_simple_remove(&mdev->irq_info.cb_ida,
181 spin_unlock_irqrestore(
182 &mdev->irq_info.mic_intr_lock, flags);
183 spin_unlock(&mdev->irq_info.mic_thread_lock);
188 spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);
189 spin_unlock(&mdev->irq_info.mic_thread_lock);
190 return MIC_NUM_OFFSETS;
194 * mic_setup_msix - Initializes MSIx interrupts.
196 * @mdev: pointer to mic_device instance
199 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
201 static int mic_setup_msix(struct mic_device *mdev, struct pci_dev *pdev)
204 int entry_size = sizeof(*mdev->irq_info.msix_entries);
206 mdev->irq_info.msix_entries = kmalloc_array(MIC_MIN_MSIX,
207 entry_size, GFP_KERNEL);
208 if (!mdev->irq_info.msix_entries) {
213 for (i = 0; i < MIC_MIN_MSIX; i++)
214 mdev->irq_info.msix_entries[i].entry = i;
216 rc = pci_enable_msix_exact(pdev, mdev->irq_info.msix_entries,
219 dev_dbg(&pdev->dev, "Error enabling MSIx. rc = %d\n", rc);
220 goto err_enable_msix;
223 mdev->irq_info.num_vectors = MIC_MIN_MSIX;
224 mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) *
225 mdev->irq_info.num_vectors), GFP_KERNEL);
227 if (!mdev->irq_info.mic_msi_map) {
232 dev_dbg(mdev->sdev->parent,
233 "%d MSIx irqs setup\n", mdev->irq_info.num_vectors);
236 pci_disable_msix(pdev);
238 kfree(mdev->irq_info.msix_entries);
240 mdev->irq_info.num_vectors = 0;
245 * mic_setup_callbacks - Initialize data structures needed
246 * to handle callbacks.
248 * @mdev: pointer to mic_device instance
250 static int mic_setup_callbacks(struct mic_device *mdev)
254 mdev->irq_info.cb_list = kmalloc_array(MIC_NUM_OFFSETS,
255 sizeof(*mdev->irq_info.cb_list),
257 if (!mdev->irq_info.cb_list)
260 for (i = 0; i < MIC_NUM_OFFSETS; i++)
261 INIT_LIST_HEAD(&mdev->irq_info.cb_list[i]);
262 ida_init(&mdev->irq_info.cb_ida);
263 spin_lock_init(&mdev->irq_info.mic_intr_lock);
264 spin_lock_init(&mdev->irq_info.mic_thread_lock);
269 * mic_release_callbacks - Uninitialize data structures needed
270 * to handle callbacks.
272 * @mdev: pointer to mic_device instance
274 static void mic_release_callbacks(struct mic_device *mdev)
277 struct list_head *pos, *tmp;
278 struct mic_intr_cb *intr_cb;
281 spin_lock(&mdev->irq_info.mic_thread_lock);
282 spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);
283 for (i = 0; i < MIC_NUM_OFFSETS; i++) {
285 if (list_empty(&mdev->irq_info.cb_list[i]))
288 list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) {
289 intr_cb = list_entry(pos, struct mic_intr_cb, list);
291 ida_simple_remove(&mdev->irq_info.cb_ida,
296 spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);
297 spin_unlock(&mdev->irq_info.mic_thread_lock);
298 ida_destroy(&mdev->irq_info.cb_ida);
299 kfree(mdev->irq_info.cb_list);
303 * mic_setup_msi - Initializes MSI interrupts.
305 * @mdev: pointer to mic_device instance
306 * @pdev: PCI device structure
308 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
310 static int mic_setup_msi(struct mic_device *mdev, struct pci_dev *pdev)
314 rc = pci_enable_msi(pdev);
316 dev_dbg(&pdev->dev, "Error enabling MSI. rc = %d\n", rc);
320 mdev->irq_info.num_vectors = 1;
321 mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) *
322 mdev->irq_info.num_vectors), GFP_KERNEL);
324 if (!mdev->irq_info.mic_msi_map) {
329 rc = mic_setup_callbacks(mdev);
331 dev_err(&pdev->dev, "Error setting up callbacks\n");
335 rc = request_threaded_irq(pdev->irq, mic_interrupt, mic_thread_fn,
338 dev_err(&pdev->dev, "Error allocating MSI interrupt\n");
339 goto err_irq_req_fail;
342 dev_dbg(&pdev->dev, "%d MSI irqs setup\n", mdev->irq_info.num_vectors);
345 mic_release_callbacks(mdev);
347 kfree(mdev->irq_info.mic_msi_map);
349 pci_disable_msi(pdev);
350 mdev->irq_info.num_vectors = 0;
355 * mic_setup_intx - Initializes legacy interrupts.
357 * @mdev: pointer to mic_device instance
358 * @pdev: PCI device structure
360 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
362 static int mic_setup_intx(struct mic_device *mdev, struct pci_dev *pdev)
368 rc = mic_setup_callbacks(mdev);
370 dev_err(&pdev->dev, "Error setting up callbacks\n");
374 rc = request_threaded_irq(pdev->irq, mic_interrupt, mic_thread_fn,
375 IRQF_SHARED, "mic-intx", mdev);
379 dev_dbg(&pdev->dev, "intx irq setup\n");
382 mic_release_callbacks(mdev);
388 * mic_next_db - Retrieve the next doorbell interrupt source id.
389 * The id is picked sequentially from the available pool of
392 * @mdev: pointer to the mic_device instance.
394 * Returns the next doorbell interrupt source.
396 int mic_next_db(struct mic_device *mdev)
400 next_db = mdev->irq_info.next_avail_src %
401 mdev->intr_info->intr_len[MIC_INTR_DB];
402 mdev->irq_info.next_avail_src++;
406 #define COOKIE_ID_SHIFT 16
407 #define GET_ENTRY(cookie) ((cookie) & 0xFFFF)
408 #define GET_OFFSET(cookie) ((cookie) >> COOKIE_ID_SHIFT)
409 #define MK_COOKIE(x, y) ((x) | (y) << COOKIE_ID_SHIFT)
412 * mic_request_threaded_irq - request an irq. mic_mutex needs
413 * to be held before calling this function.
415 * @mdev: pointer to mic_device instance
416 * @handler: The callback function that handles the interrupt.
417 * The function needs to call ack_interrupts
418 * (mdev->ops->ack_interrupt(mdev)) when handling the interrupts.
419 * @thread_fn: thread fn required by request_threaded_irq.
420 * @name: The ASCII name of the callee requesting the irq.
421 * @data: private data that is returned back when calling the
423 * @intr_src: The source id of the requester. Its the doorbell id
424 * for Doorbell interrupts and DMA channel id for DMA interrupts.
425 * @type: The type of interrupt. Values defined in mic_intr_type
427 * returns: The cookie that is transparent to the caller. Passed
428 * back when calling mic_free_irq. An appropriate error code
429 * is returned on failure. Caller needs to use IS_ERR(return_val)
430 * to check for failure and PTR_ERR(return_val) to obtained the
435 mic_request_threaded_irq(struct mic_device *mdev,
436 irq_handler_t handler, irq_handler_t thread_fn,
437 const char *name, void *data, int intr_src,
438 enum mic_intr_type type)
442 struct msix_entry *msix = NULL;
443 unsigned long cookie = 0;
445 struct mic_intr_cb *intr_cb;
446 struct pci_dev *pdev = container_of(mdev->sdev->parent,
447 struct pci_dev, dev);
449 offset = mic_map_src_to_offset(mdev, intr_src, type);
450 if (offset >= MIC_NUM_OFFSETS) {
451 dev_err(mdev->sdev->parent,
452 "Error mapping index %d to a valid source id.\n",
458 if (mdev->irq_info.num_vectors > 1) {
459 msix = mic_get_available_vector(mdev);
461 dev_err(mdev->sdev->parent,
462 "No MSIx vectors available for use.\n");
467 rc = request_threaded_irq(msix->vector, handler, thread_fn,
470 dev_dbg(mdev->sdev->parent,
471 "request irq failed rc = %d\n", rc);
475 mdev->irq_info.mic_msi_map[entry] |= BIT(offset);
476 mdev->intr_ops->program_msi_to_src_map(mdev,
477 entry, offset, true);
478 cookie = MK_COOKIE(entry, offset);
479 dev_dbg(mdev->sdev->parent, "irq: %d assigned for src: %d\n",
480 msix->vector, intr_src);
482 intr_cb = mic_register_intr_callback(mdev, offset, handler,
484 if (IS_ERR(intr_cb)) {
485 dev_err(mdev->sdev->parent,
486 "No available callback entries for use\n");
487 rc = PTR_ERR(intr_cb);
492 if (pci_dev_msi_enabled(pdev)) {
493 mdev->irq_info.mic_msi_map[entry] |= (1 << offset);
494 mdev->intr_ops->program_msi_to_src_map(mdev,
495 entry, offset, true);
497 cookie = MK_COOKIE(entry, intr_cb->cb_id);
498 dev_dbg(mdev->sdev->parent, "callback %d registered for src: %d\n",
499 intr_cb->cb_id, intr_src);
501 return (struct mic_irq *)cookie;
507 * mic_free_irq - free irq. mic_mutex
508 * needs to be held before calling this function.
510 * @mdev: pointer to mic_device instance
511 * @cookie: cookie obtained during a successful call to mic_request_threaded_irq
512 * @data: private data specified by the calling function during the
513 * mic_request_threaded_irq
517 void mic_free_irq(struct mic_device *mdev,
518 struct mic_irq *cookie, void *data)
524 struct pci_dev *pdev = container_of(mdev->sdev->parent,
525 struct pci_dev, dev);
527 entry = GET_ENTRY((unsigned long)cookie);
528 offset = GET_OFFSET((unsigned long)cookie);
529 if (mdev->irq_info.num_vectors > 1) {
530 if (entry >= mdev->irq_info.num_vectors) {
531 dev_warn(mdev->sdev->parent,
532 "entry %d should be < num_irq %d\n",
533 entry, mdev->irq_info.num_vectors);
536 irq = mdev->irq_info.msix_entries[entry].vector;
538 mdev->irq_info.mic_msi_map[entry] &= ~(BIT(offset));
539 mdev->intr_ops->program_msi_to_src_map(mdev,
540 entry, offset, false);
542 dev_dbg(mdev->sdev->parent, "irq: %d freed\n", irq);
545 src_id = mic_unregister_intr_callback(mdev, offset);
546 if (src_id >= MIC_NUM_OFFSETS) {
547 dev_warn(mdev->sdev->parent, "Error unregistering callback\n");
550 if (pci_dev_msi_enabled(pdev)) {
551 mdev->irq_info.mic_msi_map[entry] &= ~(BIT(src_id));
552 mdev->intr_ops->program_msi_to_src_map(mdev,
553 entry, src_id, false);
555 dev_dbg(mdev->sdev->parent, "callback %d unregistered for src: %d\n",
561 * mic_setup_interrupts - Initializes interrupts.
563 * @mdev: pointer to mic_device instance
564 * @pdev: PCI device structure
566 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
568 int mic_setup_interrupts(struct mic_device *mdev, struct pci_dev *pdev)
572 rc = mic_setup_msix(mdev, pdev);
576 rc = mic_setup_msi(mdev, pdev);
580 rc = mic_setup_intx(mdev, pdev);
582 dev_err(mdev->sdev->parent, "no usable interrupts\n");
586 mdev->intr_ops->enable_interrupts(mdev);
591 * mic_free_interrupts - Frees interrupts setup by mic_setup_interrupts
593 * @mdev: pointer to mic_device instance
594 * @pdev: PCI device structure
598 void mic_free_interrupts(struct mic_device *mdev, struct pci_dev *pdev)
602 mdev->intr_ops->disable_interrupts(mdev);
603 if (mdev->irq_info.num_vectors > 1) {
604 for (i = 0; i < mdev->irq_info.num_vectors; i++) {
605 if (mdev->irq_info.mic_msi_map[i])
606 dev_warn(&pdev->dev, "irq %d may still be in use.\n",
607 mdev->irq_info.msix_entries[i].vector);
609 kfree(mdev->irq_info.mic_msi_map);
610 kfree(mdev->irq_info.msix_entries);
611 pci_disable_msix(pdev);
613 if (pci_dev_msi_enabled(pdev)) {
614 free_irq(pdev->irq, mdev);
615 kfree(mdev->irq_info.mic_msi_map);
616 pci_disable_msi(pdev);
618 free_irq(pdev->irq, mdev);
620 mic_release_callbacks(mdev);
625 * mic_intr_restore - Restore MIC interrupt registers.
627 * @mdev: pointer to mic_device instance.
629 * Restore the interrupt registers to values previously
630 * stored in the SW data structures. mic_mutex needs to
631 * be held before calling this function.
635 void mic_intr_restore(struct mic_device *mdev)
638 struct pci_dev *pdev = container_of(mdev->sdev->parent,
639 struct pci_dev, dev);
641 if (!pci_dev_msi_enabled(pdev))
644 for (entry = 0; entry < mdev->irq_info.num_vectors; entry++) {
645 for (offset = 0; offset < MIC_NUM_OFFSETS; offset++) {
646 if (mdev->irq_info.mic_msi_map[entry] & BIT(offset))
647 mdev->intr_ops->program_msi_to_src_map(mdev,
648 entry, offset, true);