2 * Intel MIC Platform Software Stack (MPSS)
4 * Copyright(c) 2013 Intel Corporation.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License, version 2, as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
15 * The full GNU General Public License is included in this distribution in
16 * the file called "COPYING".
18 * Intel MIC Host driver.
21 #include <linux/pci.h>
22 #include <linux/interrupt.h>
24 #include "../common/mic_dev.h"
25 #include "mic_device.h"
27 static irqreturn_t mic_thread_fn(int irq, void *dev)
29 struct mic_device *mdev = dev;
30 struct mic_intr_info *intr_info = mdev->intr_info;
31 struct mic_irq_info *irq_info = &mdev->irq_info;
32 struct mic_intr_cb *intr_cb;
33 struct pci_dev *pdev = mdev->pdev;
36 spin_lock(&irq_info->mic_thread_lock);
37 for (i = intr_info->intr_start_idx[MIC_INTR_DB];
38 i < intr_info->intr_len[MIC_INTR_DB]; i++)
39 if (test_and_clear_bit(i, &irq_info->mask)) {
40 list_for_each_entry(intr_cb, &irq_info->cb_list[i],
42 if (intr_cb->thread_fn)
43 intr_cb->thread_fn(pdev->irq,
46 spin_unlock(&irq_info->mic_thread_lock);
50 * mic_interrupt - Generic interrupt handler for
51 * MSI and INTx based interrupts.
53 static irqreturn_t mic_interrupt(int irq, void *dev)
55 struct mic_device *mdev = dev;
56 struct mic_intr_info *intr_info = mdev->intr_info;
57 struct mic_irq_info *irq_info = &mdev->irq_info;
58 struct mic_intr_cb *intr_cb;
59 struct pci_dev *pdev = mdev->pdev;
63 mask = mdev->ops->ack_interrupt(mdev);
67 spin_lock(&irq_info->mic_intr_lock);
68 for (i = intr_info->intr_start_idx[MIC_INTR_DB];
69 i < intr_info->intr_len[MIC_INTR_DB]; i++)
71 list_for_each_entry(intr_cb, &irq_info->cb_list[i],
74 intr_cb->handler(pdev->irq,
76 set_bit(i, &irq_info->mask);
78 spin_unlock(&irq_info->mic_intr_lock);
79 return IRQ_WAKE_THREAD;
82 /* Return the interrupt offset from the index. Index is 0 based. */
83 static u16 mic_map_src_to_offset(struct mic_device *mdev,
84 int intr_src, enum mic_intr_type type)
86 if (type >= MIC_NUM_INTR_TYPES)
87 return MIC_NUM_OFFSETS;
88 if (intr_src >= mdev->intr_info->intr_len[type])
89 return MIC_NUM_OFFSETS;
91 return mdev->intr_info->intr_start_idx[type] + intr_src;
94 /* Return next available msix_entry. */
95 static struct msix_entry *mic_get_available_vector(struct mic_device *mdev)
98 struct mic_irq_info *info = &mdev->irq_info;
100 for (i = 0; i < info->num_vectors; i++)
101 if (!info->mic_msi_map[i])
102 return &info->msix_entries[i];
107 * mic_register_intr_callback - Register a callback handler for the
110 * @mdev: pointer to the mic_device instance
111 * @idx: The source id to be registered.
112 * @handler: The function to be called when the source id receives
114 * @thread_fn: thread fn. corresponding to the handler
115 * @data: Private data of the requester.
116 * Return the callback structure that was registered or an
117 * appropriate error on failure.
119 static struct mic_intr_cb *mic_register_intr_callback(struct mic_device *mdev,
120 u8 idx, irq_handler_t handler, irq_handler_t thread_fn,
123 struct mic_intr_cb *intr_cb;
126 intr_cb = kmalloc(sizeof(*intr_cb), GFP_KERNEL);
129 return ERR_PTR(-ENOMEM);
131 intr_cb->handler = handler;
132 intr_cb->thread_fn = thread_fn;
133 intr_cb->data = data;
134 intr_cb->cb_id = ida_simple_get(&mdev->irq_info.cb_ida,
136 if (intr_cb->cb_id < 0) {
141 spin_lock(&mdev->irq_info.mic_thread_lock);
142 spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);
143 list_add_tail(&intr_cb->list, &mdev->irq_info.cb_list[idx]);
144 spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);
145 spin_unlock(&mdev->irq_info.mic_thread_lock);
154 * mic_unregister_intr_callback - Unregister the callback handler
155 * identified by its callback id.
157 * @mdev: pointer to the mic_device instance
158 * @idx: The callback structure id to be unregistered.
159 * Return the source id that was unregistered or MIC_NUM_OFFSETS if no
160 * such callback handler was found.
162 static u8 mic_unregister_intr_callback(struct mic_device *mdev, u32 idx)
164 struct list_head *pos, *tmp;
165 struct mic_intr_cb *intr_cb;
169 spin_lock(&mdev->irq_info.mic_thread_lock);
170 spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);
171 for (i = 0; i < MIC_NUM_OFFSETS; i++) {
172 list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) {
173 intr_cb = list_entry(pos, struct mic_intr_cb, list);
174 if (intr_cb->cb_id == idx) {
176 ida_simple_remove(&mdev->irq_info.cb_ida,
179 spin_unlock_irqrestore(
180 &mdev->irq_info.mic_intr_lock, flags);
181 spin_unlock(&mdev->irq_info.mic_thread_lock);
186 spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);
187 spin_unlock(&mdev->irq_info.mic_thread_lock);
188 return MIC_NUM_OFFSETS;
192 * mic_setup_msix - Initializes MSIx interrupts.
194 * @mdev: pointer to mic_device instance
197 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
199 static int mic_setup_msix(struct mic_device *mdev, struct pci_dev *pdev)
202 int entry_size = sizeof(*mdev->irq_info.msix_entries);
204 mdev->irq_info.msix_entries = kmalloc_array(MIC_MIN_MSIX,
205 entry_size, GFP_KERNEL);
206 if (!mdev->irq_info.msix_entries) {
211 for (i = 0; i < MIC_MIN_MSIX; i++)
212 mdev->irq_info.msix_entries[i].entry = i;
214 rc = pci_enable_msix_exact(pdev, mdev->irq_info.msix_entries,
217 dev_dbg(&pdev->dev, "Error enabling MSIx. rc = %d\n", rc);
218 goto err_enable_msix;
221 mdev->irq_info.num_vectors = MIC_MIN_MSIX;
222 mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) *
223 mdev->irq_info.num_vectors), GFP_KERNEL);
225 if (!mdev->irq_info.mic_msi_map) {
230 dev_dbg(&mdev->pdev->dev,
231 "%d MSIx irqs setup\n", mdev->irq_info.num_vectors);
234 pci_disable_msix(pdev);
236 kfree(mdev->irq_info.msix_entries);
238 mdev->irq_info.num_vectors = 0;
243 * mic_setup_callbacks - Initialize data structures needed
244 * to handle callbacks.
246 * @mdev: pointer to mic_device instance
248 static int mic_setup_callbacks(struct mic_device *mdev)
252 mdev->irq_info.cb_list = kmalloc_array(MIC_NUM_OFFSETS,
253 sizeof(*mdev->irq_info.cb_list),
255 if (!mdev->irq_info.cb_list)
258 for (i = 0; i < MIC_NUM_OFFSETS; i++)
259 INIT_LIST_HEAD(&mdev->irq_info.cb_list[i]);
260 ida_init(&mdev->irq_info.cb_ida);
261 spin_lock_init(&mdev->irq_info.mic_intr_lock);
262 spin_lock_init(&mdev->irq_info.mic_thread_lock);
267 * mic_release_callbacks - Uninitialize data structures needed
268 * to handle callbacks.
270 * @mdev: pointer to mic_device instance
272 static void mic_release_callbacks(struct mic_device *mdev)
275 struct list_head *pos, *tmp;
276 struct mic_intr_cb *intr_cb;
279 spin_lock(&mdev->irq_info.mic_thread_lock);
280 spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);
281 for (i = 0; i < MIC_NUM_OFFSETS; i++) {
282 if (list_empty(&mdev->irq_info.cb_list[i]))
285 list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) {
286 intr_cb = list_entry(pos, struct mic_intr_cb, list);
288 ida_simple_remove(&mdev->irq_info.cb_ida,
293 spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);
294 spin_unlock(&mdev->irq_info.mic_thread_lock);
295 ida_destroy(&mdev->irq_info.cb_ida);
296 kfree(mdev->irq_info.cb_list);
300 * mic_setup_msi - Initializes MSI interrupts.
302 * @mdev: pointer to mic_device instance
303 * @pdev: PCI device structure
305 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
307 static int mic_setup_msi(struct mic_device *mdev, struct pci_dev *pdev)
311 rc = pci_enable_msi(pdev);
313 dev_dbg(&pdev->dev, "Error enabling MSI. rc = %d\n", rc);
317 mdev->irq_info.num_vectors = 1;
318 mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) *
319 mdev->irq_info.num_vectors), GFP_KERNEL);
321 if (!mdev->irq_info.mic_msi_map) {
326 rc = mic_setup_callbacks(mdev);
328 dev_err(&pdev->dev, "Error setting up callbacks\n");
332 rc = request_threaded_irq(pdev->irq, mic_interrupt, mic_thread_fn,
335 dev_err(&pdev->dev, "Error allocating MSI interrupt\n");
336 goto err_irq_req_fail;
339 dev_dbg(&pdev->dev, "%d MSI irqs setup\n", mdev->irq_info.num_vectors);
342 mic_release_callbacks(mdev);
344 kfree(mdev->irq_info.mic_msi_map);
346 pci_disable_msi(pdev);
347 mdev->irq_info.num_vectors = 0;
352 * mic_setup_intx - Initializes legacy interrupts.
354 * @mdev: pointer to mic_device instance
355 * @pdev: PCI device structure
357 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
359 static int mic_setup_intx(struct mic_device *mdev, struct pci_dev *pdev)
365 rc = mic_setup_callbacks(mdev);
367 dev_err(&pdev->dev, "Error setting up callbacks\n");
371 rc = request_threaded_irq(pdev->irq, mic_interrupt, mic_thread_fn,
372 IRQF_SHARED, "mic-intx", mdev);
376 dev_dbg(&pdev->dev, "intx irq setup\n");
379 mic_release_callbacks(mdev);
385 * mic_next_db - Retrieve the next doorbell interrupt source id.
386 * The id is picked sequentially from the available pool of
389 * @mdev: pointer to the mic_device instance.
391 * Returns the next doorbell interrupt source.
393 int mic_next_db(struct mic_device *mdev)
397 next_db = mdev->irq_info.next_avail_src %
398 mdev->intr_info->intr_len[MIC_INTR_DB];
399 mdev->irq_info.next_avail_src++;
403 #define COOKIE_ID_SHIFT 16
404 #define GET_ENTRY(cookie) ((cookie) & 0xFFFF)
405 #define GET_OFFSET(cookie) ((cookie) >> COOKIE_ID_SHIFT)
406 #define MK_COOKIE(x, y) ((x) | (y) << COOKIE_ID_SHIFT)
409 * mic_request_threaded_irq - request an irq. mic_mutex needs
410 * to be held before calling this function.
412 * @mdev: pointer to mic_device instance
413 * @handler: The callback function that handles the interrupt.
414 * The function needs to call ack_interrupts
415 * (mdev->ops->ack_interrupt(mdev)) when handling the interrupts.
416 * @thread_fn: thread fn required by request_threaded_irq.
417 * @name: The ASCII name of the callee requesting the irq.
418 * @data: private data that is returned back when calling the
420 * @intr_src: The source id of the requester. Its the doorbell id
421 * for Doorbell interrupts and DMA channel id for DMA interrupts.
422 * @type: The type of interrupt. Values defined in mic_intr_type
424 * returns: The cookie that is transparent to the caller. Passed
425 * back when calling mic_free_irq. An appropriate error code
426 * is returned on failure. Caller needs to use IS_ERR(return_val)
427 * to check for failure and PTR_ERR(return_val) to obtained the
432 mic_request_threaded_irq(struct mic_device *mdev,
433 irq_handler_t handler, irq_handler_t thread_fn,
434 const char *name, void *data, int intr_src,
435 enum mic_intr_type type)
439 struct msix_entry *msix = NULL;
440 unsigned long cookie = 0;
442 struct mic_intr_cb *intr_cb;
443 struct pci_dev *pdev = mdev->pdev;
445 offset = mic_map_src_to_offset(mdev, intr_src, type);
446 if (offset >= MIC_NUM_OFFSETS) {
447 dev_err(&mdev->pdev->dev,
448 "Error mapping index %d to a valid source id.\n",
454 if (mdev->irq_info.num_vectors > 1) {
455 msix = mic_get_available_vector(mdev);
457 dev_err(&mdev->pdev->dev,
458 "No MSIx vectors available for use.\n");
463 rc = request_threaded_irq(msix->vector, handler, thread_fn,
466 dev_dbg(&mdev->pdev->dev,
467 "request irq failed rc = %d\n", rc);
471 mdev->irq_info.mic_msi_map[entry] |= BIT(offset);
472 mdev->intr_ops->program_msi_to_src_map(mdev,
473 entry, offset, true);
474 cookie = MK_COOKIE(entry, offset);
475 dev_dbg(&mdev->pdev->dev, "irq: %d assigned for src: %d\n",
476 msix->vector, intr_src);
478 intr_cb = mic_register_intr_callback(mdev, offset, handler,
480 if (IS_ERR(intr_cb)) {
481 dev_err(&mdev->pdev->dev,
482 "No available callback entries for use\n");
483 rc = PTR_ERR(intr_cb);
488 if (pci_dev_msi_enabled(pdev)) {
489 mdev->irq_info.mic_msi_map[entry] |= (1 << offset);
490 mdev->intr_ops->program_msi_to_src_map(mdev,
491 entry, offset, true);
493 cookie = MK_COOKIE(entry, intr_cb->cb_id);
494 dev_dbg(&mdev->pdev->dev, "callback %d registered for src: %d\n",
495 intr_cb->cb_id, intr_src);
497 return (struct mic_irq *)cookie;
503 * mic_free_irq - free irq. mic_mutex
504 * needs to be held before calling this function.
506 * @mdev: pointer to mic_device instance
507 * @cookie: cookie obtained during a successful call to mic_request_threaded_irq
508 * @data: private data specified by the calling function during the
509 * mic_request_threaded_irq
513 void mic_free_irq(struct mic_device *mdev,
514 struct mic_irq *cookie, void *data)
520 struct pci_dev *pdev = mdev->pdev;
522 entry = GET_ENTRY((unsigned long)cookie);
523 offset = GET_OFFSET((unsigned long)cookie);
524 if (mdev->irq_info.num_vectors > 1) {
525 if (entry >= mdev->irq_info.num_vectors) {
526 dev_warn(&mdev->pdev->dev,
527 "entry %d should be < num_irq %d\n",
528 entry, mdev->irq_info.num_vectors);
531 irq = mdev->irq_info.msix_entries[entry].vector;
533 mdev->irq_info.mic_msi_map[entry] &= ~(BIT(offset));
534 mdev->intr_ops->program_msi_to_src_map(mdev,
535 entry, offset, false);
537 dev_dbg(&mdev->pdev->dev, "irq: %d freed\n", irq);
540 src_id = mic_unregister_intr_callback(mdev, offset);
541 if (src_id >= MIC_NUM_OFFSETS) {
542 dev_warn(&mdev->pdev->dev, "Error unregistering callback\n");
545 if (pci_dev_msi_enabled(pdev)) {
546 mdev->irq_info.mic_msi_map[entry] &= ~(BIT(src_id));
547 mdev->intr_ops->program_msi_to_src_map(mdev,
548 entry, src_id, false);
550 dev_dbg(&mdev->pdev->dev, "callback %d unregistered for src: %d\n",
556 * mic_setup_interrupts - Initializes interrupts.
558 * @mdev: pointer to mic_device instance
559 * @pdev: PCI device structure
561 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
563 int mic_setup_interrupts(struct mic_device *mdev, struct pci_dev *pdev)
567 rc = mic_setup_msix(mdev, pdev);
571 rc = mic_setup_msi(mdev, pdev);
575 rc = mic_setup_intx(mdev, pdev);
577 dev_err(&mdev->pdev->dev, "no usable interrupts\n");
581 mdev->intr_ops->enable_interrupts(mdev);
586 * mic_free_interrupts - Frees interrupts setup by mic_setup_interrupts
588 * @mdev: pointer to mic_device instance
589 * @pdev: PCI device structure
593 void mic_free_interrupts(struct mic_device *mdev, struct pci_dev *pdev)
597 mdev->intr_ops->disable_interrupts(mdev);
598 if (mdev->irq_info.num_vectors > 1) {
599 for (i = 0; i < mdev->irq_info.num_vectors; i++) {
600 if (mdev->irq_info.mic_msi_map[i])
601 dev_warn(&pdev->dev, "irq %d may still be in use.\n",
602 mdev->irq_info.msix_entries[i].vector);
604 kfree(mdev->irq_info.mic_msi_map);
605 kfree(mdev->irq_info.msix_entries);
606 pci_disable_msix(pdev);
608 if (pci_dev_msi_enabled(pdev)) {
609 free_irq(pdev->irq, mdev);
610 kfree(mdev->irq_info.mic_msi_map);
611 pci_disable_msi(pdev);
613 free_irq(pdev->irq, mdev);
615 mic_release_callbacks(mdev);
620 * mic_intr_restore - Restore MIC interrupt registers.
622 * @mdev: pointer to mic_device instance.
624 * Restore the interrupt registers to values previously
625 * stored in the SW data structures. mic_mutex needs to
626 * be held before calling this function.
630 void mic_intr_restore(struct mic_device *mdev)
633 struct pci_dev *pdev = mdev->pdev;
635 if (!pci_dev_msi_enabled(pdev))
638 for (entry = 0; entry < mdev->irq_info.num_vectors; entry++) {
639 for (offset = 0; offset < MIC_NUM_OFFSETS; offset++) {
640 if (mdev->irq_info.mic_msi_map[entry] & BIT(offset))
641 mdev->intr_ops->program_msi_to_src_map(mdev,
642 entry, offset, true);