]>
git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blob - drivers/misc/mic/host/mic_intr.c
b4ca6c884d198d68dcee378083db37decfa907e8
2 * Intel MIC Platform Software Stack (MPSS)
4 * Copyright(c) 2013 Intel Corporation.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License, version 2, as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
15 * The full GNU General Public License is included in this distribution in
16 * the file called "COPYING".
18 * Intel MIC Host driver.
21 #include <linux/pci.h>
22 #include <linux/interrupt.h>
24 #include "../common/mic_dev.h"
25 #include "mic_device.h"
27 static irqreturn_t
mic_thread_fn(int irq
, void *dev
)
29 struct mic_device
*mdev
= dev
;
30 struct mic_intr_info
*intr_info
= mdev
->intr_info
;
31 struct mic_irq_info
*irq_info
= &mdev
->irq_info
;
32 struct mic_intr_cb
*intr_cb
;
33 struct pci_dev
*pdev
= container_of(mdev
->sdev
->parent
,
37 spin_lock(&irq_info
->mic_thread_lock
);
38 for (i
= intr_info
->intr_start_idx
[MIC_INTR_DB
];
39 i
< intr_info
->intr_len
[MIC_INTR_DB
]; i
++)
40 if (test_and_clear_bit(i
, &irq_info
->mask
)) {
41 list_for_each_entry(intr_cb
, &irq_info
->cb_list
[i
],
43 if (intr_cb
->thread_fn
)
44 intr_cb
->thread_fn(pdev
->irq
,
47 spin_unlock(&irq_info
->mic_thread_lock
);
51 * mic_interrupt - Generic interrupt handler for
52 * MSI and INTx based interrupts.
54 static irqreturn_t
mic_interrupt(int irq
, void *dev
)
56 struct mic_device
*mdev
= dev
;
57 struct mic_intr_info
*intr_info
= mdev
->intr_info
;
58 struct mic_irq_info
*irq_info
= &mdev
->irq_info
;
59 struct mic_intr_cb
*intr_cb
;
60 struct pci_dev
*pdev
= container_of(mdev
->sdev
->parent
,
65 mask
= mdev
->ops
->ack_interrupt(mdev
);
69 spin_lock(&irq_info
->mic_intr_lock
);
70 for (i
= intr_info
->intr_start_idx
[MIC_INTR_DB
];
71 i
< intr_info
->intr_len
[MIC_INTR_DB
]; i
++)
73 list_for_each_entry(intr_cb
, &irq_info
->cb_list
[i
],
76 intr_cb
->handler(pdev
->irq
,
78 set_bit(i
, &irq_info
->mask
);
80 spin_unlock(&irq_info
->mic_intr_lock
);
81 return IRQ_WAKE_THREAD
;
84 /* Return the interrupt offset from the index. Index is 0 based. */
85 static u16
mic_map_src_to_offset(struct mic_device
*mdev
,
86 int intr_src
, enum mic_intr_type type
)
88 if (type
>= MIC_NUM_INTR_TYPES
)
89 return MIC_NUM_OFFSETS
;
90 if (intr_src
>= mdev
->intr_info
->intr_len
[type
])
91 return MIC_NUM_OFFSETS
;
93 return mdev
->intr_info
->intr_start_idx
[type
] + intr_src
;
96 /* Return next available msix_entry. */
97 static struct msix_entry
*mic_get_available_vector(struct mic_device
*mdev
)
100 struct mic_irq_info
*info
= &mdev
->irq_info
;
102 for (i
= 0; i
< info
->num_vectors
; i
++)
103 if (!info
->mic_msi_map
[i
])
104 return &info
->msix_entries
[i
];
109 * mic_register_intr_callback - Register a callback handler for the
112 * @mdev: pointer to the mic_device instance
113 * @idx: The source id to be registered.
114 * @handler: The function to be called when the source id receives
116 * @thread_fn: thread fn. corresponding to the handler
117 * @data: Private data of the requester.
118 * Return the callback structure that was registered or an
119 * appropriate error on failure.
121 static struct mic_intr_cb
*mic_register_intr_callback(struct mic_device
*mdev
,
122 u8 idx
, irq_handler_t handler
, irq_handler_t thread_fn
,
125 struct mic_intr_cb
*intr_cb
;
128 intr_cb
= kmalloc(sizeof(*intr_cb
), GFP_KERNEL
);
131 return ERR_PTR(-ENOMEM
);
133 intr_cb
->handler
= handler
;
134 intr_cb
->thread_fn
= thread_fn
;
135 intr_cb
->data
= data
;
136 intr_cb
->cb_id
= ida_simple_get(&mdev
->irq_info
.cb_ida
,
138 if (intr_cb
->cb_id
< 0) {
143 spin_lock(&mdev
->irq_info
.mic_thread_lock
);
144 spin_lock_irqsave(&mdev
->irq_info
.mic_intr_lock
, flags
);
145 list_add_tail(&intr_cb
->list
, &mdev
->irq_info
.cb_list
[idx
]);
146 spin_unlock_irqrestore(&mdev
->irq_info
.mic_intr_lock
, flags
);
147 spin_unlock(&mdev
->irq_info
.mic_thread_lock
);
156 * mic_unregister_intr_callback - Unregister the callback handler
157 * identified by its callback id.
159 * @mdev: pointer to the mic_device instance
160 * @idx: The callback structure id to be unregistered.
161 * Return the source id that was unregistered or MIC_NUM_OFFSETS if no
162 * such callback handler was found.
164 static u8
mic_unregister_intr_callback(struct mic_device
*mdev
, u32 idx
)
166 struct list_head
*pos
, *tmp
;
167 struct mic_intr_cb
*intr_cb
;
171 spin_lock(&mdev
->irq_info
.mic_thread_lock
);
172 spin_lock_irqsave(&mdev
->irq_info
.mic_intr_lock
, flags
);
173 for (i
= 0; i
< MIC_NUM_OFFSETS
; i
++) {
174 list_for_each_safe(pos
, tmp
, &mdev
->irq_info
.cb_list
[i
]) {
175 intr_cb
= list_entry(pos
, struct mic_intr_cb
, list
);
176 if (intr_cb
->cb_id
== idx
) {
178 ida_simple_remove(&mdev
->irq_info
.cb_ida
,
181 spin_unlock_irqrestore(
182 &mdev
->irq_info
.mic_intr_lock
, flags
);
183 spin_unlock(&mdev
->irq_info
.mic_thread_lock
);
188 spin_unlock_irqrestore(&mdev
->irq_info
.mic_intr_lock
, flags
);
189 spin_unlock(&mdev
->irq_info
.mic_thread_lock
);
190 return MIC_NUM_OFFSETS
;
194 * mic_setup_msix - Initializes MSIx interrupts.
196 * @mdev: pointer to mic_device instance
199 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
201 static int mic_setup_msix(struct mic_device
*mdev
, struct pci_dev
*pdev
)
204 int entry_size
= sizeof(*mdev
->irq_info
.msix_entries
);
206 mdev
->irq_info
.msix_entries
= kmalloc_array(MIC_MIN_MSIX
,
207 entry_size
, GFP_KERNEL
);
208 if (!mdev
->irq_info
.msix_entries
) {
213 for (i
= 0; i
< MIC_MIN_MSIX
; i
++)
214 mdev
->irq_info
.msix_entries
[i
].entry
= i
;
216 rc
= pci_enable_msix_exact(pdev
, mdev
->irq_info
.msix_entries
,
219 dev_dbg(&pdev
->dev
, "Error enabling MSIx. rc = %d\n", rc
);
220 goto err_enable_msix
;
223 mdev
->irq_info
.num_vectors
= MIC_MIN_MSIX
;
224 mdev
->irq_info
.mic_msi_map
= kzalloc((sizeof(u32
) *
225 mdev
->irq_info
.num_vectors
), GFP_KERNEL
);
227 if (!mdev
->irq_info
.mic_msi_map
) {
232 dev_dbg(mdev
->sdev
->parent
,
233 "%d MSIx irqs setup\n", mdev
->irq_info
.num_vectors
);
236 pci_disable_msix(pdev
);
238 kfree(mdev
->irq_info
.msix_entries
);
240 mdev
->irq_info
.num_vectors
= 0;
245 * mic_setup_callbacks - Initialize data structures needed
246 * to handle callbacks.
248 * @mdev: pointer to mic_device instance
250 static int mic_setup_callbacks(struct mic_device
*mdev
)
254 mdev
->irq_info
.cb_list
= kmalloc_array(MIC_NUM_OFFSETS
,
255 sizeof(*mdev
->irq_info
.cb_list
),
257 if (!mdev
->irq_info
.cb_list
)
260 for (i
= 0; i
< MIC_NUM_OFFSETS
; i
++)
261 INIT_LIST_HEAD(&mdev
->irq_info
.cb_list
[i
]);
262 ida_init(&mdev
->irq_info
.cb_ida
);
263 spin_lock_init(&mdev
->irq_info
.mic_intr_lock
);
264 spin_lock_init(&mdev
->irq_info
.mic_thread_lock
);
269 * mic_release_callbacks - Uninitialize data structures needed
270 * to handle callbacks.
272 * @mdev: pointer to mic_device instance
274 static void mic_release_callbacks(struct mic_device
*mdev
)
277 struct list_head
*pos
, *tmp
;
278 struct mic_intr_cb
*intr_cb
;
281 spin_lock(&mdev
->irq_info
.mic_thread_lock
);
282 spin_lock_irqsave(&mdev
->irq_info
.mic_intr_lock
, flags
);
283 for (i
= 0; i
< MIC_NUM_OFFSETS
; i
++) {
285 if (list_empty(&mdev
->irq_info
.cb_list
[i
]))
288 list_for_each_safe(pos
, tmp
, &mdev
->irq_info
.cb_list
[i
]) {
289 intr_cb
= list_entry(pos
, struct mic_intr_cb
, list
);
291 ida_simple_remove(&mdev
->irq_info
.cb_ida
,
296 spin_unlock_irqrestore(&mdev
->irq_info
.mic_intr_lock
, flags
);
297 spin_unlock(&mdev
->irq_info
.mic_thread_lock
);
298 ida_destroy(&mdev
->irq_info
.cb_ida
);
299 kfree(mdev
->irq_info
.cb_list
);
303 * mic_setup_msi - Initializes MSI interrupts.
305 * @mdev: pointer to mic_device instance
306 * @pdev: PCI device structure
308 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
310 static int mic_setup_msi(struct mic_device
*mdev
, struct pci_dev
*pdev
)
314 rc
= pci_enable_msi(pdev
);
316 dev_dbg(&pdev
->dev
, "Error enabling MSI. rc = %d\n", rc
);
320 mdev
->irq_info
.num_vectors
= 1;
321 mdev
->irq_info
.mic_msi_map
= kzalloc((sizeof(u32
) *
322 mdev
->irq_info
.num_vectors
), GFP_KERNEL
);
324 if (!mdev
->irq_info
.mic_msi_map
) {
329 rc
= mic_setup_callbacks(mdev
);
331 dev_err(&pdev
->dev
, "Error setting up callbacks\n");
335 rc
= request_threaded_irq(pdev
->irq
, mic_interrupt
, mic_thread_fn
,
338 dev_err(&pdev
->dev
, "Error allocating MSI interrupt\n");
339 goto err_irq_req_fail
;
342 dev_dbg(&pdev
->dev
, "%d MSI irqs setup\n", mdev
->irq_info
.num_vectors
);
345 mic_release_callbacks(mdev
);
347 kfree(mdev
->irq_info
.mic_msi_map
);
349 pci_disable_msi(pdev
);
350 mdev
->irq_info
.num_vectors
= 0;
355 * mic_setup_intx - Initializes legacy interrupts.
357 * @mdev: pointer to mic_device instance
358 * @pdev: PCI device structure
360 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
362 static int mic_setup_intx(struct mic_device
*mdev
, struct pci_dev
*pdev
)
368 rc
= mic_setup_callbacks(mdev
);
370 dev_err(&pdev
->dev
, "Error setting up callbacks\n");
374 rc
= request_threaded_irq(pdev
->irq
, mic_interrupt
, mic_thread_fn
,
375 IRQF_SHARED
, "mic-intx", mdev
);
379 dev_dbg(&pdev
->dev
, "intx irq setup\n");
382 mic_release_callbacks(mdev
);
388 * mic_next_db - Retrieve the next doorbell interrupt source id.
389 * The id is picked sequentially from the available pool of
392 * @mdev: pointer to the mic_device instance.
394 * Returns the next doorbell interrupt source.
396 int mic_next_db(struct mic_device
*mdev
)
400 next_db
= mdev
->irq_info
.next_avail_src
%
401 mdev
->intr_info
->intr_len
[MIC_INTR_DB
];
402 mdev
->irq_info
.next_avail_src
++;
406 #define COOKIE_ID_SHIFT 16
407 #define GET_ENTRY(cookie) ((cookie) & 0xFFFF)
408 #define GET_OFFSET(cookie) ((cookie) >> COOKIE_ID_SHIFT)
409 #define MK_COOKIE(x, y) ((x) | (y) << COOKIE_ID_SHIFT)
412 * mic_request_threaded_irq - request an irq. mic_mutex needs
413 * to be held before calling this function.
415 * @mdev: pointer to mic_device instance
416 * @handler: The callback function that handles the interrupt.
417 * The function needs to call ack_interrupts
418 * (mdev->ops->ack_interrupt(mdev)) when handling the interrupts.
419 * @thread_fn: thread fn required by request_threaded_irq.
420 * @name: The ASCII name of the callee requesting the irq.
421 * @data: private data that is returned back when calling the
423 * @intr_src: The source id of the requester. Its the doorbell id
424 * for Doorbell interrupts and DMA channel id for DMA interrupts.
425 * @type: The type of interrupt. Values defined in mic_intr_type
427 * returns: The cookie that is transparent to the caller. Passed
428 * back when calling mic_free_irq. An appropriate error code
429 * is returned on failure. Caller needs to use IS_ERR(return_val)
430 * to check for failure and PTR_ERR(return_val) to obtained the
435 mic_request_threaded_irq(struct mic_device
*mdev
,
436 irq_handler_t handler
, irq_handler_t thread_fn
,
437 const char *name
, void *data
, int intr_src
,
438 enum mic_intr_type type
)
442 struct msix_entry
*msix
= NULL
;
443 unsigned long cookie
= 0;
445 struct mic_intr_cb
*intr_cb
;
446 struct pci_dev
*pdev
= container_of(mdev
->sdev
->parent
,
447 struct pci_dev
, dev
);
449 offset
= mic_map_src_to_offset(mdev
, intr_src
, type
);
450 if (offset
>= MIC_NUM_OFFSETS
) {
451 dev_err(mdev
->sdev
->parent
,
452 "Error mapping index %d to a valid source id.\n",
458 if (mdev
->irq_info
.num_vectors
> 1) {
459 msix
= mic_get_available_vector(mdev
);
461 dev_err(mdev
->sdev
->parent
,
462 "No MSIx vectors available for use.\n");
467 rc
= request_threaded_irq(msix
->vector
, handler
, thread_fn
,
470 dev_dbg(mdev
->sdev
->parent
,
471 "request irq failed rc = %d\n", rc
);
475 mdev
->irq_info
.mic_msi_map
[entry
] |= BIT(offset
);
476 mdev
->intr_ops
->program_msi_to_src_map(mdev
,
477 entry
, offset
, true);
478 cookie
= MK_COOKIE(entry
, offset
);
479 dev_dbg(mdev
->sdev
->parent
, "irq: %d assigned for src: %d\n",
480 msix
->vector
, intr_src
);
482 intr_cb
= mic_register_intr_callback(mdev
, offset
, handler
,
484 if (IS_ERR(intr_cb
)) {
485 dev_err(mdev
->sdev
->parent
,
486 "No available callback entries for use\n");
487 rc
= PTR_ERR(intr_cb
);
492 if (pci_dev_msi_enabled(pdev
)) {
493 mdev
->irq_info
.mic_msi_map
[entry
] |= (1 << offset
);
494 mdev
->intr_ops
->program_msi_to_src_map(mdev
,
495 entry
, offset
, true);
497 cookie
= MK_COOKIE(entry
, intr_cb
->cb_id
);
498 dev_dbg(mdev
->sdev
->parent
, "callback %d registered for src: %d\n",
499 intr_cb
->cb_id
, intr_src
);
501 return (struct mic_irq
*)cookie
;
507 * mic_free_irq - free irq. mic_mutex
508 * needs to be held before calling this function.
510 * @mdev: pointer to mic_device instance
511 * @cookie: cookie obtained during a successful call to mic_request_threaded_irq
512 * @data: private data specified by the calling function during the
513 * mic_request_threaded_irq
517 void mic_free_irq(struct mic_device
*mdev
,
518 struct mic_irq
*cookie
, void *data
)
524 struct pci_dev
*pdev
= container_of(mdev
->sdev
->parent
,
525 struct pci_dev
, dev
);
527 entry
= GET_ENTRY((unsigned long)cookie
);
528 offset
= GET_OFFSET((unsigned long)cookie
);
529 if (mdev
->irq_info
.num_vectors
> 1) {
530 if (entry
>= mdev
->irq_info
.num_vectors
) {
531 dev_warn(mdev
->sdev
->parent
,
532 "entry %d should be < num_irq %d\n",
533 entry
, mdev
->irq_info
.num_vectors
);
536 irq
= mdev
->irq_info
.msix_entries
[entry
].vector
;
538 mdev
->irq_info
.mic_msi_map
[entry
] &= ~(BIT(offset
));
539 mdev
->intr_ops
->program_msi_to_src_map(mdev
,
540 entry
, offset
, false);
542 dev_dbg(mdev
->sdev
->parent
, "irq: %d freed\n", irq
);
545 src_id
= mic_unregister_intr_callback(mdev
, offset
);
546 if (src_id
>= MIC_NUM_OFFSETS
) {
547 dev_warn(mdev
->sdev
->parent
, "Error unregistering callback\n");
550 if (pci_dev_msi_enabled(pdev
)) {
551 mdev
->irq_info
.mic_msi_map
[entry
] &= ~(BIT(src_id
));
552 mdev
->intr_ops
->program_msi_to_src_map(mdev
,
553 entry
, src_id
, false);
555 dev_dbg(mdev
->sdev
->parent
, "callback %d unregistered for src: %d\n",
561 * mic_setup_interrupts - Initializes interrupts.
563 * @mdev: pointer to mic_device instance
564 * @pdev: PCI device structure
566 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
568 int mic_setup_interrupts(struct mic_device
*mdev
, struct pci_dev
*pdev
)
572 rc
= mic_setup_msix(mdev
, pdev
);
576 rc
= mic_setup_msi(mdev
, pdev
);
580 rc
= mic_setup_intx(mdev
, pdev
);
582 dev_err(mdev
->sdev
->parent
, "no usable interrupts\n");
586 mdev
->intr_ops
->enable_interrupts(mdev
);
591 * mic_free_interrupts - Frees interrupts setup by mic_setup_interrupts
593 * @mdev: pointer to mic_device instance
594 * @pdev: PCI device structure
598 void mic_free_interrupts(struct mic_device
*mdev
, struct pci_dev
*pdev
)
602 mdev
->intr_ops
->disable_interrupts(mdev
);
603 if (mdev
->irq_info
.num_vectors
> 1) {
604 for (i
= 0; i
< mdev
->irq_info
.num_vectors
; i
++) {
605 if (mdev
->irq_info
.mic_msi_map
[i
])
606 dev_warn(&pdev
->dev
, "irq %d may still be in use.\n",
607 mdev
->irq_info
.msix_entries
[i
].vector
);
609 kfree(mdev
->irq_info
.mic_msi_map
);
610 kfree(mdev
->irq_info
.msix_entries
);
611 pci_disable_msix(pdev
);
613 if (pci_dev_msi_enabled(pdev
)) {
614 free_irq(pdev
->irq
, mdev
);
615 kfree(mdev
->irq_info
.mic_msi_map
);
616 pci_disable_msi(pdev
);
618 free_irq(pdev
->irq
, mdev
);
620 mic_release_callbacks(mdev
);
625 * mic_intr_restore - Restore MIC interrupt registers.
627 * @mdev: pointer to mic_device instance.
629 * Restore the interrupt registers to values previously
630 * stored in the SW data structures. mic_mutex needs to
631 * be held before calling this function.
635 void mic_intr_restore(struct mic_device
*mdev
)
638 struct pci_dev
*pdev
= container_of(mdev
->sdev
->parent
,
639 struct pci_dev
, dev
);
641 if (!pci_dev_msi_enabled(pdev
))
644 for (entry
= 0; entry
< mdev
->irq_info
.num_vectors
; entry
++) {
645 for (offset
= 0; offset
< MIC_NUM_OFFSETS
; offset
++) {
646 if (mdev
->irq_info
.mic_msi_map
[entry
] & BIT(offset
))
647 mdev
->intr_ops
->program_msi_to_src_map(mdev
,
648 entry
, offset
, true);