1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2014 Intel Corp.
4 * Author: Jiang Liu <jiang.liu@linux.intel.com>
6 * This file is licensed under GPLv2.
8 * This file contains common code to support Message Signaled Interrupts for
9 * PCI compatible and non PCI compatible devices.
11 #include <linux/types.h>
12 #include <linux/device.h>
13 #include <linux/irq.h>
14 #include <linux/irqdomain.h>
15 #include <linux/msi.h>
16 #include <linux/slab.h>
17 #include <linux/pci.h>
19 #include "internals.h"
22 * alloc_msi_entry - Allocate an initialized msi_desc
23 * @dev: Pointer to the device for which this is allocated
24 * @nvec: The number of vectors used in this entry
25 * @affinity: Optional pointer to an affinity mask array size of @nvec
27 * If @affinity is not %NULL then an affinity array[@nvec] is allocated
28 * and the affinity masks and flags from @affinity are copied.
30 * Return: pointer to allocated &msi_desc on success or %NULL on failure
32 struct msi_desc
*alloc_msi_entry(struct device
*dev
, int nvec
,
33 const struct irq_affinity_desc
*affinity
)
35 struct msi_desc
*desc
;
37 desc
= kzalloc(sizeof(*desc
), GFP_KERNEL
);
41 INIT_LIST_HEAD(&desc
->list
);
43 desc
->nvec_used
= nvec
;
45 desc
->affinity
= kmemdup(affinity
,
46 nvec
* sizeof(*desc
->affinity
), GFP_KERNEL
);
47 if (!desc
->affinity
) {
56 void free_msi_entry(struct msi_desc
*entry
)
58 kfree(entry
->affinity
);
62 void __get_cached_msi_msg(struct msi_desc
*entry
, struct msi_msg
*msg
)
67 void get_cached_msi_msg(unsigned int irq
, struct msi_msg
*msg
)
69 struct msi_desc
*entry
= irq_get_msi_desc(irq
);
71 __get_cached_msi_msg(entry
, msg
);
73 EXPORT_SYMBOL_GPL(get_cached_msi_msg
);
75 static ssize_t
msi_mode_show(struct device
*dev
, struct device_attribute
*attr
,
78 struct msi_desc
*entry
;
83 retval
= kstrtoul(attr
->attr
.name
, 10, &irq
);
87 entry
= irq_get_msi_desc(irq
);
92 is_msix
= entry
->msi_attrib
.is_msix
;
94 return sysfs_emit(buf
, "%s\n", is_msix
? "msix" : "msi");
98 * msi_populate_sysfs - Populate msi_irqs sysfs entries for devices
99 * @dev: The device(PCI, platform etc) who will get sysfs entries
101 * Return attribute_group ** so that specific bus MSI can save it to
102 * somewhere during initilizing msi irqs. If devices has no MSI irq,
103 * return NULL; if it fails to populate sysfs, return ERR_PTR
105 const struct attribute_group
**msi_populate_sysfs(struct device
*dev
)
107 const struct attribute_group
**msi_irq_groups
;
108 struct attribute
**msi_attrs
, *msi_attr
;
109 struct device_attribute
*msi_dev_attr
;
110 struct attribute_group
*msi_irq_group
;
111 struct msi_desc
*entry
;
117 /* Determine how many msi entries we have */
118 for_each_msi_entry(entry
, dev
)
119 num_msi
+= entry
->nvec_used
;
123 /* Dynamically create the MSI attributes for the device */
124 msi_attrs
= kcalloc(num_msi
+ 1, sizeof(void *), GFP_KERNEL
);
126 return ERR_PTR(-ENOMEM
);
128 for_each_msi_entry(entry
, dev
) {
129 for (i
= 0; i
< entry
->nvec_used
; i
++) {
130 msi_dev_attr
= kzalloc(sizeof(*msi_dev_attr
), GFP_KERNEL
);
133 msi_attrs
[count
] = &msi_dev_attr
->attr
;
135 sysfs_attr_init(&msi_dev_attr
->attr
);
136 msi_dev_attr
->attr
.name
= kasprintf(GFP_KERNEL
, "%d",
138 if (!msi_dev_attr
->attr
.name
)
140 msi_dev_attr
->attr
.mode
= 0444;
141 msi_dev_attr
->show
= msi_mode_show
;
146 msi_irq_group
= kzalloc(sizeof(*msi_irq_group
), GFP_KERNEL
);
149 msi_irq_group
->name
= "msi_irqs";
150 msi_irq_group
->attrs
= msi_attrs
;
152 msi_irq_groups
= kcalloc(2, sizeof(void *), GFP_KERNEL
);
154 goto error_irq_group
;
155 msi_irq_groups
[0] = msi_irq_group
;
157 ret
= sysfs_create_groups(&dev
->kobj
, msi_irq_groups
);
159 goto error_irq_groups
;
161 return msi_irq_groups
;
164 kfree(msi_irq_groups
);
166 kfree(msi_irq_group
);
169 msi_attr
= msi_attrs
[count
];
171 msi_dev_attr
= container_of(msi_attr
, struct device_attribute
, attr
);
172 kfree(msi_attr
->name
);
175 msi_attr
= msi_attrs
[count
];
182 * msi_destroy_sysfs - Destroy msi_irqs sysfs entries for devices
183 * @dev: The device(PCI, platform etc) who will remove sysfs entries
184 * @msi_irq_groups: attribute_group for device msi_irqs entries
186 void msi_destroy_sysfs(struct device
*dev
, const struct attribute_group
**msi_irq_groups
)
188 struct device_attribute
*dev_attr
;
189 struct attribute
**msi_attrs
;
192 if (msi_irq_groups
) {
193 sysfs_remove_groups(&dev
->kobj
, msi_irq_groups
);
194 msi_attrs
= msi_irq_groups
[0]->attrs
;
195 while (msi_attrs
[count
]) {
196 dev_attr
= container_of(msi_attrs
[count
],
197 struct device_attribute
, attr
);
198 kfree(dev_attr
->attr
.name
);
203 kfree(msi_irq_groups
[0]);
204 kfree(msi_irq_groups
);
208 #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN
209 static inline void irq_chip_write_msi_msg(struct irq_data
*data
,
212 data
->chip
->irq_write_msi_msg(data
, msg
);
215 static void msi_check_level(struct irq_domain
*domain
, struct msi_msg
*msg
)
217 struct msi_domain_info
*info
= domain
->host_data
;
220 * If the MSI provider has messed with the second message and
221 * not advertized that it is level-capable, signal the breakage.
223 WARN_ON(!((info
->flags
& MSI_FLAG_LEVEL_CAPABLE
) &&
224 (info
->chip
->flags
& IRQCHIP_SUPPORTS_LEVEL_MSI
)) &&
225 (msg
[1].address_lo
|| msg
[1].address_hi
|| msg
[1].data
));
229 * msi_domain_set_affinity - Generic affinity setter function for MSI domains
230 * @irq_data: The irq data associated to the interrupt
231 * @mask: The affinity mask to set
232 * @force: Flag to enforce setting (disable online checks)
234 * Intended to be used by MSI interrupt controllers which are
235 * implemented with hierarchical domains.
237 * Return: IRQ_SET_MASK_* result code
239 int msi_domain_set_affinity(struct irq_data
*irq_data
,
240 const struct cpumask
*mask
, bool force
)
242 struct irq_data
*parent
= irq_data
->parent_data
;
243 struct msi_msg msg
[2] = { [1] = { }, };
246 ret
= parent
->chip
->irq_set_affinity(parent
, mask
, force
);
247 if (ret
>= 0 && ret
!= IRQ_SET_MASK_OK_DONE
) {
248 BUG_ON(irq_chip_compose_msi_msg(irq_data
, msg
));
249 msi_check_level(irq_data
->domain
, msg
);
250 irq_chip_write_msi_msg(irq_data
, msg
);
256 static int msi_domain_activate(struct irq_domain
*domain
,
257 struct irq_data
*irq_data
, bool early
)
259 struct msi_msg msg
[2] = { [1] = { }, };
261 BUG_ON(irq_chip_compose_msi_msg(irq_data
, msg
));
262 msi_check_level(irq_data
->domain
, msg
);
263 irq_chip_write_msi_msg(irq_data
, msg
);
267 static void msi_domain_deactivate(struct irq_domain
*domain
,
268 struct irq_data
*irq_data
)
270 struct msi_msg msg
[2];
272 memset(msg
, 0, sizeof(msg
));
273 irq_chip_write_msi_msg(irq_data
, msg
);
276 static int msi_domain_alloc(struct irq_domain
*domain
, unsigned int virq
,
277 unsigned int nr_irqs
, void *arg
)
279 struct msi_domain_info
*info
= domain
->host_data
;
280 struct msi_domain_ops
*ops
= info
->ops
;
281 irq_hw_number_t hwirq
= ops
->get_hwirq(info
, arg
);
284 if (irq_find_mapping(domain
, hwirq
) > 0)
287 if (domain
->parent
) {
288 ret
= irq_domain_alloc_irqs_parent(domain
, virq
, nr_irqs
, arg
);
293 for (i
= 0; i
< nr_irqs
; i
++) {
294 ret
= ops
->msi_init(domain
, info
, virq
+ i
, hwirq
+ i
, arg
);
297 for (i
--; i
> 0; i
--)
298 ops
->msi_free(domain
, info
, virq
+ i
);
300 irq_domain_free_irqs_top(domain
, virq
, nr_irqs
);
308 static void msi_domain_free(struct irq_domain
*domain
, unsigned int virq
,
309 unsigned int nr_irqs
)
311 struct msi_domain_info
*info
= domain
->host_data
;
314 if (info
->ops
->msi_free
) {
315 for (i
= 0; i
< nr_irqs
; i
++)
316 info
->ops
->msi_free(domain
, info
, virq
+ i
);
318 irq_domain_free_irqs_top(domain
, virq
, nr_irqs
);
321 static const struct irq_domain_ops msi_domain_ops
= {
322 .alloc
= msi_domain_alloc
,
323 .free
= msi_domain_free
,
324 .activate
= msi_domain_activate
,
325 .deactivate
= msi_domain_deactivate
,
328 static irq_hw_number_t
msi_domain_ops_get_hwirq(struct msi_domain_info
*info
,
329 msi_alloc_info_t
*arg
)
334 static int msi_domain_ops_prepare(struct irq_domain
*domain
, struct device
*dev
,
335 int nvec
, msi_alloc_info_t
*arg
)
337 memset(arg
, 0, sizeof(*arg
));
341 static void msi_domain_ops_set_desc(msi_alloc_info_t
*arg
,
342 struct msi_desc
*desc
)
347 static int msi_domain_ops_init(struct irq_domain
*domain
,
348 struct msi_domain_info
*info
,
349 unsigned int virq
, irq_hw_number_t hwirq
,
350 msi_alloc_info_t
*arg
)
352 irq_domain_set_hwirq_and_chip(domain
, virq
, hwirq
, info
->chip
,
354 if (info
->handler
&& info
->handler_name
) {
355 __irq_set_handler(virq
, info
->handler
, 0, info
->handler_name
);
356 if (info
->handler_data
)
357 irq_set_handler_data(virq
, info
->handler_data
);
362 static int msi_domain_ops_check(struct irq_domain
*domain
,
363 struct msi_domain_info
*info
,
369 static struct msi_domain_ops msi_domain_ops_default
= {
370 .get_hwirq
= msi_domain_ops_get_hwirq
,
371 .msi_init
= msi_domain_ops_init
,
372 .msi_check
= msi_domain_ops_check
,
373 .msi_prepare
= msi_domain_ops_prepare
,
374 .set_desc
= msi_domain_ops_set_desc
,
375 .domain_alloc_irqs
= __msi_domain_alloc_irqs
,
376 .domain_free_irqs
= __msi_domain_free_irqs
,
379 static void msi_domain_update_dom_ops(struct msi_domain_info
*info
)
381 struct msi_domain_ops
*ops
= info
->ops
;
384 info
->ops
= &msi_domain_ops_default
;
388 if (ops
->domain_alloc_irqs
== NULL
)
389 ops
->domain_alloc_irqs
= msi_domain_ops_default
.domain_alloc_irqs
;
390 if (ops
->domain_free_irqs
== NULL
)
391 ops
->domain_free_irqs
= msi_domain_ops_default
.domain_free_irqs
;
393 if (!(info
->flags
& MSI_FLAG_USE_DEF_DOM_OPS
))
396 if (ops
->get_hwirq
== NULL
)
397 ops
->get_hwirq
= msi_domain_ops_default
.get_hwirq
;
398 if (ops
->msi_init
== NULL
)
399 ops
->msi_init
= msi_domain_ops_default
.msi_init
;
400 if (ops
->msi_check
== NULL
)
401 ops
->msi_check
= msi_domain_ops_default
.msi_check
;
402 if (ops
->msi_prepare
== NULL
)
403 ops
->msi_prepare
= msi_domain_ops_default
.msi_prepare
;
404 if (ops
->set_desc
== NULL
)
405 ops
->set_desc
= msi_domain_ops_default
.set_desc
;
408 static void msi_domain_update_chip_ops(struct msi_domain_info
*info
)
410 struct irq_chip
*chip
= info
->chip
;
412 BUG_ON(!chip
|| !chip
->irq_mask
|| !chip
->irq_unmask
);
413 if (!chip
->irq_set_affinity
)
414 chip
->irq_set_affinity
= msi_domain_set_affinity
;
418 * msi_create_irq_domain - Create an MSI interrupt domain
419 * @fwnode: Optional fwnode of the interrupt controller
420 * @info: MSI domain info
421 * @parent: Parent irq domain
423 * Return: pointer to the created &struct irq_domain or %NULL on failure
425 struct irq_domain
*msi_create_irq_domain(struct fwnode_handle
*fwnode
,
426 struct msi_domain_info
*info
,
427 struct irq_domain
*parent
)
429 struct irq_domain
*domain
;
431 msi_domain_update_dom_ops(info
);
432 if (info
->flags
& MSI_FLAG_USE_DEF_CHIP_OPS
)
433 msi_domain_update_chip_ops(info
);
435 domain
= irq_domain_create_hierarchy(parent
, IRQ_DOMAIN_FLAG_MSI
, 0,
436 fwnode
, &msi_domain_ops
, info
);
438 if (domain
&& !domain
->name
&& info
->chip
)
439 domain
->name
= info
->chip
->name
;
444 int msi_domain_prepare_irqs(struct irq_domain
*domain
, struct device
*dev
,
445 int nvec
, msi_alloc_info_t
*arg
)
447 struct msi_domain_info
*info
= domain
->host_data
;
448 struct msi_domain_ops
*ops
= info
->ops
;
451 ret
= ops
->msi_check(domain
, info
, dev
);
453 ret
= ops
->msi_prepare(domain
, dev
, nvec
, arg
);
458 int msi_domain_populate_irqs(struct irq_domain
*domain
, struct device
*dev
,
459 int virq
, int nvec
, msi_alloc_info_t
*arg
)
461 struct msi_domain_info
*info
= domain
->host_data
;
462 struct msi_domain_ops
*ops
= info
->ops
;
463 struct msi_desc
*desc
;
466 for_each_msi_entry(desc
, dev
) {
467 /* Don't even try the multi-MSI brain damage. */
468 if (WARN_ON(!desc
->irq
|| desc
->nvec_used
!= 1)) {
473 if (!(desc
->irq
>= virq
&& desc
->irq
< (virq
+ nvec
)))
476 ops
->set_desc(arg
, desc
);
477 /* Assumes the domain mutex is held! */
478 ret
= irq_domain_alloc_irqs_hierarchy(domain
, desc
->irq
, 1,
483 irq_set_msi_desc_off(desc
->irq
, 0, desc
);
487 /* Mop up the damage */
488 for_each_msi_entry(desc
, dev
) {
489 if (!(desc
->irq
>= virq
&& desc
->irq
< (virq
+ nvec
)))
492 irq_domain_free_irqs_common(domain
, desc
->irq
, 1);
500 * Carefully check whether the device can use reservation mode. If
501 * reservation mode is enabled then the early activation will assign a
502 * dummy vector to the device. If the PCI/MSI device does not support
503 * masking of the entry then this can result in spurious interrupts when
504 * the device driver is not absolutely careful. But even then a malfunction
505 * of the hardware could result in a spurious interrupt on the dummy vector
506 * and render the device unusable. If the entry can be masked then the core
507 * logic will prevent the spurious interrupt and reservation mode can be
508 * used. For now reservation mode is restricted to PCI/MSI.
510 static bool msi_check_reservation_mode(struct irq_domain
*domain
,
511 struct msi_domain_info
*info
,
514 struct msi_desc
*desc
;
516 switch(domain
->bus_token
) {
517 case DOMAIN_BUS_PCI_MSI
:
518 case DOMAIN_BUS_VMD_MSI
:
524 if (!(info
->flags
& MSI_FLAG_MUST_REACTIVATE
))
527 if (IS_ENABLED(CONFIG_PCI_MSI
) && pci_msi_ignore_mask
)
531 * Checking the first MSI descriptor is sufficient. MSIX supports
532 * masking and MSI does so when the maskbit is set.
534 desc
= first_msi_entry(dev
);
535 return desc
->msi_attrib
.is_msix
|| desc
->msi_attrib
.maskbit
;
538 int __msi_domain_alloc_irqs(struct irq_domain
*domain
, struct device
*dev
,
541 struct msi_domain_info
*info
= domain
->host_data
;
542 struct msi_domain_ops
*ops
= info
->ops
;
543 struct irq_data
*irq_data
;
544 struct msi_desc
*desc
;
545 msi_alloc_info_t arg
= { };
549 ret
= msi_domain_prepare_irqs(domain
, dev
, nvec
, &arg
);
553 for_each_msi_entry(desc
, dev
) {
554 ops
->set_desc(&arg
, desc
);
556 virq
= __irq_domain_alloc_irqs(domain
, -1, desc
->nvec_used
,
557 dev_to_node(dev
), &arg
, false,
561 if (ops
->handle_error
)
562 ret
= ops
->handle_error(domain
, desc
, ret
);
564 ops
->msi_finish(&arg
, ret
);
568 for (i
= 0; i
< desc
->nvec_used
; i
++) {
569 irq_set_msi_desc_off(virq
, i
, desc
);
570 irq_debugfs_copy_devname(virq
+ i
, dev
);
575 ops
->msi_finish(&arg
, 0);
577 can_reserve
= msi_check_reservation_mode(domain
, info
, dev
);
580 * This flag is set by the PCI layer as we need to activate
581 * the MSI entries before the PCI layer enables MSI in the
582 * card. Otherwise the card latches a random msi message.
584 if (!(info
->flags
& MSI_FLAG_ACTIVATE_EARLY
))
587 for_each_msi_vector(desc
, i
, dev
) {
588 if (desc
->irq
== i
) {
590 dev_dbg(dev
, "irq [%d-%d] for MSI\n",
591 virq
, virq
+ desc
->nvec_used
- 1);
594 irq_data
= irq_domain_get_irq_data(domain
, i
);
596 irqd_clr_can_reserve(irq_data
);
597 if (domain
->flags
& IRQ_DOMAIN_MSI_NOMASK_QUIRK
)
598 irqd_set_msi_nomask_quirk(irq_data
);
600 ret
= irq_domain_activate_irq(irq_data
, can_reserve
);
607 * If these interrupts use reservation mode, clear the activated bit
608 * so request_irq() will assign the final vector.
611 for_each_msi_vector(desc
, i
, dev
) {
612 irq_data
= irq_domain_get_irq_data(domain
, i
);
613 irqd_clr_activated(irq_data
);
619 msi_domain_free_irqs(domain
, dev
);
624 * msi_domain_alloc_irqs - Allocate interrupts from a MSI interrupt domain
625 * @domain: The domain to allocate from
626 * @dev: Pointer to device struct of the device for which the interrupts
628 * @nvec: The number of interrupts to allocate
630 * Return: %0 on success or an error code.
632 int msi_domain_alloc_irqs(struct irq_domain
*domain
, struct device
*dev
,
635 struct msi_domain_info
*info
= domain
->host_data
;
636 struct msi_domain_ops
*ops
= info
->ops
;
638 return ops
->domain_alloc_irqs(domain
, dev
, nvec
);
641 void __msi_domain_free_irqs(struct irq_domain
*domain
, struct device
*dev
)
643 struct irq_data
*irq_data
;
644 struct msi_desc
*desc
;
647 for_each_msi_vector(desc
, i
, dev
) {
648 irq_data
= irq_domain_get_irq_data(domain
, i
);
649 if (irqd_is_activated(irq_data
))
650 irq_domain_deactivate_irq(irq_data
);
653 for_each_msi_entry(desc
, dev
) {
655 * We might have failed to allocate an MSI early
656 * enough that there is no IRQ associated to this
657 * entry. If that's the case, don't do anything.
660 irq_domain_free_irqs(desc
->irq
, desc
->nvec_used
);
667 * msi_domain_free_irqs - Free interrupts from a MSI interrupt @domain associated to @dev
668 * @domain: The domain to managing the interrupts
669 * @dev: Pointer to device struct of the device for which the interrupts
672 void msi_domain_free_irqs(struct irq_domain
*domain
, struct device
*dev
)
674 struct msi_domain_info
*info
= domain
->host_data
;
675 struct msi_domain_ops
*ops
= info
->ops
;
677 return ops
->domain_free_irqs(domain
, dev
);
681 * msi_get_domain_info - Get the MSI interrupt domain info for @domain
682 * @domain: The interrupt domain to retrieve data from
684 * Return: the pointer to the msi_domain_info stored in @domain->host_data.
686 struct msi_domain_info
*msi_get_domain_info(struct irq_domain
*domain
)
688 return (struct msi_domain_info
*)domain
->host_data
;
691 #endif /* CONFIG_GENERIC_MSI_IRQ_DOMAIN */