2 * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 #include <linux/acpi.h>
19 #include <linux/acpi_iort.h>
20 #include <linux/bitmap.h>
21 #include <linux/cpu.h>
22 #include <linux/crash_dump.h>
23 #include <linux/delay.h>
24 #include <linux/dma-iommu.h>
25 #include <linux/efi.h>
26 #include <linux/interrupt.h>
27 #include <linux/irqdomain.h>
28 #include <linux/list.h>
29 #include <linux/list_sort.h>
30 #include <linux/log2.h>
31 #include <linux/memblock.h>
33 #include <linux/msi.h>
35 #include <linux/of_address.h>
36 #include <linux/of_irq.h>
37 #include <linux/of_pci.h>
38 #include <linux/of_platform.h>
39 #include <linux/percpu.h>
40 #include <linux/slab.h>
41 #include <linux/syscore_ops.h>
43 #include <linux/irqchip.h>
44 #include <linux/irqchip/arm-gic-v3.h>
45 #include <linux/irqchip/arm-gic-v4.h>
47 #include <asm/cputype.h>
48 #include <asm/exception.h>
50 #include "irq-gic-common.h"
52 #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING (1ULL << 0)
53 #define ITS_FLAGS_WORKAROUND_CAVIUM_22375 (1ULL << 1)
54 #define ITS_FLAGS_WORKAROUND_CAVIUM_23144 (1ULL << 2)
55 #define ITS_FLAGS_SAVE_SUSPEND_STATE (1ULL << 3)
57 #define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING (1 << 0)
58 #define RDIST_FLAGS_RD_TABLES_PREALLOCATED (1 << 1)
60 static u32 lpi_id_bits
;
63 * We allocate memory for PROPBASE to cover 2 ^ lpi_id_bits LPIs to
64 * deal with (one configuration byte per interrupt). PENDBASE has to
65 * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI).
67 #define LPI_NRBITS lpi_id_bits
68 #define LPI_PROPBASE_SZ ALIGN(BIT(LPI_NRBITS), SZ_64K)
69 #define LPI_PENDBASE_SZ ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K)
71 #define LPI_PROP_DEFAULT_PRIO GICD_INT_DEF_PRI
74 * Collection structure - just an ID, and a redistributor address to
75 * ping. We use one per CPU as a bag of interrupts assigned to this
78 struct its_collection
{
84 * The ITS_BASER structure - contains memory information, cached
85 * value of BASER register configuration and ITS page size.
97 * The ITS structure - contains most of the infrastructure, with the
98 * top-level MSI domain, the command queue, the collections, and the
99 * list of devices writing to it.
103 struct list_head entry
;
105 phys_addr_t phys_base
;
106 struct its_cmd_block
*cmd_base
;
107 struct its_cmd_block
*cmd_write
;
108 struct its_baser tables
[GITS_BASER_NR_REGS
];
109 struct its_collection
*collections
;
110 struct fwnode_handle
*fwnode_handle
;
111 u64 (*get_msi_base
)(struct its_device
*its_dev
);
114 struct list_head its_device_list
;
116 unsigned long list_nr
;
120 unsigned int msi_domain_flags
;
121 u32 pre_its_base
; /* for Socionext Synquacer */
123 int vlpi_redist_offset
;
126 #define ITS_ITT_ALIGN SZ_256
128 /* The maximum number of VPEID bits supported by VLPI commands */
129 #define ITS_MAX_VPEID_BITS (16)
130 #define ITS_MAX_VPEID (1 << (ITS_MAX_VPEID_BITS))
132 /* Convert page order to size in bytes */
133 #define PAGE_ORDER_TO_SIZE(o) (PAGE_SIZE << (o))
135 struct event_lpi_map
{
136 unsigned long *lpi_map
;
138 irq_hw_number_t lpi_base
;
140 struct mutex vlpi_lock
;
142 struct its_vlpi_map
*vlpi_maps
;
147 * The ITS view of a device - belongs to an ITS, owns an interrupt
148 * translation table, and a list of interrupts. If it some of its
149 * LPIs are injected into a guest (GICv4), the event_map.vm field
150 * indicates which one.
153 struct list_head entry
;
154 struct its_node
*its
;
155 struct event_lpi_map event_map
;
163 struct its_device
*dev
;
164 struct its_vpe
**vpes
;
168 static LIST_HEAD(its_nodes
);
169 static DEFINE_RAW_SPINLOCK(its_lock
);
170 static struct rdists
*gic_rdists
;
171 static struct irq_domain
*its_parent
;
173 static unsigned long its_list_map
;
174 static u16 vmovp_seq_num
;
175 static DEFINE_RAW_SPINLOCK(vmovp_lock
);
177 static DEFINE_IDA(its_vpeid_ida
);
179 #define gic_data_rdist() (raw_cpu_ptr(gic_rdists->rdist))
180 #define gic_data_rdist_cpu(cpu) (per_cpu_ptr(gic_rdists->rdist, cpu))
181 #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base)
182 #define gic_data_rdist_vlpi_base() (gic_data_rdist_rd_base() + SZ_128K)
184 static struct its_collection
*dev_event_to_col(struct its_device
*its_dev
,
187 struct its_node
*its
= its_dev
->its
;
189 return its
->collections
+ its_dev
->event_map
.col_map
[event
];
192 static struct its_collection
*valid_col(struct its_collection
*col
)
194 if (WARN_ON_ONCE(col
->target_address
& GENMASK_ULL(0, 15)))
200 static struct its_vpe
*valid_vpe(struct its_node
*its
, struct its_vpe
*vpe
)
202 if (valid_col(its
->collections
+ vpe
->col_idx
))
209 * ITS command descriptors - parameters to be encoded in a command
212 struct its_cmd_desc
{
215 struct its_device
*dev
;
220 struct its_device
*dev
;
225 struct its_device
*dev
;
230 struct its_device
*dev
;
235 struct its_collection
*col
;
240 struct its_device
*dev
;
246 struct its_device
*dev
;
247 struct its_collection
*col
;
252 struct its_device
*dev
;
257 struct its_collection
*col
;
266 struct its_collection
*col
;
272 struct its_device
*dev
;
280 struct its_device
*dev
;
287 struct its_collection
*col
;
295 * The ITS command block, which is what the ITS actually parses.
297 struct its_cmd_block
{
301 #define ITS_CMD_QUEUE_SZ SZ_64K
302 #define ITS_CMD_QUEUE_NR_ENTRIES (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block))
304 typedef struct its_collection
*(*its_cmd_builder_t
)(struct its_node
*,
305 struct its_cmd_block
*,
306 struct its_cmd_desc
*);
308 typedef struct its_vpe
*(*its_cmd_vbuilder_t
)(struct its_node
*,
309 struct its_cmd_block
*,
310 struct its_cmd_desc
*);
312 static void its_mask_encode(u64
*raw_cmd
, u64 val
, int h
, int l
)
314 u64 mask
= GENMASK_ULL(h
, l
);
316 *raw_cmd
|= (val
<< l
) & mask
;
319 static void its_encode_cmd(struct its_cmd_block
*cmd
, u8 cmd_nr
)
321 its_mask_encode(&cmd
->raw_cmd
[0], cmd_nr
, 7, 0);
324 static void its_encode_devid(struct its_cmd_block
*cmd
, u32 devid
)
326 its_mask_encode(&cmd
->raw_cmd
[0], devid
, 63, 32);
329 static void its_encode_event_id(struct its_cmd_block
*cmd
, u32 id
)
331 its_mask_encode(&cmd
->raw_cmd
[1], id
, 31, 0);
334 static void its_encode_phys_id(struct its_cmd_block
*cmd
, u32 phys_id
)
336 its_mask_encode(&cmd
->raw_cmd
[1], phys_id
, 63, 32);
339 static void its_encode_size(struct its_cmd_block
*cmd
, u8 size
)
341 its_mask_encode(&cmd
->raw_cmd
[1], size
, 4, 0);
344 static void its_encode_itt(struct its_cmd_block
*cmd
, u64 itt_addr
)
346 its_mask_encode(&cmd
->raw_cmd
[2], itt_addr
>> 8, 51, 8);
349 static void its_encode_valid(struct its_cmd_block
*cmd
, int valid
)
351 its_mask_encode(&cmd
->raw_cmd
[2], !!valid
, 63, 63);
354 static void its_encode_target(struct its_cmd_block
*cmd
, u64 target_addr
)
356 its_mask_encode(&cmd
->raw_cmd
[2], target_addr
>> 16, 51, 16);
359 static void its_encode_collection(struct its_cmd_block
*cmd
, u16 col
)
361 its_mask_encode(&cmd
->raw_cmd
[2], col
, 15, 0);
364 static void its_encode_vpeid(struct its_cmd_block
*cmd
, u16 vpeid
)
366 its_mask_encode(&cmd
->raw_cmd
[1], vpeid
, 47, 32);
369 static void its_encode_virt_id(struct its_cmd_block
*cmd
, u32 virt_id
)
371 its_mask_encode(&cmd
->raw_cmd
[2], virt_id
, 31, 0);
374 static void its_encode_db_phys_id(struct its_cmd_block
*cmd
, u32 db_phys_id
)
376 its_mask_encode(&cmd
->raw_cmd
[2], db_phys_id
, 63, 32);
379 static void its_encode_db_valid(struct its_cmd_block
*cmd
, bool db_valid
)
381 its_mask_encode(&cmd
->raw_cmd
[2], db_valid
, 0, 0);
384 static void its_encode_seq_num(struct its_cmd_block
*cmd
, u16 seq_num
)
386 its_mask_encode(&cmd
->raw_cmd
[0], seq_num
, 47, 32);
389 static void its_encode_its_list(struct its_cmd_block
*cmd
, u16 its_list
)
391 its_mask_encode(&cmd
->raw_cmd
[1], its_list
, 15, 0);
394 static void its_encode_vpt_addr(struct its_cmd_block
*cmd
, u64 vpt_pa
)
396 its_mask_encode(&cmd
->raw_cmd
[3], vpt_pa
>> 16, 51, 16);
399 static void its_encode_vpt_size(struct its_cmd_block
*cmd
, u8 vpt_size
)
401 its_mask_encode(&cmd
->raw_cmd
[3], vpt_size
, 4, 0);
404 static inline void its_fixup_cmd(struct its_cmd_block
*cmd
)
406 /* Let's fixup BE commands */
407 cmd
->raw_cmd
[0] = cpu_to_le64(cmd
->raw_cmd
[0]);
408 cmd
->raw_cmd
[1] = cpu_to_le64(cmd
->raw_cmd
[1]);
409 cmd
->raw_cmd
[2] = cpu_to_le64(cmd
->raw_cmd
[2]);
410 cmd
->raw_cmd
[3] = cpu_to_le64(cmd
->raw_cmd
[3]);
413 static struct its_collection
*its_build_mapd_cmd(struct its_node
*its
,
414 struct its_cmd_block
*cmd
,
415 struct its_cmd_desc
*desc
)
417 unsigned long itt_addr
;
418 u8 size
= ilog2(desc
->its_mapd_cmd
.dev
->nr_ites
);
420 itt_addr
= virt_to_phys(desc
->its_mapd_cmd
.dev
->itt
);
421 itt_addr
= ALIGN(itt_addr
, ITS_ITT_ALIGN
);
423 its_encode_cmd(cmd
, GITS_CMD_MAPD
);
424 its_encode_devid(cmd
, desc
->its_mapd_cmd
.dev
->device_id
);
425 its_encode_size(cmd
, size
- 1);
426 its_encode_itt(cmd
, itt_addr
);
427 its_encode_valid(cmd
, desc
->its_mapd_cmd
.valid
);
434 static struct its_collection
*its_build_mapc_cmd(struct its_node
*its
,
435 struct its_cmd_block
*cmd
,
436 struct its_cmd_desc
*desc
)
438 its_encode_cmd(cmd
, GITS_CMD_MAPC
);
439 its_encode_collection(cmd
, desc
->its_mapc_cmd
.col
->col_id
);
440 its_encode_target(cmd
, desc
->its_mapc_cmd
.col
->target_address
);
441 its_encode_valid(cmd
, desc
->its_mapc_cmd
.valid
);
445 return desc
->its_mapc_cmd
.col
;
448 static struct its_collection
*its_build_mapti_cmd(struct its_node
*its
,
449 struct its_cmd_block
*cmd
,
450 struct its_cmd_desc
*desc
)
452 struct its_collection
*col
;
454 col
= dev_event_to_col(desc
->its_mapti_cmd
.dev
,
455 desc
->its_mapti_cmd
.event_id
);
457 its_encode_cmd(cmd
, GITS_CMD_MAPTI
);
458 its_encode_devid(cmd
, desc
->its_mapti_cmd
.dev
->device_id
);
459 its_encode_event_id(cmd
, desc
->its_mapti_cmd
.event_id
);
460 its_encode_phys_id(cmd
, desc
->its_mapti_cmd
.phys_id
);
461 its_encode_collection(cmd
, col
->col_id
);
465 return valid_col(col
);
468 static struct its_collection
*its_build_movi_cmd(struct its_node
*its
,
469 struct its_cmd_block
*cmd
,
470 struct its_cmd_desc
*desc
)
472 struct its_collection
*col
;
474 col
= dev_event_to_col(desc
->its_movi_cmd
.dev
,
475 desc
->its_movi_cmd
.event_id
);
477 its_encode_cmd(cmd
, GITS_CMD_MOVI
);
478 its_encode_devid(cmd
, desc
->its_movi_cmd
.dev
->device_id
);
479 its_encode_event_id(cmd
, desc
->its_movi_cmd
.event_id
);
480 its_encode_collection(cmd
, desc
->its_movi_cmd
.col
->col_id
);
484 return valid_col(col
);
487 static struct its_collection
*its_build_discard_cmd(struct its_node
*its
,
488 struct its_cmd_block
*cmd
,
489 struct its_cmd_desc
*desc
)
491 struct its_collection
*col
;
493 col
= dev_event_to_col(desc
->its_discard_cmd
.dev
,
494 desc
->its_discard_cmd
.event_id
);
496 its_encode_cmd(cmd
, GITS_CMD_DISCARD
);
497 its_encode_devid(cmd
, desc
->its_discard_cmd
.dev
->device_id
);
498 its_encode_event_id(cmd
, desc
->its_discard_cmd
.event_id
);
502 return valid_col(col
);
505 static struct its_collection
*its_build_inv_cmd(struct its_node
*its
,
506 struct its_cmd_block
*cmd
,
507 struct its_cmd_desc
*desc
)
509 struct its_collection
*col
;
511 col
= dev_event_to_col(desc
->its_inv_cmd
.dev
,
512 desc
->its_inv_cmd
.event_id
);
514 its_encode_cmd(cmd
, GITS_CMD_INV
);
515 its_encode_devid(cmd
, desc
->its_inv_cmd
.dev
->device_id
);
516 its_encode_event_id(cmd
, desc
->its_inv_cmd
.event_id
);
520 return valid_col(col
);
523 static struct its_collection
*its_build_int_cmd(struct its_node
*its
,
524 struct its_cmd_block
*cmd
,
525 struct its_cmd_desc
*desc
)
527 struct its_collection
*col
;
529 col
= dev_event_to_col(desc
->its_int_cmd
.dev
,
530 desc
->its_int_cmd
.event_id
);
532 its_encode_cmd(cmd
, GITS_CMD_INT
);
533 its_encode_devid(cmd
, desc
->its_int_cmd
.dev
->device_id
);
534 its_encode_event_id(cmd
, desc
->its_int_cmd
.event_id
);
538 return valid_col(col
);
541 static struct its_collection
*its_build_clear_cmd(struct its_node
*its
,
542 struct its_cmd_block
*cmd
,
543 struct its_cmd_desc
*desc
)
545 struct its_collection
*col
;
547 col
= dev_event_to_col(desc
->its_clear_cmd
.dev
,
548 desc
->its_clear_cmd
.event_id
);
550 its_encode_cmd(cmd
, GITS_CMD_CLEAR
);
551 its_encode_devid(cmd
, desc
->its_clear_cmd
.dev
->device_id
);
552 its_encode_event_id(cmd
, desc
->its_clear_cmd
.event_id
);
556 return valid_col(col
);
559 static struct its_collection
*its_build_invall_cmd(struct its_node
*its
,
560 struct its_cmd_block
*cmd
,
561 struct its_cmd_desc
*desc
)
563 its_encode_cmd(cmd
, GITS_CMD_INVALL
);
564 its_encode_collection(cmd
, desc
->its_mapc_cmd
.col
->col_id
);
571 static struct its_vpe
*its_build_vinvall_cmd(struct its_node
*its
,
572 struct its_cmd_block
*cmd
,
573 struct its_cmd_desc
*desc
)
575 its_encode_cmd(cmd
, GITS_CMD_VINVALL
);
576 its_encode_vpeid(cmd
, desc
->its_vinvall_cmd
.vpe
->vpe_id
);
580 return valid_vpe(its
, desc
->its_vinvall_cmd
.vpe
);
583 static struct its_vpe
*its_build_vmapp_cmd(struct its_node
*its
,
584 struct its_cmd_block
*cmd
,
585 struct its_cmd_desc
*desc
)
587 unsigned long vpt_addr
;
590 vpt_addr
= virt_to_phys(page_address(desc
->its_vmapp_cmd
.vpe
->vpt_page
));
591 target
= desc
->its_vmapp_cmd
.col
->target_address
+ its
->vlpi_redist_offset
;
593 its_encode_cmd(cmd
, GITS_CMD_VMAPP
);
594 its_encode_vpeid(cmd
, desc
->its_vmapp_cmd
.vpe
->vpe_id
);
595 its_encode_valid(cmd
, desc
->its_vmapp_cmd
.valid
);
596 its_encode_target(cmd
, target
);
597 its_encode_vpt_addr(cmd
, vpt_addr
);
598 its_encode_vpt_size(cmd
, LPI_NRBITS
- 1);
602 return valid_vpe(its
, desc
->its_vmapp_cmd
.vpe
);
605 static struct its_vpe
*its_build_vmapti_cmd(struct its_node
*its
,
606 struct its_cmd_block
*cmd
,
607 struct its_cmd_desc
*desc
)
611 if (desc
->its_vmapti_cmd
.db_enabled
)
612 db
= desc
->its_vmapti_cmd
.vpe
->vpe_db_lpi
;
616 its_encode_cmd(cmd
, GITS_CMD_VMAPTI
);
617 its_encode_devid(cmd
, desc
->its_vmapti_cmd
.dev
->device_id
);
618 its_encode_vpeid(cmd
, desc
->its_vmapti_cmd
.vpe
->vpe_id
);
619 its_encode_event_id(cmd
, desc
->its_vmapti_cmd
.event_id
);
620 its_encode_db_phys_id(cmd
, db
);
621 its_encode_virt_id(cmd
, desc
->its_vmapti_cmd
.virt_id
);
625 return valid_vpe(its
, desc
->its_vmapti_cmd
.vpe
);
628 static struct its_vpe
*its_build_vmovi_cmd(struct its_node
*its
,
629 struct its_cmd_block
*cmd
,
630 struct its_cmd_desc
*desc
)
634 if (desc
->its_vmovi_cmd
.db_enabled
)
635 db
= desc
->its_vmovi_cmd
.vpe
->vpe_db_lpi
;
639 its_encode_cmd(cmd
, GITS_CMD_VMOVI
);
640 its_encode_devid(cmd
, desc
->its_vmovi_cmd
.dev
->device_id
);
641 its_encode_vpeid(cmd
, desc
->its_vmovi_cmd
.vpe
->vpe_id
);
642 its_encode_event_id(cmd
, desc
->its_vmovi_cmd
.event_id
);
643 its_encode_db_phys_id(cmd
, db
);
644 its_encode_db_valid(cmd
, true);
648 return valid_vpe(its
, desc
->its_vmovi_cmd
.vpe
);
651 static struct its_vpe
*its_build_vmovp_cmd(struct its_node
*its
,
652 struct its_cmd_block
*cmd
,
653 struct its_cmd_desc
*desc
)
657 target
= desc
->its_vmovp_cmd
.col
->target_address
+ its
->vlpi_redist_offset
;
658 its_encode_cmd(cmd
, GITS_CMD_VMOVP
);
659 its_encode_seq_num(cmd
, desc
->its_vmovp_cmd
.seq_num
);
660 its_encode_its_list(cmd
, desc
->its_vmovp_cmd
.its_list
);
661 its_encode_vpeid(cmd
, desc
->its_vmovp_cmd
.vpe
->vpe_id
);
662 its_encode_target(cmd
, target
);
666 return valid_vpe(its
, desc
->its_vmovp_cmd
.vpe
);
669 static u64
its_cmd_ptr_to_offset(struct its_node
*its
,
670 struct its_cmd_block
*ptr
)
672 return (ptr
- its
->cmd_base
) * sizeof(*ptr
);
675 static int its_queue_full(struct its_node
*its
)
680 widx
= its
->cmd_write
- its
->cmd_base
;
681 ridx
= readl_relaxed(its
->base
+ GITS_CREADR
) / sizeof(struct its_cmd_block
);
683 /* This is incredibly unlikely to happen, unless the ITS locks up. */
684 if (((widx
+ 1) % ITS_CMD_QUEUE_NR_ENTRIES
) == ridx
)
690 static struct its_cmd_block
*its_allocate_entry(struct its_node
*its
)
692 struct its_cmd_block
*cmd
;
693 u32 count
= 1000000; /* 1s! */
695 while (its_queue_full(its
)) {
698 pr_err_ratelimited("ITS queue not draining\n");
705 cmd
= its
->cmd_write
++;
707 /* Handle queue wrapping */
708 if (its
->cmd_write
== (its
->cmd_base
+ ITS_CMD_QUEUE_NR_ENTRIES
))
709 its
->cmd_write
= its
->cmd_base
;
720 static struct its_cmd_block
*its_post_commands(struct its_node
*its
)
722 u64 wr
= its_cmd_ptr_to_offset(its
, its
->cmd_write
);
724 writel_relaxed(wr
, its
->base
+ GITS_CWRITER
);
726 return its
->cmd_write
;
729 static void its_flush_cmd(struct its_node
*its
, struct its_cmd_block
*cmd
)
732 * Make sure the commands written to memory are observable by
735 if (its
->flags
& ITS_FLAGS_CMDQ_NEEDS_FLUSHING
)
736 gic_flush_dcache_to_poc(cmd
, sizeof(*cmd
));
741 static int its_wait_for_range_completion(struct its_node
*its
,
742 struct its_cmd_block
*from
,
743 struct its_cmd_block
*to
)
745 u64 rd_idx
, from_idx
, to_idx
;
746 u32 count
= 1000000; /* 1s! */
748 from_idx
= its_cmd_ptr_to_offset(its
, from
);
749 to_idx
= its_cmd_ptr_to_offset(its
, to
);
752 rd_idx
= readl_relaxed(its
->base
+ GITS_CREADR
);
755 if (from_idx
< to_idx
&& rd_idx
>= to_idx
)
759 if (from_idx
>= to_idx
&& rd_idx
>= to_idx
&& rd_idx
< from_idx
)
764 pr_err_ratelimited("ITS queue timeout (%llu %llu %llu)\n",
765 from_idx
, to_idx
, rd_idx
);
775 /* Warning, macro hell follows */
776 #define BUILD_SINGLE_CMD_FUNC(name, buildtype, synctype, buildfn) \
777 void name(struct its_node *its, \
779 struct its_cmd_desc *desc) \
781 struct its_cmd_block *cmd, *sync_cmd, *next_cmd; \
782 synctype *sync_obj; \
783 unsigned long flags; \
785 raw_spin_lock_irqsave(&its->lock, flags); \
787 cmd = its_allocate_entry(its); \
788 if (!cmd) { /* We're soooooo screewed... */ \
789 raw_spin_unlock_irqrestore(&its->lock, flags); \
792 sync_obj = builder(its, cmd, desc); \
793 its_flush_cmd(its, cmd); \
796 sync_cmd = its_allocate_entry(its); \
800 buildfn(its, sync_cmd, sync_obj); \
801 its_flush_cmd(its, sync_cmd); \
805 next_cmd = its_post_commands(its); \
806 raw_spin_unlock_irqrestore(&its->lock, flags); \
808 if (its_wait_for_range_completion(its, cmd, next_cmd)) \
809 pr_err_ratelimited("ITS cmd %ps failed\n", builder); \
812 static void its_build_sync_cmd(struct its_node
*its
,
813 struct its_cmd_block
*sync_cmd
,
814 struct its_collection
*sync_col
)
816 its_encode_cmd(sync_cmd
, GITS_CMD_SYNC
);
817 its_encode_target(sync_cmd
, sync_col
->target_address
);
819 its_fixup_cmd(sync_cmd
);
822 static BUILD_SINGLE_CMD_FUNC(its_send_single_command
, its_cmd_builder_t
,
823 struct its_collection
, its_build_sync_cmd
)
825 static void its_build_vsync_cmd(struct its_node
*its
,
826 struct its_cmd_block
*sync_cmd
,
827 struct its_vpe
*sync_vpe
)
829 its_encode_cmd(sync_cmd
, GITS_CMD_VSYNC
);
830 its_encode_vpeid(sync_cmd
, sync_vpe
->vpe_id
);
832 its_fixup_cmd(sync_cmd
);
835 static BUILD_SINGLE_CMD_FUNC(its_send_single_vcommand
, its_cmd_vbuilder_t
,
836 struct its_vpe
, its_build_vsync_cmd
)
838 static void its_send_int(struct its_device
*dev
, u32 event_id
)
840 struct its_cmd_desc desc
;
842 desc
.its_int_cmd
.dev
= dev
;
843 desc
.its_int_cmd
.event_id
= event_id
;
845 its_send_single_command(dev
->its
, its_build_int_cmd
, &desc
);
848 static void its_send_clear(struct its_device
*dev
, u32 event_id
)
850 struct its_cmd_desc desc
;
852 desc
.its_clear_cmd
.dev
= dev
;
853 desc
.its_clear_cmd
.event_id
= event_id
;
855 its_send_single_command(dev
->its
, its_build_clear_cmd
, &desc
);
858 static void its_send_inv(struct its_device
*dev
, u32 event_id
)
860 struct its_cmd_desc desc
;
862 desc
.its_inv_cmd
.dev
= dev
;
863 desc
.its_inv_cmd
.event_id
= event_id
;
865 its_send_single_command(dev
->its
, its_build_inv_cmd
, &desc
);
868 static void its_send_mapd(struct its_device
*dev
, int valid
)
870 struct its_cmd_desc desc
;
872 desc
.its_mapd_cmd
.dev
= dev
;
873 desc
.its_mapd_cmd
.valid
= !!valid
;
875 its_send_single_command(dev
->its
, its_build_mapd_cmd
, &desc
);
878 static void its_send_mapc(struct its_node
*its
, struct its_collection
*col
,
881 struct its_cmd_desc desc
;
883 desc
.its_mapc_cmd
.col
= col
;
884 desc
.its_mapc_cmd
.valid
= !!valid
;
886 its_send_single_command(its
, its_build_mapc_cmd
, &desc
);
889 static void its_send_mapti(struct its_device
*dev
, u32 irq_id
, u32 id
)
891 struct its_cmd_desc desc
;
893 desc
.its_mapti_cmd
.dev
= dev
;
894 desc
.its_mapti_cmd
.phys_id
= irq_id
;
895 desc
.its_mapti_cmd
.event_id
= id
;
897 its_send_single_command(dev
->its
, its_build_mapti_cmd
, &desc
);
900 static void its_send_movi(struct its_device
*dev
,
901 struct its_collection
*col
, u32 id
)
903 struct its_cmd_desc desc
;
905 desc
.its_movi_cmd
.dev
= dev
;
906 desc
.its_movi_cmd
.col
= col
;
907 desc
.its_movi_cmd
.event_id
= id
;
909 its_send_single_command(dev
->its
, its_build_movi_cmd
, &desc
);
912 static void its_send_discard(struct its_device
*dev
, u32 id
)
914 struct its_cmd_desc desc
;
916 desc
.its_discard_cmd
.dev
= dev
;
917 desc
.its_discard_cmd
.event_id
= id
;
919 its_send_single_command(dev
->its
, its_build_discard_cmd
, &desc
);
922 static void its_send_invall(struct its_node
*its
, struct its_collection
*col
)
924 struct its_cmd_desc desc
;
926 desc
.its_invall_cmd
.col
= col
;
928 its_send_single_command(its
, its_build_invall_cmd
, &desc
);
931 static void its_send_vmapti(struct its_device
*dev
, u32 id
)
933 struct its_vlpi_map
*map
= &dev
->event_map
.vlpi_maps
[id
];
934 struct its_cmd_desc desc
;
936 desc
.its_vmapti_cmd
.vpe
= map
->vpe
;
937 desc
.its_vmapti_cmd
.dev
= dev
;
938 desc
.its_vmapti_cmd
.virt_id
= map
->vintid
;
939 desc
.its_vmapti_cmd
.event_id
= id
;
940 desc
.its_vmapti_cmd
.db_enabled
= map
->db_enabled
;
942 its_send_single_vcommand(dev
->its
, its_build_vmapti_cmd
, &desc
);
945 static void its_send_vmovi(struct its_device
*dev
, u32 id
)
947 struct its_vlpi_map
*map
= &dev
->event_map
.vlpi_maps
[id
];
948 struct its_cmd_desc desc
;
950 desc
.its_vmovi_cmd
.vpe
= map
->vpe
;
951 desc
.its_vmovi_cmd
.dev
= dev
;
952 desc
.its_vmovi_cmd
.event_id
= id
;
953 desc
.its_vmovi_cmd
.db_enabled
= map
->db_enabled
;
955 its_send_single_vcommand(dev
->its
, its_build_vmovi_cmd
, &desc
);
958 static void its_send_vmapp(struct its_node
*its
,
959 struct its_vpe
*vpe
, bool valid
)
961 struct its_cmd_desc desc
;
963 desc
.its_vmapp_cmd
.vpe
= vpe
;
964 desc
.its_vmapp_cmd
.valid
= valid
;
965 desc
.its_vmapp_cmd
.col
= &its
->collections
[vpe
->col_idx
];
967 its_send_single_vcommand(its
, its_build_vmapp_cmd
, &desc
);
970 static void its_send_vmovp(struct its_vpe
*vpe
)
972 struct its_cmd_desc desc
;
973 struct its_node
*its
;
975 int col_id
= vpe
->col_idx
;
977 desc
.its_vmovp_cmd
.vpe
= vpe
;
978 desc
.its_vmovp_cmd
.its_list
= (u16
)its_list_map
;
981 its
= list_first_entry(&its_nodes
, struct its_node
, entry
);
982 desc
.its_vmovp_cmd
.seq_num
= 0;
983 desc
.its_vmovp_cmd
.col
= &its
->collections
[col_id
];
984 its_send_single_vcommand(its
, its_build_vmovp_cmd
, &desc
);
989 * Yet another marvel of the architecture. If using the
990 * its_list "feature", we need to make sure that all ITSs
991 * receive all VMOVP commands in the same order. The only way
992 * to guarantee this is to make vmovp a serialization point.
996 raw_spin_lock_irqsave(&vmovp_lock
, flags
);
998 desc
.its_vmovp_cmd
.seq_num
= vmovp_seq_num
++;
1001 list_for_each_entry(its
, &its_nodes
, entry
) {
1005 if (!vpe
->its_vm
->vlpi_count
[its
->list_nr
])
1008 desc
.its_vmovp_cmd
.col
= &its
->collections
[col_id
];
1009 its_send_single_vcommand(its
, its_build_vmovp_cmd
, &desc
);
1012 raw_spin_unlock_irqrestore(&vmovp_lock
, flags
);
1015 static void its_send_vinvall(struct its_node
*its
, struct its_vpe
*vpe
)
1017 struct its_cmd_desc desc
;
1019 desc
.its_vinvall_cmd
.vpe
= vpe
;
1020 its_send_single_vcommand(its
, its_build_vinvall_cmd
, &desc
);
1024 * irqchip functions - assumes MSI, mostly.
1027 static inline u32
its_get_event_id(struct irq_data
*d
)
1029 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
1030 return d
->hwirq
- its_dev
->event_map
.lpi_base
;
1033 static void lpi_write_config(struct irq_data
*d
, u8 clr
, u8 set
)
1035 irq_hw_number_t hwirq
;
1039 if (irqd_is_forwarded_to_vcpu(d
)) {
1040 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
1041 u32 event
= its_get_event_id(d
);
1042 struct its_vlpi_map
*map
;
1044 va
= page_address(its_dev
->event_map
.vm
->vprop_page
);
1045 map
= &its_dev
->event_map
.vlpi_maps
[event
];
1046 hwirq
= map
->vintid
;
1048 /* Remember the updated property */
1049 map
->properties
&= ~clr
;
1050 map
->properties
|= set
| LPI_PROP_GROUP1
;
1052 va
= gic_rdists
->prop_table_va
;
1056 cfg
= va
+ hwirq
- 8192;
1058 *cfg
|= set
| LPI_PROP_GROUP1
;
1061 * Make the above write visible to the redistributors.
1062 * And yes, we're flushing exactly: One. Single. Byte.
1065 if (gic_rdists
->flags
& RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING
)
1066 gic_flush_dcache_to_poc(cfg
, sizeof(*cfg
));
1071 static void lpi_update_config(struct irq_data
*d
, u8 clr
, u8 set
)
1073 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
1075 lpi_write_config(d
, clr
, set
);
1076 its_send_inv(its_dev
, its_get_event_id(d
));
1079 static void its_vlpi_set_doorbell(struct irq_data
*d
, bool enable
)
1081 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
1082 u32 event
= its_get_event_id(d
);
1084 if (its_dev
->event_map
.vlpi_maps
[event
].db_enabled
== enable
)
1087 its_dev
->event_map
.vlpi_maps
[event
].db_enabled
= enable
;
1090 * More fun with the architecture:
1092 * Ideally, we'd issue a VMAPTI to set the doorbell to its LPI
1093 * value or to 1023, depending on the enable bit. But that
1094 * would be issueing a mapping for an /existing/ DevID+EventID
1095 * pair, which is UNPREDICTABLE. Instead, let's issue a VMOVI
1096 * to the /same/ vPE, using this opportunity to adjust the
1097 * doorbell. Mouahahahaha. We loves it, Precious.
1099 its_send_vmovi(its_dev
, event
);
1102 static void its_mask_irq(struct irq_data
*d
)
1104 if (irqd_is_forwarded_to_vcpu(d
))
1105 its_vlpi_set_doorbell(d
, false);
1107 lpi_update_config(d
, LPI_PROP_ENABLED
, 0);
1110 static void its_unmask_irq(struct irq_data
*d
)
1112 if (irqd_is_forwarded_to_vcpu(d
))
1113 its_vlpi_set_doorbell(d
, true);
1115 lpi_update_config(d
, 0, LPI_PROP_ENABLED
);
1118 static int its_set_affinity(struct irq_data
*d
, const struct cpumask
*mask_val
,
1122 const struct cpumask
*cpu_mask
= cpu_online_mask
;
1123 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
1124 struct its_collection
*target_col
;
1125 u32 id
= its_get_event_id(d
);
1127 /* A forwarded interrupt should use irq_set_vcpu_affinity */
1128 if (irqd_is_forwarded_to_vcpu(d
))
1131 /* lpi cannot be routed to a redistributor that is on a foreign node */
1132 if (its_dev
->its
->flags
& ITS_FLAGS_WORKAROUND_CAVIUM_23144
) {
1133 if (its_dev
->its
->numa_node
>= 0) {
1134 cpu_mask
= cpumask_of_node(its_dev
->its
->numa_node
);
1135 if (!cpumask_intersects(mask_val
, cpu_mask
))
1140 cpu
= cpumask_any_and(mask_val
, cpu_mask
);
1142 if (cpu
>= nr_cpu_ids
)
1145 /* don't set the affinity when the target cpu is same as current one */
1146 if (cpu
!= its_dev
->event_map
.col_map
[id
]) {
1147 target_col
= &its_dev
->its
->collections
[cpu
];
1148 its_send_movi(its_dev
, target_col
, id
);
1149 its_dev
->event_map
.col_map
[id
] = cpu
;
1150 irq_data_update_effective_affinity(d
, cpumask_of(cpu
));
1153 return IRQ_SET_MASK_OK_DONE
;
1156 static u64
its_irq_get_msi_base(struct its_device
*its_dev
)
1158 struct its_node
*its
= its_dev
->its
;
1160 return its
->phys_base
+ GITS_TRANSLATER
;
1163 static void its_irq_compose_msi_msg(struct irq_data
*d
, struct msi_msg
*msg
)
1165 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
1166 struct its_node
*its
;
1170 addr
= its
->get_msi_base(its_dev
);
1172 msg
->address_lo
= lower_32_bits(addr
);
1173 msg
->address_hi
= upper_32_bits(addr
);
1174 msg
->data
= its_get_event_id(d
);
1176 iommu_dma_map_msi_msg(d
->irq
, msg
);
1179 static int its_irq_set_irqchip_state(struct irq_data
*d
,
1180 enum irqchip_irq_state which
,
1183 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
1184 u32 event
= its_get_event_id(d
);
1186 if (which
!= IRQCHIP_STATE_PENDING
)
1190 its_send_int(its_dev
, event
);
1192 its_send_clear(its_dev
, event
);
1197 static void its_map_vm(struct its_node
*its
, struct its_vm
*vm
)
1199 unsigned long flags
;
1201 /* Not using the ITS list? Everything is always mapped. */
1205 raw_spin_lock_irqsave(&vmovp_lock
, flags
);
1208 * If the VM wasn't mapped yet, iterate over the vpes and get
1211 vm
->vlpi_count
[its
->list_nr
]++;
1213 if (vm
->vlpi_count
[its
->list_nr
] == 1) {
1216 for (i
= 0; i
< vm
->nr_vpes
; i
++) {
1217 struct its_vpe
*vpe
= vm
->vpes
[i
];
1218 struct irq_data
*d
= irq_get_irq_data(vpe
->irq
);
1220 /* Map the VPE to the first possible CPU */
1221 vpe
->col_idx
= cpumask_first(cpu_online_mask
);
1222 its_send_vmapp(its
, vpe
, true);
1223 its_send_vinvall(its
, vpe
);
1224 irq_data_update_effective_affinity(d
, cpumask_of(vpe
->col_idx
));
1228 raw_spin_unlock_irqrestore(&vmovp_lock
, flags
);
1231 static void its_unmap_vm(struct its_node
*its
, struct its_vm
*vm
)
1233 unsigned long flags
;
1235 /* Not using the ITS list? Everything is always mapped. */
1239 raw_spin_lock_irqsave(&vmovp_lock
, flags
);
1241 if (!--vm
->vlpi_count
[its
->list_nr
]) {
1244 for (i
= 0; i
< vm
->nr_vpes
; i
++)
1245 its_send_vmapp(its
, vm
->vpes
[i
], false);
1248 raw_spin_unlock_irqrestore(&vmovp_lock
, flags
);
1251 static int its_vlpi_map(struct irq_data
*d
, struct its_cmd_info
*info
)
1253 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
1254 u32 event
= its_get_event_id(d
);
1260 mutex_lock(&its_dev
->event_map
.vlpi_lock
);
1262 if (!its_dev
->event_map
.vm
) {
1263 struct its_vlpi_map
*maps
;
1265 maps
= kcalloc(its_dev
->event_map
.nr_lpis
, sizeof(*maps
),
1272 its_dev
->event_map
.vm
= info
->map
->vm
;
1273 its_dev
->event_map
.vlpi_maps
= maps
;
1274 } else if (its_dev
->event_map
.vm
!= info
->map
->vm
) {
1279 /* Get our private copy of the mapping information */
1280 its_dev
->event_map
.vlpi_maps
[event
] = *info
->map
;
1282 if (irqd_is_forwarded_to_vcpu(d
)) {
1283 /* Already mapped, move it around */
1284 its_send_vmovi(its_dev
, event
);
1286 /* Ensure all the VPEs are mapped on this ITS */
1287 its_map_vm(its_dev
->its
, info
->map
->vm
);
1290 * Flag the interrupt as forwarded so that we can
1291 * start poking the virtual property table.
1293 irqd_set_forwarded_to_vcpu(d
);
1295 /* Write out the property to the prop table */
1296 lpi_write_config(d
, 0xff, info
->map
->properties
);
1298 /* Drop the physical mapping */
1299 its_send_discard(its_dev
, event
);
1301 /* and install the virtual one */
1302 its_send_vmapti(its_dev
, event
);
1304 /* Increment the number of VLPIs */
1305 its_dev
->event_map
.nr_vlpis
++;
1309 mutex_unlock(&its_dev
->event_map
.vlpi_lock
);
1313 static int its_vlpi_get(struct irq_data
*d
, struct its_cmd_info
*info
)
1315 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
1316 u32 event
= its_get_event_id(d
);
1319 mutex_lock(&its_dev
->event_map
.vlpi_lock
);
1321 if (!its_dev
->event_map
.vm
||
1322 !its_dev
->event_map
.vlpi_maps
[event
].vm
) {
1327 /* Copy our mapping information to the incoming request */
1328 *info
->map
= its_dev
->event_map
.vlpi_maps
[event
];
1331 mutex_unlock(&its_dev
->event_map
.vlpi_lock
);
1335 static int its_vlpi_unmap(struct irq_data
*d
)
1337 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
1338 u32 event
= its_get_event_id(d
);
1341 mutex_lock(&its_dev
->event_map
.vlpi_lock
);
1343 if (!its_dev
->event_map
.vm
|| !irqd_is_forwarded_to_vcpu(d
)) {
1348 /* Drop the virtual mapping */
1349 its_send_discard(its_dev
, event
);
1351 /* and restore the physical one */
1352 irqd_clr_forwarded_to_vcpu(d
);
1353 its_send_mapti(its_dev
, d
->hwirq
, event
);
1354 lpi_update_config(d
, 0xff, (LPI_PROP_DEFAULT_PRIO
|
1358 /* Potentially unmap the VM from this ITS */
1359 its_unmap_vm(its_dev
->its
, its_dev
->event_map
.vm
);
1362 * Drop the refcount and make the device available again if
1363 * this was the last VLPI.
1365 if (!--its_dev
->event_map
.nr_vlpis
) {
1366 its_dev
->event_map
.vm
= NULL
;
1367 kfree(its_dev
->event_map
.vlpi_maps
);
1371 mutex_unlock(&its_dev
->event_map
.vlpi_lock
);
1375 static int its_vlpi_prop_update(struct irq_data
*d
, struct its_cmd_info
*info
)
1377 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
1379 if (!its_dev
->event_map
.vm
|| !irqd_is_forwarded_to_vcpu(d
))
1382 if (info
->cmd_type
== PROP_UPDATE_AND_INV_VLPI
)
1383 lpi_update_config(d
, 0xff, info
->config
);
1385 lpi_write_config(d
, 0xff, info
->config
);
1386 its_vlpi_set_doorbell(d
, !!(info
->config
& LPI_PROP_ENABLED
));
1391 static int its_irq_set_vcpu_affinity(struct irq_data
*d
, void *vcpu_info
)
1393 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
1394 struct its_cmd_info
*info
= vcpu_info
;
1397 if (!its_dev
->its
->is_v4
)
1400 /* Unmap request? */
1402 return its_vlpi_unmap(d
);
1404 switch (info
->cmd_type
) {
1406 return its_vlpi_map(d
, info
);
1409 return its_vlpi_get(d
, info
);
1411 case PROP_UPDATE_VLPI
:
1412 case PROP_UPDATE_AND_INV_VLPI
:
1413 return its_vlpi_prop_update(d
, info
);
1420 static struct irq_chip its_irq_chip
= {
1422 .irq_mask
= its_mask_irq
,
1423 .irq_unmask
= its_unmask_irq
,
1424 .irq_eoi
= irq_chip_eoi_parent
,
1425 .irq_set_affinity
= its_set_affinity
,
1426 .irq_compose_msi_msg
= its_irq_compose_msi_msg
,
1427 .irq_set_irqchip_state
= its_irq_set_irqchip_state
,
1428 .irq_set_vcpu_affinity
= its_irq_set_vcpu_affinity
,
1433 * How we allocate LPIs:
1435 * lpi_range_list contains ranges of LPIs that are to available to
1436 * allocate from. To allocate LPIs, just pick the first range that
1437 * fits the required allocation, and reduce it by the required
1438 * amount. Once empty, remove the range from the list.
1440 * To free a range of LPIs, add a free range to the list, sort it and
1441 * merge the result if the new range happens to be adjacent to an
1442 * already free block.
1444 * The consequence of the above is that allocation is cost is low, but
1445 * freeing is expensive. We assumes that freeing rarely occurs.
1447 #define ITS_MAX_LPI_NRBITS 16 /* 64K LPIs */
1449 static DEFINE_MUTEX(lpi_range_lock
);
1450 static LIST_HEAD(lpi_range_list
);
1453 struct list_head entry
;
1458 static struct lpi_range
*mk_lpi_range(u32 base
, u32 span
)
1460 struct lpi_range
*range
;
1462 range
= kzalloc(sizeof(*range
), GFP_KERNEL
);
1464 INIT_LIST_HEAD(&range
->entry
);
1465 range
->base_id
= base
;
1472 static int lpi_range_cmp(void *priv
, struct list_head
*a
, struct list_head
*b
)
1474 struct lpi_range
*ra
, *rb
;
1476 ra
= container_of(a
, struct lpi_range
, entry
);
1477 rb
= container_of(b
, struct lpi_range
, entry
);
1479 return rb
->base_id
- ra
->base_id
;
1482 static void merge_lpi_ranges(void)
1484 struct lpi_range
*range
, *tmp
;
1486 list_for_each_entry_safe(range
, tmp
, &lpi_range_list
, entry
) {
1487 if (!list_is_last(&range
->entry
, &lpi_range_list
) &&
1488 (tmp
->base_id
== (range
->base_id
+ range
->span
))) {
1489 tmp
->base_id
= range
->base_id
;
1490 tmp
->span
+= range
->span
;
1491 list_del(&range
->entry
);
1497 static int alloc_lpi_range(u32 nr_lpis
, u32
*base
)
1499 struct lpi_range
*range
, *tmp
;
1502 mutex_lock(&lpi_range_lock
);
1504 list_for_each_entry_safe(range
, tmp
, &lpi_range_list
, entry
) {
1505 if (range
->span
>= nr_lpis
) {
1506 *base
= range
->base_id
;
1507 range
->base_id
+= nr_lpis
;
1508 range
->span
-= nr_lpis
;
1510 if (range
->span
== 0) {
1511 list_del(&range
->entry
);
1520 mutex_unlock(&lpi_range_lock
);
1522 pr_debug("ITS: alloc %u:%u\n", *base
, nr_lpis
);
1526 static int free_lpi_range(u32 base
, u32 nr_lpis
)
1528 struct lpi_range
*new;
1531 mutex_lock(&lpi_range_lock
);
1533 new = mk_lpi_range(base
, nr_lpis
);
1539 list_add(&new->entry
, &lpi_range_list
);
1540 list_sort(NULL
, &lpi_range_list
, lpi_range_cmp
);
1543 mutex_unlock(&lpi_range_lock
);
1547 static int __init
its_lpi_init(u32 id_bits
)
1549 u32 lpis
= (1UL << id_bits
) - 8192;
1553 numlpis
= 1UL << GICD_TYPER_NUM_LPIS(gic_rdists
->gicd_typer
);
1555 if (numlpis
> 2 && !WARN_ON(numlpis
> lpis
)) {
1557 pr_info("ITS: Using hypervisor restricted LPI range [%u]\n",
1562 * Initializing the allocator is just the same as freeing the
1563 * full range of LPIs.
1565 err
= free_lpi_range(8192, lpis
);
1566 pr_debug("ITS: Allocator initialized for %u LPIs\n", lpis
);
1570 static unsigned long *its_lpi_alloc(int nr_irqs
, u32
*base
, int *nr_ids
)
1572 unsigned long *bitmap
= NULL
;
1576 err
= alloc_lpi_range(nr_irqs
, base
);
1581 } while (nr_irqs
> 0);
1586 bitmap
= kcalloc(BITS_TO_LONGS(nr_irqs
), sizeof (long), GFP_ATOMIC
);
1594 *base
= *nr_ids
= 0;
1599 static void its_lpi_free(unsigned long *bitmap
, u32 base
, u32 nr_ids
)
1601 WARN_ON(free_lpi_range(base
, nr_ids
));
1605 static void gic_reset_prop_table(void *va
)
1607 /* Priority 0xa0, Group-1, disabled */
1608 memset(va
, LPI_PROP_DEFAULT_PRIO
| LPI_PROP_GROUP1
, LPI_PROPBASE_SZ
);
1610 /* Make sure the GIC will observe the written configuration */
1611 gic_flush_dcache_to_poc(va
, LPI_PROPBASE_SZ
);
1614 static struct page
*its_allocate_prop_table(gfp_t gfp_flags
)
1616 struct page
*prop_page
;
1618 prop_page
= alloc_pages(gfp_flags
, get_order(LPI_PROPBASE_SZ
));
1622 gic_reset_prop_table(page_address(prop_page
));
1627 static void its_free_prop_table(struct page
*prop_page
)
1629 free_pages((unsigned long)page_address(prop_page
),
1630 get_order(LPI_PROPBASE_SZ
));
1633 static bool gic_check_reserved_range(phys_addr_t addr
, unsigned long size
)
1635 phys_addr_t start
, end
, addr_end
;
1639 * We don't bother checking for a kdump kernel as by
1640 * construction, the LPI tables are out of this kernel's
1643 if (is_kdump_kernel())
1646 addr_end
= addr
+ size
- 1;
1648 for_each_reserved_mem_region(i
, &start
, &end
) {
1649 if (addr
>= start
&& addr_end
<= end
)
1653 /* Not found, not a good sign... */
1654 pr_warn("GICv3: Expected reserved range [%pa:%pa], not found\n",
1656 add_taint(TAINT_CRAP
, LOCKDEP_STILL_OK
);
1660 static int gic_reserve_range(phys_addr_t addr
, unsigned long size
)
1662 if (efi_enabled(EFI_CONFIG_TABLES
))
1663 return efi_mem_reserve_persistent(addr
, size
);
1668 static int __init
its_setup_lpi_prop_table(void)
1670 if (gic_rdists
->flags
& RDIST_FLAGS_RD_TABLES_PREALLOCATED
) {
1673 val
= gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER
);
1674 lpi_id_bits
= (val
& GICR_PROPBASER_IDBITS_MASK
) + 1;
1676 gic_rdists
->prop_table_pa
= val
& GENMASK_ULL(51, 12);
1677 gic_rdists
->prop_table_va
= memremap(gic_rdists
->prop_table_pa
,
1680 gic_reset_prop_table(gic_rdists
->prop_table_va
);
1684 lpi_id_bits
= min_t(u32
,
1685 GICD_TYPER_ID_BITS(gic_rdists
->gicd_typer
),
1686 ITS_MAX_LPI_NRBITS
);
1687 page
= its_allocate_prop_table(GFP_NOWAIT
);
1689 pr_err("Failed to allocate PROPBASE\n");
1693 gic_rdists
->prop_table_pa
= page_to_phys(page
);
1694 gic_rdists
->prop_table_va
= page_address(page
);
1695 WARN_ON(gic_reserve_range(gic_rdists
->prop_table_pa
,
1699 pr_info("GICv3: using LPI property table @%pa\n",
1700 &gic_rdists
->prop_table_pa
);
1702 return its_lpi_init(lpi_id_bits
);
1705 static const char *its_base_type_string
[] = {
1706 [GITS_BASER_TYPE_DEVICE
] = "Devices",
1707 [GITS_BASER_TYPE_VCPU
] = "Virtual CPUs",
1708 [GITS_BASER_TYPE_RESERVED3
] = "Reserved (3)",
1709 [GITS_BASER_TYPE_COLLECTION
] = "Interrupt Collections",
1710 [GITS_BASER_TYPE_RESERVED5
] = "Reserved (5)",
1711 [GITS_BASER_TYPE_RESERVED6
] = "Reserved (6)",
1712 [GITS_BASER_TYPE_RESERVED7
] = "Reserved (7)",
1715 static u64
its_read_baser(struct its_node
*its
, struct its_baser
*baser
)
1717 u32 idx
= baser
- its
->tables
;
1719 return gits_read_baser(its
->base
+ GITS_BASER
+ (idx
<< 3));
1722 static void its_write_baser(struct its_node
*its
, struct its_baser
*baser
,
1725 u32 idx
= baser
- its
->tables
;
1727 gits_write_baser(val
, its
->base
+ GITS_BASER
+ (idx
<< 3));
1728 baser
->val
= its_read_baser(its
, baser
);
1731 static int its_setup_baser(struct its_node
*its
, struct its_baser
*baser
,
1732 u64 cache
, u64 shr
, u32 psz
, u32 order
,
1735 u64 val
= its_read_baser(its
, baser
);
1736 u64 esz
= GITS_BASER_ENTRY_SIZE(val
);
1737 u64 type
= GITS_BASER_TYPE(val
);
1738 u64 baser_phys
, tmp
;
1743 alloc_pages
= (PAGE_ORDER_TO_SIZE(order
) / psz
);
1744 if (alloc_pages
> GITS_BASER_PAGES_MAX
) {
1745 pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n",
1746 &its
->phys_base
, its_base_type_string
[type
],
1747 alloc_pages
, GITS_BASER_PAGES_MAX
);
1748 alloc_pages
= GITS_BASER_PAGES_MAX
;
1749 order
= get_order(GITS_BASER_PAGES_MAX
* psz
);
1752 base
= (void *)__get_free_pages(GFP_KERNEL
| __GFP_ZERO
, order
);
1756 baser_phys
= virt_to_phys(base
);
1758 /* Check if the physical address of the memory is above 48bits */
1759 if (IS_ENABLED(CONFIG_ARM64_64K_PAGES
) && (baser_phys
>> 48)) {
1761 /* 52bit PA is supported only when PageSize=64K */
1762 if (psz
!= SZ_64K
) {
1763 pr_err("ITS: no 52bit PA support when psz=%d\n", psz
);
1764 free_pages((unsigned long)base
, order
);
1768 /* Convert 52bit PA to 48bit field */
1769 baser_phys
= GITS_BASER_PHYS_52_to_48(baser_phys
);
1774 (type
<< GITS_BASER_TYPE_SHIFT
) |
1775 ((esz
- 1) << GITS_BASER_ENTRY_SIZE_SHIFT
) |
1776 ((alloc_pages
- 1) << GITS_BASER_PAGES_SHIFT
) |
1781 val
|= indirect
? GITS_BASER_INDIRECT
: 0x0;
1785 val
|= GITS_BASER_PAGE_SIZE_4K
;
1788 val
|= GITS_BASER_PAGE_SIZE_16K
;
1791 val
|= GITS_BASER_PAGE_SIZE_64K
;
1795 its_write_baser(its
, baser
, val
);
1798 if ((val
^ tmp
) & GITS_BASER_SHAREABILITY_MASK
) {
1800 * Shareability didn't stick. Just use
1801 * whatever the read reported, which is likely
1802 * to be the only thing this redistributor
1803 * supports. If that's zero, make it
1804 * non-cacheable as well.
1806 shr
= tmp
& GITS_BASER_SHAREABILITY_MASK
;
1808 cache
= GITS_BASER_nC
;
1809 gic_flush_dcache_to_poc(base
, PAGE_ORDER_TO_SIZE(order
));
1814 if ((val
^ tmp
) & GITS_BASER_PAGE_SIZE_MASK
) {
1816 * Page size didn't stick. Let's try a smaller
1817 * size and retry. If we reach 4K, then
1818 * something is horribly wrong...
1820 free_pages((unsigned long)base
, order
);
1826 goto retry_alloc_baser
;
1829 goto retry_alloc_baser
;
1834 pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n",
1835 &its
->phys_base
, its_base_type_string
[type
],
1837 free_pages((unsigned long)base
, order
);
1841 baser
->order
= order
;
1844 tmp
= indirect
? GITS_LVL1_ENTRY_SIZE
: esz
;
1846 pr_info("ITS@%pa: allocated %d %s @%lx (%s, esz %d, psz %dK, shr %d)\n",
1847 &its
->phys_base
, (int)(PAGE_ORDER_TO_SIZE(order
) / (int)tmp
),
1848 its_base_type_string
[type
],
1849 (unsigned long)virt_to_phys(base
),
1850 indirect
? "indirect" : "flat", (int)esz
,
1851 psz
/ SZ_1K
, (int)shr
>> GITS_BASER_SHAREABILITY_SHIFT
);
1856 static bool its_parse_indirect_baser(struct its_node
*its
,
1857 struct its_baser
*baser
,
1858 u32 psz
, u32
*order
, u32 ids
)
1860 u64 tmp
= its_read_baser(its
, baser
);
1861 u64 type
= GITS_BASER_TYPE(tmp
);
1862 u64 esz
= GITS_BASER_ENTRY_SIZE(tmp
);
1863 u64 val
= GITS_BASER_InnerShareable
| GITS_BASER_RaWaWb
;
1864 u32 new_order
= *order
;
1865 bool indirect
= false;
1867 /* No need to enable Indirection if memory requirement < (psz*2)bytes */
1868 if ((esz
<< ids
) > (psz
* 2)) {
1870 * Find out whether hw supports a single or two-level table by
1871 * table by reading bit at offset '62' after writing '1' to it.
1873 its_write_baser(its
, baser
, val
| GITS_BASER_INDIRECT
);
1874 indirect
= !!(baser
->val
& GITS_BASER_INDIRECT
);
1878 * The size of the lvl2 table is equal to ITS page size
1879 * which is 'psz'. For computing lvl1 table size,
1880 * subtract ID bits that sparse lvl2 table from 'ids'
1881 * which is reported by ITS hardware times lvl1 table
1884 ids
-= ilog2(psz
/ (int)esz
);
1885 esz
= GITS_LVL1_ENTRY_SIZE
;
1890 * Allocate as many entries as required to fit the
1891 * range of device IDs that the ITS can grok... The ID
1892 * space being incredibly sparse, this results in a
1893 * massive waste of memory if two-level device table
1894 * feature is not supported by hardware.
1896 new_order
= max_t(u32
, get_order(esz
<< ids
), new_order
);
1897 if (new_order
>= MAX_ORDER
) {
1898 new_order
= MAX_ORDER
- 1;
1899 ids
= ilog2(PAGE_ORDER_TO_SIZE(new_order
) / (int)esz
);
1900 pr_warn("ITS@%pa: %s Table too large, reduce ids %u->%u\n",
1901 &its
->phys_base
, its_base_type_string
[type
],
1902 its
->device_ids
, ids
);
1910 static void its_free_tables(struct its_node
*its
)
1914 for (i
= 0; i
< GITS_BASER_NR_REGS
; i
++) {
1915 if (its
->tables
[i
].base
) {
1916 free_pages((unsigned long)its
->tables
[i
].base
,
1917 its
->tables
[i
].order
);
1918 its
->tables
[i
].base
= NULL
;
1923 static int its_alloc_tables(struct its_node
*its
)
1925 u64 shr
= GITS_BASER_InnerShareable
;
1926 u64 cache
= GITS_BASER_RaWaWb
;
1930 if (its
->flags
& ITS_FLAGS_WORKAROUND_CAVIUM_22375
)
1931 /* erratum 24313: ignore memory access type */
1932 cache
= GITS_BASER_nCnB
;
1934 for (i
= 0; i
< GITS_BASER_NR_REGS
; i
++) {
1935 struct its_baser
*baser
= its
->tables
+ i
;
1936 u64 val
= its_read_baser(its
, baser
);
1937 u64 type
= GITS_BASER_TYPE(val
);
1938 u32 order
= get_order(psz
);
1939 bool indirect
= false;
1942 case GITS_BASER_TYPE_NONE
:
1945 case GITS_BASER_TYPE_DEVICE
:
1946 indirect
= its_parse_indirect_baser(its
, baser
,
1949 case GITS_BASER_TYPE_VCPU
:
1950 indirect
= its_parse_indirect_baser(its
, baser
,
1952 ITS_MAX_VPEID_BITS
);
1956 err
= its_setup_baser(its
, baser
, cache
, shr
, psz
, order
, indirect
);
1958 its_free_tables(its
);
1962 /* Update settings which will be used for next BASERn */
1964 cache
= baser
->val
& GITS_BASER_CACHEABILITY_MASK
;
1965 shr
= baser
->val
& GITS_BASER_SHAREABILITY_MASK
;
1971 static int its_alloc_collections(struct its_node
*its
)
1975 its
->collections
= kcalloc(nr_cpu_ids
, sizeof(*its
->collections
),
1977 if (!its
->collections
)
1980 for (i
= 0; i
< nr_cpu_ids
; i
++)
1981 its
->collections
[i
].target_address
= ~0ULL;
1986 static struct page
*its_allocate_pending_table(gfp_t gfp_flags
)
1988 struct page
*pend_page
;
1990 pend_page
= alloc_pages(gfp_flags
| __GFP_ZERO
,
1991 get_order(LPI_PENDBASE_SZ
));
1995 /* Make sure the GIC will observe the zero-ed page */
1996 gic_flush_dcache_to_poc(page_address(pend_page
), LPI_PENDBASE_SZ
);
2001 static void its_free_pending_table(struct page
*pt
)
2003 free_pages((unsigned long)page_address(pt
), get_order(LPI_PENDBASE_SZ
));
2007 * Booting with kdump and LPIs enabled is generally fine. Any other
2008 * case is wrong in the absence of firmware/EFI support.
2010 static bool enabled_lpis_allowed(void)
2015 /* Check whether the property table is in a reserved region */
2016 val
= gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER
);
2017 addr
= val
& GENMASK_ULL(51, 12);
2019 return gic_check_reserved_range(addr
, LPI_PROPBASE_SZ
);
2022 static int __init
allocate_lpi_tables(void)
2028 * If LPIs are enabled while we run this from the boot CPU,
2029 * flag the RD tables as pre-allocated if the stars do align.
2031 val
= readl_relaxed(gic_data_rdist_rd_base() + GICR_CTLR
);
2032 if ((val
& GICR_CTLR_ENABLE_LPIS
) && enabled_lpis_allowed()) {
2033 gic_rdists
->flags
|= (RDIST_FLAGS_RD_TABLES_PREALLOCATED
|
2034 RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING
);
2035 pr_info("GICv3: Using preallocated redistributor tables\n");
2038 err
= its_setup_lpi_prop_table();
2043 * We allocate all the pending tables anyway, as we may have a
2044 * mix of RDs that have had LPIs enabled, and some that
2045 * don't. We'll free the unused ones as each CPU comes online.
2047 for_each_possible_cpu(cpu
) {
2048 struct page
*pend_page
;
2050 pend_page
= its_allocate_pending_table(GFP_NOWAIT
);
2052 pr_err("Failed to allocate PENDBASE for CPU%d\n", cpu
);
2056 gic_data_rdist_cpu(cpu
)->pend_page
= pend_page
;
2062 static void its_cpu_init_lpis(void)
2064 void __iomem
*rbase
= gic_data_rdist_rd_base();
2065 struct page
*pend_page
;
2069 if (gic_data_rdist()->lpi_enabled
)
2072 val
= readl_relaxed(rbase
+ GICR_CTLR
);
2073 if ((gic_rdists
->flags
& RDIST_FLAGS_RD_TABLES_PREALLOCATED
) &&
2074 (val
& GICR_CTLR_ENABLE_LPIS
)) {
2076 * Check that we get the same property table on all
2077 * RDs. If we don't, this is hopeless.
2079 paddr
= gicr_read_propbaser(rbase
+ GICR_PROPBASER
);
2080 paddr
&= GENMASK_ULL(51, 12);
2081 if (WARN_ON(gic_rdists
->prop_table_pa
!= paddr
))
2082 add_taint(TAINT_CRAP
, LOCKDEP_STILL_OK
);
2084 paddr
= gicr_read_pendbaser(rbase
+ GICR_PENDBASER
);
2085 paddr
&= GENMASK_ULL(51, 16);
2087 WARN_ON(!gic_check_reserved_range(paddr
, LPI_PENDBASE_SZ
));
2088 its_free_pending_table(gic_data_rdist()->pend_page
);
2089 gic_data_rdist()->pend_page
= NULL
;
2094 pend_page
= gic_data_rdist()->pend_page
;
2095 paddr
= page_to_phys(pend_page
);
2096 WARN_ON(gic_reserve_range(paddr
, LPI_PENDBASE_SZ
));
2099 val
= (gic_rdists
->prop_table_pa
|
2100 GICR_PROPBASER_InnerShareable
|
2101 GICR_PROPBASER_RaWaWb
|
2102 ((LPI_NRBITS
- 1) & GICR_PROPBASER_IDBITS_MASK
));
2104 gicr_write_propbaser(val
, rbase
+ GICR_PROPBASER
);
2105 tmp
= gicr_read_propbaser(rbase
+ GICR_PROPBASER
);
2107 if ((tmp
^ val
) & GICR_PROPBASER_SHAREABILITY_MASK
) {
2108 if (!(tmp
& GICR_PROPBASER_SHAREABILITY_MASK
)) {
2110 * The HW reports non-shareable, we must
2111 * remove the cacheability attributes as
2114 val
&= ~(GICR_PROPBASER_SHAREABILITY_MASK
|
2115 GICR_PROPBASER_CACHEABILITY_MASK
);
2116 val
|= GICR_PROPBASER_nC
;
2117 gicr_write_propbaser(val
, rbase
+ GICR_PROPBASER
);
2119 pr_info_once("GIC: using cache flushing for LPI property table\n");
2120 gic_rdists
->flags
|= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING
;
2124 val
= (page_to_phys(pend_page
) |
2125 GICR_PENDBASER_InnerShareable
|
2126 GICR_PENDBASER_RaWaWb
);
2128 gicr_write_pendbaser(val
, rbase
+ GICR_PENDBASER
);
2129 tmp
= gicr_read_pendbaser(rbase
+ GICR_PENDBASER
);
2131 if (!(tmp
& GICR_PENDBASER_SHAREABILITY_MASK
)) {
2133 * The HW reports non-shareable, we must remove the
2134 * cacheability attributes as well.
2136 val
&= ~(GICR_PENDBASER_SHAREABILITY_MASK
|
2137 GICR_PENDBASER_CACHEABILITY_MASK
);
2138 val
|= GICR_PENDBASER_nC
;
2139 gicr_write_pendbaser(val
, rbase
+ GICR_PENDBASER
);
2143 val
= readl_relaxed(rbase
+ GICR_CTLR
);
2144 val
|= GICR_CTLR_ENABLE_LPIS
;
2145 writel_relaxed(val
, rbase
+ GICR_CTLR
);
2147 /* Make sure the GIC has seen the above */
2150 gic_data_rdist()->lpi_enabled
= true;
2151 pr_info("GICv3: CPU%d: using %s LPI pending table @%pa\n",
2153 gic_data_rdist()->pend_page
? "allocated" : "reserved",
2157 static void its_cpu_init_collection(struct its_node
*its
)
2159 int cpu
= smp_processor_id();
2162 /* avoid cross node collections and its mapping */
2163 if (its
->flags
& ITS_FLAGS_WORKAROUND_CAVIUM_23144
) {
2164 struct device_node
*cpu_node
;
2166 cpu_node
= of_get_cpu_node(cpu
, NULL
);
2167 if (its
->numa_node
!= NUMA_NO_NODE
&&
2168 its
->numa_node
!= of_node_to_nid(cpu_node
))
2173 * We now have to bind each collection to its target
2176 if (gic_read_typer(its
->base
+ GITS_TYPER
) & GITS_TYPER_PTA
) {
2178 * This ITS wants the physical address of the
2181 target
= gic_data_rdist()->phys_base
;
2183 /* This ITS wants a linear CPU number. */
2184 target
= gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER
);
2185 target
= GICR_TYPER_CPU_NUMBER(target
) << 16;
2188 /* Perform collection mapping */
2189 its
->collections
[cpu
].target_address
= target
;
2190 its
->collections
[cpu
].col_id
= cpu
;
2192 its_send_mapc(its
, &its
->collections
[cpu
], 1);
2193 its_send_invall(its
, &its
->collections
[cpu
]);
2196 static void its_cpu_init_collections(void)
2198 struct its_node
*its
;
2200 raw_spin_lock(&its_lock
);
2202 list_for_each_entry(its
, &its_nodes
, entry
)
2203 its_cpu_init_collection(its
);
2205 raw_spin_unlock(&its_lock
);
2208 static struct its_device
*its_find_device(struct its_node
*its
, u32 dev_id
)
2210 struct its_device
*its_dev
= NULL
, *tmp
;
2211 unsigned long flags
;
2213 raw_spin_lock_irqsave(&its
->lock
, flags
);
2215 list_for_each_entry(tmp
, &its
->its_device_list
, entry
) {
2216 if (tmp
->device_id
== dev_id
) {
2222 raw_spin_unlock_irqrestore(&its
->lock
, flags
);
2227 static struct its_baser
*its_get_baser(struct its_node
*its
, u32 type
)
2231 for (i
= 0; i
< GITS_BASER_NR_REGS
; i
++) {
2232 if (GITS_BASER_TYPE(its
->tables
[i
].val
) == type
)
2233 return &its
->tables
[i
];
2239 static bool its_alloc_table_entry(struct its_baser
*baser
, u32 id
)
2245 /* Don't allow device id that exceeds single, flat table limit */
2246 esz
= GITS_BASER_ENTRY_SIZE(baser
->val
);
2247 if (!(baser
->val
& GITS_BASER_INDIRECT
))
2248 return (id
< (PAGE_ORDER_TO_SIZE(baser
->order
) / esz
));
2250 /* Compute 1st level table index & check if that exceeds table limit */
2251 idx
= id
>> ilog2(baser
->psz
/ esz
);
2252 if (idx
>= (PAGE_ORDER_TO_SIZE(baser
->order
) / GITS_LVL1_ENTRY_SIZE
))
2255 table
= baser
->base
;
2257 /* Allocate memory for 2nd level table */
2259 page
= alloc_pages(GFP_KERNEL
| __GFP_ZERO
, get_order(baser
->psz
));
2263 /* Flush Lvl2 table to PoC if hw doesn't support coherency */
2264 if (!(baser
->val
& GITS_BASER_SHAREABILITY_MASK
))
2265 gic_flush_dcache_to_poc(page_address(page
), baser
->psz
);
2267 table
[idx
] = cpu_to_le64(page_to_phys(page
) | GITS_BASER_VALID
);
2269 /* Flush Lvl1 entry to PoC if hw doesn't support coherency */
2270 if (!(baser
->val
& GITS_BASER_SHAREABILITY_MASK
))
2271 gic_flush_dcache_to_poc(table
+ idx
, GITS_LVL1_ENTRY_SIZE
);
2273 /* Ensure updated table contents are visible to ITS hardware */
2280 static bool its_alloc_device_table(struct its_node
*its
, u32 dev_id
)
2282 struct its_baser
*baser
;
2284 baser
= its_get_baser(its
, GITS_BASER_TYPE_DEVICE
);
2286 /* Don't allow device id that exceeds ITS hardware limit */
2288 return (ilog2(dev_id
) < its
->device_ids
);
2290 return its_alloc_table_entry(baser
, dev_id
);
2293 static bool its_alloc_vpe_table(u32 vpe_id
)
2295 struct its_node
*its
;
2298 * Make sure the L2 tables are allocated on *all* v4 ITSs. We
2299 * could try and only do it on ITSs corresponding to devices
2300 * that have interrupts targeted at this VPE, but the
2301 * complexity becomes crazy (and you have tons of memory
2304 list_for_each_entry(its
, &its_nodes
, entry
) {
2305 struct its_baser
*baser
;
2310 baser
= its_get_baser(its
, GITS_BASER_TYPE_VCPU
);
2314 if (!its_alloc_table_entry(baser
, vpe_id
))
2321 static struct its_device
*its_create_device(struct its_node
*its
, u32 dev_id
,
2322 int nvecs
, bool alloc_lpis
)
2324 struct its_device
*dev
;
2325 unsigned long *lpi_map
= NULL
;
2326 unsigned long flags
;
2327 u16
*col_map
= NULL
;
2334 if (!its_alloc_device_table(its
, dev_id
))
2337 if (WARN_ON(!is_power_of_2(nvecs
)))
2338 nvecs
= roundup_pow_of_two(nvecs
);
2340 dev
= kzalloc(sizeof(*dev
), GFP_KERNEL
);
2342 * Even if the device wants a single LPI, the ITT must be
2343 * sized as a power of two (and you need at least one bit...).
2345 nr_ites
= max(2, nvecs
);
2346 sz
= nr_ites
* its
->ite_size
;
2347 sz
= max(sz
, ITS_ITT_ALIGN
) + ITS_ITT_ALIGN
- 1;
2348 itt
= kzalloc(sz
, GFP_KERNEL
);
2350 lpi_map
= its_lpi_alloc(nvecs
, &lpi_base
, &nr_lpis
);
2352 col_map
= kcalloc(nr_lpis
, sizeof(*col_map
),
2355 col_map
= kcalloc(nr_ites
, sizeof(*col_map
), GFP_KERNEL
);
2360 if (!dev
|| !itt
|| !col_map
|| (!lpi_map
&& alloc_lpis
)) {
2368 gic_flush_dcache_to_poc(itt
, sz
);
2372 dev
->nr_ites
= nr_ites
;
2373 dev
->event_map
.lpi_map
= lpi_map
;
2374 dev
->event_map
.col_map
= col_map
;
2375 dev
->event_map
.lpi_base
= lpi_base
;
2376 dev
->event_map
.nr_lpis
= nr_lpis
;
2377 mutex_init(&dev
->event_map
.vlpi_lock
);
2378 dev
->device_id
= dev_id
;
2379 INIT_LIST_HEAD(&dev
->entry
);
2381 raw_spin_lock_irqsave(&its
->lock
, flags
);
2382 list_add(&dev
->entry
, &its
->its_device_list
);
2383 raw_spin_unlock_irqrestore(&its
->lock
, flags
);
2385 /* Map device to its ITT */
2386 its_send_mapd(dev
, 1);
2391 static void its_free_device(struct its_device
*its_dev
)
2393 unsigned long flags
;
2395 raw_spin_lock_irqsave(&its_dev
->its
->lock
, flags
);
2396 list_del(&its_dev
->entry
);
2397 raw_spin_unlock_irqrestore(&its_dev
->its
->lock
, flags
);
2398 kfree(its_dev
->itt
);
2402 static int its_alloc_device_irq(struct its_device
*dev
, irq_hw_number_t
*hwirq
)
2406 idx
= find_first_zero_bit(dev
->event_map
.lpi_map
,
2407 dev
->event_map
.nr_lpis
);
2408 if (idx
== dev
->event_map
.nr_lpis
)
2411 *hwirq
= dev
->event_map
.lpi_base
+ idx
;
2412 set_bit(idx
, dev
->event_map
.lpi_map
);
2417 static int its_msi_prepare(struct irq_domain
*domain
, struct device
*dev
,
2418 int nvec
, msi_alloc_info_t
*info
)
2420 struct its_node
*its
;
2421 struct its_device
*its_dev
;
2422 struct msi_domain_info
*msi_info
;
2426 * We ignore "dev" entierely, and rely on the dev_id that has
2427 * been passed via the scratchpad. This limits this domain's
2428 * usefulness to upper layers that definitely know that they
2429 * are built on top of the ITS.
2431 dev_id
= info
->scratchpad
[0].ul
;
2433 msi_info
= msi_get_domain_info(domain
);
2434 its
= msi_info
->data
;
2436 if (!gic_rdists
->has_direct_lpi
&&
2438 vpe_proxy
.dev
->its
== its
&&
2439 dev_id
== vpe_proxy
.dev
->device_id
) {
2440 /* Bad luck. Get yourself a better implementation */
2441 WARN_ONCE(1, "DevId %x clashes with GICv4 VPE proxy device\n",
2446 its_dev
= its_find_device(its
, dev_id
);
2449 * We already have seen this ID, probably through
2450 * another alias (PCI bridge of some sort). No need to
2451 * create the device.
2453 pr_debug("Reusing ITT for devID %x\n", dev_id
);
2457 its_dev
= its_create_device(its
, dev_id
, nvec
, true);
2461 pr_debug("ITT %d entries, %d bits\n", nvec
, ilog2(nvec
));
2463 info
->scratchpad
[0].ptr
= its_dev
;
2467 static struct msi_domain_ops its_msi_domain_ops
= {
2468 .msi_prepare
= its_msi_prepare
,
2471 static int its_irq_gic_domain_alloc(struct irq_domain
*domain
,
2473 irq_hw_number_t hwirq
)
2475 struct irq_fwspec fwspec
;
2477 if (irq_domain_get_of_node(domain
->parent
)) {
2478 fwspec
.fwnode
= domain
->parent
->fwnode
;
2479 fwspec
.param_count
= 3;
2480 fwspec
.param
[0] = GIC_IRQ_TYPE_LPI
;
2481 fwspec
.param
[1] = hwirq
;
2482 fwspec
.param
[2] = IRQ_TYPE_EDGE_RISING
;
2483 } else if (is_fwnode_irqchip(domain
->parent
->fwnode
)) {
2484 fwspec
.fwnode
= domain
->parent
->fwnode
;
2485 fwspec
.param_count
= 2;
2486 fwspec
.param
[0] = hwirq
;
2487 fwspec
.param
[1] = IRQ_TYPE_EDGE_RISING
;
2492 return irq_domain_alloc_irqs_parent(domain
, virq
, 1, &fwspec
);
2495 static int its_irq_domain_alloc(struct irq_domain
*domain
, unsigned int virq
,
2496 unsigned int nr_irqs
, void *args
)
2498 msi_alloc_info_t
*info
= args
;
2499 struct its_device
*its_dev
= info
->scratchpad
[0].ptr
;
2500 irq_hw_number_t hwirq
;
2504 for (i
= 0; i
< nr_irqs
; i
++) {
2505 err
= its_alloc_device_irq(its_dev
, &hwirq
);
2509 err
= its_irq_gic_domain_alloc(domain
, virq
+ i
, hwirq
);
2513 irq_domain_set_hwirq_and_chip(domain
, virq
+ i
,
2514 hwirq
, &its_irq_chip
, its_dev
);
2515 irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(virq
+ i
)));
2516 pr_debug("ID:%d pID:%d vID:%d\n",
2517 (int)(hwirq
- its_dev
->event_map
.lpi_base
),
2518 (int) hwirq
, virq
+ i
);
2524 static int its_irq_domain_activate(struct irq_domain
*domain
,
2525 struct irq_data
*d
, bool reserve
)
2527 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
2528 u32 event
= its_get_event_id(d
);
2529 const struct cpumask
*cpu_mask
= cpu_online_mask
;
2532 /* get the cpu_mask of local node */
2533 if (its_dev
->its
->numa_node
>= 0)
2534 cpu_mask
= cpumask_of_node(its_dev
->its
->numa_node
);
2536 /* Bind the LPI to the first possible CPU */
2537 cpu
= cpumask_first_and(cpu_mask
, cpu_online_mask
);
2538 if (cpu
>= nr_cpu_ids
) {
2539 if (its_dev
->its
->flags
& ITS_FLAGS_WORKAROUND_CAVIUM_23144
)
2542 cpu
= cpumask_first(cpu_online_mask
);
2545 its_dev
->event_map
.col_map
[event
] = cpu
;
2546 irq_data_update_effective_affinity(d
, cpumask_of(cpu
));
2548 /* Map the GIC IRQ and event to the device */
2549 its_send_mapti(its_dev
, d
->hwirq
, event
);
2553 static void its_irq_domain_deactivate(struct irq_domain
*domain
,
2556 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
2557 u32 event
= its_get_event_id(d
);
2559 /* Stop the delivery of interrupts */
2560 its_send_discard(its_dev
, event
);
2563 static void its_irq_domain_free(struct irq_domain
*domain
, unsigned int virq
,
2564 unsigned int nr_irqs
)
2566 struct irq_data
*d
= irq_domain_get_irq_data(domain
, virq
);
2567 struct its_device
*its_dev
= irq_data_get_irq_chip_data(d
);
2570 for (i
= 0; i
< nr_irqs
; i
++) {
2571 struct irq_data
*data
= irq_domain_get_irq_data(domain
,
2573 u32 event
= its_get_event_id(data
);
2575 /* Mark interrupt index as unused */
2576 clear_bit(event
, its_dev
->event_map
.lpi_map
);
2578 /* Nuke the entry in the domain */
2579 irq_domain_reset_irq_data(data
);
2582 /* If all interrupts have been freed, start mopping the floor */
2583 if (bitmap_empty(its_dev
->event_map
.lpi_map
,
2584 its_dev
->event_map
.nr_lpis
)) {
2585 its_lpi_free(its_dev
->event_map
.lpi_map
,
2586 its_dev
->event_map
.lpi_base
,
2587 its_dev
->event_map
.nr_lpis
);
2588 kfree(its_dev
->event_map
.col_map
);
2590 /* Unmap device/itt */
2591 its_send_mapd(its_dev
, 0);
2592 its_free_device(its_dev
);
2595 irq_domain_free_irqs_parent(domain
, virq
, nr_irqs
);
2598 static const struct irq_domain_ops its_domain_ops
= {
2599 .alloc
= its_irq_domain_alloc
,
2600 .free
= its_irq_domain_free
,
2601 .activate
= its_irq_domain_activate
,
2602 .deactivate
= its_irq_domain_deactivate
,
2608 * If a GICv4 doesn't implement Direct LPIs (which is extremely
2609 * likely), the only way to perform an invalidate is to use a fake
2610 * device to issue an INV command, implying that the LPI has first
2611 * been mapped to some event on that device. Since this is not exactly
2612 * cheap, we try to keep that mapping around as long as possible, and
2613 * only issue an UNMAP if we're short on available slots.
2615 * Broken by design(tm).
2617 static void its_vpe_db_proxy_unmap_locked(struct its_vpe
*vpe
)
2619 /* Already unmapped? */
2620 if (vpe
->vpe_proxy_event
== -1)
2623 its_send_discard(vpe_proxy
.dev
, vpe
->vpe_proxy_event
);
2624 vpe_proxy
.vpes
[vpe
->vpe_proxy_event
] = NULL
;
2627 * We don't track empty slots at all, so let's move the
2628 * next_victim pointer if we can quickly reuse that slot
2629 * instead of nuking an existing entry. Not clear that this is
2630 * always a win though, and this might just generate a ripple
2631 * effect... Let's just hope VPEs don't migrate too often.
2633 if (vpe_proxy
.vpes
[vpe_proxy
.next_victim
])
2634 vpe_proxy
.next_victim
= vpe
->vpe_proxy_event
;
2636 vpe
->vpe_proxy_event
= -1;
2639 static void its_vpe_db_proxy_unmap(struct its_vpe
*vpe
)
2641 if (!gic_rdists
->has_direct_lpi
) {
2642 unsigned long flags
;
2644 raw_spin_lock_irqsave(&vpe_proxy
.lock
, flags
);
2645 its_vpe_db_proxy_unmap_locked(vpe
);
2646 raw_spin_unlock_irqrestore(&vpe_proxy
.lock
, flags
);
2650 static void its_vpe_db_proxy_map_locked(struct its_vpe
*vpe
)
2652 /* Already mapped? */
2653 if (vpe
->vpe_proxy_event
!= -1)
2656 /* This slot was already allocated. Kick the other VPE out. */
2657 if (vpe_proxy
.vpes
[vpe_proxy
.next_victim
])
2658 its_vpe_db_proxy_unmap_locked(vpe_proxy
.vpes
[vpe_proxy
.next_victim
]);
2660 /* Map the new VPE instead */
2661 vpe_proxy
.vpes
[vpe_proxy
.next_victim
] = vpe
;
2662 vpe
->vpe_proxy_event
= vpe_proxy
.next_victim
;
2663 vpe_proxy
.next_victim
= (vpe_proxy
.next_victim
+ 1) % vpe_proxy
.dev
->nr_ites
;
2665 vpe_proxy
.dev
->event_map
.col_map
[vpe
->vpe_proxy_event
] = vpe
->col_idx
;
2666 its_send_mapti(vpe_proxy
.dev
, vpe
->vpe_db_lpi
, vpe
->vpe_proxy_event
);
2669 static void its_vpe_db_proxy_move(struct its_vpe
*vpe
, int from
, int to
)
2671 unsigned long flags
;
2672 struct its_collection
*target_col
;
2674 if (gic_rdists
->has_direct_lpi
) {
2675 void __iomem
*rdbase
;
2677 rdbase
= per_cpu_ptr(gic_rdists
->rdist
, from
)->rd_base
;
2678 gic_write_lpir(vpe
->vpe_db_lpi
, rdbase
+ GICR_CLRLPIR
);
2679 while (gic_read_lpir(rdbase
+ GICR_SYNCR
) & 1)
2685 raw_spin_lock_irqsave(&vpe_proxy
.lock
, flags
);
2687 its_vpe_db_proxy_map_locked(vpe
);
2689 target_col
= &vpe_proxy
.dev
->its
->collections
[to
];
2690 its_send_movi(vpe_proxy
.dev
, target_col
, vpe
->vpe_proxy_event
);
2691 vpe_proxy
.dev
->event_map
.col_map
[vpe
->vpe_proxy_event
] = to
;
2693 raw_spin_unlock_irqrestore(&vpe_proxy
.lock
, flags
);
2696 static int its_vpe_set_affinity(struct irq_data
*d
,
2697 const struct cpumask
*mask_val
,
2700 struct its_vpe
*vpe
= irq_data_get_irq_chip_data(d
);
2701 int cpu
= cpumask_first(mask_val
);
2704 * Changing affinity is mega expensive, so let's be as lazy as
2705 * we can and only do it if we really have to. Also, if mapped
2706 * into the proxy device, we need to move the doorbell
2707 * interrupt to its new location.
2709 if (vpe
->col_idx
!= cpu
) {
2710 int from
= vpe
->col_idx
;
2713 its_send_vmovp(vpe
);
2714 its_vpe_db_proxy_move(vpe
, from
, cpu
);
2717 irq_data_update_effective_affinity(d
, cpumask_of(cpu
));
2719 return IRQ_SET_MASK_OK_DONE
;
2722 static void its_vpe_schedule(struct its_vpe
*vpe
)
2724 void __iomem
*vlpi_base
= gic_data_rdist_vlpi_base();
2727 /* Schedule the VPE */
2728 val
= virt_to_phys(page_address(vpe
->its_vm
->vprop_page
)) &
2729 GENMASK_ULL(51, 12);
2730 val
|= (LPI_NRBITS
- 1) & GICR_VPROPBASER_IDBITS_MASK
;
2731 val
|= GICR_VPROPBASER_RaWb
;
2732 val
|= GICR_VPROPBASER_InnerShareable
;
2733 gits_write_vpropbaser(val
, vlpi_base
+ GICR_VPROPBASER
);
2735 val
= virt_to_phys(page_address(vpe
->vpt_page
)) &
2736 GENMASK_ULL(51, 16);
2737 val
|= GICR_VPENDBASER_RaWaWb
;
2738 val
|= GICR_VPENDBASER_NonShareable
;
2740 * There is no good way of finding out if the pending table is
2741 * empty as we can race against the doorbell interrupt very
2742 * easily. So in the end, vpe->pending_last is only an
2743 * indication that the vcpu has something pending, not one
2744 * that the pending table is empty. A good implementation
2745 * would be able to read its coarse map pretty quickly anyway,
2746 * making this a tolerable issue.
2748 val
|= GICR_VPENDBASER_PendingLast
;
2749 val
|= vpe
->idai
? GICR_VPENDBASER_IDAI
: 0;
2750 val
|= GICR_VPENDBASER_Valid
;
2751 gits_write_vpendbaser(val
, vlpi_base
+ GICR_VPENDBASER
);
2754 static void its_vpe_deschedule(struct its_vpe
*vpe
)
2756 void __iomem
*vlpi_base
= gic_data_rdist_vlpi_base();
2757 u32 count
= 1000000; /* 1s! */
2761 /* We're being scheduled out */
2762 val
= gits_read_vpendbaser(vlpi_base
+ GICR_VPENDBASER
);
2763 val
&= ~GICR_VPENDBASER_Valid
;
2764 gits_write_vpendbaser(val
, vlpi_base
+ GICR_VPENDBASER
);
2767 val
= gits_read_vpendbaser(vlpi_base
+ GICR_VPENDBASER
);
2768 clean
= !(val
& GICR_VPENDBASER_Dirty
);
2774 } while (!clean
&& count
);
2776 if (unlikely(!clean
&& !count
)) {
2777 pr_err_ratelimited("ITS virtual pending table not cleaning\n");
2779 vpe
->pending_last
= true;
2781 vpe
->idai
= !!(val
& GICR_VPENDBASER_IDAI
);
2782 vpe
->pending_last
= !!(val
& GICR_VPENDBASER_PendingLast
);
2786 static void its_vpe_invall(struct its_vpe
*vpe
)
2788 struct its_node
*its
;
2790 list_for_each_entry(its
, &its_nodes
, entry
) {
2794 if (its_list_map
&& !vpe
->its_vm
->vlpi_count
[its
->list_nr
])
2798 * Sending a VINVALL to a single ITS is enough, as all
2799 * we need is to reach the redistributors.
2801 its_send_vinvall(its
, vpe
);
2806 static int its_vpe_set_vcpu_affinity(struct irq_data
*d
, void *vcpu_info
)
2808 struct its_vpe
*vpe
= irq_data_get_irq_chip_data(d
);
2809 struct its_cmd_info
*info
= vcpu_info
;
2811 switch (info
->cmd_type
) {
2813 its_vpe_schedule(vpe
);
2816 case DESCHEDULE_VPE
:
2817 its_vpe_deschedule(vpe
);
2821 its_vpe_invall(vpe
);
2829 static void its_vpe_send_cmd(struct its_vpe
*vpe
,
2830 void (*cmd
)(struct its_device
*, u32
))
2832 unsigned long flags
;
2834 raw_spin_lock_irqsave(&vpe_proxy
.lock
, flags
);
2836 its_vpe_db_proxy_map_locked(vpe
);
2837 cmd(vpe_proxy
.dev
, vpe
->vpe_proxy_event
);
2839 raw_spin_unlock_irqrestore(&vpe_proxy
.lock
, flags
);
2842 static void its_vpe_send_inv(struct irq_data
*d
)
2844 struct its_vpe
*vpe
= irq_data_get_irq_chip_data(d
);
2846 if (gic_rdists
->has_direct_lpi
) {
2847 void __iomem
*rdbase
;
2849 rdbase
= per_cpu_ptr(gic_rdists
->rdist
, vpe
->col_idx
)->rd_base
;
2850 gic_write_lpir(vpe
->vpe_db_lpi
, rdbase
+ GICR_INVLPIR
);
2851 while (gic_read_lpir(rdbase
+ GICR_SYNCR
) & 1)
2854 its_vpe_send_cmd(vpe
, its_send_inv
);
2858 static void its_vpe_mask_irq(struct irq_data
*d
)
2861 * We need to unmask the LPI, which is described by the parent
2862 * irq_data. Instead of calling into the parent (which won't
2863 * exactly do the right thing, let's simply use the
2864 * parent_data pointer. Yes, I'm naughty.
2866 lpi_write_config(d
->parent_data
, LPI_PROP_ENABLED
, 0);
2867 its_vpe_send_inv(d
);
2870 static void its_vpe_unmask_irq(struct irq_data
*d
)
2872 /* Same hack as above... */
2873 lpi_write_config(d
->parent_data
, 0, LPI_PROP_ENABLED
);
2874 its_vpe_send_inv(d
);
2877 static int its_vpe_set_irqchip_state(struct irq_data
*d
,
2878 enum irqchip_irq_state which
,
2881 struct its_vpe
*vpe
= irq_data_get_irq_chip_data(d
);
2883 if (which
!= IRQCHIP_STATE_PENDING
)
2886 if (gic_rdists
->has_direct_lpi
) {
2887 void __iomem
*rdbase
;
2889 rdbase
= per_cpu_ptr(gic_rdists
->rdist
, vpe
->col_idx
)->rd_base
;
2891 gic_write_lpir(vpe
->vpe_db_lpi
, rdbase
+ GICR_SETLPIR
);
2893 gic_write_lpir(vpe
->vpe_db_lpi
, rdbase
+ GICR_CLRLPIR
);
2894 while (gic_read_lpir(rdbase
+ GICR_SYNCR
) & 1)
2899 its_vpe_send_cmd(vpe
, its_send_int
);
2901 its_vpe_send_cmd(vpe
, its_send_clear
);
2907 static struct irq_chip its_vpe_irq_chip
= {
2908 .name
= "GICv4-vpe",
2909 .irq_mask
= its_vpe_mask_irq
,
2910 .irq_unmask
= its_vpe_unmask_irq
,
2911 .irq_eoi
= irq_chip_eoi_parent
,
2912 .irq_set_affinity
= its_vpe_set_affinity
,
2913 .irq_set_irqchip_state
= its_vpe_set_irqchip_state
,
2914 .irq_set_vcpu_affinity
= its_vpe_set_vcpu_affinity
,
2917 static int its_vpe_id_alloc(void)
2919 return ida_simple_get(&its_vpeid_ida
, 0, ITS_MAX_VPEID
, GFP_KERNEL
);
2922 static void its_vpe_id_free(u16 id
)
2924 ida_simple_remove(&its_vpeid_ida
, id
);
2927 static int its_vpe_init(struct its_vpe
*vpe
)
2929 struct page
*vpt_page
;
2932 /* Allocate vpe_id */
2933 vpe_id
= its_vpe_id_alloc();
2938 vpt_page
= its_allocate_pending_table(GFP_KERNEL
);
2940 its_vpe_id_free(vpe_id
);
2944 if (!its_alloc_vpe_table(vpe_id
)) {
2945 its_vpe_id_free(vpe_id
);
2946 its_free_pending_table(vpe
->vpt_page
);
2950 vpe
->vpe_id
= vpe_id
;
2951 vpe
->vpt_page
= vpt_page
;
2952 vpe
->vpe_proxy_event
= -1;
2957 static void its_vpe_teardown(struct its_vpe
*vpe
)
2959 its_vpe_db_proxy_unmap(vpe
);
2960 its_vpe_id_free(vpe
->vpe_id
);
2961 its_free_pending_table(vpe
->vpt_page
);
2964 static void its_vpe_irq_domain_free(struct irq_domain
*domain
,
2966 unsigned int nr_irqs
)
2968 struct its_vm
*vm
= domain
->host_data
;
2971 irq_domain_free_irqs_parent(domain
, virq
, nr_irqs
);
2973 for (i
= 0; i
< nr_irqs
; i
++) {
2974 struct irq_data
*data
= irq_domain_get_irq_data(domain
,
2976 struct its_vpe
*vpe
= irq_data_get_irq_chip_data(data
);
2978 BUG_ON(vm
!= vpe
->its_vm
);
2980 clear_bit(data
->hwirq
, vm
->db_bitmap
);
2981 its_vpe_teardown(vpe
);
2982 irq_domain_reset_irq_data(data
);
2985 if (bitmap_empty(vm
->db_bitmap
, vm
->nr_db_lpis
)) {
2986 its_lpi_free(vm
->db_bitmap
, vm
->db_lpi_base
, vm
->nr_db_lpis
);
2987 its_free_prop_table(vm
->vprop_page
);
2991 static int its_vpe_irq_domain_alloc(struct irq_domain
*domain
, unsigned int virq
,
2992 unsigned int nr_irqs
, void *args
)
2994 struct its_vm
*vm
= args
;
2995 unsigned long *bitmap
;
2996 struct page
*vprop_page
;
2997 int base
, nr_ids
, i
, err
= 0;
3001 bitmap
= its_lpi_alloc(roundup_pow_of_two(nr_irqs
), &base
, &nr_ids
);
3005 if (nr_ids
< nr_irqs
) {
3006 its_lpi_free(bitmap
, base
, nr_ids
);
3010 vprop_page
= its_allocate_prop_table(GFP_KERNEL
);
3012 its_lpi_free(bitmap
, base
, nr_ids
);
3016 vm
->db_bitmap
= bitmap
;
3017 vm
->db_lpi_base
= base
;
3018 vm
->nr_db_lpis
= nr_ids
;
3019 vm
->vprop_page
= vprop_page
;
3021 for (i
= 0; i
< nr_irqs
; i
++) {
3022 vm
->vpes
[i
]->vpe_db_lpi
= base
+ i
;
3023 err
= its_vpe_init(vm
->vpes
[i
]);
3026 err
= its_irq_gic_domain_alloc(domain
, virq
+ i
,
3027 vm
->vpes
[i
]->vpe_db_lpi
);
3030 irq_domain_set_hwirq_and_chip(domain
, virq
+ i
, i
,
3031 &its_vpe_irq_chip
, vm
->vpes
[i
]);
3037 its_vpe_irq_domain_free(domain
, virq
, i
- 1);
3039 its_lpi_free(bitmap
, base
, nr_ids
);
3040 its_free_prop_table(vprop_page
);
3046 static int its_vpe_irq_domain_activate(struct irq_domain
*domain
,
3047 struct irq_data
*d
, bool reserve
)
3049 struct its_vpe
*vpe
= irq_data_get_irq_chip_data(d
);
3050 struct its_node
*its
;
3052 /* If we use the list map, we issue VMAPP on demand... */
3056 /* Map the VPE to the first possible CPU */
3057 vpe
->col_idx
= cpumask_first(cpu_online_mask
);
3059 list_for_each_entry(its
, &its_nodes
, entry
) {
3063 its_send_vmapp(its
, vpe
, true);
3064 its_send_vinvall(its
, vpe
);
3067 irq_data_update_effective_affinity(d
, cpumask_of(vpe
->col_idx
));
3072 static void its_vpe_irq_domain_deactivate(struct irq_domain
*domain
,
3075 struct its_vpe
*vpe
= irq_data_get_irq_chip_data(d
);
3076 struct its_node
*its
;
3079 * If we use the list map, we unmap the VPE once no VLPIs are
3080 * associated with the VM.
3085 list_for_each_entry(its
, &its_nodes
, entry
) {
3089 its_send_vmapp(its
, vpe
, false);
3093 static const struct irq_domain_ops its_vpe_domain_ops
= {
3094 .alloc
= its_vpe_irq_domain_alloc
,
3095 .free
= its_vpe_irq_domain_free
,
3096 .activate
= its_vpe_irq_domain_activate
,
3097 .deactivate
= its_vpe_irq_domain_deactivate
,
3100 static int its_force_quiescent(void __iomem
*base
)
3102 u32 count
= 1000000; /* 1s */
3105 val
= readl_relaxed(base
+ GITS_CTLR
);
3107 * GIC architecture specification requires the ITS to be both
3108 * disabled and quiescent for writes to GITS_BASER<n> or
3109 * GITS_CBASER to not have UNPREDICTABLE results.
3111 if ((val
& GITS_CTLR_QUIESCENT
) && !(val
& GITS_CTLR_ENABLE
))
3114 /* Disable the generation of all interrupts to this ITS */
3115 val
&= ~(GITS_CTLR_ENABLE
| GITS_CTLR_ImDe
);
3116 writel_relaxed(val
, base
+ GITS_CTLR
);
3118 /* Poll GITS_CTLR and wait until ITS becomes quiescent */
3120 val
= readl_relaxed(base
+ GITS_CTLR
);
3121 if (val
& GITS_CTLR_QUIESCENT
)
3133 static bool __maybe_unused
its_enable_quirk_cavium_22375(void *data
)
3135 struct its_node
*its
= data
;
3137 /* erratum 22375: only alloc 8MB table size */
3138 its
->device_ids
= 0x14; /* 20 bits, 8MB */
3139 its
->flags
|= ITS_FLAGS_WORKAROUND_CAVIUM_22375
;
3144 static bool __maybe_unused
its_enable_quirk_cavium_23144(void *data
)
3146 struct its_node
*its
= data
;
3148 its
->flags
|= ITS_FLAGS_WORKAROUND_CAVIUM_23144
;
3153 static bool __maybe_unused
its_enable_quirk_qdf2400_e0065(void *data
)
3155 struct its_node
*its
= data
;
3157 /* On QDF2400, the size of the ITE is 16Bytes */
3163 static u64
its_irq_get_msi_base_pre_its(struct its_device
*its_dev
)
3165 struct its_node
*its
= its_dev
->its
;
3168 * The Socionext Synquacer SoC has a so-called 'pre-ITS',
3169 * which maps 32-bit writes targeted at a separate window of
3170 * size '4 << device_id_bits' onto writes to GITS_TRANSLATER
3171 * with device ID taken from bits [device_id_bits + 1:2] of
3172 * the window offset.
3174 return its
->pre_its_base
+ (its_dev
->device_id
<< 2);
3177 static bool __maybe_unused
its_enable_quirk_socionext_synquacer(void *data
)
3179 struct its_node
*its
= data
;
3180 u32 pre_its_window
[2];
3183 if (!fwnode_property_read_u32_array(its
->fwnode_handle
,
3184 "socionext,synquacer-pre-its",
3186 ARRAY_SIZE(pre_its_window
))) {
3188 its
->pre_its_base
= pre_its_window
[0];
3189 its
->get_msi_base
= its_irq_get_msi_base_pre_its
;
3191 ids
= ilog2(pre_its_window
[1]) - 2;
3192 if (its
->device_ids
> ids
)
3193 its
->device_ids
= ids
;
3195 /* the pre-ITS breaks isolation, so disable MSI remapping */
3196 its
->msi_domain_flags
&= ~IRQ_DOMAIN_FLAG_MSI_REMAP
;
3202 static bool __maybe_unused
its_enable_quirk_hip07_161600802(void *data
)
3204 struct its_node
*its
= data
;
3207 * Hip07 insists on using the wrong address for the VLPI
3208 * page. Trick it into doing the right thing...
3210 its
->vlpi_redist_offset
= SZ_128K
;
3214 static const struct gic_quirk its_quirks
[] = {
3215 #ifdef CONFIG_CAVIUM_ERRATUM_22375
3217 .desc
= "ITS: Cavium errata 22375, 24313",
3218 .iidr
= 0xa100034c, /* ThunderX pass 1.x */
3220 .init
= its_enable_quirk_cavium_22375
,
3223 #ifdef CONFIG_CAVIUM_ERRATUM_23144
3225 .desc
= "ITS: Cavium erratum 23144",
3226 .iidr
= 0xa100034c, /* ThunderX pass 1.x */
3228 .init
= its_enable_quirk_cavium_23144
,
3231 #ifdef CONFIG_QCOM_QDF2400_ERRATUM_0065
3233 .desc
= "ITS: QDF2400 erratum 0065",
3234 .iidr
= 0x00001070, /* QDF2400 ITS rev 1.x */
3236 .init
= its_enable_quirk_qdf2400_e0065
,
3239 #ifdef CONFIG_SOCIONEXT_SYNQUACER_PREITS
3242 * The Socionext Synquacer SoC incorporates ARM's own GIC-500
3243 * implementation, but with a 'pre-ITS' added that requires
3244 * special handling in software.
3246 .desc
= "ITS: Socionext Synquacer pre-ITS",
3249 .init
= its_enable_quirk_socionext_synquacer
,
3252 #ifdef CONFIG_HISILICON_ERRATUM_161600802
3254 .desc
= "ITS: Hip07 erratum 161600802",
3257 .init
= its_enable_quirk_hip07_161600802
,
3264 static void its_enable_quirks(struct its_node
*its
)
3266 u32 iidr
= readl_relaxed(its
->base
+ GITS_IIDR
);
3268 gic_enable_quirks(iidr
, its_quirks
, its
);
3271 static int its_save_disable(void)
3273 struct its_node
*its
;
3276 raw_spin_lock(&its_lock
);
3277 list_for_each_entry(its
, &its_nodes
, entry
) {
3280 if (!(its
->flags
& ITS_FLAGS_SAVE_SUSPEND_STATE
))
3284 its
->ctlr_save
= readl_relaxed(base
+ GITS_CTLR
);
3285 err
= its_force_quiescent(base
);
3287 pr_err("ITS@%pa: failed to quiesce: %d\n",
3288 &its
->phys_base
, err
);
3289 writel_relaxed(its
->ctlr_save
, base
+ GITS_CTLR
);
3293 its
->cbaser_save
= gits_read_cbaser(base
+ GITS_CBASER
);
3298 list_for_each_entry_continue_reverse(its
, &its_nodes
, entry
) {
3301 if (!(its
->flags
& ITS_FLAGS_SAVE_SUSPEND_STATE
))
3305 writel_relaxed(its
->ctlr_save
, base
+ GITS_CTLR
);
3308 raw_spin_unlock(&its_lock
);
3313 static void its_restore_enable(void)
3315 struct its_node
*its
;
3318 raw_spin_lock(&its_lock
);
3319 list_for_each_entry(its
, &its_nodes
, entry
) {
3323 if (!(its
->flags
& ITS_FLAGS_SAVE_SUSPEND_STATE
))
3329 * Make sure that the ITS is disabled. If it fails to quiesce,
3330 * don't restore it since writing to CBASER or BASER<n>
3331 * registers is undefined according to the GIC v3 ITS
3334 ret
= its_force_quiescent(base
);
3336 pr_err("ITS@%pa: failed to quiesce on resume: %d\n",
3337 &its
->phys_base
, ret
);
3341 gits_write_cbaser(its
->cbaser_save
, base
+ GITS_CBASER
);
3344 * Writing CBASER resets CREADR to 0, so make CWRITER and
3345 * cmd_write line up with it.
3347 its
->cmd_write
= its
->cmd_base
;
3348 gits_write_cwriter(0, base
+ GITS_CWRITER
);
3350 /* Restore GITS_BASER from the value cache. */
3351 for (i
= 0; i
< GITS_BASER_NR_REGS
; i
++) {
3352 struct its_baser
*baser
= &its
->tables
[i
];
3354 if (!(baser
->val
& GITS_BASER_VALID
))
3357 its_write_baser(its
, baser
, baser
->val
);
3359 writel_relaxed(its
->ctlr_save
, base
+ GITS_CTLR
);
3362 * Reinit the collection if it's stored in the ITS. This is
3363 * indicated by the col_id being less than the HCC field.
3364 * CID < HCC as specified in the GIC v3 Documentation.
3366 if (its
->collections
[smp_processor_id()].col_id
<
3367 GITS_TYPER_HCC(gic_read_typer(base
+ GITS_TYPER
)))
3368 its_cpu_init_collection(its
);
3370 raw_spin_unlock(&its_lock
);
3373 static struct syscore_ops its_syscore_ops
= {
3374 .suspend
= its_save_disable
,
3375 .resume
= its_restore_enable
,
3378 static int its_init_domain(struct fwnode_handle
*handle
, struct its_node
*its
)
3380 struct irq_domain
*inner_domain
;
3381 struct msi_domain_info
*info
;
3383 info
= kzalloc(sizeof(*info
), GFP_KERNEL
);
3387 inner_domain
= irq_domain_create_tree(handle
, &its_domain_ops
, its
);
3388 if (!inner_domain
) {
3393 inner_domain
->parent
= its_parent
;
3394 irq_domain_update_bus_token(inner_domain
, DOMAIN_BUS_NEXUS
);
3395 inner_domain
->flags
|= its
->msi_domain_flags
;
3396 info
->ops
= &its_msi_domain_ops
;
3398 inner_domain
->host_data
= info
;
3403 static int its_init_vpe_domain(void)
3405 struct its_node
*its
;
3409 if (gic_rdists
->has_direct_lpi
) {
3410 pr_info("ITS: Using DirectLPI for VPE invalidation\n");
3414 /* Any ITS will do, even if not v4 */
3415 its
= list_first_entry(&its_nodes
, struct its_node
, entry
);
3417 entries
= roundup_pow_of_two(nr_cpu_ids
);
3418 vpe_proxy
.vpes
= kcalloc(entries
, sizeof(*vpe_proxy
.vpes
),
3420 if (!vpe_proxy
.vpes
) {
3421 pr_err("ITS: Can't allocate GICv4 proxy device array\n");
3425 /* Use the last possible DevID */
3426 devid
= GENMASK(its
->device_ids
- 1, 0);
3427 vpe_proxy
.dev
= its_create_device(its
, devid
, entries
, false);
3428 if (!vpe_proxy
.dev
) {
3429 kfree(vpe_proxy
.vpes
);
3430 pr_err("ITS: Can't allocate GICv4 proxy device\n");
3434 BUG_ON(entries
> vpe_proxy
.dev
->nr_ites
);
3436 raw_spin_lock_init(&vpe_proxy
.lock
);
3437 vpe_proxy
.next_victim
= 0;
3438 pr_info("ITS: Allocated DevID %x as GICv4 proxy device (%d slots)\n",
3439 devid
, vpe_proxy
.dev
->nr_ites
);
3444 static int __init
its_compute_its_list_map(struct resource
*res
,
3445 void __iomem
*its_base
)
3451 * This is assumed to be done early enough that we're
3452 * guaranteed to be single-threaded, hence no
3453 * locking. Should this change, we should address
3456 its_number
= find_first_zero_bit(&its_list_map
, GICv4_ITS_LIST_MAX
);
3457 if (its_number
>= GICv4_ITS_LIST_MAX
) {
3458 pr_err("ITS@%pa: No ITSList entry available!\n",
3463 ctlr
= readl_relaxed(its_base
+ GITS_CTLR
);
3464 ctlr
&= ~GITS_CTLR_ITS_NUMBER
;
3465 ctlr
|= its_number
<< GITS_CTLR_ITS_NUMBER_SHIFT
;
3466 writel_relaxed(ctlr
, its_base
+ GITS_CTLR
);
3467 ctlr
= readl_relaxed(its_base
+ GITS_CTLR
);
3468 if ((ctlr
& GITS_CTLR_ITS_NUMBER
) != (its_number
<< GITS_CTLR_ITS_NUMBER_SHIFT
)) {
3469 its_number
= ctlr
& GITS_CTLR_ITS_NUMBER
;
3470 its_number
>>= GITS_CTLR_ITS_NUMBER_SHIFT
;
3473 if (test_and_set_bit(its_number
, &its_list_map
)) {
3474 pr_err("ITS@%pa: Duplicate ITSList entry %d\n",
3475 &res
->start
, its_number
);
3482 static int __init
its_probe_one(struct resource
*res
,
3483 struct fwnode_handle
*handle
, int numa_node
)
3485 struct its_node
*its
;
3486 void __iomem
*its_base
;
3488 u64 baser
, tmp
, typer
;
3491 its_base
= ioremap(res
->start
, resource_size(res
));
3493 pr_warn("ITS@%pa: Unable to map ITS registers\n", &res
->start
);
3497 val
= readl_relaxed(its_base
+ GITS_PIDR2
) & GIC_PIDR2_ARCH_MASK
;
3498 if (val
!= 0x30 && val
!= 0x40) {
3499 pr_warn("ITS@%pa: No ITS detected, giving up\n", &res
->start
);
3504 err
= its_force_quiescent(its_base
);
3506 pr_warn("ITS@%pa: Failed to quiesce, giving up\n", &res
->start
);
3510 pr_info("ITS %pR\n", res
);
3512 its
= kzalloc(sizeof(*its
), GFP_KERNEL
);
3518 raw_spin_lock_init(&its
->lock
);
3519 INIT_LIST_HEAD(&its
->entry
);
3520 INIT_LIST_HEAD(&its
->its_device_list
);
3521 typer
= gic_read_typer(its_base
+ GITS_TYPER
);
3522 its
->base
= its_base
;
3523 its
->phys_base
= res
->start
;
3524 its
->ite_size
= GITS_TYPER_ITT_ENTRY_SIZE(typer
);
3525 its
->device_ids
= GITS_TYPER_DEVBITS(typer
);
3526 its
->is_v4
= !!(typer
& GITS_TYPER_VLPIS
);
3528 if (!(typer
& GITS_TYPER_VMOVP
)) {
3529 err
= its_compute_its_list_map(res
, its_base
);
3535 pr_info("ITS@%pa: Using ITS number %d\n",
3538 pr_info("ITS@%pa: Single VMOVP capable\n", &res
->start
);
3542 its
->numa_node
= numa_node
;
3544 its
->cmd_base
= (void *)__get_free_pages(GFP_KERNEL
| __GFP_ZERO
,
3545 get_order(ITS_CMD_QUEUE_SZ
));
3546 if (!its
->cmd_base
) {
3550 its
->cmd_write
= its
->cmd_base
;
3551 its
->fwnode_handle
= handle
;
3552 its
->get_msi_base
= its_irq_get_msi_base
;
3553 its
->msi_domain_flags
= IRQ_DOMAIN_FLAG_MSI_REMAP
;
3555 its_enable_quirks(its
);
3557 err
= its_alloc_tables(its
);
3561 err
= its_alloc_collections(its
);
3563 goto out_free_tables
;
3565 baser
= (virt_to_phys(its
->cmd_base
) |
3566 GITS_CBASER_RaWaWb
|
3567 GITS_CBASER_InnerShareable
|
3568 (ITS_CMD_QUEUE_SZ
/ SZ_4K
- 1) |
3571 gits_write_cbaser(baser
, its
->base
+ GITS_CBASER
);
3572 tmp
= gits_read_cbaser(its
->base
+ GITS_CBASER
);
3574 if ((tmp
^ baser
) & GITS_CBASER_SHAREABILITY_MASK
) {
3575 if (!(tmp
& GITS_CBASER_SHAREABILITY_MASK
)) {
3577 * The HW reports non-shareable, we must
3578 * remove the cacheability attributes as
3581 baser
&= ~(GITS_CBASER_SHAREABILITY_MASK
|
3582 GITS_CBASER_CACHEABILITY_MASK
);
3583 baser
|= GITS_CBASER_nC
;
3584 gits_write_cbaser(baser
, its
->base
+ GITS_CBASER
);
3586 pr_info("ITS: using cache flushing for cmd queue\n");
3587 its
->flags
|= ITS_FLAGS_CMDQ_NEEDS_FLUSHING
;
3590 gits_write_cwriter(0, its
->base
+ GITS_CWRITER
);
3591 ctlr
= readl_relaxed(its
->base
+ GITS_CTLR
);
3592 ctlr
|= GITS_CTLR_ENABLE
;
3594 ctlr
|= GITS_CTLR_ImDe
;
3595 writel_relaxed(ctlr
, its
->base
+ GITS_CTLR
);
3597 if (GITS_TYPER_HCC(typer
))
3598 its
->flags
|= ITS_FLAGS_SAVE_SUSPEND_STATE
;
3600 err
= its_init_domain(handle
, its
);
3602 goto out_free_tables
;
3604 raw_spin_lock(&its_lock
);
3605 list_add(&its
->entry
, &its_nodes
);
3606 raw_spin_unlock(&its_lock
);
3611 its_free_tables(its
);
3613 free_pages((unsigned long)its
->cmd_base
, get_order(ITS_CMD_QUEUE_SZ
));
3618 pr_err("ITS@%pa: failed probing (%d)\n", &res
->start
, err
);
3622 static bool gic_rdists_supports_plpis(void)
3624 return !!(gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER
) & GICR_TYPER_PLPIS
);
3627 static int redist_disable_lpis(void)
3629 void __iomem
*rbase
= gic_data_rdist_rd_base();
3630 u64 timeout
= USEC_PER_SEC
;
3633 if (!gic_rdists_supports_plpis()) {
3634 pr_info("CPU%d: LPIs not supported\n", smp_processor_id());
3638 val
= readl_relaxed(rbase
+ GICR_CTLR
);
3639 if (!(val
& GICR_CTLR_ENABLE_LPIS
))
3643 * If coming via a CPU hotplug event, we don't need to disable
3644 * LPIs before trying to re-enable them. They are already
3645 * configured and all is well in the world.
3647 * If running with preallocated tables, there is nothing to do.
3649 if (gic_data_rdist()->lpi_enabled
||
3650 (gic_rdists
->flags
& RDIST_FLAGS_RD_TABLES_PREALLOCATED
))
3654 * From that point on, we only try to do some damage control.
3656 pr_warn("GICv3: CPU%d: Booted with LPIs enabled, memory probably corrupted\n",
3657 smp_processor_id());
3658 add_taint(TAINT_CRAP
, LOCKDEP_STILL_OK
);
3661 val
&= ~GICR_CTLR_ENABLE_LPIS
;
3662 writel_relaxed(val
, rbase
+ GICR_CTLR
);
3664 /* Make sure any change to GICR_CTLR is observable by the GIC */
3668 * Software must observe RWP==0 after clearing GICR_CTLR.EnableLPIs
3669 * from 1 to 0 before programming GICR_PEND{PROP}BASER registers.
3670 * Error out if we time out waiting for RWP to clear.
3672 while (readl_relaxed(rbase
+ GICR_CTLR
) & GICR_CTLR_RWP
) {
3674 pr_err("CPU%d: Timeout while disabling LPIs\n",
3675 smp_processor_id());
3683 * After it has been written to 1, it is IMPLEMENTATION
3684 * DEFINED whether GICR_CTLR.EnableLPI becomes RES1 or can be
3685 * cleared to 0. Error out if clearing the bit failed.
3687 if (readl_relaxed(rbase
+ GICR_CTLR
) & GICR_CTLR_ENABLE_LPIS
) {
3688 pr_err("CPU%d: Failed to disable LPIs\n", smp_processor_id());
3695 int its_cpu_init(void)
3697 if (!list_empty(&its_nodes
)) {
3700 ret
= redist_disable_lpis();
3704 its_cpu_init_lpis();
3705 its_cpu_init_collections();
3711 static const struct of_device_id its_device_id
[] = {
3712 { .compatible
= "arm,gic-v3-its", },
3716 static int __init
its_of_probe(struct device_node
*node
)
3718 struct device_node
*np
;
3719 struct resource res
;
3721 for (np
= of_find_matching_node(node
, its_device_id
); np
;
3722 np
= of_find_matching_node(np
, its_device_id
)) {
3723 if (!of_device_is_available(np
))
3725 if (!of_property_read_bool(np
, "msi-controller")) {
3726 pr_warn("%pOF: no msi-controller property, ITS ignored\n",
3731 if (of_address_to_resource(np
, 0, &res
)) {
3732 pr_warn("%pOF: no regs?\n", np
);
3736 its_probe_one(&res
, &np
->fwnode
, of_node_to_nid(np
));
3743 #define ACPI_GICV3_ITS_MEM_SIZE (SZ_128K)
3745 #ifdef CONFIG_ACPI_NUMA
3746 struct its_srat_map
{
3753 static struct its_srat_map
*its_srat_maps __initdata
;
3754 static int its_in_srat __initdata
;
3756 static int __init
acpi_get_its_numa_node(u32 its_id
)
3760 for (i
= 0; i
< its_in_srat
; i
++) {
3761 if (its_id
== its_srat_maps
[i
].its_id
)
3762 return its_srat_maps
[i
].numa_node
;
3764 return NUMA_NO_NODE
;
3767 static int __init
gic_acpi_match_srat_its(struct acpi_subtable_header
*header
,
3768 const unsigned long end
)
3773 static int __init
gic_acpi_parse_srat_its(struct acpi_subtable_header
*header
,
3774 const unsigned long end
)
3777 struct acpi_srat_gic_its_affinity
*its_affinity
;
3779 its_affinity
= (struct acpi_srat_gic_its_affinity
*)header
;
3783 if (its_affinity
->header
.length
< sizeof(*its_affinity
)) {
3784 pr_err("SRAT: Invalid header length %d in ITS affinity\n",
3785 its_affinity
->header
.length
);
3789 node
= acpi_map_pxm_to_node(its_affinity
->proximity_domain
);
3791 if (node
== NUMA_NO_NODE
|| node
>= MAX_NUMNODES
) {
3792 pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node
);
3796 its_srat_maps
[its_in_srat
].numa_node
= node
;
3797 its_srat_maps
[its_in_srat
].its_id
= its_affinity
->its_id
;
3799 pr_info("SRAT: PXM %d -> ITS %d -> Node %d\n",
3800 its_affinity
->proximity_domain
, its_affinity
->its_id
, node
);
3805 static void __init
acpi_table_parse_srat_its(void)
3809 count
= acpi_table_parse_entries(ACPI_SIG_SRAT
,
3810 sizeof(struct acpi_table_srat
),
3811 ACPI_SRAT_TYPE_GIC_ITS_AFFINITY
,
3812 gic_acpi_match_srat_its
, 0);
3816 its_srat_maps
= kmalloc_array(count
, sizeof(struct its_srat_map
),
3818 if (!its_srat_maps
) {
3819 pr_warn("SRAT: Failed to allocate memory for its_srat_maps!\n");
3823 acpi_table_parse_entries(ACPI_SIG_SRAT
,
3824 sizeof(struct acpi_table_srat
),
3825 ACPI_SRAT_TYPE_GIC_ITS_AFFINITY
,
3826 gic_acpi_parse_srat_its
, 0);
3829 /* free the its_srat_maps after ITS probing */
3830 static void __init
acpi_its_srat_maps_free(void)
3832 kfree(its_srat_maps
);
3835 static void __init
acpi_table_parse_srat_its(void) { }
3836 static int __init
acpi_get_its_numa_node(u32 its_id
) { return NUMA_NO_NODE
; }
3837 static void __init
acpi_its_srat_maps_free(void) { }
3840 static int __init
gic_acpi_parse_madt_its(struct acpi_subtable_header
*header
,
3841 const unsigned long end
)
3843 struct acpi_madt_generic_translator
*its_entry
;
3844 struct fwnode_handle
*dom_handle
;
3845 struct resource res
;
3848 its_entry
= (struct acpi_madt_generic_translator
*)header
;
3849 memset(&res
, 0, sizeof(res
));
3850 res
.start
= its_entry
->base_address
;
3851 res
.end
= its_entry
->base_address
+ ACPI_GICV3_ITS_MEM_SIZE
- 1;
3852 res
.flags
= IORESOURCE_MEM
;
3854 dom_handle
= irq_domain_alloc_fwnode((void *)its_entry
->base_address
);
3856 pr_err("ITS@%pa: Unable to allocate GICv3 ITS domain token\n",
3861 err
= iort_register_domain_token(its_entry
->translation_id
, res
.start
,
3864 pr_err("ITS@%pa: Unable to register GICv3 ITS domain token (ITS ID %d) to IORT\n",
3865 &res
.start
, its_entry
->translation_id
);
3869 err
= its_probe_one(&res
, dom_handle
,
3870 acpi_get_its_numa_node(its_entry
->translation_id
));
3874 iort_deregister_domain_token(its_entry
->translation_id
);
3876 irq_domain_free_fwnode(dom_handle
);
3880 static void __init
its_acpi_probe(void)
3882 acpi_table_parse_srat_its();
3883 acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR
,
3884 gic_acpi_parse_madt_its
, 0);
3885 acpi_its_srat_maps_free();
3888 static void __init
its_acpi_probe(void) { }
3891 int __init
its_init(struct fwnode_handle
*handle
, struct rdists
*rdists
,
3892 struct irq_domain
*parent_domain
)
3894 struct device_node
*of_node
;
3895 struct its_node
*its
;
3896 bool has_v4
= false;
3899 its_parent
= parent_domain
;
3900 of_node
= to_of_node(handle
);
3902 its_of_probe(of_node
);
3906 if (list_empty(&its_nodes
)) {
3907 pr_warn("ITS: No ITS available, not enabling LPIs\n");
3911 gic_rdists
= rdists
;
3913 err
= allocate_lpi_tables();
3917 list_for_each_entry(its
, &its_nodes
, entry
)
3918 has_v4
|= its
->is_v4
;
3920 if (has_v4
& rdists
->has_vlpis
) {
3921 if (its_init_vpe_domain() ||
3922 its_init_v4(parent_domain
, &its_vpe_domain_ops
)) {
3923 rdists
->has_vlpis
= false;
3924 pr_err("ITS: Disabling GICv4 support\n");
3928 register_syscore_ops(&its_syscore_ops
);