]>
Commit | Line | Data |
---|---|---|
cc2d3216 MZ |
1 | /* |
2 | * Copyright (C) 2013, 2014 ARM Limited, All Rights Reserved. | |
3 | * Author: Marc Zyngier <marc.zyngier@arm.com> | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify | |
6 | * it under the terms of the GNU General Public License version 2 as | |
7 | * published by the Free Software Foundation. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, | |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | * GNU General Public License for more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License | |
15 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
16 | */ | |
17 | ||
18 | #include <linux/bitmap.h> | |
19 | #include <linux/cpu.h> | |
20 | #include <linux/delay.h> | |
21 | #include <linux/interrupt.h> | |
22 | #include <linux/log2.h> | |
23 | #include <linux/mm.h> | |
24 | #include <linux/msi.h> | |
25 | #include <linux/of.h> | |
26 | #include <linux/of_address.h> | |
27 | #include <linux/of_irq.h> | |
28 | #include <linux/of_pci.h> | |
29 | #include <linux/of_platform.h> | |
30 | #include <linux/percpu.h> | |
31 | #include <linux/slab.h> | |
32 | ||
33 | #include <linux/irqchip/arm-gic-v3.h> | |
34 | ||
35 | #include <asm/cacheflush.h> | |
36 | #include <asm/cputype.h> | |
37 | #include <asm/exception.h> | |
38 | ||
39 | #include "irqchip.h" | |
40 | ||
41 | #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING (1 << 0) | |
42 | ||
43 | /* | |
44 | * Collection structure - just an ID, and a redistributor address to | |
45 | * ping. We use one per CPU as a bag of interrupts assigned to this | |
46 | * CPU. | |
47 | */ | |
48 | struct its_collection { | |
49 | u64 target_address; | |
50 | u16 col_id; | |
51 | }; | |
52 | ||
53 | /* | |
54 | * The ITS structure - contains most of the infrastructure, with the | |
55 | * msi_controller, the command queue, the collections, and the list of | |
56 | * devices writing to it. | |
57 | */ | |
58 | struct its_node { | |
59 | raw_spinlock_t lock; | |
60 | struct list_head entry; | |
61 | struct msi_controller msi_chip; | |
62 | struct irq_domain *domain; | |
63 | void __iomem *base; | |
64 | unsigned long phys_base; | |
65 | struct its_cmd_block *cmd_base; | |
66 | struct its_cmd_block *cmd_write; | |
67 | void *tables[GITS_BASER_NR_REGS]; | |
68 | struct its_collection *collections; | |
69 | struct list_head its_device_list; | |
70 | u64 flags; | |
71 | u32 ite_size; | |
72 | }; | |
73 | ||
74 | #define ITS_ITT_ALIGN SZ_256 | |
75 | ||
76 | /* | |
77 | * The ITS view of a device - belongs to an ITS, a collection, owns an | |
78 | * interrupt translation table, and a list of interrupts. | |
79 | */ | |
80 | struct its_device { | |
81 | struct list_head entry; | |
82 | struct its_node *its; | |
83 | struct its_collection *collection; | |
84 | void *itt; | |
85 | unsigned long *lpi_map; | |
86 | irq_hw_number_t lpi_base; | |
87 | int nr_lpis; | |
88 | u32 nr_ites; | |
89 | u32 device_id; | |
90 | }; | |
91 | ||
92 | /* | |
93 | * ITS command descriptors - parameters to be encoded in a command | |
94 | * block. | |
95 | */ | |
96 | struct its_cmd_desc { | |
97 | union { | |
98 | struct { | |
99 | struct its_device *dev; | |
100 | u32 event_id; | |
101 | } its_inv_cmd; | |
102 | ||
103 | struct { | |
104 | struct its_device *dev; | |
105 | u32 event_id; | |
106 | } its_int_cmd; | |
107 | ||
108 | struct { | |
109 | struct its_device *dev; | |
110 | int valid; | |
111 | } its_mapd_cmd; | |
112 | ||
113 | struct { | |
114 | struct its_collection *col; | |
115 | int valid; | |
116 | } its_mapc_cmd; | |
117 | ||
118 | struct { | |
119 | struct its_device *dev; | |
120 | u32 phys_id; | |
121 | u32 event_id; | |
122 | } its_mapvi_cmd; | |
123 | ||
124 | struct { | |
125 | struct its_device *dev; | |
126 | struct its_collection *col; | |
127 | u32 id; | |
128 | } its_movi_cmd; | |
129 | ||
130 | struct { | |
131 | struct its_device *dev; | |
132 | u32 event_id; | |
133 | } its_discard_cmd; | |
134 | ||
135 | struct { | |
136 | struct its_collection *col; | |
137 | } its_invall_cmd; | |
138 | }; | |
139 | }; | |
140 | ||
141 | /* | |
142 | * The ITS command block, which is what the ITS actually parses. | |
143 | */ | |
144 | struct its_cmd_block { | |
145 | u64 raw_cmd[4]; | |
146 | }; | |
147 | ||
148 | #define ITS_CMD_QUEUE_SZ SZ_64K | |
149 | #define ITS_CMD_QUEUE_NR_ENTRIES (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block)) | |
150 | ||
151 | typedef struct its_collection *(*its_cmd_builder_t)(struct its_cmd_block *, | |
152 | struct its_cmd_desc *); | |
153 | ||
154 | static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr) | |
155 | { | |
156 | cmd->raw_cmd[0] &= ~0xffUL; | |
157 | cmd->raw_cmd[0] |= cmd_nr; | |
158 | } | |
159 | ||
160 | static void its_encode_devid(struct its_cmd_block *cmd, u32 devid) | |
161 | { | |
162 | cmd->raw_cmd[0] &= ~(0xffffUL << 32); | |
163 | cmd->raw_cmd[0] |= ((u64)devid) << 32; | |
164 | } | |
165 | ||
166 | static void its_encode_event_id(struct its_cmd_block *cmd, u32 id) | |
167 | { | |
168 | cmd->raw_cmd[1] &= ~0xffffffffUL; | |
169 | cmd->raw_cmd[1] |= id; | |
170 | } | |
171 | ||
172 | static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id) | |
173 | { | |
174 | cmd->raw_cmd[1] &= 0xffffffffUL; | |
175 | cmd->raw_cmd[1] |= ((u64)phys_id) << 32; | |
176 | } | |
177 | ||
178 | static void its_encode_size(struct its_cmd_block *cmd, u8 size) | |
179 | { | |
180 | cmd->raw_cmd[1] &= ~0x1fUL; | |
181 | cmd->raw_cmd[1] |= size & 0x1f; | |
182 | } | |
183 | ||
184 | static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr) | |
185 | { | |
186 | cmd->raw_cmd[2] &= ~0xffffffffffffUL; | |
187 | cmd->raw_cmd[2] |= itt_addr & 0xffffffffff00UL; | |
188 | } | |
189 | ||
190 | static void its_encode_valid(struct its_cmd_block *cmd, int valid) | |
191 | { | |
192 | cmd->raw_cmd[2] &= ~(1UL << 63); | |
193 | cmd->raw_cmd[2] |= ((u64)!!valid) << 63; | |
194 | } | |
195 | ||
196 | static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr) | |
197 | { | |
198 | cmd->raw_cmd[2] &= ~(0xffffffffUL << 16); | |
199 | cmd->raw_cmd[2] |= (target_addr & (0xffffffffUL << 16)); | |
200 | } | |
201 | ||
202 | static void its_encode_collection(struct its_cmd_block *cmd, u16 col) | |
203 | { | |
204 | cmd->raw_cmd[2] &= ~0xffffUL; | |
205 | cmd->raw_cmd[2] |= col; | |
206 | } | |
207 | ||
208 | static inline void its_fixup_cmd(struct its_cmd_block *cmd) | |
209 | { | |
210 | /* Let's fixup BE commands */ | |
211 | cmd->raw_cmd[0] = cpu_to_le64(cmd->raw_cmd[0]); | |
212 | cmd->raw_cmd[1] = cpu_to_le64(cmd->raw_cmd[1]); | |
213 | cmd->raw_cmd[2] = cpu_to_le64(cmd->raw_cmd[2]); | |
214 | cmd->raw_cmd[3] = cpu_to_le64(cmd->raw_cmd[3]); | |
215 | } | |
216 | ||
217 | static struct its_collection *its_build_mapd_cmd(struct its_cmd_block *cmd, | |
218 | struct its_cmd_desc *desc) | |
219 | { | |
220 | unsigned long itt_addr; | |
221 | u8 size = order_base_2(desc->its_mapd_cmd.dev->nr_ites); | |
222 | ||
223 | itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt); | |
224 | itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN); | |
225 | ||
226 | its_encode_cmd(cmd, GITS_CMD_MAPD); | |
227 | its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id); | |
228 | its_encode_size(cmd, size - 1); | |
229 | its_encode_itt(cmd, itt_addr); | |
230 | its_encode_valid(cmd, desc->its_mapd_cmd.valid); | |
231 | ||
232 | its_fixup_cmd(cmd); | |
233 | ||
234 | return desc->its_mapd_cmd.dev->collection; | |
235 | } | |
236 | ||
237 | static struct its_collection *its_build_mapc_cmd(struct its_cmd_block *cmd, | |
238 | struct its_cmd_desc *desc) | |
239 | { | |
240 | its_encode_cmd(cmd, GITS_CMD_MAPC); | |
241 | its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id); | |
242 | its_encode_target(cmd, desc->its_mapc_cmd.col->target_address); | |
243 | its_encode_valid(cmd, desc->its_mapc_cmd.valid); | |
244 | ||
245 | its_fixup_cmd(cmd); | |
246 | ||
247 | return desc->its_mapc_cmd.col; | |
248 | } | |
249 | ||
250 | static struct its_collection *its_build_mapvi_cmd(struct its_cmd_block *cmd, | |
251 | struct its_cmd_desc *desc) | |
252 | { | |
253 | its_encode_cmd(cmd, GITS_CMD_MAPVI); | |
254 | its_encode_devid(cmd, desc->its_mapvi_cmd.dev->device_id); | |
255 | its_encode_event_id(cmd, desc->its_mapvi_cmd.event_id); | |
256 | its_encode_phys_id(cmd, desc->its_mapvi_cmd.phys_id); | |
257 | its_encode_collection(cmd, desc->its_mapvi_cmd.dev->collection->col_id); | |
258 | ||
259 | its_fixup_cmd(cmd); | |
260 | ||
261 | return desc->its_mapvi_cmd.dev->collection; | |
262 | } | |
263 | ||
264 | static struct its_collection *its_build_movi_cmd(struct its_cmd_block *cmd, | |
265 | struct its_cmd_desc *desc) | |
266 | { | |
267 | its_encode_cmd(cmd, GITS_CMD_MOVI); | |
268 | its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id); | |
269 | its_encode_event_id(cmd, desc->its_movi_cmd.id); | |
270 | its_encode_collection(cmd, desc->its_movi_cmd.col->col_id); | |
271 | ||
272 | its_fixup_cmd(cmd); | |
273 | ||
274 | return desc->its_movi_cmd.dev->collection; | |
275 | } | |
276 | ||
277 | static struct its_collection *its_build_discard_cmd(struct its_cmd_block *cmd, | |
278 | struct its_cmd_desc *desc) | |
279 | { | |
280 | its_encode_cmd(cmd, GITS_CMD_DISCARD); | |
281 | its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id); | |
282 | its_encode_event_id(cmd, desc->its_discard_cmd.event_id); | |
283 | ||
284 | its_fixup_cmd(cmd); | |
285 | ||
286 | return desc->its_discard_cmd.dev->collection; | |
287 | } | |
288 | ||
289 | static struct its_collection *its_build_inv_cmd(struct its_cmd_block *cmd, | |
290 | struct its_cmd_desc *desc) | |
291 | { | |
292 | its_encode_cmd(cmd, GITS_CMD_INV); | |
293 | its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id); | |
294 | its_encode_event_id(cmd, desc->its_inv_cmd.event_id); | |
295 | ||
296 | its_fixup_cmd(cmd); | |
297 | ||
298 | return desc->its_inv_cmd.dev->collection; | |
299 | } | |
300 | ||
301 | static struct its_collection *its_build_invall_cmd(struct its_cmd_block *cmd, | |
302 | struct its_cmd_desc *desc) | |
303 | { | |
304 | its_encode_cmd(cmd, GITS_CMD_INVALL); | |
305 | its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id); | |
306 | ||
307 | its_fixup_cmd(cmd); | |
308 | ||
309 | return NULL; | |
310 | } | |
311 | ||
312 | static u64 its_cmd_ptr_to_offset(struct its_node *its, | |
313 | struct its_cmd_block *ptr) | |
314 | { | |
315 | return (ptr - its->cmd_base) * sizeof(*ptr); | |
316 | } | |
317 | ||
318 | static int its_queue_full(struct its_node *its) | |
319 | { | |
320 | int widx; | |
321 | int ridx; | |
322 | ||
323 | widx = its->cmd_write - its->cmd_base; | |
324 | ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block); | |
325 | ||
326 | /* This is incredibly unlikely to happen, unless the ITS locks up. */ | |
327 | if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx) | |
328 | return 1; | |
329 | ||
330 | return 0; | |
331 | } | |
332 | ||
333 | static struct its_cmd_block *its_allocate_entry(struct its_node *its) | |
334 | { | |
335 | struct its_cmd_block *cmd; | |
336 | u32 count = 1000000; /* 1s! */ | |
337 | ||
338 | while (its_queue_full(its)) { | |
339 | count--; | |
340 | if (!count) { | |
341 | pr_err_ratelimited("ITS queue not draining\n"); | |
342 | return NULL; | |
343 | } | |
344 | cpu_relax(); | |
345 | udelay(1); | |
346 | } | |
347 | ||
348 | cmd = its->cmd_write++; | |
349 | ||
350 | /* Handle queue wrapping */ | |
351 | if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES)) | |
352 | its->cmd_write = its->cmd_base; | |
353 | ||
354 | return cmd; | |
355 | } | |
356 | ||
357 | static struct its_cmd_block *its_post_commands(struct its_node *its) | |
358 | { | |
359 | u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write); | |
360 | ||
361 | writel_relaxed(wr, its->base + GITS_CWRITER); | |
362 | ||
363 | return its->cmd_write; | |
364 | } | |
365 | ||
366 | static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd) | |
367 | { | |
368 | /* | |
369 | * Make sure the commands written to memory are observable by | |
370 | * the ITS. | |
371 | */ | |
372 | if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING) | |
373 | __flush_dcache_area(cmd, sizeof(*cmd)); | |
374 | else | |
375 | dsb(ishst); | |
376 | } | |
377 | ||
378 | static void its_wait_for_range_completion(struct its_node *its, | |
379 | struct its_cmd_block *from, | |
380 | struct its_cmd_block *to) | |
381 | { | |
382 | u64 rd_idx, from_idx, to_idx; | |
383 | u32 count = 1000000; /* 1s! */ | |
384 | ||
385 | from_idx = its_cmd_ptr_to_offset(its, from); | |
386 | to_idx = its_cmd_ptr_to_offset(its, to); | |
387 | ||
388 | while (1) { | |
389 | rd_idx = readl_relaxed(its->base + GITS_CREADR); | |
390 | if (rd_idx >= to_idx || rd_idx < from_idx) | |
391 | break; | |
392 | ||
393 | count--; | |
394 | if (!count) { | |
395 | pr_err_ratelimited("ITS queue timeout\n"); | |
396 | return; | |
397 | } | |
398 | cpu_relax(); | |
399 | udelay(1); | |
400 | } | |
401 | } | |
402 | ||
403 | static void its_send_single_command(struct its_node *its, | |
404 | its_cmd_builder_t builder, | |
405 | struct its_cmd_desc *desc) | |
406 | { | |
407 | struct its_cmd_block *cmd, *sync_cmd, *next_cmd; | |
408 | struct its_collection *sync_col; | |
409 | ||
410 | raw_spin_lock(&its->lock); | |
411 | ||
412 | cmd = its_allocate_entry(its); | |
413 | if (!cmd) { /* We're soooooo screewed... */ | |
414 | pr_err_ratelimited("ITS can't allocate, dropping command\n"); | |
415 | raw_spin_unlock(&its->lock); | |
416 | return; | |
417 | } | |
418 | sync_col = builder(cmd, desc); | |
419 | its_flush_cmd(its, cmd); | |
420 | ||
421 | if (sync_col) { | |
422 | sync_cmd = its_allocate_entry(its); | |
423 | if (!sync_cmd) { | |
424 | pr_err_ratelimited("ITS can't SYNC, skipping\n"); | |
425 | goto post; | |
426 | } | |
427 | its_encode_cmd(sync_cmd, GITS_CMD_SYNC); | |
428 | its_encode_target(sync_cmd, sync_col->target_address); | |
429 | its_fixup_cmd(sync_cmd); | |
430 | its_flush_cmd(its, sync_cmd); | |
431 | } | |
432 | ||
433 | post: | |
434 | next_cmd = its_post_commands(its); | |
435 | raw_spin_unlock(&its->lock); | |
436 | ||
437 | its_wait_for_range_completion(its, cmd, next_cmd); | |
438 | } | |
439 | ||
440 | static void its_send_inv(struct its_device *dev, u32 event_id) | |
441 | { | |
442 | struct its_cmd_desc desc; | |
443 | ||
444 | desc.its_inv_cmd.dev = dev; | |
445 | desc.its_inv_cmd.event_id = event_id; | |
446 | ||
447 | its_send_single_command(dev->its, its_build_inv_cmd, &desc); | |
448 | } | |
449 | ||
450 | static void its_send_mapd(struct its_device *dev, int valid) | |
451 | { | |
452 | struct its_cmd_desc desc; | |
453 | ||
454 | desc.its_mapd_cmd.dev = dev; | |
455 | desc.its_mapd_cmd.valid = !!valid; | |
456 | ||
457 | its_send_single_command(dev->its, its_build_mapd_cmd, &desc); | |
458 | } | |
459 | ||
460 | static void its_send_mapc(struct its_node *its, struct its_collection *col, | |
461 | int valid) | |
462 | { | |
463 | struct its_cmd_desc desc; | |
464 | ||
465 | desc.its_mapc_cmd.col = col; | |
466 | desc.its_mapc_cmd.valid = !!valid; | |
467 | ||
468 | its_send_single_command(its, its_build_mapc_cmd, &desc); | |
469 | } | |
470 | ||
471 | static void its_send_mapvi(struct its_device *dev, u32 irq_id, u32 id) | |
472 | { | |
473 | struct its_cmd_desc desc; | |
474 | ||
475 | desc.its_mapvi_cmd.dev = dev; | |
476 | desc.its_mapvi_cmd.phys_id = irq_id; | |
477 | desc.its_mapvi_cmd.event_id = id; | |
478 | ||
479 | its_send_single_command(dev->its, its_build_mapvi_cmd, &desc); | |
480 | } | |
481 | ||
482 | static void its_send_movi(struct its_device *dev, | |
483 | struct its_collection *col, u32 id) | |
484 | { | |
485 | struct its_cmd_desc desc; | |
486 | ||
487 | desc.its_movi_cmd.dev = dev; | |
488 | desc.its_movi_cmd.col = col; | |
489 | desc.its_movi_cmd.id = id; | |
490 | ||
491 | its_send_single_command(dev->its, its_build_movi_cmd, &desc); | |
492 | } | |
493 | ||
494 | static void its_send_discard(struct its_device *dev, u32 id) | |
495 | { | |
496 | struct its_cmd_desc desc; | |
497 | ||
498 | desc.its_discard_cmd.dev = dev; | |
499 | desc.its_discard_cmd.event_id = id; | |
500 | ||
501 | its_send_single_command(dev->its, its_build_discard_cmd, &desc); | |
502 | } | |
503 | ||
504 | static void its_send_invall(struct its_node *its, struct its_collection *col) | |
505 | { | |
506 | struct its_cmd_desc desc; | |
507 | ||
508 | desc.its_invall_cmd.col = col; | |
509 | ||
510 | its_send_single_command(its, its_build_invall_cmd, &desc); | |
511 | } |