]>
Commit | Line | Data |
---|---|---|
74afab7a JL |
1 | /* |
2 | * Local APIC related interfaces to support IOAPIC, MSI, HT_IRQ etc. | |
3 | * | |
4 | * Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo | |
5 | * Moved from arch/x86/kernel/apic/io_apic.c. | |
b5dc8e6c JL |
6 | * Jiang Liu <jiang.liu@linux.intel.com> |
7 | * Enable support of hierarchical irqdomains | |
74afab7a JL |
8 | * |
9 | * This program is free software; you can redistribute it and/or modify | |
10 | * it under the terms of the GNU General Public License version 2 as | |
11 | * published by the Free Software Foundation. | |
12 | */ | |
13 | #include <linux/interrupt.h> | |
65d7ed57 | 14 | #include <linux/seq_file.h> |
74afab7a JL |
15 | #include <linux/init.h> |
16 | #include <linux/compiler.h> | |
74afab7a | 17 | #include <linux/slab.h> |
d746d1eb | 18 | #include <asm/irqdomain.h> |
74afab7a JL |
19 | #include <asm/hw_irq.h> |
20 | #include <asm/apic.h> | |
21 | #include <asm/i8259.h> | |
22 | #include <asm/desc.h> | |
23 | #include <asm/irq_remapping.h> | |
24 | ||
8d1e3dca TG |
25 | #include <asm/trace/irq_vectors.h> |
26 | ||
7f3262ed | 27 | struct apic_chip_data { |
ba224fea TG |
28 | struct irq_cfg hw_irq_cfg; |
29 | unsigned int vector; | |
30 | unsigned int prev_vector; | |
029c6e1c TG |
31 | unsigned int cpu; |
32 | unsigned int prev_cpu; | |
69cde000 | 33 | unsigned int irq; |
dccfe314 | 34 | struct hlist_node clist; |
7f3262ed JL |
35 | u8 move_in_progress : 1; |
36 | }; | |
37 | ||
b5dc8e6c | 38 | struct irq_domain *x86_vector_domain; |
c8f3e518 | 39 | EXPORT_SYMBOL_GPL(x86_vector_domain); |
74afab7a | 40 | static DEFINE_RAW_SPINLOCK(vector_lock); |
69cde000 | 41 | static cpumask_var_t vector_searchmask; |
b5dc8e6c | 42 | static struct irq_chip lapic_controller; |
0fa115da | 43 | static struct irq_matrix *vector_matrix; |
dccfe314 TG |
44 | #ifdef CONFIG_SMP |
45 | static DEFINE_PER_CPU(struct hlist_head, cleanup_list); | |
46 | #endif | |
74afab7a JL |
47 | |
48 | void lock_vector_lock(void) | |
49 | { | |
50 | /* Used to the online set of cpus does not change | |
51 | * during assign_irq_vector. | |
52 | */ | |
53 | raw_spin_lock(&vector_lock); | |
54 | } | |
55 | ||
56 | void unlock_vector_lock(void) | |
57 | { | |
58 | raw_spin_unlock(&vector_lock); | |
59 | } | |
60 | ||
99a1482d TG |
61 | void init_irq_alloc_info(struct irq_alloc_info *info, |
62 | const struct cpumask *mask) | |
63 | { | |
64 | memset(info, 0, sizeof(*info)); | |
65 | info->mask = mask; | |
66 | } | |
67 | ||
68 | void copy_irq_alloc_info(struct irq_alloc_info *dst, struct irq_alloc_info *src) | |
69 | { | |
70 | if (src) | |
71 | *dst = *src; | |
72 | else | |
73 | memset(dst, 0, sizeof(*dst)); | |
74 | } | |
75 | ||
86ba6551 | 76 | static struct apic_chip_data *apic_chip_data(struct irq_data *irqd) |
74afab7a | 77 | { |
86ba6551 | 78 | if (!irqd) |
b5dc8e6c JL |
79 | return NULL; |
80 | ||
86ba6551 TG |
81 | while (irqd->parent_data) |
82 | irqd = irqd->parent_data; | |
b5dc8e6c | 83 | |
86ba6551 | 84 | return irqd->chip_data; |
74afab7a JL |
85 | } |
86 | ||
86ba6551 | 87 | struct irq_cfg *irqd_cfg(struct irq_data *irqd) |
7f3262ed | 88 | { |
86ba6551 | 89 | struct apic_chip_data *apicd = apic_chip_data(irqd); |
7f3262ed | 90 | |
ba224fea | 91 | return apicd ? &apicd->hw_irq_cfg : NULL; |
7f3262ed | 92 | } |
c8f3e518 | 93 | EXPORT_SYMBOL_GPL(irqd_cfg); |
7f3262ed JL |
94 | |
95 | struct irq_cfg *irq_cfg(unsigned int irq) | |
74afab7a | 96 | { |
7f3262ed JL |
97 | return irqd_cfg(irq_get_irq_data(irq)); |
98 | } | |
74afab7a | 99 | |
7f3262ed JL |
100 | static struct apic_chip_data *alloc_apic_chip_data(int node) |
101 | { | |
86ba6551 | 102 | struct apic_chip_data *apicd; |
7f3262ed | 103 | |
86ba6551 | 104 | apicd = kzalloc_node(sizeof(*apicd), GFP_KERNEL, node); |
69cde000 TG |
105 | if (apicd) |
106 | INIT_HLIST_NODE(&apicd->clist); | |
86ba6551 | 107 | return apicd; |
74afab7a JL |
108 | } |
109 | ||
86ba6551 | 110 | static void free_apic_chip_data(struct apic_chip_data *apicd) |
74afab7a | 111 | { |
69cde000 | 112 | kfree(apicd); |
74afab7a JL |
113 | } |
114 | ||
ba224fea TG |
115 | static void apic_update_irq_cfg(struct irq_data *irqd, unsigned int vector, |
116 | unsigned int cpu) | |
74afab7a | 117 | { |
69cde000 | 118 | struct apic_chip_data *apicd = apic_chip_data(irqd); |
74afab7a | 119 | |
69cde000 | 120 | lockdep_assert_held(&vector_lock); |
74afab7a | 121 | |
ba224fea TG |
122 | apicd->hw_irq_cfg.vector = vector; |
123 | apicd->hw_irq_cfg.dest_apicid = apic->calc_dest_apicid(cpu); | |
124 | irq_data_update_effective_affinity(irqd, cpumask_of(cpu)); | |
125 | trace_vector_config(irqd->irq, vector, cpu, | |
126 | apicd->hw_irq_cfg.dest_apicid); | |
69cde000 | 127 | } |
74afab7a | 128 | |
69cde000 TG |
129 | static void apic_update_vector(struct irq_data *irqd, unsigned int newvec, |
130 | unsigned int newcpu) | |
131 | { | |
132 | struct apic_chip_data *apicd = apic_chip_data(irqd); | |
133 | struct irq_desc *desc = irq_data_to_desc(irqd); | |
74afab7a | 134 | |
69cde000 | 135 | lockdep_assert_held(&vector_lock); |
74afab7a | 136 | |
ba224fea | 137 | trace_vector_update(irqd->irq, newvec, newcpu, apicd->vector, |
69cde000 | 138 | apicd->cpu); |
74afab7a | 139 | |
69cde000 | 140 | /* Setup the vector move, if required */ |
ba224fea | 141 | if (apicd->vector && cpu_online(apicd->cpu)) { |
69cde000 | 142 | apicd->move_in_progress = true; |
ba224fea | 143 | apicd->prev_vector = apicd->vector; |
69cde000 TG |
144 | apicd->prev_cpu = apicd->cpu; |
145 | } else { | |
ba224fea | 146 | apicd->prev_vector = 0; |
69cde000 | 147 | } |
74afab7a | 148 | |
ba224fea | 149 | apicd->vector = newvec; |
69cde000 TG |
150 | apicd->cpu = newcpu; |
151 | BUG_ON(!IS_ERR_OR_NULL(per_cpu(vector_irq, newcpu)[newvec])); | |
152 | per_cpu(vector_irq, newcpu)[newvec] = desc; | |
153 | } | |
74afab7a | 154 | |
69cde000 TG |
155 | static int allocate_vector(struct irq_data *irqd, const struct cpumask *dest) |
156 | { | |
157 | struct apic_chip_data *apicd = apic_chip_data(irqd); | |
69cde000 | 158 | unsigned int cpu = apicd->cpu; |
ba224fea TG |
159 | int vector = apicd->vector; |
160 | ||
161 | lockdep_assert_held(&vector_lock); | |
74afab7a | 162 | |
3716fd27 | 163 | /* |
69cde000 TG |
164 | * If the current target CPU is online and in the new requested |
165 | * affinity mask, there is no point in moving the interrupt from | |
166 | * one CPU to another. | |
3716fd27 | 167 | */ |
69cde000 TG |
168 | if (vector && cpu_online(cpu) && cpumask_test_cpu(cpu, dest)) |
169 | return 0; | |
170 | ||
171 | vector = irq_matrix_alloc(vector_matrix, dest, false, &cpu); | |
172 | if (vector > 0) | |
173 | apic_update_vector(irqd, vector, cpu); | |
174 | trace_vector_alloc(irqd->irq, vector, false, vector); | |
175 | return vector; | |
176 | } | |
177 | ||
178 | static int assign_vector_locked(struct irq_data *irqd, | |
179 | const struct cpumask *dest) | |
180 | { | |
ba224fea | 181 | struct apic_chip_data *apicd = apic_chip_data(irqd); |
69cde000 TG |
182 | int vector = allocate_vector(irqd, dest); |
183 | ||
184 | if (vector < 0) | |
185 | return vector; | |
186 | ||
ba224fea | 187 | apic_update_irq_cfg(irqd, apicd->vector, apicd->cpu); |
3716fd27 | 188 | return 0; |
74afab7a JL |
189 | } |
190 | ||
69cde000 | 191 | static int assign_irq_vector(struct irq_data *irqd, const struct cpumask *dest) |
74afab7a | 192 | { |
74afab7a | 193 | unsigned long flags; |
69cde000 | 194 | int ret; |
74afab7a JL |
195 | |
196 | raw_spin_lock_irqsave(&vector_lock, flags); | |
69cde000 TG |
197 | cpumask_and(vector_searchmask, dest, cpu_online_mask); |
198 | ret = assign_vector_locked(irqd, vector_searchmask); | |
74afab7a | 199 | raw_spin_unlock_irqrestore(&vector_lock, flags); |
69cde000 | 200 | return ret; |
74afab7a JL |
201 | } |
202 | ||
69cde000 TG |
203 | static int assign_irq_vector_policy(struct irq_data *irqd, |
204 | struct irq_alloc_info *info, int node) | |
486ca539 | 205 | { |
258d86ee | 206 | if (info->mask) |
69cde000 | 207 | return assign_irq_vector(irqd, info->mask); |
486ca539 | 208 | if (node != NUMA_NO_NODE && |
69cde000 | 209 | !assign_irq_vector(irqd, cpumask_of_node(node))) |
486ca539 | 210 | return 0; |
69cde000 | 211 | return assign_irq_vector(irqd, cpu_online_mask); |
486ca539 JL |
212 | } |
213 | ||
69cde000 | 214 | static void clear_irq_vector(struct irq_data *irqd) |
74afab7a | 215 | { |
69cde000 | 216 | struct apic_chip_data *apicd = apic_chip_data(irqd); |
ba224fea | 217 | unsigned int vector = apicd->vector; |
74afab7a | 218 | |
69cde000 | 219 | lockdep_assert_held(&vector_lock); |
ba224fea | 220 | |
dccfe314 | 221 | if (!vector) |
1bdb8970 | 222 | return; |
74afab7a | 223 | |
ba224fea | 224 | trace_vector_clear(irqd->irq, vector, apicd->cpu, apicd->prev_vector, |
69cde000 TG |
225 | apicd->prev_cpu); |
226 | ||
dccfe314 | 227 | per_cpu(vector_irq, apicd->cpu)[vector] = VECTOR_UNUSED; |
69cde000 | 228 | irq_matrix_free(vector_matrix, apicd->cpu, vector, false); |
ba224fea | 229 | apicd->vector = 0; |
74afab7a | 230 | |
dccfe314 | 231 | /* Clean up move in progress */ |
ba224fea | 232 | vector = apicd->prev_vector; |
dccfe314 | 233 | if (!vector) |
74afab7a | 234 | return; |
74afab7a | 235 | |
dccfe314 | 236 | per_cpu(vector_irq, apicd->prev_cpu)[vector] = VECTOR_UNUSED; |
69cde000 | 237 | irq_matrix_free(vector_matrix, apicd->prev_cpu, vector, false); |
ba224fea | 238 | apicd->prev_vector = 0; |
86ba6551 | 239 | apicd->move_in_progress = 0; |
dccfe314 | 240 | hlist_del_init(&apicd->clist); |
74afab7a JL |
241 | } |
242 | ||
b5dc8e6c JL |
243 | static void x86_vector_free_irqs(struct irq_domain *domain, |
244 | unsigned int virq, unsigned int nr_irqs) | |
245 | { | |
86ba6551 TG |
246 | struct apic_chip_data *apicd; |
247 | struct irq_data *irqd; | |
111abeba | 248 | unsigned long flags; |
b5dc8e6c JL |
249 | int i; |
250 | ||
251 | for (i = 0; i < nr_irqs; i++) { | |
86ba6551 TG |
252 | irqd = irq_domain_get_irq_data(x86_vector_domain, virq + i); |
253 | if (irqd && irqd->chip_data) { | |
111abeba | 254 | raw_spin_lock_irqsave(&vector_lock, flags); |
69cde000 | 255 | clear_irq_vector(irqd); |
86ba6551 TG |
256 | apicd = irqd->chip_data; |
257 | irq_domain_reset_irq_data(irqd); | |
111abeba | 258 | raw_spin_unlock_irqrestore(&vector_lock, flags); |
86ba6551 | 259 | free_apic_chip_data(apicd); |
b5dc8e6c JL |
260 | } |
261 | } | |
262 | } | |
263 | ||
264 | static int x86_vector_alloc_irqs(struct irq_domain *domain, unsigned int virq, | |
265 | unsigned int nr_irqs, void *arg) | |
266 | { | |
267 | struct irq_alloc_info *info = arg; | |
86ba6551 TG |
268 | struct apic_chip_data *apicd; |
269 | struct irq_data *irqd; | |
69cde000 | 270 | unsigned long flags; |
5f2dbbc5 | 271 | int i, err, node; |
b5dc8e6c JL |
272 | |
273 | if (disable_apic) | |
274 | return -ENXIO; | |
275 | ||
276 | /* Currently vector allocator can't guarantee contiguous allocations */ | |
277 | if ((info->flags & X86_IRQ_ALLOC_CONTIGUOUS_VECTORS) && nr_irqs > 1) | |
278 | return -ENOSYS; | |
279 | ||
b5dc8e6c | 280 | for (i = 0; i < nr_irqs; i++) { |
86ba6551 TG |
281 | irqd = irq_domain_get_irq_data(domain, virq + i); |
282 | BUG_ON(!irqd); | |
283 | node = irq_data_get_node(irqd); | |
4ef76eb6 TG |
284 | WARN_ON_ONCE(irqd->chip_data); |
285 | apicd = alloc_apic_chip_data(node); | |
86ba6551 | 286 | if (!apicd) { |
b5dc8e6c JL |
287 | err = -ENOMEM; |
288 | goto error; | |
289 | } | |
290 | ||
69cde000 | 291 | apicd->irq = virq + i; |
86ba6551 TG |
292 | irqd->chip = &lapic_controller; |
293 | irqd->chip_data = apicd; | |
294 | irqd->hwirq = virq + i; | |
295 | irqd_set_single_target(irqd); | |
4ef76eb6 | 296 | /* |
69cde000 TG |
297 | * Legacy vectors are already assigned when the IOAPIC |
298 | * takes them over. They stay on the same vector. This is | |
299 | * required for check_timer() to work correctly as it might | |
300 | * switch back to legacy mode. Only update the hardware | |
301 | * config. | |
4ef76eb6 TG |
302 | */ |
303 | if (info->flags & X86_IRQ_ALLOC_LEGACY) { | |
ba224fea | 304 | apicd->vector = ISA_IRQ_VECTOR(virq + i); |
4ef76eb6 | 305 | apicd->cpu = 0; |
69cde000 TG |
306 | trace_vector_setup(virq + i, true, 0); |
307 | raw_spin_lock_irqsave(&vector_lock, flags); | |
ba224fea | 308 | apic_update_irq_cfg(irqd, apicd->vector, apicd->cpu); |
69cde000 TG |
309 | raw_spin_unlock_irqrestore(&vector_lock, flags); |
310 | continue; | |
4ef76eb6 TG |
311 | } |
312 | ||
69cde000 TG |
313 | err = assign_irq_vector_policy(irqd, info, node); |
314 | trace_vector_setup(virq + i, false, err); | |
b5dc8e6c JL |
315 | if (err) |
316 | goto error; | |
317 | } | |
318 | ||
319 | return 0; | |
320 | ||
321 | error: | |
322 | x86_vector_free_irqs(domain, virq, i + 1); | |
323 | return err; | |
324 | } | |
325 | ||
65d7ed57 TG |
326 | #ifdef CONFIG_GENERIC_IRQ_DEBUGFS |
327 | void x86_vector_debug_show(struct seq_file *m, struct irq_domain *d, | |
328 | struct irq_data *irqd, int ind) | |
329 | { | |
ba224fea | 330 | unsigned int cpu, vector, prev_cpu, prev_vector; |
65d7ed57 TG |
331 | struct apic_chip_data *apicd; |
332 | unsigned long flags; | |
333 | int irq; | |
334 | ||
335 | if (!irqd) { | |
336 | irq_matrix_debug_show(m, vector_matrix, ind); | |
337 | return; | |
338 | } | |
339 | ||
340 | irq = irqd->irq; | |
341 | if (irq < nr_legacy_irqs() && !test_bit(irq, &io_apic_irqs)) { | |
342 | seq_printf(m, "%*sVector: %5d\n", ind, "", ISA_IRQ_VECTOR(irq)); | |
343 | seq_printf(m, "%*sTarget: Legacy PIC all CPUs\n", ind, ""); | |
344 | return; | |
345 | } | |
346 | ||
347 | apicd = irqd->chip_data; | |
348 | if (!apicd) { | |
349 | seq_printf(m, "%*sVector: Not assigned\n", ind, ""); | |
350 | return; | |
351 | } | |
352 | ||
353 | raw_spin_lock_irqsave(&vector_lock, flags); | |
354 | cpu = apicd->cpu; | |
ba224fea | 355 | vector = apicd->vector; |
65d7ed57 | 356 | prev_cpu = apicd->prev_cpu; |
ba224fea | 357 | prev_vector = apicd->prev_vector; |
65d7ed57 | 358 | raw_spin_unlock_irqrestore(&vector_lock, flags); |
ba224fea | 359 | seq_printf(m, "%*sVector: %5u\n", ind, "", vector); |
65d7ed57 | 360 | seq_printf(m, "%*sTarget: %5u\n", ind, "", cpu); |
ba224fea TG |
361 | if (prev_vector) { |
362 | seq_printf(m, "%*sPrevious vector: %5u\n", ind, "", prev_vector); | |
65d7ed57 TG |
363 | seq_printf(m, "%*sPrevious target: %5u\n", ind, "", prev_cpu); |
364 | } | |
365 | } | |
366 | #endif | |
367 | ||
eb18cf55 | 368 | static const struct irq_domain_ops x86_vector_domain_ops = { |
65d7ed57 TG |
369 | .alloc = x86_vector_alloc_irqs, |
370 | .free = x86_vector_free_irqs, | |
371 | #ifdef CONFIG_GENERIC_IRQ_DEBUGFS | |
372 | .debug_show = x86_vector_debug_show, | |
373 | #endif | |
b5dc8e6c JL |
374 | }; |
375 | ||
11d686e9 JL |
376 | int __init arch_probe_nr_irqs(void) |
377 | { | |
378 | int nr; | |
379 | ||
380 | if (nr_irqs > (NR_VECTORS * nr_cpu_ids)) | |
381 | nr_irqs = NR_VECTORS * nr_cpu_ids; | |
382 | ||
383 | nr = (gsi_top + nr_legacy_irqs()) + 8 * nr_cpu_ids; | |
384 | #if defined(CONFIG_PCI_MSI) || defined(CONFIG_HT_IRQ) | |
385 | /* | |
386 | * for MSI and HT dyn irq | |
387 | */ | |
388 | if (gsi_top <= NR_IRQS_LEGACY) | |
389 | nr += 8 * nr_cpu_ids; | |
390 | else | |
391 | nr += gsi_top * 16; | |
392 | #endif | |
393 | if (nr < nr_irqs) | |
394 | nr_irqs = nr; | |
395 | ||
8c058b0b VK |
396 | /* |
397 | * We don't know if PIC is present at this point so we need to do | |
398 | * probe() to get the right number of legacy IRQs. | |
399 | */ | |
400 | return legacy_pic->probe(); | |
11d686e9 JL |
401 | } |
402 | ||
0fa115da TG |
403 | void lapic_assign_legacy_vector(unsigned int irq, bool replace) |
404 | { | |
405 | /* | |
406 | * Use assign system here so it wont get accounted as allocated | |
407 | * and moveable in the cpu hotplug check and it prevents managed | |
408 | * irq reservation from touching it. | |
409 | */ | |
410 | irq_matrix_assign_system(vector_matrix, ISA_IRQ_VECTOR(irq), replace); | |
411 | } | |
412 | ||
413 | void __init lapic_assign_system_vectors(void) | |
414 | { | |
415 | unsigned int i, vector = 0; | |
416 | ||
417 | for_each_set_bit_from(vector, system_vectors, NR_VECTORS) | |
418 | irq_matrix_assign_system(vector_matrix, vector, false); | |
419 | ||
420 | if (nr_legacy_irqs() > 1) | |
421 | lapic_assign_legacy_vector(PIC_CASCADE_IR, false); | |
422 | ||
423 | /* System vectors are reserved, online it */ | |
424 | irq_matrix_online(vector_matrix); | |
425 | ||
426 | /* Mark the preallocated legacy interrupts */ | |
427 | for (i = 0; i < nr_legacy_irqs(); i++) { | |
428 | if (i != PIC_CASCADE_IR) | |
429 | irq_matrix_assign(vector_matrix, ISA_IRQ_VECTOR(i)); | |
430 | } | |
431 | } | |
432 | ||
11d686e9 JL |
433 | int __init arch_early_irq_init(void) |
434 | { | |
9d35f859 TG |
435 | struct fwnode_handle *fn; |
436 | ||
9d35f859 TG |
437 | fn = irq_domain_alloc_named_fwnode("VECTOR"); |
438 | BUG_ON(!fn); | |
439 | x86_vector_domain = irq_domain_create_tree(fn, &x86_vector_domain_ops, | |
440 | NULL); | |
b5dc8e6c | 441 | BUG_ON(x86_vector_domain == NULL); |
9d35f859 | 442 | irq_domain_free_fwnode(fn); |
b5dc8e6c JL |
443 | irq_set_default_host(x86_vector_domain); |
444 | ||
52f518a3 | 445 | arch_init_msi_domain(x86_vector_domain); |
49e07d8f | 446 | arch_init_htirq_domain(x86_vector_domain); |
52f518a3 | 447 | |
3716fd27 | 448 | BUG_ON(!alloc_cpumask_var(&vector_searchmask, GFP_KERNEL)); |
f7fa7aee | 449 | |
0fa115da TG |
450 | /* |
451 | * Allocate the vector matrix allocator data structure and limit the | |
452 | * search area. | |
453 | */ | |
454 | vector_matrix = irq_alloc_matrix(NR_VECTORS, FIRST_EXTERNAL_VECTOR, | |
455 | FIRST_SYSTEM_VECTOR); | |
456 | BUG_ON(!vector_matrix); | |
457 | ||
11d686e9 JL |
458 | return arch_early_ioapic_init(); |
459 | } | |
460 | ||
ba801640 | 461 | #ifdef CONFIG_SMP |
f0cc6cca TG |
462 | /* Temporary hack to keep things working */ |
463 | static void vector_update_shutdown_irqs(void) | |
74afab7a | 464 | { |
a782a7e4 | 465 | struct irq_desc *desc; |
f0cc6cca | 466 | int irq; |
74afab7a | 467 | |
a782a7e4 | 468 | for_each_irq_desc(irq, desc) { |
f0cc6cca TG |
469 | struct irq_data *irqd = irq_desc_get_irq_data(desc); |
470 | struct apic_chip_data *ad = apic_chip_data(irqd); | |
74afab7a | 471 | |
ba224fea | 472 | if (!ad || !ad->vector || ad->cpu != smp_processor_id()) |
69cde000 | 473 | continue; |
ba224fea TG |
474 | this_cpu_write(vector_irq[ad->vector], desc); |
475 | irq_matrix_assign(vector_matrix, ad->vector); | |
74afab7a | 476 | } |
74afab7a JL |
477 | } |
478 | ||
f0cc6cca TG |
479 | static struct irq_desc *__setup_vector_irq(int vector) |
480 | { | |
481 | int isairq = vector - ISA_IRQ_VECTOR(0); | |
482 | ||
483 | /* Check whether the irq is in the legacy space */ | |
484 | if (isairq < 0 || isairq >= nr_legacy_irqs()) | |
485 | return VECTOR_UNUSED; | |
486 | /* Check whether the irq is handled by the IOAPIC */ | |
487 | if (test_bit(isairq, &io_apic_irqs)) | |
488 | return VECTOR_UNUSED; | |
489 | return irq_to_desc(isairq); | |
490 | } | |
491 | ||
0fa115da TG |
492 | /* Online the local APIC infrastructure and initialize the vectors */ |
493 | void lapic_online(void) | |
74afab7a | 494 | { |
f0cc6cca | 495 | unsigned int vector; |
74afab7a | 496 | |
5a3f75e3 | 497 | lockdep_assert_held(&vector_lock); |
0fa115da TG |
498 | |
499 | /* Online the vector matrix array for this CPU */ | |
500 | irq_matrix_online(vector_matrix); | |
501 | ||
74afab7a | 502 | /* |
f0cc6cca TG |
503 | * The interrupt affinity logic never targets interrupts to offline |
504 | * CPUs. The exception are the legacy PIC interrupts. In general | |
505 | * they are only targeted to CPU0, but depending on the platform | |
506 | * they can be distributed to any online CPU in hardware. The | |
507 | * kernel has no influence on that. So all active legacy vectors | |
508 | * must be installed on all CPUs. All non legacy interrupts can be | |
509 | * cleared. | |
74afab7a | 510 | */ |
f0cc6cca TG |
511 | for (vector = 0; vector < NR_VECTORS; vector++) |
512 | this_cpu_write(vector_irq[vector], __setup_vector_irq(vector)); | |
74afab7a | 513 | |
f0cc6cca TG |
514 | /* |
515 | * Until the rewrite of the managed interrupt management is in | |
516 | * place it's necessary to walk the irq descriptors and check for | |
517 | * interrupts which are targeted at this CPU. | |
518 | */ | |
519 | vector_update_shutdown_irqs(); | |
74afab7a JL |
520 | } |
521 | ||
0fa115da TG |
522 | void lapic_offline(void) |
523 | { | |
524 | lock_vector_lock(); | |
525 | irq_matrix_offline(vector_matrix); | |
526 | unlock_vector_lock(); | |
527 | } | |
528 | ||
ba801640 TG |
529 | static int apic_set_affinity(struct irq_data *irqd, |
530 | const struct cpumask *dest, bool force) | |
531 | { | |
532 | int err; | |
533 | ||
534 | if (!IS_ENABLED(CONFIG_SMP)) | |
535 | return -EPERM; | |
536 | ||
537 | if (!cpumask_intersects(dest, cpu_online_mask)) | |
538 | return -EINVAL; | |
539 | ||
540 | err = assign_irq_vector(irqd, dest); | |
541 | return err ? err : IRQ_SET_MASK_OK; | |
542 | } | |
543 | ||
544 | #else | |
545 | # define apic_set_affinity NULL | |
546 | #endif | |
547 | ||
86ba6551 | 548 | static int apic_retrigger_irq(struct irq_data *irqd) |
74afab7a | 549 | { |
86ba6551 | 550 | struct apic_chip_data *apicd = apic_chip_data(irqd); |
74afab7a | 551 | unsigned long flags; |
74afab7a JL |
552 | |
553 | raw_spin_lock_irqsave(&vector_lock, flags); | |
ba224fea | 554 | apic->send_IPI(apicd->cpu, apicd->vector); |
74afab7a JL |
555 | raw_spin_unlock_irqrestore(&vector_lock, flags); |
556 | ||
557 | return 1; | |
558 | } | |
559 | ||
86ba6551 | 560 | void apic_ack_edge(struct irq_data *irqd) |
74afab7a | 561 | { |
86ba6551 TG |
562 | irq_complete_move(irqd_cfg(irqd)); |
563 | irq_move_irq(irqd); | |
74afab7a JL |
564 | ack_APIC_irq(); |
565 | } | |
566 | ||
b5dc8e6c | 567 | static struct irq_chip lapic_controller = { |
8947dfb2 | 568 | .name = "APIC", |
b5dc8e6c | 569 | .irq_ack = apic_ack_edge, |
68f9f440 | 570 | .irq_set_affinity = apic_set_affinity, |
b5dc8e6c JL |
571 | .irq_retrigger = apic_retrigger_irq, |
572 | }; | |
573 | ||
74afab7a | 574 | #ifdef CONFIG_SMP |
c6c2002b | 575 | |
69cde000 TG |
576 | static void free_moved_vector(struct apic_chip_data *apicd) |
577 | { | |
ba224fea | 578 | unsigned int vector = apicd->prev_vector; |
69cde000 TG |
579 | unsigned int cpu = apicd->prev_cpu; |
580 | ||
581 | trace_vector_free_moved(apicd->irq, vector, false); | |
582 | irq_matrix_free(vector_matrix, cpu, vector, false); | |
583 | __this_cpu_write(vector_irq[vector], VECTOR_UNUSED); | |
584 | hlist_del_init(&apicd->clist); | |
ba224fea | 585 | apicd->prev_vector = 0; |
69cde000 TG |
586 | apicd->move_in_progress = 0; |
587 | } | |
588 | ||
c4158ff5 | 589 | asmlinkage __visible void __irq_entry smp_irq_move_cleanup_interrupt(void) |
74afab7a | 590 | { |
dccfe314 TG |
591 | struct hlist_head *clhead = this_cpu_ptr(&cleanup_list); |
592 | struct apic_chip_data *apicd; | |
593 | struct hlist_node *tmp; | |
74afab7a | 594 | |
6af7faf6 | 595 | entering_ack_irq(); |
df54c493 TG |
596 | /* Prevent vectors vanishing under us */ |
597 | raw_spin_lock(&vector_lock); | |
598 | ||
dccfe314 | 599 | hlist_for_each_entry_safe(apicd, tmp, clhead, clist) { |
ba224fea | 600 | unsigned int irr, vector = apicd->prev_vector; |
74afab7a | 601 | |
74afab7a | 602 | /* |
dccfe314 TG |
603 | * Paranoia: Check if the vector that needs to be cleaned |
604 | * up is registered at the APICs IRR. If so, then this is | |
605 | * not the best time to clean it up. Clean it up in the | |
74afab7a | 606 | * next attempt by sending another IRQ_MOVE_CLEANUP_VECTOR |
dccfe314 TG |
607 | * to this CPU. IRQ_MOVE_CLEANUP_VECTOR is the lowest |
608 | * priority external vector, so on return from this | |
609 | * interrupt the device interrupt will happen first. | |
74afab7a | 610 | */ |
dccfe314 TG |
611 | irr = apic_read(APIC_IRR + (vector / 32 * 0x10)); |
612 | if (irr & (1U << (vector % 32))) { | |
74afab7a | 613 | apic->send_IPI_self(IRQ_MOVE_CLEANUP_VECTOR); |
dccfe314 | 614 | continue; |
74afab7a | 615 | } |
69cde000 | 616 | free_moved_vector(apicd); |
74afab7a JL |
617 | } |
618 | ||
df54c493 | 619 | raw_spin_unlock(&vector_lock); |
6af7faf6 | 620 | exiting_irq(); |
74afab7a JL |
621 | } |
622 | ||
dccfe314 TG |
623 | static void __send_cleanup_vector(struct apic_chip_data *apicd) |
624 | { | |
625 | unsigned int cpu; | |
626 | ||
627 | raw_spin_lock(&vector_lock); | |
628 | apicd->move_in_progress = 0; | |
629 | cpu = apicd->prev_cpu; | |
630 | if (cpu_online(cpu)) { | |
631 | hlist_add_head(&apicd->clist, per_cpu_ptr(&cleanup_list, cpu)); | |
632 | apic->send_IPI(cpu, IRQ_MOVE_CLEANUP_VECTOR); | |
633 | } else { | |
ba224fea | 634 | apicd->prev_vector = 0; |
dccfe314 TG |
635 | } |
636 | raw_spin_unlock(&vector_lock); | |
637 | } | |
638 | ||
639 | void send_cleanup_vector(struct irq_cfg *cfg) | |
640 | { | |
641 | struct apic_chip_data *apicd; | |
642 | ||
ba224fea | 643 | apicd = container_of(cfg, struct apic_chip_data, hw_irq_cfg); |
dccfe314 TG |
644 | if (apicd->move_in_progress) |
645 | __send_cleanup_vector(apicd); | |
646 | } | |
647 | ||
74afab7a JL |
648 | static void __irq_complete_move(struct irq_cfg *cfg, unsigned vector) |
649 | { | |
86ba6551 | 650 | struct apic_chip_data *apicd; |
74afab7a | 651 | |
ba224fea | 652 | apicd = container_of(cfg, struct apic_chip_data, hw_irq_cfg); |
86ba6551 | 653 | if (likely(!apicd->move_in_progress)) |
74afab7a JL |
654 | return; |
655 | ||
ba224fea | 656 | if (vector == apicd->vector && apicd->cpu == smp_processor_id()) |
86ba6551 | 657 | __send_cleanup_vector(apicd); |
74afab7a JL |
658 | } |
659 | ||
660 | void irq_complete_move(struct irq_cfg *cfg) | |
661 | { | |
662 | __irq_complete_move(cfg, ~get_irq_regs()->orig_ax); | |
663 | } | |
664 | ||
90a2282e | 665 | /* |
551adc60 | 666 | * Called from fixup_irqs() with @desc->lock held and interrupts disabled. |
90a2282e TG |
667 | */ |
668 | void irq_force_complete_move(struct irq_desc *desc) | |
74afab7a | 669 | { |
86ba6551 | 670 | struct apic_chip_data *apicd; |
dccfe314 TG |
671 | struct irq_data *irqd; |
672 | unsigned int vector; | |
56d7d2f4 | 673 | |
db91aa79 MW |
674 | /* |
675 | * The function is called for all descriptors regardless of which | |
676 | * irqdomain they belong to. For example if an IRQ is provided by | |
677 | * an irq_chip as part of a GPIO driver, the chip data for that | |
678 | * descriptor is specific to the irq_chip in question. | |
679 | * | |
680 | * Check first that the chip_data is what we expect | |
681 | * (apic_chip_data) before touching it any further. | |
682 | */ | |
86ba6551 | 683 | irqd = irq_domain_get_irq_data(x86_vector_domain, |
dccfe314 | 684 | irq_desc_get_irq(desc)); |
86ba6551 | 685 | if (!irqd) |
db91aa79 MW |
686 | return; |
687 | ||
dccfe314 | 688 | raw_spin_lock(&vector_lock); |
86ba6551 | 689 | apicd = apic_chip_data(irqd); |
dccfe314 TG |
690 | if (!apicd) |
691 | goto unlock; | |
db91aa79 | 692 | |
dccfe314 | 693 | /* |
ba224fea | 694 | * If prev_vector is empty, no action required. |
dccfe314 | 695 | */ |
ba224fea | 696 | vector = apicd->prev_vector; |
dccfe314 TG |
697 | if (!vector) |
698 | goto unlock; | |
74afab7a | 699 | |
56d7d2f4 | 700 | /* |
dccfe314 | 701 | * This is tricky. If the cleanup of the old vector has not been |
98229aa3 TG |
702 | * done yet, then the following setaffinity call will fail with |
703 | * -EBUSY. This can leave the interrupt in a stale state. | |
704 | * | |
551adc60 TG |
705 | * All CPUs are stuck in stop machine with interrupts disabled so |
706 | * calling __irq_complete_move() would be completely pointless. | |
dccfe314 | 707 | * |
551adc60 TG |
708 | * 1) The interrupt is in move_in_progress state. That means that we |
709 | * have not seen an interrupt since the io_apic was reprogrammed to | |
710 | * the new vector. | |
711 | * | |
712 | * 2) The interrupt has fired on the new vector, but the cleanup IPIs | |
713 | * have not been processed yet. | |
714 | */ | |
86ba6551 | 715 | if (apicd->move_in_progress) { |
98229aa3 | 716 | /* |
551adc60 TG |
717 | * In theory there is a race: |
718 | * | |
719 | * set_ioapic(new_vector) <-- Interrupt is raised before update | |
720 | * is effective, i.e. it's raised on | |
721 | * the old vector. | |
722 | * | |
723 | * So if the target cpu cannot handle that interrupt before | |
724 | * the old vector is cleaned up, we get a spurious interrupt | |
725 | * and in the worst case the ioapic irq line becomes stale. | |
726 | * | |
727 | * But in case of cpu hotplug this should be a non issue | |
728 | * because if the affinity update happens right before all | |
729 | * cpus rendevouz in stop machine, there is no way that the | |
730 | * interrupt can be blocked on the target cpu because all cpus | |
731 | * loops first with interrupts enabled in stop machine, so the | |
732 | * old vector is not yet cleaned up when the interrupt fires. | |
733 | * | |
734 | * So the only way to run into this issue is if the delivery | |
735 | * of the interrupt on the apic/system bus would be delayed | |
736 | * beyond the point where the target cpu disables interrupts | |
737 | * in stop machine. I doubt that it can happen, but at least | |
738 | * there is a theroretical chance. Virtualization might be | |
739 | * able to expose this, but AFAICT the IOAPIC emulation is not | |
740 | * as stupid as the real hardware. | |
741 | * | |
742 | * Anyway, there is nothing we can do about that at this point | |
743 | * w/o refactoring the whole fixup_irq() business completely. | |
744 | * We print at least the irq number and the old vector number, | |
745 | * so we have the necessary information when a problem in that | |
746 | * area arises. | |
98229aa3 | 747 | */ |
551adc60 | 748 | pr_warn("IRQ fixup: irq %d move in progress, old vector %d\n", |
dccfe314 | 749 | irqd->irq, vector); |
98229aa3 | 750 | } |
69cde000 | 751 | free_moved_vector(apicd); |
dccfe314 | 752 | unlock: |
56d7d2f4 | 753 | raw_spin_unlock(&vector_lock); |
74afab7a | 754 | } |
74afab7a JL |
755 | #endif |
756 | ||
74afab7a JL |
757 | static void __init print_APIC_field(int base) |
758 | { | |
759 | int i; | |
760 | ||
761 | printk(KERN_DEBUG); | |
762 | ||
763 | for (i = 0; i < 8; i++) | |
764 | pr_cont("%08x", apic_read(base + i*0x10)); | |
765 | ||
766 | pr_cont("\n"); | |
767 | } | |
768 | ||
769 | static void __init print_local_APIC(void *dummy) | |
770 | { | |
771 | unsigned int i, v, ver, maxlvt; | |
772 | u64 icr; | |
773 | ||
849d3569 JL |
774 | pr_debug("printing local APIC contents on CPU#%d/%d:\n", |
775 | smp_processor_id(), hard_smp_processor_id()); | |
74afab7a | 776 | v = apic_read(APIC_ID); |
849d3569 | 777 | pr_info("... APIC ID: %08x (%01x)\n", v, read_apic_id()); |
74afab7a | 778 | v = apic_read(APIC_LVR); |
849d3569 | 779 | pr_info("... APIC VERSION: %08x\n", v); |
74afab7a JL |
780 | ver = GET_APIC_VERSION(v); |
781 | maxlvt = lapic_get_maxlvt(); | |
782 | ||
783 | v = apic_read(APIC_TASKPRI); | |
849d3569 | 784 | pr_debug("... APIC TASKPRI: %08x (%02x)\n", v, v & APIC_TPRI_MASK); |
74afab7a JL |
785 | |
786 | /* !82489DX */ | |
787 | if (APIC_INTEGRATED(ver)) { | |
788 | if (!APIC_XAPIC(ver)) { | |
789 | v = apic_read(APIC_ARBPRI); | |
849d3569 JL |
790 | pr_debug("... APIC ARBPRI: %08x (%02x)\n", |
791 | v, v & APIC_ARBPRI_MASK); | |
74afab7a JL |
792 | } |
793 | v = apic_read(APIC_PROCPRI); | |
849d3569 | 794 | pr_debug("... APIC PROCPRI: %08x\n", v); |
74afab7a JL |
795 | } |
796 | ||
797 | /* | |
798 | * Remote read supported only in the 82489DX and local APIC for | |
799 | * Pentium processors. | |
800 | */ | |
801 | if (!APIC_INTEGRATED(ver) || maxlvt == 3) { | |
802 | v = apic_read(APIC_RRR); | |
849d3569 | 803 | pr_debug("... APIC RRR: %08x\n", v); |
74afab7a JL |
804 | } |
805 | ||
806 | v = apic_read(APIC_LDR); | |
849d3569 | 807 | pr_debug("... APIC LDR: %08x\n", v); |
74afab7a JL |
808 | if (!x2apic_enabled()) { |
809 | v = apic_read(APIC_DFR); | |
849d3569 | 810 | pr_debug("... APIC DFR: %08x\n", v); |
74afab7a JL |
811 | } |
812 | v = apic_read(APIC_SPIV); | |
849d3569 | 813 | pr_debug("... APIC SPIV: %08x\n", v); |
74afab7a | 814 | |
849d3569 | 815 | pr_debug("... APIC ISR field:\n"); |
74afab7a | 816 | print_APIC_field(APIC_ISR); |
849d3569 | 817 | pr_debug("... APIC TMR field:\n"); |
74afab7a | 818 | print_APIC_field(APIC_TMR); |
849d3569 | 819 | pr_debug("... APIC IRR field:\n"); |
74afab7a JL |
820 | print_APIC_field(APIC_IRR); |
821 | ||
822 | /* !82489DX */ | |
823 | if (APIC_INTEGRATED(ver)) { | |
824 | /* Due to the Pentium erratum 3AP. */ | |
825 | if (maxlvt > 3) | |
826 | apic_write(APIC_ESR, 0); | |
827 | ||
828 | v = apic_read(APIC_ESR); | |
849d3569 | 829 | pr_debug("... APIC ESR: %08x\n", v); |
74afab7a JL |
830 | } |
831 | ||
832 | icr = apic_icr_read(); | |
849d3569 JL |
833 | pr_debug("... APIC ICR: %08x\n", (u32)icr); |
834 | pr_debug("... APIC ICR2: %08x\n", (u32)(icr >> 32)); | |
74afab7a JL |
835 | |
836 | v = apic_read(APIC_LVTT); | |
849d3569 | 837 | pr_debug("... APIC LVTT: %08x\n", v); |
74afab7a JL |
838 | |
839 | if (maxlvt > 3) { | |
840 | /* PC is LVT#4. */ | |
841 | v = apic_read(APIC_LVTPC); | |
849d3569 | 842 | pr_debug("... APIC LVTPC: %08x\n", v); |
74afab7a JL |
843 | } |
844 | v = apic_read(APIC_LVT0); | |
849d3569 | 845 | pr_debug("... APIC LVT0: %08x\n", v); |
74afab7a | 846 | v = apic_read(APIC_LVT1); |
849d3569 | 847 | pr_debug("... APIC LVT1: %08x\n", v); |
74afab7a JL |
848 | |
849 | if (maxlvt > 2) { | |
850 | /* ERR is LVT#3. */ | |
851 | v = apic_read(APIC_LVTERR); | |
849d3569 | 852 | pr_debug("... APIC LVTERR: %08x\n", v); |
74afab7a JL |
853 | } |
854 | ||
855 | v = apic_read(APIC_TMICT); | |
849d3569 | 856 | pr_debug("... APIC TMICT: %08x\n", v); |
74afab7a | 857 | v = apic_read(APIC_TMCCT); |
849d3569 | 858 | pr_debug("... APIC TMCCT: %08x\n", v); |
74afab7a | 859 | v = apic_read(APIC_TDCR); |
849d3569 | 860 | pr_debug("... APIC TDCR: %08x\n", v); |
74afab7a JL |
861 | |
862 | if (boot_cpu_has(X86_FEATURE_EXTAPIC)) { | |
863 | v = apic_read(APIC_EFEAT); | |
864 | maxlvt = (v >> 16) & 0xff; | |
849d3569 | 865 | pr_debug("... APIC EFEAT: %08x\n", v); |
74afab7a | 866 | v = apic_read(APIC_ECTRL); |
849d3569 | 867 | pr_debug("... APIC ECTRL: %08x\n", v); |
74afab7a JL |
868 | for (i = 0; i < maxlvt; i++) { |
869 | v = apic_read(APIC_EILVTn(i)); | |
849d3569 | 870 | pr_debug("... APIC EILVT%d: %08x\n", i, v); |
74afab7a JL |
871 | } |
872 | } | |
873 | pr_cont("\n"); | |
874 | } | |
875 | ||
876 | static void __init print_local_APICs(int maxcpu) | |
877 | { | |
878 | int cpu; | |
879 | ||
880 | if (!maxcpu) | |
881 | return; | |
882 | ||
883 | preempt_disable(); | |
884 | for_each_online_cpu(cpu) { | |
885 | if (cpu >= maxcpu) | |
886 | break; | |
887 | smp_call_function_single(cpu, print_local_APIC, NULL, 1); | |
888 | } | |
889 | preempt_enable(); | |
890 | } | |
891 | ||
892 | static void __init print_PIC(void) | |
893 | { | |
894 | unsigned int v; | |
895 | unsigned long flags; | |
896 | ||
897 | if (!nr_legacy_irqs()) | |
898 | return; | |
899 | ||
849d3569 | 900 | pr_debug("\nprinting PIC contents\n"); |
74afab7a JL |
901 | |
902 | raw_spin_lock_irqsave(&i8259A_lock, flags); | |
903 | ||
904 | v = inb(0xa1) << 8 | inb(0x21); | |
849d3569 | 905 | pr_debug("... PIC IMR: %04x\n", v); |
74afab7a JL |
906 | |
907 | v = inb(0xa0) << 8 | inb(0x20); | |
849d3569 | 908 | pr_debug("... PIC IRR: %04x\n", v); |
74afab7a JL |
909 | |
910 | outb(0x0b, 0xa0); | |
911 | outb(0x0b, 0x20); | |
912 | v = inb(0xa0) << 8 | inb(0x20); | |
913 | outb(0x0a, 0xa0); | |
914 | outb(0x0a, 0x20); | |
915 | ||
916 | raw_spin_unlock_irqrestore(&i8259A_lock, flags); | |
917 | ||
849d3569 | 918 | pr_debug("... PIC ISR: %04x\n", v); |
74afab7a JL |
919 | |
920 | v = inb(0x4d1) << 8 | inb(0x4d0); | |
849d3569 | 921 | pr_debug("... PIC ELCR: %04x\n", v); |
74afab7a JL |
922 | } |
923 | ||
924 | static int show_lapic __initdata = 1; | |
925 | static __init int setup_show_lapic(char *arg) | |
926 | { | |
927 | int num = -1; | |
928 | ||
929 | if (strcmp(arg, "all") == 0) { | |
930 | show_lapic = CONFIG_NR_CPUS; | |
931 | } else { | |
932 | get_option(&arg, &num); | |
933 | if (num >= 0) | |
934 | show_lapic = num; | |
935 | } | |
936 | ||
937 | return 1; | |
938 | } | |
939 | __setup("show_lapic=", setup_show_lapic); | |
940 | ||
941 | static int __init print_ICs(void) | |
942 | { | |
943 | if (apic_verbosity == APIC_QUIET) | |
944 | return 0; | |
945 | ||
946 | print_PIC(); | |
947 | ||
948 | /* don't print out if apic is not there */ | |
93984fbd | 949 | if (!boot_cpu_has(X86_FEATURE_APIC) && !apic_from_smp_config()) |
74afab7a JL |
950 | return 0; |
951 | ||
952 | print_local_APICs(show_lapic); | |
953 | print_IO_APICs(); | |
954 | ||
955 | return 0; | |
956 | } | |
957 | ||
958 | late_initcall(print_ICs); |