]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
1da177e4 | 2 | /* |
1da177e4 LT |
3 | * Copyright (C) 1992, 1998 Linus Torvalds, Ingo Molnar |
4 | * | |
5 | * This file contains the lowest level x86_64-specific interrupt | |
6 | * entry and irq statistics code. All the remaining irq logic is | |
7 | * done by the generic kernel/irq/ code and in the | |
8 | * x86_64-specific irq controller code. (e.g. i8259.c and | |
9 | * io_apic.c.) | |
10 | */ | |
11 | ||
12 | #include <linux/kernel_stat.h> | |
13 | #include <linux/interrupt.h> | |
447ae316 | 14 | #include <linux/irq.h> |
1da177e4 | 15 | #include <linux/seq_file.h> |
76e4f660 | 16 | #include <linux/delay.h> |
bcbc4f20 | 17 | #include <linux/ftrace.h> |
5f66b2a0 JSR |
18 | #include <linux/uaccess.h> |
19 | #include <linux/smp.h> | |
68db0cf1 | 20 | #include <linux/sched/task_stack.h> |
bf5882ab TG |
21 | |
22 | #include <asm/cpu_entry_area.h> | |
db1cc7ae | 23 | #include <asm/softirq_stack.h> |
eb6555c8 | 24 | #include <asm/irq_stack.h> |
1da177e4 | 25 | #include <asm/io_apic.h> |
3819cd48 | 26 | #include <asm/apic.h> |
1da177e4 | 27 | |
e6401c13 AL |
28 | DEFINE_PER_CPU_PAGE_ALIGNED(struct irq_stack, irq_stack_backing_store) __visible; |
29 | DECLARE_INIT_PER_CPU(irq_stack_backing_store); | |
30 | ||
18b7a6be AL |
31 | #ifdef CONFIG_VMAP_STACK |
32 | /* | |
33 | * VMAP the backing store with guard pages | |
34 | */ | |
35 | static int map_irq_stack(unsigned int cpu) | |
36 | { | |
37 | char *stack = (char *)per_cpu_ptr(&irq_stack_backing_store, cpu); | |
38 | struct page *pages[IRQ_STACK_SIZE / PAGE_SIZE]; | |
39 | void *va; | |
40 | int i; | |
41 | ||
42 | for (i = 0; i < IRQ_STACK_SIZE / PAGE_SIZE; i++) { | |
43 | phys_addr_t pa = per_cpu_ptr_to_phys(stack + (i << PAGE_SHIFT)); | |
44 | ||
45 | pages[i] = pfn_to_page(pa >> PAGE_SHIFT); | |
46 | } | |
47 | ||
03488011 | 48 | va = vmap(pages, IRQ_STACK_SIZE / PAGE_SIZE, VM_MAP, PAGE_KERNEL); |
18b7a6be AL |
49 | if (!va) |
50 | return -ENOMEM; | |
51 | ||
951c2a51 TG |
52 | /* Store actual TOS to avoid adjustment in the hotpath */ |
53 | per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8; | |
18b7a6be AL |
54 | return 0; |
55 | } | |
56 | #else | |
57 | /* | |
58 | * If VMAP stacks are disabled due to KASAN, just use the per cpu | |
59 | * backing store without guard pages. | |
60 | */ | |
0ac26104 TG |
61 | static int map_irq_stack(unsigned int cpu) |
62 | { | |
e6401c13 | 63 | void *va = per_cpu_ptr(&irq_stack_backing_store, cpu); |
0ac26104 | 64 | |
951c2a51 TG |
65 | /* Store actual TOS to avoid adjustment in the hotpath */ |
66 | per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8; | |
0ac26104 TG |
67 | return 0; |
68 | } | |
18b7a6be | 69 | #endif |
0ac26104 TG |
70 | |
71 | int irq_init_percpu_irqstack(unsigned int cpu) | |
72 | { | |
73 | if (per_cpu(hardirq_stack_ptr, cpu)) | |
74 | return 0; | |
75 | return map_irq_stack(cpu); | |
76 | } |