]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
1da177e4 LT |
2 | #include <linux/mm.h> |
3 | #include <linux/sched.h> | |
b17b0153 | 4 | #include <linux/sched/debug.h> |
1da177e4 LT |
5 | #include <linux/init_task.h> |
6 | #include <linux/fs.h> | |
7 | ||
7c0f6ba6 | 8 | #include <linux/uaccess.h> |
1da177e4 LT |
9 | #include <asm/processor.h> |
10 | #include <asm/desc.h> | |
7d8d8cfd | 11 | #include <asm/traps.h> |
1da177e4 | 12 | |
3dab307e | 13 | #define ptr_ok(x) ((x) > PAGE_OFFSET && (x) < PAGE_OFFSET + MAXMEM) |
1da177e4 | 14 | |
7d8d8cfd | 15 | #define TSS(x) this_cpu_read(cpu_tss_rw.x86_tss.x) |
1da177e4 | 16 | |
7d8d8cfd | 17 | static void set_df_gdt_entry(unsigned int cpu); |
dc4e0021 | 18 | |
7d8d8cfd AL |
19 | /* |
20 | * Called by double_fault with CR0.TS and EFLAGS.NT cleared. The CPU thinks | |
21 | * we're running the doublefault task. Cannot return. | |
22 | */ | |
c29c775a | 23 | asmlinkage noinstr void __noreturn doublefault_shim(void) |
7d8d8cfd AL |
24 | { |
25 | unsigned long cr2; | |
26 | struct pt_regs regs; | |
1da177e4 | 27 | |
7d8d8cfd | 28 | BUILD_BUG_ON(sizeof(struct doublefault_stack) != PAGE_SIZE); |
1da177e4 | 29 | |
7d8d8cfd | 30 | cr2 = native_read_cr2(); |
1da177e4 | 31 | |
7d8d8cfd AL |
32 | /* Reset back to the normal kernel task. */ |
33 | force_reload_TR(); | |
34 | set_df_gdt_entry(smp_processor_id()); | |
1da177e4 | 35 | |
7d8d8cfd | 36 | trace_hardirqs_off(); |
1da177e4 | 37 | |
7d8d8cfd AL |
38 | /* |
39 | * Fill in pt_regs. A downside of doing this in C is that the unwinder | |
40 | * won't see it (no ENCODE_FRAME_POINTER), so a nested stack dump | |
41 | * won't successfully unwind to the source of the double fault. | |
c29c775a | 42 | * The main dump from exc_double_fault() is fine, though, since it |
7d8d8cfd AL |
43 | * uses these regs directly. |
44 | * | |
45 | * If anyone ever cares, this could be moved to asm. | |
46 | */ | |
47 | regs.ss = TSS(ss); | |
48 | regs.__ssh = 0; | |
49 | regs.sp = TSS(sp); | |
50 | regs.flags = TSS(flags); | |
51 | regs.cs = TSS(cs); | |
52 | /* We won't go through the entry asm, so we can leave __csh as 0. */ | |
53 | regs.__csh = 0; | |
54 | regs.ip = TSS(ip); | |
55 | regs.orig_ax = 0; | |
56 | regs.gs = TSS(gs); | |
57 | regs.__gsh = 0; | |
58 | regs.fs = TSS(fs); | |
59 | regs.__fsh = 0; | |
60 | regs.es = TSS(es); | |
61 | regs.__esh = 0; | |
62 | regs.ds = TSS(ds); | |
63 | regs.__dsh = 0; | |
64 | regs.ax = TSS(ax); | |
65 | regs.bp = TSS(bp); | |
66 | regs.di = TSS(di); | |
67 | regs.si = TSS(si); | |
68 | regs.dx = TSS(dx); | |
69 | regs.cx = TSS(cx); | |
70 | regs.bx = TSS(bx); | |
71 | ||
c29c775a | 72 | exc_double_fault(®s, 0, cr2); |
1da177e4 | 73 | |
7d8d8cfd AL |
74 | /* |
75 | * x86_32 does not save the original CR3 anywhere on a task switch. | |
76 | * This means that, even if we wanted to return, we would need to find | |
77 | * some way to reconstruct CR3. We could make a credible guess based | |
78 | * on cpu_tlbstate, but that would be racy and would not account for | |
79 | * PTI. | |
80 | * | |
81 | * Instead, don't bother. We can return through | |
82 | * rewind_stack_do_exit() instead. | |
83 | */ | |
84 | panic("cannot return from double fault\n"); | |
1da177e4 LT |
85 | } |
86 | ||
dc4e0021 AL |
87 | DEFINE_PER_CPU_PAGE_ALIGNED(struct doublefault_stack, doublefault_stack) = { |
88 | .tss = { | |
89 | /* | |
90 | * No sp0 or ss0 -- we never run CPL != 0 with this TSS | |
91 | * active. sp is filled in later. | |
92 | */ | |
93 | .ldt = 0, | |
ecc7e37d | 94 | .io_bitmap_base = IO_BITMAP_OFFSET_INVALID, |
7fb983b4 | 95 | |
c29c775a | 96 | .ip = (unsigned long) asm_exc_double_fault, |
7d8d8cfd | 97 | .flags = X86_EFLAGS_FIXED, |
dc4e0021 AL |
98 | .es = __USER_DS, |
99 | .cs = __KERNEL_CS, | |
100 | .ss = __KERNEL_DS, | |
101 | .ds = __USER_DS, | |
102 | .fs = __KERNEL_PERCPU, | |
3fb0fdb3 | 103 | .gs = 0, |
7fb983b4 | 104 | |
dc4e0021 AL |
105 | .__cr3 = __pa_nodebug(swapper_pg_dir), |
106 | }, | |
1da177e4 | 107 | }; |
dc4e0021 | 108 | |
7d8d8cfd AL |
109 | static void set_df_gdt_entry(unsigned int cpu) |
110 | { | |
111 | /* Set up doublefault TSS pointer in the GDT */ | |
112 | __set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, | |
113 | &get_cpu_entry_area(cpu)->doublefault_stack.tss); | |
114 | ||
115 | } | |
116 | ||
dc4e0021 AL |
117 | void doublefault_init_cpu_tss(void) |
118 | { | |
119 | unsigned int cpu = smp_processor_id(); | |
120 | struct cpu_entry_area *cea = get_cpu_entry_area(cpu); | |
121 | ||
122 | /* | |
123 | * The linker isn't smart enough to initialize percpu variables that | |
124 | * point to other places in percpu space. | |
125 | */ | |
126 | this_cpu_write(doublefault_stack.tss.sp, | |
127 | (unsigned long)&cea->doublefault_stack.stack + | |
128 | sizeof(doublefault_stack.stack)); | |
129 | ||
7d8d8cfd | 130 | set_df_gdt_entry(cpu); |
dc4e0021 | 131 | } |