]>
Commit | Line | Data |
---|---|---|
d3561b7f RR |
1 | /* Paravirtualization interfaces |
2 | Copyright (C) 2006 Rusty Russell IBM Corporation | |
3 | ||
4 | This program is free software; you can redistribute it and/or modify | |
5 | it under the terms of the GNU General Public License as published by | |
6 | the Free Software Foundation; either version 2 of the License, or | |
7 | (at your option) any later version. | |
8 | ||
9 | This program is distributed in the hope that it will be useful, | |
10 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | GNU General Public License for more details. | |
13 | ||
14 | You should have received a copy of the GNU General Public License | |
15 | along with this program; if not, write to the Free Software | |
16 | Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | |
b1df07bd GOC |
17 | |
18 | 2007 - x86_64 support added by Glauber de Oliveira Costa, Red Hat Inc | |
d3561b7f | 19 | */ |
b1df07bd | 20 | |
d3561b7f RR |
21 | #include <linux/errno.h> |
22 | #include <linux/module.h> | |
23 | #include <linux/efi.h> | |
24 | #include <linux/bcd.h> | |
ce6234b5 | 25 | #include <linux/highmem.h> |
d3561b7f RR |
26 | |
27 | #include <asm/bug.h> | |
28 | #include <asm/paravirt.h> | |
29 | #include <asm/desc.h> | |
30 | #include <asm/setup.h> | |
31 | #include <asm/arch_hooks.h> | |
32 | #include <asm/time.h> | |
33 | #include <asm/irq.h> | |
34 | #include <asm/delay.h> | |
13623d79 RR |
35 | #include <asm/fixmap.h> |
36 | #include <asm/apic.h> | |
da181a8b | 37 | #include <asm/tlbflush.h> |
6cb9a835 | 38 | #include <asm/timer.h> |
d3561b7f RR |
39 | |
40 | /* nop stub */ | |
45876233 | 41 | void _paravirt_nop(void) |
d3561b7f RR |
42 | { |
43 | } | |
44 | ||
45 | static void __init default_banner(void) | |
46 | { | |
47 | printk(KERN_INFO "Booting paravirtualized kernel on %s\n", | |
93b1eab3 | 48 | pv_info.name); |
d3561b7f RR |
49 | } |
50 | ||
51 | char *memory_setup(void) | |
52 | { | |
93b1eab3 | 53 | return pv_init_ops.memory_setup(); |
d3561b7f RR |
54 | } |
55 | ||
139ec7c4 | 56 | /* Simple instruction patching code. */ |
93b1eab3 JF |
57 | #define DEF_NATIVE(ops, name, code) \ |
58 | extern const char start_##ops##_##name[], end_##ops##_##name[]; \ | |
59 | asm("start_" #ops "_" #name ": " code "; end_" #ops "_" #name ":") | |
60 | ||
61 | DEF_NATIVE(pv_irq_ops, irq_disable, "cli"); | |
62 | DEF_NATIVE(pv_irq_ops, irq_enable, "sti"); | |
63 | DEF_NATIVE(pv_irq_ops, restore_fl, "push %eax; popf"); | |
64 | DEF_NATIVE(pv_irq_ops, save_fl, "pushf; pop %eax"); | |
65 | DEF_NATIVE(pv_cpu_ops, iret, "iret"); | |
6abcd98f | 66 | DEF_NATIVE(pv_cpu_ops, irq_enable_syscall_ret, "sti; sysexit"); |
93b1eab3 JF |
67 | DEF_NATIVE(pv_mmu_ops, read_cr2, "mov %cr2, %eax"); |
68 | DEF_NATIVE(pv_mmu_ops, write_cr3, "mov %eax, %cr3"); | |
69 | DEF_NATIVE(pv_mmu_ops, read_cr3, "mov %cr3, %eax"); | |
70 | DEF_NATIVE(pv_cpu_ops, clts, "clts"); | |
71 | DEF_NATIVE(pv_cpu_ops, read_tsc, "rdtsc"); | |
72 | ||
73 | /* Undefined instruction for dealing with missing ops pointers. */ | |
74 | static const unsigned char ud2a[] = { 0x0f, 0x0b }; | |
139ec7c4 | 75 | |
ab144f5e AK |
76 | static unsigned native_patch(u8 type, u16 clobbers, void *ibuf, |
77 | unsigned long addr, unsigned len) | |
139ec7c4 | 78 | { |
63f70270 JF |
79 | const unsigned char *start, *end; |
80 | unsigned ret; | |
81 | ||
82 | switch(type) { | |
93b1eab3 JF |
83 | #define SITE(ops, x) \ |
84 | case PARAVIRT_PATCH(ops.x): \ | |
85 | start = start_##ops##_##x; \ | |
86 | end = end_##ops##_##x; \ | |
87 | goto patch_site | |
88 | ||
89 | SITE(pv_irq_ops, irq_disable); | |
90 | SITE(pv_irq_ops, irq_enable); | |
91 | SITE(pv_irq_ops, restore_fl); | |
92 | SITE(pv_irq_ops, save_fl); | |
93 | SITE(pv_cpu_ops, iret); | |
6abcd98f | 94 | SITE(pv_cpu_ops, irq_enable_syscall_ret); |
93b1eab3 JF |
95 | SITE(pv_mmu_ops, read_cr2); |
96 | SITE(pv_mmu_ops, read_cr3); | |
97 | SITE(pv_mmu_ops, write_cr3); | |
98 | SITE(pv_cpu_ops, clts); | |
99 | SITE(pv_cpu_ops, read_tsc); | |
63f70270 JF |
100 | #undef SITE |
101 | ||
102 | patch_site: | |
ab144f5e | 103 | ret = paravirt_patch_insns(ibuf, len, start, end); |
63f70270 JF |
104 | break; |
105 | ||
63f70270 | 106 | default: |
ab144f5e | 107 | ret = paravirt_patch_default(type, clobbers, ibuf, addr, len); |
63f70270 JF |
108 | break; |
109 | } | |
110 | ||
111 | return ret; | |
112 | } | |
113 | ||
114 | unsigned paravirt_patch_nop(void) | |
115 | { | |
116 | return 0; | |
117 | } | |
118 | ||
119 | unsigned paravirt_patch_ignore(unsigned len) | |
120 | { | |
121 | return len; | |
122 | } | |
123 | ||
19d36ccd AK |
124 | struct branch { |
125 | unsigned char opcode; | |
126 | u32 delta; | |
127 | } __attribute__((packed)); | |
128 | ||
ab144f5e AK |
129 | unsigned paravirt_patch_call(void *insnbuf, |
130 | const void *target, u16 tgt_clobbers, | |
131 | unsigned long addr, u16 site_clobbers, | |
63f70270 JF |
132 | unsigned len) |
133 | { | |
ab144f5e AK |
134 | struct branch *b = insnbuf; |
135 | unsigned long delta = (unsigned long)target - (addr+5); | |
63f70270 JF |
136 | |
137 | if (tgt_clobbers & ~site_clobbers) | |
138 | return len; /* target would clobber too much for this site */ | |
139 | if (len < 5) | |
140 | return len; /* call too long for patch site */ | |
139ec7c4 | 141 | |
ab144f5e AK |
142 | b->opcode = 0xe8; /* call */ |
143 | b->delta = delta; | |
144 | BUILD_BUG_ON(sizeof(*b) != 5); | |
139ec7c4 | 145 | |
63f70270 JF |
146 | return 5; |
147 | } | |
148 | ||
93b1eab3 | 149 | unsigned paravirt_patch_jmp(void *insnbuf, const void *target, |
ab144f5e | 150 | unsigned long addr, unsigned len) |
63f70270 | 151 | { |
ab144f5e AK |
152 | struct branch *b = insnbuf; |
153 | unsigned long delta = (unsigned long)target - (addr+5); | |
63f70270 JF |
154 | |
155 | if (len < 5) | |
156 | return len; /* call too long for patch site */ | |
157 | ||
ab144f5e AK |
158 | b->opcode = 0xe9; /* jmp */ |
159 | b->delta = delta; | |
63f70270 JF |
160 | |
161 | return 5; | |
162 | } | |
163 | ||
93b1eab3 JF |
164 | /* Neat trick to map patch type back to the call within the |
165 | * corresponding structure. */ | |
166 | static void *get_call_destination(u8 type) | |
167 | { | |
168 | struct paravirt_patch_template tmpl = { | |
169 | .pv_init_ops = pv_init_ops, | |
93b1eab3 JF |
170 | .pv_time_ops = pv_time_ops, |
171 | .pv_cpu_ops = pv_cpu_ops, | |
172 | .pv_irq_ops = pv_irq_ops, | |
173 | .pv_apic_ops = pv_apic_ops, | |
174 | .pv_mmu_ops = pv_mmu_ops, | |
175 | }; | |
176 | return *((void **)&tmpl + type); | |
177 | } | |
178 | ||
ab144f5e AK |
179 | unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf, |
180 | unsigned long addr, unsigned len) | |
63f70270 | 181 | { |
93b1eab3 | 182 | void *opfunc = get_call_destination(type); |
63f70270 JF |
183 | unsigned ret; |
184 | ||
185 | if (opfunc == NULL) | |
186 | /* If there's no function, patch it with a ud2a (BUG) */ | |
93b1eab3 | 187 | ret = paravirt_patch_insns(insnbuf, len, ud2a, ud2a+sizeof(ud2a)); |
63f70270 JF |
188 | else if (opfunc == paravirt_nop) |
189 | /* If the operation is a nop, then nop the callsite */ | |
190 | ret = paravirt_patch_nop(); | |
93b1eab3 | 191 | else if (type == PARAVIRT_PATCH(pv_cpu_ops.iret) || |
6abcd98f | 192 | type == PARAVIRT_PATCH(pv_cpu_ops.irq_enable_syscall_ret)) |
63f70270 | 193 | /* If operation requires a jmp, then jmp */ |
93b1eab3 | 194 | ret = paravirt_patch_jmp(insnbuf, opfunc, addr, len); |
63f70270 JF |
195 | else |
196 | /* Otherwise call the function; assume target could | |
197 | clobber any caller-save reg */ | |
ab144f5e AK |
198 | ret = paravirt_patch_call(insnbuf, opfunc, CLBR_ANY, |
199 | addr, clobbers, len); | |
63f70270 JF |
200 | |
201 | return ret; | |
202 | } | |
203 | ||
ab144f5e | 204 | unsigned paravirt_patch_insns(void *insnbuf, unsigned len, |
63f70270 JF |
205 | const char *start, const char *end) |
206 | { | |
207 | unsigned insn_len = end - start; | |
139ec7c4 | 208 | |
63f70270 JF |
209 | if (insn_len > len || start == NULL) |
210 | insn_len = len; | |
211 | else | |
ab144f5e | 212 | memcpy(insnbuf, start, insn_len); |
139ec7c4 | 213 | |
139ec7c4 RR |
214 | return insn_len; |
215 | } | |
216 | ||
d3561b7f RR |
217 | void init_IRQ(void) |
218 | { | |
93b1eab3 | 219 | pv_irq_ops.init_IRQ(); |
d3561b7f RR |
220 | } |
221 | ||
1a1eecd1 | 222 | static void native_flush_tlb(void) |
da181a8b RR |
223 | { |
224 | __native_flush_tlb(); | |
225 | } | |
226 | ||
227 | /* | |
228 | * Global pages have to be flushed a bit differently. Not a real | |
229 | * performance problem because this does not happen often. | |
230 | */ | |
1a1eecd1 | 231 | static void native_flush_tlb_global(void) |
da181a8b RR |
232 | { |
233 | __native_flush_tlb_global(); | |
234 | } | |
235 | ||
63f70270 | 236 | static void native_flush_tlb_single(unsigned long addr) |
da181a8b RR |
237 | { |
238 | __native_flush_tlb_single(addr); | |
239 | } | |
240 | ||
d3561b7f | 241 | /* These are in entry.S */ |
1a1eecd1 | 242 | extern void native_iret(void); |
6abcd98f | 243 | extern void native_irq_enable_syscall_ret(void); |
d3561b7f RR |
244 | |
245 | static int __init print_banner(void) | |
246 | { | |
93b1eab3 | 247 | pv_init_ops.banner(); |
d3561b7f RR |
248 | return 0; |
249 | } | |
250 | core_initcall(print_banner); | |
251 | ||
d572929c JF |
252 | static struct resource reserve_ioports = { |
253 | .start = 0, | |
254 | .end = IO_SPACE_LIMIT, | |
255 | .name = "paravirt-ioport", | |
256 | .flags = IORESOURCE_IO | IORESOURCE_BUSY, | |
257 | }; | |
258 | ||
259 | static struct resource reserve_iomem = { | |
260 | .start = 0, | |
261 | .end = -1, | |
262 | .name = "paravirt-iomem", | |
263 | .flags = IORESOURCE_MEM | IORESOURCE_BUSY, | |
264 | }; | |
265 | ||
266 | /* | |
267 | * Reserve the whole legacy IO space to prevent any legacy drivers | |
268 | * from wasting time probing for their hardware. This is a fairly | |
269 | * brute-force approach to disabling all non-virtual drivers. | |
270 | * | |
271 | * Note that this must be called very early to have any effect. | |
272 | */ | |
273 | int paravirt_disable_iospace(void) | |
274 | { | |
275 | int ret; | |
276 | ||
277 | ret = request_resource(&ioport_resource, &reserve_ioports); | |
278 | if (ret == 0) { | |
279 | ret = request_resource(&iomem_resource, &reserve_iomem); | |
280 | if (ret) | |
281 | release_resource(&reserve_ioports); | |
282 | } | |
283 | ||
284 | return ret; | |
285 | } | |
286 | ||
8965c1c0 JF |
287 | static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LAZY_NONE; |
288 | ||
289 | static inline void enter_lazy(enum paravirt_lazy_mode mode) | |
290 | { | |
291 | BUG_ON(x86_read_percpu(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE); | |
292 | BUG_ON(preemptible()); | |
293 | ||
294 | x86_write_percpu(paravirt_lazy_mode, mode); | |
295 | } | |
296 | ||
297 | void paravirt_leave_lazy(enum paravirt_lazy_mode mode) | |
298 | { | |
299 | BUG_ON(x86_read_percpu(paravirt_lazy_mode) != mode); | |
300 | BUG_ON(preemptible()); | |
301 | ||
302 | x86_write_percpu(paravirt_lazy_mode, PARAVIRT_LAZY_NONE); | |
303 | } | |
304 | ||
305 | void paravirt_enter_lazy_mmu(void) | |
306 | { | |
307 | enter_lazy(PARAVIRT_LAZY_MMU); | |
308 | } | |
309 | ||
310 | void paravirt_leave_lazy_mmu(void) | |
311 | { | |
312 | paravirt_leave_lazy(PARAVIRT_LAZY_MMU); | |
313 | } | |
314 | ||
315 | void paravirt_enter_lazy_cpu(void) | |
316 | { | |
317 | enter_lazy(PARAVIRT_LAZY_CPU); | |
318 | } | |
319 | ||
320 | void paravirt_leave_lazy_cpu(void) | |
321 | { | |
322 | paravirt_leave_lazy(PARAVIRT_LAZY_CPU); | |
323 | } | |
324 | ||
325 | enum paravirt_lazy_mode paravirt_get_lazy_mode(void) | |
326 | { | |
327 | return x86_read_percpu(paravirt_lazy_mode); | |
328 | } | |
329 | ||
93b1eab3 | 330 | struct pv_info pv_info = { |
d3561b7f RR |
331 | .name = "bare hardware", |
332 | .paravirt_enabled = 0, | |
333 | .kernel_rpl = 0, | |
5311ab62 | 334 | .shared_kernel_pmd = 1, /* Only used when CONFIG_X86_PAE is set */ |
93b1eab3 | 335 | }; |
d3561b7f | 336 | |
93b1eab3 JF |
337 | struct pv_init_ops pv_init_ops = { |
338 | .patch = native_patch, | |
d3561b7f | 339 | .banner = default_banner, |
45876233 | 340 | .arch_setup = paravirt_nop, |
d3561b7f | 341 | .memory_setup = machine_specific_memory_setup, |
93b1eab3 JF |
342 | }; |
343 | ||
344 | struct pv_time_ops pv_time_ops = { | |
345 | .time_init = hpet_time_init, | |
d3561b7f RR |
346 | .get_wallclock = native_get_wallclock, |
347 | .set_wallclock = native_set_wallclock, | |
93b1eab3 JF |
348 | .sched_clock = native_sched_clock, |
349 | .get_cpu_khz = native_calculate_cpu_khz, | |
350 | }; | |
351 | ||
352 | struct pv_irq_ops pv_irq_ops = { | |
d3561b7f | 353 | .init_IRQ = native_init_IRQ, |
93b1eab3 JF |
354 | .save_fl = native_save_fl, |
355 | .restore_fl = native_restore_fl, | |
356 | .irq_disable = native_irq_disable, | |
357 | .irq_enable = native_irq_enable, | |
358 | .safe_halt = native_safe_halt, | |
359 | .halt = native_halt, | |
360 | }; | |
d3561b7f | 361 | |
93b1eab3 | 362 | struct pv_cpu_ops pv_cpu_ops = { |
d3561b7f RR |
363 | .cpuid = native_cpuid, |
364 | .get_debugreg = native_get_debugreg, | |
365 | .set_debugreg = native_set_debugreg, | |
366 | .clts = native_clts, | |
367 | .read_cr0 = native_read_cr0, | |
368 | .write_cr0 = native_write_cr0, | |
d3561b7f RR |
369 | .read_cr4 = native_read_cr4, |
370 | .read_cr4_safe = native_read_cr4_safe, | |
371 | .write_cr4 = native_write_cr4, | |
d3561b7f | 372 | .wbinvd = native_wbinvd, |
90a0a06a RR |
373 | .read_msr = native_read_msr_safe, |
374 | .write_msr = native_write_msr_safe, | |
d3561b7f RR |
375 | .read_tsc = native_read_tsc, |
376 | .read_pmc = native_read_pmc, | |
377 | .load_tr_desc = native_load_tr_desc, | |
378 | .set_ldt = native_set_ldt, | |
379 | .load_gdt = native_load_gdt, | |
380 | .load_idt = native_load_idt, | |
381 | .store_gdt = native_store_gdt, | |
382 | .store_idt = native_store_idt, | |
383 | .store_tr = native_store_tr, | |
384 | .load_tls = native_load_tls, | |
75b8bb3e | 385 | .write_ldt_entry = native_write_ldt_entry, |
014b15be | 386 | .write_gdt_entry = native_write_gdt_entry, |
8d947344 | 387 | .write_idt_entry = native_write_idt_entry, |
faca6227 | 388 | .load_sp0 = native_load_sp0, |
d3561b7f | 389 | |
6abcd98f | 390 | .irq_enable_syscall_ret = native_irq_enable_syscall_ret, |
93b1eab3 JF |
391 | .iret = native_iret, |
392 | ||
d3561b7f RR |
393 | .set_iopl_mask = native_set_iopl_mask, |
394 | .io_delay = native_io_delay, | |
8965c1c0 JF |
395 | |
396 | .lazy_mode = { | |
397 | .enter = paravirt_nop, | |
398 | .leave = paravirt_nop, | |
399 | }, | |
93b1eab3 | 400 | }; |
d3561b7f | 401 | |
93b1eab3 | 402 | struct pv_apic_ops pv_apic_ops = { |
13623d79 RR |
403 | #ifdef CONFIG_X86_LOCAL_APIC |
404 | .apic_write = native_apic_write, | |
405 | .apic_write_atomic = native_apic_write_atomic, | |
406 | .apic_read = native_apic_read, | |
bbab4f3b ZA |
407 | .setup_boot_clock = setup_boot_APIC_clock, |
408 | .setup_secondary_clock = setup_secondary_APIC_clock, | |
0260c196 | 409 | .startup_ipi_hook = paravirt_nop, |
13623d79 | 410 | #endif |
93b1eab3 JF |
411 | }; |
412 | ||
93b1eab3 | 413 | struct pv_mmu_ops pv_mmu_ops = { |
b239fb25 JF |
414 | .pagetable_setup_start = native_pagetable_setup_start, |
415 | .pagetable_setup_done = native_pagetable_setup_done, | |
416 | ||
93b1eab3 JF |
417 | .read_cr2 = native_read_cr2, |
418 | .write_cr2 = native_write_cr2, | |
419 | .read_cr3 = native_read_cr3, | |
420 | .write_cr3 = native_write_cr3, | |
421 | ||
da181a8b RR |
422 | .flush_tlb_user = native_flush_tlb, |
423 | .flush_tlb_kernel = native_flush_tlb_global, | |
424 | .flush_tlb_single = native_flush_tlb_single, | |
d4c10477 | 425 | .flush_tlb_others = native_flush_tlb_others, |
da181a8b | 426 | |
45876233 JF |
427 | .alloc_pt = paravirt_nop, |
428 | .alloc_pd = paravirt_nop, | |
429 | .alloc_pd_clone = paravirt_nop, | |
430 | .release_pt = paravirt_nop, | |
431 | .release_pd = paravirt_nop, | |
c119ecce | 432 | |
da181a8b RR |
433 | .set_pte = native_set_pte, |
434 | .set_pte_at = native_set_pte_at, | |
435 | .set_pmd = native_set_pmd, | |
45876233 JF |
436 | .pte_update = paravirt_nop, |
437 | .pte_update_defer = paravirt_nop, | |
3dc494e8 | 438 | |
ce6234b5 JF |
439 | #ifdef CONFIG_HIGHPTE |
440 | .kmap_atomic_pte = kmap_atomic, | |
441 | #endif | |
442 | ||
da181a8b RR |
443 | #ifdef CONFIG_X86_PAE |
444 | .set_pte_atomic = native_set_pte_atomic, | |
445 | .set_pte_present = native_set_pte_present, | |
446 | .set_pud = native_set_pud, | |
447 | .pte_clear = native_pte_clear, | |
448 | .pmd_clear = native_pmd_clear, | |
3dc494e8 JF |
449 | |
450 | .pmd_val = native_pmd_val, | |
451 | .make_pmd = native_make_pmd, | |
da181a8b RR |
452 | #endif |
453 | ||
3dc494e8 JF |
454 | .pte_val = native_pte_val, |
455 | .pgd_val = native_pgd_val, | |
456 | ||
457 | .make_pte = native_make_pte, | |
458 | .make_pgd = native_make_pgd, | |
459 | ||
d6dd61c8 JF |
460 | .dup_mmap = paravirt_nop, |
461 | .exit_mmap = paravirt_nop, | |
462 | .activate_mm = paravirt_nop, | |
8965c1c0 JF |
463 | |
464 | .lazy_mode = { | |
465 | .enter = paravirt_nop, | |
466 | .leave = paravirt_nop, | |
467 | }, | |
d3561b7f | 468 | }; |
0dbe5a11 | 469 | |
93b1eab3 | 470 | EXPORT_SYMBOL_GPL(pv_time_ops); |
f97b8954 JF |
471 | EXPORT_SYMBOL (pv_cpu_ops); |
472 | EXPORT_SYMBOL (pv_mmu_ops); | |
93b1eab3 JF |
473 | EXPORT_SYMBOL_GPL(pv_apic_ops); |
474 | EXPORT_SYMBOL_GPL(pv_info); | |
475 | EXPORT_SYMBOL (pv_irq_ops); |