]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - virt/kvm/kvm_main.c
projects / mirror_ubuntu-bionic-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
KVM: x86: Makefile clean up
[mirror_ubuntu-bionic-kernel.git] / virt / kvm / kvm_main.c
1 /*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
8 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
9 *
10 * Authors:
11 * Avi Kivity <avi@qumranet.com>
12 * Yaniv Kamay <yaniv@qumranet.com>
13 *
14 * This work is licensed under the terms of the GNU GPL, version 2. See
15 * the COPYING file in the top-level directory.
16 *
17 */
18
19 #include "iodev.h"
20
21 #include <linux/kvm_host.h>
22 #include <linux/kvm.h>
23 #include <linux/module.h>
24 #include <linux/errno.h>
25 #include <linux/percpu.h>
26 #include <linux/mm.h>
27 #include <linux/miscdevice.h>
28 #include <linux/vmalloc.h>
29 #include <linux/reboot.h>
30 #include <linux/debugfs.h>
31 #include <linux/highmem.h>
32 #include <linux/file.h>
33 #include <linux/sysdev.h>
34 #include <linux/cpu.h>
35 #include <linux/sched.h>
36 #include <linux/cpumask.h>
37 #include <linux/smp.h>
38 #include <linux/anon_inodes.h>
39 #include <linux/profile.h>
40 #include <linux/kvm_para.h>
41 #include <linux/pagemap.h>
42 #include <linux/mman.h>
43 #include <linux/swap.h>
44 #include <linux/bitops.h>
45 #include <linux/spinlock.h>
46 #include <linux/compat.h>
47 #include <linux/srcu.h>
48 #include <linux/hugetlb.h>
49 #include <linux/slab.h>
50
51 #include <asm/processor.h>
52 #include <asm/io.h>
53 #include <asm/uaccess.h>
54 #include <asm/pgtable.h>
55 #include <asm-generic/bitops/le.h>
56
57 #include "coalesced_mmio.h"
58 #include "async_pf.h"
59
60 #define CREATE_TRACE_POINTS
61 #include <trace/events/kvm.h>
62
63 MODULE_AUTHOR("Qumranet");
64 MODULE_LICENSE("GPL");
65
66 /*
67 * Ordering of locks:
68 *
69 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock
70 */
71
72 DEFINE_SPINLOCK(kvm_lock);
73 LIST_HEAD(vm_list);
74
75 static cpumask_var_t cpus_hardware_enabled;
76 static int kvm_usage_count = 0;
77 static atomic_t hardware_enable_failed;
78
79 struct kmem_cache *kvm_vcpu_cache;
80 EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
81
82 static __read_mostly struct preempt_ops kvm_preempt_ops;
83
84 struct dentry *kvm_debugfs_dir;
85
86 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
87 unsigned long arg);
88 static int hardware_enable_all(void);
89 static void hardware_disable_all(void);
90
91 static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
92
93 static bool kvm_rebooting;
94
95 static bool largepages_enabled = true;
96
97 static struct page *hwpoison_page;
98 static pfn_t hwpoison_pfn;
99
100 static struct page *fault_page;
101 static pfn_t fault_pfn;
102
103 inline int kvm_is_mmio_pfn(pfn_t pfn)
104 {
105 if (pfn_valid(pfn)) {
106 struct page *page = compound_head(pfn_to_page(pfn));
107 return PageReserved(page);
108 }
109
110 return true;
111 }
112
113 /*
114 * Switches to specified vcpu, until a matching vcpu_put()
115 */
116 void vcpu_load(struct kvm_vcpu *vcpu)
117 {
118 int cpu;
119
120 mutex_lock(&vcpu->mutex);
121 cpu = get_cpu();
122 preempt_notifier_register(&vcpu->preempt_notifier);
123 kvm_arch_vcpu_load(vcpu, cpu);
124 put_cpu();
125 }
126
127 void vcpu_put(struct kvm_vcpu *vcpu)
128 {
129 preempt_disable();
130 kvm_arch_vcpu_put(vcpu);
131 preempt_notifier_unregister(&vcpu->preempt_notifier);
132 preempt_enable();
133 mutex_unlock(&vcpu->mutex);
134 }
135
136 static void ack_flush(void *_completed)
137 {
138 }
139
140 static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
141 {
142 int i, cpu, me;
143 cpumask_var_t cpus;
144 bool called = true;
145 struct kvm_vcpu *vcpu;
146
147 zalloc_cpumask_var(&cpus, GFP_ATOMIC);
148
149 raw_spin_lock(&kvm->requests_lock);
150 me = smp_processor_id();
151 kvm_for_each_vcpu(i, vcpu, kvm) {
152 if (kvm_make_check_request(req, vcpu))
153 continue;
154 cpu = vcpu->cpu;
155 if (cpus != NULL && cpu != -1 && cpu != me)
156 cpumask_set_cpu(cpu, cpus);
157 }
158 if (unlikely(cpus == NULL))
159 smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1);
160 else if (!cpumask_empty(cpus))
161 smp_call_function_many(cpus, ack_flush, NULL, 1);
162 else
163 called = false;
164 raw_spin_unlock(&kvm->requests_lock);
165 free_cpumask_var(cpus);
166 return called;
167 }
168
169 void kvm_flush_remote_tlbs(struct kvm *kvm)
170 {
171 if (make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
172 ++kvm->stat.remote_tlb_flush;
173 }
174
175 void kvm_reload_remote_mmus(struct kvm *kvm)
176 {
177 make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
178 }
179
180 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
181 {
182 struct page *page;
183 int r;
184
185 mutex_init(&vcpu->mutex);
186 vcpu->cpu = -1;
187 vcpu->kvm = kvm;
188 vcpu->vcpu_id = id;
189 init_waitqueue_head(&vcpu->wq);
190 kvm_async_pf_vcpu_init(vcpu);
191
192 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
193 if (!page) {
194 r = -ENOMEM;
195 goto fail;
196 }
197 vcpu->run = page_address(page);
198
199 r = kvm_arch_vcpu_init(vcpu);
200 if (r < 0)
201 goto fail_free_run;
202 return 0;
203
204 fail_free_run:
205 free_page((unsigned long)vcpu->run);
206 fail:
207 return r;
208 }
209 EXPORT_SYMBOL_GPL(kvm_vcpu_init);
210
211 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
212 {
213 kvm_arch_vcpu_uninit(vcpu);
214 free_page((unsigned long)vcpu->run);
215 }
216 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
217
218 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
219 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
220 {
221 return container_of(mn, struct kvm, mmu_notifier);
222 }
223
224 static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
225 struct mm_struct *mm,
226 unsigned long address)
227 {
228 struct kvm *kvm = mmu_notifier_to_kvm(mn);
229 int need_tlb_flush, idx;
230
231 /*
232 * When ->invalidate_page runs, the linux pte has been zapped
233 * already but the page is still allocated until
234 * ->invalidate_page returns. So if we increase the sequence
235 * here the kvm page fault will notice if the spte can't be
236 * established because the page is going to be freed. If
237 * instead the kvm page fault establishes the spte before
238 * ->invalidate_page runs, kvm_unmap_hva will release it
239 * before returning.
240 *
241 * The sequence increase only need to be seen at spin_unlock
242 * time, and not at spin_lock time.
243 *
244 * Increasing the sequence after the spin_unlock would be
245 * unsafe because the kvm page fault could then establish the
246 * pte after kvm_unmap_hva returned, without noticing the page
247 * is going to be freed.
248 */
249 idx = srcu_read_lock(&kvm->srcu);
250 spin_lock(&kvm->mmu_lock);
251 kvm->mmu_notifier_seq++;
252 need_tlb_flush = kvm_unmap_hva(kvm, address);
253 spin_unlock(&kvm->mmu_lock);
254 srcu_read_unlock(&kvm->srcu, idx);
255
256 /* we've to flush the tlb before the pages can be freed */
257 if (need_tlb_flush)
258 kvm_flush_remote_tlbs(kvm);
259
260 }
261
262 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
263 struct mm_struct *mm,
264 unsigned long address,
265 pte_t pte)
266 {
267 struct kvm *kvm = mmu_notifier_to_kvm(mn);
268 int idx;
269
270 idx = srcu_read_lock(&kvm->srcu);
271 spin_lock(&kvm->mmu_lock);
272 kvm->mmu_notifier_seq++;
273 kvm_set_spte_hva(kvm, address, pte);
274 spin_unlock(&kvm->mmu_lock);
275 srcu_read_unlock(&kvm->srcu, idx);
276 }
277
278 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
279 struct mm_struct *mm,
280 unsigned long start,
281 unsigned long end)
282 {
283 struct kvm *kvm = mmu_notifier_to_kvm(mn);
284 int need_tlb_flush = 0, idx;
285
286 idx = srcu_read_lock(&kvm->srcu);
287 spin_lock(&kvm->mmu_lock);
288 /*
289 * The count increase must become visible at unlock time as no
290 * spte can be established without taking the mmu_lock and
291 * count is also read inside the mmu_lock critical section.
292 */
293 kvm->mmu_notifier_count++;
294 for (; start < end; start += PAGE_SIZE)
295 need_tlb_flush |= kvm_unmap_hva(kvm, start);
296 spin_unlock(&kvm->mmu_lock);
297 srcu_read_unlock(&kvm->srcu, idx);
298
299 /* we've to flush the tlb before the pages can be freed */
300 if (need_tlb_flush)
301 kvm_flush_remote_tlbs(kvm);
302 }
303
304 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
305 struct mm_struct *mm,
306 unsigned long start,
307 unsigned long end)
308 {
309 struct kvm *kvm = mmu_notifier_to_kvm(mn);
310
311 spin_lock(&kvm->mmu_lock);
312 /*
313 * This sequence increase will notify the kvm page fault that
314 * the page that is going to be mapped in the spte could have
315 * been freed.
316 */
317 kvm->mmu_notifier_seq++;
318 /*
319 * The above sequence increase must be visible before the
320 * below count decrease but both values are read by the kvm
321 * page fault under mmu_lock spinlock so we don't need to add
322 * a smb_wmb() here in between the two.
323 */
324 kvm->mmu_notifier_count--;
325 spin_unlock(&kvm->mmu_lock);
326
327 BUG_ON(kvm->mmu_notifier_count < 0);
328 }
329
330 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
331 struct mm_struct *mm,
332 unsigned long address)
333 {
334 struct kvm *kvm = mmu_notifier_to_kvm(mn);
335 int young, idx;
336
337 idx = srcu_read_lock(&kvm->srcu);
338 spin_lock(&kvm->mmu_lock);
339 young = kvm_age_hva(kvm, address);
340 spin_unlock(&kvm->mmu_lock);
341 srcu_read_unlock(&kvm->srcu, idx);
342
343 if (young)
344 kvm_flush_remote_tlbs(kvm);
345
346 return young;
347 }
348
349 static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
350 struct mm_struct *mm)
351 {
352 struct kvm *kvm = mmu_notifier_to_kvm(mn);
353 int idx;
354
355 idx = srcu_read_lock(&kvm->srcu);
356 kvm_arch_flush_shadow(kvm);
357 srcu_read_unlock(&kvm->srcu, idx);
358 }
359
360 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
361 .invalidate_page = kvm_mmu_notifier_invalidate_page,
362 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
363 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end,
364 .clear_flush_young = kvm_mmu_notifier_clear_flush_young,
365 .change_pte = kvm_mmu_notifier_change_pte,
366 .release = kvm_mmu_notifier_release,
367 };
368
369 static int kvm_init_mmu_notifier(struct kvm *kvm)
370 {
371 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
372 return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
373 }
374
375 #else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */
376
377 static int kvm_init_mmu_notifier(struct kvm *kvm)
378 {
379 return 0;
380 }
381
382 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
383
384 static struct kvm *kvm_create_vm(void)
385 {
386 int r = 0, i;
387 struct kvm *kvm = kvm_arch_create_vm();
388
389 if (IS_ERR(kvm))
390 goto out;
391
392 r = hardware_enable_all();
393 if (r)
394 goto out_err_nodisable;
395
396 #ifdef CONFIG_HAVE_KVM_IRQCHIP
397 INIT_HLIST_HEAD(&kvm->mask_notifier_list);
398 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
399 #endif
400
401 r = -ENOMEM;
402 kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
403 if (!kvm->memslots)
404 goto out_err_nosrcu;
405 if (init_srcu_struct(&kvm->srcu))
406 goto out_err_nosrcu;
407 for (i = 0; i < KVM_NR_BUSES; i++) {
408 kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
409 GFP_KERNEL);
410 if (!kvm->buses[i])
411 goto out_err;
412 }
413
414 r = kvm_init_mmu_notifier(kvm);
415 if (r)
416 goto out_err;
417
418 kvm->mm = current->mm;
419 atomic_inc(&kvm->mm->mm_count);
420 spin_lock_init(&kvm->mmu_lock);
421 raw_spin_lock_init(&kvm->requests_lock);
422 kvm_eventfd_init(kvm);
423 mutex_init(&kvm->lock);
424 mutex_init(&kvm->irq_lock);
425 mutex_init(&kvm->slots_lock);
426 atomic_set(&kvm->users_count, 1);
427 spin_lock(&kvm_lock);
428 list_add(&kvm->vm_list, &vm_list);
429 spin_unlock(&kvm_lock);
430 out:
431 return kvm;
432
433 out_err:
434 cleanup_srcu_struct(&kvm->srcu);
435 out_err_nosrcu:
436 hardware_disable_all();
437 out_err_nodisable:
438 for (i = 0; i < KVM_NR_BUSES; i++)
439 kfree(kvm->buses[i]);
440 kfree(kvm->memslots);
441 kfree(kvm);
442 return ERR_PTR(r);
443 }
444
445 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
446 {
447 if (!memslot->dirty_bitmap)
448 return;
449
450 if (2 * kvm_dirty_bitmap_bytes(memslot) > PAGE_SIZE)
451 vfree(memslot->dirty_bitmap_head);
452 else
453 kfree(memslot->dirty_bitmap_head);
454
455 memslot->dirty_bitmap = NULL;
456 memslot->dirty_bitmap_head = NULL;
457 }
458
459 /*
460 * Free any memory in @free but not in @dont.
461 */
462 static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
463 struct kvm_memory_slot *dont)
464 {
465 int i;
466
467 if (!dont || free->rmap != dont->rmap)
468 vfree(free->rmap);
469
470 if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
471 kvm_destroy_dirty_bitmap(free);
472
473
474 for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) {
475 if (!dont || free->lpage_info[i] != dont->lpage_info[i]) {
476 vfree(free->lpage_info[i]);
477 free->lpage_info[i] = NULL;
478 }
479 }
480
481 free->npages = 0;
482 free->rmap = NULL;
483 }
484
485 void kvm_free_physmem(struct kvm *kvm)
486 {
487 int i;
488 struct kvm_memslots *slots = kvm->memslots;
489
490 for (i = 0; i < slots->nmemslots; ++i)
491 kvm_free_physmem_slot(&slots->memslots[i], NULL);
492
493 kfree(kvm->memslots);
494 }
495
496 static void kvm_destroy_vm(struct kvm *kvm)
497 {
498 int i;
499 struct mm_struct *mm = kvm->mm;
500
501 kvm_arch_sync_events(kvm);
502 spin_lock(&kvm_lock);
503 list_del(&kvm->vm_list);
504 spin_unlock(&kvm_lock);
505 kvm_free_irq_routing(kvm);
506 for (i = 0; i < KVM_NR_BUSES; i++)
507 kvm_io_bus_destroy(kvm->buses[i]);
508 kvm_coalesced_mmio_free(kvm);
509 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
510 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
511 #else
512 kvm_arch_flush_shadow(kvm);
513 #endif
514 kvm_arch_destroy_vm(kvm);
515 hardware_disable_all();
516 mmdrop(mm);
517 }
518
519 void kvm_get_kvm(struct kvm *kvm)
520 {
521 atomic_inc(&kvm->users_count);
522 }
523 EXPORT_SYMBOL_GPL(kvm_get_kvm);
524
525 void kvm_put_kvm(struct kvm *kvm)
526 {
527 if (atomic_dec_and_test(&kvm->users_count))
528 kvm_destroy_vm(kvm);
529 }
530 EXPORT_SYMBOL_GPL(kvm_put_kvm);
531
532
533 static int kvm_vm_release(struct inode *inode, struct file *filp)
534 {
535 struct kvm *kvm = filp->private_data;
536
537 kvm_irqfd_release(kvm);
538
539 kvm_put_kvm(kvm);
540 return 0;
541 }
542
543 /*
544 * Allocation size is twice as large as the actual dirty bitmap size.
545 * This makes it possible to do double buffering: see x86's
546 * kvm_vm_ioctl_get_dirty_log().
547 */
548 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
549 {
550 unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
551
552 if (dirty_bytes > PAGE_SIZE)
553 memslot->dirty_bitmap = vzalloc(dirty_bytes);
554 else
555 memslot->dirty_bitmap = kzalloc(dirty_bytes, GFP_KERNEL);
556
557 if (!memslot->dirty_bitmap)
558 return -ENOMEM;
559
560 memslot->dirty_bitmap_head = memslot->dirty_bitmap;
561 return 0;
562 }
563
564 /*
565 * Allocate some memory and give it an address in the guest physical address
566 * space.
567 *
568 * Discontiguous memory is allowed, mostly for framebuffers.
569 *
570 * Must be called holding mmap_sem for write.
571 */
572 int __kvm_set_memory_region(struct kvm *kvm,
573 struct kvm_userspace_memory_region *mem,
574 int user_alloc)
575 {
576 int r, flush_shadow = 0;
577 gfn_t base_gfn;
578 unsigned long npages;
579 unsigned long i;
580 struct kvm_memory_slot *memslot;
581 struct kvm_memory_slot old, new;
582 struct kvm_memslots *slots, *old_memslots;
583
584 r = -EINVAL;
585 /* General sanity checks */
586 if (mem->memory_size & (PAGE_SIZE - 1))
587 goto out;
588 if (mem->guest_phys_addr & (PAGE_SIZE - 1))
589 goto out;
590 if (user_alloc && (mem->userspace_addr & (PAGE_SIZE - 1)))
591 goto out;
592 if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS)
593 goto out;
594 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
595 goto out;
596
597 memslot = &kvm->memslots->memslots[mem->slot];
598 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
599 npages = mem->memory_size >> PAGE_SHIFT;
600
601 r = -EINVAL;
602 if (npages > KVM_MEM_MAX_NR_PAGES)
603 goto out;
604
605 if (!npages)
606 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
607
608 new = old = *memslot;
609
610 new.id = mem->slot;
611 new.base_gfn = base_gfn;
612 new.npages = npages;
613 new.flags = mem->flags;
614
615 /* Disallow changing a memory slot's size. */
616 r = -EINVAL;
617 if (npages && old.npages && npages != old.npages)
618 goto out_free;
619
620 /* Check for overlaps */
621 r = -EEXIST;
622 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
623 struct kvm_memory_slot *s = &kvm->memslots->memslots[i];
624
625 if (s == memslot || !s->npages)
626 continue;
627 if (!((base_gfn + npages <= s->base_gfn) ||
628 (base_gfn >= s->base_gfn + s->npages)))
629 goto out_free;
630 }
631
632 /* Free page dirty bitmap if unneeded */
633 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
634 new.dirty_bitmap = NULL;
635
636 r = -ENOMEM;
637
638 /* Allocate if a slot is being created */
639 #ifndef CONFIG_S390
640 if (npages && !new.rmap) {
641 new.rmap = vzalloc(npages * sizeof(*new.rmap));
642
643 if (!new.rmap)
644 goto out_free;
645
646 new.user_alloc = user_alloc;
647 new.userspace_addr = mem->userspace_addr;
648 }
649 if (!npages)
650 goto skip_lpage;
651
652 for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) {
653 unsigned long ugfn;
654 unsigned long j;
655 int lpages;
656 int level = i + 2;
657
658 /* Avoid unused variable warning if no large pages */
659 (void)level;
660
661 if (new.lpage_info[i])
662 continue;
663
664 lpages = 1 + ((base_gfn + npages - 1)
665 >> KVM_HPAGE_GFN_SHIFT(level));
666 lpages -= base_gfn >> KVM_HPAGE_GFN_SHIFT(level);
667
668 new.lpage_info[i] = vzalloc(lpages * sizeof(*new.lpage_info[i]));
669
670 if (!new.lpage_info[i])
671 goto out_free;
672
673 if (base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1))
674 new.lpage_info[i][0].write_count = 1;
675 if ((base_gfn+npages) & (KVM_PAGES_PER_HPAGE(level) - 1))
676 new.lpage_info[i][lpages - 1].write_count = 1;
677 ugfn = new.userspace_addr >> PAGE_SHIFT;
678 /*
679 * If the gfn and userspace address are not aligned wrt each
680 * other, or if explicitly asked to, disable large page
681 * support for this slot
682 */
683 if ((base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1) ||
684 !largepages_enabled)
685 for (j = 0; j < lpages; ++j)
686 new.lpage_info[i][j].write_count = 1;
687 }
688
689 skip_lpage:
690
691 /* Allocate page dirty bitmap if needed */
692 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
693 if (kvm_create_dirty_bitmap(&new) < 0)
694 goto out_free;
695 /* destroy any largepage mappings for dirty tracking */
696 if (old.npages)
697 flush_shadow = 1;
698 }
699 #else /* not defined CONFIG_S390 */
700 new.user_alloc = user_alloc;
701 if (user_alloc)
702 new.userspace_addr = mem->userspace_addr;
703 #endif /* not defined CONFIG_S390 */
704
705 if (!npages) {
706 r = -ENOMEM;
707 slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
708 if (!slots)
709 goto out_free;
710 memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots));
711 if (mem->slot >= slots->nmemslots)
712 slots->nmemslots = mem->slot + 1;
713 slots->generation++;
714 slots->memslots[mem->slot].flags |= KVM_MEMSLOT_INVALID;
715
716 old_memslots = kvm->memslots;
717 rcu_assign_pointer(kvm->memslots, slots);
718 synchronize_srcu_expedited(&kvm->srcu);
719 /* From this point no new shadow pages pointing to a deleted
720 * memslot will be created.
721 *
722 * validation of sp->gfn happens in:
723 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
724 * - kvm_is_visible_gfn (mmu_check_roots)
725 */
726 kvm_arch_flush_shadow(kvm);
727 kfree(old_memslots);
728 }
729
730 r = kvm_arch_prepare_memory_region(kvm, &new, old, mem, user_alloc);
731 if (r)
732 goto out_free;
733
734 /* map the pages in iommu page table */
735 if (npages) {
736 r = kvm_iommu_map_pages(kvm, &new);
737 if (r)
738 goto out_free;
739 }
740
741 r = -ENOMEM;
742 slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
743 if (!slots)
744 goto out_free;
745 memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots));
746 if (mem->slot >= slots->nmemslots)
747 slots->nmemslots = mem->slot + 1;
748 slots->generation++;
749
750 /* actual memory is freed via old in kvm_free_physmem_slot below */
751 if (!npages) {
752 new.rmap = NULL;
753 new.dirty_bitmap = NULL;
754 for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i)
755 new.lpage_info[i] = NULL;
756 }
757
758 slots->memslots[mem->slot] = new;
759 old_memslots = kvm->memslots;
760 rcu_assign_pointer(kvm->memslots, slots);
761 synchronize_srcu_expedited(&kvm->srcu);
762
763 kvm_arch_commit_memory_region(kvm, mem, old, user_alloc);
764
765 kvm_free_physmem_slot(&old, &new);
766 kfree(old_memslots);
767
768 if (flush_shadow)
769 kvm_arch_flush_shadow(kvm);
770
771 return 0;
772
773 out_free:
774 kvm_free_physmem_slot(&new, &old);
775 out:
776 return r;
777
778 }
779 EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
780
781 int kvm_set_memory_region(struct kvm *kvm,
782 struct kvm_userspace_memory_region *mem,
783 int user_alloc)
784 {
785 int r;
786
787 mutex_lock(&kvm->slots_lock);
788 r = __kvm_set_memory_region(kvm, mem, user_alloc);
789 mutex_unlock(&kvm->slots_lock);
790 return r;
791 }
792 EXPORT_SYMBOL_GPL(kvm_set_memory_region);
793
794 int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
795 struct
796 kvm_userspace_memory_region *mem,
797 int user_alloc)
798 {
799 if (mem->slot >= KVM_MEMORY_SLOTS)
800 return -EINVAL;
801 return kvm_set_memory_region(kvm, mem, user_alloc);
802 }
803
804 int kvm_get_dirty_log(struct kvm *kvm,
805 struct kvm_dirty_log *log, int *is_dirty)
806 {
807 struct kvm_memory_slot *memslot;
808 int r, i;
809 unsigned long n;
810 unsigned long any = 0;
811
812 r = -EINVAL;
813 if (log->slot >= KVM_MEMORY_SLOTS)
814 goto out;
815
816 memslot = &kvm->memslots->memslots[log->slot];
817 r = -ENOENT;
818 if (!memslot->dirty_bitmap)
819 goto out;
820
821 n = kvm_dirty_bitmap_bytes(memslot);
822
823 for (i = 0; !any && i < n/sizeof(long); ++i)
824 any = memslot->dirty_bitmap[i];
825
826 r = -EFAULT;
827 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
828 goto out;
829
830 if (any)
831 *is_dirty = 1;
832
833 r = 0;
834 out:
835 return r;
836 }
837
838 void kvm_disable_largepages(void)
839 {
840 largepages_enabled = false;
841 }
842 EXPORT_SYMBOL_GPL(kvm_disable_largepages);
843
844 int is_error_page(struct page *page)
845 {
846 return page == bad_page || page == hwpoison_page || page == fault_page;
847 }
848 EXPORT_SYMBOL_GPL(is_error_page);
849
850 int is_error_pfn(pfn_t pfn)
851 {
852 return pfn == bad_pfn || pfn == hwpoison_pfn || pfn == fault_pfn;
853 }
854 EXPORT_SYMBOL_GPL(is_error_pfn);
855
856 int is_hwpoison_pfn(pfn_t pfn)
857 {
858 return pfn == hwpoison_pfn;
859 }
860 EXPORT_SYMBOL_GPL(is_hwpoison_pfn);
861
862 int is_fault_pfn(pfn_t pfn)
863 {
864 return pfn == fault_pfn;
865 }
866 EXPORT_SYMBOL_GPL(is_fault_pfn);
867
868 static inline unsigned long bad_hva(void)
869 {
870 return PAGE_OFFSET;
871 }
872
873 int kvm_is_error_hva(unsigned long addr)
874 {
875 return addr == bad_hva();
876 }
877 EXPORT_SYMBOL_GPL(kvm_is_error_hva);
878
879 static struct kvm_memory_slot *__gfn_to_memslot(struct kvm_memslots *slots,
880 gfn_t gfn)
881 {
882 int i;
883
884 for (i = 0; i < slots->nmemslots; ++i) {
885 struct kvm_memory_slot *memslot = &slots->memslots[i];
886
887 if (gfn >= memslot->base_gfn
888 && gfn < memslot->base_gfn + memslot->npages)
889 return memslot;
890 }
891 return NULL;
892 }
893
894 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
895 {
896 return __gfn_to_memslot(kvm_memslots(kvm), gfn);
897 }
898 EXPORT_SYMBOL_GPL(gfn_to_memslot);
899
900 int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
901 {
902 int i;
903 struct kvm_memslots *slots = kvm_memslots(kvm);
904
905 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
906 struct kvm_memory_slot *memslot = &slots->memslots[i];
907
908 if (memslot->flags & KVM_MEMSLOT_INVALID)
909 continue;
910
911 if (gfn >= memslot->base_gfn
912 && gfn < memslot->base_gfn + memslot->npages)
913 return 1;
914 }
915 return 0;
916 }
917 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
918
919 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
920 {
921 struct vm_area_struct *vma;
922 unsigned long addr, size;
923
924 size = PAGE_SIZE;
925
926 addr = gfn_to_hva(kvm, gfn);
927 if (kvm_is_error_hva(addr))
928 return PAGE_SIZE;
929
930 down_read(&current->mm->mmap_sem);
931 vma = find_vma(current->mm, addr);
932 if (!vma)
933 goto out;
934
935 size = vma_kernel_pagesize(vma);
936
937 out:
938 up_read(&current->mm->mmap_sem);
939
940 return size;
941 }
942
943 int memslot_id(struct kvm *kvm, gfn_t gfn)
944 {
945 int i;
946 struct kvm_memslots *slots = kvm_memslots(kvm);
947 struct kvm_memory_slot *memslot = NULL;
948
949 for (i = 0; i < slots->nmemslots; ++i) {
950 memslot = &slots->memslots[i];
951
952 if (gfn >= memslot->base_gfn
953 && gfn < memslot->base_gfn + memslot->npages)
954 break;
955 }
956
957 return memslot - slots->memslots;
958 }
959
960 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
961 gfn_t *nr_pages)
962 {
963 if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
964 return bad_hva();
965
966 if (nr_pages)
967 *nr_pages = slot->npages - (gfn - slot->base_gfn);
968
969 return gfn_to_hva_memslot(slot, gfn);
970 }
971
972 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
973 {
974 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
975 }
976 EXPORT_SYMBOL_GPL(gfn_to_hva);
977
978 static pfn_t get_fault_pfn(void)
979 {
980 get_page(fault_page);
981 return fault_pfn;
982 }
983
984 static pfn_t hva_to_pfn(struct kvm *kvm, unsigned long addr, bool atomic,
985 bool *async, bool write_fault, bool *writable)
986 {
987 struct page *page[1];
988 int npages = 0;
989 pfn_t pfn;
990
991 /* we can do it either atomically or asynchronously, not both */
992 BUG_ON(atomic && async);
993
994 BUG_ON(!write_fault && !writable);
995
996 if (writable)
997 *writable = true;
998
999 if (atomic || async)
1000 npages = __get_user_pages_fast(addr, 1, 1, page);
1001
1002 if (unlikely(npages != 1) && !atomic) {
1003 might_sleep();
1004
1005 if (writable)
1006 *writable = write_fault;
1007
1008 npages = get_user_pages_fast(addr, 1, write_fault, page);
1009
1010 /* map read fault as writable if possible */
1011 if (unlikely(!write_fault) && npages == 1) {
1012 struct page *wpage[1];
1013
1014 npages = __get_user_pages_fast(addr, 1, 1, wpage);
1015 if (npages == 1) {
1016 *writable = true;
1017 put_page(page[0]);
1018 page[0] = wpage[0];
1019 }
1020 npages = 1;
1021 }
1022 }
1023
1024 if (unlikely(npages != 1)) {
1025 struct vm_area_struct *vma;
1026
1027 if (atomic)
1028 return get_fault_pfn();
1029
1030 down_read(&current->mm->mmap_sem);
1031 if (is_hwpoison_address(addr)) {
1032 up_read(&current->mm->mmap_sem);
1033 get_page(hwpoison_page);
1034 return page_to_pfn(hwpoison_page);
1035 }
1036
1037 vma = find_vma_intersection(current->mm, addr, addr+1);
1038
1039 if (vma == NULL)
1040 pfn = get_fault_pfn();
1041 else if ((vma->vm_flags & VM_PFNMAP)) {
1042 pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) +
1043 vma->vm_pgoff;
1044 BUG_ON(!kvm_is_mmio_pfn(pfn));
1045 } else {
1046 if (async && (vma->vm_flags & VM_WRITE))
1047 *async = true;
1048 pfn = get_fault_pfn();
1049 }
1050 up_read(&current->mm->mmap_sem);
1051 } else
1052 pfn = page_to_pfn(page[0]);
1053
1054 return pfn;
1055 }
1056
1057 pfn_t hva_to_pfn_atomic(struct kvm *kvm, unsigned long addr)
1058 {
1059 return hva_to_pfn(kvm, addr, true, NULL, true, NULL);
1060 }
1061 EXPORT_SYMBOL_GPL(hva_to_pfn_atomic);
1062
1063 static pfn_t __gfn_to_pfn(struct kvm *kvm, gfn_t gfn, bool atomic, bool *async,
1064 bool write_fault, bool *writable)
1065 {
1066 unsigned long addr;
1067
1068 if (async)
1069 *async = false;
1070
1071 addr = gfn_to_hva(kvm, gfn);
1072 if (kvm_is_error_hva(addr)) {
1073 get_page(bad_page);
1074 return page_to_pfn(bad_page);
1075 }
1076
1077 return hva_to_pfn(kvm, addr, atomic, async, write_fault, writable);
1078 }
1079
1080 pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
1081 {
1082 return __gfn_to_pfn(kvm, gfn, true, NULL, true, NULL);
1083 }
1084 EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);
1085
1086 pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async,
1087 bool write_fault, bool *writable)
1088 {
1089 return __gfn_to_pfn(kvm, gfn, false, async, write_fault, writable);
1090 }
1091 EXPORT_SYMBOL_GPL(gfn_to_pfn_async);
1092
1093 pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
1094 {
1095 return __gfn_to_pfn(kvm, gfn, false, NULL, true, NULL);
1096 }
1097 EXPORT_SYMBOL_GPL(gfn_to_pfn);
1098
1099 pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
1100 bool *writable)
1101 {
1102 return __gfn_to_pfn(kvm, gfn, false, NULL, write_fault, writable);
1103 }
1104 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);
1105
1106 pfn_t gfn_to_pfn_memslot(struct kvm *kvm,
1107 struct kvm_memory_slot *slot, gfn_t gfn)
1108 {
1109 unsigned long addr = gfn_to_hva_memslot(slot, gfn);
1110 return hva_to_pfn(kvm, addr, false, NULL, true, NULL);
1111 }
1112
1113 int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages,
1114 int nr_pages)
1115 {
1116 unsigned long addr;
1117 gfn_t entry;
1118
1119 addr = gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, &entry);
1120 if (kvm_is_error_hva(addr))
1121 return -1;
1122
1123 if (entry < nr_pages)
1124 return 0;
1125
1126 return __get_user_pages_fast(addr, nr_pages, 1, pages);
1127 }
1128 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);
1129
1130 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
1131 {
1132 pfn_t pfn;
1133
1134 pfn = gfn_to_pfn(kvm, gfn);
1135 if (!kvm_is_mmio_pfn(pfn))
1136 return pfn_to_page(pfn);
1137
1138 WARN_ON(kvm_is_mmio_pfn(pfn));
1139
1140 get_page(bad_page);
1141 return bad_page;
1142 }
1143
1144 EXPORT_SYMBOL_GPL(gfn_to_page);
1145
1146 void kvm_release_page_clean(struct page *page)
1147 {
1148 kvm_release_pfn_clean(page_to_pfn(page));
1149 }
1150 EXPORT_SYMBOL_GPL(kvm_release_page_clean);
1151
1152 void kvm_release_pfn_clean(pfn_t pfn)
1153 {
1154 if (!kvm_is_mmio_pfn(pfn))
1155 put_page(pfn_to_page(pfn));
1156 }
1157 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
1158
1159 void kvm_release_page_dirty(struct page *page)
1160 {
1161 kvm_release_pfn_dirty(page_to_pfn(page));
1162 }
1163 EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
1164
1165 void kvm_release_pfn_dirty(pfn_t pfn)
1166 {
1167 kvm_set_pfn_dirty(pfn);
1168 kvm_release_pfn_clean(pfn);
1169 }
1170 EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);
1171
1172 void kvm_set_page_dirty(struct page *page)
1173 {
1174 kvm_set_pfn_dirty(page_to_pfn(page));
1175 }
1176 EXPORT_SYMBOL_GPL(kvm_set_page_dirty);
1177
1178 void kvm_set_pfn_dirty(pfn_t pfn)
1179 {
1180 if (!kvm_is_mmio_pfn(pfn)) {
1181 struct page *page = pfn_to_page(pfn);
1182 if (!PageReserved(page))
1183 SetPageDirty(page);
1184 }
1185 }
1186 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
1187
1188 void kvm_set_pfn_accessed(pfn_t pfn)
1189 {
1190 if (!kvm_is_mmio_pfn(pfn))
1191 mark_page_accessed(pfn_to_page(pfn));
1192 }
1193 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
1194
1195 void kvm_get_pfn(pfn_t pfn)
1196 {
1197 if (!kvm_is_mmio_pfn(pfn))
1198 get_page(pfn_to_page(pfn));
1199 }
1200 EXPORT_SYMBOL_GPL(kvm_get_pfn);
1201
1202 static int next_segment(unsigned long len, int offset)
1203 {
1204 if (len > PAGE_SIZE - offset)
1205 return PAGE_SIZE - offset;
1206 else
1207 return len;
1208 }
1209
1210 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
1211 int len)
1212 {
1213 int r;
1214 unsigned long addr;
1215
1216 addr = gfn_to_hva(kvm, gfn);
1217 if (kvm_is_error_hva(addr))
1218 return -EFAULT;
1219 r = copy_from_user(data, (void __user *)addr + offset, len);
1220 if (r)
1221 return -EFAULT;
1222 return 0;
1223 }
1224 EXPORT_SYMBOL_GPL(kvm_read_guest_page);
1225
1226 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
1227 {
1228 gfn_t gfn = gpa >> PAGE_SHIFT;
1229 int seg;
1230 int offset = offset_in_page(gpa);
1231 int ret;
1232
1233 while ((seg = next_segment(len, offset)) != 0) {
1234 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
1235 if (ret < 0)
1236 return ret;
1237 offset = 0;
1238 len -= seg;
1239 data += seg;
1240 ++gfn;
1241 }
1242 return 0;
1243 }
1244 EXPORT_SYMBOL_GPL(kvm_read_guest);
1245
1246 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
1247 unsigned long len)
1248 {
1249 int r;
1250 unsigned long addr;
1251 gfn_t gfn = gpa >> PAGE_SHIFT;
1252 int offset = offset_in_page(gpa);
1253
1254 addr = gfn_to_hva(kvm, gfn);
1255 if (kvm_is_error_hva(addr))
1256 return -EFAULT;
1257 pagefault_disable();
1258 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
1259 pagefault_enable();
1260 if (r)
1261 return -EFAULT;
1262 return 0;
1263 }
1264 EXPORT_SYMBOL(kvm_read_guest_atomic);
1265
1266 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
1267 int offset, int len)
1268 {
1269 int r;
1270 unsigned long addr;
1271
1272 addr = gfn_to_hva(kvm, gfn);
1273 if (kvm_is_error_hva(addr))
1274 return -EFAULT;
1275 r = copy_to_user((void __user *)addr + offset, data, len);
1276 if (r)
1277 return -EFAULT;
1278 mark_page_dirty(kvm, gfn);
1279 return 0;
1280 }
1281 EXPORT_SYMBOL_GPL(kvm_write_guest_page);
1282
1283 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
1284 unsigned long len)
1285 {
1286 gfn_t gfn = gpa >> PAGE_SHIFT;
1287 int seg;
1288 int offset = offset_in_page(gpa);
1289 int ret;
1290
1291 while ((seg = next_segment(len, offset)) != 0) {
1292 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
1293 if (ret < 0)
1294 return ret;
1295 offset = 0;
1296 len -= seg;
1297 data += seg;
1298 ++gfn;
1299 }
1300 return 0;
1301 }
1302
1303 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1304 gpa_t gpa)
1305 {
1306 struct kvm_memslots *slots = kvm_memslots(kvm);
1307 int offset = offset_in_page(gpa);
1308 gfn_t gfn = gpa >> PAGE_SHIFT;
1309
1310 ghc->gpa = gpa;
1311 ghc->generation = slots->generation;
1312 ghc->memslot = __gfn_to_memslot(slots, gfn);
1313 ghc->hva = gfn_to_hva_many(ghc->memslot, gfn, NULL);
1314 if (!kvm_is_error_hva(ghc->hva))
1315 ghc->hva += offset;
1316 else
1317 return -EFAULT;
1318
1319 return 0;
1320 }
1321 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);
1322
1323 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1324 void *data, unsigned long len)
1325 {
1326 struct kvm_memslots *slots = kvm_memslots(kvm);
1327 int r;
1328
1329 if (slots->generation != ghc->generation)
1330 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa);
1331
1332 if (kvm_is_error_hva(ghc->hva))
1333 return -EFAULT;
1334
1335 r = copy_to_user((void __user *)ghc->hva, data, len);
1336 if (r)
1337 return -EFAULT;
1338 mark_page_dirty_in_slot(kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT);
1339
1340 return 0;
1341 }
1342 EXPORT_SYMBOL_GPL(kvm_write_guest_cached);
1343
1344 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
1345 {
1346 return kvm_write_guest_page(kvm, gfn, (const void *) empty_zero_page,
1347 offset, len);
1348 }
1349 EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
1350
1351 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
1352 {
1353 gfn_t gfn = gpa >> PAGE_SHIFT;
1354 int seg;
1355 int offset = offset_in_page(gpa);
1356 int ret;
1357
1358 while ((seg = next_segment(len, offset)) != 0) {
1359 ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
1360 if (ret < 0)
1361 return ret;
1362 offset = 0;
1363 len -= seg;
1364 ++gfn;
1365 }
1366 return 0;
1367 }
1368 EXPORT_SYMBOL_GPL(kvm_clear_guest);
1369
1370 void mark_page_dirty_in_slot(struct kvm *kvm, struct kvm_memory_slot *memslot,
1371 gfn_t gfn)
1372 {
1373 if (memslot && memslot->dirty_bitmap) {
1374 unsigned long rel_gfn = gfn - memslot->base_gfn;
1375
1376 generic___set_le_bit(rel_gfn, memslot->dirty_bitmap);
1377 }
1378 }
1379
1380 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
1381 {
1382 struct kvm_memory_slot *memslot;
1383
1384 memslot = gfn_to_memslot(kvm, gfn);
1385 mark_page_dirty_in_slot(kvm, memslot, gfn);
1386 }
1387
1388 /*
1389 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
1390 */
1391 void kvm_vcpu_block(struct kvm_vcpu *vcpu)
1392 {
1393 DEFINE_WAIT(wait);
1394
1395 for (;;) {
1396 prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
1397
1398 if (kvm_arch_vcpu_runnable(vcpu)) {
1399 kvm_make_request(KVM_REQ_UNHALT, vcpu);
1400 break;
1401 }
1402 if (kvm_cpu_has_pending_timer(vcpu))
1403 break;
1404 if (signal_pending(current))
1405 break;
1406
1407 schedule();
1408 }
1409
1410 finish_wait(&vcpu->wq, &wait);
1411 }
1412
1413 void kvm_resched(struct kvm_vcpu *vcpu)
1414 {
1415 if (!need_resched())
1416 return;
1417 cond_resched();
1418 }
1419 EXPORT_SYMBOL_GPL(kvm_resched);
1420
1421 void kvm_vcpu_on_spin(struct kvm_vcpu *vcpu)
1422 {
1423 ktime_t expires;
1424 DEFINE_WAIT(wait);
1425
1426 prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
1427
1428 /* Sleep for 100 us, and hope lock-holder got scheduled */
1429 expires = ktime_add_ns(ktime_get(), 100000UL);
1430 schedule_hrtimeout(&expires, HRTIMER_MODE_ABS);
1431
1432 finish_wait(&vcpu->wq, &wait);
1433 }
1434 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
1435
1436 static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1437 {
1438 struct kvm_vcpu *vcpu = vma->vm_file->private_data;
1439 struct page *page;
1440
1441 if (vmf->pgoff == 0)
1442 page = virt_to_page(vcpu->run);
1443 #ifdef CONFIG_X86
1444 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
1445 page = virt_to_page(vcpu->arch.pio_data);
1446 #endif
1447 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
1448 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
1449 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
1450 #endif
1451 else
1452 return VM_FAULT_SIGBUS;
1453 get_page(page);
1454 vmf->page = page;
1455 return 0;
1456 }
1457
1458 static const struct vm_operations_struct kvm_vcpu_vm_ops = {
1459 .fault = kvm_vcpu_fault,
1460 };
1461
1462 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
1463 {
1464 vma->vm_ops = &kvm_vcpu_vm_ops;
1465 return 0;
1466 }
1467
1468 static int kvm_vcpu_release(struct inode *inode, struct file *filp)
1469 {
1470 struct kvm_vcpu *vcpu = filp->private_data;
1471
1472 kvm_put_kvm(vcpu->kvm);
1473 return 0;
1474 }
1475
1476 static struct file_operations kvm_vcpu_fops = {
1477 .release = kvm_vcpu_release,
1478 .unlocked_ioctl = kvm_vcpu_ioctl,
1479 .compat_ioctl = kvm_vcpu_ioctl,
1480 .mmap = kvm_vcpu_mmap,
1481 .llseek = noop_llseek,
1482 };
1483
1484 /*
1485 * Allocates an inode for the vcpu.
1486 */
1487 static int create_vcpu_fd(struct kvm_vcpu *vcpu)
1488 {
1489 return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR);
1490 }
1491
1492 /*
1493 * Creates some virtual cpus. Good luck creating more than one.
1494 */
1495 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
1496 {
1497 int r;
1498 struct kvm_vcpu *vcpu, *v;
1499
1500 vcpu = kvm_arch_vcpu_create(kvm, id);
1501 if (IS_ERR(vcpu))
1502 return PTR_ERR(vcpu);
1503
1504 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
1505
1506 r = kvm_arch_vcpu_setup(vcpu);
1507 if (r)
1508 return r;
1509
1510 mutex_lock(&kvm->lock);
1511 if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) {
1512 r = -EINVAL;
1513 goto vcpu_destroy;
1514 }
1515
1516 kvm_for_each_vcpu(r, v, kvm)
1517 if (v->vcpu_id == id) {
1518 r = -EEXIST;
1519 goto vcpu_destroy;
1520 }
1521
1522 BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
1523
1524 /* Now it's all set up, let userspace reach it */
1525 kvm_get_kvm(kvm);
1526 r = create_vcpu_fd(vcpu);
1527 if (r < 0) {
1528 kvm_put_kvm(kvm);
1529 goto vcpu_destroy;
1530 }
1531
1532 kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
1533 smp_wmb();
1534 atomic_inc(&kvm->online_vcpus);
1535
1536 #ifdef CONFIG_KVM_APIC_ARCHITECTURE
1537 if (kvm->bsp_vcpu_id == id)
1538 kvm->bsp_vcpu = vcpu;
1539 #endif
1540 mutex_unlock(&kvm->lock);
1541 return r;
1542
1543 vcpu_destroy:
1544 mutex_unlock(&kvm->lock);
1545 kvm_arch_vcpu_destroy(vcpu);
1546 return r;
1547 }
1548
1549 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
1550 {
1551 if (sigset) {
1552 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
1553 vcpu->sigset_active = 1;
1554 vcpu->sigset = *sigset;
1555 } else
1556 vcpu->sigset_active = 0;
1557 return 0;
1558 }
1559
1560 static long kvm_vcpu_ioctl(struct file *filp,
1561 unsigned int ioctl, unsigned long arg)
1562 {
1563 struct kvm_vcpu *vcpu = filp->private_data;
1564 void __user *argp = (void __user *)arg;
1565 int r;
1566 struct kvm_fpu *fpu = NULL;
1567 struct kvm_sregs *kvm_sregs = NULL;
1568
1569 if (vcpu->kvm->mm != current->mm)
1570 return -EIO;
1571
1572 #if defined(CONFIG_S390) || defined(CONFIG_PPC)
1573 /*
1574 * Special cases: vcpu ioctls that are asynchronous to vcpu execution,
1575 * so vcpu_load() would break it.
1576 */
1577 if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_INTERRUPT)
1578 return kvm_arch_vcpu_ioctl(filp, ioctl, arg);
1579 #endif
1580
1581
1582 vcpu_load(vcpu);
1583 switch (ioctl) {
1584 case KVM_RUN:
1585 r = -EINVAL;
1586 if (arg)
1587 goto out;
1588 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
1589 trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
1590 break;
1591 case KVM_GET_REGS: {
1592 struct kvm_regs *kvm_regs;
1593
1594 r = -ENOMEM;
1595 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
1596 if (!kvm_regs)
1597 goto out;
1598 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
1599 if (r)
1600 goto out_free1;
1601 r = -EFAULT;
1602 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
1603 goto out_free1;
1604 r = 0;
1605 out_free1:
1606 kfree(kvm_regs);
1607 break;
1608 }
1609 case KVM_SET_REGS: {
1610 struct kvm_regs *kvm_regs;
1611
1612 r = -ENOMEM;
1613 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
1614 if (!kvm_regs)
1615 goto out;
1616 r = -EFAULT;
1617 if (copy_from_user(kvm_regs, argp, sizeof(struct kvm_regs)))
1618 goto out_free2;
1619 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
1620 if (r)
1621 goto out_free2;
1622 r = 0;
1623 out_free2:
1624 kfree(kvm_regs);
1625 break;
1626 }
1627 case KVM_GET_SREGS: {
1628 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
1629 r = -ENOMEM;
1630 if (!kvm_sregs)
1631 goto out;
1632 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
1633 if (r)
1634 goto out;
1635 r = -EFAULT;
1636 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
1637 goto out;
1638 r = 0;
1639 break;
1640 }
1641 case KVM_SET_SREGS: {
1642 kvm_sregs = kmalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
1643 r = -ENOMEM;
1644 if (!kvm_sregs)
1645 goto out;
1646 r = -EFAULT;
1647 if (copy_from_user(kvm_sregs, argp, sizeof(struct kvm_sregs)))
1648 goto out;
1649 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
1650 if (r)
1651 goto out;
1652 r = 0;
1653 break;
1654 }
1655 case KVM_GET_MP_STATE: {
1656 struct kvm_mp_state mp_state;
1657
1658 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
1659 if (r)
1660 goto out;
1661 r = -EFAULT;
1662 if (copy_to_user(argp, &mp_state, sizeof mp_state))
1663 goto out;
1664 r = 0;
1665 break;
1666 }
1667 case KVM_SET_MP_STATE: {
1668 struct kvm_mp_state mp_state;
1669
1670 r = -EFAULT;
1671 if (copy_from_user(&mp_state, argp, sizeof mp_state))
1672 goto out;
1673 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
1674 if (r)
1675 goto out;
1676 r = 0;
1677 break;
1678 }
1679 case KVM_TRANSLATE: {
1680 struct kvm_translation tr;
1681
1682 r = -EFAULT;
1683 if (copy_from_user(&tr, argp, sizeof tr))
1684 goto out;
1685 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
1686 if (r)
1687 goto out;
1688 r = -EFAULT;
1689 if (copy_to_user(argp, &tr, sizeof tr))
1690 goto out;
1691 r = 0;
1692 break;
1693 }
1694 case KVM_SET_GUEST_DEBUG: {
1695 struct kvm_guest_debug dbg;
1696
1697 r = -EFAULT;
1698 if (copy_from_user(&dbg, argp, sizeof dbg))
1699 goto out;
1700 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
1701 if (r)
1702 goto out;
1703 r = 0;
1704 break;
1705 }
1706 case KVM_SET_SIGNAL_MASK: {
1707 struct kvm_signal_mask __user *sigmask_arg = argp;
1708 struct kvm_signal_mask kvm_sigmask;
1709 sigset_t sigset, *p;
1710
1711 p = NULL;
1712 if (argp) {
1713 r = -EFAULT;
1714 if (copy_from_user(&kvm_sigmask, argp,
1715 sizeof kvm_sigmask))
1716 goto out;
1717 r = -EINVAL;
1718 if (kvm_sigmask.len != sizeof sigset)
1719 goto out;
1720 r = -EFAULT;
1721 if (copy_from_user(&sigset, sigmask_arg->sigset,
1722 sizeof sigset))
1723 goto out;
1724 p = &sigset;
1725 }
1726 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
1727 break;
1728 }
1729 case KVM_GET_FPU: {
1730 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
1731 r = -ENOMEM;
1732 if (!fpu)
1733 goto out;
1734 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
1735 if (r)
1736 goto out;
1737 r = -EFAULT;
1738 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
1739 goto out;
1740 r = 0;
1741 break;
1742 }
1743 case KVM_SET_FPU: {
1744 fpu = kmalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
1745 r = -ENOMEM;
1746 if (!fpu)
1747 goto out;
1748 r = -EFAULT;
1749 if (copy_from_user(fpu, argp, sizeof(struct kvm_fpu)))
1750 goto out;
1751 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
1752 if (r)
1753 goto out;
1754 r = 0;
1755 break;
1756 }
1757 default:
1758 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
1759 }
1760 out:
1761 vcpu_put(vcpu);
1762 kfree(fpu);
1763 kfree(kvm_sregs);
1764 return r;
1765 }
1766
1767 static long kvm_vm_ioctl(struct file *filp,
1768 unsigned int ioctl, unsigned long arg)
1769 {
1770 struct kvm *kvm = filp->private_data;
1771 void __user *argp = (void __user *)arg;
1772 int r;
1773
1774 if (kvm->mm != current->mm)
1775 return -EIO;
1776 switch (ioctl) {
1777 case KVM_CREATE_VCPU:
1778 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
1779 if (r < 0)
1780 goto out;
1781 break;
1782 case KVM_SET_USER_MEMORY_REGION: {
1783 struct kvm_userspace_memory_region kvm_userspace_mem;
1784
1785 r = -EFAULT;
1786 if (copy_from_user(&kvm_userspace_mem, argp,
1787 sizeof kvm_userspace_mem))
1788 goto out;
1789
1790 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1);
1791 if (r)
1792 goto out;
1793 break;
1794 }
1795 case KVM_GET_DIRTY_LOG: {
1796 struct kvm_dirty_log log;
1797
1798 r = -EFAULT;
1799 if (copy_from_user(&log, argp, sizeof log))
1800 goto out;
1801 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
1802 if (r)
1803 goto out;
1804 break;
1805 }
1806 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
1807 case KVM_REGISTER_COALESCED_MMIO: {
1808 struct kvm_coalesced_mmio_zone zone;
1809 r = -EFAULT;
1810 if (copy_from_user(&zone, argp, sizeof zone))
1811 goto out;
1812 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
1813 if (r)
1814 goto out;
1815 r = 0;
1816 break;
1817 }
1818 case KVM_UNREGISTER_COALESCED_MMIO: {
1819 struct kvm_coalesced_mmio_zone zone;
1820 r = -EFAULT;
1821 if (copy_from_user(&zone, argp, sizeof zone))
1822 goto out;
1823 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
1824 if (r)
1825 goto out;
1826 r = 0;
1827 break;
1828 }
1829 #endif
1830 case KVM_IRQFD: {
1831 struct kvm_irqfd data;
1832
1833 r = -EFAULT;
1834 if (copy_from_user(&data, argp, sizeof data))
1835 goto out;
1836 r = kvm_irqfd(kvm, data.fd, data.gsi, data.flags);
1837 break;
1838 }
1839 case KVM_IOEVENTFD: {
1840 struct kvm_ioeventfd data;
1841
1842 r = -EFAULT;
1843 if (copy_from_user(&data, argp, sizeof data))
1844 goto out;
1845 r = kvm_ioeventfd(kvm, &data);
1846 break;
1847 }
1848 #ifdef CONFIG_KVM_APIC_ARCHITECTURE
1849 case KVM_SET_BOOT_CPU_ID:
1850 r = 0;
1851 mutex_lock(&kvm->lock);
1852 if (atomic_read(&kvm->online_vcpus) != 0)
1853 r = -EBUSY;
1854 else
1855 kvm->bsp_vcpu_id = arg;
1856 mutex_unlock(&kvm->lock);
1857 break;
1858 #endif
1859 default:
1860 r = kvm_arch_vm_ioctl(filp, ioctl, arg);
1861 if (r == -ENOTTY)
1862 r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg);
1863 }
1864 out:
1865 return r;
1866 }
1867
1868 #ifdef CONFIG_COMPAT
1869 struct compat_kvm_dirty_log {
1870 __u32 slot;
1871 __u32 padding1;
1872 union {
1873 compat_uptr_t dirty_bitmap; /* one bit per page */
1874 __u64 padding2;
1875 };
1876 };
1877
1878 static long kvm_vm_compat_ioctl(struct file *filp,
1879 unsigned int ioctl, unsigned long arg)
1880 {
1881 struct kvm *kvm = filp->private_data;
1882 int r;
1883
1884 if (kvm->mm != current->mm)
1885 return -EIO;
1886 switch (ioctl) {
1887 case KVM_GET_DIRTY_LOG: {
1888 struct compat_kvm_dirty_log compat_log;
1889 struct kvm_dirty_log log;
1890
1891 r = -EFAULT;
1892 if (copy_from_user(&compat_log, (void __user *)arg,
1893 sizeof(compat_log)))
1894 goto out;
1895 log.slot = compat_log.slot;
1896 log.padding1 = compat_log.padding1;
1897 log.padding2 = compat_log.padding2;
1898 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
1899
1900 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
1901 if (r)
1902 goto out;
1903 break;
1904 }
1905 default:
1906 r = kvm_vm_ioctl(filp, ioctl, arg);
1907 }
1908
1909 out:
1910 return r;
1911 }
1912 #endif
1913
1914 static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1915 {
1916 struct page *page[1];
1917 unsigned long addr;
1918 int npages;
1919 gfn_t gfn = vmf->pgoff;
1920 struct kvm *kvm = vma->vm_file->private_data;
1921
1922 addr = gfn_to_hva(kvm, gfn);
1923 if (kvm_is_error_hva(addr))
1924 return VM_FAULT_SIGBUS;
1925
1926 npages = get_user_pages(current, current->mm, addr, 1, 1, 0, page,
1927 NULL);
1928 if (unlikely(npages != 1))
1929 return VM_FAULT_SIGBUS;
1930
1931 vmf->page = page[0];
1932 return 0;
1933 }
1934
1935 static const struct vm_operations_struct kvm_vm_vm_ops = {
1936 .fault = kvm_vm_fault,
1937 };
1938
1939 static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
1940 {
1941 vma->vm_ops = &kvm_vm_vm_ops;
1942 return 0;
1943 }
1944
1945 static struct file_operations kvm_vm_fops = {
1946 .release = kvm_vm_release,
1947 .unlocked_ioctl = kvm_vm_ioctl,
1948 #ifdef CONFIG_COMPAT
1949 .compat_ioctl = kvm_vm_compat_ioctl,
1950 #endif
1951 .mmap = kvm_vm_mmap,
1952 .llseek = noop_llseek,
1953 };
1954
1955 static int kvm_dev_ioctl_create_vm(void)
1956 {
1957 int r;
1958 struct kvm *kvm;
1959
1960 kvm = kvm_create_vm();
1961 if (IS_ERR(kvm))
1962 return PTR_ERR(kvm);
1963 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
1964 r = kvm_coalesced_mmio_init(kvm);
1965 if (r < 0) {
1966 kvm_put_kvm(kvm);
1967 return r;
1968 }
1969 #endif
1970 r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
1971 if (r < 0)
1972 kvm_put_kvm(kvm);
1973
1974 return r;
1975 }
1976
1977 static long kvm_dev_ioctl_check_extension_generic(long arg)
1978 {
1979 switch (arg) {
1980 case KVM_CAP_USER_MEMORY:
1981 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
1982 case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
1983 #ifdef CONFIG_KVM_APIC_ARCHITECTURE
1984 case KVM_CAP_SET_BOOT_CPU_ID:
1985 #endif
1986 case KVM_CAP_INTERNAL_ERROR_DATA:
1987 return 1;
1988 #ifdef CONFIG_HAVE_KVM_IRQCHIP
1989 case KVM_CAP_IRQ_ROUTING:
1990 return KVM_MAX_IRQ_ROUTES;
1991 #endif
1992 default:
1993 break;
1994 }
1995 return kvm_dev_ioctl_check_extension(arg);
1996 }
1997
1998 static long kvm_dev_ioctl(struct file *filp,
1999 unsigned int ioctl, unsigned long arg)
2000 {
2001 long r = -EINVAL;
2002
2003 switch (ioctl) {
2004 case KVM_GET_API_VERSION:
2005 r = -EINVAL;
2006 if (arg)
2007 goto out;
2008 r = KVM_API_VERSION;
2009 break;
2010 case KVM_CREATE_VM:
2011 r = -EINVAL;
2012 if (arg)
2013 goto out;
2014 r = kvm_dev_ioctl_create_vm();
2015 break;
2016 case KVM_CHECK_EXTENSION:
2017 r = kvm_dev_ioctl_check_extension_generic(arg);
2018 break;
2019 case KVM_GET_VCPU_MMAP_SIZE:
2020 r = -EINVAL;
2021 if (arg)
2022 goto out;
2023 r = PAGE_SIZE; /* struct kvm_run */
2024 #ifdef CONFIG_X86
2025 r += PAGE_SIZE; /* pio data page */
2026 #endif
2027 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2028 r += PAGE_SIZE; /* coalesced mmio ring page */
2029 #endif
2030 break;
2031 case KVM_TRACE_ENABLE:
2032 case KVM_TRACE_PAUSE:
2033 case KVM_TRACE_DISABLE:
2034 r = -EOPNOTSUPP;
2035 break;
2036 default:
2037 return kvm_arch_dev_ioctl(filp, ioctl, arg);
2038 }
2039 out:
2040 return r;
2041 }
2042
2043 static struct file_operations kvm_chardev_ops = {
2044 .unlocked_ioctl = kvm_dev_ioctl,
2045 .compat_ioctl = kvm_dev_ioctl,
2046 .llseek = noop_llseek,
2047 };
2048
2049 static struct miscdevice kvm_dev = {
2050 KVM_MINOR,
2051 "kvm",
2052 &kvm_chardev_ops,
2053 };
2054
2055 static void hardware_enable(void *junk)
2056 {
2057 int cpu = raw_smp_processor_id();
2058 int r;
2059
2060 if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
2061 return;
2062
2063 cpumask_set_cpu(cpu, cpus_hardware_enabled);
2064
2065 r = kvm_arch_hardware_enable(NULL);
2066
2067 if (r) {
2068 cpumask_clear_cpu(cpu, cpus_hardware_enabled);
2069 atomic_inc(&hardware_enable_failed);
2070 printk(KERN_INFO "kvm: enabling virtualization on "
2071 "CPU%d failed\n", cpu);
2072 }
2073 }
2074
2075 static void hardware_disable(void *junk)
2076 {
2077 int cpu = raw_smp_processor_id();
2078
2079 if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
2080 return;
2081 cpumask_clear_cpu(cpu, cpus_hardware_enabled);
2082 kvm_arch_hardware_disable(NULL);
2083 }
2084
2085 static void hardware_disable_all_nolock(void)
2086 {
2087 BUG_ON(!kvm_usage_count);
2088
2089 kvm_usage_count--;
2090 if (!kvm_usage_count)
2091 on_each_cpu(hardware_disable, NULL, 1);
2092 }
2093
2094 static void hardware_disable_all(void)
2095 {
2096 spin_lock(&kvm_lock);
2097 hardware_disable_all_nolock();
2098 spin_unlock(&kvm_lock);
2099 }
2100
2101 static int hardware_enable_all(void)
2102 {
2103 int r = 0;
2104
2105 spin_lock(&kvm_lock);
2106
2107 kvm_usage_count++;
2108 if (kvm_usage_count == 1) {
2109 atomic_set(&hardware_enable_failed, 0);
2110 on_each_cpu(hardware_enable, NULL, 1);
2111
2112 if (atomic_read(&hardware_enable_failed)) {
2113 hardware_disable_all_nolock();
2114 r = -EBUSY;
2115 }
2116 }
2117
2118 spin_unlock(&kvm_lock);
2119
2120 return r;
2121 }
2122
2123 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
2124 void *v)
2125 {
2126 int cpu = (long)v;
2127
2128 if (!kvm_usage_count)
2129 return NOTIFY_OK;
2130
2131 val &= ~CPU_TASKS_FROZEN;
2132 switch (val) {
2133 case CPU_DYING:
2134 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
2135 cpu);
2136 hardware_disable(NULL);
2137 break;
2138 case CPU_STARTING:
2139 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
2140 cpu);
2141 spin_lock(&kvm_lock);
2142 hardware_enable(NULL);
2143 spin_unlock(&kvm_lock);
2144 break;
2145 }
2146 return NOTIFY_OK;
2147 }
2148
2149
2150 asmlinkage void kvm_handle_fault_on_reboot(void)
2151 {
2152 if (kvm_rebooting) {
2153 /* spin while reset goes on */
2154 local_irq_enable();
2155 while (true)
2156 cpu_relax();
2157 }
2158 /* Fault while not rebooting. We want the trace. */
2159 BUG();
2160 }
2161 EXPORT_SYMBOL_GPL(kvm_handle_fault_on_reboot);
2162
2163 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
2164 void *v)
2165 {
2166 /*
2167 * Some (well, at least mine) BIOSes hang on reboot if
2168 * in vmx root mode.
2169 *
2170 * And Intel TXT required VMX off for all cpu when system shutdown.
2171 */
2172 printk(KERN_INFO "kvm: exiting hardware virtualization\n");
2173 kvm_rebooting = true;
2174 on_each_cpu(hardware_disable, NULL, 1);
2175 return NOTIFY_OK;
2176 }
2177
2178 static struct notifier_block kvm_reboot_notifier = {
2179 .notifier_call = kvm_reboot,
2180 .priority = 0,
2181 };
2182
2183 static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
2184 {
2185 int i;
2186
2187 for (i = 0; i < bus->dev_count; i++) {
2188 struct kvm_io_device *pos = bus->devs[i];
2189
2190 kvm_iodevice_destructor(pos);
2191 }
2192 kfree(bus);
2193 }
2194
2195 /* kvm_io_bus_write - called under kvm->slots_lock */
2196 int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
2197 int len, const void *val)
2198 {
2199 int i;
2200 struct kvm_io_bus *bus;
2201
2202 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
2203 for (i = 0; i < bus->dev_count; i++)
2204 if (!kvm_iodevice_write(bus->devs[i], addr, len, val))
2205 return 0;
2206 return -EOPNOTSUPP;
2207 }
2208
2209 /* kvm_io_bus_read - called under kvm->slots_lock */
2210 int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
2211 int len, void *val)
2212 {
2213 int i;
2214 struct kvm_io_bus *bus;
2215
2216 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
2217 for (i = 0; i < bus->dev_count; i++)
2218 if (!kvm_iodevice_read(bus->devs[i], addr, len, val))
2219 return 0;
2220 return -EOPNOTSUPP;
2221 }
2222
2223 /* Caller must hold slots_lock. */
2224 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx,
2225 struct kvm_io_device *dev)
2226 {
2227 struct kvm_io_bus *new_bus, *bus;
2228
2229 bus = kvm->buses[bus_idx];
2230 if (bus->dev_count > NR_IOBUS_DEVS-1)
2231 return -ENOSPC;
2232
2233 new_bus = kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL);
2234 if (!new_bus)
2235 return -ENOMEM;
2236 memcpy(new_bus, bus, sizeof(struct kvm_io_bus));
2237 new_bus->devs[new_bus->dev_count++] = dev;
2238 rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
2239 synchronize_srcu_expedited(&kvm->srcu);
2240 kfree(bus);
2241
2242 return 0;
2243 }
2244
2245 /* Caller must hold slots_lock. */
2246 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
2247 struct kvm_io_device *dev)
2248 {
2249 int i, r;
2250 struct kvm_io_bus *new_bus, *bus;
2251
2252 new_bus = kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL);
2253 if (!new_bus)
2254 return -ENOMEM;
2255
2256 bus = kvm->buses[bus_idx];
2257 memcpy(new_bus, bus, sizeof(struct kvm_io_bus));
2258
2259 r = -ENOENT;
2260 for (i = 0; i < new_bus->dev_count; i++)
2261 if (new_bus->devs[i] == dev) {
2262 r = 0;
2263 new_bus->devs[i] = new_bus->devs[--new_bus->dev_count];
2264 break;
2265 }
2266
2267 if (r) {
2268 kfree(new_bus);
2269 return r;
2270 }
2271
2272 rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
2273 synchronize_srcu_expedited(&kvm->srcu);
2274 kfree(bus);
2275 return r;
2276 }
2277
2278 static struct notifier_block kvm_cpu_notifier = {
2279 .notifier_call = kvm_cpu_hotplug,
2280 };
2281
2282 static int vm_stat_get(void *_offset, u64 *val)
2283 {
2284 unsigned offset = (long)_offset;
2285 struct kvm *kvm;
2286
2287 *val = 0;
2288 spin_lock(&kvm_lock);
2289 list_for_each_entry(kvm, &vm_list, vm_list)
2290 *val += *(u32 *)((void *)kvm + offset);
2291 spin_unlock(&kvm_lock);
2292 return 0;
2293 }
2294
2295 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");
2296
2297 static int vcpu_stat_get(void *_offset, u64 *val)
2298 {
2299 unsigned offset = (long)_offset;
2300 struct kvm *kvm;
2301 struct kvm_vcpu *vcpu;
2302 int i;
2303
2304 *val = 0;
2305 spin_lock(&kvm_lock);
2306 list_for_each_entry(kvm, &vm_list, vm_list)
2307 kvm_for_each_vcpu(i, vcpu, kvm)
2308 *val += *(u32 *)((void *)vcpu + offset);
2309
2310 spin_unlock(&kvm_lock);
2311 return 0;
2312 }
2313
2314 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");
2315
2316 static const struct file_operations *stat_fops[] = {
2317 [KVM_STAT_VCPU] = &vcpu_stat_fops,
2318 [KVM_STAT_VM] = &vm_stat_fops,
2319 };
2320
2321 static void kvm_init_debug(void)
2322 {
2323 struct kvm_stats_debugfs_item *p;
2324
2325 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
2326 for (p = debugfs_entries; p->name; ++p)
2327 p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
2328 (void *)(long)p->offset,
2329 stat_fops[p->kind]);
2330 }
2331
2332 static void kvm_exit_debug(void)
2333 {
2334 struct kvm_stats_debugfs_item *p;
2335
2336 for (p = debugfs_entries; p->name; ++p)
2337 debugfs_remove(p->dentry);
2338 debugfs_remove(kvm_debugfs_dir);
2339 }
2340
2341 static int kvm_suspend(struct sys_device *dev, pm_message_t state)
2342 {
2343 if (kvm_usage_count)
2344 hardware_disable(NULL);
2345 return 0;
2346 }
2347
2348 static int kvm_resume(struct sys_device *dev)
2349 {
2350 if (kvm_usage_count) {
2351 WARN_ON(spin_is_locked(&kvm_lock));
2352 hardware_enable(NULL);
2353 }
2354 return 0;
2355 }
2356
2357 static struct sysdev_class kvm_sysdev_class = {
2358 .name = "kvm",
2359 .suspend = kvm_suspend,
2360 .resume = kvm_resume,
2361 };
2362
2363 static struct sys_device kvm_sysdev = {
2364 .id = 0,
2365 .cls = &kvm_sysdev_class,
2366 };
2367
2368 struct page *bad_page;
2369 pfn_t bad_pfn;
2370
2371 static inline
2372 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
2373 {
2374 return container_of(pn, struct kvm_vcpu, preempt_notifier);
2375 }
2376
2377 static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
2378 {
2379 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
2380
2381 kvm_arch_vcpu_load(vcpu, cpu);
2382 }
2383
2384 static void kvm_sched_out(struct preempt_notifier *pn,
2385 struct task_struct *next)
2386 {
2387 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
2388
2389 kvm_arch_vcpu_put(vcpu);
2390 }
2391
2392 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
2393 struct module *module)
2394 {
2395 int r;
2396 int cpu;
2397
2398 r = kvm_arch_init(opaque);
2399 if (r)
2400 goto out_fail;
2401
2402 bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
2403
2404 if (bad_page == NULL) {
2405 r = -ENOMEM;
2406 goto out;
2407 }
2408
2409 bad_pfn = page_to_pfn(bad_page);
2410
2411 hwpoison_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
2412
2413 if (hwpoison_page == NULL) {
2414 r = -ENOMEM;
2415 goto out_free_0;
2416 }
2417
2418 hwpoison_pfn = page_to_pfn(hwpoison_page);
2419
2420 fault_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
2421
2422 if (fault_page == NULL) {
2423 r = -ENOMEM;
2424 goto out_free_0;
2425 }
2426
2427 fault_pfn = page_to_pfn(fault_page);
2428
2429 if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
2430 r = -ENOMEM;
2431 goto out_free_0;
2432 }
2433
2434 r = kvm_arch_hardware_setup();
2435 if (r < 0)
2436 goto out_free_0a;
2437
2438 for_each_online_cpu(cpu) {
2439 smp_call_function_single(cpu,
2440 kvm_arch_check_processor_compat,
2441 &r, 1);
2442 if (r < 0)
2443 goto out_free_1;
2444 }
2445
2446 r = register_cpu_notifier(&kvm_cpu_notifier);
2447 if (r)
2448 goto out_free_2;
2449 register_reboot_notifier(&kvm_reboot_notifier);
2450
2451 r = sysdev_class_register(&kvm_sysdev_class);
2452 if (r)
2453 goto out_free_3;
2454
2455 r = sysdev_register(&kvm_sysdev);
2456 if (r)
2457 goto out_free_4;
2458
2459 /* A kmem cache lets us meet the alignment requirements of fx_save. */
2460 if (!vcpu_align)
2461 vcpu_align = __alignof__(struct kvm_vcpu);
2462 kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align,
2463 0, NULL);
2464 if (!kvm_vcpu_cache) {
2465 r = -ENOMEM;
2466 goto out_free_5;
2467 }
2468
2469 r = kvm_async_pf_init();
2470 if (r)
2471 goto out_free;
2472
2473 kvm_chardev_ops.owner = module;
2474 kvm_vm_fops.owner = module;
2475 kvm_vcpu_fops.owner = module;
2476
2477 r = misc_register(&kvm_dev);
2478 if (r) {
2479 printk(KERN_ERR "kvm: misc device register failed\n");
2480 goto out_unreg;
2481 }
2482
2483 kvm_preempt_ops.sched_in = kvm_sched_in;
2484 kvm_preempt_ops.sched_out = kvm_sched_out;
2485
2486 kvm_init_debug();
2487
2488 return 0;
2489
2490 out_unreg:
2491 kvm_async_pf_deinit();
2492 out_free:
2493 kmem_cache_destroy(kvm_vcpu_cache);
2494 out_free_5:
2495 sysdev_unregister(&kvm_sysdev);
2496 out_free_4:
2497 sysdev_class_unregister(&kvm_sysdev_class);
2498 out_free_3:
2499 unregister_reboot_notifier(&kvm_reboot_notifier);
2500 unregister_cpu_notifier(&kvm_cpu_notifier);
2501 out_free_2:
2502 out_free_1:
2503 kvm_arch_hardware_unsetup();
2504 out_free_0a:
2505 free_cpumask_var(cpus_hardware_enabled);
2506 out_free_0:
2507 if (fault_page)
2508 __free_page(fault_page);
2509 if (hwpoison_page)
2510 __free_page(hwpoison_page);
2511 __free_page(bad_page);
2512 out:
2513 kvm_arch_exit();
2514 out_fail:
2515 return r;
2516 }
2517 EXPORT_SYMBOL_GPL(kvm_init);
2518
2519 void kvm_exit(void)
2520 {
2521 kvm_exit_debug();
2522 misc_deregister(&kvm_dev);
2523 kmem_cache_destroy(kvm_vcpu_cache);
2524 kvm_async_pf_deinit();
2525 sysdev_unregister(&kvm_sysdev);
2526 sysdev_class_unregister(&kvm_sysdev_class);
2527 unregister_reboot_notifier(&kvm_reboot_notifier);
2528 unregister_cpu_notifier(&kvm_cpu_notifier);
2529 on_each_cpu(hardware_disable, NULL, 1);
2530 kvm_arch_hardware_unsetup();
2531 kvm_arch_exit();
2532 free_cpumask_var(cpus_hardware_enabled);
2533 __free_page(hwpoison_page);
2534 __free_page(bad_page);
2535 }
2536 EXPORT_SYMBOL_GPL(kvm_exit);
ubuntu-bionic-kernel mirror