]>
git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blob - virt/kvm/async_pf.c
1 // SPDX-License-Identifier: GPL-2.0-only
3 * kvm asynchronous fault support
5 * Copyright 2010 Red Hat, Inc.
8 * Gleb Natapov <gleb@redhat.com>
11 #include <linux/kvm_host.h>
12 #include <linux/slab.h>
13 #include <linux/module.h>
14 #include <linux/mmu_context.h>
15 #include <linux/sched/mm.h>
18 #include <trace/events/kvm.h>
20 static struct kmem_cache
*async_pf_cache
;
22 int kvm_async_pf_init(void)
24 async_pf_cache
= KMEM_CACHE(kvm_async_pf
, 0);
32 void kvm_async_pf_deinit(void)
34 kmem_cache_destroy(async_pf_cache
);
35 async_pf_cache
= NULL
;
38 void kvm_async_pf_vcpu_init(struct kvm_vcpu
*vcpu
)
40 INIT_LIST_HEAD(&vcpu
->async_pf
.done
);
41 INIT_LIST_HEAD(&vcpu
->async_pf
.queue
);
42 spin_lock_init(&vcpu
->async_pf
.lock
);
45 static void async_pf_execute(struct work_struct
*work
)
47 struct kvm_async_pf
*apf
=
48 container_of(work
, struct kvm_async_pf
, work
);
49 struct mm_struct
*mm
= apf
->mm
;
50 struct kvm_vcpu
*vcpu
= apf
->vcpu
;
51 unsigned long addr
= apf
->addr
;
52 gpa_t cr2_or_gpa
= apf
->cr2_or_gpa
;
59 * This work is run asynchronously to the task which owns
60 * mm and might be done in another context, so we must
64 get_user_pages_remote(NULL
, mm
, addr
, 1, FOLL_WRITE
, NULL
, NULL
,
69 if (IS_ENABLED(CONFIG_KVM_ASYNC_PF_SYNC
))
70 kvm_arch_async_page_present(vcpu
, apf
);
72 spin_lock(&vcpu
->async_pf
.lock
);
73 first
= list_empty(&vcpu
->async_pf
.done
);
74 list_add_tail(&apf
->link
, &vcpu
->async_pf
.done
);
76 spin_unlock(&vcpu
->async_pf
.lock
);
78 if (!IS_ENABLED(CONFIG_KVM_ASYNC_PF_SYNC
) && first
)
79 kvm_arch_async_page_present_queued(vcpu
);
82 * apf may be freed by kvm_check_async_pf_completion() after
86 trace_kvm_async_pf_completed(addr
, cr2_or_gpa
);
88 rcuwait_wake_up(&vcpu
->wait
);
91 kvm_put_kvm(vcpu
->kvm
);
94 void kvm_clear_async_pf_completion_queue(struct kvm_vcpu
*vcpu
)
96 spin_lock(&vcpu
->async_pf
.lock
);
98 /* cancel outstanding work queue item */
99 while (!list_empty(&vcpu
->async_pf
.queue
)) {
100 struct kvm_async_pf
*work
=
101 list_first_entry(&vcpu
->async_pf
.queue
,
102 typeof(*work
), queue
);
103 list_del(&work
->queue
);
106 * We know it's present in vcpu->async_pf.done, do
112 spin_unlock(&vcpu
->async_pf
.lock
);
113 #ifdef CONFIG_KVM_ASYNC_PF_SYNC
114 flush_work(&work
->work
);
116 if (cancel_work_sync(&work
->work
)) {
118 kvm_put_kvm(vcpu
->kvm
); /* == work->vcpu->kvm */
119 kmem_cache_free(async_pf_cache
, work
);
122 spin_lock(&vcpu
->async_pf
.lock
);
125 while (!list_empty(&vcpu
->async_pf
.done
)) {
126 struct kvm_async_pf
*work
=
127 list_first_entry(&vcpu
->async_pf
.done
,
128 typeof(*work
), link
);
129 list_del(&work
->link
);
130 kmem_cache_free(async_pf_cache
, work
);
132 spin_unlock(&vcpu
->async_pf
.lock
);
134 vcpu
->async_pf
.queued
= 0;
137 void kvm_check_async_pf_completion(struct kvm_vcpu
*vcpu
)
139 struct kvm_async_pf
*work
;
141 while (!list_empty_careful(&vcpu
->async_pf
.done
) &&
142 kvm_arch_can_dequeue_async_page_present(vcpu
)) {
143 spin_lock(&vcpu
->async_pf
.lock
);
144 work
= list_first_entry(&vcpu
->async_pf
.done
, typeof(*work
),
146 list_del(&work
->link
);
147 spin_unlock(&vcpu
->async_pf
.lock
);
149 kvm_arch_async_page_ready(vcpu
, work
);
150 if (!IS_ENABLED(CONFIG_KVM_ASYNC_PF_SYNC
))
151 kvm_arch_async_page_present(vcpu
, work
);
153 list_del(&work
->queue
);
154 vcpu
->async_pf
.queued
--;
155 kmem_cache_free(async_pf_cache
, work
);
159 int kvm_setup_async_pf(struct kvm_vcpu
*vcpu
, gpa_t cr2_or_gpa
,
160 unsigned long hva
, struct kvm_arch_async_pf
*arch
)
162 struct kvm_async_pf
*work
;
164 if (vcpu
->async_pf
.queued
>= ASYNC_PF_PER_VCPU
)
167 /* Arch specific code should not do async PF in this case */
168 if (unlikely(kvm_is_error_hva(hva
)))
172 * do alloc nowait since if we are going to sleep anyway we
173 * may as well sleep faulting in page
175 work
= kmem_cache_zalloc(async_pf_cache
, GFP_NOWAIT
| __GFP_NOWARN
);
179 work
->wakeup_all
= false;
181 work
->cr2_or_gpa
= cr2_or_gpa
;
184 work
->mm
= current
->mm
;
186 kvm_get_kvm(work
->vcpu
->kvm
);
188 INIT_WORK(&work
->work
, async_pf_execute
);
190 list_add_tail(&work
->queue
, &vcpu
->async_pf
.queue
);
191 vcpu
->async_pf
.queued
++;
192 work
->notpresent_injected
= kvm_arch_async_page_not_present(vcpu
, work
);
194 schedule_work(&work
->work
);
199 int kvm_async_pf_wakeup_all(struct kvm_vcpu
*vcpu
)
201 struct kvm_async_pf
*work
;
204 if (!list_empty_careful(&vcpu
->async_pf
.done
))
207 work
= kmem_cache_zalloc(async_pf_cache
, GFP_ATOMIC
);
211 work
->wakeup_all
= true;
212 INIT_LIST_HEAD(&work
->queue
); /* for list_del to work */
214 spin_lock(&vcpu
->async_pf
.lock
);
215 first
= list_empty(&vcpu
->async_pf
.done
);
216 list_add_tail(&work
->link
, &vcpu
->async_pf
.done
);
217 spin_unlock(&vcpu
->async_pf
.lock
);
219 if (!IS_ENABLED(CONFIG_KVM_ASYNC_PF_SYNC
) && first
)
220 kvm_arch_async_page_present_queued(vcpu
);
222 vcpu
->async_pf
.queued
++;