1 // SPDX-License-Identifier: GPL-2.0
4 #include "mmu_internal.h"
10 #include <asm/cmpxchg.h>
11 #include <trace/events/kvm.h>
13 static bool __read_mostly tdp_mmu_enabled
= true;
14 module_param_named(tdp_mmu
, tdp_mmu_enabled
, bool, 0644);
16 /* Initializes the TDP MMU for the VM, if enabled. */
17 bool kvm_mmu_init_tdp_mmu(struct kvm
*kvm
)
19 if (!tdp_enabled
|| !READ_ONCE(tdp_mmu_enabled
))
22 /* This should not be changed for the lifetime of the VM. */
23 kvm
->arch
.tdp_mmu_enabled
= true;
25 INIT_LIST_HEAD(&kvm
->arch
.tdp_mmu_roots
);
26 spin_lock_init(&kvm
->arch
.tdp_mmu_pages_lock
);
27 INIT_LIST_HEAD(&kvm
->arch
.tdp_mmu_pages
);
32 static __always_inline
void kvm_lockdep_assert_mmu_lock_held(struct kvm
*kvm
,
36 lockdep_assert_held_read(&kvm
->mmu_lock
);
38 lockdep_assert_held_write(&kvm
->mmu_lock
);
41 void kvm_mmu_uninit_tdp_mmu(struct kvm
*kvm
)
43 if (!kvm
->arch
.tdp_mmu_enabled
)
46 WARN_ON(!list_empty(&kvm
->arch
.tdp_mmu_pages
));
47 WARN_ON(!list_empty(&kvm
->arch
.tdp_mmu_roots
));
50 * Ensure that all the outstanding RCU callbacks to free shadow pages
51 * can run before the VM is torn down.
56 static bool zap_gfn_range(struct kvm
*kvm
, struct kvm_mmu_page
*root
,
57 gfn_t start
, gfn_t end
, bool can_yield
, bool flush
,
60 static void tdp_mmu_free_sp(struct kvm_mmu_page
*sp
)
62 free_page((unsigned long)sp
->spt
);
63 kmem_cache_free(mmu_page_header_cache
, sp
);
67 * This is called through call_rcu in order to free TDP page table memory
68 * safely with respect to other kernel threads that may be operating on
70 * By only accessing TDP MMU page table memory in an RCU read critical
71 * section, and freeing it after a grace period, lockless access to that
72 * memory won't use it after it is freed.
74 static void tdp_mmu_free_sp_rcu_callback(struct rcu_head
*head
)
76 struct kvm_mmu_page
*sp
= container_of(head
, struct kvm_mmu_page
,
82 void kvm_tdp_mmu_put_root(struct kvm
*kvm
, struct kvm_mmu_page
*root
,
85 kvm_lockdep_assert_mmu_lock_held(kvm
, shared
);
87 if (!refcount_dec_and_test(&root
->tdp_mmu_root_count
))
90 WARN_ON(!root
->tdp_mmu_page
);
92 spin_lock(&kvm
->arch
.tdp_mmu_pages_lock
);
93 list_del_rcu(&root
->link
);
94 spin_unlock(&kvm
->arch
.tdp_mmu_pages_lock
);
96 zap_gfn_range(kvm
, root
, 0, -1ull, false, false, shared
);
98 call_rcu(&root
->rcu_head
, tdp_mmu_free_sp_rcu_callback
);
102 * Finds the next valid root after root (or the first valid root if root
103 * is NULL), takes a reference on it, and returns that next root. If root
104 * is not NULL, this thread should have already taken a reference on it, and
105 * that reference will be dropped. If no valid root is found, this
106 * function will return NULL.
108 static struct kvm_mmu_page
*tdp_mmu_next_root(struct kvm
*kvm
,
109 struct kvm_mmu_page
*prev_root
,
112 struct kvm_mmu_page
*next_root
;
117 next_root
= list_next_or_null_rcu(&kvm
->arch
.tdp_mmu_roots
,
119 typeof(*prev_root
), link
);
121 next_root
= list_first_or_null_rcu(&kvm
->arch
.tdp_mmu_roots
,
122 typeof(*next_root
), link
);
124 while (next_root
&& !kvm_tdp_mmu_get_root(kvm
, next_root
))
125 next_root
= list_next_or_null_rcu(&kvm
->arch
.tdp_mmu_roots
,
126 &next_root
->link
, typeof(*next_root
), link
);
131 kvm_tdp_mmu_put_root(kvm
, prev_root
, shared
);
137 * Note: this iterator gets and puts references to the roots it iterates over.
138 * This makes it safe to release the MMU lock and yield within the loop, but
139 * if exiting the loop early, the caller must drop the reference to the most
140 * recent root. (Unless keeping a live reference is desirable.)
142 * If shared is set, this function is operating under the MMU lock in read
143 * mode. In the unlikely event that this thread must free a root, the lock
144 * will be temporarily dropped and reacquired in write mode.
146 #define for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared) \
147 for (_root = tdp_mmu_next_root(_kvm, NULL, _shared); \
149 _root = tdp_mmu_next_root(_kvm, _root, _shared)) \
150 if (kvm_mmu_page_as_id(_root) != _as_id) { \
153 #define for_each_tdp_mmu_root(_kvm, _root, _as_id) \
154 list_for_each_entry_rcu(_root, &_kvm->arch.tdp_mmu_roots, link, \
155 lockdep_is_held_type(&kvm->mmu_lock, 0) || \
156 lockdep_is_held(&kvm->arch.tdp_mmu_pages_lock)) \
157 if (kvm_mmu_page_as_id(_root) != _as_id) { \
160 static union kvm_mmu_page_role
page_role_for_level(struct kvm_vcpu
*vcpu
,
163 union kvm_mmu_page_role role
;
165 role
= vcpu
->arch
.mmu
->mmu_role
.base
;
168 role
.gpte_is_8_bytes
= true;
169 role
.access
= ACC_ALL
;
174 static struct kvm_mmu_page
*alloc_tdp_mmu_page(struct kvm_vcpu
*vcpu
, gfn_t gfn
,
177 struct kvm_mmu_page
*sp
;
179 sp
= kvm_mmu_memory_cache_alloc(&vcpu
->arch
.mmu_page_header_cache
);
180 sp
->spt
= kvm_mmu_memory_cache_alloc(&vcpu
->arch
.mmu_shadow_page_cache
);
181 set_page_private(virt_to_page(sp
->spt
), (unsigned long)sp
);
183 sp
->role
.word
= page_role_for_level(vcpu
, level
).word
;
185 sp
->tdp_mmu_page
= true;
187 trace_kvm_mmu_get_page(sp
, true);
192 hpa_t
kvm_tdp_mmu_get_vcpu_root_hpa(struct kvm_vcpu
*vcpu
)
194 union kvm_mmu_page_role role
;
195 struct kvm
*kvm
= vcpu
->kvm
;
196 struct kvm_mmu_page
*root
;
198 lockdep_assert_held_write(&kvm
->mmu_lock
);
200 role
= page_role_for_level(vcpu
, vcpu
->arch
.mmu
->shadow_root_level
);
202 /* Check for an existing root before allocating a new one. */
203 for_each_tdp_mmu_root(kvm
, root
, kvm_mmu_role_as_id(role
)) {
204 if (root
->role
.word
== role
.word
&&
205 kvm_tdp_mmu_get_root(kvm
, root
))
209 root
= alloc_tdp_mmu_page(vcpu
, 0, vcpu
->arch
.mmu
->shadow_root_level
);
210 refcount_set(&root
->tdp_mmu_root_count
, 1);
212 spin_lock(&kvm
->arch
.tdp_mmu_pages_lock
);
213 list_add_rcu(&root
->link
, &kvm
->arch
.tdp_mmu_roots
);
214 spin_unlock(&kvm
->arch
.tdp_mmu_pages_lock
);
217 return __pa(root
->spt
);
220 static void handle_changed_spte(struct kvm
*kvm
, int as_id
, gfn_t gfn
,
221 u64 old_spte
, u64 new_spte
, int level
,
224 static void handle_changed_spte_acc_track(u64 old_spte
, u64 new_spte
, int level
)
226 if (!is_shadow_present_pte(old_spte
) || !is_last_spte(old_spte
, level
))
229 if (is_accessed_spte(old_spte
) &&
230 (!is_shadow_present_pte(new_spte
) || !is_accessed_spte(new_spte
) ||
231 spte_to_pfn(old_spte
) != spte_to_pfn(new_spte
)))
232 kvm_set_pfn_accessed(spte_to_pfn(old_spte
));
235 static void handle_changed_spte_dirty_log(struct kvm
*kvm
, int as_id
, gfn_t gfn
,
236 u64 old_spte
, u64 new_spte
, int level
)
239 struct kvm_memory_slot
*slot
;
241 if (level
> PG_LEVEL_4K
)
244 pfn_changed
= spte_to_pfn(old_spte
) != spte_to_pfn(new_spte
);
246 if ((!is_writable_pte(old_spte
) || pfn_changed
) &&
247 is_writable_pte(new_spte
)) {
248 slot
= __gfn_to_memslot(__kvm_memslots(kvm
, as_id
), gfn
);
249 mark_page_dirty_in_slot(kvm
, slot
, gfn
);
254 * tdp_mmu_link_page - Add a new page to the list of pages used by the TDP MMU
258 * @account_nx: This page replaces a NX large page and should be marked for
261 static void tdp_mmu_link_page(struct kvm
*kvm
, struct kvm_mmu_page
*sp
,
264 spin_lock(&kvm
->arch
.tdp_mmu_pages_lock
);
265 list_add(&sp
->link
, &kvm
->arch
.tdp_mmu_pages
);
267 account_huge_nx_page(kvm
, sp
);
268 spin_unlock(&kvm
->arch
.tdp_mmu_pages_lock
);
272 * tdp_mmu_unlink_page - Remove page from the list of pages used by the TDP MMU
275 * @sp: the page to be removed
276 * @shared: This operation may not be running under the exclusive use of
277 * the MMU lock and the operation must synchronize with other
278 * threads that might be adding or removing pages.
280 static void tdp_mmu_unlink_page(struct kvm
*kvm
, struct kvm_mmu_page
*sp
,
284 spin_lock(&kvm
->arch
.tdp_mmu_pages_lock
);
286 lockdep_assert_held_write(&kvm
->mmu_lock
);
289 if (sp
->lpage_disallowed
)
290 unaccount_huge_nx_page(kvm
, sp
);
293 spin_unlock(&kvm
->arch
.tdp_mmu_pages_lock
);
297 * handle_removed_tdp_mmu_page - handle a pt removed from the TDP structure
300 * @pt: the page removed from the paging structure
301 * @shared: This operation may not be running under the exclusive use
302 * of the MMU lock and the operation must synchronize with other
303 * threads that might be modifying SPTEs.
305 * Given a page table that has been removed from the TDP paging structure,
306 * iterates through the page table to clear SPTEs and free child page tables.
308 * Note that pt is passed in as a tdp_ptep_t, but it does not need RCU
309 * protection. Since this thread removed it from the paging structure,
310 * this thread will be responsible for ensuring the page is freed. Hence the
311 * early rcu_dereferences in the function.
313 static void handle_removed_tdp_mmu_page(struct kvm
*kvm
, tdp_ptep_t pt
,
316 struct kvm_mmu_page
*sp
= sptep_to_sp(rcu_dereference(pt
));
317 int level
= sp
->role
.level
;
318 gfn_t base_gfn
= sp
->gfn
;
321 trace_kvm_mmu_prepare_zap_page(sp
);
323 tdp_mmu_unlink_page(kvm
, sp
, shared
);
325 for (i
= 0; i
< PT64_ENT_PER_PAGE
; i
++) {
326 u64
*sptep
= rcu_dereference(pt
) + i
;
327 gfn_t gfn
= base_gfn
+ i
* KVM_PAGES_PER_HPAGE(level
);
332 * Set the SPTE to a nonpresent value that other
333 * threads will not overwrite. If the SPTE was
334 * already marked as removed then another thread
335 * handling a page fault could overwrite it, so
336 * set the SPTE until it is set from some other
337 * value to the removed SPTE value.
340 old_child_spte
= xchg(sptep
, REMOVED_SPTE
);
341 if (!is_removed_spte(old_child_spte
))
347 * If the SPTE is not MMU-present, there is no backing
348 * page associated with the SPTE and so no side effects
349 * that need to be recorded, and exclusive ownership of
350 * mmu_lock ensures the SPTE can't be made present.
351 * Note, zapping MMIO SPTEs is also unnecessary as they
352 * are guarded by the memslots generation, not by being
355 old_child_spte
= READ_ONCE(*sptep
);
356 if (!is_shadow_present_pte(old_child_spte
))
360 * Marking the SPTE as a removed SPTE is not
361 * strictly necessary here as the MMU lock will
362 * stop other threads from concurrently modifying
363 * this SPTE. Using the removed SPTE value keeps
364 * the two branches consistent and simplifies
367 WRITE_ONCE(*sptep
, REMOVED_SPTE
);
369 handle_changed_spte(kvm
, kvm_mmu_page_as_id(sp
), gfn
,
370 old_child_spte
, REMOVED_SPTE
, level
,
374 kvm_flush_remote_tlbs_with_address(kvm
, base_gfn
,
375 KVM_PAGES_PER_HPAGE(level
+ 1));
377 call_rcu(&sp
->rcu_head
, tdp_mmu_free_sp_rcu_callback
);
381 * __handle_changed_spte - handle bookkeeping associated with an SPTE change
383 * @as_id: the address space of the paging structure the SPTE was a part of
384 * @gfn: the base GFN that was mapped by the SPTE
385 * @old_spte: The value of the SPTE before the change
386 * @new_spte: The value of the SPTE after the change
387 * @level: the level of the PT the SPTE is part of in the paging structure
388 * @shared: This operation may not be running under the exclusive use of
389 * the MMU lock and the operation must synchronize with other
390 * threads that might be modifying SPTEs.
392 * Handle bookkeeping that might result from the modification of a SPTE.
393 * This function must be called for all TDP SPTE modifications.
395 static void __handle_changed_spte(struct kvm
*kvm
, int as_id
, gfn_t gfn
,
396 u64 old_spte
, u64 new_spte
, int level
,
399 bool was_present
= is_shadow_present_pte(old_spte
);
400 bool is_present
= is_shadow_present_pte(new_spte
);
401 bool was_leaf
= was_present
&& is_last_spte(old_spte
, level
);
402 bool is_leaf
= is_present
&& is_last_spte(new_spte
, level
);
403 bool pfn_changed
= spte_to_pfn(old_spte
) != spte_to_pfn(new_spte
);
405 WARN_ON(level
> PT64_ROOT_MAX_LEVEL
);
406 WARN_ON(level
< PG_LEVEL_4K
);
407 WARN_ON(gfn
& (KVM_PAGES_PER_HPAGE(level
) - 1));
410 * If this warning were to trigger it would indicate that there was a
411 * missing MMU notifier or a race with some notifier handler.
412 * A present, leaf SPTE should never be directly replaced with another
413 * present leaf SPTE pointing to a different PFN. A notifier handler
414 * should be zapping the SPTE before the main MM's page table is
415 * changed, or the SPTE should be zeroed, and the TLBs flushed by the
416 * thread before replacement.
418 if (was_leaf
&& is_leaf
&& pfn_changed
) {
419 pr_err("Invalid SPTE change: cannot replace a present leaf\n"
420 "SPTE with another present leaf SPTE mapping a\n"
422 "as_id: %d gfn: %llx old_spte: %llx new_spte: %llx level: %d",
423 as_id
, gfn
, old_spte
, new_spte
, level
);
426 * Crash the host to prevent error propagation and guest data
432 if (old_spte
== new_spte
)
435 trace_kvm_tdp_mmu_spte_changed(as_id
, gfn
, level
, old_spte
, new_spte
);
438 * The only times a SPTE should be changed from a non-present to
439 * non-present state is when an MMIO entry is installed/modified/
440 * removed. In that case, there is nothing to do here.
442 if (!was_present
&& !is_present
) {
444 * If this change does not involve a MMIO SPTE or removed SPTE,
445 * it is unexpected. Log the change, though it should not
446 * impact the guest since both the former and current SPTEs
449 if (WARN_ON(!is_mmio_spte(old_spte
) &&
450 !is_mmio_spte(new_spte
) &&
451 !is_removed_spte(new_spte
)))
452 pr_err("Unexpected SPTE change! Nonpresent SPTEs\n"
453 "should not be replaced with another,\n"
454 "different nonpresent SPTE, unless one or both\n"
455 "are MMIO SPTEs, or the new SPTE is\n"
456 "a temporary removed SPTE.\n"
457 "as_id: %d gfn: %llx old_spte: %llx new_spte: %llx level: %d",
458 as_id
, gfn
, old_spte
, new_spte
, level
);
462 if (is_leaf
!= was_leaf
)
463 kvm_update_page_stats(kvm
, level
, is_leaf
? 1 : -1);
465 if (was_leaf
&& is_dirty_spte(old_spte
) &&
466 (!is_present
|| !is_dirty_spte(new_spte
) || pfn_changed
))
467 kvm_set_pfn_dirty(spte_to_pfn(old_spte
));
470 * Recursively handle child PTs if the change removed a subtree from
471 * the paging structure.
473 if (was_present
&& !was_leaf
&& (pfn_changed
|| !is_present
))
474 handle_removed_tdp_mmu_page(kvm
,
475 spte_to_child_pt(old_spte
, level
), shared
);
478 static void handle_changed_spte(struct kvm
*kvm
, int as_id
, gfn_t gfn
,
479 u64 old_spte
, u64 new_spte
, int level
,
482 __handle_changed_spte(kvm
, as_id
, gfn
, old_spte
, new_spte
, level
,
484 handle_changed_spte_acc_track(old_spte
, new_spte
, level
);
485 handle_changed_spte_dirty_log(kvm
, as_id
, gfn
, old_spte
,
490 * tdp_mmu_set_spte_atomic_no_dirty_log - Set a TDP MMU SPTE atomically
491 * and handle the associated bookkeeping, but do not mark the page dirty
492 * in KVM's dirty bitmaps.
495 * @iter: a tdp_iter instance currently on the SPTE that should be set
496 * @new_spte: The value the SPTE should be set to
497 * Returns: true if the SPTE was set, false if it was not. If false is returned,
498 * this function will have no side-effects.
500 static inline bool tdp_mmu_set_spte_atomic_no_dirty_log(struct kvm
*kvm
,
501 struct tdp_iter
*iter
,
504 lockdep_assert_held_read(&kvm
->mmu_lock
);
507 * Do not change removed SPTEs. Only the thread that froze the SPTE
510 if (is_removed_spte(iter
->old_spte
))
514 * Note, fast_pf_fix_direct_spte() can also modify TDP MMU SPTEs and
515 * does not hold the mmu_lock.
517 if (cmpxchg64(rcu_dereference(iter
->sptep
), iter
->old_spte
,
518 new_spte
) != iter
->old_spte
)
521 __handle_changed_spte(kvm
, iter
->as_id
, iter
->gfn
, iter
->old_spte
,
522 new_spte
, iter
->level
, true);
523 handle_changed_spte_acc_track(iter
->old_spte
, new_spte
, iter
->level
);
529 * tdp_mmu_map_set_spte_atomic - Set a leaf TDP MMU SPTE atomically to resolve a
532 * @vcpu: The vcpu instance that took the TDP page fault.
533 * @iter: a tdp_iter instance currently on the SPTE that should be set
534 * @new_spte: The value the SPTE should be set to
536 * Returns: true if the SPTE was set, false if it was not. If false is returned,
537 * this function will have no side-effects.
539 static inline bool tdp_mmu_map_set_spte_atomic(struct kvm_vcpu
*vcpu
,
540 struct tdp_iter
*iter
,
543 struct kvm
*kvm
= vcpu
->kvm
;
545 if (!tdp_mmu_set_spte_atomic_no_dirty_log(kvm
, iter
, new_spte
))
549 * Use kvm_vcpu_gfn_to_memslot() instead of going through
550 * handle_changed_spte_dirty_log() to leverage vcpu->last_used_slot.
552 if (is_writable_pte(new_spte
)) {
553 struct kvm_memory_slot
*slot
= kvm_vcpu_gfn_to_memslot(vcpu
, iter
->gfn
);
555 if (slot
&& kvm_slot_dirty_track_enabled(slot
)) {
556 /* Enforced by kvm_mmu_hugepage_adjust. */
557 WARN_ON_ONCE(iter
->level
> PG_LEVEL_4K
);
558 mark_page_dirty_in_slot(kvm
, slot
, iter
->gfn
);
565 static inline bool tdp_mmu_zap_spte_atomic(struct kvm
*kvm
,
566 struct tdp_iter
*iter
)
569 * Freeze the SPTE by setting it to a special,
570 * non-present value. This will stop other threads from
571 * immediately installing a present entry in its place
572 * before the TLBs are flushed.
574 if (!tdp_mmu_set_spte_atomic_no_dirty_log(kvm
, iter
, REMOVED_SPTE
))
577 kvm_flush_remote_tlbs_with_address(kvm
, iter
->gfn
,
578 KVM_PAGES_PER_HPAGE(iter
->level
));
581 * No other thread can overwrite the removed SPTE as they
582 * must either wait on the MMU lock or use
583 * tdp_mmu_set_spte_atomic which will not overwrite the
584 * special removed SPTE value. No bookkeeping is needed
585 * here since the SPTE is going from non-present
588 WRITE_ONCE(*rcu_dereference(iter
->sptep
), 0);
595 * __tdp_mmu_set_spte - Set a TDP MMU SPTE and handle the associated bookkeeping
597 * @iter: a tdp_iter instance currently on the SPTE that should be set
598 * @new_spte: The value the SPTE should be set to
599 * @record_acc_track: Notify the MM subsystem of changes to the accessed state
600 * of the page. Should be set unless handling an MMU
601 * notifier for access tracking. Leaving record_acc_track
602 * unset in that case prevents page accesses from being
604 * @record_dirty_log: Record the page as dirty in the dirty bitmap if
605 * appropriate for the change being made. Should be set
606 * unless performing certain dirty logging operations.
607 * Leaving record_dirty_log unset in that case prevents page
608 * writes from being double counted.
610 static inline void __tdp_mmu_set_spte(struct kvm
*kvm
, struct tdp_iter
*iter
,
611 u64 new_spte
, bool record_acc_track
,
612 bool record_dirty_log
)
614 lockdep_assert_held_write(&kvm
->mmu_lock
);
617 * No thread should be using this function to set SPTEs to the
618 * temporary removed SPTE value.
619 * If operating under the MMU lock in read mode, tdp_mmu_set_spte_atomic
620 * should be used. If operating under the MMU lock in write mode, the
621 * use of the removed SPTE should not be necessary.
623 WARN_ON(is_removed_spte(iter
->old_spte
));
625 WRITE_ONCE(*rcu_dereference(iter
->sptep
), new_spte
);
627 __handle_changed_spte(kvm
, iter
->as_id
, iter
->gfn
, iter
->old_spte
,
628 new_spte
, iter
->level
, false);
629 if (record_acc_track
)
630 handle_changed_spte_acc_track(iter
->old_spte
, new_spte
,
632 if (record_dirty_log
)
633 handle_changed_spte_dirty_log(kvm
, iter
->as_id
, iter
->gfn
,
634 iter
->old_spte
, new_spte
,
638 static inline void tdp_mmu_set_spte(struct kvm
*kvm
, struct tdp_iter
*iter
,
641 __tdp_mmu_set_spte(kvm
, iter
, new_spte
, true, true);
644 static inline void tdp_mmu_set_spte_no_acc_track(struct kvm
*kvm
,
645 struct tdp_iter
*iter
,
648 __tdp_mmu_set_spte(kvm
, iter
, new_spte
, false, true);
651 static inline void tdp_mmu_set_spte_no_dirty_log(struct kvm
*kvm
,
652 struct tdp_iter
*iter
,
655 __tdp_mmu_set_spte(kvm
, iter
, new_spte
, true, false);
658 #define tdp_root_for_each_pte(_iter, _root, _start, _end) \
659 for_each_tdp_pte(_iter, _root->spt, _root->role.level, _start, _end)
661 #define tdp_root_for_each_leaf_pte(_iter, _root, _start, _end) \
662 tdp_root_for_each_pte(_iter, _root, _start, _end) \
663 if (!is_shadow_present_pte(_iter.old_spte) || \
664 !is_last_spte(_iter.old_spte, _iter.level)) \
668 #define tdp_mmu_for_each_pte(_iter, _mmu, _start, _end) \
669 for_each_tdp_pte(_iter, __va(_mmu->root_hpa), \
670 _mmu->shadow_root_level, _start, _end)
673 * Yield if the MMU lock is contended or this thread needs to return control
676 * If this function should yield and flush is set, it will perform a remote
677 * TLB flush before yielding.
679 * If this function yields, it will also reset the tdp_iter's walk over the
680 * paging structure and the calling function should skip to the next
681 * iteration to allow the iterator to continue its traversal from the
682 * paging structure root.
684 * Return true if this function yielded and the iterator's traversal was reset.
685 * Return false if a yield was not needed.
687 static inline bool tdp_mmu_iter_cond_resched(struct kvm
*kvm
,
688 struct tdp_iter
*iter
, bool flush
,
691 /* Ensure forward progress has been made before yielding. */
692 if (iter
->next_last_level_gfn
== iter
->yielded_gfn
)
695 if (need_resched() || rwlock_needbreak(&kvm
->mmu_lock
)) {
699 kvm_flush_remote_tlbs(kvm
);
702 cond_resched_rwlock_read(&kvm
->mmu_lock
);
704 cond_resched_rwlock_write(&kvm
->mmu_lock
);
708 WARN_ON(iter
->gfn
> iter
->next_last_level_gfn
);
710 tdp_iter_restart(iter
);
719 * Tears down the mappings for the range of gfns, [start, end), and frees the
720 * non-root pages mapping GFNs strictly within that range. Returns true if
721 * SPTEs have been cleared and a TLB flush is needed before releasing the
724 * If can_yield is true, will release the MMU lock and reschedule if the
725 * scheduler needs the CPU or there is contention on the MMU lock. If this
726 * function cannot yield, it will not release the MMU lock or reschedule and
727 * the caller must ensure it does not supply too large a GFN range, or the
728 * operation can cause a soft lockup.
730 * If shared is true, this thread holds the MMU lock in read mode and must
731 * account for the possibility that other threads are modifying the paging
732 * structures concurrently. If shared is false, this thread should hold the
733 * MMU lock in write mode.
735 static bool zap_gfn_range(struct kvm
*kvm
, struct kvm_mmu_page
*root
,
736 gfn_t start
, gfn_t end
, bool can_yield
, bool flush
,
739 gfn_t max_gfn_host
= 1ULL << (shadow_phys_bits
- PAGE_SHIFT
);
740 bool zap_all
= (start
== 0 && end
>= max_gfn_host
);
741 struct tdp_iter iter
;
744 * No need to try to step down in the iterator when zapping all SPTEs,
745 * zapping the top-level non-leaf SPTEs will recurse on their children.
747 int min_level
= zap_all
? root
->role
.level
: PG_LEVEL_4K
;
750 * Bound the walk at host.MAXPHYADDR, guest accesses beyond that will
751 * hit a #PF(RSVD) and never get to an EPT Violation/Misconfig / #NPF,
752 * and so KVM will never install a SPTE for such addresses.
754 end
= min(end
, max_gfn_host
);
756 kvm_lockdep_assert_mmu_lock_held(kvm
, shared
);
760 for_each_tdp_pte_min_level(iter
, root
->spt
, root
->role
.level
,
761 min_level
, start
, end
) {
764 tdp_mmu_iter_cond_resched(kvm
, &iter
, flush
, shared
)) {
769 if (!is_shadow_present_pte(iter
.old_spte
))
773 * If this is a non-last-level SPTE that covers a larger range
774 * than should be zapped, continue, and zap the mappings at a
775 * lower level, except when zapping all SPTEs.
779 iter
.gfn
+ KVM_PAGES_PER_HPAGE(iter
.level
) > end
) &&
780 !is_last_spte(iter
.old_spte
, iter
.level
))
784 tdp_mmu_set_spte(kvm
, &iter
, 0);
786 } else if (!tdp_mmu_zap_spte_atomic(kvm
, &iter
)) {
788 * The iter must explicitly re-read the SPTE because
789 * the atomic cmpxchg failed.
791 iter
.old_spte
= READ_ONCE(*rcu_dereference(iter
.sptep
));
801 * Tears down the mappings for the range of gfns, [start, end), and frees the
802 * non-root pages mapping GFNs strictly within that range. Returns true if
803 * SPTEs have been cleared and a TLB flush is needed before releasing the
806 bool __kvm_tdp_mmu_zap_gfn_range(struct kvm
*kvm
, int as_id
, gfn_t start
,
807 gfn_t end
, bool can_yield
, bool flush
)
809 struct kvm_mmu_page
*root
;
811 for_each_tdp_mmu_root_yield_safe(kvm
, root
, as_id
, false)
812 flush
= zap_gfn_range(kvm
, root
, start
, end
, can_yield
, flush
,
818 void kvm_tdp_mmu_zap_all(struct kvm
*kvm
)
823 for (i
= 0; i
< KVM_ADDRESS_SPACE_NUM
; i
++)
824 flush
= kvm_tdp_mmu_zap_gfn_range(kvm
, i
, 0, -1ull, flush
);
827 kvm_flush_remote_tlbs(kvm
);
830 static struct kvm_mmu_page
*next_invalidated_root(struct kvm
*kvm
,
831 struct kvm_mmu_page
*prev_root
)
833 struct kvm_mmu_page
*next_root
;
836 next_root
= list_next_or_null_rcu(&kvm
->arch
.tdp_mmu_roots
,
838 typeof(*prev_root
), link
);
840 next_root
= list_first_or_null_rcu(&kvm
->arch
.tdp_mmu_roots
,
841 typeof(*next_root
), link
);
843 while (next_root
&& !(next_root
->role
.invalid
&&
844 refcount_read(&next_root
->tdp_mmu_root_count
)))
845 next_root
= list_next_or_null_rcu(&kvm
->arch
.tdp_mmu_roots
,
847 typeof(*next_root
), link
);
853 * Since kvm_tdp_mmu_zap_all_fast has acquired a reference to each
854 * invalidated root, they will not be freed until this function drops the
855 * reference. Before dropping that reference, tear down the paging
856 * structure so that whichever thread does drop the last reference
857 * only has to do a trivial amount of work. Since the roots are invalid,
858 * no new SPTEs should be created under them.
860 void kvm_tdp_mmu_zap_invalidated_roots(struct kvm
*kvm
)
862 struct kvm_mmu_page
*next_root
;
863 struct kvm_mmu_page
*root
;
866 lockdep_assert_held_read(&kvm
->mmu_lock
);
870 root
= next_invalidated_root(kvm
, NULL
);
873 next_root
= next_invalidated_root(kvm
, root
);
877 flush
= zap_gfn_range(kvm
, root
, 0, -1ull, true, flush
, true);
880 * Put the reference acquired in
881 * kvm_tdp_mmu_invalidate_roots
883 kvm_tdp_mmu_put_root(kvm
, root
, true);
893 kvm_flush_remote_tlbs(kvm
);
897 * Mark each TDP MMU root as invalid so that other threads
898 * will drop their references and allow the root count to
901 * Also take a reference on all roots so that this thread
902 * can do the bulk of the work required to free the roots
903 * once they are invalidated. Without this reference, a
904 * vCPU thread might drop the last reference to a root and
905 * get stuck with tearing down the entire paging structure.
907 * Roots which have a zero refcount should be skipped as
908 * they're already being torn down.
909 * Already invalid roots should be referenced again so that
910 * they aren't freed before kvm_tdp_mmu_zap_all_fast is
913 * This has essentially the same effect for the TDP MMU
914 * as updating mmu_valid_gen does for the shadow MMU.
916 void kvm_tdp_mmu_invalidate_all_roots(struct kvm
*kvm
)
918 struct kvm_mmu_page
*root
;
920 lockdep_assert_held_write(&kvm
->mmu_lock
);
921 list_for_each_entry(root
, &kvm
->arch
.tdp_mmu_roots
, link
)
922 if (refcount_inc_not_zero(&root
->tdp_mmu_root_count
))
923 root
->role
.invalid
= true;
927 * Installs a last-level SPTE to handle a TDP page fault.
928 * (NPT/EPT violation/misconfiguration)
930 static int tdp_mmu_map_handle_target_level(struct kvm_vcpu
*vcpu
, int write
,
932 struct tdp_iter
*iter
,
933 kvm_pfn_t pfn
, bool prefault
)
936 int ret
= RET_PF_FIXED
;
937 int make_spte_ret
= 0;
939 if (unlikely(is_noslot_pfn(pfn
)))
940 new_spte
= make_mmio_spte(vcpu
, iter
->gfn
, ACC_ALL
);
942 make_spte_ret
= make_spte(vcpu
, ACC_ALL
, iter
->level
, iter
->gfn
,
943 pfn
, iter
->old_spte
, prefault
, true,
944 map_writable
, !shadow_accessed_mask
,
947 if (new_spte
== iter
->old_spte
)
948 ret
= RET_PF_SPURIOUS
;
949 else if (!tdp_mmu_map_set_spte_atomic(vcpu
, iter
, new_spte
))
953 * If the page fault was caused by a write but the page is write
954 * protected, emulation is needed. If the emulation was skipped,
955 * the vCPU would have the same fault again.
957 if (make_spte_ret
& SET_SPTE_WRITE_PROTECTED_PT
) {
959 ret
= RET_PF_EMULATE
;
960 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT
, vcpu
);
963 /* If a MMIO SPTE is installed, the MMIO will need to be emulated. */
964 if (unlikely(is_mmio_spte(new_spte
))) {
965 trace_mark_mmio_spte(rcu_dereference(iter
->sptep
), iter
->gfn
,
967 ret
= RET_PF_EMULATE
;
969 trace_kvm_mmu_set_spte(iter
->level
, iter
->gfn
,
970 rcu_dereference(iter
->sptep
));
974 * Increase pf_fixed in both RET_PF_EMULATE and RET_PF_FIXED to be
975 * consistent with legacy MMU behavior.
977 if (ret
!= RET_PF_SPURIOUS
)
978 vcpu
->stat
.pf_fixed
++;
984 * Handle a TDP page fault (NPT/EPT violation/misconfiguration) by installing
985 * page tables and SPTEs to translate the faulting guest physical address.
987 int kvm_tdp_mmu_map(struct kvm_vcpu
*vcpu
, gpa_t gpa
, u32 error_code
,
988 int map_writable
, int max_level
, kvm_pfn_t pfn
,
991 bool nx_huge_page_workaround_enabled
= is_nx_huge_page_enabled();
992 bool write
= error_code
& PFERR_WRITE_MASK
;
993 bool exec
= error_code
& PFERR_FETCH_MASK
;
994 bool huge_page_disallowed
= exec
&& nx_huge_page_workaround_enabled
;
995 struct kvm_mmu
*mmu
= vcpu
->arch
.mmu
;
996 struct tdp_iter iter
;
997 struct kvm_mmu_page
*sp
;
1001 gfn_t gfn
= gpa
>> PAGE_SHIFT
;
1005 level
= kvm_mmu_hugepage_adjust(vcpu
, gfn
, max_level
, &pfn
,
1006 huge_page_disallowed
, &req_level
);
1008 trace_kvm_mmu_spte_requested(gpa
, level
, pfn
);
1012 tdp_mmu_for_each_pte(iter
, mmu
, gfn
, gfn
+ 1) {
1013 if (nx_huge_page_workaround_enabled
)
1014 disallowed_hugepage_adjust(iter
.old_spte
, gfn
,
1015 iter
.level
, &pfn
, &level
);
1017 if (iter
.level
== level
)
1021 * If there is an SPTE mapping a large page at a higher level
1022 * than the target, that SPTE must be cleared and replaced
1023 * with a non-leaf SPTE.
1025 if (is_shadow_present_pte(iter
.old_spte
) &&
1026 is_large_pte(iter
.old_spte
)) {
1027 if (!tdp_mmu_zap_spte_atomic(vcpu
->kvm
, &iter
))
1031 * The iter must explicitly re-read the spte here
1032 * because the new value informs the !present
1035 iter
.old_spte
= READ_ONCE(*rcu_dereference(iter
.sptep
));
1038 if (!is_shadow_present_pte(iter
.old_spte
)) {
1040 * If SPTE has been frozen by another thread, just
1041 * give up and retry, avoiding unnecessary page table
1042 * allocation and free.
1044 if (is_removed_spte(iter
.old_spte
))
1047 sp
= alloc_tdp_mmu_page(vcpu
, iter
.gfn
, iter
.level
- 1);
1050 new_spte
= make_nonleaf_spte(child_pt
,
1051 !shadow_accessed_mask
);
1053 if (tdp_mmu_set_spte_atomic_no_dirty_log(vcpu
->kvm
, &iter
, new_spte
)) {
1054 tdp_mmu_link_page(vcpu
->kvm
, sp
,
1055 huge_page_disallowed
&&
1056 req_level
>= iter
.level
);
1058 trace_kvm_mmu_get_page(sp
, true);
1060 tdp_mmu_free_sp(sp
);
1066 if (iter
.level
!= level
) {
1068 return RET_PF_RETRY
;
1071 ret
= tdp_mmu_map_handle_target_level(vcpu
, write
, map_writable
, &iter
,
1078 bool kvm_tdp_mmu_unmap_gfn_range(struct kvm
*kvm
, struct kvm_gfn_range
*range
,
1081 struct kvm_mmu_page
*root
;
1083 for_each_tdp_mmu_root(kvm
, root
, range
->slot
->as_id
)
1084 flush
= zap_gfn_range(kvm
, root
, range
->start
, range
->end
,
1085 range
->may_block
, flush
, false);
1090 typedef bool (*tdp_handler_t
)(struct kvm
*kvm
, struct tdp_iter
*iter
,
1091 struct kvm_gfn_range
*range
);
1093 static __always_inline
bool kvm_tdp_mmu_handle_gfn(struct kvm
*kvm
,
1094 struct kvm_gfn_range
*range
,
1095 tdp_handler_t handler
)
1097 struct kvm_mmu_page
*root
;
1098 struct tdp_iter iter
;
1104 * Don't support rescheduling, none of the MMU notifiers that funnel
1105 * into this helper allow blocking; it'd be dead, wasteful code.
1107 for_each_tdp_mmu_root(kvm
, root
, range
->slot
->as_id
) {
1108 tdp_root_for_each_leaf_pte(iter
, root
, range
->start
, range
->end
)
1109 ret
|= handler(kvm
, &iter
, range
);
1118 * Mark the SPTEs range of GFNs [start, end) unaccessed and return non-zero
1119 * if any of the GFNs in the range have been accessed.
1121 static bool age_gfn_range(struct kvm
*kvm
, struct tdp_iter
*iter
,
1122 struct kvm_gfn_range
*range
)
1126 /* If we have a non-accessed entry we don't need to change the pte. */
1127 if (!is_accessed_spte(iter
->old_spte
))
1130 new_spte
= iter
->old_spte
;
1132 if (spte_ad_enabled(new_spte
)) {
1133 new_spte
&= ~shadow_accessed_mask
;
1136 * Capture the dirty status of the page, so that it doesn't get
1137 * lost when the SPTE is marked for access tracking.
1139 if (is_writable_pte(new_spte
))
1140 kvm_set_pfn_dirty(spte_to_pfn(new_spte
));
1142 new_spte
= mark_spte_for_access_track(new_spte
);
1145 tdp_mmu_set_spte_no_acc_track(kvm
, iter
, new_spte
);
1150 bool kvm_tdp_mmu_age_gfn_range(struct kvm
*kvm
, struct kvm_gfn_range
*range
)
1152 return kvm_tdp_mmu_handle_gfn(kvm
, range
, age_gfn_range
);
1155 static bool test_age_gfn(struct kvm
*kvm
, struct tdp_iter
*iter
,
1156 struct kvm_gfn_range
*range
)
1158 return is_accessed_spte(iter
->old_spte
);
1161 bool kvm_tdp_mmu_test_age_gfn(struct kvm
*kvm
, struct kvm_gfn_range
*range
)
1163 return kvm_tdp_mmu_handle_gfn(kvm
, range
, test_age_gfn
);
1166 static bool set_spte_gfn(struct kvm
*kvm
, struct tdp_iter
*iter
,
1167 struct kvm_gfn_range
*range
)
1171 /* Huge pages aren't expected to be modified without first being zapped. */
1172 WARN_ON(pte_huge(range
->pte
) || range
->start
+ 1 != range
->end
);
1174 if (iter
->level
!= PG_LEVEL_4K
||
1175 !is_shadow_present_pte(iter
->old_spte
))
1179 * Note, when changing a read-only SPTE, it's not strictly necessary to
1180 * zero the SPTE before setting the new PFN, but doing so preserves the
1181 * invariant that the PFN of a present * leaf SPTE can never change.
1182 * See __handle_changed_spte().
1184 tdp_mmu_set_spte(kvm
, iter
, 0);
1186 if (!pte_write(range
->pte
)) {
1187 new_spte
= kvm_mmu_changed_pte_notifier_make_spte(iter
->old_spte
,
1188 pte_pfn(range
->pte
));
1190 tdp_mmu_set_spte(kvm
, iter
, new_spte
);
1197 * Handle the changed_pte MMU notifier for the TDP MMU.
1198 * data is a pointer to the new pte_t mapping the HVA specified by the MMU
1200 * Returns non-zero if a flush is needed before releasing the MMU lock.
1202 bool kvm_tdp_mmu_set_spte_gfn(struct kvm
*kvm
, struct kvm_gfn_range
*range
)
1204 bool flush
= kvm_tdp_mmu_handle_gfn(kvm
, range
, set_spte_gfn
);
1206 /* FIXME: return 'flush' instead of flushing here. */
1208 kvm_flush_remote_tlbs_with_address(kvm
, range
->start
, 1);
1214 * Remove write access from all SPTEs at or above min_level that map GFNs
1215 * [start, end). Returns true if an SPTE has been changed and the TLBs need to
1218 static bool wrprot_gfn_range(struct kvm
*kvm
, struct kvm_mmu_page
*root
,
1219 gfn_t start
, gfn_t end
, int min_level
)
1221 struct tdp_iter iter
;
1223 bool spte_set
= false;
1227 BUG_ON(min_level
> KVM_MAX_HUGEPAGE_LEVEL
);
1229 for_each_tdp_pte_min_level(iter
, root
->spt
, root
->role
.level
,
1230 min_level
, start
, end
) {
1232 if (tdp_mmu_iter_cond_resched(kvm
, &iter
, false, true))
1235 if (!is_shadow_present_pte(iter
.old_spte
) ||
1236 !is_last_spte(iter
.old_spte
, iter
.level
) ||
1237 !(iter
.old_spte
& PT_WRITABLE_MASK
))
1240 new_spte
= iter
.old_spte
& ~PT_WRITABLE_MASK
;
1242 if (!tdp_mmu_set_spte_atomic_no_dirty_log(kvm
, &iter
,
1245 * The iter must explicitly re-read the SPTE because
1246 * the atomic cmpxchg failed.
1248 iter
.old_spte
= READ_ONCE(*rcu_dereference(iter
.sptep
));
1259 * Remove write access from all the SPTEs mapping GFNs in the memslot. Will
1260 * only affect leaf SPTEs down to min_level.
1261 * Returns true if an SPTE has been changed and the TLBs need to be flushed.
1263 bool kvm_tdp_mmu_wrprot_slot(struct kvm
*kvm
,
1264 const struct kvm_memory_slot
*slot
, int min_level
)
1266 struct kvm_mmu_page
*root
;
1267 bool spte_set
= false;
1269 lockdep_assert_held_read(&kvm
->mmu_lock
);
1271 for_each_tdp_mmu_root_yield_safe(kvm
, root
, slot
->as_id
, true)
1272 spte_set
|= wrprot_gfn_range(kvm
, root
, slot
->base_gfn
,
1273 slot
->base_gfn
+ slot
->npages
, min_level
);
1279 * Clear the dirty status of all the SPTEs mapping GFNs in the memslot. If
1280 * AD bits are enabled, this will involve clearing the dirty bit on each SPTE.
1281 * If AD bits are not enabled, this will require clearing the writable bit on
1282 * each SPTE. Returns true if an SPTE has been changed and the TLBs need to
1285 static bool clear_dirty_gfn_range(struct kvm
*kvm
, struct kvm_mmu_page
*root
,
1286 gfn_t start
, gfn_t end
)
1288 struct tdp_iter iter
;
1290 bool spte_set
= false;
1294 tdp_root_for_each_leaf_pte(iter
, root
, start
, end
) {
1296 if (tdp_mmu_iter_cond_resched(kvm
, &iter
, false, true))
1299 if (spte_ad_need_write_protect(iter
.old_spte
)) {
1300 if (is_writable_pte(iter
.old_spte
))
1301 new_spte
= iter
.old_spte
& ~PT_WRITABLE_MASK
;
1305 if (iter
.old_spte
& shadow_dirty_mask
)
1306 new_spte
= iter
.old_spte
& ~shadow_dirty_mask
;
1311 if (!tdp_mmu_set_spte_atomic_no_dirty_log(kvm
, &iter
,
1314 * The iter must explicitly re-read the SPTE because
1315 * the atomic cmpxchg failed.
1317 iter
.old_spte
= READ_ONCE(*rcu_dereference(iter
.sptep
));
1328 * Clear the dirty status of all the SPTEs mapping GFNs in the memslot. If
1329 * AD bits are enabled, this will involve clearing the dirty bit on each SPTE.
1330 * If AD bits are not enabled, this will require clearing the writable bit on
1331 * each SPTE. Returns true if an SPTE has been changed and the TLBs need to
1334 bool kvm_tdp_mmu_clear_dirty_slot(struct kvm
*kvm
,
1335 const struct kvm_memory_slot
*slot
)
1337 struct kvm_mmu_page
*root
;
1338 bool spte_set
= false;
1340 lockdep_assert_held_read(&kvm
->mmu_lock
);
1342 for_each_tdp_mmu_root_yield_safe(kvm
, root
, slot
->as_id
, true)
1343 spte_set
|= clear_dirty_gfn_range(kvm
, root
, slot
->base_gfn
,
1344 slot
->base_gfn
+ slot
->npages
);
1350 * Clears the dirty status of all the 4k SPTEs mapping GFNs for which a bit is
1351 * set in mask, starting at gfn. The given memslot is expected to contain all
1352 * the GFNs represented by set bits in the mask. If AD bits are enabled,
1353 * clearing the dirty status will involve clearing the dirty bit on each SPTE
1354 * or, if AD bits are not enabled, clearing the writable bit on each SPTE.
1356 static void clear_dirty_pt_masked(struct kvm
*kvm
, struct kvm_mmu_page
*root
,
1357 gfn_t gfn
, unsigned long mask
, bool wrprot
)
1359 struct tdp_iter iter
;
1364 tdp_root_for_each_leaf_pte(iter
, root
, gfn
+ __ffs(mask
),
1365 gfn
+ BITS_PER_LONG
) {
1369 if (iter
.level
> PG_LEVEL_4K
||
1370 !(mask
& (1UL << (iter
.gfn
- gfn
))))
1373 mask
&= ~(1UL << (iter
.gfn
- gfn
));
1375 if (wrprot
|| spte_ad_need_write_protect(iter
.old_spte
)) {
1376 if (is_writable_pte(iter
.old_spte
))
1377 new_spte
= iter
.old_spte
& ~PT_WRITABLE_MASK
;
1381 if (iter
.old_spte
& shadow_dirty_mask
)
1382 new_spte
= iter
.old_spte
& ~shadow_dirty_mask
;
1387 tdp_mmu_set_spte_no_dirty_log(kvm
, &iter
, new_spte
);
1394 * Clears the dirty status of all the 4k SPTEs mapping GFNs for which a bit is
1395 * set in mask, starting at gfn. The given memslot is expected to contain all
1396 * the GFNs represented by set bits in the mask. If AD bits are enabled,
1397 * clearing the dirty status will involve clearing the dirty bit on each SPTE
1398 * or, if AD bits are not enabled, clearing the writable bit on each SPTE.
1400 void kvm_tdp_mmu_clear_dirty_pt_masked(struct kvm
*kvm
,
1401 struct kvm_memory_slot
*slot
,
1402 gfn_t gfn
, unsigned long mask
,
1405 struct kvm_mmu_page
*root
;
1407 lockdep_assert_held_write(&kvm
->mmu_lock
);
1408 for_each_tdp_mmu_root(kvm
, root
, slot
->as_id
)
1409 clear_dirty_pt_masked(kvm
, root
, gfn
, mask
, wrprot
);
1413 * Clear leaf entries which could be replaced by large mappings, for
1414 * GFNs within the slot.
1416 static bool zap_collapsible_spte_range(struct kvm
*kvm
,
1417 struct kvm_mmu_page
*root
,
1418 const struct kvm_memory_slot
*slot
,
1421 gfn_t start
= slot
->base_gfn
;
1422 gfn_t end
= start
+ slot
->npages
;
1423 struct tdp_iter iter
;
1428 tdp_root_for_each_pte(iter
, root
, start
, end
) {
1430 if (tdp_mmu_iter_cond_resched(kvm
, &iter
, flush
, true)) {
1435 if (!is_shadow_present_pte(iter
.old_spte
) ||
1436 !is_last_spte(iter
.old_spte
, iter
.level
))
1439 pfn
= spte_to_pfn(iter
.old_spte
);
1440 if (kvm_is_reserved_pfn(pfn
) ||
1441 iter
.level
>= kvm_mmu_max_mapping_level(kvm
, slot
, iter
.gfn
,
1445 if (!tdp_mmu_zap_spte_atomic(kvm
, &iter
)) {
1447 * The iter must explicitly re-read the SPTE because
1448 * the atomic cmpxchg failed.
1450 iter
.old_spte
= READ_ONCE(*rcu_dereference(iter
.sptep
));
1462 * Clear non-leaf entries (and free associated page tables) which could
1463 * be replaced by large mappings, for GFNs within the slot.
1465 bool kvm_tdp_mmu_zap_collapsible_sptes(struct kvm
*kvm
,
1466 const struct kvm_memory_slot
*slot
,
1469 struct kvm_mmu_page
*root
;
1471 lockdep_assert_held_read(&kvm
->mmu_lock
);
1473 for_each_tdp_mmu_root_yield_safe(kvm
, root
, slot
->as_id
, true)
1474 flush
= zap_collapsible_spte_range(kvm
, root
, slot
, flush
);
1480 * Removes write access on the last level SPTE mapping this GFN and unsets the
1481 * MMU-writable bit to ensure future writes continue to be intercepted.
1482 * Returns true if an SPTE was set and a TLB flush is needed.
1484 static bool write_protect_gfn(struct kvm
*kvm
, struct kvm_mmu_page
*root
,
1485 gfn_t gfn
, int min_level
)
1487 struct tdp_iter iter
;
1489 bool spte_set
= false;
1491 BUG_ON(min_level
> KVM_MAX_HUGEPAGE_LEVEL
);
1495 for_each_tdp_pte_min_level(iter
, root
->spt
, root
->role
.level
,
1496 min_level
, gfn
, gfn
+ 1) {
1497 if (!is_shadow_present_pte(iter
.old_spte
) ||
1498 !is_last_spte(iter
.old_spte
, iter
.level
))
1501 if (!is_writable_pte(iter
.old_spte
))
1504 new_spte
= iter
.old_spte
&
1505 ~(PT_WRITABLE_MASK
| shadow_mmu_writable_mask
);
1507 tdp_mmu_set_spte(kvm
, &iter
, new_spte
);
1517 * Removes write access on the last level SPTE mapping this GFN and unsets the
1518 * MMU-writable bit to ensure future writes continue to be intercepted.
1519 * Returns true if an SPTE was set and a TLB flush is needed.
1521 bool kvm_tdp_mmu_write_protect_gfn(struct kvm
*kvm
,
1522 struct kvm_memory_slot
*slot
, gfn_t gfn
,
1525 struct kvm_mmu_page
*root
;
1526 bool spte_set
= false;
1528 lockdep_assert_held_write(&kvm
->mmu_lock
);
1529 for_each_tdp_mmu_root(kvm
, root
, slot
->as_id
)
1530 spte_set
|= write_protect_gfn(kvm
, root
, gfn
, min_level
);
1536 * Return the level of the lowest level SPTE added to sptes.
1537 * That SPTE may be non-present.
1539 * Must be called between kvm_tdp_mmu_walk_lockless_{begin,end}.
1541 int kvm_tdp_mmu_get_walk(struct kvm_vcpu
*vcpu
, u64 addr
, u64
*sptes
,
1544 struct tdp_iter iter
;
1545 struct kvm_mmu
*mmu
= vcpu
->arch
.mmu
;
1546 gfn_t gfn
= addr
>> PAGE_SHIFT
;
1549 *root_level
= vcpu
->arch
.mmu
->shadow_root_level
;
1551 tdp_mmu_for_each_pte(iter
, mmu
, gfn
, gfn
+ 1) {
1553 sptes
[leaf
] = iter
.old_spte
;
1560 * Returns the last level spte pointer of the shadow page walk for the given
1561 * gpa, and sets *spte to the spte value. This spte may be non-preset. If no
1562 * walk could be performed, returns NULL and *spte does not contain valid data.
1565 * - Must be called between kvm_tdp_mmu_walk_lockless_{begin,end}.
1566 * - The returned sptep must not be used after kvm_tdp_mmu_walk_lockless_end.
1568 * WARNING: This function is only intended to be called during fast_page_fault.
1570 u64
*kvm_tdp_mmu_fast_pf_get_last_sptep(struct kvm_vcpu
*vcpu
, u64 addr
,
1573 struct tdp_iter iter
;
1574 struct kvm_mmu
*mmu
= vcpu
->arch
.mmu
;
1575 gfn_t gfn
= addr
>> PAGE_SHIFT
;
1576 tdp_ptep_t sptep
= NULL
;
1578 tdp_mmu_for_each_pte(iter
, mmu
, gfn
, gfn
+ 1) {
1579 *spte
= iter
.old_spte
;
1584 * Perform the rcu_dereference to get the raw spte pointer value since
1585 * we are passing it up to fast_page_fault, which is shared with the
1586 * legacy MMU and thus does not retain the TDP MMU-specific __rcu
1589 * This is safe since fast_page_fault obeys the contracts of this
1590 * function as well as all TDP MMU contracts around modifying SPTEs
1591 * outside of mmu_lock.
1593 return rcu_dereference(sptep
);