1 // SPDX-License-Identifier: GPL-2.0
4 #include "mmu_internal.h"
11 static bool __read_mostly tdp_mmu_enabled = false;
12 module_param_named(tdp_mmu, tdp_mmu_enabled, bool, 0644);
15 static bool is_tdp_mmu_enabled(void)
18 return tdp_enabled && READ_ONCE(tdp_mmu_enabled);
21 #endif /* CONFIG_X86_64 */
24 /* Initializes the TDP MMU for the VM, if enabled. */
25 void kvm_mmu_init_tdp_mmu(struct kvm *kvm)
27 if (!is_tdp_mmu_enabled())
30 /* This should not be changed for the lifetime of the VM. */
31 kvm->arch.tdp_mmu_enabled = true;
33 INIT_LIST_HEAD(&kvm->arch.tdp_mmu_roots);
34 INIT_LIST_HEAD(&kvm->arch.tdp_mmu_pages);
37 void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm)
39 if (!kvm->arch.tdp_mmu_enabled)
42 WARN_ON(!list_empty(&kvm->arch.tdp_mmu_roots));
45 static void tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root)
47 if (kvm_mmu_put_root(kvm, root))
48 kvm_tdp_mmu_free_root(kvm, root);
51 static inline bool tdp_mmu_next_root_valid(struct kvm *kvm,
52 struct kvm_mmu_page *root)
54 lockdep_assert_held(&kvm->mmu_lock);
56 if (list_entry_is_head(root, &kvm->arch.tdp_mmu_roots, link))
59 kvm_mmu_get_root(kvm, root);
64 static inline struct kvm_mmu_page *tdp_mmu_next_root(struct kvm *kvm,
65 struct kvm_mmu_page *root)
67 struct kvm_mmu_page *next_root;
69 next_root = list_next_entry(root, link);
70 tdp_mmu_put_root(kvm, root);
75 * Note: this iterator gets and puts references to the roots it iterates over.
76 * This makes it safe to release the MMU lock and yield within the loop, but
77 * if exiting the loop early, the caller must drop the reference to the most
78 * recent root. (Unless keeping a live reference is desirable.)
80 #define for_each_tdp_mmu_root_yield_safe(_kvm, _root) \
81 for (_root = list_first_entry(&_kvm->arch.tdp_mmu_roots, \
82 typeof(*_root), link); \
83 tdp_mmu_next_root_valid(_kvm, _root); \
84 _root = tdp_mmu_next_root(_kvm, _root))
86 #define for_each_tdp_mmu_root(_kvm, _root) \
87 list_for_each_entry(_root, &_kvm->arch.tdp_mmu_roots, link)
89 bool is_tdp_mmu_root(struct kvm *kvm, hpa_t hpa)
91 struct kvm_mmu_page *sp;
93 if (!kvm->arch.tdp_mmu_enabled)
95 if (WARN_ON(!VALID_PAGE(hpa)))
98 sp = to_shadow_page(hpa);
102 return sp->tdp_mmu_page && sp->root_count;
105 static bool zap_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root,
106 gfn_t start, gfn_t end, bool can_yield, bool flush);
108 void kvm_tdp_mmu_free_root(struct kvm *kvm, struct kvm_mmu_page *root)
110 gfn_t max_gfn = 1ULL << (shadow_phys_bits - PAGE_SHIFT);
112 lockdep_assert_held(&kvm->mmu_lock);
114 WARN_ON(root->root_count);
115 WARN_ON(!root->tdp_mmu_page);
117 list_del(&root->link);
119 zap_gfn_range(kvm, root, 0, max_gfn, false, false);
121 free_page((unsigned long)root->spt);
122 kmem_cache_free(mmu_page_header_cache, root);
125 static union kvm_mmu_page_role page_role_for_level(struct kvm_vcpu *vcpu,
128 union kvm_mmu_page_role role;
130 role = vcpu->arch.mmu->mmu_role.base;
133 role.gpte_is_8_bytes = true;
134 role.access = ACC_ALL;
139 static struct kvm_mmu_page *alloc_tdp_mmu_page(struct kvm_vcpu *vcpu, gfn_t gfn,
142 struct kvm_mmu_page *sp;
144 sp = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache);
145 sp->spt = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_shadow_page_cache);
146 set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
148 sp->role.word = page_role_for_level(vcpu, level).word;
150 sp->tdp_mmu_page = true;
155 static struct kvm_mmu_page *get_tdp_mmu_vcpu_root(struct kvm_vcpu *vcpu)
157 union kvm_mmu_page_role role;
158 struct kvm *kvm = vcpu->kvm;
159 struct kvm_mmu_page *root;
161 role = page_role_for_level(vcpu, vcpu->arch.mmu->shadow_root_level);
163 spin_lock(&kvm->mmu_lock);
165 /* Check for an existing root before allocating a new one. */
166 for_each_tdp_mmu_root(kvm, root) {
167 if (root->role.word == role.word) {
168 kvm_mmu_get_root(kvm, root);
169 spin_unlock(&kvm->mmu_lock);
174 root = alloc_tdp_mmu_page(vcpu, 0, vcpu->arch.mmu->shadow_root_level);
175 root->root_count = 1;
177 list_add(&root->link, &kvm->arch.tdp_mmu_roots);
179 spin_unlock(&kvm->mmu_lock);
184 hpa_t kvm_tdp_mmu_get_vcpu_root_hpa(struct kvm_vcpu *vcpu)
186 struct kvm_mmu_page *root;
188 root = get_tdp_mmu_vcpu_root(vcpu);
192 return __pa(root->spt);
195 static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn,
196 u64 old_spte, u64 new_spte, int level);
198 static int kvm_mmu_page_as_id(struct kvm_mmu_page *sp)
200 return sp->role.smm ? 1 : 0;
203 static void handle_changed_spte_acc_track(u64 old_spte, u64 new_spte, int level)
205 bool pfn_changed = spte_to_pfn(old_spte) != spte_to_pfn(new_spte);
207 if (!is_shadow_present_pte(old_spte) || !is_last_spte(old_spte, level))
210 if (is_accessed_spte(old_spte) &&
211 (!is_accessed_spte(new_spte) || pfn_changed))
212 kvm_set_pfn_accessed(spte_to_pfn(old_spte));
215 static void handle_changed_spte_dirty_log(struct kvm *kvm, int as_id, gfn_t gfn,
216 u64 old_spte, u64 new_spte, int level)
219 struct kvm_memory_slot *slot;
221 if (level > PG_LEVEL_4K)
224 pfn_changed = spte_to_pfn(old_spte) != spte_to_pfn(new_spte);
226 if ((!is_writable_pte(old_spte) || pfn_changed) &&
227 is_writable_pte(new_spte)) {
228 slot = __gfn_to_memslot(__kvm_memslots(kvm, as_id), gfn);
229 mark_page_dirty_in_slot(slot, gfn);
234 * handle_changed_spte - handle bookkeeping associated with an SPTE change
236 * @as_id: the address space of the paging structure the SPTE was a part of
237 * @gfn: the base GFN that was mapped by the SPTE
238 * @old_spte: The value of the SPTE before the change
239 * @new_spte: The value of the SPTE after the change
240 * @level: the level of the PT the SPTE is part of in the paging structure
242 * Handle bookkeeping that might result from the modification of a SPTE.
243 * This function must be called for all TDP SPTE modifications.
245 static void __handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn,
246 u64 old_spte, u64 new_spte, int level)
248 bool was_present = is_shadow_present_pte(old_spte);
249 bool is_present = is_shadow_present_pte(new_spte);
250 bool was_leaf = was_present && is_last_spte(old_spte, level);
251 bool is_leaf = is_present && is_last_spte(new_spte, level);
252 bool pfn_changed = spte_to_pfn(old_spte) != spte_to_pfn(new_spte);
254 struct kvm_mmu_page *sp;
258 WARN_ON(level > PT64_ROOT_MAX_LEVEL);
259 WARN_ON(level < PG_LEVEL_4K);
260 WARN_ON(gfn & (KVM_PAGES_PER_HPAGE(level) - 1));
263 * If this warning were to trigger it would indicate that there was a
264 * missing MMU notifier or a race with some notifier handler.
265 * A present, leaf SPTE should never be directly replaced with another
266 * present leaf SPTE pointing to a differnt PFN. A notifier handler
267 * should be zapping the SPTE before the main MM's page table is
268 * changed, or the SPTE should be zeroed, and the TLBs flushed by the
269 * thread before replacement.
271 if (was_leaf && is_leaf && pfn_changed) {
272 pr_err("Invalid SPTE change: cannot replace a present leaf\n"
273 "SPTE with another present leaf SPTE mapping a\n"
275 "as_id: %d gfn: %llx old_spte: %llx new_spte: %llx level: %d",
276 as_id, gfn, old_spte, new_spte, level);
279 * Crash the host to prevent error propagation and guest data
285 if (old_spte == new_spte)
289 * The only times a SPTE should be changed from a non-present to
290 * non-present state is when an MMIO entry is installed/modified/
291 * removed. In that case, there is nothing to do here.
293 if (!was_present && !is_present) {
295 * If this change does not involve a MMIO SPTE, it is
296 * unexpected. Log the change, though it should not impact the
297 * guest since both the former and current SPTEs are nonpresent.
299 if (WARN_ON(!is_mmio_spte(old_spte) && !is_mmio_spte(new_spte)))
300 pr_err("Unexpected SPTE change! Nonpresent SPTEs\n"
301 "should not be replaced with another,\n"
302 "different nonpresent SPTE, unless one or both\n"
304 "as_id: %d gfn: %llx old_spte: %llx new_spte: %llx level: %d",
305 as_id, gfn, old_spte, new_spte, level);
310 if (was_leaf && is_dirty_spte(old_spte) &&
311 (!is_dirty_spte(new_spte) || pfn_changed))
312 kvm_set_pfn_dirty(spte_to_pfn(old_spte));
315 * Recursively handle child PTs if the change removed a subtree from
316 * the paging structure.
318 if (was_present && !was_leaf && (pfn_changed || !is_present)) {
319 pt = spte_to_child_pt(old_spte, level);
320 sp = sptep_to_sp(pt);
324 if (sp->lpage_disallowed)
325 unaccount_huge_nx_page(kvm, sp);
327 for (i = 0; i < PT64_ENT_PER_PAGE; i++) {
328 old_child_spte = READ_ONCE(*(pt + i));
329 WRITE_ONCE(*(pt + i), 0);
330 handle_changed_spte(kvm, as_id,
331 gfn + (i * KVM_PAGES_PER_HPAGE(level - 1)),
332 old_child_spte, 0, level - 1);
335 kvm_flush_remote_tlbs_with_address(kvm, gfn,
336 KVM_PAGES_PER_HPAGE(level));
338 free_page((unsigned long)pt);
339 kmem_cache_free(mmu_page_header_cache, sp);
343 static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn,
344 u64 old_spte, u64 new_spte, int level)
346 __handle_changed_spte(kvm, as_id, gfn, old_spte, new_spte, level);
347 handle_changed_spte_acc_track(old_spte, new_spte, level);
348 handle_changed_spte_dirty_log(kvm, as_id, gfn, old_spte,
352 static inline void __tdp_mmu_set_spte(struct kvm *kvm, struct tdp_iter *iter,
353 u64 new_spte, bool record_acc_track,
354 bool record_dirty_log)
356 u64 *root_pt = tdp_iter_root_pt(iter);
357 struct kvm_mmu_page *root = sptep_to_sp(root_pt);
358 int as_id = kvm_mmu_page_as_id(root);
360 WRITE_ONCE(*iter->sptep, new_spte);
362 __handle_changed_spte(kvm, as_id, iter->gfn, iter->old_spte, new_spte,
364 if (record_acc_track)
365 handle_changed_spte_acc_track(iter->old_spte, new_spte,
367 if (record_dirty_log)
368 handle_changed_spte_dirty_log(kvm, as_id, iter->gfn,
369 iter->old_spte, new_spte,
373 static inline void tdp_mmu_set_spte(struct kvm *kvm, struct tdp_iter *iter,
376 __tdp_mmu_set_spte(kvm, iter, new_spte, true, true);
379 static inline void tdp_mmu_set_spte_no_acc_track(struct kvm *kvm,
380 struct tdp_iter *iter,
383 __tdp_mmu_set_spte(kvm, iter, new_spte, false, true);
386 static inline void tdp_mmu_set_spte_no_dirty_log(struct kvm *kvm,
387 struct tdp_iter *iter,
390 __tdp_mmu_set_spte(kvm, iter, new_spte, true, false);
393 #define tdp_root_for_each_pte(_iter, _root, _start, _end) \
394 for_each_tdp_pte(_iter, _root->spt, _root->role.level, _start, _end)
396 #define tdp_root_for_each_leaf_pte(_iter, _root, _start, _end) \
397 tdp_root_for_each_pte(_iter, _root, _start, _end) \
398 if (!is_shadow_present_pte(_iter.old_spte) || \
399 !is_last_spte(_iter.old_spte, _iter.level)) \
403 #define tdp_mmu_for_each_pte(_iter, _mmu, _start, _end) \
404 for_each_tdp_pte(_iter, __va(_mmu->root_hpa), \
405 _mmu->shadow_root_level, _start, _end)
408 * Yield if the MMU lock is contended or this thread needs to return control
411 * If this function should yield and flush is set, it will perform a remote
412 * TLB flush before yielding.
414 * If this function yields, it will also reset the tdp_iter's walk over the
415 * paging structure and the calling function should skip to the next
416 * iteration to allow the iterator to continue its traversal from the
417 * paging structure root.
419 * Return true if this function yielded and the iterator's traversal was reset.
420 * Return false if a yield was not needed.
422 static inline bool tdp_mmu_iter_cond_resched(struct kvm *kvm,
423 struct tdp_iter *iter, bool flush)
425 /* Ensure forward progress has been made before yielding. */
426 if (iter->next_last_level_gfn == iter->yielded_gfn)
429 if (need_resched() || spin_needbreak(&kvm->mmu_lock)) {
431 kvm_flush_remote_tlbs(kvm);
433 cond_resched_lock(&kvm->mmu_lock);
435 WARN_ON(iter->gfn > iter->next_last_level_gfn);
437 tdp_iter_start(iter, iter->pt_path[iter->root_level - 1],
438 iter->root_level, iter->min_level,
439 iter->next_last_level_gfn);
448 * Tears down the mappings for the range of gfns, [start, end), and frees the
449 * non-root pages mapping GFNs strictly within that range. Returns true if
450 * SPTEs have been cleared and a TLB flush is needed before releasing the
452 * If can_yield is true, will release the MMU lock and reschedule if the
453 * scheduler needs the CPU or there is contention on the MMU lock. If this
454 * function cannot yield, it will not release the MMU lock or reschedule and
455 * the caller must ensure it does not supply too large a GFN range, or the
456 * operation can cause a soft lockup. Note, in some use cases a flush may be
457 * required by prior actions. Ensure the pending flush is performed prior to
460 static bool zap_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root,
461 gfn_t start, gfn_t end, bool can_yield, bool flush)
463 struct tdp_iter iter;
465 tdp_root_for_each_pte(iter, root, start, end) {
467 tdp_mmu_iter_cond_resched(kvm, &iter, flush)) {
472 if (!is_shadow_present_pte(iter.old_spte))
476 * If this is a non-last-level SPTE that covers a larger range
477 * than should be zapped, continue, and zap the mappings at a
480 if ((iter.gfn < start ||
481 iter.gfn + KVM_PAGES_PER_HPAGE(iter.level) > end) &&
482 !is_last_spte(iter.old_spte, iter.level))
485 tdp_mmu_set_spte(kvm, &iter, 0);
493 * Tears down the mappings for the range of gfns, [start, end), and frees the
494 * non-root pages mapping GFNs strictly within that range. Returns true if
495 * SPTEs have been cleared and a TLB flush is needed before releasing the
498 bool __kvm_tdp_mmu_zap_gfn_range(struct kvm *kvm, gfn_t start, gfn_t end,
501 struct kvm_mmu_page *root;
504 for_each_tdp_mmu_root_yield_safe(kvm, root)
505 flush = zap_gfn_range(kvm, root, start, end, can_yield, flush);
510 void kvm_tdp_mmu_zap_all(struct kvm *kvm)
512 gfn_t max_gfn = 1ULL << (shadow_phys_bits - PAGE_SHIFT);
515 flush = kvm_tdp_mmu_zap_gfn_range(kvm, 0, max_gfn);
517 kvm_flush_remote_tlbs(kvm);
521 * Installs a last-level SPTE to handle a TDP page fault.
522 * (NPT/EPT violation/misconfiguration)
524 static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu, int write,
526 struct tdp_iter *iter,
527 kvm_pfn_t pfn, bool prefault)
530 int ret = RET_PF_FIXED;
531 int make_spte_ret = 0;
533 if (unlikely(is_noslot_pfn(pfn))) {
534 new_spte = make_mmio_spte(vcpu, iter->gfn, ACC_ALL);
535 trace_mark_mmio_spte(iter->sptep, iter->gfn, new_spte);
537 make_spte_ret = make_spte(vcpu, ACC_ALL, iter->level, iter->gfn,
538 pfn, iter->old_spte, prefault, true,
539 map_writable, !shadow_accessed_mask,
542 if (new_spte == iter->old_spte)
543 ret = RET_PF_SPURIOUS;
545 tdp_mmu_set_spte(vcpu->kvm, iter, new_spte);
548 * If the page fault was caused by a write but the page is write
549 * protected, emulation is needed. If the emulation was skipped,
550 * the vCPU would have the same fault again.
552 if (make_spte_ret & SET_SPTE_WRITE_PROTECTED_PT) {
554 ret = RET_PF_EMULATE;
555 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
558 /* If a MMIO SPTE is installed, the MMIO will need to be emulated. */
559 if (unlikely(is_mmio_spte(new_spte)))
560 ret = RET_PF_EMULATE;
562 trace_kvm_mmu_set_spte(iter->level, iter->gfn, iter->sptep);
564 vcpu->stat.pf_fixed++;
570 * Handle a TDP page fault (NPT/EPT violation/misconfiguration) by installing
571 * page tables and SPTEs to translate the faulting guest physical address.
573 int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code,
574 int map_writable, int max_level, kvm_pfn_t pfn,
577 bool nx_huge_page_workaround_enabled = is_nx_huge_page_enabled();
578 bool write = error_code & PFERR_WRITE_MASK;
579 bool exec = error_code & PFERR_FETCH_MASK;
580 bool huge_page_disallowed = exec && nx_huge_page_workaround_enabled;
581 struct kvm_mmu *mmu = vcpu->arch.mmu;
582 struct tdp_iter iter;
583 struct kvm_mmu_page *sp;
587 gfn_t gfn = gpa >> PAGE_SHIFT;
591 if (WARN_ON(!VALID_PAGE(vcpu->arch.mmu->root_hpa)))
593 if (WARN_ON(!is_tdp_mmu_root(vcpu->kvm, vcpu->arch.mmu->root_hpa)))
596 level = kvm_mmu_hugepage_adjust(vcpu, gfn, max_level, &pfn,
597 huge_page_disallowed, &req_level);
599 trace_kvm_mmu_spte_requested(gpa, level, pfn);
600 tdp_mmu_for_each_pte(iter, mmu, gfn, gfn + 1) {
601 if (nx_huge_page_workaround_enabled)
602 disallowed_hugepage_adjust(iter.old_spte, gfn,
603 iter.level, &pfn, &level);
605 if (iter.level == level)
609 * If there is an SPTE mapping a large page at a higher level
610 * than the target, that SPTE must be cleared and replaced
611 * with a non-leaf SPTE.
613 if (is_shadow_present_pte(iter.old_spte) &&
614 is_large_pte(iter.old_spte)) {
615 tdp_mmu_set_spte(vcpu->kvm, &iter, 0);
617 kvm_flush_remote_tlbs_with_address(vcpu->kvm, iter.gfn,
618 KVM_PAGES_PER_HPAGE(iter.level));
621 * The iter must explicitly re-read the spte here
622 * because the new value informs the !present
625 iter.old_spte = READ_ONCE(*iter.sptep);
628 if (!is_shadow_present_pte(iter.old_spte)) {
629 sp = alloc_tdp_mmu_page(vcpu, iter.gfn, iter.level);
630 list_add(&sp->link, &vcpu->kvm->arch.tdp_mmu_pages);
632 clear_page(child_pt);
633 new_spte = make_nonleaf_spte(child_pt,
634 !shadow_accessed_mask);
636 trace_kvm_mmu_get_page(sp, true);
637 if (huge_page_disallowed && req_level >= iter.level)
638 account_huge_nx_page(vcpu->kvm, sp);
640 tdp_mmu_set_spte(vcpu->kvm, &iter, new_spte);
644 if (WARN_ON(iter.level != level))
647 ret = tdp_mmu_map_handle_target_level(vcpu, write, map_writable, &iter,
653 static int kvm_tdp_mmu_handle_hva_range(struct kvm *kvm, unsigned long start,
654 unsigned long end, unsigned long data,
655 int (*handler)(struct kvm *kvm, struct kvm_memory_slot *slot,
656 struct kvm_mmu_page *root, gfn_t start,
657 gfn_t end, unsigned long data))
659 struct kvm_memslots *slots;
660 struct kvm_memory_slot *memslot;
661 struct kvm_mmu_page *root;
665 for_each_tdp_mmu_root_yield_safe(kvm, root) {
666 as_id = kvm_mmu_page_as_id(root);
667 slots = __kvm_memslots(kvm, as_id);
668 kvm_for_each_memslot(memslot, slots) {
669 unsigned long hva_start, hva_end;
670 gfn_t gfn_start, gfn_end;
672 hva_start = max(start, memslot->userspace_addr);
673 hva_end = min(end, memslot->userspace_addr +
674 (memslot->npages << PAGE_SHIFT));
675 if (hva_start >= hva_end)
678 * {gfn(page) | page intersects with [hva_start, hva_end)} =
679 * {gfn_start, gfn_start+1, ..., gfn_end-1}.
681 gfn_start = hva_to_gfn_memslot(hva_start, memslot);
682 gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
684 ret |= handler(kvm, memslot, root, gfn_start,
692 static int zap_gfn_range_hva_wrapper(struct kvm *kvm,
693 struct kvm_memory_slot *slot,
694 struct kvm_mmu_page *root, gfn_t start,
695 gfn_t end, unsigned long unused)
697 return zap_gfn_range(kvm, root, start, end, false, false);
700 int kvm_tdp_mmu_zap_hva_range(struct kvm *kvm, unsigned long start,
703 return kvm_tdp_mmu_handle_hva_range(kvm, start, end, 0,
704 zap_gfn_range_hva_wrapper);
708 * Mark the SPTEs range of GFNs [start, end) unaccessed and return non-zero
709 * if any of the GFNs in the range have been accessed.
711 static int age_gfn_range(struct kvm *kvm, struct kvm_memory_slot *slot,
712 struct kvm_mmu_page *root, gfn_t start, gfn_t end,
713 unsigned long unused)
715 struct tdp_iter iter;
719 tdp_root_for_each_leaf_pte(iter, root, start, end) {
721 * If we have a non-accessed entry we don't need to change the
724 if (!is_accessed_spte(iter.old_spte))
727 new_spte = iter.old_spte;
729 if (spte_ad_enabled(new_spte)) {
730 clear_bit((ffs(shadow_accessed_mask) - 1),
731 (unsigned long *)&new_spte);
734 * Capture the dirty status of the page, so that it doesn't get
735 * lost when the SPTE is marked for access tracking.
737 if (is_writable_pte(new_spte))
738 kvm_set_pfn_dirty(spte_to_pfn(new_spte));
740 new_spte = mark_spte_for_access_track(new_spte);
742 new_spte &= ~shadow_dirty_mask;
744 tdp_mmu_set_spte_no_acc_track(kvm, &iter, new_spte);
751 int kvm_tdp_mmu_age_hva_range(struct kvm *kvm, unsigned long start,
754 return kvm_tdp_mmu_handle_hva_range(kvm, start, end, 0,
758 static int test_age_gfn(struct kvm *kvm, struct kvm_memory_slot *slot,
759 struct kvm_mmu_page *root, gfn_t gfn, gfn_t unused,
760 unsigned long unused2)
762 struct tdp_iter iter;
764 tdp_root_for_each_leaf_pte(iter, root, gfn, gfn + 1)
765 if (is_accessed_spte(iter.old_spte))
771 int kvm_tdp_mmu_test_age_hva(struct kvm *kvm, unsigned long hva)
773 return kvm_tdp_mmu_handle_hva_range(kvm, hva, hva + 1, 0,
778 * Handle the changed_pte MMU notifier for the TDP MMU.
779 * data is a pointer to the new pte_t mapping the HVA specified by the MMU
781 * Returns non-zero if a flush is needed before releasing the MMU lock.
783 static int set_tdp_spte(struct kvm *kvm, struct kvm_memory_slot *slot,
784 struct kvm_mmu_page *root, gfn_t gfn, gfn_t unused,
787 struct tdp_iter iter;
788 pte_t *ptep = (pte_t *)data;
793 WARN_ON(pte_huge(*ptep));
795 new_pfn = pte_pfn(*ptep);
797 tdp_root_for_each_pte(iter, root, gfn, gfn + 1) {
798 if (iter.level != PG_LEVEL_4K)
801 if (!is_shadow_present_pte(iter.old_spte))
804 tdp_mmu_set_spte(kvm, &iter, 0);
806 kvm_flush_remote_tlbs_with_address(kvm, iter.gfn, 1);
808 if (!pte_write(*ptep)) {
809 new_spte = kvm_mmu_changed_pte_notifier_make_spte(
810 iter.old_spte, new_pfn);
812 tdp_mmu_set_spte(kvm, &iter, new_spte);
819 kvm_flush_remote_tlbs_with_address(kvm, gfn, 1);
824 int kvm_tdp_mmu_set_spte_hva(struct kvm *kvm, unsigned long address,
827 return kvm_tdp_mmu_handle_hva_range(kvm, address, address + 1,
828 (unsigned long)host_ptep,
833 * Remove write access from all the SPTEs mapping GFNs [start, end). If
834 * skip_4k is set, SPTEs that map 4k pages, will not be write-protected.
835 * Returns true if an SPTE has been changed and the TLBs need to be flushed.
837 static bool wrprot_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root,
838 gfn_t start, gfn_t end, int min_level)
840 struct tdp_iter iter;
842 bool spte_set = false;
844 BUG_ON(min_level > KVM_MAX_HUGEPAGE_LEVEL);
846 for_each_tdp_pte_min_level(iter, root->spt, root->role.level,
847 min_level, start, end) {
848 if (tdp_mmu_iter_cond_resched(kvm, &iter, false))
851 if (!is_shadow_present_pte(iter.old_spte) ||
852 !is_last_spte(iter.old_spte, iter.level))
855 new_spte = iter.old_spte & ~PT_WRITABLE_MASK;
857 tdp_mmu_set_spte_no_dirty_log(kvm, &iter, new_spte);
864 * Remove write access from all the SPTEs mapping GFNs in the memslot. Will
865 * only affect leaf SPTEs down to min_level.
866 * Returns true if an SPTE has been changed and the TLBs need to be flushed.
868 bool kvm_tdp_mmu_wrprot_slot(struct kvm *kvm, struct kvm_memory_slot *slot,
871 struct kvm_mmu_page *root;
873 bool spte_set = false;
875 for_each_tdp_mmu_root_yield_safe(kvm, root) {
876 root_as_id = kvm_mmu_page_as_id(root);
877 if (root_as_id != slot->as_id)
880 spte_set |= wrprot_gfn_range(kvm, root, slot->base_gfn,
881 slot->base_gfn + slot->npages, min_level);
888 * Clear the dirty status of all the SPTEs mapping GFNs in the memslot. If
889 * AD bits are enabled, this will involve clearing the dirty bit on each SPTE.
890 * If AD bits are not enabled, this will require clearing the writable bit on
891 * each SPTE. Returns true if an SPTE has been changed and the TLBs need to
894 static bool clear_dirty_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root,
895 gfn_t start, gfn_t end)
897 struct tdp_iter iter;
899 bool spte_set = false;
901 tdp_root_for_each_leaf_pte(iter, root, start, end) {
902 if (tdp_mmu_iter_cond_resched(kvm, &iter, false))
905 if (!is_shadow_present_pte(iter.old_spte))
908 if (spte_ad_need_write_protect(iter.old_spte)) {
909 if (is_writable_pte(iter.old_spte))
910 new_spte = iter.old_spte & ~PT_WRITABLE_MASK;
914 if (iter.old_spte & shadow_dirty_mask)
915 new_spte = iter.old_spte & ~shadow_dirty_mask;
920 tdp_mmu_set_spte_no_dirty_log(kvm, &iter, new_spte);
927 * Clear the dirty status of all the SPTEs mapping GFNs in the memslot. If
928 * AD bits are enabled, this will involve clearing the dirty bit on each SPTE.
929 * If AD bits are not enabled, this will require clearing the writable bit on
930 * each SPTE. Returns true if an SPTE has been changed and the TLBs need to
933 bool kvm_tdp_mmu_clear_dirty_slot(struct kvm *kvm, struct kvm_memory_slot *slot)
935 struct kvm_mmu_page *root;
937 bool spte_set = false;
939 for_each_tdp_mmu_root_yield_safe(kvm, root) {
940 root_as_id = kvm_mmu_page_as_id(root);
941 if (root_as_id != slot->as_id)
944 spte_set |= clear_dirty_gfn_range(kvm, root, slot->base_gfn,
945 slot->base_gfn + slot->npages);
952 * Clears the dirty status of all the 4k SPTEs mapping GFNs for which a bit is
953 * set in mask, starting at gfn. The given memslot is expected to contain all
954 * the GFNs represented by set bits in the mask. If AD bits are enabled,
955 * clearing the dirty status will involve clearing the dirty bit on each SPTE
956 * or, if AD bits are not enabled, clearing the writable bit on each SPTE.
958 static void clear_dirty_pt_masked(struct kvm *kvm, struct kvm_mmu_page *root,
959 gfn_t gfn, unsigned long mask, bool wrprot)
961 struct tdp_iter iter;
964 tdp_root_for_each_leaf_pte(iter, root, gfn + __ffs(mask),
965 gfn + BITS_PER_LONG) {
969 if (iter.level > PG_LEVEL_4K ||
970 !(mask & (1UL << (iter.gfn - gfn))))
973 if (wrprot || spte_ad_need_write_protect(iter.old_spte)) {
974 if (is_writable_pte(iter.old_spte))
975 new_spte = iter.old_spte & ~PT_WRITABLE_MASK;
979 if (iter.old_spte & shadow_dirty_mask)
980 new_spte = iter.old_spte & ~shadow_dirty_mask;
985 tdp_mmu_set_spte_no_dirty_log(kvm, &iter, new_spte);
987 mask &= ~(1UL << (iter.gfn - gfn));
992 * Clears the dirty status of all the 4k SPTEs mapping GFNs for which a bit is
993 * set in mask, starting at gfn. The given memslot is expected to contain all
994 * the GFNs represented by set bits in the mask. If AD bits are enabled,
995 * clearing the dirty status will involve clearing the dirty bit on each SPTE
996 * or, if AD bits are not enabled, clearing the writable bit on each SPTE.
998 void kvm_tdp_mmu_clear_dirty_pt_masked(struct kvm *kvm,
999 struct kvm_memory_slot *slot,
1000 gfn_t gfn, unsigned long mask,
1003 struct kvm_mmu_page *root;
1006 lockdep_assert_held(&kvm->mmu_lock);
1007 for_each_tdp_mmu_root(kvm, root) {
1008 root_as_id = kvm_mmu_page_as_id(root);
1009 if (root_as_id != slot->as_id)
1012 clear_dirty_pt_masked(kvm, root, gfn, mask, wrprot);
1017 * Set the dirty status of all the SPTEs mapping GFNs in the memslot. This is
1018 * only used for PML, and so will involve setting the dirty bit on each SPTE.
1019 * Returns true if an SPTE has been changed and the TLBs need to be flushed.
1021 static bool set_dirty_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root,
1022 gfn_t start, gfn_t end)
1024 struct tdp_iter iter;
1026 bool spte_set = false;
1028 tdp_root_for_each_pte(iter, root, start, end) {
1029 if (tdp_mmu_iter_cond_resched(kvm, &iter, false))
1032 if (!is_shadow_present_pte(iter.old_spte))
1035 new_spte = iter.old_spte | shadow_dirty_mask;
1037 tdp_mmu_set_spte(kvm, &iter, new_spte);
1045 * Set the dirty status of all the SPTEs mapping GFNs in the memslot. This is
1046 * only used for PML, and so will involve setting the dirty bit on each SPTE.
1047 * Returns true if an SPTE has been changed and the TLBs need to be flushed.
1049 bool kvm_tdp_mmu_slot_set_dirty(struct kvm *kvm, struct kvm_memory_slot *slot)
1051 struct kvm_mmu_page *root;
1053 bool spte_set = false;
1055 for_each_tdp_mmu_root_yield_safe(kvm, root) {
1056 root_as_id = kvm_mmu_page_as_id(root);
1057 if (root_as_id != slot->as_id)
1060 spte_set |= set_dirty_gfn_range(kvm, root, slot->base_gfn,
1061 slot->base_gfn + slot->npages);
1067 * Clear leaf entries which could be replaced by large mappings, for
1068 * GFNs within the slot.
1070 static void zap_collapsible_spte_range(struct kvm *kvm,
1071 struct kvm_mmu_page *root,
1072 gfn_t start, gfn_t end)
1074 struct tdp_iter iter;
1076 bool spte_set = false;
1078 tdp_root_for_each_pte(iter, root, start, end) {
1079 if (tdp_mmu_iter_cond_resched(kvm, &iter, spte_set)) {
1084 if (!is_shadow_present_pte(iter.old_spte) ||
1085 !is_last_spte(iter.old_spte, iter.level))
1088 pfn = spte_to_pfn(iter.old_spte);
1089 if (kvm_is_reserved_pfn(pfn) ||
1090 (!PageCompound(pfn_to_page(pfn)) &&
1091 !kvm_is_zone_device_pfn(pfn)))
1094 tdp_mmu_set_spte(kvm, &iter, 0);
1100 kvm_flush_remote_tlbs(kvm);
1104 * Clear non-leaf entries (and free associated page tables) which could
1105 * be replaced by large mappings, for GFNs within the slot.
1107 void kvm_tdp_mmu_zap_collapsible_sptes(struct kvm *kvm,
1108 const struct kvm_memory_slot *slot)
1110 struct kvm_mmu_page *root;
1113 for_each_tdp_mmu_root_yield_safe(kvm, root) {
1114 root_as_id = kvm_mmu_page_as_id(root);
1115 if (root_as_id != slot->as_id)
1118 zap_collapsible_spte_range(kvm, root, slot->base_gfn,
1119 slot->base_gfn + slot->npages);
1124 * Removes write access on the last level SPTE mapping this GFN and unsets the
1125 * SPTE_MMU_WRITABLE bit to ensure future writes continue to be intercepted.
1126 * Returns true if an SPTE was set and a TLB flush is needed.
1128 static bool write_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root,
1131 struct tdp_iter iter;
1133 bool spte_set = false;
1135 tdp_root_for_each_leaf_pte(iter, root, gfn, gfn + 1) {
1136 new_spte = iter.old_spte &
1137 ~(PT_WRITABLE_MASK | SPTE_MMU_WRITEABLE);
1139 if (new_spte == iter.old_spte)
1142 tdp_mmu_set_spte(kvm, &iter, new_spte);
1150 * Removes write access on the last level SPTE mapping this GFN and unsets the
1151 * SPTE_MMU_WRITABLE bit to ensure future writes continue to be intercepted.
1152 * Returns true if an SPTE was set and a TLB flush is needed.
1154 bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm,
1155 struct kvm_memory_slot *slot, gfn_t gfn)
1157 struct kvm_mmu_page *root;
1159 bool spte_set = false;
1161 lockdep_assert_held(&kvm->mmu_lock);
1162 for_each_tdp_mmu_root(kvm, root) {
1163 root_as_id = kvm_mmu_page_as_id(root);
1164 if (root_as_id != slot->as_id)
1167 spte_set |= write_protect_gfn(kvm, root, gfn);
1173 * Return the level of the lowest level SPTE added to sptes.
1174 * That SPTE may be non-present.
1176 int kvm_tdp_mmu_get_walk(struct kvm_vcpu *vcpu, u64 addr, u64 *sptes,
1179 struct tdp_iter iter;
1180 struct kvm_mmu *mmu = vcpu->arch.mmu;
1181 gfn_t gfn = addr >> PAGE_SHIFT;
1184 *root_level = vcpu->arch.mmu->shadow_root_level;
1186 tdp_mmu_for_each_pte(iter, mmu, gfn, gfn + 1) {
1188 sptes[leaf - 1] = iter.old_spte;