arch/x86/kvm/mmu/page_track.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Support KVM gust page tracking
   4  *
   5  * This feature allows us to track page access in guest. Currently, only
   6  * write access is tracked.
   7  *
   8  * Copyright(C) 2015 Intel Corporation.
   9  *
  10  * Author:
  11  *   Xiao Guangrong <guangrong.xiao@linux.intel.com>
  12  */
  13
  14 #include <linux/kvm_host.h>
  15 #include <linux/rculist.h>
  16
  17 #include <asm/kvm_page_track.h>
  18
  19 #include "mmu.h"
  20 #include "mmu_internal.h"
  21
  22 bool kvm_page_track_write_tracking_enabled(struct kvm *kvm)
  23 {
  24         return IS_ENABLED(CONFIG_KVM_EXTERNAL_WRITE_TRACKING) ||
  25                !tdp_enabled || kvm_shadow_root_allocated(kvm);
  26 }
  27
  28 void kvm_page_track_free_memslot(struct kvm_memory_slot *slot)
  29 {
  30         int i;
  31
  32         for (i = 0; i < KVM_PAGE_TRACK_MAX; i++) {
  33                 kvfree(slot->arch.gfn_track[i]);
  34                 slot->arch.gfn_track[i] = NULL;
  35         }
  36 }
  37
  38 int kvm_page_track_create_memslot(struct kvm *kvm,
  39                                   struct kvm_memory_slot *slot,
  40                                   unsigned long npages)
  41 {
  42         int i;
  43
  44         for (i = 0; i < KVM_PAGE_TRACK_MAX; i++) {
  45                 if (i == KVM_PAGE_TRACK_WRITE &&
  46                     !kvm_page_track_write_tracking_enabled(kvm))
  47                         continue;
  48
  49                 slot->arch.gfn_track[i] =
  50                         __vcalloc(npages, sizeof(*slot->arch.gfn_track[i]),
  51                                   GFP_KERNEL_ACCOUNT);
  52                 if (!slot->arch.gfn_track[i])
  53                         goto track_free;
  54         }
  55
  56         return 0;
  57
  58 track_free:
  59         kvm_page_track_free_memslot(slot);
  60         return -ENOMEM;
  61 }
  62
  63 static inline bool page_track_mode_is_valid(enum kvm_page_track_mode mode)
  64 {
  65         if (mode < 0 || mode >= KVM_PAGE_TRACK_MAX)
  66                 return false;
  67
  68         return true;
  69 }
  70
  71 int kvm_page_track_write_tracking_alloc(struct kvm_memory_slot *slot)
  72 {
  73         unsigned short *gfn_track;
  74
  75         if (slot->arch.gfn_track[KVM_PAGE_TRACK_WRITE])
  76                 return 0;
  77
  78         gfn_track = __vcalloc(slot->npages, sizeof(*gfn_track),
  79                               GFP_KERNEL_ACCOUNT);
  80         if (gfn_track == NULL)
  81                 return -ENOMEM;
  82
  83         slot->arch.gfn_track[KVM_PAGE_TRACK_WRITE] = gfn_track;
  84         return 0;
  85 }
  86
  87 static void update_gfn_track(struct kvm_memory_slot *slot, gfn_t gfn,
  88                              enum kvm_page_track_mode mode, short count)
  89 {
  90         int index, val;
  91
  92         index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K);
  93
  94         val = slot->arch.gfn_track[mode][index];
  95
  96         if (WARN_ON(val + count < 0 || val + count > USHRT_MAX))
  97                 return;
  98
  99         slot->arch.gfn_track[mode][index] += count;
 100 }
 101
 102 /*
 103  * add guest page to the tracking pool so that corresponding access on that
 104  * page will be intercepted.
 105  *
 106  * It should be called under the protection both of mmu-lock and kvm->srcu
 107  * or kvm->slots_lock.
 108  *
 109  * @kvm: the guest instance we are interested in.
 110  * @slot: the @gfn belongs to.
 111  * @gfn: the guest page.
 112  * @mode: tracking mode, currently only write track is supported.
 113  */
 114 void kvm_slot_page_track_add_page(struct kvm *kvm,
 115                                   struct kvm_memory_slot *slot, gfn_t gfn,
 116                                   enum kvm_page_track_mode mode)
 117 {
 118
 119         if (WARN_ON(!page_track_mode_is_valid(mode)))
 120                 return;
 121
 122         if (WARN_ON(mode == KVM_PAGE_TRACK_WRITE &&
 123                     !kvm_page_track_write_tracking_enabled(kvm)))
 124                 return;
 125
 126         update_gfn_track(slot, gfn, mode, 1);
 127
 128         /*
 129          * new track stops large page mapping for the
 130          * tracked page.
 131          */
 132         kvm_mmu_gfn_disallow_lpage(slot, gfn);
 133
 134         if (mode == KVM_PAGE_TRACK_WRITE)
 135                 if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn, PG_LEVEL_4K))
 136                         kvm_flush_remote_tlbs(kvm);
 137 }
 138 EXPORT_SYMBOL_GPL(kvm_slot_page_track_add_page);
 139
 140 /*
 141  * remove the guest page from the tracking pool which stops the interception
 142  * of corresponding access on that page. It is the opposed operation of
 143  * kvm_slot_page_track_add_page().
 144  *
 145  * It should be called under the protection both of mmu-lock and kvm->srcu
 146  * or kvm->slots_lock.
 147  *
 148  * @kvm: the guest instance we are interested in.
 149  * @slot: the @gfn belongs to.
 150  * @gfn: the guest page.
 151  * @mode: tracking mode, currently only write track is supported.
 152  */
 153 void kvm_slot_page_track_remove_page(struct kvm *kvm,
 154                                      struct kvm_memory_slot *slot, gfn_t gfn,
 155                                      enum kvm_page_track_mode mode)
 156 {
 157         if (WARN_ON(!page_track_mode_is_valid(mode)))
 158                 return;
 159
 160         if (WARN_ON(mode == KVM_PAGE_TRACK_WRITE &&
 161                     !kvm_page_track_write_tracking_enabled(kvm)))
 162                 return;
 163
 164         update_gfn_track(slot, gfn, mode, -1);
 165
 166         /*
 167          * allow large page mapping for the tracked page
 168          * after the tracker is gone.
 169          */
 170         kvm_mmu_gfn_allow_lpage(slot, gfn);
 171 }
 172 EXPORT_SYMBOL_GPL(kvm_slot_page_track_remove_page);
 173
 174 /*
 175  * check if the corresponding access on the specified guest page is tracked.
 176  */
 177 bool kvm_slot_page_track_is_active(struct kvm *kvm,
 178                                    const struct kvm_memory_slot *slot,
 179                                    gfn_t gfn, enum kvm_page_track_mode mode)
 180 {
 181         int index;
 182
 183         if (WARN_ON(!page_track_mode_is_valid(mode)))
 184                 return false;
 185
 186         if (!slot)
 187                 return false;
 188
 189         if (mode == KVM_PAGE_TRACK_WRITE &&
 190             !kvm_page_track_write_tracking_enabled(kvm))
 191                 return false;
 192
 193         index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K);
 194         return !!READ_ONCE(slot->arch.gfn_track[mode][index]);
 195 }
 196
 197 void kvm_page_track_cleanup(struct kvm *kvm)
 198 {
 199         struct kvm_page_track_notifier_head *head;
 200
 201         head = &kvm->arch.track_notifier_head;
 202         cleanup_srcu_struct(&head->track_srcu);
 203 }
 204
 205 int kvm_page_track_init(struct kvm *kvm)
 206 {
 207         struct kvm_page_track_notifier_head *head;
 208
 209         head = &kvm->arch.track_notifier_head;
 210         INIT_HLIST_HEAD(&head->track_notifier_list);
 211         return init_srcu_struct(&head->track_srcu);
 212 }
 213
 214 /*
 215  * register the notifier so that event interception for the tracked guest
 216  * pages can be received.
 217  */
 218 void
 219 kvm_page_track_register_notifier(struct kvm *kvm,
 220                                  struct kvm_page_track_notifier_node *n)
 221 {
 222         struct kvm_page_track_notifier_head *head;
 223
 224         head = &kvm->arch.track_notifier_head;
 225
 226         write_lock(&kvm->mmu_lock);
 227         hlist_add_head_rcu(&n->node, &head->track_notifier_list);
 228         write_unlock(&kvm->mmu_lock);
 229 }
 230 EXPORT_SYMBOL_GPL(kvm_page_track_register_notifier);
 231
 232 /*
 233  * stop receiving the event interception. It is the opposed operation of
 234  * kvm_page_track_register_notifier().
 235  */
 236 void
 237 kvm_page_track_unregister_notifier(struct kvm *kvm,
 238                                    struct kvm_page_track_notifier_node *n)
 239 {
 240         struct kvm_page_track_notifier_head *head;
 241
 242         head = &kvm->arch.track_notifier_head;
 243
 244         write_lock(&kvm->mmu_lock);
 245         hlist_del_rcu(&n->node);
 246         write_unlock(&kvm->mmu_lock);
 247         synchronize_srcu(&head->track_srcu);
 248 }
 249 EXPORT_SYMBOL_GPL(kvm_page_track_unregister_notifier);
 250
 251 /*
 252  * Notify the node that write access is intercepted and write emulation is
 253  * finished at this time.
 254  *
 255  * The node should figure out if the written page is the one that node is
 256  * interested in by itself.
 257  */
 258 void kvm_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new,
 259                           int bytes)
 260 {
 261         struct kvm_page_track_notifier_head *head;
 262         struct kvm_page_track_notifier_node *n;
 263         int idx;
 264
 265         head = &vcpu->kvm->arch.track_notifier_head;
 266
 267         if (hlist_empty(&head->track_notifier_list))
 268                 return;
 269
 270         idx = srcu_read_lock(&head->track_srcu);
 271         hlist_for_each_entry_srcu(n, &head->track_notifier_list, node,
 272                                 srcu_read_lock_held(&head->track_srcu))
 273                 if (n->track_write)
 274                         n->track_write(vcpu, gpa, new, bytes, n);
 275         srcu_read_unlock(&head->track_srcu, idx);
 276 }
 277
 278 /*
 279  * Notify the node that memory slot is being removed or moved so that it can
 280  * drop write-protection for the pages in the memory slot.
 281  *
 282  * The node should figure out it has any write-protected pages in this slot
 283  * by itself.
 284  */
 285 void kvm_page_track_flush_slot(struct kvm *kvm, struct kvm_memory_slot *slot)
 286 {
 287         struct kvm_page_track_notifier_head *head;
 288         struct kvm_page_track_notifier_node *n;
 289         int idx;
 290
 291         head = &kvm->arch.track_notifier_head;
 292
 293         if (hlist_empty(&head->track_notifier_list))
 294                 return;
 295
 296         idx = srcu_read_lock(&head->track_srcu);
 297         hlist_for_each_entry_srcu(n, &head->track_notifier_list, node,
 298                                 srcu_read_lock_held(&head->track_srcu))
 299                 if (n->track_flush_slot)
 300                         n->track_flush_slot(kvm, slot, n);
 301         srcu_read_unlock(&head->track_srcu, idx);
 302 }