]>
Commit | Line | Data |
---|---|---|
21ebbeda XG |
1 | /* |
2 | * Support KVM gust page tracking | |
3 | * | |
4 | * This feature allows us to track page access in guest. Currently, only | |
5 | * write access is tracked. | |
6 | * | |
7 | * Copyright(C) 2015 Intel Corporation. | |
8 | * | |
9 | * Author: | |
10 | * Xiao Guangrong <guangrong.xiao@linux.intel.com> | |
11 | * | |
12 | * This work is licensed under the terms of the GNU GPL, version 2. See | |
13 | * the COPYING file in the top-level directory. | |
14 | */ | |
15 | ||
16 | #include <linux/kvm_host.h> | |
b2d09103 IM |
17 | #include <linux/rculist.h> |
18 | ||
21ebbeda XG |
19 | #include <asm/kvm_host.h> |
20 | #include <asm/kvm_page_track.h> | |
21 | ||
22 | #include "mmu.h" | |
23 | ||
24 | void kvm_page_track_free_memslot(struct kvm_memory_slot *free, | |
25 | struct kvm_memory_slot *dont) | |
26 | { | |
27 | int i; | |
28 | ||
29 | for (i = 0; i < KVM_PAGE_TRACK_MAX; i++) | |
30 | if (!dont || free->arch.gfn_track[i] != | |
31 | dont->arch.gfn_track[i]) { | |
32 | kvfree(free->arch.gfn_track[i]); | |
33 | free->arch.gfn_track[i] = NULL; | |
34 | } | |
35 | } | |
36 | ||
37 | int kvm_page_track_create_memslot(struct kvm_memory_slot *slot, | |
38 | unsigned long npages) | |
39 | { | |
40 | int i; | |
41 | ||
42 | for (i = 0; i < KVM_PAGE_TRACK_MAX; i++) { | |
43 | slot->arch.gfn_track[i] = kvm_kvzalloc(npages * | |
44 | sizeof(*slot->arch.gfn_track[i])); | |
45 | if (!slot->arch.gfn_track[i]) | |
46 | goto track_free; | |
47 | } | |
48 | ||
49 | return 0; | |
50 | ||
51 | track_free: | |
52 | kvm_page_track_free_memslot(slot, NULL); | |
53 | return -ENOMEM; | |
54 | } | |
f29d4d78 XG |
55 | |
56 | static inline bool page_track_mode_is_valid(enum kvm_page_track_mode mode) | |
57 | { | |
58 | if (mode < 0 || mode >= KVM_PAGE_TRACK_MAX) | |
59 | return false; | |
60 | ||
61 | return true; | |
62 | } | |
63 | ||
64 | static void update_gfn_track(struct kvm_memory_slot *slot, gfn_t gfn, | |
65 | enum kvm_page_track_mode mode, short count) | |
66 | { | |
67 | int index, val; | |
68 | ||
69 | index = gfn_to_index(gfn, slot->base_gfn, PT_PAGE_TABLE_LEVEL); | |
70 | ||
71 | val = slot->arch.gfn_track[mode][index]; | |
72 | ||
73 | if (WARN_ON(val + count < 0 || val + count > USHRT_MAX)) | |
74 | return; | |
75 | ||
76 | slot->arch.gfn_track[mode][index] += count; | |
77 | } | |
78 | ||
79 | /* | |
80 | * add guest page to the tracking pool so that corresponding access on that | |
81 | * page will be intercepted. | |
82 | * | |
83 | * It should be called under the protection both of mmu-lock and kvm->srcu | |
84 | * or kvm->slots_lock. | |
85 | * | |
86 | * @kvm: the guest instance we are interested in. | |
87 | * @slot: the @gfn belongs to. | |
88 | * @gfn: the guest page. | |
89 | * @mode: tracking mode, currently only write track is supported. | |
90 | */ | |
91 | void kvm_slot_page_track_add_page(struct kvm *kvm, | |
92 | struct kvm_memory_slot *slot, gfn_t gfn, | |
93 | enum kvm_page_track_mode mode) | |
94 | { | |
95 | ||
96 | if (WARN_ON(!page_track_mode_is_valid(mode))) | |
97 | return; | |
98 | ||
99 | update_gfn_track(slot, gfn, mode, 1); | |
100 | ||
101 | /* | |
102 | * new track stops large page mapping for the | |
103 | * tracked page. | |
104 | */ | |
105 | kvm_mmu_gfn_disallow_lpage(slot, gfn); | |
106 | ||
107 | if (mode == KVM_PAGE_TRACK_WRITE) | |
108 | if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn)) | |
109 | kvm_flush_remote_tlbs(kvm); | |
110 | } | |
871b7ef2 | 111 | EXPORT_SYMBOL_GPL(kvm_slot_page_track_add_page); |
f29d4d78 XG |
112 | |
113 | /* | |
114 | * remove the guest page from the tracking pool which stops the interception | |
115 | * of corresponding access on that page. It is the opposed operation of | |
116 | * kvm_slot_page_track_add_page(). | |
117 | * | |
118 | * It should be called under the protection both of mmu-lock and kvm->srcu | |
119 | * or kvm->slots_lock. | |
120 | * | |
121 | * @kvm: the guest instance we are interested in. | |
122 | * @slot: the @gfn belongs to. | |
123 | * @gfn: the guest page. | |
124 | * @mode: tracking mode, currently only write track is supported. | |
125 | */ | |
126 | void kvm_slot_page_track_remove_page(struct kvm *kvm, | |
127 | struct kvm_memory_slot *slot, gfn_t gfn, | |
128 | enum kvm_page_track_mode mode) | |
129 | { | |
130 | if (WARN_ON(!page_track_mode_is_valid(mode))) | |
131 | return; | |
132 | ||
133 | update_gfn_track(slot, gfn, mode, -1); | |
134 | ||
135 | /* | |
136 | * allow large page mapping for the tracked page | |
137 | * after the tracker is gone. | |
138 | */ | |
139 | kvm_mmu_gfn_allow_lpage(slot, gfn); | |
140 | } | |
871b7ef2 | 141 | EXPORT_SYMBOL_GPL(kvm_slot_page_track_remove_page); |
3d0c27ad XG |
142 | |
143 | /* | |
144 | * check if the corresponding access on the specified guest page is tracked. | |
145 | */ | |
146 | bool kvm_page_track_is_active(struct kvm_vcpu *vcpu, gfn_t gfn, | |
147 | enum kvm_page_track_mode mode) | |
148 | { | |
a6adb106 PB |
149 | struct kvm_memory_slot *slot; |
150 | int index; | |
3d0c27ad XG |
151 | |
152 | if (WARN_ON(!page_track_mode_is_valid(mode))) | |
153 | return false; | |
154 | ||
a6adb106 PB |
155 | slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); |
156 | if (!slot) | |
157 | return false; | |
158 | ||
159 | index = gfn_to_index(gfn, slot->base_gfn, PT_PAGE_TABLE_LEVEL); | |
3d0c27ad XG |
160 | return !!ACCESS_ONCE(slot->arch.gfn_track[mode][index]); |
161 | } | |
0eb05bf2 | 162 | |
2beb6dad PB |
163 | void kvm_page_track_cleanup(struct kvm *kvm) |
164 | { | |
165 | struct kvm_page_track_notifier_head *head; | |
166 | ||
167 | head = &kvm->arch.track_notifier_head; | |
168 | cleanup_srcu_struct(&head->track_srcu); | |
169 | } | |
170 | ||
0eb05bf2 XG |
171 | void kvm_page_track_init(struct kvm *kvm) |
172 | { | |
173 | struct kvm_page_track_notifier_head *head; | |
174 | ||
175 | head = &kvm->arch.track_notifier_head; | |
176 | init_srcu_struct(&head->track_srcu); | |
177 | INIT_HLIST_HEAD(&head->track_notifier_list); | |
178 | } | |
179 | ||
180 | /* | |
181 | * register the notifier so that event interception for the tracked guest | |
182 | * pages can be received. | |
183 | */ | |
184 | void | |
185 | kvm_page_track_register_notifier(struct kvm *kvm, | |
186 | struct kvm_page_track_notifier_node *n) | |
187 | { | |
188 | struct kvm_page_track_notifier_head *head; | |
189 | ||
190 | head = &kvm->arch.track_notifier_head; | |
191 | ||
192 | spin_lock(&kvm->mmu_lock); | |
193 | hlist_add_head_rcu(&n->node, &head->track_notifier_list); | |
194 | spin_unlock(&kvm->mmu_lock); | |
195 | } | |
871b7ef2 | 196 | EXPORT_SYMBOL_GPL(kvm_page_track_register_notifier); |
0eb05bf2 XG |
197 | |
198 | /* | |
199 | * stop receiving the event interception. It is the opposed operation of | |
200 | * kvm_page_track_register_notifier(). | |
201 | */ | |
202 | void | |
203 | kvm_page_track_unregister_notifier(struct kvm *kvm, | |
204 | struct kvm_page_track_notifier_node *n) | |
205 | { | |
206 | struct kvm_page_track_notifier_head *head; | |
207 | ||
208 | head = &kvm->arch.track_notifier_head; | |
209 | ||
210 | spin_lock(&kvm->mmu_lock); | |
211 | hlist_del_rcu(&n->node); | |
212 | spin_unlock(&kvm->mmu_lock); | |
213 | synchronize_srcu(&head->track_srcu); | |
214 | } | |
871b7ef2 | 215 | EXPORT_SYMBOL_GPL(kvm_page_track_unregister_notifier); |
0eb05bf2 XG |
216 | |
217 | /* | |
218 | * Notify the node that write access is intercepted and write emulation is | |
219 | * finished at this time. | |
220 | * | |
221 | * The node should figure out if the written page is the one that node is | |
222 | * interested in by itself. | |
223 | */ | |
224 | void kvm_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new, | |
225 | int bytes) | |
226 | { | |
227 | struct kvm_page_track_notifier_head *head; | |
228 | struct kvm_page_track_notifier_node *n; | |
229 | int idx; | |
230 | ||
231 | head = &vcpu->kvm->arch.track_notifier_head; | |
232 | ||
233 | if (hlist_empty(&head->track_notifier_list)) | |
234 | return; | |
235 | ||
236 | idx = srcu_read_lock(&head->track_srcu); | |
237 | hlist_for_each_entry_rcu(n, &head->track_notifier_list, node) | |
238 | if (n->track_write) | |
d126363d | 239 | n->track_write(vcpu, gpa, new, bytes, n); |
0eb05bf2 XG |
240 | srcu_read_unlock(&head->track_srcu, idx); |
241 | } | |
ae7cd873 XC |
242 | |
243 | /* | |
244 | * Notify the node that memory slot is being removed or moved so that it can | |
245 | * drop write-protection for the pages in the memory slot. | |
246 | * | |
247 | * The node should figure out it has any write-protected pages in this slot | |
248 | * by itself. | |
249 | */ | |
250 | void kvm_page_track_flush_slot(struct kvm *kvm, struct kvm_memory_slot *slot) | |
251 | { | |
252 | struct kvm_page_track_notifier_head *head; | |
253 | struct kvm_page_track_notifier_node *n; | |
254 | int idx; | |
255 | ||
256 | head = &kvm->arch.track_notifier_head; | |
257 | ||
258 | if (hlist_empty(&head->track_notifier_list)) | |
259 | return; | |
260 | ||
261 | idx = srcu_read_lock(&head->track_srcu); | |
262 | hlist_for_each_entry_rcu(n, &head->track_notifier_list, node) | |
263 | if (n->track_flush_slot) | |
d126363d | 264 | n->track_flush_slot(kvm, slot, n); |
ae7cd873 XC |
265 | srcu_read_unlock(&head->track_srcu, idx); |
266 | } |