]>
Commit | Line | Data |
---|---|---|
eaf78265 JR |
1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* | |
3 | * Kernel-based Virtual Machine driver for Linux | |
4 | * | |
5 | * AMD SVM-SEV support | |
6 | * | |
7 | * Copyright 2010 Red Hat, Inc. and/or its affiliates. | |
8 | */ | |
9 | ||
10 | #include <linux/kvm_types.h> | |
11 | #include <linux/kvm_host.h> | |
12 | #include <linux/kernel.h> | |
13 | #include <linux/highmem.h> | |
14 | #include <linux/psp-sev.h> | |
b2bce0a5 | 15 | #include <linux/pagemap.h> |
eaf78265 | 16 | #include <linux/swap.h> |
7aef27f0 | 17 | #include <linux/misc_cgroup.h> |
add5e2f0 | 18 | #include <linux/processor.h> |
d523ab6b | 19 | #include <linux/trace_events.h> |
86137773 | 20 | #include <asm/fpu/internal.h> |
eaf78265 | 21 | |
784a4661 | 22 | #include <asm/pkru.h> |
8640ca58 TL |
23 | #include <asm/trapnr.h> |
24 | ||
eaf78265 JR |
25 | #include "x86.h" |
26 | #include "svm.h" | |
35a78319 | 27 | #include "svm_ops.h" |
291bd20d | 28 | #include "cpuid.h" |
d523ab6b | 29 | #include "trace.h" |
eaf78265 | 30 | |
7aef27f0 VS |
31 | #ifndef CONFIG_KVM_AMD_SEV |
32 | /* | |
33 | * When this config is not defined, SEV feature is not supported and APIs in | |
34 | * this file are not used but this file still gets compiled into the KVM AMD | |
35 | * module. | |
36 | * | |
37 | * We will not have MISC_CG_RES_SEV and MISC_CG_RES_SEV_ES entries in the enum | |
38 | * misc_res_type {} defined in linux/misc_cgroup.h. | |
39 | * | |
40 | * Below macros allow compilation to succeed. | |
41 | */ | |
42 | #define MISC_CG_RES_SEV MISC_CG_RES_TYPES | |
43 | #define MISC_CG_RES_SEV_ES MISC_CG_RES_TYPES | |
44 | #endif | |
45 | ||
a479c334 | 46 | #ifdef CONFIG_KVM_AMD_SEV |
e8126bda | 47 | /* enable/disable SEV support */ |
6c2c7bf5 | 48 | static bool sev_enabled = true; |
8d364a07 | 49 | module_param_named(sev, sev_enabled, bool, 0444); |
e8126bda SC |
50 | |
51 | /* enable/disable SEV-ES support */ | |
6c2c7bf5 | 52 | static bool sev_es_enabled = true; |
8d364a07 | 53 | module_param_named(sev_es, sev_es_enabled, bool, 0444); |
a479c334 SC |
54 | #else |
55 | #define sev_enabled false | |
56 | #define sev_es_enabled false | |
57 | #endif /* CONFIG_KVM_AMD_SEV */ | |
e8126bda | 58 | |
1edc1459 | 59 | static u8 sev_enc_bit; |
eaf78265 JR |
60 | static DECLARE_RWSEM(sev_deactivate_lock); |
61 | static DEFINE_MUTEX(sev_bitmap_lock); | |
62 | unsigned int max_sev_asid; | |
63 | static unsigned int min_sev_asid; | |
d3d1af85 | 64 | static unsigned long sev_me_mask; |
bb2baeb2 | 65 | static unsigned int nr_asids; |
eaf78265 JR |
66 | static unsigned long *sev_asid_bitmap; |
67 | static unsigned long *sev_reclaim_asid_bitmap; | |
eaf78265 JR |
68 | |
69 | struct enc_region { | |
70 | struct list_head list; | |
71 | unsigned long npages; | |
72 | struct page **pages; | |
73 | unsigned long uaddr; | |
74 | unsigned long size; | |
75 | }; | |
76 | ||
469bb32b SC |
77 | /* Called with the sev_bitmap_lock held, or on shutdown */ |
78 | static int sev_flush_asids(int min_asid, int max_asid) | |
eaf78265 | 79 | { |
bb2baeb2 | 80 | int ret, asid, error = 0; |
469bb32b SC |
81 | |
82 | /* Check if there are any ASIDs to reclaim before performing a flush */ | |
bb2baeb2 MZ |
83 | asid = find_next_bit(sev_reclaim_asid_bitmap, nr_asids, min_asid); |
84 | if (asid > max_asid) | |
469bb32b | 85 | return -EBUSY; |
eaf78265 JR |
86 | |
87 | /* | |
88 | * DEACTIVATE will clear the WBINVD indicator causing DF_FLUSH to fail, | |
89 | * so it must be guarded. | |
90 | */ | |
91 | down_write(&sev_deactivate_lock); | |
92 | ||
93 | wbinvd_on_all_cpus(); | |
94 | ret = sev_guest_df_flush(&error); | |
95 | ||
96 | up_write(&sev_deactivate_lock); | |
97 | ||
98 | if (ret) | |
99 | pr_err("SEV: DF_FLUSH failed, ret=%d, error=%#x\n", ret, error); | |
100 | ||
101 | return ret; | |
102 | } | |
103 | ||
54526d1f NT |
104 | static inline bool is_mirroring_enc_context(struct kvm *kvm) |
105 | { | |
106 | return !!to_kvm_svm(kvm)->sev_info.enc_context_owner; | |
107 | } | |
108 | ||
eaf78265 | 109 | /* Must be called with the sev_bitmap_lock held */ |
80675b3a | 110 | static bool __sev_recycle_asids(int min_asid, int max_asid) |
eaf78265 | 111 | { |
469bb32b | 112 | if (sev_flush_asids(min_asid, max_asid)) |
eaf78265 JR |
113 | return false; |
114 | ||
80675b3a | 115 | /* The flush process will flush all reclaimable SEV and SEV-ES ASIDs */ |
eaf78265 | 116 | bitmap_xor(sev_asid_bitmap, sev_asid_bitmap, sev_reclaim_asid_bitmap, |
bb2baeb2 MZ |
117 | nr_asids); |
118 | bitmap_zero(sev_reclaim_asid_bitmap, nr_asids); | |
eaf78265 JR |
119 | |
120 | return true; | |
121 | } | |
122 | ||
80675b3a | 123 | static int sev_asid_new(struct kvm_sev_info *sev) |
eaf78265 | 124 | { |
bb2baeb2 | 125 | int asid, min_asid, max_asid, ret; |
eaf78265 | 126 | bool retry = true; |
7aef27f0 VS |
127 | enum misc_res_type type; |
128 | ||
129 | type = sev->es_active ? MISC_CG_RES_SEV_ES : MISC_CG_RES_SEV; | |
130 | WARN_ON(sev->misc_cg); | |
131 | sev->misc_cg = get_current_misc_cg(); | |
132 | ret = misc_cg_try_charge(type, sev->misc_cg, 1); | |
133 | if (ret) { | |
134 | put_misc_cg(sev->misc_cg); | |
135 | sev->misc_cg = NULL; | |
136 | return ret; | |
137 | } | |
eaf78265 JR |
138 | |
139 | mutex_lock(&sev_bitmap_lock); | |
140 | ||
141 | /* | |
80675b3a TL |
142 | * SEV-enabled guests must use asid from min_sev_asid to max_sev_asid. |
143 | * SEV-ES-enabled guest can use from 1 to min_sev_asid - 1. | |
eaf78265 | 144 | */ |
bb2baeb2 | 145 | min_asid = sev->es_active ? 1 : min_sev_asid; |
80675b3a | 146 | max_asid = sev->es_active ? min_sev_asid - 1 : max_sev_asid; |
eaf78265 | 147 | again: |
bb2baeb2 MZ |
148 | asid = find_next_zero_bit(sev_asid_bitmap, max_asid + 1, min_asid); |
149 | if (asid > max_asid) { | |
80675b3a | 150 | if (retry && __sev_recycle_asids(min_asid, max_asid)) { |
eaf78265 JR |
151 | retry = false; |
152 | goto again; | |
153 | } | |
154 | mutex_unlock(&sev_bitmap_lock); | |
7aef27f0 VS |
155 | ret = -EBUSY; |
156 | goto e_uncharge; | |
eaf78265 JR |
157 | } |
158 | ||
bb2baeb2 | 159 | __set_bit(asid, sev_asid_bitmap); |
eaf78265 JR |
160 | |
161 | mutex_unlock(&sev_bitmap_lock); | |
162 | ||
bb2baeb2 | 163 | return asid; |
7aef27f0 VS |
164 | e_uncharge: |
165 | misc_cg_uncharge(type, sev->misc_cg, 1); | |
166 | put_misc_cg(sev->misc_cg); | |
167 | sev->misc_cg = NULL; | |
168 | return ret; | |
eaf78265 JR |
169 | } |
170 | ||
171 | static int sev_get_asid(struct kvm *kvm) | |
172 | { | |
173 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
174 | ||
175 | return sev->asid; | |
176 | } | |
177 | ||
7aef27f0 | 178 | static void sev_asid_free(struct kvm_sev_info *sev) |
eaf78265 JR |
179 | { |
180 | struct svm_cpu_data *sd; | |
bb2baeb2 | 181 | int cpu; |
7aef27f0 | 182 | enum misc_res_type type; |
eaf78265 JR |
183 | |
184 | mutex_lock(&sev_bitmap_lock); | |
185 | ||
bb2baeb2 | 186 | __set_bit(sev->asid, sev_reclaim_asid_bitmap); |
eaf78265 JR |
187 | |
188 | for_each_possible_cpu(cpu) { | |
189 | sd = per_cpu(svm_data, cpu); | |
179c6c27 | 190 | sd->sev_vmcbs[sev->asid] = NULL; |
eaf78265 JR |
191 | } |
192 | ||
193 | mutex_unlock(&sev_bitmap_lock); | |
7aef27f0 VS |
194 | |
195 | type = sev->es_active ? MISC_CG_RES_SEV_ES : MISC_CG_RES_SEV; | |
196 | misc_cg_uncharge(type, sev->misc_cg, 1); | |
197 | put_misc_cg(sev->misc_cg); | |
198 | sev->misc_cg = NULL; | |
eaf78265 JR |
199 | } |
200 | ||
934002cd | 201 | static void sev_decommission(unsigned int handle) |
eaf78265 | 202 | { |
238eca82 | 203 | struct sev_data_decommission decommission; |
934002cd AG |
204 | |
205 | if (!handle) | |
206 | return; | |
207 | ||
208 | decommission.handle = handle; | |
209 | sev_guest_decommission(&decommission, NULL); | |
210 | } | |
211 | ||
212 | static void sev_unbind_asid(struct kvm *kvm, unsigned int handle) | |
213 | { | |
238eca82 | 214 | struct sev_data_deactivate deactivate; |
eaf78265 JR |
215 | |
216 | if (!handle) | |
217 | return; | |
218 | ||
238eca82 | 219 | deactivate.handle = handle; |
eaf78265 JR |
220 | |
221 | /* Guard DEACTIVATE against WBINVD/DF_FLUSH used in ASID recycling */ | |
222 | down_read(&sev_deactivate_lock); | |
238eca82 | 223 | sev_guest_deactivate(&deactivate, NULL); |
eaf78265 JR |
224 | up_read(&sev_deactivate_lock); |
225 | ||
934002cd | 226 | sev_decommission(handle); |
eaf78265 JR |
227 | } |
228 | ||
229 | static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp) | |
230 | { | |
231 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
9fa1521d | 232 | bool es_active = argp->id == KVM_SEV_ES_INIT; |
eaf78265 JR |
233 | int asid, ret; |
234 | ||
8727906f SC |
235 | if (kvm->created_vcpus) |
236 | return -EINVAL; | |
237 | ||
eaf78265 JR |
238 | ret = -EBUSY; |
239 | if (unlikely(sev->active)) | |
240 | return ret; | |
241 | ||
fd49e8ee | 242 | sev->es_active = es_active; |
80675b3a | 243 | asid = sev_asid_new(sev); |
eaf78265 | 244 | if (asid < 0) |
fd49e8ee | 245 | goto e_no_asid; |
7aef27f0 | 246 | sev->asid = asid; |
eaf78265 JR |
247 | |
248 | ret = sev_platform_init(&argp->error); | |
249 | if (ret) | |
250 | goto e_free; | |
251 | ||
252 | sev->active = true; | |
253 | sev->asid = asid; | |
254 | INIT_LIST_HEAD(&sev->regions_list); | |
255 | ||
256 | return 0; | |
257 | ||
258 | e_free: | |
7aef27f0 VS |
259 | sev_asid_free(sev); |
260 | sev->asid = 0; | |
fd49e8ee PB |
261 | e_no_asid: |
262 | sev->es_active = false; | |
eaf78265 JR |
263 | return ret; |
264 | } | |
265 | ||
266 | static int sev_bind_asid(struct kvm *kvm, unsigned int handle, int *error) | |
267 | { | |
238eca82 | 268 | struct sev_data_activate activate; |
eaf78265 JR |
269 | int asid = sev_get_asid(kvm); |
270 | int ret; | |
271 | ||
eaf78265 | 272 | /* activate ASID on the given handle */ |
238eca82 SC |
273 | activate.handle = handle; |
274 | activate.asid = asid; | |
275 | ret = sev_guest_activate(&activate, error); | |
eaf78265 JR |
276 | |
277 | return ret; | |
278 | } | |
279 | ||
280 | static int __sev_issue_cmd(int fd, int id, void *data, int *error) | |
281 | { | |
282 | struct fd f; | |
283 | int ret; | |
284 | ||
285 | f = fdget(fd); | |
286 | if (!f.file) | |
287 | return -EBADF; | |
288 | ||
289 | ret = sev_issue_cmd_external_user(f.file, id, data, error); | |
290 | ||
291 | fdput(f); | |
292 | return ret; | |
293 | } | |
294 | ||
295 | static int sev_issue_cmd(struct kvm *kvm, int id, void *data, int *error) | |
296 | { | |
297 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
298 | ||
299 | return __sev_issue_cmd(sev->fd, id, data, error); | |
300 | } | |
301 | ||
302 | static int sev_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp) | |
303 | { | |
304 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
238eca82 | 305 | struct sev_data_launch_start start; |
eaf78265 JR |
306 | struct kvm_sev_launch_start params; |
307 | void *dh_blob, *session_blob; | |
308 | int *error = &argp->error; | |
309 | int ret; | |
310 | ||
311 | if (!sev_guest(kvm)) | |
312 | return -ENOTTY; | |
313 | ||
314 | if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, sizeof(params))) | |
315 | return -EFAULT; | |
316 | ||
238eca82 | 317 | memset(&start, 0, sizeof(start)); |
eaf78265 JR |
318 | |
319 | dh_blob = NULL; | |
320 | if (params.dh_uaddr) { | |
321 | dh_blob = psp_copy_user_blob(params.dh_uaddr, params.dh_len); | |
238eca82 SC |
322 | if (IS_ERR(dh_blob)) |
323 | return PTR_ERR(dh_blob); | |
eaf78265 | 324 | |
238eca82 SC |
325 | start.dh_cert_address = __sme_set(__pa(dh_blob)); |
326 | start.dh_cert_len = params.dh_len; | |
eaf78265 JR |
327 | } |
328 | ||
329 | session_blob = NULL; | |
330 | if (params.session_uaddr) { | |
331 | session_blob = psp_copy_user_blob(params.session_uaddr, params.session_len); | |
332 | if (IS_ERR(session_blob)) { | |
333 | ret = PTR_ERR(session_blob); | |
334 | goto e_free_dh; | |
335 | } | |
336 | ||
238eca82 SC |
337 | start.session_address = __sme_set(__pa(session_blob)); |
338 | start.session_len = params.session_len; | |
eaf78265 JR |
339 | } |
340 | ||
238eca82 SC |
341 | start.handle = params.handle; |
342 | start.policy = params.policy; | |
eaf78265 JR |
343 | |
344 | /* create memory encryption context */ | |
238eca82 | 345 | ret = __sev_issue_cmd(argp->sev_fd, SEV_CMD_LAUNCH_START, &start, error); |
eaf78265 JR |
346 | if (ret) |
347 | goto e_free_session; | |
348 | ||
349 | /* Bind ASID to this guest */ | |
238eca82 | 350 | ret = sev_bind_asid(kvm, start.handle, error); |
934002cd AG |
351 | if (ret) { |
352 | sev_decommission(start.handle); | |
eaf78265 | 353 | goto e_free_session; |
934002cd | 354 | } |
eaf78265 JR |
355 | |
356 | /* return handle to userspace */ | |
238eca82 | 357 | params.handle = start.handle; |
eaf78265 | 358 | if (copy_to_user((void __user *)(uintptr_t)argp->data, ¶ms, sizeof(params))) { |
238eca82 | 359 | sev_unbind_asid(kvm, start.handle); |
eaf78265 JR |
360 | ret = -EFAULT; |
361 | goto e_free_session; | |
362 | } | |
363 | ||
238eca82 | 364 | sev->handle = start.handle; |
eaf78265 JR |
365 | sev->fd = argp->sev_fd; |
366 | ||
367 | e_free_session: | |
368 | kfree(session_blob); | |
369 | e_free_dh: | |
370 | kfree(dh_blob); | |
eaf78265 JR |
371 | return ret; |
372 | } | |
373 | ||
374 | static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr, | |
375 | unsigned long ulen, unsigned long *n, | |
376 | int write) | |
377 | { | |
378 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
78824fab JH |
379 | unsigned long npages, size; |
380 | int npinned; | |
eaf78265 JR |
381 | unsigned long locked, lock_limit; |
382 | struct page **pages; | |
383 | unsigned long first, last; | |
ff2bd9ff | 384 | int ret; |
eaf78265 | 385 | |
19a23da5 PG |
386 | lockdep_assert_held(&kvm->lock); |
387 | ||
eaf78265 | 388 | if (ulen == 0 || uaddr + ulen < uaddr) |
a8d908b5 | 389 | return ERR_PTR(-EINVAL); |
eaf78265 JR |
390 | |
391 | /* Calculate number of pages. */ | |
392 | first = (uaddr & PAGE_MASK) >> PAGE_SHIFT; | |
393 | last = ((uaddr + ulen - 1) & PAGE_MASK) >> PAGE_SHIFT; | |
394 | npages = (last - first + 1); | |
395 | ||
396 | locked = sev->pages_locked + npages; | |
397 | lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; | |
398 | if (locked > lock_limit && !capable(CAP_IPC_LOCK)) { | |
399 | pr_err("SEV: %lu locked pages exceed the lock limit of %lu.\n", locked, lock_limit); | |
a8d908b5 | 400 | return ERR_PTR(-ENOMEM); |
eaf78265 JR |
401 | } |
402 | ||
78824fab | 403 | if (WARN_ON_ONCE(npages > INT_MAX)) |
a8d908b5 | 404 | return ERR_PTR(-EINVAL); |
78824fab | 405 | |
eaf78265 JR |
406 | /* Avoid using vmalloc for smaller buffers. */ |
407 | size = npages * sizeof(struct page *); | |
408 | if (size > PAGE_SIZE) | |
88dca4ca | 409 | pages = __vmalloc(size, GFP_KERNEL_ACCOUNT | __GFP_ZERO); |
eaf78265 JR |
410 | else |
411 | pages = kmalloc(size, GFP_KERNEL_ACCOUNT); | |
412 | ||
413 | if (!pages) | |
a8d908b5 | 414 | return ERR_PTR(-ENOMEM); |
eaf78265 JR |
415 | |
416 | /* Pin the user virtual address. */ | |
dc42c8ae | 417 | npinned = pin_user_pages_fast(uaddr, npages, write ? FOLL_WRITE : 0, pages); |
eaf78265 JR |
418 | if (npinned != npages) { |
419 | pr_err("SEV: Failure locking %lu pages.\n", npages); | |
ff2bd9ff | 420 | ret = -ENOMEM; |
eaf78265 JR |
421 | goto err; |
422 | } | |
423 | ||
424 | *n = npages; | |
425 | sev->pages_locked = locked; | |
426 | ||
427 | return pages; | |
428 | ||
429 | err: | |
ff2bd9ff | 430 | if (npinned > 0) |
dc42c8ae | 431 | unpin_user_pages(pages, npinned); |
eaf78265 JR |
432 | |
433 | kvfree(pages); | |
ff2bd9ff | 434 | return ERR_PTR(ret); |
eaf78265 JR |
435 | } |
436 | ||
437 | static void sev_unpin_memory(struct kvm *kvm, struct page **pages, | |
438 | unsigned long npages) | |
439 | { | |
440 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
441 | ||
dc42c8ae | 442 | unpin_user_pages(pages, npages); |
eaf78265 JR |
443 | kvfree(pages); |
444 | sev->pages_locked -= npages; | |
445 | } | |
446 | ||
447 | static void sev_clflush_pages(struct page *pages[], unsigned long npages) | |
448 | { | |
449 | uint8_t *page_virtual; | |
450 | unsigned long i; | |
451 | ||
e1ebb2b4 KS |
452 | if (this_cpu_has(X86_FEATURE_SME_COHERENT) || npages == 0 || |
453 | pages == NULL) | |
eaf78265 JR |
454 | return; |
455 | ||
456 | for (i = 0; i < npages; i++) { | |
457 | page_virtual = kmap_atomic(pages[i]); | |
458 | clflush_cache_range(page_virtual, PAGE_SIZE); | |
459 | kunmap_atomic(page_virtual); | |
460 | } | |
461 | } | |
462 | ||
463 | static unsigned long get_num_contig_pages(unsigned long idx, | |
464 | struct page **inpages, unsigned long npages) | |
465 | { | |
466 | unsigned long paddr, next_paddr; | |
467 | unsigned long i = idx + 1, pages = 1; | |
468 | ||
469 | /* find the number of contiguous pages starting from idx */ | |
470 | paddr = __sme_page_pa(inpages[idx]); | |
471 | while (i < npages) { | |
472 | next_paddr = __sme_page_pa(inpages[i++]); | |
473 | if ((paddr + PAGE_SIZE) == next_paddr) { | |
474 | pages++; | |
475 | paddr = next_paddr; | |
476 | continue; | |
477 | } | |
478 | break; | |
479 | } | |
480 | ||
481 | return pages; | |
482 | } | |
483 | ||
484 | static int sev_launch_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp) | |
485 | { | |
486 | unsigned long vaddr, vaddr_end, next_vaddr, npages, pages, size, i; | |
487 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
488 | struct kvm_sev_launch_update_data params; | |
238eca82 | 489 | struct sev_data_launch_update_data data; |
eaf78265 JR |
490 | struct page **inpages; |
491 | int ret; | |
492 | ||
493 | if (!sev_guest(kvm)) | |
494 | return -ENOTTY; | |
495 | ||
496 | if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, sizeof(params))) | |
497 | return -EFAULT; | |
498 | ||
eaf78265 JR |
499 | vaddr = params.uaddr; |
500 | size = params.len; | |
501 | vaddr_end = vaddr + size; | |
502 | ||
503 | /* Lock the user memory. */ | |
504 | inpages = sev_pin_memory(kvm, vaddr, size, &npages, 1); | |
238eca82 SC |
505 | if (IS_ERR(inpages)) |
506 | return PTR_ERR(inpages); | |
eaf78265 JR |
507 | |
508 | /* | |
14e3dd8d PB |
509 | * Flush (on non-coherent CPUs) before LAUNCH_UPDATE encrypts pages in |
510 | * place; the cache may contain the data that was written unencrypted. | |
eaf78265 JR |
511 | */ |
512 | sev_clflush_pages(inpages, npages); | |
513 | ||
238eca82 SC |
514 | data.reserved = 0; |
515 | data.handle = sev->handle; | |
516 | ||
eaf78265 JR |
517 | for (i = 0; vaddr < vaddr_end; vaddr = next_vaddr, i += pages) { |
518 | int offset, len; | |
519 | ||
520 | /* | |
521 | * If the user buffer is not page-aligned, calculate the offset | |
522 | * within the page. | |
523 | */ | |
524 | offset = vaddr & (PAGE_SIZE - 1); | |
525 | ||
526 | /* Calculate the number of pages that can be encrypted in one go. */ | |
527 | pages = get_num_contig_pages(i, inpages, npages); | |
528 | ||
529 | len = min_t(size_t, ((pages * PAGE_SIZE) - offset), size); | |
530 | ||
238eca82 SC |
531 | data.len = len; |
532 | data.address = __sme_page_pa(inpages[i]) + offset; | |
533 | ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_DATA, &data, &argp->error); | |
eaf78265 JR |
534 | if (ret) |
535 | goto e_unpin; | |
536 | ||
537 | size -= len; | |
538 | next_vaddr = vaddr + len; | |
539 | } | |
540 | ||
541 | e_unpin: | |
542 | /* content of memory is updated, mark pages dirty */ | |
543 | for (i = 0; i < npages; i++) { | |
544 | set_page_dirty_lock(inpages[i]); | |
545 | mark_page_accessed(inpages[i]); | |
546 | } | |
547 | /* unlock the user pages */ | |
548 | sev_unpin_memory(kvm, inpages, npages); | |
eaf78265 JR |
549 | return ret; |
550 | } | |
551 | ||
ad73109a TL |
552 | static int sev_es_sync_vmsa(struct vcpu_svm *svm) |
553 | { | |
554 | struct vmcb_save_area *save = &svm->vmcb->save; | |
555 | ||
556 | /* Check some debug related fields before encrypting the VMSA */ | |
557 | if (svm->vcpu.guest_debug || (save->dr7 & ~DR7_FIXED_1)) | |
558 | return -EINVAL; | |
559 | ||
560 | /* Sync registgers */ | |
561 | save->rax = svm->vcpu.arch.regs[VCPU_REGS_RAX]; | |
562 | save->rbx = svm->vcpu.arch.regs[VCPU_REGS_RBX]; | |
563 | save->rcx = svm->vcpu.arch.regs[VCPU_REGS_RCX]; | |
564 | save->rdx = svm->vcpu.arch.regs[VCPU_REGS_RDX]; | |
565 | save->rsp = svm->vcpu.arch.regs[VCPU_REGS_RSP]; | |
566 | save->rbp = svm->vcpu.arch.regs[VCPU_REGS_RBP]; | |
567 | save->rsi = svm->vcpu.arch.regs[VCPU_REGS_RSI]; | |
568 | save->rdi = svm->vcpu.arch.regs[VCPU_REGS_RDI]; | |
d45f89f7 | 569 | #ifdef CONFIG_X86_64 |
ad73109a TL |
570 | save->r8 = svm->vcpu.arch.regs[VCPU_REGS_R8]; |
571 | save->r9 = svm->vcpu.arch.regs[VCPU_REGS_R9]; | |
572 | save->r10 = svm->vcpu.arch.regs[VCPU_REGS_R10]; | |
573 | save->r11 = svm->vcpu.arch.regs[VCPU_REGS_R11]; | |
574 | save->r12 = svm->vcpu.arch.regs[VCPU_REGS_R12]; | |
575 | save->r13 = svm->vcpu.arch.regs[VCPU_REGS_R13]; | |
576 | save->r14 = svm->vcpu.arch.regs[VCPU_REGS_R14]; | |
577 | save->r15 = svm->vcpu.arch.regs[VCPU_REGS_R15]; | |
d45f89f7 | 578 | #endif |
ad73109a TL |
579 | save->rip = svm->vcpu.arch.regs[VCPU_REGS_RIP]; |
580 | ||
581 | /* Sync some non-GPR registers before encrypting */ | |
582 | save->xcr0 = svm->vcpu.arch.xcr0; | |
583 | save->pkru = svm->vcpu.arch.pkru; | |
584 | save->xss = svm->vcpu.arch.ia32_xss; | |
d0f9f826 | 585 | save->dr6 = svm->vcpu.arch.dr6; |
ad73109a TL |
586 | |
587 | /* | |
588 | * SEV-ES will use a VMSA that is pointed to by the VMCB, not | |
589 | * the traditional VMSA that is part of the VMCB. Copy the | |
590 | * traditional VMSA as it has been built so far (in prep | |
591 | * for LAUNCH_UPDATE_VMSA) to be the initial SEV-ES state. | |
592 | */ | |
593 | memcpy(svm->vmsa, save, sizeof(*save)); | |
594 | ||
595 | return 0; | |
596 | } | |
597 | ||
bb18a677 PG |
598 | static int __sev_launch_update_vmsa(struct kvm *kvm, struct kvm_vcpu *vcpu, |
599 | int *error) | |
ad73109a | 600 | { |
238eca82 | 601 | struct sev_data_launch_update_vmsa vmsa; |
bb18a677 PG |
602 | struct vcpu_svm *svm = to_svm(vcpu); |
603 | int ret; | |
604 | ||
605 | /* Perform some pre-encryption checks against the VMSA */ | |
606 | ret = sev_es_sync_vmsa(svm); | |
607 | if (ret) | |
608 | return ret; | |
609 | ||
610 | /* | |
611 | * The LAUNCH_UPDATE_VMSA command will perform in-place encryption of | |
612 | * the VMSA memory content (i.e it will write the same memory region | |
613 | * with the guest's key), so invalidate it first. | |
614 | */ | |
615 | clflush_cache_range(svm->vmsa, PAGE_SIZE); | |
616 | ||
617 | vmsa.reserved = 0; | |
618 | vmsa.handle = to_kvm_svm(kvm)->sev_info.handle; | |
619 | vmsa.address = __sme_pa(svm->vmsa); | |
620 | vmsa.len = PAGE_SIZE; | |
baa1e5ca PG |
621 | ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_VMSA, &vmsa, error); |
622 | if (ret) | |
623 | return ret; | |
624 | ||
625 | vcpu->arch.guest_state_protected = true; | |
626 | return 0; | |
bb18a677 PG |
627 | } |
628 | ||
629 | static int sev_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) | |
630 | { | |
c36b16d2 | 631 | struct kvm_vcpu *vcpu; |
ad73109a TL |
632 | int i, ret; |
633 | ||
634 | if (!sev_es_guest(kvm)) | |
635 | return -ENOTTY; | |
636 | ||
c36b16d2 | 637 | kvm_for_each_vcpu(i, vcpu, kvm) { |
bb18a677 | 638 | ret = mutex_lock_killable(&vcpu->mutex); |
ad73109a | 639 | if (ret) |
238eca82 | 640 | return ret; |
ad73109a | 641 | |
bb18a677 | 642 | ret = __sev_launch_update_vmsa(kvm, vcpu, &argp->error); |
ad73109a | 643 | |
bb18a677 | 644 | mutex_unlock(&vcpu->mutex); |
ad73109a | 645 | if (ret) |
238eca82 | 646 | return ret; |
ad73109a TL |
647 | } |
648 | ||
238eca82 | 649 | return 0; |
ad73109a TL |
650 | } |
651 | ||
eaf78265 JR |
652 | static int sev_launch_measure(struct kvm *kvm, struct kvm_sev_cmd *argp) |
653 | { | |
654 | void __user *measure = (void __user *)(uintptr_t)argp->data; | |
655 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
238eca82 | 656 | struct sev_data_launch_measure data; |
eaf78265 JR |
657 | struct kvm_sev_launch_measure params; |
658 | void __user *p = NULL; | |
659 | void *blob = NULL; | |
660 | int ret; | |
661 | ||
662 | if (!sev_guest(kvm)) | |
663 | return -ENOTTY; | |
664 | ||
665 | if (copy_from_user(¶ms, measure, sizeof(params))) | |
666 | return -EFAULT; | |
667 | ||
238eca82 | 668 | memset(&data, 0, sizeof(data)); |
eaf78265 JR |
669 | |
670 | /* User wants to query the blob length */ | |
671 | if (!params.len) | |
672 | goto cmd; | |
673 | ||
674 | p = (void __user *)(uintptr_t)params.uaddr; | |
675 | if (p) { | |
238eca82 SC |
676 | if (params.len > SEV_FW_BLOB_MAX_SIZE) |
677 | return -EINVAL; | |
eaf78265 | 678 | |
eba04b20 | 679 | blob = kmalloc(params.len, GFP_KERNEL_ACCOUNT); |
eaf78265 | 680 | if (!blob) |
238eca82 | 681 | return -ENOMEM; |
eaf78265 | 682 | |
238eca82 SC |
683 | data.address = __psp_pa(blob); |
684 | data.len = params.len; | |
eaf78265 JR |
685 | } |
686 | ||
687 | cmd: | |
238eca82 SC |
688 | data.handle = sev->handle; |
689 | ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_MEASURE, &data, &argp->error); | |
eaf78265 JR |
690 | |
691 | /* | |
692 | * If we query the session length, FW responded with expected data. | |
693 | */ | |
694 | if (!params.len) | |
695 | goto done; | |
696 | ||
697 | if (ret) | |
698 | goto e_free_blob; | |
699 | ||
700 | if (blob) { | |
701 | if (copy_to_user(p, blob, params.len)) | |
702 | ret = -EFAULT; | |
703 | } | |
704 | ||
705 | done: | |
238eca82 | 706 | params.len = data.len; |
eaf78265 JR |
707 | if (copy_to_user(measure, ¶ms, sizeof(params))) |
708 | ret = -EFAULT; | |
709 | e_free_blob: | |
710 | kfree(blob); | |
eaf78265 JR |
711 | return ret; |
712 | } | |
713 | ||
714 | static int sev_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp) | |
715 | { | |
716 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
238eca82 | 717 | struct sev_data_launch_finish data; |
eaf78265 JR |
718 | |
719 | if (!sev_guest(kvm)) | |
720 | return -ENOTTY; | |
721 | ||
238eca82 SC |
722 | data.handle = sev->handle; |
723 | return sev_issue_cmd(kvm, SEV_CMD_LAUNCH_FINISH, &data, &argp->error); | |
eaf78265 JR |
724 | } |
725 | ||
726 | static int sev_guest_status(struct kvm *kvm, struct kvm_sev_cmd *argp) | |
727 | { | |
728 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
729 | struct kvm_sev_guest_status params; | |
238eca82 | 730 | struct sev_data_guest_status data; |
eaf78265 JR |
731 | int ret; |
732 | ||
733 | if (!sev_guest(kvm)) | |
734 | return -ENOTTY; | |
735 | ||
238eca82 | 736 | memset(&data, 0, sizeof(data)); |
eaf78265 | 737 | |
238eca82 SC |
738 | data.handle = sev->handle; |
739 | ret = sev_issue_cmd(kvm, SEV_CMD_GUEST_STATUS, &data, &argp->error); | |
eaf78265 | 740 | if (ret) |
238eca82 | 741 | return ret; |
eaf78265 | 742 | |
238eca82 SC |
743 | params.policy = data.policy; |
744 | params.state = data.state; | |
745 | params.handle = data.handle; | |
eaf78265 JR |
746 | |
747 | if (copy_to_user((void __user *)(uintptr_t)argp->data, ¶ms, sizeof(params))) | |
748 | ret = -EFAULT; | |
238eca82 | 749 | |
eaf78265 JR |
750 | return ret; |
751 | } | |
752 | ||
753 | static int __sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src, | |
754 | unsigned long dst, int size, | |
755 | int *error, bool enc) | |
756 | { | |
757 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
238eca82 | 758 | struct sev_data_dbg data; |
eaf78265 | 759 | |
238eca82 SC |
760 | data.reserved = 0; |
761 | data.handle = sev->handle; | |
762 | data.dst_addr = dst; | |
763 | data.src_addr = src; | |
764 | data.len = size; | |
eaf78265 | 765 | |
238eca82 SC |
766 | return sev_issue_cmd(kvm, |
767 | enc ? SEV_CMD_DBG_ENCRYPT : SEV_CMD_DBG_DECRYPT, | |
768 | &data, error); | |
eaf78265 JR |
769 | } |
770 | ||
771 | static int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr, | |
772 | unsigned long dst_paddr, int sz, int *err) | |
773 | { | |
774 | int offset; | |
775 | ||
776 | /* | |
777 | * Its safe to read more than we are asked, caller should ensure that | |
778 | * destination has enough space. | |
779 | */ | |
eaf78265 | 780 | offset = src_paddr & 15; |
854c57f0 | 781 | src_paddr = round_down(src_paddr, 16); |
eaf78265 JR |
782 | sz = round_up(sz + offset, 16); |
783 | ||
784 | return __sev_issue_dbg_cmd(kvm, src_paddr, dst_paddr, sz, err, false); | |
785 | } | |
786 | ||
787 | static int __sev_dbg_decrypt_user(struct kvm *kvm, unsigned long paddr, | |
368340a3 | 788 | void __user *dst_uaddr, |
eaf78265 JR |
789 | unsigned long dst_paddr, |
790 | int size, int *err) | |
791 | { | |
792 | struct page *tpage = NULL; | |
793 | int ret, offset; | |
794 | ||
795 | /* if inputs are not 16-byte then use intermediate buffer */ | |
796 | if (!IS_ALIGNED(dst_paddr, 16) || | |
797 | !IS_ALIGNED(paddr, 16) || | |
798 | !IS_ALIGNED(size, 16)) { | |
799 | tpage = (void *)alloc_page(GFP_KERNEL); | |
800 | if (!tpage) | |
801 | return -ENOMEM; | |
802 | ||
803 | dst_paddr = __sme_page_pa(tpage); | |
804 | } | |
805 | ||
806 | ret = __sev_dbg_decrypt(kvm, paddr, dst_paddr, size, err); | |
807 | if (ret) | |
808 | goto e_free; | |
809 | ||
810 | if (tpage) { | |
811 | offset = paddr & 15; | |
368340a3 | 812 | if (copy_to_user(dst_uaddr, page_address(tpage) + offset, size)) |
eaf78265 JR |
813 | ret = -EFAULT; |
814 | } | |
815 | ||
816 | e_free: | |
817 | if (tpage) | |
818 | __free_page(tpage); | |
819 | ||
820 | return ret; | |
821 | } | |
822 | ||
823 | static int __sev_dbg_encrypt_user(struct kvm *kvm, unsigned long paddr, | |
368340a3 | 824 | void __user *vaddr, |
eaf78265 | 825 | unsigned long dst_paddr, |
368340a3 | 826 | void __user *dst_vaddr, |
eaf78265 JR |
827 | int size, int *error) |
828 | { | |
829 | struct page *src_tpage = NULL; | |
830 | struct page *dst_tpage = NULL; | |
831 | int ret, len = size; | |
832 | ||
833 | /* If source buffer is not aligned then use an intermediate buffer */ | |
368340a3 | 834 | if (!IS_ALIGNED((unsigned long)vaddr, 16)) { |
eaf78265 JR |
835 | src_tpage = alloc_page(GFP_KERNEL); |
836 | if (!src_tpage) | |
837 | return -ENOMEM; | |
838 | ||
368340a3 | 839 | if (copy_from_user(page_address(src_tpage), vaddr, size)) { |
eaf78265 JR |
840 | __free_page(src_tpage); |
841 | return -EFAULT; | |
842 | } | |
843 | ||
844 | paddr = __sme_page_pa(src_tpage); | |
845 | } | |
846 | ||
847 | /* | |
848 | * If destination buffer or length is not aligned then do read-modify-write: | |
849 | * - decrypt destination in an intermediate buffer | |
850 | * - copy the source buffer in an intermediate buffer | |
851 | * - use the intermediate buffer as source buffer | |
852 | */ | |
368340a3 | 853 | if (!IS_ALIGNED((unsigned long)dst_vaddr, 16) || !IS_ALIGNED(size, 16)) { |
eaf78265 JR |
854 | int dst_offset; |
855 | ||
856 | dst_tpage = alloc_page(GFP_KERNEL); | |
857 | if (!dst_tpage) { | |
858 | ret = -ENOMEM; | |
859 | goto e_free; | |
860 | } | |
861 | ||
862 | ret = __sev_dbg_decrypt(kvm, dst_paddr, | |
863 | __sme_page_pa(dst_tpage), size, error); | |
864 | if (ret) | |
865 | goto e_free; | |
866 | ||
867 | /* | |
868 | * If source is kernel buffer then use memcpy() otherwise | |
869 | * copy_from_user(). | |
870 | */ | |
871 | dst_offset = dst_paddr & 15; | |
872 | ||
873 | if (src_tpage) | |
874 | memcpy(page_address(dst_tpage) + dst_offset, | |
875 | page_address(src_tpage), size); | |
876 | else { | |
877 | if (copy_from_user(page_address(dst_tpage) + dst_offset, | |
368340a3 | 878 | vaddr, size)) { |
eaf78265 JR |
879 | ret = -EFAULT; |
880 | goto e_free; | |
881 | } | |
882 | } | |
883 | ||
884 | paddr = __sme_page_pa(dst_tpage); | |
885 | dst_paddr = round_down(dst_paddr, 16); | |
886 | len = round_up(size, 16); | |
887 | } | |
888 | ||
889 | ret = __sev_issue_dbg_cmd(kvm, paddr, dst_paddr, len, error, true); | |
890 | ||
891 | e_free: | |
892 | if (src_tpage) | |
893 | __free_page(src_tpage); | |
894 | if (dst_tpage) | |
895 | __free_page(dst_tpage); | |
896 | return ret; | |
897 | } | |
898 | ||
899 | static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec) | |
900 | { | |
901 | unsigned long vaddr, vaddr_end, next_vaddr; | |
902 | unsigned long dst_vaddr; | |
903 | struct page **src_p, **dst_p; | |
904 | struct kvm_sev_dbg debug; | |
905 | unsigned long n; | |
906 | unsigned int size; | |
907 | int ret; | |
908 | ||
909 | if (!sev_guest(kvm)) | |
910 | return -ENOTTY; | |
911 | ||
912 | if (copy_from_user(&debug, (void __user *)(uintptr_t)argp->data, sizeof(debug))) | |
913 | return -EFAULT; | |
914 | ||
915 | if (!debug.len || debug.src_uaddr + debug.len < debug.src_uaddr) | |
916 | return -EINVAL; | |
917 | if (!debug.dst_uaddr) | |
918 | return -EINVAL; | |
919 | ||
920 | vaddr = debug.src_uaddr; | |
921 | size = debug.len; | |
922 | vaddr_end = vaddr + size; | |
923 | dst_vaddr = debug.dst_uaddr; | |
924 | ||
925 | for (; vaddr < vaddr_end; vaddr = next_vaddr) { | |
926 | int len, s_off, d_off; | |
927 | ||
928 | /* lock userspace source and destination page */ | |
929 | src_p = sev_pin_memory(kvm, vaddr & PAGE_MASK, PAGE_SIZE, &n, 0); | |
ff2bd9ff DC |
930 | if (IS_ERR(src_p)) |
931 | return PTR_ERR(src_p); | |
eaf78265 JR |
932 | |
933 | dst_p = sev_pin_memory(kvm, dst_vaddr & PAGE_MASK, PAGE_SIZE, &n, 1); | |
ff2bd9ff | 934 | if (IS_ERR(dst_p)) { |
eaf78265 | 935 | sev_unpin_memory(kvm, src_p, n); |
ff2bd9ff | 936 | return PTR_ERR(dst_p); |
eaf78265 JR |
937 | } |
938 | ||
939 | /* | |
14e3dd8d PB |
940 | * Flush (on non-coherent CPUs) before DBG_{DE,EN}CRYPT read or modify |
941 | * the pages; flush the destination too so that future accesses do not | |
942 | * see stale data. | |
eaf78265 JR |
943 | */ |
944 | sev_clflush_pages(src_p, 1); | |
945 | sev_clflush_pages(dst_p, 1); | |
946 | ||
947 | /* | |
948 | * Since user buffer may not be page aligned, calculate the | |
949 | * offset within the page. | |
950 | */ | |
951 | s_off = vaddr & ~PAGE_MASK; | |
952 | d_off = dst_vaddr & ~PAGE_MASK; | |
953 | len = min_t(size_t, (PAGE_SIZE - s_off), size); | |
954 | ||
955 | if (dec) | |
956 | ret = __sev_dbg_decrypt_user(kvm, | |
957 | __sme_page_pa(src_p[0]) + s_off, | |
368340a3 | 958 | (void __user *)dst_vaddr, |
eaf78265 JR |
959 | __sme_page_pa(dst_p[0]) + d_off, |
960 | len, &argp->error); | |
961 | else | |
962 | ret = __sev_dbg_encrypt_user(kvm, | |
963 | __sme_page_pa(src_p[0]) + s_off, | |
368340a3 | 964 | (void __user *)vaddr, |
eaf78265 | 965 | __sme_page_pa(dst_p[0]) + d_off, |
368340a3 | 966 | (void __user *)dst_vaddr, |
eaf78265 JR |
967 | len, &argp->error); |
968 | ||
969 | sev_unpin_memory(kvm, src_p, n); | |
970 | sev_unpin_memory(kvm, dst_p, n); | |
971 | ||
972 | if (ret) | |
973 | goto err; | |
974 | ||
975 | next_vaddr = vaddr + len; | |
976 | dst_vaddr = dst_vaddr + len; | |
977 | size -= len; | |
978 | } | |
979 | err: | |
980 | return ret; | |
981 | } | |
982 | ||
983 | static int sev_launch_secret(struct kvm *kvm, struct kvm_sev_cmd *argp) | |
984 | { | |
985 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
238eca82 | 986 | struct sev_data_launch_secret data; |
eaf78265 JR |
987 | struct kvm_sev_launch_secret params; |
988 | struct page **pages; | |
989 | void *blob, *hdr; | |
50085bee | 990 | unsigned long n, i; |
eaf78265 JR |
991 | int ret, offset; |
992 | ||
993 | if (!sev_guest(kvm)) | |
994 | return -ENOTTY; | |
995 | ||
996 | if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, sizeof(params))) | |
997 | return -EFAULT; | |
998 | ||
999 | pages = sev_pin_memory(kvm, params.guest_uaddr, params.guest_len, &n, 1); | |
a8d908b5 PB |
1000 | if (IS_ERR(pages)) |
1001 | return PTR_ERR(pages); | |
eaf78265 | 1002 | |
50085bee | 1003 | /* |
14e3dd8d PB |
1004 | * Flush (on non-coherent CPUs) before LAUNCH_SECRET encrypts pages in |
1005 | * place; the cache may contain the data that was written unencrypted. | |
50085bee CC |
1006 | */ |
1007 | sev_clflush_pages(pages, n); | |
1008 | ||
eaf78265 JR |
1009 | /* |
1010 | * The secret must be copied into contiguous memory region, lets verify | |
1011 | * that userspace memory pages are contiguous before we issue command. | |
1012 | */ | |
1013 | if (get_num_contig_pages(0, pages, n) != n) { | |
1014 | ret = -EINVAL; | |
1015 | goto e_unpin_memory; | |
1016 | } | |
1017 | ||
238eca82 | 1018 | memset(&data, 0, sizeof(data)); |
eaf78265 JR |
1019 | |
1020 | offset = params.guest_uaddr & (PAGE_SIZE - 1); | |
238eca82 SC |
1021 | data.guest_address = __sme_page_pa(pages[0]) + offset; |
1022 | data.guest_len = params.guest_len; | |
eaf78265 JR |
1023 | |
1024 | blob = psp_copy_user_blob(params.trans_uaddr, params.trans_len); | |
1025 | if (IS_ERR(blob)) { | |
1026 | ret = PTR_ERR(blob); | |
238eca82 | 1027 | goto e_unpin_memory; |
eaf78265 JR |
1028 | } |
1029 | ||
238eca82 SC |
1030 | data.trans_address = __psp_pa(blob); |
1031 | data.trans_len = params.trans_len; | |
eaf78265 JR |
1032 | |
1033 | hdr = psp_copy_user_blob(params.hdr_uaddr, params.hdr_len); | |
1034 | if (IS_ERR(hdr)) { | |
1035 | ret = PTR_ERR(hdr); | |
1036 | goto e_free_blob; | |
1037 | } | |
238eca82 SC |
1038 | data.hdr_address = __psp_pa(hdr); |
1039 | data.hdr_len = params.hdr_len; | |
eaf78265 | 1040 | |
238eca82 SC |
1041 | data.handle = sev->handle; |
1042 | ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_SECRET, &data, &argp->error); | |
eaf78265 JR |
1043 | |
1044 | kfree(hdr); | |
1045 | ||
1046 | e_free_blob: | |
1047 | kfree(blob); | |
eaf78265 | 1048 | e_unpin_memory: |
50085bee CC |
1049 | /* content of memory is updated, mark pages dirty */ |
1050 | for (i = 0; i < n; i++) { | |
1051 | set_page_dirty_lock(pages[i]); | |
1052 | mark_page_accessed(pages[i]); | |
1053 | } | |
eaf78265 JR |
1054 | sev_unpin_memory(kvm, pages, n); |
1055 | return ret; | |
1056 | } | |
1057 | ||
2c07ded0 BS |
1058 | static int sev_get_attestation_report(struct kvm *kvm, struct kvm_sev_cmd *argp) |
1059 | { | |
1060 | void __user *report = (void __user *)(uintptr_t)argp->data; | |
1061 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
238eca82 | 1062 | struct sev_data_attestation_report data; |
2c07ded0 BS |
1063 | struct kvm_sev_attestation_report params; |
1064 | void __user *p; | |
1065 | void *blob = NULL; | |
1066 | int ret; | |
1067 | ||
1068 | if (!sev_guest(kvm)) | |
1069 | return -ENOTTY; | |
1070 | ||
1071 | if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, sizeof(params))) | |
1072 | return -EFAULT; | |
1073 | ||
238eca82 | 1074 | memset(&data, 0, sizeof(data)); |
2c07ded0 BS |
1075 | |
1076 | /* User wants to query the blob length */ | |
1077 | if (!params.len) | |
1078 | goto cmd; | |
1079 | ||
1080 | p = (void __user *)(uintptr_t)params.uaddr; | |
1081 | if (p) { | |
238eca82 SC |
1082 | if (params.len > SEV_FW_BLOB_MAX_SIZE) |
1083 | return -EINVAL; | |
2c07ded0 | 1084 | |
eba04b20 | 1085 | blob = kmalloc(params.len, GFP_KERNEL_ACCOUNT); |
2c07ded0 | 1086 | if (!blob) |
238eca82 | 1087 | return -ENOMEM; |
2c07ded0 | 1088 | |
238eca82 SC |
1089 | data.address = __psp_pa(blob); |
1090 | data.len = params.len; | |
1091 | memcpy(data.mnonce, params.mnonce, sizeof(params.mnonce)); | |
2c07ded0 BS |
1092 | } |
1093 | cmd: | |
238eca82 SC |
1094 | data.handle = sev->handle; |
1095 | ret = sev_issue_cmd(kvm, SEV_CMD_ATTESTATION_REPORT, &data, &argp->error); | |
2c07ded0 BS |
1096 | /* |
1097 | * If we query the session length, FW responded with expected data. | |
1098 | */ | |
1099 | if (!params.len) | |
1100 | goto done; | |
1101 | ||
1102 | if (ret) | |
1103 | goto e_free_blob; | |
1104 | ||
1105 | if (blob) { | |
1106 | if (copy_to_user(p, blob, params.len)) | |
1107 | ret = -EFAULT; | |
1108 | } | |
1109 | ||
1110 | done: | |
238eca82 | 1111 | params.len = data.len; |
2c07ded0 BS |
1112 | if (copy_to_user(report, ¶ms, sizeof(params))) |
1113 | ret = -EFAULT; | |
1114 | e_free_blob: | |
1115 | kfree(blob); | |
2c07ded0 BS |
1116 | return ret; |
1117 | } | |
1118 | ||
4cfdd47d BS |
1119 | /* Userspace wants to query session length. */ |
1120 | static int | |
1121 | __sev_send_start_query_session_length(struct kvm *kvm, struct kvm_sev_cmd *argp, | |
1122 | struct kvm_sev_send_start *params) | |
1123 | { | |
1124 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
238eca82 | 1125 | struct sev_data_send_start data; |
4cfdd47d BS |
1126 | int ret; |
1127 | ||
4f13d471 | 1128 | memset(&data, 0, sizeof(data)); |
238eca82 SC |
1129 | data.handle = sev->handle; |
1130 | ret = sev_issue_cmd(kvm, SEV_CMD_SEND_START, &data, &argp->error); | |
4cfdd47d | 1131 | |
238eca82 | 1132 | params->session_len = data.session_len; |
4cfdd47d BS |
1133 | if (copy_to_user((void __user *)(uintptr_t)argp->data, params, |
1134 | sizeof(struct kvm_sev_send_start))) | |
1135 | ret = -EFAULT; | |
1136 | ||
4cfdd47d BS |
1137 | return ret; |
1138 | } | |
1139 | ||
1140 | static int sev_send_start(struct kvm *kvm, struct kvm_sev_cmd *argp) | |
1141 | { | |
1142 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
238eca82 | 1143 | struct sev_data_send_start data; |
4cfdd47d BS |
1144 | struct kvm_sev_send_start params; |
1145 | void *amd_certs, *session_data; | |
1146 | void *pdh_cert, *plat_certs; | |
1147 | int ret; | |
1148 | ||
1149 | if (!sev_guest(kvm)) | |
1150 | return -ENOTTY; | |
1151 | ||
1152 | if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, | |
1153 | sizeof(struct kvm_sev_send_start))) | |
1154 | return -EFAULT; | |
1155 | ||
1156 | /* if session_len is zero, userspace wants to query the session length */ | |
1157 | if (!params.session_len) | |
1158 | return __sev_send_start_query_session_length(kvm, argp, | |
1159 | ¶ms); | |
1160 | ||
1161 | /* some sanity checks */ | |
1162 | if (!params.pdh_cert_uaddr || !params.pdh_cert_len || | |
1163 | !params.session_uaddr || params.session_len > SEV_FW_BLOB_MAX_SIZE) | |
1164 | return -EINVAL; | |
1165 | ||
1166 | /* allocate the memory to hold the session data blob */ | |
1167 | session_data = kmalloc(params.session_len, GFP_KERNEL_ACCOUNT); | |
1168 | if (!session_data) | |
1169 | return -ENOMEM; | |
1170 | ||
1171 | /* copy the certificate blobs from userspace */ | |
1172 | pdh_cert = psp_copy_user_blob(params.pdh_cert_uaddr, | |
1173 | params.pdh_cert_len); | |
1174 | if (IS_ERR(pdh_cert)) { | |
1175 | ret = PTR_ERR(pdh_cert); | |
1176 | goto e_free_session; | |
1177 | } | |
1178 | ||
1179 | plat_certs = psp_copy_user_blob(params.plat_certs_uaddr, | |
1180 | params.plat_certs_len); | |
1181 | if (IS_ERR(plat_certs)) { | |
1182 | ret = PTR_ERR(plat_certs); | |
1183 | goto e_free_pdh; | |
1184 | } | |
1185 | ||
1186 | amd_certs = psp_copy_user_blob(params.amd_certs_uaddr, | |
1187 | params.amd_certs_len); | |
1188 | if (IS_ERR(amd_certs)) { | |
1189 | ret = PTR_ERR(amd_certs); | |
1190 | goto e_free_plat_cert; | |
1191 | } | |
1192 | ||
4cfdd47d | 1193 | /* populate the FW SEND_START field with system physical address */ |
238eca82 SC |
1194 | memset(&data, 0, sizeof(data)); |
1195 | data.pdh_cert_address = __psp_pa(pdh_cert); | |
1196 | data.pdh_cert_len = params.pdh_cert_len; | |
1197 | data.plat_certs_address = __psp_pa(plat_certs); | |
1198 | data.plat_certs_len = params.plat_certs_len; | |
1199 | data.amd_certs_address = __psp_pa(amd_certs); | |
1200 | data.amd_certs_len = params.amd_certs_len; | |
1201 | data.session_address = __psp_pa(session_data); | |
1202 | data.session_len = params.session_len; | |
1203 | data.handle = sev->handle; | |
1204 | ||
1205 | ret = sev_issue_cmd(kvm, SEV_CMD_SEND_START, &data, &argp->error); | |
4cfdd47d BS |
1206 | |
1207 | if (!ret && copy_to_user((void __user *)(uintptr_t)params.session_uaddr, | |
1208 | session_data, params.session_len)) { | |
1209 | ret = -EFAULT; | |
238eca82 | 1210 | goto e_free_amd_cert; |
4cfdd47d BS |
1211 | } |
1212 | ||
238eca82 SC |
1213 | params.policy = data.policy; |
1214 | params.session_len = data.session_len; | |
4cfdd47d BS |
1215 | if (copy_to_user((void __user *)(uintptr_t)argp->data, ¶ms, |
1216 | sizeof(struct kvm_sev_send_start))) | |
1217 | ret = -EFAULT; | |
1218 | ||
4cfdd47d BS |
1219 | e_free_amd_cert: |
1220 | kfree(amd_certs); | |
1221 | e_free_plat_cert: | |
1222 | kfree(plat_certs); | |
1223 | e_free_pdh: | |
1224 | kfree(pdh_cert); | |
1225 | e_free_session: | |
1226 | kfree(session_data); | |
1227 | return ret; | |
1228 | } | |
1229 | ||
d3d1af85 BS |
1230 | /* Userspace wants to query either header or trans length. */ |
1231 | static int | |
1232 | __sev_send_update_data_query_lengths(struct kvm *kvm, struct kvm_sev_cmd *argp, | |
1233 | struct kvm_sev_send_update_data *params) | |
1234 | { | |
1235 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
238eca82 | 1236 | struct sev_data_send_update_data data; |
d3d1af85 BS |
1237 | int ret; |
1238 | ||
4f13d471 | 1239 | memset(&data, 0, sizeof(data)); |
238eca82 SC |
1240 | data.handle = sev->handle; |
1241 | ret = sev_issue_cmd(kvm, SEV_CMD_SEND_UPDATE_DATA, &data, &argp->error); | |
d3d1af85 | 1242 | |
238eca82 SC |
1243 | params->hdr_len = data.hdr_len; |
1244 | params->trans_len = data.trans_len; | |
d3d1af85 BS |
1245 | |
1246 | if (copy_to_user((void __user *)(uintptr_t)argp->data, params, | |
1247 | sizeof(struct kvm_sev_send_update_data))) | |
1248 | ret = -EFAULT; | |
1249 | ||
d3d1af85 BS |
1250 | return ret; |
1251 | } | |
1252 | ||
1253 | static int sev_send_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp) | |
1254 | { | |
1255 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
238eca82 | 1256 | struct sev_data_send_update_data data; |
d3d1af85 BS |
1257 | struct kvm_sev_send_update_data params; |
1258 | void *hdr, *trans_data; | |
1259 | struct page **guest_page; | |
1260 | unsigned long n; | |
1261 | int ret, offset; | |
1262 | ||
1263 | if (!sev_guest(kvm)) | |
1264 | return -ENOTTY; | |
1265 | ||
1266 | if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, | |
1267 | sizeof(struct kvm_sev_send_update_data))) | |
1268 | return -EFAULT; | |
1269 | ||
1270 | /* userspace wants to query either header or trans length */ | |
1271 | if (!params.trans_len || !params.hdr_len) | |
1272 | return __sev_send_update_data_query_lengths(kvm, argp, ¶ms); | |
1273 | ||
1274 | if (!params.trans_uaddr || !params.guest_uaddr || | |
1275 | !params.guest_len || !params.hdr_uaddr) | |
1276 | return -EINVAL; | |
1277 | ||
1278 | /* Check if we are crossing the page boundary */ | |
1279 | offset = params.guest_uaddr & (PAGE_SIZE - 1); | |
1280 | if ((params.guest_len + offset > PAGE_SIZE)) | |
1281 | return -EINVAL; | |
1282 | ||
1283 | /* Pin guest memory */ | |
1284 | guest_page = sev_pin_memory(kvm, params.guest_uaddr & PAGE_MASK, | |
1285 | PAGE_SIZE, &n, 0); | |
c7a1b2b6 SC |
1286 | if (IS_ERR(guest_page)) |
1287 | return PTR_ERR(guest_page); | |
d3d1af85 BS |
1288 | |
1289 | /* allocate memory for header and transport buffer */ | |
1290 | ret = -ENOMEM; | |
1291 | hdr = kmalloc(params.hdr_len, GFP_KERNEL_ACCOUNT); | |
1292 | if (!hdr) | |
1293 | goto e_unpin; | |
1294 | ||
1295 | trans_data = kmalloc(params.trans_len, GFP_KERNEL_ACCOUNT); | |
1296 | if (!trans_data) | |
1297 | goto e_free_hdr; | |
1298 | ||
238eca82 SC |
1299 | memset(&data, 0, sizeof(data)); |
1300 | data.hdr_address = __psp_pa(hdr); | |
1301 | data.hdr_len = params.hdr_len; | |
1302 | data.trans_address = __psp_pa(trans_data); | |
1303 | data.trans_len = params.trans_len; | |
d3d1af85 BS |
1304 | |
1305 | /* The SEND_UPDATE_DATA command requires C-bit to be always set. */ | |
238eca82 SC |
1306 | data.guest_address = (page_to_pfn(guest_page[0]) << PAGE_SHIFT) + offset; |
1307 | data.guest_address |= sev_me_mask; | |
1308 | data.guest_len = params.guest_len; | |
1309 | data.handle = sev->handle; | |
d3d1af85 | 1310 | |
238eca82 | 1311 | ret = sev_issue_cmd(kvm, SEV_CMD_SEND_UPDATE_DATA, &data, &argp->error); |
d3d1af85 BS |
1312 | |
1313 | if (ret) | |
238eca82 | 1314 | goto e_free_trans_data; |
d3d1af85 BS |
1315 | |
1316 | /* copy transport buffer to user space */ | |
1317 | if (copy_to_user((void __user *)(uintptr_t)params.trans_uaddr, | |
1318 | trans_data, params.trans_len)) { | |
1319 | ret = -EFAULT; | |
238eca82 | 1320 | goto e_free_trans_data; |
d3d1af85 BS |
1321 | } |
1322 | ||
1323 | /* Copy packet header to userspace. */ | |
b4a69392 SC |
1324 | if (copy_to_user((void __user *)(uintptr_t)params.hdr_uaddr, hdr, |
1325 | params.hdr_len)) | |
1326 | ret = -EFAULT; | |
d3d1af85 | 1327 | |
d3d1af85 BS |
1328 | e_free_trans_data: |
1329 | kfree(trans_data); | |
1330 | e_free_hdr: | |
1331 | kfree(hdr); | |
1332 | e_unpin: | |
1333 | sev_unpin_memory(kvm, guest_page, n); | |
1334 | ||
1335 | return ret; | |
1336 | } | |
1337 | ||
fddecf6a BS |
1338 | static int sev_send_finish(struct kvm *kvm, struct kvm_sev_cmd *argp) |
1339 | { | |
1340 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
238eca82 | 1341 | struct sev_data_send_finish data; |
fddecf6a BS |
1342 | |
1343 | if (!sev_guest(kvm)) | |
1344 | return -ENOTTY; | |
1345 | ||
238eca82 SC |
1346 | data.handle = sev->handle; |
1347 | return sev_issue_cmd(kvm, SEV_CMD_SEND_FINISH, &data, &argp->error); | |
fddecf6a BS |
1348 | } |
1349 | ||
5569e2e7 SR |
1350 | static int sev_send_cancel(struct kvm *kvm, struct kvm_sev_cmd *argp) |
1351 | { | |
1352 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
238eca82 | 1353 | struct sev_data_send_cancel data; |
5569e2e7 SR |
1354 | |
1355 | if (!sev_guest(kvm)) | |
1356 | return -ENOTTY; | |
1357 | ||
238eca82 SC |
1358 | data.handle = sev->handle; |
1359 | return sev_issue_cmd(kvm, SEV_CMD_SEND_CANCEL, &data, &argp->error); | |
5569e2e7 SR |
1360 | } |
1361 | ||
af43cbbf BS |
1362 | static int sev_receive_start(struct kvm *kvm, struct kvm_sev_cmd *argp) |
1363 | { | |
1364 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
238eca82 | 1365 | struct sev_data_receive_start start; |
af43cbbf BS |
1366 | struct kvm_sev_receive_start params; |
1367 | int *error = &argp->error; | |
1368 | void *session_data; | |
1369 | void *pdh_data; | |
1370 | int ret; | |
1371 | ||
1372 | if (!sev_guest(kvm)) | |
1373 | return -ENOTTY; | |
1374 | ||
1375 | /* Get parameter from the userspace */ | |
1376 | if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, | |
1377 | sizeof(struct kvm_sev_receive_start))) | |
1378 | return -EFAULT; | |
1379 | ||
1380 | /* some sanity checks */ | |
1381 | if (!params.pdh_uaddr || !params.pdh_len || | |
1382 | !params.session_uaddr || !params.session_len) | |
1383 | return -EINVAL; | |
1384 | ||
1385 | pdh_data = psp_copy_user_blob(params.pdh_uaddr, params.pdh_len); | |
1386 | if (IS_ERR(pdh_data)) | |
1387 | return PTR_ERR(pdh_data); | |
1388 | ||
1389 | session_data = psp_copy_user_blob(params.session_uaddr, | |
1390 | params.session_len); | |
1391 | if (IS_ERR(session_data)) { | |
1392 | ret = PTR_ERR(session_data); | |
1393 | goto e_free_pdh; | |
1394 | } | |
1395 | ||
238eca82 SC |
1396 | memset(&start, 0, sizeof(start)); |
1397 | start.handle = params.handle; | |
1398 | start.policy = params.policy; | |
1399 | start.pdh_cert_address = __psp_pa(pdh_data); | |
1400 | start.pdh_cert_len = params.pdh_len; | |
1401 | start.session_address = __psp_pa(session_data); | |
1402 | start.session_len = params.session_len; | |
af43cbbf BS |
1403 | |
1404 | /* create memory encryption context */ | |
238eca82 | 1405 | ret = __sev_issue_cmd(argp->sev_fd, SEV_CMD_RECEIVE_START, &start, |
af43cbbf BS |
1406 | error); |
1407 | if (ret) | |
238eca82 | 1408 | goto e_free_session; |
af43cbbf BS |
1409 | |
1410 | /* Bind ASID to this guest */ | |
238eca82 | 1411 | ret = sev_bind_asid(kvm, start.handle, error); |
f1815e0a MZ |
1412 | if (ret) { |
1413 | sev_decommission(start.handle); | |
238eca82 | 1414 | goto e_free_session; |
f1815e0a | 1415 | } |
af43cbbf | 1416 | |
238eca82 | 1417 | params.handle = start.handle; |
af43cbbf BS |
1418 | if (copy_to_user((void __user *)(uintptr_t)argp->data, |
1419 | ¶ms, sizeof(struct kvm_sev_receive_start))) { | |
1420 | ret = -EFAULT; | |
238eca82 SC |
1421 | sev_unbind_asid(kvm, start.handle); |
1422 | goto e_free_session; | |
af43cbbf BS |
1423 | } |
1424 | ||
238eca82 | 1425 | sev->handle = start.handle; |
af43cbbf BS |
1426 | sev->fd = argp->sev_fd; |
1427 | ||
af43cbbf BS |
1428 | e_free_session: |
1429 | kfree(session_data); | |
1430 | e_free_pdh: | |
1431 | kfree(pdh_data); | |
1432 | ||
1433 | return ret; | |
1434 | } | |
1435 | ||
15fb7de1 BS |
1436 | static int sev_receive_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp) |
1437 | { | |
1438 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
1439 | struct kvm_sev_receive_update_data params; | |
238eca82 | 1440 | struct sev_data_receive_update_data data; |
15fb7de1 BS |
1441 | void *hdr = NULL, *trans = NULL; |
1442 | struct page **guest_page; | |
1443 | unsigned long n; | |
1444 | int ret, offset; | |
1445 | ||
1446 | if (!sev_guest(kvm)) | |
1447 | return -EINVAL; | |
1448 | ||
1449 | if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, | |
1450 | sizeof(struct kvm_sev_receive_update_data))) | |
1451 | return -EFAULT; | |
1452 | ||
1453 | if (!params.hdr_uaddr || !params.hdr_len || | |
1454 | !params.guest_uaddr || !params.guest_len || | |
1455 | !params.trans_uaddr || !params.trans_len) | |
1456 | return -EINVAL; | |
1457 | ||
1458 | /* Check if we are crossing the page boundary */ | |
1459 | offset = params.guest_uaddr & (PAGE_SIZE - 1); | |
1460 | if ((params.guest_len + offset > PAGE_SIZE)) | |
1461 | return -EINVAL; | |
1462 | ||
1463 | hdr = psp_copy_user_blob(params.hdr_uaddr, params.hdr_len); | |
1464 | if (IS_ERR(hdr)) | |
1465 | return PTR_ERR(hdr); | |
1466 | ||
1467 | trans = psp_copy_user_blob(params.trans_uaddr, params.trans_len); | |
1468 | if (IS_ERR(trans)) { | |
1469 | ret = PTR_ERR(trans); | |
1470 | goto e_free_hdr; | |
1471 | } | |
1472 | ||
238eca82 SC |
1473 | memset(&data, 0, sizeof(data)); |
1474 | data.hdr_address = __psp_pa(hdr); | |
1475 | data.hdr_len = params.hdr_len; | |
1476 | data.trans_address = __psp_pa(trans); | |
1477 | data.trans_len = params.trans_len; | |
15fb7de1 BS |
1478 | |
1479 | /* Pin guest memory */ | |
15fb7de1 | 1480 | guest_page = sev_pin_memory(kvm, params.guest_uaddr & PAGE_MASK, |
50c03801 | 1481 | PAGE_SIZE, &n, 1); |
c7a1b2b6 SC |
1482 | if (IS_ERR(guest_page)) { |
1483 | ret = PTR_ERR(guest_page); | |
238eca82 | 1484 | goto e_free_trans; |
c7a1b2b6 | 1485 | } |
15fb7de1 | 1486 | |
c8c340a9 MK |
1487 | /* |
1488 | * Flush (on non-coherent CPUs) before RECEIVE_UPDATE_DATA, the PSP | |
1489 | * encrypts the written data with the guest's key, and the cache may | |
1490 | * contain dirty, unencrypted data. | |
1491 | */ | |
1492 | sev_clflush_pages(guest_page, n); | |
1493 | ||
15fb7de1 | 1494 | /* The RECEIVE_UPDATE_DATA command requires C-bit to be always set. */ |
238eca82 SC |
1495 | data.guest_address = (page_to_pfn(guest_page[0]) << PAGE_SHIFT) + offset; |
1496 | data.guest_address |= sev_me_mask; | |
1497 | data.guest_len = params.guest_len; | |
1498 | data.handle = sev->handle; | |
15fb7de1 | 1499 | |
238eca82 | 1500 | ret = sev_issue_cmd(kvm, SEV_CMD_RECEIVE_UPDATE_DATA, &data, |
15fb7de1 BS |
1501 | &argp->error); |
1502 | ||
1503 | sev_unpin_memory(kvm, guest_page, n); | |
1504 | ||
15fb7de1 BS |
1505 | e_free_trans: |
1506 | kfree(trans); | |
1507 | e_free_hdr: | |
1508 | kfree(hdr); | |
1509 | ||
1510 | return ret; | |
1511 | } | |
1512 | ||
6a443def BS |
1513 | static int sev_receive_finish(struct kvm *kvm, struct kvm_sev_cmd *argp) |
1514 | { | |
1515 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
238eca82 | 1516 | struct sev_data_receive_finish data; |
6a443def BS |
1517 | |
1518 | if (!sev_guest(kvm)) | |
1519 | return -ENOTTY; | |
1520 | ||
238eca82 SC |
1521 | data.handle = sev->handle; |
1522 | return sev_issue_cmd(kvm, SEV_CMD_RECEIVE_FINISH, &data, &argp->error); | |
6a443def BS |
1523 | } |
1524 | ||
5b92b6ca PG |
1525 | static bool cmd_allowed_from_miror(u32 cmd_id) |
1526 | { | |
1527 | /* | |
1528 | * Allow mirrors VM to call KVM_SEV_LAUNCH_UPDATE_VMSA to enable SEV-ES | |
1529 | * active mirror VMs. Also allow the debugging and status commands. | |
1530 | */ | |
1531 | if (cmd_id == KVM_SEV_LAUNCH_UPDATE_VMSA || | |
1532 | cmd_id == KVM_SEV_GUEST_STATUS || cmd_id == KVM_SEV_DBG_DECRYPT || | |
1533 | cmd_id == KVM_SEV_DBG_ENCRYPT) | |
1534 | return true; | |
1535 | ||
1536 | return false; | |
1537 | } | |
1538 | ||
eaf78265 JR |
1539 | int svm_mem_enc_op(struct kvm *kvm, void __user *argp) |
1540 | { | |
1541 | struct kvm_sev_cmd sev_cmd; | |
1542 | int r; | |
1543 | ||
a5c1c5aa | 1544 | if (!sev_enabled) |
eaf78265 JR |
1545 | return -ENOTTY; |
1546 | ||
1547 | if (!argp) | |
1548 | return 0; | |
1549 | ||
1550 | if (copy_from_user(&sev_cmd, argp, sizeof(struct kvm_sev_cmd))) | |
1551 | return -EFAULT; | |
1552 | ||
1553 | mutex_lock(&kvm->lock); | |
1554 | ||
5b92b6ca PG |
1555 | /* Only the enc_context_owner handles some memory enc operations. */ |
1556 | if (is_mirroring_enc_context(kvm) && | |
1557 | !cmd_allowed_from_miror(sev_cmd.id)) { | |
54526d1f NT |
1558 | r = -EINVAL; |
1559 | goto out; | |
1560 | } | |
1561 | ||
eaf78265 | 1562 | switch (sev_cmd.id) { |
9fa1521d | 1563 | case KVM_SEV_ES_INIT: |
8d364a07 | 1564 | if (!sev_es_enabled) { |
9fa1521d SC |
1565 | r = -ENOTTY; |
1566 | goto out; | |
1567 | } | |
1568 | fallthrough; | |
eaf78265 JR |
1569 | case KVM_SEV_INIT: |
1570 | r = sev_guest_init(kvm, &sev_cmd); | |
1571 | break; | |
1572 | case KVM_SEV_LAUNCH_START: | |
1573 | r = sev_launch_start(kvm, &sev_cmd); | |
1574 | break; | |
1575 | case KVM_SEV_LAUNCH_UPDATE_DATA: | |
1576 | r = sev_launch_update_data(kvm, &sev_cmd); | |
1577 | break; | |
ad73109a TL |
1578 | case KVM_SEV_LAUNCH_UPDATE_VMSA: |
1579 | r = sev_launch_update_vmsa(kvm, &sev_cmd); | |
1580 | break; | |
eaf78265 JR |
1581 | case KVM_SEV_LAUNCH_MEASURE: |
1582 | r = sev_launch_measure(kvm, &sev_cmd); | |
1583 | break; | |
1584 | case KVM_SEV_LAUNCH_FINISH: | |
1585 | r = sev_launch_finish(kvm, &sev_cmd); | |
1586 | break; | |
1587 | case KVM_SEV_GUEST_STATUS: | |
1588 | r = sev_guest_status(kvm, &sev_cmd); | |
1589 | break; | |
1590 | case KVM_SEV_DBG_DECRYPT: | |
1591 | r = sev_dbg_crypt(kvm, &sev_cmd, true); | |
1592 | break; | |
1593 | case KVM_SEV_DBG_ENCRYPT: | |
1594 | r = sev_dbg_crypt(kvm, &sev_cmd, false); | |
1595 | break; | |
1596 | case KVM_SEV_LAUNCH_SECRET: | |
1597 | r = sev_launch_secret(kvm, &sev_cmd); | |
1598 | break; | |
2c07ded0 BS |
1599 | case KVM_SEV_GET_ATTESTATION_REPORT: |
1600 | r = sev_get_attestation_report(kvm, &sev_cmd); | |
1601 | break; | |
4cfdd47d BS |
1602 | case KVM_SEV_SEND_START: |
1603 | r = sev_send_start(kvm, &sev_cmd); | |
1604 | break; | |
d3d1af85 BS |
1605 | case KVM_SEV_SEND_UPDATE_DATA: |
1606 | r = sev_send_update_data(kvm, &sev_cmd); | |
1607 | break; | |
fddecf6a BS |
1608 | case KVM_SEV_SEND_FINISH: |
1609 | r = sev_send_finish(kvm, &sev_cmd); | |
1610 | break; | |
5569e2e7 SR |
1611 | case KVM_SEV_SEND_CANCEL: |
1612 | r = sev_send_cancel(kvm, &sev_cmd); | |
1613 | break; | |
af43cbbf BS |
1614 | case KVM_SEV_RECEIVE_START: |
1615 | r = sev_receive_start(kvm, &sev_cmd); | |
1616 | break; | |
15fb7de1 BS |
1617 | case KVM_SEV_RECEIVE_UPDATE_DATA: |
1618 | r = sev_receive_update_data(kvm, &sev_cmd); | |
1619 | break; | |
6a443def BS |
1620 | case KVM_SEV_RECEIVE_FINISH: |
1621 | r = sev_receive_finish(kvm, &sev_cmd); | |
1622 | break; | |
eaf78265 JR |
1623 | default: |
1624 | r = -EINVAL; | |
1625 | goto out; | |
1626 | } | |
1627 | ||
1628 | if (copy_to_user(argp, &sev_cmd, sizeof(struct kvm_sev_cmd))) | |
1629 | r = -EFAULT; | |
1630 | ||
1631 | out: | |
1632 | mutex_unlock(&kvm->lock); | |
1633 | return r; | |
1634 | } | |
1635 | ||
1636 | int svm_register_enc_region(struct kvm *kvm, | |
1637 | struct kvm_enc_region *range) | |
1638 | { | |
1639 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
1640 | struct enc_region *region; | |
1641 | int ret = 0; | |
1642 | ||
1643 | if (!sev_guest(kvm)) | |
1644 | return -ENOTTY; | |
1645 | ||
54526d1f NT |
1646 | /* If kvm is mirroring encryption context it isn't responsible for it */ |
1647 | if (is_mirroring_enc_context(kvm)) | |
1648 | return -EINVAL; | |
1649 | ||
eaf78265 JR |
1650 | if (range->addr > ULONG_MAX || range->size > ULONG_MAX) |
1651 | return -EINVAL; | |
1652 | ||
1653 | region = kzalloc(sizeof(*region), GFP_KERNEL_ACCOUNT); | |
1654 | if (!region) | |
1655 | return -ENOMEM; | |
1656 | ||
19a23da5 | 1657 | mutex_lock(&kvm->lock); |
eaf78265 | 1658 | region->pages = sev_pin_memory(kvm, range->addr, range->size, ®ion->npages, 1); |
a8d908b5 PB |
1659 | if (IS_ERR(region->pages)) { |
1660 | ret = PTR_ERR(region->pages); | |
19a23da5 | 1661 | mutex_unlock(&kvm->lock); |
eaf78265 JR |
1662 | goto e_free; |
1663 | } | |
1664 | ||
19a23da5 PG |
1665 | region->uaddr = range->addr; |
1666 | region->size = range->size; | |
1667 | ||
1668 | list_add_tail(®ion->list, &sev->regions_list); | |
1669 | mutex_unlock(&kvm->lock); | |
1670 | ||
eaf78265 JR |
1671 | /* |
1672 | * The guest may change the memory encryption attribute from C=0 -> C=1 | |
1673 | * or vice versa for this memory range. Lets make sure caches are | |
1674 | * flushed to ensure that guest data gets written into memory with | |
1675 | * correct C-bit. | |
1676 | */ | |
1677 | sev_clflush_pages(region->pages, region->npages); | |
1678 | ||
eaf78265 JR |
1679 | return ret; |
1680 | ||
1681 | e_free: | |
1682 | kfree(region); | |
1683 | return ret; | |
1684 | } | |
1685 | ||
1686 | static struct enc_region * | |
1687 | find_enc_region(struct kvm *kvm, struct kvm_enc_region *range) | |
1688 | { | |
1689 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
1690 | struct list_head *head = &sev->regions_list; | |
1691 | struct enc_region *i; | |
1692 | ||
1693 | list_for_each_entry(i, head, list) { | |
1694 | if (i->uaddr == range->addr && | |
1695 | i->size == range->size) | |
1696 | return i; | |
1697 | } | |
1698 | ||
1699 | return NULL; | |
1700 | } | |
1701 | ||
1702 | static void __unregister_enc_region_locked(struct kvm *kvm, | |
1703 | struct enc_region *region) | |
1704 | { | |
1705 | sev_unpin_memory(kvm, region->pages, region->npages); | |
1706 | list_del(®ion->list); | |
1707 | kfree(region); | |
1708 | } | |
1709 | ||
1710 | int svm_unregister_enc_region(struct kvm *kvm, | |
1711 | struct kvm_enc_region *range) | |
1712 | { | |
1713 | struct enc_region *region; | |
1714 | int ret; | |
1715 | ||
54526d1f NT |
1716 | /* If kvm is mirroring encryption context it isn't responsible for it */ |
1717 | if (is_mirroring_enc_context(kvm)) | |
1718 | return -EINVAL; | |
1719 | ||
eaf78265 JR |
1720 | mutex_lock(&kvm->lock); |
1721 | ||
1722 | if (!sev_guest(kvm)) { | |
1723 | ret = -ENOTTY; | |
1724 | goto failed; | |
1725 | } | |
1726 | ||
1727 | region = find_enc_region(kvm, range); | |
1728 | if (!region) { | |
1729 | ret = -EINVAL; | |
1730 | goto failed; | |
1731 | } | |
1732 | ||
1733 | /* | |
1734 | * Ensure that all guest tagged cache entries are flushed before | |
1735 | * releasing the pages back to the system for use. CLFLUSH will | |
1736 | * not do this, so issue a WBINVD. | |
1737 | */ | |
1738 | wbinvd_on_all_cpus(); | |
1739 | ||
1740 | __unregister_enc_region_locked(kvm, region); | |
1741 | ||
1742 | mutex_unlock(&kvm->lock); | |
1743 | return 0; | |
1744 | ||
1745 | failed: | |
1746 | mutex_unlock(&kvm->lock); | |
1747 | return ret; | |
1748 | } | |
1749 | ||
54526d1f NT |
1750 | int svm_vm_copy_asid_from(struct kvm *kvm, unsigned int source_fd) |
1751 | { | |
1752 | struct file *source_kvm_file; | |
1753 | struct kvm *source_kvm; | |
f43c887c | 1754 | struct kvm_sev_info source_sev, *mirror_sev; |
54526d1f NT |
1755 | int ret; |
1756 | ||
1757 | source_kvm_file = fget(source_fd); | |
1758 | if (!file_is_kvm(source_kvm_file)) { | |
1759 | ret = -EBADF; | |
1760 | goto e_source_put; | |
1761 | } | |
1762 | ||
1763 | source_kvm = source_kvm_file->private_data; | |
1764 | mutex_lock(&source_kvm->lock); | |
1765 | ||
1766 | if (!sev_guest(source_kvm)) { | |
1767 | ret = -EINVAL; | |
1768 | goto e_source_unlock; | |
1769 | } | |
1770 | ||
1771 | /* Mirrors of mirrors should work, but let's not get silly */ | |
1772 | if (is_mirroring_enc_context(source_kvm) || source_kvm == kvm) { | |
1773 | ret = -EINVAL; | |
1774 | goto e_source_unlock; | |
1775 | } | |
1776 | ||
f43c887c PG |
1777 | memcpy(&source_sev, &to_kvm_svm(source_kvm)->sev_info, |
1778 | sizeof(source_sev)); | |
54526d1f NT |
1779 | |
1780 | /* | |
1781 | * The mirror kvm holds an enc_context_owner ref so its asid can't | |
1782 | * disappear until we're done with it | |
1783 | */ | |
1784 | kvm_get_kvm(source_kvm); | |
1785 | ||
1786 | fput(source_kvm_file); | |
1787 | mutex_unlock(&source_kvm->lock); | |
1788 | mutex_lock(&kvm->lock); | |
1789 | ||
e02e5d0e SC |
1790 | /* |
1791 | * Disallow out-of-band SEV/SEV-ES init if the target is already an | |
1792 | * SEV guest, or if vCPUs have been created. KVM relies on vCPUs being | |
1793 | * created after SEV/SEV-ES initialization, e.g. to init intercepts. | |
1794 | */ | |
1795 | if (sev_guest(kvm) || kvm->created_vcpus) { | |
54526d1f NT |
1796 | ret = -EINVAL; |
1797 | goto e_mirror_unlock; | |
1798 | } | |
1799 | ||
1800 | /* Set enc_context_owner and copy its encryption context over */ | |
1801 | mirror_sev = &to_kvm_svm(kvm)->sev_info; | |
1802 | mirror_sev->enc_context_owner = source_kvm; | |
54526d1f | 1803 | mirror_sev->active = true; |
f43c887c PG |
1804 | mirror_sev->asid = source_sev.asid; |
1805 | mirror_sev->fd = source_sev.fd; | |
1806 | mirror_sev->es_active = source_sev.es_active; | |
1807 | mirror_sev->handle = source_sev.handle; | |
0af65292 | 1808 | INIT_LIST_HEAD(&mirror_sev->regions_list); |
f43c887c PG |
1809 | /* |
1810 | * Do not copy ap_jump_table. Since the mirror does not share the same | |
1811 | * KVM contexts as the original, and they may have different | |
1812 | * memory-views. | |
1813 | */ | |
54526d1f NT |
1814 | |
1815 | mutex_unlock(&kvm->lock); | |
1816 | return 0; | |
1817 | ||
1818 | e_mirror_unlock: | |
1819 | mutex_unlock(&kvm->lock); | |
1820 | kvm_put_kvm(source_kvm); | |
1821 | return ret; | |
1822 | e_source_unlock: | |
1823 | mutex_unlock(&source_kvm->lock); | |
1824 | e_source_put: | |
8899a5fc CIK |
1825 | if (source_kvm_file) |
1826 | fput(source_kvm_file); | |
54526d1f NT |
1827 | return ret; |
1828 | } | |
1829 | ||
eaf78265 JR |
1830 | void sev_vm_destroy(struct kvm *kvm) |
1831 | { | |
1832 | struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; | |
1833 | struct list_head *head = &sev->regions_list; | |
1834 | struct list_head *pos, *q; | |
1835 | ||
1836 | if (!sev_guest(kvm)) | |
1837 | return; | |
1838 | ||
54526d1f NT |
1839 | /* If this is a mirror_kvm release the enc_context_owner and skip sev cleanup */ |
1840 | if (is_mirroring_enc_context(kvm)) { | |
1841 | kvm_put_kvm(sev->enc_context_owner); | |
1842 | return; | |
1843 | } | |
1844 | ||
eaf78265 JR |
1845 | mutex_lock(&kvm->lock); |
1846 | ||
1847 | /* | |
1848 | * Ensure that all guest tagged cache entries are flushed before | |
1849 | * releasing the pages back to the system for use. CLFLUSH will | |
1850 | * not do this, so issue a WBINVD. | |
1851 | */ | |
1852 | wbinvd_on_all_cpus(); | |
1853 | ||
1854 | /* | |
1855 | * if userspace was terminated before unregistering the memory regions | |
1856 | * then lets unpin all the registered memory. | |
1857 | */ | |
1858 | if (!list_empty(head)) { | |
1859 | list_for_each_safe(pos, q, head) { | |
1860 | __unregister_enc_region_locked(kvm, | |
1861 | list_entry(pos, struct enc_region, list)); | |
7be74942 | 1862 | cond_resched(); |
eaf78265 JR |
1863 | } |
1864 | } | |
1865 | ||
1866 | mutex_unlock(&kvm->lock); | |
1867 | ||
1868 | sev_unbind_asid(kvm, sev->handle); | |
7aef27f0 | 1869 | sev_asid_free(sev); |
eaf78265 JR |
1870 | } |
1871 | ||
d9db0fd6 PB |
1872 | void __init sev_set_cpu_caps(void) |
1873 | { | |
8d364a07 | 1874 | if (!sev_enabled) |
d9db0fd6 | 1875 | kvm_cpu_cap_clear(X86_FEATURE_SEV); |
8d364a07 | 1876 | if (!sev_es_enabled) |
d9db0fd6 PB |
1877 | kvm_cpu_cap_clear(X86_FEATURE_SEV_ES); |
1878 | } | |
1879 | ||
916391a2 | 1880 | void __init sev_hardware_setup(void) |
eaf78265 | 1881 | { |
a479c334 | 1882 | #ifdef CONFIG_KVM_AMD_SEV |
7aef27f0 | 1883 | unsigned int eax, ebx, ecx, edx, sev_asid_count, sev_es_asid_count; |
916391a2 TL |
1884 | bool sev_es_supported = false; |
1885 | bool sev_supported = false; | |
1886 | ||
a479c334 | 1887 | if (!sev_enabled || !npt_enabled) |
e8126bda SC |
1888 | goto out; |
1889 | ||
916391a2 TL |
1890 | /* Does the CPU support SEV? */ |
1891 | if (!boot_cpu_has(X86_FEATURE_SEV)) | |
1892 | goto out; | |
1893 | ||
1894 | /* Retrieve SEV CPUID information */ | |
1895 | cpuid(0x8000001f, &eax, &ebx, &ecx, &edx); | |
1896 | ||
1edc1459 TL |
1897 | /* Set encryption bit location for SEV-ES guests */ |
1898 | sev_enc_bit = ebx & 0x3f; | |
1899 | ||
eaf78265 | 1900 | /* Maximum number of encrypted guests supported simultaneously */ |
916391a2 | 1901 | max_sev_asid = ecx; |
8cb756b7 | 1902 | if (!max_sev_asid) |
916391a2 | 1903 | goto out; |
eaf78265 JR |
1904 | |
1905 | /* Minimum ASID value that should be used for SEV guest */ | |
916391a2 | 1906 | min_sev_asid = edx; |
d3d1af85 | 1907 | sev_me_mask = 1UL << (ebx & 0x3f); |
eaf78265 | 1908 | |
bb2baeb2 MZ |
1909 | /* |
1910 | * Initialize SEV ASID bitmaps. Allocate space for ASID 0 in the bitmap, | |
1911 | * even though it's never used, so that the bitmap is indexed by the | |
1912 | * actual ASID. | |
1913 | */ | |
1914 | nr_asids = max_sev_asid + 1; | |
1915 | sev_asid_bitmap = bitmap_zalloc(nr_asids, GFP_KERNEL); | |
eaf78265 | 1916 | if (!sev_asid_bitmap) |
916391a2 | 1917 | goto out; |
eaf78265 | 1918 | |
bb2baeb2 | 1919 | sev_reclaim_asid_bitmap = bitmap_zalloc(nr_asids, GFP_KERNEL); |
f31b88b3 SC |
1920 | if (!sev_reclaim_asid_bitmap) { |
1921 | bitmap_free(sev_asid_bitmap); | |
1922 | sev_asid_bitmap = NULL; | |
916391a2 | 1923 | goto out; |
f31b88b3 | 1924 | } |
eaf78265 | 1925 | |
7aef27f0 VS |
1926 | sev_asid_count = max_sev_asid - min_sev_asid + 1; |
1927 | if (misc_cg_set_capacity(MISC_CG_RES_SEV, sev_asid_count)) | |
1928 | goto out; | |
1929 | ||
1930 | pr_info("SEV supported: %u ASIDs\n", sev_asid_count); | |
916391a2 | 1931 | sev_supported = true; |
eaf78265 | 1932 | |
916391a2 | 1933 | /* SEV-ES support requested? */ |
8d364a07 | 1934 | if (!sev_es_enabled) |
916391a2 TL |
1935 | goto out; |
1936 | ||
1937 | /* Does the CPU support SEV-ES? */ | |
1938 | if (!boot_cpu_has(X86_FEATURE_SEV_ES)) | |
1939 | goto out; | |
1940 | ||
1941 | /* Has the system been allocated ASIDs for SEV-ES? */ | |
1942 | if (min_sev_asid == 1) | |
1943 | goto out; | |
1944 | ||
7aef27f0 VS |
1945 | sev_es_asid_count = min_sev_asid - 1; |
1946 | if (misc_cg_set_capacity(MISC_CG_RES_SEV_ES, sev_es_asid_count)) | |
1947 | goto out; | |
1948 | ||
1949 | pr_info("SEV-ES supported: %u ASIDs\n", sev_es_asid_count); | |
916391a2 TL |
1950 | sev_es_supported = true; |
1951 | ||
1952 | out: | |
8d364a07 SC |
1953 | sev_enabled = sev_supported; |
1954 | sev_es_enabled = sev_es_supported; | |
a479c334 | 1955 | #endif |
eaf78265 JR |
1956 | } |
1957 | ||
1958 | void sev_hardware_teardown(void) | |
1959 | { | |
a5c1c5aa | 1960 | if (!sev_enabled) |
9ef1530c PB |
1961 | return; |
1962 | ||
469bb32b | 1963 | /* No need to take sev_bitmap_lock, all VMs have been destroyed. */ |
bb2baeb2 | 1964 | sev_flush_asids(1, max_sev_asid); |
469bb32b | 1965 | |
eaf78265 JR |
1966 | bitmap_free(sev_asid_bitmap); |
1967 | bitmap_free(sev_reclaim_asid_bitmap); | |
469bb32b | 1968 | |
7aef27f0 VS |
1969 | misc_cg_set_capacity(MISC_CG_RES_SEV, 0); |
1970 | misc_cg_set_capacity(MISC_CG_RES_SEV_ES, 0); | |
eaf78265 | 1971 | } |
eaf78265 | 1972 | |
b95c221c SC |
1973 | int sev_cpu_init(struct svm_cpu_data *sd) |
1974 | { | |
a5c1c5aa | 1975 | if (!sev_enabled) |
b95c221c SC |
1976 | return 0; |
1977 | ||
bb2baeb2 | 1978 | sd->sev_vmcbs = kcalloc(nr_asids, sizeof(void *), GFP_KERNEL); |
b95c221c SC |
1979 | if (!sd->sev_vmcbs) |
1980 | return -ENOMEM; | |
1981 | ||
1982 | return 0; | |
eaf78265 JR |
1983 | } |
1984 | ||
add5e2f0 TL |
1985 | /* |
1986 | * Pages used by hardware to hold guest encrypted state must be flushed before | |
1987 | * returning them to the system. | |
1988 | */ | |
1989 | static void sev_flush_guest_memory(struct vcpu_svm *svm, void *va, | |
1990 | unsigned long len) | |
1991 | { | |
1992 | /* | |
1993 | * If hardware enforced cache coherency for encrypted mappings of the | |
1994 | * same physical page is supported, nothing to do. | |
1995 | */ | |
1996 | if (boot_cpu_has(X86_FEATURE_SME_COHERENT)) | |
1997 | return; | |
1998 | ||
1999 | /* | |
2000 | * If the VM Page Flush MSR is supported, use it to flush the page | |
2001 | * (using the page virtual address and the guest ASID). | |
2002 | */ | |
2003 | if (boot_cpu_has(X86_FEATURE_VM_PAGE_FLUSH)) { | |
2004 | struct kvm_sev_info *sev; | |
2005 | unsigned long va_start; | |
2006 | u64 start, stop; | |
2007 | ||
2008 | /* Align start and stop to page boundaries. */ | |
2009 | va_start = (unsigned long)va; | |
2010 | start = (u64)va_start & PAGE_MASK; | |
2011 | stop = PAGE_ALIGN((u64)va_start + len); | |
2012 | ||
2013 | if (start < stop) { | |
2014 | sev = &to_kvm_svm(svm->vcpu.kvm)->sev_info; | |
2015 | ||
2016 | while (start < stop) { | |
2017 | wrmsrl(MSR_AMD64_VM_PAGE_FLUSH, | |
2018 | start | sev->asid); | |
2019 | ||
2020 | start += PAGE_SIZE; | |
2021 | } | |
2022 | ||
2023 | return; | |
2024 | } | |
2025 | ||
2026 | WARN(1, "Address overflow, using WBINVD\n"); | |
2027 | } | |
2028 | ||
2029 | /* | |
2030 | * Hardware should always have one of the above features, | |
2031 | * but if not, use WBINVD and issue a warning. | |
2032 | */ | |
2033 | WARN_ONCE(1, "Using WBINVD to flush guest memory\n"); | |
2034 | wbinvd_on_all_cpus(); | |
2035 | } | |
2036 | ||
2037 | void sev_free_vcpu(struct kvm_vcpu *vcpu) | |
2038 | { | |
2039 | struct vcpu_svm *svm; | |
2040 | ||
2041 | if (!sev_es_guest(vcpu->kvm)) | |
2042 | return; | |
2043 | ||
2044 | svm = to_svm(vcpu); | |
2045 | ||
2046 | if (vcpu->arch.guest_state_protected) | |
2047 | sev_flush_guest_memory(svm, svm->vmsa, PAGE_SIZE); | |
2048 | __free_page(virt_to_page(svm->vmsa)); | |
8f423a80 TL |
2049 | |
2050 | if (svm->ghcb_sa_free) | |
2051 | kfree(svm->ghcb_sa); | |
add5e2f0 TL |
2052 | } |
2053 | ||
291bd20d TL |
2054 | static void dump_ghcb(struct vcpu_svm *svm) |
2055 | { | |
2056 | struct ghcb *ghcb = svm->ghcb; | |
2057 | unsigned int nbits; | |
2058 | ||
2059 | /* Re-use the dump_invalid_vmcb module parameter */ | |
2060 | if (!dump_invalid_vmcb) { | |
2061 | pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n"); | |
2062 | return; | |
2063 | } | |
2064 | ||
2065 | nbits = sizeof(ghcb->save.valid_bitmap) * 8; | |
2066 | ||
2067 | pr_err("GHCB (GPA=%016llx):\n", svm->vmcb->control.ghcb_gpa); | |
2068 | pr_err("%-20s%016llx is_valid: %u\n", "sw_exit_code", | |
2069 | ghcb->save.sw_exit_code, ghcb_sw_exit_code_is_valid(ghcb)); | |
2070 | pr_err("%-20s%016llx is_valid: %u\n", "sw_exit_info_1", | |
2071 | ghcb->save.sw_exit_info_1, ghcb_sw_exit_info_1_is_valid(ghcb)); | |
2072 | pr_err("%-20s%016llx is_valid: %u\n", "sw_exit_info_2", | |
2073 | ghcb->save.sw_exit_info_2, ghcb_sw_exit_info_2_is_valid(ghcb)); | |
2074 | pr_err("%-20s%016llx is_valid: %u\n", "sw_scratch", | |
2075 | ghcb->save.sw_scratch, ghcb_sw_scratch_is_valid(ghcb)); | |
2076 | pr_err("%-20s%*pb\n", "valid_bitmap", nbits, ghcb->save.valid_bitmap); | |
2077 | } | |
2078 | ||
2079 | static void sev_es_sync_to_ghcb(struct vcpu_svm *svm) | |
2080 | { | |
2081 | struct kvm_vcpu *vcpu = &svm->vcpu; | |
2082 | struct ghcb *ghcb = svm->ghcb; | |
2083 | ||
2084 | /* | |
2085 | * The GHCB protocol so far allows for the following data | |
2086 | * to be returned: | |
2087 | * GPRs RAX, RBX, RCX, RDX | |
2088 | * | |
25009140 SC |
2089 | * Copy their values, even if they may not have been written during the |
2090 | * VM-Exit. It's the guest's responsibility to not consume random data. | |
291bd20d | 2091 | */ |
25009140 SC |
2092 | ghcb_set_rax(ghcb, vcpu->arch.regs[VCPU_REGS_RAX]); |
2093 | ghcb_set_rbx(ghcb, vcpu->arch.regs[VCPU_REGS_RBX]); | |
2094 | ghcb_set_rcx(ghcb, vcpu->arch.regs[VCPU_REGS_RCX]); | |
2095 | ghcb_set_rdx(ghcb, vcpu->arch.regs[VCPU_REGS_RDX]); | |
291bd20d TL |
2096 | } |
2097 | ||
2098 | static void sev_es_sync_from_ghcb(struct vcpu_svm *svm) | |
2099 | { | |
2100 | struct vmcb_control_area *control = &svm->vmcb->control; | |
2101 | struct kvm_vcpu *vcpu = &svm->vcpu; | |
2102 | struct ghcb *ghcb = svm->ghcb; | |
2103 | u64 exit_code; | |
2104 | ||
2105 | /* | |
2106 | * The GHCB protocol so far allows for the following data | |
2107 | * to be supplied: | |
2108 | * GPRs RAX, RBX, RCX, RDX | |
2109 | * XCR0 | |
2110 | * CPL | |
2111 | * | |
2112 | * VMMCALL allows the guest to provide extra registers. KVM also | |
2113 | * expects RSI for hypercalls, so include that, too. | |
2114 | * | |
2115 | * Copy their values to the appropriate location if supplied. | |
2116 | */ | |
2117 | memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs)); | |
2118 | ||
2119 | vcpu->arch.regs[VCPU_REGS_RAX] = ghcb_get_rax_if_valid(ghcb); | |
2120 | vcpu->arch.regs[VCPU_REGS_RBX] = ghcb_get_rbx_if_valid(ghcb); | |
2121 | vcpu->arch.regs[VCPU_REGS_RCX] = ghcb_get_rcx_if_valid(ghcb); | |
2122 | vcpu->arch.regs[VCPU_REGS_RDX] = ghcb_get_rdx_if_valid(ghcb); | |
2123 | vcpu->arch.regs[VCPU_REGS_RSI] = ghcb_get_rsi_if_valid(ghcb); | |
2124 | ||
2125 | svm->vmcb->save.cpl = ghcb_get_cpl_if_valid(ghcb); | |
2126 | ||
2127 | if (ghcb_xcr0_is_valid(ghcb)) { | |
2128 | vcpu->arch.xcr0 = ghcb_get_xcr0(ghcb); | |
2129 | kvm_update_cpuid_runtime(vcpu); | |
2130 | } | |
2131 | ||
2132 | /* Copy the GHCB exit information into the VMCB fields */ | |
2133 | exit_code = ghcb_get_sw_exit_code(ghcb); | |
2134 | control->exit_code = lower_32_bits(exit_code); | |
2135 | control->exit_code_hi = upper_32_bits(exit_code); | |
2136 | control->exit_info_1 = ghcb_get_sw_exit_info_1(ghcb); | |
2137 | control->exit_info_2 = ghcb_get_sw_exit_info_2(ghcb); | |
2138 | ||
2139 | /* Clear the valid entries fields */ | |
2140 | memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); | |
2141 | } | |
2142 | ||
2143 | static int sev_es_validate_vmgexit(struct vcpu_svm *svm) | |
2144 | { | |
2145 | struct kvm_vcpu *vcpu; | |
2146 | struct ghcb *ghcb; | |
2147 | u64 exit_code = 0; | |
2148 | ||
2149 | ghcb = svm->ghcb; | |
2150 | ||
2151 | /* Only GHCB Usage code 0 is supported */ | |
2152 | if (ghcb->ghcb_usage) | |
2153 | goto vmgexit_err; | |
2154 | ||
2155 | /* | |
2156 | * Retrieve the exit code now even though is may not be marked valid | |
2157 | * as it could help with debugging. | |
2158 | */ | |
2159 | exit_code = ghcb_get_sw_exit_code(ghcb); | |
2160 | ||
2161 | if (!ghcb_sw_exit_code_is_valid(ghcb) || | |
2162 | !ghcb_sw_exit_info_1_is_valid(ghcb) || | |
2163 | !ghcb_sw_exit_info_2_is_valid(ghcb)) | |
2164 | goto vmgexit_err; | |
2165 | ||
2166 | switch (ghcb_get_sw_exit_code(ghcb)) { | |
2167 | case SVM_EXIT_READ_DR7: | |
2168 | break; | |
2169 | case SVM_EXIT_WRITE_DR7: | |
2170 | if (!ghcb_rax_is_valid(ghcb)) | |
2171 | goto vmgexit_err; | |
2172 | break; | |
2173 | case SVM_EXIT_RDTSC: | |
2174 | break; | |
2175 | case SVM_EXIT_RDPMC: | |
2176 | if (!ghcb_rcx_is_valid(ghcb)) | |
2177 | goto vmgexit_err; | |
2178 | break; | |
2179 | case SVM_EXIT_CPUID: | |
2180 | if (!ghcb_rax_is_valid(ghcb) || | |
2181 | !ghcb_rcx_is_valid(ghcb)) | |
2182 | goto vmgexit_err; | |
2183 | if (ghcb_get_rax(ghcb) == 0xd) | |
2184 | if (!ghcb_xcr0_is_valid(ghcb)) | |
2185 | goto vmgexit_err; | |
2186 | break; | |
2187 | case SVM_EXIT_INVD: | |
2188 | break; | |
2189 | case SVM_EXIT_IOIO: | |
7ed9abfe TL |
2190 | if (ghcb_get_sw_exit_info_1(ghcb) & SVM_IOIO_STR_MASK) { |
2191 | if (!ghcb_sw_scratch_is_valid(ghcb)) | |
291bd20d | 2192 | goto vmgexit_err; |
7ed9abfe TL |
2193 | } else { |
2194 | if (!(ghcb_get_sw_exit_info_1(ghcb) & SVM_IOIO_TYPE_MASK)) | |
2195 | if (!ghcb_rax_is_valid(ghcb)) | |
2196 | goto vmgexit_err; | |
2197 | } | |
291bd20d TL |
2198 | break; |
2199 | case SVM_EXIT_MSR: | |
2200 | if (!ghcb_rcx_is_valid(ghcb)) | |
2201 | goto vmgexit_err; | |
2202 | if (ghcb_get_sw_exit_info_1(ghcb)) { | |
2203 | if (!ghcb_rax_is_valid(ghcb) || | |
2204 | !ghcb_rdx_is_valid(ghcb)) | |
2205 | goto vmgexit_err; | |
2206 | } | |
2207 | break; | |
2208 | case SVM_EXIT_VMMCALL: | |
2209 | if (!ghcb_rax_is_valid(ghcb) || | |
2210 | !ghcb_cpl_is_valid(ghcb)) | |
2211 | goto vmgexit_err; | |
2212 | break; | |
2213 | case SVM_EXIT_RDTSCP: | |
2214 | break; | |
2215 | case SVM_EXIT_WBINVD: | |
2216 | break; | |
2217 | case SVM_EXIT_MONITOR: | |
2218 | if (!ghcb_rax_is_valid(ghcb) || | |
2219 | !ghcb_rcx_is_valid(ghcb) || | |
2220 | !ghcb_rdx_is_valid(ghcb)) | |
2221 | goto vmgexit_err; | |
2222 | break; | |
2223 | case SVM_EXIT_MWAIT: | |
2224 | if (!ghcb_rax_is_valid(ghcb) || | |
2225 | !ghcb_rcx_is_valid(ghcb)) | |
2226 | goto vmgexit_err; | |
2227 | break; | |
8f423a80 TL |
2228 | case SVM_VMGEXIT_MMIO_READ: |
2229 | case SVM_VMGEXIT_MMIO_WRITE: | |
2230 | if (!ghcb_sw_scratch_is_valid(ghcb)) | |
2231 | goto vmgexit_err; | |
2232 | break; | |
4444dfe4 | 2233 | case SVM_VMGEXIT_NMI_COMPLETE: |
647daca2 | 2234 | case SVM_VMGEXIT_AP_HLT_LOOP: |
8640ca58 | 2235 | case SVM_VMGEXIT_AP_JUMP_TABLE: |
291bd20d TL |
2236 | case SVM_VMGEXIT_UNSUPPORTED_EVENT: |
2237 | break; | |
2238 | default: | |
2239 | goto vmgexit_err; | |
2240 | } | |
2241 | ||
2242 | return 0; | |
2243 | ||
2244 | vmgexit_err: | |
2245 | vcpu = &svm->vcpu; | |
2246 | ||
2247 | if (ghcb->ghcb_usage) { | |
2248 | vcpu_unimpl(vcpu, "vmgexit: ghcb usage %#x is not valid\n", | |
2249 | ghcb->ghcb_usage); | |
2250 | } else { | |
2251 | vcpu_unimpl(vcpu, "vmgexit: exit reason %#llx is not valid\n", | |
2252 | exit_code); | |
2253 | dump_ghcb(svm); | |
2254 | } | |
2255 | ||
2256 | vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; | |
2257 | vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON; | |
2258 | vcpu->run->internal.ndata = 2; | |
2259 | vcpu->run->internal.data[0] = exit_code; | |
2260 | vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu; | |
2261 | ||
2262 | return -EINVAL; | |
2263 | } | |
2264 | ||
ce7ea0cf | 2265 | void sev_es_unmap_ghcb(struct vcpu_svm *svm) |
291bd20d TL |
2266 | { |
2267 | if (!svm->ghcb) | |
2268 | return; | |
2269 | ||
8f423a80 TL |
2270 | if (svm->ghcb_sa_free) { |
2271 | /* | |
2272 | * The scratch area lives outside the GHCB, so there is a | |
2273 | * buffer that, depending on the operation performed, may | |
2274 | * need to be synced, then freed. | |
2275 | */ | |
2276 | if (svm->ghcb_sa_sync) { | |
2277 | kvm_write_guest(svm->vcpu.kvm, | |
2278 | ghcb_get_sw_scratch(svm->ghcb), | |
2279 | svm->ghcb_sa, svm->ghcb_sa_len); | |
2280 | svm->ghcb_sa_sync = false; | |
2281 | } | |
2282 | ||
2283 | kfree(svm->ghcb_sa); | |
2284 | svm->ghcb_sa = NULL; | |
2285 | svm->ghcb_sa_free = false; | |
2286 | } | |
2287 | ||
d523ab6b TL |
2288 | trace_kvm_vmgexit_exit(svm->vcpu.vcpu_id, svm->ghcb); |
2289 | ||
291bd20d TL |
2290 | sev_es_sync_to_ghcb(svm); |
2291 | ||
2292 | kvm_vcpu_unmap(&svm->vcpu, &svm->ghcb_map, true); | |
2293 | svm->ghcb = NULL; | |
2294 | } | |
2295 | ||
eaf78265 JR |
2296 | void pre_sev_run(struct vcpu_svm *svm, int cpu) |
2297 | { | |
2298 | struct svm_cpu_data *sd = per_cpu(svm_data, cpu); | |
2299 | int asid = sev_get_asid(svm->vcpu.kvm); | |
2300 | ||
2301 | /* Assign the asid allocated with this SEV guest */ | |
dee734a7 | 2302 | svm->asid = asid; |
eaf78265 JR |
2303 | |
2304 | /* | |
2305 | * Flush guest TLB: | |
2306 | * | |
2307 | * 1) when different VMCB for the same ASID is to be run on the same host CPU. | |
2308 | * 2) or this VMCB was executed on different host CPU in previous VMRUNs. | |
2309 | */ | |
2310 | if (sd->sev_vmcbs[asid] == svm->vmcb && | |
8a14fe4f | 2311 | svm->vcpu.arch.last_vmentry_cpu == cpu) |
eaf78265 JR |
2312 | return; |
2313 | ||
eaf78265 JR |
2314 | sd->sev_vmcbs[asid] = svm->vmcb; |
2315 | svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID; | |
06e7852c | 2316 | vmcb_mark_dirty(svm->vmcb, VMCB_ASID); |
eaf78265 | 2317 | } |
291bd20d | 2318 | |
8f423a80 | 2319 | #define GHCB_SCRATCH_AREA_LIMIT (16ULL * PAGE_SIZE) |
2963281b | 2320 | static int setup_vmgexit_scratch(struct vcpu_svm *svm, bool sync, u64 len) |
8f423a80 TL |
2321 | { |
2322 | struct vmcb_control_area *control = &svm->vmcb->control; | |
2323 | struct ghcb *ghcb = svm->ghcb; | |
2324 | u64 ghcb_scratch_beg, ghcb_scratch_end; | |
2325 | u64 scratch_gpa_beg, scratch_gpa_end; | |
2326 | void *scratch_va; | |
2327 | ||
2328 | scratch_gpa_beg = ghcb_get_sw_scratch(ghcb); | |
2329 | if (!scratch_gpa_beg) { | |
2330 | pr_err("vmgexit: scratch gpa not provided\n"); | |
2963281b | 2331 | return -EINVAL; |
8f423a80 TL |
2332 | } |
2333 | ||
2334 | scratch_gpa_end = scratch_gpa_beg + len; | |
2335 | if (scratch_gpa_end < scratch_gpa_beg) { | |
2336 | pr_err("vmgexit: scratch length (%#llx) not valid for scratch address (%#llx)\n", | |
2337 | len, scratch_gpa_beg); | |
2963281b | 2338 | return -EINVAL; |
8f423a80 TL |
2339 | } |
2340 | ||
2341 | if ((scratch_gpa_beg & PAGE_MASK) == control->ghcb_gpa) { | |
2342 | /* Scratch area begins within GHCB */ | |
2343 | ghcb_scratch_beg = control->ghcb_gpa + | |
2344 | offsetof(struct ghcb, shared_buffer); | |
2345 | ghcb_scratch_end = control->ghcb_gpa + | |
2346 | offsetof(struct ghcb, reserved_1); | |
2347 | ||
2348 | /* | |
2349 | * If the scratch area begins within the GHCB, it must be | |
2350 | * completely contained in the GHCB shared buffer area. | |
2351 | */ | |
2352 | if (scratch_gpa_beg < ghcb_scratch_beg || | |
2353 | scratch_gpa_end > ghcb_scratch_end) { | |
2354 | pr_err("vmgexit: scratch area is outside of GHCB shared buffer area (%#llx - %#llx)\n", | |
2355 | scratch_gpa_beg, scratch_gpa_end); | |
2963281b | 2356 | return -EINVAL; |
8f423a80 TL |
2357 | } |
2358 | ||
2359 | scratch_va = (void *)svm->ghcb; | |
2360 | scratch_va += (scratch_gpa_beg - control->ghcb_gpa); | |
2361 | } else { | |
2362 | /* | |
2363 | * The guest memory must be read into a kernel buffer, so | |
2364 | * limit the size | |
2365 | */ | |
2366 | if (len > GHCB_SCRATCH_AREA_LIMIT) { | |
2367 | pr_err("vmgexit: scratch area exceeds KVM limits (%#llx requested, %#llx limit)\n", | |
2368 | len, GHCB_SCRATCH_AREA_LIMIT); | |
2963281b | 2369 | return -EINVAL; |
8f423a80 | 2370 | } |
eba04b20 | 2371 | scratch_va = kzalloc(len, GFP_KERNEL_ACCOUNT); |
8f423a80 | 2372 | if (!scratch_va) |
2963281b | 2373 | return -ENOMEM; |
8f423a80 TL |
2374 | |
2375 | if (kvm_read_guest(svm->vcpu.kvm, scratch_gpa_beg, scratch_va, len)) { | |
2376 | /* Unable to copy scratch area from guest */ | |
2377 | pr_err("vmgexit: kvm_read_guest for scratch area failed\n"); | |
2378 | ||
2379 | kfree(scratch_va); | |
2963281b | 2380 | return -EFAULT; |
8f423a80 TL |
2381 | } |
2382 | ||
2383 | /* | |
2384 | * The scratch area is outside the GHCB. The operation will | |
2385 | * dictate whether the buffer needs to be synced before running | |
2386 | * the vCPU next time (i.e. a read was requested so the data | |
2387 | * must be written back to the guest memory). | |
2388 | */ | |
2389 | svm->ghcb_sa_sync = sync; | |
2390 | svm->ghcb_sa_free = true; | |
2391 | } | |
2392 | ||
2393 | svm->ghcb_sa = scratch_va; | |
2394 | svm->ghcb_sa_len = len; | |
2395 | ||
2963281b | 2396 | return 0; |
8f423a80 TL |
2397 | } |
2398 | ||
d3694667 TL |
2399 | static void set_ghcb_msr_bits(struct vcpu_svm *svm, u64 value, u64 mask, |
2400 | unsigned int pos) | |
2401 | { | |
2402 | svm->vmcb->control.ghcb_gpa &= ~(mask << pos); | |
2403 | svm->vmcb->control.ghcb_gpa |= (value & mask) << pos; | |
2404 | } | |
2405 | ||
2406 | static u64 get_ghcb_msr_bits(struct vcpu_svm *svm, u64 mask, unsigned int pos) | |
2407 | { | |
2408 | return (svm->vmcb->control.ghcb_gpa >> pos) & mask; | |
2409 | } | |
2410 | ||
1edc1459 TL |
2411 | static void set_ghcb_msr(struct vcpu_svm *svm, u64 value) |
2412 | { | |
2413 | svm->vmcb->control.ghcb_gpa = value; | |
2414 | } | |
2415 | ||
291bd20d TL |
2416 | static int sev_handle_vmgexit_msr_protocol(struct vcpu_svm *svm) |
2417 | { | |
1edc1459 | 2418 | struct vmcb_control_area *control = &svm->vmcb->control; |
d3694667 | 2419 | struct kvm_vcpu *vcpu = &svm->vcpu; |
1edc1459 | 2420 | u64 ghcb_info; |
d3694667 | 2421 | int ret = 1; |
1edc1459 TL |
2422 | |
2423 | ghcb_info = control->ghcb_gpa & GHCB_MSR_INFO_MASK; | |
2424 | ||
59e38b58 TL |
2425 | trace_kvm_vmgexit_msr_protocol_enter(svm->vcpu.vcpu_id, |
2426 | control->ghcb_gpa); | |
2427 | ||
1edc1459 TL |
2428 | switch (ghcb_info) { |
2429 | case GHCB_MSR_SEV_INFO_REQ: | |
2430 | set_ghcb_msr(svm, GHCB_MSR_SEV_INFO(GHCB_VERSION_MAX, | |
2431 | GHCB_VERSION_MIN, | |
2432 | sev_enc_bit)); | |
2433 | break; | |
d3694667 TL |
2434 | case GHCB_MSR_CPUID_REQ: { |
2435 | u64 cpuid_fn, cpuid_reg, cpuid_value; | |
2436 | ||
2437 | cpuid_fn = get_ghcb_msr_bits(svm, | |
2438 | GHCB_MSR_CPUID_FUNC_MASK, | |
2439 | GHCB_MSR_CPUID_FUNC_POS); | |
2440 | ||
2441 | /* Initialize the registers needed by the CPUID intercept */ | |
2442 | vcpu->arch.regs[VCPU_REGS_RAX] = cpuid_fn; | |
2443 | vcpu->arch.regs[VCPU_REGS_RCX] = 0; | |
2444 | ||
63129754 | 2445 | ret = svm_invoke_exit_handler(vcpu, SVM_EXIT_CPUID); |
d3694667 TL |
2446 | if (!ret) { |
2447 | ret = -EINVAL; | |
2448 | break; | |
2449 | } | |
2450 | ||
2451 | cpuid_reg = get_ghcb_msr_bits(svm, | |
2452 | GHCB_MSR_CPUID_REG_MASK, | |
2453 | GHCB_MSR_CPUID_REG_POS); | |
2454 | if (cpuid_reg == 0) | |
2455 | cpuid_value = vcpu->arch.regs[VCPU_REGS_RAX]; | |
2456 | else if (cpuid_reg == 1) | |
2457 | cpuid_value = vcpu->arch.regs[VCPU_REGS_RBX]; | |
2458 | else if (cpuid_reg == 2) | |
2459 | cpuid_value = vcpu->arch.regs[VCPU_REGS_RCX]; | |
2460 | else | |
2461 | cpuid_value = vcpu->arch.regs[VCPU_REGS_RDX]; | |
2462 | ||
2463 | set_ghcb_msr_bits(svm, cpuid_value, | |
2464 | GHCB_MSR_CPUID_VALUE_MASK, | |
2465 | GHCB_MSR_CPUID_VALUE_POS); | |
2466 | ||
2467 | set_ghcb_msr_bits(svm, GHCB_MSR_CPUID_RESP, | |
2468 | GHCB_MSR_INFO_MASK, | |
2469 | GHCB_MSR_INFO_POS); | |
2470 | break; | |
2471 | } | |
e1d71116 TL |
2472 | case GHCB_MSR_TERM_REQ: { |
2473 | u64 reason_set, reason_code; | |
2474 | ||
2475 | reason_set = get_ghcb_msr_bits(svm, | |
2476 | GHCB_MSR_TERM_REASON_SET_MASK, | |
2477 | GHCB_MSR_TERM_REASON_SET_POS); | |
2478 | reason_code = get_ghcb_msr_bits(svm, | |
2479 | GHCB_MSR_TERM_REASON_MASK, | |
2480 | GHCB_MSR_TERM_REASON_POS); | |
2481 | pr_info("SEV-ES guest requested termination: %#llx:%#llx\n", | |
2482 | reason_set, reason_code); | |
2483 | fallthrough; | |
2484 | } | |
1edc1459 | 2485 | default: |
d3694667 | 2486 | ret = -EINVAL; |
1edc1459 TL |
2487 | } |
2488 | ||
59e38b58 TL |
2489 | trace_kvm_vmgexit_msr_protocol_exit(svm->vcpu.vcpu_id, |
2490 | control->ghcb_gpa, ret); | |
2491 | ||
d3694667 | 2492 | return ret; |
291bd20d TL |
2493 | } |
2494 | ||
63129754 | 2495 | int sev_handle_vmgexit(struct kvm_vcpu *vcpu) |
291bd20d | 2496 | { |
63129754 | 2497 | struct vcpu_svm *svm = to_svm(vcpu); |
291bd20d TL |
2498 | struct vmcb_control_area *control = &svm->vmcb->control; |
2499 | u64 ghcb_gpa, exit_code; | |
2500 | struct ghcb *ghcb; | |
2501 | int ret; | |
2502 | ||
2503 | /* Validate the GHCB */ | |
2504 | ghcb_gpa = control->ghcb_gpa; | |
2505 | if (ghcb_gpa & GHCB_MSR_INFO_MASK) | |
2506 | return sev_handle_vmgexit_msr_protocol(svm); | |
2507 | ||
2508 | if (!ghcb_gpa) { | |
63129754 | 2509 | vcpu_unimpl(vcpu, "vmgexit: GHCB gpa is not set\n"); |
291bd20d TL |
2510 | return -EINVAL; |
2511 | } | |
2512 | ||
63129754 | 2513 | if (kvm_vcpu_map(vcpu, ghcb_gpa >> PAGE_SHIFT, &svm->ghcb_map)) { |
291bd20d | 2514 | /* Unable to map GHCB from guest */ |
63129754 | 2515 | vcpu_unimpl(vcpu, "vmgexit: error mapping GHCB [%#llx] from guest\n", |
291bd20d TL |
2516 | ghcb_gpa); |
2517 | return -EINVAL; | |
2518 | } | |
2519 | ||
2520 | svm->ghcb = svm->ghcb_map.hva; | |
2521 | ghcb = svm->ghcb_map.hva; | |
2522 | ||
63129754 | 2523 | trace_kvm_vmgexit_enter(vcpu->vcpu_id, ghcb); |
d523ab6b | 2524 | |
291bd20d TL |
2525 | exit_code = ghcb_get_sw_exit_code(ghcb); |
2526 | ||
2527 | ret = sev_es_validate_vmgexit(svm); | |
2528 | if (ret) | |
2529 | return ret; | |
2530 | ||
2531 | sev_es_sync_from_ghcb(svm); | |
2532 | ghcb_set_sw_exit_info_1(ghcb, 0); | |
2533 | ghcb_set_sw_exit_info_2(ghcb, 0); | |
2534 | ||
291bd20d | 2535 | switch (exit_code) { |
8f423a80 | 2536 | case SVM_VMGEXIT_MMIO_READ: |
2963281b SC |
2537 | ret = setup_vmgexit_scratch(svm, true, control->exit_info_2); |
2538 | if (ret) | |
8f423a80 TL |
2539 | break; |
2540 | ||
63129754 | 2541 | ret = kvm_sev_es_mmio_read(vcpu, |
8f423a80 TL |
2542 | control->exit_info_1, |
2543 | control->exit_info_2, | |
2544 | svm->ghcb_sa); | |
2545 | break; | |
2546 | case SVM_VMGEXIT_MMIO_WRITE: | |
2963281b SC |
2547 | ret = setup_vmgexit_scratch(svm, false, control->exit_info_2); |
2548 | if (ret) | |
8f423a80 TL |
2549 | break; |
2550 | ||
63129754 | 2551 | ret = kvm_sev_es_mmio_write(vcpu, |
8f423a80 TL |
2552 | control->exit_info_1, |
2553 | control->exit_info_2, | |
2554 | svm->ghcb_sa); | |
2555 | break; | |
4444dfe4 | 2556 | case SVM_VMGEXIT_NMI_COMPLETE: |
63129754 | 2557 | ret = svm_invoke_exit_handler(vcpu, SVM_EXIT_IRET); |
4444dfe4 | 2558 | break; |
647daca2 | 2559 | case SVM_VMGEXIT_AP_HLT_LOOP: |
63129754 | 2560 | ret = kvm_emulate_ap_reset_hold(vcpu); |
647daca2 | 2561 | break; |
8640ca58 | 2562 | case SVM_VMGEXIT_AP_JUMP_TABLE: { |
63129754 | 2563 | struct kvm_sev_info *sev = &to_kvm_svm(vcpu->kvm)->sev_info; |
8640ca58 TL |
2564 | |
2565 | switch (control->exit_info_1) { | |
2566 | case 0: | |
2567 | /* Set AP jump table address */ | |
2568 | sev->ap_jump_table = control->exit_info_2; | |
2569 | break; | |
2570 | case 1: | |
2571 | /* Get AP jump table address */ | |
2572 | ghcb_set_sw_exit_info_2(ghcb, sev->ap_jump_table); | |
2573 | break; | |
2574 | default: | |
2575 | pr_err("svm: vmgexit: unsupported AP jump table request - exit_info_1=%#llx\n", | |
2576 | control->exit_info_1); | |
2577 | ghcb_set_sw_exit_info_1(ghcb, 1); | |
2578 | ghcb_set_sw_exit_info_2(ghcb, | |
2579 | X86_TRAP_UD | | |
2580 | SVM_EVTINJ_TYPE_EXEPT | | |
2581 | SVM_EVTINJ_VALID); | |
2582 | } | |
2583 | ||
2584 | ret = 1; | |
2585 | break; | |
2586 | } | |
291bd20d | 2587 | case SVM_VMGEXIT_UNSUPPORTED_EVENT: |
63129754 | 2588 | vcpu_unimpl(vcpu, |
291bd20d TL |
2589 | "vmgexit: unsupported event - exit_info_1=%#llx, exit_info_2=%#llx\n", |
2590 | control->exit_info_1, control->exit_info_2); | |
2963281b | 2591 | ret = -EINVAL; |
291bd20d TL |
2592 | break; |
2593 | default: | |
63129754 | 2594 | ret = svm_invoke_exit_handler(vcpu, exit_code); |
291bd20d TL |
2595 | } |
2596 | ||
2597 | return ret; | |
2598 | } | |
7ed9abfe TL |
2599 | |
2600 | int sev_es_string_io(struct vcpu_svm *svm, int size, unsigned int port, int in) | |
2601 | { | |
9b0971ca PB |
2602 | int count; |
2603 | int bytes; | |
2963281b | 2604 | int r; |
9b0971ca PB |
2605 | |
2606 | if (svm->vmcb->control.exit_info_2 > INT_MAX) | |
2607 | return -EINVAL; | |
2608 | ||
2609 | count = svm->vmcb->control.exit_info_2; | |
2610 | if (unlikely(check_mul_overflow(count, size, &bytes))) | |
2611 | return -EINVAL; | |
2612 | ||
2963281b SC |
2613 | r = setup_vmgexit_scratch(svm, in, bytes); |
2614 | if (r) | |
2615 | return r; | |
7ed9abfe | 2616 | |
9b0971ca | 2617 | return kvm_sev_es_string_io(&svm->vcpu, size, port, svm->ghcb_sa, count, in); |
7ed9abfe | 2618 | } |
376c6d28 TL |
2619 | |
2620 | void sev_es_init_vmcb(struct vcpu_svm *svm) | |
2621 | { | |
2622 | struct kvm_vcpu *vcpu = &svm->vcpu; | |
2623 | ||
2624 | svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ES_ENABLE; | |
2625 | svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK; | |
2626 | ||
2627 | /* | |
2628 | * An SEV-ES guest requires a VMSA area that is a separate from the | |
2629 | * VMCB page. Do not include the encryption mask on the VMSA physical | |
2630 | * address since hardware will access it using the guest key. | |
2631 | */ | |
2632 | svm->vmcb->control.vmsa_pa = __pa(svm->vmsa); | |
2633 | ||
2634 | /* Can't intercept CR register access, HV can't modify CR registers */ | |
2635 | svm_clr_intercept(svm, INTERCEPT_CR0_READ); | |
2636 | svm_clr_intercept(svm, INTERCEPT_CR4_READ); | |
2637 | svm_clr_intercept(svm, INTERCEPT_CR8_READ); | |
2638 | svm_clr_intercept(svm, INTERCEPT_CR0_WRITE); | |
2639 | svm_clr_intercept(svm, INTERCEPT_CR4_WRITE); | |
2640 | svm_clr_intercept(svm, INTERCEPT_CR8_WRITE); | |
2641 | ||
2642 | svm_clr_intercept(svm, INTERCEPT_SELECTIVE_CR0); | |
2643 | ||
2644 | /* Track EFER/CR register changes */ | |
2645 | svm_set_intercept(svm, TRAP_EFER_WRITE); | |
2646 | svm_set_intercept(svm, TRAP_CR0_WRITE); | |
2647 | svm_set_intercept(svm, TRAP_CR4_WRITE); | |
2648 | svm_set_intercept(svm, TRAP_CR8_WRITE); | |
2649 | ||
2650 | /* No support for enable_vmware_backdoor */ | |
2651 | clr_exception_intercept(svm, GP_VECTOR); | |
2652 | ||
2653 | /* Can't intercept XSETBV, HV can't modify XCR0 directly */ | |
2654 | svm_clr_intercept(svm, INTERCEPT_XSETBV); | |
2655 | ||
2656 | /* Clear intercepts on selected MSRs */ | |
2657 | set_msr_interception(vcpu, svm->msrpm, MSR_EFER, 1, 1); | |
2658 | set_msr_interception(vcpu, svm->msrpm, MSR_IA32_CR_PAT, 1, 1); | |
2659 | set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1); | |
2660 | set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1); | |
2661 | set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1); | |
2662 | set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1); | |
2663 | } | |
2664 | ||
2665 | void sev_es_create_vcpu(struct vcpu_svm *svm) | |
2666 | { | |
2667 | /* | |
2668 | * Set the GHCB MSR value as per the GHCB specification when creating | |
2669 | * a vCPU for an SEV-ES guest. | |
2670 | */ | |
2671 | set_ghcb_msr(svm, GHCB_MSR_SEV_INFO(GHCB_VERSION_MAX, | |
2672 | GHCB_VERSION_MIN, | |
2673 | sev_enc_bit)); | |
2674 | } | |
86137773 | 2675 | |
a7fc06dd | 2676 | void sev_es_prepare_guest_switch(struct vcpu_svm *svm, unsigned int cpu) |
86137773 TL |
2677 | { |
2678 | struct svm_cpu_data *sd = per_cpu(svm_data, cpu); | |
2679 | struct vmcb_save_area *hostsa; | |
86137773 TL |
2680 | |
2681 | /* | |
2682 | * As an SEV-ES guest, hardware will restore the host state on VMEXIT, | |
2683 | * of which one step is to perform a VMLOAD. Since hardware does not | |
2684 | * perform a VMSAVE on VMRUN, the host savearea must be updated. | |
2685 | */ | |
35a78319 | 2686 | vmsave(__sme_page_pa(sd->save_area)); |
86137773 | 2687 | |
86137773 TL |
2688 | /* XCR0 is restored on VMEXIT, save the current host value */ |
2689 | hostsa = (struct vmcb_save_area *)(page_address(sd->save_area) + 0x400); | |
2690 | hostsa->xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); | |
2691 | ||
d9f6e12f | 2692 | /* PKRU is restored on VMEXIT, save the current host value */ |
86137773 TL |
2693 | hostsa->pkru = read_pkru(); |
2694 | ||
2695 | /* MSR_IA32_XSS is restored on VMEXIT, save the currnet host value */ | |
2696 | hostsa->xss = host_xss; | |
2697 | } | |
2698 | ||
647daca2 TL |
2699 | void sev_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector) |
2700 | { | |
2701 | struct vcpu_svm *svm = to_svm(vcpu); | |
2702 | ||
2703 | /* First SIPI: Use the values as initially set by the VMM */ | |
2704 | if (!svm->received_first_sipi) { | |
2705 | svm->received_first_sipi = true; | |
2706 | return; | |
2707 | } | |
2708 | ||
2709 | /* | |
2710 | * Subsequent SIPI: Return from an AP Reset Hold VMGEXIT, where | |
2711 | * the guest will set the CS and RIP. Set SW_EXIT_INFO_2 to a | |
2712 | * non-zero value. | |
2713 | */ | |
a3ba26ec TL |
2714 | if (!svm->ghcb) |
2715 | return; | |
2716 | ||
647daca2 TL |
2717 | ghcb_set_sw_exit_info_2(svm->ghcb, 1); |
2718 | } |