2 * KVM Microsoft Hyper-V emulation
4 * derived from arch/x86/kvm/x86.c
6 * Copyright (C) 2006 Qumranet, Inc.
7 * Copyright (C) 2008 Qumranet, Inc.
8 * Copyright IBM Corporation, 2008
9 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
10 * Copyright (C) 2015 Andrey Smetanin <asmetanin@virtuozzo.com>
13 * Avi Kivity <avi@qumranet.com>
14 * Yaniv Kamay <yaniv@qumranet.com>
15 * Amit Shah <amit.shah@qumranet.com>
16 * Ben-Ami Yassour <benami@il.ibm.com>
17 * Andrey Smetanin <asmetanin@virtuozzo.com>
19 * This work is licensed under the terms of the GNU GPL, version 2. See
20 * the COPYING file in the top-level directory.
29 #include <linux/kvm_host.h>
30 #include <linux/highmem.h>
31 #include <linux/sched/cputime.h>
33 #include <asm/apicdef.h>
34 #include <trace/events/kvm.h>
38 static inline u64
synic_read_sint(struct kvm_vcpu_hv_synic
*synic
, int sint
)
40 return atomic64_read(&synic
->sint
[sint
]);
43 static inline int synic_get_sint_vector(u64 sint_value
)
45 if (sint_value
& HV_SYNIC_SINT_MASKED
)
47 return sint_value
& HV_SYNIC_SINT_VECTOR_MASK
;
50 static bool synic_has_vector_connected(struct kvm_vcpu_hv_synic
*synic
,
55 for (i
= 0; i
< ARRAY_SIZE(synic
->sint
); i
++) {
56 if (synic_get_sint_vector(synic_read_sint(synic
, i
)) == vector
)
62 static bool synic_has_vector_auto_eoi(struct kvm_vcpu_hv_synic
*synic
,
68 for (i
= 0; i
< ARRAY_SIZE(synic
->sint
); i
++) {
69 sint_value
= synic_read_sint(synic
, i
);
70 if (synic_get_sint_vector(sint_value
) == vector
&&
71 sint_value
& HV_SYNIC_SINT_AUTO_EOI
)
77 static int synic_set_sint(struct kvm_vcpu_hv_synic
*synic
, int sint
,
82 vector
= data
& HV_SYNIC_SINT_VECTOR_MASK
;
83 if (vector
< 16 && !host
)
86 * Guest may configure multiple SINTs to use the same vector, so
87 * we maintain a bitmap of vectors handled by synic, and a
88 * bitmap of vectors with auto-eoi behavior. The bitmaps are
89 * updated here, and atomically queried on fast paths.
92 atomic64_set(&synic
->sint
[sint
], data
);
94 if (synic_has_vector_connected(synic
, vector
))
95 __set_bit(vector
, synic
->vec_bitmap
);
97 __clear_bit(vector
, synic
->vec_bitmap
);
99 if (synic_has_vector_auto_eoi(synic
, vector
))
100 __set_bit(vector
, synic
->auto_eoi_bitmap
);
102 __clear_bit(vector
, synic
->auto_eoi_bitmap
);
104 /* Load SynIC vectors into EOI exit bitmap */
105 kvm_make_request(KVM_REQ_SCAN_IOAPIC
, synic_to_vcpu(synic
));
109 static struct kvm_vcpu
*get_vcpu_by_vpidx(struct kvm
*kvm
, u32 vpidx
)
111 struct kvm_vcpu
*vcpu
= NULL
;
114 if (vpidx
< KVM_MAX_VCPUS
)
115 vcpu
= kvm_get_vcpu(kvm
, vpidx
);
116 if (vcpu
&& vcpu_to_hv_vcpu(vcpu
)->vp_index
== vpidx
)
118 kvm_for_each_vcpu(i
, vcpu
, kvm
)
119 if (vcpu_to_hv_vcpu(vcpu
)->vp_index
== vpidx
)
124 static struct kvm_vcpu_hv_synic
*synic_get(struct kvm
*kvm
, u32 vpidx
)
126 struct kvm_vcpu
*vcpu
;
127 struct kvm_vcpu_hv_synic
*synic
;
129 vcpu
= get_vcpu_by_vpidx(kvm
, vpidx
);
132 synic
= vcpu_to_synic(vcpu
);
133 return (synic
->active
) ? synic
: NULL
;
136 static void synic_clear_sint_msg_pending(struct kvm_vcpu_hv_synic
*synic
,
139 struct kvm_vcpu
*vcpu
= synic_to_vcpu(synic
);
142 struct hv_message
*msg
;
143 struct hv_message_page
*msg_page
;
145 gpa
= synic
->msg_page
& PAGE_MASK
;
146 page
= kvm_vcpu_gfn_to_page(vcpu
, gpa
>> PAGE_SHIFT
);
147 if (is_error_page(page
)) {
148 vcpu_err(vcpu
, "Hyper-V SynIC can't get msg page, gpa 0x%llx\n",
152 msg_page
= kmap_atomic(page
);
154 msg
= &msg_page
->sint_message
[sint
];
155 msg
->header
.message_flags
.msg_pending
= 0;
157 kunmap_atomic(msg_page
);
158 kvm_release_page_dirty(page
);
159 kvm_vcpu_mark_page_dirty(vcpu
, gpa
>> PAGE_SHIFT
);
162 static void kvm_hv_notify_acked_sint(struct kvm_vcpu
*vcpu
, u32 sint
)
164 struct kvm
*kvm
= vcpu
->kvm
;
165 struct kvm_vcpu_hv_synic
*synic
= vcpu_to_synic(vcpu
);
166 struct kvm_vcpu_hv
*hv_vcpu
= vcpu_to_hv_vcpu(vcpu
);
167 struct kvm_vcpu_hv_stimer
*stimer
;
168 int gsi
, idx
, stimers_pending
;
170 trace_kvm_hv_notify_acked_sint(vcpu
->vcpu_id
, sint
);
172 if (synic
->msg_page
& HV_SYNIC_SIMP_ENABLE
)
173 synic_clear_sint_msg_pending(synic
, sint
);
175 /* Try to deliver pending Hyper-V SynIC timers messages */
177 for (idx
= 0; idx
< ARRAY_SIZE(hv_vcpu
->stimer
); idx
++) {
178 stimer
= &hv_vcpu
->stimer
[idx
];
179 if (stimer
->msg_pending
&&
180 (stimer
->config
& HV_STIMER_ENABLE
) &&
181 HV_STIMER_SINT(stimer
->config
) == sint
) {
182 set_bit(stimer
->index
,
183 hv_vcpu
->stimer_pending_bitmap
);
188 kvm_make_request(KVM_REQ_HV_STIMER
, vcpu
);
190 idx
= srcu_read_lock(&kvm
->irq_srcu
);
191 gsi
= atomic_read(&synic
->sint_to_gsi
[sint
]);
193 kvm_notify_acked_gsi(kvm
, gsi
);
194 srcu_read_unlock(&kvm
->irq_srcu
, idx
);
197 static void synic_exit(struct kvm_vcpu_hv_synic
*synic
, u32 msr
)
199 struct kvm_vcpu
*vcpu
= synic_to_vcpu(synic
);
200 struct kvm_vcpu_hv
*hv_vcpu
= &vcpu
->arch
.hyperv
;
202 hv_vcpu
->exit
.type
= KVM_EXIT_HYPERV_SYNIC
;
203 hv_vcpu
->exit
.u
.synic
.msr
= msr
;
204 hv_vcpu
->exit
.u
.synic
.control
= synic
->control
;
205 hv_vcpu
->exit
.u
.synic
.evt_page
= synic
->evt_page
;
206 hv_vcpu
->exit
.u
.synic
.msg_page
= synic
->msg_page
;
208 kvm_make_request(KVM_REQ_HV_EXIT
, vcpu
);
211 static int synic_set_msr(struct kvm_vcpu_hv_synic
*synic
,
212 u32 msr
, u64 data
, bool host
)
214 struct kvm_vcpu
*vcpu
= synic_to_vcpu(synic
);
220 trace_kvm_hv_synic_set_msr(vcpu
->vcpu_id
, msr
, data
, host
);
224 case HV_X64_MSR_SCONTROL
:
225 synic
->control
= data
;
227 synic_exit(synic
, msr
);
229 case HV_X64_MSR_SVERSION
:
234 synic
->version
= data
;
236 case HV_X64_MSR_SIEFP
:
237 if ((data
& HV_SYNIC_SIEFP_ENABLE
) && !host
&&
238 !synic
->dont_zero_synic_pages
)
239 if (kvm_clear_guest(vcpu
->kvm
,
240 data
& PAGE_MASK
, PAGE_SIZE
)) {
244 synic
->evt_page
= data
;
246 synic_exit(synic
, msr
);
248 case HV_X64_MSR_SIMP
:
249 if ((data
& HV_SYNIC_SIMP_ENABLE
) && !host
&&
250 !synic
->dont_zero_synic_pages
)
251 if (kvm_clear_guest(vcpu
->kvm
,
252 data
& PAGE_MASK
, PAGE_SIZE
)) {
256 synic
->msg_page
= data
;
258 synic_exit(synic
, msr
);
260 case HV_X64_MSR_EOM
: {
263 for (i
= 0; i
< ARRAY_SIZE(synic
->sint
); i
++)
264 kvm_hv_notify_acked_sint(vcpu
, i
);
267 case HV_X64_MSR_SINT0
... HV_X64_MSR_SINT15
:
268 ret
= synic_set_sint(synic
, msr
- HV_X64_MSR_SINT0
, data
, host
);
277 static int synic_get_msr(struct kvm_vcpu_hv_synic
*synic
, u32 msr
, u64
*pdata
)
286 case HV_X64_MSR_SCONTROL
:
287 *pdata
= synic
->control
;
289 case HV_X64_MSR_SVERSION
:
290 *pdata
= synic
->version
;
292 case HV_X64_MSR_SIEFP
:
293 *pdata
= synic
->evt_page
;
295 case HV_X64_MSR_SIMP
:
296 *pdata
= synic
->msg_page
;
301 case HV_X64_MSR_SINT0
... HV_X64_MSR_SINT15
:
302 *pdata
= atomic64_read(&synic
->sint
[msr
- HV_X64_MSR_SINT0
]);
311 static int synic_set_irq(struct kvm_vcpu_hv_synic
*synic
, u32 sint
)
313 struct kvm_vcpu
*vcpu
= synic_to_vcpu(synic
);
314 struct kvm_lapic_irq irq
;
317 if (sint
>= ARRAY_SIZE(synic
->sint
))
320 vector
= synic_get_sint_vector(synic_read_sint(synic
, sint
));
324 memset(&irq
, 0, sizeof(irq
));
325 irq
.shorthand
= APIC_DEST_SELF
;
326 irq
.dest_mode
= APIC_DEST_PHYSICAL
;
327 irq
.delivery_mode
= APIC_DM_FIXED
;
331 ret
= kvm_irq_delivery_to_apic(vcpu
->kvm
, vcpu
->arch
.apic
, &irq
, NULL
);
332 trace_kvm_hv_synic_set_irq(vcpu
->vcpu_id
, sint
, irq
.vector
, ret
);
336 int kvm_hv_synic_set_irq(struct kvm
*kvm
, u32 vpidx
, u32 sint
)
338 struct kvm_vcpu_hv_synic
*synic
;
340 synic
= synic_get(kvm
, vpidx
);
344 return synic_set_irq(synic
, sint
);
347 void kvm_hv_synic_send_eoi(struct kvm_vcpu
*vcpu
, int vector
)
349 struct kvm_vcpu_hv_synic
*synic
= vcpu_to_synic(vcpu
);
352 trace_kvm_hv_synic_send_eoi(vcpu
->vcpu_id
, vector
);
354 for (i
= 0; i
< ARRAY_SIZE(synic
->sint
); i
++)
355 if (synic_get_sint_vector(synic_read_sint(synic
, i
)) == vector
)
356 kvm_hv_notify_acked_sint(vcpu
, i
);
359 static int kvm_hv_set_sint_gsi(struct kvm
*kvm
, u32 vpidx
, u32 sint
, int gsi
)
361 struct kvm_vcpu_hv_synic
*synic
;
363 synic
= synic_get(kvm
, vpidx
);
367 if (sint
>= ARRAY_SIZE(synic
->sint_to_gsi
))
370 atomic_set(&synic
->sint_to_gsi
[sint
], gsi
);
374 void kvm_hv_irq_routing_update(struct kvm
*kvm
)
376 struct kvm_irq_routing_table
*irq_rt
;
377 struct kvm_kernel_irq_routing_entry
*e
;
380 irq_rt
= srcu_dereference_check(kvm
->irq_routing
, &kvm
->irq_srcu
,
381 lockdep_is_held(&kvm
->irq_lock
));
383 for (gsi
= 0; gsi
< irq_rt
->nr_rt_entries
; gsi
++) {
384 hlist_for_each_entry(e
, &irq_rt
->map
[gsi
], link
) {
385 if (e
->type
== KVM_IRQ_ROUTING_HV_SINT
)
386 kvm_hv_set_sint_gsi(kvm
, e
->hv_sint
.vcpu
,
387 e
->hv_sint
.sint
, gsi
);
392 static void synic_init(struct kvm_vcpu_hv_synic
*synic
)
396 memset(synic
, 0, sizeof(*synic
));
397 synic
->version
= HV_SYNIC_VERSION_1
;
398 for (i
= 0; i
< ARRAY_SIZE(synic
->sint
); i
++) {
399 atomic64_set(&synic
->sint
[i
], HV_SYNIC_SINT_MASKED
);
400 atomic_set(&synic
->sint_to_gsi
[i
], -1);
404 static u64
get_time_ref_counter(struct kvm
*kvm
)
406 struct kvm_hv
*hv
= &kvm
->arch
.hyperv
;
407 struct kvm_vcpu
*vcpu
;
411 * The guest has not set up the TSC page or the clock isn't
412 * stable, fall back to get_kvmclock_ns.
414 if (!hv
->tsc_ref
.tsc_sequence
)
415 return div_u64(get_kvmclock_ns(kvm
), 100);
417 vcpu
= kvm_get_vcpu(kvm
, 0);
418 tsc
= kvm_read_l1_tsc(vcpu
, rdtsc());
419 return mul_u64_u64_shr(tsc
, hv
->tsc_ref
.tsc_scale
, 64)
420 + hv
->tsc_ref
.tsc_offset
;
423 static void stimer_mark_pending(struct kvm_vcpu_hv_stimer
*stimer
,
426 struct kvm_vcpu
*vcpu
= stimer_to_vcpu(stimer
);
428 set_bit(stimer
->index
,
429 vcpu_to_hv_vcpu(vcpu
)->stimer_pending_bitmap
);
430 kvm_make_request(KVM_REQ_HV_STIMER
, vcpu
);
435 static void stimer_cleanup(struct kvm_vcpu_hv_stimer
*stimer
)
437 struct kvm_vcpu
*vcpu
= stimer_to_vcpu(stimer
);
439 trace_kvm_hv_stimer_cleanup(stimer_to_vcpu(stimer
)->vcpu_id
,
442 hrtimer_cancel(&stimer
->timer
);
443 clear_bit(stimer
->index
,
444 vcpu_to_hv_vcpu(vcpu
)->stimer_pending_bitmap
);
445 stimer
->msg_pending
= false;
446 stimer
->exp_time
= 0;
449 static enum hrtimer_restart
stimer_timer_callback(struct hrtimer
*timer
)
451 struct kvm_vcpu_hv_stimer
*stimer
;
453 stimer
= container_of(timer
, struct kvm_vcpu_hv_stimer
, timer
);
454 trace_kvm_hv_stimer_callback(stimer_to_vcpu(stimer
)->vcpu_id
,
456 stimer_mark_pending(stimer
, true);
458 return HRTIMER_NORESTART
;
462 * stimer_start() assumptions:
463 * a) stimer->count is not equal to 0
464 * b) stimer->config has HV_STIMER_ENABLE flag
466 static int stimer_start(struct kvm_vcpu_hv_stimer
*stimer
)
471 time_now
= get_time_ref_counter(stimer_to_vcpu(stimer
)->kvm
);
472 ktime_now
= ktime_get();
474 if (stimer
->config
& HV_STIMER_PERIODIC
) {
475 if (stimer
->exp_time
) {
476 if (time_now
>= stimer
->exp_time
) {
479 div64_u64_rem(time_now
- stimer
->exp_time
,
480 stimer
->count
, &remainder
);
482 time_now
+ (stimer
->count
- remainder
);
485 stimer
->exp_time
= time_now
+ stimer
->count
;
487 trace_kvm_hv_stimer_start_periodic(
488 stimer_to_vcpu(stimer
)->vcpu_id
,
490 time_now
, stimer
->exp_time
);
492 hrtimer_start(&stimer
->timer
,
493 ktime_add_ns(ktime_now
,
494 100 * (stimer
->exp_time
- time_now
)),
498 stimer
->exp_time
= stimer
->count
;
499 if (time_now
>= stimer
->count
) {
501 * Expire timer according to Hypervisor Top-Level Functional
502 * specification v4(15.3.1):
503 * "If a one shot is enabled and the specified count is in
504 * the past, it will expire immediately."
506 stimer_mark_pending(stimer
, false);
510 trace_kvm_hv_stimer_start_one_shot(stimer_to_vcpu(stimer
)->vcpu_id
,
512 time_now
, stimer
->count
);
514 hrtimer_start(&stimer
->timer
,
515 ktime_add_ns(ktime_now
, 100 * (stimer
->count
- time_now
)),
520 static int stimer_set_config(struct kvm_vcpu_hv_stimer
*stimer
, u64 config
,
523 trace_kvm_hv_stimer_set_config(stimer_to_vcpu(stimer
)->vcpu_id
,
524 stimer
->index
, config
, host
);
526 stimer_cleanup(stimer
);
527 if ((stimer
->config
& HV_STIMER_ENABLE
) && HV_STIMER_SINT(config
) == 0)
528 config
&= ~HV_STIMER_ENABLE
;
529 stimer
->config
= config
;
530 stimer_mark_pending(stimer
, false);
534 static int stimer_set_count(struct kvm_vcpu_hv_stimer
*stimer
, u64 count
,
537 trace_kvm_hv_stimer_set_count(stimer_to_vcpu(stimer
)->vcpu_id
,
538 stimer
->index
, count
, host
);
540 stimer_cleanup(stimer
);
541 stimer
->count
= count
;
542 if (stimer
->count
== 0)
543 stimer
->config
&= ~HV_STIMER_ENABLE
;
544 else if (stimer
->config
& HV_STIMER_AUTOENABLE
)
545 stimer
->config
|= HV_STIMER_ENABLE
;
546 stimer_mark_pending(stimer
, false);
550 static int stimer_get_config(struct kvm_vcpu_hv_stimer
*stimer
, u64
*pconfig
)
552 *pconfig
= stimer
->config
;
556 static int stimer_get_count(struct kvm_vcpu_hv_stimer
*stimer
, u64
*pcount
)
558 *pcount
= stimer
->count
;
562 static int synic_deliver_msg(struct kvm_vcpu_hv_synic
*synic
, u32 sint
,
563 struct hv_message
*src_msg
)
565 struct kvm_vcpu
*vcpu
= synic_to_vcpu(synic
);
568 struct hv_message
*dst_msg
;
570 struct hv_message_page
*msg_page
;
572 if (!(synic
->msg_page
& HV_SYNIC_SIMP_ENABLE
))
575 gpa
= synic
->msg_page
& PAGE_MASK
;
576 page
= kvm_vcpu_gfn_to_page(vcpu
, gpa
>> PAGE_SHIFT
);
577 if (is_error_page(page
))
580 msg_page
= kmap_atomic(page
);
581 dst_msg
= &msg_page
->sint_message
[sint
];
582 if (sync_cmpxchg(&dst_msg
->header
.message_type
, HVMSG_NONE
,
583 src_msg
->header
.message_type
) != HVMSG_NONE
) {
584 dst_msg
->header
.message_flags
.msg_pending
= 1;
587 memcpy(&dst_msg
->u
.payload
, &src_msg
->u
.payload
,
588 src_msg
->header
.payload_size
);
589 dst_msg
->header
.message_type
= src_msg
->header
.message_type
;
590 dst_msg
->header
.payload_size
= src_msg
->header
.payload_size
;
591 r
= synic_set_irq(synic
, sint
);
597 kunmap_atomic(msg_page
);
598 kvm_release_page_dirty(page
);
599 kvm_vcpu_mark_page_dirty(vcpu
, gpa
>> PAGE_SHIFT
);
603 static int stimer_send_msg(struct kvm_vcpu_hv_stimer
*stimer
)
605 struct kvm_vcpu
*vcpu
= stimer_to_vcpu(stimer
);
606 struct hv_message
*msg
= &stimer
->msg
;
607 struct hv_timer_message_payload
*payload
=
608 (struct hv_timer_message_payload
*)&msg
->u
.payload
;
610 payload
->expiration_time
= stimer
->exp_time
;
611 payload
->delivery_time
= get_time_ref_counter(vcpu
->kvm
);
612 return synic_deliver_msg(vcpu_to_synic(vcpu
),
613 HV_STIMER_SINT(stimer
->config
), msg
);
616 static void stimer_expiration(struct kvm_vcpu_hv_stimer
*stimer
)
620 stimer
->msg_pending
= true;
621 r
= stimer_send_msg(stimer
);
622 trace_kvm_hv_stimer_expiration(stimer_to_vcpu(stimer
)->vcpu_id
,
625 stimer
->msg_pending
= false;
626 if (!(stimer
->config
& HV_STIMER_PERIODIC
))
627 stimer
->config
&= ~HV_STIMER_ENABLE
;
631 void kvm_hv_process_stimers(struct kvm_vcpu
*vcpu
)
633 struct kvm_vcpu_hv
*hv_vcpu
= vcpu_to_hv_vcpu(vcpu
);
634 struct kvm_vcpu_hv_stimer
*stimer
;
635 u64 time_now
, exp_time
;
638 for (i
= 0; i
< ARRAY_SIZE(hv_vcpu
->stimer
); i
++)
639 if (test_and_clear_bit(i
, hv_vcpu
->stimer_pending_bitmap
)) {
640 stimer
= &hv_vcpu
->stimer
[i
];
641 if (stimer
->config
& HV_STIMER_ENABLE
) {
642 exp_time
= stimer
->exp_time
;
646 get_time_ref_counter(vcpu
->kvm
);
647 if (time_now
>= exp_time
)
648 stimer_expiration(stimer
);
651 if ((stimer
->config
& HV_STIMER_ENABLE
) &&
653 stimer_start(stimer
);
655 stimer_cleanup(stimer
);
660 void kvm_hv_vcpu_uninit(struct kvm_vcpu
*vcpu
)
662 struct kvm_vcpu_hv
*hv_vcpu
= vcpu_to_hv_vcpu(vcpu
);
665 for (i
= 0; i
< ARRAY_SIZE(hv_vcpu
->stimer
); i
++)
666 stimer_cleanup(&hv_vcpu
->stimer
[i
]);
669 static void stimer_prepare_msg(struct kvm_vcpu_hv_stimer
*stimer
)
671 struct hv_message
*msg
= &stimer
->msg
;
672 struct hv_timer_message_payload
*payload
=
673 (struct hv_timer_message_payload
*)&msg
->u
.payload
;
675 memset(&msg
->header
, 0, sizeof(msg
->header
));
676 msg
->header
.message_type
= HVMSG_TIMER_EXPIRED
;
677 msg
->header
.payload_size
= sizeof(*payload
);
679 payload
->timer_index
= stimer
->index
;
680 payload
->expiration_time
= 0;
681 payload
->delivery_time
= 0;
684 static void stimer_init(struct kvm_vcpu_hv_stimer
*stimer
, int timer_index
)
686 memset(stimer
, 0, sizeof(*stimer
));
687 stimer
->index
= timer_index
;
688 hrtimer_init(&stimer
->timer
, CLOCK_MONOTONIC
, HRTIMER_MODE_ABS
);
689 stimer
->timer
.function
= stimer_timer_callback
;
690 stimer_prepare_msg(stimer
);
693 void kvm_hv_vcpu_init(struct kvm_vcpu
*vcpu
)
695 struct kvm_vcpu_hv
*hv_vcpu
= vcpu_to_hv_vcpu(vcpu
);
698 synic_init(&hv_vcpu
->synic
);
700 bitmap_zero(hv_vcpu
->stimer_pending_bitmap
, HV_SYNIC_STIMER_COUNT
);
701 for (i
= 0; i
< ARRAY_SIZE(hv_vcpu
->stimer
); i
++)
702 stimer_init(&hv_vcpu
->stimer
[i
], i
);
705 void kvm_hv_vcpu_postcreate(struct kvm_vcpu
*vcpu
)
707 struct kvm_vcpu_hv
*hv_vcpu
= vcpu_to_hv_vcpu(vcpu
);
709 hv_vcpu
->vp_index
= kvm_vcpu_get_idx(vcpu
);
712 int kvm_hv_activate_synic(struct kvm_vcpu
*vcpu
, bool dont_zero_synic_pages
)
714 struct kvm_vcpu_hv_synic
*synic
= vcpu_to_synic(vcpu
);
717 * Hyper-V SynIC auto EOI SINT's are
718 * not compatible with APICV, so deactivate APICV
720 kvm_vcpu_deactivate_apicv(vcpu
);
721 synic
->active
= true;
722 synic
->dont_zero_synic_pages
= dont_zero_synic_pages
;
726 static bool kvm_hv_msr_partition_wide(u32 msr
)
731 case HV_X64_MSR_GUEST_OS_ID
:
732 case HV_X64_MSR_HYPERCALL
:
733 case HV_X64_MSR_REFERENCE_TSC
:
734 case HV_X64_MSR_TIME_REF_COUNT
:
735 case HV_X64_MSR_CRASH_CTL
:
736 case HV_X64_MSR_CRASH_P0
... HV_X64_MSR_CRASH_P4
:
737 case HV_X64_MSR_RESET
:
745 static int kvm_hv_msr_get_crash_data(struct kvm_vcpu
*vcpu
,
746 u32 index
, u64
*pdata
)
748 struct kvm_hv
*hv
= &vcpu
->kvm
->arch
.hyperv
;
750 if (WARN_ON_ONCE(index
>= ARRAY_SIZE(hv
->hv_crash_param
)))
753 *pdata
= hv
->hv_crash_param
[index
];
757 static int kvm_hv_msr_get_crash_ctl(struct kvm_vcpu
*vcpu
, u64
*pdata
)
759 struct kvm_hv
*hv
= &vcpu
->kvm
->arch
.hyperv
;
761 *pdata
= hv
->hv_crash_ctl
;
765 static int kvm_hv_msr_set_crash_ctl(struct kvm_vcpu
*vcpu
, u64 data
, bool host
)
767 struct kvm_hv
*hv
= &vcpu
->kvm
->arch
.hyperv
;
770 hv
->hv_crash_ctl
= data
& HV_X64_MSR_CRASH_CTL_NOTIFY
;
772 if (!host
&& (data
& HV_X64_MSR_CRASH_CTL_NOTIFY
)) {
774 vcpu_debug(vcpu
, "hv crash (0x%llx 0x%llx 0x%llx 0x%llx 0x%llx)\n",
775 hv
->hv_crash_param
[0],
776 hv
->hv_crash_param
[1],
777 hv
->hv_crash_param
[2],
778 hv
->hv_crash_param
[3],
779 hv
->hv_crash_param
[4]);
781 /* Send notification about crash to user space */
782 kvm_make_request(KVM_REQ_HV_CRASH
, vcpu
);
788 static int kvm_hv_msr_set_crash_data(struct kvm_vcpu
*vcpu
,
791 struct kvm_hv
*hv
= &vcpu
->kvm
->arch
.hyperv
;
793 if (WARN_ON_ONCE(index
>= ARRAY_SIZE(hv
->hv_crash_param
)))
796 hv
->hv_crash_param
[index
] = data
;
801 * The kvmclock and Hyper-V TSC page use similar formulas, and converting
802 * between them is possible:
805 * nsec = (ticks - tsc_timestamp) * tsc_to_system_mul * 2^(tsc_shift-32)
809 * nsec/100 = ticks * scale / 2^64 + offset
811 * When tsc_timestamp = system_time = 0, offset is zero in the Hyper-V formula.
812 * By dividing the kvmclock formula by 100 and equating what's left we get:
813 * ticks * scale / 2^64 = ticks * tsc_to_system_mul * 2^(tsc_shift-32) / 100
814 * scale / 2^64 = tsc_to_system_mul * 2^(tsc_shift-32) / 100
815 * scale = tsc_to_system_mul * 2^(32+tsc_shift) / 100
817 * Now expand the kvmclock formula and divide by 100:
818 * nsec = ticks * tsc_to_system_mul * 2^(tsc_shift-32)
819 * - tsc_timestamp * tsc_to_system_mul * 2^(tsc_shift-32)
821 * nsec/100 = ticks * tsc_to_system_mul * 2^(tsc_shift-32) / 100
822 * - tsc_timestamp * tsc_to_system_mul * 2^(tsc_shift-32) / 100
823 * + system_time / 100
825 * Replace tsc_to_system_mul * 2^(tsc_shift-32) / 100 by scale / 2^64:
826 * nsec/100 = ticks * scale / 2^64
827 * - tsc_timestamp * scale / 2^64
828 * + system_time / 100
830 * Equate with the Hyper-V formula so that ticks * scale / 2^64 cancels out:
831 * offset = system_time / 100 - tsc_timestamp * scale / 2^64
833 * These two equivalencies are implemented in this function.
835 static bool compute_tsc_page_parameters(struct pvclock_vcpu_time_info
*hv_clock
,
836 HV_REFERENCE_TSC_PAGE
*tsc_ref
)
840 if (!(hv_clock
->flags
& PVCLOCK_TSC_STABLE_BIT
))
844 * check if scale would overflow, if so we use the time ref counter
845 * tsc_to_system_mul * 2^(tsc_shift+32) / 100 >= 2^64
846 * tsc_to_system_mul / 100 >= 2^(32-tsc_shift)
847 * tsc_to_system_mul >= 100 * 2^(32-tsc_shift)
849 max_mul
= 100ull << (32 - hv_clock
->tsc_shift
);
850 if (hv_clock
->tsc_to_system_mul
>= max_mul
)
854 * Otherwise compute the scale and offset according to the formulas
858 mul_u64_u32_div(1ULL << (32 + hv_clock
->tsc_shift
),
859 hv_clock
->tsc_to_system_mul
,
862 tsc_ref
->tsc_offset
= hv_clock
->system_time
;
863 do_div(tsc_ref
->tsc_offset
, 100);
864 tsc_ref
->tsc_offset
-=
865 mul_u64_u64_shr(hv_clock
->tsc_timestamp
, tsc_ref
->tsc_scale
, 64);
869 void kvm_hv_setup_tsc_page(struct kvm
*kvm
,
870 struct pvclock_vcpu_time_info
*hv_clock
)
872 struct kvm_hv
*hv
= &kvm
->arch
.hyperv
;
876 BUILD_BUG_ON(sizeof(tsc_seq
) != sizeof(hv
->tsc_ref
.tsc_sequence
));
877 BUILD_BUG_ON(offsetof(HV_REFERENCE_TSC_PAGE
, tsc_sequence
) != 0);
879 if (!(hv
->hv_tsc_page
& HV_X64_MSR_TSC_REFERENCE_ENABLE
))
882 mutex_lock(&kvm
->arch
.hyperv
.hv_lock
);
883 if (!(hv
->hv_tsc_page
& HV_X64_MSR_TSC_REFERENCE_ENABLE
))
886 gfn
= hv
->hv_tsc_page
>> HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT
;
888 * Because the TSC parameters only vary when there is a
889 * change in the master clock, do not bother with caching.
891 if (unlikely(kvm_read_guest(kvm
, gfn_to_gpa(gfn
),
892 &tsc_seq
, sizeof(tsc_seq
))))
896 * While we're computing and writing the parameters, force the
897 * guest to use the time reference count MSR.
899 hv
->tsc_ref
.tsc_sequence
= 0;
900 if (kvm_write_guest(kvm
, gfn_to_gpa(gfn
),
901 &hv
->tsc_ref
, sizeof(hv
->tsc_ref
.tsc_sequence
)))
904 if (!compute_tsc_page_parameters(hv_clock
, &hv
->tsc_ref
))
907 /* Ensure sequence is zero before writing the rest of the struct. */
909 if (kvm_write_guest(kvm
, gfn_to_gpa(gfn
), &hv
->tsc_ref
, sizeof(hv
->tsc_ref
)))
913 * Now switch to the TSC page mechanism by writing the sequence.
916 if (tsc_seq
== 0xFFFFFFFF || tsc_seq
== 0)
919 /* Write the struct entirely before the non-zero sequence. */
922 hv
->tsc_ref
.tsc_sequence
= tsc_seq
;
923 kvm_write_guest(kvm
, gfn_to_gpa(gfn
),
924 &hv
->tsc_ref
, sizeof(hv
->tsc_ref
.tsc_sequence
));
926 mutex_unlock(&kvm
->arch
.hyperv
.hv_lock
);
929 static int kvm_hv_set_msr_pw(struct kvm_vcpu
*vcpu
, u32 msr
, u64 data
,
932 struct kvm
*kvm
= vcpu
->kvm
;
933 struct kvm_hv
*hv
= &kvm
->arch
.hyperv
;
936 case HV_X64_MSR_GUEST_OS_ID
:
937 hv
->hv_guest_os_id
= data
;
938 /* setting guest os id to zero disables hypercall page */
939 if (!hv
->hv_guest_os_id
)
940 hv
->hv_hypercall
&= ~HV_X64_MSR_HYPERCALL_ENABLE
;
942 case HV_X64_MSR_HYPERCALL
: {
947 /* if guest os id is not set hypercall should remain disabled */
948 if (!hv
->hv_guest_os_id
)
950 if (!(data
& HV_X64_MSR_HYPERCALL_ENABLE
)) {
951 hv
->hv_hypercall
= data
;
954 gfn
= data
>> HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT
;
955 addr
= gfn_to_hva(kvm
, gfn
);
956 if (kvm_is_error_hva(addr
))
958 kvm_x86_ops
->patch_hypercall(vcpu
, instructions
);
959 ((unsigned char *)instructions
)[3] = 0xc3; /* ret */
960 if (__copy_to_user((void __user
*)addr
, instructions
, 4))
962 hv
->hv_hypercall
= data
;
963 mark_page_dirty(kvm
, gfn
);
966 case HV_X64_MSR_REFERENCE_TSC
:
967 hv
->hv_tsc_page
= data
;
968 if (hv
->hv_tsc_page
& HV_X64_MSR_TSC_REFERENCE_ENABLE
)
969 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE
, vcpu
);
971 case HV_X64_MSR_CRASH_P0
... HV_X64_MSR_CRASH_P4
:
972 return kvm_hv_msr_set_crash_data(vcpu
,
973 msr
- HV_X64_MSR_CRASH_P0
,
975 case HV_X64_MSR_CRASH_CTL
:
976 return kvm_hv_msr_set_crash_ctl(vcpu
, data
, host
);
977 case HV_X64_MSR_RESET
:
979 vcpu_debug(vcpu
, "hyper-v reset requested\n");
980 kvm_make_request(KVM_REQ_HV_RESET
, vcpu
);
984 vcpu_unimpl(vcpu
, "Hyper-V uhandled wrmsr: 0x%x data 0x%llx\n",
991 /* Calculate cpu time spent by current task in 100ns units */
992 static u64
current_task_runtime_100ns(void)
996 task_cputime_adjusted(current
, &utime
, &stime
);
998 return div_u64(utime
+ stime
, 100);
1001 static int kvm_hv_set_msr(struct kvm_vcpu
*vcpu
, u32 msr
, u64 data
, bool host
)
1003 struct kvm_vcpu_hv
*hv
= &vcpu
->arch
.hyperv
;
1006 case HV_X64_MSR_VP_INDEX
:
1009 hv
->vp_index
= (u32
)data
;
1011 case HV_X64_MSR_APIC_ASSIST_PAGE
: {
1015 if (!(data
& HV_X64_MSR_APIC_ASSIST_PAGE_ENABLE
)) {
1016 hv
->hv_vapic
= data
;
1017 if (kvm_lapic_enable_pv_eoi(vcpu
, 0))
1021 gfn
= data
>> HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT
;
1022 addr
= kvm_vcpu_gfn_to_hva(vcpu
, gfn
);
1023 if (kvm_is_error_hva(addr
))
1025 if (__clear_user((void __user
*)addr
, PAGE_SIZE
))
1027 hv
->hv_vapic
= data
;
1028 kvm_vcpu_mark_page_dirty(vcpu
, gfn
);
1029 if (kvm_lapic_enable_pv_eoi(vcpu
,
1030 gfn_to_gpa(gfn
) | KVM_MSR_ENABLED
))
1034 case HV_X64_MSR_EOI
:
1035 return kvm_hv_vapic_msr_write(vcpu
, APIC_EOI
, data
);
1036 case HV_X64_MSR_ICR
:
1037 return kvm_hv_vapic_msr_write(vcpu
, APIC_ICR
, data
);
1038 case HV_X64_MSR_TPR
:
1039 return kvm_hv_vapic_msr_write(vcpu
, APIC_TASKPRI
, data
);
1040 case HV_X64_MSR_VP_RUNTIME
:
1043 hv
->runtime_offset
= data
- current_task_runtime_100ns();
1045 case HV_X64_MSR_SCONTROL
:
1046 case HV_X64_MSR_SVERSION
:
1047 case HV_X64_MSR_SIEFP
:
1048 case HV_X64_MSR_SIMP
:
1049 case HV_X64_MSR_EOM
:
1050 case HV_X64_MSR_SINT0
... HV_X64_MSR_SINT15
:
1051 return synic_set_msr(vcpu_to_synic(vcpu
), msr
, data
, host
);
1052 case HV_X64_MSR_STIMER0_CONFIG
:
1053 case HV_X64_MSR_STIMER1_CONFIG
:
1054 case HV_X64_MSR_STIMER2_CONFIG
:
1055 case HV_X64_MSR_STIMER3_CONFIG
: {
1056 int timer_index
= (msr
- HV_X64_MSR_STIMER0_CONFIG
)/2;
1058 return stimer_set_config(vcpu_to_stimer(vcpu
, timer_index
),
1061 case HV_X64_MSR_STIMER0_COUNT
:
1062 case HV_X64_MSR_STIMER1_COUNT
:
1063 case HV_X64_MSR_STIMER2_COUNT
:
1064 case HV_X64_MSR_STIMER3_COUNT
: {
1065 int timer_index
= (msr
- HV_X64_MSR_STIMER0_COUNT
)/2;
1067 return stimer_set_count(vcpu_to_stimer(vcpu
, timer_index
),
1071 vcpu_unimpl(vcpu
, "Hyper-V uhandled wrmsr: 0x%x data 0x%llx\n",
1079 static int kvm_hv_get_msr_pw(struct kvm_vcpu
*vcpu
, u32 msr
, u64
*pdata
)
1082 struct kvm
*kvm
= vcpu
->kvm
;
1083 struct kvm_hv
*hv
= &kvm
->arch
.hyperv
;
1086 case HV_X64_MSR_GUEST_OS_ID
:
1087 data
= hv
->hv_guest_os_id
;
1089 case HV_X64_MSR_HYPERCALL
:
1090 data
= hv
->hv_hypercall
;
1092 case HV_X64_MSR_TIME_REF_COUNT
:
1093 data
= get_time_ref_counter(kvm
);
1095 case HV_X64_MSR_REFERENCE_TSC
:
1096 data
= hv
->hv_tsc_page
;
1098 case HV_X64_MSR_CRASH_P0
... HV_X64_MSR_CRASH_P4
:
1099 return kvm_hv_msr_get_crash_data(vcpu
,
1100 msr
- HV_X64_MSR_CRASH_P0
,
1102 case HV_X64_MSR_CRASH_CTL
:
1103 return kvm_hv_msr_get_crash_ctl(vcpu
, pdata
);
1104 case HV_X64_MSR_RESET
:
1108 vcpu_unimpl(vcpu
, "Hyper-V unhandled rdmsr: 0x%x\n", msr
);
1116 static int kvm_hv_get_msr(struct kvm_vcpu
*vcpu
, u32 msr
, u64
*pdata
)
1119 struct kvm_vcpu_hv
*hv
= &vcpu
->arch
.hyperv
;
1122 case HV_X64_MSR_VP_INDEX
:
1123 data
= hv
->vp_index
;
1125 case HV_X64_MSR_EOI
:
1126 return kvm_hv_vapic_msr_read(vcpu
, APIC_EOI
, pdata
);
1127 case HV_X64_MSR_ICR
:
1128 return kvm_hv_vapic_msr_read(vcpu
, APIC_ICR
, pdata
);
1129 case HV_X64_MSR_TPR
:
1130 return kvm_hv_vapic_msr_read(vcpu
, APIC_TASKPRI
, pdata
);
1131 case HV_X64_MSR_APIC_ASSIST_PAGE
:
1132 data
= hv
->hv_vapic
;
1134 case HV_X64_MSR_VP_RUNTIME
:
1135 data
= current_task_runtime_100ns() + hv
->runtime_offset
;
1137 case HV_X64_MSR_SCONTROL
:
1138 case HV_X64_MSR_SVERSION
:
1139 case HV_X64_MSR_SIEFP
:
1140 case HV_X64_MSR_SIMP
:
1141 case HV_X64_MSR_EOM
:
1142 case HV_X64_MSR_SINT0
... HV_X64_MSR_SINT15
:
1143 return synic_get_msr(vcpu_to_synic(vcpu
), msr
, pdata
);
1144 case HV_X64_MSR_STIMER0_CONFIG
:
1145 case HV_X64_MSR_STIMER1_CONFIG
:
1146 case HV_X64_MSR_STIMER2_CONFIG
:
1147 case HV_X64_MSR_STIMER3_CONFIG
: {
1148 int timer_index
= (msr
- HV_X64_MSR_STIMER0_CONFIG
)/2;
1150 return stimer_get_config(vcpu_to_stimer(vcpu
, timer_index
),
1153 case HV_X64_MSR_STIMER0_COUNT
:
1154 case HV_X64_MSR_STIMER1_COUNT
:
1155 case HV_X64_MSR_STIMER2_COUNT
:
1156 case HV_X64_MSR_STIMER3_COUNT
: {
1157 int timer_index
= (msr
- HV_X64_MSR_STIMER0_COUNT
)/2;
1159 return stimer_get_count(vcpu_to_stimer(vcpu
, timer_index
),
1163 vcpu_unimpl(vcpu
, "Hyper-V unhandled rdmsr: 0x%x\n", msr
);
1170 int kvm_hv_set_msr_common(struct kvm_vcpu
*vcpu
, u32 msr
, u64 data
, bool host
)
1172 if (kvm_hv_msr_partition_wide(msr
)) {
1175 mutex_lock(&vcpu
->kvm
->arch
.hyperv
.hv_lock
);
1176 r
= kvm_hv_set_msr_pw(vcpu
, msr
, data
, host
);
1177 mutex_unlock(&vcpu
->kvm
->arch
.hyperv
.hv_lock
);
1180 return kvm_hv_set_msr(vcpu
, msr
, data
, host
);
1183 int kvm_hv_get_msr_common(struct kvm_vcpu
*vcpu
, u32 msr
, u64
*pdata
)
1185 if (kvm_hv_msr_partition_wide(msr
)) {
1188 mutex_lock(&vcpu
->kvm
->arch
.hyperv
.hv_lock
);
1189 r
= kvm_hv_get_msr_pw(vcpu
, msr
, pdata
);
1190 mutex_unlock(&vcpu
->kvm
->arch
.hyperv
.hv_lock
);
1193 return kvm_hv_get_msr(vcpu
, msr
, pdata
);
1196 bool kvm_hv_hypercall_enabled(struct kvm
*kvm
)
1198 return READ_ONCE(kvm
->arch
.hyperv
.hv_hypercall
) & HV_X64_MSR_HYPERCALL_ENABLE
;
1201 static void kvm_hv_hypercall_set_result(struct kvm_vcpu
*vcpu
, u64 result
)
1205 longmode
= is_64_bit_mode(vcpu
);
1207 kvm_register_write(vcpu
, VCPU_REGS_RAX
, result
);
1209 kvm_register_write(vcpu
, VCPU_REGS_RDX
, result
>> 32);
1210 kvm_register_write(vcpu
, VCPU_REGS_RAX
, result
& 0xffffffff);
1214 static int kvm_hv_hypercall_complete_userspace(struct kvm_vcpu
*vcpu
)
1216 struct kvm_run
*run
= vcpu
->run
;
1218 kvm_hv_hypercall_set_result(vcpu
, run
->hyperv
.u
.hcall
.result
);
1222 int kvm_hv_hypercall(struct kvm_vcpu
*vcpu
)
1224 u64 param
, ingpa
, outgpa
, ret
;
1225 uint16_t code
, rep_idx
, rep_cnt
, res
= HV_STATUS_SUCCESS
, rep_done
= 0;
1226 bool fast
, longmode
;
1229 * hypercall generates UD from non zero cpl and real mode
1232 if (kvm_x86_ops
->get_cpl(vcpu
) != 0 || !is_protmode(vcpu
)) {
1233 kvm_queue_exception(vcpu
, UD_VECTOR
);
1237 longmode
= is_64_bit_mode(vcpu
);
1240 param
= ((u64
)kvm_register_read(vcpu
, VCPU_REGS_RDX
) << 32) |
1241 (kvm_register_read(vcpu
, VCPU_REGS_RAX
) & 0xffffffff);
1242 ingpa
= ((u64
)kvm_register_read(vcpu
, VCPU_REGS_RBX
) << 32) |
1243 (kvm_register_read(vcpu
, VCPU_REGS_RCX
) & 0xffffffff);
1244 outgpa
= ((u64
)kvm_register_read(vcpu
, VCPU_REGS_RDI
) << 32) |
1245 (kvm_register_read(vcpu
, VCPU_REGS_RSI
) & 0xffffffff);
1247 #ifdef CONFIG_X86_64
1249 param
= kvm_register_read(vcpu
, VCPU_REGS_RCX
);
1250 ingpa
= kvm_register_read(vcpu
, VCPU_REGS_RDX
);
1251 outgpa
= kvm_register_read(vcpu
, VCPU_REGS_R8
);
1255 code
= param
& 0xffff;
1256 fast
= (param
>> 16) & 0x1;
1257 rep_cnt
= (param
>> 32) & 0xfff;
1258 rep_idx
= (param
>> 48) & 0xfff;
1260 trace_kvm_hv_hypercall(code
, fast
, rep_cnt
, rep_idx
, ingpa
, outgpa
);
1262 /* Hypercall continuation is not supported yet */
1263 if (rep_cnt
|| rep_idx
) {
1264 res
= HV_STATUS_INVALID_HYPERCALL_CODE
;
1269 case HVCALL_NOTIFY_LONG_SPIN_WAIT
:
1270 kvm_vcpu_on_spin(vcpu
);
1272 case HVCALL_POST_MESSAGE
:
1273 case HVCALL_SIGNAL_EVENT
:
1274 /* don't bother userspace if it has no way to handle it */
1275 if (!vcpu_to_synic(vcpu
)->active
) {
1276 res
= HV_STATUS_INVALID_HYPERCALL_CODE
;
1279 vcpu
->run
->exit_reason
= KVM_EXIT_HYPERV
;
1280 vcpu
->run
->hyperv
.type
= KVM_EXIT_HYPERV_HCALL
;
1281 vcpu
->run
->hyperv
.u
.hcall
.input
= param
;
1282 vcpu
->run
->hyperv
.u
.hcall
.params
[0] = ingpa
;
1283 vcpu
->run
->hyperv
.u
.hcall
.params
[1] = outgpa
;
1284 vcpu
->arch
.complete_userspace_io
=
1285 kvm_hv_hypercall_complete_userspace
;
1288 res
= HV_STATUS_INVALID_HYPERCALL_CODE
;
1293 ret
= res
| (((u64
)rep_done
& 0xfff) << 32);
1294 kvm_hv_hypercall_set_result(vcpu
, ret
);