KVM: halt_polling: provide a way to qualify wakeups during poll

Some wakeups should not be considered a sucessful poll. For example on
s390 I/O interrupts are usually floating, which means that _ALL_ CPUs
would be considered runnable - letting all vCPUs poll all the time for
transactional like workload, even if one vCPU would be enough.
This can result in huge CPU usage for large guests.
This patch lets architectures provide a way to qualify wakeups if they
should be considered a good/bad wakeups in regard to polls.

For s390 the implementation will fence of halt polling for anything but
known good, single vCPU events. The s390 implementation for floating
interrupts does a wakeup for one vCPU, but the interrupt will be delivered
by whatever CPU checks first for a pending interrupt. We prefer the
woken up CPU by marking the poll of this CPU as "good" poll.
This code will also mark several other wakeup reasons like IPI or
expired timers as "good". This will of course also mark some events as
not sucessful. As  KVM on z runs always as a 2nd level hypervisor,
we prefer to not poll, unless we are really sure, though.

This patch successfully limits the CPU usage for cases like uperf 1byte
transactional ping pong workload or wakeup heavy workload like OLTP
while still providing a proper speedup.

This also introduced a new vcpu stat "halt_poll_no_tuning" that marks
wakeups that are considered not good for polling.

Signed-off-by: Christian Borntraeger <borntraeger@de.ibm.com>
Acked-by: Radim Krčmář <rkrcmar@redhat.com> (for an earlier version)
Cc: David Matlack <dmatlack@google.com>
Cc: Wanpeng Li <kernellwp@gmail.com>
[Rename config symbol. - Paolo]
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index 385070180c2587df524fed552ac0f36b71cac26b..4cd8732796ab6ff3cafef1f6b11462d2a25215ea 100644 (file)
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -187,6 +187,7 @@ struct kvm_vm_stat {
 struct kvm_vcpu_stat {
        u32 halt_successful_poll;
        u32 halt_attempted_poll;
+       u32 halt_poll_invalid;
        u32 halt_wakeup;
        u32 hvc_exit_stat;
        u64 wfe_exit_stat;
@@ -282,6 +283,7 @@ static inline void kvm_arch_hardware_unsetup(void) {}
 static inline void kvm_arch_sync_events(struct kvm *kvm) {}
 static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
 static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
+static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {}
 
 static inline void kvm_arm_init_debug(void) {}
 static inline void kvm_arm_setup_debug(struct kvm_vcpu *vcpu) {}

diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index f5c6bd2541ef4d16c6f3b26cee01cc013d50055d..d49399d9890d358ad027a15e0a3e44ee13ba744f 100644 (file)
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -293,6 +293,7 @@ struct kvm_vm_stat {
 struct kvm_vcpu_stat {
        u32 halt_successful_poll;
        u32 halt_attempted_poll;
+       u32 halt_poll_invalid;
        u32 halt_wakeup;
        u32 hvc_exit_stat;
        u64 wfe_exit_stat;
@@ -357,6 +358,7 @@ static inline void kvm_arch_hardware_unsetup(void) {}
 static inline void kvm_arch_sync_events(struct kvm *kvm) {}
 static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
 static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
+static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {}
 
 void kvm_arm_init_debug(void);
 void kvm_arm_setup_debug(struct kvm_vcpu *vcpu);

diff --git a/arch/mips/include/asm/kvm_host.h b/arch/mips/include/asm/kvm_host.h
index 942b8f6bf35b463d46bc5f5abe292f6e1ec7844a..9a37a1044032dc45733607d901ac20c4f8465798 100644 (file)
--- a/arch/mips/include/asm/kvm_host.h
+++ b/arch/mips/include/asm/kvm_host.h
@@ -122,6 +122,7 @@ struct kvm_vcpu_stat {
        u32 flush_dcache_exits;
        u32 halt_successful_poll;
        u32 halt_attempted_poll;
+       u32 halt_poll_invalid;
        u32 halt_wakeup;
 };
 
@@ -812,5 +813,6 @@ static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
 static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
 static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {}
 static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {}
+static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {}
 
 #endif /* __MIPS_KVM_HOST_H__ */

diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c
index 23b20946323899ef20142184c08e8c5dde20784d..dc052fb5c7a2f80cdc20b7bf282b0cb09fc30a2e 100644 (file)
--- a/arch/mips/kvm/mips.c
+++ b/arch/mips/kvm/mips.c
@@ -56,6 +56,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
        { "flush_dcache", VCPU_STAT(flush_dcache_exits), KVM_STAT_VCPU },
        { "halt_successful_poll", VCPU_STAT(halt_successful_poll), KVM_STAT_VCPU },
        { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll), KVM_STAT_VCPU },
+       { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid), KVM_STAT_VCPU },
        { "halt_wakeup",  VCPU_STAT(halt_wakeup),        KVM_STAT_VCPU },
        {NULL}
 };

diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h
index a07645c17818e50dacad5af2fe36c309a3462666..ec35af34a3fb411d6d8f536be997daac9e3b57ce 100644 (file)
--- a/arch/powerpc/include/asm/kvm_host.h
+++ b/arch/powerpc/include/asm/kvm_host.h
@@ -116,6 +116,7 @@ struct kvm_vcpu_stat {
        u32 ext_intr_exits;
        u32 halt_successful_poll;
        u32 halt_attempted_poll;
+       u32 halt_poll_invalid;
        u32 halt_wakeup;
        u32 dbell_exits;
        u32 gdbell_exits;
@@ -727,5 +728,6 @@ static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
 static inline void kvm_arch_exit(void) {}
 static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {}
 static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {}
+static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {}
 
 #endif /* __POWERPC_KVM_HOST_H__ */

diff --git a/arch/powerpc/kvm/book3s.c b/arch/powerpc/kvm/book3s.c
index b34220d2aa42c96380b8f2d8fcbe1de630f5c31e..47018fcbf7d6aa3dd02caeef441dccd52ed476a4 100644 (file)
--- a/arch/powerpc/kvm/book3s.c
+++ b/arch/powerpc/kvm/book3s.c
@@ -54,6 +54,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
        { "queue_intr",  VCPU_STAT(queue_intr) },
        { "halt_successful_poll", VCPU_STAT(halt_successful_poll), },
        { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll), },
+       { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
        { "halt_wakeup", VCPU_STAT(halt_wakeup) },
        { "pf_storage",  VCPU_STAT(pf_storage) },
        { "sp_storage",  VCPU_STAT(sp_storage) },

diff --git a/arch/powerpc/kvm/booke.c b/arch/powerpc/kvm/booke.c
index 4d66f44a165789df702242c622024a269da745a2..4afae695899ad99373c033e53978603879bf078c 100644 (file)
--- a/arch/powerpc/kvm/booke.c
+++ b/arch/powerpc/kvm/booke.c
@@ -64,6 +64,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
        { "ext_intr",   VCPU_STAT(ext_intr_exits) },
        { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
        { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
+       { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
        { "halt_wakeup", VCPU_STAT(halt_wakeup) },
        { "doorbell", VCPU_STAT(dbell_exits) },
        { "guest doorbell", VCPU_STAT(gdbell_exits) },

diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
index 9282ccf1d1366b7340fd213473eab006830713d5..53d794538067bc8f3c6ca1365e1c5a967d8f02e6 100644 (file)
--- a/arch/s390/include/asm/kvm_host.h
+++ b/arch/s390/include/asm/kvm_host.h
@@ -247,6 +247,7 @@ struct kvm_vcpu_stat {
        u32 exit_instruction;
        u32 halt_successful_poll;
        u32 halt_attempted_poll;
+       u32 halt_poll_invalid;
        u32 halt_wakeup;
        u32 instruction_lctl;
        u32 instruction_lctlg;
@@ -696,4 +697,6 @@ static inline void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
 static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {}
 static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {}
 
+void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu);
+
 #endif

diff --git a/arch/s390/kvm/Kconfig b/arch/s390/kvm/Kconfig
index 5ea5af3c7db72479a69710a6606d8dddd6143b7a..b1900239b0ab2cbe27e7b5be3f78d8fdacbfe5dd 100644 (file)
--- a/arch/s390/kvm/Kconfig
+++ b/arch/s390/kvm/Kconfig
@@ -28,6 +28,7 @@ config KVM
        select HAVE_KVM_IRQCHIP
        select HAVE_KVM_IRQFD
        select HAVE_KVM_IRQ_ROUTING
+       select HAVE_KVM_INVALID_WAKEUPS
        select SRCU
        select KVM_VFIO
        ---help---

diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c
index e55040467eb506cfd7fc82c9d485bd015a247fa6..5a80af740d3eefb86155b8cf4680da79d8770170 100644 (file)
--- a/arch/s390/kvm/interrupt.c
+++ b/arch/s390/kvm/interrupt.c
@@ -977,6 +977,11 @@ no_timer:
 
 void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu)
 {
+       /*
+        * We cannot move this into the if, as the CPU might be already
+        * in kvm_vcpu_block without having the waitqueue set (polling)
+        */
+       vcpu->valid_wakeup = true;
        if (swait_active(&vcpu->wq)) {
                /*
                 * The vcpu gave up the cpu voluntarily, mark it as a good

diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index c597201a5ca9ba0cbdbc2dbd67ba0df1437604a4..6d8ec3ac9dd8ec675d4d138a83888fa371138bcb 100644 (file)
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -65,6 +65,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
        { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
        { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
        { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
+       { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
        { "halt_wakeup", VCPU_STAT(halt_wakeup) },
        { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
        { "instruction_lctl", VCPU_STAT(instruction_lctl) },
@@ -2992,6 +2993,11 @@ static inline unsigned long nonhyp_mask(int i)
        return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
 }
 
+void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
+{
+       vcpu->valid_wakeup = false;
+}
+
 static int __init kvm_s390_init(void)
 {
        int i;

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index c66e26280707a9c5d24914a41a6cd22b62d461c3..c99494b4bdf7b17d9cf0e9f0f32c12d49f354680 100644 (file)
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -803,6 +803,7 @@ struct kvm_vcpu_stat {
        u32 halt_exits;
        u32 halt_successful_poll;
        u32 halt_attempted_poll;
+       u32 halt_poll_invalid;
        u32 halt_wakeup;
        u32 request_irq_exits;
        u32 irq_exits;
@@ -1342,5 +1343,6 @@ void kvm_set_msi_irq(struct kvm_kernel_irq_routing_entry *e,
 
 static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {}
 static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {}
+static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {}
 
 #endif /* _ASM_X86_KVM_HOST_H */

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 6c774cdf553c69e600e6b90fe90aed2d77d081c4..bcef92fc41d8082fda2d8f744d1111af8273e6a3 100644 (file)
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -161,6 +161,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
        { "halt_exits", VCPU_STAT(halt_exits) },
        { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
        { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
+       { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
        { "halt_wakeup", VCPU_STAT(halt_wakeup) },
        { "hypercalls", VCPU_STAT(hypercalls) },
        { "request_irq", VCPU_STAT(request_irq_exits) },

diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 92a0229044fb0c8148f4d5c227de5b3856610fba..bbcd921d7cb0155a897e74543975624cdc511f70 100644 (file)
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -229,6 +229,7 @@ struct kvm_vcpu {
        sigset_t sigset;
        struct kvm_vcpu_stat stat;
        unsigned int halt_poll_ns;
+       bool valid_wakeup;
 
 #ifdef CONFIG_HAS_IOMEM
        int mmio_needed;
@@ -1196,4 +1197,18 @@ int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,
                                  uint32_t guest_irq, bool set);
 #endif /* CONFIG_HAVE_KVM_IRQ_BYPASS */
 
+#ifdef CONFIG_HAVE_KVM_INVALID_WAKEUPS
+/* If we wakeup during the poll time, was it a sucessful poll? */
+static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu)
+{
+       return vcpu->valid_wakeup;
+}
+
+#else
+static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu)
+{
+       return true;
+}
+#endif /* CONFIG_HAVE_KVM_INVALID_WAKEUPS */
+
 #endif

diff --git a/include/trace/events/kvm.h b/include/trace/events/kvm.h
index aa69253ecc7dd94df775361ac6c6f5b70c6e0dfc..526fb3d2e43a416fd14f47bc33b8f7e034426145 100644 (file)
--- a/include/trace/events/kvm.h
+++ b/include/trace/events/kvm.h
@@ -38,22 +38,25 @@ TRACE_EVENT(kvm_userspace_exit,
 );
 
 TRACE_EVENT(kvm_vcpu_wakeup,
-           TP_PROTO(__u64 ns, bool waited),
-           TP_ARGS(ns, waited),
+           TP_PROTO(__u64 ns, bool waited, bool valid),
+           TP_ARGS(ns, waited, valid),
 
        TP_STRUCT__entry(
                __field(        __u64,          ns              )
                __field(        bool,           waited          )
+               __field(        bool,           valid           )
        ),
 
        TP_fast_assign(
                __entry->ns             = ns;
                __entry->waited         = waited;
+               __entry->valid          = valid;
        ),
 
-       TP_printk("%s time %lld ns",
+       TP_printk("%s time %lld ns, polling %s",
                  __entry->waited ? "wait" : "poll",
-                 __entry->ns)
+                 __entry->ns,
+                 __entry->valid ? "valid" : "invalid")
 );
 
 #if defined(CONFIG_HAVE_KVM_IRQFD)

diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig
index 7a79b68535836d51796fb424d4ba95c15eed96ad..e5d6108f5e8596aa26780d39754a45c8c88ed8f9 100644 (file)
--- a/virt/kvm/Kconfig
+++ b/virt/kvm/Kconfig
@@ -41,6 +41,9 @@ config KVM_VFIO
 config HAVE_KVM_ARCH_TLB_FLUSH_ALL
        bool
 
+config HAVE_KVM_INVALID_WAKEUPS
+       bool
+
 config KVM_GENERIC_DIRTYLOG_READ_PROTECT
        bool
 

diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index ed3d9bb18a5666b325d94a185d03fe76511cf45e..21f6498d52e30190e20806caa48e685c33e808ce 100644 (file)
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -2028,6 +2028,8 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu)
                         */
                        if (kvm_vcpu_check_block(vcpu) < 0) {
                                ++vcpu->stat.halt_successful_poll;
+                               if (!vcpu_valid_wakeup(vcpu))
+                                       ++vcpu->stat.halt_poll_invalid;
                                goto out;
                        }
                        cur = ktime_get();
@@ -2057,7 +2059,8 @@ out:
                if (block_ns <= vcpu->halt_poll_ns)
                        ;
                /* we had a long block, shrink polling */
-               else if (vcpu->halt_poll_ns && block_ns > halt_poll_ns)
+               else if (!vcpu_valid_wakeup(vcpu) ||
+                       (vcpu->halt_poll_ns && block_ns > halt_poll_ns))
                        shrink_halt_poll_ns(vcpu);
                /* we had a short halt and our poll time is too small */
                else if (vcpu->halt_poll_ns < halt_poll_ns &&
@@ -2066,7 +2069,8 @@ out:
        } else
                vcpu->halt_poll_ns = 0;
 
-       trace_kvm_vcpu_wakeup(block_ns, waited);
+       trace_kvm_vcpu_wakeup(block_ns, waited, vcpu_valid_wakeup(vcpu));
+       kvm_arch_vcpu_block_finish(vcpu);
 }
 EXPORT_SYMBOL_GPL(kvm_vcpu_block);