2 * kvm eventfd support - use eventfd objects to signal various KVM events
4 * Copyright 2009 Novell. All Rights Reserved.
5 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
8 * Gregory Haskins <ghaskins@novell.com>
10 * This file is free software; you can redistribute it and/or modify
11 * it under the terms of version 2 of the GNU General Public License
12 * as published by the Free Software Foundation.
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software Foundation,
21 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
24 #include <linux/kvm_host.h>
25 #include <linux/kvm.h>
26 #include <linux/workqueue.h>
27 #include <linux/syscalls.h>
28 #include <linux/wait.h>
29 #include <linux/poll.h>
30 #include <linux/file.h>
31 #include <linux/list.h>
32 #include <linux/eventfd.h>
33 #include <linux/kernel.h>
34 #include <linux/srcu.h>
35 #include <linux/slab.h>
36 #include <linux/seqlock.h>
37 #include <trace/events/kvm.h>
41 #ifdef CONFIG_HAVE_KVM_IRQFD
43 * --------------------------------------------------------------------
44 * irqfd: Allows an fd to be used to inject an interrupt to the guest
46 * Credit goes to Avi Kivity for the original idea.
47 * --------------------------------------------------------------------
51 * Resampling irqfds are a special variety of irqfds used to emulate
52 * level triggered interrupts. The interrupt is asserted on eventfd
53 * trigger. On acknowledgement through the irq ack notifier, the
54 * interrupt is de-asserted and userspace is notified through the
55 * resamplefd. All resamplers on the same gsi are de-asserted
56 * together, so we don't need to track the state of each individual
57 * user. We can also therefore share the same irq source ID.
59 struct _irqfd_resampler
{
62 * List of resampling struct _irqfd objects sharing this gsi.
63 * RCU list modified under kvm->irqfds.resampler_lock
65 struct list_head list
;
66 struct kvm_irq_ack_notifier notifier
;
68 * Entry in list of kvm->irqfd.resampler_list. Use for sharing
69 * resamplers among irqfds on the same gsi.
70 * Accessed and modified under kvm->irqfds.resampler_lock
72 struct list_head link
;
76 /* Used for MSI fast-path */
79 /* Update side is protected by irqfds.lock */
80 struct kvm_kernel_irq_routing_entry irq_entry
;
81 seqcount_t irq_entry_sc
;
82 /* Used for level IRQ fast-path */
84 struct work_struct inject
;
85 /* The resampler used by this irqfd (resampler-only) */
86 struct _irqfd_resampler
*resampler
;
87 /* Eventfd notified on resample (resampler-only) */
88 struct eventfd_ctx
*resamplefd
;
89 /* Entry in list of irqfds for a resampler (resampler-only) */
90 struct list_head resampler_link
;
91 /* Used for setup/shutdown */
92 struct eventfd_ctx
*eventfd
;
93 struct list_head list
;
95 struct work_struct shutdown
;
98 static struct workqueue_struct
*irqfd_cleanup_wq
;
101 irqfd_inject(struct work_struct
*work
)
103 struct _irqfd
*irqfd
= container_of(work
, struct _irqfd
, inject
);
104 struct kvm
*kvm
= irqfd
->kvm
;
106 if (!irqfd
->resampler
) {
107 kvm_set_irq(kvm
, KVM_USERSPACE_IRQ_SOURCE_ID
, irqfd
->gsi
, 1,
109 kvm_set_irq(kvm
, KVM_USERSPACE_IRQ_SOURCE_ID
, irqfd
->gsi
, 0,
112 kvm_set_irq(kvm
, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID
,
113 irqfd
->gsi
, 1, false);
117 * Since resampler irqfds share an IRQ source ID, we de-assert once
118 * then notify all of the resampler irqfds using this GSI. We can't
119 * do multiple de-asserts or we risk racing with incoming re-asserts.
122 irqfd_resampler_ack(struct kvm_irq_ack_notifier
*kian
)
124 struct _irqfd_resampler
*resampler
;
126 struct _irqfd
*irqfd
;
129 resampler
= container_of(kian
, struct _irqfd_resampler
, notifier
);
130 kvm
= resampler
->kvm
;
132 kvm_set_irq(kvm
, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID
,
133 resampler
->notifier
.gsi
, 0, false);
135 idx
= srcu_read_lock(&kvm
->irq_srcu
);
137 list_for_each_entry_rcu(irqfd
, &resampler
->list
, resampler_link
)
138 eventfd_signal(irqfd
->resamplefd
, 1);
140 srcu_read_unlock(&kvm
->irq_srcu
, idx
);
144 irqfd_resampler_shutdown(struct _irqfd
*irqfd
)
146 struct _irqfd_resampler
*resampler
= irqfd
->resampler
;
147 struct kvm
*kvm
= resampler
->kvm
;
149 mutex_lock(&kvm
->irqfds
.resampler_lock
);
151 list_del_rcu(&irqfd
->resampler_link
);
152 synchronize_srcu(&kvm
->irq_srcu
);
154 if (list_empty(&resampler
->list
)) {
155 list_del(&resampler
->link
);
156 kvm_unregister_irq_ack_notifier(kvm
, &resampler
->notifier
);
157 kvm_set_irq(kvm
, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID
,
158 resampler
->notifier
.gsi
, 0, false);
162 mutex_unlock(&kvm
->irqfds
.resampler_lock
);
166 * Race-free decouple logic (ordering is critical)
169 irqfd_shutdown(struct work_struct
*work
)
171 struct _irqfd
*irqfd
= container_of(work
, struct _irqfd
, shutdown
);
175 * Synchronize with the wait-queue and unhook ourselves to prevent
178 eventfd_ctx_remove_wait_queue(irqfd
->eventfd
, &irqfd
->wait
, &cnt
);
181 * We know no new events will be scheduled at this point, so block
182 * until all previously outstanding events have completed
184 flush_work(&irqfd
->inject
);
186 if (irqfd
->resampler
) {
187 irqfd_resampler_shutdown(irqfd
);
188 eventfd_ctx_put(irqfd
->resamplefd
);
192 * It is now safe to release the object's resources
194 eventfd_ctx_put(irqfd
->eventfd
);
199 /* assumes kvm->irqfds.lock is held */
201 irqfd_is_active(struct _irqfd
*irqfd
)
203 return list_empty(&irqfd
->list
) ? false : true;
207 * Mark the irqfd as inactive and schedule it for removal
209 * assumes kvm->irqfds.lock is held
212 irqfd_deactivate(struct _irqfd
*irqfd
)
214 BUG_ON(!irqfd_is_active(irqfd
));
216 list_del_init(&irqfd
->list
);
218 queue_work(irqfd_cleanup_wq
, &irqfd
->shutdown
);
222 * Called with wqh->lock held and interrupts disabled
225 irqfd_wakeup(wait_queue_t
*wait
, unsigned mode
, int sync
, void *key
)
227 struct _irqfd
*irqfd
= container_of(wait
, struct _irqfd
, wait
);
228 unsigned long flags
= (unsigned long)key
;
229 struct kvm_kernel_irq_routing_entry irq
;
230 struct kvm
*kvm
= irqfd
->kvm
;
234 if (flags
& POLLIN
) {
235 idx
= srcu_read_lock(&kvm
->irq_srcu
);
237 seq
= read_seqcount_begin(&irqfd
->irq_entry_sc
);
238 irq
= irqfd
->irq_entry
;
239 } while (read_seqcount_retry(&irqfd
->irq_entry_sc
, seq
));
240 /* An event has been signaled, inject an interrupt */
241 if (irq
.type
== KVM_IRQ_ROUTING_MSI
)
242 kvm_set_msi(&irq
, kvm
, KVM_USERSPACE_IRQ_SOURCE_ID
, 1,
245 schedule_work(&irqfd
->inject
);
246 srcu_read_unlock(&kvm
->irq_srcu
, idx
);
249 if (flags
& POLLHUP
) {
250 /* The eventfd is closing, detach from KVM */
253 spin_lock_irqsave(&kvm
->irqfds
.lock
, flags
);
256 * We must check if someone deactivated the irqfd before
257 * we could acquire the irqfds.lock since the item is
258 * deactivated from the KVM side before it is unhooked from
259 * the wait-queue. If it is already deactivated, we can
260 * simply return knowing the other side will cleanup for us.
261 * We cannot race against the irqfd going away since the
262 * other side is required to acquire wqh->lock, which we hold
264 if (irqfd_is_active(irqfd
))
265 irqfd_deactivate(irqfd
);
267 spin_unlock_irqrestore(&kvm
->irqfds
.lock
, flags
);
274 irqfd_ptable_queue_proc(struct file
*file
, wait_queue_head_t
*wqh
,
277 struct _irqfd
*irqfd
= container_of(pt
, struct _irqfd
, pt
);
278 add_wait_queue(wqh
, &irqfd
->wait
);
281 /* Must be called under irqfds.lock */
282 static void irqfd_update(struct kvm
*kvm
, struct _irqfd
*irqfd
)
284 struct kvm_kernel_irq_routing_entry
*e
;
285 struct kvm_kernel_irq_routing_entry entries
[KVM_NR_IRQCHIPS
];
288 n_entries
= kvm_irq_map_gsi(kvm
, entries
, irqfd
->gsi
);
290 write_seqcount_begin(&irqfd
->irq_entry_sc
);
292 irqfd
->irq_entry
.type
= 0;
295 for (i
= 0; i
< n_entries
; ++i
, ++e
) {
296 /* Only fast-path MSI. */
297 if (e
->type
== KVM_IRQ_ROUTING_MSI
)
298 irqfd
->irq_entry
= *e
;
301 write_seqcount_end(&irqfd
->irq_entry_sc
);
305 kvm_irqfd_assign(struct kvm
*kvm
, struct kvm_irqfd
*args
)
307 struct _irqfd
*irqfd
, *tmp
;
309 struct eventfd_ctx
*eventfd
= NULL
, *resamplefd
= NULL
;
314 irqfd
= kzalloc(sizeof(*irqfd
), GFP_KERNEL
);
319 irqfd
->gsi
= args
->gsi
;
320 INIT_LIST_HEAD(&irqfd
->list
);
321 INIT_WORK(&irqfd
->inject
, irqfd_inject
);
322 INIT_WORK(&irqfd
->shutdown
, irqfd_shutdown
);
323 seqcount_init(&irqfd
->irq_entry_sc
);
331 eventfd
= eventfd_ctx_fileget(f
.file
);
332 if (IS_ERR(eventfd
)) {
333 ret
= PTR_ERR(eventfd
);
337 irqfd
->eventfd
= eventfd
;
339 if (args
->flags
& KVM_IRQFD_FLAG_RESAMPLE
) {
340 struct _irqfd_resampler
*resampler
;
342 resamplefd
= eventfd_ctx_fdget(args
->resamplefd
);
343 if (IS_ERR(resamplefd
)) {
344 ret
= PTR_ERR(resamplefd
);
348 irqfd
->resamplefd
= resamplefd
;
349 INIT_LIST_HEAD(&irqfd
->resampler_link
);
351 mutex_lock(&kvm
->irqfds
.resampler_lock
);
353 list_for_each_entry(resampler
,
354 &kvm
->irqfds
.resampler_list
, link
) {
355 if (resampler
->notifier
.gsi
== irqfd
->gsi
) {
356 irqfd
->resampler
= resampler
;
361 if (!irqfd
->resampler
) {
362 resampler
= kzalloc(sizeof(*resampler
), GFP_KERNEL
);
365 mutex_unlock(&kvm
->irqfds
.resampler_lock
);
369 resampler
->kvm
= kvm
;
370 INIT_LIST_HEAD(&resampler
->list
);
371 resampler
->notifier
.gsi
= irqfd
->gsi
;
372 resampler
->notifier
.irq_acked
= irqfd_resampler_ack
;
373 INIT_LIST_HEAD(&resampler
->link
);
375 list_add(&resampler
->link
, &kvm
->irqfds
.resampler_list
);
376 kvm_register_irq_ack_notifier(kvm
,
377 &resampler
->notifier
);
378 irqfd
->resampler
= resampler
;
381 list_add_rcu(&irqfd
->resampler_link
, &irqfd
->resampler
->list
);
382 synchronize_srcu(&kvm
->irq_srcu
);
384 mutex_unlock(&kvm
->irqfds
.resampler_lock
);
388 * Install our own custom wake-up handling so we are notified via
389 * a callback whenever someone signals the underlying eventfd
391 init_waitqueue_func_entry(&irqfd
->wait
, irqfd_wakeup
);
392 init_poll_funcptr(&irqfd
->pt
, irqfd_ptable_queue_proc
);
394 spin_lock_irq(&kvm
->irqfds
.lock
);
397 list_for_each_entry(tmp
, &kvm
->irqfds
.items
, list
) {
398 if (irqfd
->eventfd
!= tmp
->eventfd
)
400 /* This fd is used for another irq already. */
402 spin_unlock_irq(&kvm
->irqfds
.lock
);
406 idx
= srcu_read_lock(&kvm
->irq_srcu
);
407 irqfd_update(kvm
, irqfd
);
408 srcu_read_unlock(&kvm
->irq_srcu
, idx
);
410 list_add_tail(&irqfd
->list
, &kvm
->irqfds
.items
);
412 spin_unlock_irq(&kvm
->irqfds
.lock
);
415 * Check if there was an event already pending on the eventfd
416 * before we registered, and trigger it as if we didn't miss it.
418 events
= f
.file
->f_op
->poll(f
.file
, &irqfd
->pt
);
421 schedule_work(&irqfd
->inject
);
424 * do not drop the file until the irqfd is fully initialized, otherwise
425 * we might race against the POLLHUP
432 if (irqfd
->resampler
)
433 irqfd_resampler_shutdown(irqfd
);
435 if (resamplefd
&& !IS_ERR(resamplefd
))
436 eventfd_ctx_put(resamplefd
);
438 if (eventfd
&& !IS_ERR(eventfd
))
439 eventfd_ctx_put(eventfd
);
448 bool kvm_irq_has_notifier(struct kvm
*kvm
, unsigned irqchip
, unsigned pin
)
450 struct kvm_irq_ack_notifier
*kian
;
453 idx
= srcu_read_lock(&kvm
->irq_srcu
);
454 gsi
= kvm_irq_map_chip_pin(kvm
, irqchip
, pin
);
456 hlist_for_each_entry_rcu(kian
, &kvm
->irq_ack_notifier_list
,
458 if (kian
->gsi
== gsi
) {
459 srcu_read_unlock(&kvm
->irq_srcu
, idx
);
463 srcu_read_unlock(&kvm
->irq_srcu
, idx
);
467 EXPORT_SYMBOL_GPL(kvm_irq_has_notifier
);
469 void kvm_notify_acked_irq(struct kvm
*kvm
, unsigned irqchip
, unsigned pin
)
471 struct kvm_irq_ack_notifier
*kian
;
474 trace_kvm_ack_irq(irqchip
, pin
);
476 idx
= srcu_read_lock(&kvm
->irq_srcu
);
477 gsi
= kvm_irq_map_chip_pin(kvm
, irqchip
, pin
);
479 hlist_for_each_entry_rcu(kian
, &kvm
->irq_ack_notifier_list
,
481 if (kian
->gsi
== gsi
)
482 kian
->irq_acked(kian
);
483 srcu_read_unlock(&kvm
->irq_srcu
, idx
);
486 void kvm_register_irq_ack_notifier(struct kvm
*kvm
,
487 struct kvm_irq_ack_notifier
*kian
)
489 mutex_lock(&kvm
->irq_lock
);
490 hlist_add_head_rcu(&kian
->link
, &kvm
->irq_ack_notifier_list
);
491 mutex_unlock(&kvm
->irq_lock
);
492 kvm_vcpu_request_scan_ioapic(kvm
);
495 void kvm_unregister_irq_ack_notifier(struct kvm
*kvm
,
496 struct kvm_irq_ack_notifier
*kian
)
498 mutex_lock(&kvm
->irq_lock
);
499 hlist_del_init_rcu(&kian
->link
);
500 mutex_unlock(&kvm
->irq_lock
);
501 synchronize_srcu(&kvm
->irq_srcu
);
502 kvm_vcpu_request_scan_ioapic(kvm
);
507 kvm_eventfd_init(struct kvm
*kvm
)
509 #ifdef CONFIG_HAVE_KVM_IRQFD
510 spin_lock_init(&kvm
->irqfds
.lock
);
511 INIT_LIST_HEAD(&kvm
->irqfds
.items
);
512 INIT_LIST_HEAD(&kvm
->irqfds
.resampler_list
);
513 mutex_init(&kvm
->irqfds
.resampler_lock
);
515 INIT_LIST_HEAD(&kvm
->ioeventfds
);
518 #ifdef CONFIG_HAVE_KVM_IRQFD
520 * shutdown any irqfd's that match fd+gsi
523 kvm_irqfd_deassign(struct kvm
*kvm
, struct kvm_irqfd
*args
)
525 struct _irqfd
*irqfd
, *tmp
;
526 struct eventfd_ctx
*eventfd
;
528 eventfd
= eventfd_ctx_fdget(args
->fd
);
530 return PTR_ERR(eventfd
);
532 spin_lock_irq(&kvm
->irqfds
.lock
);
534 list_for_each_entry_safe(irqfd
, tmp
, &kvm
->irqfds
.items
, list
) {
535 if (irqfd
->eventfd
== eventfd
&& irqfd
->gsi
== args
->gsi
) {
537 * This clearing of irq_entry.type is needed for when
538 * another thread calls kvm_irq_routing_update before
539 * we flush workqueue below (we synchronize with
540 * kvm_irq_routing_update using irqfds.lock).
542 write_seqcount_begin(&irqfd
->irq_entry_sc
);
543 irqfd
->irq_entry
.type
= 0;
544 write_seqcount_end(&irqfd
->irq_entry_sc
);
545 irqfd_deactivate(irqfd
);
549 spin_unlock_irq(&kvm
->irqfds
.lock
);
550 eventfd_ctx_put(eventfd
);
553 * Block until we know all outstanding shutdown jobs have completed
554 * so that we guarantee there will not be any more interrupts on this
555 * gsi once this deassign function returns.
557 flush_workqueue(irqfd_cleanup_wq
);
563 kvm_irqfd(struct kvm
*kvm
, struct kvm_irqfd
*args
)
565 if (args
->flags
& ~(KVM_IRQFD_FLAG_DEASSIGN
| KVM_IRQFD_FLAG_RESAMPLE
))
568 if (args
->flags
& KVM_IRQFD_FLAG_DEASSIGN
)
569 return kvm_irqfd_deassign(kvm
, args
);
571 return kvm_irqfd_assign(kvm
, args
);
575 * This function is called as the kvm VM fd is being released. Shutdown all
576 * irqfds that still remain open
579 kvm_irqfd_release(struct kvm
*kvm
)
581 struct _irqfd
*irqfd
, *tmp
;
583 spin_lock_irq(&kvm
->irqfds
.lock
);
585 list_for_each_entry_safe(irqfd
, tmp
, &kvm
->irqfds
.items
, list
)
586 irqfd_deactivate(irqfd
);
588 spin_unlock_irq(&kvm
->irqfds
.lock
);
591 * Block until we know all outstanding shutdown jobs have completed
592 * since we do not take a kvm* reference.
594 flush_workqueue(irqfd_cleanup_wq
);
599 * Take note of a change in irq routing.
600 * Caller must invoke synchronize_srcu(&kvm->irq_srcu) afterwards.
602 void kvm_irq_routing_update(struct kvm
*kvm
)
604 struct _irqfd
*irqfd
;
606 spin_lock_irq(&kvm
->irqfds
.lock
);
608 list_for_each_entry(irqfd
, &kvm
->irqfds
.items
, list
)
609 irqfd_update(kvm
, irqfd
);
611 spin_unlock_irq(&kvm
->irqfds
.lock
);
615 * create a host-wide workqueue for issuing deferred shutdown requests
616 * aggregated from all vm* instances. We need our own isolated single-thread
617 * queue to prevent deadlock against flushing the normal work-queue.
619 int kvm_irqfd_init(void)
621 irqfd_cleanup_wq
= create_singlethread_workqueue("kvm-irqfd-cleanup");
622 if (!irqfd_cleanup_wq
)
628 void kvm_irqfd_exit(void)
630 destroy_workqueue(irqfd_cleanup_wq
);
635 * --------------------------------------------------------------------
636 * ioeventfd: translate a PIO/MMIO memory write to an eventfd signal.
638 * userspace can register a PIO/MMIO address with an eventfd for receiving
639 * notification when the memory has been touched.
640 * --------------------------------------------------------------------
644 struct list_head list
;
647 struct eventfd_ctx
*eventfd
;
649 struct kvm_io_device dev
;
654 static inline struct _ioeventfd
*
655 to_ioeventfd(struct kvm_io_device
*dev
)
657 return container_of(dev
, struct _ioeventfd
, dev
);
661 ioeventfd_release(struct _ioeventfd
*p
)
663 eventfd_ctx_put(p
->eventfd
);
669 ioeventfd_in_range(struct _ioeventfd
*p
, gpa_t addr
, int len
, const void *val
)
674 /* address must be precise for a hit */
678 /* length = 0 means only look at the address, so always a hit */
681 if (len
!= p
->length
)
682 /* address-range must be precise for a hit */
686 /* all else equal, wildcard is always a hit */
689 /* otherwise, we have to actually compare the data */
691 BUG_ON(!IS_ALIGNED((unsigned long)val
, len
));
710 return _val
== p
->datamatch
? true : false;
713 /* MMIO/PIO writes trigger an event if the addr/val match */
715 ioeventfd_write(struct kvm_io_device
*this, gpa_t addr
, int len
,
718 struct _ioeventfd
*p
= to_ioeventfd(this);
720 if (!ioeventfd_in_range(p
, addr
, len
, val
))
723 eventfd_signal(p
->eventfd
, 1);
728 * This function is called as KVM is completely shutting down. We do not
729 * need to worry about locking just nuke anything we have as quickly as possible
732 ioeventfd_destructor(struct kvm_io_device
*this)
734 struct _ioeventfd
*p
= to_ioeventfd(this);
736 ioeventfd_release(p
);
739 static const struct kvm_io_device_ops ioeventfd_ops
= {
740 .write
= ioeventfd_write
,
741 .destructor
= ioeventfd_destructor
,
744 /* assumes kvm->slots_lock held */
746 ioeventfd_check_collision(struct kvm
*kvm
, struct _ioeventfd
*p
)
748 struct _ioeventfd
*_p
;
750 list_for_each_entry(_p
, &kvm
->ioeventfds
, list
)
751 if (_p
->bus_idx
== p
->bus_idx
&&
752 _p
->addr
== p
->addr
&&
753 (!_p
->length
|| !p
->length
||
754 (_p
->length
== p
->length
&&
755 (_p
->wildcard
|| p
->wildcard
||
756 _p
->datamatch
== p
->datamatch
))))
762 static enum kvm_bus
ioeventfd_bus_from_flags(__u32 flags
)
764 if (flags
& KVM_IOEVENTFD_FLAG_PIO
)
766 if (flags
& KVM_IOEVENTFD_FLAG_VIRTIO_CCW_NOTIFY
)
767 return KVM_VIRTIO_CCW_NOTIFY_BUS
;
772 kvm_assign_ioeventfd(struct kvm
*kvm
, struct kvm_ioeventfd
*args
)
774 enum kvm_bus bus_idx
;
775 struct _ioeventfd
*p
;
776 struct eventfd_ctx
*eventfd
;
779 bus_idx
= ioeventfd_bus_from_flags(args
->flags
);
780 /* must be natural-word sized, or 0 to ignore length */
792 /* check for range overflow */
793 if (args
->addr
+ args
->len
< args
->addr
)
796 /* check for extra flags that we don't understand */
797 if (args
->flags
& ~KVM_IOEVENTFD_VALID_FLAG_MASK
)
800 /* ioeventfd with no length can't be combined with DATAMATCH */
802 args
->flags
& (KVM_IOEVENTFD_FLAG_PIO
|
803 KVM_IOEVENTFD_FLAG_DATAMATCH
))
806 eventfd
= eventfd_ctx_fdget(args
->fd
);
808 return PTR_ERR(eventfd
);
810 p
= kzalloc(sizeof(*p
), GFP_KERNEL
);
816 INIT_LIST_HEAD(&p
->list
);
817 p
->addr
= args
->addr
;
818 p
->bus_idx
= bus_idx
;
819 p
->length
= args
->len
;
820 p
->eventfd
= eventfd
;
822 /* The datamatch feature is optional, otherwise this is a wildcard */
823 if (args
->flags
& KVM_IOEVENTFD_FLAG_DATAMATCH
)
824 p
->datamatch
= args
->datamatch
;
828 mutex_lock(&kvm
->slots_lock
);
830 /* Verify that there isn't a match already */
831 if (ioeventfd_check_collision(kvm
, p
)) {
836 kvm_iodevice_init(&p
->dev
, &ioeventfd_ops
);
838 ret
= kvm_io_bus_register_dev(kvm
, bus_idx
, p
->addr
, p
->length
,
843 /* When length is ignored, MMIO is also put on a separate bus, for
846 if (!args
->len
&& !(args
->flags
& KVM_IOEVENTFD_FLAG_PIO
)) {
847 ret
= kvm_io_bus_register_dev(kvm
, KVM_FAST_MMIO_BUS
,
848 p
->addr
, 0, &p
->dev
);
853 kvm
->buses
[bus_idx
]->ioeventfd_count
++;
854 list_add_tail(&p
->list
, &kvm
->ioeventfds
);
856 mutex_unlock(&kvm
->slots_lock
);
861 kvm_io_bus_unregister_dev(kvm
, bus_idx
, &p
->dev
);
863 mutex_unlock(&kvm
->slots_lock
);
867 eventfd_ctx_put(eventfd
);
873 kvm_deassign_ioeventfd(struct kvm
*kvm
, struct kvm_ioeventfd
*args
)
875 enum kvm_bus bus_idx
;
876 struct _ioeventfd
*p
, *tmp
;
877 struct eventfd_ctx
*eventfd
;
880 bus_idx
= ioeventfd_bus_from_flags(args
->flags
);
881 eventfd
= eventfd_ctx_fdget(args
->fd
);
883 return PTR_ERR(eventfd
);
885 mutex_lock(&kvm
->slots_lock
);
887 list_for_each_entry_safe(p
, tmp
, &kvm
->ioeventfds
, list
) {
888 bool wildcard
= !(args
->flags
& KVM_IOEVENTFD_FLAG_DATAMATCH
);
890 if (p
->bus_idx
!= bus_idx
||
891 p
->eventfd
!= eventfd
||
892 p
->addr
!= args
->addr
||
893 p
->length
!= args
->len
||
894 p
->wildcard
!= wildcard
)
897 if (!p
->wildcard
&& p
->datamatch
!= args
->datamatch
)
900 kvm_io_bus_unregister_dev(kvm
, bus_idx
, &p
->dev
);
902 kvm_io_bus_unregister_dev(kvm
, KVM_FAST_MMIO_BUS
,
905 kvm
->buses
[bus_idx
]->ioeventfd_count
--;
906 ioeventfd_release(p
);
911 mutex_unlock(&kvm
->slots_lock
);
913 eventfd_ctx_put(eventfd
);
919 kvm_ioeventfd(struct kvm
*kvm
, struct kvm_ioeventfd
*args
)
921 if (args
->flags
& KVM_IOEVENTFD_FLAG_DEASSIGN
)
922 return kvm_deassign_ioeventfd(kvm
, args
);
924 return kvm_assign_ioeventfd(kvm
, args
);