4 * Copyright (C) 2012 Red Hat, Inc. All rights reserved.
5 * Author: Alex Williamson <alex.williamson@redhat.com>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
11 * Derived from original vfio:
12 * Copyright 2010 Cisco Systems, Inc. All rights reserved.
13 * Author: Tom Lyon, pugs@cisco.com
16 #include <linux/cdev.h>
17 #include <linux/compat.h>
18 #include <linux/device.h>
19 #include <linux/file.h>
20 #include <linux/anon_inodes.h>
22 #include <linux/idr.h>
23 #include <linux/iommu.h>
24 #include <linux/list.h>
25 #include <linux/miscdevice.h>
26 #include <linux/module.h>
27 #include <linux/mutex.h>
28 #include <linux/rwsem.h>
29 #include <linux/sched.h>
30 #include <linux/slab.h>
31 #include <linux/stat.h>
32 #include <linux/string.h>
33 #include <linux/uaccess.h>
34 #include <linux/vfio.h>
35 #include <linux/wait.h>
37 #define DRIVER_VERSION "0.3"
38 #define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
39 #define DRIVER_DESC "VFIO - User Level meta-driver"
43 struct list_head iommu_drivers_list
;
44 struct mutex iommu_drivers_lock
;
45 struct list_head group_list
;
47 struct mutex group_lock
;
48 struct cdev group_cdev
;
50 wait_queue_head_t release_q
;
53 struct vfio_iommu_driver
{
54 const struct vfio_iommu_driver_ops
*ops
;
55 struct list_head vfio_next
;
58 struct vfio_container
{
60 struct list_head group_list
;
61 struct rw_semaphore group_lock
;
62 struct vfio_iommu_driver
*iommu_driver
;
66 struct vfio_unbound_dev
{
68 struct list_head unbound_next
;
74 atomic_t container_users
;
75 struct iommu_group
*iommu_group
;
76 struct vfio_container
*container
;
77 struct list_head device_list
;
78 struct mutex device_lock
;
80 struct notifier_block nb
;
81 struct list_head vfio_next
;
82 struct list_head container_next
;
83 struct list_head unbound_list
;
84 struct mutex unbound_lock
;
91 const struct vfio_device_ops
*ops
;
92 struct vfio_group
*group
;
93 struct list_head group_next
;
98 * IOMMU driver registration
100 int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops
*ops
)
102 struct vfio_iommu_driver
*driver
, *tmp
;
104 driver
= kzalloc(sizeof(*driver
), GFP_KERNEL
);
110 mutex_lock(&vfio
.iommu_drivers_lock
);
112 /* Check for duplicates */
113 list_for_each_entry(tmp
, &vfio
.iommu_drivers_list
, vfio_next
) {
114 if (tmp
->ops
== ops
) {
115 mutex_unlock(&vfio
.iommu_drivers_lock
);
121 list_add(&driver
->vfio_next
, &vfio
.iommu_drivers_list
);
123 mutex_unlock(&vfio
.iommu_drivers_lock
);
127 EXPORT_SYMBOL_GPL(vfio_register_iommu_driver
);
129 void vfio_unregister_iommu_driver(const struct vfio_iommu_driver_ops
*ops
)
131 struct vfio_iommu_driver
*driver
;
133 mutex_lock(&vfio
.iommu_drivers_lock
);
134 list_for_each_entry(driver
, &vfio
.iommu_drivers_list
, vfio_next
) {
135 if (driver
->ops
== ops
) {
136 list_del(&driver
->vfio_next
);
137 mutex_unlock(&vfio
.iommu_drivers_lock
);
142 mutex_unlock(&vfio
.iommu_drivers_lock
);
144 EXPORT_SYMBOL_GPL(vfio_unregister_iommu_driver
);
147 * Group minor allocation/free - both called with vfio.group_lock held
149 static int vfio_alloc_group_minor(struct vfio_group
*group
)
151 return idr_alloc(&vfio
.group_idr
, group
, 0, MINORMASK
+ 1, GFP_KERNEL
);
154 static void vfio_free_group_minor(int minor
)
156 idr_remove(&vfio
.group_idr
, minor
);
159 static int vfio_iommu_group_notifier(struct notifier_block
*nb
,
160 unsigned long action
, void *data
);
161 static void vfio_group_get(struct vfio_group
*group
);
164 * Container objects - containers are created when /dev/vfio/vfio is
165 * opened, but their lifecycle extends until the last user is done, so
166 * it's freed via kref. Must support container/group/device being
167 * closed in any order.
169 static void vfio_container_get(struct vfio_container
*container
)
171 kref_get(&container
->kref
);
174 static void vfio_container_release(struct kref
*kref
)
176 struct vfio_container
*container
;
177 container
= container_of(kref
, struct vfio_container
, kref
);
182 static void vfio_container_put(struct vfio_container
*container
)
184 kref_put(&container
->kref
, vfio_container_release
);
187 static void vfio_group_unlock_and_free(struct vfio_group
*group
)
189 mutex_unlock(&vfio
.group_lock
);
191 * Unregister outside of lock. A spurious callback is harmless now
192 * that the group is no longer in vfio.group_list.
194 iommu_group_unregister_notifier(group
->iommu_group
, &group
->nb
);
199 * Group objects - create, release, get, put, search
201 static struct vfio_group
*vfio_create_group(struct iommu_group
*iommu_group
)
203 struct vfio_group
*group
, *tmp
;
207 group
= kzalloc(sizeof(*group
), GFP_KERNEL
);
209 return ERR_PTR(-ENOMEM
);
211 kref_init(&group
->kref
);
212 INIT_LIST_HEAD(&group
->device_list
);
213 mutex_init(&group
->device_lock
);
214 INIT_LIST_HEAD(&group
->unbound_list
);
215 mutex_init(&group
->unbound_lock
);
216 atomic_set(&group
->container_users
, 0);
217 atomic_set(&group
->opened
, 0);
218 group
->iommu_group
= iommu_group
;
220 group
->nb
.notifier_call
= vfio_iommu_group_notifier
;
223 * blocking notifiers acquire a rwsem around registering and hold
224 * it around callback. Therefore, need to register outside of
225 * vfio.group_lock to avoid A-B/B-A contention. Our callback won't
226 * do anything unless it can find the group in vfio.group_list, so
227 * no harm in registering early.
229 ret
= iommu_group_register_notifier(iommu_group
, &group
->nb
);
235 mutex_lock(&vfio
.group_lock
);
237 /* Did we race creating this group? */
238 list_for_each_entry(tmp
, &vfio
.group_list
, vfio_next
) {
239 if (tmp
->iommu_group
== iommu_group
) {
241 vfio_group_unlock_and_free(group
);
246 minor
= vfio_alloc_group_minor(group
);
248 vfio_group_unlock_and_free(group
);
249 return ERR_PTR(minor
);
252 dev
= device_create(vfio
.class, NULL
,
253 MKDEV(MAJOR(vfio
.group_devt
), minor
),
254 group
, "%d", iommu_group_id(iommu_group
));
256 vfio_free_group_minor(minor
);
257 vfio_group_unlock_and_free(group
);
258 return (struct vfio_group
*)dev
; /* ERR_PTR */
261 group
->minor
= minor
;
264 list_add(&group
->vfio_next
, &vfio
.group_list
);
266 mutex_unlock(&vfio
.group_lock
);
271 /* called with vfio.group_lock held */
272 static void vfio_group_release(struct kref
*kref
)
274 struct vfio_group
*group
= container_of(kref
, struct vfio_group
, kref
);
275 struct vfio_unbound_dev
*unbound
, *tmp
;
276 struct iommu_group
*iommu_group
= group
->iommu_group
;
278 WARN_ON(!list_empty(&group
->device_list
));
280 list_for_each_entry_safe(unbound
, tmp
,
281 &group
->unbound_list
, unbound_next
) {
282 list_del(&unbound
->unbound_next
);
286 device_destroy(vfio
.class, MKDEV(MAJOR(vfio
.group_devt
), group
->minor
));
287 list_del(&group
->vfio_next
);
288 vfio_free_group_minor(group
->minor
);
289 vfio_group_unlock_and_free(group
);
290 iommu_group_put(iommu_group
);
293 static void vfio_group_put(struct vfio_group
*group
)
295 kref_put_mutex(&group
->kref
, vfio_group_release
, &vfio
.group_lock
);
298 /* Assume group_lock or group reference is held */
299 static void vfio_group_get(struct vfio_group
*group
)
301 kref_get(&group
->kref
);
305 * Not really a try as we will sleep for mutex, but we need to make
306 * sure the group pointer is valid under lock and get a reference.
308 static struct vfio_group
*vfio_group_try_get(struct vfio_group
*group
)
310 struct vfio_group
*target
= group
;
312 mutex_lock(&vfio
.group_lock
);
313 list_for_each_entry(group
, &vfio
.group_list
, vfio_next
) {
314 if (group
== target
) {
315 vfio_group_get(group
);
316 mutex_unlock(&vfio
.group_lock
);
320 mutex_unlock(&vfio
.group_lock
);
326 struct vfio_group
*vfio_group_get_from_iommu(struct iommu_group
*iommu_group
)
328 struct vfio_group
*group
;
330 mutex_lock(&vfio
.group_lock
);
331 list_for_each_entry(group
, &vfio
.group_list
, vfio_next
) {
332 if (group
->iommu_group
== iommu_group
) {
333 vfio_group_get(group
);
334 mutex_unlock(&vfio
.group_lock
);
338 mutex_unlock(&vfio
.group_lock
);
343 static struct vfio_group
*vfio_group_get_from_minor(int minor
)
345 struct vfio_group
*group
;
347 mutex_lock(&vfio
.group_lock
);
348 group
= idr_find(&vfio
.group_idr
, minor
);
350 mutex_unlock(&vfio
.group_lock
);
353 vfio_group_get(group
);
354 mutex_unlock(&vfio
.group_lock
);
360 * Device objects - create, release, get, put, search
363 struct vfio_device
*vfio_group_create_device(struct vfio_group
*group
,
365 const struct vfio_device_ops
*ops
,
368 struct vfio_device
*device
;
370 device
= kzalloc(sizeof(*device
), GFP_KERNEL
);
372 return ERR_PTR(-ENOMEM
);
374 kref_init(&device
->kref
);
376 device
->group
= group
;
378 device
->device_data
= device_data
;
379 dev_set_drvdata(dev
, device
);
381 /* No need to get group_lock, caller has group reference */
382 vfio_group_get(group
);
384 mutex_lock(&group
->device_lock
);
385 list_add(&device
->group_next
, &group
->device_list
);
386 mutex_unlock(&group
->device_lock
);
391 static void vfio_device_release(struct kref
*kref
)
393 struct vfio_device
*device
= container_of(kref
,
394 struct vfio_device
, kref
);
395 struct vfio_group
*group
= device
->group
;
397 list_del(&device
->group_next
);
398 mutex_unlock(&group
->device_lock
);
400 dev_set_drvdata(device
->dev
, NULL
);
404 /* vfio_del_group_dev may be waiting for this device */
405 wake_up(&vfio
.release_q
);
408 /* Device reference always implies a group reference */
409 void vfio_device_put(struct vfio_device
*device
)
411 struct vfio_group
*group
= device
->group
;
412 kref_put_mutex(&device
->kref
, vfio_device_release
, &group
->device_lock
);
413 vfio_group_put(group
);
415 EXPORT_SYMBOL_GPL(vfio_device_put
);
417 static void vfio_device_get(struct vfio_device
*device
)
419 vfio_group_get(device
->group
);
420 kref_get(&device
->kref
);
423 static struct vfio_device
*vfio_group_get_device(struct vfio_group
*group
,
426 struct vfio_device
*device
;
428 mutex_lock(&group
->device_lock
);
429 list_for_each_entry(device
, &group
->device_list
, group_next
) {
430 if (device
->dev
== dev
) {
431 vfio_device_get(device
);
432 mutex_unlock(&group
->device_lock
);
436 mutex_unlock(&group
->device_lock
);
441 * Whitelist some drivers that we know are safe (no dma) or just sit on
442 * a device. It's not always practical to leave a device within a group
443 * driverless as it could get re-bound to something unsafe.
445 static const char * const vfio_driver_whitelist
[] = { "pci-stub", "pcieport" };
447 static bool vfio_whitelisted_driver(struct device_driver
*drv
)
451 for (i
= 0; i
< ARRAY_SIZE(vfio_driver_whitelist
); i
++) {
452 if (!strcmp(drv
->name
, vfio_driver_whitelist
[i
]))
460 * A vfio group is viable for use by userspace if all devices are in
461 * one of the following states:
463 * - bound to a vfio driver
464 * - bound to a whitelisted driver
466 * We use two methods to determine whether a device is bound to a vfio
467 * driver. The first is to test whether the device exists in the vfio
468 * group. The second is to test if the device exists on the group
469 * unbound_list, indicating it's in the middle of transitioning from
470 * a vfio driver to driver-less.
472 static int vfio_dev_viable(struct device
*dev
, void *data
)
474 struct vfio_group
*group
= data
;
475 struct vfio_device
*device
;
476 struct device_driver
*drv
= ACCESS_ONCE(dev
->driver
);
477 struct vfio_unbound_dev
*unbound
;
480 mutex_lock(&group
->unbound_lock
);
481 list_for_each_entry(unbound
, &group
->unbound_list
, unbound_next
) {
482 if (dev
== unbound
->dev
) {
487 mutex_unlock(&group
->unbound_lock
);
489 if (!ret
|| !drv
|| vfio_whitelisted_driver(drv
))
492 device
= vfio_group_get_device(group
, dev
);
494 vfio_device_put(device
);
502 * Async device support
504 static int vfio_group_nb_add_dev(struct vfio_group
*group
, struct device
*dev
)
506 struct vfio_device
*device
;
508 /* Do we already know about it? We shouldn't */
509 device
= vfio_group_get_device(group
, dev
);
510 if (WARN_ON_ONCE(device
)) {
511 vfio_device_put(device
);
515 /* Nothing to do for idle groups */
516 if (!atomic_read(&group
->container_users
))
519 /* TODO Prevent device auto probing */
520 WARN("Device %s added to live group %d!\n", dev_name(dev
),
521 iommu_group_id(group
->iommu_group
));
526 static int vfio_group_nb_verify(struct vfio_group
*group
, struct device
*dev
)
528 /* We don't care what happens when the group isn't in use */
529 if (!atomic_read(&group
->container_users
))
532 return vfio_dev_viable(dev
, group
);
535 static int vfio_iommu_group_notifier(struct notifier_block
*nb
,
536 unsigned long action
, void *data
)
538 struct vfio_group
*group
= container_of(nb
, struct vfio_group
, nb
);
539 struct device
*dev
= data
;
540 struct vfio_unbound_dev
*unbound
;
543 * Need to go through a group_lock lookup to get a reference or we
544 * risk racing a group being removed. Ignore spurious notifies.
546 group
= vfio_group_try_get(group
);
551 case IOMMU_GROUP_NOTIFY_ADD_DEVICE
:
552 vfio_group_nb_add_dev(group
, dev
);
554 case IOMMU_GROUP_NOTIFY_DEL_DEVICE
:
556 * Nothing to do here. If the device is in use, then the
557 * vfio sub-driver should block the remove callback until
558 * it is unused. If the device is unused or attached to a
559 * stub driver, then it should be released and we don't
560 * care that it will be going away.
563 case IOMMU_GROUP_NOTIFY_BIND_DRIVER
:
564 pr_debug("%s: Device %s, group %d binding to driver\n",
565 __func__
, dev_name(dev
),
566 iommu_group_id(group
->iommu_group
));
568 case IOMMU_GROUP_NOTIFY_BOUND_DRIVER
:
569 pr_debug("%s: Device %s, group %d bound to driver %s\n",
570 __func__
, dev_name(dev
),
571 iommu_group_id(group
->iommu_group
), dev
->driver
->name
);
572 BUG_ON(vfio_group_nb_verify(group
, dev
));
574 case IOMMU_GROUP_NOTIFY_UNBIND_DRIVER
:
575 pr_debug("%s: Device %s, group %d unbinding from driver %s\n",
576 __func__
, dev_name(dev
),
577 iommu_group_id(group
->iommu_group
), dev
->driver
->name
);
579 case IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER
:
580 pr_debug("%s: Device %s, group %d unbound from driver\n",
581 __func__
, dev_name(dev
),
582 iommu_group_id(group
->iommu_group
));
584 * XXX An unbound device in a live group is ok, but we'd
585 * really like to avoid the above BUG_ON by preventing other
586 * drivers from binding to it. Once that occurs, we have to
587 * stop the system to maintain isolation. At a minimum, we'd
588 * want a toggle to disable driver auto probe for this device.
591 mutex_lock(&group
->unbound_lock
);
592 list_for_each_entry(unbound
,
593 &group
->unbound_list
, unbound_next
) {
594 if (dev
== unbound
->dev
) {
595 list_del(&unbound
->unbound_next
);
600 mutex_unlock(&group
->unbound_lock
);
604 vfio_group_put(group
);
611 int vfio_add_group_dev(struct device
*dev
,
612 const struct vfio_device_ops
*ops
, void *device_data
)
614 struct iommu_group
*iommu_group
;
615 struct vfio_group
*group
;
616 struct vfio_device
*device
;
618 iommu_group
= iommu_group_get(dev
);
622 group
= vfio_group_get_from_iommu(iommu_group
);
624 group
= vfio_create_group(iommu_group
);
626 iommu_group_put(iommu_group
);
627 return PTR_ERR(group
);
631 * A found vfio_group already holds a reference to the
632 * iommu_group. A created vfio_group keeps the reference.
634 iommu_group_put(iommu_group
);
637 device
= vfio_group_get_device(group
, dev
);
639 WARN(1, "Device %s already exists on group %d\n",
640 dev_name(dev
), iommu_group_id(iommu_group
));
641 vfio_device_put(device
);
642 vfio_group_put(group
);
646 device
= vfio_group_create_device(group
, dev
, ops
, device_data
);
647 if (IS_ERR(device
)) {
648 vfio_group_put(group
);
649 return PTR_ERR(device
);
653 * Drop all but the vfio_device reference. The vfio_device holds
654 * a reference to the vfio_group, which holds a reference to the
657 vfio_group_put(group
);
661 EXPORT_SYMBOL_GPL(vfio_add_group_dev
);
664 * Get a reference to the vfio_device for a device. Even if the
665 * caller thinks they own the device, they could be racing with a
666 * release call path, so we can't trust drvdata for the shortcut.
667 * Go the long way around, from the iommu_group to the vfio_group
668 * to the vfio_device.
670 struct vfio_device
*vfio_device_get_from_dev(struct device
*dev
)
672 struct iommu_group
*iommu_group
;
673 struct vfio_group
*group
;
674 struct vfio_device
*device
;
676 iommu_group
= iommu_group_get(dev
);
680 group
= vfio_group_get_from_iommu(iommu_group
);
681 iommu_group_put(iommu_group
);
685 device
= vfio_group_get_device(group
, dev
);
686 vfio_group_put(group
);
690 EXPORT_SYMBOL_GPL(vfio_device_get_from_dev
);
692 static struct vfio_device
*vfio_device_get_from_name(struct vfio_group
*group
,
695 struct vfio_device
*device
;
697 mutex_lock(&group
->device_lock
);
698 list_for_each_entry(device
, &group
->device_list
, group_next
) {
699 if (!strcmp(dev_name(device
->dev
), buf
)) {
700 vfio_device_get(device
);
704 mutex_unlock(&group
->device_lock
);
710 * Caller must hold a reference to the vfio_device
712 void *vfio_device_data(struct vfio_device
*device
)
714 return device
->device_data
;
716 EXPORT_SYMBOL_GPL(vfio_device_data
);
718 /* Given a referenced group, check if it contains the device */
719 static bool vfio_dev_present(struct vfio_group
*group
, struct device
*dev
)
721 struct vfio_device
*device
;
723 device
= vfio_group_get_device(group
, dev
);
727 vfio_device_put(device
);
732 * Decrement the device reference count and wait for the device to be
733 * removed. Open file descriptors for the device... */
734 void *vfio_del_group_dev(struct device
*dev
)
736 struct vfio_device
*device
= dev_get_drvdata(dev
);
737 struct vfio_group
*group
= device
->group
;
738 void *device_data
= device
->device_data
;
739 struct vfio_unbound_dev
*unbound
;
742 bool interrupted
= false;
745 * The group exists so long as we have a device reference. Get
746 * a group reference and use it to scan for the device going away.
748 vfio_group_get(group
);
751 * When the device is removed from the group, the group suddenly
752 * becomes non-viable; the device has a driver (until the unbind
753 * completes), but it's not present in the group. This is bad news
754 * for any external users that need to re-acquire a group reference
755 * in order to match and release their existing reference. To
756 * solve this, we track such devices on the unbound_list to bridge
757 * the gap until they're fully unbound.
759 unbound
= kzalloc(sizeof(*unbound
), GFP_KERNEL
);
762 mutex_lock(&group
->unbound_lock
);
763 list_add(&unbound
->unbound_next
, &group
->unbound_list
);
764 mutex_unlock(&group
->unbound_lock
);
768 vfio_device_put(device
);
771 * If the device is still present in the group after the above
772 * 'put', then it is in use and we need to request it from the
773 * bus driver. The driver may in turn need to request the
774 * device from the user. We send the request on an arbitrary
775 * interval with counter to allow the driver to take escalating
776 * measures to release the device if it has the ability to do so.
779 device
= vfio_group_get_device(group
, dev
);
783 if (device
->ops
->request
)
784 device
->ops
->request(device_data
, i
++);
786 vfio_device_put(device
);
789 ret
= wait_event_timeout(vfio
.release_q
,
790 !vfio_dev_present(group
, dev
), HZ
* 10);
792 ret
= wait_event_interruptible_timeout(vfio
.release_q
,
793 !vfio_dev_present(group
, dev
), HZ
* 10);
794 if (ret
== -ERESTARTSYS
) {
797 "Device is currently in use, task"
799 "blocked until device is released",
800 current
->comm
, task_pid_nr(current
));
805 vfio_group_put(group
);
809 EXPORT_SYMBOL_GPL(vfio_del_group_dev
);
812 * VFIO base fd, /dev/vfio/vfio
814 static long vfio_ioctl_check_extension(struct vfio_container
*container
,
817 struct vfio_iommu_driver
*driver
;
820 down_read(&container
->group_lock
);
822 driver
= container
->iommu_driver
;
825 /* No base extensions yet */
828 * If no driver is set, poll all registered drivers for
829 * extensions and return the first positive result. If
830 * a driver is already set, further queries will be passed
831 * only to that driver.
834 mutex_lock(&vfio
.iommu_drivers_lock
);
835 list_for_each_entry(driver
, &vfio
.iommu_drivers_list
,
837 if (!try_module_get(driver
->ops
->owner
))
840 ret
= driver
->ops
->ioctl(NULL
,
841 VFIO_CHECK_EXTENSION
,
843 module_put(driver
->ops
->owner
);
847 mutex_unlock(&vfio
.iommu_drivers_lock
);
849 ret
= driver
->ops
->ioctl(container
->iommu_data
,
850 VFIO_CHECK_EXTENSION
, arg
);
853 up_read(&container
->group_lock
);
858 /* hold write lock on container->group_lock */
859 static int __vfio_container_attach_groups(struct vfio_container
*container
,
860 struct vfio_iommu_driver
*driver
,
863 struct vfio_group
*group
;
866 list_for_each_entry(group
, &container
->group_list
, container_next
) {
867 ret
= driver
->ops
->attach_group(data
, group
->iommu_group
);
875 list_for_each_entry_continue_reverse(group
, &container
->group_list
,
877 driver
->ops
->detach_group(data
, group
->iommu_group
);
883 static long vfio_ioctl_set_iommu(struct vfio_container
*container
,
886 struct vfio_iommu_driver
*driver
;
889 down_write(&container
->group_lock
);
892 * The container is designed to be an unprivileged interface while
893 * the group can be assigned to specific users. Therefore, only by
894 * adding a group to a container does the user get the privilege of
895 * enabling the iommu, which may allocate finite resources. There
896 * is no unset_iommu, but by removing all the groups from a container,
897 * the container is deprivileged and returns to an unset state.
899 if (list_empty(&container
->group_list
) || container
->iommu_driver
) {
900 up_write(&container
->group_lock
);
904 mutex_lock(&vfio
.iommu_drivers_lock
);
905 list_for_each_entry(driver
, &vfio
.iommu_drivers_list
, vfio_next
) {
908 if (!try_module_get(driver
->ops
->owner
))
912 * The arg magic for SET_IOMMU is the same as CHECK_EXTENSION,
913 * so test which iommu driver reported support for this
914 * extension and call open on them. We also pass them the
915 * magic, allowing a single driver to support multiple
916 * interfaces if they'd like.
918 if (driver
->ops
->ioctl(NULL
, VFIO_CHECK_EXTENSION
, arg
) <= 0) {
919 module_put(driver
->ops
->owner
);
923 /* module reference holds the driver we're working on */
924 mutex_unlock(&vfio
.iommu_drivers_lock
);
926 data
= driver
->ops
->open(arg
);
929 module_put(driver
->ops
->owner
);
930 goto skip_drivers_unlock
;
933 ret
= __vfio_container_attach_groups(container
, driver
, data
);
935 container
->iommu_driver
= driver
;
936 container
->iommu_data
= data
;
938 driver
->ops
->release(data
);
939 module_put(driver
->ops
->owner
);
942 goto skip_drivers_unlock
;
945 mutex_unlock(&vfio
.iommu_drivers_lock
);
947 up_write(&container
->group_lock
);
952 static long vfio_fops_unl_ioctl(struct file
*filep
,
953 unsigned int cmd
, unsigned long arg
)
955 struct vfio_container
*container
= filep
->private_data
;
956 struct vfio_iommu_driver
*driver
;
964 case VFIO_GET_API_VERSION
:
965 ret
= VFIO_API_VERSION
;
967 case VFIO_CHECK_EXTENSION
:
968 ret
= vfio_ioctl_check_extension(container
, arg
);
971 ret
= vfio_ioctl_set_iommu(container
, arg
);
974 down_read(&container
->group_lock
);
976 driver
= container
->iommu_driver
;
977 data
= container
->iommu_data
;
979 if (driver
) /* passthrough all unrecognized ioctls */
980 ret
= driver
->ops
->ioctl(data
, cmd
, arg
);
982 up_read(&container
->group_lock
);
989 static long vfio_fops_compat_ioctl(struct file
*filep
,
990 unsigned int cmd
, unsigned long arg
)
992 arg
= (unsigned long)compat_ptr(arg
);
993 return vfio_fops_unl_ioctl(filep
, cmd
, arg
);
995 #endif /* CONFIG_COMPAT */
997 static int vfio_fops_open(struct inode
*inode
, struct file
*filep
)
999 struct vfio_container
*container
;
1001 container
= kzalloc(sizeof(*container
), GFP_KERNEL
);
1005 INIT_LIST_HEAD(&container
->group_list
);
1006 init_rwsem(&container
->group_lock
);
1007 kref_init(&container
->kref
);
1009 filep
->private_data
= container
;
1014 static int vfio_fops_release(struct inode
*inode
, struct file
*filep
)
1016 struct vfio_container
*container
= filep
->private_data
;
1018 filep
->private_data
= NULL
;
1020 vfio_container_put(container
);
1026 * Once an iommu driver is set, we optionally pass read/write/mmap
1027 * on to the driver, allowing management interfaces beyond ioctl.
1029 static ssize_t
vfio_fops_read(struct file
*filep
, char __user
*buf
,
1030 size_t count
, loff_t
*ppos
)
1032 struct vfio_container
*container
= filep
->private_data
;
1033 struct vfio_iommu_driver
*driver
;
1034 ssize_t ret
= -EINVAL
;
1036 down_read(&container
->group_lock
);
1038 driver
= container
->iommu_driver
;
1039 if (likely(driver
&& driver
->ops
->read
))
1040 ret
= driver
->ops
->read(container
->iommu_data
,
1043 up_read(&container
->group_lock
);
1048 static ssize_t
vfio_fops_write(struct file
*filep
, const char __user
*buf
,
1049 size_t count
, loff_t
*ppos
)
1051 struct vfio_container
*container
= filep
->private_data
;
1052 struct vfio_iommu_driver
*driver
;
1053 ssize_t ret
= -EINVAL
;
1055 down_read(&container
->group_lock
);
1057 driver
= container
->iommu_driver
;
1058 if (likely(driver
&& driver
->ops
->write
))
1059 ret
= driver
->ops
->write(container
->iommu_data
,
1062 up_read(&container
->group_lock
);
1067 static int vfio_fops_mmap(struct file
*filep
, struct vm_area_struct
*vma
)
1069 struct vfio_container
*container
= filep
->private_data
;
1070 struct vfio_iommu_driver
*driver
;
1073 down_read(&container
->group_lock
);
1075 driver
= container
->iommu_driver
;
1076 if (likely(driver
&& driver
->ops
->mmap
))
1077 ret
= driver
->ops
->mmap(container
->iommu_data
, vma
);
1079 up_read(&container
->group_lock
);
1084 static const struct file_operations vfio_fops
= {
1085 .owner
= THIS_MODULE
,
1086 .open
= vfio_fops_open
,
1087 .release
= vfio_fops_release
,
1088 .read
= vfio_fops_read
,
1089 .write
= vfio_fops_write
,
1090 .unlocked_ioctl
= vfio_fops_unl_ioctl
,
1091 #ifdef CONFIG_COMPAT
1092 .compat_ioctl
= vfio_fops_compat_ioctl
,
1094 .mmap
= vfio_fops_mmap
,
1098 * VFIO Group fd, /dev/vfio/$GROUP
1100 static void __vfio_group_unset_container(struct vfio_group
*group
)
1102 struct vfio_container
*container
= group
->container
;
1103 struct vfio_iommu_driver
*driver
;
1105 down_write(&container
->group_lock
);
1107 driver
= container
->iommu_driver
;
1109 driver
->ops
->detach_group(container
->iommu_data
,
1110 group
->iommu_group
);
1112 group
->container
= NULL
;
1113 list_del(&group
->container_next
);
1115 /* Detaching the last group deprivileges a container, remove iommu */
1116 if (driver
&& list_empty(&container
->group_list
)) {
1117 driver
->ops
->release(container
->iommu_data
);
1118 module_put(driver
->ops
->owner
);
1119 container
->iommu_driver
= NULL
;
1120 container
->iommu_data
= NULL
;
1123 up_write(&container
->group_lock
);
1125 vfio_container_put(container
);
1129 * VFIO_GROUP_UNSET_CONTAINER should fail if there are other users or
1130 * if there was no container to unset. Since the ioctl is called on
1131 * the group, we know that still exists, therefore the only valid
1132 * transition here is 1->0.
1134 static int vfio_group_unset_container(struct vfio_group
*group
)
1136 int users
= atomic_cmpxchg(&group
->container_users
, 1, 0);
1143 __vfio_group_unset_container(group
);
1149 * When removing container users, anything that removes the last user
1150 * implicitly removes the group from the container. That is, if the
1151 * group file descriptor is closed, as well as any device file descriptors,
1152 * the group is free.
1154 static void vfio_group_try_dissolve_container(struct vfio_group
*group
)
1156 if (0 == atomic_dec_if_positive(&group
->container_users
))
1157 __vfio_group_unset_container(group
);
1160 static int vfio_group_set_container(struct vfio_group
*group
, int container_fd
)
1163 struct vfio_container
*container
;
1164 struct vfio_iommu_driver
*driver
;
1167 if (atomic_read(&group
->container_users
))
1170 f
= fdget(container_fd
);
1174 /* Sanity check, is this really our fd? */
1175 if (f
.file
->f_op
!= &vfio_fops
) {
1180 container
= f
.file
->private_data
;
1181 WARN_ON(!container
); /* fget ensures we don't race vfio_release */
1183 down_write(&container
->group_lock
);
1185 driver
= container
->iommu_driver
;
1187 ret
= driver
->ops
->attach_group(container
->iommu_data
,
1188 group
->iommu_group
);
1193 group
->container
= container
;
1194 list_add(&group
->container_next
, &container
->group_list
);
1196 /* Get a reference on the container and mark a user within the group */
1197 vfio_container_get(container
);
1198 atomic_inc(&group
->container_users
);
1201 up_write(&container
->group_lock
);
1206 static bool vfio_group_viable(struct vfio_group
*group
)
1208 return (iommu_group_for_each_dev(group
->iommu_group
,
1209 group
, vfio_dev_viable
) == 0);
1212 static const struct file_operations vfio_device_fops
;
1214 static int vfio_group_get_device_fd(struct vfio_group
*group
, char *buf
)
1216 struct vfio_device
*device
;
1220 if (0 == atomic_read(&group
->container_users
) ||
1221 !group
->container
->iommu_driver
|| !vfio_group_viable(group
))
1224 device
= vfio_device_get_from_name(group
, buf
);
1228 ret
= device
->ops
->open(device
->device_data
);
1230 vfio_device_put(device
);
1235 * We can't use anon_inode_getfd() because we need to modify
1236 * the f_mode flags directly to allow more than just ioctls
1238 ret
= get_unused_fd_flags(O_CLOEXEC
);
1240 device
->ops
->release(device
->device_data
);
1241 vfio_device_put(device
);
1245 filep
= anon_inode_getfile("[vfio-device]", &vfio_device_fops
,
1247 if (IS_ERR(filep
)) {
1249 ret
= PTR_ERR(filep
);
1250 device
->ops
->release(device
->device_data
);
1251 vfio_device_put(device
);
1256 * TODO: add an anon_inode interface to do this.
1257 * Appears to be missing by lack of need rather than
1258 * explicitly prevented. Now there's need.
1260 filep
->f_mode
|= (FMODE_LSEEK
| FMODE_PREAD
| FMODE_PWRITE
);
1262 atomic_inc(&group
->container_users
);
1264 fd_install(ret
, filep
);
1269 static long vfio_group_fops_unl_ioctl(struct file
*filep
,
1270 unsigned int cmd
, unsigned long arg
)
1272 struct vfio_group
*group
= filep
->private_data
;
1276 case VFIO_GROUP_GET_STATUS
:
1278 struct vfio_group_status status
;
1279 unsigned long minsz
;
1281 minsz
= offsetofend(struct vfio_group_status
, flags
);
1283 if (copy_from_user(&status
, (void __user
*)arg
, minsz
))
1286 if (status
.argsz
< minsz
)
1291 if (vfio_group_viable(group
))
1292 status
.flags
|= VFIO_GROUP_FLAGS_VIABLE
;
1294 if (group
->container
)
1295 status
.flags
|= VFIO_GROUP_FLAGS_CONTAINER_SET
;
1297 if (copy_to_user((void __user
*)arg
, &status
, minsz
))
1303 case VFIO_GROUP_SET_CONTAINER
:
1307 if (get_user(fd
, (int __user
*)arg
))
1313 ret
= vfio_group_set_container(group
, fd
);
1316 case VFIO_GROUP_UNSET_CONTAINER
:
1317 ret
= vfio_group_unset_container(group
);
1319 case VFIO_GROUP_GET_DEVICE_FD
:
1323 buf
= strndup_user((const char __user
*)arg
, PAGE_SIZE
);
1325 return PTR_ERR(buf
);
1327 ret
= vfio_group_get_device_fd(group
, buf
);
1336 #ifdef CONFIG_COMPAT
1337 static long vfio_group_fops_compat_ioctl(struct file
*filep
,
1338 unsigned int cmd
, unsigned long arg
)
1340 arg
= (unsigned long)compat_ptr(arg
);
1341 return vfio_group_fops_unl_ioctl(filep
, cmd
, arg
);
1343 #endif /* CONFIG_COMPAT */
1345 static int vfio_group_fops_open(struct inode
*inode
, struct file
*filep
)
1347 struct vfio_group
*group
;
1350 group
= vfio_group_get_from_minor(iminor(inode
));
1354 /* Do we need multiple instances of the group open? Seems not. */
1355 opened
= atomic_cmpxchg(&group
->opened
, 0, 1);
1357 vfio_group_put(group
);
1361 /* Is something still in use from a previous open? */
1362 if (group
->container
) {
1363 atomic_dec(&group
->opened
);
1364 vfio_group_put(group
);
1368 filep
->private_data
= group
;
1373 static int vfio_group_fops_release(struct inode
*inode
, struct file
*filep
)
1375 struct vfio_group
*group
= filep
->private_data
;
1377 filep
->private_data
= NULL
;
1379 vfio_group_try_dissolve_container(group
);
1381 atomic_dec(&group
->opened
);
1383 vfio_group_put(group
);
1388 static const struct file_operations vfio_group_fops
= {
1389 .owner
= THIS_MODULE
,
1390 .unlocked_ioctl
= vfio_group_fops_unl_ioctl
,
1391 #ifdef CONFIG_COMPAT
1392 .compat_ioctl
= vfio_group_fops_compat_ioctl
,
1394 .open
= vfio_group_fops_open
,
1395 .release
= vfio_group_fops_release
,
1401 static int vfio_device_fops_release(struct inode
*inode
, struct file
*filep
)
1403 struct vfio_device
*device
= filep
->private_data
;
1405 device
->ops
->release(device
->device_data
);
1407 vfio_group_try_dissolve_container(device
->group
);
1409 vfio_device_put(device
);
1414 static long vfio_device_fops_unl_ioctl(struct file
*filep
,
1415 unsigned int cmd
, unsigned long arg
)
1417 struct vfio_device
*device
= filep
->private_data
;
1419 if (unlikely(!device
->ops
->ioctl
))
1422 return device
->ops
->ioctl(device
->device_data
, cmd
, arg
);
1425 static ssize_t
vfio_device_fops_read(struct file
*filep
, char __user
*buf
,
1426 size_t count
, loff_t
*ppos
)
1428 struct vfio_device
*device
= filep
->private_data
;
1430 if (unlikely(!device
->ops
->read
))
1433 return device
->ops
->read(device
->device_data
, buf
, count
, ppos
);
1436 static ssize_t
vfio_device_fops_write(struct file
*filep
,
1437 const char __user
*buf
,
1438 size_t count
, loff_t
*ppos
)
1440 struct vfio_device
*device
= filep
->private_data
;
1442 if (unlikely(!device
->ops
->write
))
1445 return device
->ops
->write(device
->device_data
, buf
, count
, ppos
);
1448 static int vfio_device_fops_mmap(struct file
*filep
, struct vm_area_struct
*vma
)
1450 struct vfio_device
*device
= filep
->private_data
;
1452 if (unlikely(!device
->ops
->mmap
))
1455 return device
->ops
->mmap(device
->device_data
, vma
);
1458 #ifdef CONFIG_COMPAT
1459 static long vfio_device_fops_compat_ioctl(struct file
*filep
,
1460 unsigned int cmd
, unsigned long arg
)
1462 arg
= (unsigned long)compat_ptr(arg
);
1463 return vfio_device_fops_unl_ioctl(filep
, cmd
, arg
);
1465 #endif /* CONFIG_COMPAT */
1467 static const struct file_operations vfio_device_fops
= {
1468 .owner
= THIS_MODULE
,
1469 .release
= vfio_device_fops_release
,
1470 .read
= vfio_device_fops_read
,
1471 .write
= vfio_device_fops_write
,
1472 .unlocked_ioctl
= vfio_device_fops_unl_ioctl
,
1473 #ifdef CONFIG_COMPAT
1474 .compat_ioctl
= vfio_device_fops_compat_ioctl
,
1476 .mmap
= vfio_device_fops_mmap
,
1480 * External user API, exported by symbols to be linked dynamically.
1482 * The protocol includes:
1483 * 1. do normal VFIO init operation:
1484 * - opening a new container;
1485 * - attaching group(s) to it;
1486 * - setting an IOMMU driver for a container.
1487 * When IOMMU is set for a container, all groups in it are
1488 * considered ready to use by an external user.
1490 * 2. User space passes a group fd to an external user.
1491 * The external user calls vfio_group_get_external_user()
1493 * - the group is initialized;
1494 * - IOMMU is set for it.
1495 * If both checks passed, vfio_group_get_external_user()
1496 * increments the container user counter to prevent
1497 * the VFIO group from disposal before KVM exits.
1499 * 3. The external user calls vfio_external_user_iommu_id()
1500 * to know an IOMMU ID.
1502 * 4. When the external KVM finishes, it calls
1503 * vfio_group_put_external_user() to release the VFIO group.
1504 * This call decrements the container user counter.
1506 struct vfio_group
*vfio_group_get_external_user(struct file
*filep
)
1508 struct vfio_group
*group
= filep
->private_data
;
1510 if (filep
->f_op
!= &vfio_group_fops
)
1511 return ERR_PTR(-EINVAL
);
1513 if (!atomic_inc_not_zero(&group
->container_users
))
1514 return ERR_PTR(-EINVAL
);
1516 if (!group
->container
->iommu_driver
||
1517 !vfio_group_viable(group
)) {
1518 atomic_dec(&group
->container_users
);
1519 return ERR_PTR(-EINVAL
);
1522 vfio_group_get(group
);
1526 EXPORT_SYMBOL_GPL(vfio_group_get_external_user
);
1528 void vfio_group_put_external_user(struct vfio_group
*group
)
1530 vfio_group_put(group
);
1531 vfio_group_try_dissolve_container(group
);
1533 EXPORT_SYMBOL_GPL(vfio_group_put_external_user
);
1535 int vfio_external_user_iommu_id(struct vfio_group
*group
)
1537 return iommu_group_id(group
->iommu_group
);
1539 EXPORT_SYMBOL_GPL(vfio_external_user_iommu_id
);
1541 long vfio_external_check_extension(struct vfio_group
*group
, unsigned long arg
)
1543 return vfio_ioctl_check_extension(group
->container
, arg
);
1545 EXPORT_SYMBOL_GPL(vfio_external_check_extension
);
1548 * Module/class support
1550 static char *vfio_devnode(struct device
*dev
, umode_t
*mode
)
1552 return kasprintf(GFP_KERNEL
, "vfio/%s", dev_name(dev
));
1555 static struct miscdevice vfio_dev
= {
1556 .minor
= VFIO_MINOR
,
1559 .nodename
= "vfio/vfio",
1560 .mode
= S_IRUGO
| S_IWUGO
,
1563 static int __init
vfio_init(void)
1567 idr_init(&vfio
.group_idr
);
1568 mutex_init(&vfio
.group_lock
);
1569 mutex_init(&vfio
.iommu_drivers_lock
);
1570 INIT_LIST_HEAD(&vfio
.group_list
);
1571 INIT_LIST_HEAD(&vfio
.iommu_drivers_list
);
1572 init_waitqueue_head(&vfio
.release_q
);
1574 ret
= misc_register(&vfio_dev
);
1576 pr_err("vfio: misc device register failed\n");
1580 /* /dev/vfio/$GROUP */
1581 vfio
.class = class_create(THIS_MODULE
, "vfio");
1582 if (IS_ERR(vfio
.class)) {
1583 ret
= PTR_ERR(vfio
.class);
1587 vfio
.class->devnode
= vfio_devnode
;
1589 ret
= alloc_chrdev_region(&vfio
.group_devt
, 0, MINORMASK
, "vfio");
1591 goto err_alloc_chrdev
;
1593 cdev_init(&vfio
.group_cdev
, &vfio_group_fops
);
1594 ret
= cdev_add(&vfio
.group_cdev
, vfio
.group_devt
, MINORMASK
);
1598 pr_info(DRIVER_DESC
" version: " DRIVER_VERSION
"\n");
1601 * Attempt to load known iommu-drivers. This gives us a working
1602 * environment without the user needing to explicitly load iommu
1605 request_module_nowait("vfio_iommu_type1");
1606 request_module_nowait("vfio_iommu_spapr_tce");
1611 unregister_chrdev_region(vfio
.group_devt
, MINORMASK
);
1613 class_destroy(vfio
.class);
1616 misc_deregister(&vfio_dev
);
1620 static void __exit
vfio_cleanup(void)
1622 WARN_ON(!list_empty(&vfio
.group_list
));
1624 idr_destroy(&vfio
.group_idr
);
1625 cdev_del(&vfio
.group_cdev
);
1626 unregister_chrdev_region(vfio
.group_devt
, MINORMASK
);
1627 class_destroy(vfio
.class);
1629 misc_deregister(&vfio_dev
);
1632 module_init(vfio_init
);
1633 module_exit(vfio_cleanup
);
1635 MODULE_VERSION(DRIVER_VERSION
);
1636 MODULE_LICENSE("GPL v2");
1637 MODULE_AUTHOR(DRIVER_AUTHOR
);
1638 MODULE_DESCRIPTION(DRIVER_DESC
);
1639 MODULE_ALIAS_MISCDEV(VFIO_MINOR
);
1640 MODULE_ALIAS("devname:vfio/vfio");