4 * Copyright (C) 2012 Red Hat, Inc. All rights reserved.
5 * Author: Alex Williamson <alex.williamson@redhat.com>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
11 * Derived from original vfio:
12 * Copyright 2010 Cisco Systems, Inc. All rights reserved.
13 * Author: Tom Lyon, pugs@cisco.com
16 #include <linux/cdev.h>
17 #include <linux/compat.h>
18 #include <linux/device.h>
19 #include <linux/file.h>
20 #include <linux/anon_inodes.h>
22 #include <linux/idr.h>
23 #include <linux/iommu.h>
24 #include <linux/list.h>
25 #include <linux/module.h>
26 #include <linux/mutex.h>
27 #include <linux/sched.h>
28 #include <linux/slab.h>
29 #include <linux/string.h>
30 #include <linux/uaccess.h>
31 #include <linux/vfio.h>
32 #include <linux/wait.h>
34 #define DRIVER_VERSION "0.3"
35 #define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
36 #define DRIVER_DESC "VFIO - User Level meta-driver"
40 struct list_head iommu_drivers_list
;
41 struct mutex iommu_drivers_lock
;
42 struct list_head group_list
;
44 struct mutex group_lock
;
45 struct cdev group_cdev
;
49 wait_queue_head_t release_q
;
52 struct vfio_iommu_driver
{
53 const struct vfio_iommu_driver_ops
*ops
;
54 struct list_head vfio_next
;
57 struct vfio_container
{
59 struct list_head group_list
;
60 struct mutex group_lock
;
61 struct vfio_iommu_driver
*iommu_driver
;
68 atomic_t container_users
;
69 struct iommu_group
*iommu_group
;
70 struct vfio_container
*container
;
71 struct list_head device_list
;
72 struct mutex device_lock
;
74 struct notifier_block nb
;
75 struct list_head vfio_next
;
76 struct list_head container_next
;
82 const struct vfio_device_ops
*ops
;
83 struct vfio_group
*group
;
84 struct list_head group_next
;
89 * IOMMU driver registration
91 int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops
*ops
)
93 struct vfio_iommu_driver
*driver
, *tmp
;
95 driver
= kzalloc(sizeof(*driver
), GFP_KERNEL
);
101 mutex_lock(&vfio
.iommu_drivers_lock
);
103 /* Check for duplicates */
104 list_for_each_entry(tmp
, &vfio
.iommu_drivers_list
, vfio_next
) {
105 if (tmp
->ops
== ops
) {
106 mutex_unlock(&vfio
.iommu_drivers_lock
);
112 list_add(&driver
->vfio_next
, &vfio
.iommu_drivers_list
);
114 mutex_unlock(&vfio
.iommu_drivers_lock
);
118 EXPORT_SYMBOL_GPL(vfio_register_iommu_driver
);
120 void vfio_unregister_iommu_driver(const struct vfio_iommu_driver_ops
*ops
)
122 struct vfio_iommu_driver
*driver
;
124 mutex_lock(&vfio
.iommu_drivers_lock
);
125 list_for_each_entry(driver
, &vfio
.iommu_drivers_list
, vfio_next
) {
126 if (driver
->ops
== ops
) {
127 list_del(&driver
->vfio_next
);
128 mutex_unlock(&vfio
.iommu_drivers_lock
);
133 mutex_unlock(&vfio
.iommu_drivers_lock
);
135 EXPORT_SYMBOL_GPL(vfio_unregister_iommu_driver
);
138 * Group minor allocation/free - both called with vfio.group_lock held
140 static int vfio_alloc_group_minor(struct vfio_group
*group
)
145 if (unlikely(idr_pre_get(&vfio
.group_idr
, GFP_KERNEL
) == 0))
148 /* index 0 is used by /dev/vfio/vfio */
149 ret
= idr_get_new_above(&vfio
.group_idr
, group
, 1, &minor
);
152 if (ret
|| minor
> MINORMASK
) {
153 if (minor
> MINORMASK
)
154 idr_remove(&vfio
.group_idr
, minor
);
161 static void vfio_free_group_minor(int minor
)
163 idr_remove(&vfio
.group_idr
, minor
);
166 static int vfio_iommu_group_notifier(struct notifier_block
*nb
,
167 unsigned long action
, void *data
);
168 static void vfio_group_get(struct vfio_group
*group
);
171 * Container objects - containers are created when /dev/vfio/vfio is
172 * opened, but their lifecycle extends until the last user is done, so
173 * it's freed via kref. Must support container/group/device being
174 * closed in any order.
176 static void vfio_container_get(struct vfio_container
*container
)
178 kref_get(&container
->kref
);
181 static void vfio_container_release(struct kref
*kref
)
183 struct vfio_container
*container
;
184 container
= container_of(kref
, struct vfio_container
, kref
);
189 static void vfio_container_put(struct vfio_container
*container
)
191 kref_put(&container
->kref
, vfio_container_release
);
195 * Group objects - create, release, get, put, search
197 static struct vfio_group
*vfio_create_group(struct iommu_group
*iommu_group
)
199 struct vfio_group
*group
, *tmp
;
203 group
= kzalloc(sizeof(*group
), GFP_KERNEL
);
205 return ERR_PTR(-ENOMEM
);
207 kref_init(&group
->kref
);
208 INIT_LIST_HEAD(&group
->device_list
);
209 mutex_init(&group
->device_lock
);
210 atomic_set(&group
->container_users
, 0);
211 group
->iommu_group
= iommu_group
;
213 group
->nb
.notifier_call
= vfio_iommu_group_notifier
;
216 * blocking notifiers acquire a rwsem around registering and hold
217 * it around callback. Therefore, need to register outside of
218 * vfio.group_lock to avoid A-B/B-A contention. Our callback won't
219 * do anything unless it can find the group in vfio.group_list, so
220 * no harm in registering early.
222 ret
= iommu_group_register_notifier(iommu_group
, &group
->nb
);
228 mutex_lock(&vfio
.group_lock
);
230 minor
= vfio_alloc_group_minor(group
);
232 mutex_unlock(&vfio
.group_lock
);
234 return ERR_PTR(minor
);
237 /* Did we race creating this group? */
238 list_for_each_entry(tmp
, &vfio
.group_list
, vfio_next
) {
239 if (tmp
->iommu_group
== iommu_group
) {
241 vfio_free_group_minor(minor
);
242 mutex_unlock(&vfio
.group_lock
);
248 dev
= device_create(vfio
.class, NULL
, MKDEV(MAJOR(vfio
.devt
), minor
),
249 group
, "%d", iommu_group_id(iommu_group
));
251 vfio_free_group_minor(minor
);
252 mutex_unlock(&vfio
.group_lock
);
254 return (struct vfio_group
*)dev
; /* ERR_PTR */
257 group
->minor
= minor
;
260 list_add(&group
->vfio_next
, &vfio
.group_list
);
262 mutex_unlock(&vfio
.group_lock
);
267 static void vfio_group_release(struct kref
*kref
)
269 struct vfio_group
*group
= container_of(kref
, struct vfio_group
, kref
);
271 WARN_ON(!list_empty(&group
->device_list
));
273 device_destroy(vfio
.class, MKDEV(MAJOR(vfio
.devt
), group
->minor
));
274 list_del(&group
->vfio_next
);
275 vfio_free_group_minor(group
->minor
);
277 mutex_unlock(&vfio
.group_lock
);
280 * Unregister outside of lock. A spurious callback is harmless now
281 * that the group is no longer in vfio.group_list.
283 iommu_group_unregister_notifier(group
->iommu_group
, &group
->nb
);
288 static void vfio_group_put(struct vfio_group
*group
)
290 mutex_lock(&vfio
.group_lock
);
292 * Release needs to unlock to unregister the notifier, so only
293 * unlock if not released.
295 if (!kref_put(&group
->kref
, vfio_group_release
))
296 mutex_unlock(&vfio
.group_lock
);
299 /* Assume group_lock or group reference is held */
300 static void vfio_group_get(struct vfio_group
*group
)
302 kref_get(&group
->kref
);
306 * Not really a try as we will sleep for mutex, but we need to make
307 * sure the group pointer is valid under lock and get a reference.
309 static struct vfio_group
*vfio_group_try_get(struct vfio_group
*group
)
311 struct vfio_group
*target
= group
;
313 mutex_lock(&vfio
.group_lock
);
314 list_for_each_entry(group
, &vfio
.group_list
, vfio_next
) {
315 if (group
== target
) {
316 vfio_group_get(group
);
317 mutex_unlock(&vfio
.group_lock
);
321 mutex_unlock(&vfio
.group_lock
);
327 struct vfio_group
*vfio_group_get_from_iommu(struct iommu_group
*iommu_group
)
329 struct vfio_group
*group
;
331 mutex_lock(&vfio
.group_lock
);
332 list_for_each_entry(group
, &vfio
.group_list
, vfio_next
) {
333 if (group
->iommu_group
== iommu_group
) {
334 vfio_group_get(group
);
335 mutex_unlock(&vfio
.group_lock
);
339 mutex_unlock(&vfio
.group_lock
);
344 static struct vfio_group
*vfio_group_get_from_minor(int minor
)
346 struct vfio_group
*group
;
348 mutex_lock(&vfio
.group_lock
);
349 group
= idr_find(&vfio
.group_idr
, minor
);
351 mutex_unlock(&vfio
.group_lock
);
354 vfio_group_get(group
);
355 mutex_unlock(&vfio
.group_lock
);
361 * Device objects - create, release, get, put, search
364 struct vfio_device
*vfio_group_create_device(struct vfio_group
*group
,
366 const struct vfio_device_ops
*ops
,
369 struct vfio_device
*device
;
372 device
= kzalloc(sizeof(*device
), GFP_KERNEL
);
374 return ERR_PTR(-ENOMEM
);
376 kref_init(&device
->kref
);
378 device
->group
= group
;
380 device
->device_data
= device_data
;
382 ret
= dev_set_drvdata(dev
, device
);
388 /* No need to get group_lock, caller has group reference */
389 vfio_group_get(group
);
391 mutex_lock(&group
->device_lock
);
392 list_add(&device
->group_next
, &group
->device_list
);
393 mutex_unlock(&group
->device_lock
);
398 static void vfio_device_release(struct kref
*kref
)
400 struct vfio_device
*device
= container_of(kref
,
401 struct vfio_device
, kref
);
402 struct vfio_group
*group
= device
->group
;
404 mutex_lock(&group
->device_lock
);
405 list_del(&device
->group_next
);
406 mutex_unlock(&group
->device_lock
);
408 dev_set_drvdata(device
->dev
, NULL
);
412 /* vfio_del_group_dev may be waiting for this device */
413 wake_up(&vfio
.release_q
);
416 /* Device reference always implies a group reference */
417 static void vfio_device_put(struct vfio_device
*device
)
419 kref_put(&device
->kref
, vfio_device_release
);
420 vfio_group_put(device
->group
);
423 static void vfio_device_get(struct vfio_device
*device
)
425 vfio_group_get(device
->group
);
426 kref_get(&device
->kref
);
429 static struct vfio_device
*vfio_group_get_device(struct vfio_group
*group
,
432 struct vfio_device
*device
;
434 mutex_lock(&group
->device_lock
);
435 list_for_each_entry(device
, &group
->device_list
, group_next
) {
436 if (device
->dev
== dev
) {
437 vfio_device_get(device
);
438 mutex_unlock(&group
->device_lock
);
442 mutex_unlock(&group
->device_lock
);
447 * Whitelist some drivers that we know are safe (no dma) or just sit on
448 * a device. It's not always practical to leave a device within a group
449 * driverless as it could get re-bound to something unsafe.
451 static const char * const vfio_driver_whitelist
[] = { "pci-stub" };
453 static bool vfio_whitelisted_driver(struct device_driver
*drv
)
457 for (i
= 0; i
< ARRAY_SIZE(vfio_driver_whitelist
); i
++) {
458 if (!strcmp(drv
->name
, vfio_driver_whitelist
[i
]))
466 * A vfio group is viable for use by userspace if all devices are either
467 * driver-less or bound to a vfio or whitelisted driver. We test the
468 * latter by the existence of a struct vfio_device matching the dev.
470 static int vfio_dev_viable(struct device
*dev
, void *data
)
472 struct vfio_group
*group
= data
;
473 struct vfio_device
*device
;
475 if (!dev
->driver
|| vfio_whitelisted_driver(dev
->driver
))
478 device
= vfio_group_get_device(group
, dev
);
480 vfio_device_put(device
);
488 * Async device support
490 static int vfio_group_nb_add_dev(struct vfio_group
*group
, struct device
*dev
)
492 struct vfio_device
*device
;
494 /* Do we already know about it? We shouldn't */
495 device
= vfio_group_get_device(group
, dev
);
496 if (WARN_ON_ONCE(device
)) {
497 vfio_device_put(device
);
501 /* Nothing to do for idle groups */
502 if (!atomic_read(&group
->container_users
))
505 /* TODO Prevent device auto probing */
506 WARN("Device %s added to live group %d!\n", dev_name(dev
),
507 iommu_group_id(group
->iommu_group
));
512 static int vfio_group_nb_del_dev(struct vfio_group
*group
, struct device
*dev
)
514 struct vfio_device
*device
;
517 * Expect to fall out here. If a device was in use, it would
518 * have been bound to a vfio sub-driver, which would have blocked
519 * in .remove at vfio_del_group_dev. Sanity check that we no
520 * longer track the device, so it's safe to remove.
522 device
= vfio_group_get_device(group
, dev
);
526 WARN("Device %s removed from live group %d!\n", dev_name(dev
),
527 iommu_group_id(group
->iommu_group
));
529 vfio_device_put(device
);
533 static int vfio_group_nb_verify(struct vfio_group
*group
, struct device
*dev
)
535 /* We don't care what happens when the group isn't in use */
536 if (!atomic_read(&group
->container_users
))
539 return vfio_dev_viable(dev
, group
);
542 static int vfio_iommu_group_notifier(struct notifier_block
*nb
,
543 unsigned long action
, void *data
)
545 struct vfio_group
*group
= container_of(nb
, struct vfio_group
, nb
);
546 struct device
*dev
= data
;
549 * Need to go through a group_lock lookup to get a reference or
550 * we risk racing a group being removed. Leave a WARN_ON for
551 * debuging, but if the group no longer exists, a spurious notify
554 group
= vfio_group_try_get(group
);
559 case IOMMU_GROUP_NOTIFY_ADD_DEVICE
:
560 vfio_group_nb_add_dev(group
, dev
);
562 case IOMMU_GROUP_NOTIFY_DEL_DEVICE
:
563 vfio_group_nb_del_dev(group
, dev
);
565 case IOMMU_GROUP_NOTIFY_BIND_DRIVER
:
566 pr_debug("%s: Device %s, group %d binding to driver\n",
567 __func__
, dev_name(dev
),
568 iommu_group_id(group
->iommu_group
));
570 case IOMMU_GROUP_NOTIFY_BOUND_DRIVER
:
571 pr_debug("%s: Device %s, group %d bound to driver %s\n",
572 __func__
, dev_name(dev
),
573 iommu_group_id(group
->iommu_group
), dev
->driver
->name
);
574 BUG_ON(vfio_group_nb_verify(group
, dev
));
576 case IOMMU_GROUP_NOTIFY_UNBIND_DRIVER
:
577 pr_debug("%s: Device %s, group %d unbinding from driver %s\n",
578 __func__
, dev_name(dev
),
579 iommu_group_id(group
->iommu_group
), dev
->driver
->name
);
581 case IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER
:
582 pr_debug("%s: Device %s, group %d unbound from driver\n",
583 __func__
, dev_name(dev
),
584 iommu_group_id(group
->iommu_group
));
586 * XXX An unbound device in a live group is ok, but we'd
587 * really like to avoid the above BUG_ON by preventing other
588 * drivers from binding to it. Once that occurs, we have to
589 * stop the system to maintain isolation. At a minimum, we'd
590 * want a toggle to disable driver auto probe for this device.
595 vfio_group_put(group
);
602 int vfio_add_group_dev(struct device
*dev
,
603 const struct vfio_device_ops
*ops
, void *device_data
)
605 struct iommu_group
*iommu_group
;
606 struct vfio_group
*group
;
607 struct vfio_device
*device
;
609 iommu_group
= iommu_group_get(dev
);
613 group
= vfio_group_get_from_iommu(iommu_group
);
615 group
= vfio_create_group(iommu_group
);
617 iommu_group_put(iommu_group
);
618 return PTR_ERR(group
);
622 device
= vfio_group_get_device(group
, dev
);
624 WARN(1, "Device %s already exists on group %d\n",
625 dev_name(dev
), iommu_group_id(iommu_group
));
626 vfio_device_put(device
);
627 vfio_group_put(group
);
628 iommu_group_put(iommu_group
);
632 device
= vfio_group_create_device(group
, dev
, ops
, device_data
);
633 if (IS_ERR(device
)) {
634 vfio_group_put(group
);
635 iommu_group_put(iommu_group
);
636 return PTR_ERR(device
);
640 * Added device holds reference to iommu_group and vfio_device
641 * (which in turn holds reference to vfio_group). Drop extra
642 * group reference used while acquiring device.
644 vfio_group_put(group
);
648 EXPORT_SYMBOL_GPL(vfio_add_group_dev
);
650 /* Test whether a struct device is present in our tracking */
651 static bool vfio_dev_present(struct device
*dev
)
653 struct iommu_group
*iommu_group
;
654 struct vfio_group
*group
;
655 struct vfio_device
*device
;
657 iommu_group
= iommu_group_get(dev
);
661 group
= vfio_group_get_from_iommu(iommu_group
);
663 iommu_group_put(iommu_group
);
667 device
= vfio_group_get_device(group
, dev
);
669 vfio_group_put(group
);
670 iommu_group_put(iommu_group
);
674 vfio_device_put(device
);
675 vfio_group_put(group
);
676 iommu_group_put(iommu_group
);
681 * Decrement the device reference count and wait for the device to be
682 * removed. Open file descriptors for the device... */
683 void *vfio_del_group_dev(struct device
*dev
)
685 struct vfio_device
*device
= dev_get_drvdata(dev
);
686 struct vfio_group
*group
= device
->group
;
687 struct iommu_group
*iommu_group
= group
->iommu_group
;
688 void *device_data
= device
->device_data
;
690 vfio_device_put(device
);
692 /* TODO send a signal to encourage this to be released */
693 wait_event(vfio
.release_q
, !vfio_dev_present(dev
));
695 iommu_group_put(iommu_group
);
699 EXPORT_SYMBOL_GPL(vfio_del_group_dev
);
702 * VFIO base fd, /dev/vfio/vfio
704 static long vfio_ioctl_check_extension(struct vfio_container
*container
,
707 struct vfio_iommu_driver
*driver
= container
->iommu_driver
;
711 /* No base extensions yet */
714 * If no driver is set, poll all registered drivers for
715 * extensions and return the first positive result. If
716 * a driver is already set, further queries will be passed
717 * only to that driver.
720 mutex_lock(&vfio
.iommu_drivers_lock
);
721 list_for_each_entry(driver
, &vfio
.iommu_drivers_list
,
723 if (!try_module_get(driver
->ops
->owner
))
726 ret
= driver
->ops
->ioctl(NULL
,
727 VFIO_CHECK_EXTENSION
,
729 module_put(driver
->ops
->owner
);
733 mutex_unlock(&vfio
.iommu_drivers_lock
);
735 ret
= driver
->ops
->ioctl(container
->iommu_data
,
736 VFIO_CHECK_EXTENSION
, arg
);
742 /* hold container->group_lock */
743 static int __vfio_container_attach_groups(struct vfio_container
*container
,
744 struct vfio_iommu_driver
*driver
,
747 struct vfio_group
*group
;
750 list_for_each_entry(group
, &container
->group_list
, container_next
) {
751 ret
= driver
->ops
->attach_group(data
, group
->iommu_group
);
759 list_for_each_entry_continue_reverse(group
, &container
->group_list
,
761 driver
->ops
->detach_group(data
, group
->iommu_group
);
767 static long vfio_ioctl_set_iommu(struct vfio_container
*container
,
770 struct vfio_iommu_driver
*driver
;
773 mutex_lock(&container
->group_lock
);
776 * The container is designed to be an unprivileged interface while
777 * the group can be assigned to specific users. Therefore, only by
778 * adding a group to a container does the user get the privilege of
779 * enabling the iommu, which may allocate finite resources. There
780 * is no unset_iommu, but by removing all the groups from a container,
781 * the container is deprivileged and returns to an unset state.
783 if (list_empty(&container
->group_list
) || container
->iommu_driver
) {
784 mutex_unlock(&container
->group_lock
);
788 mutex_lock(&vfio
.iommu_drivers_lock
);
789 list_for_each_entry(driver
, &vfio
.iommu_drivers_list
, vfio_next
) {
792 if (!try_module_get(driver
->ops
->owner
))
796 * The arg magic for SET_IOMMU is the same as CHECK_EXTENSION,
797 * so test which iommu driver reported support for this
798 * extension and call open on them. We also pass them the
799 * magic, allowing a single driver to support multiple
800 * interfaces if they'd like.
802 if (driver
->ops
->ioctl(NULL
, VFIO_CHECK_EXTENSION
, arg
) <= 0) {
803 module_put(driver
->ops
->owner
);
807 /* module reference holds the driver we're working on */
808 mutex_unlock(&vfio
.iommu_drivers_lock
);
810 data
= driver
->ops
->open(arg
);
813 module_put(driver
->ops
->owner
);
814 goto skip_drivers_unlock
;
817 ret
= __vfio_container_attach_groups(container
, driver
, data
);
819 container
->iommu_driver
= driver
;
820 container
->iommu_data
= data
;
822 driver
->ops
->release(data
);
823 module_put(driver
->ops
->owner
);
826 goto skip_drivers_unlock
;
829 mutex_unlock(&vfio
.iommu_drivers_lock
);
831 mutex_unlock(&container
->group_lock
);
836 static long vfio_fops_unl_ioctl(struct file
*filep
,
837 unsigned int cmd
, unsigned long arg
)
839 struct vfio_container
*container
= filep
->private_data
;
840 struct vfio_iommu_driver
*driver
;
847 driver
= container
->iommu_driver
;
848 data
= container
->iommu_data
;
851 case VFIO_GET_API_VERSION
:
852 ret
= VFIO_API_VERSION
;
854 case VFIO_CHECK_EXTENSION
:
855 ret
= vfio_ioctl_check_extension(container
, arg
);
858 ret
= vfio_ioctl_set_iommu(container
, arg
);
861 if (driver
) /* passthrough all unrecognized ioctls */
862 ret
= driver
->ops
->ioctl(data
, cmd
, arg
);
869 static long vfio_fops_compat_ioctl(struct file
*filep
,
870 unsigned int cmd
, unsigned long arg
)
872 arg
= (unsigned long)compat_ptr(arg
);
873 return vfio_fops_unl_ioctl(filep
, cmd
, arg
);
875 #endif /* CONFIG_COMPAT */
877 static int vfio_fops_open(struct inode
*inode
, struct file
*filep
)
879 struct vfio_container
*container
;
881 container
= kzalloc(sizeof(*container
), GFP_KERNEL
);
885 INIT_LIST_HEAD(&container
->group_list
);
886 mutex_init(&container
->group_lock
);
887 kref_init(&container
->kref
);
889 filep
->private_data
= container
;
894 static int vfio_fops_release(struct inode
*inode
, struct file
*filep
)
896 struct vfio_container
*container
= filep
->private_data
;
898 filep
->private_data
= NULL
;
900 vfio_container_put(container
);
906 * Once an iommu driver is set, we optionally pass read/write/mmap
907 * on to the driver, allowing management interfaces beyond ioctl.
909 static ssize_t
vfio_fops_read(struct file
*filep
, char __user
*buf
,
910 size_t count
, loff_t
*ppos
)
912 struct vfio_container
*container
= filep
->private_data
;
913 struct vfio_iommu_driver
*driver
= container
->iommu_driver
;
915 if (unlikely(!driver
|| !driver
->ops
->read
))
918 return driver
->ops
->read(container
->iommu_data
, buf
, count
, ppos
);
921 static ssize_t
vfio_fops_write(struct file
*filep
, const char __user
*buf
,
922 size_t count
, loff_t
*ppos
)
924 struct vfio_container
*container
= filep
->private_data
;
925 struct vfio_iommu_driver
*driver
= container
->iommu_driver
;
927 if (unlikely(!driver
|| !driver
->ops
->write
))
930 return driver
->ops
->write(container
->iommu_data
, buf
, count
, ppos
);
933 static int vfio_fops_mmap(struct file
*filep
, struct vm_area_struct
*vma
)
935 struct vfio_container
*container
= filep
->private_data
;
936 struct vfio_iommu_driver
*driver
= container
->iommu_driver
;
938 if (unlikely(!driver
|| !driver
->ops
->mmap
))
941 return driver
->ops
->mmap(container
->iommu_data
, vma
);
944 static const struct file_operations vfio_fops
= {
945 .owner
= THIS_MODULE
,
946 .open
= vfio_fops_open
,
947 .release
= vfio_fops_release
,
948 .read
= vfio_fops_read
,
949 .write
= vfio_fops_write
,
950 .unlocked_ioctl
= vfio_fops_unl_ioctl
,
952 .compat_ioctl
= vfio_fops_compat_ioctl
,
954 .mmap
= vfio_fops_mmap
,
958 * VFIO Group fd, /dev/vfio/$GROUP
960 static void __vfio_group_unset_container(struct vfio_group
*group
)
962 struct vfio_container
*container
= group
->container
;
963 struct vfio_iommu_driver
*driver
;
965 mutex_lock(&container
->group_lock
);
967 driver
= container
->iommu_driver
;
969 driver
->ops
->detach_group(container
->iommu_data
,
972 group
->container
= NULL
;
973 list_del(&group
->container_next
);
975 /* Detaching the last group deprivileges a container, remove iommu */
976 if (driver
&& list_empty(&container
->group_list
)) {
977 driver
->ops
->release(container
->iommu_data
);
978 module_put(driver
->ops
->owner
);
979 container
->iommu_driver
= NULL
;
980 container
->iommu_data
= NULL
;
983 mutex_unlock(&container
->group_lock
);
985 vfio_container_put(container
);
989 * VFIO_GROUP_UNSET_CONTAINER should fail if there are other users or
990 * if there was no container to unset. Since the ioctl is called on
991 * the group, we know that still exists, therefore the only valid
992 * transition here is 1->0.
994 static int vfio_group_unset_container(struct vfio_group
*group
)
996 int users
= atomic_cmpxchg(&group
->container_users
, 1, 0);
1003 __vfio_group_unset_container(group
);
1009 * When removing container users, anything that removes the last user
1010 * implicitly removes the group from the container. That is, if the
1011 * group file descriptor is closed, as well as any device file descriptors,
1012 * the group is free.
1014 static void vfio_group_try_dissolve_container(struct vfio_group
*group
)
1016 if (0 == atomic_dec_if_positive(&group
->container_users
))
1017 __vfio_group_unset_container(group
);
1020 static int vfio_group_set_container(struct vfio_group
*group
, int container_fd
)
1023 struct vfio_container
*container
;
1024 struct vfio_iommu_driver
*driver
;
1027 if (atomic_read(&group
->container_users
))
1030 filep
= fget(container_fd
);
1034 /* Sanity check, is this really our fd? */
1035 if (filep
->f_op
!= &vfio_fops
) {
1040 container
= filep
->private_data
;
1041 WARN_ON(!container
); /* fget ensures we don't race vfio_release */
1043 mutex_lock(&container
->group_lock
);
1045 driver
= container
->iommu_driver
;
1047 ret
= driver
->ops
->attach_group(container
->iommu_data
,
1048 group
->iommu_group
);
1053 group
->container
= container
;
1054 list_add(&group
->container_next
, &container
->group_list
);
1056 /* Get a reference on the container and mark a user within the group */
1057 vfio_container_get(container
);
1058 atomic_inc(&group
->container_users
);
1061 mutex_unlock(&container
->group_lock
);
1067 static bool vfio_group_viable(struct vfio_group
*group
)
1069 return (iommu_group_for_each_dev(group
->iommu_group
,
1070 group
, vfio_dev_viable
) == 0);
1073 static const struct file_operations vfio_device_fops
;
1075 static int vfio_group_get_device_fd(struct vfio_group
*group
, char *buf
)
1077 struct vfio_device
*device
;
1081 if (0 == atomic_read(&group
->container_users
) ||
1082 !group
->container
->iommu_driver
|| !vfio_group_viable(group
))
1085 mutex_lock(&group
->device_lock
);
1086 list_for_each_entry(device
, &group
->device_list
, group_next
) {
1087 if (strcmp(dev_name(device
->dev
), buf
))
1090 ret
= device
->ops
->open(device
->device_data
);
1094 * We can't use anon_inode_getfd() because we need to modify
1095 * the f_mode flags directly to allow more than just ioctls
1097 ret
= get_unused_fd();
1099 device
->ops
->release(device
->device_data
);
1103 filep
= anon_inode_getfile("[vfio-device]", &vfio_device_fops
,
1105 if (IS_ERR(filep
)) {
1107 ret
= PTR_ERR(filep
);
1108 device
->ops
->release(device
->device_data
);
1113 * TODO: add an anon_inode interface to do this.
1114 * Appears to be missing by lack of need rather than
1115 * explicitly prevented. Now there's need.
1117 filep
->f_mode
|= (FMODE_LSEEK
| FMODE_PREAD
| FMODE_PWRITE
);
1119 fd_install(ret
, filep
);
1121 vfio_device_get(device
);
1122 atomic_inc(&group
->container_users
);
1125 mutex_unlock(&group
->device_lock
);
1130 static long vfio_group_fops_unl_ioctl(struct file
*filep
,
1131 unsigned int cmd
, unsigned long arg
)
1133 struct vfio_group
*group
= filep
->private_data
;
1137 case VFIO_GROUP_GET_STATUS
:
1139 struct vfio_group_status status
;
1140 unsigned long minsz
;
1142 minsz
= offsetofend(struct vfio_group_status
, flags
);
1144 if (copy_from_user(&status
, (void __user
*)arg
, minsz
))
1147 if (status
.argsz
< minsz
)
1152 if (vfio_group_viable(group
))
1153 status
.flags
|= VFIO_GROUP_FLAGS_VIABLE
;
1155 if (group
->container
)
1156 status
.flags
|= VFIO_GROUP_FLAGS_CONTAINER_SET
;
1158 if (copy_to_user((void __user
*)arg
, &status
, minsz
))
1164 case VFIO_GROUP_SET_CONTAINER
:
1168 if (get_user(fd
, (int __user
*)arg
))
1174 ret
= vfio_group_set_container(group
, fd
);
1177 case VFIO_GROUP_UNSET_CONTAINER
:
1178 ret
= vfio_group_unset_container(group
);
1180 case VFIO_GROUP_GET_DEVICE_FD
:
1184 buf
= strndup_user((const char __user
*)arg
, PAGE_SIZE
);
1186 return PTR_ERR(buf
);
1188 ret
= vfio_group_get_device_fd(group
, buf
);
1197 #ifdef CONFIG_COMPAT
1198 static long vfio_group_fops_compat_ioctl(struct file
*filep
,
1199 unsigned int cmd
, unsigned long arg
)
1201 arg
= (unsigned long)compat_ptr(arg
);
1202 return vfio_group_fops_unl_ioctl(filep
, cmd
, arg
);
1204 #endif /* CONFIG_COMPAT */
1206 static int vfio_group_fops_open(struct inode
*inode
, struct file
*filep
)
1208 struct vfio_group
*group
;
1210 group
= vfio_group_get_from_minor(iminor(inode
));
1214 if (group
->container
) {
1215 vfio_group_put(group
);
1219 filep
->private_data
= group
;
1224 static int vfio_group_fops_release(struct inode
*inode
, struct file
*filep
)
1226 struct vfio_group
*group
= filep
->private_data
;
1228 filep
->private_data
= NULL
;
1230 vfio_group_try_dissolve_container(group
);
1232 vfio_group_put(group
);
1237 static const struct file_operations vfio_group_fops
= {
1238 .owner
= THIS_MODULE
,
1239 .unlocked_ioctl
= vfio_group_fops_unl_ioctl
,
1240 #ifdef CONFIG_COMPAT
1241 .compat_ioctl
= vfio_group_fops_compat_ioctl
,
1243 .open
= vfio_group_fops_open
,
1244 .release
= vfio_group_fops_release
,
1250 static int vfio_device_fops_release(struct inode
*inode
, struct file
*filep
)
1252 struct vfio_device
*device
= filep
->private_data
;
1254 device
->ops
->release(device
->device_data
);
1256 vfio_group_try_dissolve_container(device
->group
);
1258 vfio_device_put(device
);
1263 static long vfio_device_fops_unl_ioctl(struct file
*filep
,
1264 unsigned int cmd
, unsigned long arg
)
1266 struct vfio_device
*device
= filep
->private_data
;
1268 if (unlikely(!device
->ops
->ioctl
))
1271 return device
->ops
->ioctl(device
->device_data
, cmd
, arg
);
1274 static ssize_t
vfio_device_fops_read(struct file
*filep
, char __user
*buf
,
1275 size_t count
, loff_t
*ppos
)
1277 struct vfio_device
*device
= filep
->private_data
;
1279 if (unlikely(!device
->ops
->read
))
1282 return device
->ops
->read(device
->device_data
, buf
, count
, ppos
);
1285 static ssize_t
vfio_device_fops_write(struct file
*filep
,
1286 const char __user
*buf
,
1287 size_t count
, loff_t
*ppos
)
1289 struct vfio_device
*device
= filep
->private_data
;
1291 if (unlikely(!device
->ops
->write
))
1294 return device
->ops
->write(device
->device_data
, buf
, count
, ppos
);
1297 static int vfio_device_fops_mmap(struct file
*filep
, struct vm_area_struct
*vma
)
1299 struct vfio_device
*device
= filep
->private_data
;
1301 if (unlikely(!device
->ops
->mmap
))
1304 return device
->ops
->mmap(device
->device_data
, vma
);
1307 #ifdef CONFIG_COMPAT
1308 static long vfio_device_fops_compat_ioctl(struct file
*filep
,
1309 unsigned int cmd
, unsigned long arg
)
1311 arg
= (unsigned long)compat_ptr(arg
);
1312 return vfio_device_fops_unl_ioctl(filep
, cmd
, arg
);
1314 #endif /* CONFIG_COMPAT */
1316 static const struct file_operations vfio_device_fops
= {
1317 .owner
= THIS_MODULE
,
1318 .release
= vfio_device_fops_release
,
1319 .read
= vfio_device_fops_read
,
1320 .write
= vfio_device_fops_write
,
1321 .unlocked_ioctl
= vfio_device_fops_unl_ioctl
,
1322 #ifdef CONFIG_COMPAT
1323 .compat_ioctl
= vfio_device_fops_compat_ioctl
,
1325 .mmap
= vfio_device_fops_mmap
,
1329 * Module/class support
1331 static char *vfio_devnode(struct device
*dev
, umode_t
*mode
)
1333 return kasprintf(GFP_KERNEL
, "vfio/%s", dev_name(dev
));
1336 static int __init
vfio_init(void)
1340 idr_init(&vfio
.group_idr
);
1341 mutex_init(&vfio
.group_lock
);
1342 mutex_init(&vfio
.iommu_drivers_lock
);
1343 INIT_LIST_HEAD(&vfio
.group_list
);
1344 INIT_LIST_HEAD(&vfio
.iommu_drivers_list
);
1345 init_waitqueue_head(&vfio
.release_q
);
1347 vfio
.class = class_create(THIS_MODULE
, "vfio");
1348 if (IS_ERR(vfio
.class)) {
1349 ret
= PTR_ERR(vfio
.class);
1353 vfio
.class->devnode
= vfio_devnode
;
1355 ret
= alloc_chrdev_region(&vfio
.devt
, 0, MINORMASK
, "vfio");
1357 goto err_base_chrdev
;
1359 cdev_init(&vfio
.cdev
, &vfio_fops
);
1360 ret
= cdev_add(&vfio
.cdev
, vfio
.devt
, 1);
1364 vfio
.dev
= device_create(vfio
.class, NULL
, vfio
.devt
, NULL
, "vfio");
1365 if (IS_ERR(vfio
.dev
)) {
1366 ret
= PTR_ERR(vfio
.dev
);
1370 /* /dev/vfio/$GROUP */
1371 cdev_init(&vfio
.group_cdev
, &vfio_group_fops
);
1372 ret
= cdev_add(&vfio
.group_cdev
,
1373 MKDEV(MAJOR(vfio
.devt
), 1), MINORMASK
- 1);
1375 goto err_groups_cdev
;
1377 pr_info(DRIVER_DESC
" version: " DRIVER_VERSION
"\n");
1380 * Attempt to load known iommu-drivers. This gives us a working
1381 * environment without the user needing to explicitly load iommu
1384 request_module_nowait("vfio_iommu_type1");
1389 device_destroy(vfio
.class, vfio
.devt
);
1391 cdev_del(&vfio
.cdev
);
1393 unregister_chrdev_region(vfio
.devt
, MINORMASK
);
1395 class_destroy(vfio
.class);
1401 static void __exit
vfio_cleanup(void)
1403 WARN_ON(!list_empty(&vfio
.group_list
));
1405 idr_destroy(&vfio
.group_idr
);
1406 cdev_del(&vfio
.group_cdev
);
1407 device_destroy(vfio
.class, vfio
.devt
);
1408 cdev_del(&vfio
.cdev
);
1409 unregister_chrdev_region(vfio
.devt
, MINORMASK
);
1410 class_destroy(vfio
.class);
1414 module_init(vfio_init
);
1415 module_exit(vfio_cleanup
);
1417 MODULE_VERSION(DRIVER_VERSION
);
1418 MODULE_LICENSE("GPL v2");
1419 MODULE_AUTHOR(DRIVER_AUTHOR
);
1420 MODULE_DESCRIPTION(DRIVER_DESC
);