2 * VFIO-KVM bridge pseudo device
4 * Copyright (C) 2013 Red Hat, Inc. All rights reserved.
5 * Author: Alex Williamson <alex.williamson@redhat.com>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
12 #include <linux/errno.h>
13 #include <linux/file.h>
14 #include <linux/kvm_host.h>
15 #include <linux/list.h>
16 #include <linux/module.h>
17 #include <linux/mutex.h>
18 #include <linux/slab.h>
19 #include <linux/uaccess.h>
20 #include <linux/vfio.h>
23 #ifdef CONFIG_SPAPR_TCE_IOMMU
24 #include <asm/kvm_ppc.h>
27 struct kvm_vfio_group
{
28 struct list_head node
;
29 struct vfio_group
*vfio_group
;
33 struct list_head group_list
;
38 static struct vfio_group
*kvm_vfio_group_get_external_user(struct file
*filep
)
40 struct vfio_group
*vfio_group
;
41 struct vfio_group
*(*fn
)(struct file
*);
43 fn
= symbol_get(vfio_group_get_external_user
);
45 return ERR_PTR(-EINVAL
);
47 vfio_group
= fn(filep
);
49 symbol_put(vfio_group_get_external_user
);
54 static void kvm_vfio_group_put_external_user(struct vfio_group
*vfio_group
)
56 void (*fn
)(struct vfio_group
*);
58 fn
= symbol_get(vfio_group_put_external_user
);
64 symbol_put(vfio_group_put_external_user
);
67 static void kvm_vfio_group_set_kvm(struct vfio_group
*group
, struct kvm
*kvm
)
69 void (*fn
)(struct vfio_group
*, struct kvm
*);
71 fn
= symbol_get(vfio_group_set_kvm
);
77 symbol_put(vfio_group_set_kvm
);
80 static bool kvm_vfio_group_is_coherent(struct vfio_group
*vfio_group
)
82 long (*fn
)(struct vfio_group
*, unsigned long);
85 fn
= symbol_get(vfio_external_check_extension
);
89 ret
= fn(vfio_group
, VFIO_DMA_CC_IOMMU
);
91 symbol_put(vfio_external_check_extension
);
96 #ifdef CONFIG_SPAPR_TCE_IOMMU
97 static int kvm_vfio_external_user_iommu_id(struct vfio_group
*vfio_group
)
99 int (*fn
)(struct vfio_group
*);
102 fn
= symbol_get(vfio_external_user_iommu_id
);
106 ret
= fn(vfio_group
);
108 symbol_put(vfio_external_user_iommu_id
);
113 static struct iommu_group
*kvm_vfio_group_get_iommu_group(
114 struct vfio_group
*group
)
116 int group_id
= kvm_vfio_external_user_iommu_id(group
);
121 return iommu_group_get_by_id(group_id
);
124 static void kvm_spapr_tce_release_vfio_group(struct kvm
*kvm
,
125 struct vfio_group
*vfio_group
)
127 struct iommu_group
*grp
= kvm_vfio_group_get_iommu_group(vfio_group
);
129 if (WARN_ON_ONCE(!grp
))
132 kvm_spapr_tce_release_iommu_group(kvm
, grp
);
133 iommu_group_put(grp
);
138 * Groups can use the same or different IOMMU domains. If the same then
139 * adding a new group may change the coherency of groups we've previously
140 * been told about. We don't want to care about any of that so we retest
141 * each group and bail as soon as we find one that's noncoherent. This
142 * means we only ever [un]register_noncoherent_dma once for the whole device.
144 static void kvm_vfio_update_coherency(struct kvm_device
*dev
)
146 struct kvm_vfio
*kv
= dev
->private;
147 bool noncoherent
= false;
148 struct kvm_vfio_group
*kvg
;
150 mutex_lock(&kv
->lock
);
152 list_for_each_entry(kvg
, &kv
->group_list
, node
) {
153 if (!kvm_vfio_group_is_coherent(kvg
->vfio_group
)) {
159 if (noncoherent
!= kv
->noncoherent
) {
160 kv
->noncoherent
= noncoherent
;
163 kvm_arch_register_noncoherent_dma(dev
->kvm
);
165 kvm_arch_unregister_noncoherent_dma(dev
->kvm
);
168 mutex_unlock(&kv
->lock
);
171 static int kvm_vfio_set_group(struct kvm_device
*dev
, long attr
, u64 arg
)
173 struct kvm_vfio
*kv
= dev
->private;
174 struct vfio_group
*vfio_group
;
175 struct kvm_vfio_group
*kvg
;
176 int32_t __user
*argp
= (int32_t __user
*)(unsigned long)arg
;
182 case KVM_DEV_VFIO_GROUP_ADD
:
183 if (get_user(fd
, argp
))
190 vfio_group
= kvm_vfio_group_get_external_user(f
.file
);
193 if (IS_ERR(vfio_group
))
194 return PTR_ERR(vfio_group
);
196 mutex_lock(&kv
->lock
);
198 list_for_each_entry(kvg
, &kv
->group_list
, node
) {
199 if (kvg
->vfio_group
== vfio_group
) {
200 mutex_unlock(&kv
->lock
);
201 kvm_vfio_group_put_external_user(vfio_group
);
206 kvg
= kzalloc(sizeof(*kvg
), GFP_KERNEL
);
208 mutex_unlock(&kv
->lock
);
209 kvm_vfio_group_put_external_user(vfio_group
);
213 list_add_tail(&kvg
->node
, &kv
->group_list
);
214 kvg
->vfio_group
= vfio_group
;
216 kvm_arch_start_assignment(dev
->kvm
);
218 mutex_unlock(&kv
->lock
);
220 kvm_vfio_group_set_kvm(vfio_group
, dev
->kvm
);
222 kvm_vfio_update_coherency(dev
);
226 case KVM_DEV_VFIO_GROUP_DEL
:
227 if (get_user(fd
, argp
))
234 vfio_group
= kvm_vfio_group_get_external_user(f
.file
);
237 if (IS_ERR(vfio_group
))
238 return PTR_ERR(vfio_group
);
242 mutex_lock(&kv
->lock
);
244 list_for_each_entry(kvg
, &kv
->group_list
, node
) {
245 if (kvg
->vfio_group
!= vfio_group
)
248 list_del(&kvg
->node
);
249 kvm_vfio_group_put_external_user(kvg
->vfio_group
);
255 kvm_arch_end_assignment(dev
->kvm
);
257 mutex_unlock(&kv
->lock
);
259 #ifdef CONFIG_SPAPR_TCE_IOMMU
260 kvm_spapr_tce_release_vfio_group(dev
->kvm
, vfio_group
);
262 kvm_vfio_group_set_kvm(vfio_group
, NULL
);
264 kvm_vfio_group_put_external_user(vfio_group
);
266 kvm_vfio_update_coherency(dev
);
270 #ifdef CONFIG_SPAPR_TCE_IOMMU
271 case KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE
: {
272 struct kvm_vfio_spapr_tce param
;
273 struct kvm_vfio
*kv
= dev
->private;
274 struct vfio_group
*vfio_group
;
275 struct kvm_vfio_group
*kvg
;
277 struct iommu_group
*grp
;
279 if (copy_from_user(¶m
, (void __user
*)arg
,
280 sizeof(struct kvm_vfio_spapr_tce
)))
283 f
= fdget(param
.groupfd
);
287 vfio_group
= kvm_vfio_group_get_external_user(f
.file
);
290 if (IS_ERR(vfio_group
))
291 return PTR_ERR(vfio_group
);
293 grp
= kvm_vfio_group_get_iommu_group(vfio_group
);
294 if (WARN_ON_ONCE(!grp
)) {
295 kvm_vfio_group_put_external_user(vfio_group
);
301 mutex_lock(&kv
->lock
);
303 list_for_each_entry(kvg
, &kv
->group_list
, node
) {
304 if (kvg
->vfio_group
!= vfio_group
)
307 ret
= kvm_spapr_tce_attach_iommu_group(dev
->kvm
,
312 mutex_unlock(&kv
->lock
);
314 iommu_group_put(grp
);
315 kvm_vfio_group_put_external_user(vfio_group
);
319 #endif /* CONFIG_SPAPR_TCE_IOMMU */
325 static int kvm_vfio_set_attr(struct kvm_device
*dev
,
326 struct kvm_device_attr
*attr
)
328 switch (attr
->group
) {
329 case KVM_DEV_VFIO_GROUP
:
330 return kvm_vfio_set_group(dev
, attr
->attr
, attr
->addr
);
336 static int kvm_vfio_has_attr(struct kvm_device
*dev
,
337 struct kvm_device_attr
*attr
)
339 switch (attr
->group
) {
340 case KVM_DEV_VFIO_GROUP
:
341 switch (attr
->attr
) {
342 case KVM_DEV_VFIO_GROUP_ADD
:
343 case KVM_DEV_VFIO_GROUP_DEL
:
344 #ifdef CONFIG_SPAPR_TCE_IOMMU
345 case KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE
:
356 static void kvm_vfio_destroy(struct kvm_device
*dev
)
358 struct kvm_vfio
*kv
= dev
->private;
359 struct kvm_vfio_group
*kvg
, *tmp
;
361 list_for_each_entry_safe(kvg
, tmp
, &kv
->group_list
, node
) {
362 #ifdef CONFIG_SPAPR_TCE_IOMMU
363 kvm_spapr_tce_release_vfio_group(dev
->kvm
, kvg
->vfio_group
);
365 kvm_vfio_group_set_kvm(kvg
->vfio_group
, NULL
);
366 kvm_vfio_group_put_external_user(kvg
->vfio_group
);
367 list_del(&kvg
->node
);
369 kvm_arch_end_assignment(dev
->kvm
);
372 kvm_vfio_update_coherency(dev
);
375 kfree(dev
); /* alloc by kvm_ioctl_create_device, free by .destroy */
378 static int kvm_vfio_create(struct kvm_device
*dev
, u32 type
);
380 static struct kvm_device_ops kvm_vfio_ops
= {
382 .create
= kvm_vfio_create
,
383 .destroy
= kvm_vfio_destroy
,
384 .set_attr
= kvm_vfio_set_attr
,
385 .has_attr
= kvm_vfio_has_attr
,
388 static int kvm_vfio_create(struct kvm_device
*dev
, u32 type
)
390 struct kvm_device
*tmp
;
393 /* Only one VFIO "device" per VM */
394 list_for_each_entry(tmp
, &dev
->kvm
->devices
, vm_node
)
395 if (tmp
->ops
== &kvm_vfio_ops
)
398 kv
= kzalloc(sizeof(*kv
), GFP_KERNEL
);
402 INIT_LIST_HEAD(&kv
->group_list
);
403 mutex_init(&kv
->lock
);
410 int kvm_vfio_ops_init(void)
412 return kvm_register_device_ops(&kvm_vfio_ops
, KVM_DEV_TYPE_VFIO
);
415 void kvm_vfio_ops_exit(void)
417 kvm_unregister_device_ops(KVM_DEV_TYPE_VFIO
);