2 * KVMGT - the implementation of Intel mediated pass-through framework for KVM
4 * Copyright(c) 2014-2016 Intel Corporation. All rights reserved.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26 * Kevin Tian <kevin.tian@intel.com>
27 * Jike Song <jike.song@intel.com>
28 * Xiaoguang Chen <xiaoguang.chen@intel.com>
31 #include <linux/init.h>
32 #include <linux/device.h>
34 #include <linux/mmu_context.h>
35 #include <linux/types.h>
36 #include <linux/list.h>
37 #include <linux/rbtree.h>
38 #include <linux/spinlock.h>
39 #include <linux/eventfd.h>
40 #include <linux/uuid.h>
41 #include <linux/kvm_host.h>
42 #include <linux/vfio.h>
43 #include <linux/mdev.h>
48 static const struct intel_gvt_ops
*intel_gvt_ops
;
50 /* helper macros copied from vfio-pci */
51 #define VFIO_PCI_OFFSET_SHIFT 40
52 #define VFIO_PCI_OFFSET_TO_INDEX(off) (off >> VFIO_PCI_OFFSET_SHIFT)
53 #define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT)
54 #define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1)
65 struct hlist_node hnode
;
68 struct kvmgt_guest_info
{
70 struct intel_vgpu
*vgpu
;
71 struct kvm_page_track_notifier_node track_node
;
72 #define NR_BKT (1 << 18)
73 struct hlist_head ptable
[NR_BKT
];
83 static inline bool handle_valid(unsigned long handle
)
85 return !!(handle
& ~0xff);
88 static int kvmgt_guest_init(struct mdev_device
*mdev
);
89 static void intel_vgpu_release_work(struct work_struct
*work
);
90 static bool kvmgt_guest_exit(struct kvmgt_guest_info
*info
);
92 static int gvt_dma_map_iova(struct intel_vgpu
*vgpu
, kvm_pfn_t pfn
,
96 struct device
*dev
= &vgpu
->gvt
->dev_priv
->drm
.pdev
->dev
;
99 if (unlikely(!pfn_valid(pfn
)))
102 page
= pfn_to_page(pfn
);
103 daddr
= dma_map_page(dev
, page
, 0, PAGE_SIZE
,
104 PCI_DMA_BIDIRECTIONAL
);
105 if (dma_mapping_error(dev
, daddr
))
108 *iova
= (unsigned long)(daddr
>> PAGE_SHIFT
);
112 static void gvt_dma_unmap_iova(struct intel_vgpu
*vgpu
, unsigned long iova
)
114 struct device
*dev
= &vgpu
->gvt
->dev_priv
->drm
.pdev
->dev
;
117 daddr
= (dma_addr_t
)(iova
<< PAGE_SHIFT
);
118 dma_unmap_page(dev
, daddr
, PAGE_SIZE
, PCI_DMA_BIDIRECTIONAL
);
121 static struct gvt_dma
*__gvt_cache_find(struct intel_vgpu
*vgpu
, gfn_t gfn
)
123 struct rb_node
*node
= vgpu
->vdev
.cache
.rb_node
;
124 struct gvt_dma
*ret
= NULL
;
127 struct gvt_dma
*itr
= rb_entry(node
, struct gvt_dma
, node
);
130 node
= node
->rb_left
;
131 else if (gfn
> itr
->gfn
)
132 node
= node
->rb_right
;
143 static unsigned long gvt_cache_find(struct intel_vgpu
*vgpu
, gfn_t gfn
)
145 struct gvt_dma
*entry
;
148 mutex_lock(&vgpu
->vdev
.cache_lock
);
150 entry
= __gvt_cache_find(vgpu
, gfn
);
151 iova
= (entry
== NULL
) ? INTEL_GVT_INVALID_ADDR
: entry
->iova
;
153 mutex_unlock(&vgpu
->vdev
.cache_lock
);
157 static void gvt_cache_add(struct intel_vgpu
*vgpu
, gfn_t gfn
,
160 struct gvt_dma
*new, *itr
;
161 struct rb_node
**link
= &vgpu
->vdev
.cache
.rb_node
, *parent
= NULL
;
163 new = kzalloc(sizeof(struct gvt_dma
), GFP_KERNEL
);
170 mutex_lock(&vgpu
->vdev
.cache_lock
);
173 itr
= rb_entry(parent
, struct gvt_dma
, node
);
177 else if (gfn
< itr
->gfn
)
178 link
= &parent
->rb_left
;
180 link
= &parent
->rb_right
;
183 rb_link_node(&new->node
, parent
, link
);
184 rb_insert_color(&new->node
, &vgpu
->vdev
.cache
);
185 mutex_unlock(&vgpu
->vdev
.cache_lock
);
189 mutex_unlock(&vgpu
->vdev
.cache_lock
);
193 static void __gvt_cache_remove_entry(struct intel_vgpu
*vgpu
,
194 struct gvt_dma
*entry
)
196 rb_erase(&entry
->node
, &vgpu
->vdev
.cache
);
200 static void gvt_cache_remove(struct intel_vgpu
*vgpu
, gfn_t gfn
)
202 struct device
*dev
= mdev_dev(vgpu
->vdev
.mdev
);
203 struct gvt_dma
*this;
207 mutex_lock(&vgpu
->vdev
.cache_lock
);
208 this = __gvt_cache_find(vgpu
, gfn
);
210 mutex_unlock(&vgpu
->vdev
.cache_lock
);
215 gvt_dma_unmap_iova(vgpu
, this->iova
);
216 rc
= vfio_unpin_pages(dev
, &g1
, 1);
218 __gvt_cache_remove_entry(vgpu
, this);
219 mutex_unlock(&vgpu
->vdev
.cache_lock
);
222 static void gvt_cache_init(struct intel_vgpu
*vgpu
)
224 vgpu
->vdev
.cache
= RB_ROOT
;
225 mutex_init(&vgpu
->vdev
.cache_lock
);
228 static void gvt_cache_destroy(struct intel_vgpu
*vgpu
)
231 struct rb_node
*node
= NULL
;
232 struct device
*dev
= mdev_dev(vgpu
->vdev
.mdev
);
235 mutex_lock(&vgpu
->vdev
.cache_lock
);
236 while ((node
= rb_first(&vgpu
->vdev
.cache
))) {
237 dma
= rb_entry(node
, struct gvt_dma
, node
);
238 gvt_dma_unmap_iova(vgpu
, dma
->iova
);
241 vfio_unpin_pages(dev
, &gfn
, 1);
242 __gvt_cache_remove_entry(vgpu
, dma
);
244 mutex_unlock(&vgpu
->vdev
.cache_lock
);
247 static struct intel_vgpu_type
*intel_gvt_find_vgpu_type(struct intel_gvt
*gvt
,
251 struct intel_vgpu_type
*t
;
252 const char *driver_name
= dev_driver_string(
253 &gvt
->dev_priv
->drm
.pdev
->dev
);
255 for (i
= 0; i
< gvt
->num_types
; i
++) {
257 if (!strncmp(t
->name
, name
+ strlen(driver_name
) + 1,
265 static ssize_t
available_instances_show(struct kobject
*kobj
,
266 struct device
*dev
, char *buf
)
268 struct intel_vgpu_type
*type
;
269 unsigned int num
= 0;
270 void *gvt
= kdev_to_i915(dev
)->gvt
;
272 type
= intel_gvt_find_vgpu_type(gvt
, kobject_name(kobj
));
276 num
= type
->avail_instance
;
278 return sprintf(buf
, "%u\n", num
);
281 static ssize_t
device_api_show(struct kobject
*kobj
, struct device
*dev
,
284 return sprintf(buf
, "%s\n", VFIO_DEVICE_API_PCI_STRING
);
287 static ssize_t
description_show(struct kobject
*kobj
, struct device
*dev
,
290 struct intel_vgpu_type
*type
;
291 void *gvt
= kdev_to_i915(dev
)->gvt
;
293 type
= intel_gvt_find_vgpu_type(gvt
, kobject_name(kobj
));
297 return sprintf(buf
, "low_gm_size: %dMB\nhigh_gm_size: %dMB\n"
298 "fence: %d\nresolution: %s\n",
299 BYTES_TO_MB(type
->low_gm_size
),
300 BYTES_TO_MB(type
->high_gm_size
),
301 type
->fence
, vgpu_edid_str(type
->resolution
));
304 static MDEV_TYPE_ATTR_RO(available_instances
);
305 static MDEV_TYPE_ATTR_RO(device_api
);
306 static MDEV_TYPE_ATTR_RO(description
);
308 static struct attribute
*type_attrs
[] = {
309 &mdev_type_attr_available_instances
.attr
,
310 &mdev_type_attr_device_api
.attr
,
311 &mdev_type_attr_description
.attr
,
315 static struct attribute_group
*intel_vgpu_type_groups
[] = {
316 [0 ... NR_MAX_INTEL_VGPU_TYPES
- 1] = NULL
,
319 static bool intel_gvt_init_vgpu_type_groups(struct intel_gvt
*gvt
)
322 struct intel_vgpu_type
*type
;
323 struct attribute_group
*group
;
325 for (i
= 0; i
< gvt
->num_types
; i
++) {
326 type
= &gvt
->types
[i
];
328 group
= kzalloc(sizeof(struct attribute_group
), GFP_KERNEL
);
332 group
->name
= type
->name
;
333 group
->attrs
= type_attrs
;
334 intel_vgpu_type_groups
[i
] = group
;
340 for (j
= 0; j
< i
; j
++) {
341 group
= intel_vgpu_type_groups
[j
];
348 static void intel_gvt_cleanup_vgpu_type_groups(struct intel_gvt
*gvt
)
351 struct attribute_group
*group
;
353 for (i
= 0; i
< gvt
->num_types
; i
++) {
354 group
= intel_vgpu_type_groups
[i
];
359 static void kvmgt_protect_table_init(struct kvmgt_guest_info
*info
)
361 hash_init(info
->ptable
);
364 static void kvmgt_protect_table_destroy(struct kvmgt_guest_info
*info
)
366 struct kvmgt_pgfn
*p
;
367 struct hlist_node
*tmp
;
370 hash_for_each_safe(info
->ptable
, i
, tmp
, p
, hnode
) {
376 static struct kvmgt_pgfn
*
377 __kvmgt_protect_table_find(struct kvmgt_guest_info
*info
, gfn_t gfn
)
379 struct kvmgt_pgfn
*p
, *res
= NULL
;
381 hash_for_each_possible(info
->ptable
, p
, hnode
, gfn
) {
391 static bool kvmgt_gfn_is_write_protected(struct kvmgt_guest_info
*info
,
394 struct kvmgt_pgfn
*p
;
396 p
= __kvmgt_protect_table_find(info
, gfn
);
400 static void kvmgt_protect_table_add(struct kvmgt_guest_info
*info
, gfn_t gfn
)
402 struct kvmgt_pgfn
*p
;
404 if (kvmgt_gfn_is_write_protected(info
, gfn
))
407 p
= kzalloc(sizeof(struct kvmgt_pgfn
), GFP_ATOMIC
);
408 if (WARN(!p
, "gfn: 0x%llx\n", gfn
))
412 hash_add(info
->ptable
, &p
->hnode
, gfn
);
415 static void kvmgt_protect_table_del(struct kvmgt_guest_info
*info
,
418 struct kvmgt_pgfn
*p
;
420 p
= __kvmgt_protect_table_find(info
, gfn
);
427 static int intel_vgpu_create(struct kobject
*kobj
, struct mdev_device
*mdev
)
429 struct intel_vgpu
*vgpu
= NULL
;
430 struct intel_vgpu_type
*type
;
435 pdev
= mdev_parent_dev(mdev
);
436 gvt
= kdev_to_i915(pdev
)->gvt
;
438 type
= intel_gvt_find_vgpu_type(gvt
, kobject_name(kobj
));
440 gvt_vgpu_err("failed to find type %s to create\n",
446 vgpu
= intel_gvt_ops
->vgpu_create(gvt
, type
);
447 if (IS_ERR_OR_NULL(vgpu
)) {
448 ret
= vgpu
== NULL
? -EFAULT
: PTR_ERR(vgpu
);
449 gvt_vgpu_err("failed to create intel vgpu: %d\n", ret
);
453 INIT_WORK(&vgpu
->vdev
.release_work
, intel_vgpu_release_work
);
455 vgpu
->vdev
.mdev
= mdev
;
456 mdev_set_drvdata(mdev
, vgpu
);
458 gvt_dbg_core("intel_vgpu_create succeeded for mdev: %s\n",
459 dev_name(mdev_dev(mdev
)));
466 static int intel_vgpu_remove(struct mdev_device
*mdev
)
468 struct intel_vgpu
*vgpu
= mdev_get_drvdata(mdev
);
470 if (handle_valid(vgpu
->handle
))
473 intel_gvt_ops
->vgpu_destroy(vgpu
);
477 static int intel_vgpu_iommu_notifier(struct notifier_block
*nb
,
478 unsigned long action
, void *data
)
480 struct intel_vgpu
*vgpu
= container_of(nb
,
482 vdev
.iommu_notifier
);
484 if (action
== VFIO_IOMMU_NOTIFY_DMA_UNMAP
) {
485 struct vfio_iommu_type1_dma_unmap
*unmap
= data
;
486 unsigned long gfn
, end_gfn
;
488 gfn
= unmap
->iova
>> PAGE_SHIFT
;
489 end_gfn
= gfn
+ unmap
->size
/ PAGE_SIZE
;
491 while (gfn
< end_gfn
)
492 gvt_cache_remove(vgpu
, gfn
++);
498 static int intel_vgpu_group_notifier(struct notifier_block
*nb
,
499 unsigned long action
, void *data
)
501 struct intel_vgpu
*vgpu
= container_of(nb
,
503 vdev
.group_notifier
);
505 /* the only action we care about */
506 if (action
== VFIO_GROUP_NOTIFY_SET_KVM
) {
507 vgpu
->vdev
.kvm
= data
;
510 schedule_work(&vgpu
->vdev
.release_work
);
516 static int intel_vgpu_open(struct mdev_device
*mdev
)
518 struct intel_vgpu
*vgpu
= mdev_get_drvdata(mdev
);
519 unsigned long events
;
522 vgpu
->vdev
.iommu_notifier
.notifier_call
= intel_vgpu_iommu_notifier
;
523 vgpu
->vdev
.group_notifier
.notifier_call
= intel_vgpu_group_notifier
;
525 events
= VFIO_IOMMU_NOTIFY_DMA_UNMAP
;
526 ret
= vfio_register_notifier(mdev_dev(mdev
), VFIO_IOMMU_NOTIFY
, &events
,
527 &vgpu
->vdev
.iommu_notifier
);
529 gvt_vgpu_err("vfio_register_notifier for iommu failed: %d\n",
534 events
= VFIO_GROUP_NOTIFY_SET_KVM
;
535 ret
= vfio_register_notifier(mdev_dev(mdev
), VFIO_GROUP_NOTIFY
, &events
,
536 &vgpu
->vdev
.group_notifier
);
538 gvt_vgpu_err("vfio_register_notifier for group failed: %d\n",
543 ret
= kvmgt_guest_init(mdev
);
547 intel_gvt_ops
->vgpu_activate(vgpu
);
549 atomic_set(&vgpu
->vdev
.released
, 0);
553 vfio_unregister_notifier(mdev_dev(mdev
), VFIO_GROUP_NOTIFY
,
554 &vgpu
->vdev
.group_notifier
);
557 vfio_unregister_notifier(mdev_dev(mdev
), VFIO_IOMMU_NOTIFY
,
558 &vgpu
->vdev
.iommu_notifier
);
563 static void __intel_vgpu_release(struct intel_vgpu
*vgpu
)
565 struct kvmgt_guest_info
*info
;
568 if (!handle_valid(vgpu
->handle
))
571 if (atomic_cmpxchg(&vgpu
->vdev
.released
, 0, 1))
574 intel_gvt_ops
->vgpu_deactivate(vgpu
);
576 ret
= vfio_unregister_notifier(mdev_dev(vgpu
->vdev
.mdev
), VFIO_IOMMU_NOTIFY
,
577 &vgpu
->vdev
.iommu_notifier
);
578 WARN(ret
, "vfio_unregister_notifier for iommu failed: %d\n", ret
);
580 ret
= vfio_unregister_notifier(mdev_dev(vgpu
->vdev
.mdev
), VFIO_GROUP_NOTIFY
,
581 &vgpu
->vdev
.group_notifier
);
582 WARN(ret
, "vfio_unregister_notifier for group failed: %d\n", ret
);
584 info
= (struct kvmgt_guest_info
*)vgpu
->handle
;
585 kvmgt_guest_exit(info
);
587 vgpu
->vdev
.kvm
= NULL
;
591 static void intel_vgpu_release(struct mdev_device
*mdev
)
593 struct intel_vgpu
*vgpu
= mdev_get_drvdata(mdev
);
595 __intel_vgpu_release(vgpu
);
598 static void intel_vgpu_release_work(struct work_struct
*work
)
600 struct intel_vgpu
*vgpu
= container_of(work
, struct intel_vgpu
,
603 __intel_vgpu_release(vgpu
);
606 static uint64_t intel_vgpu_get_bar0_addr(struct intel_vgpu
*vgpu
)
608 u32 start_lo
, start_hi
;
610 int pos
= PCI_BASE_ADDRESS_0
;
612 start_lo
= (*(u32
*)(vgpu
->cfg_space
.virtual_cfg_space
+ pos
)) &
613 PCI_BASE_ADDRESS_MEM_MASK
;
614 mem_type
= (*(u32
*)(vgpu
->cfg_space
.virtual_cfg_space
+ pos
)) &
615 PCI_BASE_ADDRESS_MEM_TYPE_MASK
;
618 case PCI_BASE_ADDRESS_MEM_TYPE_64
:
619 start_hi
= (*(u32
*)(vgpu
->cfg_space
.virtual_cfg_space
622 case PCI_BASE_ADDRESS_MEM_TYPE_32
:
623 case PCI_BASE_ADDRESS_MEM_TYPE_1M
:
624 /* 1M mem BAR treated as 32-bit BAR */
626 /* mem unknown type treated as 32-bit BAR */
631 return ((u64
)start_hi
<< 32) | start_lo
;
634 static ssize_t
intel_vgpu_rw(struct mdev_device
*mdev
, char *buf
,
635 size_t count
, loff_t
*ppos
, bool is_write
)
637 struct intel_vgpu
*vgpu
= mdev_get_drvdata(mdev
);
638 unsigned int index
= VFIO_PCI_OFFSET_TO_INDEX(*ppos
);
639 uint64_t pos
= *ppos
& VFIO_PCI_OFFSET_MASK
;
643 if (index
>= VFIO_PCI_NUM_REGIONS
) {
644 gvt_vgpu_err("invalid index: %u\n", index
);
649 case VFIO_PCI_CONFIG_REGION_INDEX
:
651 ret
= intel_gvt_ops
->emulate_cfg_write(vgpu
, pos
,
654 ret
= intel_gvt_ops
->emulate_cfg_read(vgpu
, pos
,
657 case VFIO_PCI_BAR0_REGION_INDEX
:
658 case VFIO_PCI_BAR1_REGION_INDEX
:
660 uint64_t bar0_start
= intel_vgpu_get_bar0_addr(vgpu
);
662 ret
= intel_gvt_ops
->emulate_mmio_write(vgpu
,
663 bar0_start
+ pos
, buf
, count
);
665 uint64_t bar0_start
= intel_vgpu_get_bar0_addr(vgpu
);
667 ret
= intel_gvt_ops
->emulate_mmio_read(vgpu
,
668 bar0_start
+ pos
, buf
, count
);
671 case VFIO_PCI_BAR2_REGION_INDEX
:
672 case VFIO_PCI_BAR3_REGION_INDEX
:
673 case VFIO_PCI_BAR4_REGION_INDEX
:
674 case VFIO_PCI_BAR5_REGION_INDEX
:
675 case VFIO_PCI_VGA_REGION_INDEX
:
676 case VFIO_PCI_ROM_REGION_INDEX
:
678 gvt_vgpu_err("unsupported region: %u\n", index
);
681 return ret
== 0 ? count
: ret
;
684 static ssize_t
intel_vgpu_read(struct mdev_device
*mdev
, char __user
*buf
,
685 size_t count
, loff_t
*ppos
)
687 unsigned int done
= 0;
693 if (count
>= 4 && !(*ppos
% 4)) {
696 ret
= intel_vgpu_rw(mdev
, (char *)&val
, sizeof(val
),
701 if (copy_to_user(buf
, &val
, sizeof(val
)))
705 } else if (count
>= 2 && !(*ppos
% 2)) {
708 ret
= intel_vgpu_rw(mdev
, (char *)&val
, sizeof(val
),
713 if (copy_to_user(buf
, &val
, sizeof(val
)))
720 ret
= intel_vgpu_rw(mdev
, &val
, sizeof(val
), ppos
,
725 if (copy_to_user(buf
, &val
, sizeof(val
)))
743 static ssize_t
intel_vgpu_write(struct mdev_device
*mdev
,
744 const char __user
*buf
,
745 size_t count
, loff_t
*ppos
)
747 unsigned int done
= 0;
753 if (count
>= 4 && !(*ppos
% 4)) {
756 if (copy_from_user(&val
, buf
, sizeof(val
)))
759 ret
= intel_vgpu_rw(mdev
, (char *)&val
, sizeof(val
),
765 } else if (count
>= 2 && !(*ppos
% 2)) {
768 if (copy_from_user(&val
, buf
, sizeof(val
)))
771 ret
= intel_vgpu_rw(mdev
, (char *)&val
,
772 sizeof(val
), ppos
, true);
780 if (copy_from_user(&val
, buf
, sizeof(val
)))
783 ret
= intel_vgpu_rw(mdev
, &val
, sizeof(val
),
802 static int intel_vgpu_mmap(struct mdev_device
*mdev
, struct vm_area_struct
*vma
)
806 unsigned long req_size
, pgoff
= 0;
808 struct intel_vgpu
*vgpu
= mdev_get_drvdata(mdev
);
810 index
= vma
->vm_pgoff
>> (VFIO_PCI_OFFSET_SHIFT
- PAGE_SHIFT
);
811 if (index
>= VFIO_PCI_ROM_REGION_INDEX
)
814 if (vma
->vm_end
< vma
->vm_start
)
816 if ((vma
->vm_flags
& VM_SHARED
) == 0)
818 if (index
!= VFIO_PCI_BAR2_REGION_INDEX
)
821 pg_prot
= vma
->vm_page_prot
;
822 virtaddr
= vma
->vm_start
;
823 req_size
= vma
->vm_end
- vma
->vm_start
;
824 pgoff
= vgpu_aperture_pa_base(vgpu
) >> PAGE_SHIFT
;
826 return remap_pfn_range(vma
, virtaddr
, pgoff
, req_size
, pg_prot
);
829 static int intel_vgpu_get_irq_count(struct intel_vgpu
*vgpu
, int type
)
831 if (type
== VFIO_PCI_INTX_IRQ_INDEX
|| type
== VFIO_PCI_MSI_IRQ_INDEX
)
837 static int intel_vgpu_set_intx_mask(struct intel_vgpu
*vgpu
,
838 unsigned int index
, unsigned int start
,
839 unsigned int count
, uint32_t flags
,
845 static int intel_vgpu_set_intx_unmask(struct intel_vgpu
*vgpu
,
846 unsigned int index
, unsigned int start
,
847 unsigned int count
, uint32_t flags
, void *data
)
852 static int intel_vgpu_set_intx_trigger(struct intel_vgpu
*vgpu
,
853 unsigned int index
, unsigned int start
, unsigned int count
,
854 uint32_t flags
, void *data
)
859 static int intel_vgpu_set_msi_trigger(struct intel_vgpu
*vgpu
,
860 unsigned int index
, unsigned int start
, unsigned int count
,
861 uint32_t flags
, void *data
)
863 struct eventfd_ctx
*trigger
;
865 if (flags
& VFIO_IRQ_SET_DATA_EVENTFD
) {
866 int fd
= *(int *)data
;
868 trigger
= eventfd_ctx_fdget(fd
);
869 if (IS_ERR(trigger
)) {
870 gvt_vgpu_err("eventfd_ctx_fdget failed\n");
871 return PTR_ERR(trigger
);
873 vgpu
->vdev
.msi_trigger
= trigger
;
879 static int intel_vgpu_set_irqs(struct intel_vgpu
*vgpu
, uint32_t flags
,
880 unsigned int index
, unsigned int start
, unsigned int count
,
883 int (*func
)(struct intel_vgpu
*vgpu
, unsigned int index
,
884 unsigned int start
, unsigned int count
, uint32_t flags
,
888 case VFIO_PCI_INTX_IRQ_INDEX
:
889 switch (flags
& VFIO_IRQ_SET_ACTION_TYPE_MASK
) {
890 case VFIO_IRQ_SET_ACTION_MASK
:
891 func
= intel_vgpu_set_intx_mask
;
893 case VFIO_IRQ_SET_ACTION_UNMASK
:
894 func
= intel_vgpu_set_intx_unmask
;
896 case VFIO_IRQ_SET_ACTION_TRIGGER
:
897 func
= intel_vgpu_set_intx_trigger
;
901 case VFIO_PCI_MSI_IRQ_INDEX
:
902 switch (flags
& VFIO_IRQ_SET_ACTION_TYPE_MASK
) {
903 case VFIO_IRQ_SET_ACTION_MASK
:
904 case VFIO_IRQ_SET_ACTION_UNMASK
:
905 /* XXX Need masking support exported */
907 case VFIO_IRQ_SET_ACTION_TRIGGER
:
908 func
= intel_vgpu_set_msi_trigger
;
917 return func(vgpu
, index
, start
, count
, flags
, data
);
920 static long intel_vgpu_ioctl(struct mdev_device
*mdev
, unsigned int cmd
,
923 struct intel_vgpu
*vgpu
= mdev_get_drvdata(mdev
);
926 gvt_dbg_core("vgpu%d ioctl, cmd: %d\n", vgpu
->id
, cmd
);
928 if (cmd
== VFIO_DEVICE_GET_INFO
) {
929 struct vfio_device_info info
;
931 minsz
= offsetofend(struct vfio_device_info
, num_irqs
);
933 if (copy_from_user(&info
, (void __user
*)arg
, minsz
))
936 if (info
.argsz
< minsz
)
939 info
.flags
= VFIO_DEVICE_FLAGS_PCI
;
940 info
.flags
|= VFIO_DEVICE_FLAGS_RESET
;
941 info
.num_regions
= VFIO_PCI_NUM_REGIONS
;
942 info
.num_irqs
= VFIO_PCI_NUM_IRQS
;
944 return copy_to_user((void __user
*)arg
, &info
, minsz
) ?
947 } else if (cmd
== VFIO_DEVICE_GET_REGION_INFO
) {
948 struct vfio_region_info info
;
949 struct vfio_info_cap caps
= { .buf
= NULL
, .size
= 0 };
951 struct vfio_region_info_cap_sparse_mmap
*sparse
= NULL
;
956 minsz
= offsetofend(struct vfio_region_info
, offset
);
958 if (copy_from_user(&info
, (void __user
*)arg
, minsz
))
961 if (info
.argsz
< minsz
)
964 switch (info
.index
) {
965 case VFIO_PCI_CONFIG_REGION_INDEX
:
966 info
.offset
= VFIO_PCI_INDEX_TO_OFFSET(info
.index
);
967 info
.size
= INTEL_GVT_MAX_CFG_SPACE_SZ
;
968 info
.flags
= VFIO_REGION_INFO_FLAG_READ
|
969 VFIO_REGION_INFO_FLAG_WRITE
;
971 case VFIO_PCI_BAR0_REGION_INDEX
:
972 info
.offset
= VFIO_PCI_INDEX_TO_OFFSET(info
.index
);
973 info
.size
= vgpu
->cfg_space
.bar
[info
.index
].size
;
979 info
.flags
= VFIO_REGION_INFO_FLAG_READ
|
980 VFIO_REGION_INFO_FLAG_WRITE
;
982 case VFIO_PCI_BAR1_REGION_INDEX
:
983 info
.offset
= VFIO_PCI_INDEX_TO_OFFSET(info
.index
);
987 case VFIO_PCI_BAR2_REGION_INDEX
:
988 info
.offset
= VFIO_PCI_INDEX_TO_OFFSET(info
.index
);
989 info
.flags
= VFIO_REGION_INFO_FLAG_CAPS
|
990 VFIO_REGION_INFO_FLAG_MMAP
|
991 VFIO_REGION_INFO_FLAG_READ
|
992 VFIO_REGION_INFO_FLAG_WRITE
;
993 info
.size
= gvt_aperture_sz(vgpu
->gvt
);
995 size
= sizeof(*sparse
) +
996 (nr_areas
* sizeof(*sparse
->areas
));
997 sparse
= kzalloc(size
, GFP_KERNEL
);
1001 sparse
->nr_areas
= nr_areas
;
1002 cap_type_id
= VFIO_REGION_INFO_CAP_SPARSE_MMAP
;
1003 sparse
->areas
[0].offset
=
1004 PAGE_ALIGN(vgpu_aperture_offset(vgpu
));
1005 sparse
->areas
[0].size
= vgpu_aperture_sz(vgpu
);
1008 case VFIO_PCI_BAR3_REGION_INDEX
... VFIO_PCI_BAR5_REGION_INDEX
:
1009 info
.offset
= VFIO_PCI_INDEX_TO_OFFSET(info
.index
);
1013 gvt_dbg_core("get region info bar:%d\n", info
.index
);
1016 case VFIO_PCI_ROM_REGION_INDEX
:
1017 case VFIO_PCI_VGA_REGION_INDEX
:
1018 gvt_dbg_core("get region info index:%d\n", info
.index
);
1022 struct vfio_region_info_cap_type cap_type
;
1024 if (info
.index
>= VFIO_PCI_NUM_REGIONS
+
1025 vgpu
->vdev
.num_regions
)
1028 i
= info
.index
- VFIO_PCI_NUM_REGIONS
;
1031 VFIO_PCI_INDEX_TO_OFFSET(info
.index
);
1032 info
.size
= vgpu
->vdev
.region
[i
].size
;
1033 info
.flags
= vgpu
->vdev
.region
[i
].flags
;
1035 cap_type
.type
= vgpu
->vdev
.region
[i
].type
;
1036 cap_type
.subtype
= vgpu
->vdev
.region
[i
].subtype
;
1038 ret
= vfio_info_add_capability(&caps
,
1039 VFIO_REGION_INFO_CAP_TYPE
,
1046 if ((info
.flags
& VFIO_REGION_INFO_FLAG_CAPS
) && sparse
) {
1047 switch (cap_type_id
) {
1048 case VFIO_REGION_INFO_CAP_SPARSE_MMAP
:
1049 ret
= vfio_info_add_capability(&caps
,
1050 VFIO_REGION_INFO_CAP_SPARSE_MMAP
,
1062 if (info
.argsz
< sizeof(info
) + caps
.size
) {
1063 info
.argsz
= sizeof(info
) + caps
.size
;
1064 info
.cap_offset
= 0;
1066 vfio_info_cap_shift(&caps
, sizeof(info
));
1067 if (copy_to_user((void __user
*)arg
+
1068 sizeof(info
), caps
.buf
,
1073 info
.cap_offset
= sizeof(info
);
1079 return copy_to_user((void __user
*)arg
, &info
, minsz
) ?
1081 } else if (cmd
== VFIO_DEVICE_GET_IRQ_INFO
) {
1082 struct vfio_irq_info info
;
1084 minsz
= offsetofend(struct vfio_irq_info
, count
);
1086 if (copy_from_user(&info
, (void __user
*)arg
, minsz
))
1089 if (info
.argsz
< minsz
|| info
.index
>= VFIO_PCI_NUM_IRQS
)
1092 switch (info
.index
) {
1093 case VFIO_PCI_INTX_IRQ_INDEX
:
1094 case VFIO_PCI_MSI_IRQ_INDEX
:
1100 info
.flags
= VFIO_IRQ_INFO_EVENTFD
;
1102 info
.count
= intel_vgpu_get_irq_count(vgpu
, info
.index
);
1104 if (info
.index
== VFIO_PCI_INTX_IRQ_INDEX
)
1105 info
.flags
|= (VFIO_IRQ_INFO_MASKABLE
|
1106 VFIO_IRQ_INFO_AUTOMASKED
);
1108 info
.flags
|= VFIO_IRQ_INFO_NORESIZE
;
1110 return copy_to_user((void __user
*)arg
, &info
, minsz
) ?
1112 } else if (cmd
== VFIO_DEVICE_SET_IRQS
) {
1113 struct vfio_irq_set hdr
;
1116 size_t data_size
= 0;
1118 minsz
= offsetofend(struct vfio_irq_set
, count
);
1120 if (copy_from_user(&hdr
, (void __user
*)arg
, minsz
))
1123 if (!(hdr
.flags
& VFIO_IRQ_SET_DATA_NONE
)) {
1124 int max
= intel_vgpu_get_irq_count(vgpu
, hdr
.index
);
1126 ret
= vfio_set_irqs_validate_and_prepare(&hdr
, max
,
1127 VFIO_PCI_NUM_IRQS
, &data_size
);
1129 gvt_vgpu_err("intel:vfio_set_irqs_validate_and_prepare failed\n");
1133 data
= memdup_user((void __user
*)(arg
+ minsz
),
1136 return PTR_ERR(data
);
1140 ret
= intel_vgpu_set_irqs(vgpu
, hdr
.flags
, hdr
.index
,
1141 hdr
.start
, hdr
.count
, data
);
1145 } else if (cmd
== VFIO_DEVICE_RESET
) {
1146 intel_gvt_ops
->vgpu_reset(vgpu
);
1153 static const struct mdev_parent_ops intel_vgpu_ops
= {
1154 .supported_type_groups
= intel_vgpu_type_groups
,
1155 .create
= intel_vgpu_create
,
1156 .remove
= intel_vgpu_remove
,
1158 .open
= intel_vgpu_open
,
1159 .release
= intel_vgpu_release
,
1161 .read
= intel_vgpu_read
,
1162 .write
= intel_vgpu_write
,
1163 .mmap
= intel_vgpu_mmap
,
1164 .ioctl
= intel_vgpu_ioctl
,
1167 static int kvmgt_host_init(struct device
*dev
, void *gvt
, const void *ops
)
1169 if (!intel_gvt_init_vgpu_type_groups(gvt
))
1172 intel_gvt_ops
= ops
;
1174 return mdev_register_device(dev
, &intel_vgpu_ops
);
1177 static void kvmgt_host_exit(struct device
*dev
, void *gvt
)
1179 intel_gvt_cleanup_vgpu_type_groups(gvt
);
1180 mdev_unregister_device(dev
);
1183 static int kvmgt_write_protect_add(unsigned long handle
, u64 gfn
)
1185 struct kvmgt_guest_info
*info
;
1187 struct kvm_memory_slot
*slot
;
1190 if (!handle_valid(handle
))
1193 info
= (struct kvmgt_guest_info
*)handle
;
1196 idx
= srcu_read_lock(&kvm
->srcu
);
1197 slot
= gfn_to_memslot(kvm
, gfn
);
1199 srcu_read_unlock(&kvm
->srcu
, idx
);
1203 spin_lock(&kvm
->mmu_lock
);
1205 if (kvmgt_gfn_is_write_protected(info
, gfn
))
1208 kvm_slot_page_track_add_page(kvm
, slot
, gfn
, KVM_PAGE_TRACK_WRITE
);
1209 kvmgt_protect_table_add(info
, gfn
);
1212 spin_unlock(&kvm
->mmu_lock
);
1213 srcu_read_unlock(&kvm
->srcu
, idx
);
1217 static int kvmgt_write_protect_remove(unsigned long handle
, u64 gfn
)
1219 struct kvmgt_guest_info
*info
;
1221 struct kvm_memory_slot
*slot
;
1224 if (!handle_valid(handle
))
1227 info
= (struct kvmgt_guest_info
*)handle
;
1230 idx
= srcu_read_lock(&kvm
->srcu
);
1231 slot
= gfn_to_memslot(kvm
, gfn
);
1233 srcu_read_unlock(&kvm
->srcu
, idx
);
1237 spin_lock(&kvm
->mmu_lock
);
1239 if (!kvmgt_gfn_is_write_protected(info
, gfn
))
1242 kvm_slot_page_track_remove_page(kvm
, slot
, gfn
, KVM_PAGE_TRACK_WRITE
);
1243 kvmgt_protect_table_del(info
, gfn
);
1246 spin_unlock(&kvm
->mmu_lock
);
1247 srcu_read_unlock(&kvm
->srcu
, idx
);
1251 static void kvmgt_page_track_write(struct kvm_vcpu
*vcpu
, gpa_t gpa
,
1252 const u8
*val
, int len
,
1253 struct kvm_page_track_notifier_node
*node
)
1255 struct kvmgt_guest_info
*info
= container_of(node
,
1256 struct kvmgt_guest_info
, track_node
);
1258 if (kvmgt_gfn_is_write_protected(info
, gpa_to_gfn(gpa
)))
1259 intel_gvt_ops
->emulate_mmio_write(info
->vgpu
, gpa
,
1263 static void kvmgt_page_track_flush_slot(struct kvm
*kvm
,
1264 struct kvm_memory_slot
*slot
,
1265 struct kvm_page_track_notifier_node
*node
)
1269 struct kvmgt_guest_info
*info
= container_of(node
,
1270 struct kvmgt_guest_info
, track_node
);
1272 spin_lock(&kvm
->mmu_lock
);
1273 for (i
= 0; i
< slot
->npages
; i
++) {
1274 gfn
= slot
->base_gfn
+ i
;
1275 if (kvmgt_gfn_is_write_protected(info
, gfn
)) {
1276 kvm_slot_page_track_remove_page(kvm
, slot
, gfn
,
1277 KVM_PAGE_TRACK_WRITE
);
1278 kvmgt_protect_table_del(info
, gfn
);
1281 spin_unlock(&kvm
->mmu_lock
);
1284 static bool __kvmgt_vgpu_exist(struct intel_vgpu
*vgpu
, struct kvm
*kvm
)
1286 struct intel_vgpu
*itr
;
1287 struct kvmgt_guest_info
*info
;
1291 mutex_lock(&vgpu
->gvt
->lock
);
1292 for_each_active_vgpu(vgpu
->gvt
, itr
, id
) {
1293 if (!handle_valid(itr
->handle
))
1296 info
= (struct kvmgt_guest_info
*)itr
->handle
;
1297 if (kvm
&& kvm
== info
->kvm
) {
1303 mutex_unlock(&vgpu
->gvt
->lock
);
1307 static int kvmgt_guest_init(struct mdev_device
*mdev
)
1309 struct kvmgt_guest_info
*info
;
1310 struct intel_vgpu
*vgpu
;
1313 vgpu
= mdev_get_drvdata(mdev
);
1314 if (handle_valid(vgpu
->handle
))
1317 kvm
= vgpu
->vdev
.kvm
;
1318 if (!kvm
|| kvm
->mm
!= current
->mm
) {
1319 gvt_vgpu_err("KVM is required to use Intel vGPU\n");
1323 if (__kvmgt_vgpu_exist(vgpu
, kvm
))
1326 info
= vzalloc(sizeof(struct kvmgt_guest_info
));
1330 vgpu
->handle
= (unsigned long)info
;
1333 kvm_get_kvm(info
->kvm
);
1335 kvmgt_protect_table_init(info
);
1336 gvt_cache_init(vgpu
);
1338 info
->track_node
.track_write
= kvmgt_page_track_write
;
1339 info
->track_node
.track_flush_slot
= kvmgt_page_track_flush_slot
;
1340 kvm_page_track_register_notifier(kvm
, &info
->track_node
);
1345 static bool kvmgt_guest_exit(struct kvmgt_guest_info
*info
)
1347 kvm_page_track_unregister_notifier(info
->kvm
, &info
->track_node
);
1348 kvm_put_kvm(info
->kvm
);
1349 kvmgt_protect_table_destroy(info
);
1350 gvt_cache_destroy(info
->vgpu
);
1356 static int kvmgt_attach_vgpu(void *vgpu
, unsigned long *handle
)
1358 /* nothing to do here */
1362 static void kvmgt_detach_vgpu(unsigned long handle
)
1364 /* nothing to do here */
1367 static int kvmgt_inject_msi(unsigned long handle
, u32 addr
, u16 data
)
1369 struct kvmgt_guest_info
*info
;
1370 struct intel_vgpu
*vgpu
;
1372 if (!handle_valid(handle
))
1375 info
= (struct kvmgt_guest_info
*)handle
;
1378 if (eventfd_signal(vgpu
->vdev
.msi_trigger
, 1) == 1)
1384 static unsigned long kvmgt_gfn_to_pfn(unsigned long handle
, unsigned long gfn
)
1386 unsigned long iova
, pfn
;
1387 struct kvmgt_guest_info
*info
;
1389 struct intel_vgpu
*vgpu
;
1392 if (!handle_valid(handle
))
1393 return INTEL_GVT_INVALID_ADDR
;
1395 info
= (struct kvmgt_guest_info
*)handle
;
1397 iova
= gvt_cache_find(info
->vgpu
, gfn
);
1398 if (iova
!= INTEL_GVT_INVALID_ADDR
)
1401 pfn
= INTEL_GVT_INVALID_ADDR
;
1402 dev
= mdev_dev(info
->vgpu
->vdev
.mdev
);
1403 rc
= vfio_pin_pages(dev
, &gfn
, 1, IOMMU_READ
| IOMMU_WRITE
, &pfn
);
1405 gvt_vgpu_err("vfio_pin_pages failed for gfn 0x%lx: %d\n",
1407 return INTEL_GVT_INVALID_ADDR
;
1409 /* transfer to host iova for GFX to use DMA */
1410 rc
= gvt_dma_map_iova(info
->vgpu
, pfn
, &iova
);
1412 gvt_vgpu_err("gvt_dma_map_iova failed for gfn: 0x%lx\n", gfn
);
1413 vfio_unpin_pages(dev
, &gfn
, 1);
1414 return INTEL_GVT_INVALID_ADDR
;
1417 gvt_cache_add(info
->vgpu
, gfn
, iova
);
1421 static int kvmgt_rw_gpa(unsigned long handle
, unsigned long gpa
,
1422 void *buf
, unsigned long len
, bool write
)
1424 struct kvmgt_guest_info
*info
;
1427 bool kthread
= current
->mm
== NULL
;
1429 if (!handle_valid(handle
))
1432 info
= (struct kvmgt_guest_info
*)handle
;
1438 idx
= srcu_read_lock(&kvm
->srcu
);
1439 ret
= write
? kvm_write_guest(kvm
, gpa
, buf
, len
) :
1440 kvm_read_guest(kvm
, gpa
, buf
, len
);
1441 srcu_read_unlock(&kvm
->srcu
, idx
);
1449 static int kvmgt_read_gpa(unsigned long handle
, unsigned long gpa
,
1450 void *buf
, unsigned long len
)
1452 return kvmgt_rw_gpa(handle
, gpa
, buf
, len
, false);
1455 static int kvmgt_write_gpa(unsigned long handle
, unsigned long gpa
,
1456 void *buf
, unsigned long len
)
1458 return kvmgt_rw_gpa(handle
, gpa
, buf
, len
, true);
1461 static unsigned long kvmgt_virt_to_pfn(void *addr
)
1463 return PFN_DOWN(__pa(addr
));
1466 struct intel_gvt_mpt kvmgt_mpt
= {
1467 .host_init
= kvmgt_host_init
,
1468 .host_exit
= kvmgt_host_exit
,
1469 .attach_vgpu
= kvmgt_attach_vgpu
,
1470 .detach_vgpu
= kvmgt_detach_vgpu
,
1471 .inject_msi
= kvmgt_inject_msi
,
1472 .from_virt_to_mfn
= kvmgt_virt_to_pfn
,
1473 .set_wp_page
= kvmgt_write_protect_add
,
1474 .unset_wp_page
= kvmgt_write_protect_remove
,
1475 .read_gpa
= kvmgt_read_gpa
,
1476 .write_gpa
= kvmgt_write_gpa
,
1477 .gfn_to_mfn
= kvmgt_gfn_to_pfn
,
1479 EXPORT_SYMBOL_GPL(kvmgt_mpt
);
1481 static int __init
kvmgt_init(void)
1486 static void __exit
kvmgt_exit(void)
1490 module_init(kvmgt_init
);
1491 module_exit(kvmgt_exit
);
1493 MODULE_LICENSE("GPL and additional rights");
1494 MODULE_AUTHOR("Intel Corporation");