1 /******************************************************************************
4 * Device for accessing (in user-space) pages that have been granted by other
7 * Copyright (c) 2006-2007, D G Murray.
8 * (c) 2009 Gerd Hoffmann <kraxel@redhat.com>
9 * (c) 2018 Oleksandr Andrushchenko, EPAM Systems Inc.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
23 #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
25 #include <linux/dma-mapping.h>
26 #include <linux/module.h>
27 #include <linux/kernel.h>
28 #include <linux/init.h>
29 #include <linux/miscdevice.h>
31 #include <linux/uaccess.h>
32 #include <linux/sched.h>
33 #include <linux/sched/mm.h>
34 #include <linux/spinlock.h>
35 #include <linux/slab.h>
36 #include <linux/highmem.h>
37 #include <linux/refcount.h>
40 #include <xen/grant_table.h>
41 #include <xen/balloon.h>
42 #include <xen/gntdev.h>
43 #include <xen/events.h>
45 #include <asm/xen/hypervisor.h>
46 #include <asm/xen/hypercall.h>
48 #include "gntdev-common.h"
49 #ifdef CONFIG_XEN_GNTDEV_DMABUF
50 #include "gntdev-dmabuf.h"
53 MODULE_LICENSE("GPL");
54 MODULE_AUTHOR("Derek G. Murray <Derek.Murray@cl.cam.ac.uk>, "
55 "Gerd Hoffmann <kraxel@redhat.com>");
56 MODULE_DESCRIPTION("User-space granted page access driver");
58 static unsigned int limit
= 64*1024;
59 module_param(limit
, uint
, 0644);
60 MODULE_PARM_DESC(limit
,
61 "Maximum number of grants that may be mapped by one mapping request");
63 static int use_ptemod
;
65 static int unmap_grant_pages(struct gntdev_grant_map
*map
,
66 int offset
, int pages
);
68 static struct miscdevice gntdev_miscdev
;
70 /* ------------------------------------------------------------------ */
72 bool gntdev_test_page_count(unsigned int count
)
74 return !count
|| count
> limit
;
77 static void gntdev_print_maps(struct gntdev_priv
*priv
,
78 char *text
, int text_index
)
81 struct gntdev_grant_map
*map
;
83 pr_debug("%s: maps list (priv %p)\n", __func__
, priv
);
84 list_for_each_entry(map
, &priv
->maps
, next
)
85 pr_debug(" index %2d, count %2d %s\n",
86 map
->index
, map
->count
,
87 map
->index
== text_index
&& text
? text
: "");
91 static void gntdev_free_map(struct gntdev_grant_map
*map
)
96 #ifdef CONFIG_XEN_GRANT_DMA_ALLOC
98 struct gnttab_dma_alloc_args args
;
100 args
.dev
= map
->dma_dev
;
101 args
.coherent
= !!(map
->dma_flags
& GNTDEV_DMA_FLAG_COHERENT
);
102 args
.nr_pages
= map
->count
;
103 args
.pages
= map
->pages
;
104 args
.frames
= map
->frames
;
105 args
.vaddr
= map
->dma_vaddr
;
106 args
.dev_bus_addr
= map
->dma_bus_addr
;
108 gnttab_dma_free_pages(&args
);
112 gnttab_free_pages(map
->count
, map
->pages
);
114 #ifdef CONFIG_XEN_GRANT_DMA_ALLOC
119 kvfree(map
->map_ops
);
120 kvfree(map
->unmap_ops
);
121 kvfree(map
->kmap_ops
);
122 kvfree(map
->kunmap_ops
);
126 struct gntdev_grant_map
*gntdev_alloc_map(struct gntdev_priv
*priv
, int count
,
129 struct gntdev_grant_map
*add
;
132 add
= kzalloc(sizeof(*add
), GFP_KERNEL
);
136 add
->grants
= kvmalloc_array(count
, sizeof(add
->grants
[0]),
138 add
->map_ops
= kvmalloc_array(count
, sizeof(add
->map_ops
[0]),
140 add
->unmap_ops
= kvmalloc_array(count
, sizeof(add
->unmap_ops
[0]),
142 add
->pages
= kvcalloc(count
, sizeof(add
->pages
[0]), GFP_KERNEL
);
143 if (NULL
== add
->grants
||
144 NULL
== add
->map_ops
||
145 NULL
== add
->unmap_ops
||
149 add
->kmap_ops
= kvmalloc_array(count
, sizeof(add
->kmap_ops
[0]),
151 add
->kunmap_ops
= kvmalloc_array(count
, sizeof(add
->kunmap_ops
[0]),
153 if (NULL
== add
->kmap_ops
|| NULL
== add
->kunmap_ops
)
157 #ifdef CONFIG_XEN_GRANT_DMA_ALLOC
158 add
->dma_flags
= dma_flags
;
161 * Check if this mapping is requested to be backed
164 if (dma_flags
& (GNTDEV_DMA_FLAG_WC
| GNTDEV_DMA_FLAG_COHERENT
)) {
165 struct gnttab_dma_alloc_args args
;
167 add
->frames
= kvcalloc(count
, sizeof(add
->frames
[0]),
172 /* Remember the device, so we can free DMA memory. */
173 add
->dma_dev
= priv
->dma_dev
;
175 args
.dev
= priv
->dma_dev
;
176 args
.coherent
= !!(dma_flags
& GNTDEV_DMA_FLAG_COHERENT
);
177 args
.nr_pages
= count
;
178 args
.pages
= add
->pages
;
179 args
.frames
= add
->frames
;
181 if (gnttab_dma_alloc_pages(&args
))
184 add
->dma_vaddr
= args
.vaddr
;
185 add
->dma_bus_addr
= args
.dev_bus_addr
;
188 if (gnttab_alloc_pages(count
, add
->pages
))
191 for (i
= 0; i
< count
; i
++) {
192 add
->grants
[i
].domid
= DOMID_INVALID
;
193 add
->grants
[i
].ref
= INVALID_GRANT_REF
;
194 add
->map_ops
[i
].handle
= INVALID_GRANT_HANDLE
;
195 add
->unmap_ops
[i
].handle
= INVALID_GRANT_HANDLE
;
197 add
->kmap_ops
[i
].handle
= INVALID_GRANT_HANDLE
;
198 add
->kunmap_ops
[i
].handle
= INVALID_GRANT_HANDLE
;
204 refcount_set(&add
->users
, 1);
209 gntdev_free_map(add
);
213 void gntdev_add_map(struct gntdev_priv
*priv
, struct gntdev_grant_map
*add
)
215 struct gntdev_grant_map
*map
;
217 list_for_each_entry(map
, &priv
->maps
, next
) {
218 if (add
->index
+ add
->count
< map
->index
) {
219 list_add_tail(&add
->next
, &map
->next
);
222 add
->index
= map
->index
+ map
->count
;
224 list_add_tail(&add
->next
, &priv
->maps
);
227 gntdev_print_maps(priv
, "[new]", add
->index
);
230 static struct gntdev_grant_map
*gntdev_find_map_index(struct gntdev_priv
*priv
,
231 int index
, int count
)
233 struct gntdev_grant_map
*map
;
235 list_for_each_entry(map
, &priv
->maps
, next
) {
236 if (map
->index
!= index
)
238 if (count
&& map
->count
!= count
)
245 void gntdev_put_map(struct gntdev_priv
*priv
, struct gntdev_grant_map
*map
)
250 if (!refcount_dec_and_test(&map
->users
))
253 if (map
->pages
&& !use_ptemod
)
254 unmap_grant_pages(map
, 0, map
->count
);
256 if (map
->notify
.flags
& UNMAP_NOTIFY_SEND_EVENT
) {
257 notify_remote_via_evtchn(map
->notify
.event
);
258 evtchn_put(map
->notify
.event
);
260 gntdev_free_map(map
);
263 /* ------------------------------------------------------------------ */
265 static int find_grant_ptes(pte_t
*pte
, unsigned long addr
, void *data
)
267 struct gntdev_grant_map
*map
= data
;
268 unsigned int pgnr
= (addr
- map
->vma
->vm_start
) >> PAGE_SHIFT
;
269 int flags
= map
->flags
| GNTMAP_application_map
| GNTMAP_contains_pte
|
270 (1 << _GNTMAP_guest_avail0
);
273 BUG_ON(pgnr
>= map
->count
);
274 pte_maddr
= arbitrary_virt_to_machine(pte
).maddr
;
276 gnttab_set_map_op(&map
->map_ops
[pgnr
], pte_maddr
, flags
,
277 map
->grants
[pgnr
].ref
,
278 map
->grants
[pgnr
].domid
);
279 gnttab_set_unmap_op(&map
->unmap_ops
[pgnr
], pte_maddr
, flags
,
280 INVALID_GRANT_HANDLE
);
284 int gntdev_map_grant_pages(struct gntdev_grant_map
*map
)
289 /* Note: it could already be mapped */
290 if (map
->map_ops
[0].handle
!= INVALID_GRANT_HANDLE
)
292 for (i
= 0; i
< map
->count
; i
++) {
293 unsigned long addr
= (unsigned long)
294 pfn_to_kaddr(page_to_pfn(map
->pages
[i
]));
295 gnttab_set_map_op(&map
->map_ops
[i
], addr
, map
->flags
,
297 map
->grants
[i
].domid
);
298 gnttab_set_unmap_op(&map
->unmap_ops
[i
], addr
,
299 map
->flags
, INVALID_GRANT_HANDLE
);
303 * Setup the map_ops corresponding to the pte entries pointing
304 * to the kernel linear addresses of the struct pages.
305 * These ptes are completely different from the user ptes dealt
306 * with find_grant_ptes.
307 * Note that GNTMAP_device_map isn't needed here: The
308 * dev_bus_addr output field gets consumed only from ->map_ops,
309 * and by not requesting it when mapping we also avoid needing
310 * to mirror dev_bus_addr into ->unmap_ops (and holding an extra
311 * reference to the page in the hypervisor).
313 unsigned int flags
= (map
->flags
& ~GNTMAP_device_map
) |
316 for (i
= 0; i
< map
->count
; i
++) {
317 unsigned long address
= (unsigned long)
318 pfn_to_kaddr(page_to_pfn(map
->pages
[i
]));
319 BUG_ON(PageHighMem(map
->pages
[i
]));
321 gnttab_set_map_op(&map
->kmap_ops
[i
], address
, flags
,
323 map
->grants
[i
].domid
);
324 gnttab_set_unmap_op(&map
->kunmap_ops
[i
], address
,
325 flags
, INVALID_GRANT_HANDLE
);
329 pr_debug("map %d+%d\n", map
->index
, map
->count
);
330 err
= gnttab_map_refs(map
->map_ops
, map
->kmap_ops
, map
->pages
,
333 for (i
= 0; i
< map
->count
; i
++) {
334 if (map
->map_ops
[i
].status
== GNTST_okay
)
335 map
->unmap_ops
[i
].handle
= map
->map_ops
[i
].handle
;
339 if (map
->flags
& GNTMAP_device_map
)
340 map
->unmap_ops
[i
].dev_bus_addr
= map
->map_ops
[i
].dev_bus_addr
;
343 if (map
->kmap_ops
[i
].status
== GNTST_okay
)
344 map
->kunmap_ops
[i
].handle
= map
->kmap_ops
[i
].handle
;
352 static int __unmap_grant_pages(struct gntdev_grant_map
*map
, int offset
,
356 struct gntab_unmap_queue_data unmap_data
;
358 if (map
->notify
.flags
& UNMAP_NOTIFY_CLEAR_BYTE
) {
359 int pgno
= (map
->notify
.addr
>> PAGE_SHIFT
);
360 if (pgno
>= offset
&& pgno
< offset
+ pages
) {
361 /* No need for kmap, pages are in lowmem */
362 uint8_t *tmp
= pfn_to_kaddr(page_to_pfn(map
->pages
[pgno
]));
363 tmp
[map
->notify
.addr
& (PAGE_SIZE
-1)] = 0;
364 map
->notify
.flags
&= ~UNMAP_NOTIFY_CLEAR_BYTE
;
368 unmap_data
.unmap_ops
= map
->unmap_ops
+ offset
;
369 unmap_data
.kunmap_ops
= use_ptemod
? map
->kunmap_ops
+ offset
: NULL
;
370 unmap_data
.pages
= map
->pages
+ offset
;
371 unmap_data
.count
= pages
;
373 err
= gnttab_unmap_refs_sync(&unmap_data
);
377 for (i
= 0; i
< pages
; i
++) {
378 if (map
->unmap_ops
[offset
+i
].status
)
380 pr_debug("unmap handle=%d st=%d\n",
381 map
->unmap_ops
[offset
+i
].handle
,
382 map
->unmap_ops
[offset
+i
].status
);
383 map
->unmap_ops
[offset
+i
].handle
= INVALID_GRANT_HANDLE
;
385 if (map
->kunmap_ops
[offset
+i
].status
)
387 pr_debug("kunmap handle=%u st=%d\n",
388 map
->kunmap_ops
[offset
+i
].handle
,
389 map
->kunmap_ops
[offset
+i
].status
);
390 map
->kunmap_ops
[offset
+i
].handle
= INVALID_GRANT_HANDLE
;
396 static int unmap_grant_pages(struct gntdev_grant_map
*map
, int offset
,
401 pr_debug("unmap %d+%d [%d+%d]\n", map
->index
, map
->count
, offset
, pages
);
403 /* It is possible the requested range will have a "hole" where we
404 * already unmapped some of the grants. Only unmap valid ranges.
406 while (pages
&& !err
) {
408 map
->unmap_ops
[offset
].handle
== INVALID_GRANT_HANDLE
) {
413 while (range
< pages
) {
414 if (map
->unmap_ops
[offset
+ range
].handle
==
415 INVALID_GRANT_HANDLE
)
419 err
= __unmap_grant_pages(map
, offset
, range
);
427 /* ------------------------------------------------------------------ */
429 static void gntdev_vma_open(struct vm_area_struct
*vma
)
431 struct gntdev_grant_map
*map
= vma
->vm_private_data
;
433 pr_debug("gntdev_vma_open %p\n", vma
);
434 refcount_inc(&map
->users
);
437 static void gntdev_vma_close(struct vm_area_struct
*vma
)
439 struct gntdev_grant_map
*map
= vma
->vm_private_data
;
440 struct file
*file
= vma
->vm_file
;
441 struct gntdev_priv
*priv
= file
->private_data
;
443 pr_debug("gntdev_vma_close %p\n", vma
);
445 WARN_ON(map
->vma
!= vma
);
446 mmu_interval_notifier_remove(&map
->notifier
);
449 vma
->vm_private_data
= NULL
;
450 gntdev_put_map(priv
, map
);
453 static struct page
*gntdev_vma_find_special_page(struct vm_area_struct
*vma
,
456 struct gntdev_grant_map
*map
= vma
->vm_private_data
;
458 return map
->pages
[(addr
- map
->pages_vm_start
) >> PAGE_SHIFT
];
461 static const struct vm_operations_struct gntdev_vmops
= {
462 .open
= gntdev_vma_open
,
463 .close
= gntdev_vma_close
,
464 .find_special_page
= gntdev_vma_find_special_page
,
467 /* ------------------------------------------------------------------ */
469 static bool gntdev_invalidate(struct mmu_interval_notifier
*mn
,
470 const struct mmu_notifier_range
*range
,
471 unsigned long cur_seq
)
473 struct gntdev_grant_map
*map
=
474 container_of(mn
, struct gntdev_grant_map
, notifier
);
475 unsigned long mstart
, mend
;
478 if (!mmu_notifier_range_blockable(range
))
482 * If the VMA is split or otherwise changed the notifier is not
483 * updated, but we don't want to process VA's outside the modified
484 * VMA. FIXME: It would be much more understandable to just prevent
485 * modifying the VMA in the first place.
487 if (map
->vma
->vm_start
>= range
->end
||
488 map
->vma
->vm_end
<= range
->start
)
491 mstart
= max(range
->start
, map
->vma
->vm_start
);
492 mend
= min(range
->end
, map
->vma
->vm_end
);
493 pr_debug("map %d+%d (%lx %lx), range %lx %lx, mrange %lx %lx\n",
494 map
->index
, map
->count
,
495 map
->vma
->vm_start
, map
->vma
->vm_end
,
496 range
->start
, range
->end
, mstart
, mend
);
497 err
= unmap_grant_pages(map
,
498 (mstart
- map
->vma
->vm_start
) >> PAGE_SHIFT
,
499 (mend
- mstart
) >> PAGE_SHIFT
);
505 static const struct mmu_interval_notifier_ops gntdev_mmu_ops
= {
506 .invalidate
= gntdev_invalidate
,
509 /* ------------------------------------------------------------------ */
511 static int gntdev_open(struct inode
*inode
, struct file
*flip
)
513 struct gntdev_priv
*priv
;
515 priv
= kzalloc(sizeof(*priv
), GFP_KERNEL
);
519 INIT_LIST_HEAD(&priv
->maps
);
520 mutex_init(&priv
->lock
);
522 #ifdef CONFIG_XEN_GNTDEV_DMABUF
523 priv
->dmabuf_priv
= gntdev_dmabuf_init(flip
);
524 if (IS_ERR(priv
->dmabuf_priv
)) {
525 int ret
= PTR_ERR(priv
->dmabuf_priv
);
532 flip
->private_data
= priv
;
533 #ifdef CONFIG_XEN_GRANT_DMA_ALLOC
534 priv
->dma_dev
= gntdev_miscdev
.this_device
;
535 dma_coerce_mask_and_coherent(priv
->dma_dev
, DMA_BIT_MASK(64));
537 pr_debug("priv %p\n", priv
);
542 static int gntdev_release(struct inode
*inode
, struct file
*flip
)
544 struct gntdev_priv
*priv
= flip
->private_data
;
545 struct gntdev_grant_map
*map
;
547 pr_debug("priv %p\n", priv
);
549 mutex_lock(&priv
->lock
);
550 while (!list_empty(&priv
->maps
)) {
551 map
= list_entry(priv
->maps
.next
,
552 struct gntdev_grant_map
, next
);
553 list_del(&map
->next
);
554 gntdev_put_map(NULL
/* already removed */, map
);
556 mutex_unlock(&priv
->lock
);
558 #ifdef CONFIG_XEN_GNTDEV_DMABUF
559 gntdev_dmabuf_fini(priv
->dmabuf_priv
);
566 static long gntdev_ioctl_map_grant_ref(struct gntdev_priv
*priv
,
567 struct ioctl_gntdev_map_grant_ref __user
*u
)
569 struct ioctl_gntdev_map_grant_ref op
;
570 struct gntdev_grant_map
*map
;
573 if (copy_from_user(&op
, u
, sizeof(op
)) != 0)
575 pr_debug("priv %p, add %d\n", priv
, op
.count
);
576 if (unlikely(gntdev_test_page_count(op
.count
)))
580 map
= gntdev_alloc_map(priv
, op
.count
, 0 /* This is not a dma-buf. */);
584 if (copy_from_user(map
->grants
, &u
->refs
,
585 sizeof(map
->grants
[0]) * op
.count
) != 0) {
586 gntdev_put_map(NULL
, map
);
590 mutex_lock(&priv
->lock
);
591 gntdev_add_map(priv
, map
);
592 op
.index
= map
->index
<< PAGE_SHIFT
;
593 mutex_unlock(&priv
->lock
);
595 if (copy_to_user(u
, &op
, sizeof(op
)) != 0)
601 static long gntdev_ioctl_unmap_grant_ref(struct gntdev_priv
*priv
,
602 struct ioctl_gntdev_unmap_grant_ref __user
*u
)
604 struct ioctl_gntdev_unmap_grant_ref op
;
605 struct gntdev_grant_map
*map
;
608 if (copy_from_user(&op
, u
, sizeof(op
)) != 0)
610 pr_debug("priv %p, del %d+%d\n", priv
, (int)op
.index
, (int)op
.count
);
612 mutex_lock(&priv
->lock
);
613 map
= gntdev_find_map_index(priv
, op
.index
>> PAGE_SHIFT
, op
.count
);
615 list_del(&map
->next
);
618 mutex_unlock(&priv
->lock
);
620 gntdev_put_map(priv
, map
);
624 static long gntdev_ioctl_get_offset_for_vaddr(struct gntdev_priv
*priv
,
625 struct ioctl_gntdev_get_offset_for_vaddr __user
*u
)
627 struct ioctl_gntdev_get_offset_for_vaddr op
;
628 struct vm_area_struct
*vma
;
629 struct gntdev_grant_map
*map
;
632 if (copy_from_user(&op
, u
, sizeof(op
)) != 0)
634 pr_debug("priv %p, offset for vaddr %lx\n", priv
, (unsigned long)op
.vaddr
);
636 mmap_read_lock(current
->mm
);
637 vma
= find_vma(current
->mm
, op
.vaddr
);
638 if (!vma
|| vma
->vm_ops
!= &gntdev_vmops
)
641 map
= vma
->vm_private_data
;
645 op
.offset
= map
->index
<< PAGE_SHIFT
;
646 op
.count
= map
->count
;
650 mmap_read_unlock(current
->mm
);
652 if (rv
== 0 && copy_to_user(u
, &op
, sizeof(op
)) != 0)
657 static long gntdev_ioctl_notify(struct gntdev_priv
*priv
, void __user
*u
)
659 struct ioctl_gntdev_unmap_notify op
;
660 struct gntdev_grant_map
*map
;
663 evtchn_port_t out_event
;
665 if (copy_from_user(&op
, u
, sizeof(op
)))
668 if (op
.action
& ~(UNMAP_NOTIFY_CLEAR_BYTE
|UNMAP_NOTIFY_SEND_EVENT
))
671 /* We need to grab a reference to the event channel we are going to use
672 * to send the notify before releasing the reference we may already have
673 * (if someone has called this ioctl twice). This is required so that
674 * it is possible to change the clear_byte part of the notification
675 * without disturbing the event channel part, which may now be the last
676 * reference to that event channel.
678 if (op
.action
& UNMAP_NOTIFY_SEND_EVENT
) {
679 if (evtchn_get(op
.event_channel_port
))
683 out_flags
= op
.action
;
684 out_event
= op
.event_channel_port
;
686 mutex_lock(&priv
->lock
);
688 list_for_each_entry(map
, &priv
->maps
, next
) {
689 uint64_t begin
= map
->index
<< PAGE_SHIFT
;
690 uint64_t end
= (map
->index
+ map
->count
) << PAGE_SHIFT
;
691 if (op
.index
>= begin
&& op
.index
< end
)
698 if ((op
.action
& UNMAP_NOTIFY_CLEAR_BYTE
) &&
699 (map
->flags
& GNTMAP_readonly
)) {
704 out_flags
= map
->notify
.flags
;
705 out_event
= map
->notify
.event
;
707 map
->notify
.flags
= op
.action
;
708 map
->notify
.addr
= op
.index
- (map
->index
<< PAGE_SHIFT
);
709 map
->notify
.event
= op
.event_channel_port
;
714 mutex_unlock(&priv
->lock
);
716 /* Drop the reference to the event channel we did not save in the map */
717 if (out_flags
& UNMAP_NOTIFY_SEND_EVENT
)
718 evtchn_put(out_event
);
723 #define GNTDEV_COPY_BATCH 16
725 struct gntdev_copy_batch
{
726 struct gnttab_copy ops
[GNTDEV_COPY_BATCH
];
727 struct page
*pages
[GNTDEV_COPY_BATCH
];
728 s16 __user
*status
[GNTDEV_COPY_BATCH
];
730 unsigned int nr_pages
;
734 static int gntdev_get_page(struct gntdev_copy_batch
*batch
, void __user
*virt
,
737 unsigned long addr
= (unsigned long)virt
;
739 unsigned long xen_pfn
;
742 ret
= pin_user_pages_fast(addr
, 1, batch
->writeable
? FOLL_WRITE
: 0, &page
);
746 batch
->pages
[batch
->nr_pages
++] = page
;
748 xen_pfn
= page_to_xen_pfn(page
) + XEN_PFN_DOWN(addr
& ~PAGE_MASK
);
749 *gfn
= pfn_to_gfn(xen_pfn
);
754 static void gntdev_put_pages(struct gntdev_copy_batch
*batch
)
756 unpin_user_pages_dirty_lock(batch
->pages
, batch
->nr_pages
, batch
->writeable
);
758 batch
->writeable
= false;
761 static int gntdev_copy(struct gntdev_copy_batch
*batch
)
765 gnttab_batch_copy(batch
->ops
, batch
->nr_ops
);
766 gntdev_put_pages(batch
);
769 * For each completed op, update the status if the op failed
770 * and all previous ops for the segment were successful.
772 for (i
= 0; i
< batch
->nr_ops
; i
++) {
773 s16 status
= batch
->ops
[i
].status
;
776 if (status
== GNTST_okay
)
779 if (__get_user(old_status
, batch
->status
[i
]))
782 if (old_status
!= GNTST_okay
)
785 if (__put_user(status
, batch
->status
[i
]))
793 static int gntdev_grant_copy_seg(struct gntdev_copy_batch
*batch
,
794 struct gntdev_grant_copy_segment
*seg
,
800 * Disallow local -> local copies since there is only space in
801 * batch->pages for one page per-op and this would be a very
802 * expensive memcpy().
804 if (!(seg
->flags
& (GNTCOPY_source_gref
| GNTCOPY_dest_gref
)))
807 /* Can't cross page if source/dest is a grant ref. */
808 if (seg
->flags
& GNTCOPY_source_gref
) {
809 if (seg
->source
.foreign
.offset
+ seg
->len
> XEN_PAGE_SIZE
)
812 if (seg
->flags
& GNTCOPY_dest_gref
) {
813 if (seg
->dest
.foreign
.offset
+ seg
->len
> XEN_PAGE_SIZE
)
817 if (put_user(GNTST_okay
, status
))
820 while (copied
< seg
->len
) {
821 struct gnttab_copy
*op
;
827 if (batch
->nr_ops
>= GNTDEV_COPY_BATCH
) {
828 ret
= gntdev_copy(batch
);
833 len
= seg
->len
- copied
;
835 op
= &batch
->ops
[batch
->nr_ops
];
838 if (seg
->flags
& GNTCOPY_source_gref
) {
839 op
->source
.u
.ref
= seg
->source
.foreign
.ref
;
840 op
->source
.domid
= seg
->source
.foreign
.domid
;
841 op
->source
.offset
= seg
->source
.foreign
.offset
+ copied
;
842 op
->flags
|= GNTCOPY_source_gref
;
844 virt
= seg
->source
.virt
+ copied
;
845 off
= (unsigned long)virt
& ~XEN_PAGE_MASK
;
846 len
= min(len
, (size_t)XEN_PAGE_SIZE
- off
);
847 batch
->writeable
= false;
849 ret
= gntdev_get_page(batch
, virt
, &gfn
);
853 op
->source
.u
.gmfn
= gfn
;
854 op
->source
.domid
= DOMID_SELF
;
855 op
->source
.offset
= off
;
858 if (seg
->flags
& GNTCOPY_dest_gref
) {
859 op
->dest
.u
.ref
= seg
->dest
.foreign
.ref
;
860 op
->dest
.domid
= seg
->dest
.foreign
.domid
;
861 op
->dest
.offset
= seg
->dest
.foreign
.offset
+ copied
;
862 op
->flags
|= GNTCOPY_dest_gref
;
864 virt
= seg
->dest
.virt
+ copied
;
865 off
= (unsigned long)virt
& ~XEN_PAGE_MASK
;
866 len
= min(len
, (size_t)XEN_PAGE_SIZE
- off
);
867 batch
->writeable
= true;
869 ret
= gntdev_get_page(batch
, virt
, &gfn
);
873 op
->dest
.u
.gmfn
= gfn
;
874 op
->dest
.domid
= DOMID_SELF
;
875 op
->dest
.offset
= off
;
881 batch
->status
[batch
->nr_ops
] = status
;
888 static long gntdev_ioctl_grant_copy(struct gntdev_priv
*priv
, void __user
*u
)
890 struct ioctl_gntdev_grant_copy copy
;
891 struct gntdev_copy_batch batch
;
895 if (copy_from_user(©
, u
, sizeof(copy
)))
901 for (i
= 0; i
< copy
.count
; i
++) {
902 struct gntdev_grant_copy_segment seg
;
904 if (copy_from_user(&seg
, ©
.segments
[i
], sizeof(seg
))) {
909 ret
= gntdev_grant_copy_seg(&batch
, &seg
, ©
.segments
[i
].status
);
916 ret
= gntdev_copy(&batch
);
920 gntdev_put_pages(&batch
);
924 static long gntdev_ioctl(struct file
*flip
,
925 unsigned int cmd
, unsigned long arg
)
927 struct gntdev_priv
*priv
= flip
->private_data
;
928 void __user
*ptr
= (void __user
*)arg
;
931 case IOCTL_GNTDEV_MAP_GRANT_REF
:
932 return gntdev_ioctl_map_grant_ref(priv
, ptr
);
934 case IOCTL_GNTDEV_UNMAP_GRANT_REF
:
935 return gntdev_ioctl_unmap_grant_ref(priv
, ptr
);
937 case IOCTL_GNTDEV_GET_OFFSET_FOR_VADDR
:
938 return gntdev_ioctl_get_offset_for_vaddr(priv
, ptr
);
940 case IOCTL_GNTDEV_SET_UNMAP_NOTIFY
:
941 return gntdev_ioctl_notify(priv
, ptr
);
943 case IOCTL_GNTDEV_GRANT_COPY
:
944 return gntdev_ioctl_grant_copy(priv
, ptr
);
946 #ifdef CONFIG_XEN_GNTDEV_DMABUF
947 case IOCTL_GNTDEV_DMABUF_EXP_FROM_REFS
:
948 return gntdev_ioctl_dmabuf_exp_from_refs(priv
, use_ptemod
, ptr
);
950 case IOCTL_GNTDEV_DMABUF_EXP_WAIT_RELEASED
:
951 return gntdev_ioctl_dmabuf_exp_wait_released(priv
, ptr
);
953 case IOCTL_GNTDEV_DMABUF_IMP_TO_REFS
:
954 return gntdev_ioctl_dmabuf_imp_to_refs(priv
, ptr
);
956 case IOCTL_GNTDEV_DMABUF_IMP_RELEASE
:
957 return gntdev_ioctl_dmabuf_imp_release(priv
, ptr
);
961 pr_debug("priv %p, unknown cmd %x\n", priv
, cmd
);
968 static int gntdev_mmap(struct file
*flip
, struct vm_area_struct
*vma
)
970 struct gntdev_priv
*priv
= flip
->private_data
;
971 int index
= vma
->vm_pgoff
;
972 int count
= vma_pages(vma
);
973 struct gntdev_grant_map
*map
;
976 if ((vma
->vm_flags
& VM_WRITE
) && !(vma
->vm_flags
& VM_SHARED
))
979 pr_debug("map %d+%d at %lx (pgoff %lx)\n",
980 index
, count
, vma
->vm_start
, vma
->vm_pgoff
);
982 mutex_lock(&priv
->lock
);
983 map
= gntdev_find_map_index(priv
, index
, count
);
986 if (use_ptemod
&& map
->vma
)
988 refcount_inc(&map
->users
);
990 vma
->vm_ops
= &gntdev_vmops
;
992 vma
->vm_flags
|= VM_DONTEXPAND
| VM_DONTDUMP
| VM_MIXEDMAP
;
995 vma
->vm_flags
|= VM_DONTCOPY
;
997 vma
->vm_private_data
= map
;
999 if ((vma
->vm_flags
& VM_WRITE
) &&
1000 (map
->flags
& GNTMAP_readonly
))
1001 goto out_unlock_put
;
1003 map
->flags
= GNTMAP_host_map
;
1004 if (!(vma
->vm_flags
& VM_WRITE
))
1005 map
->flags
|= GNTMAP_readonly
;
1010 err
= mmu_interval_notifier_insert_locked(
1011 &map
->notifier
, vma
->vm_mm
, vma
->vm_start
,
1012 vma
->vm_end
- vma
->vm_start
, &gntdev_mmu_ops
);
1015 goto out_unlock_put
;
1018 mutex_unlock(&priv
->lock
);
1022 * gntdev takes the address of the PTE in find_grant_ptes() and
1023 * passes it to the hypervisor in gntdev_map_grant_pages(). The
1024 * purpose of the notifier is to prevent the hypervisor pointer
1025 * to the PTE from going stale.
1027 * Since this vma's mappings can't be touched without the
1028 * mmap_lock, and we are holding it now, there is no need for
1029 * the notifier_range locking pattern.
1031 mmu_interval_read_begin(&map
->notifier
);
1033 map
->pages_vm_start
= vma
->vm_start
;
1034 err
= apply_to_page_range(vma
->vm_mm
, vma
->vm_start
,
1035 vma
->vm_end
- vma
->vm_start
,
1036 find_grant_ptes
, map
);
1038 pr_warn("find_grant_ptes() failure.\n");
1043 err
= gntdev_map_grant_pages(map
);
1048 err
= vm_map_pages_zero(vma
, map
->pages
, map
->count
);
1056 mutex_unlock(&priv
->lock
);
1060 mutex_unlock(&priv
->lock
);
1063 unmap_grant_pages(map
, 0, map
->count
);
1065 mmu_interval_notifier_remove(&map
->notifier
);
1069 gntdev_put_map(priv
, map
);
1073 static const struct file_operations gntdev_fops
= {
1074 .owner
= THIS_MODULE
,
1075 .open
= gntdev_open
,
1076 .release
= gntdev_release
,
1077 .mmap
= gntdev_mmap
,
1078 .unlocked_ioctl
= gntdev_ioctl
1081 static struct miscdevice gntdev_miscdev
= {
1082 .minor
= MISC_DYNAMIC_MINOR
,
1083 .name
= "xen/gntdev",
1084 .fops
= &gntdev_fops
,
1087 /* ------------------------------------------------------------------ */
1089 static int __init
gntdev_init(void)
1096 use_ptemod
= !xen_feature(XENFEAT_auto_translated_physmap
);
1098 err
= misc_register(&gntdev_miscdev
);
1100 pr_err("Could not register gntdev device\n");
1106 static void __exit
gntdev_exit(void)
1108 misc_deregister(&gntdev_miscdev
);
1111 module_init(gntdev_init
);
1112 module_exit(gntdev_exit
);
1114 /* ------------------------------------------------------------------ */