1 // SPDX-License-Identifier: GPL-2.0 OR MIT
4 * Xen frontend/backend page directory based shared buffer
7 * Copyright (C) 2018 EPAM Systems Inc.
9 * Author: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
12 #include <linux/module.h>
13 #include <linux/errno.h>
16 #include <asm/xen/hypervisor.h>
17 #include <xen/balloon.h>
19 #include <xen/xenbus.h>
20 #include <xen/interface/io/ring.h>
22 #include <xen/xen-front-pgdir-shbuf.h>
24 #ifndef GRANT_INVALID_REF
26 * FIXME: usage of grant reference 0 as invalid grant reference:
27 * grant reference 0 is valid, but never exposed to a PV driver,
28 * because of the fact it is already in use/reserved by the PV console.
30 #define GRANT_INVALID_REF 0
34 * This structure represents the structure of a shared page
35 * that contains grant references to the pages of the shared
36 * buffer. This structure is common to many Xen para-virtualized
37 * protocols at include/xen/interface/io/
39 struct xen_page_directory
{
40 grant_ref_t gref_dir_next_page
;
41 grant_ref_t gref
[1]; /* Variable length */
45 * Shared buffer ops which are differently implemented
46 * depending on the allocation mode, e.g. if the buffer
47 * is allocated by the corresponding backend or frontend.
48 * Some of the operations.
50 struct xen_front_pgdir_shbuf_ops
{
52 * Calculate number of grefs required to handle this buffer,
53 * e.g. if grefs are required for page directory only or the buffer
56 void (*calc_num_grefs
)(struct xen_front_pgdir_shbuf
*buf
);
58 /* Fill page directory according to para-virtual display protocol. */
59 void (*fill_page_dir
)(struct xen_front_pgdir_shbuf
*buf
);
61 /* Claim grant references for the pages of the buffer. */
62 int (*grant_refs_for_buffer
)(struct xen_front_pgdir_shbuf
*buf
,
63 grant_ref_t
*priv_gref_head
, int gref_idx
);
65 /* Map grant references of the buffer. */
66 int (*map
)(struct xen_front_pgdir_shbuf
*buf
);
68 /* Unmap grant references of the buffer. */
69 int (*unmap
)(struct xen_front_pgdir_shbuf
*buf
);
73 * Get granted reference to the very first page of the
74 * page directory. Usually this is passed to the backend,
75 * so it can find/fill the grant references to the buffer's
78 * \param buf shared buffer which page directory is of interest.
79 * \return granted reference to the very first page of the
83 xen_front_pgdir_shbuf_get_dir_start(struct xen_front_pgdir_shbuf
*buf
)
86 return GRANT_INVALID_REF
;
90 EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_get_dir_start
);
93 * Map granted references of the shared buffer.
95 * Depending on the shared buffer mode of allocation
96 * (be_alloc flag) this can either do nothing (for buffers
97 * shared by the frontend itself) or map the provided granted
98 * references onto the backing storage (buf->pages).
100 * \param buf shared buffer which grants to be maped.
101 * \return zero on success or a negative number on failure.
103 int xen_front_pgdir_shbuf_map(struct xen_front_pgdir_shbuf
*buf
)
105 if (buf
->ops
&& buf
->ops
->map
)
106 return buf
->ops
->map(buf
);
108 /* No need to map own grant references. */
111 EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_map
);
114 * Unmap granted references of the shared buffer.
116 * Depending on the shared buffer mode of allocation
117 * (be_alloc flag) this can either do nothing (for buffers
118 * shared by the frontend itself) or unmap the provided granted
121 * \param buf shared buffer which grants to be unmaped.
122 * \return zero on success or a negative number on failure.
124 int xen_front_pgdir_shbuf_unmap(struct xen_front_pgdir_shbuf
*buf
)
126 if (buf
->ops
&& buf
->ops
->unmap
)
127 return buf
->ops
->unmap(buf
);
129 /* No need to unmap own grant references. */
132 EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_unmap
);
135 * Free all the resources of the shared buffer.
137 * \param buf shared buffer which resources to be freed.
139 void xen_front_pgdir_shbuf_free(struct xen_front_pgdir_shbuf
*buf
)
144 for (i
= 0; i
< buf
->num_grefs
; i
++)
145 if (buf
->grefs
[i
] != GRANT_INVALID_REF
)
146 gnttab_end_foreign_access(buf
->grefs
[i
],
150 kfree(buf
->directory
);
152 EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_free
);
155 * Number of grefs a page can hold with respect to the
156 * struct xen_page_directory header.
158 #define XEN_NUM_GREFS_PER_PAGE ((PAGE_SIZE - \
159 offsetof(struct xen_page_directory, \
160 gref)) / sizeof(grant_ref_t))
163 * Get the number of pages the page directory consumes itself.
165 * \param buf shared buffer.
167 static int get_num_pages_dir(struct xen_front_pgdir_shbuf
*buf
)
169 return DIV_ROUND_UP(buf
->num_pages
, XEN_NUM_GREFS_PER_PAGE
);
173 * Calculate the number of grant references needed to share the buffer
174 * and its pages when backend allocates the buffer.
176 * \param buf shared buffer.
178 static void backend_calc_num_grefs(struct xen_front_pgdir_shbuf
*buf
)
180 /* Only for pages the page directory consumes itself. */
181 buf
->num_grefs
= get_num_pages_dir(buf
);
185 * Calculate the number of grant references needed to share the buffer
186 * and its pages when frontend allocates the buffer.
188 * \param buf shared buffer.
190 static void guest_calc_num_grefs(struct xen_front_pgdir_shbuf
*buf
)
193 * Number of pages the page directory consumes itself
194 * plus grefs for the buffer pages.
196 buf
->num_grefs
= get_num_pages_dir(buf
) + buf
->num_pages
;
199 #define xen_page_to_vaddr(page) \
200 ((uintptr_t)pfn_to_kaddr(page_to_xen_pfn(page)))
203 * Unmap the buffer previously mapped with grant references
204 * provided by the backend.
206 * \param buf shared buffer.
207 * \return zero on success or a negative number on failure.
209 static int backend_unmap(struct xen_front_pgdir_shbuf
*buf
)
211 struct gnttab_unmap_grant_ref
*unmap_ops
;
214 if (!buf
->pages
|| !buf
->backend_map_handles
|| !buf
->grefs
)
217 unmap_ops
= kcalloc(buf
->num_pages
, sizeof(*unmap_ops
),
222 for (i
= 0; i
< buf
->num_pages
; i
++) {
225 addr
= xen_page_to_vaddr(buf
->pages
[i
]);
226 gnttab_set_unmap_op(&unmap_ops
[i
], addr
, GNTMAP_host_map
,
227 buf
->backend_map_handles
[i
]);
230 ret
= gnttab_unmap_refs(unmap_ops
, NULL
, buf
->pages
,
233 for (i
= 0; i
< buf
->num_pages
; i
++) {
234 if (unlikely(unmap_ops
[i
].status
!= GNTST_okay
))
235 dev_err(&buf
->xb_dev
->dev
,
236 "Failed to unmap page %d: %d\n",
237 i
, unmap_ops
[i
].status
);
241 dev_err(&buf
->xb_dev
->dev
,
242 "Failed to unmap grant references, ret %d", ret
);
245 kfree(buf
->backend_map_handles
);
246 buf
->backend_map_handles
= NULL
;
251 * Map the buffer with grant references provided by the backend.
253 * \param buf shared buffer.
254 * \return zero on success or a negative number on failure.
256 static int backend_map(struct xen_front_pgdir_shbuf
*buf
)
258 struct gnttab_map_grant_ref
*map_ops
= NULL
;
260 int ret
, cur_gref
, cur_dir_page
, cur_page
, grefs_left
;
262 map_ops
= kcalloc(buf
->num_pages
, sizeof(*map_ops
), GFP_KERNEL
);
266 buf
->backend_map_handles
= kcalloc(buf
->num_pages
,
267 sizeof(*buf
->backend_map_handles
),
269 if (!buf
->backend_map_handles
) {
275 * Read page directory to get grefs from the backend: for external
276 * buffer we only allocate buf->grefs for the page directory,
277 * so buf->num_grefs has number of pages in the page directory itself.
279 ptr
= buf
->directory
;
280 grefs_left
= buf
->num_pages
;
282 for (cur_dir_page
= 0; cur_dir_page
< buf
->num_grefs
; cur_dir_page
++) {
283 struct xen_page_directory
*page_dir
=
284 (struct xen_page_directory
*)ptr
;
285 int to_copy
= XEN_NUM_GREFS_PER_PAGE
;
287 if (to_copy
> grefs_left
)
288 to_copy
= grefs_left
;
290 for (cur_gref
= 0; cur_gref
< to_copy
; cur_gref
++) {
293 addr
= xen_page_to_vaddr(buf
->pages
[cur_page
]);
294 gnttab_set_map_op(&map_ops
[cur_page
], addr
,
296 page_dir
->gref
[cur_gref
],
297 buf
->xb_dev
->otherend_id
);
301 grefs_left
-= to_copy
;
304 ret
= gnttab_map_refs(map_ops
, NULL
, buf
->pages
, buf
->num_pages
);
306 /* Save handles even if error, so we can unmap. */
307 for (cur_page
= 0; cur_page
< buf
->num_pages
; cur_page
++) {
308 if (likely(map_ops
[cur_page
].status
== GNTST_okay
)) {
309 buf
->backend_map_handles
[cur_page
] =
310 map_ops
[cur_page
].handle
;
312 buf
->backend_map_handles
[cur_page
] =
313 INVALID_GRANT_HANDLE
;
316 dev_err(&buf
->xb_dev
->dev
,
317 "Failed to map page %d: %d\n",
318 cur_page
, map_ops
[cur_page
].status
);
323 dev_err(&buf
->xb_dev
->dev
,
324 "Failed to map grant references, ret %d", ret
);
333 * Fill page directory with grant references to the pages of the
334 * page directory itself.
336 * The grant references to the buffer pages are provided by the
337 * backend in this case.
339 * \param buf shared buffer.
341 static void backend_fill_page_dir(struct xen_front_pgdir_shbuf
*buf
)
343 struct xen_page_directory
*page_dir
;
345 int i
, num_pages_dir
;
347 ptr
= buf
->directory
;
348 num_pages_dir
= get_num_pages_dir(buf
);
350 /* Fill only grefs for the page directory itself. */
351 for (i
= 0; i
< num_pages_dir
- 1; i
++) {
352 page_dir
= (struct xen_page_directory
*)ptr
;
354 page_dir
->gref_dir_next_page
= buf
->grefs
[i
+ 1];
357 /* Last page must say there is no more pages. */
358 page_dir
= (struct xen_page_directory
*)ptr
;
359 page_dir
->gref_dir_next_page
= GRANT_INVALID_REF
;
363 * Fill page directory with grant references to the pages of the
364 * page directory and the buffer we share with the backend.
366 * \param buf shared buffer.
368 static void guest_fill_page_dir(struct xen_front_pgdir_shbuf
*buf
)
371 int cur_gref
, grefs_left
, to_copy
, i
, num_pages_dir
;
373 ptr
= buf
->directory
;
374 num_pages_dir
= get_num_pages_dir(buf
);
377 * While copying, skip grefs at start, they are for pages
378 * granted for the page directory itself.
380 cur_gref
= num_pages_dir
;
381 grefs_left
= buf
->num_pages
;
382 for (i
= 0; i
< num_pages_dir
; i
++) {
383 struct xen_page_directory
*page_dir
=
384 (struct xen_page_directory
*)ptr
;
386 if (grefs_left
<= XEN_NUM_GREFS_PER_PAGE
) {
387 to_copy
= grefs_left
;
388 page_dir
->gref_dir_next_page
= GRANT_INVALID_REF
;
390 to_copy
= XEN_NUM_GREFS_PER_PAGE
;
391 page_dir
->gref_dir_next_page
= buf
->grefs
[i
+ 1];
393 memcpy(&page_dir
->gref
, &buf
->grefs
[cur_gref
],
394 to_copy
* sizeof(grant_ref_t
));
396 grefs_left
-= to_copy
;
402 * Grant references to the frontend's buffer pages.
404 * These will be shared with the backend, so it can
405 * access the buffer's data.
407 * \param buf shared buffer.
408 * \return zero on success or a negative number on failure.
410 static int guest_grant_refs_for_buffer(struct xen_front_pgdir_shbuf
*buf
,
411 grant_ref_t
*priv_gref_head
,
414 int i
, cur_ref
, otherend_id
;
416 otherend_id
= buf
->xb_dev
->otherend_id
;
417 for (i
= 0; i
< buf
->num_pages
; i
++) {
418 cur_ref
= gnttab_claim_grant_reference(priv_gref_head
);
422 gnttab_grant_foreign_access_ref(cur_ref
, otherend_id
,
423 xen_page_to_gfn(buf
->pages
[i
]),
425 buf
->grefs
[gref_idx
++] = cur_ref
;
431 * Grant all the references needed to share the buffer.
433 * Grant references to the page directory pages and, if
434 * needed, also to the pages of the shared buffer data.
436 * \param buf shared buffer.
437 * \return zero on success or a negative number on failure.
439 static int grant_references(struct xen_front_pgdir_shbuf
*buf
)
441 grant_ref_t priv_gref_head
;
442 int ret
, i
, j
, cur_ref
;
443 int otherend_id
, num_pages_dir
;
445 ret
= gnttab_alloc_grant_references(buf
->num_grefs
, &priv_gref_head
);
447 dev_err(&buf
->xb_dev
->dev
,
448 "Cannot allocate grant references\n");
452 otherend_id
= buf
->xb_dev
->otherend_id
;
454 num_pages_dir
= get_num_pages_dir(buf
);
455 for (i
= 0; i
< num_pages_dir
; i
++) {
458 cur_ref
= gnttab_claim_grant_reference(&priv_gref_head
);
462 frame
= xen_page_to_gfn(virt_to_page(buf
->directory
+
464 gnttab_grant_foreign_access_ref(cur_ref
, otherend_id
, frame
, 0);
465 buf
->grefs
[j
++] = cur_ref
;
468 if (buf
->ops
->grant_refs_for_buffer
) {
469 ret
= buf
->ops
->grant_refs_for_buffer(buf
, &priv_gref_head
, j
);
474 gnttab_free_grant_references(priv_gref_head
);
479 * Allocate all required structures to mange shared buffer.
481 * \param buf shared buffer.
482 * \return zero on success or a negative number on failure.
484 static int alloc_storage(struct xen_front_pgdir_shbuf
*buf
)
486 buf
->grefs
= kcalloc(buf
->num_grefs
, sizeof(*buf
->grefs
), GFP_KERNEL
);
490 buf
->directory
= kcalloc(get_num_pages_dir(buf
), PAGE_SIZE
, GFP_KERNEL
);
498 * For backend allocated buffers we don't need grant_refs_for_buffer
499 * as those grant references are allocated at backend side.
501 static const struct xen_front_pgdir_shbuf_ops backend_ops
= {
502 .calc_num_grefs
= backend_calc_num_grefs
,
503 .fill_page_dir
= backend_fill_page_dir
,
505 .unmap
= backend_unmap
509 * For locally granted references we do not need to map/unmap
512 static const struct xen_front_pgdir_shbuf_ops local_ops
= {
513 .calc_num_grefs
= guest_calc_num_grefs
,
514 .fill_page_dir
= guest_fill_page_dir
,
515 .grant_refs_for_buffer
= guest_grant_refs_for_buffer
,
519 * Allocate a new instance of a shared buffer.
521 * \param cfg configuration to be used while allocating a new shared buffer.
522 * \return zero on success or a negative number on failure.
524 int xen_front_pgdir_shbuf_alloc(struct xen_front_pgdir_shbuf_cfg
*cfg
)
526 struct xen_front_pgdir_shbuf
*buf
= cfg
->pgdir
;
530 buf
->ops
= &backend_ops
;
532 buf
->ops
= &local_ops
;
533 buf
->xb_dev
= cfg
->xb_dev
;
534 buf
->num_pages
= cfg
->num_pages
;
535 buf
->pages
= cfg
->pages
;
537 buf
->ops
->calc_num_grefs(buf
);
539 ret
= alloc_storage(buf
);
543 ret
= grant_references(buf
);
547 buf
->ops
->fill_page_dir(buf
);
552 xen_front_pgdir_shbuf_free(buf
);
555 EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_alloc
);
557 MODULE_DESCRIPTION("Xen frontend/backend page directory based "
558 "shared buffer handling");
559 MODULE_AUTHOR("Oleksandr Andrushchenko");
560 MODULE_LICENSE("GPL");