]>
Commit | Line | Data |
---|---|---|
1a59d1b8 | 1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
f87d0fbb RR |
2 | /* |
3 | * Linux host-side vring helpers; for when the kernel needs to access | |
4 | * someone else's vring. | |
5 | * | |
6 | * Copyright IBM Corporation, 2013. | |
7 | * Parts taken from drivers/vhost/vhost.c Copyright 2009 Red Hat, Inc. | |
8 | * | |
f87d0fbb RR |
9 | * Written by: Rusty Russell <rusty@rustcorp.com.au> |
10 | */ | |
11 | #ifndef _LINUX_VRINGH_H | |
12 | #define _LINUX_VRINGH_H | |
13 | #include <uapi/linux/virtio_ring.h> | |
b9f7ac8c | 14 | #include <linux/virtio_byteorder.h> |
f87d0fbb RR |
15 | #include <linux/uio.h> |
16 | #include <linux/slab.h> | |
3302363a | 17 | #if IS_REACHABLE(CONFIG_VHOST_IOTLB) |
9ad9c49c JW |
18 | #include <linux/dma-direction.h> |
19 | #include <linux/vhost_iotlb.h> | |
3302363a | 20 | #endif |
f87d0fbb RR |
21 | #include <asm/barrier.h> |
22 | ||
23 | /* virtio_ring with information needed for host access. */ | |
24 | struct vringh { | |
b9f7ac8c MT |
25 | /* Everything is little endian */ |
26 | bool little_endian; | |
27 | ||
f87d0fbb RR |
28 | /* Guest publishes used event idx (note: we always do). */ |
29 | bool event_indices; | |
30 | ||
31 | /* Can we get away with weak barriers? */ | |
32 | bool weak_barriers; | |
33 | ||
34 | /* Last available index we saw (ie. where we're up to). */ | |
35 | u16 last_avail_idx; | |
36 | ||
37 | /* Last index we used. */ | |
38 | u16 last_used_idx; | |
39 | ||
40 | /* How many descriptors we've completed since last need_notify(). */ | |
41 | u32 completed; | |
42 | ||
43 | /* The vring (note: it may contain user pointers!) */ | |
44 | struct vring vring; | |
3beee86a | 45 | |
9ad9c49c JW |
46 | /* IOTLB for this vring */ |
47 | struct vhost_iotlb *iotlb; | |
48 | ||
f53d9910 SG |
49 | /* spinlock to synchronize IOTLB accesses */ |
50 | spinlock_t *iotlb_lock; | |
51 | ||
3beee86a SB |
52 | /* The function to call to notify the guest about added buffers */ |
53 | void (*notify)(struct vringh *); | |
54 | }; | |
55 | ||
56 | /** | |
57 | * struct vringh_config_ops - ops for creating a host vring from a virtio driver | |
58 | * @find_vrhs: find the host vrings and instantiate them | |
59 | * vdev: the virtio_device | |
60 | * nhvrs: the number of host vrings to find | |
61 | * hvrs: on success, includes new host vrings | |
62 | * callbacks: array of driver callbacks, for each host vring | |
63 | * include a NULL entry for vqs that do not need a callback | |
64 | * Returns 0 on success or error status | |
65 | * @del_vrhs: free the host vrings found by find_vrhs(). | |
66 | */ | |
67 | struct virtio_device; | |
68 | typedef void vrh_callback_t(struct virtio_device *, struct vringh *); | |
69 | struct vringh_config_ops { | |
70 | int (*find_vrhs)(struct virtio_device *vdev, unsigned nhvrs, | |
71 | struct vringh *vrhs[], vrh_callback_t *callbacks[]); | |
72 | void (*del_vrhs)(struct virtio_device *vdev); | |
f87d0fbb RR |
73 | }; |
74 | ||
75 | /* The memory the vring can access, and what offset to apply. */ | |
76 | struct vringh_range { | |
77 | u64 start, end_incl; | |
78 | u64 offset; | |
79 | }; | |
80 | ||
81 | /** | |
82 | * struct vringh_iov - iovec mangler. | |
83 | * | |
84 | * Mangles iovec in place, and restores it. | |
85 | * Remaining data is iov + i, of used - i elements. | |
86 | */ | |
87 | struct vringh_iov { | |
88 | struct iovec *iov; | |
89 | size_t consumed; /* Within iov[i] */ | |
90 | unsigned i, used, max_num; | |
91 | }; | |
92 | ||
93 | /** | |
94 | * struct vringh_iov - kvec mangler. | |
95 | * | |
96 | * Mangles kvec in place, and restores it. | |
97 | * Remaining data is iov + i, of used - i elements. | |
98 | */ | |
99 | struct vringh_kiov { | |
100 | struct kvec *iov; | |
101 | size_t consumed; /* Within iov[i] */ | |
102 | unsigned i, used, max_num; | |
103 | }; | |
104 | ||
105 | /* Flag on max_num to indicate we're kmalloced. */ | |
106 | #define VRINGH_IOV_ALLOCATED 0x8000000 | |
107 | ||
108 | /* Helpers for userspace vrings. */ | |
b97a8a90 | 109 | int vringh_init_user(struct vringh *vrh, u64 features, |
f87d0fbb | 110 | unsigned int num, bool weak_barriers, |
a865e420 MT |
111 | vring_desc_t __user *desc, |
112 | vring_avail_t __user *avail, | |
113 | vring_used_t __user *used); | |
f87d0fbb RR |
114 | |
115 | static inline void vringh_iov_init(struct vringh_iov *iov, | |
116 | struct iovec *iovec, unsigned num) | |
117 | { | |
118 | iov->used = iov->i = 0; | |
119 | iov->consumed = 0; | |
120 | iov->max_num = num; | |
121 | iov->iov = iovec; | |
122 | } | |
123 | ||
124 | static inline void vringh_iov_reset(struct vringh_iov *iov) | |
125 | { | |
126 | iov->iov[iov->i].iov_len += iov->consumed; | |
127 | iov->iov[iov->i].iov_base -= iov->consumed; | |
128 | iov->consumed = 0; | |
129 | iov->i = 0; | |
130 | } | |
131 | ||
132 | static inline void vringh_iov_cleanup(struct vringh_iov *iov) | |
133 | { | |
134 | if (iov->max_num & VRINGH_IOV_ALLOCATED) | |
135 | kfree(iov->iov); | |
136 | iov->max_num = iov->used = iov->i = iov->consumed = 0; | |
137 | iov->iov = NULL; | |
138 | } | |
139 | ||
140 | /* Convert a descriptor into iovecs. */ | |
141 | int vringh_getdesc_user(struct vringh *vrh, | |
142 | struct vringh_iov *riov, | |
143 | struct vringh_iov *wiov, | |
144 | bool (*getrange)(struct vringh *vrh, | |
145 | u64 addr, struct vringh_range *r), | |
146 | u16 *head); | |
147 | ||
148 | /* Copy bytes from readable vsg, consuming it (and incrementing wiov->i). */ | |
149 | ssize_t vringh_iov_pull_user(struct vringh_iov *riov, void *dst, size_t len); | |
150 | ||
151 | /* Copy bytes into writable vsg, consuming it (and incrementing wiov->i). */ | |
152 | ssize_t vringh_iov_push_user(struct vringh_iov *wiov, | |
153 | const void *src, size_t len); | |
154 | ||
155 | /* Mark a descriptor as used. */ | |
156 | int vringh_complete_user(struct vringh *vrh, u16 head, u32 len); | |
157 | int vringh_complete_multi_user(struct vringh *vrh, | |
158 | const struct vring_used_elem used[], | |
159 | unsigned num_used); | |
160 | ||
161 | /* Pretend we've never seen descriptor (for easy error handling). */ | |
162 | void vringh_abandon_user(struct vringh *vrh, unsigned int num); | |
163 | ||
164 | /* Do we need to fire the eventfd to notify the other side? */ | |
165 | int vringh_need_notify_user(struct vringh *vrh); | |
166 | ||
167 | bool vringh_notify_enable_user(struct vringh *vrh); | |
168 | void vringh_notify_disable_user(struct vringh *vrh); | |
169 | ||
170 | /* Helpers for kernelspace vrings. */ | |
b97a8a90 | 171 | int vringh_init_kern(struct vringh *vrh, u64 features, |
f87d0fbb RR |
172 | unsigned int num, bool weak_barriers, |
173 | struct vring_desc *desc, | |
174 | struct vring_avail *avail, | |
175 | struct vring_used *used); | |
176 | ||
177 | static inline void vringh_kiov_init(struct vringh_kiov *kiov, | |
178 | struct kvec *kvec, unsigned num) | |
179 | { | |
180 | kiov->used = kiov->i = 0; | |
181 | kiov->consumed = 0; | |
182 | kiov->max_num = num; | |
183 | kiov->iov = kvec; | |
184 | } | |
185 | ||
186 | static inline void vringh_kiov_reset(struct vringh_kiov *kiov) | |
187 | { | |
188 | kiov->iov[kiov->i].iov_len += kiov->consumed; | |
189 | kiov->iov[kiov->i].iov_base -= kiov->consumed; | |
190 | kiov->consumed = 0; | |
191 | kiov->i = 0; | |
192 | } | |
193 | ||
194 | static inline void vringh_kiov_cleanup(struct vringh_kiov *kiov) | |
195 | { | |
196 | if (kiov->max_num & VRINGH_IOV_ALLOCATED) | |
197 | kfree(kiov->iov); | |
198 | kiov->max_num = kiov->used = kiov->i = kiov->consumed = 0; | |
199 | kiov->iov = NULL; | |
200 | } | |
201 | ||
14c9ac05 SG |
202 | static inline size_t vringh_kiov_length(struct vringh_kiov *kiov) |
203 | { | |
204 | size_t len = 0; | |
205 | int i; | |
206 | ||
207 | for (i = kiov->i; i < kiov->used; i++) | |
208 | len += kiov->iov[i].iov_len; | |
209 | ||
210 | return len; | |
211 | } | |
212 | ||
b8c06ad4 SG |
213 | void vringh_kiov_advance(struct vringh_kiov *kiov, size_t len); |
214 | ||
f87d0fbb RR |
215 | int vringh_getdesc_kern(struct vringh *vrh, |
216 | struct vringh_kiov *riov, | |
217 | struct vringh_kiov *wiov, | |
218 | u16 *head, | |
219 | gfp_t gfp); | |
220 | ||
221 | ssize_t vringh_iov_pull_kern(struct vringh_kiov *riov, void *dst, size_t len); | |
222 | ssize_t vringh_iov_push_kern(struct vringh_kiov *wiov, | |
223 | const void *src, size_t len); | |
224 | void vringh_abandon_kern(struct vringh *vrh, unsigned int num); | |
225 | int vringh_complete_kern(struct vringh *vrh, u16 head, u32 len); | |
226 | ||
227 | bool vringh_notify_enable_kern(struct vringh *vrh); | |
228 | void vringh_notify_disable_kern(struct vringh *vrh); | |
229 | ||
230 | int vringh_need_notify_kern(struct vringh *vrh); | |
231 | ||
3beee86a SB |
232 | /* Notify the guest about buffers added to the used ring */ |
233 | static inline void vringh_notify(struct vringh *vrh) | |
234 | { | |
235 | if (vrh->notify) | |
236 | vrh->notify(vrh); | |
237 | } | |
238 | ||
5da7b160 GK |
239 | static inline bool vringh_is_little_endian(const struct vringh *vrh) |
240 | { | |
7d824109 GK |
241 | return vrh->little_endian || |
242 | virtio_legacy_is_little_endian(); | |
5da7b160 GK |
243 | } |
244 | ||
b9f7ac8c MT |
245 | static inline u16 vringh16_to_cpu(const struct vringh *vrh, __virtio16 val) |
246 | { | |
5da7b160 | 247 | return __virtio16_to_cpu(vringh_is_little_endian(vrh), val); |
b9f7ac8c MT |
248 | } |
249 | ||
250 | static inline __virtio16 cpu_to_vringh16(const struct vringh *vrh, u16 val) | |
251 | { | |
5da7b160 | 252 | return __cpu_to_virtio16(vringh_is_little_endian(vrh), val); |
b9f7ac8c MT |
253 | } |
254 | ||
255 | static inline u32 vringh32_to_cpu(const struct vringh *vrh, __virtio32 val) | |
256 | { | |
5da7b160 | 257 | return __virtio32_to_cpu(vringh_is_little_endian(vrh), val); |
b9f7ac8c MT |
258 | } |
259 | ||
260 | static inline __virtio32 cpu_to_vringh32(const struct vringh *vrh, u32 val) | |
261 | { | |
5da7b160 | 262 | return __cpu_to_virtio32(vringh_is_little_endian(vrh), val); |
b9f7ac8c MT |
263 | } |
264 | ||
265 | static inline u64 vringh64_to_cpu(const struct vringh *vrh, __virtio64 val) | |
266 | { | |
5da7b160 | 267 | return __virtio64_to_cpu(vringh_is_little_endian(vrh), val); |
b9f7ac8c MT |
268 | } |
269 | ||
270 | static inline __virtio64 cpu_to_vringh64(const struct vringh *vrh, u64 val) | |
271 | { | |
5da7b160 | 272 | return __cpu_to_virtio64(vringh_is_little_endian(vrh), val); |
b9f7ac8c | 273 | } |
9ad9c49c | 274 | |
3302363a MT |
275 | #if IS_REACHABLE(CONFIG_VHOST_IOTLB) |
276 | ||
f53d9910 SG |
277 | void vringh_set_iotlb(struct vringh *vrh, struct vhost_iotlb *iotlb, |
278 | spinlock_t *iotlb_lock); | |
9ad9c49c JW |
279 | |
280 | int vringh_init_iotlb(struct vringh *vrh, u64 features, | |
281 | unsigned int num, bool weak_barriers, | |
282 | struct vring_desc *desc, | |
283 | struct vring_avail *avail, | |
284 | struct vring_used *used); | |
285 | ||
286 | int vringh_getdesc_iotlb(struct vringh *vrh, | |
287 | struct vringh_kiov *riov, | |
288 | struct vringh_kiov *wiov, | |
289 | u16 *head, | |
290 | gfp_t gfp); | |
291 | ||
292 | ssize_t vringh_iov_pull_iotlb(struct vringh *vrh, | |
293 | struct vringh_kiov *riov, | |
294 | void *dst, size_t len); | |
295 | ssize_t vringh_iov_push_iotlb(struct vringh *vrh, | |
296 | struct vringh_kiov *wiov, | |
297 | const void *src, size_t len); | |
298 | ||
299 | void vringh_abandon_iotlb(struct vringh *vrh, unsigned int num); | |
300 | ||
301 | int vringh_complete_iotlb(struct vringh *vrh, u16 head, u32 len); | |
302 | ||
303 | bool vringh_notify_enable_iotlb(struct vringh *vrh); | |
304 | void vringh_notify_disable_iotlb(struct vringh *vrh); | |
305 | ||
306 | int vringh_need_notify_iotlb(struct vringh *vrh); | |
307 | ||
3302363a MT |
308 | #endif /* CONFIG_VHOST_IOTLB */ |
309 | ||
f87d0fbb | 310 | #endif /* _LINUX_VRINGH_H */ |