]>
Commit | Line | Data |
---|---|---|
bcabbcca OBC |
1 | /* |
2 | * Virtio-based remote processor messaging bus | |
3 | * | |
4 | * Copyright (C) 2011 Texas Instruments, Inc. | |
5 | * Copyright (C) 2011 Google, Inc. | |
6 | * | |
7 | * Ohad Ben-Cohen <ohad@wizery.com> | |
8 | * Brian Swetland <swetland@google.com> | |
9 | * | |
10 | * This software is licensed under the terms of the GNU General Public | |
11 | * License version 2, as published by the Free Software Foundation, and | |
12 | * may be copied, distributed, and modified under those terms. | |
13 | * | |
14 | * This program is distributed in the hope that it will be useful, | |
15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
17 | * GNU General Public License for more details. | |
18 | */ | |
19 | ||
20 | #define pr_fmt(fmt) "%s: " fmt, __func__ | |
21 | ||
22 | #include <linux/kernel.h> | |
23 | #include <linux/module.h> | |
24 | #include <linux/virtio.h> | |
25 | #include <linux/virtio_ids.h> | |
26 | #include <linux/virtio_config.h> | |
27 | #include <linux/scatterlist.h> | |
28 | #include <linux/dma-mapping.h> | |
29 | #include <linux/slab.h> | |
30 | #include <linux/idr.h> | |
31 | #include <linux/jiffies.h> | |
32 | #include <linux/sched.h> | |
33 | #include <linux/wait.h> | |
34 | #include <linux/rpmsg.h> | |
35 | #include <linux/mutex.h> | |
36 | ||
37 | /** | |
38 | * struct virtproc_info - virtual remote processor state | |
39 | * @vdev: the virtio device | |
40 | * @rvq: rx virtqueue | |
41 | * @svq: tx virtqueue | |
42 | * @rbufs: kernel address of rx buffers | |
43 | * @sbufs: kernel address of tx buffers | |
44 | * @last_sbuf: index of last tx buffer used | |
45 | * @bufs_dma: dma base addr of the buffers | |
46 | * @tx_lock: protects svq, sbufs and sleepers, to allow concurrent senders. | |
47 | * sending a message might require waking up a dozing remote | |
48 | * processor, which involves sleeping, hence the mutex. | |
49 | * @endpoints: idr of local endpoints, allows fast retrieval | |
50 | * @endpoints_lock: lock of the endpoints set | |
51 | * @sendq: wait queue of sending contexts waiting for a tx buffers | |
52 | * @sleepers: number of senders that are waiting for a tx buffer | |
53 | * @ns_ept: the bus's name service endpoint | |
54 | * | |
55 | * This structure stores the rpmsg state of a given virtio remote processor | |
56 | * device (there might be several virtio proc devices for each physical | |
57 | * remote processor). | |
58 | */ | |
59 | struct virtproc_info { | |
60 | struct virtio_device *vdev; | |
61 | struct virtqueue *rvq, *svq; | |
62 | void *rbufs, *sbufs; | |
63 | int last_sbuf; | |
64 | dma_addr_t bufs_dma; | |
65 | struct mutex tx_lock; | |
66 | struct idr endpoints; | |
67 | struct mutex endpoints_lock; | |
68 | wait_queue_head_t sendq; | |
69 | atomic_t sleepers; | |
70 | struct rpmsg_endpoint *ns_ept; | |
71 | }; | |
72 | ||
73 | /** | |
74 | * struct rpmsg_channel_info - internal channel info representation | |
75 | * @name: name of service | |
76 | * @src: local address | |
77 | * @dst: destination address | |
78 | */ | |
79 | struct rpmsg_channel_info { | |
80 | char name[RPMSG_NAME_SIZE]; | |
81 | u32 src; | |
82 | u32 dst; | |
83 | }; | |
84 | ||
85 | #define to_rpmsg_channel(d) container_of(d, struct rpmsg_channel, dev) | |
86 | #define to_rpmsg_driver(d) container_of(d, struct rpmsg_driver, drv) | |
87 | ||
88 | /* | |
89 | * We're allocating 512 buffers of 512 bytes for communications, and then | |
90 | * using the first 256 buffers for RX, and the last 256 buffers for TX. | |
91 | * | |
92 | * Each buffer will have 16 bytes for the msg header and 496 bytes for | |
93 | * the payload. | |
94 | * | |
95 | * This will require a total space of 256KB for the buffers. | |
96 | * | |
97 | * We might also want to add support for user-provided buffers in time. | |
98 | * This will allow bigger buffer size flexibility, and can also be used | |
99 | * to achieve zero-copy messaging. | |
100 | * | |
101 | * Note that these numbers are purely a decision of this driver - we | |
102 | * can change this without changing anything in the firmware of the remote | |
103 | * processor. | |
104 | */ | |
105 | #define RPMSG_NUM_BUFS (512) | |
106 | #define RPMSG_BUF_SIZE (512) | |
107 | #define RPMSG_TOTAL_BUF_SPACE (RPMSG_NUM_BUFS * RPMSG_BUF_SIZE) | |
108 | ||
109 | /* | |
110 | * Local addresses are dynamically allocated on-demand. | |
111 | * We do not dynamically assign addresses from the low 1024 range, | |
112 | * in order to reserve that address range for predefined services. | |
113 | */ | |
114 | #define RPMSG_RESERVED_ADDRESSES (1024) | |
115 | ||
116 | /* Address 53 is reserved for advertising remote services */ | |
117 | #define RPMSG_NS_ADDR (53) | |
118 | ||
119 | /* sysfs show configuration fields */ | |
120 | #define rpmsg_show_attr(field, path, format_string) \ | |
121 | static ssize_t \ | |
122 | field##_show(struct device *dev, \ | |
123 | struct device_attribute *attr, char *buf) \ | |
124 | { \ | |
125 | struct rpmsg_channel *rpdev = to_rpmsg_channel(dev); \ | |
126 | \ | |
127 | return sprintf(buf, format_string, rpdev->path); \ | |
128 | } | |
129 | ||
130 | /* for more info, see Documentation/ABI/testing/sysfs-bus-rpmsg */ | |
131 | rpmsg_show_attr(name, id.name, "%s\n"); | |
132 | rpmsg_show_attr(src, src, "0x%x\n"); | |
133 | rpmsg_show_attr(dst, dst, "0x%x\n"); | |
134 | rpmsg_show_attr(announce, announce ? "true" : "false", "%s\n"); | |
135 | ||
136 | /* | |
137 | * Unique (and free running) index for rpmsg devices. | |
138 | * | |
139 | * Yeah, we're not recycling those numbers (yet?). will be easy | |
140 | * to change if/when we want to. | |
141 | */ | |
142 | static unsigned int rpmsg_dev_index; | |
143 | ||
144 | static ssize_t modalias_show(struct device *dev, | |
145 | struct device_attribute *attr, char *buf) | |
146 | { | |
147 | struct rpmsg_channel *rpdev = to_rpmsg_channel(dev); | |
148 | ||
149 | return sprintf(buf, RPMSG_DEVICE_MODALIAS_FMT "\n", rpdev->id.name); | |
150 | } | |
151 | ||
152 | static struct device_attribute rpmsg_dev_attrs[] = { | |
153 | __ATTR_RO(name), | |
154 | __ATTR_RO(modalias), | |
155 | __ATTR_RO(dst), | |
156 | __ATTR_RO(src), | |
157 | __ATTR_RO(announce), | |
158 | __ATTR_NULL | |
159 | }; | |
160 | ||
161 | /* rpmsg devices and drivers are matched using the service name */ | |
162 | static inline int rpmsg_id_match(const struct rpmsg_channel *rpdev, | |
163 | const struct rpmsg_device_id *id) | |
164 | { | |
165 | return strncmp(id->name, rpdev->id.name, RPMSG_NAME_SIZE) == 0; | |
166 | } | |
167 | ||
168 | /* match rpmsg channel and rpmsg driver */ | |
169 | static int rpmsg_dev_match(struct device *dev, struct device_driver *drv) | |
170 | { | |
171 | struct rpmsg_channel *rpdev = to_rpmsg_channel(dev); | |
172 | struct rpmsg_driver *rpdrv = to_rpmsg_driver(drv); | |
173 | const struct rpmsg_device_id *ids = rpdrv->id_table; | |
174 | unsigned int i; | |
175 | ||
176 | for (i = 0; ids[i].name[0]; i++) | |
177 | if (rpmsg_id_match(rpdev, &ids[i])) | |
178 | return 1; | |
179 | ||
180 | return 0; | |
181 | } | |
182 | ||
183 | static int rpmsg_uevent(struct device *dev, struct kobj_uevent_env *env) | |
184 | { | |
185 | struct rpmsg_channel *rpdev = to_rpmsg_channel(dev); | |
186 | ||
187 | return add_uevent_var(env, "MODALIAS=" RPMSG_DEVICE_MODALIAS_FMT, | |
188 | rpdev->id.name); | |
189 | } | |
190 | ||
191 | /* for more info, see below documentation of rpmsg_create_ept() */ | |
192 | static struct rpmsg_endpoint *__rpmsg_create_ept(struct virtproc_info *vrp, | |
193 | struct rpmsg_channel *rpdev, rpmsg_rx_cb_t cb, | |
194 | void *priv, u32 addr) | |
195 | { | |
196 | int err, tmpaddr, request; | |
197 | struct rpmsg_endpoint *ept; | |
198 | struct device *dev = rpdev ? &rpdev->dev : &vrp->vdev->dev; | |
199 | ||
200 | if (!idr_pre_get(&vrp->endpoints, GFP_KERNEL)) | |
201 | return NULL; | |
202 | ||
203 | ept = kzalloc(sizeof(*ept), GFP_KERNEL); | |
204 | if (!ept) { | |
205 | dev_err(dev, "failed to kzalloc a new ept\n"); | |
206 | return NULL; | |
207 | } | |
208 | ||
209 | ept->rpdev = rpdev; | |
210 | ept->cb = cb; | |
211 | ept->priv = priv; | |
212 | ||
213 | /* do we need to allocate a local address ? */ | |
214 | request = addr == RPMSG_ADDR_ANY ? RPMSG_RESERVED_ADDRESSES : addr; | |
215 | ||
216 | mutex_lock(&vrp->endpoints_lock); | |
217 | ||
218 | /* bind the endpoint to an rpmsg address (and allocate one if needed) */ | |
219 | err = idr_get_new_above(&vrp->endpoints, ept, request, &tmpaddr); | |
220 | if (err) { | |
221 | dev_err(dev, "idr_get_new_above failed: %d\n", err); | |
222 | goto free_ept; | |
223 | } | |
224 | ||
225 | /* make sure the user's address request is fulfilled, if relevant */ | |
226 | if (addr != RPMSG_ADDR_ANY && tmpaddr != addr) { | |
227 | dev_err(dev, "address 0x%x already in use\n", addr); | |
228 | goto rem_idr; | |
229 | } | |
230 | ||
231 | ept->addr = tmpaddr; | |
232 | ||
233 | mutex_unlock(&vrp->endpoints_lock); | |
234 | ||
235 | return ept; | |
236 | ||
237 | rem_idr: | |
238 | idr_remove(&vrp->endpoints, request); | |
239 | free_ept: | |
240 | mutex_unlock(&vrp->endpoints_lock); | |
241 | kfree(ept); | |
242 | return NULL; | |
243 | } | |
244 | ||
245 | /** | |
246 | * rpmsg_create_ept() - create a new rpmsg_endpoint | |
247 | * @rpdev: rpmsg channel device | |
248 | * @cb: rx callback handler | |
249 | * @priv: private data for the driver's use | |
250 | * @addr: local rpmsg address to bind with @cb | |
251 | * | |
252 | * Every rpmsg address in the system is bound to an rx callback (so when | |
253 | * inbound messages arrive, they are dispatched by the rpmsg bus using the | |
254 | * appropriate callback handler) by means of an rpmsg_endpoint struct. | |
255 | * | |
256 | * This function allows drivers to create such an endpoint, and by that, | |
257 | * bind a callback, and possibly some private data too, to an rpmsg address | |
258 | * (either one that is known in advance, or one that will be dynamically | |
259 | * assigned for them). | |
260 | * | |
261 | * Simple rpmsg drivers need not call rpmsg_create_ept, because an endpoint | |
262 | * is already created for them when they are probed by the rpmsg bus | |
263 | * (using the rx callback provided when they registered to the rpmsg bus). | |
264 | * | |
265 | * So things should just work for simple drivers: they already have an | |
266 | * endpoint, their rx callback is bound to their rpmsg address, and when | |
267 | * relevant inbound messages arrive (i.e. messages which their dst address | |
268 | * equals to the src address of their rpmsg channel), the driver's handler | |
269 | * is invoked to process it. | |
270 | * | |
271 | * That said, more complicated drivers might do need to allocate | |
272 | * additional rpmsg addresses, and bind them to different rx callbacks. | |
273 | * To accomplish that, those drivers need to call this function. | |
274 | * | |
275 | * Drivers should provide their @rpdev channel (so the new endpoint would belong | |
276 | * to the same remote processor their channel belongs to), an rx callback | |
277 | * function, an optional private data (which is provided back when the | |
278 | * rx callback is invoked), and an address they want to bind with the | |
279 | * callback. If @addr is RPMSG_ADDR_ANY, then rpmsg_create_ept will | |
280 | * dynamically assign them an available rpmsg address (drivers should have | |
281 | * a very good reason why not to always use RPMSG_ADDR_ANY here). | |
282 | * | |
283 | * Returns a pointer to the endpoint on success, or NULL on error. | |
284 | */ | |
285 | struct rpmsg_endpoint *rpmsg_create_ept(struct rpmsg_channel *rpdev, | |
286 | rpmsg_rx_cb_t cb, void *priv, u32 addr) | |
287 | { | |
288 | return __rpmsg_create_ept(rpdev->vrp, rpdev, cb, priv, addr); | |
289 | } | |
290 | EXPORT_SYMBOL(rpmsg_create_ept); | |
291 | ||
292 | /** | |
293 | * rpmsg_destroy_ept() - destroy an existing rpmsg endpoint | |
294 | * @ept: endpoing to destroy | |
295 | * | |
296 | * Should be used by drivers to destroy an rpmsg endpoint previously | |
297 | * created with rpmsg_create_ept(). | |
298 | */ | |
299 | void rpmsg_destroy_ept(struct rpmsg_endpoint *ept) | |
300 | { | |
301 | struct virtproc_info *vrp = ept->rpdev->vrp; | |
302 | ||
303 | mutex_lock(&vrp->endpoints_lock); | |
304 | idr_remove(&vrp->endpoints, ept->addr); | |
305 | mutex_unlock(&vrp->endpoints_lock); | |
306 | ||
307 | kfree(ept); | |
308 | } | |
309 | EXPORT_SYMBOL(rpmsg_destroy_ept); | |
310 | ||
311 | /* | |
312 | * when an rpmsg driver is probed with a channel, we seamlessly create | |
313 | * it an endpoint, binding its rx callback to a unique local rpmsg | |
314 | * address. | |
315 | * | |
316 | * if we need to, we also announce about this channel to the remote | |
317 | * processor (needed in case the driver is exposing an rpmsg service). | |
318 | */ | |
319 | static int rpmsg_dev_probe(struct device *dev) | |
320 | { | |
321 | struct rpmsg_channel *rpdev = to_rpmsg_channel(dev); | |
322 | struct rpmsg_driver *rpdrv = to_rpmsg_driver(rpdev->dev.driver); | |
323 | struct virtproc_info *vrp = rpdev->vrp; | |
324 | struct rpmsg_endpoint *ept; | |
325 | int err; | |
326 | ||
327 | ept = rpmsg_create_ept(rpdev, rpdrv->callback, NULL, rpdev->src); | |
328 | if (!ept) { | |
329 | dev_err(dev, "failed to create endpoint\n"); | |
330 | err = -ENOMEM; | |
331 | goto out; | |
332 | } | |
333 | ||
334 | rpdev->ept = ept; | |
335 | rpdev->src = ept->addr; | |
336 | ||
337 | err = rpdrv->probe(rpdev); | |
338 | if (err) { | |
339 | dev_err(dev, "%s: failed: %d\n", __func__, err); | |
340 | rpmsg_destroy_ept(ept); | |
341 | goto out; | |
342 | } | |
343 | ||
344 | /* need to tell remote processor's name service about this channel ? */ | |
345 | if (rpdev->announce && | |
346 | virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) { | |
347 | struct rpmsg_ns_msg nsm; | |
348 | ||
349 | strncpy(nsm.name, rpdev->id.name, RPMSG_NAME_SIZE); | |
350 | nsm.addr = rpdev->src; | |
351 | nsm.flags = RPMSG_NS_CREATE; | |
352 | ||
353 | err = rpmsg_sendto(rpdev, &nsm, sizeof(nsm), RPMSG_NS_ADDR); | |
354 | if (err) | |
355 | dev_err(dev, "failed to announce service %d\n", err); | |
356 | } | |
357 | ||
358 | out: | |
359 | return err; | |
360 | } | |
361 | ||
362 | static int rpmsg_dev_remove(struct device *dev) | |
363 | { | |
364 | struct rpmsg_channel *rpdev = to_rpmsg_channel(dev); | |
365 | struct rpmsg_driver *rpdrv = to_rpmsg_driver(rpdev->dev.driver); | |
366 | struct virtproc_info *vrp = rpdev->vrp; | |
367 | int err = 0; | |
368 | ||
369 | /* tell remote processor's name service we're removing this channel */ | |
370 | if (rpdev->announce && | |
371 | virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) { | |
372 | struct rpmsg_ns_msg nsm; | |
373 | ||
374 | strncpy(nsm.name, rpdev->id.name, RPMSG_NAME_SIZE); | |
375 | nsm.addr = rpdev->src; | |
376 | nsm.flags = RPMSG_NS_DESTROY; | |
377 | ||
378 | err = rpmsg_sendto(rpdev, &nsm, sizeof(nsm), RPMSG_NS_ADDR); | |
379 | if (err) | |
380 | dev_err(dev, "failed to announce service %d\n", err); | |
381 | } | |
382 | ||
383 | rpdrv->remove(rpdev); | |
384 | ||
385 | rpmsg_destroy_ept(rpdev->ept); | |
386 | ||
387 | return err; | |
388 | } | |
389 | ||
390 | static struct bus_type rpmsg_bus = { | |
391 | .name = "rpmsg", | |
392 | .match = rpmsg_dev_match, | |
393 | .dev_attrs = rpmsg_dev_attrs, | |
394 | .uevent = rpmsg_uevent, | |
395 | .probe = rpmsg_dev_probe, | |
396 | .remove = rpmsg_dev_remove, | |
397 | }; | |
398 | ||
399 | /** | |
400 | * register_rpmsg_driver() - register an rpmsg driver with the rpmsg bus | |
401 | * @rpdrv: pointer to a struct rpmsg_driver | |
402 | * | |
403 | * Returns 0 on success, and an appropriate error value on failure. | |
404 | */ | |
405 | int register_rpmsg_driver(struct rpmsg_driver *rpdrv) | |
406 | { | |
407 | rpdrv->drv.bus = &rpmsg_bus; | |
408 | return driver_register(&rpdrv->drv); | |
409 | } | |
410 | EXPORT_SYMBOL(register_rpmsg_driver); | |
411 | ||
412 | /** | |
413 | * unregister_rpmsg_driver() - unregister an rpmsg driver from the rpmsg bus | |
414 | * @rpdrv: pointer to a struct rpmsg_driver | |
415 | * | |
416 | * Returns 0 on success, and an appropriate error value on failure. | |
417 | */ | |
418 | void unregister_rpmsg_driver(struct rpmsg_driver *rpdrv) | |
419 | { | |
420 | driver_unregister(&rpdrv->drv); | |
421 | } | |
422 | EXPORT_SYMBOL(unregister_rpmsg_driver); | |
423 | ||
424 | static void rpmsg_release_device(struct device *dev) | |
425 | { | |
426 | struct rpmsg_channel *rpdev = to_rpmsg_channel(dev); | |
427 | ||
428 | kfree(rpdev); | |
429 | } | |
430 | ||
431 | /* | |
432 | * match an rpmsg channel with a channel info struct. | |
433 | * this is used to make sure we're not creating rpmsg devices for channels | |
434 | * that already exist. | |
435 | */ | |
436 | static int rpmsg_channel_match(struct device *dev, void *data) | |
437 | { | |
438 | struct rpmsg_channel_info *chinfo = data; | |
439 | struct rpmsg_channel *rpdev = to_rpmsg_channel(dev); | |
440 | ||
441 | if (chinfo->src != RPMSG_ADDR_ANY && chinfo->src != rpdev->src) | |
442 | return 0; | |
443 | ||
444 | if (chinfo->dst != RPMSG_ADDR_ANY && chinfo->dst != rpdev->dst) | |
445 | return 0; | |
446 | ||
447 | if (strncmp(chinfo->name, rpdev->id.name, RPMSG_NAME_SIZE)) | |
448 | return 0; | |
449 | ||
450 | /* found a match ! */ | |
451 | return 1; | |
452 | } | |
453 | ||
454 | /* | |
455 | * create an rpmsg channel using its name and address info. | |
456 | * this function will be used to create both static and dynamic | |
457 | * channels. | |
458 | */ | |
459 | static struct rpmsg_channel *rpmsg_create_channel(struct virtproc_info *vrp, | |
460 | struct rpmsg_channel_info *chinfo) | |
461 | { | |
462 | struct rpmsg_channel *rpdev; | |
463 | struct device *tmp, *dev = &vrp->vdev->dev; | |
464 | int ret; | |
465 | ||
466 | /* make sure a similar channel doesn't already exist */ | |
467 | tmp = device_find_child(dev, chinfo, rpmsg_channel_match); | |
468 | if (tmp) { | |
469 | /* decrement the matched device's refcount back */ | |
470 | put_device(tmp); | |
471 | dev_err(dev, "channel %s:%x:%x already exist\n", | |
472 | chinfo->name, chinfo->src, chinfo->dst); | |
473 | return NULL; | |
474 | } | |
475 | ||
476 | rpdev = kzalloc(sizeof(struct rpmsg_channel), GFP_KERNEL); | |
477 | if (!rpdev) { | |
478 | pr_err("kzalloc failed\n"); | |
479 | return NULL; | |
480 | } | |
481 | ||
482 | rpdev->vrp = vrp; | |
483 | rpdev->src = chinfo->src; | |
484 | rpdev->dst = chinfo->dst; | |
485 | ||
486 | /* | |
487 | * rpmsg server channels has predefined local address (for now), | |
488 | * and their existence needs to be announced remotely | |
489 | */ | |
490 | rpdev->announce = rpdev->src != RPMSG_ADDR_ANY ? true : false; | |
491 | ||
492 | strncpy(rpdev->id.name, chinfo->name, RPMSG_NAME_SIZE); | |
493 | ||
494 | /* very simple device indexing plumbing which is enough for now */ | |
495 | dev_set_name(&rpdev->dev, "rpmsg%d", rpmsg_dev_index++); | |
496 | ||
497 | rpdev->dev.parent = &vrp->vdev->dev; | |
498 | rpdev->dev.bus = &rpmsg_bus; | |
499 | rpdev->dev.release = rpmsg_release_device; | |
500 | ||
501 | ret = device_register(&rpdev->dev); | |
502 | if (ret) { | |
503 | dev_err(dev, "device_register failed: %d\n", ret); | |
504 | put_device(&rpdev->dev); | |
505 | return NULL; | |
506 | } | |
507 | ||
508 | return rpdev; | |
509 | } | |
510 | ||
511 | /* | |
512 | * find an existing channel using its name + address properties, | |
513 | * and destroy it | |
514 | */ | |
515 | static int rpmsg_destroy_channel(struct virtproc_info *vrp, | |
516 | struct rpmsg_channel_info *chinfo) | |
517 | { | |
518 | struct virtio_device *vdev = vrp->vdev; | |
519 | struct device *dev; | |
520 | ||
521 | dev = device_find_child(&vdev->dev, chinfo, rpmsg_channel_match); | |
522 | if (!dev) | |
523 | return -EINVAL; | |
524 | ||
525 | device_unregister(dev); | |
526 | ||
527 | put_device(dev); | |
528 | ||
529 | return 0; | |
530 | } | |
531 | ||
532 | /* super simple buffer "allocator" that is just enough for now */ | |
533 | static void *get_a_tx_buf(struct virtproc_info *vrp) | |
534 | { | |
535 | unsigned int len; | |
536 | void *ret; | |
537 | ||
538 | /* support multiple concurrent senders */ | |
539 | mutex_lock(&vrp->tx_lock); | |
540 | ||
541 | /* | |
542 | * either pick the next unused tx buffer | |
543 | * (half of our buffers are used for sending messages) | |
544 | */ | |
545 | if (vrp->last_sbuf < RPMSG_NUM_BUFS / 2) | |
546 | ret = vrp->sbufs + RPMSG_BUF_SIZE * vrp->last_sbuf++; | |
547 | /* or recycle a used one */ | |
548 | else | |
549 | ret = virtqueue_get_buf(vrp->svq, &len); | |
550 | ||
551 | mutex_unlock(&vrp->tx_lock); | |
552 | ||
553 | return ret; | |
554 | } | |
555 | ||
556 | /** | |
557 | * rpmsg_upref_sleepers() - enable "tx-complete" interrupts, if needed | |
558 | * @vrp: virtual remote processor state | |
559 | * | |
560 | * This function is called before a sender is blocked, waiting for | |
561 | * a tx buffer to become available. | |
562 | * | |
563 | * If we already have blocking senders, this function merely increases | |
564 | * the "sleepers" reference count, and exits. | |
565 | * | |
566 | * Otherwise, if this is the first sender to block, we also enable | |
567 | * virtio's tx callbacks, so we'd be immediately notified when a tx | |
568 | * buffer is consumed (we rely on virtio's tx callback in order | |
569 | * to wake up sleeping senders as soon as a tx buffer is used by the | |
570 | * remote processor). | |
571 | */ | |
572 | static void rpmsg_upref_sleepers(struct virtproc_info *vrp) | |
573 | { | |
574 | /* support multiple concurrent senders */ | |
575 | mutex_lock(&vrp->tx_lock); | |
576 | ||
577 | /* are we the first sleeping context waiting for tx buffers ? */ | |
578 | if (atomic_inc_return(&vrp->sleepers) == 1) | |
579 | /* enable "tx-complete" interrupts before dozing off */ | |
580 | virtqueue_enable_cb(vrp->svq); | |
581 | ||
582 | mutex_unlock(&vrp->tx_lock); | |
583 | } | |
584 | ||
585 | /** | |
586 | * rpmsg_downref_sleepers() - disable "tx-complete" interrupts, if needed | |
587 | * @vrp: virtual remote processor state | |
588 | * | |
589 | * This function is called after a sender, that waited for a tx buffer | |
590 | * to become available, is unblocked. | |
591 | * | |
592 | * If we still have blocking senders, this function merely decreases | |
593 | * the "sleepers" reference count, and exits. | |
594 | * | |
595 | * Otherwise, if there are no more blocking senders, we also disable | |
596 | * virtio's tx callbacks, to avoid the overhead incurred with handling | |
597 | * those (now redundant) interrupts. | |
598 | */ | |
599 | static void rpmsg_downref_sleepers(struct virtproc_info *vrp) | |
600 | { | |
601 | /* support multiple concurrent senders */ | |
602 | mutex_lock(&vrp->tx_lock); | |
603 | ||
604 | /* are we the last sleeping context waiting for tx buffers ? */ | |
605 | if (atomic_dec_and_test(&vrp->sleepers)) | |
606 | /* disable "tx-complete" interrupts */ | |
607 | virtqueue_disable_cb(vrp->svq); | |
608 | ||
609 | mutex_unlock(&vrp->tx_lock); | |
610 | } | |
611 | ||
612 | /** | |
613 | * rpmsg_send_offchannel_raw() - send a message across to the remote processor | |
614 | * @rpdev: the rpmsg channel | |
615 | * @src: source address | |
616 | * @dst: destination address | |
617 | * @data: payload of message | |
618 | * @len: length of payload | |
619 | * @wait: indicates whether caller should block in case no TX buffers available | |
620 | * | |
621 | * This function is the base implementation for all of the rpmsg sending API. | |
622 | * | |
623 | * It will send @data of length @len to @dst, and say it's from @src. The | |
624 | * message will be sent to the remote processor which the @rpdev channel | |
625 | * belongs to. | |
626 | * | |
627 | * The message is sent using one of the TX buffers that are available for | |
628 | * communication with this remote processor. | |
629 | * | |
630 | * If @wait is true, the caller will be blocked until either a TX buffer is | |
631 | * available, or 15 seconds elapses (we don't want callers to | |
632 | * sleep indefinitely due to misbehaving remote processors), and in that | |
633 | * case -ERESTARTSYS is returned. The number '15' itself was picked | |
634 | * arbitrarily; there's little point in asking drivers to provide a timeout | |
635 | * value themselves. | |
636 | * | |
637 | * Otherwise, if @wait is false, and there are no TX buffers available, | |
638 | * the function will immediately fail, and -ENOMEM will be returned. | |
639 | * | |
640 | * Normally drivers shouldn't use this function directly; instead, drivers | |
641 | * should use the appropriate rpmsg_{try}send{to, _offchannel} API | |
642 | * (see include/linux/rpmsg.h). | |
643 | * | |
644 | * Returns 0 on success and an appropriate error value on failure. | |
645 | */ | |
646 | int rpmsg_send_offchannel_raw(struct rpmsg_channel *rpdev, u32 src, u32 dst, | |
647 | void *data, int len, bool wait) | |
648 | { | |
649 | struct virtproc_info *vrp = rpdev->vrp; | |
650 | struct device *dev = &rpdev->dev; | |
651 | struct scatterlist sg; | |
652 | struct rpmsg_hdr *msg; | |
653 | int err; | |
654 | ||
655 | /* bcasting isn't allowed */ | |
656 | if (src == RPMSG_ADDR_ANY || dst == RPMSG_ADDR_ANY) { | |
657 | dev_err(dev, "invalid addr (src 0x%x, dst 0x%x)\n", src, dst); | |
658 | return -EINVAL; | |
659 | } | |
660 | ||
661 | /* | |
662 | * We currently use fixed-sized buffers, and therefore the payload | |
663 | * length is limited. | |
664 | * | |
665 | * One of the possible improvements here is either to support | |
666 | * user-provided buffers (and then we can also support zero-copy | |
667 | * messaging), or to improve the buffer allocator, to support | |
668 | * variable-length buffer sizes. | |
669 | */ | |
670 | if (len > RPMSG_BUF_SIZE - sizeof(struct rpmsg_hdr)) { | |
671 | dev_err(dev, "message is too big (%d)\n", len); | |
672 | return -EMSGSIZE; | |
673 | } | |
674 | ||
675 | /* grab a buffer */ | |
676 | msg = get_a_tx_buf(vrp); | |
677 | if (!msg && !wait) | |
678 | return -ENOMEM; | |
679 | ||
680 | /* no free buffer ? wait for one (but bail after 15 seconds) */ | |
681 | while (!msg) { | |
682 | /* enable "tx-complete" interrupts, if not already enabled */ | |
683 | rpmsg_upref_sleepers(vrp); | |
684 | ||
685 | /* | |
686 | * sleep until a free buffer is available or 15 secs elapse. | |
687 | * the timeout period is not configurable because there's | |
688 | * little point in asking drivers to specify that. | |
689 | * if later this happens to be required, it'd be easy to add. | |
690 | */ | |
691 | err = wait_event_interruptible_timeout(vrp->sendq, | |
692 | (msg = get_a_tx_buf(vrp)), | |
693 | msecs_to_jiffies(15000)); | |
694 | ||
695 | /* disable "tx-complete" interrupts if we're the last sleeper */ | |
696 | rpmsg_downref_sleepers(vrp); | |
697 | ||
698 | /* timeout ? */ | |
699 | if (!err) { | |
700 | dev_err(dev, "timeout waiting for a tx buffer\n"); | |
701 | return -ERESTARTSYS; | |
702 | } | |
703 | } | |
704 | ||
705 | msg->len = len; | |
706 | msg->flags = 0; | |
707 | msg->src = src; | |
708 | msg->dst = dst; | |
709 | msg->reserved = 0; | |
710 | memcpy(msg->data, data, len); | |
711 | ||
712 | dev_dbg(dev, "TX From 0x%x, To 0x%x, Len %d, Flags %d, Reserved %d\n", | |
713 | msg->src, msg->dst, msg->len, | |
714 | msg->flags, msg->reserved); | |
715 | print_hex_dump(KERN_DEBUG, "rpmsg_virtio TX: ", DUMP_PREFIX_NONE, 16, 1, | |
716 | msg, sizeof(*msg) + msg->len, true); | |
717 | ||
718 | sg_init_one(&sg, msg, sizeof(*msg) + len); | |
719 | ||
720 | mutex_lock(&vrp->tx_lock); | |
721 | ||
722 | /* add message to the remote processor's virtqueue */ | |
b719587e | 723 | err = virtqueue_add_buf(vrp->svq, &sg, 1, 0, msg, GFP_KERNEL); |
bcabbcca OBC |
724 | if (err < 0) { |
725 | /* | |
726 | * need to reclaim the buffer here, otherwise it's lost | |
727 | * (memory won't leak, but rpmsg won't use it again for TX). | |
728 | * this will wait for a buffer management overhaul. | |
729 | */ | |
b719587e | 730 | dev_err(dev, "virtqueue_add_buf failed: %d\n", err); |
bcabbcca OBC |
731 | goto out; |
732 | } | |
733 | ||
734 | /* tell the remote processor it has a pending message to read */ | |
735 | virtqueue_kick(vrp->svq); | |
736 | ||
737 | err = 0; | |
738 | out: | |
739 | mutex_unlock(&vrp->tx_lock); | |
740 | return err; | |
741 | } | |
742 | EXPORT_SYMBOL(rpmsg_send_offchannel_raw); | |
743 | ||
744 | /* called when an rx buffer is used, and it's time to digest a message */ | |
745 | static void rpmsg_recv_done(struct virtqueue *rvq) | |
746 | { | |
747 | struct rpmsg_hdr *msg; | |
748 | unsigned int len; | |
749 | struct rpmsg_endpoint *ept; | |
750 | struct scatterlist sg; | |
751 | struct virtproc_info *vrp = rvq->vdev->priv; | |
752 | struct device *dev = &rvq->vdev->dev; | |
753 | int err; | |
754 | ||
755 | msg = virtqueue_get_buf(rvq, &len); | |
756 | if (!msg) { | |
757 | dev_err(dev, "uhm, incoming signal, but no used buffer ?\n"); | |
758 | return; | |
759 | } | |
760 | ||
761 | dev_dbg(dev, "From: 0x%x, To: 0x%x, Len: %d, Flags: %d, Reserved: %d\n", | |
762 | msg->src, msg->dst, msg->len, | |
763 | msg->flags, msg->reserved); | |
764 | print_hex_dump(KERN_DEBUG, "rpmsg_virtio RX: ", DUMP_PREFIX_NONE, 16, 1, | |
765 | msg, sizeof(*msg) + msg->len, true); | |
766 | ||
767 | /* use the dst addr to fetch the callback of the appropriate user */ | |
768 | mutex_lock(&vrp->endpoints_lock); | |
769 | ept = idr_find(&vrp->endpoints, msg->dst); | |
770 | mutex_unlock(&vrp->endpoints_lock); | |
771 | ||
772 | if (ept && ept->cb) | |
773 | ept->cb(ept->rpdev, msg->data, msg->len, ept->priv, msg->src); | |
774 | else | |
775 | dev_warn(dev, "msg received with no recepient\n"); | |
776 | ||
777 | sg_init_one(&sg, msg, sizeof(*msg) + len); | |
778 | ||
779 | /* add the buffer back to the remote processor's virtqueue */ | |
b719587e | 780 | err = virtqueue_add_buf(vrp->rvq, &sg, 0, 1, msg, GFP_KERNEL); |
bcabbcca OBC |
781 | if (err < 0) { |
782 | dev_err(dev, "failed to add a virtqueue buffer: %d\n", err); | |
783 | return; | |
784 | } | |
785 | ||
786 | /* tell the remote processor we added another available rx buffer */ | |
787 | virtqueue_kick(vrp->rvq); | |
788 | } | |
789 | ||
790 | /* | |
791 | * This is invoked whenever the remote processor completed processing | |
792 | * a TX msg we just sent it, and the buffer is put back to the used ring. | |
793 | * | |
794 | * Normally, though, we suppress this "tx complete" interrupt in order to | |
795 | * avoid the incurred overhead. | |
796 | */ | |
797 | static void rpmsg_xmit_done(struct virtqueue *svq) | |
798 | { | |
799 | struct virtproc_info *vrp = svq->vdev->priv; | |
800 | ||
801 | dev_dbg(&svq->vdev->dev, "%s\n", __func__); | |
802 | ||
803 | /* wake up potential senders that are waiting for a tx buffer */ | |
804 | wake_up_interruptible(&vrp->sendq); | |
805 | } | |
806 | ||
807 | /* invoked when a name service announcement arrives */ | |
808 | static void rpmsg_ns_cb(struct rpmsg_channel *rpdev, void *data, int len, | |
809 | void *priv, u32 src) | |
810 | { | |
811 | struct rpmsg_ns_msg *msg = data; | |
812 | struct rpmsg_channel *newch; | |
813 | struct rpmsg_channel_info chinfo; | |
814 | struct virtproc_info *vrp = priv; | |
815 | struct device *dev = &vrp->vdev->dev; | |
816 | int ret; | |
817 | ||
818 | print_hex_dump(KERN_DEBUG, "NS announcement: ", | |
819 | DUMP_PREFIX_NONE, 16, 1, | |
820 | data, len, true); | |
821 | ||
822 | if (len != sizeof(*msg)) { | |
823 | dev_err(dev, "malformed ns msg (%d)\n", len); | |
824 | return; | |
825 | } | |
826 | ||
827 | /* | |
828 | * the name service ept does _not_ belong to a real rpmsg channel, | |
829 | * and is handled by the rpmsg bus itself. | |
830 | * for sanity reasons, make sure a valid rpdev has _not_ sneaked | |
831 | * in somehow. | |
832 | */ | |
833 | if (rpdev) { | |
834 | dev_err(dev, "anomaly: ns ept has an rpdev handle\n"); | |
835 | return; | |
836 | } | |
837 | ||
838 | /* don't trust the remote processor for null terminating the name */ | |
839 | msg->name[RPMSG_NAME_SIZE - 1] = '\0'; | |
840 | ||
841 | dev_info(dev, "%sing channel %s addr 0x%x\n", | |
842 | msg->flags & RPMSG_NS_DESTROY ? "destroy" : "creat", | |
843 | msg->name, msg->addr); | |
844 | ||
845 | strncpy(chinfo.name, msg->name, sizeof(chinfo.name)); | |
846 | chinfo.src = RPMSG_ADDR_ANY; | |
847 | chinfo.dst = msg->addr; | |
848 | ||
849 | if (msg->flags & RPMSG_NS_DESTROY) { | |
850 | ret = rpmsg_destroy_channel(vrp, &chinfo); | |
851 | if (ret) | |
852 | dev_err(dev, "rpmsg_destroy_channel failed: %d\n", ret); | |
853 | } else { | |
854 | newch = rpmsg_create_channel(vrp, &chinfo); | |
855 | if (!newch) | |
856 | dev_err(dev, "rpmsg_create_channel failed\n"); | |
857 | } | |
858 | } | |
859 | ||
860 | static int rpmsg_probe(struct virtio_device *vdev) | |
861 | { | |
862 | vq_callback_t *vq_cbs[] = { rpmsg_recv_done, rpmsg_xmit_done }; | |
863 | const char *names[] = { "input", "output" }; | |
864 | struct virtqueue *vqs[2]; | |
865 | struct virtproc_info *vrp; | |
866 | void *bufs_va; | |
867 | int err = 0, i; | |
868 | ||
869 | vrp = kzalloc(sizeof(*vrp), GFP_KERNEL); | |
870 | if (!vrp) | |
871 | return -ENOMEM; | |
872 | ||
873 | vrp->vdev = vdev; | |
874 | ||
875 | idr_init(&vrp->endpoints); | |
876 | mutex_init(&vrp->endpoints_lock); | |
877 | mutex_init(&vrp->tx_lock); | |
878 | init_waitqueue_head(&vrp->sendq); | |
879 | ||
880 | /* We expect two virtqueues, rx and tx (and in this order) */ | |
881 | err = vdev->config->find_vqs(vdev, 2, vqs, vq_cbs, names); | |
882 | if (err) | |
883 | goto free_vrp; | |
884 | ||
885 | vrp->rvq = vqs[0]; | |
886 | vrp->svq = vqs[1]; | |
887 | ||
888 | /* allocate coherent memory for the buffers */ | |
889 | bufs_va = dma_alloc_coherent(vdev->dev.parent, RPMSG_TOTAL_BUF_SPACE, | |
890 | &vrp->bufs_dma, GFP_KERNEL); | |
891 | if (!bufs_va) | |
892 | goto vqs_del; | |
893 | ||
894 | dev_dbg(&vdev->dev, "buffers: va %p, dma 0x%x\n", bufs_va, | |
895 | vrp->bufs_dma); | |
896 | ||
897 | /* half of the buffers is dedicated for RX */ | |
898 | vrp->rbufs = bufs_va; | |
899 | ||
900 | /* and half is dedicated for TX */ | |
901 | vrp->sbufs = bufs_va + RPMSG_TOTAL_BUF_SPACE / 2; | |
902 | ||
903 | /* set up the receive buffers */ | |
904 | for (i = 0; i < RPMSG_NUM_BUFS / 2; i++) { | |
905 | struct scatterlist sg; | |
906 | void *cpu_addr = vrp->rbufs + i * RPMSG_BUF_SIZE; | |
907 | ||
908 | sg_init_one(&sg, cpu_addr, RPMSG_BUF_SIZE); | |
909 | ||
b719587e | 910 | err = virtqueue_add_buf(vrp->rvq, &sg, 0, 1, cpu_addr, |
bcabbcca OBC |
911 | GFP_KERNEL); |
912 | WARN_ON(err < 0); /* sanity check; this can't really happen */ | |
913 | } | |
914 | ||
915 | /* suppress "tx-complete" interrupts */ | |
916 | virtqueue_disable_cb(vrp->svq); | |
917 | ||
918 | vdev->priv = vrp; | |
919 | ||
920 | /* if supported by the remote processor, enable the name service */ | |
921 | if (virtio_has_feature(vdev, VIRTIO_RPMSG_F_NS)) { | |
922 | /* a dedicated endpoint handles the name service msgs */ | |
923 | vrp->ns_ept = __rpmsg_create_ept(vrp, NULL, rpmsg_ns_cb, | |
924 | vrp, RPMSG_NS_ADDR); | |
925 | if (!vrp->ns_ept) { | |
926 | dev_err(&vdev->dev, "failed to create the ns ept\n"); | |
927 | err = -ENOMEM; | |
928 | goto free_coherent; | |
929 | } | |
930 | } | |
931 | ||
932 | /* tell the remote processor it can start sending messages */ | |
933 | virtqueue_kick(vrp->rvq); | |
934 | ||
935 | dev_info(&vdev->dev, "rpmsg host is online\n"); | |
936 | ||
937 | return 0; | |
938 | ||
939 | free_coherent: | |
940 | dma_free_coherent(vdev->dev.parent, RPMSG_TOTAL_BUF_SPACE, bufs_va, | |
941 | vrp->bufs_dma); | |
942 | vqs_del: | |
943 | vdev->config->del_vqs(vrp->vdev); | |
944 | free_vrp: | |
945 | kfree(vrp); | |
946 | return err; | |
947 | } | |
948 | ||
949 | static int rpmsg_remove_device(struct device *dev, void *data) | |
950 | { | |
951 | device_unregister(dev); | |
952 | ||
953 | return 0; | |
954 | } | |
955 | ||
956 | static void __devexit rpmsg_remove(struct virtio_device *vdev) | |
957 | { | |
958 | struct virtproc_info *vrp = vdev->priv; | |
959 | int ret; | |
960 | ||
961 | vdev->config->reset(vdev); | |
962 | ||
963 | ret = device_for_each_child(&vdev->dev, NULL, rpmsg_remove_device); | |
964 | if (ret) | |
965 | dev_warn(&vdev->dev, "can't remove rpmsg device: %d\n", ret); | |
966 | ||
967 | idr_remove_all(&vrp->endpoints); | |
968 | idr_destroy(&vrp->endpoints); | |
969 | ||
970 | vdev->config->del_vqs(vrp->vdev); | |
971 | ||
972 | dma_free_coherent(vdev->dev.parent, RPMSG_TOTAL_BUF_SPACE, | |
973 | vrp->rbufs, vrp->bufs_dma); | |
974 | ||
975 | kfree(vrp); | |
976 | } | |
977 | ||
978 | static struct virtio_device_id id_table[] = { | |
979 | { VIRTIO_ID_RPMSG, VIRTIO_DEV_ANY_ID }, | |
980 | { 0 }, | |
981 | }; | |
982 | ||
983 | static unsigned int features[] = { | |
984 | VIRTIO_RPMSG_F_NS, | |
985 | }; | |
986 | ||
987 | static struct virtio_driver virtio_ipc_driver = { | |
988 | .feature_table = features, | |
989 | .feature_table_size = ARRAY_SIZE(features), | |
990 | .driver.name = KBUILD_MODNAME, | |
991 | .driver.owner = THIS_MODULE, | |
992 | .id_table = id_table, | |
993 | .probe = rpmsg_probe, | |
994 | .remove = __devexit_p(rpmsg_remove), | |
995 | }; | |
996 | ||
997 | static int __init rpmsg_init(void) | |
998 | { | |
999 | int ret; | |
1000 | ||
1001 | ret = bus_register(&rpmsg_bus); | |
1002 | if (ret) { | |
1003 | pr_err("failed to register rpmsg bus: %d\n", ret); | |
1004 | return ret; | |
1005 | } | |
1006 | ||
1007 | ret = register_virtio_driver(&virtio_ipc_driver); | |
1008 | if (ret) { | |
1009 | pr_err("failed to register virtio driver: %d\n", ret); | |
1010 | bus_unregister(&rpmsg_bus); | |
1011 | } | |
1012 | ||
1013 | return ret; | |
1014 | } | |
1015 | module_init(rpmsg_init); | |
1016 | ||
1017 | static void __exit rpmsg_fini(void) | |
1018 | { | |
1019 | unregister_virtio_driver(&virtio_ipc_driver); | |
1020 | bus_unregister(&rpmsg_bus); | |
1021 | } | |
1022 | module_exit(rpmsg_fini); | |
1023 | ||
1024 | MODULE_DEVICE_TABLE(virtio, id_table); | |
1025 | MODULE_DESCRIPTION("Virtio-based remote processor messaging bus"); | |
1026 | MODULE_LICENSE("GPL v2"); |