]>
Commit | Line | Data |
---|---|---|
9c92ab61 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
ac8954a4 OBC |
2 | /* |
3 | * Remote processor messaging transport (OMAP platform-specific bits) | |
4 | * | |
5 | * Copyright (C) 2011 Texas Instruments, Inc. | |
6 | * Copyright (C) 2011 Google, Inc. | |
7 | * | |
8 | * Ohad Ben-Cohen <ohad@wizery.com> | |
9 | * Brian Swetland <swetland@google.com> | |
ac8954a4 OBC |
10 | */ |
11 | ||
0a0f0d8b | 12 | #include <linux/dma-map-ops.h> |
ac8954a4 | 13 | #include <linux/export.h> |
086d0872 | 14 | #include <linux/of_reserved_mem.h> |
ac8954a4 | 15 | #include <linux/remoteproc.h> |
ac8954a4 OBC |
16 | #include <linux/virtio.h> |
17 | #include <linux/virtio_config.h> | |
18 | #include <linux/virtio_ids.h> | |
19 | #include <linux/virtio_ring.h> | |
20 | #include <linux/err.h> | |
21 | #include <linux/kref.h> | |
22 | #include <linux/slab.h> | |
23 | ||
24 | #include "remoteproc_internal.h" | |
25 | ||
ac8954a4 | 26 | /* kick the remote processor, and let it know which virtqueue to poke at */ |
46f9c2b9 | 27 | static bool rproc_virtio_notify(struct virtqueue *vq) |
ac8954a4 | 28 | { |
7a186941 OBC |
29 | struct rproc_vring *rvring = vq->priv; |
30 | struct rproc *rproc = rvring->rvdev->rproc; | |
31 | int notifyid = rvring->notifyid; | |
ac8954a4 | 32 | |
b5ab5e24 | 33 | dev_dbg(&rproc->dev, "kicking vq index: %d\n", notifyid); |
ac8954a4 | 34 | |
7a186941 | 35 | rproc->ops->kick(rproc, notifyid); |
46f9c2b9 | 36 | return true; |
ac8954a4 OBC |
37 | } |
38 | ||
39 | /** | |
40 | * rproc_vq_interrupt() - tell remoteproc that a virtqueue is interrupted | |
41 | * @rproc: handle to the remote processor | |
7a186941 | 42 | * @notifyid: index of the signalled virtqueue (unique per this @rproc) |
ac8954a4 OBC |
43 | * |
44 | * This function should be called by the platform-specific rproc driver, | |
45 | * when the remote processor signals that a specific virtqueue has pending | |
46 | * messages available. | |
47 | * | |
f2867434 | 48 | * Return: IRQ_NONE if no message was found in the @notifyid virtqueue, |
ac8954a4 OBC |
49 | * and otherwise returns IRQ_HANDLED. |
50 | */ | |
7a186941 | 51 | irqreturn_t rproc_vq_interrupt(struct rproc *rproc, int notifyid) |
ac8954a4 | 52 | { |
7a186941 OBC |
53 | struct rproc_vring *rvring; |
54 | ||
b5ab5e24 | 55 | dev_dbg(&rproc->dev, "vq index %d is interrupted\n", notifyid); |
7a186941 OBC |
56 | |
57 | rvring = idr_find(&rproc->notifyids, notifyid); | |
58 | if (!rvring || !rvring->vq) | |
59 | return IRQ_NONE; | |
60 | ||
61 | return vring_interrupt(0, rvring->vq); | |
ac8954a4 OBC |
62 | } |
63 | EXPORT_SYMBOL(rproc_vq_interrupt); | |
64 | ||
65 | static struct virtqueue *rp_find_vq(struct virtio_device *vdev, | |
f145928d | 66 | unsigned int id, |
ac8954a4 | 67 | void (*callback)(struct virtqueue *vq), |
f94682dd | 68 | const char *name, bool ctx) |
ac8954a4 | 69 | { |
7a186941 | 70 | struct rproc_vdev *rvdev = vdev_to_rvdev(vdev); |
ac8954a4 | 71 | struct rproc *rproc = vdev_to_rproc(vdev); |
b5ab5e24 | 72 | struct device *dev = &rproc->dev; |
c6aed238 | 73 | struct rproc_mem_entry *mem; |
7a186941 | 74 | struct rproc_vring *rvring; |
c6aed238 | 75 | struct fw_rsc_vdev *rsc; |
ac8954a4 OBC |
76 | struct virtqueue *vq; |
77 | void *addr; | |
a863af5d | 78 | int len, size; |
ac8954a4 | 79 | |
7a186941 OBC |
80 | /* we're temporarily limited to two virtqueues per rvdev */ |
81 | if (id >= ARRAY_SIZE(rvdev->vring)) | |
82 | return ERR_PTR(-EINVAL); | |
83 | ||
6457f126 MT |
84 | if (!name) |
85 | return NULL; | |
86 | ||
c6aed238 LP |
87 | /* Search allocated memory region by name */ |
88 | mem = rproc_find_carveout_by_name(rproc, "vdev%dvring%d", rvdev->index, | |
89 | id); | |
90 | if (!mem || !mem->va) | |
91 | return ERR_PTR(-ENOMEM); | |
92 | ||
6db20ea8 | 93 | rvring = &rvdev->vring[id]; |
c6aed238 | 94 | addr = mem->va; |
7a186941 | 95 | len = rvring->len; |
ac8954a4 | 96 | |
7a186941 OBC |
97 | /* zero vring */ |
98 | size = vring_size(len, rvring->align); | |
99 | memset(addr, 0, size); | |
ac8954a4 | 100 | |
276ec993 | 101 | dev_dbg(dev, "vring%d: va %pK qsz %d notifyid %d\n", |
730f84ce | 102 | id, addr, len, rvring->notifyid); |
ac8954a4 | 103 | |
dd6da1c5 OBC |
104 | /* |
105 | * Create the new vq, and tell virtio we're not interested in | |
106 | * the 'weak' smp barriers, since we're talking with a real device. | |
107 | */ | |
f94682dd MT |
108 | vq = vring_new_virtqueue(id, len, rvring->align, vdev, false, ctx, |
109 | addr, rproc_virtio_notify, callback, name); | |
ac8954a4 | 110 | if (!vq) { |
b5ab5e24 | 111 | dev_err(dev, "vring_new_virtqueue %s failed\n", name); |
6db20ea8 | 112 | rproc_free_vring(rvring); |
7a186941 | 113 | return ERR_PTR(-ENOMEM); |
ac8954a4 OBC |
114 | } |
115 | ||
7a186941 OBC |
116 | rvring->vq = vq; |
117 | vq->priv = rvring; | |
ac8954a4 | 118 | |
c6aed238 LP |
119 | /* Update vring in resource table */ |
120 | rsc = (void *)rproc->table_ptr + rvdev->rsc_offset; | |
121 | rsc->vring[id].da = mem->da; | |
122 | ||
ac8954a4 | 123 | return vq; |
ac8954a4 OBC |
124 | } |
125 | ||
dab55bba | 126 | static void __rproc_virtio_del_vqs(struct virtio_device *vdev) |
ac8954a4 OBC |
127 | { |
128 | struct virtqueue *vq, *n; | |
7a186941 | 129 | struct rproc_vring *rvring; |
ac8954a4 OBC |
130 | |
131 | list_for_each_entry_safe(vq, n, &vdev->vqs, list) { | |
7a186941 OBC |
132 | rvring = vq->priv; |
133 | rvring->vq = NULL; | |
ac8954a4 | 134 | vring_del_virtqueue(vq); |
ac8954a4 | 135 | } |
ac8954a4 OBC |
136 | } |
137 | ||
dab55bba OBC |
138 | static void rproc_virtio_del_vqs(struct virtio_device *vdev) |
139 | { | |
dab55bba OBC |
140 | __rproc_virtio_del_vqs(vdev); |
141 | } | |
142 | ||
f145928d | 143 | static int rproc_virtio_find_vqs(struct virtio_device *vdev, unsigned int nvqs, |
730f84ce AS |
144 | struct virtqueue *vqs[], |
145 | vq_callback_t *callbacks[], | |
fb5e31d9 | 146 | const char * const names[], |
f94682dd | 147 | const bool * ctx, |
fb5e31d9 | 148 | struct irq_affinity *desc) |
ac8954a4 | 149 | { |
a229989d | 150 | int i, ret, queue_idx = 0; |
ac8954a4 | 151 | |
ac8954a4 | 152 | for (i = 0; i < nvqs; ++i) { |
a229989d WW |
153 | if (!names[i]) { |
154 | vqs[i] = NULL; | |
155 | continue; | |
156 | } | |
157 | ||
158 | vqs[i] = rp_find_vq(vdev, queue_idx++, callbacks[i], names[i], | |
f94682dd | 159 | ctx ? ctx[i] : false); |
ac8954a4 OBC |
160 | if (IS_ERR(vqs[i])) { |
161 | ret = PTR_ERR(vqs[i]); | |
162 | goto error; | |
163 | } | |
164 | } | |
165 | ||
166 | return 0; | |
167 | ||
168 | error: | |
dab55bba | 169 | __rproc_virtio_del_vqs(vdev); |
ac8954a4 OBC |
170 | return ret; |
171 | } | |
172 | ||
ac8954a4 OBC |
173 | static u8 rproc_virtio_get_status(struct virtio_device *vdev) |
174 | { | |
92b38f85 SB |
175 | struct rproc_vdev *rvdev = vdev_to_rvdev(vdev); |
176 | struct fw_rsc_vdev *rsc; | |
177 | ||
178 | rsc = (void *)rvdev->rproc->table_ptr + rvdev->rsc_offset; | |
179 | ||
180 | return rsc->status; | |
ac8954a4 OBC |
181 | } |
182 | ||
183 | static void rproc_virtio_set_status(struct virtio_device *vdev, u8 status) | |
184 | { | |
92b38f85 SB |
185 | struct rproc_vdev *rvdev = vdev_to_rvdev(vdev); |
186 | struct fw_rsc_vdev *rsc; | |
187 | ||
188 | rsc = (void *)rvdev->rproc->table_ptr + rvdev->rsc_offset; | |
189 | ||
190 | rsc->status = status; | |
7a186941 | 191 | dev_dbg(&vdev->dev, "status: %d\n", status); |
ac8954a4 OBC |
192 | } |
193 | ||
194 | static void rproc_virtio_reset(struct virtio_device *vdev) | |
195 | { | |
92b38f85 SB |
196 | struct rproc_vdev *rvdev = vdev_to_rvdev(vdev); |
197 | struct fw_rsc_vdev *rsc; | |
198 | ||
199 | rsc = (void *)rvdev->rproc->table_ptr + rvdev->rsc_offset; | |
200 | ||
201 | rsc->status = 0; | |
ac8954a4 OBC |
202 | dev_dbg(&vdev->dev, "reset !\n"); |
203 | } | |
204 | ||
205 | /* provide the vdev features as retrieved from the firmware */ | |
d0254773 | 206 | static u64 rproc_virtio_get_features(struct virtio_device *vdev) |
ac8954a4 | 207 | { |
7a186941 | 208 | struct rproc_vdev *rvdev = vdev_to_rvdev(vdev); |
92b38f85 SB |
209 | struct fw_rsc_vdev *rsc; |
210 | ||
211 | rsc = (void *)rvdev->rproc->table_ptr + rvdev->rsc_offset; | |
ac8954a4 | 212 | |
92b38f85 | 213 | return rsc->dfeatures; |
ac8954a4 OBC |
214 | } |
215 | ||
3a814fdf TB |
216 | static void rproc_transport_features(struct virtio_device *vdev) |
217 | { | |
218 | /* | |
219 | * Packed ring isn't enabled on remoteproc for now, | |
220 | * because remoteproc uses vring_new_virtqueue() which | |
221 | * creates virtio rings on preallocated memory. | |
222 | */ | |
223 | __virtio_clear_bit(vdev, VIRTIO_F_RING_PACKED); | |
224 | } | |
225 | ||
5c609a5e | 226 | static int rproc_virtio_finalize_features(struct virtio_device *vdev) |
ac8954a4 | 227 | { |
7a186941 | 228 | struct rproc_vdev *rvdev = vdev_to_rvdev(vdev); |
92b38f85 SB |
229 | struct fw_rsc_vdev *rsc; |
230 | ||
231 | rsc = (void *)rvdev->rproc->table_ptr + rvdev->rsc_offset; | |
ac8954a4 OBC |
232 | |
233 | /* Give virtio_ring a chance to accept features */ | |
234 | vring_transport_features(vdev); | |
235 | ||
3a814fdf TB |
236 | /* Give virtio_rproc a chance to accept features. */ |
237 | rproc_transport_features(vdev); | |
238 | ||
93d389f8 MT |
239 | /* Make sure we don't have any features > 32 bits! */ |
240 | BUG_ON((u32)vdev->features != vdev->features); | |
241 | ||
ac8954a4 OBC |
242 | /* |
243 | * Remember the finalized features of our vdev, and provide it | |
244 | * to the remote processor once it is powered on. | |
ac8954a4 | 245 | */ |
e16e12be | 246 | rsc->gfeatures = vdev->features; |
5c609a5e MT |
247 | |
248 | return 0; | |
92b38f85 SB |
249 | } |
250 | ||
f145928d AS |
251 | static void rproc_virtio_get(struct virtio_device *vdev, unsigned int offset, |
252 | void *buf, unsigned int len) | |
92b38f85 SB |
253 | { |
254 | struct rproc_vdev *rvdev = vdev_to_rvdev(vdev); | |
255 | struct fw_rsc_vdev *rsc; | |
256 | void *cfg; | |
257 | ||
258 | rsc = (void *)rvdev->rproc->table_ptr + rvdev->rsc_offset; | |
259 | cfg = &rsc->vring[rsc->num_of_vrings]; | |
260 | ||
261 | if (offset + len > rsc->config_len || offset + len < len) { | |
262 | dev_err(&vdev->dev, "rproc_virtio_get: access out of bounds\n"); | |
263 | return; | |
264 | } | |
265 | ||
266 | memcpy(buf, cfg + offset, len); | |
267 | } | |
268 | ||
f145928d AS |
269 | static void rproc_virtio_set(struct virtio_device *vdev, unsigned int offset, |
270 | const void *buf, unsigned int len) | |
92b38f85 SB |
271 | { |
272 | struct rproc_vdev *rvdev = vdev_to_rvdev(vdev); | |
273 | struct fw_rsc_vdev *rsc; | |
274 | void *cfg; | |
275 | ||
276 | rsc = (void *)rvdev->rproc->table_ptr + rvdev->rsc_offset; | |
277 | cfg = &rsc->vring[rsc->num_of_vrings]; | |
278 | ||
279 | if (offset + len > rsc->config_len || offset + len < len) { | |
280 | dev_err(&vdev->dev, "rproc_virtio_set: access out of bounds\n"); | |
281 | return; | |
282 | } | |
283 | ||
284 | memcpy(cfg + offset, buf, len); | |
ac8954a4 OBC |
285 | } |
286 | ||
93503932 | 287 | static const struct virtio_config_ops rproc_virtio_config_ops = { |
ac8954a4 OBC |
288 | .get_features = rproc_virtio_get_features, |
289 | .finalize_features = rproc_virtio_finalize_features, | |
290 | .find_vqs = rproc_virtio_find_vqs, | |
291 | .del_vqs = rproc_virtio_del_vqs, | |
292 | .reset = rproc_virtio_reset, | |
293 | .set_status = rproc_virtio_set_status, | |
294 | .get_status = rproc_virtio_get_status, | |
92b38f85 SB |
295 | .get = rproc_virtio_get, |
296 | .set = rproc_virtio_set, | |
ac8954a4 OBC |
297 | }; |
298 | ||
299 | /* | |
300 | * This function is called whenever vdev is released, and is responsible | |
7183a2a7 | 301 | * to decrement the remote processor's refcount which was taken when vdev was |
ac8954a4 OBC |
302 | * added. |
303 | * | |
304 | * Never call this function directly; it will be called by the driver | |
305 | * core when needed. | |
306 | */ | |
aab8d802 | 307 | static void rproc_virtio_dev_release(struct device *dev) |
ac8954a4 OBC |
308 | { |
309 | struct virtio_device *vdev = dev_to_virtio(dev); | |
6db20ea8 | 310 | struct rproc_vdev *rvdev = vdev_to_rvdev(vdev); |
ac8954a4 OBC |
311 | struct rproc *rproc = vdev_to_rproc(vdev); |
312 | ||
d4c036fe LP |
313 | kfree(vdev); |
314 | ||
aab8d802 | 315 | kref_put(&rvdev->refcount, rproc_vdev_release); |
6db20ea8 | 316 | |
7183a2a7 | 317 | put_device(&rproc->dev); |
ac8954a4 OBC |
318 | } |
319 | ||
320 | /** | |
7a186941 OBC |
321 | * rproc_add_virtio_dev() - register an rproc-induced virtio device |
322 | * @rvdev: the remote vdev | |
2e7d4c2c | 323 | * @id: the device type identification (used to match it with a driver). |
ac8954a4 | 324 | * |
7a186941 OBC |
325 | * This function registers a virtio device. This vdev's partent is |
326 | * the rproc device. | |
ac8954a4 | 327 | * |
f2867434 | 328 | * Return: 0 on success or an appropriate error value otherwise |
ac8954a4 | 329 | */ |
7a186941 | 330 | int rproc_add_virtio_dev(struct rproc_vdev *rvdev, int id) |
ac8954a4 | 331 | { |
7a186941 | 332 | struct rproc *rproc = rvdev->rproc; |
086d0872 | 333 | struct device *dev = &rvdev->dev; |
d4c036fe | 334 | struct virtio_device *vdev; |
086d0872 | 335 | struct rproc_mem_entry *mem; |
ac8954a4 OBC |
336 | int ret; |
337 | ||
791c13b7 NS |
338 | if (rproc->ops->kick == NULL) { |
339 | ret = -EINVAL; | |
2fb75cea | 340 | dev_err(dev, ".kick method not defined for %s\n", rproc->name); |
791c13b7 NS |
341 | goto out; |
342 | } | |
343 | ||
086d0872 LP |
344 | /* Try to find dedicated vdev buffer carveout */ |
345 | mem = rproc_find_carveout_by_name(rproc, "vdev%dbuffer", rvdev->index); | |
346 | if (mem) { | |
347 | phys_addr_t pa; | |
348 | ||
349 | if (mem->of_resm_idx != -1) { | |
350 | struct device_node *np = rproc->dev.parent->of_node; | |
351 | ||
352 | /* Associate reserved memory to vdev device */ | |
353 | ret = of_reserved_mem_device_init_by_idx(dev, np, | |
354 | mem->of_resm_idx); | |
355 | if (ret) { | |
356 | dev_err(dev, "Can't associate reserved memory\n"); | |
357 | goto out; | |
358 | } | |
359 | } else { | |
360 | if (mem->va) { | |
361 | dev_warn(dev, "vdev %d buffer already mapped\n", | |
362 | rvdev->index); | |
363 | pa = rproc_va_to_pa(mem->va); | |
364 | } else { | |
365 | /* Use dma address as carveout no memmapped yet */ | |
366 | pa = (phys_addr_t)mem->dma; | |
367 | } | |
368 | ||
369 | /* Associate vdev buffer memory pool to vdev subdev */ | |
370 | ret = dma_declare_coherent_memory(dev, pa, | |
371 | mem->da, | |
d664ce75 | 372 | mem->len); |
086d0872 LP |
373 | if (ret < 0) { |
374 | dev_err(dev, "Failed to associate buffer\n"); | |
375 | goto out; | |
376 | } | |
377 | } | |
db9178a4 TK |
378 | } else { |
379 | struct device_node *np = rproc->dev.parent->of_node; | |
380 | ||
381 | /* | |
382 | * If we don't have dedicated buffer, just attempt to re-assign | |
383 | * the reserved memory from our parent. A default memory-region | |
384 | * at index 0 from the parent's memory-regions is assigned for | |
385 | * the rvdev dev to allocate from. Failure is non-critical and | |
386 | * the allocations will fall back to global pools, so don't | |
387 | * check return value either. | |
388 | */ | |
389 | of_reserved_mem_device_init_by_idx(dev, np, 0); | |
086d0872 LP |
390 | } |
391 | ||
d4c036fe LP |
392 | /* Allocate virtio device */ |
393 | vdev = kzalloc(sizeof(*vdev), GFP_KERNEL); | |
394 | if (!vdev) { | |
395 | ret = -ENOMEM; | |
396 | goto out; | |
397 | } | |
7a186941 OBC |
398 | vdev->id.device = id, |
399 | vdev->config = &rproc_virtio_config_ops, | |
400 | vdev->dev.parent = dev; | |
aab8d802 | 401 | vdev->dev.release = rproc_virtio_dev_release; |
ac8954a4 OBC |
402 | |
403 | /* | |
404 | * We're indirectly making a non-temporary copy of the rproc pointer | |
405 | * here, because drivers probed with this vdev will indirectly | |
406 | * access the wrapping rproc. | |
407 | * | |
408 | * Therefore we must increment the rproc refcount here, and decrement | |
409 | * it _only_ when the vdev is released. | |
410 | */ | |
7183a2a7 | 411 | get_device(&rproc->dev); |
ac8954a4 | 412 | |
aab8d802 BA |
413 | /* Reference the vdev and vring allocations */ |
414 | kref_get(&rvdev->refcount); | |
415 | ||
7a186941 | 416 | ret = register_virtio_device(vdev); |
ac8954a4 | 417 | if (ret) { |
900a163e | 418 | put_device(&vdev->dev); |
ac8954a4 | 419 | dev_err(dev, "failed to register vdev: %d\n", ret); |
7a186941 | 420 | goto out; |
ac8954a4 OBC |
421 | } |
422 | ||
7a186941 OBC |
423 | dev_info(dev, "registered %s (type %d)\n", dev_name(&vdev->dev), id); |
424 | ||
425 | out: | |
ac8954a4 OBC |
426 | return ret; |
427 | } | |
428 | ||
429 | /** | |
7a186941 | 430 | * rproc_remove_virtio_dev() - remove an rproc-induced virtio device |
d4c036fe LP |
431 | * @dev: the virtio device |
432 | * @data: must be null | |
ac8954a4 | 433 | * |
7a186941 | 434 | * This function unregisters an existing virtio device. |
f2867434 SA |
435 | * |
436 | * Return: 0 | |
ac8954a4 | 437 | */ |
d4c036fe | 438 | int rproc_remove_virtio_dev(struct device *dev, void *data) |
ac8954a4 | 439 | { |
d4c036fe LP |
440 | struct virtio_device *vdev = dev_to_virtio(dev); |
441 | ||
442 | unregister_virtio_device(vdev); | |
443 | return 0; | |
ac8954a4 | 444 | } |