1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2011-2018, The Linux Foundation. All rights reserved.
3 // Copyright (c) 2018, Linaro Limited
5 #include <linux/completion.h>
6 #include <linux/device.h>
7 #include <linux/dma-buf.h>
8 #include <linux/dma-mapping.h>
10 #include <linux/list.h>
11 #include <linux/miscdevice.h>
12 #include <linux/module.h>
13 #include <linux/of_address.h>
15 #include <linux/sort.h>
16 #include <linux/of_platform.h>
17 #include <linux/rpmsg.h>
18 #include <linux/scatterlist.h>
19 #include <linux/slab.h>
20 #include <uapi/misc/fastrpc.h>
22 #define ADSP_DOMAIN_ID (0)
23 #define MDSP_DOMAIN_ID (1)
24 #define SDSP_DOMAIN_ID (2)
25 #define CDSP_DOMAIN_ID (3)
26 #define FASTRPC_DEV_MAX 4 /* adsp, mdsp, slpi, cdsp*/
27 #define FASTRPC_MAX_SESSIONS 9 /*8 compute, 1 cpz*/
28 #define FASTRPC_ALIGN 128
29 #define FASTRPC_MAX_FDLIST 16
30 #define FASTRPC_MAX_CRCLIST 64
31 #define FASTRPC_PHYS(p) ((p) & 0xffffffff)
32 #define FASTRPC_CTX_MAX (256)
33 #define FASTRPC_INIT_HANDLE 1
34 #define FASTRPC_CTXID_MASK (0xFF0)
35 #define INIT_FILELEN_MAX (64 * 1024 * 1024)
36 #define FASTRPC_DEVICE_NAME "fastrpc"
38 /* Retrives number of input buffers from the scalars parameter */
39 #define REMOTE_SCALARS_INBUFS(sc) (((sc) >> 16) & 0x0ff)
41 /* Retrives number of output buffers from the scalars parameter */
42 #define REMOTE_SCALARS_OUTBUFS(sc) (((sc) >> 8) & 0x0ff)
44 /* Retrives number of input handles from the scalars parameter */
45 #define REMOTE_SCALARS_INHANDLES(sc) (((sc) >> 4) & 0x0f)
47 /* Retrives number of output handles from the scalars parameter */
48 #define REMOTE_SCALARS_OUTHANDLES(sc) ((sc) & 0x0f)
50 #define REMOTE_SCALARS_LENGTH(sc) (REMOTE_SCALARS_INBUFS(sc) + \
51 REMOTE_SCALARS_OUTBUFS(sc) + \
52 REMOTE_SCALARS_INHANDLES(sc)+ \
53 REMOTE_SCALARS_OUTHANDLES(sc))
54 #define FASTRPC_BUILD_SCALARS(attr, method, in, out, oin, oout) \
55 (((attr & 0x07) << 29) | \
56 ((method & 0x1f) << 24) | \
57 ((in & 0xff) << 16) | \
58 ((out & 0xff) << 8) | \
59 ((oin & 0x0f) << 4) | \
62 #define FASTRPC_SCALARS(method, in, out) \
63 FASTRPC_BUILD_SCALARS(0, method, in, out, 0, 0)
65 #define FASTRPC_CREATE_PROCESS_NARGS 6
66 /* Remote Method id table */
67 #define FASTRPC_RMID_INIT_ATTACH 0
68 #define FASTRPC_RMID_INIT_RELEASE 1
69 #define FASTRPC_RMID_INIT_CREATE 6
70 #define FASTRPC_RMID_INIT_CREATE_ATTR 7
71 #define FASTRPC_RMID_INIT_CREATE_STATIC 8
73 #define miscdev_to_cctx(d) container_of(d, struct fastrpc_channel_ctx, miscdev)
75 static const char *domains
[FASTRPC_DEV_MAX
] = { "adsp", "mdsp",
77 struct fastrpc_phy_page
{
78 u64 addr
; /* physical address */
79 u64 size
; /* size of contiguous region */
82 struct fastrpc_invoke_buf
{
83 u32 num
; /* number of contiguous regions */
84 u32 pgidx
; /* index to start of contiguous region */
87 struct fastrpc_remote_arg
{
93 int pid
; /* process group id */
94 int tid
; /* thread id */
95 u64 ctx
; /* invoke caller context */
96 u32 handle
; /* handle to invoke */
97 u32 sc
; /* scalars structure describing the data */
98 u64 addr
; /* physical address */
99 u64 size
; /* size of contiguous region */
102 struct fastrpc_invoke_rsp
{
103 u64 ctx
; /* invoke caller context */
104 int retval
; /* invoke return value */
107 struct fastrpc_buf_overlap
{
117 struct fastrpc_user
*fl
;
118 struct dma_buf
*dmabuf
;
123 /* Lock for dma buf attachments */
125 struct list_head attachments
;
128 struct fastrpc_dma_buf_attachment
{
131 struct list_head node
;
135 struct list_head node
;
136 struct fastrpc_user
*fl
;
139 struct sg_table
*table
;
140 struct dma_buf_attachment
*attach
;
145 struct kref refcount
;
148 struct fastrpc_invoke_ctx
{
158 struct kref refcount
;
159 struct list_head node
; /* list of ctxs */
160 struct completion work
;
161 struct work_struct put_work
;
162 struct fastrpc_msg msg
;
163 struct fastrpc_user
*fl
;
164 struct fastrpc_remote_arg
*rpra
;
165 struct fastrpc_map
**maps
;
166 struct fastrpc_buf
*buf
;
167 struct fastrpc_invoke_args
*args
;
168 struct fastrpc_buf_overlap
*olaps
;
169 struct fastrpc_channel_ctx
*cctx
;
172 struct fastrpc_session_ctx
{
179 struct fastrpc_channel_ctx
{
182 struct rpmsg_device
*rpdev
;
183 struct fastrpc_session_ctx session
[FASTRPC_MAX_SESSIONS
];
186 struct list_head users
;
187 struct miscdevice miscdev
;
188 struct kref refcount
;
191 struct fastrpc_user
{
192 struct list_head user
;
193 struct list_head maps
;
194 struct list_head pending
;
196 struct fastrpc_channel_ctx
*cctx
;
197 struct fastrpc_session_ctx
*sctx
;
198 struct fastrpc_buf
*init_mem
;
204 /* lock for allocations */
208 static void fastrpc_free_map(struct kref
*ref
)
210 struct fastrpc_map
*map
;
212 map
= container_of(ref
, struct fastrpc_map
, refcount
);
215 dma_buf_unmap_attachment(map
->attach
, map
->table
,
217 dma_buf_detach(map
->buf
, map
->attach
);
218 dma_buf_put(map
->buf
);
224 static void fastrpc_map_put(struct fastrpc_map
*map
)
227 kref_put(&map
->refcount
, fastrpc_free_map
);
230 static void fastrpc_map_get(struct fastrpc_map
*map
)
233 kref_get(&map
->refcount
);
236 static int fastrpc_map_find(struct fastrpc_user
*fl
, int fd
,
237 struct fastrpc_map
**ppmap
)
239 struct fastrpc_map
*map
= NULL
;
241 mutex_lock(&fl
->mutex
);
242 list_for_each_entry(map
, &fl
->maps
, node
) {
244 fastrpc_map_get(map
);
246 mutex_unlock(&fl
->mutex
);
250 mutex_unlock(&fl
->mutex
);
255 static void fastrpc_buf_free(struct fastrpc_buf
*buf
)
257 dma_free_coherent(buf
->dev
, buf
->size
, buf
->virt
,
258 FASTRPC_PHYS(buf
->phys
));
262 static int fastrpc_buf_alloc(struct fastrpc_user
*fl
, struct device
*dev
,
263 u64 size
, struct fastrpc_buf
**obuf
)
265 struct fastrpc_buf
*buf
;
267 buf
= kzalloc(sizeof(*buf
), GFP_KERNEL
);
271 INIT_LIST_HEAD(&buf
->attachments
);
272 mutex_init(&buf
->lock
);
280 buf
->virt
= dma_alloc_coherent(dev
, buf
->size
, (dma_addr_t
*)&buf
->phys
,
283 mutex_destroy(&buf
->lock
);
288 if (fl
->sctx
&& fl
->sctx
->sid
)
289 buf
->phys
+= ((u64
)fl
->sctx
->sid
<< 32);
296 static void fastrpc_channel_ctx_free(struct kref
*ref
)
298 struct fastrpc_channel_ctx
*cctx
;
300 cctx
= container_of(ref
, struct fastrpc_channel_ctx
, refcount
);
305 static void fastrpc_channel_ctx_get(struct fastrpc_channel_ctx
*cctx
)
307 kref_get(&cctx
->refcount
);
310 static void fastrpc_channel_ctx_put(struct fastrpc_channel_ctx
*cctx
)
312 kref_put(&cctx
->refcount
, fastrpc_channel_ctx_free
);
315 static void fastrpc_context_free(struct kref
*ref
)
317 struct fastrpc_invoke_ctx
*ctx
;
318 struct fastrpc_channel_ctx
*cctx
;
322 ctx
= container_of(ref
, struct fastrpc_invoke_ctx
, refcount
);
325 for (i
= 0; i
< ctx
->nscalars
; i
++)
326 fastrpc_map_put(ctx
->maps
[i
]);
329 fastrpc_buf_free(ctx
->buf
);
331 spin_lock_irqsave(&cctx
->lock
, flags
);
332 idr_remove(&cctx
->ctx_idr
, ctx
->ctxid
>> 4);
333 spin_unlock_irqrestore(&cctx
->lock
, flags
);
339 fastrpc_channel_ctx_put(cctx
);
342 static void fastrpc_context_get(struct fastrpc_invoke_ctx
*ctx
)
344 kref_get(&ctx
->refcount
);
347 static void fastrpc_context_put(struct fastrpc_invoke_ctx
*ctx
)
349 kref_put(&ctx
->refcount
, fastrpc_context_free
);
352 static void fastrpc_context_put_wq(struct work_struct
*work
)
354 struct fastrpc_invoke_ctx
*ctx
=
355 container_of(work
, struct fastrpc_invoke_ctx
, put_work
);
357 fastrpc_context_put(ctx
);
360 #define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1)
361 static int olaps_cmp(const void *a
, const void *b
)
363 struct fastrpc_buf_overlap
*pa
= (struct fastrpc_buf_overlap
*)a
;
364 struct fastrpc_buf_overlap
*pb
= (struct fastrpc_buf_overlap
*)b
;
365 /* sort with lowest starting buffer first */
366 int st
= CMP(pa
->start
, pb
->start
);
367 /* sort with highest ending buffer first */
368 int ed
= CMP(pb
->end
, pa
->end
);
370 return st
== 0 ? ed
: st
;
373 static void fastrpc_get_buff_overlaps(struct fastrpc_invoke_ctx
*ctx
)
378 for (i
= 0; i
< ctx
->nbufs
; ++i
) {
379 ctx
->olaps
[i
].start
= ctx
->args
[i
].ptr
;
380 ctx
->olaps
[i
].end
= ctx
->olaps
[i
].start
+ ctx
->args
[i
].length
;
381 ctx
->olaps
[i
].raix
= i
;
384 sort(ctx
->olaps
, ctx
->nbufs
, sizeof(*ctx
->olaps
), olaps_cmp
, NULL
);
386 for (i
= 0; i
< ctx
->nbufs
; ++i
) {
387 /* Falling inside previous range */
388 if (ctx
->olaps
[i
].start
< max_end
) {
389 ctx
->olaps
[i
].mstart
= max_end
;
390 ctx
->olaps
[i
].mend
= ctx
->olaps
[i
].end
;
391 ctx
->olaps
[i
].offset
= max_end
- ctx
->olaps
[i
].start
;
393 if (ctx
->olaps
[i
].end
> max_end
) {
394 max_end
= ctx
->olaps
[i
].end
;
396 ctx
->olaps
[i
].mend
= 0;
397 ctx
->olaps
[i
].mstart
= 0;
401 ctx
->olaps
[i
].mend
= ctx
->olaps
[i
].end
;
402 ctx
->olaps
[i
].mstart
= ctx
->olaps
[i
].start
;
403 ctx
->olaps
[i
].offset
= 0;
404 max_end
= ctx
->olaps
[i
].end
;
409 static struct fastrpc_invoke_ctx
*fastrpc_context_alloc(
410 struct fastrpc_user
*user
, u32 kernel
, u32 sc
,
411 struct fastrpc_invoke_args
*args
)
413 struct fastrpc_channel_ctx
*cctx
= user
->cctx
;
414 struct fastrpc_invoke_ctx
*ctx
= NULL
;
418 ctx
= kzalloc(sizeof(*ctx
), GFP_KERNEL
);
420 return ERR_PTR(-ENOMEM
);
422 INIT_LIST_HEAD(&ctx
->node
);
424 ctx
->nscalars
= REMOTE_SCALARS_LENGTH(sc
);
425 ctx
->nbufs
= REMOTE_SCALARS_INBUFS(sc
) +
426 REMOTE_SCALARS_OUTBUFS(sc
);
429 ctx
->maps
= kcalloc(ctx
->nscalars
,
430 sizeof(*ctx
->maps
), GFP_KERNEL
);
433 return ERR_PTR(-ENOMEM
);
435 ctx
->olaps
= kcalloc(ctx
->nscalars
,
436 sizeof(*ctx
->olaps
), GFP_KERNEL
);
440 return ERR_PTR(-ENOMEM
);
443 fastrpc_get_buff_overlaps(ctx
);
446 /* Released in fastrpc_context_put() */
447 fastrpc_channel_ctx_get(cctx
);
451 ctx
->pid
= current
->pid
;
452 ctx
->tgid
= user
->tgid
;
454 init_completion(&ctx
->work
);
455 INIT_WORK(&ctx
->put_work
, fastrpc_context_put_wq
);
457 spin_lock(&user
->lock
);
458 list_add_tail(&ctx
->node
, &user
->pending
);
459 spin_unlock(&user
->lock
);
461 spin_lock_irqsave(&cctx
->lock
, flags
);
462 ret
= idr_alloc_cyclic(&cctx
->ctx_idr
, ctx
, 1,
463 FASTRPC_CTX_MAX
, GFP_ATOMIC
);
465 spin_unlock_irqrestore(&cctx
->lock
, flags
);
468 ctx
->ctxid
= ret
<< 4;
469 spin_unlock_irqrestore(&cctx
->lock
, flags
);
471 kref_init(&ctx
->refcount
);
475 spin_lock(&user
->lock
);
476 list_del(&ctx
->node
);
477 spin_unlock(&user
->lock
);
478 fastrpc_channel_ctx_put(cctx
);
486 static struct sg_table
*
487 fastrpc_map_dma_buf(struct dma_buf_attachment
*attachment
,
488 enum dma_data_direction dir
)
490 struct fastrpc_dma_buf_attachment
*a
= attachment
->priv
;
491 struct sg_table
*table
;
495 if (!dma_map_sg(attachment
->dev
, table
->sgl
, table
->nents
, dir
))
496 return ERR_PTR(-ENOMEM
);
501 static void fastrpc_unmap_dma_buf(struct dma_buf_attachment
*attach
,
502 struct sg_table
*table
,
503 enum dma_data_direction dir
)
505 dma_unmap_sg(attach
->dev
, table
->sgl
, table
->nents
, dir
);
508 static void fastrpc_release(struct dma_buf
*dmabuf
)
510 struct fastrpc_buf
*buffer
= dmabuf
->priv
;
512 fastrpc_buf_free(buffer
);
515 static int fastrpc_dma_buf_attach(struct dma_buf
*dmabuf
,
516 struct dma_buf_attachment
*attachment
)
518 struct fastrpc_dma_buf_attachment
*a
;
519 struct fastrpc_buf
*buffer
= dmabuf
->priv
;
522 a
= kzalloc(sizeof(*a
), GFP_KERNEL
);
526 ret
= dma_get_sgtable(buffer
->dev
, &a
->sgt
, buffer
->virt
,
527 FASTRPC_PHYS(buffer
->phys
), buffer
->size
);
529 dev_err(buffer
->dev
, "failed to get scatterlist from DMA API\n");
533 a
->dev
= attachment
->dev
;
534 INIT_LIST_HEAD(&a
->node
);
535 attachment
->priv
= a
;
537 mutex_lock(&buffer
->lock
);
538 list_add(&a
->node
, &buffer
->attachments
);
539 mutex_unlock(&buffer
->lock
);
544 static void fastrpc_dma_buf_detatch(struct dma_buf
*dmabuf
,
545 struct dma_buf_attachment
*attachment
)
547 struct fastrpc_dma_buf_attachment
*a
= attachment
->priv
;
548 struct fastrpc_buf
*buffer
= dmabuf
->priv
;
550 mutex_lock(&buffer
->lock
);
552 mutex_unlock(&buffer
->lock
);
553 sg_free_table(&a
->sgt
);
557 static void *fastrpc_kmap(struct dma_buf
*dmabuf
, unsigned long pgnum
)
559 struct fastrpc_buf
*buf
= dmabuf
->priv
;
561 return buf
->virt
? buf
->virt
+ pgnum
* PAGE_SIZE
: NULL
;
564 static void *fastrpc_vmap(struct dma_buf
*dmabuf
)
566 struct fastrpc_buf
*buf
= dmabuf
->priv
;
571 static int fastrpc_mmap(struct dma_buf
*dmabuf
,
572 struct vm_area_struct
*vma
)
574 struct fastrpc_buf
*buf
= dmabuf
->priv
;
575 size_t size
= vma
->vm_end
- vma
->vm_start
;
577 return dma_mmap_coherent(buf
->dev
, vma
, buf
->virt
,
578 FASTRPC_PHYS(buf
->phys
), size
);
581 static const struct dma_buf_ops fastrpc_dma_buf_ops
= {
582 .attach
= fastrpc_dma_buf_attach
,
583 .detach
= fastrpc_dma_buf_detatch
,
584 .map_dma_buf
= fastrpc_map_dma_buf
,
585 .unmap_dma_buf
= fastrpc_unmap_dma_buf
,
586 .mmap
= fastrpc_mmap
,
588 .vmap
= fastrpc_vmap
,
589 .release
= fastrpc_release
,
592 static int fastrpc_map_create(struct fastrpc_user
*fl
, int fd
,
593 u64 len
, struct fastrpc_map
**ppmap
)
595 struct fastrpc_session_ctx
*sess
= fl
->sctx
;
596 struct fastrpc_map
*map
= NULL
;
599 if (!fastrpc_map_find(fl
, fd
, ppmap
))
602 map
= kzalloc(sizeof(*map
), GFP_KERNEL
);
606 INIT_LIST_HEAD(&map
->node
);
609 map
->buf
= dma_buf_get(fd
);
610 if (IS_ERR(map
->buf
)) {
611 err
= PTR_ERR(map
->buf
);
615 map
->attach
= dma_buf_attach(map
->buf
, sess
->dev
);
616 if (IS_ERR(map
->attach
)) {
617 dev_err(sess
->dev
, "Failed to attach dmabuf\n");
618 err
= PTR_ERR(map
->attach
);
622 map
->table
= dma_buf_map_attachment(map
->attach
, DMA_BIDIRECTIONAL
);
623 if (IS_ERR(map
->table
)) {
624 err
= PTR_ERR(map
->table
);
628 map
->phys
= sg_dma_address(map
->table
->sgl
);
629 map
->phys
+= ((u64
)fl
->sctx
->sid
<< 32);
631 map
->va
= sg_virt(map
->table
->sgl
);
633 kref_init(&map
->refcount
);
635 spin_lock(&fl
->lock
);
636 list_add_tail(&map
->node
, &fl
->maps
);
637 spin_unlock(&fl
->lock
);
643 dma_buf_detach(map
->buf
, map
->attach
);
645 dma_buf_put(map
->buf
);
653 * Fastrpc payload buffer with metadata looks like:
655 * >>>>>> START of METADATA <<<<<<<<<
656 * +---------------------------------+
658 * | type:(struct fastrpc_remote_arg)|
660 * +---------------------------------+
661 * | Invoke Buffer list |
662 * | type:(struct fastrpc_invoke_buf)|
664 * +---------------------------------+
666 * | type:(struct fastrpc_phy_page) |
668 * +---------------------------------+
670 * |(can be specific to SoC/Firmware)|
671 * +---------------------------------+
672 * >>>>>>>> END of METADATA <<<<<<<<<
673 * +---------------------------------+
676 * +---------------------------------+
679 static int fastrpc_get_meta_size(struct fastrpc_invoke_ctx
*ctx
)
683 size
= (sizeof(struct fastrpc_remote_arg
) +
684 sizeof(struct fastrpc_invoke_buf
) +
685 sizeof(struct fastrpc_phy_page
)) * ctx
->nscalars
+
686 sizeof(u64
) * FASTRPC_MAX_FDLIST
+
687 sizeof(u32
) * FASTRPC_MAX_CRCLIST
;
692 static u64
fastrpc_get_payload_size(struct fastrpc_invoke_ctx
*ctx
, int metalen
)
697 size
= ALIGN(metalen
, FASTRPC_ALIGN
);
698 for (i
= 0; i
< ctx
->nscalars
; i
++) {
699 if (ctx
->args
[i
].fd
== 0 || ctx
->args
[i
].fd
== -1) {
701 if (ctx
->olaps
[i
].offset
== 0)
702 size
= ALIGN(size
, FASTRPC_ALIGN
);
704 size
+= (ctx
->olaps
[i
].mend
- ctx
->olaps
[i
].mstart
);
711 static int fastrpc_create_maps(struct fastrpc_invoke_ctx
*ctx
)
713 struct device
*dev
= ctx
->fl
->sctx
->dev
;
716 for (i
= 0; i
< ctx
->nscalars
; ++i
) {
717 /* Make sure reserved field is set to 0 */
718 if (ctx
->args
[i
].reserved
)
721 if (ctx
->args
[i
].fd
== 0 || ctx
->args
[i
].fd
== -1 ||
722 ctx
->args
[i
].length
== 0)
725 err
= fastrpc_map_create(ctx
->fl
, ctx
->args
[i
].fd
,
726 ctx
->args
[i
].length
, &ctx
->maps
[i
]);
728 dev_err(dev
, "Error Creating map %d\n", err
);
736 static int fastrpc_get_args(u32 kernel
, struct fastrpc_invoke_ctx
*ctx
)
738 struct device
*dev
= ctx
->fl
->sctx
->dev
;
739 struct fastrpc_remote_arg
*rpra
;
740 struct fastrpc_invoke_buf
*list
;
741 struct fastrpc_phy_page
*pages
;
742 int inbufs
, i
, oix
, err
= 0;
743 u64 len
, rlen
, pkt_size
;
744 u64 pg_start
, pg_end
;
748 inbufs
= REMOTE_SCALARS_INBUFS(ctx
->sc
);
749 metalen
= fastrpc_get_meta_size(ctx
);
750 pkt_size
= fastrpc_get_payload_size(ctx
, metalen
);
752 err
= fastrpc_create_maps(ctx
);
756 ctx
->msg_sz
= pkt_size
;
758 err
= fastrpc_buf_alloc(ctx
->fl
, dev
, pkt_size
, &ctx
->buf
);
762 rpra
= ctx
->buf
->virt
;
763 list
= ctx
->buf
->virt
+ ctx
->nscalars
* sizeof(*rpra
);
764 pages
= ctx
->buf
->virt
+ ctx
->nscalars
* (sizeof(*list
) +
766 args
= (uintptr_t)ctx
->buf
->virt
+ metalen
;
767 rlen
= pkt_size
- metalen
;
770 for (oix
= 0; oix
< ctx
->nbufs
; ++oix
) {
773 i
= ctx
->olaps
[oix
].raix
;
774 len
= ctx
->args
[i
].length
;
778 list
[i
].num
= len
? 1 : 0;
785 struct vm_area_struct
*vma
= NULL
;
787 rpra
[i
].pv
= (u64
) ctx
->args
[i
].ptr
;
788 pages
[i
].addr
= ctx
->maps
[i
]->phys
;
790 vma
= find_vma(current
->mm
, ctx
->args
[i
].ptr
);
792 pages
[i
].addr
+= ctx
->args
[i
].ptr
-
795 pg_start
= (ctx
->args
[i
].ptr
& PAGE_MASK
) >> PAGE_SHIFT
;
796 pg_end
= ((ctx
->args
[i
].ptr
+ len
- 1) & PAGE_MASK
) >>
798 pages
[i
].size
= (pg_end
- pg_start
+ 1) * PAGE_SIZE
;
802 if (ctx
->olaps
[oix
].offset
== 0) {
803 rlen
-= ALIGN(args
, FASTRPC_ALIGN
) - args
;
804 args
= ALIGN(args
, FASTRPC_ALIGN
);
807 mlen
= ctx
->olaps
[oix
].mend
- ctx
->olaps
[oix
].mstart
;
812 rpra
[i
].pv
= args
- ctx
->olaps
[oix
].offset
;
813 pages
[i
].addr
= ctx
->buf
->phys
-
814 ctx
->olaps
[oix
].offset
+
816 pages
[i
].addr
= pages
[i
].addr
& PAGE_MASK
;
818 pg_start
= (args
& PAGE_MASK
) >> PAGE_SHIFT
;
819 pg_end
= ((args
+ len
- 1) & PAGE_MASK
) >> PAGE_SHIFT
;
820 pages
[i
].size
= (pg_end
- pg_start
+ 1) * PAGE_SIZE
;
825 if (i
< inbufs
&& !ctx
->maps
[i
]) {
826 void *dst
= (void *)(uintptr_t)rpra
[i
].pv
;
827 void *src
= (void *)(uintptr_t)ctx
->args
[i
].ptr
;
830 if (copy_from_user(dst
, (void __user
*)src
,
836 memcpy(dst
, src
, len
);
841 for (i
= ctx
->nbufs
; i
< ctx
->nscalars
; ++i
) {
842 rpra
[i
].pv
= (u64
) ctx
->args
[i
].ptr
;
843 rpra
[i
].len
= ctx
->args
[i
].length
;
844 list
[i
].num
= ctx
->args
[i
].length
? 1 : 0;
846 pages
[i
].addr
= ctx
->maps
[i
]->phys
;
847 pages
[i
].size
= ctx
->maps
[i
]->size
;
852 dev_err(dev
, "Error: get invoke args failed:%d\n", err
);
857 static int fastrpc_put_args(struct fastrpc_invoke_ctx
*ctx
,
860 struct fastrpc_remote_arg
*rpra
= ctx
->rpra
;
863 inbufs
= REMOTE_SCALARS_INBUFS(ctx
->sc
);
865 for (i
= inbufs
; i
< ctx
->nbufs
; ++i
) {
866 void *src
= (void *)(uintptr_t)rpra
[i
].pv
;
867 void *dst
= (void *)(uintptr_t)ctx
->args
[i
].ptr
;
868 u64 len
= rpra
[i
].len
;
871 if (copy_to_user((void __user
*)dst
, src
, len
))
874 memcpy(dst
, src
, len
);
881 static int fastrpc_invoke_send(struct fastrpc_session_ctx
*sctx
,
882 struct fastrpc_invoke_ctx
*ctx
,
883 u32 kernel
, uint32_t handle
)
885 struct fastrpc_channel_ctx
*cctx
;
886 struct fastrpc_user
*fl
= ctx
->fl
;
887 struct fastrpc_msg
*msg
= &ctx
->msg
;
891 msg
->tid
= current
->pid
;
896 msg
->ctx
= ctx
->ctxid
| fl
->pd
;
897 msg
->handle
= handle
;
899 msg
->addr
= ctx
->buf
? ctx
->buf
->phys
: 0;
900 msg
->size
= roundup(ctx
->msg_sz
, PAGE_SIZE
);
901 fastrpc_context_get(ctx
);
903 return rpmsg_send(cctx
->rpdev
->ept
, (void *)msg
, sizeof(*msg
));
906 static int fastrpc_internal_invoke(struct fastrpc_user
*fl
, u32 kernel
,
908 struct fastrpc_invoke_args
*args
)
910 struct fastrpc_invoke_ctx
*ctx
= NULL
;
916 if (!fl
->cctx
->rpdev
)
919 ctx
= fastrpc_context_alloc(fl
, kernel
, sc
, args
);
924 err
= fastrpc_get_args(kernel
, ctx
);
929 /* make sure that all CPU memory writes are seen by DSP */
931 /* Send invoke buffer to remote dsp */
932 err
= fastrpc_invoke_send(fl
->sctx
, ctx
, kernel
, handle
);
936 /* Wait for remote dsp to respond or time out */
937 err
= wait_for_completion_interruptible(&ctx
->work
);
941 /* Check the response from remote dsp */
947 /* make sure that all memory writes by DSP are seen by CPU */
949 /* populate all the output buffers with results */
950 err
= fastrpc_put_args(ctx
, kernel
);
956 /* We are done with this compute context, remove it from pending list */
957 spin_lock(&fl
->lock
);
958 list_del(&ctx
->node
);
959 spin_unlock(&fl
->lock
);
960 fastrpc_context_put(ctx
);
963 dev_dbg(fl
->sctx
->dev
, "Error: Invoke Failed %d\n", err
);
968 static int fastrpc_init_create_process(struct fastrpc_user
*fl
,
971 struct fastrpc_init_create init
;
972 struct fastrpc_invoke_args
*args
;
973 struct fastrpc_phy_page pages
[1];
974 struct fastrpc_map
*map
= NULL
;
975 struct fastrpc_buf
*imem
= NULL
;
988 args
= kcalloc(FASTRPC_CREATE_PROCESS_NARGS
, sizeof(*args
), GFP_KERNEL
);
992 if (copy_from_user(&init
, argp
, sizeof(init
))) {
997 if (init
.filelen
> INIT_FILELEN_MAX
) {
1002 inbuf
.pgid
= fl
->tgid
;
1003 inbuf
.namelen
= strlen(current
->comm
) + 1;
1004 inbuf
.filelen
= init
.filelen
;
1006 inbuf
.attrs
= init
.attrs
;
1007 inbuf
.siglen
= init
.siglen
;
1010 if (init
.filelen
&& init
.filefd
) {
1011 err
= fastrpc_map_create(fl
, init
.filefd
, init
.filelen
, &map
);
1016 memlen
= ALIGN(max(INIT_FILELEN_MAX
, (int)init
.filelen
* 4),
1018 err
= fastrpc_buf_alloc(fl
, fl
->sctx
->dev
, memlen
,
1023 fl
->init_mem
= imem
;
1024 args
[0].ptr
= (u64
)(uintptr_t)&inbuf
;
1025 args
[0].length
= sizeof(inbuf
);
1028 args
[1].ptr
= (u64
)(uintptr_t)current
->comm
;
1029 args
[1].length
= inbuf
.namelen
;
1032 args
[2].ptr
= (u64
) init
.file
;
1033 args
[2].length
= inbuf
.filelen
;
1034 args
[2].fd
= init
.filefd
;
1036 pages
[0].addr
= imem
->phys
;
1037 pages
[0].size
= imem
->size
;
1039 args
[3].ptr
= (u64
)(uintptr_t) pages
;
1040 args
[3].length
= 1 * sizeof(*pages
);
1043 args
[4].ptr
= (u64
)(uintptr_t)&inbuf
.attrs
;
1044 args
[4].length
= sizeof(inbuf
.attrs
);
1047 args
[5].ptr
= (u64
)(uintptr_t) &inbuf
.siglen
;
1048 args
[5].length
= sizeof(inbuf
.siglen
);
1051 sc
= FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE
, 4, 0);
1053 sc
= FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_ATTR
, 6, 0);
1055 err
= fastrpc_internal_invoke(fl
, true, FASTRPC_INIT_HANDLE
,
1065 fl
->init_mem
= NULL
;
1066 fastrpc_buf_free(imem
);
1069 spin_lock(&fl
->lock
);
1070 list_del(&map
->node
);
1071 spin_unlock(&fl
->lock
);
1072 fastrpc_map_put(map
);
1080 static struct fastrpc_session_ctx
*fastrpc_session_alloc(
1081 struct fastrpc_channel_ctx
*cctx
)
1083 struct fastrpc_session_ctx
*session
= NULL
;
1084 unsigned long flags
;
1087 spin_lock_irqsave(&cctx
->lock
, flags
);
1088 for (i
= 0; i
< cctx
->sesscount
; i
++) {
1089 if (!cctx
->session
[i
].used
&& cctx
->session
[i
].valid
) {
1090 cctx
->session
[i
].used
= true;
1091 session
= &cctx
->session
[i
];
1095 spin_unlock_irqrestore(&cctx
->lock
, flags
);
1100 static void fastrpc_session_free(struct fastrpc_channel_ctx
*cctx
,
1101 struct fastrpc_session_ctx
*session
)
1103 unsigned long flags
;
1105 spin_lock_irqsave(&cctx
->lock
, flags
);
1106 session
->used
= false;
1107 spin_unlock_irqrestore(&cctx
->lock
, flags
);
1110 static int fastrpc_release_current_dsp_process(struct fastrpc_user
*fl
)
1112 struct fastrpc_invoke_args args
[1];
1117 args
[0].ptr
= (u64
)(uintptr_t) &tgid
;
1118 args
[0].length
= sizeof(tgid
);
1120 args
[0].reserved
= 0;
1121 sc
= FASTRPC_SCALARS(FASTRPC_RMID_INIT_RELEASE
, 1, 0);
1123 return fastrpc_internal_invoke(fl
, true, FASTRPC_INIT_HANDLE
,
1127 static int fastrpc_device_release(struct inode
*inode
, struct file
*file
)
1129 struct fastrpc_user
*fl
= (struct fastrpc_user
*)file
->private_data
;
1130 struct fastrpc_channel_ctx
*cctx
= fl
->cctx
;
1131 struct fastrpc_invoke_ctx
*ctx
, *n
;
1132 struct fastrpc_map
*map
, *m
;
1133 unsigned long flags
;
1135 fastrpc_release_current_dsp_process(fl
);
1137 spin_lock_irqsave(&cctx
->lock
, flags
);
1138 list_del(&fl
->user
);
1139 spin_unlock_irqrestore(&cctx
->lock
, flags
);
1142 fastrpc_buf_free(fl
->init_mem
);
1144 list_for_each_entry_safe(ctx
, n
, &fl
->pending
, node
) {
1145 list_del(&ctx
->node
);
1146 fastrpc_context_put(ctx
);
1149 list_for_each_entry_safe(map
, m
, &fl
->maps
, node
) {
1150 list_del(&map
->node
);
1151 fastrpc_map_put(map
);
1154 fastrpc_session_free(cctx
, fl
->sctx
);
1155 fastrpc_channel_ctx_put(cctx
);
1157 mutex_destroy(&fl
->mutex
);
1159 file
->private_data
= NULL
;
1164 static int fastrpc_device_open(struct inode
*inode
, struct file
*filp
)
1166 struct fastrpc_channel_ctx
*cctx
= miscdev_to_cctx(filp
->private_data
);
1167 struct fastrpc_user
*fl
= NULL
;
1168 unsigned long flags
;
1170 fl
= kzalloc(sizeof(*fl
), GFP_KERNEL
);
1174 /* Released in fastrpc_device_release() */
1175 fastrpc_channel_ctx_get(cctx
);
1177 filp
->private_data
= fl
;
1178 spin_lock_init(&fl
->lock
);
1179 mutex_init(&fl
->mutex
);
1180 INIT_LIST_HEAD(&fl
->pending
);
1181 INIT_LIST_HEAD(&fl
->maps
);
1182 INIT_LIST_HEAD(&fl
->user
);
1183 fl
->tgid
= current
->tgid
;
1186 fl
->sctx
= fastrpc_session_alloc(cctx
);
1188 dev_err(&cctx
->rpdev
->dev
, "No session available\n");
1189 mutex_destroy(&fl
->mutex
);
1195 spin_lock_irqsave(&cctx
->lock
, flags
);
1196 list_add_tail(&fl
->user
, &cctx
->users
);
1197 spin_unlock_irqrestore(&cctx
->lock
, flags
);
1202 static int fastrpc_dmabuf_alloc(struct fastrpc_user
*fl
, char __user
*argp
)
1204 struct fastrpc_alloc_dma_buf bp
;
1205 DEFINE_DMA_BUF_EXPORT_INFO(exp_info
);
1206 struct fastrpc_buf
*buf
= NULL
;
1209 if (copy_from_user(&bp
, argp
, sizeof(bp
)))
1212 err
= fastrpc_buf_alloc(fl
, fl
->sctx
->dev
, bp
.size
, &buf
);
1215 exp_info
.ops
= &fastrpc_dma_buf_ops
;
1216 exp_info
.size
= bp
.size
;
1217 exp_info
.flags
= O_RDWR
;
1218 exp_info
.priv
= buf
;
1219 buf
->dmabuf
= dma_buf_export(&exp_info
);
1220 if (IS_ERR(buf
->dmabuf
)) {
1221 err
= PTR_ERR(buf
->dmabuf
);
1222 fastrpc_buf_free(buf
);
1226 bp
.fd
= dma_buf_fd(buf
->dmabuf
, O_ACCMODE
);
1228 dma_buf_put(buf
->dmabuf
);
1232 if (copy_to_user(argp
, &bp
, sizeof(bp
))) {
1233 dma_buf_put(buf
->dmabuf
);
1240 static int fastrpc_init_attach(struct fastrpc_user
*fl
)
1242 struct fastrpc_invoke_args args
[1];
1243 int tgid
= fl
->tgid
;
1246 args
[0].ptr
= (u64
)(uintptr_t) &tgid
;
1247 args
[0].length
= sizeof(tgid
);
1249 args
[0].reserved
= 0;
1250 sc
= FASTRPC_SCALARS(FASTRPC_RMID_INIT_ATTACH
, 1, 0);
1253 return fastrpc_internal_invoke(fl
, true, FASTRPC_INIT_HANDLE
,
1257 static int fastrpc_invoke(struct fastrpc_user
*fl
, char __user
*argp
)
1259 struct fastrpc_invoke_args
*args
= NULL
;
1260 struct fastrpc_invoke inv
;
1264 if (copy_from_user(&inv
, argp
, sizeof(inv
)))
1267 /* nscalars is truncated here to max supported value */
1268 nscalars
= REMOTE_SCALARS_LENGTH(inv
.sc
);
1270 args
= kcalloc(nscalars
, sizeof(*args
), GFP_KERNEL
);
1274 if (copy_from_user(args
, (void __user
*)(uintptr_t)inv
.args
,
1275 nscalars
* sizeof(*args
))) {
1281 err
= fastrpc_internal_invoke(fl
, false, inv
.handle
, inv
.sc
, args
);
1287 static long fastrpc_device_ioctl(struct file
*file
, unsigned int cmd
,
1290 struct fastrpc_user
*fl
= (struct fastrpc_user
*)file
->private_data
;
1291 char __user
*argp
= (char __user
*)arg
;
1295 case FASTRPC_IOCTL_INVOKE
:
1296 err
= fastrpc_invoke(fl
, argp
);
1298 case FASTRPC_IOCTL_INIT_ATTACH
:
1299 err
= fastrpc_init_attach(fl
);
1301 case FASTRPC_IOCTL_INIT_CREATE
:
1302 err
= fastrpc_init_create_process(fl
, argp
);
1304 case FASTRPC_IOCTL_ALLOC_DMA_BUFF
:
1305 err
= fastrpc_dmabuf_alloc(fl
, argp
);
1315 static const struct file_operations fastrpc_fops
= {
1316 .open
= fastrpc_device_open
,
1317 .release
= fastrpc_device_release
,
1318 .unlocked_ioctl
= fastrpc_device_ioctl
,
1319 .compat_ioctl
= fastrpc_device_ioctl
,
1322 static int fastrpc_cb_probe(struct platform_device
*pdev
)
1324 struct fastrpc_channel_ctx
*cctx
;
1325 struct fastrpc_session_ctx
*sess
;
1326 struct device
*dev
= &pdev
->dev
;
1327 int i
, sessions
= 0;
1328 unsigned long flags
;
1331 cctx
= dev_get_drvdata(dev
->parent
);
1335 of_property_read_u32(dev
->of_node
, "qcom,nsessions", &sessions
);
1337 spin_lock_irqsave(&cctx
->lock
, flags
);
1338 sess
= &cctx
->session
[cctx
->sesscount
];
1342 dev_set_drvdata(dev
, sess
);
1344 if (of_property_read_u32(dev
->of_node
, "reg", &sess
->sid
))
1345 dev_info(dev
, "FastRPC Session ID not specified in DT\n");
1348 struct fastrpc_session_ctx
*dup_sess
;
1350 for (i
= 1; i
< sessions
; i
++) {
1351 if (cctx
->sesscount
++ >= FASTRPC_MAX_SESSIONS
)
1353 dup_sess
= &cctx
->session
[cctx
->sesscount
];
1354 memcpy(dup_sess
, sess
, sizeof(*dup_sess
));
1358 spin_unlock_irqrestore(&cctx
->lock
, flags
);
1359 rc
= dma_set_mask(dev
, DMA_BIT_MASK(32));
1361 dev_err(dev
, "32-bit DMA enable failed\n");
1368 static int fastrpc_cb_remove(struct platform_device
*pdev
)
1370 struct fastrpc_channel_ctx
*cctx
= dev_get_drvdata(pdev
->dev
.parent
);
1371 struct fastrpc_session_ctx
*sess
= dev_get_drvdata(&pdev
->dev
);
1372 unsigned long flags
;
1375 spin_lock_irqsave(&cctx
->lock
, flags
);
1376 for (i
= 1; i
< FASTRPC_MAX_SESSIONS
; i
++) {
1377 if (cctx
->session
[i
].sid
== sess
->sid
) {
1378 cctx
->session
[i
].valid
= false;
1382 spin_unlock_irqrestore(&cctx
->lock
, flags
);
1387 static const struct of_device_id fastrpc_match_table
[] = {
1388 { .compatible
= "qcom,fastrpc-compute-cb", },
1392 static struct platform_driver fastrpc_cb_driver
= {
1393 .probe
= fastrpc_cb_probe
,
1394 .remove
= fastrpc_cb_remove
,
1396 .name
= "qcom,fastrpc-cb",
1397 .of_match_table
= fastrpc_match_table
,
1398 .suppress_bind_attrs
= true,
1402 static int fastrpc_rpmsg_probe(struct rpmsg_device
*rpdev
)
1404 struct device
*rdev
= &rpdev
->dev
;
1405 struct fastrpc_channel_ctx
*data
;
1406 int i
, err
, domain_id
= -1;
1409 err
= of_property_read_string(rdev
->of_node
, "label", &domain
);
1411 dev_info(rdev
, "FastRPC Domain not specified in DT\n");
1415 for (i
= 0; i
<= CDSP_DOMAIN_ID
; i
++) {
1416 if (!strcmp(domains
[i
], domain
)) {
1422 if (domain_id
< 0) {
1423 dev_info(rdev
, "FastRPC Invalid Domain ID %d\n", domain_id
);
1427 data
= kzalloc(sizeof(*data
), GFP_KERNEL
);
1431 data
->miscdev
.minor
= MISC_DYNAMIC_MINOR
;
1432 data
->miscdev
.name
= kasprintf(GFP_KERNEL
, "fastrpc-%s",
1433 domains
[domain_id
]);
1434 data
->miscdev
.fops
= &fastrpc_fops
;
1435 err
= misc_register(&data
->miscdev
);
1439 kref_init(&data
->refcount
);
1441 dev_set_drvdata(&rpdev
->dev
, data
);
1442 dma_set_mask_and_coherent(rdev
, DMA_BIT_MASK(32));
1443 INIT_LIST_HEAD(&data
->users
);
1444 spin_lock_init(&data
->lock
);
1445 idr_init(&data
->ctx_idr
);
1446 data
->domain_id
= domain_id
;
1447 data
->rpdev
= rpdev
;
1449 return of_platform_populate(rdev
->of_node
, NULL
, NULL
, rdev
);
1452 static void fastrpc_notify_users(struct fastrpc_user
*user
)
1454 struct fastrpc_invoke_ctx
*ctx
;
1456 spin_lock(&user
->lock
);
1457 list_for_each_entry(ctx
, &user
->pending
, node
)
1458 complete(&ctx
->work
);
1459 spin_unlock(&user
->lock
);
1462 static void fastrpc_rpmsg_remove(struct rpmsg_device
*rpdev
)
1464 struct fastrpc_channel_ctx
*cctx
= dev_get_drvdata(&rpdev
->dev
);
1465 struct fastrpc_user
*user
;
1466 unsigned long flags
;
1468 spin_lock_irqsave(&cctx
->lock
, flags
);
1469 list_for_each_entry(user
, &cctx
->users
, user
)
1470 fastrpc_notify_users(user
);
1471 spin_unlock_irqrestore(&cctx
->lock
, flags
);
1473 misc_deregister(&cctx
->miscdev
);
1474 of_platform_depopulate(&rpdev
->dev
);
1477 fastrpc_channel_ctx_put(cctx
);
1480 static int fastrpc_rpmsg_callback(struct rpmsg_device
*rpdev
, void *data
,
1481 int len
, void *priv
, u32 addr
)
1483 struct fastrpc_channel_ctx
*cctx
= dev_get_drvdata(&rpdev
->dev
);
1484 struct fastrpc_invoke_rsp
*rsp
= data
;
1485 struct fastrpc_invoke_ctx
*ctx
;
1486 unsigned long flags
;
1487 unsigned long ctxid
;
1489 if (len
< sizeof(*rsp
))
1492 ctxid
= ((rsp
->ctx
& FASTRPC_CTXID_MASK
) >> 4);
1494 spin_lock_irqsave(&cctx
->lock
, flags
);
1495 ctx
= idr_find(&cctx
->ctx_idr
, ctxid
);
1496 spin_unlock_irqrestore(&cctx
->lock
, flags
);
1499 dev_err(&rpdev
->dev
, "No context ID matches response\n");
1503 ctx
->retval
= rsp
->retval
;
1504 complete(&ctx
->work
);
1507 * The DMA buffer associated with the context cannot be freed in
1508 * interrupt context so schedule it through a worker thread to
1509 * avoid a kernel BUG.
1511 schedule_work(&ctx
->put_work
);
1516 static const struct of_device_id fastrpc_rpmsg_of_match
[] = {
1517 { .compatible
= "qcom,fastrpc" },
1520 MODULE_DEVICE_TABLE(of
, fastrpc_rpmsg_of_match
);
1522 static struct rpmsg_driver fastrpc_driver
= {
1523 .probe
= fastrpc_rpmsg_probe
,
1524 .remove
= fastrpc_rpmsg_remove
,
1525 .callback
= fastrpc_rpmsg_callback
,
1527 .name
= "qcom,fastrpc",
1528 .of_match_table
= fastrpc_rpmsg_of_match
,
1532 static int fastrpc_init(void)
1536 ret
= platform_driver_register(&fastrpc_cb_driver
);
1538 pr_err("fastrpc: failed to register cb driver\n");
1542 ret
= register_rpmsg_driver(&fastrpc_driver
);
1544 pr_err("fastrpc: failed to register rpmsg driver\n");
1545 platform_driver_unregister(&fastrpc_cb_driver
);
1551 module_init(fastrpc_init
);
1553 static void fastrpc_exit(void)
1555 platform_driver_unregister(&fastrpc_cb_driver
);
1556 unregister_rpmsg_driver(&fastrpc_driver
);
1558 module_exit(fastrpc_exit
);
1560 MODULE_LICENSE("GPL v2");