2 * Copyright (c) 2010-2012 Broadcom. All rights reserved.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions, and the following disclaimer,
9 * without modification.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. The names of the above-listed copyright holders may not be used
14 * to endorse or promote products derived from this software without
15 * specific prior written permission.
17 * ALTERNATIVELY, this software may be distributed under the terms of the
18 * GNU General Public License ("GPL") version 2, as published by the Free
19 * Software Foundation.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
22 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
23 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
25 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
26 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
27 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
28 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
29 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
30 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
31 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <linux/kernel.h>
35 #include <linux/types.h>
36 #include <linux/errno.h>
37 #include <linux/interrupt.h>
38 #include <linux/pagemap.h>
39 #include <linux/dma-mapping.h>
41 #include <linux/platform_device.h>
42 #include <linux/uaccess.h>
45 #include <soc/bcm2835/raspberrypi-firmware.h>
47 #define TOTAL_SLOTS (VCHIQ_SLOT_ZERO_SLOTS + 2 * 32)
49 #include "vchiq_arm.h"
50 #include "vchiq_connected.h"
51 #include "vchiq_killable.h"
52 #include "vchiq_pagelist.h"
54 #define MAX_FRAGMENTS (VCHIQ_NUM_CURRENT_BULKS * 2)
56 #define VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX 0
57 #define VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX 1
62 typedef struct vchiq_2835_state_struct
{
64 VCHIQ_ARM_STATE_T arm_state
;
65 } VCHIQ_2835_ARM_STATE_T
;
67 struct vchiq_pagelist_info
{
69 size_t pagelist_buffer_size
;
71 enum dma_data_direction dma_dir
;
72 unsigned int num_pages
;
73 unsigned int pages_need_release
;
75 struct scatterlist
*scatterlist
;
76 unsigned int scatterlist_mapped
;
79 static void __iomem
*g_regs
;
80 static unsigned int g_cache_line_size
= sizeof(CACHE_LINE_SIZE
);
81 static unsigned int g_fragments_size
;
82 static char *g_fragments_base
;
83 static char *g_free_fragments
;
84 static struct semaphore g_free_fragments_sema
;
85 static struct device
*g_dev
;
87 extern int vchiq_arm_log_level
;
89 static DEFINE_SEMAPHORE(g_free_fragments_mutex
);
92 vchiq_doorbell_irq(int irq
, void *dev_id
);
94 static struct vchiq_pagelist_info
*
95 create_pagelist(char __user
*buf
, size_t count
, unsigned short type
,
96 struct task_struct
*task
);
99 free_pagelist(struct vchiq_pagelist_info
*pagelistinfo
,
102 int vchiq_platform_init(struct platform_device
*pdev
, VCHIQ_STATE_T
*state
)
104 struct device
*dev
= &pdev
->dev
;
105 struct rpi_firmware
*fw
= platform_get_drvdata(pdev
);
106 VCHIQ_SLOT_ZERO_T
*vchiq_slot_zero
;
107 struct resource
*res
;
109 dma_addr_t slot_phys
;
111 int slot_mem_size
, frag_mem_size
;
115 * VCHI messages between the CPU and firmware use
116 * 32-bit bus addresses.
118 err
= dma_set_mask_and_coherent(dev
, DMA_BIT_MASK(32));
123 err
= of_property_read_u32(dev
->of_node
, "cache-line-size",
127 dev_err(dev
, "Missing cache-line-size property\n");
131 g_fragments_size
= 2 * g_cache_line_size
;
133 /* Allocate space for the channels in coherent memory */
134 slot_mem_size
= PAGE_ALIGN(TOTAL_SLOTS
* VCHIQ_SLOT_SIZE
);
135 frag_mem_size
= PAGE_ALIGN(g_fragments_size
* MAX_FRAGMENTS
);
137 slot_mem
= dmam_alloc_coherent(dev
, slot_mem_size
+ frag_mem_size
,
138 &slot_phys
, GFP_KERNEL
);
140 dev_err(dev
, "could not allocate DMA memory\n");
144 WARN_ON(((unsigned long)slot_mem
& (PAGE_SIZE
- 1)) != 0);
146 vchiq_slot_zero
= vchiq_init_slots(slot_mem
, slot_mem_size
);
147 if (!vchiq_slot_zero
)
150 vchiq_slot_zero
->platform_data
[VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX
] =
151 (int)slot_phys
+ slot_mem_size
;
152 vchiq_slot_zero
->platform_data
[VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX
] =
155 g_fragments_base
= (char *)slot_mem
+ slot_mem_size
;
156 slot_mem_size
+= frag_mem_size
;
158 g_free_fragments
= g_fragments_base
;
159 for (i
= 0; i
< (MAX_FRAGMENTS
- 1); i
++) {
160 *(char **)&g_fragments_base
[i
*g_fragments_size
] =
161 &g_fragments_base
[(i
+ 1)*g_fragments_size
];
163 *(char **)&g_fragments_base
[i
* g_fragments_size
] = NULL
;
164 sema_init(&g_free_fragments_sema
, MAX_FRAGMENTS
);
166 if (vchiq_init_state(state
, vchiq_slot_zero
, 0) != VCHIQ_SUCCESS
)
169 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
170 g_regs
= devm_ioremap_resource(&pdev
->dev
, res
);
172 return PTR_ERR(g_regs
);
174 irq
= platform_get_irq(pdev
, 0);
176 dev_err(dev
, "failed to get IRQ\n");
180 err
= devm_request_irq(dev
, irq
, vchiq_doorbell_irq
, IRQF_IRQPOLL
,
181 "VCHIQ doorbell", state
);
183 dev_err(dev
, "failed to register irq=%d\n", irq
);
187 /* Send the base address of the slots to VideoCore */
188 channelbase
= slot_phys
;
189 err
= rpi_firmware_property(fw
, RPI_FIRMWARE_VCHIQ_INIT
,
190 &channelbase
, sizeof(channelbase
));
191 if (err
|| channelbase
) {
192 dev_err(dev
, "failed to set channelbase\n");
193 return err
? : -ENXIO
;
197 vchiq_log_info(vchiq_arm_log_level
,
198 "vchiq_init - done (slots %pK, phys %pad)",
199 vchiq_slot_zero
, &slot_phys
);
201 vchiq_call_connected_callbacks();
207 vchiq_platform_init_state(VCHIQ_STATE_T
*state
)
209 VCHIQ_STATUS_T status
= VCHIQ_SUCCESS
;
211 state
->platform_state
= kzalloc(sizeof(VCHIQ_2835_ARM_STATE_T
), GFP_KERNEL
);
212 ((VCHIQ_2835_ARM_STATE_T
*)state
->platform_state
)->inited
= 1;
213 status
= vchiq_arm_init_state(state
, &((VCHIQ_2835_ARM_STATE_T
*)state
->platform_state
)->arm_state
);
214 if (status
!= VCHIQ_SUCCESS
)
216 ((VCHIQ_2835_ARM_STATE_T
*)state
->platform_state
)->inited
= 0;
222 vchiq_platform_get_arm_state(VCHIQ_STATE_T
*state
)
224 if (!((VCHIQ_2835_ARM_STATE_T
*)state
->platform_state
)->inited
)
228 return &((VCHIQ_2835_ARM_STATE_T
*)state
->platform_state
)->arm_state
;
232 remote_event_signal(REMOTE_EVENT_T
*event
)
238 dsb(sy
); /* data barrier operation */
241 writel(0, g_regs
+ BELL2
); /* trigger vc interrupt */
245 vchiq_prepare_bulk_data(VCHIQ_BULK_T
*bulk
, VCHI_MEM_HANDLE_T memhandle
,
246 void *offset
, int size
, int dir
)
248 struct vchiq_pagelist_info
*pagelistinfo
;
250 WARN_ON(memhandle
!= VCHI_MEM_HANDLE_INVALID
);
252 pagelistinfo
= create_pagelist((char __user
*)offset
, size
,
253 (dir
== VCHIQ_BULK_RECEIVE
)
261 bulk
->handle
= memhandle
;
262 bulk
->data
= (void *)(unsigned long)pagelistinfo
->dma_addr
;
265 * Store the pagelistinfo address in remote_data,
266 * which isn't used by the slave.
268 bulk
->remote_data
= pagelistinfo
;
270 return VCHIQ_SUCCESS
;
274 vchiq_complete_bulk(VCHIQ_BULK_T
*bulk
)
276 if (bulk
&& bulk
->remote_data
&& bulk
->actual
)
277 free_pagelist((struct vchiq_pagelist_info
*)bulk
->remote_data
,
282 vchiq_transfer_bulk(VCHIQ_BULK_T
*bulk
)
285 * This should only be called on the master (VideoCore) side, but
286 * provide an implementation to avoid the need for ifdefery.
292 vchiq_dump_platform_state(void *dump_context
)
297 len
= snprintf(buf
, sizeof(buf
),
298 " Platform: 2835 (VC master)");
299 vchiq_dump(dump_context
, buf
, len
+ 1);
303 vchiq_platform_suspend(VCHIQ_STATE_T
*state
)
309 vchiq_platform_resume(VCHIQ_STATE_T
*state
)
311 return VCHIQ_SUCCESS
;
315 vchiq_platform_paused(VCHIQ_STATE_T
*state
)
320 vchiq_platform_resumed(VCHIQ_STATE_T
*state
)
325 vchiq_platform_videocore_wanted(VCHIQ_STATE_T
*state
)
327 return 1; // autosuspend not supported - videocore always wanted
331 vchiq_platform_use_suspend_timer(void)
336 vchiq_dump_platform_use_state(VCHIQ_STATE_T
*state
)
338 vchiq_log_info(vchiq_arm_log_level
, "Suspend timer not in use");
341 vchiq_platform_handle_timeout(VCHIQ_STATE_T
*state
)
350 vchiq_doorbell_irq(int irq
, void *dev_id
)
352 VCHIQ_STATE_T
*state
= dev_id
;
353 irqreturn_t ret
= IRQ_NONE
;
356 /* Read (and clear) the doorbell */
357 status
= readl(g_regs
+ BELL0
);
359 if (status
& 0x4) { /* Was the doorbell rung? */
360 remote_event_pollall(state
);
368 cleaup_pagelistinfo(struct vchiq_pagelist_info
*pagelistinfo
)
370 if (pagelistinfo
->scatterlist_mapped
) {
371 dma_unmap_sg(g_dev
, pagelistinfo
->scatterlist
,
372 pagelistinfo
->num_pages
, pagelistinfo
->dma_dir
);
375 if (pagelistinfo
->pages_need_release
) {
378 for (i
= 0; i
< pagelistinfo
->num_pages
; i
++)
379 put_page(pagelistinfo
->pages
[i
]);
382 dma_free_coherent(g_dev
, pagelistinfo
->pagelist_buffer_size
,
383 pagelistinfo
->pagelist
, pagelistinfo
->dma_addr
);
386 /* There is a potential problem with partial cache lines (pages?)
387 ** at the ends of the block when reading. If the CPU accessed anything in
388 ** the same line (page?) then it may have pulled old data into the cache,
389 ** obscuring the new data underneath. We can solve this by transferring the
390 ** partial cache lines separately, and allowing the ARM to copy into the
394 static struct vchiq_pagelist_info
*
395 create_pagelist(char __user
*buf
, size_t count
, unsigned short type
,
396 struct task_struct
*task
)
398 PAGELIST_T
*pagelist
;
399 struct vchiq_pagelist_info
*pagelistinfo
;
402 unsigned int num_pages
, offset
, i
, k
;
404 size_t pagelist_size
;
405 struct scatterlist
*scatterlist
, *sg
;
409 offset
= ((unsigned int)(unsigned long)buf
& (PAGE_SIZE
- 1));
410 num_pages
= DIV_ROUND_UP(count
+ offset
, PAGE_SIZE
);
412 pagelist_size
= sizeof(PAGELIST_T
) +
413 (num_pages
* sizeof(u32
)) +
414 (num_pages
* sizeof(pages
[0]) +
415 (num_pages
* sizeof(struct scatterlist
))) +
416 sizeof(struct vchiq_pagelist_info
);
418 /* Allocate enough storage to hold the page pointers and the page
421 pagelist
= dma_zalloc_coherent(g_dev
,
426 vchiq_log_trace(vchiq_arm_log_level
, "create_pagelist - %pK",
431 addrs
= pagelist
->addrs
;
432 pages
= (struct page
**)(addrs
+ num_pages
);
433 scatterlist
= (struct scatterlist
*)(pages
+ num_pages
);
434 pagelistinfo
= (struct vchiq_pagelist_info
*)
435 (scatterlist
+ num_pages
);
437 pagelist
->length
= count
;
438 pagelist
->type
= type
;
439 pagelist
->offset
= offset
;
441 /* Populate the fields of the pagelistinfo structure */
442 pagelistinfo
->pagelist
= pagelist
;
443 pagelistinfo
->pagelist_buffer_size
= pagelist_size
;
444 pagelistinfo
->dma_addr
= dma_addr
;
445 pagelistinfo
->dma_dir
= (type
== PAGELIST_WRITE
) ?
446 DMA_TO_DEVICE
: DMA_FROM_DEVICE
;
447 pagelistinfo
->num_pages
= num_pages
;
448 pagelistinfo
->pages_need_release
= 0;
449 pagelistinfo
->pages
= pages
;
450 pagelistinfo
->scatterlist
= scatterlist
;
451 pagelistinfo
->scatterlist_mapped
= 0;
453 if (is_vmalloc_addr(buf
)) {
454 unsigned long length
= count
;
455 unsigned int off
= offset
;
457 for (actual_pages
= 0; actual_pages
< num_pages
;
459 struct page
*pg
= vmalloc_to_page(buf
+ (actual_pages
*
461 size_t bytes
= PAGE_SIZE
- off
;
465 pages
[actual_pages
] = pg
;
469 /* do not try and release vmalloc pages */
471 down_read(&task
->mm
->mmap_sem
);
472 actual_pages
= get_user_pages(
473 (unsigned long)buf
& ~(PAGE_SIZE
- 1),
475 (type
== PAGELIST_READ
) ? FOLL_WRITE
: 0,
478 up_read(&task
->mm
->mmap_sem
);
480 if (actual_pages
!= num_pages
) {
481 vchiq_log_info(vchiq_arm_log_level
,
482 "create_pagelist - only %d/%d pages locked",
486 /* This is probably due to the process being killed */
487 while (actual_pages
> 0)
490 put_page(pages
[actual_pages
]);
492 cleaup_pagelistinfo(pagelistinfo
);
495 /* release user pages */
496 pagelistinfo
->pages_need_release
= 1;
500 * Initialize the scatterlist so that the magic cookie
501 * is filled if debugging is enabled
503 sg_init_table(scatterlist
, num_pages
);
504 /* Now set the pages for each scatterlist */
505 for (i
= 0; i
< num_pages
; i
++)
506 sg_set_page(scatterlist
+ i
, pages
[i
], PAGE_SIZE
, 0);
508 dma_buffers
= dma_map_sg(g_dev
,
511 pagelistinfo
->dma_dir
);
513 if (dma_buffers
== 0) {
514 cleaup_pagelistinfo(pagelistinfo
);
518 pagelistinfo
->scatterlist_mapped
= 1;
520 /* Combine adjacent blocks for performance */
522 for_each_sg(scatterlist
, sg
, dma_buffers
, i
) {
523 u32 len
= sg_dma_len(sg
);
524 u32 addr
= sg_dma_address(sg
);
526 /* Note: addrs is the address + page_count - 1
527 * The firmware expects the block to be page
528 * aligned and a multiple of the page size
531 WARN_ON(len
& ~PAGE_MASK
);
532 WARN_ON(addr
& ~PAGE_MASK
);
534 ((addrs
[k
- 1] & PAGE_MASK
) |
535 ((addrs
[k
- 1] & ~PAGE_MASK
) + 1) << PAGE_SHIFT
)
537 addrs
[k
- 1] += (len
>> PAGE_SHIFT
);
539 addrs
[k
++] = addr
| ((len
>> PAGE_SHIFT
) - 1);
543 /* Partial cache lines (fragments) require special measures */
544 if ((type
== PAGELIST_READ
) &&
545 ((pagelist
->offset
& (g_cache_line_size
- 1)) ||
546 ((pagelist
->offset
+ pagelist
->length
) &
547 (g_cache_line_size
- 1)))) {
550 if (down_interruptible(&g_free_fragments_sema
) != 0) {
551 cleaup_pagelistinfo(pagelistinfo
);
555 WARN_ON(g_free_fragments
== NULL
);
557 down(&g_free_fragments_mutex
);
558 fragments
= g_free_fragments
;
559 WARN_ON(fragments
== NULL
);
560 g_free_fragments
= *(char **) g_free_fragments
;
561 up(&g_free_fragments_mutex
);
562 pagelist
->type
= PAGELIST_READ_WITH_FRAGMENTS
+
563 (fragments
- g_fragments_base
) / g_fragments_size
;
570 free_pagelist(struct vchiq_pagelist_info
*pagelistinfo
,
574 PAGELIST_T
*pagelist
= pagelistinfo
->pagelist
;
575 struct page
**pages
= pagelistinfo
->pages
;
576 unsigned int num_pages
= pagelistinfo
->num_pages
;
578 vchiq_log_trace(vchiq_arm_log_level
, "free_pagelist - %pK, %d",
579 pagelistinfo
->pagelist
, actual
);
582 * NOTE: dma_unmap_sg must be called before the
583 * cpu can touch any of the data/pages.
585 dma_unmap_sg(g_dev
, pagelistinfo
->scatterlist
,
586 pagelistinfo
->num_pages
, pagelistinfo
->dma_dir
);
587 pagelistinfo
->scatterlist_mapped
= 0;
589 /* Deal with any partial cache lines (fragments) */
590 if (pagelist
->type
>= PAGELIST_READ_WITH_FRAGMENTS
) {
591 char *fragments
= g_fragments_base
+
592 (pagelist
->type
- PAGELIST_READ_WITH_FRAGMENTS
) *
594 int head_bytes
, tail_bytes
;
596 head_bytes
= (g_cache_line_size
- pagelist
->offset
) &
597 (g_cache_line_size
- 1);
598 tail_bytes
= (pagelist
->offset
+ actual
) &
599 (g_cache_line_size
- 1);
601 if ((actual
>= 0) && (head_bytes
!= 0)) {
602 if (head_bytes
> actual
)
605 memcpy((char *)page_address(pages
[0]) +
610 if ((actual
>= 0) && (head_bytes
< actual
) &&
612 memcpy((char *)page_address(pages
[num_pages
- 1]) +
613 ((pagelist
->offset
+ actual
) &
614 (PAGE_SIZE
- 1) & ~(g_cache_line_size
- 1)),
615 fragments
+ g_cache_line_size
,
619 down(&g_free_fragments_mutex
);
620 *(char **)fragments
= g_free_fragments
;
621 g_free_fragments
= fragments
;
622 up(&g_free_fragments_mutex
);
623 up(&g_free_fragments_sema
);
626 /* Need to mark all the pages dirty. */
627 if (pagelist
->type
!= PAGELIST_WRITE
&&
628 pagelistinfo
->pages_need_release
) {
629 for (i
= 0; i
< num_pages
; i
++)
630 set_page_dirty(pages
[i
]);
633 cleaup_pagelistinfo(pagelistinfo
);