2 * Copyright (c) 2010-2012 Broadcom. All rights reserved.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions, and the following disclaimer,
9 * without modification.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. The names of the above-listed copyright holders may not be used
14 * to endorse or promote products derived from this software without
15 * specific prior written permission.
17 * ALTERNATIVELY, this software may be distributed under the terms of the
18 * GNU General Public License ("GPL") version 2, as published by the Free
19 * Software Foundation.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
22 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
23 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
25 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
26 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
27 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
28 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
29 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
30 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
31 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <linux/kernel.h>
35 #include <linux/types.h>
36 #include <linux/errno.h>
37 #include <linux/interrupt.h>
38 #include <linux/pagemap.h>
39 #include <linux/dma-mapping.h>
40 #include <linux/version.h>
42 #include <linux/platform_device.h>
43 #include <linux/uaccess.h>
45 #include <asm/pgtable.h>
46 #include <soc/bcm2835/raspberrypi-firmware.h>
48 #define dmac_map_area __glue(_CACHE,_dma_map_area)
49 #define dmac_unmap_area __glue(_CACHE,_dma_unmap_area)
51 extern void dmac_map_area(const void *, size_t, int);
52 extern void dmac_unmap_area(const void *, size_t, int);
54 #define TOTAL_SLOTS (VCHIQ_SLOT_ZERO_SLOTS + 2 * 32)
56 #define VCHIQ_ARM_ADDRESS(x) ((void *)((char *)x + g_virt_to_bus_offset))
58 #include "vchiq_arm.h"
59 #include "vchiq_2835.h"
60 #include "vchiq_connected.h"
61 #include "vchiq_killable.h"
63 #define MAX_FRAGMENTS (VCHIQ_NUM_CURRENT_BULKS * 2)
68 typedef struct vchiq_2835_state_struct
{
70 VCHIQ_ARM_STATE_T arm_state
;
71 } VCHIQ_2835_ARM_STATE_T
;
73 static void __iomem
*g_regs
;
74 static unsigned int g_cache_line_size
= sizeof(CACHE_LINE_SIZE
);
75 static unsigned int g_fragments_size
;
76 static char *g_fragments_base
;
77 static char *g_free_fragments
;
78 static struct semaphore g_free_fragments_sema
;
79 static unsigned long g_virt_to_bus_offset
;
81 extern int vchiq_arm_log_level
;
83 static DEFINE_SEMAPHORE(g_free_fragments_mutex
);
86 vchiq_doorbell_irq(int irq
, void *dev_id
);
89 create_pagelist(char __user
*buf
, size_t count
, unsigned short type
,
90 struct task_struct
*task
, PAGELIST_T
** ppagelist
);
93 free_pagelist(PAGELIST_T
*pagelist
, int actual
);
95 int vchiq_platform_init(struct platform_device
*pdev
, VCHIQ_STATE_T
*state
)
97 struct device
*dev
= &pdev
->dev
;
98 struct rpi_firmware
*fw
= platform_get_drvdata(pdev
);
99 VCHIQ_SLOT_ZERO_T
*vchiq_slot_zero
;
100 struct resource
*res
;
102 dma_addr_t slot_phys
;
104 int slot_mem_size
, frag_mem_size
;
107 g_virt_to_bus_offset
= virt_to_dma(dev
, (void *)0);
109 (void)of_property_read_u32(dev
->of_node
, "cache-line-size",
111 g_fragments_size
= 2 * g_cache_line_size
;
113 /* Allocate space for the channels in coherent memory */
114 slot_mem_size
= PAGE_ALIGN(TOTAL_SLOTS
* VCHIQ_SLOT_SIZE
);
115 frag_mem_size
= PAGE_ALIGN(g_fragments_size
* MAX_FRAGMENTS
);
117 slot_mem
= dmam_alloc_coherent(dev
, slot_mem_size
+ frag_mem_size
,
118 &slot_phys
, GFP_KERNEL
);
120 dev_err(dev
, "could not allocate DMA memory\n");
124 WARN_ON(((int)slot_mem
& (PAGE_SIZE
- 1)) != 0);
126 vchiq_slot_zero
= vchiq_init_slots(slot_mem
, slot_mem_size
);
127 if (!vchiq_slot_zero
)
130 vchiq_slot_zero
->platform_data
[VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX
] =
131 (int)slot_phys
+ slot_mem_size
;
132 vchiq_slot_zero
->platform_data
[VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX
] =
135 g_fragments_base
= (char *)slot_mem
+ slot_mem_size
;
136 slot_mem_size
+= frag_mem_size
;
138 g_free_fragments
= g_fragments_base
;
139 for (i
= 0; i
< (MAX_FRAGMENTS
- 1); i
++) {
140 *(char **)&g_fragments_base
[i
*g_fragments_size
] =
141 &g_fragments_base
[(i
+ 1)*g_fragments_size
];
143 *(char **)&g_fragments_base
[i
* g_fragments_size
] = NULL
;
144 sema_init(&g_free_fragments_sema
, MAX_FRAGMENTS
);
146 if (vchiq_init_state(state
, vchiq_slot_zero
, 0) != VCHIQ_SUCCESS
)
149 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
150 g_regs
= devm_ioremap_resource(&pdev
->dev
, res
);
152 return PTR_ERR(g_regs
);
154 irq
= platform_get_irq(pdev
, 0);
156 dev_err(dev
, "failed to get IRQ\n");
160 err
= devm_request_irq(dev
, irq
, vchiq_doorbell_irq
, IRQF_IRQPOLL
,
161 "VCHIQ doorbell", state
);
163 dev_err(dev
, "failed to register irq=%d\n", irq
);
167 /* Send the base address of the slots to VideoCore */
168 channelbase
= slot_phys
;
169 err
= rpi_firmware_property(fw
, RPI_FIRMWARE_VCHIQ_INIT
,
170 &channelbase
, sizeof(channelbase
));
171 if (err
|| channelbase
) {
172 dev_err(dev
, "failed to set channelbase\n");
173 return err
? : -ENXIO
;
176 vchiq_log_info(vchiq_arm_log_level
,
177 "vchiq_init - done (slots %x, phys %pad)",
178 (unsigned int)vchiq_slot_zero
, &slot_phys
);
180 vchiq_call_connected_callbacks();
186 vchiq_platform_init_state(VCHIQ_STATE_T
*state
)
188 VCHIQ_STATUS_T status
= VCHIQ_SUCCESS
;
189 state
->platform_state
= kzalloc(sizeof(VCHIQ_2835_ARM_STATE_T
), GFP_KERNEL
);
190 ((VCHIQ_2835_ARM_STATE_T
*)state
->platform_state
)->inited
= 1;
191 status
= vchiq_arm_init_state(state
, &((VCHIQ_2835_ARM_STATE_T
*)state
->platform_state
)->arm_state
);
192 if(status
!= VCHIQ_SUCCESS
)
194 ((VCHIQ_2835_ARM_STATE_T
*)state
->platform_state
)->inited
= 0;
200 vchiq_platform_get_arm_state(VCHIQ_STATE_T
*state
)
202 if(!((VCHIQ_2835_ARM_STATE_T
*)state
->platform_state
)->inited
)
206 return &((VCHIQ_2835_ARM_STATE_T
*)state
->platform_state
)->arm_state
;
210 remote_event_signal(REMOTE_EVENT_T
*event
)
216 dsb(); /* data barrier operation */
219 writel(0, g_regs
+ BELL2
); /* trigger vc interrupt */
223 vchiq_copy_from_user(void *dst
, const void *src
, int size
)
225 if ((uint32_t)src
< TASK_SIZE
) {
226 return copy_from_user(dst
, src
, size
);
228 memcpy(dst
, src
, size
);
234 vchiq_prepare_bulk_data(VCHIQ_BULK_T
*bulk
, VCHI_MEM_HANDLE_T memhandle
,
235 void *offset
, int size
, int dir
)
237 PAGELIST_T
*pagelist
;
240 WARN_ON(memhandle
!= VCHI_MEM_HANDLE_INVALID
);
242 ret
= create_pagelist((char __user
*)offset
, size
,
243 (dir
== VCHIQ_BULK_RECEIVE
)
251 bulk
->handle
= memhandle
;
252 bulk
->data
= VCHIQ_ARM_ADDRESS(pagelist
);
254 /* Store the pagelist address in remote_data, which isn't used by the
256 bulk
->remote_data
= pagelist
;
258 return VCHIQ_SUCCESS
;
262 vchiq_complete_bulk(VCHIQ_BULK_T
*bulk
)
264 if (bulk
&& bulk
->remote_data
&& bulk
->actual
)
265 free_pagelist((PAGELIST_T
*)bulk
->remote_data
, bulk
->actual
);
269 vchiq_transfer_bulk(VCHIQ_BULK_T
*bulk
)
272 * This should only be called on the master (VideoCore) side, but
273 * provide an implementation to avoid the need for ifdefery.
279 vchiq_dump_platform_state(void *dump_context
)
283 len
= snprintf(buf
, sizeof(buf
),
284 " Platform: 2835 (VC master)");
285 vchiq_dump(dump_context
, buf
, len
+ 1);
289 vchiq_platform_suspend(VCHIQ_STATE_T
*state
)
295 vchiq_platform_resume(VCHIQ_STATE_T
*state
)
297 return VCHIQ_SUCCESS
;
301 vchiq_platform_paused(VCHIQ_STATE_T
*state
)
306 vchiq_platform_resumed(VCHIQ_STATE_T
*state
)
311 vchiq_platform_videocore_wanted(VCHIQ_STATE_T
* state
)
313 return 1; // autosuspend not supported - videocore always wanted
317 vchiq_platform_use_suspend_timer(void)
322 vchiq_dump_platform_use_state(VCHIQ_STATE_T
*state
)
324 vchiq_log_info(vchiq_arm_log_level
, "Suspend timer not in use");
327 vchiq_platform_handle_timeout(VCHIQ_STATE_T
*state
)
336 vchiq_doorbell_irq(int irq
, void *dev_id
)
338 VCHIQ_STATE_T
*state
= dev_id
;
339 irqreturn_t ret
= IRQ_NONE
;
342 /* Read (and clear) the doorbell */
343 status
= readl(g_regs
+ BELL0
);
345 if (status
& 0x4) { /* Was the doorbell rung? */
346 remote_event_pollall(state
);
353 /* There is a potential problem with partial cache lines (pages?)
354 ** at the ends of the block when reading. If the CPU accessed anything in
355 ** the same line (page?) then it may have pulled old data into the cache,
356 ** obscuring the new data underneath. We can solve this by transferring the
357 ** partial cache lines separately, and allowing the ARM to copy into the
360 ** N.B. This implementation plays slightly fast and loose with the Linux
361 ** driver programming rules, e.g. its use of dmac_map_area instead of
362 ** dma_map_single, but it isn't a multi-platform driver and it benefits
363 ** from increased speed as a result.
367 create_pagelist(char __user
*buf
, size_t count
, unsigned short type
,
368 struct task_struct
*task
, PAGELIST_T
** ppagelist
)
370 PAGELIST_T
*pagelist
;
372 unsigned long *addrs
;
373 unsigned int num_pages
, offset
, i
;
374 char *addr
, *base_addr
, *next_addr
;
375 int run
, addridx
, actual_pages
;
376 unsigned long *need_release
;
378 offset
= (unsigned int)buf
& (PAGE_SIZE
- 1);
379 num_pages
= (count
+ offset
+ PAGE_SIZE
- 1) / PAGE_SIZE
;
383 /* Allocate enough storage to hold the page pointers and the page
386 pagelist
= kmalloc(sizeof(PAGELIST_T
) +
387 (num_pages
* sizeof(unsigned long)) +
388 sizeof(unsigned long) +
389 (num_pages
* sizeof(pages
[0])),
392 vchiq_log_trace(vchiq_arm_log_level
,
393 "create_pagelist - %x", (unsigned int)pagelist
);
397 addrs
= pagelist
->addrs
;
398 need_release
= (unsigned long *)(addrs
+ num_pages
);
399 pages
= (struct page
**)(addrs
+ num_pages
+ 1);
401 if (is_vmalloc_addr(buf
)) {
402 int dir
= (type
== PAGELIST_WRITE
) ?
403 DMA_TO_DEVICE
: DMA_FROM_DEVICE
;
404 unsigned long length
= count
;
405 unsigned int off
= offset
;
407 for (actual_pages
= 0; actual_pages
< num_pages
;
409 struct page
*pg
= vmalloc_to_page(buf
+ (actual_pages
*
411 size_t bytes
= PAGE_SIZE
- off
;
415 pages
[actual_pages
] = pg
;
416 dmac_map_area(page_address(pg
) + off
, bytes
, dir
);
420 *need_release
= 0; /* do not try and release vmalloc pages */
422 down_read(&task
->mm
->mmap_sem
);
423 actual_pages
= get_user_pages(task
, task
->mm
,
424 (unsigned long)buf
& ~(PAGE_SIZE
- 1),
426 (type
== PAGELIST_READ
) /*Write */ ,
430 up_read(&task
->mm
->mmap_sem
);
432 if (actual_pages
!= num_pages
) {
433 vchiq_log_info(vchiq_arm_log_level
,
434 "create_pagelist - only %d/%d pages locked",
438 /* This is probably due to the process being killed */
439 while (actual_pages
> 0)
442 page_cache_release(pages
[actual_pages
]);
445 if (actual_pages
== 0)
446 actual_pages
= -ENOMEM
;
449 *need_release
= 1; /* release user pages */
452 pagelist
->length
= count
;
453 pagelist
->type
= type
;
454 pagelist
->offset
= offset
;
456 /* Group the pages into runs of contiguous pages */
458 base_addr
= VCHIQ_ARM_ADDRESS(page_address(pages
[0]));
459 next_addr
= base_addr
+ PAGE_SIZE
;
463 for (i
= 1; i
< num_pages
; i
++) {
464 addr
= VCHIQ_ARM_ADDRESS(page_address(pages
[i
]));
465 if ((addr
== next_addr
) && (run
< (PAGE_SIZE
- 1))) {
466 next_addr
+= PAGE_SIZE
;
469 addrs
[addridx
] = (unsigned long)base_addr
+ run
;
472 next_addr
= addr
+ PAGE_SIZE
;
477 addrs
[addridx
] = (unsigned long)base_addr
+ run
;
480 /* Partial cache lines (fragments) require special measures */
481 if ((type
== PAGELIST_READ
) &&
482 ((pagelist
->offset
& (g_cache_line_size
- 1)) ||
483 ((pagelist
->offset
+ pagelist
->length
) &
484 (g_cache_line_size
- 1)))) {
487 if (down_interruptible(&g_free_fragments_sema
) != 0) {
492 WARN_ON(g_free_fragments
== NULL
);
494 down(&g_free_fragments_mutex
);
495 fragments
= g_free_fragments
;
496 WARN_ON(fragments
== NULL
);
497 g_free_fragments
= *(char **) g_free_fragments
;
498 up(&g_free_fragments_mutex
);
499 pagelist
->type
= PAGELIST_READ_WITH_FRAGMENTS
+
500 (fragments
- g_fragments_base
) / g_fragments_size
;
503 dmac_flush_range(pagelist
, addrs
+ num_pages
);
505 *ppagelist
= pagelist
;
511 free_pagelist(PAGELIST_T
*pagelist
, int actual
)
513 unsigned long *need_release
;
515 unsigned int num_pages
, i
;
517 vchiq_log_trace(vchiq_arm_log_level
,
518 "free_pagelist - %x, %d", (unsigned int)pagelist
, actual
);
521 (pagelist
->length
+ pagelist
->offset
+ PAGE_SIZE
- 1) /
524 need_release
= (unsigned long *)(pagelist
->addrs
+ num_pages
);
525 pages
= (struct page
**)(pagelist
->addrs
+ num_pages
+ 1);
527 /* Deal with any partial cache lines (fragments) */
528 if (pagelist
->type
>= PAGELIST_READ_WITH_FRAGMENTS
) {
529 char *fragments
= g_fragments_base
+
530 (pagelist
->type
- PAGELIST_READ_WITH_FRAGMENTS
) *
532 int head_bytes
, tail_bytes
;
533 head_bytes
= (g_cache_line_size
- pagelist
->offset
) &
534 (g_cache_line_size
- 1);
535 tail_bytes
= (pagelist
->offset
+ actual
) &
536 (g_cache_line_size
- 1);
538 if ((actual
>= 0) && (head_bytes
!= 0)) {
539 if (head_bytes
> actual
)
542 memcpy((char *)page_address(pages
[0]) +
547 if ((actual
>= 0) && (head_bytes
< actual
) &&
549 memcpy((char *)page_address(pages
[num_pages
- 1]) +
550 ((pagelist
->offset
+ actual
) &
551 (PAGE_SIZE
- 1) & ~(g_cache_line_size
- 1)),
552 fragments
+ g_cache_line_size
,
556 down(&g_free_fragments_mutex
);
557 *(char **)fragments
= g_free_fragments
;
558 g_free_fragments
= fragments
;
559 up(&g_free_fragments_mutex
);
560 up(&g_free_fragments_sema
);
564 unsigned int length
= pagelist
->length
;
565 unsigned int offset
= pagelist
->offset
;
567 for (i
= 0; i
< num_pages
; i
++) {
568 struct page
*pg
= pages
[i
];
570 if (pagelist
->type
!= PAGELIST_WRITE
) {
571 unsigned int bytes
= PAGE_SIZE
- offset
;
575 dmac_unmap_area(page_address(pg
) + offset
,
576 bytes
, DMA_FROM_DEVICE
);
581 page_cache_release(pg
);