]>
Commit | Line | Data |
---|---|---|
71bad7f0 | 1 | /** |
2 | * Copyright (c) 2010-2012 Broadcom. All rights reserved. | |
3 | * | |
4 | * Redistribution and use in source and binary forms, with or without | |
5 | * modification, are permitted provided that the following conditions | |
6 | * are met: | |
7 | * 1. Redistributions of source code must retain the above copyright | |
8 | * notice, this list of conditions, and the following disclaimer, | |
9 | * without modification. | |
10 | * 2. Redistributions in binary form must reproduce the above copyright | |
11 | * notice, this list of conditions and the following disclaimer in the | |
12 | * documentation and/or other materials provided with the distribution. | |
13 | * 3. The names of the above-listed copyright holders may not be used | |
14 | * to endorse or promote products derived from this software without | |
15 | * specific prior written permission. | |
16 | * | |
17 | * ALTERNATIVELY, this software may be distributed under the terms of the | |
18 | * GNU General Public License ("GPL") version 2, as published by the Free | |
19 | * Software Foundation. | |
20 | * | |
21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS | |
22 | * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | |
23 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
24 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR | |
25 | * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, | |
26 | * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, | |
27 | * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR | |
28 | * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF | |
29 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING | |
30 | * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS | |
31 | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
32 | */ | |
33 | ||
34 | #include <linux/kernel.h> | |
35 | #include <linux/types.h> | |
36 | #include <linux/errno.h> | |
37 | #include <linux/interrupt.h> | |
38 | #include <linux/pagemap.h> | |
39 | #include <linux/dma-mapping.h> | |
71bad7f0 | 40 | #include <linux/io.h> |
41 | #include <linux/platform_device.h> | |
42 | #include <linux/uaccess.h> | |
6d80d636 | 43 | #include <linux/mm.h> |
71bad7f0 | 44 | #include <linux/of.h> |
71bad7f0 | 45 | #include <soc/bcm2835/raspberrypi-firmware.h> |
46 | ||
71bad7f0 | 47 | #define TOTAL_SLOTS (VCHIQ_SLOT_ZERO_SLOTS + 2 * 32) |
48 | ||
71bad7f0 | 49 | #include "vchiq_arm.h" |
71bad7f0 | 50 | #include "vchiq_connected.h" |
51 | #include "vchiq_killable.h" | |
946d61ac | 52 | #include "vchiq_pagelist.h" |
71bad7f0 | 53 | |
54 | #define MAX_FRAGMENTS (VCHIQ_NUM_CURRENT_BULKS * 2) | |
55 | ||
946d61ac SW |
56 | #define VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX 0 |
57 | #define VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX 1 | |
58 | ||
71bad7f0 | 59 | #define BELL0 0x00 |
60 | #define BELL2 0x08 | |
61 | ||
62 | typedef struct vchiq_2835_state_struct { | |
f306ed07 SW |
63 | int inited; |
64 | VCHIQ_ARM_STATE_T arm_state; | |
71bad7f0 | 65 | } VCHIQ_2835_ARM_STATE_T; |
66 | ||
4807f2c0 MZ |
67 | struct vchiq_pagelist_info { |
68 | PAGELIST_T *pagelist; | |
69 | size_t pagelist_buffer_size; | |
70 | dma_addr_t dma_addr; | |
71 | enum dma_data_direction dma_dir; | |
72 | unsigned int num_pages; | |
73 | unsigned int pages_need_release; | |
74 | struct page **pages; | |
75 | struct scatterlist *scatterlist; | |
76 | unsigned int scatterlist_mapped; | |
77 | }; | |
78 | ||
71bad7f0 | 79 | static void __iomem *g_regs; |
80 | static unsigned int g_cache_line_size = sizeof(CACHE_LINE_SIZE); | |
81 | static unsigned int g_fragments_size; | |
82 | static char *g_fragments_base; | |
83 | static char *g_free_fragments; | |
84 | static struct semaphore g_free_fragments_sema; | |
cf9caf19 | 85 | static struct device *g_dev; |
71bad7f0 | 86 | |
87 | extern int vchiq_arm_log_level; | |
88 | ||
89 | static DEFINE_SEMAPHORE(g_free_fragments_mutex); | |
90 | ||
91 | static irqreturn_t | |
92 | vchiq_doorbell_irq(int irq, void *dev_id); | |
93 | ||
4807f2c0 | 94 | static struct vchiq_pagelist_info * |
71bad7f0 | 95 | create_pagelist(char __user *buf, size_t count, unsigned short type, |
4807f2c0 | 96 | struct task_struct *task); |
71bad7f0 | 97 | |
98 | static void | |
4807f2c0 MZ |
99 | free_pagelist(struct vchiq_pagelist_info *pagelistinfo, |
100 | int actual); | |
71bad7f0 | 101 | |
102 | int vchiq_platform_init(struct platform_device *pdev, VCHIQ_STATE_T *state) | |
103 | { | |
104 | struct device *dev = &pdev->dev; | |
105 | struct rpi_firmware *fw = platform_get_drvdata(pdev); | |
106 | VCHIQ_SLOT_ZERO_T *vchiq_slot_zero; | |
107 | struct resource *res; | |
108 | void *slot_mem; | |
109 | dma_addr_t slot_phys; | |
110 | u32 channelbase; | |
111 | int slot_mem_size, frag_mem_size; | |
112 | int err, irq, i; | |
113 | ||
c01cc53d MZ |
114 | /* |
115 | * VCHI messages between the CPU and firmware use | |
116 | * 32-bit bus addresses. | |
117 | */ | |
118 | err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); | |
119 | ||
120 | if (err < 0) | |
121 | return err; | |
122 | ||
6cf1bf63 | 123 | err = of_property_read_u32(dev->of_node, "cache-line-size", |
71bad7f0 | 124 | &g_cache_line_size); |
6cf1bf63 MZ |
125 | |
126 | if (err) { | |
127 | dev_err(dev, "Missing cache-line-size property\n"); | |
128 | return -ENODEV; | |
129 | } | |
130 | ||
71bad7f0 | 131 | g_fragments_size = 2 * g_cache_line_size; |
132 | ||
133 | /* Allocate space for the channels in coherent memory */ | |
134 | slot_mem_size = PAGE_ALIGN(TOTAL_SLOTS * VCHIQ_SLOT_SIZE); | |
135 | frag_mem_size = PAGE_ALIGN(g_fragments_size * MAX_FRAGMENTS); | |
136 | ||
137 | slot_mem = dmam_alloc_coherent(dev, slot_mem_size + frag_mem_size, | |
138 | &slot_phys, GFP_KERNEL); | |
139 | if (!slot_mem) { | |
140 | dev_err(dev, "could not allocate DMA memory\n"); | |
141 | return -ENOMEM; | |
142 | } | |
143 | ||
cf9caf19 | 144 | WARN_ON(((unsigned long)slot_mem & (PAGE_SIZE - 1)) != 0); |
71bad7f0 | 145 | |
146 | vchiq_slot_zero = vchiq_init_slots(slot_mem, slot_mem_size); | |
147 | if (!vchiq_slot_zero) | |
148 | return -EINVAL; | |
149 | ||
150 | vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX] = | |
151 | (int)slot_phys + slot_mem_size; | |
152 | vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX] = | |
153 | MAX_FRAGMENTS; | |
154 | ||
155 | g_fragments_base = (char *)slot_mem + slot_mem_size; | |
156 | slot_mem_size += frag_mem_size; | |
157 | ||
158 | g_free_fragments = g_fragments_base; | |
159 | for (i = 0; i < (MAX_FRAGMENTS - 1); i++) { | |
160 | *(char **)&g_fragments_base[i*g_fragments_size] = | |
161 | &g_fragments_base[(i + 1)*g_fragments_size]; | |
162 | } | |
163 | *(char **)&g_fragments_base[i * g_fragments_size] = NULL; | |
164 | sema_init(&g_free_fragments_sema, MAX_FRAGMENTS); | |
165 | ||
166 | if (vchiq_init_state(state, vchiq_slot_zero, 0) != VCHIQ_SUCCESS) | |
167 | return -EINVAL; | |
168 | ||
169 | res = platform_get_resource(pdev, IORESOURCE_MEM, 0); | |
170 | g_regs = devm_ioremap_resource(&pdev->dev, res); | |
171 | if (IS_ERR(g_regs)) | |
172 | return PTR_ERR(g_regs); | |
173 | ||
174 | irq = platform_get_irq(pdev, 0); | |
175 | if (irq <= 0) { | |
176 | dev_err(dev, "failed to get IRQ\n"); | |
177 | return irq; | |
178 | } | |
179 | ||
180 | err = devm_request_irq(dev, irq, vchiq_doorbell_irq, IRQF_IRQPOLL, | |
181 | "VCHIQ doorbell", state); | |
182 | if (err) { | |
183 | dev_err(dev, "failed to register irq=%d\n", irq); | |
184 | return err; | |
185 | } | |
186 | ||
187 | /* Send the base address of the slots to VideoCore */ | |
188 | channelbase = slot_phys; | |
189 | err = rpi_firmware_property(fw, RPI_FIRMWARE_VCHIQ_INIT, | |
190 | &channelbase, sizeof(channelbase)); | |
191 | if (err || channelbase) { | |
192 | dev_err(dev, "failed to set channelbase\n"); | |
193 | return err ? : -ENXIO; | |
194 | } | |
195 | ||
cf9caf19 | 196 | g_dev = dev; |
71bad7f0 | 197 | vchiq_log_info(vchiq_arm_log_level, |
df044ebf GKH |
198 | "vchiq_init - done (slots %pK, phys %pad)", |
199 | vchiq_slot_zero, &slot_phys); | |
71bad7f0 | 200 | |
201 | vchiq_call_connected_callbacks(); | |
202 | ||
f306ed07 | 203 | return 0; |
71bad7f0 | 204 | } |
205 | ||
206 | VCHIQ_STATUS_T | |
207 | vchiq_platform_init_state(VCHIQ_STATE_T *state) | |
208 | { | |
f306ed07 | 209 | VCHIQ_STATUS_T status = VCHIQ_SUCCESS; |
6e475350 | 210 | |
f306ed07 | 211 | state->platform_state = kzalloc(sizeof(VCHIQ_2835_ARM_STATE_T), GFP_KERNEL); |
b33050d0 SW |
212 | ((VCHIQ_2835_ARM_STATE_T *)state->platform_state)->inited = 1; |
213 | status = vchiq_arm_init_state(state, &((VCHIQ_2835_ARM_STATE_T *)state->platform_state)->arm_state); | |
214 | if (status != VCHIQ_SUCCESS) | |
f306ed07 | 215 | { |
b33050d0 | 216 | ((VCHIQ_2835_ARM_STATE_T *)state->platform_state)->inited = 0; |
f306ed07 SW |
217 | } |
218 | return status; | |
71bad7f0 | 219 | } |
220 | ||
221 | VCHIQ_ARM_STATE_T* | |
222 | vchiq_platform_get_arm_state(VCHIQ_STATE_T *state) | |
223 | { | |
b33050d0 | 224 | if (!((VCHIQ_2835_ARM_STATE_T *)state->platform_state)->inited) |
f306ed07 SW |
225 | { |
226 | BUG(); | |
227 | } | |
b33050d0 | 228 | return &((VCHIQ_2835_ARM_STATE_T *)state->platform_state)->arm_state; |
71bad7f0 | 229 | } |
230 | ||
231 | void | |
232 | remote_event_signal(REMOTE_EVENT_T *event) | |
233 | { | |
234 | wmb(); | |
235 | ||
236 | event->fired = 1; | |
237 | ||
35b7ebda | 238 | dsb(sy); /* data barrier operation */ |
71bad7f0 | 239 | |
240 | if (event->armed) | |
241 | writel(0, g_regs + BELL2); /* trigger vc interrupt */ | |
242 | } | |
243 | ||
71bad7f0 | 244 | VCHIQ_STATUS_T |
245 | vchiq_prepare_bulk_data(VCHIQ_BULK_T *bulk, VCHI_MEM_HANDLE_T memhandle, | |
246 | void *offset, int size, int dir) | |
247 | { | |
4807f2c0 | 248 | struct vchiq_pagelist_info *pagelistinfo; |
71bad7f0 | 249 | |
250 | WARN_ON(memhandle != VCHI_MEM_HANDLE_INVALID); | |
251 | ||
4807f2c0 MZ |
252 | pagelistinfo = create_pagelist((char __user *)offset, size, |
253 | (dir == VCHIQ_BULK_RECEIVE) | |
254 | ? PAGELIST_READ | |
255 | : PAGELIST_WRITE, | |
256 | current); | |
cf9caf19 | 257 | |
4807f2c0 | 258 | if (!pagelistinfo) |
71bad7f0 | 259 | return VCHIQ_ERROR; |
260 | ||
261 | bulk->handle = memhandle; | |
4807f2c0 | 262 | bulk->data = (void *)(unsigned long)pagelistinfo->dma_addr; |
71bad7f0 | 263 | |
4807f2c0 MZ |
264 | /* |
265 | * Store the pagelistinfo address in remote_data, | |
266 | * which isn't used by the slave. | |
267 | */ | |
268 | bulk->remote_data = pagelistinfo; | |
71bad7f0 | 269 | |
270 | return VCHIQ_SUCCESS; | |
271 | } | |
272 | ||
273 | void | |
274 | vchiq_complete_bulk(VCHIQ_BULK_T *bulk) | |
275 | { | |
276 | if (bulk && bulk->remote_data && bulk->actual) | |
4807f2c0 MZ |
277 | free_pagelist((struct vchiq_pagelist_info *)bulk->remote_data, |
278 | bulk->actual); | |
71bad7f0 | 279 | } |
280 | ||
281 | void | |
282 | vchiq_transfer_bulk(VCHIQ_BULK_T *bulk) | |
283 | { | |
284 | /* | |
285 | * This should only be called on the master (VideoCore) side, but | |
286 | * provide an implementation to avoid the need for ifdefery. | |
287 | */ | |
288 | BUG(); | |
289 | } | |
290 | ||
291 | void | |
292 | vchiq_dump_platform_state(void *dump_context) | |
293 | { | |
294 | char buf[80]; | |
295 | int len; | |
6e475350 | 296 | |
71bad7f0 | 297 | len = snprintf(buf, sizeof(buf), |
298 | " Platform: 2835 (VC master)"); | |
299 | vchiq_dump(dump_context, buf, len + 1); | |
300 | } | |
301 | ||
302 | VCHIQ_STATUS_T | |
303 | vchiq_platform_suspend(VCHIQ_STATE_T *state) | |
304 | { | |
f306ed07 | 305 | return VCHIQ_ERROR; |
71bad7f0 | 306 | } |
307 | ||
308 | VCHIQ_STATUS_T | |
309 | vchiq_platform_resume(VCHIQ_STATE_T *state) | |
310 | { | |
f306ed07 | 311 | return VCHIQ_SUCCESS; |
71bad7f0 | 312 | } |
313 | ||
314 | void | |
315 | vchiq_platform_paused(VCHIQ_STATE_T *state) | |
316 | { | |
317 | } | |
318 | ||
319 | void | |
320 | vchiq_platform_resumed(VCHIQ_STATE_T *state) | |
321 | { | |
322 | } | |
323 | ||
324 | int | |
b33050d0 | 325 | vchiq_platform_videocore_wanted(VCHIQ_STATE_T *state) |
71bad7f0 | 326 | { |
f306ed07 | 327 | return 1; // autosuspend not supported - videocore always wanted |
71bad7f0 | 328 | } |
329 | ||
330 | int | |
331 | vchiq_platform_use_suspend_timer(void) | |
332 | { | |
f306ed07 | 333 | return 0; |
71bad7f0 | 334 | } |
335 | void | |
336 | vchiq_dump_platform_use_state(VCHIQ_STATE_T *state) | |
337 | { | |
338 | vchiq_log_info(vchiq_arm_log_level, "Suspend timer not in use"); | |
339 | } | |
340 | void | |
341 | vchiq_platform_handle_timeout(VCHIQ_STATE_T *state) | |
342 | { | |
343 | (void)state; | |
344 | } | |
345 | /* | |
346 | * Local functions | |
347 | */ | |
348 | ||
349 | static irqreturn_t | |
350 | vchiq_doorbell_irq(int irq, void *dev_id) | |
351 | { | |
352 | VCHIQ_STATE_T *state = dev_id; | |
353 | irqreturn_t ret = IRQ_NONE; | |
354 | unsigned int status; | |
355 | ||
356 | /* Read (and clear) the doorbell */ | |
357 | status = readl(g_regs + BELL0); | |
358 | ||
359 | if (status & 0x4) { /* Was the doorbell rung? */ | |
360 | remote_event_pollall(state); | |
361 | ret = IRQ_HANDLED; | |
362 | } | |
363 | ||
364 | return ret; | |
365 | } | |
366 | ||
4807f2c0 MZ |
367 | static void |
368 | cleaup_pagelistinfo(struct vchiq_pagelist_info *pagelistinfo) | |
369 | { | |
370 | if (pagelistinfo->scatterlist_mapped) { | |
371 | dma_unmap_sg(g_dev, pagelistinfo->scatterlist, | |
372 | pagelistinfo->num_pages, pagelistinfo->dma_dir); | |
373 | } | |
374 | ||
375 | if (pagelistinfo->pages_need_release) { | |
376 | unsigned int i; | |
377 | ||
378 | for (i = 0; i < pagelistinfo->num_pages; i++) | |
379 | put_page(pagelistinfo->pages[i]); | |
380 | } | |
381 | ||
382 | dma_free_coherent(g_dev, pagelistinfo->pagelist_buffer_size, | |
383 | pagelistinfo->pagelist, pagelistinfo->dma_addr); | |
384 | } | |
385 | ||
71bad7f0 | 386 | /* There is a potential problem with partial cache lines (pages?) |
387 | ** at the ends of the block when reading. If the CPU accessed anything in | |
388 | ** the same line (page?) then it may have pulled old data into the cache, | |
389 | ** obscuring the new data underneath. We can solve this by transferring the | |
390 | ** partial cache lines separately, and allowing the ARM to copy into the | |
391 | ** cached area. | |
71bad7f0 | 392 | */ |
393 | ||
4807f2c0 | 394 | static struct vchiq_pagelist_info * |
71bad7f0 | 395 | create_pagelist(char __user *buf, size_t count, unsigned short type, |
4807f2c0 | 396 | struct task_struct *task) |
71bad7f0 | 397 | { |
398 | PAGELIST_T *pagelist; | |
4807f2c0 | 399 | struct vchiq_pagelist_info *pagelistinfo; |
71bad7f0 | 400 | struct page **pages; |
cf9caf19 MZ |
401 | u32 *addrs; |
402 | unsigned int num_pages, offset, i, k; | |
403 | int actual_pages; | |
cf9caf19 MZ |
404 | size_t pagelist_size; |
405 | struct scatterlist *scatterlist, *sg; | |
406 | int dma_buffers; | |
4807f2c0 | 407 | dma_addr_t dma_addr; |
71bad7f0 | 408 | |
cf9caf19 | 409 | offset = ((unsigned int)(unsigned long)buf & (PAGE_SIZE - 1)); |
0169acae | 410 | num_pages = DIV_ROUND_UP(count + offset, PAGE_SIZE); |
71bad7f0 | 411 | |
cf9caf19 | 412 | pagelist_size = sizeof(PAGELIST_T) + |
4807f2c0 | 413 | (num_pages * sizeof(u32)) + |
cf9caf19 | 414 | (num_pages * sizeof(pages[0]) + |
4807f2c0 MZ |
415 | (num_pages * sizeof(struct scatterlist))) + |
416 | sizeof(struct vchiq_pagelist_info); | |
cf9caf19 | 417 | |
71bad7f0 | 418 | /* Allocate enough storage to hold the page pointers and the page |
419 | ** list | |
420 | */ | |
cf9caf19 MZ |
421 | pagelist = dma_zalloc_coherent(g_dev, |
422 | pagelist_size, | |
4807f2c0 | 423 | &dma_addr, |
cf9caf19 | 424 | GFP_KERNEL); |
71bad7f0 | 425 | |
df044ebf GKH |
426 | vchiq_log_trace(vchiq_arm_log_level, "create_pagelist - %pK", |
427 | pagelist); | |
71bad7f0 | 428 | if (!pagelist) |
4807f2c0 MZ |
429 | return NULL; |
430 | ||
431 | addrs = pagelist->addrs; | |
432 | pages = (struct page **)(addrs + num_pages); | |
433 | scatterlist = (struct scatterlist *)(pages + num_pages); | |
434 | pagelistinfo = (struct vchiq_pagelist_info *) | |
435 | (scatterlist + num_pages); | |
71bad7f0 | 436 | |
4807f2c0 MZ |
437 | pagelist->length = count; |
438 | pagelist->type = type; | |
439 | pagelist->offset = offset; | |
440 | ||
441 | /* Populate the fields of the pagelistinfo structure */ | |
442 | pagelistinfo->pagelist = pagelist; | |
443 | pagelistinfo->pagelist_buffer_size = pagelist_size; | |
444 | pagelistinfo->dma_addr = dma_addr; | |
445 | pagelistinfo->dma_dir = (type == PAGELIST_WRITE) ? | |
446 | DMA_TO_DEVICE : DMA_FROM_DEVICE; | |
447 | pagelistinfo->num_pages = num_pages; | |
448 | pagelistinfo->pages_need_release = 0; | |
449 | pagelistinfo->pages = pages; | |
450 | pagelistinfo->scatterlist = scatterlist; | |
451 | pagelistinfo->scatterlist_mapped = 0; | |
71bad7f0 | 452 | |
453 | if (is_vmalloc_addr(buf)) { | |
71bad7f0 | 454 | unsigned long length = count; |
455 | unsigned int off = offset; | |
456 | ||
457 | for (actual_pages = 0; actual_pages < num_pages; | |
458 | actual_pages++) { | |
459 | struct page *pg = vmalloc_to_page(buf + (actual_pages * | |
460 | PAGE_SIZE)); | |
461 | size_t bytes = PAGE_SIZE - off; | |
462 | ||
463 | if (bytes > length) | |
464 | bytes = length; | |
465 | pages[actual_pages] = pg; | |
71bad7f0 | 466 | length -= bytes; |
467 | off = 0; | |
468 | } | |
4807f2c0 | 469 | /* do not try and release vmalloc pages */ |
71bad7f0 | 470 | } else { |
471 | down_read(&task->mm->mmap_sem); | |
166beccd | 472 | actual_pages = get_user_pages( |
71bad7f0 | 473 | (unsigned long)buf & ~(PAGE_SIZE - 1), |
474 | num_pages, | |
768ae309 | 475 | (type == PAGELIST_READ) ? FOLL_WRITE : 0, |
71bad7f0 | 476 | pages, |
477 | NULL /*vmas */); | |
478 | up_read(&task->mm->mmap_sem); | |
479 | ||
480 | if (actual_pages != num_pages) { | |
481 | vchiq_log_info(vchiq_arm_log_level, | |
482 | "create_pagelist - only %d/%d pages locked", | |
483 | actual_pages, | |
484 | num_pages); | |
485 | ||
486 | /* This is probably due to the process being killed */ | |
487 | while (actual_pages > 0) | |
488 | { | |
489 | actual_pages--; | |
232664b3 | 490 | put_page(pages[actual_pages]); |
71bad7f0 | 491 | } |
4807f2c0 MZ |
492 | cleaup_pagelistinfo(pagelistinfo); |
493 | return NULL; | |
71bad7f0 | 494 | } |
4807f2c0 MZ |
495 | /* release user pages */ |
496 | pagelistinfo->pages_need_release = 1; | |
71bad7f0 | 497 | } |
498 | ||
dc5424ae MZ |
499 | /* |
500 | * Initialize the scatterlist so that the magic cookie | |
501 | * is filled if debugging is enabled | |
502 | */ | |
503 | sg_init_table(scatterlist, num_pages); | |
504 | /* Now set the pages for each scatterlist */ | |
ff92b9e3 PE |
505 | for (i = 0; i < num_pages; i++) { |
506 | unsigned int len = PAGE_SIZE - offset; | |
507 | ||
508 | if (len > count) | |
509 | len = count; | |
510 | sg_set_page(scatterlist + i, pages[i], len, offset); | |
511 | offset = 0; | |
512 | count -= len; | |
513 | } | |
cf9caf19 MZ |
514 | |
515 | dma_buffers = dma_map_sg(g_dev, | |
516 | scatterlist, | |
517 | num_pages, | |
4807f2c0 | 518 | pagelistinfo->dma_dir); |
71bad7f0 | 519 | |
cf9caf19 | 520 | if (dma_buffers == 0) { |
4807f2c0 MZ |
521 | cleaup_pagelistinfo(pagelistinfo); |
522 | return NULL; | |
cf9caf19 MZ |
523 | } |
524 | ||
4807f2c0 MZ |
525 | pagelistinfo->scatterlist_mapped = 1; |
526 | ||
cf9caf19 MZ |
527 | /* Combine adjacent blocks for performance */ |
528 | k = 0; | |
529 | for_each_sg(scatterlist, sg, dma_buffers, i) { | |
530 | u32 len = sg_dma_len(sg); | |
531 | u32 addr = sg_dma_address(sg); | |
532 | ||
533 | /* Note: addrs is the address + page_count - 1 | |
ff92b9e3 | 534 | * The firmware expects blocks after the first to be page- |
cf9caf19 MZ |
535 | * aligned and a multiple of the page size |
536 | */ | |
537 | WARN_ON(len == 0); | |
ff92b9e3 PE |
538 | WARN_ON(i && (i != (dma_buffers - 1)) && (len & ~PAGE_MASK)); |
539 | WARN_ON(i && (addr & ~PAGE_MASK)); | |
cf9caf19 | 540 | if (k > 0 && |
ff92b9e3 PE |
541 | ((addrs[k - 1] & PAGE_MASK) + |
542 | (((addrs[k - 1] & ~PAGE_MASK) + 1) << PAGE_SHIFT)) | |
543 | == (addr & PAGE_MASK)) | |
544 | addrs[k - 1] += ((len + PAGE_SIZE - 1) >> PAGE_SHIFT); | |
545 | else | |
546 | addrs[k++] = (addr & PAGE_MASK) | | |
547 | (((len + PAGE_SIZE - 1) >> PAGE_SHIFT) - 1); | |
71bad7f0 | 548 | } |
549 | ||
71bad7f0 | 550 | /* Partial cache lines (fragments) require special measures */ |
551 | if ((type == PAGELIST_READ) && | |
552 | ((pagelist->offset & (g_cache_line_size - 1)) || | |
553 | ((pagelist->offset + pagelist->length) & | |
554 | (g_cache_line_size - 1)))) { | |
555 | char *fragments; | |
556 | ||
557 | if (down_interruptible(&g_free_fragments_sema) != 0) { | |
4807f2c0 MZ |
558 | cleaup_pagelistinfo(pagelistinfo); |
559 | return NULL; | |
71bad7f0 | 560 | } |
561 | ||
562 | WARN_ON(g_free_fragments == NULL); | |
563 | ||
564 | down(&g_free_fragments_mutex); | |
565 | fragments = g_free_fragments; | |
566 | WARN_ON(fragments == NULL); | |
567 | g_free_fragments = *(char **) g_free_fragments; | |
568 | up(&g_free_fragments_mutex); | |
569 | pagelist->type = PAGELIST_READ_WITH_FRAGMENTS + | |
570 | (fragments - g_fragments_base) / g_fragments_size; | |
571 | } | |
572 | ||
4807f2c0 | 573 | return pagelistinfo; |
71bad7f0 | 574 | } |
575 | ||
576 | static void | |
4807f2c0 MZ |
577 | free_pagelist(struct vchiq_pagelist_info *pagelistinfo, |
578 | int actual) | |
71bad7f0 | 579 | { |
4807f2c0 MZ |
580 | unsigned int i; |
581 | PAGELIST_T *pagelist = pagelistinfo->pagelist; | |
582 | struct page **pages = pagelistinfo->pages; | |
583 | unsigned int num_pages = pagelistinfo->num_pages; | |
71bad7f0 | 584 | |
df044ebf | 585 | vchiq_log_trace(vchiq_arm_log_level, "free_pagelist - %pK, %d", |
4807f2c0 | 586 | pagelistinfo->pagelist, actual); |
71bad7f0 | 587 | |
4807f2c0 MZ |
588 | /* |
589 | * NOTE: dma_unmap_sg must be called before the | |
590 | * cpu can touch any of the data/pages. | |
591 | */ | |
592 | dma_unmap_sg(g_dev, pagelistinfo->scatterlist, | |
593 | pagelistinfo->num_pages, pagelistinfo->dma_dir); | |
594 | pagelistinfo->scatterlist_mapped = 0; | |
71bad7f0 | 595 | |
596 | /* Deal with any partial cache lines (fragments) */ | |
597 | if (pagelist->type >= PAGELIST_READ_WITH_FRAGMENTS) { | |
598 | char *fragments = g_fragments_base + | |
599 | (pagelist->type - PAGELIST_READ_WITH_FRAGMENTS) * | |
600 | g_fragments_size; | |
601 | int head_bytes, tail_bytes; | |
6e475350 | 602 | |
71bad7f0 | 603 | head_bytes = (g_cache_line_size - pagelist->offset) & |
604 | (g_cache_line_size - 1); | |
605 | tail_bytes = (pagelist->offset + actual) & | |
606 | (g_cache_line_size - 1); | |
607 | ||
608 | if ((actual >= 0) && (head_bytes != 0)) { | |
609 | if (head_bytes > actual) | |
610 | head_bytes = actual; | |
611 | ||
612 | memcpy((char *)page_address(pages[0]) + | |
613 | pagelist->offset, | |
614 | fragments, | |
615 | head_bytes); | |
616 | } | |
617 | if ((actual >= 0) && (head_bytes < actual) && | |
618 | (tail_bytes != 0)) { | |
619 | memcpy((char *)page_address(pages[num_pages - 1]) + | |
620 | ((pagelist->offset + actual) & | |
621 | (PAGE_SIZE - 1) & ~(g_cache_line_size - 1)), | |
622 | fragments + g_cache_line_size, | |
623 | tail_bytes); | |
624 | } | |
625 | ||
626 | down(&g_free_fragments_mutex); | |
627 | *(char **)fragments = g_free_fragments; | |
628 | g_free_fragments = fragments; | |
629 | up(&g_free_fragments_mutex); | |
630 | up(&g_free_fragments_sema); | |
631 | } | |
632 | ||
4807f2c0 MZ |
633 | /* Need to mark all the pages dirty. */ |
634 | if (pagelist->type != PAGELIST_WRITE && | |
635 | pagelistinfo->pages_need_release) { | |
636 | for (i = 0; i < num_pages; i++) | |
637 | set_page_dirty(pages[i]); | |
71bad7f0 | 638 | } |
639 | ||
4807f2c0 | 640 | cleaup_pagelistinfo(pagelistinfo); |
71bad7f0 | 641 | } |