]>
Commit | Line | Data |
---|---|---|
6579324a TB |
1 | /* |
2 | * Tegra host1x Command DMA | |
3 | * | |
4 | * Copyright (c) 2010-2013, NVIDIA Corporation. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify it | |
7 | * under the terms and conditions of the GNU General Public License, | |
8 | * version 2, as published by the Free Software Foundation. | |
9 | * | |
10 | * This program is distributed in the hope it will be useful, but WITHOUT | |
11 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
12 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
13 | * more details. | |
14 | * | |
15 | * You should have received a copy of the GNU General Public License | |
16 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
17 | */ | |
18 | ||
19 | ||
20 | #include <asm/cacheflush.h> | |
21 | #include <linux/device.h> | |
22 | #include <linux/dma-mapping.h> | |
35d747a8 | 23 | #include <linux/host1x.h> |
6579324a TB |
24 | #include <linux/interrupt.h> |
25 | #include <linux/kernel.h> | |
26 | #include <linux/kfifo.h> | |
27 | #include <linux/slab.h> | |
28 | #include <trace/events/host1x.h> | |
29 | ||
30 | #include "cdma.h" | |
31 | #include "channel.h" | |
32 | #include "dev.h" | |
33 | #include "debug.h" | |
6579324a TB |
34 | #include "job.h" |
35 | ||
36 | /* | |
37 | * push_buffer | |
38 | * | |
39 | * The push buffer is a circular array of words to be fetched by command DMA. | |
40 | * Note that it works slightly differently to the sync queue; fence == pos | |
41 | * means that the push buffer is full, not empty. | |
42 | */ | |
43 | ||
44 | #define HOST1X_PUSHBUFFER_SLOTS 512 | |
45 | ||
46 | /* | |
47 | * Clean up push buffer resources | |
48 | */ | |
49 | static void host1x_pushbuffer_destroy(struct push_buffer *pb) | |
50 | { | |
51 | struct host1x_cdma *cdma = pb_to_cdma(pb); | |
52 | struct host1x *host1x = cdma_to_host1x(cdma); | |
53 | ||
54 | if (pb->phys != 0) | |
f6e45661 LR |
55 | dma_free_wc(host1x->dev, pb->size_bytes + 4, pb->mapped, |
56 | pb->phys); | |
6579324a TB |
57 | |
58 | pb->mapped = NULL; | |
59 | pb->phys = 0; | |
60 | } | |
61 | ||
62 | /* | |
63 | * Init push buffer resources | |
64 | */ | |
65 | static int host1x_pushbuffer_init(struct push_buffer *pb) | |
66 | { | |
67 | struct host1x_cdma *cdma = pb_to_cdma(pb); | |
68 | struct host1x *host1x = cdma_to_host1x(cdma); | |
69 | ||
70 | pb->mapped = NULL; | |
71 | pb->phys = 0; | |
72 | pb->size_bytes = HOST1X_PUSHBUFFER_SLOTS * 8; | |
73 | ||
74 | /* initialize buffer pointers */ | |
75 | pb->fence = pb->size_bytes - 8; | |
76 | pb->pos = 0; | |
77 | ||
78 | /* allocate and map pushbuffer memory */ | |
f6e45661 LR |
79 | pb->mapped = dma_alloc_wc(host1x->dev, pb->size_bytes + 4, &pb->phys, |
80 | GFP_KERNEL); | |
6579324a TB |
81 | if (!pb->mapped) |
82 | goto fail; | |
83 | ||
84 | host1x_hw_pushbuffer_init(host1x, pb); | |
85 | ||
86 | return 0; | |
87 | ||
88 | fail: | |
89 | host1x_pushbuffer_destroy(pb); | |
90 | return -ENOMEM; | |
91 | } | |
92 | ||
93 | /* | |
94 | * Push two words to the push buffer | |
95 | * Caller must ensure push buffer is not full | |
96 | */ | |
97 | static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2) | |
98 | { | |
ebb2475c TR |
99 | u32 *p = (u32 *)((void *)pb->mapped + pb->pos); |
100 | ||
101 | WARN_ON(pb->pos == pb->fence); | |
6579324a TB |
102 | *(p++) = op1; |
103 | *(p++) = op2; | |
ebb2475c | 104 | pb->pos = (pb->pos + 8) & (pb->size_bytes - 1); |
6579324a TB |
105 | } |
106 | ||
107 | /* | |
108 | * Pop a number of two word slots from the push buffer | |
109 | * Caller must ensure push buffer is not empty | |
110 | */ | |
111 | static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots) | |
112 | { | |
113 | /* Advance the next write position */ | |
114 | pb->fence = (pb->fence + slots * 8) & (pb->size_bytes - 1); | |
115 | } | |
116 | ||
117 | /* | |
118 | * Return the number of two word slots free in the push buffer | |
119 | */ | |
120 | static u32 host1x_pushbuffer_space(struct push_buffer *pb) | |
121 | { | |
122 | return ((pb->fence - pb->pos) & (pb->size_bytes - 1)) / 8; | |
123 | } | |
124 | ||
125 | /* | |
126 | * Sleep (if necessary) until the requested event happens | |
127 | * - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty. | |
128 | * - Returns 1 | |
129 | * - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer | |
130 | * - Return the amount of space (> 0) | |
131 | * Must be called with the cdma lock held. | |
132 | */ | |
133 | unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma, | |
134 | enum cdma_event event) | |
135 | { | |
136 | for (;;) { | |
0b8070d1 | 137 | struct push_buffer *pb = &cdma->push_buffer; |
6579324a TB |
138 | unsigned int space; |
139 | ||
0b8070d1 TR |
140 | switch (event) { |
141 | case CDMA_EVENT_SYNC_QUEUE_EMPTY: | |
6579324a | 142 | space = list_empty(&cdma->sync_queue) ? 1 : 0; |
0b8070d1 TR |
143 | break; |
144 | ||
145 | case CDMA_EVENT_PUSH_BUFFER_SPACE: | |
6579324a | 146 | space = host1x_pushbuffer_space(pb); |
0b8070d1 TR |
147 | break; |
148 | ||
149 | default: | |
6579324a TB |
150 | WARN_ON(1); |
151 | return -EINVAL; | |
152 | } | |
153 | ||
154 | if (space) | |
155 | return space; | |
156 | ||
157 | trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev), | |
158 | event); | |
159 | ||
160 | /* If somebody has managed to already start waiting, yield */ | |
161 | if (cdma->event != CDMA_EVENT_NONE) { | |
162 | mutex_unlock(&cdma->lock); | |
163 | schedule(); | |
164 | mutex_lock(&cdma->lock); | |
165 | continue; | |
166 | } | |
0b8070d1 | 167 | |
6579324a TB |
168 | cdma->event = event; |
169 | ||
170 | mutex_unlock(&cdma->lock); | |
171 | down(&cdma->sem); | |
172 | mutex_lock(&cdma->lock); | |
173 | } | |
0b8070d1 | 174 | |
6579324a TB |
175 | return 0; |
176 | } | |
177 | ||
178 | /* | |
179 | * Start timer that tracks the time spent by the job. | |
180 | * Must be called with the cdma lock held. | |
181 | */ | |
182 | static void cdma_start_timer_locked(struct host1x_cdma *cdma, | |
183 | struct host1x_job *job) | |
184 | { | |
185 | struct host1x *host = cdma_to_host1x(cdma); | |
186 | ||
187 | if (cdma->timeout.client) { | |
188 | /* timer already started */ | |
189 | return; | |
190 | } | |
191 | ||
192 | cdma->timeout.client = job->client; | |
193 | cdma->timeout.syncpt = host1x_syncpt_get(host, job->syncpt_id); | |
194 | cdma->timeout.syncpt_val = job->syncpt_end; | |
195 | cdma->timeout.start_ktime = ktime_get(); | |
196 | ||
197 | schedule_delayed_work(&cdma->timeout.wq, | |
198 | msecs_to_jiffies(job->timeout)); | |
199 | } | |
200 | ||
201 | /* | |
202 | * Stop timer when a buffer submission completes. | |
203 | * Must be called with the cdma lock held. | |
204 | */ | |
205 | static void stop_cdma_timer_locked(struct host1x_cdma *cdma) | |
206 | { | |
207 | cancel_delayed_work(&cdma->timeout.wq); | |
208 | cdma->timeout.client = 0; | |
209 | } | |
210 | ||
211 | /* | |
212 | * For all sync queue entries that have already finished according to the | |
213 | * current sync point registers: | |
214 | * - unpin & unref their mems | |
215 | * - pop their push buffer slots | |
216 | * - remove them from the sync queue | |
217 | * This is normally called from the host code's worker thread, but can be | |
218 | * called manually if necessary. | |
219 | * Must be called with the cdma lock held. | |
220 | */ | |
221 | static void update_cdma_locked(struct host1x_cdma *cdma) | |
222 | { | |
223 | bool signal = false; | |
224 | struct host1x *host1x = cdma_to_host1x(cdma); | |
225 | struct host1x_job *job, *n; | |
226 | ||
227 | /* If CDMA is stopped, queue is cleared and we can return */ | |
228 | if (!cdma->running) | |
229 | return; | |
230 | ||
231 | /* | |
232 | * Walk the sync queue, reading the sync point registers as necessary, | |
233 | * to consume as many sync queue entries as possible without blocking | |
234 | */ | |
235 | list_for_each_entry_safe(job, n, &cdma->sync_queue, list) { | |
236 | struct host1x_syncpt *sp = | |
237 | host1x_syncpt_get(host1x, job->syncpt_id); | |
238 | ||
239 | /* Check whether this syncpt has completed, and bail if not */ | |
240 | if (!host1x_syncpt_is_expired(sp, job->syncpt_end)) { | |
241 | /* Start timer on next pending syncpt */ | |
242 | if (job->timeout) | |
243 | cdma_start_timer_locked(cdma, job); | |
0b8070d1 | 244 | |
6579324a TB |
245 | break; |
246 | } | |
247 | ||
248 | /* Cancel timeout, when a buffer completes */ | |
249 | if (cdma->timeout.client) | |
250 | stop_cdma_timer_locked(cdma); | |
251 | ||
252 | /* Unpin the memory */ | |
253 | host1x_job_unpin(job); | |
254 | ||
255 | /* Pop push buffer slots */ | |
256 | if (job->num_slots) { | |
257 | struct push_buffer *pb = &cdma->push_buffer; | |
6df633d0 | 258 | |
6579324a | 259 | host1x_pushbuffer_pop(pb, job->num_slots); |
0b8070d1 | 260 | |
6579324a TB |
261 | if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE) |
262 | signal = true; | |
263 | } | |
264 | ||
265 | list_del(&job->list); | |
266 | host1x_job_put(job); | |
267 | } | |
268 | ||
269 | if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY && | |
270 | list_empty(&cdma->sync_queue)) | |
271 | signal = true; | |
272 | ||
273 | if (signal) { | |
274 | cdma->event = CDMA_EVENT_NONE; | |
275 | up(&cdma->sem); | |
276 | } | |
277 | } | |
278 | ||
279 | void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma, | |
280 | struct device *dev) | |
281 | { | |
6579324a | 282 | struct host1x *host1x = cdma_to_host1x(cdma); |
0b8070d1 TR |
283 | u32 restart_addr, syncpt_incrs, syncpt_val; |
284 | struct host1x_job *job = NULL; | |
6579324a TB |
285 | |
286 | syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt); | |
287 | ||
288 | dev_dbg(dev, "%s: starting cleanup (thresh %d)\n", | |
289 | __func__, syncpt_val); | |
290 | ||
291 | /* | |
292 | * Move the sync_queue read pointer to the first entry that hasn't | |
293 | * completed based on the current HW syncpt value. It's likely there | |
294 | * won't be any (i.e. we're still at the head), but covers the case | |
295 | * where a syncpt incr happens just prior/during the teardown. | |
296 | */ | |
297 | ||
298 | dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n", | |
299 | __func__); | |
300 | ||
301 | list_for_each_entry(job, &cdma->sync_queue, list) { | |
302 | if (syncpt_val < job->syncpt_end) | |
303 | break; | |
304 | ||
305 | host1x_job_dump(dev, job); | |
306 | } | |
307 | ||
308 | /* | |
309 | * Walk the sync_queue, first incrementing with the CPU syncpts that | |
310 | * are partially executed (the first buffer) or fully skipped while | |
311 | * still in the current context (slots are also NOP-ed). | |
312 | * | |
313 | * At the point contexts are interleaved, syncpt increments must be | |
314 | * done inline with the pushbuffer from a GATHER buffer to maintain | |
315 | * the order (slots are modified to be a GATHER of syncpt incrs). | |
316 | * | |
317 | * Note: save in restart_addr the location where the timed out buffer | |
318 | * started in the PB, so we can start the refetch from there (with the | |
319 | * modified NOP-ed PB slots). This lets things appear to have completed | |
320 | * properly for this buffer and resources are freed. | |
321 | */ | |
322 | ||
323 | dev_dbg(dev, "%s: perform CPU incr on pending same ctx buffers\n", | |
324 | __func__); | |
325 | ||
326 | if (!list_empty(&cdma->sync_queue)) | |
327 | restart_addr = job->first_get; | |
328 | else | |
329 | restart_addr = cdma->last_pos; | |
330 | ||
331 | /* do CPU increments as long as this context continues */ | |
332 | list_for_each_entry_from(job, &cdma->sync_queue, list) { | |
333 | /* different context, gets us out of this loop */ | |
334 | if (job->client != cdma->timeout.client) | |
335 | break; | |
336 | ||
337 | /* won't need a timeout when replayed */ | |
338 | job->timeout = 0; | |
339 | ||
340 | syncpt_incrs = job->syncpt_end - syncpt_val; | |
341 | dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs); | |
342 | ||
343 | host1x_job_dump(dev, job); | |
344 | ||
345 | /* safe to use CPU to incr syncpts */ | |
346 | host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get, | |
347 | syncpt_incrs, job->syncpt_end, | |
348 | job->num_slots); | |
349 | ||
350 | syncpt_val += syncpt_incrs; | |
351 | } | |
352 | ||
6df633d0 TR |
353 | /* |
354 | * The following sumbits from the same client may be dependent on the | |
6579324a | 355 | * failed submit and therefore they may fail. Force a small timeout |
6df633d0 TR |
356 | * to make the queue cleanup faster. |
357 | */ | |
6579324a TB |
358 | |
359 | list_for_each_entry_from(job, &cdma->sync_queue, list) | |
360 | if (job->client == cdma->timeout.client) | |
361 | job->timeout = min_t(unsigned int, job->timeout, 500); | |
362 | ||
363 | dev_dbg(dev, "%s: finished sync_queue modification\n", __func__); | |
364 | ||
365 | /* roll back DMAGET and start up channel again */ | |
366 | host1x_hw_cdma_resume(host1x, cdma, restart_addr); | |
367 | } | |
368 | ||
369 | /* | |
370 | * Create a cdma | |
371 | */ | |
372 | int host1x_cdma_init(struct host1x_cdma *cdma) | |
373 | { | |
374 | int err; | |
375 | ||
376 | mutex_init(&cdma->lock); | |
377 | sema_init(&cdma->sem, 0); | |
378 | ||
379 | INIT_LIST_HEAD(&cdma->sync_queue); | |
380 | ||
381 | cdma->event = CDMA_EVENT_NONE; | |
382 | cdma->running = false; | |
383 | cdma->torndown = false; | |
384 | ||
385 | err = host1x_pushbuffer_init(&cdma->push_buffer); | |
386 | if (err) | |
387 | return err; | |
0b8070d1 | 388 | |
6579324a TB |
389 | return 0; |
390 | } | |
391 | ||
392 | /* | |
393 | * Destroy a cdma | |
394 | */ | |
395 | int host1x_cdma_deinit(struct host1x_cdma *cdma) | |
396 | { | |
397 | struct push_buffer *pb = &cdma->push_buffer; | |
398 | struct host1x *host1x = cdma_to_host1x(cdma); | |
399 | ||
400 | if (cdma->running) { | |
401 | pr_warn("%s: CDMA still running\n", __func__); | |
402 | return -EBUSY; | |
403 | } | |
404 | ||
405 | host1x_pushbuffer_destroy(pb); | |
406 | host1x_hw_cdma_timeout_destroy(host1x, cdma); | |
407 | ||
408 | return 0; | |
409 | } | |
410 | ||
411 | /* | |
412 | * Begin a cdma submit | |
413 | */ | |
414 | int host1x_cdma_begin(struct host1x_cdma *cdma, struct host1x_job *job) | |
415 | { | |
416 | struct host1x *host1x = cdma_to_host1x(cdma); | |
417 | ||
418 | mutex_lock(&cdma->lock); | |
419 | ||
420 | if (job->timeout) { | |
421 | /* init state on first submit with timeout value */ | |
422 | if (!cdma->timeout.initialized) { | |
423 | int err; | |
6df633d0 | 424 | |
6579324a TB |
425 | err = host1x_hw_cdma_timeout_init(host1x, cdma, |
426 | job->syncpt_id); | |
427 | if (err) { | |
428 | mutex_unlock(&cdma->lock); | |
429 | return err; | |
430 | } | |
431 | } | |
432 | } | |
0b8070d1 | 433 | |
6579324a TB |
434 | if (!cdma->running) |
435 | host1x_hw_cdma_start(host1x, cdma); | |
436 | ||
437 | cdma->slots_free = 0; | |
438 | cdma->slots_used = 0; | |
439 | cdma->first_get = cdma->push_buffer.pos; | |
440 | ||
441 | trace_host1x_cdma_begin(dev_name(job->channel->dev)); | |
442 | return 0; | |
443 | } | |
444 | ||
445 | /* | |
446 | * Push two words into a push buffer slot | |
447 | * Blocks as necessary if the push buffer is full. | |
448 | */ | |
449 | void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2) | |
450 | { | |
451 | struct host1x *host1x = cdma_to_host1x(cdma); | |
452 | struct push_buffer *pb = &cdma->push_buffer; | |
453 | u32 slots_free = cdma->slots_free; | |
454 | ||
6236451d TB |
455 | if (host1x_debug_trace_cmdbuf) |
456 | trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma)->dev), | |
457 | op1, op2); | |
458 | ||
6579324a TB |
459 | if (slots_free == 0) { |
460 | host1x_hw_cdma_flush(host1x, cdma); | |
461 | slots_free = host1x_cdma_wait_locked(cdma, | |
462 | CDMA_EVENT_PUSH_BUFFER_SPACE); | |
463 | } | |
0b8070d1 | 464 | |
6579324a TB |
465 | cdma->slots_free = slots_free - 1; |
466 | cdma->slots_used++; | |
467 | host1x_pushbuffer_push(pb, op1, op2); | |
468 | } | |
469 | ||
470 | /* | |
471 | * End a cdma submit | |
472 | * Kick off DMA, add job to the sync queue, and a number of slots to be freed | |
473 | * from the pushbuffer. The handles for a submit must all be pinned at the same | |
474 | * time, but they can be unpinned in smaller chunks. | |
475 | */ | |
476 | void host1x_cdma_end(struct host1x_cdma *cdma, | |
477 | struct host1x_job *job) | |
478 | { | |
479 | struct host1x *host1x = cdma_to_host1x(cdma); | |
480 | bool idle = list_empty(&cdma->sync_queue); | |
481 | ||
482 | host1x_hw_cdma_flush(host1x, cdma); | |
483 | ||
484 | job->first_get = cdma->first_get; | |
485 | job->num_slots = cdma->slots_used; | |
486 | host1x_job_get(job); | |
487 | list_add_tail(&job->list, &cdma->sync_queue); | |
488 | ||
489 | /* start timer on idle -> active transitions */ | |
490 | if (job->timeout && idle) | |
491 | cdma_start_timer_locked(cdma, job); | |
492 | ||
493 | trace_host1x_cdma_end(dev_name(job->channel->dev)); | |
494 | mutex_unlock(&cdma->lock); | |
495 | } | |
496 | ||
497 | /* | |
498 | * Update cdma state according to current sync point values | |
499 | */ | |
500 | void host1x_cdma_update(struct host1x_cdma *cdma) | |
501 | { | |
502 | mutex_lock(&cdma->lock); | |
503 | update_cdma_locked(cdma); | |
504 | mutex_unlock(&cdma->lock); | |
505 | } |