2 * Copyright (C) 2007 Ben Skeggs.
5 * Permission is hereby granted, free of charge, to any person obtaining
6 * a copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sublicense, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
13 * The above copyright notice and this permission notice (including the
14 * next paragraph) shall be included in all copies or substantial
15 * portions of the Software.
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
29 #include <linux/ktime.h>
30 #include <linux/hrtimer.h>
31 #include <trace/events/fence.h>
33 #include <nvif/notify.h>
34 #include <nvif/event.h>
36 #include "nouveau_drm.h"
37 #include "nouveau_dma.h"
38 #include "nouveau_fence.h"
40 static const struct fence_ops nouveau_fence_ops_uevent
;
41 static const struct fence_ops nouveau_fence_ops_legacy
;
43 static inline struct nouveau_fence
*
44 from_fence(struct fence
*fence
)
46 return container_of(fence
, struct nouveau_fence
, base
);
49 static inline struct nouveau_fence_chan
*
50 nouveau_fctx(struct nouveau_fence
*fence
)
52 return container_of(fence
->base
.lock
, struct nouveau_fence_chan
, lock
);
56 nouveau_fence_signal(struct nouveau_fence
*fence
)
58 fence_signal_locked(&fence
->base
);
59 list_del(&fence
->head
);
61 if (test_bit(FENCE_FLAG_USER_BITS
, &fence
->base
.flags
)) {
62 struct nouveau_fence_chan
*fctx
= nouveau_fctx(fence
);
64 if (!--fctx
->notify_ref
)
65 nvif_notify_put(&fctx
->notify
);
68 fence_put(&fence
->base
);
71 static struct nouveau_fence
*
72 nouveau_local_fence(struct fence
*fence
, struct nouveau_drm
*drm
) {
73 struct nouveau_fence_priv
*priv
= (void*)drm
->fence
;
75 if (fence
->ops
!= &nouveau_fence_ops_legacy
&&
76 fence
->ops
!= &nouveau_fence_ops_uevent
)
79 if (fence
->context
< priv
->context_base
||
80 fence
->context
>= priv
->context_base
+ priv
->contexts
)
83 return from_fence(fence
);
87 nouveau_fence_context_del(struct nouveau_fence_chan
*fctx
)
89 struct nouveau_fence
*fence
;
91 nvif_notify_fini(&fctx
->notify
);
93 spin_lock_irq(&fctx
->lock
);
94 while (!list_empty(&fctx
->pending
)) {
95 fence
= list_entry(fctx
->pending
.next
, typeof(*fence
), head
);
97 nouveau_fence_signal(fence
);
98 fence
->channel
= NULL
;
100 spin_unlock_irq(&fctx
->lock
);
104 nouveau_fence_context_put(struct kref
*fence_ref
)
106 kfree(container_of(fence_ref
, struct nouveau_fence_chan
, fence_ref
));
110 nouveau_fence_context_free(struct nouveau_fence_chan
*fctx
)
112 kref_put(&fctx
->fence_ref
, nouveau_fence_context_put
);
116 nouveau_fence_update(struct nouveau_channel
*chan
, struct nouveau_fence_chan
*fctx
)
118 struct nouveau_fence
*fence
;
120 u32 seq
= fctx
->read(chan
);
122 while (!list_empty(&fctx
->pending
)) {
123 fence
= list_entry(fctx
->pending
.next
, typeof(*fence
), head
);
125 if ((int)(seq
- fence
->base
.seqno
) < 0)
128 nouveau_fence_signal(fence
);
133 nouveau_fence_wait_uevent_handler(struct nvif_notify
*notify
)
135 struct nouveau_fence_chan
*fctx
=
136 container_of(notify
, typeof(*fctx
), notify
);
139 spin_lock_irqsave(&fctx
->lock
, flags
);
140 if (!list_empty(&fctx
->pending
)) {
141 struct nouveau_fence
*fence
;
143 fence
= list_entry(fctx
->pending
.next
, typeof(*fence
), head
);
144 nouveau_fence_update(fence
->channel
, fctx
);
146 spin_unlock_irqrestore(&fctx
->lock
, flags
);
148 /* Always return keep here. NVIF refcount is handled with nouveau_fence_update */
149 return NVIF_NOTIFY_KEEP
;
153 nouveau_fence_context_new(struct nouveau_channel
*chan
, struct nouveau_fence_chan
*fctx
)
155 struct nouveau_fence_priv
*priv
= (void*)chan
->drm
->fence
;
156 struct nouveau_cli
*cli
= (void *)nvif_client(chan
->object
);
159 INIT_LIST_HEAD(&fctx
->flip
);
160 INIT_LIST_HEAD(&fctx
->pending
);
161 spin_lock_init(&fctx
->lock
);
162 fctx
->context
= priv
->context_base
+ chan
->chid
;
164 if (chan
== chan
->drm
->cechan
)
165 strcpy(fctx
->name
, "copy engine channel");
166 else if (chan
== chan
->drm
->channel
)
167 strcpy(fctx
->name
, "generic kernel channel");
169 strcpy(fctx
->name
, nvkm_client(&cli
->base
)->name
);
171 kref_init(&fctx
->fence_ref
);
175 ret
= nvif_notify_init(chan
->object
, NULL
,
176 nouveau_fence_wait_uevent_handler
, false,
177 G82_CHANNEL_DMA_V0_NTFY_UEVENT
,
178 &(struct nvif_notify_uevent_req
) { },
179 sizeof(struct nvif_notify_uevent_req
),
180 sizeof(struct nvif_notify_uevent_rep
),
186 struct nouveau_fence_work
{
187 struct work_struct work
;
189 void (*func
)(void *);
194 nouveau_fence_work_handler(struct work_struct
*kwork
)
196 struct nouveau_fence_work
*work
= container_of(kwork
, typeof(*work
), work
);
197 work
->func(work
->data
);
201 static void nouveau_fence_work_cb(struct fence
*fence
, struct fence_cb
*cb
)
203 struct nouveau_fence_work
*work
= container_of(cb
, typeof(*work
), cb
);
205 schedule_work(&work
->work
);
209 nouveau_fence_work(struct fence
*fence
,
210 void (*func
)(void *), void *data
)
212 struct nouveau_fence_work
*work
;
214 if (fence_is_signaled(fence
))
217 work
= kmalloc(sizeof(*work
), GFP_KERNEL
);
220 * this might not be a nouveau fence any more,
221 * so force a lazy wait here
223 WARN_ON(nouveau_fence_wait((struct nouveau_fence
*)fence
,
228 INIT_WORK(&work
->work
, nouveau_fence_work_handler
);
232 if (fence_add_callback(fence
, &work
->cb
, nouveau_fence_work_cb
) < 0)
243 nouveau_fence_emit(struct nouveau_fence
*fence
, struct nouveau_channel
*chan
)
245 struct nouveau_fence_chan
*fctx
= chan
->fence
;
246 struct nouveau_fence_priv
*priv
= (void*)chan
->drm
->fence
;
249 fence
->channel
= chan
;
250 fence
->timeout
= jiffies
+ (15 * HZ
);
253 fence_init(&fence
->base
, &nouveau_fence_ops_uevent
,
254 &fctx
->lock
, fctx
->context
, ++fctx
->sequence
);
256 fence_init(&fence
->base
, &nouveau_fence_ops_legacy
,
257 &fctx
->lock
, fctx
->context
, ++fctx
->sequence
);
258 kref_get(&fctx
->fence_ref
);
260 trace_fence_emit(&fence
->base
);
261 ret
= fctx
->emit(fence
);
263 fence_get(&fence
->base
);
264 spin_lock_irq(&fctx
->lock
);
265 nouveau_fence_update(chan
, fctx
);
266 list_add_tail(&fence
->head
, &fctx
->pending
);
267 spin_unlock_irq(&fctx
->lock
);
274 nouveau_fence_done(struct nouveau_fence
*fence
)
276 if (fence
->base
.ops
== &nouveau_fence_ops_legacy
||
277 fence
->base
.ops
== &nouveau_fence_ops_uevent
) {
278 struct nouveau_fence_chan
*fctx
= nouveau_fctx(fence
);
281 if (test_bit(FENCE_FLAG_SIGNALED_BIT
, &fence
->base
.flags
))
284 spin_lock_irqsave(&fctx
->lock
, flags
);
285 nouveau_fence_update(fence
->channel
, fctx
);
286 spin_unlock_irqrestore(&fctx
->lock
, flags
);
288 return fence_is_signaled(&fence
->base
);
292 nouveau_fence_wait_legacy(struct fence
*f
, bool intr
, long wait
)
294 struct nouveau_fence
*fence
= from_fence(f
);
295 unsigned long sleep_time
= NSEC_PER_MSEC
/ 1000;
296 unsigned long t
= jiffies
, timeout
= t
+ wait
;
298 while (!nouveau_fence_done(fence
)) {
303 if (wait
!= MAX_SCHEDULE_TIMEOUT
&& time_after_eq(t
, timeout
)) {
304 __set_current_state(TASK_RUNNING
);
308 __set_current_state(intr
? TASK_INTERRUPTIBLE
:
309 TASK_UNINTERRUPTIBLE
);
311 kt
= ktime_set(0, sleep_time
);
312 schedule_hrtimeout(&kt
, HRTIMER_MODE_REL
);
314 if (sleep_time
> NSEC_PER_MSEC
)
315 sleep_time
= NSEC_PER_MSEC
;
317 if (intr
&& signal_pending(current
))
321 __set_current_state(TASK_RUNNING
);
327 nouveau_fence_wait_busy(struct nouveau_fence
*fence
, bool intr
)
331 while (!nouveau_fence_done(fence
)) {
332 if (time_after_eq(jiffies
, fence
->timeout
)) {
337 __set_current_state(intr
?
339 TASK_UNINTERRUPTIBLE
);
341 if (intr
&& signal_pending(current
)) {
347 __set_current_state(TASK_RUNNING
);
352 nouveau_fence_wait(struct nouveau_fence
*fence
, bool lazy
, bool intr
)
357 return nouveau_fence_wait_busy(fence
, intr
);
359 ret
= fence_wait_timeout(&fence
->base
, intr
, 15 * HZ
);
369 nouveau_fence_sync(struct nouveau_bo
*nvbo
, struct nouveau_channel
*chan
, bool exclusive
, bool intr
)
371 struct nouveau_fence_chan
*fctx
= chan
->fence
;
373 struct reservation_object
*resv
= nvbo
->bo
.resv
;
374 struct reservation_object_list
*fobj
;
375 struct nouveau_fence
*f
;
379 ret
= reservation_object_reserve_shared(resv
);
385 fobj
= reservation_object_get_list(resv
);
386 fence
= reservation_object_get_excl(resv
);
388 if (fence
&& (!exclusive
|| !fobj
|| !fobj
->shared_count
)) {
389 struct nouveau_channel
*prev
= NULL
;
391 f
= nouveau_local_fence(fence
, chan
->drm
);
395 if (!prev
|| (prev
!= chan
&& (ret
= fctx
->sync(f
, prev
, chan
))))
396 ret
= fence_wait(fence
, intr
);
401 if (!exclusive
|| !fobj
)
404 for (i
= 0; i
< fobj
->shared_count
&& !ret
; ++i
) {
405 struct nouveau_channel
*prev
= NULL
;
407 fence
= rcu_dereference_protected(fobj
->shared
[i
],
408 reservation_object_held(resv
));
410 f
= nouveau_local_fence(fence
, chan
->drm
);
414 if (!prev
|| (prev
!= chan
&& (ret
= fctx
->sync(f
, prev
, chan
))))
415 ret
= fence_wait(fence
, intr
);
425 nouveau_fence_unref(struct nouveau_fence
**pfence
)
428 fence_put(&(*pfence
)->base
);
433 nouveau_fence_new(struct nouveau_channel
*chan
, bool sysmem
,
434 struct nouveau_fence
**pfence
)
436 struct nouveau_fence
*fence
;
439 if (unlikely(!chan
->fence
))
442 fence
= kzalloc(sizeof(*fence
), GFP_KERNEL
);
446 fence
->sysmem
= sysmem
;
448 ret
= nouveau_fence_emit(fence
, chan
);
450 nouveau_fence_unref(&fence
);
456 static const char *nouveau_fence_get_get_driver_name(struct fence
*fence
)
461 static const char *nouveau_fence_get_timeline_name(struct fence
*f
)
463 struct nouveau_fence
*fence
= from_fence(f
);
464 struct nouveau_fence_chan
*fctx
= nouveau_fctx(fence
);
466 return fence
->channel
? fctx
->name
: "dead channel";
470 * In an ideal world, read would not assume the channel context is still alive.
471 * This function may be called from another device, running into free memory as a
472 * result. The drm node should still be there, so we can derive the index from
475 static bool nouveau_fence_is_signaled(struct fence
*f
)
477 struct nouveau_fence
*fence
= from_fence(f
);
478 struct nouveau_fence_chan
*fctx
= nouveau_fctx(fence
);
479 struct nouveau_channel
*chan
= fence
->channel
;
481 return (int)(fctx
->read(chan
) - fence
->base
.seqno
) >= 0;
484 static bool nouveau_fence_no_signaling(struct fence
*f
)
486 struct nouveau_fence
*fence
= from_fence(f
);
489 * caller should have a reference on the fence,
490 * else fence could get freed here
492 WARN_ON(atomic_read(&fence
->base
.refcount
.refcount
) <= 1);
495 * This needs uevents to work correctly, but fence_add_callback relies on
496 * being able to enable signaling. It will still get signaled eventually,
497 * just not right away.
499 if (nouveau_fence_is_signaled(f
)) {
500 list_del(&fence
->head
);
502 fence_put(&fence
->base
);
509 static void nouveau_fence_release(struct fence
*f
)
511 struct nouveau_fence
*fence
= from_fence(f
);
512 struct nouveau_fence_chan
*fctx
= nouveau_fctx(fence
);
514 kref_put(&fctx
->fence_ref
, nouveau_fence_context_put
);
515 fence_free(&fence
->base
);
518 static const struct fence_ops nouveau_fence_ops_legacy
= {
519 .get_driver_name
= nouveau_fence_get_get_driver_name
,
520 .get_timeline_name
= nouveau_fence_get_timeline_name
,
521 .enable_signaling
= nouveau_fence_no_signaling
,
522 .signaled
= nouveau_fence_is_signaled
,
523 .wait
= nouveau_fence_wait_legacy
,
524 .release
= nouveau_fence_release
527 static bool nouveau_fence_enable_signaling(struct fence
*f
)
529 struct nouveau_fence
*fence
= from_fence(f
);
530 struct nouveau_fence_chan
*fctx
= nouveau_fctx(fence
);
533 if (!fctx
->notify_ref
++)
534 nvif_notify_get(&fctx
->notify
);
536 ret
= nouveau_fence_no_signaling(f
);
538 set_bit(FENCE_FLAG_USER_BITS
, &fence
->base
.flags
);
539 else if (!--fctx
->notify_ref
)
540 nvif_notify_put(&fctx
->notify
);
545 static const struct fence_ops nouveau_fence_ops_uevent
= {
546 .get_driver_name
= nouveau_fence_get_get_driver_name
,
547 .get_timeline_name
= nouveau_fence_get_timeline_name
,
548 .enable_signaling
= nouveau_fence_enable_signaling
,
549 .signaled
= nouveau_fence_is_signaled
,
550 .wait
= fence_default_wait
,