2 * Copyright © 2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 #include <linux/kthread.h>
26 #include <uapi/linux/sched/types.h>
30 static unsigned int __intel_breadcrumbs_wakeup(struct intel_breadcrumbs
*b
)
32 struct intel_wait
*wait
;
33 unsigned int result
= 0;
35 lockdep_assert_held(&b
->irq_lock
);
39 result
= ENGINE_WAKEUP_WAITER
;
40 if (wake_up_process(wait
->tsk
))
41 result
|= ENGINE_WAKEUP_ASLEEP
;
47 unsigned int intel_engine_wakeup(struct intel_engine_cs
*engine
)
49 struct intel_breadcrumbs
*b
= &engine
->breadcrumbs
;
53 spin_lock_irqsave(&b
->irq_lock
, flags
);
54 result
= __intel_breadcrumbs_wakeup(b
);
55 spin_unlock_irqrestore(&b
->irq_lock
, flags
);
60 static unsigned long wait_timeout(void)
62 return round_jiffies_up(jiffies
+ DRM_I915_HANGCHECK_JIFFIES
);
65 static noinline
void missed_breadcrumb(struct intel_engine_cs
*engine
)
67 DRM_DEBUG_DRIVER("%s missed breadcrumb at %pS, irq posted? %s, current seqno=%x, last=%x\n",
68 engine
->name
, __builtin_return_address(0),
69 yesno(test_bit(ENGINE_IRQ_BREADCRUMB
,
70 &engine
->irq_posted
)),
71 intel_engine_get_seqno(engine
),
72 intel_engine_last_submit(engine
));
74 set_bit(engine
->id
, &engine
->i915
->gpu_error
.missed_irq_rings
);
77 static void intel_breadcrumbs_hangcheck(struct timer_list
*t
)
79 struct intel_engine_cs
*engine
= from_timer(engine
, t
,
80 breadcrumbs
.hangcheck
);
81 struct intel_breadcrumbs
*b
= &engine
->breadcrumbs
;
86 if (b
->hangcheck_interrupts
!= atomic_read(&engine
->irq_count
)) {
87 b
->hangcheck_interrupts
= atomic_read(&engine
->irq_count
);
88 mod_timer(&b
->hangcheck
, wait_timeout());
92 /* We keep the hangcheck timer alive until we disarm the irq, even
93 * if there are no waiters at present.
95 * If the waiter was currently running, assume it hasn't had a chance
96 * to process the pending interrupt (e.g, low priority task on a loaded
97 * system) and wait until it sleeps before declaring a missed interrupt.
99 * If the waiter was asleep (and not even pending a wakeup), then we
100 * must have missed an interrupt as the GPU has stopped advancing
101 * but we still have a waiter. Assuming all batches complete within
102 * DRM_I915_HANGCHECK_JIFFIES [1.5s]!
104 if (intel_engine_wakeup(engine
) & ENGINE_WAKEUP_ASLEEP
) {
105 missed_breadcrumb(engine
);
106 mod_timer(&engine
->breadcrumbs
.fake_irq
, jiffies
+ 1);
108 mod_timer(&b
->hangcheck
, wait_timeout());
112 static void intel_breadcrumbs_fake_irq(struct timer_list
*t
)
114 struct intel_engine_cs
*engine
= from_timer(engine
, t
,
115 breadcrumbs
.fake_irq
);
116 struct intel_breadcrumbs
*b
= &engine
->breadcrumbs
;
118 /* The timer persists in case we cannot enable interrupts,
119 * or if we have previously seen seqno/interrupt incoherency
120 * ("missed interrupt" syndrome, better known as a "missed breadcrumb").
121 * Here the worker will wake up every jiffie in order to kick the
122 * oldest waiter to do the coherent seqno check.
125 spin_lock_irq(&b
->irq_lock
);
126 if (!__intel_breadcrumbs_wakeup(b
))
127 __intel_engine_disarm_breadcrumbs(engine
);
128 spin_unlock_irq(&b
->irq_lock
);
132 mod_timer(&b
->fake_irq
, jiffies
+ 1);
134 /* Ensure that even if the GPU hangs, we get woken up.
136 * However, note that if no one is waiting, we never notice
137 * a gpu hang. Eventually, we will have to wait for a resource
138 * held by the GPU and so trigger a hangcheck. In the most
139 * pathological case, this will be upon memory starvation! To
140 * prevent this, we also queue the hangcheck from the retire
143 i915_queue_hangcheck(engine
->i915
);
146 static void irq_enable(struct intel_engine_cs
*engine
)
148 /* Enabling the IRQ may miss the generation of the interrupt, but
149 * we still need to force the barrier before reading the seqno,
152 set_bit(ENGINE_IRQ_BREADCRUMB
, &engine
->irq_posted
);
154 /* Caller disables interrupts */
155 spin_lock(&engine
->i915
->irq_lock
);
156 engine
->irq_enable(engine
);
157 spin_unlock(&engine
->i915
->irq_lock
);
160 static void irq_disable(struct intel_engine_cs
*engine
)
162 /* Caller disables interrupts */
163 spin_lock(&engine
->i915
->irq_lock
);
164 engine
->irq_disable(engine
);
165 spin_unlock(&engine
->i915
->irq_lock
);
168 void __intel_engine_disarm_breadcrumbs(struct intel_engine_cs
*engine
)
170 struct intel_breadcrumbs
*b
= &engine
->breadcrumbs
;
172 lockdep_assert_held(&b
->irq_lock
);
173 GEM_BUG_ON(b
->irq_wait
);
175 if (b
->irq_enabled
) {
177 b
->irq_enabled
= false;
180 b
->irq_armed
= false;
183 void intel_engine_disarm_breadcrumbs(struct intel_engine_cs
*engine
)
185 struct intel_breadcrumbs
*b
= &engine
->breadcrumbs
;
186 struct intel_wait
*wait
, *n
, *first
;
189 goto wakeup_signaler
;
191 /* We only disarm the irq when we are idle (all requests completed),
192 * so if the bottom-half remains asleep, it missed the request
196 spin_lock_irq(&b
->rb_lock
);
198 spin_lock(&b
->irq_lock
);
199 first
= fetch_and_zero(&b
->irq_wait
);
200 __intel_engine_disarm_breadcrumbs(engine
);
201 spin_unlock(&b
->irq_lock
);
203 rbtree_postorder_for_each_entry_safe(wait
, n
, &b
->waiters
, node
) {
204 RB_CLEAR_NODE(&wait
->node
);
205 if (wake_up_process(wait
->tsk
) && wait
== first
)
206 missed_breadcrumb(engine
);
208 b
->waiters
= RB_ROOT
;
210 spin_unlock_irq(&b
->rb_lock
);
213 * The signaling thread may be asleep holding a reference to a request,
214 * that had its signaling cancelled prior to being preempted. We need
215 * to kick the signaler, just in case, to release any such reference.
218 wake_up_process(b
->signaler
);
221 static bool use_fake_irq(const struct intel_breadcrumbs
*b
)
223 const struct intel_engine_cs
*engine
=
224 container_of(b
, struct intel_engine_cs
, breadcrumbs
);
226 if (!test_bit(engine
->id
, &engine
->i915
->gpu_error
.missed_irq_rings
))
229 /* Only start with the heavy weight fake irq timer if we have not
230 * seen any interrupts since enabling it the first time. If the
231 * interrupts are still arriving, it means we made a mistake in our
232 * engine->seqno_barrier(), a timing error that should be transient
233 * and unlikely to reoccur.
235 return atomic_read(&engine
->irq_count
) == b
->hangcheck_interrupts
;
238 static void enable_fake_irq(struct intel_breadcrumbs
*b
)
240 /* Ensure we never sleep indefinitely */
241 if (!b
->irq_enabled
|| use_fake_irq(b
))
242 mod_timer(&b
->fake_irq
, jiffies
+ 1);
244 mod_timer(&b
->hangcheck
, wait_timeout());
247 static bool __intel_breadcrumbs_enable_irq(struct intel_breadcrumbs
*b
)
249 struct intel_engine_cs
*engine
=
250 container_of(b
, struct intel_engine_cs
, breadcrumbs
);
251 struct drm_i915_private
*i915
= engine
->i915
;
253 lockdep_assert_held(&b
->irq_lock
);
257 /* The breadcrumb irq will be disarmed on the interrupt after the
258 * waiters are signaled. This gives us a single interrupt window in
259 * which we can add a new waiter and avoid the cost of re-enabling
263 GEM_BUG_ON(b
->irq_enabled
);
265 if (I915_SELFTEST_ONLY(b
->mock
)) {
266 /* For our mock objects we want to avoid interaction
267 * with the real hardware (which is not set up). So
268 * we simply pretend we have enabled the powerwell
269 * and the irq, and leave it up to the mock
270 * implementation to call intel_engine_wakeup()
271 * itself when it wants to simulate a user interrupt,
276 /* Since we are waiting on a request, the GPU should be busy
277 * and should have its own rpm reference. This is tracked
278 * by i915->gt.awake, we can forgo holding our own wakref
279 * for the interrupt as before i915->gt.awake is released (when
280 * the driver is idle) we disarm the breadcrumbs.
283 /* No interrupts? Kick the waiter every jiffie! */
284 if (intel_irqs_enabled(i915
)) {
285 if (!test_bit(engine
->id
, &i915
->gpu_error
.test_irq_rings
))
287 b
->irq_enabled
= true;
294 static inline struct intel_wait
*to_wait(struct rb_node
*node
)
296 return rb_entry(node
, struct intel_wait
, node
);
299 static inline void __intel_breadcrumbs_finish(struct intel_breadcrumbs
*b
,
300 struct intel_wait
*wait
)
302 lockdep_assert_held(&b
->rb_lock
);
303 GEM_BUG_ON(b
->irq_wait
== wait
);
305 /* This request is completed, so remove it from the tree, mark it as
306 * complete, and *then* wake up the associated task. N.B. when the
307 * task wakes up, it will find the empty rb_node, discern that it
308 * has already been removed from the tree and skip the serialisation
309 * of the b->rb_lock and b->irq_lock. This means that the destruction
310 * of the intel_wait is not serialised with the interrupt handler
311 * by the waiter - it must instead be serialised by the caller.
313 rb_erase(&wait
->node
, &b
->waiters
);
314 RB_CLEAR_NODE(&wait
->node
);
316 wake_up_process(wait
->tsk
); /* implicit smp_wmb() */
319 static inline void __intel_breadcrumbs_next(struct intel_engine_cs
*engine
,
320 struct rb_node
*next
)
322 struct intel_breadcrumbs
*b
= &engine
->breadcrumbs
;
324 spin_lock(&b
->irq_lock
);
325 GEM_BUG_ON(!b
->irq_armed
);
326 GEM_BUG_ON(!b
->irq_wait
);
327 b
->irq_wait
= to_wait(next
);
328 spin_unlock(&b
->irq_lock
);
330 /* We always wake up the next waiter that takes over as the bottom-half
331 * as we may delegate not only the irq-seqno barrier to the next waiter
332 * but also the task of waking up concurrent waiters.
335 wake_up_process(to_wait(next
)->tsk
);
338 static bool __intel_engine_add_wait(struct intel_engine_cs
*engine
,
339 struct intel_wait
*wait
)
341 struct intel_breadcrumbs
*b
= &engine
->breadcrumbs
;
342 struct rb_node
**p
, *parent
, *completed
;
346 /* Insert the request into the retirement ordered list
347 * of waiters by walking the rbtree. If we are the oldest
348 * seqno in the tree (the first to be retired), then
349 * set ourselves as the bottom-half.
351 * As we descend the tree, prune completed branches since we hold the
352 * spinlock we know that the first_waiter must be delayed and can
353 * reduce some of the sequential wake up latency if we take action
354 * ourselves and wake up the completed tasks in parallel. Also, by
355 * removing stale elements in the tree, we may be able to reduce the
356 * ping-pong between the old bottom-half and ourselves as first-waiter.
362 seqno
= intel_engine_get_seqno(engine
);
364 /* If the request completed before we managed to grab the spinlock,
365 * return now before adding ourselves to the rbtree. We let the
366 * current bottom-half handle any pending wakeups and instead
367 * try and get out of the way quickly.
369 if (i915_seqno_passed(seqno
, wait
->seqno
)) {
370 RB_CLEAR_NODE(&wait
->node
);
374 p
= &b
->waiters
.rb_node
;
377 if (wait
->seqno
== to_wait(parent
)->seqno
) {
378 /* We have multiple waiters on the same seqno, select
379 * the highest priority task (that with the smallest
380 * task->prio) to serve as the bottom-half for this
383 if (wait
->tsk
->prio
> to_wait(parent
)->tsk
->prio
) {
384 p
= &parent
->rb_right
;
387 p
= &parent
->rb_left
;
389 } else if (i915_seqno_passed(wait
->seqno
,
390 to_wait(parent
)->seqno
)) {
391 p
= &parent
->rb_right
;
392 if (i915_seqno_passed(seqno
, to_wait(parent
)->seqno
))
397 p
= &parent
->rb_left
;
400 rb_link_node(&wait
->node
, parent
, p
);
401 rb_insert_color(&wait
->node
, &b
->waiters
);
404 spin_lock(&b
->irq_lock
);
406 /* After assigning ourselves as the new bottom-half, we must
407 * perform a cursory check to prevent a missed interrupt.
408 * Either we miss the interrupt whilst programming the hardware,
409 * or if there was a previous waiter (for a later seqno) they
410 * may be woken instead of us (due to the inherent race
411 * in the unlocked read of b->irq_seqno_bh in the irq handler)
412 * and so we miss the wake up.
414 armed
= __intel_breadcrumbs_enable_irq(b
);
415 spin_unlock(&b
->irq_lock
);
419 /* Advance the bottom-half (b->irq_wait) before we wake up
420 * the waiters who may scribble over their intel_wait
421 * just as the interrupt handler is dereferencing it via
425 struct rb_node
*next
= rb_next(completed
);
426 GEM_BUG_ON(next
== &wait
->node
);
427 __intel_breadcrumbs_next(engine
, next
);
431 struct intel_wait
*crumb
= to_wait(completed
);
432 completed
= rb_prev(completed
);
433 __intel_breadcrumbs_finish(b
, crumb
);
437 GEM_BUG_ON(!b
->irq_wait
);
438 GEM_BUG_ON(!b
->irq_armed
);
439 GEM_BUG_ON(rb_first(&b
->waiters
) != &b
->irq_wait
->node
);
444 bool intel_engine_add_wait(struct intel_engine_cs
*engine
,
445 struct intel_wait
*wait
)
447 struct intel_breadcrumbs
*b
= &engine
->breadcrumbs
;
450 spin_lock_irq(&b
->rb_lock
);
451 armed
= __intel_engine_add_wait(engine
, wait
);
452 spin_unlock_irq(&b
->rb_lock
);
456 /* Make the caller recheck if its request has already started. */
457 return i915_seqno_passed(intel_engine_get_seqno(engine
),
461 static inline bool chain_wakeup(struct rb_node
*rb
, int priority
)
463 return rb
&& to_wait(rb
)->tsk
->prio
<= priority
;
466 static inline int wakeup_priority(struct intel_breadcrumbs
*b
,
467 struct task_struct
*tsk
)
469 if (tsk
== b
->signaler
)
475 static void __intel_engine_remove_wait(struct intel_engine_cs
*engine
,
476 struct intel_wait
*wait
)
478 struct intel_breadcrumbs
*b
= &engine
->breadcrumbs
;
480 lockdep_assert_held(&b
->rb_lock
);
482 if (RB_EMPTY_NODE(&wait
->node
))
485 if (b
->irq_wait
== wait
) {
486 const int priority
= wakeup_priority(b
, wait
->tsk
);
487 struct rb_node
*next
;
489 /* We are the current bottom-half. Find the next candidate,
490 * the first waiter in the queue on the remaining oldest
491 * request. As multiple seqnos may complete in the time it
492 * takes us to wake up and find the next waiter, we have to
493 * wake up that waiter for it to perform its own coherent
496 next
= rb_next(&wait
->node
);
497 if (chain_wakeup(next
, priority
)) {
498 /* If the next waiter is already complete,
499 * wake it up and continue onto the next waiter. So
500 * if have a small herd, they will wake up in parallel
501 * rather than sequentially, which should reduce
502 * the overall latency in waking all the completed
505 * However, waking up a chain adds extra latency to
506 * the first_waiter. This is undesirable if that
507 * waiter is a high priority task.
509 u32 seqno
= intel_engine_get_seqno(engine
);
511 while (i915_seqno_passed(seqno
, to_wait(next
)->seqno
)) {
512 struct rb_node
*n
= rb_next(next
);
514 __intel_breadcrumbs_finish(b
, to_wait(next
));
516 if (!chain_wakeup(next
, priority
))
521 __intel_breadcrumbs_next(engine
, next
);
523 GEM_BUG_ON(rb_first(&b
->waiters
) == &wait
->node
);
526 GEM_BUG_ON(RB_EMPTY_NODE(&wait
->node
));
527 rb_erase(&wait
->node
, &b
->waiters
);
528 RB_CLEAR_NODE(&wait
->node
);
531 GEM_BUG_ON(b
->irq_wait
== wait
);
532 GEM_BUG_ON(rb_first(&b
->waiters
) !=
533 (b
->irq_wait
? &b
->irq_wait
->node
: NULL
));
536 void intel_engine_remove_wait(struct intel_engine_cs
*engine
,
537 struct intel_wait
*wait
)
539 struct intel_breadcrumbs
*b
= &engine
->breadcrumbs
;
541 /* Quick check to see if this waiter was already decoupled from
542 * the tree by the bottom-half to avoid contention on the spinlock
545 if (RB_EMPTY_NODE(&wait
->node
)) {
546 GEM_BUG_ON(READ_ONCE(b
->irq_wait
) == wait
);
550 spin_lock_irq(&b
->rb_lock
);
551 __intel_engine_remove_wait(engine
, wait
);
552 spin_unlock_irq(&b
->rb_lock
);
555 static bool signal_valid(const struct drm_i915_gem_request
*request
)
557 return intel_wait_check_request(&request
->signaling
.wait
, request
);
560 static bool signal_complete(const struct drm_i915_gem_request
*request
)
565 /* If another process served as the bottom-half it may have already
566 * signalled that this wait is already completed.
568 if (intel_wait_complete(&request
->signaling
.wait
))
569 return signal_valid(request
);
571 /* Carefully check if the request is complete, giving time for the
572 * seqno to be visible or if the GPU hung.
574 if (__i915_request_irq_complete(request
))
580 static struct drm_i915_gem_request
*to_signaler(struct rb_node
*rb
)
582 return rb_entry(rb
, struct drm_i915_gem_request
, signaling
.node
);
585 static void signaler_set_rtpriority(void)
587 struct sched_param param
= { .sched_priority
= 1 };
589 sched_setscheduler_nocheck(current
, SCHED_FIFO
, ¶m
);
592 static int intel_breadcrumbs_signaler(void *arg
)
594 struct intel_engine_cs
*engine
= arg
;
595 struct intel_breadcrumbs
*b
= &engine
->breadcrumbs
;
596 struct drm_i915_gem_request
*request
;
598 /* Install ourselves with high priority to reduce signalling latency */
599 signaler_set_rtpriority();
602 bool do_schedule
= true;
604 set_current_state(TASK_INTERRUPTIBLE
);
606 /* We are either woken up by the interrupt bottom-half,
607 * or by a client adding a new signaller. In both cases,
608 * the GPU seqno may have advanced beyond our oldest signal.
609 * If it has, propagate the signal, remove the waiter and
610 * check again with the next oldest signal. Otherwise we
611 * need to wait for a new interrupt from the GPU or for
615 request
= rcu_dereference(b
->first_signal
);
617 request
= i915_gem_request_get_rcu(request
);
619 if (signal_complete(request
)) {
621 dma_fence_signal(&request
->fence
);
622 local_bh_enable(); /* kick start the tasklets */
624 spin_lock_irq(&b
->rb_lock
);
626 /* Wake up all other completed waiters and select the
627 * next bottom-half for the next user interrupt.
629 __intel_engine_remove_wait(engine
,
630 &request
->signaling
.wait
);
632 /* Find the next oldest signal. Note that as we have
633 * not been holding the lock, another client may
634 * have installed an even older signal than the one
635 * we just completed - so double check we are still
636 * the oldest before picking the next one.
638 if (request
== rcu_access_pointer(b
->first_signal
)) {
640 rb_next(&request
->signaling
.node
);
641 rcu_assign_pointer(b
->first_signal
,
642 rb
? to_signaler(rb
) : NULL
);
644 rb_erase(&request
->signaling
.node
, &b
->signals
);
645 RB_CLEAR_NODE(&request
->signaling
.node
);
647 spin_unlock_irq(&b
->rb_lock
);
649 i915_gem_request_put(request
);
651 /* If the engine is saturated we may be continually
652 * processing completed requests. This angers the
653 * NMI watchdog if we never let anything else
654 * have access to the CPU. Let's pretend to be nice
655 * and relinquish the CPU if we burn through the
656 * entire RT timeslice!
658 do_schedule
= need_resched();
661 if (unlikely(do_schedule
)) {
662 if (kthread_should_park())
665 if (unlikely(kthread_should_stop())) {
666 i915_gem_request_put(request
);
672 i915_gem_request_put(request
);
674 __set_current_state(TASK_RUNNING
);
679 void intel_engine_enable_signaling(struct drm_i915_gem_request
*request
,
682 struct intel_engine_cs
*engine
= request
->engine
;
683 struct intel_breadcrumbs
*b
= &engine
->breadcrumbs
;
686 /* Note that we may be called from an interrupt handler on another
687 * device (e.g. nouveau signaling a fence completion causing us
688 * to submit a request, and so enable signaling). As such,
689 * we need to make sure that all other users of b->rb_lock protect
690 * against interrupts, i.e. use spin_lock_irqsave.
693 /* locked by dma_fence_enable_sw_signaling() (irqsafe fence->lock) */
694 GEM_BUG_ON(!irqs_disabled());
695 lockdep_assert_held(&request
->lock
);
697 seqno
= i915_gem_request_global_seqno(request
);
701 request
->signaling
.wait
.tsk
= b
->signaler
;
702 request
->signaling
.wait
.request
= request
;
703 request
->signaling
.wait
.seqno
= seqno
;
704 i915_gem_request_get(request
);
706 spin_lock(&b
->rb_lock
);
708 /* First add ourselves into the list of waiters, but register our
709 * bottom-half as the signaller thread. As per usual, only the oldest
710 * waiter (not just signaller) is tasked as the bottom-half waking
711 * up all completed waiters after the user interrupt.
713 * If we are the oldest waiter, enable the irq (after which we
714 * must double check that the seqno did not complete).
716 wakeup
&= __intel_engine_add_wait(engine
, &request
->signaling
.wait
);
718 if (!__i915_gem_request_completed(request
, seqno
)) {
719 struct rb_node
*parent
, **p
;
722 /* Now insert ourselves into the retirement ordered list of
723 * signals on this engine. We track the oldest seqno as that
724 * will be the first signal to complete.
728 p
= &b
->signals
.rb_node
;
731 if (i915_seqno_passed(seqno
,
732 to_signaler(parent
)->signaling
.wait
.seqno
)) {
733 p
= &parent
->rb_right
;
736 p
= &parent
->rb_left
;
739 rb_link_node(&request
->signaling
.node
, parent
, p
);
740 rb_insert_color(&request
->signaling
.node
, &b
->signals
);
742 rcu_assign_pointer(b
->first_signal
, request
);
744 __intel_engine_remove_wait(engine
, &request
->signaling
.wait
);
745 i915_gem_request_put(request
);
749 spin_unlock(&b
->rb_lock
);
752 wake_up_process(b
->signaler
);
755 void intel_engine_cancel_signaling(struct drm_i915_gem_request
*request
)
757 struct intel_engine_cs
*engine
= request
->engine
;
758 struct intel_breadcrumbs
*b
= &engine
->breadcrumbs
;
760 GEM_BUG_ON(!irqs_disabled());
761 lockdep_assert_held(&request
->lock
);
762 GEM_BUG_ON(!request
->signaling
.wait
.seqno
);
764 spin_lock(&b
->rb_lock
);
766 if (!RB_EMPTY_NODE(&request
->signaling
.node
)) {
767 if (request
== rcu_access_pointer(b
->first_signal
)) {
769 rb_next(&request
->signaling
.node
);
770 rcu_assign_pointer(b
->first_signal
,
771 rb
? to_signaler(rb
) : NULL
);
773 rb_erase(&request
->signaling
.node
, &b
->signals
);
774 RB_CLEAR_NODE(&request
->signaling
.node
);
775 i915_gem_request_put(request
);
778 __intel_engine_remove_wait(engine
, &request
->signaling
.wait
);
780 spin_unlock(&b
->rb_lock
);
782 request
->signaling
.wait
.seqno
= 0;
785 int intel_engine_init_breadcrumbs(struct intel_engine_cs
*engine
)
787 struct intel_breadcrumbs
*b
= &engine
->breadcrumbs
;
788 struct task_struct
*tsk
;
790 spin_lock_init(&b
->rb_lock
);
791 spin_lock_init(&b
->irq_lock
);
793 timer_setup(&b
->fake_irq
, intel_breadcrumbs_fake_irq
, 0);
794 timer_setup(&b
->hangcheck
, intel_breadcrumbs_hangcheck
, 0);
796 /* Spawn a thread to provide a common bottom-half for all signals.
797 * As this is an asynchronous interface we cannot steal the current
798 * task for handling the bottom-half to the user interrupt, therefore
799 * we create a thread to do the coherent seqno dance after the
800 * interrupt and then signal the waitqueue (via the dma-buf/fence).
802 tsk
= kthread_run(intel_breadcrumbs_signaler
, engine
,
803 "i915/signal:%d", engine
->id
);
812 static void cancel_fake_irq(struct intel_engine_cs
*engine
)
814 struct intel_breadcrumbs
*b
= &engine
->breadcrumbs
;
816 del_timer_sync(&b
->hangcheck
);
817 del_timer_sync(&b
->fake_irq
);
818 clear_bit(engine
->id
, &engine
->i915
->gpu_error
.missed_irq_rings
);
821 void intel_engine_reset_breadcrumbs(struct intel_engine_cs
*engine
)
823 struct intel_breadcrumbs
*b
= &engine
->breadcrumbs
;
825 cancel_fake_irq(engine
);
826 spin_lock_irq(&b
->irq_lock
);
833 /* We set the IRQ_BREADCRUMB bit when we enable the irq presuming the
834 * GPU is active and may have already executed the MI_USER_INTERRUPT
835 * before the CPU is ready to receive. However, the engine is currently
836 * idle (we haven't started it yet), there is no possibility for a
837 * missed interrupt as we enabled the irq and so we can clear the
838 * immediate wakeup (until a real interrupt arrives for the waiter).
840 clear_bit(ENGINE_IRQ_BREADCRUMB
, &engine
->irq_posted
);
845 spin_unlock_irq(&b
->irq_lock
);
848 void intel_engine_fini_breadcrumbs(struct intel_engine_cs
*engine
)
850 struct intel_breadcrumbs
*b
= &engine
->breadcrumbs
;
852 /* The engines should be idle and all requests accounted for! */
853 WARN_ON(READ_ONCE(b
->irq_wait
));
854 WARN_ON(!RB_EMPTY_ROOT(&b
->waiters
));
855 WARN_ON(rcu_access_pointer(b
->first_signal
));
856 WARN_ON(!RB_EMPTY_ROOT(&b
->signals
));
858 if (!IS_ERR_OR_NULL(b
->signaler
))
859 kthread_stop(b
->signaler
);
861 cancel_fake_irq(engine
);
864 bool intel_breadcrumbs_busy(struct intel_engine_cs
*engine
)
866 struct intel_breadcrumbs
*b
= &engine
->breadcrumbs
;
869 spin_lock_irq(&b
->rb_lock
);
872 wake_up_process(b
->irq_wait
->tsk
);
876 if (rcu_access_pointer(b
->first_signal
)) {
877 wake_up_process(b
->signaler
);
881 spin_unlock_irq(&b
->rb_lock
);
886 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
887 #include "selftests/intel_breadcrumbs.c"