[mirror_ubuntu-jammy-kernel.git] / drivers / gpu / drm / i915 / i915_sw_fence.c

/*
 * (C) Copyright 2016 Intel Corporation
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; version 2
 * of the License.
 */

#include <linux/slab.h>
#include <linux/dma-fence.h>
#include <linux/irq_work.h>
#include <linux/reservation.h>

#include "i915_sw_fence.h"
#include "i915_selftest.h"

#define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */

static DEFINE_SPINLOCK(i915_sw_fence_lock);

enum {
	DEBUG_FENCE_IDLE = 0,
	DEBUG_FENCE_NOTIFY,
};

#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS

static void *i915_sw_fence_debug_hint(void *addr)
{
	return (void *)(((struct i915_sw_fence *)addr)->flags & I915_SW_FENCE_MASK);
}

static struct debug_obj_descr i915_sw_fence_debug_descr = {
	.name = "i915_sw_fence",
	.debug_hint = i915_sw_fence_debug_hint,
};

static inline void debug_fence_init(struct i915_sw_fence *fence)
{
	debug_object_init(fence, &i915_sw_fence_debug_descr);
}

static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
{
	debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr);
}

static inline void debug_fence_activate(struct i915_sw_fence *fence)
{
	debug_object_activate(fence, &i915_sw_fence_debug_descr);
}

static inline void debug_fence_set_state(struct i915_sw_fence *fence,
					 int old, int new)
{
	debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);
}

static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
{
	debug_object_deactivate(fence, &i915_sw_fence_debug_descr);
}

static inline void debug_fence_destroy(struct i915_sw_fence *fence)
{
	debug_object_destroy(fence, &i915_sw_fence_debug_descr);
}

static inline void debug_fence_free(struct i915_sw_fence *fence)
{
	debug_object_free(fence, &i915_sw_fence_debug_descr);
	smp_wmb(); /* flush the change in state before reallocation */
}

static inline void debug_fence_assert(struct i915_sw_fence *fence)
{
	debug_object_assert_init(fence, &i915_sw_fence_debug_descr);
}

#else

static inline void debug_fence_init(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_activate(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_set_state(struct i915_sw_fence *fence,
					 int old, int new)
{
}

static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_destroy(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_free(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_assert(struct i915_sw_fence *fence)
{
}

#endif

static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
				  enum i915_sw_fence_notify state)
{
	i915_sw_fence_notify_t fn;

	fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK);
	return fn(fence, state);
}

#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
void i915_sw_fence_fini(struct i915_sw_fence *fence)
{
	debug_fence_free(fence);
}
#endif

static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
					struct list_head *continuation)
{
	wait_queue_head_t *x = &fence->wait;
	wait_queue_entry_t *pos, *next;
	unsigned long flags;

	debug_fence_deactivate(fence);
	atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */

	/*
	 * To prevent unbounded recursion as we traverse the graph of
	 * i915_sw_fences, we move the entry list from this, the next ready
	 * fence, to the tail of the original fence's entry list
	 * (and so added to the list to be woken).
	 */

	spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation);
	if (continuation) {
		list_for_each_entry_safe(pos, next, &x->head, entry) {
			if (pos->func == autoremove_wake_function)
				pos->func(pos, TASK_NORMAL, 0, continuation);
			else
				list_move_tail(&pos->entry, continuation);
		}
	} else {
		LIST_HEAD(extra);

		do {
			list_for_each_entry_safe(pos, next, &x->head, entry)
				pos->func(pos, TASK_NORMAL, 0, &extra);

			if (list_empty(&extra))
				break;

			list_splice_tail_init(&extra, &x->head);
		} while (1);
	}
	spin_unlock_irqrestore(&x->lock, flags);

	debug_fence_assert(fence);
}

static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
				     struct list_head *continuation)
{
	debug_fence_assert(fence);

	if (!atomic_dec_and_test(&fence->pending))
		return;

	debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);

	if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
		return;

	debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);

	__i915_sw_fence_wake_up_all(fence, continuation);

	debug_fence_destroy(fence);
	__i915_sw_fence_notify(fence, FENCE_FREE);
}

static void i915_sw_fence_complete(struct i915_sw_fence *fence)
{
	debug_fence_assert(fence);

	if (WARN_ON(i915_sw_fence_done(fence)))
		return;

	__i915_sw_fence_complete(fence, NULL);
}

static void i915_sw_fence_await(struct i915_sw_fence *fence)
{
	debug_fence_assert(fence);
	WARN_ON(atomic_inc_return(&fence->pending) <= 1);
}

void __i915_sw_fence_init(struct i915_sw_fence *fence,
			  i915_sw_fence_notify_t fn,
			  const char *name,
			  struct lock_class_key *key)
{
	BUG_ON(!fn || (unsigned long)fn & ~I915_SW_FENCE_MASK);

	debug_fence_init(fence);

	__init_waitqueue_head(&fence->wait, name, key);
	atomic_set(&fence->pending, 1);
	fence->flags = (unsigned long)fn;
}

void i915_sw_fence_commit(struct i915_sw_fence *fence)
{
	debug_fence_activate(fence);
	i915_sw_fence_complete(fence);
}

static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key)
{
	list_del(&wq->entry);
	__i915_sw_fence_complete(wq->private, key);

	if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
		kfree(wq);
	return 0;
}

static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
				    const struct i915_sw_fence * const signaler)
{
	wait_queue_entry_t *wq;

	if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
		return false;

	if (fence == signaler)
		return true;

	list_for_each_entry(wq, &fence->wait.head, entry) {
		if (wq->func != i915_sw_fence_wake)
			continue;

		if (__i915_sw_fence_check_if_after(wq->private, signaler))
			return true;
	}

	return false;
}

static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
{
	wait_queue_entry_t *wq;

	if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
		return;

	list_for_each_entry(wq, &fence->wait.head, entry) {
		if (wq->func != i915_sw_fence_wake)
			continue;

		__i915_sw_fence_clear_checked_bit(wq->private);
	}
}

static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
				  const struct i915_sw_fence * const signaler)
{
	unsigned long flags;
	bool err;

	if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG))
		return false;

	spin_lock_irqsave(&i915_sw_fence_lock, flags);
	err = __i915_sw_fence_check_if_after(fence, signaler);
	__i915_sw_fence_clear_checked_bit(fence);
	spin_unlock_irqrestore(&i915_sw_fence_lock, flags);

	return err;
}

static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
					  struct i915_sw_fence *signaler,
					  wait_queue_entry_t *wq, gfp_t gfp)
{
	unsigned long flags;
	int pending;

	debug_fence_assert(fence);

	if (i915_sw_fence_done(signaler))
		return 0;

	debug_fence_assert(signaler);

	/* The dependency graph must be acyclic. */
	if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
		return -EINVAL;

	pending = 0;
	if (!wq) {
		wq = kmalloc(sizeof(*wq), gfp);
		if (!wq) {
			if (!gfpflags_allow_blocking(gfp))
				return -ENOMEM;

			i915_sw_fence_wait(signaler);
			return 0;
		}

		pending |= I915_SW_FENCE_FLAG_ALLOC;
	}

	INIT_LIST_HEAD(&wq->entry);
	wq->flags = pending;
	wq->func = i915_sw_fence_wake;
	wq->private = fence;

	i915_sw_fence_await(fence);

	spin_lock_irqsave(&signaler->wait.lock, flags);
	if (likely(!i915_sw_fence_done(signaler))) {
		__add_wait_queue_entry_tail(&signaler->wait, wq);
		pending = 1;
	} else {
		i915_sw_fence_wake(wq, 0, 0, NULL);
		pending = 0;
	}
	spin_unlock_irqrestore(&signaler->wait.lock, flags);

	return pending;
}

int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
				 struct i915_sw_fence *signaler,
				 wait_queue_entry_t *wq)
{
	return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
}

int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
				     struct i915_sw_fence *signaler,
				     gfp_t gfp)
{
	return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
}

struct i915_sw_dma_fence_cb {
	struct dma_fence_cb base;
	struct i915_sw_fence *fence;
	struct dma_fence *dma;
	struct timer_list timer;
	struct irq_work work;
	struct rcu_head rcu;
};

static void timer_i915_sw_fence_wake(struct timer_list *t)
{
	struct i915_sw_dma_fence_cb *cb = from_timer(cb, t, timer);
	struct i915_sw_fence *fence;

	fence = xchg(&cb->fence, NULL);
	if (!fence)
		return;

	pr_warn("asynchronous wait on fence %s:%s:%x timed out\n",
		cb->dma->ops->get_driver_name(cb->dma),
		cb->dma->ops->get_timeline_name(cb->dma),
		cb->dma->seqno);

	i915_sw_fence_complete(fence);
}

static void dma_i915_sw_fence_wake(struct dma_fence *dma,
				   struct dma_fence_cb *data)
{
	struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);
	struct i915_sw_fence *fence;

	fence = xchg(&cb->fence, NULL);
	if (fence)
		i915_sw_fence_complete(fence);

	irq_work_queue(&cb->work);
}

static void irq_i915_sw_fence_work(struct irq_work *wrk)
{
	struct i915_sw_dma_fence_cb *cb = container_of(wrk, typeof(*cb), work);

	del_timer_sync(&cb->timer);
	dma_fence_put(cb->dma);

	kfree_rcu(cb, rcu);
}

int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
				  struct dma_fence *dma,
				  unsigned long timeout,
				  gfp_t gfp)
{
	struct i915_sw_dma_fence_cb *cb;
	int ret;

	debug_fence_assert(fence);

	if (dma_fence_is_signaled(dma))
		return 0;

	cb = kmalloc(sizeof(*cb), gfp);
	if (!cb) {
		if (!gfpflags_allow_blocking(gfp))
			return -ENOMEM;

		return dma_fence_wait(dma, false);
	}

	cb->fence = fence;
	i915_sw_fence_await(fence);

	cb->dma = NULL;
	timer_setup(&cb->timer, timer_i915_sw_fence_wake, TIMER_IRQSAFE);
	init_irq_work(&cb->work, irq_i915_sw_fence_work);
	if (timeout) {
		cb->dma = dma_fence_get(dma);
		mod_timer(&cb->timer, round_jiffies_up(jiffies + timeout));
	}

	ret = dma_fence_add_callback(dma, &cb->base, dma_i915_sw_fence_wake);
	if (ret == 0) {
		ret = 1;
	} else {
		dma_i915_sw_fence_wake(dma, &cb->base);
		if (ret == -ENOENT) /* fence already signaled */
			ret = 0;
	}

	return ret;
}

int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
				    struct reservation_object *resv,
				    const struct dma_fence_ops *exclude,
				    bool write,
				    unsigned long timeout,
				    gfp_t gfp)
{
	struct dma_fence *excl;
	int ret = 0, pending;

	debug_fence_assert(fence);

	if (write) {
		struct dma_fence **shared;
		unsigned int count, i;

		ret = reservation_object_get_fences_rcu(resv,
							&excl, &count, &shared);
		if (ret)
			return ret;

		for (i = 0; i < count; i++) {
			if (shared[i]->ops == exclude)
				continue;

			pending = i915_sw_fence_await_dma_fence(fence,
								shared[i],
								timeout,
								gfp);
			if (pending < 0) {
				ret = pending;
				break;
			}

			ret |= pending;
		}

		for (i = 0; i < count; i++)
			dma_fence_put(shared[i]);
		kfree(shared);
	} else {
		excl = reservation_object_get_excl_rcu(resv);
	}

	if (ret >= 0 && excl && excl->ops != exclude) {
		pending = i915_sw_fence_await_dma_fence(fence,
							excl,
							timeout,
							gfp);
		if (pending < 0)
			ret = pending;
		else
			ret |= pending;
	}

	dma_fence_put(excl);

	return ret;
}

#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "selftests/lib_sw_fence.c"
#include "selftests/i915_sw_fence.c"
#endif
Commit	Line	Data
e68a139f CW	1	/*
	2	* (C) Copyright 2016 Intel Corporation
	3	*
	4	* This program is free software; you can redistribute it and/or
	5	* modify it under the terms of the GNU General Public License
	6	* as published by the Free Software Foundation; version 2
	7	* of the License.
	8	*/
	9
	10	#include <linux/slab.h>
f54d1867	11	#include <linux/dma-fence.h>
81c0ed21	12	#include <linux/irq_work.h>
e68a139f CW	13	#include <linux/reservation.h>
	14
	15	#include "i915_sw_fence.h"
47624cc3	16	#include "i915_selftest.h"
e68a139f	17
7e941861 CW	18	#define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */
7e941861 CW	19
e68a139f CW	20	static DEFINE_SPINLOCK(i915_sw_fence_lock);
e68a139f CW	21
fc158405 CW	22	enum {
	23	DEBUG_FENCE_IDLE = 0,
	24	DEBUG_FENCE_NOTIFY,
	25	};
	26
	27	#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
	28
	29	static void i915_sw_fence_debug_hint(void addr)
	30	{
	31	return (void )(((struct i915_sw_fence )addr)->flags & I915_SW_FENCE_MASK);
	32	}
	33
	34	static struct debug_obj_descr i915_sw_fence_debug_descr = {
	35	.name = "i915_sw_fence",
	36	.debug_hint = i915_sw_fence_debug_hint,
	37	};
	38
	39	static inline void debug_fence_init(struct i915_sw_fence *fence)
	40	{
	41	debug_object_init(fence, &i915_sw_fence_debug_descr);
	42	}
	43
214707fc CW	44	static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
	45	{
	46	debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr);
	47	}
	48
fc158405 CW	49	static inline void debug_fence_activate(struct i915_sw_fence *fence)
	50	{
	51	debug_object_activate(fence, &i915_sw_fence_debug_descr);
	52	}
	53
	54	static inline void debug_fence_set_state(struct i915_sw_fence *fence,
	55	int old, int new)
	56	{
	57	debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);
	58	}
	59
	60	static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
	61	{
	62	debug_object_deactivate(fence, &i915_sw_fence_debug_descr);
	63	}
	64
	65	static inline void debug_fence_destroy(struct i915_sw_fence *fence)
	66	{
	67	debug_object_destroy(fence, &i915_sw_fence_debug_descr);
	68	}
	69
	70	static inline void debug_fence_free(struct i915_sw_fence *fence)
	71	{
	72	debug_object_free(fence, &i915_sw_fence_debug_descr);
6f13f29f	73	smp_wmb(); /* flush the change in state before reallocation */
fc158405 CW	74	}
	75
	76	static inline void debug_fence_assert(struct i915_sw_fence *fence)
	77	{
	78	debug_object_assert_init(fence, &i915_sw_fence_debug_descr);
	79	}
	80
	81	#else
	82
	83	static inline void debug_fence_init(struct i915_sw_fence *fence)
	84	{
	85	}
	86
214707fc CW	87	static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
	88	{
	89	}
	90
fc158405 CW	91	static inline void debug_fence_activate(struct i915_sw_fence *fence)
	92	{
	93	}
	94
	95	static inline void debug_fence_set_state(struct i915_sw_fence *fence,
	96	int old, int new)
	97	{
	98	}
	99
	100	static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
	101	{
	102	}
	103
	104	static inline void debug_fence_destroy(struct i915_sw_fence *fence)
	105	{
	106	}
	107
	108	static inline void debug_fence_free(struct i915_sw_fence *fence)
	109	{
	110	}
	111
	112	static inline void debug_fence_assert(struct i915_sw_fence *fence)
	113	{
	114	}
	115
	116	#endif
	117
e68a139f CW	118	static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
	119	enum i915_sw_fence_notify state)
	120	{
	121	i915_sw_fence_notify_t fn;
	122
	123	fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK);
	124	return fn(fence, state);
	125	}
	126
fc158405 CW	127	#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
	128	void i915_sw_fence_fini(struct i915_sw_fence *fence)
	129	{
	130	debug_fence_free(fence);
	131	}
	132	#endif
	133
e68a139f CW	134	static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
	135	struct list_head *continuation)
	136	{
	137	wait_queue_head_t *x = &fence->wait;
ac6424b9	138	wait_queue_entry_t pos, next;
e68a139f CW	139	unsigned long flags;
e68a139f CW	140
fc158405	141	debug_fence_deactivate(fence);
e68a139f CW	142	atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */
	143
	144	/*
	145	* To prevent unbounded recursion as we traverse the graph of
2055da97 IM	146	* i915_sw_fences, we move the entry list from this, the next ready
2055da97 IM	147	* fence, to the tail of the original fence's entry list
e68a139f CW	148	* (and so added to the list to be woken).
	149	*/
	150
	151	spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation);
	152	if (continuation) {
2055da97	153	list_for_each_entry_safe(pos, next, &x->head, entry) {
e68a139f CW	154	if (pos->func == autoremove_wake_function)
	155	pos->func(pos, TASK_NORMAL, 0, continuation);
	156	else
2055da97	157	list_move_tail(&pos->entry, continuation);
e68a139f CW	158	}
	159	} else {
	160	LIST_HEAD(extra);
	161
	162	do {
2055da97	163	list_for_each_entry_safe(pos, next, &x->head, entry)
e68a139f CW	164	pos->func(pos, TASK_NORMAL, 0, &extra);
	165
	166	if (list_empty(&extra))
	167	break;
	168
2055da97	169	list_splice_tail_init(&extra, &x->head);
e68a139f CW	170	} while (1);
	171	}
	172	spin_unlock_irqrestore(&x->lock, flags);
fc158405 CW	173
fc158405 CW	174	debug_fence_assert(fence);
e68a139f CW	175	}
	176
	177	static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
	178	struct list_head *continuation)
	179	{
fc158405 CW	180	debug_fence_assert(fence);
fc158405 CW	181
e68a139f CW	182	if (!atomic_dec_and_test(&fence->pending))
	183	return;
	184
fc158405 CW	185	debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);
fc158405 CW	186
9310cb7f	187	if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
e68a139f CW	188	return;
e68a139f CW	189
fc158405 CW	190	debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);
fc158405 CW	191
e68a139f	192	__i915_sw_fence_wake_up_all(fence, continuation);
9310cb7f CW	193
	194	debug_fence_destroy(fence);
	195	__i915_sw_fence_notify(fence, FENCE_FREE);
e68a139f CW	196	}
	197
	198	static void i915_sw_fence_complete(struct i915_sw_fence *fence)
	199	{
fc158405 CW	200	debug_fence_assert(fence);
fc158405 CW	201
e68a139f CW	202	if (WARN_ON(i915_sw_fence_done(fence)))
	203	return;
	204
	205	__i915_sw_fence_complete(fence, NULL);
	206	}
	207
	208	static void i915_sw_fence_await(struct i915_sw_fence *fence)
	209	{
fc158405	210	debug_fence_assert(fence);
e68a139f CW	211	WARN_ON(atomic_inc_return(&fence->pending) <= 1);
	212	}
	213
556b7487 CW	214	void __i915_sw_fence_init(struct i915_sw_fence *fence,
	215	i915_sw_fence_notify_t fn,
	216	const char *name,
	217	struct lock_class_key *key)
e68a139f	218	{
9310cb7f	219	BUG_ON(!fn \|\| (unsigned long)fn & ~I915_SW_FENCE_MASK);
e68a139f	220
fc158405 CW	221	debug_fence_init(fence);
fc158405 CW	222
556b7487	223	__init_waitqueue_head(&fence->wait, name, key);
e68a139f CW	224	atomic_set(&fence->pending, 1);
	225	fence->flags = (unsigned long)fn;
	226	}
	227
fc158405 CW	228	void i915_sw_fence_commit(struct i915_sw_fence *fence)
	229	{
	230	debug_fence_activate(fence);
9310cb7f	231	i915_sw_fence_complete(fence);
fc158405 CW	232	}
fc158405 CW	233
ac6424b9	234	static int i915_sw_fence_wake(wait_queue_entry_t wq, unsigned mode, int flags, void key)
e68a139f	235	{
2055da97	236	list_del(&wq->entry);
e68a139f	237	__i915_sw_fence_complete(wq->private, key);
9310cb7f	238
7e941861 CW	239	if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
7e941861 CW	240	kfree(wq);
e68a139f CW	241	return 0;
	242	}
	243
	244	static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
	245	const struct i915_sw_fence * const signaler)
	246	{
ac6424b9	247	wait_queue_entry_t *wq;
e68a139f CW	248
	249	if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
	250	return false;
	251
	252	if (fence == signaler)
	253	return true;
	254
2055da97	255	list_for_each_entry(wq, &fence->wait.head, entry) {
e68a139f CW	256	if (wq->func != i915_sw_fence_wake)
	257	continue;
	258
	259	if (__i915_sw_fence_check_if_after(wq->private, signaler))
	260	return true;
	261	}
	262
	263	return false;
	264	}
	265
	266	static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
	267	{
ac6424b9	268	wait_queue_entry_t *wq;
e68a139f CW	269
	270	if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
	271	return;
	272
2055da97	273	list_for_each_entry(wq, &fence->wait.head, entry) {
e68a139f CW	274	if (wq->func != i915_sw_fence_wake)
	275	continue;
	276
	277	__i915_sw_fence_clear_checked_bit(wq->private);
	278	}
	279	}
	280
	281	static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
	282	const struct i915_sw_fence * const signaler)
	283	{
	284	unsigned long flags;
	285	bool err;
	286
47624cc3	287	if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG))
e68a139f CW	288	return false;
	289
	290	spin_lock_irqsave(&i915_sw_fence_lock, flags);
	291	err = __i915_sw_fence_check_if_after(fence, signaler);
	292	__i915_sw_fence_clear_checked_bit(fence);
	293	spin_unlock_irqrestore(&i915_sw_fence_lock, flags);
	294
	295	return err;
	296	}
	297
7e941861 CW	298	static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
7e941861 CW	299	struct i915_sw_fence *signaler,
ac6424b9	300	wait_queue_entry_t *wq, gfp_t gfp)
e68a139f CW	301	{
	302	unsigned long flags;
	303	int pending;
	304
fc158405 CW	305	debug_fence_assert(fence);
fc158405 CW	306
e68a139f CW	307	if (i915_sw_fence_done(signaler))
	308	return 0;
	309
fc158405 CW	310	debug_fence_assert(signaler);
fc158405 CW	311
e68a139f CW	312	/* The dependency graph must be acyclic. */
	313	if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
	314	return -EINVAL;
	315
7e941861 CW	316	pending = 0;
	317	if (!wq) {
	318	wq = kmalloc(sizeof(*wq), gfp);
	319	if (!wq) {
	320	if (!gfpflags_allow_blocking(gfp))
	321	return -ENOMEM;
	322
	323	i915_sw_fence_wait(signaler);
	324	return 0;
	325	}
	326
	327	pending \|= I915_SW_FENCE_FLAG_ALLOC;
	328	}
	329
2055da97	330	INIT_LIST_HEAD(&wq->entry);
7e941861	331	wq->flags = pending;
e68a139f	332	wq->func = i915_sw_fence_wake;
9310cb7f	333	wq->private = fence;
e68a139f CW	334
	335	i915_sw_fence_await(fence);
	336
	337	spin_lock_irqsave(&signaler->wait.lock, flags);
	338	if (likely(!i915_sw_fence_done(signaler))) {
ac6424b9	339	__add_wait_queue_entry_tail(&signaler->wait, wq);
e68a139f CW	340	pending = 1;
	341	} else {
	342	i915_sw_fence_wake(wq, 0, 0, NULL);
	343	pending = 0;
	344	}
	345	spin_unlock_irqrestore(&signaler->wait.lock, flags);
	346
	347	return pending;
	348	}
	349
7e941861 CW	350	int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
7e941861 CW	351	struct i915_sw_fence *signaler,
ac6424b9	352	wait_queue_entry_t *wq)
7e941861 CW	353	{
	354	return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
	355	}
	356
	357	int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
	358	struct i915_sw_fence *signaler,
	359	gfp_t gfp)
	360	{
	361	return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
	362	}
	363
f54d1867 CW	364	struct i915_sw_dma_fence_cb {
f54d1867 CW	365	struct dma_fence_cb base;
e68a139f	366	struct i915_sw_fence *fence;
f54d1867	367	struct dma_fence *dma;
e68a139f	368	struct timer_list timer;
81c0ed21	369	struct irq_work work;
2cf654db	370	struct rcu_head rcu;
e68a139f CW	371	};
e68a139f CW	372
39cbf2aa	373	static void timer_i915_sw_fence_wake(struct timer_list *t)
e68a139f	374	{
39cbf2aa	375	struct i915_sw_dma_fence_cb *cb = from_timer(cb, t, timer);
81c0ed21 CW	376	struct i915_sw_fence *fence;
	377
	378	fence = xchg(&cb->fence, NULL);
	379	if (!fence)
	380	return;
e68a139f	381
8dfe162a JP	382	pr_warn("asynchronous wait on fence %s:%s:%x timed out\n",
	383	cb->dma->ops->get_driver_name(cb->dma),
	384	cb->dma->ops->get_timeline_name(cb->dma),
	385	cb->dma->seqno);
e68a139f	386
81c0ed21	387	i915_sw_fence_complete(fence);
e68a139f CW	388	}
e68a139f CW	389
f54d1867 CW	390	static void dma_i915_sw_fence_wake(struct dma_fence *dma,
f54d1867 CW	391	struct dma_fence_cb *data)
e68a139f	392	{
f54d1867	393	struct i915_sw_dma_fence_cb cb = container_of(data, typeof(cb), base);
81c0ed21 CW	394	struct i915_sw_fence *fence;
	395
	396	fence = xchg(&cb->fence, NULL);
	397	if (fence)
	398	i915_sw_fence_complete(fence);
	399
	400	irq_work_queue(&cb->work);
	401	}
	402
	403	static void irq_i915_sw_fence_work(struct irq_work *wrk)
	404	{
	405	struct i915_sw_dma_fence_cb cb = container_of(wrk, typeof(cb), work);
e68a139f CW	406
e68a139f CW	407	del_timer_sync(&cb->timer);
f54d1867	408	dma_fence_put(cb->dma);
e68a139f	409
2cf654db	410	kfree_rcu(cb, rcu);
e68a139f CW	411	}
	412
	413	int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
f54d1867	414	struct dma_fence *dma,
e68a139f CW	415	unsigned long timeout,
	416	gfp_t gfp)
	417	{
f54d1867	418	struct i915_sw_dma_fence_cb *cb;
e68a139f CW	419	int ret;
e68a139f CW	420
fc158405 CW	421	debug_fence_assert(fence);
fc158405 CW	422
f54d1867	423	if (dma_fence_is_signaled(dma))
e68a139f CW	424	return 0;
	425
	426	cb = kmalloc(sizeof(*cb), gfp);
	427	if (!cb) {
	428	if (!gfpflags_allow_blocking(gfp))
	429	return -ENOMEM;
	430
f54d1867	431	return dma_fence_wait(dma, false);
e68a139f CW	432	}
e68a139f CW	433
9310cb7f	434	cb->fence = fence;
e68a139f CW	435	i915_sw_fence_await(fence);
	436
	437	cb->dma = NULL;
39cbf2aa	438	timer_setup(&cb->timer, timer_i915_sw_fence_wake, TIMER_IRQSAFE);
81c0ed21	439	init_irq_work(&cb->work, irq_i915_sw_fence_work);
e68a139f	440	if (timeout) {
f54d1867	441	cb->dma = dma_fence_get(dma);
e68a139f CW	442	mod_timer(&cb->timer, round_jiffies_up(jiffies + timeout));
	443	}
	444
f54d1867	445	ret = dma_fence_add_callback(dma, &cb->base, dma_i915_sw_fence_wake);
e68a139f CW	446	if (ret == 0) {
	447	ret = 1;
	448	} else {
	449	dma_i915_sw_fence_wake(dma, &cb->base);
	450	if (ret == -ENOENT) /* fence already signaled */
	451	ret = 0;
	452	}
	453
	454	return ret;
	455	}
	456
	457	int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
	458	struct reservation_object *resv,
f54d1867	459	const struct dma_fence_ops *exclude,
e68a139f CW	460	bool write,
	461	unsigned long timeout,
	462	gfp_t gfp)
	463	{
f54d1867	464	struct dma_fence *excl;
e68a139f CW	465	int ret = 0, pending;
e68a139f CW	466
fc158405 CW	467	debug_fence_assert(fence);
fc158405 CW	468
e68a139f	469	if (write) {
f54d1867	470	struct dma_fence **shared;
e68a139f CW	471	unsigned int count, i;
	472
	473	ret = reservation_object_get_fences_rcu(resv,
	474	&excl, &count, &shared);
	475	if (ret)
	476	return ret;
	477
	478	for (i = 0; i < count; i++) {
	479	if (shared[i]->ops == exclude)
	480	continue;
	481
	482	pending = i915_sw_fence_await_dma_fence(fence,
	483	shared[i],
	484	timeout,
	485	gfp);
	486	if (pending < 0) {
	487	ret = pending;
	488	break;
	489	}
	490
	491	ret \|= pending;
	492	}
	493
	494	for (i = 0; i < count; i++)
f54d1867	495	dma_fence_put(shared[i]);
e68a139f CW	496	kfree(shared);
	497	} else {
	498	excl = reservation_object_get_excl_rcu(resv);
	499	}
	500
	501	if (ret >= 0 && excl && excl->ops != exclude) {
	502	pending = i915_sw_fence_await_dma_fence(fence,
	503	excl,
	504	timeout,
	505	gfp);
	506	if (pending < 0)
	507	ret = pending;
	508	else
	509	ret \|= pending;
	510	}
	511
f54d1867	512	dma_fence_put(excl);
e68a139f CW	513
	514	return ret;
	515	}
47624cc3 CW	516
47624cc3 CW	517	#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
214707fc	518	#include "selftests/lib_sw_fence.c"
47624cc3 CW	519	#include "selftests/i915_sw_fence.c"
47624cc3 CW	520	#endif