[mirror_ubuntu-hirsute-kernel.git] / drivers / gpu / drm / i915 / i915_sw_fence.c

/*
 * (C) Copyright 2016 Intel Corporation
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; version 2
 * of the License.
 */

#include <linux/slab.h>
#include <linux/dma-fence.h>
#include <linux/reservation.h>

#include "i915_sw_fence.h"

#define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */

static DEFINE_SPINLOCK(i915_sw_fence_lock);

enum {
	DEBUG_FENCE_IDLE = 0,
	DEBUG_FENCE_NOTIFY,
};

#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS

static void *i915_sw_fence_debug_hint(void *addr)
{
	return (void *)(((struct i915_sw_fence *)addr)->flags & I915_SW_FENCE_MASK);
}

static struct debug_obj_descr i915_sw_fence_debug_descr = {
	.name = "i915_sw_fence",
	.debug_hint = i915_sw_fence_debug_hint,
};

static inline void debug_fence_init(struct i915_sw_fence *fence)
{
	debug_object_init(fence, &i915_sw_fence_debug_descr);
}

static inline void debug_fence_activate(struct i915_sw_fence *fence)
{
	debug_object_activate(fence, &i915_sw_fence_debug_descr);
}

static inline void debug_fence_set_state(struct i915_sw_fence *fence,
					 int old, int new)
{
	debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);
}

static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
{
	debug_object_deactivate(fence, &i915_sw_fence_debug_descr);
}

static inline void debug_fence_destroy(struct i915_sw_fence *fence)
{
	debug_object_destroy(fence, &i915_sw_fence_debug_descr);
}

static inline void debug_fence_free(struct i915_sw_fence *fence)
{
	debug_object_free(fence, &i915_sw_fence_debug_descr);
	smp_wmb(); /* flush the change in state before reallocation */
}

static inline void debug_fence_assert(struct i915_sw_fence *fence)
{
	debug_object_assert_init(fence, &i915_sw_fence_debug_descr);
}

#else

static inline void debug_fence_init(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_activate(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_set_state(struct i915_sw_fence *fence,
					 int old, int new)
{
}

static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_destroy(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_free(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_assert(struct i915_sw_fence *fence)
{
}

#endif

static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
				  enum i915_sw_fence_notify state)
{
	i915_sw_fence_notify_t fn;

	fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK);
	return fn(fence, state);
}

#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
void i915_sw_fence_fini(struct i915_sw_fence *fence)
{
	debug_fence_free(fence);
}
#endif

static void i915_sw_fence_release(struct kref *kref)
{
	struct i915_sw_fence *fence = container_of(kref, typeof(*fence), kref);

	WARN_ON(atomic_read(&fence->pending) > 0);
	debug_fence_destroy(fence);

	if (fence->flags & I915_SW_FENCE_MASK) {
		__i915_sw_fence_notify(fence, FENCE_FREE);
	} else {
		i915_sw_fence_fini(fence);
		kfree(fence);
	}
}

static void i915_sw_fence_put(struct i915_sw_fence *fence)
{
	debug_fence_assert(fence);
	kref_put(&fence->kref, i915_sw_fence_release);
}

static struct i915_sw_fence *i915_sw_fence_get(struct i915_sw_fence *fence)
{
	debug_fence_assert(fence);
	kref_get(&fence->kref);
	return fence;
}

static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
					struct list_head *continuation)
{
	wait_queue_head_t *x = &fence->wait;
	wait_queue_entry_t *pos, *next;
	unsigned long flags;

	debug_fence_deactivate(fence);
	atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */

	/*
	 * To prevent unbounded recursion as we traverse the graph of
	 * i915_sw_fences, we move the task_list from this, the next ready
	 * fence, to the tail of the original fence's task_list
	 * (and so added to the list to be woken).
	 */

	spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation);
	if (continuation) {
		list_for_each_entry_safe(pos, next, &x->task_list, task_list) {
			if (pos->func == autoremove_wake_function)
				pos->func(pos, TASK_NORMAL, 0, continuation);
			else
				list_move_tail(&pos->task_list, continuation);
		}
	} else {
		LIST_HEAD(extra);

		do {
			list_for_each_entry_safe(pos, next,
						 &x->task_list, task_list)
				pos->func(pos, TASK_NORMAL, 0, &extra);

			if (list_empty(&extra))
				break;

			list_splice_tail_init(&extra, &x->task_list);
		} while (1);
	}
	spin_unlock_irqrestore(&x->lock, flags);

	debug_fence_assert(fence);
}

static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
				     struct list_head *continuation)
{
	debug_fence_assert(fence);

	if (!atomic_dec_and_test(&fence->pending))
		return;

	debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);

	if (fence->flags & I915_SW_FENCE_MASK &&
	    __i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
		return;

	debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);

	__i915_sw_fence_wake_up_all(fence, continuation);
}

static void i915_sw_fence_complete(struct i915_sw_fence *fence)
{
	debug_fence_assert(fence);

	if (WARN_ON(i915_sw_fence_done(fence)))
		return;

	__i915_sw_fence_complete(fence, NULL);
}

static void i915_sw_fence_await(struct i915_sw_fence *fence)
{
	debug_fence_assert(fence);
	WARN_ON(atomic_inc_return(&fence->pending) <= 1);
}

void __i915_sw_fence_init(struct i915_sw_fence *fence,
			  i915_sw_fence_notify_t fn,
			  const char *name,
			  struct lock_class_key *key)
{
	BUG_ON((unsigned long)fn & ~I915_SW_FENCE_MASK);

	debug_fence_init(fence);

	__init_waitqueue_head(&fence->wait, name, key);
	kref_init(&fence->kref);
	atomic_set(&fence->pending, 1);
	fence->flags = (unsigned long)fn;
}

static void __i915_sw_fence_commit(struct i915_sw_fence *fence)
{
	i915_sw_fence_complete(fence);
	i915_sw_fence_put(fence);
}

void i915_sw_fence_commit(struct i915_sw_fence *fence)
{
	debug_fence_activate(fence);
	__i915_sw_fence_commit(fence);
}

static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key)
{
	list_del(&wq->task_list);
	__i915_sw_fence_complete(wq->private, key);
	i915_sw_fence_put(wq->private);
	if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
		kfree(wq);
	return 0;
}

static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
				    const struct i915_sw_fence * const signaler)
{
	wait_queue_entry_t *wq;

	if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
		return false;

	if (fence == signaler)
		return true;

	list_for_each_entry(wq, &fence->wait.task_list, task_list) {
		if (wq->func != i915_sw_fence_wake)
			continue;

		if (__i915_sw_fence_check_if_after(wq->private, signaler))
			return true;
	}

	return false;
}

static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
{
	wait_queue_entry_t *wq;

	if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
		return;

	list_for_each_entry(wq, &fence->wait.task_list, task_list) {
		if (wq->func != i915_sw_fence_wake)
			continue;

		__i915_sw_fence_clear_checked_bit(wq->private);
	}
}

static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
				  const struct i915_sw_fence * const signaler)
{
	unsigned long flags;
	bool err;

	if (!IS_ENABLED(CONFIG_I915_SW_FENCE_CHECK_DAG))
		return false;

	spin_lock_irqsave(&i915_sw_fence_lock, flags);
	err = __i915_sw_fence_check_if_after(fence, signaler);
	__i915_sw_fence_clear_checked_bit(fence);
	spin_unlock_irqrestore(&i915_sw_fence_lock, flags);

	return err;
}

static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
					  struct i915_sw_fence *signaler,
					  wait_queue_entry_t *wq, gfp_t gfp)
{
	unsigned long flags;
	int pending;

	debug_fence_assert(fence);

	if (i915_sw_fence_done(signaler))
		return 0;

	debug_fence_assert(signaler);

	/* The dependency graph must be acyclic. */
	if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
		return -EINVAL;

	pending = 0;
	if (!wq) {
		wq = kmalloc(sizeof(*wq), gfp);
		if (!wq) {
			if (!gfpflags_allow_blocking(gfp))
				return -ENOMEM;

			i915_sw_fence_wait(signaler);
			return 0;
		}

		pending |= I915_SW_FENCE_FLAG_ALLOC;
	}

	INIT_LIST_HEAD(&wq->task_list);
	wq->flags = pending;
	wq->func = i915_sw_fence_wake;
	wq->private = i915_sw_fence_get(fence);

	i915_sw_fence_await(fence);

	spin_lock_irqsave(&signaler->wait.lock, flags);
	if (likely(!i915_sw_fence_done(signaler))) {
		__add_wait_queue_entry_tail(&signaler->wait, wq);
		pending = 1;
	} else {
		i915_sw_fence_wake(wq, 0, 0, NULL);
		pending = 0;
	}
	spin_unlock_irqrestore(&signaler->wait.lock, flags);

	return pending;
}

int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
				 struct i915_sw_fence *signaler,
				 wait_queue_entry_t *wq)
{
	return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
}

int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
				     struct i915_sw_fence *signaler,
				     gfp_t gfp)
{
	return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
}

struct i915_sw_dma_fence_cb {
	struct dma_fence_cb base;
	struct i915_sw_fence *fence;
	struct dma_fence *dma;
	struct timer_list timer;
};

static void timer_i915_sw_fence_wake(unsigned long data)
{
	struct i915_sw_dma_fence_cb *cb = (struct i915_sw_dma_fence_cb *)data;

	pr_warn("asynchronous wait on fence %s:%s:%x timed out\n",
		cb->dma->ops->get_driver_name(cb->dma),
		cb->dma->ops->get_timeline_name(cb->dma),
		cb->dma->seqno);
	dma_fence_put(cb->dma);
	cb->dma = NULL;

	__i915_sw_fence_commit(cb->fence);
	cb->timer.function = NULL;
}

static void dma_i915_sw_fence_wake(struct dma_fence *dma,
				   struct dma_fence_cb *data)
{
	struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);

	del_timer_sync(&cb->timer);
	if (cb->timer.function)
		__i915_sw_fence_commit(cb->fence);
	dma_fence_put(cb->dma);

	kfree(cb);
}

int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
				  struct dma_fence *dma,
				  unsigned long timeout,
				  gfp_t gfp)
{
	struct i915_sw_dma_fence_cb *cb;
	int ret;

	debug_fence_assert(fence);

	if (dma_fence_is_signaled(dma))
		return 0;

	cb = kmalloc(sizeof(*cb), gfp);
	if (!cb) {
		if (!gfpflags_allow_blocking(gfp))
			return -ENOMEM;

		return dma_fence_wait(dma, false);
	}

	cb->fence = i915_sw_fence_get(fence);
	i915_sw_fence_await(fence);

	cb->dma = NULL;
	__setup_timer(&cb->timer,
		      timer_i915_sw_fence_wake, (unsigned long)cb,
		      TIMER_IRQSAFE);
	if (timeout) {
		cb->dma = dma_fence_get(dma);
		mod_timer(&cb->timer, round_jiffies_up(jiffies + timeout));
	}

	ret = dma_fence_add_callback(dma, &cb->base, dma_i915_sw_fence_wake);
	if (ret == 0) {
		ret = 1;
	} else {
		dma_i915_sw_fence_wake(dma, &cb->base);
		if (ret == -ENOENT) /* fence already signaled */
			ret = 0;
	}

	return ret;
}

int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
				    struct reservation_object *resv,
				    const struct dma_fence_ops *exclude,
				    bool write,
				    unsigned long timeout,
				    gfp_t gfp)
{
	struct dma_fence *excl;
	int ret = 0, pending;

	debug_fence_assert(fence);

	if (write) {
		struct dma_fence **shared;
		unsigned int count, i;

		ret = reservation_object_get_fences_rcu(resv,
							&excl, &count, &shared);
		if (ret)
			return ret;

		for (i = 0; i < count; i++) {
			if (shared[i]->ops == exclude)
				continue;

			pending = i915_sw_fence_await_dma_fence(fence,
								shared[i],
								timeout,
								gfp);
			if (pending < 0) {
				ret = pending;
				break;
			}

			ret |= pending;
		}

		for (i = 0; i < count; i++)
			dma_fence_put(shared[i]);
		kfree(shared);
	} else {
		excl = reservation_object_get_excl_rcu(resv);
	}

	if (ret >= 0 && excl && excl->ops != exclude) {
		pending = i915_sw_fence_await_dma_fence(fence,
							excl,
							timeout,
							gfp);
		if (pending < 0)
			ret = pending;
		else
			ret |= pending;
	}

	dma_fence_put(excl);

	return ret;
}
Commit	Line	Data
e68a139f CW	1	/*
	2	* (C) Copyright 2016 Intel Corporation
	3	*
	4	* This program is free software; you can redistribute it and/or
	5	* modify it under the terms of the GNU General Public License
	6	* as published by the Free Software Foundation; version 2
	7	* of the License.
	8	*/
	9
	10	#include <linux/slab.h>
f54d1867	11	#include <linux/dma-fence.h>
e68a139f CW	12	#include <linux/reservation.h>
	13
	14	#include "i915_sw_fence.h"
	15
7e941861 CW	16	#define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */
7e941861 CW	17
e68a139f CW	18	static DEFINE_SPINLOCK(i915_sw_fence_lock);
e68a139f CW	19
fc158405 CW	20	enum {
	21	DEBUG_FENCE_IDLE = 0,
	22	DEBUG_FENCE_NOTIFY,
	23	};
	24
	25	#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
	26
	27	static void i915_sw_fence_debug_hint(void addr)
	28	{
	29	return (void )(((struct i915_sw_fence )addr)->flags & I915_SW_FENCE_MASK);
	30	}
	31
	32	static struct debug_obj_descr i915_sw_fence_debug_descr = {
	33	.name = "i915_sw_fence",
	34	.debug_hint = i915_sw_fence_debug_hint,
	35	};
	36
	37	static inline void debug_fence_init(struct i915_sw_fence *fence)
	38	{
	39	debug_object_init(fence, &i915_sw_fence_debug_descr);
	40	}
	41
	42	static inline void debug_fence_activate(struct i915_sw_fence *fence)
	43	{
	44	debug_object_activate(fence, &i915_sw_fence_debug_descr);
	45	}
	46
	47	static inline void debug_fence_set_state(struct i915_sw_fence *fence,
	48	int old, int new)
	49	{
	50	debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);
	51	}
	52
	53	static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
	54	{
	55	debug_object_deactivate(fence, &i915_sw_fence_debug_descr);
	56	}
	57
	58	static inline void debug_fence_destroy(struct i915_sw_fence *fence)
	59	{
	60	debug_object_destroy(fence, &i915_sw_fence_debug_descr);
	61	}
	62
	63	static inline void debug_fence_free(struct i915_sw_fence *fence)
	64	{
	65	debug_object_free(fence, &i915_sw_fence_debug_descr);
6f13f29f	66	smp_wmb(); /* flush the change in state before reallocation */
fc158405 CW	67	}
	68
	69	static inline void debug_fence_assert(struct i915_sw_fence *fence)
	70	{
	71	debug_object_assert_init(fence, &i915_sw_fence_debug_descr);
	72	}
	73
	74	#else
	75
	76	static inline void debug_fence_init(struct i915_sw_fence *fence)
	77	{
	78	}
	79
	80	static inline void debug_fence_activate(struct i915_sw_fence *fence)
	81	{
	82	}
	83
	84	static inline void debug_fence_set_state(struct i915_sw_fence *fence,
	85	int old, int new)
	86	{
	87	}
	88
	89	static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
	90	{
	91	}
	92
	93	static inline void debug_fence_destroy(struct i915_sw_fence *fence)
	94	{
	95	}
	96
	97	static inline void debug_fence_free(struct i915_sw_fence *fence)
	98	{
	99	}
	100
	101	static inline void debug_fence_assert(struct i915_sw_fence *fence)
	102	{
	103	}
	104
	105	#endif
	106
e68a139f CW	107	static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
	108	enum i915_sw_fence_notify state)
	109	{
	110	i915_sw_fence_notify_t fn;
	111
	112	fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK);
	113	return fn(fence, state);
	114	}
	115
fc158405 CW	116	#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
	117	void i915_sw_fence_fini(struct i915_sw_fence *fence)
	118	{
	119	debug_fence_free(fence);
	120	}
	121	#endif
	122
	123	static void i915_sw_fence_release(struct kref *kref)
e68a139f CW	124	{
	125	struct i915_sw_fence fence = container_of(kref, typeof(fence), kref);
	126
	127	WARN_ON(atomic_read(&fence->pending) > 0);
fc158405	128	debug_fence_destroy(fence);
e68a139f	129
fc158405	130	if (fence->flags & I915_SW_FENCE_MASK) {
e68a139f	131	__i915_sw_fence_notify(fence, FENCE_FREE);
fc158405 CW	132	} else {
fc158405 CW	133	i915_sw_fence_fini(fence);
e68a139f	134	kfree(fence);
fc158405	135	}
e68a139f CW	136	}
	137
	138	static void i915_sw_fence_put(struct i915_sw_fence *fence)
	139	{
fc158405 CW	140	debug_fence_assert(fence);
fc158405 CW	141	kref_put(&fence->kref, i915_sw_fence_release);
e68a139f CW	142	}
	143
	144	static struct i915_sw_fence i915_sw_fence_get(struct i915_sw_fence fence)
	145	{
fc158405	146	debug_fence_assert(fence);
e68a139f CW	147	kref_get(&fence->kref);
	148	return fence;
	149	}
	150
	151	static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
	152	struct list_head *continuation)
	153	{
	154	wait_queue_head_t *x = &fence->wait;
ac6424b9	155	wait_queue_entry_t pos, next;
e68a139f CW	156	unsigned long flags;
e68a139f CW	157
fc158405	158	debug_fence_deactivate(fence);
e68a139f CW	159	atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */
	160
	161	/*
	162	* To prevent unbounded recursion as we traverse the graph of
	163	* i915_sw_fences, we move the task_list from this, the next ready
	164	* fence, to the tail of the original fence's task_list
	165	* (and so added to the list to be woken).
	166	*/
	167
	168	spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation);
	169	if (continuation) {
	170	list_for_each_entry_safe(pos, next, &x->task_list, task_list) {
	171	if (pos->func == autoremove_wake_function)
	172	pos->func(pos, TASK_NORMAL, 0, continuation);
	173	else
	174	list_move_tail(&pos->task_list, continuation);
	175	}
	176	} else {
	177	LIST_HEAD(extra);
	178
	179	do {
	180	list_for_each_entry_safe(pos, next,
	181	&x->task_list, task_list)
	182	pos->func(pos, TASK_NORMAL, 0, &extra);
	183
	184	if (list_empty(&extra))
	185	break;
	186
	187	list_splice_tail_init(&extra, &x->task_list);
	188	} while (1);
	189	}
	190	spin_unlock_irqrestore(&x->lock, flags);
fc158405 CW	191
fc158405 CW	192	debug_fence_assert(fence);
e68a139f CW	193	}
	194
	195	static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
	196	struct list_head *continuation)
	197	{
fc158405 CW	198	debug_fence_assert(fence);
fc158405 CW	199
e68a139f CW	200	if (!atomic_dec_and_test(&fence->pending))
	201	return;
	202
fc158405 CW	203	debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);
fc158405 CW	204
e68a139f CW	205	if (fence->flags & I915_SW_FENCE_MASK &&
	206	__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
	207	return;
	208
fc158405 CW	209	debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);
fc158405 CW	210
e68a139f CW	211	__i915_sw_fence_wake_up_all(fence, continuation);
	212	}
	213
	214	static void i915_sw_fence_complete(struct i915_sw_fence *fence)
	215	{
fc158405 CW	216	debug_fence_assert(fence);
fc158405 CW	217
e68a139f CW	218	if (WARN_ON(i915_sw_fence_done(fence)))
	219	return;
	220
	221	__i915_sw_fence_complete(fence, NULL);
	222	}
	223
	224	static void i915_sw_fence_await(struct i915_sw_fence *fence)
	225	{
fc158405	226	debug_fence_assert(fence);
e68a139f CW	227	WARN_ON(atomic_inc_return(&fence->pending) <= 1);
	228	}
	229
556b7487 CW	230	void __i915_sw_fence_init(struct i915_sw_fence *fence,
	231	i915_sw_fence_notify_t fn,
	232	const char *name,
	233	struct lock_class_key *key)
e68a139f CW	234	{
	235	BUG_ON((unsigned long)fn & ~I915_SW_FENCE_MASK);
	236
fc158405 CW	237	debug_fence_init(fence);
fc158405 CW	238
556b7487	239	__init_waitqueue_head(&fence->wait, name, key);
e68a139f CW	240	kref_init(&fence->kref);
	241	atomic_set(&fence->pending, 1);
	242	fence->flags = (unsigned long)fn;
	243	}
	244
fc158405	245	static void __i915_sw_fence_commit(struct i915_sw_fence *fence)
e68a139f CW	246	{
	247	i915_sw_fence_complete(fence);
	248	i915_sw_fence_put(fence);
	249	}
	250
fc158405 CW	251	void i915_sw_fence_commit(struct i915_sw_fence *fence)
	252	{
	253	debug_fence_activate(fence);
	254	__i915_sw_fence_commit(fence);
	255	}
	256
ac6424b9	257	static int i915_sw_fence_wake(wait_queue_entry_t wq, unsigned mode, int flags, void key)
e68a139f CW	258	{
	259	list_del(&wq->task_list);
	260	__i915_sw_fence_complete(wq->private, key);
	261	i915_sw_fence_put(wq->private);
7e941861 CW	262	if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
7e941861 CW	263	kfree(wq);
e68a139f CW	264	return 0;
	265	}
	266
	267	static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
	268	const struct i915_sw_fence * const signaler)
	269	{
ac6424b9	270	wait_queue_entry_t *wq;
e68a139f CW	271
	272	if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
	273	return false;
	274
	275	if (fence == signaler)
	276	return true;
	277
	278	list_for_each_entry(wq, &fence->wait.task_list, task_list) {
	279	if (wq->func != i915_sw_fence_wake)
	280	continue;
	281
	282	if (__i915_sw_fence_check_if_after(wq->private, signaler))
	283	return true;
	284	}
	285
	286	return false;
	287	}
	288
	289	static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
	290	{
ac6424b9	291	wait_queue_entry_t *wq;
e68a139f CW	292
	293	if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
	294	return;
	295
	296	list_for_each_entry(wq, &fence->wait.task_list, task_list) {
	297	if (wq->func != i915_sw_fence_wake)
	298	continue;
	299
	300	__i915_sw_fence_clear_checked_bit(wq->private);
	301	}
	302	}
	303
	304	static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
	305	const struct i915_sw_fence * const signaler)
	306	{
	307	unsigned long flags;
	308	bool err;
	309
	310	if (!IS_ENABLED(CONFIG_I915_SW_FENCE_CHECK_DAG))
	311	return false;
	312
	313	spin_lock_irqsave(&i915_sw_fence_lock, flags);
	314	err = __i915_sw_fence_check_if_after(fence, signaler);
	315	__i915_sw_fence_clear_checked_bit(fence);
	316	spin_unlock_irqrestore(&i915_sw_fence_lock, flags);
	317
	318	return err;
	319	}
	320
7e941861 CW	321	static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
7e941861 CW	322	struct i915_sw_fence *signaler,
ac6424b9	323	wait_queue_entry_t *wq, gfp_t gfp)
e68a139f CW	324	{
	325	unsigned long flags;
	326	int pending;
	327
fc158405 CW	328	debug_fence_assert(fence);
fc158405 CW	329
e68a139f CW	330	if (i915_sw_fence_done(signaler))
	331	return 0;
	332
fc158405 CW	333	debug_fence_assert(signaler);
fc158405 CW	334
e68a139f CW	335	/* The dependency graph must be acyclic. */
	336	if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
	337	return -EINVAL;
	338
7e941861 CW	339	pending = 0;
	340	if (!wq) {
	341	wq = kmalloc(sizeof(*wq), gfp);
	342	if (!wq) {
	343	if (!gfpflags_allow_blocking(gfp))
	344	return -ENOMEM;
	345
	346	i915_sw_fence_wait(signaler);
	347	return 0;
	348	}
	349
	350	pending \|= I915_SW_FENCE_FLAG_ALLOC;
	351	}
	352
e68a139f	353	INIT_LIST_HEAD(&wq->task_list);
7e941861	354	wq->flags = pending;
e68a139f CW	355	wq->func = i915_sw_fence_wake;
	356	wq->private = i915_sw_fence_get(fence);
	357
	358	i915_sw_fence_await(fence);
	359
	360	spin_lock_irqsave(&signaler->wait.lock, flags);
	361	if (likely(!i915_sw_fence_done(signaler))) {
ac6424b9	362	__add_wait_queue_entry_tail(&signaler->wait, wq);
e68a139f CW	363	pending = 1;
	364	} else {
	365	i915_sw_fence_wake(wq, 0, 0, NULL);
	366	pending = 0;
	367	}
	368	spin_unlock_irqrestore(&signaler->wait.lock, flags);
	369
	370	return pending;
	371	}
	372
7e941861 CW	373	int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
7e941861 CW	374	struct i915_sw_fence *signaler,
ac6424b9	375	wait_queue_entry_t *wq)
7e941861 CW	376	{
	377	return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
	378	}
	379
	380	int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
	381	struct i915_sw_fence *signaler,
	382	gfp_t gfp)
	383	{
	384	return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
	385	}
	386
f54d1867 CW	387	struct i915_sw_dma_fence_cb {
f54d1867 CW	388	struct dma_fence_cb base;
e68a139f	389	struct i915_sw_fence *fence;
f54d1867	390	struct dma_fence *dma;
e68a139f CW	391	struct timer_list timer;
	392	};
	393
	394	static void timer_i915_sw_fence_wake(unsigned long data)
	395	{
f54d1867	396	struct i915_sw_dma_fence_cb cb = (struct i915_sw_dma_fence_cb )data;
e68a139f	397
8dfe162a JP	398	pr_warn("asynchronous wait on fence %s:%s:%x timed out\n",
	399	cb->dma->ops->get_driver_name(cb->dma),
	400	cb->dma->ops->get_timeline_name(cb->dma),
	401	cb->dma->seqno);
f54d1867	402	dma_fence_put(cb->dma);
e68a139f CW	403	cb->dma = NULL;
e68a139f CW	404
fc158405	405	__i915_sw_fence_commit(cb->fence);
e68a139f CW	406	cb->timer.function = NULL;
	407	}
	408
f54d1867 CW	409	static void dma_i915_sw_fence_wake(struct dma_fence *dma,
f54d1867 CW	410	struct dma_fence_cb *data)
e68a139f	411	{
f54d1867	412	struct i915_sw_dma_fence_cb cb = container_of(data, typeof(cb), base);
e68a139f CW	413
	414	del_timer_sync(&cb->timer);
	415	if (cb->timer.function)
fc158405	416	__i915_sw_fence_commit(cb->fence);
f54d1867	417	dma_fence_put(cb->dma);
e68a139f CW	418
	419	kfree(cb);
	420	}
	421
	422	int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
f54d1867	423	struct dma_fence *dma,
e68a139f CW	424	unsigned long timeout,
	425	gfp_t gfp)
	426	{
f54d1867	427	struct i915_sw_dma_fence_cb *cb;
e68a139f CW	428	int ret;
e68a139f CW	429
fc158405 CW	430	debug_fence_assert(fence);
fc158405 CW	431
f54d1867	432	if (dma_fence_is_signaled(dma))
e68a139f CW	433	return 0;
	434
	435	cb = kmalloc(sizeof(*cb), gfp);
	436	if (!cb) {
	437	if (!gfpflags_allow_blocking(gfp))
	438	return -ENOMEM;
	439
f54d1867	440	return dma_fence_wait(dma, false);
e68a139f CW	441	}
	442
	443	cb->fence = i915_sw_fence_get(fence);
	444	i915_sw_fence_await(fence);
	445
	446	cb->dma = NULL;
	447	__setup_timer(&cb->timer,
	448	timer_i915_sw_fence_wake, (unsigned long)cb,
	449	TIMER_IRQSAFE);
	450	if (timeout) {
f54d1867	451	cb->dma = dma_fence_get(dma);
e68a139f CW	452	mod_timer(&cb->timer, round_jiffies_up(jiffies + timeout));
	453	}
	454
f54d1867	455	ret = dma_fence_add_callback(dma, &cb->base, dma_i915_sw_fence_wake);
e68a139f CW	456	if (ret == 0) {
	457	ret = 1;
	458	} else {
	459	dma_i915_sw_fence_wake(dma, &cb->base);
	460	if (ret == -ENOENT) /* fence already signaled */
	461	ret = 0;
	462	}
	463
	464	return ret;
	465	}
	466
	467	int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
	468	struct reservation_object *resv,
f54d1867	469	const struct dma_fence_ops *exclude,
e68a139f CW	470	bool write,
	471	unsigned long timeout,
	472	gfp_t gfp)
	473	{
f54d1867	474	struct dma_fence *excl;
e68a139f CW	475	int ret = 0, pending;
e68a139f CW	476
fc158405 CW	477	debug_fence_assert(fence);
fc158405 CW	478
e68a139f	479	if (write) {
f54d1867	480	struct dma_fence **shared;
e68a139f CW	481	unsigned int count, i;
	482
	483	ret = reservation_object_get_fences_rcu(resv,
	484	&excl, &count, &shared);
	485	if (ret)
	486	return ret;
	487
	488	for (i = 0; i < count; i++) {
	489	if (shared[i]->ops == exclude)
	490	continue;
	491
	492	pending = i915_sw_fence_await_dma_fence(fence,
	493	shared[i],
	494	timeout,
	495	gfp);
	496	if (pending < 0) {
	497	ret = pending;
	498	break;
	499	}
	500
	501	ret \|= pending;
	502	}
	503
	504	for (i = 0; i < count; i++)
f54d1867	505	dma_fence_put(shared[i]);
e68a139f CW	506	kfree(shared);
	507	} else {
	508	excl = reservation_object_get_excl_rcu(resv);
	509	}
	510
	511	if (ret >= 0 && excl && excl->ops != exclude) {
	512	pending = i915_sw_fence_await_dma_fence(fence,
	513	excl,
	514	timeout,
	515	gfp);
	516	if (pending < 0)
	517	ret = pending;
	518	else
	519	ret \|= pending;
	520	}
	521
f54d1867	522	dma_fence_put(excl);
e68a139f CW	523
	524	return ret;
	525	}