[mirror_ubuntu-focal-kernel.git] / drivers / gpu / drm / i915 / selftests / intel_breadcrumbs.c

/*
 * Copyright © 2016 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 */

#include "../i915_selftest.h"
#include "i915_random.h"

#include "mock_gem_device.h"
#include "mock_engine.h"

static int check_rbtree(struct intel_engine_cs *engine,
			const unsigned long *bitmap,
			const struct intel_wait *waiters,
			const int count)
{
	struct intel_breadcrumbs *b = &engine->breadcrumbs;
	struct rb_node *rb;
	int n;

	if (&b->irq_wait->node != rb_first(&b->waiters)) {
		pr_err("First waiter does not match first element of wait-tree\n");
		return -EINVAL;
	}

	n = find_first_bit(bitmap, count);
	for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) {
		struct intel_wait *w = container_of(rb, typeof(*w), node);
		int idx = w - waiters;

		if (!test_bit(idx, bitmap)) {
			pr_err("waiter[%d, seqno=%d] removed but still in wait-tree\n",
			       idx, w->seqno);
			return -EINVAL;
		}

		if (n != idx) {
			pr_err("waiter[%d, seqno=%d] does not match expected next element in tree [%d]\n",
			       idx, w->seqno, n);
			return -EINVAL;
		}

		n = find_next_bit(bitmap, count, n + 1);
	}

	return 0;
}

static int check_completion(struct intel_engine_cs *engine,
			    const unsigned long *bitmap,
			    const struct intel_wait *waiters,
			    const int count)
{
	int n;

	for (n = 0; n < count; n++) {
		if (intel_wait_complete(&waiters[n]) != !!test_bit(n, bitmap))
			continue;

		pr_err("waiter[%d, seqno=%d] is %s, but expected %s\n",
		       n, waiters[n].seqno,
		       intel_wait_complete(&waiters[n]) ? "complete" : "active",
		       test_bit(n, bitmap) ? "active" : "complete");
		return -EINVAL;
	}

	return 0;
}

static int check_rbtree_empty(struct intel_engine_cs *engine)
{
	struct intel_breadcrumbs *b = &engine->breadcrumbs;

	if (b->irq_wait) {
		pr_err("Empty breadcrumbs still has a waiter\n");
		return -EINVAL;
	}

	if (!RB_EMPTY_ROOT(&b->waiters)) {
		pr_err("Empty breadcrumbs, but wait-tree not empty\n");
		return -EINVAL;
	}

	return 0;
}

static int igt_random_insert_remove(void *arg)
{
	const u32 seqno_bias = 0x1000;
	I915_RND_STATE(prng);
	struct intel_engine_cs *engine = arg;
	struct intel_wait *waiters;
	const int count = 4096;
	unsigned int *order;
	unsigned long *bitmap;
	int err = -ENOMEM;
	int n;

	mock_engine_reset(engine);

	waiters = kvmalloc_array(count, sizeof(*waiters), GFP_TEMPORARY);
	if (!waiters)
		goto out_engines;

	bitmap = kcalloc(DIV_ROUND_UP(count, BITS_PER_LONG), sizeof(*bitmap),
			 GFP_TEMPORARY);
	if (!bitmap)
		goto out_waiters;

	order = i915_random_order(count, &prng);
	if (!order)
		goto out_bitmap;

	for (n = 0; n < count; n++)
		intel_wait_init_for_seqno(&waiters[n], seqno_bias + n);

	err = check_rbtree(engine, bitmap, waiters, count);
	if (err)
		goto out_order;

	/* Add and remove waiters into the rbtree in random order. At each
	 * step, we verify that the rbtree is correctly ordered.
	 */
	for (n = 0; n < count; n++) {
		int i = order[n];

		intel_engine_add_wait(engine, &waiters[i]);
		__set_bit(i, bitmap);

		err = check_rbtree(engine, bitmap, waiters, count);
		if (err)
			goto out_order;
	}

	i915_random_reorder(order, count, &prng);
	for (n = 0; n < count; n++) {
		int i = order[n];

		intel_engine_remove_wait(engine, &waiters[i]);
		__clear_bit(i, bitmap);

		err = check_rbtree(engine, bitmap, waiters, count);
		if (err)
			goto out_order;
	}

	err = check_rbtree_empty(engine);
out_order:
	kfree(order);
out_bitmap:
	kfree(bitmap);
out_waiters:
	kvfree(waiters);
out_engines:
	mock_engine_flush(engine);
	return err;
}

static int igt_insert_complete(void *arg)
{
	const u32 seqno_bias = 0x1000;
	struct intel_engine_cs *engine = arg;
	struct intel_wait *waiters;
	const int count = 4096;
	unsigned long *bitmap;
	int err = -ENOMEM;
	int n, m;

	mock_engine_reset(engine);

	waiters = kvmalloc_array(count, sizeof(*waiters), GFP_TEMPORARY);
	if (!waiters)
		goto out_engines;

	bitmap = kcalloc(DIV_ROUND_UP(count, BITS_PER_LONG), sizeof(*bitmap),
			 GFP_TEMPORARY);
	if (!bitmap)
		goto out_waiters;

	for (n = 0; n < count; n++) {
		intel_wait_init_for_seqno(&waiters[n], n + seqno_bias);
		intel_engine_add_wait(engine, &waiters[n]);
		__set_bit(n, bitmap);
	}
	err = check_rbtree(engine, bitmap, waiters, count);
	if (err)
		goto out_bitmap;

	/* On each step, we advance the seqno so that several waiters are then
	 * complete (we increase the seqno by increasingly larger values to
	 * retire more and more waiters at once). All retired waiters should
	 * be woken and removed from the rbtree, and so that we check.
	 */
	for (n = 0; n < count; n = m) {
		int seqno = 2 * n;

		GEM_BUG_ON(find_first_bit(bitmap, count) != n);

		if (intel_wait_complete(&waiters[n])) {
			pr_err("waiter[%d, seqno=%d] completed too early\n",
			       n, waiters[n].seqno);
			err = -EINVAL;
			goto out_bitmap;
		}

		/* complete the following waiters */
		mock_seqno_advance(engine, seqno + seqno_bias);
		for (m = n; m <= seqno; m++) {
			if (m == count)
				break;

			GEM_BUG_ON(!test_bit(m, bitmap));
			__clear_bit(m, bitmap);
		}

		intel_engine_remove_wait(engine, &waiters[n]);
		RB_CLEAR_NODE(&waiters[n].node);

		err = check_rbtree(engine, bitmap, waiters, count);
		if (err) {
			pr_err("rbtree corrupt after seqno advance to %d\n",
			       seqno + seqno_bias);
			goto out_bitmap;
		}

		err = check_completion(engine, bitmap, waiters, count);
		if (err) {
			pr_err("completions after seqno advance to %d failed\n",
			       seqno + seqno_bias);
			goto out_bitmap;
		}
	}

	err = check_rbtree_empty(engine);
out_bitmap:
	kfree(bitmap);
out_waiters:
	kvfree(waiters);
out_engines:
	mock_engine_flush(engine);
	return err;
}

struct igt_wakeup {
	struct task_struct *tsk;
	atomic_t *ready, *set, *done;
	struct intel_engine_cs *engine;
	unsigned long flags;
#define STOP 0
#define IDLE 1
	wait_queue_head_t *wq;
	u32 seqno;
};

static int wait_atomic(atomic_t *p)
{
	schedule();
	return 0;
}

static int wait_atomic_timeout(atomic_t *p)
{
	return schedule_timeout(10 * HZ) ? 0 : -ETIMEDOUT;
}

static bool wait_for_ready(struct igt_wakeup *w)
{
	DEFINE_WAIT(ready);

	set_bit(IDLE, &w->flags);
	if (atomic_dec_and_test(w->done))
		wake_up_atomic_t(w->done);

	if (test_bit(STOP, &w->flags))
		goto out;

	for (;;) {
		prepare_to_wait(w->wq, &ready, TASK_INTERRUPTIBLE);
		if (atomic_read(w->ready) == 0)
			break;

		schedule();
	}
	finish_wait(w->wq, &ready);

out:
	clear_bit(IDLE, &w->flags);
	if (atomic_dec_and_test(w->set))
		wake_up_atomic_t(w->set);

	return !test_bit(STOP, &w->flags);
}

static int igt_wakeup_thread(void *arg)
{
	struct igt_wakeup *w = arg;
	struct intel_wait wait;

	while (wait_for_ready(w)) {
		GEM_BUG_ON(kthread_should_stop());

		intel_wait_init_for_seqno(&wait, w->seqno);
		intel_engine_add_wait(w->engine, &wait);
		for (;;) {
			set_current_state(TASK_UNINTERRUPTIBLE);
			if (i915_seqno_passed(intel_engine_get_seqno(w->engine),
					      w->seqno))
				break;

			if (test_bit(STOP, &w->flags)) /* emergency escape */
				break;

			schedule();
		}
		intel_engine_remove_wait(w->engine, &wait);
		__set_current_state(TASK_RUNNING);
	}

	return 0;
}

static void igt_wake_all_sync(atomic_t *ready,
			      atomic_t *set,
			      atomic_t *done,
			      wait_queue_head_t *wq,
			      int count)
{
	atomic_set(set, count);
	atomic_set(ready, 0);
	wake_up_all(wq);

	wait_on_atomic_t(set, wait_atomic, TASK_UNINTERRUPTIBLE);
	atomic_set(ready, count);
	atomic_set(done, count);
}

static int igt_wakeup(void *arg)
{
	I915_RND_STATE(prng);
	const int state = TASK_UNINTERRUPTIBLE;
	struct intel_engine_cs *engine = arg;
	struct igt_wakeup *waiters;
	DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
	const int count = 4096;
	const u32 max_seqno = count / 4;
	atomic_t ready, set, done;
	int err = -ENOMEM;
	int n, step;

	mock_engine_reset(engine);

	waiters = kvmalloc_array(count, sizeof(*waiters), GFP_TEMPORARY);
	if (!waiters)
		goto out_engines;

	/* Create a large number of threads, each waiting on a random seqno.
	 * Multiple waiters will be waiting for the same seqno.
	 */
	atomic_set(&ready, count);
	for (n = 0; n < count; n++) {
		waiters[n].wq = &wq;
		waiters[n].ready = &ready;
		waiters[n].set = &set;
		waiters[n].done = &done;
		waiters[n].engine = engine;
		waiters[n].flags = BIT(IDLE);

		waiters[n].tsk = kthread_run(igt_wakeup_thread, &waiters[n],
					     "i915/igt:%d", n);
		if (IS_ERR(waiters[n].tsk))
			goto out_waiters;

		get_task_struct(waiters[n].tsk);
	}

	for (step = 1; step <= max_seqno; step <<= 1) {
		u32 seqno;

		/* The waiter threads start paused as we assign them a random
		 * seqno and reset the engine. Once the engine is reset,
		 * we signal that the threads may begin their wait upon their
		 * seqno.
		 */
		for (n = 0; n < count; n++) {
			GEM_BUG_ON(!test_bit(IDLE, &waiters[n].flags));
			waiters[n].seqno =
				1 + prandom_u32_state(&prng) % max_seqno;
		}
		mock_seqno_advance(engine, 0);
		igt_wake_all_sync(&ready, &set, &done, &wq, count);

		/* Simulate the GPU doing chunks of work, with one or more
		 * seqno appearing to finish at the same time. A random number
		 * of threads will be waiting upon the update and hopefully be
		 * woken.
		 */
		for (seqno = 1; seqno <= max_seqno + step; seqno += step) {
			usleep_range(50, 500);
			mock_seqno_advance(engine, seqno);
		}
		GEM_BUG_ON(intel_engine_get_seqno(engine) < 1 + max_seqno);

		/* With the seqno now beyond any of the waiting threads, they
		 * should all be woken, see that they are complete and signal
		 * that they are ready for the next test. We wait until all
		 * threads are complete and waiting for us (i.e. not a seqno).
		 */
		err = wait_on_atomic_t(&done, wait_atomic_timeout, state);
		if (err) {
			pr_err("Timed out waiting for %d remaining waiters\n",
			       atomic_read(&done));
			break;
		}

		err = check_rbtree_empty(engine);
		if (err)
			break;
	}

out_waiters:
	for (n = 0; n < count; n++) {
		if (IS_ERR(waiters[n].tsk))
			break;

		set_bit(STOP, &waiters[n].flags);
	}
	mock_seqno_advance(engine, INT_MAX); /* wakeup any broken waiters */
	igt_wake_all_sync(&ready, &set, &done, &wq, n);

	for (n = 0; n < count; n++) {
		if (IS_ERR(waiters[n].tsk))
			break;

		kthread_stop(waiters[n].tsk);
		put_task_struct(waiters[n].tsk);
	}

	kvfree(waiters);
out_engines:
	mock_engine_flush(engine);
	return err;
}

int intel_breadcrumbs_mock_selftests(void)
{
	static const struct i915_subtest tests[] = {
		SUBTEST(igt_random_insert_remove),
		SUBTEST(igt_insert_complete),
		SUBTEST(igt_wakeup),
	};
	struct drm_i915_private *i915;
	int err;

	i915 = mock_gem_device();
	if (!i915)
		return -ENOMEM;

	err = i915_subtests(tests, i915->engine[RCS]);
	drm_dev_unref(&i915->drm);

	return err;
}
Commit	Line	Data
f97fbf96 CW	1	/*
	2	* Copyright © 2016 Intel Corporation
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice (including the next
	12	* paragraph) shall be included in all copies or substantial portions of the
	13	* Software.
	14	*
	15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
	20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
	21	* IN THE SOFTWARE.
	22	*
	23	*/
	24
	25	#include "../i915_selftest.h"
	26	#include "i915_random.h"
	27
0daf0113	28	#include "mock_gem_device.h"
f97fbf96 CW	29	#include "mock_engine.h"
	30
	31	static int check_rbtree(struct intel_engine_cs *engine,
	32	const unsigned long *bitmap,
	33	const struct intel_wait *waiters,
	34	const int count)
	35	{
	36	struct intel_breadcrumbs *b = &engine->breadcrumbs;
	37	struct rb_node *rb;
	38	int n;
	39
61d3dc70	40	if (&b->irq_wait->node != rb_first(&b->waiters)) {
f97fbf96 CW	41	pr_err("First waiter does not match first element of wait-tree\n");
	42	return -EINVAL;
	43	}
	44
	45	n = find_first_bit(bitmap, count);
	46	for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) {
	47	struct intel_wait w = container_of(rb, typeof(w), node);
	48	int idx = w - waiters;
	49
	50	if (!test_bit(idx, bitmap)) {
	51	pr_err("waiter[%d, seqno=%d] removed but still in wait-tree\n",
	52	idx, w->seqno);
	53	return -EINVAL;
	54	}
	55
	56	if (n != idx) {
	57	pr_err("waiter[%d, seqno=%d] does not match expected next element in tree [%d]\n",
	58	idx, w->seqno, n);
	59	return -EINVAL;
	60	}
	61
	62	n = find_next_bit(bitmap, count, n + 1);
	63	}
	64
	65	return 0;
	66	}
	67
ae1f8090 CW	68	static int check_completion(struct intel_engine_cs *engine,
	69	const unsigned long *bitmap,
	70	const struct intel_wait *waiters,
	71	const int count)
	72	{
	73	int n;
	74
	75	for (n = 0; n < count; n++) {
	76	if (intel_wait_complete(&waiters[n]) != !!test_bit(n, bitmap))
	77	continue;
	78
	79	pr_err("waiter[%d, seqno=%d] is %s, but expected %s\n",
	80	n, waiters[n].seqno,
	81	intel_wait_complete(&waiters[n]) ? "complete" : "active",
	82	test_bit(n, bitmap) ? "active" : "complete");
	83	return -EINVAL;
	84	}
	85
	86	return 0;
	87	}
	88
f97fbf96 CW	89	static int check_rbtree_empty(struct intel_engine_cs *engine)
	90	{
	91	struct intel_breadcrumbs *b = &engine->breadcrumbs;
	92
61d3dc70	93	if (b->irq_wait) {
f97fbf96 CW	94	pr_err("Empty breadcrumbs still has a waiter\n");
	95	return -EINVAL;
	96	}
	97
	98	if (!RB_EMPTY_ROOT(&b->waiters)) {
	99	pr_err("Empty breadcrumbs, but wait-tree not empty\n");
	100	return -EINVAL;
	101	}
	102
	103	return 0;
	104	}
	105
	106	static int igt_random_insert_remove(void *arg)
	107	{
	108	const u32 seqno_bias = 0x1000;
	109	I915_RND_STATE(prng);
	110	struct intel_engine_cs *engine = arg;
	111	struct intel_wait *waiters;
	112	const int count = 4096;
	113	unsigned int *order;
	114	unsigned long *bitmap;
	115	int err = -ENOMEM;
	116	int n;
	117
	118	mock_engine_reset(engine);
	119
2098105e	120	waiters = kvmalloc_array(count, sizeof(*waiters), GFP_TEMPORARY);
f97fbf96 CW	121	if (!waiters)
	122	goto out_engines;
	123
	124	bitmap = kcalloc(DIV_ROUND_UP(count, BITS_PER_LONG), sizeof(*bitmap),
	125	GFP_TEMPORARY);
	126	if (!bitmap)
	127	goto out_waiters;
	128
	129	order = i915_random_order(count, &prng);
	130	if (!order)
	131	goto out_bitmap;
	132
	133	for (n = 0; n < count; n++)
754c9fd5	134	intel_wait_init_for_seqno(&waiters[n], seqno_bias + n);
f97fbf96 CW	135
	136	err = check_rbtree(engine, bitmap, waiters, count);
	137	if (err)
	138	goto out_order;
	139
	140	/* Add and remove waiters into the rbtree in random order. At each
	141	* step, we verify that the rbtree is correctly ordered.
	142	*/
	143	for (n = 0; n < count; n++) {
	144	int i = order[n];
	145
	146	intel_engine_add_wait(engine, &waiters[i]);
	147	__set_bit(i, bitmap);
	148
	149	err = check_rbtree(engine, bitmap, waiters, count);
	150	if (err)
	151	goto out_order;
	152	}
	153
	154	i915_random_reorder(order, count, &prng);
	155	for (n = 0; n < count; n++) {
	156	int i = order[n];
	157
	158	intel_engine_remove_wait(engine, &waiters[i]);
	159	__clear_bit(i, bitmap);
	160
	161	err = check_rbtree(engine, bitmap, waiters, count);
	162	if (err)
	163	goto out_order;
	164	}
	165
	166	err = check_rbtree_empty(engine);
	167	out_order:
	168	kfree(order);
	169	out_bitmap:
	170	kfree(bitmap);
	171	out_waiters:
2098105e	172	kvfree(waiters);
f97fbf96 CW	173	out_engines:
	174	mock_engine_flush(engine);
	175	return err;
	176	}
	177
ae1f8090 CW	178	static int igt_insert_complete(void *arg)
	179	{
	180	const u32 seqno_bias = 0x1000;
	181	struct intel_engine_cs *engine = arg;
	182	struct intel_wait *waiters;
	183	const int count = 4096;
	184	unsigned long *bitmap;
	185	int err = -ENOMEM;
	186	int n, m;
	187
	188	mock_engine_reset(engine);
	189
2098105e	190	waiters = kvmalloc_array(count, sizeof(*waiters), GFP_TEMPORARY);
ae1f8090 CW	191	if (!waiters)
	192	goto out_engines;
	193
	194	bitmap = kcalloc(DIV_ROUND_UP(count, BITS_PER_LONG), sizeof(*bitmap),
	195	GFP_TEMPORARY);
	196	if (!bitmap)
	197	goto out_waiters;
	198
	199	for (n = 0; n < count; n++) {
754c9fd5	200	intel_wait_init_for_seqno(&waiters[n], n + seqno_bias);
ae1f8090 CW	201	intel_engine_add_wait(engine, &waiters[n]);
	202	__set_bit(n, bitmap);
	203	}
	204	err = check_rbtree(engine, bitmap, waiters, count);
	205	if (err)
	206	goto out_bitmap;
	207
	208	/* On each step, we advance the seqno so that several waiters are then
	209	* complete (we increase the seqno by increasingly larger values to
	210	* retire more and more waiters at once). All retired waiters should
	211	* be woken and removed from the rbtree, and so that we check.
	212	*/
	213	for (n = 0; n < count; n = m) {
	214	int seqno = 2 * n;
	215
	216	GEM_BUG_ON(find_first_bit(bitmap, count) != n);
	217
	218	if (intel_wait_complete(&waiters[n])) {
	219	pr_err("waiter[%d, seqno=%d] completed too early\n",
	220	n, waiters[n].seqno);
	221	err = -EINVAL;
	222	goto out_bitmap;
	223	}
	224
	225	/* complete the following waiters */
	226	mock_seqno_advance(engine, seqno + seqno_bias);
	227	for (m = n; m <= seqno; m++) {
	228	if (m == count)
	229	break;
	230
	231	GEM_BUG_ON(!test_bit(m, bitmap));
	232	__clear_bit(m, bitmap);
	233	}
	234
	235	intel_engine_remove_wait(engine, &waiters[n]);
	236	RB_CLEAR_NODE(&waiters[n].node);
	237
	238	err = check_rbtree(engine, bitmap, waiters, count);
	239	if (err) {
	240	pr_err("rbtree corrupt after seqno advance to %d\n",
	241	seqno + seqno_bias);
	242	goto out_bitmap;
	243	}
	244
	245	err = check_completion(engine, bitmap, waiters, count);
	246	if (err) {
	247	pr_err("completions after seqno advance to %d failed\n",
	248	seqno + seqno_bias);
	249	goto out_bitmap;
	250	}
	251	}
	252
	253	err = check_rbtree_empty(engine);
	254	out_bitmap:
	255	kfree(bitmap);
	256	out_waiters:
2098105e	257	kvfree(waiters);
ae1f8090 CW	258	out_engines:
	259	mock_engine_flush(engine);
	260	return err;
	261	}
	262
e62e8ad1 CW	263	struct igt_wakeup {
	264	struct task_struct *tsk;
	265	atomic_t ready, set, *done;
	266	struct intel_engine_cs *engine;
	267	unsigned long flags;
	268	#define STOP 0
	269	#define IDLE 1
	270	wait_queue_head_t *wq;
	271	u32 seqno;
	272	};
	273
	274	static int wait_atomic(atomic_t *p)
	275	{
	276	schedule();
	277	return 0;
	278	}
	279
	280	static int wait_atomic_timeout(atomic_t *p)
	281	{
	282	return schedule_timeout(10 * HZ) ? 0 : -ETIMEDOUT;
	283	}
	284
	285	static bool wait_for_ready(struct igt_wakeup *w)
	286	{
	287	DEFINE_WAIT(ready);
	288
	289	set_bit(IDLE, &w->flags);
	290	if (atomic_dec_and_test(w->done))
	291	wake_up_atomic_t(w->done);
	292
	293	if (test_bit(STOP, &w->flags))
	294	goto out;
	295
	296	for (;;) {
	297	prepare_to_wait(w->wq, &ready, TASK_INTERRUPTIBLE);
	298	if (atomic_read(w->ready) == 0)
	299	break;
	300
	301	schedule();
	302	}
	303	finish_wait(w->wq, &ready);
	304
	305	out:
	306	clear_bit(IDLE, &w->flags);
	307	if (atomic_dec_and_test(w->set))
	308	wake_up_atomic_t(w->set);
	309
	310	return !test_bit(STOP, &w->flags);
	311	}
	312
	313	static int igt_wakeup_thread(void *arg)
	314	{
	315	struct igt_wakeup *w = arg;
	316	struct intel_wait wait;
	317
	318	while (wait_for_ready(w)) {
	319	GEM_BUG_ON(kthread_should_stop());
	320
754c9fd5	321	intel_wait_init_for_seqno(&wait, w->seqno);
e62e8ad1 CW	322	intel_engine_add_wait(w->engine, &wait);
	323	for (;;) {
	324	set_current_state(TASK_UNINTERRUPTIBLE);
	325	if (i915_seqno_passed(intel_engine_get_seqno(w->engine),
	326	w->seqno))
	327	break;
	328
	329	if (test_bit(STOP, &w->flags)) /* emergency escape */
	330	break;
	331
	332	schedule();
	333	}
	334	intel_engine_remove_wait(w->engine, &wait);
	335	__set_current_state(TASK_RUNNING);
	336	}
	337
	338	return 0;
	339	}
	340
	341	static void igt_wake_all_sync(atomic_t *ready,
	342	atomic_t *set,
	343	atomic_t *done,
	344	wait_queue_head_t *wq,
	345	int count)
	346	{
	347	atomic_set(set, count);
	348	atomic_set(ready, 0);
	349	wake_up_all(wq);
	350
	351	wait_on_atomic_t(set, wait_atomic, TASK_UNINTERRUPTIBLE);
	352	atomic_set(ready, count);
	353	atomic_set(done, count);
	354	}
	355
	356	static int igt_wakeup(void *arg)
	357	{
	358	I915_RND_STATE(prng);
	359	const int state = TASK_UNINTERRUPTIBLE;
	360	struct intel_engine_cs *engine = arg;
	361	struct igt_wakeup *waiters;
	362	DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
	363	const int count = 4096;
	364	const u32 max_seqno = count / 4;
	365	atomic_t ready, set, done;
	366	int err = -ENOMEM;
	367	int n, step;
	368
	369	mock_engine_reset(engine);
	370
2098105e	371	waiters = kvmalloc_array(count, sizeof(*waiters), GFP_TEMPORARY);
e62e8ad1 CW	372	if (!waiters)
	373	goto out_engines;
	374
	375	/* Create a large number of threads, each waiting on a random seqno.
	376	* Multiple waiters will be waiting for the same seqno.
	377	*/
	378	atomic_set(&ready, count);
	379	for (n = 0; n < count; n++) {
	380	waiters[n].wq = &wq;
	381	waiters[n].ready = &ready;
	382	waiters[n].set = &set;
	383	waiters[n].done = &done;
	384	waiters[n].engine = engine;
	385	waiters[n].flags = BIT(IDLE);
	386
	387	waiters[n].tsk = kthread_run(igt_wakeup_thread, &waiters[n],
	388	"i915/igt:%d", n);
	389	if (IS_ERR(waiters[n].tsk))
	390	goto out_waiters;
	391
	392	get_task_struct(waiters[n].tsk);
	393	}
	394
	395	for (step = 1; step <= max_seqno; step <<= 1) {
	396	u32 seqno;
	397
	398	/* The waiter threads start paused as we assign them a random
	399	* seqno and reset the engine. Once the engine is reset,
	400	* we signal that the threads may begin their wait upon their
	401	* seqno.
	402	*/
	403	for (n = 0; n < count; n++) {
	404	GEM_BUG_ON(!test_bit(IDLE, &waiters[n].flags));
	405	waiters[n].seqno =
	406	1 + prandom_u32_state(&prng) % max_seqno;
	407	}
	408	mock_seqno_advance(engine, 0);
	409	igt_wake_all_sync(&ready, &set, &done, &wq, count);
	410
	411	/* Simulate the GPU doing chunks of work, with one or more
	412	* seqno appearing to finish at the same time. A random number
	413	* of threads will be waiting upon the update and hopefully be
	414	* woken.
	415	*/
	416	for (seqno = 1; seqno <= max_seqno + step; seqno += step) {
	417	usleep_range(50, 500);
	418	mock_seqno_advance(engine, seqno);
	419	}
	420	GEM_BUG_ON(intel_engine_get_seqno(engine) < 1 + max_seqno);
	421
	422	/* With the seqno now beyond any of the waiting threads, they
	423	* should all be woken, see that they are complete and signal
	424	* that they are ready for the next test. We wait until all
	425	* threads are complete and waiting for us (i.e. not a seqno).
	426	*/
	427	err = wait_on_atomic_t(&done, wait_atomic_timeout, state);
	428	if (err) {
	429	pr_err("Timed out waiting for %d remaining waiters\n",
	430	atomic_read(&done));
	431	break;
	432	}
	433
	434	err = check_rbtree_empty(engine);
	435	if (err)
436	break;
437	}
438
439	out_waiters:
440	for (n = 0; n < count; n++) {
441	if (IS_ERR(waiters[n].tsk))
442	break;
443
444	set_bit(STOP, &waiters[n].flags);
445	}
446	mock_seqno_advance(engine, INT_MAX); /* wakeup any broken waiters */
447	igt_wake_all_sync(&ready, &set, &done, &wq, n);
448
449	for (n = 0; n < count; n++) {
450	if (IS_ERR(waiters[n].tsk))
451	break;
452
453	kthread_stop(waiters[n].tsk);
454	put_task_struct(waiters[n].tsk);
455	}
456
2098105e	457	kvfree(waiters);
e62e8ad1 CW	458	out_engines:
	459	mock_engine_flush(engine);
	460	return err;
	461	}
	462
f97fbf96 CW	463	int intel_breadcrumbs_mock_selftests(void)
	464	{
	465	static const struct i915_subtest tests[] = {
	466	SUBTEST(igt_random_insert_remove),
ae1f8090	467	SUBTEST(igt_insert_complete),
e62e8ad1	468	SUBTEST(igt_wakeup),
f97fbf96	469	};
0daf0113	470	struct drm_i915_private *i915;
f97fbf96 CW	471	int err;
f97fbf96 CW	472
0daf0113 CW	473	i915 = mock_gem_device();
0daf0113 CW	474	if (!i915)
f97fbf96 CW	475	return -ENOMEM;
f97fbf96 CW	476
0daf0113 CW	477	err = i915_subtests(tests, i915->engine[RCS]);
0daf0113 CW	478	drm_dev_unref(&i915->drm);
f97fbf96 CW	479
	480	return err;
	481	}