]>
Commit | Line | Data |
---|---|---|
f97fbf96 CW |
1 | /* |
2 | * Copyright © 2016 Intel Corporation | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice (including the next | |
12 | * paragraph) shall be included in all copies or substantial portions of the | |
13 | * Software. | |
14 | * | |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS | |
21 | * IN THE SOFTWARE. | |
22 | * | |
23 | */ | |
24 | ||
25 | #include "../i915_selftest.h" | |
26 | #include "i915_random.h" | |
27 | ||
0daf0113 | 28 | #include "mock_gem_device.h" |
f97fbf96 CW |
29 | #include "mock_engine.h" |
30 | ||
31 | static int check_rbtree(struct intel_engine_cs *engine, | |
32 | const unsigned long *bitmap, | |
33 | const struct intel_wait *waiters, | |
34 | const int count) | |
35 | { | |
36 | struct intel_breadcrumbs *b = &engine->breadcrumbs; | |
37 | struct rb_node *rb; | |
38 | int n; | |
39 | ||
61d3dc70 | 40 | if (&b->irq_wait->node != rb_first(&b->waiters)) { |
f97fbf96 CW |
41 | pr_err("First waiter does not match first element of wait-tree\n"); |
42 | return -EINVAL; | |
43 | } | |
44 | ||
45 | n = find_first_bit(bitmap, count); | |
46 | for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) { | |
47 | struct intel_wait *w = container_of(rb, typeof(*w), node); | |
48 | int idx = w - waiters; | |
49 | ||
50 | if (!test_bit(idx, bitmap)) { | |
51 | pr_err("waiter[%d, seqno=%d] removed but still in wait-tree\n", | |
52 | idx, w->seqno); | |
53 | return -EINVAL; | |
54 | } | |
55 | ||
56 | if (n != idx) { | |
57 | pr_err("waiter[%d, seqno=%d] does not match expected next element in tree [%d]\n", | |
58 | idx, w->seqno, n); | |
59 | return -EINVAL; | |
60 | } | |
61 | ||
62 | n = find_next_bit(bitmap, count, n + 1); | |
63 | } | |
64 | ||
65 | return 0; | |
66 | } | |
67 | ||
ae1f8090 CW |
68 | static int check_completion(struct intel_engine_cs *engine, |
69 | const unsigned long *bitmap, | |
70 | const struct intel_wait *waiters, | |
71 | const int count) | |
72 | { | |
73 | int n; | |
74 | ||
75 | for (n = 0; n < count; n++) { | |
76 | if (intel_wait_complete(&waiters[n]) != !!test_bit(n, bitmap)) | |
77 | continue; | |
78 | ||
79 | pr_err("waiter[%d, seqno=%d] is %s, but expected %s\n", | |
80 | n, waiters[n].seqno, | |
81 | intel_wait_complete(&waiters[n]) ? "complete" : "active", | |
82 | test_bit(n, bitmap) ? "active" : "complete"); | |
83 | return -EINVAL; | |
84 | } | |
85 | ||
86 | return 0; | |
87 | } | |
88 | ||
f97fbf96 CW |
89 | static int check_rbtree_empty(struct intel_engine_cs *engine) |
90 | { | |
91 | struct intel_breadcrumbs *b = &engine->breadcrumbs; | |
92 | ||
61d3dc70 | 93 | if (b->irq_wait) { |
f97fbf96 CW |
94 | pr_err("Empty breadcrumbs still has a waiter\n"); |
95 | return -EINVAL; | |
96 | } | |
97 | ||
98 | if (!RB_EMPTY_ROOT(&b->waiters)) { | |
99 | pr_err("Empty breadcrumbs, but wait-tree not empty\n"); | |
100 | return -EINVAL; | |
101 | } | |
102 | ||
103 | return 0; | |
104 | } | |
105 | ||
106 | static int igt_random_insert_remove(void *arg) | |
107 | { | |
108 | const u32 seqno_bias = 0x1000; | |
109 | I915_RND_STATE(prng); | |
110 | struct intel_engine_cs *engine = arg; | |
111 | struct intel_wait *waiters; | |
112 | const int count = 4096; | |
113 | unsigned int *order; | |
114 | unsigned long *bitmap; | |
115 | int err = -ENOMEM; | |
116 | int n; | |
117 | ||
118 | mock_engine_reset(engine); | |
119 | ||
0ee931c4 | 120 | waiters = kvmalloc_array(count, sizeof(*waiters), GFP_KERNEL); |
f97fbf96 CW |
121 | if (!waiters) |
122 | goto out_engines; | |
123 | ||
124 | bitmap = kcalloc(DIV_ROUND_UP(count, BITS_PER_LONG), sizeof(*bitmap), | |
0ee931c4 | 125 | GFP_KERNEL); |
f97fbf96 CW |
126 | if (!bitmap) |
127 | goto out_waiters; | |
128 | ||
129 | order = i915_random_order(count, &prng); | |
130 | if (!order) | |
131 | goto out_bitmap; | |
132 | ||
133 | for (n = 0; n < count; n++) | |
754c9fd5 | 134 | intel_wait_init_for_seqno(&waiters[n], seqno_bias + n); |
f97fbf96 CW |
135 | |
136 | err = check_rbtree(engine, bitmap, waiters, count); | |
137 | if (err) | |
138 | goto out_order; | |
139 | ||
140 | /* Add and remove waiters into the rbtree in random order. At each | |
141 | * step, we verify that the rbtree is correctly ordered. | |
142 | */ | |
143 | for (n = 0; n < count; n++) { | |
144 | int i = order[n]; | |
145 | ||
146 | intel_engine_add_wait(engine, &waiters[i]); | |
147 | __set_bit(i, bitmap); | |
148 | ||
149 | err = check_rbtree(engine, bitmap, waiters, count); | |
150 | if (err) | |
151 | goto out_order; | |
152 | } | |
153 | ||
154 | i915_random_reorder(order, count, &prng); | |
155 | for (n = 0; n < count; n++) { | |
156 | int i = order[n]; | |
157 | ||
158 | intel_engine_remove_wait(engine, &waiters[i]); | |
159 | __clear_bit(i, bitmap); | |
160 | ||
161 | err = check_rbtree(engine, bitmap, waiters, count); | |
162 | if (err) | |
163 | goto out_order; | |
164 | } | |
165 | ||
166 | err = check_rbtree_empty(engine); | |
167 | out_order: | |
168 | kfree(order); | |
169 | out_bitmap: | |
170 | kfree(bitmap); | |
171 | out_waiters: | |
2098105e | 172 | kvfree(waiters); |
f97fbf96 CW |
173 | out_engines: |
174 | mock_engine_flush(engine); | |
175 | return err; | |
176 | } | |
177 | ||
ae1f8090 CW |
178 | static int igt_insert_complete(void *arg) |
179 | { | |
180 | const u32 seqno_bias = 0x1000; | |
181 | struct intel_engine_cs *engine = arg; | |
182 | struct intel_wait *waiters; | |
183 | const int count = 4096; | |
184 | unsigned long *bitmap; | |
185 | int err = -ENOMEM; | |
186 | int n, m; | |
187 | ||
188 | mock_engine_reset(engine); | |
189 | ||
0ee931c4 | 190 | waiters = kvmalloc_array(count, sizeof(*waiters), GFP_KERNEL); |
ae1f8090 CW |
191 | if (!waiters) |
192 | goto out_engines; | |
193 | ||
194 | bitmap = kcalloc(DIV_ROUND_UP(count, BITS_PER_LONG), sizeof(*bitmap), | |
0ee931c4 | 195 | GFP_KERNEL); |
ae1f8090 CW |
196 | if (!bitmap) |
197 | goto out_waiters; | |
198 | ||
199 | for (n = 0; n < count; n++) { | |
754c9fd5 | 200 | intel_wait_init_for_seqno(&waiters[n], n + seqno_bias); |
ae1f8090 CW |
201 | intel_engine_add_wait(engine, &waiters[n]); |
202 | __set_bit(n, bitmap); | |
203 | } | |
204 | err = check_rbtree(engine, bitmap, waiters, count); | |
205 | if (err) | |
206 | goto out_bitmap; | |
207 | ||
208 | /* On each step, we advance the seqno so that several waiters are then | |
209 | * complete (we increase the seqno by increasingly larger values to | |
210 | * retire more and more waiters at once). All retired waiters should | |
211 | * be woken and removed from the rbtree, and so that we check. | |
212 | */ | |
213 | for (n = 0; n < count; n = m) { | |
214 | int seqno = 2 * n; | |
215 | ||
216 | GEM_BUG_ON(find_first_bit(bitmap, count) != n); | |
217 | ||
218 | if (intel_wait_complete(&waiters[n])) { | |
219 | pr_err("waiter[%d, seqno=%d] completed too early\n", | |
220 | n, waiters[n].seqno); | |
221 | err = -EINVAL; | |
222 | goto out_bitmap; | |
223 | } | |
224 | ||
225 | /* complete the following waiters */ | |
226 | mock_seqno_advance(engine, seqno + seqno_bias); | |
227 | for (m = n; m <= seqno; m++) { | |
228 | if (m == count) | |
229 | break; | |
230 | ||
231 | GEM_BUG_ON(!test_bit(m, bitmap)); | |
232 | __clear_bit(m, bitmap); | |
233 | } | |
234 | ||
235 | intel_engine_remove_wait(engine, &waiters[n]); | |
236 | RB_CLEAR_NODE(&waiters[n].node); | |
237 | ||
238 | err = check_rbtree(engine, bitmap, waiters, count); | |
239 | if (err) { | |
240 | pr_err("rbtree corrupt after seqno advance to %d\n", | |
241 | seqno + seqno_bias); | |
242 | goto out_bitmap; | |
243 | } | |
244 | ||
245 | err = check_completion(engine, bitmap, waiters, count); | |
246 | if (err) { | |
247 | pr_err("completions after seqno advance to %d failed\n", | |
248 | seqno + seqno_bias); | |
249 | goto out_bitmap; | |
250 | } | |
251 | } | |
252 | ||
253 | err = check_rbtree_empty(engine); | |
254 | out_bitmap: | |
255 | kfree(bitmap); | |
256 | out_waiters: | |
2098105e | 257 | kvfree(waiters); |
ae1f8090 CW |
258 | out_engines: |
259 | mock_engine_flush(engine); | |
260 | return err; | |
261 | } | |
262 | ||
e62e8ad1 CW |
263 | struct igt_wakeup { |
264 | struct task_struct *tsk; | |
265 | atomic_t *ready, *set, *done; | |
266 | struct intel_engine_cs *engine; | |
267 | unsigned long flags; | |
268 | #define STOP 0 | |
269 | #define IDLE 1 | |
270 | wait_queue_head_t *wq; | |
271 | u32 seqno; | |
272 | }; | |
273 | ||
5e4def20 | 274 | static int wait_atomic_timeout(atomic_t *p, unsigned int mode) |
e62e8ad1 CW |
275 | { |
276 | return schedule_timeout(10 * HZ) ? 0 : -ETIMEDOUT; | |
277 | } | |
278 | ||
279 | static bool wait_for_ready(struct igt_wakeup *w) | |
280 | { | |
281 | DEFINE_WAIT(ready); | |
282 | ||
283 | set_bit(IDLE, &w->flags); | |
284 | if (atomic_dec_and_test(w->done)) | |
285 | wake_up_atomic_t(w->done); | |
286 | ||
287 | if (test_bit(STOP, &w->flags)) | |
288 | goto out; | |
289 | ||
290 | for (;;) { | |
291 | prepare_to_wait(w->wq, &ready, TASK_INTERRUPTIBLE); | |
292 | if (atomic_read(w->ready) == 0) | |
293 | break; | |
294 | ||
295 | schedule(); | |
296 | } | |
297 | finish_wait(w->wq, &ready); | |
298 | ||
299 | out: | |
300 | clear_bit(IDLE, &w->flags); | |
301 | if (atomic_dec_and_test(w->set)) | |
302 | wake_up_atomic_t(w->set); | |
303 | ||
304 | return !test_bit(STOP, &w->flags); | |
305 | } | |
306 | ||
307 | static int igt_wakeup_thread(void *arg) | |
308 | { | |
309 | struct igt_wakeup *w = arg; | |
310 | struct intel_wait wait; | |
311 | ||
312 | while (wait_for_ready(w)) { | |
313 | GEM_BUG_ON(kthread_should_stop()); | |
314 | ||
754c9fd5 | 315 | intel_wait_init_for_seqno(&wait, w->seqno); |
e62e8ad1 CW |
316 | intel_engine_add_wait(w->engine, &wait); |
317 | for (;;) { | |
318 | set_current_state(TASK_UNINTERRUPTIBLE); | |
319 | if (i915_seqno_passed(intel_engine_get_seqno(w->engine), | |
320 | w->seqno)) | |
321 | break; | |
322 | ||
323 | if (test_bit(STOP, &w->flags)) /* emergency escape */ | |
324 | break; | |
325 | ||
326 | schedule(); | |
327 | } | |
328 | intel_engine_remove_wait(w->engine, &wait); | |
329 | __set_current_state(TASK_RUNNING); | |
330 | } | |
331 | ||
332 | return 0; | |
333 | } | |
334 | ||
335 | static void igt_wake_all_sync(atomic_t *ready, | |
336 | atomic_t *set, | |
337 | atomic_t *done, | |
338 | wait_queue_head_t *wq, | |
339 | int count) | |
340 | { | |
341 | atomic_set(set, count); | |
342 | atomic_set(ready, 0); | |
343 | wake_up_all(wq); | |
344 | ||
5e4def20 | 345 | wait_on_atomic_t(set, atomic_t_wait, TASK_UNINTERRUPTIBLE); |
e62e8ad1 CW |
346 | atomic_set(ready, count); |
347 | atomic_set(done, count); | |
348 | } | |
349 | ||
350 | static int igt_wakeup(void *arg) | |
351 | { | |
352 | I915_RND_STATE(prng); | |
353 | const int state = TASK_UNINTERRUPTIBLE; | |
354 | struct intel_engine_cs *engine = arg; | |
355 | struct igt_wakeup *waiters; | |
356 | DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq); | |
357 | const int count = 4096; | |
358 | const u32 max_seqno = count / 4; | |
359 | atomic_t ready, set, done; | |
360 | int err = -ENOMEM; | |
361 | int n, step; | |
362 | ||
363 | mock_engine_reset(engine); | |
364 | ||
0ee931c4 | 365 | waiters = kvmalloc_array(count, sizeof(*waiters), GFP_KERNEL); |
e62e8ad1 CW |
366 | if (!waiters) |
367 | goto out_engines; | |
368 | ||
369 | /* Create a large number of threads, each waiting on a random seqno. | |
370 | * Multiple waiters will be waiting for the same seqno. | |
371 | */ | |
372 | atomic_set(&ready, count); | |
373 | for (n = 0; n < count; n++) { | |
374 | waiters[n].wq = &wq; | |
375 | waiters[n].ready = &ready; | |
376 | waiters[n].set = &set; | |
377 | waiters[n].done = &done; | |
378 | waiters[n].engine = engine; | |
379 | waiters[n].flags = BIT(IDLE); | |
380 | ||
381 | waiters[n].tsk = kthread_run(igt_wakeup_thread, &waiters[n], | |
382 | "i915/igt:%d", n); | |
383 | if (IS_ERR(waiters[n].tsk)) | |
384 | goto out_waiters; | |
385 | ||
386 | get_task_struct(waiters[n].tsk); | |
387 | } | |
388 | ||
389 | for (step = 1; step <= max_seqno; step <<= 1) { | |
390 | u32 seqno; | |
391 | ||
392 | /* The waiter threads start paused as we assign them a random | |
393 | * seqno and reset the engine. Once the engine is reset, | |
394 | * we signal that the threads may begin their wait upon their | |
395 | * seqno. | |
396 | */ | |
397 | for (n = 0; n < count; n++) { | |
398 | GEM_BUG_ON(!test_bit(IDLE, &waiters[n].flags)); | |
399 | waiters[n].seqno = | |
400 | 1 + prandom_u32_state(&prng) % max_seqno; | |
401 | } | |
402 | mock_seqno_advance(engine, 0); | |
403 | igt_wake_all_sync(&ready, &set, &done, &wq, count); | |
404 | ||
405 | /* Simulate the GPU doing chunks of work, with one or more | |
406 | * seqno appearing to finish at the same time. A random number | |
407 | * of threads will be waiting upon the update and hopefully be | |
408 | * woken. | |
409 | */ | |
410 | for (seqno = 1; seqno <= max_seqno + step; seqno += step) { | |
411 | usleep_range(50, 500); | |
412 | mock_seqno_advance(engine, seqno); | |
413 | } | |
414 | GEM_BUG_ON(intel_engine_get_seqno(engine) < 1 + max_seqno); | |
415 | ||
416 | /* With the seqno now beyond any of the waiting threads, they | |
417 | * should all be woken, see that they are complete and signal | |
418 | * that they are ready for the next test. We wait until all | |
419 | * threads are complete and waiting for us (i.e. not a seqno). | |
420 | */ | |
421 | err = wait_on_atomic_t(&done, wait_atomic_timeout, state); | |
422 | if (err) { | |
423 | pr_err("Timed out waiting for %d remaining waiters\n", | |
424 | atomic_read(&done)); | |
425 | break; | |
426 | } | |
427 | ||
428 | err = check_rbtree_empty(engine); | |
429 | if (err) | |
430 | break; | |
431 | } | |
432 | ||
433 | out_waiters: | |
434 | for (n = 0; n < count; n++) { | |
435 | if (IS_ERR(waiters[n].tsk)) | |
436 | break; | |
437 | ||
438 | set_bit(STOP, &waiters[n].flags); | |
439 | } | |
440 | mock_seqno_advance(engine, INT_MAX); /* wakeup any broken waiters */ | |
441 | igt_wake_all_sync(&ready, &set, &done, &wq, n); | |
442 | ||
443 | for (n = 0; n < count; n++) { | |
444 | if (IS_ERR(waiters[n].tsk)) | |
445 | break; | |
446 | ||
447 | kthread_stop(waiters[n].tsk); | |
448 | put_task_struct(waiters[n].tsk); | |
449 | } | |
450 | ||
2098105e | 451 | kvfree(waiters); |
e62e8ad1 CW |
452 | out_engines: |
453 | mock_engine_flush(engine); | |
454 | return err; | |
455 | } | |
456 | ||
f97fbf96 CW |
457 | int intel_breadcrumbs_mock_selftests(void) |
458 | { | |
459 | static const struct i915_subtest tests[] = { | |
460 | SUBTEST(igt_random_insert_remove), | |
ae1f8090 | 461 | SUBTEST(igt_insert_complete), |
e62e8ad1 | 462 | SUBTEST(igt_wakeup), |
f97fbf96 | 463 | }; |
0daf0113 | 464 | struct drm_i915_private *i915; |
f97fbf96 CW |
465 | int err; |
466 | ||
0daf0113 CW |
467 | i915 = mock_gem_device(); |
468 | if (!i915) | |
f97fbf96 CW |
469 | return -ENOMEM; |
470 | ||
0daf0113 CW |
471 | err = i915_subtests(tests, i915->engine[RCS]); |
472 | drm_dev_unref(&i915->drm); | |
f97fbf96 CW |
473 | |
474 | return err; | |
475 | } |