]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/gpu/drm/i915/intel_engine_cs.c
projects / mirror_ubuntu-bionic-kernel.git / blob
? search:


8dd8e7750e52777b736d9c1820626e37efbacfe4
[mirror_ubuntu-bionic-kernel.git] / drivers / gpu / drm / i915 / intel_engine_cs.c
1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25 #include <drm/drm_print.h>
26
27 #include "i915_drv.h"
28 #include "i915_vgpu.h"
29 #include "intel_ringbuffer.h"
30 #include "intel_lrc.h"
31
32 /* Haswell does have the CXT_SIZE register however it does not appear to be
33 * valid. Now, docs explain in dwords what is in the context object. The full
34 * size is 70720 bytes, however, the power context and execlist context will
35 * never be saved (power context is stored elsewhere, and execlists don't work
36 * on HSW) - so the final size, including the extra state required for the
37 * Resource Streamer, is 66944 bytes, which rounds to 17 pages.
38 */
39 #define HSW_CXT_TOTAL_SIZE (17 * PAGE_SIZE)
40 /* Same as Haswell, but 72064 bytes now. */
41 #define GEN8_CXT_TOTAL_SIZE (18 * PAGE_SIZE)
42
43 #define GEN8_LR_CONTEXT_RENDER_SIZE (20 * PAGE_SIZE)
44 #define GEN9_LR_CONTEXT_RENDER_SIZE (22 * PAGE_SIZE)
45 #define GEN10_LR_CONTEXT_RENDER_SIZE (18 * PAGE_SIZE)
46
47 #define GEN8_LR_CONTEXT_OTHER_SIZE ( 2 * PAGE_SIZE)
48
49 struct engine_class_info {
50 const char *name;
51 int (*init_legacy)(struct intel_engine_cs *engine);
52 int (*init_execlists)(struct intel_engine_cs *engine);
53
54 u8 uabi_class;
55 };
56
57 static const struct engine_class_info intel_engine_classes[] = {
58 [RENDER_CLASS] = {
59 .name = "rcs",
60 .init_execlists = logical_render_ring_init,
61 .init_legacy = intel_init_render_ring_buffer,
62 .uabi_class = I915_ENGINE_CLASS_RENDER,
63 },
64 [COPY_ENGINE_CLASS] = {
65 .name = "bcs",
66 .init_execlists = logical_xcs_ring_init,
67 .init_legacy = intel_init_blt_ring_buffer,
68 .uabi_class = I915_ENGINE_CLASS_COPY,
69 },
70 [VIDEO_DECODE_CLASS] = {
71 .name = "vcs",
72 .init_execlists = logical_xcs_ring_init,
73 .init_legacy = intel_init_bsd_ring_buffer,
74 .uabi_class = I915_ENGINE_CLASS_VIDEO,
75 },
76 [VIDEO_ENHANCEMENT_CLASS] = {
77 .name = "vecs",
78 .init_execlists = logical_xcs_ring_init,
79 .init_legacy = intel_init_vebox_ring_buffer,
80 .uabi_class = I915_ENGINE_CLASS_VIDEO_ENHANCE,
81 },
82 };
83
84 struct engine_info {
85 unsigned int hw_id;
86 unsigned int uabi_id;
87 u8 class;
88 u8 instance;
89 u32 mmio_base;
90 unsigned irq_shift;
91 };
92
93 static const struct engine_info intel_engines[] = {
94 [RCS] = {
95 .hw_id = RCS_HW,
96 .uabi_id = I915_EXEC_RENDER,
97 .class = RENDER_CLASS,
98 .instance = 0,
99 .mmio_base = RENDER_RING_BASE,
100 .irq_shift = GEN8_RCS_IRQ_SHIFT,
101 },
102 [BCS] = {
103 .hw_id = BCS_HW,
104 .uabi_id = I915_EXEC_BLT,
105 .class = COPY_ENGINE_CLASS,
106 .instance = 0,
107 .mmio_base = BLT_RING_BASE,
108 .irq_shift = GEN8_BCS_IRQ_SHIFT,
109 },
110 [VCS] = {
111 .hw_id = VCS_HW,
112 .uabi_id = I915_EXEC_BSD,
113 .class = VIDEO_DECODE_CLASS,
114 .instance = 0,
115 .mmio_base = GEN6_BSD_RING_BASE,
116 .irq_shift = GEN8_VCS1_IRQ_SHIFT,
117 },
118 [VCS2] = {
119 .hw_id = VCS2_HW,
120 .uabi_id = I915_EXEC_BSD,
121 .class = VIDEO_DECODE_CLASS,
122 .instance = 1,
123 .mmio_base = GEN8_BSD2_RING_BASE,
124 .irq_shift = GEN8_VCS2_IRQ_SHIFT,
125 },
126 [VECS] = {
127 .hw_id = VECS_HW,
128 .uabi_id = I915_EXEC_VEBOX,
129 .class = VIDEO_ENHANCEMENT_CLASS,
130 .instance = 0,
131 .mmio_base = VEBOX_RING_BASE,
132 .irq_shift = GEN8_VECS_IRQ_SHIFT,
133 },
134 };
135
136 /**
137 * ___intel_engine_context_size() - return the size of the context for an engine
138 * @dev_priv: i915 device private
139 * @class: engine class
140 *
141 * Each engine class may require a different amount of space for a context
142 * image.
143 *
144 * Return: size (in bytes) of an engine class specific context image
145 *
146 * Note: this size includes the HWSP, which is part of the context image
147 * in LRC mode, but does not include the "shared data page" used with
148 * GuC submission. The caller should account for this if using the GuC.
149 */
150 static u32
151 __intel_engine_context_size(struct drm_i915_private *dev_priv, u8 class)
152 {
153 u32 cxt_size;
154
155 BUILD_BUG_ON(I915_GTT_PAGE_SIZE != PAGE_SIZE);
156
157 switch (class) {
158 case RENDER_CLASS:
159 switch (INTEL_GEN(dev_priv)) {
160 default:
161 MISSING_CASE(INTEL_GEN(dev_priv));
162 case 10:
163 return GEN10_LR_CONTEXT_RENDER_SIZE;
164 case 9:
165 return GEN9_LR_CONTEXT_RENDER_SIZE;
166 case 8:
167 return i915_modparams.enable_execlists ?
168 GEN8_LR_CONTEXT_RENDER_SIZE :
169 GEN8_CXT_TOTAL_SIZE;
170 case 7:
171 if (IS_HASWELL(dev_priv))
172 return HSW_CXT_TOTAL_SIZE;
173
174 cxt_size = I915_READ(GEN7_CXT_SIZE);
175 return round_up(GEN7_CXT_TOTAL_SIZE(cxt_size) * 64,
176 PAGE_SIZE);
177 case 6:
178 cxt_size = I915_READ(CXT_SIZE);
179 return round_up(GEN6_CXT_TOTAL_SIZE(cxt_size) * 64,
180 PAGE_SIZE);
181 case 5:
182 case 4:
183 case 3:
184 case 2:
185 /* For the special day when i810 gets merged. */
186 case 1:
187 return 0;
188 }
189 break;
190 default:
191 MISSING_CASE(class);
192 case VIDEO_DECODE_CLASS:
193 case VIDEO_ENHANCEMENT_CLASS:
194 case COPY_ENGINE_CLASS:
195 if (INTEL_GEN(dev_priv) < 8)
196 return 0;
197 return GEN8_LR_CONTEXT_OTHER_SIZE;
198 }
199 }
200
201 static int
202 intel_engine_setup(struct drm_i915_private *dev_priv,
203 enum intel_engine_id id)
204 {
205 const struct engine_info *info = &intel_engines[id];
206 const struct engine_class_info *class_info;
207 struct intel_engine_cs *engine;
208
209 GEM_BUG_ON(info->class >= ARRAY_SIZE(intel_engine_classes));
210 class_info = &intel_engine_classes[info->class];
211
212 GEM_BUG_ON(dev_priv->engine[id]);
213 engine = kzalloc(sizeof(*engine), GFP_KERNEL);
214 if (!engine)
215 return -ENOMEM;
216
217 engine->id = id;
218 engine->i915 = dev_priv;
219 WARN_ON(snprintf(engine->name, sizeof(engine->name), "%s%u",
220 class_info->name, info->instance) >=
221 sizeof(engine->name));
222 engine->hw_id = engine->guc_id = info->hw_id;
223 engine->mmio_base = info->mmio_base;
224 engine->irq_shift = info->irq_shift;
225 engine->class = info->class;
226 engine->instance = info->instance;
227
228 engine->uabi_id = info->uabi_id;
229 engine->uabi_class = class_info->uabi_class;
230
231 engine->context_size = __intel_engine_context_size(dev_priv,
232 engine->class);
233 if (WARN_ON(engine->context_size > BIT(20)))
234 engine->context_size = 0;
235
236 /* Nothing to do here, execute in order of dependencies */
237 engine->schedule = NULL;
238
239 ATOMIC_INIT_NOTIFIER_HEAD(&engine->context_status_notifier);
240
241 dev_priv->engine[id] = engine;
242 return 0;
243 }
244
245 /**
246 * intel_engines_init_mmio() - allocate and prepare the Engine Command Streamers
247 * @dev_priv: i915 device private
248 *
249 * Return: non-zero if the initialization failed.
250 */
251 int intel_engines_init_mmio(struct drm_i915_private *dev_priv)
252 {
253 struct intel_device_info *device_info = mkwrite_device_info(dev_priv);
254 const unsigned int ring_mask = INTEL_INFO(dev_priv)->ring_mask;
255 struct intel_engine_cs *engine;
256 enum intel_engine_id id;
257 unsigned int mask = 0;
258 unsigned int i;
259 int err;
260
261 WARN_ON(ring_mask == 0);
262 WARN_ON(ring_mask &
263 GENMASK(sizeof(mask) * BITS_PER_BYTE - 1, I915_NUM_ENGINES));
264
265 for (i = 0; i < ARRAY_SIZE(intel_engines); i++) {
266 if (!HAS_ENGINE(dev_priv, i))
267 continue;
268
269 err = intel_engine_setup(dev_priv, i);
270 if (err)
271 goto cleanup;
272
273 mask |= ENGINE_MASK(i);
274 }
275
276 /*
277 * Catch failures to update intel_engines table when the new engines
278 * are added to the driver by a warning and disabling the forgotten
279 * engines.
280 */
281 if (WARN_ON(mask != ring_mask))
282 device_info->ring_mask = mask;
283
284 /* We always presume we have at least RCS available for later probing */
285 if (WARN_ON(!HAS_ENGINE(dev_priv, RCS))) {
286 err = -ENODEV;
287 goto cleanup;
288 }
289
290 device_info->num_rings = hweight32(mask);
291
292 return 0;
293
294 cleanup:
295 for_each_engine(engine, dev_priv, id)
296 kfree(engine);
297 return err;
298 }
299
300 /**
301 * intel_engines_init() - init the Engine Command Streamers
302 * @dev_priv: i915 device private
303 *
304 * Return: non-zero if the initialization failed.
305 */
306 int intel_engines_init(struct drm_i915_private *dev_priv)
307 {
308 struct intel_engine_cs *engine;
309 enum intel_engine_id id, err_id;
310 int err;
311
312 for_each_engine(engine, dev_priv, id) {
313 const struct engine_class_info *class_info =
314 &intel_engine_classes[engine->class];
315 int (*init)(struct intel_engine_cs *engine);
316
317 if (i915_modparams.enable_execlists)
318 init = class_info->init_execlists;
319 else
320 init = class_info->init_legacy;
321
322 err = -EINVAL;
323 err_id = id;
324
325 if (GEM_WARN_ON(!init))
326 goto cleanup;
327
328 err = init(engine);
329 if (err)
330 goto cleanup;
331
332 GEM_BUG_ON(!engine->submit_request);
333 }
334
335 return 0;
336
337 cleanup:
338 for_each_engine(engine, dev_priv, id) {
339 if (id >= err_id) {
340 kfree(engine);
341 dev_priv->engine[id] = NULL;
342 } else {
343 dev_priv->gt.cleanup_engine(engine);
344 }
345 }
346 return err;
347 }
348
349 void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno)
350 {
351 struct drm_i915_private *dev_priv = engine->i915;
352
353 /* Our semaphore implementation is strictly monotonic (i.e. we proceed
354 * so long as the semaphore value in the register/page is greater
355 * than the sync value), so whenever we reset the seqno,
356 * so long as we reset the tracking semaphore value to 0, it will
357 * always be before the next request's seqno. If we don't reset
358 * the semaphore value, then when the seqno moves backwards all
359 * future waits will complete instantly (causing rendering corruption).
360 */
361 if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) {
362 I915_WRITE(RING_SYNC_0(engine->mmio_base), 0);
363 I915_WRITE(RING_SYNC_1(engine->mmio_base), 0);
364 if (HAS_VEBOX(dev_priv))
365 I915_WRITE(RING_SYNC_2(engine->mmio_base), 0);
366 }
367 if (dev_priv->semaphore) {
368 struct page *page = i915_vma_first_page(dev_priv->semaphore);
369 void *semaphores;
370
371 /* Semaphores are in noncoherent memory, flush to be safe */
372 semaphores = kmap_atomic(page);
373 memset(semaphores + GEN8_SEMAPHORE_OFFSET(engine->id, 0),
374 0, I915_NUM_ENGINES * gen8_semaphore_seqno_size);
375 drm_clflush_virt_range(semaphores + GEN8_SEMAPHORE_OFFSET(engine->id, 0),
376 I915_NUM_ENGINES * gen8_semaphore_seqno_size);
377 kunmap_atomic(semaphores);
378 }
379
380 intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
381 clear_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted);
382
383 /* After manually advancing the seqno, fake the interrupt in case
384 * there are any waiters for that seqno.
385 */
386 intel_engine_wakeup(engine);
387
388 GEM_BUG_ON(intel_engine_get_seqno(engine) != seqno);
389 }
390
391 static void intel_engine_init_timeline(struct intel_engine_cs *engine)
392 {
393 engine->timeline = &engine->i915->gt.global_timeline.engine[engine->id];
394 }
395
396 static bool csb_force_mmio(struct drm_i915_private *i915)
397 {
398 /*
399 * IOMMU adds unpredictable latency causing the CSB write (from the
400 * GPU into the HWSP) to only be visible some time after the interrupt
401 * (missed breadcrumb syndrome).
402 */
403 if (intel_vtd_active())
404 return true;
405
406 /* Older GVT emulation depends upon intercepting CSB mmio */
407 if (intel_vgpu_active(i915) && !intel_vgpu_has_hwsp_emulation(i915))
408 return true;
409
410 return false;
411 }
412
413 static void intel_engine_init_execlist(struct intel_engine_cs *engine)
414 {
415 struct intel_engine_execlists * const execlists = &engine->execlists;
416
417 execlists->csb_use_mmio = csb_force_mmio(engine->i915);
418
419 execlists->port_mask = 1;
420 BUILD_BUG_ON_NOT_POWER_OF_2(execlists_num_ports(execlists));
421 GEM_BUG_ON(execlists_num_ports(execlists) > EXECLIST_MAX_PORTS);
422
423 execlists->queue = RB_ROOT;
424 execlists->first = NULL;
425 }
426
427 /**
428 * intel_engines_setup_common - setup engine state not requiring hw access
429 * @engine: Engine to setup.
430 *
431 * Initializes @engine@ structure members shared between legacy and execlists
432 * submission modes which do not require hardware access.
433 *
434 * Typically done early in the submission mode specific engine setup stage.
435 */
436 void intel_engine_setup_common(struct intel_engine_cs *engine)
437 {
438 intel_engine_init_execlist(engine);
439
440 intel_engine_init_timeline(engine);
441 intel_engine_init_hangcheck(engine);
442 i915_gem_batch_pool_init(engine, &engine->batch_pool);
443
444 intel_engine_init_cmd_parser(engine);
445 }
446
447 int intel_engine_create_scratch(struct intel_engine_cs *engine, int size)
448 {
449 struct drm_i915_gem_object *obj;
450 struct i915_vma *vma;
451 int ret;
452
453 WARN_ON(engine->scratch);
454
455 obj = i915_gem_object_create_stolen(engine->i915, size);
456 if (!obj)
457 obj = i915_gem_object_create_internal(engine->i915, size);
458 if (IS_ERR(obj)) {
459 DRM_ERROR("Failed to allocate scratch page\n");
460 return PTR_ERR(obj);
461 }
462
463 vma = i915_vma_instance(obj, &engine->i915->ggtt.base, NULL);
464 if (IS_ERR(vma)) {
465 ret = PTR_ERR(vma);
466 goto err_unref;
467 }
468
469 ret = i915_vma_pin(vma, 0, 4096, PIN_GLOBAL | PIN_HIGH);
470 if (ret)
471 goto err_unref;
472
473 engine->scratch = vma;
474 DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
475 engine->name, i915_ggtt_offset(vma));
476 return 0;
477
478 err_unref:
479 i915_gem_object_put(obj);
480 return ret;
481 }
482
483 static void intel_engine_cleanup_scratch(struct intel_engine_cs *engine)
484 {
485 i915_vma_unpin_and_release(&engine->scratch);
486 }
487
488 static void cleanup_phys_status_page(struct intel_engine_cs *engine)
489 {
490 struct drm_i915_private *dev_priv = engine->i915;
491
492 if (!dev_priv->status_page_dmah)
493 return;
494
495 drm_pci_free(&dev_priv->drm, dev_priv->status_page_dmah);
496 engine->status_page.page_addr = NULL;
497 }
498
499 static void cleanup_status_page(struct intel_engine_cs *engine)
500 {
501 struct i915_vma *vma;
502 struct drm_i915_gem_object *obj;
503
504 vma = fetch_and_zero(&engine->status_page.vma);
505 if (!vma)
506 return;
507
508 obj = vma->obj;
509
510 i915_vma_unpin(vma);
511 i915_vma_close(vma);
512
513 i915_gem_object_unpin_map(obj);
514 __i915_gem_object_release_unless_active(obj);
515 }
516
517 static int init_status_page(struct intel_engine_cs *engine)
518 {
519 struct drm_i915_gem_object *obj;
520 struct i915_vma *vma;
521 unsigned int flags;
522 void *vaddr;
523 int ret;
524
525 obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
526 if (IS_ERR(obj)) {
527 DRM_ERROR("Failed to allocate status page\n");
528 return PTR_ERR(obj);
529 }
530
531 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
532 if (ret)
533 goto err;
534
535 vma = i915_vma_instance(obj, &engine->i915->ggtt.base, NULL);
536 if (IS_ERR(vma)) {
537 ret = PTR_ERR(vma);
538 goto err;
539 }
540
541 flags = PIN_GLOBAL;
542 if (!HAS_LLC(engine->i915))
543 /* On g33, we cannot place HWS above 256MiB, so
544 * restrict its pinning to the low mappable arena.
545 * Though this restriction is not documented for
546 * gen4, gen5, or byt, they also behave similarly
547 * and hang if the HWS is placed at the top of the
548 * GTT. To generalise, it appears that all !llc
549 * platforms have issues with us placing the HWS
550 * above the mappable region (even though we never
551 * actually map it).
552 */
553 flags |= PIN_MAPPABLE;
554 else
555 flags |= PIN_HIGH;
556 ret = i915_vma_pin(vma, 0, 4096, flags);
557 if (ret)
558 goto err;
559
560 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
561 if (IS_ERR(vaddr)) {
562 ret = PTR_ERR(vaddr);
563 goto err_unpin;
564 }
565
566 engine->status_page.vma = vma;
567 engine->status_page.ggtt_offset = i915_ggtt_offset(vma);
568 engine->status_page.page_addr = memset(vaddr, 0, PAGE_SIZE);
569
570 DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
571 engine->name, i915_ggtt_offset(vma));
572 return 0;
573
574 err_unpin:
575 i915_vma_unpin(vma);
576 err:
577 i915_gem_object_put(obj);
578 return ret;
579 }
580
581 static int init_phys_status_page(struct intel_engine_cs *engine)
582 {
583 struct drm_i915_private *dev_priv = engine->i915;
584
585 GEM_BUG_ON(engine->id != RCS);
586
587 dev_priv->status_page_dmah =
588 drm_pci_alloc(&dev_priv->drm, PAGE_SIZE, PAGE_SIZE);
589 if (!dev_priv->status_page_dmah)
590 return -ENOMEM;
591
592 engine->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
593 memset(engine->status_page.page_addr, 0, PAGE_SIZE);
594
595 return 0;
596 }
597
598 /**
599 * intel_engines_init_common - initialize cengine state which might require hw access
600 * @engine: Engine to initialize.
601 *
602 * Initializes @engine@ structure members shared between legacy and execlists
603 * submission modes which do require hardware access.
604 *
605 * Typcally done at later stages of submission mode specific engine setup.
606 *
607 * Returns zero on success or an error code on failure.
608 */
609 int intel_engine_init_common(struct intel_engine_cs *engine)
610 {
611 struct intel_ring *ring;
612 int ret;
613
614 engine->set_default_submission(engine);
615
616 /* We may need to do things with the shrinker which
617 * require us to immediately switch back to the default
618 * context. This can cause a problem as pinning the
619 * default context also requires GTT space which may not
620 * be available. To avoid this we always pin the default
621 * context.
622 */
623 ring = engine->context_pin(engine, engine->i915->kernel_context);
624 if (IS_ERR(ring))
625 return PTR_ERR(ring);
626
627 /*
628 * Similarly the preempt context must always be available so that
629 * we can interrupt the engine at any time.
630 */
631 if (INTEL_INFO(engine->i915)->has_logical_ring_preemption) {
632 ring = engine->context_pin(engine,
633 engine->i915->preempt_context);
634 if (IS_ERR(ring)) {
635 ret = PTR_ERR(ring);
636 goto err_unpin_kernel;
637 }
638 }
639
640 ret = intel_engine_init_breadcrumbs(engine);
641 if (ret)
642 goto err_unpin_preempt;
643
644 ret = i915_gem_render_state_init(engine);
645 if (ret)
646 goto err_breadcrumbs;
647
648 if (HWS_NEEDS_PHYSICAL(engine->i915))
649 ret = init_phys_status_page(engine);
650 else
651 ret = init_status_page(engine);
652 if (ret)
653 goto err_rs_fini;
654
655 return 0;
656
657 err_rs_fini:
658 i915_gem_render_state_fini(engine);
659 err_breadcrumbs:
660 intel_engine_fini_breadcrumbs(engine);
661 err_unpin_preempt:
662 if (INTEL_INFO(engine->i915)->has_logical_ring_preemption)
663 engine->context_unpin(engine, engine->i915->preempt_context);
664 err_unpin_kernel:
665 engine->context_unpin(engine, engine->i915->kernel_context);
666 return ret;
667 }
668
669 /**
670 * intel_engines_cleanup_common - cleans up the engine state created by
671 * the common initiailizers.
672 * @engine: Engine to cleanup.
673 *
674 * This cleans up everything created by the common helpers.
675 */
676 void intel_engine_cleanup_common(struct intel_engine_cs *engine)
677 {
678 intel_engine_cleanup_scratch(engine);
679
680 if (HWS_NEEDS_PHYSICAL(engine->i915))
681 cleanup_phys_status_page(engine);
682 else
683 cleanup_status_page(engine);
684
685 i915_gem_render_state_fini(engine);
686 intel_engine_fini_breadcrumbs(engine);
687 intel_engine_cleanup_cmd_parser(engine);
688 i915_gem_batch_pool_fini(&engine->batch_pool);
689
690 if (INTEL_INFO(engine->i915)->has_logical_ring_preemption)
691 engine->context_unpin(engine, engine->i915->preempt_context);
692 engine->context_unpin(engine, engine->i915->kernel_context);
693 }
694
695 u64 intel_engine_get_active_head(struct intel_engine_cs *engine)
696 {
697 struct drm_i915_private *dev_priv = engine->i915;
698 u64 acthd;
699
700 if (INTEL_GEN(dev_priv) >= 8)
701 acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base),
702 RING_ACTHD_UDW(engine->mmio_base));
703 else if (INTEL_GEN(dev_priv) >= 4)
704 acthd = I915_READ(RING_ACTHD(engine->mmio_base));
705 else
706 acthd = I915_READ(ACTHD);
707
708 return acthd;
709 }
710
711 u64 intel_engine_get_last_batch_head(struct intel_engine_cs *engine)
712 {
713 struct drm_i915_private *dev_priv = engine->i915;
714 u64 bbaddr;
715
716 if (INTEL_GEN(dev_priv) >= 8)
717 bbaddr = I915_READ64_2x32(RING_BBADDR(engine->mmio_base),
718 RING_BBADDR_UDW(engine->mmio_base));
719 else
720 bbaddr = I915_READ(RING_BBADDR(engine->mmio_base));
721
722 return bbaddr;
723 }
724
725 const char *i915_cache_level_str(struct drm_i915_private *i915, int type)
726 {
727 switch (type) {
728 case I915_CACHE_NONE: return " uncached";
729 case I915_CACHE_LLC: return HAS_LLC(i915) ? " LLC" : " snooped";
730 case I915_CACHE_L3_LLC: return " L3+LLC";
731 case I915_CACHE_WT: return " WT";
732 default: return "";
733 }
734 }
735
736 static inline uint32_t
737 read_subslice_reg(struct drm_i915_private *dev_priv, int slice,
738 int subslice, i915_reg_t reg)
739 {
740 uint32_t mcr;
741 uint32_t ret;
742 enum forcewake_domains fw_domains;
743
744 fw_domains = intel_uncore_forcewake_for_reg(dev_priv, reg,
745 FW_REG_READ);
746 fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
747 GEN8_MCR_SELECTOR,
748 FW_REG_READ | FW_REG_WRITE);
749
750 spin_lock_irq(&dev_priv->uncore.lock);
751 intel_uncore_forcewake_get__locked(dev_priv, fw_domains);
752
753 mcr = I915_READ_FW(GEN8_MCR_SELECTOR);
754 /*
755 * The HW expects the slice and sublice selectors to be reset to 0
756 * after reading out the registers.
757 */
758 WARN_ON_ONCE(mcr & (GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK));
759 mcr &= ~(GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK);
760 mcr |= GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice);
761 I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);
762
763 ret = I915_READ_FW(reg);
764
765 mcr &= ~(GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK);
766 I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);
767
768 intel_uncore_forcewake_put__locked(dev_priv, fw_domains);
769 spin_unlock_irq(&dev_priv->uncore.lock);
770
771 return ret;
772 }
773
774 /* NB: please notice the memset */
775 void intel_engine_get_instdone(struct intel_engine_cs *engine,
776 struct intel_instdone *instdone)
777 {
778 struct drm_i915_private *dev_priv = engine->i915;
779 u32 mmio_base = engine->mmio_base;
780 int slice;
781 int subslice;
782
783 memset(instdone, 0, sizeof(*instdone));
784
785 switch (INTEL_GEN(dev_priv)) {
786 default:
787 instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
788
789 if (engine->id != RCS)
790 break;
791
792 instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
793 for_each_instdone_slice_subslice(dev_priv, slice, subslice) {
794 instdone->sampler[slice][subslice] =
795 read_subslice_reg(dev_priv, slice, subslice,
796 GEN7_SAMPLER_INSTDONE);
797 instdone->row[slice][subslice] =
798 read_subslice_reg(dev_priv, slice, subslice,
799 GEN7_ROW_INSTDONE);
800 }
801 break;
802 case 7:
803 instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
804
805 if (engine->id != RCS)
806 break;
807
808 instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
809 instdone->sampler[0][0] = I915_READ(GEN7_SAMPLER_INSTDONE);
810 instdone->row[0][0] = I915_READ(GEN7_ROW_INSTDONE);
811
812 break;
813 case 6:
814 case 5:
815 case 4:
816 instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
817
818 if (engine->id == RCS)
819 /* HACK: Using the wrong struct member */
820 instdone->slice_common = I915_READ(GEN4_INSTDONE1);
821 break;
822 case 3:
823 case 2:
824 instdone->instdone = I915_READ(GEN2_INSTDONE);
825 break;
826 }
827 }
828
829 static int wa_add(struct drm_i915_private *dev_priv,
830 i915_reg_t addr,
831 const u32 mask, const u32 val)
832 {
833 const u32 idx = dev_priv->workarounds.count;
834
835 if (WARN_ON(idx >= I915_MAX_WA_REGS))
836 return -ENOSPC;
837
838 dev_priv->workarounds.reg[idx].addr = addr;
839 dev_priv->workarounds.reg[idx].value = val;
840 dev_priv->workarounds.reg[idx].mask = mask;
841
842 dev_priv->workarounds.count++;
843
844 return 0;
845 }
846
847 #define WA_REG(addr, mask, val) do { \
848 const int r = wa_add(dev_priv, (addr), (mask), (val)); \
849 if (r) \
850 return r; \
851 } while (0)
852
853 #define WA_SET_BIT_MASKED(addr, mask) \
854 WA_REG(addr, (mask), _MASKED_BIT_ENABLE(mask))
855
856 #define WA_CLR_BIT_MASKED(addr, mask) \
857 WA_REG(addr, (mask), _MASKED_BIT_DISABLE(mask))
858
859 #define WA_SET_FIELD_MASKED(addr, mask, value) \
860 WA_REG(addr, mask, _MASKED_FIELD(mask, value))
861
862 static int wa_ring_whitelist_reg(struct intel_engine_cs *engine,
863 i915_reg_t reg)
864 {
865 struct drm_i915_private *dev_priv = engine->i915;
866 struct i915_workarounds *wa = &dev_priv->workarounds;
867 const uint32_t index = wa->hw_whitelist_count[engine->id];
868
869 if (WARN_ON(index >= RING_MAX_NONPRIV_SLOTS))
870 return -EINVAL;
871
872 I915_WRITE(RING_FORCE_TO_NONPRIV(engine->mmio_base, index),
873 i915_mmio_reg_offset(reg));
874 wa->hw_whitelist_count[engine->id]++;
875
876 return 0;
877 }
878
879 static int gen8_init_workarounds(struct intel_engine_cs *engine)
880 {
881 struct drm_i915_private *dev_priv = engine->i915;
882
883 WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
884
885 /* WaDisableAsyncFlipPerfMode:bdw,chv */
886 WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
887
888 /* WaDisablePartialInstShootdown:bdw,chv */
889 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
890 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
891
892 /* Use Force Non-Coherent whenever executing a 3D context. This is a
893 * workaround for for a possible hang in the unlikely event a TLB
894 * invalidation occurs during a PSD flush.
895 */
896 /* WaForceEnableNonCoherent:bdw,chv */
897 /* WaHdcDisableFetchWhenMasked:bdw,chv */
898 WA_SET_BIT_MASKED(HDC_CHICKEN0,
899 HDC_DONOT_FETCH_MEM_WHEN_MASKED |
900 HDC_FORCE_NON_COHERENT);
901
902 /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
903 * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
904 * polygons in the same 8x4 pixel/sample area to be processed without
905 * stalling waiting for the earlier ones to write to Hierarchical Z
906 * buffer."
907 *
908 * This optimization is off by default for BDW and CHV; turn it on.
909 */
910 WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
911
912 /* Wa4x4STCOptimizationDisable:bdw,chv */
913 WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
914
915 /*
916 * BSpec recommends 8x4 when MSAA is used,
917 * however in practice 16x4 seems fastest.
918 *
919 * Note that PS/WM thread counts depend on the WIZ hashing
920 * disable bit, which we don't touch here, but it's good
921 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
922 */
923 WA_SET_FIELD_MASKED(GEN7_GT_MODE,
924 GEN6_WIZ_HASHING_MASK,
925 GEN6_WIZ_HASHING_16x4);
926
927 return 0;
928 }
929
930 static int bdw_init_workarounds(struct intel_engine_cs *engine)
931 {
932 struct drm_i915_private *dev_priv = engine->i915;
933 int ret;
934
935 ret = gen8_init_workarounds(engine);
936 if (ret)
937 return ret;
938
939 /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
940 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
941
942 /* WaDisableDopClockGating:bdw
943 *
944 * Also see the related UCGTCL1 write in broadwell_init_clock_gating()
945 * to disable EUTC clock gating.
946 */
947 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
948 DOP_CLOCK_GATING_DISABLE);
949
950 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
951 GEN8_SAMPLER_POWER_BYPASS_DIS);
952
953 WA_SET_BIT_MASKED(HDC_CHICKEN0,
954 /* WaForceContextSaveRestoreNonCoherent:bdw */
955 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
956 /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
957 (IS_BDW_GT3(dev_priv) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
958
959 return 0;
960 }
961
962 static int chv_init_workarounds(struct intel_engine_cs *engine)
963 {
964 struct drm_i915_private *dev_priv = engine->i915;
965 int ret;
966
967 ret = gen8_init_workarounds(engine);
968 if (ret)
969 return ret;
970
971 /* WaDisableThreadStallDopClockGating:chv */
972 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
973
974 /* Improve HiZ throughput on CHV. */
975 WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
976
977 return 0;
978 }
979
980 static int gen9_init_workarounds(struct intel_engine_cs *engine)
981 {
982 struct drm_i915_private *dev_priv = engine->i915;
983 int ret;
984
985 /* WaConextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
986 I915_WRITE(GEN9_CSFE_CHICKEN1_RCS, _MASKED_BIT_ENABLE(GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE));
987
988 /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
989 I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) |
990 GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
991
992 /* WaDisableKillLogic:bxt,skl,kbl */
993 if (!IS_COFFEELAKE(dev_priv))
994 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
995 ECOCHK_DIS_TLB);
996
997 if (HAS_LLC(dev_priv)) {
998 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
999 *
1000 * Must match Display Engine. See
1001 * WaCompressedResourceDisplayNewHashMode.
1002 */
1003 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1004 GEN9_PBE_COMPRESSED_HASH_SELECTION);
1005 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
1006 GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
1007
1008 I915_WRITE(MMCD_MISC_CTRL,
1009 I915_READ(MMCD_MISC_CTRL) |
1010 MMCD_PCLA |
1011 MMCD_HOTSPOT_EN);
1012 }
1013
1014 /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
1015 /* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
1016 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
1017 FLOW_CONTROL_ENABLE |
1018 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
1019
1020 /* Syncing dependencies between camera and graphics:skl,bxt,kbl */
1021 if (!IS_COFFEELAKE(dev_priv))
1022 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
1023 GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
1024
1025 /* WaDisableDgMirrorFixInHalfSliceChicken5:bxt */
1026 if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
1027 WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
1028 GEN9_DG_MIRROR_FIX_ENABLE);
1029
1030 /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:bxt */
1031 if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
1032 WA_SET_BIT_MASKED(GEN7_COMMON_SLICE_CHICKEN1,
1033 GEN9_RHWO_OPTIMIZATION_DISABLE);
1034 /*
1035 * WA also requires GEN9_SLICE_COMMON_ECO_CHICKEN0[14:14] to be set
1036 * but we do that in per ctx batchbuffer as there is an issue
1037 * with this register not getting restored on ctx restore
1038 */
1039 }
1040
1041 /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
1042 /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
1043 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
1044 GEN9_ENABLE_YV12_BUGFIX |
1045 GEN9_ENABLE_GPGPU_PREEMPTION);
1046
1047 /* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
1048 /* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
1049 WA_SET_BIT_MASKED(CACHE_MODE_1, (GEN8_4x4_STC_OPTIMIZATION_DISABLE |
1050 GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE));
1051
1052 /* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
1053 WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
1054 GEN9_CCS_TLB_PREFETCH_ENABLE);
1055
1056 /* WaDisableMaskBasedCammingInRCC:bxt */
1057 if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
1058 WA_SET_BIT_MASKED(SLICE_ECO_CHICKEN0,
1059 PIXEL_MASK_CAMMING_DISABLE);
1060
1061 /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
1062 WA_SET_BIT_MASKED(HDC_CHICKEN0,
1063 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
1064 HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
1065
1066 /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
1067 * both tied to WaForceContextSaveRestoreNonCoherent
1068 * in some hsds for skl. We keep the tie for all gen9. The
1069 * documentation is a bit hazy and so we want to get common behaviour,
1070 * even though there is no clear evidence we would need both on kbl/bxt.
1071 * This area has been source of system hangs so we play it safe
1072 * and mimic the skl regardless of what bspec says.
1073 *
1074 * Use Force Non-Coherent whenever executing a 3D context. This
1075 * is a workaround for a possible hang in the unlikely event
1076 * a TLB invalidation occurs during a PSD flush.
1077 */
1078
1079 /* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
1080 WA_SET_BIT_MASKED(HDC_CHICKEN0,
1081 HDC_FORCE_NON_COHERENT);
1082
1083 /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
1084 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
1085 BDW_DISABLE_HDC_INVALIDATION);
1086
1087 /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
1088 if (IS_SKYLAKE(dev_priv) ||
1089 IS_KABYLAKE(dev_priv) ||
1090 IS_COFFEELAKE(dev_priv) ||
1091 IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0))
1092 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
1093 GEN8_SAMPLER_POWER_BYPASS_DIS);
1094
1095 /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
1096 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
1097
1098 /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
1099 I915_WRITE(GEN8_L3SQCREG4, (I915_READ(GEN8_L3SQCREG4) |
1100 GEN8_LQSC_FLUSH_COHERENT_LINES));
1101
1102 /*
1103 * Supporting preemption with fine-granularity requires changes in the
1104 * batch buffer programming. Since we can't break old userspace, we
1105 * need to set our default preemption level to safe value. Userspace is
1106 * still able to use more fine-grained preemption levels, since in
1107 * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
1108 * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
1109 * not real HW workarounds, but merely a way to start using preemption
1110 * while maintaining old contract with userspace.
1111 */
1112
1113 /* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
1114 WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
1115
1116 /* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
1117 WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_GPGPU_LEVEL_MASK,
1118 GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
1119
1120 /* WaClearHIZ_WM_CHICKEN3:bxt,glk */
1121 if (IS_GEN9_LP(dev_priv))
1122 WA_SET_BIT_MASKED(GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ);
1123
1124 /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
1125 ret = wa_ring_whitelist_reg(engine, GEN9_CTX_PREEMPT_REG);
1126 if (ret)
1127 return ret;
1128
1129 /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
1130 I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
1131 _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
1132 ret = wa_ring_whitelist_reg(engine, GEN8_CS_CHICKEN1);
1133 if (ret)
1134 return ret;
1135
1136 /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
1137 ret = wa_ring_whitelist_reg(engine, GEN8_HDC_CHICKEN1);
1138 if (ret)
1139 return ret;
1140
1141 return 0;
1142 }
1143
1144 static int skl_tune_iz_hashing(struct intel_engine_cs *engine)
1145 {
1146 struct drm_i915_private *dev_priv = engine->i915;
1147 u8 vals[3] = { 0, 0, 0 };
1148 unsigned int i;
1149
1150 for (i = 0; i < 3; i++) {
1151 u8 ss;
1152
1153 /*
1154 * Only consider slices where one, and only one, subslice has 7
1155 * EUs
1156 */
1157 if (!is_power_of_2(INTEL_INFO(dev_priv)->sseu.subslice_7eu[i]))
1158 continue;
1159
1160 /*
1161 * subslice_7eu[i] != 0 (because of the check above) and
1162 * ss_max == 4 (maximum number of subslices possible per slice)
1163 *
1164 * -> 0 <= ss <= 3;
1165 */
1166 ss = ffs(INTEL_INFO(dev_priv)->sseu.subslice_7eu[i]) - 1;
1167 vals[i] = 3 - ss;
1168 }
1169
1170 if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
1171 return 0;
1172
1173 /* Tune IZ hashing. See intel_device_info_runtime_init() */
1174 WA_SET_FIELD_MASKED(GEN7_GT_MODE,
1175 GEN9_IZ_HASHING_MASK(2) |
1176 GEN9_IZ_HASHING_MASK(1) |
1177 GEN9_IZ_HASHING_MASK(0),
1178 GEN9_IZ_HASHING(2, vals[2]) |
1179 GEN9_IZ_HASHING(1, vals[1]) |
1180 GEN9_IZ_HASHING(0, vals[0]));
1181
1182 return 0;
1183 }
1184
1185 static int skl_init_workarounds(struct intel_engine_cs *engine)
1186 {
1187 struct drm_i915_private *dev_priv = engine->i915;
1188 int ret;
1189
1190 ret = gen9_init_workarounds(engine);
1191 if (ret)
1192 return ret;
1193
1194 /* WaEnableGapsTsvCreditFix:skl */
1195 I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
1196 GEN9_GAPS_TSV_CREDIT_DISABLE));
1197
1198 /* WaDisableGafsUnitClkGating:skl */
1199 I915_WRITE(GEN7_UCGCTL4, (I915_READ(GEN7_UCGCTL4) |
1200 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE));
1201
1202 /* WaInPlaceDecompressionHang:skl */
1203 if (IS_SKL_REVID(dev_priv, SKL_REVID_H0, REVID_FOREVER))
1204 I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
1205 (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
1206 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
1207
1208 /* WaDisableLSQCROPERFforOCL:skl */
1209 ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
1210 if (ret)
1211 return ret;
1212
1213 return skl_tune_iz_hashing(engine);
1214 }
1215
1216 static int bxt_init_workarounds(struct intel_engine_cs *engine)
1217 {
1218 struct drm_i915_private *dev_priv = engine->i915;
1219 int ret;
1220
1221 ret = gen9_init_workarounds(engine);
1222 if (ret)
1223 return ret;
1224
1225 /* WaStoreMultiplePTEenable:bxt */
1226 /* This is a requirement according to Hardware specification */
1227 if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
1228 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_TLBPF);
1229
1230 /* WaSetClckGatingDisableMedia:bxt */
1231 if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
1232 I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) &
1233 ~GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE));
1234 }
1235
1236 /* WaDisableThreadStallDopClockGating:bxt */
1237 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
1238 STALL_DOP_GATING_DISABLE);
1239
1240 /* WaDisablePooledEuLoadBalancingFix:bxt */
1241 if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER)) {
1242 I915_WRITE(FF_SLICE_CS_CHICKEN2,
1243 _MASKED_BIT_ENABLE(GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE));
1244 }
1245
1246 /* WaDisableSbeCacheDispatchPortSharing:bxt */
1247 if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0)) {
1248 WA_SET_BIT_MASKED(
1249 GEN7_HALF_SLICE_CHICKEN1,
1250 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
1251 }
1252
1253 /* WaDisableObjectLevelPreemptionForTrifanOrPolygon:bxt */
1254 /* WaDisableObjectLevelPreemptionForInstancedDraw:bxt */
1255 /* WaDisableObjectLevelPreemtionForInstanceId:bxt */
1256 /* WaDisableLSQCROPERFforOCL:bxt */
1257 if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
1258 ret = wa_ring_whitelist_reg(engine, GEN9_CS_DEBUG_MODE1);
1259 if (ret)
1260 return ret;
1261
1262 ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
1263 if (ret)
1264 return ret;
1265 }
1266
1267 /* WaProgramL3SqcReg1DefaultForPerf:bxt */
1268 if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER)) {
1269 u32 val = I915_READ(GEN8_L3SQCREG1);
1270 val &= ~L3_PRIO_CREDITS_MASK;
1271 val |= L3_GENERAL_PRIO_CREDITS(62) | L3_HIGH_PRIO_CREDITS(2);
1272 I915_WRITE(GEN8_L3SQCREG1, val);
1273 }
1274
1275 /* WaToEnableHwFixForPushConstHWBug:bxt */
1276 if (IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER))
1277 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1278 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1279
1280 /* WaInPlaceDecompressionHang:bxt */
1281 if (IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER))
1282 I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
1283 (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
1284 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
1285
1286 return 0;
1287 }
1288
1289 static int cnl_init_workarounds(struct intel_engine_cs *engine)
1290 {
1291 struct drm_i915_private *dev_priv = engine->i915;
1292 int ret;
1293
1294 /* WaDisableI2mCycleOnWRPort:cnl (pre-prod) */
1295 if (IS_CNL_REVID(dev_priv, CNL_REVID_B0, CNL_REVID_B0))
1296 I915_WRITE(GAMT_CHKN_BIT_REG,
1297 (I915_READ(GAMT_CHKN_BIT_REG) |
1298 GAMT_CHKN_DISABLE_I2M_CYCLE_ON_WR_PORT));
1299
1300 /* WaForceContextSaveRestoreNonCoherent:cnl */
1301 WA_SET_BIT_MASKED(CNL_HDC_CHICKEN0,
1302 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT);
1303
1304 /* WaThrottleEUPerfToAvoidTDBackPressure:cnl(pre-prod) */
1305 if (IS_CNL_REVID(dev_priv, CNL_REVID_B0, CNL_REVID_B0))
1306 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, THROTTLE_12_5);
1307
1308 /* WaDisableReplayBufferBankArbitrationOptimization:cnl */
1309 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1310 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1311
1312 /* WaDisableEnhancedSBEVertexCaching:cnl (pre-prod) */
1313 if (IS_CNL_REVID(dev_priv, 0, CNL_REVID_B0))
1314 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1315 GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE);
1316
1317 /* WaInPlaceDecompressionHang:cnl */
1318 I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
1319 (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
1320 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
1321
1322 /* WaPushConstantDereferenceHoldDisable:cnl */
1323 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE);
1324
1325 /* FtrEnableFastAnisoL1BankingFix: cnl */
1326 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, CNL_FAST_ANISO_L1_BANKING_FIX);
1327
1328 /* WaDisable3DMidCmdPreemption:cnl */
1329 WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
1330
1331 /* WaDisableGPGPUMidCmdPreemption:cnl */
1332 WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_GPGPU_LEVEL_MASK,
1333 GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
1334
1335 /* WaEnablePreemptionGranularityControlByUMD:cnl */
1336 I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
1337 _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
1338 ret= wa_ring_whitelist_reg(engine, GEN8_CS_CHICKEN1);
1339 if (ret)
1340 return ret;
1341
1342 return 0;
1343 }
1344
1345 static int kbl_init_workarounds(struct intel_engine_cs *engine)
1346 {
1347 struct drm_i915_private *dev_priv = engine->i915;
1348 int ret;
1349
1350 ret = gen9_init_workarounds(engine);
1351 if (ret)
1352 return ret;
1353
1354 /* WaEnableGapsTsvCreditFix:kbl */
1355 I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
1356 GEN9_GAPS_TSV_CREDIT_DISABLE));
1357
1358 /* WaDisableDynamicCreditSharing:kbl */
1359 if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
1360 I915_WRITE(GAMT_CHKN_BIT_REG,
1361 (I915_READ(GAMT_CHKN_BIT_REG) |
1362 GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING));
1363
1364 /* WaDisableFenceDestinationToSLM:kbl (pre-prod) */
1365 if (IS_KBL_REVID(dev_priv, KBL_REVID_A0, KBL_REVID_A0))
1366 WA_SET_BIT_MASKED(HDC_CHICKEN0,
1367 HDC_FENCE_DEST_SLM_DISABLE);
1368
1369 /* WaToEnableHwFixForPushConstHWBug:kbl */
1370 if (IS_KBL_REVID(dev_priv, KBL_REVID_C0, REVID_FOREVER))
1371 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1372 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1373
1374 /* WaDisableGafsUnitClkGating:kbl */
1375 I915_WRITE(GEN7_UCGCTL4, (I915_READ(GEN7_UCGCTL4) |
1376 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE));
1377
1378 /* WaDisableSbeCacheDispatchPortSharing:kbl */
1379 WA_SET_BIT_MASKED(
1380 GEN7_HALF_SLICE_CHICKEN1,
1381 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
1382
1383 /* WaInPlaceDecompressionHang:kbl */
1384 I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
1385 (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
1386 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
1387
1388 /* WaDisableLSQCROPERFforOCL:kbl */
1389 ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
1390 if (ret)
1391 return ret;
1392
1393 return 0;
1394 }
1395
1396 static int glk_init_workarounds(struct intel_engine_cs *engine)
1397 {
1398 struct drm_i915_private *dev_priv = engine->i915;
1399 int ret;
1400
1401 ret = gen9_init_workarounds(engine);
1402 if (ret)
1403 return ret;
1404
1405 /* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
1406 ret = wa_ring_whitelist_reg(engine, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1407 if (ret)
1408 return ret;
1409
1410 /* WaToEnableHwFixForPushConstHWBug:glk */
1411 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1412 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1413
1414 return 0;
1415 }
1416
1417 static int cfl_init_workarounds(struct intel_engine_cs *engine)
1418 {
1419 struct drm_i915_private *dev_priv = engine->i915;
1420 int ret;
1421
1422 ret = gen9_init_workarounds(engine);
1423 if (ret)
1424 return ret;
1425
1426 /* WaEnableGapsTsvCreditFix:cfl */
1427 I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
1428 GEN9_GAPS_TSV_CREDIT_DISABLE));
1429
1430 /* WaToEnableHwFixForPushConstHWBug:cfl */
1431 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1432 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1433
1434 /* WaDisableGafsUnitClkGating:cfl */
1435 I915_WRITE(GEN7_UCGCTL4, (I915_READ(GEN7_UCGCTL4) |
1436 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE));
1437
1438 /* WaDisableSbeCacheDispatchPortSharing:cfl */
1439 WA_SET_BIT_MASKED(
1440 GEN7_HALF_SLICE_CHICKEN1,
1441 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
1442
1443 /* WaInPlaceDecompressionHang:cfl */
1444 I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
1445 (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
1446 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
1447
1448 return 0;
1449 }
1450
1451 int init_workarounds_ring(struct intel_engine_cs *engine)
1452 {
1453 struct drm_i915_private *dev_priv = engine->i915;
1454 int err;
1455
1456 WARN_ON(engine->id != RCS);
1457
1458 dev_priv->workarounds.count = 0;
1459 dev_priv->workarounds.hw_whitelist_count[engine->id] = 0;
1460
1461 if (IS_BROADWELL(dev_priv))
1462 err = bdw_init_workarounds(engine);
1463 else if (IS_CHERRYVIEW(dev_priv))
1464 err = chv_init_workarounds(engine);
1465 else if (IS_SKYLAKE(dev_priv))
1466 err = skl_init_workarounds(engine);
1467 else if (IS_BROXTON(dev_priv))
1468 err = bxt_init_workarounds(engine);
1469 else if (IS_KABYLAKE(dev_priv))
1470 err = kbl_init_workarounds(engine);
1471 else if (IS_GEMINILAKE(dev_priv))
1472 err = glk_init_workarounds(engine);
1473 else if (IS_COFFEELAKE(dev_priv))
1474 err = cfl_init_workarounds(engine);
1475 else if (IS_CANNONLAKE(dev_priv))
1476 err = cnl_init_workarounds(engine);
1477 else
1478 err = 0;
1479 if (err)
1480 return err;
1481
1482 DRM_DEBUG_DRIVER("%s: Number of context specific w/a: %d\n",
1483 engine->name, dev_priv->workarounds.count);
1484 return 0;
1485 }
1486
1487 int intel_ring_workarounds_emit(struct drm_i915_gem_request *req)
1488 {
1489 struct i915_workarounds *w = &req->i915->workarounds;
1490 u32 *cs;
1491 int ret, i;
1492
1493 if (w->count == 0)
1494 return 0;
1495
1496 ret = req->engine->emit_flush(req, EMIT_BARRIER);
1497 if (ret)
1498 return ret;
1499
1500 cs = intel_ring_begin(req, (w->count * 2 + 2));
1501 if (IS_ERR(cs))
1502 return PTR_ERR(cs);
1503
1504 *cs++ = MI_LOAD_REGISTER_IMM(w->count);
1505 for (i = 0; i < w->count; i++) {
1506 *cs++ = i915_mmio_reg_offset(w->reg[i].addr);
1507 *cs++ = w->reg[i].value;
1508 }
1509 *cs++ = MI_NOOP;
1510
1511 intel_ring_advance(req, cs);
1512
1513 ret = req->engine->emit_flush(req, EMIT_BARRIER);
1514 if (ret)
1515 return ret;
1516
1517 return 0;
1518 }
1519
1520 static bool ring_is_idle(struct intel_engine_cs *engine)
1521 {
1522 struct drm_i915_private *dev_priv = engine->i915;
1523 bool idle = true;
1524
1525 intel_runtime_pm_get(dev_priv);
1526
1527 /* First check that no commands are left in the ring */
1528 if ((I915_READ_HEAD(engine) & HEAD_ADDR) !=
1529 (I915_READ_TAIL(engine) & TAIL_ADDR))
1530 idle = false;
1531
1532 /* No bit for gen2, so assume the CS parser is idle */
1533 if (INTEL_GEN(dev_priv) > 2 && !(I915_READ_MODE(engine) & MODE_IDLE))
1534 idle = false;
1535
1536 intel_runtime_pm_put(dev_priv);
1537
1538 return idle;
1539 }
1540
1541 /**
1542 * intel_engine_is_idle() - Report if the engine has finished process all work
1543 * @engine: the intel_engine_cs
1544 *
1545 * Return true if there are no requests pending, nothing left to be submitted
1546 * to hardware, and that the engine is idle.
1547 */
1548 bool intel_engine_is_idle(struct intel_engine_cs *engine)
1549 {
1550 struct drm_i915_private *dev_priv = engine->i915;
1551
1552 /* More white lies, if wedged, hw state is inconsistent */
1553 if (i915_terminally_wedged(&dev_priv->gpu_error))
1554 return true;
1555
1556 /* Any inflight/incomplete requests? */
1557 if (!i915_seqno_passed(intel_engine_get_seqno(engine),
1558 intel_engine_last_submit(engine)))
1559 return false;
1560
1561 if (I915_SELFTEST_ONLY(engine->breadcrumbs.mock))
1562 return true;
1563
1564 /* Interrupt/tasklet pending? */
1565 if (test_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted))
1566 return false;
1567
1568 /* Waiting to drain ELSP? */
1569 if (READ_ONCE(engine->execlists.active))
1570 return false;
1571
1572 /* ELSP is empty, but there are ready requests? */
1573 if (READ_ONCE(engine->execlists.first))
1574 return false;
1575
1576 /* Ring stopped? */
1577 if (!ring_is_idle(engine))
1578 return false;
1579
1580 return true;
1581 }
1582
1583 bool intel_engines_are_idle(struct drm_i915_private *dev_priv)
1584 {
1585 struct intel_engine_cs *engine;
1586 enum intel_engine_id id;
1587
1588 if (READ_ONCE(dev_priv->gt.active_requests))
1589 return false;
1590
1591 /* If the driver is wedged, HW state may be very inconsistent and
1592 * report that it is still busy, even though we have stopped using it.
1593 */
1594 if (i915_terminally_wedged(&dev_priv->gpu_error))
1595 return true;
1596
1597 for_each_engine(engine, dev_priv, id) {
1598 if (!intel_engine_is_idle(engine))
1599 return false;
1600 }
1601
1602 return true;
1603 }
1604
1605 /**
1606 * intel_engine_has_kernel_context:
1607 * @engine: the engine
1608 *
1609 * Returns true if the last context to be executed on this engine, or has been
1610 * executed if the engine is already idle, is the kernel context
1611 * (#i915.kernel_context).
1612 */
1613 bool intel_engine_has_kernel_context(const struct intel_engine_cs *engine)
1614 {
1615 const struct i915_gem_context * const kernel_context =
1616 engine->i915->kernel_context;
1617 struct drm_i915_gem_request *rq;
1618
1619 lockdep_assert_held(&engine->i915->drm.struct_mutex);
1620
1621 /*
1622 * Check the last context seen by the engine. If active, it will be
1623 * the last request that remains in the timeline. When idle, it is
1624 * the last executed context as tracked by retirement.
1625 */
1626 rq = __i915_gem_active_peek(&engine->timeline->last_request);
1627 if (rq)
1628 return rq->ctx == kernel_context;
1629 else
1630 return engine->last_retired_context == kernel_context;
1631 }
1632
1633 void intel_engines_reset_default_submission(struct drm_i915_private *i915)
1634 {
1635 struct intel_engine_cs *engine;
1636 enum intel_engine_id id;
1637
1638 for_each_engine(engine, i915, id)
1639 engine->set_default_submission(engine);
1640 }
1641
1642 void intel_engines_mark_idle(struct drm_i915_private *i915)
1643 {
1644 struct intel_engine_cs *engine;
1645 enum intel_engine_id id;
1646
1647 for_each_engine(engine, i915, id) {
1648 intel_engine_disarm_breadcrumbs(engine);
1649 i915_gem_batch_pool_fini(&engine->batch_pool);
1650 tasklet_kill(&engine->execlists.irq_tasklet);
1651 engine->execlists.no_priolist = false;
1652 }
1653 }
1654
1655 bool intel_engine_can_store_dword(struct intel_engine_cs *engine)
1656 {
1657 switch (INTEL_GEN(engine->i915)) {
1658 case 2:
1659 return false; /* uses physical not virtual addresses */
1660 case 3:
1661 /* maybe only uses physical not virtual addresses */
1662 return !(IS_I915G(engine->i915) || IS_I915GM(engine->i915));
1663 case 6:
1664 return engine->class != VIDEO_DECODE_CLASS; /* b0rked */
1665 default:
1666 return true;
1667 }
1668 }
1669
1670 static void print_request(struct drm_printer *m,
1671 struct drm_i915_gem_request *rq,
1672 const char *prefix)
1673 {
1674 drm_printf(m, "%s%x%s [%x:%x] prio=%d @ %dms: %s\n", prefix,
1675 rq->global_seqno,
1676 i915_gem_request_completed(rq) ? "!" : "",
1677 rq->ctx->hw_id, rq->fence.seqno,
1678 rq->priotree.priority,
1679 jiffies_to_msecs(jiffies - rq->emitted_jiffies),
1680 rq->timeline->common->name);
1681 }
1682
1683 void intel_engine_dump(struct intel_engine_cs *engine, struct drm_printer *m)
1684 {
1685 struct intel_breadcrumbs * const b = &engine->breadcrumbs;
1686 const struct intel_engine_execlists * const execlists = &engine->execlists;
1687 struct i915_gpu_error * const error = &engine->i915->gpu_error;
1688 struct drm_i915_private *dev_priv = engine->i915;
1689 struct drm_i915_gem_request *rq;
1690 struct rb_node *rb;
1691 u64 addr;
1692
1693 drm_printf(m, "%s\n", engine->name);
1694 drm_printf(m, "\tcurrent seqno %x, last %x, hangcheck %x [%d ms], inflight %d\n",
1695 intel_engine_get_seqno(engine),
1696 intel_engine_last_submit(engine),
1697 engine->hangcheck.seqno,
1698 jiffies_to_msecs(jiffies - engine->hangcheck.action_timestamp),
1699 engine->timeline->inflight_seqnos);
1700 drm_printf(m, "\tReset count: %d\n",
1701 i915_reset_engine_count(error, engine));
1702
1703 rcu_read_lock();
1704
1705 drm_printf(m, "\tRequests:\n");
1706
1707 rq = list_first_entry(&engine->timeline->requests,
1708 struct drm_i915_gem_request, link);
1709 if (&rq->link != &engine->timeline->requests)
1710 print_request(m, rq, "\t\tfirst ");
1711
1712 rq = list_last_entry(&engine->timeline->requests,
1713 struct drm_i915_gem_request, link);
1714 if (&rq->link != &engine->timeline->requests)
1715 print_request(m, rq, "\t\tlast ");
1716
1717 rq = i915_gem_find_active_request(engine);
1718 if (rq) {
1719 print_request(m, rq, "\t\tactive ");
1720 drm_printf(m,
1721 "\t\t[head %04x, postfix %04x, tail %04x, batch 0x%08x_%08x]\n",
1722 rq->head, rq->postfix, rq->tail,
1723 rq->batch ? upper_32_bits(rq->batch->node.start) : ~0u,
1724 rq->batch ? lower_32_bits(rq->batch->node.start) : ~0u);
1725 }
1726
1727 drm_printf(m, "\tRING_START: 0x%08x [0x%08x]\n",
1728 I915_READ(RING_START(engine->mmio_base)),
1729 rq ? i915_ggtt_offset(rq->ring->vma) : 0);
1730 drm_printf(m, "\tRING_HEAD: 0x%08x [0x%08x]\n",
1731 I915_READ(RING_HEAD(engine->mmio_base)) & HEAD_ADDR,
1732 rq ? rq->ring->head : 0);
1733 drm_printf(m, "\tRING_TAIL: 0x%08x [0x%08x]\n",
1734 I915_READ(RING_TAIL(engine->mmio_base)) & TAIL_ADDR,
1735 rq ? rq->ring->tail : 0);
1736 drm_printf(m, "\tRING_CTL: 0x%08x [%s]\n",
1737 I915_READ(RING_CTL(engine->mmio_base)),
1738 I915_READ(RING_CTL(engine->mmio_base)) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? "waiting" : "");
1739
1740 rcu_read_unlock();
1741
1742 addr = intel_engine_get_active_head(engine);
1743 drm_printf(m, "\tACTHD: 0x%08x_%08x\n",
1744 upper_32_bits(addr), lower_32_bits(addr));
1745 addr = intel_engine_get_last_batch_head(engine);
1746 drm_printf(m, "\tBBADDR: 0x%08x_%08x\n",
1747 upper_32_bits(addr), lower_32_bits(addr));
1748
1749 if (i915_modparams.enable_execlists) {
1750 const u32 *hws = &engine->status_page.page_addr[I915_HWS_CSB_BUF0_INDEX];
1751 u32 ptr, read, write;
1752 unsigned int idx;
1753
1754 drm_printf(m, "\tExeclist status: 0x%08x %08x\n",
1755 I915_READ(RING_EXECLIST_STATUS_LO(engine)),
1756 I915_READ(RING_EXECLIST_STATUS_HI(engine)));
1757
1758 ptr = I915_READ(RING_CONTEXT_STATUS_PTR(engine));
1759 read = GEN8_CSB_READ_PTR(ptr);
1760 write = GEN8_CSB_WRITE_PTR(ptr);
1761 drm_printf(m, "\tExeclist CSB read %d [%d cached], write %d [%d from hws], interrupt posted? %s\n",
1762 read, execlists->csb_head,
1763 write,
1764 intel_read_status_page(engine, intel_hws_csb_write_index(engine->i915)),
1765 yesno(test_bit(ENGINE_IRQ_EXECLIST,
1766 &engine->irq_posted)));
1767 if (read >= GEN8_CSB_ENTRIES)
1768 read = 0;
1769 if (write >= GEN8_CSB_ENTRIES)
1770 write = 0;
1771 if (read > write)
1772 write += GEN8_CSB_ENTRIES;
1773 while (read < write) {
1774 idx = ++read % GEN8_CSB_ENTRIES;
1775 drm_printf(m, "\tExeclist CSB[%d]: 0x%08x [0x%08x in hwsp], context: %d [%d in hwsp]\n",
1776 idx,
1777 I915_READ(RING_CONTEXT_STATUS_BUF_LO(engine, idx)),
1778 hws[idx * 2],
1779 I915_READ(RING_CONTEXT_STATUS_BUF_HI(engine, idx)),
1780 hws[idx * 2 + 1]);
1781 }
1782
1783 rcu_read_lock();
1784 for (idx = 0; idx < execlists_num_ports(execlists); idx++) {
1785 unsigned int count;
1786
1787 rq = port_unpack(&execlists->port[idx], &count);
1788 if (rq) {
1789 drm_printf(m, "\t\tELSP[%d] count=%d, ",
1790 idx, count);
1791 print_request(m, rq, "rq: ");
1792 } else {
1793 drm_printf(m, "\t\tELSP[%d] idle\n",
1794 idx);
1795 }
1796 }
1797 drm_printf(m, "\t\tHW active? 0x%x\n", execlists->active);
1798 rcu_read_unlock();
1799 } else if (INTEL_GEN(dev_priv) > 6) {
1800 drm_printf(m, "\tPP_DIR_BASE: 0x%08x\n",
1801 I915_READ(RING_PP_DIR_BASE(engine)));
1802 drm_printf(m, "\tPP_DIR_BASE_READ: 0x%08x\n",
1803 I915_READ(RING_PP_DIR_BASE_READ(engine)));
1804 drm_printf(m, "\tPP_DIR_DCLV: 0x%08x\n",
1805 I915_READ(RING_PP_DIR_DCLV(engine)));
1806 }
1807
1808 spin_lock_irq(&engine->timeline->lock);
1809 list_for_each_entry(rq, &engine->timeline->requests, link)
1810 print_request(m, rq, "\t\tE ");
1811 for (rb = execlists->first; rb; rb = rb_next(rb)) {
1812 struct i915_priolist *p =
1813 rb_entry(rb, typeof(*p), node);
1814
1815 list_for_each_entry(rq, &p->requests, priotree.link)
1816 print_request(m, rq, "\t\tQ ");
1817 }
1818 spin_unlock_irq(&engine->timeline->lock);
1819
1820 spin_lock_irq(&b->rb_lock);
1821 for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) {
1822 struct intel_wait *w = rb_entry(rb, typeof(*w), node);
1823
1824 drm_printf(m, "\t%s [%d] waiting for %x\n",
1825 w->tsk->comm, w->tsk->pid, w->seqno);
1826 }
1827 spin_unlock_irq(&b->rb_lock);
1828
1829 drm_printf(m, "\n");
1830 }
1831
1832 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1833 #include "selftests/mock_engine.c"
1834 #endif
ubuntu-bionic-kernel mirror