drivers/gpu/drm/i915/intel_engine_cs.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <drm/drm_print.h>
  26
  27 #include "i915_drv.h"
  28 #include "i915_vgpu.h"
  29 #include "intel_ringbuffer.h"
  30 #include "intel_lrc.h"
  31
  32 /* Haswell does have the CXT_SIZE register however it does not appear to be
  33  * valid. Now, docs explain in dwords what is in the context object. The full
  34  * size is 70720 bytes, however, the power context and execlist context will
  35  * never be saved (power context is stored elsewhere, and execlists don't work
  36  * on HSW) - so the final size, including the extra state required for the
  37  * Resource Streamer, is 66944 bytes, which rounds to 17 pages.
  38  */
  39 #define HSW_CXT_TOTAL_SIZE              (17 * PAGE_SIZE)
  40 /* Same as Haswell, but 72064 bytes now. */
  41 #define GEN8_CXT_TOTAL_SIZE             (18 * PAGE_SIZE)
  42
  43 #define GEN8_LR_CONTEXT_RENDER_SIZE     (20 * PAGE_SIZE)
  44 #define GEN9_LR_CONTEXT_RENDER_SIZE     (22 * PAGE_SIZE)
  45 #define GEN10_LR_CONTEXT_RENDER_SIZE    (18 * PAGE_SIZE)
  46
  47 #define GEN8_LR_CONTEXT_OTHER_SIZE      ( 2 * PAGE_SIZE)
  48
  49 struct engine_class_info {
  50         const char *name;
  51         int (*init_legacy)(struct intel_engine_cs *engine);
  52         int (*init_execlists)(struct intel_engine_cs *engine);
  53
  54         u8 uabi_class;
  55 };
  56
  57 static const struct engine_class_info intel_engine_classes[] = {
  58         [RENDER_CLASS] = {
  59                 .name = "rcs",
  60                 .init_execlists = logical_render_ring_init,
  61                 .init_legacy = intel_init_render_ring_buffer,
  62                 .uabi_class = I915_ENGINE_CLASS_RENDER,
  63         },
  64         [COPY_ENGINE_CLASS] = {
  65                 .name = "bcs",
  66                 .init_execlists = logical_xcs_ring_init,
  67                 .init_legacy = intel_init_blt_ring_buffer,
  68                 .uabi_class = I915_ENGINE_CLASS_COPY,
  69         },
  70         [VIDEO_DECODE_CLASS] = {
  71                 .name = "vcs",
  72                 .init_execlists = logical_xcs_ring_init,
  73                 .init_legacy = intel_init_bsd_ring_buffer,
  74                 .uabi_class = I915_ENGINE_CLASS_VIDEO,
  75         },
  76         [VIDEO_ENHANCEMENT_CLASS] = {
  77                 .name = "vecs",
  78                 .init_execlists = logical_xcs_ring_init,
  79                 .init_legacy = intel_init_vebox_ring_buffer,
  80                 .uabi_class = I915_ENGINE_CLASS_VIDEO_ENHANCE,
  81         },
  82 };
  83
  84 struct engine_info {
  85         unsigned int hw_id;
  86         unsigned int uabi_id;
  87         u8 class;
  88         u8 instance;
  89         u32 mmio_base;
  90         unsigned irq_shift;
  91 };
  92
  93 static const struct engine_info intel_engines[] = {
  94         [RCS] = {
  95                 .hw_id = RCS_HW,
  96                 .uabi_id = I915_EXEC_RENDER,
  97                 .class = RENDER_CLASS,
  98                 .instance = 0,
  99                 .mmio_base = RENDER_RING_BASE,
 100                 .irq_shift = GEN8_RCS_IRQ_SHIFT,
 101         },
 102         [BCS] = {
 103                 .hw_id = BCS_HW,
 104                 .uabi_id = I915_EXEC_BLT,
 105                 .class = COPY_ENGINE_CLASS,
 106                 .instance = 0,
 107                 .mmio_base = BLT_RING_BASE,
 108                 .irq_shift = GEN8_BCS_IRQ_SHIFT,
 109         },
 110         [VCS] = {
 111                 .hw_id = VCS_HW,
 112                 .uabi_id = I915_EXEC_BSD,
 113                 .class = VIDEO_DECODE_CLASS,
 114                 .instance = 0,
 115                 .mmio_base = GEN6_BSD_RING_BASE,
 116                 .irq_shift = GEN8_VCS1_IRQ_SHIFT,
 117         },
 118         [VCS2] = {
 119                 .hw_id = VCS2_HW,
 120                 .uabi_id = I915_EXEC_BSD,
 121                 .class = VIDEO_DECODE_CLASS,
 122                 .instance = 1,
 123                 .mmio_base = GEN8_BSD2_RING_BASE,
 124                 .irq_shift = GEN8_VCS2_IRQ_SHIFT,
 125         },
 126         [VECS] = {
 127                 .hw_id = VECS_HW,
 128                 .uabi_id = I915_EXEC_VEBOX,
 129                 .class = VIDEO_ENHANCEMENT_CLASS,
 130                 .instance = 0,
 131                 .mmio_base = VEBOX_RING_BASE,
 132                 .irq_shift = GEN8_VECS_IRQ_SHIFT,
 133         },
 134 };
 135
 136 /**
 137  * ___intel_engine_context_size() - return the size of the context for an engine
 138  * @dev_priv: i915 device private
 139  * @class: engine class
 140  *
 141  * Each engine class may require a different amount of space for a context
 142  * image.
 143  *
 144  * Return: size (in bytes) of an engine class specific context image
 145  *
 146  * Note: this size includes the HWSP, which is part of the context image
 147  * in LRC mode, but does not include the "shared data page" used with
 148  * GuC submission. The caller should account for this if using the GuC.
 149  */
 150 static u32
 151 __intel_engine_context_size(struct drm_i915_private *dev_priv, u8 class)
 152 {
 153         u32 cxt_size;
 154
 155         BUILD_BUG_ON(I915_GTT_PAGE_SIZE != PAGE_SIZE);
 156
 157         switch (class) {
 158         case RENDER_CLASS:
 159                 switch (INTEL_GEN(dev_priv)) {
 160                 default:
 161                         MISSING_CASE(INTEL_GEN(dev_priv));
 162                 case 10:
 163                         return GEN10_LR_CONTEXT_RENDER_SIZE;
 164                 case 9:
 165                         return GEN9_LR_CONTEXT_RENDER_SIZE;
 166                 case 8:
 167                         return i915_modparams.enable_execlists ?
 168                                GEN8_LR_CONTEXT_RENDER_SIZE :
 169                                GEN8_CXT_TOTAL_SIZE;
 170                 case 7:
 171                         if (IS_HASWELL(dev_priv))
 172                                 return HSW_CXT_TOTAL_SIZE;
 173
 174                         cxt_size = I915_READ(GEN7_CXT_SIZE);
 175                         return round_up(GEN7_CXT_TOTAL_SIZE(cxt_size) * 64,
 176                                         PAGE_SIZE);
 177                 case 6:
 178                         cxt_size = I915_READ(CXT_SIZE);
 179                         return round_up(GEN6_CXT_TOTAL_SIZE(cxt_size) * 64,
 180                                         PAGE_SIZE);
 181                 case 5:
 182                 case 4:
 183                 case 3:
 184                 case 2:
 185                 /* For the special day when i810 gets merged. */
 186                 case 1:
 187                         return 0;
 188                 }
 189                 break;
 190         default:
 191                 MISSING_CASE(class);
 192         case VIDEO_DECODE_CLASS:
 193         case VIDEO_ENHANCEMENT_CLASS:
 194         case COPY_ENGINE_CLASS:
 195                 if (INTEL_GEN(dev_priv) < 8)
 196                         return 0;
 197                 return GEN8_LR_CONTEXT_OTHER_SIZE;
 198         }
 199 }
 200
 201 static int
 202 intel_engine_setup(struct drm_i915_private *dev_priv,
 203                    enum intel_engine_id id)
 204 {
 205         const struct engine_info *info = &intel_engines[id];
 206         const struct engine_class_info *class_info;
 207         struct intel_engine_cs *engine;
 208
 209         GEM_BUG_ON(info->class >= ARRAY_SIZE(intel_engine_classes));
 210         class_info = &intel_engine_classes[info->class];
 211
 212         GEM_BUG_ON(dev_priv->engine[id]);
 213         engine = kzalloc(sizeof(*engine), GFP_KERNEL);
 214         if (!engine)
 215                 return -ENOMEM;
 216
 217         engine->id = id;
 218         engine->i915 = dev_priv;
 219         WARN_ON(snprintf(engine->name, sizeof(engine->name), "%s%u",
 220                          class_info->name, info->instance) >=
 221                 sizeof(engine->name));
 222         engine->hw_id = engine->guc_id = info->hw_id;
 223         engine->mmio_base = info->mmio_base;
 224         engine->irq_shift = info->irq_shift;
 225         engine->class = info->class;
 226         engine->instance = info->instance;
 227
 228         engine->uabi_id = info->uabi_id;
 229         engine->uabi_class = class_info->uabi_class;
 230
 231         engine->context_size = __intel_engine_context_size(dev_priv,
 232                                                            engine->class);
 233         if (WARN_ON(engine->context_size > BIT(20)))
 234                 engine->context_size = 0;
 235
 236         /* Nothing to do here, execute in order of dependencies */
 237         engine->schedule = NULL;
 238
 239         ATOMIC_INIT_NOTIFIER_HEAD(&engine->context_status_notifier);
 240
 241         dev_priv->engine[id] = engine;
 242         return 0;
 243 }
 244
 245 /**
 246  * intel_engines_init_mmio() - allocate and prepare the Engine Command Streamers
 247  * @dev_priv: i915 device private
 248  *
 249  * Return: non-zero if the initialization failed.
 250  */
 251 int intel_engines_init_mmio(struct drm_i915_private *dev_priv)
 252 {
 253         struct intel_device_info *device_info = mkwrite_device_info(dev_priv);
 254         const unsigned int ring_mask = INTEL_INFO(dev_priv)->ring_mask;
 255         struct intel_engine_cs *engine;
 256         enum intel_engine_id id;
 257         unsigned int mask = 0;
 258         unsigned int i;
 259         int err;
 260
 261         WARN_ON(ring_mask == 0);
 262         WARN_ON(ring_mask &
 263                 GENMASK(sizeof(mask) * BITS_PER_BYTE - 1, I915_NUM_ENGINES));
 264
 265         for (i = 0; i < ARRAY_SIZE(intel_engines); i++) {
 266                 if (!HAS_ENGINE(dev_priv, i))
 267                         continue;
 268
 269                 err = intel_engine_setup(dev_priv, i);
 270                 if (err)
 271                         goto cleanup;
 272
 273                 mask |= ENGINE_MASK(i);
 274         }
 275
 276         /*
 277          * Catch failures to update intel_engines table when the new engines
 278          * are added to the driver by a warning and disabling the forgotten
 279          * engines.
 280          */
 281         if (WARN_ON(mask != ring_mask))
 282                 device_info->ring_mask = mask;
 283
 284         /* We always presume we have at least RCS available for later probing */
 285         if (WARN_ON(!HAS_ENGINE(dev_priv, RCS))) {
 286                 err = -ENODEV;
 287                 goto cleanup;
 288         }
 289
 290         device_info->num_rings = hweight32(mask);
 291
 292         return 0;
 293
 294 cleanup:
 295         for_each_engine(engine, dev_priv, id)
 296                 kfree(engine);
 297         return err;
 298 }
 299
 300 /**
 301  * intel_engines_init() - init the Engine Command Streamers
 302  * @dev_priv: i915 device private
 303  *
 304  * Return: non-zero if the initialization failed.
 305  */
 306 int intel_engines_init(struct drm_i915_private *dev_priv)
 307 {
 308         struct intel_engine_cs *engine;
 309         enum intel_engine_id id, err_id;
 310         int err;
 311
 312         for_each_engine(engine, dev_priv, id) {
 313                 const struct engine_class_info *class_info =
 314                         &intel_engine_classes[engine->class];
 315                 int (*init)(struct intel_engine_cs *engine);
 316
 317                 if (i915_modparams.enable_execlists)
 318                         init = class_info->init_execlists;
 319                 else
 320                         init = class_info->init_legacy;
 321
 322                 err = -EINVAL;
 323                 err_id = id;
 324
 325                 if (GEM_WARN_ON(!init))
 326                         goto cleanup;
 327
 328                 err = init(engine);
 329                 if (err)
 330                         goto cleanup;
 331
 332                 GEM_BUG_ON(!engine->submit_request);
 333         }
 334
 335         return 0;
 336
 337 cleanup:
 338         for_each_engine(engine, dev_priv, id) {
 339                 if (id >= err_id) {
 340                         kfree(engine);
 341                         dev_priv->engine[id] = NULL;
 342                 } else {
 343                         dev_priv->gt.cleanup_engine(engine);
 344                 }
 345         }
 346         return err;
 347 }
 348
 349 void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno)
 350 {
 351         struct drm_i915_private *dev_priv = engine->i915;
 352
 353         /* Our semaphore implementation is strictly monotonic (i.e. we proceed
 354          * so long as the semaphore value in the register/page is greater
 355          * than the sync value), so whenever we reset the seqno,
 356          * so long as we reset the tracking semaphore value to 0, it will
 357          * always be before the next request's seqno. If we don't reset
 358          * the semaphore value, then when the seqno moves backwards all
 359          * future waits will complete instantly (causing rendering corruption).
 360          */
 361         if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) {
 362                 I915_WRITE(RING_SYNC_0(engine->mmio_base), 0);
 363                 I915_WRITE(RING_SYNC_1(engine->mmio_base), 0);
 364                 if (HAS_VEBOX(dev_priv))
 365                         I915_WRITE(RING_SYNC_2(engine->mmio_base), 0);
 366         }
 367         if (dev_priv->semaphore) {
 368                 struct page *page = i915_vma_first_page(dev_priv->semaphore);
 369                 void *semaphores;
 370
 371                 /* Semaphores are in noncoherent memory, flush to be safe */
 372                 semaphores = kmap_atomic(page);
 373                 memset(semaphores + GEN8_SEMAPHORE_OFFSET(engine->id, 0),
 374                        0, I915_NUM_ENGINES * gen8_semaphore_seqno_size);
 375                 drm_clflush_virt_range(semaphores + GEN8_SEMAPHORE_OFFSET(engine->id, 0),
 376                                        I915_NUM_ENGINES * gen8_semaphore_seqno_size);
 377                 kunmap_atomic(semaphores);
 378         }
 379
 380         intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
 381         clear_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted);
 382
 383         /* After manually advancing the seqno, fake the interrupt in case
 384          * there are any waiters for that seqno.
 385          */
 386         intel_engine_wakeup(engine);
 387
 388         GEM_BUG_ON(intel_engine_get_seqno(engine) != seqno);
 389 }
 390
 391 static void intel_engine_init_timeline(struct intel_engine_cs *engine)
 392 {
 393         engine->timeline = &engine->i915->gt.global_timeline.engine[engine->id];
 394 }
 395
 396 static bool csb_force_mmio(struct drm_i915_private *i915)
 397 {
 398         /*
 399          * IOMMU adds unpredictable latency causing the CSB write (from the
 400          * GPU into the HWSP) to only be visible some time after the interrupt
 401          * (missed breadcrumb syndrome).
 402          */
 403         if (intel_vtd_active())
 404                 return true;
 405
 406         /* Older GVT emulation depends upon intercepting CSB mmio */
 407         if (intel_vgpu_active(i915) && !intel_vgpu_has_hwsp_emulation(i915))
 408                 return true;
 409
 410         return false;
 411 }
 412
 413 static void intel_engine_init_execlist(struct intel_engine_cs *engine)
 414 {
 415         struct intel_engine_execlists * const execlists = &engine->execlists;
 416
 417         execlists->csb_use_mmio = csb_force_mmio(engine->i915);
 418
 419         execlists->port_mask = 1;
 420         BUILD_BUG_ON_NOT_POWER_OF_2(execlists_num_ports(execlists));
 421         GEM_BUG_ON(execlists_num_ports(execlists) > EXECLIST_MAX_PORTS);
 422
 423         execlists->queue = RB_ROOT;
 424         execlists->first = NULL;
 425 }
 426
 427 /**
 428  * intel_engines_setup_common - setup engine state not requiring hw access
 429  * @engine: Engine to setup.
 430  *
 431  * Initializes @engine@ structure members shared between legacy and execlists
 432  * submission modes which do not require hardware access.
 433  *
 434  * Typically done early in the submission mode specific engine setup stage.
 435  */
 436 void intel_engine_setup_common(struct intel_engine_cs *engine)
 437 {
 438         intel_engine_init_execlist(engine);
 439
 440         intel_engine_init_timeline(engine);
 441         intel_engine_init_hangcheck(engine);
 442         i915_gem_batch_pool_init(engine, &engine->batch_pool);
 443
 444         intel_engine_init_cmd_parser(engine);
 445 }
 446
 447 int intel_engine_create_scratch(struct intel_engine_cs *engine, int size)
 448 {
 449         struct drm_i915_gem_object *obj;
 450         struct i915_vma *vma;
 451         int ret;
 452
 453         WARN_ON(engine->scratch);
 454
 455         obj = i915_gem_object_create_stolen(engine->i915, size);
 456         if (!obj)
 457                 obj = i915_gem_object_create_internal(engine->i915, size);
 458         if (IS_ERR(obj)) {
 459                 DRM_ERROR("Failed to allocate scratch page\n");
 460                 return PTR_ERR(obj);
 461         }
 462
 463         vma = i915_vma_instance(obj, &engine->i915->ggtt.base, NULL);
 464         if (IS_ERR(vma)) {
 465                 ret = PTR_ERR(vma);
 466                 goto err_unref;
 467         }
 468
 469         ret = i915_vma_pin(vma, 0, 4096, PIN_GLOBAL | PIN_HIGH);
 470         if (ret)
 471                 goto err_unref;
 472
 473         engine->scratch = vma;
 474         DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
 475                          engine->name, i915_ggtt_offset(vma));
 476         return 0;
 477
 478 err_unref:
 479         i915_gem_object_put(obj);
 480         return ret;
 481 }
 482
 483 static void intel_engine_cleanup_scratch(struct intel_engine_cs *engine)
 484 {
 485         i915_vma_unpin_and_release(&engine->scratch);
 486 }
 487
 488 static void cleanup_phys_status_page(struct intel_engine_cs *engine)
 489 {
 490         struct drm_i915_private *dev_priv = engine->i915;
 491
 492         if (!dev_priv->status_page_dmah)
 493                 return;
 494
 495         drm_pci_free(&dev_priv->drm, dev_priv->status_page_dmah);
 496         engine->status_page.page_addr = NULL;
 497 }
 498
 499 static void cleanup_status_page(struct intel_engine_cs *engine)
 500 {
 501         struct i915_vma *vma;
 502         struct drm_i915_gem_object *obj;
 503
 504         vma = fetch_and_zero(&engine->status_page.vma);
 505         if (!vma)
 506                 return;
 507
 508         obj = vma->obj;
 509
 510         i915_vma_unpin(vma);
 511         i915_vma_close(vma);
 512
 513         i915_gem_object_unpin_map(obj);
 514         __i915_gem_object_release_unless_active(obj);
 515 }
 516
 517 static int init_status_page(struct intel_engine_cs *engine)
 518 {
 519         struct drm_i915_gem_object *obj;
 520         struct i915_vma *vma;
 521         unsigned int flags;
 522         void *vaddr;
 523         int ret;
 524
 525         obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
 526         if (IS_ERR(obj)) {
 527                 DRM_ERROR("Failed to allocate status page\n");
 528                 return PTR_ERR(obj);
 529         }
 530
 531         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
 532         if (ret)
 533                 goto err;
 534
 535         vma = i915_vma_instance(obj, &engine->i915->ggtt.base, NULL);
 536         if (IS_ERR(vma)) {
 537                 ret = PTR_ERR(vma);
 538                 goto err;
 539         }
 540
 541         flags = PIN_GLOBAL;
 542         if (!HAS_LLC(engine->i915))
 543                 /* On g33, we cannot place HWS above 256MiB, so
 544                  * restrict its pinning to the low mappable arena.
 545                  * Though this restriction is not documented for
 546                  * gen4, gen5, or byt, they also behave similarly
 547                  * and hang if the HWS is placed at the top of the
 548                  * GTT. To generalise, it appears that all !llc
 549                  * platforms have issues with us placing the HWS
 550                  * above the mappable region (even though we never
 551                  * actually map it).
 552                  */
 553                 flags |= PIN_MAPPABLE;
 554         else
 555                 flags |= PIN_HIGH;
 556         ret = i915_vma_pin(vma, 0, 4096, flags);
 557         if (ret)
 558                 goto err;
 559
 560         vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
 561         if (IS_ERR(vaddr)) {
 562                 ret = PTR_ERR(vaddr);
 563                 goto err_unpin;
 564         }
 565
 566         engine->status_page.vma = vma;
 567         engine->status_page.ggtt_offset = i915_ggtt_offset(vma);
 568         engine->status_page.page_addr = memset(vaddr, 0, PAGE_SIZE);
 569
 570         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
 571                          engine->name, i915_ggtt_offset(vma));
 572         return 0;
 573
 574 err_unpin:
 575         i915_vma_unpin(vma);
 576 err:
 577         i915_gem_object_put(obj);
 578         return ret;
 579 }
 580
 581 static int init_phys_status_page(struct intel_engine_cs *engine)
 582 {
 583         struct drm_i915_private *dev_priv = engine->i915;
 584
 585         GEM_BUG_ON(engine->id != RCS);
 586
 587         dev_priv->status_page_dmah =
 588                 drm_pci_alloc(&dev_priv->drm, PAGE_SIZE, PAGE_SIZE);
 589         if (!dev_priv->status_page_dmah)
 590                 return -ENOMEM;
 591
 592         engine->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
 593         memset(engine->status_page.page_addr, 0, PAGE_SIZE);
 594
 595         return 0;
 596 }
 597
 598 /**
 599  * intel_engines_init_common - initialize cengine state which might require hw access
 600  * @engine: Engine to initialize.
 601  *
 602  * Initializes @engine@ structure members shared between legacy and execlists
 603  * submission modes which do require hardware access.
 604  *
 605  * Typcally done at later stages of submission mode specific engine setup.
 606  *
 607  * Returns zero on success or an error code on failure.
 608  */
 609 int intel_engine_init_common(struct intel_engine_cs *engine)
 610 {
 611         struct intel_ring *ring;
 612         int ret;
 613
 614         engine->set_default_submission(engine);
 615
 616         /* We may need to do things with the shrinker which
 617          * require us to immediately switch back to the default
 618          * context. This can cause a problem as pinning the
 619          * default context also requires GTT space which may not
 620          * be available. To avoid this we always pin the default
 621          * context.
 622          */
 623         ring = engine->context_pin(engine, engine->i915->kernel_context);
 624         if (IS_ERR(ring))
 625                 return PTR_ERR(ring);
 626
 627         /*
 628          * Similarly the preempt context must always be available so that
 629          * we can interrupt the engine at any time.
 630          */
 631         if (INTEL_INFO(engine->i915)->has_logical_ring_preemption) {
 632                 ring = engine->context_pin(engine,
 633                                            engine->i915->preempt_context);
 634                 if (IS_ERR(ring)) {
 635                         ret = PTR_ERR(ring);
 636                         goto err_unpin_kernel;
 637                 }
 638         }
 639
 640         ret = intel_engine_init_breadcrumbs(engine);
 641         if (ret)
 642                 goto err_unpin_preempt;
 643
 644         ret = i915_gem_render_state_init(engine);
 645         if (ret)
 646                 goto err_breadcrumbs;
 647
 648         if (HWS_NEEDS_PHYSICAL(engine->i915))
 649                 ret = init_phys_status_page(engine);
 650         else
 651                 ret = init_status_page(engine);
 652         if (ret)
 653                 goto err_rs_fini;
 654
 655         return 0;
 656
 657 err_rs_fini:
 658         i915_gem_render_state_fini(engine);
 659 err_breadcrumbs:
 660         intel_engine_fini_breadcrumbs(engine);
 661 err_unpin_preempt:
 662         if (INTEL_INFO(engine->i915)->has_logical_ring_preemption)
 663                 engine->context_unpin(engine, engine->i915->preempt_context);
 664 err_unpin_kernel:
 665         engine->context_unpin(engine, engine->i915->kernel_context);
 666         return ret;
 667 }
 668
 669 /**
 670  * intel_engines_cleanup_common - cleans up the engine state created by
 671  *                                the common initiailizers.
 672  * @engine: Engine to cleanup.
 673  *
 674  * This cleans up everything created by the common helpers.
 675  */
 676 void intel_engine_cleanup_common(struct intel_engine_cs *engine)
 677 {
 678         intel_engine_cleanup_scratch(engine);
 679
 680         if (HWS_NEEDS_PHYSICAL(engine->i915))
 681                 cleanup_phys_status_page(engine);
 682         else
 683                 cleanup_status_page(engine);
 684
 685         i915_gem_render_state_fini(engine);
 686         intel_engine_fini_breadcrumbs(engine);
 687         intel_engine_cleanup_cmd_parser(engine);
 688         i915_gem_batch_pool_fini(&engine->batch_pool);
 689
 690         if (engine->default_state)
 691                 i915_gem_object_put(engine->default_state);
 692
 693         if (INTEL_INFO(engine->i915)->has_logical_ring_preemption)
 694                 engine->context_unpin(engine, engine->i915->preempt_context);
 695         engine->context_unpin(engine, engine->i915->kernel_context);
 696 }
 697
 698 u64 intel_engine_get_active_head(struct intel_engine_cs *engine)
 699 {
 700         struct drm_i915_private *dev_priv = engine->i915;
 701         u64 acthd;
 702
 703         if (INTEL_GEN(dev_priv) >= 8)
 704                 acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base),
 705                                          RING_ACTHD_UDW(engine->mmio_base));
 706         else if (INTEL_GEN(dev_priv) >= 4)
 707                 acthd = I915_READ(RING_ACTHD(engine->mmio_base));
 708         else
 709                 acthd = I915_READ(ACTHD);
 710
 711         return acthd;
 712 }
 713
 714 u64 intel_engine_get_last_batch_head(struct intel_engine_cs *engine)
 715 {
 716         struct drm_i915_private *dev_priv = engine->i915;
 717         u64 bbaddr;
 718
 719         if (INTEL_GEN(dev_priv) >= 8)
 720                 bbaddr = I915_READ64_2x32(RING_BBADDR(engine->mmio_base),
 721                                           RING_BBADDR_UDW(engine->mmio_base));
 722         else
 723                 bbaddr = I915_READ(RING_BBADDR(engine->mmio_base));
 724
 725         return bbaddr;
 726 }
 727
 728 const char *i915_cache_level_str(struct drm_i915_private *i915, int type)
 729 {
 730         switch (type) {
 731         case I915_CACHE_NONE: return " uncached";
 732         case I915_CACHE_LLC: return HAS_LLC(i915) ? " LLC" : " snooped";
 733         case I915_CACHE_L3_LLC: return " L3+LLC";
 734         case I915_CACHE_WT: return " WT";
 735         default: return "";
 736         }
 737 }
 738
 739 static inline uint32_t
 740 read_subslice_reg(struct drm_i915_private *dev_priv, int slice,
 741                   int subslice, i915_reg_t reg)
 742 {
 743         uint32_t mcr;
 744         uint32_t ret;
 745         enum forcewake_domains fw_domains;
 746
 747         fw_domains = intel_uncore_forcewake_for_reg(dev_priv, reg,
 748                                                     FW_REG_READ);
 749         fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
 750                                                      GEN8_MCR_SELECTOR,
 751                                                      FW_REG_READ | FW_REG_WRITE);
 752
 753         spin_lock_irq(&dev_priv->uncore.lock);
 754         intel_uncore_forcewake_get__locked(dev_priv, fw_domains);
 755
 756         mcr = I915_READ_FW(GEN8_MCR_SELECTOR);
 757         /*
 758          * The HW expects the slice and sublice selectors to be reset to 0
 759          * after reading out the registers.
 760          */
 761         WARN_ON_ONCE(mcr & (GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK));
 762         mcr &= ~(GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK);
 763         mcr |= GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice);
 764         I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);
 765
 766         ret = I915_READ_FW(reg);
 767
 768         mcr &= ~(GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK);
 769         I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);
 770
 771         intel_uncore_forcewake_put__locked(dev_priv, fw_domains);
 772         spin_unlock_irq(&dev_priv->uncore.lock);
 773
 774         return ret;
 775 }
 776
 777 /* NB: please notice the memset */
 778 void intel_engine_get_instdone(struct intel_engine_cs *engine,
 779                                struct intel_instdone *instdone)
 780 {
 781         struct drm_i915_private *dev_priv = engine->i915;
 782         u32 mmio_base = engine->mmio_base;
 783         int slice;
 784         int subslice;
 785
 786         memset(instdone, 0, sizeof(*instdone));
 787
 788         switch (INTEL_GEN(dev_priv)) {
 789         default:
 790                 instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
 791
 792                 if (engine->id != RCS)
 793                         break;
 794
 795                 instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
 796                 for_each_instdone_slice_subslice(dev_priv, slice, subslice) {
 797                         instdone->sampler[slice][subslice] =
 798                                 read_subslice_reg(dev_priv, slice, subslice,
 799                                                   GEN7_SAMPLER_INSTDONE);
 800                         instdone->row[slice][subslice] =
 801                                 read_subslice_reg(dev_priv, slice, subslice,
 802                                                   GEN7_ROW_INSTDONE);
 803                 }
 804                 break;
 805         case 7:
 806                 instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
 807
 808                 if (engine->id != RCS)
 809                         break;
 810
 811                 instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
 812                 instdone->sampler[0][0] = I915_READ(GEN7_SAMPLER_INSTDONE);
 813                 instdone->row[0][0] = I915_READ(GEN7_ROW_INSTDONE);
 814
 815                 break;
 816         case 6:
 817         case 5:
 818         case 4:
 819                 instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
 820
 821                 if (engine->id == RCS)
 822                         /* HACK: Using the wrong struct member */
 823                         instdone->slice_common = I915_READ(GEN4_INSTDONE1);
 824                 break;
 825         case 3:
 826         case 2:
 827                 instdone->instdone = I915_READ(GEN2_INSTDONE);
 828                 break;
 829         }
 830 }
 831
 832 static int wa_add(struct drm_i915_private *dev_priv,
 833                   i915_reg_t addr,
 834                   const u32 mask, const u32 val)
 835 {
 836         const u32 idx = dev_priv->workarounds.count;
 837
 838         if (WARN_ON(idx >= I915_MAX_WA_REGS))
 839                 return -ENOSPC;
 840
 841         dev_priv->workarounds.reg[idx].addr = addr;
 842         dev_priv->workarounds.reg[idx].value = val;
 843         dev_priv->workarounds.reg[idx].mask = mask;
 844
 845         dev_priv->workarounds.count++;
 846
 847         return 0;
 848 }
 849
 850 #define WA_REG(addr, mask, val) do { \
 851                 const int r = wa_add(dev_priv, (addr), (mask), (val)); \
 852                 if (r) \
 853                         return r; \
 854         } while (0)
 855
 856 #define WA_SET_BIT_MASKED(addr, mask) \
 857         WA_REG(addr, (mask), _MASKED_BIT_ENABLE(mask))
 858
 859 #define WA_CLR_BIT_MASKED(addr, mask) \
 860         WA_REG(addr, (mask), _MASKED_BIT_DISABLE(mask))
 861
 862 #define WA_SET_FIELD_MASKED(addr, mask, value) \
 863         WA_REG(addr, mask, _MASKED_FIELD(mask, value))
 864
 865 static int wa_ring_whitelist_reg(struct intel_engine_cs *engine,
 866                                  i915_reg_t reg)
 867 {
 868         struct drm_i915_private *dev_priv = engine->i915;
 869         struct i915_workarounds *wa = &dev_priv->workarounds;
 870         const uint32_t index = wa->hw_whitelist_count[engine->id];
 871
 872         if (WARN_ON(index >= RING_MAX_NONPRIV_SLOTS))
 873                 return -EINVAL;
 874
 875         I915_WRITE(RING_FORCE_TO_NONPRIV(engine->mmio_base, index),
 876                    i915_mmio_reg_offset(reg));
 877         wa->hw_whitelist_count[engine->id]++;
 878
 879         return 0;
 880 }
 881
 882 static int gen8_init_workarounds(struct intel_engine_cs *engine)
 883 {
 884         struct drm_i915_private *dev_priv = engine->i915;
 885
 886         WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
 887
 888         /* WaDisableAsyncFlipPerfMode:bdw,chv */
 889         WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
 890
 891         /* WaDisablePartialInstShootdown:bdw,chv */
 892         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 893                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 894
 895         /* Use Force Non-Coherent whenever executing a 3D context. This is a
 896          * workaround for for a possible hang in the unlikely event a TLB
 897          * invalidation occurs during a PSD flush.
 898          */
 899         /* WaForceEnableNonCoherent:bdw,chv */
 900         /* WaHdcDisableFetchWhenMasked:bdw,chv */
 901         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 902                           HDC_DONOT_FETCH_MEM_WHEN_MASKED |
 903                           HDC_FORCE_NON_COHERENT);
 904
 905         /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
 906          * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
 907          *  polygons in the same 8x4 pixel/sample area to be processed without
 908          *  stalling waiting for the earlier ones to write to Hierarchical Z
 909          *  buffer."
 910          *
 911          * This optimization is off by default for BDW and CHV; turn it on.
 912          */
 913         WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
 914
 915         /* Wa4x4STCOptimizationDisable:bdw,chv */
 916         WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
 917
 918         /*
 919          * BSpec recommends 8x4 when MSAA is used,
 920          * however in practice 16x4 seems fastest.
 921          *
 922          * Note that PS/WM thread counts depend on the WIZ hashing
 923          * disable bit, which we don't touch here, but it's good
 924          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 925          */
 926         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
 927                             GEN6_WIZ_HASHING_MASK,
 928                             GEN6_WIZ_HASHING_16x4);
 929
 930         return 0;
 931 }
 932
 933 static int bdw_init_workarounds(struct intel_engine_cs *engine)
 934 {
 935         struct drm_i915_private *dev_priv = engine->i915;
 936         int ret;
 937
 938         ret = gen8_init_workarounds(engine);
 939         if (ret)
 940                 return ret;
 941
 942         /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
 943         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 944
 945         /* WaDisableDopClockGating:bdw
 946          *
 947          * Also see the related UCGTCL1 write in broadwell_init_clock_gating()
 948          * to disable EUTC clock gating.
 949          */
 950         WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 951                           DOP_CLOCK_GATING_DISABLE);
 952
 953         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 954                           GEN8_SAMPLER_POWER_BYPASS_DIS);
 955
 956         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 957                           /* WaForceContextSaveRestoreNonCoherent:bdw */
 958                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 959                           /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
 960                           (IS_BDW_GT3(dev_priv) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
 961
 962         return 0;
 963 }
 964
 965 static int chv_init_workarounds(struct intel_engine_cs *engine)
 966 {
 967         struct drm_i915_private *dev_priv = engine->i915;
 968         int ret;
 969
 970         ret = gen8_init_workarounds(engine);
 971         if (ret)
 972                 return ret;
 973
 974         /* WaDisableThreadStallDopClockGating:chv */
 975         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 976
 977         /* Improve HiZ throughput on CHV. */
 978         WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
 979
 980         return 0;
 981 }
 982
 983 static int gen9_init_workarounds(struct intel_engine_cs *engine)
 984 {
 985         struct drm_i915_private *dev_priv = engine->i915;
 986         int ret;
 987
 988         /* WaConextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
 989         I915_WRITE(GEN9_CSFE_CHICKEN1_RCS, _MASKED_BIT_ENABLE(GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE));
 990
 991         /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
 992         I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) |
 993                    GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
 994
 995         /* WaDisableKillLogic:bxt,skl,kbl */
 996         if (!IS_COFFEELAKE(dev_priv))
 997                 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
 998                            ECOCHK_DIS_TLB);
 999
1000         if (HAS_LLC(dev_priv)) {
1001                 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
1002                  *
1003                  * Must match Display Engine. See
1004                  * WaCompressedResourceDisplayNewHashMode.
1005                  */
1006                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1007                                   GEN9_PBE_COMPRESSED_HASH_SELECTION);
1008                 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
1009                                   GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
1010
1011                 I915_WRITE(MMCD_MISC_CTRL,
1012                            I915_READ(MMCD_MISC_CTRL) |
1013                            MMCD_PCLA |
1014                            MMCD_HOTSPOT_EN);
1015         }
1016
1017         /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
1018         /* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
1019         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
1020                           FLOW_CONTROL_ENABLE |
1021                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
1022
1023         /* Syncing dependencies between camera and graphics:skl,bxt,kbl */
1024         if (!IS_COFFEELAKE(dev_priv))
1025                 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
1026                                   GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
1027
1028         /* WaDisableDgMirrorFixInHalfSliceChicken5:bxt */
1029         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
1030                 WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
1031                                   GEN9_DG_MIRROR_FIX_ENABLE);
1032
1033         /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:bxt */
1034         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
1035                 WA_SET_BIT_MASKED(GEN7_COMMON_SLICE_CHICKEN1,
1036                                   GEN9_RHWO_OPTIMIZATION_DISABLE);
1037                 /*
1038                  * WA also requires GEN9_SLICE_COMMON_ECO_CHICKEN0[14:14] to be set
1039                  * but we do that in per ctx batchbuffer as there is an issue
1040                  * with this register not getting restored on ctx restore
1041                  */
1042         }
1043
1044         /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
1045         /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
1046         WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
1047                           GEN9_ENABLE_YV12_BUGFIX |
1048                           GEN9_ENABLE_GPGPU_PREEMPTION);
1049
1050         /* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
1051         /* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
1052         WA_SET_BIT_MASKED(CACHE_MODE_1, (GEN8_4x4_STC_OPTIMIZATION_DISABLE |
1053                                          GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE));
1054
1055         /* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
1056         WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
1057                           GEN9_CCS_TLB_PREFETCH_ENABLE);
1058
1059         /* WaDisableMaskBasedCammingInRCC:bxt */
1060         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
1061                 WA_SET_BIT_MASKED(SLICE_ECO_CHICKEN0,
1062                                   PIXEL_MASK_CAMMING_DISABLE);
1063
1064         /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
1065         WA_SET_BIT_MASKED(HDC_CHICKEN0,
1066                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
1067                           HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
1068
1069         /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
1070          * both tied to WaForceContextSaveRestoreNonCoherent
1071          * in some hsds for skl. We keep the tie for all gen9. The
1072          * documentation is a bit hazy and so we want to get common behaviour,
1073          * even though there is no clear evidence we would need both on kbl/bxt.
1074          * This area has been source of system hangs so we play it safe
1075          * and mimic the skl regardless of what bspec says.
1076          *
1077          * Use Force Non-Coherent whenever executing a 3D context. This
1078          * is a workaround for a possible hang in the unlikely event
1079          * a TLB invalidation occurs during a PSD flush.
1080          */
1081
1082         /* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
1083         WA_SET_BIT_MASKED(HDC_CHICKEN0,
1084                           HDC_FORCE_NON_COHERENT);
1085
1086         /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
1087         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
1088                    BDW_DISABLE_HDC_INVALIDATION);
1089
1090         /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
1091         if (IS_SKYLAKE(dev_priv) ||
1092             IS_KABYLAKE(dev_priv) ||
1093             IS_COFFEELAKE(dev_priv) ||
1094             IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0))
1095                 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
1096                                   GEN8_SAMPLER_POWER_BYPASS_DIS);
1097
1098         /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
1099         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
1100
1101         /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
1102         I915_WRITE(GEN8_L3SQCREG4, (I915_READ(GEN8_L3SQCREG4) |
1103                                     GEN8_LQSC_FLUSH_COHERENT_LINES));
1104
1105         /*
1106          * Supporting preemption with fine-granularity requires changes in the
1107          * batch buffer programming. Since we can't break old userspace, we
1108          * need to set our default preemption level to safe value. Userspace is
1109          * still able to use more fine-grained preemption levels, since in
1110          * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
1111          * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
1112          * not real HW workarounds, but merely a way to start using preemption
1113          * while maintaining old contract with userspace.
1114          */
1115
1116         /* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
1117         WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
1118
1119         /* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
1120         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_GPGPU_LEVEL_MASK,
1121                             GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
1122
1123         /* WaClearHIZ_WM_CHICKEN3:bxt,glk */
1124         if (IS_GEN9_LP(dev_priv))
1125                 WA_SET_BIT_MASKED(GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ);
1126
1127         /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
1128         ret = wa_ring_whitelist_reg(engine, GEN9_CTX_PREEMPT_REG);
1129         if (ret)
1130                 return ret;
1131
1132         /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
1133         I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
1134                    _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
1135         ret = wa_ring_whitelist_reg(engine, GEN8_CS_CHICKEN1);
1136         if (ret)
1137                 return ret;
1138
1139         /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
1140         ret = wa_ring_whitelist_reg(engine, GEN8_HDC_CHICKEN1);
1141         if (ret)
1142                 return ret;
1143
1144         return 0;
1145 }
1146
1147 static int skl_tune_iz_hashing(struct intel_engine_cs *engine)
1148 {
1149         struct drm_i915_private *dev_priv = engine->i915;
1150         u8 vals[3] = { 0, 0, 0 };
1151         unsigned int i;
1152
1153         for (i = 0; i < 3; i++) {
1154                 u8 ss;
1155
1156                 /*
1157                  * Only consider slices where one, and only one, subslice has 7
1158                  * EUs
1159                  */
1160                 if (!is_power_of_2(INTEL_INFO(dev_priv)->sseu.subslice_7eu[i]))
1161                         continue;
1162
1163                 /*
1164                  * subslice_7eu[i] != 0 (because of the check above) and
1165                  * ss_max == 4 (maximum number of subslices possible per slice)
1166                  *
1167                  * ->    0 <= ss <= 3;
1168                  */
1169                 ss = ffs(INTEL_INFO(dev_priv)->sseu.subslice_7eu[i]) - 1;
1170                 vals[i] = 3 - ss;
1171         }
1172
1173         if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
1174                 return 0;
1175
1176         /* Tune IZ hashing. See intel_device_info_runtime_init() */
1177         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
1178                             GEN9_IZ_HASHING_MASK(2) |
1179                             GEN9_IZ_HASHING_MASK(1) |
1180                             GEN9_IZ_HASHING_MASK(0),
1181                             GEN9_IZ_HASHING(2, vals[2]) |
1182                             GEN9_IZ_HASHING(1, vals[1]) |
1183                             GEN9_IZ_HASHING(0, vals[0]));
1184
1185         return 0;
1186 }
1187
1188 static int skl_init_workarounds(struct intel_engine_cs *engine)
1189 {
1190         struct drm_i915_private *dev_priv = engine->i915;
1191         int ret;
1192
1193         ret = gen9_init_workarounds(engine);
1194         if (ret)
1195                 return ret;
1196
1197         /* WaEnableGapsTsvCreditFix:skl */
1198         I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
1199                                    GEN9_GAPS_TSV_CREDIT_DISABLE));
1200
1201         /* WaDisableGafsUnitClkGating:skl */
1202         I915_WRITE(GEN7_UCGCTL4, (I915_READ(GEN7_UCGCTL4) |
1203                                   GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE));
1204
1205         /* WaInPlaceDecompressionHang:skl */
1206         if (IS_SKL_REVID(dev_priv, SKL_REVID_H0, REVID_FOREVER))
1207                 I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
1208                            (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
1209                             GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
1210
1211         /* WaDisableLSQCROPERFforOCL:skl */
1212         ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
1213         if (ret)
1214                 return ret;
1215
1216         return skl_tune_iz_hashing(engine);
1217 }
1218
1219 static int bxt_init_workarounds(struct intel_engine_cs *engine)
1220 {
1221         struct drm_i915_private *dev_priv = engine->i915;
1222         int ret;
1223
1224         ret = gen9_init_workarounds(engine);
1225         if (ret)
1226                 return ret;
1227
1228         /* WaStoreMultiplePTEenable:bxt */
1229         /* This is a requirement according to Hardware specification */
1230         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
1231                 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_TLBPF);
1232
1233         /* WaSetClckGatingDisableMedia:bxt */
1234         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
1235                 I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) &
1236                                             ~GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE));
1237         }
1238
1239         /* WaDisableThreadStallDopClockGating:bxt */
1240         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
1241                           STALL_DOP_GATING_DISABLE);
1242
1243         /* WaDisablePooledEuLoadBalancingFix:bxt */
1244         if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER)) {
1245                 I915_WRITE(FF_SLICE_CS_CHICKEN2,
1246                            _MASKED_BIT_ENABLE(GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE));
1247         }
1248
1249         /* WaDisableSbeCacheDispatchPortSharing:bxt */
1250         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0)) {
1251                 WA_SET_BIT_MASKED(
1252                         GEN7_HALF_SLICE_CHICKEN1,
1253                         GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
1254         }
1255
1256         /* WaDisableObjectLevelPreemptionForTrifanOrPolygon:bxt */
1257         /* WaDisableObjectLevelPreemptionForInstancedDraw:bxt */
1258         /* WaDisableObjectLevelPreemtionForInstanceId:bxt */
1259         /* WaDisableLSQCROPERFforOCL:bxt */
1260         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
1261                 ret = wa_ring_whitelist_reg(engine, GEN9_CS_DEBUG_MODE1);
1262                 if (ret)
1263                         return ret;
1264
1265                 ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
1266                 if (ret)
1267                         return ret;
1268         }
1269
1270         /* WaProgramL3SqcReg1DefaultForPerf:bxt */
1271         if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER)) {
1272                 u32 val = I915_READ(GEN8_L3SQCREG1);
1273                 val &= ~L3_PRIO_CREDITS_MASK;
1274                 val |= L3_GENERAL_PRIO_CREDITS(62) | L3_HIGH_PRIO_CREDITS(2);
1275                 I915_WRITE(GEN8_L3SQCREG1, val);
1276         }
1277
1278         /* WaToEnableHwFixForPushConstHWBug:bxt */
1279         if (IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER))
1280                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1281                                   GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1282
1283         /* WaInPlaceDecompressionHang:bxt */
1284         if (IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER))
1285                 I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
1286                            (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
1287                             GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
1288
1289         return 0;
1290 }
1291
1292 static int cnl_init_workarounds(struct intel_engine_cs *engine)
1293 {
1294         struct drm_i915_private *dev_priv = engine->i915;
1295         int ret;
1296
1297         /* WaDisableI2mCycleOnWRPort:cnl (pre-prod) */
1298         if (IS_CNL_REVID(dev_priv, CNL_REVID_B0, CNL_REVID_B0))
1299                 I915_WRITE(GAMT_CHKN_BIT_REG,
1300                            (I915_READ(GAMT_CHKN_BIT_REG) |
1301                             GAMT_CHKN_DISABLE_I2M_CYCLE_ON_WR_PORT));
1302
1303         /* WaForceContextSaveRestoreNonCoherent:cnl */
1304         WA_SET_BIT_MASKED(CNL_HDC_CHICKEN0,
1305                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT);
1306
1307         /* WaThrottleEUPerfToAvoidTDBackPressure:cnl(pre-prod) */
1308         if (IS_CNL_REVID(dev_priv, CNL_REVID_B0, CNL_REVID_B0))
1309                 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, THROTTLE_12_5);
1310
1311         /* WaDisableReplayBufferBankArbitrationOptimization:cnl */
1312         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1313                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1314
1315         /* WaDisableEnhancedSBEVertexCaching:cnl (pre-prod) */
1316         if (IS_CNL_REVID(dev_priv, 0, CNL_REVID_B0))
1317                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1318                                   GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE);
1319
1320         /* WaInPlaceDecompressionHang:cnl */
1321         I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
1322                    (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
1323                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
1324
1325         /* WaPushConstantDereferenceHoldDisable:cnl */
1326         WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE);
1327
1328         /* FtrEnableFastAnisoL1BankingFix: cnl */
1329         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, CNL_FAST_ANISO_L1_BANKING_FIX);
1330
1331         /* WaDisable3DMidCmdPreemption:cnl */
1332         WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
1333
1334         /* WaDisableGPGPUMidCmdPreemption:cnl */
1335         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_GPGPU_LEVEL_MASK,
1336                             GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
1337
1338         /* WaEnablePreemptionGranularityControlByUMD:cnl */
1339         I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
1340                    _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
1341         ret= wa_ring_whitelist_reg(engine, GEN8_CS_CHICKEN1);
1342         if (ret)
1343                 return ret;
1344
1345         return 0;
1346 }
1347
1348 static int kbl_init_workarounds(struct intel_engine_cs *engine)
1349 {
1350         struct drm_i915_private *dev_priv = engine->i915;
1351         int ret;
1352
1353         ret = gen9_init_workarounds(engine);
1354         if (ret)
1355                 return ret;
1356
1357         /* WaEnableGapsTsvCreditFix:kbl */
1358         I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
1359                                    GEN9_GAPS_TSV_CREDIT_DISABLE));
1360
1361         /* WaDisableDynamicCreditSharing:kbl */
1362         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
1363                 I915_WRITE(GAMT_CHKN_BIT_REG,
1364                            (I915_READ(GAMT_CHKN_BIT_REG) |
1365                             GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING));
1366
1367         /* WaDisableFenceDestinationToSLM:kbl (pre-prod) */
1368         if (IS_KBL_REVID(dev_priv, KBL_REVID_A0, KBL_REVID_A0))
1369                 WA_SET_BIT_MASKED(HDC_CHICKEN0,
1370                                   HDC_FENCE_DEST_SLM_DISABLE);
1371
1372         /* WaToEnableHwFixForPushConstHWBug:kbl */
1373         if (IS_KBL_REVID(dev_priv, KBL_REVID_C0, REVID_FOREVER))
1374                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1375                                   GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1376
1377         /* WaDisableGafsUnitClkGating:kbl */
1378         I915_WRITE(GEN7_UCGCTL4, (I915_READ(GEN7_UCGCTL4) |
1379                                   GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE));
1380
1381         /* WaDisableSbeCacheDispatchPortSharing:kbl */
1382         WA_SET_BIT_MASKED(
1383                 GEN7_HALF_SLICE_CHICKEN1,
1384                 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
1385
1386         /* WaInPlaceDecompressionHang:kbl */
1387         I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
1388                    (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
1389                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
1390
1391         /* WaDisableLSQCROPERFforOCL:kbl */
1392         ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
1393         if (ret)
1394                 return ret;
1395
1396         return 0;
1397 }
1398
1399 static int glk_init_workarounds(struct intel_engine_cs *engine)
1400 {
1401         struct drm_i915_private *dev_priv = engine->i915;
1402         int ret;
1403
1404         ret = gen9_init_workarounds(engine);
1405         if (ret)
1406                 return ret;
1407
1408         /* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
1409         ret = wa_ring_whitelist_reg(engine, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1410         if (ret)
1411                 return ret;
1412
1413         /* WaToEnableHwFixForPushConstHWBug:glk */
1414         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1415                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1416
1417         return 0;
1418 }
1419
1420 static int cfl_init_workarounds(struct intel_engine_cs *engine)
1421 {
1422         struct drm_i915_private *dev_priv = engine->i915;
1423         int ret;
1424
1425         ret = gen9_init_workarounds(engine);
1426         if (ret)
1427                 return ret;
1428
1429         /* WaEnableGapsTsvCreditFix:cfl */
1430         I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
1431                                    GEN9_GAPS_TSV_CREDIT_DISABLE));
1432
1433         /* WaToEnableHwFixForPushConstHWBug:cfl */
1434         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1435                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1436
1437         /* WaDisableGafsUnitClkGating:cfl */
1438         I915_WRITE(GEN7_UCGCTL4, (I915_READ(GEN7_UCGCTL4) |
1439                                   GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE));
1440
1441         /* WaDisableSbeCacheDispatchPortSharing:cfl */
1442         WA_SET_BIT_MASKED(
1443                 GEN7_HALF_SLICE_CHICKEN1,
1444                 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
1445
1446         /* WaInPlaceDecompressionHang:cfl */
1447         I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
1448                    (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
1449                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
1450
1451         return 0;
1452 }
1453
1454 int init_workarounds_ring(struct intel_engine_cs *engine)
1455 {
1456         struct drm_i915_private *dev_priv = engine->i915;
1457         int err;
1458
1459         WARN_ON(engine->id != RCS);
1460
1461         dev_priv->workarounds.count = 0;
1462         dev_priv->workarounds.hw_whitelist_count[engine->id] = 0;
1463
1464         if (IS_BROADWELL(dev_priv))
1465                 err = bdw_init_workarounds(engine);
1466         else if (IS_CHERRYVIEW(dev_priv))
1467                 err = chv_init_workarounds(engine);
1468         else if (IS_SKYLAKE(dev_priv))
1469                 err =  skl_init_workarounds(engine);
1470         else if (IS_BROXTON(dev_priv))
1471                 err = bxt_init_workarounds(engine);
1472         else if (IS_KABYLAKE(dev_priv))
1473                 err = kbl_init_workarounds(engine);
1474         else if (IS_GEMINILAKE(dev_priv))
1475                 err =  glk_init_workarounds(engine);
1476         else if (IS_COFFEELAKE(dev_priv))
1477                 err = cfl_init_workarounds(engine);
1478         else if (IS_CANNONLAKE(dev_priv))
1479                 err = cnl_init_workarounds(engine);
1480         else
1481                 err = 0;
1482         if (err)
1483                 return err;
1484
1485         DRM_DEBUG_DRIVER("%s: Number of context specific w/a: %d\n",
1486                          engine->name, dev_priv->workarounds.count);
1487         return 0;
1488 }
1489
1490 int intel_ring_workarounds_emit(struct drm_i915_gem_request *req)
1491 {
1492         struct i915_workarounds *w = &req->i915->workarounds;
1493         u32 *cs;
1494         int ret, i;
1495
1496         if (w->count == 0)
1497                 return 0;
1498
1499         ret = req->engine->emit_flush(req, EMIT_BARRIER);
1500         if (ret)
1501                 return ret;
1502
1503         cs = intel_ring_begin(req, (w->count * 2 + 2));
1504         if (IS_ERR(cs))
1505                 return PTR_ERR(cs);
1506
1507         *cs++ = MI_LOAD_REGISTER_IMM(w->count);
1508         for (i = 0; i < w->count; i++) {
1509                 *cs++ = i915_mmio_reg_offset(w->reg[i].addr);
1510                 *cs++ = w->reg[i].value;
1511         }
1512         *cs++ = MI_NOOP;
1513
1514         intel_ring_advance(req, cs);
1515
1516         ret = req->engine->emit_flush(req, EMIT_BARRIER);
1517         if (ret)
1518                 return ret;
1519
1520         return 0;
1521 }
1522
1523 static bool ring_is_idle(struct intel_engine_cs *engine)
1524 {
1525         struct drm_i915_private *dev_priv = engine->i915;
1526         bool idle = true;
1527
1528         intel_runtime_pm_get(dev_priv);
1529
1530         /* First check that no commands are left in the ring */
1531         if ((I915_READ_HEAD(engine) & HEAD_ADDR) !=
1532             (I915_READ_TAIL(engine) & TAIL_ADDR))
1533                 idle = false;
1534
1535         /* No bit for gen2, so assume the CS parser is idle */
1536         if (INTEL_GEN(dev_priv) > 2 && !(I915_READ_MODE(engine) & MODE_IDLE))
1537                 idle = false;
1538
1539         intel_runtime_pm_put(dev_priv);
1540
1541         return idle;
1542 }
1543
1544 /**
1545  * intel_engine_is_idle() - Report if the engine has finished process all work
1546  * @engine: the intel_engine_cs
1547  *
1548  * Return true if there are no requests pending, nothing left to be submitted
1549  * to hardware, and that the engine is idle.
1550  */
1551 bool intel_engine_is_idle(struct intel_engine_cs *engine)
1552 {
1553         struct drm_i915_private *dev_priv = engine->i915;
1554
1555         /* More white lies, if wedged, hw state is inconsistent */
1556         if (i915_terminally_wedged(&dev_priv->gpu_error))
1557                 return true;
1558
1559         /* Any inflight/incomplete requests? */
1560         if (!i915_seqno_passed(intel_engine_get_seqno(engine),
1561                                intel_engine_last_submit(engine)))
1562                 return false;
1563
1564         if (I915_SELFTEST_ONLY(engine->breadcrumbs.mock))
1565                 return true;
1566
1567         /* Interrupt/tasklet pending? */
1568         if (test_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted))
1569                 return false;
1570
1571         /* Waiting to drain ELSP? */
1572         if (READ_ONCE(engine->execlists.active))
1573                 return false;
1574
1575         /* ELSP is empty, but there are ready requests? */
1576         if (READ_ONCE(engine->execlists.first))
1577                 return false;
1578
1579         /* Ring stopped? */
1580         if (!ring_is_idle(engine))
1581                 return false;
1582
1583         return true;
1584 }
1585
1586 bool intel_engines_are_idle(struct drm_i915_private *dev_priv)
1587 {
1588         struct intel_engine_cs *engine;
1589         enum intel_engine_id id;
1590
1591         if (READ_ONCE(dev_priv->gt.active_requests))
1592                 return false;
1593
1594         /* If the driver is wedged, HW state may be very inconsistent and
1595          * report that it is still busy, even though we have stopped using it.
1596          */
1597         if (i915_terminally_wedged(&dev_priv->gpu_error))
1598                 return true;
1599
1600         for_each_engine(engine, dev_priv, id) {
1601                 if (!intel_engine_is_idle(engine))
1602                         return false;
1603         }
1604
1605         return true;
1606 }
1607
1608 /**
1609  * intel_engine_has_kernel_context:
1610  * @engine: the engine
1611  *
1612  * Returns true if the last context to be executed on this engine, or has been
1613  * executed if the engine is already idle, is the kernel context
1614  * (#i915.kernel_context).
1615  */
1616 bool intel_engine_has_kernel_context(const struct intel_engine_cs *engine)
1617 {
1618         const struct i915_gem_context * const kernel_context =
1619                 engine->i915->kernel_context;
1620         struct drm_i915_gem_request *rq;
1621
1622         lockdep_assert_held(&engine->i915->drm.struct_mutex);
1623
1624         /*
1625          * Check the last context seen by the engine. If active, it will be
1626          * the last request that remains in the timeline. When idle, it is
1627          * the last executed context as tracked by retirement.
1628          */
1629         rq = __i915_gem_active_peek(&engine->timeline->last_request);
1630         if (rq)
1631                 return rq->ctx == kernel_context;
1632         else
1633                 return engine->last_retired_context == kernel_context;
1634 }
1635
1636 void intel_engines_reset_default_submission(struct drm_i915_private *i915)
1637 {
1638         struct intel_engine_cs *engine;
1639         enum intel_engine_id id;
1640
1641         for_each_engine(engine, i915, id)
1642                 engine->set_default_submission(engine);
1643 }
1644
1645 void intel_engines_mark_idle(struct drm_i915_private *i915)
1646 {
1647         struct intel_engine_cs *engine;
1648         enum intel_engine_id id;
1649
1650         for_each_engine(engine, i915, id) {
1651                 intel_engine_disarm_breadcrumbs(engine);
1652                 i915_gem_batch_pool_fini(&engine->batch_pool);
1653                 tasklet_kill(&engine->execlists.irq_tasklet);
1654                 engine->execlists.no_priolist = false;
1655         }
1656 }
1657
1658 bool intel_engine_can_store_dword(struct intel_engine_cs *engine)
1659 {
1660         switch (INTEL_GEN(engine->i915)) {
1661         case 2:
1662                 return false; /* uses physical not virtual addresses */
1663         case 3:
1664                 /* maybe only uses physical not virtual addresses */
1665                 return !(IS_I915G(engine->i915) || IS_I915GM(engine->i915));
1666         case 6:
1667                 return engine->class != VIDEO_DECODE_CLASS; /* b0rked */
1668         default:
1669                 return true;
1670         }
1671 }
1672
1673 unsigned int intel_engines_has_context_isolation(struct drm_i915_private *i915)
1674 {
1675         struct intel_engine_cs *engine;
1676         enum intel_engine_id id;
1677         unsigned int which;
1678
1679         which = 0;
1680         for_each_engine(engine, i915, id)
1681                 if (engine->default_state)
1682                         which |= BIT(engine->uabi_class);
1683
1684         return which;
1685 }
1686
1687 static void print_request(struct drm_printer *m,
1688                           struct drm_i915_gem_request *rq,
1689                           const char *prefix)
1690 {
1691         drm_printf(m, "%s%x%s [%x:%x] prio=%d @ %dms: %s\n", prefix,
1692                    rq->global_seqno,
1693                    i915_gem_request_completed(rq) ? "!" : "",
1694                    rq->ctx->hw_id, rq->fence.seqno,
1695                    rq->priotree.priority,
1696                    jiffies_to_msecs(jiffies - rq->emitted_jiffies),
1697                    rq->timeline->common->name);
1698 }
1699
1700 void intel_engine_dump(struct intel_engine_cs *engine, struct drm_printer *m)
1701 {
1702         struct intel_breadcrumbs * const b = &engine->breadcrumbs;
1703         const struct intel_engine_execlists * const execlists = &engine->execlists;
1704         struct i915_gpu_error * const error = &engine->i915->gpu_error;
1705         struct drm_i915_private *dev_priv = engine->i915;
1706         struct drm_i915_gem_request *rq;
1707         struct rb_node *rb;
1708         u64 addr;
1709
1710         drm_printf(m, "%s\n", engine->name);
1711         drm_printf(m, "\tcurrent seqno %x, last %x, hangcheck %x [%d ms], inflight %d\n",
1712                    intel_engine_get_seqno(engine),
1713                    intel_engine_last_submit(engine),
1714                    engine->hangcheck.seqno,
1715                    jiffies_to_msecs(jiffies - engine->hangcheck.action_timestamp),
1716                    engine->timeline->inflight_seqnos);
1717         drm_printf(m, "\tReset count: %d\n",
1718                    i915_reset_engine_count(error, engine));
1719
1720         rcu_read_lock();
1721
1722         drm_printf(m, "\tRequests:\n");
1723
1724         rq = list_first_entry(&engine->timeline->requests,
1725                               struct drm_i915_gem_request, link);
1726         if (&rq->link != &engine->timeline->requests)
1727                 print_request(m, rq, "\t\tfirst  ");
1728
1729         rq = list_last_entry(&engine->timeline->requests,
1730                              struct drm_i915_gem_request, link);
1731         if (&rq->link != &engine->timeline->requests)
1732                 print_request(m, rq, "\t\tlast   ");
1733
1734         rq = i915_gem_find_active_request(engine);
1735         if (rq) {
1736                 print_request(m, rq, "\t\tactive ");
1737                 drm_printf(m,
1738                            "\t\t[head %04x, postfix %04x, tail %04x, batch 0x%08x_%08x]\n",
1739                            rq->head, rq->postfix, rq->tail,
1740                            rq->batch ? upper_32_bits(rq->batch->node.start) : ~0u,
1741                            rq->batch ? lower_32_bits(rq->batch->node.start) : ~0u);
1742         }
1743
1744         drm_printf(m, "\tRING_START: 0x%08x [0x%08x]\n",
1745                    I915_READ(RING_START(engine->mmio_base)),
1746                    rq ? i915_ggtt_offset(rq->ring->vma) : 0);
1747         drm_printf(m, "\tRING_HEAD:  0x%08x [0x%08x]\n",
1748                    I915_READ(RING_HEAD(engine->mmio_base)) & HEAD_ADDR,
1749                    rq ? rq->ring->head : 0);
1750         drm_printf(m, "\tRING_TAIL:  0x%08x [0x%08x]\n",
1751                    I915_READ(RING_TAIL(engine->mmio_base)) & TAIL_ADDR,
1752                    rq ? rq->ring->tail : 0);
1753         drm_printf(m, "\tRING_CTL:   0x%08x [%s]\n",
1754                    I915_READ(RING_CTL(engine->mmio_base)),
1755                    I915_READ(RING_CTL(engine->mmio_base)) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? "waiting" : "");
1756
1757         rcu_read_unlock();
1758
1759         addr = intel_engine_get_active_head(engine);
1760         drm_printf(m, "\tACTHD:  0x%08x_%08x\n",
1761                    upper_32_bits(addr), lower_32_bits(addr));
1762         addr = intel_engine_get_last_batch_head(engine);
1763         drm_printf(m, "\tBBADDR: 0x%08x_%08x\n",
1764                    upper_32_bits(addr), lower_32_bits(addr));
1765
1766         if (i915_modparams.enable_execlists) {
1767                 const u32 *hws = &engine->status_page.page_addr[I915_HWS_CSB_BUF0_INDEX];
1768                 u32 ptr, read, write;
1769                 unsigned int idx;
1770
1771                 drm_printf(m, "\tExeclist status: 0x%08x %08x\n",
1772                            I915_READ(RING_EXECLIST_STATUS_LO(engine)),
1773                            I915_READ(RING_EXECLIST_STATUS_HI(engine)));
1774
1775                 ptr = I915_READ(RING_CONTEXT_STATUS_PTR(engine));
1776                 read = GEN8_CSB_READ_PTR(ptr);
1777                 write = GEN8_CSB_WRITE_PTR(ptr);
1778                 drm_printf(m, "\tExeclist CSB read %d [%d cached], write %d [%d from hws], interrupt posted? %s\n",
1779                            read, execlists->csb_head,
1780                            write,
1781                            intel_read_status_page(engine, intel_hws_csb_write_index(engine->i915)),
1782                            yesno(test_bit(ENGINE_IRQ_EXECLIST,
1783                                           &engine->irq_posted)));
1784                 if (read >= GEN8_CSB_ENTRIES)
1785                         read = 0;
1786                 if (write >= GEN8_CSB_ENTRIES)
1787                         write = 0;
1788                 if (read > write)
1789                         write += GEN8_CSB_ENTRIES;
1790                 while (read < write) {
1791                         idx = ++read % GEN8_CSB_ENTRIES;
1792                         drm_printf(m, "\tExeclist CSB[%d]: 0x%08x [0x%08x in hwsp], context: %d [%d in hwsp]\n",
1793                                    idx,
1794                                    I915_READ(RING_CONTEXT_STATUS_BUF_LO(engine, idx)),
1795                                    hws[idx * 2],
1796                                    I915_READ(RING_CONTEXT_STATUS_BUF_HI(engine, idx)),
1797                                    hws[idx * 2 + 1]);
1798                 }
1799
1800                 rcu_read_lock();
1801                 for (idx = 0; idx < execlists_num_ports(execlists); idx++) {
1802                         unsigned int count;
1803
1804                         rq = port_unpack(&execlists->port[idx], &count);
1805                         if (rq) {
1806                                 drm_printf(m, "\t\tELSP[%d] count=%d, ",
1807                                            idx, count);
1808                                 print_request(m, rq, "rq: ");
1809                         } else {
1810                                 drm_printf(m, "\t\tELSP[%d] idle\n",
1811                                            idx);
1812                         }
1813                 }
1814                 drm_printf(m, "\t\tHW active? 0x%x\n", execlists->active);
1815                 rcu_read_unlock();
1816         } else if (INTEL_GEN(dev_priv) > 6) {
1817                 drm_printf(m, "\tPP_DIR_BASE: 0x%08x\n",
1818                            I915_READ(RING_PP_DIR_BASE(engine)));
1819                 drm_printf(m, "\tPP_DIR_BASE_READ: 0x%08x\n",
1820                            I915_READ(RING_PP_DIR_BASE_READ(engine)));
1821                 drm_printf(m, "\tPP_DIR_DCLV: 0x%08x\n",
1822                            I915_READ(RING_PP_DIR_DCLV(engine)));
1823         }
1824
1825         spin_lock_irq(&engine->timeline->lock);
1826         list_for_each_entry(rq, &engine->timeline->requests, link)
1827                 print_request(m, rq, "\t\tE ");
1828         for (rb = execlists->first; rb; rb = rb_next(rb)) {
1829                 struct i915_priolist *p =
1830                         rb_entry(rb, typeof(*p), node);
1831
1832                 list_for_each_entry(rq, &p->requests, priotree.link)
1833                         print_request(m, rq, "\t\tQ ");
1834         }
1835         spin_unlock_irq(&engine->timeline->lock);
1836
1837         spin_lock_irq(&b->rb_lock);
1838         for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) {
1839                 struct intel_wait *w = rb_entry(rb, typeof(*w), node);
1840
1841                 drm_printf(m, "\t%s [%d] waiting for %x\n",
1842                            w->tsk->comm, w->tsk->pid, w->seqno);
1843         }
1844         spin_unlock_irq(&b->rb_lock);
1845
1846         drm_printf(m, "\n");
1847 }
1848
1849 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1850 #include "selftests/mock_engine.c"
1851 #endif