[mirror_ubuntu-focal-kernel.git] / drivers / gpu / drm / i915 / intel_engine_cs.c

/*
 * Copyright © 2016 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 */

#include "i915_drv.h"
#include "intel_ringbuffer.h"
#include "intel_lrc.h"

static const struct engine_info {
	const char *name;
	unsigned exec_id;
	enum intel_engine_hw_id hw_id;
	u32 mmio_base;
	unsigned irq_shift;
	int (*init_legacy)(struct intel_engine_cs *engine);
	int (*init_execlists)(struct intel_engine_cs *engine);
} intel_engines[] = {
	[RCS] = {
		.name = "render ring",
		.exec_id = I915_EXEC_RENDER,
		.hw_id = RCS_HW,
		.mmio_base = RENDER_RING_BASE,
		.irq_shift = GEN8_RCS_IRQ_SHIFT,
		.init_execlists = logical_render_ring_init,
		.init_legacy = intel_init_render_ring_buffer,
	},
	[BCS] = {
		.name = "blitter ring",
		.exec_id = I915_EXEC_BLT,
		.hw_id = BCS_HW,
		.mmio_base = BLT_RING_BASE,
		.irq_shift = GEN8_BCS_IRQ_SHIFT,
		.init_execlists = logical_xcs_ring_init,
		.init_legacy = intel_init_blt_ring_buffer,
	},
	[VCS] = {
		.name = "bsd ring",
		.exec_id = I915_EXEC_BSD,
		.hw_id = VCS_HW,
		.mmio_base = GEN6_BSD_RING_BASE,
		.irq_shift = GEN8_VCS1_IRQ_SHIFT,
		.init_execlists = logical_xcs_ring_init,
		.init_legacy = intel_init_bsd_ring_buffer,
	},
	[VCS2] = {
		.name = "bsd2 ring",
		.exec_id = I915_EXEC_BSD,
		.hw_id = VCS2_HW,
		.mmio_base = GEN8_BSD2_RING_BASE,
		.irq_shift = GEN8_VCS2_IRQ_SHIFT,
		.init_execlists = logical_xcs_ring_init,
		.init_legacy = intel_init_bsd2_ring_buffer,
	},
	[VECS] = {
		.name = "video enhancement ring",
		.exec_id = I915_EXEC_VEBOX,
		.hw_id = VECS_HW,
		.mmio_base = VEBOX_RING_BASE,
		.irq_shift = GEN8_VECS_IRQ_SHIFT,
		.init_execlists = logical_xcs_ring_init,
		.init_legacy = intel_init_vebox_ring_buffer,
	},
};

static int
intel_engine_setup(struct drm_i915_private *dev_priv,
		   enum intel_engine_id id)
{
	const struct engine_info *info = &intel_engines[id];
	struct intel_engine_cs *engine;

	GEM_BUG_ON(dev_priv->engine[id]);
	engine = kzalloc(sizeof(*engine), GFP_KERNEL);
	if (!engine)
		return -ENOMEM;

	engine->id = id;
	engine->i915 = dev_priv;
	engine->name = info->name;
	engine->exec_id = info->exec_id;
	engine->hw_id = engine->guc_id = info->hw_id;
	engine->mmio_base = info->mmio_base;
	engine->irq_shift = info->irq_shift;

	/* Nothing to do here, execute in order of dependencies */
	engine->schedule = NULL;

	dev_priv->engine[id] = engine;
	return 0;
}

/**
 * intel_engines_init() - allocate, populate and init the Engine Command Streamers
 * @dev_priv: i915 device private
 *
 * Return: non-zero if the initialization failed.
 */
int intel_engines_init(struct drm_i915_private *dev_priv)
{
	struct intel_device_info *device_info = mkwrite_device_info(dev_priv);
	unsigned int ring_mask = INTEL_INFO(dev_priv)->ring_mask;
	unsigned int mask = 0;
	int (*init)(struct intel_engine_cs *engine);
	struct intel_engine_cs *engine;
	enum intel_engine_id id;
	unsigned int i;
	int ret;

	WARN_ON(ring_mask == 0);
	WARN_ON(ring_mask &
		GENMASK(sizeof(mask) * BITS_PER_BYTE - 1, I915_NUM_ENGINES));

	for (i = 0; i < ARRAY_SIZE(intel_engines); i++) {
		if (!HAS_ENGINE(dev_priv, i))
			continue;

		if (i915.enable_execlists)
			init = intel_engines[i].init_execlists;
		else
			init = intel_engines[i].init_legacy;

		if (!init)
			continue;

		ret = intel_engine_setup(dev_priv, i);
		if (ret)
			goto cleanup;

		ret = init(dev_priv->engine[i]);
		if (ret)
			goto cleanup;

		mask |= ENGINE_MASK(i);
	}

	/*
	 * Catch failures to update intel_engines table when the new engines
	 * are added to the driver by a warning and disabling the forgotten
	 * engines.
	 */
	if (WARN_ON(mask != ring_mask))
		device_info->ring_mask = mask;

	device_info->num_rings = hweight32(mask);

	return 0;

cleanup:
	for_each_engine(engine, dev_priv, id) {
		if (i915.enable_execlists)
			intel_logical_ring_cleanup(engine);
		else
			intel_engine_cleanup(engine);
	}

	return ret;
}

void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno)
{
	struct drm_i915_private *dev_priv = engine->i915;

	/* Our semaphore implementation is strictly monotonic (i.e. we proceed
	 * so long as the semaphore value in the register/page is greater
	 * than the sync value), so whenever we reset the seqno,
	 * so long as we reset the tracking semaphore value to 0, it will
	 * always be before the next request's seqno. If we don't reset
	 * the semaphore value, then when the seqno moves backwards all
	 * future waits will complete instantly (causing rendering corruption).
	 */
	if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) {
		I915_WRITE(RING_SYNC_0(engine->mmio_base), 0);
		I915_WRITE(RING_SYNC_1(engine->mmio_base), 0);
		if (HAS_VEBOX(dev_priv))
			I915_WRITE(RING_SYNC_2(engine->mmio_base), 0);
	}
	if (dev_priv->semaphore) {
		struct page *page = i915_vma_first_page(dev_priv->semaphore);
		void *semaphores;

		/* Semaphores are in noncoherent memory, flush to be safe */
		semaphores = kmap(page);
		memset(semaphores + GEN8_SEMAPHORE_OFFSET(engine->id, 0),
		       0, I915_NUM_ENGINES * gen8_semaphore_seqno_size);
		drm_clflush_virt_range(semaphores + GEN8_SEMAPHORE_OFFSET(engine->id, 0),
				       I915_NUM_ENGINES * gen8_semaphore_seqno_size);
		kunmap(page);
	}

	intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
	if (engine->irq_seqno_barrier)
		engine->irq_seqno_barrier(engine);

	GEM_BUG_ON(i915_gem_active_isset(&engine->timeline->last_request));
	engine->timeline->last_submitted_seqno = seqno;

	engine->hangcheck.seqno = seqno;

	/* After manually advancing the seqno, fake the interrupt in case
	 * there are any waiters for that seqno.
	 */
	intel_engine_wakeup(engine);
}

static void intel_engine_init_timeline(struct intel_engine_cs *engine)
{
	engine->timeline = &engine->i915->gt.global_timeline.engine[engine->id];
}

/**
 * intel_engines_setup_common - setup engine state not requiring hw access
 * @engine: Engine to setup.
 *
 * Initializes @engine@ structure members shared between legacy and execlists
 * submission modes which do not require hardware access.
 *
 * Typically done early in the submission mode specific engine setup stage.
 */
void intel_engine_setup_common(struct intel_engine_cs *engine)
{
	engine->execlist_queue = RB_ROOT;
	engine->execlist_first = NULL;

	intel_engine_init_timeline(engine);
	intel_engine_init_hangcheck(engine);
	i915_gem_batch_pool_init(engine, &engine->batch_pool);

	intel_engine_init_cmd_parser(engine);
}

int intel_engine_create_scratch(struct intel_engine_cs *engine, int size)
{
	struct drm_i915_gem_object *obj;
	struct i915_vma *vma;
	int ret;

	WARN_ON(engine->scratch);

	obj = i915_gem_object_create_stolen(engine->i915, size);
	if (!obj)
		obj = i915_gem_object_create_internal(engine->i915, size);
	if (IS_ERR(obj)) {
		DRM_ERROR("Failed to allocate scratch page\n");
		return PTR_ERR(obj);
	}

	vma = i915_vma_create(obj, &engine->i915->ggtt.base, NULL);
	if (IS_ERR(vma)) {
		ret = PTR_ERR(vma);
		goto err_unref;
	}

	ret = i915_vma_pin(vma, 0, 4096, PIN_GLOBAL | PIN_HIGH);
	if (ret)
		goto err_unref;

	engine->scratch = vma;
	DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
			 engine->name, i915_ggtt_offset(vma));
	return 0;

err_unref:
	i915_gem_object_put(obj);
	return ret;
}

static void intel_engine_cleanup_scratch(struct intel_engine_cs *engine)
{
	i915_vma_unpin_and_release(&engine->scratch);
}

/**
 * intel_engines_init_common - initialize cengine state which might require hw access
 * @engine: Engine to initialize.
 *
 * Initializes @engine@ structure members shared between legacy and execlists
 * submission modes which do require hardware access.
 *
 * Typcally done at later stages of submission mode specific engine setup.
 *
 * Returns zero on success or an error code on failure.
 */
int intel_engine_init_common(struct intel_engine_cs *engine)
{
	int ret;

	/* We may need to do things with the shrinker which
	 * require us to immediately switch back to the default
	 * context. This can cause a problem as pinning the
	 * default context also requires GTT space which may not
	 * be available. To avoid this we always pin the default
	 * context.
	 */
	ret = engine->context_pin(engine, engine->i915->kernel_context);
	if (ret)
		return ret;

	ret = intel_engine_init_breadcrumbs(engine);
	if (ret)
		goto err_unpin;

	ret = i915_gem_render_state_init(engine);
	if (ret)
		goto err_unpin;

	return 0;

err_unpin:
	engine->context_unpin(engine, engine->i915->kernel_context);
	return ret;
}

/**
 * intel_engines_cleanup_common - cleans up the engine state created by
 *                                the common initiailizers.
 * @engine: Engine to cleanup.
 *
 * This cleans up everything created by the common helpers.
 */
void intel_engine_cleanup_common(struct intel_engine_cs *engine)
{
	intel_engine_cleanup_scratch(engine);

	i915_gem_render_state_fini(engine);
	intel_engine_fini_breadcrumbs(engine);
	intel_engine_cleanup_cmd_parser(engine);
	i915_gem_batch_pool_fini(&engine->batch_pool);

	engine->context_unpin(engine, engine->i915->kernel_context);
}

u64 intel_engine_get_active_head(struct intel_engine_cs *engine)
{
	struct drm_i915_private *dev_priv = engine->i915;
	u64 acthd;

	if (INTEL_GEN(dev_priv) >= 8)
		acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base),
					 RING_ACTHD_UDW(engine->mmio_base));
	else if (INTEL_GEN(dev_priv) >= 4)
		acthd = I915_READ(RING_ACTHD(engine->mmio_base));
	else
		acthd = I915_READ(ACTHD);

	return acthd;
}

u64 intel_engine_get_last_batch_head(struct intel_engine_cs *engine)
{
	struct drm_i915_private *dev_priv = engine->i915;
	u64 bbaddr;

	if (INTEL_GEN(dev_priv) >= 8)
		bbaddr = I915_READ64_2x32(RING_BBADDR(engine->mmio_base),
					  RING_BBADDR_UDW(engine->mmio_base));
	else
		bbaddr = I915_READ(RING_BBADDR(engine->mmio_base));

	return bbaddr;
}

const char *i915_cache_level_str(struct drm_i915_private *i915, int type)
{
	switch (type) {
	case I915_CACHE_NONE: return " uncached";
	case I915_CACHE_LLC: return HAS_LLC(i915) ? " LLC" : " snooped";
	case I915_CACHE_L3_LLC: return " L3+LLC";
	case I915_CACHE_WT: return " WT";
	default: return "";
	}
}

static inline uint32_t
read_subslice_reg(struct drm_i915_private *dev_priv, int slice,
		  int subslice, i915_reg_t reg)
{
	uint32_t mcr;
	uint32_t ret;
	enum forcewake_domains fw_domains;

	fw_domains = intel_uncore_forcewake_for_reg(dev_priv, reg,
						    FW_REG_READ);
	fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
						     GEN8_MCR_SELECTOR,
						     FW_REG_READ | FW_REG_WRITE);

	spin_lock_irq(&dev_priv->uncore.lock);
	intel_uncore_forcewake_get__locked(dev_priv, fw_domains);

	mcr = I915_READ_FW(GEN8_MCR_SELECTOR);
	/*
	 * The HW expects the slice and sublice selectors to be reset to 0
	 * after reading out the registers.
	 */
	WARN_ON_ONCE(mcr & (GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK));
	mcr &= ~(GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK);
	mcr |= GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice);
	I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);

	ret = I915_READ_FW(reg);

	mcr &= ~(GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK);
	I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);

	intel_uncore_forcewake_put__locked(dev_priv, fw_domains);
	spin_unlock_irq(&dev_priv->uncore.lock);

	return ret;
}

/* NB: please notice the memset */
void intel_engine_get_instdone(struct intel_engine_cs *engine,
			       struct intel_instdone *instdone)
{
	struct drm_i915_private *dev_priv = engine->i915;
	u32 mmio_base = engine->mmio_base;
	int slice;
	int subslice;

	memset(instdone, 0, sizeof(*instdone));

	switch (INTEL_GEN(dev_priv)) {
	default:
		instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));

		if (engine->id != RCS)
			break;

		instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
		for_each_instdone_slice_subslice(dev_priv, slice, subslice) {
			instdone->sampler[slice][subslice] =
				read_subslice_reg(dev_priv, slice, subslice,
						  GEN7_SAMPLER_INSTDONE);
			instdone->row[slice][subslice] =
				read_subslice_reg(dev_priv, slice, subslice,
						  GEN7_ROW_INSTDONE);
		}
		break;
	case 7:
		instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));

		if (engine->id != RCS)
			break;

		instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
		instdone->sampler[0][0] = I915_READ(GEN7_SAMPLER_INSTDONE);
		instdone->row[0][0] = I915_READ(GEN7_ROW_INSTDONE);

		break;
	case 6:
	case 5:
	case 4:
		instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));

		if (engine->id == RCS)
			/* HACK: Using the wrong struct member */
			instdone->slice_common = I915_READ(GEN4_INSTDONE1);
		break;
	case 3:
	case 2:
		instdone->instdone = I915_READ(GEN2_INSTDONE);
		break;
	}
}
Commit	Line	Data
88d2ba2e TU	1	/*
	2	* Copyright © 2016 Intel Corporation
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice (including the next
	12	* paragraph) shall be included in all copies or substantial portions of the
	13	* Software.
	14	*
	15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
	20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
	21	* IN THE SOFTWARE.
	22	*
	23	*/
	24
	25	#include "i915_drv.h"
	26	#include "intel_ringbuffer.h"
	27	#include "intel_lrc.h"
	28
	29	static const struct engine_info {
	30	const char *name;
	31	unsigned exec_id;
5ec2cf7e	32	enum intel_engine_hw_id hw_id;
88d2ba2e TU	33	u32 mmio_base;
	34	unsigned irq_shift;
	35	int (init_legacy)(struct intel_engine_cs engine);
	36	int (init_execlists)(struct intel_engine_cs engine);
	37	} intel_engines[] = {
	38	[RCS] = {
	39	.name = "render ring",
	40	.exec_id = I915_EXEC_RENDER,
5ec2cf7e	41	.hw_id = RCS_HW,
88d2ba2e TU	42	.mmio_base = RENDER_RING_BASE,
	43	.irq_shift = GEN8_RCS_IRQ_SHIFT,
	44	.init_execlists = logical_render_ring_init,
	45	.init_legacy = intel_init_render_ring_buffer,
	46	},
	47	[BCS] = {
	48	.name = "blitter ring",
	49	.exec_id = I915_EXEC_BLT,
5ec2cf7e	50	.hw_id = BCS_HW,
88d2ba2e TU	51	.mmio_base = BLT_RING_BASE,
	52	.irq_shift = GEN8_BCS_IRQ_SHIFT,
	53	.init_execlists = logical_xcs_ring_init,
	54	.init_legacy = intel_init_blt_ring_buffer,
	55	},
	56	[VCS] = {
	57	.name = "bsd ring",
	58	.exec_id = I915_EXEC_BSD,
5ec2cf7e	59	.hw_id = VCS_HW,
88d2ba2e TU	60	.mmio_base = GEN6_BSD_RING_BASE,
	61	.irq_shift = GEN8_VCS1_IRQ_SHIFT,
	62	.init_execlists = logical_xcs_ring_init,
	63	.init_legacy = intel_init_bsd_ring_buffer,
	64	},
	65	[VCS2] = {
	66	.name = "bsd2 ring",
	67	.exec_id = I915_EXEC_BSD,
5ec2cf7e	68	.hw_id = VCS2_HW,
88d2ba2e TU	69	.mmio_base = GEN8_BSD2_RING_BASE,
	70	.irq_shift = GEN8_VCS2_IRQ_SHIFT,
	71	.init_execlists = logical_xcs_ring_init,
	72	.init_legacy = intel_init_bsd2_ring_buffer,
	73	},
	74	[VECS] = {
	75	.name = "video enhancement ring",
	76	.exec_id = I915_EXEC_VEBOX,
5ec2cf7e	77	.hw_id = VECS_HW,
88d2ba2e TU	78	.mmio_base = VEBOX_RING_BASE,
	79	.irq_shift = GEN8_VECS_IRQ_SHIFT,
	80	.init_execlists = logical_xcs_ring_init,
	81	.init_legacy = intel_init_vebox_ring_buffer,
	82	},
	83	};
	84
3b3f1650	85	static int
88d2ba2e TU	86	intel_engine_setup(struct drm_i915_private *dev_priv,
	87	enum intel_engine_id id)
	88	{
	89	const struct engine_info *info = &intel_engines[id];
3b3f1650 AG	90	struct intel_engine_cs *engine;
	91
	92	GEM_BUG_ON(dev_priv->engine[id]);
	93	engine = kzalloc(sizeof(*engine), GFP_KERNEL);
	94	if (!engine)
	95	return -ENOMEM;
88d2ba2e TU	96
	97	engine->id = id;
	98	engine->i915 = dev_priv;
	99	engine->name = info->name;
	100	engine->exec_id = info->exec_id;
5ec2cf7e	101	engine->hw_id = engine->guc_id = info->hw_id;
88d2ba2e TU	102	engine->mmio_base = info->mmio_base;
	103	engine->irq_shift = info->irq_shift;
	104
0de9136d CW	105	/* Nothing to do here, execute in order of dependencies */
	106	engine->schedule = NULL;
	107
3b3f1650 AG	108	dev_priv->engine[id] = engine;
3b3f1650 AG	109	return 0;
88d2ba2e TU	110	}
	111
	112	/**
	113	* intel_engines_init() - allocate, populate and init the Engine Command Streamers
bf9e8429	114	* @dev_priv: i915 device private
88d2ba2e TU	115	*
	116	* Return: non-zero if the initialization failed.
	117	*/
bf9e8429	118	int intel_engines_init(struct drm_i915_private *dev_priv)
88d2ba2e	119	{
c1bb1145	120	struct intel_device_info *device_info = mkwrite_device_info(dev_priv);
70006ad6	121	unsigned int ring_mask = INTEL_INFO(dev_priv)->ring_mask;
88d2ba2e TU	122	unsigned int mask = 0;
88d2ba2e TU	123	int (init)(struct intel_engine_cs engine);
3b3f1650 AG	124	struct intel_engine_cs *engine;
3b3f1650 AG	125	enum intel_engine_id id;
88d2ba2e TU	126	unsigned int i;
	127	int ret;
	128
70006ad6 TU	129	WARN_ON(ring_mask == 0);
70006ad6 TU	130	WARN_ON(ring_mask &
88d2ba2e TU	131	GENMASK(sizeof(mask) * BITS_PER_BYTE - 1, I915_NUM_ENGINES));
	132
	133	for (i = 0; i < ARRAY_SIZE(intel_engines); i++) {
	134	if (!HAS_ENGINE(dev_priv, i))
	135	continue;
	136
	137	if (i915.enable_execlists)
	138	init = intel_engines[i].init_execlists;
	139	else
	140	init = intel_engines[i].init_legacy;
	141
	142	if (!init)
	143	continue;
	144
3b3f1650 AG	145	ret = intel_engine_setup(dev_priv, i);
	146	if (ret)
	147	goto cleanup;
	148
	149	ret = init(dev_priv->engine[i]);
88d2ba2e TU	150	if (ret)
	151	goto cleanup;
	152
	153	mask \|= ENGINE_MASK(i);
	154	}
	155
	156	/*
	157	* Catch failures to update intel_engines table when the new engines
	158	* are added to the driver by a warning and disabling the forgotten
	159	* engines.
	160	*/
70006ad6	161	if (WARN_ON(mask != ring_mask))
c1bb1145 TU	162	device_info->ring_mask = mask;
	163
	164	device_info->num_rings = hweight32(mask);
88d2ba2e TU	165
	166	return 0;
	167
	168	cleanup:
3b3f1650	169	for_each_engine(engine, dev_priv, id) {
88d2ba2e	170	if (i915.enable_execlists)
3b3f1650	171	intel_logical_ring_cleanup(engine);
88d2ba2e	172	else
3b3f1650	173	intel_engine_cleanup(engine);
88d2ba2e TU	174	}
	175
	176	return ret;
	177	}
	178
73cb9701	179	void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno)
57f275a2 CW	180	{
	181	struct drm_i915_private *dev_priv = engine->i915;
	182
	183	/* Our semaphore implementation is strictly monotonic (i.e. we proceed
	184	* so long as the semaphore value in the register/page is greater
	185	* than the sync value), so whenever we reset the seqno,
	186	* so long as we reset the tracking semaphore value to 0, it will
	187	* always be before the next request's seqno. If we don't reset
	188	* the semaphore value, then when the seqno moves backwards all
	189	* future waits will complete instantly (causing rendering corruption).
	190	*/
	191	if (IS_GEN6(dev_priv) \|\| IS_GEN7(dev_priv)) {
	192	I915_WRITE(RING_SYNC_0(engine->mmio_base), 0);
	193	I915_WRITE(RING_SYNC_1(engine->mmio_base), 0);
	194	if (HAS_VEBOX(dev_priv))
	195	I915_WRITE(RING_SYNC_2(engine->mmio_base), 0);
	196	}
51d545d0 CW	197	if (dev_priv->semaphore) {
	198	struct page *page = i915_vma_first_page(dev_priv->semaphore);
	199	void *semaphores;
	200
	201	/* Semaphores are in noncoherent memory, flush to be safe */
	202	semaphores = kmap(page);
57f275a2 CW	203	memset(semaphores + GEN8_SEMAPHORE_OFFSET(engine->id, 0),
57f275a2 CW	204	0, I915_NUM_ENGINES * gen8_semaphore_seqno_size);
51d545d0 CW	205	drm_clflush_virt_range(semaphores + GEN8_SEMAPHORE_OFFSET(engine->id, 0),
51d545d0 CW	206	I915_NUM_ENGINES * gen8_semaphore_seqno_size);
57f275a2 CW	207	kunmap(page);
57f275a2 CW	208	}
57f275a2 CW	209
	210	intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
	211	if (engine->irq_seqno_barrier)
	212	engine->irq_seqno_barrier(engine);
73cb9701 CW	213
	214	GEM_BUG_ON(i915_gem_active_isset(&engine->timeline->last_request));
	215	engine->timeline->last_submitted_seqno = seqno;
57f275a2 CW	216
	217	engine->hangcheck.seqno = seqno;
	218
	219	/* After manually advancing the seqno, fake the interrupt in case
	220	* there are any waiters for that seqno.
	221	*/
	222	intel_engine_wakeup(engine);
	223	}
	224
73cb9701	225	static void intel_engine_init_timeline(struct intel_engine_cs *engine)
dcff85c8	226	{
73cb9701	227	engine->timeline = &engine->i915->gt.global_timeline.engine[engine->id];
dcff85c8 CW	228	}
dcff85c8 CW	229
019bf277 TU	230	/**
	231	* intel_engines_setup_common - setup engine state not requiring hw access
	232	* @engine: Engine to setup.
	233	*
	234	* Initializes @engine@ structure members shared between legacy and execlists
	235	* submission modes which do not require hardware access.
	236	*
	237	* Typically done early in the submission mode specific engine setup stage.
	238	*/
	239	void intel_engine_setup_common(struct intel_engine_cs *engine)
	240	{
20311bd3 CW	241	engine->execlist_queue = RB_ROOT;
20311bd3 CW	242	engine->execlist_first = NULL;
019bf277	243
73cb9701	244	intel_engine_init_timeline(engine);
019bf277	245	intel_engine_init_hangcheck(engine);
115003e9	246	i915_gem_batch_pool_init(engine, &engine->batch_pool);
7756e454 CW	247
7756e454 CW	248	intel_engine_init_cmd_parser(engine);
019bf277 TU	249	}
019bf277 TU	250
adc320c4 CW	251	int intel_engine_create_scratch(struct intel_engine_cs *engine, int size)
	252	{
	253	struct drm_i915_gem_object *obj;
	254	struct i915_vma *vma;
	255	int ret;
	256
	257	WARN_ON(engine->scratch);
	258
187685cb	259	obj = i915_gem_object_create_stolen(engine->i915, size);
adc320c4	260	if (!obj)
920cf419	261	obj = i915_gem_object_create_internal(engine->i915, size);
adc320c4 CW	262	if (IS_ERR(obj)) {
	263	DRM_ERROR("Failed to allocate scratch page\n");
	264	return PTR_ERR(obj);
	265	}
	266
	267	vma = i915_vma_create(obj, &engine->i915->ggtt.base, NULL);
	268	if (IS_ERR(vma)) {
	269	ret = PTR_ERR(vma);
	270	goto err_unref;
	271	}
	272
	273	ret = i915_vma_pin(vma, 0, 4096, PIN_GLOBAL \| PIN_HIGH);
	274	if (ret)
	275	goto err_unref;
	276
	277	engine->scratch = vma;
bde13ebd CW	278	DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
bde13ebd CW	279	engine->name, i915_ggtt_offset(vma));
adc320c4 CW	280	return 0;
	281
	282	err_unref:
	283	i915_gem_object_put(obj);
	284	return ret;
	285	}
	286
	287	static void intel_engine_cleanup_scratch(struct intel_engine_cs *engine)
	288	{
19880c4a	289	i915_vma_unpin_and_release(&engine->scratch);
adc320c4 CW	290	}
adc320c4 CW	291
019bf277 TU	292	/**
	293	* intel_engines_init_common - initialize cengine state which might require hw access
	294	* @engine: Engine to initialize.
	295	*
	296	* Initializes @engine@ structure members shared between legacy and execlists
	297	* submission modes which do require hardware access.
	298	*
	299	* Typcally done at later stages of submission mode specific engine setup.
	300	*
	301	* Returns zero on success or an error code on failure.
	302	*/
	303	int intel_engine_init_common(struct intel_engine_cs *engine)
	304	{
	305	int ret;
	306
e8a9c58f CW	307	/* We may need to do things with the shrinker which
	308	* require us to immediately switch back to the default
	309	* context. This can cause a problem as pinning the
	310	* default context also requires GTT space which may not
	311	* be available. To avoid this we always pin the default
	312	* context.
	313	*/
	314	ret = engine->context_pin(engine, engine->i915->kernel_context);
019bf277 TU	315	if (ret)
	316	return ret;
	317
e8a9c58f CW	318	ret = intel_engine_init_breadcrumbs(engine);
	319	if (ret)
	320	goto err_unpin;
	321
4e50f082 CW	322	ret = i915_gem_render_state_init(engine);
4e50f082 CW	323	if (ret)
e8a9c58f	324	goto err_unpin;
4e50f082	325
7756e454	326	return 0;
e8a9c58f CW	327
	328	err_unpin:
	329	engine->context_unpin(engine, engine->i915->kernel_context);
	330	return ret;
019bf277	331	}
96a945aa CW	332
	333	/**
	334	* intel_engines_cleanup_common - cleans up the engine state created by
	335	* the common initiailizers.
	336	* @engine: Engine to cleanup.
	337	*
	338	* This cleans up everything created by the common helpers.
	339	*/
	340	void intel_engine_cleanup_common(struct intel_engine_cs *engine)
	341	{
adc320c4 CW	342	intel_engine_cleanup_scratch(engine);
adc320c4 CW	343
4e50f082	344	i915_gem_render_state_fini(engine);
96a945aa	345	intel_engine_fini_breadcrumbs(engine);
7756e454	346	intel_engine_cleanup_cmd_parser(engine);
96a945aa	347	i915_gem_batch_pool_fini(&engine->batch_pool);
e8a9c58f CW	348
e8a9c58f CW	349	engine->context_unpin(engine, engine->i915->kernel_context);
96a945aa	350	}
1b36595f CW	351
	352	u64 intel_engine_get_active_head(struct intel_engine_cs *engine)
	353	{
	354	struct drm_i915_private *dev_priv = engine->i915;
	355	u64 acthd;
	356
	357	if (INTEL_GEN(dev_priv) >= 8)
	358	acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base),
	359	RING_ACTHD_UDW(engine->mmio_base));
	360	else if (INTEL_GEN(dev_priv) >= 4)
	361	acthd = I915_READ(RING_ACTHD(engine->mmio_base));
	362	else
	363	acthd = I915_READ(ACTHD);
	364
	365	return acthd;
	366	}
	367
	368	u64 intel_engine_get_last_batch_head(struct intel_engine_cs *engine)
	369	{
	370	struct drm_i915_private *dev_priv = engine->i915;
	371	u64 bbaddr;
	372
	373	if (INTEL_GEN(dev_priv) >= 8)
	374	bbaddr = I915_READ64_2x32(RING_BBADDR(engine->mmio_base),
	375	RING_BBADDR_UDW(engine->mmio_base));
	376	else
	377	bbaddr = I915_READ(RING_BBADDR(engine->mmio_base));
	378
	379	return bbaddr;
	380	}
0e704476 CW	381
	382	const char i915_cache_level_str(struct drm_i915_private i915, int type)
	383	{
	384	switch (type) {
	385	case I915_CACHE_NONE: return " uncached";
	386	case I915_CACHE_LLC: return HAS_LLC(i915) ? " LLC" : " snooped";
	387	case I915_CACHE_L3_LLC: return " L3+LLC";
	388	case I915_CACHE_WT: return " WT";
	389	default: return "";
	390	}
	391	}
	392
	393	static inline uint32_t
	394	read_subslice_reg(struct drm_i915_private *dev_priv, int slice,
	395	int subslice, i915_reg_t reg)
	396	{
	397	uint32_t mcr;
	398	uint32_t ret;
	399	enum forcewake_domains fw_domains;
	400
	401	fw_domains = intel_uncore_forcewake_for_reg(dev_priv, reg,
	402	FW_REG_READ);
	403	fw_domains \|= intel_uncore_forcewake_for_reg(dev_priv,
	404	GEN8_MCR_SELECTOR,
	405	FW_REG_READ \| FW_REG_WRITE);
	406
	407	spin_lock_irq(&dev_priv->uncore.lock);
	408	intel_uncore_forcewake_get__locked(dev_priv, fw_domains);
	409
	410	mcr = I915_READ_FW(GEN8_MCR_SELECTOR);
	411	/*
	412	* The HW expects the slice and sublice selectors to be reset to 0
	413	* after reading out the registers.
	414	*/
	415	WARN_ON_ONCE(mcr & (GEN8_MCR_SLICE_MASK \| GEN8_MCR_SUBSLICE_MASK));
	416	mcr &= ~(GEN8_MCR_SLICE_MASK \| GEN8_MCR_SUBSLICE_MASK);
	417	mcr \|= GEN8_MCR_SLICE(slice) \| GEN8_MCR_SUBSLICE(subslice);
	418	I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);
	419
	420	ret = I915_READ_FW(reg);
	421
	422	mcr &= ~(GEN8_MCR_SLICE_MASK \| GEN8_MCR_SUBSLICE_MASK);
	423	I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);
	424
	425	intel_uncore_forcewake_put__locked(dev_priv, fw_domains);
	426	spin_unlock_irq(&dev_priv->uncore.lock);
	427
	428	return ret;
	429	}
	430
	431	/* NB: please notice the memset */
	432	void intel_engine_get_instdone(struct intel_engine_cs *engine,
	433	struct intel_instdone *instdone)
	434	{
	435	struct drm_i915_private *dev_priv = engine->i915;
	436	u32 mmio_base = engine->mmio_base;
	437	int slice;
	438	int subslice;
	439
	440	memset(instdone, 0, sizeof(*instdone));
	441
	442	switch (INTEL_GEN(dev_priv)) {
	443	default:
	444	instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
445
446	if (engine->id != RCS)
447	break;
448
449	instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
450	for_each_instdone_slice_subslice(dev_priv, slice, subslice) {
451	instdone->sampler[slice][subslice] =
452	read_subslice_reg(dev_priv, slice, subslice,
453	GEN7_SAMPLER_INSTDONE);
454	instdone->row[slice][subslice] =
455	read_subslice_reg(dev_priv, slice, subslice,
456	GEN7_ROW_INSTDONE);
457	}
458	break;
459	case 7:
460	instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
461
462	if (engine->id != RCS)
463	break;
464
465	instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
466	instdone->sampler[0][0] = I915_READ(GEN7_SAMPLER_INSTDONE);
467	instdone->row[0][0] = I915_READ(GEN7_ROW_INSTDONE);
468
469	break;
470	case 6:
471	case 5:
472	case 4:
473	instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
474
475	if (engine->id == RCS)
476	/* HACK: Using the wrong struct member */
477	instdone->slice_common = I915_READ(GEN4_INSTDONE1);
478	break;
479	case 3:
480	case 2:
481	instdone->instdone = I915_READ(GEN2_INSTDONE);
482	break;
483	}
484	}