[mirror_ubuntu-bionic-kernel.git] / drivers / gpu / drm / i915 / i915_gem_evict.c

/*
 * Copyright © 2008-2010 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eric Anholt <eric@anholt.net>
 *    Chris Wilson <chris@chris-wilson.co.uuk>
 *
 */

#include "drmP.h"
#include "drm.h"
#include "i915_drv.h"
#include "i915_drm.h"
#include "i915_trace.h"

static bool
mark_free(struct drm_i915_gem_object *obj, struct list_head *unwind)
{
	list_add(&obj->exec_list, unwind);
	drm_gem_object_reference(&obj->base);
	return drm_mm_scan_add_block(obj->gtt_space);
}

int
i915_gem_evict_something(struct drm_device *dev, int min_size,
			 unsigned alignment, bool mappable)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	struct list_head eviction_list, unwind_list;
	struct drm_i915_gem_object *obj;
	int ret = 0;

	i915_gem_retire_requests(dev);

	/* Re-check for free space after retiring requests */
	if (mappable) {
		if (drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
						min_size, alignment, 0,
						dev_priv->mm.gtt_mappable_end,
						0))
			return 0;
	} else {
		if (drm_mm_search_free(&dev_priv->mm.gtt_space,
				       min_size, alignment, 0))
			return 0;
	}

	trace_i915_gem_evict(dev, min_size, alignment, mappable);

	/*
	 * The goal is to evict objects and amalgamate space in LRU order.
	 * The oldest idle objects reside on the inactive list, which is in
	 * retirement order. The next objects to retire are those on the (per
	 * ring) active list that do not have an outstanding flush. Once the
	 * hardware reports completion (the seqno is updated after the
	 * batchbuffer has been finished) the clean buffer objects would
	 * be retired to the inactive list. Any dirty objects would be added
	 * to the tail of the flushing list. So after processing the clean
	 * active objects we need to emit a MI_FLUSH to retire the flushing
	 * list, hence the retirement order of the flushing list is in
	 * advance of the dirty objects on the active lists.
	 *
	 * The retirement sequence is thus:
	 *   1. Inactive objects (already retired)
	 *   2. Clean active objects
	 *   3. Flushing list
	 *   4. Dirty active objects.
	 *
	 * On each list, the oldest objects lie at the HEAD with the freshest
	 * object on the TAIL.
	 */

	INIT_LIST_HEAD(&unwind_list);
	if (mappable)
		drm_mm_init_scan_with_range(&dev_priv->mm.gtt_space, min_size,
					    alignment, 0,
					    dev_priv->mm.gtt_mappable_end);
	else
		drm_mm_init_scan(&dev_priv->mm.gtt_space, min_size, alignment);

	/* First see if there is a large enough contiguous idle region... */
	list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) {
		if (mark_free(obj, &unwind_list))
			goto found;
	}

	/* Now merge in the soon-to-be-expired objects... */
	list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
		/* Does the object require an outstanding flush? */
		if (obj->base.write_domain || obj->pin_count)
			continue;

		if (mark_free(obj, &unwind_list))
			goto found;
	}

	/* Finally add anything with a pending flush (in order of retirement) */
	list_for_each_entry(obj, &dev_priv->mm.flushing_list, mm_list) {
		if (obj->pin_count)
			continue;

		if (mark_free(obj, &unwind_list))
			goto found;
	}
	list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
		if (! obj->base.write_domain || obj->pin_count)
			continue;

		if (mark_free(obj, &unwind_list))
			goto found;
	}

	/* Nothing found, clean up and bail out! */
	while (!list_empty(&unwind_list)) {
		obj = list_first_entry(&unwind_list,
				       struct drm_i915_gem_object,
				       exec_list);

		ret = drm_mm_scan_remove_block(obj->gtt_space);
		BUG_ON(ret);

		list_del_init(&obj->exec_list);
		drm_gem_object_unreference(&obj->base);
	}

	/* We expect the caller to unpin, evict all and try again, or give up.
	 * So calling i915_gem_evict_everything() is unnecessary.
	 */
	return -ENOSPC;

found:
	/* drm_mm doesn't allow any other other operations while
	 * scanning, therefore store to be evicted objects on a
	 * temporary list. */
	INIT_LIST_HEAD(&eviction_list);
	while (!list_empty(&unwind_list)) {
		obj = list_first_entry(&unwind_list,
				       struct drm_i915_gem_object,
				       exec_list);
		if (drm_mm_scan_remove_block(obj->gtt_space)) {
			list_move(&obj->exec_list, &eviction_list);
			continue;
		}
		list_del_init(&obj->exec_list);
		drm_gem_object_unreference(&obj->base);
	}

	/* Unbinding will emit any required flushes */
	while (!list_empty(&eviction_list)) {
		obj = list_first_entry(&eviction_list,
				       struct drm_i915_gem_object,
				       exec_list);
		if (ret == 0)
			ret = i915_gem_object_unbind(obj);

		list_del_init(&obj->exec_list);
		drm_gem_object_unreference(&obj->base);
	}

	return ret;
}

int
i915_gem_evict_everything(struct drm_device *dev, bool purgeable_only)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	int ret;
	bool lists_empty;

	lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
		       list_empty(&dev_priv->mm.flushing_list) &&
		       list_empty(&dev_priv->mm.active_list));
	if (lists_empty)
		return -ENOSPC;

	trace_i915_gem_evict_everything(dev, purgeable_only);

	/* Flush everything (on to the inactive lists) and evict */
	ret = i915_gpu_idle(dev);
	if (ret)
		return ret;

	BUG_ON(!list_empty(&dev_priv->mm.flushing_list));

	return i915_gem_evict_inactive(dev, purgeable_only);
}

/** Unbinds all inactive objects. */
int
i915_gem_evict_inactive(struct drm_device *dev, bool purgeable_only)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	struct drm_i915_gem_object *obj, *next;

	list_for_each_entry_safe(obj, next,
				 &dev_priv->mm.inactive_list, mm_list) {
		if (!purgeable_only || obj->madv != I915_MADV_WILLNEED) {
			int ret = i915_gem_object_unbind(obj);
			if (ret)
				return ret;
		}
	}

	return 0;
}
Commit	Line	Data
b47eb4a2 CW	1	/*
	2	* Copyright © 2008-2010 Intel Corporation
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice (including the next
	12	* paragraph) shall be included in all copies or substantial portions of the
	13	* Software.
	14	*
	15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
	20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
	21	* IN THE SOFTWARE.
	22	*
	23	* Authors:
	24	* Eric Anholt <eric@anholt.net>
	25	* Chris Wilson <chris@chris-wilson.co.uuk>
	26	*
	27	*/
	28
	29	#include "drmP.h"
	30	#include "drm.h"
	31	#include "i915_drv.h"
	32	#include "i915_drm.h"
db53a302	33	#include "i915_trace.h"
b47eb4a2	34
cd377ea9	35	static bool
05394f39	36	mark_free(struct drm_i915_gem_object obj, struct list_head unwind)
b47eb4a2	37	{
432e58ed	38	list_add(&obj->exec_list, unwind);
05394f39 CW	39	drm_gem_object_reference(&obj->base);
05394f39 CW	40	return drm_mm_scan_add_block(obj->gtt_space);
b47eb4a2 CW	41	}
	42
	43	int
a6e0aa42	44	i915_gem_evict_something(struct drm_device *dev, int min_size,
5eac3ab4	45	unsigned alignment, bool mappable)
b47eb4a2 CW	46	{
b47eb4a2 CW	47	drm_i915_private_t *dev_priv = dev->dev_private;
cd377ea9	48	struct list_head eviction_list, unwind_list;
05394f39	49	struct drm_i915_gem_object *obj;
cd377ea9	50	int ret = 0;
b47eb4a2	51
cd377ea9	52	i915_gem_retire_requests(dev);
b47eb4a2	53
cd377ea9	54	/* Re-check for free space after retiring requests */
a6e0aa42 DV	55	if (mappable) {
a6e0aa42 DV	56	if (drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
5eac3ab4	57	min_size, alignment, 0,
a6e0aa42 DV	58	dev_priv->mm.gtt_mappable_end,
	59	0))
	60	return 0;
	61	} else {
	62	if (drm_mm_search_free(&dev_priv->mm.gtt_space,
	63	min_size, alignment, 0))
	64	return 0;
	65	}
b47eb4a2	66
db53a302 CW	67	trace_i915_gem_evict(dev, min_size, alignment, mappable);
db53a302 CW	68
cd377ea9 CW	69	/*
	70	* The goal is to evict objects and amalgamate space in LRU order.
	71	* The oldest idle objects reside on the inactive list, which is in
	72	* retirement order. The next objects to retire are those on the (per
	73	* ring) active list that do not have an outstanding flush. Once the
	74	* hardware reports completion (the seqno is updated after the
	75	* batchbuffer has been finished) the clean buffer objects would
	76	* be retired to the inactive list. Any dirty objects would be added
	77	* to the tail of the flushing list. So after processing the clean
	78	* active objects we need to emit a MI_FLUSH to retire the flushing
	79	* list, hence the retirement order of the flushing list is in
	80	* advance of the dirty objects on the active lists.
	81	*
	82	* The retirement sequence is thus:
	83	* 1. Inactive objects (already retired)
	84	* 2. Clean active objects
	85	* 3. Flushing list
	86	* 4. Dirty active objects.
	87	*
	88	* On each list, the oldest objects lie at the HEAD with the freshest
	89	* object on the TAIL.
	90	*/
	91
	92	INIT_LIST_HEAD(&unwind_list);
a6e0aa42 DV	93	if (mappable)
	94	drm_mm_init_scan_with_range(&dev_priv->mm.gtt_space, min_size,
	95	alignment, 0,
	96	dev_priv->mm.gtt_mappable_end);
	97	else
	98	drm_mm_init_scan(&dev_priv->mm.gtt_space, min_size, alignment);
cd377ea9 CW	99
cd377ea9 CW	100	/* First see if there is a large enough contiguous idle region... */
05394f39 CW	101	list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) {
05394f39 CW	102	if (mark_free(obj, &unwind_list))
cd377ea9 CW	103	goto found;
cd377ea9 CW	104	}
b47eb4a2	105
cd377ea9	106	/* Now merge in the soon-to-be-expired objects... */
05394f39	107	list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
cd377ea9	108	/* Does the object require an outstanding flush? */
05394f39	109	if (obj->base.write_domain \|\| obj->pin_count)
b47eb4a2	110	continue;
b47eb4a2	111
05394f39	112	if (mark_free(obj, &unwind_list))
cd377ea9 CW	113	goto found;
cd377ea9 CW	114	}
b47eb4a2	115
cd377ea9	116	/* Finally add anything with a pending flush (in order of retirement) */
05394f39 CW	117	list_for_each_entry(obj, &dev_priv->mm.flushing_list, mm_list) {
05394f39 CW	118	if (obj->pin_count)
cd377ea9	119	continue;
b47eb4a2	120
05394f39	121	if (mark_free(obj, &unwind_list))
cd377ea9 CW	122	goto found;
cd377ea9 CW	123	}
05394f39 CW	124	list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
05394f39 CW	125	if (! obj->base.write_domain \|\| obj->pin_count)
b47eb4a2	126	continue;
b47eb4a2	127
05394f39	128	if (mark_free(obj, &unwind_list))
cd377ea9 CW	129	goto found;
	130	}
	131
	132	/* Nothing found, clean up and bail out! */
092de6f2 CW	133	while (!list_empty(&unwind_list)) {
	134	obj = list_first_entry(&unwind_list,
	135	struct drm_i915_gem_object,
	136	exec_list);
	137
05394f39	138	ret = drm_mm_scan_remove_block(obj->gtt_space);
cd377ea9	139	BUG_ON(ret);
092de6f2 CW	140
092de6f2 CW	141	list_del_init(&obj->exec_list);
05394f39	142	drm_gem_object_unreference(&obj->base);
cd377ea9 CW	143	}
	144
	145	/* We expect the caller to unpin, evict all and try again, or give up.
	146	* So calling i915_gem_evict_everything() is unnecessary.
	147	*/
	148	return -ENOSPC;
	149
	150	found:
e39a0150 CW	151	/* drm_mm doesn't allow any other other operations while
	152	* scanning, therefore store to be evicted objects on a
	153	* temporary list. */
cd377ea9	154	INIT_LIST_HEAD(&eviction_list);
e39a0150	155	while (!list_empty(&unwind_list)) {
05394f39 CW	156	obj = list_first_entry(&unwind_list,
05394f39 CW	157	struct drm_i915_gem_object,
432e58ed	158	exec_list);
05394f39	159	if (drm_mm_scan_remove_block(obj->gtt_space)) {
432e58ed	160	list_move(&obj->exec_list, &eviction_list);
e39a0150 CW	161	continue;
e39a0150 CW	162	}
432e58ed	163	list_del_init(&obj->exec_list);
05394f39	164	drm_gem_object_unreference(&obj->base);
cd377ea9	165	}
b47eb4a2	166
cd377ea9	167	/* Unbinding will emit any required flushes */
e39a0150	168	while (!list_empty(&eviction_list)) {
05394f39 CW	169	obj = list_first_entry(&eviction_list,
05394f39 CW	170	struct drm_i915_gem_object,
432e58ed	171	exec_list);
e39a0150	172	if (ret == 0)
05394f39	173	ret = i915_gem_object_unbind(obj);
092de6f2	174
432e58ed	175	list_del_init(&obj->exec_list);
05394f39	176	drm_gem_object_unreference(&obj->base);
b47eb4a2	177	}
cd377ea9	178
e39a0150	179	return ret;
b47eb4a2 CW	180	}
	181
	182	int
5eac3ab4	183	i915_gem_evict_everything(struct drm_device *dev, bool purgeable_only)
b47eb4a2 CW	184	{
	185	drm_i915_private_t *dev_priv = dev->dev_private;
	186	int ret;
	187	bool lists_empty;
	188
b47eb4a2 CW	189	lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
b47eb4a2 CW	190	list_empty(&dev_priv->mm.flushing_list) &&
395b70be	191	list_empty(&dev_priv->mm.active_list));
b47eb4a2 CW	192	if (lists_empty)
	193	return -ENOSPC;
	194
db53a302 CW	195	trace_i915_gem_evict_everything(dev, purgeable_only);
db53a302 CW	196
b47eb4a2 CW	197	/* Flush everything (on to the inactive lists) and evict */
	198	ret = i915_gpu_idle(dev);
	199	if (ret)
	200	return ret;
	201
	202	BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
	203
5eac3ab4	204	return i915_gem_evict_inactive(dev, purgeable_only);
b47eb4a2 CW	205	}
	206
	207	/** Unbinds all inactive objects. */
	208	int
5eac3ab4	209	i915_gem_evict_inactive(struct drm_device *dev, bool purgeable_only)
b47eb4a2 CW	210	{
b47eb4a2 CW	211	drm_i915_private_t *dev_priv = dev->dev_private;
5eac3ab4 CW	212	struct drm_i915_gem_object obj, next;
	213
	214	list_for_each_entry_safe(obj, next,
	215	&dev_priv->mm.inactive_list, mm_list) {
	216	if (!purgeable_only \|\| obj->madv != I915_MADV_WILLNEED) {
05394f39	217	int ret = i915_gem_object_unbind(obj);
5eac3ab4 CW	218	if (ret)
5eac3ab4 CW	219	return ret;
b47eb4a2 CW	220	}
	221	}
	222
	223	return 0;
	224	}