[mirror_ubuntu-jammy-kernel.git] / drivers / gpu / drm / amd / amdgpu / amdgpu_sa.c

/*
 * Copyright 2011 Red Hat Inc.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 */
/*
 * Authors:
 *    Jerome Glisse <glisse@freedesktop.org>
 */
/* Algorithm:
 *
 * We store the last allocated bo in "hole", we always try to allocate
 * after the last allocated bo. Principle is that in a linear GPU ring
 * progression was is after last is the oldest bo we allocated and thus
 * the first one that should no longer be in use by the GPU.
 *
 * If it's not the case we skip over the bo after last to the closest
 * done bo if such one exist. If none exist and we are not asked to
 * block we report failure to allocate.
 *
 * If we are asked to block we wait on all the oldest fence of all
 * rings. We just wait for any of those fence to complete.
 */

#include "amdgpu.h"

static void amdgpu_sa_bo_remove_locked(struct amdgpu_sa_bo *sa_bo);
static void amdgpu_sa_bo_try_free(struct amdgpu_sa_manager *sa_manager);

int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,
			      struct amdgpu_sa_manager *sa_manager,
			      unsigned size, u32 align, u32 domain)
{
	int i, r;

	init_waitqueue_head(&sa_manager->wq);
	sa_manager->bo = NULL;
	sa_manager->size = size;
	sa_manager->domain = domain;
	sa_manager->align = align;
	sa_manager->hole = &sa_manager->olist;
	INIT_LIST_HEAD(&sa_manager->olist);
	for (i = 0; i < AMDGPU_SA_NUM_FENCE_LISTS; ++i)
		INIT_LIST_HEAD(&sa_manager->flist[i]);

	r = amdgpu_bo_create_kernel(adev, size, align, domain, &sa_manager->bo,
				&sa_manager->gpu_addr, &sa_manager->cpu_ptr);
	if (r) {
		dev_err(adev->dev, "(%d) failed to allocate bo for manager\n", r);
		return r;
	}

	memset(sa_manager->cpu_ptr, 0, sa_manager->size);
	return r;
}

void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev,
			       struct amdgpu_sa_manager *sa_manager)
{
	struct amdgpu_sa_bo *sa_bo, *tmp;

	if (sa_manager->bo == NULL) {
		dev_err(adev->dev, "no bo for sa manager\n");
		return;
	}

	if (!list_empty(&sa_manager->olist)) {
		sa_manager->hole = &sa_manager->olist,
		amdgpu_sa_bo_try_free(sa_manager);
		if (!list_empty(&sa_manager->olist)) {
			dev_err(adev->dev, "sa_manager is not empty, clearing anyway\n");
		}
	}
	list_for_each_entry_safe(sa_bo, tmp, &sa_manager->olist, olist) {
		amdgpu_sa_bo_remove_locked(sa_bo);
	}

	amdgpu_bo_free_kernel(&sa_manager->bo, &sa_manager->gpu_addr, &sa_manager->cpu_ptr);
	sa_manager->size = 0;
}

static void amdgpu_sa_bo_remove_locked(struct amdgpu_sa_bo *sa_bo)
{
	struct amdgpu_sa_manager *sa_manager = sa_bo->manager;
	if (sa_manager->hole == &sa_bo->olist) {
		sa_manager->hole = sa_bo->olist.prev;
	}
	list_del_init(&sa_bo->olist);
	list_del_init(&sa_bo->flist);
	dma_fence_put(sa_bo->fence);
	kfree(sa_bo);
}

static void amdgpu_sa_bo_try_free(struct amdgpu_sa_manager *sa_manager)
{
	struct amdgpu_sa_bo *sa_bo, *tmp;

	if (sa_manager->hole->next == &sa_manager->olist)
		return;

	sa_bo = list_entry(sa_manager->hole->next, struct amdgpu_sa_bo, olist);
	list_for_each_entry_safe_from(sa_bo, tmp, &sa_manager->olist, olist) {
		if (sa_bo->fence == NULL ||
		    !dma_fence_is_signaled(sa_bo->fence)) {
			return;
		}
		amdgpu_sa_bo_remove_locked(sa_bo);
	}
}

static inline unsigned amdgpu_sa_bo_hole_soffset(struct amdgpu_sa_manager *sa_manager)
{
	struct list_head *hole = sa_manager->hole;

	if (hole != &sa_manager->olist) {
		return list_entry(hole, struct amdgpu_sa_bo, olist)->eoffset;
	}
	return 0;
}

static inline unsigned amdgpu_sa_bo_hole_eoffset(struct amdgpu_sa_manager *sa_manager)
{
	struct list_head *hole = sa_manager->hole;

	if (hole->next != &sa_manager->olist) {
		return list_entry(hole->next, struct amdgpu_sa_bo, olist)->soffset;
	}
	return sa_manager->size;
}

static bool amdgpu_sa_bo_try_alloc(struct amdgpu_sa_manager *sa_manager,
				   struct amdgpu_sa_bo *sa_bo,
				   unsigned size, unsigned align)
{
	unsigned soffset, eoffset, wasted;

	soffset = amdgpu_sa_bo_hole_soffset(sa_manager);
	eoffset = amdgpu_sa_bo_hole_eoffset(sa_manager);
	wasted = (align - (soffset % align)) % align;

	if ((eoffset - soffset) >= (size + wasted)) {
		soffset += wasted;

		sa_bo->manager = sa_manager;
		sa_bo->soffset = soffset;
		sa_bo->eoffset = soffset + size;
		list_add(&sa_bo->olist, sa_manager->hole);
		INIT_LIST_HEAD(&sa_bo->flist);
		sa_manager->hole = &sa_bo->olist;
		return true;
	}
	return false;
}

/**
 * amdgpu_sa_event - Check if we can stop waiting
 *
 * @sa_manager: pointer to the sa_manager
 * @size: number of bytes we want to allocate
 * @align: alignment we need to match
 *
 * Check if either there is a fence we can wait for or
 * enough free memory to satisfy the allocation directly
 */
static bool amdgpu_sa_event(struct amdgpu_sa_manager *sa_manager,
			    unsigned size, unsigned align)
{
	unsigned soffset, eoffset, wasted;
	int i;

	for (i = 0; i < AMDGPU_SA_NUM_FENCE_LISTS; ++i)
		if (!list_empty(&sa_manager->flist[i]))
			return true;

	soffset = amdgpu_sa_bo_hole_soffset(sa_manager);
	eoffset = amdgpu_sa_bo_hole_eoffset(sa_manager);
	wasted = (align - (soffset % align)) % align;

	if ((eoffset - soffset) >= (size + wasted)) {
		return true;
	}

	return false;
}

static bool amdgpu_sa_bo_next_hole(struct amdgpu_sa_manager *sa_manager,
				   struct dma_fence **fences,
				   unsigned *tries)
{
	struct amdgpu_sa_bo *best_bo = NULL;
	unsigned i, soffset, best, tmp;

	/* if hole points to the end of the buffer */
	if (sa_manager->hole->next == &sa_manager->olist) {
		/* try again with its beginning */
		sa_manager->hole = &sa_manager->olist;
		return true;
	}

	soffset = amdgpu_sa_bo_hole_soffset(sa_manager);
	/* to handle wrap around we add sa_manager->size */
	best = sa_manager->size * 2;
	/* go over all fence list and try to find the closest sa_bo
	 * of the current last
	 */
	for (i = 0; i < AMDGPU_SA_NUM_FENCE_LISTS; ++i) {
		struct amdgpu_sa_bo *sa_bo;

		fences[i] = NULL;

		if (list_empty(&sa_manager->flist[i]))
			continue;

		sa_bo = list_first_entry(&sa_manager->flist[i],
					 struct amdgpu_sa_bo, flist);

		if (!dma_fence_is_signaled(sa_bo->fence)) {
			fences[i] = sa_bo->fence;
			continue;
		}

		/* limit the number of tries each ring gets */
		if (tries[i] > 2) {
			continue;
		}

		tmp = sa_bo->soffset;
		if (tmp < soffset) {
			/* wrap around, pretend it's after */
			tmp += sa_manager->size;
		}
		tmp -= soffset;
		if (tmp < best) {
			/* this sa bo is the closest one */
			best = tmp;
			best_bo = sa_bo;
		}
	}

	if (best_bo) {
		uint32_t idx = best_bo->fence->context;

		idx %= AMDGPU_SA_NUM_FENCE_LISTS;
		++tries[idx];
		sa_manager->hole = best_bo->olist.prev;

		/* we knew that this one is signaled,
		   so it's save to remote it */
		amdgpu_sa_bo_remove_locked(best_bo);
		return true;
	}
	return false;
}

int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
		     struct amdgpu_sa_bo **sa_bo,
		     unsigned size, unsigned align)
{
	struct dma_fence *fences[AMDGPU_SA_NUM_FENCE_LISTS];
	unsigned tries[AMDGPU_SA_NUM_FENCE_LISTS];
	unsigned count;
	int i, r;
	signed long t;

	if (WARN_ON_ONCE(align > sa_manager->align))
		return -EINVAL;

	if (WARN_ON_ONCE(size > sa_manager->size))
		return -EINVAL;

	*sa_bo = kmalloc(sizeof(struct amdgpu_sa_bo), GFP_KERNEL);
	if (!(*sa_bo))
		return -ENOMEM;
	(*sa_bo)->manager = sa_manager;
	(*sa_bo)->fence = NULL;
	INIT_LIST_HEAD(&(*sa_bo)->olist);
	INIT_LIST_HEAD(&(*sa_bo)->flist);

	spin_lock(&sa_manager->wq.lock);
	do {
		for (i = 0; i < AMDGPU_SA_NUM_FENCE_LISTS; ++i)
			tries[i] = 0;

		do {
			amdgpu_sa_bo_try_free(sa_manager);

			if (amdgpu_sa_bo_try_alloc(sa_manager, *sa_bo,
						   size, align)) {
				spin_unlock(&sa_manager->wq.lock);
				return 0;
			}

			/* see if we can skip over some allocations */
		} while (amdgpu_sa_bo_next_hole(sa_manager, fences, tries));

		for (i = 0, count = 0; i < AMDGPU_SA_NUM_FENCE_LISTS; ++i)
			if (fences[i])
				fences[count++] = dma_fence_get(fences[i]);

		if (count) {
			spin_unlock(&sa_manager->wq.lock);
			t = dma_fence_wait_any_timeout(fences, count, false,
						       MAX_SCHEDULE_TIMEOUT,
						       NULL);
			for (i = 0; i < count; ++i)
				dma_fence_put(fences[i]);

			r = (t > 0) ? 0 : t;
			spin_lock(&sa_manager->wq.lock);
		} else {
			/* if we have nothing to wait for block */
			r = wait_event_interruptible_locked(
				sa_manager->wq,
				amdgpu_sa_event(sa_manager, size, align)
			);
		}

	} while (!r);

	spin_unlock(&sa_manager->wq.lock);
	kfree(*sa_bo);
	*sa_bo = NULL;
	return r;
}

void amdgpu_sa_bo_free(struct amdgpu_device *adev, struct amdgpu_sa_bo **sa_bo,
		       struct dma_fence *fence)
{
	struct amdgpu_sa_manager *sa_manager;

	if (sa_bo == NULL || *sa_bo == NULL) {
		return;
	}

	sa_manager = (*sa_bo)->manager;
	spin_lock(&sa_manager->wq.lock);
	if (fence && !dma_fence_is_signaled(fence)) {
		uint32_t idx;

		(*sa_bo)->fence = dma_fence_get(fence);
		idx = fence->context % AMDGPU_SA_NUM_FENCE_LISTS;
		list_add_tail(&(*sa_bo)->flist, &sa_manager->flist[idx]);
	} else {
		amdgpu_sa_bo_remove_locked(*sa_bo);
	}
	wake_up_all_locked(&sa_manager->wq);
	spin_unlock(&sa_manager->wq.lock);
	*sa_bo = NULL;
}

#if defined(CONFIG_DEBUG_FS)

void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager,
				  struct seq_file *m)
{
	struct amdgpu_sa_bo *i;

	spin_lock(&sa_manager->wq.lock);
	list_for_each_entry(i, &sa_manager->olist, olist) {
		uint64_t soffset = i->soffset + sa_manager->gpu_addr;
		uint64_t eoffset = i->eoffset + sa_manager->gpu_addr;
		if (&i->olist == sa_manager->hole) {
			seq_printf(m, ">");
		} else {
			seq_printf(m, " ");
		}
		seq_printf(m, "[0x%010llx 0x%010llx] size %8lld",
			   soffset, eoffset, eoffset - soffset);

		if (i->fence)
			seq_printf(m, " protected by 0x%016llx on context %llu",
				   i->fence->seqno, i->fence->context);

		seq_printf(m, "\n");
	}
	spin_unlock(&sa_manager->wq.lock);
}
#endif
Commit	Line	Data
d38ceaf9 AD	1	/*
	2	* Copyright 2011 Red Hat Inc.
	3	* All Rights Reserved.
	4	*
	5	* Permission is hereby granted, free of charge, to any person obtaining a
	6	* copy of this software and associated documentation files (the
	7	* "Software"), to deal in the Software without restriction, including
	8	* without limitation the rights to use, copy, modify, merge, publish,
	9	* distribute, sub license, and/or sell copies of the Software, and to
	10	* permit persons to whom the Software is furnished to do so, subject to
	11	* the following conditions:
	12	*
	13	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	14	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	15	* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
	16	* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
	17	* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
	18	* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
	19	* USE OR OTHER DEALINGS IN THE SOFTWARE.
	20	*
	21	* The above copyright notice and this permission notice (including the
	22	* next paragraph) shall be included in all copies or substantial portions
	23	* of the Software.
	24	*
	25	*/
	26	/*
	27	* Authors:
	28	* Jerome Glisse <glisse@freedesktop.org>
	29	*/
	30	/* Algorithm:
	31	*
	32	* We store the last allocated bo in "hole", we always try to allocate
	33	* after the last allocated bo. Principle is that in a linear GPU ring
	34	* progression was is after last is the oldest bo we allocated and thus
	35	* the first one that should no longer be in use by the GPU.
	36	*
	37	* If it's not the case we skip over the bo after last to the closest
	38	* done bo if such one exist. If none exist and we are not asked to
	39	* block we report failure to allocate.
	40	*
	41	* If we are asked to block we wait on all the oldest fence of all
	42	* rings. We just wait for any of those fence to complete.
	43	*/
fdf2f6c5	44
d38ceaf9 AD	45	#include "amdgpu.h"
	46
	47	static void amdgpu_sa_bo_remove_locked(struct amdgpu_sa_bo *sa_bo);
	48	static void amdgpu_sa_bo_try_free(struct amdgpu_sa_manager *sa_manager);
	49
	50	int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,
	51	struct amdgpu_sa_manager *sa_manager,
	52	unsigned size, u32 align, u32 domain)
	53	{
	54	int i, r;
	55
	56	init_waitqueue_head(&sa_manager->wq);
	57	sa_manager->bo = NULL;
	58	sa_manager->size = size;
	59	sa_manager->domain = domain;
	60	sa_manager->align = align;
	61	sa_manager->hole = &sa_manager->olist;
	62	INIT_LIST_HEAD(&sa_manager->olist);
6ba60b89	63	for (i = 0; i < AMDGPU_SA_NUM_FENCE_LISTS; ++i)
d38ceaf9	64	INIT_LIST_HEAD(&sa_manager->flist[i]);
d38ceaf9	65
bffe07b8 ML	66	r = amdgpu_bo_create_kernel(adev, size, align, domain, &sa_manager->bo,
bffe07b8 ML	67	&sa_manager->gpu_addr, &sa_manager->cpu_ptr);
d38ceaf9 AD	68	if (r) {
	69	dev_err(adev->dev, "(%d) failed to allocate bo for manager\n", r);
	70	return r;
	71	}
	72
bffe07b8	73	memset(sa_manager->cpu_ptr, 0, sa_manager->size);
d38ceaf9 AD	74	return r;
	75	}
	76
	77	void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev,
f3729f7b	78	struct amdgpu_sa_manager *sa_manager)
d38ceaf9 AD	79	{
	80	struct amdgpu_sa_bo sa_bo, tmp;
	81
bffe07b8 ML	82	if (sa_manager->bo == NULL) {
	83	dev_err(adev->dev, "no bo for sa manager\n");
	84	return;
	85	}
	86
d38ceaf9 AD	87	if (!list_empty(&sa_manager->olist)) {
	88	sa_manager->hole = &sa_manager->olist,
	89	amdgpu_sa_bo_try_free(sa_manager);
	90	if (!list_empty(&sa_manager->olist)) {
	91	dev_err(adev->dev, "sa_manager is not empty, clearing anyway\n");
	92	}
	93	}
	94	list_for_each_entry_safe(sa_bo, tmp, &sa_manager->olist, olist) {
	95	amdgpu_sa_bo_remove_locked(sa_bo);
	96	}
d38ceaf9	97
bffe07b8 ML	98	amdgpu_bo_free_kernel(&sa_manager->bo, &sa_manager->gpu_addr, &sa_manager->cpu_ptr);
bffe07b8 ML	99	sa_manager->size = 0;
d38ceaf9 AD	100	}
	101
	102	static void amdgpu_sa_bo_remove_locked(struct amdgpu_sa_bo *sa_bo)
	103	{
	104	struct amdgpu_sa_manager *sa_manager = sa_bo->manager;
	105	if (sa_manager->hole == &sa_bo->olist) {
	106	sa_manager->hole = sa_bo->olist.prev;
	107	}
	108	list_del_init(&sa_bo->olist);
	109	list_del_init(&sa_bo->flist);
f54d1867	110	dma_fence_put(sa_bo->fence);
d38ceaf9 AD	111	kfree(sa_bo);
	112	}
	113
	114	static void amdgpu_sa_bo_try_free(struct amdgpu_sa_manager *sa_manager)
	115	{
	116	struct amdgpu_sa_bo sa_bo, tmp;
	117
	118	if (sa_manager->hole->next == &sa_manager->olist)
	119	return;
	120
	121	sa_bo = list_entry(sa_manager->hole->next, struct amdgpu_sa_bo, olist);
	122	list_for_each_entry_safe_from(sa_bo, tmp, &sa_manager->olist, olist) {
3cdb8119	123	if (sa_bo->fence == NULL \|\|
f54d1867	124	!dma_fence_is_signaled(sa_bo->fence)) {
d38ceaf9 AD	125	return;
	126	}
	127	amdgpu_sa_bo_remove_locked(sa_bo);
	128	}
	129	}
	130
	131	static inline unsigned amdgpu_sa_bo_hole_soffset(struct amdgpu_sa_manager *sa_manager)
	132	{
	133	struct list_head *hole = sa_manager->hole;
	134
	135	if (hole != &sa_manager->olist) {
	136	return list_entry(hole, struct amdgpu_sa_bo, olist)->eoffset;
	137	}
	138	return 0;
	139	}
	140
	141	static inline unsigned amdgpu_sa_bo_hole_eoffset(struct amdgpu_sa_manager *sa_manager)
	142	{
	143	struct list_head *hole = sa_manager->hole;
	144
	145	if (hole->next != &sa_manager->olist) {
	146	return list_entry(hole->next, struct amdgpu_sa_bo, olist)->soffset;
	147	}
	148	return sa_manager->size;
	149	}
	150
	151	static bool amdgpu_sa_bo_try_alloc(struct amdgpu_sa_manager *sa_manager,
	152	struct amdgpu_sa_bo *sa_bo,
	153	unsigned size, unsigned align)
	154	{
	155	unsigned soffset, eoffset, wasted;
	156
	157	soffset = amdgpu_sa_bo_hole_soffset(sa_manager);
	158	eoffset = amdgpu_sa_bo_hole_eoffset(sa_manager);
	159	wasted = (align - (soffset % align)) % align;
	160
	161	if ((eoffset - soffset) >= (size + wasted)) {
	162	soffset += wasted;
	163
	164	sa_bo->manager = sa_manager;
	165	sa_bo->soffset = soffset;
	166	sa_bo->eoffset = soffset + size;
	167	list_add(&sa_bo->olist, sa_manager->hole);
	168	INIT_LIST_HEAD(&sa_bo->flist);
	169	sa_manager->hole = &sa_bo->olist;
	170	return true;
	171	}
	172	return false;
	173	}
	174
	175	/**
	176	* amdgpu_sa_event - Check if we can stop waiting
	177	*
	178	* @sa_manager: pointer to the sa_manager
	179	* @size: number of bytes we want to allocate
	180	* @align: alignment we need to match
	181	*
	182	* Check if either there is a fence we can wait for or
	183	* enough free memory to satisfy the allocation directly
	184	*/
	185	static bool amdgpu_sa_event(struct amdgpu_sa_manager *sa_manager,
	186	unsigned size, unsigned align)
	187	{
	188	unsigned soffset, eoffset, wasted;
189	int i;
190
6ba60b89 CK	191	for (i = 0; i < AMDGPU_SA_NUM_FENCE_LISTS; ++i)
6ba60b89 CK	192	if (!list_empty(&sa_manager->flist[i]))
d38ceaf9	193	return true;
d38ceaf9 AD	194
	195	soffset = amdgpu_sa_bo_hole_soffset(sa_manager);
	196	eoffset = amdgpu_sa_bo_hole_eoffset(sa_manager);
	197	wasted = (align - (soffset % align)) % align;
	198
	199	if ((eoffset - soffset) >= (size + wasted)) {
	200	return true;
	201	}
	202
	203	return false;
	204	}
	205
	206	static bool amdgpu_sa_bo_next_hole(struct amdgpu_sa_manager *sa_manager,
f54d1867	207	struct dma_fence **fences,
d38ceaf9 AD	208	unsigned *tries)
	209	{
	210	struct amdgpu_sa_bo *best_bo = NULL;
	211	unsigned i, soffset, best, tmp;
	212
	213	/* if hole points to the end of the buffer */
	214	if (sa_manager->hole->next == &sa_manager->olist) {
	215	/* try again with its beginning */
	216	sa_manager->hole = &sa_manager->olist;
	217	return true;
	218	}
	219
	220	soffset = amdgpu_sa_bo_hole_soffset(sa_manager);
	221	/* to handle wrap around we add sa_manager->size */
	222	best = sa_manager->size * 2;
	223	/* go over all fence list and try to find the closest sa_bo
	224	* of the current last
	225	*/
6ba60b89	226	for (i = 0; i < AMDGPU_SA_NUM_FENCE_LISTS; ++i) {
d38ceaf9 AD	227	struct amdgpu_sa_bo *sa_bo;
d38ceaf9 AD	228
6a15f3ff MD	229	fences[i] = NULL;
6a15f3ff MD	230
6ba60b89	231	if (list_empty(&sa_manager->flist[i]))
d38ceaf9	232	continue;
d38ceaf9 AD	233
	234	sa_bo = list_first_entry(&sa_manager->flist[i],
	235	struct amdgpu_sa_bo, flist);
	236
f54d1867	237	if (!dma_fence_is_signaled(sa_bo->fence)) {
d38ceaf9 AD	238	fences[i] = sa_bo->fence;
	239	continue;
	240	}
	241
	242	/* limit the number of tries each ring gets */
	243	if (tries[i] > 2) {
	244	continue;
	245	}
	246
	247	tmp = sa_bo->soffset;
	248	if (tmp < soffset) {
	249	/* wrap around, pretend it's after */
	250	tmp += sa_manager->size;
	251	}
	252	tmp -= soffset;
	253	if (tmp < best) {
	254	/* this sa bo is the closest one */
	255	best = tmp;
	256	best_bo = sa_bo;
	257	}
	258	}
	259
	260	if (best_bo) {
6ba60b89 CK	261	uint32_t idx = best_bo->fence->context;
	262
	263	idx %= AMDGPU_SA_NUM_FENCE_LISTS;
4ce9891e	264	++tries[idx];
d38ceaf9 AD	265	sa_manager->hole = best_bo->olist.prev;
	266
	267	/* we knew that this one is signaled,
	268	so it's save to remote it */
	269	amdgpu_sa_bo_remove_locked(best_bo);
	270	return true;
	271	}
	272	return false;
	273	}
	274
bbf0b345	275	int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
d38ceaf9 AD	276	struct amdgpu_sa_bo **sa_bo,
	277	unsigned size, unsigned align)
	278	{
f54d1867	279	struct dma_fence *fences[AMDGPU_SA_NUM_FENCE_LISTS];
6ba60b89	280	unsigned tries[AMDGPU_SA_NUM_FENCE_LISTS];
ee327caf	281	unsigned count;
d38ceaf9	282	int i, r;
a8f5bf0b	283	signed long t;
d38ceaf9	284
fe6b2ad9 CK	285	if (WARN_ON_ONCE(align > sa_manager->align))
	286	return -EINVAL;
	287
	288	if (WARN_ON_ONCE(size > sa_manager->size))
	289	return -EINVAL;
d38ceaf9 AD	290
d38ceaf9 AD	291	*sa_bo = kmalloc(sizeof(struct amdgpu_sa_bo), GFP_KERNEL);
3f12325a	292	if (!(*sa_bo))
d38ceaf9	293	return -ENOMEM;
d38ceaf9 AD	294	(*sa_bo)->manager = sa_manager;
	295	(*sa_bo)->fence = NULL;
	296	INIT_LIST_HEAD(&(*sa_bo)->olist);
	297	INIT_LIST_HEAD(&(*sa_bo)->flist);
	298
	299	spin_lock(&sa_manager->wq.lock);
	300	do {
6a15f3ff	301	for (i = 0; i < AMDGPU_SA_NUM_FENCE_LISTS; ++i)
d38ceaf9	302	tries[i] = 0;
d38ceaf9 AD	303
	304	do {
	305	amdgpu_sa_bo_try_free(sa_manager);
	306
	307	if (amdgpu_sa_bo_try_alloc(sa_manager, *sa_bo,
	308	size, align)) {
	309	spin_unlock(&sa_manager->wq.lock);
	310	return 0;
	311	}
	312
	313	/* see if we can skip over some allocations */
	314	} while (amdgpu_sa_bo_next_hole(sa_manager, fences, tries));
	315
6ba60b89	316	for (i = 0, count = 0; i < AMDGPU_SA_NUM_FENCE_LISTS; ++i)
ee327caf	317	if (fences[i])
f54d1867	318	fences[count++] = dma_fence_get(fences[i]);
ee327caf CK	319
	320	if (count) {
	321	spin_unlock(&sa_manager->wq.lock);
f54d1867	322	t = dma_fence_wait_any_timeout(fences, count, false,
eef18a82 JZ	323	MAX_SCHEDULE_TIMEOUT,
eef18a82 JZ	324	NULL);
a8d81b36	325	for (i = 0; i < count; ++i)
f54d1867	326	dma_fence_put(fences[i]);
a8d81b36	327
ee327caf CK	328	r = (t > 0) ? 0 : t;
	329	spin_lock(&sa_manager->wq.lock);
	330	} else {
	331	/* if we have nothing to wait for block */
d38ceaf9 AD	332	r = wait_event_interruptible_locked(
	333	sa_manager->wq,
	334	amdgpu_sa_event(sa_manager, size, align)
	335	);
	336	}
	337
	338	} while (!r);
	339
	340	spin_unlock(&sa_manager->wq.lock);
	341	kfree(*sa_bo);
	342	*sa_bo = NULL;
	343	return r;
	344	}
	345
	346	void amdgpu_sa_bo_free(struct amdgpu_device adev, struct amdgpu_sa_bo *sa_bo,
f54d1867	347	struct dma_fence *fence)
d38ceaf9 AD	348	{
	349	struct amdgpu_sa_manager *sa_manager;
	350
	351	if (sa_bo == NULL \|\| *sa_bo == NULL) {
	352	return;
	353	}
	354
	355	sa_manager = (*sa_bo)->manager;
	356	spin_lock(&sa_manager->wq.lock);
f54d1867	357	if (fence && !dma_fence_is_signaled(fence)) {
4ce9891e	358	uint32_t idx;
6ba60b89	359
f54d1867	360	(*sa_bo)->fence = dma_fence_get(fence);
6ba60b89	361	idx = fence->context % AMDGPU_SA_NUM_FENCE_LISTS;
4ce9891e	362	list_add_tail(&(*sa_bo)->flist, &sa_manager->flist[idx]);
d38ceaf9 AD	363	} else {
	364	amdgpu_sa_bo_remove_locked(*sa_bo);
	365	}
	366	wake_up_all_locked(&sa_manager->wq);
	367	spin_unlock(&sa_manager->wq.lock);
	368	*sa_bo = NULL;
	369	}
	370
	371	#if defined(CONFIG_DEBUG_FS)
4f839a24	372
d38ceaf9 AD	373	void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager,
	374	struct seq_file *m)
	375	{
	376	struct amdgpu_sa_bo *i;
	377
	378	spin_lock(&sa_manager->wq.lock);
	379	list_for_each_entry(i, &sa_manager->olist, olist) {
	380	uint64_t soffset = i->soffset + sa_manager->gpu_addr;
	381	uint64_t eoffset = i->eoffset + sa_manager->gpu_addr;
	382	if (&i->olist == sa_manager->hole) {
	383	seq_printf(m, ">");
	384	} else {
	385	seq_printf(m, " ");
	386	}
	387	seq_printf(m, "[0x%010llx 0x%010llx] size %8lld",
	388	soffset, eoffset, eoffset - soffset);
6ba60b89	389
4f839a24	390	if (i->fence)
b312d8ca	391	seq_printf(m, " protected by 0x%016llx on context %llu",
6ba60b89 CK	392	i->fence->seqno, i->fence->context);
6ba60b89 CK	393
d38ceaf9 AD	394	seq_printf(m, "\n");
	395	}
	396	spin_unlock(&sa_manager->wq.lock);
	397	}
	398	#endif