[mirror_ubuntu-bionic-kernel.git] / drivers / gpu / drm / msm / adreno / a5xx_preempt.c

/* Copyright (c) 2017 The Linux Foundation. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 and
 * only version 2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 */

#include "msm_gem.h"
#include "a5xx_gpu.h"

/*
 * Try to transition the preemption state from old to new. Return
 * true on success or false if the original state wasn't 'old'
 */
static inline bool try_preempt_state(struct a5xx_gpu *a5xx_gpu,
		enum preempt_state old, enum preempt_state new)
{
	enum preempt_state cur = atomic_cmpxchg(&a5xx_gpu->preempt_state,
		old, new);

	return (cur == old);
}

/*
 * Force the preemption state to the specified state.  This is used in cases
 * where the current state is known and won't change
 */
static inline void set_preempt_state(struct a5xx_gpu *gpu,
		enum preempt_state new)
{
	/*
	 * preempt_state may be read by other cores trying to trigger a
	 * preemption or in the interrupt handler so barriers are needed
	 * before...
	 */
	smp_mb__before_atomic();
	atomic_set(&gpu->preempt_state, new);
	/* ... and after*/
	smp_mb__after_atomic();
}

/* Write the most recent wptr for the given ring into the hardware */
static inline void update_wptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
{
	unsigned long flags;
	uint32_t wptr;

	if (!ring)
		return;

	spin_lock_irqsave(&ring->lock, flags);
	wptr = get_wptr(ring);
	spin_unlock_irqrestore(&ring->lock, flags);

	gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr);
}

/* Return the highest priority ringbuffer with something in it */
static struct msm_ringbuffer *get_next_ring(struct msm_gpu *gpu)
{
	unsigned long flags;
	int i;

	for (i = 0; i < gpu->nr_rings; i++) {
		bool empty;
		struct msm_ringbuffer *ring = gpu->rb[i];

		spin_lock_irqsave(&ring->lock, flags);
		empty = (get_wptr(ring) == ring->memptrs->rptr);
		spin_unlock_irqrestore(&ring->lock, flags);

		if (!empty)
			return ring;
	}

	return NULL;
}

static void a5xx_preempt_timer(struct timer_list *t)
{
	struct a5xx_gpu *a5xx_gpu = from_timer(a5xx_gpu, t, preempt_timer);
	struct msm_gpu *gpu = &a5xx_gpu->base.base;
	struct drm_device *dev = gpu->dev;
	struct msm_drm_private *priv = dev->dev_private;

	if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_FAULTED))
		return;

	dev_err(dev->dev, "%s: preemption timed out\n", gpu->name);
	queue_work(priv->wq, &gpu->recover_work);
}

/* Try to trigger a preemption switch */
void a5xx_preempt_trigger(struct msm_gpu *gpu)
{
	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
	unsigned long flags;
	struct msm_ringbuffer *ring;

	if (gpu->nr_rings == 1)
		return;

	/*
	 * Try to start preemption by moving from NONE to START. If
	 * unsuccessful, a preemption is already in flight
	 */
	if (!try_preempt_state(a5xx_gpu, PREEMPT_NONE, PREEMPT_START))
		return;

	/* Get the next ring to preempt to */
	ring = get_next_ring(gpu);

	/*
	 * If no ring is populated or the highest priority ring is the current
	 * one do nothing except to update the wptr to the latest and greatest
	 */
	if (!ring || (a5xx_gpu->cur_ring == ring)) {
		/*
		 * Its possible that while a preemption request is in progress
		 * from an irq context, a user context trying to submit might
		 * fail to update the write pointer, because it determines
		 * that the preempt state is not PREEMPT_NONE.
		 *
		 * Close the race by introducing an intermediate
		 * state PREEMPT_ABORT to let the submit path
		 * know that the ringbuffer is not going to change
		 * and can safely update the write pointer.
		 */

		set_preempt_state(a5xx_gpu, PREEMPT_ABORT);
		update_wptr(gpu, a5xx_gpu->cur_ring);
		set_preempt_state(a5xx_gpu, PREEMPT_NONE);
		return;
	}

	/* Make sure the wptr doesn't update while we're in motion */
	spin_lock_irqsave(&ring->lock, flags);
	a5xx_gpu->preempt[ring->id]->wptr = get_wptr(ring);
	spin_unlock_irqrestore(&ring->lock, flags);

	/* Set the address of the incoming preemption record */
	gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO,
		REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_HI,
		a5xx_gpu->preempt_iova[ring->id]);

	a5xx_gpu->next_ring = ring;

	/* Start a timer to catch a stuck preemption */
	mod_timer(&a5xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000));

	/* Set the preemption state to triggered */
	set_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED);

	/* Make sure everything is written before hitting the button */
	wmb();

	/* And actually start the preemption */
	gpu_write(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL, 1);
}

void a5xx_preempt_irq(struct msm_gpu *gpu)
{
	uint32_t status;
	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
	struct drm_device *dev = gpu->dev;
	struct msm_drm_private *priv = dev->dev_private;

	if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_PENDING))
		return;

	/* Delete the preemption watchdog timer */
	del_timer(&a5xx_gpu->preempt_timer);

	/*
	 * The hardware should be setting CP_CONTEXT_SWITCH_CNTL to zero before
	 * firing the interrupt, but there is a non zero chance of a hardware
	 * condition or a software race that could set it again before we have a
	 * chance to finish. If that happens, log and go for recovery
	 */
	status = gpu_read(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL);
	if (unlikely(status)) {
		set_preempt_state(a5xx_gpu, PREEMPT_FAULTED);
		dev_err(dev->dev, "%s: Preemption failed to complete\n",
			gpu->name);
		queue_work(priv->wq, &gpu->recover_work);
		return;
	}

	a5xx_gpu->cur_ring = a5xx_gpu->next_ring;
	a5xx_gpu->next_ring = NULL;

	update_wptr(gpu, a5xx_gpu->cur_ring);

	set_preempt_state(a5xx_gpu, PREEMPT_NONE);
}

void a5xx_preempt_hw_init(struct msm_gpu *gpu)
{
	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
	int i;

	for (i = 0; i < gpu->nr_rings; i++) {
		a5xx_gpu->preempt[i]->wptr = 0;
		a5xx_gpu->preempt[i]->rptr = 0;
		a5xx_gpu->preempt[i]->rbase = gpu->rb[i]->iova;
	}

	/* Write a 0 to signal that we aren't switching pagetables */
	gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO,
		REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI, 0);

	/* Reset the preemption state */
	set_preempt_state(a5xx_gpu, PREEMPT_NONE);

	/* Always come up on rb 0 */
	a5xx_gpu->cur_ring = gpu->rb[0];
}

static int preempt_init_ring(struct a5xx_gpu *a5xx_gpu,
		struct msm_ringbuffer *ring)
{
	struct adreno_gpu *adreno_gpu = &a5xx_gpu->base;
	struct msm_gpu *gpu = &adreno_gpu->base;
	struct a5xx_preempt_record *ptr;
	struct drm_gem_object *bo = NULL;
	u64 iova = 0;

	ptr = msm_gem_kernel_new(gpu->dev,
		A5XX_PREEMPT_RECORD_SIZE + A5XX_PREEMPT_COUNTER_SIZE,
		MSM_BO_UNCACHED, gpu->aspace, &bo, &iova);

	if (IS_ERR(ptr))
		return PTR_ERR(ptr);

	a5xx_gpu->preempt_bo[ring->id] = bo;
	a5xx_gpu->preempt_iova[ring->id] = iova;
	a5xx_gpu->preempt[ring->id] = ptr;

	/* Set up the defaults on the preemption record */

	ptr->magic = A5XX_PREEMPT_RECORD_MAGIC;
	ptr->info = 0;
	ptr->data = 0;
	ptr->cntl = MSM_GPU_RB_CNTL_DEFAULT;
	ptr->rptr_addr = rbmemptr(ring, rptr);
	ptr->counter = iova + A5XX_PREEMPT_RECORD_SIZE;

	return 0;
}

void a5xx_preempt_fini(struct msm_gpu *gpu)
{
	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
	int i;

	for (i = 0; i < gpu->nr_rings; i++) {
		if (!a5xx_gpu->preempt_bo[i])
			continue;

		msm_gem_put_vaddr(a5xx_gpu->preempt_bo[i]);

		if (a5xx_gpu->preempt_iova[i])
			msm_gem_put_iova(a5xx_gpu->preempt_bo[i], gpu->aspace);

		drm_gem_object_unreference(a5xx_gpu->preempt_bo[i]);
		a5xx_gpu->preempt_bo[i] = NULL;
	}
}

void a5xx_preempt_init(struct msm_gpu *gpu)
{
	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
	int i;

	/* No preemption if we only have one ring */
	if (gpu->nr_rings <= 1)
		return;

	for (i = 0; i < gpu->nr_rings; i++) {
		if (preempt_init_ring(a5xx_gpu, gpu->rb[i])) {
			/*
			 * On any failure our adventure is over. Clean up and
			 * set nr_rings to 1 to force preemption off
			 */
			a5xx_preempt_fini(gpu);
			gpu->nr_rings = 1;

			return;
		}
	}

	timer_setup(&a5xx_gpu->preempt_timer, a5xx_preempt_timer, 0);
}
Commit	Line	Data
b1fc2839 JC	1	/* Copyright (c) 2017 The Linux Foundation. All rights reserved.
	2	*
	3	* This program is free software; you can redistribute it and/or modify
	4	* it under the terms of the GNU General Public License version 2 and
	5	* only version 2 as published by the Free Software Foundation.
	6	*
	7	* This program is distributed in the hope that it will be useful,
	8	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	9	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	10	* GNU General Public License for more details.
	11	*
	12	*/
	13
	14	#include "msm_gem.h"
	15	#include "a5xx_gpu.h"
	16
	17	/*
	18	* Try to transition the preemption state from old to new. Return
	19	* true on success or false if the original state wasn't 'old'
	20	*/
	21	static inline bool try_preempt_state(struct a5xx_gpu *a5xx_gpu,
	22	enum preempt_state old, enum preempt_state new)
	23	{
	24	enum preempt_state cur = atomic_cmpxchg(&a5xx_gpu->preempt_state,
	25	old, new);
	26
	27	return (cur == old);
	28	}
	29
	30	/*
	31	* Force the preemption state to the specified state. This is used in cases
	32	* where the current state is known and won't change
	33	*/
	34	static inline void set_preempt_state(struct a5xx_gpu *gpu,
	35	enum preempt_state new)
	36	{
	37	/*
	38	* preempt_state may be read by other cores trying to trigger a
	39	* preemption or in the interrupt handler so barriers are needed
	40	* before...
	41	*/
	42	smp_mb__before_atomic();
	43	atomic_set(&gpu->preempt_state, new);
	44	/* ... and after*/
	45	smp_mb__after_atomic();
	46	}
	47
	48	/* Write the most recent wptr for the given ring into the hardware */
	49	static inline void update_wptr(struct msm_gpu gpu, struct msm_ringbuffer ring)
	50	{
	51	unsigned long flags;
	52	uint32_t wptr;
	53
	54	if (!ring)
	55	return;
	56
	57	spin_lock_irqsave(&ring->lock, flags);
	58	wptr = get_wptr(ring);
	59	spin_unlock_irqrestore(&ring->lock, flags);
	60
	61	gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr);
	62	}
	63
	64	/* Return the highest priority ringbuffer with something in it */
65	static struct msm_ringbuffer get_next_ring(struct msm_gpu gpu)
66	{
67	unsigned long flags;
68	int i;
69
70	for (i = 0; i < gpu->nr_rings; i++) {
71	bool empty;
72	struct msm_ringbuffer *ring = gpu->rb[i];
73
74	spin_lock_irqsave(&ring->lock, flags);
75	empty = (get_wptr(ring) == ring->memptrs->rptr);
76	spin_unlock_irqrestore(&ring->lock, flags);
77
78	if (!empty)
79	return ring;
80	}
81
82	return NULL;
83	}
84
e99e88a9	85	static void a5xx_preempt_timer(struct timer_list *t)
b1fc2839	86	{
e99e88a9	87	struct a5xx_gpu *a5xx_gpu = from_timer(a5xx_gpu, t, preempt_timer);
b1fc2839 JC	88	struct msm_gpu *gpu = &a5xx_gpu->base.base;
	89	struct drm_device *dev = gpu->dev;
	90	struct msm_drm_private *priv = dev->dev_private;
	91
	92	if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_FAULTED))
	93	return;
	94
	95	dev_err(dev->dev, "%s: preemption timed out\n", gpu->name);
	96	queue_work(priv->wq, &gpu->recover_work);
	97	}
	98
	99	/* Try to trigger a preemption switch */
	100	void a5xx_preempt_trigger(struct msm_gpu *gpu)
	101	{
	102	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
	103	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
	104	unsigned long flags;
	105	struct msm_ringbuffer *ring;
	106
	107	if (gpu->nr_rings == 1)
	108	return;
	109
	110	/*
	111	* Try to start preemption by moving from NONE to START. If
	112	* unsuccessful, a preemption is already in flight
	113	*/
	114	if (!try_preempt_state(a5xx_gpu, PREEMPT_NONE, PREEMPT_START))
	115	return;
	116
	117	/* Get the next ring to preempt to */
	118	ring = get_next_ring(gpu);
	119
	120	/*
	121	* If no ring is populated or the highest priority ring is the current
	122	* one do nothing except to update the wptr to the latest and greatest
	123	*/
	124	if (!ring \|\| (a5xx_gpu->cur_ring == ring)) {
	125	/*
	126	* Its possible that while a preemption request is in progress
	127	* from an irq context, a user context trying to submit might
	128	* fail to update the write pointer, because it determines
	129	* that the preempt state is not PREEMPT_NONE.
	130	*
	131	* Close the race by introducing an intermediate
	132	* state PREEMPT_ABORT to let the submit path
	133	* know that the ringbuffer is not going to change
	134	* and can safely update the write pointer.
	135	*/
	136
	137	set_preempt_state(a5xx_gpu, PREEMPT_ABORT);
	138	update_wptr(gpu, a5xx_gpu->cur_ring);
	139	set_preempt_state(a5xx_gpu, PREEMPT_NONE);
	140	return;
	141	}
	142
	143	/* Make sure the wptr doesn't update while we're in motion */
	144	spin_lock_irqsave(&ring->lock, flags);
	145	a5xx_gpu->preempt[ring->id]->wptr = get_wptr(ring);
	146	spin_unlock_irqrestore(&ring->lock, flags);
	147
	148	/* Set the address of the incoming preemption record */
	149	gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO,
	150	REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_HI,
	151	a5xx_gpu->preempt_iova[ring->id]);
152
153	a5xx_gpu->next_ring = ring;
154
155	/* Start a timer to catch a stuck preemption */
156	mod_timer(&a5xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000));
157
158	/* Set the preemption state to triggered */
159	set_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED);
160
161	/* Make sure everything is written before hitting the button */
162	wmb();
163
164	/* And actually start the preemption */
165	gpu_write(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL, 1);
166	}
167
168	void a5xx_preempt_irq(struct msm_gpu *gpu)
169	{
170	uint32_t status;
171	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
172	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
173	struct drm_device *dev = gpu->dev;
174	struct msm_drm_private *priv = dev->dev_private;
175
176	if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_PENDING))
177	return;
178
179	/* Delete the preemption watchdog timer */
180	del_timer(&a5xx_gpu->preempt_timer);
181
182	/*
183	* The hardware should be setting CP_CONTEXT_SWITCH_CNTL to zero before
184	* firing the interrupt, but there is a non zero chance of a hardware
185	* condition or a software race that could set it again before we have a
186	* chance to finish. If that happens, log and go for recovery
187	*/
188	status = gpu_read(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL);
189	if (unlikely(status)) {
190	set_preempt_state(a5xx_gpu, PREEMPT_FAULTED);
191	dev_err(dev->dev, "%s: Preemption failed to complete\n",
192	gpu->name);
193	queue_work(priv->wq, &gpu->recover_work);
194	return;
195	}
196
197	a5xx_gpu->cur_ring = a5xx_gpu->next_ring;
198	a5xx_gpu->next_ring = NULL;
199
200	update_wptr(gpu, a5xx_gpu->cur_ring);
201
202	set_preempt_state(a5xx_gpu, PREEMPT_NONE);
203	}
204
205	void a5xx_preempt_hw_init(struct msm_gpu *gpu)
206	{
207	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
208	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
209	int i;
210
211	for (i = 0; i < gpu->nr_rings; i++) {
212	a5xx_gpu->preempt[i]->wptr = 0;
213	a5xx_gpu->preempt[i]->rptr = 0;
214	a5xx_gpu->preempt[i]->rbase = gpu->rb[i]->iova;
215	}
216
217	/* Write a 0 to signal that we aren't switching pagetables */
218	gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO,
219	REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI, 0);
220
221	/* Reset the preemption state */
222	set_preempt_state(a5xx_gpu, PREEMPT_NONE);
223
224	/* Always come up on rb 0 */
225	a5xx_gpu->cur_ring = gpu->rb[0];
226	}
227
228	static int preempt_init_ring(struct a5xx_gpu *a5xx_gpu,
229	struct msm_ringbuffer *ring)
230	{
231	struct adreno_gpu *adreno_gpu = &a5xx_gpu->base;
232	struct msm_gpu *gpu = &adreno_gpu->base;
233	struct a5xx_preempt_record *ptr;
234	struct drm_gem_object *bo = NULL;
235	u64 iova = 0;
236
237	ptr = msm_gem_kernel_new(gpu->dev,
238	A5XX_PREEMPT_RECORD_SIZE + A5XX_PREEMPT_COUNTER_SIZE,
239	MSM_BO_UNCACHED, gpu->aspace, &bo, &iova);
240
241	if (IS_ERR(ptr))
242	return PTR_ERR(ptr);
243
244	a5xx_gpu->preempt_bo[ring->id] = bo;
245	a5xx_gpu->preempt_iova[ring->id] = iova;
246	a5xx_gpu->preempt[ring->id] = ptr;
247
248	/* Set up the defaults on the preemption record */
249
250	ptr->magic = A5XX_PREEMPT_RECORD_MAGIC;
251	ptr->info = 0;
252	ptr->data = 0;
253	ptr->cntl = MSM_GPU_RB_CNTL_DEFAULT;
254	ptr->rptr_addr = rbmemptr(ring, rptr);
255	ptr->counter = iova + A5XX_PREEMPT_RECORD_SIZE;
256
257	return 0;
258	}
259
260	void a5xx_preempt_fini(struct msm_gpu *gpu)
261	{
262	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
263	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
264	int i;
265
266	for (i = 0; i < gpu->nr_rings; i++) {
267	if (!a5xx_gpu->preempt_bo[i])
268	continue;
269
270	msm_gem_put_vaddr(a5xx_gpu->preempt_bo[i]);
271
272	if (a5xx_gpu->preempt_iova[i])
273	msm_gem_put_iova(a5xx_gpu->preempt_bo[i], gpu->aspace);
274
275	drm_gem_object_unreference(a5xx_gpu->preempt_bo[i]);
276	a5xx_gpu->preempt_bo[i] = NULL;
277	}
278	}
279
280	void a5xx_preempt_init(struct msm_gpu *gpu)
281	{
282	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
283	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
284	int i;
285
286	/* No preemption if we only have one ring */
287	if (gpu->nr_rings <= 1)
288	return;
289
290	for (i = 0; i < gpu->nr_rings; i++) {
291	if (preempt_init_ring(a5xx_gpu, gpu->rb[i])) {
292	/*
293	* On any failure our adventure is over. Clean up and
294	* set nr_rings to 1 to force preemption off
295	*/
296	a5xx_preempt_fini(gpu);
297	gpu->nr_rings = 1;
298
299	return;
300	}
301	}
302
e99e88a9	303	timer_setup(&a5xx_gpu->preempt_timer, a5xx_preempt_timer, 0);
b1fc2839	304	}