[mirror_ubuntu-focal-kernel.git] / drivers / gpu / drm / msm / msm_gem_submit.c

/*
 * Copyright (C) 2013 Red Hat
 * Author: Rob Clark <robdclark@gmail.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published by
 * the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "msm_drv.h"
#include "msm_gpu.h"
#include "msm_gem.h"

/*
 * Cmdstream submission:
 */

/* make sure these don't conflict w/ MSM_SUBMIT_BO_x */
#define BO_VALID    0x8000   /* is current addr in cmdstream correct/valid? */
#define BO_LOCKED   0x4000
#define BO_PINNED   0x2000

static struct msm_gem_submit *submit_create(struct drm_device *dev,
		struct msm_gpu *gpu, int nr)
{
	struct msm_gem_submit *submit;
	int sz = sizeof(*submit) + (nr * sizeof(submit->bos[0]));

	submit = kmalloc(sz, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
	if (!submit)
		return NULL;

	submit->dev = dev;
	submit->gpu = gpu;
	submit->pid = get_pid(task_pid(current));

	/* initially, until copy_from_user() and bo lookup succeeds: */
	submit->nr_bos = 0;
	submit->nr_cmds = 0;

	INIT_LIST_HEAD(&submit->bo_list);
	ww_acquire_init(&submit->ticket, &reservation_ww_class);

	return submit;
}

void msm_gem_submit_free(struct msm_gem_submit *submit)
{
	fence_put(submit->fence);
	list_del(&submit->node);
	put_pid(submit->pid);
	kfree(submit);
}

static int submit_lookup_objects(struct msm_gem_submit *submit,
		struct drm_msm_gem_submit *args, struct drm_file *file)
{
	unsigned i;
	int ret = 0;

	spin_lock(&file->table_lock);

	for (i = 0; i < args->nr_bos; i++) {
		struct drm_msm_gem_submit_bo submit_bo;
		struct drm_gem_object *obj;
		struct msm_gem_object *msm_obj;
		void __user *userptr =
			u64_to_user_ptr(args->bos + (i * sizeof(submit_bo)));

		ret = copy_from_user(&submit_bo, userptr, sizeof(submit_bo));
		if (ret) {
			ret = -EFAULT;
			goto out_unlock;
		}

		if (submit_bo.flags & ~MSM_SUBMIT_BO_FLAGS) {
			DRM_ERROR("invalid flags: %x\n", submit_bo.flags);
			ret = -EINVAL;
			goto out_unlock;
		}

		submit->bos[i].flags = submit_bo.flags;
		/* in validate_objects() we figure out if this is true: */
		submit->bos[i].iova  = submit_bo.presumed;

		/* normally use drm_gem_object_lookup(), but for bulk lookup
		 * all under single table_lock just hit object_idr directly:
		 */
		obj = idr_find(&file->object_idr, submit_bo.handle);
		if (!obj) {
			DRM_ERROR("invalid handle %u at index %u\n", submit_bo.handle, i);
			ret = -EINVAL;
			goto out_unlock;
		}

		msm_obj = to_msm_bo(obj);

		if (!list_empty(&msm_obj->submit_entry)) {
			DRM_ERROR("handle %u at index %u already on submit list\n",
					submit_bo.handle, i);
			ret = -EINVAL;
			goto out_unlock;
		}

		drm_gem_object_reference(obj);

		submit->bos[i].obj = msm_obj;

		list_add_tail(&msm_obj->submit_entry, &submit->bo_list);
	}

out_unlock:
	submit->nr_bos = i;
	spin_unlock(&file->table_lock);

	return ret;
}

static void submit_unlock_unpin_bo(struct msm_gem_submit *submit, int i)
{
	struct msm_gem_object *msm_obj = submit->bos[i].obj;

	if (submit->bos[i].flags & BO_PINNED)
		msm_gem_put_iova(&msm_obj->base, submit->gpu->id);

	if (submit->bos[i].flags & BO_LOCKED)
		ww_mutex_unlock(&msm_obj->resv->lock);

	if (!(submit->bos[i].flags & BO_VALID))
		submit->bos[i].iova = 0;

	submit->bos[i].flags &= ~(BO_LOCKED | BO_PINNED);
}

/* This is where we make sure all the bo's are reserved and pin'd: */
static int submit_lock_objects(struct msm_gem_submit *submit)
{
	int contended, slow_locked = -1, i, ret = 0;

retry:
	for (i = 0; i < submit->nr_bos; i++) {
		struct msm_gem_object *msm_obj = submit->bos[i].obj;

		if (slow_locked == i)
			slow_locked = -1;

		contended = i;

		if (!(submit->bos[i].flags & BO_LOCKED)) {
			ret = ww_mutex_lock_interruptible(&msm_obj->resv->lock,
					&submit->ticket);
			if (ret)
				goto fail;
			submit->bos[i].flags |= BO_LOCKED;
		}
	}

	ww_acquire_done(&submit->ticket);

	return 0;

fail:
	for (; i >= 0; i--)
		submit_unlock_unpin_bo(submit, i);

	if (slow_locked > 0)
		submit_unlock_unpin_bo(submit, slow_locked);

	if (ret == -EDEADLK) {
		struct msm_gem_object *msm_obj = submit->bos[contended].obj;
		/* we lost out in a seqno race, lock and retry.. */
		ret = ww_mutex_lock_slow_interruptible(&msm_obj->resv->lock,
				&submit->ticket);
		if (!ret) {
			submit->bos[contended].flags |= BO_LOCKED;
			slow_locked = contended;
			goto retry;
		}
	}

	return ret;
}

static int submit_fence_sync(struct msm_gem_submit *submit)
{
	int i, ret = 0;

	for (i = 0; i < submit->nr_bos; i++) {
		struct msm_gem_object *msm_obj = submit->bos[i].obj;
		bool write = submit->bos[i].flags & MSM_SUBMIT_BO_WRITE;

		ret = msm_gem_sync_object(&msm_obj->base, submit->gpu->fctx, write);
		if (ret)
			break;
	}

	return ret;
}

static int submit_pin_objects(struct msm_gem_submit *submit)
{
	int i, ret = 0;

	submit->valid = true;

	for (i = 0; i < submit->nr_bos; i++) {
		struct msm_gem_object *msm_obj = submit->bos[i].obj;
		uint32_t iova;

		/* if locking succeeded, pin bo: */
		ret = msm_gem_get_iova_locked(&msm_obj->base,
				submit->gpu->id, &iova);

		if (ret)
			break;

		submit->bos[i].flags |= BO_PINNED;

		if (iova == submit->bos[i].iova) {
			submit->bos[i].flags |= BO_VALID;
		} else {
			submit->bos[i].iova = iova;
			/* iova changed, so address in cmdstream is not valid: */
			submit->bos[i].flags &= ~BO_VALID;
			submit->valid = false;
		}
	}

	return ret;
}

static int submit_bo(struct msm_gem_submit *submit, uint32_t idx,
		struct msm_gem_object **obj, uint32_t *iova, bool *valid)
{
	if (idx >= submit->nr_bos) {
		DRM_ERROR("invalid buffer index: %u (out of %u)\n",
				idx, submit->nr_bos);
		return -EINVAL;
	}

	if (obj)
		*obj = submit->bos[idx].obj;
	if (iova)
		*iova = submit->bos[idx].iova;
	if (valid)
		*valid = !!(submit->bos[idx].flags & BO_VALID);

	return 0;
}

/* process the reloc's and patch up the cmdstream as needed: */
static int submit_reloc(struct msm_gem_submit *submit, struct msm_gem_object *obj,
		uint32_t offset, uint32_t nr_relocs, uint64_t relocs)
{
	uint32_t i, last_offset = 0;
	uint32_t *ptr;
	int ret;

	if (offset % 4) {
		DRM_ERROR("non-aligned cmdstream buffer: %u\n", offset);
		return -EINVAL;
	}

	/* For now, just map the entire thing.  Eventually we probably
	 * to do it page-by-page, w/ kmap() if not vmap()d..
	 */
	ptr = msm_gem_vaddr_locked(&obj->base);

	if (IS_ERR(ptr)) {
		ret = PTR_ERR(ptr);
		DBG("failed to map: %d", ret);
		return ret;
	}

	for (i = 0; i < nr_relocs; i++) {
		struct drm_msm_gem_submit_reloc submit_reloc;
		void __user *userptr =
			u64_to_user_ptr(relocs + (i * sizeof(submit_reloc)));
		uint32_t iova, off;
		bool valid;

		ret = copy_from_user(&submit_reloc, userptr, sizeof(submit_reloc));
		if (ret)
			return -EFAULT;

		if (submit_reloc.submit_offset % 4) {
			DRM_ERROR("non-aligned reloc offset: %u\n",
					submit_reloc.submit_offset);
			return -EINVAL;
		}

		/* offset in dwords: */
		off = submit_reloc.submit_offset / 4;

		if ((off >= (obj->base.size / 4)) ||
				(off < last_offset)) {
			DRM_ERROR("invalid offset %u at reloc %u\n", off, i);
			return -EINVAL;
		}

		ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova, &valid);
		if (ret)
			return ret;

		if (valid)
			continue;

		iova += submit_reloc.reloc_offset;

		if (submit_reloc.shift < 0)
			iova >>= -submit_reloc.shift;
		else
			iova <<= submit_reloc.shift;

		ptr[off] = iova | submit_reloc.or;

		last_offset = off;
	}

	return 0;
}

static void submit_cleanup(struct msm_gem_submit *submit)
{
	unsigned i;

	for (i = 0; i < submit->nr_bos; i++) {
		struct msm_gem_object *msm_obj = submit->bos[i].obj;
		submit_unlock_unpin_bo(submit, i);
		list_del_init(&msm_obj->submit_entry);
		drm_gem_object_unreference(&msm_obj->base);
	}

	ww_acquire_fini(&submit->ticket);
}

int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
		struct drm_file *file)
{
	struct msm_drm_private *priv = dev->dev_private;
	struct drm_msm_gem_submit *args = data;
	struct msm_file_private *ctx = file->driver_priv;
	struct msm_gem_submit *submit;
	struct msm_gpu *gpu = priv->gpu;
	unsigned i;
	int ret;

	if (!gpu)
		return -ENXIO;

	/* for now, we just have 3d pipe.. eventually this would need to
	 * be more clever to dispatch to appropriate gpu module:
	 */
	if (args->pipe != MSM_PIPE_3D0)
		return -EINVAL;

	if (args->nr_cmds > MAX_CMDS)
		return -EINVAL;

	submit = submit_create(dev, gpu, args->nr_bos);
	if (!submit)
		return -ENOMEM;

	mutex_lock(&dev->struct_mutex);

	ret = submit_lookup_objects(submit, args, file);
	if (ret)
		goto out;

	ret = submit_lock_objects(submit);
	if (ret)
		goto out;

	ret = submit_fence_sync(submit);
	if (ret)
		goto out;

	ret = submit_pin_objects(submit);
	if (ret)
		goto out;

	for (i = 0; i < args->nr_cmds; i++) {
		struct drm_msm_gem_submit_cmd submit_cmd;
		void __user *userptr =
			u64_to_user_ptr(args->cmds + (i * sizeof(submit_cmd)));
		struct msm_gem_object *msm_obj;
		uint32_t iova;

		ret = copy_from_user(&submit_cmd, userptr, sizeof(submit_cmd));
		if (ret) {
			ret = -EFAULT;
			goto out;
		}

		/* validate input from userspace: */
		switch (submit_cmd.type) {
		case MSM_SUBMIT_CMD_BUF:
		case MSM_SUBMIT_CMD_IB_TARGET_BUF:
		case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
			break;
		default:
			DRM_ERROR("invalid type: %08x\n", submit_cmd.type);
			ret = -EINVAL;
			goto out;
		}

		ret = submit_bo(submit, submit_cmd.submit_idx,
				&msm_obj, &iova, NULL);
		if (ret)
			goto out;

		if (submit_cmd.size % 4) {
			DRM_ERROR("non-aligned cmdstream buffer size: %u\n",
					submit_cmd.size);
			ret = -EINVAL;
			goto out;
		}

		if ((submit_cmd.size + submit_cmd.submit_offset) >=
				msm_obj->base.size) {
			DRM_ERROR("invalid cmdstream size: %u\n", submit_cmd.size);
			ret = -EINVAL;
			goto out;
		}

		submit->cmd[i].type = submit_cmd.type;
		submit->cmd[i].size = submit_cmd.size / 4;
		submit->cmd[i].iova = iova + submit_cmd.submit_offset;
		submit->cmd[i].idx  = submit_cmd.submit_idx;

		if (submit->valid)
			continue;

		ret = submit_reloc(submit, msm_obj, submit_cmd.submit_offset,
				submit_cmd.nr_relocs, submit_cmd.relocs);
		if (ret)
			goto out;
	}

	submit->nr_cmds = i;

	ret = msm_gpu_submit(gpu, submit, ctx);

	args->fence = submit->fence->seqno;

out:
	submit_cleanup(submit);
	if (ret)
		msm_gem_submit_free(submit);
	mutex_unlock(&dev->struct_mutex);
	return ret;
}
Commit	Line	Data
7198e6b0 RC	1	/*
	2	* Copyright (C) 2013 Red Hat
	3	* Author: Rob Clark <robdclark@gmail.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify it
	6	* under the terms of the GNU General Public License version 2 as published by
	7	* the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful, but WITHOUT
	10	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	11	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	12	* more details.
	13	*
	14	* You should have received a copy of the GNU General Public License along with
	15	* this program. If not, see <http://www.gnu.org/licenses/>.
	16	*/
	17
	18	#include "msm_drv.h"
	19	#include "msm_gpu.h"
	20	#include "msm_gem.h"
	21
	22	/*
	23	* Cmdstream submission:
	24	*/
	25
7198e6b0	26	/* make sure these don't conflict w/ MSM_SUBMIT_BO_x */
340faef2	27	#define BO_VALID 0x8000 /* is current addr in cmdstream correct/valid? */
7198e6b0 RC	28	#define BO_LOCKED 0x4000
	29	#define BO_PINNED 0x2000
	30
7198e6b0 RC	31	static struct msm_gem_submit submit_create(struct drm_device dev,
	32	struct msm_gpu *gpu, int nr)
	33	{
	34	struct msm_gem_submit *submit;
	35	int sz = sizeof(submit) + (nr sizeof(submit->bos[0]));
	36
	37	submit = kmalloc(sz, GFP_TEMPORARY \| __GFP_NOWARN \| __GFP_NORETRY);
6860b56c RC	38	if (!submit)
6860b56c RC	39	return NULL;
7198e6b0	40
6860b56c RC	41	submit->dev = dev;
6860b56c RC	42	submit->gpu = gpu;
4816b626	43	submit->pid = get_pid(task_pid(current));
7198e6b0	44
6860b56c RC	45	/* initially, until copy_from_user() and bo lookup succeeds: */
	46	submit->nr_bos = 0;
	47	submit->nr_cmds = 0;
	48
	49	INIT_LIST_HEAD(&submit->bo_list);
	50	ww_acquire_init(&submit->ticket, &reservation_ww_class);
7198e6b0 RC	51
	52	return submit;
	53	}
	54
40e6815b RC	55	void msm_gem_submit_free(struct msm_gem_submit *submit)
	56	{
	57	fence_put(submit->fence);
	58	list_del(&submit->node);
4816b626	59	put_pid(submit->pid);
40e6815b RC	60	kfree(submit);
	61	}
	62
7198e6b0 RC	63	static int submit_lookup_objects(struct msm_gem_submit *submit,
	64	struct drm_msm_gem_submit args, struct drm_file file)
	65	{
	66	unsigned i;
	67	int ret = 0;
	68
	69	spin_lock(&file->table_lock);
	70
	71	for (i = 0; i < args->nr_bos; i++) {
	72	struct drm_msm_gem_submit_bo submit_bo;
	73	struct drm_gem_object *obj;
	74	struct msm_gem_object *msm_obj;
	75	void __user *userptr =
3ed605bc	76	u64_to_user_ptr(args->bos + (i * sizeof(submit_bo)));
7198e6b0 RC	77
	78	ret = copy_from_user(&submit_bo, userptr, sizeof(submit_bo));
	79	if (ret) {
	80	ret = -EFAULT;
	81	goto out_unlock;
	82	}
	83
93ddb0d3	84	if (submit_bo.flags & ~MSM_SUBMIT_BO_FLAGS) {
19872533	85	DRM_ERROR("invalid flags: %x\n", submit_bo.flags);
7198e6b0 RC	86	ret = -EINVAL;
	87	goto out_unlock;
	88	}
	89
	90	submit->bos[i].flags = submit_bo.flags;
	91	/* in validate_objects() we figure out if this is true: */
	92	submit->bos[i].iova = submit_bo.presumed;
	93
	94	/* normally use drm_gem_object_lookup(), but for bulk lookup
	95	* all under single table_lock just hit object_idr directly:
	96	*/
	97	obj = idr_find(&file->object_idr, submit_bo.handle);
	98	if (!obj) {
19872533	99	DRM_ERROR("invalid handle %u at index %u\n", submit_bo.handle, i);
7198e6b0 RC	100	ret = -EINVAL;
	101	goto out_unlock;
	102	}
	103
	104	msm_obj = to_msm_bo(obj);
	105
	106	if (!list_empty(&msm_obj->submit_entry)) {
19872533	107	DRM_ERROR("handle %u at index %u already on submit list\n",
7198e6b0 RC	108	submit_bo.handle, i);
	109	ret = -EINVAL;
	110	goto out_unlock;
	111	}
	112
	113	drm_gem_object_reference(obj);
	114
	115	submit->bos[i].obj = msm_obj;
	116
	117	list_add_tail(&msm_obj->submit_entry, &submit->bo_list);
	118	}
	119
	120	out_unlock:
	121	submit->nr_bos = i;
	122	spin_unlock(&file->table_lock);
	123
	124	return ret;
	125	}
	126
	127	static void submit_unlock_unpin_bo(struct msm_gem_submit *submit, int i)
	128	{
	129	struct msm_gem_object *msm_obj = submit->bos[i].obj;
	130
	131	if (submit->bos[i].flags & BO_PINNED)
	132	msm_gem_put_iova(&msm_obj->base, submit->gpu->id);
	133
	134	if (submit->bos[i].flags & BO_LOCKED)
	135	ww_mutex_unlock(&msm_obj->resv->lock);
	136
	137	if (!(submit->bos[i].flags & BO_VALID))
	138	submit->bos[i].iova = 0;
	139
	140	submit->bos[i].flags &= ~(BO_LOCKED \| BO_PINNED);
	141	}
	142
	143	/* This is where we make sure all the bo's are reserved and pin'd: */
340faef2	144	static int submit_lock_objects(struct msm_gem_submit *submit)
7198e6b0 RC	145	{
	146	int contended, slow_locked = -1, i, ret = 0;
	147
	148	retry:
7198e6b0 RC	149	for (i = 0; i < submit->nr_bos; i++) {
7198e6b0 RC	150	struct msm_gem_object *msm_obj = submit->bos[i].obj;
7198e6b0 RC	151
	152	if (slow_locked == i)
	153	slow_locked = -1;
	154
	155	contended = i;
	156
	157	if (!(submit->bos[i].flags & BO_LOCKED)) {
	158	ret = ww_mutex_lock_interruptible(&msm_obj->resv->lock,
	159	&submit->ticket);
	160	if (ret)
	161	goto fail;
	162	submit->bos[i].flags \|= BO_LOCKED;
	163	}
7198e6b0 RC	164	}
	165
	166	ww_acquire_done(&submit->ticket);
	167
	168	return 0;
	169
	170	fail:
	171	for (; i >= 0; i--)
	172	submit_unlock_unpin_bo(submit, i);
	173
	174	if (slow_locked > 0)
	175	submit_unlock_unpin_bo(submit, slow_locked);
	176
	177	if (ret == -EDEADLK) {
	178	struct msm_gem_object *msm_obj = submit->bos[contended].obj;
	179	/* we lost out in a seqno race, lock and retry.. */
	180	ret = ww_mutex_lock_slow_interruptible(&msm_obj->resv->lock,
	181	&submit->ticket);
	182	if (!ret) {
	183	submit->bos[contended].flags \|= BO_LOCKED;
	184	slow_locked = contended;
	185	goto retry;
	186	}
	187	}
	188
	189	return ret;
	190	}
	191
b6295f9a RC	192	static int submit_fence_sync(struct msm_gem_submit *submit)
	193	{
	194	int i, ret = 0;
	195
	196	for (i = 0; i < submit->nr_bos; i++) {
	197	struct msm_gem_object *msm_obj = submit->bos[i].obj;
	198	bool write = submit->bos[i].flags & MSM_SUBMIT_BO_WRITE;
	199
	200	ret = msm_gem_sync_object(&msm_obj->base, submit->gpu->fctx, write);
	201	if (ret)
	202	break;
	203	}
	204
	205	return ret;
	206	}
	207
340faef2 RC	208	static int submit_pin_objects(struct msm_gem_submit *submit)
	209	{
	210	int i, ret = 0;
	211
	212	submit->valid = true;
	213
	214	for (i = 0; i < submit->nr_bos; i++) {
	215	struct msm_gem_object *msm_obj = submit->bos[i].obj;
	216	uint32_t iova;
	217
	218	/* if locking succeeded, pin bo: */
	219	ret = msm_gem_get_iova_locked(&msm_obj->base,
	220	submit->gpu->id, &iova);
	221
	222	if (ret)
	223	break;
	224
	225	submit->bos[i].flags \|= BO_PINNED;
	226
	227	if (iova == submit->bos[i].iova) {
	228	submit->bos[i].flags \|= BO_VALID;
	229	} else {
	230	submit->bos[i].iova = iova;
	231	/* iova changed, so address in cmdstream is not valid: */
	232	submit->bos[i].flags &= ~BO_VALID;
	233	submit->valid = false;
	234	}
	235	}
	236
	237	return ret;
	238	}
	239
7198e6b0 RC	240	static int submit_bo(struct msm_gem_submit *submit, uint32_t idx,
	241	struct msm_gem_object *obj, uint32_t iova, bool *valid)
	242	{
	243	if (idx >= submit->nr_bos) {
19872533 RC	244	DRM_ERROR("invalid buffer index: %u (out of %u)\n",
	245	idx, submit->nr_bos);
	246	return -EINVAL;
7198e6b0 RC	247	}
	248
	249	if (obj)
	250	*obj = submit->bos[idx].obj;
	251	if (iova)
	252	*iova = submit->bos[idx].iova;
	253	if (valid)
	254	*valid = !!(submit->bos[idx].flags & BO_VALID);
	255
	256	return 0;
	257	}
	258
	259	/* process the reloc's and patch up the cmdstream as needed: */
	260	static int submit_reloc(struct msm_gem_submit submit, struct msm_gem_object obj,
	261	uint32_t offset, uint32_t nr_relocs, uint64_t relocs)
	262	{
	263	uint32_t i, last_offset = 0;
	264	uint32_t *ptr;
	265	int ret;
	266
	267	if (offset % 4) {
19872533	268	DRM_ERROR("non-aligned cmdstream buffer: %u\n", offset);
7198e6b0 RC	269	return -EINVAL;
	270	}
	271
	272	/* For now, just map the entire thing. Eventually we probably
	273	* to do it page-by-page, w/ kmap() if not vmap()d..
	274	*/
c2703b13	275	ptr = msm_gem_vaddr_locked(&obj->base);
7198e6b0 RC	276
	277	if (IS_ERR(ptr)) {
	278	ret = PTR_ERR(ptr);
	279	DBG("failed to map: %d", ret);
	280	return ret;
	281	}
	282
	283	for (i = 0; i < nr_relocs; i++) {
	284	struct drm_msm_gem_submit_reloc submit_reloc;
	285	void __user *userptr =
3ed605bc	286	u64_to_user_ptr(relocs + (i * sizeof(submit_reloc)));
7198e6b0 RC	287	uint32_t iova, off;
	288	bool valid;
	289
	290	ret = copy_from_user(&submit_reloc, userptr, sizeof(submit_reloc));
	291	if (ret)
	292	return -EFAULT;
	293
	294	if (submit_reloc.submit_offset % 4) {
19872533	295	DRM_ERROR("non-aligned reloc offset: %u\n",
7198e6b0 RC	296	submit_reloc.submit_offset);
	297	return -EINVAL;
	298	}
	299
	300	/* offset in dwords: */
	301	off = submit_reloc.submit_offset / 4;
	302
	303	if ((off >= (obj->base.size / 4)) \|\|
	304	(off < last_offset)) {
19872533	305	DRM_ERROR("invalid offset %u at reloc %u\n", off, i);
7198e6b0 RC	306	return -EINVAL;
	307	}
	308
	309	ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova, &valid);
	310	if (ret)
	311	return ret;
	312
	313	if (valid)
	314	continue;
	315
	316	iova += submit_reloc.reloc_offset;
	317
	318	if (submit_reloc.shift < 0)
	319	iova >>= -submit_reloc.shift;
	320	else
	321	iova <<= submit_reloc.shift;
	322
	323	ptr[off] = iova \| submit_reloc.or;
	324
	325	last_offset = off;
	326	}
	327
	328	return 0;
	329	}
	330
40e6815b	331	static void submit_cleanup(struct msm_gem_submit *submit)
7198e6b0 RC	332	{
	333	unsigned i;
	334
7198e6b0 RC	335	for (i = 0; i < submit->nr_bos; i++) {
	336	struct msm_gem_object *msm_obj = submit->bos[i].obj;
	337	submit_unlock_unpin_bo(submit, i);
	338	list_del_init(&msm_obj->submit_entry);
	339	drm_gem_object_unreference(&msm_obj->base);
	340	}
7198e6b0 RC	341
7198e6b0 RC	342	ww_acquire_fini(&submit->ticket);
7198e6b0 RC	343	}
	344
	345	int msm_ioctl_gem_submit(struct drm_device dev, void data,
	346	struct drm_file *file)
	347	{
	348	struct msm_drm_private *priv = dev->dev_private;
	349	struct drm_msm_gem_submit *args = data;
	350	struct msm_file_private *ctx = file->driver_priv;
	351	struct msm_gem_submit *submit;
c01a958e	352	struct msm_gpu *gpu = priv->gpu;
7198e6b0 RC	353	unsigned i;
	354	int ret;
	355
c01a958e RC	356	if (!gpu)
	357	return -ENXIO;
	358
7198e6b0 RC	359	/* for now, we just have 3d pipe.. eventually this would need to
	360	* be more clever to dispatch to appropriate gpu module:
	361	*/
	362	if (args->pipe != MSM_PIPE_3D0)
	363	return -EINVAL;
	364
7198e6b0 RC	365	if (args->nr_cmds > MAX_CMDS)
	366	return -EINVAL;
	367
	368	submit = submit_create(dev, gpu, args->nr_bos);
687f084a RC	369	if (!submit)
	370	return -ENOMEM;
	371
	372	mutex_lock(&dev->struct_mutex);
7198e6b0 RC	373
	374	ret = submit_lookup_objects(submit, args, file);
	375	if (ret)
	376	goto out;
	377
340faef2 RC	378	ret = submit_lock_objects(submit);
	379	if (ret)
	380	goto out;
	381
b6295f9a RC	382	ret = submit_fence_sync(submit);
	383	if (ret)
	384	goto out;
	385
340faef2	386	ret = submit_pin_objects(submit);
7198e6b0 RC	387	if (ret)
	388	goto out;
	389
	390	for (i = 0; i < args->nr_cmds; i++) {
	391	struct drm_msm_gem_submit_cmd submit_cmd;
	392	void __user *userptr =
3ed605bc	393	u64_to_user_ptr(args->cmds + (i * sizeof(submit_cmd)));
7198e6b0 RC	394	struct msm_gem_object *msm_obj;
	395	uint32_t iova;
	396
	397	ret = copy_from_user(&submit_cmd, userptr, sizeof(submit_cmd));
	398	if (ret) {
	399	ret = -EFAULT;
	400	goto out;
	401	}
	402
93ddb0d3 RC	403	/* validate input from userspace: */
	404	switch (submit_cmd.type) {
	405	case MSM_SUBMIT_CMD_BUF:
	406	case MSM_SUBMIT_CMD_IB_TARGET_BUF:
	407	case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
	408	break;
	409	default:
	410	DRM_ERROR("invalid type: %08x\n", submit_cmd.type);
	411	ret = -EINVAL;
	412	goto out;
	413	}
	414
7198e6b0 RC	415	ret = submit_bo(submit, submit_cmd.submit_idx,
	416	&msm_obj, &iova, NULL);
	417	if (ret)
	418	goto out;
	419
	420	if (submit_cmd.size % 4) {
19872533	421	DRM_ERROR("non-aligned cmdstream buffer size: %u\n",
7198e6b0 RC	422	submit_cmd.size);
	423	ret = -EINVAL;
	424	goto out;
	425	}
	426
19872533 RC	427	if ((submit_cmd.size + submit_cmd.submit_offset) >=
	428	msm_obj->base.size) {
	429	DRM_ERROR("invalid cmdstream size: %u\n", submit_cmd.size);
7198e6b0 RC	430	ret = -EINVAL;
	431	goto out;
	432	}
	433
	434	submit->cmd[i].type = submit_cmd.type;
	435	submit->cmd[i].size = submit_cmd.size / 4;
	436	submit->cmd[i].iova = iova + submit_cmd.submit_offset;
a7d3c950	437	submit->cmd[i].idx = submit_cmd.submit_idx;
7198e6b0 RC	438
	439	if (submit->valid)
	440	continue;
	441
	442	ret = submit_reloc(submit, msm_obj, submit_cmd.submit_offset,
	443	submit_cmd.nr_relocs, submit_cmd.relocs);
	444	if (ret)
	445	goto out;
	446	}
	447
	448	submit->nr_cmds = i;
	449
	450	ret = msm_gpu_submit(gpu, submit, ctx);
	451
b6295f9a	452	args->fence = submit->fence->seqno;
7198e6b0 RC	453
7198e6b0 RC	454	out:
40e6815b RC	455	submit_cleanup(submit);
	456	if (ret)
	457	msm_gem_submit_free(submit);
c2703b13	458	mutex_unlock(&dev->struct_mutex);
7198e6b0 RC	459	return ret;
7198e6b0 RC	460	}