[mirror_ubuntu-zesty-kernel.git] / drivers / gpu / drm / nouveau / nouveau_channel.c

/*
 * Copyright 2005-2006 Stephane Marchesin
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

#include "drmP.h"
#include "drm.h"
#include "nouveau_drv.h"
#include "nouveau_drm.h"
#include "nouveau_dma.h"

static int
nouveau_channel_pushbuf_ctxdma_init(struct nouveau_channel *chan)
{
	struct drm_device *dev = chan->dev;
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct nouveau_bo *pb = chan->pushbuf_bo;
	struct nouveau_gpuobj *pushbuf = NULL;
	int ret = 0;

	if (dev_priv->card_type >= NV_50) {
		if (dev_priv->card_type < NV_C0) {
			ret = nouveau_gpuobj_dma_new(chan,
						     NV_CLASS_DMA_IN_MEMORY, 0,
						     (1ULL << 40),
						     NV_MEM_ACCESS_RO,
						     NV_MEM_TARGET_VM,
						     &pushbuf);
		}
		chan->pushbuf_base = pb->bo.offset;
	} else
	if (pb->bo.mem.mem_type == TTM_PL_TT) {
		ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 0,
					     dev_priv->gart_info.aper_size,
					     NV_MEM_ACCESS_RO,
					     NV_MEM_TARGET_GART, &pushbuf);
		chan->pushbuf_base = pb->bo.mem.start << PAGE_SHIFT;
	} else
	if (dev_priv->card_type != NV_04) {
		ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 0,
					     dev_priv->fb_available_size,
					     NV_MEM_ACCESS_RO,
					     NV_MEM_TARGET_VRAM, &pushbuf);
		chan->pushbuf_base = pb->bo.mem.start << PAGE_SHIFT;
	} else {
		/* NV04 cmdbuf hack, from original ddx.. not sure of it's
		 * exact reason for existing :)  PCI access to cmdbuf in
		 * VRAM.
		 */
		ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
					     pci_resource_start(dev->pdev, 1),
					     dev_priv->fb_available_size,
					     NV_MEM_ACCESS_RO,
					     NV_MEM_TARGET_PCI, &pushbuf);
		chan->pushbuf_base = pb->bo.mem.start << PAGE_SHIFT;
	}

	nouveau_gpuobj_ref(pushbuf, &chan->pushbuf);
	nouveau_gpuobj_ref(NULL, &pushbuf);
	return ret;
}

static struct nouveau_bo *
nouveau_channel_user_pushbuf_alloc(struct drm_device *dev)
{
	struct nouveau_bo *pushbuf = NULL;
	int location, ret;

	if (nouveau_vram_pushbuf)
		location = TTM_PL_FLAG_VRAM;
	else
		location = TTM_PL_FLAG_TT;

	ret = nouveau_bo_new(dev, NULL, 65536, 0, location, 0, 0x0000, &pushbuf);
	if (ret) {
		NV_ERROR(dev, "error allocating DMA push buffer: %d\n", ret);
		return NULL;
	}

	ret = nouveau_bo_pin(pushbuf, location);
	if (ret) {
		NV_ERROR(dev, "error pinning DMA push buffer: %d\n", ret);
		nouveau_bo_ref(NULL, &pushbuf);
		return NULL;
	}

	ret = nouveau_bo_map(pushbuf);
	if (ret) {
		nouveau_bo_unpin(pushbuf);
		nouveau_bo_ref(NULL, &pushbuf);
		return NULL;
	}

	return pushbuf;
}

/* allocates and initializes a fifo for user space consumption */
int
nouveau_channel_alloc(struct drm_device *dev, struct nouveau_channel **chan_ret,
		      struct drm_file *file_priv,
		      uint32_t vram_handle, uint32_t gart_handle)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
	struct nouveau_channel *chan;
	unsigned long flags;
	int ret;

	/* allocate and lock channel structure */
	chan = kzalloc(sizeof(*chan), GFP_KERNEL);
	if (!chan)
		return -ENOMEM;
	chan->dev = dev;
	chan->file_priv = file_priv;
	chan->vram_handle = vram_handle;
	chan->gart_handle = gart_handle;

	kref_init(&chan->ref);
	atomic_set(&chan->users, 1);
	mutex_init(&chan->mutex);
	mutex_lock(&chan->mutex);

	/* allocate hw channel id */
	spin_lock_irqsave(&dev_priv->channels.lock, flags);
	for (chan->id = 0; chan->id < pfifo->channels; chan->id++) {
		if (!dev_priv->channels.ptr[chan->id]) {
			nouveau_channel_ref(chan, &dev_priv->channels.ptr[chan->id]);
			break;
		}
	}
	spin_unlock_irqrestore(&dev_priv->channels.lock, flags);

	if (chan->id == pfifo->channels) {
		mutex_unlock(&chan->mutex);
		kfree(chan);
		return -ENODEV;
	}

	NV_DEBUG(dev, "initialising channel %d\n", chan->id);
	INIT_LIST_HEAD(&chan->nvsw.vbl_wait);
	INIT_LIST_HEAD(&chan->nvsw.flip);
	INIT_LIST_HEAD(&chan->fence.pending);

	/* Allocate DMA push buffer */
	chan->pushbuf_bo = nouveau_channel_user_pushbuf_alloc(dev);
	if (!chan->pushbuf_bo) {
		ret = -ENOMEM;
		NV_ERROR(dev, "pushbuf %d\n", ret);
		nouveau_channel_put(&chan);
		return ret;
	}

	nouveau_dma_pre_init(chan);
	chan->user_put = 0x40;
	chan->user_get = 0x44;

	/* Allocate space for per-channel fixed notifier memory */
	ret = nouveau_notifier_init_channel(chan);
	if (ret) {
		NV_ERROR(dev, "ntfy %d\n", ret);
		nouveau_channel_put(&chan);
		return ret;
	}

	/* Setup channel's default objects */
	ret = nouveau_gpuobj_channel_init(chan, vram_handle, gart_handle);
	if (ret) {
		NV_ERROR(dev, "gpuobj %d\n", ret);
		nouveau_channel_put(&chan);
		return ret;
	}

	/* Create a dma object for the push buffer */
	ret = nouveau_channel_pushbuf_ctxdma_init(chan);
	if (ret) {
		NV_ERROR(dev, "pbctxdma %d\n", ret);
		nouveau_channel_put(&chan);
		return ret;
	}

	/* disable the fifo caches */
	pfifo->reassign(dev, false);

	/* Construct initial RAMFC for new channel */
	ret = pfifo->create_context(chan);
	if (ret) {
		nouveau_channel_put(&chan);
		return ret;
	}

	pfifo->reassign(dev, true);

	ret = nouveau_dma_init(chan);
	if (!ret)
		ret = nouveau_fence_channel_init(chan);
	if (ret) {
		nouveau_channel_put(&chan);
		return ret;
	}

	nouveau_debugfs_channel_init(chan);

	NV_DEBUG(dev, "channel %d initialised\n", chan->id);
	*chan_ret = chan;
	return 0;
}

struct nouveau_channel *
nouveau_channel_get_unlocked(struct nouveau_channel *ref)
{
	struct nouveau_channel *chan = NULL;

	if (likely(ref && atomic_inc_not_zero(&ref->users)))
		nouveau_channel_ref(ref, &chan);

	return chan;
}

struct nouveau_channel *
nouveau_channel_get(struct drm_device *dev, struct drm_file *file_priv, int id)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct nouveau_channel *chan;
	unsigned long flags;

	if (unlikely(id < 0 || id >= NOUVEAU_MAX_CHANNEL_NR))
		return ERR_PTR(-EINVAL);

	spin_lock_irqsave(&dev_priv->channels.lock, flags);
	chan = nouveau_channel_get_unlocked(dev_priv->channels.ptr[id]);
	spin_unlock_irqrestore(&dev_priv->channels.lock, flags);

	if (unlikely(!chan))
		return ERR_PTR(-EINVAL);

	if (unlikely(file_priv && chan->file_priv != file_priv)) {
		nouveau_channel_put_unlocked(&chan);
		return ERR_PTR(-EINVAL);
	}

	mutex_lock(&chan->mutex);
	return chan;
}

void
nouveau_channel_put_unlocked(struct nouveau_channel **pchan)
{
	struct nouveau_channel *chan = *pchan;
	struct drm_device *dev = chan->dev;
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
	struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
	struct nouveau_crypt_engine *pcrypt = &dev_priv->engine.crypt;
	unsigned long flags;

	/* decrement the refcount, and we're done if there's still refs */
	if (likely(!atomic_dec_and_test(&chan->users))) {
		nouveau_channel_ref(NULL, pchan);
		return;
	}

	/* no one wants the channel anymore */
	NV_DEBUG(dev, "freeing channel %d\n", chan->id);
	nouveau_debugfs_channel_fini(chan);

	/* give it chance to idle */
	nouveau_channel_idle(chan);

	/* ensure all outstanding fences are signaled.  they should be if the
	 * above attempts at idling were OK, but if we failed this'll tell TTM
	 * we're done with the buffers.
	 */
	nouveau_fence_channel_fini(chan);

	/* boot it off the hardware */
	pfifo->reassign(dev, false);

	/* We want to give pgraph a chance to idle and get rid of all
	 * potential errors. We need to do this without the context
	 * switch lock held, otherwise the irq handler is unable to
	 * process them.
	 */
	if (pgraph->channel(dev) == chan)
		nouveau_wait_for_idle(dev);

	/* destroy the engine specific contexts */
	pfifo->destroy_context(chan);
	pgraph->destroy_context(chan);
	if (pcrypt->destroy_context)
		pcrypt->destroy_context(chan);

	pfifo->reassign(dev, true);

	/* aside from its resources, the channel should now be dead,
	 * remove it from the channel list
	 */
	spin_lock_irqsave(&dev_priv->channels.lock, flags);
	nouveau_channel_ref(NULL, &dev_priv->channels.ptr[chan->id]);
	spin_unlock_irqrestore(&dev_priv->channels.lock, flags);

	/* destroy any resources the channel owned */
	nouveau_gpuobj_ref(NULL, &chan->pushbuf);
	if (chan->pushbuf_bo) {
		nouveau_bo_unmap(chan->pushbuf_bo);
		nouveau_bo_unpin(chan->pushbuf_bo);
		nouveau_bo_ref(NULL, &chan->pushbuf_bo);
	}
	nouveau_gpuobj_channel_takedown(chan);
	nouveau_notifier_takedown_channel(chan);

	nouveau_channel_ref(NULL, pchan);
}

void
nouveau_channel_put(struct nouveau_channel **pchan)
{
	mutex_unlock(&(*pchan)->mutex);
	nouveau_channel_put_unlocked(pchan);
}

static void
nouveau_channel_del(struct kref *ref)
{
	struct nouveau_channel *chan =
		container_of(ref, struct nouveau_channel, ref);

	kfree(chan);
}

void
nouveau_channel_ref(struct nouveau_channel *chan,
		    struct nouveau_channel **pchan)
{
	if (chan)
		kref_get(&chan->ref);

	if (*pchan)
		kref_put(&(*pchan)->ref, nouveau_channel_del);

	*pchan = chan;
}

void
nouveau_channel_idle(struct nouveau_channel *chan)
{
	struct drm_device *dev = chan->dev;
	struct nouveau_fence *fence = NULL;
	int ret;

	nouveau_fence_update(chan);

	if (chan->fence.sequence != chan->fence.sequence_ack) {
		ret = nouveau_fence_new(chan, &fence, true);
		if (!ret) {
			ret = nouveau_fence_wait(fence, false, false);
			nouveau_fence_unref(&fence);
		}

		if (ret)
			NV_ERROR(dev, "Failed to idle channel %d.\n", chan->id);
	}
}

/* cleans up all the fifos from file_priv */
void
nouveau_channel_cleanup(struct drm_device *dev, struct drm_file *file_priv)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct nouveau_engine *engine = &dev_priv->engine;
	struct nouveau_channel *chan;
	int i;

	NV_DEBUG(dev, "clearing FIFO enables from file_priv\n");
	for (i = 0; i < engine->fifo.channels; i++) {
		chan = nouveau_channel_get(dev, file_priv, i);
		if (IS_ERR(chan))
			continue;

		atomic_dec(&chan->users);
		nouveau_channel_put(&chan);
	}
}


/***********************************
 * ioctls wrapping the functions
 ***********************************/

static int
nouveau_ioctl_fifo_alloc(struct drm_device *dev, void *data,
			 struct drm_file *file_priv)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct drm_nouveau_channel_alloc *init = data;
	struct nouveau_channel *chan;
	int ret;

	if (dev_priv->engine.graph.accel_blocked)
		return -ENODEV;

	if (init->fb_ctxdma_handle == ~0 || init->tt_ctxdma_handle == ~0)
		return -EINVAL;

	ret = nouveau_channel_alloc(dev, &chan, file_priv,
				    init->fb_ctxdma_handle,
				    init->tt_ctxdma_handle);
	if (ret)
		return ret;
	init->channel  = chan->id;

	if (chan->dma.ib_max)
		init->pushbuf_domains = NOUVEAU_GEM_DOMAIN_VRAM |
					NOUVEAU_GEM_DOMAIN_GART;
	else if (chan->pushbuf_bo->bo.mem.mem_type == TTM_PL_VRAM)
		init->pushbuf_domains = NOUVEAU_GEM_DOMAIN_VRAM;
	else
		init->pushbuf_domains = NOUVEAU_GEM_DOMAIN_GART;

	if (dev_priv->card_type < NV_C0) {
		init->subchan[0].handle = NvM2MF;
		if (dev_priv->card_type < NV_50)
			init->subchan[0].grclass = 0x0039;
		else
			init->subchan[0].grclass = 0x5039;
		init->subchan[1].handle = NvSw;
		init->subchan[1].grclass = NV_SW;
		init->nr_subchan = 2;
	} else {
		init->subchan[0].handle  = 0x9039;
		init->subchan[0].grclass = 0x9039;
		init->nr_subchan = 1;
	}

	/* Named memory object area */
	ret = drm_gem_handle_create(file_priv, chan->notifier_bo->gem,
				    &init->notifier_handle);

	if (ret == 0)
		atomic_inc(&chan->users); /* userspace reference */
	nouveau_channel_put(&chan);
	return ret;
}

static int
nouveau_ioctl_fifo_free(struct drm_device *dev, void *data,
			struct drm_file *file_priv)
{
	struct drm_nouveau_channel_free *req = data;
	struct nouveau_channel *chan;

	chan = nouveau_channel_get(dev, file_priv, req->channel);
	if (IS_ERR(chan))
		return PTR_ERR(chan);

	atomic_dec(&chan->users);
	nouveau_channel_put(&chan);
	return 0;
}

/***********************************
 * finally, the ioctl table
 ***********************************/

struct drm_ioctl_desc nouveau_ioctls[] = {
	DRM_IOCTL_DEF_DRV(NOUVEAU_GETPARAM, nouveau_ioctl_getparam, DRM_UNLOCKED|DRM_AUTH),
	DRM_IOCTL_DEF_DRV(NOUVEAU_SETPARAM, nouveau_ioctl_setparam, DRM_UNLOCKED|DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
	DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_ALLOC, nouveau_ioctl_fifo_alloc, DRM_UNLOCKED|DRM_AUTH),
	DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_FREE, nouveau_ioctl_fifo_free, DRM_UNLOCKED|DRM_AUTH),
	DRM_IOCTL_DEF_DRV(NOUVEAU_GROBJ_ALLOC, nouveau_ioctl_grobj_alloc, DRM_UNLOCKED|DRM_AUTH),
	DRM_IOCTL_DEF_DRV(NOUVEAU_NOTIFIEROBJ_ALLOC, nouveau_ioctl_notifier_alloc, DRM_UNLOCKED|DRM_AUTH),
	DRM_IOCTL_DEF_DRV(NOUVEAU_GPUOBJ_FREE, nouveau_ioctl_gpuobj_free, DRM_UNLOCKED|DRM_AUTH),
	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_NEW, nouveau_gem_ioctl_new, DRM_UNLOCKED|DRM_AUTH),
	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_PUSHBUF, nouveau_gem_ioctl_pushbuf, DRM_UNLOCKED|DRM_AUTH),
	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_PREP, nouveau_gem_ioctl_cpu_prep, DRM_UNLOCKED|DRM_AUTH),
	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_FINI, nouveau_gem_ioctl_cpu_fini, DRM_UNLOCKED|DRM_AUTH),
	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_INFO, nouveau_gem_ioctl_info, DRM_UNLOCKED|DRM_AUTH),
};

int nouveau_max_ioctl = DRM_ARRAY_SIZE(nouveau_ioctls);
Commit	Line	Data
6ee73861 BS	1	/*
	2	* Copyright 2005-2006 Stephane Marchesin
	3	* All Rights Reserved.
	4	*
	5	* Permission is hereby granted, free of charge, to any person obtaining a
	6	* copy of this software and associated documentation files (the "Software"),
	7	* to deal in the Software without restriction, including without limitation
	8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	9	* and/or sell copies of the Software, and to permit persons to whom the
	10	* Software is furnished to do so, subject to the following conditions:
	11	*
	12	* The above copyright notice and this permission notice (including the next
	13	* paragraph) shall be included in all copies or substantial portions of the
	14	* Software.
	15	*
	16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	19	* PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
	20	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	21	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
	22	* DEALINGS IN THE SOFTWARE.
	23	*/
	24
	25	#include "drmP.h"
	26	#include "drm.h"
	27	#include "nouveau_drv.h"
	28	#include "nouveau_drm.h"
	29	#include "nouveau_dma.h"
	30
	31	static int
	32	nouveau_channel_pushbuf_ctxdma_init(struct nouveau_channel *chan)
	33	{
	34	struct drm_device *dev = chan->dev;
	35	struct drm_nouveau_private *dev_priv = dev->dev_private;
	36	struct nouveau_bo *pb = chan->pushbuf_bo;
	37	struct nouveau_gpuobj *pushbuf = NULL;
ea5f2786	38	int ret = 0;
6ee73861	39
d87897d4	40	if (dev_priv->card_type >= NV_50) {
96545299 BS	41	if (dev_priv->card_type < NV_C0) {
	42	ret = nouveau_gpuobj_dma_new(chan,
	43	NV_CLASS_DMA_IN_MEMORY, 0,
	44	(1ULL << 40),
	45	NV_MEM_ACCESS_RO,
	46	NV_MEM_TARGET_VM,
	47	&pushbuf);
	48	}
d87897d4 BS	49	chan->pushbuf_base = pb->bo.offset;
d87897d4 BS	50	} else
6ee73861	51	if (pb->bo.mem.mem_type == TTM_PL_TT) {
7f4a195f BS	52	ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 0,
	53	dev_priv->gart_info.aper_size,
	54	NV_MEM_ACCESS_RO,
	55	NV_MEM_TARGET_GART, &pushbuf);
d961db75	56	chan->pushbuf_base = pb->bo.mem.start << PAGE_SHIFT;
6ee73861 BS	57	} else
	58	if (dev_priv->card_type != NV_04) {
	59	ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 0,
	60	dev_priv->fb_available_size,
7f4a195f BS	61	NV_MEM_ACCESS_RO,
7f4a195f BS	62	NV_MEM_TARGET_VRAM, &pushbuf);
d961db75	63	chan->pushbuf_base = pb->bo.mem.start << PAGE_SHIFT;
6ee73861 BS	64	} else {
	65	/* NV04 cmdbuf hack, from original ddx.. not sure of it's
	66	* exact reason for existing :) PCI access to cmdbuf in
	67	* VRAM.
	68	*/
	69	ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
7f4a195f	70	pci_resource_start(dev->pdev, 1),
6ee73861	71	dev_priv->fb_available_size,
7f4a195f BS	72	NV_MEM_ACCESS_RO,
7f4a195f BS	73	NV_MEM_TARGET_PCI, &pushbuf);
d961db75	74	chan->pushbuf_base = pb->bo.mem.start << PAGE_SHIFT;
6ee73861 BS	75	}
6ee73861 BS	76
a8eaebc6 BS	77	nouveau_gpuobj_ref(pushbuf, &chan->pushbuf);
a8eaebc6 BS	78	nouveau_gpuobj_ref(NULL, &pushbuf);
96545299	79	return ret;
6ee73861 BS	80	}
	81
	82	static struct nouveau_bo *
	83	nouveau_channel_user_pushbuf_alloc(struct drm_device *dev)
	84	{
	85	struct nouveau_bo *pushbuf = NULL;
	86	int location, ret;
	87
	88	if (nouveau_vram_pushbuf)
	89	location = TTM_PL_FLAG_VRAM;
	90	else
	91	location = TTM_PL_FLAG_TT;
	92
d550c41e	93	ret = nouveau_bo_new(dev, NULL, 65536, 0, location, 0, 0x0000, &pushbuf);
6ee73861 BS	94	if (ret) {
	95	NV_ERROR(dev, "error allocating DMA push buffer: %d\n", ret);
	96	return NULL;
	97	}
	98
	99	ret = nouveau_bo_pin(pushbuf, location);
	100	if (ret) {
	101	NV_ERROR(dev, "error pinning DMA push buffer: %d\n", ret);
	102	nouveau_bo_ref(NULL, &pushbuf);
	103	return NULL;
	104	}
	105
96545299 BS	106	ret = nouveau_bo_map(pushbuf);
	107	if (ret) {
	108	nouveau_bo_unpin(pushbuf);
	109	nouveau_bo_ref(NULL, &pushbuf);
	110	return NULL;
	111	}
	112
6ee73861 BS	113	return pushbuf;
	114	}
	115
	116	/* allocates and initializes a fifo for user space consumption */
	117	int
	118	nouveau_channel_alloc(struct drm_device dev, struct nouveau_channel *chan_ret,
	119	struct drm_file *file_priv,
cff5c133	120	uint32_t vram_handle, uint32_t gart_handle)
6ee73861 BS	121	{
6ee73861 BS	122	struct drm_nouveau_private *dev_priv = dev->dev_private;
6ee73861 BS	123	struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
6ee73861 BS	124	struct nouveau_channel *chan;
cff5c133	125	unsigned long flags;
d908175c	126	int ret;
6ee73861	127
cff5c133 BS	128	/* allocate and lock channel structure */
	129	chan = kzalloc(sizeof(*chan), GFP_KERNEL);
	130	if (!chan)
6ee73861	131	return -ENOMEM;
6ee73861	132	chan->dev = dev;
6ee73861 BS	133	chan->file_priv = file_priv;
6ee73861 BS	134	chan->vram_handle = vram_handle;
cff5c133 BS	135	chan->gart_handle = gart_handle;
cff5c133 BS	136
f091a3d4 FJ	137	kref_init(&chan->ref);
f091a3d4 FJ	138	atomic_set(&chan->users, 1);
6a6b73f2	139	mutex_init(&chan->mutex);
cff5c133	140	mutex_lock(&chan->mutex);
6ee73861	141
cff5c133 BS	142	/* allocate hw channel id */
	143	spin_lock_irqsave(&dev_priv->channels.lock, flags);
	144	for (chan->id = 0; chan->id < pfifo->channels; chan->id++) {
	145	if (!dev_priv->channels.ptr[chan->id]) {
f091a3d4	146	nouveau_channel_ref(chan, &dev_priv->channels.ptr[chan->id]);
cff5c133 BS	147	break;
	148	}
	149	}
	150	spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
	151
	152	if (chan->id == pfifo->channels) {
	153	mutex_unlock(&chan->mutex);
	154	kfree(chan);
	155	return -ENODEV;
	156	}
	157
	158	NV_DEBUG(dev, "initialising channel %d\n", chan->id);
	159	INIT_LIST_HEAD(&chan->nvsw.vbl_wait);
332b242f	160	INIT_LIST_HEAD(&chan->nvsw.flip);
cff5c133	161	INIT_LIST_HEAD(&chan->fence.pending);
6ee73861 BS	162
	163	/* Allocate DMA push buffer */
	164	chan->pushbuf_bo = nouveau_channel_user_pushbuf_alloc(dev);
	165	if (!chan->pushbuf_bo) {
	166	ret = -ENOMEM;
	167	NV_ERROR(dev, "pushbuf %d\n", ret);
cff5c133	168	nouveau_channel_put(&chan);
6ee73861 BS	169	return ret;
	170	}
	171
75c99da6	172	nouveau_dma_pre_init(chan);
6ee73861 BS	173	chan->user_put = 0x40;
	174	chan->user_get = 0x44;
	175
	176	/* Allocate space for per-channel fixed notifier memory */
	177	ret = nouveau_notifier_init_channel(chan);
	178	if (ret) {
	179	NV_ERROR(dev, "ntfy %d\n", ret);
cff5c133	180	nouveau_channel_put(&chan);
6ee73861 BS	181	return ret;
	182	}
	183
	184	/* Setup channel's default objects */
cff5c133	185	ret = nouveau_gpuobj_channel_init(chan, vram_handle, gart_handle);
6ee73861 BS	186	if (ret) {
6ee73861 BS	187	NV_ERROR(dev, "gpuobj %d\n", ret);
cff5c133	188	nouveau_channel_put(&chan);
6ee73861 BS	189	return ret;
	190	}
	191
	192	/* Create a dma object for the push buffer */
	193	ret = nouveau_channel_pushbuf_ctxdma_init(chan);
	194	if (ret) {
	195	NV_ERROR(dev, "pbctxdma %d\n", ret);
cff5c133	196	nouveau_channel_put(&chan);
6ee73861 BS	197	return ret;
	198	}
	199
	200	/* disable the fifo caches */
	201	pfifo->reassign(dev, false);
	202
25985edc	203	/* Construct initial RAMFC for new channel */
6ee73861 BS	204	ret = pfifo->create_context(chan);
6ee73861 BS	205	if (ret) {
cff5c133	206	nouveau_channel_put(&chan);
6ee73861 BS	207	return ret;
	208	}
	209
	210	pfifo->reassign(dev, true);
	211
	212	ret = nouveau_dma_init(chan);
	213	if (!ret)
2730723b	214	ret = nouveau_fence_channel_init(chan);
6ee73861	215	if (ret) {
cff5c133	216	nouveau_channel_put(&chan);
6ee73861 BS	217	return ret;
	218	}
	219
	220	nouveau_debugfs_channel_init(chan);
	221
cff5c133	222	NV_DEBUG(dev, "channel %d initialised\n", chan->id);
6ee73861 BS	223	*chan_ret = chan;
	224	return 0;
	225	}
	226
feeb0aec FJ	227	struct nouveau_channel *
	228	nouveau_channel_get_unlocked(struct nouveau_channel *ref)
	229	{
f091a3d4	230	struct nouveau_channel *chan = NULL;
feeb0aec	231
f091a3d4 FJ	232	if (likely(ref && atomic_inc_not_zero(&ref->users)))
	233	nouveau_channel_ref(ref, &chan);
	234
	235	return chan;
feeb0aec FJ	236	}
feeb0aec FJ	237
cff5c133 BS	238	struct nouveau_channel *
	239	nouveau_channel_get(struct drm_device dev, struct drm_file file_priv, int id)
	240	{
	241	struct drm_nouveau_private *dev_priv = dev->dev_private;
feeb0aec	242	struct nouveau_channel *chan;
cff5c133 BS	243	unsigned long flags;
cff5c133 BS	244
8e91182b MH	245	if (unlikely(id < 0 \|\| id >= NOUVEAU_MAX_CHANNEL_NR))
	246	return ERR_PTR(-EINVAL);
	247
cff5c133	248	spin_lock_irqsave(&dev_priv->channels.lock, flags);
feeb0aec FJ	249	chan = nouveau_channel_get_unlocked(dev_priv->channels.ptr[id]);
feeb0aec FJ	250	spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
cff5c133	251
feeb0aec	252	if (unlikely(!chan))
cff5c133	253	return ERR_PTR(-EINVAL);
cff5c133	254
feeb0aec FJ	255	if (unlikely(file_priv && chan->file_priv != file_priv)) {
feeb0aec FJ	256	nouveau_channel_put_unlocked(&chan);
cff5c133 BS	257	return ERR_PTR(-EINVAL);
	258	}
	259
cff5c133 BS	260	mutex_lock(&chan->mutex);
	261	return chan;
	262	}
	263
6ee73861	264	void
feeb0aec	265	nouveau_channel_put_unlocked(struct nouveau_channel **pchan)
6ee73861	266	{
cff5c133	267	struct nouveau_channel chan = pchan;
6ee73861 BS	268	struct drm_device *dev = chan->dev;
6ee73861 BS	269	struct drm_nouveau_private *dev_priv = dev->dev_private;
6ee73861	270	struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
cff5c133	271	struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
bd2e597d	272	struct nouveau_crypt_engine *pcrypt = &dev_priv->engine.crypt;
6ee73861	273	unsigned long flags;
6ee73861	274
cff5c133	275	/* decrement the refcount, and we're done if there's still refs */
f091a3d4 FJ	276	if (likely(!atomic_dec_and_test(&chan->users))) {
f091a3d4 FJ	277	nouveau_channel_ref(NULL, pchan);
cff5c133 BS	278	return;
cff5c133 BS	279	}
6ee73861	280
25985edc	281	/* no one wants the channel anymore */
cff5c133	282	NV_DEBUG(dev, "freeing channel %d\n", chan->id);
6ee73861 BS	283	nouveau_debugfs_channel_fini(chan);
6ee73861 BS	284
cff5c133	285	/* give it chance to idle */
6dccd311	286	nouveau_channel_idle(chan);
6ee73861	287
cff5c133	288	/* ensure all outstanding fences are signaled. they should be if the
6ee73861 BS	289	* above attempts at idling were OK, but if we failed this'll tell TTM
	290	* we're done with the buffers.
	291	*/
2730723b	292	nouveau_fence_channel_fini(chan);
6ee73861	293
cff5c133	294	/* boot it off the hardware */
6ee73861 BS	295	pfifo->reassign(dev, false);
6ee73861 BS	296
3945e475 FJ	297	/* We want to give pgraph a chance to idle and get rid of all
	298	* potential errors. We need to do this without the context
	299	* switch lock held, otherwise the irq handler is unable to
	300	* process them.
ff9e5279 MM	301	*/
	302	if (pgraph->channel(dev) == chan)
	303	nouveau_wait_for_idle(dev);
	304
3945e475	305	/* destroy the engine specific contexts */
6ee73861	306	pfifo->destroy_context(chan);
3945e475	307	pgraph->destroy_context(chan);
bd2e597d BS	308	if (pcrypt->destroy_context)
bd2e597d BS	309	pcrypt->destroy_context(chan);
6ee73861 BS	310
	311	pfifo->reassign(dev, true);
	312
cff5c133 BS	313	/* aside from its resources, the channel should now be dead,
	314	* remove it from the channel list
	315	*/
	316	spin_lock_irqsave(&dev_priv->channels.lock, flags);
f091a3d4	317	nouveau_channel_ref(NULL, &dev_priv->channels.ptr[chan->id]);
cff5c133 BS	318	spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
	319
	320	/* destroy any resources the channel owned */
a8eaebc6	321	nouveau_gpuobj_ref(NULL, &chan->pushbuf);
6ee73861	322	if (chan->pushbuf_bo) {
9d59e8a1	323	nouveau_bo_unmap(chan->pushbuf_bo);
6ee73861 BS	324	nouveau_bo_unpin(chan->pushbuf_bo);
	325	nouveau_bo_ref(NULL, &chan->pushbuf_bo);
	326	}
	327	nouveau_gpuobj_channel_takedown(chan);
	328	nouveau_notifier_takedown_channel(chan);
6ee73861	329
f091a3d4	330	nouveau_channel_ref(NULL, pchan);
6ee73861 BS	331	}
6ee73861 BS	332
feeb0aec FJ	333	void
	334	nouveau_channel_put(struct nouveau_channel **pchan)
	335	{
	336	mutex_unlock(&(*pchan)->mutex);
	337	nouveau_channel_put_unlocked(pchan);
	338	}
	339
f091a3d4 FJ	340	static void
	341	nouveau_channel_del(struct kref *ref)
	342	{
	343	struct nouveau_channel *chan =
	344	container_of(ref, struct nouveau_channel, ref);
	345
f091a3d4 FJ	346	kfree(chan);
	347	}
	348
	349	void
	350	nouveau_channel_ref(struct nouveau_channel *chan,
	351	struct nouveau_channel **pchan)
	352	{
	353	if (chan)
	354	kref_get(&chan->ref);
	355
	356	if (*pchan)
	357	kref_put(&(*pchan)->ref, nouveau_channel_del);
	358
	359	*pchan = chan;
	360	}
	361
6dccd311 FJ	362	void
	363	nouveau_channel_idle(struct nouveau_channel *chan)
	364	{
	365	struct drm_device *dev = chan->dev;
	366	struct nouveau_fence *fence = NULL;
	367	int ret;
	368
	369	nouveau_fence_update(chan);
	370
	371	if (chan->fence.sequence != chan->fence.sequence_ack) {
	372	ret = nouveau_fence_new(chan, &fence, true);
	373	if (!ret) {
	374	ret = nouveau_fence_wait(fence, false, false);
	375	nouveau_fence_unref(&fence);
	376	}
	377
	378	if (ret)
	379	NV_ERROR(dev, "Failed to idle channel %d.\n", chan->id);
	380	}
	381	}
	382
6ee73861 BS	383	/* cleans up all the fifos from file_priv */
	384	void
	385	nouveau_channel_cleanup(struct drm_device dev, struct drm_file file_priv)
	386	{
	387	struct drm_nouveau_private *dev_priv = dev->dev_private;
	388	struct nouveau_engine *engine = &dev_priv->engine;
cff5c133	389	struct nouveau_channel *chan;
6ee73861 BS	390	int i;
	391
	392	NV_DEBUG(dev, "clearing FIFO enables from file_priv\n");
	393	for (i = 0; i < engine->fifo.channels; i++) {
cff5c133 BS	394	chan = nouveau_channel_get(dev, file_priv, i);
	395	if (IS_ERR(chan))
	396	continue;
6ee73861	397
f091a3d4	398	atomic_dec(&chan->users);
cff5c133	399	nouveau_channel_put(&chan);
6ee73861 BS	400	}
	401	}
	402
6ee73861 BS	403
	404	/***********************************
	405	* ioctls wrapping the functions
	406	***********************************/
	407
	408	static int
	409	nouveau_ioctl_fifo_alloc(struct drm_device dev, void data,
	410	struct drm_file *file_priv)
	411	{
	412	struct drm_nouveau_private *dev_priv = dev->dev_private;
	413	struct drm_nouveau_channel_alloc *init = data;
	414	struct nouveau_channel *chan;
	415	int ret;
	416
6ee73861 BS	417	if (dev_priv->engine.graph.accel_blocked)
	418	return -ENODEV;
	419
	420	if (init->fb_ctxdma_handle == ~0 \|\| init->tt_ctxdma_handle == ~0)
	421	return -EINVAL;
	422
	423	ret = nouveau_channel_alloc(dev, &chan, file_priv,
	424	init->fb_ctxdma_handle,
	425	init->tt_ctxdma_handle);
	426	if (ret)
	427	return ret;
	428	init->channel = chan->id;
	429
a1606a95 BS	430	if (chan->dma.ib_max)
	431	init->pushbuf_domains = NOUVEAU_GEM_DOMAIN_VRAM \|
	432	NOUVEAU_GEM_DOMAIN_GART;
	433	else if (chan->pushbuf_bo->bo.mem.mem_type == TTM_PL_VRAM)
	434	init->pushbuf_domains = NOUVEAU_GEM_DOMAIN_VRAM;
	435	else
	436	init->pushbuf_domains = NOUVEAU_GEM_DOMAIN_GART;
	437
2a55c9a7 BS	438	if (dev_priv->card_type < NV_C0) {
	439	init->subchan[0].handle = NvM2MF;
	440	if (dev_priv->card_type < NV_50)
	441	init->subchan[0].grclass = 0x0039;
	442	else
	443	init->subchan[0].grclass = 0x5039;
	444	init->subchan[1].handle = NvSw;
	445	init->subchan[1].grclass = NV_SW;
	446	init->nr_subchan = 2;
	447	} else {
	448	init->subchan[0].handle = 0x9039;
	449	init->subchan[0].grclass = 0x9039;
	450	init->nr_subchan = 1;
	451	}
6ee73861 BS	452
	453	/* Named memory object area */
	454	ret = drm_gem_handle_create(file_priv, chan->notifier_bo->gem,
	455	&init->notifier_handle);
6ee73861	456
cff5c133	457	if (ret == 0)
f091a3d4	458	atomic_inc(&chan->users); /* userspace reference */
cff5c133 BS	459	nouveau_channel_put(&chan);
cff5c133 BS	460	return ret;
6ee73861 BS	461	}
	462
	463	static int
	464	nouveau_ioctl_fifo_free(struct drm_device dev, void data,
	465	struct drm_file *file_priv)
	466	{
cff5c133	467	struct drm_nouveau_channel_free *req = data;
6ee73861 BS	468	struct nouveau_channel *chan;
6ee73861 BS	469
cff5c133 BS	470	chan = nouveau_channel_get(dev, file_priv, req->channel);
	471	if (IS_ERR(chan))
	472	return PTR_ERR(chan);
6ee73861	473
f091a3d4	474	atomic_dec(&chan->users);
cff5c133	475	nouveau_channel_put(&chan);
6ee73861 BS	476	return 0;
	477	}
	478
	479	/***********************************
	480	* finally, the ioctl table
	481	***********************************/
	482
	483	struct drm_ioctl_desc nouveau_ioctls[] = {
b12120a5 BS	484	DRM_IOCTL_DEF_DRV(NOUVEAU_GETPARAM, nouveau_ioctl_getparam, DRM_UNLOCKED\|DRM_AUTH),
	485	DRM_IOCTL_DEF_DRV(NOUVEAU_SETPARAM, nouveau_ioctl_setparam, DRM_UNLOCKED\|DRM_AUTH\|DRM_MASTER\|DRM_ROOT_ONLY),
	486	DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_ALLOC, nouveau_ioctl_fifo_alloc, DRM_UNLOCKED\|DRM_AUTH),
	487	DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_FREE, nouveau_ioctl_fifo_free, DRM_UNLOCKED\|DRM_AUTH),
	488	DRM_IOCTL_DEF_DRV(NOUVEAU_GROBJ_ALLOC, nouveau_ioctl_grobj_alloc, DRM_UNLOCKED\|DRM_AUTH),
	489	DRM_IOCTL_DEF_DRV(NOUVEAU_NOTIFIEROBJ_ALLOC, nouveau_ioctl_notifier_alloc, DRM_UNLOCKED\|DRM_AUTH),
	490	DRM_IOCTL_DEF_DRV(NOUVEAU_GPUOBJ_FREE, nouveau_ioctl_gpuobj_free, DRM_UNLOCKED\|DRM_AUTH),
	491	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_NEW, nouveau_gem_ioctl_new, DRM_UNLOCKED\|DRM_AUTH),
	492	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_PUSHBUF, nouveau_gem_ioctl_pushbuf, DRM_UNLOCKED\|DRM_AUTH),
	493	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_PREP, nouveau_gem_ioctl_cpu_prep, DRM_UNLOCKED\|DRM_AUTH),
	494	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_FINI, nouveau_gem_ioctl_cpu_fini, DRM_UNLOCKED\|DRM_AUTH),
	495	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_INFO, nouveau_gem_ioctl_info, DRM_UNLOCKED\|DRM_AUTH),
6ee73861 BS	496	};
	497
	498	int nouveau_max_ioctl = DRM_ARRAY_SIZE(nouveau_ioctls);