[mirror_ubuntu-bionic-kernel.git] / drivers / gpu / drm / nouveau / nvkm / subdev / instmem / nv50.c

/*
 * Copyright 2012 Red Hat Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * Authors: Ben Skeggs
 */
#define nv50_instmem(p) container_of((p), struct nv50_instmem, base)
#include "priv.h"

#include <core/memory.h>
#include <subdev/bar.h>
#include <subdev/fb.h>
#include <subdev/mmu.h>

struct nv50_instmem {
	struct nvkm_instmem base;
	u64 addr;

	/* Mappings that can be evicted when BAR2 space has been exhausted. */
	struct list_head lru;
};

/******************************************************************************
 * instmem object implementation
 *****************************************************************************/
#define nv50_instobj(p) container_of((p), struct nv50_instobj, base.memory)

struct nv50_instobj {
	struct nvkm_instobj base;
	struct nv50_instmem *imem;
	struct nvkm_memory *ram;
	struct nvkm_vma bar;
	refcount_t maps;
	void *map;
	struct list_head lru;
};

static void
nv50_instobj_wr32_slow(struct nvkm_memory *memory, u64 offset, u32 data)
{
	struct nv50_instobj *iobj = nv50_instobj(memory);
	struct nv50_instmem *imem = iobj->imem;
	struct nvkm_device *device = imem->base.subdev.device;
	u64 base = (nvkm_memory_addr(iobj->ram) + offset) & 0xffffff00000ULL;
	u64 addr = (nvkm_memory_addr(iobj->ram) + offset) & 0x000000fffffULL;
	unsigned long flags;

	spin_lock_irqsave(&imem->base.lock, flags);
	if (unlikely(imem->addr != base)) {
		nvkm_wr32(device, 0x001700, base >> 16);
		imem->addr = base;
	}
	nvkm_wr32(device, 0x700000 + addr, data);
	spin_unlock_irqrestore(&imem->base.lock, flags);
}

static u32
nv50_instobj_rd32_slow(struct nvkm_memory *memory, u64 offset)
{
	struct nv50_instobj *iobj = nv50_instobj(memory);
	struct nv50_instmem *imem = iobj->imem;
	struct nvkm_device *device = imem->base.subdev.device;
	u64 base = (nvkm_memory_addr(iobj->ram) + offset) & 0xffffff00000ULL;
	u64 addr = (nvkm_memory_addr(iobj->ram) + offset) & 0x000000fffffULL;
	u32 data;
	unsigned long flags;

	spin_lock_irqsave(&imem->base.lock, flags);
	if (unlikely(imem->addr != base)) {
		nvkm_wr32(device, 0x001700, base >> 16);
		imem->addr = base;
	}
	data = nvkm_rd32(device, 0x700000 + addr);
	spin_unlock_irqrestore(&imem->base.lock, flags);
	return data;
}

static const struct nvkm_memory_ptrs
nv50_instobj_slow = {
	.rd32 = nv50_instobj_rd32_slow,
	.wr32 = nv50_instobj_wr32_slow,
};

static void
nv50_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data)
{
	iowrite32_native(data, nv50_instobj(memory)->map + offset);
}

static u32
nv50_instobj_rd32(struct nvkm_memory *memory, u64 offset)
{
	return ioread32_native(nv50_instobj(memory)->map + offset);
}

static const struct nvkm_memory_ptrs
nv50_instobj_fast = {
	.rd32 = nv50_instobj_rd32,
	.wr32 = nv50_instobj_wr32,
};

static void
nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm)
{
	struct nv50_instmem *imem = iobj->imem;
	struct nv50_instobj *eobj;
	struct nvkm_memory *memory = &iobj->base.memory;
	struct nvkm_subdev *subdev = &imem->base.subdev;
	struct nvkm_device *device = subdev->device;
	struct nvkm_vma bar = {}, ebar;
	u64 size = nvkm_memory_size(memory);
	void *emap;
	int ret;

	/* Attempt to allocate BAR2 address-space and map the object
	 * into it.  The lock has to be dropped while doing this due
	 * to the possibility of recursion for page table allocation.
	 */
	mutex_unlock(&subdev->mutex);
	while ((ret = nvkm_vm_get(vmm, size, 12, NV_MEM_ACCESS_RW, &bar))) {
		/* Evict unused mappings, and keep retrying until we either
		 * succeed,or there's no more objects left on the LRU.
		 */
		mutex_lock(&subdev->mutex);
		eobj = list_first_entry_or_null(&imem->lru, typeof(*eobj), lru);
		if (eobj) {
			nvkm_debug(subdev, "evict %016llx %016llx @ %016llx\n",
				   nvkm_memory_addr(&eobj->base.memory),
				   nvkm_memory_size(&eobj->base.memory),
				   eobj->bar.offset);
			list_del_init(&eobj->lru);
			ebar = eobj->bar;
			eobj->bar.node = NULL;
			emap = eobj->map;
			eobj->map = NULL;
		}
		mutex_unlock(&subdev->mutex);
		if (!eobj)
			break;
		iounmap(emap);
		nvkm_vm_put(&ebar);
	}

	if (ret == 0)
		ret = nvkm_memory_map(memory, 0, vmm, &bar, NULL, 0);
	mutex_lock(&subdev->mutex);
	if (ret || iobj->bar.node) {
		/* We either failed, or another thread beat us. */
		mutex_unlock(&subdev->mutex);
		nvkm_vm_put(&bar);
		mutex_lock(&subdev->mutex);
		return;
	}

	/* Make the mapping visible to the host. */
	iobj->bar = bar;
	iobj->map = ioremap_wc(device->func->resource_addr(device, 3) +
			       (u32)iobj->bar.offset, size);
	if (!iobj->map) {
		nvkm_warn(subdev, "PRAMIN ioremap failed\n");
		nvkm_vm_put(&iobj->bar);
	}
}

static int
nv50_instobj_map(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm,
		 struct nvkm_vma *vma, void *argv, u32 argc)
{
	memory = nv50_instobj(memory)->ram;
	return nvkm_memory_map(memory, offset, vmm, vma, argv, argc);
}

static void
nv50_instobj_release(struct nvkm_memory *memory)
{
	struct nv50_instobj *iobj = nv50_instobj(memory);
	struct nv50_instmem *imem = iobj->imem;
	struct nvkm_subdev *subdev = &imem->base.subdev;

	wmb();
	nvkm_bar_flush(subdev->device->bar);

	if (refcount_dec_and_mutex_lock(&iobj->maps, &subdev->mutex)) {
		/* Add the now-unused mapping to the LRU instead of directly
		 * unmapping it here, in case we need to map it again later.
		 */
		if (likely(iobj->lru.next) && iobj->map) {
			BUG_ON(!list_empty(&iobj->lru));
			list_add_tail(&iobj->lru, &imem->lru);
		}

		/* Switch back to NULL accessors when last map is gone. */
		iobj->base.memory.ptrs = NULL;
		mutex_unlock(&subdev->mutex);
	}
}

static void __iomem *
nv50_instobj_acquire(struct nvkm_memory *memory)
{
	struct nv50_instobj *iobj = nv50_instobj(memory);
	struct nvkm_instmem *imem = &iobj->imem->base;
	struct nvkm_vmm *vmm;
	void __iomem *map = NULL;

	/* Already mapped? */
	if (refcount_inc_not_zero(&iobj->maps))
		return iobj->map;

	/* Take the lock, and re-check that another thread hasn't
	 * already mapped the object in the meantime.
	 */
	mutex_lock(&imem->subdev.mutex);
	if (refcount_inc_not_zero(&iobj->maps)) {
		mutex_unlock(&imem->subdev.mutex);
		return iobj->map;
	}

	/* Attempt to get a direct CPU mapping of the object. */
	if ((vmm = nvkm_bar_bar2_vmm(imem->subdev.device))) {
		if (!iobj->map)
			nv50_instobj_kmap(iobj, vmm);
		map = iobj->map;
	}

	if (!refcount_inc_not_zero(&iobj->maps)) {
		/* Exclude object from eviction while it's being accessed. */
		if (likely(iobj->lru.next))
			list_del_init(&iobj->lru);

		if (map)
			iobj->base.memory.ptrs = &nv50_instobj_fast;
		else
			iobj->base.memory.ptrs = &nv50_instobj_slow;
		refcount_inc(&iobj->maps);
	}

	mutex_unlock(&imem->subdev.mutex);
	return map;
}

static void
nv50_instobj_boot(struct nvkm_memory *memory, struct nvkm_vmm *vmm)
{
	struct nv50_instobj *iobj = nv50_instobj(memory);
	struct nvkm_instmem *imem = &iobj->imem->base;

	/* Exclude bootstrapped objects (ie. the page tables for the
	 * instmem BAR itself) from eviction.
	 */
	mutex_lock(&imem->subdev.mutex);
	if (likely(iobj->lru.next)) {
		list_del_init(&iobj->lru);
		iobj->lru.next = NULL;
	}

	nv50_instobj_kmap(iobj, vmm);
	nvkm_instmem_boot(imem);
	mutex_unlock(&imem->subdev.mutex);
}

static u64
nv50_instobj_size(struct nvkm_memory *memory)
{
	return nvkm_memory_size(nv50_instobj(memory)->ram);
}

static u64
nv50_instobj_addr(struct nvkm_memory *memory)
{
	return nvkm_memory_addr(nv50_instobj(memory)->ram);
}

static enum nvkm_memory_target
nv50_instobj_target(struct nvkm_memory *memory)
{
	return nvkm_memory_target(nv50_instobj(memory)->ram);
}

static void *
nv50_instobj_dtor(struct nvkm_memory *memory)
{
	struct nv50_instobj *iobj = nv50_instobj(memory);
	struct nvkm_instmem *imem = &iobj->imem->base;
	struct nvkm_vma bar;
	void *map = map;

	mutex_lock(&imem->subdev.mutex);
	if (likely(iobj->lru.next))
		list_del(&iobj->lru);
	map = iobj->map;
	bar = iobj->bar;
	mutex_unlock(&imem->subdev.mutex);

	if (map) {
		iounmap(map);
		nvkm_vm_put(&bar);
	}

	nvkm_memory_unref(&iobj->ram);
	nvkm_instobj_dtor(imem, &iobj->base);
	return iobj;
}

static const struct nvkm_memory_func
nv50_instobj_func = {
	.dtor = nv50_instobj_dtor,
	.target = nv50_instobj_target,
	.size = nv50_instobj_size,
	.addr = nv50_instobj_addr,
	.boot = nv50_instobj_boot,
	.acquire = nv50_instobj_acquire,
	.release = nv50_instobj_release,
	.map = nv50_instobj_map,
};

static int
nv50_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero,
		 struct nvkm_memory **pmemory)
{
	struct nv50_instmem *imem = nv50_instmem(base);
	struct nv50_instobj *iobj;
	struct nvkm_device *device = imem->base.subdev.device;
	u8 page = max(order_base_2(align), 12);

	if (!(iobj = kzalloc(sizeof(*iobj), GFP_KERNEL)))
		return -ENOMEM;
	*pmemory = &iobj->base.memory;

	nvkm_instobj_ctor(&nv50_instobj_func, &imem->base, &iobj->base);
	iobj->imem = imem;
	refcount_set(&iobj->maps, 0);
	INIT_LIST_HEAD(&iobj->lru);

	return nvkm_ram_get(device, 0, 1, page, size, true, true, &iobj->ram);
}

/******************************************************************************
 * instmem subdev implementation
 *****************************************************************************/

static void
nv50_instmem_fini(struct nvkm_instmem *base)
{
	nv50_instmem(base)->addr = ~0ULL;
}

static const struct nvkm_instmem_func
nv50_instmem = {
	.fini = nv50_instmem_fini,
	.memory_new = nv50_instobj_new,
	.zero = false,
};

int
nv50_instmem_new(struct nvkm_device *device, int index,
		 struct nvkm_instmem **pimem)
{
	struct nv50_instmem *imem;

	if (!(imem = kzalloc(sizeof(*imem), GFP_KERNEL)))
		return -ENOMEM;
	nvkm_instmem_ctor(&nv50_instmem, device, index, &imem->base);
	INIT_LIST_HEAD(&imem->lru);
	*pimem = &imem->base;
	return 0;
}
Commit	Line	Data
6ee73861	1	/*
3863c9bc	2	* Copyright 2012 Red Hat Inc.
6ee73861	3	*
3863c9bc BS	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
6ee73861	10	*
3863c9bc BS	11	* The above copyright notice and this permission notice shall be included in
3863c9bc BS	12	* all copies or substantial portions of the Software.
6ee73861	13	*
3863c9bc BS	14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	20	* OTHER DEALINGS IN THE SOFTWARE.
6ee73861	21	*
3863c9bc	22	* Authors: Ben Skeggs
6ee73861	23	*/
d8e83994	24	#define nv50_instmem(p) container_of((p), struct nv50_instmem, base)
78b2b4e7	25	#include "priv.h"
6ee73861	26
d8e83994 BS	27	#include <core/memory.h>
d8e83994 BS	28	#include <subdev/bar.h>
3863c9bc	29	#include <subdev/fb.h>
d8e83994	30	#include <subdev/mmu.h>
24a4ae86	31
c44c06ae	32	struct nv50_instmem {
78b2b4e7	33	struct nvkm_instmem base;
3863c9bc	34	u64 addr;
03edf1b3 BS	35
	36	/* Mappings that can be evicted when BAR2 space has been exhausted. */
	37	struct list_head lru;
6ee73861 BS	38	};
6ee73861 BS	39
d8e83994 BS	40	/******************************************************************************
	41	* instmem object implementation
	42	*****************************************************************************/
be55287a	43	#define nv50_instobj(p) container_of((p), struct nv50_instobj, base.memory)
d8e83994	44
c44c06ae	45	struct nv50_instobj {
be55287a	46	struct nvkm_instobj base;
d8e83994	47	struct nv50_instmem *imem;
7f4f82af	48	struct nvkm_memory *ram;
d8e83994	49	struct nvkm_vma bar;
be55287a	50	refcount_t maps;
d8e83994	51	void *map;
03edf1b3	52	struct list_head lru;
3863c9bc	53	};
fbd2895e	54
07bbc1c5 BS	55	static void
07bbc1c5 BS	56	nv50_instobj_wr32_slow(struct nvkm_memory *memory, u64 offset, u32 data)
d8e83994	57	{
07bbc1c5 BS	58	struct nv50_instobj *iobj = nv50_instobj(memory);
	59	struct nv50_instmem *imem = iobj->imem;
	60	struct nvkm_device *device = imem->base.subdev.device;
7f4f82af BS	61	u64 base = (nvkm_memory_addr(iobj->ram) + offset) & 0xffffff00000ULL;
7f4f82af BS	62	u64 addr = (nvkm_memory_addr(iobj->ram) + offset) & 0x000000fffffULL;
af515ec8	63	unsigned long flags;
d8e83994	64
af515ec8	65	spin_lock_irqsave(&imem->base.lock, flags);
07bbc1c5 BS	66	if (unlikely(imem->addr != base)) {
	67	nvkm_wr32(device, 0x001700, base >> 16);
	68	imem->addr = base;
	69	}
	70	nvkm_wr32(device, 0x700000 + addr, data);
af515ec8	71	spin_unlock_irqrestore(&imem->base.lock, flags);
d8e83994 BS	72	}
d8e83994 BS	73
07bbc1c5 BS	74	static u32
07bbc1c5 BS	75	nv50_instobj_rd32_slow(struct nvkm_memory *memory, u64 offset)
d8e83994	76	{
07bbc1c5 BS	77	struct nv50_instobj *iobj = nv50_instobj(memory);
	78	struct nv50_instmem *imem = iobj->imem;
	79	struct nvkm_device *device = imem->base.subdev.device;
7f4f82af BS	80	u64 base = (nvkm_memory_addr(iobj->ram) + offset) & 0xffffff00000ULL;
7f4f82af BS	81	u64 addr = (nvkm_memory_addr(iobj->ram) + offset) & 0x000000fffffULL;
07bbc1c5	82	u32 data;
af515ec8	83	unsigned long flags;
07bbc1c5	84
af515ec8	85	spin_lock_irqsave(&imem->base.lock, flags);
07bbc1c5 BS	86	if (unlikely(imem->addr != base)) {
	87	nvkm_wr32(device, 0x001700, base >> 16);
	88	imem->addr = base;
	89	}
	90	data = nvkm_rd32(device, 0x700000 + addr);
af515ec8	91	spin_unlock_irqrestore(&imem->base.lock, flags);
07bbc1c5	92	return data;
d8e83994 BS	93	}
d8e83994 BS	94
07bbc1c5 BS	95	static const struct nvkm_memory_ptrs
	96	nv50_instobj_slow = {
	97	.rd32 = nv50_instobj_rd32_slow,
	98	.wr32 = nv50_instobj_wr32_slow,
	99	};
	100
be55287a BS	101	static void
	102	nv50_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data)
	103	{
	104	iowrite32_native(data, nv50_instobj(memory)->map + offset);
	105	}
	106
	107	static u32
	108	nv50_instobj_rd32(struct nvkm_memory *memory, u64 offset)
	109	{
	110	return ioread32_native(nv50_instobj(memory)->map + offset);
	111	}
	112
	113	static const struct nvkm_memory_ptrs
	114	nv50_instobj_fast = {
	115	.rd32 = nv50_instobj_rd32,
	116	.wr32 = nv50_instobj_wr32,
	117	};
	118
f584bde6 BS	119	static void
	120	nv50_instobj_kmap(struct nv50_instobj iobj, struct nvkm_vmm vmm)
	121	{
be55287a	122	struct nv50_instmem *imem = iobj->imem;
03edf1b3	123	struct nv50_instobj *eobj;
be55287a BS	124	struct nvkm_memory *memory = &iobj->base.memory;
be55287a BS	125	struct nvkm_subdev *subdev = &imem->base.subdev;
f584bde6	126	struct nvkm_device *device = subdev->device;
03edf1b3	127	struct nvkm_vma bar = {}, ebar;
f584bde6	128	u64 size = nvkm_memory_size(memory);
03edf1b3	129	void *emap;
f584bde6 BS	130	int ret;
f584bde6 BS	131
be55287a BS	132	/* Attempt to allocate BAR2 address-space and map the object
	133	* into it. The lock has to be dropped while doing this due
	134	* to the possibility of recursion for page table allocation.
	135	*/
	136	mutex_unlock(&subdev->mutex);
03edf1b3 BS	137	while ((ret = nvkm_vm_get(vmm, size, 12, NV_MEM_ACCESS_RW, &bar))) {
	138	/* Evict unused mappings, and keep retrying until we either
	139	* succeed,or there's no more objects left on the LRU.
	140	*/
	141	mutex_lock(&subdev->mutex);
	142	eobj = list_first_entry_or_null(&imem->lru, typeof(*eobj), lru);
	143	if (eobj) {
	144	nvkm_debug(subdev, "evict %016llx %016llx @ %016llx\n",
	145	nvkm_memory_addr(&eobj->base.memory),
	146	nvkm_memory_size(&eobj->base.memory),
	147	eobj->bar.offset);
	148	list_del_init(&eobj->lru);
	149	ebar = eobj->bar;
	150	eobj->bar.node = NULL;
	151	emap = eobj->map;
	152	eobj->map = NULL;
	153	}
	154	mutex_unlock(&subdev->mutex);
	155	if (!eobj)
	156	break;
	157	iounmap(emap);
	158	nvkm_vm_put(&ebar);
	159	}
	160
be55287a	161	if (ret == 0)
19a82e49	162	ret = nvkm_memory_map(memory, 0, vmm, &bar, NULL, 0);
be55287a BS	163	mutex_lock(&subdev->mutex);
	164	if (ret \|\| iobj->bar.node) {
	165	/* We either failed, or another thread beat us. */
	166	mutex_unlock(&subdev->mutex);
	167	nvkm_vm_put(&bar);
	168	mutex_lock(&subdev->mutex);
	169	return;
	170	}
	171
	172	/* Make the mapping visible to the host. */
	173	iobj->bar = bar;
dfcbd550 BS	174	iobj->map = ioremap_wc(device->func->resource_addr(device, 3) +
dfcbd550 BS	175	(u32)iobj->bar.offset, size);
be55287a BS	176	if (!iobj->map) {
	177	nvkm_warn(subdev, "PRAMIN ioremap failed\n");
	178	nvkm_vm_put(&iobj->bar);
f584bde6 BS	179	}
	180	}
	181
19a82e49 BS	182	static int
	183	nv50_instobj_map(struct nvkm_memory memory, u64 offset, struct nvkm_vmm vmm,
	184	struct nvkm_vma vma, void argv, u32 argc)
d8e83994	185	{
7f4f82af BS	186	memory = nv50_instobj(memory)->ram;
7f4f82af BS	187	return nvkm_memory_map(memory, offset, vmm, vma, argv, argc);
d8e83994 BS	188	}
	189
	190	static void
	191	nv50_instobj_release(struct nvkm_memory *memory)
	192	{
be55287a BS	193	struct nv50_instobj *iobj = nv50_instobj(memory);
	194	struct nv50_instmem *imem = iobj->imem;
	195	struct nvkm_subdev *subdev = &imem->base.subdev;
	196
dfcbd550	197	wmb();
be55287a BS	198	nvkm_bar_flush(subdev->device->bar);
	199
	200	if (refcount_dec_and_mutex_lock(&iobj->maps, &subdev->mutex)) {
03edf1b3 BS	201	/* Add the now-unused mapping to the LRU instead of directly
	202	* unmapping it here, in case we need to map it again later.
	203	*/
	204	if (likely(iobj->lru.next) && iobj->map) {
	205	BUG_ON(!list_empty(&iobj->lru));
	206	list_add_tail(&iobj->lru, &imem->lru);
	207	}
	208
be55287a	209	/* Switch back to NULL accessors when last map is gone. */
ffd937bb	210	iobj->base.memory.ptrs = NULL;
be55287a BS	211	mutex_unlock(&subdev->mutex);
be55287a BS	212	}
d8e83994 BS	213	}
	214
	215	static void __iomem *
	216	nv50_instobj_acquire(struct nvkm_memory *memory)
	217	{
	218	struct nv50_instobj *iobj = nv50_instobj(memory);
be55287a BS	219	struct nvkm_instmem *imem = &iobj->imem->base;
	220	struct nvkm_vmm *vmm;
	221	void __iomem *map = NULL;
d8e83994	222
be55287a BS	223	/* Already mapped? */
be55287a BS	224	if (refcount_inc_not_zero(&iobj->maps))
d8e83994 BS	225	return iobj->map;
d8e83994 BS	226
be55287a BS	227	/* Take the lock, and re-check that another thread hasn't
	228	* already mapped the object in the meantime.
	229	*/
	230	mutex_lock(&imem->subdev.mutex);
	231	if (refcount_inc_not_zero(&iobj->maps)) {
	232	mutex_unlock(&imem->subdev.mutex);
	233	return iobj->map;
	234	}
	235
	236	/* Attempt to get a direct CPU mapping of the object. */
69b136f2 BS	237	if ((vmm = nvkm_bar_bar2_vmm(imem->subdev.device))) {
	238	if (!iobj->map)
	239	nv50_instobj_kmap(iobj, vmm);
	240	map = iobj->map;
	241	}
be55287a BS	242
be55287a BS	243	if (!refcount_inc_not_zero(&iobj->maps)) {
03edf1b3 BS	244	/* Exclude object from eviction while it's being accessed. */
	245	if (likely(iobj->lru.next))
	246	list_del_init(&iobj->lru);
	247
be55287a BS	248	if (map)
	249	iobj->base.memory.ptrs = &nv50_instobj_fast;
	250	else
	251	iobj->base.memory.ptrs = &nv50_instobj_slow;
	252	refcount_inc(&iobj->maps);
	253	}
	254
	255	mutex_unlock(&imem->subdev.mutex);
	256	return map;
d8e83994	257	}
6ee73861	258
07bbc1c5	259	static void
f584bde6	260	nv50_instobj_boot(struct nvkm_memory memory, struct nvkm_vmm vmm)
6ee73861	261	{
d8e83994	262	struct nv50_instobj *iobj = nv50_instobj(memory);
be55287a BS	263	struct nvkm_instmem *imem = &iobj->imem->base;
be55287a BS	264
03edf1b3 BS	265	/* Exclude bootstrapped objects (ie. the page tables for the
	266	* instmem BAR itself) from eviction.
	267	*/
be55287a	268	mutex_lock(&imem->subdev.mutex);
03edf1b3 BS	269	if (likely(iobj->lru.next)) {
	270	list_del_init(&iobj->lru);
	271	iobj->lru.next = NULL;
	272	}
	273
f584bde6	274	nv50_instobj_kmap(iobj, vmm);
b00b8430	275	nvkm_instmem_boot(imem);
be55287a	276	mutex_unlock(&imem->subdev.mutex);
6ee73861 BS	277	}
6ee73861 BS	278
07bbc1c5 BS	279	static u64
07bbc1c5 BS	280	nv50_instobj_size(struct nvkm_memory *memory)
6ee73861	281	{
7f4f82af	282	return nvkm_memory_size(nv50_instobj(memory)->ram);
07bbc1c5	283	}
dc1e5c0d	284
07bbc1c5 BS	285	static u64
	286	nv50_instobj_addr(struct nvkm_memory *memory)
	287	{
7f4f82af	288	return nvkm_memory_addr(nv50_instobj(memory)->ram);
6ee73861 BS	289	}
6ee73861 BS	290
07bbc1c5 BS	291	static enum nvkm_memory_target
07bbc1c5 BS	292	nv50_instobj_target(struct nvkm_memory *memory)
ab606194	293	{
7f4f82af	294	return nvkm_memory_target(nv50_instobj(memory)->ram);
ab606194 BS	295	}
ab606194 BS	296
d8e83994 BS	297	static void *
	298	nv50_instobj_dtor(struct nvkm_memory *memory)
	299	{
	300	struct nv50_instobj *iobj = nv50_instobj(memory);
be55287a	301	struct nvkm_instmem *imem = &iobj->imem->base;
03edf1b3 BS	302	struct nvkm_vma bar;
	303	void *map = map;
	304
	305	mutex_lock(&imem->subdev.mutex);
	306	if (likely(iobj->lru.next))
	307	list_del(&iobj->lru);
	308	map = iobj->map;
	309	bar = iobj->bar;
	310	mutex_unlock(&imem->subdev.mutex);
	311
	312	if (map) {
	313	iounmap(map);
	314	nvkm_vm_put(&bar);
d8e83994	315	}
03edf1b3	316
7f4f82af	317	nvkm_memory_unref(&iobj->ram);
be55287a	318	nvkm_instobj_dtor(imem, &iobj->base);
d8e83994 BS	319	return iobj;
	320	}
	321
	322	static const struct nvkm_memory_func
	323	nv50_instobj_func = {
	324	.dtor = nv50_instobj_dtor,
	325	.target = nv50_instobj_target,
	326	.size = nv50_instobj_size,
	327	.addr = nv50_instobj_addr,
	328	.boot = nv50_instobj_boot,
	329	.acquire = nv50_instobj_acquire,
	330	.release = nv50_instobj_release,
d8e83994 BS	331	.map = nv50_instobj_map,
	332	};
	333
ab606194	334	static int
d8e83994 BS	335	nv50_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero,
d8e83994 BS	336	struct nvkm_memory **pmemory)
ab606194	337	{
d8e83994 BS	338	struct nv50_instmem *imem = nv50_instmem(base);
d8e83994 BS	339	struct nv50_instobj *iobj;
7f4f82af BS	340	struct nvkm_device *device = imem->base.subdev.device;
7f4f82af BS	341	u8 page = max(order_base_2(align), 12);
ab606194	342
d8e83994 BS	343	if (!(iobj = kzalloc(sizeof(*iobj), GFP_KERNEL)))
d8e83994 BS	344	return -ENOMEM;
be55287a	345	*pmemory = &iobj->base.memory;
ab606194	346
be55287a	347	nvkm_instobj_ctor(&nv50_instobj_func, &imem->base, &iobj->base);
d8e83994	348	iobj->imem = imem;
be55287a	349	refcount_set(&iobj->maps, 0);
03edf1b3	350	INIT_LIST_HEAD(&iobj->lru);
d8e83994	351
7f4f82af	352	return nvkm_ram_get(device, 0, 1, page, size, true, true, &iobj->ram);
ab606194 BS	353	}
ab606194 BS	354
24a4ae86 BS	355	/******************************************************************************
	356	* instmem subdev implementation
	357	*****************************************************************************/
	358
b7a2bc18 BS	359	static void
b7a2bc18 BS	360	nv50_instmem_fini(struct nvkm_instmem *base)
6ee73861	361	{
b7a2bc18	362	nv50_instmem(base)->addr = ~0ULL;
6ee73861 BS	363	}
6ee73861 BS	364
b7a2bc18 BS	365	static const struct nvkm_instmem_func
	366	nv50_instmem = {
	367	.fini = nv50_instmem_fini,
	368	.memory_new = nv50_instobj_new,
b7a2bc18 BS	369	.zero = false,
	370	};
	371
	372	int
	373	nv50_instmem_new(struct nvkm_device *device, int index,
	374	struct nvkm_instmem **pimem)
6ee73861	375	{
c44c06ae	376	struct nv50_instmem *imem;
6ee73861	377
b7a2bc18 BS	378	if (!(imem = kzalloc(sizeof(*imem), GFP_KERNEL)))
	379	return -ENOMEM;
	380	nvkm_instmem_ctor(&nv50_instmem, device, index, &imem->base);
03edf1b3	381	INIT_LIST_HEAD(&imem->lru);
b7a2bc18	382	*pimem = &imem->base;
6ee73861 BS	383	return 0;
6ee73861 BS	384	}