[mirror_ubuntu-bionic-kernel.git] / drivers / gpu / drm / i915 / i915_gem_tiling.c

/*
 * Copyright © 2008 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eric Anholt <eric@anholt.net>
 *
 */

#include <linux/string.h>
#include <linux/bitops.h>
#include <drm/drmP.h>
#include <drm/i915_drm.h>
#include "i915_drv.h"

/**
 * DOC: buffer object tiling
 *
 * i915_gem_set_tiling_ioctl() and i915_gem_get_tiling_ioctl() is the userspace
 * interface to declare fence register requirements.
 *
 * In principle GEM doesn't care at all about the internal data layout of an
 * object, and hence it also doesn't care about tiling or swizzling. There's two
 * exceptions:
 *
 * - For X and Y tiling the hardware provides detilers for CPU access, so called
 *   fences. Since there's only a limited amount of them the kernel must manage
 *   these, and therefore userspace must tell the kernel the object tiling if it
 *   wants to use fences for detiling.
 * - On gen3 and gen4 platforms have a swizzling pattern for tiled objects which
 *   depends upon the physical page frame number. When swapping such objects the
 *   page frame number might change and the kernel must be able to fix this up
 *   and hence now the tiling. Note that on a subset of platforms with
 *   asymmetric memory channel population the swizzling pattern changes in an
 *   unknown way, and for those the kernel simply forbids swapping completely.
 *
 * Since neither of this applies for new tiling layouts on modern platforms like
 * W, Ys and Yf tiling GEM only allows object tiling to be set to X or Y tiled.
 * Anything else can be handled in userspace entirely without the kernel's
 * invovlement.
 */

/**
 * i915_gem_fence_size - required global GTT size for a fence
 * @i915: i915 device
 * @size: object size
 * @tiling: tiling mode
 * @stride: tiling stride
 *
 * Return the required global GTT size for a fence (view of a tiled object),
 * taking into account potential fence register mapping.
 */
u32 i915_gem_fence_size(struct drm_i915_private *i915,
			u32 size, unsigned int tiling, unsigned int stride)
{
	u32 ggtt_size;

	GEM_BUG_ON(!size);

	if (tiling == I915_TILING_NONE)
		return size;

	GEM_BUG_ON(!stride);

	if (INTEL_GEN(i915) >= 4) {
		stride *= i915_gem_tile_height(tiling);
		GEM_BUG_ON(!IS_ALIGNED(stride, I965_FENCE_PAGE));
		return roundup(size, stride);
	}

	/* Previous chips need a power-of-two fence region when tiling */
	if (IS_GEN3(i915))
		ggtt_size = 1024*1024;
	else
		ggtt_size = 512*1024;

	while (ggtt_size < size)
		ggtt_size <<= 1;

	return ggtt_size;
}

/**
 * i915_gem_fence_alignment - required global GTT alignment for a fence
 * @i915: i915 device
 * @size: object size
 * @tiling: tiling mode
 * @stride: tiling stride
 *
 * Return the required global GTT alignment for a fence (a view of a tiled
 * object), taking into account potential fence register mapping.
 */
u32 i915_gem_fence_alignment(struct drm_i915_private *i915, u32 size,
			     unsigned int tiling, unsigned int stride)
{
	GEM_BUG_ON(!size);

	/*
	 * Minimum alignment is 4k (GTT page size), but might be greater
	 * if a fence register is needed for the object.
	 */
	if (tiling == I915_TILING_NONE)
		return I915_GTT_MIN_ALIGNMENT;

	if (INTEL_GEN(i915) >= 4)
		return I965_FENCE_PAGE;

	/*
	 * Previous chips need to be aligned to the size of the smallest
	 * fence register that can contain the object.
	 */
	return i915_gem_fence_size(i915, size, tiling, stride);
}

/* Check pitch constriants for all chips & tiling formats */
static bool
i915_tiling_ok(struct drm_i915_gem_object *obj,
	       unsigned int tiling, unsigned int stride)
{
	struct drm_i915_private *i915 = to_i915(obj->base.dev);
	unsigned int tile_width;

	/* Linear is always fine */
	if (tiling == I915_TILING_NONE)
		return true;

	if (tiling > I915_TILING_LAST)
		return false;

	/* check maximum stride & object size */
	/* i965+ stores the end address of the gtt mapping in the fence
	 * reg, so dont bother to check the size */
	if (INTEL_GEN(i915) >= 7) {
		if (stride / 128 > GEN7_FENCE_MAX_PITCH_VAL)
			return false;
	} else if (INTEL_GEN(i915) >= 4) {
		if (stride / 128 > I965_FENCE_MAX_PITCH_VAL)
			return false;
	} else {
		if (stride > 8192)
			return false;

		if (!is_power_of_2(stride))
			return false;

		if (IS_GEN3(i915)) {
			if (obj->base.size > I830_FENCE_MAX_SIZE_VAL << 20)
				return false;
		} else {
			if (obj->base.size > I830_FENCE_MAX_SIZE_VAL << 19)
				return false;
		}
	}

	if (IS_GEN2(i915) ||
	    (tiling == I915_TILING_Y && HAS_128_BYTE_Y_TILING(i915)))
		tile_width = 128;
	else
		tile_width = 512;

	if (!stride || !IS_ALIGNED(stride, tile_width))
		return false;

	return true;
}

static bool i915_vma_fence_prepare(struct i915_vma *vma,
				   int tiling_mode, unsigned int stride)
{
	struct drm_i915_private *i915 = vma->vm->i915;
	u32 size, alignment;

	if (!i915_vma_is_map_and_fenceable(vma))
		return true;

	size = i915_gem_fence_size(i915, vma->size, tiling_mode, stride);
	if (vma->node.size < size)
		return false;

	alignment = i915_gem_fence_alignment(i915, vma->size, tiling_mode, stride);
	if (!IS_ALIGNED(vma->node.start, alignment))
		return false;

	return true;
}

/* Make the current GTT allocation valid for the change in tiling. */
static int
i915_gem_object_fence_prepare(struct drm_i915_gem_object *obj,
			      int tiling_mode, unsigned int stride)
{
	struct i915_vma *vma;
	int ret;

	if (tiling_mode == I915_TILING_NONE)
		return 0;

	list_for_each_entry(vma, &obj->vma_list, obj_link) {
		if (!i915_vma_is_ggtt(vma))
			break;

		if (i915_vma_fence_prepare(vma, tiling_mode, stride))
			continue;

		ret = i915_vma_unbind(vma);
		if (ret)
			return ret;
	}

	return 0;
}

int
i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,
			   unsigned int tiling, unsigned int stride)
{
	struct drm_i915_private *i915 = to_i915(obj->base.dev);
	struct i915_vma *vma;
	int err;

	/* Make sure we don't cross-contaminate obj->tiling_and_stride */
	BUILD_BUG_ON(I915_TILING_LAST & STRIDE_MASK);

	GEM_BUG_ON(!i915_tiling_ok(obj, tiling, stride));
	GEM_BUG_ON(!stride ^ (tiling == I915_TILING_NONE));
	lockdep_assert_held(&i915->drm.struct_mutex);

	if ((tiling | stride) == obj->tiling_and_stride)
		return 0;

	if (obj->framebuffer_references)
		return -EBUSY;

	/* We need to rebind the object if its current allocation
	 * no longer meets the alignment restrictions for its new
	 * tiling mode. Otherwise we can just leave it alone, but
	 * need to ensure that any fence register is updated before
	 * the next fenced (either through the GTT or by the BLT unit
	 * on older GPUs) access.
	 *
	 * After updating the tiling parameters, we then flag whether
	 * we need to update an associated fence register. Note this
	 * has to also include the unfenced register the GPU uses
	 * whilst executing a fenced command for an untiled object.
	 */

	err = i915_gem_object_fence_prepare(obj, tiling, stride);
	if (err)
		return err;

	/* If the memory has unknown (i.e. varying) swizzling, we pin the
	 * pages to prevent them being swapped out and causing corruption
	 * due to the change in swizzling.
	 */
	mutex_lock(&obj->mm.lock);
	if (obj->mm.pages &&
	    obj->mm.madv == I915_MADV_WILLNEED &&
	    i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
		if (tiling == I915_TILING_NONE) {
			GEM_BUG_ON(!obj->mm.quirked);
			__i915_gem_object_unpin_pages(obj);
			obj->mm.quirked = false;
		}
		if (!i915_gem_object_is_tiled(obj)) {
			GEM_BUG_ON(!obj->mm.quirked);
			__i915_gem_object_pin_pages(obj);
			obj->mm.quirked = true;
		}
	}
	mutex_unlock(&obj->mm.lock);

	list_for_each_entry(vma, &obj->vma_list, obj_link) {
		if (!i915_vma_is_ggtt(vma))
			break;

		vma->fence_size =
			i915_gem_fence_size(i915, vma->size, tiling, stride);
		vma->fence_alignment =
			i915_gem_fence_alignment(i915,
						 vma->size, tiling, stride);

		if (vma->fence)
			vma->fence->dirty = true;
	}

	obj->tiling_and_stride = tiling | stride;

	/* Force the fence to be reacquired for GTT access */
	i915_gem_release_mmap(obj);

	/* Try to preallocate memory required to save swizzling on put-pages */
	if (i915_gem_object_needs_bit17_swizzle(obj)) {
		if (!obj->bit_17) {
			obj->bit_17 = kcalloc(BITS_TO_LONGS(obj->base.size >> PAGE_SHIFT),
					      sizeof(long), GFP_KERNEL);
		}
	} else {
		kfree(obj->bit_17);
		obj->bit_17 = NULL;
	}

	return 0;
}

/**
 * i915_gem_set_tiling_ioctl - IOCTL handler to set tiling mode
 * @dev: DRM device
 * @data: data pointer for the ioctl
 * @file: DRM file for the ioctl call
 *
 * Sets the tiling mode of an object, returning the required swizzling of
 * bit 6 of addresses in the object.
 *
 * Called by the user via ioctl.
 *
 * Returns:
 * Zero on success, negative errno on failure.
 */
int
i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data,
			  struct drm_file *file)
{
	struct drm_i915_gem_set_tiling *args = data;
	struct drm_i915_gem_object *obj;
	int err;

	obj = i915_gem_object_lookup(file, args->handle);
	if (!obj)
		return -ENOENT;

	if (!i915_tiling_ok(obj, args->tiling_mode, args->stride)) {
		err = -EINVAL;
		goto err;
	}

	if (args->tiling_mode == I915_TILING_NONE) {
		args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
		args->stride = 0;
	} else {
		if (args->tiling_mode == I915_TILING_X)
			args->swizzle_mode = to_i915(dev)->mm.bit_6_swizzle_x;
		else
			args->swizzle_mode = to_i915(dev)->mm.bit_6_swizzle_y;

		/* Hide bit 17 swizzling from the user.  This prevents old Mesa
		 * from aborting the application on sw fallbacks to bit 17,
		 * and we use the pread/pwrite bit17 paths to swizzle for it.
		 * If there was a user that was relying on the swizzle
		 * information for drm_intel_bo_map()ed reads/writes this would
		 * break it, but we don't have any of those.
		 */
		if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17)
			args->swizzle_mode = I915_BIT_6_SWIZZLE_9;
		if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
			args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;

		/* If we can't handle the swizzling, make it untiled. */
		if (args->swizzle_mode == I915_BIT_6_SWIZZLE_UNKNOWN) {
			args->tiling_mode = I915_TILING_NONE;
			args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
			args->stride = 0;
		}
	}

	err = mutex_lock_interruptible(&dev->struct_mutex);
	if (err)
		goto err;

	err = i915_gem_object_set_tiling(obj, args->tiling_mode, args->stride);
	mutex_unlock(&dev->struct_mutex);

	/* We have to maintain this existing ABI... */
	args->stride = i915_gem_object_get_stride(obj);
	args->tiling_mode = i915_gem_object_get_tiling(obj);

err:
	i915_gem_object_put(obj);
	return err;
}

/**
 * i915_gem_get_tiling_ioctl - IOCTL handler to get tiling mode
 * @dev: DRM device
 * @data: data pointer for the ioctl
 * @file: DRM file for the ioctl call
 *
 * Returns the current tiling mode and required bit 6 swizzling for the object.
 *
 * Called by the user via ioctl.
 *
 * Returns:
 * Zero on success, negative errno on failure.
 */
int
i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data,
			  struct drm_file *file)
{
	struct drm_i915_gem_get_tiling *args = data;
	struct drm_i915_private *dev_priv = to_i915(dev);
	struct drm_i915_gem_object *obj;
	int err = -ENOENT;

	rcu_read_lock();
	obj = i915_gem_object_lookup_rcu(file, args->handle);
	if (obj) {
		args->tiling_mode =
			READ_ONCE(obj->tiling_and_stride) & TILING_MASK;
		err = 0;
	}
	rcu_read_unlock();
	if (unlikely(err))
		return err;

	switch (args->tiling_mode) {
	case I915_TILING_X:
		args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x;
		break;
	case I915_TILING_Y:
		args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y;
		break;
	default:
	case I915_TILING_NONE:
		args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
		break;
	}

	/* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */
	if (dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
		args->phys_swizzle_mode = I915_BIT_6_SWIZZLE_UNKNOWN;
	else
		args->phys_swizzle_mode = args->swizzle_mode;
	if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17)
		args->swizzle_mode = I915_BIT_6_SWIZZLE_9;
	if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
		args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;

	return 0;
}
Commit	Line	Data
673a394b EA	1	/*
	2	* Copyright © 2008 Intel Corporation
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice (including the next
	12	* paragraph) shall be included in all copies or substantial portions of the
	13	* Software.
	14	*
	15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
	20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
	21	* IN THE SOFTWARE.
	22	*
	23	* Authors:
	24	* Eric Anholt <eric@anholt.net>
	25	*
	26	*/
	27
760285e7 DH	28	#include <linux/string.h>
	29	#include <linux/bitops.h>
	30	#include <drm/drmP.h>
	31	#include <drm/i915_drm.h>
673a394b EA	32	#include "i915_drv.h"
673a394b EA	33
3271dca4 DV	34	/**
3271dca4 DV	35	* DOC: buffer object tiling
673a394b	36	*
111dbcab CW	37	* i915_gem_set_tiling_ioctl() and i915_gem_get_tiling_ioctl() is the userspace
111dbcab CW	38	* interface to declare fence register requirements.
673a394b	39	*
3271dca4 DV	40	* In principle GEM doesn't care at all about the internal data layout of an
	41	* object, and hence it also doesn't care about tiling or swizzling. There's two
	42	* exceptions:
673a394b	43	*
3271dca4 DV	44	* - For X and Y tiling the hardware provides detilers for CPU access, so called
	45	* fences. Since there's only a limited amount of them the kernel must manage
	46	* these, and therefore userspace must tell the kernel the object tiling if it
	47	* wants to use fences for detiling.
	48	* - On gen3 and gen4 platforms have a swizzling pattern for tiled objects which
	49	* depends upon the physical page frame number. When swapping such objects the
	50	* page frame number might change and the kernel must be able to fix this up
	51	* and hence now the tiling. Note that on a subset of platforms with
	52	* asymmetric memory channel population the swizzling pattern changes in an
	53	* unknown way, and for those the kernel simply forbids swapping completely.
673a394b	54	*
3271dca4 DV	55	* Since neither of this applies for new tiling layouts on modern platforms like
	56	* W, Ys and Yf tiling GEM only allows object tiling to be set to X or Y tiled.
	57	* Anything else can be handled in userspace entirely without the kernel's
	58	* invovlement.
673a394b EA	59	*/
673a394b EA	60
91d4e0aa CW	61	/**
	62	* i915_gem_fence_size - required global GTT size for a fence
	63	* @i915: i915 device
	64	* @size: object size
	65	* @tiling: tiling mode
	66	* @stride: tiling stride
	67	*
	68	* Return the required global GTT size for a fence (view of a tiled object),
	69	* taking into account potential fence register mapping.
	70	*/
	71	u32 i915_gem_fence_size(struct drm_i915_private *i915,
	72	u32 size, unsigned int tiling, unsigned int stride)
	73	{
	74	u32 ggtt_size;
	75
	76	GEM_BUG_ON(!size);
	77
	78	if (tiling == I915_TILING_NONE)
	79	return size;
	80
	81	GEM_BUG_ON(!stride);
	82
	83	if (INTEL_GEN(i915) >= 4) {
	84	stride *= i915_gem_tile_height(tiling);
f51455d4	85	GEM_BUG_ON(!IS_ALIGNED(stride, I965_FENCE_PAGE));
91d4e0aa CW	86	return roundup(size, stride);
	87	}
	88
	89	/* Previous chips need a power-of-two fence region when tiling */
	90	if (IS_GEN3(i915))
	91	ggtt_size = 1024*1024;
	92	else
	93	ggtt_size = 512*1024;
	94
	95	while (ggtt_size < size)
	96	ggtt_size <<= 1;
	97
	98	return ggtt_size;
	99	}
	100
	101	/**
	102	* i915_gem_fence_alignment - required global GTT alignment for a fence
	103	* @i915: i915 device
	104	* @size: object size
	105	* @tiling: tiling mode
	106	* @stride: tiling stride
	107	*
	108	* Return the required global GTT alignment for a fence (a view of a tiled
	109	* object), taking into account potential fence register mapping.
	110	*/
	111	u32 i915_gem_fence_alignment(struct drm_i915_private *i915, u32 size,
	112	unsigned int tiling, unsigned int stride)
	113	{
	114	GEM_BUG_ON(!size);
	115
	116	/*
	117	* Minimum alignment is 4k (GTT page size), but might be greater
	118	* if a fence register is needed for the object.
	119	*/
f51455d4 CW	120	if (tiling == I915_TILING_NONE)
	121	return I915_GTT_MIN_ALIGNMENT;
	122
	123	if (INTEL_GEN(i915) >= 4)
	124	return I965_FENCE_PAGE;
91d4e0aa CW	125
	126	/*
	127	* Previous chips need to be aligned to the size of the smallest
	128	* fence register that can contain the object.
	129	*/
	130	return i915_gem_fence_size(i915, size, tiling, stride);
	131	}
	132
0f973f27	133	/* Check pitch constriants for all chips & tiling formats */
a00b10c3	134	static bool
957870f9 CW	135	i915_tiling_ok(struct drm_i915_gem_object *obj,
957870f9 CW	136	unsigned int tiling, unsigned int stride)
0f973f27	137	{
957870f9 CW	138	struct drm_i915_private *i915 = to_i915(obj->base.dev);
957870f9 CW	139	unsigned int tile_width;
0f973f27 JB	140
0f973f27 JB	141	/* Linear is always fine */
957870f9	142	if (tiling == I915_TILING_NONE)
0f973f27 JB	143	return true;
0f973f27 JB	144
957870f9	145	if (tiling > I915_TILING_LAST)
deeb1519 CW	146	return false;
deeb1519 CW	147
8d7773a3	148	/* check maximum stride & object size */
3a062478 VS	149	/* i965+ stores the end address of the gtt mapping in the fence
3a062478 VS	150	* reg, so dont bother to check the size */
957870f9	151	if (INTEL_GEN(i915) >= 7) {
3a062478 VS	152	if (stride / 128 > GEN7_FENCE_MAX_PITCH_VAL)
3a062478 VS	153	return false;
957870f9	154	} else if (INTEL_GEN(i915) >= 4) {
8d7773a3 DV	155	if (stride / 128 > I965_FENCE_MAX_PITCH_VAL)
8d7773a3 DV	156	return false;
a6c45cf0	157	} else {
c36a2a6d	158	if (stride > 8192)
8d7773a3	159	return false;
e76a16de	160
a3a1e533 CW	161	if (!is_power_of_2(stride))
	162	return false;
	163
957870f9 CW	164	if (IS_GEN3(i915)) {
957870f9 CW	165	if (obj->base.size > I830_FENCE_MAX_SIZE_VAL << 20)
c36a2a6d DV	166	return false;
c36a2a6d DV	167	} else {
957870f9	168	if (obj->base.size > I830_FENCE_MAX_SIZE_VAL << 19)
c36a2a6d DV	169	return false;
c36a2a6d DV	170	}
8d7773a3 DV	171	}
8d7773a3 DV	172
957870f9 CW	173	if (IS_GEN2(i915) \|\|
	174	(tiling == I915_TILING_Y && HAS_128_BYTE_Y_TILING(i915)))
	175	tile_width = 128;
	176	else
	177	tile_width = 512;
	178
52da22e7	179	if (!stride \|\| !IS_ALIGNED(stride, tile_width))
fe48d8de VS	180	return false;
fe48d8de VS	181
a3a1e533	182	return true;
0f973f27 JB	183	}
0f973f27 JB	184
5b30694b CW	185	static bool i915_vma_fence_prepare(struct i915_vma *vma,
5b30694b CW	186	int tiling_mode, unsigned int stride)
49ef5294	187	{
944397f0 CW	188	struct drm_i915_private *i915 = vma->vm->i915;
944397f0 CW	189	u32 size, alignment;
49ef5294 CW	190
	191	if (!i915_vma_is_map_and_fenceable(vma))
	192	return true;
	193
91d4e0aa	194	size = i915_gem_fence_size(i915, vma->size, tiling_mode, stride);
49ef5294 CW	195	if (vma->node.size < size)
	196	return false;
	197
91d4e0aa	198	alignment = i915_gem_fence_alignment(i915, vma->size, tiling_mode, stride);
f51455d4	199	if (!IS_ALIGNED(vma->node.start, alignment))
49ef5294 CW	200	return false;
	201
	202	return true;
	203	}
	204
f23eda8c CW	205	/* Make the current GTT allocation valid for the change in tiling. */
f23eda8c CW	206	static int
5b30694b CW	207	i915_gem_object_fence_prepare(struct drm_i915_gem_object *obj,
5b30694b CW	208	int tiling_mode, unsigned int stride)
52dc7d32	209	{
f23eda8c	210	struct i915_vma *vma;
49ef5294	211	int ret;
52dc7d32 CW	212
52dc7d32 CW	213	if (tiling_mode == I915_TILING_NONE)
f23eda8c	214	return 0;
52dc7d32	215
49ef5294	216	list_for_each_entry(vma, &obj->vma_list, obj_link) {
944397f0 CW	217	if (!i915_vma_is_ggtt(vma))
	218	break;
	219
5b30694b	220	if (i915_vma_fence_prepare(vma, tiling_mode, stride))
49ef5294	221	continue;
a6c45cf0	222
49ef5294 CW	223	ret = i915_vma_unbind(vma);
	224	if (ret)
	225	return ret;
df153158 CW	226	}
df153158 CW	227
f23eda8c	228	return 0;
52dc7d32 CW	229	}
52dc7d32 CW	230
957870f9 CW	231	int
	232	i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,
	233	unsigned int tiling, unsigned int stride)
	234	{
	235	struct drm_i915_private *i915 = to_i915(obj->base.dev);
	236	struct i915_vma *vma;
	237	int err;
	238
	239	/* Make sure we don't cross-contaminate obj->tiling_and_stride */
	240	BUILD_BUG_ON(I915_TILING_LAST & STRIDE_MASK);
	241
	242	GEM_BUG_ON(!i915_tiling_ok(obj, tiling, stride));
	243	GEM_BUG_ON(!stride ^ (tiling == I915_TILING_NONE));
	244	lockdep_assert_held(&i915->drm.struct_mutex);
	245
	246	if ((tiling \| stride) == obj->tiling_and_stride)
	247	return 0;
	248
	249	if (obj->framebuffer_references)
	250	return -EBUSY;
	251
	252	/* We need to rebind the object if its current allocation
	253	* no longer meets the alignment restrictions for its new
	254	* tiling mode. Otherwise we can just leave it alone, but
	255	* need to ensure that any fence register is updated before
	256	* the next fenced (either through the GTT or by the BLT unit
	257	* on older GPUs) access.
	258	*
	259	* After updating the tiling parameters, we then flag whether
	260	* we need to update an associated fence register. Note this
	261	* has to also include the unfenced register the GPU uses
	262	* whilst executing a fenced command for an untiled object.
	263	*/
	264
	265	err = i915_gem_object_fence_prepare(obj, tiling, stride);
	266	if (err)
	267	return err;
	268
	269	/* If the memory has unknown (i.e. varying) swizzling, we pin the
	270	* pages to prevent them being swapped out and causing corruption
	271	* due to the change in swizzling.
	272	*/
	273	mutex_lock(&obj->mm.lock);
	274	if (obj->mm.pages &&
	275	obj->mm.madv == I915_MADV_WILLNEED &&
	276	i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
	277	if (tiling == I915_TILING_NONE) {
	278	GEM_BUG_ON(!obj->mm.quirked);
	279	__i915_gem_object_unpin_pages(obj);
	280	obj->mm.quirked = false;
	281	}
	282	if (!i915_gem_object_is_tiled(obj)) {
	283	GEM_BUG_ON(!obj->mm.quirked);
	284	__i915_gem_object_pin_pages(obj);
	285	obj->mm.quirked = true;
	286	}
	287	}
	288	mutex_unlock(&obj->mm.lock);
	289
	290	list_for_each_entry(vma, &obj->vma_list, obj_link) {
	291	if (!i915_vma_is_ggtt(vma))
	292	break;
	293
	294	vma->fence_size =
295	i915_gem_fence_size(i915, vma->size, tiling, stride);
296	vma->fence_alignment =
297	i915_gem_fence_alignment(i915,
298	vma->size, tiling, stride);
299
300	if (vma->fence)
301	vma->fence->dirty = true;
302	}
303
304	obj->tiling_and_stride = tiling \| stride;
305
306	/* Force the fence to be reacquired for GTT access */
307	i915_gem_release_mmap(obj);
308
309	/* Try to preallocate memory required to save swizzling on put-pages */
310	if (i915_gem_object_needs_bit17_swizzle(obj)) {
311	if (!obj->bit_17) {
312	obj->bit_17 = kcalloc(BITS_TO_LONGS(obj->base.size >> PAGE_SHIFT),
313	sizeof(long), GFP_KERNEL);
314	}
315	} else {
316	kfree(obj->bit_17);
317	obj->bit_17 = NULL;
318	}
319
320	return 0;
321	}
322
673a394b	323	/**
111dbcab	324	* i915_gem_set_tiling_ioctl - IOCTL handler to set tiling mode
3271dca4 DV	325	* @dev: DRM device
	326	* @data: data pointer for the ioctl
	327	* @file: DRM file for the ioctl call
	328	*
673a394b EA	329	* Sets the tiling mode of an object, returning the required swizzling of
673a394b EA	330	* bit 6 of addresses in the object.
3271dca4 DV	331	*
	332	* Called by the user via ioctl.
	333	*
	334	* Returns:
	335	* Zero on success, negative errno on failure.
673a394b EA	336	*/
673a394b EA	337	int
111dbcab CW	338	i915_gem_set_tiling_ioctl(struct drm_device dev, void data,
111dbcab CW	339	struct drm_file *file)
673a394b EA	340	{
673a394b EA	341	struct drm_i915_gem_set_tiling *args = data;
05394f39	342	struct drm_i915_gem_object *obj;
957870f9	343	int err;
3e510a8e	344
03ac0642 CW	345	obj = i915_gem_object_lookup(file, args->handle);
03ac0642 CW	346	if (!obj)
bf79cb91	347	return -ENOENT;
673a394b	348
957870f9 CW	349	if (!i915_tiling_ok(obj, args->tiling_mode, args->stride)) {
957870f9 CW	350	err = -EINVAL;
6c31a614	351	goto err;
31770bd4 DV	352	}
31770bd4 DV	353
673a394b	354	if (args->tiling_mode == I915_TILING_NONE) {
673a394b	355	args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
52dc7d32	356	args->stride = 0;
673a394b EA	357	} else {
673a394b EA	358	if (args->tiling_mode == I915_TILING_X)
957870f9	359	args->swizzle_mode = to_i915(dev)->mm.bit_6_swizzle_x;
673a394b	360	else
957870f9	361	args->swizzle_mode = to_i915(dev)->mm.bit_6_swizzle_y;
280b713b EA	362
	363	/* Hide bit 17 swizzling from the user. This prevents old Mesa
	364	* from aborting the application on sw fallbacks to bit 17,
	365	* and we use the pread/pwrite bit17 paths to swizzle for it.
	366	* If there was a user that was relying on the swizzle
	367	* information for drm_intel_bo_map()ed reads/writes this would
	368	* break it, but we don't have any of those.
	369	*/
	370	if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17)
	371	args->swizzle_mode = I915_BIT_6_SWIZZLE_9;
	372	if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
	373	args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;
	374
673a394b EA	375	/* If we can't handle the swizzling, make it untiled. */
	376	if (args->swizzle_mode == I915_BIT_6_SWIZZLE_UNKNOWN) {
	377	args->tiling_mode = I915_TILING_NONE;
	378	args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
52dc7d32	379	args->stride = 0;
673a394b EA	380	}
673a394b EA	381	}
0f973f27	382
957870f9 CW	383	err = mutex_lock_interruptible(&dev->struct_mutex);
	384	if (err)
	385	goto err;
467cffba	386
957870f9 CW	387	err = i915_gem_object_set_tiling(obj, args->tiling_mode, args->stride);
	388	mutex_unlock(&dev->struct_mutex);
	389
	390	/* We have to maintain this existing ABI... */
3e510a8e CW	391	args->stride = i915_gem_object_get_stride(obj);
3e510a8e CW	392	args->tiling_mode = i915_gem_object_get_tiling(obj);
e9b73c67	393
6c31a614	394	err:
f8c417cd	395	i915_gem_object_put(obj);
f23eda8c	396	return err;
673a394b EA	397	}
	398
	399	/**
111dbcab	400	* i915_gem_get_tiling_ioctl - IOCTL handler to get tiling mode
3271dca4 DV	401	* @dev: DRM device
	402	* @data: data pointer for the ioctl
	403	* @file: DRM file for the ioctl call
	404	*
673a394b	405	* Returns the current tiling mode and required bit 6 swizzling for the object.
3271dca4 DV	406	*
	407	* Called by the user via ioctl.
	408	*
	409	* Returns:
	410	* Zero on success, negative errno on failure.
673a394b EA	411	*/
673a394b EA	412	int
111dbcab CW	413	i915_gem_get_tiling_ioctl(struct drm_device dev, void data,
111dbcab CW	414	struct drm_file *file)
673a394b EA	415	{
673a394b EA	416	struct drm_i915_gem_get_tiling *args = data;
fac5e23e	417	struct drm_i915_private *dev_priv = to_i915(dev);
05394f39	418	struct drm_i915_gem_object *obj;
fbbd37b3 CW	419	int err = -ENOENT;
	420
	421	rcu_read_lock();
	422	obj = i915_gem_object_lookup_rcu(file, args->handle);
	423	if (obj) {
	424	args->tiling_mode =
	425	READ_ONCE(obj->tiling_and_stride) & TILING_MASK;
	426	err = 0;
	427	}
	428	rcu_read_unlock();
	429	if (unlikely(err))
	430	return err;
673a394b	431
9ad36761	432	switch (args->tiling_mode) {
673a394b EA	433	case I915_TILING_X:
	434	args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x;
	435	break;
	436	case I915_TILING_Y:
	437	args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y;
	438	break;
fbbd37b3	439	default:
673a394b EA	440	case I915_TILING_NONE:
	441	args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
	442	break;
673a394b EA	443	}
673a394b EA	444
280b713b	445	/* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */
5eb3e5a5 CW	446	if (dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
	447	args->phys_swizzle_mode = I915_BIT_6_SWIZZLE_UNKNOWN;
	448	else
	449	args->phys_swizzle_mode = args->swizzle_mode;
280b713b EA	450	if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17)
	451	args->swizzle_mode = I915_BIT_6_SWIZZLE_9;
	452	if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
	453	args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;
	454
673a394b EA	455	return 0;
673a394b EA	456	}