]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blame - drivers/gpu/drm/i915/i915_gem.c
projects / mirror_ubuntu-hirsute-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: Record fence registers on error.
[mirror_ubuntu-hirsute-kernel.git] / drivers / gpu / drm / i915 / i915_gem.c
CommitLineData
673a394b
EA		 1/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
1c5d22f7 32#include "i915_trace.h"
652c393a 33#include "intel_drv.h"
5a0e3ad6 34#include <linux/slab.h>
673a394b 35#include <linux/swap.h>
79e53945 36#include <linux/pci.h>
673a394b 37
0f8c6d7c
CW		 38struct change_domains {
39 uint32_t invalidate_domains;
40 uint32_t flush_domains;
41 uint32_t flush_rings;
42};
43
05394f39 44static int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj,
ba3d8d74 45 bool pipelined);
05394f39
CW		 46static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
47static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
48static int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
e47c68e9 49 int write);
05394f39 50static int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
e47c68e9
EA		 51 uint64_t offset,
52 uint64_t size);
05394f39
CW		 53static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj);
54static int i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
2cf34d7b 55 bool interruptible);
05394f39 56static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
a00b10c3 57 unsigned alignment,
75e9e915 58 bool map_and_fenceable);
05394f39
CW		 59static void i915_gem_clear_fence_reg(struct drm_i915_gem_object *obj);
60static int i915_gem_phys_pwrite(struct drm_device *dev,
61 struct drm_i915_gem_object *obj,
71acb5eb 62 struct drm_i915_gem_pwrite *args,
05394f39
CW		 63 struct drm_file *file);
64static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
673a394b 65
17250b71
CW		 66static int i915_gem_inactive_shrink(struct shrinker *shrinker,
67 int nr_to_scan,
68 gfp_t gfp_mask);
69
31169714 70
73aa808f
CW		 71/* some bookkeeping */
72static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
73 size_t size)
74{
75 dev_priv->mm.object_count++;
76 dev_priv->mm.object_memory += size;
77}
78
79static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
80 size_t size)
81{
82 dev_priv->mm.object_count--;
83 dev_priv->mm.object_memory -= size;
84}
85
86static void i915_gem_info_add_gtt(struct drm_i915_private *dev_priv,
a00b10c3 87 struct drm_i915_gem_object *obj)
73aa808f
CW		 88{
89 dev_priv->mm.gtt_count++;
a00b10c3
CW		 90 dev_priv->mm.gtt_memory += obj->gtt_space->size;
91 if (obj->gtt_offset < dev_priv->mm.gtt_mappable_end) {
fb7d516a 92 dev_priv->mm.mappable_gtt_used +=
a00b10c3
CW		 93 min_t(size_t, obj->gtt_space->size,
94 dev_priv->mm.gtt_mappable_end - obj->gtt_offset);
fb7d516a 95 }
93a37f20 96 list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
73aa808f
CW		 97}
98
99static void i915_gem_info_remove_gtt(struct drm_i915_private *dev_priv,
a00b10c3 100 struct drm_i915_gem_object *obj)
73aa808f
CW		 101{
102 dev_priv->mm.gtt_count--;
a00b10c3
CW		 103 dev_priv->mm.gtt_memory -= obj->gtt_space->size;
104 if (obj->gtt_offset < dev_priv->mm.gtt_mappable_end) {
fb7d516a 105 dev_priv->mm.mappable_gtt_used -=
a00b10c3
CW		 106 min_t(size_t, obj->gtt_space->size,
107 dev_priv->mm.gtt_mappable_end - obj->gtt_offset);
fb7d516a 108 }
93a37f20 109 list_del_init(&obj->gtt_list);
fb7d516a
DV		 110}
111
112/**
113 * Update the mappable working set counters. Call _only_ when there is a change
114 * in one of (pin|fault)_mappable and update *_mappable _before_ calling.
115 * @mappable: new state the changed mappable flag (either pin_ or fault_).
116 */
117static void
118i915_gem_info_update_mappable(struct drm_i915_private *dev_priv,
a00b10c3 119 struct drm_i915_gem_object *obj,
fb7d516a
DV		 120 bool mappable)
121{
fb7d516a 122 if (mappable) {
a00b10c3 123 if (obj->pin_mappable && obj->fault_mappable)
fb7d516a
DV		 124 /* Combined state was already mappable. */
125 return;
126 dev_priv->mm.gtt_mappable_count++;
a00b10c3 127 dev_priv->mm.gtt_mappable_memory += obj->gtt_space->size;
fb7d516a 128 } else {
a00b10c3 129 if (obj->pin_mappable || obj->fault_mappable)
fb7d516a
DV		 130 /* Combined state still mappable. */
131 return;
132 dev_priv->mm.gtt_mappable_count--;
a00b10c3 133 dev_priv->mm.gtt_mappable_memory -= obj->gtt_space->size;
fb7d516a 134 }
73aa808f
CW		 135}
136
137static void i915_gem_info_add_pin(struct drm_i915_private *dev_priv,
a00b10c3 138 struct drm_i915_gem_object *obj,
fb7d516a 139 bool mappable)
73aa808f
CW		 140{
141 dev_priv->mm.pin_count++;
a00b10c3 142 dev_priv->mm.pin_memory += obj->gtt_space->size;
fb7d516a 143 if (mappable) {
a00b10c3 144 obj->pin_mappable = true;
fb7d516a
DV		 145 i915_gem_info_update_mappable(dev_priv, obj, true);
146 }
73aa808f
CW		 147}
148
149static void i915_gem_info_remove_pin(struct drm_i915_private *dev_priv,
a00b10c3 150 struct drm_i915_gem_object *obj)
73aa808f
CW		 151{
152 dev_priv->mm.pin_count--;
a00b10c3
CW		 153 dev_priv->mm.pin_memory -= obj->gtt_space->size;
154 if (obj->pin_mappable) {
155 obj->pin_mappable = false;
fb7d516a
DV		 156 i915_gem_info_update_mappable(dev_priv, obj, false);
157 }
73aa808f
CW		 158}
159
30dbf0c0
CW		 160int
161i915_gem_check_is_wedged(struct drm_device *dev)
162{
163 struct drm_i915_private *dev_priv = dev->dev_private;
164 struct completion *x = &dev_priv->error_completion;
165 unsigned long flags;
166 int ret;
167
168 if (!atomic_read(&dev_priv->mm.wedged))
169 return 0;
170
171 ret = wait_for_completion_interruptible(x);
172 if (ret)
173 return ret;
174
175 /* Success, we reset the GPU! */
176 if (!atomic_read(&dev_priv->mm.wedged))
177 return 0;
178
179 /* GPU is hung, bump the completion count to account for
180 * the token we just consumed so that we never hit zero and
181 * end up waiting upon a subsequent completion event that
182 * will never happen.
183 */
184 spin_lock_irqsave(&x->wait.lock, flags);
185 x->done++;
186 spin_unlock_irqrestore(&x->wait.lock, flags);
187 return -EIO;
188}
189
76c1dec1
CW		 190static int i915_mutex_lock_interruptible(struct drm_device *dev)
191{
192 struct drm_i915_private *dev_priv = dev->dev_private;
193 int ret;
194
195 ret = i915_gem_check_is_wedged(dev);
196 if (ret)
197 return ret;
198
199 ret = mutex_lock_interruptible(&dev->struct_mutex);
200 if (ret)
201 return ret;
202
203 if (atomic_read(&dev_priv->mm.wedged)) {
204 mutex_unlock(&dev->struct_mutex);
205 return -EAGAIN;
206 }
207
23bc5982 208 WARN_ON(i915_verify_lists(dev));
76c1dec1
CW		 209 return 0;
210}
30dbf0c0 211
7d1c4804 212static inline bool
05394f39 213i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
7d1c4804 214{
05394f39 215 return obj->gtt_space && !obj->active && obj->pin_count == 0;
7d1c4804
CW		 216}
217
73aa808f
CW		 218int i915_gem_do_init(struct drm_device *dev,
219 unsigned long start,
53984635 220 unsigned long mappable_end,
79e53945 221 unsigned long end)
673a394b
EA		 222{
223 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 224
79e53945
JB		 225 if (start >= end ||
226 (start & (PAGE_SIZE - 1)) != 0 ||
227 (end & (PAGE_SIZE - 1)) != 0) {
673a394b
EA		 228 return -EINVAL;
229 }
230
79e53945
JB		 231 drm_mm_init(&dev_priv->mm.gtt_space, start,
232 end - start);
673a394b 233
73aa808f 234 dev_priv->mm.gtt_total = end - start;
fb7d516a 235 dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
53984635 236 dev_priv->mm.gtt_mappable_end = mappable_end;
79e53945
JB		 237
238 return 0;
239}
673a394b 240
79e53945
JB		 241int
242i915_gem_init_ioctl(struct drm_device *dev, void *data,
05394f39 243 struct drm_file *file)
79e53945
JB		 244{
245 struct drm_i915_gem_init *args = data;
246 int ret;
247
248 mutex_lock(&dev->struct_mutex);
53984635 249 ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
673a394b
EA		 250 mutex_unlock(&dev->struct_mutex);
251
79e53945 252 return ret;
673a394b
EA		 253}
254
5a125c3c
EA		 255int
256i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
05394f39 257 struct drm_file *file)
5a125c3c 258{
73aa808f 259 struct drm_i915_private *dev_priv = dev->dev_private;
5a125c3c 260 struct drm_i915_gem_get_aperture *args = data;
5a125c3c
EA		 261
262 if (!(dev->driver->driver_features & DRIVER_GEM))
263 return -ENODEV;
264
73aa808f
CW		 265 mutex_lock(&dev->struct_mutex);
266 args->aper_size = dev_priv->mm.gtt_total;
267 args->aper_available_size = args->aper_size - dev_priv->mm.pin_memory;
268 mutex_unlock(&dev->struct_mutex);
5a125c3c
EA		 269
270 return 0;
271}
272
673a394b
EA		 273
274/**
275 * Creates a new mm object and returns a handle to it.
276 */
277int
278i915_gem_create_ioctl(struct drm_device *dev, void *data,
05394f39 279 struct drm_file *file)
673a394b
EA		 280{
281 struct drm_i915_gem_create *args = data;
05394f39 282 struct drm_i915_gem_object *obj;
a1a2d1d3
PP		 283 int ret;
284 u32 handle;
673a394b
EA		 285
286 args->size = roundup(args->size, PAGE_SIZE);
287
288 /* Allocate the new object */
ac52bc56 289 obj = i915_gem_alloc_object(dev, args->size);
673a394b
EA		 290 if (obj == NULL)
291 return -ENOMEM;
292
05394f39 293 ret = drm_gem_handle_create(file, &obj->base, &handle);
1dfd9754 294 if (ret) {
05394f39
CW		 295 drm_gem_object_release(&obj->base);
296 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
202f2fef 297 kfree(obj);
673a394b 298 return ret;
1dfd9754 299 }
673a394b 300
202f2fef 301 /* drop reference from allocate - handle holds it now */
05394f39 302 drm_gem_object_unreference(&obj->base);
202f2fef
CW		 303 trace_i915_gem_object_create(obj);
304
1dfd9754 305 args->handle = handle;
673a394b
EA		 306 return 0;
307}
308
05394f39 309static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
280b713b 310{
05394f39 311 drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
280b713b
EA		 312
313 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
05394f39 314 obj->tiling_mode != I915_TILING_NONE;
280b713b
EA		 315}
316
99a03df5 317static inline void
40123c1f
EA		 318slow_shmem_copy(struct page *dst_page,
319 int dst_offset,
320 struct page *src_page,
321 int src_offset,
322 int length)
323{
324 char *dst_vaddr, *src_vaddr;
325
99a03df5
CW		 326 dst_vaddr = kmap(dst_page);
327 src_vaddr = kmap(src_page);
40123c1f
EA		 328
329 memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
330
99a03df5
CW		 331 kunmap(src_page);
332 kunmap(dst_page);
40123c1f
EA		 333}
334
99a03df5 335static inline void
280b713b
EA		 336slow_shmem_bit17_copy(struct page *gpu_page,
337 int gpu_offset,
338 struct page *cpu_page,
339 int cpu_offset,
340 int length,
341 int is_read)
342{
343 char *gpu_vaddr, *cpu_vaddr;
344
345 /* Use the unswizzled path if this page isn't affected. */
346 if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
347 if (is_read)
348 return slow_shmem_copy(cpu_page, cpu_offset,
349 gpu_page, gpu_offset, length);
350 else
351 return slow_shmem_copy(gpu_page, gpu_offset,
352 cpu_page, cpu_offset, length);
353 }
354
99a03df5
CW		 355 gpu_vaddr = kmap(gpu_page);
356 cpu_vaddr = kmap(cpu_page);
280b713b
EA		 357
358 /* Copy the data, XORing A6 with A17 (1). The user already knows he's
359 * XORing with the other bits (A9 for Y, A9 and A10 for X)
360 */
361 while (length > 0) {
362 int cacheline_end = ALIGN(gpu_offset + 1, 64);
363 int this_length = min(cacheline_end - gpu_offset, length);
364 int swizzled_gpu_offset = gpu_offset ^ 64;
365
366 if (is_read) {
367 memcpy(cpu_vaddr + cpu_offset,
368 gpu_vaddr + swizzled_gpu_offset,
369 this_length);
370 } else {
371 memcpy(gpu_vaddr + swizzled_gpu_offset,
372 cpu_vaddr + cpu_offset,
373 this_length);
374 }
375 cpu_offset += this_length;
376 gpu_offset += this_length;
377 length -= this_length;
378 }
379
99a03df5
CW		 380 kunmap(cpu_page);
381 kunmap(gpu_page);
280b713b
EA		 382}
383
eb01459f
EA		 384/**
385 * This is the fast shmem pread path, which attempts to copy_from_user directly
386 * from the backing pages of the object to the user's address space. On a
387 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
388 */
389static int
05394f39
CW		 390i915_gem_shmem_pread_fast(struct drm_device *dev,
391 struct drm_i915_gem_object *obj,
eb01459f 392 struct drm_i915_gem_pread *args,
05394f39 393 struct drm_file *file)
eb01459f 394{
05394f39 395 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
eb01459f 396 ssize_t remain;
e5281ccd 397 loff_t offset;
eb01459f
EA		 398 char __user *user_data;
399 int page_offset, page_length;
eb01459f
EA		 400
401 user_data = (char __user *) (uintptr_t) args->data_ptr;
402 remain = args->size;
403
eb01459f
EA		 404 offset = args->offset;
405
406 while (remain > 0) {
e5281ccd
CW		 407 struct page *page;
408 char *vaddr;
409 int ret;
410
eb01459f
EA		 411 /* Operation in this page
412 *
eb01459f
EA		 413 * page_offset = offset within page
414 * page_length = bytes to copy for this page
415 */
eb01459f
EA		 416 page_offset = offset & (PAGE_SIZE-1);
417 page_length = remain;
418 if ((page_offset + remain) > PAGE_SIZE)
419 page_length = PAGE_SIZE - page_offset;
420
e5281ccd
CW		 421 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
422 GFP_HIGHUSER | __GFP_RECLAIMABLE);
423 if (IS_ERR(page))
424 return PTR_ERR(page);
425
426 vaddr = kmap_atomic(page);
427 ret = __copy_to_user_inatomic(user_data,
428 vaddr + page_offset,
429 page_length);
430 kunmap_atomic(vaddr);
431
432 mark_page_accessed(page);
433 page_cache_release(page);
434 if (ret)
4f27b75d 435 return -EFAULT;
eb01459f
EA		 436
437 remain -= page_length;
438 user_data += page_length;
439 offset += page_length;
440 }
441
4f27b75d 442 return 0;
eb01459f
EA		 443}
444
445/**
446 * This is the fallback shmem pread path, which allocates temporary storage
447 * in kernel space to copy_to_user into outside of the struct_mutex, so we
448 * can copy out of the object's backing pages while holding the struct mutex
449 * and not take page faults.
450 */
451static int
05394f39
CW		 452i915_gem_shmem_pread_slow(struct drm_device *dev,
453 struct drm_i915_gem_object *obj,
eb01459f 454 struct drm_i915_gem_pread *args,
05394f39 455 struct drm_file *file)
eb01459f 456{
05394f39 457 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
eb01459f
EA		 458 struct mm_struct *mm = current->mm;
459 struct page **user_pages;
460 ssize_t remain;
461 loff_t offset, pinned_pages, i;
462 loff_t first_data_page, last_data_page, num_pages;
e5281ccd
CW		 463 int shmem_page_offset;
464 int data_page_index, data_page_offset;
eb01459f
EA		 465 int page_length;
466 int ret;
467 uint64_t data_ptr = args->data_ptr;
280b713b 468 int do_bit17_swizzling;
eb01459f
EA		 469
470 remain = args->size;
471
472 /* Pin the user pages containing the data. We can't fault while
473 * holding the struct mutex, yet we want to hold it while
474 * dereferencing the user data.
475 */
476 first_data_page = data_ptr / PAGE_SIZE;
477 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
478 num_pages = last_data_page - first_data_page + 1;
479
4f27b75d 480 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
eb01459f
EA		 481 if (user_pages == NULL)
482 return -ENOMEM;
483
4f27b75d 484 mutex_unlock(&dev->struct_mutex);
eb01459f
EA		 485 down_read(&mm->mmap_sem);
486 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
e5e9ecde 487 num_pages, 1, 0, user_pages, NULL);
eb01459f 488 up_read(&mm->mmap_sem);
4f27b75d 489 mutex_lock(&dev->struct_mutex);
eb01459f
EA		 490 if (pinned_pages < num_pages) {
491 ret = -EFAULT;
4f27b75d 492 goto out;
eb01459f
EA		 493 }
494
4f27b75d
CW		 495 ret = i915_gem_object_set_cpu_read_domain_range(obj,
496 args->offset,
497 args->size);
07f73f69 498 if (ret)
4f27b75d 499 goto out;
eb01459f 500
4f27b75d 501 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
eb01459f 502
eb01459f
EA		 503 offset = args->offset;
504
505 while (remain > 0) {
e5281ccd
CW		 506 struct page *page;
507
eb01459f
EA		 508 /* Operation in this page
509 *
eb01459f
EA		 510 * shmem_page_offset = offset within page in shmem file
511 * data_page_index = page number in get_user_pages return
512 * data_page_offset = offset with data_page_index page.
513 * page_length = bytes to copy for this page
514 */
eb01459f
EA		 515 shmem_page_offset = offset & ~PAGE_MASK;
516 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
517 data_page_offset = data_ptr & ~PAGE_MASK;
518
519 page_length = remain;
520 if ((shmem_page_offset + page_length) > PAGE_SIZE)
521 page_length = PAGE_SIZE - shmem_page_offset;
522 if ((data_page_offset + page_length) > PAGE_SIZE)
523 page_length = PAGE_SIZE - data_page_offset;
524
e5281ccd
CW		 525 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
526 GFP_HIGHUSER | __GFP_RECLAIMABLE);
527 if (IS_ERR(page))
528 return PTR_ERR(page);
529
280b713b 530 if (do_bit17_swizzling) {
e5281ccd 531 slow_shmem_bit17_copy(page,
280b713b 532 shmem_page_offset,
99a03df5
CW		 533 user_pages[data_page_index],
534 data_page_offset,
535 page_length,
536 1);
537 } else {
538 slow_shmem_copy(user_pages[data_page_index],
539 data_page_offset,
e5281ccd 540 page,
99a03df5
CW		 541 shmem_page_offset,
542 page_length);
280b713b 543 }
eb01459f 544
e5281ccd
CW		 545 mark_page_accessed(page);
546 page_cache_release(page);
547
eb01459f
EA		 548 remain -= page_length;
549 data_ptr += page_length;
550 offset += page_length;
551 }
552
4f27b75d 553out:
eb01459f
EA		 554 for (i = 0; i < pinned_pages; i++) {
555 SetPageDirty(user_pages[i]);
e5281ccd 556 mark_page_accessed(user_pages[i]);
eb01459f
EA		 557 page_cache_release(user_pages[i]);
558 }
8e7d2b2c 559 drm_free_large(user_pages);
eb01459f
EA		 560
561 return ret;
562}
563
673a394b
EA		 564/**
565 * Reads data from the object referenced by handle.
566 *
567 * On error, the contents of *data are undefined.
568 */
569int
570i915_gem_pread_ioctl(struct drm_device *dev, void *data,
05394f39 571 struct drm_file *file)
673a394b
EA		 572{
573 struct drm_i915_gem_pread *args = data;
05394f39 574 struct drm_i915_gem_object *obj;
35b62a89 575 int ret = 0;
673a394b 576
51311d0a
CW		 577 if (args->size == 0)
578 return 0;
579
580 if (!access_ok(VERIFY_WRITE,
581 (char __user *)(uintptr_t)args->data_ptr,
582 args->size))
583 return -EFAULT;
584
585 ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,
586 args->size);
587 if (ret)
588 return -EFAULT;
589
4f27b75d 590 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 591 if (ret)
4f27b75d 592 return ret;
673a394b 593
05394f39 594 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1d7cfea1
CW		 595 if (obj == NULL) {
596 ret = -ENOENT;
597 goto unlock;
4f27b75d 598 }
673a394b 599
7dcd2499 600 /* Bounds check source. */
05394f39
CW		 601 if (args->offset > obj->base.size ||
602 args->size > obj->base.size - args->offset) {
ce9d419d 603 ret = -EINVAL;
35b62a89 604 goto out;
ce9d419d
CW		 605 }
606
4f27b75d
CW		 607 ret = i915_gem_object_set_cpu_read_domain_range(obj,
608 args->offset,
609 args->size);
610 if (ret)
e5281ccd 611 goto out;
4f27b75d
CW		 612
613 ret = -EFAULT;
614 if (!i915_gem_object_needs_bit17_swizzle(obj))
05394f39 615 ret = i915_gem_shmem_pread_fast(dev, obj, args, file);
4f27b75d 616 if (ret == -EFAULT)
05394f39 617 ret = i915_gem_shmem_pread_slow(dev, obj, args, file);
673a394b 618
35b62a89 619out:
05394f39 620 drm_gem_object_unreference(&obj->base);
1d7cfea1 621unlock:
4f27b75d 622 mutex_unlock(&dev->struct_mutex);
eb01459f 623 return ret;
673a394b
EA		 624}
625
0839ccb8
KP		 626/* This is the fast write path which cannot handle
627 * page faults in the source data
9b7530cc 628 */
0839ccb8
KP		 629
630static inline int
631fast_user_write(struct io_mapping *mapping,
632 loff_t page_base, int page_offset,
633 char __user *user_data,
634 int length)
9b7530cc 635{
9b7530cc 636 char *vaddr_atomic;
0839ccb8 637 unsigned long unwritten;
9b7530cc 638
3e4d3af5 639 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
0839ccb8
KP		 640 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
641 user_data, length);
3e4d3af5 642 io_mapping_unmap_atomic(vaddr_atomic);
fbd5a26d 643 return unwritten;
0839ccb8
KP		 644}
645
646/* Here's the write path which can sleep for
647 * page faults
648 */
649
ab34c226 650static inline void
3de09aa3
EA		 651slow_kernel_write(struct io_mapping *mapping,
652 loff_t gtt_base, int gtt_offset,
653 struct page *user_page, int user_offset,
654 int length)
0839ccb8 655{
ab34c226
CW		 656 char __iomem *dst_vaddr;
657 char *src_vaddr;
0839ccb8 658
ab34c226
CW		 659 dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
660 src_vaddr = kmap(user_page);
661
662 memcpy_toio(dst_vaddr + gtt_offset,
663 src_vaddr + user_offset,
664 length);
665
666 kunmap(user_page);
667 io_mapping_unmap(dst_vaddr);
9b7530cc
LT		 668}
669
3de09aa3
EA		 670/**
671 * This is the fast pwrite path, where we copy the data directly from the
672 * user into the GTT, uncached.
673 */
673a394b 674static int
05394f39
CW		 675i915_gem_gtt_pwrite_fast(struct drm_device *dev,
676 struct drm_i915_gem_object *obj,
3de09aa3 677 struct drm_i915_gem_pwrite *args,
05394f39 678 struct drm_file *file)
673a394b 679{
0839ccb8 680 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 681 ssize_t remain;
0839ccb8 682 loff_t offset, page_base;
673a394b 683 char __user *user_data;
0839ccb8 684 int page_offset, page_length;
673a394b
EA		 685
686 user_data = (char __user *) (uintptr_t) args->data_ptr;
687 remain = args->size;
673a394b 688
05394f39 689 offset = obj->gtt_offset + args->offset;
673a394b
EA		 690
691 while (remain > 0) {
692 /* Operation in this page
693 *
0839ccb8
KP		 694 * page_base = page offset within aperture
695 * page_offset = offset within page
696 * page_length = bytes to copy for this page
673a394b 697 */
0839ccb8
KP		 698 page_base = (offset & ~(PAGE_SIZE-1));
699 page_offset = offset & (PAGE_SIZE-1);
700 page_length = remain;
701 if ((page_offset + remain) > PAGE_SIZE)
702 page_length = PAGE_SIZE - page_offset;
703
0839ccb8 704 /* If we get a fault while copying data, then (presumably) our
3de09aa3
EA		 705 * source page isn't available. Return the error and we'll
706 * retry in the slow path.
0839ccb8 707 */
fbd5a26d
CW		 708 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
709 page_offset, user_data, page_length))
710
711 return -EFAULT;
673a394b 712
0839ccb8
KP		 713 remain -= page_length;
714 user_data += page_length;
715 offset += page_length;
673a394b 716 }
673a394b 717
fbd5a26d 718 return 0;
673a394b
EA		 719}
720
3de09aa3
EA		 721/**
722 * This is the fallback GTT pwrite path, which uses get_user_pages to pin
723 * the memory and maps it using kmap_atomic for copying.
724 *
725 * This code resulted in x11perf -rgb10text consuming about 10% more CPU
726 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
727 */
3043c60c 728static int
05394f39
CW		 729i915_gem_gtt_pwrite_slow(struct drm_device *dev,
730 struct drm_i915_gem_object *obj,
3de09aa3 731 struct drm_i915_gem_pwrite *args,
05394f39 732 struct drm_file *file)
673a394b 733{
3de09aa3
EA		 734 drm_i915_private_t *dev_priv = dev->dev_private;
735 ssize_t remain;
736 loff_t gtt_page_base, offset;
737 loff_t first_data_page, last_data_page, num_pages;
738 loff_t pinned_pages, i;
739 struct page **user_pages;
740 struct mm_struct *mm = current->mm;
741 int gtt_page_offset, data_page_offset, data_page_index, page_length;
673a394b 742 int ret;
3de09aa3
EA		 743 uint64_t data_ptr = args->data_ptr;
744
745 remain = args->size;
746
747 /* Pin the user pages containing the data. We can't fault while
748 * holding the struct mutex, and all of the pwrite implementations
749 * want to hold it while dereferencing the user data.
750 */
751 first_data_page = data_ptr / PAGE_SIZE;
752 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
753 num_pages = last_data_page - first_data_page + 1;
754
fbd5a26d 755 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
3de09aa3
EA		 756 if (user_pages == NULL)
757 return -ENOMEM;
758
fbd5a26d 759 mutex_unlock(&dev->struct_mutex);
3de09aa3
EA		 760 down_read(&mm->mmap_sem);
761 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
762 num_pages, 0, 0, user_pages, NULL);
763 up_read(&mm->mmap_sem);
fbd5a26d 764 mutex_lock(&dev->struct_mutex);
3de09aa3
EA		 765 if (pinned_pages < num_pages) {
766 ret = -EFAULT;
767 goto out_unpin_pages;
768 }
673a394b 769
3de09aa3
EA		 770 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
771 if (ret)
fbd5a26d 772 goto out_unpin_pages;
3de09aa3 773
05394f39 774 offset = obj->gtt_offset + args->offset;
3de09aa3
EA		 775
776 while (remain > 0) {
777 /* Operation in this page
778 *
779 * gtt_page_base = page offset within aperture
780 * gtt_page_offset = offset within page in aperture
781 * data_page_index = page number in get_user_pages return
782 * data_page_offset = offset with data_page_index page.
783 * page_length = bytes to copy for this page
784 */
785 gtt_page_base = offset & PAGE_MASK;
786 gtt_page_offset = offset & ~PAGE_MASK;
787 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
788 data_page_offset = data_ptr & ~PAGE_MASK;
789
790 page_length = remain;
791 if ((gtt_page_offset + page_length) > PAGE_SIZE)
792 page_length = PAGE_SIZE - gtt_page_offset;
793 if ((data_page_offset + page_length) > PAGE_SIZE)
794 page_length = PAGE_SIZE - data_page_offset;
795
ab34c226
CW		 796 slow_kernel_write(dev_priv->mm.gtt_mapping,
797 gtt_page_base, gtt_page_offset,
798 user_pages[data_page_index],
799 data_page_offset,
800 page_length);
3de09aa3
EA		 801
802 remain -= page_length;
803 offset += page_length;
804 data_ptr += page_length;
805 }
806
3de09aa3
EA		 807out_unpin_pages:
808 for (i = 0; i < pinned_pages; i++)
809 page_cache_release(user_pages[i]);
8e7d2b2c 810 drm_free_large(user_pages);
3de09aa3
EA		 811
812 return ret;
813}
814
40123c1f
EA		 815/**
816 * This is the fast shmem pwrite path, which attempts to directly
817 * copy_from_user into the kmapped pages backing the object.
818 */
3043c60c 819static int
05394f39
CW		 820i915_gem_shmem_pwrite_fast(struct drm_device *dev,
821 struct drm_i915_gem_object *obj,
40123c1f 822 struct drm_i915_gem_pwrite *args,
05394f39 823 struct drm_file *file)
673a394b 824{
05394f39 825 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
40123c1f 826 ssize_t remain;
e5281ccd 827 loff_t offset;
40123c1f
EA		 828 char __user *user_data;
829 int page_offset, page_length;
40123c1f
EA		 830
831 user_data = (char __user *) (uintptr_t) args->data_ptr;
832 remain = args->size;
673a394b 833
40123c1f 834 offset = args->offset;
05394f39 835 obj->dirty = 1;
40123c1f
EA		 836
837 while (remain > 0) {
e5281ccd
CW		 838 struct page *page;
839 char *vaddr;
840 int ret;
841
40123c1f
EA		 842 /* Operation in this page
843 *
40123c1f
EA		 844 * page_offset = offset within page
845 * page_length = bytes to copy for this page
846 */
40123c1f
EA		 847 page_offset = offset & (PAGE_SIZE-1);
848 page_length = remain;
849 if ((page_offset + remain) > PAGE_SIZE)
850 page_length = PAGE_SIZE - page_offset;
851
e5281ccd
CW		 852 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
853 GFP_HIGHUSER | __GFP_RECLAIMABLE);
854 if (IS_ERR(page))
855 return PTR_ERR(page);
856
857 vaddr = kmap_atomic(page, KM_USER0);
858 ret = __copy_from_user_inatomic(vaddr + page_offset,
859 user_data,
860 page_length);
861 kunmap_atomic(vaddr, KM_USER0);
862
863 set_page_dirty(page);
864 mark_page_accessed(page);
865 page_cache_release(page);
866
867 /* If we get a fault while copying data, then (presumably) our
868 * source page isn't available. Return the error and we'll
869 * retry in the slow path.
870 */
871 if (ret)
fbd5a26d 872 return -EFAULT;
40123c1f
EA		 873
874 remain -= page_length;
875 user_data += page_length;
876 offset += page_length;
877 }
878
fbd5a26d 879 return 0;
40123c1f
EA		 880}
881
882/**
883 * This is the fallback shmem pwrite path, which uses get_user_pages to pin
884 * the memory and maps it using kmap_atomic for copying.
885 *
886 * This avoids taking mmap_sem for faulting on the user's address while the
887 * struct_mutex is held.
888 */
889static int
05394f39
CW		 890i915_gem_shmem_pwrite_slow(struct drm_device *dev,
891 struct drm_i915_gem_object *obj,
40123c1f 892 struct drm_i915_gem_pwrite *args,
05394f39 893 struct drm_file *file)
40123c1f 894{
05394f39 895 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
40123c1f
EA		 896 struct mm_struct *mm = current->mm;
897 struct page **user_pages;
898 ssize_t remain;
899 loff_t offset, pinned_pages, i;
900 loff_t first_data_page, last_data_page, num_pages;
e5281ccd 901 int shmem_page_offset;
40123c1f
EA		 902 int data_page_index, data_page_offset;
903 int page_length;
904 int ret;
905 uint64_t data_ptr = args->data_ptr;
280b713b 906 int do_bit17_swizzling;
40123c1f
EA		 907
908 remain = args->size;
909
910 /* Pin the user pages containing the data. We can't fault while
911 * holding the struct mutex, and all of the pwrite implementations
912 * want to hold it while dereferencing the user data.
913 */
914 first_data_page = data_ptr / PAGE_SIZE;
915 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
916 num_pages = last_data_page - first_data_page + 1;
917
4f27b75d 918 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
40123c1f
EA		 919 if (user_pages == NULL)
920 return -ENOMEM;
921
fbd5a26d 922 mutex_unlock(&dev->struct_mutex);
40123c1f
EA		 923 down_read(&mm->mmap_sem);
924 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
925 num_pages, 0, 0, user_pages, NULL);
926 up_read(&mm->mmap_sem);
fbd5a26d 927 mutex_lock(&dev->struct_mutex);
40123c1f
EA		 928 if (pinned_pages < num_pages) {
929 ret = -EFAULT;
fbd5a26d 930 goto out;
673a394b
EA		 931 }
932
fbd5a26d 933 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
07f73f69 934 if (ret)
fbd5a26d 935 goto out;
40123c1f 936
fbd5a26d 937 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
40123c1f 938
673a394b 939 offset = args->offset;
05394f39 940 obj->dirty = 1;
673a394b 941
40123c1f 942 while (remain > 0) {
e5281ccd
CW		 943 struct page *page;
944
40123c1f
EA		 945 /* Operation in this page
946 *
40123c1f
EA		 947 * shmem_page_offset = offset within page in shmem file
948 * data_page_index = page number in get_user_pages return
949 * data_page_offset = offset with data_page_index page.
950 * page_length = bytes to copy for this page
951 */
40123c1f
EA		 952 shmem_page_offset = offset & ~PAGE_MASK;
953 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
954 data_page_offset = data_ptr & ~PAGE_MASK;
955
956 page_length = remain;
957 if ((shmem_page_offset + page_length) > PAGE_SIZE)
958 page_length = PAGE_SIZE - shmem_page_offset;
959 if ((data_page_offset + page_length) > PAGE_SIZE)
960 page_length = PAGE_SIZE - data_page_offset;
961
e5281ccd
CW		 962 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
963 GFP_HIGHUSER | __GFP_RECLAIMABLE);
964 if (IS_ERR(page)) {
965 ret = PTR_ERR(page);
966 goto out;
967 }
968
280b713b 969 if (do_bit17_swizzling) {
e5281ccd 970 slow_shmem_bit17_copy(page,
280b713b
EA		 971 shmem_page_offset,
972 user_pages[data_page_index],
973 data_page_offset,
99a03df5
CW		 974 page_length,
975 0);
976 } else {
e5281ccd 977 slow_shmem_copy(page,
99a03df5
CW		 978 shmem_page_offset,
979 user_pages[data_page_index],
980 data_page_offset,
981 page_length);
280b713b 982 }
40123c1f 983
e5281ccd
CW		 984 set_page_dirty(page);
985 mark_page_accessed(page);
986 page_cache_release(page);
987
40123c1f
EA		 988 remain -= page_length;
989 data_ptr += page_length;
990 offset += page_length;
673a394b
EA		 991 }
992
fbd5a26d 993out:
40123c1f
EA		 994 for (i = 0; i < pinned_pages; i++)
995 page_cache_release(user_pages[i]);
8e7d2b2c 996 drm_free_large(user_pages);
673a394b 997
40123c1f 998 return ret;
673a394b
EA		 999}
1000
1001/**
1002 * Writes data to the object referenced by handle.
1003 *
1004 * On error, the contents of the buffer that were to be modified are undefined.
1005 */
1006int
1007i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
fbd5a26d 1008 struct drm_file *file)
673a394b
EA		 1009{
1010 struct drm_i915_gem_pwrite *args = data;
05394f39 1011 struct drm_i915_gem_object *obj;
51311d0a
CW		 1012 int ret;
1013
1014 if (args->size == 0)
1015 return 0;
1016
1017 if (!access_ok(VERIFY_READ,
1018 (char __user *)(uintptr_t)args->data_ptr,
1019 args->size))
1020 return -EFAULT;
1021
1022 ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
1023 args->size);
1024 if (ret)
1025 return -EFAULT;
673a394b 1026
fbd5a26d 1027 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 1028 if (ret)
fbd5a26d 1029 return ret;
1d7cfea1 1030
05394f39 1031 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1d7cfea1
CW		 1032 if (obj == NULL) {
1033 ret = -ENOENT;
1034 goto unlock;
fbd5a26d 1035 }
673a394b 1036
7dcd2499 1037 /* Bounds check destination. */
05394f39
CW		 1038 if (args->offset > obj->base.size ||
1039 args->size > obj->base.size - args->offset) {
ce9d419d 1040 ret = -EINVAL;
35b62a89 1041 goto out;
ce9d419d
CW		 1042 }
1043
673a394b
EA		 1044 /* We can only do the GTT pwrite on untiled buffers, as otherwise
1045 * it would end up going through the fenced access, and we'll get
1046 * different detiling behavior between reading and writing.
1047 * pread/pwrite currently are reading and writing from the CPU
1048 * perspective, requiring manual detiling by the client.
1049 */
05394f39 1050 if (obj->phys_obj)
fbd5a26d 1051 ret = i915_gem_phys_pwrite(dev, obj, args, file);
05394f39
CW		 1052 else if (obj->tiling_mode == I915_TILING_NONE &&
1053 obj->gtt_space &&
1054 obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
75e9e915 1055 ret = i915_gem_object_pin(obj, 0, true);
fbd5a26d
CW		 1056 if (ret)
1057 goto out;
1058
1059 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
1060 if (ret)
1061 goto out_unpin;
1062
1063 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1064 if (ret == -EFAULT)
1065 ret = i915_gem_gtt_pwrite_slow(dev, obj, args, file);
1066
1067out_unpin:
1068 i915_gem_object_unpin(obj);
40123c1f 1069 } else {
fbd5a26d
CW		 1070 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1071 if (ret)
e5281ccd 1072 goto out;
673a394b 1073
fbd5a26d
CW		 1074 ret = -EFAULT;
1075 if (!i915_gem_object_needs_bit17_swizzle(obj))
1076 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
1077 if (ret == -EFAULT)
1078 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
fbd5a26d 1079 }
673a394b 1080
35b62a89 1081out:
05394f39 1082 drm_gem_object_unreference(&obj->base);
1d7cfea1 1083unlock:
fbd5a26d 1084 mutex_unlock(&dev->struct_mutex);
673a394b
EA		 1085 return ret;
1086}
1087
1088/**
2ef7eeaa
EA		 1089 * Called when user space prepares to use an object with the CPU, either
1090 * through the mmap ioctl's mapping or a GTT mapping.
673a394b
EA		 1091 */
1092int
1093i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
05394f39 1094 struct drm_file *file)
673a394b 1095{
a09ba7fa 1096 struct drm_i915_private *dev_priv = dev->dev_private;
673a394b 1097 struct drm_i915_gem_set_domain *args = data;
05394f39 1098 struct drm_i915_gem_object *obj;
2ef7eeaa
EA		 1099 uint32_t read_domains = args->read_domains;
1100 uint32_t write_domain = args->write_domain;
673a394b
EA		 1101 int ret;
1102
1103 if (!(dev->driver->driver_features & DRIVER_GEM))
1104 return -ENODEV;
1105
2ef7eeaa 1106 /* Only handle setting domains to types used by the CPU. */
21d509e3 1107 if (write_domain & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 1108 return -EINVAL;
1109
21d509e3 1110 if (read_domains & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 1111 return -EINVAL;
1112
1113 /* Having something in the write domain implies it's in the read
1114 * domain, and only that read domain. Enforce that in the request.
1115 */
1116 if (write_domain != 0 && read_domains != write_domain)
1117 return -EINVAL;
1118
76c1dec1 1119 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 1120 if (ret)
76c1dec1 1121 return ret;
1d7cfea1 1122
05394f39 1123 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1d7cfea1
CW		 1124 if (obj == NULL) {
1125 ret = -ENOENT;
1126 goto unlock;
76c1dec1 1127 }
673a394b 1128
652c393a
JB		 1129 intel_mark_busy(dev, obj);
1130
2ef7eeaa
EA		 1131 if (read_domains & I915_GEM_DOMAIN_GTT) {
1132 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
02354392 1133
a09ba7fa
EA		 1134 /* Update the LRU on the fence for the CPU access that's
1135 * about to occur.
1136 */
05394f39 1137 if (obj->fence_reg != I915_FENCE_REG_NONE) {
007cc8ac 1138 struct drm_i915_fence_reg *reg =
05394f39 1139 &dev_priv->fence_regs[obj->fence_reg];
007cc8ac 1140 list_move_tail(&reg->lru_list,
a09ba7fa
EA		 1141 &dev_priv->mm.fence_list);
1142 }
1143
02354392
EA		 1144 /* Silently promote "you're not bound, there was nothing to do"
1145 * to success, since the client was just asking us to
1146 * make sure everything was done.
1147 */
1148 if (ret == -EINVAL)
1149 ret = 0;
2ef7eeaa 1150 } else {
e47c68e9 1151 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
2ef7eeaa
EA		 1152 }
1153
7d1c4804 1154 /* Maintain LRU order of "inactive" objects */
05394f39
CW		 1155 if (ret == 0 && i915_gem_object_is_inactive(obj))
1156 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
7d1c4804 1157
05394f39 1158 drm_gem_object_unreference(&obj->base);
1d7cfea1 1159unlock:
673a394b
EA		 1160 mutex_unlock(&dev->struct_mutex);
1161 return ret;
1162}
1163
1164/**
1165 * Called when user space has done writes to this buffer
1166 */
1167int
1168i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
05394f39 1169 struct drm_file *file)
673a394b
EA		 1170{
1171 struct drm_i915_gem_sw_finish *args = data;
05394f39 1172 struct drm_i915_gem_object *obj;
673a394b
EA		 1173 int ret = 0;
1174
1175 if (!(dev->driver->driver_features & DRIVER_GEM))
1176 return -ENODEV;
1177
76c1dec1 1178 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 1179 if (ret)
76c1dec1 1180 return ret;
1d7cfea1 1181
05394f39 1182 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
673a394b 1183 if (obj == NULL) {
1d7cfea1
CW		 1184 ret = -ENOENT;
1185 goto unlock;
673a394b
EA		 1186 }
1187
673a394b 1188 /* Pinned buffers may be scanout, so flush the cache */
05394f39 1189 if (obj->pin_count)
e47c68e9
EA		 1190 i915_gem_object_flush_cpu_write_domain(obj);
1191
05394f39 1192 drm_gem_object_unreference(&obj->base);
1d7cfea1 1193unlock:
673a394b
EA		 1194 mutex_unlock(&dev->struct_mutex);
1195 return ret;
1196}
1197
1198/**
1199 * Maps the contents of an object, returning the address it is mapped
1200 * into.
1201 *
1202 * While the mapping holds a reference on the contents of the object, it doesn't
1203 * imply a ref on the object itself.
1204 */
1205int
1206i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
05394f39 1207 struct drm_file *file)
673a394b 1208{
da761a6e 1209 struct drm_i915_private *dev_priv = dev->dev_private;
673a394b
EA		 1210 struct drm_i915_gem_mmap *args = data;
1211 struct drm_gem_object *obj;
1212 loff_t offset;
1213 unsigned long addr;
1214
1215 if (!(dev->driver->driver_features & DRIVER_GEM))
1216 return -ENODEV;
1217
05394f39 1218 obj = drm_gem_object_lookup(dev, file, args->handle);
673a394b 1219 if (obj == NULL)
bf79cb91 1220 return -ENOENT;
673a394b 1221
da761a6e
CW		 1222 if (obj->size > dev_priv->mm.gtt_mappable_end) {
1223 drm_gem_object_unreference_unlocked(obj);
1224 return -E2BIG;
1225 }
1226
673a394b
EA		 1227 offset = args->offset;
1228
1229 down_write(&current->mm->mmap_sem);
1230 addr = do_mmap(obj->filp, 0, args->size,
1231 PROT_READ | PROT_WRITE, MAP_SHARED,
1232 args->offset);
1233 up_write(&current->mm->mmap_sem);
bc9025bd 1234 drm_gem_object_unreference_unlocked(obj);
673a394b
EA		 1235 if (IS_ERR((void *)addr))
1236 return addr;
1237
1238 args->addr_ptr = (uint64_t) addr;
1239
1240 return 0;
1241}
1242
de151cf6
JB		 1243/**
1244 * i915_gem_fault - fault a page into the GTT
1245 * vma: VMA in question
1246 * vmf: fault info
1247 *
1248 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1249 * from userspace. The fault handler takes care of binding the object to
1250 * the GTT (if needed), allocating and programming a fence register (again,
1251 * only if needed based on whether the old reg is still valid or the object
1252 * is tiled) and inserting a new PTE into the faulting process.
1253 *
1254 * Note that the faulting process may involve evicting existing objects
1255 * from the GTT and/or fence registers to make room. So performance may
1256 * suffer if the GTT working set is large or there are few fence registers
1257 * left.
1258 */
1259int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1260{
05394f39
CW		 1261 struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1262 struct drm_device *dev = obj->base.dev;
7d1c4804 1263 drm_i915_private_t *dev_priv = dev->dev_private;
de151cf6
JB		 1264 pgoff_t page_offset;
1265 unsigned long pfn;
1266 int ret = 0;
0f973f27 1267 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
de151cf6
JB		 1268
1269 /* We don't use vmf->pgoff since that has the fake offset */
1270 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1271 PAGE_SHIFT;
1272
1273 /* Now bind it into the GTT if needed */
1274 mutex_lock(&dev->struct_mutex);
05394f39 1275 BUG_ON(obj->pin_count && !obj->pin_mappable);
a00b10c3 1276
05394f39
CW		 1277 if (obj->gtt_space) {
1278 if (!obj->map_and_fenceable) {
a00b10c3
CW		 1279 ret = i915_gem_object_unbind(obj);
1280 if (ret)
1281 goto unlock;
1282 }
1283 }
16e809ac 1284
05394f39 1285 if (!obj->gtt_space) {
75e9e915 1286 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
c715089f
CW		 1287 if (ret)
1288 goto unlock;
de151cf6
JB		 1289 }
1290
4a684a41
CW		 1291 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1292 if (ret)
1293 goto unlock;
1294
05394f39
CW		 1295 if (!obj->fault_mappable) {
1296 obj->fault_mappable = true;
1297 i915_gem_info_update_mappable(dev_priv, obj, true);
fb7d516a
DV		 1298 }
1299
de151cf6 1300 /* Need a new fence register? */
05394f39 1301 if (obj->tiling_mode != I915_TILING_NONE) {
2cf34d7b 1302 ret = i915_gem_object_get_fence_reg(obj, true);
c715089f
CW		 1303 if (ret)
1304 goto unlock;
d9ddcb96 1305 }
de151cf6 1306
05394f39
CW		 1307 if (i915_gem_object_is_inactive(obj))
1308 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
7d1c4804 1309
05394f39 1310 pfn = ((dev->agp->base + obj->gtt_offset) >> PAGE_SHIFT) +
de151cf6
JB		 1311 page_offset;
1312
1313 /* Finally, remap it using the new GTT offset */
1314 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
c715089f 1315unlock:
de151cf6
JB		 1316 mutex_unlock(&dev->struct_mutex);
1317
1318 switch (ret) {
045e769a
CW		 1319 case -EAGAIN:
1320 set_need_resched();
c715089f
CW		 1321 case 0:
1322 case -ERESTARTSYS:
1323 return VM_FAULT_NOPAGE;
de151cf6 1324 case -ENOMEM:
de151cf6 1325 return VM_FAULT_OOM;
de151cf6 1326 default:
c715089f 1327 return VM_FAULT_SIGBUS;
de151cf6
JB		 1328 }
1329}
1330
1331/**
1332 * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1333 * @obj: obj in question
1334 *
1335 * GEM memory mapping works by handing back to userspace a fake mmap offset
1336 * it can use in a subsequent mmap(2) call. The DRM core code then looks
1337 * up the object based on the offset and sets up the various memory mapping
1338 * structures.
1339 *
1340 * This routine allocates and attaches a fake offset for @obj.
1341 */
1342static int
05394f39 1343i915_gem_create_mmap_offset(struct drm_i915_gem_object *obj)
de151cf6 1344{
05394f39 1345 struct drm_device *dev = obj->base.dev;
de151cf6 1346 struct drm_gem_mm *mm = dev->mm_private;
de151cf6 1347 struct drm_map_list *list;
f77d390c 1348 struct drm_local_map *map;
de151cf6
JB		 1349 int ret = 0;
1350
1351 /* Set the object up for mmap'ing */
05394f39 1352 list = &obj->base.map_list;
9a298b2a 1353 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
de151cf6
JB		 1354 if (!list->map)
1355 return -ENOMEM;
1356
1357 map = list->map;
1358 map->type = _DRM_GEM;
05394f39 1359 map->size = obj->base.size;
de151cf6
JB		 1360 map->handle = obj;
1361
1362 /* Get a DRM GEM mmap offset allocated... */
1363 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
05394f39
CW		 1364 obj->base.size / PAGE_SIZE,
1365 0, 0);
de151cf6 1366 if (!list->file_offset_node) {
05394f39
CW		 1367 DRM_ERROR("failed to allocate offset for bo %d\n",
1368 obj->base.name);
9e0ae534 1369 ret = -ENOSPC;
de151cf6
JB		 1370 goto out_free_list;
1371 }
1372
1373 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
05394f39
CW		 1374 obj->base.size / PAGE_SIZE,
1375 0);
de151cf6
JB		 1376 if (!list->file_offset_node) {
1377 ret = -ENOMEM;
1378 goto out_free_list;
1379 }
1380
1381 list->hash.key = list->file_offset_node->start;
9e0ae534
CW		 1382 ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1383 if (ret) {
de151cf6
JB		 1384 DRM_ERROR("failed to add to map hash\n");
1385 goto out_free_mm;
1386 }
1387
de151cf6
JB		 1388 return 0;
1389
1390out_free_mm:
1391 drm_mm_put_block(list->file_offset_node);
1392out_free_list:
9a298b2a 1393 kfree(list->map);
39a01d1f 1394 list->map = NULL;
de151cf6
JB		 1395
1396 return ret;
1397}
1398
901782b2
CW		 1399/**
1400 * i915_gem_release_mmap - remove physical page mappings
1401 * @obj: obj in question
1402 *
af901ca1 1403 * Preserve the reservation of the mmapping with the DRM core code, but
901782b2
CW		 1404 * relinquish ownership of the pages back to the system.
1405 *
1406 * It is vital that we remove the page mapping if we have mapped a tiled
1407 * object through the GTT and then lose the fence register due to
1408 * resource pressure. Similarly if the object has been moved out of the
1409 * aperture, than pages mapped into userspace must be revoked. Removing the
1410 * mapping will then trigger a page fault on the next user access, allowing
1411 * fixup by i915_gem_fault().
1412 */
d05ca301 1413void
05394f39 1414i915_gem_release_mmap(struct drm_i915_gem_object *obj)
901782b2 1415{
05394f39 1416 struct drm_device *dev = obj->base.dev;
fb7d516a 1417 struct drm_i915_private *dev_priv = dev->dev_private;
901782b2 1418
05394f39 1419 if (unlikely(obj->base.map_list.map && dev->dev_mapping))
901782b2 1420 unmap_mapping_range(dev->dev_mapping,
05394f39
CW		 1421 (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
1422 obj->base.size, 1);
fb7d516a 1423
05394f39
CW		 1424 if (obj->fault_mappable) {
1425 obj->fault_mappable = false;
1426 i915_gem_info_update_mappable(dev_priv, obj, false);
fb7d516a 1427 }
901782b2
CW		 1428}
1429
ab00b3e5 1430static void
05394f39 1431i915_gem_free_mmap_offset(struct drm_i915_gem_object *obj)
ab00b3e5 1432{
05394f39 1433 struct drm_device *dev = obj->base.dev;
ab00b3e5 1434 struct drm_gem_mm *mm = dev->mm_private;
05394f39 1435 struct drm_map_list *list = &obj->base.map_list;
ab00b3e5 1436
ab00b3e5 1437 drm_ht_remove_item(&mm->offset_hash, &list->hash);
39a01d1f
CW		 1438 drm_mm_put_block(list->file_offset_node);
1439 kfree(list->map);
1440 list->map = NULL;
ab00b3e5
JB		 1441}
1442
92b88aeb
CW		 1443static uint32_t
1444i915_gem_get_gtt_size(struct drm_i915_gem_object *obj)
1445{
1446 struct drm_device *dev = obj->base.dev;
1447 uint32_t size;
1448
1449 if (INTEL_INFO(dev)->gen >= 4 ||
1450 obj->tiling_mode == I915_TILING_NONE)
1451 return obj->base.size;
1452
1453 /* Previous chips need a power-of-two fence region when tiling */
1454 if (INTEL_INFO(dev)->gen == 3)
1455 size = 1024*1024;
1456 else
1457 size = 512*1024;
1458
1459 while (size < obj->base.size)
1460 size <<= 1;
1461
1462 return size;
1463}
1464
de151cf6
JB		 1465/**
1466 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1467 * @obj: object to check
1468 *
1469 * Return the required GTT alignment for an object, taking into account
5e783301 1470 * potential fence register mapping.
de151cf6
JB		 1471 */
1472static uint32_t
05394f39 1473i915_gem_get_gtt_alignment(struct drm_i915_gem_object *obj)
de151cf6 1474{
05394f39 1475 struct drm_device *dev = obj->base.dev;
de151cf6
JB		 1476
1477 /*
1478 * Minimum alignment is 4k (GTT page size), but might be greater
1479 * if a fence register is needed for the object.
1480 */
a00b10c3 1481 if (INTEL_INFO(dev)->gen >= 4 ||
05394f39 1482 obj->tiling_mode == I915_TILING_NONE)
de151cf6
JB		 1483 return 4096;
1484
a00b10c3
CW		 1485 /*
1486 * Previous chips need to be aligned to the size of the smallest
1487 * fence register that can contain the object.
1488 */
05394f39 1489 return i915_gem_get_gtt_size(obj);
a00b10c3
CW		 1490}
1491
5e783301
DV		 1492/**
1493 * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
1494 * unfenced object
1495 * @obj: object to check
1496 *
1497 * Return the required GTT alignment for an object, only taking into account
1498 * unfenced tiled surface requirements.
1499 */
1500static uint32_t
05394f39 1501i915_gem_get_unfenced_gtt_alignment(struct drm_i915_gem_object *obj)
5e783301 1502{
05394f39 1503 struct drm_device *dev = obj->base.dev;
5e783301
DV		 1504 int tile_height;
1505
1506 /*
1507 * Minimum alignment is 4k (GTT page size) for sane hw.
1508 */
1509 if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
05394f39 1510 obj->tiling_mode == I915_TILING_NONE)
5e783301
DV		 1511 return 4096;
1512
1513 /*
1514 * Older chips need unfenced tiled buffers to be aligned to the left
1515 * edge of an even tile row (where tile rows are counted as if the bo is
1516 * placed in a fenced gtt region).
1517 */
1518 if (IS_GEN2(dev) ||
05394f39 1519 (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)))
5e783301
DV		 1520 tile_height = 32;
1521 else
1522 tile_height = 8;
1523
05394f39 1524 return tile_height * obj->stride * 2;
5e783301
DV		 1525}
1526
de151cf6
JB		 1527/**
1528 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1529 * @dev: DRM device
1530 * @data: GTT mapping ioctl data
05394f39 1531 * @file: GEM object info
de151cf6
JB		 1532 *
1533 * Simply returns the fake offset to userspace so it can mmap it.
1534 * The mmap call will end up in drm_gem_mmap(), which will set things
1535 * up so we can get faults in the handler above.
1536 *
1537 * The fault handler will take care of binding the object into the GTT
1538 * (since it may have been evicted to make room for something), allocating
1539 * a fence register, and mapping the appropriate aperture address into
1540 * userspace.
1541 */
1542int
1543i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
05394f39 1544 struct drm_file *file)
de151cf6 1545{
da761a6e 1546 struct drm_i915_private *dev_priv = dev->dev_private;
de151cf6 1547 struct drm_i915_gem_mmap_gtt *args = data;
05394f39 1548 struct drm_i915_gem_object *obj;
de151cf6
JB		 1549 int ret;
1550
1551 if (!(dev->driver->driver_features & DRIVER_GEM))
1552 return -ENODEV;
1553
76c1dec1 1554 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 1555 if (ret)
76c1dec1 1556 return ret;
de151cf6 1557
05394f39 1558 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1d7cfea1
CW		 1559 if (obj == NULL) {
1560 ret = -ENOENT;
1561 goto unlock;
1562 }
de151cf6 1563
05394f39 1564 if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
da761a6e
CW		 1565 ret = -E2BIG;
1566 goto unlock;
1567 }
1568
05394f39 1569 if (obj->madv != I915_MADV_WILLNEED) {
ab18282d 1570 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1d7cfea1
CW		 1571 ret = -EINVAL;
1572 goto out;
ab18282d
CW		 1573 }
1574
05394f39 1575 if (!obj->base.map_list.map) {
de151cf6 1576 ret = i915_gem_create_mmap_offset(obj);
1d7cfea1
CW		 1577 if (ret)
1578 goto out;
de151cf6
JB		 1579 }
1580
05394f39 1581 args->offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
de151cf6 1582
1d7cfea1 1583out:
05394f39 1584 drm_gem_object_unreference(&obj->base);
1d7cfea1 1585unlock:
de151cf6 1586 mutex_unlock(&dev->struct_mutex);
1d7cfea1 1587 return ret;
de151cf6
JB		 1588}
1589
e5281ccd 1590static int
05394f39 1591i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
e5281ccd
CW		 1592 gfp_t gfpmask)
1593{
e5281ccd
CW		 1594 int page_count, i;
1595 struct address_space *mapping;
1596 struct inode *inode;
1597 struct page *page;
1598
1599 /* Get the list of pages out of our struct file. They'll be pinned
1600 * at this point until we release them.
1601 */
05394f39
CW		 1602 page_count = obj->base.size / PAGE_SIZE;
1603 BUG_ON(obj->pages != NULL);
1604 obj->pages = drm_malloc_ab(page_count, sizeof(struct page *));
1605 if (obj->pages == NULL)
e5281ccd
CW		 1606 return -ENOMEM;
1607
05394f39 1608 inode = obj->base.filp->f_path.dentry->d_inode;
e5281ccd
CW		 1609 mapping = inode->i_mapping;
1610 for (i = 0; i < page_count; i++) {
1611 page = read_cache_page_gfp(mapping, i,
1612 GFP_HIGHUSER |
1613 __GFP_COLD |
1614 __GFP_RECLAIMABLE |
1615 gfpmask);
1616 if (IS_ERR(page))
1617 goto err_pages;
1618
05394f39 1619 obj->pages[i] = page;
e5281ccd
CW		 1620 }
1621
05394f39 1622 if (obj->tiling_mode != I915_TILING_NONE)
e5281ccd
CW		 1623 i915_gem_object_do_bit_17_swizzle(obj);
1624
1625 return 0;
1626
1627err_pages:
1628 while (i--)
05394f39 1629 page_cache_release(obj->pages[i]);
e5281ccd 1630
05394f39
CW		 1631 drm_free_large(obj->pages);
1632 obj->pages = NULL;
e5281ccd
CW		 1633 return PTR_ERR(page);
1634}
1635
5cdf5881 1636static void
05394f39 1637i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
673a394b 1638{
05394f39 1639 int page_count = obj->base.size / PAGE_SIZE;
673a394b
EA		 1640 int i;
1641
05394f39 1642 BUG_ON(obj->madv == __I915_MADV_PURGED);
673a394b 1643
05394f39 1644 if (obj->tiling_mode != I915_TILING_NONE)
280b713b
EA		 1645 i915_gem_object_save_bit_17_swizzle(obj);
1646
05394f39
CW		 1647 if (obj->madv == I915_MADV_DONTNEED)
1648 obj->dirty = 0;
3ef94daa
CW		 1649
1650 for (i = 0; i < page_count; i++) {
05394f39
CW		 1651 if (obj->dirty)
1652 set_page_dirty(obj->pages[i]);
3ef94daa 1653
05394f39
CW		 1654 if (obj->madv == I915_MADV_WILLNEED)
1655 mark_page_accessed(obj->pages[i]);
3ef94daa 1656
05394f39 1657 page_cache_release(obj->pages[i]);
3ef94daa 1658 }
05394f39 1659 obj->dirty = 0;
673a394b 1660
05394f39
CW		 1661 drm_free_large(obj->pages);
1662 obj->pages = NULL;
673a394b
EA		 1663}
1664
a56ba56c
CW		 1665static uint32_t
1666i915_gem_next_request_seqno(struct drm_device *dev,
1667 struct intel_ring_buffer *ring)
1668{
1669 drm_i915_private_t *dev_priv = dev->dev_private;
5d97eb69 1670 return ring->outstanding_lazy_request = dev_priv->next_seqno;
a56ba56c
CW		 1671}
1672
673a394b 1673static void
05394f39 1674i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
852835f3 1675 struct intel_ring_buffer *ring)
673a394b 1676{
05394f39 1677 struct drm_device *dev = obj->base.dev;
69dc4987 1678 struct drm_i915_private *dev_priv = dev->dev_private;
a56ba56c 1679 uint32_t seqno = i915_gem_next_request_seqno(dev, ring);
617dbe27 1680
852835f3 1681 BUG_ON(ring == NULL);
05394f39 1682 obj->ring = ring;
673a394b
EA		 1683
1684 /* Add a reference if we're newly entering the active list. */
05394f39
CW		 1685 if (!obj->active) {
1686 drm_gem_object_reference(&obj->base);
1687 obj->active = 1;
673a394b 1688 }
e35a41de 1689
673a394b 1690 /* Move from whatever list we were on to the tail of execution. */
05394f39
CW		 1691 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
1692 list_move_tail(&obj->ring_list, &ring->active_list);
1693 obj->last_rendering_seqno = seqno;
673a394b
EA		 1694}
1695
ce44b0ea 1696static void
05394f39 1697i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
ce44b0ea 1698{
05394f39 1699 struct drm_device *dev = obj->base.dev;
ce44b0ea 1700 drm_i915_private_t *dev_priv = dev->dev_private;
ce44b0ea 1701
05394f39
CW		 1702 BUG_ON(!obj->active);
1703 list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
1704 list_del_init(&obj->ring_list);
1705 obj->last_rendering_seqno = 0;
ce44b0ea 1706}
673a394b 1707
963b4836
CW		 1708/* Immediately discard the backing storage */
1709static void
05394f39 1710i915_gem_object_truncate(struct drm_i915_gem_object *obj)
963b4836 1711{
bb6baf76 1712 struct inode *inode;
963b4836 1713
ae9fed6b
CW		 1714 /* Our goal here is to return as much of the memory as
1715 * is possible back to the system as we are called from OOM.
1716 * To do this we must instruct the shmfs to drop all of its
1717 * backing pages, *now*. Here we mirror the actions taken
1718 * when by shmem_delete_inode() to release the backing store.
1719 */
05394f39 1720 inode = obj->base.filp->f_path.dentry->d_inode;
ae9fed6b
CW		 1721 truncate_inode_pages(inode->i_mapping, 0);
1722 if (inode->i_op->truncate_range)
1723 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
bb6baf76 1724
05394f39 1725 obj->madv = __I915_MADV_PURGED;
963b4836
CW		 1726}
1727
1728static inline int
05394f39 1729i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
963b4836 1730{
05394f39 1731 return obj->madv == I915_MADV_DONTNEED;
963b4836
CW		 1732}
1733
673a394b 1734static void
05394f39 1735i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
673a394b 1736{
05394f39 1737 struct drm_device *dev = obj->base.dev;
673a394b 1738 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 1739
05394f39
CW		 1740 if (obj->pin_count != 0)
1741 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
673a394b 1742 else
05394f39
CW		 1743 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1744 list_del_init(&obj->ring_list);
673a394b 1745
05394f39 1746 BUG_ON(!list_empty(&obj->gpu_write_list));
99fcb766 1747
05394f39
CW		 1748 obj->last_rendering_seqno = 0;
1749 obj->ring = NULL;
1750 if (obj->active) {
1751 obj->active = 0;
1752 drm_gem_object_unreference(&obj->base);
673a394b 1753 }
23bc5982 1754 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 1755}
1756
63560396
DV		 1757static void
1758i915_gem_process_flushing_list(struct drm_device *dev,
8a1a49f9 1759 uint32_t flush_domains,
852835f3 1760 struct intel_ring_buffer *ring)
63560396
DV		 1761{
1762 drm_i915_private_t *dev_priv = dev->dev_private;
05394f39 1763 struct drm_i915_gem_object *obj, *next;
63560396 1764
05394f39 1765 list_for_each_entry_safe(obj, next,
64193406 1766 &ring->gpu_write_list,
63560396 1767 gpu_write_list) {
05394f39
CW		 1768 if (obj->base.write_domain & flush_domains) {
1769 uint32_t old_write_domain = obj->base.write_domain;
63560396 1770
05394f39
CW		 1771 obj->base.write_domain = 0;
1772 list_del_init(&obj->gpu_write_list);
617dbe27 1773 i915_gem_object_move_to_active(obj, ring);
63560396
DV		 1774
1775 /* update the fence lru list */
05394f39 1776 if (obj->fence_reg != I915_FENCE_REG_NONE) {
007cc8ac 1777 struct drm_i915_fence_reg *reg =
05394f39 1778 &dev_priv->fence_regs[obj->fence_reg];
007cc8ac 1779 list_move_tail(&reg->lru_list,
63560396 1780 &dev_priv->mm.fence_list);
007cc8ac 1781 }
63560396
DV		 1782
1783 trace_i915_gem_object_change_domain(obj,
05394f39 1784 obj->base.read_domains,
63560396
DV		 1785 old_write_domain);
1786 }
1787 }
1788}
8187a2b7 1789
3cce469c 1790int
8a1a49f9 1791i915_add_request(struct drm_device *dev,
f787a5f5 1792 struct drm_file *file,
8dc5d147 1793 struct drm_i915_gem_request *request,
8a1a49f9 1794 struct intel_ring_buffer *ring)
673a394b
EA		 1795{
1796 drm_i915_private_t *dev_priv = dev->dev_private;
f787a5f5 1797 struct drm_i915_file_private *file_priv = NULL;
673a394b
EA		 1798 uint32_t seqno;
1799 int was_empty;
3cce469c
CW		 1800 int ret;
1801
1802 BUG_ON(request == NULL);
673a394b 1803
f787a5f5
CW		 1804 if (file != NULL)
1805 file_priv = file->driver_priv;
b962442e 1806
3cce469c
CW		 1807 ret = ring->add_request(ring, &seqno);
1808 if (ret)
1809 return ret;
673a394b 1810
a56ba56c 1811 ring->outstanding_lazy_request = false;
673a394b
EA		 1812
1813 request->seqno = seqno;
852835f3 1814 request->ring = ring;
673a394b 1815 request->emitted_jiffies = jiffies;
852835f3
ZN		 1816 was_empty = list_empty(&ring->request_list);
1817 list_add_tail(&request->list, &ring->request_list);
1818
f787a5f5 1819 if (file_priv) {
1c25595f 1820 spin_lock(&file_priv->mm.lock);
f787a5f5 1821 request->file_priv = file_priv;
b962442e 1822 list_add_tail(&request->client_list,
f787a5f5 1823 &file_priv->mm.request_list);
1c25595f 1824 spin_unlock(&file_priv->mm.lock);
b962442e 1825 }
673a394b 1826
f65d9421 1827 if (!dev_priv->mm.suspended) {
b3b079db
CW		 1828 mod_timer(&dev_priv->hangcheck_timer,
1829 jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
f65d9421 1830 if (was_empty)
b3b079db
CW		 1831 queue_delayed_work(dev_priv->wq,
1832 &dev_priv->mm.retire_work, HZ);
f65d9421 1833 }
3cce469c 1834 return 0;
673a394b
EA		 1835}
1836
1837/**
1838 * Command execution barrier
1839 *
1840 * Ensures that all commands in the ring are finished
1841 * before signalling the CPU
1842 */
8a1a49f9 1843static void
852835f3 1844i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
673a394b 1845{
673a394b 1846 uint32_t flush_domains = 0;
673a394b
EA		 1847
1848 /* The sampler always gets flushed on i965 (sigh) */
a6c45cf0 1849 if (INTEL_INFO(dev)->gen >= 4)
673a394b 1850 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
852835f3 1851
78501eac 1852 ring->flush(ring, I915_GEM_DOMAIN_COMMAND, flush_domains);
673a394b
EA		 1853}
1854
f787a5f5
CW		 1855static inline void
1856i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
673a394b 1857{
1c25595f 1858 struct drm_i915_file_private *file_priv = request->file_priv;
673a394b 1859
1c25595f
CW		 1860 if (!file_priv)
1861 return;
1c5d22f7 1862
1c25595f
CW		 1863 spin_lock(&file_priv->mm.lock);
1864 list_del(&request->client_list);
1865 request->file_priv = NULL;
1866 spin_unlock(&file_priv->mm.lock);
673a394b 1867}
673a394b 1868
dfaae392
CW		 1869static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1870 struct intel_ring_buffer *ring)
9375e446 1871{
dfaae392
CW		 1872 while (!list_empty(&ring->request_list)) {
1873 struct drm_i915_gem_request *request;
673a394b 1874
dfaae392
CW		 1875 request = list_first_entry(&ring->request_list,
1876 struct drm_i915_gem_request,
1877 list);
de151cf6 1878
dfaae392 1879 list_del(&request->list);
f787a5f5 1880 i915_gem_request_remove_from_client(request);
dfaae392
CW		 1881 kfree(request);
1882 }
673a394b 1883
dfaae392 1884 while (!list_empty(&ring->active_list)) {
05394f39 1885 struct drm_i915_gem_object *obj;
9375e446 1886
05394f39
CW		 1887 obj = list_first_entry(&ring->active_list,
1888 struct drm_i915_gem_object,
1889 ring_list);
9375e446 1890
05394f39
CW		 1891 obj->base.write_domain = 0;
1892 list_del_init(&obj->gpu_write_list);
1893 i915_gem_object_move_to_inactive(obj);
673a394b
EA		 1894 }
1895}
1896
069efc1d 1897void i915_gem_reset(struct drm_device *dev)
673a394b 1898{
77f01230 1899 struct drm_i915_private *dev_priv = dev->dev_private;
05394f39 1900 struct drm_i915_gem_object *obj;
069efc1d 1901 int i;
673a394b 1902
dfaae392 1903 i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring);
87acb0a5 1904 i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring);
549f7365 1905 i915_gem_reset_ring_lists(dev_priv, &dev_priv->blt_ring);
dfaae392
CW		 1906
1907 /* Remove anything from the flushing lists. The GPU cache is likely
1908 * to be lost on reset along with the data, so simply move the
1909 * lost bo to the inactive list.
1910 */
1911 while (!list_empty(&dev_priv->mm.flushing_list)) {
05394f39
CW		 1912 obj= list_first_entry(&dev_priv->mm.flushing_list,
1913 struct drm_i915_gem_object,
1914 mm_list);
dfaae392 1915
05394f39
CW		 1916 obj->base.write_domain = 0;
1917 list_del_init(&obj->gpu_write_list);
1918 i915_gem_object_move_to_inactive(obj);
dfaae392
CW		 1919 }
1920
1921 /* Move everything out of the GPU domains to ensure we do any
1922 * necessary invalidation upon reuse.
1923 */
05394f39 1924 list_for_each_entry(obj,
77f01230 1925 &dev_priv->mm.inactive_list,
69dc4987 1926 mm_list)
77f01230 1927 {
05394f39 1928 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
77f01230 1929 }
069efc1d
CW		 1930
1931 /* The fence registers are invalidated so clear them out */
1932 for (i = 0; i < 16; i++) {
1933 struct drm_i915_fence_reg *reg;
1934
1935 reg = &dev_priv->fence_regs[i];
1936 if (!reg->obj)
1937 continue;
1938
1939 i915_gem_clear_fence_reg(reg->obj);
1940 }
673a394b
EA		 1941}
1942
1943/**
1944 * This function clears the request list as sequence numbers are passed.
1945 */
b09a1fec
CW		 1946static void
1947i915_gem_retire_requests_ring(struct drm_device *dev,
1948 struct intel_ring_buffer *ring)
673a394b
EA		 1949{
1950 drm_i915_private_t *dev_priv = dev->dev_private;
1951 uint32_t seqno;
1952
b84d5f0c
CW		 1953 if (!ring->status_page.page_addr ||
1954 list_empty(&ring->request_list))
6c0594a3
KW		 1955 return;
1956
23bc5982 1957 WARN_ON(i915_verify_lists(dev));
673a394b 1958
78501eac 1959 seqno = ring->get_seqno(ring);
852835f3 1960 while (!list_empty(&ring->request_list)) {
673a394b 1961 struct drm_i915_gem_request *request;
673a394b 1962
852835f3 1963 request = list_first_entry(&ring->request_list,
673a394b
EA		 1964 struct drm_i915_gem_request,
1965 list);
673a394b 1966
dfaae392 1967 if (!i915_seqno_passed(seqno, request->seqno))
b84d5f0c
CW		 1968 break;
1969
1970 trace_i915_gem_request_retire(dev, request->seqno);
1971
1972 list_del(&request->list);
f787a5f5 1973 i915_gem_request_remove_from_client(request);
b84d5f0c
CW		 1974 kfree(request);
1975 }
673a394b 1976
b84d5f0c
CW		 1977 /* Move any buffers on the active list that are no longer referenced
1978 * by the ringbuffer to the flushing/inactive lists as appropriate.
1979 */
1980 while (!list_empty(&ring->active_list)) {
05394f39 1981 struct drm_i915_gem_object *obj;
b84d5f0c 1982
05394f39
CW		 1983 obj= list_first_entry(&ring->active_list,
1984 struct drm_i915_gem_object,
1985 ring_list);
673a394b 1986
05394f39 1987 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
673a394b 1988 break;
b84d5f0c 1989
05394f39 1990 if (obj->base.write_domain != 0)
b84d5f0c
CW		 1991 i915_gem_object_move_to_flushing(obj);
1992 else
1993 i915_gem_object_move_to_inactive(obj);
673a394b 1994 }
9d34e5db
CW		 1995
1996 if (unlikely (dev_priv->trace_irq_seqno &&
1997 i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
78501eac 1998 ring->user_irq_put(ring);
9d34e5db
CW		 1999 dev_priv->trace_irq_seqno = 0;
2000 }
23bc5982
CW		 2001
2002 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 2003}
2004
b09a1fec
CW		 2005void
2006i915_gem_retire_requests(struct drm_device *dev)
2007{
2008 drm_i915_private_t *dev_priv = dev->dev_private;
2009
be72615b 2010 if (!list_empty(&dev_priv->mm.deferred_free_list)) {
05394f39 2011 struct drm_i915_gem_object *obj, *next;
be72615b
CW		 2012
2013 /* We must be careful that during unbind() we do not
2014 * accidentally infinitely recurse into retire requests.
2015 * Currently:
2016 * retire -> free -> unbind -> wait -> retire_ring
2017 */
05394f39 2018 list_for_each_entry_safe(obj, next,
be72615b 2019 &dev_priv->mm.deferred_free_list,
69dc4987 2020 mm_list)
05394f39 2021 i915_gem_free_object_tail(obj);
be72615b
CW		 2022 }
2023
b09a1fec 2024 i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
87acb0a5 2025 i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
549f7365 2026 i915_gem_retire_requests_ring(dev, &dev_priv->blt_ring);
b09a1fec
CW		 2027}
2028
75ef9da2 2029static void
673a394b
EA		 2030i915_gem_retire_work_handler(struct work_struct *work)
2031{
2032 drm_i915_private_t *dev_priv;
2033 struct drm_device *dev;
2034
2035 dev_priv = container_of(work, drm_i915_private_t,
2036 mm.retire_work.work);
2037 dev = dev_priv->dev;
2038
891b48cf
CW		 2039 /* Come back later if the device is busy... */
2040 if (!mutex_trylock(&dev->struct_mutex)) {
2041 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
2042 return;
2043 }
2044
b09a1fec 2045 i915_gem_retire_requests(dev);
d1b851fc 2046
6dbe2772 2047 if (!dev_priv->mm.suspended &&
d1b851fc 2048 (!list_empty(&dev_priv->render_ring.request_list) ||
549f7365
CW		 2049 !list_empty(&dev_priv->bsd_ring.request_list) ||
2050 !list_empty(&dev_priv->blt_ring.request_list)))
9c9fe1f8 2051 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
673a394b
EA		 2052 mutex_unlock(&dev->struct_mutex);
2053}
2054
5a5a0c64 2055int
852835f3 2056i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
8a1a49f9 2057 bool interruptible, struct intel_ring_buffer *ring)
673a394b
EA		 2058{
2059 drm_i915_private_t *dev_priv = dev->dev_private;
802c7eb6 2060 u32 ier;
673a394b
EA		 2061 int ret = 0;
2062
2063 BUG_ON(seqno == 0);
2064
ba1234d1 2065 if (atomic_read(&dev_priv->mm.wedged))
30dbf0c0
CW		 2066 return -EAGAIN;
2067
5d97eb69 2068 if (seqno == ring->outstanding_lazy_request) {
3cce469c
CW		 2069 struct drm_i915_gem_request *request;
2070
2071 request = kzalloc(sizeof(*request), GFP_KERNEL);
2072 if (request == NULL)
e35a41de 2073 return -ENOMEM;
3cce469c
CW		 2074
2075 ret = i915_add_request(dev, NULL, request, ring);
2076 if (ret) {
2077 kfree(request);
2078 return ret;
2079 }
2080
2081 seqno = request->seqno;
e35a41de 2082 }
ffed1d09 2083
78501eac 2084 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
bad720ff 2085 if (HAS_PCH_SPLIT(dev))
036a4a7d
ZW		 2086 ier = I915_READ(DEIER) | I915_READ(GTIER);
2087 else
2088 ier = I915_READ(IER);
802c7eb6
JB		 2089 if (!ier) {
2090 DRM_ERROR("something (likely vbetool) disabled "
2091 "interrupts, re-enabling\n");
2092 i915_driver_irq_preinstall(dev);
2093 i915_driver_irq_postinstall(dev);
2094 }
2095
1c5d22f7
CW		 2096 trace_i915_gem_request_wait_begin(dev, seqno);
2097
b2223497 2098 ring->waiting_seqno = seqno;
78501eac 2099 ring->user_irq_get(ring);
48764bf4 2100 if (interruptible)
852835f3 2101 ret = wait_event_interruptible(ring->irq_queue,
78501eac 2102 i915_seqno_passed(ring->get_seqno(ring), seqno)
852835f3 2103 || atomic_read(&dev_priv->mm.wedged));
48764bf4 2104 else
852835f3 2105 wait_event(ring->irq_queue,
78501eac 2106 i915_seqno_passed(ring->get_seqno(ring), seqno)
852835f3 2107 || atomic_read(&dev_priv->mm.wedged));
48764bf4 2108
78501eac 2109 ring->user_irq_put(ring);
b2223497 2110 ring->waiting_seqno = 0;
1c5d22f7
CW		 2111
2112 trace_i915_gem_request_wait_end(dev, seqno);
673a394b 2113 }
ba1234d1 2114 if (atomic_read(&dev_priv->mm.wedged))
30dbf0c0 2115 ret = -EAGAIN;
673a394b
EA		 2116
2117 if (ret && ret != -ERESTARTSYS)
8bff917c 2118 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
78501eac 2119 __func__, ret, seqno, ring->get_seqno(ring),
8bff917c 2120 dev_priv->next_seqno);
673a394b
EA		 2121
2122 /* Directly dispatch request retiring. While we have the work queue
2123 * to handle this, the waiter on a request often wants an associated
2124 * buffer to have made it to the inactive list, and we would need
2125 * a separate wait queue to handle that.
2126 */
2127 if (ret == 0)
b09a1fec 2128 i915_gem_retire_requests_ring(dev, ring);
673a394b
EA		 2129
2130 return ret;
2131}
2132
48764bf4
DV		 2133/**
2134 * Waits for a sequence number to be signaled, and cleans up the
2135 * request and object lists appropriately for that event.
2136 */
2137static int
852835f3 2138i915_wait_request(struct drm_device *dev, uint32_t seqno,
a56ba56c 2139 struct intel_ring_buffer *ring)
48764bf4 2140{
852835f3 2141 return i915_do_wait_request(dev, seqno, 1, ring);
48764bf4
DV		 2142}
2143
20f0cd55 2144static void
9220434a
CW		 2145i915_gem_flush_ring(struct drm_device *dev,
2146 struct intel_ring_buffer *ring,
2147 uint32_t invalidate_domains,
2148 uint32_t flush_domains)
2149{
78501eac 2150 ring->flush(ring, invalidate_domains, flush_domains);
9220434a
CW		 2151 i915_gem_process_flushing_list(dev, flush_domains, ring);
2152}
2153
8187a2b7
ZN		 2154static void
2155i915_gem_flush(struct drm_device *dev,
2156 uint32_t invalidate_domains,
9220434a
CW		 2157 uint32_t flush_domains,
2158 uint32_t flush_rings)
8187a2b7
ZN		 2159{
2160 drm_i915_private_t *dev_priv = dev->dev_private;
8bff917c 2161
8187a2b7 2162 if (flush_domains & I915_GEM_DOMAIN_CPU)
40ce6575 2163 intel_gtt_chipset_flush();
8bff917c 2164
9220434a
CW		 2165 if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
2166 if (flush_rings & RING_RENDER)
05394f39 2167 i915_gem_flush_ring(dev, &dev_priv->render_ring,
9220434a
CW		 2168 invalidate_domains, flush_domains);
2169 if (flush_rings & RING_BSD)
05394f39 2170 i915_gem_flush_ring(dev, &dev_priv->bsd_ring,
9220434a 2171 invalidate_domains, flush_domains);
549f7365 2172 if (flush_rings & RING_BLT)
05394f39 2173 i915_gem_flush_ring(dev, &dev_priv->blt_ring,
549f7365 2174 invalidate_domains, flush_domains);
9220434a 2175 }
8187a2b7
ZN		 2176}
2177
673a394b
EA		 2178/**
2179 * Ensures that all rendering to the object has completed and the object is
2180 * safe to unbind from the GTT or access from the CPU.
2181 */
2182static int
05394f39 2183i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
2cf34d7b 2184 bool interruptible)
673a394b 2185{
05394f39 2186 struct drm_device *dev = obj->base.dev;
673a394b
EA		 2187 int ret;
2188
e47c68e9
EA		 2189 /* This function only exists to support waiting for existing rendering,
2190 * not for emitting required flushes.
673a394b 2191 */
05394f39 2192 BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
673a394b
EA		 2193
2194 /* If there is rendering queued on the buffer being evicted, wait for
2195 * it.
2196 */
05394f39 2197 if (obj->active) {
2cf34d7b 2198 ret = i915_do_wait_request(dev,
05394f39 2199 obj->last_rendering_seqno,
2cf34d7b 2200 interruptible,
05394f39 2201 obj->ring);
2cf34d7b 2202 if (ret)
673a394b
EA		 2203 return ret;
2204 }
2205
2206 return 0;
2207}
2208
2209/**
2210 * Unbinds an object from the GTT aperture.
2211 */
0f973f27 2212int
05394f39 2213i915_gem_object_unbind(struct drm_i915_gem_object *obj)
673a394b 2214{
05394f39 2215 struct drm_device *dev = obj->base.dev;
73aa808f 2216 struct drm_i915_private *dev_priv = dev->dev_private;
673a394b
EA		 2217 int ret = 0;
2218
05394f39 2219 if (obj->gtt_space == NULL)
673a394b
EA		 2220 return 0;
2221
05394f39 2222 if (obj->pin_count != 0) {
673a394b
EA		 2223 DRM_ERROR("Attempting to unbind pinned buffer\n");
2224 return -EINVAL;
2225 }
2226
5323fd04
EA		 2227 /* blow away mappings if mapped through GTT */
2228 i915_gem_release_mmap(obj);
2229
673a394b
EA		 2230 /* Move the object to the CPU domain to ensure that
2231 * any possible CPU writes while it's not in the GTT
2232 * are flushed when we go to remap it. This will
2233 * also ensure that all pending GPU writes are finished
2234 * before we unbind.
2235 */
e47c68e9 2236 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
8dc1775d 2237 if (ret == -ERESTARTSYS)
673a394b 2238 return ret;
8dc1775d
CW		 2239 /* Continue on if we fail due to EIO, the GPU is hung so we
2240 * should be safe and we need to cleanup or else we might
2241 * cause memory corruption through use-after-free.
2242 */
812ed492
CW		 2243 if (ret) {
2244 i915_gem_clflush_object(obj);
05394f39 2245 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
812ed492 2246 }
673a394b 2247
96b47b65 2248 /* release the fence reg _after_ flushing */
05394f39 2249 if (obj->fence_reg != I915_FENCE_REG_NONE)
96b47b65
DV		 2250 i915_gem_clear_fence_reg(obj);
2251
7c2e6fdf 2252 i915_gem_gtt_unbind_object(obj);
673a394b 2253
e5281ccd 2254 i915_gem_object_put_pages_gtt(obj);
673a394b 2255
05394f39
CW		 2256 i915_gem_info_remove_gtt(dev_priv, obj);
2257 list_del_init(&obj->mm_list);
75e9e915 2258 /* Avoid an unnecessary call to unbind on rebind. */
05394f39 2259 obj->map_and_fenceable = true;
673a394b 2260
05394f39
CW		 2261 drm_mm_put_block(obj->gtt_space);
2262 obj->gtt_space = NULL;
2263 obj->gtt_offset = 0;
673a394b 2264
05394f39 2265 if (i915_gem_object_is_purgeable(obj))
963b4836
CW		 2266 i915_gem_object_truncate(obj);
2267
1c5d22f7
CW		 2268 trace_i915_gem_object_unbind(obj);
2269
8dc1775d 2270 return ret;
673a394b
EA		 2271}
2272
a56ba56c
CW		 2273static int i915_ring_idle(struct drm_device *dev,
2274 struct intel_ring_buffer *ring)
2275{
395b70be 2276 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
64193406
CW		 2277 return 0;
2278
05394f39 2279 i915_gem_flush_ring(dev, ring,
a56ba56c
CW		 2280 I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2281 return i915_wait_request(dev,
2282 i915_gem_next_request_seqno(dev, ring),
2283 ring);
2284}
2285
b47eb4a2 2286int
4df2faf4
DV		 2287i915_gpu_idle(struct drm_device *dev)
2288{
2289 drm_i915_private_t *dev_priv = dev->dev_private;
2290 bool lists_empty;
852835f3 2291 int ret;
4df2faf4 2292
d1b851fc 2293 lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
395b70be 2294 list_empty(&dev_priv->mm.active_list));
4df2faf4
DV		 2295 if (lists_empty)
2296 return 0;
2297
2298 /* Flush everything onto the inactive list. */
a56ba56c 2299 ret = i915_ring_idle(dev, &dev_priv->render_ring);
8a1a49f9
DV		 2300 if (ret)
2301 return ret;
d1b851fc 2302
87acb0a5
CW		 2303 ret = i915_ring_idle(dev, &dev_priv->bsd_ring);
2304 if (ret)
2305 return ret;
d1b851fc 2306
549f7365
CW		 2307 ret = i915_ring_idle(dev, &dev_priv->blt_ring);
2308 if (ret)
2309 return ret;
4df2faf4 2310
8a1a49f9 2311 return 0;
4df2faf4
DV		 2312}
2313
05394f39 2314static void sandybridge_write_fence_reg(struct drm_i915_gem_object *obj)
4e901fdc 2315{
05394f39 2316 struct drm_device *dev = obj->base.dev;
4e901fdc 2317 drm_i915_private_t *dev_priv = dev->dev_private;
05394f39
CW		 2318 u32 size = obj->gtt_space->size;
2319 int regnum = obj->fence_reg;
4e901fdc
EA		 2320 uint64_t val;
2321
05394f39 2322 val = (uint64_t)((obj->gtt_offset + size - 4096) &
4e901fdc 2323 0xfffff000) << 32;
05394f39
CW		 2324 val |= obj->gtt_offset & 0xfffff000;
2325 val |= (uint64_t)((obj->stride / 128) - 1) <<
4e901fdc
EA		 2326 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2327
05394f39 2328 if (obj->tiling_mode == I915_TILING_Y)
4e901fdc
EA		 2329 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2330 val |= I965_FENCE_REG_VALID;
2331
2332 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2333}
2334
05394f39 2335static void i965_write_fence_reg(struct drm_i915_gem_object *obj)
de151cf6 2336{
05394f39 2337 struct drm_device *dev = obj->base.dev;
de151cf6 2338 drm_i915_private_t *dev_priv = dev->dev_private;
05394f39
CW		 2339 u32 size = obj->gtt_space->size;
2340 int regnum = obj->fence_reg;
de151cf6
JB		 2341 uint64_t val;
2342
05394f39 2343 val = (uint64_t)((obj->gtt_offset + size - 4096) &
de151cf6 2344 0xfffff000) << 32;
05394f39
CW		 2345 val |= obj->gtt_offset & 0xfffff000;
2346 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2347 if (obj->tiling_mode == I915_TILING_Y)
de151cf6
JB		 2348 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2349 val |= I965_FENCE_REG_VALID;
2350
2351 I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2352}
2353
05394f39 2354static void i915_write_fence_reg(struct drm_i915_gem_object *obj)
de151cf6 2355{
05394f39 2356 struct drm_device *dev = obj->base.dev;
de151cf6 2357 drm_i915_private_t *dev_priv = dev->dev_private;
05394f39 2358 u32 size = obj->gtt_space->size;
a00b10c3 2359 uint32_t fence_reg, val, pitch_val;
0f973f27 2360 int tile_width;
de151cf6 2361
05394f39
CW		 2362 if ((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
2363 (obj->gtt_offset & (size - 1))) {
a00b10c3 2364 WARN(1, "%s: object 0x%08x [fenceable? %d] not 1M or size (0x%08x) aligned [gtt_space offset=%lx, size=%lx]\n",
05394f39
CW		 2365 __func__, obj->gtt_offset, obj->map_and_fenceable, size,
2366 obj->gtt_space->start, obj->gtt_space->size);
de151cf6
JB		 2367 return;
2368 }
2369
05394f39 2370 if (obj->tiling_mode == I915_TILING_Y &&
0f973f27
JB		 2371 HAS_128_BYTE_Y_TILING(dev))
2372 tile_width = 128;
de151cf6 2373 else
0f973f27
JB		 2374 tile_width = 512;
2375
2376 /* Note: pitch better be a power of two tile widths */
05394f39 2377 pitch_val = obj->stride / tile_width;
0f973f27 2378 pitch_val = ffs(pitch_val) - 1;
de151cf6 2379
05394f39 2380 if (obj->tiling_mode == I915_TILING_Y &&
c36a2a6d
DV		 2381 HAS_128_BYTE_Y_TILING(dev))
2382 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2383 else
2384 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2385
05394f39
CW		 2386 val = obj->gtt_offset;
2387 if (obj->tiling_mode == I915_TILING_Y)
de151cf6 2388 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
a00b10c3 2389 val |= I915_FENCE_SIZE_BITS(size);
de151cf6
JB		 2390 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2391 val |= I830_FENCE_REG_VALID;
2392
05394f39 2393 fence_reg = obj->fence_reg;
a00b10c3
CW		 2394 if (fence_reg < 8)
2395 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
dc529a4f 2396 else
a00b10c3 2397 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
dc529a4f 2398 I915_WRITE(fence_reg, val);
de151cf6
JB		 2399}
2400
05394f39 2401static void i830_write_fence_reg(struct drm_i915_gem_object *obj)
de151cf6 2402{
05394f39 2403 struct drm_device *dev = obj->base.dev;
de151cf6 2404 drm_i915_private_t *dev_priv = dev->dev_private;
05394f39
CW		 2405 u32 size = obj->gtt_space->size;
2406 int regnum = obj->fence_reg;
de151cf6
JB		 2407 uint32_t val;
2408 uint32_t pitch_val;
8d7773a3 2409 uint32_t fence_size_bits;
de151cf6 2410
05394f39
CW		 2411 if ((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
2412 (obj->gtt_offset & (obj->base.size - 1))) {
8d7773a3 2413 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
05394f39 2414 __func__, obj->gtt_offset);
de151cf6
JB		 2415 return;
2416 }
2417
05394f39 2418 pitch_val = obj->stride / 128;
e76a16de
EA		 2419 pitch_val = ffs(pitch_val) - 1;
2420 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2421
05394f39
CW		 2422 val = obj->gtt_offset;
2423 if (obj->tiling_mode == I915_TILING_Y)
de151cf6 2424 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
a00b10c3 2425 fence_size_bits = I830_FENCE_SIZE_BITS(size);
8d7773a3
DV		 2426 WARN_ON(fence_size_bits & ~0x00000f00);
2427 val |= fence_size_bits;
de151cf6
JB		 2428 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2429 val |= I830_FENCE_REG_VALID;
2430
2431 I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
de151cf6
JB		 2432}
2433
2cf34d7b
CW		 2434static int i915_find_fence_reg(struct drm_device *dev,
2435 bool interruptible)
ae3db24a 2436{
ae3db24a 2437 struct drm_i915_private *dev_priv = dev->dev_private;
a00b10c3 2438 struct drm_i915_fence_reg *reg;
05394f39 2439 struct drm_i915_gem_object *obj = NULL;
ae3db24a
DV		 2440 int i, avail, ret;
2441
2442 /* First try to find a free reg */
2443 avail = 0;
2444 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2445 reg = &dev_priv->fence_regs[i];
2446 if (!reg->obj)
2447 return i;
2448
05394f39
CW		 2449 if (!reg->obj->pin_count)
2450 avail++;
ae3db24a
DV		 2451 }
2452
2453 if (avail == 0)
2454 return -ENOSPC;
2455
2456 /* None available, try to steal one or wait for a user to finish */
a00b10c3 2457 avail = I915_FENCE_REG_NONE;
007cc8ac
DV		 2458 list_for_each_entry(reg, &dev_priv->mm.fence_list,
2459 lru_list) {
05394f39
CW		 2460 obj = reg->obj;
2461 if (obj->pin_count)
ae3db24a
DV		 2462 continue;
2463
2464 /* found one! */
05394f39 2465 avail = obj->fence_reg;
ae3db24a
DV		 2466 break;
2467 }
2468
a00b10c3 2469 BUG_ON(avail == I915_FENCE_REG_NONE);
ae3db24a
DV		 2470
2471 /* We only have a reference on obj from the active list. put_fence_reg
2472 * might drop that one, causing a use-after-free in it. So hold a
2473 * private reference to obj like the other callers of put_fence_reg
2474 * (set_tiling ioctl) do. */
05394f39
CW		 2475 drm_gem_object_reference(&obj->base);
2476 ret = i915_gem_object_put_fence_reg(obj, interruptible);
2477 drm_gem_object_unreference(&obj->base);
ae3db24a
DV		 2478 if (ret != 0)
2479 return ret;
2480
a00b10c3 2481 return avail;
ae3db24a
DV		 2482}
2483
de151cf6
JB		 2484/**
2485 * i915_gem_object_get_fence_reg - set up a fence reg for an object
2486 * @obj: object to map through a fence reg
2487 *
2488 * When mapping objects through the GTT, userspace wants to be able to write
2489 * to them without having to worry about swizzling if the object is tiled.
2490 *
2491 * This function walks the fence regs looking for a free one for @obj,
2492 * stealing one if it can't find any.
2493 *
2494 * It then sets up the reg based on the object's properties: address, pitch
2495 * and tiling format.
2496 */
8c4b8c3f 2497int
05394f39 2498i915_gem_object_get_fence_reg(struct drm_i915_gem_object *obj,
2cf34d7b 2499 bool interruptible)
de151cf6 2500{
05394f39 2501 struct drm_device *dev = obj->base.dev;
79e53945 2502 struct drm_i915_private *dev_priv = dev->dev_private;
de151cf6 2503 struct drm_i915_fence_reg *reg = NULL;
ae3db24a 2504 int ret;
de151cf6 2505
a09ba7fa 2506 /* Just update our place in the LRU if our fence is getting used. */
05394f39
CW		 2507 if (obj->fence_reg != I915_FENCE_REG_NONE) {
2508 reg = &dev_priv->fence_regs[obj->fence_reg];
007cc8ac 2509 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
a09ba7fa
EA		 2510 return 0;
2511 }
2512
05394f39 2513 switch (obj->tiling_mode) {
de151cf6
JB		 2514 case I915_TILING_NONE:
2515 WARN(1, "allocating a fence for non-tiled object?\n");
2516 break;
2517 case I915_TILING_X:
05394f39 2518 if (!obj->stride)
0f973f27 2519 return -EINVAL;
05394f39 2520 WARN((obj->stride & (512 - 1)),
0f973f27 2521 "object 0x%08x is X tiled but has non-512B pitch\n",
05394f39 2522 obj->gtt_offset);
de151cf6
JB		 2523 break;
2524 case I915_TILING_Y:
05394f39 2525 if (!obj->stride)
0f973f27 2526 return -EINVAL;
05394f39 2527 WARN((obj->stride & (128 - 1)),
0f973f27 2528 "object 0x%08x is Y tiled but has non-128B pitch\n",
05394f39 2529 obj->gtt_offset);
de151cf6
JB		 2530 break;
2531 }
2532
2cf34d7b 2533 ret = i915_find_fence_reg(dev, interruptible);
ae3db24a
DV		 2534 if (ret < 0)
2535 return ret;
de151cf6 2536
05394f39
CW		 2537 obj->fence_reg = ret;
2538 reg = &dev_priv->fence_regs[obj->fence_reg];
007cc8ac 2539 list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
a09ba7fa 2540
de151cf6
JB		 2541 reg->obj = obj;
2542
e259befd
CW		 2543 switch (INTEL_INFO(dev)->gen) {
2544 case 6:
a00b10c3 2545 sandybridge_write_fence_reg(obj);
e259befd
CW		 2546 break;
2547 case 5:
2548 case 4:
a00b10c3 2549 i965_write_fence_reg(obj);
e259befd
CW		 2550 break;
2551 case 3:
a00b10c3 2552 i915_write_fence_reg(obj);
e259befd
CW		 2553 break;
2554 case 2:
a00b10c3 2555 i830_write_fence_reg(obj);
e259befd
CW		 2556 break;
2557 }
d9ddcb96 2558
a00b10c3 2559 trace_i915_gem_object_get_fence(obj,
05394f39
CW		 2560 obj->fence_reg,
2561 obj->tiling_mode);
1c5d22f7 2562
d9ddcb96 2563 return 0;
de151cf6
JB		 2564}
2565
2566/**
2567 * i915_gem_clear_fence_reg - clear out fence register info
2568 * @obj: object to clear
2569 *
2570 * Zeroes out the fence register itself and clears out the associated
05394f39 2571 * data structures in dev_priv and obj.
de151cf6
JB		 2572 */
2573static void
05394f39 2574i915_gem_clear_fence_reg(struct drm_i915_gem_object *obj)
de151cf6 2575{
05394f39 2576 struct drm_device *dev = obj->base.dev;
79e53945 2577 drm_i915_private_t *dev_priv = dev->dev_private;
05394f39 2578 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[obj->fence_reg];
e259befd 2579 uint32_t fence_reg;
de151cf6 2580
e259befd
CW		 2581 switch (INTEL_INFO(dev)->gen) {
2582 case 6:
4e901fdc 2583 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
05394f39 2584 (obj->fence_reg * 8), 0);
e259befd
CW		 2585 break;
2586 case 5:
2587 case 4:
05394f39 2588 I915_WRITE64(FENCE_REG_965_0 + (obj->fence_reg * 8), 0);
e259befd
CW		 2589 break;
2590 case 3:
05394f39
CW		 2591 if (obj->fence_reg >= 8)
2592 fence_reg = FENCE_REG_945_8 + (obj->fence_reg - 8) * 4;
dc529a4f 2593 else
e259befd 2594 case 2:
05394f39 2595 fence_reg = FENCE_REG_830_0 + obj->fence_reg * 4;
dc529a4f
EA		 2596
2597 I915_WRITE(fence_reg, 0);
e259befd 2598 break;
dc529a4f 2599 }
de151cf6 2600
007cc8ac 2601 reg->obj = NULL;
05394f39 2602 obj->fence_reg = I915_FENCE_REG_NONE;
007cc8ac 2603 list_del_init(&reg->lru_list);
de151cf6
JB		 2604}
2605
52dc7d32
CW		 2606/**
2607 * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2608 * to the buffer to finish, and then resets the fence register.
2609 * @obj: tiled object holding a fence register.
2cf34d7b 2610 * @bool: whether the wait upon the fence is interruptible
52dc7d32
CW		 2611 *
2612 * Zeroes out the fence register itself and clears out the associated
05394f39 2613 * data structures in dev_priv and obj.
52dc7d32
CW		 2614 */
2615int
05394f39 2616i915_gem_object_put_fence_reg(struct drm_i915_gem_object *obj,
2cf34d7b 2617 bool interruptible)
52dc7d32 2618{
05394f39 2619 struct drm_device *dev = obj->base.dev;
53640e1d 2620 struct drm_i915_private *dev_priv = dev->dev_private;
53640e1d 2621 struct drm_i915_fence_reg *reg;
52dc7d32 2622
05394f39 2623 if (obj->fence_reg == I915_FENCE_REG_NONE)
52dc7d32
CW		 2624 return 0;
2625
10ae9bd2
DV		 2626 /* If we've changed tiling, GTT-mappings of the object
2627 * need to re-fault to ensure that the correct fence register
2628 * setup is in place.
2629 */
2630 i915_gem_release_mmap(obj);
2631
52dc7d32
CW		 2632 /* On the i915, GPU access to tiled buffers is via a fence,
2633 * therefore we must wait for any outstanding access to complete
2634 * before clearing the fence.
2635 */
05394f39 2636 reg = &dev_priv->fence_regs[obj->fence_reg];
53640e1d 2637 if (reg->gpu) {
52dc7d32
CW		 2638 int ret;
2639
2cf34d7b 2640 ret = i915_gem_object_flush_gpu_write_domain(obj, true);
0bc23aad 2641 if (ret)
2dafb1e0
CW		 2642 return ret;
2643
2cf34d7b 2644 ret = i915_gem_object_wait_rendering(obj, interruptible);
0bc23aad 2645 if (ret)
52dc7d32 2646 return ret;
53640e1d
CW		 2647
2648 reg->gpu = false;
52dc7d32
CW		 2649 }
2650
4a726612 2651 i915_gem_object_flush_gtt_write_domain(obj);
0bc23aad 2652 i915_gem_clear_fence_reg(obj);
52dc7d32
CW		 2653
2654 return 0;
2655}
2656
673a394b
EA		 2657/**
2658 * Finds free space in the GTT aperture and binds the object there.
2659 */
2660static int
05394f39 2661i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
920afa77 2662 unsigned alignment,
75e9e915 2663 bool map_and_fenceable)
673a394b 2664{
05394f39 2665 struct drm_device *dev = obj->base.dev;
673a394b 2666 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 2667 struct drm_mm_node *free_space;
a00b10c3 2668 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
5e783301 2669 u32 size, fence_size, fence_alignment, unfenced_alignment;
75e9e915 2670 bool mappable, fenceable;
07f73f69 2671 int ret;
673a394b 2672
05394f39 2673 if (obj->madv != I915_MADV_WILLNEED) {
3ef94daa
CW		 2674 DRM_ERROR("Attempting to bind a purgeable object\n");
2675 return -EINVAL;
2676 }
2677
05394f39
CW		 2678 fence_size = i915_gem_get_gtt_size(obj);
2679 fence_alignment = i915_gem_get_gtt_alignment(obj);
2680 unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(obj);
a00b10c3 2681
673a394b 2682 if (alignment == 0)
5e783301
DV		 2683 alignment = map_and_fenceable ? fence_alignment :
2684 unfenced_alignment;
75e9e915 2685 if (map_and_fenceable && alignment & (fence_alignment - 1)) {
673a394b
EA		 2686 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2687 return -EINVAL;
2688 }
2689
05394f39 2690 size = map_and_fenceable ? fence_size : obj->base.size;
a00b10c3 2691
654fc607
CW		 2692 /* If the object is bigger than the entire aperture, reject it early
2693 * before evicting everything in a vain attempt to find space.
2694 */
05394f39 2695 if (obj->base.size >
75e9e915 2696 (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
654fc607
CW		 2697 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2698 return -E2BIG;
2699 }
2700
673a394b 2701 search_free:
75e9e915 2702 if (map_and_fenceable)
920afa77
DV		 2703 free_space =
2704 drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
a00b10c3 2705 size, alignment, 0,
920afa77
DV		 2706 dev_priv->mm.gtt_mappable_end,
2707 0);
2708 else
2709 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
a00b10c3 2710 size, alignment, 0);
920afa77
DV		 2711
2712 if (free_space != NULL) {
75e9e915 2713 if (map_and_fenceable)
05394f39 2714 obj->gtt_space =
920afa77 2715 drm_mm_get_block_range_generic(free_space,
a00b10c3 2716 size, alignment, 0,
920afa77
DV		 2717 dev_priv->mm.gtt_mappable_end,
2718 0);
2719 else
05394f39 2720 obj->gtt_space =
a00b10c3 2721 drm_mm_get_block(free_space, size, alignment);
920afa77 2722 }
05394f39 2723 if (obj->gtt_space == NULL) {
673a394b
EA		 2724 /* If the gtt is empty and we're still having trouble
2725 * fitting our object in, we're out of memory.
2726 */
75e9e915
DV		 2727 ret = i915_gem_evict_something(dev, size, alignment,
2728 map_and_fenceable);
9731129c 2729 if (ret)
673a394b 2730 return ret;
9731129c 2731
673a394b
EA		 2732 goto search_free;
2733 }
2734
e5281ccd 2735 ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
673a394b 2736 if (ret) {
05394f39
CW		 2737 drm_mm_put_block(obj->gtt_space);
2738 obj->gtt_space = NULL;
07f73f69
CW		 2739
2740 if (ret == -ENOMEM) {
2741 /* first try to clear up some space from the GTT */
a00b10c3 2742 ret = i915_gem_evict_something(dev, size,
75e9e915
DV		 2743 alignment,
2744 map_and_fenceable);
07f73f69 2745 if (ret) {
07f73f69 2746 /* now try to shrink everyone else */
4bdadb97
CW		 2747 if (gfpmask) {
2748 gfpmask = 0;
2749 goto search_free;
07f73f69
CW		 2750 }
2751
2752 return ret;
2753 }
2754
2755 goto search_free;
2756 }
2757
673a394b
EA		 2758 return ret;
2759 }
2760
7c2e6fdf
DV		 2761 ret = i915_gem_gtt_bind_object(obj);
2762 if (ret) {
e5281ccd 2763 i915_gem_object_put_pages_gtt(obj);
05394f39
CW		 2764 drm_mm_put_block(obj->gtt_space);
2765 obj->gtt_space = NULL;
07f73f69 2766
a00b10c3 2767 ret = i915_gem_evict_something(dev, size,
75e9e915 2768 alignment, map_and_fenceable);
9731129c 2769 if (ret)
07f73f69 2770 return ret;
07f73f69
CW		 2771
2772 goto search_free;
673a394b 2773 }
673a394b 2774
05394f39 2775 obj->gtt_offset = obj->gtt_space->start;
fb7d516a 2776
bf1a1092 2777 /* keep track of bounds object by adding it to the inactive list */
05394f39
CW		 2778 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2779 i915_gem_info_add_gtt(dev_priv, obj);
bf1a1092 2780
673a394b
EA		 2781 /* Assert that the object is not currently in any GPU domain. As it
2782 * wasn't in the GTT, there shouldn't be any way it could have been in
2783 * a GPU cache
2784 */
05394f39
CW		 2785 BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2786 BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
673a394b 2787
05394f39 2788 trace_i915_gem_object_bind(obj, obj->gtt_offset, map_and_fenceable);
1c5d22f7 2789
75e9e915 2790 fenceable =
05394f39
CW		 2791 obj->gtt_space->size == fence_size &&
2792 (obj->gtt_space->start & (fence_alignment -1)) == 0;
a00b10c3 2793
75e9e915 2794 mappable =
05394f39 2795 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
a00b10c3 2796
05394f39 2797 obj->map_and_fenceable = mappable && fenceable;
75e9e915 2798
673a394b
EA		 2799 return 0;
2800}
2801
2802void
05394f39 2803i915_gem_clflush_object(struct drm_i915_gem_object *obj)
673a394b 2804{
673a394b
EA		 2805 /* If we don't have a page list set up, then we're not pinned
2806 * to GPU, and we can ignore the cache flush because it'll happen
2807 * again at bind time.
2808 */
05394f39 2809 if (obj->pages == NULL)
673a394b
EA		 2810 return;
2811
1c5d22f7 2812 trace_i915_gem_object_clflush(obj);
cfa16a0d 2813
05394f39 2814 drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
673a394b
EA		 2815}
2816
e47c68e9 2817/** Flushes any GPU write domain for the object if it's dirty. */
2dafb1e0 2818static int
05394f39 2819i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj,
ba3d8d74 2820 bool pipelined)
e47c68e9 2821{
05394f39 2822 struct drm_device *dev = obj->base.dev;
e47c68e9 2823
05394f39 2824 if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
2dafb1e0 2825 return 0;
e47c68e9
EA		 2826
2827 /* Queue the GPU write cache flushing we need. */
05394f39
CW		 2828 i915_gem_flush_ring(dev, obj->ring, 0, obj->base.write_domain);
2829 BUG_ON(obj->base.write_domain);
1c5d22f7 2830
ba3d8d74
DV		 2831 if (pipelined)
2832 return 0;
2833
2cf34d7b 2834 return i915_gem_object_wait_rendering(obj, true);
e47c68e9
EA		 2835}
2836
2837/** Flushes the GTT write domain for the object if it's dirty. */
2838static void
05394f39 2839i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
e47c68e9 2840{
1c5d22f7
CW		 2841 uint32_t old_write_domain;
2842
05394f39 2843 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
e47c68e9
EA		 2844 return;
2845
2846 /* No actual flushing is required for the GTT write domain. Writes
2847 * to it immediately go to main memory as far as we know, so there's
2848 * no chipset flush. It also doesn't land in render cache.
2849 */
4a684a41
CW		 2850 i915_gem_release_mmap(obj);
2851
05394f39
CW		 2852 old_write_domain = obj->base.write_domain;
2853 obj->base.write_domain = 0;
1c5d22f7
CW		 2854
2855 trace_i915_gem_object_change_domain(obj,
05394f39 2856 obj->base.read_domains,
1c5d22f7 2857 old_write_domain);
e47c68e9
EA		 2858}
2859
2860/** Flushes the CPU write domain for the object if it's dirty. */
2861static void
05394f39 2862i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
e47c68e9 2863{
1c5d22f7 2864 uint32_t old_write_domain;
e47c68e9 2865
05394f39 2866 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
e47c68e9
EA		 2867 return;
2868
2869 i915_gem_clflush_object(obj);
40ce6575 2870 intel_gtt_chipset_flush();
05394f39
CW		 2871 old_write_domain = obj->base.write_domain;
2872 obj->base.write_domain = 0;
1c5d22f7
CW		 2873
2874 trace_i915_gem_object_change_domain(obj,
05394f39 2875 obj->base.read_domains,
1c5d22f7 2876 old_write_domain);
e47c68e9
EA		 2877}
2878
2ef7eeaa
EA		 2879/**
2880 * Moves a single object to the GTT read, and possibly write domain.
2881 *
2882 * This function returns when the move is complete, including waiting on
2883 * flushes to occur.
2884 */
79e53945 2885int
05394f39 2886i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, int write)
2ef7eeaa 2887{
1c5d22f7 2888 uint32_t old_write_domain, old_read_domains;
e47c68e9 2889 int ret;
2ef7eeaa 2890
02354392 2891 /* Not valid to be called on unbound objects. */
05394f39 2892 if (obj->gtt_space == NULL)
02354392
EA		 2893 return -EINVAL;
2894
ba3d8d74 2895 ret = i915_gem_object_flush_gpu_write_domain(obj, false);
2dafb1e0
CW		 2896 if (ret != 0)
2897 return ret;
2898
7213342d 2899 i915_gem_object_flush_cpu_write_domain(obj);
1c5d22f7 2900
ba3d8d74 2901 if (write) {
2cf34d7b 2902 ret = i915_gem_object_wait_rendering(obj, true);
ba3d8d74
DV		 2903 if (ret)
2904 return ret;
ba3d8d74 2905 }
e47c68e9 2906
05394f39
CW		 2907 old_write_domain = obj->base.write_domain;
2908 old_read_domains = obj->base.read_domains;
1c5d22f7 2909
e47c68e9
EA		 2910 /* It should now be out of any other write domains, and we can update
2911 * the domain values for our changes.
2912 */
05394f39
CW		 2913 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2914 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
e47c68e9 2915 if (write) {
05394f39
CW		 2916 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
2917 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
2918 obj->dirty = 1;
2ef7eeaa
EA		 2919 }
2920
1c5d22f7
CW		 2921 trace_i915_gem_object_change_domain(obj,
2922 old_read_domains,
2923 old_write_domain);
2924
e47c68e9
EA		 2925 return 0;
2926}
2927
b9241ea3
ZW		 2928/*
2929 * Prepare buffer for display plane. Use uninterruptible for possible flush
2930 * wait, as in modesetting process we're not supposed to be interrupted.
2931 */
2932int
05394f39 2933i915_gem_object_set_to_display_plane(struct drm_i915_gem_object *obj,
48b956c5 2934 bool pipelined)
b9241ea3 2935{
ba3d8d74 2936 uint32_t old_read_domains;
b9241ea3
ZW		 2937 int ret;
2938
2939 /* Not valid to be called on unbound objects. */
05394f39 2940 if (obj->gtt_space == NULL)
b9241ea3
ZW		 2941 return -EINVAL;
2942
ced270fa 2943 ret = i915_gem_object_flush_gpu_write_domain(obj, true);
2dafb1e0
CW		 2944 if (ret)
2945 return ret;
b9241ea3 2946
ced270fa
CW		 2947 /* Currently, we are always called from an non-interruptible context. */
2948 if (!pipelined) {
2949 ret = i915_gem_object_wait_rendering(obj, false);
2950 if (ret)
b9241ea3
ZW		 2951 return ret;
2952 }
2953
b118c1e3
CW		 2954 i915_gem_object_flush_cpu_write_domain(obj);
2955
05394f39
CW		 2956 old_read_domains = obj->base.read_domains;
2957 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
b9241ea3
ZW		 2958
2959 trace_i915_gem_object_change_domain(obj,
2960 old_read_domains,
05394f39 2961 obj->base.write_domain);
b9241ea3
ZW		 2962
2963 return 0;
2964}
2965
85345517
CW		 2966int
2967i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj,
2968 bool interruptible)
2969{
2970 if (!obj->active)
2971 return 0;
2972
2973 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS)
05394f39 2974 i915_gem_flush_ring(obj->base.dev, obj->ring,
85345517
CW		 2975 0, obj->base.write_domain);
2976
05394f39 2977 return i915_gem_object_wait_rendering(obj, interruptible);
85345517
CW		 2978}
2979
e47c68e9
EA		 2980/**
2981 * Moves a single object to the CPU read, and possibly write domain.
2982 *
2983 * This function returns when the move is complete, including waiting on
2984 * flushes to occur.
2985 */
2986static int
05394f39 2987i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, int write)
e47c68e9 2988{
1c5d22f7 2989 uint32_t old_write_domain, old_read_domains;
e47c68e9
EA		 2990 int ret;
2991
ba3d8d74 2992 ret = i915_gem_object_flush_gpu_write_domain(obj, false);
e47c68e9
EA		 2993 if (ret != 0)
2994 return ret;
2ef7eeaa 2995
e47c68e9 2996 i915_gem_object_flush_gtt_write_domain(obj);
2ef7eeaa 2997
e47c68e9
EA		 2998 /* If we have a partially-valid cache of the object in the CPU,
2999 * finish invalidating it and free the per-page flags.
2ef7eeaa 3000 */
e47c68e9 3001 i915_gem_object_set_to_full_cpu_read_domain(obj);
2ef7eeaa 3002
7213342d 3003 if (write) {
2cf34d7b 3004 ret = i915_gem_object_wait_rendering(obj, true);
7213342d
CW		 3005 if (ret)
3006 return ret;
3007 }
3008
05394f39
CW		 3009 old_write_domain = obj->base.write_domain;
3010 old_read_domains = obj->base.read_domains;
1c5d22f7 3011
e47c68e9 3012 /* Flush the CPU cache if it's still invalid. */
05394f39 3013 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2ef7eeaa 3014 i915_gem_clflush_object(obj);
2ef7eeaa 3015
05394f39 3016 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
2ef7eeaa
EA		 3017 }
3018
3019 /* It should now be out of any other write domains, and we can update
3020 * the domain values for our changes.
3021 */
05394f39 3022 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
e47c68e9
EA		 3023
3024 /* If we're writing through the CPU, then the GPU read domains will
3025 * need to be invalidated at next use.
3026 */
3027 if (write) {
05394f39
CW		 3028 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3029 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
e47c68e9 3030 }
2ef7eeaa 3031
1c5d22f7
CW		 3032 trace_i915_gem_object_change_domain(obj,
3033 old_read_domains,
3034 old_write_domain);
3035
2ef7eeaa
EA		 3036 return 0;
3037}
3038
673a394b
EA		 3039/*
3040 * Set the next domain for the specified object. This
3041 * may not actually perform the necessary flushing/invaliding though,
3042 * as that may want to be batched with other set_domain operations
3043 *
3044 * This is (we hope) the only really tricky part of gem. The goal
3045 * is fairly simple -- track which caches hold bits of the object
3046 * and make sure they remain coherent. A few concrete examples may
3047 * help to explain how it works. For shorthand, we use the notation
3048 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
3049 * a pair of read and write domain masks.
3050 *
3051 * Case 1: the batch buffer
3052 *
3053 * 1. Allocated
3054 * 2. Written by CPU
3055 * 3. Mapped to GTT
3056 * 4. Read by GPU
3057 * 5. Unmapped from GTT
3058 * 6. Freed
3059 *
3060 * Let's take these a step at a time
3061 *
3062 * 1. Allocated
3063 * Pages allocated from the kernel may still have
3064 * cache contents, so we set them to (CPU, CPU) always.
3065 * 2. Written by CPU (using pwrite)
3066 * The pwrite function calls set_domain (CPU, CPU) and
3067 * this function does nothing (as nothing changes)
3068 * 3. Mapped by GTT
3069 * This function asserts that the object is not
3070 * currently in any GPU-based read or write domains
3071 * 4. Read by GPU
3072 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
3073 * As write_domain is zero, this function adds in the
3074 * current read domains (CPU+COMMAND, 0).
3075 * flush_domains is set to CPU.
3076 * invalidate_domains is set to COMMAND
3077 * clflush is run to get data out of the CPU caches
3078 * then i915_dev_set_domain calls i915_gem_flush to
3079 * emit an MI_FLUSH and drm_agp_chipset_flush
3080 * 5. Unmapped from GTT
3081 * i915_gem_object_unbind calls set_domain (CPU, CPU)
3082 * flush_domains and invalidate_domains end up both zero
3083 * so no flushing/invalidating happens
3084 * 6. Freed
3085 * yay, done
3086 *
3087 * Case 2: The shared render buffer
3088 *
3089 * 1. Allocated
3090 * 2. Mapped to GTT
3091 * 3. Read/written by GPU
3092 * 4. set_domain to (CPU,CPU)
3093 * 5. Read/written by CPU
3094 * 6. Read/written by GPU
3095 *
3096 * 1. Allocated
3097 * Same as last example, (CPU, CPU)
3098 * 2. Mapped to GTT
3099 * Nothing changes (assertions find that it is not in the GPU)
3100 * 3. Read/written by GPU
3101 * execbuffer calls set_domain (RENDER, RENDER)
3102 * flush_domains gets CPU
3103 * invalidate_domains gets GPU
3104 * clflush (obj)
3105 * MI_FLUSH and drm_agp_chipset_flush
3106 * 4. set_domain (CPU, CPU)
3107 * flush_domains gets GPU
3108 * invalidate_domains gets CPU
3109 * wait_rendering (obj) to make sure all drawing is complete.
3110 * This will include an MI_FLUSH to get the data from GPU
3111 * to memory
3112 * clflush (obj) to invalidate the CPU cache
3113 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3114 * 5. Read/written by CPU
3115 * cache lines are loaded and dirtied
3116 * 6. Read written by GPU
3117 * Same as last GPU access
3118 *
3119 * Case 3: The constant buffer
3120 *
3121 * 1. Allocated
3122 * 2. Written by CPU
3123 * 3. Read by GPU
3124 * 4. Updated (written) by CPU again
3125 * 5. Read by GPU
3126 *
3127 * 1. Allocated
3128 * (CPU, CPU)
3129 * 2. Written by CPU
3130 * (CPU, CPU)
3131 * 3. Read by GPU
3132 * (CPU+RENDER, 0)
3133 * flush_domains = CPU
3134 * invalidate_domains = RENDER
3135 * clflush (obj)
3136 * MI_FLUSH
3137 * drm_agp_chipset_flush
3138 * 4. Updated (written) by CPU again
3139 * (CPU, CPU)
3140 * flush_domains = 0 (no previous write domain)
3141 * invalidate_domains = 0 (no new read domains)
3142 * 5. Read by GPU
3143 * (CPU+RENDER, 0)
3144 * flush_domains = CPU
3145 * invalidate_domains = RENDER
3146 * clflush (obj)
3147 * MI_FLUSH
3148 * drm_agp_chipset_flush
3149 */
c0d90829 3150static void
05394f39 3151i915_gem_object_set_to_gpu_domain(struct drm_i915_gem_object *obj,
0f8c6d7c
CW		 3152 struct intel_ring_buffer *ring,
3153 struct change_domains *cd)
673a394b 3154{
05394f39 3155 uint32_t invalidate_domains = 0, flush_domains = 0;
652c393a 3156
673a394b
EA		 3157 /*
3158 * If the object isn't moving to a new write domain,
3159 * let the object stay in multiple read domains
3160 */
05394f39
CW		 3161 if (obj->base.pending_write_domain == 0)
3162 obj->base.pending_read_domains |= obj->base.read_domains;
673a394b
EA		 3163
3164 /*
3165 * Flush the current write domain if
3166 * the new read domains don't match. Invalidate
3167 * any read domains which differ from the old
3168 * write domain
3169 */
05394f39
CW		 3170 if (obj->base.write_domain &&
3171 (obj->base.write_domain != obj->base.pending_read_domains ||
3172 obj->ring != ring)) {
3173 flush_domains |= obj->base.write_domain;
8b0e378a 3174 invalidate_domains |=
05394f39 3175 obj->base.pending_read_domains & ~obj->base.write_domain;
673a394b
EA		 3176 }
3177 /*
3178 * Invalidate any read caches which may have
3179 * stale data. That is, any new read domains.
3180 */
05394f39 3181 invalidate_domains |= obj->base.pending_read_domains & ~obj->base.read_domains;
3d2a812a 3182 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
673a394b 3183 i915_gem_clflush_object(obj);
673a394b 3184
4a684a41
CW		 3185 /* blow away mappings if mapped through GTT */
3186 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_GTT)
3187 i915_gem_release_mmap(obj);
3188
efbeed96
EA		 3189 /* The actual obj->write_domain will be updated with
3190 * pending_write_domain after we emit the accumulated flush for all
3191 * of our domain changes in execbuffers (which clears objects'
3192 * write_domains). So if we have a current write domain that we
3193 * aren't changing, set pending_write_domain to that.
3194 */
05394f39
CW		 3195 if (flush_domains == 0 && obj->base.pending_write_domain == 0)
3196 obj->base.pending_write_domain = obj->base.write_domain;
673a394b 3197
0f8c6d7c
CW		 3198 cd->invalidate_domains |= invalidate_domains;
3199 cd->flush_domains |= flush_domains;
b6651458 3200 if (flush_domains & I915_GEM_GPU_DOMAINS)
05394f39 3201 cd->flush_rings |= obj->ring->id;
b6651458 3202 if (invalidate_domains & I915_GEM_GPU_DOMAINS)
0f8c6d7c 3203 cd->flush_rings |= ring->id;
673a394b
EA		 3204}
3205
3206/**
e47c68e9 3207 * Moves the object from a partially CPU read to a full one.
673a394b 3208 *
e47c68e9
EA		 3209 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3210 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
673a394b 3211 */
e47c68e9 3212static void
05394f39 3213i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj)
673a394b 3214{
05394f39 3215 if (!obj->page_cpu_valid)
e47c68e9
EA		 3216 return;
3217
3218 /* If we're partially in the CPU read domain, finish moving it in.
3219 */
05394f39 3220 if (obj->base.read_domains & I915_GEM_DOMAIN_CPU) {
e47c68e9
EA		 3221 int i;
3222
05394f39
CW		 3223 for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) {
3224 if (obj->page_cpu_valid[i])
e47c68e9 3225 continue;
05394f39 3226 drm_clflush_pages(obj->pages + i, 1);
e47c68e9 3227 }
e47c68e9
EA		 3228 }
3229
3230 /* Free the page_cpu_valid mappings which are now stale, whether
3231 * or not we've got I915_GEM_DOMAIN_CPU.
3232 */
05394f39
CW		 3233 kfree(obj->page_cpu_valid);
3234 obj->page_cpu_valid = NULL;
e47c68e9
EA		 3235}
3236
3237/**
3238 * Set the CPU read domain on a range of the object.
3239 *
3240 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3241 * not entirely valid. The page_cpu_valid member of the object flags which
3242 * pages have been flushed, and will be respected by
3243 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3244 * of the whole object.
3245 *
3246 * This function returns when the move is complete, including waiting on
3247 * flushes to occur.
3248 */
3249static int
05394f39 3250i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
e47c68e9
EA		 3251 uint64_t offset, uint64_t size)
3252{
1c5d22f7 3253 uint32_t old_read_domains;
e47c68e9 3254 int i, ret;
673a394b 3255
05394f39 3256 if (offset == 0 && size == obj->base.size)
e47c68e9 3257 return i915_gem_object_set_to_cpu_domain(obj, 0);
673a394b 3258
ba3d8d74 3259 ret = i915_gem_object_flush_gpu_write_domain(obj, false);
e47c68e9 3260 if (ret != 0)
6a47baa6 3261 return ret;
e47c68e9
EA		 3262 i915_gem_object_flush_gtt_write_domain(obj);
3263
3264 /* If we're already fully in the CPU read domain, we're done. */
05394f39
CW		 3265 if (obj->page_cpu_valid == NULL &&
3266 (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)
e47c68e9 3267 return 0;
673a394b 3268
e47c68e9
EA		 3269 /* Otherwise, create/clear the per-page CPU read domain flag if we're
3270 * newly adding I915_GEM_DOMAIN_CPU
3271 */
05394f39
CW		 3272 if (obj->page_cpu_valid == NULL) {
3273 obj->page_cpu_valid = kzalloc(obj->base.size / PAGE_SIZE,
3274 GFP_KERNEL);
3275 if (obj->page_cpu_valid == NULL)
e47c68e9 3276 return -ENOMEM;
05394f39
CW		 3277 } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
3278 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);
673a394b
EA		 3279
3280 /* Flush the cache on any pages that are still invalid from the CPU's
3281 * perspective.
3282 */
e47c68e9
EA		 3283 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3284 i++) {
05394f39 3285 if (obj->page_cpu_valid[i])
673a394b
EA		 3286 continue;
3287
05394f39 3288 drm_clflush_pages(obj->pages + i, 1);
673a394b 3289
05394f39 3290 obj->page_cpu_valid[i] = 1;
673a394b
EA		 3291 }
3292
e47c68e9
EA		 3293 /* It should now be out of any other write domains, and we can update
3294 * the domain values for our changes.
3295 */
05394f39 3296 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
e47c68e9 3297
05394f39
CW		 3298 old_read_domains = obj->base.read_domains;
3299 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
e47c68e9 3300
1c5d22f7
CW		 3301 trace_i915_gem_object_change_domain(obj,
3302 old_read_domains,
05394f39 3303 obj->base.write_domain);
1c5d22f7 3304
673a394b
EA		 3305 return 0;
3306}
3307
673a394b 3308static int
bcf50e27
CW		 3309i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
3310 struct drm_file *file_priv,
3311 struct drm_i915_gem_exec_object2 *entry,
3312 struct drm_i915_gem_relocation_entry *reloc)
673a394b 3313{
9af90d19 3314 struct drm_device *dev = obj->base.dev;
bcf50e27
CW		 3315 struct drm_gem_object *target_obj;
3316 uint32_t target_offset;
3317 int ret = -EINVAL;
673a394b 3318
bcf50e27
CW		 3319 target_obj = drm_gem_object_lookup(dev, file_priv,
3320 reloc->target_handle);
3321 if (target_obj == NULL)
3322 return -ENOENT;
673a394b 3323
bcf50e27 3324 target_offset = to_intel_bo(target_obj)->gtt_offset;
76446cac 3325
bcf50e27
CW		 3326#if WATCH_RELOC
3327 DRM_INFO("%s: obj %p offset %08x target %d "
3328 "read %08x write %08x gtt %08x "
3329 "presumed %08x delta %08x\n",
3330 __func__,
3331 obj,
3332 (int) reloc->offset,
3333 (int) reloc->target_handle,
3334 (int) reloc->read_domains,
3335 (int) reloc->write_domain,
3336 (int) target_offset,
3337 (int) reloc->presumed_offset,
3338 reloc->delta);
3339#endif
673a394b 3340
bcf50e27
CW		 3341 /* The target buffer should have appeared before us in the
3342 * exec_object list, so it should have a GTT space bound by now.
3343 */
3344 if (target_offset == 0) {
3345 DRM_ERROR("No GTT space found for object %d\n",
3346 reloc->target_handle);
3347 goto err;
3348 }
9af90d19 3349
bcf50e27
CW		 3350 /* Validate that the target is in a valid r/w GPU domain */
3351 if (reloc->write_domain & (reloc->write_domain - 1)) {
3352 DRM_ERROR("reloc with multiple write domains: "
3353 "obj %p target %d offset %d "
3354 "read %08x write %08x",
3355 obj, reloc->target_handle,
3356 (int) reloc->offset,
3357 reloc->read_domains,
3358 reloc->write_domain);
3359 goto err;
3360 }
3361 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3362 reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3363 DRM_ERROR("reloc with read/write CPU domains: "
3364 "obj %p target %d offset %d "
3365 "read %08x write %08x",
3366 obj, reloc->target_handle,
3367 (int) reloc->offset,
3368 reloc->read_domains,
3369 reloc->write_domain);
3370 goto err;
3371 }
3372 if (reloc->write_domain && target_obj->pending_write_domain &&
3373 reloc->write_domain != target_obj->pending_write_domain) {
3374 DRM_ERROR("Write domain conflict: "
3375 "obj %p target %d offset %d "
3376 "new %08x old %08x\n",
3377 obj, reloc->target_handle,
3378 (int) reloc->offset,
3379 reloc->write_domain,
3380 target_obj->pending_write_domain);
3381 goto err;
3382 }
673a394b 3383
bcf50e27
CW		 3384 target_obj->pending_read_domains |= reloc->read_domains;
3385 target_obj->pending_write_domain |= reloc->write_domain;
8542a0bb 3386
bcf50e27
CW		 3387 /* If the relocation already has the right value in it, no
3388 * more work needs to be done.
3389 */
3390 if (target_offset == reloc->presumed_offset)
3391 goto out;
673a394b 3392
bcf50e27
CW		 3393 /* Check that the relocation address is valid... */
3394 if (reloc->offset > obj->base.size - 4) {
3395 DRM_ERROR("Relocation beyond object bounds: "
3396 "obj %p target %d offset %d size %d.\n",
3397 obj, reloc->target_handle,
3398 (int) reloc->offset,
3399 (int) obj->base.size);
3400 goto err;
3401 }
3402 if (reloc->offset & 3) {
3403 DRM_ERROR("Relocation not 4-byte aligned: "
3404 "obj %p target %d offset %d.\n",
3405 obj, reloc->target_handle,
3406 (int) reloc->offset);
3407 goto err;
3408 }
673a394b 3409
bcf50e27
CW		 3410 /* and points to somewhere within the target object. */
3411 if (reloc->delta >= target_obj->size) {
3412 DRM_ERROR("Relocation beyond target object bounds: "
3413 "obj %p target %d delta %d size %d.\n",
3414 obj, reloc->target_handle,
3415 (int) reloc->delta,
3416 (int) target_obj->size);
3417 goto err;
3418 }
673a394b 3419
bcf50e27
CW		 3420 reloc->delta += target_offset;
3421 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
3422 uint32_t page_offset = reloc->offset & ~PAGE_MASK;
3423 char *vaddr;
673a394b 3424
bcf50e27
CW		 3425 vaddr = kmap_atomic(obj->pages[reloc->offset >> PAGE_SHIFT]);
3426 *(uint32_t *)(vaddr + page_offset) = reloc->delta;
3427 kunmap_atomic(vaddr);
3428 } else {
3429 struct drm_i915_private *dev_priv = dev->dev_private;
3430 uint32_t __iomem *reloc_entry;
3431 void __iomem *reloc_page;
8542a0bb 3432
05394f39 3433 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
bcf50e27
CW		 3434 if (ret)
3435 goto err;
673a394b 3436
bcf50e27
CW		 3437 /* Map the page containing the relocation we're going to perform. */
3438 reloc->offset += obj->gtt_offset;
3439 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3440 reloc->offset & PAGE_MASK);
3441 reloc_entry = (uint32_t __iomem *)
3442 (reloc_page + (reloc->offset & ~PAGE_MASK));
3443 iowrite32(reloc->delta, reloc_entry);
3444 io_mapping_unmap_atomic(reloc_page);
3445 }
673a394b 3446
bcf50e27
CW		 3447 /* and update the user's relocation entry */
3448 reloc->presumed_offset = target_offset;
b962442e 3449
bcf50e27
CW		 3450out:
3451 ret = 0;
3452err:
3453 drm_gem_object_unreference(target_obj);
3454 return ret;
3455}
b962442e 3456
bcf50e27
CW		 3457static int
3458i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj,
3459 struct drm_file *file_priv,
3460 struct drm_i915_gem_exec_object2 *entry)
3461{
3462 struct drm_i915_gem_relocation_entry __user *user_relocs;
3463 int i, ret;
3464
3465 user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
3466 for (i = 0; i < entry->relocation_count; i++) {
3467 struct drm_i915_gem_relocation_entry reloc;
3468
3469 if (__copy_from_user_inatomic(&reloc,
3470 user_relocs+i,
3471 sizeof(reloc)))
3472 return -EFAULT;
3473
3474 ret = i915_gem_execbuffer_relocate_entry(obj, file_priv, entry, &reloc);
3475 if (ret)
3476 return ret;
b962442e 3477
b5dc608c 3478 if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset,
bcf50e27
CW		 3479 &reloc.presumed_offset,
3480 sizeof(reloc.presumed_offset)))
3481 return -EFAULT;
b962442e 3482 }
b962442e 3483
bcf50e27
CW		 3484 return 0;
3485}
3486
3487static int
3488i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj,
3489 struct drm_file *file_priv,
3490 struct drm_i915_gem_exec_object2 *entry,
3491 struct drm_i915_gem_relocation_entry *relocs)
3492{
3493 int i, ret;
3494
3495 for (i = 0; i < entry->relocation_count; i++) {
3496 ret = i915_gem_execbuffer_relocate_entry(obj, file_priv, entry, &relocs[i]);
3497 if (ret)
3498 return ret;
3499 }
3500
3501 return 0;
673a394b
EA		 3502}
3503
40a5f0de 3504static int
bcf50e27
CW		 3505i915_gem_execbuffer_relocate(struct drm_device *dev,
3506 struct drm_file *file,
05394f39 3507 struct drm_i915_gem_object **object_list,
bcf50e27
CW		 3508 struct drm_i915_gem_exec_object2 *exec_list,
3509 int count)
3510{
3511 int i, ret;
3512
3513 for (i = 0; i < count; i++) {
05394f39 3514 struct drm_i915_gem_object *obj = object_list[i];
bcf50e27
CW		 3515 obj->base.pending_read_domains = 0;
3516 obj->base.pending_write_domain = 0;
3517 ret = i915_gem_execbuffer_relocate_object(obj, file,
3518 &exec_list[i]);
3519 if (ret)
3520 return ret;
3521 }
3522
3523 return 0;
673a394b
EA		 3524}
3525
40a5f0de 3526static int
bcf50e27
CW		 3527i915_gem_execbuffer_reserve(struct drm_device *dev,
3528 struct drm_file *file,
05394f39 3529 struct drm_i915_gem_object **object_list,
bcf50e27
CW		 3530 struct drm_i915_gem_exec_object2 *exec_list,
3531 int count)
40a5f0de 3532{
9af90d19
CW		 3533 struct drm_i915_private *dev_priv = dev->dev_private;
3534 int ret, i, retry;
40a5f0de 3535
9af90d19 3536 /* attempt to pin all of the buffers into the GTT */
5eac3ab4
CW		 3537 retry = 0;
3538 do {
9af90d19
CW		 3539 ret = 0;
3540 for (i = 0; i < count; i++) {
3541 struct drm_i915_gem_exec_object2 *entry = &exec_list[i];
05394f39 3542 struct drm_i915_gem_object *obj = object_list[i];
9af90d19
CW		 3543 bool need_fence =
3544 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3545 obj->tiling_mode != I915_TILING_NONE;
3546
16e809ac
DV		 3547 /* g33/pnv can't fence buffers in the unmappable part */
3548 bool need_mappable =
3549 entry->relocation_count ? true : need_fence;
3550
9af90d19 3551 /* Check fence reg constraints and rebind if necessary */
75e9e915 3552 if (need_mappable && !obj->map_and_fenceable) {
05394f39 3553 ret = i915_gem_object_unbind(obj);
9af90d19
CW		 3554 if (ret)
3555 break;
3556 }
40a5f0de 3557
05394f39 3558 ret = i915_gem_object_pin(obj,
16e809ac 3559 entry->alignment,
75e9e915 3560 need_mappable);
9af90d19
CW		 3561 if (ret)
3562 break;
40a5f0de 3563
9af90d19
CW		 3564 /*
3565 * Pre-965 chips need a fence register set up in order
3566 * to properly handle blits to/from tiled surfaces.
3567 */
3568 if (need_fence) {
05394f39 3569 ret = i915_gem_object_get_fence_reg(obj, true);
9af90d19 3570 if (ret) {
05394f39 3571 i915_gem_object_unpin(obj);
9af90d19
CW		 3572 break;
3573 }
40a5f0de 3574
9af90d19
CW		 3575 dev_priv->fence_regs[obj->fence_reg].gpu = true;
3576 }
40a5f0de 3577
9af90d19 3578 entry->offset = obj->gtt_offset;
40a5f0de
EA		 3579 }
3580
9af90d19
CW		 3581 while (i--)
3582 i915_gem_object_unpin(object_list[i]);
3583
5eac3ab4 3584 if (ret != -ENOSPC || retry > 1)
9af90d19
CW		 3585 return ret;
3586
5eac3ab4
CW		 3587 /* First attempt, just clear anything that is purgeable.
3588 * Second attempt, clear the entire GTT.
3589 */
3590 ret = i915_gem_evict_everything(dev, retry == 0);
9af90d19
CW		 3591 if (ret)
3592 return ret;
40a5f0de 3593
5eac3ab4
CW		 3594 retry++;
3595 } while (1);
40a5f0de
EA		 3596}
3597
bcf50e27
CW		 3598static int
3599i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
3600 struct drm_file *file,
05394f39 3601 struct drm_i915_gem_object **object_list,
bcf50e27
CW		 3602 struct drm_i915_gem_exec_object2 *exec_list,
3603 int count)
3604{
3605 struct drm_i915_gem_relocation_entry *reloc;
3606 int i, total, ret;
3607
05394f39
CW		 3608 for (i = 0; i < count; i++)
3609 object_list[i]->in_execbuffer = false;
bcf50e27
CW		 3610
3611 mutex_unlock(&dev->struct_mutex);
3612
3613 total = 0;
3614 for (i = 0; i < count; i++)
3615 total += exec_list[i].relocation_count;
3616
3617 reloc = drm_malloc_ab(total, sizeof(*reloc));
3618 if (reloc == NULL) {
3619 mutex_lock(&dev->struct_mutex);
3620 return -ENOMEM;
3621 }
3622
3623 total = 0;
3624 for (i = 0; i < count; i++) {
3625 struct drm_i915_gem_relocation_entry __user *user_relocs;
3626
3627 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3628
3629 if (copy_from_user(reloc+total, user_relocs,
3630 exec_list[i].relocation_count *
3631 sizeof(*reloc))) {
3632 ret = -EFAULT;
3633 mutex_lock(&dev->struct_mutex);
3634 goto err;
3635 }
3636
3637 total += exec_list[i].relocation_count;
3638 }
3639
3640 ret = i915_mutex_lock_interruptible(dev);
3641 if (ret) {
3642 mutex_lock(&dev->struct_mutex);
3643 goto err;
3644 }
3645
3646 ret = i915_gem_execbuffer_reserve(dev, file,
3647 object_list, exec_list,
3648 count);
3649 if (ret)
3650 goto err;
3651
3652 total = 0;
3653 for (i = 0; i < count; i++) {
05394f39 3654 struct drm_i915_gem_object *obj = object_list[i];
bcf50e27
CW		 3655 obj->base.pending_read_domains = 0;
3656 obj->base.pending_write_domain = 0;
3657 ret = i915_gem_execbuffer_relocate_object_slow(obj, file,
3658 &exec_list[i],
3659 reloc + total);
3660 if (ret)
3661 goto err;
3662
3663 total += exec_list[i].relocation_count;
3664 }
3665
3666 /* Leave the user relocations as are, this is the painfully slow path,
3667 * and we want to avoid the complication of dropping the lock whilst
3668 * having buffers reserved in the aperture and so causing spurious
3669 * ENOSPC for random operations.
3670 */
3671
3672err:
3673 drm_free_large(reloc);
3674 return ret;
3675}
3676
13b29289
CW		 3677static int
3678i915_gem_execbuffer_move_to_gpu(struct drm_device *dev,
3679 struct drm_file *file,
3680 struct intel_ring_buffer *ring,
05394f39 3681 struct drm_i915_gem_object **objects,
13b29289
CW		 3682 int count)
3683{
0f8c6d7c 3684 struct change_domains cd;
13b29289
CW		 3685 int ret, i;
3686
0f8c6d7c
CW		 3687 cd.invalidate_domains = 0;
3688 cd.flush_domains = 0;
3689 cd.flush_rings = 0;
13b29289 3690 for (i = 0; i < count; i++)
0f8c6d7c 3691 i915_gem_object_set_to_gpu_domain(objects[i], ring, &cd);
13b29289 3692
0f8c6d7c 3693 if (cd.invalidate_domains | cd.flush_domains) {
13b29289
CW		 3694#if WATCH_EXEC
3695 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3696 __func__,
0f8c6d7c
CW		 3697 cd.invalidate_domains,
3698 cd.flush_domains);
13b29289 3699#endif
05394f39 3700 i915_gem_flush(dev,
0f8c6d7c
CW		 3701 cd.invalidate_domains,
3702 cd.flush_domains,
3703 cd.flush_rings);
13b29289
CW		 3704 }
3705
3706 for (i = 0; i < count; i++) {
05394f39 3707 struct drm_i915_gem_object *obj = objects[i];
13b29289
CW		 3708 /* XXX replace with semaphores */
3709 if (obj->ring && ring != obj->ring) {
05394f39 3710 ret = i915_gem_object_wait_rendering(obj, true);
13b29289
CW		 3711 if (ret)
3712 return ret;
3713 }
3714 }
3715
3716 return 0;
3717}
3718
673a394b
EA		 3719/* Throttle our rendering by waiting until the ring has completed our requests
3720 * emitted over 20 msec ago.
3721 *
b962442e
EA		 3722 * Note that if we were to use the current jiffies each time around the loop,
3723 * we wouldn't escape the function with any frames outstanding if the time to
3724 * render a frame was over 20ms.
3725 *
673a394b
EA		 3726 * This should get us reasonable parallelism between CPU and GPU but also
3727 * relatively low latency when blocking on a particular request to finish.
3728 */
40a5f0de 3729static int
f787a5f5 3730i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
40a5f0de 3731{
f787a5f5
CW		 3732 struct drm_i915_private *dev_priv = dev->dev_private;
3733 struct drm_i915_file_private *file_priv = file->driver_priv;
b962442e 3734 unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
f787a5f5
CW		 3735 struct drm_i915_gem_request *request;
3736 struct intel_ring_buffer *ring = NULL;
3737 u32 seqno = 0;
3738 int ret;
93533c29 3739
1c25595f 3740 spin_lock(&file_priv->mm.lock);
f787a5f5 3741 list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
b962442e
EA		 3742 if (time_after_eq(request->emitted_jiffies, recent_enough))
3743 break;
40a5f0de 3744
f787a5f5
CW		 3745 ring = request->ring;
3746 seqno = request->seqno;
b962442e 3747 }
1c25595f 3748 spin_unlock(&file_priv->mm.lock);
40a5f0de 3749
f787a5f5
CW		 3750 if (seqno == 0)
3751 return 0;
2bc43b5c 3752
f787a5f5 3753 ret = 0;
78501eac 3754 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
f787a5f5
CW		 3755 /* And wait for the seqno passing without holding any locks and
3756 * causing extra latency for others. This is safe as the irq
3757 * generation is designed to be run atomically and so is
3758 * lockless.
3759 */
78501eac 3760 ring->user_irq_get(ring);
f787a5f5 3761 ret = wait_event_interruptible(ring->irq_queue,
78501eac 3762 i915_seqno_passed(ring->get_seqno(ring), seqno)
f787a5f5 3763 || atomic_read(&dev_priv->mm.wedged));
78501eac 3764 ring->user_irq_put(ring);
40a5f0de 3765
f787a5f5
CW		 3766 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3767 ret = -EIO;
40a5f0de
EA		 3768 }
3769
f787a5f5
CW		 3770 if (ret == 0)
3771 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
40a5f0de
EA		 3772
3773 return ret;
3774}
3775
83d60795 3776static int
2549d6c2
CW		 3777i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec,
3778 uint64_t exec_offset)
83d60795
CW		 3779{
3780 uint32_t exec_start, exec_len;
3781
3782 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3783 exec_len = (uint32_t) exec->batch_len;
3784
3785 if ((exec_start | exec_len) & 0x7)
3786 return -EINVAL;
3787
3788 if (!exec_start)
3789 return -EINVAL;
3790
3791 return 0;
3792}
3793
6b95a207 3794static int
2549d6c2
CW		 3795validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
3796 int count)
6b95a207 3797{
2549d6c2 3798 int i;
6b95a207 3799
2549d6c2
CW		 3800 for (i = 0; i < count; i++) {
3801 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
d1d78830 3802 int length; /* limited by fault_in_pages_readable() */
6b95a207 3803
d1d78830
CW		 3804 /* First check for malicious input causing overflow */
3805 if (exec[i].relocation_count >
3806 INT_MAX / sizeof(struct drm_i915_gem_relocation_entry))
3807 return -EINVAL;
6b95a207 3808
d1d78830
CW		 3809 length = exec[i].relocation_count *
3810 sizeof(struct drm_i915_gem_relocation_entry);
2549d6c2
CW		 3811 if (!access_ok(VERIFY_READ, ptr, length))
3812 return -EFAULT;
40a5f0de 3813
b5dc608c
CW		 3814 /* we may also need to update the presumed offsets */
3815 if (!access_ok(VERIFY_WRITE, ptr, length))
3816 return -EFAULT;
3817
2549d6c2
CW		 3818 if (fault_in_pages_readable(ptr, length))
3819 return -EFAULT;
6b95a207 3820 }
6b95a207 3821
83d60795 3822 return 0;
6b95a207
KH		 3823}
3824
8dc5d147 3825static int
76446cac 3826i915_gem_do_execbuffer(struct drm_device *dev, void *data,
9af90d19 3827 struct drm_file *file,
76446cac
JB		 3828 struct drm_i915_gem_execbuffer2 *args,
3829 struct drm_i915_gem_exec_object2 *exec_list)
673a394b
EA		 3830{
3831 drm_i915_private_t *dev_priv = dev->dev_private;
05394f39
CW		 3832 struct drm_i915_gem_object **object_list = NULL;
3833 struct drm_i915_gem_object *batch_obj;
201361a5 3834 struct drm_clip_rect *cliprects = NULL;
8dc5d147 3835 struct drm_i915_gem_request *request = NULL;
9af90d19 3836 int ret, i, flips;
673a394b 3837 uint64_t exec_offset;
673a394b 3838
852835f3
ZN		 3839 struct intel_ring_buffer *ring = NULL;
3840
30dbf0c0
CW		 3841 ret = i915_gem_check_is_wedged(dev);
3842 if (ret)
3843 return ret;
3844
2549d6c2
CW		 3845 ret = validate_exec_list(exec_list, args->buffer_count);
3846 if (ret)
3847 return ret;
3848
673a394b
EA		 3849#if WATCH_EXEC
3850 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3851 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3852#endif
549f7365
CW		 3853 switch (args->flags & I915_EXEC_RING_MASK) {
3854 case I915_EXEC_DEFAULT:
3855 case I915_EXEC_RENDER:
3856 ring = &dev_priv->render_ring;
3857 break;
3858 case I915_EXEC_BSD:
d1b851fc 3859 if (!HAS_BSD(dev)) {
549f7365 3860 DRM_ERROR("execbuf with invalid ring (BSD)\n");
d1b851fc
ZN		 3861 return -EINVAL;
3862 }
3863 ring = &dev_priv->bsd_ring;
549f7365
CW		 3864 break;
3865 case I915_EXEC_BLT:
3866 if (!HAS_BLT(dev)) {
3867 DRM_ERROR("execbuf with invalid ring (BLT)\n");
3868 return -EINVAL;
3869 }
3870 ring = &dev_priv->blt_ring;
3871 break;
3872 default:
3873 DRM_ERROR("execbuf with unknown ring: %d\n",
3874 (int)(args->flags & I915_EXEC_RING_MASK));
3875 return -EINVAL;
d1b851fc
ZN		 3876 }
3877
4f481ed2
EA		 3878 if (args->buffer_count < 1) {
3879 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3880 return -EINVAL;
3881 }
c8e0f93a 3882 object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
76446cac
JB		 3883 if (object_list == NULL) {
3884 DRM_ERROR("Failed to allocate object list for %d buffers\n",
673a394b
EA		 3885 args->buffer_count);
3886 ret = -ENOMEM;
3887 goto pre_mutex_err;
3888 }
673a394b 3889
201361a5 3890 if (args->num_cliprects != 0) {
9a298b2a
EA		 3891 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3892 GFP_KERNEL);
a40e8d31
OA		 3893 if (cliprects == NULL) {
3894 ret = -ENOMEM;
201361a5 3895 goto pre_mutex_err;
a40e8d31 3896 }
201361a5
EA		 3897
3898 ret = copy_from_user(cliprects,
3899 (struct drm_clip_rect __user *)
3900 (uintptr_t) args->cliprects_ptr,
3901 sizeof(*cliprects) * args->num_cliprects);
3902 if (ret != 0) {
3903 DRM_ERROR("copy %d cliprects failed: %d\n",
3904 args->num_cliprects, ret);
c877cdce 3905 ret = -EFAULT;
201361a5
EA		 3906 goto pre_mutex_err;
3907 }
3908 }
3909
8dc5d147
CW		 3910 request = kzalloc(sizeof(*request), GFP_KERNEL);
3911 if (request == NULL) {
3912 ret = -ENOMEM;
40a5f0de 3913 goto pre_mutex_err;
8dc5d147 3914 }
40a5f0de 3915
76c1dec1
CW		 3916 ret = i915_mutex_lock_interruptible(dev);
3917 if (ret)
a198bc80 3918 goto pre_mutex_err;
673a394b
EA		 3919
3920 if (dev_priv->mm.suspended) {
673a394b 3921 mutex_unlock(&dev->struct_mutex);
a198bc80
CW		 3922 ret = -EBUSY;
3923 goto pre_mutex_err;
673a394b
EA		 3924 }
3925
ac94a962 3926 /* Look up object handles */
673a394b 3927 for (i = 0; i < args->buffer_count; i++) {
05394f39 3928 struct drm_i915_gem_object *obj;
7e318e18 3929
05394f39
CW		 3930 obj = to_intel_bo (drm_gem_object_lookup(dev, file,
3931 exec_list[i].handle));
3932 if (obj == NULL) {
673a394b
EA		 3933 DRM_ERROR("Invalid object handle %d at index %d\n",
3934 exec_list[i].handle, i);
0ce907f8 3935 /* prevent error path from reading uninitialized data */
05394f39 3936 args->buffer_count = i;
bf79cb91 3937 ret = -ENOENT;
673a394b
EA		 3938 goto err;
3939 }
05394f39 3940 object_list[i] = obj;
b70d11da 3941
05394f39 3942 if (obj->in_execbuffer) {
b70d11da 3943 DRM_ERROR("Object %p appears more than once in object list\n",
05394f39 3944 obj);
0ce907f8
CW		 3945 /* prevent error path from reading uninitialized data */
3946 args->buffer_count = i + 1;
bf79cb91 3947 ret = -EINVAL;
b70d11da
KH		 3948 goto err;
3949 }
05394f39 3950 obj->in_execbuffer = true;
ac94a962 3951 }
673a394b 3952
9af90d19 3953 /* Move the objects en-masse into the GTT, evicting if necessary. */
bcf50e27
CW		 3954 ret = i915_gem_execbuffer_reserve(dev, file,
3955 object_list, exec_list,
3956 args->buffer_count);
9af90d19
CW		 3957 if (ret)
3958 goto err;
ac94a962 3959
9af90d19 3960 /* The objects are in their final locations, apply the relocations. */
bcf50e27
CW		 3961 ret = i915_gem_execbuffer_relocate(dev, file,
3962 object_list, exec_list,
3963 args->buffer_count);
3964 if (ret) {
3965 if (ret == -EFAULT) {
3966 ret = i915_gem_execbuffer_relocate_slow(dev, file,
3967 object_list,
3968 exec_list,
3969 args->buffer_count);
3970 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
3971 }
9af90d19 3972 if (ret)
ac94a962 3973 goto err;
673a394b
EA		 3974 }
3975
3976 /* Set the pending read domains for the batch buffer to COMMAND */
3977 batch_obj = object_list[args->buffer_count-1];
05394f39 3978 if (batch_obj->base.pending_write_domain) {
5f26a2c7
CW		 3979 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3980 ret = -EINVAL;
3981 goto err;
3982 }
05394f39 3983 batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
673a394b 3984
9af90d19 3985 /* Sanity check the batch buffer */
05394f39 3986 exec_offset = batch_obj->gtt_offset;
9af90d19 3987 ret = i915_gem_check_execbuffer(args, exec_offset);
83d60795
CW		 3988 if (ret != 0) {
3989 DRM_ERROR("execbuf with invalid offset/length\n");
3990 goto err;
3991 }
3992
13b29289
CW		 3993 ret = i915_gem_execbuffer_move_to_gpu(dev, file, ring,
3994 object_list, args->buffer_count);
3995 if (ret)
3996 goto err;
673a394b 3997
673a394b
EA		 3998#if WATCH_COHERENCY
3999 for (i = 0; i < args->buffer_count; i++) {
4000 i915_gem_object_check_coherency(object_list[i],
4001 exec_list[i].handle);
4002 }
4003#endif
4004
673a394b 4005#if WATCH_EXEC
6911a9b8 4006 i915_gem_dump_object(batch_obj,
673a394b
EA		 4007 args->batch_len,
4008 __func__,
4009 ~0);
4010#endif
4011
e59f2bac
CW		 4012 /* Check for any pending flips. As we only maintain a flip queue depth
4013 * of 1, we can simply insert a WAIT for the next display flip prior
4014 * to executing the batch and avoid stalling the CPU.
4015 */
4016 flips = 0;
4017 for (i = 0; i < args->buffer_count; i++) {
05394f39
CW		 4018 if (object_list[i]->base.write_domain)
4019 flips |= atomic_read(&object_list[i]->pending_flip);
e59f2bac
CW		 4020 }
4021 if (flips) {
4022 int plane, flip_mask;
4023
4024 for (plane = 0; flips >> plane; plane++) {
4025 if (((flips >> plane) & 1) == 0)
4026 continue;
4027
4028 if (plane)
4029 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
4030 else
4031 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
4032
e1f99ce6
CW		 4033 ret = intel_ring_begin(ring, 2);
4034 if (ret)
4035 goto err;
4036
78501eac
CW		 4037 intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
4038 intel_ring_emit(ring, MI_NOOP);
4039 intel_ring_advance(ring);
e59f2bac
CW		 4040 }
4041 }
4042
673a394b 4043 /* Exec the batchbuffer */
78501eac 4044 ret = ring->dispatch_execbuffer(ring, args, cliprects, exec_offset);
673a394b
EA		 4045 if (ret) {
4046 DRM_ERROR("dispatch failed %d\n", ret);
4047 goto err;
4048 }
4049
673a394b 4050 for (i = 0; i < args->buffer_count; i++) {
05394f39 4051 struct drm_i915_gem_object *obj = object_list[i];
673a394b 4052
05394f39
CW		 4053 obj->base.read_domains = obj->base.pending_read_domains;
4054 obj->base.write_domain = obj->base.pending_write_domain;
7e318e18 4055
617dbe27 4056 i915_gem_object_move_to_active(obj, ring);
05394f39
CW		 4057 if (obj->base.write_domain) {
4058 obj->dirty = 1;
4059 list_move_tail(&obj->gpu_write_list,
64193406 4060 &ring->gpu_write_list);
7e318e18
CW		 4061 intel_mark_busy(dev, obj);
4062 }
4063
4064 trace_i915_gem_object_change_domain(obj,
05394f39
CW		 4065 obj->base.read_domains,
4066 obj->base.write_domain);
673a394b 4067 }
673a394b 4068
7e318e18
CW		 4069 /*
4070 * Ensure that the commands in the batch buffer are
4071 * finished before the interrupt fires
4072 */
4073 i915_retire_commands(dev, ring);
4074
3cce469c 4075 if (i915_add_request(dev, file, request, ring))
5d97eb69 4076 i915_gem_next_request_seqno(dev, ring);
3cce469c
CW		 4077 else
4078 request = NULL;
673a394b 4079
673a394b 4080err:
b70d11da 4081 for (i = 0; i < args->buffer_count; i++) {
05394f39
CW		 4082 object_list[i]->in_execbuffer = false;
4083 drm_gem_object_unreference(&object_list[i]->base);
b70d11da 4084 }
673a394b 4085
673a394b
EA		 4086 mutex_unlock(&dev->struct_mutex);
4087
93533c29 4088pre_mutex_err:
8e7d2b2c 4089 drm_free_large(object_list);
9a298b2a 4090 kfree(cliprects);
8dc5d147 4091 kfree(request);
673a394b
EA		 4092
4093 return ret;
4094}
4095
76446cac
JB		 4096/*
4097 * Legacy execbuffer just creates an exec2 list from the original exec object
4098 * list array and passes it to the real function.
4099 */
4100int
4101i915_gem_execbuffer(struct drm_device *dev, void *data,
05394f39 4102 struct drm_file *file)
76446cac
JB		 4103{
4104 struct drm_i915_gem_execbuffer *args = data;
4105 struct drm_i915_gem_execbuffer2 exec2;
4106 struct drm_i915_gem_exec_object *exec_list = NULL;
4107 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4108 int ret, i;
4109
4110#if WATCH_EXEC
4111 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4112 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4113#endif
4114
4115 if (args->buffer_count < 1) {
4116 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
4117 return -EINVAL;
4118 }
4119
4120 /* Copy in the exec list from userland */
4121 exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
4122 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4123 if (exec_list == NULL || exec2_list == NULL) {
4124 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4125 args->buffer_count);
4126 drm_free_large(exec_list);
4127 drm_free_large(exec2_list);
4128 return -ENOMEM;
4129 }
4130 ret = copy_from_user(exec_list,
4131 (struct drm_i915_relocation_entry __user *)
4132 (uintptr_t) args->buffers_ptr,
4133 sizeof(*exec_list) * args->buffer_count);
4134 if (ret != 0) {
4135 DRM_ERROR("copy %d exec entries failed %d\n",
4136 args->buffer_count, ret);
4137 drm_free_large(exec_list);
4138 drm_free_large(exec2_list);
4139 return -EFAULT;
4140 }
4141
4142 for (i = 0; i < args->buffer_count; i++) {
4143 exec2_list[i].handle = exec_list[i].handle;
4144 exec2_list[i].relocation_count = exec_list[i].relocation_count;
4145 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
4146 exec2_list[i].alignment = exec_list[i].alignment;
4147 exec2_list[i].offset = exec_list[i].offset;
a6c45cf0 4148 if (INTEL_INFO(dev)->gen < 4)
76446cac
JB		 4149 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
4150 else
4151 exec2_list[i].flags = 0;
4152 }
4153
4154 exec2.buffers_ptr = args->buffers_ptr;
4155 exec2.buffer_count = args->buffer_count;
4156 exec2.batch_start_offset = args->batch_start_offset;
4157 exec2.batch_len = args->batch_len;
4158 exec2.DR1 = args->DR1;
4159 exec2.DR4 = args->DR4;
4160 exec2.num_cliprects = args->num_cliprects;
4161 exec2.cliprects_ptr = args->cliprects_ptr;
852835f3 4162 exec2.flags = I915_EXEC_RENDER;
76446cac 4163
05394f39 4164 ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
76446cac
JB		 4165 if (!ret) {
4166 /* Copy the new buffer offsets back to the user's exec list. */
4167 for (i = 0; i < args->buffer_count; i++)
4168 exec_list[i].offset = exec2_list[i].offset;
4169 /* ... and back out to userspace */
4170 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4171 (uintptr_t) args->buffers_ptr,
4172 exec_list,
4173 sizeof(*exec_list) * args->buffer_count);
4174 if (ret) {
4175 ret = -EFAULT;
4176 DRM_ERROR("failed to copy %d exec entries "
4177 "back to user (%d)\n",
4178 args->buffer_count, ret);
4179 }
76446cac
JB		 4180 }
4181
4182 drm_free_large(exec_list);
4183 drm_free_large(exec2_list);
4184 return ret;
4185}
4186
4187int
4188i915_gem_execbuffer2(struct drm_device *dev, void *data,
05394f39 4189 struct drm_file *file)
76446cac
JB		 4190{
4191 struct drm_i915_gem_execbuffer2 *args = data;
4192 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4193 int ret;
4194
4195#if WATCH_EXEC
4196 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4197 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4198#endif
4199
4200 if (args->buffer_count < 1) {
4201 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4202 return -EINVAL;
4203 }
4204
4205 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4206 if (exec2_list == NULL) {
4207 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4208 args->buffer_count);
4209 return -ENOMEM;
4210 }
4211 ret = copy_from_user(exec2_list,
4212 (struct drm_i915_relocation_entry __user *)
4213 (uintptr_t) args->buffers_ptr,
4214 sizeof(*exec2_list) * args->buffer_count);
4215 if (ret != 0) {
4216 DRM_ERROR("copy %d exec entries failed %d\n",
4217 args->buffer_count, ret);
4218 drm_free_large(exec2_list);
4219 return -EFAULT;
4220 }
4221
05394f39 4222 ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
76446cac
JB		 4223 if (!ret) {
4224 /* Copy the new buffer offsets back to the user's exec list. */
4225 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4226 (uintptr_t) args->buffers_ptr,
4227 exec2_list,
4228 sizeof(*exec2_list) * args->buffer_count);
4229 if (ret) {
4230 ret = -EFAULT;
4231 DRM_ERROR("failed to copy %d exec entries "
4232 "back to user (%d)\n",
4233 args->buffer_count, ret);
4234 }
4235 }
4236
4237 drm_free_large(exec2_list);
4238 return ret;
4239}
4240
673a394b 4241int
05394f39
CW		 4242i915_gem_object_pin(struct drm_i915_gem_object *obj,
4243 uint32_t alignment,
75e9e915 4244 bool map_and_fenceable)
673a394b 4245{
05394f39 4246 struct drm_device *dev = obj->base.dev;
f13d3f73 4247 struct drm_i915_private *dev_priv = dev->dev_private;
673a394b
EA		 4248 int ret;
4249
05394f39 4250 BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
75e9e915 4251 BUG_ON(map_and_fenceable && !map_and_fenceable);
23bc5982 4252 WARN_ON(i915_verify_lists(dev));
ac0c6b5a 4253
05394f39
CW		 4254 if (obj->gtt_space != NULL) {
4255 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
4256 (map_and_fenceable && !obj->map_and_fenceable)) {
4257 WARN(obj->pin_count,
ae7d49d8 4258 "bo is already pinned with incorrect alignment:"
75e9e915
DV		 4259 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
4260 " obj->map_and_fenceable=%d\n",
05394f39 4261 obj->gtt_offset, alignment,
75e9e915 4262 map_and_fenceable,
05394f39 4263 obj->map_and_fenceable);
ac0c6b5a
CW		 4264 ret = i915_gem_object_unbind(obj);
4265 if (ret)
4266 return ret;
4267 }
4268 }
4269
05394f39 4270 if (obj->gtt_space == NULL) {
a00b10c3 4271 ret = i915_gem_object_bind_to_gtt(obj, alignment,
75e9e915 4272 map_and_fenceable);
9731129c 4273 if (ret)
673a394b 4274 return ret;
22c344e9 4275 }
76446cac 4276
05394f39
CW		 4277 if (obj->pin_count++ == 0) {
4278 i915_gem_info_add_pin(dev_priv, obj, map_and_fenceable);
4279 if (!obj->active)
4280 list_move_tail(&obj->mm_list,
f13d3f73 4281 &dev_priv->mm.pinned_list);
673a394b 4282 }
05394f39 4283 BUG_ON(!obj->pin_mappable && map_and_fenceable);
673a394b 4284
23bc5982 4285 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 4286 return 0;
4287}
4288
4289void
05394f39 4290i915_gem_object_unpin(struct drm_i915_gem_object *obj)
673a394b 4291{
05394f39 4292 struct drm_device *dev = obj->base.dev;
673a394b 4293 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 4294
23bc5982 4295 WARN_ON(i915_verify_lists(dev));
05394f39
CW		 4296 BUG_ON(obj->pin_count == 0);
4297 BUG_ON(obj->gtt_space == NULL);
673a394b 4298
05394f39
CW		 4299 if (--obj->pin_count == 0) {
4300 if (!obj->active)
4301 list_move_tail(&obj->mm_list,
673a394b 4302 &dev_priv->mm.inactive_list);
05394f39 4303 i915_gem_info_remove_pin(dev_priv, obj);
673a394b 4304 }
23bc5982 4305 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 4306}
4307
4308int
4309i915_gem_pin_ioctl(struct drm_device *dev, void *data,
05394f39 4310 struct drm_file *file)
673a394b
EA		 4311{
4312 struct drm_i915_gem_pin *args = data;
05394f39 4313 struct drm_i915_gem_object *obj;
673a394b
EA		 4314 int ret;
4315
1d7cfea1
CW		 4316 ret = i915_mutex_lock_interruptible(dev);
4317 if (ret)
4318 return ret;
673a394b 4319
05394f39 4320 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
673a394b 4321 if (obj == NULL) {
1d7cfea1
CW		 4322 ret = -ENOENT;
4323 goto unlock;
673a394b 4324 }
673a394b 4325
05394f39 4326 if (obj->madv != I915_MADV_WILLNEED) {
bb6baf76 4327 DRM_ERROR("Attempting to pin a purgeable buffer\n");
1d7cfea1
CW		 4328 ret = -EINVAL;
4329 goto out;
3ef94daa
CW		 4330 }
4331
05394f39 4332 if (obj->pin_filp != NULL && obj->pin_filp != file) {
79e53945
JB		 4333 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4334 args->handle);
1d7cfea1
CW		 4335 ret = -EINVAL;
4336 goto out;
79e53945
JB		 4337 }
4338
05394f39
CW		 4339 obj->user_pin_count++;
4340 obj->pin_filp = file;
4341 if (obj->user_pin_count == 1) {
75e9e915 4342 ret = i915_gem_object_pin(obj, args->alignment, true);
1d7cfea1
CW		 4343 if (ret)
4344 goto out;
673a394b
EA		 4345 }
4346
4347 /* XXX - flush the CPU caches for pinned objects
4348 * as the X server doesn't manage domains yet
4349 */
e47c68e9 4350 i915_gem_object_flush_cpu_write_domain(obj);
05394f39 4351 args->offset = obj->gtt_offset;
1d7cfea1 4352out:
05394f39 4353 drm_gem_object_unreference(&obj->base);
1d7cfea1 4354unlock:
673a394b 4355 mutex_unlock(&dev->struct_mutex);
1d7cfea1 4356 return ret;
673a394b
EA		 4357}
4358
4359int
4360i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
05394f39 4361 struct drm_file *file)
673a394b
EA		 4362{
4363 struct drm_i915_gem_pin *args = data;
05394f39 4364 struct drm_i915_gem_object *obj;
76c1dec1 4365 int ret;
673a394b 4366
1d7cfea1
CW		 4367 ret = i915_mutex_lock_interruptible(dev);
4368 if (ret)
4369 return ret;
673a394b 4370
05394f39 4371 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
673a394b 4372 if (obj == NULL) {
1d7cfea1
CW		 4373 ret = -ENOENT;
4374 goto unlock;
673a394b 4375 }
76c1dec1 4376
05394f39 4377 if (obj->pin_filp != file) {
79e53945
JB		 4378 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4379 args->handle);
1d7cfea1
CW		 4380 ret = -EINVAL;
4381 goto out;
79e53945 4382 }
05394f39
CW		 4383 obj->user_pin_count--;
4384 if (obj->user_pin_count == 0) {
4385 obj->pin_filp = NULL;
79e53945
JB		 4386 i915_gem_object_unpin(obj);
4387 }
673a394b 4388
1d7cfea1 4389out:
05394f39 4390 drm_gem_object_unreference(&obj->base);
1d7cfea1 4391unlock:
673a394b 4392 mutex_unlock(&dev->struct_mutex);
1d7cfea1 4393 return ret;
673a394b
EA		 4394}
4395
4396int
4397i915_gem_busy_ioctl(struct drm_device *dev, void *data,
05394f39 4398 struct drm_file *file)
673a394b
EA		 4399{
4400 struct drm_i915_gem_busy *args = data;
05394f39 4401 struct drm_i915_gem_object *obj;
30dbf0c0
CW		 4402 int ret;
4403
76c1dec1 4404 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 4405 if (ret)
76c1dec1 4406 return ret;
673a394b 4407
05394f39 4408 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
673a394b 4409 if (obj == NULL) {
1d7cfea1
CW		 4410 ret = -ENOENT;
4411 goto unlock;
673a394b 4412 }
d1b851fc 4413
0be555b6
CW		 4414 /* Count all active objects as busy, even if they are currently not used
4415 * by the gpu. Users of this interface expect objects to eventually
4416 * become non-busy without any further actions, therefore emit any
4417 * necessary flushes here.
c4de0a5d 4418 */
05394f39 4419 args->busy = obj->active;
0be555b6
CW		 4420 if (args->busy) {
4421 /* Unconditionally flush objects, even when the gpu still uses this
4422 * object. Userspace calling this function indicates that it wants to
4423 * use this buffer rather sooner than later, so issuing the required
4424 * flush earlier is beneficial.
4425 */
05394f39
CW		 4426 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS)
4427 i915_gem_flush_ring(dev, obj->ring,
4428 0, obj->base.write_domain);
0be555b6
CW		 4429
4430 /* Update the active list for the hardware's current position.
4431 * Otherwise this only updates on a delayed timer or when irqs
4432 * are actually unmasked, and our working set ends up being
4433 * larger than required.
4434 */
05394f39 4435 i915_gem_retire_requests_ring(dev, obj->ring);
0be555b6 4436
05394f39 4437 args->busy = obj->active;
0be555b6 4438 }
673a394b 4439
05394f39 4440 drm_gem_object_unreference(&obj->base);
1d7cfea1 4441unlock:
673a394b 4442 mutex_unlock(&dev->struct_mutex);
1d7cfea1 4443 return ret;
673a394b
EA		 4444}
4445
4446int
4447i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4448 struct drm_file *file_priv)
4449{
4450 return i915_gem_ring_throttle(dev, file_priv);
4451}
4452
3ef94daa
CW		 4453int
4454i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4455 struct drm_file *file_priv)
4456{
4457 struct drm_i915_gem_madvise *args = data;
05394f39 4458 struct drm_i915_gem_object *obj;
76c1dec1 4459 int ret;
3ef94daa
CW		 4460
4461 switch (args->madv) {
4462 case I915_MADV_DONTNEED:
4463 case I915_MADV_WILLNEED:
4464 break;
4465 default:
4466 return -EINVAL;
4467 }
4468
1d7cfea1
CW		 4469 ret = i915_mutex_lock_interruptible(dev);
4470 if (ret)
4471 return ret;
4472
05394f39 4473 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
3ef94daa 4474 if (obj == NULL) {
1d7cfea1
CW		 4475 ret = -ENOENT;
4476 goto unlock;
3ef94daa 4477 }
3ef94daa 4478
05394f39 4479 if (obj->pin_count) {
1d7cfea1
CW		 4480 ret = -EINVAL;
4481 goto out;
3ef94daa
CW		 4482 }
4483
05394f39
CW		 4484 if (obj->madv != __I915_MADV_PURGED)
4485 obj->madv = args->madv;
3ef94daa 4486
2d7ef395 4487 /* if the object is no longer bound, discard its backing storage */
05394f39
CW		 4488 if (i915_gem_object_is_purgeable(obj) &&
4489 obj->gtt_space == NULL)
2d7ef395
CW		 4490 i915_gem_object_truncate(obj);
4491
05394f39 4492 args->retained = obj->madv != __I915_MADV_PURGED;
bb6baf76 4493
1d7cfea1 4494out:
05394f39 4495 drm_gem_object_unreference(&obj->base);
1d7cfea1 4496unlock:
3ef94daa 4497 mutex_unlock(&dev->struct_mutex);
1d7cfea1 4498 return ret;
3ef94daa
CW		 4499}
4500
05394f39
CW		 4501struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
4502 size_t size)
ac52bc56 4503{
73aa808f 4504 struct drm_i915_private *dev_priv = dev->dev_private;
c397b908 4505 struct drm_i915_gem_object *obj;
ac52bc56 4506
c397b908
DV		 4507 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4508 if (obj == NULL)
4509 return NULL;
673a394b 4510
c397b908
DV		 4511 if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4512 kfree(obj);
4513 return NULL;
4514 }
673a394b 4515
73aa808f
CW		 4516 i915_gem_info_add_obj(dev_priv, size);
4517
c397b908
DV		 4518 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4519 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
673a394b 4520
c397b908 4521 obj->agp_type = AGP_USER_MEMORY;
62b8b215 4522 obj->base.driver_private = NULL;
c397b908 4523 obj->fence_reg = I915_FENCE_REG_NONE;
69dc4987 4524 INIT_LIST_HEAD(&obj->mm_list);
93a37f20 4525 INIT_LIST_HEAD(&obj->gtt_list);
69dc4987 4526 INIT_LIST_HEAD(&obj->ring_list);
c397b908 4527 INIT_LIST_HEAD(&obj->gpu_write_list);
c397b908 4528 obj->madv = I915_MADV_WILLNEED;
75e9e915
DV		 4529 /* Avoid an unnecessary call to unbind on the first bind. */
4530 obj->map_and_fenceable = true;
de151cf6 4531
05394f39 4532 return obj;
c397b908
DV		 4533}
4534
4535int i915_gem_init_object(struct drm_gem_object *obj)
4536{
4537 BUG();
de151cf6 4538
673a394b
EA		 4539 return 0;
4540}
4541
05394f39 4542static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
673a394b 4543{
05394f39 4544 struct drm_device *dev = obj->base.dev;
be72615b 4545 drm_i915_private_t *dev_priv = dev->dev_private;
be72615b 4546 int ret;
673a394b 4547
be72615b
CW		 4548 ret = i915_gem_object_unbind(obj);
4549 if (ret == -ERESTARTSYS) {
05394f39 4550 list_move(&obj->mm_list,
be72615b
CW		 4551 &dev_priv->mm.deferred_free_list);
4552 return;
4553 }
673a394b 4554
05394f39 4555 if (obj->base.map_list.map)
7e616158 4556 i915_gem_free_mmap_offset(obj);
de151cf6 4557
05394f39
CW		 4558 drm_gem_object_release(&obj->base);
4559 i915_gem_info_remove_obj(dev_priv, obj->base.size);
c397b908 4560
05394f39
CW		 4561 kfree(obj->page_cpu_valid);
4562 kfree(obj->bit_17);
4563 kfree(obj);
673a394b
EA		 4564}
4565
05394f39 4566void i915_gem_free_object(struct drm_gem_object *gem_obj)
be72615b 4567{
05394f39
CW		 4568 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4569 struct drm_device *dev = obj->base.dev;
be72615b
CW		 4570
4571 trace_i915_gem_object_destroy(obj);
4572
05394f39 4573 while (obj->pin_count > 0)
be72615b
CW		 4574 i915_gem_object_unpin(obj);
4575
05394f39 4576 if (obj->phys_obj)
be72615b
CW		 4577 i915_gem_detach_phys_object(dev, obj);
4578
4579 i915_gem_free_object_tail(obj);
4580}
4581
29105ccc
CW		 4582int
4583i915_gem_idle(struct drm_device *dev)
4584{
4585 drm_i915_private_t *dev_priv = dev->dev_private;
4586 int ret;
28dfe52a 4587
29105ccc 4588 mutex_lock(&dev->struct_mutex);
1c5d22f7 4589
87acb0a5 4590 if (dev_priv->mm.suspended) {
29105ccc
CW		 4591 mutex_unlock(&dev->struct_mutex);
4592 return 0;
28dfe52a
EA		 4593 }
4594
29105ccc 4595 ret = i915_gpu_idle(dev);
6dbe2772
KP		 4596 if (ret) {
4597 mutex_unlock(&dev->struct_mutex);
673a394b 4598 return ret;
6dbe2772 4599 }
673a394b 4600
29105ccc
CW		 4601 /* Under UMS, be paranoid and evict. */
4602 if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
5eac3ab4 4603 ret = i915_gem_evict_inactive(dev, false);
29105ccc
CW		 4604 if (ret) {
4605 mutex_unlock(&dev->struct_mutex);
4606 return ret;
4607 }
4608 }
4609
4610 /* Hack! Don't let anybody do execbuf while we don't control the chip.
4611 * We need to replace this with a semaphore, or something.
4612 * And not confound mm.suspended!
4613 */
4614 dev_priv->mm.suspended = 1;
bc0c7f14 4615 del_timer_sync(&dev_priv->hangcheck_timer);
29105ccc
CW		 4616
4617 i915_kernel_lost_context(dev);
6dbe2772 4618 i915_gem_cleanup_ringbuffer(dev);
29105ccc 4619
6dbe2772
KP		 4620 mutex_unlock(&dev->struct_mutex);
4621
29105ccc
CW		 4622 /* Cancel the retire work handler, which should be idle now. */
4623 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4624
673a394b
EA		 4625 return 0;
4626}
4627
e552eb70
JB		 4628/*
4629 * 965+ support PIPE_CONTROL commands, which provide finer grained control
4630 * over cache flushing.
4631 */
8187a2b7 4632static int
e552eb70
JB		 4633i915_gem_init_pipe_control(struct drm_device *dev)
4634{
4635 drm_i915_private_t *dev_priv = dev->dev_private;
05394f39 4636 struct drm_i915_gem_object *obj;
e552eb70
JB		 4637 int ret;
4638
34dc4d44 4639 obj = i915_gem_alloc_object(dev, 4096);
e552eb70
JB		 4640 if (obj == NULL) {
4641 DRM_ERROR("Failed to allocate seqno page\n");
4642 ret = -ENOMEM;
4643 goto err;
4644 }
05394f39 4645 obj->agp_type = AGP_USER_CACHED_MEMORY;
e552eb70 4646
75e9e915 4647 ret = i915_gem_object_pin(obj, 4096, true);
e552eb70
JB		 4648 if (ret)
4649 goto err_unref;
4650
05394f39
CW		 4651 dev_priv->seqno_gfx_addr = obj->gtt_offset;
4652 dev_priv->seqno_page = kmap(obj->pages[0]);
e552eb70
JB		 4653 if (dev_priv->seqno_page == NULL)
4654 goto err_unpin;
4655
4656 dev_priv->seqno_obj = obj;
4657 memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4658
4659 return 0;
4660
4661err_unpin:
4662 i915_gem_object_unpin(obj);
4663err_unref:
05394f39 4664 drm_gem_object_unreference(&obj->base);
e552eb70
JB		 4665err:
4666 return ret;
4667}
4668
8187a2b7
ZN		 4669
4670static void
e552eb70
JB		 4671i915_gem_cleanup_pipe_control(struct drm_device *dev)
4672{
4673 drm_i915_private_t *dev_priv = dev->dev_private;
05394f39 4674 struct drm_i915_gem_object *obj;
e552eb70
JB		 4675
4676 obj = dev_priv->seqno_obj;
05394f39 4677 kunmap(obj->pages[0]);
e552eb70 4678 i915_gem_object_unpin(obj);
05394f39 4679 drm_gem_object_unreference(&obj->base);
e552eb70
JB		 4680 dev_priv->seqno_obj = NULL;
4681
4682 dev_priv->seqno_page = NULL;
673a394b
EA		 4683}
4684
8187a2b7
ZN		 4685int
4686i915_gem_init_ringbuffer(struct drm_device *dev)
4687{
4688 drm_i915_private_t *dev_priv = dev->dev_private;
4689 int ret;
68f95ba9 4690
8187a2b7
ZN		 4691 if (HAS_PIPE_CONTROL(dev)) {
4692 ret = i915_gem_init_pipe_control(dev);
4693 if (ret)
4694 return ret;
4695 }
68f95ba9 4696
5c1143bb 4697 ret = intel_init_render_ring_buffer(dev);
68f95ba9
CW		 4698 if (ret)
4699 goto cleanup_pipe_control;
4700
4701 if (HAS_BSD(dev)) {
5c1143bb 4702 ret = intel_init_bsd_ring_buffer(dev);
68f95ba9
CW		 4703 if (ret)
4704 goto cleanup_render_ring;
d1b851fc 4705 }
68f95ba9 4706
549f7365
CW		 4707 if (HAS_BLT(dev)) {
4708 ret = intel_init_blt_ring_buffer(dev);
4709 if (ret)
4710 goto cleanup_bsd_ring;
4711 }
4712
6f392d54
CW		 4713 dev_priv->next_seqno = 1;
4714
68f95ba9
CW		 4715 return 0;
4716
549f7365 4717cleanup_bsd_ring:
78501eac 4718 intel_cleanup_ring_buffer(&dev_priv->bsd_ring);
68f95ba9 4719cleanup_render_ring:
78501eac 4720 intel_cleanup_ring_buffer(&dev_priv->render_ring);
68f95ba9
CW		 4721cleanup_pipe_control:
4722 if (HAS_PIPE_CONTROL(dev))
4723 i915_gem_cleanup_pipe_control(dev);
8187a2b7
ZN		 4724 return ret;
4725}
4726
4727void
4728i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4729{
4730 drm_i915_private_t *dev_priv = dev->dev_private;
4731
78501eac
CW		 4732 intel_cleanup_ring_buffer(&dev_priv->render_ring);
4733 intel_cleanup_ring_buffer(&dev_priv->bsd_ring);
4734 intel_cleanup_ring_buffer(&dev_priv->blt_ring);
8187a2b7
ZN		 4735 if (HAS_PIPE_CONTROL(dev))
4736 i915_gem_cleanup_pipe_control(dev);
4737}
4738
673a394b
EA		 4739int
4740i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4741 struct drm_file *file_priv)
4742{
4743 drm_i915_private_t *dev_priv = dev->dev_private;
4744 int ret;
4745
79e53945
JB		 4746 if (drm_core_check_feature(dev, DRIVER_MODESET))
4747 return 0;
4748
ba1234d1 4749 if (atomic_read(&dev_priv->mm.wedged)) {
673a394b 4750 DRM_ERROR("Reenabling wedged hardware, good luck\n");
ba1234d1 4751 atomic_set(&dev_priv->mm.wedged, 0);
673a394b
EA		 4752 }
4753
673a394b 4754 mutex_lock(&dev->struct_mutex);
9bb2d6f9
EA		 4755 dev_priv->mm.suspended = 0;
4756
4757 ret = i915_gem_init_ringbuffer(dev);
d816f6ac
WF		 4758 if (ret != 0) {
4759 mutex_unlock(&dev->struct_mutex);
9bb2d6f9 4760 return ret;
d816f6ac 4761 }
9bb2d6f9 4762
69dc4987 4763 BUG_ON(!list_empty(&dev_priv->mm.active_list));
852835f3 4764 BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
87acb0a5 4765 BUG_ON(!list_empty(&dev_priv->bsd_ring.active_list));
549f7365 4766 BUG_ON(!list_empty(&dev_priv->blt_ring.active_list));
673a394b
EA		 4767 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4768 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
852835f3 4769 BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
87acb0a5 4770 BUG_ON(!list_empty(&dev_priv->bsd_ring.request_list));
549f7365 4771 BUG_ON(!list_empty(&dev_priv->blt_ring.request_list));
673a394b 4772 mutex_unlock(&dev->struct_mutex);
dbb19d30 4773
5f35308b
CW		 4774 ret = drm_irq_install(dev);
4775 if (ret)
4776 goto cleanup_ringbuffer;
dbb19d30 4777
673a394b 4778 return 0;
5f35308b
CW		 4779
4780cleanup_ringbuffer:
4781 mutex_lock(&dev->struct_mutex);
4782 i915_gem_cleanup_ringbuffer(dev);
4783 dev_priv->mm.suspended = 1;
4784 mutex_unlock(&dev->struct_mutex);
4785
4786 return ret;
673a394b
EA		 4787}
4788
4789int
4790i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4791 struct drm_file *file_priv)
4792{
79e53945
JB		 4793 if (drm_core_check_feature(dev, DRIVER_MODESET))
4794 return 0;
4795
dbb19d30 4796 drm_irq_uninstall(dev);
e6890f6f 4797 return i915_gem_idle(dev);
673a394b
EA		 4798}
4799
4800void
4801i915_gem_lastclose(struct drm_device *dev)
4802{
4803 int ret;
673a394b 4804
e806b495
EA		 4805 if (drm_core_check_feature(dev, DRIVER_MODESET))
4806 return;
4807
6dbe2772
KP		 4808 ret = i915_gem_idle(dev);
4809 if (ret)
4810 DRM_ERROR("failed to idle hardware: %d\n", ret);
673a394b
EA		 4811}
4812
64193406
CW		 4813static void
4814init_ring_lists(struct intel_ring_buffer *ring)
4815{
4816 INIT_LIST_HEAD(&ring->active_list);
4817 INIT_LIST_HEAD(&ring->request_list);
4818 INIT_LIST_HEAD(&ring->gpu_write_list);
4819}
4820
673a394b
EA		 4821void
4822i915_gem_load(struct drm_device *dev)
4823{
b5aa8a0f 4824 int i;
673a394b
EA		 4825 drm_i915_private_t *dev_priv = dev->dev_private;
4826
69dc4987 4827 INIT_LIST_HEAD(&dev_priv->mm.active_list);
673a394b
EA		 4828 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4829 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
f13d3f73 4830 INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
a09ba7fa 4831 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
be72615b 4832 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
93a37f20 4833 INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
64193406
CW		 4834 init_ring_lists(&dev_priv->render_ring);
4835 init_ring_lists(&dev_priv->bsd_ring);
4836 init_ring_lists(&dev_priv->blt_ring);
007cc8ac
DV		 4837 for (i = 0; i < 16; i++)
4838 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
673a394b
EA		 4839 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4840 i915_gem_retire_work_handler);
30dbf0c0 4841 init_completion(&dev_priv->error_completion);
31169714 4842
94400120
DA		 4843 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4844 if (IS_GEN3(dev)) {
4845 u32 tmp = I915_READ(MI_ARB_STATE);
4846 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4847 /* arb state is a masked write, so set bit + bit in mask */
4848 tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4849 I915_WRITE(MI_ARB_STATE, tmp);
4850 }
4851 }
4852
de151cf6 4853 /* Old X drivers will take 0-2 for front, back, depth buffers */
b397c836
EA		 4854 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4855 dev_priv->fence_reg_start = 3;
de151cf6 4856
a6c45cf0 4857 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
de151cf6
JB		 4858 dev_priv->num_fence_regs = 16;
4859 else
4860 dev_priv->num_fence_regs = 8;
4861
b5aa8a0f 4862 /* Initialize fence registers to zero */
a6c45cf0
CW		 4863 switch (INTEL_INFO(dev)->gen) {
4864 case 6:
4865 for (i = 0; i < 16; i++)
4866 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), 0);
4867 break;
4868 case 5:
4869 case 4:
b5aa8a0f
GH		 4870 for (i = 0; i < 16; i++)
4871 I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
a6c45cf0
CW		 4872 break;
4873 case 3:
b5aa8a0f
GH		 4874 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4875 for (i = 0; i < 8; i++)
4876 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
a6c45cf0
CW		 4877 case 2:
4878 for (i = 0; i < 8; i++)
4879 I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4880 break;
b5aa8a0f 4881 }
673a394b 4882 i915_gem_detect_bit_6_swizzle(dev);
6b95a207 4883 init_waitqueue_head(&dev_priv->pending_flip_queue);
17250b71
CW		 4884
4885 dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
4886 dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
4887 register_shrinker(&dev_priv->mm.inactive_shrinker);
673a394b 4888}
71acb5eb
DA		 4889
4890/*
4891 * Create a physically contiguous memory object for this object
4892 * e.g. for cursor + overlay regs
4893 */
995b6762
CW		 4894static int i915_gem_init_phys_object(struct drm_device *dev,
4895 int id, int size, int align)
71acb5eb
DA		 4896{
4897 drm_i915_private_t *dev_priv = dev->dev_private;
4898 struct drm_i915_gem_phys_object *phys_obj;
4899 int ret;
4900
4901 if (dev_priv->mm.phys_objs[id - 1] || !size)
4902 return 0;
4903
9a298b2a 4904 phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
71acb5eb
DA		 4905 if (!phys_obj)
4906 return -ENOMEM;
4907
4908 phys_obj->id = id;
4909
6eeefaf3 4910 phys_obj->handle = drm_pci_alloc(dev, size, align);
71acb5eb
DA		 4911 if (!phys_obj->handle) {
4912 ret = -ENOMEM;
4913 goto kfree_obj;
4914 }
4915#ifdef CONFIG_X86
4916 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4917#endif
4918
4919 dev_priv->mm.phys_objs[id - 1] = phys_obj;
4920
4921 return 0;
4922kfree_obj:
9a298b2a 4923 kfree(phys_obj);
71acb5eb
DA		 4924 return ret;
4925}
4926
995b6762 4927static void i915_gem_free_phys_object(struct drm_device *dev, int id)
71acb5eb
DA		 4928{
4929 drm_i915_private_t *dev_priv = dev->dev_private;
4930 struct drm_i915_gem_phys_object *phys_obj;
4931
4932 if (!dev_priv->mm.phys_objs[id - 1])
4933 return;
4934
4935 phys_obj = dev_priv->mm.phys_objs[id - 1];
4936 if (phys_obj->cur_obj) {
4937 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4938 }
4939
4940#ifdef CONFIG_X86
4941 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4942#endif
4943 drm_pci_free(dev, phys_obj->handle);
4944 kfree(phys_obj);
4945 dev_priv->mm.phys_objs[id - 1] = NULL;
4946}
4947
4948void i915_gem_free_all_phys_object(struct drm_device *dev)
4949{
4950 int i;
4951
260883c8 4952 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
71acb5eb
DA		 4953 i915_gem_free_phys_object(dev, i);
4954}
4955
4956void i915_gem_detach_phys_object(struct drm_device *dev,
05394f39 4957 struct drm_i915_gem_object *obj)
71acb5eb 4958{
05394f39 4959 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
e5281ccd 4960 char *vaddr;
71acb5eb 4961 int i;
71acb5eb
DA		 4962 int page_count;
4963
05394f39 4964 if (!obj->phys_obj)
71acb5eb 4965 return;
05394f39 4966 vaddr = obj->phys_obj->handle->vaddr;
71acb5eb 4967
05394f39 4968 page_count = obj->base.size / PAGE_SIZE;
71acb5eb 4969 for (i = 0; i < page_count; i++) {
e5281ccd
CW		 4970 struct page *page = read_cache_page_gfp(mapping, i,
4971 GFP_HIGHUSER | __GFP_RECLAIMABLE);
4972 if (!IS_ERR(page)) {
4973 char *dst = kmap_atomic(page);
4974 memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
4975 kunmap_atomic(dst);
4976
4977 drm_clflush_pages(&page, 1);
4978
4979 set_page_dirty(page);
4980 mark_page_accessed(page);
4981 page_cache_release(page);
4982 }
71acb5eb 4983 }
40ce6575 4984 intel_gtt_chipset_flush();
d78b47b9 4985
05394f39
CW		 4986 obj->phys_obj->cur_obj = NULL;
4987 obj->phys_obj = NULL;
71acb5eb
DA		 4988}
4989
4990int
4991i915_gem_attach_phys_object(struct drm_device *dev,
05394f39 4992 struct drm_i915_gem_object *obj,
6eeefaf3
CW		 4993 int id,
4994 int align)
71acb5eb 4995{
05394f39 4996 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
71acb5eb 4997 drm_i915_private_t *dev_priv = dev->dev_private;
71acb5eb
DA		 4998 int ret = 0;
4999 int page_count;
5000 int i;
5001
5002 if (id > I915_MAX_PHYS_OBJECT)
5003 return -EINVAL;
5004
05394f39
CW		 5005 if (obj->phys_obj) {
5006 if (obj->phys_obj->id == id)
71acb5eb
DA		 5007 return 0;
5008 i915_gem_detach_phys_object(dev, obj);
5009 }
5010
71acb5eb
DA		 5011 /* create a new object */
5012 if (!dev_priv->mm.phys_objs[id - 1]) {
5013 ret = i915_gem_init_phys_object(dev, id,
05394f39 5014 obj->base.size, align);
71acb5eb 5015 if (ret) {
05394f39
CW		 5016 DRM_ERROR("failed to init phys object %d size: %zu\n",
5017 id, obj->base.size);
e5281ccd 5018 return ret;
71acb5eb
DA		 5019 }
5020 }
5021
5022 /* bind to the object */
05394f39
CW		 5023 obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
5024 obj->phys_obj->cur_obj = obj;
71acb5eb 5025
05394f39 5026 page_count = obj->base.size / PAGE_SIZE;
71acb5eb
DA		 5027
5028 for (i = 0; i < page_count; i++) {
e5281ccd
CW		 5029 struct page *page;
5030 char *dst, *src;
5031
5032 page = read_cache_page_gfp(mapping, i,
5033 GFP_HIGHUSER | __GFP_RECLAIMABLE);
5034 if (IS_ERR(page))
5035 return PTR_ERR(page);
71acb5eb 5036
ff75b9bc 5037 src = kmap_atomic(page);
05394f39 5038 dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
71acb5eb 5039 memcpy(dst, src, PAGE_SIZE);
3e4d3af5 5040 kunmap_atomic(src);
71acb5eb 5041
e5281ccd
CW		 5042 mark_page_accessed(page);
5043 page_cache_release(page);
5044 }
d78b47b9 5045
71acb5eb 5046 return 0;
71acb5eb
DA		 5047}
5048
5049static int
05394f39
CW		 5050i915_gem_phys_pwrite(struct drm_device *dev,
5051 struct drm_i915_gem_object *obj,
71acb5eb
DA		 5052 struct drm_i915_gem_pwrite *args,
5053 struct drm_file *file_priv)
5054{
05394f39 5055 void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
b47b30cc 5056 char __user *user_data = (char __user *) (uintptr_t) args->data_ptr;
71acb5eb 5057
b47b30cc
CW		 5058 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
5059 unsigned long unwritten;
5060
5061 /* The physical object once assigned is fixed for the lifetime
5062 * of the obj, so we can safely drop the lock and continue
5063 * to access vaddr.
5064 */
5065 mutex_unlock(&dev->struct_mutex);
5066 unwritten = copy_from_user(vaddr, user_data, args->size);
5067 mutex_lock(&dev->struct_mutex);
5068 if (unwritten)
5069 return -EFAULT;
5070 }
71acb5eb 5071
40ce6575 5072 intel_gtt_chipset_flush();
71acb5eb
DA		 5073 return 0;
5074}
b962442e 5075
f787a5f5 5076void i915_gem_release(struct drm_device *dev, struct drm_file *file)
b962442e 5077{
f787a5f5 5078 struct drm_i915_file_private *file_priv = file->driver_priv;
b962442e
EA		 5079
5080 /* Clean up our request list when the client is going away, so that
5081 * later retire_requests won't dereference our soon-to-be-gone
5082 * file_priv.
5083 */
1c25595f 5084 spin_lock(&file_priv->mm.lock);
f787a5f5
CW		 5085 while (!list_empty(&file_priv->mm.request_list)) {
5086 struct drm_i915_gem_request *request;
5087
5088 request = list_first_entry(&file_priv->mm.request_list,
5089 struct drm_i915_gem_request,
5090 client_list);
5091 list_del(&request->client_list);
5092 request->file_priv = NULL;
5093 }
1c25595f 5094 spin_unlock(&file_priv->mm.lock);
b962442e 5095}
31169714 5096
1637ef41
CW		 5097static int
5098i915_gpu_is_active(struct drm_device *dev)
5099{
5100 drm_i915_private_t *dev_priv = dev->dev_private;
5101 int lists_empty;
5102
1637ef41 5103 lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
17250b71 5104 list_empty(&dev_priv->mm.active_list);
1637ef41
CW		 5105
5106 return !lists_empty;
5107}
5108
31169714 5109static int
17250b71
CW		 5110i915_gem_inactive_shrink(struct shrinker *shrinker,
5111 int nr_to_scan,
5112 gfp_t gfp_mask)
31169714 5113{
17250b71
CW		 5114 struct drm_i915_private *dev_priv =
5115 container_of(shrinker,
5116 struct drm_i915_private,
5117 mm.inactive_shrinker);
5118 struct drm_device *dev = dev_priv->dev;
5119 struct drm_i915_gem_object *obj, *next;
5120 int cnt;
5121
5122 if (!mutex_trylock(&dev->struct_mutex))
bbe2e11a 5123 return 0;
31169714
CW		 5124
5125 /* "fast-path" to count number of available objects */
5126 if (nr_to_scan == 0) {
17250b71
CW		 5127 cnt = 0;
5128 list_for_each_entry(obj,
5129 &dev_priv->mm.inactive_list,
5130 mm_list)
5131 cnt++;
5132 mutex_unlock(&dev->struct_mutex);
5133 return cnt / 100 * sysctl_vfs_cache_pressure;
31169714
CW		 5134 }
5135
1637ef41 5136rescan:
31169714 5137 /* first scan for clean buffers */
17250b71 5138 i915_gem_retire_requests(dev);
31169714 5139
17250b71
CW		 5140 list_for_each_entry_safe(obj, next,
5141 &dev_priv->mm.inactive_list,
5142 mm_list) {
5143 if (i915_gem_object_is_purgeable(obj)) {
05394f39 5144 i915_gem_object_unbind(obj);
17250b71
CW		 5145 if (--nr_to_scan == 0)
5146 break;
31169714 5147 }
31169714
CW		 5148 }
5149
5150 /* second pass, evict/count anything still on the inactive list */
17250b71
CW		 5151 cnt = 0;
5152 list_for_each_entry_safe(obj, next,
5153 &dev_priv->mm.inactive_list,
5154 mm_list) {
5155 if (nr_to_scan) {
05394f39 5156 i915_gem_object_unbind(obj);
17250b71
CW		 5157 nr_to_scan--;
5158 } else
5159 cnt++;
5160 }
5161
5162 if (nr_to_scan && i915_gpu_is_active(dev)) {
1637ef41
CW		 5163 /*
5164 * We are desperate for pages, so as a last resort, wait
5165 * for the GPU to finish and discard whatever we can.
5166 * This has a dramatic impact to reduce the number of
5167 * OOM-killer events whilst running the GPU aggressively.
5168 */
17250b71 5169 if (i915_gpu_idle(dev) == 0)
1637ef41
CW		 5170 goto rescan;
5171 }
17250b71
CW		 5172 mutex_unlock(&dev->struct_mutex);
5173 return cnt / 100 * sysctl_vfs_cache_pressure;
31169714 5174}
Ubuntu Hirsute Linux kernel mirror