]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - drivers/gpu/drm/i915/i915_gem.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
nouveau: add PRIME support
[mirror_ubuntu-bionic-kernel.git] / drivers / gpu / drm / i915 / i915_gem.c
CommitLineData
673a394b
EA		 1/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
1c5d22f7 32#include "i915_trace.h"
652c393a 33#include "intel_drv.h"
5949eac4 34#include <linux/shmem_fs.h>
5a0e3ad6 35#include <linux/slab.h>
673a394b 36#include <linux/swap.h>
79e53945 37#include <linux/pci.h>
673a394b 38
88241785 39static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
05394f39
CW		 40static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
41static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
88241785
CW		 42static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
43 unsigned alignment,
44 bool map_and_fenceable);
05394f39
CW		 45static int i915_gem_phys_pwrite(struct drm_device *dev,
46 struct drm_i915_gem_object *obj,
71acb5eb 47 struct drm_i915_gem_pwrite *args,
05394f39 48 struct drm_file *file);
673a394b 49
61050808
CW		 50static void i915_gem_write_fence(struct drm_device *dev, int reg,
51 struct drm_i915_gem_object *obj);
52static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
53 struct drm_i915_fence_reg *fence,
54 bool enable);
55
17250b71 56static int i915_gem_inactive_shrink(struct shrinker *shrinker,
1495f230 57 struct shrink_control *sc);
8c59967c 58static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
31169714 59
61050808
CW		 60static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
61{
62 if (obj->tiling_mode)
63 i915_gem_release_mmap(obj);
64
65 /* As we do not have an associated fence register, we will force
66 * a tiling change if we ever need to acquire one.
67 */
5d82e3e6 68 obj->fence_dirty = false;
61050808
CW		 69 obj->fence_reg = I915_FENCE_REG_NONE;
70}
71
73aa808f
CW		 72/* some bookkeeping */
73static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
74 size_t size)
75{
76 dev_priv->mm.object_count++;
77 dev_priv->mm.object_memory += size;
78}
79
80static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
81 size_t size)
82{
83 dev_priv->mm.object_count--;
84 dev_priv->mm.object_memory -= size;
85}
86
21dd3734
CW		 87static int
88i915_gem_wait_for_error(struct drm_device *dev)
30dbf0c0
CW		 89{
90 struct drm_i915_private *dev_priv = dev->dev_private;
91 struct completion *x = &dev_priv->error_completion;
92 unsigned long flags;
93 int ret;
94
95 if (!atomic_read(&dev_priv->mm.wedged))
96 return 0;
97
98 ret = wait_for_completion_interruptible(x);
99 if (ret)
100 return ret;
101
21dd3734
CW		 102 if (atomic_read(&dev_priv->mm.wedged)) {
103 /* GPU is hung, bump the completion count to account for
104 * the token we just consumed so that we never hit zero and
105 * end up waiting upon a subsequent completion event that
106 * will never happen.
107 */
108 spin_lock_irqsave(&x->wait.lock, flags);
109 x->done++;
110 spin_unlock_irqrestore(&x->wait.lock, flags);
111 }
112 return 0;
30dbf0c0
CW		 113}
114
54cf91dc 115int i915_mutex_lock_interruptible(struct drm_device *dev)
76c1dec1 116{
76c1dec1
CW		 117 int ret;
118
21dd3734 119 ret = i915_gem_wait_for_error(dev);
76c1dec1
CW		 120 if (ret)
121 return ret;
122
123 ret = mutex_lock_interruptible(&dev->struct_mutex);
124 if (ret)
125 return ret;
126
23bc5982 127 WARN_ON(i915_verify_lists(dev));
76c1dec1
CW		 128 return 0;
129}
30dbf0c0 130
7d1c4804 131static inline bool
05394f39 132i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
7d1c4804 133{
1b50247a 134 return !obj->active;
7d1c4804
CW		 135}
136
79e53945
JB		 137int
138i915_gem_init_ioctl(struct drm_device *dev, void *data,
05394f39 139 struct drm_file *file)
79e53945
JB		 140{
141 struct drm_i915_gem_init *args = data;
2021746e 142
7bb6fb8d
DV		 143 if (drm_core_check_feature(dev, DRIVER_MODESET))
144 return -ENODEV;
145
2021746e
CW		 146 if (args->gtt_start >= args->gtt_end ||
147 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
148 return -EINVAL;
79e53945 149
f534bc0b
DV		 150 /* GEM with user mode setting was never supported on ilk and later. */
151 if (INTEL_INFO(dev)->gen >= 5)
152 return -ENODEV;
153
79e53945 154 mutex_lock(&dev->struct_mutex);
644ec02b
DV		 155 i915_gem_init_global_gtt(dev, args->gtt_start,
156 args->gtt_end, args->gtt_end);
673a394b
EA		 157 mutex_unlock(&dev->struct_mutex);
158
2021746e 159 return 0;
673a394b
EA		 160}
161
5a125c3c
EA		 162int
163i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
05394f39 164 struct drm_file *file)
5a125c3c 165{
73aa808f 166 struct drm_i915_private *dev_priv = dev->dev_private;
5a125c3c 167 struct drm_i915_gem_get_aperture *args = data;
6299f992
CW		 168 struct drm_i915_gem_object *obj;
169 size_t pinned;
5a125c3c 170
6299f992 171 pinned = 0;
73aa808f 172 mutex_lock(&dev->struct_mutex);
1b50247a
CW		 173 list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list)
174 if (obj->pin_count)
175 pinned += obj->gtt_space->size;
73aa808f 176 mutex_unlock(&dev->struct_mutex);
5a125c3c 177
6299f992 178 args->aper_size = dev_priv->mm.gtt_total;
0206e353 179 args->aper_available_size = args->aper_size - pinned;
6299f992 180
5a125c3c
EA		 181 return 0;
182}
183
ff72145b
DA		 184static int
185i915_gem_create(struct drm_file *file,
186 struct drm_device *dev,
187 uint64_t size,
188 uint32_t *handle_p)
673a394b 189{
05394f39 190 struct drm_i915_gem_object *obj;
a1a2d1d3
PP		 191 int ret;
192 u32 handle;
673a394b 193
ff72145b 194 size = roundup(size, PAGE_SIZE);
8ffc0246
CW		 195 if (size == 0)
196 return -EINVAL;
673a394b
EA		 197
198 /* Allocate the new object */
ff72145b 199 obj = i915_gem_alloc_object(dev, size);
673a394b
EA		 200 if (obj == NULL)
201 return -ENOMEM;
202
05394f39 203 ret = drm_gem_handle_create(file, &obj->base, &handle);
1dfd9754 204 if (ret) {
05394f39
CW		 205 drm_gem_object_release(&obj->base);
206 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
202f2fef 207 kfree(obj);
673a394b 208 return ret;
1dfd9754 209 }
673a394b 210
202f2fef 211 /* drop reference from allocate - handle holds it now */
05394f39 212 drm_gem_object_unreference(&obj->base);
202f2fef
CW		 213 trace_i915_gem_object_create(obj);
214
ff72145b 215 *handle_p = handle;
673a394b
EA		 216 return 0;
217}
218
ff72145b
DA		 219int
220i915_gem_dumb_create(struct drm_file *file,
221 struct drm_device *dev,
222 struct drm_mode_create_dumb *args)
223{
224 /* have to work out size/pitch and return them */
ed0291fd 225 args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
ff72145b
DA		 226 args->size = args->pitch * args->height;
227 return i915_gem_create(file, dev,
228 args->size, &args->handle);
229}
230
231int i915_gem_dumb_destroy(struct drm_file *file,
232 struct drm_device *dev,
233 uint32_t handle)
234{
235 return drm_gem_handle_delete(file, handle);
236}
237
238/**
239 * Creates a new mm object and returns a handle to it.
240 */
241int
242i915_gem_create_ioctl(struct drm_device *dev, void *data,
243 struct drm_file *file)
244{
245 struct drm_i915_gem_create *args = data;
63ed2cb2 246
ff72145b
DA		 247 return i915_gem_create(file, dev,
248 args->size, &args->handle);
249}
250
05394f39 251static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
280b713b 252{
05394f39 253 drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
280b713b
EA		 254
255 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
05394f39 256 obj->tiling_mode != I915_TILING_NONE;
280b713b
EA		 257}
258
8461d226
DV		 259static inline int
260__copy_to_user_swizzled(char __user *cpu_vaddr,
261 const char *gpu_vaddr, int gpu_offset,
262 int length)
263{
264 int ret, cpu_offset = 0;
265
266 while (length > 0) {
267 int cacheline_end = ALIGN(gpu_offset + 1, 64);
268 int this_length = min(cacheline_end - gpu_offset, length);
269 int swizzled_gpu_offset = gpu_offset ^ 64;
270
271 ret = __copy_to_user(cpu_vaddr + cpu_offset,
272 gpu_vaddr + swizzled_gpu_offset,
273 this_length);
274 if (ret)
275 return ret + length;
276
277 cpu_offset += this_length;
278 gpu_offset += this_length;
279 length -= this_length;
280 }
281
282 return 0;
283}
284
8c59967c 285static inline int
4f0c7cfb
BW		 286__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
287 const char __user *cpu_vaddr,
8c59967c
DV		 288 int length)
289{
290 int ret, cpu_offset = 0;
291
292 while (length > 0) {
293 int cacheline_end = ALIGN(gpu_offset + 1, 64);
294 int this_length = min(cacheline_end - gpu_offset, length);
295 int swizzled_gpu_offset = gpu_offset ^ 64;
296
297 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
298 cpu_vaddr + cpu_offset,
299 this_length);
300 if (ret)
301 return ret + length;
302
303 cpu_offset += this_length;
304 gpu_offset += this_length;
305 length -= this_length;
306 }
307
308 return 0;
309}
310
d174bd64
DV		 311/* Per-page copy function for the shmem pread fastpath.
312 * Flushes invalid cachelines before reading the target if
313 * needs_clflush is set. */
eb01459f 314static int
d174bd64
DV		 315shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
316 char __user *user_data,
317 bool page_do_bit17_swizzling, bool needs_clflush)
318{
319 char *vaddr;
320 int ret;
321
e7e58eb5 322 if (unlikely(page_do_bit17_swizzling))
d174bd64
DV		 323 return -EINVAL;
324
325 vaddr = kmap_atomic(page);
326 if (needs_clflush)
327 drm_clflush_virt_range(vaddr + shmem_page_offset,
328 page_length);
329 ret = __copy_to_user_inatomic(user_data,
330 vaddr + shmem_page_offset,
331 page_length);
332 kunmap_atomic(vaddr);
333
334 return ret;
335}
336
23c18c71
DV		 337static void
338shmem_clflush_swizzled_range(char *addr, unsigned long length,
339 bool swizzled)
340{
e7e58eb5 341 if (unlikely(swizzled)) {
23c18c71
DV		 342 unsigned long start = (unsigned long) addr;
343 unsigned long end = (unsigned long) addr + length;
344
345 /* For swizzling simply ensure that we always flush both
346 * channels. Lame, but simple and it works. Swizzled
347 * pwrite/pread is far from a hotpath - current userspace
348 * doesn't use it at all. */
349 start = round_down(start, 128);
350 end = round_up(end, 128);
351
352 drm_clflush_virt_range((void *)start, end - start);
353 } else {
354 drm_clflush_virt_range(addr, length);
355 }
356
357}
358
d174bd64
DV		 359/* Only difference to the fast-path function is that this can handle bit17
360 * and uses non-atomic copy and kmap functions. */
361static int
362shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
363 char __user *user_data,
364 bool page_do_bit17_swizzling, bool needs_clflush)
365{
366 char *vaddr;
367 int ret;
368
369 vaddr = kmap(page);
370 if (needs_clflush)
23c18c71
DV		 371 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
372 page_length,
373 page_do_bit17_swizzling);
d174bd64
DV		 374
375 if (page_do_bit17_swizzling)
376 ret = __copy_to_user_swizzled(user_data,
377 vaddr, shmem_page_offset,
378 page_length);
379 else
380 ret = __copy_to_user(user_data,
381 vaddr + shmem_page_offset,
382 page_length);
383 kunmap(page);
384
385 return ret;
386}
387
eb01459f 388static int
dbf7bff0
DV		 389i915_gem_shmem_pread(struct drm_device *dev,
390 struct drm_i915_gem_object *obj,
391 struct drm_i915_gem_pread *args,
392 struct drm_file *file)
eb01459f 393{
05394f39 394 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
8461d226 395 char __user *user_data;
eb01459f 396 ssize_t remain;
8461d226 397 loff_t offset;
eb2c0c81 398 int shmem_page_offset, page_length, ret = 0;
8461d226 399 int obj_do_bit17_swizzling, page_do_bit17_swizzling;
dbf7bff0 400 int hit_slowpath = 0;
96d79b52 401 int prefaulted = 0;
8489731c 402 int needs_clflush = 0;
692a576b 403 int release_page;
eb01459f 404
8461d226 405 user_data = (char __user *) (uintptr_t) args->data_ptr;
eb01459f
EA		 406 remain = args->size;
407
8461d226 408 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
eb01459f 409
8489731c
DV		 410 if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
411 /* If we're not in the cpu read domain, set ourself into the gtt
412 * read domain and manually flush cachelines (if required). This
413 * optimizes for the case when the gpu will dirty the data
414 * anyway again before the next pread happens. */
415 if (obj->cache_level == I915_CACHE_NONE)
416 needs_clflush = 1;
417 ret = i915_gem_object_set_to_gtt_domain(obj, false);
418 if (ret)
419 return ret;
420 }
eb01459f 421
8461d226 422 offset = args->offset;
eb01459f
EA		 423
424 while (remain > 0) {
e5281ccd
CW		 425 struct page *page;
426
eb01459f
EA		 427 /* Operation in this page
428 *
eb01459f 429 * shmem_page_offset = offset within page in shmem file
eb01459f
EA		 430 * page_length = bytes to copy for this page
431 */
c8cbbb8b 432 shmem_page_offset = offset_in_page(offset);
eb01459f
EA		 433 page_length = remain;
434 if ((shmem_page_offset + page_length) > PAGE_SIZE)
435 page_length = PAGE_SIZE - shmem_page_offset;
eb01459f 436
692a576b
DV		 437 if (obj->pages) {
438 page = obj->pages[offset >> PAGE_SHIFT];
439 release_page = 0;
440 } else {
441 page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
442 if (IS_ERR(page)) {
443 ret = PTR_ERR(page);
444 goto out;
445 }
446 release_page = 1;
b65552f0 447 }
e5281ccd 448
8461d226
DV		 449 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
450 (page_to_phys(page) & (1 << 17)) != 0;
451
d174bd64
DV		 452 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
453 user_data, page_do_bit17_swizzling,
454 needs_clflush);
455 if (ret == 0)
456 goto next_page;
dbf7bff0
DV		 457
458 hit_slowpath = 1;
692a576b 459 page_cache_get(page);
dbf7bff0
DV		 460 mutex_unlock(&dev->struct_mutex);
461
96d79b52 462 if (!prefaulted) {
f56f821f 463 ret = fault_in_multipages_writeable(user_data, remain);
96d79b52
DV		 464 /* Userspace is tricking us, but we've already clobbered
465 * its pages with the prefault and promised to write the
466 * data up to the first fault. Hence ignore any errors
467 * and just continue. */
468 (void)ret;
469 prefaulted = 1;
470 }
eb01459f 471
d174bd64
DV		 472 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
473 user_data, page_do_bit17_swizzling,
474 needs_clflush);
eb01459f 475
dbf7bff0 476 mutex_lock(&dev->struct_mutex);
e5281ccd 477 page_cache_release(page);
dbf7bff0 478next_page:
e5281ccd 479 mark_page_accessed(page);
692a576b
DV		 480 if (release_page)
481 page_cache_release(page);
e5281ccd 482
8461d226
DV		 483 if (ret) {
484 ret = -EFAULT;
485 goto out;
486 }
487
eb01459f 488 remain -= page_length;
8461d226 489 user_data += page_length;
eb01459f
EA		 490 offset += page_length;
491 }
492
4f27b75d 493out:
dbf7bff0
DV		 494 if (hit_slowpath) {
495 /* Fixup: Kill any reinstated backing storage pages */
496 if (obj->madv == __I915_MADV_PURGED)
497 i915_gem_object_truncate(obj);
498 }
eb01459f
EA		 499
500 return ret;
501}
502
673a394b
EA		 503/**
504 * Reads data from the object referenced by handle.
505 *
506 * On error, the contents of *data are undefined.
507 */
508int
509i915_gem_pread_ioctl(struct drm_device *dev, void *data,
05394f39 510 struct drm_file *file)
673a394b
EA		 511{
512 struct drm_i915_gem_pread *args = data;
05394f39 513 struct drm_i915_gem_object *obj;
35b62a89 514 int ret = 0;
673a394b 515
51311d0a
CW		 516 if (args->size == 0)
517 return 0;
518
519 if (!access_ok(VERIFY_WRITE,
520 (char __user *)(uintptr_t)args->data_ptr,
521 args->size))
522 return -EFAULT;
523
4f27b75d 524 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 525 if (ret)
4f27b75d 526 return ret;
673a394b 527
05394f39 528 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
c8725226 529 if (&obj->base == NULL) {
1d7cfea1
CW		 530 ret = -ENOENT;
531 goto unlock;
4f27b75d 532 }
673a394b 533
7dcd2499 534 /* Bounds check source. */
05394f39
CW		 535 if (args->offset > obj->base.size ||
536 args->size > obj->base.size - args->offset) {
ce9d419d 537 ret = -EINVAL;
35b62a89 538 goto out;
ce9d419d
CW		 539 }
540
db53a302
CW		 541 trace_i915_gem_object_pread(obj, args->offset, args->size);
542
dbf7bff0 543 ret = i915_gem_shmem_pread(dev, obj, args, file);
673a394b 544
35b62a89 545out:
05394f39 546 drm_gem_object_unreference(&obj->base);
1d7cfea1 547unlock:
4f27b75d 548 mutex_unlock(&dev->struct_mutex);
eb01459f 549 return ret;
673a394b
EA		 550}
551
0839ccb8
KP		 552/* This is the fast write path which cannot handle
553 * page faults in the source data
9b7530cc 554 */
0839ccb8
KP		 555
556static inline int
557fast_user_write(struct io_mapping *mapping,
558 loff_t page_base, int page_offset,
559 char __user *user_data,
560 int length)
9b7530cc 561{
4f0c7cfb
BW		 562 void __iomem *vaddr_atomic;
563 void *vaddr;
0839ccb8 564 unsigned long unwritten;
9b7530cc 565
3e4d3af5 566 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
4f0c7cfb
BW		 567 /* We can use the cpu mem copy function because this is X86. */
568 vaddr = (void __force*)vaddr_atomic + page_offset;
569 unwritten = __copy_from_user_inatomic_nocache(vaddr,
0839ccb8 570 user_data, length);
3e4d3af5 571 io_mapping_unmap_atomic(vaddr_atomic);
fbd5a26d 572 return unwritten;
0839ccb8
KP		 573}
574
3de09aa3
EA		 575/**
576 * This is the fast pwrite path, where we copy the data directly from the
577 * user into the GTT, uncached.
578 */
673a394b 579static int
05394f39
CW		 580i915_gem_gtt_pwrite_fast(struct drm_device *dev,
581 struct drm_i915_gem_object *obj,
3de09aa3 582 struct drm_i915_gem_pwrite *args,
05394f39 583 struct drm_file *file)
673a394b 584{
0839ccb8 585 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 586 ssize_t remain;
0839ccb8 587 loff_t offset, page_base;
673a394b 588 char __user *user_data;
935aaa69
DV		 589 int page_offset, page_length, ret;
590
591 ret = i915_gem_object_pin(obj, 0, true);
592 if (ret)
593 goto out;
594
595 ret = i915_gem_object_set_to_gtt_domain(obj, true);
596 if (ret)
597 goto out_unpin;
598
599 ret = i915_gem_object_put_fence(obj);
600 if (ret)
601 goto out_unpin;
673a394b
EA		 602
603 user_data = (char __user *) (uintptr_t) args->data_ptr;
604 remain = args->size;
673a394b 605
05394f39 606 offset = obj->gtt_offset + args->offset;
673a394b
EA		 607
608 while (remain > 0) {
609 /* Operation in this page
610 *
0839ccb8
KP		 611 * page_base = page offset within aperture
612 * page_offset = offset within page
613 * page_length = bytes to copy for this page
673a394b 614 */
c8cbbb8b
CW		 615 page_base = offset & PAGE_MASK;
616 page_offset = offset_in_page(offset);
0839ccb8
KP		 617 page_length = remain;
618 if ((page_offset + remain) > PAGE_SIZE)
619 page_length = PAGE_SIZE - page_offset;
620
0839ccb8 621 /* If we get a fault while copying data, then (presumably) our
3de09aa3
EA		 622 * source page isn't available. Return the error and we'll
623 * retry in the slow path.
0839ccb8 624 */
fbd5a26d 625 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
935aaa69
DV		 626 page_offset, user_data, page_length)) {
627 ret = -EFAULT;
628 goto out_unpin;
629 }
673a394b 630
0839ccb8
KP		 631 remain -= page_length;
632 user_data += page_length;
633 offset += page_length;
673a394b 634 }
673a394b 635
935aaa69
DV		 636out_unpin:
637 i915_gem_object_unpin(obj);
638out:
3de09aa3 639 return ret;
673a394b
EA		 640}
641
d174bd64
DV		 642/* Per-page copy function for the shmem pwrite fastpath.
643 * Flushes invalid cachelines before writing to the target if
644 * needs_clflush_before is set and flushes out any written cachelines after
645 * writing if needs_clflush is set. */
3043c60c 646static int
d174bd64
DV		 647shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
648 char __user *user_data,
649 bool page_do_bit17_swizzling,
650 bool needs_clflush_before,
651 bool needs_clflush_after)
673a394b 652{
d174bd64 653 char *vaddr;
673a394b 654 int ret;
3de09aa3 655
e7e58eb5 656 if (unlikely(page_do_bit17_swizzling))
d174bd64 657 return -EINVAL;
3de09aa3 658
d174bd64
DV		 659 vaddr = kmap_atomic(page);
660 if (needs_clflush_before)
661 drm_clflush_virt_range(vaddr + shmem_page_offset,
662 page_length);
663 ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
664 user_data,
665 page_length);
666 if (needs_clflush_after)
667 drm_clflush_virt_range(vaddr + shmem_page_offset,
668 page_length);
669 kunmap_atomic(vaddr);
3de09aa3
EA		 670
671 return ret;
672}
673
d174bd64
DV		 674/* Only difference to the fast-path function is that this can handle bit17
675 * and uses non-atomic copy and kmap functions. */
3043c60c 676static int
d174bd64
DV		 677shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
678 char __user *user_data,
679 bool page_do_bit17_swizzling,
680 bool needs_clflush_before,
681 bool needs_clflush_after)
673a394b 682{
d174bd64
DV		 683 char *vaddr;
684 int ret;
e5281ccd 685
d174bd64 686 vaddr = kmap(page);
e7e58eb5 687 if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
23c18c71
DV		 688 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
689 page_length,
690 page_do_bit17_swizzling);
d174bd64
DV		 691 if (page_do_bit17_swizzling)
692 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
e5281ccd
CW		 693 user_data,
694 page_length);
d174bd64
DV		 695 else
696 ret = __copy_from_user(vaddr + shmem_page_offset,
697 user_data,
698 page_length);
699 if (needs_clflush_after)
23c18c71
DV		 700 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
701 page_length,
702 page_do_bit17_swizzling);
d174bd64 703 kunmap(page);
40123c1f 704
d174bd64 705 return ret;
40123c1f
EA		 706}
707
40123c1f 708static int
e244a443
DV		 709i915_gem_shmem_pwrite(struct drm_device *dev,
710 struct drm_i915_gem_object *obj,
711 struct drm_i915_gem_pwrite *args,
712 struct drm_file *file)
40123c1f 713{
05394f39 714 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
40123c1f 715 ssize_t remain;
8c59967c
DV		 716 loff_t offset;
717 char __user *user_data;
eb2c0c81 718 int shmem_page_offset, page_length, ret = 0;
8c59967c 719 int obj_do_bit17_swizzling, page_do_bit17_swizzling;
e244a443 720 int hit_slowpath = 0;
58642885
DV		 721 int needs_clflush_after = 0;
722 int needs_clflush_before = 0;
692a576b 723 int release_page;
40123c1f 724
8c59967c 725 user_data = (char __user *) (uintptr_t) args->data_ptr;
40123c1f
EA		 726 remain = args->size;
727
8c59967c 728 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
40123c1f 729
58642885
DV		 730 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
731 /* If we're not in the cpu write domain, set ourself into the gtt
732 * write domain and manually flush cachelines (if required). This
733 * optimizes for the case when the gpu will use the data
734 * right away and we therefore have to clflush anyway. */
735 if (obj->cache_level == I915_CACHE_NONE)
736 needs_clflush_after = 1;
737 ret = i915_gem_object_set_to_gtt_domain(obj, true);
738 if (ret)
739 return ret;
740 }
741 /* Same trick applies for invalidate partially written cachelines before
742 * writing. */
743 if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
744 && obj->cache_level == I915_CACHE_NONE)
745 needs_clflush_before = 1;
746
673a394b 747 offset = args->offset;
05394f39 748 obj->dirty = 1;
673a394b 749
40123c1f 750 while (remain > 0) {
e5281ccd 751 struct page *page;
58642885 752 int partial_cacheline_write;
e5281ccd 753
40123c1f
EA		 754 /* Operation in this page
755 *
40123c1f 756 * shmem_page_offset = offset within page in shmem file
40123c1f
EA		 757 * page_length = bytes to copy for this page
758 */
c8cbbb8b 759 shmem_page_offset = offset_in_page(offset);
40123c1f
EA		 760
761 page_length = remain;
762 if ((shmem_page_offset + page_length) > PAGE_SIZE)
763 page_length = PAGE_SIZE - shmem_page_offset;
40123c1f 764
58642885
DV		 765 /* If we don't overwrite a cacheline completely we need to be
766 * careful to have up-to-date data by first clflushing. Don't
767 * overcomplicate things and flush the entire patch. */
768 partial_cacheline_write = needs_clflush_before &&
769 ((shmem_page_offset | page_length)
770 & (boot_cpu_data.x86_clflush_size - 1));
771
692a576b
DV		 772 if (obj->pages) {
773 page = obj->pages[offset >> PAGE_SHIFT];
774 release_page = 0;
775 } else {
776 page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
777 if (IS_ERR(page)) {
778 ret = PTR_ERR(page);
779 goto out;
780 }
781 release_page = 1;
e5281ccd
CW		 782 }
783
8c59967c
DV		 784 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
785 (page_to_phys(page) & (1 << 17)) != 0;
786
d174bd64
DV		 787 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
788 user_data, page_do_bit17_swizzling,
789 partial_cacheline_write,
790 needs_clflush_after);
791 if (ret == 0)
792 goto next_page;
e244a443
DV		 793
794 hit_slowpath = 1;
692a576b 795 page_cache_get(page);
e244a443
DV		 796 mutex_unlock(&dev->struct_mutex);
797
d174bd64
DV		 798 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
799 user_data, page_do_bit17_swizzling,
800 partial_cacheline_write,
801 needs_clflush_after);
40123c1f 802
e244a443 803 mutex_lock(&dev->struct_mutex);
692a576b 804 page_cache_release(page);
e244a443 805next_page:
e5281ccd
CW		 806 set_page_dirty(page);
807 mark_page_accessed(page);
692a576b
DV		 808 if (release_page)
809 page_cache_release(page);
e5281ccd 810
8c59967c
DV		 811 if (ret) {
812 ret = -EFAULT;
813 goto out;
814 }
815
40123c1f 816 remain -= page_length;
8c59967c 817 user_data += page_length;
40123c1f 818 offset += page_length;
673a394b
EA		 819 }
820
fbd5a26d 821out:
e244a443
DV		 822 if (hit_slowpath) {
823 /* Fixup: Kill any reinstated backing storage pages */
824 if (obj->madv == __I915_MADV_PURGED)
825 i915_gem_object_truncate(obj);
826 /* and flush dirty cachelines in case the object isn't in the cpu write
827 * domain anymore. */
828 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
829 i915_gem_clflush_object(obj);
830 intel_gtt_chipset_flush();
831 }
8c59967c 832 }
673a394b 833
58642885
DV		 834 if (needs_clflush_after)
835 intel_gtt_chipset_flush();
836
40123c1f 837 return ret;
673a394b
EA		 838}
839
840/**
841 * Writes data to the object referenced by handle.
842 *
843 * On error, the contents of the buffer that were to be modified are undefined.
844 */
845int
846i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
fbd5a26d 847 struct drm_file *file)
673a394b
EA		 848{
849 struct drm_i915_gem_pwrite *args = data;
05394f39 850 struct drm_i915_gem_object *obj;
51311d0a
CW		 851 int ret;
852
853 if (args->size == 0)
854 return 0;
855
856 if (!access_ok(VERIFY_READ,
857 (char __user *)(uintptr_t)args->data_ptr,
858 args->size))
859 return -EFAULT;
860
f56f821f
DV		 861 ret = fault_in_multipages_readable((char __user *)(uintptr_t)args->data_ptr,
862 args->size);
51311d0a
CW		 863 if (ret)
864 return -EFAULT;
673a394b 865
fbd5a26d 866 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 867 if (ret)
fbd5a26d 868 return ret;
1d7cfea1 869
05394f39 870 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
c8725226 871 if (&obj->base == NULL) {
1d7cfea1
CW		 872 ret = -ENOENT;
873 goto unlock;
fbd5a26d 874 }
673a394b 875
7dcd2499 876 /* Bounds check destination. */
05394f39
CW		 877 if (args->offset > obj->base.size ||
878 args->size > obj->base.size - args->offset) {
ce9d419d 879 ret = -EINVAL;
35b62a89 880 goto out;
ce9d419d
CW		 881 }
882
db53a302
CW		 883 trace_i915_gem_object_pwrite(obj, args->offset, args->size);
884
935aaa69 885 ret = -EFAULT;
673a394b
EA		 886 /* We can only do the GTT pwrite on untiled buffers, as otherwise
887 * it would end up going through the fenced access, and we'll get
888 * different detiling behavior between reading and writing.
889 * pread/pwrite currently are reading and writing from the CPU
890 * perspective, requiring manual detiling by the client.
891 */
5c0480f2 892 if (obj->phys_obj) {
fbd5a26d 893 ret = i915_gem_phys_pwrite(dev, obj, args, file);
5c0480f2
DV		 894 goto out;
895 }
896
897 if (obj->gtt_space &&
3ae53783 898 obj->cache_level == I915_CACHE_NONE &&
c07496fa 899 obj->tiling_mode == I915_TILING_NONE &&
ffc62976 900 obj->map_and_fenceable &&
5c0480f2 901 obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
fbd5a26d 902 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
935aaa69
DV		 903 /* Note that the gtt paths might fail with non-page-backed user
904 * pointers (e.g. gtt mappings when moving data between
905 * textures). Fallback to the shmem path in that case. */
fbd5a26d 906 }
673a394b 907
5c0480f2 908 if (ret == -EFAULT)
935aaa69 909 ret = i915_gem_shmem_pwrite(dev, obj, args, file);
5c0480f2 910
35b62a89 911out:
05394f39 912 drm_gem_object_unreference(&obj->base);
1d7cfea1 913unlock:
fbd5a26d 914 mutex_unlock(&dev->struct_mutex);
673a394b
EA		 915 return ret;
916}
917
918/**
2ef7eeaa
EA		 919 * Called when user space prepares to use an object with the CPU, either
920 * through the mmap ioctl's mapping or a GTT mapping.
673a394b
EA		 921 */
922int
923i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
05394f39 924 struct drm_file *file)
673a394b
EA		 925{
926 struct drm_i915_gem_set_domain *args = data;
05394f39 927 struct drm_i915_gem_object *obj;
2ef7eeaa
EA		 928 uint32_t read_domains = args->read_domains;
929 uint32_t write_domain = args->write_domain;
673a394b
EA		 930 int ret;
931
2ef7eeaa 932 /* Only handle setting domains to types used by the CPU. */
21d509e3 933 if (write_domain & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 934 return -EINVAL;
935
21d509e3 936 if (read_domains & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 937 return -EINVAL;
938
939 /* Having something in the write domain implies it's in the read
940 * domain, and only that read domain. Enforce that in the request.
941 */
942 if (write_domain != 0 && read_domains != write_domain)
943 return -EINVAL;
944
76c1dec1 945 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 946 if (ret)
76c1dec1 947 return ret;
1d7cfea1 948
05394f39 949 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
c8725226 950 if (&obj->base == NULL) {
1d7cfea1
CW		 951 ret = -ENOENT;
952 goto unlock;
76c1dec1 953 }
673a394b 954
2ef7eeaa
EA		 955 if (read_domains & I915_GEM_DOMAIN_GTT) {
956 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
02354392
EA		 957
958 /* Silently promote "you're not bound, there was nothing to do"
959 * to success, since the client was just asking us to
960 * make sure everything was done.
961 */
962 if (ret == -EINVAL)
963 ret = 0;
2ef7eeaa 964 } else {
e47c68e9 965 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
2ef7eeaa
EA		 966 }
967
05394f39 968 drm_gem_object_unreference(&obj->base);
1d7cfea1 969unlock:
673a394b
EA		 970 mutex_unlock(&dev->struct_mutex);
971 return ret;
972}
973
974/**
975 * Called when user space has done writes to this buffer
976 */
977int
978i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
05394f39 979 struct drm_file *file)
673a394b
EA		 980{
981 struct drm_i915_gem_sw_finish *args = data;
05394f39 982 struct drm_i915_gem_object *obj;
673a394b
EA		 983 int ret = 0;
984
76c1dec1 985 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 986 if (ret)
76c1dec1 987 return ret;
1d7cfea1 988
05394f39 989 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
c8725226 990 if (&obj->base == NULL) {
1d7cfea1
CW		 991 ret = -ENOENT;
992 goto unlock;
673a394b
EA		 993 }
994
673a394b 995 /* Pinned buffers may be scanout, so flush the cache */
05394f39 996 if (obj->pin_count)
e47c68e9
EA		 997 i915_gem_object_flush_cpu_write_domain(obj);
998
05394f39 999 drm_gem_object_unreference(&obj->base);
1d7cfea1 1000unlock:
673a394b
EA		 1001 mutex_unlock(&dev->struct_mutex);
1002 return ret;
1003}
1004
1005/**
1006 * Maps the contents of an object, returning the address it is mapped
1007 * into.
1008 *
1009 * While the mapping holds a reference on the contents of the object, it doesn't
1010 * imply a ref on the object itself.
1011 */
1012int
1013i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
05394f39 1014 struct drm_file *file)
673a394b
EA		 1015{
1016 struct drm_i915_gem_mmap *args = data;
1017 struct drm_gem_object *obj;
673a394b
EA		 1018 unsigned long addr;
1019
05394f39 1020 obj = drm_gem_object_lookup(dev, file, args->handle);
673a394b 1021 if (obj == NULL)
bf79cb91 1022 return -ENOENT;
673a394b 1023
6be5ceb0 1024 addr = vm_mmap(obj->filp, 0, args->size,
673a394b
EA		 1025 PROT_READ | PROT_WRITE, MAP_SHARED,
1026 args->offset);
bc9025bd 1027 drm_gem_object_unreference_unlocked(obj);
673a394b
EA		 1028 if (IS_ERR((void *)addr))
1029 return addr;
1030
1031 args->addr_ptr = (uint64_t) addr;
1032
1033 return 0;
1034}
1035
de151cf6
JB		 1036/**
1037 * i915_gem_fault - fault a page into the GTT
1038 * vma: VMA in question
1039 * vmf: fault info
1040 *
1041 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1042 * from userspace. The fault handler takes care of binding the object to
1043 * the GTT (if needed), allocating and programming a fence register (again,
1044 * only if needed based on whether the old reg is still valid or the object
1045 * is tiled) and inserting a new PTE into the faulting process.
1046 *
1047 * Note that the faulting process may involve evicting existing objects
1048 * from the GTT and/or fence registers to make room. So performance may
1049 * suffer if the GTT working set is large or there are few fence registers
1050 * left.
1051 */
1052int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1053{
05394f39
CW		 1054 struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1055 struct drm_device *dev = obj->base.dev;
7d1c4804 1056 drm_i915_private_t *dev_priv = dev->dev_private;
de151cf6
JB		 1057 pgoff_t page_offset;
1058 unsigned long pfn;
1059 int ret = 0;
0f973f27 1060 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
de151cf6
JB		 1061
1062 /* We don't use vmf->pgoff since that has the fake offset */
1063 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1064 PAGE_SHIFT;
1065
d9bc7e9f
CW		 1066 ret = i915_mutex_lock_interruptible(dev);
1067 if (ret)
1068 goto out;
a00b10c3 1069
db53a302
CW		 1070 trace_i915_gem_object_fault(obj, page_offset, true, write);
1071
d9bc7e9f 1072 /* Now bind it into the GTT if needed */
919926ae
CW		 1073 if (!obj->map_and_fenceable) {
1074 ret = i915_gem_object_unbind(obj);
1075 if (ret)
1076 goto unlock;
a00b10c3 1077 }
05394f39 1078 if (!obj->gtt_space) {
75e9e915 1079 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
c715089f
CW		 1080 if (ret)
1081 goto unlock;
de151cf6 1082
e92d03bf
EA		 1083 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1084 if (ret)
1085 goto unlock;
1086 }
4a684a41 1087
74898d7e
DV		 1088 if (!obj->has_global_gtt_mapping)
1089 i915_gem_gtt_bind_object(obj, obj->cache_level);
1090
06d98131 1091 ret = i915_gem_object_get_fence(obj);
d9e86c0e
CW		 1092 if (ret)
1093 goto unlock;
de151cf6 1094
05394f39
CW		 1095 if (i915_gem_object_is_inactive(obj))
1096 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
7d1c4804 1097
6299f992
CW		 1098 obj->fault_mappable = true;
1099
05394f39 1100 pfn = ((dev->agp->base + obj->gtt_offset) >> PAGE_SHIFT) +
de151cf6
JB		 1101 page_offset;
1102
1103 /* Finally, remap it using the new GTT offset */
1104 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
c715089f 1105unlock:
de151cf6 1106 mutex_unlock(&dev->struct_mutex);
d9bc7e9f 1107out:
de151cf6 1108 switch (ret) {
d9bc7e9f 1109 case -EIO:
045e769a 1110 case -EAGAIN:
d9bc7e9f
CW		 1111 /* Give the error handler a chance to run and move the
1112 * objects off the GPU active list. Next time we service the
1113 * fault, we should be able to transition the page into the
1114 * GTT without touching the GPU (and so avoid further
1115 * EIO/EGAIN). If the GPU is wedged, then there is no issue
1116 * with coherency, just lost writes.
1117 */
045e769a 1118 set_need_resched();
c715089f
CW		 1119 case 0:
1120 case -ERESTARTSYS:
bed636ab 1121 case -EINTR:
c715089f 1122 return VM_FAULT_NOPAGE;
de151cf6 1123 case -ENOMEM:
de151cf6 1124 return VM_FAULT_OOM;
de151cf6 1125 default:
c715089f 1126 return VM_FAULT_SIGBUS;
de151cf6
JB		 1127 }
1128}
1129
901782b2
CW		 1130/**
1131 * i915_gem_release_mmap - remove physical page mappings
1132 * @obj: obj in question
1133 *
af901ca1 1134 * Preserve the reservation of the mmapping with the DRM core code, but
901782b2
CW		 1135 * relinquish ownership of the pages back to the system.
1136 *
1137 * It is vital that we remove the page mapping if we have mapped a tiled
1138 * object through the GTT and then lose the fence register due to
1139 * resource pressure. Similarly if the object has been moved out of the
1140 * aperture, than pages mapped into userspace must be revoked. Removing the
1141 * mapping will then trigger a page fault on the next user access, allowing
1142 * fixup by i915_gem_fault().
1143 */
d05ca301 1144void
05394f39 1145i915_gem_release_mmap(struct drm_i915_gem_object *obj)
901782b2 1146{
6299f992
CW		 1147 if (!obj->fault_mappable)
1148 return;
901782b2 1149
f6e47884
CW		 1150 if (obj->base.dev->dev_mapping)
1151 unmap_mapping_range(obj->base.dev->dev_mapping,
1152 (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
1153 obj->base.size, 1);
fb7d516a 1154
6299f992 1155 obj->fault_mappable = false;
901782b2
CW		 1156}
1157
92b88aeb 1158static uint32_t
e28f8711 1159i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
92b88aeb 1160{
e28f8711 1161 uint32_t gtt_size;
92b88aeb
CW		 1162
1163 if (INTEL_INFO(dev)->gen >= 4 ||
e28f8711
CW		 1164 tiling_mode == I915_TILING_NONE)
1165 return size;
92b88aeb
CW		 1166
1167 /* Previous chips need a power-of-two fence region when tiling */
1168 if (INTEL_INFO(dev)->gen == 3)
e28f8711 1169 gtt_size = 1024*1024;
92b88aeb 1170 else
e28f8711 1171 gtt_size = 512*1024;
92b88aeb 1172
e28f8711
CW		 1173 while (gtt_size < size)
1174 gtt_size <<= 1;
92b88aeb 1175
e28f8711 1176 return gtt_size;
92b88aeb
CW		 1177}
1178
de151cf6
JB		 1179/**
1180 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1181 * @obj: object to check
1182 *
1183 * Return the required GTT alignment for an object, taking into account
5e783301 1184 * potential fence register mapping.
de151cf6
JB		 1185 */
1186static uint32_t
e28f8711
CW		 1187i915_gem_get_gtt_alignment(struct drm_device *dev,
1188 uint32_t size,
1189 int tiling_mode)
de151cf6 1190{
de151cf6
JB		 1191 /*
1192 * Minimum alignment is 4k (GTT page size), but might be greater
1193 * if a fence register is needed for the object.
1194 */
a00b10c3 1195 if (INTEL_INFO(dev)->gen >= 4 ||
e28f8711 1196 tiling_mode == I915_TILING_NONE)
de151cf6
JB		 1197 return 4096;
1198
a00b10c3
CW		 1199 /*
1200 * Previous chips need to be aligned to the size of the smallest
1201 * fence register that can contain the object.
1202 */
e28f8711 1203 return i915_gem_get_gtt_size(dev, size, tiling_mode);
a00b10c3
CW		 1204}
1205
5e783301
DV		 1206/**
1207 * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
1208 * unfenced object
e28f8711
CW		 1209 * @dev: the device
1210 * @size: size of the object
1211 * @tiling_mode: tiling mode of the object
5e783301
DV		 1212 *
1213 * Return the required GTT alignment for an object, only taking into account
1214 * unfenced tiled surface requirements.
1215 */
467cffba 1216uint32_t
e28f8711
CW		 1217i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
1218 uint32_t size,
1219 int tiling_mode)
5e783301 1220{
5e783301
DV		 1221 /*
1222 * Minimum alignment is 4k (GTT page size) for sane hw.
1223 */
1224 if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
e28f8711 1225 tiling_mode == I915_TILING_NONE)
5e783301
DV		 1226 return 4096;
1227
e28f8711
CW		 1228 /* Previous hardware however needs to be aligned to a power-of-two
1229 * tile height. The simplest method for determining this is to reuse
1230 * the power-of-tile object size.
5e783301 1231 */
e28f8711 1232 return i915_gem_get_gtt_size(dev, size, tiling_mode);
5e783301
DV		 1233}
1234
de151cf6 1235int
ff72145b
DA		 1236i915_gem_mmap_gtt(struct drm_file *file,
1237 struct drm_device *dev,
1238 uint32_t handle,
1239 uint64_t *offset)
de151cf6 1240{
da761a6e 1241 struct drm_i915_private *dev_priv = dev->dev_private;
05394f39 1242 struct drm_i915_gem_object *obj;
de151cf6
JB		 1243 int ret;
1244
76c1dec1 1245 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 1246 if (ret)
76c1dec1 1247 return ret;
de151cf6 1248
ff72145b 1249 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
c8725226 1250 if (&obj->base == NULL) {
1d7cfea1
CW		 1251 ret = -ENOENT;
1252 goto unlock;
1253 }
de151cf6 1254
05394f39 1255 if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
da761a6e 1256 ret = -E2BIG;
ff56b0bc 1257 goto out;
da761a6e
CW		 1258 }
1259
05394f39 1260 if (obj->madv != I915_MADV_WILLNEED) {
ab18282d 1261 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1d7cfea1
CW		 1262 ret = -EINVAL;
1263 goto out;
ab18282d
CW		 1264 }
1265
05394f39 1266 if (!obj->base.map_list.map) {
b464e9a2 1267 ret = drm_gem_create_mmap_offset(&obj->base);
1d7cfea1
CW		 1268 if (ret)
1269 goto out;
de151cf6
JB		 1270 }
1271
ff72145b 1272 *offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
de151cf6 1273
1d7cfea1 1274out:
05394f39 1275 drm_gem_object_unreference(&obj->base);
1d7cfea1 1276unlock:
de151cf6 1277 mutex_unlock(&dev->struct_mutex);
1d7cfea1 1278 return ret;
de151cf6
JB		 1279}
1280
ff72145b
DA		 1281/**
1282 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1283 * @dev: DRM device
1284 * @data: GTT mapping ioctl data
1285 * @file: GEM object info
1286 *
1287 * Simply returns the fake offset to userspace so it can mmap it.
1288 * The mmap call will end up in drm_gem_mmap(), which will set things
1289 * up so we can get faults in the handler above.
1290 *
1291 * The fault handler will take care of binding the object into the GTT
1292 * (since it may have been evicted to make room for something), allocating
1293 * a fence register, and mapping the appropriate aperture address into
1294 * userspace.
1295 */
1296int
1297i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1298 struct drm_file *file)
1299{
1300 struct drm_i915_gem_mmap_gtt *args = data;
1301
ff72145b
DA		 1302 return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
1303}
1304
1305
e5281ccd 1306static int
05394f39 1307i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
e5281ccd
CW		 1308 gfp_t gfpmask)
1309{
e5281ccd
CW		 1310 int page_count, i;
1311 struct address_space *mapping;
1312 struct inode *inode;
1313 struct page *page;
1314
1315 /* Get the list of pages out of our struct file. They'll be pinned
1316 * at this point until we release them.
1317 */
05394f39
CW		 1318 page_count = obj->base.size / PAGE_SIZE;
1319 BUG_ON(obj->pages != NULL);
1320 obj->pages = drm_malloc_ab(page_count, sizeof(struct page *));
1321 if (obj->pages == NULL)
e5281ccd
CW		 1322 return -ENOMEM;
1323
05394f39 1324 inode = obj->base.filp->f_path.dentry->d_inode;
e5281ccd 1325 mapping = inode->i_mapping;
5949eac4
HD		 1326 gfpmask |= mapping_gfp_mask(mapping);
1327
e5281ccd 1328 for (i = 0; i < page_count; i++) {
5949eac4 1329 page = shmem_read_mapping_page_gfp(mapping, i, gfpmask);
e5281ccd
CW		 1330 if (IS_ERR(page))
1331 goto err_pages;
1332
05394f39 1333 obj->pages[i] = page;
e5281ccd
CW		 1334 }
1335
6dacfd2f 1336 if (i915_gem_object_needs_bit17_swizzle(obj))
e5281ccd
CW		 1337 i915_gem_object_do_bit_17_swizzle(obj);
1338
1339 return 0;
1340
1341err_pages:
1342 while (i--)
05394f39 1343 page_cache_release(obj->pages[i]);
e5281ccd 1344
05394f39
CW		 1345 drm_free_large(obj->pages);
1346 obj->pages = NULL;
e5281ccd
CW		 1347 return PTR_ERR(page);
1348}
1349
5cdf5881 1350static void
05394f39 1351i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
673a394b 1352{
05394f39 1353 int page_count = obj->base.size / PAGE_SIZE;
673a394b
EA		 1354 int i;
1355
05394f39 1356 BUG_ON(obj->madv == __I915_MADV_PURGED);
673a394b 1357
6dacfd2f 1358 if (i915_gem_object_needs_bit17_swizzle(obj))
280b713b
EA		 1359 i915_gem_object_save_bit_17_swizzle(obj);
1360
05394f39
CW		 1361 if (obj->madv == I915_MADV_DONTNEED)
1362 obj->dirty = 0;
3ef94daa
CW		 1363
1364 for (i = 0; i < page_count; i++) {
05394f39
CW		 1365 if (obj->dirty)
1366 set_page_dirty(obj->pages[i]);
3ef94daa 1367
05394f39
CW		 1368 if (obj->madv == I915_MADV_WILLNEED)
1369 mark_page_accessed(obj->pages[i]);
3ef94daa 1370
05394f39 1371 page_cache_release(obj->pages[i]);
3ef94daa 1372 }
05394f39 1373 obj->dirty = 0;
673a394b 1374
05394f39
CW		 1375 drm_free_large(obj->pages);
1376 obj->pages = NULL;
673a394b
EA		 1377}
1378
54cf91dc 1379void
05394f39 1380i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1ec14ad3
CW		 1381 struct intel_ring_buffer *ring,
1382 u32 seqno)
673a394b 1383{
05394f39 1384 struct drm_device *dev = obj->base.dev;
69dc4987 1385 struct drm_i915_private *dev_priv = dev->dev_private;
617dbe27 1386
852835f3 1387 BUG_ON(ring == NULL);
05394f39 1388 obj->ring = ring;
673a394b
EA		 1389
1390 /* Add a reference if we're newly entering the active list. */
05394f39
CW		 1391 if (!obj->active) {
1392 drm_gem_object_reference(&obj->base);
1393 obj->active = 1;
673a394b 1394 }
e35a41de 1395
673a394b 1396 /* Move from whatever list we were on to the tail of execution. */
05394f39
CW		 1397 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
1398 list_move_tail(&obj->ring_list, &ring->active_list);
caea7476 1399
05394f39 1400 obj->last_rendering_seqno = seqno;
caea7476 1401
7dd49065 1402 if (obj->fenced_gpu_access) {
caea7476 1403 obj->last_fenced_seqno = seqno;
caea7476 1404
7dd49065
CW		 1405 /* Bump MRU to take account of the delayed flush */
1406 if (obj->fence_reg != I915_FENCE_REG_NONE) {
1407 struct drm_i915_fence_reg *reg;
1408
1409 reg = &dev_priv->fence_regs[obj->fence_reg];
1410 list_move_tail(&reg->lru_list,
1411 &dev_priv->mm.fence_list);
1412 }
caea7476
CW		 1413 }
1414}
1415
1416static void
1417i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
1418{
1419 list_del_init(&obj->ring_list);
1420 obj->last_rendering_seqno = 0;
15a13bbd 1421 obj->last_fenced_seqno = 0;
673a394b
EA		 1422}
1423
ce44b0ea 1424static void
05394f39 1425i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
ce44b0ea 1426{
05394f39 1427 struct drm_device *dev = obj->base.dev;
ce44b0ea 1428 drm_i915_private_t *dev_priv = dev->dev_private;
ce44b0ea 1429
05394f39
CW		 1430 BUG_ON(!obj->active);
1431 list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
caea7476
CW		 1432
1433 i915_gem_object_move_off_active(obj);
1434}
1435
1436static void
1437i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1438{
1439 struct drm_device *dev = obj->base.dev;
1440 struct drm_i915_private *dev_priv = dev->dev_private;
1441
1b50247a 1442 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
caea7476
CW		 1443
1444 BUG_ON(!list_empty(&obj->gpu_write_list));
1445 BUG_ON(!obj->active);
1446 obj->ring = NULL;
1447
1448 i915_gem_object_move_off_active(obj);
1449 obj->fenced_gpu_access = false;
caea7476
CW		 1450
1451 obj->active = 0;
87ca9c8a 1452 obj->pending_gpu_write = false;
caea7476
CW		 1453 drm_gem_object_unreference(&obj->base);
1454
1455 WARN_ON(i915_verify_lists(dev));
ce44b0ea 1456}
673a394b 1457
963b4836
CW		 1458/* Immediately discard the backing storage */
1459static void
05394f39 1460i915_gem_object_truncate(struct drm_i915_gem_object *obj)
963b4836 1461{
bb6baf76 1462 struct inode *inode;
963b4836 1463
ae9fed6b
CW		 1464 /* Our goal here is to return as much of the memory as
1465 * is possible back to the system as we are called from OOM.
1466 * To do this we must instruct the shmfs to drop all of its
e2377fe0 1467 * backing pages, *now*.
ae9fed6b 1468 */
05394f39 1469 inode = obj->base.filp->f_path.dentry->d_inode;
e2377fe0 1470 shmem_truncate_range(inode, 0, (loff_t)-1);
bb6baf76 1471
a14917ee
CW		 1472 if (obj->base.map_list.map)
1473 drm_gem_free_mmap_offset(&obj->base);
1474
05394f39 1475 obj->madv = __I915_MADV_PURGED;
963b4836
CW		 1476}
1477
1478static inline int
05394f39 1479i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
963b4836 1480{
05394f39 1481 return obj->madv == I915_MADV_DONTNEED;
963b4836
CW		 1482}
1483
63560396 1484static void
db53a302
CW		 1485i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
1486 uint32_t flush_domains)
63560396 1487{
05394f39 1488 struct drm_i915_gem_object *obj, *next;
63560396 1489
05394f39 1490 list_for_each_entry_safe(obj, next,
64193406 1491 &ring->gpu_write_list,
63560396 1492 gpu_write_list) {
05394f39
CW		 1493 if (obj->base.write_domain & flush_domains) {
1494 uint32_t old_write_domain = obj->base.write_domain;
63560396 1495
05394f39
CW		 1496 obj->base.write_domain = 0;
1497 list_del_init(&obj->gpu_write_list);
1ec14ad3 1498 i915_gem_object_move_to_active(obj, ring,
db53a302 1499 i915_gem_next_request_seqno(ring));
63560396 1500
63560396 1501 trace_i915_gem_object_change_domain(obj,
05394f39 1502 obj->base.read_domains,
63560396
DV		 1503 old_write_domain);
1504 }
1505 }
1506}
8187a2b7 1507
53d227f2
DV		 1508static u32
1509i915_gem_get_seqno(struct drm_device *dev)
1510{
1511 drm_i915_private_t *dev_priv = dev->dev_private;
1512 u32 seqno = dev_priv->next_seqno;
1513
1514 /* reserve 0 for non-seqno */
1515 if (++dev_priv->next_seqno == 0)
1516 dev_priv->next_seqno = 1;
1517
1518 return seqno;
1519}
1520
1521u32
1522i915_gem_next_request_seqno(struct intel_ring_buffer *ring)
1523{
1524 if (ring->outstanding_lazy_request == 0)
1525 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev);
1526
1527 return ring->outstanding_lazy_request;
1528}
1529
3cce469c 1530int
db53a302 1531i915_add_request(struct intel_ring_buffer *ring,
f787a5f5 1532 struct drm_file *file,
db53a302 1533 struct drm_i915_gem_request *request)
673a394b 1534{
db53a302 1535 drm_i915_private_t *dev_priv = ring->dev->dev_private;
673a394b 1536 uint32_t seqno;
a71d8d94 1537 u32 request_ring_position;
673a394b 1538 int was_empty;
3cce469c
CW		 1539 int ret;
1540
1541 BUG_ON(request == NULL);
53d227f2 1542 seqno = i915_gem_next_request_seqno(ring);
673a394b 1543
a71d8d94
CW		 1544 /* Record the position of the start of the request so that
1545 * should we detect the updated seqno part-way through the
1546 * GPU processing the request, we never over-estimate the
1547 * position of the head.
1548 */
1549 request_ring_position = intel_ring_get_tail(ring);
1550
3cce469c
CW		 1551 ret = ring->add_request(ring, &seqno);
1552 if (ret)
1553 return ret;
673a394b 1554
db53a302 1555 trace_i915_gem_request_add(ring, seqno);
673a394b
EA		 1556
1557 request->seqno = seqno;
852835f3 1558 request->ring = ring;
a71d8d94 1559 request->tail = request_ring_position;
673a394b 1560 request->emitted_jiffies = jiffies;
852835f3
ZN		 1561 was_empty = list_empty(&ring->request_list);
1562 list_add_tail(&request->list, &ring->request_list);
1563
db53a302
CW		 1564 if (file) {
1565 struct drm_i915_file_private *file_priv = file->driver_priv;
1566
1c25595f 1567 spin_lock(&file_priv->mm.lock);
f787a5f5 1568 request->file_priv = file_priv;
b962442e 1569 list_add_tail(&request->client_list,
f787a5f5 1570 &file_priv->mm.request_list);
1c25595f 1571 spin_unlock(&file_priv->mm.lock);
b962442e 1572 }
673a394b 1573
5391d0cf 1574 ring->outstanding_lazy_request = 0;
db53a302 1575
f65d9421 1576 if (!dev_priv->mm.suspended) {
3e0dc6b0
BW		 1577 if (i915_enable_hangcheck) {
1578 mod_timer(&dev_priv->hangcheck_timer,
1579 jiffies +
1580 msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
1581 }
f65d9421 1582 if (was_empty)
b3b079db
CW		 1583 queue_delayed_work(dev_priv->wq,
1584 &dev_priv->mm.retire_work, HZ);
f65d9421 1585 }
3cce469c 1586 return 0;
673a394b
EA		 1587}
1588
f787a5f5
CW		 1589static inline void
1590i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
673a394b 1591{
1c25595f 1592 struct drm_i915_file_private *file_priv = request->file_priv;
673a394b 1593
1c25595f
CW		 1594 if (!file_priv)
1595 return;
1c5d22f7 1596
1c25595f 1597 spin_lock(&file_priv->mm.lock);
09bfa517
HRK		 1598 if (request->file_priv) {
1599 list_del(&request->client_list);
1600 request->file_priv = NULL;
1601 }
1c25595f 1602 spin_unlock(&file_priv->mm.lock);
673a394b 1603}
673a394b 1604
dfaae392
CW		 1605static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1606 struct intel_ring_buffer *ring)
9375e446 1607{
dfaae392
CW		 1608 while (!list_empty(&ring->request_list)) {
1609 struct drm_i915_gem_request *request;
673a394b 1610
dfaae392
CW		 1611 request = list_first_entry(&ring->request_list,
1612 struct drm_i915_gem_request,
1613 list);
de151cf6 1614
dfaae392 1615 list_del(&request->list);
f787a5f5 1616 i915_gem_request_remove_from_client(request);
dfaae392
CW		 1617 kfree(request);
1618 }
673a394b 1619
dfaae392 1620 while (!list_empty(&ring->active_list)) {
05394f39 1621 struct drm_i915_gem_object *obj;
9375e446 1622
05394f39
CW		 1623 obj = list_first_entry(&ring->active_list,
1624 struct drm_i915_gem_object,
1625 ring_list);
9375e446 1626
05394f39
CW		 1627 obj->base.write_domain = 0;
1628 list_del_init(&obj->gpu_write_list);
1629 i915_gem_object_move_to_inactive(obj);
673a394b
EA		 1630 }
1631}
1632
312817a3
CW		 1633static void i915_gem_reset_fences(struct drm_device *dev)
1634{
1635 struct drm_i915_private *dev_priv = dev->dev_private;
1636 int i;
1637
4b9de737 1638 for (i = 0; i < dev_priv->num_fence_regs; i++) {
312817a3 1639 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
7d2cb39c 1640
ada726c7 1641 i915_gem_write_fence(dev, i, NULL);
7d2cb39c 1642
ada726c7
CW		 1643 if (reg->obj)
1644 i915_gem_object_fence_lost(reg->obj);
7d2cb39c 1645
ada726c7
CW		 1646 reg->pin_count = 0;
1647 reg->obj = NULL;
1648 INIT_LIST_HEAD(&reg->lru_list);
312817a3 1649 }
ada726c7
CW		 1650
1651 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
312817a3
CW		 1652}
1653
069efc1d 1654void i915_gem_reset(struct drm_device *dev)
673a394b 1655{
77f01230 1656 struct drm_i915_private *dev_priv = dev->dev_private;
05394f39 1657 struct drm_i915_gem_object *obj;
b4519513 1658 struct intel_ring_buffer *ring;
1ec14ad3 1659 int i;
673a394b 1660
b4519513
CW		 1661 for_each_ring(ring, dev_priv, i)
1662 i915_gem_reset_ring_lists(dev_priv, ring);
dfaae392
CW		 1663
1664 /* Remove anything from the flushing lists. The GPU cache is likely
1665 * to be lost on reset along with the data, so simply move the
1666 * lost bo to the inactive list.
1667 */
1668 while (!list_empty(&dev_priv->mm.flushing_list)) {
0206e353 1669 obj = list_first_entry(&dev_priv->mm.flushing_list,
05394f39
CW		 1670 struct drm_i915_gem_object,
1671 mm_list);
dfaae392 1672
05394f39
CW		 1673 obj->base.write_domain = 0;
1674 list_del_init(&obj->gpu_write_list);
1675 i915_gem_object_move_to_inactive(obj);
dfaae392
CW		 1676 }
1677
1678 /* Move everything out of the GPU domains to ensure we do any
1679 * necessary invalidation upon reuse.
1680 */
05394f39 1681 list_for_each_entry(obj,
77f01230 1682 &dev_priv->mm.inactive_list,
69dc4987 1683 mm_list)
77f01230 1684 {
05394f39 1685 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
77f01230 1686 }
069efc1d
CW		 1687
1688 /* The fence registers are invalidated so clear them out */
312817a3 1689 i915_gem_reset_fences(dev);
673a394b
EA		 1690}
1691
1692/**
1693 * This function clears the request list as sequence numbers are passed.
1694 */
a71d8d94 1695void
db53a302 1696i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
673a394b 1697{
673a394b 1698 uint32_t seqno;
1ec14ad3 1699 int i;
673a394b 1700
db53a302 1701 if (list_empty(&ring->request_list))
6c0594a3
KW		 1702 return;
1703
db53a302 1704 WARN_ON(i915_verify_lists(ring->dev));
673a394b 1705
78501eac 1706 seqno = ring->get_seqno(ring);
1ec14ad3 1707
076e2c0e 1708 for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
1ec14ad3
CW		 1709 if (seqno >= ring->sync_seqno[i])
1710 ring->sync_seqno[i] = 0;
1711
852835f3 1712 while (!list_empty(&ring->request_list)) {
673a394b 1713 struct drm_i915_gem_request *request;
673a394b 1714
852835f3 1715 request = list_first_entry(&ring->request_list,
673a394b
EA		 1716 struct drm_i915_gem_request,
1717 list);
673a394b 1718
dfaae392 1719 if (!i915_seqno_passed(seqno, request->seqno))
b84d5f0c
CW		 1720 break;
1721
db53a302 1722 trace_i915_gem_request_retire(ring, request->seqno);
a71d8d94
CW		 1723 /* We know the GPU must have read the request to have
1724 * sent us the seqno + interrupt, so use the position
1725 * of tail of the request to update the last known position
1726 * of the GPU head.
1727 */
1728 ring->last_retired_head = request->tail;
b84d5f0c
CW		 1729
1730 list_del(&request->list);
f787a5f5 1731 i915_gem_request_remove_from_client(request);
b84d5f0c
CW		 1732 kfree(request);
1733 }
673a394b 1734
b84d5f0c
CW		 1735 /* Move any buffers on the active list that are no longer referenced
1736 * by the ringbuffer to the flushing/inactive lists as appropriate.
1737 */
1738 while (!list_empty(&ring->active_list)) {
05394f39 1739 struct drm_i915_gem_object *obj;
b84d5f0c 1740
0206e353 1741 obj = list_first_entry(&ring->active_list,
05394f39
CW		 1742 struct drm_i915_gem_object,
1743 ring_list);
673a394b 1744
05394f39 1745 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
673a394b 1746 break;
b84d5f0c 1747
05394f39 1748 if (obj->base.write_domain != 0)
b84d5f0c
CW		 1749 i915_gem_object_move_to_flushing(obj);
1750 else
1751 i915_gem_object_move_to_inactive(obj);
673a394b 1752 }
9d34e5db 1753
db53a302
CW		 1754 if (unlikely(ring->trace_irq_seqno &&
1755 i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
1ec14ad3 1756 ring->irq_put(ring);
db53a302 1757 ring->trace_irq_seqno = 0;
9d34e5db 1758 }
23bc5982 1759
db53a302 1760 WARN_ON(i915_verify_lists(ring->dev));
673a394b
EA		 1761}
1762
b09a1fec
CW		 1763void
1764i915_gem_retire_requests(struct drm_device *dev)
1765{
1766 drm_i915_private_t *dev_priv = dev->dev_private;
b4519513 1767 struct intel_ring_buffer *ring;
1ec14ad3 1768 int i;
b09a1fec 1769
b4519513
CW		 1770 for_each_ring(ring, dev_priv, i)
1771 i915_gem_retire_requests_ring(ring);
b09a1fec
CW		 1772}
1773
75ef9da2 1774static void
673a394b
EA		 1775i915_gem_retire_work_handler(struct work_struct *work)
1776{
1777 drm_i915_private_t *dev_priv;
1778 struct drm_device *dev;
b4519513 1779 struct intel_ring_buffer *ring;
0a58705b
CW		 1780 bool idle;
1781 int i;
673a394b
EA		 1782
1783 dev_priv = container_of(work, drm_i915_private_t,
1784 mm.retire_work.work);
1785 dev = dev_priv->dev;
1786
891b48cf
CW		 1787 /* Come back later if the device is busy... */
1788 if (!mutex_trylock(&dev->struct_mutex)) {
1789 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1790 return;
1791 }
1792
b09a1fec 1793 i915_gem_retire_requests(dev);
d1b851fc 1794
0a58705b
CW		 1795 /* Send a periodic flush down the ring so we don't hold onto GEM
1796 * objects indefinitely.
1797 */
1798 idle = true;
b4519513 1799 for_each_ring(ring, dev_priv, i) {
0a58705b
CW		 1800 if (!list_empty(&ring->gpu_write_list)) {
1801 struct drm_i915_gem_request *request;
1802 int ret;
1803
db53a302
CW		 1804 ret = i915_gem_flush_ring(ring,
1805 0, I915_GEM_GPU_DOMAINS);
0a58705b
CW		 1806 request = kzalloc(sizeof(*request), GFP_KERNEL);
1807 if (ret || request == NULL ||
db53a302 1808 i915_add_request(ring, NULL, request))
0a58705b
CW		 1809 kfree(request);
1810 }
1811
1812 idle &= list_empty(&ring->request_list);
1813 }
1814
1815 if (!dev_priv->mm.suspended && !idle)
9c9fe1f8 1816 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
0a58705b 1817
673a394b
EA		 1818 mutex_unlock(&dev->struct_mutex);
1819}
1820
b4aca010
BW		 1821static int
1822i915_gem_check_wedge(struct drm_i915_private *dev_priv)
1823{
1824 BUG_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex));
1825
1826 if (atomic_read(&dev_priv->mm.wedged)) {
1827 struct completion *x = &dev_priv->error_completion;
1828 bool recovery_complete;
1829 unsigned long flags;
1830
1831 /* Give the error handler a chance to run. */
1832 spin_lock_irqsave(&x->wait.lock, flags);
1833 recovery_complete = x->done > 0;
1834 spin_unlock_irqrestore(&x->wait.lock, flags);
1835
1836 return recovery_complete ? -EIO : -EAGAIN;
1837 }
1838
1839 return 0;
1840}
1841
1842/*
1843 * Compare seqno against outstanding lazy request. Emit a request if they are
1844 * equal.
1845 */
1846static int
1847i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
1848{
1849 int ret = 0;
1850
1851 BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
1852
1853 if (seqno == ring->outstanding_lazy_request) {
1854 struct drm_i915_gem_request *request;
1855
1856 request = kzalloc(sizeof(*request), GFP_KERNEL);
1857 if (request == NULL)
1858 return -ENOMEM;
1859
1860 ret = i915_add_request(ring, NULL, request);
1861 if (ret) {
1862 kfree(request);
1863 return ret;
1864 }
1865
1866 BUG_ON(seqno != request->seqno);
1867 }
1868
1869 return ret;
1870}
1871
604dd3ec
BW		 1872static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
1873 bool interruptible)
1874{
1875 drm_i915_private_t *dev_priv = ring->dev->dev_private;
1876 int ret = 0;
1877
1878 if (i915_seqno_passed(ring->get_seqno(ring), seqno))
1879 return 0;
1880
1881 trace_i915_gem_request_wait_begin(ring, seqno);
1882 if (WARN_ON(!ring->irq_get(ring)))
1883 return -ENODEV;
1884
1885#define EXIT_COND \
1886 (i915_seqno_passed(ring->get_seqno(ring), seqno) || \
1887 atomic_read(&dev_priv->mm.wedged))
1888
1889 if (interruptible)
1890 ret = wait_event_interruptible(ring->irq_queue,
1891 EXIT_COND);
1892 else
1893 wait_event(ring->irq_queue, EXIT_COND);
1894
1895 ring->irq_put(ring);
1896 trace_i915_gem_request_wait_end(ring, seqno);
1897#undef EXIT_COND
1898
1899 return ret;
1900}
1901
db53a302
CW		 1902/**
1903 * Waits for a sequence number to be signaled, and cleans up the
1904 * request and object lists appropriately for that event.
1905 */
5a5a0c64 1906int
db53a302 1907i915_wait_request(struct intel_ring_buffer *ring,
b2da9fe5 1908 uint32_t seqno)
673a394b 1909{
db53a302 1910 drm_i915_private_t *dev_priv = ring->dev->dev_private;
673a394b
EA		 1911 int ret = 0;
1912
1913 BUG_ON(seqno == 0);
1914
b4aca010
BW		 1915 ret = i915_gem_check_wedge(dev_priv);
1916 if (ret)
1917 return ret;
3cce469c 1918
b4aca010
BW		 1919 ret = i915_gem_check_olr(ring, seqno);
1920 if (ret)
1921 return ret;
ffed1d09 1922
604dd3ec 1923 ret = __wait_seqno(ring, seqno, dev_priv->mm.interruptible);
ba1234d1 1924 if (atomic_read(&dev_priv->mm.wedged))
30dbf0c0 1925 ret = -EAGAIN;
673a394b 1926
673a394b
EA		 1927 return ret;
1928}
1929
673a394b
EA		 1930/**
1931 * Ensures that all rendering to the object has completed and the object is
1932 * safe to unbind from the GTT or access from the CPU.
1933 */
54cf91dc 1934int
ce453d81 1935i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
673a394b 1936{
673a394b
EA		 1937 int ret;
1938
e47c68e9
EA		 1939 /* This function only exists to support waiting for existing rendering,
1940 * not for emitting required flushes.
673a394b 1941 */
05394f39 1942 BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
673a394b
EA		 1943
1944 /* If there is rendering queued on the buffer being evicted, wait for
1945 * it.
1946 */
05394f39 1947 if (obj->active) {
b2da9fe5 1948 ret = i915_wait_request(obj->ring, obj->last_rendering_seqno);
2cf34d7b 1949 if (ret)
673a394b 1950 return ret;
b2da9fe5 1951 i915_gem_retire_requests_ring(obj->ring);
673a394b
EA		 1952 }
1953
1954 return 0;
1955}
1956
5816d648
BW		 1957/**
1958 * i915_gem_object_sync - sync an object to a ring.
1959 *
1960 * @obj: object which may be in use on another ring.
1961 * @to: ring we wish to use the object on. May be NULL.
1962 *
1963 * This code is meant to abstract object synchronization with the GPU.
1964 * Calling with NULL implies synchronizing the object with the CPU
1965 * rather than a particular GPU ring.
1966 *
1967 * Returns 0 if successful, else propagates up the lower layer error.
1968 */
2911a35b
BW		 1969int
1970i915_gem_object_sync(struct drm_i915_gem_object *obj,
1971 struct intel_ring_buffer *to)
1972{
1973 struct intel_ring_buffer *from = obj->ring;
1974 u32 seqno;
1975 int ret, idx;
1976
1977 if (from == NULL || to == from)
1978 return 0;
1979
5816d648 1980 if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
2911a35b
BW		 1981 return i915_gem_object_wait_rendering(obj);
1982
1983 idx = intel_ring_sync_index(from, to);
1984
1985 seqno = obj->last_rendering_seqno;
1986 if (seqno <= from->sync_seqno[idx])
1987 return 0;
1988
b4aca010
BW		 1989 ret = i915_gem_check_olr(obj->ring, seqno);
1990 if (ret)
1991 return ret;
2911a35b 1992
1500f7ea 1993 ret = to->sync_to(to, from, seqno);
e3a5a225
BW		 1994 if (!ret)
1995 from->sync_seqno[idx] = seqno;
2911a35b 1996
e3a5a225 1997 return ret;
2911a35b
BW		 1998}
1999
b5ffc9bc
CW		 2000static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
2001{
2002 u32 old_write_domain, old_read_domains;
2003
b5ffc9bc
CW		 2004 /* Act a barrier for all accesses through the GTT */
2005 mb();
2006
2007 /* Force a pagefault for domain tracking on next user access */
2008 i915_gem_release_mmap(obj);
2009
b97c3d9c
KP		 2010 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
2011 return;
2012
b5ffc9bc
CW		 2013 old_read_domains = obj->base.read_domains;
2014 old_write_domain = obj->base.write_domain;
2015
2016 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
2017 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2018
2019 trace_i915_gem_object_change_domain(obj,
2020 old_read_domains,
2021 old_write_domain);
2022}
2023
673a394b
EA		 2024/**
2025 * Unbinds an object from the GTT aperture.
2026 */
0f973f27 2027int
05394f39 2028i915_gem_object_unbind(struct drm_i915_gem_object *obj)
673a394b 2029{
7bddb01f 2030 drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
673a394b
EA		 2031 int ret = 0;
2032
05394f39 2033 if (obj->gtt_space == NULL)
673a394b
EA		 2034 return 0;
2035
05394f39 2036 if (obj->pin_count != 0) {
673a394b
EA		 2037 DRM_ERROR("Attempting to unbind pinned buffer\n");
2038 return -EINVAL;
2039 }
2040
a8198eea 2041 ret = i915_gem_object_finish_gpu(obj);
1488fc08 2042 if (ret)
a8198eea
CW		 2043 return ret;
2044 /* Continue on if we fail due to EIO, the GPU is hung so we
2045 * should be safe and we need to cleanup or else we might
2046 * cause memory corruption through use-after-free.
2047 */
2048
b5ffc9bc 2049 i915_gem_object_finish_gtt(obj);
5323fd04 2050
673a394b
EA		 2051 /* Move the object to the CPU domain to ensure that
2052 * any possible CPU writes while it's not in the GTT
a8198eea 2053 * are flushed when we go to remap it.
673a394b 2054 */
a8198eea
CW		 2055 if (ret == 0)
2056 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
8dc1775d 2057 if (ret == -ERESTARTSYS)
673a394b 2058 return ret;
812ed492 2059 if (ret) {
a8198eea
CW		 2060 /* In the event of a disaster, abandon all caches and
2061 * hope for the best.
2062 */
812ed492 2063 i915_gem_clflush_object(obj);
05394f39 2064 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
812ed492 2065 }
673a394b 2066
96b47b65 2067 /* release the fence reg _after_ flushing */
d9e86c0e 2068 ret = i915_gem_object_put_fence(obj);
1488fc08 2069 if (ret)
d9e86c0e 2070 return ret;
96b47b65 2071
db53a302
CW		 2072 trace_i915_gem_object_unbind(obj);
2073
74898d7e
DV		 2074 if (obj->has_global_gtt_mapping)
2075 i915_gem_gtt_unbind_object(obj);
7bddb01f
DV		 2076 if (obj->has_aliasing_ppgtt_mapping) {
2077 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
2078 obj->has_aliasing_ppgtt_mapping = 0;
2079 }
74163907 2080 i915_gem_gtt_finish_object(obj);
7bddb01f 2081
e5281ccd 2082 i915_gem_object_put_pages_gtt(obj);
673a394b 2083
6299f992 2084 list_del_init(&obj->gtt_list);
05394f39 2085 list_del_init(&obj->mm_list);
75e9e915 2086 /* Avoid an unnecessary call to unbind on rebind. */
05394f39 2087 obj->map_and_fenceable = true;
673a394b 2088
05394f39
CW		 2089 drm_mm_put_block(obj->gtt_space);
2090 obj->gtt_space = NULL;
2091 obj->gtt_offset = 0;
673a394b 2092
05394f39 2093 if (i915_gem_object_is_purgeable(obj))
963b4836
CW		 2094 i915_gem_object_truncate(obj);
2095
8dc1775d 2096 return ret;
673a394b
EA		 2097}
2098
88241785 2099int
db53a302 2100i915_gem_flush_ring(struct intel_ring_buffer *ring,
54cf91dc
CW		 2101 uint32_t invalidate_domains,
2102 uint32_t flush_domains)
2103{
88241785
CW		 2104 int ret;
2105
36d527de
CW		 2106 if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
2107 return 0;
2108
db53a302
CW		 2109 trace_i915_gem_ring_flush(ring, invalidate_domains, flush_domains);
2110
88241785
CW		 2111 ret = ring->flush(ring, invalidate_domains, flush_domains);
2112 if (ret)
2113 return ret;
2114
36d527de
CW		 2115 if (flush_domains & I915_GEM_GPU_DOMAINS)
2116 i915_gem_process_flushing_list(ring, flush_domains);
2117
88241785 2118 return 0;
54cf91dc
CW		 2119}
2120
b2da9fe5 2121static int i915_ring_idle(struct intel_ring_buffer *ring)
a56ba56c 2122{
88241785
CW		 2123 int ret;
2124
395b70be 2125 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
64193406
CW		 2126 return 0;
2127
88241785 2128 if (!list_empty(&ring->gpu_write_list)) {
db53a302 2129 ret = i915_gem_flush_ring(ring,
0ac74c6b 2130 I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
88241785
CW		 2131 if (ret)
2132 return ret;
2133 }
2134
b2da9fe5 2135 return i915_wait_request(ring, i915_gem_next_request_seqno(ring));
a56ba56c
CW		 2136}
2137
b2da9fe5 2138int i915_gpu_idle(struct drm_device *dev)
4df2faf4
DV		 2139{
2140 drm_i915_private_t *dev_priv = dev->dev_private;
b4519513 2141 struct intel_ring_buffer *ring;
1ec14ad3 2142 int ret, i;
4df2faf4 2143
4df2faf4 2144 /* Flush everything onto the inactive list. */
b4519513
CW		 2145 for_each_ring(ring, dev_priv, i) {
2146 ret = i915_ring_idle(ring);
1ec14ad3
CW		 2147 if (ret)
2148 return ret;
b4519513
CW		 2149
2150 /* Is the device fubar? */
2151 if (WARN_ON(!list_empty(&ring->gpu_write_list)))
2152 return -EBUSY;
1ec14ad3 2153 }
4df2faf4 2154
8a1a49f9 2155 return 0;
4df2faf4
DV		 2156}
2157
9ce079e4
CW		 2158static void sandybridge_write_fence_reg(struct drm_device *dev, int reg,
2159 struct drm_i915_gem_object *obj)
4e901fdc 2160{
4e901fdc 2161 drm_i915_private_t *dev_priv = dev->dev_private;
4e901fdc
EA		 2162 uint64_t val;
2163
9ce079e4
CW		 2164 if (obj) {
2165 u32 size = obj->gtt_space->size;
4e901fdc 2166
9ce079e4
CW		 2167 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2168 0xfffff000) << 32;
2169 val |= obj->gtt_offset & 0xfffff000;
2170 val |= (uint64_t)((obj->stride / 128) - 1) <<
2171 SANDYBRIDGE_FENCE_PITCH_SHIFT;
4e901fdc 2172
9ce079e4
CW		 2173 if (obj->tiling_mode == I915_TILING_Y)
2174 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2175 val |= I965_FENCE_REG_VALID;
2176 } else
2177 val = 0;
c6642782 2178
9ce079e4
CW		 2179 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
2180 POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
4e901fdc
EA		 2181}
2182
9ce079e4
CW		 2183static void i965_write_fence_reg(struct drm_device *dev, int reg,
2184 struct drm_i915_gem_object *obj)
de151cf6 2185{
de151cf6 2186 drm_i915_private_t *dev_priv = dev->dev_private;
de151cf6
JB		 2187 uint64_t val;
2188
9ce079e4
CW		 2189 if (obj) {
2190 u32 size = obj->gtt_space->size;
de151cf6 2191
9ce079e4
CW		 2192 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2193 0xfffff000) << 32;
2194 val |= obj->gtt_offset & 0xfffff000;
2195 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2196 if (obj->tiling_mode == I915_TILING_Y)
2197 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2198 val |= I965_FENCE_REG_VALID;
2199 } else
2200 val = 0;
c6642782 2201
9ce079e4
CW		 2202 I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
2203 POSTING_READ(FENCE_REG_965_0 + reg * 8);
de151cf6
JB		 2204}
2205
9ce079e4
CW		 2206static void i915_write_fence_reg(struct drm_device *dev, int reg,
2207 struct drm_i915_gem_object *obj)
de151cf6 2208{
de151cf6 2209 drm_i915_private_t *dev_priv = dev->dev_private;
9ce079e4 2210 u32 val;
de151cf6 2211
9ce079e4
CW		 2212 if (obj) {
2213 u32 size = obj->gtt_space->size;
2214 int pitch_val;
2215 int tile_width;
c6642782 2216
9ce079e4
CW		 2217 WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
2218 (size & -size) != size ||
2219 (obj->gtt_offset & (size - 1)),
2220 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2221 obj->gtt_offset, obj->map_and_fenceable, size);
c6642782 2222
9ce079e4
CW		 2223 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
2224 tile_width = 128;
2225 else
2226 tile_width = 512;
2227
2228 /* Note: pitch better be a power of two tile widths */
2229 pitch_val = obj->stride / tile_width;
2230 pitch_val = ffs(pitch_val) - 1;
2231
2232 val = obj->gtt_offset;
2233 if (obj->tiling_mode == I915_TILING_Y)
2234 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2235 val |= I915_FENCE_SIZE_BITS(size);
2236 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2237 val |= I830_FENCE_REG_VALID;
2238 } else
2239 val = 0;
2240
2241 if (reg < 8)
2242 reg = FENCE_REG_830_0 + reg * 4;
2243 else
2244 reg = FENCE_REG_945_8 + (reg - 8) * 4;
2245
2246 I915_WRITE(reg, val);
2247 POSTING_READ(reg);
de151cf6
JB		 2248}
2249
9ce079e4
CW		 2250static void i830_write_fence_reg(struct drm_device *dev, int reg,
2251 struct drm_i915_gem_object *obj)
de151cf6 2252{
de151cf6 2253 drm_i915_private_t *dev_priv = dev->dev_private;
de151cf6 2254 uint32_t val;
de151cf6 2255
9ce079e4
CW		 2256 if (obj) {
2257 u32 size = obj->gtt_space->size;
2258 uint32_t pitch_val;
de151cf6 2259
9ce079e4
CW		 2260 WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
2261 (size & -size) != size ||
2262 (obj->gtt_offset & (size - 1)),
2263 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
2264 obj->gtt_offset, size);
e76a16de 2265
9ce079e4
CW		 2266 pitch_val = obj->stride / 128;
2267 pitch_val = ffs(pitch_val) - 1;
de151cf6 2268
9ce079e4
CW		 2269 val = obj->gtt_offset;
2270 if (obj->tiling_mode == I915_TILING_Y)
2271 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2272 val |= I830_FENCE_SIZE_BITS(size);
2273 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2274 val |= I830_FENCE_REG_VALID;
2275 } else
2276 val = 0;
c6642782 2277
9ce079e4
CW		 2278 I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
2279 POSTING_READ(FENCE_REG_830_0 + reg * 4);
2280}
2281
2282static void i915_gem_write_fence(struct drm_device *dev, int reg,
2283 struct drm_i915_gem_object *obj)
2284{
2285 switch (INTEL_INFO(dev)->gen) {
2286 case 7:
2287 case 6: sandybridge_write_fence_reg(dev, reg, obj); break;
2288 case 5:
2289 case 4: i965_write_fence_reg(dev, reg, obj); break;
2290 case 3: i915_write_fence_reg(dev, reg, obj); break;
2291 case 2: i830_write_fence_reg(dev, reg, obj); break;
2292 default: break;
2293 }
de151cf6
JB		 2294}
2295
61050808
CW		 2296static inline int fence_number(struct drm_i915_private *dev_priv,
2297 struct drm_i915_fence_reg *fence)
2298{
2299 return fence - dev_priv->fence_regs;
2300}
2301
2302static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
2303 struct drm_i915_fence_reg *fence,
2304 bool enable)
2305{
2306 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2307 int reg = fence_number(dev_priv, fence);
2308
2309 i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
2310
2311 if (enable) {
2312 obj->fence_reg = reg;
2313 fence->obj = obj;
2314 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
2315 } else {
2316 obj->fence_reg = I915_FENCE_REG_NONE;
2317 fence->obj = NULL;
2318 list_del_init(&fence->lru_list);
2319 }
2320}
2321
d9e86c0e 2322static int
a360bb1a 2323i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
d9e86c0e
CW		 2324{
2325 int ret;
2326
2327 if (obj->fenced_gpu_access) {
88241785 2328 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
1c293ea3 2329 ret = i915_gem_flush_ring(obj->ring,
88241785
CW		 2330 0, obj->base.write_domain);
2331 if (ret)
2332 return ret;
2333 }
d9e86c0e
CW		 2334
2335 obj->fenced_gpu_access = false;
2336 }
2337
1c293ea3 2338 if (obj->last_fenced_seqno) {
b2da9fe5 2339 ret = i915_wait_request(obj->ring, obj->last_fenced_seqno);
18991845
CW		 2340 if (ret)
2341 return ret;
d9e86c0e
CW		 2342
2343 obj->last_fenced_seqno = 0;
d9e86c0e
CW		 2344 }
2345
63256ec5
CW		 2346 /* Ensure that all CPU reads are completed before installing a fence
2347 * and all writes before removing the fence.
2348 */
2349 if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
2350 mb();
2351
d9e86c0e
CW		 2352 return 0;
2353}
2354
2355int
2356i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2357{
61050808 2358 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
d9e86c0e
CW		 2359 int ret;
2360
a360bb1a 2361 ret = i915_gem_object_flush_fence(obj);
d9e86c0e
CW		 2362 if (ret)
2363 return ret;
2364
61050808
CW		 2365 if (obj->fence_reg == I915_FENCE_REG_NONE)
2366 return 0;
d9e86c0e 2367
61050808
CW		 2368 i915_gem_object_update_fence(obj,
2369 &dev_priv->fence_regs[obj->fence_reg],
2370 false);
2371 i915_gem_object_fence_lost(obj);
d9e86c0e
CW		 2372
2373 return 0;
2374}
2375
2376static struct drm_i915_fence_reg *
a360bb1a 2377i915_find_fence_reg(struct drm_device *dev)
ae3db24a 2378{
ae3db24a 2379 struct drm_i915_private *dev_priv = dev->dev_private;
8fe301ad 2380 struct drm_i915_fence_reg *reg, *avail;
d9e86c0e 2381 int i;
ae3db24a
DV		 2382
2383 /* First try to find a free reg */
d9e86c0e 2384 avail = NULL;
ae3db24a
DV		 2385 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2386 reg = &dev_priv->fence_regs[i];
2387 if (!reg->obj)
d9e86c0e 2388 return reg;
ae3db24a 2389
1690e1eb 2390 if (!reg->pin_count)
d9e86c0e 2391 avail = reg;
ae3db24a
DV		 2392 }
2393
d9e86c0e
CW		 2394 if (avail == NULL)
2395 return NULL;
ae3db24a
DV		 2396
2397 /* None available, try to steal one or wait for a user to finish */
d9e86c0e 2398 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
1690e1eb 2399 if (reg->pin_count)
ae3db24a
DV		 2400 continue;
2401
8fe301ad 2402 return reg;
ae3db24a
DV		 2403 }
2404
8fe301ad 2405 return NULL;
ae3db24a
DV		 2406}
2407
de151cf6 2408/**
9a5a53b3 2409 * i915_gem_object_get_fence - set up fencing for an object
de151cf6
JB		 2410 * @obj: object to map through a fence reg
2411 *
2412 * When mapping objects through the GTT, userspace wants to be able to write
2413 * to them without having to worry about swizzling if the object is tiled.
de151cf6
JB		 2414 * This function walks the fence regs looking for a free one for @obj,
2415 * stealing one if it can't find any.
2416 *
2417 * It then sets up the reg based on the object's properties: address, pitch
2418 * and tiling format.
9a5a53b3
CW		 2419 *
2420 * For an untiled surface, this removes any existing fence.
de151cf6 2421 */
8c4b8c3f 2422int
06d98131 2423i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
de151cf6 2424{
05394f39 2425 struct drm_device *dev = obj->base.dev;
79e53945 2426 struct drm_i915_private *dev_priv = dev->dev_private;
14415745 2427 bool enable = obj->tiling_mode != I915_TILING_NONE;
d9e86c0e 2428 struct drm_i915_fence_reg *reg;
ae3db24a 2429 int ret;
de151cf6 2430
14415745
CW		 2431 /* Have we updated the tiling parameters upon the object and so
2432 * will need to serialise the write to the associated fence register?
2433 */
5d82e3e6 2434 if (obj->fence_dirty) {
14415745
CW		 2435 ret = i915_gem_object_flush_fence(obj);
2436 if (ret)
2437 return ret;
2438 }
9a5a53b3 2439
d9e86c0e 2440 /* Just update our place in the LRU if our fence is getting reused. */
05394f39
CW		 2441 if (obj->fence_reg != I915_FENCE_REG_NONE) {
2442 reg = &dev_priv->fence_regs[obj->fence_reg];
5d82e3e6 2443 if (!obj->fence_dirty) {
14415745
CW		 2444 list_move_tail(&reg->lru_list,
2445 &dev_priv->mm.fence_list);
2446 return 0;
2447 }
2448 } else if (enable) {
2449 reg = i915_find_fence_reg(dev);
2450 if (reg == NULL)
2451 return -EDEADLK;
d9e86c0e 2452
14415745
CW		 2453 if (reg->obj) {
2454 struct drm_i915_gem_object *old = reg->obj;
2455
2456 ret = i915_gem_object_flush_fence(old);
29c5a587
CW		 2457 if (ret)
2458 return ret;
2459
14415745 2460 i915_gem_object_fence_lost(old);
29c5a587 2461 }
14415745 2462 } else
a09ba7fa 2463 return 0;
a09ba7fa 2464
14415745 2465 i915_gem_object_update_fence(obj, reg, enable);
5d82e3e6 2466 obj->fence_dirty = false;
14415745 2467
9ce079e4 2468 return 0;
de151cf6
JB		 2469}
2470
673a394b
EA		 2471/**
2472 * Finds free space in the GTT aperture and binds the object there.
2473 */
2474static int
05394f39 2475i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
920afa77 2476 unsigned alignment,
75e9e915 2477 bool map_and_fenceable)
673a394b 2478{
05394f39 2479 struct drm_device *dev = obj->base.dev;
673a394b 2480 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 2481 struct drm_mm_node *free_space;
a00b10c3 2482 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
5e783301 2483 u32 size, fence_size, fence_alignment, unfenced_alignment;
75e9e915 2484 bool mappable, fenceable;
07f73f69 2485 int ret;
673a394b 2486
05394f39 2487 if (obj->madv != I915_MADV_WILLNEED) {
3ef94daa
CW		 2488 DRM_ERROR("Attempting to bind a purgeable object\n");
2489 return -EINVAL;
2490 }
2491
e28f8711
CW		 2492 fence_size = i915_gem_get_gtt_size(dev,
2493 obj->base.size,
2494 obj->tiling_mode);
2495 fence_alignment = i915_gem_get_gtt_alignment(dev,
2496 obj->base.size,
2497 obj->tiling_mode);
2498 unfenced_alignment =
2499 i915_gem_get_unfenced_gtt_alignment(dev,
2500 obj->base.size,
2501 obj->tiling_mode);
a00b10c3 2502
673a394b 2503 if (alignment == 0)
5e783301
DV		 2504 alignment = map_and_fenceable ? fence_alignment :
2505 unfenced_alignment;
75e9e915 2506 if (map_and_fenceable && alignment & (fence_alignment - 1)) {
673a394b
EA		 2507 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2508 return -EINVAL;
2509 }
2510
05394f39 2511 size = map_and_fenceable ? fence_size : obj->base.size;
a00b10c3 2512
654fc607
CW		 2513 /* If the object is bigger than the entire aperture, reject it early
2514 * before evicting everything in a vain attempt to find space.
2515 */
05394f39 2516 if (obj->base.size >
75e9e915 2517 (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
654fc607
CW		 2518 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2519 return -E2BIG;
2520 }
2521
673a394b 2522 search_free:
75e9e915 2523 if (map_and_fenceable)
920afa77
DV		 2524 free_space =
2525 drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
a00b10c3 2526 size, alignment, 0,
920afa77
DV		 2527 dev_priv->mm.gtt_mappable_end,
2528 0);
2529 else
2530 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
a00b10c3 2531 size, alignment, 0);
920afa77
DV		 2532
2533 if (free_space != NULL) {
75e9e915 2534 if (map_and_fenceable)
05394f39 2535 obj->gtt_space =
920afa77 2536 drm_mm_get_block_range_generic(free_space,
a00b10c3 2537 size, alignment, 0,
920afa77
DV		 2538 dev_priv->mm.gtt_mappable_end,
2539 0);
2540 else
05394f39 2541 obj->gtt_space =
a00b10c3 2542 drm_mm_get_block(free_space, size, alignment);
920afa77 2543 }
05394f39 2544 if (obj->gtt_space == NULL) {
673a394b
EA		 2545 /* If the gtt is empty and we're still having trouble
2546 * fitting our object in, we're out of memory.
2547 */
75e9e915
DV		 2548 ret = i915_gem_evict_something(dev, size, alignment,
2549 map_and_fenceable);
9731129c 2550 if (ret)
673a394b 2551 return ret;
9731129c 2552
673a394b
EA		 2553 goto search_free;
2554 }
2555
e5281ccd 2556 ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
673a394b 2557 if (ret) {
05394f39
CW		 2558 drm_mm_put_block(obj->gtt_space);
2559 obj->gtt_space = NULL;
07f73f69
CW		 2560
2561 if (ret == -ENOMEM) {
809b6334
CW		 2562 /* first try to reclaim some memory by clearing the GTT */
2563 ret = i915_gem_evict_everything(dev, false);
07f73f69 2564 if (ret) {
07f73f69 2565 /* now try to shrink everyone else */
4bdadb97
CW		 2566 if (gfpmask) {
2567 gfpmask = 0;
2568 goto search_free;
07f73f69
CW		 2569 }
2570
809b6334 2571 return -ENOMEM;
07f73f69
CW		 2572 }
2573
2574 goto search_free;
2575 }
2576
673a394b
EA		 2577 return ret;
2578 }
2579
74163907 2580 ret = i915_gem_gtt_prepare_object(obj);
7c2e6fdf 2581 if (ret) {
e5281ccd 2582 i915_gem_object_put_pages_gtt(obj);
05394f39
CW		 2583 drm_mm_put_block(obj->gtt_space);
2584 obj->gtt_space = NULL;
07f73f69 2585
809b6334 2586 if (i915_gem_evict_everything(dev, false))
07f73f69 2587 return ret;
07f73f69
CW		 2588
2589 goto search_free;
673a394b 2590 }
673a394b 2591
0ebb9829
DV		 2592 if (!dev_priv->mm.aliasing_ppgtt)
2593 i915_gem_gtt_bind_object(obj, obj->cache_level);
673a394b 2594
6299f992 2595 list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
05394f39 2596 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
bf1a1092 2597
673a394b
EA		 2598 /* Assert that the object is not currently in any GPU domain. As it
2599 * wasn't in the GTT, there shouldn't be any way it could have been in
2600 * a GPU cache
2601 */
05394f39
CW		 2602 BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2603 BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
673a394b 2604
6299f992 2605 obj->gtt_offset = obj->gtt_space->start;
1c5d22f7 2606
75e9e915 2607 fenceable =
05394f39 2608 obj->gtt_space->size == fence_size &&
0206e353 2609 (obj->gtt_space->start & (fence_alignment - 1)) == 0;
a00b10c3 2610
75e9e915 2611 mappable =
05394f39 2612 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
a00b10c3 2613
05394f39 2614 obj->map_and_fenceable = mappable && fenceable;
75e9e915 2615
db53a302 2616 trace_i915_gem_object_bind(obj, map_and_fenceable);
673a394b
EA		 2617 return 0;
2618}
2619
2620void
05394f39 2621i915_gem_clflush_object(struct drm_i915_gem_object *obj)
673a394b 2622{
673a394b
EA		 2623 /* If we don't have a page list set up, then we're not pinned
2624 * to GPU, and we can ignore the cache flush because it'll happen
2625 * again at bind time.
2626 */
05394f39 2627 if (obj->pages == NULL)
673a394b
EA		 2628 return;
2629
9c23f7fc
CW		 2630 /* If the GPU is snooping the contents of the CPU cache,
2631 * we do not need to manually clear the CPU cache lines. However,
2632 * the caches are only snooped when the render cache is
2633 * flushed/invalidated. As we always have to emit invalidations
2634 * and flushes when moving into and out of the RENDER domain, correct
2635 * snooping behaviour occurs naturally as the result of our domain
2636 * tracking.
2637 */
2638 if (obj->cache_level != I915_CACHE_NONE)
2639 return;
2640
1c5d22f7 2641 trace_i915_gem_object_clflush(obj);
cfa16a0d 2642
05394f39 2643 drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
673a394b
EA		 2644}
2645
e47c68e9 2646/** Flushes any GPU write domain for the object if it's dirty. */
88241785 2647static int
3619df03 2648i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
e47c68e9 2649{
05394f39 2650 if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
88241785 2651 return 0;
e47c68e9
EA		 2652
2653 /* Queue the GPU write cache flushing we need. */
db53a302 2654 return i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
e47c68e9
EA		 2655}
2656
2657/** Flushes the GTT write domain for the object if it's dirty. */
2658static void
05394f39 2659i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
e47c68e9 2660{
1c5d22f7
CW		 2661 uint32_t old_write_domain;
2662
05394f39 2663 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
e47c68e9
EA		 2664 return;
2665
63256ec5 2666 /* No actual flushing is required for the GTT write domain. Writes
e47c68e9
EA		 2667 * to it immediately go to main memory as far as we know, so there's
2668 * no chipset flush. It also doesn't land in render cache.
63256ec5
CW		 2669 *
2670 * However, we do have to enforce the order so that all writes through
2671 * the GTT land before any writes to the device, such as updates to
2672 * the GATT itself.
e47c68e9 2673 */
63256ec5
CW		 2674 wmb();
2675
05394f39
CW		 2676 old_write_domain = obj->base.write_domain;
2677 obj->base.write_domain = 0;
1c5d22f7
CW		 2678
2679 trace_i915_gem_object_change_domain(obj,
05394f39 2680 obj->base.read_domains,
1c5d22f7 2681 old_write_domain);
e47c68e9
EA		 2682}
2683
2684/** Flushes the CPU write domain for the object if it's dirty. */
2685static void
05394f39 2686i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
e47c68e9 2687{
1c5d22f7 2688 uint32_t old_write_domain;
e47c68e9 2689
05394f39 2690 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
e47c68e9
EA		 2691 return;
2692
2693 i915_gem_clflush_object(obj);
40ce6575 2694 intel_gtt_chipset_flush();
05394f39
CW		 2695 old_write_domain = obj->base.write_domain;
2696 obj->base.write_domain = 0;
1c5d22f7
CW		 2697
2698 trace_i915_gem_object_change_domain(obj,
05394f39 2699 obj->base.read_domains,
1c5d22f7 2700 old_write_domain);
e47c68e9
EA		 2701}
2702
2ef7eeaa
EA		 2703/**
2704 * Moves a single object to the GTT read, and possibly write domain.
2705 *
2706 * This function returns when the move is complete, including waiting on
2707 * flushes to occur.
2708 */
79e53945 2709int
2021746e 2710i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
2ef7eeaa 2711{
8325a09d 2712 drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
1c5d22f7 2713 uint32_t old_write_domain, old_read_domains;
e47c68e9 2714 int ret;
2ef7eeaa 2715
02354392 2716 /* Not valid to be called on unbound objects. */
05394f39 2717 if (obj->gtt_space == NULL)
02354392
EA		 2718 return -EINVAL;
2719
8d7e3de1
CW		 2720 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
2721 return 0;
2722
88241785
CW		 2723 ret = i915_gem_object_flush_gpu_write_domain(obj);
2724 if (ret)
2725 return ret;
2726
87ca9c8a 2727 if (obj->pending_gpu_write || write) {
ce453d81 2728 ret = i915_gem_object_wait_rendering(obj);
87ca9c8a
CW		 2729 if (ret)
2730 return ret;
2731 }
2dafb1e0 2732
7213342d 2733 i915_gem_object_flush_cpu_write_domain(obj);
1c5d22f7 2734
05394f39
CW		 2735 old_write_domain = obj->base.write_domain;
2736 old_read_domains = obj->base.read_domains;
1c5d22f7 2737
e47c68e9
EA		 2738 /* It should now be out of any other write domains, and we can update
2739 * the domain values for our changes.
2740 */
05394f39
CW		 2741 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2742 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
e47c68e9 2743 if (write) {
05394f39
CW		 2744 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
2745 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
2746 obj->dirty = 1;
2ef7eeaa
EA		 2747 }
2748
1c5d22f7
CW		 2749 trace_i915_gem_object_change_domain(obj,
2750 old_read_domains,
2751 old_write_domain);
2752
8325a09d
CW		 2753 /* And bump the LRU for this access */
2754 if (i915_gem_object_is_inactive(obj))
2755 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2756
e47c68e9
EA		 2757 return 0;
2758}
2759
e4ffd173
CW		 2760int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
2761 enum i915_cache_level cache_level)
2762{
7bddb01f
DV		 2763 struct drm_device *dev = obj->base.dev;
2764 drm_i915_private_t *dev_priv = dev->dev_private;
e4ffd173
CW		 2765 int ret;
2766
2767 if (obj->cache_level == cache_level)
2768 return 0;
2769
2770 if (obj->pin_count) {
2771 DRM_DEBUG("can not change the cache level of pinned objects\n");
2772 return -EBUSY;
2773 }
2774
2775 if (obj->gtt_space) {
2776 ret = i915_gem_object_finish_gpu(obj);
2777 if (ret)
2778 return ret;
2779
2780 i915_gem_object_finish_gtt(obj);
2781
2782 /* Before SandyBridge, you could not use tiling or fence
2783 * registers with snooped memory, so relinquish any fences
2784 * currently pointing to our region in the aperture.
2785 */
2786 if (INTEL_INFO(obj->base.dev)->gen < 6) {
2787 ret = i915_gem_object_put_fence(obj);
2788 if (ret)
2789 return ret;
2790 }
2791
74898d7e
DV		 2792 if (obj->has_global_gtt_mapping)
2793 i915_gem_gtt_bind_object(obj, cache_level);
7bddb01f
DV		 2794 if (obj->has_aliasing_ppgtt_mapping)
2795 i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
2796 obj, cache_level);
e4ffd173
CW		 2797 }
2798
2799 if (cache_level == I915_CACHE_NONE) {
2800 u32 old_read_domains, old_write_domain;
2801
2802 /* If we're coming from LLC cached, then we haven't
2803 * actually been tracking whether the data is in the
2804 * CPU cache or not, since we only allow one bit set
2805 * in obj->write_domain and have been skipping the clflushes.
2806 * Just set it to the CPU cache for now.
2807 */
2808 WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
2809 WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
2810
2811 old_read_domains = obj->base.read_domains;
2812 old_write_domain = obj->base.write_domain;
2813
2814 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2815 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2816
2817 trace_i915_gem_object_change_domain(obj,
2818 old_read_domains,
2819 old_write_domain);
2820 }
2821
2822 obj->cache_level = cache_level;
2823 return 0;
2824}
2825
b9241ea3 2826/*
2da3b9b9
CW		 2827 * Prepare buffer for display plane (scanout, cursors, etc).
2828 * Can be called from an uninterruptible phase (modesetting) and allows
2829 * any flushes to be pipelined (for pageflips).
b9241ea3
ZW		 2830 */
2831int
2da3b9b9
CW		 2832i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
2833 u32 alignment,
919926ae 2834 struct intel_ring_buffer *pipelined)
b9241ea3 2835{
2da3b9b9 2836 u32 old_read_domains, old_write_domain;
b9241ea3
ZW		 2837 int ret;
2838
88241785
CW		 2839 ret = i915_gem_object_flush_gpu_write_domain(obj);
2840 if (ret)
2841 return ret;
2842
0be73284 2843 if (pipelined != obj->ring) {
2911a35b
BW		 2844 ret = i915_gem_object_sync(obj, pipelined);
2845 if (ret)
b9241ea3
ZW		 2846 return ret;
2847 }
2848
a7ef0640
EA		 2849 /* The display engine is not coherent with the LLC cache on gen6. As
2850 * a result, we make sure that the pinning that is about to occur is
2851 * done with uncached PTEs. This is lowest common denominator for all
2852 * chipsets.
2853 *
2854 * However for gen6+, we could do better by using the GFDT bit instead
2855 * of uncaching, which would allow us to flush all the LLC-cached data
2856 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
2857 */
2858 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
2859 if (ret)
2860 return ret;
2861
2da3b9b9
CW		 2862 /* As the user may map the buffer once pinned in the display plane
2863 * (e.g. libkms for the bootup splash), we have to ensure that we
2864 * always use map_and_fenceable for all scanout buffers.
2865 */
2866 ret = i915_gem_object_pin(obj, alignment, true);
2867 if (ret)
2868 return ret;
2869
b118c1e3
CW		 2870 i915_gem_object_flush_cpu_write_domain(obj);
2871
2da3b9b9 2872 old_write_domain = obj->base.write_domain;
05394f39 2873 old_read_domains = obj->base.read_domains;
2da3b9b9
CW		 2874
2875 /* It should now be out of any other write domains, and we can update
2876 * the domain values for our changes.
2877 */
2878 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
05394f39 2879 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
b9241ea3
ZW		 2880
2881 trace_i915_gem_object_change_domain(obj,
2882 old_read_domains,
2da3b9b9 2883 old_write_domain);
b9241ea3
ZW		 2884
2885 return 0;
2886}
2887
85345517 2888int
a8198eea 2889i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
85345517 2890{
88241785
CW		 2891 int ret;
2892
a8198eea 2893 if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
85345517
CW		 2894 return 0;
2895
88241785 2896 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
db53a302 2897 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
88241785
CW		 2898 if (ret)
2899 return ret;
2900 }
85345517 2901
c501ae7f
CW		 2902 ret = i915_gem_object_wait_rendering(obj);
2903 if (ret)
2904 return ret;
2905
a8198eea
CW		 2906 /* Ensure that we invalidate the GPU's caches and TLBs. */
2907 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
c501ae7f 2908 return 0;
85345517
CW		 2909}
2910
e47c68e9
EA		 2911/**
2912 * Moves a single object to the CPU read, and possibly write domain.
2913 *
2914 * This function returns when the move is complete, including waiting on
2915 * flushes to occur.
2916 */
dabdfe02 2917int
919926ae 2918i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
e47c68e9 2919{
1c5d22f7 2920 uint32_t old_write_domain, old_read_domains;
e47c68e9
EA		 2921 int ret;
2922
8d7e3de1
CW		 2923 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
2924 return 0;
2925
88241785
CW		 2926 ret = i915_gem_object_flush_gpu_write_domain(obj);
2927 if (ret)
2928 return ret;
2929
f8413190
CW		 2930 if (write || obj->pending_gpu_write) {
2931 ret = i915_gem_object_wait_rendering(obj);
2932 if (ret)
2933 return ret;
2934 }
2ef7eeaa 2935
e47c68e9 2936 i915_gem_object_flush_gtt_write_domain(obj);
2ef7eeaa 2937
05394f39
CW		 2938 old_write_domain = obj->base.write_domain;
2939 old_read_domains = obj->base.read_domains;
1c5d22f7 2940
e47c68e9 2941 /* Flush the CPU cache if it's still invalid. */
05394f39 2942 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2ef7eeaa 2943 i915_gem_clflush_object(obj);
2ef7eeaa 2944
05394f39 2945 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
2ef7eeaa
EA		 2946 }
2947
2948 /* It should now be out of any other write domains, and we can update
2949 * the domain values for our changes.
2950 */
05394f39 2951 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
e47c68e9
EA		 2952
2953 /* If we're writing through the CPU, then the GPU read domains will
2954 * need to be invalidated at next use.
2955 */
2956 if (write) {
05394f39
CW		 2957 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2958 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
e47c68e9 2959 }
2ef7eeaa 2960
1c5d22f7
CW		 2961 trace_i915_gem_object_change_domain(obj,
2962 old_read_domains,
2963 old_write_domain);
2964
2ef7eeaa
EA		 2965 return 0;
2966}
2967
673a394b
EA		 2968/* Throttle our rendering by waiting until the ring has completed our requests
2969 * emitted over 20 msec ago.
2970 *
b962442e
EA		 2971 * Note that if we were to use the current jiffies each time around the loop,
2972 * we wouldn't escape the function with any frames outstanding if the time to
2973 * render a frame was over 20ms.
2974 *
673a394b
EA		 2975 * This should get us reasonable parallelism between CPU and GPU but also
2976 * relatively low latency when blocking on a particular request to finish.
2977 */
40a5f0de 2978static int
f787a5f5 2979i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
40a5f0de 2980{
f787a5f5
CW		 2981 struct drm_i915_private *dev_priv = dev->dev_private;
2982 struct drm_i915_file_private *file_priv = file->driver_priv;
b962442e 2983 unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
f787a5f5
CW		 2984 struct drm_i915_gem_request *request;
2985 struct intel_ring_buffer *ring = NULL;
2986 u32 seqno = 0;
2987 int ret;
93533c29 2988
e110e8d6
CW		 2989 if (atomic_read(&dev_priv->mm.wedged))
2990 return -EIO;
2991
1c25595f 2992 spin_lock(&file_priv->mm.lock);
f787a5f5 2993 list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
b962442e
EA		 2994 if (time_after_eq(request->emitted_jiffies, recent_enough))
2995 break;
40a5f0de 2996
f787a5f5
CW		 2997 ring = request->ring;
2998 seqno = request->seqno;
b962442e 2999 }
1c25595f 3000 spin_unlock(&file_priv->mm.lock);
40a5f0de 3001
f787a5f5
CW		 3002 if (seqno == 0)
3003 return 0;
2bc43b5c 3004
3b88cc0d 3005 ret = __wait_seqno(ring, seqno, true);
f787a5f5
CW		 3006 if (ret == 0)
3007 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
40a5f0de
EA		 3008
3009 return ret;
3010}
3011
673a394b 3012int
05394f39
CW		 3013i915_gem_object_pin(struct drm_i915_gem_object *obj,
3014 uint32_t alignment,
75e9e915 3015 bool map_and_fenceable)
673a394b 3016{
673a394b
EA		 3017 int ret;
3018
05394f39 3019 BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
ac0c6b5a 3020
05394f39
CW		 3021 if (obj->gtt_space != NULL) {
3022 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
3023 (map_and_fenceable && !obj->map_and_fenceable)) {
3024 WARN(obj->pin_count,
ae7d49d8 3025 "bo is already pinned with incorrect alignment:"
75e9e915
DV		 3026 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
3027 " obj->map_and_fenceable=%d\n",
05394f39 3028 obj->gtt_offset, alignment,
75e9e915 3029 map_and_fenceable,
05394f39 3030 obj->map_and_fenceable);
ac0c6b5a
CW		 3031 ret = i915_gem_object_unbind(obj);
3032 if (ret)
3033 return ret;
3034 }
3035 }
3036
05394f39 3037 if (obj->gtt_space == NULL) {
a00b10c3 3038 ret = i915_gem_object_bind_to_gtt(obj, alignment,
75e9e915 3039 map_and_fenceable);
9731129c 3040 if (ret)
673a394b 3041 return ret;
22c344e9 3042 }
76446cac 3043
74898d7e
DV		 3044 if (!obj->has_global_gtt_mapping && map_and_fenceable)
3045 i915_gem_gtt_bind_object(obj, obj->cache_level);
3046
1b50247a 3047 obj->pin_count++;
6299f992 3048 obj->pin_mappable |= map_and_fenceable;
673a394b
EA		 3049
3050 return 0;
3051}
3052
3053void
05394f39 3054i915_gem_object_unpin(struct drm_i915_gem_object *obj)
673a394b 3055{
05394f39
CW		 3056 BUG_ON(obj->pin_count == 0);
3057 BUG_ON(obj->gtt_space == NULL);
673a394b 3058
1b50247a 3059 if (--obj->pin_count == 0)
6299f992 3060 obj->pin_mappable = false;
673a394b
EA		 3061}
3062
3063int
3064i915_gem_pin_ioctl(struct drm_device *dev, void *data,
05394f39 3065 struct drm_file *file)
673a394b
EA		 3066{
3067 struct drm_i915_gem_pin *args = data;
05394f39 3068 struct drm_i915_gem_object *obj;
673a394b
EA		 3069 int ret;
3070
1d7cfea1
CW		 3071 ret = i915_mutex_lock_interruptible(dev);
3072 if (ret)
3073 return ret;
673a394b 3074
05394f39 3075 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
c8725226 3076 if (&obj->base == NULL) {
1d7cfea1
CW		 3077 ret = -ENOENT;
3078 goto unlock;
673a394b 3079 }
673a394b 3080
05394f39 3081 if (obj->madv != I915_MADV_WILLNEED) {
bb6baf76 3082 DRM_ERROR("Attempting to pin a purgeable buffer\n");
1d7cfea1
CW		 3083 ret = -EINVAL;
3084 goto out;
3ef94daa
CW		 3085 }
3086
05394f39 3087 if (obj->pin_filp != NULL && obj->pin_filp != file) {
79e53945
JB		 3088 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
3089 args->handle);
1d7cfea1
CW		 3090 ret = -EINVAL;
3091 goto out;
79e53945
JB		 3092 }
3093
05394f39
CW		 3094 obj->user_pin_count++;
3095 obj->pin_filp = file;
3096 if (obj->user_pin_count == 1) {
75e9e915 3097 ret = i915_gem_object_pin(obj, args->alignment, true);
1d7cfea1
CW		 3098 if (ret)
3099 goto out;
673a394b
EA		 3100 }
3101
3102 /* XXX - flush the CPU caches for pinned objects
3103 * as the X server doesn't manage domains yet
3104 */
e47c68e9 3105 i915_gem_object_flush_cpu_write_domain(obj);
05394f39 3106 args->offset = obj->gtt_offset;
1d7cfea1 3107out:
05394f39 3108 drm_gem_object_unreference(&obj->base);
1d7cfea1 3109unlock:
673a394b 3110 mutex_unlock(&dev->struct_mutex);
1d7cfea1 3111 return ret;
673a394b
EA		 3112}
3113
3114int
3115i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
05394f39 3116 struct drm_file *file)
673a394b
EA		 3117{
3118 struct drm_i915_gem_pin *args = data;
05394f39 3119 struct drm_i915_gem_object *obj;
76c1dec1 3120 int ret;
673a394b 3121
1d7cfea1
CW		 3122 ret = i915_mutex_lock_interruptible(dev);
3123 if (ret)
3124 return ret;
673a394b 3125
05394f39 3126 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
c8725226 3127 if (&obj->base == NULL) {
1d7cfea1
CW		 3128 ret = -ENOENT;
3129 goto unlock;
673a394b 3130 }
76c1dec1 3131
05394f39 3132 if (obj->pin_filp != file) {
79e53945
JB		 3133 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
3134 args->handle);
1d7cfea1
CW		 3135 ret = -EINVAL;
3136 goto out;
79e53945 3137 }
05394f39
CW		 3138 obj->user_pin_count--;
3139 if (obj->user_pin_count == 0) {
3140 obj->pin_filp = NULL;
79e53945
JB		 3141 i915_gem_object_unpin(obj);
3142 }
673a394b 3143
1d7cfea1 3144out:
05394f39 3145 drm_gem_object_unreference(&obj->base);
1d7cfea1 3146unlock:
673a394b 3147 mutex_unlock(&dev->struct_mutex);
1d7cfea1 3148 return ret;
673a394b
EA		 3149}
3150
3151int
3152i915_gem_busy_ioctl(struct drm_device *dev, void *data,
05394f39 3153 struct drm_file *file)
673a394b
EA		 3154{
3155 struct drm_i915_gem_busy *args = data;
05394f39 3156 struct drm_i915_gem_object *obj;
30dbf0c0
CW		 3157 int ret;
3158
76c1dec1 3159 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 3160 if (ret)
76c1dec1 3161 return ret;
673a394b 3162
05394f39 3163 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
c8725226 3164 if (&obj->base == NULL) {
1d7cfea1
CW		 3165 ret = -ENOENT;
3166 goto unlock;
673a394b 3167 }
d1b851fc 3168
0be555b6
CW		 3169 /* Count all active objects as busy, even if they are currently not used
3170 * by the gpu. Users of this interface expect objects to eventually
3171 * become non-busy without any further actions, therefore emit any
3172 * necessary flushes here.
c4de0a5d 3173 */
05394f39 3174 args->busy = obj->active;
0be555b6
CW		 3175 if (args->busy) {
3176 /* Unconditionally flush objects, even when the gpu still uses this
3177 * object. Userspace calling this function indicates that it wants to
3178 * use this buffer rather sooner than later, so issuing the required
3179 * flush earlier is beneficial.
3180 */
1a1c6976 3181 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
db53a302 3182 ret = i915_gem_flush_ring(obj->ring,
88241785 3183 0, obj->base.write_domain);
b4aca010
BW		 3184 } else {
3185 ret = i915_gem_check_olr(obj->ring,
3186 obj->last_rendering_seqno);
7a194876 3187 }
0be555b6
CW		 3188
3189 /* Update the active list for the hardware's current position.
3190 * Otherwise this only updates on a delayed timer or when irqs
3191 * are actually unmasked, and our working set ends up being
3192 * larger than required.
3193 */
db53a302 3194 i915_gem_retire_requests_ring(obj->ring);
0be555b6 3195
05394f39 3196 args->busy = obj->active;
0be555b6 3197 }
673a394b 3198
05394f39 3199 drm_gem_object_unreference(&obj->base);
1d7cfea1 3200unlock:
673a394b 3201 mutex_unlock(&dev->struct_mutex);
1d7cfea1 3202 return ret;
673a394b
EA		 3203}
3204
3205int
3206i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
3207 struct drm_file *file_priv)
3208{
0206e353 3209 return i915_gem_ring_throttle(dev, file_priv);
673a394b
EA		 3210}
3211
3ef94daa
CW		 3212int
3213i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
3214 struct drm_file *file_priv)
3215{
3216 struct drm_i915_gem_madvise *args = data;
05394f39 3217 struct drm_i915_gem_object *obj;
76c1dec1 3218 int ret;
3ef94daa
CW		 3219
3220 switch (args->madv) {
3221 case I915_MADV_DONTNEED:
3222 case I915_MADV_WILLNEED:
3223 break;
3224 default:
3225 return -EINVAL;
3226 }
3227
1d7cfea1
CW		 3228 ret = i915_mutex_lock_interruptible(dev);
3229 if (ret)
3230 return ret;
3231
05394f39 3232 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
c8725226 3233 if (&obj->base == NULL) {
1d7cfea1
CW		 3234 ret = -ENOENT;
3235 goto unlock;
3ef94daa 3236 }
3ef94daa 3237
05394f39 3238 if (obj->pin_count) {
1d7cfea1
CW		 3239 ret = -EINVAL;
3240 goto out;
3ef94daa
CW		 3241 }
3242
05394f39
CW		 3243 if (obj->madv != __I915_MADV_PURGED)
3244 obj->madv = args->madv;
3ef94daa 3245
2d7ef395 3246 /* if the object is no longer bound, discard its backing storage */
05394f39
CW		 3247 if (i915_gem_object_is_purgeable(obj) &&
3248 obj->gtt_space == NULL)
2d7ef395
CW		 3249 i915_gem_object_truncate(obj);
3250
05394f39 3251 args->retained = obj->madv != __I915_MADV_PURGED;
bb6baf76 3252
1d7cfea1 3253out:
05394f39 3254 drm_gem_object_unreference(&obj->base);
1d7cfea1 3255unlock:
3ef94daa 3256 mutex_unlock(&dev->struct_mutex);
1d7cfea1 3257 return ret;
3ef94daa
CW		 3258}
3259
05394f39
CW		 3260struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
3261 size_t size)
ac52bc56 3262{
73aa808f 3263 struct drm_i915_private *dev_priv = dev->dev_private;
c397b908 3264 struct drm_i915_gem_object *obj;
5949eac4 3265 struct address_space *mapping;
ac52bc56 3266
c397b908
DV		 3267 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
3268 if (obj == NULL)
3269 return NULL;
673a394b 3270
c397b908
DV		 3271 if (drm_gem_object_init(dev, &obj->base, size) != 0) {
3272 kfree(obj);
3273 return NULL;
3274 }
673a394b 3275
5949eac4
HD		 3276 mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
3277 mapping_set_gfp_mask(mapping, GFP_HIGHUSER | __GFP_RECLAIMABLE);
3278
73aa808f
CW		 3279 i915_gem_info_add_obj(dev_priv, size);
3280
c397b908
DV		 3281 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3282 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
673a394b 3283
3d29b842
ED		 3284 if (HAS_LLC(dev)) {
3285 /* On some devices, we can have the GPU use the LLC (the CPU
a1871112
EA		 3286 * cache) for about a 10% performance improvement
3287 * compared to uncached. Graphics requests other than
3288 * display scanout are coherent with the CPU in
3289 * accessing this cache. This means in this mode we
3290 * don't need to clflush on the CPU side, and on the
3291 * GPU side we only need to flush internal caches to
3292 * get data visible to the CPU.
3293 *
3294 * However, we maintain the display planes as UC, and so
3295 * need to rebind when first used as such.
3296 */
3297 obj->cache_level = I915_CACHE_LLC;
3298 } else
3299 obj->cache_level = I915_CACHE_NONE;
3300
62b8b215 3301 obj->base.driver_private = NULL;
c397b908 3302 obj->fence_reg = I915_FENCE_REG_NONE;
69dc4987 3303 INIT_LIST_HEAD(&obj->mm_list);
93a37f20 3304 INIT_LIST_HEAD(&obj->gtt_list);
69dc4987 3305 INIT_LIST_HEAD(&obj->ring_list);
432e58ed 3306 INIT_LIST_HEAD(&obj->exec_list);
c397b908 3307 INIT_LIST_HEAD(&obj->gpu_write_list);
c397b908 3308 obj->madv = I915_MADV_WILLNEED;
75e9e915
DV		 3309 /* Avoid an unnecessary call to unbind on the first bind. */
3310 obj->map_and_fenceable = true;
de151cf6 3311
05394f39 3312 return obj;
c397b908
DV		 3313}
3314
3315int i915_gem_init_object(struct drm_gem_object *obj)
3316{
3317 BUG();
de151cf6 3318
673a394b
EA		 3319 return 0;
3320}
3321
1488fc08 3322void i915_gem_free_object(struct drm_gem_object *gem_obj)
673a394b 3323{
1488fc08 3324 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
05394f39 3325 struct drm_device *dev = obj->base.dev;
be72615b 3326 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 3327
26e12f89
CW		 3328 trace_i915_gem_object_destroy(obj);
3329
1488fc08
CW		 3330 if (obj->phys_obj)
3331 i915_gem_detach_phys_object(dev, obj);
3332
3333 obj->pin_count = 0;
3334 if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
3335 bool was_interruptible;
3336
3337 was_interruptible = dev_priv->mm.interruptible;
3338 dev_priv->mm.interruptible = false;
3339
3340 WARN_ON(i915_gem_object_unbind(obj));
3341
3342 dev_priv->mm.interruptible = was_interruptible;
3343 }
3344
05394f39 3345 if (obj->base.map_list.map)
b464e9a2 3346 drm_gem_free_mmap_offset(&obj->base);
de151cf6 3347
05394f39
CW		 3348 drm_gem_object_release(&obj->base);
3349 i915_gem_info_remove_obj(dev_priv, obj->base.size);
c397b908 3350
05394f39
CW		 3351 kfree(obj->bit_17);
3352 kfree(obj);
673a394b
EA		 3353}
3354
29105ccc
CW		 3355int
3356i915_gem_idle(struct drm_device *dev)
3357{
3358 drm_i915_private_t *dev_priv = dev->dev_private;
3359 int ret;
28dfe52a 3360
29105ccc 3361 mutex_lock(&dev->struct_mutex);
1c5d22f7 3362
87acb0a5 3363 if (dev_priv->mm.suspended) {
29105ccc
CW		 3364 mutex_unlock(&dev->struct_mutex);
3365 return 0;
28dfe52a
EA		 3366 }
3367
b2da9fe5 3368 ret = i915_gpu_idle(dev);
6dbe2772
KP		 3369 if (ret) {
3370 mutex_unlock(&dev->struct_mutex);
673a394b 3371 return ret;
6dbe2772 3372 }
b2da9fe5 3373 i915_gem_retire_requests(dev);
673a394b 3374
29105ccc 3375 /* Under UMS, be paranoid and evict. */
a39d7efc
CW		 3376 if (!drm_core_check_feature(dev, DRIVER_MODESET))
3377 i915_gem_evict_everything(dev, false);
29105ccc 3378
312817a3
CW		 3379 i915_gem_reset_fences(dev);
3380
29105ccc
CW		 3381 /* Hack! Don't let anybody do execbuf while we don't control the chip.
3382 * We need to replace this with a semaphore, or something.
3383 * And not confound mm.suspended!
3384 */
3385 dev_priv->mm.suspended = 1;
bc0c7f14 3386 del_timer_sync(&dev_priv->hangcheck_timer);
29105ccc
CW		 3387
3388 i915_kernel_lost_context(dev);
6dbe2772 3389 i915_gem_cleanup_ringbuffer(dev);
29105ccc 3390
6dbe2772
KP		 3391 mutex_unlock(&dev->struct_mutex);
3392
29105ccc
CW		 3393 /* Cancel the retire work handler, which should be idle now. */
3394 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3395
673a394b
EA		 3396 return 0;
3397}
3398
f691e2f4
DV		 3399void i915_gem_init_swizzling(struct drm_device *dev)
3400{
3401 drm_i915_private_t *dev_priv = dev->dev_private;
3402
11782b02 3403 if (INTEL_INFO(dev)->gen < 5 ||
f691e2f4
DV		 3404 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
3405 return;
3406
3407 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
3408 DISP_TILE_SURFACE_SWIZZLING);
3409
11782b02
DV		 3410 if (IS_GEN5(dev))
3411 return;
3412
f691e2f4
DV		 3413 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
3414 if (IS_GEN6(dev))
6b26c86d 3415 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
f691e2f4 3416 else
6b26c86d 3417 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
f691e2f4 3418}
e21af88d
DV		 3419
3420void i915_gem_init_ppgtt(struct drm_device *dev)
3421{
3422 drm_i915_private_t *dev_priv = dev->dev_private;
3423 uint32_t pd_offset;
3424 struct intel_ring_buffer *ring;
55a254ac
DV		 3425 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
3426 uint32_t __iomem *pd_addr;
3427 uint32_t pd_entry;
e21af88d
DV		 3428 int i;
3429
3430 if (!dev_priv->mm.aliasing_ppgtt)
3431 return;
3432
55a254ac
DV		 3433
3434 pd_addr = dev_priv->mm.gtt->gtt + ppgtt->pd_offset/sizeof(uint32_t);
3435 for (i = 0; i < ppgtt->num_pd_entries; i++) {
3436 dma_addr_t pt_addr;
3437
3438 if (dev_priv->mm.gtt->needs_dmar)
3439 pt_addr = ppgtt->pt_dma_addr[i];
3440 else
3441 pt_addr = page_to_phys(ppgtt->pt_pages[i]);
3442
3443 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
3444 pd_entry |= GEN6_PDE_VALID;
3445
3446 writel(pd_entry, pd_addr + i);
3447 }
3448 readl(pd_addr);
3449
3450 pd_offset = ppgtt->pd_offset;
e21af88d
DV		 3451 pd_offset /= 64; /* in cachelines, */
3452 pd_offset <<= 16;
3453
3454 if (INTEL_INFO(dev)->gen == 6) {
48ecfa10
DV		 3455 uint32_t ecochk, gab_ctl, ecobits;
3456
3457 ecobits = I915_READ(GAC_ECO_BITS);
3458 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
be901a5a
DV		 3459
3460 gab_ctl = I915_READ(GAB_CTL);
3461 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
3462
3463 ecochk = I915_READ(GAM_ECOCHK);
e21af88d
DV		 3464 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
3465 ECOCHK_PPGTT_CACHE64B);
6b26c86d 3466 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
e21af88d
DV		 3467 } else if (INTEL_INFO(dev)->gen >= 7) {
3468 I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B);
3469 /* GFX_MODE is per-ring on gen7+ */
3470 }
3471
b4519513 3472 for_each_ring(ring, dev_priv, i) {
e21af88d
DV		 3473 if (INTEL_INFO(dev)->gen >= 7)
3474 I915_WRITE(RING_MODE_GEN7(ring),
6b26c86d 3475 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
e21af88d
DV		 3476
3477 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
3478 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
3479 }
3480}
3481
8187a2b7 3482int
f691e2f4 3483i915_gem_init_hw(struct drm_device *dev)
8187a2b7
ZN		 3484{
3485 drm_i915_private_t *dev_priv = dev->dev_private;
3486 int ret;
68f95ba9 3487
f691e2f4
DV		 3488 i915_gem_init_swizzling(dev);
3489
5c1143bb 3490 ret = intel_init_render_ring_buffer(dev);
68f95ba9 3491 if (ret)
b6913e4b 3492 return ret;
68f95ba9
CW		 3493
3494 if (HAS_BSD(dev)) {
5c1143bb 3495 ret = intel_init_bsd_ring_buffer(dev);
68f95ba9
CW		 3496 if (ret)
3497 goto cleanup_render_ring;
d1b851fc 3498 }
68f95ba9 3499
549f7365
CW		 3500 if (HAS_BLT(dev)) {
3501 ret = intel_init_blt_ring_buffer(dev);
3502 if (ret)
3503 goto cleanup_bsd_ring;
3504 }
3505
6f392d54
CW		 3506 dev_priv->next_seqno = 1;
3507
e21af88d
DV		 3508 i915_gem_init_ppgtt(dev);
3509
68f95ba9
CW		 3510 return 0;
3511
549f7365 3512cleanup_bsd_ring:
1ec14ad3 3513 intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
68f95ba9 3514cleanup_render_ring:
1ec14ad3 3515 intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
8187a2b7
ZN		 3516 return ret;
3517}
3518
1070a42b
CW		 3519static bool
3520intel_enable_ppgtt(struct drm_device *dev)
3521{
3522 if (i915_enable_ppgtt >= 0)
3523 return i915_enable_ppgtt;
3524
3525#ifdef CONFIG_INTEL_IOMMU
3526 /* Disable ppgtt on SNB if VT-d is on. */
3527 if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
3528 return false;
3529#endif
3530
3531 return true;
3532}
3533
3534int i915_gem_init(struct drm_device *dev)
3535{
3536 struct drm_i915_private *dev_priv = dev->dev_private;
3537 unsigned long gtt_size, mappable_size;
3538 int ret;
3539
3540 gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
3541 mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
3542
3543 mutex_lock(&dev->struct_mutex);
3544 if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
3545 /* PPGTT pdes are stolen from global gtt ptes, so shrink the
3546 * aperture accordingly when using aliasing ppgtt. */
3547 gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
3548
3549 i915_gem_init_global_gtt(dev, 0, mappable_size, gtt_size);
3550
3551 ret = i915_gem_init_aliasing_ppgtt(dev);
3552 if (ret) {
3553 mutex_unlock(&dev->struct_mutex);
3554 return ret;
3555 }
3556 } else {
3557 /* Let GEM Manage all of the aperture.
3558 *
3559 * However, leave one page at the end still bound to the scratch
3560 * page. There are a number of places where the hardware
3561 * apparently prefetches past the end of the object, and we've
3562 * seen multiple hangs with the GPU head pointer stuck in a
3563 * batchbuffer bound at the last page of the aperture. One page
3564 * should be enough to keep any prefetching inside of the
3565 * aperture.
3566 */
3567 i915_gem_init_global_gtt(dev, 0, mappable_size,
3568 gtt_size);
3569 }
3570
3571 ret = i915_gem_init_hw(dev);
3572 mutex_unlock(&dev->struct_mutex);
3573 if (ret) {
3574 i915_gem_cleanup_aliasing_ppgtt(dev);
3575 return ret;
3576 }
3577
53ca26ca
DV		 3578 /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */
3579 if (!drm_core_check_feature(dev, DRIVER_MODESET))
3580 dev_priv->dri1.allow_batchbuffer = 1;
1070a42b
CW		 3581 return 0;
3582}
3583
8187a2b7
ZN		 3584void
3585i915_gem_cleanup_ringbuffer(struct drm_device *dev)
3586{
3587 drm_i915_private_t *dev_priv = dev->dev_private;
b4519513 3588 struct intel_ring_buffer *ring;
1ec14ad3 3589 int i;
8187a2b7 3590
b4519513
CW		 3591 for_each_ring(ring, dev_priv, i)
3592 intel_cleanup_ring_buffer(ring);
8187a2b7
ZN		 3593}
3594
673a394b
EA		 3595int
3596i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
3597 struct drm_file *file_priv)
3598{
3599 drm_i915_private_t *dev_priv = dev->dev_private;
b4519513 3600 int ret;
673a394b 3601
79e53945
JB		 3602 if (drm_core_check_feature(dev, DRIVER_MODESET))
3603 return 0;
3604
ba1234d1 3605 if (atomic_read(&dev_priv->mm.wedged)) {
673a394b 3606 DRM_ERROR("Reenabling wedged hardware, good luck\n");
ba1234d1 3607 atomic_set(&dev_priv->mm.wedged, 0);
673a394b
EA		 3608 }
3609
673a394b 3610 mutex_lock(&dev->struct_mutex);
9bb2d6f9
EA		 3611 dev_priv->mm.suspended = 0;
3612
f691e2f4 3613 ret = i915_gem_init_hw(dev);
d816f6ac
WF		 3614 if (ret != 0) {
3615 mutex_unlock(&dev->struct_mutex);
9bb2d6f9 3616 return ret;
d816f6ac 3617 }
9bb2d6f9 3618
69dc4987 3619 BUG_ON(!list_empty(&dev_priv->mm.active_list));
673a394b
EA		 3620 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
3621 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
673a394b 3622 mutex_unlock(&dev->struct_mutex);
dbb19d30 3623
5f35308b
CW		 3624 ret = drm_irq_install(dev);
3625 if (ret)
3626 goto cleanup_ringbuffer;
dbb19d30 3627
673a394b 3628 return 0;
5f35308b
CW		 3629
3630cleanup_ringbuffer:
3631 mutex_lock(&dev->struct_mutex);
3632 i915_gem_cleanup_ringbuffer(dev);
3633 dev_priv->mm.suspended = 1;
3634 mutex_unlock(&dev->struct_mutex);
3635
3636 return ret;
673a394b
EA		 3637}
3638
3639int
3640i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
3641 struct drm_file *file_priv)
3642{
79e53945
JB		 3643 if (drm_core_check_feature(dev, DRIVER_MODESET))
3644 return 0;
3645
dbb19d30 3646 drm_irq_uninstall(dev);
e6890f6f 3647 return i915_gem_idle(dev);
673a394b
EA		 3648}
3649
3650void
3651i915_gem_lastclose(struct drm_device *dev)
3652{
3653 int ret;
673a394b 3654
e806b495
EA		 3655 if (drm_core_check_feature(dev, DRIVER_MODESET))
3656 return;
3657
6dbe2772
KP		 3658 ret = i915_gem_idle(dev);
3659 if (ret)
3660 DRM_ERROR("failed to idle hardware: %d\n", ret);
673a394b
EA		 3661}
3662
64193406
CW		 3663static void
3664init_ring_lists(struct intel_ring_buffer *ring)
3665{
3666 INIT_LIST_HEAD(&ring->active_list);
3667 INIT_LIST_HEAD(&ring->request_list);
3668 INIT_LIST_HEAD(&ring->gpu_write_list);
3669}
3670
673a394b
EA		 3671void
3672i915_gem_load(struct drm_device *dev)
3673{
b5aa8a0f 3674 int i;
673a394b
EA		 3675 drm_i915_private_t *dev_priv = dev->dev_private;
3676
69dc4987 3677 INIT_LIST_HEAD(&dev_priv->mm.active_list);
673a394b
EA		 3678 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
3679 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
a09ba7fa 3680 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
93a37f20 3681 INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
1ec14ad3
CW		 3682 for (i = 0; i < I915_NUM_RINGS; i++)
3683 init_ring_lists(&dev_priv->ring[i]);
4b9de737 3684 for (i = 0; i < I915_MAX_NUM_FENCES; i++)
007cc8ac 3685 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
673a394b
EA		 3686 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
3687 i915_gem_retire_work_handler);
30dbf0c0 3688 init_completion(&dev_priv->error_completion);
31169714 3689
94400120
DA		 3690 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
3691 if (IS_GEN3(dev)) {
50743298
DV		 3692 I915_WRITE(MI_ARB_STATE,
3693 _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
94400120
DA		 3694 }
3695
72bfa19c
CW		 3696 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
3697
de151cf6 3698 /* Old X drivers will take 0-2 for front, back, depth buffers */
b397c836
EA		 3699 if (!drm_core_check_feature(dev, DRIVER_MODESET))
3700 dev_priv->fence_reg_start = 3;
de151cf6 3701
a6c45cf0 3702 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
de151cf6
JB		 3703 dev_priv->num_fence_regs = 16;
3704 else
3705 dev_priv->num_fence_regs = 8;
3706
b5aa8a0f 3707 /* Initialize fence registers to zero */
ada726c7 3708 i915_gem_reset_fences(dev);
10ed13e4 3709
673a394b 3710 i915_gem_detect_bit_6_swizzle(dev);
6b95a207 3711 init_waitqueue_head(&dev_priv->pending_flip_queue);
17250b71 3712
ce453d81
CW		 3713 dev_priv->mm.interruptible = true;
3714
17250b71
CW		 3715 dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
3716 dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
3717 register_shrinker(&dev_priv->mm.inactive_shrinker);
673a394b 3718}
71acb5eb
DA		 3719
3720/*
3721 * Create a physically contiguous memory object for this object
3722 * e.g. for cursor + overlay regs
3723 */
995b6762
CW		 3724static int i915_gem_init_phys_object(struct drm_device *dev,
3725 int id, int size, int align)
71acb5eb
DA		 3726{
3727 drm_i915_private_t *dev_priv = dev->dev_private;
3728 struct drm_i915_gem_phys_object *phys_obj;
3729 int ret;
3730
3731 if (dev_priv->mm.phys_objs[id - 1] || !size)
3732 return 0;
3733
9a298b2a 3734 phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
71acb5eb
DA		 3735 if (!phys_obj)
3736 return -ENOMEM;
3737
3738 phys_obj->id = id;
3739
6eeefaf3 3740 phys_obj->handle = drm_pci_alloc(dev, size, align);
71acb5eb
DA		 3741 if (!phys_obj->handle) {
3742 ret = -ENOMEM;
3743 goto kfree_obj;
3744 }
3745#ifdef CONFIG_X86
3746 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
3747#endif
3748
3749 dev_priv->mm.phys_objs[id - 1] = phys_obj;
3750
3751 return 0;
3752kfree_obj:
9a298b2a 3753 kfree(phys_obj);
71acb5eb
DA		 3754 return ret;
3755}
3756
995b6762 3757static void i915_gem_free_phys_object(struct drm_device *dev, int id)
71acb5eb
DA		 3758{
3759 drm_i915_private_t *dev_priv = dev->dev_private;
3760 struct drm_i915_gem_phys_object *phys_obj;
3761
3762 if (!dev_priv->mm.phys_objs[id - 1])
3763 return;
3764
3765 phys_obj = dev_priv->mm.phys_objs[id - 1];
3766 if (phys_obj->cur_obj) {
3767 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
3768 }
3769
3770#ifdef CONFIG_X86
3771 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
3772#endif
3773 drm_pci_free(dev, phys_obj->handle);
3774 kfree(phys_obj);
3775 dev_priv->mm.phys_objs[id - 1] = NULL;
3776}
3777
3778void i915_gem_free_all_phys_object(struct drm_device *dev)
3779{
3780 int i;
3781
260883c8 3782 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
71acb5eb
DA		 3783 i915_gem_free_phys_object(dev, i);
3784}
3785
3786void i915_gem_detach_phys_object(struct drm_device *dev,
05394f39 3787 struct drm_i915_gem_object *obj)
71acb5eb 3788{
05394f39 3789 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
e5281ccd 3790 char *vaddr;
71acb5eb 3791 int i;
71acb5eb
DA		 3792 int page_count;
3793
05394f39 3794 if (!obj->phys_obj)
71acb5eb 3795 return;
05394f39 3796 vaddr = obj->phys_obj->handle->vaddr;
71acb5eb 3797
05394f39 3798 page_count = obj->base.size / PAGE_SIZE;
71acb5eb 3799 for (i = 0; i < page_count; i++) {
5949eac4 3800 struct page *page = shmem_read_mapping_page(mapping, i);
e5281ccd
CW		 3801 if (!IS_ERR(page)) {
3802 char *dst = kmap_atomic(page);
3803 memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
3804 kunmap_atomic(dst);
3805
3806 drm_clflush_pages(&page, 1);
3807
3808 set_page_dirty(page);
3809 mark_page_accessed(page);
3810 page_cache_release(page);
3811 }
71acb5eb 3812 }
40ce6575 3813 intel_gtt_chipset_flush();
d78b47b9 3814
05394f39
CW		 3815 obj->phys_obj->cur_obj = NULL;
3816 obj->phys_obj = NULL;
71acb5eb
DA		 3817}
3818
3819int
3820i915_gem_attach_phys_object(struct drm_device *dev,
05394f39 3821 struct drm_i915_gem_object *obj,
6eeefaf3
CW		 3822 int id,
3823 int align)
71acb5eb 3824{
05394f39 3825 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
71acb5eb 3826 drm_i915_private_t *dev_priv = dev->dev_private;
71acb5eb
DA		 3827 int ret = 0;
3828 int page_count;
3829 int i;
3830
3831 if (id > I915_MAX_PHYS_OBJECT)
3832 return -EINVAL;
3833
05394f39
CW		 3834 if (obj->phys_obj) {
3835 if (obj->phys_obj->id == id)
71acb5eb
DA		 3836 return 0;
3837 i915_gem_detach_phys_object(dev, obj);
3838 }
3839
71acb5eb
DA		 3840 /* create a new object */
3841 if (!dev_priv->mm.phys_objs[id - 1]) {
3842 ret = i915_gem_init_phys_object(dev, id,
05394f39 3843 obj->base.size, align);
71acb5eb 3844 if (ret) {
05394f39
CW		 3845 DRM_ERROR("failed to init phys object %d size: %zu\n",
3846 id, obj->base.size);
e5281ccd 3847 return ret;
71acb5eb
DA		 3848 }
3849 }
3850
3851 /* bind to the object */
05394f39
CW		 3852 obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
3853 obj->phys_obj->cur_obj = obj;
71acb5eb 3854
05394f39 3855 page_count = obj->base.size / PAGE_SIZE;
71acb5eb
DA		 3856
3857 for (i = 0; i < page_count; i++) {
e5281ccd
CW		 3858 struct page *page;
3859 char *dst, *src;
3860
5949eac4 3861 page = shmem_read_mapping_page(mapping, i);
e5281ccd
CW		 3862 if (IS_ERR(page))
3863 return PTR_ERR(page);
71acb5eb 3864
ff75b9bc 3865 src = kmap_atomic(page);
05394f39 3866 dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
71acb5eb 3867 memcpy(dst, src, PAGE_SIZE);
3e4d3af5 3868 kunmap_atomic(src);
71acb5eb 3869
e5281ccd
CW		 3870 mark_page_accessed(page);
3871 page_cache_release(page);
3872 }
d78b47b9 3873
71acb5eb 3874 return 0;
71acb5eb
DA		 3875}
3876
3877static int
05394f39
CW		 3878i915_gem_phys_pwrite(struct drm_device *dev,
3879 struct drm_i915_gem_object *obj,
71acb5eb
DA		 3880 struct drm_i915_gem_pwrite *args,
3881 struct drm_file *file_priv)
3882{
05394f39 3883 void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
b47b30cc 3884 char __user *user_data = (char __user *) (uintptr_t) args->data_ptr;
71acb5eb 3885
b47b30cc
CW		 3886 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
3887 unsigned long unwritten;
3888
3889 /* The physical object once assigned is fixed for the lifetime
3890 * of the obj, so we can safely drop the lock and continue
3891 * to access vaddr.
3892 */
3893 mutex_unlock(&dev->struct_mutex);
3894 unwritten = copy_from_user(vaddr, user_data, args->size);
3895 mutex_lock(&dev->struct_mutex);
3896 if (unwritten)
3897 return -EFAULT;
3898 }
71acb5eb 3899
40ce6575 3900 intel_gtt_chipset_flush();
71acb5eb
DA		 3901 return 0;
3902}
b962442e 3903
f787a5f5 3904void i915_gem_release(struct drm_device *dev, struct drm_file *file)
b962442e 3905{
f787a5f5 3906 struct drm_i915_file_private *file_priv = file->driver_priv;
b962442e
EA		 3907
3908 /* Clean up our request list when the client is going away, so that
3909 * later retire_requests won't dereference our soon-to-be-gone
3910 * file_priv.
3911 */
1c25595f 3912 spin_lock(&file_priv->mm.lock);
f787a5f5
CW		 3913 while (!list_empty(&file_priv->mm.request_list)) {
3914 struct drm_i915_gem_request *request;
3915
3916 request = list_first_entry(&file_priv->mm.request_list,
3917 struct drm_i915_gem_request,
3918 client_list);
3919 list_del(&request->client_list);
3920 request->file_priv = NULL;
3921 }
1c25595f 3922 spin_unlock(&file_priv->mm.lock);
b962442e 3923}
31169714 3924
1637ef41
CW		 3925static int
3926i915_gpu_is_active(struct drm_device *dev)
3927{
3928 drm_i915_private_t *dev_priv = dev->dev_private;
3929 int lists_empty;
3930
1637ef41 3931 lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
17250b71 3932 list_empty(&dev_priv->mm.active_list);
1637ef41
CW		 3933
3934 return !lists_empty;
3935}
3936
31169714 3937static int
1495f230 3938i915_gem_inactive_shrink(struct shrinker *shrinker, struct shrink_control *sc)
31169714 3939{
17250b71
CW		 3940 struct drm_i915_private *dev_priv =
3941 container_of(shrinker,
3942 struct drm_i915_private,
3943 mm.inactive_shrinker);
3944 struct drm_device *dev = dev_priv->dev;
3945 struct drm_i915_gem_object *obj, *next;
1495f230 3946 int nr_to_scan = sc->nr_to_scan;
17250b71
CW		 3947 int cnt;
3948
3949 if (!mutex_trylock(&dev->struct_mutex))
bbe2e11a 3950 return 0;
31169714
CW		 3951
3952 /* "fast-path" to count number of available objects */
3953 if (nr_to_scan == 0) {
17250b71
CW		 3954 cnt = 0;
3955 list_for_each_entry(obj,
3956 &dev_priv->mm.inactive_list,
3957 mm_list)
3958 cnt++;
3959 mutex_unlock(&dev->struct_mutex);
3960 return cnt / 100 * sysctl_vfs_cache_pressure;
31169714
CW		 3961 }
3962
1637ef41 3963rescan:
31169714 3964 /* first scan for clean buffers */
17250b71 3965 i915_gem_retire_requests(dev);
31169714 3966
17250b71
CW		 3967 list_for_each_entry_safe(obj, next,
3968 &dev_priv->mm.inactive_list,
3969 mm_list) {
3970 if (i915_gem_object_is_purgeable(obj)) {
2021746e
CW		 3971 if (i915_gem_object_unbind(obj) == 0 &&
3972 --nr_to_scan == 0)
17250b71 3973 break;
31169714 3974 }
31169714
CW		 3975 }
3976
3977 /* second pass, evict/count anything still on the inactive list */
17250b71
CW		 3978 cnt = 0;
3979 list_for_each_entry_safe(obj, next,
3980 &dev_priv->mm.inactive_list,
3981 mm_list) {
2021746e
CW		 3982 if (nr_to_scan &&
3983 i915_gem_object_unbind(obj) == 0)
17250b71 3984 nr_to_scan--;
2021746e 3985 else
17250b71
CW		 3986 cnt++;
3987 }
3988
3989 if (nr_to_scan && i915_gpu_is_active(dev)) {
1637ef41
CW		 3990 /*
3991 * We are desperate for pages, so as a last resort, wait
3992 * for the GPU to finish and discard whatever we can.
3993 * This has a dramatic impact to reduce the number of
3994 * OOM-killer events whilst running the GPU aggressively.
3995 */
b2da9fe5 3996 if (i915_gpu_idle(dev) == 0)
1637ef41
CW		 3997 goto rescan;
3998 }
17250b71
CW		 3999 mutex_unlock(&dev->struct_mutex);
4000 return cnt / 100 * sysctl_vfs_cache_pressure;
31169714 4001}
ubuntu-bionic-kernel mirror