]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/gpu/drm/i915/i915_gem.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: don't allow tiling changes on pinned buffers v2
[mirror_ubuntu-artful-kernel.git] / drivers / gpu / drm / i915 / i915_gem.c
CommitLineData
673a394b
EA		 1/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
1c5d22f7 32#include "i915_trace.h"
652c393a 33#include "intel_drv.h"
5a0e3ad6 34#include <linux/slab.h>
673a394b 35#include <linux/swap.h>
79e53945 36#include <linux/pci.h>
673a394b 37
28dfe52a
EA		 38#define I915_GEM_GPU_DOMAINS (~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
39
e47c68e9
EA		 40static void i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
41static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
42static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
e47c68e9
EA		 43static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
44 int write);
45static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
46 uint64_t offset,
47 uint64_t size);
48static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
673a394b 49static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
de151cf6
JB		 50static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
51 unsigned alignment);
de151cf6 52static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
07f73f69 53static int i915_gem_evict_something(struct drm_device *dev, int min_size);
ab5ee576 54static int i915_gem_evict_from_inactive_list(struct drm_device *dev);
71acb5eb
DA		 55static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
56 struct drm_i915_gem_pwrite *args,
57 struct drm_file *file_priv);
673a394b 58
31169714
CW		 59static LIST_HEAD(shrink_list);
60static DEFINE_SPINLOCK(shrink_list_lock);
61
79e53945
JB		 62int i915_gem_do_init(struct drm_device *dev, unsigned long start,
63 unsigned long end)
673a394b
EA		 64{
65 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 66
79e53945
JB		 67 if (start >= end ||
68 (start & (PAGE_SIZE - 1)) != 0 ||
69 (end & (PAGE_SIZE - 1)) != 0) {
673a394b
EA		 70 return -EINVAL;
71 }
72
79e53945
JB		 73 drm_mm_init(&dev_priv->mm.gtt_space, start,
74 end - start);
673a394b 75
79e53945
JB		 76 dev->gtt_total = (uint32_t) (end - start);
77
78 return 0;
79}
673a394b 80
79e53945
JB		 81int
82i915_gem_init_ioctl(struct drm_device *dev, void *data,
83 struct drm_file *file_priv)
84{
85 struct drm_i915_gem_init *args = data;
86 int ret;
87
88 mutex_lock(&dev->struct_mutex);
89 ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
673a394b
EA		 90 mutex_unlock(&dev->struct_mutex);
91
79e53945 92 return ret;
673a394b
EA		 93}
94
5a125c3c
EA		 95int
96i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
97 struct drm_file *file_priv)
98{
5a125c3c 99 struct drm_i915_gem_get_aperture *args = data;
5a125c3c
EA		 100
101 if (!(dev->driver->driver_features & DRIVER_GEM))
102 return -ENODEV;
103
104 args->aper_size = dev->gtt_total;
2678d9d6
KP		 105 args->aper_available_size = (args->aper_size -
106 atomic_read(&dev->pin_memory));
5a125c3c
EA		 107
108 return 0;
109}
110
673a394b
EA		 111
112/**
113 * Creates a new mm object and returns a handle to it.
114 */
115int
116i915_gem_create_ioctl(struct drm_device *dev, void *data,
117 struct drm_file *file_priv)
118{
119 struct drm_i915_gem_create *args = data;
120 struct drm_gem_object *obj;
a1a2d1d3
PP		 121 int ret;
122 u32 handle;
673a394b
EA		 123
124 args->size = roundup(args->size, PAGE_SIZE);
125
126 /* Allocate the new object */
ac52bc56 127 obj = i915_gem_alloc_object(dev, args->size);
673a394b
EA		 128 if (obj == NULL)
129 return -ENOMEM;
130
131 ret = drm_gem_handle_create(file_priv, obj, &handle);
bc9025bd 132 drm_gem_object_handle_unreference_unlocked(obj);
673a394b
EA		 133
134 if (ret)
135 return ret;
136
137 args->handle = handle;
138
139 return 0;
140}
141
eb01459f
EA		 142static inline int
143fast_shmem_read(struct page **pages,
144 loff_t page_base, int page_offset,
145 char __user *data,
146 int length)
147{
148 char __iomem *vaddr;
2bc43b5c 149 int unwritten;
eb01459f
EA		 150
151 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
152 if (vaddr == NULL)
153 return -ENOMEM;
2bc43b5c 154 unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
eb01459f
EA		 155 kunmap_atomic(vaddr, KM_USER0);
156
2bc43b5c
FM		 157 if (unwritten)
158 return -EFAULT;
159
160 return 0;
eb01459f
EA		 161}
162
280b713b
EA		 163static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
164{
165 drm_i915_private_t *dev_priv = obj->dev->dev_private;
23010e43 166 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
280b713b
EA		 167
168 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
169 obj_priv->tiling_mode != I915_TILING_NONE;
170}
171
40123c1f
EA		 172static inline int
173slow_shmem_copy(struct page *dst_page,
174 int dst_offset,
175 struct page *src_page,
176 int src_offset,
177 int length)
178{
179 char *dst_vaddr, *src_vaddr;
180
181 dst_vaddr = kmap_atomic(dst_page, KM_USER0);
182 if (dst_vaddr == NULL)
183 return -ENOMEM;
184
185 src_vaddr = kmap_atomic(src_page, KM_USER1);
186 if (src_vaddr == NULL) {
187 kunmap_atomic(dst_vaddr, KM_USER0);
188 return -ENOMEM;
189 }
190
191 memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
192
193 kunmap_atomic(src_vaddr, KM_USER1);
194 kunmap_atomic(dst_vaddr, KM_USER0);
195
196 return 0;
197}
198
280b713b
EA		 199static inline int
200slow_shmem_bit17_copy(struct page *gpu_page,
201 int gpu_offset,
202 struct page *cpu_page,
203 int cpu_offset,
204 int length,
205 int is_read)
206{
207 char *gpu_vaddr, *cpu_vaddr;
208
209 /* Use the unswizzled path if this page isn't affected. */
210 if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
211 if (is_read)
212 return slow_shmem_copy(cpu_page, cpu_offset,
213 gpu_page, gpu_offset, length);
214 else
215 return slow_shmem_copy(gpu_page, gpu_offset,
216 cpu_page, cpu_offset, length);
217 }
218
219 gpu_vaddr = kmap_atomic(gpu_page, KM_USER0);
220 if (gpu_vaddr == NULL)
221 return -ENOMEM;
222
223 cpu_vaddr = kmap_atomic(cpu_page, KM_USER1);
224 if (cpu_vaddr == NULL) {
225 kunmap_atomic(gpu_vaddr, KM_USER0);
226 return -ENOMEM;
227 }
228
229 /* Copy the data, XORing A6 with A17 (1). The user already knows he's
230 * XORing with the other bits (A9 for Y, A9 and A10 for X)
231 */
232 while (length > 0) {
233 int cacheline_end = ALIGN(gpu_offset + 1, 64);
234 int this_length = min(cacheline_end - gpu_offset, length);
235 int swizzled_gpu_offset = gpu_offset ^ 64;
236
237 if (is_read) {
238 memcpy(cpu_vaddr + cpu_offset,
239 gpu_vaddr + swizzled_gpu_offset,
240 this_length);
241 } else {
242 memcpy(gpu_vaddr + swizzled_gpu_offset,
243 cpu_vaddr + cpu_offset,
244 this_length);
245 }
246 cpu_offset += this_length;
247 gpu_offset += this_length;
248 length -= this_length;
249 }
250
251 kunmap_atomic(cpu_vaddr, KM_USER1);
252 kunmap_atomic(gpu_vaddr, KM_USER0);
253
254 return 0;
255}
256
eb01459f
EA		 257/**
258 * This is the fast shmem pread path, which attempts to copy_from_user directly
259 * from the backing pages of the object to the user's address space. On a
260 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
261 */
262static int
263i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
264 struct drm_i915_gem_pread *args,
265 struct drm_file *file_priv)
266{
23010e43 267 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
eb01459f
EA		 268 ssize_t remain;
269 loff_t offset, page_base;
270 char __user *user_data;
271 int page_offset, page_length;
272 int ret;
273
274 user_data = (char __user *) (uintptr_t) args->data_ptr;
275 remain = args->size;
276
277 mutex_lock(&dev->struct_mutex);
278
4bdadb97 279 ret = i915_gem_object_get_pages(obj, 0);
eb01459f
EA		 280 if (ret != 0)
281 goto fail_unlock;
282
283 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
284 args->size);
285 if (ret != 0)
286 goto fail_put_pages;
287
23010e43 288 obj_priv = to_intel_bo(obj);
eb01459f
EA		 289 offset = args->offset;
290
291 while (remain > 0) {
292 /* Operation in this page
293 *
294 * page_base = page offset within aperture
295 * page_offset = offset within page
296 * page_length = bytes to copy for this page
297 */
298 page_base = (offset & ~(PAGE_SIZE-1));
299 page_offset = offset & (PAGE_SIZE-1);
300 page_length = remain;
301 if ((page_offset + remain) > PAGE_SIZE)
302 page_length = PAGE_SIZE - page_offset;
303
304 ret = fast_shmem_read(obj_priv->pages,
305 page_base, page_offset,
306 user_data, page_length);
307 if (ret)
308 goto fail_put_pages;
309
310 remain -= page_length;
311 user_data += page_length;
312 offset += page_length;
313 }
314
315fail_put_pages:
316 i915_gem_object_put_pages(obj);
317fail_unlock:
318 mutex_unlock(&dev->struct_mutex);
319
320 return ret;
321}
322
07f73f69
CW		 323static int
324i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
325{
326 int ret;
327
4bdadb97 328 ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
07f73f69
CW		 329
330 /* If we've insufficient memory to map in the pages, attempt
331 * to make some space by throwing out some old buffers.
332 */
333 if (ret == -ENOMEM) {
334 struct drm_device *dev = obj->dev;
07f73f69
CW		 335
336 ret = i915_gem_evict_something(dev, obj->size);
337 if (ret)
338 return ret;
339
4bdadb97 340 ret = i915_gem_object_get_pages(obj, 0);
07f73f69
CW		 341 }
342
343 return ret;
344}
345
eb01459f
EA		 346/**
347 * This is the fallback shmem pread path, which allocates temporary storage
348 * in kernel space to copy_to_user into outside of the struct_mutex, so we
349 * can copy out of the object's backing pages while holding the struct mutex
350 * and not take page faults.
351 */
352static int
353i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
354 struct drm_i915_gem_pread *args,
355 struct drm_file *file_priv)
356{
23010e43 357 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
eb01459f
EA		 358 struct mm_struct *mm = current->mm;
359 struct page **user_pages;
360 ssize_t remain;
361 loff_t offset, pinned_pages, i;
362 loff_t first_data_page, last_data_page, num_pages;
363 int shmem_page_index, shmem_page_offset;
364 int data_page_index, data_page_offset;
365 int page_length;
366 int ret;
367 uint64_t data_ptr = args->data_ptr;
280b713b 368 int do_bit17_swizzling;
eb01459f
EA		 369
370 remain = args->size;
371
372 /* Pin the user pages containing the data. We can't fault while
373 * holding the struct mutex, yet we want to hold it while
374 * dereferencing the user data.
375 */
376 first_data_page = data_ptr / PAGE_SIZE;
377 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
378 num_pages = last_data_page - first_data_page + 1;
379
8e7d2b2c 380 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
eb01459f
EA		 381 if (user_pages == NULL)
382 return -ENOMEM;
383
384 down_read(&mm->mmap_sem);
385 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
e5e9ecde 386 num_pages, 1, 0, user_pages, NULL);
eb01459f
EA		 387 up_read(&mm->mmap_sem);
388 if (pinned_pages < num_pages) {
389 ret = -EFAULT;
390 goto fail_put_user_pages;
391 }
392
280b713b
EA		 393 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
394
eb01459f
EA		 395 mutex_lock(&dev->struct_mutex);
396
07f73f69
CW		 397 ret = i915_gem_object_get_pages_or_evict(obj);
398 if (ret)
eb01459f
EA		 399 goto fail_unlock;
400
401 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
402 args->size);
403 if (ret != 0)
404 goto fail_put_pages;
405
23010e43 406 obj_priv = to_intel_bo(obj);
eb01459f
EA		 407 offset = args->offset;
408
409 while (remain > 0) {
410 /* Operation in this page
411 *
412 * shmem_page_index = page number within shmem file
413 * shmem_page_offset = offset within page in shmem file
414 * data_page_index = page number in get_user_pages return
415 * data_page_offset = offset with data_page_index page.
416 * page_length = bytes to copy for this page
417 */
418 shmem_page_index = offset / PAGE_SIZE;
419 shmem_page_offset = offset & ~PAGE_MASK;
420 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
421 data_page_offset = data_ptr & ~PAGE_MASK;
422
423 page_length = remain;
424 if ((shmem_page_offset + page_length) > PAGE_SIZE)
425 page_length = PAGE_SIZE - shmem_page_offset;
426 if ((data_page_offset + page_length) > PAGE_SIZE)
427 page_length = PAGE_SIZE - data_page_offset;
428
280b713b
EA		 429 if (do_bit17_swizzling) {
430 ret = slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
431 shmem_page_offset,
432 user_pages[data_page_index],
433 data_page_offset,
434 page_length,
435 1);
436 } else {
437 ret = slow_shmem_copy(user_pages[data_page_index],
438 data_page_offset,
439 obj_priv->pages[shmem_page_index],
440 shmem_page_offset,
441 page_length);
442 }
eb01459f
EA		 443 if (ret)
444 goto fail_put_pages;
445
446 remain -= page_length;
447 data_ptr += page_length;
448 offset += page_length;
449 }
450
451fail_put_pages:
452 i915_gem_object_put_pages(obj);
453fail_unlock:
454 mutex_unlock(&dev->struct_mutex);
455fail_put_user_pages:
456 for (i = 0; i < pinned_pages; i++) {
457 SetPageDirty(user_pages[i]);
458 page_cache_release(user_pages[i]);
459 }
8e7d2b2c 460 drm_free_large(user_pages);
eb01459f
EA		 461
462 return ret;
463}
464
673a394b
EA		 465/**
466 * Reads data from the object referenced by handle.
467 *
468 * On error, the contents of *data are undefined.
469 */
470int
471i915_gem_pread_ioctl(struct drm_device *dev, void *data,
472 struct drm_file *file_priv)
473{
474 struct drm_i915_gem_pread *args = data;
475 struct drm_gem_object *obj;
476 struct drm_i915_gem_object *obj_priv;
673a394b
EA		 477 int ret;
478
479 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
480 if (obj == NULL)
481 return -EBADF;
23010e43 482 obj_priv = to_intel_bo(obj);
673a394b
EA		 483
484 /* Bounds check source.
485 *
486 * XXX: This could use review for overflow issues...
487 */
488 if (args->offset > obj->size || args->size > obj->size ||
489 args->offset + args->size > obj->size) {
bc9025bd 490 drm_gem_object_unreference_unlocked(obj);
673a394b
EA		 491 return -EINVAL;
492 }
493
280b713b 494 if (i915_gem_object_needs_bit17_swizzle(obj)) {
eb01459f 495 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
280b713b
EA		 496 } else {
497 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
498 if (ret != 0)
499 ret = i915_gem_shmem_pread_slow(dev, obj, args,
500 file_priv);
501 }
673a394b 502
bc9025bd 503 drm_gem_object_unreference_unlocked(obj);
673a394b 504
eb01459f 505 return ret;
673a394b
EA		 506}
507
0839ccb8
KP		 508/* This is the fast write path which cannot handle
509 * page faults in the source data
9b7530cc 510 */
0839ccb8
KP		 511
512static inline int
513fast_user_write(struct io_mapping *mapping,
514 loff_t page_base, int page_offset,
515 char __user *user_data,
516 int length)
9b7530cc 517{
9b7530cc 518 char *vaddr_atomic;
0839ccb8 519 unsigned long unwritten;
9b7530cc 520
0839ccb8
KP		 521 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
522 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
523 user_data, length);
524 io_mapping_unmap_atomic(vaddr_atomic);
525 if (unwritten)
526 return -EFAULT;
527 return 0;
528}
529
530/* Here's the write path which can sleep for
531 * page faults
532 */
533
534static inline int
3de09aa3
EA		 535slow_kernel_write(struct io_mapping *mapping,
536 loff_t gtt_base, int gtt_offset,
537 struct page *user_page, int user_offset,
538 int length)
0839ccb8 539{
3de09aa3 540 char *src_vaddr, *dst_vaddr;
0839ccb8
KP		 541 unsigned long unwritten;
542
3de09aa3
EA		 543 dst_vaddr = io_mapping_map_atomic_wc(mapping, gtt_base);
544 src_vaddr = kmap_atomic(user_page, KM_USER1);
545 unwritten = __copy_from_user_inatomic_nocache(dst_vaddr + gtt_offset,
546 src_vaddr + user_offset,
547 length);
548 kunmap_atomic(src_vaddr, KM_USER1);
549 io_mapping_unmap_atomic(dst_vaddr);
0839ccb8
KP		 550 if (unwritten)
551 return -EFAULT;
9b7530cc 552 return 0;
9b7530cc
LT		 553}
554
40123c1f
EA		 555static inline int
556fast_shmem_write(struct page **pages,
557 loff_t page_base, int page_offset,
558 char __user *data,
559 int length)
560{
561 char __iomem *vaddr;
d0088775 562 unsigned long unwritten;
40123c1f
EA		 563
564 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
565 if (vaddr == NULL)
566 return -ENOMEM;
d0088775 567 unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
40123c1f
EA		 568 kunmap_atomic(vaddr, KM_USER0);
569
d0088775
DA		 570 if (unwritten)
571 return -EFAULT;
40123c1f
EA		 572 return 0;
573}
574
3de09aa3
EA		 575/**
576 * This is the fast pwrite path, where we copy the data directly from the
577 * user into the GTT, uncached.
578 */
673a394b 579static int
3de09aa3
EA		 580i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
581 struct drm_i915_gem_pwrite *args,
582 struct drm_file *file_priv)
673a394b 583{
23010e43 584 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
0839ccb8 585 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 586 ssize_t remain;
0839ccb8 587 loff_t offset, page_base;
673a394b 588 char __user *user_data;
0839ccb8
KP		 589 int page_offset, page_length;
590 int ret;
673a394b
EA		 591
592 user_data = (char __user *) (uintptr_t) args->data_ptr;
593 remain = args->size;
594 if (!access_ok(VERIFY_READ, user_data, remain))
595 return -EFAULT;
596
597
598 mutex_lock(&dev->struct_mutex);
599 ret = i915_gem_object_pin(obj, 0);
600 if (ret) {
601 mutex_unlock(&dev->struct_mutex);
602 return ret;
603 }
2ef7eeaa 604 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
673a394b
EA		 605 if (ret)
606 goto fail;
607
23010e43 608 obj_priv = to_intel_bo(obj);
673a394b 609 offset = obj_priv->gtt_offset + args->offset;
673a394b
EA		 610
611 while (remain > 0) {
612 /* Operation in this page
613 *
0839ccb8
KP		 614 * page_base = page offset within aperture
615 * page_offset = offset within page
616 * page_length = bytes to copy for this page
673a394b 617 */
0839ccb8
KP		 618 page_base = (offset & ~(PAGE_SIZE-1));
619 page_offset = offset & (PAGE_SIZE-1);
620 page_length = remain;
621 if ((page_offset + remain) > PAGE_SIZE)
622 page_length = PAGE_SIZE - page_offset;
623
624 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
625 page_offset, user_data, page_length);
626
627 /* If we get a fault while copying data, then (presumably) our
3de09aa3
EA		 628 * source page isn't available. Return the error and we'll
629 * retry in the slow path.
0839ccb8 630 */
3de09aa3
EA		 631 if (ret)
632 goto fail;
673a394b 633
0839ccb8
KP		 634 remain -= page_length;
635 user_data += page_length;
636 offset += page_length;
673a394b 637 }
673a394b
EA		 638
639fail:
640 i915_gem_object_unpin(obj);
641 mutex_unlock(&dev->struct_mutex);
642
643 return ret;
644}
645
3de09aa3
EA		 646/**
647 * This is the fallback GTT pwrite path, which uses get_user_pages to pin
648 * the memory and maps it using kmap_atomic for copying.
649 *
650 * This code resulted in x11perf -rgb10text consuming about 10% more CPU
651 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
652 */
3043c60c 653static int
3de09aa3
EA		 654i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
655 struct drm_i915_gem_pwrite *args,
656 struct drm_file *file_priv)
673a394b 657{
23010e43 658 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3de09aa3
EA		 659 drm_i915_private_t *dev_priv = dev->dev_private;
660 ssize_t remain;
661 loff_t gtt_page_base, offset;
662 loff_t first_data_page, last_data_page, num_pages;
663 loff_t pinned_pages, i;
664 struct page **user_pages;
665 struct mm_struct *mm = current->mm;
666 int gtt_page_offset, data_page_offset, data_page_index, page_length;
673a394b 667 int ret;
3de09aa3
EA		 668 uint64_t data_ptr = args->data_ptr;
669
670 remain = args->size;
671
672 /* Pin the user pages containing the data. We can't fault while
673 * holding the struct mutex, and all of the pwrite implementations
674 * want to hold it while dereferencing the user data.
675 */
676 first_data_page = data_ptr / PAGE_SIZE;
677 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
678 num_pages = last_data_page - first_data_page + 1;
679
8e7d2b2c 680 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
3de09aa3
EA		 681 if (user_pages == NULL)
682 return -ENOMEM;
683
684 down_read(&mm->mmap_sem);
685 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
686 num_pages, 0, 0, user_pages, NULL);
687 up_read(&mm->mmap_sem);
688 if (pinned_pages < num_pages) {
689 ret = -EFAULT;
690 goto out_unpin_pages;
691 }
673a394b
EA		 692
693 mutex_lock(&dev->struct_mutex);
3de09aa3
EA		 694 ret = i915_gem_object_pin(obj, 0);
695 if (ret)
696 goto out_unlock;
697
698 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
699 if (ret)
700 goto out_unpin_object;
701
23010e43 702 obj_priv = to_intel_bo(obj);
3de09aa3
EA		 703 offset = obj_priv->gtt_offset + args->offset;
704
705 while (remain > 0) {
706 /* Operation in this page
707 *
708 * gtt_page_base = page offset within aperture
709 * gtt_page_offset = offset within page in aperture
710 * data_page_index = page number in get_user_pages return
711 * data_page_offset = offset with data_page_index page.
712 * page_length = bytes to copy for this page
713 */
714 gtt_page_base = offset & PAGE_MASK;
715 gtt_page_offset = offset & ~PAGE_MASK;
716 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
717 data_page_offset = data_ptr & ~PAGE_MASK;
718
719 page_length = remain;
720 if ((gtt_page_offset + page_length) > PAGE_SIZE)
721 page_length = PAGE_SIZE - gtt_page_offset;
722 if ((data_page_offset + page_length) > PAGE_SIZE)
723 page_length = PAGE_SIZE - data_page_offset;
724
725 ret = slow_kernel_write(dev_priv->mm.gtt_mapping,
726 gtt_page_base, gtt_page_offset,
727 user_pages[data_page_index],
728 data_page_offset,
729 page_length);
730
731 /* If we get a fault while copying data, then (presumably) our
732 * source page isn't available. Return the error and we'll
733 * retry in the slow path.
734 */
735 if (ret)
736 goto out_unpin_object;
737
738 remain -= page_length;
739 offset += page_length;
740 data_ptr += page_length;
741 }
742
743out_unpin_object:
744 i915_gem_object_unpin(obj);
745out_unlock:
746 mutex_unlock(&dev->struct_mutex);
747out_unpin_pages:
748 for (i = 0; i < pinned_pages; i++)
749 page_cache_release(user_pages[i]);
8e7d2b2c 750 drm_free_large(user_pages);
3de09aa3
EA		 751
752 return ret;
753}
754
40123c1f
EA		 755/**
756 * This is the fast shmem pwrite path, which attempts to directly
757 * copy_from_user into the kmapped pages backing the object.
758 */
3043c60c 759static int
40123c1f
EA		 760i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
761 struct drm_i915_gem_pwrite *args,
762 struct drm_file *file_priv)
673a394b 763{
23010e43 764 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
40123c1f
EA		 765 ssize_t remain;
766 loff_t offset, page_base;
767 char __user *user_data;
768 int page_offset, page_length;
673a394b 769 int ret;
40123c1f
EA		 770
771 user_data = (char __user *) (uintptr_t) args->data_ptr;
772 remain = args->size;
673a394b
EA		 773
774 mutex_lock(&dev->struct_mutex);
775
4bdadb97 776 ret = i915_gem_object_get_pages(obj, 0);
40123c1f
EA		 777 if (ret != 0)
778 goto fail_unlock;
673a394b 779
e47c68e9 780 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
40123c1f
EA		 781 if (ret != 0)
782 goto fail_put_pages;
783
23010e43 784 obj_priv = to_intel_bo(obj);
40123c1f
EA		 785 offset = args->offset;
786 obj_priv->dirty = 1;
787
788 while (remain > 0) {
789 /* Operation in this page
790 *
791 * page_base = page offset within aperture
792 * page_offset = offset within page
793 * page_length = bytes to copy for this page
794 */
795 page_base = (offset & ~(PAGE_SIZE-1));
796 page_offset = offset & (PAGE_SIZE-1);
797 page_length = remain;
798 if ((page_offset + remain) > PAGE_SIZE)
799 page_length = PAGE_SIZE - page_offset;
800
801 ret = fast_shmem_write(obj_priv->pages,
802 page_base, page_offset,
803 user_data, page_length);
804 if (ret)
805 goto fail_put_pages;
806
807 remain -= page_length;
808 user_data += page_length;
809 offset += page_length;
810 }
811
812fail_put_pages:
813 i915_gem_object_put_pages(obj);
814fail_unlock:
815 mutex_unlock(&dev->struct_mutex);
816
817 return ret;
818}
819
820/**
821 * This is the fallback shmem pwrite path, which uses get_user_pages to pin
822 * the memory and maps it using kmap_atomic for copying.
823 *
824 * This avoids taking mmap_sem for faulting on the user's address while the
825 * struct_mutex is held.
826 */
827static int
828i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
829 struct drm_i915_gem_pwrite *args,
830 struct drm_file *file_priv)
831{
23010e43 832 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
40123c1f
EA		 833 struct mm_struct *mm = current->mm;
834 struct page **user_pages;
835 ssize_t remain;
836 loff_t offset, pinned_pages, i;
837 loff_t first_data_page, last_data_page, num_pages;
838 int shmem_page_index, shmem_page_offset;
839 int data_page_index, data_page_offset;
840 int page_length;
841 int ret;
842 uint64_t data_ptr = args->data_ptr;
280b713b 843 int do_bit17_swizzling;
40123c1f
EA		 844
845 remain = args->size;
846
847 /* Pin the user pages containing the data. We can't fault while
848 * holding the struct mutex, and all of the pwrite implementations
849 * want to hold it while dereferencing the user data.
850 */
851 first_data_page = data_ptr / PAGE_SIZE;
852 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
853 num_pages = last_data_page - first_data_page + 1;
854
8e7d2b2c 855 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
40123c1f
EA		 856 if (user_pages == NULL)
857 return -ENOMEM;
858
859 down_read(&mm->mmap_sem);
860 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
861 num_pages, 0, 0, user_pages, NULL);
862 up_read(&mm->mmap_sem);
863 if (pinned_pages < num_pages) {
864 ret = -EFAULT;
865 goto fail_put_user_pages;
673a394b
EA		 866 }
867
280b713b
EA		 868 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
869
40123c1f
EA		 870 mutex_lock(&dev->struct_mutex);
871
07f73f69
CW		 872 ret = i915_gem_object_get_pages_or_evict(obj);
873 if (ret)
40123c1f
EA		 874 goto fail_unlock;
875
876 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
877 if (ret != 0)
878 goto fail_put_pages;
879
23010e43 880 obj_priv = to_intel_bo(obj);
673a394b 881 offset = args->offset;
40123c1f 882 obj_priv->dirty = 1;
673a394b 883
40123c1f
EA		 884 while (remain > 0) {
885 /* Operation in this page
886 *
887 * shmem_page_index = page number within shmem file
888 * shmem_page_offset = offset within page in shmem file
889 * data_page_index = page number in get_user_pages return
890 * data_page_offset = offset with data_page_index page.
891 * page_length = bytes to copy for this page
892 */
893 shmem_page_index = offset / PAGE_SIZE;
894 shmem_page_offset = offset & ~PAGE_MASK;
895 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
896 data_page_offset = data_ptr & ~PAGE_MASK;
897
898 page_length = remain;
899 if ((shmem_page_offset + page_length) > PAGE_SIZE)
900 page_length = PAGE_SIZE - shmem_page_offset;
901 if ((data_page_offset + page_length) > PAGE_SIZE)
902 page_length = PAGE_SIZE - data_page_offset;
903
280b713b
EA		 904 if (do_bit17_swizzling) {
905 ret = slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
906 shmem_page_offset,
907 user_pages[data_page_index],
908 data_page_offset,
909 page_length,
910 0);
911 } else {
912 ret = slow_shmem_copy(obj_priv->pages[shmem_page_index],
913 shmem_page_offset,
914 user_pages[data_page_index],
915 data_page_offset,
916 page_length);
917 }
40123c1f
EA		 918 if (ret)
919 goto fail_put_pages;
920
921 remain -= page_length;
922 data_ptr += page_length;
923 offset += page_length;
673a394b
EA		 924 }
925
40123c1f
EA		 926fail_put_pages:
927 i915_gem_object_put_pages(obj);
928fail_unlock:
673a394b 929 mutex_unlock(&dev->struct_mutex);
40123c1f
EA		 930fail_put_user_pages:
931 for (i = 0; i < pinned_pages; i++)
932 page_cache_release(user_pages[i]);
8e7d2b2c 933 drm_free_large(user_pages);
673a394b 934
40123c1f 935 return ret;
673a394b
EA		 936}
937
938/**
939 * Writes data to the object referenced by handle.
940 *
941 * On error, the contents of the buffer that were to be modified are undefined.
942 */
943int
944i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
945 struct drm_file *file_priv)
946{
947 struct drm_i915_gem_pwrite *args = data;
948 struct drm_gem_object *obj;
949 struct drm_i915_gem_object *obj_priv;
950 int ret = 0;
951
952 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
953 if (obj == NULL)
954 return -EBADF;
23010e43 955 obj_priv = to_intel_bo(obj);
673a394b
EA		 956
957 /* Bounds check destination.
958 *
959 * XXX: This could use review for overflow issues...
960 */
961 if (args->offset > obj->size || args->size > obj->size ||
962 args->offset + args->size > obj->size) {
bc9025bd 963 drm_gem_object_unreference_unlocked(obj);
673a394b
EA		 964 return -EINVAL;
965 }
966
967 /* We can only do the GTT pwrite on untiled buffers, as otherwise
968 * it would end up going through the fenced access, and we'll get
969 * different detiling behavior between reading and writing.
970 * pread/pwrite currently are reading and writing from the CPU
971 * perspective, requiring manual detiling by the client.
972 */
71acb5eb
DA		 973 if (obj_priv->phys_obj)
974 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
975 else if (obj_priv->tiling_mode == I915_TILING_NONE &&
3de09aa3
EA		 976 dev->gtt_total != 0) {
977 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
978 if (ret == -EFAULT) {
979 ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
980 file_priv);
981 }
280b713b
EA		 982 } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
983 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
40123c1f
EA		 984 } else {
985 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
986 if (ret == -EFAULT) {
987 ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
988 file_priv);
989 }
990 }
673a394b
EA		 991
992#if WATCH_PWRITE
993 if (ret)
994 DRM_INFO("pwrite failed %d\n", ret);
995#endif
996
bc9025bd 997 drm_gem_object_unreference_unlocked(obj);
673a394b
EA		 998
999 return ret;
1000}
1001
1002/**
2ef7eeaa
EA		 1003 * Called when user space prepares to use an object with the CPU, either
1004 * through the mmap ioctl's mapping or a GTT mapping.
673a394b
EA		 1005 */
1006int
1007i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1008 struct drm_file *file_priv)
1009{
a09ba7fa 1010 struct drm_i915_private *dev_priv = dev->dev_private;
673a394b
EA		 1011 struct drm_i915_gem_set_domain *args = data;
1012 struct drm_gem_object *obj;
652c393a 1013 struct drm_i915_gem_object *obj_priv;
2ef7eeaa
EA		 1014 uint32_t read_domains = args->read_domains;
1015 uint32_t write_domain = args->write_domain;
673a394b
EA		 1016 int ret;
1017
1018 if (!(dev->driver->driver_features & DRIVER_GEM))
1019 return -ENODEV;
1020
2ef7eeaa 1021 /* Only handle setting domains to types used by the CPU. */
21d509e3 1022 if (write_domain & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 1023 return -EINVAL;
1024
21d509e3 1025 if (read_domains & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 1026 return -EINVAL;
1027
1028 /* Having something in the write domain implies it's in the read
1029 * domain, and only that read domain. Enforce that in the request.
1030 */
1031 if (write_domain != 0 && read_domains != write_domain)
1032 return -EINVAL;
1033
673a394b
EA		 1034 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1035 if (obj == NULL)
1036 return -EBADF;
23010e43 1037 obj_priv = to_intel_bo(obj);
673a394b
EA		 1038
1039 mutex_lock(&dev->struct_mutex);
652c393a
JB		 1040
1041 intel_mark_busy(dev, obj);
1042
673a394b 1043#if WATCH_BUF
cfd43c02 1044 DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
2ef7eeaa 1045 obj, obj->size, read_domains, write_domain);
673a394b 1046#endif
2ef7eeaa
EA		 1047 if (read_domains & I915_GEM_DOMAIN_GTT) {
1048 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
02354392 1049
a09ba7fa
EA		 1050 /* Update the LRU on the fence for the CPU access that's
1051 * about to occur.
1052 */
1053 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1054 list_move_tail(&obj_priv->fence_list,
1055 &dev_priv->mm.fence_list);
1056 }
1057
02354392
EA		 1058 /* Silently promote "you're not bound, there was nothing to do"
1059 * to success, since the client was just asking us to
1060 * make sure everything was done.
1061 */
1062 if (ret == -EINVAL)
1063 ret = 0;
2ef7eeaa 1064 } else {
e47c68e9 1065 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
2ef7eeaa
EA		 1066 }
1067
673a394b
EA		 1068 drm_gem_object_unreference(obj);
1069 mutex_unlock(&dev->struct_mutex);
1070 return ret;
1071}
1072
1073/**
1074 * Called when user space has done writes to this buffer
1075 */
1076int
1077i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1078 struct drm_file *file_priv)
1079{
1080 struct drm_i915_gem_sw_finish *args = data;
1081 struct drm_gem_object *obj;
1082 struct drm_i915_gem_object *obj_priv;
1083 int ret = 0;
1084
1085 if (!(dev->driver->driver_features & DRIVER_GEM))
1086 return -ENODEV;
1087
1088 mutex_lock(&dev->struct_mutex);
1089 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1090 if (obj == NULL) {
1091 mutex_unlock(&dev->struct_mutex);
1092 return -EBADF;
1093 }
1094
1095#if WATCH_BUF
cfd43c02 1096 DRM_INFO("%s: sw_finish %d (%p %zd)\n",
673a394b
EA		 1097 __func__, args->handle, obj, obj->size);
1098#endif
23010e43 1099 obj_priv = to_intel_bo(obj);
673a394b
EA		 1100
1101 /* Pinned buffers may be scanout, so flush the cache */
e47c68e9
EA		 1102 if (obj_priv->pin_count)
1103 i915_gem_object_flush_cpu_write_domain(obj);
1104
673a394b
EA		 1105 drm_gem_object_unreference(obj);
1106 mutex_unlock(&dev->struct_mutex);
1107 return ret;
1108}
1109
1110/**
1111 * Maps the contents of an object, returning the address it is mapped
1112 * into.
1113 *
1114 * While the mapping holds a reference on the contents of the object, it doesn't
1115 * imply a ref on the object itself.
1116 */
1117int
1118i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1119 struct drm_file *file_priv)
1120{
1121 struct drm_i915_gem_mmap *args = data;
1122 struct drm_gem_object *obj;
1123 loff_t offset;
1124 unsigned long addr;
1125
1126 if (!(dev->driver->driver_features & DRIVER_GEM))
1127 return -ENODEV;
1128
1129 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1130 if (obj == NULL)
1131 return -EBADF;
1132
1133 offset = args->offset;
1134
1135 down_write(&current->mm->mmap_sem);
1136 addr = do_mmap(obj->filp, 0, args->size,
1137 PROT_READ | PROT_WRITE, MAP_SHARED,
1138 args->offset);
1139 up_write(&current->mm->mmap_sem);
bc9025bd 1140 drm_gem_object_unreference_unlocked(obj);
673a394b
EA		 1141 if (IS_ERR((void *)addr))
1142 return addr;
1143
1144 args->addr_ptr = (uint64_t) addr;
1145
1146 return 0;
1147}
1148
de151cf6
JB		 1149/**
1150 * i915_gem_fault - fault a page into the GTT
1151 * vma: VMA in question
1152 * vmf: fault info
1153 *
1154 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1155 * from userspace. The fault handler takes care of binding the object to
1156 * the GTT (if needed), allocating and programming a fence register (again,
1157 * only if needed based on whether the old reg is still valid or the object
1158 * is tiled) and inserting a new PTE into the faulting process.
1159 *
1160 * Note that the faulting process may involve evicting existing objects
1161 * from the GTT and/or fence registers to make room. So performance may
1162 * suffer if the GTT working set is large or there are few fence registers
1163 * left.
1164 */
1165int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1166{
1167 struct drm_gem_object *obj = vma->vm_private_data;
1168 struct drm_device *dev = obj->dev;
1169 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 1170 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6
JB		 1171 pgoff_t page_offset;
1172 unsigned long pfn;
1173 int ret = 0;
0f973f27 1174 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
de151cf6
JB		 1175
1176 /* We don't use vmf->pgoff since that has the fake offset */
1177 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1178 PAGE_SHIFT;
1179
1180 /* Now bind it into the GTT if needed */
1181 mutex_lock(&dev->struct_mutex);
1182 if (!obj_priv->gtt_space) {
e67b8ce1 1183 ret = i915_gem_object_bind_to_gtt(obj, 0);
c715089f
CW		 1184 if (ret)
1185 goto unlock;
07f4f3e8 1186
14b60391 1187 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
07f4f3e8
KH		 1188
1189 ret = i915_gem_object_set_to_gtt_domain(obj, write);
c715089f
CW		 1190 if (ret)
1191 goto unlock;
de151cf6
JB		 1192 }
1193
1194 /* Need a new fence register? */
a09ba7fa 1195 if (obj_priv->tiling_mode != I915_TILING_NONE) {
8c4b8c3f 1196 ret = i915_gem_object_get_fence_reg(obj);
c715089f
CW		 1197 if (ret)
1198 goto unlock;
d9ddcb96 1199 }
de151cf6
JB		 1200
1201 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1202 page_offset;
1203
1204 /* Finally, remap it using the new GTT offset */
1205 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
c715089f 1206unlock:
de151cf6
JB		 1207 mutex_unlock(&dev->struct_mutex);
1208
1209 switch (ret) {
c715089f
CW		 1210 case 0:
1211 case -ERESTARTSYS:
1212 return VM_FAULT_NOPAGE;
de151cf6
JB		 1213 case -ENOMEM:
1214 case -EAGAIN:
1215 return VM_FAULT_OOM;
de151cf6 1216 default:
c715089f 1217 return VM_FAULT_SIGBUS;
de151cf6
JB		 1218 }
1219}
1220
1221/**
1222 * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1223 * @obj: obj in question
1224 *
1225 * GEM memory mapping works by handing back to userspace a fake mmap offset
1226 * it can use in a subsequent mmap(2) call. The DRM core code then looks
1227 * up the object based on the offset and sets up the various memory mapping
1228 * structures.
1229 *
1230 * This routine allocates and attaches a fake offset for @obj.
1231 */
1232static int
1233i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1234{
1235 struct drm_device *dev = obj->dev;
1236 struct drm_gem_mm *mm = dev->mm_private;
23010e43 1237 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6 1238 struct drm_map_list *list;
f77d390c 1239 struct drm_local_map *map;
de151cf6
JB		 1240 int ret = 0;
1241
1242 /* Set the object up for mmap'ing */
1243 list = &obj->map_list;
9a298b2a 1244 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
de151cf6
JB		 1245 if (!list->map)
1246 return -ENOMEM;
1247
1248 map = list->map;
1249 map->type = _DRM_GEM;
1250 map->size = obj->size;
1251 map->handle = obj;
1252
1253 /* Get a DRM GEM mmap offset allocated... */
1254 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1255 obj->size / PAGE_SIZE, 0, 0);
1256 if (!list->file_offset_node) {
1257 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1258 ret = -ENOMEM;
1259 goto out_free_list;
1260 }
1261
1262 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1263 obj->size / PAGE_SIZE, 0);
1264 if (!list->file_offset_node) {
1265 ret = -ENOMEM;
1266 goto out_free_list;
1267 }
1268
1269 list->hash.key = list->file_offset_node->start;
1270 if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) {
1271 DRM_ERROR("failed to add to map hash\n");
5618ca6a 1272 ret = -ENOMEM;
de151cf6
JB		 1273 goto out_free_mm;
1274 }
1275
1276 /* By now we should be all set, any drm_mmap request on the offset
1277 * below will get to our mmap & fault handler */
1278 obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1279
1280 return 0;
1281
1282out_free_mm:
1283 drm_mm_put_block(list->file_offset_node);
1284out_free_list:
9a298b2a 1285 kfree(list->map);
de151cf6
JB		 1286
1287 return ret;
1288}
1289
901782b2
CW		 1290/**
1291 * i915_gem_release_mmap - remove physical page mappings
1292 * @obj: obj in question
1293 *
af901ca1 1294 * Preserve the reservation of the mmapping with the DRM core code, but
901782b2
CW		 1295 * relinquish ownership of the pages back to the system.
1296 *
1297 * It is vital that we remove the page mapping if we have mapped a tiled
1298 * object through the GTT and then lose the fence register due to
1299 * resource pressure. Similarly if the object has been moved out of the
1300 * aperture, than pages mapped into userspace must be revoked. Removing the
1301 * mapping will then trigger a page fault on the next user access, allowing
1302 * fixup by i915_gem_fault().
1303 */
d05ca301 1304void
901782b2
CW		 1305i915_gem_release_mmap(struct drm_gem_object *obj)
1306{
1307 struct drm_device *dev = obj->dev;
23010e43 1308 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
901782b2
CW		 1309
1310 if (dev->dev_mapping)
1311 unmap_mapping_range(dev->dev_mapping,
1312 obj_priv->mmap_offset, obj->size, 1);
1313}
1314
ab00b3e5
JB		 1315static void
1316i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1317{
1318 struct drm_device *dev = obj->dev;
23010e43 1319 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
ab00b3e5
JB		 1320 struct drm_gem_mm *mm = dev->mm_private;
1321 struct drm_map_list *list;
1322
1323 list = &obj->map_list;
1324 drm_ht_remove_item(&mm->offset_hash, &list->hash);
1325
1326 if (list->file_offset_node) {
1327 drm_mm_put_block(list->file_offset_node);
1328 list->file_offset_node = NULL;
1329 }
1330
1331 if (list->map) {
9a298b2a 1332 kfree(list->map);
ab00b3e5
JB		 1333 list->map = NULL;
1334 }
1335
1336 obj_priv->mmap_offset = 0;
1337}
1338
de151cf6
JB		 1339/**
1340 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1341 * @obj: object to check
1342 *
1343 * Return the required GTT alignment for an object, taking into account
1344 * potential fence register mapping if needed.
1345 */
1346static uint32_t
1347i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1348{
1349 struct drm_device *dev = obj->dev;
23010e43 1350 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6
JB		 1351 int start, i;
1352
1353 /*
1354 * Minimum alignment is 4k (GTT page size), but might be greater
1355 * if a fence register is needed for the object.
1356 */
1357 if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE)
1358 return 4096;
1359
1360 /*
1361 * Previous chips need to be aligned to the size of the smallest
1362 * fence register that can contain the object.
1363 */
1364 if (IS_I9XX(dev))
1365 start = 1024*1024;
1366 else
1367 start = 512*1024;
1368
1369 for (i = start; i < obj->size; i <<= 1)
1370 ;
1371
1372 return i;
1373}
1374
1375/**
1376 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1377 * @dev: DRM device
1378 * @data: GTT mapping ioctl data
1379 * @file_priv: GEM object info
1380 *
1381 * Simply returns the fake offset to userspace so it can mmap it.
1382 * The mmap call will end up in drm_gem_mmap(), which will set things
1383 * up so we can get faults in the handler above.
1384 *
1385 * The fault handler will take care of binding the object into the GTT
1386 * (since it may have been evicted to make room for something), allocating
1387 * a fence register, and mapping the appropriate aperture address into
1388 * userspace.
1389 */
1390int
1391i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1392 struct drm_file *file_priv)
1393{
1394 struct drm_i915_gem_mmap_gtt *args = data;
1395 struct drm_i915_private *dev_priv = dev->dev_private;
1396 struct drm_gem_object *obj;
1397 struct drm_i915_gem_object *obj_priv;
1398 int ret;
1399
1400 if (!(dev->driver->driver_features & DRIVER_GEM))
1401 return -ENODEV;
1402
1403 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1404 if (obj == NULL)
1405 return -EBADF;
1406
1407 mutex_lock(&dev->struct_mutex);
1408
23010e43 1409 obj_priv = to_intel_bo(obj);
de151cf6 1410
ab18282d
CW		 1411 if (obj_priv->madv != I915_MADV_WILLNEED) {
1412 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1413 drm_gem_object_unreference(obj);
1414 mutex_unlock(&dev->struct_mutex);
1415 return -EINVAL;
1416 }
1417
1418
de151cf6
JB		 1419 if (!obj_priv->mmap_offset) {
1420 ret = i915_gem_create_mmap_offset(obj);
13af1062
CW		 1421 if (ret) {
1422 drm_gem_object_unreference(obj);
1423 mutex_unlock(&dev->struct_mutex);
de151cf6 1424 return ret;
13af1062 1425 }
de151cf6
JB		 1426 }
1427
1428 args->offset = obj_priv->mmap_offset;
1429
de151cf6
JB		 1430 /*
1431 * Pull it into the GTT so that we have a page list (makes the
1432 * initial fault faster and any subsequent flushing possible).
1433 */
1434 if (!obj_priv->agp_mem) {
e67b8ce1 1435 ret = i915_gem_object_bind_to_gtt(obj, 0);
de151cf6
JB		 1436 if (ret) {
1437 drm_gem_object_unreference(obj);
1438 mutex_unlock(&dev->struct_mutex);
1439 return ret;
1440 }
14b60391 1441 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
de151cf6
JB		 1442 }
1443
1444 drm_gem_object_unreference(obj);
1445 mutex_unlock(&dev->struct_mutex);
1446
1447 return 0;
1448}
1449
6911a9b8 1450void
856fa198 1451i915_gem_object_put_pages(struct drm_gem_object *obj)
673a394b 1452{
23010e43 1453 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 1454 int page_count = obj->size / PAGE_SIZE;
1455 int i;
1456
856fa198 1457 BUG_ON(obj_priv->pages_refcount == 0);
bb6baf76 1458 BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
673a394b 1459
856fa198
EA		 1460 if (--obj_priv->pages_refcount != 0)
1461 return;
673a394b 1462
280b713b
EA		 1463 if (obj_priv->tiling_mode != I915_TILING_NONE)
1464 i915_gem_object_save_bit_17_swizzle(obj);
1465
3ef94daa 1466 if (obj_priv->madv == I915_MADV_DONTNEED)
13a05fd9 1467 obj_priv->dirty = 0;
3ef94daa
CW		 1468
1469 for (i = 0; i < page_count; i++) {
3ef94daa
CW		 1470 if (obj_priv->dirty)
1471 set_page_dirty(obj_priv->pages[i]);
1472
1473 if (obj_priv->madv == I915_MADV_WILLNEED)
856fa198 1474 mark_page_accessed(obj_priv->pages[i]);
3ef94daa
CW		 1475
1476 page_cache_release(obj_priv->pages[i]);
1477 }
673a394b
EA		 1478 obj_priv->dirty = 0;
1479
8e7d2b2c 1480 drm_free_large(obj_priv->pages);
856fa198 1481 obj_priv->pages = NULL;
673a394b
EA		 1482}
1483
1484static void
ce44b0ea 1485i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno)
673a394b
EA		 1486{
1487 struct drm_device *dev = obj->dev;
1488 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 1489 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 1490
1491 /* Add a reference if we're newly entering the active list. */
1492 if (!obj_priv->active) {
1493 drm_gem_object_reference(obj);
1494 obj_priv->active = 1;
1495 }
1496 /* Move from whatever list we were on to the tail of execution. */
5e118f41 1497 spin_lock(&dev_priv->mm.active_list_lock);
673a394b
EA		 1498 list_move_tail(&obj_priv->list,
1499 &dev_priv->mm.active_list);
5e118f41 1500 spin_unlock(&dev_priv->mm.active_list_lock);
ce44b0ea 1501 obj_priv->last_rendering_seqno = seqno;
673a394b
EA		 1502}
1503
ce44b0ea
EA		 1504static void
1505i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1506{
1507 struct drm_device *dev = obj->dev;
1508 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 1509 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
ce44b0ea
EA		 1510
1511 BUG_ON(!obj_priv->active);
1512 list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1513 obj_priv->last_rendering_seqno = 0;
1514}
673a394b 1515
963b4836
CW		 1516/* Immediately discard the backing storage */
1517static void
1518i915_gem_object_truncate(struct drm_gem_object *obj)
1519{
23010e43 1520 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
bb6baf76 1521 struct inode *inode;
963b4836 1522
bb6baf76
CW		 1523 inode = obj->filp->f_path.dentry->d_inode;
1524 if (inode->i_op->truncate)
1525 inode->i_op->truncate (inode);
1526
1527 obj_priv->madv = __I915_MADV_PURGED;
963b4836
CW		 1528}
1529
1530static inline int
1531i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1532{
1533 return obj_priv->madv == I915_MADV_DONTNEED;
1534}
1535
673a394b
EA		 1536static void
1537i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1538{
1539 struct drm_device *dev = obj->dev;
1540 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 1541 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 1542
1543 i915_verify_inactive(dev, __FILE__, __LINE__);
1544 if (obj_priv->pin_count != 0)
1545 list_del_init(&obj_priv->list);
1546 else
1547 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1548
99fcb766
DV		 1549 BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1550
ce44b0ea 1551 obj_priv->last_rendering_seqno = 0;
673a394b
EA		 1552 if (obj_priv->active) {
1553 obj_priv->active = 0;
1554 drm_gem_object_unreference(obj);
1555 }
1556 i915_verify_inactive(dev, __FILE__, __LINE__);
1557}
1558
63560396
DV		 1559static void
1560i915_gem_process_flushing_list(struct drm_device *dev,
1561 uint32_t flush_domains, uint32_t seqno)
1562{
1563 drm_i915_private_t *dev_priv = dev->dev_private;
1564 struct drm_i915_gem_object *obj_priv, *next;
1565
1566 list_for_each_entry_safe(obj_priv, next,
1567 &dev_priv->mm.gpu_write_list,
1568 gpu_write_list) {
a8089e84 1569 struct drm_gem_object *obj = &obj_priv->base;
63560396
DV		 1570
1571 if ((obj->write_domain & flush_domains) ==
1572 obj->write_domain) {
1573 uint32_t old_write_domain = obj->write_domain;
1574
1575 obj->write_domain = 0;
1576 list_del_init(&obj_priv->gpu_write_list);
1577 i915_gem_object_move_to_active(obj, seqno);
1578
1579 /* update the fence lru list */
1580 if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
1581 list_move_tail(&obj_priv->fence_list,
1582 &dev_priv->mm.fence_list);
1583
1584 trace_i915_gem_object_change_domain(obj,
1585 obj->read_domains,
1586 old_write_domain);
1587 }
1588 }
1589}
1590
e552eb70
JB		 1591#define PIPE_CONTROL_FLUSH(addr) \
1592 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | \
1593 PIPE_CONTROL_DEPTH_STALL); \
1594 OUT_RING(addr | PIPE_CONTROL_GLOBAL_GTT); \
1595 OUT_RING(0); \
1596 OUT_RING(0); \
1597
673a394b
EA		 1598/**
1599 * Creates a new sequence number, emitting a write of it to the status page
1600 * plus an interrupt, which will trigger i915_user_interrupt_handler.
1601 *
1602 * Must be called with struct_lock held.
1603 *
1604 * Returned sequence numbers are nonzero on success.
1605 */
5a5a0c64 1606uint32_t
b962442e
EA		 1607i915_add_request(struct drm_device *dev, struct drm_file *file_priv,
1608 uint32_t flush_domains)
673a394b
EA		 1609{
1610 drm_i915_private_t *dev_priv = dev->dev_private;
b962442e 1611 struct drm_i915_file_private *i915_file_priv = NULL;
673a394b
EA		 1612 struct drm_i915_gem_request *request;
1613 uint32_t seqno;
1614 int was_empty;
1615 RING_LOCALS;
1616
b962442e
EA		 1617 if (file_priv != NULL)
1618 i915_file_priv = file_priv->driver_priv;
1619
9a298b2a 1620 request = kzalloc(sizeof(*request), GFP_KERNEL);
673a394b
EA		 1621 if (request == NULL)
1622 return 0;
1623
1624 /* Grab the seqno we're going to make this request be, and bump the
1625 * next (skipping 0 so it can be the reserved no-seqno value).
1626 */
1627 seqno = dev_priv->mm.next_gem_seqno;
1628 dev_priv->mm.next_gem_seqno++;
1629 if (dev_priv->mm.next_gem_seqno == 0)
1630 dev_priv->mm.next_gem_seqno++;
1631
e552eb70
JB		 1632 if (HAS_PIPE_CONTROL(dev)) {
1633 u32 scratch_addr = dev_priv->seqno_gfx_addr + 128;
673a394b 1634
e552eb70
JB		 1635 /*
1636 * Workaround qword write incoherence by flushing the
1637 * PIPE_NOTIFY buffers out to memory before requesting
1638 * an interrupt.
1639 */
1640 BEGIN_LP_RING(32);
1641 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
1642 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
1643 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
1644 OUT_RING(seqno);
1645 OUT_RING(0);
1646 PIPE_CONTROL_FLUSH(scratch_addr);
1647 scratch_addr += 128; /* write to separate cachelines */
1648 PIPE_CONTROL_FLUSH(scratch_addr);
1649 scratch_addr += 128;
1650 PIPE_CONTROL_FLUSH(scratch_addr);
1651 scratch_addr += 128;
1652 PIPE_CONTROL_FLUSH(scratch_addr);
1653 scratch_addr += 128;
1654 PIPE_CONTROL_FLUSH(scratch_addr);
1655 scratch_addr += 128;
1656 PIPE_CONTROL_FLUSH(scratch_addr);
1657 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
1658 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
1659 PIPE_CONTROL_NOTIFY);
1660 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
1661 OUT_RING(seqno);
1662 OUT_RING(0);
1663 ADVANCE_LP_RING();
1664 } else {
1665 BEGIN_LP_RING(4);
1666 OUT_RING(MI_STORE_DWORD_INDEX);
1667 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1668 OUT_RING(seqno);
1669
1670 OUT_RING(MI_USER_INTERRUPT);
1671 ADVANCE_LP_RING();
1672 }
673a394b 1673
44d98a61 1674 DRM_DEBUG_DRIVER("%d\n", seqno);
673a394b
EA		 1675
1676 request->seqno = seqno;
1677 request->emitted_jiffies = jiffies;
673a394b
EA		 1678 was_empty = list_empty(&dev_priv->mm.request_list);
1679 list_add_tail(&request->list, &dev_priv->mm.request_list);
b962442e
EA		 1680 if (i915_file_priv) {
1681 list_add_tail(&request->client_list,
1682 &i915_file_priv->mm.request_list);
1683 } else {
1684 INIT_LIST_HEAD(&request->client_list);
1685 }
673a394b 1686
ce44b0ea
EA		 1687 /* Associate any objects on the flushing list matching the write
1688 * domain we're flushing with our flush.
1689 */
63560396
DV		 1690 if (flush_domains != 0)
1691 i915_gem_process_flushing_list(dev, flush_domains, seqno);
ce44b0ea 1692
f65d9421
BG		 1693 if (!dev_priv->mm.suspended) {
1694 mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD);
1695 if (was_empty)
1696 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1697 }
673a394b
EA		 1698 return seqno;
1699}
1700
1701/**
1702 * Command execution barrier
1703 *
1704 * Ensures that all commands in the ring are finished
1705 * before signalling the CPU
1706 */
3043c60c 1707static uint32_t
673a394b
EA		 1708i915_retire_commands(struct drm_device *dev)
1709{
1710 drm_i915_private_t *dev_priv = dev->dev_private;
1711 uint32_t cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
1712 uint32_t flush_domains = 0;
1713 RING_LOCALS;
1714
1715 /* The sampler always gets flushed on i965 (sigh) */
1716 if (IS_I965G(dev))
1717 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1718 BEGIN_LP_RING(2);
1719 OUT_RING(cmd);
1720 OUT_RING(0); /* noop */
1721 ADVANCE_LP_RING();
1722 return flush_domains;
1723}
1724
1725/**
1726 * Moves buffers associated only with the given active seqno from the active
1727 * to inactive list, potentially freeing them.
1728 */
1729static void
1730i915_gem_retire_request(struct drm_device *dev,
1731 struct drm_i915_gem_request *request)
1732{
1733 drm_i915_private_t *dev_priv = dev->dev_private;
1734
1c5d22f7
CW		 1735 trace_i915_gem_request_retire(dev, request->seqno);
1736
673a394b
EA		 1737 /* Move any buffers on the active list that are no longer referenced
1738 * by the ringbuffer to the flushing/inactive lists as appropriate.
1739 */
5e118f41 1740 spin_lock(&dev_priv->mm.active_list_lock);
673a394b
EA		 1741 while (!list_empty(&dev_priv->mm.active_list)) {
1742 struct drm_gem_object *obj;
1743 struct drm_i915_gem_object *obj_priv;
1744
1745 obj_priv = list_first_entry(&dev_priv->mm.active_list,
1746 struct drm_i915_gem_object,
1747 list);
a8089e84 1748 obj = &obj_priv->base;
673a394b
EA		 1749
1750 /* If the seqno being retired doesn't match the oldest in the
1751 * list, then the oldest in the list must still be newer than
1752 * this seqno.
1753 */
1754 if (obj_priv->last_rendering_seqno != request->seqno)
5e118f41 1755 goto out;
de151cf6 1756
673a394b
EA		 1757#if WATCH_LRU
1758 DRM_INFO("%s: retire %d moves to inactive list %p\n",
1759 __func__, request->seqno, obj);
1760#endif
1761
ce44b0ea
EA		 1762 if (obj->write_domain != 0)
1763 i915_gem_object_move_to_flushing(obj);
68c84342
SL		 1764 else {
1765 /* Take a reference on the object so it won't be
1766 * freed while the spinlock is held. The list
1767 * protection for this spinlock is safe when breaking
1768 * the lock like this since the next thing we do
1769 * is just get the head of the list again.
1770 */
1771 drm_gem_object_reference(obj);
673a394b 1772 i915_gem_object_move_to_inactive(obj);
68c84342
SL		 1773 spin_unlock(&dev_priv->mm.active_list_lock);
1774 drm_gem_object_unreference(obj);
1775 spin_lock(&dev_priv->mm.active_list_lock);
1776 }
673a394b 1777 }
5e118f41
CW		 1778out:
1779 spin_unlock(&dev_priv->mm.active_list_lock);
673a394b
EA		 1780}
1781
1782/**
1783 * Returns true if seq1 is later than seq2.
1784 */
22be1724 1785bool
673a394b
EA		 1786i915_seqno_passed(uint32_t seq1, uint32_t seq2)
1787{
1788 return (int32_t)(seq1 - seq2) >= 0;
1789}
1790
1791uint32_t
1792i915_get_gem_seqno(struct drm_device *dev)
1793{
1794 drm_i915_private_t *dev_priv = dev->dev_private;
1795
1918ad77 1796 if (HAS_PIPE_CONTROL(dev))
e552eb70
JB		 1797 return ((volatile u32 *)(dev_priv->seqno_page))[0];
1798 else
1799 return READ_HWSP(dev_priv, I915_GEM_HWS_INDEX);
673a394b
EA		 1800}
1801
1802/**
1803 * This function clears the request list as sequence numbers are passed.
1804 */
1805void
1806i915_gem_retire_requests(struct drm_device *dev)
1807{
1808 drm_i915_private_t *dev_priv = dev->dev_private;
1809 uint32_t seqno;
1810
9d34e5db 1811 if (!dev_priv->hw_status_page || list_empty(&dev_priv->mm.request_list))
6c0594a3
KW		 1812 return;
1813
673a394b
EA		 1814 seqno = i915_get_gem_seqno(dev);
1815
1816 while (!list_empty(&dev_priv->mm.request_list)) {
1817 struct drm_i915_gem_request *request;
1818 uint32_t retiring_seqno;
1819
1820 request = list_first_entry(&dev_priv->mm.request_list,
1821 struct drm_i915_gem_request,
1822 list);
1823 retiring_seqno = request->seqno;
1824
1825 if (i915_seqno_passed(seqno, retiring_seqno) ||
ba1234d1 1826 atomic_read(&dev_priv->mm.wedged)) {
673a394b
EA		 1827 i915_gem_retire_request(dev, request);
1828
1829 list_del(&request->list);
b962442e 1830 list_del(&request->client_list);
9a298b2a 1831 kfree(request);
673a394b
EA		 1832 } else
1833 break;
1834 }
9d34e5db
CW		 1835
1836 if (unlikely (dev_priv->trace_irq_seqno &&
1837 i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
1838 i915_user_irq_put(dev);
1839 dev_priv->trace_irq_seqno = 0;
1840 }
673a394b
EA		 1841}
1842
1843void
1844i915_gem_retire_work_handler(struct work_struct *work)
1845{
1846 drm_i915_private_t *dev_priv;
1847 struct drm_device *dev;
1848
1849 dev_priv = container_of(work, drm_i915_private_t,
1850 mm.retire_work.work);
1851 dev = dev_priv->dev;
1852
1853 mutex_lock(&dev->struct_mutex);
1854 i915_gem_retire_requests(dev);
6dbe2772
KP		 1855 if (!dev_priv->mm.suspended &&
1856 !list_empty(&dev_priv->mm.request_list))
9c9fe1f8 1857 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
673a394b
EA		 1858 mutex_unlock(&dev->struct_mutex);
1859}
1860
5a5a0c64 1861int
48764bf4 1862i915_do_wait_request(struct drm_device *dev, uint32_t seqno, int interruptible)
673a394b
EA		 1863{
1864 drm_i915_private_t *dev_priv = dev->dev_private;
802c7eb6 1865 u32 ier;
673a394b
EA		 1866 int ret = 0;
1867
1868 BUG_ON(seqno == 0);
1869
ba1234d1 1870 if (atomic_read(&dev_priv->mm.wedged))
ffed1d09
BG		 1871 return -EIO;
1872
673a394b 1873 if (!i915_seqno_passed(i915_get_gem_seqno(dev), seqno)) {
bad720ff 1874 if (HAS_PCH_SPLIT(dev))
036a4a7d
ZW		 1875 ier = I915_READ(DEIER) | I915_READ(GTIER);
1876 else
1877 ier = I915_READ(IER);
802c7eb6
JB		 1878 if (!ier) {
1879 DRM_ERROR("something (likely vbetool) disabled "
1880 "interrupts, re-enabling\n");
1881 i915_driver_irq_preinstall(dev);
1882 i915_driver_irq_postinstall(dev);
1883 }
1884
1c5d22f7
CW		 1885 trace_i915_gem_request_wait_begin(dev, seqno);
1886
673a394b
EA		 1887 dev_priv->mm.waiting_gem_seqno = seqno;
1888 i915_user_irq_get(dev);
48764bf4
DV		 1889 if (interruptible)
1890 ret = wait_event_interruptible(dev_priv->irq_queue,
1891 i915_seqno_passed(i915_get_gem_seqno(dev), seqno) ||
1892 atomic_read(&dev_priv->mm.wedged));
1893 else
1894 wait_event(dev_priv->irq_queue,
1895 i915_seqno_passed(i915_get_gem_seqno(dev), seqno) ||
1896 atomic_read(&dev_priv->mm.wedged));
1897
673a394b
EA		 1898 i915_user_irq_put(dev);
1899 dev_priv->mm.waiting_gem_seqno = 0;
1c5d22f7
CW		 1900
1901 trace_i915_gem_request_wait_end(dev, seqno);
673a394b 1902 }
ba1234d1 1903 if (atomic_read(&dev_priv->mm.wedged))
673a394b
EA		 1904 ret = -EIO;
1905
1906 if (ret && ret != -ERESTARTSYS)
1907 DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
1908 __func__, ret, seqno, i915_get_gem_seqno(dev));
1909
1910 /* Directly dispatch request retiring. While we have the work queue
1911 * to handle this, the waiter on a request often wants an associated
1912 * buffer to have made it to the inactive list, and we would need
1913 * a separate wait queue to handle that.
1914 */
1915 if (ret == 0)
1916 i915_gem_retire_requests(dev);
1917
1918 return ret;
1919}
1920
48764bf4
DV		 1921/**
1922 * Waits for a sequence number to be signaled, and cleans up the
1923 * request and object lists appropriately for that event.
1924 */
1925static int
1926i915_wait_request(struct drm_device *dev, uint32_t seqno)
1927{
1928 return i915_do_wait_request(dev, seqno, 1);
1929}
1930
673a394b
EA		 1931static void
1932i915_gem_flush(struct drm_device *dev,
1933 uint32_t invalidate_domains,
1934 uint32_t flush_domains)
1935{
1936 drm_i915_private_t *dev_priv = dev->dev_private;
1937 uint32_t cmd;
1938 RING_LOCALS;
1939
1940#if WATCH_EXEC
1941 DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
1942 invalidate_domains, flush_domains);
1943#endif
1c5d22f7
CW		 1944 trace_i915_gem_request_flush(dev, dev_priv->mm.next_gem_seqno,
1945 invalidate_domains, flush_domains);
673a394b
EA		 1946
1947 if (flush_domains & I915_GEM_DOMAIN_CPU)
1948 drm_agp_chipset_flush(dev);
1949
21d509e3 1950 if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
673a394b
EA		 1951 /*
1952 * read/write caches:
1953 *
1954 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
1955 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
1956 * also flushed at 2d versus 3d pipeline switches.
1957 *
1958 * read-only caches:
1959 *
1960 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
1961 * MI_READ_FLUSH is set, and is always flushed on 965.
1962 *
1963 * I915_GEM_DOMAIN_COMMAND may not exist?
1964 *
1965 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
1966 * invalidated when MI_EXE_FLUSH is set.
1967 *
1968 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
1969 * invalidated with every MI_FLUSH.
1970 *
1971 * TLBs:
1972 *
1973 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
1974 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
1975 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
1976 * are flushed at any MI_FLUSH.
1977 */
1978
1979 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
1980 if ((invalidate_domains|flush_domains) &
1981 I915_GEM_DOMAIN_RENDER)
1982 cmd &= ~MI_NO_WRITE_FLUSH;
1983 if (!IS_I965G(dev)) {
1984 /*
1985 * On the 965, the sampler cache always gets flushed
1986 * and this bit is reserved.
1987 */
1988 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
1989 cmd |= MI_READ_FLUSH;
1990 }
1991 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
1992 cmd |= MI_EXE_FLUSH;
1993
1994#if WATCH_EXEC
1995 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
1996#endif
1997 BEGIN_LP_RING(2);
1998 OUT_RING(cmd);
48764bf4 1999 OUT_RING(MI_NOOP);
673a394b
EA		 2000 ADVANCE_LP_RING();
2001 }
2002}
2003
2004/**
2005 * Ensures that all rendering to the object has completed and the object is
2006 * safe to unbind from the GTT or access from the CPU.
2007 */
2008static int
2009i915_gem_object_wait_rendering(struct drm_gem_object *obj)
2010{
2011 struct drm_device *dev = obj->dev;
23010e43 2012 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 2013 int ret;
2014
e47c68e9
EA		 2015 /* This function only exists to support waiting for existing rendering,
2016 * not for emitting required flushes.
673a394b 2017 */
e47c68e9 2018 BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
673a394b
EA		 2019
2020 /* If there is rendering queued on the buffer being evicted, wait for
2021 * it.
2022 */
2023 if (obj_priv->active) {
2024#if WATCH_BUF
2025 DRM_INFO("%s: object %p wait for seqno %08x\n",
2026 __func__, obj, obj_priv->last_rendering_seqno);
2027#endif
2028 ret = i915_wait_request(dev, obj_priv->last_rendering_seqno);
2029 if (ret != 0)
2030 return ret;
2031 }
2032
2033 return 0;
2034}
2035
2036/**
2037 * Unbinds an object from the GTT aperture.
2038 */
0f973f27 2039int
673a394b
EA		 2040i915_gem_object_unbind(struct drm_gem_object *obj)
2041{
2042 struct drm_device *dev = obj->dev;
4a87b8ca 2043 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2044 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 2045 int ret = 0;
2046
2047#if WATCH_BUF
2048 DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
2049 DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
2050#endif
2051 if (obj_priv->gtt_space == NULL)
2052 return 0;
2053
2054 if (obj_priv->pin_count != 0) {
2055 DRM_ERROR("Attempting to unbind pinned buffer\n");
2056 return -EINVAL;
2057 }
2058
5323fd04
EA		 2059 /* blow away mappings if mapped through GTT */
2060 i915_gem_release_mmap(obj);
2061
673a394b
EA		 2062 /* Move the object to the CPU domain to ensure that
2063 * any possible CPU writes while it's not in the GTT
2064 * are flushed when we go to remap it. This will
2065 * also ensure that all pending GPU writes are finished
2066 * before we unbind.
2067 */
e47c68e9 2068 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
673a394b 2069 if (ret) {
e47c68e9
EA		 2070 if (ret != -ERESTARTSYS)
2071 DRM_ERROR("set_domain failed: %d\n", ret);
673a394b
EA		 2072 return ret;
2073 }
2074
5323fd04
EA		 2075 BUG_ON(obj_priv->active);
2076
96b47b65
DV		 2077 /* release the fence reg _after_ flushing */
2078 if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2079 i915_gem_clear_fence_reg(obj);
2080
673a394b
EA		 2081 if (obj_priv->agp_mem != NULL) {
2082 drm_unbind_agp(obj_priv->agp_mem);
2083 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2084 obj_priv->agp_mem = NULL;
2085 }
2086
856fa198 2087 i915_gem_object_put_pages(obj);
a32808c0 2088 BUG_ON(obj_priv->pages_refcount);
673a394b
EA		 2089
2090 if (obj_priv->gtt_space) {
2091 atomic_dec(&dev->gtt_count);
2092 atomic_sub(obj->size, &dev->gtt_memory);
2093
2094 drm_mm_put_block(obj_priv->gtt_space);
2095 obj_priv->gtt_space = NULL;
2096 }
2097
2098 /* Remove ourselves from the LRU list if present. */
4a87b8ca 2099 spin_lock(&dev_priv->mm.active_list_lock);
673a394b
EA		 2100 if (!list_empty(&obj_priv->list))
2101 list_del_init(&obj_priv->list);
4a87b8ca 2102 spin_unlock(&dev_priv->mm.active_list_lock);
673a394b 2103
963b4836
CW		 2104 if (i915_gem_object_is_purgeable(obj_priv))
2105 i915_gem_object_truncate(obj);
2106
1c5d22f7
CW		 2107 trace_i915_gem_object_unbind(obj);
2108
673a394b
EA		 2109 return 0;
2110}
2111
07f73f69
CW		 2112static struct drm_gem_object *
2113i915_gem_find_inactive_object(struct drm_device *dev, int min_size)
2114{
2115 drm_i915_private_t *dev_priv = dev->dev_private;
2116 struct drm_i915_gem_object *obj_priv;
2117 struct drm_gem_object *best = NULL;
2118 struct drm_gem_object *first = NULL;
2119
2120 /* Try to find the smallest clean object */
2121 list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) {
a8089e84 2122 struct drm_gem_object *obj = &obj_priv->base;
07f73f69 2123 if (obj->size >= min_size) {
963b4836
CW		 2124 if ((!obj_priv->dirty ||
2125 i915_gem_object_is_purgeable(obj_priv)) &&
07f73f69
CW		 2126 (!best || obj->size < best->size)) {
2127 best = obj;
2128 if (best->size == min_size)
2129 return best;
2130 }
2131 if (!first)
2132 first = obj;
2133 }
2134 }
2135
2136 return best ? best : first;
2137}
2138
4df2faf4
DV		 2139static int
2140i915_gpu_idle(struct drm_device *dev)
2141{
2142 drm_i915_private_t *dev_priv = dev->dev_private;
2143 bool lists_empty;
2144 uint32_t seqno;
2145
2146 spin_lock(&dev_priv->mm.active_list_lock);
2147 lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
2148 list_empty(&dev_priv->mm.active_list);
2149 spin_unlock(&dev_priv->mm.active_list_lock);
2150
2151 if (lists_empty)
2152 return 0;
2153
2154 /* Flush everything onto the inactive list. */
2155 i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2156 seqno = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS);
2157 if (seqno == 0)
2158 return -ENOMEM;
2159
2160 return i915_wait_request(dev, seqno);
2161}
2162
673a394b 2163static int
07f73f69
CW		 2164i915_gem_evict_everything(struct drm_device *dev)
2165{
2166 drm_i915_private_t *dev_priv = dev->dev_private;
07f73f69
CW		 2167 int ret;
2168 bool lists_empty;
2169
07f73f69
CW		 2170 spin_lock(&dev_priv->mm.active_list_lock);
2171 lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
2172 list_empty(&dev_priv->mm.flushing_list) &&
2173 list_empty(&dev_priv->mm.active_list));
2174 spin_unlock(&dev_priv->mm.active_list_lock);
2175
9731129c 2176 if (lists_empty)
07f73f69 2177 return -ENOSPC;
07f73f69
CW		 2178
2179 /* Flush everything (on to the inactive lists) and evict */
4df2faf4 2180 ret = i915_gpu_idle(dev);
07f73f69
CW		 2181 if (ret)
2182 return ret;
2183
99fcb766
DV		 2184 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
2185
ab5ee576 2186 ret = i915_gem_evict_from_inactive_list(dev);
07f73f69
CW		 2187 if (ret)
2188 return ret;
2189
2190 spin_lock(&dev_priv->mm.active_list_lock);
2191 lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
2192 list_empty(&dev_priv->mm.flushing_list) &&
2193 list_empty(&dev_priv->mm.active_list));
2194 spin_unlock(&dev_priv->mm.active_list_lock);
2195 BUG_ON(!lists_empty);
2196
2197 return 0;
2198}
2199
673a394b 2200static int
07f73f69 2201i915_gem_evict_something(struct drm_device *dev, int min_size)
673a394b
EA		 2202{
2203 drm_i915_private_t *dev_priv = dev->dev_private;
2204 struct drm_gem_object *obj;
07f73f69 2205 int ret;
673a394b
EA		 2206
2207 for (;;) {
07f73f69
CW		 2208 i915_gem_retire_requests(dev);
2209
673a394b
EA		 2210 /* If there's an inactive buffer available now, grab it
2211 * and be done.
2212 */
07f73f69
CW		 2213 obj = i915_gem_find_inactive_object(dev, min_size);
2214 if (obj) {
2215 struct drm_i915_gem_object *obj_priv;
2216
673a394b
EA		 2217#if WATCH_LRU
2218 DRM_INFO("%s: evicting %p\n", __func__, obj);
2219#endif
23010e43 2220 obj_priv = to_intel_bo(obj);
07f73f69 2221 BUG_ON(obj_priv->pin_count != 0);
673a394b
EA		 2222 BUG_ON(obj_priv->active);
2223
2224 /* Wait on the rendering and unbind the buffer. */
07f73f69 2225 return i915_gem_object_unbind(obj);
673a394b
EA		 2226 }
2227
2228 /* If we didn't get anything, but the ring is still processing
07f73f69
CW		 2229 * things, wait for the next to finish and hopefully leave us
2230 * a buffer to evict.
673a394b
EA		 2231 */
2232 if (!list_empty(&dev_priv->mm.request_list)) {
2233 struct drm_i915_gem_request *request;
2234
2235 request = list_first_entry(&dev_priv->mm.request_list,
2236 struct drm_i915_gem_request,
2237 list);
2238
2239 ret = i915_wait_request(dev, request->seqno);
2240 if (ret)
07f73f69 2241 return ret;
673a394b 2242
07f73f69 2243 continue;
673a394b
EA		 2244 }
2245
2246 /* If we didn't have anything on the request list but there
2247 * are buffers awaiting a flush, emit one and try again.
2248 * When we wait on it, those buffers waiting for that flush
2249 * will get moved to inactive.
2250 */
2251 if (!list_empty(&dev_priv->mm.flushing_list)) {
07f73f69 2252 struct drm_i915_gem_object *obj_priv;
673a394b 2253
9a1e2582
CW		 2254 /* Find an object that we can immediately reuse */
2255 list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, list) {
a8089e84 2256 obj = &obj_priv->base;
9a1e2582
CW		 2257 if (obj->size >= min_size)
2258 break;
673a394b 2259
9a1e2582
CW		 2260 obj = NULL;
2261 }
673a394b 2262
9a1e2582
CW		 2263 if (obj != NULL) {
2264 uint32_t seqno;
673a394b 2265
9a1e2582
CW		 2266 i915_gem_flush(dev,
2267 obj->write_domain,
2268 obj->write_domain);
2269 seqno = i915_add_request(dev, NULL, obj->write_domain);
2270 if (seqno == 0)
2271 return -ENOMEM;
9a1e2582
CW		 2272 continue;
2273 }
673a394b
EA		 2274 }
2275
07f73f69
CW		 2276 /* If we didn't do any of the above, there's no single buffer
2277 * large enough to swap out for the new one, so just evict
2278 * everything and start again. (This should be rare.)
673a394b 2279 */
9731129c 2280 if (!list_empty (&dev_priv->mm.inactive_list))
ab5ee576 2281 return i915_gem_evict_from_inactive_list(dev);
9731129c 2282 else
07f73f69 2283 return i915_gem_evict_everything(dev);
ac94a962 2284 }
ac94a962
KP		 2285}
2286
6911a9b8 2287int
4bdadb97
CW		 2288i915_gem_object_get_pages(struct drm_gem_object *obj,
2289 gfp_t gfpmask)
673a394b 2290{
23010e43 2291 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 2292 int page_count, i;
2293 struct address_space *mapping;
2294 struct inode *inode;
2295 struct page *page;
673a394b 2296
856fa198 2297 if (obj_priv->pages_refcount++ != 0)
673a394b
EA		 2298 return 0;
2299
2300 /* Get the list of pages out of our struct file. They'll be pinned
2301 * at this point until we release them.
2302 */
2303 page_count = obj->size / PAGE_SIZE;
856fa198 2304 BUG_ON(obj_priv->pages != NULL);
8e7d2b2c 2305 obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
856fa198 2306 if (obj_priv->pages == NULL) {
856fa198 2307 obj_priv->pages_refcount--;
673a394b
EA		 2308 return -ENOMEM;
2309 }
2310
2311 inode = obj->filp->f_path.dentry->d_inode;
2312 mapping = inode->i_mapping;
2313 for (i = 0; i < page_count; i++) {
4bdadb97
CW		 2314 page = read_cache_page_gfp(mapping, i,
2315 mapping_gfp_mask (mapping) |
2316 __GFP_COLD |
2317 gfpmask);
1f2b1013
CW		 2318 if (IS_ERR(page))
2319 goto err_pages;
2320
856fa198 2321 obj_priv->pages[i] = page;
673a394b 2322 }
280b713b
EA		 2323
2324 if (obj_priv->tiling_mode != I915_TILING_NONE)
2325 i915_gem_object_do_bit_17_swizzle(obj);
2326
673a394b 2327 return 0;
1f2b1013
CW		 2328
2329err_pages:
2330 while (i--)
2331 page_cache_release(obj_priv->pages[i]);
2332
2333 drm_free_large(obj_priv->pages);
2334 obj_priv->pages = NULL;
2335 obj_priv->pages_refcount--;
2336 return PTR_ERR(page);
673a394b
EA		 2337}
2338
4e901fdc
EA		 2339static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2340{
2341 struct drm_gem_object *obj = reg->obj;
2342 struct drm_device *dev = obj->dev;
2343 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2344 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4e901fdc
EA		 2345 int regnum = obj_priv->fence_reg;
2346 uint64_t val;
2347
2348 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2349 0xfffff000) << 32;
2350 val |= obj_priv->gtt_offset & 0xfffff000;
2351 val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2352 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2353
2354 if (obj_priv->tiling_mode == I915_TILING_Y)
2355 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2356 val |= I965_FENCE_REG_VALID;
2357
2358 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2359}
2360
de151cf6
JB		 2361static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2362{
2363 struct drm_gem_object *obj = reg->obj;
2364 struct drm_device *dev = obj->dev;
2365 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2366 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6
JB		 2367 int regnum = obj_priv->fence_reg;
2368 uint64_t val;
2369
2370 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2371 0xfffff000) << 32;
2372 val |= obj_priv->gtt_offset & 0xfffff000;
2373 val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2374 if (obj_priv->tiling_mode == I915_TILING_Y)
2375 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2376 val |= I965_FENCE_REG_VALID;
2377
2378 I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2379}
2380
2381static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2382{
2383 struct drm_gem_object *obj = reg->obj;
2384 struct drm_device *dev = obj->dev;
2385 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2386 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6 2387 int regnum = obj_priv->fence_reg;
0f973f27 2388 int tile_width;
dc529a4f 2389 uint32_t fence_reg, val;
de151cf6
JB		 2390 uint32_t pitch_val;
2391
2392 if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2393 (obj_priv->gtt_offset & (obj->size - 1))) {
f06da264 2394 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
0f973f27 2395 __func__, obj_priv->gtt_offset, obj->size);
de151cf6
JB		 2396 return;
2397 }
2398
0f973f27
JB		 2399 if (obj_priv->tiling_mode == I915_TILING_Y &&
2400 HAS_128_BYTE_Y_TILING(dev))
2401 tile_width = 128;
de151cf6 2402 else
0f973f27
JB		 2403 tile_width = 512;
2404
2405 /* Note: pitch better be a power of two tile widths */
2406 pitch_val = obj_priv->stride / tile_width;
2407 pitch_val = ffs(pitch_val) - 1;
de151cf6 2408
c36a2a6d
DV		 2409 if (obj_priv->tiling_mode == I915_TILING_Y &&
2410 HAS_128_BYTE_Y_TILING(dev))
2411 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2412 else
2413 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2414
de151cf6
JB		 2415 val = obj_priv->gtt_offset;
2416 if (obj_priv->tiling_mode == I915_TILING_Y)
2417 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2418 val |= I915_FENCE_SIZE_BITS(obj->size);
2419 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2420 val |= I830_FENCE_REG_VALID;
2421
dc529a4f
EA		 2422 if (regnum < 8)
2423 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2424 else
2425 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2426 I915_WRITE(fence_reg, val);
de151cf6
JB		 2427}
2428
2429static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2430{
2431 struct drm_gem_object *obj = reg->obj;
2432 struct drm_device *dev = obj->dev;
2433 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2434 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6
JB		 2435 int regnum = obj_priv->fence_reg;
2436 uint32_t val;
2437 uint32_t pitch_val;
8d7773a3 2438 uint32_t fence_size_bits;
de151cf6 2439
8d7773a3 2440 if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
de151cf6 2441 (obj_priv->gtt_offset & (obj->size - 1))) {
8d7773a3 2442 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
0f973f27 2443 __func__, obj_priv->gtt_offset);
de151cf6
JB		 2444 return;
2445 }
2446
e76a16de
EA		 2447 pitch_val = obj_priv->stride / 128;
2448 pitch_val = ffs(pitch_val) - 1;
2449 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2450
de151cf6
JB		 2451 val = obj_priv->gtt_offset;
2452 if (obj_priv->tiling_mode == I915_TILING_Y)
2453 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
8d7773a3
DV		 2454 fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2455 WARN_ON(fence_size_bits & ~0x00000f00);
2456 val |= fence_size_bits;
de151cf6
JB		 2457 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2458 val |= I830_FENCE_REG_VALID;
2459
2460 I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
de151cf6
JB		 2461}
2462
ae3db24a
DV		 2463static int i915_find_fence_reg(struct drm_device *dev)
2464{
2465 struct drm_i915_fence_reg *reg = NULL;
2466 struct drm_i915_gem_object *obj_priv = NULL;
2467 struct drm_i915_private *dev_priv = dev->dev_private;
2468 struct drm_gem_object *obj = NULL;
2469 int i, avail, ret;
2470
2471 /* First try to find a free reg */
2472 avail = 0;
2473 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2474 reg = &dev_priv->fence_regs[i];
2475 if (!reg->obj)
2476 return i;
2477
23010e43 2478 obj_priv = to_intel_bo(reg->obj);
ae3db24a
DV		 2479 if (!obj_priv->pin_count)
2480 avail++;
2481 }
2482
2483 if (avail == 0)
2484 return -ENOSPC;
2485
2486 /* None available, try to steal one or wait for a user to finish */
2487 i = I915_FENCE_REG_NONE;
2488 list_for_each_entry(obj_priv, &dev_priv->mm.fence_list,
2489 fence_list) {
a8089e84 2490 obj = &obj_priv->base;
ae3db24a
DV		 2491
2492 if (obj_priv->pin_count)
2493 continue;
2494
2495 /* found one! */
2496 i = obj_priv->fence_reg;
2497 break;
2498 }
2499
2500 BUG_ON(i == I915_FENCE_REG_NONE);
2501
2502 /* We only have a reference on obj from the active list. put_fence_reg
2503 * might drop that one, causing a use-after-free in it. So hold a
2504 * private reference to obj like the other callers of put_fence_reg
2505 * (set_tiling ioctl) do. */
2506 drm_gem_object_reference(obj);
2507 ret = i915_gem_object_put_fence_reg(obj);
2508 drm_gem_object_unreference(obj);
2509 if (ret != 0)
2510 return ret;
2511
2512 return i;
2513}
2514
de151cf6
JB		 2515/**
2516 * i915_gem_object_get_fence_reg - set up a fence reg for an object
2517 * @obj: object to map through a fence reg
2518 *
2519 * When mapping objects through the GTT, userspace wants to be able to write
2520 * to them without having to worry about swizzling if the object is tiled.
2521 *
2522 * This function walks the fence regs looking for a free one for @obj,
2523 * stealing one if it can't find any.
2524 *
2525 * It then sets up the reg based on the object's properties: address, pitch
2526 * and tiling format.
2527 */
8c4b8c3f
CW		 2528int
2529i915_gem_object_get_fence_reg(struct drm_gem_object *obj)
de151cf6
JB		 2530{
2531 struct drm_device *dev = obj->dev;
79e53945 2532 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 2533 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6 2534 struct drm_i915_fence_reg *reg = NULL;
ae3db24a 2535 int ret;
de151cf6 2536
a09ba7fa
EA		 2537 /* Just update our place in the LRU if our fence is getting used. */
2538 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2539 list_move_tail(&obj_priv->fence_list, &dev_priv->mm.fence_list);
2540 return 0;
2541 }
2542
de151cf6
JB		 2543 switch (obj_priv->tiling_mode) {
2544 case I915_TILING_NONE:
2545 WARN(1, "allocating a fence for non-tiled object?\n");
2546 break;
2547 case I915_TILING_X:
0f973f27
JB		 2548 if (!obj_priv->stride)
2549 return -EINVAL;
2550 WARN((obj_priv->stride & (512 - 1)),
2551 "object 0x%08x is X tiled but has non-512B pitch\n",
2552 obj_priv->gtt_offset);
de151cf6
JB		 2553 break;
2554 case I915_TILING_Y:
0f973f27
JB		 2555 if (!obj_priv->stride)
2556 return -EINVAL;
2557 WARN((obj_priv->stride & (128 - 1)),
2558 "object 0x%08x is Y tiled but has non-128B pitch\n",
2559 obj_priv->gtt_offset);
de151cf6
JB		 2560 break;
2561 }
2562
ae3db24a
DV		 2563 ret = i915_find_fence_reg(dev);
2564 if (ret < 0)
2565 return ret;
de151cf6 2566
ae3db24a
DV		 2567 obj_priv->fence_reg = ret;
2568 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
a09ba7fa
EA		 2569 list_add_tail(&obj_priv->fence_list, &dev_priv->mm.fence_list);
2570
de151cf6
JB		 2571 reg->obj = obj;
2572
4e901fdc
EA		 2573 if (IS_GEN6(dev))
2574 sandybridge_write_fence_reg(reg);
2575 else if (IS_I965G(dev))
de151cf6
JB		 2576 i965_write_fence_reg(reg);
2577 else if (IS_I9XX(dev))
2578 i915_write_fence_reg(reg);
2579 else
2580 i830_write_fence_reg(reg);
d9ddcb96 2581
ae3db24a
DV		 2582 trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
2583 obj_priv->tiling_mode);
1c5d22f7 2584
d9ddcb96 2585 return 0;
de151cf6
JB		 2586}
2587
2588/**
2589 * i915_gem_clear_fence_reg - clear out fence register info
2590 * @obj: object to clear
2591 *
2592 * Zeroes out the fence register itself and clears out the associated
2593 * data structures in dev_priv and obj_priv.
2594 */
2595static void
2596i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2597{
2598 struct drm_device *dev = obj->dev;
79e53945 2599 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2600 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6 2601
4e901fdc
EA		 2602 if (IS_GEN6(dev)) {
2603 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2604 (obj_priv->fence_reg * 8), 0);
2605 } else if (IS_I965G(dev)) {
de151cf6 2606 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
4e901fdc 2607 } else {
dc529a4f
EA		 2608 uint32_t fence_reg;
2609
2610 if (obj_priv->fence_reg < 8)
2611 fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2612 else
2613 fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg -
2614 8) * 4;
2615
2616 I915_WRITE(fence_reg, 0);
2617 }
de151cf6
JB		 2618
2619 dev_priv->fence_regs[obj_priv->fence_reg].obj = NULL;
2620 obj_priv->fence_reg = I915_FENCE_REG_NONE;
a09ba7fa 2621 list_del_init(&obj_priv->fence_list);
de151cf6
JB		 2622}
2623
52dc7d32
CW		 2624/**
2625 * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2626 * to the buffer to finish, and then resets the fence register.
2627 * @obj: tiled object holding a fence register.
2628 *
2629 * Zeroes out the fence register itself and clears out the associated
2630 * data structures in dev_priv and obj_priv.
2631 */
2632int
2633i915_gem_object_put_fence_reg(struct drm_gem_object *obj)
2634{
2635 struct drm_device *dev = obj->dev;
23010e43 2636 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
52dc7d32
CW		 2637
2638 if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2639 return 0;
2640
10ae9bd2
DV		 2641 /* If we've changed tiling, GTT-mappings of the object
2642 * need to re-fault to ensure that the correct fence register
2643 * setup is in place.
2644 */
2645 i915_gem_release_mmap(obj);
2646
52dc7d32
CW		 2647 /* On the i915, GPU access to tiled buffers is via a fence,
2648 * therefore we must wait for any outstanding access to complete
2649 * before clearing the fence.
2650 */
2651 if (!IS_I965G(dev)) {
2652 int ret;
2653
2654 i915_gem_object_flush_gpu_write_domain(obj);
52dc7d32
CW		 2655 ret = i915_gem_object_wait_rendering(obj);
2656 if (ret != 0)
2657 return ret;
2658 }
2659
4a726612 2660 i915_gem_object_flush_gtt_write_domain(obj);
52dc7d32
CW		 2661 i915_gem_clear_fence_reg (obj);
2662
2663 return 0;
2664}
2665
673a394b
EA		 2666/**
2667 * Finds free space in the GTT aperture and binds the object there.
2668 */
2669static int
2670i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2671{
2672 struct drm_device *dev = obj->dev;
2673 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2674 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b 2675 struct drm_mm_node *free_space;
4bdadb97 2676 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
07f73f69 2677 int ret;
673a394b 2678
bb6baf76 2679 if (obj_priv->madv != I915_MADV_WILLNEED) {
3ef94daa
CW		 2680 DRM_ERROR("Attempting to bind a purgeable object\n");
2681 return -EINVAL;
2682 }
2683
673a394b 2684 if (alignment == 0)
0f973f27 2685 alignment = i915_gem_get_gtt_alignment(obj);
8d7773a3 2686 if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
673a394b
EA		 2687 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2688 return -EINVAL;
2689 }
2690
2691 search_free:
2692 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2693 obj->size, alignment, 0);
2694 if (free_space != NULL) {
2695 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2696 alignment);
2697 if (obj_priv->gtt_space != NULL) {
2698 obj_priv->gtt_space->private = obj;
2699 obj_priv->gtt_offset = obj_priv->gtt_space->start;
2700 }
2701 }
2702 if (obj_priv->gtt_space == NULL) {
2703 /* If the gtt is empty and we're still having trouble
2704 * fitting our object in, we're out of memory.
2705 */
2706#if WATCH_LRU
2707 DRM_INFO("%s: GTT full, evicting something\n", __func__);
2708#endif
07f73f69 2709 ret = i915_gem_evict_something(dev, obj->size);
9731129c 2710 if (ret)
673a394b 2711 return ret;
9731129c 2712
673a394b
EA		 2713 goto search_free;
2714 }
2715
2716#if WATCH_BUF
cfd43c02 2717 DRM_INFO("Binding object of size %zd at 0x%08x\n",
673a394b
EA		 2718 obj->size, obj_priv->gtt_offset);
2719#endif
4bdadb97 2720 ret = i915_gem_object_get_pages(obj, gfpmask);
673a394b
EA		 2721 if (ret) {
2722 drm_mm_put_block(obj_priv->gtt_space);
2723 obj_priv->gtt_space = NULL;
07f73f69
CW		 2724
2725 if (ret == -ENOMEM) {
2726 /* first try to clear up some space from the GTT */
2727 ret = i915_gem_evict_something(dev, obj->size);
2728 if (ret) {
07f73f69 2729 /* now try to shrink everyone else */
4bdadb97
CW		 2730 if (gfpmask) {
2731 gfpmask = 0;
2732 goto search_free;
07f73f69
CW		 2733 }
2734
2735 return ret;
2736 }
2737
2738 goto search_free;
2739 }
2740
673a394b
EA		 2741 return ret;
2742 }
2743
673a394b
EA		 2744 /* Create an AGP memory structure pointing at our pages, and bind it
2745 * into the GTT.
2746 */
2747 obj_priv->agp_mem = drm_agp_bind_pages(dev,
856fa198 2748 obj_priv->pages,
07f73f69 2749 obj->size >> PAGE_SHIFT,
ba1eb1d8
KP		 2750 obj_priv->gtt_offset,
2751 obj_priv->agp_type);
673a394b 2752 if (obj_priv->agp_mem == NULL) {
856fa198 2753 i915_gem_object_put_pages(obj);
673a394b
EA		 2754 drm_mm_put_block(obj_priv->gtt_space);
2755 obj_priv->gtt_space = NULL;
07f73f69
CW		 2756
2757 ret = i915_gem_evict_something(dev, obj->size);
9731129c 2758 if (ret)
07f73f69 2759 return ret;
07f73f69
CW		 2760
2761 goto search_free;
673a394b
EA		 2762 }
2763 atomic_inc(&dev->gtt_count);
2764 atomic_add(obj->size, &dev->gtt_memory);
2765
2766 /* Assert that the object is not currently in any GPU domain. As it
2767 * wasn't in the GTT, there shouldn't be any way it could have been in
2768 * a GPU cache
2769 */
21d509e3
CW		 2770 BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2771 BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
673a394b 2772
1c5d22f7
CW		 2773 trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2774
673a394b
EA		 2775 return 0;
2776}
2777
2778void
2779i915_gem_clflush_object(struct drm_gem_object *obj)
2780{
23010e43 2781 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 2782
2783 /* If we don't have a page list set up, then we're not pinned
2784 * to GPU, and we can ignore the cache flush because it'll happen
2785 * again at bind time.
2786 */
856fa198 2787 if (obj_priv->pages == NULL)
673a394b
EA		 2788 return;
2789
1c5d22f7 2790 trace_i915_gem_object_clflush(obj);
cfa16a0d 2791
856fa198 2792 drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
673a394b
EA		 2793}
2794
e47c68e9
EA		 2795/** Flushes any GPU write domain for the object if it's dirty. */
2796static void
2797i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
2798{
2799 struct drm_device *dev = obj->dev;
1c5d22f7 2800 uint32_t old_write_domain;
e47c68e9
EA		 2801
2802 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2803 return;
2804
2805 /* Queue the GPU write cache flushing we need. */
1c5d22f7 2806 old_write_domain = obj->write_domain;
e47c68e9 2807 i915_gem_flush(dev, 0, obj->write_domain);
922a2efc 2808 (void) i915_add_request(dev, NULL, obj->write_domain);
99fcb766 2809 BUG_ON(obj->write_domain);
1c5d22f7
CW		 2810
2811 trace_i915_gem_object_change_domain(obj,
2812 obj->read_domains,
2813 old_write_domain);
e47c68e9
EA		 2814}
2815
2816/** Flushes the GTT write domain for the object if it's dirty. */
2817static void
2818i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2819{
1c5d22f7
CW		 2820 uint32_t old_write_domain;
2821
e47c68e9
EA		 2822 if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2823 return;
2824
2825 /* No actual flushing is required for the GTT write domain. Writes
2826 * to it immediately go to main memory as far as we know, so there's
2827 * no chipset flush. It also doesn't land in render cache.
2828 */
1c5d22f7 2829 old_write_domain = obj->write_domain;
e47c68e9 2830 obj->write_domain = 0;
1c5d22f7
CW		 2831
2832 trace_i915_gem_object_change_domain(obj,
2833 obj->read_domains,
2834 old_write_domain);
e47c68e9
EA		 2835}
2836
2837/** Flushes the CPU write domain for the object if it's dirty. */
2838static void
2839i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2840{
2841 struct drm_device *dev = obj->dev;
1c5d22f7 2842 uint32_t old_write_domain;
e47c68e9
EA		 2843
2844 if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2845 return;
2846
2847 i915_gem_clflush_object(obj);
2848 drm_agp_chipset_flush(dev);
1c5d22f7 2849 old_write_domain = obj->write_domain;
e47c68e9 2850 obj->write_domain = 0;
1c5d22f7
CW		 2851
2852 trace_i915_gem_object_change_domain(obj,
2853 obj->read_domains,
2854 old_write_domain);
e47c68e9
EA		 2855}
2856
6b95a207
KH		 2857void
2858i915_gem_object_flush_write_domain(struct drm_gem_object *obj)
2859{
2860 switch (obj->write_domain) {
2861 case I915_GEM_DOMAIN_GTT:
2862 i915_gem_object_flush_gtt_write_domain(obj);
2863 break;
2864 case I915_GEM_DOMAIN_CPU:
2865 i915_gem_object_flush_cpu_write_domain(obj);
2866 break;
2867 default:
2868 i915_gem_object_flush_gpu_write_domain(obj);
2869 break;
2870 }
2871}
2872
2ef7eeaa
EA		 2873/**
2874 * Moves a single object to the GTT read, and possibly write domain.
2875 *
2876 * This function returns when the move is complete, including waiting on
2877 * flushes to occur.
2878 */
79e53945 2879int
2ef7eeaa
EA		 2880i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2881{
23010e43 2882 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1c5d22f7 2883 uint32_t old_write_domain, old_read_domains;
e47c68e9 2884 int ret;
2ef7eeaa 2885
02354392
EA		 2886 /* Not valid to be called on unbound objects. */
2887 if (obj_priv->gtt_space == NULL)
2888 return -EINVAL;
2889
e47c68e9
EA		 2890 i915_gem_object_flush_gpu_write_domain(obj);
2891 /* Wait on any GPU rendering and flushing to occur. */
2892 ret = i915_gem_object_wait_rendering(obj);
2893 if (ret != 0)
2894 return ret;
2895
1c5d22f7
CW		 2896 old_write_domain = obj->write_domain;
2897 old_read_domains = obj->read_domains;
2898
e47c68e9
EA		 2899 /* If we're writing through the GTT domain, then CPU and GPU caches
2900 * will need to be invalidated at next use.
2ef7eeaa 2901 */
e47c68e9
EA		 2902 if (write)
2903 obj->read_domains &= I915_GEM_DOMAIN_GTT;
2ef7eeaa 2904
e47c68e9 2905 i915_gem_object_flush_cpu_write_domain(obj);
2ef7eeaa 2906
e47c68e9
EA		 2907 /* It should now be out of any other write domains, and we can update
2908 * the domain values for our changes.
2909 */
2910 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2911 obj->read_domains |= I915_GEM_DOMAIN_GTT;
2912 if (write) {
2913 obj->write_domain = I915_GEM_DOMAIN_GTT;
2914 obj_priv->dirty = 1;
2ef7eeaa
EA		 2915 }
2916
1c5d22f7
CW		 2917 trace_i915_gem_object_change_domain(obj,
2918 old_read_domains,
2919 old_write_domain);
2920
e47c68e9
EA		 2921 return 0;
2922}
2923
b9241ea3
ZW		 2924/*
2925 * Prepare buffer for display plane. Use uninterruptible for possible flush
2926 * wait, as in modesetting process we're not supposed to be interrupted.
2927 */
2928int
2929i915_gem_object_set_to_display_plane(struct drm_gem_object *obj)
2930{
2931 struct drm_device *dev = obj->dev;
23010e43 2932 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
b9241ea3
ZW		 2933 uint32_t old_write_domain, old_read_domains;
2934 int ret;
2935
2936 /* Not valid to be called on unbound objects. */
2937 if (obj_priv->gtt_space == NULL)
2938 return -EINVAL;
2939
2940 i915_gem_object_flush_gpu_write_domain(obj);
2941
2942 /* Wait on any GPU rendering and flushing to occur. */
2943 if (obj_priv->active) {
2944#if WATCH_BUF
2945 DRM_INFO("%s: object %p wait for seqno %08x\n",
2946 __func__, obj, obj_priv->last_rendering_seqno);
2947#endif
2948 ret = i915_do_wait_request(dev, obj_priv->last_rendering_seqno, 0);
2949 if (ret != 0)
2950 return ret;
2951 }
2952
2953 old_write_domain = obj->write_domain;
2954 old_read_domains = obj->read_domains;
2955
2956 obj->read_domains &= I915_GEM_DOMAIN_GTT;
2957
2958 i915_gem_object_flush_cpu_write_domain(obj);
2959
2960 /* It should now be out of any other write domains, and we can update
2961 * the domain values for our changes.
2962 */
2963 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2964 obj->read_domains |= I915_GEM_DOMAIN_GTT;
2965 obj->write_domain = I915_GEM_DOMAIN_GTT;
2966 obj_priv->dirty = 1;
2967
2968 trace_i915_gem_object_change_domain(obj,
2969 old_read_domains,
2970 old_write_domain);
2971
2972 return 0;
2973}
2974
e47c68e9
EA		 2975/**
2976 * Moves a single object to the CPU read, and possibly write domain.
2977 *
2978 * This function returns when the move is complete, including waiting on
2979 * flushes to occur.
2980 */
2981static int
2982i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2983{
1c5d22f7 2984 uint32_t old_write_domain, old_read_domains;
e47c68e9
EA		 2985 int ret;
2986
2987 i915_gem_object_flush_gpu_write_domain(obj);
2ef7eeaa 2988 /* Wait on any GPU rendering and flushing to occur. */
e47c68e9
EA		 2989 ret = i915_gem_object_wait_rendering(obj);
2990 if (ret != 0)
2991 return ret;
2ef7eeaa 2992
e47c68e9 2993 i915_gem_object_flush_gtt_write_domain(obj);
2ef7eeaa 2994
e47c68e9
EA		 2995 /* If we have a partially-valid cache of the object in the CPU,
2996 * finish invalidating it and free the per-page flags.
2ef7eeaa 2997 */
e47c68e9 2998 i915_gem_object_set_to_full_cpu_read_domain(obj);
2ef7eeaa 2999
1c5d22f7
CW		 3000 old_write_domain = obj->write_domain;
3001 old_read_domains = obj->read_domains;
3002
e47c68e9
EA		 3003 /* Flush the CPU cache if it's still invalid. */
3004 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2ef7eeaa 3005 i915_gem_clflush_object(obj);
2ef7eeaa 3006
e47c68e9 3007 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2ef7eeaa
EA		 3008 }
3009
3010 /* It should now be out of any other write domains, and we can update
3011 * the domain values for our changes.
3012 */
e47c68e9
EA		 3013 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3014
3015 /* If we're writing through the CPU, then the GPU read domains will
3016 * need to be invalidated at next use.
3017 */
3018 if (write) {
3019 obj->read_domains &= I915_GEM_DOMAIN_CPU;
3020 obj->write_domain = I915_GEM_DOMAIN_CPU;
3021 }
2ef7eeaa 3022
1c5d22f7
CW		 3023 trace_i915_gem_object_change_domain(obj,
3024 old_read_domains,
3025 old_write_domain);
3026
2ef7eeaa
EA		 3027 return 0;
3028}
3029
673a394b
EA		 3030/*
3031 * Set the next domain for the specified object. This
3032 * may not actually perform the necessary flushing/invaliding though,
3033 * as that may want to be batched with other set_domain operations
3034 *
3035 * This is (we hope) the only really tricky part of gem. The goal
3036 * is fairly simple -- track which caches hold bits of the object
3037 * and make sure they remain coherent. A few concrete examples may
3038 * help to explain how it works. For shorthand, we use the notation
3039 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
3040 * a pair of read and write domain masks.
3041 *
3042 * Case 1: the batch buffer
3043 *
3044 * 1. Allocated
3045 * 2. Written by CPU
3046 * 3. Mapped to GTT
3047 * 4. Read by GPU
3048 * 5. Unmapped from GTT
3049 * 6. Freed
3050 *
3051 * Let's take these a step at a time
3052 *
3053 * 1. Allocated
3054 * Pages allocated from the kernel may still have
3055 * cache contents, so we set them to (CPU, CPU) always.
3056 * 2. Written by CPU (using pwrite)
3057 * The pwrite function calls set_domain (CPU, CPU) and
3058 * this function does nothing (as nothing changes)
3059 * 3. Mapped by GTT
3060 * This function asserts that the object is not
3061 * currently in any GPU-based read or write domains
3062 * 4. Read by GPU
3063 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
3064 * As write_domain is zero, this function adds in the
3065 * current read domains (CPU+COMMAND, 0).
3066 * flush_domains is set to CPU.
3067 * invalidate_domains is set to COMMAND
3068 * clflush is run to get data out of the CPU caches
3069 * then i915_dev_set_domain calls i915_gem_flush to
3070 * emit an MI_FLUSH and drm_agp_chipset_flush
3071 * 5. Unmapped from GTT
3072 * i915_gem_object_unbind calls set_domain (CPU, CPU)
3073 * flush_domains and invalidate_domains end up both zero
3074 * so no flushing/invalidating happens
3075 * 6. Freed
3076 * yay, done
3077 *
3078 * Case 2: The shared render buffer
3079 *
3080 * 1. Allocated
3081 * 2. Mapped to GTT
3082 * 3. Read/written by GPU
3083 * 4. set_domain to (CPU,CPU)
3084 * 5. Read/written by CPU
3085 * 6. Read/written by GPU
3086 *
3087 * 1. Allocated
3088 * Same as last example, (CPU, CPU)
3089 * 2. Mapped to GTT
3090 * Nothing changes (assertions find that it is not in the GPU)
3091 * 3. Read/written by GPU
3092 * execbuffer calls set_domain (RENDER, RENDER)
3093 * flush_domains gets CPU
3094 * invalidate_domains gets GPU
3095 * clflush (obj)
3096 * MI_FLUSH and drm_agp_chipset_flush
3097 * 4. set_domain (CPU, CPU)
3098 * flush_domains gets GPU
3099 * invalidate_domains gets CPU
3100 * wait_rendering (obj) to make sure all drawing is complete.
3101 * This will include an MI_FLUSH to get the data from GPU
3102 * to memory
3103 * clflush (obj) to invalidate the CPU cache
3104 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3105 * 5. Read/written by CPU
3106 * cache lines are loaded and dirtied
3107 * 6. Read written by GPU
3108 * Same as last GPU access
3109 *
3110 * Case 3: The constant buffer
3111 *
3112 * 1. Allocated
3113 * 2. Written by CPU
3114 * 3. Read by GPU
3115 * 4. Updated (written) by CPU again
3116 * 5. Read by GPU
3117 *
3118 * 1. Allocated
3119 * (CPU, CPU)
3120 * 2. Written by CPU
3121 * (CPU, CPU)
3122 * 3. Read by GPU
3123 * (CPU+RENDER, 0)
3124 * flush_domains = CPU
3125 * invalidate_domains = RENDER
3126 * clflush (obj)
3127 * MI_FLUSH
3128 * drm_agp_chipset_flush
3129 * 4. Updated (written) by CPU again
3130 * (CPU, CPU)
3131 * flush_domains = 0 (no previous write domain)
3132 * invalidate_domains = 0 (no new read domains)
3133 * 5. Read by GPU
3134 * (CPU+RENDER, 0)
3135 * flush_domains = CPU
3136 * invalidate_domains = RENDER
3137 * clflush (obj)
3138 * MI_FLUSH
3139 * drm_agp_chipset_flush
3140 */
c0d90829 3141static void
8b0e378a 3142i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
673a394b
EA		 3143{
3144 struct drm_device *dev = obj->dev;
23010e43 3145 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 3146 uint32_t invalidate_domains = 0;
3147 uint32_t flush_domains = 0;
1c5d22f7 3148 uint32_t old_read_domains;
e47c68e9 3149
8b0e378a
EA		 3150 BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
3151 BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
673a394b 3152
652c393a
JB		 3153 intel_mark_busy(dev, obj);
3154
673a394b
EA		 3155#if WATCH_BUF
3156 DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
3157 __func__, obj,
8b0e378a
EA		 3158 obj->read_domains, obj->pending_read_domains,
3159 obj->write_domain, obj->pending_write_domain);
673a394b
EA		 3160#endif
3161 /*
3162 * If the object isn't moving to a new write domain,
3163 * let the object stay in multiple read domains
3164 */
8b0e378a
EA		 3165 if (obj->pending_write_domain == 0)
3166 obj->pending_read_domains |= obj->read_domains;
673a394b
EA		 3167 else
3168 obj_priv->dirty = 1;
3169
3170 /*
3171 * Flush the current write domain if
3172 * the new read domains don't match. Invalidate
3173 * any read domains which differ from the old
3174 * write domain
3175 */
8b0e378a
EA		 3176 if (obj->write_domain &&
3177 obj->write_domain != obj->pending_read_domains) {
673a394b 3178 flush_domains |= obj->write_domain;
8b0e378a
EA		 3179 invalidate_domains |=
3180 obj->pending_read_domains & ~obj->write_domain;
673a394b
EA		 3181 }
3182 /*
3183 * Invalidate any read caches which may have
3184 * stale data. That is, any new read domains.
3185 */
8b0e378a 3186 invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
673a394b
EA		 3187 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
3188#if WATCH_BUF
3189 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
3190 __func__, flush_domains, invalidate_domains);
3191#endif
673a394b
EA		 3192 i915_gem_clflush_object(obj);
3193 }
3194
1c5d22f7
CW		 3195 old_read_domains = obj->read_domains;
3196
efbeed96
EA		 3197 /* The actual obj->write_domain will be updated with
3198 * pending_write_domain after we emit the accumulated flush for all
3199 * of our domain changes in execbuffers (which clears objects'
3200 * write_domains). So if we have a current write domain that we
3201 * aren't changing, set pending_write_domain to that.
3202 */
3203 if (flush_domains == 0 && obj->pending_write_domain == 0)
3204 obj->pending_write_domain = obj->write_domain;
8b0e378a 3205 obj->read_domains = obj->pending_read_domains;
673a394b
EA		 3206
3207 dev->invalidate_domains |= invalidate_domains;
3208 dev->flush_domains |= flush_domains;
3209#if WATCH_BUF
3210 DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
3211 __func__,
3212 obj->read_domains, obj->write_domain,
3213 dev->invalidate_domains, dev->flush_domains);
3214#endif
1c5d22f7
CW		 3215
3216 trace_i915_gem_object_change_domain(obj,
3217 old_read_domains,
3218 obj->write_domain);
673a394b
EA		 3219}
3220
3221/**
e47c68e9 3222 * Moves the object from a partially CPU read to a full one.
673a394b 3223 *
e47c68e9
EA		 3224 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3225 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
673a394b 3226 */
e47c68e9
EA		 3227static void
3228i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
673a394b 3229{
23010e43 3230 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b 3231
e47c68e9
EA		 3232 if (!obj_priv->page_cpu_valid)
3233 return;
3234
3235 /* If we're partially in the CPU read domain, finish moving it in.
3236 */
3237 if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3238 int i;
3239
3240 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3241 if (obj_priv->page_cpu_valid[i])
3242 continue;
856fa198 3243 drm_clflush_pages(obj_priv->pages + i, 1);
e47c68e9 3244 }
e47c68e9
EA		 3245 }
3246
3247 /* Free the page_cpu_valid mappings which are now stale, whether
3248 * or not we've got I915_GEM_DOMAIN_CPU.
3249 */
9a298b2a 3250 kfree(obj_priv->page_cpu_valid);
e47c68e9
EA		 3251 obj_priv->page_cpu_valid = NULL;
3252}
3253
3254/**
3255 * Set the CPU read domain on a range of the object.
3256 *
3257 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3258 * not entirely valid. The page_cpu_valid member of the object flags which
3259 * pages have been flushed, and will be respected by
3260 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3261 * of the whole object.
3262 *
3263 * This function returns when the move is complete, including waiting on
3264 * flushes to occur.
3265 */
3266static int
3267i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3268 uint64_t offset, uint64_t size)
3269{
23010e43 3270 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1c5d22f7 3271 uint32_t old_read_domains;
e47c68e9 3272 int i, ret;
673a394b 3273
e47c68e9
EA		 3274 if (offset == 0 && size == obj->size)
3275 return i915_gem_object_set_to_cpu_domain(obj, 0);
673a394b 3276
e47c68e9
EA		 3277 i915_gem_object_flush_gpu_write_domain(obj);
3278 /* Wait on any GPU rendering and flushing to occur. */
6a47baa6 3279 ret = i915_gem_object_wait_rendering(obj);
e47c68e9 3280 if (ret != 0)
6a47baa6 3281 return ret;
e47c68e9
EA		 3282 i915_gem_object_flush_gtt_write_domain(obj);
3283
3284 /* If we're already fully in the CPU read domain, we're done. */
3285 if (obj_priv->page_cpu_valid == NULL &&
3286 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3287 return 0;
673a394b 3288
e47c68e9
EA		 3289 /* Otherwise, create/clear the per-page CPU read domain flag if we're
3290 * newly adding I915_GEM_DOMAIN_CPU
3291 */
673a394b 3292 if (obj_priv->page_cpu_valid == NULL) {
9a298b2a
EA		 3293 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3294 GFP_KERNEL);
e47c68e9
EA		 3295 if (obj_priv->page_cpu_valid == NULL)
3296 return -ENOMEM;
3297 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3298 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
673a394b
EA		 3299
3300 /* Flush the cache on any pages that are still invalid from the CPU's
3301 * perspective.
3302 */
e47c68e9
EA		 3303 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3304 i++) {
673a394b
EA		 3305 if (obj_priv->page_cpu_valid[i])
3306 continue;
3307
856fa198 3308 drm_clflush_pages(obj_priv->pages + i, 1);
673a394b
EA		 3309
3310 obj_priv->page_cpu_valid[i] = 1;
3311 }
3312
e47c68e9
EA		 3313 /* It should now be out of any other write domains, and we can update
3314 * the domain values for our changes.
3315 */
3316 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3317
1c5d22f7 3318 old_read_domains = obj->read_domains;
e47c68e9
EA		 3319 obj->read_domains |= I915_GEM_DOMAIN_CPU;
3320
1c5d22f7
CW		 3321 trace_i915_gem_object_change_domain(obj,
3322 old_read_domains,
3323 obj->write_domain);
3324
673a394b
EA		 3325 return 0;
3326}
3327
673a394b
EA		 3328/**
3329 * Pin an object to the GTT and evaluate the relocations landing in it.
3330 */
3331static int
3332i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3333 struct drm_file *file_priv,
76446cac 3334 struct drm_i915_gem_exec_object2 *entry,
40a5f0de 3335 struct drm_i915_gem_relocation_entry *relocs)
673a394b
EA		 3336{
3337 struct drm_device *dev = obj->dev;
0839ccb8 3338 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 3339 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b 3340 int i, ret;
0839ccb8 3341 void __iomem *reloc_page;
76446cac
JB		 3342 bool need_fence;
3343
3344 need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3345 obj_priv->tiling_mode != I915_TILING_NONE;
3346
3347 /* Check fence reg constraints and rebind if necessary */
f590d279
OA		 3348 if (need_fence && !i915_gem_object_fence_offset_ok(obj,
3349 obj_priv->tiling_mode))
76446cac 3350 i915_gem_object_unbind(obj);
673a394b
EA		 3351
3352 /* Choose the GTT offset for our buffer and put it there. */
3353 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
3354 if (ret)
3355 return ret;
3356
76446cac
JB		 3357 /*
3358 * Pre-965 chips need a fence register set up in order to
3359 * properly handle blits to/from tiled surfaces.
3360 */
3361 if (need_fence) {
3362 ret = i915_gem_object_get_fence_reg(obj);
3363 if (ret != 0) {
3364 if (ret != -EBUSY && ret != -ERESTARTSYS)
3365 DRM_ERROR("Failure to install fence: %d\n",
3366 ret);
3367 i915_gem_object_unpin(obj);
3368 return ret;
3369 }
3370 }
3371
673a394b
EA		 3372 entry->offset = obj_priv->gtt_offset;
3373
673a394b
EA		 3374 /* Apply the relocations, using the GTT aperture to avoid cache
3375 * flushing requirements.
3376 */
3377 for (i = 0; i < entry->relocation_count; i++) {
40a5f0de 3378 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
673a394b
EA		 3379 struct drm_gem_object *target_obj;
3380 struct drm_i915_gem_object *target_obj_priv;
3043c60c
EA		 3381 uint32_t reloc_val, reloc_offset;
3382 uint32_t __iomem *reloc_entry;
673a394b 3383
673a394b 3384 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
40a5f0de 3385 reloc->target_handle);
673a394b
EA		 3386 if (target_obj == NULL) {
3387 i915_gem_object_unpin(obj);
3388 return -EBADF;
3389 }
23010e43 3390 target_obj_priv = to_intel_bo(target_obj);
673a394b 3391
8542a0bb
CW		 3392#if WATCH_RELOC
3393 DRM_INFO("%s: obj %p offset %08x target %d "
3394 "read %08x write %08x gtt %08x "
3395 "presumed %08x delta %08x\n",
3396 __func__,
3397 obj,
3398 (int) reloc->offset,
3399 (int) reloc->target_handle,
3400 (int) reloc->read_domains,
3401 (int) reloc->write_domain,
3402 (int) target_obj_priv->gtt_offset,
3403 (int) reloc->presumed_offset,
3404 reloc->delta);
3405#endif
3406
673a394b
EA		 3407 /* The target buffer should have appeared before us in the
3408 * exec_object list, so it should have a GTT space bound by now.
3409 */
3410 if (target_obj_priv->gtt_space == NULL) {
3411 DRM_ERROR("No GTT space found for object %d\n",
40a5f0de 3412 reloc->target_handle);
673a394b
EA		 3413 drm_gem_object_unreference(target_obj);
3414 i915_gem_object_unpin(obj);
3415 return -EINVAL;
3416 }
3417
8542a0bb 3418 /* Validate that the target is in a valid r/w GPU domain */
16edd550
DV		 3419 if (reloc->write_domain & (reloc->write_domain - 1)) {
3420 DRM_ERROR("reloc with multiple write domains: "
3421 "obj %p target %d offset %d "
3422 "read %08x write %08x",
3423 obj, reloc->target_handle,
3424 (int) reloc->offset,
3425 reloc->read_domains,
3426 reloc->write_domain);
3427 return -EINVAL;
3428 }
40a5f0de
EA		 3429 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3430 reloc->read_domains & I915_GEM_DOMAIN_CPU) {
e47c68e9
EA		 3431 DRM_ERROR("reloc with read/write CPU domains: "
3432 "obj %p target %d offset %d "
3433 "read %08x write %08x",
40a5f0de
EA		 3434 obj, reloc->target_handle,
3435 (int) reloc->offset,
3436 reloc->read_domains,
3437 reloc->write_domain);
491152b8
CW		 3438 drm_gem_object_unreference(target_obj);
3439 i915_gem_object_unpin(obj);
e47c68e9
EA		 3440 return -EINVAL;
3441 }
40a5f0de
EA		 3442 if (reloc->write_domain && target_obj->pending_write_domain &&
3443 reloc->write_domain != target_obj->pending_write_domain) {
673a394b
EA		 3444 DRM_ERROR("Write domain conflict: "
3445 "obj %p target %d offset %d "
3446 "new %08x old %08x\n",
40a5f0de
EA		 3447 obj, reloc->target_handle,
3448 (int) reloc->offset,
3449 reloc->write_domain,
673a394b
EA		 3450 target_obj->pending_write_domain);
3451 drm_gem_object_unreference(target_obj);
3452 i915_gem_object_unpin(obj);
3453 return -EINVAL;
3454 }
3455
40a5f0de
EA		 3456 target_obj->pending_read_domains |= reloc->read_domains;
3457 target_obj->pending_write_domain |= reloc->write_domain;
673a394b
EA		 3458
3459 /* If the relocation already has the right value in it, no
3460 * more work needs to be done.
3461 */
40a5f0de 3462 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
673a394b
EA		 3463 drm_gem_object_unreference(target_obj);
3464 continue;
3465 }
3466
8542a0bb
CW		 3467 /* Check that the relocation address is valid... */
3468 if (reloc->offset > obj->size - 4) {
3469 DRM_ERROR("Relocation beyond object bounds: "
3470 "obj %p target %d offset %d size %d.\n",
3471 obj, reloc->target_handle,
3472 (int) reloc->offset, (int) obj->size);
3473 drm_gem_object_unreference(target_obj);
3474 i915_gem_object_unpin(obj);
3475 return -EINVAL;
3476 }
3477 if (reloc->offset & 3) {
3478 DRM_ERROR("Relocation not 4-byte aligned: "
3479 "obj %p target %d offset %d.\n",
3480 obj, reloc->target_handle,
3481 (int) reloc->offset);
3482 drm_gem_object_unreference(target_obj);
3483 i915_gem_object_unpin(obj);
3484 return -EINVAL;
3485 }
3486
3487 /* and points to somewhere within the target object. */
3488 if (reloc->delta >= target_obj->size) {
3489 DRM_ERROR("Relocation beyond target object bounds: "
3490 "obj %p target %d delta %d size %d.\n",
3491 obj, reloc->target_handle,
3492 (int) reloc->delta, (int) target_obj->size);
3493 drm_gem_object_unreference(target_obj);
3494 i915_gem_object_unpin(obj);
3495 return -EINVAL;
3496 }
3497
2ef7eeaa
EA		 3498 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3499 if (ret != 0) {
3500 drm_gem_object_unreference(target_obj);
3501 i915_gem_object_unpin(obj);
3502 return -EINVAL;
673a394b
EA		 3503 }
3504
3505 /* Map the page containing the relocation we're going to
3506 * perform.
3507 */
40a5f0de 3508 reloc_offset = obj_priv->gtt_offset + reloc->offset;
0839ccb8
KP		 3509 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3510 (reloc_offset &
3511 ~(PAGE_SIZE - 1)));
3043c60c 3512 reloc_entry = (uint32_t __iomem *)(reloc_page +
0839ccb8 3513 (reloc_offset & (PAGE_SIZE - 1)));
40a5f0de 3514 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
673a394b
EA		 3515
3516#if WATCH_BUF
3517 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
40a5f0de 3518 obj, (unsigned int) reloc->offset,
673a394b
EA		 3519 readl(reloc_entry), reloc_val);
3520#endif
3521 writel(reloc_val, reloc_entry);
0839ccb8 3522 io_mapping_unmap_atomic(reloc_page);
673a394b 3523
40a5f0de
EA		 3524 /* The updated presumed offset for this entry will be
3525 * copied back out to the user.
673a394b 3526 */
40a5f0de 3527 reloc->presumed_offset = target_obj_priv->gtt_offset;
673a394b
EA		 3528
3529 drm_gem_object_unreference(target_obj);
3530 }
3531
673a394b
EA		 3532#if WATCH_BUF
3533 if (0)
3534 i915_gem_dump_object(obj, 128, __func__, ~0);
3535#endif
3536 return 0;
3537}
3538
3539/** Dispatch a batchbuffer to the ring
3540 */
3541static int
3542i915_dispatch_gem_execbuffer(struct drm_device *dev,
76446cac 3543 struct drm_i915_gem_execbuffer2 *exec,
201361a5 3544 struct drm_clip_rect *cliprects,
673a394b
EA		 3545 uint64_t exec_offset)
3546{
3547 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b
EA		 3548 int nbox = exec->num_cliprects;
3549 int i = 0, count;
83d60795 3550 uint32_t exec_start, exec_len;
673a394b
EA		 3551 RING_LOCALS;
3552
3553 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3554 exec_len = (uint32_t) exec->batch_len;
3555
8f0dc5bf 3556 trace_i915_gem_request_submit(dev, dev_priv->mm.next_gem_seqno + 1);
1c5d22f7 3557
673a394b
EA		 3558 count = nbox ? nbox : 1;
3559
3560 for (i = 0; i < count; i++) {
3561 if (i < nbox) {
201361a5 3562 int ret = i915_emit_box(dev, cliprects, i,
673a394b
EA		 3563 exec->DR1, exec->DR4);
3564 if (ret)
3565 return ret;
3566 }
3567
3568 if (IS_I830(dev) || IS_845G(dev)) {
3569 BEGIN_LP_RING(4);
3570 OUT_RING(MI_BATCH_BUFFER);
3571 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
3572 OUT_RING(exec_start + exec_len - 4);
3573 OUT_RING(0);
3574 ADVANCE_LP_RING();
3575 } else {
3576 BEGIN_LP_RING(2);
3577 if (IS_I965G(dev)) {
3578 OUT_RING(MI_BATCH_BUFFER_START |
3579 (2 << 6) |
3580 MI_BATCH_NON_SECURE_I965);
3581 OUT_RING(exec_start);
3582 } else {
3583 OUT_RING(MI_BATCH_BUFFER_START |
3584 (2 << 6));
3585 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
3586 }
3587 ADVANCE_LP_RING();
3588 }
3589 }
3590
3591 /* XXX breadcrumb */
3592 return 0;
3593}
3594
3595/* Throttle our rendering by waiting until the ring has completed our requests
3596 * emitted over 20 msec ago.
3597 *
b962442e
EA		 3598 * Note that if we were to use the current jiffies each time around the loop,
3599 * we wouldn't escape the function with any frames outstanding if the time to
3600 * render a frame was over 20ms.
3601 *
673a394b
EA		 3602 * This should get us reasonable parallelism between CPU and GPU but also
3603 * relatively low latency when blocking on a particular request to finish.
3604 */
3605static int
3606i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
3607{
3608 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
3609 int ret = 0;
b962442e 3610 unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
673a394b
EA		 3611
3612 mutex_lock(&dev->struct_mutex);
b962442e
EA		 3613 while (!list_empty(&i915_file_priv->mm.request_list)) {
3614 struct drm_i915_gem_request *request;
3615
3616 request = list_first_entry(&i915_file_priv->mm.request_list,
3617 struct drm_i915_gem_request,
3618 client_list);
3619
3620 if (time_after_eq(request->emitted_jiffies, recent_enough))
3621 break;
3622
3623 ret = i915_wait_request(dev, request->seqno);
3624 if (ret != 0)
3625 break;
3626 }
673a394b 3627 mutex_unlock(&dev->struct_mutex);
b962442e 3628
673a394b
EA		 3629 return ret;
3630}
3631
40a5f0de 3632static int
76446cac 3633i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
40a5f0de
EA		 3634 uint32_t buffer_count,
3635 struct drm_i915_gem_relocation_entry **relocs)
3636{
3637 uint32_t reloc_count = 0, reloc_index = 0, i;
3638 int ret;
3639
3640 *relocs = NULL;
3641 for (i = 0; i < buffer_count; i++) {
3642 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3643 return -EINVAL;
3644 reloc_count += exec_list[i].relocation_count;
3645 }
3646
8e7d2b2c 3647 *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
76446cac
JB		 3648 if (*relocs == NULL) {
3649 DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
40a5f0de 3650 return -ENOMEM;
76446cac 3651 }
40a5f0de
EA		 3652
3653 for (i = 0; i < buffer_count; i++) {
3654 struct drm_i915_gem_relocation_entry __user *user_relocs;
3655
3656 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3657
3658 ret = copy_from_user(&(*relocs)[reloc_index],
3659 user_relocs,
3660 exec_list[i].relocation_count *
3661 sizeof(**relocs));
3662 if (ret != 0) {
8e7d2b2c 3663 drm_free_large(*relocs);
40a5f0de 3664 *relocs = NULL;
2bc43b5c 3665 return -EFAULT;
40a5f0de
EA		 3666 }
3667
3668 reloc_index += exec_list[i].relocation_count;
3669 }
3670
2bc43b5c 3671 return 0;
40a5f0de
EA		 3672}
3673
3674static int
76446cac 3675i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
40a5f0de
EA		 3676 uint32_t buffer_count,
3677 struct drm_i915_gem_relocation_entry *relocs)
3678{
3679 uint32_t reloc_count = 0, i;
2bc43b5c 3680 int ret = 0;
40a5f0de 3681
93533c29
CW		 3682 if (relocs == NULL)
3683 return 0;
3684
40a5f0de
EA		 3685 for (i = 0; i < buffer_count; i++) {
3686 struct drm_i915_gem_relocation_entry __user *user_relocs;
2bc43b5c 3687 int unwritten;
40a5f0de
EA		 3688
3689 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3690
2bc43b5c
FM		 3691 unwritten = copy_to_user(user_relocs,
3692 &relocs[reloc_count],
3693 exec_list[i].relocation_count *
3694 sizeof(*relocs));
3695
3696 if (unwritten) {
3697 ret = -EFAULT;
3698 goto err;
40a5f0de
EA		 3699 }
3700
3701 reloc_count += exec_list[i].relocation_count;
3702 }
3703
2bc43b5c 3704err:
8e7d2b2c 3705 drm_free_large(relocs);
40a5f0de
EA		 3706
3707 return ret;
3708}
3709
83d60795 3710static int
76446cac 3711i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
83d60795
CW		 3712 uint64_t exec_offset)
3713{
3714 uint32_t exec_start, exec_len;
3715
3716 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3717 exec_len = (uint32_t) exec->batch_len;
3718
3719 if ((exec_start | exec_len) & 0x7)
3720 return -EINVAL;
3721
3722 if (!exec_start)
3723 return -EINVAL;
3724
3725 return 0;
3726}
3727
6b95a207
KH		 3728static int
3729i915_gem_wait_for_pending_flip(struct drm_device *dev,
3730 struct drm_gem_object **object_list,
3731 int count)
3732{
3733 drm_i915_private_t *dev_priv = dev->dev_private;
3734 struct drm_i915_gem_object *obj_priv;
3735 DEFINE_WAIT(wait);
3736 int i, ret = 0;
3737
3738 for (;;) {
3739 prepare_to_wait(&dev_priv->pending_flip_queue,
3740 &wait, TASK_INTERRUPTIBLE);
3741 for (i = 0; i < count; i++) {
23010e43 3742 obj_priv = to_intel_bo(object_list[i]);
6b95a207
KH		 3743 if (atomic_read(&obj_priv->pending_flip) > 0)
3744 break;
3745 }
3746 if (i == count)
3747 break;
3748
3749 if (!signal_pending(current)) {
3750 mutex_unlock(&dev->struct_mutex);
3751 schedule();
3752 mutex_lock(&dev->struct_mutex);
3753 continue;
3754 }
3755 ret = -ERESTARTSYS;
3756 break;
3757 }
3758 finish_wait(&dev_priv->pending_flip_queue, &wait);
3759
3760 return ret;
3761}
3762
673a394b 3763int
76446cac
JB		 3764i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3765 struct drm_file *file_priv,
3766 struct drm_i915_gem_execbuffer2 *args,
3767 struct drm_i915_gem_exec_object2 *exec_list)
673a394b
EA		 3768{
3769 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b
EA		 3770 struct drm_gem_object **object_list = NULL;
3771 struct drm_gem_object *batch_obj;
b70d11da 3772 struct drm_i915_gem_object *obj_priv;
201361a5 3773 struct drm_clip_rect *cliprects = NULL;
93533c29 3774 struct drm_i915_gem_relocation_entry *relocs = NULL;
76446cac 3775 int ret = 0, ret2, i, pinned = 0;
673a394b 3776 uint64_t exec_offset;
40a5f0de 3777 uint32_t seqno, flush_domains, reloc_index;
6b95a207 3778 int pin_tries, flips;
673a394b
EA		 3779
3780#if WATCH_EXEC
3781 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3782 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3783#endif
3784
4f481ed2
EA		 3785 if (args->buffer_count < 1) {
3786 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3787 return -EINVAL;
3788 }
c8e0f93a 3789 object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
76446cac
JB		 3790 if (object_list == NULL) {
3791 DRM_ERROR("Failed to allocate object list for %d buffers\n",
673a394b
EA		 3792 args->buffer_count);
3793 ret = -ENOMEM;
3794 goto pre_mutex_err;
3795 }
673a394b 3796
201361a5 3797 if (args->num_cliprects != 0) {
9a298b2a
EA		 3798 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3799 GFP_KERNEL);
a40e8d31
OA		 3800 if (cliprects == NULL) {
3801 ret = -ENOMEM;
201361a5 3802 goto pre_mutex_err;
a40e8d31 3803 }
201361a5
EA		 3804
3805 ret = copy_from_user(cliprects,
3806 (struct drm_clip_rect __user *)
3807 (uintptr_t) args->cliprects_ptr,
3808 sizeof(*cliprects) * args->num_cliprects);
3809 if (ret != 0) {
3810 DRM_ERROR("copy %d cliprects failed: %d\n",
3811 args->num_cliprects, ret);
3812 goto pre_mutex_err;
3813 }
3814 }
3815
40a5f0de
EA		 3816 ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3817 &relocs);
3818 if (ret != 0)
3819 goto pre_mutex_err;
3820
673a394b
EA		 3821 mutex_lock(&dev->struct_mutex);
3822
3823 i915_verify_inactive(dev, __FILE__, __LINE__);
3824
ba1234d1 3825 if (atomic_read(&dev_priv->mm.wedged)) {
673a394b 3826 mutex_unlock(&dev->struct_mutex);
a198bc80
CW		 3827 ret = -EIO;
3828 goto pre_mutex_err;
673a394b
EA		 3829 }
3830
3831 if (dev_priv->mm.suspended) {
673a394b 3832 mutex_unlock(&dev->struct_mutex);
a198bc80
CW		 3833 ret = -EBUSY;
3834 goto pre_mutex_err;
673a394b
EA		 3835 }
3836
ac94a962 3837 /* Look up object handles */
6b95a207 3838 flips = 0;
673a394b
EA		 3839 for (i = 0; i < args->buffer_count; i++) {
3840 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3841 exec_list[i].handle);
3842 if (object_list[i] == NULL) {
3843 DRM_ERROR("Invalid object handle %d at index %d\n",
3844 exec_list[i].handle, i);
0ce907f8
CW		 3845 /* prevent error path from reading uninitialized data */
3846 args->buffer_count = i + 1;
673a394b
EA		 3847 ret = -EBADF;
3848 goto err;
3849 }
b70d11da 3850
23010e43 3851 obj_priv = to_intel_bo(object_list[i]);
b70d11da
KH		 3852 if (obj_priv->in_execbuffer) {
3853 DRM_ERROR("Object %p appears more than once in object list\n",
3854 object_list[i]);
0ce907f8
CW		 3855 /* prevent error path from reading uninitialized data */
3856 args->buffer_count = i + 1;
b70d11da
KH		 3857 ret = -EBADF;
3858 goto err;
3859 }
3860 obj_priv->in_execbuffer = true;
6b95a207
KH		 3861 flips += atomic_read(&obj_priv->pending_flip);
3862 }
3863
3864 if (flips > 0) {
3865 ret = i915_gem_wait_for_pending_flip(dev, object_list,
3866 args->buffer_count);
3867 if (ret)
3868 goto err;
ac94a962 3869 }
673a394b 3870
ac94a962
KP		 3871 /* Pin and relocate */
3872 for (pin_tries = 0; ; pin_tries++) {
3873 ret = 0;
40a5f0de
EA		 3874 reloc_index = 0;
3875
ac94a962
KP		 3876 for (i = 0; i < args->buffer_count; i++) {
3877 object_list[i]->pending_read_domains = 0;
3878 object_list[i]->pending_write_domain = 0;
3879 ret = i915_gem_object_pin_and_relocate(object_list[i],
3880 file_priv,
40a5f0de
EA		 3881 &exec_list[i],
3882 &relocs[reloc_index]);
ac94a962
KP		 3883 if (ret)
3884 break;
3885 pinned = i + 1;
40a5f0de 3886 reloc_index += exec_list[i].relocation_count;
ac94a962
KP		 3887 }
3888 /* success */
3889 if (ret == 0)
3890 break;
3891
3892 /* error other than GTT full, or we've already tried again */
2939e1f5 3893 if (ret != -ENOSPC || pin_tries >= 1) {
07f73f69
CW		 3894 if (ret != -ERESTARTSYS) {
3895 unsigned long long total_size = 0;
3896 for (i = 0; i < args->buffer_count; i++)
3897 total_size += object_list[i]->size;
3898 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes: %d\n",
3899 pinned+1, args->buffer_count,
3900 total_size, ret);
3901 DRM_ERROR("%d objects [%d pinned], "
3902 "%d object bytes [%d pinned], "
3903 "%d/%d gtt bytes\n",
3904 atomic_read(&dev->object_count),
3905 atomic_read(&dev->pin_count),
3906 atomic_read(&dev->object_memory),
3907 atomic_read(&dev->pin_memory),
3908 atomic_read(&dev->gtt_memory),
3909 dev->gtt_total);
3910 }
673a394b
EA		 3911 goto err;
3912 }
ac94a962
KP		 3913
3914 /* unpin all of our buffers */
3915 for (i = 0; i < pinned; i++)
3916 i915_gem_object_unpin(object_list[i]);
b1177636 3917 pinned = 0;
ac94a962
KP		 3918
3919 /* evict everyone we can from the aperture */
3920 ret = i915_gem_evict_everything(dev);
07f73f69 3921 if (ret && ret != -ENOSPC)
ac94a962 3922 goto err;
673a394b
EA		 3923 }
3924
3925 /* Set the pending read domains for the batch buffer to COMMAND */
3926 batch_obj = object_list[args->buffer_count-1];
5f26a2c7
CW		 3927 if (batch_obj->pending_write_domain) {
3928 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3929 ret = -EINVAL;
3930 goto err;
3931 }
3932 batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
673a394b 3933
83d60795
CW		 3934 /* Sanity check the batch buffer, prior to moving objects */
3935 exec_offset = exec_list[args->buffer_count - 1].offset;
3936 ret = i915_gem_check_execbuffer (args, exec_offset);
3937 if (ret != 0) {
3938 DRM_ERROR("execbuf with invalid offset/length\n");
3939 goto err;
3940 }
3941
673a394b
EA		 3942 i915_verify_inactive(dev, __FILE__, __LINE__);
3943
646f0f6e
KP		 3944 /* Zero the global flush/invalidate flags. These
3945 * will be modified as new domains are computed
3946 * for each object
3947 */
3948 dev->invalidate_domains = 0;
3949 dev->flush_domains = 0;
3950
673a394b
EA		 3951 for (i = 0; i < args->buffer_count; i++) {
3952 struct drm_gem_object *obj = object_list[i];
673a394b 3953
646f0f6e 3954 /* Compute new gpu domains and update invalidate/flush */
8b0e378a 3955 i915_gem_object_set_to_gpu_domain(obj);
673a394b
EA		 3956 }
3957
3958 i915_verify_inactive(dev, __FILE__, __LINE__);
3959
646f0f6e
KP		 3960 if (dev->invalidate_domains | dev->flush_domains) {
3961#if WATCH_EXEC
3962 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3963 __func__,
3964 dev->invalidate_domains,
3965 dev->flush_domains);
3966#endif
3967 i915_gem_flush(dev,
3968 dev->invalidate_domains,
3969 dev->flush_domains);
99fcb766 3970 if (dev->flush_domains & I915_GEM_GPU_DOMAINS)
b962442e
EA		 3971 (void)i915_add_request(dev, file_priv,
3972 dev->flush_domains);
646f0f6e 3973 }
673a394b 3974
efbeed96
EA		 3975 for (i = 0; i < args->buffer_count; i++) {
3976 struct drm_gem_object *obj = object_list[i];
23010e43 3977 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1c5d22f7 3978 uint32_t old_write_domain = obj->write_domain;
efbeed96
EA		 3979
3980 obj->write_domain = obj->pending_write_domain;
99fcb766
DV		 3981 if (obj->write_domain)
3982 list_move_tail(&obj_priv->gpu_write_list,
3983 &dev_priv->mm.gpu_write_list);
3984 else
3985 list_del_init(&obj_priv->gpu_write_list);
3986
1c5d22f7
CW		 3987 trace_i915_gem_object_change_domain(obj,
3988 obj->read_domains,
3989 old_write_domain);
efbeed96
EA		 3990 }
3991
673a394b
EA		 3992 i915_verify_inactive(dev, __FILE__, __LINE__);
3993
3994#if WATCH_COHERENCY
3995 for (i = 0; i < args->buffer_count; i++) {
3996 i915_gem_object_check_coherency(object_list[i],
3997 exec_list[i].handle);
3998 }
3999#endif
4000
673a394b 4001#if WATCH_EXEC
6911a9b8 4002 i915_gem_dump_object(batch_obj,
673a394b
EA		 4003 args->batch_len,
4004 __func__,
4005 ~0);
4006#endif
4007
673a394b 4008 /* Exec the batchbuffer */
201361a5 4009 ret = i915_dispatch_gem_execbuffer(dev, args, cliprects, exec_offset);
673a394b
EA		 4010 if (ret) {
4011 DRM_ERROR("dispatch failed %d\n", ret);
4012 goto err;
4013 }
4014
4015 /*
4016 * Ensure that the commands in the batch buffer are
4017 * finished before the interrupt fires
4018 */
4019 flush_domains = i915_retire_commands(dev);
4020
4021 i915_verify_inactive(dev, __FILE__, __LINE__);
4022
4023 /*
4024 * Get a seqno representing the execution of the current buffer,
4025 * which we can wait on. We would like to mitigate these interrupts,
4026 * likely by only creating seqnos occasionally (so that we have
4027 * *some* interrupts representing completion of buffers that we can
4028 * wait on when trying to clear up gtt space).
4029 */
b962442e 4030 seqno = i915_add_request(dev, file_priv, flush_domains);
673a394b 4031 BUG_ON(seqno == 0);
673a394b
EA		 4032 for (i = 0; i < args->buffer_count; i++) {
4033 struct drm_gem_object *obj = object_list[i];
673a394b 4034
ce44b0ea 4035 i915_gem_object_move_to_active(obj, seqno);
673a394b
EA		 4036#if WATCH_LRU
4037 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
4038#endif
4039 }
4040#if WATCH_LRU
4041 i915_dump_lru(dev, __func__);
4042#endif
4043
4044 i915_verify_inactive(dev, __FILE__, __LINE__);
4045
673a394b 4046err:
aad87dff
JL		 4047 for (i = 0; i < pinned; i++)
4048 i915_gem_object_unpin(object_list[i]);
4049
b70d11da
KH		 4050 for (i = 0; i < args->buffer_count; i++) {
4051 if (object_list[i]) {
23010e43 4052 obj_priv = to_intel_bo(object_list[i]);
b70d11da
KH		 4053 obj_priv->in_execbuffer = false;
4054 }
aad87dff 4055 drm_gem_object_unreference(object_list[i]);
b70d11da 4056 }
673a394b 4057
673a394b
EA		 4058 mutex_unlock(&dev->struct_mutex);
4059
93533c29 4060pre_mutex_err:
40a5f0de
EA		 4061 /* Copy the updated relocations out regardless of current error
4062 * state. Failure to update the relocs would mean that the next
4063 * time userland calls execbuf, it would do so with presumed offset
4064 * state that didn't match the actual object state.
4065 */
4066 ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
4067 relocs);
4068 if (ret2 != 0) {
4069 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
4070
4071 if (ret == 0)
4072 ret = ret2;
4073 }
4074
8e7d2b2c 4075 drm_free_large(object_list);
9a298b2a 4076 kfree(cliprects);
673a394b
EA		 4077
4078 return ret;
4079}
4080
76446cac
JB		 4081/*
4082 * Legacy execbuffer just creates an exec2 list from the original exec object
4083 * list array and passes it to the real function.
4084 */
4085int
4086i915_gem_execbuffer(struct drm_device *dev, void *data,
4087 struct drm_file *file_priv)
4088{
4089 struct drm_i915_gem_execbuffer *args = data;
4090 struct drm_i915_gem_execbuffer2 exec2;
4091 struct drm_i915_gem_exec_object *exec_list = NULL;
4092 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4093 int ret, i;
4094
4095#if WATCH_EXEC
4096 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4097 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4098#endif
4099
4100 if (args->buffer_count < 1) {
4101 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
4102 return -EINVAL;
4103 }
4104
4105 /* Copy in the exec list from userland */
4106 exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
4107 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4108 if (exec_list == NULL || exec2_list == NULL) {
4109 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4110 args->buffer_count);
4111 drm_free_large(exec_list);
4112 drm_free_large(exec2_list);
4113 return -ENOMEM;
4114 }
4115 ret = copy_from_user(exec_list,
4116 (struct drm_i915_relocation_entry __user *)
4117 (uintptr_t) args->buffers_ptr,
4118 sizeof(*exec_list) * args->buffer_count);
4119 if (ret != 0) {
4120 DRM_ERROR("copy %d exec entries failed %d\n",
4121 args->buffer_count, ret);
4122 drm_free_large(exec_list);
4123 drm_free_large(exec2_list);
4124 return -EFAULT;
4125 }
4126
4127 for (i = 0; i < args->buffer_count; i++) {
4128 exec2_list[i].handle = exec_list[i].handle;
4129 exec2_list[i].relocation_count = exec_list[i].relocation_count;
4130 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
4131 exec2_list[i].alignment = exec_list[i].alignment;
4132 exec2_list[i].offset = exec_list[i].offset;
4133 if (!IS_I965G(dev))
4134 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
4135 else
4136 exec2_list[i].flags = 0;
4137 }
4138
4139 exec2.buffers_ptr = args->buffers_ptr;
4140 exec2.buffer_count = args->buffer_count;
4141 exec2.batch_start_offset = args->batch_start_offset;
4142 exec2.batch_len = args->batch_len;
4143 exec2.DR1 = args->DR1;
4144 exec2.DR4 = args->DR4;
4145 exec2.num_cliprects = args->num_cliprects;
4146 exec2.cliprects_ptr = args->cliprects_ptr;
4147 exec2.flags = 0;
4148
4149 ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
4150 if (!ret) {
4151 /* Copy the new buffer offsets back to the user's exec list. */
4152 for (i = 0; i < args->buffer_count; i++)
4153 exec_list[i].offset = exec2_list[i].offset;
4154 /* ... and back out to userspace */
4155 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4156 (uintptr_t) args->buffers_ptr,
4157 exec_list,
4158 sizeof(*exec_list) * args->buffer_count);
4159 if (ret) {
4160 ret = -EFAULT;
4161 DRM_ERROR("failed to copy %d exec entries "
4162 "back to user (%d)\n",
4163 args->buffer_count, ret);
4164 }
76446cac
JB		 4165 }
4166
4167 drm_free_large(exec_list);
4168 drm_free_large(exec2_list);
4169 return ret;
4170}
4171
4172int
4173i915_gem_execbuffer2(struct drm_device *dev, void *data,
4174 struct drm_file *file_priv)
4175{
4176 struct drm_i915_gem_execbuffer2 *args = data;
4177 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4178 int ret;
4179
4180#if WATCH_EXEC
4181 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4182 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4183#endif
4184
4185 if (args->buffer_count < 1) {
4186 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4187 return -EINVAL;
4188 }
4189
4190 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4191 if (exec2_list == NULL) {
4192 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4193 args->buffer_count);
4194 return -ENOMEM;
4195 }
4196 ret = copy_from_user(exec2_list,
4197 (struct drm_i915_relocation_entry __user *)
4198 (uintptr_t) args->buffers_ptr,
4199 sizeof(*exec2_list) * args->buffer_count);
4200 if (ret != 0) {
4201 DRM_ERROR("copy %d exec entries failed %d\n",
4202 args->buffer_count, ret);
4203 drm_free_large(exec2_list);
4204 return -EFAULT;
4205 }
4206
4207 ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4208 if (!ret) {
4209 /* Copy the new buffer offsets back to the user's exec list. */
4210 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4211 (uintptr_t) args->buffers_ptr,
4212 exec2_list,
4213 sizeof(*exec2_list) * args->buffer_count);
4214 if (ret) {
4215 ret = -EFAULT;
4216 DRM_ERROR("failed to copy %d exec entries "
4217 "back to user (%d)\n",
4218 args->buffer_count, ret);
4219 }
4220 }
4221
4222 drm_free_large(exec2_list);
4223 return ret;
4224}
4225
673a394b
EA		 4226int
4227i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4228{
4229 struct drm_device *dev = obj->dev;
23010e43 4230 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 4231 int ret;
4232
4233 i915_verify_inactive(dev, __FILE__, __LINE__);
4234 if (obj_priv->gtt_space == NULL) {
4235 ret = i915_gem_object_bind_to_gtt(obj, alignment);
9731129c 4236 if (ret)
673a394b 4237 return ret;
22c344e9 4238 }
76446cac 4239
673a394b
EA		 4240 obj_priv->pin_count++;
4241
4242 /* If the object is not active and not pending a flush,
4243 * remove it from the inactive list
4244 */
4245 if (obj_priv->pin_count == 1) {
4246 atomic_inc(&dev->pin_count);
4247 atomic_add(obj->size, &dev->pin_memory);
4248 if (!obj_priv->active &&
21d509e3 4249 (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0 &&
673a394b
EA		 4250 !list_empty(&obj_priv->list))
4251 list_del_init(&obj_priv->list);
4252 }
4253 i915_verify_inactive(dev, __FILE__, __LINE__);
4254
4255 return 0;
4256}
4257
4258void
4259i915_gem_object_unpin(struct drm_gem_object *obj)
4260{
4261 struct drm_device *dev = obj->dev;
4262 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 4263 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 4264
4265 i915_verify_inactive(dev, __FILE__, __LINE__);
4266 obj_priv->pin_count--;
4267 BUG_ON(obj_priv->pin_count < 0);
4268 BUG_ON(obj_priv->gtt_space == NULL);
4269
4270 /* If the object is no longer pinned, and is
4271 * neither active nor being flushed, then stick it on
4272 * the inactive list
4273 */
4274 if (obj_priv->pin_count == 0) {
4275 if (!obj_priv->active &&
21d509e3 4276 (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
673a394b
EA		 4277 list_move_tail(&obj_priv->list,
4278 &dev_priv->mm.inactive_list);
4279 atomic_dec(&dev->pin_count);
4280 atomic_sub(obj->size, &dev->pin_memory);
4281 }
4282 i915_verify_inactive(dev, __FILE__, __LINE__);
4283}
4284
4285int
4286i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4287 struct drm_file *file_priv)
4288{
4289 struct drm_i915_gem_pin *args = data;
4290 struct drm_gem_object *obj;
4291 struct drm_i915_gem_object *obj_priv;
4292 int ret;
4293
4294 mutex_lock(&dev->struct_mutex);
4295
4296 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4297 if (obj == NULL) {
4298 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
4299 args->handle);
4300 mutex_unlock(&dev->struct_mutex);
4301 return -EBADF;
4302 }
23010e43 4303 obj_priv = to_intel_bo(obj);
673a394b 4304
bb6baf76
CW		 4305 if (obj_priv->madv != I915_MADV_WILLNEED) {
4306 DRM_ERROR("Attempting to pin a purgeable buffer\n");
3ef94daa
CW		 4307 drm_gem_object_unreference(obj);
4308 mutex_unlock(&dev->struct_mutex);
4309 return -EINVAL;
4310 }
4311
79e53945
JB		 4312 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4313 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4314 args->handle);
96dec61d 4315 drm_gem_object_unreference(obj);
673a394b 4316 mutex_unlock(&dev->struct_mutex);
79e53945
JB		 4317 return -EINVAL;
4318 }
4319
4320 obj_priv->user_pin_count++;
4321 obj_priv->pin_filp = file_priv;
4322 if (obj_priv->user_pin_count == 1) {
4323 ret = i915_gem_object_pin(obj, args->alignment);
4324 if (ret != 0) {
4325 drm_gem_object_unreference(obj);
4326 mutex_unlock(&dev->struct_mutex);
4327 return ret;
4328 }
673a394b
EA		 4329 }
4330
4331 /* XXX - flush the CPU caches for pinned objects
4332 * as the X server doesn't manage domains yet
4333 */
e47c68e9 4334 i915_gem_object_flush_cpu_write_domain(obj);
673a394b
EA		 4335 args->offset = obj_priv->gtt_offset;
4336 drm_gem_object_unreference(obj);
4337 mutex_unlock(&dev->struct_mutex);
4338
4339 return 0;
4340}
4341
4342int
4343i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4344 struct drm_file *file_priv)
4345{
4346 struct drm_i915_gem_pin *args = data;
4347 struct drm_gem_object *obj;
79e53945 4348 struct drm_i915_gem_object *obj_priv;
673a394b
EA		 4349
4350 mutex_lock(&dev->struct_mutex);
4351
4352 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4353 if (obj == NULL) {
4354 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
4355 args->handle);
4356 mutex_unlock(&dev->struct_mutex);
4357 return -EBADF;
4358 }
4359
23010e43 4360 obj_priv = to_intel_bo(obj);
79e53945
JB		 4361 if (obj_priv->pin_filp != file_priv) {
4362 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4363 args->handle);
4364 drm_gem_object_unreference(obj);
4365 mutex_unlock(&dev->struct_mutex);
4366 return -EINVAL;
4367 }
4368 obj_priv->user_pin_count--;
4369 if (obj_priv->user_pin_count == 0) {
4370 obj_priv->pin_filp = NULL;
4371 i915_gem_object_unpin(obj);
4372 }
673a394b
EA		 4373
4374 drm_gem_object_unreference(obj);
4375 mutex_unlock(&dev->struct_mutex);
4376 return 0;
4377}
4378
4379int
4380i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4381 struct drm_file *file_priv)
4382{
4383 struct drm_i915_gem_busy *args = data;
4384 struct drm_gem_object *obj;
4385 struct drm_i915_gem_object *obj_priv;
4386
673a394b
EA		 4387 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4388 if (obj == NULL) {
4389 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
4390 args->handle);
673a394b
EA		 4391 return -EBADF;
4392 }
4393
b1ce786c 4394 mutex_lock(&dev->struct_mutex);
f21289b3
EA		 4395 /* Update the active list for the hardware's current position.
4396 * Otherwise this only updates on a delayed timer or when irqs are
4397 * actually unmasked, and our working set ends up being larger than
4398 * required.
4399 */
4400 i915_gem_retire_requests(dev);
4401
23010e43 4402 obj_priv = to_intel_bo(obj);
c4de0a5d
EA		 4403 /* Don't count being on the flushing list against the object being
4404 * done. Otherwise, a buffer left on the flushing list but not getting
4405 * flushed (because nobody's flushing that domain) won't ever return
4406 * unbusy and get reused by libdrm's bo cache. The other expected
4407 * consumer of this interface, OpenGL's occlusion queries, also specs
4408 * that the objects get unbusy "eventually" without any interference.
4409 */
4410 args->busy = obj_priv->active && obj_priv->last_rendering_seqno != 0;
673a394b
EA		 4411
4412 drm_gem_object_unreference(obj);
4413 mutex_unlock(&dev->struct_mutex);
4414 return 0;
4415}
4416
4417int
4418i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4419 struct drm_file *file_priv)
4420{
4421 return i915_gem_ring_throttle(dev, file_priv);
4422}
4423
3ef94daa
CW		 4424int
4425i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4426 struct drm_file *file_priv)
4427{
4428 struct drm_i915_gem_madvise *args = data;
4429 struct drm_gem_object *obj;
4430 struct drm_i915_gem_object *obj_priv;
4431
4432 switch (args->madv) {
4433 case I915_MADV_DONTNEED:
4434 case I915_MADV_WILLNEED:
4435 break;
4436 default:
4437 return -EINVAL;
4438 }
4439
4440 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4441 if (obj == NULL) {
4442 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
4443 args->handle);
4444 return -EBADF;
4445 }
4446
4447 mutex_lock(&dev->struct_mutex);
23010e43 4448 obj_priv = to_intel_bo(obj);
3ef94daa
CW		 4449
4450 if (obj_priv->pin_count) {
4451 drm_gem_object_unreference(obj);
4452 mutex_unlock(&dev->struct_mutex);
4453
4454 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
4455 return -EINVAL;
4456 }
4457
bb6baf76
CW		 4458 if (obj_priv->madv != __I915_MADV_PURGED)
4459 obj_priv->madv = args->madv;
3ef94daa 4460
2d7ef395
CW		 4461 /* if the object is no longer bound, discard its backing storage */
4462 if (i915_gem_object_is_purgeable(obj_priv) &&
4463 obj_priv->gtt_space == NULL)
4464 i915_gem_object_truncate(obj);
4465
bb6baf76
CW		 4466 args->retained = obj_priv->madv != __I915_MADV_PURGED;
4467
3ef94daa
CW		 4468 drm_gem_object_unreference(obj);
4469 mutex_unlock(&dev->struct_mutex);
4470
4471 return 0;
4472}
4473
ac52bc56
DV		 4474struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4475 size_t size)
4476{
c397b908 4477 struct drm_i915_gem_object *obj;
ac52bc56 4478
c397b908
DV		 4479 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4480 if (obj == NULL)
4481 return NULL;
673a394b 4482
c397b908
DV		 4483 if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4484 kfree(obj);
4485 return NULL;
4486 }
673a394b 4487
c397b908
DV		 4488 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4489 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
673a394b 4490
c397b908 4491 obj->agp_type = AGP_USER_MEMORY;
ba1eb1d8 4492
62b8b215 4493 obj->base.driver_private = NULL;
c397b908
DV		 4494 obj->fence_reg = I915_FENCE_REG_NONE;
4495 INIT_LIST_HEAD(&obj->list);
4496 INIT_LIST_HEAD(&obj->gpu_write_list);
4497 INIT_LIST_HEAD(&obj->fence_list);
4498 obj->madv = I915_MADV_WILLNEED;
de151cf6 4499
c397b908
DV		 4500 trace_i915_gem_object_create(&obj->base);
4501
4502 return &obj->base;
4503}
4504
4505int i915_gem_init_object(struct drm_gem_object *obj)
4506{
4507 BUG();
de151cf6 4508
673a394b
EA		 4509 return 0;
4510}
4511
4512void i915_gem_free_object(struct drm_gem_object *obj)
4513{
de151cf6 4514 struct drm_device *dev = obj->dev;
23010e43 4515 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b 4516
1c5d22f7
CW		 4517 trace_i915_gem_object_destroy(obj);
4518
673a394b
EA		 4519 while (obj_priv->pin_count > 0)
4520 i915_gem_object_unpin(obj);
4521
71acb5eb
DA		 4522 if (obj_priv->phys_obj)
4523 i915_gem_detach_phys_object(dev, obj);
4524
673a394b
EA		 4525 i915_gem_object_unbind(obj);
4526
7e616158
CW		 4527 if (obj_priv->mmap_offset)
4528 i915_gem_free_mmap_offset(obj);
de151cf6 4529
c397b908
DV		 4530 drm_gem_object_release(obj);
4531
9a298b2a 4532 kfree(obj_priv->page_cpu_valid);
280b713b 4533 kfree(obj_priv->bit_17);
c397b908 4534 kfree(obj_priv);
673a394b
EA		 4535}
4536
ab5ee576 4537/** Unbinds all inactive objects. */
673a394b 4538static int
ab5ee576 4539i915_gem_evict_from_inactive_list(struct drm_device *dev)
673a394b 4540{
ab5ee576 4541 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 4542
ab5ee576
CW		 4543 while (!list_empty(&dev_priv->mm.inactive_list)) {
4544 struct drm_gem_object *obj;
4545 int ret;
673a394b 4546
a8089e84
DV		 4547 obj = &list_first_entry(&dev_priv->mm.inactive_list,
4548 struct drm_i915_gem_object,
4549 list)->base;
673a394b
EA		 4550
4551 ret = i915_gem_object_unbind(obj);
4552 if (ret != 0) {
ab5ee576 4553 DRM_ERROR("Error unbinding object: %d\n", ret);
673a394b
EA		 4554 return ret;
4555 }
4556 }
4557
673a394b
EA		 4558 return 0;
4559}
4560
29105ccc
CW		 4561int
4562i915_gem_idle(struct drm_device *dev)
4563{
4564 drm_i915_private_t *dev_priv = dev->dev_private;
4565 int ret;
28dfe52a 4566
29105ccc 4567 mutex_lock(&dev->struct_mutex);
1c5d22f7 4568
29105ccc
CW		 4569 if (dev_priv->mm.suspended || dev_priv->ring.ring_obj == NULL) {
4570 mutex_unlock(&dev->struct_mutex);
4571 return 0;
28dfe52a
EA		 4572 }
4573
29105ccc 4574 ret = i915_gpu_idle(dev);
6dbe2772
KP		 4575 if (ret) {
4576 mutex_unlock(&dev->struct_mutex);
673a394b 4577 return ret;
6dbe2772 4578 }
673a394b 4579
29105ccc
CW		 4580 /* Under UMS, be paranoid and evict. */
4581 if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4582 ret = i915_gem_evict_from_inactive_list(dev);
4583 if (ret) {
4584 mutex_unlock(&dev->struct_mutex);
4585 return ret;
4586 }
4587 }
4588
4589 /* Hack! Don't let anybody do execbuf while we don't control the chip.
4590 * We need to replace this with a semaphore, or something.
4591 * And not confound mm.suspended!
4592 */
4593 dev_priv->mm.suspended = 1;
4594 del_timer(&dev_priv->hangcheck_timer);
4595
4596 i915_kernel_lost_context(dev);
6dbe2772 4597 i915_gem_cleanup_ringbuffer(dev);
29105ccc 4598
6dbe2772
KP		 4599 mutex_unlock(&dev->struct_mutex);
4600
29105ccc
CW		 4601 /* Cancel the retire work handler, which should be idle now. */
4602 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4603
673a394b
EA		 4604 return 0;
4605}
4606
e552eb70
JB		 4607/*
4608 * 965+ support PIPE_CONTROL commands, which provide finer grained control
4609 * over cache flushing.
4610 */
4611static int
4612i915_gem_init_pipe_control(struct drm_device *dev)
4613{
4614 drm_i915_private_t *dev_priv = dev->dev_private;
4615 struct drm_gem_object *obj;
4616 struct drm_i915_gem_object *obj_priv;
4617 int ret;
4618
34dc4d44 4619 obj = i915_gem_alloc_object(dev, 4096);
e552eb70
JB		 4620 if (obj == NULL) {
4621 DRM_ERROR("Failed to allocate seqno page\n");
4622 ret = -ENOMEM;
4623 goto err;
4624 }
4625 obj_priv = to_intel_bo(obj);
4626 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4627
4628 ret = i915_gem_object_pin(obj, 4096);
4629 if (ret)
4630 goto err_unref;
4631
4632 dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4633 dev_priv->seqno_page = kmap(obj_priv->pages[0]);
4634 if (dev_priv->seqno_page == NULL)
4635 goto err_unpin;
4636
4637 dev_priv->seqno_obj = obj;
4638 memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4639
4640 return 0;
4641
4642err_unpin:
4643 i915_gem_object_unpin(obj);
4644err_unref:
4645 drm_gem_object_unreference(obj);
4646err:
4647 return ret;
4648}
4649
673a394b
EA		 4650static int
4651i915_gem_init_hws(struct drm_device *dev)
4652{
4653 drm_i915_private_t *dev_priv = dev->dev_private;
4654 struct drm_gem_object *obj;
4655 struct drm_i915_gem_object *obj_priv;
4656 int ret;
4657
4658 /* If we need a physical address for the status page, it's already
4659 * initialized at driver load time.
4660 */
4661 if (!I915_NEED_GFX_HWS(dev))
4662 return 0;
4663
ac52bc56 4664 obj = i915_gem_alloc_object(dev, 4096);
673a394b
EA		 4665 if (obj == NULL) {
4666 DRM_ERROR("Failed to allocate status page\n");
e552eb70
JB		 4667 ret = -ENOMEM;
4668 goto err;
673a394b 4669 }
23010e43 4670 obj_priv = to_intel_bo(obj);
ba1eb1d8 4671 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
673a394b
EA		 4672
4673 ret = i915_gem_object_pin(obj, 4096);
4674 if (ret != 0) {
4675 drm_gem_object_unreference(obj);
e552eb70 4676 goto err_unref;
673a394b
EA		 4677 }
4678
4679 dev_priv->status_gfx_addr = obj_priv->gtt_offset;
673a394b 4680
856fa198 4681 dev_priv->hw_status_page = kmap(obj_priv->pages[0]);
ba1eb1d8 4682 if (dev_priv->hw_status_page == NULL) {
673a394b
EA		 4683 DRM_ERROR("Failed to map status page.\n");
4684 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
e552eb70
JB		 4685 ret = -EINVAL;
4686 goto err_unpin;
673a394b 4687 }
e552eb70
JB		 4688
4689 if (HAS_PIPE_CONTROL(dev)) {
4690 ret = i915_gem_init_pipe_control(dev);
4691 if (ret)
4692 goto err_unpin;
4693 }
4694
673a394b 4695 dev_priv->hws_obj = obj;
673a394b 4696 memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
f6e450a6
EA		 4697 if (IS_GEN6(dev)) {
4698 I915_WRITE(HWS_PGA_GEN6, dev_priv->status_gfx_addr);
4699 I915_READ(HWS_PGA_GEN6); /* posting read */
4700 } else {
4701 I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
4702 I915_READ(HWS_PGA); /* posting read */
4703 }
44d98a61 4704 DRM_DEBUG_DRIVER("hws offset: 0x%08x\n", dev_priv->status_gfx_addr);
673a394b
EA		 4705
4706 return 0;
e552eb70
JB		 4707
4708err_unpin:
4709 i915_gem_object_unpin(obj);
4710err_unref:
4711 drm_gem_object_unreference(obj);
4712err:
4713 return 0;
4714}
4715
4716static void
4717i915_gem_cleanup_pipe_control(struct drm_device *dev)
4718{
4719 drm_i915_private_t *dev_priv = dev->dev_private;
4720 struct drm_gem_object *obj;
4721 struct drm_i915_gem_object *obj_priv;
4722
4723 obj = dev_priv->seqno_obj;
4724 obj_priv = to_intel_bo(obj);
4725 kunmap(obj_priv->pages[0]);
4726 i915_gem_object_unpin(obj);
4727 drm_gem_object_unreference(obj);
4728 dev_priv->seqno_obj = NULL;
4729
4730 dev_priv->seqno_page = NULL;
673a394b
EA		 4731}
4732
85a7bb98
CW		 4733static void
4734i915_gem_cleanup_hws(struct drm_device *dev)
4735{
4736 drm_i915_private_t *dev_priv = dev->dev_private;
bab2d1f6
CW		 4737 struct drm_gem_object *obj;
4738 struct drm_i915_gem_object *obj_priv;
85a7bb98
CW		 4739
4740 if (dev_priv->hws_obj == NULL)
4741 return;
4742
bab2d1f6 4743 obj = dev_priv->hws_obj;
23010e43 4744 obj_priv = to_intel_bo(obj);
bab2d1f6 4745
856fa198 4746 kunmap(obj_priv->pages[0]);
85a7bb98
CW		 4747 i915_gem_object_unpin(obj);
4748 drm_gem_object_unreference(obj);
4749 dev_priv->hws_obj = NULL;
bab2d1f6 4750
85a7bb98
CW		 4751 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
4752 dev_priv->hw_status_page = NULL;
4753
e552eb70
JB		 4754 if (HAS_PIPE_CONTROL(dev))
4755 i915_gem_cleanup_pipe_control(dev);
4756
85a7bb98
CW		 4757 /* Write high address into HWS_PGA when disabling. */
4758 I915_WRITE(HWS_PGA, 0x1ffff000);
4759}
4760
79e53945 4761int
673a394b
EA		 4762i915_gem_init_ringbuffer(struct drm_device *dev)
4763{
4764 drm_i915_private_t *dev_priv = dev->dev_private;
4765 struct drm_gem_object *obj;
4766 struct drm_i915_gem_object *obj_priv;
79e53945 4767 drm_i915_ring_buffer_t *ring = &dev_priv->ring;
673a394b 4768 int ret;
50aa253d 4769 u32 head;
673a394b
EA		 4770
4771 ret = i915_gem_init_hws(dev);
4772 if (ret != 0)
4773 return ret;
4774
ac52bc56 4775 obj = i915_gem_alloc_object(dev, 128 * 1024);
673a394b
EA		 4776 if (obj == NULL) {
4777 DRM_ERROR("Failed to allocate ringbuffer\n");
85a7bb98 4778 i915_gem_cleanup_hws(dev);
673a394b
EA		 4779 return -ENOMEM;
4780 }
23010e43 4781 obj_priv = to_intel_bo(obj);
673a394b
EA		 4782
4783 ret = i915_gem_object_pin(obj, 4096);
4784 if (ret != 0) {
4785 drm_gem_object_unreference(obj);
85a7bb98 4786 i915_gem_cleanup_hws(dev);
673a394b
EA		 4787 return ret;
4788 }
4789
4790 /* Set up the kernel mapping for the ring. */
79e53945 4791 ring->Size = obj->size;
673a394b 4792
79e53945
JB		 4793 ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
4794 ring->map.size = obj->size;
4795 ring->map.type = 0;
4796 ring->map.flags = 0;
4797 ring->map.mtrr = 0;
673a394b 4798
79e53945
JB		 4799 drm_core_ioremap_wc(&ring->map, dev);
4800 if (ring->map.handle == NULL) {
673a394b
EA		 4801 DRM_ERROR("Failed to map ringbuffer.\n");
4802 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
47ed185a 4803 i915_gem_object_unpin(obj);
673a394b 4804 drm_gem_object_unreference(obj);
85a7bb98 4805 i915_gem_cleanup_hws(dev);
673a394b
EA		 4806 return -EINVAL;
4807 }
79e53945
JB		 4808 ring->ring_obj = obj;
4809 ring->virtual_start = ring->map.handle;
673a394b
EA		 4810
4811 /* Stop the ring if it's running. */
4812 I915_WRITE(PRB0_CTL, 0);
673a394b 4813 I915_WRITE(PRB0_TAIL, 0);
50aa253d 4814 I915_WRITE(PRB0_HEAD, 0);
673a394b
EA		 4815
4816 /* Initialize the ring. */
4817 I915_WRITE(PRB0_START, obj_priv->gtt_offset);
50aa253d
KP		 4818 head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4819
4820 /* G45 ring initialization fails to reset head to zero */
4821 if (head != 0) {
4822 DRM_ERROR("Ring head not reset to zero "
4823 "ctl %08x head %08x tail %08x start %08x\n",
4824 I915_READ(PRB0_CTL),
4825 I915_READ(PRB0_HEAD),
4826 I915_READ(PRB0_TAIL),
4827 I915_READ(PRB0_START));
4828 I915_WRITE(PRB0_HEAD, 0);
4829
4830 DRM_ERROR("Ring head forced to zero "
4831 "ctl %08x head %08x tail %08x start %08x\n",
4832 I915_READ(PRB0_CTL),
4833 I915_READ(PRB0_HEAD),
4834 I915_READ(PRB0_TAIL),
4835 I915_READ(PRB0_START));
4836 }
4837
673a394b
EA		 4838 I915_WRITE(PRB0_CTL,
4839 ((obj->size - 4096) & RING_NR_PAGES) |
4840 RING_NO_REPORT |
4841 RING_VALID);
4842
50aa253d
KP		 4843 head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4844
4845 /* If the head is still not zero, the ring is dead */
4846 if (head != 0) {
4847 DRM_ERROR("Ring initialization failed "
4848 "ctl %08x head %08x tail %08x start %08x\n",
4849 I915_READ(PRB0_CTL),
4850 I915_READ(PRB0_HEAD),
4851 I915_READ(PRB0_TAIL),
4852 I915_READ(PRB0_START));
4853 return -EIO;
4854 }
4855
673a394b 4856 /* Update our cache of the ring state */
79e53945
JB		 4857 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4858 i915_kernel_lost_context(dev);
4859 else {
4860 ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4861 ring->tail = I915_READ(PRB0_TAIL) & TAIL_ADDR;
4862 ring->space = ring->head - (ring->tail + 8);
4863 if (ring->space < 0)
4864 ring->space += ring->Size;
4865 }
673a394b 4866
71cf39b1
EA		 4867 if (IS_I9XX(dev) && !IS_GEN3(dev)) {
4868 I915_WRITE(MI_MODE,
4869 (VS_TIMER_DISPATCH) << 16 | VS_TIMER_DISPATCH);
4870 }
4871
673a394b
EA		 4872 return 0;
4873}
4874
79e53945 4875void
673a394b
EA		 4876i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4877{
4878 drm_i915_private_t *dev_priv = dev->dev_private;
4879
4880 if (dev_priv->ring.ring_obj == NULL)
4881 return;
4882
4883 drm_core_ioremapfree(&dev_priv->ring.map, dev);
4884
4885 i915_gem_object_unpin(dev_priv->ring.ring_obj);
4886 drm_gem_object_unreference(dev_priv->ring.ring_obj);
4887 dev_priv->ring.ring_obj = NULL;
4888 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
4889
85a7bb98 4890 i915_gem_cleanup_hws(dev);
673a394b
EA		 4891}
4892
4893int
4894i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4895 struct drm_file *file_priv)
4896{
4897 drm_i915_private_t *dev_priv = dev->dev_private;
4898 int ret;
4899
79e53945
JB		 4900 if (drm_core_check_feature(dev, DRIVER_MODESET))
4901 return 0;
4902
ba1234d1 4903 if (atomic_read(&dev_priv->mm.wedged)) {
673a394b 4904 DRM_ERROR("Reenabling wedged hardware, good luck\n");
ba1234d1 4905 atomic_set(&dev_priv->mm.wedged, 0);
673a394b
EA		 4906 }
4907
673a394b 4908 mutex_lock(&dev->struct_mutex);
9bb2d6f9
EA		 4909 dev_priv->mm.suspended = 0;
4910
4911 ret = i915_gem_init_ringbuffer(dev);
d816f6ac
WF		 4912 if (ret != 0) {
4913 mutex_unlock(&dev->struct_mutex);
9bb2d6f9 4914 return ret;
d816f6ac 4915 }
9bb2d6f9 4916
5e118f41 4917 spin_lock(&dev_priv->mm.active_list_lock);
673a394b 4918 BUG_ON(!list_empty(&dev_priv->mm.active_list));
5e118f41
CW		 4919 spin_unlock(&dev_priv->mm.active_list_lock);
4920
673a394b
EA		 4921 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4922 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4923 BUG_ON(!list_empty(&dev_priv->mm.request_list));
673a394b 4924 mutex_unlock(&dev->struct_mutex);
dbb19d30
KH		 4925
4926 drm_irq_install(dev);
4927
673a394b
EA		 4928 return 0;
4929}
4930
4931int
4932i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4933 struct drm_file *file_priv)
4934{
79e53945
JB		 4935 if (drm_core_check_feature(dev, DRIVER_MODESET))
4936 return 0;
4937
dbb19d30 4938 drm_irq_uninstall(dev);
e6890f6f 4939 return i915_gem_idle(dev);
673a394b
EA		 4940}
4941
4942void
4943i915_gem_lastclose(struct drm_device *dev)
4944{
4945 int ret;
673a394b 4946
e806b495
EA		 4947 if (drm_core_check_feature(dev, DRIVER_MODESET))
4948 return;
4949
6dbe2772
KP		 4950 ret = i915_gem_idle(dev);
4951 if (ret)
4952 DRM_ERROR("failed to idle hardware: %d\n", ret);
673a394b
EA		 4953}
4954
4955void
4956i915_gem_load(struct drm_device *dev)
4957{
b5aa8a0f 4958 int i;
673a394b
EA		 4959 drm_i915_private_t *dev_priv = dev->dev_private;
4960
5e118f41 4961 spin_lock_init(&dev_priv->mm.active_list_lock);
673a394b
EA		 4962 INIT_LIST_HEAD(&dev_priv->mm.active_list);
4963 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
99fcb766 4964 INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
673a394b
EA		 4965 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4966 INIT_LIST_HEAD(&dev_priv->mm.request_list);
a09ba7fa 4967 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
673a394b
EA		 4968 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4969 i915_gem_retire_work_handler);
4970 dev_priv->mm.next_gem_seqno = 1;
4971
31169714
CW		 4972 spin_lock(&shrink_list_lock);
4973 list_add(&dev_priv->mm.shrink_list, &shrink_list);
4974 spin_unlock(&shrink_list_lock);
4975
de151cf6 4976 /* Old X drivers will take 0-2 for front, back, depth buffers */
b397c836
EA		 4977 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4978 dev_priv->fence_reg_start = 3;
de151cf6 4979
0f973f27 4980 if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
de151cf6
JB		 4981 dev_priv->num_fence_regs = 16;
4982 else
4983 dev_priv->num_fence_regs = 8;
4984
b5aa8a0f
GH		 4985 /* Initialize fence registers to zero */
4986 if (IS_I965G(dev)) {
4987 for (i = 0; i < 16; i++)
4988 I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4989 } else {
4990 for (i = 0; i < 8; i++)
4991 I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4992 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4993 for (i = 0; i < 8; i++)
4994 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4995 }
673a394b 4996 i915_gem_detect_bit_6_swizzle(dev);
6b95a207 4997 init_waitqueue_head(&dev_priv->pending_flip_queue);
673a394b 4998}
71acb5eb
DA		 4999
5000/*
5001 * Create a physically contiguous memory object for this object
5002 * e.g. for cursor + overlay regs
5003 */
5004int i915_gem_init_phys_object(struct drm_device *dev,
5005 int id, int size)
5006{
5007 drm_i915_private_t *dev_priv = dev->dev_private;
5008 struct drm_i915_gem_phys_object *phys_obj;
5009 int ret;
5010
5011 if (dev_priv->mm.phys_objs[id - 1] || !size)
5012 return 0;
5013
9a298b2a 5014 phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
71acb5eb
DA		 5015 if (!phys_obj)
5016 return -ENOMEM;
5017
5018 phys_obj->id = id;
5019
e6be8d9d 5020 phys_obj->handle = drm_pci_alloc(dev, size, 0);
71acb5eb
DA		 5021 if (!phys_obj->handle) {
5022 ret = -ENOMEM;
5023 goto kfree_obj;
5024 }
5025#ifdef CONFIG_X86
5026 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
5027#endif
5028
5029 dev_priv->mm.phys_objs[id - 1] = phys_obj;
5030
5031 return 0;
5032kfree_obj:
9a298b2a 5033 kfree(phys_obj);
71acb5eb
DA		 5034 return ret;
5035}
5036
5037void i915_gem_free_phys_object(struct drm_device *dev, int id)
5038{
5039 drm_i915_private_t *dev_priv = dev->dev_private;
5040 struct drm_i915_gem_phys_object *phys_obj;
5041
5042 if (!dev_priv->mm.phys_objs[id - 1])
5043 return;
5044
5045 phys_obj = dev_priv->mm.phys_objs[id - 1];
5046 if (phys_obj->cur_obj) {
5047 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
5048 }
5049
5050#ifdef CONFIG_X86
5051 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
5052#endif
5053 drm_pci_free(dev, phys_obj->handle);
5054 kfree(phys_obj);
5055 dev_priv->mm.phys_objs[id - 1] = NULL;
5056}
5057
5058void i915_gem_free_all_phys_object(struct drm_device *dev)
5059{
5060 int i;
5061
260883c8 5062 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
71acb5eb
DA		 5063 i915_gem_free_phys_object(dev, i);
5064}
5065
5066void i915_gem_detach_phys_object(struct drm_device *dev,
5067 struct drm_gem_object *obj)
5068{
5069 struct drm_i915_gem_object *obj_priv;
5070 int i;
5071 int ret;
5072 int page_count;
5073
23010e43 5074 obj_priv = to_intel_bo(obj);
71acb5eb
DA		 5075 if (!obj_priv->phys_obj)
5076 return;
5077
4bdadb97 5078 ret = i915_gem_object_get_pages(obj, 0);
71acb5eb
DA		 5079 if (ret)
5080 goto out;
5081
5082 page_count = obj->size / PAGE_SIZE;
5083
5084 for (i = 0; i < page_count; i++) {
856fa198 5085 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
71acb5eb
DA		 5086 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
5087
5088 memcpy(dst, src, PAGE_SIZE);
5089 kunmap_atomic(dst, KM_USER0);
5090 }
856fa198 5091 drm_clflush_pages(obj_priv->pages, page_count);
71acb5eb 5092 drm_agp_chipset_flush(dev);
d78b47b9
CW		 5093
5094 i915_gem_object_put_pages(obj);
71acb5eb
DA		 5095out:
5096 obj_priv->phys_obj->cur_obj = NULL;
5097 obj_priv->phys_obj = NULL;
5098}
5099
5100int
5101i915_gem_attach_phys_object(struct drm_device *dev,
5102 struct drm_gem_object *obj, int id)
5103{
5104 drm_i915_private_t *dev_priv = dev->dev_private;
5105 struct drm_i915_gem_object *obj_priv;
5106 int ret = 0;
5107 int page_count;
5108 int i;
5109
5110 if (id > I915_MAX_PHYS_OBJECT)
5111 return -EINVAL;
5112
23010e43 5113 obj_priv = to_intel_bo(obj);
71acb5eb
DA		 5114
5115 if (obj_priv->phys_obj) {
5116 if (obj_priv->phys_obj->id == id)
5117 return 0;
5118 i915_gem_detach_phys_object(dev, obj);
5119 }
5120
5121
5122 /* create a new object */
5123 if (!dev_priv->mm.phys_objs[id - 1]) {
5124 ret = i915_gem_init_phys_object(dev, id,
5125 obj->size);
5126 if (ret) {
aeb565df 5127 DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
71acb5eb
DA		 5128 goto out;
5129 }
5130 }
5131
5132 /* bind to the object */
5133 obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
5134 obj_priv->phys_obj->cur_obj = obj;
5135
4bdadb97 5136 ret = i915_gem_object_get_pages(obj, 0);
71acb5eb
DA		 5137 if (ret) {
5138 DRM_ERROR("failed to get page list\n");
5139 goto out;
5140 }
5141
5142 page_count = obj->size / PAGE_SIZE;
5143
5144 for (i = 0; i < page_count; i++) {
856fa198 5145 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
71acb5eb
DA		 5146 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
5147
5148 memcpy(dst, src, PAGE_SIZE);
5149 kunmap_atomic(src, KM_USER0);
5150 }
5151
d78b47b9
CW		 5152 i915_gem_object_put_pages(obj);
5153
71acb5eb
DA		 5154 return 0;
5155out:
5156 return ret;
5157}
5158
5159static int
5160i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
5161 struct drm_i915_gem_pwrite *args,
5162 struct drm_file *file_priv)
5163{
23010e43 5164 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
71acb5eb
DA		 5165 void *obj_addr;
5166 int ret;
5167 char __user *user_data;
5168
5169 user_data = (char __user *) (uintptr_t) args->data_ptr;
5170 obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
5171
44d98a61 5172 DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
71acb5eb
DA		 5173 ret = copy_from_user(obj_addr, user_data, args->size);
5174 if (ret)
5175 return -EFAULT;
5176
5177 drm_agp_chipset_flush(dev);
5178 return 0;
5179}
b962442e
EA		 5180
5181void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv)
5182{
5183 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
5184
5185 /* Clean up our request list when the client is going away, so that
5186 * later retire_requests won't dereference our soon-to-be-gone
5187 * file_priv.
5188 */
5189 mutex_lock(&dev->struct_mutex);
5190 while (!list_empty(&i915_file_priv->mm.request_list))
5191 list_del_init(i915_file_priv->mm.request_list.next);
5192 mutex_unlock(&dev->struct_mutex);
5193}
31169714 5194
1637ef41
CW		 5195static int
5196i915_gpu_is_active(struct drm_device *dev)
5197{
5198 drm_i915_private_t *dev_priv = dev->dev_private;
5199 int lists_empty;
5200
5201 spin_lock(&dev_priv->mm.active_list_lock);
5202 lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
5203 list_empty(&dev_priv->mm.active_list);
5204 spin_unlock(&dev_priv->mm.active_list_lock);
5205
5206 return !lists_empty;
5207}
5208
31169714
CW		 5209static int
5210i915_gem_shrink(int nr_to_scan, gfp_t gfp_mask)
5211{
5212 drm_i915_private_t *dev_priv, *next_dev;
5213 struct drm_i915_gem_object *obj_priv, *next_obj;
5214 int cnt = 0;
5215 int would_deadlock = 1;
5216
5217 /* "fast-path" to count number of available objects */
5218 if (nr_to_scan == 0) {
5219 spin_lock(&shrink_list_lock);
5220 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5221 struct drm_device *dev = dev_priv->dev;
5222
5223 if (mutex_trylock(&dev->struct_mutex)) {
5224 list_for_each_entry(obj_priv,
5225 &dev_priv->mm.inactive_list,
5226 list)
5227 cnt++;
5228 mutex_unlock(&dev->struct_mutex);
5229 }
5230 }
5231 spin_unlock(&shrink_list_lock);
5232
5233 return (cnt / 100) * sysctl_vfs_cache_pressure;
5234 }
5235
5236 spin_lock(&shrink_list_lock);
5237
1637ef41 5238rescan:
31169714
CW		 5239 /* first scan for clean buffers */
5240 list_for_each_entry_safe(dev_priv, next_dev,
5241 &shrink_list, mm.shrink_list) {
5242 struct drm_device *dev = dev_priv->dev;
5243
5244 if (! mutex_trylock(&dev->struct_mutex))
5245 continue;
5246
5247 spin_unlock(&shrink_list_lock);
5248
5249 i915_gem_retire_requests(dev);
5250
5251 list_for_each_entry_safe(obj_priv, next_obj,
5252 &dev_priv->mm.inactive_list,
5253 list) {
5254 if (i915_gem_object_is_purgeable(obj_priv)) {
a8089e84 5255 i915_gem_object_unbind(&obj_priv->base);
31169714
CW		 5256 if (--nr_to_scan <= 0)
5257 break;
5258 }
5259 }
5260
5261 spin_lock(&shrink_list_lock);
5262 mutex_unlock(&dev->struct_mutex);
5263
963b4836
CW		 5264 would_deadlock = 0;
5265
31169714
CW		 5266 if (nr_to_scan <= 0)
5267 break;
5268 }
5269
5270 /* second pass, evict/count anything still on the inactive list */
5271 list_for_each_entry_safe(dev_priv, next_dev,
5272 &shrink_list, mm.shrink_list) {
5273 struct drm_device *dev = dev_priv->dev;
5274
5275 if (! mutex_trylock(&dev->struct_mutex))
5276 continue;
5277
5278 spin_unlock(&shrink_list_lock);
5279
5280 list_for_each_entry_safe(obj_priv, next_obj,
5281 &dev_priv->mm.inactive_list,
5282 list) {
5283 if (nr_to_scan > 0) {
a8089e84 5284 i915_gem_object_unbind(&obj_priv->base);
31169714
CW		 5285 nr_to_scan--;
5286 } else
5287 cnt++;
5288 }
5289
5290 spin_lock(&shrink_list_lock);
5291 mutex_unlock(&dev->struct_mutex);
5292
5293 would_deadlock = 0;
5294 }
5295
1637ef41
CW		 5296 if (nr_to_scan) {
5297 int active = 0;
5298
5299 /*
5300 * We are desperate for pages, so as a last resort, wait
5301 * for the GPU to finish and discard whatever we can.
5302 * This has a dramatic impact to reduce the number of
5303 * OOM-killer events whilst running the GPU aggressively.
5304 */
5305 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5306 struct drm_device *dev = dev_priv->dev;
5307
5308 if (!mutex_trylock(&dev->struct_mutex))
5309 continue;
5310
5311 spin_unlock(&shrink_list_lock);
5312
5313 if (i915_gpu_is_active(dev)) {
5314 i915_gpu_idle(dev);
5315 active++;
5316 }
5317
5318 spin_lock(&shrink_list_lock);
5319 mutex_unlock(&dev->struct_mutex);
5320 }
5321
5322 if (active)
5323 goto rescan;
5324 }
5325
31169714
CW		 5326 spin_unlock(&shrink_list_lock);
5327
5328 if (would_deadlock)
5329 return -1;
5330 else if (cnt > 0)
5331 return (cnt / 100) * sysctl_vfs_cache_pressure;
5332 else
5333 return 0;
5334}
5335
5336static struct shrinker shrinker = {
5337 .shrink = i915_gem_shrink,
5338 .seeks = DEFAULT_SEEKS,
5339};
5340
5341__init void
5342i915_gem_shrinker_init(void)
5343{
5344 register_shrinker(&shrinker);
5345}
5346
5347__exit void
5348i915_gem_shrinker_exit(void)
5349{
5350 unregister_shrinker(&shrinker);
5351}
Ubuntu Artful kernel mirror