2 * Copyright © 2008-2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Eric Anholt <eric@anholt.net>
28 #include <drm/drm_vma_manager.h>
29 #include <drm/i915_drm.h>
30 #include <linux/dma-fence-array.h>
31 #include <linux/kthread.h>
32 #include <linux/dma-resv.h>
33 #include <linux/shmem_fs.h>
34 #include <linux/slab.h>
35 #include <linux/stop_machine.h>
36 #include <linux/swap.h>
37 #include <linux/pci.h>
38 #include <linux/dma-buf.h>
39 #include <linux/mman.h>
41 #include "display/intel_display.h"
42 #include "display/intel_frontbuffer.h"
44 #include "gem/i915_gem_clflush.h"
45 #include "gem/i915_gem_context.h"
46 #include "gem/i915_gem_ioctls.h"
47 #include "gem/i915_gem_pm.h"
48 #include "gem/i915_gemfs.h"
49 #include "gt/intel_engine_user.h"
50 #include "gt/intel_gt.h"
51 #include "gt/intel_gt_pm.h"
52 #include "gt/intel_mocs.h"
53 #include "gt/intel_reset.h"
54 #include "gt/intel_renderstate.h"
55 #include "gt/intel_workarounds.h"
58 #include "i915_scatterlist.h"
59 #include "i915_trace.h"
60 #include "i915_vgpu.h"
65 insert_mappable_node(struct i915_ggtt
*ggtt
, struct drm_mm_node
*node
, u32 size
)
69 err
= mutex_lock_interruptible(&ggtt
->vm
.mutex
);
73 memset(node
, 0, sizeof(*node
));
74 err
= drm_mm_insert_node_in_range(&ggtt
->vm
.mm
, node
,
75 size
, 0, I915_COLOR_UNEVICTABLE
,
76 0, ggtt
->mappable_end
,
79 mutex_unlock(&ggtt
->vm
.mutex
);
85 remove_mappable_node(struct i915_ggtt
*ggtt
, struct drm_mm_node
*node
)
87 mutex_lock(&ggtt
->vm
.mutex
);
88 drm_mm_remove_node(node
);
89 mutex_unlock(&ggtt
->vm
.mutex
);
93 i915_gem_get_aperture_ioctl(struct drm_device
*dev
, void *data
,
94 struct drm_file
*file
)
96 struct i915_ggtt
*ggtt
= &to_i915(dev
)->ggtt
;
97 struct drm_i915_gem_get_aperture
*args
= data
;
101 if (mutex_lock_interruptible(&ggtt
->vm
.mutex
))
104 pinned
= ggtt
->vm
.reserved
;
105 list_for_each_entry(vma
, &ggtt
->vm
.bound_list
, vm_link
)
106 if (i915_vma_is_pinned(vma
))
107 pinned
+= vma
->node
.size
;
109 mutex_unlock(&ggtt
->vm
.mutex
);
111 args
->aper_size
= ggtt
->vm
.total
;
112 args
->aper_available_size
= args
->aper_size
- pinned
;
117 int i915_gem_object_unbind(struct drm_i915_gem_object
*obj
,
120 struct i915_vma
*vma
;
121 LIST_HEAD(still_in_list
);
124 spin_lock(&obj
->vma
.lock
);
125 while (!ret
&& (vma
= list_first_entry_or_null(&obj
->vma
.list
,
128 struct i915_address_space
*vm
= vma
->vm
;
131 if (!i915_vm_tryopen(vm
))
134 list_move_tail(&vma
->obj_link
, &still_in_list
);
135 spin_unlock(&obj
->vma
.lock
);
137 if (flags
& I915_GEM_OBJECT_UNBIND_ACTIVE
||
138 !i915_vma_is_active(vma
))
139 ret
= i915_vma_unbind(vma
);
142 spin_lock(&obj
->vma
.lock
);
144 list_splice(&still_in_list
, &obj
->vma
.list
);
145 spin_unlock(&obj
->vma
.lock
);
151 i915_gem_phys_pwrite(struct drm_i915_gem_object
*obj
,
152 struct drm_i915_gem_pwrite
*args
,
153 struct drm_file
*file
)
155 void *vaddr
= obj
->phys_handle
->vaddr
+ args
->offset
;
156 char __user
*user_data
= u64_to_user_ptr(args
->data_ptr
);
159 * We manually control the domain here and pretend that it
160 * remains coherent i.e. in the GTT domain, like shmem_pwrite.
162 intel_frontbuffer_invalidate(obj
->frontbuffer
, ORIGIN_CPU
);
164 if (copy_from_user(vaddr
, user_data
, args
->size
))
167 drm_clflush_virt_range(vaddr
, args
->size
);
168 intel_gt_chipset_flush(&to_i915(obj
->base
.dev
)->gt
);
170 intel_frontbuffer_flush(obj
->frontbuffer
, ORIGIN_CPU
);
175 i915_gem_create(struct drm_file
*file
,
176 struct drm_i915_private
*dev_priv
,
180 struct drm_i915_gem_object
*obj
;
185 size
= round_up(*size_p
, PAGE_SIZE
);
189 /* Allocate the new object */
190 obj
= i915_gem_object_create_shmem(dev_priv
, size
);
194 ret
= drm_gem_handle_create(file
, &obj
->base
, &handle
);
195 /* drop reference from allocate - handle holds it now */
196 i915_gem_object_put(obj
);
206 i915_gem_dumb_create(struct drm_file
*file
,
207 struct drm_device
*dev
,
208 struct drm_mode_create_dumb
*args
)
210 int cpp
= DIV_ROUND_UP(args
->bpp
, 8);
215 format
= DRM_FORMAT_C8
;
218 format
= DRM_FORMAT_RGB565
;
221 format
= DRM_FORMAT_XRGB8888
;
227 /* have to work out size/pitch and return them */
228 args
->pitch
= ALIGN(args
->width
* cpp
, 64);
230 /* align stride to page size so that we can remap */
231 if (args
->pitch
> intel_plane_fb_max_stride(to_i915(dev
), format
,
232 DRM_FORMAT_MOD_LINEAR
))
233 args
->pitch
= ALIGN(args
->pitch
, 4096);
235 args
->size
= args
->pitch
* args
->height
;
236 return i915_gem_create(file
, to_i915(dev
),
237 &args
->size
, &args
->handle
);
241 * Creates a new mm object and returns a handle to it.
242 * @dev: drm device pointer
243 * @data: ioctl data blob
244 * @file: drm file pointer
247 i915_gem_create_ioctl(struct drm_device
*dev
, void *data
,
248 struct drm_file
*file
)
250 struct drm_i915_private
*dev_priv
= to_i915(dev
);
251 struct drm_i915_gem_create
*args
= data
;
253 i915_gem_flush_free_objects(dev_priv
);
255 return i915_gem_create(file
, dev_priv
,
256 &args
->size
, &args
->handle
);
260 shmem_pread(struct page
*page
, int offset
, int len
, char __user
*user_data
,
269 drm_clflush_virt_range(vaddr
+ offset
, len
);
271 ret
= __copy_to_user(user_data
, vaddr
+ offset
, len
);
275 return ret
? -EFAULT
: 0;
279 i915_gem_shmem_pread(struct drm_i915_gem_object
*obj
,
280 struct drm_i915_gem_pread
*args
)
282 unsigned int needs_clflush
;
283 unsigned int idx
, offset
;
284 struct dma_fence
*fence
;
285 char __user
*user_data
;
289 ret
= i915_gem_object_prepare_read(obj
, &needs_clflush
);
293 fence
= i915_gem_object_lock_fence(obj
);
294 i915_gem_object_finish_access(obj
);
299 user_data
= u64_to_user_ptr(args
->data_ptr
);
300 offset
= offset_in_page(args
->offset
);
301 for (idx
= args
->offset
>> PAGE_SHIFT
; remain
; idx
++) {
302 struct page
*page
= i915_gem_object_get_page(obj
, idx
);
303 unsigned int length
= min_t(u64
, remain
, PAGE_SIZE
- offset
);
305 ret
= shmem_pread(page
, offset
, length
, user_data
,
315 i915_gem_object_unlock_fence(obj
, fence
);
320 gtt_user_read(struct io_mapping
*mapping
,
321 loff_t base
, int offset
,
322 char __user
*user_data
, int length
)
325 unsigned long unwritten
;
327 /* We can use the cpu mem copy function because this is X86. */
328 vaddr
= io_mapping_map_atomic_wc(mapping
, base
);
329 unwritten
= __copy_to_user_inatomic(user_data
,
330 (void __force
*)vaddr
+ offset
,
332 io_mapping_unmap_atomic(vaddr
);
334 vaddr
= io_mapping_map_wc(mapping
, base
, PAGE_SIZE
);
335 unwritten
= copy_to_user(user_data
,
336 (void __force
*)vaddr
+ offset
,
338 io_mapping_unmap(vaddr
);
344 i915_gem_gtt_pread(struct drm_i915_gem_object
*obj
,
345 const struct drm_i915_gem_pread
*args
)
347 struct drm_i915_private
*i915
= to_i915(obj
->base
.dev
);
348 struct i915_ggtt
*ggtt
= &i915
->ggtt
;
349 intel_wakeref_t wakeref
;
350 struct drm_mm_node node
;
351 struct dma_fence
*fence
;
352 void __user
*user_data
;
353 struct i915_vma
*vma
;
357 wakeref
= intel_runtime_pm_get(&i915
->runtime_pm
);
358 vma
= ERR_PTR(-ENODEV
);
359 if (!i915_gem_object_is_tiled(obj
))
360 vma
= i915_gem_object_ggtt_pin(obj
, NULL
, 0, 0,
362 PIN_NONBLOCK
/* NOWARN */ |
365 node
.start
= i915_ggtt_offset(vma
);
368 ret
= insert_mappable_node(ggtt
, &node
, PAGE_SIZE
);
371 GEM_BUG_ON(!drm_mm_node_allocated(&node
));
374 ret
= i915_gem_object_lock_interruptible(obj
);
378 ret
= i915_gem_object_set_to_gtt_domain(obj
, false);
380 i915_gem_object_unlock(obj
);
384 fence
= i915_gem_object_lock_fence(obj
);
385 i915_gem_object_unlock(obj
);
391 user_data
= u64_to_user_ptr(args
->data_ptr
);
393 offset
= args
->offset
;
396 /* Operation in this page
398 * page_base = page offset within aperture
399 * page_offset = offset within page
400 * page_length = bytes to copy for this page
402 u32 page_base
= node
.start
;
403 unsigned page_offset
= offset_in_page(offset
);
404 unsigned page_length
= PAGE_SIZE
- page_offset
;
405 page_length
= remain
< page_length
? remain
: page_length
;
406 if (drm_mm_node_allocated(&node
)) {
407 ggtt
->vm
.insert_page(&ggtt
->vm
,
408 i915_gem_object_get_dma_address(obj
, offset
>> PAGE_SHIFT
),
409 node
.start
, I915_CACHE_NONE
, 0);
411 page_base
+= offset
& PAGE_MASK
;
414 if (gtt_user_read(&ggtt
->iomap
, page_base
, page_offset
,
415 user_data
, page_length
)) {
420 remain
-= page_length
;
421 user_data
+= page_length
;
422 offset
+= page_length
;
425 i915_gem_object_unlock_fence(obj
, fence
);
427 if (drm_mm_node_allocated(&node
)) {
428 ggtt
->vm
.clear_range(&ggtt
->vm
, node
.start
, node
.size
);
429 remove_mappable_node(ggtt
, &node
);
434 intel_runtime_pm_put(&i915
->runtime_pm
, wakeref
);
439 * Reads data from the object referenced by handle.
440 * @dev: drm device pointer
441 * @data: ioctl data blob
442 * @file: drm file pointer
444 * On error, the contents of *data are undefined.
447 i915_gem_pread_ioctl(struct drm_device
*dev
, void *data
,
448 struct drm_file
*file
)
450 struct drm_i915_gem_pread
*args
= data
;
451 struct drm_i915_gem_object
*obj
;
457 if (!access_ok(u64_to_user_ptr(args
->data_ptr
),
461 obj
= i915_gem_object_lookup(file
, args
->handle
);
465 /* Bounds check source. */
466 if (range_overflows_t(u64
, args
->offset
, args
->size
, obj
->base
.size
)) {
471 trace_i915_gem_object_pread(obj
, args
->offset
, args
->size
);
473 ret
= i915_gem_object_wait(obj
,
474 I915_WAIT_INTERRUPTIBLE
,
475 MAX_SCHEDULE_TIMEOUT
);
479 ret
= i915_gem_object_pin_pages(obj
);
483 ret
= i915_gem_shmem_pread(obj
, args
);
484 if (ret
== -EFAULT
|| ret
== -ENODEV
)
485 ret
= i915_gem_gtt_pread(obj
, args
);
487 i915_gem_object_unpin_pages(obj
);
489 i915_gem_object_put(obj
);
493 /* This is the fast write path which cannot handle
494 * page faults in the source data
498 ggtt_write(struct io_mapping
*mapping
,
499 loff_t base
, int offset
,
500 char __user
*user_data
, int length
)
503 unsigned long unwritten
;
505 /* We can use the cpu mem copy function because this is X86. */
506 vaddr
= io_mapping_map_atomic_wc(mapping
, base
);
507 unwritten
= __copy_from_user_inatomic_nocache((void __force
*)vaddr
+ offset
,
509 io_mapping_unmap_atomic(vaddr
);
511 vaddr
= io_mapping_map_wc(mapping
, base
, PAGE_SIZE
);
512 unwritten
= copy_from_user((void __force
*)vaddr
+ offset
,
514 io_mapping_unmap(vaddr
);
521 * This is the fast pwrite path, where we copy the data directly from the
522 * user into the GTT, uncached.
523 * @obj: i915 GEM object
524 * @args: pwrite arguments structure
527 i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object
*obj
,
528 const struct drm_i915_gem_pwrite
*args
)
530 struct drm_i915_private
*i915
= to_i915(obj
->base
.dev
);
531 struct i915_ggtt
*ggtt
= &i915
->ggtt
;
532 struct intel_runtime_pm
*rpm
= &i915
->runtime_pm
;
533 intel_wakeref_t wakeref
;
534 struct drm_mm_node node
;
535 struct dma_fence
*fence
;
536 struct i915_vma
*vma
;
538 void __user
*user_data
;
541 if (i915_gem_object_has_struct_page(obj
)) {
543 * Avoid waking the device up if we can fallback, as
544 * waking/resuming is very slow (worst-case 10-100 ms
545 * depending on PCI sleeps and our own resume time).
546 * This easily dwarfs any performance advantage from
547 * using the cache bypass of indirect GGTT access.
549 wakeref
= intel_runtime_pm_get_if_in_use(rpm
);
553 /* No backing pages, no fallback, we must force GGTT access */
554 wakeref
= intel_runtime_pm_get(rpm
);
557 vma
= ERR_PTR(-ENODEV
);
558 if (!i915_gem_object_is_tiled(obj
))
559 vma
= i915_gem_object_ggtt_pin(obj
, NULL
, 0, 0,
561 PIN_NONBLOCK
/* NOWARN */ |
564 node
.start
= i915_ggtt_offset(vma
);
567 ret
= insert_mappable_node(ggtt
, &node
, PAGE_SIZE
);
570 GEM_BUG_ON(!drm_mm_node_allocated(&node
));
573 ret
= i915_gem_object_lock_interruptible(obj
);
577 ret
= i915_gem_object_set_to_gtt_domain(obj
, true);
579 i915_gem_object_unlock(obj
);
583 fence
= i915_gem_object_lock_fence(obj
);
584 i915_gem_object_unlock(obj
);
590 intel_frontbuffer_invalidate(obj
->frontbuffer
, ORIGIN_CPU
);
592 user_data
= u64_to_user_ptr(args
->data_ptr
);
593 offset
= args
->offset
;
596 /* Operation in this page
598 * page_base = page offset within aperture
599 * page_offset = offset within page
600 * page_length = bytes to copy for this page
602 u32 page_base
= node
.start
;
603 unsigned int page_offset
= offset_in_page(offset
);
604 unsigned int page_length
= PAGE_SIZE
- page_offset
;
605 page_length
= remain
< page_length
? remain
: page_length
;
606 if (drm_mm_node_allocated(&node
)) {
607 /* flush the write before we modify the GGTT */
608 intel_gt_flush_ggtt_writes(ggtt
->vm
.gt
);
609 ggtt
->vm
.insert_page(&ggtt
->vm
,
610 i915_gem_object_get_dma_address(obj
, offset
>> PAGE_SHIFT
),
611 node
.start
, I915_CACHE_NONE
, 0);
612 wmb(); /* flush modifications to the GGTT (insert_page) */
614 page_base
+= offset
& PAGE_MASK
;
616 /* If we get a fault while copying data, then (presumably) our
617 * source page isn't available. Return the error and we'll
618 * retry in the slow path.
619 * If the object is non-shmem backed, we retry again with the
620 * path that handles page fault.
622 if (ggtt_write(&ggtt
->iomap
, page_base
, page_offset
,
623 user_data
, page_length
)) {
628 remain
-= page_length
;
629 user_data
+= page_length
;
630 offset
+= page_length
;
632 intel_frontbuffer_flush(obj
->frontbuffer
, ORIGIN_CPU
);
634 i915_gem_object_unlock_fence(obj
, fence
);
636 intel_gt_flush_ggtt_writes(ggtt
->vm
.gt
);
637 if (drm_mm_node_allocated(&node
)) {
638 ggtt
->vm
.clear_range(&ggtt
->vm
, node
.start
, node
.size
);
639 remove_mappable_node(ggtt
, &node
);
644 intel_runtime_pm_put(rpm
, wakeref
);
648 /* Per-page copy function for the shmem pwrite fastpath.
649 * Flushes invalid cachelines before writing to the target if
650 * needs_clflush_before is set and flushes out any written cachelines after
651 * writing if needs_clflush is set.
654 shmem_pwrite(struct page
*page
, int offset
, int len
, char __user
*user_data
,
655 bool needs_clflush_before
,
656 bool needs_clflush_after
)
663 if (needs_clflush_before
)
664 drm_clflush_virt_range(vaddr
+ offset
, len
);
666 ret
= __copy_from_user(vaddr
+ offset
, user_data
, len
);
667 if (!ret
&& needs_clflush_after
)
668 drm_clflush_virt_range(vaddr
+ offset
, len
);
672 return ret
? -EFAULT
: 0;
676 i915_gem_shmem_pwrite(struct drm_i915_gem_object
*obj
,
677 const struct drm_i915_gem_pwrite
*args
)
679 unsigned int partial_cacheline_write
;
680 unsigned int needs_clflush
;
681 unsigned int offset
, idx
;
682 struct dma_fence
*fence
;
683 void __user
*user_data
;
687 ret
= i915_gem_object_prepare_write(obj
, &needs_clflush
);
691 fence
= i915_gem_object_lock_fence(obj
);
692 i915_gem_object_finish_access(obj
);
696 /* If we don't overwrite a cacheline completely we need to be
697 * careful to have up-to-date data by first clflushing. Don't
698 * overcomplicate things and flush the entire patch.
700 partial_cacheline_write
= 0;
701 if (needs_clflush
& CLFLUSH_BEFORE
)
702 partial_cacheline_write
= boot_cpu_data
.x86_clflush_size
- 1;
704 user_data
= u64_to_user_ptr(args
->data_ptr
);
706 offset
= offset_in_page(args
->offset
);
707 for (idx
= args
->offset
>> PAGE_SHIFT
; remain
; idx
++) {
708 struct page
*page
= i915_gem_object_get_page(obj
, idx
);
709 unsigned int length
= min_t(u64
, remain
, PAGE_SIZE
- offset
);
711 ret
= shmem_pwrite(page
, offset
, length
, user_data
,
712 (offset
| length
) & partial_cacheline_write
,
713 needs_clflush
& CLFLUSH_AFTER
);
722 intel_frontbuffer_flush(obj
->frontbuffer
, ORIGIN_CPU
);
723 i915_gem_object_unlock_fence(obj
, fence
);
729 * Writes data to the object referenced by handle.
731 * @data: ioctl data blob
734 * On error, the contents of the buffer that were to be modified are undefined.
737 i915_gem_pwrite_ioctl(struct drm_device
*dev
, void *data
,
738 struct drm_file
*file
)
740 struct drm_i915_gem_pwrite
*args
= data
;
741 struct drm_i915_gem_object
*obj
;
747 if (!access_ok(u64_to_user_ptr(args
->data_ptr
), args
->size
))
750 obj
= i915_gem_object_lookup(file
, args
->handle
);
754 /* Bounds check destination. */
755 if (range_overflows_t(u64
, args
->offset
, args
->size
, obj
->base
.size
)) {
760 /* Writes not allowed into this read-only object */
761 if (i915_gem_object_is_readonly(obj
)) {
766 trace_i915_gem_object_pwrite(obj
, args
->offset
, args
->size
);
769 if (obj
->ops
->pwrite
)
770 ret
= obj
->ops
->pwrite(obj
, args
);
774 ret
= i915_gem_object_wait(obj
,
775 I915_WAIT_INTERRUPTIBLE
|
777 MAX_SCHEDULE_TIMEOUT
);
781 ret
= i915_gem_object_pin_pages(obj
);
786 /* We can only do the GTT pwrite on untiled buffers, as otherwise
787 * it would end up going through the fenced access, and we'll get
788 * different detiling behavior between reading and writing.
789 * pread/pwrite currently are reading and writing from the CPU
790 * perspective, requiring manual detiling by the client.
792 if (!i915_gem_object_has_struct_page(obj
) ||
793 cpu_write_needs_clflush(obj
))
794 /* Note that the gtt paths might fail with non-page-backed user
795 * pointers (e.g. gtt mappings when moving data between
796 * textures). Fallback to the shmem path in that case.
798 ret
= i915_gem_gtt_pwrite_fast(obj
, args
);
800 if (ret
== -EFAULT
|| ret
== -ENOSPC
) {
801 if (obj
->phys_handle
)
802 ret
= i915_gem_phys_pwrite(obj
, args
, file
);
804 ret
= i915_gem_shmem_pwrite(obj
, args
);
807 i915_gem_object_unpin_pages(obj
);
809 i915_gem_object_put(obj
);
814 * Called when user space has done writes to this buffer
816 * @data: ioctl data blob
820 i915_gem_sw_finish_ioctl(struct drm_device
*dev
, void *data
,
821 struct drm_file
*file
)
823 struct drm_i915_gem_sw_finish
*args
= data
;
824 struct drm_i915_gem_object
*obj
;
826 obj
= i915_gem_object_lookup(file
, args
->handle
);
831 * Proxy objects are barred from CPU access, so there is no
832 * need to ban sw_finish as it is a nop.
835 /* Pinned buffers may be scanout, so flush the cache */
836 i915_gem_object_flush_if_display(obj
);
837 i915_gem_object_put(obj
);
842 void i915_gem_runtime_suspend(struct drm_i915_private
*i915
)
844 struct drm_i915_gem_object
*obj
, *on
;
848 * Only called during RPM suspend. All users of the userfault_list
849 * must be holding an RPM wakeref to ensure that this can not
850 * run concurrently with themselves (and use the struct_mutex for
851 * protection between themselves).
854 list_for_each_entry_safe(obj
, on
,
855 &i915
->ggtt
.userfault_list
, userfault_link
)
856 __i915_gem_object_release_mmap(obj
);
859 * The fence will be lost when the device powers down. If any were
860 * in use by hardware (i.e. they are pinned), we should not be powering
861 * down! All other fences will be reacquired by the user upon waking.
863 for (i
= 0; i
< i915
->ggtt
.num_fences
; i
++) {
864 struct i915_fence_reg
*reg
= &i915
->ggtt
.fence_regs
[i
];
867 * Ideally we want to assert that the fence register is not
868 * live at this point (i.e. that no piece of code will be
869 * trying to write through fence + GTT, as that both violates
870 * our tracking of activity and associated locking/barriers,
871 * but also is illegal given that the hw is powered down).
873 * Previously we used reg->pin_count as a "liveness" indicator.
874 * That is not sufficient, and we need a more fine-grained
875 * tool if we want to have a sanity check here.
881 GEM_BUG_ON(i915_vma_has_userfault(reg
->vma
));
887 i915_gem_object_ggtt_pin(struct drm_i915_gem_object
*obj
,
888 const struct i915_ggtt_view
*view
,
893 struct drm_i915_private
*dev_priv
= to_i915(obj
->base
.dev
);
894 struct i915_address_space
*vm
= &dev_priv
->ggtt
.vm
;
895 struct i915_vma
*vma
;
898 if (i915_gem_object_never_bind_ggtt(obj
))
899 return ERR_PTR(-ENODEV
);
901 if (flags
& PIN_MAPPABLE
&&
902 (!view
|| view
->type
== I915_GGTT_VIEW_NORMAL
)) {
903 /* If the required space is larger than the available
904 * aperture, we will not able to find a slot for the
905 * object and unbinding the object now will be in
906 * vain. Worse, doing so may cause us to ping-pong
907 * the object in and out of the Global GTT and
908 * waste a lot of cycles under the mutex.
910 if (obj
->base
.size
> dev_priv
->ggtt
.mappable_end
)
911 return ERR_PTR(-E2BIG
);
913 /* If NONBLOCK is set the caller is optimistically
914 * trying to cache the full object within the mappable
915 * aperture, and *must* have a fallback in place for
916 * situations where we cannot bind the object. We
917 * can be a little more lax here and use the fallback
918 * more often to avoid costly migrations of ourselves
919 * and other objects within the aperture.
921 * Half-the-aperture is used as a simple heuristic.
922 * More interesting would to do search for a free
923 * block prior to making the commitment to unbind.
924 * That caters for the self-harm case, and with a
925 * little more heuristics (e.g. NOFAULT, NOEVICT)
926 * we could try to minimise harm to others.
928 if (flags
& PIN_NONBLOCK
&&
929 obj
->base
.size
> dev_priv
->ggtt
.mappable_end
/ 2)
930 return ERR_PTR(-ENOSPC
);
933 vma
= i915_vma_instance(obj
, vm
, view
);
937 if (i915_vma_misplaced(vma
, size
, alignment
, flags
)) {
938 if (flags
& PIN_NONBLOCK
) {
939 if (i915_vma_is_pinned(vma
) || i915_vma_is_active(vma
))
940 return ERR_PTR(-ENOSPC
);
942 if (flags
& PIN_MAPPABLE
&&
943 vma
->fence_size
> dev_priv
->ggtt
.mappable_end
/ 2)
944 return ERR_PTR(-ENOSPC
);
947 ret
= i915_vma_unbind(vma
);
952 if (vma
->fence
&& !i915_gem_object_is_tiled(obj
)) {
953 mutex_lock(&vma
->vm
->mutex
);
954 ret
= i915_vma_revoke_fence(vma
);
955 mutex_unlock(&vma
->vm
->mutex
);
960 ret
= i915_vma_pin(vma
, size
, alignment
, flags
| PIN_GLOBAL
);
968 i915_gem_madvise_ioctl(struct drm_device
*dev
, void *data
,
969 struct drm_file
*file_priv
)
971 struct drm_i915_private
*i915
= to_i915(dev
);
972 struct drm_i915_gem_madvise
*args
= data
;
973 struct drm_i915_gem_object
*obj
;
976 switch (args
->madv
) {
977 case I915_MADV_DONTNEED
:
978 case I915_MADV_WILLNEED
:
984 obj
= i915_gem_object_lookup(file_priv
, args
->handle
);
988 err
= mutex_lock_interruptible(&obj
->mm
.lock
);
992 if (i915_gem_object_has_pages(obj
) &&
993 i915_gem_object_is_tiled(obj
) &&
994 i915
->quirks
& QUIRK_PIN_SWIZZLED_PAGES
) {
995 if (obj
->mm
.madv
== I915_MADV_WILLNEED
) {
996 GEM_BUG_ON(!obj
->mm
.quirked
);
997 __i915_gem_object_unpin_pages(obj
);
998 obj
->mm
.quirked
= false;
1000 if (args
->madv
== I915_MADV_WILLNEED
) {
1001 GEM_BUG_ON(obj
->mm
.quirked
);
1002 __i915_gem_object_pin_pages(obj
);
1003 obj
->mm
.quirked
= true;
1007 if (obj
->mm
.madv
!= __I915_MADV_PURGED
)
1008 obj
->mm
.madv
= args
->madv
;
1010 if (i915_gem_object_has_pages(obj
)) {
1011 struct list_head
*list
;
1013 if (i915_gem_object_is_shrinkable(obj
)) {
1014 unsigned long flags
;
1016 spin_lock_irqsave(&i915
->mm
.obj_lock
, flags
);
1018 if (obj
->mm
.madv
!= I915_MADV_WILLNEED
)
1019 list
= &i915
->mm
.purge_list
;
1021 list
= &i915
->mm
.shrink_list
;
1022 list_move_tail(&obj
->mm
.link
, list
);
1024 spin_unlock_irqrestore(&i915
->mm
.obj_lock
, flags
);
1028 /* if the object is no longer attached, discard its backing storage */
1029 if (obj
->mm
.madv
== I915_MADV_DONTNEED
&&
1030 !i915_gem_object_has_pages(obj
))
1031 i915_gem_object_truncate(obj
);
1033 args
->retained
= obj
->mm
.madv
!= __I915_MADV_PURGED
;
1034 mutex_unlock(&obj
->mm
.lock
);
1037 i915_gem_object_put(obj
);
1041 void i915_gem_sanitize(struct drm_i915_private
*i915
)
1043 intel_wakeref_t wakeref
;
1047 wakeref
= intel_runtime_pm_get(&i915
->runtime_pm
);
1048 intel_uncore_forcewake_get(&i915
->uncore
, FORCEWAKE_ALL
);
1051 * As we have just resumed the machine and woken the device up from
1052 * deep PCI sleep (presumably D3_cold), assume the HW has been reset
1053 * back to defaults, recovering from whatever wedged state we left it
1054 * in and so worth trying to use the device once more.
1056 if (intel_gt_is_wedged(&i915
->gt
))
1057 intel_gt_unset_wedged(&i915
->gt
);
1060 * If we inherit context state from the BIOS or earlier occupants
1061 * of the GPU, the GPU may be in an inconsistent state when we
1062 * try to take over. The only way to remove the earlier state
1063 * is by resetting. However, resetting on earlier gen is tricky as
1064 * it may impact the display and we are uncertain about the stability
1065 * of the reset, so this could be applied to even earlier gen.
1067 intel_gt_sanitize(&i915
->gt
, false);
1069 intel_uncore_forcewake_put(&i915
->uncore
, FORCEWAKE_ALL
);
1070 intel_runtime_pm_put(&i915
->runtime_pm
, wakeref
);
1073 static int __intel_engines_record_defaults(struct drm_i915_private
*i915
)
1075 struct i915_request
*requests
[I915_NUM_ENGINES
] = {};
1076 struct intel_engine_cs
*engine
;
1077 enum intel_engine_id id
;
1081 * As we reset the gpu during very early sanitisation, the current
1082 * register state on the GPU should reflect its defaults values.
1083 * We load a context onto the hw (with restore-inhibit), then switch
1084 * over to a second context to save that default register state. We
1085 * can then prime every new context with that state so they all start
1086 * from the same default HW values.
1089 for_each_engine(engine
, i915
, id
) {
1090 struct intel_context
*ce
;
1091 struct i915_request
*rq
;
1093 /* We must be able to switch to something! */
1094 GEM_BUG_ON(!engine
->kernel_context
);
1095 engine
->serial
++; /* force the kernel context switch */
1097 ce
= intel_context_create(i915
->kernel_context
, engine
);
1103 rq
= intel_context_create_request(ce
);
1106 intel_context_put(ce
);
1110 err
= intel_engine_emit_ctx_wa(rq
);
1115 * Failing to program the MOCS is non-fatal.The system will not
1116 * run at peak performance. So warn the user and carry on.
1118 err
= intel_mocs_emit(rq
);
1120 dev_notice(i915
->drm
.dev
,
1121 "Failed to program MOCS registers; expect performance issues.\n");
1123 err
= intel_renderstate_emit(rq
);
1128 requests
[id
] = i915_request_get(rq
);
1129 i915_request_add(rq
);
1134 /* Flush the default context image to memory, and enable powersaving. */
1135 if (!i915_gem_load_power_context(i915
)) {
1140 for (id
= 0; id
< ARRAY_SIZE(requests
); id
++) {
1141 struct i915_request
*rq
;
1142 struct i915_vma
*state
;
1149 /* We want to be able to unbind the state from the GGTT */
1150 GEM_BUG_ON(intel_context_is_pinned(rq
->hw_context
));
1152 state
= rq
->hw_context
->state
;
1157 * As we will hold a reference to the logical state, it will
1158 * not be torn down with the context, and importantly the
1159 * object will hold onto its vma (making it possible for a
1160 * stray GTT write to corrupt our defaults). Unmap the vma
1161 * from the GTT to prevent such accidents and reclaim the
1164 err
= i915_vma_unbind(state
);
1168 i915_gem_object_lock(state
->obj
);
1169 err
= i915_gem_object_set_to_cpu_domain(state
->obj
, false);
1170 i915_gem_object_unlock(state
->obj
);
1174 i915_gem_object_set_cache_coherency(state
->obj
, I915_CACHE_LLC
);
1176 /* Check we can acquire the image of the context state */
1177 vaddr
= i915_gem_object_pin_map(state
->obj
, I915_MAP_FORCE_WB
);
1178 if (IS_ERR(vaddr
)) {
1179 err
= PTR_ERR(vaddr
);
1183 rq
->engine
->default_state
= i915_gem_object_get(state
->obj
);
1184 i915_gem_object_unpin_map(state
->obj
);
1189 * If we have to abandon now, we expect the engines to be idle
1190 * and ready to be torn-down. The quickest way we can accomplish
1191 * this is by declaring ourselves wedged.
1194 intel_gt_set_wedged(&i915
->gt
);
1196 for (id
= 0; id
< ARRAY_SIZE(requests
); id
++) {
1197 struct intel_context
*ce
;
1198 struct i915_request
*rq
;
1204 ce
= rq
->hw_context
;
1205 i915_request_put(rq
);
1206 intel_context_put(ce
);
1211 static int intel_engines_verify_workarounds(struct drm_i915_private
*i915
)
1213 struct intel_engine_cs
*engine
;
1214 enum intel_engine_id id
;
1217 if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM
))
1220 for_each_engine(engine
, i915
, id
) {
1221 if (intel_engine_verify_workarounds(engine
, "load"))
1228 int i915_gem_init(struct drm_i915_private
*dev_priv
)
1232 /* We need to fallback to 4K pages if host doesn't support huge gtt. */
1233 if (intel_vgpu_active(dev_priv
) && !intel_vgpu_has_huge_gtt(dev_priv
))
1234 mkwrite_device_info(dev_priv
)->page_sizes
=
1235 I915_GTT_PAGE_SIZE_4K
;
1237 intel_timelines_init(dev_priv
);
1239 ret
= i915_gem_init_userptr(dev_priv
);
1243 intel_uc_fetch_firmwares(&dev_priv
->gt
.uc
);
1244 intel_wopcm_init(&dev_priv
->wopcm
);
1246 /* This is just a security blanket to placate dragons.
1247 * On some systems, we very sporadically observe that the first TLBs
1248 * used by the CS may be stale, despite us poking the TLB reset. If
1249 * we hold the forcewake during initialisation these problems
1250 * just magically go away.
1252 intel_uncore_forcewake_get(&dev_priv
->uncore
, FORCEWAKE_ALL
);
1254 ret
= i915_init_ggtt(dev_priv
);
1256 GEM_BUG_ON(ret
== -EIO
);
1260 intel_gt_init(&dev_priv
->gt
);
1262 ret
= intel_engines_setup(dev_priv
);
1264 GEM_BUG_ON(ret
== -EIO
);
1268 ret
= i915_gem_init_contexts(dev_priv
);
1270 GEM_BUG_ON(ret
== -EIO
);
1274 ret
= intel_engines_init(dev_priv
);
1276 GEM_BUG_ON(ret
== -EIO
);
1280 intel_init_gt_powersave(dev_priv
);
1282 intel_uc_init(&dev_priv
->gt
.uc
);
1284 ret
= intel_gt_init_hw(&dev_priv
->gt
);
1288 /* Only when the HW is re-initialised, can we replay the requests */
1289 ret
= intel_gt_resume(&dev_priv
->gt
);
1294 * Despite its name intel_init_clock_gating applies both display
1295 * clock gating workarounds; GT mmio workarounds and the occasional
1296 * GT power context workaround. Worse, sometimes it includes a context
1297 * register workaround which we need to apply before we record the
1298 * default HW state for all contexts.
1300 * FIXME: break up the workarounds and apply them at the right time!
1302 intel_init_clock_gating(dev_priv
);
1304 ret
= intel_engines_verify_workarounds(dev_priv
);
1308 ret
= __intel_engines_record_defaults(dev_priv
);
1312 ret
= i915_inject_load_error(dev_priv
, -ENODEV
);
1316 ret
= i915_inject_load_error(dev_priv
, -EIO
);
1320 intel_uncore_forcewake_put(&dev_priv
->uncore
, FORCEWAKE_ALL
);
1325 * Unwinding is complicated by that we want to handle -EIO to mean
1326 * disable GPU submission but keep KMS alive. We want to mark the
1327 * HW as irrevisibly wedged, but keep enough state around that the
1328 * driver doesn't explode during runtime.
1331 intel_gt_set_wedged_on_init(&dev_priv
->gt
);
1332 i915_gem_suspend(dev_priv
);
1333 i915_gem_suspend_late(dev_priv
);
1335 i915_gem_drain_workqueue(dev_priv
);
1337 intel_uc_fini_hw(&dev_priv
->gt
.uc
);
1340 intel_uc_fini(&dev_priv
->gt
.uc
);
1341 intel_engines_cleanup(dev_priv
);
1345 i915_gem_driver_release__contexts(dev_priv
);
1347 intel_gt_driver_release(&dev_priv
->gt
);
1349 intel_uncore_forcewake_put(&dev_priv
->uncore
, FORCEWAKE_ALL
);
1352 intel_uc_cleanup_firmwares(&dev_priv
->gt
.uc
);
1353 i915_gem_cleanup_userptr(dev_priv
);
1354 intel_timelines_fini(dev_priv
);
1359 * Allow engines or uC initialisation to fail by marking the GPU
1360 * as wedged. But we only want to do this when the GPU is angry,
1361 * for all other failure, such as an allocation failure, bail.
1363 if (!intel_gt_is_wedged(&dev_priv
->gt
)) {
1364 i915_probe_error(dev_priv
,
1365 "Failed to initialize GPU, declaring it wedged!\n");
1366 intel_gt_set_wedged(&dev_priv
->gt
);
1369 /* Minimal basic recovery for KMS */
1370 ret
= i915_ggtt_enable_hw(dev_priv
);
1371 i915_gem_restore_gtt_mappings(dev_priv
);
1372 i915_gem_restore_fences(dev_priv
);
1373 intel_init_clock_gating(dev_priv
);
1376 i915_gem_drain_freed_objects(dev_priv
);
1380 void i915_gem_driver_register(struct drm_i915_private
*i915
)
1382 i915_gem_driver_register__shrinker(i915
);
1384 intel_engines_driver_register(i915
);
1387 void i915_gem_driver_unregister(struct drm_i915_private
*i915
)
1389 i915_gem_driver_unregister__shrinker(i915
);
1392 void i915_gem_driver_remove(struct drm_i915_private
*dev_priv
)
1394 intel_wakeref_auto_fini(&dev_priv
->ggtt
.userfault_wakeref
);
1396 i915_gem_suspend_late(dev_priv
);
1397 intel_gt_driver_remove(&dev_priv
->gt
);
1399 /* Flush any outstanding unpin_work. */
1400 i915_gem_drain_workqueue(dev_priv
);
1402 intel_uc_fini_hw(&dev_priv
->gt
.uc
);
1403 intel_uc_fini(&dev_priv
->gt
.uc
);
1405 i915_gem_drain_freed_objects(dev_priv
);
1408 void i915_gem_driver_release(struct drm_i915_private
*dev_priv
)
1410 intel_engines_cleanup(dev_priv
);
1411 i915_gem_driver_release__contexts(dev_priv
);
1412 intel_gt_driver_release(&dev_priv
->gt
);
1414 intel_wa_list_free(&dev_priv
->gt_wa_list
);
1416 intel_uc_cleanup_firmwares(&dev_priv
->gt
.uc
);
1417 i915_gem_cleanup_userptr(dev_priv
);
1418 intel_timelines_fini(dev_priv
);
1420 i915_gem_drain_freed_objects(dev_priv
);
1422 WARN_ON(!list_empty(&dev_priv
->gem
.contexts
.list
));
1425 void i915_gem_init_mmio(struct drm_i915_private
*i915
)
1427 i915_gem_sanitize(i915
);
1430 static void i915_gem_init__mm(struct drm_i915_private
*i915
)
1432 spin_lock_init(&i915
->mm
.obj_lock
);
1434 init_llist_head(&i915
->mm
.free_list
);
1436 INIT_LIST_HEAD(&i915
->mm
.purge_list
);
1437 INIT_LIST_HEAD(&i915
->mm
.shrink_list
);
1439 i915_gem_init__objects(i915
);
1442 void i915_gem_init_early(struct drm_i915_private
*dev_priv
)
1446 i915_gem_init__mm(dev_priv
);
1447 i915_gem_init__pm(dev_priv
);
1449 spin_lock_init(&dev_priv
->fb_tracking
.lock
);
1451 err
= i915_gemfs_init(dev_priv
);
1453 DRM_NOTE("Unable to create a private tmpfs mount, hugepage support will be disabled(%d).\n", err
);
1456 void i915_gem_cleanup_early(struct drm_i915_private
*dev_priv
)
1458 i915_gem_drain_freed_objects(dev_priv
);
1459 GEM_BUG_ON(!llist_empty(&dev_priv
->mm
.free_list
));
1460 GEM_BUG_ON(atomic_read(&dev_priv
->mm
.free_count
));
1461 WARN_ON(dev_priv
->mm
.shrink_count
);
1463 i915_gemfs_fini(dev_priv
);
1466 int i915_gem_freeze(struct drm_i915_private
*dev_priv
)
1468 /* Discard all purgeable objects, let userspace recover those as
1469 * required after resuming.
1471 i915_gem_shrink_all(dev_priv
);
1476 int i915_gem_freeze_late(struct drm_i915_private
*i915
)
1478 struct drm_i915_gem_object
*obj
;
1479 intel_wakeref_t wakeref
;
1482 * Called just before we write the hibernation image.
1484 * We need to update the domain tracking to reflect that the CPU
1485 * will be accessing all the pages to create and restore from the
1486 * hibernation, and so upon restoration those pages will be in the
1489 * To make sure the hibernation image contains the latest state,
1490 * we update that state just before writing out the image.
1492 * To try and reduce the hibernation image, we manually shrink
1493 * the objects as well, see i915_gem_freeze()
1496 wakeref
= intel_runtime_pm_get(&i915
->runtime_pm
);
1498 i915_gem_shrink(i915
, -1UL, NULL
, ~0);
1499 i915_gem_drain_freed_objects(i915
);
1501 list_for_each_entry(obj
, &i915
->mm
.shrink_list
, mm
.link
) {
1502 i915_gem_object_lock(obj
);
1503 WARN_ON(i915_gem_object_set_to_cpu_domain(obj
, true));
1504 i915_gem_object_unlock(obj
);
1507 intel_runtime_pm_put(&i915
->runtime_pm
, wakeref
);
1512 void i915_gem_release(struct drm_device
*dev
, struct drm_file
*file
)
1514 struct drm_i915_file_private
*file_priv
= file
->driver_priv
;
1515 struct i915_request
*request
;
1517 /* Clean up our request list when the client is going away, so that
1518 * later retire_requests won't dereference our soon-to-be-gone
1521 spin_lock(&file_priv
->mm
.lock
);
1522 list_for_each_entry(request
, &file_priv
->mm
.request_list
, client_link
)
1523 request
->file_priv
= NULL
;
1524 spin_unlock(&file_priv
->mm
.lock
);
1527 int i915_gem_open(struct drm_i915_private
*i915
, struct drm_file
*file
)
1529 struct drm_i915_file_private
*file_priv
;
1534 file_priv
= kzalloc(sizeof(*file_priv
), GFP_KERNEL
);
1538 file
->driver_priv
= file_priv
;
1539 file_priv
->dev_priv
= i915
;
1540 file_priv
->file
= file
;
1542 spin_lock_init(&file_priv
->mm
.lock
);
1543 INIT_LIST_HEAD(&file_priv
->mm
.request_list
);
1545 file_priv
->bsd_engine
= -1;
1546 file_priv
->hang_timestamp
= jiffies
;
1548 ret
= i915_gem_context_open(i915
, file
);
1555 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1556 #include "selftests/mock_gem_device.c"
1557 #include "selftests/i915_gem.c"