2 * SPDX-License-Identifier: MIT
4 * Copyright © 2014-2016 Intel Corporation
7 #include <linux/anon_inodes.h>
8 #include <linux/mman.h>
9 #include <linux/pfn_t.h>
10 #include <linux/sizes.h>
12 #include "gt/intel_gt.h"
13 #include "gt/intel_gt_requests.h"
16 #include "i915_gem_gtt.h"
17 #include "i915_gem_ioctls.h"
18 #include "i915_gem_object.h"
19 #include "i915_gem_mman.h"
20 #include "i915_trace.h"
21 #include "i915_user_extensions.h"
22 #include "i915_gem_ttm.h"
26 __vma_matches(struct vm_area_struct
*vma
, struct file
*filp
,
27 unsigned long addr
, unsigned long size
)
29 if (vma
->vm_file
!= filp
)
32 return vma
->vm_start
== addr
&&
33 (vma
->vm_end
- vma
->vm_start
) == PAGE_ALIGN(size
);
37 * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address
40 * @data: ioctl data blob
43 * While the mapping holds a reference on the contents of the object, it doesn't
44 * imply a ref on the object itself.
48 * DRM driver writers who look a this function as an example for how to do GEM
49 * mmap support, please don't implement mmap support like here. The modern way
50 * to implement DRM mmap support is with an mmap offset ioctl (like
51 * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
52 * That way debug tooling like valgrind will understand what's going on, hiding
53 * the mmap call in a driver private ioctl will break that. The i915 driver only
54 * does cpu mmaps this way because we didn't know better.
57 i915_gem_mmap_ioctl(struct drm_device
*dev
, void *data
,
58 struct drm_file
*file
)
60 struct drm_i915_private
*i915
= to_i915(dev
);
61 struct drm_i915_gem_mmap
*args
= data
;
62 struct drm_i915_gem_object
*obj
;
66 * mmap ioctl is disallowed for all discrete platforms,
67 * and for all platforms with GRAPHICS_VER > 12.
69 if (IS_DGFX(i915
) || GRAPHICS_VER(i915
) > 12)
72 if (args
->flags
& ~(I915_MMAP_WC
))
75 if (args
->flags
& I915_MMAP_WC
&& !boot_cpu_has(X86_FEATURE_PAT
))
78 obj
= i915_gem_object_lookup(file
, args
->handle
);
82 /* prime objects have no backing filp to GEM mmap
85 if (!obj
->base
.filp
) {
90 if (range_overflows(args
->offset
, args
->size
, (u64
)obj
->base
.size
)) {
95 addr
= vm_mmap(obj
->base
.filp
, 0, args
->size
,
96 PROT_READ
| PROT_WRITE
, MAP_SHARED
,
98 if (IS_ERR_VALUE(addr
))
101 if (args
->flags
& I915_MMAP_WC
) {
102 struct mm_struct
*mm
= current
->mm
;
103 struct vm_area_struct
*vma
;
105 if (mmap_write_lock_killable(mm
)) {
109 vma
= find_vma(mm
, addr
);
110 if (vma
&& __vma_matches(vma
, obj
->base
.filp
, addr
, args
->size
))
112 pgprot_writecombine(vm_get_page_prot(vma
->vm_flags
));
115 mmap_write_unlock(mm
);
116 if (IS_ERR_VALUE(addr
))
119 i915_gem_object_put(obj
);
121 args
->addr_ptr
= (u64
)addr
;
125 i915_gem_object_put(obj
);
129 static unsigned int tile_row_pages(const struct drm_i915_gem_object
*obj
)
131 return i915_gem_object_get_tile_row_size(obj
) >> PAGE_SHIFT
;
135 * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps
137 * A history of the GTT mmap interface:
139 * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to
140 * aligned and suitable for fencing, and still fit into the available
141 * mappable space left by the pinned display objects. A classic problem
142 * we called the page-fault-of-doom where we would ping-pong between
143 * two objects that could not fit inside the GTT and so the memcpy
144 * would page one object in at the expense of the other between every
147 * 1 - Objects can be any size, and have any compatible fencing (X Y, or none
148 * as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the
149 * object is too large for the available space (or simply too large
150 * for the mappable aperture!), a view is created instead and faulted
151 * into userspace. (This view is aligned and sized appropriately for
154 * 2 - Recognise WC as a separate cache domain so that we can flush the
155 * delayed writes via GTT before performing direct access via WC.
157 * 3 - Remove implicit set-domain(GTT) and synchronisation on initial
158 * pagefault; swapin remains transparent.
160 * 4 - Support multiple fault handlers per object depending on object's
161 * backing storage (a.k.a. MMAP_OFFSET).
165 * * snoopable objects cannot be accessed via the GTT. It can cause machine
166 * hangs on some architectures, corruption on others. An attempt to service
167 * a GTT page fault from a snoopable object will generate a SIGBUS.
169 * * the object must be able to fit into RAM (physical memory, though no
170 * limited to the mappable aperture).
175 * * a new GTT page fault will synchronize rendering from the GPU and flush
176 * all data to system memory. Subsequent access will not be synchronized.
178 * * all mappings are revoked on runtime device suspend.
180 * * there are only 8, 16 or 32 fence registers to share between all users
181 * (older machines require fence register for display and blitter access
182 * as well). Contention of the fence registers will cause the previous users
183 * to be unmapped and any new access will generate new page faults.
185 * * running out of memory while servicing a fault may generate a SIGBUS,
186 * rather than the expected SIGSEGV.
188 int i915_gem_mmap_gtt_version(void)
193 static inline struct i915_ggtt_view
194 compute_partial_view(const struct drm_i915_gem_object
*obj
,
198 struct i915_ggtt_view view
;
200 if (i915_gem_object_is_tiled(obj
))
201 chunk
= roundup(chunk
, tile_row_pages(obj
) ?: 1);
203 view
.type
= I915_GGTT_VIEW_PARTIAL
;
204 view
.partial
.offset
= rounddown(page_offset
, chunk
);
206 min_t(unsigned int, chunk
,
207 (obj
->base
.size
>> PAGE_SHIFT
) - view
.partial
.offset
);
209 /* If the partial covers the entire object, just create a normal VMA. */
210 if (chunk
>= obj
->base
.size
>> PAGE_SHIFT
)
211 view
.type
= I915_GGTT_VIEW_NORMAL
;
216 static vm_fault_t
i915_error_to_vmf_fault(int err
)
220 WARN_ONCE(err
, "unhandled error in %s: %i\n", __func__
, err
);
222 case -EIO
: /* shmemfs failure from swap device */
223 case -EFAULT
: /* purged object */
224 case -ENODEV
: /* bad object, how did you get here! */
225 case -ENXIO
: /* unable to access backing store (on device) */
226 return VM_FAULT_SIGBUS
;
228 case -ENOMEM
: /* our allocation failure */
233 case -ENOSPC
: /* transient failure to evict? */
238 * EBUSY is ok: this just means that another thread
239 * already did the job.
241 return VM_FAULT_NOPAGE
;
245 static vm_fault_t
vm_fault_cpu(struct vm_fault
*vmf
)
247 struct vm_area_struct
*area
= vmf
->vma
;
248 struct i915_mmap_offset
*mmo
= area
->vm_private_data
;
249 struct drm_i915_gem_object
*obj
= mmo
->obj
;
250 resource_size_t iomap
;
253 /* Sanity check that we allow writing into this object */
254 if (unlikely(i915_gem_object_is_readonly(obj
) &&
255 area
->vm_flags
& VM_WRITE
))
256 return VM_FAULT_SIGBUS
;
258 if (i915_gem_object_lock_interruptible(obj
, NULL
))
259 return VM_FAULT_NOPAGE
;
261 err
= i915_gem_object_pin_pages(obj
);
266 if (!i915_gem_object_has_struct_page(obj
)) {
267 iomap
= obj
->mm
.region
->iomap
.base
;
268 iomap
-= obj
->mm
.region
->region
.start
;
271 /* PTEs are revoked in obj->ops->put_pages() */
272 err
= remap_io_sg(area
,
273 area
->vm_start
, area
->vm_end
- area
->vm_start
,
274 obj
->mm
.pages
->sgl
, iomap
);
276 if (area
->vm_flags
& VM_WRITE
) {
277 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj
));
278 obj
->mm
.dirty
= true;
281 i915_gem_object_unpin_pages(obj
);
284 i915_gem_object_unlock(obj
);
285 return i915_error_to_vmf_fault(err
);
288 static vm_fault_t
vm_fault_gtt(struct vm_fault
*vmf
)
290 #define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT)
291 struct vm_area_struct
*area
= vmf
->vma
;
292 struct i915_mmap_offset
*mmo
= area
->vm_private_data
;
293 struct drm_i915_gem_object
*obj
= mmo
->obj
;
294 struct drm_device
*dev
= obj
->base
.dev
;
295 struct drm_i915_private
*i915
= to_i915(dev
);
296 struct intel_runtime_pm
*rpm
= &i915
->runtime_pm
;
297 struct i915_ggtt
*ggtt
= &i915
->ggtt
;
298 bool write
= area
->vm_flags
& VM_WRITE
;
299 struct i915_gem_ww_ctx ww
;
300 intel_wakeref_t wakeref
;
301 struct i915_vma
*vma
;
306 /* We don't use vmf->pgoff since that has the fake offset */
307 page_offset
= (vmf
->address
- area
->vm_start
) >> PAGE_SHIFT
;
309 trace_i915_gem_object_fault(obj
, page_offset
, true, write
);
311 wakeref
= intel_runtime_pm_get(rpm
);
313 i915_gem_ww_ctx_init(&ww
, true);
315 ret
= i915_gem_object_lock(obj
, &ww
);
319 /* Sanity check that we allow writing into this object */
320 if (i915_gem_object_is_readonly(obj
) && write
) {
325 ret
= i915_gem_object_pin_pages(obj
);
329 ret
= intel_gt_reset_trylock(ggtt
->vm
.gt
, &srcu
);
333 /* Now pin it into the GTT as needed */
334 vma
= i915_gem_object_ggtt_pin_ww(obj
, &ww
, NULL
, 0, 0,
336 PIN_NONBLOCK
/* NOWARN */ |
338 if (IS_ERR(vma
) && vma
!= ERR_PTR(-EDEADLK
)) {
339 /* Use a partial view if it is bigger than available space */
340 struct i915_ggtt_view view
=
341 compute_partial_view(obj
, page_offset
, MIN_CHUNK_PAGES
);
344 flags
= PIN_MAPPABLE
| PIN_NOSEARCH
;
345 if (view
.type
== I915_GGTT_VIEW_NORMAL
)
346 flags
|= PIN_NONBLOCK
; /* avoid warnings for pinned */
349 * Userspace is now writing through an untracked VMA, abandon
350 * all hope that the hardware is able to track future writes.
353 vma
= i915_gem_object_ggtt_pin_ww(obj
, &ww
, &view
, 0, 0, flags
);
354 if (IS_ERR(vma
) && vma
!= ERR_PTR(-EDEADLK
)) {
355 flags
= PIN_MAPPABLE
;
356 view
.type
= I915_GGTT_VIEW_PARTIAL
;
357 vma
= i915_gem_object_ggtt_pin_ww(obj
, &ww
, &view
, 0, 0, flags
);
360 /* The entire mappable GGTT is pinned? Unexpected! */
361 GEM_BUG_ON(vma
== ERR_PTR(-ENOSPC
));
368 /* Access to snoopable pages through the GTT is incoherent. */
369 if (obj
->cache_level
!= I915_CACHE_NONE
&& !HAS_LLC(i915
)) {
374 ret
= i915_vma_pin_fence(vma
);
378 /* Finally, remap it using the new GTT offset */
379 ret
= remap_io_mapping(area
,
380 area
->vm_start
+ (vma
->ggtt_view
.partial
.offset
<< PAGE_SHIFT
),
381 (ggtt
->gmadr
.start
+ vma
->node
.start
) >> PAGE_SHIFT
,
382 min_t(u64
, vma
->size
, area
->vm_end
- area
->vm_start
),
387 assert_rpm_wakelock_held(rpm
);
389 /* Mark as being mmapped into userspace for later revocation */
390 mutex_lock(&i915
->ggtt
.vm
.mutex
);
391 if (!i915_vma_set_userfault(vma
) && !obj
->userfault_count
++)
392 list_add(&obj
->userfault_link
, &i915
->ggtt
.userfault_list
);
393 mutex_unlock(&i915
->ggtt
.vm
.mutex
);
395 /* Track the mmo associated with the fenced vma */
398 if (IS_ACTIVE(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND
))
399 intel_wakeref_auto(&i915
->ggtt
.userfault_wakeref
,
400 msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND
));
403 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj
));
404 i915_vma_set_ggtt_write(vma
);
405 obj
->mm
.dirty
= true;
409 i915_vma_unpin_fence(vma
);
411 __i915_vma_unpin(vma
);
413 intel_gt_reset_unlock(ggtt
->vm
.gt
, srcu
);
415 i915_gem_object_unpin_pages(obj
);
417 if (ret
== -EDEADLK
) {
418 ret
= i915_gem_ww_ctx_backoff(&ww
);
422 i915_gem_ww_ctx_fini(&ww
);
423 intel_runtime_pm_put(rpm
, wakeref
);
424 return i915_error_to_vmf_fault(ret
);
428 vm_access(struct vm_area_struct
*area
, unsigned long addr
,
429 void *buf
, int len
, int write
)
431 struct i915_mmap_offset
*mmo
= area
->vm_private_data
;
432 struct drm_i915_gem_object
*obj
= mmo
->obj
;
433 struct i915_gem_ww_ctx ww
;
437 if (i915_gem_object_is_readonly(obj
) && write
)
440 addr
-= area
->vm_start
;
441 if (range_overflows_t(u64
, addr
, len
, obj
->base
.size
))
444 i915_gem_ww_ctx_init(&ww
, true);
446 err
= i915_gem_object_lock(obj
, &ww
);
450 /* As this is primarily for debugging, let's focus on simplicity */
451 vaddr
= i915_gem_object_pin_map(obj
, I915_MAP_FORCE_WC
);
453 err
= PTR_ERR(vaddr
);
458 memcpy(vaddr
+ addr
, buf
, len
);
459 __i915_gem_object_flush_map(obj
, addr
, len
);
461 memcpy(buf
, vaddr
+ addr
, len
);
464 i915_gem_object_unpin_map(obj
);
466 if (err
== -EDEADLK
) {
467 err
= i915_gem_ww_ctx_backoff(&ww
);
471 i915_gem_ww_ctx_fini(&ww
);
479 void __i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object
*obj
)
481 struct i915_vma
*vma
;
483 GEM_BUG_ON(!obj
->userfault_count
);
485 for_each_ggtt_vma(vma
, obj
)
486 i915_vma_revoke_mmap(vma
);
488 GEM_BUG_ON(obj
->userfault_count
);
492 * It is vital that we remove the page mapping if we have mapped a tiled
493 * object through the GTT and then lose the fence register due to
494 * resource pressure. Similarly if the object has been moved out of the
495 * aperture, than pages mapped into userspace must be revoked. Removing the
496 * mapping will then trigger a page fault on the next user access, allowing
497 * fixup by vm_fault_gtt().
499 void i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object
*obj
)
501 struct drm_i915_private
*i915
= to_i915(obj
->base
.dev
);
502 intel_wakeref_t wakeref
;
505 * Serialisation between user GTT access and our code depends upon
506 * revoking the CPU's PTE whilst the mutex is held. The next user
507 * pagefault then has to wait until we release the mutex.
509 * Note that RPM complicates somewhat by adding an additional
510 * requirement that operations to the GGTT be made holding the RPM
513 wakeref
= intel_runtime_pm_get(&i915
->runtime_pm
);
514 mutex_lock(&i915
->ggtt
.vm
.mutex
);
516 if (!obj
->userfault_count
)
519 __i915_gem_object_release_mmap_gtt(obj
);
522 * Ensure that the CPU's PTE are revoked and there are not outstanding
523 * memory transactions from userspace before we return. The TLB
524 * flushing implied above by changing the PTE above *should* be
525 * sufficient, an extra barrier here just provides us with a bit
526 * of paranoid documentation about our requirement to serialise
527 * memory writes before touching registers / GSM.
532 mutex_unlock(&i915
->ggtt
.vm
.mutex
);
533 intel_runtime_pm_put(&i915
->runtime_pm
, wakeref
);
536 void i915_gem_object_release_mmap_offset(struct drm_i915_gem_object
*obj
)
538 struct i915_mmap_offset
*mmo
, *mn
;
540 spin_lock(&obj
->mmo
.lock
);
541 rbtree_postorder_for_each_entry_safe(mmo
, mn
,
542 &obj
->mmo
.offsets
, offset
) {
544 * vma_node_unmap for GTT mmaps handled already in
545 * __i915_gem_object_release_mmap_gtt
547 if (mmo
->mmap_type
== I915_MMAP_TYPE_GTT
)
550 spin_unlock(&obj
->mmo
.lock
);
551 drm_vma_node_unmap(&mmo
->vma_node
,
552 obj
->base
.dev
->anon_inode
->i_mapping
);
553 spin_lock(&obj
->mmo
.lock
);
555 spin_unlock(&obj
->mmo
.lock
);
558 static struct i915_mmap_offset
*
559 lookup_mmo(struct drm_i915_gem_object
*obj
,
560 enum i915_mmap_type mmap_type
)
564 spin_lock(&obj
->mmo
.lock
);
565 rb
= obj
->mmo
.offsets
.rb_node
;
567 struct i915_mmap_offset
*mmo
=
568 rb_entry(rb
, typeof(*mmo
), offset
);
570 if (mmo
->mmap_type
== mmap_type
) {
571 spin_unlock(&obj
->mmo
.lock
);
575 if (mmo
->mmap_type
< mmap_type
)
580 spin_unlock(&obj
->mmo
.lock
);
585 static struct i915_mmap_offset
*
586 insert_mmo(struct drm_i915_gem_object
*obj
, struct i915_mmap_offset
*mmo
)
588 struct rb_node
*rb
, **p
;
590 spin_lock(&obj
->mmo
.lock
);
592 p
= &obj
->mmo
.offsets
.rb_node
;
594 struct i915_mmap_offset
*pos
;
597 pos
= rb_entry(rb
, typeof(*pos
), offset
);
599 if (pos
->mmap_type
== mmo
->mmap_type
) {
600 spin_unlock(&obj
->mmo
.lock
);
601 drm_vma_offset_remove(obj
->base
.dev
->vma_offset_manager
,
607 if (pos
->mmap_type
< mmo
->mmap_type
)
612 rb_link_node(&mmo
->offset
, rb
, p
);
613 rb_insert_color(&mmo
->offset
, &obj
->mmo
.offsets
);
614 spin_unlock(&obj
->mmo
.lock
);
619 static struct i915_mmap_offset
*
620 mmap_offset_attach(struct drm_i915_gem_object
*obj
,
621 enum i915_mmap_type mmap_type
,
622 struct drm_file
*file
)
624 struct drm_i915_private
*i915
= to_i915(obj
->base
.dev
);
625 struct i915_mmap_offset
*mmo
;
628 GEM_BUG_ON(obj
->ops
->mmap_offset
|| obj
->ops
->mmap_ops
);
630 mmo
= lookup_mmo(obj
, mmap_type
);
634 mmo
= kmalloc(sizeof(*mmo
), GFP_KERNEL
);
636 return ERR_PTR(-ENOMEM
);
639 mmo
->mmap_type
= mmap_type
;
640 drm_vma_node_reset(&mmo
->vma_node
);
642 err
= drm_vma_offset_add(obj
->base
.dev
->vma_offset_manager
,
643 &mmo
->vma_node
, obj
->base
.size
/ PAGE_SIZE
);
647 /* Attempt to reap some mmap space from dead objects */
648 err
= intel_gt_retire_requests_timeout(&i915
->gt
, MAX_SCHEDULE_TIMEOUT
,
653 i915_gem_drain_freed_objects(i915
);
654 err
= drm_vma_offset_add(obj
->base
.dev
->vma_offset_manager
,
655 &mmo
->vma_node
, obj
->base
.size
/ PAGE_SIZE
);
660 mmo
= insert_mmo(obj
, mmo
);
661 GEM_BUG_ON(lookup_mmo(obj
, mmap_type
) != mmo
);
664 drm_vma_node_allow(&mmo
->vma_node
, file
);
673 __assign_mmap_offset(struct drm_i915_gem_object
*obj
,
674 enum i915_mmap_type mmap_type
,
675 u64
*offset
, struct drm_file
*file
)
677 struct i915_mmap_offset
*mmo
;
679 if (i915_gem_object_never_mmap(obj
))
682 if (obj
->ops
->mmap_offset
) {
683 if (mmap_type
!= I915_MMAP_TYPE_FIXED
)
686 *offset
= obj
->ops
->mmap_offset(obj
);
690 if (mmap_type
== I915_MMAP_TYPE_FIXED
)
693 if (mmap_type
!= I915_MMAP_TYPE_GTT
&&
694 !i915_gem_object_has_struct_page(obj
) &&
695 !i915_gem_object_has_iomem(obj
))
698 mmo
= mmap_offset_attach(obj
, mmap_type
, file
);
702 *offset
= drm_vma_node_offset_addr(&mmo
->vma_node
);
707 __assign_mmap_offset_handle(struct drm_file
*file
,
709 enum i915_mmap_type mmap_type
,
712 struct drm_i915_gem_object
*obj
;
715 obj
= i915_gem_object_lookup(file
, handle
);
719 err
= i915_gem_object_lock_interruptible(obj
, NULL
);
722 err
= __assign_mmap_offset(obj
, mmap_type
, offset
, file
);
723 i915_gem_object_unlock(obj
);
725 i915_gem_object_put(obj
);
730 i915_gem_dumb_mmap_offset(struct drm_file
*file
,
731 struct drm_device
*dev
,
735 enum i915_mmap_type mmap_type
;
737 if (HAS_LMEM(to_i915(dev
)))
738 mmap_type
= I915_MMAP_TYPE_FIXED
;
739 else if (boot_cpu_has(X86_FEATURE_PAT
))
740 mmap_type
= I915_MMAP_TYPE_WC
;
741 else if (!i915_ggtt_has_aperture(&to_i915(dev
)->ggtt
))
744 mmap_type
= I915_MMAP_TYPE_GTT
;
746 return __assign_mmap_offset_handle(file
, handle
, mmap_type
, offset
);
750 * i915_gem_mmap_offset_ioctl - prepare an object for GTT mmap'ing
752 * @data: GTT mapping ioctl data
753 * @file: GEM object info
755 * Simply returns the fake offset to userspace so it can mmap it.
756 * The mmap call will end up in drm_gem_mmap(), which will set things
757 * up so we can get faults in the handler above.
759 * The fault handler will take care of binding the object into the GTT
760 * (since it may have been evicted to make room for something), allocating
761 * a fence register, and mapping the appropriate aperture address into
765 i915_gem_mmap_offset_ioctl(struct drm_device
*dev
, void *data
,
766 struct drm_file
*file
)
768 struct drm_i915_private
*i915
= to_i915(dev
);
769 struct drm_i915_gem_mmap_offset
*args
= data
;
770 enum i915_mmap_type type
;
774 * Historically we failed to check args.pad and args.offset
775 * and so we cannot use those fields for user input and we cannot
776 * add -EINVAL for them as the ABI is fixed, i.e. old userspace
777 * may be feeding in garbage in those fields.
779 * if (args->pad) return -EINVAL; is verbotten!
782 err
= i915_user_extensions(u64_to_user_ptr(args
->extensions
),
787 switch (args
->flags
) {
788 case I915_MMAP_OFFSET_GTT
:
789 if (!i915_ggtt_has_aperture(&i915
->ggtt
))
791 type
= I915_MMAP_TYPE_GTT
;
794 case I915_MMAP_OFFSET_WC
:
795 if (!boot_cpu_has(X86_FEATURE_PAT
))
797 type
= I915_MMAP_TYPE_WC
;
800 case I915_MMAP_OFFSET_WB
:
801 type
= I915_MMAP_TYPE_WB
;
804 case I915_MMAP_OFFSET_UC
:
805 if (!boot_cpu_has(X86_FEATURE_PAT
))
807 type
= I915_MMAP_TYPE_UC
;
810 case I915_MMAP_OFFSET_FIXED
:
811 type
= I915_MMAP_TYPE_FIXED
;
818 return __assign_mmap_offset_handle(file
, args
->handle
, type
, &args
->offset
);
821 static void vm_open(struct vm_area_struct
*vma
)
823 struct i915_mmap_offset
*mmo
= vma
->vm_private_data
;
824 struct drm_i915_gem_object
*obj
= mmo
->obj
;
827 i915_gem_object_get(obj
);
830 static void vm_close(struct vm_area_struct
*vma
)
832 struct i915_mmap_offset
*mmo
= vma
->vm_private_data
;
833 struct drm_i915_gem_object
*obj
= mmo
->obj
;
836 i915_gem_object_put(obj
);
839 static const struct vm_operations_struct vm_ops_gtt
= {
840 .fault
= vm_fault_gtt
,
846 static const struct vm_operations_struct vm_ops_cpu
= {
847 .fault
= vm_fault_cpu
,
853 static int singleton_release(struct inode
*inode
, struct file
*file
)
855 struct drm_i915_private
*i915
= file
->private_data
;
857 cmpxchg(&i915
->gem
.mmap_singleton
, file
, NULL
);
858 drm_dev_put(&i915
->drm
);
863 static const struct file_operations singleton_fops
= {
864 .owner
= THIS_MODULE
,
865 .release
= singleton_release
,
868 static struct file
*mmap_singleton(struct drm_i915_private
*i915
)
873 file
= READ_ONCE(i915
->gem
.mmap_singleton
);
874 if (file
&& !get_file_rcu(file
))
880 file
= anon_inode_getfile("i915.gem", &singleton_fops
, i915
, O_RDWR
);
884 /* Everyone shares a single global address space */
885 file
->f_mapping
= i915
->drm
.anon_inode
->i_mapping
;
887 smp_store_mb(i915
->gem
.mmap_singleton
, file
);
888 drm_dev_get(&i915
->drm
);
894 * This overcomes the limitation in drm_gem_mmap's assignment of a
895 * drm_gem_object as the vma->vm_private_data. Since we need to
896 * be able to resolve multiple mmap offsets which could be tied
897 * to a single gem object.
899 int i915_gem_mmap(struct file
*filp
, struct vm_area_struct
*vma
)
901 struct drm_vma_offset_node
*node
;
902 struct drm_file
*priv
= filp
->private_data
;
903 struct drm_device
*dev
= priv
->minor
->dev
;
904 struct drm_i915_gem_object
*obj
= NULL
;
905 struct i915_mmap_offset
*mmo
= NULL
;
908 if (drm_dev_is_unplugged(dev
))
912 drm_vma_offset_lock_lookup(dev
->vma_offset_manager
);
913 node
= drm_vma_offset_exact_lookup_locked(dev
->vma_offset_manager
,
916 if (node
&& drm_vma_node_is_allowed(node
, priv
)) {
918 * Skip 0-refcnted objects as it is in the process of being
919 * destroyed and will be invalid when the vma manager lock
922 if (!node
->driver_private
) {
923 mmo
= container_of(node
, struct i915_mmap_offset
, vma_node
);
924 obj
= i915_gem_object_get_rcu(mmo
->obj
);
926 GEM_BUG_ON(obj
&& obj
->ops
->mmap_ops
);
928 obj
= i915_gem_object_get_rcu
929 (container_of(node
, struct drm_i915_gem_object
,
932 GEM_BUG_ON(obj
&& !obj
->ops
->mmap_ops
);
935 drm_vma_offset_unlock_lookup(dev
->vma_offset_manager
);
938 return node
? -EACCES
: -EINVAL
;
940 if (i915_gem_object_is_readonly(obj
)) {
941 if (vma
->vm_flags
& VM_WRITE
) {
942 i915_gem_object_put(obj
);
945 vma
->vm_flags
&= ~VM_MAYWRITE
;
948 anon
= mmap_singleton(to_i915(dev
));
950 i915_gem_object_put(obj
);
951 return PTR_ERR(anon
);
954 vma
->vm_flags
|= VM_PFNMAP
| VM_DONTEXPAND
| VM_DONTDUMP
| VM_IO
;
957 * We keep the ref on mmo->obj, not vm_file, but we require
958 * vma->vm_file->f_mapping, see vma_link(), for later revocation.
959 * Our userspace is accustomed to having per-file resource cleanup
960 * (i.e. contexts, objects and requests) on their close(fd), which
961 * requires avoiding extraneous references to their filp, hence why
962 * we prefer to use an anonymous file for their mmaps.
964 vma_set_file(vma
, anon
);
965 /* Drop the initial creation reference, the vma is now holding one. */
968 if (obj
->ops
->mmap_ops
) {
969 vma
->vm_page_prot
= pgprot_decrypted(vm_get_page_prot(vma
->vm_flags
));
970 vma
->vm_ops
= obj
->ops
->mmap_ops
;
971 vma
->vm_private_data
= node
->driver_private
;
975 vma
->vm_private_data
= mmo
;
977 switch (mmo
->mmap_type
) {
978 case I915_MMAP_TYPE_WC
:
980 pgprot_writecombine(vm_get_page_prot(vma
->vm_flags
));
981 vma
->vm_ops
= &vm_ops_cpu
;
984 case I915_MMAP_TYPE_FIXED
:
987 case I915_MMAP_TYPE_WB
:
988 vma
->vm_page_prot
= vm_get_page_prot(vma
->vm_flags
);
989 vma
->vm_ops
= &vm_ops_cpu
;
992 case I915_MMAP_TYPE_UC
:
994 pgprot_noncached(vm_get_page_prot(vma
->vm_flags
));
995 vma
->vm_ops
= &vm_ops_cpu
;
998 case I915_MMAP_TYPE_GTT
:
1000 pgprot_writecombine(vm_get_page_prot(vma
->vm_flags
));
1001 vma
->vm_ops
= &vm_ops_gtt
;
1004 vma
->vm_page_prot
= pgprot_decrypted(vma
->vm_page_prot
);
1009 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1010 #include "selftests/i915_gem_mman.c"