]>
Commit | Line | Data |
---|---|---|
b414fcd5 CW |
1 | /* |
2 | * SPDX-License-Identifier: MIT | |
3 | * | |
4 | * Copyright © 2014-2016 Intel Corporation | |
5 | */ | |
6 | ||
f17b8980 | 7 | #include <linux/anon_inodes.h> |
b414fcd5 | 8 | #include <linux/mman.h> |
cc662126 | 9 | #include <linux/pfn_t.h> |
b414fcd5 CW |
10 | #include <linux/sizes.h> |
11 | ||
cb823ed9 | 12 | #include "gt/intel_gt.h" |
66101975 | 13 | #include "gt/intel_gt_requests.h" |
cb823ed9 | 14 | |
b414fcd5 CW |
15 | #include "i915_drv.h" |
16 | #include "i915_gem_gtt.h" | |
17 | #include "i915_gem_ioctls.h" | |
18 | #include "i915_gem_object.h" | |
cc662126 | 19 | #include "i915_gem_mman.h" |
a09d9a80 | 20 | #include "i915_trace.h" |
126d5de3 | 21 | #include "i915_user_extensions.h" |
cf3e3e86 | 22 | #include "i915_gem_ttm.h" |
b414fcd5 | 23 | #include "i915_vma.h" |
b414fcd5 CW |
24 | |
25 | static inline bool | |
26 | __vma_matches(struct vm_area_struct *vma, struct file *filp, | |
27 | unsigned long addr, unsigned long size) | |
28 | { | |
29 | if (vma->vm_file != filp) | |
30 | return false; | |
31 | ||
32 | return vma->vm_start == addr && | |
33 | (vma->vm_end - vma->vm_start) == PAGE_ALIGN(size); | |
34 | } | |
35 | ||
36 | /** | |
37 | * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address | |
38 | * it is mapped to. | |
39 | * @dev: drm device | |
40 | * @data: ioctl data blob | |
41 | * @file: drm file | |
42 | * | |
43 | * While the mapping holds a reference on the contents of the object, it doesn't | |
44 | * imply a ref on the object itself. | |
45 | * | |
46 | * IMPORTANT: | |
47 | * | |
48 | * DRM driver writers who look a this function as an example for how to do GEM | |
49 | * mmap support, please don't implement mmap support like here. The modern way | |
50 | * to implement DRM mmap support is with an mmap offset ioctl (like | |
51 | * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly. | |
52 | * That way debug tooling like valgrind will understand what's going on, hiding | |
53 | * the mmap call in a driver private ioctl will break that. The i915 driver only | |
54 | * does cpu mmaps this way because we didn't know better. | |
55 | */ | |
56 | int | |
57 | i915_gem_mmap_ioctl(struct drm_device *dev, void *data, | |
58 | struct drm_file *file) | |
59 | { | |
35cbd91e | 60 | struct drm_i915_private *i915 = to_i915(dev); |
b414fcd5 CW |
61 | struct drm_i915_gem_mmap *args = data; |
62 | struct drm_i915_gem_object *obj; | |
63 | unsigned long addr; | |
64 | ||
d3f3baa3 TH |
65 | /* |
66 | * mmap ioctl is disallowed for all discrete platforms, | |
67 | * and for all platforms with GRAPHICS_VER > 12. | |
35cbd91e | 68 | */ |
1625a893 | 69 | if (IS_DGFX(i915) || GRAPHICS_VER_FULL(i915) > IP_VER(12, 0)) |
35cbd91e ML |
70 | return -EOPNOTSUPP; |
71 | ||
b414fcd5 CW |
72 | if (args->flags & ~(I915_MMAP_WC)) |
73 | return -EINVAL; | |
74 | ||
75 | if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT)) | |
76 | return -ENODEV; | |
77 | ||
78 | obj = i915_gem_object_lookup(file, args->handle); | |
79 | if (!obj) | |
80 | return -ENOENT; | |
81 | ||
82 | /* prime objects have no backing filp to GEM mmap | |
83 | * pages from. | |
84 | */ | |
85 | if (!obj->base.filp) { | |
86 | addr = -ENXIO; | |
87 | goto err; | |
88 | } | |
89 | ||
90 | if (range_overflows(args->offset, args->size, (u64)obj->base.size)) { | |
91 | addr = -EINVAL; | |
92 | goto err; | |
93 | } | |
94 | ||
95 | addr = vm_mmap(obj->base.filp, 0, args->size, | |
96 | PROT_READ | PROT_WRITE, MAP_SHARED, | |
97 | args->offset); | |
98 | if (IS_ERR_VALUE(addr)) | |
99 | goto err; | |
100 | ||
101 | if (args->flags & I915_MMAP_WC) { | |
102 | struct mm_struct *mm = current->mm; | |
103 | struct vm_area_struct *vma; | |
104 | ||
d8ed45c5 | 105 | if (mmap_write_lock_killable(mm)) { |
b414fcd5 CW |
106 | addr = -EINTR; |
107 | goto err; | |
108 | } | |
109 | vma = find_vma(mm, addr); | |
110 | if (vma && __vma_matches(vma, obj->base.filp, addr, args->size)) | |
111 | vma->vm_page_prot = | |
112 | pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); | |
113 | else | |
114 | addr = -ENOMEM; | |
d8ed45c5 | 115 | mmap_write_unlock(mm); |
b414fcd5 CW |
116 | if (IS_ERR_VALUE(addr)) |
117 | goto err; | |
b414fcd5 CW |
118 | } |
119 | i915_gem_object_put(obj); | |
120 | ||
121 | args->addr_ptr = (u64)addr; | |
122 | return 0; | |
123 | ||
124 | err: | |
125 | i915_gem_object_put(obj); | |
126 | return addr; | |
127 | } | |
128 | ||
129 | static unsigned int tile_row_pages(const struct drm_i915_gem_object *obj) | |
130 | { | |
131 | return i915_gem_object_get_tile_row_size(obj) >> PAGE_SHIFT; | |
132 | } | |
133 | ||
134 | /** | |
135 | * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps | |
136 | * | |
137 | * A history of the GTT mmap interface: | |
138 | * | |
139 | * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to | |
140 | * aligned and suitable for fencing, and still fit into the available | |
141 | * mappable space left by the pinned display objects. A classic problem | |
142 | * we called the page-fault-of-doom where we would ping-pong between | |
143 | * two objects that could not fit inside the GTT and so the memcpy | |
144 | * would page one object in at the expense of the other between every | |
145 | * single byte. | |
146 | * | |
147 | * 1 - Objects can be any size, and have any compatible fencing (X Y, or none | |
148 | * as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the | |
149 | * object is too large for the available space (or simply too large | |
150 | * for the mappable aperture!), a view is created instead and faulted | |
151 | * into userspace. (This view is aligned and sized appropriately for | |
152 | * fenced access.) | |
153 | * | |
154 | * 2 - Recognise WC as a separate cache domain so that we can flush the | |
155 | * delayed writes via GTT before performing direct access via WC. | |
156 | * | |
157 | * 3 - Remove implicit set-domain(GTT) and synchronisation on initial | |
158 | * pagefault; swapin remains transparent. | |
159 | * | |
cc662126 AJ |
160 | * 4 - Support multiple fault handlers per object depending on object's |
161 | * backing storage (a.k.a. MMAP_OFFSET). | |
162 | * | |
b414fcd5 CW |
163 | * Restrictions: |
164 | * | |
165 | * * snoopable objects cannot be accessed via the GTT. It can cause machine | |
166 | * hangs on some architectures, corruption on others. An attempt to service | |
167 | * a GTT page fault from a snoopable object will generate a SIGBUS. | |
168 | * | |
169 | * * the object must be able to fit into RAM (physical memory, though no | |
170 | * limited to the mappable aperture). | |
171 | * | |
172 | * | |
173 | * Caveats: | |
174 | * | |
175 | * * a new GTT page fault will synchronize rendering from the GPU and flush | |
176 | * all data to system memory. Subsequent access will not be synchronized. | |
177 | * | |
178 | * * all mappings are revoked on runtime device suspend. | |
179 | * | |
180 | * * there are only 8, 16 or 32 fence registers to share between all users | |
181 | * (older machines require fence register for display and blitter access | |
182 | * as well). Contention of the fence registers will cause the previous users | |
183 | * to be unmapped and any new access will generate new page faults. | |
184 | * | |
185 | * * running out of memory while servicing a fault may generate a SIGBUS, | |
186 | * rather than the expected SIGSEGV. | |
187 | */ | |
188 | int i915_gem_mmap_gtt_version(void) | |
189 | { | |
cc662126 | 190 | return 4; |
b414fcd5 CW |
191 | } |
192 | ||
193 | static inline struct i915_ggtt_view | |
194 | compute_partial_view(const struct drm_i915_gem_object *obj, | |
195 | pgoff_t page_offset, | |
196 | unsigned int chunk) | |
197 | { | |
198 | struct i915_ggtt_view view; | |
199 | ||
200 | if (i915_gem_object_is_tiled(obj)) | |
ed52c62d | 201 | chunk = roundup(chunk, tile_row_pages(obj) ?: 1); |
b414fcd5 CW |
202 | |
203 | view.type = I915_GGTT_VIEW_PARTIAL; | |
204 | view.partial.offset = rounddown(page_offset, chunk); | |
205 | view.partial.size = | |
206 | min_t(unsigned int, chunk, | |
207 | (obj->base.size >> PAGE_SHIFT) - view.partial.offset); | |
208 | ||
209 | /* If the partial covers the entire object, just create a normal VMA. */ | |
210 | if (chunk >= obj->base.size >> PAGE_SHIFT) | |
211 | view.type = I915_GGTT_VIEW_NORMAL; | |
212 | ||
213 | return view; | |
214 | } | |
215 | ||
cc662126 AJ |
216 | static vm_fault_t i915_error_to_vmf_fault(int err) |
217 | { | |
218 | switch (err) { | |
219 | default: | |
220 | WARN_ONCE(err, "unhandled error in %s: %i\n", __func__, err); | |
df561f66 | 221 | fallthrough; |
cc662126 AJ |
222 | case -EIO: /* shmemfs failure from swap device */ |
223 | case -EFAULT: /* purged object */ | |
224 | case -ENODEV: /* bad object, how did you get here! */ | |
4e598fad | 225 | case -ENXIO: /* unable to access backing store (on device) */ |
cc662126 AJ |
226 | return VM_FAULT_SIGBUS; |
227 | ||
cc662126 AJ |
228 | case -ENOMEM: /* our allocation failure */ |
229 | return VM_FAULT_OOM; | |
230 | ||
231 | case 0: | |
232 | case -EAGAIN: | |
552e01f6 | 233 | case -ENOSPC: /* transient failure to evict? */ |
cc662126 AJ |
234 | case -ERESTARTSYS: |
235 | case -EINTR: | |
236 | case -EBUSY: | |
237 | /* | |
238 | * EBUSY is ok: this just means that another thread | |
239 | * already did the job. | |
240 | */ | |
241 | return VM_FAULT_NOPAGE; | |
242 | } | |
243 | } | |
244 | ||
245 | static vm_fault_t vm_fault_cpu(struct vm_fault *vmf) | |
246 | { | |
247 | struct vm_area_struct *area = vmf->vma; | |
248 | struct i915_mmap_offset *mmo = area->vm_private_data; | |
249 | struct drm_i915_gem_object *obj = mmo->obj; | |
4e598fad | 250 | resource_size_t iomap; |
cc662126 AJ |
251 | int err; |
252 | ||
cc662126 | 253 | /* Sanity check that we allow writing into this object */ |
1764b992 AJ |
254 | if (unlikely(i915_gem_object_is_readonly(obj) && |
255 | area->vm_flags & VM_WRITE)) | |
256 | return VM_FAULT_SIGBUS; | |
cc662126 | 257 | |
9fa1f478 ML |
258 | if (i915_gem_object_lock_interruptible(obj, NULL)) |
259 | return VM_FAULT_NOPAGE; | |
260 | ||
cc662126 AJ |
261 | err = i915_gem_object_pin_pages(obj); |
262 | if (err) | |
1764b992 | 263 | goto out; |
cc662126 | 264 | |
4e598fad | 265 | iomap = -1; |
c471748d | 266 | if (!i915_gem_object_has_struct_page(obj)) { |
4e598fad AJ |
267 | iomap = obj->mm.region->iomap.base; |
268 | iomap -= obj->mm.region->region.start; | |
269 | } | |
270 | ||
cc662126 | 271 | /* PTEs are revoked in obj->ops->put_pages() */ |
4e598fad AJ |
272 | err = remap_io_sg(area, |
273 | area->vm_start, area->vm_end - area->vm_start, | |
274 | obj->mm.pages->sgl, iomap); | |
cc662126 | 275 | |
1764b992 | 276 | if (area->vm_flags & VM_WRITE) { |
cc662126 | 277 | GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); |
cc662126 AJ |
278 | obj->mm.dirty = true; |
279 | } | |
280 | ||
281 | i915_gem_object_unpin_pages(obj); | |
282 | ||
1764b992 | 283 | out: |
9fa1f478 | 284 | i915_gem_object_unlock(obj); |
1764b992 | 285 | return i915_error_to_vmf_fault(err); |
cc662126 AJ |
286 | } |
287 | ||
288 | static vm_fault_t vm_fault_gtt(struct vm_fault *vmf) | |
b414fcd5 CW |
289 | { |
290 | #define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT) | |
291 | struct vm_area_struct *area = vmf->vma; | |
cc662126 AJ |
292 | struct i915_mmap_offset *mmo = area->vm_private_data; |
293 | struct drm_i915_gem_object *obj = mmo->obj; | |
b414fcd5 CW |
294 | struct drm_device *dev = obj->base.dev; |
295 | struct drm_i915_private *i915 = to_i915(dev); | |
d858d569 | 296 | struct intel_runtime_pm *rpm = &i915->runtime_pm; |
b414fcd5 CW |
297 | struct i915_ggtt *ggtt = &i915->ggtt; |
298 | bool write = area->vm_flags & VM_WRITE; | |
3c0ffa27 | 299 | struct i915_gem_ww_ctx ww; |
b414fcd5 CW |
300 | intel_wakeref_t wakeref; |
301 | struct i915_vma *vma; | |
302 | pgoff_t page_offset; | |
303 | int srcu; | |
304 | int ret; | |
305 | ||
b414fcd5 CW |
306 | /* We don't use vmf->pgoff since that has the fake offset */ |
307 | page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT; | |
308 | ||
309 | trace_i915_gem_object_fault(obj, page_offset, true, write); | |
310 | ||
3c0ffa27 ML |
311 | wakeref = intel_runtime_pm_get(rpm); |
312 | ||
313 | i915_gem_ww_ctx_init(&ww, true); | |
314 | retry: | |
315 | ret = i915_gem_object_lock(obj, &ww); | |
b414fcd5 | 316 | if (ret) |
3c0ffa27 | 317 | goto err_rpm; |
b414fcd5 | 318 | |
3c0ffa27 ML |
319 | /* Sanity check that we allow writing into this object */ |
320 | if (i915_gem_object_is_readonly(obj) && write) { | |
321 | ret = -EFAULT; | |
322 | goto err_rpm; | |
323 | } | |
b414fcd5 | 324 | |
3c0ffa27 | 325 | ret = i915_gem_object_pin_pages(obj); |
eebab60f | 326 | if (ret) |
b414fcd5 | 327 | goto err_rpm; |
b414fcd5 | 328 | |
3c0ffa27 ML |
329 | ret = intel_gt_reset_trylock(ggtt->vm.gt, &srcu); |
330 | if (ret) | |
331 | goto err_pages; | |
332 | ||
b414fcd5 | 333 | /* Now pin it into the GTT as needed */ |
3c0ffa27 ML |
334 | vma = i915_gem_object_ggtt_pin_ww(obj, &ww, NULL, 0, 0, |
335 | PIN_MAPPABLE | | |
336 | PIN_NONBLOCK /* NOWARN */ | | |
337 | PIN_NOEVICT); | |
338 | if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) { | |
b414fcd5 CW |
339 | /* Use a partial view if it is bigger than available space */ |
340 | struct i915_ggtt_view view = | |
341 | compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES); | |
342 | unsigned int flags; | |
343 | ||
6846895f | 344 | flags = PIN_MAPPABLE | PIN_NOSEARCH; |
b414fcd5 CW |
345 | if (view.type == I915_GGTT_VIEW_NORMAL) |
346 | flags |= PIN_NONBLOCK; /* avoid warnings for pinned */ | |
347 | ||
348 | /* | |
349 | * Userspace is now writing through an untracked VMA, abandon | |
350 | * all hope that the hardware is able to track future writes. | |
351 | */ | |
b414fcd5 | 352 | |
3c0ffa27 ML |
353 | vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags); |
354 | if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) { | |
b414fcd5 CW |
355 | flags = PIN_MAPPABLE; |
356 | view.type = I915_GGTT_VIEW_PARTIAL; | |
3c0ffa27 | 357 | vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags); |
b414fcd5 | 358 | } |
8f9fb61c CW |
359 | |
360 | /* The entire mappable GGTT is pinned? Unexpected! */ | |
361 | GEM_BUG_ON(vma == ERR_PTR(-ENOSPC)); | |
b414fcd5 CW |
362 | } |
363 | if (IS_ERR(vma)) { | |
364 | ret = PTR_ERR(vma); | |
2850748e CW |
365 | goto err_reset; |
366 | } | |
367 | ||
368 | /* Access to snoopable pages through the GTT is incoherent. */ | |
369 | if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(i915)) { | |
370 | ret = -EFAULT; | |
371 | goto err_unpin; | |
b414fcd5 CW |
372 | } |
373 | ||
374 | ret = i915_vma_pin_fence(vma); | |
375 | if (ret) | |
376 | goto err_unpin; | |
377 | ||
378 | /* Finally, remap it using the new GTT offset */ | |
0e4fe0c9 MA |
379 | ret = remap_io_mapping(area, |
380 | area->vm_start + (vma->ggtt_view.partial.offset << PAGE_SHIFT), | |
381 | (ggtt->gmadr.start + vma->node.start) >> PAGE_SHIFT, | |
382 | min_t(u64, vma->size, area->vm_end - area->vm_start), | |
383 | &ggtt->iomap); | |
b414fcd5 CW |
384 | if (ret) |
385 | goto err_fence; | |
386 | ||
d858d569 | 387 | assert_rpm_wakelock_held(rpm); |
b7d151ba CW |
388 | |
389 | /* Mark as being mmapped into userspace for later revocation */ | |
390 | mutex_lock(&i915->ggtt.vm.mutex); | |
b414fcd5 | 391 | if (!i915_vma_set_userfault(vma) && !obj->userfault_count++) |
0cf289bd | 392 | list_add(&obj->userfault_link, &i915->ggtt.userfault_list); |
b7d151ba CW |
393 | mutex_unlock(&i915->ggtt.vm.mutex); |
394 | ||
cc662126 AJ |
395 | /* Track the mmo associated with the fenced vma */ |
396 | vma->mmo = mmo; | |
397 | ||
babaab2f | 398 | if (IS_ACTIVE(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)) |
0cf289bd | 399 | intel_wakeref_auto(&i915->ggtt.userfault_wakeref, |
b414fcd5 | 400 | msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)); |
b414fcd5 | 401 | |
5028851c CW |
402 | if (write) { |
403 | GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); | |
404 | i915_vma_set_ggtt_write(vma); | |
405 | obj->mm.dirty = true; | |
406 | } | |
b414fcd5 CW |
407 | |
408 | err_fence: | |
409 | i915_vma_unpin_fence(vma); | |
410 | err_unpin: | |
411 | __i915_vma_unpin(vma); | |
b414fcd5 | 412 | err_reset: |
cb823ed9 | 413 | intel_gt_reset_unlock(ggtt->vm.gt, srcu); |
3c0ffa27 ML |
414 | err_pages: |
415 | i915_gem_object_unpin_pages(obj); | |
b414fcd5 | 416 | err_rpm: |
3c0ffa27 ML |
417 | if (ret == -EDEADLK) { |
418 | ret = i915_gem_ww_ctx_backoff(&ww); | |
419 | if (!ret) | |
420 | goto retry; | |
421 | } | |
422 | i915_gem_ww_ctx_fini(&ww); | |
d858d569 | 423 | intel_runtime_pm_put(rpm, wakeref); |
cc662126 | 424 | return i915_error_to_vmf_fault(ret); |
b414fcd5 CW |
425 | } |
426 | ||
9f909e21 CW |
427 | static int |
428 | vm_access(struct vm_area_struct *area, unsigned long addr, | |
429 | void *buf, int len, int write) | |
430 | { | |
431 | struct i915_mmap_offset *mmo = area->vm_private_data; | |
432 | struct drm_i915_gem_object *obj = mmo->obj; | |
52665fe7 | 433 | struct i915_gem_ww_ctx ww; |
9f909e21 | 434 | void *vaddr; |
52665fe7 | 435 | int err = 0; |
9f909e21 CW |
436 | |
437 | if (i915_gem_object_is_readonly(obj) && write) | |
438 | return -EACCES; | |
439 | ||
440 | addr -= area->vm_start; | |
c40bad05 | 441 | if (range_overflows_t(u64, addr, len, obj->base.size)) |
9f909e21 CW |
442 | return -EINVAL; |
443 | ||
52665fe7 ML |
444 | i915_gem_ww_ctx_init(&ww, true); |
445 | retry: | |
446 | err = i915_gem_object_lock(obj, &ww); | |
447 | if (err) | |
448 | goto out; | |
449 | ||
9f909e21 CW |
450 | /* As this is primarily for debugging, let's focus on simplicity */ |
451 | vaddr = i915_gem_object_pin_map(obj, I915_MAP_FORCE_WC); | |
52665fe7 ML |
452 | if (IS_ERR(vaddr)) { |
453 | err = PTR_ERR(vaddr); | |
454 | goto out; | |
455 | } | |
9f909e21 CW |
456 | |
457 | if (write) { | |
458 | memcpy(vaddr + addr, buf, len); | |
459 | __i915_gem_object_flush_map(obj, addr, len); | |
460 | } else { | |
461 | memcpy(buf, vaddr + addr, len); | |
462 | } | |
463 | ||
464 | i915_gem_object_unpin_map(obj); | |
52665fe7 ML |
465 | out: |
466 | if (err == -EDEADLK) { | |
467 | err = i915_gem_ww_ctx_backoff(&ww); | |
468 | if (!err) | |
469 | goto retry; | |
470 | } | |
471 | i915_gem_ww_ctx_fini(&ww); | |
472 | ||
473 | if (err) | |
474 | return err; | |
9f909e21 CW |
475 | |
476 | return len; | |
477 | } | |
478 | ||
cc662126 | 479 | void __i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj) |
b414fcd5 CW |
480 | { |
481 | struct i915_vma *vma; | |
482 | ||
483 | GEM_BUG_ON(!obj->userfault_count); | |
484 | ||
b414fcd5 | 485 | for_each_ggtt_vma(vma, obj) |
cc662126 AJ |
486 | i915_vma_revoke_mmap(vma); |
487 | ||
488 | GEM_BUG_ON(obj->userfault_count); | |
b414fcd5 CW |
489 | } |
490 | ||
cc662126 | 491 | /* |
b414fcd5 CW |
492 | * It is vital that we remove the page mapping if we have mapped a tiled |
493 | * object through the GTT and then lose the fence register due to | |
494 | * resource pressure. Similarly if the object has been moved out of the | |
495 | * aperture, than pages mapped into userspace must be revoked. Removing the | |
496 | * mapping will then trigger a page fault on the next user access, allowing | |
cc662126 | 497 | * fixup by vm_fault_gtt(). |
b414fcd5 | 498 | */ |
cb2baf42 | 499 | void i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj) |
b414fcd5 CW |
500 | { |
501 | struct drm_i915_private *i915 = to_i915(obj->base.dev); | |
502 | intel_wakeref_t wakeref; | |
503 | ||
cc662126 AJ |
504 | /* |
505 | * Serialisation between user GTT access and our code depends upon | |
b414fcd5 CW |
506 | * revoking the CPU's PTE whilst the mutex is held. The next user |
507 | * pagefault then has to wait until we release the mutex. | |
508 | * | |
509 | * Note that RPM complicates somewhat by adding an additional | |
510 | * requirement that operations to the GGTT be made holding the RPM | |
511 | * wakeref. | |
512 | */ | |
d858d569 | 513 | wakeref = intel_runtime_pm_get(&i915->runtime_pm); |
b7d151ba | 514 | mutex_lock(&i915->ggtt.vm.mutex); |
b414fcd5 CW |
515 | |
516 | if (!obj->userfault_count) | |
517 | goto out; | |
518 | ||
cc662126 | 519 | __i915_gem_object_release_mmap_gtt(obj); |
b414fcd5 | 520 | |
cc662126 AJ |
521 | /* |
522 | * Ensure that the CPU's PTE are revoked and there are not outstanding | |
b414fcd5 CW |
523 | * memory transactions from userspace before we return. The TLB |
524 | * flushing implied above by changing the PTE above *should* be | |
525 | * sufficient, an extra barrier here just provides us with a bit | |
526 | * of paranoid documentation about our requirement to serialise | |
527 | * memory writes before touching registers / GSM. | |
528 | */ | |
529 | wmb(); | |
530 | ||
531 | out: | |
b7d151ba | 532 | mutex_unlock(&i915->ggtt.vm.mutex); |
d858d569 | 533 | intel_runtime_pm_put(&i915->runtime_pm, wakeref); |
b414fcd5 CW |
534 | } |
535 | ||
cc662126 AJ |
536 | void i915_gem_object_release_mmap_offset(struct drm_i915_gem_object *obj) |
537 | { | |
78655598 | 538 | struct i915_mmap_offset *mmo, *mn; |
cc662126 AJ |
539 | |
540 | spin_lock(&obj->mmo.lock); | |
78655598 CW |
541 | rbtree_postorder_for_each_entry_safe(mmo, mn, |
542 | &obj->mmo.offsets, offset) { | |
cc662126 AJ |
543 | /* |
544 | * vma_node_unmap for GTT mmaps handled already in | |
545 | * __i915_gem_object_release_mmap_gtt | |
546 | */ | |
547 | if (mmo->mmap_type == I915_MMAP_TYPE_GTT) | |
548 | continue; | |
549 | ||
550 | spin_unlock(&obj->mmo.lock); | |
551 | drm_vma_node_unmap(&mmo->vma_node, | |
552 | obj->base.dev->anon_inode->i_mapping); | |
553 | spin_lock(&obj->mmo.lock); | |
554 | } | |
555 | spin_unlock(&obj->mmo.lock); | |
556 | } | |
557 | ||
78655598 CW |
558 | static struct i915_mmap_offset * |
559 | lookup_mmo(struct drm_i915_gem_object *obj, | |
560 | enum i915_mmap_type mmap_type) | |
561 | { | |
562 | struct rb_node *rb; | |
563 | ||
564 | spin_lock(&obj->mmo.lock); | |
565 | rb = obj->mmo.offsets.rb_node; | |
566 | while (rb) { | |
567 | struct i915_mmap_offset *mmo = | |
568 | rb_entry(rb, typeof(*mmo), offset); | |
569 | ||
570 | if (mmo->mmap_type == mmap_type) { | |
571 | spin_unlock(&obj->mmo.lock); | |
572 | return mmo; | |
573 | } | |
574 | ||
575 | if (mmo->mmap_type < mmap_type) | |
576 | rb = rb->rb_right; | |
577 | else | |
578 | rb = rb->rb_left; | |
579 | } | |
580 | spin_unlock(&obj->mmo.lock); | |
581 | ||
582 | return NULL; | |
583 | } | |
584 | ||
585 | static struct i915_mmap_offset * | |
586 | insert_mmo(struct drm_i915_gem_object *obj, struct i915_mmap_offset *mmo) | |
587 | { | |
588 | struct rb_node *rb, **p; | |
589 | ||
590 | spin_lock(&obj->mmo.lock); | |
591 | rb = NULL; | |
592 | p = &obj->mmo.offsets.rb_node; | |
593 | while (*p) { | |
594 | struct i915_mmap_offset *pos; | |
595 | ||
596 | rb = *p; | |
597 | pos = rb_entry(rb, typeof(*pos), offset); | |
598 | ||
599 | if (pos->mmap_type == mmo->mmap_type) { | |
600 | spin_unlock(&obj->mmo.lock); | |
601 | drm_vma_offset_remove(obj->base.dev->vma_offset_manager, | |
602 | &mmo->vma_node); | |
603 | kfree(mmo); | |
604 | return pos; | |
605 | } | |
606 | ||
607 | if (pos->mmap_type < mmo->mmap_type) | |
608 | p = &rb->rb_right; | |
609 | else | |
610 | p = &rb->rb_left; | |
611 | } | |
612 | rb_link_node(&mmo->offset, rb, p); | |
613 | rb_insert_color(&mmo->offset, &obj->mmo.offsets); | |
614 | spin_unlock(&obj->mmo.lock); | |
615 | ||
616 | return mmo; | |
617 | } | |
618 | ||
cc662126 AJ |
619 | static struct i915_mmap_offset * |
620 | mmap_offset_attach(struct drm_i915_gem_object *obj, | |
621 | enum i915_mmap_type mmap_type, | |
622 | struct drm_file *file) | |
b414fcd5 CW |
623 | { |
624 | struct drm_i915_private *i915 = to_i915(obj->base.dev); | |
cc662126 | 625 | struct i915_mmap_offset *mmo; |
b414fcd5 CW |
626 | int err; |
627 | ||
cf3e3e86 ML |
628 | GEM_BUG_ON(obj->ops->mmap_offset || obj->ops->mmap_ops); |
629 | ||
78655598 CW |
630 | mmo = lookup_mmo(obj, mmap_type); |
631 | if (mmo) | |
632 | goto out; | |
633 | ||
cc662126 AJ |
634 | mmo = kmalloc(sizeof(*mmo), GFP_KERNEL); |
635 | if (!mmo) | |
636 | return ERR_PTR(-ENOMEM); | |
637 | ||
638 | mmo->obj = obj; | |
cc662126 AJ |
639 | mmo->mmap_type = mmap_type; |
640 | drm_vma_node_reset(&mmo->vma_node); | |
641 | ||
78655598 CW |
642 | err = drm_vma_offset_add(obj->base.dev->vma_offset_manager, |
643 | &mmo->vma_node, obj->base.size / PAGE_SIZE); | |
b414fcd5 | 644 | if (likely(!err)) |
78655598 | 645 | goto insert; |
b414fcd5 CW |
646 | |
647 | /* Attempt to reap some mmap space from dead objects */ | |
b97060a9 MB |
648 | err = intel_gt_retire_requests_timeout(&i915->gt, MAX_SCHEDULE_TIMEOUT, |
649 | NULL); | |
789ed955 | 650 | if (err) |
cc662126 | 651 | goto err; |
b414fcd5 | 652 | |
789ed955 | 653 | i915_gem_drain_freed_objects(i915); |
78655598 CW |
654 | err = drm_vma_offset_add(obj->base.dev->vma_offset_manager, |
655 | &mmo->vma_node, obj->base.size / PAGE_SIZE); | |
cc662126 AJ |
656 | if (err) |
657 | goto err; | |
658 | ||
78655598 CW |
659 | insert: |
660 | mmo = insert_mmo(obj, mmo); | |
661 | GEM_BUG_ON(lookup_mmo(obj, mmap_type) != mmo); | |
cc662126 AJ |
662 | out: |
663 | if (file) | |
664 | drm_vma_node_allow(&mmo->vma_node, file); | |
cc662126 AJ |
665 | return mmo; |
666 | ||
667 | err: | |
668 | kfree(mmo); | |
669 | return ERR_PTR(err); | |
b414fcd5 CW |
670 | } |
671 | ||
cc662126 | 672 | static int |
cf3e3e86 | 673 | __assign_mmap_offset(struct drm_i915_gem_object *obj, |
cc662126 | 674 | enum i915_mmap_type mmap_type, |
cf3e3e86 | 675 | u64 *offset, struct drm_file *file) |
b414fcd5 | 676 | { |
cc662126 | 677 | struct i915_mmap_offset *mmo; |
bee0a70a | 678 | |
cf3e3e86 ML |
679 | if (i915_gem_object_never_mmap(obj)) |
680 | return -ENODEV; | |
b414fcd5 | 681 | |
cf3e3e86 | 682 | if (obj->ops->mmap_offset) { |
7961c5b6 ML |
683 | if (mmap_type != I915_MMAP_TYPE_FIXED) |
684 | return -ENODEV; | |
685 | ||
cf3e3e86 ML |
686 | *offset = obj->ops->mmap_offset(obj); |
687 | return 0; | |
a4311745 CW |
688 | } |
689 | ||
7961c5b6 ML |
690 | if (mmap_type == I915_MMAP_TYPE_FIXED) |
691 | return -ENODEV; | |
692 | ||
cc662126 | 693 | if (mmap_type != I915_MMAP_TYPE_GTT && |
c471748d | 694 | !i915_gem_object_has_struct_page(obj) && |
0ff37575 | 695 | !i915_gem_object_has_iomem(obj)) |
cf3e3e86 | 696 | return -ENODEV; |
cc662126 AJ |
697 | |
698 | mmo = mmap_offset_attach(obj, mmap_type, file); | |
cf3e3e86 ML |
699 | if (IS_ERR(mmo)) |
700 | return PTR_ERR(mmo); | |
b414fcd5 | 701 | |
cc662126 | 702 | *offset = drm_vma_node_offset_addr(&mmo->vma_node); |
cf3e3e86 ML |
703 | return 0; |
704 | } | |
705 | ||
706 | static int | |
707 | __assign_mmap_offset_handle(struct drm_file *file, | |
708 | u32 handle, | |
709 | enum i915_mmap_type mmap_type, | |
710 | u64 *offset) | |
711 | { | |
712 | struct drm_i915_gem_object *obj; | |
713 | int err; | |
714 | ||
715 | obj = i915_gem_object_lookup(file, handle); | |
716 | if (!obj) | |
717 | return -ENOENT; | |
718 | ||
0ff37575 TH |
719 | err = i915_gem_object_lock_interruptible(obj, NULL); |
720 | if (err) | |
721 | goto out_put; | |
cf3e3e86 | 722 | err = __assign_mmap_offset(obj, mmap_type, offset, file); |
0ff37575 TH |
723 | i915_gem_object_unlock(obj); |
724 | out_put: | |
b414fcd5 | 725 | i915_gem_object_put(obj); |
cc662126 AJ |
726 | return err; |
727 | } | |
728 | ||
729 | int | |
730 | i915_gem_dumb_mmap_offset(struct drm_file *file, | |
731 | struct drm_device *dev, | |
732 | u32 handle, | |
733 | u64 *offset) | |
734 | { | |
735 | enum i915_mmap_type mmap_type; | |
736 | ||
7961c5b6 ML |
737 | if (HAS_LMEM(to_i915(dev))) |
738 | mmap_type = I915_MMAP_TYPE_FIXED; | |
739 | else if (boot_cpu_has(X86_FEATURE_PAT)) | |
cc662126 AJ |
740 | mmap_type = I915_MMAP_TYPE_WC; |
741 | else if (!i915_ggtt_has_aperture(&to_i915(dev)->ggtt)) | |
742 | return -ENODEV; | |
743 | else | |
744 | mmap_type = I915_MMAP_TYPE_GTT; | |
745 | ||
cf3e3e86 | 746 | return __assign_mmap_offset_handle(file, handle, mmap_type, offset); |
b414fcd5 CW |
747 | } |
748 | ||
749 | /** | |
cc662126 | 750 | * i915_gem_mmap_offset_ioctl - prepare an object for GTT mmap'ing |
b414fcd5 CW |
751 | * @dev: DRM device |
752 | * @data: GTT mapping ioctl data | |
753 | * @file: GEM object info | |
754 | * | |
755 | * Simply returns the fake offset to userspace so it can mmap it. | |
756 | * The mmap call will end up in drm_gem_mmap(), which will set things | |
757 | * up so we can get faults in the handler above. | |
758 | * | |
759 | * The fault handler will take care of binding the object into the GTT | |
760 | * (since it may have been evicted to make room for something), allocating | |
761 | * a fence register, and mapping the appropriate aperture address into | |
762 | * userspace. | |
763 | */ | |
764 | int | |
cc662126 AJ |
765 | i915_gem_mmap_offset_ioctl(struct drm_device *dev, void *data, |
766 | struct drm_file *file) | |
b414fcd5 | 767 | { |
cc662126 AJ |
768 | struct drm_i915_private *i915 = to_i915(dev); |
769 | struct drm_i915_gem_mmap_offset *args = data; | |
770 | enum i915_mmap_type type; | |
126d5de3 | 771 | int err; |
cc662126 | 772 | |
8d65859a CW |
773 | /* |
774 | * Historically we failed to check args.pad and args.offset | |
775 | * and so we cannot use those fields for user input and we cannot | |
776 | * add -EINVAL for them as the ABI is fixed, i.e. old userspace | |
777 | * may be feeding in garbage in those fields. | |
778 | * | |
779 | * if (args->pad) return -EINVAL; is verbotten! | |
780 | */ | |
781 | ||
126d5de3 CW |
782 | err = i915_user_extensions(u64_to_user_ptr(args->extensions), |
783 | NULL, 0, NULL); | |
784 | if (err) | |
785 | return err; | |
cc662126 AJ |
786 | |
787 | switch (args->flags) { | |
788 | case I915_MMAP_OFFSET_GTT: | |
789 | if (!i915_ggtt_has_aperture(&i915->ggtt)) | |
790 | return -ENODEV; | |
791 | type = I915_MMAP_TYPE_GTT; | |
792 | break; | |
793 | ||
794 | case I915_MMAP_OFFSET_WC: | |
795 | if (!boot_cpu_has(X86_FEATURE_PAT)) | |
796 | return -ENODEV; | |
797 | type = I915_MMAP_TYPE_WC; | |
798 | break; | |
799 | ||
800 | case I915_MMAP_OFFSET_WB: | |
801 | type = I915_MMAP_TYPE_WB; | |
802 | break; | |
803 | ||
804 | case I915_MMAP_OFFSET_UC: | |
805 | if (!boot_cpu_has(X86_FEATURE_PAT)) | |
806 | return -ENODEV; | |
807 | type = I915_MMAP_TYPE_UC; | |
808 | break; | |
809 | ||
7961c5b6 ML |
810 | case I915_MMAP_OFFSET_FIXED: |
811 | type = I915_MMAP_TYPE_FIXED; | |
812 | break; | |
813 | ||
cc662126 AJ |
814 | default: |
815 | return -EINVAL; | |
816 | } | |
b414fcd5 | 817 | |
cf3e3e86 | 818 | return __assign_mmap_offset_handle(file, args->handle, type, &args->offset); |
cc662126 AJ |
819 | } |
820 | ||
821 | static void vm_open(struct vm_area_struct *vma) | |
822 | { | |
823 | struct i915_mmap_offset *mmo = vma->vm_private_data; | |
824 | struct drm_i915_gem_object *obj = mmo->obj; | |
825 | ||
826 | GEM_BUG_ON(!obj); | |
827 | i915_gem_object_get(obj); | |
828 | } | |
829 | ||
830 | static void vm_close(struct vm_area_struct *vma) | |
831 | { | |
832 | struct i915_mmap_offset *mmo = vma->vm_private_data; | |
833 | struct drm_i915_gem_object *obj = mmo->obj; | |
834 | ||
835 | GEM_BUG_ON(!obj); | |
836 | i915_gem_object_put(obj); | |
837 | } | |
838 | ||
839 | static const struct vm_operations_struct vm_ops_gtt = { | |
840 | .fault = vm_fault_gtt, | |
9f909e21 | 841 | .access = vm_access, |
cc662126 AJ |
842 | .open = vm_open, |
843 | .close = vm_close, | |
844 | }; | |
845 | ||
846 | static const struct vm_operations_struct vm_ops_cpu = { | |
847 | .fault = vm_fault_cpu, | |
9f909e21 | 848 | .access = vm_access, |
cc662126 AJ |
849 | .open = vm_open, |
850 | .close = vm_close, | |
851 | }; | |
852 | ||
f17b8980 CW |
853 | static int singleton_release(struct inode *inode, struct file *file) |
854 | { | |
855 | struct drm_i915_private *i915 = file->private_data; | |
856 | ||
857 | cmpxchg(&i915->gem.mmap_singleton, file, NULL); | |
858 | drm_dev_put(&i915->drm); | |
859 | ||
860 | return 0; | |
861 | } | |
862 | ||
863 | static const struct file_operations singleton_fops = { | |
864 | .owner = THIS_MODULE, | |
865 | .release = singleton_release, | |
866 | }; | |
867 | ||
868 | static struct file *mmap_singleton(struct drm_i915_private *i915) | |
869 | { | |
870 | struct file *file; | |
871 | ||
872 | rcu_read_lock(); | |
4aea5a9e | 873 | file = READ_ONCE(i915->gem.mmap_singleton); |
f17b8980 CW |
874 | if (file && !get_file_rcu(file)) |
875 | file = NULL; | |
876 | rcu_read_unlock(); | |
877 | if (file) | |
878 | return file; | |
879 | ||
880 | file = anon_inode_getfile("i915.gem", &singleton_fops, i915, O_RDWR); | |
881 | if (IS_ERR(file)) | |
882 | return file; | |
883 | ||
884 | /* Everyone shares a single global address space */ | |
885 | file->f_mapping = i915->drm.anon_inode->i_mapping; | |
886 | ||
887 | smp_store_mb(i915->gem.mmap_singleton, file); | |
888 | drm_dev_get(&i915->drm); | |
889 | ||
890 | return file; | |
891 | } | |
892 | ||
cc662126 AJ |
893 | /* |
894 | * This overcomes the limitation in drm_gem_mmap's assignment of a | |
895 | * drm_gem_object as the vma->vm_private_data. Since we need to | |
896 | * be able to resolve multiple mmap offsets which could be tied | |
897 | * to a single gem object. | |
898 | */ | |
899 | int i915_gem_mmap(struct file *filp, struct vm_area_struct *vma) | |
900 | { | |
901 | struct drm_vma_offset_node *node; | |
902 | struct drm_file *priv = filp->private_data; | |
903 | struct drm_device *dev = priv->minor->dev; | |
280d14a6 | 904 | struct drm_i915_gem_object *obj = NULL; |
cc662126 | 905 | struct i915_mmap_offset *mmo = NULL; |
f17b8980 | 906 | struct file *anon; |
cc662126 AJ |
907 | |
908 | if (drm_dev_is_unplugged(dev)) | |
909 | return -ENODEV; | |
910 | ||
280d14a6 | 911 | rcu_read_lock(); |
cc662126 AJ |
912 | drm_vma_offset_lock_lookup(dev->vma_offset_manager); |
913 | node = drm_vma_offset_exact_lookup_locked(dev->vma_offset_manager, | |
914 | vma->vm_pgoff, | |
915 | vma_pages(vma)); | |
280d14a6 | 916 | if (node && drm_vma_node_is_allowed(node, priv)) { |
cc662126 AJ |
917 | /* |
918 | * Skip 0-refcnted objects as it is in the process of being | |
919 | * destroyed and will be invalid when the vma manager lock | |
920 | * is released. | |
921 | */ | |
cf3e3e86 ML |
922 | if (!node->driver_private) { |
923 | mmo = container_of(node, struct i915_mmap_offset, vma_node); | |
924 | obj = i915_gem_object_get_rcu(mmo->obj); | |
925 | ||
926 | GEM_BUG_ON(obj && obj->ops->mmap_ops); | |
927 | } else { | |
928 | obj = i915_gem_object_get_rcu | |
929 | (container_of(node, struct drm_i915_gem_object, | |
930 | base.vma_node)); | |
931 | ||
932 | GEM_BUG_ON(obj && !obj->ops->mmap_ops); | |
933 | } | |
cc662126 AJ |
934 | } |
935 | drm_vma_offset_unlock_lookup(dev->vma_offset_manager); | |
280d14a6 | 936 | rcu_read_unlock(); |
cc662126 | 937 | if (!obj) |
280d14a6 | 938 | return node ? -EACCES : -EINVAL; |
cc662126 | 939 | |
280d14a6 | 940 | if (i915_gem_object_is_readonly(obj)) { |
cc662126 | 941 | if (vma->vm_flags & VM_WRITE) { |
280d14a6 | 942 | i915_gem_object_put(obj); |
cc662126 AJ |
943 | return -EINVAL; |
944 | } | |
945 | vma->vm_flags &= ~VM_MAYWRITE; | |
946 | } | |
947 | ||
280d14a6 | 948 | anon = mmap_singleton(to_i915(dev)); |
f17b8980 | 949 | if (IS_ERR(anon)) { |
280d14a6 | 950 | i915_gem_object_put(obj); |
f17b8980 CW |
951 | return PTR_ERR(anon); |
952 | } | |
953 | ||
0ff37575 | 954 | vma->vm_flags |= VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO; |
cc662126 | 955 | |
f17b8980 CW |
956 | /* |
957 | * We keep the ref on mmo->obj, not vm_file, but we require | |
958 | * vma->vm_file->f_mapping, see vma_link(), for later revocation. | |
959 | * Our userspace is accustomed to having per-file resource cleanup | |
960 | * (i.e. contexts, objects and requests) on their close(fd), which | |
961 | * requires avoiding extraneous references to their filp, hence why | |
962 | * we prefer to use an anonymous file for their mmaps. | |
963 | */ | |
295992fb CK |
964 | vma_set_file(vma, anon); |
965 | /* Drop the initial creation reference, the vma is now holding one. */ | |
966 | fput(anon); | |
f17b8980 | 967 | |
cf3e3e86 ML |
968 | if (obj->ops->mmap_ops) { |
969 | vma->vm_page_prot = pgprot_decrypted(vm_get_page_prot(vma->vm_flags)); | |
970 | vma->vm_ops = obj->ops->mmap_ops; | |
971 | vma->vm_private_data = node->driver_private; | |
972 | return 0; | |
973 | } | |
974 | ||
975 | vma->vm_private_data = mmo; | |
976 | ||
cc662126 AJ |
977 | switch (mmo->mmap_type) { |
978 | case I915_MMAP_TYPE_WC: | |
979 | vma->vm_page_prot = | |
980 | pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); | |
981 | vma->vm_ops = &vm_ops_cpu; | |
982 | break; | |
983 | ||
7961c5b6 ML |
984 | case I915_MMAP_TYPE_FIXED: |
985 | GEM_WARN_ON(1); | |
986 | fallthrough; | |
cc662126 AJ |
987 | case I915_MMAP_TYPE_WB: |
988 | vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); | |
989 | vma->vm_ops = &vm_ops_cpu; | |
990 | break; | |
991 | ||
992 | case I915_MMAP_TYPE_UC: | |
993 | vma->vm_page_prot = | |
994 | pgprot_noncached(vm_get_page_prot(vma->vm_flags)); | |
995 | vma->vm_ops = &vm_ops_cpu; | |
996 | break; | |
997 | ||
998 | case I915_MMAP_TYPE_GTT: | |
999 | vma->vm_page_prot = | |
1000 | pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); | |
1001 | vma->vm_ops = &vm_ops_gtt; | |
1002 | break; | |
1003 | } | |
1004 | vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); | |
1005 | ||
1006 | return 0; | |
b414fcd5 CW |
1007 | } |
1008 | ||
1009 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) | |
1010 | #include "selftests/i915_gem_mman.c" | |
1011 | #endif |