2 * Copyright © 2010 Daniel Vetter
3 * Copyright © 2011-2014 Intel Corporation
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
26 #include <linux/slab.h> /* fault-inject.h is not standalone! */
28 #include <linux/fault-inject.h>
29 #include <linux/log2.h>
30 #include <linux/random.h>
31 #include <linux/seq_file.h>
32 #include <linux/stop_machine.h>
34 #include <asm/set_memory.h>
37 #include <drm/i915_drm.h>
40 #include "i915_vgpu.h"
41 #include "i915_trace.h"
42 #include "intel_drv.h"
43 #include "intel_frontbuffer.h"
45 #define I915_GFP_DMA (GFP_KERNEL | __GFP_HIGHMEM)
48 * DOC: Global GTT views
50 * Background and previous state
52 * Historically objects could exists (be bound) in global GTT space only as
53 * singular instances with a view representing all of the object's backing pages
54 * in a linear fashion. This view will be called a normal view.
56 * To support multiple views of the same object, where the number of mapped
57 * pages is not equal to the backing store, or where the layout of the pages
58 * is not linear, concept of a GGTT view was added.
60 * One example of an alternative view is a stereo display driven by a single
61 * image. In this case we would have a framebuffer looking like this
67 * Above would represent a normal GGTT view as normally mapped for GPU or CPU
68 * rendering. In contrast, fed to the display engine would be an alternative
69 * view which could look something like this:
74 * In this example both the size and layout of pages in the alternative view is
75 * different from the normal view.
77 * Implementation and usage
79 * GGTT views are implemented using VMAs and are distinguished via enum
80 * i915_ggtt_view_type and struct i915_ggtt_view.
82 * A new flavour of core GEM functions which work with GGTT bound objects were
83 * added with the _ggtt_ infix, and sometimes with _view postfix to avoid
84 * renaming in large amounts of code. They take the struct i915_ggtt_view
85 * parameter encapsulating all metadata required to implement a view.
87 * As a helper for callers which are only interested in the normal view,
88 * globally const i915_ggtt_view_normal singleton instance exists. All old core
89 * GEM API functions, the ones not taking the view parameter, are operating on,
90 * or with the normal GGTT view.
92 * Code wanting to add or use a new GGTT view needs to:
94 * 1. Add a new enum with a suitable name.
95 * 2. Extend the metadata in the i915_ggtt_view structure if required.
96 * 3. Add support to i915_get_vma_pages().
98 * New views are required to build a scatter-gather table from within the
99 * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and
100 * exists for the lifetime of an VMA.
102 * Core API is designed to have copy semantics which means that passed in
103 * struct i915_ggtt_view does not need to be persistent (left around after
104 * calling the core API functions).
109 i915_get_ggtt_vma_pages(struct i915_vma
*vma
);
111 static void gen6_ggtt_invalidate(struct drm_i915_private
*dev_priv
)
113 /* Note that as an uncached mmio write, this should flush the
114 * WCB of the writes into the GGTT before it triggers the invalidate.
116 I915_WRITE(GFX_FLSH_CNTL_GEN6
, GFX_FLSH_CNTL_EN
);
119 static void guc_ggtt_invalidate(struct drm_i915_private
*dev_priv
)
121 gen6_ggtt_invalidate(dev_priv
);
122 I915_WRITE(GEN8_GTCR
, GEN8_GTCR_INVALIDATE
);
125 static void gmch_ggtt_invalidate(struct drm_i915_private
*dev_priv
)
127 intel_gtt_chipset_flush();
130 static inline void i915_ggtt_invalidate(struct drm_i915_private
*i915
)
132 i915
->ggtt
.invalidate(i915
);
135 int intel_sanitize_enable_ppgtt(struct drm_i915_private
*dev_priv
,
139 bool has_full_48bit_ppgtt
;
141 if (!dev_priv
->info
.has_aliasing_ppgtt
)
144 has_full_ppgtt
= dev_priv
->info
.has_full_ppgtt
;
145 has_full_48bit_ppgtt
= dev_priv
->info
.has_full_48bit_ppgtt
;
147 if (intel_vgpu_active(dev_priv
)) {
148 /* GVT-g has no support for 32bit ppgtt */
149 has_full_ppgtt
= false;
150 has_full_48bit_ppgtt
= intel_vgpu_has_full_48bit_ppgtt(dev_priv
);
154 * We don't allow disabling PPGTT for gen9+ as it's a requirement for
155 * execlists, the sole mechanism available to submit work.
157 if (enable_ppgtt
== 0 && INTEL_GEN(dev_priv
) < 9)
160 if (enable_ppgtt
== 1)
163 if (enable_ppgtt
== 2 && has_full_ppgtt
)
166 if (enable_ppgtt
== 3 && has_full_48bit_ppgtt
)
169 /* Disable ppgtt on SNB if VT-d is on. */
170 if (IS_GEN6(dev_priv
) && intel_vtd_active()) {
171 DRM_INFO("Disabling PPGTT because VT-d is on\n");
175 /* Early VLV doesn't have this */
176 if (IS_VALLEYVIEW(dev_priv
) && dev_priv
->drm
.pdev
->revision
< 0xb) {
177 DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n");
181 if (INTEL_GEN(dev_priv
) >= 8 && i915_modparams
.enable_execlists
) {
182 if (has_full_48bit_ppgtt
)
192 static int ppgtt_bind_vma(struct i915_vma
*vma
,
193 enum i915_cache_level cache_level
,
199 if (!(vma
->flags
& I915_VMA_LOCAL_BIND
)) {
200 ret
= vma
->vm
->allocate_va_range(vma
->vm
, vma
->node
.start
,
206 /* Currently applicable only to VLV */
209 pte_flags
|= PTE_READ_ONLY
;
211 vma
->vm
->insert_entries(vma
->vm
, vma
, cache_level
, pte_flags
);
216 static void ppgtt_unbind_vma(struct i915_vma
*vma
)
218 vma
->vm
->clear_range(vma
->vm
, vma
->node
.start
, vma
->size
);
221 static int ppgtt_set_pages(struct i915_vma
*vma
)
223 GEM_BUG_ON(vma
->pages
);
225 vma
->pages
= vma
->obj
->mm
.pages
;
227 vma
->page_sizes
= vma
->obj
->mm
.page_sizes
;
232 static void clear_pages(struct i915_vma
*vma
)
234 GEM_BUG_ON(!vma
->pages
);
236 if (vma
->pages
!= vma
->obj
->mm
.pages
) {
237 sg_free_table(vma
->pages
);
242 memset(&vma
->page_sizes
, 0, sizeof(vma
->page_sizes
));
245 static gen8_pte_t
gen8_pte_encode(dma_addr_t addr
,
246 enum i915_cache_level level
)
248 gen8_pte_t pte
= _PAGE_PRESENT
| _PAGE_RW
;
252 case I915_CACHE_NONE
:
253 pte
|= PPAT_UNCACHED
;
256 pte
|= PPAT_DISPLAY_ELLC
;
266 static gen8_pde_t
gen8_pde_encode(const dma_addr_t addr
,
267 const enum i915_cache_level level
)
269 gen8_pde_t pde
= _PAGE_PRESENT
| _PAGE_RW
;
271 if (level
!= I915_CACHE_NONE
)
272 pde
|= PPAT_CACHED_PDE
;
274 pde
|= PPAT_UNCACHED
;
278 #define gen8_pdpe_encode gen8_pde_encode
279 #define gen8_pml4e_encode gen8_pde_encode
281 static gen6_pte_t
snb_pte_encode(dma_addr_t addr
,
282 enum i915_cache_level level
,
285 gen6_pte_t pte
= GEN6_PTE_VALID
;
286 pte
|= GEN6_PTE_ADDR_ENCODE(addr
);
289 case I915_CACHE_L3_LLC
:
291 pte
|= GEN6_PTE_CACHE_LLC
;
293 case I915_CACHE_NONE
:
294 pte
|= GEN6_PTE_UNCACHED
;
303 static gen6_pte_t
ivb_pte_encode(dma_addr_t addr
,
304 enum i915_cache_level level
,
307 gen6_pte_t pte
= GEN6_PTE_VALID
;
308 pte
|= GEN6_PTE_ADDR_ENCODE(addr
);
311 case I915_CACHE_L3_LLC
:
312 pte
|= GEN7_PTE_CACHE_L3_LLC
;
315 pte
|= GEN6_PTE_CACHE_LLC
;
317 case I915_CACHE_NONE
:
318 pte
|= GEN6_PTE_UNCACHED
;
327 static gen6_pte_t
byt_pte_encode(dma_addr_t addr
,
328 enum i915_cache_level level
,
331 gen6_pte_t pte
= GEN6_PTE_VALID
;
332 pte
|= GEN6_PTE_ADDR_ENCODE(addr
);
334 if (!(flags
& PTE_READ_ONLY
))
335 pte
|= BYT_PTE_WRITEABLE
;
337 if (level
!= I915_CACHE_NONE
)
338 pte
|= BYT_PTE_SNOOPED_BY_CPU_CACHES
;
343 static gen6_pte_t
hsw_pte_encode(dma_addr_t addr
,
344 enum i915_cache_level level
,
347 gen6_pte_t pte
= GEN6_PTE_VALID
;
348 pte
|= HSW_PTE_ADDR_ENCODE(addr
);
350 if (level
!= I915_CACHE_NONE
)
351 pte
|= HSW_WB_LLC_AGE3
;
356 static gen6_pte_t
iris_pte_encode(dma_addr_t addr
,
357 enum i915_cache_level level
,
360 gen6_pte_t pte
= GEN6_PTE_VALID
;
361 pte
|= HSW_PTE_ADDR_ENCODE(addr
);
364 case I915_CACHE_NONE
:
367 pte
|= HSW_WT_ELLC_LLC_AGE3
;
370 pte
|= HSW_WB_ELLC_LLC_AGE3
;
377 static struct page
*vm_alloc_page(struct i915_address_space
*vm
, gfp_t gfp
)
379 struct pagevec
*pvec
= &vm
->free_pages
;
381 if (I915_SELFTEST_ONLY(should_fail(&vm
->fault_attr
, 1)))
382 i915_gem_shrink_all(vm
->i915
);
384 if (likely(pvec
->nr
))
385 return pvec
->pages
[--pvec
->nr
];
388 return alloc_page(gfp
);
390 /* A placeholder for a specific mutex to guard the WC stash */
391 lockdep_assert_held(&vm
->i915
->drm
.struct_mutex
);
393 /* Look in our global stash of WC pages... */
394 pvec
= &vm
->i915
->mm
.wc_stash
;
395 if (likely(pvec
->nr
))
396 return pvec
->pages
[--pvec
->nr
];
398 /* Otherwise batch allocate pages to amoritize cost of set_pages_wc. */
402 page
= alloc_page(gfp
);
406 pvec
->pages
[pvec
->nr
++] = page
;
407 } while (pagevec_space(pvec
));
409 if (unlikely(!pvec
->nr
))
412 set_pages_array_wc(pvec
->pages
, pvec
->nr
);
414 return pvec
->pages
[--pvec
->nr
];
417 static void vm_free_pages_release(struct i915_address_space
*vm
,
420 struct pagevec
*pvec
= &vm
->free_pages
;
422 GEM_BUG_ON(!pagevec_count(pvec
));
424 if (vm
->pt_kmap_wc
) {
425 struct pagevec
*stash
= &vm
->i915
->mm
.wc_stash
;
427 /* When we use WC, first fill up the global stash and then
428 * only if full immediately free the overflow.
431 lockdep_assert_held(&vm
->i915
->drm
.struct_mutex
);
432 if (pagevec_space(stash
)) {
434 stash
->pages
[stash
->nr
++] =
435 pvec
->pages
[--pvec
->nr
];
438 } while (pagevec_space(stash
));
440 /* As we have made some room in the VM's free_pages,
441 * we can wait for it to fill again. Unless we are
442 * inside i915_address_space_fini() and must
443 * immediately release the pages!
449 set_pages_array_wb(pvec
->pages
, pvec
->nr
);
452 __pagevec_release(pvec
);
455 static void vm_free_page(struct i915_address_space
*vm
, struct page
*page
)
457 if (!pagevec_add(&vm
->free_pages
, page
))
458 vm_free_pages_release(vm
, false);
461 static int __setup_page_dma(struct i915_address_space
*vm
,
462 struct i915_page_dma
*p
,
465 p
->page
= vm_alloc_page(vm
, gfp
| __GFP_NOWARN
| __GFP_NORETRY
);
466 if (unlikely(!p
->page
))
469 p
->daddr
= dma_map_page(vm
->dma
, p
->page
, 0, PAGE_SIZE
,
470 PCI_DMA_BIDIRECTIONAL
);
471 if (unlikely(dma_mapping_error(vm
->dma
, p
->daddr
))) {
472 vm_free_page(vm
, p
->page
);
479 static int setup_page_dma(struct i915_address_space
*vm
,
480 struct i915_page_dma
*p
)
482 return __setup_page_dma(vm
, p
, I915_GFP_DMA
);
485 static void cleanup_page_dma(struct i915_address_space
*vm
,
486 struct i915_page_dma
*p
)
488 dma_unmap_page(vm
->dma
, p
->daddr
, PAGE_SIZE
, PCI_DMA_BIDIRECTIONAL
);
489 vm_free_page(vm
, p
->page
);
492 #define kmap_atomic_px(px) kmap_atomic(px_base(px)->page)
494 #define setup_px(vm, px) setup_page_dma((vm), px_base(px))
495 #define cleanup_px(vm, px) cleanup_page_dma((vm), px_base(px))
496 #define fill_px(ppgtt, px, v) fill_page_dma((vm), px_base(px), (v))
497 #define fill32_px(ppgtt, px, v) fill_page_dma_32((vm), px_base(px), (v))
499 static void fill_page_dma(struct i915_address_space
*vm
,
500 struct i915_page_dma
*p
,
503 u64
* const vaddr
= kmap_atomic(p
->page
);
505 memset64(vaddr
, val
, PAGE_SIZE
/ sizeof(val
));
507 kunmap_atomic(vaddr
);
510 static void fill_page_dma_32(struct i915_address_space
*vm
,
511 struct i915_page_dma
*p
,
514 fill_page_dma(vm
, p
, (u64
)v
<< 32 | v
);
518 setup_scratch_page(struct i915_address_space
*vm
, gfp_t gfp
)
520 struct page
*page
= NULL
;
525 * In order to utilize 64K pages for an object with a size < 2M, we will
526 * need to support a 64K scratch page, given that every 16th entry for a
527 * page-table operating in 64K mode must point to a properly aligned 64K
528 * region, including any PTEs which happen to point to scratch.
530 * This is only relevant for the 48b PPGTT where we support
531 * huge-gtt-pages, see also i915_vma_insert().
533 * TODO: we should really consider write-protecting the scratch-page and
534 * sharing between ppgtt
536 if (i915_vm_is_48bit(vm
) &&
537 HAS_PAGE_SIZES(vm
->i915
, I915_GTT_PAGE_SIZE_64K
)) {
538 order
= get_order(I915_GTT_PAGE_SIZE_64K
);
539 page
= alloc_pages(gfp
| __GFP_ZERO
| __GFP_NOWARN
, order
);
541 addr
= dma_map_page(vm
->dma
, page
, 0,
542 I915_GTT_PAGE_SIZE_64K
,
543 PCI_DMA_BIDIRECTIONAL
);
544 if (unlikely(dma_mapping_error(vm
->dma
, addr
))) {
545 __free_pages(page
, order
);
549 if (!IS_ALIGNED(addr
, I915_GTT_PAGE_SIZE_64K
)) {
550 dma_unmap_page(vm
->dma
, addr
,
551 I915_GTT_PAGE_SIZE_64K
,
552 PCI_DMA_BIDIRECTIONAL
);
553 __free_pages(page
, order
);
561 page
= alloc_page(gfp
| __GFP_ZERO
);
565 addr
= dma_map_page(vm
->dma
, page
, 0, PAGE_SIZE
,
566 PCI_DMA_BIDIRECTIONAL
);
567 if (unlikely(dma_mapping_error(vm
->dma
, addr
))) {
573 vm
->scratch_page
.page
= page
;
574 vm
->scratch_page
.daddr
= addr
;
575 vm
->scratch_page
.order
= order
;
580 static void cleanup_scratch_page(struct i915_address_space
*vm
)
582 struct i915_page_dma
*p
= &vm
->scratch_page
;
584 dma_unmap_page(vm
->dma
, p
->daddr
, BIT(p
->order
) << PAGE_SHIFT
,
585 PCI_DMA_BIDIRECTIONAL
);
586 __free_pages(p
->page
, p
->order
);
589 static struct i915_page_table
*alloc_pt(struct i915_address_space
*vm
)
591 struct i915_page_table
*pt
;
593 pt
= kmalloc(sizeof(*pt
), GFP_KERNEL
| __GFP_NOWARN
);
595 return ERR_PTR(-ENOMEM
);
597 if (unlikely(setup_px(vm
, pt
))) {
599 return ERR_PTR(-ENOMEM
);
606 static void free_pt(struct i915_address_space
*vm
, struct i915_page_table
*pt
)
612 static void gen8_initialize_pt(struct i915_address_space
*vm
,
613 struct i915_page_table
*pt
)
616 gen8_pte_encode(vm
->scratch_page
.daddr
, I915_CACHE_LLC
));
619 static void gen6_initialize_pt(struct i915_address_space
*vm
,
620 struct i915_page_table
*pt
)
623 vm
->pte_encode(vm
->scratch_page
.daddr
, I915_CACHE_LLC
, 0));
626 static struct i915_page_directory
*alloc_pd(struct i915_address_space
*vm
)
628 struct i915_page_directory
*pd
;
630 pd
= kzalloc(sizeof(*pd
), GFP_KERNEL
| __GFP_NOWARN
);
632 return ERR_PTR(-ENOMEM
);
634 if (unlikely(setup_px(vm
, pd
))) {
636 return ERR_PTR(-ENOMEM
);
643 static void free_pd(struct i915_address_space
*vm
,
644 struct i915_page_directory
*pd
)
650 static void gen8_initialize_pd(struct i915_address_space
*vm
,
651 struct i915_page_directory
*pd
)
656 gen8_pde_encode(px_dma(vm
->scratch_pt
), I915_CACHE_LLC
));
657 for (i
= 0; i
< I915_PDES
; i
++)
658 pd
->page_table
[i
] = vm
->scratch_pt
;
661 static int __pdp_init(struct i915_address_space
*vm
,
662 struct i915_page_directory_pointer
*pdp
)
664 const unsigned int pdpes
= i915_pdpes_per_pdp(vm
);
667 pdp
->page_directory
= kmalloc_array(pdpes
, sizeof(*pdp
->page_directory
),
668 GFP_KERNEL
| __GFP_NOWARN
);
669 if (unlikely(!pdp
->page_directory
))
672 for (i
= 0; i
< pdpes
; i
++)
673 pdp
->page_directory
[i
] = vm
->scratch_pd
;
678 static void __pdp_fini(struct i915_page_directory_pointer
*pdp
)
680 kfree(pdp
->page_directory
);
681 pdp
->page_directory
= NULL
;
684 static inline bool use_4lvl(const struct i915_address_space
*vm
)
686 return i915_vm_is_48bit(vm
);
689 static struct i915_page_directory_pointer
*
690 alloc_pdp(struct i915_address_space
*vm
)
692 struct i915_page_directory_pointer
*pdp
;
695 WARN_ON(!use_4lvl(vm
));
697 pdp
= kzalloc(sizeof(*pdp
), GFP_KERNEL
);
699 return ERR_PTR(-ENOMEM
);
701 ret
= __pdp_init(vm
, pdp
);
705 ret
= setup_px(vm
, pdp
);
719 static void free_pdp(struct i915_address_space
*vm
,
720 struct i915_page_directory_pointer
*pdp
)
731 static void gen8_initialize_pdp(struct i915_address_space
*vm
,
732 struct i915_page_directory_pointer
*pdp
)
734 gen8_ppgtt_pdpe_t scratch_pdpe
;
736 scratch_pdpe
= gen8_pdpe_encode(px_dma(vm
->scratch_pd
), I915_CACHE_LLC
);
738 fill_px(vm
, pdp
, scratch_pdpe
);
741 static void gen8_initialize_pml4(struct i915_address_space
*vm
,
742 struct i915_pml4
*pml4
)
747 gen8_pml4e_encode(px_dma(vm
->scratch_pdp
), I915_CACHE_LLC
));
748 for (i
= 0; i
< GEN8_PML4ES_PER_PML4
; i
++)
749 pml4
->pdps
[i
] = vm
->scratch_pdp
;
752 /* Broadwell Page Directory Pointer Descriptors */
753 static int gen8_write_pdp(struct drm_i915_gem_request
*req
,
757 struct intel_engine_cs
*engine
= req
->engine
;
762 cs
= intel_ring_begin(req
, 6);
766 *cs
++ = MI_LOAD_REGISTER_IMM(1);
767 *cs
++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(engine
, entry
));
768 *cs
++ = upper_32_bits(addr
);
769 *cs
++ = MI_LOAD_REGISTER_IMM(1);
770 *cs
++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(engine
, entry
));
771 *cs
++ = lower_32_bits(addr
);
772 intel_ring_advance(req
, cs
);
777 static int gen8_mm_switch_3lvl(struct i915_hw_ppgtt
*ppgtt
,
778 struct drm_i915_gem_request
*req
)
782 for (i
= GEN8_3LVL_PDPES
- 1; i
>= 0; i
--) {
783 const dma_addr_t pd_daddr
= i915_page_dir_dma_addr(ppgtt
, i
);
785 ret
= gen8_write_pdp(req
, i
, pd_daddr
);
793 static int gen8_mm_switch_4lvl(struct i915_hw_ppgtt
*ppgtt
,
794 struct drm_i915_gem_request
*req
)
796 return gen8_write_pdp(req
, 0, px_dma(&ppgtt
->pml4
));
799 /* PDE TLBs are a pain to invalidate on GEN8+. When we modify
800 * the page table structures, we mark them dirty so that
801 * context switching/execlist queuing code takes extra steps
802 * to ensure that tlbs are flushed.
804 static void mark_tlbs_dirty(struct i915_hw_ppgtt
*ppgtt
)
806 ppgtt
->pd_dirty_rings
= INTEL_INFO(ppgtt
->base
.i915
)->ring_mask
;
809 /* Removes entries from a single page table, releasing it if it's empty.
810 * Caller can use the return value to update higher-level entries.
812 static bool gen8_ppgtt_clear_pt(struct i915_address_space
*vm
,
813 struct i915_page_table
*pt
,
814 u64 start
, u64 length
)
816 unsigned int num_entries
= gen8_pte_count(start
, length
);
817 unsigned int pte
= gen8_pte_index(start
);
818 unsigned int pte_end
= pte
+ num_entries
;
819 const gen8_pte_t scratch_pte
=
820 gen8_pte_encode(vm
->scratch_page
.daddr
, I915_CACHE_LLC
);
823 GEM_BUG_ON(num_entries
> pt
->used_ptes
);
825 pt
->used_ptes
-= num_entries
;
829 vaddr
= kmap_atomic_px(pt
);
830 while (pte
< pte_end
)
831 vaddr
[pte
++] = scratch_pte
;
832 kunmap_atomic(vaddr
);
837 static void gen8_ppgtt_set_pde(struct i915_address_space
*vm
,
838 struct i915_page_directory
*pd
,
839 struct i915_page_table
*pt
,
844 pd
->page_table
[pde
] = pt
;
846 vaddr
= kmap_atomic_px(pd
);
847 vaddr
[pde
] = gen8_pde_encode(px_dma(pt
), I915_CACHE_LLC
);
848 kunmap_atomic(vaddr
);
851 static bool gen8_ppgtt_clear_pd(struct i915_address_space
*vm
,
852 struct i915_page_directory
*pd
,
853 u64 start
, u64 length
)
855 struct i915_page_table
*pt
;
858 gen8_for_each_pde(pt
, pd
, start
, length
, pde
) {
859 GEM_BUG_ON(pt
== vm
->scratch_pt
);
861 if (!gen8_ppgtt_clear_pt(vm
, pt
, start
, length
))
864 gen8_ppgtt_set_pde(vm
, pd
, vm
->scratch_pt
, pde
);
865 GEM_BUG_ON(!pd
->used_pdes
);
871 return !pd
->used_pdes
;
874 static void gen8_ppgtt_set_pdpe(struct i915_address_space
*vm
,
875 struct i915_page_directory_pointer
*pdp
,
876 struct i915_page_directory
*pd
,
879 gen8_ppgtt_pdpe_t
*vaddr
;
881 pdp
->page_directory
[pdpe
] = pd
;
885 vaddr
= kmap_atomic_px(pdp
);
886 vaddr
[pdpe
] = gen8_pdpe_encode(px_dma(pd
), I915_CACHE_LLC
);
887 kunmap_atomic(vaddr
);
890 /* Removes entries from a single page dir pointer, releasing it if it's empty.
891 * Caller can use the return value to update higher-level entries
893 static bool gen8_ppgtt_clear_pdp(struct i915_address_space
*vm
,
894 struct i915_page_directory_pointer
*pdp
,
895 u64 start
, u64 length
)
897 struct i915_page_directory
*pd
;
900 gen8_for_each_pdpe(pd
, pdp
, start
, length
, pdpe
) {
901 GEM_BUG_ON(pd
== vm
->scratch_pd
);
903 if (!gen8_ppgtt_clear_pd(vm
, pd
, start
, length
))
906 gen8_ppgtt_set_pdpe(vm
, pdp
, vm
->scratch_pd
, pdpe
);
907 GEM_BUG_ON(!pdp
->used_pdpes
);
913 return !pdp
->used_pdpes
;
916 static void gen8_ppgtt_clear_3lvl(struct i915_address_space
*vm
,
917 u64 start
, u64 length
)
919 gen8_ppgtt_clear_pdp(vm
, &i915_vm_to_ppgtt(vm
)->pdp
, start
, length
);
922 static void gen8_ppgtt_set_pml4e(struct i915_pml4
*pml4
,
923 struct i915_page_directory_pointer
*pdp
,
926 gen8_ppgtt_pml4e_t
*vaddr
;
928 pml4
->pdps
[pml4e
] = pdp
;
930 vaddr
= kmap_atomic_px(pml4
);
931 vaddr
[pml4e
] = gen8_pml4e_encode(px_dma(pdp
), I915_CACHE_LLC
);
932 kunmap_atomic(vaddr
);
935 /* Removes entries from a single pml4.
936 * This is the top-level structure in 4-level page tables used on gen8+.
937 * Empty entries are always scratch pml4e.
939 static void gen8_ppgtt_clear_4lvl(struct i915_address_space
*vm
,
940 u64 start
, u64 length
)
942 struct i915_hw_ppgtt
*ppgtt
= i915_vm_to_ppgtt(vm
);
943 struct i915_pml4
*pml4
= &ppgtt
->pml4
;
944 struct i915_page_directory_pointer
*pdp
;
947 GEM_BUG_ON(!use_4lvl(vm
));
949 gen8_for_each_pml4e(pdp
, pml4
, start
, length
, pml4e
) {
950 GEM_BUG_ON(pdp
== vm
->scratch_pdp
);
952 if (!gen8_ppgtt_clear_pdp(vm
, pdp
, start
, length
))
955 gen8_ppgtt_set_pml4e(pml4
, vm
->scratch_pdp
, pml4e
);
961 static inline struct sgt_dma
{
962 struct scatterlist
*sg
;
964 } sgt_dma(struct i915_vma
*vma
) {
965 struct scatterlist
*sg
= vma
->pages
->sgl
;
966 dma_addr_t addr
= sg_dma_address(sg
);
967 return (struct sgt_dma
) { sg
, addr
, addr
+ sg
->length
};
970 struct gen8_insert_pte
{
977 static __always_inline
struct gen8_insert_pte
gen8_insert_pte(u64 start
)
979 return (struct gen8_insert_pte
) {
980 gen8_pml4e_index(start
),
981 gen8_pdpe_index(start
),
982 gen8_pde_index(start
),
983 gen8_pte_index(start
),
987 static __always_inline
bool
988 gen8_ppgtt_insert_pte_entries(struct i915_hw_ppgtt
*ppgtt
,
989 struct i915_page_directory_pointer
*pdp
,
990 struct sgt_dma
*iter
,
991 struct gen8_insert_pte
*idx
,
992 enum i915_cache_level cache_level
)
994 struct i915_page_directory
*pd
;
995 const gen8_pte_t pte_encode
= gen8_pte_encode(0, cache_level
);
999 GEM_BUG_ON(idx
->pdpe
>= i915_pdpes_per_pdp(&ppgtt
->base
));
1000 pd
= pdp
->page_directory
[idx
->pdpe
];
1001 vaddr
= kmap_atomic_px(pd
->page_table
[idx
->pde
]);
1003 vaddr
[idx
->pte
] = pte_encode
| iter
->dma
;
1005 iter
->dma
+= PAGE_SIZE
;
1006 if (iter
->dma
>= iter
->max
) {
1007 iter
->sg
= __sg_next(iter
->sg
);
1013 iter
->dma
= sg_dma_address(iter
->sg
);
1014 iter
->max
= iter
->dma
+ iter
->sg
->length
;
1017 if (++idx
->pte
== GEN8_PTES
) {
1020 if (++idx
->pde
== I915_PDES
) {
1023 /* Limited by sg length for 3lvl */
1024 if (++idx
->pdpe
== GEN8_PML4ES_PER_PML4
) {
1030 GEM_BUG_ON(idx
->pdpe
>= i915_pdpes_per_pdp(&ppgtt
->base
));
1031 pd
= pdp
->page_directory
[idx
->pdpe
];
1034 kunmap_atomic(vaddr
);
1035 vaddr
= kmap_atomic_px(pd
->page_table
[idx
->pde
]);
1038 kunmap_atomic(vaddr
);
1043 static void gen8_ppgtt_insert_3lvl(struct i915_address_space
*vm
,
1044 struct i915_vma
*vma
,
1045 enum i915_cache_level cache_level
,
1048 struct i915_hw_ppgtt
*ppgtt
= i915_vm_to_ppgtt(vm
);
1049 struct sgt_dma iter
= sgt_dma(vma
);
1050 struct gen8_insert_pte idx
= gen8_insert_pte(vma
->node
.start
);
1052 gen8_ppgtt_insert_pte_entries(ppgtt
, &ppgtt
->pdp
, &iter
, &idx
,
1055 vma
->page_sizes
.gtt
= I915_GTT_PAGE_SIZE
;
1058 static void gen8_ppgtt_insert_huge_entries(struct i915_vma
*vma
,
1059 struct i915_page_directory_pointer
**pdps
,
1060 struct sgt_dma
*iter
,
1061 enum i915_cache_level cache_level
)
1063 const gen8_pte_t pte_encode
= gen8_pte_encode(0, cache_level
);
1064 u64 start
= vma
->node
.start
;
1065 dma_addr_t rem
= iter
->sg
->length
;
1068 struct gen8_insert_pte idx
= gen8_insert_pte(start
);
1069 struct i915_page_directory_pointer
*pdp
= pdps
[idx
.pml4e
];
1070 struct i915_page_directory
*pd
= pdp
->page_directory
[idx
.pdpe
];
1071 unsigned int page_size
;
1072 bool maybe_64K
= false;
1073 gen8_pte_t encode
= pte_encode
;
1077 if (vma
->page_sizes
.sg
& I915_GTT_PAGE_SIZE_2M
&&
1078 IS_ALIGNED(iter
->dma
, I915_GTT_PAGE_SIZE_2M
) &&
1079 rem
>= I915_GTT_PAGE_SIZE_2M
&& !idx
.pte
) {
1082 page_size
= I915_GTT_PAGE_SIZE_2M
;
1084 encode
|= GEN8_PDE_PS_2M
;
1086 vaddr
= kmap_atomic_px(pd
);
1088 struct i915_page_table
*pt
= pd
->page_table
[idx
.pde
];
1092 page_size
= I915_GTT_PAGE_SIZE
;
1095 vma
->page_sizes
.sg
& I915_GTT_PAGE_SIZE_64K
&&
1096 IS_ALIGNED(iter
->dma
, I915_GTT_PAGE_SIZE_64K
) &&
1097 (IS_ALIGNED(rem
, I915_GTT_PAGE_SIZE_64K
) ||
1098 rem
>= (max
- index
) << PAGE_SHIFT
))
1101 vaddr
= kmap_atomic_px(pt
);
1105 GEM_BUG_ON(iter
->sg
->length
< page_size
);
1106 vaddr
[index
++] = encode
| iter
->dma
;
1109 iter
->dma
+= page_size
;
1111 if (iter
->dma
>= iter
->max
) {
1112 iter
->sg
= __sg_next(iter
->sg
);
1116 rem
= iter
->sg
->length
;
1117 iter
->dma
= sg_dma_address(iter
->sg
);
1118 iter
->max
= iter
->dma
+ rem
;
1120 if (maybe_64K
&& index
< max
&&
1121 !(IS_ALIGNED(iter
->dma
, I915_GTT_PAGE_SIZE_64K
) &&
1122 (IS_ALIGNED(rem
, I915_GTT_PAGE_SIZE_64K
) ||
1123 rem
>= (max
- index
) << PAGE_SHIFT
)))
1126 if (unlikely(!IS_ALIGNED(iter
->dma
, page_size
)))
1129 } while (rem
>= page_size
&& index
< max
);
1131 kunmap_atomic(vaddr
);
1134 * Is it safe to mark the 2M block as 64K? -- Either we have
1135 * filled whole page-table with 64K entries, or filled part of
1136 * it and have reached the end of the sg table and we have
1141 (i915_vm_has_scratch_64K(vma
->vm
) &&
1142 !iter
->sg
&& IS_ALIGNED(vma
->node
.start
+
1144 I915_GTT_PAGE_SIZE_2M
)))) {
1145 vaddr
= kmap_atomic_px(pd
);
1146 vaddr
[idx
.pde
] |= GEN8_PDE_IPS_64K
;
1147 kunmap_atomic(vaddr
);
1148 page_size
= I915_GTT_PAGE_SIZE_64K
;
1151 vma
->page_sizes
.gtt
|= page_size
;
1155 static void gen8_ppgtt_insert_4lvl(struct i915_address_space
*vm
,
1156 struct i915_vma
*vma
,
1157 enum i915_cache_level cache_level
,
1160 struct i915_hw_ppgtt
*ppgtt
= i915_vm_to_ppgtt(vm
);
1161 struct sgt_dma iter
= sgt_dma(vma
);
1162 struct i915_page_directory_pointer
**pdps
= ppgtt
->pml4
.pdps
;
1164 if (vma
->page_sizes
.sg
> I915_GTT_PAGE_SIZE
) {
1165 gen8_ppgtt_insert_huge_entries(vma
, pdps
, &iter
, cache_level
);
1167 struct gen8_insert_pte idx
= gen8_insert_pte(vma
->node
.start
);
1169 while (gen8_ppgtt_insert_pte_entries(ppgtt
, pdps
[idx
.pml4e
++],
1170 &iter
, &idx
, cache_level
))
1171 GEM_BUG_ON(idx
.pml4e
>= GEN8_PML4ES_PER_PML4
);
1173 vma
->page_sizes
.gtt
= I915_GTT_PAGE_SIZE
;
1177 static void gen8_free_page_tables(struct i915_address_space
*vm
,
1178 struct i915_page_directory
*pd
)
1185 for (i
= 0; i
< I915_PDES
; i
++) {
1186 if (pd
->page_table
[i
] != vm
->scratch_pt
)
1187 free_pt(vm
, pd
->page_table
[i
]);
1191 static int gen8_init_scratch(struct i915_address_space
*vm
)
1195 ret
= setup_scratch_page(vm
, I915_GFP_DMA
);
1199 vm
->scratch_pt
= alloc_pt(vm
);
1200 if (IS_ERR(vm
->scratch_pt
)) {
1201 ret
= PTR_ERR(vm
->scratch_pt
);
1202 goto free_scratch_page
;
1205 vm
->scratch_pd
= alloc_pd(vm
);
1206 if (IS_ERR(vm
->scratch_pd
)) {
1207 ret
= PTR_ERR(vm
->scratch_pd
);
1212 vm
->scratch_pdp
= alloc_pdp(vm
);
1213 if (IS_ERR(vm
->scratch_pdp
)) {
1214 ret
= PTR_ERR(vm
->scratch_pdp
);
1219 gen8_initialize_pt(vm
, vm
->scratch_pt
);
1220 gen8_initialize_pd(vm
, vm
->scratch_pd
);
1222 gen8_initialize_pdp(vm
, vm
->scratch_pdp
);
1227 free_pd(vm
, vm
->scratch_pd
);
1229 free_pt(vm
, vm
->scratch_pt
);
1231 cleanup_scratch_page(vm
);
1236 static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt
*ppgtt
, bool create
)
1238 struct i915_address_space
*vm
= &ppgtt
->base
;
1239 struct drm_i915_private
*dev_priv
= vm
->i915
;
1240 enum vgt_g2v_type msg
;
1244 const u64 daddr
= px_dma(&ppgtt
->pml4
);
1246 I915_WRITE(vgtif_reg(pdp
[0].lo
), lower_32_bits(daddr
));
1247 I915_WRITE(vgtif_reg(pdp
[0].hi
), upper_32_bits(daddr
));
1249 msg
= (create
? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE
:
1250 VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY
);
1252 for (i
= 0; i
< GEN8_3LVL_PDPES
; i
++) {
1253 const u64 daddr
= i915_page_dir_dma_addr(ppgtt
, i
);
1255 I915_WRITE(vgtif_reg(pdp
[i
].lo
), lower_32_bits(daddr
));
1256 I915_WRITE(vgtif_reg(pdp
[i
].hi
), upper_32_bits(daddr
));
1259 msg
= (create
? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE
:
1260 VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY
);
1263 I915_WRITE(vgtif_reg(g2v_notify
), msg
);
1268 static void gen8_free_scratch(struct i915_address_space
*vm
)
1271 free_pdp(vm
, vm
->scratch_pdp
);
1272 free_pd(vm
, vm
->scratch_pd
);
1273 free_pt(vm
, vm
->scratch_pt
);
1274 cleanup_scratch_page(vm
);
1277 static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space
*vm
,
1278 struct i915_page_directory_pointer
*pdp
)
1280 const unsigned int pdpes
= i915_pdpes_per_pdp(vm
);
1283 for (i
= 0; i
< pdpes
; i
++) {
1284 if (pdp
->page_directory
[i
] == vm
->scratch_pd
)
1287 gen8_free_page_tables(vm
, pdp
->page_directory
[i
]);
1288 free_pd(vm
, pdp
->page_directory
[i
]);
1294 static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt
*ppgtt
)
1298 for (i
= 0; i
< GEN8_PML4ES_PER_PML4
; i
++) {
1299 if (ppgtt
->pml4
.pdps
[i
] == ppgtt
->base
.scratch_pdp
)
1302 gen8_ppgtt_cleanup_3lvl(&ppgtt
->base
, ppgtt
->pml4
.pdps
[i
]);
1305 cleanup_px(&ppgtt
->base
, &ppgtt
->pml4
);
1308 static void gen8_ppgtt_cleanup(struct i915_address_space
*vm
)
1310 struct drm_i915_private
*dev_priv
= vm
->i915
;
1311 struct i915_hw_ppgtt
*ppgtt
= i915_vm_to_ppgtt(vm
);
1313 if (intel_vgpu_active(dev_priv
))
1314 gen8_ppgtt_notify_vgt(ppgtt
, false);
1317 gen8_ppgtt_cleanup_4lvl(ppgtt
);
1319 gen8_ppgtt_cleanup_3lvl(&ppgtt
->base
, &ppgtt
->pdp
);
1321 gen8_free_scratch(vm
);
1324 static int gen8_ppgtt_alloc_pd(struct i915_address_space
*vm
,
1325 struct i915_page_directory
*pd
,
1326 u64 start
, u64 length
)
1328 struct i915_page_table
*pt
;
1332 gen8_for_each_pde(pt
, pd
, start
, length
, pde
) {
1333 int count
= gen8_pte_count(start
, length
);
1335 if (pt
== vm
->scratch_pt
) {
1340 if (count
< GEN8_PTES
|| intel_vgpu_active(vm
->i915
))
1341 gen8_initialize_pt(vm
, pt
);
1343 gen8_ppgtt_set_pde(vm
, pd
, pt
, pde
);
1345 GEM_BUG_ON(pd
->used_pdes
> I915_PDES
);
1348 pt
->used_ptes
+= count
;
1353 gen8_ppgtt_clear_pd(vm
, pd
, from
, start
- from
);
1357 static int gen8_ppgtt_alloc_pdp(struct i915_address_space
*vm
,
1358 struct i915_page_directory_pointer
*pdp
,
1359 u64 start
, u64 length
)
1361 struct i915_page_directory
*pd
;
1366 gen8_for_each_pdpe(pd
, pdp
, start
, length
, pdpe
) {
1367 if (pd
== vm
->scratch_pd
) {
1372 gen8_initialize_pd(vm
, pd
);
1373 gen8_ppgtt_set_pdpe(vm
, pdp
, pd
, pdpe
);
1375 GEM_BUG_ON(pdp
->used_pdpes
> i915_pdpes_per_pdp(vm
));
1377 mark_tlbs_dirty(i915_vm_to_ppgtt(vm
));
1380 ret
= gen8_ppgtt_alloc_pd(vm
, pd
, start
, length
);
1388 if (!pd
->used_pdes
) {
1389 gen8_ppgtt_set_pdpe(vm
, pdp
, vm
->scratch_pd
, pdpe
);
1390 GEM_BUG_ON(!pdp
->used_pdpes
);
1395 gen8_ppgtt_clear_pdp(vm
, pdp
, from
, start
- from
);
1399 static int gen8_ppgtt_alloc_3lvl(struct i915_address_space
*vm
,
1400 u64 start
, u64 length
)
1402 return gen8_ppgtt_alloc_pdp(vm
,
1403 &i915_vm_to_ppgtt(vm
)->pdp
, start
, length
);
1406 static int gen8_ppgtt_alloc_4lvl(struct i915_address_space
*vm
,
1407 u64 start
, u64 length
)
1409 struct i915_hw_ppgtt
*ppgtt
= i915_vm_to_ppgtt(vm
);
1410 struct i915_pml4
*pml4
= &ppgtt
->pml4
;
1411 struct i915_page_directory_pointer
*pdp
;
1416 gen8_for_each_pml4e(pdp
, pml4
, start
, length
, pml4e
) {
1417 if (pml4
->pdps
[pml4e
] == vm
->scratch_pdp
) {
1418 pdp
= alloc_pdp(vm
);
1422 gen8_initialize_pdp(vm
, pdp
);
1423 gen8_ppgtt_set_pml4e(pml4
, pdp
, pml4e
);
1426 ret
= gen8_ppgtt_alloc_pdp(vm
, pdp
, start
, length
);
1434 if (!pdp
->used_pdpes
) {
1435 gen8_ppgtt_set_pml4e(pml4
, vm
->scratch_pdp
, pml4e
);
1439 gen8_ppgtt_clear_4lvl(vm
, from
, start
- from
);
1443 static void gen8_dump_pdp(struct i915_hw_ppgtt
*ppgtt
,
1444 struct i915_page_directory_pointer
*pdp
,
1445 u64 start
, u64 length
,
1446 gen8_pte_t scratch_pte
,
1449 struct i915_address_space
*vm
= &ppgtt
->base
;
1450 struct i915_page_directory
*pd
;
1453 gen8_for_each_pdpe(pd
, pdp
, start
, length
, pdpe
) {
1454 struct i915_page_table
*pt
;
1455 u64 pd_len
= length
;
1456 u64 pd_start
= start
;
1459 if (pdp
->page_directory
[pdpe
] == ppgtt
->base
.scratch_pd
)
1462 seq_printf(m
, "\tPDPE #%d\n", pdpe
);
1463 gen8_for_each_pde(pt
, pd
, pd_start
, pd_len
, pde
) {
1465 gen8_pte_t
*pt_vaddr
;
1467 if (pd
->page_table
[pde
] == ppgtt
->base
.scratch_pt
)
1470 pt_vaddr
= kmap_atomic_px(pt
);
1471 for (pte
= 0; pte
< GEN8_PTES
; pte
+= 4) {
1472 u64 va
= (pdpe
<< GEN8_PDPE_SHIFT
|
1473 pde
<< GEN8_PDE_SHIFT
|
1474 pte
<< GEN8_PTE_SHIFT
);
1478 for (i
= 0; i
< 4; i
++)
1479 if (pt_vaddr
[pte
+ i
] != scratch_pte
)
1484 seq_printf(m
, "\t\t0x%llx [%03d,%03d,%04d]: =", va
, pdpe
, pde
, pte
);
1485 for (i
= 0; i
< 4; i
++) {
1486 if (pt_vaddr
[pte
+ i
] != scratch_pte
)
1487 seq_printf(m
, " %llx", pt_vaddr
[pte
+ i
]);
1489 seq_puts(m
, " SCRATCH ");
1493 kunmap_atomic(pt_vaddr
);
1498 static void gen8_dump_ppgtt(struct i915_hw_ppgtt
*ppgtt
, struct seq_file
*m
)
1500 struct i915_address_space
*vm
= &ppgtt
->base
;
1501 const gen8_pte_t scratch_pte
=
1502 gen8_pte_encode(vm
->scratch_page
.daddr
, I915_CACHE_LLC
);
1503 u64 start
= 0, length
= ppgtt
->base
.total
;
1507 struct i915_pml4
*pml4
= &ppgtt
->pml4
;
1508 struct i915_page_directory_pointer
*pdp
;
1510 gen8_for_each_pml4e(pdp
, pml4
, start
, length
, pml4e
) {
1511 if (pml4
->pdps
[pml4e
] == ppgtt
->base
.scratch_pdp
)
1514 seq_printf(m
, " PML4E #%llu\n", pml4e
);
1515 gen8_dump_pdp(ppgtt
, pdp
, start
, length
, scratch_pte
, m
);
1518 gen8_dump_pdp(ppgtt
, &ppgtt
->pdp
, start
, length
, scratch_pte
, m
);
1522 static int gen8_preallocate_top_level_pdp(struct i915_hw_ppgtt
*ppgtt
)
1524 struct i915_address_space
*vm
= &ppgtt
->base
;
1525 struct i915_page_directory_pointer
*pdp
= &ppgtt
->pdp
;
1526 struct i915_page_directory
*pd
;
1527 u64 start
= 0, length
= ppgtt
->base
.total
;
1531 gen8_for_each_pdpe(pd
, pdp
, start
, length
, pdpe
) {
1536 gen8_initialize_pd(vm
, pd
);
1537 gen8_ppgtt_set_pdpe(vm
, pdp
, pd
, pdpe
);
1541 pdp
->used_pdpes
++; /* never remove */
1546 gen8_for_each_pdpe(pd
, pdp
, from
, start
, pdpe
) {
1547 gen8_ppgtt_set_pdpe(vm
, pdp
, vm
->scratch_pd
, pdpe
);
1550 pdp
->used_pdpes
= 0;
1555 * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
1556 * with a net effect resembling a 2-level page table in normal x86 terms. Each
1557 * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
1561 static int gen8_ppgtt_init(struct i915_hw_ppgtt
*ppgtt
)
1563 struct i915_address_space
*vm
= &ppgtt
->base
;
1564 struct drm_i915_private
*dev_priv
= vm
->i915
;
1567 ppgtt
->base
.total
= USES_FULL_48BIT_PPGTT(dev_priv
) ?
1571 /* There are only few exceptions for gen >=6. chv and bxt.
1572 * And we are not sure about the latter so play safe for now.
1574 if (IS_CHERRYVIEW(dev_priv
) || IS_BROXTON(dev_priv
))
1575 ppgtt
->base
.pt_kmap_wc
= true;
1577 ret
= gen8_init_scratch(&ppgtt
->base
);
1579 ppgtt
->base
.total
= 0;
1584 ret
= setup_px(&ppgtt
->base
, &ppgtt
->pml4
);
1588 gen8_initialize_pml4(&ppgtt
->base
, &ppgtt
->pml4
);
1590 ppgtt
->switch_mm
= gen8_mm_switch_4lvl
;
1591 ppgtt
->base
.allocate_va_range
= gen8_ppgtt_alloc_4lvl
;
1592 ppgtt
->base
.insert_entries
= gen8_ppgtt_insert_4lvl
;
1593 ppgtt
->base
.clear_range
= gen8_ppgtt_clear_4lvl
;
1595 ret
= __pdp_init(&ppgtt
->base
, &ppgtt
->pdp
);
1599 if (intel_vgpu_active(dev_priv
)) {
1600 ret
= gen8_preallocate_top_level_pdp(ppgtt
);
1602 __pdp_fini(&ppgtt
->pdp
);
1607 ppgtt
->switch_mm
= gen8_mm_switch_3lvl
;
1608 ppgtt
->base
.allocate_va_range
= gen8_ppgtt_alloc_3lvl
;
1609 ppgtt
->base
.insert_entries
= gen8_ppgtt_insert_3lvl
;
1610 ppgtt
->base
.clear_range
= gen8_ppgtt_clear_3lvl
;
1613 if (intel_vgpu_active(dev_priv
))
1614 gen8_ppgtt_notify_vgt(ppgtt
, true);
1616 ppgtt
->base
.cleanup
= gen8_ppgtt_cleanup
;
1617 ppgtt
->base
.unbind_vma
= ppgtt_unbind_vma
;
1618 ppgtt
->base
.bind_vma
= ppgtt_bind_vma
;
1619 ppgtt
->base
.set_pages
= ppgtt_set_pages
;
1620 ppgtt
->base
.clear_pages
= clear_pages
;
1621 ppgtt
->debug_dump
= gen8_dump_ppgtt
;
1626 gen8_free_scratch(&ppgtt
->base
);
1630 static void gen6_dump_ppgtt(struct i915_hw_ppgtt
*ppgtt
, struct seq_file
*m
)
1632 struct i915_address_space
*vm
= &ppgtt
->base
;
1633 struct i915_page_table
*unused
;
1634 gen6_pte_t scratch_pte
;
1635 u32 pd_entry
, pte
, pde
;
1636 u32 start
= 0, length
= ppgtt
->base
.total
;
1638 scratch_pte
= vm
->pte_encode(vm
->scratch_page
.daddr
,
1641 gen6_for_each_pde(unused
, &ppgtt
->pd
, start
, length
, pde
) {
1643 gen6_pte_t
*pt_vaddr
;
1644 const dma_addr_t pt_addr
= px_dma(ppgtt
->pd
.page_table
[pde
]);
1645 pd_entry
= readl(ppgtt
->pd_addr
+ pde
);
1646 expected
= (GEN6_PDE_ADDR_ENCODE(pt_addr
) | GEN6_PDE_VALID
);
1648 if (pd_entry
!= expected
)
1649 seq_printf(m
, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n",
1653 seq_printf(m
, "\tPDE: %x\n", pd_entry
);
1655 pt_vaddr
= kmap_atomic_px(ppgtt
->pd
.page_table
[pde
]);
1657 for (pte
= 0; pte
< GEN6_PTES
; pte
+=4) {
1659 (pde
* PAGE_SIZE
* GEN6_PTES
) +
1663 for (i
= 0; i
< 4; i
++)
1664 if (pt_vaddr
[pte
+ i
] != scratch_pte
)
1669 seq_printf(m
, "\t\t0x%lx [%03d,%04d]: =", va
, pde
, pte
);
1670 for (i
= 0; i
< 4; i
++) {
1671 if (pt_vaddr
[pte
+ i
] != scratch_pte
)
1672 seq_printf(m
, " %08x", pt_vaddr
[pte
+ i
]);
1674 seq_puts(m
, " SCRATCH ");
1678 kunmap_atomic(pt_vaddr
);
1682 /* Write pde (index) from the page directory @pd to the page table @pt */
1683 static inline void gen6_write_pde(const struct i915_hw_ppgtt
*ppgtt
,
1684 const unsigned int pde
,
1685 const struct i915_page_table
*pt
)
1687 /* Caller needs to make sure the write completes if necessary */
1688 writel_relaxed(GEN6_PDE_ADDR_ENCODE(px_dma(pt
)) | GEN6_PDE_VALID
,
1689 ppgtt
->pd_addr
+ pde
);
1692 /* Write all the page tables found in the ppgtt structure to incrementing page
1694 static void gen6_write_page_range(struct i915_hw_ppgtt
*ppgtt
,
1695 u32 start
, u32 length
)
1697 struct i915_page_table
*pt
;
1700 gen6_for_each_pde(pt
, &ppgtt
->pd
, start
, length
, pde
)
1701 gen6_write_pde(ppgtt
, pde
, pt
);
1703 mark_tlbs_dirty(ppgtt
);
1707 static inline u32
get_pd_offset(struct i915_hw_ppgtt
*ppgtt
)
1709 GEM_BUG_ON(ppgtt
->pd
.base
.ggtt_offset
& 0x3f);
1710 return ppgtt
->pd
.base
.ggtt_offset
<< 10;
1713 static int hsw_mm_switch(struct i915_hw_ppgtt
*ppgtt
,
1714 struct drm_i915_gem_request
*req
)
1716 struct intel_engine_cs
*engine
= req
->engine
;
1719 /* NB: TLBs must be flushed and invalidated before a switch */
1720 cs
= intel_ring_begin(req
, 6);
1724 *cs
++ = MI_LOAD_REGISTER_IMM(2);
1725 *cs
++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine
));
1726 *cs
++ = PP_DIR_DCLV_2G
;
1727 *cs
++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine
));
1728 *cs
++ = get_pd_offset(ppgtt
);
1730 intel_ring_advance(req
, cs
);
1735 static int gen7_mm_switch(struct i915_hw_ppgtt
*ppgtt
,
1736 struct drm_i915_gem_request
*req
)
1738 struct intel_engine_cs
*engine
= req
->engine
;
1741 /* NB: TLBs must be flushed and invalidated before a switch */
1742 cs
= intel_ring_begin(req
, 6);
1746 *cs
++ = MI_LOAD_REGISTER_IMM(2);
1747 *cs
++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine
));
1748 *cs
++ = PP_DIR_DCLV_2G
;
1749 *cs
++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine
));
1750 *cs
++ = get_pd_offset(ppgtt
);
1752 intel_ring_advance(req
, cs
);
1757 static int gen6_mm_switch(struct i915_hw_ppgtt
*ppgtt
,
1758 struct drm_i915_gem_request
*req
)
1760 struct intel_engine_cs
*engine
= req
->engine
;
1761 struct drm_i915_private
*dev_priv
= req
->i915
;
1763 I915_WRITE(RING_PP_DIR_DCLV(engine
), PP_DIR_DCLV_2G
);
1764 I915_WRITE(RING_PP_DIR_BASE(engine
), get_pd_offset(ppgtt
));
1768 static void gen8_ppgtt_enable(struct drm_i915_private
*dev_priv
)
1770 struct intel_engine_cs
*engine
;
1771 enum intel_engine_id id
;
1773 for_each_engine(engine
, dev_priv
, id
) {
1774 u32 four_level
= USES_FULL_48BIT_PPGTT(dev_priv
) ?
1775 GEN8_GFX_PPGTT_48B
: 0;
1776 I915_WRITE(RING_MODE_GEN7(engine
),
1777 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE
| four_level
));
1781 static void gen7_ppgtt_enable(struct drm_i915_private
*dev_priv
)
1783 struct intel_engine_cs
*engine
;
1784 u32 ecochk
, ecobits
;
1785 enum intel_engine_id id
;
1787 ecobits
= I915_READ(GAC_ECO_BITS
);
1788 I915_WRITE(GAC_ECO_BITS
, ecobits
| ECOBITS_PPGTT_CACHE64B
);
1790 ecochk
= I915_READ(GAM_ECOCHK
);
1791 if (IS_HASWELL(dev_priv
)) {
1792 ecochk
|= ECOCHK_PPGTT_WB_HSW
;
1794 ecochk
|= ECOCHK_PPGTT_LLC_IVB
;
1795 ecochk
&= ~ECOCHK_PPGTT_GFDT_IVB
;
1797 I915_WRITE(GAM_ECOCHK
, ecochk
);
1799 for_each_engine(engine
, dev_priv
, id
) {
1800 /* GFX_MODE is per-ring on gen7+ */
1801 I915_WRITE(RING_MODE_GEN7(engine
),
1802 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE
));
1806 static void gen6_ppgtt_enable(struct drm_i915_private
*dev_priv
)
1808 u32 ecochk
, gab_ctl
, ecobits
;
1810 ecobits
= I915_READ(GAC_ECO_BITS
);
1811 I915_WRITE(GAC_ECO_BITS
, ecobits
| ECOBITS_SNB_BIT
|
1812 ECOBITS_PPGTT_CACHE64B
);
1814 gab_ctl
= I915_READ(GAB_CTL
);
1815 I915_WRITE(GAB_CTL
, gab_ctl
| GAB_CTL_CONT_AFTER_PAGEFAULT
);
1817 ecochk
= I915_READ(GAM_ECOCHK
);
1818 I915_WRITE(GAM_ECOCHK
, ecochk
| ECOCHK_SNB_BIT
| ECOCHK_PPGTT_CACHE64B
);
1820 I915_WRITE(GFX_MODE
, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE
));
1823 /* PPGTT support for Sandybdrige/Gen6 and later */
1824 static void gen6_ppgtt_clear_range(struct i915_address_space
*vm
,
1825 u64 start
, u64 length
)
1827 struct i915_hw_ppgtt
*ppgtt
= i915_vm_to_ppgtt(vm
);
1828 unsigned int first_entry
= start
>> PAGE_SHIFT
;
1829 unsigned int pde
= first_entry
/ GEN6_PTES
;
1830 unsigned int pte
= first_entry
% GEN6_PTES
;
1831 unsigned int num_entries
= length
>> PAGE_SHIFT
;
1832 gen6_pte_t scratch_pte
=
1833 vm
->pte_encode(vm
->scratch_page
.daddr
, I915_CACHE_LLC
, 0);
1835 while (num_entries
) {
1836 struct i915_page_table
*pt
= ppgtt
->pd
.page_table
[pde
++];
1837 unsigned int end
= min(pte
+ num_entries
, GEN6_PTES
);
1840 num_entries
-= end
- pte
;
1842 /* Note that the hw doesn't support removing PDE on the fly
1843 * (they are cached inside the context with no means to
1844 * invalidate the cache), so we can only reset the PTE
1845 * entries back to scratch.
1848 vaddr
= kmap_atomic_px(pt
);
1850 vaddr
[pte
++] = scratch_pte
;
1851 } while (pte
< end
);
1852 kunmap_atomic(vaddr
);
1858 static void gen6_ppgtt_insert_entries(struct i915_address_space
*vm
,
1859 struct i915_vma
*vma
,
1860 enum i915_cache_level cache_level
,
1863 struct i915_hw_ppgtt
*ppgtt
= i915_vm_to_ppgtt(vm
);
1864 unsigned first_entry
= vma
->node
.start
>> PAGE_SHIFT
;
1865 unsigned act_pt
= first_entry
/ GEN6_PTES
;
1866 unsigned act_pte
= first_entry
% GEN6_PTES
;
1867 const u32 pte_encode
= vm
->pte_encode(0, cache_level
, flags
);
1868 struct sgt_dma iter
= sgt_dma(vma
);
1871 vaddr
= kmap_atomic_px(ppgtt
->pd
.page_table
[act_pt
]);
1873 vaddr
[act_pte
] = pte_encode
| GEN6_PTE_ADDR_ENCODE(iter
.dma
);
1875 iter
.dma
+= PAGE_SIZE
;
1876 if (iter
.dma
== iter
.max
) {
1877 iter
.sg
= __sg_next(iter
.sg
);
1881 iter
.dma
= sg_dma_address(iter
.sg
);
1882 iter
.max
= iter
.dma
+ iter
.sg
->length
;
1885 if (++act_pte
== GEN6_PTES
) {
1886 kunmap_atomic(vaddr
);
1887 vaddr
= kmap_atomic_px(ppgtt
->pd
.page_table
[++act_pt
]);
1891 kunmap_atomic(vaddr
);
1893 vma
->page_sizes
.gtt
= I915_GTT_PAGE_SIZE
;
1896 static int gen6_alloc_va_range(struct i915_address_space
*vm
,
1897 u64 start
, u64 length
)
1899 struct i915_hw_ppgtt
*ppgtt
= i915_vm_to_ppgtt(vm
);
1900 struct i915_page_table
*pt
;
1905 gen6_for_each_pde(pt
, &ppgtt
->pd
, start
, length
, pde
) {
1906 if (pt
== vm
->scratch_pt
) {
1911 gen6_initialize_pt(vm
, pt
);
1912 ppgtt
->pd
.page_table
[pde
] = pt
;
1913 gen6_write_pde(ppgtt
, pde
, pt
);
1919 mark_tlbs_dirty(ppgtt
);
1926 gen6_ppgtt_clear_range(vm
, from
, start
);
1930 static int gen6_init_scratch(struct i915_address_space
*vm
)
1934 ret
= setup_scratch_page(vm
, I915_GFP_DMA
);
1938 vm
->scratch_pt
= alloc_pt(vm
);
1939 if (IS_ERR(vm
->scratch_pt
)) {
1940 cleanup_scratch_page(vm
);
1941 return PTR_ERR(vm
->scratch_pt
);
1944 gen6_initialize_pt(vm
, vm
->scratch_pt
);
1949 static void gen6_free_scratch(struct i915_address_space
*vm
)
1951 free_pt(vm
, vm
->scratch_pt
);
1952 cleanup_scratch_page(vm
);
1955 static void gen6_ppgtt_cleanup(struct i915_address_space
*vm
)
1957 struct i915_hw_ppgtt
*ppgtt
= i915_vm_to_ppgtt(vm
);
1958 struct i915_page_directory
*pd
= &ppgtt
->pd
;
1959 struct i915_page_table
*pt
;
1962 drm_mm_remove_node(&ppgtt
->node
);
1964 gen6_for_all_pdes(pt
, pd
, pde
)
1965 if (pt
!= vm
->scratch_pt
)
1968 gen6_free_scratch(vm
);
1971 static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt
*ppgtt
)
1973 struct i915_address_space
*vm
= &ppgtt
->base
;
1974 struct drm_i915_private
*dev_priv
= ppgtt
->base
.i915
;
1975 struct i915_ggtt
*ggtt
= &dev_priv
->ggtt
;
1978 /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The
1979 * allocator works in address space sizes, so it's multiplied by page
1980 * size. We allocate at the top of the GTT to avoid fragmentation.
1982 BUG_ON(!drm_mm_initialized(&ggtt
->base
.mm
));
1984 ret
= gen6_init_scratch(vm
);
1988 ret
= i915_gem_gtt_insert(&ggtt
->base
, &ppgtt
->node
,
1989 GEN6_PD_SIZE
, GEN6_PD_ALIGN
,
1990 I915_COLOR_UNEVICTABLE
,
1991 0, ggtt
->base
.total
,
1996 if (ppgtt
->node
.start
< ggtt
->mappable_end
)
1997 DRM_DEBUG("Forced to use aperture for PDEs\n");
1999 ppgtt
->pd
.base
.ggtt_offset
=
2000 ppgtt
->node
.start
/ PAGE_SIZE
* sizeof(gen6_pte_t
);
2002 ppgtt
->pd_addr
= (gen6_pte_t __iomem
*)ggtt
->gsm
+
2003 ppgtt
->pd
.base
.ggtt_offset
/ sizeof(gen6_pte_t
);
2008 gen6_free_scratch(vm
);
2012 static int gen6_ppgtt_alloc(struct i915_hw_ppgtt
*ppgtt
)
2014 return gen6_ppgtt_allocate_page_directories(ppgtt
);
2017 static void gen6_scratch_va_range(struct i915_hw_ppgtt
*ppgtt
,
2018 u64 start
, u64 length
)
2020 struct i915_page_table
*unused
;
2023 gen6_for_each_pde(unused
, &ppgtt
->pd
, start
, length
, pde
)
2024 ppgtt
->pd
.page_table
[pde
] = ppgtt
->base
.scratch_pt
;
2027 static int gen6_ppgtt_init(struct i915_hw_ppgtt
*ppgtt
)
2029 struct drm_i915_private
*dev_priv
= ppgtt
->base
.i915
;
2030 struct i915_ggtt
*ggtt
= &dev_priv
->ggtt
;
2033 ppgtt
->base
.pte_encode
= ggtt
->base
.pte_encode
;
2034 if (intel_vgpu_active(dev_priv
) || IS_GEN6(dev_priv
))
2035 ppgtt
->switch_mm
= gen6_mm_switch
;
2036 else if (IS_HASWELL(dev_priv
))
2037 ppgtt
->switch_mm
= hsw_mm_switch
;
2038 else if (IS_GEN7(dev_priv
))
2039 ppgtt
->switch_mm
= gen7_mm_switch
;
2043 ret
= gen6_ppgtt_alloc(ppgtt
);
2047 ppgtt
->base
.total
= I915_PDES
* GEN6_PTES
* PAGE_SIZE
;
2049 gen6_scratch_va_range(ppgtt
, 0, ppgtt
->base
.total
);
2050 gen6_write_page_range(ppgtt
, 0, ppgtt
->base
.total
);
2052 ret
= gen6_alloc_va_range(&ppgtt
->base
, 0, ppgtt
->base
.total
);
2054 gen6_ppgtt_cleanup(&ppgtt
->base
);
2058 ppgtt
->base
.clear_range
= gen6_ppgtt_clear_range
;
2059 ppgtt
->base
.insert_entries
= gen6_ppgtt_insert_entries
;
2060 ppgtt
->base
.unbind_vma
= ppgtt_unbind_vma
;
2061 ppgtt
->base
.bind_vma
= ppgtt_bind_vma
;
2062 ppgtt
->base
.set_pages
= ppgtt_set_pages
;
2063 ppgtt
->base
.clear_pages
= clear_pages
;
2064 ppgtt
->base
.cleanup
= gen6_ppgtt_cleanup
;
2065 ppgtt
->debug_dump
= gen6_dump_ppgtt
;
2067 DRM_DEBUG_DRIVER("Allocated pde space (%lldM) at GTT entry: %llx\n",
2068 ppgtt
->node
.size
>> 20,
2069 ppgtt
->node
.start
/ PAGE_SIZE
);
2071 DRM_DEBUG_DRIVER("Adding PPGTT at offset %x\n",
2072 ppgtt
->pd
.base
.ggtt_offset
<< 10);
2077 static int __hw_ppgtt_init(struct i915_hw_ppgtt
*ppgtt
,
2078 struct drm_i915_private
*dev_priv
)
2080 ppgtt
->base
.i915
= dev_priv
;
2081 ppgtt
->base
.dma
= &dev_priv
->drm
.pdev
->dev
;
2083 if (INTEL_INFO(dev_priv
)->gen
< 8)
2084 return gen6_ppgtt_init(ppgtt
);
2086 return gen8_ppgtt_init(ppgtt
);
2089 static void i915_address_space_init(struct i915_address_space
*vm
,
2090 struct drm_i915_private
*dev_priv
,
2093 i915_gem_timeline_init(dev_priv
, &vm
->timeline
, name
);
2095 drm_mm_init(&vm
->mm
, 0, vm
->total
);
2096 vm
->mm
.head_node
.color
= I915_COLOR_UNEVICTABLE
;
2098 INIT_LIST_HEAD(&vm
->active_list
);
2099 INIT_LIST_HEAD(&vm
->inactive_list
);
2100 INIT_LIST_HEAD(&vm
->unbound_list
);
2102 list_add_tail(&vm
->global_link
, &dev_priv
->vm_list
);
2103 pagevec_init(&vm
->free_pages
);
2106 static void i915_address_space_fini(struct i915_address_space
*vm
)
2108 if (pagevec_count(&vm
->free_pages
))
2109 vm_free_pages_release(vm
, true);
2111 i915_gem_timeline_fini(&vm
->timeline
);
2112 drm_mm_takedown(&vm
->mm
);
2113 list_del(&vm
->global_link
);
2116 static void gtt_write_workarounds(struct drm_i915_private
*dev_priv
)
2118 /* This function is for gtt related workarounds. This function is
2119 * called on driver load and after a GPU reset, so you can place
2120 * workarounds here even if they get overwritten by GPU reset.
2122 /* WaIncreaseDefaultTLBEntries:chv,bdw,skl,bxt,kbl,glk,cfl,cnl */
2123 if (IS_BROADWELL(dev_priv
))
2124 I915_WRITE(GEN8_L3_LRA_1_GPGPU
, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW
);
2125 else if (IS_CHERRYVIEW(dev_priv
))
2126 I915_WRITE(GEN8_L3_LRA_1_GPGPU
, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_CHV
);
2127 else if (IS_GEN9_BC(dev_priv
) || IS_GEN10(dev_priv
))
2128 I915_WRITE(GEN8_L3_LRA_1_GPGPU
, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL
);
2129 else if (IS_GEN9_LP(dev_priv
))
2130 I915_WRITE(GEN8_L3_LRA_1_GPGPU
, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT
);
2133 * To support 64K PTEs we need to first enable the use of the
2134 * Intermediate-Page-Size(IPS) bit of the PDE field via some magical
2135 * mmio, otherwise the page-walker will simply ignore the IPS bit. This
2136 * shouldn't be needed after GEN10.
2138 * 64K pages were first introduced from BDW+, although technically they
2139 * only *work* from gen9+. For pre-BDW we instead have the option for
2140 * 32K pages, but we don't currently have any support for it in our
2143 if (HAS_PAGE_SIZES(dev_priv
, I915_GTT_PAGE_SIZE_64K
) &&
2144 INTEL_GEN(dev_priv
) <= 10)
2145 I915_WRITE(GEN8_GAMW_ECO_DEV_RW_IA
,
2146 I915_READ(GEN8_GAMW_ECO_DEV_RW_IA
) |
2147 GAMW_ECO_ENABLE_64K_IPS_FIELD
);
2150 int i915_ppgtt_init_hw(struct drm_i915_private
*dev_priv
)
2152 gtt_write_workarounds(dev_priv
);
2154 /* In the case of execlists, PPGTT is enabled by the context descriptor
2155 * and the PDPs are contained within the context itself. We don't
2156 * need to do anything here. */
2157 if (i915_modparams
.enable_execlists
)
2160 if (!USES_PPGTT(dev_priv
))
2163 if (IS_GEN6(dev_priv
))
2164 gen6_ppgtt_enable(dev_priv
);
2165 else if (IS_GEN7(dev_priv
))
2166 gen7_ppgtt_enable(dev_priv
);
2167 else if (INTEL_GEN(dev_priv
) >= 8)
2168 gen8_ppgtt_enable(dev_priv
);
2170 MISSING_CASE(INTEL_GEN(dev_priv
));
2175 struct i915_hw_ppgtt
*
2176 i915_ppgtt_create(struct drm_i915_private
*dev_priv
,
2177 struct drm_i915_file_private
*fpriv
,
2180 struct i915_hw_ppgtt
*ppgtt
;
2183 ppgtt
= kzalloc(sizeof(*ppgtt
), GFP_KERNEL
);
2185 return ERR_PTR(-ENOMEM
);
2187 ret
= __hw_ppgtt_init(ppgtt
, dev_priv
);
2190 return ERR_PTR(ret
);
2193 kref_init(&ppgtt
->ref
);
2194 i915_address_space_init(&ppgtt
->base
, dev_priv
, name
);
2195 ppgtt
->base
.file
= fpriv
;
2197 trace_i915_ppgtt_create(&ppgtt
->base
);
2202 void i915_ppgtt_close(struct i915_address_space
*vm
)
2204 struct list_head
*phases
[] = {
2211 GEM_BUG_ON(vm
->closed
);
2214 for (phase
= phases
; *phase
; phase
++) {
2215 struct i915_vma
*vma
, *vn
;
2217 list_for_each_entry_safe(vma
, vn
, *phase
, vm_link
)
2218 if (!i915_vma_is_closed(vma
))
2219 i915_vma_close(vma
);
2223 void i915_ppgtt_release(struct kref
*kref
)
2225 struct i915_hw_ppgtt
*ppgtt
=
2226 container_of(kref
, struct i915_hw_ppgtt
, ref
);
2228 trace_i915_ppgtt_release(&ppgtt
->base
);
2230 /* vmas should already be unbound and destroyed */
2231 WARN_ON(!list_empty(&ppgtt
->base
.active_list
));
2232 WARN_ON(!list_empty(&ppgtt
->base
.inactive_list
));
2233 WARN_ON(!list_empty(&ppgtt
->base
.unbound_list
));
2235 ppgtt
->base
.cleanup(&ppgtt
->base
);
2236 i915_address_space_fini(&ppgtt
->base
);
2240 /* Certain Gen5 chipsets require require idling the GPU before
2241 * unmapping anything from the GTT when VT-d is enabled.
2243 static bool needs_idle_maps(struct drm_i915_private
*dev_priv
)
2245 /* Query intel_iommu to see if we need the workaround. Presumably that
2248 return IS_GEN5(dev_priv
) && IS_MOBILE(dev_priv
) && intel_vtd_active();
2251 void i915_check_and_clear_faults(struct drm_i915_private
*dev_priv
)
2253 struct intel_engine_cs
*engine
;
2254 enum intel_engine_id id
;
2256 if (INTEL_INFO(dev_priv
)->gen
< 6)
2259 for_each_engine(engine
, dev_priv
, id
) {
2261 fault_reg
= I915_READ(RING_FAULT_REG(engine
));
2262 if (fault_reg
& RING_FAULT_VALID
) {
2263 DRM_DEBUG_DRIVER("Unexpected fault\n"
2265 "\tAddress space: %s\n"
2268 fault_reg
& PAGE_MASK
,
2269 fault_reg
& RING_FAULT_GTTSEL_MASK
? "GGTT" : "PPGTT",
2270 RING_FAULT_SRCID(fault_reg
),
2271 RING_FAULT_FAULT_TYPE(fault_reg
));
2272 I915_WRITE(RING_FAULT_REG(engine
),
2273 fault_reg
& ~RING_FAULT_VALID
);
2277 /* Engine specific init may not have been done till this point. */
2278 if (dev_priv
->engine
[RCS
])
2279 POSTING_READ(RING_FAULT_REG(dev_priv
->engine
[RCS
]));
2282 void i915_gem_suspend_gtt_mappings(struct drm_i915_private
*dev_priv
)
2284 struct i915_ggtt
*ggtt
= &dev_priv
->ggtt
;
2286 /* Don't bother messing with faults pre GEN6 as we have little
2287 * documentation supporting that it's a good idea.
2289 if (INTEL_GEN(dev_priv
) < 6)
2292 i915_check_and_clear_faults(dev_priv
);
2294 ggtt
->base
.clear_range(&ggtt
->base
, 0, ggtt
->base
.total
);
2296 i915_ggtt_invalidate(dev_priv
);
2299 int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object
*obj
,
2300 struct sg_table
*pages
)
2303 if (dma_map_sg(&obj
->base
.dev
->pdev
->dev
,
2304 pages
->sgl
, pages
->nents
,
2305 PCI_DMA_BIDIRECTIONAL
))
2308 /* If the DMA remap fails, one cause can be that we have
2309 * too many objects pinned in a small remapping table,
2310 * such as swiotlb. Incrementally purge all other objects and
2311 * try again - if there are no more pages to remove from
2312 * the DMA remapper, i915_gem_shrink will return 0.
2314 GEM_BUG_ON(obj
->mm
.pages
== pages
);
2315 } while (i915_gem_shrink(to_i915(obj
->base
.dev
),
2316 obj
->base
.size
>> PAGE_SHIFT
, NULL
,
2318 I915_SHRINK_UNBOUND
|
2319 I915_SHRINK_ACTIVE
));
2324 static void gen8_set_pte(void __iomem
*addr
, gen8_pte_t pte
)
2329 static void gen8_ggtt_insert_page(struct i915_address_space
*vm
,
2332 enum i915_cache_level level
,
2335 struct i915_ggtt
*ggtt
= i915_vm_to_ggtt(vm
);
2336 gen8_pte_t __iomem
*pte
=
2337 (gen8_pte_t __iomem
*)ggtt
->gsm
+ (offset
>> PAGE_SHIFT
);
2339 gen8_set_pte(pte
, gen8_pte_encode(addr
, level
));
2341 ggtt
->invalidate(vm
->i915
);
2344 static void gen8_ggtt_insert_entries(struct i915_address_space
*vm
,
2345 struct i915_vma
*vma
,
2346 enum i915_cache_level level
,
2349 struct i915_ggtt
*ggtt
= i915_vm_to_ggtt(vm
);
2350 struct sgt_iter sgt_iter
;
2351 gen8_pte_t __iomem
*gtt_entries
;
2352 const gen8_pte_t pte_encode
= gen8_pte_encode(0, level
);
2355 gtt_entries
= (gen8_pte_t __iomem
*)ggtt
->gsm
;
2356 gtt_entries
+= vma
->node
.start
>> PAGE_SHIFT
;
2357 for_each_sgt_dma(addr
, sgt_iter
, vma
->pages
)
2358 gen8_set_pte(gtt_entries
++, pte_encode
| addr
);
2362 /* This next bit makes the above posting read even more important. We
2363 * want to flush the TLBs only after we're certain all the PTE updates
2366 ggtt
->invalidate(vm
->i915
);
2369 static void gen6_ggtt_insert_page(struct i915_address_space
*vm
,
2372 enum i915_cache_level level
,
2375 struct i915_ggtt
*ggtt
= i915_vm_to_ggtt(vm
);
2376 gen6_pte_t __iomem
*pte
=
2377 (gen6_pte_t __iomem
*)ggtt
->gsm
+ (offset
>> PAGE_SHIFT
);
2379 iowrite32(vm
->pte_encode(addr
, level
, flags
), pte
);
2381 ggtt
->invalidate(vm
->i915
);
2385 * Binds an object into the global gtt with the specified cache level. The object
2386 * will be accessible to the GPU via commands whose operands reference offsets
2387 * within the global GTT as well as accessible by the GPU through the GMADR
2388 * mapped BAR (dev_priv->mm.gtt->gtt).
2390 static void gen6_ggtt_insert_entries(struct i915_address_space
*vm
,
2391 struct i915_vma
*vma
,
2392 enum i915_cache_level level
,
2395 struct i915_ggtt
*ggtt
= i915_vm_to_ggtt(vm
);
2396 gen6_pte_t __iomem
*entries
= (gen6_pte_t __iomem
*)ggtt
->gsm
;
2397 unsigned int i
= vma
->node
.start
>> PAGE_SHIFT
;
2398 struct sgt_iter iter
;
2400 for_each_sgt_dma(addr
, iter
, vma
->pages
)
2401 iowrite32(vm
->pte_encode(addr
, level
, flags
), &entries
[i
++]);
2404 /* This next bit makes the above posting read even more important. We
2405 * want to flush the TLBs only after we're certain all the PTE updates
2408 ggtt
->invalidate(vm
->i915
);
2411 static void nop_clear_range(struct i915_address_space
*vm
,
2412 u64 start
, u64 length
)
2416 static void gen8_ggtt_clear_range(struct i915_address_space
*vm
,
2417 u64 start
, u64 length
)
2419 struct i915_ggtt
*ggtt
= i915_vm_to_ggtt(vm
);
2420 unsigned first_entry
= start
>> PAGE_SHIFT
;
2421 unsigned num_entries
= length
>> PAGE_SHIFT
;
2422 const gen8_pte_t scratch_pte
=
2423 gen8_pte_encode(vm
->scratch_page
.daddr
, I915_CACHE_LLC
);
2424 gen8_pte_t __iomem
*gtt_base
=
2425 (gen8_pte_t __iomem
*)ggtt
->gsm
+ first_entry
;
2426 const int max_entries
= ggtt_total_entries(ggtt
) - first_entry
;
2429 if (WARN(num_entries
> max_entries
,
2430 "First entry = %d; Num entries = %d (max=%d)\n",
2431 first_entry
, num_entries
, max_entries
))
2432 num_entries
= max_entries
;
2434 for (i
= 0; i
< num_entries
; i
++)
2435 gen8_set_pte(>t_base
[i
], scratch_pte
);
2438 static void bxt_vtd_ggtt_wa(struct i915_address_space
*vm
)
2440 struct drm_i915_private
*dev_priv
= vm
->i915
;
2443 * Make sure the internal GAM fifo has been cleared of all GTT
2444 * writes before exiting stop_machine(). This guarantees that
2445 * any aperture accesses waiting to start in another process
2446 * cannot back up behind the GTT writes causing a hang.
2447 * The register can be any arbitrary GAM register.
2449 POSTING_READ(GFX_FLSH_CNTL_GEN6
);
2452 struct insert_page
{
2453 struct i915_address_space
*vm
;
2456 enum i915_cache_level level
;
2459 static int bxt_vtd_ggtt_insert_page__cb(void *_arg
)
2461 struct insert_page
*arg
= _arg
;
2463 gen8_ggtt_insert_page(arg
->vm
, arg
->addr
, arg
->offset
, arg
->level
, 0);
2464 bxt_vtd_ggtt_wa(arg
->vm
);
2469 static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space
*vm
,
2472 enum i915_cache_level level
,
2475 struct insert_page arg
= { vm
, addr
, offset
, level
};
2477 stop_machine(bxt_vtd_ggtt_insert_page__cb
, &arg
, NULL
);
2480 struct insert_entries
{
2481 struct i915_address_space
*vm
;
2482 struct i915_vma
*vma
;
2483 enum i915_cache_level level
;
2486 static int bxt_vtd_ggtt_insert_entries__cb(void *_arg
)
2488 struct insert_entries
*arg
= _arg
;
2490 gen8_ggtt_insert_entries(arg
->vm
, arg
->vma
, arg
->level
, 0);
2491 bxt_vtd_ggtt_wa(arg
->vm
);
2496 static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space
*vm
,
2497 struct i915_vma
*vma
,
2498 enum i915_cache_level level
,
2501 struct insert_entries arg
= { vm
, vma
, level
};
2503 stop_machine(bxt_vtd_ggtt_insert_entries__cb
, &arg
, NULL
);
2506 struct clear_range
{
2507 struct i915_address_space
*vm
;
2512 static int bxt_vtd_ggtt_clear_range__cb(void *_arg
)
2514 struct clear_range
*arg
= _arg
;
2516 gen8_ggtt_clear_range(arg
->vm
, arg
->start
, arg
->length
);
2517 bxt_vtd_ggtt_wa(arg
->vm
);
2522 static void bxt_vtd_ggtt_clear_range__BKL(struct i915_address_space
*vm
,
2526 struct clear_range arg
= { vm
, start
, length
};
2528 stop_machine(bxt_vtd_ggtt_clear_range__cb
, &arg
, NULL
);
2531 static void gen6_ggtt_clear_range(struct i915_address_space
*vm
,
2532 u64 start
, u64 length
)
2534 struct i915_ggtt
*ggtt
= i915_vm_to_ggtt(vm
);
2535 unsigned first_entry
= start
>> PAGE_SHIFT
;
2536 unsigned num_entries
= length
>> PAGE_SHIFT
;
2537 gen6_pte_t scratch_pte
, __iomem
*gtt_base
=
2538 (gen6_pte_t __iomem
*)ggtt
->gsm
+ first_entry
;
2539 const int max_entries
= ggtt_total_entries(ggtt
) - first_entry
;
2542 if (WARN(num_entries
> max_entries
,
2543 "First entry = %d; Num entries = %d (max=%d)\n",
2544 first_entry
, num_entries
, max_entries
))
2545 num_entries
= max_entries
;
2547 scratch_pte
= vm
->pte_encode(vm
->scratch_page
.daddr
,
2550 for (i
= 0; i
< num_entries
; i
++)
2551 iowrite32(scratch_pte
, >t_base
[i
]);
2554 static void i915_ggtt_insert_page(struct i915_address_space
*vm
,
2557 enum i915_cache_level cache_level
,
2560 unsigned int flags
= (cache_level
== I915_CACHE_NONE
) ?
2561 AGP_USER_MEMORY
: AGP_USER_CACHED_MEMORY
;
2563 intel_gtt_insert_page(addr
, offset
>> PAGE_SHIFT
, flags
);
2566 static void i915_ggtt_insert_entries(struct i915_address_space
*vm
,
2567 struct i915_vma
*vma
,
2568 enum i915_cache_level cache_level
,
2571 unsigned int flags
= (cache_level
== I915_CACHE_NONE
) ?
2572 AGP_USER_MEMORY
: AGP_USER_CACHED_MEMORY
;
2574 intel_gtt_insert_sg_entries(vma
->pages
, vma
->node
.start
>> PAGE_SHIFT
,
2578 static void i915_ggtt_clear_range(struct i915_address_space
*vm
,
2579 u64 start
, u64 length
)
2581 intel_gtt_clear_range(start
>> PAGE_SHIFT
, length
>> PAGE_SHIFT
);
2584 static int ggtt_bind_vma(struct i915_vma
*vma
,
2585 enum i915_cache_level cache_level
,
2588 struct drm_i915_private
*i915
= vma
->vm
->i915
;
2589 struct drm_i915_gem_object
*obj
= vma
->obj
;
2592 /* Currently applicable only to VLV */
2595 pte_flags
|= PTE_READ_ONLY
;
2597 intel_runtime_pm_get(i915
);
2598 vma
->vm
->insert_entries(vma
->vm
, vma
, cache_level
, pte_flags
);
2599 intel_runtime_pm_put(i915
);
2601 vma
->page_sizes
.gtt
= I915_GTT_PAGE_SIZE
;
2604 * Without aliasing PPGTT there's no difference between
2605 * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
2606 * upgrade to both bound if we bind either to avoid double-binding.
2608 vma
->flags
|= I915_VMA_GLOBAL_BIND
| I915_VMA_LOCAL_BIND
;
2613 static void ggtt_unbind_vma(struct i915_vma
*vma
)
2615 struct drm_i915_private
*i915
= vma
->vm
->i915
;
2617 intel_runtime_pm_get(i915
);
2618 vma
->vm
->clear_range(vma
->vm
, vma
->node
.start
, vma
->size
);
2619 intel_runtime_pm_put(i915
);
2622 static int aliasing_gtt_bind_vma(struct i915_vma
*vma
,
2623 enum i915_cache_level cache_level
,
2626 struct drm_i915_private
*i915
= vma
->vm
->i915
;
2630 /* Currently applicable only to VLV */
2632 if (vma
->obj
->gt_ro
)
2633 pte_flags
|= PTE_READ_ONLY
;
2635 if (flags
& I915_VMA_LOCAL_BIND
) {
2636 struct i915_hw_ppgtt
*appgtt
= i915
->mm
.aliasing_ppgtt
;
2638 if (!(vma
->flags
& I915_VMA_LOCAL_BIND
) &&
2639 appgtt
->base
.allocate_va_range
) {
2640 ret
= appgtt
->base
.allocate_va_range(&appgtt
->base
,
2647 appgtt
->base
.insert_entries(&appgtt
->base
, vma
, cache_level
,
2651 if (flags
& I915_VMA_GLOBAL_BIND
) {
2652 intel_runtime_pm_get(i915
);
2653 vma
->vm
->insert_entries(vma
->vm
, vma
, cache_level
, pte_flags
);
2654 intel_runtime_pm_put(i915
);
2660 static void aliasing_gtt_unbind_vma(struct i915_vma
*vma
)
2662 struct drm_i915_private
*i915
= vma
->vm
->i915
;
2664 if (vma
->flags
& I915_VMA_GLOBAL_BIND
) {
2665 intel_runtime_pm_get(i915
);
2666 vma
->vm
->clear_range(vma
->vm
, vma
->node
.start
, vma
->size
);
2667 intel_runtime_pm_put(i915
);
2670 if (vma
->flags
& I915_VMA_LOCAL_BIND
) {
2671 struct i915_address_space
*vm
= &i915
->mm
.aliasing_ppgtt
->base
;
2673 vm
->clear_range(vm
, vma
->node
.start
, vma
->size
);
2677 void i915_gem_gtt_finish_pages(struct drm_i915_gem_object
*obj
,
2678 struct sg_table
*pages
)
2680 struct drm_i915_private
*dev_priv
= to_i915(obj
->base
.dev
);
2681 struct device
*kdev
= &dev_priv
->drm
.pdev
->dev
;
2682 struct i915_ggtt
*ggtt
= &dev_priv
->ggtt
;
2684 if (unlikely(ggtt
->do_idle_maps
)) {
2685 if (i915_gem_wait_for_idle(dev_priv
, 0)) {
2686 DRM_ERROR("Failed to wait for idle; VT'd may hang.\n");
2687 /* Wait a bit, in hopes it avoids the hang */
2692 dma_unmap_sg(kdev
, pages
->sgl
, pages
->nents
, PCI_DMA_BIDIRECTIONAL
);
2695 static int ggtt_set_pages(struct i915_vma
*vma
)
2699 GEM_BUG_ON(vma
->pages
);
2701 ret
= i915_get_ggtt_vma_pages(vma
);
2705 vma
->page_sizes
= vma
->obj
->mm
.page_sizes
;
2710 static void i915_gtt_color_adjust(const struct drm_mm_node
*node
,
2711 unsigned long color
,
2715 if (node
->allocated
&& node
->color
!= color
)
2716 *start
+= I915_GTT_PAGE_SIZE
;
2718 /* Also leave a space between the unallocated reserved node after the
2719 * GTT and any objects within the GTT, i.e. we use the color adjustment
2720 * to insert a guard page to prevent prefetches crossing over the
2723 node
= list_next_entry(node
, node_list
);
2724 if (node
->color
!= color
)
2725 *end
-= I915_GTT_PAGE_SIZE
;
2728 int i915_gem_init_aliasing_ppgtt(struct drm_i915_private
*i915
)
2730 struct i915_ggtt
*ggtt
= &i915
->ggtt
;
2731 struct i915_hw_ppgtt
*ppgtt
;
2734 ppgtt
= i915_ppgtt_create(i915
, ERR_PTR(-EPERM
), "[alias]");
2736 return PTR_ERR(ppgtt
);
2738 if (WARN_ON(ppgtt
->base
.total
< ggtt
->base
.total
)) {
2743 if (ppgtt
->base
.allocate_va_range
) {
2744 /* Note we only pre-allocate as far as the end of the global
2745 * GTT. On 48b / 4-level page-tables, the difference is very,
2746 * very significant! We have to preallocate as GVT/vgpu does
2747 * not like the page directory disappearing.
2749 err
= ppgtt
->base
.allocate_va_range(&ppgtt
->base
,
2750 0, ggtt
->base
.total
);
2755 i915
->mm
.aliasing_ppgtt
= ppgtt
;
2757 WARN_ON(ggtt
->base
.bind_vma
!= ggtt_bind_vma
);
2758 ggtt
->base
.bind_vma
= aliasing_gtt_bind_vma
;
2760 WARN_ON(ggtt
->base
.unbind_vma
!= ggtt_unbind_vma
);
2761 ggtt
->base
.unbind_vma
= aliasing_gtt_unbind_vma
;
2766 i915_ppgtt_put(ppgtt
);
2770 void i915_gem_fini_aliasing_ppgtt(struct drm_i915_private
*i915
)
2772 struct i915_ggtt
*ggtt
= &i915
->ggtt
;
2773 struct i915_hw_ppgtt
*ppgtt
;
2775 ppgtt
= fetch_and_zero(&i915
->mm
.aliasing_ppgtt
);
2779 i915_ppgtt_put(ppgtt
);
2781 ggtt
->base
.bind_vma
= ggtt_bind_vma
;
2782 ggtt
->base
.unbind_vma
= ggtt_unbind_vma
;
2785 int i915_gem_init_ggtt(struct drm_i915_private
*dev_priv
)
2787 /* Let GEM Manage all of the aperture.
2789 * However, leave one page at the end still bound to the scratch page.
2790 * There are a number of places where the hardware apparently prefetches
2791 * past the end of the object, and we've seen multiple hangs with the
2792 * GPU head pointer stuck in a batchbuffer bound at the last page of the
2793 * aperture. One page should be enough to keep any prefetching inside
2796 struct i915_ggtt
*ggtt
= &dev_priv
->ggtt
;
2797 unsigned long hole_start
, hole_end
;
2798 struct drm_mm_node
*entry
;
2801 ret
= intel_vgt_balloon(dev_priv
);
2805 /* Reserve a mappable slot for our lockless error capture */
2806 ret
= drm_mm_insert_node_in_range(&ggtt
->base
.mm
, &ggtt
->error_capture
,
2807 PAGE_SIZE
, 0, I915_COLOR_UNEVICTABLE
,
2808 0, ggtt
->mappable_end
,
2813 /* Clear any non-preallocated blocks */
2814 drm_mm_for_each_hole(entry
, &ggtt
->base
.mm
, hole_start
, hole_end
) {
2815 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
2816 hole_start
, hole_end
);
2817 ggtt
->base
.clear_range(&ggtt
->base
, hole_start
,
2818 hole_end
- hole_start
);
2821 /* And finally clear the reserved guard page */
2822 ggtt
->base
.clear_range(&ggtt
->base
,
2823 ggtt
->base
.total
- PAGE_SIZE
, PAGE_SIZE
);
2825 if (USES_PPGTT(dev_priv
) && !USES_FULL_PPGTT(dev_priv
)) {
2826 ret
= i915_gem_init_aliasing_ppgtt(dev_priv
);
2834 drm_mm_remove_node(&ggtt
->error_capture
);
2839 * i915_ggtt_cleanup_hw - Clean up GGTT hardware initialization
2840 * @dev_priv: i915 device
2842 void i915_ggtt_cleanup_hw(struct drm_i915_private
*dev_priv
)
2844 struct i915_ggtt
*ggtt
= &dev_priv
->ggtt
;
2845 struct i915_vma
*vma
, *vn
;
2846 struct pagevec
*pvec
;
2848 ggtt
->base
.closed
= true;
2850 mutex_lock(&dev_priv
->drm
.struct_mutex
);
2851 WARN_ON(!list_empty(&ggtt
->base
.active_list
));
2852 list_for_each_entry_safe(vma
, vn
, &ggtt
->base
.inactive_list
, vm_link
)
2853 WARN_ON(i915_vma_unbind(vma
));
2854 mutex_unlock(&dev_priv
->drm
.struct_mutex
);
2856 i915_gem_cleanup_stolen(&dev_priv
->drm
);
2858 mutex_lock(&dev_priv
->drm
.struct_mutex
);
2859 i915_gem_fini_aliasing_ppgtt(dev_priv
);
2861 if (drm_mm_node_allocated(&ggtt
->error_capture
))
2862 drm_mm_remove_node(&ggtt
->error_capture
);
2864 if (drm_mm_initialized(&ggtt
->base
.mm
)) {
2865 intel_vgt_deballoon(dev_priv
);
2866 i915_address_space_fini(&ggtt
->base
);
2869 ggtt
->base
.cleanup(&ggtt
->base
);
2871 pvec
= &dev_priv
->mm
.wc_stash
;
2873 set_pages_array_wb(pvec
->pages
, pvec
->nr
);
2874 __pagevec_release(pvec
);
2877 mutex_unlock(&dev_priv
->drm
.struct_mutex
);
2879 arch_phys_wc_del(ggtt
->mtrr
);
2880 io_mapping_fini(&ggtt
->mappable
);
2883 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl
)
2885 snb_gmch_ctl
>>= SNB_GMCH_GGMS_SHIFT
;
2886 snb_gmch_ctl
&= SNB_GMCH_GGMS_MASK
;
2887 return snb_gmch_ctl
<< 20;
2890 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl
)
2892 bdw_gmch_ctl
>>= BDW_GMCH_GGMS_SHIFT
;
2893 bdw_gmch_ctl
&= BDW_GMCH_GGMS_MASK
;
2895 bdw_gmch_ctl
= 1 << bdw_gmch_ctl
;
2897 #ifdef CONFIG_X86_32
2898 /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */
2899 if (bdw_gmch_ctl
> 4)
2903 return bdw_gmch_ctl
<< 20;
2906 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl
)
2908 gmch_ctrl
>>= SNB_GMCH_GGMS_SHIFT
;
2909 gmch_ctrl
&= SNB_GMCH_GGMS_MASK
;
2912 return 1 << (20 + gmch_ctrl
);
2917 static size_t gen6_get_stolen_size(u16 snb_gmch_ctl
)
2919 snb_gmch_ctl
>>= SNB_GMCH_GMS_SHIFT
;
2920 snb_gmch_ctl
&= SNB_GMCH_GMS_MASK
;
2921 return (size_t)snb_gmch_ctl
<< 25; /* 32 MB units */
2924 static size_t gen8_get_stolen_size(u16 bdw_gmch_ctl
)
2926 bdw_gmch_ctl
>>= BDW_GMCH_GMS_SHIFT
;
2927 bdw_gmch_ctl
&= BDW_GMCH_GMS_MASK
;
2928 return (size_t)bdw_gmch_ctl
<< 25; /* 32 MB units */
2931 static size_t chv_get_stolen_size(u16 gmch_ctrl
)
2933 gmch_ctrl
>>= SNB_GMCH_GMS_SHIFT
;
2934 gmch_ctrl
&= SNB_GMCH_GMS_MASK
;
2937 * 0x0 to 0x10: 32MB increments starting at 0MB
2938 * 0x11 to 0x16: 4MB increments starting at 8MB
2939 * 0x17 to 0x1d: 4MB increments start at 36MB
2941 if (gmch_ctrl
< 0x11)
2942 return (size_t)gmch_ctrl
<< 25;
2943 else if (gmch_ctrl
< 0x17)
2944 return (size_t)(gmch_ctrl
- 0x11 + 2) << 22;
2946 return (size_t)(gmch_ctrl
- 0x17 + 9) << 22;
2949 static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl
)
2951 gen9_gmch_ctl
>>= BDW_GMCH_GMS_SHIFT
;
2952 gen9_gmch_ctl
&= BDW_GMCH_GMS_MASK
;
2954 if (gen9_gmch_ctl
< 0xf0)
2955 return (size_t)gen9_gmch_ctl
<< 25; /* 32 MB units */
2957 /* 4MB increments starting at 0xf0 for 4MB */
2958 return (size_t)(gen9_gmch_ctl
- 0xf0 + 1) << 22;
2961 static int ggtt_probe_common(struct i915_ggtt
*ggtt
, u64 size
)
2963 struct drm_i915_private
*dev_priv
= ggtt
->base
.i915
;
2964 struct pci_dev
*pdev
= dev_priv
->drm
.pdev
;
2965 phys_addr_t phys_addr
;
2968 /* For Modern GENs the PTEs and register space are split in the BAR */
2969 phys_addr
= pci_resource_start(pdev
, 0) + pci_resource_len(pdev
, 0) / 2;
2972 * On BXT+/CNL+ writes larger than 64 bit to the GTT pagetable range
2973 * will be dropped. For WC mappings in general we have 64 byte burst
2974 * writes when the WC buffer is flushed, so we can't use it, but have to
2975 * resort to an uncached mapping. The WC issue is easily caught by the
2976 * readback check when writing GTT PTE entries.
2978 if (IS_GEN9_LP(dev_priv
) || INTEL_GEN(dev_priv
) >= 10)
2979 ggtt
->gsm
= ioremap_nocache(phys_addr
, size
);
2981 ggtt
->gsm
= ioremap_wc(phys_addr
, size
);
2983 DRM_ERROR("Failed to map the ggtt page table\n");
2987 ret
= setup_scratch_page(&ggtt
->base
, GFP_DMA32
);
2989 DRM_ERROR("Scratch setup failed\n");
2990 /* iounmap will also get called at remove, but meh */
2998 static struct intel_ppat_entry
*
2999 __alloc_ppat_entry(struct intel_ppat
*ppat
, unsigned int index
, u8 value
)
3001 struct intel_ppat_entry
*entry
= &ppat
->entries
[index
];
3003 GEM_BUG_ON(index
>= ppat
->max_entries
);
3004 GEM_BUG_ON(test_bit(index
, ppat
->used
));
3007 entry
->value
= value
;
3008 kref_init(&entry
->ref
);
3009 set_bit(index
, ppat
->used
);
3010 set_bit(index
, ppat
->dirty
);
3015 static void __free_ppat_entry(struct intel_ppat_entry
*entry
)
3017 struct intel_ppat
*ppat
= entry
->ppat
;
3018 unsigned int index
= entry
- ppat
->entries
;
3020 GEM_BUG_ON(index
>= ppat
->max_entries
);
3021 GEM_BUG_ON(!test_bit(index
, ppat
->used
));
3023 entry
->value
= ppat
->clear_value
;
3024 clear_bit(index
, ppat
->used
);
3025 set_bit(index
, ppat
->dirty
);
3029 * intel_ppat_get - get a usable PPAT entry
3030 * @i915: i915 device instance
3031 * @value: the PPAT value required by the caller
3033 * The function tries to search if there is an existing PPAT entry which
3034 * matches with the required value. If perfectly matched, the existing PPAT
3035 * entry will be used. If only partially matched, it will try to check if
3036 * there is any available PPAT index. If yes, it will allocate a new PPAT
3037 * index for the required entry and update the HW. If not, the partially
3038 * matched entry will be used.
3040 const struct intel_ppat_entry
*
3041 intel_ppat_get(struct drm_i915_private
*i915
, u8 value
)
3043 struct intel_ppat
*ppat
= &i915
->ppat
;
3044 struct intel_ppat_entry
*entry
;
3045 unsigned int scanned
, best_score
;
3048 GEM_BUG_ON(!ppat
->max_entries
);
3050 scanned
= best_score
= 0;
3051 for_each_set_bit(i
, ppat
->used
, ppat
->max_entries
) {
3054 score
= ppat
->match(ppat
->entries
[i
].value
, value
);
3055 if (score
> best_score
) {
3056 entry
= &ppat
->entries
[i
];
3057 if (score
== INTEL_PPAT_PERFECT_MATCH
) {
3058 kref_get(&entry
->ref
);
3066 if (scanned
== ppat
->max_entries
) {
3068 return ERR_PTR(-ENOSPC
);
3070 kref_get(&entry
->ref
);
3074 i
= find_first_zero_bit(ppat
->used
, ppat
->max_entries
);
3075 entry
= __alloc_ppat_entry(ppat
, i
, value
);
3076 ppat
->update_hw(i915
);
3080 static void release_ppat(struct kref
*kref
)
3082 struct intel_ppat_entry
*entry
=
3083 container_of(kref
, struct intel_ppat_entry
, ref
);
3084 struct drm_i915_private
*i915
= entry
->ppat
->i915
;
3086 __free_ppat_entry(entry
);
3087 entry
->ppat
->update_hw(i915
);
3091 * intel_ppat_put - put back the PPAT entry got from intel_ppat_get()
3092 * @entry: an intel PPAT entry
3094 * Put back the PPAT entry got from intel_ppat_get(). If the PPAT index of the
3095 * entry is dynamically allocated, its reference count will be decreased. Once
3096 * the reference count becomes into zero, the PPAT index becomes free again.
3098 void intel_ppat_put(const struct intel_ppat_entry
*entry
)
3100 struct intel_ppat
*ppat
= entry
->ppat
;
3101 unsigned int index
= entry
- ppat
->entries
;
3103 GEM_BUG_ON(!ppat
->max_entries
);
3105 kref_put(&ppat
->entries
[index
].ref
, release_ppat
);
3108 static void cnl_private_pat_update_hw(struct drm_i915_private
*dev_priv
)
3110 struct intel_ppat
*ppat
= &dev_priv
->ppat
;
3113 for_each_set_bit(i
, ppat
->dirty
, ppat
->max_entries
) {
3114 I915_WRITE(GEN10_PAT_INDEX(i
), ppat
->entries
[i
].value
);
3115 clear_bit(i
, ppat
->dirty
);
3119 static void bdw_private_pat_update_hw(struct drm_i915_private
*dev_priv
)
3121 struct intel_ppat
*ppat
= &dev_priv
->ppat
;
3125 for (i
= 0; i
< ppat
->max_entries
; i
++)
3126 pat
|= GEN8_PPAT(i
, ppat
->entries
[i
].value
);
3128 bitmap_clear(ppat
->dirty
, 0, ppat
->max_entries
);
3130 I915_WRITE(GEN8_PRIVATE_PAT_LO
, lower_32_bits(pat
));
3131 I915_WRITE(GEN8_PRIVATE_PAT_HI
, upper_32_bits(pat
));
3134 static unsigned int bdw_private_pat_match(u8 src
, u8 dst
)
3136 unsigned int score
= 0;
3143 /* Cache attribute has to be matched. */
3144 if (GEN8_PPAT_GET_CA(src
) != GEN8_PPAT_GET_CA(dst
))
3149 if (GEN8_PPAT_GET_TC(src
) == GEN8_PPAT_GET_TC(dst
))
3152 if (GEN8_PPAT_GET_AGE(src
) == GEN8_PPAT_GET_AGE(dst
))
3155 if (score
== (AGE_MATCH
| TC_MATCH
| CA_MATCH
))
3156 return INTEL_PPAT_PERFECT_MATCH
;
3161 static unsigned int chv_private_pat_match(u8 src
, u8 dst
)
3163 return (CHV_PPAT_GET_SNOOP(src
) == CHV_PPAT_GET_SNOOP(dst
)) ?
3164 INTEL_PPAT_PERFECT_MATCH
: 0;
3167 static void cnl_setup_private_ppat(struct intel_ppat
*ppat
)
3169 ppat
->max_entries
= 8;
3170 ppat
->update_hw
= cnl_private_pat_update_hw
;
3171 ppat
->match
= bdw_private_pat_match
;
3172 ppat
->clear_value
= GEN8_PPAT_WB
| GEN8_PPAT_LLCELLC
| GEN8_PPAT_AGE(3);
3174 /* XXX: spec is unclear if this is still needed for CNL+ */
3175 if (!USES_PPGTT(ppat
->i915
)) {
3176 __alloc_ppat_entry(ppat
, 0, GEN8_PPAT_UC
);
3180 __alloc_ppat_entry(ppat
, 0, GEN8_PPAT_WB
| GEN8_PPAT_LLC
);
3181 __alloc_ppat_entry(ppat
, 1, GEN8_PPAT_WC
| GEN8_PPAT_LLCELLC
);
3182 __alloc_ppat_entry(ppat
, 2, GEN8_PPAT_WT
| GEN8_PPAT_LLCELLC
);
3183 __alloc_ppat_entry(ppat
, 3, GEN8_PPAT_UC
);
3184 __alloc_ppat_entry(ppat
, 4, GEN8_PPAT_WB
| GEN8_PPAT_LLCELLC
| GEN8_PPAT_AGE(0));
3185 __alloc_ppat_entry(ppat
, 5, GEN8_PPAT_WB
| GEN8_PPAT_LLCELLC
| GEN8_PPAT_AGE(1));
3186 __alloc_ppat_entry(ppat
, 6, GEN8_PPAT_WB
| GEN8_PPAT_LLCELLC
| GEN8_PPAT_AGE(2));
3187 __alloc_ppat_entry(ppat
, 7, GEN8_PPAT_WB
| GEN8_PPAT_LLCELLC
| GEN8_PPAT_AGE(3));
3190 /* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
3191 * bits. When using advanced contexts each context stores its own PAT, but
3192 * writing this data shouldn't be harmful even in those cases. */
3193 static void bdw_setup_private_ppat(struct intel_ppat
*ppat
)
3195 ppat
->max_entries
= 8;
3196 ppat
->update_hw
= bdw_private_pat_update_hw
;
3197 ppat
->match
= bdw_private_pat_match
;
3198 ppat
->clear_value
= GEN8_PPAT_WB
| GEN8_PPAT_LLCELLC
| GEN8_PPAT_AGE(3);
3200 if (!USES_PPGTT(ppat
->i915
)) {
3201 /* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry,
3202 * so RTL will always use the value corresponding to
3204 * So let's disable cache for GGTT to avoid screen corruptions.
3205 * MOCS still can be used though.
3206 * - System agent ggtt writes (i.e. cpu gtt mmaps) already work
3207 * before this patch, i.e. the same uncached + snooping access
3208 * like on gen6/7 seems to be in effect.
3209 * - So this just fixes blitter/render access. Again it looks
3210 * like it's not just uncached access, but uncached + snooping.
3211 * So we can still hold onto all our assumptions wrt cpu
3212 * clflushing on LLC machines.
3214 __alloc_ppat_entry(ppat
, 0, GEN8_PPAT_UC
);
3218 __alloc_ppat_entry(ppat
, 0, GEN8_PPAT_WB
| GEN8_PPAT_LLC
); /* for normal objects, no eLLC */
3219 __alloc_ppat_entry(ppat
, 1, GEN8_PPAT_WC
| GEN8_PPAT_LLCELLC
); /* for something pointing to ptes? */
3220 __alloc_ppat_entry(ppat
, 2, GEN8_PPAT_WT
| GEN8_PPAT_LLCELLC
); /* for scanout with eLLC */
3221 __alloc_ppat_entry(ppat
, 3, GEN8_PPAT_UC
); /* Uncached objects, mostly for scanout */
3222 __alloc_ppat_entry(ppat
, 4, GEN8_PPAT_WB
| GEN8_PPAT_LLCELLC
| GEN8_PPAT_AGE(0));
3223 __alloc_ppat_entry(ppat
, 5, GEN8_PPAT_WB
| GEN8_PPAT_LLCELLC
| GEN8_PPAT_AGE(1));
3224 __alloc_ppat_entry(ppat
, 6, GEN8_PPAT_WB
| GEN8_PPAT_LLCELLC
| GEN8_PPAT_AGE(2));
3225 __alloc_ppat_entry(ppat
, 7, GEN8_PPAT_WB
| GEN8_PPAT_LLCELLC
| GEN8_PPAT_AGE(3));
3228 static void chv_setup_private_ppat(struct intel_ppat
*ppat
)
3230 ppat
->max_entries
= 8;
3231 ppat
->update_hw
= bdw_private_pat_update_hw
;
3232 ppat
->match
= chv_private_pat_match
;
3233 ppat
->clear_value
= CHV_PPAT_SNOOP
;
3236 * Map WB on BDW to snooped on CHV.
3238 * Only the snoop bit has meaning for CHV, the rest is
3241 * The hardware will never snoop for certain types of accesses:
3242 * - CPU GTT (GMADR->GGTT->no snoop->memory)
3243 * - PPGTT page tables
3244 * - some other special cycles
3246 * As with BDW, we also need to consider the following for GT accesses:
3247 * "For GGTT, there is NO pat_sel[2:0] from the entry,
3248 * so RTL will always use the value corresponding to
3250 * Which means we must set the snoop bit in PAT entry 0
3251 * in order to keep the global status page working.
3254 __alloc_ppat_entry(ppat
, 0, CHV_PPAT_SNOOP
);
3255 __alloc_ppat_entry(ppat
, 1, 0);
3256 __alloc_ppat_entry(ppat
, 2, 0);
3257 __alloc_ppat_entry(ppat
, 3, 0);
3258 __alloc_ppat_entry(ppat
, 4, CHV_PPAT_SNOOP
);
3259 __alloc_ppat_entry(ppat
, 5, CHV_PPAT_SNOOP
);
3260 __alloc_ppat_entry(ppat
, 6, CHV_PPAT_SNOOP
);
3261 __alloc_ppat_entry(ppat
, 7, CHV_PPAT_SNOOP
);
3264 static void gen6_gmch_remove(struct i915_address_space
*vm
)
3266 struct i915_ggtt
*ggtt
= i915_vm_to_ggtt(vm
);
3269 cleanup_scratch_page(vm
);
3272 static void setup_private_pat(struct drm_i915_private
*dev_priv
)
3274 struct intel_ppat
*ppat
= &dev_priv
->ppat
;
3277 ppat
->i915
= dev_priv
;
3279 if (INTEL_GEN(dev_priv
) >= 10)
3280 cnl_setup_private_ppat(ppat
);
3281 else if (IS_CHERRYVIEW(dev_priv
) || IS_GEN9_LP(dev_priv
))
3282 chv_setup_private_ppat(ppat
);
3284 bdw_setup_private_ppat(ppat
);
3286 GEM_BUG_ON(ppat
->max_entries
> INTEL_MAX_PPAT_ENTRIES
);
3288 for_each_clear_bit(i
, ppat
->used
, ppat
->max_entries
) {
3289 ppat
->entries
[i
].value
= ppat
->clear_value
;
3290 ppat
->entries
[i
].ppat
= ppat
;
3291 set_bit(i
, ppat
->dirty
);
3294 ppat
->update_hw(dev_priv
);
3297 static int gen8_gmch_probe(struct i915_ggtt
*ggtt
)
3299 struct drm_i915_private
*dev_priv
= ggtt
->base
.i915
;
3300 struct pci_dev
*pdev
= dev_priv
->drm
.pdev
;
3305 /* TODO: We're not aware of mappable constraints on gen8 yet */
3306 ggtt
->mappable_base
= pci_resource_start(pdev
, 2);
3307 ggtt
->mappable_end
= pci_resource_len(pdev
, 2);
3309 err
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(39));
3311 err
= pci_set_consistent_dma_mask(pdev
, DMA_BIT_MASK(39));
3313 DRM_ERROR("Can't set DMA mask/consistent mask (%d)\n", err
);
3315 pci_read_config_word(pdev
, SNB_GMCH_CTRL
, &snb_gmch_ctl
);
3317 if (INTEL_GEN(dev_priv
) >= 9) {
3318 ggtt
->stolen_size
= gen9_get_stolen_size(snb_gmch_ctl
);
3319 size
= gen8_get_total_gtt_size(snb_gmch_ctl
);
3320 } else if (IS_CHERRYVIEW(dev_priv
)) {
3321 ggtt
->stolen_size
= chv_get_stolen_size(snb_gmch_ctl
);
3322 size
= chv_get_total_gtt_size(snb_gmch_ctl
);
3324 ggtt
->stolen_size
= gen8_get_stolen_size(snb_gmch_ctl
);
3325 size
= gen8_get_total_gtt_size(snb_gmch_ctl
);
3328 ggtt
->base
.total
= (size
/ sizeof(gen8_pte_t
)) << PAGE_SHIFT
;
3329 ggtt
->base
.cleanup
= gen6_gmch_remove
;
3330 ggtt
->base
.bind_vma
= ggtt_bind_vma
;
3331 ggtt
->base
.unbind_vma
= ggtt_unbind_vma
;
3332 ggtt
->base
.set_pages
= ggtt_set_pages
;
3333 ggtt
->base
.clear_pages
= clear_pages
;
3334 ggtt
->base
.insert_page
= gen8_ggtt_insert_page
;
3335 ggtt
->base
.clear_range
= nop_clear_range
;
3336 if (!USES_FULL_PPGTT(dev_priv
) || intel_scanout_needs_vtd_wa(dev_priv
))
3337 ggtt
->base
.clear_range
= gen8_ggtt_clear_range
;
3339 ggtt
->base
.insert_entries
= gen8_ggtt_insert_entries
;
3341 /* Serialize GTT updates with aperture access on BXT if VT-d is on. */
3342 if (intel_ggtt_update_needs_vtd_wa(dev_priv
)) {
3343 ggtt
->base
.insert_entries
= bxt_vtd_ggtt_insert_entries__BKL
;
3344 ggtt
->base
.insert_page
= bxt_vtd_ggtt_insert_page__BKL
;
3345 if (ggtt
->base
.clear_range
!= nop_clear_range
)
3346 ggtt
->base
.clear_range
= bxt_vtd_ggtt_clear_range__BKL
;
3349 ggtt
->invalidate
= gen6_ggtt_invalidate
;
3351 setup_private_pat(dev_priv
);
3353 return ggtt_probe_common(ggtt
, size
);
3356 static int gen6_gmch_probe(struct i915_ggtt
*ggtt
)
3358 struct drm_i915_private
*dev_priv
= ggtt
->base
.i915
;
3359 struct pci_dev
*pdev
= dev_priv
->drm
.pdev
;
3364 ggtt
->mappable_base
= pci_resource_start(pdev
, 2);
3365 ggtt
->mappable_end
= pci_resource_len(pdev
, 2);
3367 /* 64/512MB is the current min/max we actually know of, but this is just
3368 * a coarse sanity check.
3370 if (ggtt
->mappable_end
< (64<<20) || ggtt
->mappable_end
> (512<<20)) {
3371 DRM_ERROR("Unknown GMADR size (%llx)\n", ggtt
->mappable_end
);
3375 err
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(40));
3377 err
= pci_set_consistent_dma_mask(pdev
, DMA_BIT_MASK(40));
3379 DRM_ERROR("Can't set DMA mask/consistent mask (%d)\n", err
);
3380 pci_read_config_word(pdev
, SNB_GMCH_CTRL
, &snb_gmch_ctl
);
3382 ggtt
->stolen_size
= gen6_get_stolen_size(snb_gmch_ctl
);
3384 size
= gen6_get_total_gtt_size(snb_gmch_ctl
);
3385 ggtt
->base
.total
= (size
/ sizeof(gen6_pte_t
)) << PAGE_SHIFT
;
3387 ggtt
->base
.clear_range
= gen6_ggtt_clear_range
;
3388 ggtt
->base
.insert_page
= gen6_ggtt_insert_page
;
3389 ggtt
->base
.insert_entries
= gen6_ggtt_insert_entries
;
3390 ggtt
->base
.bind_vma
= ggtt_bind_vma
;
3391 ggtt
->base
.unbind_vma
= ggtt_unbind_vma
;
3392 ggtt
->base
.set_pages
= ggtt_set_pages
;
3393 ggtt
->base
.clear_pages
= clear_pages
;
3394 ggtt
->base
.cleanup
= gen6_gmch_remove
;
3396 ggtt
->invalidate
= gen6_ggtt_invalidate
;
3398 if (HAS_EDRAM(dev_priv
))
3399 ggtt
->base
.pte_encode
= iris_pte_encode
;
3400 else if (IS_HASWELL(dev_priv
))
3401 ggtt
->base
.pte_encode
= hsw_pte_encode
;
3402 else if (IS_VALLEYVIEW(dev_priv
))
3403 ggtt
->base
.pte_encode
= byt_pte_encode
;
3404 else if (INTEL_GEN(dev_priv
) >= 7)
3405 ggtt
->base
.pte_encode
= ivb_pte_encode
;
3407 ggtt
->base
.pte_encode
= snb_pte_encode
;
3409 return ggtt_probe_common(ggtt
, size
);
3412 static void i915_gmch_remove(struct i915_address_space
*vm
)
3414 intel_gmch_remove();
3417 static int i915_gmch_probe(struct i915_ggtt
*ggtt
)
3419 struct drm_i915_private
*dev_priv
= ggtt
->base
.i915
;
3422 ret
= intel_gmch_probe(dev_priv
->bridge_dev
, dev_priv
->drm
.pdev
, NULL
);
3424 DRM_ERROR("failed to set up gmch\n");
3428 intel_gtt_get(&ggtt
->base
.total
,
3430 &ggtt
->mappable_base
,
3431 &ggtt
->mappable_end
);
3433 ggtt
->do_idle_maps
= needs_idle_maps(dev_priv
);
3434 ggtt
->base
.insert_page
= i915_ggtt_insert_page
;
3435 ggtt
->base
.insert_entries
= i915_ggtt_insert_entries
;
3436 ggtt
->base
.clear_range
= i915_ggtt_clear_range
;
3437 ggtt
->base
.bind_vma
= ggtt_bind_vma
;
3438 ggtt
->base
.unbind_vma
= ggtt_unbind_vma
;
3439 ggtt
->base
.set_pages
= ggtt_set_pages
;
3440 ggtt
->base
.clear_pages
= clear_pages
;
3441 ggtt
->base
.cleanup
= i915_gmch_remove
;
3443 ggtt
->invalidate
= gmch_ggtt_invalidate
;
3445 if (unlikely(ggtt
->do_idle_maps
))
3446 DRM_INFO("applying Ironlake quirks for intel_iommu\n");
3452 * i915_ggtt_probe_hw - Probe GGTT hardware location
3453 * @dev_priv: i915 device
3455 int i915_ggtt_probe_hw(struct drm_i915_private
*dev_priv
)
3457 struct i915_ggtt
*ggtt
= &dev_priv
->ggtt
;
3460 ggtt
->base
.i915
= dev_priv
;
3461 ggtt
->base
.dma
= &dev_priv
->drm
.pdev
->dev
;
3463 if (INTEL_GEN(dev_priv
) <= 5)
3464 ret
= i915_gmch_probe(ggtt
);
3465 else if (INTEL_GEN(dev_priv
) < 8)
3466 ret
= gen6_gmch_probe(ggtt
);
3468 ret
= gen8_gmch_probe(ggtt
);
3472 /* Trim the GGTT to fit the GuC mappable upper range (when enabled).
3473 * This is easier than doing range restriction on the fly, as we
3474 * currently don't have any bits spare to pass in this upper
3477 if (HAS_GUC(dev_priv
) && i915_modparams
.enable_guc_loading
) {
3478 ggtt
->base
.total
= min_t(u64
, ggtt
->base
.total
, GUC_GGTT_TOP
);
3479 ggtt
->mappable_end
= min(ggtt
->mappable_end
, ggtt
->base
.total
);
3482 if ((ggtt
->base
.total
- 1) >> 32) {
3483 DRM_ERROR("We never expected a Global GTT with more than 32bits"
3484 " of address space! Found %lldM!\n",
3485 ggtt
->base
.total
>> 20);
3486 ggtt
->base
.total
= 1ULL << 32;
3487 ggtt
->mappable_end
= min(ggtt
->mappable_end
, ggtt
->base
.total
);
3490 if (ggtt
->mappable_end
> ggtt
->base
.total
) {
3491 DRM_ERROR("mappable aperture extends past end of GGTT,"
3492 " aperture=%llx, total=%llx\n",
3493 ggtt
->mappable_end
, ggtt
->base
.total
);
3494 ggtt
->mappable_end
= ggtt
->base
.total
;
3497 /* GMADR is the PCI mmio aperture into the global GTT. */
3498 DRM_INFO("Memory usable by graphics device = %lluM\n",
3499 ggtt
->base
.total
>> 20);
3500 DRM_DEBUG_DRIVER("GMADR size = %lldM\n", ggtt
->mappable_end
>> 20);
3501 DRM_DEBUG_DRIVER("GTT stolen size = %uM\n", ggtt
->stolen_size
>> 20);
3502 if (intel_vtd_active())
3503 DRM_INFO("VT-d active for gfx access\n");
3509 * i915_ggtt_init_hw - Initialize GGTT hardware
3510 * @dev_priv: i915 device
3512 int i915_ggtt_init_hw(struct drm_i915_private
*dev_priv
)
3514 struct i915_ggtt
*ggtt
= &dev_priv
->ggtt
;
3517 INIT_LIST_HEAD(&dev_priv
->vm_list
);
3519 /* Note that we use page colouring to enforce a guard page at the
3520 * end of the address space. This is required as the CS may prefetch
3521 * beyond the end of the batch buffer, across the page boundary,
3522 * and beyond the end of the GTT if we do not provide a guard.
3524 mutex_lock(&dev_priv
->drm
.struct_mutex
);
3525 i915_address_space_init(&ggtt
->base
, dev_priv
, "[global]");
3526 if (!HAS_LLC(dev_priv
) && !USES_PPGTT(dev_priv
))
3527 ggtt
->base
.mm
.color_adjust
= i915_gtt_color_adjust
;
3528 mutex_unlock(&dev_priv
->drm
.struct_mutex
);
3530 if (!io_mapping_init_wc(&dev_priv
->ggtt
.mappable
,
3531 dev_priv
->ggtt
.mappable_base
,
3532 dev_priv
->ggtt
.mappable_end
)) {
3534 goto out_gtt_cleanup
;
3537 ggtt
->mtrr
= arch_phys_wc_add(ggtt
->mappable_base
, ggtt
->mappable_end
);
3540 * Initialise stolen early so that we may reserve preallocated
3541 * objects for the BIOS to KMS transition.
3543 ret
= i915_gem_init_stolen(dev_priv
);
3545 goto out_gtt_cleanup
;
3550 ggtt
->base
.cleanup(&ggtt
->base
);
3554 int i915_ggtt_enable_hw(struct drm_i915_private
*dev_priv
)
3556 if (INTEL_GEN(dev_priv
) < 6 && !intel_enable_gtt())
3562 void i915_ggtt_enable_guc(struct drm_i915_private
*i915
)
3564 GEM_BUG_ON(i915
->ggtt
.invalidate
!= gen6_ggtt_invalidate
);
3566 i915
->ggtt
.invalidate
= guc_ggtt_invalidate
;
3569 void i915_ggtt_disable_guc(struct drm_i915_private
*i915
)
3571 /* We should only be called after i915_ggtt_enable_guc() */
3572 GEM_BUG_ON(i915
->ggtt
.invalidate
!= guc_ggtt_invalidate
);
3574 i915
->ggtt
.invalidate
= gen6_ggtt_invalidate
;
3577 void i915_gem_restore_gtt_mappings(struct drm_i915_private
*dev_priv
)
3579 struct i915_ggtt
*ggtt
= &dev_priv
->ggtt
;
3580 struct drm_i915_gem_object
*obj
, *on
;
3582 i915_check_and_clear_faults(dev_priv
);
3584 /* First fill our portion of the GTT with scratch pages */
3585 ggtt
->base
.clear_range(&ggtt
->base
, 0, ggtt
->base
.total
);
3587 ggtt
->base
.closed
= true; /* skip rewriting PTE on VMA unbind */
3589 /* clflush objects bound into the GGTT and rebind them. */
3590 list_for_each_entry_safe(obj
, on
, &dev_priv
->mm
.bound_list
, mm
.link
) {
3591 bool ggtt_bound
= false;
3592 struct i915_vma
*vma
;
3594 list_for_each_entry(vma
, &obj
->vma_list
, obj_link
) {
3595 if (vma
->vm
!= &ggtt
->base
)
3598 if (!i915_vma_unbind(vma
))
3601 WARN_ON(i915_vma_bind(vma
, obj
->cache_level
,
3607 WARN_ON(i915_gem_object_set_to_gtt_domain(obj
, false));
3610 ggtt
->base
.closed
= false;
3612 if (INTEL_GEN(dev_priv
) >= 8) {
3613 struct intel_ppat
*ppat
= &dev_priv
->ppat
;
3615 bitmap_set(ppat
->dirty
, 0, ppat
->max_entries
);
3616 dev_priv
->ppat
.update_hw(dev_priv
);
3620 if (USES_PPGTT(dev_priv
)) {
3621 struct i915_address_space
*vm
;
3623 list_for_each_entry(vm
, &dev_priv
->vm_list
, global_link
) {
3624 struct i915_hw_ppgtt
*ppgtt
;
3626 if (i915_is_ggtt(vm
))
3627 ppgtt
= dev_priv
->mm
.aliasing_ppgtt
;
3629 ppgtt
= i915_vm_to_ppgtt(vm
);
3631 gen6_write_page_range(ppgtt
, 0, ppgtt
->base
.total
);
3635 i915_ggtt_invalidate(dev_priv
);
3638 static struct scatterlist
*
3639 rotate_pages(const dma_addr_t
*in
, unsigned int offset
,
3640 unsigned int width
, unsigned int height
,
3641 unsigned int stride
,
3642 struct sg_table
*st
, struct scatterlist
*sg
)
3644 unsigned int column
, row
;
3645 unsigned int src_idx
;
3647 for (column
= 0; column
< width
; column
++) {
3648 src_idx
= stride
* (height
- 1) + column
;
3649 for (row
= 0; row
< height
; row
++) {
3651 /* We don't need the pages, but need to initialize
3652 * the entries so the sg list can be happily traversed.
3653 * The only thing we need are DMA addresses.
3655 sg_set_page(sg
, NULL
, PAGE_SIZE
, 0);
3656 sg_dma_address(sg
) = in
[offset
+ src_idx
];
3657 sg_dma_len(sg
) = PAGE_SIZE
;
3666 static noinline
struct sg_table
*
3667 intel_rotate_pages(struct intel_rotation_info
*rot_info
,
3668 struct drm_i915_gem_object
*obj
)
3670 const unsigned long n_pages
= obj
->base
.size
/ PAGE_SIZE
;
3671 unsigned int size
= intel_rotation_info_size(rot_info
);
3672 struct sgt_iter sgt_iter
;
3673 dma_addr_t dma_addr
;
3675 dma_addr_t
*page_addr_list
;
3676 struct sg_table
*st
;
3677 struct scatterlist
*sg
;
3680 /* Allocate a temporary list of source pages for random access. */
3681 page_addr_list
= kvmalloc_array(n_pages
,
3684 if (!page_addr_list
)
3685 return ERR_PTR(ret
);
3687 /* Allocate target SG list. */
3688 st
= kmalloc(sizeof(*st
), GFP_KERNEL
);
3692 ret
= sg_alloc_table(st
, size
, GFP_KERNEL
);
3696 /* Populate source page list from the object. */
3698 for_each_sgt_dma(dma_addr
, sgt_iter
, obj
->mm
.pages
)
3699 page_addr_list
[i
++] = dma_addr
;
3701 GEM_BUG_ON(i
!= n_pages
);
3705 for (i
= 0 ; i
< ARRAY_SIZE(rot_info
->plane
); i
++) {
3706 sg
= rotate_pages(page_addr_list
, rot_info
->plane
[i
].offset
,
3707 rot_info
->plane
[i
].width
, rot_info
->plane
[i
].height
,
3708 rot_info
->plane
[i
].stride
, st
, sg
);
3711 DRM_DEBUG_KMS("Created rotated page mapping for object size %zu (%ux%u tiles, %u pages)\n",
3712 obj
->base
.size
, rot_info
->plane
[0].width
, rot_info
->plane
[0].height
, size
);
3714 kvfree(page_addr_list
);
3721 kvfree(page_addr_list
);
3723 DRM_DEBUG_KMS("Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n",
3724 obj
->base
.size
, rot_info
->plane
[0].width
, rot_info
->plane
[0].height
, size
);
3726 return ERR_PTR(ret
);
3729 static noinline
struct sg_table
*
3730 intel_partial_pages(const struct i915_ggtt_view
*view
,
3731 struct drm_i915_gem_object
*obj
)
3733 struct sg_table
*st
;
3734 struct scatterlist
*sg
, *iter
;
3735 unsigned int count
= view
->partial
.size
;
3736 unsigned int offset
;
3739 st
= kmalloc(sizeof(*st
), GFP_KERNEL
);
3743 ret
= sg_alloc_table(st
, count
, GFP_KERNEL
);
3747 iter
= i915_gem_object_get_sg(obj
, view
->partial
.offset
, &offset
);
3755 len
= min(iter
->length
- (offset
<< PAGE_SHIFT
),
3756 count
<< PAGE_SHIFT
);
3757 sg_set_page(sg
, NULL
, len
, 0);
3758 sg_dma_address(sg
) =
3759 sg_dma_address(iter
) + (offset
<< PAGE_SHIFT
);
3760 sg_dma_len(sg
) = len
;
3763 count
-= len
>> PAGE_SHIFT
;
3770 iter
= __sg_next(iter
);
3777 return ERR_PTR(ret
);
3781 i915_get_ggtt_vma_pages(struct i915_vma
*vma
)
3785 /* The vma->pages are only valid within the lifespan of the borrowed
3786 * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so
3787 * must be the vma->pages. A simple rule is that vma->pages must only
3788 * be accessed when the obj->mm.pages are pinned.
3790 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma
->obj
));
3792 switch (vma
->ggtt_view
.type
) {
3793 case I915_GGTT_VIEW_NORMAL
:
3794 vma
->pages
= vma
->obj
->mm
.pages
;
3797 case I915_GGTT_VIEW_ROTATED
:
3799 intel_rotate_pages(&vma
->ggtt_view
.rotated
, vma
->obj
);
3802 case I915_GGTT_VIEW_PARTIAL
:
3803 vma
->pages
= intel_partial_pages(&vma
->ggtt_view
, vma
->obj
);
3807 WARN_ONCE(1, "GGTT view %u not implemented!\n",
3808 vma
->ggtt_view
.type
);
3813 if (unlikely(IS_ERR(vma
->pages
))) {
3814 ret
= PTR_ERR(vma
->pages
);
3816 DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n",
3817 vma
->ggtt_view
.type
, ret
);
3823 * i915_gem_gtt_reserve - reserve a node in an address_space (GTT)
3824 * @vm: the &struct i915_address_space
3825 * @node: the &struct drm_mm_node (typically i915_vma.mode)
3826 * @size: how much space to allocate inside the GTT,
3827 * must be #I915_GTT_PAGE_SIZE aligned
3828 * @offset: where to insert inside the GTT,
3829 * must be #I915_GTT_MIN_ALIGNMENT aligned, and the node
3830 * (@offset + @size) must fit within the address space
3831 * @color: color to apply to node, if this node is not from a VMA,
3832 * color must be #I915_COLOR_UNEVICTABLE
3833 * @flags: control search and eviction behaviour
3835 * i915_gem_gtt_reserve() tries to insert the @node at the exact @offset inside
3836 * the address space (using @size and @color). If the @node does not fit, it
3837 * tries to evict any overlapping nodes from the GTT, including any
3838 * neighbouring nodes if the colors do not match (to ensure guard pages between
3839 * differing domains). See i915_gem_evict_for_node() for the gory details
3840 * on the eviction algorithm. #PIN_NONBLOCK may used to prevent waiting on
3841 * evicting active overlapping objects, and any overlapping node that is pinned
3842 * or marked as unevictable will also result in failure.
3844 * Returns: 0 on success, -ENOSPC if no suitable hole is found, -EINTR if
3845 * asked to wait for eviction and interrupted.
3847 int i915_gem_gtt_reserve(struct i915_address_space
*vm
,
3848 struct drm_mm_node
*node
,
3849 u64 size
, u64 offset
, unsigned long color
,
3855 GEM_BUG_ON(!IS_ALIGNED(size
, I915_GTT_PAGE_SIZE
));
3856 GEM_BUG_ON(!IS_ALIGNED(offset
, I915_GTT_MIN_ALIGNMENT
));
3857 GEM_BUG_ON(range_overflows(offset
, size
, vm
->total
));
3858 GEM_BUG_ON(vm
== &vm
->i915
->mm
.aliasing_ppgtt
->base
);
3859 GEM_BUG_ON(drm_mm_node_allocated(node
));
3862 node
->start
= offset
;
3863 node
->color
= color
;
3865 err
= drm_mm_reserve_node(&vm
->mm
, node
);
3869 if (flags
& PIN_NOEVICT
)
3872 err
= i915_gem_evict_for_node(vm
, node
, flags
);
3874 err
= drm_mm_reserve_node(&vm
->mm
, node
);
3879 static u64
random_offset(u64 start
, u64 end
, u64 len
, u64 align
)
3883 GEM_BUG_ON(range_overflows(start
, len
, end
));
3884 GEM_BUG_ON(round_up(start
, align
) > round_down(end
- len
, align
));
3886 range
= round_down(end
- len
, align
) - round_up(start
, align
);
3888 if (sizeof(unsigned long) == sizeof(u64
)) {
3889 addr
= get_random_long();
3891 addr
= get_random_int();
3892 if (range
> U32_MAX
) {
3894 addr
|= get_random_int();
3897 div64_u64_rem(addr
, range
, &addr
);
3901 return round_up(start
, align
);
3905 * i915_gem_gtt_insert - insert a node into an address_space (GTT)
3906 * @vm: the &struct i915_address_space
3907 * @node: the &struct drm_mm_node (typically i915_vma.node)
3908 * @size: how much space to allocate inside the GTT,
3909 * must be #I915_GTT_PAGE_SIZE aligned
3910 * @alignment: required alignment of starting offset, may be 0 but
3911 * if specified, this must be a power-of-two and at least
3912 * #I915_GTT_MIN_ALIGNMENT
3913 * @color: color to apply to node
3914 * @start: start of any range restriction inside GTT (0 for all),
3915 * must be #I915_GTT_PAGE_SIZE aligned
3916 * @end: end of any range restriction inside GTT (U64_MAX for all),
3917 * must be #I915_GTT_PAGE_SIZE aligned if not U64_MAX
3918 * @flags: control search and eviction behaviour
3920 * i915_gem_gtt_insert() first searches for an available hole into which
3921 * is can insert the node. The hole address is aligned to @alignment and
3922 * its @size must then fit entirely within the [@start, @end] bounds. The
3923 * nodes on either side of the hole must match @color, or else a guard page
3924 * will be inserted between the two nodes (or the node evicted). If no
3925 * suitable hole is found, first a victim is randomly selected and tested
3926 * for eviction, otherwise then the LRU list of objects within the GTT
3927 * is scanned to find the first set of replacement nodes to create the hole.
3928 * Those old overlapping nodes are evicted from the GTT (and so must be
3929 * rebound before any future use). Any node that is currently pinned cannot
3930 * be evicted (see i915_vma_pin()). Similar if the node's VMA is currently
3931 * active and #PIN_NONBLOCK is specified, that node is also skipped when
3932 * searching for an eviction candidate. See i915_gem_evict_something() for
3933 * the gory details on the eviction algorithm.
3935 * Returns: 0 on success, -ENOSPC if no suitable hole is found, -EINTR if
3936 * asked to wait for eviction and interrupted.
3938 int i915_gem_gtt_insert(struct i915_address_space
*vm
,
3939 struct drm_mm_node
*node
,
3940 u64 size
, u64 alignment
, unsigned long color
,
3941 u64 start
, u64 end
, unsigned int flags
)
3943 enum drm_mm_insert_mode mode
;
3947 lockdep_assert_held(&vm
->i915
->drm
.struct_mutex
);
3949 GEM_BUG_ON(!IS_ALIGNED(size
, I915_GTT_PAGE_SIZE
));
3950 GEM_BUG_ON(alignment
&& !is_power_of_2(alignment
));
3951 GEM_BUG_ON(alignment
&& !IS_ALIGNED(alignment
, I915_GTT_MIN_ALIGNMENT
));
3952 GEM_BUG_ON(start
>= end
);
3953 GEM_BUG_ON(start
> 0 && !IS_ALIGNED(start
, I915_GTT_PAGE_SIZE
));
3954 GEM_BUG_ON(end
< U64_MAX
&& !IS_ALIGNED(end
, I915_GTT_PAGE_SIZE
));
3955 GEM_BUG_ON(vm
== &vm
->i915
->mm
.aliasing_ppgtt
->base
);
3956 GEM_BUG_ON(drm_mm_node_allocated(node
));
3958 if (unlikely(range_overflows(start
, size
, end
)))
3961 if (unlikely(round_up(start
, alignment
) > round_down(end
- size
, alignment
)))
3964 mode
= DRM_MM_INSERT_BEST
;
3965 if (flags
& PIN_HIGH
)
3966 mode
= DRM_MM_INSERT_HIGH
;
3967 if (flags
& PIN_MAPPABLE
)
3968 mode
= DRM_MM_INSERT_LOW
;
3970 /* We only allocate in PAGE_SIZE/GTT_PAGE_SIZE (4096) chunks,
3971 * so we know that we always have a minimum alignment of 4096.
3972 * The drm_mm range manager is optimised to return results
3973 * with zero alignment, so where possible use the optimal
3976 BUILD_BUG_ON(I915_GTT_MIN_ALIGNMENT
> I915_GTT_PAGE_SIZE
);
3977 if (alignment
<= I915_GTT_MIN_ALIGNMENT
)
3980 err
= drm_mm_insert_node_in_range(&vm
->mm
, node
,
3981 size
, alignment
, color
,
3986 if (flags
& PIN_NOEVICT
)
3989 /* No free space, pick a slot at random.
3991 * There is a pathological case here using a GTT shared between
3992 * mmap and GPU (i.e. ggtt/aliasing_ppgtt but not full-ppgtt):
3994 * |<-- 256 MiB aperture -->||<-- 1792 MiB unmappable -->|
3995 * (64k objects) (448k objects)
3997 * Now imagine that the eviction LRU is ordered top-down (just because
3998 * pathology meets real life), and that we need to evict an object to
3999 * make room inside the aperture. The eviction scan then has to walk
4000 * the 448k list before it finds one within range. And now imagine that
4001 * it has to search for a new hole between every byte inside the memcpy,
4002 * for several simultaneous clients.
4004 * On a full-ppgtt system, if we have run out of available space, there
4005 * will be lots and lots of objects in the eviction list! Again,
4006 * searching that LRU list may be slow if we are also applying any
4007 * range restrictions (e.g. restriction to low 4GiB) and so, for
4008 * simplicity and similarilty between different GTT, try the single
4009 * random replacement first.
4011 offset
= random_offset(start
, end
,
4012 size
, alignment
?: I915_GTT_MIN_ALIGNMENT
);
4013 err
= i915_gem_gtt_reserve(vm
, node
, size
, offset
, color
, flags
);
4017 /* Randomly selected placement is pinned, do a search */
4018 err
= i915_gem_evict_something(vm
, size
, alignment
, color
,
4023 return drm_mm_insert_node_in_range(&vm
->mm
, node
,
4024 size
, alignment
, color
,
4025 start
, end
, DRM_MM_INSERT_EVICT
);
4028 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
4029 #include "selftests/mock_gtt.c"
4030 #include "selftests/i915_gem_gtt.c"