]>
git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c
e0fa0cae79607767313ade8e8d4076659b53c283
2 * Copyright 2010 Red Hat Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
27 #include <subdev/bar.h>
28 #include <subdev/fb.h>
30 #include <nvif/if500d.h>
31 #include <nvif/if900d.h>
34 struct nvkm_mmu_pt
*pt
;
35 struct list_head head
;
42 nvkm_mmu_ptp_put(struct nvkm_mmu
*mmu
, bool force
, struct nvkm_mmu_pt
*pt
)
44 const int slot
= pt
->base
>> pt
->ptp
->shift
;
45 struct nvkm_mmu_ptp
*ptp
= pt
->ptp
;
47 /* If there were no free slots in the parent allocation before,
48 * there will be now, so return PTP to the cache.
51 list_add(&ptp
->head
, &mmu
->ptp
.list
);
52 ptp
->free
|= BIT(slot
);
54 /* If there's no more sub-allocations, destroy PTP. */
55 if (ptp
->free
== ptp
->mask
) {
56 nvkm_mmu_ptc_put(mmu
, force
, &ptp
->pt
);
65 nvkm_mmu_ptp_get(struct nvkm_mmu
*mmu
, u32 size
, bool zero
)
67 struct nvkm_mmu_pt
*pt
;
68 struct nvkm_mmu_ptp
*ptp
;
71 if (!(pt
= kzalloc(sizeof(*pt
), GFP_KERNEL
)))
74 ptp
= list_first_entry_or_null(&mmu
->ptp
.list
, typeof(*ptp
), head
);
76 /* Need to allocate a new parent to sub-allocate from. */
77 if (!(ptp
= kmalloc(sizeof(*ptp
), GFP_KERNEL
))) {
82 ptp
->pt
= nvkm_mmu_ptc_get(mmu
, 0x1000, 0x1000, false);
89 ptp
->shift
= order_base_2(size
);
90 slot
= nvkm_memory_size(ptp
->pt
->memory
) >> ptp
->shift
;
91 ptp
->mask
= (1 << slot
) - 1;
92 ptp
->free
= ptp
->mask
;
93 list_add(&ptp
->head
, &mmu
->ptp
.list
);
98 /* Sub-allocate from parent object, removing PTP from cache
99 * if there's no more free slots left.
101 slot
= __ffs(ptp
->free
);
102 ptp
->free
&= ~BIT(slot
);
104 list_del(&ptp
->head
);
106 pt
->memory
= pt
->ptp
->pt
->memory
;
107 pt
->base
= slot
<< ptp
->shift
;
108 pt
->addr
= pt
->ptp
->pt
->addr
+ pt
->base
;
112 struct nvkm_mmu_ptc
{
113 struct list_head head
;
114 struct list_head item
;
119 static inline struct nvkm_mmu_ptc
*
120 nvkm_mmu_ptc_find(struct nvkm_mmu
*mmu
, u32 size
)
122 struct nvkm_mmu_ptc
*ptc
;
124 list_for_each_entry(ptc
, &mmu
->ptc
.list
, head
) {
125 if (ptc
->size
== size
)
129 ptc
= kmalloc(sizeof(*ptc
), GFP_KERNEL
);
131 INIT_LIST_HEAD(&ptc
->item
);
134 list_add(&ptc
->head
, &mmu
->ptc
.list
);
141 nvkm_mmu_ptc_put(struct nvkm_mmu
*mmu
, bool force
, struct nvkm_mmu_pt
**ppt
)
143 struct nvkm_mmu_pt
*pt
= *ppt
;
145 /* Handle sub-allocated page tables. */
147 mutex_lock(&mmu
->ptp
.mutex
);
148 nvkm_mmu_ptp_put(mmu
, force
, pt
);
149 mutex_unlock(&mmu
->ptp
.mutex
);
153 /* Either cache or free the object. */
154 mutex_lock(&mmu
->ptc
.mutex
);
155 if (pt
->ptc
->refs
< 8 /* Heuristic. */ && !force
) {
156 list_add_tail(&pt
->head
, &pt
->ptc
->item
);
159 nvkm_memory_unref(&pt
->memory
);
162 mutex_unlock(&mmu
->ptc
.mutex
);
167 nvkm_mmu_ptc_get(struct nvkm_mmu
*mmu
, u32 size
, u32 align
, bool zero
)
169 struct nvkm_mmu_ptc
*ptc
;
170 struct nvkm_mmu_pt
*pt
;
173 /* Sub-allocated page table (ie. GP100 LPT). */
174 if (align
< 0x1000) {
175 mutex_lock(&mmu
->ptp
.mutex
);
176 pt
= nvkm_mmu_ptp_get(mmu
, align
, zero
);
177 mutex_unlock(&mmu
->ptp
.mutex
);
181 /* Lookup cache for this page table size. */
182 mutex_lock(&mmu
->ptc
.mutex
);
183 ptc
= nvkm_mmu_ptc_find(mmu
, size
);
185 mutex_unlock(&mmu
->ptc
.mutex
);
189 /* If there's a free PT in the cache, reuse it. */
190 pt
= list_first_entry_or_null(&ptc
->item
, typeof(*pt
), head
);
193 nvkm_fo64(pt
->memory
, 0, 0, size
>> 3);
196 mutex_unlock(&mmu
->ptc
.mutex
);
199 mutex_unlock(&mmu
->ptc
.mutex
);
201 /* No such luck, we need to allocate. */
202 if (!(pt
= kmalloc(sizeof(*pt
), GFP_KERNEL
)))
207 ret
= nvkm_memory_new(mmu
->subdev
.device
, NVKM_MEM_TARGET_INST
,
208 size
, align
, zero
, &pt
->memory
);
215 pt
->addr
= nvkm_memory_addr(pt
->memory
);
220 nvkm_vm_map_(const struct nvkm_vmm_page
*page
, struct nvkm_vma
*vma
, u64 delta
,
221 struct nvkm_mem
*mem
, nvkm_vmm_pte_func fn
,
222 struct nvkm_vmm_map
*map
)
225 struct nv50_vmm_map_v0 nv50
;
226 struct gf100_vmm_map_v0 gf100
;
228 struct nvkm_vmm
*vmm
= vma
->vm
;
233 map
->memory
= mem
->memory
;
236 if (vmm
->func
->valid
) {
237 switch (vmm
->mmu
->subdev
.device
->card_type
) {
239 args
.nv50
.version
= 0;
240 args
.nv50
.ro
= !(vma
->access
& NV_MEM_ACCESS_WO
);
241 args
.nv50
.priv
= !!(vma
->access
& NV_MEM_ACCESS_SYS
);
242 args
.nv50
.kind
= (mem
->memtype
& 0x07f);
243 args
.nv50
.comp
= (mem
->memtype
& 0x180) >> 7;
245 argc
= sizeof(args
.nv50
);
251 args
.gf100
.version
= 0;
252 args
.gf100
.vol
= (nvkm_memory_target(map
->memory
) != NVKM_MEM_TARGET_VRAM
);
253 args
.gf100
.ro
= !(vma
->access
& NV_MEM_ACCESS_WO
);
254 args
.gf100
.priv
= !!(vma
->access
& NV_MEM_ACCESS_SYS
);
255 args
.gf100
.kind
= (mem
->memtype
& 0x0ff);
257 argc
= sizeof(args
.gf100
);
264 ret
= vmm
->func
->valid(vmm
, argv
, argc
, map
);
269 mutex_lock(&vmm
->mutex
);
270 nvkm_vmm_ptes_map(vmm
, page
, vma
->node
->addr
+ delta
,
271 vma
->node
->size
, map
, fn
);
272 mutex_unlock(&vmm
->mutex
);
274 nvkm_memory_tags_put(vma
->node
->memory
, vmm
->mmu
->subdev
.device
, &vma
->node
->tags
);
275 nvkm_memory_unref(&vma
->node
->memory
);
276 vma
->node
->memory
= nvkm_memory_ref(map
->memory
);
277 vma
->node
->tags
= map
->tags
;
281 nvkm_mmu_ptc_dump(struct nvkm_mmu
*mmu
)
283 struct nvkm_mmu_ptc
*ptc
;
284 list_for_each_entry(ptc
, &mmu
->ptc
.list
, head
) {
285 struct nvkm_mmu_pt
*pt
, *tt
;
286 list_for_each_entry_safe(pt
, tt
, &ptc
->item
, head
) {
287 nvkm_memory_unref(&pt
->memory
);
295 nvkm_mmu_ptc_fini(struct nvkm_mmu
*mmu
)
297 struct nvkm_mmu_ptc
*ptc
, *ptct
;
299 list_for_each_entry_safe(ptc
, ptct
, &mmu
->ptc
.list
, head
) {
300 WARN_ON(!list_empty(&ptc
->item
));
301 list_del(&ptc
->head
);
307 nvkm_mmu_ptc_init(struct nvkm_mmu
*mmu
)
309 mutex_init(&mmu
->ptc
.mutex
);
310 INIT_LIST_HEAD(&mmu
->ptc
.list
);
311 mutex_init(&mmu
->ptp
.mutex
);
312 INIT_LIST_HEAD(&mmu
->ptp
.list
);
316 nvkm_vm_map_at(struct nvkm_vma
*vma
, u64 delta
, struct nvkm_mem
*node
)
318 const struct nvkm_vmm_page
*page
= &vma
->vm
->func
->page
[vma
->node
->page
];
319 if (page
->desc
->func
->unmap
) {
320 struct nvkm_vmm_map map
= { .mem
= node
->mem
};
321 nvkm_vm_map_(page
, vma
, delta
, node
, page
->desc
->func
->mem
, &map
);
327 nvkm_vm_map_sg_table(struct nvkm_vma
*vma
, u64 delta
, u64 length
,
328 struct nvkm_mem
*mem
)
330 const struct nvkm_vmm_page
*page
= &vma
->vm
->func
->page
[vma
->node
->page
];
331 if (page
->desc
->func
->unmap
) {
332 struct nvkm_vmm_map map
= { .sgl
= mem
->sg
->sgl
};
333 nvkm_vm_map_(page
, vma
, delta
, mem
, page
->desc
->func
->sgl
, &map
);
339 nvkm_vm_map_sg(struct nvkm_vma
*vma
, u64 delta
, u64 length
,
340 struct nvkm_mem
*mem
)
342 const struct nvkm_vmm_page
*page
= &vma
->vm
->func
->page
[vma
->node
->page
];
343 if (page
->desc
->func
->unmap
) {
344 struct nvkm_vmm_map map
= { .dma
= mem
->pages
};
345 nvkm_vm_map_(page
, vma
, delta
, mem
, page
->desc
->func
->dma
, &map
);
351 nvkm_vm_map(struct nvkm_vma
*vma
, struct nvkm_mem
*node
)
354 nvkm_vm_map_sg_table(vma
, 0, node
->size
<< 12, node
);
357 nvkm_vm_map_sg(vma
, 0, node
->size
<< 12, node
);
359 nvkm_vm_map_at(vma
, 0, node
);
363 nvkm_vm_unmap(struct nvkm_vma
*vma
)
365 nvkm_vmm_unmap(vma
->vm
, vma
->node
);
369 nvkm_vm_get(struct nvkm_vm
*vm
, u64 size
, u32 page_shift
, u32 access
,
370 struct nvkm_vma
*vma
)
374 mutex_lock(&vm
->mutex
);
375 ret
= nvkm_vmm_get_locked(vm
, true, false, false, page_shift
, 0,
377 mutex_unlock(&vm
->mutex
);
384 nvkm_vm_ref(vm
, &vma
->vm
, NULL
);
385 vma
->offset
= vma
->addr
= vma
->node
->addr
;
386 vma
->access
= access
;
391 nvkm_vm_put(struct nvkm_vma
*vma
)
393 nvkm_vmm_put(vma
->vm
, &vma
->node
);
394 nvkm_vm_ref(NULL
, &vma
->vm
, NULL
);
398 nvkm_vm_boot(struct nvkm_vm
*vm
, u64 size
)
400 return nvkm_vmm_boot(vm
);
404 nvkm_vm_new(struct nvkm_device
*device
, u64 offset
, u64 length
, u64 mm_offset
,
405 struct lock_class_key
*key
, struct nvkm_vm
**pvm
)
407 struct nvkm_mmu
*mmu
= device
->mmu
;
410 if (mmu
->func
->vmm
.ctor
) {
411 int ret
= mmu
->func
->vmm
.ctor(mmu
, mm_offset
,
412 offset
+ length
- mm_offset
,
413 NULL
, 0, key
, "legacy", pvm
);
415 nvkm_vm_ref(NULL
, pvm
, NULL
);
426 nvkm_vm_ref(struct nvkm_vm
*ref
, struct nvkm_vm
**ptr
, struct nvkm_memory
*inst
)
430 int ret
= nvkm_vmm_join(ref
, inst
);
439 nvkm_vmm_part(*ptr
, inst
);
448 nvkm_mmu_type(struct nvkm_mmu
*mmu
, int heap
, u8 type
)
450 if (heap
>= 0 && !WARN_ON(mmu
->type_nr
== ARRAY_SIZE(mmu
->type
))) {
451 mmu
->type
[mmu
->type_nr
].type
= type
| mmu
->heap
[heap
].type
;
452 mmu
->type
[mmu
->type_nr
].heap
= heap
;
458 nvkm_mmu_heap(struct nvkm_mmu
*mmu
, u8 type
, u64 size
)
461 if (!WARN_ON(mmu
->heap_nr
== ARRAY_SIZE(mmu
->heap
))) {
462 mmu
->heap
[mmu
->heap_nr
].type
= type
;
463 mmu
->heap
[mmu
->heap_nr
].size
= size
;
464 return mmu
->heap_nr
++;
471 nvkm_mmu_host(struct nvkm_mmu
*mmu
)
473 struct nvkm_device
*device
= mmu
->subdev
.device
;
474 u8 type
= NVKM_MEM_KIND
* !!mmu
->func
->kind_sys
;
477 /* Non-mappable system memory. */
478 heap
= nvkm_mmu_heap(mmu
, NVKM_MEM_HOST
, ~0ULL);
479 nvkm_mmu_type(mmu
, heap
, type
);
481 /* Non-coherent, cached, system memory.
483 * Block-linear mappings of system memory must be done through
484 * BAR1, and cannot be supported on systems where we're unable
485 * to map BAR1 with write-combining.
487 type
|= NVKM_MEM_MAPPABLE
;
488 if (!device
->bar
|| device
->bar
->iomap_uncached
)
489 nvkm_mmu_type(mmu
, heap
, type
& ~NVKM_MEM_KIND
);
491 nvkm_mmu_type(mmu
, heap
, type
);
493 /* Coherent, cached, system memory.
495 * Unsupported on systems that aren't able to support snooped
496 * mappings, and also for block-linear mappings which must be
499 type
|= NVKM_MEM_COHERENT
;
500 if (device
->func
->cpu_coherent
)
501 nvkm_mmu_type(mmu
, heap
, type
& ~NVKM_MEM_KIND
);
503 /* Uncached system memory. */
504 nvkm_mmu_type(mmu
, heap
, type
|= NVKM_MEM_UNCACHED
);
508 nvkm_mmu_vram(struct nvkm_mmu
*mmu
)
510 struct nvkm_device
*device
= mmu
->subdev
.device
;
511 struct nvkm_mm
*mm
= &device
->fb
->ram
->vram
;
512 const u32 sizeN
= nvkm_mm_heap_size(mm
, NVKM_RAM_MM_NORMAL
);
513 const u32 sizeU
= nvkm_mm_heap_size(mm
, NVKM_RAM_MM_NOMAP
);
514 const u32 sizeM
= nvkm_mm_heap_size(mm
, NVKM_RAM_MM_MIXED
);
515 u8 type
= NVKM_MEM_KIND
* !!mmu
->func
->kind
;
516 u8 heap
= NVKM_MEM_VRAM
;
517 int heapM
, heapN
, heapU
;
519 /* Mixed-memory doesn't support compression or display. */
520 heapM
= nvkm_mmu_heap(mmu
, heap
, sizeM
<< NVKM_RAM_MM_SHIFT
);
522 heap
|= NVKM_MEM_COMP
;
523 heap
|= NVKM_MEM_DISP
;
524 heapN
= nvkm_mmu_heap(mmu
, heap
, sizeN
<< NVKM_RAM_MM_SHIFT
);
525 heapU
= nvkm_mmu_heap(mmu
, heap
, sizeU
<< NVKM_RAM_MM_SHIFT
);
527 /* Add non-mappable VRAM types first so that they're preferred
528 * over anything else. Mixed-memory will be slower than other
529 * heaps, it's prioritised last.
531 nvkm_mmu_type(mmu
, heapU
, type
);
532 nvkm_mmu_type(mmu
, heapN
, type
);
533 nvkm_mmu_type(mmu
, heapM
, type
);
535 /* Add host memory types next, under the assumption that users
536 * wanting mappable memory want to use them as staging buffers
541 /* Mappable VRAM types go last, as they're basically the worst
542 * possible type to ask for unless there's no other choice.
545 /* Write-combined BAR1 access. */
546 type
|= NVKM_MEM_MAPPABLE
;
547 if (!device
->bar
->iomap_uncached
) {
548 nvkm_mmu_type(mmu
, heapN
, type
);
549 nvkm_mmu_type(mmu
, heapM
, type
);
552 /* Uncached BAR1 access. */
553 type
|= NVKM_MEM_COHERENT
;
554 type
|= NVKM_MEM_UNCACHED
;
555 nvkm_mmu_type(mmu
, heapN
, type
);
556 nvkm_mmu_type(mmu
, heapM
, type
);
561 nvkm_mmu_oneinit(struct nvkm_subdev
*subdev
)
563 struct nvkm_mmu
*mmu
= nvkm_mmu(subdev
);
565 /* Determine available memory types. */
566 if (mmu
->subdev
.device
->fb
&& mmu
->subdev
.device
->fb
->ram
)
571 if (mmu
->func
->vmm
.global
) {
572 int ret
= nvkm_vmm_new(subdev
->device
, 0, 0, NULL
, 0, NULL
,
582 nvkm_mmu_init(struct nvkm_subdev
*subdev
)
584 struct nvkm_mmu
*mmu
= nvkm_mmu(subdev
);
586 mmu
->func
->init(mmu
);
591 nvkm_mmu_dtor(struct nvkm_subdev
*subdev
)
593 struct nvkm_mmu
*mmu
= nvkm_mmu(subdev
);
595 nvkm_vmm_unref(&mmu
->vmm
);
597 nvkm_mmu_ptc_fini(mmu
);
601 static const struct nvkm_subdev_func
603 .dtor
= nvkm_mmu_dtor
,
604 .oneinit
= nvkm_mmu_oneinit
,
605 .init
= nvkm_mmu_init
,
609 nvkm_mmu_ctor(const struct nvkm_mmu_func
*func
, struct nvkm_device
*device
,
610 int index
, struct nvkm_mmu
*mmu
)
612 nvkm_subdev_ctor(&nvkm_mmu
, device
, index
, &mmu
->subdev
);
614 mmu
->limit
= func
->limit
;
615 mmu
->dma_bits
= func
->dma_bits
;
616 mmu
->lpg_shift
= func
->lpg_shift
;
617 nvkm_mmu_ptc_init(mmu
);
618 mmu
->user
.ctor
= nvkm_ummu_new
;
619 mmu
->user
.base
= func
->mmu
.user
;
623 nvkm_mmu_new_(const struct nvkm_mmu_func
*func
, struct nvkm_device
*device
,
624 int index
, struct nvkm_mmu
**pmmu
)
626 if (!(*pmmu
= kzalloc(sizeof(**pmmu
), GFP_KERNEL
)))
628 nvkm_mmu_ctor(func
, device
, index
, *pmmu
);