2 * Copyright 2017 Red Hat Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 #define NVKM_VMM_LEVELS_MAX 5
25 #include <subdev/fb.h>
28 nvkm_vmm_pt_del(struct nvkm_vmm_pt
**ppgt
)
30 struct nvkm_vmm_pt
*pgt
= *ppgt
;
39 static struct nvkm_vmm_pt
*
40 nvkm_vmm_pt_new(const struct nvkm_vmm_desc
*desc
, bool sparse
,
41 const struct nvkm_vmm_page
*page
)
43 const u32 pten
= 1 << desc
->bits
;
44 struct nvkm_vmm_pt
*pgt
;
47 if (desc
->type
> PGT
) {
48 if (desc
->type
== SPT
) {
49 const struct nvkm_vmm_desc
*pair
= page
[-1].desc
;
50 lpte
= pten
>> (desc
->bits
- pair
->bits
);
56 if (!(pgt
= kzalloc(sizeof(*pgt
) + lpte
, GFP_KERNEL
)))
58 pgt
->page
= page
? page
->shift
: 0;
61 if (desc
->type
== PGD
) {
62 pgt
->pde
= kvzalloc(sizeof(*pgt
->pde
) * pten
, GFP_KERNEL
);
72 struct nvkm_vmm_iter
{
73 const struct nvkm_vmm_page
*page
;
74 const struct nvkm_vmm_desc
*desc
;
78 u32 pte
[NVKM_VMM_LEVELS_MAX
];
79 struct nvkm_vmm_pt
*pt
[NVKM_VMM_LEVELS_MAX
];
83 #ifdef CONFIG_NOUVEAU_DEBUG_MMU
85 nvkm_vmm_desc_type(const struct nvkm_vmm_desc
*desc
)
88 case PGD
: return "PGD";
89 case PGT
: return "PGT";
90 case SPT
: return "SPT";
91 case LPT
: return "LPT";
98 nvkm_vmm_trace(struct nvkm_vmm_iter
*it
, char *buf
)
101 for (lvl
= it
->max
; lvl
>= 0; lvl
--) {
103 buf
+= sprintf(buf
, "%05x:", it
->pte
[lvl
]);
105 buf
+= sprintf(buf
, "xxxxx:");
109 #define TRA(i,f,a...) do { \
110 char _buf[NVKM_VMM_LEVELS_MAX * 7]; \
111 struct nvkm_vmm_iter *_it = (i); \
112 nvkm_vmm_trace(_it, _buf); \
113 VMM_TRACE(_it->vmm, "%s "f, _buf, ##a); \
116 #define TRA(i,f,a...)
120 nvkm_vmm_flush_mark(struct nvkm_vmm_iter
*it
)
122 it
->flush
= min(it
->flush
, it
->max
- it
->lvl
);
126 nvkm_vmm_flush(struct nvkm_vmm_iter
*it
)
128 if (it
->flush
!= NVKM_VMM_LEVELS_MAX
) {
129 if (it
->vmm
->func
->flush
) {
130 TRA(it
, "flush: %d", it
->flush
);
131 it
->vmm
->func
->flush(it
->vmm
, it
->flush
);
133 it
->flush
= NVKM_VMM_LEVELS_MAX
;
138 nvkm_vmm_unref_pdes(struct nvkm_vmm_iter
*it
)
140 const struct nvkm_vmm_desc
*desc
= it
->desc
;
141 const int type
= desc
[it
->lvl
].type
== SPT
;
142 struct nvkm_vmm_pt
*pgd
= it
->pt
[it
->lvl
+ 1];
143 struct nvkm_vmm_pt
*pgt
= it
->pt
[it
->lvl
];
144 struct nvkm_mmu_pt
*pt
= pgt
->pt
[type
];
145 struct nvkm_vmm
*vmm
= it
->vmm
;
146 u32 pdei
= it
->pte
[it
->lvl
+ 1];
148 /* Recurse up the tree, unreferencing/destroying unneeded PDs. */
150 if (--pgd
->refs
[0]) {
151 const struct nvkm_vmm_desc_func
*func
= desc
[it
->lvl
].func
;
152 /* PD has other valid PDEs, so we need a proper update. */
153 TRA(it
, "PDE unmap %s", nvkm_vmm_desc_type(&desc
[it
->lvl
- 1]));
154 pgt
->pt
[type
] = NULL
;
155 if (!pgt
->refs
[!type
]) {
156 /* PDE no longer required. */
159 func
->sparse(vmm
, pgd
->pt
[0], pdei
, 1);
160 pgd
->pde
[pdei
] = NVKM_VMM_PDE_SPARSE
;
162 func
->unmap(vmm
, pgd
->pt
[0], pdei
, 1);
163 pgd
->pde
[pdei
] = NULL
;
166 /* Special handling for Tesla-class GPUs,
167 * where there's no central PD, but each
168 * instance has its own embedded PD.
170 func
->pde(vmm
, pgd
, pdei
);
171 pgd
->pde
[pdei
] = NULL
;
174 /* PDE was pointing at dual-PTs and we're removing
175 * one of them, leaving the other in place.
177 func
->pde(vmm
, pgd
, pdei
);
180 /* GPU may have cached the PTs, flush before freeing. */
181 nvkm_vmm_flush_mark(it
);
184 /* PD has no valid PDEs left, so we can just destroy it. */
185 nvkm_vmm_unref_pdes(it
);
189 TRA(it
, "PDE free %s", nvkm_vmm_desc_type(&desc
[it
->lvl
- 1]));
190 nvkm_mmu_ptc_put(vmm
->mmu
, vmm
->bootstrapped
, &pt
);
191 if (!pgt
->refs
[!type
])
192 nvkm_vmm_pt_del(&pgt
);
197 nvkm_vmm_unref_sptes(struct nvkm_vmm_iter
*it
, struct nvkm_vmm_pt
*pgt
,
198 const struct nvkm_vmm_desc
*desc
, u32 ptei
, u32 ptes
)
200 const struct nvkm_vmm_desc
*pair
= it
->page
[-1].desc
;
201 const u32 sptb
= desc
->bits
- pair
->bits
;
202 const u32 sptn
= 1 << sptb
;
203 struct nvkm_vmm
*vmm
= it
->vmm
;
204 u32 spti
= ptei
& (sptn
- 1), lpti
, pteb
;
206 /* Determine how many SPTEs are being touched under each LPTE,
207 * and drop reference counts.
209 for (lpti
= ptei
>> sptb
; ptes
; spti
= 0, lpti
++) {
210 const u32 pten
= min(sptn
- spti
, ptes
);
211 pgt
->pte
[lpti
] -= pten
;
215 /* We're done here if there's no corresponding LPT. */
219 for (ptei
= pteb
= ptei
>> sptb
; ptei
< lpti
; pteb
= ptei
) {
220 /* Skip over any LPTEs that still have valid SPTEs. */
221 if (pgt
->pte
[pteb
] & NVKM_VMM_PTE_SPTES
) {
222 for (ptes
= 1, ptei
++; ptei
< lpti
; ptes
++, ptei
++) {
223 if (!(pgt
->pte
[ptei
] & NVKM_VMM_PTE_SPTES
))
229 /* As there's no more non-UNMAPPED SPTEs left in the range
230 * covered by a number of LPTEs, the LPTEs once again take
231 * control over their address range.
233 * Determine how many LPTEs need to transition state.
235 pgt
->pte
[ptei
] &= ~NVKM_VMM_PTE_VALID
;
236 for (ptes
= 1, ptei
++; ptei
< lpti
; ptes
++, ptei
++) {
237 if (pgt
->pte
[ptei
] & NVKM_VMM_PTE_SPTES
)
239 pgt
->pte
[ptei
] &= ~NVKM_VMM_PTE_VALID
;
242 if (pgt
->pte
[pteb
] & NVKM_VMM_PTE_SPARSE
) {
243 TRA(it
, "LPTE %05x: U -> S %d PTEs", pteb
, ptes
);
244 pair
->func
->sparse(vmm
, pgt
->pt
[0], pteb
, ptes
);
246 if (pair
->func
->invalid
) {
247 /* If the MMU supports it, restore the LPTE to the
248 * INVALID state to tell the MMU there is no point
249 * trying to fetch the corresponding SPTEs.
251 TRA(it
, "LPTE %05x: U -> I %d PTEs", pteb
, ptes
);
252 pair
->func
->invalid(vmm
, pgt
->pt
[0], pteb
, ptes
);
258 nvkm_vmm_unref_ptes(struct nvkm_vmm_iter
*it
, u32 ptei
, u32 ptes
)
260 const struct nvkm_vmm_desc
*desc
= it
->desc
;
261 const int type
= desc
->type
== SPT
;
262 struct nvkm_vmm_pt
*pgt
= it
->pt
[0];
264 /* Drop PTE references. */
265 pgt
->refs
[type
] -= ptes
;
267 /* Dual-PTs need special handling, unless PDE becoming invalid. */
268 if (desc
->type
== SPT
&& (pgt
->refs
[0] || pgt
->refs
[1]))
269 nvkm_vmm_unref_sptes(it
, pgt
, desc
, ptei
, ptes
);
271 /* PT no longer neeed? Destroy it. */
272 if (!pgt
->refs
[type
]) {
274 TRA(it
, "%s empty", nvkm_vmm_desc_type(desc
));
276 nvkm_vmm_unref_pdes(it
);
277 return false; /* PTE writes for unmap() not necessary. */
284 nvkm_vmm_ref_sptes(struct nvkm_vmm_iter
*it
, struct nvkm_vmm_pt
*pgt
,
285 const struct nvkm_vmm_desc
*desc
, u32 ptei
, u32 ptes
)
287 const struct nvkm_vmm_desc
*pair
= it
->page
[-1].desc
;
288 const u32 sptb
= desc
->bits
- pair
->bits
;
289 const u32 sptn
= 1 << sptb
;
290 struct nvkm_vmm
*vmm
= it
->vmm
;
291 u32 spti
= ptei
& (sptn
- 1), lpti
, pteb
;
293 /* Determine how many SPTEs are being touched under each LPTE,
294 * and increase reference counts.
296 for (lpti
= ptei
>> sptb
; ptes
; spti
= 0, lpti
++) {
297 const u32 pten
= min(sptn
- spti
, ptes
);
298 pgt
->pte
[lpti
] += pten
;
302 /* We're done here if there's no corresponding LPT. */
306 for (ptei
= pteb
= ptei
>> sptb
; ptei
< lpti
; pteb
= ptei
) {
307 /* Skip over any LPTEs that already have valid SPTEs. */
308 if (pgt
->pte
[pteb
] & NVKM_VMM_PTE_VALID
) {
309 for (ptes
= 1, ptei
++; ptei
< lpti
; ptes
++, ptei
++) {
310 if (!(pgt
->pte
[ptei
] & NVKM_VMM_PTE_VALID
))
316 /* As there are now non-UNMAPPED SPTEs in the range covered
317 * by a number of LPTEs, we need to transfer control of the
318 * address range to the SPTEs.
320 * Determine how many LPTEs need to transition state.
322 pgt
->pte
[ptei
] |= NVKM_VMM_PTE_VALID
;
323 for (ptes
= 1, ptei
++; ptei
< lpti
; ptes
++, ptei
++) {
324 if (pgt
->pte
[ptei
] & NVKM_VMM_PTE_VALID
)
326 pgt
->pte
[ptei
] |= NVKM_VMM_PTE_VALID
;
329 if (pgt
->pte
[pteb
] & NVKM_VMM_PTE_SPARSE
) {
330 const u32 spti
= pteb
* sptn
;
331 const u32 sptc
= ptes
* sptn
;
332 /* The entire LPTE is marked as sparse, we need
333 * to make sure that the SPTEs are too.
335 TRA(it
, "SPTE %05x: U -> S %d PTEs", spti
, sptc
);
336 desc
->func
->sparse(vmm
, pgt
->pt
[1], spti
, sptc
);
337 /* Sparse LPTEs prevent SPTEs from being accessed. */
338 TRA(it
, "LPTE %05x: S -> U %d PTEs", pteb
, ptes
);
339 pair
->func
->unmap(vmm
, pgt
->pt
[0], pteb
, ptes
);
341 if (pair
->func
->invalid
) {
342 /* MMU supports blocking SPTEs by marking an LPTE
343 * as INVALID. We need to reverse that here.
345 TRA(it
, "LPTE %05x: I -> U %d PTEs", pteb
, ptes
);
346 pair
->func
->unmap(vmm
, pgt
->pt
[0], pteb
, ptes
);
352 nvkm_vmm_ref_ptes(struct nvkm_vmm_iter
*it
, u32 ptei
, u32 ptes
)
354 const struct nvkm_vmm_desc
*desc
= it
->desc
;
355 const int type
= desc
->type
== SPT
;
356 struct nvkm_vmm_pt
*pgt
= it
->pt
[0];
358 /* Take PTE references. */
359 pgt
->refs
[type
] += ptes
;
361 /* Dual-PTs need special handling. */
362 if (desc
->type
== SPT
)
363 nvkm_vmm_ref_sptes(it
, pgt
, desc
, ptei
, ptes
);
369 nvkm_vmm_sparse_ptes(const struct nvkm_vmm_desc
*desc
,
370 struct nvkm_vmm_pt
*pgt
, u32 ptei
, u32 ptes
)
372 if (desc
->type
== PGD
) {
374 pgt
->pde
[ptei
++] = NVKM_VMM_PDE_SPARSE
;
376 if (desc
->type
== LPT
) {
377 memset(&pgt
->pte
[ptei
], NVKM_VMM_PTE_SPARSE
, ptes
);
382 nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter
*it
, u32 ptei
, u32 ptes
)
384 struct nvkm_vmm_pt
*pt
= it
->pt
[0];
385 if (it
->desc
->type
== PGD
)
386 memset(&pt
->pde
[ptei
], 0x00, sizeof(pt
->pde
[0]) * ptes
);
388 if (it
->desc
->type
== LPT
)
389 memset(&pt
->pte
[ptei
], 0x00, sizeof(pt
->pte
[0]) * ptes
);
390 return nvkm_vmm_unref_ptes(it
, ptei
, ptes
);
394 nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter
*it
, u32 ptei
, u32 ptes
)
396 nvkm_vmm_sparse_ptes(it
->desc
, it
->pt
[0], ptei
, ptes
);
397 return nvkm_vmm_ref_ptes(it
, ptei
, ptes
);
401 nvkm_vmm_ref_hwpt(struct nvkm_vmm_iter
*it
, struct nvkm_vmm_pt
*pgd
, u32 pdei
)
403 const struct nvkm_vmm_desc
*desc
= &it
->desc
[it
->lvl
- 1];
404 const int type
= desc
->type
== SPT
;
405 struct nvkm_vmm_pt
*pgt
= pgd
->pde
[pdei
];
406 const bool zero
= !pgt
->sparse
&& !desc
->func
->invalid
;
407 struct nvkm_vmm
*vmm
= it
->vmm
;
408 struct nvkm_mmu
*mmu
= vmm
->mmu
;
409 struct nvkm_mmu_pt
*pt
;
410 u32 pten
= 1 << desc
->bits
;
411 u32 pteb
, ptei
, ptes
;
412 u32 size
= desc
->size
* pten
;
416 pgt
->pt
[type
] = nvkm_mmu_ptc_get(mmu
, size
, desc
->align
, zero
);
417 if (!pgt
->pt
[type
]) {
419 nvkm_vmm_unref_pdes(it
);
428 if (desc
->type
== LPT
&& pgt
->refs
[1]) {
429 /* SPT already exists covering the same range as this LPT,
430 * which means we need to be careful that any LPTEs which
431 * overlap valid SPTEs are unmapped as opposed to invalid
432 * or sparse, which would prevent the MMU from looking at
433 * the SPTEs on some GPUs.
435 for (ptei
= pteb
= 0; ptei
< pten
; pteb
= ptei
) {
436 bool spte
= pgt
->pte
[ptei
] & NVKM_VMM_PTE_SPTES
;
437 for (ptes
= 1, ptei
++; ptei
< pten
; ptes
++, ptei
++) {
438 bool next
= pgt
->pte
[ptei
] & NVKM_VMM_PTE_SPTES
;
445 desc
->func
->sparse(vmm
, pt
, pteb
, ptes
);
447 desc
->func
->invalid(vmm
, pt
, pteb
, ptes
);
448 memset(&pgt
->pte
[pteb
], 0x00, ptes
);
450 desc
->func
->unmap(vmm
, pt
, pteb
, ptes
);
452 pgt
->pte
[pteb
++] |= NVKM_VMM_PTE_VALID
;
457 nvkm_vmm_sparse_ptes(desc
, pgt
, 0, pten
);
458 desc
->func
->sparse(vmm
, pt
, 0, pten
);
460 desc
->func
->invalid(vmm
, pt
, 0, pten
);
465 TRA(it
, "PDE write %s", nvkm_vmm_desc_type(desc
));
466 it
->desc
[it
->lvl
].func
->pde(it
->vmm
, pgd
, pdei
);
467 nvkm_vmm_flush_mark(it
);
472 nvkm_vmm_ref_swpt(struct nvkm_vmm_iter
*it
, struct nvkm_vmm_pt
*pgd
, u32 pdei
)
474 const struct nvkm_vmm_desc
*desc
= &it
->desc
[it
->lvl
- 1];
475 struct nvkm_vmm_pt
*pgt
= pgd
->pde
[pdei
];
477 pgt
= nvkm_vmm_pt_new(desc
, NVKM_VMM_PDE_SPARSED(pgt
), it
->page
);
480 nvkm_vmm_unref_pdes(it
);
484 pgd
->pde
[pdei
] = pgt
;
489 nvkm_vmm_iter(struct nvkm_vmm
*vmm
, const struct nvkm_vmm_page
*page
,
490 u64 addr
, u64 size
, const char *name
, bool ref
,
491 bool (*REF_PTES
)(struct nvkm_vmm_iter
*, u32
, u32
),
492 nvkm_vmm_pte_func MAP_PTES
, struct nvkm_vmm_map
*map
,
493 nvkm_vmm_pxe_func CLR_PTES
)
495 const struct nvkm_vmm_desc
*desc
= page
->desc
;
496 struct nvkm_vmm_iter it
;
497 u64 bits
= addr
>> page
->shift
;
502 it
.cnt
= size
>> page
->shift
;
503 it
.flush
= NVKM_VMM_LEVELS_MAX
;
505 /* Deconstruct address into PTE indices for each mapping level. */
506 for (it
.lvl
= 0; desc
[it
.lvl
].bits
; it
.lvl
++) {
507 it
.pte
[it
.lvl
] = bits
& ((1 << desc
[it
.lvl
].bits
) - 1);
508 bits
>>= desc
[it
.lvl
].bits
;
511 it
.pt
[it
.max
] = vmm
->pd
;
514 TRA(&it
, "%s: %016llx %016llx %d %lld PTEs", name
,
515 addr
, size
, page
->shift
, it
.cnt
);
518 /* Depth-first traversal of page tables. */
520 struct nvkm_vmm_pt
*pgt
= it
.pt
[it
.lvl
];
521 const int type
= desc
->type
== SPT
;
522 const u32 pten
= 1 << desc
->bits
;
523 const u32 ptei
= it
.pte
[0];
524 const u32 ptes
= min_t(u64
, it
.cnt
, pten
- ptei
);
526 /* Walk down the tree, finding page tables for each level. */
527 for (; it
.lvl
; it
.lvl
--) {
528 const u32 pdei
= it
.pte
[it
.lvl
];
529 struct nvkm_vmm_pt
*pgd
= pgt
;
532 if (ref
&& NVKM_VMM_PDE_INVALID(pgd
->pde
[pdei
])) {
533 if (!nvkm_vmm_ref_swpt(&it
, pgd
, pdei
))
536 it
.pt
[it
.lvl
- 1] = pgt
= pgd
->pde
[pdei
];
540 * This is a separate step from above due to GF100 and
541 * newer having dual page tables at some levels, which
542 * are refcounted independently.
544 if (ref
&& !pgt
->refs
[desc
[it
.lvl
- 1].type
== SPT
]) {
545 if (!nvkm_vmm_ref_hwpt(&it
, pgd
, pdei
))
550 /* Handle PTE updates. */
551 if (!REF_PTES
|| REF_PTES(&it
, ptei
, ptes
)) {
552 struct nvkm_mmu_pt
*pt
= pgt
->pt
[type
];
553 if (MAP_PTES
|| CLR_PTES
) {
555 MAP_PTES(vmm
, pt
, ptei
, ptes
, map
);
557 CLR_PTES(vmm
, pt
, ptei
, ptes
);
558 nvkm_vmm_flush_mark(&it
);
562 /* Walk back up the tree to the next position. */
563 it
.pte
[it
.lvl
] += ptes
;
566 while (it
.pte
[it
.lvl
] == (1 << desc
[it
.lvl
].bits
)) {
567 it
.pte
[it
.lvl
++] = 0;
577 /* Reconstruct the failure address so the caller is able to
578 * reverse any partially completed operations.
580 addr
= it
.pte
[it
.max
--];
582 addr
= addr
<< desc
[it
.max
].bits
;
583 addr
|= it
.pte
[it
.max
];
586 return addr
<< page
->shift
;
590 nvkm_vmm_ptes_sparse_put(struct nvkm_vmm
*vmm
, const struct nvkm_vmm_page
*page
,
593 nvkm_vmm_iter(vmm
, page
, addr
, size
, "sparse unref", false,
594 nvkm_vmm_sparse_unref_ptes
, NULL
, NULL
,
595 page
->desc
->func
->invalid
?
596 page
->desc
->func
->invalid
: page
->desc
->func
->unmap
);
600 nvkm_vmm_ptes_sparse_get(struct nvkm_vmm
*vmm
, const struct nvkm_vmm_page
*page
,
603 if ((page
->type
& NVKM_VMM_PAGE_SPARSE
)) {
604 u64 fail
= nvkm_vmm_iter(vmm
, page
, addr
, size
, "sparse ref",
605 true, nvkm_vmm_sparse_ref_ptes
, NULL
,
606 NULL
, page
->desc
->func
->sparse
);
608 if ((size
= fail
- addr
))
609 nvkm_vmm_ptes_sparse_put(vmm
, page
, addr
, size
);
618 nvkm_vmm_ptes_sparse(struct nvkm_vmm
*vmm
, u64 addr
, u64 size
, bool ref
)
620 const struct nvkm_vmm_page
*page
= vmm
->func
->page
;
626 /* Limit maximum page size based on remaining size. */
627 while (size
< (1ULL << page
[m
].shift
))
631 /* Find largest page size suitable for alignment. */
632 while (!IS_ALIGNED(addr
, 1ULL << page
[i
].shift
))
635 /* Determine number of PTEs at this page size. */
637 /* Limited to alignment boundary of next page size. */
638 u64 next
= 1ULL << page
[i
- 1].shift
;
639 u64 part
= ALIGN(addr
, next
) - addr
;
640 if (size
- part
>= next
)
641 block
= (part
>> page
[i
].shift
) << page
[i
].shift
;
643 block
= (size
>> page
[i
].shift
) << page
[i
].shift
;
645 block
= (size
>> page
[i
].shift
) << page
[i
].shift
;;
648 /* Perform operation. */
650 int ret
= nvkm_vmm_ptes_sparse_get(vmm
, &page
[i
], addr
, block
);
652 if ((size
= addr
- start
))
653 nvkm_vmm_ptes_sparse(vmm
, start
, size
, false);
657 nvkm_vmm_ptes_sparse_put(vmm
, &page
[i
], addr
, block
);
668 nvkm_vmm_ptes_unmap_put(struct nvkm_vmm
*vmm
, const struct nvkm_vmm_page
*page
,
669 u64 addr
, u64 size
, bool sparse
)
671 const struct nvkm_vmm_desc_func
*func
= page
->desc
->func
;
672 nvkm_vmm_iter(vmm
, page
, addr
, size
, "unmap + unref",
673 false, nvkm_vmm_unref_ptes
, NULL
, NULL
,
674 sparse
? func
->sparse
: func
->invalid
? func
->invalid
:
679 nvkm_vmm_ptes_get_map(struct nvkm_vmm
*vmm
, const struct nvkm_vmm_page
*page
,
680 u64 addr
, u64 size
, struct nvkm_vmm_map
*map
,
681 nvkm_vmm_pte_func func
)
683 u64 fail
= nvkm_vmm_iter(vmm
, page
, addr
, size
, "ref + map", true,
684 nvkm_vmm_ref_ptes
, func
, map
, NULL
);
686 if ((size
= fail
- addr
))
687 nvkm_vmm_ptes_unmap_put(vmm
, page
, addr
, size
, false);
694 nvkm_vmm_ptes_unmap(struct nvkm_vmm
*vmm
, const struct nvkm_vmm_page
*page
,
695 u64 addr
, u64 size
, bool sparse
)
697 const struct nvkm_vmm_desc_func
*func
= page
->desc
->func
;
698 nvkm_vmm_iter(vmm
, page
, addr
, size
, "unmap", false, NULL
, NULL
, NULL
,
699 sparse
? func
->sparse
: func
->invalid
? func
->invalid
:
704 nvkm_vmm_ptes_map(struct nvkm_vmm
*vmm
, const struct nvkm_vmm_page
*page
,
705 u64 addr
, u64 size
, struct nvkm_vmm_map
*map
,
706 nvkm_vmm_pte_func func
)
708 nvkm_vmm_iter(vmm
, page
, addr
, size
, "map", false,
709 NULL
, func
, map
, NULL
);
713 nvkm_vmm_ptes_put(struct nvkm_vmm
*vmm
, const struct nvkm_vmm_page
*page
,
716 nvkm_vmm_iter(vmm
, page
, addr
, size
, "unref", false,
717 nvkm_vmm_unref_ptes
, NULL
, NULL
, NULL
);
721 nvkm_vmm_ptes_get(struct nvkm_vmm
*vmm
, const struct nvkm_vmm_page
*page
,
724 u64 fail
= nvkm_vmm_iter(vmm
, page
, addr
, size
, "ref", true,
725 nvkm_vmm_ref_ptes
, NULL
, NULL
, NULL
);
728 nvkm_vmm_ptes_put(vmm
, page
, addr
, fail
- addr
);
734 static inline struct nvkm_vma
*
735 nvkm_vma_new(u64 addr
, u64 size
)
737 struct nvkm_vma
*vma
= kzalloc(sizeof(*vma
), GFP_KERNEL
);
741 vma
->page
= NVKM_VMA_PAGE_NONE
;
742 vma
->refd
= NVKM_VMA_PAGE_NONE
;
748 nvkm_vma_tail(struct nvkm_vma
*vma
, u64 tail
)
750 struct nvkm_vma
*new;
752 BUG_ON(vma
->size
== tail
);
754 if (!(new = nvkm_vma_new(vma
->addr
+ (vma
->size
- tail
), tail
)))
758 new->mapref
= vma
->mapref
;
759 new->sparse
= vma
->sparse
;
760 new->page
= vma
->page
;
761 new->refd
= vma
->refd
;
762 new->used
= vma
->used
;
763 new->part
= vma
->part
;
764 new->user
= vma
->user
;
765 new->busy
= vma
->busy
;
766 list_add(&new->head
, &vma
->head
);
771 nvkm_vmm_free_insert(struct nvkm_vmm
*vmm
, struct nvkm_vma
*vma
)
773 struct rb_node
**ptr
= &vmm
->free
.rb_node
;
774 struct rb_node
*parent
= NULL
;
777 struct nvkm_vma
*this = rb_entry(*ptr
, typeof(*this), tree
);
779 if (vma
->size
< this->size
)
780 ptr
= &parent
->rb_left
;
782 if (vma
->size
> this->size
)
783 ptr
= &parent
->rb_right
;
785 if (vma
->addr
< this->addr
)
786 ptr
= &parent
->rb_left
;
788 if (vma
->addr
> this->addr
)
789 ptr
= &parent
->rb_right
;
794 rb_link_node(&vma
->tree
, parent
, ptr
);
795 rb_insert_color(&vma
->tree
, &vmm
->free
);
799 nvkm_vmm_node_insert(struct nvkm_vmm
*vmm
, struct nvkm_vma
*vma
)
801 struct rb_node
**ptr
= &vmm
->root
.rb_node
;
802 struct rb_node
*parent
= NULL
;
805 struct nvkm_vma
*this = rb_entry(*ptr
, typeof(*this), tree
);
807 if (vma
->addr
< this->addr
)
808 ptr
= &parent
->rb_left
;
810 if (vma
->addr
> this->addr
)
811 ptr
= &parent
->rb_right
;
816 rb_link_node(&vma
->tree
, parent
, ptr
);
817 rb_insert_color(&vma
->tree
, &vmm
->root
);
821 nvkm_vmm_node_search(struct nvkm_vmm
*vmm
, u64 addr
)
823 struct rb_node
*node
= vmm
->root
.rb_node
;
825 struct nvkm_vma
*vma
= rb_entry(node
, typeof(*vma
), tree
);
826 if (addr
< vma
->addr
)
827 node
= node
->rb_left
;
829 if (addr
>= vma
->addr
+ vma
->size
)
830 node
= node
->rb_right
;
838 nvkm_vmm_dtor(struct nvkm_vmm
*vmm
)
840 struct nvkm_vma
*vma
;
841 struct rb_node
*node
;
843 while ((node
= rb_first(&vmm
->root
))) {
844 struct nvkm_vma
*vma
= rb_entry(node
, typeof(*vma
), tree
);
845 nvkm_vmm_put(vmm
, &vma
);
848 if (vmm
->bootstrapped
) {
849 const struct nvkm_vmm_page
*page
= vmm
->func
->page
;
850 const u64 limit
= vmm
->limit
- vmm
->start
;
852 while (page
[1].shift
)
855 nvkm_mmu_ptc_dump(vmm
->mmu
);
856 nvkm_vmm_ptes_put(vmm
, page
, vmm
->start
, limit
);
859 vma
= list_first_entry(&vmm
->list
, typeof(*vma
), head
);
860 list_del(&vma
->head
);
862 WARN_ON(!list_empty(&vmm
->list
));
865 dma_free_coherent(vmm
->mmu
->subdev
.device
->dev
, 16 * 1024,
866 vmm
->nullp
, vmm
->null
);
870 nvkm_mmu_ptc_put(vmm
->mmu
, true, &vmm
->pd
->pt
[0]);
871 nvkm_vmm_pt_del(&vmm
->pd
);
876 nvkm_vmm_ctor(const struct nvkm_vmm_func
*func
, struct nvkm_mmu
*mmu
,
877 u32 pd_header
, u64 addr
, u64 size
, struct lock_class_key
*key
,
878 const char *name
, struct nvkm_vmm
*vmm
)
880 static struct lock_class_key _key
;
881 const struct nvkm_vmm_page
*page
= func
->page
;
882 const struct nvkm_vmm_desc
*desc
;
883 struct nvkm_vma
*vma
;
884 int levels
, bits
= 0;
889 vmm
->debug
= mmu
->subdev
.debug
;
890 kref_init(&vmm
->kref
);
892 __mutex_init(&vmm
->mutex
, "&vmm->mutex", key
? key
: &_key
);
894 /* Locate the smallest page size supported by the backend, it will
895 * have the the deepest nesting of page tables.
897 while (page
[1].shift
)
900 /* Locate the structure that describes the layout of the top-level
901 * page table, and determine the number of valid bits in a virtual
904 for (levels
= 0, desc
= page
->desc
; desc
->bits
; desc
++, levels
++)
909 if (WARN_ON(levels
> NVKM_VMM_LEVELS_MAX
))
913 vmm
->limit
= size
? (addr
+ size
) : (1ULL << bits
);
914 if (vmm
->start
> vmm
->limit
|| vmm
->limit
> (1ULL << bits
))
917 /* Allocate top-level page table. */
918 vmm
->pd
= nvkm_vmm_pt_new(desc
, false, NULL
);
921 vmm
->pd
->refs
[0] = 1;
922 INIT_LIST_HEAD(&vmm
->join
);
924 /* ... and the GPU storage for it, except on Tesla-class GPUs that
925 * have the PD embedded in the instance structure.
928 const u32 size
= pd_header
+ desc
->size
* (1 << desc
->bits
);
929 vmm
->pd
->pt
[0] = nvkm_mmu_ptc_get(mmu
, size
, desc
->align
, true);
934 /* Initialise address-space MM. */
935 INIT_LIST_HEAD(&vmm
->list
);
939 if (!(vma
= nvkm_vma_new(vmm
->start
, vmm
->limit
- vmm
->start
)))
942 nvkm_vmm_free_insert(vmm
, vma
);
943 list_add(&vma
->head
, &vmm
->list
);
948 nvkm_vmm_new_(const struct nvkm_vmm_func
*func
, struct nvkm_mmu
*mmu
,
949 u32 hdr
, u64 addr
, u64 size
, struct lock_class_key
*key
,
950 const char *name
, struct nvkm_vmm
**pvmm
)
952 if (!(*pvmm
= kzalloc(sizeof(**pvmm
), GFP_KERNEL
)))
954 return nvkm_vmm_ctor(func
, mmu
, hdr
, addr
, size
, key
, name
, *pvmm
);
957 #define node(root, dir) ((root)->head.dir == &vmm->list) ? NULL : \
958 list_entry((root)->head.dir, struct nvkm_vma, head)
961 nvkm_vmm_unmap_region(struct nvkm_vmm
*vmm
, struct nvkm_vma
*vma
)
963 struct nvkm_vma
*next
;
965 nvkm_memory_tags_put(vma
->memory
, vmm
->mmu
->subdev
.device
, &vma
->tags
);
966 nvkm_memory_unref(&vma
->memory
);
969 struct nvkm_vma
*prev
= node(vma
, prev
);
971 prev
->size
+= vma
->size
;
972 rb_erase(&vma
->tree
, &vmm
->root
);
973 list_del(&vma
->head
);
979 next
= node(vma
, next
);
980 if (next
&& next
->part
) {
982 vma
->size
+= next
->size
;
983 rb_erase(&next
->tree
, &vmm
->root
);
984 list_del(&next
->head
);
991 nvkm_vmm_unmap_locked(struct nvkm_vmm
*vmm
, struct nvkm_vma
*vma
)
993 const struct nvkm_vmm_page
*page
= &vmm
->func
->page
[vma
->refd
];
996 nvkm_vmm_ptes_unmap_put(vmm
, page
, vma
->addr
, vma
->size
, vma
->sparse
);
997 vma
->refd
= NVKM_VMA_PAGE_NONE
;
999 nvkm_vmm_ptes_unmap(vmm
, page
, vma
->addr
, vma
->size
, vma
->sparse
);
1002 nvkm_vmm_unmap_region(vmm
, vma
);
1006 nvkm_vmm_unmap(struct nvkm_vmm
*vmm
, struct nvkm_vma
*vma
)
1009 mutex_lock(&vmm
->mutex
);
1010 nvkm_vmm_unmap_locked(vmm
, vma
);
1011 mutex_unlock(&vmm
->mutex
);
1016 nvkm_vmm_map_valid(struct nvkm_vmm
*vmm
, struct nvkm_vma
*vma
,
1017 void *argv
, u32 argc
, struct nvkm_vmm_map
*map
)
1019 switch (nvkm_memory_target(map
->memory
)) {
1020 case NVKM_MEM_TARGET_VRAM
:
1021 if (!(map
->page
->type
& NVKM_VMM_PAGE_VRAM
)) {
1022 VMM_DEBUG(vmm
, "%d !VRAM", map
->page
->shift
);
1026 case NVKM_MEM_TARGET_HOST
:
1027 case NVKM_MEM_TARGET_NCOH
:
1028 if (!(map
->page
->type
& NVKM_VMM_PAGE_HOST
)) {
1029 VMM_DEBUG(vmm
, "%d !HOST", map
->page
->shift
);
1038 if (!IS_ALIGNED( vma
->addr
, 1ULL << map
->page
->shift
) ||
1039 !IS_ALIGNED((u64
)vma
->size
, 1ULL << map
->page
->shift
) ||
1040 !IS_ALIGNED( map
->offset
, 1ULL << map
->page
->shift
) ||
1041 nvkm_memory_page(map
->memory
) < map
->page
->shift
) {
1042 VMM_DEBUG(vmm
, "alignment %016llx %016llx %016llx %d %d",
1043 vma
->addr
, (u64
)vma
->size
, map
->offset
, map
->page
->shift
,
1044 nvkm_memory_page(map
->memory
));
1048 return vmm
->func
->valid(vmm
, argv
, argc
, map
);
1052 nvkm_vmm_map_choose(struct nvkm_vmm
*vmm
, struct nvkm_vma
*vma
,
1053 void *argv
, u32 argc
, struct nvkm_vmm_map
*map
)
1055 for (map
->page
= vmm
->func
->page
; map
->page
->shift
; map
->page
++) {
1056 VMM_DEBUG(vmm
, "trying %d", map
->page
->shift
);
1057 if (!nvkm_vmm_map_valid(vmm
, vma
, argv
, argc
, map
))
1064 nvkm_vmm_map_locked(struct nvkm_vmm
*vmm
, struct nvkm_vma
*vma
,
1065 void *argv
, u32 argc
, struct nvkm_vmm_map
*map
)
1067 nvkm_vmm_pte_func func
;
1070 /* Make sure we won't overrun the end of the memory object. */
1071 if (unlikely(nvkm_memory_size(map
->memory
) < map
->offset
+ vma
->size
)) {
1072 VMM_DEBUG(vmm
, "overrun %016llx %016llx %016llx",
1073 nvkm_memory_size(map
->memory
),
1074 map
->offset
, (u64
)vma
->size
);
1078 /* Check remaining arguments for validity. */
1079 if (vma
->page
== NVKM_VMA_PAGE_NONE
&&
1080 vma
->refd
== NVKM_VMA_PAGE_NONE
) {
1081 /* Find the largest page size we can perform the mapping at. */
1082 const u32 debug
= vmm
->debug
;
1084 ret
= nvkm_vmm_map_choose(vmm
, vma
, argv
, argc
, map
);
1087 VMM_DEBUG(vmm
, "invalid at any page size");
1088 nvkm_vmm_map_choose(vmm
, vma
, argv
, argc
, map
);
1092 /* Page size of the VMA is already pre-determined. */
1093 if (vma
->refd
!= NVKM_VMA_PAGE_NONE
)
1094 map
->page
= &vmm
->func
->page
[vma
->refd
];
1096 map
->page
= &vmm
->func
->page
[vma
->page
];
1098 ret
= nvkm_vmm_map_valid(vmm
, vma
, argv
, argc
, map
);
1100 VMM_DEBUG(vmm
, "invalid %d\n", ret
);
1105 /* Deal with the 'offset' argument, and fetch the backend function. */
1106 map
->off
= map
->offset
;
1108 for (; map
->off
; map
->mem
= map
->mem
->next
) {
1109 u64 size
= (u64
)map
->mem
->length
<< NVKM_RAM_MM_SHIFT
;
1110 if (size
> map
->off
)
1114 func
= map
->page
->desc
->func
->mem
;
1117 for (; map
->off
; map
->sgl
= sg_next(map
->sgl
)) {
1118 u64 size
= sg_dma_len(map
->sgl
);
1119 if (size
> map
->off
)
1123 func
= map
->page
->desc
->func
->sgl
;
1125 map
->dma
+= map
->offset
>> PAGE_SHIFT
;
1126 map
->off
= map
->offset
& PAGE_MASK
;
1127 func
= map
->page
->desc
->func
->dma
;
1130 /* Perform the map. */
1131 if (vma
->refd
== NVKM_VMA_PAGE_NONE
) {
1132 ret
= nvkm_vmm_ptes_get_map(vmm
, map
->page
, vma
->addr
, vma
->size
, map
, func
);
1136 vma
->refd
= map
->page
- vmm
->func
->page
;
1138 nvkm_vmm_ptes_map(vmm
, map
->page
, vma
->addr
, vma
->size
, map
, func
);
1141 nvkm_memory_tags_put(vma
->memory
, vmm
->mmu
->subdev
.device
, &vma
->tags
);
1142 nvkm_memory_unref(&vma
->memory
);
1143 vma
->memory
= nvkm_memory_ref(map
->memory
);
1144 vma
->tags
= map
->tags
;
1149 nvkm_vmm_map(struct nvkm_vmm
*vmm
, struct nvkm_vma
*vma
, void *argv
, u32 argc
,
1150 struct nvkm_vmm_map
*map
)
1153 mutex_lock(&vmm
->mutex
);
1154 ret
= nvkm_vmm_map_locked(vmm
, vma
, argv
, argc
, map
);
1156 mutex_unlock(&vmm
->mutex
);
1161 nvkm_vmm_put_region(struct nvkm_vmm
*vmm
, struct nvkm_vma
*vma
)
1163 struct nvkm_vma
*prev
, *next
;
1165 if ((prev
= node(vma
, prev
)) && !prev
->used
) {
1166 rb_erase(&prev
->tree
, &vmm
->free
);
1167 list_del(&prev
->head
);
1168 vma
->addr
= prev
->addr
;
1169 vma
->size
+= prev
->size
;
1173 if ((next
= node(vma
, next
)) && !next
->used
) {
1174 rb_erase(&next
->tree
, &vmm
->free
);
1175 list_del(&next
->head
);
1176 vma
->size
+= next
->size
;
1180 nvkm_vmm_free_insert(vmm
, vma
);
1184 nvkm_vmm_put_locked(struct nvkm_vmm
*vmm
, struct nvkm_vma
*vma
)
1186 const struct nvkm_vmm_page
*page
= vmm
->func
->page
;
1187 struct nvkm_vma
*next
= vma
;
1191 if (vma
->mapref
|| !vma
->sparse
) {
1193 const bool map
= next
->memory
!= NULL
;
1194 const u8 refd
= next
->refd
;
1195 const u64 addr
= next
->addr
;
1196 u64 size
= next
->size
;
1198 /* Merge regions that are in the same state. */
1199 while ((next
= node(next
, next
)) && next
->part
&&
1200 (next
->memory
!= NULL
) == map
&&
1201 (next
->refd
== refd
))
1205 /* Region(s) are mapped, merge the unmap
1206 * and dereference into a single walk of
1209 nvkm_vmm_ptes_unmap_put(vmm
, &page
[refd
], addr
,
1212 if (refd
!= NVKM_VMA_PAGE_NONE
) {
1213 /* Drop allocation-time PTE references. */
1214 nvkm_vmm_ptes_put(vmm
, &page
[refd
], addr
, size
);
1216 } while (next
&& next
->part
);
1219 /* Merge any mapped regions that were split from the initial
1220 * address-space allocation back into the allocated VMA, and
1221 * release memory/compression resources.
1226 nvkm_vmm_unmap_region(vmm
, next
);
1227 } while ((next
= node(vma
, next
)) && next
->part
);
1229 if (vma
->sparse
&& !vma
->mapref
) {
1230 /* Sparse region that was allocated with a fixed page size,
1231 * meaning all relevant PTEs were referenced once when the
1232 * region was allocated, and remained that way, regardless
1233 * of whether memory was mapped into it afterwards.
1235 * The process of unmapping, unsparsing, and dereferencing
1236 * PTEs can be done in a single page tree walk.
1238 nvkm_vmm_ptes_sparse_put(vmm
, &page
[vma
->refd
], vma
->addr
, vma
->size
);
1241 /* Sparse region that wasn't allocated with a fixed page size,
1242 * PTE references were taken both at allocation time (to make
1243 * the GPU see the region as sparse), and when mapping memory
1246 * The latter was handled above, and the remaining references
1247 * are dealt with here.
1249 nvkm_vmm_ptes_sparse(vmm
, vma
->addr
, vma
->size
, false);
1252 /* Remove VMA from the list of allocated nodes. */
1253 rb_erase(&vma
->tree
, &vmm
->root
);
1255 /* Merge VMA back into the free list. */
1256 vma
->page
= NVKM_VMA_PAGE_NONE
;
1257 vma
->refd
= NVKM_VMA_PAGE_NONE
;
1260 nvkm_vmm_put_region(vmm
, vma
);
1264 nvkm_vmm_put(struct nvkm_vmm
*vmm
, struct nvkm_vma
**pvma
)
1266 struct nvkm_vma
*vma
= *pvma
;
1268 mutex_lock(&vmm
->mutex
);
1269 nvkm_vmm_put_locked(vmm
, vma
);
1270 mutex_unlock(&vmm
->mutex
);
1276 nvkm_vmm_get_locked(struct nvkm_vmm
*vmm
, bool getref
, bool mapref
, bool sparse
,
1277 u8 shift
, u8 align
, u64 size
, struct nvkm_vma
**pvma
)
1279 const struct nvkm_vmm_page
*page
= &vmm
->func
->page
[NVKM_VMA_PAGE_NONE
];
1280 struct rb_node
*node
= NULL
, *temp
;
1281 struct nvkm_vma
*vma
= NULL
, *tmp
;
1285 VMM_TRACE(vmm
, "getref %d mapref %d sparse %d "
1286 "shift: %d align: %d size: %016llx",
1287 getref
, mapref
, sparse
, shift
, align
, size
);
1289 /* Zero-sized, or lazily-allocated sparse VMAs, make no sense. */
1290 if (unlikely(!size
|| (!getref
&& !mapref
&& sparse
))) {
1291 VMM_DEBUG(vmm
, "args %016llx %d %d %d",
1292 size
, getref
, mapref
, sparse
);
1296 /* Tesla-class GPUs can only select page size per-PDE, which means
1297 * we're required to know the mapping granularity up-front to find
1298 * a suitable region of address-space.
1300 * The same goes if we're requesting up-front allocation of PTES.
1302 if (unlikely((getref
|| vmm
->func
->page_block
) && !shift
)) {
1303 VMM_DEBUG(vmm
, "page size required: %d %016llx",
1304 getref
, vmm
->func
->page_block
);
1308 /* If a specific page size was requested, determine its index and
1309 * make sure the requested size is a multiple of the page size.
1312 for (page
= vmm
->func
->page
; page
->shift
; page
++) {
1313 if (shift
== page
->shift
)
1317 if (!page
->shift
|| !IS_ALIGNED(size
, 1ULL << page
->shift
)) {
1318 VMM_DEBUG(vmm
, "page %d %016llx", shift
, size
);
1321 align
= max_t(u8
, align
, shift
);
1323 align
= max_t(u8
, align
, 12);
1326 /* Locate smallest block that can possibly satisfy the allocation. */
1327 temp
= vmm
->free
.rb_node
;
1329 struct nvkm_vma
*this = rb_entry(temp
, typeof(*this), tree
);
1330 if (this->size
< size
) {
1331 temp
= temp
->rb_right
;
1334 temp
= temp
->rb_left
;
1338 if (unlikely(!node
))
1341 /* Take into account alignment restrictions, trying larger blocks
1342 * in turn until we find a suitable free block.
1345 struct nvkm_vma
*this = rb_entry(node
, typeof(*this), tree
);
1346 struct nvkm_vma
*prev
= node(this, prev
);
1347 struct nvkm_vma
*next
= node(this, next
);
1348 const int p
= page
- vmm
->func
->page
;
1351 if (vmm
->func
->page_block
&& prev
&& prev
->page
!= p
)
1352 addr
= roundup(addr
, vmm
->func
->page_block
);
1353 addr
= ALIGN(addr
, 1ULL << align
);
1355 tail
= this->addr
+ this->size
;
1356 if (vmm
->func
->page_block
&& next
&& next
->page
!= p
)
1357 tail
= rounddown(tail
, vmm
->func
->page_block
);
1359 if (addr
<= tail
&& tail
- addr
>= size
) {
1360 rb_erase(&this->tree
, &vmm
->free
);
1364 } while ((node
= rb_next(node
)));
1369 /* If the VMA we found isn't already exactly the requested size,
1370 * it needs to be split, and the remaining free blocks returned.
1372 if (addr
!= vma
->addr
) {
1373 if (!(tmp
= nvkm_vma_tail(vma
, vma
->size
+ vma
->addr
- addr
))) {
1374 nvkm_vmm_put_region(vmm
, vma
);
1377 nvkm_vmm_free_insert(vmm
, vma
);
1381 if (size
!= vma
->size
) {
1382 if (!(tmp
= nvkm_vma_tail(vma
, vma
->size
- size
))) {
1383 nvkm_vmm_put_region(vmm
, vma
);
1386 nvkm_vmm_free_insert(vmm
, tmp
);
1389 /* Pre-allocate page tables and/or setup sparse mappings. */
1390 if (sparse
&& getref
)
1391 ret
= nvkm_vmm_ptes_sparse_get(vmm
, page
, vma
->addr
, vma
->size
);
1393 ret
= nvkm_vmm_ptes_sparse(vmm
, vma
->addr
, vma
->size
, true);
1395 ret
= nvkm_vmm_ptes_get(vmm
, page
, vma
->addr
, vma
->size
);
1399 nvkm_vmm_put_region(vmm
, vma
);
1403 vma
->mapref
= mapref
&& !getref
;
1404 vma
->sparse
= sparse
;
1405 vma
->page
= page
- vmm
->func
->page
;
1406 vma
->refd
= getref
? vma
->page
: NVKM_VMA_PAGE_NONE
;
1408 nvkm_vmm_node_insert(vmm
, vma
);
1414 nvkm_vmm_get(struct nvkm_vmm
*vmm
, u8 page
, u64 size
, struct nvkm_vma
**pvma
)
1417 mutex_lock(&vmm
->mutex
);
1418 ret
= nvkm_vmm_get_locked(vmm
, false, true, false, page
, 0, size
, pvma
);
1419 mutex_unlock(&vmm
->mutex
);
1424 nvkm_vmm_part(struct nvkm_vmm
*vmm
, struct nvkm_memory
*inst
)
1426 if (vmm
->func
->part
&& inst
) {
1427 mutex_lock(&vmm
->mutex
);
1428 vmm
->func
->part(vmm
, inst
);
1429 mutex_unlock(&vmm
->mutex
);
1434 nvkm_vmm_join(struct nvkm_vmm
*vmm
, struct nvkm_memory
*inst
)
1437 if (vmm
->func
->join
) {
1438 mutex_lock(&vmm
->mutex
);
1439 ret
= vmm
->func
->join(vmm
, inst
);
1440 mutex_unlock(&vmm
->mutex
);
1446 nvkm_vmm_boot_ptes(struct nvkm_vmm_iter
*it
, u32 ptei
, u32 ptes
)
1448 const struct nvkm_vmm_desc
*desc
= it
->desc
;
1449 const int type
= desc
->type
== SPT
;
1450 nvkm_memory_boot(it
->pt
[0]->pt
[type
]->memory
, it
->vmm
);
1455 nvkm_vmm_boot(struct nvkm_vmm
*vmm
)
1457 const struct nvkm_vmm_page
*page
= vmm
->func
->page
;
1458 const u64 limit
= vmm
->limit
- vmm
->start
;
1461 while (page
[1].shift
)
1464 ret
= nvkm_vmm_ptes_get(vmm
, page
, vmm
->start
, limit
);
1468 nvkm_vmm_iter(vmm
, page
, vmm
->start
, limit
, "bootstrap", false,
1469 nvkm_vmm_boot_ptes
, NULL
, NULL
, NULL
);
1470 vmm
->bootstrapped
= true;
1475 nvkm_vmm_del(struct kref
*kref
)
1477 struct nvkm_vmm
*vmm
= container_of(kref
, typeof(*vmm
), kref
);
1483 nvkm_vmm_unref(struct nvkm_vmm
**pvmm
)
1485 struct nvkm_vmm
*vmm
= *pvmm
;
1487 kref_put(&vmm
->kref
, nvkm_vmm_del
);
1493 nvkm_vmm_ref(struct nvkm_vmm
*vmm
)
1496 kref_get(&vmm
->kref
);
1501 nvkm_vmm_new(struct nvkm_device
*device
, u64 addr
, u64 size
, void *argv
,
1502 u32 argc
, struct lock_class_key
*key
, const char *name
,
1503 struct nvkm_vmm
**pvmm
)
1505 struct nvkm_mmu
*mmu
= device
->mmu
;
1506 struct nvkm_vmm
*vmm
= NULL
;
1508 ret
= mmu
->func
->vmm
.ctor(mmu
, addr
, size
, argv
, argc
, key
, name
, &vmm
);
1510 nvkm_vmm_unref(&vmm
);