2 * Copyright 2017 Red Hat Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 #define NVKM_VMM_LEVELS_MAX 5
26 nvkm_vmm_pt_del(struct nvkm_vmm_pt
**ppgt
)
28 struct nvkm_vmm_pt
*pgt
= *ppgt
;
37 static struct nvkm_vmm_pt
*
38 nvkm_vmm_pt_new(const struct nvkm_vmm_desc
*desc
, bool sparse
,
39 const struct nvkm_vmm_page
*page
)
41 const u32 pten
= 1 << desc
->bits
;
42 struct nvkm_vmm_pt
*pgt
;
45 if (desc
->type
> PGT
) {
46 if (desc
->type
== SPT
) {
47 const struct nvkm_vmm_desc
*pair
= page
[-1].desc
;
48 lpte
= pten
>> (desc
->bits
- pair
->bits
);
54 if (!(pgt
= kzalloc(sizeof(*pgt
) + lpte
, GFP_KERNEL
)))
56 pgt
->page
= page
? page
->shift
: 0;
59 if (desc
->type
== PGD
) {
60 pgt
->pde
= kvzalloc(sizeof(*pgt
->pde
) * pten
, GFP_KERNEL
);
70 struct nvkm_vmm_iter
{
71 const struct nvkm_vmm_page
*page
;
72 const struct nvkm_vmm_desc
*desc
;
76 u32 pte
[NVKM_VMM_LEVELS_MAX
];
77 struct nvkm_vmm_pt
*pt
[NVKM_VMM_LEVELS_MAX
];
81 #ifdef CONFIG_NOUVEAU_DEBUG_MMU
83 nvkm_vmm_desc_type(const struct nvkm_vmm_desc
*desc
)
86 case PGD
: return "PGD";
87 case PGT
: return "PGT";
88 case SPT
: return "SPT";
89 case LPT
: return "LPT";
96 nvkm_vmm_trace(struct nvkm_vmm_iter
*it
, char *buf
)
99 for (lvl
= it
->max
; lvl
>= 0; lvl
--) {
101 buf
+= sprintf(buf
, "%05x:", it
->pte
[lvl
]);
103 buf
+= sprintf(buf
, "xxxxx:");
107 #define TRA(i,f,a...) do { \
108 char _buf[NVKM_VMM_LEVELS_MAX * 7]; \
109 struct nvkm_vmm_iter *_it = (i); \
110 nvkm_vmm_trace(_it, _buf); \
111 VMM_TRACE(_it->vmm, "%s "f, _buf, ##a); \
114 #define TRA(i,f,a...)
118 nvkm_vmm_flush_mark(struct nvkm_vmm_iter
*it
)
120 it
->flush
= min(it
->flush
, it
->max
- it
->lvl
);
124 nvkm_vmm_flush(struct nvkm_vmm_iter
*it
)
126 if (it
->flush
!= NVKM_VMM_LEVELS_MAX
) {
127 if (it
->vmm
->func
->flush
) {
128 TRA(it
, "flush: %d", it
->flush
);
129 it
->vmm
->func
->flush(it
->vmm
, it
->flush
);
131 it
->flush
= NVKM_VMM_LEVELS_MAX
;
136 nvkm_vmm_unref_pdes(struct nvkm_vmm_iter
*it
)
138 const struct nvkm_vmm_desc
*desc
= it
->desc
;
139 const int type
= desc
[it
->lvl
].type
== SPT
;
140 struct nvkm_vmm_pt
*pgd
= it
->pt
[it
->lvl
+ 1];
141 struct nvkm_vmm_pt
*pgt
= it
->pt
[it
->lvl
];
142 struct nvkm_mmu_pt
*pt
= pgt
->pt
[type
];
143 struct nvkm_vmm
*vmm
= it
->vmm
;
144 u32 pdei
= it
->pte
[it
->lvl
+ 1];
146 /* Recurse up the tree, unreferencing/destroying unneeded PDs. */
148 if (--pgd
->refs
[0]) {
149 const struct nvkm_vmm_desc_func
*func
= desc
[it
->lvl
].func
;
150 /* PD has other valid PDEs, so we need a proper update. */
151 TRA(it
, "PDE unmap %s", nvkm_vmm_desc_type(&desc
[it
->lvl
- 1]));
152 pgt
->pt
[type
] = NULL
;
153 if (!pgt
->refs
[!type
]) {
154 /* PDE no longer required. */
157 func
->sparse(vmm
, pgd
->pt
[0], pdei
, 1);
158 pgd
->pde
[pdei
] = NVKM_VMM_PDE_SPARSE
;
160 func
->unmap(vmm
, pgd
->pt
[0], pdei
, 1);
161 pgd
->pde
[pdei
] = NULL
;
164 /* Special handling for Tesla-class GPUs,
165 * where there's no central PD, but each
166 * instance has its own embedded PD.
168 func
->pde(vmm
, pgd
, pdei
);
169 pgd
->pde
[pdei
] = NULL
;
172 /* PDE was pointing at dual-PTs and we're removing
173 * one of them, leaving the other in place.
175 func
->pde(vmm
, pgd
, pdei
);
178 /* GPU may have cached the PTs, flush before freeing. */
179 nvkm_vmm_flush_mark(it
);
182 /* PD has no valid PDEs left, so we can just destroy it. */
183 nvkm_vmm_unref_pdes(it
);
187 TRA(it
, "PDE free %s", nvkm_vmm_desc_type(&desc
[it
->lvl
- 1]));
188 nvkm_mmu_ptc_put(vmm
->mmu
, vmm
->bootstrapped
, &pt
);
189 if (!pgt
->refs
[!type
])
190 nvkm_vmm_pt_del(&pgt
);
195 nvkm_vmm_unref_sptes(struct nvkm_vmm_iter
*it
, struct nvkm_vmm_pt
*pgt
,
196 const struct nvkm_vmm_desc
*desc
, u32 ptei
, u32 ptes
)
198 const struct nvkm_vmm_desc
*pair
= it
->page
[-1].desc
;
199 const u32 sptb
= desc
->bits
- pair
->bits
;
200 const u32 sptn
= 1 << sptb
;
201 struct nvkm_vmm
*vmm
= it
->vmm
;
202 u32 spti
= ptei
& (sptn
- 1), lpti
, pteb
;
204 /* Determine how many SPTEs are being touched under each LPTE,
205 * and drop reference counts.
207 for (lpti
= ptei
>> sptb
; ptes
; spti
= 0, lpti
++) {
208 const u32 pten
= min(sptn
- spti
, ptes
);
209 pgt
->pte
[lpti
] -= pten
;
213 /* We're done here if there's no corresponding LPT. */
217 for (ptei
= pteb
= ptei
>> sptb
; ptei
< lpti
; pteb
= ptei
) {
218 /* Skip over any LPTEs that still have valid SPTEs. */
219 if (pgt
->pte
[pteb
] & NVKM_VMM_PTE_SPTES
) {
220 for (ptes
= 1, ptei
++; ptei
< lpti
; ptes
++, ptei
++) {
221 if (!(pgt
->pte
[ptei
] & NVKM_VMM_PTE_SPTES
))
227 /* As there's no more non-UNMAPPED SPTEs left in the range
228 * covered by a number of LPTEs, the LPTEs once again take
229 * control over their address range.
231 * Determine how many LPTEs need to transition state.
233 pgt
->pte
[ptei
] &= ~NVKM_VMM_PTE_VALID
;
234 for (ptes
= 1, ptei
++; ptei
< lpti
; ptes
++, ptei
++) {
235 if (pgt
->pte
[ptei
] & NVKM_VMM_PTE_SPTES
)
237 pgt
->pte
[ptei
] &= ~NVKM_VMM_PTE_VALID
;
240 if (pgt
->pte
[pteb
] & NVKM_VMM_PTE_SPARSE
) {
241 TRA(it
, "LPTE %05x: U -> S %d PTEs", pteb
, ptes
);
242 pair
->func
->sparse(vmm
, pgt
->pt
[0], pteb
, ptes
);
244 if (pair
->func
->invalid
) {
245 /* If the MMU supports it, restore the LPTE to the
246 * INVALID state to tell the MMU there is no point
247 * trying to fetch the corresponding SPTEs.
249 TRA(it
, "LPTE %05x: U -> I %d PTEs", pteb
, ptes
);
250 pair
->func
->invalid(vmm
, pgt
->pt
[0], pteb
, ptes
);
256 nvkm_vmm_unref_ptes(struct nvkm_vmm_iter
*it
, u32 ptei
, u32 ptes
)
258 const struct nvkm_vmm_desc
*desc
= it
->desc
;
259 const int type
= desc
->type
== SPT
;
260 struct nvkm_vmm_pt
*pgt
= it
->pt
[0];
262 /* Drop PTE references. */
263 pgt
->refs
[type
] -= ptes
;
265 /* Dual-PTs need special handling, unless PDE becoming invalid. */
266 if (desc
->type
== SPT
&& (pgt
->refs
[0] || pgt
->refs
[1]))
267 nvkm_vmm_unref_sptes(it
, pgt
, desc
, ptei
, ptes
);
269 /* PT no longer neeed? Destroy it. */
270 if (!pgt
->refs
[type
]) {
272 TRA(it
, "%s empty", nvkm_vmm_desc_type(desc
));
274 nvkm_vmm_unref_pdes(it
);
275 return false; /* PTE writes for unmap() not necessary. */
282 nvkm_vmm_ref_sptes(struct nvkm_vmm_iter
*it
, struct nvkm_vmm_pt
*pgt
,
283 const struct nvkm_vmm_desc
*desc
, u32 ptei
, u32 ptes
)
285 const struct nvkm_vmm_desc
*pair
= it
->page
[-1].desc
;
286 const u32 sptb
= desc
->bits
- pair
->bits
;
287 const u32 sptn
= 1 << sptb
;
288 struct nvkm_vmm
*vmm
= it
->vmm
;
289 u32 spti
= ptei
& (sptn
- 1), lpti
, pteb
;
291 /* Determine how many SPTEs are being touched under each LPTE,
292 * and increase reference counts.
294 for (lpti
= ptei
>> sptb
; ptes
; spti
= 0, lpti
++) {
295 const u32 pten
= min(sptn
- spti
, ptes
);
296 pgt
->pte
[lpti
] += pten
;
300 /* We're done here if there's no corresponding LPT. */
304 for (ptei
= pteb
= ptei
>> sptb
; ptei
< lpti
; pteb
= ptei
) {
305 /* Skip over any LPTEs that already have valid SPTEs. */
306 if (pgt
->pte
[pteb
] & NVKM_VMM_PTE_VALID
) {
307 for (ptes
= 1, ptei
++; ptei
< lpti
; ptes
++, ptei
++) {
308 if (!(pgt
->pte
[ptei
] & NVKM_VMM_PTE_VALID
))
314 /* As there are now non-UNMAPPED SPTEs in the range covered
315 * by a number of LPTEs, we need to transfer control of the
316 * address range to the SPTEs.
318 * Determine how many LPTEs need to transition state.
320 pgt
->pte
[ptei
] |= NVKM_VMM_PTE_VALID
;
321 for (ptes
= 1, ptei
++; ptei
< lpti
; ptes
++, ptei
++) {
322 if (pgt
->pte
[ptei
] & NVKM_VMM_PTE_VALID
)
324 pgt
->pte
[ptei
] |= NVKM_VMM_PTE_VALID
;
327 if (pgt
->pte
[pteb
] & NVKM_VMM_PTE_SPARSE
) {
328 const u32 spti
= pteb
* sptn
;
329 const u32 sptc
= ptes
* sptn
;
330 /* The entire LPTE is marked as sparse, we need
331 * to make sure that the SPTEs are too.
333 TRA(it
, "SPTE %05x: U -> S %d PTEs", spti
, sptc
);
334 desc
->func
->sparse(vmm
, pgt
->pt
[1], spti
, sptc
);
335 /* Sparse LPTEs prevent SPTEs from being accessed. */
336 TRA(it
, "LPTE %05x: S -> U %d PTEs", pteb
, ptes
);
337 pair
->func
->unmap(vmm
, pgt
->pt
[0], pteb
, ptes
);
339 if (pair
->func
->invalid
) {
340 /* MMU supports blocking SPTEs by marking an LPTE
341 * as INVALID. We need to reverse that here.
343 TRA(it
, "LPTE %05x: I -> U %d PTEs", pteb
, ptes
);
344 pair
->func
->unmap(vmm
, pgt
->pt
[0], pteb
, ptes
);
350 nvkm_vmm_ref_ptes(struct nvkm_vmm_iter
*it
, u32 ptei
, u32 ptes
)
352 const struct nvkm_vmm_desc
*desc
= it
->desc
;
353 const int type
= desc
->type
== SPT
;
354 struct nvkm_vmm_pt
*pgt
= it
->pt
[0];
356 /* Take PTE references. */
357 pgt
->refs
[type
] += ptes
;
359 /* Dual-PTs need special handling. */
360 if (desc
->type
== SPT
)
361 nvkm_vmm_ref_sptes(it
, pgt
, desc
, ptei
, ptes
);
367 nvkm_vmm_sparse_ptes(const struct nvkm_vmm_desc
*desc
,
368 struct nvkm_vmm_pt
*pgt
, u32 ptei
, u32 ptes
)
370 if (desc
->type
== PGD
) {
372 pgt
->pde
[ptei
++] = NVKM_VMM_PDE_SPARSE
;
374 if (desc
->type
== LPT
) {
375 memset(&pgt
->pte
[ptei
], NVKM_VMM_PTE_SPARSE
, ptes
);
380 nvkm_vmm_ref_hwpt(struct nvkm_vmm_iter
*it
, struct nvkm_vmm_pt
*pgd
, u32 pdei
)
382 const struct nvkm_vmm_desc
*desc
= &it
->desc
[it
->lvl
- 1];
383 const int type
= desc
->type
== SPT
;
384 struct nvkm_vmm_pt
*pgt
= pgd
->pde
[pdei
];
385 const bool zero
= !pgt
->sparse
&& !desc
->func
->invalid
;
386 struct nvkm_vmm
*vmm
= it
->vmm
;
387 struct nvkm_mmu
*mmu
= vmm
->mmu
;
388 struct nvkm_mmu_pt
*pt
;
389 u32 pten
= 1 << desc
->bits
;
390 u32 pteb
, ptei
, ptes
;
391 u32 size
= desc
->size
* pten
;
395 pgt
->pt
[type
] = nvkm_mmu_ptc_get(mmu
, size
, desc
->align
, zero
);
396 if (!pgt
->pt
[type
]) {
398 nvkm_vmm_unref_pdes(it
);
407 if (desc
->type
== LPT
&& pgt
->refs
[1]) {
408 /* SPT already exists covering the same range as this LPT,
409 * which means we need to be careful that any LPTEs which
410 * overlap valid SPTEs are unmapped as opposed to invalid
411 * or sparse, which would prevent the MMU from looking at
412 * the SPTEs on some GPUs.
414 for (ptei
= pteb
= 0; ptei
< pten
; pteb
= ptei
) {
415 bool spte
= pgt
->pte
[ptei
] & NVKM_VMM_PTE_SPTES
;
416 for (ptes
= 1, ptei
++; ptei
< pten
; ptes
++, ptei
++) {
417 bool next
= pgt
->pte
[ptei
] & NVKM_VMM_PTE_SPTES
;
424 desc
->func
->sparse(vmm
, pt
, pteb
, ptes
);
426 desc
->func
->invalid(vmm
, pt
, pteb
, ptes
);
427 memset(&pgt
->pte
[pteb
], 0x00, ptes
);
429 desc
->func
->unmap(vmm
, pt
, pteb
, ptes
);
431 pgt
->pte
[pteb
++] |= NVKM_VMM_PTE_VALID
;
436 nvkm_vmm_sparse_ptes(desc
, pgt
, 0, pten
);
437 desc
->func
->sparse(vmm
, pt
, 0, pten
);
439 desc
->func
->invalid(vmm
, pt
, 0, pten
);
444 TRA(it
, "PDE write %s", nvkm_vmm_desc_type(desc
));
445 it
->desc
[it
->lvl
].func
->pde(it
->vmm
, pgd
, pdei
);
446 nvkm_vmm_flush_mark(it
);
451 nvkm_vmm_ref_swpt(struct nvkm_vmm_iter
*it
, struct nvkm_vmm_pt
*pgd
, u32 pdei
)
453 const struct nvkm_vmm_desc
*desc
= &it
->desc
[it
->lvl
- 1];
454 struct nvkm_vmm_pt
*pgt
= pgd
->pde
[pdei
];
456 pgt
= nvkm_vmm_pt_new(desc
, NVKM_VMM_PDE_SPARSED(pgt
), it
->page
);
459 nvkm_vmm_unref_pdes(it
);
463 pgd
->pde
[pdei
] = pgt
;
468 nvkm_vmm_iter(struct nvkm_vmm
*vmm
, const struct nvkm_vmm_page
*page
,
469 u64 addr
, u64 size
, const char *name
, bool ref
,
470 bool (*REF_PTES
)(struct nvkm_vmm_iter
*, u32
, u32
),
471 nvkm_vmm_pte_func MAP_PTES
, struct nvkm_vmm_map
*map
,
472 nvkm_vmm_pxe_func CLR_PTES
)
474 const struct nvkm_vmm_desc
*desc
= page
->desc
;
475 struct nvkm_vmm_iter it
;
476 u64 bits
= addr
>> page
->shift
;
481 it
.cnt
= size
>> page
->shift
;
482 it
.flush
= NVKM_VMM_LEVELS_MAX
;
484 /* Deconstruct address into PTE indices for each mapping level. */
485 for (it
.lvl
= 0; desc
[it
.lvl
].bits
; it
.lvl
++) {
486 it
.pte
[it
.lvl
] = bits
& ((1 << desc
[it
.lvl
].bits
) - 1);
487 bits
>>= desc
[it
.lvl
].bits
;
490 it
.pt
[it
.max
] = vmm
->pd
;
493 TRA(&it
, "%s: %016llx %016llx %d %lld PTEs", name
,
494 addr
, size
, page
->shift
, it
.cnt
);
497 /* Depth-first traversal of page tables. */
499 struct nvkm_vmm_pt
*pgt
= it
.pt
[it
.lvl
];
500 const int type
= desc
->type
== SPT
;
501 const u32 pten
= 1 << desc
->bits
;
502 const u32 ptei
= it
.pte
[0];
503 const u32 ptes
= min_t(u64
, it
.cnt
, pten
- ptei
);
505 /* Walk down the tree, finding page tables for each level. */
506 for (; it
.lvl
; it
.lvl
--) {
507 const u32 pdei
= it
.pte
[it
.lvl
];
508 struct nvkm_vmm_pt
*pgd
= pgt
;
511 if (ref
&& NVKM_VMM_PDE_INVALID(pgd
->pde
[pdei
])) {
512 if (!nvkm_vmm_ref_swpt(&it
, pgd
, pdei
))
515 it
.pt
[it
.lvl
- 1] = pgt
= pgd
->pde
[pdei
];
519 * This is a separate step from above due to GF100 and
520 * newer having dual page tables at some levels, which
521 * are refcounted independently.
523 if (ref
&& !pgt
->refs
[desc
[it
.lvl
- 1].type
== SPT
]) {
524 if (!nvkm_vmm_ref_hwpt(&it
, pgd
, pdei
))
529 /* Handle PTE updates. */
530 if (!REF_PTES
|| REF_PTES(&it
, ptei
, ptes
)) {
531 struct nvkm_mmu_pt
*pt
= pgt
->pt
[type
];
532 if (MAP_PTES
|| CLR_PTES
) {
534 MAP_PTES(vmm
, pt
, ptei
, ptes
, map
);
536 CLR_PTES(vmm
, pt
, ptei
, ptes
);
537 nvkm_vmm_flush_mark(&it
);
541 /* Walk back up the tree to the next position. */
542 it
.pte
[it
.lvl
] += ptes
;
545 while (it
.pte
[it
.lvl
] == (1 << desc
[it
.lvl
].bits
)) {
546 it
.pte
[it
.lvl
++] = 0;
556 /* Reconstruct the failure address so the caller is able to
557 * reverse any partially completed operations.
559 addr
= it
.pte
[it
.max
--];
561 addr
= addr
<< desc
[it
.max
].bits
;
562 addr
|= it
.pte
[it
.max
];
565 return addr
<< page
->shift
;
569 nvkm_vmm_ptes_unmap(struct nvkm_vmm
*vmm
, const struct nvkm_vmm_page
*page
,
570 u64 addr
, u64 size
, bool sparse
)
572 const struct nvkm_vmm_desc_func
*func
= page
->desc
->func
;
573 nvkm_vmm_iter(vmm
, page
, addr
, size
, "unmap", false, NULL
, NULL
, NULL
,
574 sparse
? func
->sparse
: func
->invalid
? func
->invalid
:
579 nvkm_vmm_ptes_map(struct nvkm_vmm
*vmm
, const struct nvkm_vmm_page
*page
,
580 u64 addr
, u64 size
, struct nvkm_vmm_map
*map
,
581 nvkm_vmm_pte_func func
)
583 nvkm_vmm_iter(vmm
, page
, addr
, size
, "map", false,
584 NULL
, func
, map
, NULL
);
588 nvkm_vmm_ptes_put(struct nvkm_vmm
*vmm
, const struct nvkm_vmm_page
*page
,
591 nvkm_vmm_iter(vmm
, page
, addr
, size
, "unref", false,
592 nvkm_vmm_unref_ptes
, NULL
, NULL
, NULL
);
596 nvkm_vmm_ptes_get(struct nvkm_vmm
*vmm
, const struct nvkm_vmm_page
*page
,
599 u64 fail
= nvkm_vmm_iter(vmm
, page
, addr
, size
, "ref", true,
600 nvkm_vmm_ref_ptes
, NULL
, NULL
, NULL
);
603 nvkm_vmm_ptes_put(vmm
, page
, addr
, fail
- addr
);
610 nvkm_vmm_dtor(struct nvkm_vmm
*vmm
)
612 if (vmm
->bootstrapped
) {
613 const struct nvkm_vmm_page
*page
= vmm
->func
->page
;
614 const u64 limit
= vmm
->limit
- vmm
->start
;
616 while (page
[1].shift
)
619 nvkm_mmu_ptc_dump(vmm
->mmu
);
620 nvkm_vmm_ptes_put(vmm
, page
, vmm
->start
, limit
);
624 dma_free_coherent(vmm
->mmu
->subdev
.device
->dev
, 16 * 1024,
625 vmm
->nullp
, vmm
->null
);
629 nvkm_mmu_ptc_put(vmm
->mmu
, true, &vmm
->pd
->pt
[0]);
630 nvkm_vmm_pt_del(&vmm
->pd
);
635 nvkm_vmm_ctor(const struct nvkm_vmm_func
*func
, struct nvkm_mmu
*mmu
,
636 u32 pd_header
, u64 addr
, u64 size
, struct lock_class_key
*key
,
637 const char *name
, struct nvkm_vmm
*vmm
)
639 static struct lock_class_key _key
;
640 const struct nvkm_vmm_page
*page
= func
->page
;
641 const struct nvkm_vmm_desc
*desc
;
642 int levels
, bits
= 0;
647 vmm
->debug
= mmu
->subdev
.debug
;
648 kref_init(&vmm
->kref
);
650 __mutex_init(&vmm
->mutex
, "&vmm->mutex", key
? key
: &_key
);
652 /* Locate the smallest page size supported by the backend, it will
653 * have the the deepest nesting of page tables.
655 while (page
[1].shift
)
658 /* Locate the structure that describes the layout of the top-level
659 * page table, and determine the number of valid bits in a virtual
662 for (levels
= 0, desc
= page
->desc
; desc
->bits
; desc
++, levels
++)
667 if (WARN_ON(levels
> NVKM_VMM_LEVELS_MAX
))
671 vmm
->limit
= size
? (addr
+ size
) : (1ULL << bits
);
672 if (vmm
->start
> vmm
->limit
|| vmm
->limit
> (1ULL << bits
))
675 /* Allocate top-level page table. */
676 vmm
->pd
= nvkm_vmm_pt_new(desc
, false, NULL
);
679 vmm
->pd
->refs
[0] = 1;
680 INIT_LIST_HEAD(&vmm
->join
);
682 /* ... and the GPU storage for it, except on Tesla-class GPUs that
683 * have the PD embedded in the instance structure.
686 const u32 size
= pd_header
+ desc
->size
* (1 << desc
->bits
);
687 vmm
->pd
->pt
[0] = nvkm_mmu_ptc_get(mmu
, size
, desc
->align
, true);
696 nvkm_vmm_new_(const struct nvkm_vmm_func
*func
, struct nvkm_mmu
*mmu
,
697 u32 hdr
, u64 addr
, u64 size
, struct lock_class_key
*key
,
698 const char *name
, struct nvkm_vmm
**pvmm
)
700 if (!(*pvmm
= kzalloc(sizeof(**pvmm
), GFP_KERNEL
)))
702 return nvkm_vmm_ctor(func
, mmu
, hdr
, addr
, size
, key
, name
, *pvmm
);
706 nvkm_vmm_boot_ptes(struct nvkm_vmm_iter
*it
, u32 ptei
, u32 ptes
)
708 const struct nvkm_vmm_desc
*desc
= it
->desc
;
709 const int type
= desc
->type
== SPT
;
710 nvkm_memory_boot(it
->pt
[0]->pt
[type
]->memory
, it
->vmm
);
715 nvkm_vmm_boot(struct nvkm_vmm
*vmm
)
717 const struct nvkm_vmm_page
*page
= vmm
->func
->page
;
718 const u64 limit
= vmm
->limit
- vmm
->start
;
721 while (page
[1].shift
)
724 ret
= nvkm_vmm_ptes_get(vmm
, page
, vmm
->start
, limit
);
728 nvkm_vmm_iter(vmm
, page
, vmm
->start
, limit
, "bootstrap", false,
729 nvkm_vmm_boot_ptes
, NULL
, NULL
, NULL
);
730 vmm
->bootstrapped
= true;