]>
Commit | Line | Data |
---|---|---|
a11c3198 BS |
1 | /* |
2 | * Copyright 2010 Red Hat Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | * Authors: Ben Skeggs | |
23 | */ | |
eea5cf0f | 24 | #include "ummu.h" |
806a7335 | 25 | #include "vmm.h" |
a11c3198 | 26 | |
51645eb7 | 27 | #include <subdev/bar.h> |
c9582455 | 28 | #include <subdev/fb.h> |
a11c3198 | 29 | |
fd542a3e | 30 | #include <nvif/if500d.h> |
b77791da | 31 | #include <nvif/if900d.h> |
fd542a3e | 32 | |
f1280394 BS |
33 | struct nvkm_mmu_ptp { |
34 | struct nvkm_mmu_pt *pt; | |
35 | struct list_head head; | |
36 | u8 shift; | |
37 | u16 mask; | |
38 | u16 free; | |
39 | }; | |
40 | ||
41 | static void | |
42 | nvkm_mmu_ptp_put(struct nvkm_mmu *mmu, bool force, struct nvkm_mmu_pt *pt) | |
43 | { | |
44 | const int slot = pt->base >> pt->ptp->shift; | |
45 | struct nvkm_mmu_ptp *ptp = pt->ptp; | |
46 | ||
47 | /* If there were no free slots in the parent allocation before, | |
48 | * there will be now, so return PTP to the cache. | |
49 | */ | |
50 | if (!ptp->free) | |
51 | list_add(&ptp->head, &mmu->ptp.list); | |
52 | ptp->free |= BIT(slot); | |
53 | ||
54 | /* If there's no more sub-allocations, destroy PTP. */ | |
55 | if (ptp->free == ptp->mask) { | |
56 | nvkm_mmu_ptc_put(mmu, force, &ptp->pt); | |
57 | list_del(&ptp->head); | |
58 | kfree(ptp); | |
59 | } | |
60 | ||
61 | kfree(pt); | |
62 | } | |
63 | ||
64 | struct nvkm_mmu_pt * | |
65 | nvkm_mmu_ptp_get(struct nvkm_mmu *mmu, u32 size, bool zero) | |
66 | { | |
67 | struct nvkm_mmu_pt *pt; | |
68 | struct nvkm_mmu_ptp *ptp; | |
69 | int slot; | |
70 | ||
71 | if (!(pt = kzalloc(sizeof(*pt), GFP_KERNEL))) | |
72 | return NULL; | |
73 | ||
74 | ptp = list_first_entry_or_null(&mmu->ptp.list, typeof(*ptp), head); | |
75 | if (!ptp) { | |
76 | /* Need to allocate a new parent to sub-allocate from. */ | |
77 | if (!(ptp = kmalloc(sizeof(*ptp), GFP_KERNEL))) { | |
78 | kfree(pt); | |
79 | return NULL; | |
80 | } | |
81 | ||
82 | ptp->pt = nvkm_mmu_ptc_get(mmu, 0x1000, 0x1000, false); | |
83 | if (!ptp->pt) { | |
84 | kfree(ptp); | |
85 | kfree(pt); | |
86 | return NULL; | |
87 | } | |
88 | ||
89 | ptp->shift = order_base_2(size); | |
90 | slot = nvkm_memory_size(ptp->pt->memory) >> ptp->shift; | |
91 | ptp->mask = (1 << slot) - 1; | |
92 | ptp->free = ptp->mask; | |
93 | list_add(&ptp->head, &mmu->ptp.list); | |
94 | } | |
95 | pt->ptp = ptp; | |
96 | pt->sub = true; | |
97 | ||
98 | /* Sub-allocate from parent object, removing PTP from cache | |
99 | * if there's no more free slots left. | |
100 | */ | |
101 | slot = __ffs(ptp->free); | |
102 | ptp->free &= ~BIT(slot); | |
103 | if (!ptp->free) | |
104 | list_del(&ptp->head); | |
105 | ||
106 | pt->memory = pt->ptp->pt->memory; | |
107 | pt->base = slot << ptp->shift; | |
108 | pt->addr = pt->ptp->pt->addr + pt->base; | |
109 | return pt; | |
110 | } | |
111 | ||
9a45ddaa BS |
112 | struct nvkm_mmu_ptc { |
113 | struct list_head head; | |
114 | struct list_head item; | |
115 | u32 size; | |
116 | u32 refs; | |
117 | }; | |
118 | ||
119 | static inline struct nvkm_mmu_ptc * | |
120 | nvkm_mmu_ptc_find(struct nvkm_mmu *mmu, u32 size) | |
121 | { | |
122 | struct nvkm_mmu_ptc *ptc; | |
123 | ||
124 | list_for_each_entry(ptc, &mmu->ptc.list, head) { | |
125 | if (ptc->size == size) | |
126 | return ptc; | |
127 | } | |
128 | ||
129 | ptc = kmalloc(sizeof(*ptc), GFP_KERNEL); | |
130 | if (ptc) { | |
131 | INIT_LIST_HEAD(&ptc->item); | |
132 | ptc->size = size; | |
133 | ptc->refs = 0; | |
134 | list_add(&ptc->head, &mmu->ptc.list); | |
135 | } | |
136 | ||
137 | return ptc; | |
138 | } | |
139 | ||
140 | void | |
141 | nvkm_mmu_ptc_put(struct nvkm_mmu *mmu, bool force, struct nvkm_mmu_pt **ppt) | |
142 | { | |
143 | struct nvkm_mmu_pt *pt = *ppt; | |
144 | if (pt) { | |
f1280394 BS |
145 | /* Handle sub-allocated page tables. */ |
146 | if (pt->sub) { | |
147 | mutex_lock(&mmu->ptp.mutex); | |
148 | nvkm_mmu_ptp_put(mmu, force, pt); | |
149 | mutex_unlock(&mmu->ptp.mutex); | |
150 | return; | |
151 | } | |
152 | ||
9a45ddaa BS |
153 | /* Either cache or free the object. */ |
154 | mutex_lock(&mmu->ptc.mutex); | |
155 | if (pt->ptc->refs < 8 /* Heuristic. */ && !force) { | |
156 | list_add_tail(&pt->head, &pt->ptc->item); | |
157 | pt->ptc->refs++; | |
158 | } else { | |
159 | nvkm_memory_unref(&pt->memory); | |
160 | kfree(pt); | |
161 | } | |
162 | mutex_unlock(&mmu->ptc.mutex); | |
163 | } | |
164 | } | |
165 | ||
166 | struct nvkm_mmu_pt * | |
167 | nvkm_mmu_ptc_get(struct nvkm_mmu *mmu, u32 size, u32 align, bool zero) | |
168 | { | |
169 | struct nvkm_mmu_ptc *ptc; | |
170 | struct nvkm_mmu_pt *pt; | |
171 | int ret; | |
172 | ||
f1280394 BS |
173 | /* Sub-allocated page table (ie. GP100 LPT). */ |
174 | if (align < 0x1000) { | |
175 | mutex_lock(&mmu->ptp.mutex); | |
176 | pt = nvkm_mmu_ptp_get(mmu, align, zero); | |
177 | mutex_unlock(&mmu->ptp.mutex); | |
178 | return pt; | |
179 | } | |
180 | ||
9a45ddaa BS |
181 | /* Lookup cache for this page table size. */ |
182 | mutex_lock(&mmu->ptc.mutex); | |
183 | ptc = nvkm_mmu_ptc_find(mmu, size); | |
184 | if (!ptc) { | |
185 | mutex_unlock(&mmu->ptc.mutex); | |
186 | return NULL; | |
187 | } | |
188 | ||
189 | /* If there's a free PT in the cache, reuse it. */ | |
190 | pt = list_first_entry_or_null(&ptc->item, typeof(*pt), head); | |
191 | if (pt) { | |
192 | if (zero) | |
193 | nvkm_fo64(pt->memory, 0, 0, size >> 3); | |
194 | list_del(&pt->head); | |
195 | ptc->refs--; | |
196 | mutex_unlock(&mmu->ptc.mutex); | |
197 | return pt; | |
198 | } | |
199 | mutex_unlock(&mmu->ptc.mutex); | |
200 | ||
201 | /* No such luck, we need to allocate. */ | |
202 | if (!(pt = kmalloc(sizeof(*pt), GFP_KERNEL))) | |
203 | return NULL; | |
204 | pt->ptc = ptc; | |
f1280394 | 205 | pt->sub = false; |
9a45ddaa BS |
206 | |
207 | ret = nvkm_memory_new(mmu->subdev.device, NVKM_MEM_TARGET_INST, | |
208 | size, align, zero, &pt->memory); | |
209 | if (ret) { | |
210 | kfree(pt); | |
211 | return NULL; | |
212 | } | |
213 | ||
214 | pt->base = 0; | |
215 | pt->addr = nvkm_memory_addr(pt->memory); | |
216 | return pt; | |
217 | } | |
218 | ||
219 | void | |
220 | nvkm_mmu_ptc_dump(struct nvkm_mmu *mmu) | |
221 | { | |
222 | struct nvkm_mmu_ptc *ptc; | |
223 | list_for_each_entry(ptc, &mmu->ptc.list, head) { | |
224 | struct nvkm_mmu_pt *pt, *tt; | |
225 | list_for_each_entry_safe(pt, tt, &ptc->item, head) { | |
226 | nvkm_memory_unref(&pt->memory); | |
227 | list_del(&pt->head); | |
228 | kfree(pt); | |
229 | } | |
230 | } | |
231 | } | |
232 | ||
233 | static void | |
234 | nvkm_mmu_ptc_fini(struct nvkm_mmu *mmu) | |
235 | { | |
236 | struct nvkm_mmu_ptc *ptc, *ptct; | |
237 | ||
238 | list_for_each_entry_safe(ptc, ptct, &mmu->ptc.list, head) { | |
239 | WARN_ON(!list_empty(&ptc->item)); | |
240 | list_del(&ptc->head); | |
241 | kfree(ptc); | |
242 | } | |
243 | } | |
244 | ||
245 | static void | |
246 | nvkm_mmu_ptc_init(struct nvkm_mmu *mmu) | |
247 | { | |
248 | mutex_init(&mmu->ptc.mutex); | |
249 | INIT_LIST_HEAD(&mmu->ptc.list); | |
f1280394 BS |
250 | mutex_init(&mmu->ptp.mutex); |
251 | INIT_LIST_HEAD(&mmu->ptp.list); | |
9a45ddaa BS |
252 | } |
253 | ||
51645eb7 BS |
254 | static void |
255 | nvkm_mmu_type(struct nvkm_mmu *mmu, int heap, u8 type) | |
256 | { | |
257 | if (heap >= 0 && !WARN_ON(mmu->type_nr == ARRAY_SIZE(mmu->type))) { | |
258 | mmu->type[mmu->type_nr].type = type | mmu->heap[heap].type; | |
259 | mmu->type[mmu->type_nr].heap = heap; | |
260 | mmu->type_nr++; | |
261 | } | |
262 | } | |
263 | ||
264 | static int | |
265 | nvkm_mmu_heap(struct nvkm_mmu *mmu, u8 type, u64 size) | |
266 | { | |
267 | if (size) { | |
268 | if (!WARN_ON(mmu->heap_nr == ARRAY_SIZE(mmu->heap))) { | |
269 | mmu->heap[mmu->heap_nr].type = type; | |
270 | mmu->heap[mmu->heap_nr].size = size; | |
271 | return mmu->heap_nr++; | |
272 | } | |
273 | } | |
274 | return -EINVAL; | |
275 | } | |
276 | ||
277 | static void | |
278 | nvkm_mmu_host(struct nvkm_mmu *mmu) | |
279 | { | |
280 | struct nvkm_device *device = mmu->subdev.device; | |
281 | u8 type = NVKM_MEM_KIND * !!mmu->func->kind_sys; | |
282 | int heap; | |
283 | ||
284 | /* Non-mappable system memory. */ | |
285 | heap = nvkm_mmu_heap(mmu, NVKM_MEM_HOST, ~0ULL); | |
286 | nvkm_mmu_type(mmu, heap, type); | |
287 | ||
288 | /* Non-coherent, cached, system memory. | |
289 | * | |
290 | * Block-linear mappings of system memory must be done through | |
291 | * BAR1, and cannot be supported on systems where we're unable | |
292 | * to map BAR1 with write-combining. | |
293 | */ | |
294 | type |= NVKM_MEM_MAPPABLE; | |
295 | if (!device->bar || device->bar->iomap_uncached) | |
296 | nvkm_mmu_type(mmu, heap, type & ~NVKM_MEM_KIND); | |
297 | else | |
298 | nvkm_mmu_type(mmu, heap, type); | |
299 | ||
300 | /* Coherent, cached, system memory. | |
301 | * | |
302 | * Unsupported on systems that aren't able to support snooped | |
303 | * mappings, and also for block-linear mappings which must be | |
304 | * done through BAR1. | |
305 | */ | |
306 | type |= NVKM_MEM_COHERENT; | |
307 | if (device->func->cpu_coherent) | |
308 | nvkm_mmu_type(mmu, heap, type & ~NVKM_MEM_KIND); | |
309 | ||
310 | /* Uncached system memory. */ | |
311 | nvkm_mmu_type(mmu, heap, type |= NVKM_MEM_UNCACHED); | |
312 | } | |
313 | ||
314 | static void | |
315 | nvkm_mmu_vram(struct nvkm_mmu *mmu) | |
316 | { | |
317 | struct nvkm_device *device = mmu->subdev.device; | |
318 | struct nvkm_mm *mm = &device->fb->ram->vram; | |
319 | const u32 sizeN = nvkm_mm_heap_size(mm, NVKM_RAM_MM_NORMAL); | |
320 | const u32 sizeU = nvkm_mm_heap_size(mm, NVKM_RAM_MM_NOMAP); | |
321 | const u32 sizeM = nvkm_mm_heap_size(mm, NVKM_RAM_MM_MIXED); | |
322 | u8 type = NVKM_MEM_KIND * !!mmu->func->kind; | |
323 | u8 heap = NVKM_MEM_VRAM; | |
324 | int heapM, heapN, heapU; | |
325 | ||
326 | /* Mixed-memory doesn't support compression or display. */ | |
327 | heapM = nvkm_mmu_heap(mmu, heap, sizeM << NVKM_RAM_MM_SHIFT); | |
328 | ||
329 | heap |= NVKM_MEM_COMP; | |
330 | heap |= NVKM_MEM_DISP; | |
331 | heapN = nvkm_mmu_heap(mmu, heap, sizeN << NVKM_RAM_MM_SHIFT); | |
332 | heapU = nvkm_mmu_heap(mmu, heap, sizeU << NVKM_RAM_MM_SHIFT); | |
333 | ||
334 | /* Add non-mappable VRAM types first so that they're preferred | |
335 | * over anything else. Mixed-memory will be slower than other | |
336 | * heaps, it's prioritised last. | |
337 | */ | |
338 | nvkm_mmu_type(mmu, heapU, type); | |
339 | nvkm_mmu_type(mmu, heapN, type); | |
340 | nvkm_mmu_type(mmu, heapM, type); | |
341 | ||
342 | /* Add host memory types next, under the assumption that users | |
343 | * wanting mappable memory want to use them as staging buffers | |
344 | * or the like. | |
345 | */ | |
346 | nvkm_mmu_host(mmu); | |
347 | ||
348 | /* Mappable VRAM types go last, as they're basically the worst | |
349 | * possible type to ask for unless there's no other choice. | |
350 | */ | |
351 | if (device->bar) { | |
352 | /* Write-combined BAR1 access. */ | |
353 | type |= NVKM_MEM_MAPPABLE; | |
354 | if (!device->bar->iomap_uncached) { | |
355 | nvkm_mmu_type(mmu, heapN, type); | |
356 | nvkm_mmu_type(mmu, heapM, type); | |
357 | } | |
358 | ||
359 | /* Uncached BAR1 access. */ | |
360 | type |= NVKM_MEM_COHERENT; | |
361 | type |= NVKM_MEM_UNCACHED; | |
362 | nvkm_mmu_type(mmu, heapN, type); | |
363 | nvkm_mmu_type(mmu, heapM, type); | |
364 | } | |
365 | } | |
366 | ||
c9582455 BS |
367 | static int |
368 | nvkm_mmu_oneinit(struct nvkm_subdev *subdev) | |
369 | { | |
370 | struct nvkm_mmu *mmu = nvkm_mmu(subdev); | |
806a7335 | 371 | |
51645eb7 BS |
372 | /* Determine available memory types. */ |
373 | if (mmu->subdev.device->fb && mmu->subdev.device->fb->ram) | |
374 | nvkm_mmu_vram(mmu); | |
375 | else | |
376 | nvkm_mmu_host(mmu); | |
377 | ||
806a7335 | 378 | if (mmu->func->vmm.global) { |
f9463a4b BS |
379 | int ret = nvkm_vmm_new(subdev->device, 0, 0, NULL, 0, NULL, |
380 | "gart", &mmu->vmm); | |
806a7335 BS |
381 | if (ret) |
382 | return ret; | |
383 | } | |
384 | ||
c9582455 BS |
385 | return 0; |
386 | } | |
387 | ||
388 | static int | |
389 | nvkm_mmu_init(struct nvkm_subdev *subdev) | |
390 | { | |
391 | struct nvkm_mmu *mmu = nvkm_mmu(subdev); | |
392 | if (mmu->func->init) | |
393 | mmu->func->init(mmu); | |
394 | return 0; | |
395 | } | |
396 | ||
397 | static void * | |
398 | nvkm_mmu_dtor(struct nvkm_subdev *subdev) | |
399 | { | |
400 | struct nvkm_mmu *mmu = nvkm_mmu(subdev); | |
9a45ddaa | 401 | |
f9463a4b | 402 | nvkm_vmm_unref(&mmu->vmm); |
9a45ddaa BS |
403 | |
404 | nvkm_mmu_ptc_fini(mmu); | |
03b0ba7b | 405 | return mmu; |
c9582455 BS |
406 | } |
407 | ||
408 | static const struct nvkm_subdev_func | |
409 | nvkm_mmu = { | |
410 | .dtor = nvkm_mmu_dtor, | |
411 | .oneinit = nvkm_mmu_oneinit, | |
412 | .init = nvkm_mmu_init, | |
413 | }; | |
414 | ||
415 | void | |
416 | nvkm_mmu_ctor(const struct nvkm_mmu_func *func, struct nvkm_device *device, | |
417 | int index, struct nvkm_mmu *mmu) | |
418 | { | |
56d06fa2 | 419 | nvkm_subdev_ctor(&nvkm_mmu, device, index, &mmu->subdev); |
c9582455 | 420 | mmu->func = func; |
c9582455 | 421 | mmu->dma_bits = func->dma_bits; |
9a45ddaa | 422 | nvkm_mmu_ptc_init(mmu); |
eea5cf0f BS |
423 | mmu->user.ctor = nvkm_ummu_new; |
424 | mmu->user.base = func->mmu.user; | |
c9582455 BS |
425 | } |
426 | ||
427 | int | |
428 | nvkm_mmu_new_(const struct nvkm_mmu_func *func, struct nvkm_device *device, | |
429 | int index, struct nvkm_mmu **pmmu) | |
430 | { | |
431 | if (!(*pmmu = kzalloc(sizeof(**pmmu), GFP_KERNEL))) | |
432 | return -ENOMEM; | |
433 | nvkm_mmu_ctor(func, device, index, *pmmu); | |
434 | return 0; | |
435 | } |