]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/gpu/drm/nouveau/nouveau_bo.c
projects / mirror_ubuntu-bionic-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
drm/nouveau: check kind validity against mmu object
[mirror_ubuntu-bionic-kernel.git] / drivers / gpu / drm / nouveau / nouveau_bo.c
1 /*
2 * Copyright 2007 Dave Airlied
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 */
24 /*
25 * Authors: Dave Airlied <airlied@linux.ie>
26 * Ben Skeggs <darktama@iinet.net.au>
27 * Jeremy Kolb <jkolb@brandeis.edu>
28 */
29
30 #include <linux/dma-mapping.h>
31 #include <linux/swiotlb.h>
32
33 #include "nouveau_drv.h"
34 #include "nouveau_dma.h"
35 #include "nouveau_fence.h"
36
37 #include "nouveau_bo.h"
38 #include "nouveau_ttm.h"
39 #include "nouveau_gem.h"
40 #include "nouveau_mem.h"
41 #include "nouveau_vmm.h"
42
43 /*
44 * NV10-NV40 tiling helpers
45 */
46
47 static void
48 nv10_bo_update_tile_region(struct drm_device *dev, struct nouveau_drm_tile *reg,
49 u32 addr, u32 size, u32 pitch, u32 flags)
50 {
51 struct nouveau_drm *drm = nouveau_drm(dev);
52 int i = reg - drm->tile.reg;
53 struct nvkm_fb *fb = nvxx_fb(&drm->client.device);
54 struct nvkm_fb_tile *tile = &fb->tile.region[i];
55
56 nouveau_fence_unref(&reg->fence);
57
58 if (tile->pitch)
59 nvkm_fb_tile_fini(fb, i, tile);
60
61 if (pitch)
62 nvkm_fb_tile_init(fb, i, addr, size, pitch, flags, tile);
63
64 nvkm_fb_tile_prog(fb, i, tile);
65 }
66
67 static struct nouveau_drm_tile *
68 nv10_bo_get_tile_region(struct drm_device *dev, int i)
69 {
70 struct nouveau_drm *drm = nouveau_drm(dev);
71 struct nouveau_drm_tile *tile = &drm->tile.reg[i];
72
73 spin_lock(&drm->tile.lock);
74
75 if (!tile->used &&
76 (!tile->fence || nouveau_fence_done(tile->fence)))
77 tile->used = true;
78 else
79 tile = NULL;
80
81 spin_unlock(&drm->tile.lock);
82 return tile;
83 }
84
85 static void
86 nv10_bo_put_tile_region(struct drm_device *dev, struct nouveau_drm_tile *tile,
87 struct dma_fence *fence)
88 {
89 struct nouveau_drm *drm = nouveau_drm(dev);
90
91 if (tile) {
92 spin_lock(&drm->tile.lock);
93 tile->fence = (struct nouveau_fence *)dma_fence_get(fence);
94 tile->used = false;
95 spin_unlock(&drm->tile.lock);
96 }
97 }
98
99 static struct nouveau_drm_tile *
100 nv10_bo_set_tiling(struct drm_device *dev, u32 addr,
101 u32 size, u32 pitch, u32 zeta)
102 {
103 struct nouveau_drm *drm = nouveau_drm(dev);
104 struct nvkm_fb *fb = nvxx_fb(&drm->client.device);
105 struct nouveau_drm_tile *tile, *found = NULL;
106 int i;
107
108 for (i = 0; i < fb->tile.regions; i++) {
109 tile = nv10_bo_get_tile_region(dev, i);
110
111 if (pitch && !found) {
112 found = tile;
113 continue;
114
115 } else if (tile && fb->tile.region[i].pitch) {
116 /* Kill an unused tile region. */
117 nv10_bo_update_tile_region(dev, tile, 0, 0, 0, 0);
118 }
119
120 nv10_bo_put_tile_region(dev, tile, NULL);
121 }
122
123 if (found)
124 nv10_bo_update_tile_region(dev, found, addr, size, pitch, zeta);
125 return found;
126 }
127
128 static void
129 nouveau_bo_del_ttm(struct ttm_buffer_object *bo)
130 {
131 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
132 struct drm_device *dev = drm->dev;
133 struct nouveau_bo *nvbo = nouveau_bo(bo);
134
135 if (unlikely(nvbo->gem.filp))
136 DRM_ERROR("bo %p still attached to GEM object\n", bo);
137 WARN_ON(nvbo->pin_refcnt > 0);
138 nv10_bo_put_tile_region(dev, nvbo->tile, NULL);
139 kfree(nvbo);
140 }
141
142 static inline u64
143 roundup_64(u64 x, u32 y)
144 {
145 x += y - 1;
146 do_div(x, y);
147 return x * y;
148 }
149
150 static void
151 nouveau_bo_fixup_align(struct nouveau_bo *nvbo, u32 flags,
152 int *align, u64 *size)
153 {
154 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
155 struct nvif_device *device = &drm->client.device;
156
157 if (device->info.family < NV_DEVICE_INFO_V0_TESLA) {
158 if (nvbo->mode) {
159 if (device->info.chipset >= 0x40) {
160 *align = 65536;
161 *size = roundup_64(*size, 64 * nvbo->mode);
162
163 } else if (device->info.chipset >= 0x30) {
164 *align = 32768;
165 *size = roundup_64(*size, 64 * nvbo->mode);
166
167 } else if (device->info.chipset >= 0x20) {
168 *align = 16384;
169 *size = roundup_64(*size, 64 * nvbo->mode);
170
171 } else if (device->info.chipset >= 0x10) {
172 *align = 16384;
173 *size = roundup_64(*size, 32 * nvbo->mode);
174 }
175 }
176 } else {
177 *size = roundup_64(*size, (1 << nvbo->page));
178 *align = max((1 << nvbo->page), *align);
179 }
180
181 *size = roundup_64(*size, PAGE_SIZE);
182 }
183
184 int
185 nouveau_bo_new(struct nouveau_cli *cli, u64 size, int align,
186 uint32_t flags, uint32_t tile_mode, uint32_t tile_flags,
187 struct sg_table *sg, struct reservation_object *robj,
188 struct nouveau_bo **pnvbo)
189 {
190 struct nouveau_drm *drm = cli->drm;
191 struct nouveau_bo *nvbo;
192 struct nvif_mmu *mmu = &cli->mmu;
193 size_t acc_size;
194 int ret;
195 int type = ttm_bo_type_device;
196
197 if (!size) {
198 NV_WARN(drm, "skipped size %016llx\n", size);
199 return -EINVAL;
200 }
201
202 if (sg)
203 type = ttm_bo_type_sg;
204
205 nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL);
206 if (!nvbo)
207 return -ENOMEM;
208 INIT_LIST_HEAD(&nvbo->head);
209 INIT_LIST_HEAD(&nvbo->entry);
210 INIT_LIST_HEAD(&nvbo->vma_list);
211 nvbo->bo.bdev = &drm->ttm.bdev;
212 nvbo->cli = cli;
213
214 if (!nvxx_device(&drm->client.device)->func->cpu_coherent)
215 nvbo->force_coherent = flags & TTM_PL_FLAG_UNCACHED;
216
217 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) {
218 nvbo->kind = (tile_flags & 0x0000ff00) >> 8;
219 if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) {
220 kfree(nvbo);
221 return -EINVAL;
222 }
223
224 nvbo->comp = mmu->kind[nvbo->kind] != nvbo->kind;
225 } else
226 if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
227 nvbo->kind = (tile_flags & 0x00007f00) >> 8;
228 nvbo->comp = (tile_flags & 0x00030000) >> 16;
229 if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) {
230 kfree(nvbo);
231 return -EINVAL;
232 }
233 } else {
234 nvbo->zeta = (tile_flags & 0x00000007);
235 }
236 nvbo->mode = tile_mode;
237 nvbo->contig = !(tile_flags & NOUVEAU_GEM_TILE_NONCONTIG);
238
239 nvbo->page = 12;
240 if (drm->client.vm) {
241 if (!(flags & TTM_PL_FLAG_TT) && size > 256 * 1024)
242 nvbo->page = drm->client.vm->mmu->lpg_shift;
243 else {
244 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI)
245 nvbo->kind = mmu->kind[nvbo->kind];
246 nvbo->comp = 0;
247 }
248 }
249
250 nouveau_bo_fixup_align(nvbo, flags, &align, &size);
251 nvbo->bo.mem.num_pages = size >> PAGE_SHIFT;
252 nouveau_bo_placement_set(nvbo, flags, 0);
253
254 acc_size = ttm_bo_dma_acc_size(&drm->ttm.bdev, size,
255 sizeof(struct nouveau_bo));
256
257 ret = ttm_bo_init(&drm->ttm.bdev, &nvbo->bo, size,
258 type, &nvbo->placement,
259 align >> PAGE_SHIFT, false, NULL, acc_size, sg,
260 robj, nouveau_bo_del_ttm);
261 if (ret) {
262 /* ttm will call nouveau_bo_del_ttm if it fails.. */
263 return ret;
264 }
265
266 *pnvbo = nvbo;
267 return 0;
268 }
269
270 static void
271 set_placement_list(struct ttm_place *pl, unsigned *n, uint32_t type, uint32_t flags)
272 {
273 *n = 0;
274
275 if (type & TTM_PL_FLAG_VRAM)
276 pl[(*n)++].flags = TTM_PL_FLAG_VRAM | flags;
277 if (type & TTM_PL_FLAG_TT)
278 pl[(*n)++].flags = TTM_PL_FLAG_TT | flags;
279 if (type & TTM_PL_FLAG_SYSTEM)
280 pl[(*n)++].flags = TTM_PL_FLAG_SYSTEM | flags;
281 }
282
283 static void
284 set_placement_range(struct nouveau_bo *nvbo, uint32_t type)
285 {
286 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
287 u32 vram_pages = drm->client.device.info.ram_size >> PAGE_SHIFT;
288 unsigned i, fpfn, lpfn;
289
290 if (drm->client.device.info.family == NV_DEVICE_INFO_V0_CELSIUS &&
291 nvbo->mode && (type & TTM_PL_FLAG_VRAM) &&
292 nvbo->bo.mem.num_pages < vram_pages / 4) {
293 /*
294 * Make sure that the color and depth buffers are handled
295 * by independent memory controller units. Up to a 9x
296 * speed up when alpha-blending and depth-test are enabled
297 * at the same time.
298 */
299 if (nvbo->zeta) {
300 fpfn = vram_pages / 2;
301 lpfn = ~0;
302 } else {
303 fpfn = 0;
304 lpfn = vram_pages / 2;
305 }
306 for (i = 0; i < nvbo->placement.num_placement; ++i) {
307 nvbo->placements[i].fpfn = fpfn;
308 nvbo->placements[i].lpfn = lpfn;
309 }
310 for (i = 0; i < nvbo->placement.num_busy_placement; ++i) {
311 nvbo->busy_placements[i].fpfn = fpfn;
312 nvbo->busy_placements[i].lpfn = lpfn;
313 }
314 }
315 }
316
317 void
318 nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t type, uint32_t busy)
319 {
320 struct ttm_placement *pl = &nvbo->placement;
321 uint32_t flags = (nvbo->force_coherent ? TTM_PL_FLAG_UNCACHED :
322 TTM_PL_MASK_CACHING) |
323 (nvbo->pin_refcnt ? TTM_PL_FLAG_NO_EVICT : 0);
324
325 pl->placement = nvbo->placements;
326 set_placement_list(nvbo->placements, &pl->num_placement,
327 type, flags);
328
329 pl->busy_placement = nvbo->busy_placements;
330 set_placement_list(nvbo->busy_placements, &pl->num_busy_placement,
331 type | busy, flags);
332
333 set_placement_range(nvbo, type);
334 }
335
336 int
337 nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype, bool contig)
338 {
339 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
340 struct ttm_buffer_object *bo = &nvbo->bo;
341 bool force = false, evict = false;
342 int ret;
343
344 ret = ttm_bo_reserve(bo, false, false, NULL);
345 if (ret)
346 return ret;
347
348 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA &&
349 memtype == TTM_PL_FLAG_VRAM && contig) {
350 if (!nvbo->contig) {
351 nvbo->contig = true;
352 force = true;
353 evict = true;
354 }
355 }
356
357 if (nvbo->pin_refcnt) {
358 if (!(memtype & (1 << bo->mem.mem_type)) || evict) {
359 NV_ERROR(drm, "bo %p pinned elsewhere: "
360 "0x%08x vs 0x%08x\n", bo,
361 1 << bo->mem.mem_type, memtype);
362 ret = -EBUSY;
363 }
364 nvbo->pin_refcnt++;
365 goto out;
366 }
367
368 if (evict) {
369 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT, 0);
370 ret = nouveau_bo_validate(nvbo, false, false);
371 if (ret)
372 goto out;
373 }
374
375 nvbo->pin_refcnt++;
376 nouveau_bo_placement_set(nvbo, memtype, 0);
377
378 /* drop pin_refcnt temporarily, so we don't trip the assertion
379 * in nouveau_bo_move() that makes sure we're not trying to
380 * move a pinned buffer
381 */
382 nvbo->pin_refcnt--;
383 ret = nouveau_bo_validate(nvbo, false, false);
384 if (ret)
385 goto out;
386 nvbo->pin_refcnt++;
387
388 switch (bo->mem.mem_type) {
389 case TTM_PL_VRAM:
390 drm->gem.vram_available -= bo->mem.size;
391 break;
392 case TTM_PL_TT:
393 drm->gem.gart_available -= bo->mem.size;
394 break;
395 default:
396 break;
397 }
398
399 out:
400 if (force && ret)
401 nvbo->contig = false;
402 ttm_bo_unreserve(bo);
403 return ret;
404 }
405
406 int
407 nouveau_bo_unpin(struct nouveau_bo *nvbo)
408 {
409 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
410 struct ttm_buffer_object *bo = &nvbo->bo;
411 int ret, ref;
412
413 ret = ttm_bo_reserve(bo, false, false, NULL);
414 if (ret)
415 return ret;
416
417 ref = --nvbo->pin_refcnt;
418 WARN_ON_ONCE(ref < 0);
419 if (ref)
420 goto out;
421
422 nouveau_bo_placement_set(nvbo, bo->mem.placement, 0);
423
424 ret = nouveau_bo_validate(nvbo, false, false);
425 if (ret == 0) {
426 switch (bo->mem.mem_type) {
427 case TTM_PL_VRAM:
428 drm->gem.vram_available += bo->mem.size;
429 break;
430 case TTM_PL_TT:
431 drm->gem.gart_available += bo->mem.size;
432 break;
433 default:
434 break;
435 }
436 }
437
438 out:
439 ttm_bo_unreserve(bo);
440 return ret;
441 }
442
443 int
444 nouveau_bo_map(struct nouveau_bo *nvbo)
445 {
446 int ret;
447
448 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
449 if (ret)
450 return ret;
451
452 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, &nvbo->kmap);
453
454 ttm_bo_unreserve(&nvbo->bo);
455 return ret;
456 }
457
458 void
459 nouveau_bo_unmap(struct nouveau_bo *nvbo)
460 {
461 if (!nvbo)
462 return;
463
464 ttm_bo_kunmap(&nvbo->kmap);
465 }
466
467 void
468 nouveau_bo_sync_for_device(struct nouveau_bo *nvbo)
469 {
470 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
471 struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm;
472 int i;
473
474 if (!ttm_dma)
475 return;
476
477 /* Don't waste time looping if the object is coherent */
478 if (nvbo->force_coherent)
479 return;
480
481 for (i = 0; i < ttm_dma->ttm.num_pages; i++)
482 dma_sync_single_for_device(drm->dev->dev,
483 ttm_dma->dma_address[i],
484 PAGE_SIZE, DMA_TO_DEVICE);
485 }
486
487 void
488 nouveau_bo_sync_for_cpu(struct nouveau_bo *nvbo)
489 {
490 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
491 struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm;
492 int i;
493
494 if (!ttm_dma)
495 return;
496
497 /* Don't waste time looping if the object is coherent */
498 if (nvbo->force_coherent)
499 return;
500
501 for (i = 0; i < ttm_dma->ttm.num_pages; i++)
502 dma_sync_single_for_cpu(drm->dev->dev, ttm_dma->dma_address[i],
503 PAGE_SIZE, DMA_FROM_DEVICE);
504 }
505
506 int
507 nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible,
508 bool no_wait_gpu)
509 {
510 int ret;
511
512 ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement,
513 interruptible, no_wait_gpu);
514 if (ret)
515 return ret;
516
517 nouveau_bo_sync_for_device(nvbo);
518
519 return 0;
520 }
521
522 void
523 nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val)
524 {
525 bool is_iomem;
526 u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
527
528 mem += index;
529
530 if (is_iomem)
531 iowrite16_native(val, (void __force __iomem *)mem);
532 else
533 *mem = val;
534 }
535
536 u32
537 nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index)
538 {
539 bool is_iomem;
540 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
541
542 mem += index;
543
544 if (is_iomem)
545 return ioread32_native((void __force __iomem *)mem);
546 else
547 return *mem;
548 }
549
550 void
551 nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val)
552 {
553 bool is_iomem;
554 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
555
556 mem += index;
557
558 if (is_iomem)
559 iowrite32_native(val, (void __force __iomem *)mem);
560 else
561 *mem = val;
562 }
563
564 static struct ttm_tt *
565 nouveau_ttm_tt_create(struct ttm_bo_device *bdev, unsigned long size,
566 uint32_t page_flags, struct page *dummy_read)
567 {
568 #if IS_ENABLED(CONFIG_AGP)
569 struct nouveau_drm *drm = nouveau_bdev(bdev);
570
571 if (drm->agp.bridge) {
572 return ttm_agp_tt_create(bdev, drm->agp.bridge, size,
573 page_flags, dummy_read);
574 }
575 #endif
576
577 return nouveau_sgdma_create_ttm(bdev, size, page_flags, dummy_read);
578 }
579
580 static int
581 nouveau_bo_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
582 {
583 /* We'll do this from user space. */
584 return 0;
585 }
586
587 static int
588 nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
589 struct ttm_mem_type_manager *man)
590 {
591 struct nouveau_drm *drm = nouveau_bdev(bdev);
592
593 switch (type) {
594 case TTM_PL_SYSTEM:
595 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
596 man->available_caching = TTM_PL_MASK_CACHING;
597 man->default_caching = TTM_PL_FLAG_CACHED;
598 break;
599 case TTM_PL_VRAM:
600 man->flags = TTM_MEMTYPE_FLAG_FIXED |
601 TTM_MEMTYPE_FLAG_MAPPABLE;
602 man->available_caching = TTM_PL_FLAG_UNCACHED |
603 TTM_PL_FLAG_WC;
604 man->default_caching = TTM_PL_FLAG_WC;
605
606 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
607 /* Some BARs do not support being ioremapped WC */
608 if (nvxx_bar(&drm->client.device)->iomap_uncached) {
609 man->available_caching = TTM_PL_FLAG_UNCACHED;
610 man->default_caching = TTM_PL_FLAG_UNCACHED;
611 }
612
613 man->func = &nouveau_vram_manager;
614 man->io_reserve_fastpath = false;
615 man->use_io_reserve_lru = true;
616 } else {
617 man->func = &ttm_bo_manager_func;
618 }
619 break;
620 case TTM_PL_TT:
621 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA)
622 man->func = &nouveau_gart_manager;
623 else
624 if (!drm->agp.bridge)
625 man->func = &nv04_gart_manager;
626 else
627 man->func = &ttm_bo_manager_func;
628
629 if (drm->agp.bridge) {
630 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
631 man->available_caching = TTM_PL_FLAG_UNCACHED |
632 TTM_PL_FLAG_WC;
633 man->default_caching = TTM_PL_FLAG_WC;
634 } else {
635 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE |
636 TTM_MEMTYPE_FLAG_CMA;
637 man->available_caching = TTM_PL_MASK_CACHING;
638 man->default_caching = TTM_PL_FLAG_CACHED;
639 }
640
641 break;
642 default:
643 return -EINVAL;
644 }
645 return 0;
646 }
647
648 static void
649 nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl)
650 {
651 struct nouveau_bo *nvbo = nouveau_bo(bo);
652
653 switch (bo->mem.mem_type) {
654 case TTM_PL_VRAM:
655 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT,
656 TTM_PL_FLAG_SYSTEM);
657 break;
658 default:
659 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_SYSTEM, 0);
660 break;
661 }
662
663 *pl = nvbo->placement;
664 }
665
666
667 static int
668 nve0_bo_move_init(struct nouveau_channel *chan, u32 handle)
669 {
670 int ret = RING_SPACE(chan, 2);
671 if (ret == 0) {
672 BEGIN_NVC0(chan, NvSubCopy, 0x0000, 1);
673 OUT_RING (chan, handle & 0x0000ffff);
674 FIRE_RING (chan);
675 }
676 return ret;
677 }
678
679 static int
680 nve0_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
681 struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg)
682 {
683 struct nouveau_mem *mem = nouveau_mem(old_reg);
684 int ret = RING_SPACE(chan, 10);
685 if (ret == 0) {
686 BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8);
687 OUT_RING (chan, upper_32_bits(mem->vma[0].addr));
688 OUT_RING (chan, lower_32_bits(mem->vma[0].addr));
689 OUT_RING (chan, upper_32_bits(mem->vma[1].addr));
690 OUT_RING (chan, lower_32_bits(mem->vma[1].addr));
691 OUT_RING (chan, PAGE_SIZE);
692 OUT_RING (chan, PAGE_SIZE);
693 OUT_RING (chan, PAGE_SIZE);
694 OUT_RING (chan, new_reg->num_pages);
695 BEGIN_IMC0(chan, NvSubCopy, 0x0300, 0x0386);
696 }
697 return ret;
698 }
699
700 static int
701 nvc0_bo_move_init(struct nouveau_channel *chan, u32 handle)
702 {
703 int ret = RING_SPACE(chan, 2);
704 if (ret == 0) {
705 BEGIN_NVC0(chan, NvSubCopy, 0x0000, 1);
706 OUT_RING (chan, handle);
707 }
708 return ret;
709 }
710
711 static int
712 nvc0_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
713 struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg)
714 {
715 struct nouveau_mem *mem = nouveau_mem(old_reg);
716 u64 src_offset = mem->vma[0].addr;
717 u64 dst_offset = mem->vma[1].addr;
718 u32 page_count = new_reg->num_pages;
719 int ret;
720
721 page_count = new_reg->num_pages;
722 while (page_count) {
723 int line_count = (page_count > 8191) ? 8191 : page_count;
724
725 ret = RING_SPACE(chan, 11);
726 if (ret)
727 return ret;
728
729 BEGIN_NVC0(chan, NvSubCopy, 0x030c, 8);
730 OUT_RING (chan, upper_32_bits(src_offset));
731 OUT_RING (chan, lower_32_bits(src_offset));
732 OUT_RING (chan, upper_32_bits(dst_offset));
733 OUT_RING (chan, lower_32_bits(dst_offset));
734 OUT_RING (chan, PAGE_SIZE);
735 OUT_RING (chan, PAGE_SIZE);
736 OUT_RING (chan, PAGE_SIZE);
737 OUT_RING (chan, line_count);
738 BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
739 OUT_RING (chan, 0x00000110);
740
741 page_count -= line_count;
742 src_offset += (PAGE_SIZE * line_count);
743 dst_offset += (PAGE_SIZE * line_count);
744 }
745
746 return 0;
747 }
748
749 static int
750 nvc0_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
751 struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg)
752 {
753 struct nouveau_mem *mem = nouveau_mem(old_reg);
754 u64 src_offset = mem->vma[0].addr;
755 u64 dst_offset = mem->vma[1].addr;
756 u32 page_count = new_reg->num_pages;
757 int ret;
758
759 page_count = new_reg->num_pages;
760 while (page_count) {
761 int line_count = (page_count > 2047) ? 2047 : page_count;
762
763 ret = RING_SPACE(chan, 12);
764 if (ret)
765 return ret;
766
767 BEGIN_NVC0(chan, NvSubCopy, 0x0238, 2);
768 OUT_RING (chan, upper_32_bits(dst_offset));
769 OUT_RING (chan, lower_32_bits(dst_offset));
770 BEGIN_NVC0(chan, NvSubCopy, 0x030c, 6);
771 OUT_RING (chan, upper_32_bits(src_offset));
772 OUT_RING (chan, lower_32_bits(src_offset));
773 OUT_RING (chan, PAGE_SIZE); /* src_pitch */
774 OUT_RING (chan, PAGE_SIZE); /* dst_pitch */
775 OUT_RING (chan, PAGE_SIZE); /* line_length */
776 OUT_RING (chan, line_count);
777 BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
778 OUT_RING (chan, 0x00100110);
779
780 page_count -= line_count;
781 src_offset += (PAGE_SIZE * line_count);
782 dst_offset += (PAGE_SIZE * line_count);
783 }
784
785 return 0;
786 }
787
788 static int
789 nva3_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
790 struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg)
791 {
792 struct nouveau_mem *mem = nouveau_mem(old_reg);
793 u64 src_offset = mem->vma[0].addr;
794 u64 dst_offset = mem->vma[1].addr;
795 u32 page_count = new_reg->num_pages;
796 int ret;
797
798 page_count = new_reg->num_pages;
799 while (page_count) {
800 int line_count = (page_count > 8191) ? 8191 : page_count;
801
802 ret = RING_SPACE(chan, 11);
803 if (ret)
804 return ret;
805
806 BEGIN_NV04(chan, NvSubCopy, 0x030c, 8);
807 OUT_RING (chan, upper_32_bits(src_offset));
808 OUT_RING (chan, lower_32_bits(src_offset));
809 OUT_RING (chan, upper_32_bits(dst_offset));
810 OUT_RING (chan, lower_32_bits(dst_offset));
811 OUT_RING (chan, PAGE_SIZE);
812 OUT_RING (chan, PAGE_SIZE);
813 OUT_RING (chan, PAGE_SIZE);
814 OUT_RING (chan, line_count);
815 BEGIN_NV04(chan, NvSubCopy, 0x0300, 1);
816 OUT_RING (chan, 0x00000110);
817
818 page_count -= line_count;
819 src_offset += (PAGE_SIZE * line_count);
820 dst_offset += (PAGE_SIZE * line_count);
821 }
822
823 return 0;
824 }
825
826 static int
827 nv98_bo_move_exec(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
828 struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg)
829 {
830 struct nouveau_mem *mem = nouveau_mem(old_reg);
831 int ret = RING_SPACE(chan, 7);
832 if (ret == 0) {
833 BEGIN_NV04(chan, NvSubCopy, 0x0320, 6);
834 OUT_RING (chan, upper_32_bits(mem->vma[0].addr));
835 OUT_RING (chan, lower_32_bits(mem->vma[0].addr));
836 OUT_RING (chan, upper_32_bits(mem->vma[1].addr));
837 OUT_RING (chan, lower_32_bits(mem->vma[1].addr));
838 OUT_RING (chan, 0x00000000 /* COPY */);
839 OUT_RING (chan, new_reg->num_pages << PAGE_SHIFT);
840 }
841 return ret;
842 }
843
844 static int
845 nv84_bo_move_exec(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
846 struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg)
847 {
848 struct nouveau_mem *mem = nouveau_mem(old_reg);
849 int ret = RING_SPACE(chan, 7);
850 if (ret == 0) {
851 BEGIN_NV04(chan, NvSubCopy, 0x0304, 6);
852 OUT_RING (chan, new_reg->num_pages << PAGE_SHIFT);
853 OUT_RING (chan, upper_32_bits(mem->vma[0].addr));
854 OUT_RING (chan, lower_32_bits(mem->vma[0].addr));
855 OUT_RING (chan, upper_32_bits(mem->vma[1].addr));
856 OUT_RING (chan, lower_32_bits(mem->vma[1].addr));
857 OUT_RING (chan, 0x00000000 /* MODE_COPY, QUERY_NONE */);
858 }
859 return ret;
860 }
861
862 static int
863 nv50_bo_move_init(struct nouveau_channel *chan, u32 handle)
864 {
865 int ret = RING_SPACE(chan, 6);
866 if (ret == 0) {
867 BEGIN_NV04(chan, NvSubCopy, 0x0000, 1);
868 OUT_RING (chan, handle);
869 BEGIN_NV04(chan, NvSubCopy, 0x0180, 3);
870 OUT_RING (chan, chan->drm->ntfy.handle);
871 OUT_RING (chan, chan->vram.handle);
872 OUT_RING (chan, chan->vram.handle);
873 }
874
875 return ret;
876 }
877
878 static int
879 nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
880 struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg)
881 {
882 struct nouveau_mem *mem = nouveau_mem(old_reg);
883 u64 length = (new_reg->num_pages << PAGE_SHIFT);
884 u64 src_offset = mem->vma[0].addr;
885 u64 dst_offset = mem->vma[1].addr;
886 int src_tiled = !!mem->kind;
887 int dst_tiled = !!nouveau_mem(new_reg)->kind;
888 int ret;
889
890 while (length) {
891 u32 amount, stride, height;
892
893 ret = RING_SPACE(chan, 18 + 6 * (src_tiled + dst_tiled));
894 if (ret)
895 return ret;
896
897 amount = min(length, (u64)(4 * 1024 * 1024));
898 stride = 16 * 4;
899 height = amount / stride;
900
901 if (src_tiled) {
902 BEGIN_NV04(chan, NvSubCopy, 0x0200, 7);
903 OUT_RING (chan, 0);
904 OUT_RING (chan, 0);
905 OUT_RING (chan, stride);
906 OUT_RING (chan, height);
907 OUT_RING (chan, 1);
908 OUT_RING (chan, 0);
909 OUT_RING (chan, 0);
910 } else {
911 BEGIN_NV04(chan, NvSubCopy, 0x0200, 1);
912 OUT_RING (chan, 1);
913 }
914 if (dst_tiled) {
915 BEGIN_NV04(chan, NvSubCopy, 0x021c, 7);
916 OUT_RING (chan, 0);
917 OUT_RING (chan, 0);
918 OUT_RING (chan, stride);
919 OUT_RING (chan, height);
920 OUT_RING (chan, 1);
921 OUT_RING (chan, 0);
922 OUT_RING (chan, 0);
923 } else {
924 BEGIN_NV04(chan, NvSubCopy, 0x021c, 1);
925 OUT_RING (chan, 1);
926 }
927
928 BEGIN_NV04(chan, NvSubCopy, 0x0238, 2);
929 OUT_RING (chan, upper_32_bits(src_offset));
930 OUT_RING (chan, upper_32_bits(dst_offset));
931 BEGIN_NV04(chan, NvSubCopy, 0x030c, 8);
932 OUT_RING (chan, lower_32_bits(src_offset));
933 OUT_RING (chan, lower_32_bits(dst_offset));
934 OUT_RING (chan, stride);
935 OUT_RING (chan, stride);
936 OUT_RING (chan, stride);
937 OUT_RING (chan, height);
938 OUT_RING (chan, 0x00000101);
939 OUT_RING (chan, 0x00000000);
940 BEGIN_NV04(chan, NvSubCopy, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
941 OUT_RING (chan, 0);
942
943 length -= amount;
944 src_offset += amount;
945 dst_offset += amount;
946 }
947
948 return 0;
949 }
950
951 static int
952 nv04_bo_move_init(struct nouveau_channel *chan, u32 handle)
953 {
954 int ret = RING_SPACE(chan, 4);
955 if (ret == 0) {
956 BEGIN_NV04(chan, NvSubCopy, 0x0000, 1);
957 OUT_RING (chan, handle);
958 BEGIN_NV04(chan, NvSubCopy, 0x0180, 1);
959 OUT_RING (chan, chan->drm->ntfy.handle);
960 }
961
962 return ret;
963 }
964
965 static inline uint32_t
966 nouveau_bo_mem_ctxdma(struct ttm_buffer_object *bo,
967 struct nouveau_channel *chan, struct ttm_mem_reg *reg)
968 {
969 if (reg->mem_type == TTM_PL_TT)
970 return NvDmaTT;
971 return chan->vram.handle;
972 }
973
974 static int
975 nv04_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
976 struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg)
977 {
978 u32 src_offset = old_reg->start << PAGE_SHIFT;
979 u32 dst_offset = new_reg->start << PAGE_SHIFT;
980 u32 page_count = new_reg->num_pages;
981 int ret;
982
983 ret = RING_SPACE(chan, 3);
984 if (ret)
985 return ret;
986
987 BEGIN_NV04(chan, NvSubCopy, NV_MEMORY_TO_MEMORY_FORMAT_DMA_SOURCE, 2);
988 OUT_RING (chan, nouveau_bo_mem_ctxdma(bo, chan, old_reg));
989 OUT_RING (chan, nouveau_bo_mem_ctxdma(bo, chan, new_reg));
990
991 page_count = new_reg->num_pages;
992 while (page_count) {
993 int line_count = (page_count > 2047) ? 2047 : page_count;
994
995 ret = RING_SPACE(chan, 11);
996 if (ret)
997 return ret;
998
999 BEGIN_NV04(chan, NvSubCopy,
1000 NV_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN, 8);
1001 OUT_RING (chan, src_offset);
1002 OUT_RING (chan, dst_offset);
1003 OUT_RING (chan, PAGE_SIZE); /* src_pitch */
1004 OUT_RING (chan, PAGE_SIZE); /* dst_pitch */
1005 OUT_RING (chan, PAGE_SIZE); /* line_length */
1006 OUT_RING (chan, line_count);
1007 OUT_RING (chan, 0x00000101);
1008 OUT_RING (chan, 0x00000000);
1009 BEGIN_NV04(chan, NvSubCopy, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
1010 OUT_RING (chan, 0);
1011
1012 page_count -= line_count;
1013 src_offset += (PAGE_SIZE * line_count);
1014 dst_offset += (PAGE_SIZE * line_count);
1015 }
1016
1017 return 0;
1018 }
1019
1020 static int
1021 nouveau_bo_move_prep(struct nouveau_drm *drm, struct ttm_buffer_object *bo,
1022 struct ttm_mem_reg *reg)
1023 {
1024 struct nouveau_mem *old_mem = nouveau_mem(&bo->mem);
1025 struct nouveau_mem *new_mem = nouveau_mem(reg);
1026 struct nvkm_vm *vmm = drm->client.vm;
1027 u64 size = (u64)reg->num_pages << PAGE_SHIFT;
1028 int ret;
1029
1030 ret = nvkm_vm_get(vmm, size, old_mem->mem.page, NV_MEM_ACCESS_RW,
1031 &old_mem->vma[0]);
1032 if (ret)
1033 return ret;
1034
1035 ret = nvkm_vm_get(vmm, size, new_mem->mem.page, NV_MEM_ACCESS_RW,
1036 &old_mem->vma[1]);
1037 if (ret) {
1038 nvkm_vm_put(&old_mem->vma[0]);
1039 return ret;
1040 }
1041
1042 ret = nouveau_mem_map(old_mem, vmm, &old_mem->vma[0]);
1043 if (ret)
1044 goto done;
1045
1046 ret = nouveau_mem_map(new_mem, vmm, &old_mem->vma[1]);
1047 done:
1048 if (ret) {
1049 nvkm_vm_put(&old_mem->vma[1]);
1050 nvkm_vm_put(&old_mem->vma[0]);
1051 }
1052 return 0;
1053 }
1054
1055 static int
1056 nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
1057 bool no_wait_gpu, struct ttm_mem_reg *new_reg)
1058 {
1059 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1060 struct nouveau_channel *chan = drm->ttm.chan;
1061 struct nouveau_cli *cli = (void *)chan->user.client;
1062 struct nouveau_fence *fence;
1063 int ret;
1064
1065 /* create temporary vmas for the transfer and attach them to the
1066 * old nvkm_mem node, these will get cleaned up after ttm has
1067 * destroyed the ttm_mem_reg
1068 */
1069 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
1070 ret = nouveau_bo_move_prep(drm, bo, new_reg);
1071 if (ret)
1072 return ret;
1073 }
1074
1075 mutex_lock_nested(&cli->mutex, SINGLE_DEPTH_NESTING);
1076 ret = nouveau_fence_sync(nouveau_bo(bo), chan, true, intr);
1077 if (ret == 0) {
1078 ret = drm->ttm.move(chan, bo, &bo->mem, new_reg);
1079 if (ret == 0) {
1080 ret = nouveau_fence_new(chan, false, &fence);
1081 if (ret == 0) {
1082 ret = ttm_bo_move_accel_cleanup(bo,
1083 &fence->base,
1084 evict,
1085 new_reg);
1086 nouveau_fence_unref(&fence);
1087 }
1088 }
1089 }
1090 mutex_unlock(&cli->mutex);
1091 return ret;
1092 }
1093
1094 void
1095 nouveau_bo_move_init(struct nouveau_drm *drm)
1096 {
1097 static const struct {
1098 const char *name;
1099 int engine;
1100 s32 oclass;
1101 int (*exec)(struct nouveau_channel *,
1102 struct ttm_buffer_object *,
1103 struct ttm_mem_reg *, struct ttm_mem_reg *);
1104 int (*init)(struct nouveau_channel *, u32 handle);
1105 } _methods[] = {
1106 { "COPY", 4, 0xc1b5, nve0_bo_move_copy, nve0_bo_move_init },
1107 { "GRCE", 0, 0xc1b5, nve0_bo_move_copy, nvc0_bo_move_init },
1108 { "COPY", 4, 0xc0b5, nve0_bo_move_copy, nve0_bo_move_init },
1109 { "GRCE", 0, 0xc0b5, nve0_bo_move_copy, nvc0_bo_move_init },
1110 { "COPY", 4, 0xb0b5, nve0_bo_move_copy, nve0_bo_move_init },
1111 { "GRCE", 0, 0xb0b5, nve0_bo_move_copy, nvc0_bo_move_init },
1112 { "COPY", 4, 0xa0b5, nve0_bo_move_copy, nve0_bo_move_init },
1113 { "GRCE", 0, 0xa0b5, nve0_bo_move_copy, nvc0_bo_move_init },
1114 { "COPY1", 5, 0x90b8, nvc0_bo_move_copy, nvc0_bo_move_init },
1115 { "COPY0", 4, 0x90b5, nvc0_bo_move_copy, nvc0_bo_move_init },
1116 { "COPY", 0, 0x85b5, nva3_bo_move_copy, nv50_bo_move_init },
1117 { "CRYPT", 0, 0x74c1, nv84_bo_move_exec, nv50_bo_move_init },
1118 { "M2MF", 0, 0x9039, nvc0_bo_move_m2mf, nvc0_bo_move_init },
1119 { "M2MF", 0, 0x5039, nv50_bo_move_m2mf, nv50_bo_move_init },
1120 { "M2MF", 0, 0x0039, nv04_bo_move_m2mf, nv04_bo_move_init },
1121 {},
1122 { "CRYPT", 0, 0x88b4, nv98_bo_move_exec, nv50_bo_move_init },
1123 }, *mthd = _methods;
1124 const char *name = "CPU";
1125 int ret;
1126
1127 do {
1128 struct nouveau_channel *chan;
1129
1130 if (mthd->engine)
1131 chan = drm->cechan;
1132 else
1133 chan = drm->channel;
1134 if (chan == NULL)
1135 continue;
1136
1137 ret = nvif_object_init(&chan->user,
1138 mthd->oclass | (mthd->engine << 16),
1139 mthd->oclass, NULL, 0,
1140 &drm->ttm.copy);
1141 if (ret == 0) {
1142 ret = mthd->init(chan, drm->ttm.copy.handle);
1143 if (ret) {
1144 nvif_object_fini(&drm->ttm.copy);
1145 continue;
1146 }
1147
1148 drm->ttm.move = mthd->exec;
1149 drm->ttm.chan = chan;
1150 name = mthd->name;
1151 break;
1152 }
1153 } while ((++mthd)->exec);
1154
1155 NV_INFO(drm, "MM: using %s for buffer copies\n", name);
1156 }
1157
1158 static int
1159 nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr,
1160 bool no_wait_gpu, struct ttm_mem_reg *new_reg)
1161 {
1162 struct ttm_place placement_memtype = {
1163 .fpfn = 0,
1164 .lpfn = 0,
1165 .flags = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING
1166 };
1167 struct ttm_placement placement;
1168 struct ttm_mem_reg tmp_reg;
1169 int ret;
1170
1171 placement.num_placement = placement.num_busy_placement = 1;
1172 placement.placement = placement.busy_placement = &placement_memtype;
1173
1174 tmp_reg = *new_reg;
1175 tmp_reg.mm_node = NULL;
1176 ret = ttm_bo_mem_space(bo, &placement, &tmp_reg, intr, no_wait_gpu);
1177 if (ret)
1178 return ret;
1179
1180 ret = ttm_tt_bind(bo->ttm, &tmp_reg);
1181 if (ret)
1182 goto out;
1183
1184 ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_gpu, &tmp_reg);
1185 if (ret)
1186 goto out;
1187
1188 ret = ttm_bo_move_ttm(bo, intr, no_wait_gpu, new_reg);
1189 out:
1190 ttm_bo_mem_put(bo, &tmp_reg);
1191 return ret;
1192 }
1193
1194 static int
1195 nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr,
1196 bool no_wait_gpu, struct ttm_mem_reg *new_reg)
1197 {
1198 struct ttm_place placement_memtype = {
1199 .fpfn = 0,
1200 .lpfn = 0,
1201 .flags = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING
1202 };
1203 struct ttm_placement placement;
1204 struct ttm_mem_reg tmp_reg;
1205 int ret;
1206
1207 placement.num_placement = placement.num_busy_placement = 1;
1208 placement.placement = placement.busy_placement = &placement_memtype;
1209
1210 tmp_reg = *new_reg;
1211 tmp_reg.mm_node = NULL;
1212 ret = ttm_bo_mem_space(bo, &placement, &tmp_reg, intr, no_wait_gpu);
1213 if (ret)
1214 return ret;
1215
1216 ret = ttm_bo_move_ttm(bo, intr, no_wait_gpu, &tmp_reg);
1217 if (ret)
1218 goto out;
1219
1220 ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_gpu, new_reg);
1221 if (ret)
1222 goto out;
1223
1224 out:
1225 ttm_bo_mem_put(bo, &tmp_reg);
1226 return ret;
1227 }
1228
1229 static void
1230 nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, bool evict,
1231 struct ttm_mem_reg *new_reg)
1232 {
1233 struct nouveau_mem *mem = new_reg ? nouveau_mem(new_reg) : NULL;
1234 struct nouveau_bo *nvbo = nouveau_bo(bo);
1235 struct nouveau_vma *vma;
1236
1237 /* ttm can now (stupidly) pass the driver bos it didn't create... */
1238 if (bo->destroy != nouveau_bo_del_ttm)
1239 return;
1240
1241 if (mem && new_reg->mem_type != TTM_PL_SYSTEM &&
1242 mem->mem.page == nvbo->page) {
1243 list_for_each_entry(vma, &nvbo->vma_list, head) {
1244 nouveau_vma_map(vma, mem);
1245 }
1246 } else {
1247 list_for_each_entry(vma, &nvbo->vma_list, head) {
1248 WARN_ON(ttm_bo_wait(bo, false, false));
1249 nouveau_vma_unmap(vma);
1250 }
1251 }
1252 }
1253
1254 static int
1255 nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_reg,
1256 struct nouveau_drm_tile **new_tile)
1257 {
1258 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1259 struct drm_device *dev = drm->dev;
1260 struct nouveau_bo *nvbo = nouveau_bo(bo);
1261 u64 offset = new_reg->start << PAGE_SHIFT;
1262
1263 *new_tile = NULL;
1264 if (new_reg->mem_type != TTM_PL_VRAM)
1265 return 0;
1266
1267 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_CELSIUS) {
1268 *new_tile = nv10_bo_set_tiling(dev, offset, new_reg->size,
1269 nvbo->mode, nvbo->zeta);
1270 }
1271
1272 return 0;
1273 }
1274
1275 static void
1276 nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo,
1277 struct nouveau_drm_tile *new_tile,
1278 struct nouveau_drm_tile **old_tile)
1279 {
1280 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1281 struct drm_device *dev = drm->dev;
1282 struct dma_fence *fence = reservation_object_get_excl(bo->resv);
1283
1284 nv10_bo_put_tile_region(dev, *old_tile, fence);
1285 *old_tile = new_tile;
1286 }
1287
1288 static int
1289 nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
1290 bool no_wait_gpu, struct ttm_mem_reg *new_reg)
1291 {
1292 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1293 struct nouveau_bo *nvbo = nouveau_bo(bo);
1294 struct ttm_mem_reg *old_reg = &bo->mem;
1295 struct nouveau_drm_tile *new_tile = NULL;
1296 int ret = 0;
1297
1298 ret = ttm_bo_wait(bo, intr, no_wait_gpu);
1299 if (ret)
1300 return ret;
1301
1302 if (nvbo->pin_refcnt)
1303 NV_WARN(drm, "Moving pinned object %p!\n", nvbo);
1304
1305 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) {
1306 ret = nouveau_bo_vm_bind(bo, new_reg, &new_tile);
1307 if (ret)
1308 return ret;
1309 }
1310
1311 /* Fake bo copy. */
1312 if (old_reg->mem_type == TTM_PL_SYSTEM && !bo->ttm) {
1313 BUG_ON(bo->mem.mm_node != NULL);
1314 bo->mem = *new_reg;
1315 new_reg->mm_node = NULL;
1316 goto out;
1317 }
1318
1319 /* Hardware assisted copy. */
1320 if (drm->ttm.move) {
1321 if (new_reg->mem_type == TTM_PL_SYSTEM)
1322 ret = nouveau_bo_move_flipd(bo, evict, intr,
1323 no_wait_gpu, new_reg);
1324 else if (old_reg->mem_type == TTM_PL_SYSTEM)
1325 ret = nouveau_bo_move_flips(bo, evict, intr,
1326 no_wait_gpu, new_reg);
1327 else
1328 ret = nouveau_bo_move_m2mf(bo, evict, intr,
1329 no_wait_gpu, new_reg);
1330 if (!ret)
1331 goto out;
1332 }
1333
1334 /* Fallback to software copy. */
1335 ret = ttm_bo_wait(bo, intr, no_wait_gpu);
1336 if (ret == 0)
1337 ret = ttm_bo_move_memcpy(bo, intr, no_wait_gpu, new_reg);
1338
1339 out:
1340 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) {
1341 if (ret)
1342 nouveau_bo_vm_cleanup(bo, NULL, &new_tile);
1343 else
1344 nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile);
1345 }
1346
1347 return ret;
1348 }
1349
1350 static int
1351 nouveau_bo_verify_access(struct ttm_buffer_object *bo, struct file *filp)
1352 {
1353 struct nouveau_bo *nvbo = nouveau_bo(bo);
1354
1355 return drm_vma_node_verify_access(&nvbo->gem.vma_node,
1356 filp->private_data);
1357 }
1358
1359 static int
1360 nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *reg)
1361 {
1362 struct ttm_mem_type_manager *man = &bdev->man[reg->mem_type];
1363 struct nouveau_drm *drm = nouveau_bdev(bdev);
1364 struct nvkm_device *device = nvxx_device(&drm->client.device);
1365 struct nouveau_mem *mem = nouveau_mem(reg);
1366 int ret;
1367
1368 reg->bus.addr = NULL;
1369 reg->bus.offset = 0;
1370 reg->bus.size = reg->num_pages << PAGE_SHIFT;
1371 reg->bus.base = 0;
1372 reg->bus.is_iomem = false;
1373 if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
1374 return -EINVAL;
1375 switch (reg->mem_type) {
1376 case TTM_PL_SYSTEM:
1377 /* System memory */
1378 return 0;
1379 case TTM_PL_TT:
1380 #if IS_ENABLED(CONFIG_AGP)
1381 if (drm->agp.bridge) {
1382 reg->bus.offset = reg->start << PAGE_SHIFT;
1383 reg->bus.base = drm->agp.base;
1384 reg->bus.is_iomem = !drm->agp.cma;
1385 }
1386 #endif
1387 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA || !mem->kind)
1388 /* untiled */
1389 break;
1390 /* fallthrough, tiled memory */
1391 case TTM_PL_VRAM:
1392 reg->bus.offset = reg->start << PAGE_SHIFT;
1393 reg->bus.base = device->func->resource_addr(device, 1);
1394 reg->bus.is_iomem = true;
1395 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
1396 struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device);
1397 int page_shift = 12;
1398 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_FERMI)
1399 page_shift = mem->mem.page;
1400
1401 ret = nvkm_vm_get(bar, mem->_mem->size << 12,
1402 page_shift, NV_MEM_ACCESS_RW,
1403 &mem->bar_vma);
1404 if (ret)
1405 return ret;
1406
1407 nvkm_vm_map(&mem->bar_vma, mem->_mem);
1408 reg->bus.offset = mem->bar_vma.offset;
1409 }
1410 break;
1411 default:
1412 return -EINVAL;
1413 }
1414 return 0;
1415 }
1416
1417 static void
1418 nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *reg)
1419 {
1420 struct nouveau_mem *mem = nouveau_mem(reg);
1421
1422 if (!mem->bar_vma.node)
1423 return;
1424
1425 nvkm_vm_put(&mem->bar_vma);
1426 }
1427
1428 static int
1429 nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
1430 {
1431 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1432 struct nouveau_bo *nvbo = nouveau_bo(bo);
1433 struct nvkm_device *device = nvxx_device(&drm->client.device);
1434 u32 mappable = device->func->resource_size(device, 1) >> PAGE_SHIFT;
1435 int i, ret;
1436
1437 /* as long as the bo isn't in vram, and isn't tiled, we've got
1438 * nothing to do here.
1439 */
1440 if (bo->mem.mem_type != TTM_PL_VRAM) {
1441 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA ||
1442 !nvbo->kind)
1443 return 0;
1444
1445 if (bo->mem.mem_type == TTM_PL_SYSTEM) {
1446 nouveau_bo_placement_set(nvbo, TTM_PL_TT, 0);
1447
1448 ret = nouveau_bo_validate(nvbo, false, false);
1449 if (ret)
1450 return ret;
1451 }
1452 return 0;
1453 }
1454
1455 /* make sure bo is in mappable vram */
1456 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA ||
1457 bo->mem.start + bo->mem.num_pages < mappable)
1458 return 0;
1459
1460 for (i = 0; i < nvbo->placement.num_placement; ++i) {
1461 nvbo->placements[i].fpfn = 0;
1462 nvbo->placements[i].lpfn = mappable;
1463 }
1464
1465 for (i = 0; i < nvbo->placement.num_busy_placement; ++i) {
1466 nvbo->busy_placements[i].fpfn = 0;
1467 nvbo->busy_placements[i].lpfn = mappable;
1468 }
1469
1470 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_VRAM, 0);
1471 return nouveau_bo_validate(nvbo, false, false);
1472 }
1473
1474 static int
1475 nouveau_ttm_tt_populate(struct ttm_tt *ttm)
1476 {
1477 struct ttm_dma_tt *ttm_dma = (void *)ttm;
1478 struct nouveau_drm *drm;
1479 struct device *dev;
1480 unsigned i;
1481 int r;
1482 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
1483
1484 if (ttm->state != tt_unpopulated)
1485 return 0;
1486
1487 if (slave && ttm->sg) {
1488 /* make userspace faulting work */
1489 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
1490 ttm_dma->dma_address, ttm->num_pages);
1491 ttm->state = tt_unbound;
1492 return 0;
1493 }
1494
1495 drm = nouveau_bdev(ttm->bdev);
1496 dev = drm->dev->dev;
1497
1498 #if IS_ENABLED(CONFIG_AGP)
1499 if (drm->agp.bridge) {
1500 return ttm_agp_tt_populate(ttm);
1501 }
1502 #endif
1503
1504 #if IS_ENABLED(CONFIG_SWIOTLB) && IS_ENABLED(CONFIG_X86)
1505 if (swiotlb_nr_tbl()) {
1506 return ttm_dma_populate((void *)ttm, dev);
1507 }
1508 #endif
1509
1510 r = ttm_pool_populate(ttm);
1511 if (r) {
1512 return r;
1513 }
1514
1515 for (i = 0; i < ttm->num_pages; i++) {
1516 dma_addr_t addr;
1517
1518 addr = dma_map_page(dev, ttm->pages[i], 0, PAGE_SIZE,
1519 DMA_BIDIRECTIONAL);
1520
1521 if (dma_mapping_error(dev, addr)) {
1522 while (i--) {
1523 dma_unmap_page(dev, ttm_dma->dma_address[i],
1524 PAGE_SIZE, DMA_BIDIRECTIONAL);
1525 ttm_dma->dma_address[i] = 0;
1526 }
1527 ttm_pool_unpopulate(ttm);
1528 return -EFAULT;
1529 }
1530
1531 ttm_dma->dma_address[i] = addr;
1532 }
1533 return 0;
1534 }
1535
1536 static void
1537 nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm)
1538 {
1539 struct ttm_dma_tt *ttm_dma = (void *)ttm;
1540 struct nouveau_drm *drm;
1541 struct device *dev;
1542 unsigned i;
1543 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
1544
1545 if (slave)
1546 return;
1547
1548 drm = nouveau_bdev(ttm->bdev);
1549 dev = drm->dev->dev;
1550
1551 #if IS_ENABLED(CONFIG_AGP)
1552 if (drm->agp.bridge) {
1553 ttm_agp_tt_unpopulate(ttm);
1554 return;
1555 }
1556 #endif
1557
1558 #if IS_ENABLED(CONFIG_SWIOTLB) && IS_ENABLED(CONFIG_X86)
1559 if (swiotlb_nr_tbl()) {
1560 ttm_dma_unpopulate((void *)ttm, dev);
1561 return;
1562 }
1563 #endif
1564
1565 for (i = 0; i < ttm->num_pages; i++) {
1566 if (ttm_dma->dma_address[i]) {
1567 dma_unmap_page(dev, ttm_dma->dma_address[i], PAGE_SIZE,
1568 DMA_BIDIRECTIONAL);
1569 }
1570 }
1571
1572 ttm_pool_unpopulate(ttm);
1573 }
1574
1575 void
1576 nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence, bool exclusive)
1577 {
1578 struct reservation_object *resv = nvbo->bo.resv;
1579
1580 if (exclusive)
1581 reservation_object_add_excl_fence(resv, &fence->base);
1582 else if (fence)
1583 reservation_object_add_shared_fence(resv, &fence->base);
1584 }
1585
1586 struct ttm_bo_driver nouveau_bo_driver = {
1587 .ttm_tt_create = &nouveau_ttm_tt_create,
1588 .ttm_tt_populate = &nouveau_ttm_tt_populate,
1589 .ttm_tt_unpopulate = &nouveau_ttm_tt_unpopulate,
1590 .invalidate_caches = nouveau_bo_invalidate_caches,
1591 .init_mem_type = nouveau_bo_init_mem_type,
1592 .eviction_valuable = ttm_bo_eviction_valuable,
1593 .evict_flags = nouveau_bo_evict_flags,
1594 .move_notify = nouveau_bo_move_ntfy,
1595 .move = nouveau_bo_move,
1596 .verify_access = nouveau_bo_verify_access,
1597 .fault_reserve_notify = &nouveau_ttm_fault_reserve_notify,
1598 .io_mem_reserve = &nouveau_ttm_io_mem_reserve,
1599 .io_mem_free = &nouveau_ttm_io_mem_free,
1600 .io_mem_pfn = ttm_bo_default_io_mem_pfn,
1601 };
ubuntu-bionic-kernel mirror