2 * Copyright 2012 Red Hat Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
26 #include <nvif/class.h>
27 #include <nvif/ioctl.h>
30 #include <core/client.h>
32 #include "nouveau_drm.h"
33 #include "nouveau_dma.h"
34 #include "nouveau_bo.h"
35 #include "nouveau_chan.h"
36 #include "nouveau_fence.h"
37 #include "nouveau_abi16.h"
39 MODULE_PARM_DESC(vram_pushbuf
, "Create DMA push buffers in VRAM");
40 int nouveau_vram_pushbuf
;
41 module_param_named(vram_pushbuf
, nouveau_vram_pushbuf
, int, 0400);
44 nouveau_channel_idle(struct nouveau_channel
*chan
)
46 if (likely(chan
&& chan
->fence
)) {
47 struct nouveau_cli
*cli
= (void *)chan
->user
.client
;
48 struct nouveau_fence
*fence
= NULL
;
51 ret
= nouveau_fence_new(chan
, false, &fence
);
53 ret
= nouveau_fence_wait(fence
, false, false);
54 nouveau_fence_unref(&fence
);
58 NV_PRINTK(err
, cli
, "failed to idle channel "
61 nvxx_client(&cli
->base
)->name
);
69 nouveau_channel_del(struct nouveau_channel
**pchan
)
71 struct nouveau_channel
*chan
= *pchan
;
74 nouveau_fence(chan
->drm
)->context_del(chan
);
75 nvif_object_fini(&chan
->nvsw
);
76 nvif_object_fini(&chan
->gart
);
77 nvif_object_fini(&chan
->vram
);
78 nvif_object_fini(&chan
->user
);
79 nvif_object_fini(&chan
->push
.ctxdma
);
80 nouveau_bo_vma_del(chan
->push
.buffer
, &chan
->push
.vma
);
81 nouveau_bo_unmap(chan
->push
.buffer
);
82 if (chan
->push
.buffer
&& chan
->push
.buffer
->pin_refcnt
)
83 nouveau_bo_unpin(chan
->push
.buffer
);
84 nouveau_bo_ref(NULL
, &chan
->push
.buffer
);
91 nouveau_channel_prep(struct nouveau_drm
*drm
, struct nvif_device
*device
,
92 u32 handle
, u32 size
, struct nouveau_channel
**pchan
)
94 struct nouveau_cli
*cli
= (void *)device
->object
.client
;
95 struct nvkm_mmu
*mmu
= nvxx_mmu(device
);
96 struct nv_dma_v0 args
= {};
97 struct nouveau_channel
*chan
;
101 chan
= *pchan
= kzalloc(sizeof(*chan
), GFP_KERNEL
);
105 chan
->device
= device
;
108 /* allocate memory for dma push buffer */
109 target
= TTM_PL_FLAG_TT
| TTM_PL_FLAG_UNCACHED
;
110 if (nouveau_vram_pushbuf
)
111 target
= TTM_PL_FLAG_VRAM
;
113 ret
= nouveau_bo_new(drm
->dev
, size
, 0, target
, 0, 0, NULL
, NULL
,
116 ret
= nouveau_bo_pin(chan
->push
.buffer
, target
, false);
118 ret
= nouveau_bo_map(chan
->push
.buffer
);
122 nouveau_channel_del(pchan
);
126 /* create dma object covering the *entire* memory space that the
127 * pushbuf lives in, this is because the GEM code requires that
128 * we be able to call out to other (indirect) push buffers
130 chan
->push
.vma
.offset
= chan
->push
.buffer
->bo
.offset
;
132 if (device
->info
.family
>= NV_DEVICE_INFO_V0_TESLA
) {
133 ret
= nouveau_bo_vma_add(chan
->push
.buffer
, cli
->vm
,
136 nouveau_channel_del(pchan
);
140 args
.target
= NV_DMA_V0_TARGET_VM
;
141 args
.access
= NV_DMA_V0_ACCESS_VM
;
143 args
.limit
= cli
->vm
->mmu
->limit
- 1;
145 if (chan
->push
.buffer
->bo
.mem
.mem_type
== TTM_PL_VRAM
) {
146 if (device
->info
.family
== NV_DEVICE_INFO_V0_TNT
) {
147 /* nv04 vram pushbuf hack, retarget to its location in
148 * the framebuffer bar rather than direct vram access..
149 * nfi why this exists, it came from the -nv ddx.
151 args
.target
= NV_DMA_V0_TARGET_PCI
;
152 args
.access
= NV_DMA_V0_ACCESS_RDWR
;
153 args
.start
= nvxx_device(device
)->func
->
154 resource_addr(nvxx_device(device
), 1);
155 args
.limit
= args
.start
+ device
->info
.ram_user
- 1;
157 args
.target
= NV_DMA_V0_TARGET_VRAM
;
158 args
.access
= NV_DMA_V0_ACCESS_RDWR
;
160 args
.limit
= device
->info
.ram_user
- 1;
163 if (chan
->drm
->agp
.bridge
) {
164 args
.target
= NV_DMA_V0_TARGET_AGP
;
165 args
.access
= NV_DMA_V0_ACCESS_RDWR
;
166 args
.start
= chan
->drm
->agp
.base
;
167 args
.limit
= chan
->drm
->agp
.base
+
168 chan
->drm
->agp
.size
- 1;
170 args
.target
= NV_DMA_V0_TARGET_VM
;
171 args
.access
= NV_DMA_V0_ACCESS_RDWR
;
173 args
.limit
= mmu
->limit
- 1;
177 ret
= nvif_object_init(&device
->object
, NVDRM_PUSH
|
178 (handle
& 0xffff), NV_DMA_FROM_MEMORY
,
179 &args
, sizeof(args
), &chan
->push
.ctxdma
);
181 nouveau_channel_del(pchan
);
189 nouveau_channel_ind(struct nouveau_drm
*drm
, struct nvif_device
*device
,
190 u32 handle
, u32 engine
, struct nouveau_channel
**pchan
)
192 static const u16 oclasses
[] = { MAXWELL_CHANNEL_GPFIFO_A
,
193 KEPLER_CHANNEL_GPFIFO_A
,
194 FERMI_CHANNEL_GPFIFO
,
198 const u16
*oclass
= oclasses
;
200 struct nv50_channel_gpfifo_v0 nv50
;
201 struct fermi_channel_gpfifo_v0 fermi
;
202 struct kepler_channel_gpfifo_a_v0 kepler
;
204 struct nouveau_channel
*chan
;
208 /* allocate dma push buffer */
209 ret
= nouveau_channel_prep(drm
, device
, handle
, 0x12000, &chan
);
214 /* create channel object */
216 if (oclass
[0] >= KEPLER_CHANNEL_GPFIFO_A
) {
217 args
.kepler
.version
= 0;
218 args
.kepler
.engine
= engine
;
219 args
.kepler
.ilength
= 0x02000;
220 args
.kepler
.ioffset
= 0x10000 + chan
->push
.vma
.offset
;
222 size
= sizeof(args
.kepler
);
224 if (oclass
[0] >= FERMI_CHANNEL_GPFIFO
) {
225 args
.fermi
.version
= 0;
226 args
.fermi
.ilength
= 0x02000;
227 args
.fermi
.ioffset
= 0x10000 + chan
->push
.vma
.offset
;
229 size
= sizeof(args
.fermi
);
231 args
.nv50
.version
= 0;
232 args
.nv50
.ilength
= 0x02000;
233 args
.nv50
.ioffset
= 0x10000 + chan
->push
.vma
.offset
;
234 args
.nv50
.pushbuf
= nvif_handle(&chan
->push
.ctxdma
);
236 size
= sizeof(args
.nv50
);
239 ret
= nvif_object_init(&device
->object
, handle
, *oclass
++,
240 &args
, size
, &chan
->user
);
242 if (chan
->user
.oclass
>= KEPLER_CHANNEL_GPFIFO_A
)
243 chan
->chid
= args
.kepler
.chid
;
245 if (chan
->user
.oclass
>= FERMI_CHANNEL_GPFIFO
)
246 chan
->chid
= args
.fermi
.chid
;
248 chan
->chid
= args
.nv50
.chid
;
253 nouveau_channel_del(pchan
);
258 nouveau_channel_dma(struct nouveau_drm
*drm
, struct nvif_device
*device
,
259 u32 handle
, struct nouveau_channel
**pchan
)
261 static const u16 oclasses
[] = { NV40_CHANNEL_DMA
,
266 const u16
*oclass
= oclasses
;
267 struct nv03_channel_dma_v0 args
;
268 struct nouveau_channel
*chan
;
271 /* allocate dma push buffer */
272 ret
= nouveau_channel_prep(drm
, device
, handle
, 0x10000, &chan
);
277 /* create channel object */
279 args
.pushbuf
= nvif_handle(&chan
->push
.ctxdma
);
280 args
.offset
= chan
->push
.vma
.offset
;
283 ret
= nvif_object_init(&device
->object
, handle
, *oclass
++,
284 &args
, sizeof(args
), &chan
->user
);
286 chan
->chid
= args
.chid
;
289 } while (ret
&& *oclass
);
291 nouveau_channel_del(pchan
);
296 nouveau_channel_init(struct nouveau_channel
*chan
, u32 vram
, u32 gart
)
298 struct nvif_device
*device
= chan
->device
;
299 struct nouveau_cli
*cli
= (void *)chan
->user
.client
;
300 struct nvkm_mmu
*mmu
= nvxx_mmu(device
);
301 struct nv_dma_v0 args
= {};
304 nvif_object_map(&chan
->user
);
306 /* allocate dma objects to cover all allowed vram, and gart */
307 if (device
->info
.family
< NV_DEVICE_INFO_V0_FERMI
) {
308 if (device
->info
.family
>= NV_DEVICE_INFO_V0_TESLA
) {
309 args
.target
= NV_DMA_V0_TARGET_VM
;
310 args
.access
= NV_DMA_V0_ACCESS_VM
;
312 args
.limit
= cli
->vm
->mmu
->limit
- 1;
314 args
.target
= NV_DMA_V0_TARGET_VRAM
;
315 args
.access
= NV_DMA_V0_ACCESS_RDWR
;
317 args
.limit
= device
->info
.ram_user
- 1;
320 ret
= nvif_object_init(&chan
->user
, vram
, NV_DMA_IN_MEMORY
,
321 &args
, sizeof(args
), &chan
->vram
);
325 if (device
->info
.family
>= NV_DEVICE_INFO_V0_TESLA
) {
326 args
.target
= NV_DMA_V0_TARGET_VM
;
327 args
.access
= NV_DMA_V0_ACCESS_VM
;
329 args
.limit
= cli
->vm
->mmu
->limit
- 1;
331 if (chan
->drm
->agp
.bridge
) {
332 args
.target
= NV_DMA_V0_TARGET_AGP
;
333 args
.access
= NV_DMA_V0_ACCESS_RDWR
;
334 args
.start
= chan
->drm
->agp
.base
;
335 args
.limit
= chan
->drm
->agp
.base
+
336 chan
->drm
->agp
.size
- 1;
338 args
.target
= NV_DMA_V0_TARGET_VM
;
339 args
.access
= NV_DMA_V0_ACCESS_RDWR
;
341 args
.limit
= mmu
->limit
- 1;
344 ret
= nvif_object_init(&chan
->user
, gart
, NV_DMA_IN_MEMORY
,
345 &args
, sizeof(args
), &chan
->gart
);
350 /* initialise dma tracking parameters */
351 switch (chan
->user
.oclass
& 0x00ff) {
354 chan
->user_put
= 0x40;
355 chan
->user_get
= 0x44;
356 chan
->dma
.max
= (0x10000 / 4) - 2;
359 chan
->user_put
= 0x40;
360 chan
->user_get
= 0x44;
361 chan
->user_get_hi
= 0x60;
362 chan
->dma
.ib_base
= 0x10000 / 4;
363 chan
->dma
.ib_max
= (0x02000 / 8) - 1;
364 chan
->dma
.ib_put
= 0;
365 chan
->dma
.ib_free
= chan
->dma
.ib_max
- chan
->dma
.ib_put
;
366 chan
->dma
.max
= chan
->dma
.ib_base
;
371 chan
->dma
.cur
= chan
->dma
.put
;
372 chan
->dma
.free
= chan
->dma
.max
- chan
->dma
.cur
;
374 ret
= RING_SPACE(chan
, NOUVEAU_DMA_SKIPS
);
378 for (i
= 0; i
< NOUVEAU_DMA_SKIPS
; i
++)
379 OUT_RING(chan
, 0x00000000);
381 /* allocate software object class (used for fences on <= nv05) */
382 if (device
->info
.family
< NV_DEVICE_INFO_V0_CELSIUS
) {
383 ret
= nvif_object_init(&chan
->user
, 0x006e,
384 NVIF_IOCTL_NEW_V0_SW_NV04
,
385 NULL
, 0, &chan
->nvsw
);
389 ret
= RING_SPACE(chan
, 2);
393 BEGIN_NV04(chan
, NvSubSw
, 0x0000, 1);
394 OUT_RING (chan
, chan
->nvsw
.handle
);
398 /* initialise synchronisation */
399 return nouveau_fence(chan
->drm
)->context_new(chan
);
403 nouveau_channel_new(struct nouveau_drm
*drm
, struct nvif_device
*device
,
404 u32 handle
, u32 arg0
, u32 arg1
,
405 struct nouveau_channel
**pchan
)
407 struct nouveau_cli
*cli
= (void *)device
->object
.client
;
411 /* hack until fencenv50 is fixed, and agp access relaxed */
412 super
= cli
->base
.super
;
413 cli
->base
.super
= true;
415 ret
= nouveau_channel_ind(drm
, device
, handle
, arg0
, pchan
);
417 NV_PRINTK(dbg
, cli
, "ib channel create, %d\n", ret
);
418 ret
= nouveau_channel_dma(drm
, device
, handle
, pchan
);
420 NV_PRINTK(dbg
, cli
, "dma channel create, %d\n", ret
);
425 ret
= nouveau_channel_init(*pchan
, arg0
, arg1
);
427 NV_PRINTK(err
, cli
, "channel failed to initialise, %d\n", ret
);
428 nouveau_channel_del(pchan
);
432 cli
->base
.super
= super
;