2 * Copyright 2012 Red Hat Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
26 #include <core/client.h>
27 #include <core/engctx.h>
28 #include <core/enum.h>
29 #include <core/handle.h>
30 #include <subdev/bar.h>
31 #include <subdev/fb.h>
32 #include <subdev/mmu.h>
33 #include <subdev/timer.h>
35 #include <nvif/class.h>
36 #include <nvif/unpack.h>
38 #define _(a,b) { (a), ((1ULL << (a)) | (b)) }
43 _(NVDEV_ENGINE_GR
, (1ULL << NVDEV_ENGINE_SW
) |
44 (1ULL << NVDEV_ENGINE_CE2
)),
45 _(NVDEV_ENGINE_MSPDEC
, 0),
46 _(NVDEV_ENGINE_MSPPP
, 0),
47 _(NVDEV_ENGINE_MSVLD
, 0),
48 _(NVDEV_ENGINE_CE0
, 0),
49 _(NVDEV_ENGINE_CE1
, 0),
50 _(NVDEV_ENGINE_MSENC
, 0),
53 #define FIFO_ENGINE_NR ARRAY_SIZE(fifo_engine)
55 struct gk104_fifo_engn
{
56 struct nvkm_gpuobj
*runlist
[2];
58 wait_queue_head_t wait
;
62 struct nvkm_fifo base
;
64 struct work_struct fault
;
67 struct gk104_fifo_engn engine
[FIFO_ENGINE_NR
];
69 struct nvkm_gpuobj
*mem
;
75 struct gk104_fifo_base
{
76 struct nvkm_fifo_base base
;
77 struct nvkm_gpuobj
*pgd
;
81 struct gk104_fifo_chan
{
82 struct nvkm_fifo_chan base
;
91 /*******************************************************************************
92 * FIFO channel objects
93 ******************************************************************************/
96 gk104_fifo_runlist_update(struct gk104_fifo
*fifo
, u32 engine
)
98 struct gk104_fifo_engn
*engn
= &fifo
->engine
[engine
];
99 struct nvkm_subdev
*subdev
= &fifo
->base
.engine
.subdev
;
100 struct nvkm_device
*device
= subdev
->device
;
101 struct nvkm_bar
*bar
= device
->bar
;
102 struct nvkm_gpuobj
*cur
;
105 mutex_lock(&nv_subdev(fifo
)->mutex
);
106 cur
= engn
->runlist
[engn
->cur_runlist
];
107 engn
->cur_runlist
= !engn
->cur_runlist
;
110 for (i
= 0, p
= 0; i
< fifo
->base
.max
; i
++) {
111 struct gk104_fifo_chan
*chan
= (void *)fifo
->base
.channel
[i
];
112 if (chan
&& chan
->state
== RUNNING
&& chan
->engine
== engine
) {
113 nvkm_wo32(cur
, p
+ 0, i
);
114 nvkm_wo32(cur
, p
+ 4, 0x00000000);
121 nvkm_wr32(device
, 0x002270, cur
->addr
>> 12);
122 nvkm_wr32(device
, 0x002274, (engine
<< 20) | (p
>> 3));
124 if (wait_event_timeout(engn
->wait
, !(nvkm_rd32(device
, 0x002284 +
125 (engine
* 0x08)) & 0x00100000),
126 msecs_to_jiffies(2000)) == 0)
127 nvkm_error(subdev
, "runlist %d update timeout\n", engine
);
128 mutex_unlock(&nv_subdev(fifo
)->mutex
);
132 gk104_fifo_context_attach(struct nvkm_object
*parent
,
133 struct nvkm_object
*object
)
135 struct nvkm_bar
*bar
= nvkm_bar(parent
);
136 struct gk104_fifo_base
*base
= (void *)parent
->parent
;
137 struct nvkm_gpuobj
*engn
= &base
->base
.gpuobj
;
138 struct nvkm_engctx
*ectx
= (void *)object
;
142 switch (nv_engidx(object
->engine
)) {
143 case NVDEV_ENGINE_SW
:
145 case NVDEV_ENGINE_CE0
:
146 case NVDEV_ENGINE_CE1
:
147 case NVDEV_ENGINE_CE2
:
148 nv_engctx(ectx
)->addr
= nv_gpuobj(base
)->addr
>> 12;
150 case NVDEV_ENGINE_GR
: addr
= 0x0210; break;
151 case NVDEV_ENGINE_MSVLD
: addr
= 0x0270; break;
152 case NVDEV_ENGINE_MSPDEC
: addr
= 0x0250; break;
153 case NVDEV_ENGINE_MSPPP
: addr
= 0x0260; break;
158 if (!ectx
->vma
.node
) {
159 ret
= nvkm_gpuobj_map_vm(nv_gpuobj(ectx
), base
->vm
,
160 NV_MEM_ACCESS_RW
, &ectx
->vma
);
164 nv_engctx(ectx
)->addr
= nv_gpuobj(base
)->addr
>> 12;
168 nvkm_wo32(engn
, addr
+ 0x00, lower_32_bits(ectx
->vma
.offset
) | 4);
169 nvkm_wo32(engn
, addr
+ 0x04, upper_32_bits(ectx
->vma
.offset
));
176 gk104_fifo_chan_kick(struct gk104_fifo_chan
*chan
)
178 struct nvkm_object
*obj
= (void *)chan
;
179 struct gk104_fifo
*fifo
= (void *)obj
->engine
;
180 struct nvkm_subdev
*subdev
= &fifo
->base
.engine
.subdev
;
181 struct nvkm_device
*device
= subdev
->device
;
183 nvkm_wr32(device
, 0x002634, chan
->base
.chid
);
184 if (nvkm_msec(device
, 2000,
185 if (!(nvkm_rd32(device
, 0x002634) & 0x00100000))
188 nvkm_error(subdev
, "channel %d [%s] kick timeout\n",
189 chan
->base
.chid
, nvkm_client_name(chan
));
197 gk104_fifo_context_detach(struct nvkm_object
*parent
, bool suspend
,
198 struct nvkm_object
*object
)
200 struct nvkm_bar
*bar
= nvkm_bar(parent
);
201 struct gk104_fifo_base
*base
= (void *)parent
->parent
;
202 struct gk104_fifo_chan
*chan
= (void *)parent
;
203 struct nvkm_gpuobj
*engn
= &base
->base
.gpuobj
;
207 switch (nv_engidx(object
->engine
)) {
208 case NVDEV_ENGINE_SW
: return 0;
209 case NVDEV_ENGINE_CE0
:
210 case NVDEV_ENGINE_CE1
:
211 case NVDEV_ENGINE_CE2
: addr
= 0x0000; break;
212 case NVDEV_ENGINE_GR
: addr
= 0x0210; break;
213 case NVDEV_ENGINE_MSVLD
: addr
= 0x0270; break;
214 case NVDEV_ENGINE_MSPDEC
: addr
= 0x0250; break;
215 case NVDEV_ENGINE_MSPPP
: addr
= 0x0260; break;
220 ret
= gk104_fifo_chan_kick(chan
);
226 nvkm_wo32(engn
, addr
+ 0x00, 0x00000000);
227 nvkm_wo32(engn
, addr
+ 0x04, 0x00000000);
236 gk104_fifo_chan_ctor(struct nvkm_object
*parent
, struct nvkm_object
*engine
,
237 struct nvkm_oclass
*oclass
, void *data
, u32 size
,
238 struct nvkm_object
**pobject
)
241 struct kepler_channel_gpfifo_a_v0 v0
;
243 struct nvkm_bar
*bar
= nvkm_bar(parent
);
244 struct gk104_fifo
*fifo
= (void *)engine
;
245 struct gk104_fifo_base
*base
= (void *)parent
;
246 struct gk104_fifo_chan
*chan
;
247 struct nvkm_subdev
*subdev
= &fifo
->base
.engine
.subdev
;
248 struct nvkm_gpuobj
*ramfc
= &base
->base
.gpuobj
;
249 u64 usermem
, ioffset
, ilength
;
252 nvif_ioctl(parent
, "create channel gpfifo size %d\n", size
);
253 if (nvif_unpack(args
->v0
, 0, 0, false)) {
254 nvif_ioctl(parent
, "create channel gpfifo vers %d pushbuf %llx "
255 "ioffset %016llx ilength %08x engine %08x\n",
256 args
->v0
.version
, args
->v0
.pushbuf
, args
->v0
.ioffset
,
257 args
->v0
.ilength
, args
->v0
.engine
);
261 for (i
= 0; i
< FIFO_ENGINE_NR
; i
++) {
262 if (args
->v0
.engine
& (1 << i
)) {
263 if (nvkm_engine(parent
, fifo_engine
[i
].subdev
)) {
264 args
->v0
.engine
= (1 << i
);
270 if (i
== FIFO_ENGINE_NR
) {
271 nvkm_error(subdev
, "unsupported engines %08x\n",
276 ret
= nvkm_fifo_channel_create(parent
, engine
, oclass
, 1,
277 fifo
->user
.bar
.offset
, 0x200,
279 fifo_engine
[i
].mask
, &chan
);
280 *pobject
= nv_object(chan
);
284 args
->v0
.chid
= chan
->base
.chid
;
286 nv_parent(chan
)->context_attach
= gk104_fifo_context_attach
;
287 nv_parent(chan
)->context_detach
= gk104_fifo_context_detach
;
290 usermem
= chan
->base
.chid
* 0x200;
291 ioffset
= args
->v0
.ioffset
;
292 ilength
= order_base_2(args
->v0
.ilength
/ 8);
294 nvkm_kmap(fifo
->user
.mem
);
295 for (i
= 0; i
< 0x200; i
+= 4)
296 nvkm_wo32(fifo
->user
.mem
, usermem
+ i
, 0x00000000);
297 nvkm_done(fifo
->user
.mem
);
300 nvkm_wo32(ramfc
, 0x08, lower_32_bits(fifo
->user
.mem
->addr
+ usermem
));
301 nvkm_wo32(ramfc
, 0x0c, upper_32_bits(fifo
->user
.mem
->addr
+ usermem
));
302 nvkm_wo32(ramfc
, 0x10, 0x0000face);
303 nvkm_wo32(ramfc
, 0x30, 0xfffff902);
304 nvkm_wo32(ramfc
, 0x48, lower_32_bits(ioffset
));
305 nvkm_wo32(ramfc
, 0x4c, upper_32_bits(ioffset
) | (ilength
<< 16));
306 nvkm_wo32(ramfc
, 0x84, 0x20400000);
307 nvkm_wo32(ramfc
, 0x94, 0x30000001);
308 nvkm_wo32(ramfc
, 0x9c, 0x00000100);
309 nvkm_wo32(ramfc
, 0xac, 0x0000001f);
310 nvkm_wo32(ramfc
, 0xe8, chan
->base
.chid
);
311 nvkm_wo32(ramfc
, 0xb8, 0xf8000000);
312 nvkm_wo32(ramfc
, 0xf8, 0x10003080); /* 0x002310 */
313 nvkm_wo32(ramfc
, 0xfc, 0x10000010); /* 0x002350 */
320 gk104_fifo_chan_init(struct nvkm_object
*object
)
322 struct nvkm_gpuobj
*base
= nv_gpuobj(object
->parent
);
323 struct gk104_fifo
*fifo
= (void *)object
->engine
;
324 struct gk104_fifo_chan
*chan
= (void *)object
;
325 struct nvkm_device
*device
= fifo
->base
.engine
.subdev
.device
;
326 u32 chid
= chan
->base
.chid
;
329 ret
= nvkm_fifo_channel_init(&chan
->base
);
333 nvkm_mask(device
, 0x800004 + (chid
* 8), 0x000f0000, chan
->engine
<< 16);
334 nvkm_wr32(device
, 0x800000 + (chid
* 8), 0x80000000 | base
->addr
>> 12);
336 if (chan
->state
== STOPPED
&& (chan
->state
= RUNNING
) == RUNNING
) {
337 nvkm_mask(device
, 0x800004 + (chid
* 8), 0x00000400, 0x00000400);
338 gk104_fifo_runlist_update(fifo
, chan
->engine
);
339 nvkm_mask(device
, 0x800004 + (chid
* 8), 0x00000400, 0x00000400);
346 gk104_fifo_chan_fini(struct nvkm_object
*object
, bool suspend
)
348 struct gk104_fifo
*fifo
= (void *)object
->engine
;
349 struct gk104_fifo_chan
*chan
= (void *)object
;
350 struct nvkm_device
*device
= fifo
->base
.engine
.subdev
.device
;
351 u32 chid
= chan
->base
.chid
;
353 if (chan
->state
== RUNNING
&& (chan
->state
= STOPPED
) == STOPPED
) {
354 nvkm_mask(device
, 0x800004 + (chid
* 8), 0x00000800, 0x00000800);
355 gk104_fifo_runlist_update(fifo
, chan
->engine
);
358 nvkm_wr32(device
, 0x800000 + (chid
* 8), 0x00000000);
359 return nvkm_fifo_channel_fini(&chan
->base
, suspend
);
363 gk104_fifo_chan_ofuncs
= {
364 .ctor
= gk104_fifo_chan_ctor
,
365 .dtor
= _nvkm_fifo_channel_dtor
,
366 .init
= gk104_fifo_chan_init
,
367 .fini
= gk104_fifo_chan_fini
,
368 .map
= _nvkm_fifo_channel_map
,
369 .rd32
= _nvkm_fifo_channel_rd32
,
370 .wr32
= _nvkm_fifo_channel_wr32
,
371 .ntfy
= _nvkm_fifo_channel_ntfy
374 static struct nvkm_oclass
375 gk104_fifo_sclass
[] = {
376 { KEPLER_CHANNEL_GPFIFO_A
, &gk104_fifo_chan_ofuncs
},
380 /*******************************************************************************
381 * FIFO context - instmem heap and vm setup
382 ******************************************************************************/
385 gk104_fifo_context_ctor(struct nvkm_object
*parent
, struct nvkm_object
*engine
,
386 struct nvkm_oclass
*oclass
, void *data
, u32 size
,
387 struct nvkm_object
**pobject
)
389 struct gk104_fifo_base
*base
;
392 ret
= nvkm_fifo_context_create(parent
, engine
, oclass
, NULL
, 0x1000,
393 0x1000, NVOBJ_FLAG_ZERO_ALLOC
, &base
);
394 *pobject
= nv_object(base
);
398 ret
= nvkm_gpuobj_new(nv_object(base
), NULL
, 0x10000, 0x1000, 0,
403 nvkm_kmap(&base
->base
.gpuobj
);
404 nvkm_wo32(&base
->base
.gpuobj
, 0x0200, lower_32_bits(base
->pgd
->addr
));
405 nvkm_wo32(&base
->base
.gpuobj
, 0x0204, upper_32_bits(base
->pgd
->addr
));
406 nvkm_wo32(&base
->base
.gpuobj
, 0x0208, 0xffffffff);
407 nvkm_wo32(&base
->base
.gpuobj
, 0x020c, 0x000000ff);
408 nvkm_done(&base
->base
.gpuobj
);
410 ret
= nvkm_vm_ref(nvkm_client(parent
)->vm
, &base
->vm
, base
->pgd
);
418 gk104_fifo_context_dtor(struct nvkm_object
*object
)
420 struct gk104_fifo_base
*base
= (void *)object
;
421 nvkm_vm_ref(NULL
, &base
->vm
, base
->pgd
);
422 nvkm_gpuobj_ref(NULL
, &base
->pgd
);
423 nvkm_fifo_context_destroy(&base
->base
);
426 static struct nvkm_oclass
427 gk104_fifo_cclass
= {
428 .handle
= NV_ENGCTX(FIFO
, 0xe0),
429 .ofuncs
= &(struct nvkm_ofuncs
) {
430 .ctor
= gk104_fifo_context_ctor
,
431 .dtor
= gk104_fifo_context_dtor
,
432 .init
= _nvkm_fifo_context_init
,
433 .fini
= _nvkm_fifo_context_fini
,
434 .rd32
= _nvkm_fifo_context_rd32
,
435 .wr32
= _nvkm_fifo_context_wr32
,
439 /*******************************************************************************
441 ******************************************************************************/
444 gk104_fifo_engidx(struct gk104_fifo
*fifo
, u32 engn
)
447 case NVDEV_ENGINE_GR
:
448 case NVDEV_ENGINE_CE2
: engn
= 0; break;
449 case NVDEV_ENGINE_MSVLD
: engn
= 1; break;
450 case NVDEV_ENGINE_MSPPP
: engn
= 2; break;
451 case NVDEV_ENGINE_MSPDEC
: engn
= 3; break;
452 case NVDEV_ENGINE_CE0
: engn
= 4; break;
453 case NVDEV_ENGINE_CE1
: engn
= 5; break;
454 case NVDEV_ENGINE_MSENC
: engn
= 6; break;
462 static inline struct nvkm_engine
*
463 gk104_fifo_engine(struct gk104_fifo
*fifo
, u32 engn
)
465 if (engn
>= ARRAY_SIZE(fifo_engine
))
467 return nvkm_engine(fifo
, fifo_engine
[engn
].subdev
);
471 gk104_fifo_recover_work(struct work_struct
*work
)
473 struct gk104_fifo
*fifo
= container_of(work
, typeof(*fifo
), fault
);
474 struct nvkm_device
*device
= fifo
->base
.engine
.subdev
.device
;
475 struct nvkm_object
*engine
;
480 spin_lock_irqsave(&fifo
->base
.lock
, flags
);
483 spin_unlock_irqrestore(&fifo
->base
.lock
, flags
);
485 for (todo
= mask
; engn
= __ffs64(todo
), todo
; todo
&= ~(1 << engn
))
486 engm
|= 1 << gk104_fifo_engidx(fifo
, engn
);
487 nvkm_mask(device
, 0x002630, engm
, engm
);
489 for (todo
= mask
; engn
= __ffs64(todo
), todo
; todo
&= ~(1 << engn
)) {
490 if ((engine
= (void *)nvkm_engine(fifo
, engn
))) {
491 nv_ofuncs(engine
)->fini(engine
, false);
492 WARN_ON(nv_ofuncs(engine
)->init(engine
));
494 gk104_fifo_runlist_update(fifo
, gk104_fifo_engidx(fifo
, engn
));
497 nvkm_wr32(device
, 0x00262c, engm
);
498 nvkm_mask(device
, 0x002630, engm
, 0x00000000);
502 gk104_fifo_recover(struct gk104_fifo
*fifo
, struct nvkm_engine
*engine
,
503 struct gk104_fifo_chan
*chan
)
505 struct nvkm_subdev
*subdev
= &fifo
->base
.engine
.subdev
;
506 struct nvkm_device
*device
= subdev
->device
;
507 u32 chid
= chan
->base
.chid
;
510 nvkm_error(subdev
, "%s engine fault on channel %d, recovering...\n",
511 nv_subdev(engine
)->name
, chid
);
513 nvkm_mask(device
, 0x800004 + (chid
* 0x08), 0x00000800, 0x00000800);
514 chan
->state
= KILLED
;
516 spin_lock_irqsave(&fifo
->base
.lock
, flags
);
517 fifo
->mask
|= 1ULL << nv_engidx(engine
);
518 spin_unlock_irqrestore(&fifo
->base
.lock
, flags
);
519 schedule_work(&fifo
->fault
);
523 gk104_fifo_swmthd(struct gk104_fifo
*fifo
, u32 chid
, u32 mthd
, u32 data
)
525 struct gk104_fifo_chan
*chan
= NULL
;
526 struct nvkm_handle
*bind
;
530 spin_lock_irqsave(&fifo
->base
.lock
, flags
);
531 if (likely(chid
>= fifo
->base
.min
&& chid
<= fifo
->base
.max
))
532 chan
= (void *)fifo
->base
.channel
[chid
];
536 bind
= nvkm_namedb_get_class(nv_namedb(chan
), 0x906e);
538 if (!mthd
|| !nv_call(bind
->object
, mthd
, data
))
540 nvkm_namedb_put(bind
);
544 spin_unlock_irqrestore(&fifo
->base
.lock
, flags
);
548 static const struct nvkm_enum
549 gk104_fifo_bind_reason
[] = {
550 { 0x01, "BIND_NOT_UNBOUND" },
551 { 0x02, "SNOOP_WITHOUT_BAR1" },
552 { 0x03, "UNBIND_WHILE_RUNNING" },
553 { 0x05, "INVALID_RUNLIST" },
554 { 0x06, "INVALID_CTX_TGT" },
555 { 0x0b, "UNBIND_WHILE_PARKED" },
560 gk104_fifo_intr_bind(struct gk104_fifo
*fifo
)
562 struct nvkm_subdev
*subdev
= &fifo
->base
.engine
.subdev
;
563 struct nvkm_device
*device
= subdev
->device
;
564 u32 intr
= nvkm_rd32(device
, 0x00252c);
565 u32 code
= intr
& 0x000000ff;
566 const struct nvkm_enum
*en
=
567 nvkm_enum_find(gk104_fifo_bind_reason
, code
);
569 nvkm_error(subdev
, "BIND_ERROR %02x [%s]\n", code
, en
? en
->name
: "");
572 static const struct nvkm_enum
573 gk104_fifo_sched_reason
[] = {
574 { 0x0a, "CTXSW_TIMEOUT" },
579 gk104_fifo_intr_sched_ctxsw(struct gk104_fifo
*fifo
)
581 struct nvkm_device
*device
= fifo
->base
.engine
.subdev
.device
;
582 struct nvkm_engine
*engine
;
583 struct gk104_fifo_chan
*chan
;
586 for (engn
= 0; engn
< ARRAY_SIZE(fifo_engine
); engn
++) {
587 u32 stat
= nvkm_rd32(device
, 0x002640 + (engn
* 0x04));
588 u32 busy
= (stat
& 0x80000000);
589 u32 next
= (stat
& 0x07ff0000) >> 16;
590 u32 chsw
= (stat
& 0x00008000);
591 u32 save
= (stat
& 0x00004000);
592 u32 load
= (stat
& 0x00002000);
593 u32 prev
= (stat
& 0x000007ff);
594 u32 chid
= load
? next
: prev
;
598 if (!(chan
= (void *)fifo
->base
.channel
[chid
]))
600 if (!(engine
= gk104_fifo_engine(fifo
, engn
)))
602 gk104_fifo_recover(fifo
, engine
, chan
);
608 gk104_fifo_intr_sched(struct gk104_fifo
*fifo
)
610 struct nvkm_subdev
*subdev
= &fifo
->base
.engine
.subdev
;
611 struct nvkm_device
*device
= subdev
->device
;
612 u32 intr
= nvkm_rd32(device
, 0x00254c);
613 u32 code
= intr
& 0x000000ff;
614 const struct nvkm_enum
*en
=
615 nvkm_enum_find(gk104_fifo_sched_reason
, code
);
617 nvkm_error(subdev
, "SCHED_ERROR %02x [%s]\n", code
, en
? en
->name
: "");
621 gk104_fifo_intr_sched_ctxsw(fifo
);
629 gk104_fifo_intr_chsw(struct gk104_fifo
*fifo
)
631 struct nvkm_subdev
*subdev
= &fifo
->base
.engine
.subdev
;
632 struct nvkm_device
*device
= subdev
->device
;
633 u32 stat
= nvkm_rd32(device
, 0x00256c);
634 nvkm_error(subdev
, "CHSW_ERROR %08x\n", stat
);
635 nvkm_wr32(device
, 0x00256c, stat
);
639 gk104_fifo_intr_dropped_fault(struct gk104_fifo
*fifo
)
641 struct nvkm_subdev
*subdev
= &fifo
->base
.engine
.subdev
;
642 struct nvkm_device
*device
= subdev
->device
;
643 u32 stat
= nvkm_rd32(device
, 0x00259c);
644 nvkm_error(subdev
, "DROPPED_MMU_FAULT %08x\n", stat
);
647 static const struct nvkm_enum
648 gk104_fifo_fault_engine
[] = {
649 { 0x00, "GR", NULL
, NVDEV_ENGINE_GR
},
650 { 0x03, "IFB", NULL
, NVDEV_ENGINE_IFB
},
651 { 0x04, "BAR1", NULL
, NVDEV_SUBDEV_BAR
},
652 { 0x05, "BAR3", NULL
, NVDEV_SUBDEV_INSTMEM
},
653 { 0x07, "PBDMA0", NULL
, NVDEV_ENGINE_FIFO
},
654 { 0x08, "PBDMA1", NULL
, NVDEV_ENGINE_FIFO
},
655 { 0x09, "PBDMA2", NULL
, NVDEV_ENGINE_FIFO
},
656 { 0x10, "MSVLD", NULL
, NVDEV_ENGINE_MSVLD
},
657 { 0x11, "MSPPP", NULL
, NVDEV_ENGINE_MSPPP
},
659 { 0x14, "MSPDEC", NULL
, NVDEV_ENGINE_MSPDEC
},
660 { 0x15, "CE0", NULL
, NVDEV_ENGINE_CE0
},
661 { 0x16, "CE1", NULL
, NVDEV_ENGINE_CE1
},
663 { 0x19, "MSENC", NULL
, NVDEV_ENGINE_MSENC
},
664 { 0x1b, "CE2", NULL
, NVDEV_ENGINE_CE2
},
668 static const struct nvkm_enum
669 gk104_fifo_fault_reason
[] = {
671 { 0x01, "PDE_SIZE" },
673 { 0x03, "VA_LIMIT_VIOLATION" },
674 { 0x04, "UNBOUND_INST_BLOCK" },
675 { 0x05, "PRIV_VIOLATION" },
676 { 0x06, "RO_VIOLATION" },
677 { 0x07, "WO_VIOLATION" },
678 { 0x08, "PITCH_MASK_VIOLATION" },
679 { 0x09, "WORK_CREATION" },
680 { 0x0a, "UNSUPPORTED_APERTURE" },
681 { 0x0b, "COMPRESSION_FAILURE" },
682 { 0x0c, "UNSUPPORTED_KIND" },
683 { 0x0d, "REGION_VIOLATION" },
684 { 0x0e, "BOTH_PTES_VALID" },
685 { 0x0f, "INFO_TYPE_POISONED" },
689 static const struct nvkm_enum
690 gk104_fifo_fault_hubclient
[] = {
698 { 0x07, "HOST_CPU" },
699 { 0x08, "HOST_CPU_NB" },
710 { 0x13, "RASTERTWOD" },
726 static const struct nvkm_enum
727 gk104_fifo_fault_gpcclient
[] = {
728 { 0x00, "L1_0" }, { 0x01, "T1_0" }, { 0x02, "PE_0" },
729 { 0x03, "L1_1" }, { 0x04, "T1_1" }, { 0x05, "PE_1" },
730 { 0x06, "L1_2" }, { 0x07, "T1_2" }, { 0x08, "PE_2" },
731 { 0x09, "L1_3" }, { 0x0a, "T1_3" }, { 0x0b, "PE_3" },
739 { 0x13, "L1_4" }, { 0x14, "T1_4" }, { 0x15, "PE_4" },
740 { 0x16, "L1_5" }, { 0x17, "T1_5" }, { 0x18, "PE_5" },
741 { 0x19, "L1_6" }, { 0x1a, "T1_6" }, { 0x1b, "PE_6" },
742 { 0x1c, "L1_7" }, { 0x1d, "T1_7" }, { 0x1e, "PE_7" },
744 { 0x20, "LTP_UTLB_0" },
745 { 0x21, "LTP_UTLB_1" },
746 { 0x22, "LTP_UTLB_2" },
747 { 0x23, "LTP_UTLB_3" },
748 { 0x24, "GPC_RGG_UTLB" },
753 gk104_fifo_intr_fault(struct gk104_fifo
*fifo
, int unit
)
755 struct nvkm_subdev
*subdev
= &fifo
->base
.engine
.subdev
;
756 struct nvkm_device
*device
= subdev
->device
;
757 u32 inst
= nvkm_rd32(device
, 0x002800 + (unit
* 0x10));
758 u32 valo
= nvkm_rd32(device
, 0x002804 + (unit
* 0x10));
759 u32 vahi
= nvkm_rd32(device
, 0x002808 + (unit
* 0x10));
760 u32 stat
= nvkm_rd32(device
, 0x00280c + (unit
* 0x10));
761 u32 gpc
= (stat
& 0x1f000000) >> 24;
762 u32 client
= (stat
& 0x00001f00) >> 8;
763 u32 write
= (stat
& 0x00000080);
764 u32 hub
= (stat
& 0x00000040);
765 u32 reason
= (stat
& 0x0000000f);
766 struct nvkm_object
*engctx
= NULL
, *object
;
767 struct nvkm_engine
*engine
= NULL
;
768 const struct nvkm_enum
*er
, *eu
, *ec
;
771 er
= nvkm_enum_find(gk104_fifo_fault_reason
, reason
);
772 eu
= nvkm_enum_find(gk104_fifo_fault_engine
, unit
);
774 ec
= nvkm_enum_find(gk104_fifo_fault_hubclient
, client
);
776 ec
= nvkm_enum_find(gk104_fifo_fault_gpcclient
, client
);
777 snprintf(gpcid
, sizeof(gpcid
), "GPC%d/", gpc
);
782 case NVDEV_SUBDEV_BAR
:
783 nvkm_mask(device
, 0x001704, 0x00000000, 0x00000000);
785 case NVDEV_SUBDEV_INSTMEM
:
786 nvkm_mask(device
, 0x001714, 0x00000000, 0x00000000);
788 case NVDEV_ENGINE_IFB
:
789 nvkm_mask(device
, 0x001718, 0x00000000, 0x00000000);
792 engine
= nvkm_engine(fifo
, eu
->data2
);
794 engctx
= nvkm_engctx_get(engine
, inst
);
800 "%s fault at %010llx engine %02x [%s] client %02x [%s%s] "
801 "reason %02x [%s] on channel %d [%010llx %s]\n",
802 write
? "write" : "read", (u64
)vahi
<< 32 | valo
,
803 unit
, eu
? eu
->name
: "", client
, gpcid
, ec
? ec
->name
: "",
804 reason
, er
? er
->name
: "", -1, (u64
)inst
<< 12,
805 nvkm_client_name(engctx
));
809 switch (nv_mclass(object
)) {
810 case KEPLER_CHANNEL_GPFIFO_A
:
811 case MAXWELL_CHANNEL_GPFIFO_A
:
812 gk104_fifo_recover(fifo
, engine
, (void *)object
);
815 object
= object
->parent
;
818 nvkm_engctx_put(engctx
);
821 static const struct nvkm_bitfield gk104_fifo_pbdma_intr_0
[] = {
822 { 0x00000001, "MEMREQ" },
823 { 0x00000002, "MEMACK_TIMEOUT" },
824 { 0x00000004, "MEMACK_EXTRA" },
825 { 0x00000008, "MEMDAT_TIMEOUT" },
826 { 0x00000010, "MEMDAT_EXTRA" },
827 { 0x00000020, "MEMFLUSH" },
828 { 0x00000040, "MEMOP" },
829 { 0x00000080, "LBCONNECT" },
830 { 0x00000100, "LBREQ" },
831 { 0x00000200, "LBACK_TIMEOUT" },
832 { 0x00000400, "LBACK_EXTRA" },
833 { 0x00000800, "LBDAT_TIMEOUT" },
834 { 0x00001000, "LBDAT_EXTRA" },
835 { 0x00002000, "GPFIFO" },
836 { 0x00004000, "GPPTR" },
837 { 0x00008000, "GPENTRY" },
838 { 0x00010000, "GPCRC" },
839 { 0x00020000, "PBPTR" },
840 { 0x00040000, "PBENTRY" },
841 { 0x00080000, "PBCRC" },
842 { 0x00100000, "XBARCONNECT" },
843 { 0x00200000, "METHOD" },
844 { 0x00400000, "METHODCRC" },
845 { 0x00800000, "DEVICE" },
846 { 0x02000000, "SEMAPHORE" },
847 { 0x04000000, "ACQUIRE" },
848 { 0x08000000, "PRI" },
849 { 0x20000000, "NO_CTXSW_SEG" },
850 { 0x40000000, "PBSEG" },
851 { 0x80000000, "SIGNATURE" },
856 gk104_fifo_intr_pbdma_0(struct gk104_fifo
*fifo
, int unit
)
858 struct nvkm_subdev
*subdev
= &fifo
->base
.engine
.subdev
;
859 struct nvkm_device
*device
= subdev
->device
;
860 u32 mask
= nvkm_rd32(device
, 0x04010c + (unit
* 0x2000));
861 u32 stat
= nvkm_rd32(device
, 0x040108 + (unit
* 0x2000)) & mask
;
862 u32 addr
= nvkm_rd32(device
, 0x0400c0 + (unit
* 0x2000));
863 u32 data
= nvkm_rd32(device
, 0x0400c4 + (unit
* 0x2000));
864 u32 chid
= nvkm_rd32(device
, 0x040120 + (unit
* 0x2000)) & 0xfff;
865 u32 subc
= (addr
& 0x00070000) >> 16;
866 u32 mthd
= (addr
& 0x00003ffc);
870 if (stat
& 0x00800000) {
871 if (!gk104_fifo_swmthd(fifo
, chid
, mthd
, data
))
873 nvkm_wr32(device
, 0x0400c0 + (unit
* 0x2000), 0x80600008);
877 nvkm_snprintbf(msg
, sizeof(msg
), gk104_fifo_pbdma_intr_0
, show
);
878 nvkm_error(subdev
, "PBDMA%d: %08x [%s] ch %d [%s] subc %d "
879 "mthd %04x data %08x\n",
880 unit
, show
, msg
, chid
,
881 nvkm_client_name_for_fifo_chid(&fifo
->base
, chid
),
885 nvkm_wr32(device
, 0x040108 + (unit
* 0x2000), stat
);
888 static const struct nvkm_bitfield gk104_fifo_pbdma_intr_1
[] = {
889 { 0x00000001, "HCE_RE_ILLEGAL_OP" },
890 { 0x00000002, "HCE_RE_ALIGNB" },
891 { 0x00000004, "HCE_PRIV" },
892 { 0x00000008, "HCE_ILLEGAL_MTHD" },
893 { 0x00000010, "HCE_ILLEGAL_CLASS" },
898 gk104_fifo_intr_pbdma_1(struct gk104_fifo
*fifo
, int unit
)
900 struct nvkm_subdev
*subdev
= &fifo
->base
.engine
.subdev
;
901 struct nvkm_device
*device
= subdev
->device
;
902 u32 mask
= nvkm_rd32(device
, 0x04014c + (unit
* 0x2000));
903 u32 stat
= nvkm_rd32(device
, 0x040148 + (unit
* 0x2000)) & mask
;
904 u32 chid
= nvkm_rd32(device
, 0x040120 + (unit
* 0x2000)) & 0xfff;
908 nvkm_snprintbf(msg
, sizeof(msg
), gk104_fifo_pbdma_intr_1
, stat
);
909 nvkm_error(subdev
, "PBDMA%d: %08x [%s] ch %d %08x %08x\n",
910 unit
, stat
, msg
, chid
,
911 nvkm_rd32(device
, 0x040150 + (unit
* 0x2000)),
912 nvkm_rd32(device
, 0x040154 + (unit
* 0x2000)));
915 nvkm_wr32(device
, 0x040148 + (unit
* 0x2000), stat
);
919 gk104_fifo_intr_runlist(struct gk104_fifo
*fifo
)
921 struct nvkm_device
*device
= fifo
->base
.engine
.subdev
.device
;
922 u32 mask
= nvkm_rd32(device
, 0x002a00);
924 u32 engn
= __ffs(mask
);
925 wake_up(&fifo
->engine
[engn
].wait
);
926 nvkm_wr32(device
, 0x002a00, 1 << engn
);
927 mask
&= ~(1 << engn
);
932 gk104_fifo_intr_engine(struct gk104_fifo
*fifo
)
934 nvkm_fifo_uevent(&fifo
->base
);
938 gk104_fifo_intr(struct nvkm_subdev
*subdev
)
940 struct gk104_fifo
*fifo
= (void *)subdev
;
941 struct nvkm_device
*device
= fifo
->base
.engine
.subdev
.device
;
942 u32 mask
= nvkm_rd32(device
, 0x002140);
943 u32 stat
= nvkm_rd32(device
, 0x002100) & mask
;
945 if (stat
& 0x00000001) {
946 gk104_fifo_intr_bind(fifo
);
947 nvkm_wr32(device
, 0x002100, 0x00000001);
951 if (stat
& 0x00000010) {
952 nvkm_error(subdev
, "PIO_ERROR\n");
953 nvkm_wr32(device
, 0x002100, 0x00000010);
957 if (stat
& 0x00000100) {
958 gk104_fifo_intr_sched(fifo
);
959 nvkm_wr32(device
, 0x002100, 0x00000100);
963 if (stat
& 0x00010000) {
964 gk104_fifo_intr_chsw(fifo
);
965 nvkm_wr32(device
, 0x002100, 0x00010000);
969 if (stat
& 0x00800000) {
970 nvkm_error(subdev
, "FB_FLUSH_TIMEOUT\n");
971 nvkm_wr32(device
, 0x002100, 0x00800000);
975 if (stat
& 0x01000000) {
976 nvkm_error(subdev
, "LB_ERROR\n");
977 nvkm_wr32(device
, 0x002100, 0x01000000);
981 if (stat
& 0x08000000) {
982 gk104_fifo_intr_dropped_fault(fifo
);
983 nvkm_wr32(device
, 0x002100, 0x08000000);
987 if (stat
& 0x10000000) {
988 u32 mask
= nvkm_rd32(device
, 0x00259c);
990 u32 unit
= __ffs(mask
);
991 gk104_fifo_intr_fault(fifo
, unit
);
992 nvkm_wr32(device
, 0x00259c, (1 << unit
));
993 mask
&= ~(1 << unit
);
998 if (stat
& 0x20000000) {
999 u32 mask
= nvkm_rd32(device
, 0x0025a0);
1001 u32 unit
= __ffs(mask
);
1002 gk104_fifo_intr_pbdma_0(fifo
, unit
);
1003 gk104_fifo_intr_pbdma_1(fifo
, unit
);
1004 nvkm_wr32(device
, 0x0025a0, (1 << unit
));
1005 mask
&= ~(1 << unit
);
1007 stat
&= ~0x20000000;
1010 if (stat
& 0x40000000) {
1011 gk104_fifo_intr_runlist(fifo
);
1012 stat
&= ~0x40000000;
1015 if (stat
& 0x80000000) {
1016 nvkm_wr32(device
, 0x002100, 0x80000000);
1017 gk104_fifo_intr_engine(fifo
);
1018 stat
&= ~0x80000000;
1022 nvkm_error(subdev
, "INTR %08x\n", stat
);
1023 nvkm_mask(device
, 0x002140, stat
, 0x00000000);
1024 nvkm_wr32(device
, 0x002100, stat
);
1029 gk104_fifo_uevent_init(struct nvkm_event
*event
, int type
, int index
)
1031 struct nvkm_fifo
*fifo
= container_of(event
, typeof(*fifo
), uevent
);
1032 struct nvkm_device
*device
= fifo
->engine
.subdev
.device
;
1033 nvkm_mask(device
, 0x002140, 0x80000000, 0x80000000);
1037 gk104_fifo_uevent_fini(struct nvkm_event
*event
, int type
, int index
)
1039 struct nvkm_fifo
*fifo
= container_of(event
, typeof(*fifo
), uevent
);
1040 struct nvkm_device
*device
= fifo
->engine
.subdev
.device
;
1041 nvkm_mask(device
, 0x002140, 0x80000000, 0x00000000);
1044 static const struct nvkm_event_func
1045 gk104_fifo_uevent_func
= {
1046 .ctor
= nvkm_fifo_uevent_ctor
,
1047 .init
= gk104_fifo_uevent_init
,
1048 .fini
= gk104_fifo_uevent_fini
,
1052 gk104_fifo_fini(struct nvkm_object
*object
, bool suspend
)
1054 struct gk104_fifo
*fifo
= (void *)object
;
1055 struct nvkm_device
*device
= fifo
->base
.engine
.subdev
.device
;
1058 ret
= nvkm_fifo_fini(&fifo
->base
, suspend
);
1062 /* allow mmu fault interrupts, even when we're not using fifo */
1063 nvkm_mask(device
, 0x002140, 0x10000000, 0x10000000);
1068 gk104_fifo_init(struct nvkm_object
*object
)
1070 struct gk104_fifo
*fifo
= (void *)object
;
1071 struct nvkm_subdev
*subdev
= &fifo
->base
.engine
.subdev
;
1072 struct nvkm_device
*device
= subdev
->device
;
1075 ret
= nvkm_fifo_init(&fifo
->base
);
1079 /* enable all available PBDMA units */
1080 nvkm_wr32(device
, 0x000204, 0xffffffff);
1081 fifo
->spoon_nr
= hweight32(nvkm_rd32(device
, 0x000204));
1082 nvkm_debug(subdev
, "%d PBDMA unit(s)\n", fifo
->spoon_nr
);
1085 for (i
= 0; i
< fifo
->spoon_nr
; i
++) {
1086 nvkm_mask(device
, 0x04013c + (i
* 0x2000), 0x10000100, 0x00000000);
1087 nvkm_wr32(device
, 0x040108 + (i
* 0x2000), 0xffffffff); /* INTR */
1088 nvkm_wr32(device
, 0x04010c + (i
* 0x2000), 0xfffffeff); /* INTREN */
1092 for (i
= 0; i
< fifo
->spoon_nr
; i
++) {
1093 nvkm_wr32(device
, 0x040148 + (i
* 0x2000), 0xffffffff); /* INTR */
1094 nvkm_wr32(device
, 0x04014c + (i
* 0x2000), 0xffffffff); /* INTREN */
1097 nvkm_wr32(device
, 0x002254, 0x10000000 | fifo
->user
.bar
.offset
>> 12);
1099 nvkm_wr32(device
, 0x002100, 0xffffffff);
1100 nvkm_wr32(device
, 0x002140, 0x7fffffff);
1105 gk104_fifo_dtor(struct nvkm_object
*object
)
1107 struct gk104_fifo
*fifo
= (void *)object
;
1110 nvkm_gpuobj_unmap(&fifo
->user
.bar
);
1111 nvkm_gpuobj_ref(NULL
, &fifo
->user
.mem
);
1113 for (i
= 0; i
< FIFO_ENGINE_NR
; i
++) {
1114 nvkm_gpuobj_ref(NULL
, &fifo
->engine
[i
].runlist
[1]);
1115 nvkm_gpuobj_ref(NULL
, &fifo
->engine
[i
].runlist
[0]);
1118 nvkm_fifo_destroy(&fifo
->base
);
1122 gk104_fifo_ctor(struct nvkm_object
*parent
, struct nvkm_object
*engine
,
1123 struct nvkm_oclass
*oclass
, void *data
, u32 size
,
1124 struct nvkm_object
**pobject
)
1126 struct gk104_fifo_impl
*impl
= (void *)oclass
;
1127 struct gk104_fifo
*fifo
;
1130 ret
= nvkm_fifo_create(parent
, engine
, oclass
, 0,
1131 impl
->channels
- 1, &fifo
);
1132 *pobject
= nv_object(fifo
);
1136 INIT_WORK(&fifo
->fault
, gk104_fifo_recover_work
);
1138 for (i
= 0; i
< FIFO_ENGINE_NR
; i
++) {
1139 ret
= nvkm_gpuobj_new(nv_object(fifo
), NULL
, 0x8000, 0x1000,
1140 0, &fifo
->engine
[i
].runlist
[0]);
1144 ret
= nvkm_gpuobj_new(nv_object(fifo
), NULL
, 0x8000, 0x1000,
1145 0, &fifo
->engine
[i
].runlist
[1]);
1149 init_waitqueue_head(&fifo
->engine
[i
].wait
);
1152 ret
= nvkm_gpuobj_new(nv_object(fifo
), NULL
, impl
->channels
* 0x200,
1153 0x1000, NVOBJ_FLAG_ZERO_ALLOC
, &fifo
->user
.mem
);
1157 ret
= nvkm_gpuobj_map(fifo
->user
.mem
, NV_MEM_ACCESS_RW
,
1162 ret
= nvkm_event_init(&gk104_fifo_uevent_func
, 1, 1, &fifo
->base
.uevent
);
1166 nv_subdev(fifo
)->unit
= 0x00000100;
1167 nv_subdev(fifo
)->intr
= gk104_fifo_intr
;
1168 nv_engine(fifo
)->cclass
= &gk104_fifo_cclass
;
1169 nv_engine(fifo
)->sclass
= gk104_fifo_sclass
;
1173 struct nvkm_oclass
*
1174 gk104_fifo_oclass
= &(struct gk104_fifo_impl
) {
1175 .base
.handle
= NV_ENGINE(FIFO
, 0xe0),
1176 .base
.ofuncs
= &(struct nvkm_ofuncs
) {
1177 .ctor
= gk104_fifo_ctor
,
1178 .dtor
= gk104_fifo_dtor
,
1179 .init
= gk104_fifo_init
,
1180 .fini
= gk104_fifo_fini
,