2 * Copyright 2012 Red Hat Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
26 #include <core/client.h>
27 #include <core/engctx.h>
28 #include <core/enum.h>
29 #include <core/handle.h>
30 #include <subdev/bar.h>
31 #include <subdev/fb.h>
32 #include <subdev/mmu.h>
33 #include <subdev/timer.h>
35 #include <nvif/class.h>
36 #include <nvif/unpack.h>
38 #define _(a,b) { (a), ((1ULL << (a)) | (b)) }
43 _(NVDEV_ENGINE_GR
, (1ULL << NVDEV_ENGINE_SW
) |
44 (1ULL << NVDEV_ENGINE_CE2
)),
45 _(NVDEV_ENGINE_MSPDEC
, 0),
46 _(NVDEV_ENGINE_MSPPP
, 0),
47 _(NVDEV_ENGINE_MSVLD
, 0),
48 _(NVDEV_ENGINE_CE0
, 0),
49 _(NVDEV_ENGINE_CE1
, 0),
50 _(NVDEV_ENGINE_MSENC
, 0),
53 #define FIFO_ENGINE_NR ARRAY_SIZE(fifo_engine)
55 struct gk104_fifo_engn
{
56 struct nvkm_gpuobj
*runlist
[2];
58 wait_queue_head_t wait
;
61 struct gk104_fifo_priv
{
62 struct nvkm_fifo base
;
64 struct work_struct fault
;
67 struct gk104_fifo_engn engine
[FIFO_ENGINE_NR
];
69 struct nvkm_gpuobj
*mem
;
75 struct gk104_fifo_base
{
76 struct nvkm_fifo_base base
;
77 struct nvkm_gpuobj
*pgd
;
81 struct gk104_fifo_chan
{
82 struct nvkm_fifo_chan base
;
91 /*******************************************************************************
92 * FIFO channel objects
93 ******************************************************************************/
96 gk104_fifo_runlist_update(struct gk104_fifo_priv
*priv
, u32 engine
)
98 struct nvkm_bar
*bar
= nvkm_bar(priv
);
99 struct gk104_fifo_engn
*engn
= &priv
->engine
[engine
];
100 struct nvkm_gpuobj
*cur
;
103 mutex_lock(&nv_subdev(priv
)->mutex
);
104 cur
= engn
->runlist
[engn
->cur_runlist
];
105 engn
->cur_runlist
= !engn
->cur_runlist
;
107 for (i
= 0, p
= 0; i
< priv
->base
.max
; i
++) {
108 struct gk104_fifo_chan
*chan
= (void *)priv
->base
.channel
[i
];
109 if (chan
&& chan
->state
== RUNNING
&& chan
->engine
== engine
) {
110 nv_wo32(cur
, p
+ 0, i
);
111 nv_wo32(cur
, p
+ 4, 0x00000000);
117 nv_wr32(priv
, 0x002270, cur
->addr
>> 12);
118 nv_wr32(priv
, 0x002274, (engine
<< 20) | (p
>> 3));
120 if (wait_event_timeout(engn
->wait
, !(nv_rd32(priv
, 0x002284 +
121 (engine
* 0x08)) & 0x00100000),
122 msecs_to_jiffies(2000)) == 0)
123 nv_error(priv
, "runlist %d update timeout\n", engine
);
124 mutex_unlock(&nv_subdev(priv
)->mutex
);
128 gk104_fifo_context_attach(struct nvkm_object
*parent
,
129 struct nvkm_object
*object
)
131 struct nvkm_bar
*bar
= nvkm_bar(parent
);
132 struct gk104_fifo_base
*base
= (void *)parent
->parent
;
133 struct nvkm_engctx
*ectx
= (void *)object
;
137 switch (nv_engidx(object
->engine
)) {
138 case NVDEV_ENGINE_SW
:
140 case NVDEV_ENGINE_CE0
:
141 case NVDEV_ENGINE_CE1
:
142 case NVDEV_ENGINE_CE2
:
143 nv_engctx(ectx
)->addr
= nv_gpuobj(base
)->addr
>> 12;
145 case NVDEV_ENGINE_GR
: addr
= 0x0210; break;
146 case NVDEV_ENGINE_MSVLD
: addr
= 0x0270; break;
147 case NVDEV_ENGINE_MSPDEC
: addr
= 0x0250; break;
148 case NVDEV_ENGINE_MSPPP
: addr
= 0x0260; break;
153 if (!ectx
->vma
.node
) {
154 ret
= nvkm_gpuobj_map_vm(nv_gpuobj(ectx
), base
->vm
,
155 NV_MEM_ACCESS_RW
, &ectx
->vma
);
159 nv_engctx(ectx
)->addr
= nv_gpuobj(base
)->addr
>> 12;
162 nv_wo32(base
, addr
+ 0x00, lower_32_bits(ectx
->vma
.offset
) | 4);
163 nv_wo32(base
, addr
+ 0x04, upper_32_bits(ectx
->vma
.offset
));
169 gk104_fifo_context_detach(struct nvkm_object
*parent
, bool suspend
,
170 struct nvkm_object
*object
)
172 struct nvkm_bar
*bar
= nvkm_bar(parent
);
173 struct gk104_fifo_priv
*priv
= (void *)parent
->engine
;
174 struct gk104_fifo_base
*base
= (void *)parent
->parent
;
175 struct gk104_fifo_chan
*chan
= (void *)parent
;
178 switch (nv_engidx(object
->engine
)) {
179 case NVDEV_ENGINE_SW
: return 0;
180 case NVDEV_ENGINE_CE0
:
181 case NVDEV_ENGINE_CE1
:
182 case NVDEV_ENGINE_CE2
: addr
= 0x0000; break;
183 case NVDEV_ENGINE_GR
: addr
= 0x0210; break;
184 case NVDEV_ENGINE_MSVLD
: addr
= 0x0270; break;
185 case NVDEV_ENGINE_MSPDEC
: addr
= 0x0250; break;
186 case NVDEV_ENGINE_MSPPP
: addr
= 0x0260; break;
191 nv_wr32(priv
, 0x002634, chan
->base
.chid
);
192 if (!nv_wait(priv
, 0x002634, 0xffffffff, chan
->base
.chid
)) {
193 nv_error(priv
, "channel %d [%s] kick timeout\n",
194 chan
->base
.chid
, nvkm_client_name(chan
));
200 nv_wo32(base
, addr
+ 0x00, 0x00000000);
201 nv_wo32(base
, addr
+ 0x04, 0x00000000);
209 gk104_fifo_chan_ctor(struct nvkm_object
*parent
, struct nvkm_object
*engine
,
210 struct nvkm_oclass
*oclass
, void *data
, u32 size
,
211 struct nvkm_object
**pobject
)
214 struct kepler_channel_gpfifo_a_v0 v0
;
216 struct nvkm_bar
*bar
= nvkm_bar(parent
);
217 struct gk104_fifo_priv
*priv
= (void *)engine
;
218 struct gk104_fifo_base
*base
= (void *)parent
;
219 struct gk104_fifo_chan
*chan
;
220 u64 usermem
, ioffset
, ilength
;
223 nv_ioctl(parent
, "create channel gpfifo size %d\n", size
);
224 if (nvif_unpack(args
->v0
, 0, 0, false)) {
225 nv_ioctl(parent
, "create channel gpfifo vers %d pushbuf %08x "
226 "ioffset %016llx ilength %08x engine %08x\n",
227 args
->v0
.version
, args
->v0
.pushbuf
, args
->v0
.ioffset
,
228 args
->v0
.ilength
, args
->v0
.engine
);
232 for (i
= 0; i
< FIFO_ENGINE_NR
; i
++) {
233 if (args
->v0
.engine
& (1 << i
)) {
234 if (nvkm_engine(parent
, fifo_engine
[i
].subdev
)) {
235 args
->v0
.engine
= (1 << i
);
241 if (i
== FIFO_ENGINE_NR
) {
242 nv_error(priv
, "unsupported engines 0x%08x\n", args
->v0
.engine
);
246 ret
= nvkm_fifo_channel_create(parent
, engine
, oclass
, 1,
247 priv
->user
.bar
.offset
, 0x200,
249 fifo_engine
[i
].mask
, &chan
);
250 *pobject
= nv_object(chan
);
254 args
->v0
.chid
= chan
->base
.chid
;
256 nv_parent(chan
)->context_attach
= gk104_fifo_context_attach
;
257 nv_parent(chan
)->context_detach
= gk104_fifo_context_detach
;
260 usermem
= chan
->base
.chid
* 0x200;
261 ioffset
= args
->v0
.ioffset
;
262 ilength
= order_base_2(args
->v0
.ilength
/ 8);
264 for (i
= 0; i
< 0x200; i
+= 4)
265 nv_wo32(priv
->user
.mem
, usermem
+ i
, 0x00000000);
267 nv_wo32(base
, 0x08, lower_32_bits(priv
->user
.mem
->addr
+ usermem
));
268 nv_wo32(base
, 0x0c, upper_32_bits(priv
->user
.mem
->addr
+ usermem
));
269 nv_wo32(base
, 0x10, 0x0000face);
270 nv_wo32(base
, 0x30, 0xfffff902);
271 nv_wo32(base
, 0x48, lower_32_bits(ioffset
));
272 nv_wo32(base
, 0x4c, upper_32_bits(ioffset
) | (ilength
<< 16));
273 nv_wo32(base
, 0x84, 0x20400000);
274 nv_wo32(base
, 0x94, 0x30000001);
275 nv_wo32(base
, 0x9c, 0x00000100);
276 nv_wo32(base
, 0xac, 0x0000001f);
277 nv_wo32(base
, 0xe8, chan
->base
.chid
);
278 nv_wo32(base
, 0xb8, 0xf8000000);
279 nv_wo32(base
, 0xf8, 0x10003080); /* 0x002310 */
280 nv_wo32(base
, 0xfc, 0x10000010); /* 0x002350 */
286 gk104_fifo_chan_init(struct nvkm_object
*object
)
288 struct nvkm_gpuobj
*base
= nv_gpuobj(object
->parent
);
289 struct gk104_fifo_priv
*priv
= (void *)object
->engine
;
290 struct gk104_fifo_chan
*chan
= (void *)object
;
291 u32 chid
= chan
->base
.chid
;
294 ret
= nvkm_fifo_channel_init(&chan
->base
);
298 nv_mask(priv
, 0x800004 + (chid
* 8), 0x000f0000, chan
->engine
<< 16);
299 nv_wr32(priv
, 0x800000 + (chid
* 8), 0x80000000 | base
->addr
>> 12);
301 if (chan
->state
== STOPPED
&& (chan
->state
= RUNNING
) == RUNNING
) {
302 nv_mask(priv
, 0x800004 + (chid
* 8), 0x00000400, 0x00000400);
303 gk104_fifo_runlist_update(priv
, chan
->engine
);
304 nv_mask(priv
, 0x800004 + (chid
* 8), 0x00000400, 0x00000400);
311 gk104_fifo_chan_fini(struct nvkm_object
*object
, bool suspend
)
313 struct gk104_fifo_priv
*priv
= (void *)object
->engine
;
314 struct gk104_fifo_chan
*chan
= (void *)object
;
315 u32 chid
= chan
->base
.chid
;
317 if (chan
->state
== RUNNING
&& (chan
->state
= STOPPED
) == STOPPED
) {
318 nv_mask(priv
, 0x800004 + (chid
* 8), 0x00000800, 0x00000800);
319 gk104_fifo_runlist_update(priv
, chan
->engine
);
322 nv_wr32(priv
, 0x800000 + (chid
* 8), 0x00000000);
323 return nvkm_fifo_channel_fini(&chan
->base
, suspend
);
327 gk104_fifo_chan_ofuncs
= {
328 .ctor
= gk104_fifo_chan_ctor
,
329 .dtor
= _nvkm_fifo_channel_dtor
,
330 .init
= gk104_fifo_chan_init
,
331 .fini
= gk104_fifo_chan_fini
,
332 .map
= _nvkm_fifo_channel_map
,
333 .rd32
= _nvkm_fifo_channel_rd32
,
334 .wr32
= _nvkm_fifo_channel_wr32
,
335 .ntfy
= _nvkm_fifo_channel_ntfy
338 static struct nvkm_oclass
339 gk104_fifo_sclass
[] = {
340 { KEPLER_CHANNEL_GPFIFO_A
, &gk104_fifo_chan_ofuncs
},
344 /*******************************************************************************
345 * FIFO context - instmem heap and vm setup
346 ******************************************************************************/
349 gk104_fifo_context_ctor(struct nvkm_object
*parent
, struct nvkm_object
*engine
,
350 struct nvkm_oclass
*oclass
, void *data
, u32 size
,
351 struct nvkm_object
**pobject
)
353 struct gk104_fifo_base
*base
;
356 ret
= nvkm_fifo_context_create(parent
, engine
, oclass
, NULL
, 0x1000,
357 0x1000, NVOBJ_FLAG_ZERO_ALLOC
, &base
);
358 *pobject
= nv_object(base
);
362 ret
= nvkm_gpuobj_new(nv_object(base
), NULL
, 0x10000, 0x1000, 0,
367 nv_wo32(base
, 0x0200, lower_32_bits(base
->pgd
->addr
));
368 nv_wo32(base
, 0x0204, upper_32_bits(base
->pgd
->addr
));
369 nv_wo32(base
, 0x0208, 0xffffffff);
370 nv_wo32(base
, 0x020c, 0x000000ff);
372 ret
= nvkm_vm_ref(nvkm_client(parent
)->vm
, &base
->vm
, base
->pgd
);
380 gk104_fifo_context_dtor(struct nvkm_object
*object
)
382 struct gk104_fifo_base
*base
= (void *)object
;
383 nvkm_vm_ref(NULL
, &base
->vm
, base
->pgd
);
384 nvkm_gpuobj_ref(NULL
, &base
->pgd
);
385 nvkm_fifo_context_destroy(&base
->base
);
388 static struct nvkm_oclass
389 gk104_fifo_cclass
= {
390 .handle
= NV_ENGCTX(FIFO
, 0xe0),
391 .ofuncs
= &(struct nvkm_ofuncs
) {
392 .ctor
= gk104_fifo_context_ctor
,
393 .dtor
= gk104_fifo_context_dtor
,
394 .init
= _nvkm_fifo_context_init
,
395 .fini
= _nvkm_fifo_context_fini
,
396 .rd32
= _nvkm_fifo_context_rd32
,
397 .wr32
= _nvkm_fifo_context_wr32
,
401 /*******************************************************************************
403 ******************************************************************************/
406 gk104_fifo_engidx(struct gk104_fifo_priv
*priv
, u32 engn
)
409 case NVDEV_ENGINE_GR
:
410 case NVDEV_ENGINE_CE2
: engn
= 0; break;
411 case NVDEV_ENGINE_MSVLD
: engn
= 1; break;
412 case NVDEV_ENGINE_MSPPP
: engn
= 2; break;
413 case NVDEV_ENGINE_MSPDEC
: engn
= 3; break;
414 case NVDEV_ENGINE_CE0
: engn
= 4; break;
415 case NVDEV_ENGINE_CE1
: engn
= 5; break;
416 case NVDEV_ENGINE_MSENC
: engn
= 6; break;
424 static inline struct nvkm_engine
*
425 gk104_fifo_engine(struct gk104_fifo_priv
*priv
, u32 engn
)
427 if (engn
>= ARRAY_SIZE(fifo_engine
))
429 return nvkm_engine(priv
, fifo_engine
[engn
].subdev
);
433 gk104_fifo_recover_work(struct work_struct
*work
)
435 struct gk104_fifo_priv
*priv
= container_of(work
, typeof(*priv
), fault
);
436 struct nvkm_object
*engine
;
441 spin_lock_irqsave(&priv
->base
.lock
, flags
);
444 spin_unlock_irqrestore(&priv
->base
.lock
, flags
);
446 for (todo
= mask
; engn
= __ffs64(todo
), todo
; todo
&= ~(1 << engn
))
447 engm
|= 1 << gk104_fifo_engidx(priv
, engn
);
448 nv_mask(priv
, 0x002630, engm
, engm
);
450 for (todo
= mask
; engn
= __ffs64(todo
), todo
; todo
&= ~(1 << engn
)) {
451 if ((engine
= (void *)nvkm_engine(priv
, engn
))) {
452 nv_ofuncs(engine
)->fini(engine
, false);
453 WARN_ON(nv_ofuncs(engine
)->init(engine
));
455 gk104_fifo_runlist_update(priv
, gk104_fifo_engidx(priv
, engn
));
458 nv_wr32(priv
, 0x00262c, engm
);
459 nv_mask(priv
, 0x002630, engm
, 0x00000000);
463 gk104_fifo_recover(struct gk104_fifo_priv
*priv
, struct nvkm_engine
*engine
,
464 struct gk104_fifo_chan
*chan
)
466 u32 chid
= chan
->base
.chid
;
469 nv_error(priv
, "%s engine fault on channel %d, recovering...\n",
470 nv_subdev(engine
)->name
, chid
);
472 nv_mask(priv
, 0x800004 + (chid
* 0x08), 0x00000800, 0x00000800);
473 chan
->state
= KILLED
;
475 spin_lock_irqsave(&priv
->base
.lock
, flags
);
476 priv
->mask
|= 1ULL << nv_engidx(engine
);
477 spin_unlock_irqrestore(&priv
->base
.lock
, flags
);
478 schedule_work(&priv
->fault
);
482 gk104_fifo_swmthd(struct gk104_fifo_priv
*priv
, u32 chid
, u32 mthd
, u32 data
)
484 struct gk104_fifo_chan
*chan
= NULL
;
485 struct nvkm_handle
*bind
;
489 spin_lock_irqsave(&priv
->base
.lock
, flags
);
490 if (likely(chid
>= priv
->base
.min
&& chid
<= priv
->base
.max
))
491 chan
= (void *)priv
->base
.channel
[chid
];
495 bind
= nvkm_namedb_get_class(nv_namedb(chan
), 0x906e);
497 if (!mthd
|| !nv_call(bind
->object
, mthd
, data
))
499 nvkm_namedb_put(bind
);
503 spin_unlock_irqrestore(&priv
->base
.lock
, flags
);
507 static const struct nvkm_enum
508 gk104_fifo_bind_reason
[] = {
509 { 0x01, "BIND_NOT_UNBOUND" },
510 { 0x02, "SNOOP_WITHOUT_BAR1" },
511 { 0x03, "UNBIND_WHILE_RUNNING" },
512 { 0x05, "INVALID_RUNLIST" },
513 { 0x06, "INVALID_CTX_TGT" },
514 { 0x0b, "UNBIND_WHILE_PARKED" },
519 gk104_fifo_intr_bind(struct gk104_fifo_priv
*priv
)
521 u32 intr
= nv_rd32(priv
, 0x00252c);
522 u32 code
= intr
& 0x000000ff;
523 const struct nvkm_enum
*en
;
526 en
= nvkm_enum_find(gk104_fifo_bind_reason
, code
);
528 snprintf(enunk
, sizeof(enunk
), "UNK%02x", code
);
530 nv_error(priv
, "BIND_ERROR [ %s ]\n", en
? en
->name
: enunk
);
533 static const struct nvkm_enum
534 gk104_fifo_sched_reason
[] = {
535 { 0x0a, "CTXSW_TIMEOUT" },
540 gk104_fifo_intr_sched_ctxsw(struct gk104_fifo_priv
*priv
)
542 struct nvkm_engine
*engine
;
543 struct gk104_fifo_chan
*chan
;
546 for (engn
= 0; engn
< ARRAY_SIZE(fifo_engine
); engn
++) {
547 u32 stat
= nv_rd32(priv
, 0x002640 + (engn
* 0x04));
548 u32 busy
= (stat
& 0x80000000);
549 u32 next
= (stat
& 0x07ff0000) >> 16;
550 u32 chsw
= (stat
& 0x00008000);
551 u32 save
= (stat
& 0x00004000);
552 u32 load
= (stat
& 0x00002000);
553 u32 prev
= (stat
& 0x000007ff);
554 u32 chid
= load
? next
: prev
;
558 if (!(chan
= (void *)priv
->base
.channel
[chid
]))
560 if (!(engine
= gk104_fifo_engine(priv
, engn
)))
562 gk104_fifo_recover(priv
, engine
, chan
);
568 gk104_fifo_intr_sched(struct gk104_fifo_priv
*priv
)
570 u32 intr
= nv_rd32(priv
, 0x00254c);
571 u32 code
= intr
& 0x000000ff;
572 const struct nvkm_enum
*en
;
575 en
= nvkm_enum_find(gk104_fifo_sched_reason
, code
);
577 snprintf(enunk
, sizeof(enunk
), "UNK%02x", code
);
579 nv_error(priv
, "SCHED_ERROR [ %s ]\n", en
? en
->name
: enunk
);
583 gk104_fifo_intr_sched_ctxsw(priv
);
591 gk104_fifo_intr_chsw(struct gk104_fifo_priv
*priv
)
593 u32 stat
= nv_rd32(priv
, 0x00256c);
594 nv_error(priv
, "CHSW_ERROR 0x%08x\n", stat
);
595 nv_wr32(priv
, 0x00256c, stat
);
599 gk104_fifo_intr_dropped_fault(struct gk104_fifo_priv
*priv
)
601 u32 stat
= nv_rd32(priv
, 0x00259c);
602 nv_error(priv
, "DROPPED_MMU_FAULT 0x%08x\n", stat
);
605 static const struct nvkm_enum
606 gk104_fifo_fault_engine
[] = {
607 { 0x00, "GR", NULL
, NVDEV_ENGINE_GR
},
608 { 0x03, "IFB", NULL
, NVDEV_ENGINE_IFB
},
609 { 0x04, "BAR1", NULL
, NVDEV_SUBDEV_BAR
},
610 { 0x05, "BAR3", NULL
, NVDEV_SUBDEV_INSTMEM
},
611 { 0x07, "PBDMA0", NULL
, NVDEV_ENGINE_FIFO
},
612 { 0x08, "PBDMA1", NULL
, NVDEV_ENGINE_FIFO
},
613 { 0x09, "PBDMA2", NULL
, NVDEV_ENGINE_FIFO
},
614 { 0x10, "MSVLD", NULL
, NVDEV_ENGINE_MSVLD
},
615 { 0x11, "MSPPP", NULL
, NVDEV_ENGINE_MSPPP
},
617 { 0x14, "MSPDEC", NULL
, NVDEV_ENGINE_MSPDEC
},
618 { 0x15, "CE0", NULL
, NVDEV_ENGINE_CE0
},
619 { 0x16, "CE1", NULL
, NVDEV_ENGINE_CE1
},
621 { 0x19, "MSENC", NULL
, NVDEV_ENGINE_MSENC
},
622 { 0x1b, "CE2", NULL
, NVDEV_ENGINE_CE2
},
626 static const struct nvkm_enum
627 gk104_fifo_fault_reason
[] = {
629 { 0x01, "PDE_SIZE" },
631 { 0x03, "VA_LIMIT_VIOLATION" },
632 { 0x04, "UNBOUND_INST_BLOCK" },
633 { 0x05, "PRIV_VIOLATION" },
634 { 0x06, "RO_VIOLATION" },
635 { 0x07, "WO_VIOLATION" },
636 { 0x08, "PITCH_MASK_VIOLATION" },
637 { 0x09, "WORK_CREATION" },
638 { 0x0a, "UNSUPPORTED_APERTURE" },
639 { 0x0b, "COMPRESSION_FAILURE" },
640 { 0x0c, "UNSUPPORTED_KIND" },
641 { 0x0d, "REGION_VIOLATION" },
642 { 0x0e, "BOTH_PTES_VALID" },
643 { 0x0f, "INFO_TYPE_POISONED" },
647 static const struct nvkm_enum
648 gk104_fifo_fault_hubclient
[] = {
656 { 0x07, "HOST_CPU" },
657 { 0x08, "HOST_CPU_NB" },
668 { 0x13, "RASTERTWOD" },
684 static const struct nvkm_enum
685 gk104_fifo_fault_gpcclient
[] = {
686 { 0x00, "L1_0" }, { 0x01, "T1_0" }, { 0x02, "PE_0" },
687 { 0x03, "L1_1" }, { 0x04, "T1_1" }, { 0x05, "PE_1" },
688 { 0x06, "L1_2" }, { 0x07, "T1_2" }, { 0x08, "PE_2" },
689 { 0x09, "L1_3" }, { 0x0a, "T1_3" }, { 0x0b, "PE_3" },
697 { 0x13, "L1_4" }, { 0x14, "T1_4" }, { 0x15, "PE_4" },
698 { 0x16, "L1_5" }, { 0x17, "T1_5" }, { 0x18, "PE_5" },
699 { 0x19, "L1_6" }, { 0x1a, "T1_6" }, { 0x1b, "PE_6" },
700 { 0x1c, "L1_7" }, { 0x1d, "T1_7" }, { 0x1e, "PE_7" },
702 { 0x20, "LTP_UTLB_0" },
703 { 0x21, "LTP_UTLB_1" },
704 { 0x22, "LTP_UTLB_2" },
705 { 0x23, "LTP_UTLB_3" },
706 { 0x24, "GPC_RGG_UTLB" },
711 gk104_fifo_intr_fault(struct gk104_fifo_priv
*priv
, int unit
)
713 u32 inst
= nv_rd32(priv
, 0x002800 + (unit
* 0x10));
714 u32 valo
= nv_rd32(priv
, 0x002804 + (unit
* 0x10));
715 u32 vahi
= nv_rd32(priv
, 0x002808 + (unit
* 0x10));
716 u32 stat
= nv_rd32(priv
, 0x00280c + (unit
* 0x10));
717 u32 gpc
= (stat
& 0x1f000000) >> 24;
718 u32 client
= (stat
& 0x00001f00) >> 8;
719 u32 write
= (stat
& 0x00000080);
720 u32 hub
= (stat
& 0x00000040);
721 u32 reason
= (stat
& 0x0000000f);
722 struct nvkm_object
*engctx
= NULL
, *object
;
723 struct nvkm_engine
*engine
= NULL
;
724 const struct nvkm_enum
*er
, *eu
, *ec
;
730 er
= nvkm_enum_find(gk104_fifo_fault_reason
, reason
);
732 snprintf(erunk
, sizeof(erunk
), "UNK%02X", reason
);
734 eu
= nvkm_enum_find(gk104_fifo_fault_engine
, unit
);
737 case NVDEV_SUBDEV_BAR
:
738 nv_mask(priv
, 0x001704, 0x00000000, 0x00000000);
740 case NVDEV_SUBDEV_INSTMEM
:
741 nv_mask(priv
, 0x001714, 0x00000000, 0x00000000);
743 case NVDEV_ENGINE_IFB
:
744 nv_mask(priv
, 0x001718, 0x00000000, 0x00000000);
747 engine
= nvkm_engine(priv
, eu
->data2
);
749 engctx
= nvkm_engctx_get(engine
, inst
);
753 snprintf(euunk
, sizeof(euunk
), "UNK%02x", unit
);
757 ec
= nvkm_enum_find(gk104_fifo_fault_hubclient
, client
);
759 ec
= nvkm_enum_find(gk104_fifo_fault_gpcclient
, client
);
760 snprintf(gpcid
, sizeof(gpcid
), "%d", gpc
);
764 snprintf(ecunk
, sizeof(ecunk
), "UNK%02x", client
);
766 nv_error(priv
, "%s fault at 0x%010llx [%s] from %s/%s%s%s%s on "
767 "channel 0x%010llx [%s]\n", write
? "write" : "read",
768 (u64
)vahi
<< 32 | valo
, er
? er
->name
: erunk
,
769 eu
? eu
->name
: euunk
, hub
? "" : "GPC", gpcid
, hub
? "" : "/",
770 ec
? ec
->name
: ecunk
, (u64
)inst
<< 12,
771 nvkm_client_name(engctx
));
775 switch (nv_mclass(object
)) {
776 case KEPLER_CHANNEL_GPFIFO_A
:
777 case MAXWELL_CHANNEL_GPFIFO_A
:
778 gk104_fifo_recover(priv
, engine
, (void *)object
);
781 object
= object
->parent
;
784 nvkm_engctx_put(engctx
);
787 static const struct nvkm_bitfield gk104_fifo_pbdma_intr_0
[] = {
788 { 0x00000001, "MEMREQ" },
789 { 0x00000002, "MEMACK_TIMEOUT" },
790 { 0x00000004, "MEMACK_EXTRA" },
791 { 0x00000008, "MEMDAT_TIMEOUT" },
792 { 0x00000010, "MEMDAT_EXTRA" },
793 { 0x00000020, "MEMFLUSH" },
794 { 0x00000040, "MEMOP" },
795 { 0x00000080, "LBCONNECT" },
796 { 0x00000100, "LBREQ" },
797 { 0x00000200, "LBACK_TIMEOUT" },
798 { 0x00000400, "LBACK_EXTRA" },
799 { 0x00000800, "LBDAT_TIMEOUT" },
800 { 0x00001000, "LBDAT_EXTRA" },
801 { 0x00002000, "GPFIFO" },
802 { 0x00004000, "GPPTR" },
803 { 0x00008000, "GPENTRY" },
804 { 0x00010000, "GPCRC" },
805 { 0x00020000, "PBPTR" },
806 { 0x00040000, "PBENTRY" },
807 { 0x00080000, "PBCRC" },
808 { 0x00100000, "XBARCONNECT" },
809 { 0x00200000, "METHOD" },
810 { 0x00400000, "METHODCRC" },
811 { 0x00800000, "DEVICE" },
812 { 0x02000000, "SEMAPHORE" },
813 { 0x04000000, "ACQUIRE" },
814 { 0x08000000, "PRI" },
815 { 0x20000000, "NO_CTXSW_SEG" },
816 { 0x40000000, "PBSEG" },
817 { 0x80000000, "SIGNATURE" },
822 gk104_fifo_intr_pbdma_0(struct gk104_fifo_priv
*priv
, int unit
)
824 u32 mask
= nv_rd32(priv
, 0x04010c + (unit
* 0x2000));
825 u32 stat
= nv_rd32(priv
, 0x040108 + (unit
* 0x2000)) & mask
;
826 u32 addr
= nv_rd32(priv
, 0x0400c0 + (unit
* 0x2000));
827 u32 data
= nv_rd32(priv
, 0x0400c4 + (unit
* 0x2000));
828 u32 chid
= nv_rd32(priv
, 0x040120 + (unit
* 0x2000)) & 0xfff;
829 u32 subc
= (addr
& 0x00070000) >> 16;
830 u32 mthd
= (addr
& 0x00003ffc);
833 if (stat
& 0x00800000) {
834 if (!gk104_fifo_swmthd(priv
, chid
, mthd
, data
))
836 nv_wr32(priv
, 0x0400c0 + (unit
* 0x2000), 0x80600008);
840 nv_error(priv
, "PBDMA%d:", unit
);
841 nvkm_bitfield_print(gk104_fifo_pbdma_intr_0
, show
);
844 "PBDMA%d: ch %d [%s] subc %d mthd 0x%04x data 0x%08x\n",
846 nvkm_client_name_for_fifo_chid(&priv
->base
, chid
),
850 nv_wr32(priv
, 0x040108 + (unit
* 0x2000), stat
);
853 static const struct nvkm_bitfield gk104_fifo_pbdma_intr_1
[] = {
854 { 0x00000001, "HCE_RE_ILLEGAL_OP" },
855 { 0x00000002, "HCE_RE_ALIGNB" },
856 { 0x00000004, "HCE_PRIV" },
857 { 0x00000008, "HCE_ILLEGAL_MTHD" },
858 { 0x00000010, "HCE_ILLEGAL_CLASS" },
863 gk104_fifo_intr_pbdma_1(struct gk104_fifo_priv
*priv
, int unit
)
865 u32 mask
= nv_rd32(priv
, 0x04014c + (unit
* 0x2000));
866 u32 stat
= nv_rd32(priv
, 0x040148 + (unit
* 0x2000)) & mask
;
867 u32 chid
= nv_rd32(priv
, 0x040120 + (unit
* 0x2000)) & 0xfff;
870 nv_error(priv
, "PBDMA%d:", unit
);
871 nvkm_bitfield_print(gk104_fifo_pbdma_intr_1
, stat
);
873 nv_error(priv
, "PBDMA%d: ch %d %08x %08x\n", unit
, chid
,
874 nv_rd32(priv
, 0x040150 + (unit
* 0x2000)),
875 nv_rd32(priv
, 0x040154 + (unit
* 0x2000)));
878 nv_wr32(priv
, 0x040148 + (unit
* 0x2000), stat
);
882 gk104_fifo_intr_runlist(struct gk104_fifo_priv
*priv
)
884 u32 mask
= nv_rd32(priv
, 0x002a00);
886 u32 engn
= __ffs(mask
);
887 wake_up(&priv
->engine
[engn
].wait
);
888 nv_wr32(priv
, 0x002a00, 1 << engn
);
889 mask
&= ~(1 << engn
);
894 gk104_fifo_intr_engine(struct gk104_fifo_priv
*priv
)
896 nvkm_fifo_uevent(&priv
->base
);
900 gk104_fifo_intr(struct nvkm_subdev
*subdev
)
902 struct gk104_fifo_priv
*priv
= (void *)subdev
;
903 u32 mask
= nv_rd32(priv
, 0x002140);
904 u32 stat
= nv_rd32(priv
, 0x002100) & mask
;
906 if (stat
& 0x00000001) {
907 gk104_fifo_intr_bind(priv
);
908 nv_wr32(priv
, 0x002100, 0x00000001);
912 if (stat
& 0x00000010) {
913 nv_error(priv
, "PIO_ERROR\n");
914 nv_wr32(priv
, 0x002100, 0x00000010);
918 if (stat
& 0x00000100) {
919 gk104_fifo_intr_sched(priv
);
920 nv_wr32(priv
, 0x002100, 0x00000100);
924 if (stat
& 0x00010000) {
925 gk104_fifo_intr_chsw(priv
);
926 nv_wr32(priv
, 0x002100, 0x00010000);
930 if (stat
& 0x00800000) {
931 nv_error(priv
, "FB_FLUSH_TIMEOUT\n");
932 nv_wr32(priv
, 0x002100, 0x00800000);
936 if (stat
& 0x01000000) {
937 nv_error(priv
, "LB_ERROR\n");
938 nv_wr32(priv
, 0x002100, 0x01000000);
942 if (stat
& 0x08000000) {
943 gk104_fifo_intr_dropped_fault(priv
);
944 nv_wr32(priv
, 0x002100, 0x08000000);
948 if (stat
& 0x10000000) {
949 u32 mask
= nv_rd32(priv
, 0x00259c);
951 u32 unit
= __ffs(mask
);
952 gk104_fifo_intr_fault(priv
, unit
);
953 nv_wr32(priv
, 0x00259c, (1 << unit
));
954 mask
&= ~(1 << unit
);
959 if (stat
& 0x20000000) {
960 u32 mask
= nv_rd32(priv
, 0x0025a0);
962 u32 unit
= __ffs(mask
);
963 gk104_fifo_intr_pbdma_0(priv
, unit
);
964 gk104_fifo_intr_pbdma_1(priv
, unit
);
965 nv_wr32(priv
, 0x0025a0, (1 << unit
));
966 mask
&= ~(1 << unit
);
971 if (stat
& 0x40000000) {
972 gk104_fifo_intr_runlist(priv
);
976 if (stat
& 0x80000000) {
977 nv_wr32(priv
, 0x002100, 0x80000000);
978 gk104_fifo_intr_engine(priv
);
983 nv_error(priv
, "INTR 0x%08x\n", stat
);
984 nv_mask(priv
, 0x002140, stat
, 0x00000000);
985 nv_wr32(priv
, 0x002100, stat
);
990 gk104_fifo_uevent_init(struct nvkm_event
*event
, int type
, int index
)
992 struct nvkm_fifo
*fifo
= container_of(event
, typeof(*fifo
), uevent
);
993 nv_mask(fifo
, 0x002140, 0x80000000, 0x80000000);
997 gk104_fifo_uevent_fini(struct nvkm_event
*event
, int type
, int index
)
999 struct nvkm_fifo
*fifo
= container_of(event
, typeof(*fifo
), uevent
);
1000 nv_mask(fifo
, 0x002140, 0x80000000, 0x00000000);
1003 static const struct nvkm_event_func
1004 gk104_fifo_uevent_func
= {
1005 .ctor
= nvkm_fifo_uevent_ctor
,
1006 .init
= gk104_fifo_uevent_init
,
1007 .fini
= gk104_fifo_uevent_fini
,
1011 gk104_fifo_fini(struct nvkm_object
*object
, bool suspend
)
1013 struct gk104_fifo_priv
*priv
= (void *)object
;
1016 ret
= nvkm_fifo_fini(&priv
->base
, suspend
);
1020 /* allow mmu fault interrupts, even when we're not using fifo */
1021 nv_mask(priv
, 0x002140, 0x10000000, 0x10000000);
1026 gk104_fifo_init(struct nvkm_object
*object
)
1028 struct gk104_fifo_priv
*priv
= (void *)object
;
1031 ret
= nvkm_fifo_init(&priv
->base
);
1035 /* enable all available PBDMA units */
1036 nv_wr32(priv
, 0x000204, 0xffffffff);
1037 priv
->spoon_nr
= hweight32(nv_rd32(priv
, 0x000204));
1038 nv_debug(priv
, "%d PBDMA unit(s)\n", priv
->spoon_nr
);
1041 for (i
= 0; i
< priv
->spoon_nr
; i
++) {
1042 nv_mask(priv
, 0x04013c + (i
* 0x2000), 0x10000100, 0x00000000);
1043 nv_wr32(priv
, 0x040108 + (i
* 0x2000), 0xffffffff); /* INTR */
1044 nv_wr32(priv
, 0x04010c + (i
* 0x2000), 0xfffffeff); /* INTREN */
1048 for (i
= 0; i
< priv
->spoon_nr
; i
++) {
1049 nv_wr32(priv
, 0x040148 + (i
* 0x2000), 0xffffffff); /* INTR */
1050 nv_wr32(priv
, 0x04014c + (i
* 0x2000), 0xffffffff); /* INTREN */
1053 nv_wr32(priv
, 0x002254, 0x10000000 | priv
->user
.bar
.offset
>> 12);
1055 nv_wr32(priv
, 0x002100, 0xffffffff);
1056 nv_wr32(priv
, 0x002140, 0x7fffffff);
1061 gk104_fifo_dtor(struct nvkm_object
*object
)
1063 struct gk104_fifo_priv
*priv
= (void *)object
;
1066 nvkm_gpuobj_unmap(&priv
->user
.bar
);
1067 nvkm_gpuobj_ref(NULL
, &priv
->user
.mem
);
1069 for (i
= 0; i
< FIFO_ENGINE_NR
; i
++) {
1070 nvkm_gpuobj_ref(NULL
, &priv
->engine
[i
].runlist
[1]);
1071 nvkm_gpuobj_ref(NULL
, &priv
->engine
[i
].runlist
[0]);
1074 nvkm_fifo_destroy(&priv
->base
);
1078 gk104_fifo_ctor(struct nvkm_object
*parent
, struct nvkm_object
*engine
,
1079 struct nvkm_oclass
*oclass
, void *data
, u32 size
,
1080 struct nvkm_object
**pobject
)
1082 struct gk104_fifo_impl
*impl
= (void *)oclass
;
1083 struct gk104_fifo_priv
*priv
;
1086 ret
= nvkm_fifo_create(parent
, engine
, oclass
, 0,
1087 impl
->channels
- 1, &priv
);
1088 *pobject
= nv_object(priv
);
1092 INIT_WORK(&priv
->fault
, gk104_fifo_recover_work
);
1094 for (i
= 0; i
< FIFO_ENGINE_NR
; i
++) {
1095 ret
= nvkm_gpuobj_new(nv_object(priv
), NULL
, 0x8000, 0x1000,
1096 0, &priv
->engine
[i
].runlist
[0]);
1100 ret
= nvkm_gpuobj_new(nv_object(priv
), NULL
, 0x8000, 0x1000,
1101 0, &priv
->engine
[i
].runlist
[1]);
1105 init_waitqueue_head(&priv
->engine
[i
].wait
);
1108 ret
= nvkm_gpuobj_new(nv_object(priv
), NULL
, impl
->channels
* 0x200,
1109 0x1000, NVOBJ_FLAG_ZERO_ALLOC
, &priv
->user
.mem
);
1113 ret
= nvkm_gpuobj_map(priv
->user
.mem
, NV_MEM_ACCESS_RW
,
1118 ret
= nvkm_event_init(&gk104_fifo_uevent_func
, 1, 1, &priv
->base
.uevent
);
1122 nv_subdev(priv
)->unit
= 0x00000100;
1123 nv_subdev(priv
)->intr
= gk104_fifo_intr
;
1124 nv_engine(priv
)->cclass
= &gk104_fifo_cclass
;
1125 nv_engine(priv
)->sclass
= gk104_fifo_sclass
;
1129 struct nvkm_oclass
*
1130 gk104_fifo_oclass
= &(struct gk104_fifo_impl
) {
1131 .base
.handle
= NV_ENGINE(FIFO
, 0xe0),
1132 .base
.ofuncs
= &(struct nvkm_ofuncs
) {
1133 .ctor
= gk104_fifo_ctor
,
1134 .dtor
= gk104_fifo_dtor
,
1135 .init
= gk104_fifo_init
,
1136 .fini
= gk104_fifo_fini
,