]>
Commit | Line | Data |
---|---|---|
5132f377 | 1 | /* |
ebb945a9 | 2 | * Copyright 2012 Red Hat Inc. |
5132f377 BS |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | * Authors: Ben Skeggs | |
23 | */ | |
05c7145d | 24 | #include "gk104.h" |
9a65a38c | 25 | #include "changk104.h" |
5132f377 | 26 | |
ebb945a9 | 27 | #include <core/client.h> |
13de7f46 | 28 | #include <core/gpuobj.h> |
358ce601 | 29 | #include <subdev/bar.h> |
0faaa47d | 30 | #include <subdev/timer.h> |
e93e198d | 31 | #include <subdev/top.h> |
61570911 | 32 | #include <engine/sw.h> |
5132f377 | 33 | |
05c7145d | 34 | #include <nvif/class.h> |
5132f377 | 35 | |
b88917fe BS |
36 | struct gk104_fifo_engine_status { |
37 | bool busy; | |
38 | bool faulted; | |
39 | bool chsw; | |
40 | bool save; | |
41 | bool load; | |
42 | struct { | |
43 | bool tsg; | |
44 | u32 id; | |
45 | } prev, next, *chan; | |
46 | }; | |
47 | ||
48 | static void | |
49 | gk104_fifo_engine_status(struct gk104_fifo *fifo, int engn, | |
50 | struct gk104_fifo_engine_status *status) | |
51 | { | |
ec5c6bda | 52 | struct nvkm_engine *engine = fifo->engine[engn].engine; |
b88917fe BS |
53 | struct nvkm_subdev *subdev = &fifo->base.engine.subdev; |
54 | struct nvkm_device *device = subdev->device; | |
55 | u32 stat = nvkm_rd32(device, 0x002640 + (engn * 0x08)); | |
56 | ||
57 | status->busy = !!(stat & 0x80000000); | |
58 | status->faulted = !!(stat & 0x40000000); | |
59 | status->next.tsg = !!(stat & 0x10000000); | |
60 | status->next.id = (stat & 0x0fff0000) >> 16; | |
61 | status->chsw = !!(stat & 0x00008000); | |
62 | status->save = !!(stat & 0x00004000); | |
63 | status->load = !!(stat & 0x00002000); | |
64 | status->prev.tsg = !!(stat & 0x00001000); | |
65 | status->prev.id = (stat & 0x00000fff); | |
ec5c6bda BS |
66 | status->chan = NULL; |
67 | ||
68 | if (status->busy && status->chsw) { | |
69 | if (status->load && status->save) { | |
70 | if (engine && nvkm_engine_chsw_load(engine)) | |
71 | status->chan = &status->next; | |
72 | else | |
73 | status->chan = &status->prev; | |
74 | } else | |
75 | if (status->load) { | |
76 | status->chan = &status->next; | |
77 | } else { | |
78 | status->chan = &status->prev; | |
79 | } | |
80 | } else | |
81 | if (status->load) { | |
82 | status->chan = &status->prev; | |
83 | } | |
b88917fe BS |
84 | |
85 | nvkm_debug(subdev, "engine %02d: busy %d faulted %d chsw %d " | |
86 | "save %d load %d %sid %d%s-> %sid %d%s\n", | |
87 | engn, status->busy, status->faulted, | |
88 | status->chsw, status->save, status->load, | |
89 | status->prev.tsg ? "tsg" : "ch", status->prev.id, | |
90 | status->chan == &status->prev ? "*" : " ", | |
91 | status->next.tsg ? "tsg" : "ch", status->next.id, | |
92 | status->chan == &status->next ? "*" : " "); | |
93 | } | |
94 | ||
98ac3f06 BS |
95 | static int |
96 | gk104_fifo_class_get(struct nvkm_fifo *base, int index, | |
97 | const struct nvkm_fifo_chan_oclass **psclass) | |
98 | { | |
99 | struct gk104_fifo *fifo = gk104_fifo(base); | |
100 | int c = 0; | |
101 | ||
102 | while ((*psclass = fifo->func->chan[c])) { | |
103 | if (c++ == index) | |
104 | return 0; | |
105 | } | |
106 | ||
107 | return c; | |
108 | } | |
109 | ||
110 | static void | |
13de7f46 | 111 | gk104_fifo_uevent_fini(struct nvkm_fifo *fifo) |
9a65a38c | 112 | { |
9a65a38c BS |
113 | struct nvkm_device *device = fifo->engine.subdev.device; |
114 | nvkm_mask(device, 0x002140, 0x80000000, 0x00000000); | |
115 | } | |
5132f377 | 116 | |
98ac3f06 | 117 | static void |
13de7f46 | 118 | gk104_fifo_uevent_init(struct nvkm_fifo *fifo) |
9a65a38c | 119 | { |
9a65a38c BS |
120 | struct nvkm_device *device = fifo->engine.subdev.device; |
121 | nvkm_mask(device, 0x002140, 0x80000000, 0x80000000); | |
122 | } | |
ebb945a9 | 123 | |
9a65a38c | 124 | void |
69aa40e2 | 125 | gk104_fifo_runlist_commit(struct gk104_fifo *fifo, int runl) |
5132f377 | 126 | { |
8f0649b5 | 127 | struct gk104_fifo_chan *chan; |
e5c5e4f5 BS |
128 | struct nvkm_subdev *subdev = &fifo->base.engine.subdev; |
129 | struct nvkm_device *device = subdev->device; | |
69aa40e2 | 130 | struct nvkm_memory *mem; |
8f0649b5 | 131 | int nr = 0; |
a2e435a1 | 132 | int target; |
5132f377 | 133 | |
13de7f46 | 134 | mutex_lock(&subdev->mutex); |
69aa40e2 BS |
135 | mem = fifo->runlist[runl].mem[fifo->runlist[runl].next]; |
136 | fifo->runlist[runl].next = !fifo->runlist[runl].next; | |
5132f377 | 137 | |
69aa40e2 BS |
138 | nvkm_kmap(mem); |
139 | list_for_each_entry(chan, &fifo->runlist[runl].chan, head) { | |
140 | nvkm_wo32(mem, (nr * 8) + 0, chan->base.chid); | |
141 | nvkm_wo32(mem, (nr * 8) + 4, 0x00000000); | |
8f0649b5 | 142 | nr++; |
5132f377 | 143 | } |
69aa40e2 | 144 | nvkm_done(mem); |
5132f377 | 145 | |
d2ee3605 BS |
146 | switch (nvkm_memory_target(mem)) { |
147 | case NVKM_MEM_TARGET_VRAM: target = 0; break; | |
148 | case NVKM_MEM_TARGET_NCOH: target = 3; break; | |
149 | default: | |
150 | WARN_ON(1); | |
2579b8b0 | 151 | goto unlock; |
d2ee3605 | 152 | } |
a2e435a1 | 153 | |
69aa40e2 | 154 | nvkm_wr32(device, 0x002270, (nvkm_memory_addr(mem) >> 12) | |
a2e435a1 | 155 | (target << 28)); |
69aa40e2 | 156 | nvkm_wr32(device, 0x002274, (runl << 20) | nr); |
87032e11 | 157 | |
69aa40e2 BS |
158 | if (wait_event_timeout(fifo->runlist[runl].wait, |
159 | !(nvkm_rd32(device, 0x002284 + (runl * 0x08)) | |
160 | & 0x00100000), | |
161 | msecs_to_jiffies(2000)) == 0) | |
162 | nvkm_error(subdev, "runlist %d update timeout\n", runl); | |
2579b8b0 | 163 | unlock: |
13de7f46 | 164 | mutex_unlock(&subdev->mutex); |
5132f377 BS |
165 | } |
166 | ||
386ffd5e BS |
167 | void |
168 | gk104_fifo_runlist_remove(struct gk104_fifo *fifo, struct gk104_fifo_chan *chan) | |
169 | { | |
170 | mutex_lock(&fifo->base.engine.subdev.mutex); | |
171 | list_del_init(&chan->head); | |
172 | mutex_unlock(&fifo->base.engine.subdev.mutex); | |
173 | } | |
174 | ||
175 | void | |
176 | gk104_fifo_runlist_insert(struct gk104_fifo *fifo, struct gk104_fifo_chan *chan) | |
177 | { | |
178 | mutex_lock(&fifo->base.engine.subdev.mutex); | |
69aa40e2 | 179 | list_add_tail(&chan->head, &fifo->runlist[chan->runl].chan); |
386ffd5e BS |
180 | mutex_unlock(&fifo->base.engine.subdev.mutex); |
181 | } | |
182 | ||
98d1e317 | 183 | static void |
55252da1 | 184 | gk104_fifo_recover_work(struct work_struct *w) |
98d1e317 | 185 | { |
55252da1 | 186 | struct gk104_fifo *fifo = container_of(w, typeof(*fifo), recover.work); |
87744403 | 187 | struct nvkm_device *device = fifo->base.engine.subdev.device; |
6cf813fb | 188 | struct nvkm_engine *engine; |
98d1e317 | 189 | unsigned long flags; |
19f89279 BS |
190 | u32 engm, runm, todo; |
191 | int engn, runl; | |
98d1e317 | 192 | |
6189f1b0 | 193 | spin_lock_irqsave(&fifo->base.lock, flags); |
19f89279 BS |
194 | runm = fifo->recover.runm; |
195 | engm = fifo->recover.engm; | |
196 | fifo->recover.engm = 0; | |
197 | fifo->recover.runm = 0; | |
6189f1b0 | 198 | spin_unlock_irqrestore(&fifo->base.lock, flags); |
98d1e317 | 199 | |
19f89279 | 200 | nvkm_mask(device, 0x002630, runm, runm); |
98d1e317 | 201 | |
19f89279 BS |
202 | for (todo = engm; engn = __ffs(todo), todo; todo &= ~BIT(engn)) { |
203 | if ((engine = fifo->engine[engn].engine)) { | |
6cf813fb BS |
204 | nvkm_subdev_fini(&engine->subdev, false); |
205 | WARN_ON(nvkm_subdev_init(&engine->subdev)); | |
98d1e317 | 206 | } |
98d1e317 BS |
207 | } |
208 | ||
19f89279 BS |
209 | for (todo = runm; runl = __ffs(todo), todo; todo &= ~BIT(runl)) |
210 | gk104_fifo_runlist_commit(fifo, runl); | |
211 | ||
212 | nvkm_wr32(device, 0x00262c, runm); | |
213 | nvkm_mask(device, 0x002630, runm, 0x00000000); | |
98d1e317 BS |
214 | } |
215 | ||
0faaa47d BS |
216 | static void gk104_fifo_recover_engn(struct gk104_fifo *fifo, int engn); |
217 | ||
98d1e317 | 218 | static void |
0faaa47d | 219 | gk104_fifo_recover_runl(struct gk104_fifo *fifo, int runl) |
98d1e317 | 220 | { |
e5c5e4f5 BS |
221 | struct nvkm_subdev *subdev = &fifo->base.engine.subdev; |
222 | struct nvkm_device *device = subdev->device; | |
0faaa47d | 223 | const u32 runm = BIT(runl); |
98d1e317 | 224 | |
6ca307b0 | 225 | assert_spin_locked(&fifo->base.lock); |
0faaa47d BS |
226 | if (fifo->recover.runm & runm) |
227 | return; | |
228 | fifo->recover.runm |= runm; | |
98d1e317 | 229 | |
0faaa47d BS |
230 | /* Block runlist to prevent channel assignment(s) from changing. */ |
231 | nvkm_mask(device, 0x002630, runm, runm); | |
98d1e317 | 232 | |
0faaa47d BS |
233 | /* Schedule recovery. */ |
234 | nvkm_warn(subdev, "runlist %d: scheduled for recovery\n", runl); | |
235 | schedule_work(&fifo->recover.work); | |
236 | } | |
237 | ||
238 | static void | |
239 | gk104_fifo_recover_chan(struct nvkm_fifo *base, int chid) | |
240 | { | |
241 | struct gk104_fifo *fifo = gk104_fifo(base); | |
242 | struct nvkm_subdev *subdev = &fifo->base.engine.subdev; | |
243 | struct nvkm_device *device = subdev->device; | |
244 | const u32 stat = nvkm_rd32(device, 0x800004 + (chid * 0x08)); | |
245 | const u32 runl = (stat & 0x000f0000) >> 16; | |
246 | const bool used = (stat & 0x00000001); | |
eaa5ed65 | 247 | unsigned long engn, engm = fifo->runlist[runl].engm; |
0faaa47d BS |
248 | struct gk104_fifo_chan *chan; |
249 | ||
250 | assert_spin_locked(&fifo->base.lock); | |
251 | if (!used) | |
252 | return; | |
253 | ||
254 | /* Lookup SW state for channel, and mark it as dead. */ | |
255 | list_for_each_entry(chan, &fifo->runlist[runl].chan, head) { | |
256 | if (chan->base.chid == chid) { | |
257 | list_del_init(&chan->head); | |
258 | chan->killed = true; | |
259 | nvkm_fifo_kevent(&fifo->base, chid); | |
19f89279 BS |
260 | break; |
261 | } | |
262 | } | |
263 | ||
0faaa47d BS |
264 | /* Disable channel. */ |
265 | nvkm_wr32(device, 0x800004 + (chid * 0x08), stat | 0x00000800); | |
266 | nvkm_warn(subdev, "channel %d: killed\n", chid); | |
eaa5ed65 BS |
267 | |
268 | /* Block channel assignments from changing during recovery. */ | |
269 | gk104_fifo_recover_runl(fifo, runl); | |
270 | ||
271 | /* Schedule recovery for any engines the channel is on. */ | |
272 | for_each_set_bit(engn, &engm, fifo->engine_nr) { | |
273 | struct gk104_fifo_engine_status status; | |
274 | gk104_fifo_engine_status(fifo, engn, &status); | |
275 | if (!status.chan || status.chan->id != chid) | |
276 | continue; | |
277 | gk104_fifo_recover_engn(fifo, engn); | |
278 | } | |
0faaa47d BS |
279 | } |
280 | ||
281 | static void | |
282 | gk104_fifo_recover_engn(struct gk104_fifo *fifo, int engn) | |
283 | { | |
3ebef76a | 284 | struct nvkm_engine *engine = fifo->engine[engn].engine; |
0faaa47d | 285 | struct nvkm_subdev *subdev = &fifo->base.engine.subdev; |
3ebef76a | 286 | struct nvkm_device *device = subdev->device; |
0faaa47d BS |
287 | const u32 runl = fifo->engine[engn].runl; |
288 | const u32 engm = BIT(engn); | |
289 | struct gk104_fifo_engine_status status; | |
3ebef76a | 290 | int mmui = -1; |
0faaa47d BS |
291 | |
292 | assert_spin_locked(&fifo->base.lock); | |
293 | if (fifo->recover.engm & engm) | |
294 | return; | |
295 | fifo->recover.engm |= engm; | |
296 | ||
297 | /* Block channel assignments from changing during recovery. */ | |
298 | gk104_fifo_recover_runl(fifo, runl); | |
299 | ||
300 | /* Determine which channel (if any) is currently on the engine. */ | |
301 | gk104_fifo_engine_status(fifo, engn, &status); | |
302 | if (status.chan) { | |
303 | /* The channel is not longer viable, kill it. */ | |
304 | gk104_fifo_recover_chan(&fifo->base, status.chan->id); | |
305 | } | |
306 | ||
3ebef76a BS |
307 | /* Determine MMU fault ID for the engine, if we're not being |
308 | * called from the fault handler already. | |
309 | */ | |
310 | if (!status.faulted && engine) { | |
311 | mmui = nvkm_top_fault_id(device, engine->subdev.index); | |
312 | if (mmui < 0) { | |
313 | const struct nvkm_enum *en = fifo->func->fault.engine; | |
314 | for (; en && en->name; en++) { | |
315 | if (en->data2 == engine->subdev.index) { | |
316 | mmui = en->value; | |
317 | break; | |
318 | } | |
319 | } | |
320 | } | |
321 | WARN_ON(mmui < 0); | |
322 | } | |
323 | ||
324 | /* Trigger a MMU fault for the engine. | |
325 | * | |
326 | * No good idea why this is needed, but nvgpu does something similar, | |
327 | * and it makes recovery from CTXSW_TIMEOUT a lot more reliable. | |
328 | */ | |
329 | if (mmui >= 0) { | |
330 | nvkm_wr32(device, 0x002a30 + (engn * 0x04), 0x00000100 | mmui); | |
331 | ||
332 | /* Wait for fault to trigger. */ | |
333 | nvkm_msec(device, 2000, | |
334 | gk104_fifo_engine_status(fifo, engn, &status); | |
335 | if (status.faulted) | |
336 | break; | |
337 | ); | |
338 | ||
339 | /* Release MMU fault trigger, and ACK the fault. */ | |
340 | nvkm_wr32(device, 0x002a30 + (engn * 0x04), 0x00000000); | |
341 | nvkm_wr32(device, 0x00259c, BIT(mmui)); | |
342 | nvkm_wr32(device, 0x002100, 0x10000000); | |
343 | } | |
344 | ||
0faaa47d BS |
345 | /* Schedule recovery. */ |
346 | nvkm_warn(subdev, "engine %d: scheduled for recovery\n", engn); | |
55252da1 | 347 | schedule_work(&fifo->recover.work); |
0faaa47d BS |
348 | } |
349 | ||
05c7145d BS |
350 | static const struct nvkm_enum |
351 | gk104_fifo_bind_reason[] = { | |
56b2f68c BS |
352 | { 0x01, "BIND_NOT_UNBOUND" }, |
353 | { 0x02, "SNOOP_WITHOUT_BAR1" }, | |
354 | { 0x03, "UNBIND_WHILE_RUNNING" }, | |
355 | { 0x05, "INVALID_RUNLIST" }, | |
356 | { 0x06, "INVALID_CTX_TGT" }, | |
357 | { 0x0b, "UNBIND_WHILE_PARKED" }, | |
358 | {} | |
359 | }; | |
360 | ||
361 | static void | |
6189f1b0 | 362 | gk104_fifo_intr_bind(struct gk104_fifo *fifo) |
56b2f68c | 363 | { |
e5c5e4f5 BS |
364 | struct nvkm_subdev *subdev = &fifo->base.engine.subdev; |
365 | struct nvkm_device *device = subdev->device; | |
87744403 | 366 | u32 intr = nvkm_rd32(device, 0x00252c); |
56b2f68c | 367 | u32 code = intr & 0x000000ff; |
e5c5e4f5 BS |
368 | const struct nvkm_enum *en = |
369 | nvkm_enum_find(gk104_fifo_bind_reason, code); | |
56b2f68c | 370 | |
e5c5e4f5 | 371 | nvkm_error(subdev, "BIND_ERROR %02x [%s]\n", code, en ? en->name : ""); |
56b2f68c BS |
372 | } |
373 | ||
05c7145d BS |
374 | static const struct nvkm_enum |
375 | gk104_fifo_sched_reason[] = { | |
e9fb9805 BS |
376 | { 0x0a, "CTXSW_TIMEOUT" }, |
377 | {} | |
378 | }; | |
379 | ||
129dcca7 | 380 | static void |
6189f1b0 | 381 | gk104_fifo_intr_sched_ctxsw(struct gk104_fifo *fifo) |
129dcca7 | 382 | { |
03f16f5f | 383 | struct nvkm_device *device = fifo->base.engine.subdev.device; |
91b9d659 | 384 | unsigned long flags, engm = 0; |
129dcca7 BS |
385 | u32 engn; |
386 | ||
03f16f5f BS |
387 | /* We need to ACK the SCHED_ERROR here, and prevent it reasserting, |
388 | * as MMU_FAULT cannot be triggered while it's pending. | |
389 | */ | |
6ca307b0 | 390 | spin_lock_irqsave(&fifo->base.lock, flags); |
03f16f5f BS |
391 | nvkm_mask(device, 0x002140, 0x00000100, 0x00000000); |
392 | nvkm_wr32(device, 0x002100, 0x00000100); | |
393 | ||
41e5171b | 394 | for (engn = 0; engn < fifo->engine_nr; engn++) { |
b88917fe BS |
395 | struct gk104_fifo_engine_status status; |
396 | ||
397 | gk104_fifo_engine_status(fifo, engn, &status); | |
398 | if (!status.busy || !status.chsw) | |
af83a677 BS |
399 | continue; |
400 | ||
91b9d659 | 401 | engm |= BIT(engn); |
129dcca7 | 402 | } |
91b9d659 BS |
403 | |
404 | for_each_set_bit(engn, &engm, fifo->engine_nr) | |
405 | gk104_fifo_recover_engn(fifo, engn); | |
406 | ||
03f16f5f | 407 | nvkm_mask(device, 0x002140, 0x00000100, 0x00000100); |
6ca307b0 | 408 | spin_unlock_irqrestore(&fifo->base.lock, flags); |
129dcca7 BS |
409 | } |
410 | ||
885f3ced | 411 | static void |
6189f1b0 | 412 | gk104_fifo_intr_sched(struct gk104_fifo *fifo) |
885f3ced | 413 | { |
e5c5e4f5 BS |
414 | struct nvkm_subdev *subdev = &fifo->base.engine.subdev; |
415 | struct nvkm_device *device = subdev->device; | |
87744403 | 416 | u32 intr = nvkm_rd32(device, 0x00254c); |
885f3ced | 417 | u32 code = intr & 0x000000ff; |
e5c5e4f5 BS |
418 | const struct nvkm_enum *en = |
419 | nvkm_enum_find(gk104_fifo_sched_reason, code); | |
0a7760e0 | 420 | |
e5c5e4f5 | 421 | nvkm_error(subdev, "SCHED_ERROR %02x [%s]\n", code, en ? en->name : ""); |
129dcca7 BS |
422 | |
423 | switch (code) { | |
424 | case 0x0a: | |
6189f1b0 | 425 | gk104_fifo_intr_sched_ctxsw(fifo); |
129dcca7 BS |
426 | break; |
427 | default: | |
428 | break; | |
429 | } | |
885f3ced BS |
430 | } |
431 | ||
432 | static void | |
6189f1b0 | 433 | gk104_fifo_intr_chsw(struct gk104_fifo *fifo) |
885f3ced | 434 | { |
e5c5e4f5 BS |
435 | struct nvkm_subdev *subdev = &fifo->base.engine.subdev; |
436 | struct nvkm_device *device = subdev->device; | |
87744403 | 437 | u32 stat = nvkm_rd32(device, 0x00256c); |
e5c5e4f5 | 438 | nvkm_error(subdev, "CHSW_ERROR %08x\n", stat); |
87744403 | 439 | nvkm_wr32(device, 0x00256c, stat); |
885f3ced BS |
440 | } |
441 | ||
442 | static void | |
6189f1b0 | 443 | gk104_fifo_intr_dropped_fault(struct gk104_fifo *fifo) |
885f3ced | 444 | { |
e5c5e4f5 BS |
445 | struct nvkm_subdev *subdev = &fifo->base.engine.subdev; |
446 | struct nvkm_device *device = subdev->device; | |
87744403 | 447 | u32 stat = nvkm_rd32(device, 0x00259c); |
e5c5e4f5 | 448 | nvkm_error(subdev, "DROPPED_MMU_FAULT %08x\n", stat); |
885f3ced BS |
449 | } |
450 | ||
e9fb9805 | 451 | static void |
6189f1b0 | 452 | gk104_fifo_intr_fault(struct gk104_fifo *fifo, int unit) |
5132f377 | 453 | { |
e5c5e4f5 BS |
454 | struct nvkm_subdev *subdev = &fifo->base.engine.subdev; |
455 | struct nvkm_device *device = subdev->device; | |
87744403 BS |
456 | u32 inst = nvkm_rd32(device, 0x002800 + (unit * 0x10)); |
457 | u32 valo = nvkm_rd32(device, 0x002804 + (unit * 0x10)); | |
458 | u32 vahi = nvkm_rd32(device, 0x002808 + (unit * 0x10)); | |
459 | u32 stat = nvkm_rd32(device, 0x00280c + (unit * 0x10)); | |
885f3ced | 460 | u32 gpc = (stat & 0x1f000000) >> 24; |
5132f377 | 461 | u32 client = (stat & 0x00001f00) >> 8; |
885f3ced BS |
462 | u32 write = (stat & 0x00000080); |
463 | u32 hub = (stat & 0x00000040); | |
464 | u32 reason = (stat & 0x0000000f); | |
05c7145d | 465 | const struct nvkm_enum *er, *eu, *ec; |
6ca307b0 BS |
466 | struct nvkm_engine *engine = NULL; |
467 | struct nvkm_fifo_chan *chan; | |
468 | unsigned long flags; | |
e50d0237 | 469 | char gpcid[8] = "", en[16] = ""; |
3534821d | 470 | int engn; |
885f3ced | 471 | |
91419acf BS |
472 | er = nvkm_enum_find(fifo->func->fault.reason, reason); |
473 | eu = nvkm_enum_find(fifo->func->fault.engine, unit); | |
e5c5e4f5 | 474 | if (hub) { |
91419acf | 475 | ec = nvkm_enum_find(fifo->func->fault.hubclient, client); |
e5c5e4f5 | 476 | } else { |
91419acf | 477 | ec = nvkm_enum_find(fifo->func->fault.gpcclient, client); |
e5c5e4f5 BS |
478 | snprintf(gpcid, sizeof(gpcid), "GPC%d/", gpc); |
479 | } | |
480 | ||
acdf7d4f | 481 | if (eu && eu->data2) { |
885f3ced | 482 | switch (eu->data2) { |
68f3f702 | 483 | case NVKM_SUBDEV_BAR: |
87744403 | 484 | nvkm_mask(device, 0x001704, 0x00000000, 0x00000000); |
885f3ced | 485 | break; |
68f3f702 | 486 | case NVKM_SUBDEV_INSTMEM: |
87744403 | 487 | nvkm_mask(device, 0x001714, 0x00000000, 0x00000000); |
885f3ced | 488 | break; |
68f3f702 | 489 | case NVKM_ENGINE_IFB: |
87744403 | 490 | nvkm_mask(device, 0x001718, 0x00000000, 0x00000000); |
885f3ced BS |
491 | break; |
492 | default: | |
13de7f46 | 493 | engine = nvkm_device_engine(device, eu->data2); |
885f3ced | 494 | break; |
cb1567c2 | 495 | } |
885f3ced BS |
496 | } |
497 | ||
e50d0237 | 498 | if (eu == NULL) { |
952eb819 | 499 | enum nvkm_devidx engidx = nvkm_top_fault(device, unit); |
e50d0237 BS |
500 | if (engidx < NVKM_SUBDEV_NR) { |
501 | const char *src = nvkm_subdev_name[engidx]; | |
502 | char *dst = en; | |
503 | do { | |
504 | *dst++ = toupper(*src++); | |
505 | } while(*src); | |
506 | engine = nvkm_device_engine(device, engidx); | |
507 | } | |
508 | } else { | |
509 | snprintf(en, sizeof(en), "%s", eu->name); | |
510 | } | |
511 | ||
3534821d BS |
512 | spin_lock_irqsave(&fifo->base.lock, flags); |
513 | chan = nvkm_fifo_chan_inst_locked(&fifo->base, (u64)inst << 12); | |
6ca307b0 | 514 | |
e5c5e4f5 BS |
515 | nvkm_error(subdev, |
516 | "%s fault at %010llx engine %02x [%s] client %02x [%s%s] " | |
517 | "reason %02x [%s] on channel %d [%010llx %s]\n", | |
518 | write ? "write" : "read", (u64)vahi << 32 | valo, | |
e50d0237 | 519 | unit, en, client, gpcid, ec ? ec->name : "", |
6ca307b0 | 520 | reason, er ? er->name : "", chan ? chan->chid : -1, |
8f0649b5 BS |
521 | (u64)inst << 12, |
522 | chan ? chan->object.client->name : "unknown"); | |
98d1e317 | 523 | |
3534821d BS |
524 | |
525 | /* Kill the channel that caused the fault. */ | |
526 | if (chan) | |
527 | gk104_fifo_recover_chan(&fifo->base, chan->chid); | |
528 | ||
529 | /* Channel recovery will probably have already done this for the | |
530 | * correct engine(s), but just in case we can't find the channel | |
531 | * information... | |
532 | */ | |
533 | for (engn = 0; engn < fifo->engine_nr && engine; engn++) { | |
534 | if (fifo->engine[engn].engine == engine) { | |
535 | gk104_fifo_recover_engn(fifo, engn); | |
536 | break; | |
537 | } | |
538 | } | |
539 | ||
540 | spin_unlock_irqrestore(&fifo->base.lock, flags); | |
5132f377 BS |
541 | } |
542 | ||
05c7145d | 543 | static const struct nvkm_bitfield gk104_fifo_pbdma_intr_0[] = { |
3d61b967 BS |
544 | { 0x00000001, "MEMREQ" }, |
545 | { 0x00000002, "MEMACK_TIMEOUT" }, | |
546 | { 0x00000004, "MEMACK_EXTRA" }, | |
547 | { 0x00000008, "MEMDAT_TIMEOUT" }, | |
548 | { 0x00000010, "MEMDAT_EXTRA" }, | |
549 | { 0x00000020, "MEMFLUSH" }, | |
550 | { 0x00000040, "MEMOP" }, | |
551 | { 0x00000080, "LBCONNECT" }, | |
552 | { 0x00000100, "LBREQ" }, | |
553 | { 0x00000200, "LBACK_TIMEOUT" }, | |
554 | { 0x00000400, "LBACK_EXTRA" }, | |
555 | { 0x00000800, "LBDAT_TIMEOUT" }, | |
556 | { 0x00001000, "LBDAT_EXTRA" }, | |
557 | { 0x00002000, "GPFIFO" }, | |
558 | { 0x00004000, "GPPTR" }, | |
559 | { 0x00008000, "GPENTRY" }, | |
560 | { 0x00010000, "GPCRC" }, | |
561 | { 0x00020000, "PBPTR" }, | |
562 | { 0x00040000, "PBENTRY" }, | |
563 | { 0x00080000, "PBCRC" }, | |
564 | { 0x00100000, "XBARCONNECT" }, | |
565 | { 0x00200000, "METHOD" }, | |
566 | { 0x00400000, "METHODCRC" }, | |
567 | { 0x00800000, "DEVICE" }, | |
568 | { 0x02000000, "SEMAPHORE" }, | |
569 | { 0x04000000, "ACQUIRE" }, | |
570 | { 0x08000000, "PRI" }, | |
571 | { 0x20000000, "NO_CTXSW_SEG" }, | |
572 | { 0x40000000, "PBSEG" }, | |
573 | { 0x80000000, "SIGNATURE" }, | |
574 | {} | |
575 | }; | |
e2b34fa0 | 576 | |
5132f377 | 577 | static void |
6189f1b0 | 578 | gk104_fifo_intr_pbdma_0(struct gk104_fifo *fifo, int unit) |
5132f377 | 579 | { |
e5c5e4f5 BS |
580 | struct nvkm_subdev *subdev = &fifo->base.engine.subdev; |
581 | struct nvkm_device *device = subdev->device; | |
87744403 BS |
582 | u32 mask = nvkm_rd32(device, 0x04010c + (unit * 0x2000)); |
583 | u32 stat = nvkm_rd32(device, 0x040108 + (unit * 0x2000)) & mask; | |
584 | u32 addr = nvkm_rd32(device, 0x0400c0 + (unit * 0x2000)); | |
585 | u32 data = nvkm_rd32(device, 0x0400c4 + (unit * 0x2000)); | |
586 | u32 chid = nvkm_rd32(device, 0x040120 + (unit * 0x2000)) & 0xfff; | |
ebb945a9 | 587 | u32 subc = (addr & 0x00070000) >> 16; |
5132f377 | 588 | u32 mthd = (addr & 0x00003ffc); |
e2b34fa0 | 589 | u32 show = stat; |
8f0649b5 BS |
590 | struct nvkm_fifo_chan *chan; |
591 | unsigned long flags; | |
e5c5e4f5 | 592 | char msg[128]; |
e2b34fa0 | 593 | |
ebb945a9 | 594 | if (stat & 0x00800000) { |
61570911 BS |
595 | if (device->sw) { |
596 | if (nvkm_sw_mthd(device->sw, chid, subc, mthd, data)) | |
597 | show &= ~0x00800000; | |
598 | } | |
ebb945a9 BS |
599 | } |
600 | ||
b4c5fc4b BS |
601 | nvkm_wr32(device, 0x0400c0 + (unit * 0x2000), 0x80600008); |
602 | ||
e2b34fa0 | 603 | if (show) { |
e5c5e4f5 | 604 | nvkm_snprintbf(msg, sizeof(msg), gk104_fifo_pbdma_intr_0, show); |
8f0649b5 BS |
605 | chan = nvkm_fifo_chan_chid(&fifo->base, chid, &flags); |
606 | nvkm_error(subdev, "PBDMA%d: %08x [%s] ch %d [%010llx %s] " | |
607 | "subc %d mthd %04x data %08x\n", | |
608 | unit, show, msg, chid, chan ? chan->inst->addr : 0, | |
609 | chan ? chan->object.client->name : "unknown", | |
e5c5e4f5 | 610 | subc, mthd, data); |
8f0649b5 | 611 | nvkm_fifo_chan_put(&fifo->base, flags, &chan); |
e2b34fa0 | 612 | } |
5132f377 | 613 | |
87744403 | 614 | nvkm_wr32(device, 0x040108 + (unit * 0x2000), stat); |
5132f377 BS |
615 | } |
616 | ||
05c7145d | 617 | static const struct nvkm_bitfield gk104_fifo_pbdma_intr_1[] = { |
70b2cc8e BS |
618 | { 0x00000001, "HCE_RE_ILLEGAL_OP" }, |
619 | { 0x00000002, "HCE_RE_ALIGNB" }, | |
620 | { 0x00000004, "HCE_PRIV" }, | |
621 | { 0x00000008, "HCE_ILLEGAL_MTHD" }, | |
622 | { 0x00000010, "HCE_ILLEGAL_CLASS" }, | |
623 | {} | |
624 | }; | |
625 | ||
626 | static void | |
6189f1b0 | 627 | gk104_fifo_intr_pbdma_1(struct gk104_fifo *fifo, int unit) |
70b2cc8e | 628 | { |
e5c5e4f5 BS |
629 | struct nvkm_subdev *subdev = &fifo->base.engine.subdev; |
630 | struct nvkm_device *device = subdev->device; | |
87744403 BS |
631 | u32 mask = nvkm_rd32(device, 0x04014c + (unit * 0x2000)); |
632 | u32 stat = nvkm_rd32(device, 0x040148 + (unit * 0x2000)) & mask; | |
633 | u32 chid = nvkm_rd32(device, 0x040120 + (unit * 0x2000)) & 0xfff; | |
e5c5e4f5 | 634 | char msg[128]; |
70b2cc8e BS |
635 | |
636 | if (stat) { | |
e5c5e4f5 BS |
637 | nvkm_snprintbf(msg, sizeof(msg), gk104_fifo_pbdma_intr_1, stat); |
638 | nvkm_error(subdev, "PBDMA%d: %08x [%s] ch %d %08x %08x\n", | |
639 | unit, stat, msg, chid, | |
640 | nvkm_rd32(device, 0x040150 + (unit * 0x2000)), | |
641 | nvkm_rd32(device, 0x040154 + (unit * 0x2000))); | |
70b2cc8e BS |
642 | } |
643 | ||
87744403 | 644 | nvkm_wr32(device, 0x040148 + (unit * 0x2000), stat); |
70b2cc8e BS |
645 | } |
646 | ||
138b873f | 647 | static void |
6189f1b0 | 648 | gk104_fifo_intr_runlist(struct gk104_fifo *fifo) |
138b873f | 649 | { |
87744403 BS |
650 | struct nvkm_device *device = fifo->base.engine.subdev.device; |
651 | u32 mask = nvkm_rd32(device, 0x002a00); | |
138b873f | 652 | while (mask) { |
69aa40e2 BS |
653 | int runl = __ffs(mask); |
654 | wake_up(&fifo->runlist[runl].wait); | |
655 | nvkm_wr32(device, 0x002a00, 1 << runl); | |
656 | mask &= ~(1 << runl); | |
138b873f BS |
657 | } |
658 | } | |
659 | ||
c074bdbc | 660 | static void |
6189f1b0 | 661 | gk104_fifo_intr_engine(struct gk104_fifo *fifo) |
c074bdbc | 662 | { |
6189f1b0 | 663 | nvkm_fifo_uevent(&fifo->base); |
c074bdbc BS |
664 | } |
665 | ||
98ac3f06 | 666 | static void |
13de7f46 | 667 | gk104_fifo_intr(struct nvkm_fifo *base) |
5132f377 | 668 | { |
13de7f46 BS |
669 | struct gk104_fifo *fifo = gk104_fifo(base); |
670 | struct nvkm_subdev *subdev = &fifo->base.engine.subdev; | |
671 | struct nvkm_device *device = subdev->device; | |
87744403 BS |
672 | u32 mask = nvkm_rd32(device, 0x002140); |
673 | u32 stat = nvkm_rd32(device, 0x002100) & mask; | |
5132f377 | 674 | |
e9fb9805 | 675 | if (stat & 0x00000001) { |
6189f1b0 | 676 | gk104_fifo_intr_bind(fifo); |
87744403 | 677 | nvkm_wr32(device, 0x002100, 0x00000001); |
e9fb9805 BS |
678 | stat &= ~0x00000001; |
679 | } | |
680 | ||
681 | if (stat & 0x00000010) { | |
e5c5e4f5 | 682 | nvkm_error(subdev, "PIO_ERROR\n"); |
87744403 | 683 | nvkm_wr32(device, 0x002100, 0x00000010); |
e9fb9805 BS |
684 | stat &= ~0x00000010; |
685 | } | |
686 | ||
5132f377 | 687 | if (stat & 0x00000100) { |
6189f1b0 | 688 | gk104_fifo_intr_sched(fifo); |
87744403 | 689 | nvkm_wr32(device, 0x002100, 0x00000100); |
5132f377 BS |
690 | stat &= ~0x00000100; |
691 | } | |
692 | ||
e9fb9805 | 693 | if (stat & 0x00010000) { |
6189f1b0 | 694 | gk104_fifo_intr_chsw(fifo); |
87744403 | 695 | nvkm_wr32(device, 0x002100, 0x00010000); |
e9fb9805 BS |
696 | stat &= ~0x00010000; |
697 | } | |
698 | ||
699 | if (stat & 0x00800000) { | |
e5c5e4f5 | 700 | nvkm_error(subdev, "FB_FLUSH_TIMEOUT\n"); |
87744403 | 701 | nvkm_wr32(device, 0x002100, 0x00800000); |
e9fb9805 BS |
702 | stat &= ~0x00800000; |
703 | } | |
704 | ||
705 | if (stat & 0x01000000) { | |
e5c5e4f5 | 706 | nvkm_error(subdev, "LB_ERROR\n"); |
87744403 | 707 | nvkm_wr32(device, 0x002100, 0x01000000); |
e9fb9805 BS |
708 | stat &= ~0x01000000; |
709 | } | |
710 | ||
711 | if (stat & 0x08000000) { | |
6189f1b0 | 712 | gk104_fifo_intr_dropped_fault(fifo); |
87744403 | 713 | nvkm_wr32(device, 0x002100, 0x08000000); |
e9fb9805 BS |
714 | stat &= ~0x08000000; |
715 | } | |
716 | ||
5132f377 | 717 | if (stat & 0x10000000) { |
87744403 | 718 | u32 mask = nvkm_rd32(device, 0x00259c); |
885f3ced BS |
719 | while (mask) { |
720 | u32 unit = __ffs(mask); | |
6189f1b0 | 721 | gk104_fifo_intr_fault(fifo, unit); |
87744403 | 722 | nvkm_wr32(device, 0x00259c, (1 << unit)); |
885f3ced | 723 | mask &= ~(1 << unit); |
5132f377 | 724 | } |
5132f377 BS |
725 | stat &= ~0x10000000; |
726 | } | |
727 | ||
728 | if (stat & 0x20000000) { | |
87744403 | 729 | u32 mask = nvkm_rd32(device, 0x0025a0); |
3d61b967 BS |
730 | while (mask) { |
731 | u32 unit = __ffs(mask); | |
6189f1b0 BS |
732 | gk104_fifo_intr_pbdma_0(fifo, unit); |
733 | gk104_fifo_intr_pbdma_1(fifo, unit); | |
87744403 | 734 | nvkm_wr32(device, 0x0025a0, (1 << unit)); |
3d61b967 | 735 | mask &= ~(1 << unit); |
5132f377 | 736 | } |
5132f377 BS |
737 | stat &= ~0x20000000; |
738 | } | |
739 | ||
740 | if (stat & 0x40000000) { | |
6189f1b0 | 741 | gk104_fifo_intr_runlist(fifo); |
5132f377 BS |
742 | stat &= ~0x40000000; |
743 | } | |
744 | ||
9bd2ddba | 745 | if (stat & 0x80000000) { |
87744403 | 746 | nvkm_wr32(device, 0x002100, 0x80000000); |
6189f1b0 | 747 | gk104_fifo_intr_engine(fifo); |
9bd2ddba BS |
748 | stat &= ~0x80000000; |
749 | } | |
750 | ||
5132f377 | 751 | if (stat) { |
e5c5e4f5 | 752 | nvkm_error(subdev, "INTR %08x\n", stat); |
87744403 BS |
753 | nvkm_mask(device, 0x002140, stat, 0x00000000); |
754 | nvkm_wr32(device, 0x002100, stat); | |
5132f377 BS |
755 | } |
756 | } | |
c420b2dc | 757 | |
98ac3f06 | 758 | static void |
13de7f46 BS |
759 | gk104_fifo_fini(struct nvkm_fifo *base) |
760 | { | |
761 | struct gk104_fifo *fifo = gk104_fifo(base); | |
762 | struct nvkm_device *device = fifo->base.engine.subdev.device; | |
55252da1 | 763 | flush_work(&fifo->recover.work); |
13de7f46 BS |
764 | /* allow mmu fault interrupts, even when we're not using fifo */ |
765 | nvkm_mask(device, 0x002140, 0x10000000, 0x10000000); | |
766 | } | |
767 | ||
98ac3f06 | 768 | static int |
13de7f46 | 769 | gk104_fifo_oneinit(struct nvkm_fifo *base) |
649ec925 | 770 | { |
13de7f46 | 771 | struct gk104_fifo *fifo = gk104_fifo(base); |
6d39b83f BS |
772 | struct nvkm_subdev *subdev = &fifo->base.engine.subdev; |
773 | struct nvkm_device *device = subdev->device; | |
e93e198d BS |
774 | int engn, runl, pbid, ret, i, j; |
775 | enum nvkm_devidx engidx; | |
41e5171b | 776 | u32 *map; |
649ec925 | 777 | |
6d39b83f BS |
778 | /* Determine number of PBDMAs by checking valid enable bits. */ |
779 | nvkm_wr32(device, 0x000204, 0xffffffff); | |
780 | fifo->pbdma_nr = hweight32(nvkm_rd32(device, 0x000204)); | |
781 | nvkm_debug(subdev, "%d PBDMA(s)\n", fifo->pbdma_nr); | |
782 | ||
41e5171b BS |
783 | /* Read PBDMA->runlist(s) mapping from HW. */ |
784 | if (!(map = kzalloc(sizeof(*map) * fifo->pbdma_nr, GFP_KERNEL))) | |
785 | return -ENOMEM; | |
786 | ||
787 | for (i = 0; i < fifo->pbdma_nr; i++) | |
788 | map[i] = nvkm_rd32(device, 0x002390 + (i * 0x04)); | |
789 | ||
e93e198d BS |
790 | /* Determine runlist configuration from topology device info. */ |
791 | i = 0; | |
952eb819 | 792 | while ((int)(engidx = nvkm_top_engine(device, i++, &runl, &engn)) >= 0) { |
41e5171b | 793 | /* Determine which PBDMA handles requests for this engine. */ |
e93e198d | 794 | for (j = 0, pbid = -1; j < fifo->pbdma_nr; j++) { |
41e5171b BS |
795 | if (map[j] & (1 << runl)) { |
796 | pbid = j; | |
797 | break; | |
798 | } | |
799 | } | |
800 | ||
1fe8c02f BS |
801 | nvkm_debug(subdev, "engine %2d: runlist %2d pbdma %2d (%s)\n", |
802 | engn, runl, pbid, nvkm_subdev_name[engidx]); | |
41e5171b | 803 | |
e93e198d BS |
804 | fifo->engine[engn].engine = nvkm_device_engine(device, engidx); |
805 | fifo->engine[engn].runl = runl; | |
806 | fifo->engine[engn].pbid = pbid; | |
807 | fifo->engine_nr = max(fifo->engine_nr, engn + 1); | |
808 | fifo->runlist[runl].engm |= 1 << engn; | |
809 | fifo->runlist_nr = max(fifo->runlist_nr, runl + 1); | |
41e5171b BS |
810 | } |
811 | ||
812 | kfree(map); | |
813 | ||
814 | for (i = 0; i < fifo->runlist_nr; i++) { | |
13de7f46 BS |
815 | ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, |
816 | 0x8000, 0x1000, false, | |
69aa40e2 | 817 | &fifo->runlist[i].mem[0]); |
13de7f46 BS |
818 | if (ret) |
819 | return ret; | |
820 | ||
821 | ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, | |
822 | 0x8000, 0x1000, false, | |
69aa40e2 | 823 | &fifo->runlist[i].mem[1]); |
13de7f46 BS |
824 | if (ret) |
825 | return ret; | |
826 | ||
69aa40e2 BS |
827 | init_waitqueue_head(&fifo->runlist[i].wait); |
828 | INIT_LIST_HEAD(&fifo->runlist[i].chan); | |
13de7f46 BS |
829 | } |
830 | ||
831 | ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, | |
832 | fifo->base.nr * 0x200, 0x1000, true, | |
833 | &fifo->user.mem); | |
649ec925 BS |
834 | if (ret) |
835 | return ret; | |
836 | ||
13de7f46 BS |
837 | ret = nvkm_bar_umap(device->bar, fifo->base.nr * 0x200, 12, |
838 | &fifo->user.bar); | |
839 | if (ret) | |
840 | return ret; | |
841 | ||
842 | nvkm_memory_map(fifo->user.mem, &fifo->user.bar, 0); | |
649ec925 BS |
843 | return 0; |
844 | } | |
845 | ||
98ac3f06 | 846 | static void |
13de7f46 | 847 | gk104_fifo_init(struct nvkm_fifo *base) |
a763951a | 848 | { |
13de7f46 | 849 | struct gk104_fifo *fifo = gk104_fifo(base); |
6d39b83f | 850 | struct nvkm_device *device = fifo->base.engine.subdev.device; |
13de7f46 | 851 | int i; |
a763951a | 852 | |
6d39b83f BS |
853 | /* Enable PBDMAs. */ |
854 | nvkm_wr32(device, 0x000204, (1 << fifo->pbdma_nr) - 1); | |
a763951a | 855 | |
39b05542 | 856 | /* PBDMA[n] */ |
6d39b83f | 857 | for (i = 0; i < fifo->pbdma_nr; i++) { |
87744403 BS |
858 | nvkm_mask(device, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000); |
859 | nvkm_wr32(device, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */ | |
860 | nvkm_wr32(device, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTREN */ | |
a763951a BS |
861 | } |
862 | ||
70b2cc8e | 863 | /* PBDMA[n].HCE */ |
6d39b83f | 864 | for (i = 0; i < fifo->pbdma_nr; i++) { |
87744403 BS |
865 | nvkm_wr32(device, 0x040148 + (i * 0x2000), 0xffffffff); /* INTR */ |
866 | nvkm_wr32(device, 0x04014c + (i * 0x2000), 0xffffffff); /* INTREN */ | |
70b2cc8e BS |
867 | } |
868 | ||
87744403 | 869 | nvkm_wr32(device, 0x002254, 0x10000000 | fifo->user.bar.offset >> 12); |
a763951a | 870 | |
87744403 BS |
871 | nvkm_wr32(device, 0x002100, 0xffffffff); |
872 | nvkm_wr32(device, 0x002140, 0x7fffffff); | |
a763951a BS |
873 | } |
874 | ||
98ac3f06 | 875 | static void * |
13de7f46 | 876 | gk104_fifo_dtor(struct nvkm_fifo *base) |
a763951a | 877 | { |
13de7f46 | 878 | struct gk104_fifo *fifo = gk104_fifo(base); |
a763951a BS |
879 | int i; |
880 | ||
358ce601 BS |
881 | nvkm_vm_put(&fifo->user.bar); |
882 | nvkm_memory_del(&fifo->user.mem); | |
a763951a | 883 | |
41e5171b | 884 | for (i = 0; i < fifo->runlist_nr; i++) { |
69aa40e2 BS |
885 | nvkm_memory_del(&fifo->runlist[i].mem[1]); |
886 | nvkm_memory_del(&fifo->runlist[i].mem[0]); | |
a763951a BS |
887 | } |
888 | ||
13de7f46 BS |
889 | return fifo; |
890 | } | |
891 | ||
98ac3f06 BS |
892 | static const struct nvkm_fifo_func |
893 | gk104_fifo_ = { | |
894 | .dtor = gk104_fifo_dtor, | |
895 | .oneinit = gk104_fifo_oneinit, | |
896 | .init = gk104_fifo_init, | |
897 | .fini = gk104_fifo_fini, | |
898 | .intr = gk104_fifo_intr, | |
899 | .uevent_init = gk104_fifo_uevent_init, | |
900 | .uevent_fini = gk104_fifo_uevent_fini, | |
0faaa47d | 901 | .recover_chan = gk104_fifo_recover_chan, |
98ac3f06 BS |
902 | .class_get = gk104_fifo_class_get, |
903 | }; | |
904 | ||
13de7f46 | 905 | int |
98ac3f06 | 906 | gk104_fifo_new_(const struct gk104_fifo_func *func, struct nvkm_device *device, |
13de7f46 BS |
907 | int index, int nr, struct nvkm_fifo **pfifo) |
908 | { | |
909 | struct gk104_fifo *fifo; | |
910 | ||
911 | if (!(fifo = kzalloc(sizeof(*fifo), GFP_KERNEL))) | |
912 | return -ENOMEM; | |
98ac3f06 | 913 | fifo->func = func; |
55252da1 | 914 | INIT_WORK(&fifo->recover.work, gk104_fifo_recover_work); |
13de7f46 BS |
915 | *pfifo = &fifo->base; |
916 | ||
98ac3f06 | 917 | return nvkm_fifo_ctor(&gk104_fifo_, device, index, nr, &fifo->base); |
a763951a BS |
918 | } |
919 | ||
91419acf BS |
920 | const struct nvkm_enum |
921 | gk104_fifo_fault_engine[] = { | |
922 | { 0x00, "GR", NULL, NVKM_ENGINE_GR }, | |
289e0827 BS |
923 | { 0x01, "DISPLAY" }, |
924 | { 0x02, "CAPTURE" }, | |
91419acf BS |
925 | { 0x03, "IFB", NULL, NVKM_ENGINE_IFB }, |
926 | { 0x04, "BAR1", NULL, NVKM_SUBDEV_BAR }, | |
289e0827 BS |
927 | { 0x05, "BAR2", NULL, NVKM_SUBDEV_INSTMEM }, |
928 | { 0x06, "SCHED" }, | |
ec884f74 BS |
929 | { 0x07, "HOST0", NULL, NVKM_ENGINE_FIFO }, |
930 | { 0x08, "HOST1", NULL, NVKM_ENGINE_FIFO }, | |
931 | { 0x09, "HOST2", NULL, NVKM_ENGINE_FIFO }, | |
932 | { 0x0a, "HOST3", NULL, NVKM_ENGINE_FIFO }, | |
933 | { 0x0b, "HOST4", NULL, NVKM_ENGINE_FIFO }, | |
934 | { 0x0c, "HOST5", NULL, NVKM_ENGINE_FIFO }, | |
935 | { 0x0d, "HOST6", NULL, NVKM_ENGINE_FIFO }, | |
936 | { 0x0e, "HOST7", NULL, NVKM_ENGINE_FIFO }, | |
289e0827 | 937 | { 0x0f, "HOSTSR" }, |
91419acf BS |
938 | { 0x10, "MSVLD", NULL, NVKM_ENGINE_MSVLD }, |
939 | { 0x11, "MSPPP", NULL, NVKM_ENGINE_MSPPP }, | |
940 | { 0x13, "PERF" }, | |
941 | { 0x14, "MSPDEC", NULL, NVKM_ENGINE_MSPDEC }, | |
942 | { 0x15, "CE0", NULL, NVKM_ENGINE_CE0 }, | |
943 | { 0x16, "CE1", NULL, NVKM_ENGINE_CE1 }, | |
944 | { 0x17, "PMU" }, | |
289e0827 | 945 | { 0x18, "PTP" }, |
91419acf BS |
946 | { 0x19, "MSENC", NULL, NVKM_ENGINE_MSENC }, |
947 | { 0x1b, "CE2", NULL, NVKM_ENGINE_CE2 }, | |
948 | {} | |
949 | }; | |
950 | ||
951 | const struct nvkm_enum | |
952 | gk104_fifo_fault_reason[] = { | |
953 | { 0x00, "PDE" }, | |
954 | { 0x01, "PDE_SIZE" }, | |
955 | { 0x02, "PTE" }, | |
956 | { 0x03, "VA_LIMIT_VIOLATION" }, | |
957 | { 0x04, "UNBOUND_INST_BLOCK" }, | |
958 | { 0x05, "PRIV_VIOLATION" }, | |
959 | { 0x06, "RO_VIOLATION" }, | |
960 | { 0x07, "WO_VIOLATION" }, | |
961 | { 0x08, "PITCH_MASK_VIOLATION" }, | |
962 | { 0x09, "WORK_CREATION" }, | |
963 | { 0x0a, "UNSUPPORTED_APERTURE" }, | |
964 | { 0x0b, "COMPRESSION_FAILURE" }, | |
965 | { 0x0c, "UNSUPPORTED_KIND" }, | |
966 | { 0x0d, "REGION_VIOLATION" }, | |
967 | { 0x0e, "BOTH_PTES_VALID" }, | |
968 | { 0x0f, "INFO_TYPE_POISONED" }, | |
969 | {} | |
970 | }; | |
971 | ||
972 | const struct nvkm_enum | |
973 | gk104_fifo_fault_hubclient[] = { | |
974 | { 0x00, "VIP" }, | |
975 | { 0x01, "CE0" }, | |
976 | { 0x02, "CE1" }, | |
977 | { 0x03, "DNISO" }, | |
978 | { 0x04, "FE" }, | |
979 | { 0x05, "FECS" }, | |
980 | { 0x06, "HOST" }, | |
981 | { 0x07, "HOST_CPU" }, | |
982 | { 0x08, "HOST_CPU_NB" }, | |
983 | { 0x09, "ISO" }, | |
984 | { 0x0a, "MMU" }, | |
985 | { 0x0b, "MSPDEC" }, | |
986 | { 0x0c, "MSPPP" }, | |
987 | { 0x0d, "MSVLD" }, | |
988 | { 0x0e, "NISO" }, | |
989 | { 0x0f, "P2P" }, | |
990 | { 0x10, "PD" }, | |
991 | { 0x11, "PERF" }, | |
992 | { 0x12, "PMU" }, | |
993 | { 0x13, "RASTERTWOD" }, | |
994 | { 0x14, "SCC" }, | |
995 | { 0x15, "SCC_NB" }, | |
996 | { 0x16, "SEC" }, | |
997 | { 0x17, "SSYNC" }, | |
998 | { 0x18, "GR_CE" }, | |
999 | { 0x19, "CE2" }, | |
1000 | { 0x1a, "XV" }, | |
1001 | { 0x1b, "MMU_NB" }, | |
1002 | { 0x1c, "MSENC" }, | |
1003 | { 0x1d, "DFALCON" }, | |
1004 | { 0x1e, "SKED" }, | |
1005 | { 0x1f, "AFALCON" }, | |
1006 | {} | |
1007 | }; | |
1008 | ||
1009 | const struct nvkm_enum | |
1010 | gk104_fifo_fault_gpcclient[] = { | |
1011 | { 0x00, "L1_0" }, { 0x01, "T1_0" }, { 0x02, "PE_0" }, | |
1012 | { 0x03, "L1_1" }, { 0x04, "T1_1" }, { 0x05, "PE_1" }, | |
1013 | { 0x06, "L1_2" }, { 0x07, "T1_2" }, { 0x08, "PE_2" }, | |
1014 | { 0x09, "L1_3" }, { 0x0a, "T1_3" }, { 0x0b, "PE_3" }, | |
1015 | { 0x0c, "RAST" }, | |
1016 | { 0x0d, "GCC" }, | |
1017 | { 0x0e, "GPCCS" }, | |
1018 | { 0x0f, "PROP_0" }, | |
1019 | { 0x10, "PROP_1" }, | |
1020 | { 0x11, "PROP_2" }, | |
1021 | { 0x12, "PROP_3" }, | |
1022 | { 0x13, "L1_4" }, { 0x14, "T1_4" }, { 0x15, "PE_4" }, | |
1023 | { 0x16, "L1_5" }, { 0x17, "T1_5" }, { 0x18, "PE_5" }, | |
1024 | { 0x19, "L1_6" }, { 0x1a, "T1_6" }, { 0x1b, "PE_6" }, | |
1025 | { 0x1c, "L1_7" }, { 0x1d, "T1_7" }, { 0x1e, "PE_7" }, | |
1026 | { 0x1f, "GPM" }, | |
1027 | { 0x20, "LTP_UTLB_0" }, | |
1028 | { 0x21, "LTP_UTLB_1" }, | |
1029 | { 0x22, "LTP_UTLB_2" }, | |
1030 | { 0x23, "LTP_UTLB_3" }, | |
1031 | { 0x24, "GPC_RGG_UTLB" }, | |
1032 | {} | |
1033 | }; | |
1034 | ||
98ac3f06 | 1035 | static const struct gk104_fifo_func |
13de7f46 | 1036 | gk104_fifo = { |
91419acf BS |
1037 | .fault.engine = gk104_fifo_fault_engine, |
1038 | .fault.reason = gk104_fifo_fault_reason, | |
1039 | .fault.hubclient = gk104_fifo_fault_hubclient, | |
1040 | .fault.gpcclient = gk104_fifo_fault_gpcclient, | |
8f0649b5 BS |
1041 | .chan = { |
1042 | &gk104_fifo_gpfifo_oclass, | |
1043 | NULL | |
1044 | }, | |
1045 | }; | |
1046 | ||
a763951a | 1047 | int |
13de7f46 | 1048 | gk104_fifo_new(struct nvkm_device *device, int index, struct nvkm_fifo **pfifo) |
ebb945a9 | 1049 | { |
13de7f46 | 1050 | return gk104_fifo_new_(&gk104_fifo, device, index, 4096, pfifo); |
ebb945a9 | 1051 | } |