]>
Commit | Line | Data |
---|---|---|
5132f377 | 1 | /* |
ebb945a9 | 2 | * Copyright 2012 Red Hat Inc. |
5132f377 BS |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | * Authors: Ben Skeggs | |
23 | */ | |
24 | ||
ebb945a9 BS |
25 | #include <core/client.h> |
26 | #include <core/handle.h> | |
27 | #include <core/namedb.h> | |
28 | #include <core/gpuobj.h> | |
29 | #include <core/engctx.h> | |
9bd2ddba | 30 | #include <core/event.h> |
bbf8906b BS |
31 | #include <nvif/unpack.h> |
32 | #include <nvif/class.h> | |
ebb945a9 | 33 | #include <core/enum.h> |
5132f377 | 34 | |
ebb945a9 BS |
35 | #include <subdev/timer.h> |
36 | #include <subdev/bar.h> | |
52225551 | 37 | #include <subdev/fb.h> |
ebb945a9 | 38 | #include <subdev/vm.h> |
5132f377 | 39 | |
ebb945a9 | 40 | #include <engine/dmaobj.h> |
a763951a BS |
41 | |
42 | #include "nve0.h" | |
5132f377 | 43 | |
507ceb15 | 44 | #define _(a,b) { (a), ((1ULL << (a)) | (b)) } |
dbff2dee | 45 | static const struct { |
507ceb15 MP |
46 | u64 subdev; |
47 | u64 mask; | |
dbff2dee | 48 | } fifo_engine[] = { |
48506d17 BS |
49 | _(NVDEV_ENGINE_GR , (1ULL << NVDEV_ENGINE_SW) | |
50 | (1ULL << NVDEV_ENGINE_COPY2)), | |
dbff2dee BS |
51 | _(NVDEV_ENGINE_VP , 0), |
52 | _(NVDEV_ENGINE_PPP , 0), | |
53 | _(NVDEV_ENGINE_BSP , 0), | |
54 | _(NVDEV_ENGINE_COPY0 , 0), | |
55 | _(NVDEV_ENGINE_COPY1 , 0), | |
56 | _(NVDEV_ENGINE_VENC , 0), | |
57 | }; | |
58 | #undef _ | |
59 | #define FIFO_ENGINE_NR ARRAY_SIZE(fifo_engine) | |
60 | ||
ebb945a9 | 61 | struct nve0_fifo_engn { |
f82c44a7 BS |
62 | struct nouveau_gpuobj *runlist[2]; |
63 | int cur_runlist; | |
138b873f | 64 | wait_queue_head_t wait; |
5132f377 BS |
65 | }; |
66 | ||
67 | struct nve0_fifo_priv { | |
ebb945a9 | 68 | struct nouveau_fifo base; |
98d1e317 BS |
69 | |
70 | struct work_struct fault; | |
71 | u64 mask; | |
72 | ||
dbff2dee | 73 | struct nve0_fifo_engn engine[FIFO_ENGINE_NR]; |
5132f377 BS |
74 | struct { |
75 | struct nouveau_gpuobj *mem; | |
76 | struct nouveau_vma bar; | |
77 | } user; | |
78 | int spoon_nr; | |
79 | }; | |
80 | ||
ebb945a9 BS |
81 | struct nve0_fifo_base { |
82 | struct nouveau_fifo_base base; | |
83 | struct nouveau_gpuobj *pgd; | |
84 | struct nouveau_vm *vm; | |
85 | }; | |
86 | ||
5132f377 | 87 | struct nve0_fifo_chan { |
c420b2dc | 88 | struct nouveau_fifo_chan base; |
5132f377 | 89 | u32 engine; |
87032e11 BS |
90 | enum { |
91 | STOPPED, | |
92 | RUNNING, | |
93 | KILLED | |
94 | } state; | |
5132f377 BS |
95 | }; |
96 | ||
ebb945a9 BS |
97 | /******************************************************************************* |
98 | * FIFO channel objects | |
99 | ******************************************************************************/ | |
100 | ||
5132f377 | 101 | static void |
f82c44a7 | 102 | nve0_fifo_runlist_update(struct nve0_fifo_priv *priv, u32 engine) |
5132f377 | 103 | { |
ebb945a9 BS |
104 | struct nouveau_bar *bar = nouveau_bar(priv); |
105 | struct nve0_fifo_engn *engn = &priv->engine[engine]; | |
5132f377 | 106 | struct nouveau_gpuobj *cur; |
ebb945a9 | 107 | int i, p; |
5132f377 | 108 | |
c2e3259b | 109 | mutex_lock(&nv_subdev(priv)->mutex); |
f82c44a7 BS |
110 | cur = engn->runlist[engn->cur_runlist]; |
111 | engn->cur_runlist = !engn->cur_runlist; | |
5132f377 | 112 | |
ebb945a9 | 113 | for (i = 0, p = 0; i < priv->base.max; i++) { |
87032e11 BS |
114 | struct nve0_fifo_chan *chan = (void *)priv->base.channel[i]; |
115 | if (chan && chan->state == RUNNING && chan->engine == engine) { | |
116 | nv_wo32(cur, p + 0, i); | |
117 | nv_wo32(cur, p + 4, 0x00000000); | |
118 | p += 8; | |
119 | } | |
5132f377 | 120 | } |
ebb945a9 | 121 | bar->flush(bar); |
5132f377 | 122 | |
ebb945a9 BS |
123 | nv_wr32(priv, 0x002270, cur->addr >> 12); |
124 | nv_wr32(priv, 0x002274, (engine << 20) | (p >> 3)); | |
87032e11 | 125 | |
5c0633e6 BS |
126 | if (wait_event_timeout(engn->wait, !(nv_rd32(priv, 0x002284 + |
127 | (engine * 0x08)) & 0x00100000), | |
128 | msecs_to_jiffies(2000)) == 0) | |
f82c44a7 | 129 | nv_error(priv, "runlist %d update timeout\n", engine); |
c2e3259b | 130 | mutex_unlock(&nv_subdev(priv)->mutex); |
5132f377 BS |
131 | } |
132 | ||
c420b2dc | 133 | static int |
ebb945a9 BS |
134 | nve0_fifo_context_attach(struct nouveau_object *parent, |
135 | struct nouveau_object *object) | |
5132f377 | 136 | { |
ebb945a9 BS |
137 | struct nouveau_bar *bar = nouveau_bar(parent); |
138 | struct nve0_fifo_base *base = (void *)parent->parent; | |
139 | struct nouveau_engctx *ectx = (void *)object; | |
140 | u32 addr; | |
141 | int ret; | |
142 | ||
143 | switch (nv_engidx(object->engine)) { | |
01672ef4 | 144 | case NVDEV_ENGINE_SW : |
448a4532 | 145 | return 0; |
dbff2dee | 146 | case NVDEV_ENGINE_COPY0: |
01672ef4 BS |
147 | case NVDEV_ENGINE_COPY1: |
148 | case NVDEV_ENGINE_COPY2: | |
448a4532 | 149 | nv_engctx(ectx)->addr = nv_gpuobj(base)->addr >> 12; |
01672ef4 BS |
150 | return 0; |
151 | case NVDEV_ENGINE_GR : addr = 0x0210; break; | |
b2f04fc6 | 152 | case NVDEV_ENGINE_BSP : addr = 0x0270; break; |
a7416d0d | 153 | case NVDEV_ENGINE_VP : addr = 0x0250; break; |
f3295b3c | 154 | case NVDEV_ENGINE_PPP : addr = 0x0260; break; |
ebb945a9 BS |
155 | default: |
156 | return -EINVAL; | |
5132f377 BS |
157 | } |
158 | ||
ebb945a9 BS |
159 | if (!ectx->vma.node) { |
160 | ret = nouveau_gpuobj_map_vm(nv_gpuobj(ectx), base->vm, | |
161 | NV_MEM_ACCESS_RW, &ectx->vma); | |
162 | if (ret) | |
163 | return ret; | |
4c2d4222 BS |
164 | |
165 | nv_engctx(ectx)->addr = nv_gpuobj(base)->addr >> 12; | |
ebb945a9 BS |
166 | } |
167 | ||
168 | nv_wo32(base, addr + 0x00, lower_32_bits(ectx->vma.offset) | 4); | |
169 | nv_wo32(base, addr + 0x04, upper_32_bits(ectx->vma.offset)); | |
170 | bar->flush(bar); | |
171 | return 0; | |
5132f377 BS |
172 | } |
173 | ||
ebb945a9 BS |
174 | static int |
175 | nve0_fifo_context_detach(struct nouveau_object *parent, bool suspend, | |
176 | struct nouveau_object *object) | |
5132f377 | 177 | { |
ebb945a9 BS |
178 | struct nouveau_bar *bar = nouveau_bar(parent); |
179 | struct nve0_fifo_priv *priv = (void *)parent->engine; | |
180 | struct nve0_fifo_base *base = (void *)parent->parent; | |
181 | struct nve0_fifo_chan *chan = (void *)parent; | |
182 | u32 addr; | |
183 | ||
184 | switch (nv_engidx(object->engine)) { | |
185 | case NVDEV_ENGINE_SW : return 0; | |
dbff2dee | 186 | case NVDEV_ENGINE_COPY0: |
01672ef4 BS |
187 | case NVDEV_ENGINE_COPY1: |
188 | case NVDEV_ENGINE_COPY2: addr = 0x0000; break; | |
189 | case NVDEV_ENGINE_GR : addr = 0x0210; break; | |
b2f04fc6 | 190 | case NVDEV_ENGINE_BSP : addr = 0x0270; break; |
a7416d0d | 191 | case NVDEV_ENGINE_VP : addr = 0x0250; break; |
f3295b3c | 192 | case NVDEV_ENGINE_PPP : addr = 0x0260; break; |
ebb945a9 BS |
193 | default: |
194 | return -EINVAL; | |
195 | } | |
196 | ||
ebb945a9 BS |
197 | nv_wr32(priv, 0x002634, chan->base.chid); |
198 | if (!nv_wait(priv, 0x002634, 0xffffffff, chan->base.chid)) { | |
93260d3c MS |
199 | nv_error(priv, "channel %d [%s] kick timeout\n", |
200 | chan->base.chid, nouveau_client_name(chan)); | |
ebb945a9 BS |
201 | if (suspend) |
202 | return -EBUSY; | |
5132f377 BS |
203 | } |
204 | ||
01672ef4 BS |
205 | if (addr) { |
206 | nv_wo32(base, addr + 0x00, 0x00000000); | |
207 | nv_wo32(base, addr + 0x04, 0x00000000); | |
208 | bar->flush(bar); | |
209 | } | |
210 | ||
ebb945a9 | 211 | return 0; |
5132f377 BS |
212 | } |
213 | ||
214 | static int | |
ebb945a9 BS |
215 | nve0_fifo_chan_ctor(struct nouveau_object *parent, |
216 | struct nouveau_object *engine, | |
217 | struct nouveau_oclass *oclass, void *data, u32 size, | |
218 | struct nouveau_object **pobject) | |
5132f377 | 219 | { |
bbf8906b BS |
220 | union { |
221 | struct kepler_channel_gpfifo_a_v0 v0; | |
222 | } *args = data; | |
ebb945a9 BS |
223 | struct nouveau_bar *bar = nouveau_bar(parent); |
224 | struct nve0_fifo_priv *priv = (void *)engine; | |
225 | struct nve0_fifo_base *base = (void *)parent; | |
226 | struct nve0_fifo_chan *chan; | |
ebb945a9 BS |
227 | u64 usermem, ioffset, ilength; |
228 | int ret, i; | |
229 | ||
bbf8906b BS |
230 | nv_ioctl(parent, "create channel gpfifo size %d\n", size); |
231 | if (nvif_unpack(args->v0, 0, 0, false)) { | |
232 | nv_ioctl(parent, "create channel gpfifo vers %d pushbuf %08x " | |
233 | "ioffset %016llx ilength %08x engine %08x\n", | |
234 | args->v0.version, args->v0.pushbuf, args->v0.ioffset, | |
235 | args->v0.ilength, args->v0.engine); | |
236 | } else | |
237 | return ret; | |
ebb945a9 | 238 | |
dbff2dee | 239 | for (i = 0; i < FIFO_ENGINE_NR; i++) { |
bbf8906b | 240 | if (args->v0.engine & (1 << i)) { |
dbff2dee | 241 | if (nouveau_engine(parent, fifo_engine[i].subdev)) { |
bbf8906b | 242 | args->v0.engine = (1 << i); |
dbff2dee BS |
243 | break; |
244 | } | |
245 | } | |
246 | } | |
247 | ||
56fbd2b6 | 248 | if (i == FIFO_ENGINE_NR) { |
bbf8906b | 249 | nv_error(priv, "unsupported engines 0x%08x\n", args->v0.engine); |
dbff2dee | 250 | return -ENODEV; |
56fbd2b6 | 251 | } |
dbff2dee | 252 | |
ebb945a9 BS |
253 | ret = nouveau_fifo_channel_create(parent, engine, oclass, 1, |
254 | priv->user.bar.offset, 0x200, | |
bbf8906b | 255 | args->v0.pushbuf, |
dbff2dee | 256 | fifo_engine[i].mask, &chan); |
ebb945a9 BS |
257 | *pobject = nv_object(chan); |
258 | if (ret) | |
259 | return ret; | |
260 | ||
bbf8906b BS |
261 | args->v0.chid = chan->base.chid; |
262 | ||
ebb945a9 BS |
263 | nv_parent(chan)->context_attach = nve0_fifo_context_attach; |
264 | nv_parent(chan)->context_detach = nve0_fifo_context_detach; | |
dbff2dee | 265 | chan->engine = i; |
ebb945a9 BS |
266 | |
267 | usermem = chan->base.chid * 0x200; | |
bbf8906b BS |
268 | ioffset = args->v0.ioffset; |
269 | ilength = order_base_2(args->v0.ilength / 8); | |
ebb945a9 BS |
270 | |
271 | for (i = 0; i < 0x200; i += 4) | |
272 | nv_wo32(priv->user.mem, usermem + i, 0x00000000); | |
273 | ||
274 | nv_wo32(base, 0x08, lower_32_bits(priv->user.mem->addr + usermem)); | |
275 | nv_wo32(base, 0x0c, upper_32_bits(priv->user.mem->addr + usermem)); | |
276 | nv_wo32(base, 0x10, 0x0000face); | |
277 | nv_wo32(base, 0x30, 0xfffff902); | |
278 | nv_wo32(base, 0x48, lower_32_bits(ioffset)); | |
279 | nv_wo32(base, 0x4c, upper_32_bits(ioffset) | (ilength << 16)); | |
280 | nv_wo32(base, 0x84, 0x20400000); | |
281 | nv_wo32(base, 0x94, 0x30000001); | |
282 | nv_wo32(base, 0x9c, 0x00000100); | |
283 | nv_wo32(base, 0xac, 0x0000001f); | |
284 | nv_wo32(base, 0xe8, chan->base.chid); | |
285 | nv_wo32(base, 0xb8, 0xf8000000); | |
286 | nv_wo32(base, 0xf8, 0x10003080); /* 0x002310 */ | |
287 | nv_wo32(base, 0xfc, 0x10000010); /* 0x002350 */ | |
288 | bar->flush(bar); | |
289 | return 0; | |
290 | } | |
5132f377 | 291 | |
ebb945a9 BS |
292 | static int |
293 | nve0_fifo_chan_init(struct nouveau_object *object) | |
294 | { | |
295 | struct nouveau_gpuobj *base = nv_gpuobj(object->parent); | |
296 | struct nve0_fifo_priv *priv = (void *)object->engine; | |
297 | struct nve0_fifo_chan *chan = (void *)object; | |
298 | u32 chid = chan->base.chid; | |
299 | int ret; | |
5132f377 | 300 | |
ebb945a9 BS |
301 | ret = nouveau_fifo_channel_init(&chan->base); |
302 | if (ret) | |
303 | return ret; | |
5132f377 | 304 | |
dbff2dee | 305 | nv_mask(priv, 0x800004 + (chid * 8), 0x000f0000, chan->engine << 16); |
ebb945a9 | 306 | nv_wr32(priv, 0x800000 + (chid * 8), 0x80000000 | base->addr >> 12); |
87032e11 BS |
307 | |
308 | if (chan->state == STOPPED && (chan->state = RUNNING) == RUNNING) { | |
309 | nv_mask(priv, 0x800004 + (chid * 8), 0x00000400, 0x00000400); | |
310 | nve0_fifo_runlist_update(priv, chan->engine); | |
311 | nv_mask(priv, 0x800004 + (chid * 8), 0x00000400, 0x00000400); | |
312 | } | |
313 | ||
ebb945a9 BS |
314 | return 0; |
315 | } | |
5132f377 | 316 | |
ebb945a9 BS |
317 | static int |
318 | nve0_fifo_chan_fini(struct nouveau_object *object, bool suspend) | |
319 | { | |
320 | struct nve0_fifo_priv *priv = (void *)object->engine; | |
321 | struct nve0_fifo_chan *chan = (void *)object; | |
322 | u32 chid = chan->base.chid; | |
5132f377 | 323 | |
87032e11 BS |
324 | if (chan->state == RUNNING && (chan->state = STOPPED) == STOPPED) { |
325 | nv_mask(priv, 0x800004 + (chid * 8), 0x00000800, 0x00000800); | |
326 | nve0_fifo_runlist_update(priv, chan->engine); | |
327 | } | |
5132f377 | 328 | |
87032e11 | 329 | nv_wr32(priv, 0x800000 + (chid * 8), 0x00000000); |
ebb945a9 BS |
330 | return nouveau_fifo_channel_fini(&chan->base, suspend); |
331 | } | |
5132f377 | 332 | |
ebb945a9 BS |
333 | static struct nouveau_ofuncs |
334 | nve0_fifo_ofuncs = { | |
335 | .ctor = nve0_fifo_chan_ctor, | |
336 | .dtor = _nouveau_fifo_channel_dtor, | |
337 | .init = nve0_fifo_chan_init, | |
338 | .fini = nve0_fifo_chan_fini, | |
339 | .rd32 = _nouveau_fifo_channel_rd32, | |
340 | .wr32 = _nouveau_fifo_channel_wr32, | |
341 | }; | |
5132f377 | 342 | |
ebb945a9 BS |
343 | static struct nouveau_oclass |
344 | nve0_fifo_sclass[] = { | |
bbf8906b | 345 | { KEPLER_CHANNEL_GPFIFO_A, &nve0_fifo_ofuncs }, |
ebb945a9 BS |
346 | {} |
347 | }; | |
348 | ||
349 | /******************************************************************************* | |
350 | * FIFO context - instmem heap and vm setup | |
351 | ******************************************************************************/ | |
5132f377 | 352 | |
c420b2dc | 353 | static int |
ebb945a9 BS |
354 | nve0_fifo_context_ctor(struct nouveau_object *parent, |
355 | struct nouveau_object *engine, | |
356 | struct nouveau_oclass *oclass, void *data, u32 size, | |
357 | struct nouveau_object **pobject) | |
c420b2dc | 358 | { |
ebb945a9 BS |
359 | struct nve0_fifo_base *base; |
360 | int ret; | |
c420b2dc | 361 | |
ebb945a9 BS |
362 | ret = nouveau_fifo_context_create(parent, engine, oclass, NULL, 0x1000, |
363 | 0x1000, NVOBJ_FLAG_ZERO_ALLOC, &base); | |
364 | *pobject = nv_object(base); | |
365 | if (ret) | |
366 | return ret; | |
c420b2dc | 367 | |
f50c8054 BS |
368 | ret = nouveau_gpuobj_new(nv_object(base), NULL, 0x10000, 0x1000, 0, |
369 | &base->pgd); | |
ebb945a9 BS |
370 | if (ret) |
371 | return ret; | |
372 | ||
373 | nv_wo32(base, 0x0200, lower_32_bits(base->pgd->addr)); | |
374 | nv_wo32(base, 0x0204, upper_32_bits(base->pgd->addr)); | |
375 | nv_wo32(base, 0x0208, 0xffffffff); | |
376 | nv_wo32(base, 0x020c, 0x000000ff); | |
377 | ||
378 | ret = nouveau_vm_ref(nouveau_client(parent)->vm, &base->vm, base->pgd); | |
379 | if (ret) | |
380 | return ret; | |
c420b2dc | 381 | |
c420b2dc BS |
382 | return 0; |
383 | } | |
384 | ||
ebb945a9 BS |
385 | static void |
386 | nve0_fifo_context_dtor(struct nouveau_object *object) | |
387 | { | |
388 | struct nve0_fifo_base *base = (void *)object; | |
389 | nouveau_vm_ref(NULL, &base->vm, base->pgd); | |
390 | nouveau_gpuobj_ref(NULL, &base->pgd); | |
391 | nouveau_fifo_context_destroy(&base->base); | |
392 | } | |
393 | ||
394 | static struct nouveau_oclass | |
395 | nve0_fifo_cclass = { | |
396 | .handle = NV_ENGCTX(FIFO, 0xe0), | |
397 | .ofuncs = &(struct nouveau_ofuncs) { | |
398 | .ctor = nve0_fifo_context_ctor, | |
399 | .dtor = nve0_fifo_context_dtor, | |
400 | .init = _nouveau_fifo_context_init, | |
401 | .fini = _nouveau_fifo_context_fini, | |
402 | .rd32 = _nouveau_fifo_context_rd32, | |
403 | .wr32 = _nouveau_fifo_context_wr32, | |
404 | }, | |
405 | }; | |
406 | ||
407 | /******************************************************************************* | |
408 | * PFIFO engine | |
409 | ******************************************************************************/ | |
410 | ||
98d1e317 BS |
411 | static inline int |
412 | nve0_fifo_engidx(struct nve0_fifo_priv *priv, u32 engn) | |
413 | { | |
414 | switch (engn) { | |
415 | case NVDEV_ENGINE_GR : | |
416 | case NVDEV_ENGINE_COPY2: engn = 0; break; | |
417 | case NVDEV_ENGINE_BSP : engn = 1; break; | |
418 | case NVDEV_ENGINE_PPP : engn = 2; break; | |
419 | case NVDEV_ENGINE_VP : engn = 3; break; | |
420 | case NVDEV_ENGINE_COPY0: engn = 4; break; | |
421 | case NVDEV_ENGINE_COPY1: engn = 5; break; | |
422 | case NVDEV_ENGINE_VENC : engn = 6; break; | |
423 | default: | |
424 | return -1; | |
425 | } | |
426 | ||
427 | return engn; | |
428 | } | |
429 | ||
129dcca7 BS |
430 | static inline struct nouveau_engine * |
431 | nve0_fifo_engine(struct nve0_fifo_priv *priv, u32 engn) | |
432 | { | |
433 | if (engn >= ARRAY_SIZE(fifo_engine)) | |
434 | return NULL; | |
435 | return nouveau_engine(priv, fifo_engine[engn].subdev); | |
436 | } | |
437 | ||
98d1e317 BS |
438 | static void |
439 | nve0_fifo_recover_work(struct work_struct *work) | |
440 | { | |
441 | struct nve0_fifo_priv *priv = container_of(work, typeof(*priv), fault); | |
442 | struct nouveau_object *engine; | |
443 | unsigned long flags; | |
444 | u32 engn, engm = 0; | |
445 | u64 mask, todo; | |
446 | ||
447 | spin_lock_irqsave(&priv->base.lock, flags); | |
448 | mask = priv->mask; | |
449 | priv->mask = 0ULL; | |
450 | spin_unlock_irqrestore(&priv->base.lock, flags); | |
451 | ||
452 | for (todo = mask; engn = __ffs64(todo), todo; todo &= ~(1 << engn)) | |
453 | engm |= 1 << nve0_fifo_engidx(priv, engn); | |
454 | nv_mask(priv, 0x002630, engm, engm); | |
455 | ||
456 | for (todo = mask; engn = __ffs64(todo), todo; todo &= ~(1 << engn)) { | |
457 | if ((engine = (void *)nouveau_engine(priv, engn))) { | |
458 | nv_ofuncs(engine)->fini(engine, false); | |
459 | WARN_ON(nv_ofuncs(engine)->init(engine)); | |
460 | } | |
461 | nve0_fifo_runlist_update(priv, nve0_fifo_engidx(priv, engn)); | |
462 | } | |
463 | ||
464 | nv_wr32(priv, 0x00262c, engm); | |
465 | nv_mask(priv, 0x002630, engm, 0x00000000); | |
466 | } | |
467 | ||
468 | static void | |
469 | nve0_fifo_recover(struct nve0_fifo_priv *priv, struct nouveau_engine *engine, | |
470 | struct nve0_fifo_chan *chan) | |
471 | { | |
472 | struct nouveau_object *engobj = nv_object(engine); | |
473 | u32 chid = chan->base.chid; | |
474 | unsigned long flags; | |
475 | ||
476 | nv_error(priv, "%s engine fault on channel %d, recovering...\n", | |
477 | nv_subdev(engine)->name, chid); | |
478 | ||
479 | nv_mask(priv, 0x800004 + (chid * 0x08), 0x00000800, 0x00000800); | |
480 | chan->state = KILLED; | |
481 | ||
482 | spin_lock_irqsave(&priv->base.lock, flags); | |
483 | priv->mask |= 1ULL << nv_engidx(engobj); | |
484 | spin_unlock_irqrestore(&priv->base.lock, flags); | |
485 | schedule_work(&priv->fault); | |
486 | } | |
487 | ||
3d61b967 BS |
488 | static int |
489 | nve0_fifo_swmthd(struct nve0_fifo_priv *priv, u32 chid, u32 mthd, u32 data) | |
490 | { | |
491 | struct nve0_fifo_chan *chan = NULL; | |
492 | struct nouveau_handle *bind; | |
493 | unsigned long flags; | |
494 | int ret = -EINVAL; | |
495 | ||
496 | spin_lock_irqsave(&priv->base.lock, flags); | |
497 | if (likely(chid >= priv->base.min && chid <= priv->base.max)) | |
498 | chan = (void *)priv->base.channel[chid]; | |
499 | if (unlikely(!chan)) | |
500 | goto out; | |
501 | ||
502 | bind = nouveau_namedb_get_class(nv_namedb(chan), 0x906e); | |
503 | if (likely(bind)) { | |
504 | if (!mthd || !nv_call(bind->object, mthd, data)) | |
505 | ret = 0; | |
506 | nouveau_namedb_put(bind); | |
507 | } | |
508 | ||
509 | out: | |
510 | spin_unlock_irqrestore(&priv->base.lock, flags); | |
511 | return ret; | |
512 | } | |
513 | ||
56b2f68c BS |
514 | static const struct nouveau_enum |
515 | nve0_fifo_bind_reason[] = { | |
516 | { 0x01, "BIND_NOT_UNBOUND" }, | |
517 | { 0x02, "SNOOP_WITHOUT_BAR1" }, | |
518 | { 0x03, "UNBIND_WHILE_RUNNING" }, | |
519 | { 0x05, "INVALID_RUNLIST" }, | |
520 | { 0x06, "INVALID_CTX_TGT" }, | |
521 | { 0x0b, "UNBIND_WHILE_PARKED" }, | |
522 | {} | |
523 | }; | |
524 | ||
525 | static void | |
526 | nve0_fifo_intr_bind(struct nve0_fifo_priv *priv) | |
527 | { | |
528 | u32 intr = nv_rd32(priv, 0x00252c); | |
529 | u32 code = intr & 0x000000ff; | |
530 | const struct nouveau_enum *en; | |
531 | char enunk[6] = ""; | |
532 | ||
533 | en = nouveau_enum_find(nve0_fifo_bind_reason, code); | |
534 | if (!en) | |
535 | snprintf(enunk, sizeof(enunk), "UNK%02x", code); | |
536 | ||
537 | nv_error(priv, "BIND_ERROR [ %s ]\n", en ? en->name : enunk); | |
538 | } | |
539 | ||
0a7760e0 BS |
540 | static const struct nouveau_enum |
541 | nve0_fifo_sched_reason[] = { | |
e9fb9805 BS |
542 | { 0x0a, "CTXSW_TIMEOUT" }, |
543 | {} | |
544 | }; | |
545 | ||
129dcca7 BS |
546 | static void |
547 | nve0_fifo_intr_sched_ctxsw(struct nve0_fifo_priv *priv) | |
548 | { | |
549 | struct nouveau_engine *engine; | |
550 | struct nve0_fifo_chan *chan; | |
551 | u32 engn; | |
552 | ||
553 | for (engn = 0; engn < ARRAY_SIZE(fifo_engine); engn++) { | |
554 | u32 stat = nv_rd32(priv, 0x002640 + (engn * 0x04)); | |
555 | u32 busy = (stat & 0x80000000); | |
556 | u32 next = (stat & 0x07ff0000) >> 16; | |
557 | u32 chsw = (stat & 0x00008000); | |
558 | u32 save = (stat & 0x00004000); | |
559 | u32 load = (stat & 0x00002000); | |
560 | u32 prev = (stat & 0x000007ff); | |
561 | u32 chid = load ? next : prev; | |
562 | (void)save; | |
563 | ||
564 | if (busy && chsw) { | |
565 | if (!(chan = (void *)priv->base.channel[chid])) | |
566 | continue; | |
567 | if (!(engine = nve0_fifo_engine(priv, engn))) | |
568 | continue; | |
569 | nve0_fifo_recover(priv, engine, chan); | |
570 | } | |
571 | } | |
572 | } | |
573 | ||
885f3ced BS |
574 | static void |
575 | nve0_fifo_intr_sched(struct nve0_fifo_priv *priv) | |
576 | { | |
577 | u32 intr = nv_rd32(priv, 0x00254c); | |
578 | u32 code = intr & 0x000000ff; | |
0a7760e0 BS |
579 | const struct nouveau_enum *en; |
580 | char enunk[6] = ""; | |
581 | ||
582 | en = nouveau_enum_find(nve0_fifo_sched_reason, code); | |
583 | if (!en) | |
584 | snprintf(enunk, sizeof(enunk), "UNK%02x", code); | |
585 | ||
586 | nv_error(priv, "SCHED_ERROR [ %s ]\n", en ? en->name : enunk); | |
129dcca7 BS |
587 | |
588 | switch (code) { | |
589 | case 0x0a: | |
590 | nve0_fifo_intr_sched_ctxsw(priv); | |
591 | break; | |
592 | default: | |
593 | break; | |
594 | } | |
885f3ced BS |
595 | } |
596 | ||
597 | static void | |
598 | nve0_fifo_intr_chsw(struct nve0_fifo_priv *priv) | |
599 | { | |
600 | u32 stat = nv_rd32(priv, 0x00256c); | |
601 | nv_error(priv, "CHSW_ERROR 0x%08x\n", stat); | |
602 | nv_wr32(priv, 0x00256c, stat); | |
603 | } | |
604 | ||
605 | static void | |
606 | nve0_fifo_intr_dropped_fault(struct nve0_fifo_priv *priv) | |
607 | { | |
608 | u32 stat = nv_rd32(priv, 0x00259c); | |
609 | nv_error(priv, "DROPPED_MMU_FAULT 0x%08x\n", stat); | |
610 | } | |
611 | ||
612 | static const struct nouveau_enum | |
613 | nve0_fifo_fault_engine[] = { | |
e1b6b14a | 614 | { 0x00, "GR", NULL, NVDEV_ENGINE_GR }, |
885f3ced | 615 | { 0x03, "IFB", NULL, NVDEV_ENGINE_IFB }, |
cb1567c2 BS |
616 | { 0x04, "BAR1", NULL, NVDEV_SUBDEV_BAR }, |
617 | { 0x05, "BAR3", NULL, NVDEV_SUBDEV_INSTMEM }, | |
e1b6b14a BS |
618 | { 0x07, "PBDMA0", NULL, NVDEV_ENGINE_FIFO }, |
619 | { 0x08, "PBDMA1", NULL, NVDEV_ENGINE_FIFO }, | |
620 | { 0x09, "PBDMA2", NULL, NVDEV_ENGINE_FIFO }, | |
621 | { 0x10, "MSVLD", NULL, NVDEV_ENGINE_BSP }, | |
622 | { 0x11, "MSPPP", NULL, NVDEV_ENGINE_PPP }, | |
623 | { 0x13, "PERF" }, | |
624 | { 0x14, "MSPDEC", NULL, NVDEV_ENGINE_VP }, | |
625 | { 0x15, "CE0", NULL, NVDEV_ENGINE_COPY0 }, | |
626 | { 0x16, "CE1", NULL, NVDEV_ENGINE_COPY1 }, | |
627 | { 0x17, "PMU" }, | |
628 | { 0x19, "MSENC", NULL, NVDEV_ENGINE_VENC }, | |
629 | { 0x1b, "CE2", NULL, NVDEV_ENGINE_COPY2 }, | |
5132f377 BS |
630 | {} |
631 | }; | |
632 | ||
885f3ced BS |
633 | static const struct nouveau_enum |
634 | nve0_fifo_fault_reason[] = { | |
e1b6b14a BS |
635 | { 0x00, "PDE" }, |
636 | { 0x01, "PDE_SIZE" }, | |
637 | { 0x02, "PTE" }, | |
638 | { 0x03, "VA_LIMIT_VIOLATION" }, | |
639 | { 0x04, "UNBOUND_INST_BLOCK" }, | |
640 | { 0x05, "PRIV_VIOLATION" }, | |
641 | { 0x06, "RO_VIOLATION" }, | |
642 | { 0x07, "WO_VIOLATION" }, | |
643 | { 0x08, "PITCH_MASK_VIOLATION" }, | |
644 | { 0x09, "WORK_CREATION" }, | |
645 | { 0x0a, "UNSUPPORTED_APERTURE" }, | |
646 | { 0x0b, "COMPRESSION_FAILURE" }, | |
647 | { 0x0c, "UNSUPPORTED_KIND" }, | |
648 | { 0x0d, "REGION_VIOLATION" }, | |
649 | { 0x0e, "BOTH_PTES_VALID" }, | |
650 | { 0x0f, "INFO_TYPE_POISONED" }, | |
5132f377 BS |
651 | {} |
652 | }; | |
653 | ||
885f3ced BS |
654 | static const struct nouveau_enum |
655 | nve0_fifo_fault_hubclient[] = { | |
e1b6b14a BS |
656 | { 0x00, "VIP" }, |
657 | { 0x01, "CE0" }, | |
658 | { 0x02, "CE1" }, | |
659 | { 0x03, "DNISO" }, | |
660 | { 0x04, "FE" }, | |
661 | { 0x05, "FECS" }, | |
662 | { 0x06, "HOST" }, | |
663 | { 0x07, "HOST_CPU" }, | |
664 | { 0x08, "HOST_CPU_NB" }, | |
665 | { 0x09, "ISO" }, | |
666 | { 0x0a, "MMU" }, | |
667 | { 0x0b, "MSPDEC" }, | |
668 | { 0x0c, "MSPPP" }, | |
669 | { 0x0d, "MSVLD" }, | |
670 | { 0x0e, "NISO" }, | |
671 | { 0x0f, "P2P" }, | |
672 | { 0x10, "PD" }, | |
673 | { 0x11, "PERF" }, | |
674 | { 0x12, "PMU" }, | |
675 | { 0x13, "RASTERTWOD" }, | |
676 | { 0x14, "SCC" }, | |
677 | { 0x15, "SCC_NB" }, | |
678 | { 0x16, "SEC" }, | |
679 | { 0x17, "SSYNC" }, | |
680 | { 0x18, "GR_COPY" }, | |
681 | { 0x19, "CE2" }, | |
682 | { 0x1a, "XV" }, | |
683 | { 0x1b, "MMU_NB" }, | |
684 | { 0x1c, "MSENC" }, | |
685 | { 0x1d, "DFALCON" }, | |
686 | { 0x1e, "SKED" }, | |
687 | { 0x1f, "AFALCON" }, | |
5132f377 BS |
688 | {} |
689 | }; | |
690 | ||
885f3ced BS |
691 | static const struct nouveau_enum |
692 | nve0_fifo_fault_gpcclient[] = { | |
e1b6b14a BS |
693 | { 0x00, "L1_0" }, { 0x01, "T1_0" }, { 0x02, "PE_0" }, |
694 | { 0x03, "L1_1" }, { 0x04, "T1_1" }, { 0x05, "PE_1" }, | |
695 | { 0x06, "L1_2" }, { 0x07, "T1_2" }, { 0x08, "PE_2" }, | |
696 | { 0x09, "L1_3" }, { 0x0a, "T1_3" }, { 0x0b, "PE_3" }, | |
697 | { 0x0c, "RAST" }, | |
698 | { 0x0d, "GCC" }, | |
699 | { 0x0e, "GPCCS" }, | |
700 | { 0x0f, "PROP_0" }, | |
701 | { 0x10, "PROP_1" }, | |
702 | { 0x11, "PROP_2" }, | |
703 | { 0x12, "PROP_3" }, | |
704 | { 0x13, "L1_4" }, { 0x14, "T1_4" }, { 0x15, "PE_4" }, | |
705 | { 0x16, "L1_5" }, { 0x17, "T1_5" }, { 0x18, "PE_5" }, | |
706 | { 0x19, "L1_6" }, { 0x1a, "T1_6" }, { 0x1b, "PE_6" }, | |
707 | { 0x1c, "L1_7" }, { 0x1d, "T1_7" }, { 0x1e, "PE_7" }, | |
708 | { 0x1f, "GPM" }, | |
709 | { 0x20, "LTP_UTLB_0" }, | |
710 | { 0x21, "LTP_UTLB_1" }, | |
711 | { 0x22, "LTP_UTLB_2" }, | |
712 | { 0x23, "LTP_UTLB_3" }, | |
713 | { 0x24, "GPC_RGG_UTLB" }, | |
5132f377 BS |
714 | {} |
715 | }; | |
716 | ||
e9fb9805 BS |
717 | static void |
718 | nve0_fifo_intr_fault(struct nve0_fifo_priv *priv, int unit) | |
5132f377 | 719 | { |
885f3ced BS |
720 | u32 inst = nv_rd32(priv, 0x002800 + (unit * 0x10)); |
721 | u32 valo = nv_rd32(priv, 0x002804 + (unit * 0x10)); | |
722 | u32 vahi = nv_rd32(priv, 0x002808 + (unit * 0x10)); | |
723 | u32 stat = nv_rd32(priv, 0x00280c + (unit * 0x10)); | |
724 | u32 gpc = (stat & 0x1f000000) >> 24; | |
5132f377 | 725 | u32 client = (stat & 0x00001f00) >> 8; |
885f3ced BS |
726 | u32 write = (stat & 0x00000080); |
727 | u32 hub = (stat & 0x00000040); | |
728 | u32 reason = (stat & 0x0000000f); | |
98d1e317 BS |
729 | struct nouveau_object *engctx = NULL, *object; |
730 | struct nouveau_engine *engine = NULL; | |
885f3ced BS |
731 | const struct nouveau_enum *er, *eu, *ec; |
732 | char erunk[6] = ""; | |
733 | char euunk[6] = ""; | |
734 | char ecunk[6] = ""; | |
735 | char gpcid[3] = ""; | |
736 | ||
737 | er = nouveau_enum_find(nve0_fifo_fault_reason, reason); | |
738 | if (!er) | |
739 | snprintf(erunk, sizeof(erunk), "UNK%02X", reason); | |
740 | ||
741 | eu = nouveau_enum_find(nve0_fifo_fault_engine, unit); | |
742 | if (eu) { | |
743 | switch (eu->data2) { | |
744 | case NVDEV_SUBDEV_BAR: | |
cb1567c2 | 745 | nv_mask(priv, 0x001704, 0x00000000, 0x00000000); |
885f3ced BS |
746 | break; |
747 | case NVDEV_SUBDEV_INSTMEM: | |
cb1567c2 | 748 | nv_mask(priv, 0x001714, 0x00000000, 0x00000000); |
885f3ced BS |
749 | break; |
750 | case NVDEV_ENGINE_IFB: | |
751 | nv_mask(priv, 0x001718, 0x00000000, 0x00000000); | |
752 | break; | |
753 | default: | |
754 | engine = nouveau_engine(priv, eu->data2); | |
755 | if (engine) | |
cb1567c2 | 756 | engctx = nouveau_engctx_get(engine, inst); |
885f3ced | 757 | break; |
cb1567c2 | 758 | } |
885f3ced BS |
759 | } else { |
760 | snprintf(euunk, sizeof(euunk), "UNK%02x", unit); | |
93260d3c | 761 | } |
885f3ced BS |
762 | |
763 | if (hub) { | |
764 | ec = nouveau_enum_find(nve0_fifo_fault_hubclient, client); | |
765 | } else { | |
766 | ec = nouveau_enum_find(nve0_fifo_fault_gpcclient, client); | |
767 | snprintf(gpcid, sizeof(gpcid), "%d", gpc); | |
768 | } | |
769 | ||
770 | if (!ec) | |
771 | snprintf(ecunk, sizeof(ecunk), "UNK%02x", client); | |
772 | ||
773 | nv_error(priv, "%s fault at 0x%010llx [%s] from %s/%s%s%s%s on " | |
774 | "channel 0x%010llx [%s]\n", write ? "write" : "read", | |
775 | (u64)vahi << 32 | valo, er ? er->name : erunk, | |
776 | eu ? eu->name : euunk, hub ? "" : "GPC", gpcid, hub ? "" : "/", | |
777 | ec ? ec->name : ecunk, (u64)inst << 12, | |
778 | nouveau_client_name(engctx)); | |
93260d3c | 779 | |
98d1e317 BS |
780 | object = engctx; |
781 | while (object) { | |
782 | switch (nv_mclass(object)) { | |
bbf8906b | 783 | case KEPLER_CHANNEL_GPFIFO_A: |
98d1e317 BS |
784 | nve0_fifo_recover(priv, engine, (void *)object); |
785 | break; | |
786 | } | |
787 | object = object->parent; | |
788 | } | |
789 | ||
93260d3c | 790 | nouveau_engctx_put(engctx); |
5132f377 BS |
791 | } |
792 | ||
3d61b967 BS |
793 | static const struct nouveau_bitfield nve0_fifo_pbdma_intr[] = { |
794 | { 0x00000001, "MEMREQ" }, | |
795 | { 0x00000002, "MEMACK_TIMEOUT" }, | |
796 | { 0x00000004, "MEMACK_EXTRA" }, | |
797 | { 0x00000008, "MEMDAT_TIMEOUT" }, | |
798 | { 0x00000010, "MEMDAT_EXTRA" }, | |
799 | { 0x00000020, "MEMFLUSH" }, | |
800 | { 0x00000040, "MEMOP" }, | |
801 | { 0x00000080, "LBCONNECT" }, | |
802 | { 0x00000100, "LBREQ" }, | |
803 | { 0x00000200, "LBACK_TIMEOUT" }, | |
804 | { 0x00000400, "LBACK_EXTRA" }, | |
805 | { 0x00000800, "LBDAT_TIMEOUT" }, | |
806 | { 0x00001000, "LBDAT_EXTRA" }, | |
807 | { 0x00002000, "GPFIFO" }, | |
808 | { 0x00004000, "GPPTR" }, | |
809 | { 0x00008000, "GPENTRY" }, | |
810 | { 0x00010000, "GPCRC" }, | |
811 | { 0x00020000, "PBPTR" }, | |
812 | { 0x00040000, "PBENTRY" }, | |
813 | { 0x00080000, "PBCRC" }, | |
814 | { 0x00100000, "XBARCONNECT" }, | |
815 | { 0x00200000, "METHOD" }, | |
816 | { 0x00400000, "METHODCRC" }, | |
817 | { 0x00800000, "DEVICE" }, | |
818 | { 0x02000000, "SEMAPHORE" }, | |
819 | { 0x04000000, "ACQUIRE" }, | |
820 | { 0x08000000, "PRI" }, | |
821 | { 0x20000000, "NO_CTXSW_SEG" }, | |
822 | { 0x40000000, "PBSEG" }, | |
823 | { 0x80000000, "SIGNATURE" }, | |
824 | {} | |
825 | }; | |
e2b34fa0 | 826 | |
5132f377 | 827 | static void |
e9fb9805 | 828 | nve0_fifo_intr_pbdma(struct nve0_fifo_priv *priv, int unit) |
5132f377 | 829 | { |
ebb945a9 BS |
830 | u32 stat = nv_rd32(priv, 0x040108 + (unit * 0x2000)); |
831 | u32 addr = nv_rd32(priv, 0x0400c0 + (unit * 0x2000)); | |
832 | u32 data = nv_rd32(priv, 0x0400c4 + (unit * 0x2000)); | |
833 | u32 chid = nv_rd32(priv, 0x040120 + (unit * 0x2000)) & 0xfff; | |
834 | u32 subc = (addr & 0x00070000) >> 16; | |
5132f377 | 835 | u32 mthd = (addr & 0x00003ffc); |
e2b34fa0 BS |
836 | u32 show = stat; |
837 | ||
ebb945a9 BS |
838 | if (stat & 0x00800000) { |
839 | if (!nve0_fifo_swmthd(priv, chid, mthd, data)) | |
840 | show &= ~0x00800000; | |
841 | } | |
842 | ||
e2b34fa0 | 843 | if (show) { |
39b05542 BS |
844 | nv_error(priv, "PBDMA%d:", unit); |
845 | nouveau_bitfield_print(nve0_fifo_pbdma_intr, show); | |
f533da10 | 846 | pr_cont("\n"); |
93260d3c | 847 | nv_error(priv, |
39b05542 | 848 | "PBDMA%d: ch %d [%s] subc %d mthd 0x%04x data 0x%08x\n", |
93260d3c MS |
849 | unit, chid, |
850 | nouveau_client_name_for_fifo_chid(&priv->base, chid), | |
851 | subc, mthd, data); | |
e2b34fa0 | 852 | } |
5132f377 | 853 | |
ebb945a9 BS |
854 | nv_wr32(priv, 0x0400c0 + (unit * 0x2000), 0x80600008); |
855 | nv_wr32(priv, 0x040108 + (unit * 0x2000), stat); | |
5132f377 BS |
856 | } |
857 | ||
138b873f BS |
858 | static void |
859 | nve0_fifo_intr_runlist(struct nve0_fifo_priv *priv) | |
860 | { | |
861 | u32 mask = nv_rd32(priv, 0x002a00); | |
862 | while (mask) { | |
863 | u32 engn = __ffs(mask); | |
864 | wake_up(&priv->engine[engn].wait); | |
865 | nv_wr32(priv, 0x002a00, 1 << engn); | |
866 | mask &= ~(1 << engn); | |
867 | } | |
868 | } | |
869 | ||
c074bdbc BS |
870 | static void |
871 | nve0_fifo_intr_engine(struct nve0_fifo_priv *priv) | |
872 | { | |
79ca2770 | 873 | nvkm_event_send(&priv->base.uevent, 1, 0, NULL, 0); |
c074bdbc BS |
874 | } |
875 | ||
5132f377 | 876 | static void |
ebb945a9 | 877 | nve0_fifo_intr(struct nouveau_subdev *subdev) |
5132f377 | 878 | { |
ebb945a9 BS |
879 | struct nve0_fifo_priv *priv = (void *)subdev; |
880 | u32 mask = nv_rd32(priv, 0x002140); | |
881 | u32 stat = nv_rd32(priv, 0x002100) & mask; | |
5132f377 | 882 | |
e9fb9805 | 883 | if (stat & 0x00000001) { |
56b2f68c | 884 | nve0_fifo_intr_bind(priv); |
e9fb9805 BS |
885 | nv_wr32(priv, 0x002100, 0x00000001); |
886 | stat &= ~0x00000001; | |
887 | } | |
888 | ||
889 | if (stat & 0x00000010) { | |
890 | nv_error(priv, "PIO_ERROR\n"); | |
891 | nv_wr32(priv, 0x002100, 0x00000010); | |
892 | stat &= ~0x00000010; | |
893 | } | |
894 | ||
5132f377 | 895 | if (stat & 0x00000100) { |
e9fb9805 | 896 | nve0_fifo_intr_sched(priv); |
ebb945a9 | 897 | nv_wr32(priv, 0x002100, 0x00000100); |
5132f377 BS |
898 | stat &= ~0x00000100; |
899 | } | |
900 | ||
e9fb9805 BS |
901 | if (stat & 0x00010000) { |
902 | nve0_fifo_intr_chsw(priv); | |
903 | nv_wr32(priv, 0x002100, 0x00010000); | |
904 | stat &= ~0x00010000; | |
905 | } | |
906 | ||
907 | if (stat & 0x00800000) { | |
908 | nv_error(priv, "FB_FLUSH_TIMEOUT\n"); | |
909 | nv_wr32(priv, 0x002100, 0x00800000); | |
910 | stat &= ~0x00800000; | |
911 | } | |
912 | ||
913 | if (stat & 0x01000000) { | |
914 | nv_error(priv, "LB_ERROR\n"); | |
915 | nv_wr32(priv, 0x002100, 0x01000000); | |
916 | stat &= ~0x01000000; | |
917 | } | |
918 | ||
919 | if (stat & 0x08000000) { | |
920 | nve0_fifo_intr_dropped_fault(priv); | |
921 | nv_wr32(priv, 0x002100, 0x08000000); | |
922 | stat &= ~0x08000000; | |
923 | } | |
924 | ||
5132f377 | 925 | if (stat & 0x10000000) { |
885f3ced BS |
926 | u32 mask = nv_rd32(priv, 0x00259c); |
927 | while (mask) { | |
928 | u32 unit = __ffs(mask); | |
929 | nve0_fifo_intr_fault(priv, unit); | |
930 | nv_wr32(priv, 0x00259c, (1 << unit)); | |
931 | mask &= ~(1 << unit); | |
5132f377 | 932 | } |
5132f377 BS |
933 | stat &= ~0x10000000; |
934 | } | |
935 | ||
936 | if (stat & 0x20000000) { | |
39b05542 | 937 | u32 mask = nv_rd32(priv, 0x0025a0); |
3d61b967 BS |
938 | while (mask) { |
939 | u32 unit = __ffs(mask); | |
e9fb9805 | 940 | nve0_fifo_intr_pbdma(priv, unit); |
3d61b967 BS |
941 | nv_wr32(priv, 0x0025a0, (1 << unit)); |
942 | mask &= ~(1 << unit); | |
5132f377 | 943 | } |
5132f377 BS |
944 | stat &= ~0x20000000; |
945 | } | |
946 | ||
947 | if (stat & 0x40000000) { | |
138b873f | 948 | nve0_fifo_intr_runlist(priv); |
5132f377 BS |
949 | stat &= ~0x40000000; |
950 | } | |
951 | ||
9bd2ddba | 952 | if (stat & 0x80000000) { |
c074bdbc | 953 | nve0_fifo_intr_engine(priv); |
9bd2ddba BS |
954 | nv_wr32(priv, 0x002100, 0x80000000); |
955 | stat &= ~0x80000000; | |
956 | } | |
957 | ||
5132f377 | 958 | if (stat) { |
7a42f492 BS |
959 | nv_error(priv, "INTR 0x%08x\n", stat); |
960 | nv_mask(priv, 0x002140, stat, 0x00000000); | |
ebb945a9 | 961 | nv_wr32(priv, 0x002100, stat); |
5132f377 BS |
962 | } |
963 | } | |
c420b2dc | 964 | |
9bd2ddba | 965 | static void |
79ca2770 | 966 | nve0_fifo_uevent_init(struct nvkm_event *event, int type, int index) |
9bd2ddba | 967 | { |
79ca2770 BS |
968 | struct nouveau_fifo *fifo = container_of(event, typeof(*fifo), uevent); |
969 | nv_mask(fifo, 0x002140, 0x80000000, 0x80000000); | |
9bd2ddba BS |
970 | } |
971 | ||
972 | static void | |
79ca2770 | 973 | nve0_fifo_uevent_fini(struct nvkm_event *event, int type, int index) |
9bd2ddba | 974 | { |
79ca2770 BS |
975 | struct nouveau_fifo *fifo = container_of(event, typeof(*fifo), uevent); |
976 | nv_mask(fifo, 0x002140, 0x80000000, 0x00000000); | |
9bd2ddba BS |
977 | } |
978 | ||
79ca2770 BS |
979 | static int |
980 | nve0_fifo_uevent_ctor(void *data, u32 size, struct nvkm_notify *notify) | |
981 | { | |
982 | if (size == 0) { | |
983 | notify->size = 0; | |
984 | notify->types = 1; | |
985 | notify->index = 0; | |
986 | return 0; | |
987 | } | |
988 | return -ENOSYS; | |
989 | } | |
990 | ||
991 | static const struct nvkm_event_func | |
992 | nve0_fifo_uevent_func = { | |
993 | .ctor = nve0_fifo_uevent_ctor, | |
994 | .init = nve0_fifo_uevent_init, | |
995 | .fini = nve0_fifo_uevent_fini, | |
996 | }; | |
997 | ||
649ec925 BS |
998 | int |
999 | nve0_fifo_fini(struct nouveau_object *object, bool suspend) | |
1000 | { | |
1001 | struct nve0_fifo_priv *priv = (void *)object; | |
1002 | int ret; | |
1003 | ||
1004 | ret = nouveau_fifo_fini(&priv->base, suspend); | |
1005 | if (ret) | |
1006 | return ret; | |
1007 | ||
1008 | /* allow mmu fault interrupts, even when we're not using fifo */ | |
1009 | nv_mask(priv, 0x002140, 0x10000000, 0x10000000); | |
1010 | return 0; | |
1011 | } | |
1012 | ||
a763951a BS |
1013 | int |
1014 | nve0_fifo_init(struct nouveau_object *object) | |
1015 | { | |
1016 | struct nve0_fifo_priv *priv = (void *)object; | |
1017 | int ret, i; | |
1018 | ||
1019 | ret = nouveau_fifo_init(&priv->base); | |
1020 | if (ret) | |
1021 | return ret; | |
1022 | ||
39b05542 | 1023 | /* enable all available PBDMA units */ |
a763951a BS |
1024 | nv_wr32(priv, 0x000204, 0xffffffff); |
1025 | priv->spoon_nr = hweight32(nv_rd32(priv, 0x000204)); | |
39b05542 | 1026 | nv_debug(priv, "%d PBDMA unit(s)\n", priv->spoon_nr); |
a763951a | 1027 | |
39b05542 | 1028 | /* PBDMA[n] */ |
a763951a BS |
1029 | for (i = 0; i < priv->spoon_nr; i++) { |
1030 | nv_mask(priv, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000); | |
1031 | nv_wr32(priv, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */ | |
1032 | nv_wr32(priv, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTREN */ | |
1033 | } | |
1034 | ||
1035 | nv_wr32(priv, 0x002254, 0x10000000 | priv->user.bar.offset >> 12); | |
1036 | ||
a763951a | 1037 | nv_wr32(priv, 0x002100, 0xffffffff); |
138b873f | 1038 | nv_wr32(priv, 0x002140, 0x7fffffff); |
a763951a BS |
1039 | return 0; |
1040 | } | |
1041 | ||
1042 | void | |
1043 | nve0_fifo_dtor(struct nouveau_object *object) | |
1044 | { | |
1045 | struct nve0_fifo_priv *priv = (void *)object; | |
1046 | int i; | |
1047 | ||
1048 | nouveau_gpuobj_unmap(&priv->user.bar); | |
1049 | nouveau_gpuobj_ref(NULL, &priv->user.mem); | |
1050 | ||
1051 | for (i = 0; i < FIFO_ENGINE_NR; i++) { | |
f82c44a7 BS |
1052 | nouveau_gpuobj_ref(NULL, &priv->engine[i].runlist[1]); |
1053 | nouveau_gpuobj_ref(NULL, &priv->engine[i].runlist[0]); | |
a763951a BS |
1054 | } |
1055 | ||
1056 | nouveau_fifo_destroy(&priv->base); | |
1057 | } | |
1058 | ||
1059 | int | |
ebb945a9 BS |
1060 | nve0_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine, |
1061 | struct nouveau_oclass *oclass, void *data, u32 size, | |
1062 | struct nouveau_object **pobject) | |
1063 | { | |
a763951a | 1064 | struct nve0_fifo_impl *impl = (void *)oclass; |
ebb945a9 | 1065 | struct nve0_fifo_priv *priv; |
8d6f585d | 1066 | int ret, i; |
ebb945a9 | 1067 | |
a763951a BS |
1068 | ret = nouveau_fifo_create(parent, engine, oclass, 0, |
1069 | impl->channels - 1, &priv); | |
ebb945a9 BS |
1070 | *pobject = nv_object(priv); |
1071 | if (ret) | |
1072 | return ret; | |
1073 | ||
98d1e317 BS |
1074 | INIT_WORK(&priv->fault, nve0_fifo_recover_work); |
1075 | ||
8d6f585d BS |
1076 | for (i = 0; i < FIFO_ENGINE_NR; i++) { |
1077 | ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x8000, 0x1000, | |
f82c44a7 | 1078 | 0, &priv->engine[i].runlist[0]); |
8d6f585d BS |
1079 | if (ret) |
1080 | return ret; | |
1081 | ||
1082 | ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x8000, 0x1000, | |
f82c44a7 | 1083 | 0, &priv->engine[i].runlist[1]); |
8d6f585d BS |
1084 | if (ret) |
1085 | return ret; | |
138b873f BS |
1086 | |
1087 | init_waitqueue_head(&priv->engine[i].wait); | |
8d6f585d BS |
1088 | } |
1089 | ||
02f0b8c8 AC |
1090 | ret = nouveau_gpuobj_new(nv_object(priv), NULL, impl->channels * 0x200, |
1091 | 0x1000, NVOBJ_FLAG_ZERO_ALLOC, &priv->user.mem); | |
ebb945a9 BS |
1092 | if (ret) |
1093 | return ret; | |
1094 | ||
1095 | ret = nouveau_gpuobj_map(priv->user.mem, NV_MEM_ACCESS_RW, | |
1096 | &priv->user.bar); | |
1097 | if (ret) | |
1098 | return ret; | |
1099 | ||
79ca2770 BS |
1100 | ret = nvkm_event_init(&nve0_fifo_uevent_func, 1, 1, &priv->base.uevent); |
1101 | if (ret) | |
1102 | return ret; | |
9bd2ddba | 1103 | |
ebb945a9 BS |
1104 | nv_subdev(priv)->unit = 0x00000100; |
1105 | nv_subdev(priv)->intr = nve0_fifo_intr; | |
1106 | nv_engine(priv)->cclass = &nve0_fifo_cclass; | |
1107 | nv_engine(priv)->sclass = nve0_fifo_sclass; | |
1108 | return 0; | |
1109 | } | |
1110 | ||
16c4f227 | 1111 | struct nouveau_oclass * |
a763951a BS |
1112 | nve0_fifo_oclass = &(struct nve0_fifo_impl) { |
1113 | .base.handle = NV_ENGINE(FIFO, 0xe0), | |
1114 | .base.ofuncs = &(struct nouveau_ofuncs) { | |
ebb945a9 BS |
1115 | .ctor = nve0_fifo_ctor, |
1116 | .dtor = nve0_fifo_dtor, | |
1117 | .init = nve0_fifo_init, | |
649ec925 | 1118 | .fini = nve0_fifo_fini, |
ebb945a9 | 1119 | }, |
a763951a BS |
1120 | .channels = 4096, |
1121 | }.base; |