From: Ben Skeggs <bskeggs@redhat.com>
Date: Wed, 1 Jun 2022 10:47:23 +0000 (+1000)
Subject: drm/nouveau/fifo: add runq
X-Git-Tag: Ubuntu-6.2.0-16.16~1330^2~20^2~79
X-Git-Url: https://git.proxmox.com/?a=commitdiff_plain;h=1c488ba96e59ce84bf1cfd321bf27252f4598bec;p=mirror_ubuntu-kernels.git

drm/nouveau/fifo: add runq

Creates an nvkm_runq for each PBDMA, these will be associated with the
relevant runlist(s) later.

Signed-off-by: Ben Skeggs <bskeggs@redhat.com>
Reviewed-by: Lyude Paul <lyude@redhat.com>
---

diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h
index b688121f1a91..895eb262345e 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h
@@ -40,6 +40,8 @@ struct nvkm_fifo {
 	struct nvkm_chid *chid;
 	struct nvkm_chid *cgid;
 
+	struct list_head runqs;
+
 	DECLARE_BITMAP(mask, NVKM_FIFO_CHID_NR);
 	int nr;
 	struct list_head chan;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild
index c985736bff2c..447001db070e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild
@@ -2,6 +2,7 @@
 nvkm-y += nvkm/engine/fifo/base.o
 nvkm-y += nvkm/engine/fifo/chan.o
 nvkm-y += nvkm/engine/fifo/chid.o
+nvkm-y += nvkm/engine/fifo/runq.o
 
 nvkm-y += nvkm/engine/fifo/nv04.o
 nvkm-y += nvkm/engine/fifo/nv10.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
index e6b0653625f5..806ca8986eb5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
@@ -24,6 +24,7 @@
 #include "priv.h"
 #include "chan.h"
 #include "chid.h"
+#include "runq.h"
 
 #include <core/gpuobj.h>
 #include <subdev/mc.h>
@@ -235,7 +236,7 @@ static int
 nvkm_fifo_oneinit(struct nvkm_engine *engine)
 {
 	struct nvkm_fifo *fifo = nvkm_fifo(engine);
-	int ret;
+	int ret, nr, i;
 
 	/* Initialise CHID/CGID allocator(s) on GPUs where they aren't per-runlist. */
 	if (fifo->func->chid_nr) {
@@ -244,6 +245,14 @@ nvkm_fifo_oneinit(struct nvkm_engine *engine)
 			return ret;
 	}
 
+	/* Create runqueues for each PBDMA. */
+	if (fifo->func->runq_nr) {
+		for (nr = fifo->func->runq_nr(fifo), i = 0; i < nr; i++) {
+			if (!nvkm_runq_new(fifo, i))
+				return -ENOMEM;
+		}
+	}
+
 	if (fifo->func->oneinit)
 		return fifo->func->oneinit(fifo);
 
@@ -260,8 +269,12 @@ static void *
 nvkm_fifo_dtor(struct nvkm_engine *engine)
 {
 	struct nvkm_fifo *fifo = nvkm_fifo(engine);
+	struct nvkm_runq *runq, *rtmp;
 	void *data = fifo;
 
+	list_for_each_entry_safe(runq, rtmp, &fifo->runqs, head)
+		nvkm_runq_del(runq);
+
 	nvkm_chid_unref(&fifo->cgid);
 	nvkm_chid_unref(&fifo->chid);
 
@@ -292,6 +305,7 @@ nvkm_fifo_ctor(const struct nvkm_fifo_func *func, struct nvkm_device *device,
 	int ret, nr;
 
 	fifo->func = func;
+	INIT_LIST_HEAD(&fifo->runqs);
 	spin_lock_init(&fifo->lock);
 	mutex_init(&fifo->mutex);
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
index d8eeca818637..14d8bc3b5956 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
@@ -23,6 +23,7 @@
  */
 #include "chan.h"
 #include "chid.h"
+#include "runq.h"
 
 #include "gf100.h"
 #include "changf100.h"
@@ -100,6 +101,10 @@ gf100_fifo_intr_pbdma(struct gf100_fifo *fifo, int unit)
 	nvkm_wr32(device, 0x040108 + (unit * 0x2000), stat);
 }
 
+static const struct nvkm_runq_func
+gf100_runq = {
+};
+
 void
 gf100_fifo_runlist_commit(struct gf100_fifo *fifo)
 {
@@ -625,6 +630,18 @@ gf100_fifo_init(struct nvkm_fifo *base)
 	nvkm_wr32(device, 0x002628, 0x00000001); /* ENGINE_INTR_EN */
 }
 
+int
+gf100_fifo_runq_nr(struct nvkm_fifo *fifo)
+{
+	struct nvkm_device *device = fifo->engine.subdev.device;
+	u32 save;
+
+	/* Determine number of PBDMAs by checking valid enable bits. */
+	save = nvkm_mask(device, 0x000204, 0xffffffff, 0xffffffff);
+	save = nvkm_mask(device, 0x000204, 0xffffffff, save);
+	return hweight32(save);
+}
+
 int
 gf100_fifo_chid_ctor(struct nvkm_fifo *fifo, int nr)
 {
@@ -640,11 +657,7 @@ gf100_fifo_oneinit(struct nvkm_fifo *base)
 	struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device);
 	int ret;
 
-	/* Determine number of PBDMAs by checking valid enable bits. */
-	nvkm_wr32(device, 0x002204, 0xffffffff);
-	fifo->pbdma_nr = hweight32(nvkm_rd32(device, 0x002204));
-	nvkm_debug(subdev, "%d PBDMA(s)\n", fifo->pbdma_nr);
-
+	fifo->pbdma_nr = fifo->base.func->runq_nr(&fifo->base);
 
 	ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x1000,
 			      false, &fifo->runlist.mem[0]);
@@ -689,6 +702,7 @@ gf100_fifo = {
 	.oneinit = gf100_fifo_oneinit,
 	.chid_nr = nv50_fifo_chid_nr,
 	.chid_ctor = gf100_fifo_chid_ctor,
+	.runq_nr = gf100_fifo_runq_nr,
 	.init = gf100_fifo_init,
 	.fini = gf100_fifo_fini,
 	.intr = gf100_fifo_intr,
@@ -697,6 +711,7 @@ gf100_fifo = {
 	.id_engine = gf100_fifo_id_engine,
 	.uevent_init = gf100_fifo_uevent_init,
 	.uevent_fini = gf100_fifo_uevent_fini,
+	.runq = &gf100_runq,
 	.cgrp = {{                            }, &nv04_cgrp },
 	.chan = {{ 0, 0, FERMI_CHANNEL_GPFIFO }, &gf100_chan, .oclass = &gf100_fifo_gpfifo_oclass },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
index c4fa4fc68d46..efebdcdbeefc 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
@@ -23,6 +23,7 @@
  */
 #include "chan.h"
 #include "chid.h"
+#include "runq.h"
 
 #include "gk104.h"
 #include "cgrp.h"
@@ -168,6 +169,10 @@ static const struct nvkm_bitfield gk104_fifo_pbdma_intr_0[] = {
 	{}
 };
 
+const struct nvkm_runq_func
+gk104_runq = {
+};
+
 void
 gk104_fifo_runlist_commit(struct gk104_fifo *fifo, int runl,
 			  struct nvkm_memory *mem, int nr)
@@ -275,18 +280,8 @@ gk104_fifo_pbdma_init(struct gk104_fifo *fifo)
 	nvkm_wr32(device, 0x000204, (1 << fifo->pbdma_nr) - 1);
 }
 
-int
-gk104_fifo_pbdma_nr(struct gk104_fifo *fifo)
-{
-	struct nvkm_device *device = fifo->base.engine.subdev.device;
-	/* Determine number of PBDMAs by checking valid enable bits. */
-	nvkm_wr32(device, 0x000204, 0xffffffff);
-	return hweight32(nvkm_rd32(device, 0x000204));
-}
-
 const struct gk104_fifo_pbdma_func
 gk104_fifo_pbdma = {
-	.nr = gk104_fifo_pbdma_nr,
 	.init = gk104_fifo_pbdma_init,
 };
 
@@ -1076,8 +1071,7 @@ gk104_fifo_oneinit(struct nvkm_fifo *base)
 	int pbid, ret, i, j;
 	u32 *map;
 
-	fifo->pbdma_nr = fifo->func->pbdma->nr(fifo);
-	nvkm_debug(subdev, "%d PBDMA(s)\n", fifo->pbdma_nr);
+	fifo->pbdma_nr = fifo->func->runq_nr(&fifo->base);
 
 	/* Read PBDMA->runlist(s) mapping from HW. */
 	if (!(map = kcalloc(fifo->pbdma_nr, sizeof(*map), GFP_KERNEL)))
@@ -1195,6 +1189,7 @@ gk104_fifo = {
 	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = gk104_fifo_chid_nr,
 	.chid_ctor = gf100_fifo_chid_ctor,
+	.runq_nr = gf100_fifo_runq_nr,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -1213,6 +1208,7 @@ gk104_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gk104_fifo_runlist,
 	.pbdma = &gk104_fifo_pbdma,
+	.runq = &gk104_runq,
 	.cgrp = {{                               }, &nv04_cgrp },
 	.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk104_chan, .ctor = &gk104_fifo_gpfifo_new },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h
index af6a004a6267..636091c7fcee 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h
@@ -82,7 +82,6 @@ void gk104_fifo_uevent_fini(struct nvkm_fifo *fifo);
 void gk104_fifo_uevent_init(struct nvkm_fifo *fifo);
 
 extern const struct gk104_fifo_pbdma_func gk104_fifo_pbdma;
-int gk104_fifo_pbdma_nr(struct gk104_fifo *);
 void gk104_fifo_pbdma_init(struct gk104_fifo *);
 extern const struct nvkm_enum gk104_fifo_fault_access[];
 extern const struct nvkm_enum gk104_fifo_fault_engine[];
@@ -106,7 +105,6 @@ extern const struct nvkm_enum gm107_fifo_fault_engine[];
 extern const struct gk104_fifo_runlist_func gm107_fifo_runlist;
 
 extern const struct gk104_fifo_pbdma_func gm200_fifo_pbdma;
-int gm200_fifo_pbdma_nr(struct gk104_fifo *);
 
 extern const struct nvkm_enum gp100_fifo_fault_engine[];
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c
index b23d50d1b351..d6d86ad47ecb 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c
@@ -75,6 +75,7 @@ gk110_fifo = {
 	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = gk104_fifo_chid_nr,
 	.chid_ctor = gk110_fifo_chid_ctor,
+	.runq_nr = gf100_fifo_runq_nr,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -93,6 +94,7 @@ gk110_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gk110_fifo_runlist,
 	.pbdma = &gk104_fifo_pbdma,
+	.runq = &gk104_runq,
 	.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp },
 	.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_B }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c
index 1dd937bdbbae..d81593722fa1 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c
@@ -21,6 +21,7 @@
  *
  * Authors: Ben Skeggs
  */
+#include "runq.h"
 #include "gk104.h"
 #include "changk104.h"
 
@@ -38,11 +39,14 @@ gk208_fifo_pbdma_init_timeout(struct gk104_fifo *fifo)
 
 const struct gk104_fifo_pbdma_func
 gk208_fifo_pbdma = {
-	.nr = gk104_fifo_pbdma_nr,
 	.init = gk104_fifo_pbdma_init,
 	.init_timeout = gk208_fifo_pbdma_init_timeout,
 };
 
+const struct nvkm_runq_func
+gk208_runq = {
+};
+
 static int
 gk208_fifo_chid_nr(struct nvkm_fifo *fifo)
 {
@@ -55,6 +59,7 @@ gk208_fifo = {
 	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = gk208_fifo_chid_nr,
 	.chid_ctor = gk110_fifo_chid_ctor,
+	.runq_nr = gf100_fifo_runq_nr,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -73,6 +78,7 @@ gk208_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gk110_fifo_runlist,
 	.pbdma = &gk208_fifo_pbdma,
+	.runq = &gk208_runq,
 	.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp },
 	.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c
index d2e78d2a868a..11227cf55941 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c
@@ -30,6 +30,7 @@ gk20a_fifo = {
 	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = nv50_fifo_chid_nr,
 	.chid_ctor = gk110_fifo_chid_ctor,
+	.runq_nr = gf100_fifo_runq_nr,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -48,6 +49,7 @@ gk20a_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gk110_fifo_runlist,
 	.pbdma = &gk208_fifo_pbdma,
+	.runq = &gk208_runq,
 	.cgrp = {{                               }, &gk110_cgrp },
 	.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c
index 32ef88a90d1e..2dc89d3a0d0c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c
@@ -115,6 +115,7 @@ gm107_fifo = {
 	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = gm107_fifo_chid_nr,
 	.chid_ctor = gk110_fifo_chid_ctor,
+	.runq_nr = gf100_fifo_runq_nr,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -133,6 +134,7 @@ gm107_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gm107_fifo_runlist,
 	.pbdma = &gk208_fifo_pbdma,
+	.runq = &gk208_runq,
 	.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp },
 	.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_B }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c
index 74891833926e..f24551c701e1 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c
@@ -27,15 +27,13 @@
 #include <nvif/class.h>
 
 int
-gm200_fifo_pbdma_nr(struct gk104_fifo *fifo)
+gm200_fifo_runq_nr(struct nvkm_fifo *fifo)
 {
-	struct nvkm_device *device = fifo->base.engine.subdev.device;
-	return nvkm_rd32(device, 0x002004) & 0x000000ff;
+	return nvkm_rd32(fifo->engine.subdev.device, 0x002004) & 0x000000ff;
 }
 
 const struct gk104_fifo_pbdma_func
 gm200_fifo_pbdma = {
-	.nr = gm200_fifo_pbdma_nr,
 	.init = gk104_fifo_pbdma_init,
 	.init_timeout = gk208_fifo_pbdma_init_timeout,
 };
@@ -52,6 +50,7 @@ gm200_fifo = {
 	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = gm200_fifo_chid_nr,
 	.chid_ctor = gk110_fifo_chid_ctor,
+	.runq_nr = gm200_fifo_runq_nr,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -70,6 +69,7 @@ gm200_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gm107_fifo_runlist,
 	.pbdma = &gm200_fifo_pbdma,
+	.runq = &gk208_runq,
 	.cgrp = {{ 0, 0,  KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp },
 	.chan = {{ 0, 0, MAXWELL_CHANNEL_GPFIFO_A }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c
index c1ad59d1223b..60bc5d314324 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c
@@ -87,6 +87,7 @@ gp100_fifo = {
 	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = gm200_fifo_chid_nr,
 	.chid_ctor = gk110_fifo_chid_ctor,
+	.runq_nr = gm200_fifo_runq_nr,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -105,6 +106,7 @@ gp100_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gm107_fifo_runlist,
 	.pbdma = &gm200_fifo_pbdma,
+	.runq = &gk208_runq,
 	.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp, .force = true },
 	.chan = {{ 0, 0, PASCAL_CHANNEL_GPFIFO_A }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c
index 9e876bd871bb..f7db1b4db0dc 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c
@@ -21,6 +21,7 @@
  */
 #include "chan.h"
 #include "cgrp.h"
+#include "runq.h"
 
 #include "gk104.h"
 #include "changk104.h"
@@ -33,6 +34,10 @@ static const struct nvkm_chan_func
 gv100_chan = {
 };
 
+const struct nvkm_runq_func
+gv100_runq = {
+};
+
 void
 gv100_fifo_runlist_chan(struct gk104_fifo_chan *chan,
 			struct nvkm_memory *memory, u32 offset)
@@ -302,6 +307,7 @@ gv100_fifo = {
 	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = gm200_fifo_chid_nr,
 	.chid_ctor = gk110_fifo_chid_ctor,
+	.runq_nr = gm200_fifo_runq_nr,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -319,6 +325,7 @@ gv100_fifo = {
 	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gv100_fifo_runlist,
 	.pbdma = &gm200_fifo_pbdma,
+	.runq = &gv100_runq,
 	.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp, .force = true },
 	.chan = {{ 0, 0,  VOLTA_CHANNEL_GPFIFO_A }, &gv100_chan, .ctor = gv100_fifo_gpfifo_new },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h
index 80280841cf81..daa594669985 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h
@@ -22,6 +22,7 @@ struct nvkm_fifo_func {
 	int (*oneinit)(struct nvkm_fifo *);
 	int (*chid_nr)(struct nvkm_fifo *);
 	int (*chid_ctor)(struct nvkm_fifo *, int nr);
+	int (*runq_nr)(struct nvkm_fifo *);
 
 	int (*info)(struct nvkm_fifo *, u64 mthd, u64 *data);
 	void (*init)(struct nvkm_fifo *);
@@ -61,11 +62,12 @@ struct nvkm_fifo_func {
 	} *runlist;
 
 	const struct gk104_fifo_pbdma_func {
-		int (*nr)(struct gk104_fifo *);
 		void (*init)(struct gk104_fifo *);
 		void (*init_timeout)(struct gk104_fifo *);
 	} *pbdma;
 
+	const struct nvkm_runq_func *runq;
+
 	struct nvkm_fifo_func_cgrp {
 		struct nvkm_sclass user;
 		const struct nvkm_cgrp_func *func;
@@ -104,6 +106,7 @@ int nv50_fifo_chid_ctor(struct nvkm_fifo *, int);
 extern const struct nvkm_chan_func g84_chan;
 
 int gf100_fifo_chid_ctor(struct nvkm_fifo *, int);
+int gf100_fifo_runq_nr(struct nvkm_fifo *);
 void gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int);
 
 int gk104_fifo_chid_nr(struct nvkm_fifo *);
@@ -113,19 +116,25 @@ void gk104_fifo_fault(struct nvkm_fifo *, struct nvkm_fault_data *);
 void gk104_fifo_recover_chan(struct nvkm_fifo *, int);
 int gk104_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *);
 struct nvkm_engine *gk104_fifo_id_engine(struct nvkm_fifo *, int);
+extern const struct nvkm_runq_func gk104_runq;
 
 int gk110_fifo_chid_ctor(struct nvkm_fifo *, int);
 extern const struct nvkm_cgrp_func gk110_cgrp;
 extern const struct nvkm_chan_func gk110_chan;
 
+extern const struct nvkm_runq_func gk208_runq;
+
 void gm107_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int);
 extern const struct nvkm_fifo_func_mmu_fault gm107_fifo_mmu_fault;
 extern const struct nvkm_chan_func gm107_chan;
 
 int gm200_fifo_chid_nr(struct nvkm_fifo *);
+int gm200_fifo_runq_nr(struct nvkm_fifo *);
 
 void gp100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int);
 
+extern const struct nvkm_runq_func gv100_runq;
+
 extern const struct nvkm_fifo_func_mmu_fault tu102_fifo_mmu_fault;
 
 int nvkm_uchan_new(struct nvkm_fifo *, struct nvkm_cgrp *, const struct nvkm_oclass *,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.c
new file mode 100644
index 000000000000..33bcf5fb3ef0
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2021 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "runq.h"
+#include "priv.h"
+
+void
+nvkm_runq_del(struct nvkm_runq *runq)
+{
+	list_del(&runq->head);
+	kfree(runq);
+}
+
+struct nvkm_runq *
+nvkm_runq_new(struct nvkm_fifo *fifo, int pbid)
+{
+	struct nvkm_runq *runq;
+
+	if (!(runq = kzalloc(sizeof(*runq), GFP_KERNEL)))
+		return NULL;
+
+	runq->func = fifo->func->runq;
+	runq->fifo = fifo;
+	runq->id = pbid;
+	list_add_tail(&runq->head, &fifo->runqs);
+	return runq;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h
new file mode 100644
index 000000000000..1832a14488c3
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: MIT */
+#ifndef __NVKM_RUNQ_H__
+#define __NVKM_RUNQ_H__
+#include <core/os.h>
+
+struct nvkm_runq {
+	const struct nvkm_runq_func {
+	} *func;
+	struct nvkm_fifo *fifo;
+	int id;
+
+	struct list_head head;
+};
+
+struct nvkm_runq *nvkm_runq_new(struct nvkm_fifo *, int pbid);
+void nvkm_runq_del(struct nvkm_runq *);
+
+#define nvkm_runq_foreach_cond(runq,fifo,cond) nvkm_list_foreach(runq, &(fifo)->runqs, head, (cond))
+
+#define RUNQ_PRINT(r,l,p,f,a...)							   \
+	nvkm_printk__(&(r)->fifo->engine.subdev, NV_DBG_##l, p, "PBDMA%d:"f, (r)->id, ##a)
+#define RUNQ_ERROR(r,f,a...) RUNQ_PRINT((r), ERROR,    err, " "f"\n", ##a)
+#define RUNQ_DEBUG(r,f,a...) RUNQ_PRINT((r), DEBUG,   info, " "f"\n", ##a)
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c
index fbc67b84a8b1..6d0431610235 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c
@@ -99,7 +99,6 @@ tu102_fifo_pbdma_init(struct gk104_fifo *fifo)
 
 static const struct gk104_fifo_pbdma_func
 tu102_fifo_pbdma = {
-	.nr = gm200_fifo_pbdma_nr,
 	.init = tu102_fifo_pbdma_init,
 	.init_timeout = gk208_fifo_pbdma_init_timeout,
 };
@@ -443,6 +442,7 @@ tu102_fifo = {
 	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = gm200_fifo_chid_nr,
 	.chid_ctor = gk110_fifo_chid_ctor,
+	.runq_nr = gm200_fifo_runq_nr,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
@@ -460,6 +460,7 @@ tu102_fifo = {
 	.recover_chan = tu102_fifo_recover_chan,
 	.runlist = &tu102_fifo_runlist,
 	.pbdma = &tu102_fifo_pbdma,
+	.runq = &gv100_runq,
 	.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp, .force = true },
 	.chan = {{ 0, 0, TURING_CHANNEL_GPFIFO_A }, &tu102_chan, .ctor = tu102_fifo_gpfifo_new },
 };