]> git.proxmox.com Git - mirror_ubuntu-eoan-kernel.git/blame - drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
projects / mirror_ubuntu-eoan-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/amdgpu: Add vega20 support on kfd probe
[mirror_ubuntu-eoan-kernel.git] / drivers / gpu / drm / amd / amdkfd / kfd_device_queue_manager.c
CommitLineData
64c7f8cf
BG		 1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
26103436
FK		 24#include <linux/ratelimit.h>
25#include <linux/printk.h>
64c7f8cf
BG		 26#include <linux/slab.h>
27#include <linux/list.h>
28#include <linux/types.h>
64c7f8cf 29#include <linux/bitops.h>
99331a51 30#include <linux/sched.h>
64c7f8cf
BG		 31#include "kfd_priv.h"
32#include "kfd_device_queue_manager.h"
33#include "kfd_mqd_manager.h"
34#include "cik_regs.h"
35#include "kfd_kernel_queue.h"
64c7f8cf
BG		 36
37/* Size of the per-pipe EOP queue */
38#define CIK_HPD_EOP_BYTES_LOG2 11
39#define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)
40
64c7f8cf
BG		 41static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
42 unsigned int pasid, unsigned int vmid);
43
44static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
45 struct queue *q,
46 struct qcm_process_device *qpd);
bcea3081 47
c4744e24
YZ		 48static int execute_queues_cpsch(struct device_queue_manager *dqm,
49 enum kfd_unmap_queues_filter filter,
50 uint32_t filter_param);
7da2bcf8 51static int unmap_queues_cpsch(struct device_queue_manager *dqm,
4465f466
YZ		 52 enum kfd_unmap_queues_filter filter,
53 uint32_t filter_param);
64c7f8cf 54
60a00956
FK		 55static int map_queues_cpsch(struct device_queue_manager *dqm);
56
bcea3081
BG		 57static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
58 struct queue *q,
59 struct qcm_process_device *qpd);
60
61static void deallocate_sdma_queue(struct device_queue_manager *dqm,
62 unsigned int sdma_queue_id);
64c7f8cf 63
73ea648d
SL		 64static void kfd_process_hw_exception(struct work_struct *work);
65
bcea3081
BG		 66static inline
67enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
64c7f8cf 68{
bcea3081 69 if (type == KFD_QUEUE_TYPE_SDMA)
85d258f9
BG		 70 return KFD_MQD_TYPE_SDMA;
71 return KFD_MQD_TYPE_CP;
64c7f8cf
BG		 72}
73
d0b63bb3
AR		 74static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe)
75{
76 int i;
77 int pipe_offset = mec * dqm->dev->shared_resources.num_pipe_per_mec
78 + pipe * dqm->dev->shared_resources.num_queue_per_pipe;
79
80 /* queue is available for KFD usage if bit is 1 */
81 for (i = 0; i < dqm->dev->shared_resources.num_queue_per_pipe; ++i)
82 if (test_bit(pipe_offset + i,
83 dqm->dev->shared_resources.queue_bitmap))
84 return true;
85 return false;
86}
87
d0b63bb3 88unsigned int get_queues_num(struct device_queue_manager *dqm)
64ea8f4a 89{
d0b63bb3
AR		 90 return bitmap_weight(dqm->dev->shared_resources.queue_bitmap,
91 KGD_MAX_QUEUES);
64ea8f4a
OG		 92}
93
d0b63bb3 94unsigned int get_queues_per_pipe(struct device_queue_manager *dqm)
64c7f8cf 95{
d0b63bb3
AR		 96 return dqm->dev->shared_resources.num_queue_per_pipe;
97}
98
99unsigned int get_pipes_per_mec(struct device_queue_manager *dqm)
100{
d0b63bb3 101 return dqm->dev->shared_resources.num_pipe_per_mec;
64c7f8cf
BG		 102}
103
98bb9222
YZ		 104static unsigned int get_num_sdma_engines(struct device_queue_manager *dqm)
105{
106 return dqm->dev->device_info->num_sdma_engines;
107}
108
109unsigned int get_num_sdma_queues(struct device_queue_manager *dqm)
110{
111 return dqm->dev->device_info->num_sdma_engines
d5094189 112 * dqm->dev->device_info->num_sdma_queues_per_engine;
98bb9222
YZ		 113}
114
a22fc854 115void program_sh_mem_settings(struct device_queue_manager *dqm,
64c7f8cf
BG		 116 struct qcm_process_device *qpd)
117{
cea405b1
XZ		 118 return dqm->dev->kfd2kgd->program_sh_mem_settings(
119 dqm->dev->kgd, qpd->vmid,
64c7f8cf
BG		 120 qpd->sh_mem_config,
121 qpd->sh_mem_ape1_base,
122 qpd->sh_mem_ape1_limit,
123 qpd->sh_mem_bases);
124}
125
ef568db7
FK		 126static int allocate_doorbell(struct qcm_process_device *qpd, struct queue *q)
127{
128 struct kfd_dev *dev = qpd->dqm->dev;
129
130 if (!KFD_IS_SOC15(dev->device_info->asic_family)) {
131 /* On pre-SOC15 chips we need to use the queue ID to
132 * preserve the user mode ABI.
133 */
134 q->doorbell_id = q->properties.queue_id;
135 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
136 /* For SDMA queues on SOC15, use static doorbell
137 * assignments based on the engine and queue.
138 */
139 q->doorbell_id = dev->shared_resources.sdma_doorbell
140 [q->properties.sdma_engine_id]
141 [q->properties.sdma_queue_id];
142 } else {
143 /* For CP queues on SOC15 reserve a free doorbell ID */
144 unsigned int found;
145
146 found = find_first_zero_bit(qpd->doorbell_bitmap,
147 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
148 if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {
149 pr_debug("No doorbells available");
150 return -EBUSY;
151 }
152 set_bit(found, qpd->doorbell_bitmap);
153 q->doorbell_id = found;
154 }
155
156 q->properties.doorbell_off =
157 kfd_doorbell_id_to_offset(dev, q->process,
158 q->doorbell_id);
159
160 return 0;
161}
162
163static void deallocate_doorbell(struct qcm_process_device *qpd,
164 struct queue *q)
165{
166 unsigned int old;
167 struct kfd_dev *dev = qpd->dqm->dev;
168
169 if (!KFD_IS_SOC15(dev->device_info->asic_family) ||
170 q->properties.type == KFD_QUEUE_TYPE_SDMA)
171 return;
172
173 old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap);
174 WARN_ON(!old);
175}
176
64c7f8cf
BG		 177static int allocate_vmid(struct device_queue_manager *dqm,
178 struct qcm_process_device *qpd,
179 struct queue *q)
180{
181 int bit, allocated_vmid;
182
183 if (dqm->vmid_bitmap == 0)
184 return -ENOMEM;
185
4252bf68
HK		 186 bit = ffs(dqm->vmid_bitmap) - 1;
187 dqm->vmid_bitmap &= ~(1 << bit);
64c7f8cf 188
44008d7a 189 allocated_vmid = bit + dqm->dev->vm_info.first_vmid_kfd;
79775b62 190 pr_debug("vmid allocation %d\n", allocated_vmid);
64c7f8cf
BG		 191 qpd->vmid = allocated_vmid;
192 q->properties.vmid = allocated_vmid;
193
194 set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid);
195 program_sh_mem_settings(dqm, qpd);
196
403575c4
FK		 197 /* qpd->page_table_base is set earlier when register_process()
198 * is called, i.e. when the first queue is created.
199 */
200 dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->kgd,
201 qpd->vmid,
202 qpd->page_table_base);
203 /* invalidate the VM context after pasid and vmid mapping is set up */
204 kfd_flush_tlb(qpd_to_pdd(qpd));
205
64c7f8cf
BG		 206 return 0;
207}
208
552764b6
FK		 209static int flush_texture_cache_nocpsch(struct kfd_dev *kdev,
210 struct qcm_process_device *qpd)
211{
f6e27ff1
FK		 212 const struct packet_manager_funcs *pmf = qpd->dqm->packets.pmf;
213 int ret;
552764b6
FK		 214
215 if (!qpd->ib_kaddr)
216 return -ENOMEM;
217
f6e27ff1
FK		 218 ret = pmf->release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr);
219 if (ret)
220 return ret;
552764b6
FK		 221
222 return kdev->kfd2kgd->submit_ib(kdev->kgd, KGD_ENGINE_MEC1, qpd->vmid,
f6e27ff1
FK		 223 qpd->ib_base, (uint32_t *)qpd->ib_kaddr,
224 pmf->release_mem_size / sizeof(uint32_t));
552764b6
FK		 225}
226
64c7f8cf
BG		 227static void deallocate_vmid(struct device_queue_manager *dqm,
228 struct qcm_process_device *qpd,
229 struct queue *q)
230{
44008d7a 231 int bit = qpd->vmid - dqm->dev->vm_info.first_vmid_kfd;
64c7f8cf 232
552764b6
FK		 233 /* On GFX v7, CP doesn't flush TC at dequeue */
234 if (q->device->device_info->asic_family == CHIP_HAWAII)
235 if (flush_texture_cache_nocpsch(q->device, qpd))
236 pr_err("Failed to flush TC\n");
237
403575c4
FK		 238 kfd_flush_tlb(qpd_to_pdd(qpd));
239
2030664b
BG		 240 /* Release the vmid mapping */
241 set_pasid_vmid_mapping(dqm, 0, qpd->vmid);
242
4252bf68 243 dqm->vmid_bitmap |= (1 << bit);
64c7f8cf
BG		 244 qpd->vmid = 0;
245 q->properties.vmid = 0;
246}
247
248static int create_queue_nocpsch(struct device_queue_manager *dqm,
249 struct queue *q,
b46cb7d7 250 struct qcm_process_device *qpd)
64c7f8cf
BG		 251{
252 int retval;
253
64c7f8cf
BG		 254 print_queue(q);
255
efeaed4d 256 dqm_lock(dqm);
64c7f8cf 257
b8cbab04 258 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
79775b62 259 pr_warn("Can't create new usermode queue because %d queues were already created\n",
b8cbab04 260 dqm->total_queue_count);
ab7c1648
KR		 261 retval = -EPERM;
262 goto out_unlock;
b8cbab04
OG		 263 }
264
64c7f8cf
BG		 265 if (list_empty(&qpd->queues_list)) {
266 retval = allocate_vmid(dqm, qpd, q);
ab7c1648
KR		 267 if (retval)
268 goto out_unlock;
64c7f8cf 269 }
64c7f8cf 270 q->properties.vmid = qpd->vmid;
26103436
FK		 271 /*
272 * Eviction state logic: we only mark active queues as evicted
273 * to avoid the overhead of restoring inactive queues later
274 */
275 if (qpd->evicted)
276 q->properties.is_evicted = (q->properties.queue_size > 0 &&
277 q->properties.queue_percent > 0 &&
278 q->properties.queue_address != 0);
64c7f8cf 279
373d7080
FK		 280 q->properties.tba_addr = qpd->tba_addr;
281 q->properties.tma_addr = qpd->tma_addr;
282
bcea3081
BG		 283 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
284 retval = create_compute_queue_nocpsch(dqm, q, qpd);
ab7c1648 285 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
bcea3081 286 retval = create_sdma_queue_nocpsch(dqm, q, qpd);
ab7c1648
KR		 287 else
288 retval = -EINVAL;
64c7f8cf 289
4eacc26b 290 if (retval) {
b46cb7d7 291 if (list_empty(&qpd->queues_list))
64c7f8cf 292 deallocate_vmid(dqm, qpd, q);
ab7c1648 293 goto out_unlock;
64c7f8cf
BG		 294 }
295
296 list_add(&q->list, &qpd->queues_list);
bc920fd4 297 qpd->queue_count++;
b6819cec
JC		 298 if (q->properties.is_active)
299 dqm->queue_count++;
64c7f8cf 300
bcea3081
BG		 301 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
302 dqm->sdma_queue_count++;
64c7f8cf 303
b8cbab04
OG		 304 /*
305 * Unconditionally increment this counter, regardless of the queue's
306 * type or whether the queue is active.
307 */
308 dqm->total_queue_count++;
309 pr_debug("Total of %d queues are accountable so far\n",
310 dqm->total_queue_count);
311
ab7c1648 312out_unlock:
efeaed4d 313 dqm_unlock(dqm);
ab7c1648 314 return retval;
64c7f8cf
BG		 315}
316
317static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
318{
319 bool set;
f0ec5b99 320 int pipe, bit, i;
64c7f8cf
BG		 321
322 set = false;
323
8eabaf54
KR		 324 for (pipe = dqm->next_pipe_to_allocate, i = 0;
325 i < get_pipes_per_mec(dqm);
d0b63bb3
AR		 326 pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) {
327
328 if (!is_pipe_enabled(dqm, 0, pipe))
329 continue;
330
64c7f8cf 331 if (dqm->allocated_queues[pipe] != 0) {
4252bf68
HK		 332 bit = ffs(dqm->allocated_queues[pipe]) - 1;
333 dqm->allocated_queues[pipe] &= ~(1 << bit);
64c7f8cf
BG		 334 q->pipe = pipe;
335 q->queue = bit;
336 set = true;
337 break;
338 }
339 }
340
991ca8ee 341 if (!set)
64c7f8cf
BG		 342 return -EBUSY;
343
79775b62 344 pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue);
64c7f8cf 345 /* horizontal hqd allocation */
d0b63bb3 346 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm);
64c7f8cf
BG		 347
348 return 0;
349}
350
351static inline void deallocate_hqd(struct device_queue_manager *dqm,
352 struct queue *q)
353{
4252bf68 354 dqm->allocated_queues[q->pipe] |= (1 << q->queue);
64c7f8cf
BG		 355}
356
357static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
358 struct queue *q,
359 struct qcm_process_device *qpd)
360{
361 int retval;
8d5f3552 362 struct mqd_manager *mqd_mgr;
64c7f8cf 363
8d5f3552
YZ		 364 mqd_mgr = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
365 if (!mqd_mgr)
64c7f8cf
BG		 366 return -ENOMEM;
367
368 retval = allocate_hqd(dqm, q);
4eacc26b 369 if (retval)
64c7f8cf
BG		 370 return retval;
371
ef568db7
FK		 372 retval = allocate_doorbell(qpd, q);
373 if (retval)
374 goto out_deallocate_hqd;
375
8d5f3552 376 retval = mqd_mgr->init_mqd(mqd_mgr, &q->mqd, &q->mqd_mem_obj,
64c7f8cf 377 &q->gart_mqd_addr, &q->properties);
ab7c1648 378 if (retval)
ef568db7 379 goto out_deallocate_doorbell;
64c7f8cf 380
79775b62
KR		 381 pr_debug("Loading mqd to hqd on pipe %d, queue %d\n",
382 q->pipe, q->queue);
030e416b 383
6a1c9510
MR		 384 dqm->dev->kfd2kgd->set_scratch_backing_va(
385 dqm->dev->kgd, qpd->sh_hidden_private_base, qpd->vmid);
386
60a00956
FK		 387 if (!q->properties.is_active)
388 return 0;
389
8d5f3552
YZ		 390 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, q->queue,
391 &q->properties, q->process->mm);
ab7c1648
KR		 392 if (retval)
393 goto out_uninit_mqd;
030e416b 394
64c7f8cf 395 return 0;
ab7c1648
KR		 396
397out_uninit_mqd:
8d5f3552 398 mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
ef568db7
FK		 399out_deallocate_doorbell:
400 deallocate_doorbell(qpd, q);
ab7c1648
KR		 401out_deallocate_hqd:
402 deallocate_hqd(dqm, q);
403
404 return retval;
64c7f8cf
BG		 405}
406
9fd3f1bf
FK		 407/* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked
408 * to avoid asynchronized access
409 */
410static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
64c7f8cf
BG		 411 struct qcm_process_device *qpd,
412 struct queue *q)
413{
414 int retval;
8d5f3552 415 struct mqd_manager *mqd_mgr;
64c7f8cf 416
8d5f3552 417 mqd_mgr = dqm->ops.get_mqd_manager(dqm,
9fd3f1bf 418 get_mqd_type_from_queue_type(q->properties.type));
8d5f3552 419 if (!mqd_mgr)
9fd3f1bf 420 return -ENOMEM;
64c7f8cf 421
c2e1b3a4 422 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
c2e1b3a4
BG		 423 deallocate_hqd(dqm, q);
424 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
c2e1b3a4
BG		 425 dqm->sdma_queue_count--;
426 deallocate_sdma_queue(dqm, q->sdma_id);
7113cd65 427 } else {
79775b62 428 pr_debug("q->properties.type %d is invalid\n",
7113cd65 429 q->properties.type);
9fd3f1bf 430 return -EINVAL;
64c7f8cf 431 }
9fd3f1bf 432 dqm->total_queue_count--;
64c7f8cf 433
ef568db7
FK		 434 deallocate_doorbell(qpd, q);
435
8d5f3552 436 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
c2e1b3a4 437 KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
b90e3fbe 438 KFD_UNMAP_LATENCY_MS,
64c7f8cf 439 q->pipe, q->queue);
9fd3f1bf
FK		 440 if (retval == -ETIME)
441 qpd->reset_wavefronts = true;
64c7f8cf 442
8d5f3552 443 mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
64c7f8cf
BG		 444
445 list_del(&q->list);
9fd3f1bf
FK		 446 if (list_empty(&qpd->queues_list)) {
447 if (qpd->reset_wavefronts) {
448 pr_warn("Resetting wave fronts (nocpsch) on dev %p\n",
449 dqm->dev);
450 /* dbgdev_wave_reset_wavefronts has to be called before
451 * deallocate_vmid(), i.e. when vmid is still in use.
452 */
453 dbgdev_wave_reset_wavefronts(dqm->dev,
454 qpd->pqm->process);
455 qpd->reset_wavefronts = false;
456 }
457
64c7f8cf 458 deallocate_vmid(dqm, qpd, q);
9fd3f1bf 459 }
bc920fd4 460 qpd->queue_count--;
b6819cec
JC		 461 if (q->properties.is_active)
462 dqm->queue_count--;
b8cbab04 463
9fd3f1bf
FK		 464 return retval;
465}
b8cbab04 466
9fd3f1bf
FK		 467static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
468 struct qcm_process_device *qpd,
469 struct queue *q)
470{
471 int retval;
472
efeaed4d 473 dqm_lock(dqm);
9fd3f1bf 474 retval = destroy_queue_nocpsch_locked(dqm, qpd, q);
efeaed4d 475 dqm_unlock(dqm);
9fd3f1bf 476
64c7f8cf
BG		 477 return retval;
478}
479
480static int update_queue(struct device_queue_manager *dqm, struct queue *q)
481{
482 int retval;
8d5f3552 483 struct mqd_manager *mqd_mgr;
26103436 484 struct kfd_process_device *pdd;
b6ffbab8 485 bool prev_active = false;
64c7f8cf 486
efeaed4d 487 dqm_lock(dqm);
26103436
FK		 488 pdd = kfd_get_process_device_data(q->device, q->process);
489 if (!pdd) {
490 retval = -ENODEV;
491 goto out_unlock;
492 }
8d5f3552 493 mqd_mgr = dqm->ops.get_mqd_manager(dqm,
0b3674ae 494 get_mqd_type_from_queue_type(q->properties.type));
8d5f3552 495 if (!mqd_mgr) {
ab7c1648
KR		 496 retval = -ENOMEM;
497 goto out_unlock;
64c7f8cf 498 }
26103436
FK		 499 /*
500 * Eviction state logic: we only mark active queues as evicted
501 * to avoid the overhead of restoring inactive queues later
502 */
503 if (pdd->qpd.evicted)
504 q->properties.is_evicted = (q->properties.queue_size > 0 &&
505 q->properties.queue_percent > 0 &&
506 q->properties.queue_address != 0);
64c7f8cf 507
60a00956
FK		 508 /* Save previous activity state for counters */
509 prev_active = q->properties.is_active;
510
511 /* Make sure the queue is unmapped before updating the MQD */
d146c5a7 512 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) {
60a00956
FK		 513 retval = unmap_queues_cpsch(dqm,
514 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
894a8293 515 if (retval) {
60a00956
FK		 516 pr_err("unmap queue failed\n");
517 goto out_unlock;
518 }
894a8293 519 } else if (prev_active &&
60a00956
FK		 520 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
521 q->properties.type == KFD_QUEUE_TYPE_SDMA)) {
8d5f3552 522 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
60a00956
FK		 523 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
524 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
525 if (retval) {
526 pr_err("destroy mqd failed\n");
527 goto out_unlock;
528 }
529 }
530
8d5f3552 531 retval = mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties);
60a00956 532
096d1a3e
FK		 533 /*
534 * check active state vs. the previous state and modify
535 * counter accordingly. map_queues_cpsch uses the
536 * dqm->queue_count to determine whether a new runlist must be
537 * uploaded.
538 */
539 if (q->properties.is_active && !prev_active)
540 dqm->queue_count++;
541 else if (!q->properties.is_active && prev_active)
542 dqm->queue_count--;
543
d146c5a7 544 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS)
60a00956 545 retval = map_queues_cpsch(dqm);
894a8293 546 else if (q->properties.is_active &&
60a00956
FK		 547 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
548 q->properties.type == KFD_QUEUE_TYPE_SDMA))
8d5f3552 549 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, q->queue,
60a00956 550 &q->properties, q->process->mm);
b6ffbab8 551
ab7c1648 552out_unlock:
efeaed4d 553 dqm_unlock(dqm);
64c7f8cf
BG		 554 return retval;
555}
556
58dcd5bf 557static struct mqd_manager *get_mqd_manager(
64c7f8cf
BG		 558 struct device_queue_manager *dqm, enum KFD_MQD_TYPE type)
559{
8d5f3552 560 struct mqd_manager *mqd_mgr;
64c7f8cf 561
32fa8219
FK		 562 if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
563 return NULL;
64c7f8cf 564
79775b62 565 pr_debug("mqd type %d\n", type);
64c7f8cf 566
8d5f3552
YZ		 567 mqd_mgr = dqm->mqd_mgrs[type];
568 if (!mqd_mgr) {
569 mqd_mgr = mqd_manager_init(type, dqm->dev);
570 if (!mqd_mgr)
79775b62 571 pr_err("mqd manager is NULL");
8d5f3552 572 dqm->mqd_mgrs[type] = mqd_mgr;
64c7f8cf
BG		 573 }
574
8d5f3552 575 return mqd_mgr;
64c7f8cf
BG		 576}
577
26103436
FK		 578static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
579 struct qcm_process_device *qpd)
580{
581 struct queue *q;
8d5f3552 582 struct mqd_manager *mqd_mgr;
26103436
FK		 583 struct kfd_process_device *pdd;
584 int retval = 0;
585
efeaed4d 586 dqm_lock(dqm);
26103436
FK		 587 if (qpd->evicted++ > 0) /* already evicted, do nothing */
588 goto out;
589
590 pdd = qpd_to_pdd(qpd);
591 pr_info_ratelimited("Evicting PASID %u queues\n",
592 pdd->process->pasid);
593
594 /* unactivate all active queues on the qpd */
595 list_for_each_entry(q, &qpd->queues_list, list) {
596 if (!q->properties.is_active)
597 continue;
8d5f3552 598 mqd_mgr = dqm->ops.get_mqd_manager(dqm,
26103436 599 get_mqd_type_from_queue_type(q->properties.type));
8d5f3552 600 if (!mqd_mgr) { /* should not be here */
26103436
FK		 601 pr_err("Cannot evict queue, mqd mgr is NULL\n");
602 retval = -ENOMEM;
603 goto out;
604 }
605 q->properties.is_evicted = true;
606 q->properties.is_active = false;
8d5f3552 607 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
26103436
FK		 608 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
609 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
610 if (retval)
611 goto out;
612 dqm->queue_count--;
613 }
614
615out:
efeaed4d 616 dqm_unlock(dqm);
26103436
FK		 617 return retval;
618}
619
620static int evict_process_queues_cpsch(struct device_queue_manager *dqm,
621 struct qcm_process_device *qpd)
622{
623 struct queue *q;
624 struct kfd_process_device *pdd;
625 int retval = 0;
626
efeaed4d 627 dqm_lock(dqm);
26103436
FK		 628 if (qpd->evicted++ > 0) /* already evicted, do nothing */
629 goto out;
630
631 pdd = qpd_to_pdd(qpd);
632 pr_info_ratelimited("Evicting PASID %u queues\n",
633 pdd->process->pasid);
634
635 /* unactivate all active queues on the qpd */
636 list_for_each_entry(q, &qpd->queues_list, list) {
637 if (!q->properties.is_active)
638 continue;
639 q->properties.is_evicted = true;
640 q->properties.is_active = false;
641 dqm->queue_count--;
642 }
643 retval = execute_queues_cpsch(dqm,
644 qpd->is_debug ?
645 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES :
646 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
647
648out:
efeaed4d 649 dqm_unlock(dqm);
26103436
FK		 650 return retval;
651}
652
653static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
654 struct qcm_process_device *qpd)
655{
656 struct queue *q;
8d5f3552 657 struct mqd_manager *mqd_mgr;
26103436 658 struct kfd_process_device *pdd;
e715c6d0 659 uint64_t pd_base;
26103436
FK		 660 int retval = 0;
661
662 pdd = qpd_to_pdd(qpd);
663 /* Retrieve PD base */
664 pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
665
efeaed4d 666 dqm_lock(dqm);
26103436
FK		 667 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
668 goto out;
669 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
670 qpd->evicted--;
671 goto out;
672 }
673
674 pr_info_ratelimited("Restoring PASID %u queues\n",
675 pdd->process->pasid);
676
677 /* Update PD Base in QPD */
678 qpd->page_table_base = pd_base;
e715c6d0 679 pr_debug("Updated PD address to 0x%llx\n", pd_base);
26103436
FK		 680
681 if (!list_empty(&qpd->queues_list)) {
682 dqm->dev->kfd2kgd->set_vm_context_page_table_base(
683 dqm->dev->kgd,
684 qpd->vmid,
685 qpd->page_table_base);
686 kfd_flush_tlb(pdd);
687 }
688
689 /* activate all active queues on the qpd */
690 list_for_each_entry(q, &qpd->queues_list, list) {
691 if (!q->properties.is_evicted)
692 continue;
8d5f3552 693 mqd_mgr = dqm->ops.get_mqd_manager(dqm,
26103436 694 get_mqd_type_from_queue_type(q->properties.type));
8d5f3552 695 if (!mqd_mgr) { /* should not be here */
26103436
FK		 696 pr_err("Cannot restore queue, mqd mgr is NULL\n");
697 retval = -ENOMEM;
698 goto out;
699 }
700 q->properties.is_evicted = false;
701 q->properties.is_active = true;
8d5f3552 702 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
26103436
FK		 703 q->queue, &q->properties,
704 q->process->mm);
705 if (retval)
706 goto out;
707 dqm->queue_count++;
708 }
709 qpd->evicted = 0;
710out:
efeaed4d 711 dqm_unlock(dqm);
26103436
FK		 712 return retval;
713}
714
715static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
716 struct qcm_process_device *qpd)
717{
718 struct queue *q;
719 struct kfd_process_device *pdd;
e715c6d0 720 uint64_t pd_base;
26103436
FK		 721 int retval = 0;
722
723 pdd = qpd_to_pdd(qpd);
724 /* Retrieve PD base */
725 pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
726
efeaed4d 727 dqm_lock(dqm);
26103436
FK		 728 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
729 goto out;
730 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
731 qpd->evicted--;
732 goto out;
733 }
734
735 pr_info_ratelimited("Restoring PASID %u queues\n",
736 pdd->process->pasid);
737
738 /* Update PD Base in QPD */
739 qpd->page_table_base = pd_base;
e715c6d0 740 pr_debug("Updated PD address to 0x%llx\n", pd_base);
26103436
FK		 741
742 /* activate all active queues on the qpd */
743 list_for_each_entry(q, &qpd->queues_list, list) {
744 if (!q->properties.is_evicted)
745 continue;
746 q->properties.is_evicted = false;
747 q->properties.is_active = true;
748 dqm->queue_count++;
749 }
750 retval = execute_queues_cpsch(dqm,
751 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
752 if (!retval)
753 qpd->evicted = 0;
754out:
efeaed4d 755 dqm_unlock(dqm);
26103436
FK		 756 return retval;
757}
758
58dcd5bf 759static int register_process(struct device_queue_manager *dqm,
64c7f8cf
BG		 760 struct qcm_process_device *qpd)
761{
762 struct device_process_node *n;
403575c4 763 struct kfd_process_device *pdd;
e715c6d0 764 uint64_t pd_base;
a22fc854 765 int retval;
64c7f8cf 766
dbf56ab1 767 n = kzalloc(sizeof(*n), GFP_KERNEL);
64c7f8cf
BG		 768 if (!n)
769 return -ENOMEM;
770
771 n->qpd = qpd;
772
403575c4
FK		 773 pdd = qpd_to_pdd(qpd);
774 /* Retrieve PD base */
775 pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
776
efeaed4d 777 dqm_lock(dqm);
64c7f8cf
BG		 778 list_add(&n->list, &dqm->queues);
779
403575c4
FK		 780 /* Update PD Base in QPD */
781 qpd->page_table_base = pd_base;
e715c6d0 782 pr_debug("Updated PD address to 0x%llx\n", pd_base);
403575c4 783
bfd5e378 784 retval = dqm->asic_ops.update_qpd(dqm, qpd);
a22fc854 785
b5aa3f4a
FK		 786 if (dqm->processes_count++ == 0)
787 dqm->dev->kfd2kgd->set_compute_idle(dqm->dev->kgd, false);
64c7f8cf 788
efeaed4d 789 dqm_unlock(dqm);
64c7f8cf 790
a22fc854 791 return retval;
64c7f8cf
BG		 792}
793
58dcd5bf 794static int unregister_process(struct device_queue_manager *dqm,
64c7f8cf
BG		 795 struct qcm_process_device *qpd)
796{
797 int retval;
798 struct device_process_node *cur, *next;
799
1e5ec956
OG		 800 pr_debug("qpd->queues_list is %s\n",
801 list_empty(&qpd->queues_list) ? "empty" : "not empty");
64c7f8cf
BG		 802
803 retval = 0;
efeaed4d 804 dqm_lock(dqm);
64c7f8cf
BG		 805
806 list_for_each_entry_safe(cur, next, &dqm->queues, list) {
807 if (qpd == cur->qpd) {
808 list_del(&cur->list);
f5d896bb 809 kfree(cur);
b5aa3f4a
FK		 810 if (--dqm->processes_count == 0)
811 dqm->dev->kfd2kgd->set_compute_idle(
812 dqm->dev->kgd, true);
64c7f8cf
BG		 813 goto out;
814 }
815 }
816 /* qpd not found in dqm list */
817 retval = 1;
818out:
efeaed4d 819 dqm_unlock(dqm);
64c7f8cf
BG		 820 return retval;
821}
822
823static int
824set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid,
825 unsigned int vmid)
826{
827 uint32_t pasid_mapping;
828
cea405b1
XZ		 829 pasid_mapping = (pasid == 0) ? 0 :
830 (uint32_t)pasid |
831 ATC_VMID_PASID_MAPPING_VALID;
832
833 return dqm->dev->kfd2kgd->set_pasid_vmid_mapping(
834 dqm->dev->kgd, pasid_mapping,
64c7f8cf
BG		 835 vmid);
836}
837
2249d558
AL		 838static void init_interrupts(struct device_queue_manager *dqm)
839{
840 unsigned int i;
841
d0b63bb3
AR		 842 for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++)
843 if (is_pipe_enabled(dqm, 0, i))
844 dqm->dev->kfd2kgd->init_interrupts(dqm->dev->kgd, i);
2249d558
AL		 845}
846
64c7f8cf
BG		 847static int initialize_nocpsch(struct device_queue_manager *dqm)
848{
86194cf8 849 int pipe, queue;
64c7f8cf 850
79775b62 851 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
64c7f8cf 852
ab7c1648
KR		 853 dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm),
854 sizeof(unsigned int), GFP_KERNEL);
855 if (!dqm->allocated_queues)
856 return -ENOMEM;
857
efeaed4d 858 mutex_init(&dqm->lock_hidden);
64c7f8cf
BG		 859 INIT_LIST_HEAD(&dqm->queues);
860 dqm->queue_count = dqm->next_pipe_to_allocate = 0;
bcea3081 861 dqm->sdma_queue_count = 0;
64c7f8cf 862
86194cf8
FK		 863 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
864 int pipe_offset = pipe * get_queues_per_pipe(dqm);
865
866 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++)
867 if (test_bit(pipe_offset + queue,
868 dqm->dev->shared_resources.queue_bitmap))
869 dqm->allocated_queues[pipe] |= 1 << queue;
870 }
64c7f8cf 871
44008d7a 872 dqm->vmid_bitmap = (1 << dqm->dev->vm_info.vmid_num_kfd) - 1;
98bb9222 873 dqm->sdma_bitmap = (1 << get_num_sdma_queues(dqm)) - 1;
64c7f8cf 874
64c7f8cf
BG		 875 return 0;
876}
877
58dcd5bf 878static void uninitialize(struct device_queue_manager *dqm)
64c7f8cf 879{
6f9d54fd
OG		 880 int i;
881
32fa8219 882 WARN_ON(dqm->queue_count > 0 || dqm->processes_count > 0);
64c7f8cf
BG		 883
884 kfree(dqm->allocated_queues);
6f9d54fd 885 for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
8d5f3552 886 kfree(dqm->mqd_mgrs[i]);
efeaed4d 887 mutex_destroy(&dqm->lock_hidden);
a86aa3ca 888 kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem);
64c7f8cf
BG		 889}
890
891static int start_nocpsch(struct device_queue_manager *dqm)
892{
2249d558 893 init_interrupts(dqm);
552764b6 894 return pm_init(&dqm->packets, dqm);
64c7f8cf
BG		 895}
896
897static int stop_nocpsch(struct device_queue_manager *dqm)
898{
552764b6 899 pm_uninit(&dqm->packets);
64c7f8cf
BG		 900 return 0;
901}
902
bcea3081
BG		 903static int allocate_sdma_queue(struct device_queue_manager *dqm,
904 unsigned int *sdma_queue_id)
905{
906 int bit;
907
908 if (dqm->sdma_bitmap == 0)
909 return -ENOMEM;
910
4252bf68
HK		 911 bit = ffs(dqm->sdma_bitmap) - 1;
912 dqm->sdma_bitmap &= ~(1 << bit);
bcea3081
BG		 913 *sdma_queue_id = bit;
914
915 return 0;
916}
917
918static void deallocate_sdma_queue(struct device_queue_manager *dqm,
919 unsigned int sdma_queue_id)
920{
98bb9222 921 if (sdma_queue_id >= get_num_sdma_queues(dqm))
bcea3081 922 return;
4252bf68 923 dqm->sdma_bitmap |= (1 << sdma_queue_id);
bcea3081
BG		 924}
925
bcea3081
BG		 926static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
927 struct queue *q,
928 struct qcm_process_device *qpd)
929{
8d5f3552 930 struct mqd_manager *mqd_mgr;
bcea3081
BG		 931 int retval;
932
8d5f3552
YZ		 933 mqd_mgr = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA);
934 if (!mqd_mgr)
bcea3081
BG		 935 return -ENOMEM;
936
937 retval = allocate_sdma_queue(dqm, &q->sdma_id);
4eacc26b 938 if (retval)
bcea3081
BG		 939 return retval;
940
98bb9222
YZ		 941 q->properties.sdma_queue_id = q->sdma_id / get_num_sdma_engines(dqm);
942 q->properties.sdma_engine_id = q->sdma_id % get_num_sdma_engines(dqm);
bcea3081 943
ef568db7
FK		 944 retval = allocate_doorbell(qpd, q);
945 if (retval)
946 goto out_deallocate_sdma_queue;
947
79775b62
KR		 948 pr_debug("SDMA id is: %d\n", q->sdma_id);
949 pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id);
950 pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
bcea3081 951
bfd5e378 952 dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
8d5f3552 953 retval = mqd_mgr->init_mqd(mqd_mgr, &q->mqd, &q->mqd_mem_obj,
bcea3081 954 &q->gart_mqd_addr, &q->properties);
ab7c1648 955 if (retval)
ef568db7 956 goto out_deallocate_doorbell;
bcea3081 957
8d5f3552
YZ		 958 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, 0, 0, &q->properties,
959 NULL);
ab7c1648
KR		 960 if (retval)
961 goto out_uninit_mqd;
4fadf6b6 962
bcea3081 963 return 0;
ab7c1648
KR		 964
965out_uninit_mqd:
8d5f3552 966 mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
ef568db7
FK		 967out_deallocate_doorbell:
968 deallocate_doorbell(qpd, q);
ab7c1648
KR		 969out_deallocate_sdma_queue:
970 deallocate_sdma_queue(dqm, q->sdma_id);
971
972 return retval;
bcea3081
BG		 973}
974
64c7f8cf
BG		 975/*
976 * Device Queue Manager implementation for cp scheduler
977 */
978
979static int set_sched_resources(struct device_queue_manager *dqm)
980{
d0b63bb3 981 int i, mec;
64c7f8cf 982 struct scheduling_resources res;
64c7f8cf 983
44008d7a 984 res.vmid_mask = dqm->dev->shared_resources.compute_vmid_bitmap;
d0b63bb3
AR		 985
986 res.queue_mask = 0;
987 for (i = 0; i < KGD_MAX_QUEUES; ++i) {
988 mec = (i / dqm->dev->shared_resources.num_queue_per_pipe)
989 / dqm->dev->shared_resources.num_pipe_per_mec;
990
991 if (!test_bit(i, dqm->dev->shared_resources.queue_bitmap))
992 continue;
993
994 /* only acquire queues from the first MEC */
995 if (mec > 0)
996 continue;
997
998 /* This situation may be hit in the future if a new HW
999 * generation exposes more than 64 queues. If so, the
8eabaf54
KR		 1000 * definition of res.queue_mask needs updating
1001 */
1d11ee89 1002 if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) {
d0b63bb3
AR		 1003 pr_err("Invalid queue enabled by amdgpu: %d\n", i);
1004 break;
1005 }
1006
1007 res.queue_mask |= (1ull << i);
1008 }
64c7f8cf
BG		 1009 res.gws_mask = res.oac_mask = res.gds_heap_base =
1010 res.gds_heap_size = 0;
1011
79775b62
KR		 1012 pr_debug("Scheduling resources:\n"
1013 "vmid mask: 0x%8X\n"
1014 "queue mask: 0x%8llX\n",
64c7f8cf
BG		 1015 res.vmid_mask, res.queue_mask);
1016
1017 return pm_send_set_resources(&dqm->packets, &res);
1018}
1019
1020static int initialize_cpsch(struct device_queue_manager *dqm)
1021{
79775b62 1022 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
64c7f8cf 1023
efeaed4d 1024 mutex_init(&dqm->lock_hidden);
64c7f8cf
BG		 1025 INIT_LIST_HEAD(&dqm->queues);
1026 dqm->queue_count = dqm->processes_count = 0;
bcea3081 1027 dqm->sdma_queue_count = 0;
64c7f8cf 1028 dqm->active_runlist = false;
98bb9222 1029 dqm->sdma_bitmap = (1 << get_num_sdma_queues(dqm)) - 1;
64c7f8cf 1030
73ea648d
SL		 1031 INIT_WORK(&dqm->hw_exception_work, kfd_process_hw_exception);
1032
bfd5e378 1033 return 0;
64c7f8cf
BG		 1034}
1035
1036static int start_cpsch(struct device_queue_manager *dqm)
1037{
64c7f8cf
BG		 1038 int retval;
1039
64c7f8cf
BG		 1040 retval = 0;
1041
1042 retval = pm_init(&dqm->packets, dqm);
4eacc26b 1043 if (retval)
64c7f8cf
BG		 1044 goto fail_packet_manager_init;
1045
1046 retval = set_sched_resources(dqm);
4eacc26b 1047 if (retval)
64c7f8cf
BG		 1048 goto fail_set_sched_resources;
1049
79775b62 1050 pr_debug("Allocating fence memory\n");
64c7f8cf
BG		 1051
1052 /* allocate fence memory on the gart */
a86aa3ca
OG		 1053 retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr),
1054 &dqm->fence_mem);
64c7f8cf 1055
4eacc26b 1056 if (retval)
64c7f8cf
BG		 1057 goto fail_allocate_vidmem;
1058
1059 dqm->fence_addr = dqm->fence_mem->cpu_ptr;
1060 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr;
2249d558
AL		 1061
1062 init_interrupts(dqm);
1063
efeaed4d 1064 dqm_lock(dqm);
73ea648d
SL		 1065 /* clear hang status when driver try to start the hw scheduler */
1066 dqm->is_hws_hang = false;
c4744e24 1067 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
efeaed4d 1068 dqm_unlock(dqm);
64c7f8cf
BG		 1069
1070 return 0;
1071fail_allocate_vidmem:
1072fail_set_sched_resources:
1073 pm_uninit(&dqm->packets);
1074fail_packet_manager_init:
1075 return retval;
1076}
1077
1078static int stop_cpsch(struct device_queue_manager *dqm)
1079{
efeaed4d 1080 dqm_lock(dqm);
4465f466 1081 unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
efeaed4d 1082 dqm_unlock(dqm);
64c7f8cf 1083
a86aa3ca 1084 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
64c7f8cf
BG		 1085 pm_uninit(&dqm->packets);
1086
1087 return 0;
1088}
1089
1090static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
1091 struct kernel_queue *kq,
1092 struct qcm_process_device *qpd)
1093{
efeaed4d 1094 dqm_lock(dqm);
b8cbab04 1095 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
79775b62 1096 pr_warn("Can't create new kernel queue because %d queues were already created\n",
b8cbab04 1097 dqm->total_queue_count);
efeaed4d 1098 dqm_unlock(dqm);
b8cbab04
OG		 1099 return -EPERM;
1100 }
1101
1102 /*
1103 * Unconditionally increment this counter, regardless of the queue's
1104 * type or whether the queue is active.
1105 */
1106 dqm->total_queue_count++;
1107 pr_debug("Total of %d queues are accountable so far\n",
1108 dqm->total_queue_count);
1109
64c7f8cf
BG		 1110 list_add(&kq->list, &qpd->priv_queue_list);
1111 dqm->queue_count++;
1112 qpd->is_debug = true;
c4744e24 1113 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
efeaed4d 1114 dqm_unlock(dqm);
64c7f8cf
BG		 1115
1116 return 0;
1117}
1118
1119static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
1120 struct kernel_queue *kq,
1121 struct qcm_process_device *qpd)
1122{
efeaed4d 1123 dqm_lock(dqm);
64c7f8cf
BG		 1124 list_del(&kq->list);
1125 dqm->queue_count--;
1126 qpd->is_debug = false;
c4744e24 1127 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
b8cbab04
OG		 1128 /*
1129 * Unconditionally decrement this counter, regardless of the queue's
1130 * type.
1131 */
8b58f261 1132 dqm->total_queue_count--;
b8cbab04
OG		 1133 pr_debug("Total of %d queues are accountable so far\n",
1134 dqm->total_queue_count);
efeaed4d 1135 dqm_unlock(dqm);
64c7f8cf
BG		 1136}
1137
1138static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
b46cb7d7 1139 struct qcm_process_device *qpd)
64c7f8cf
BG		 1140{
1141 int retval;
8d5f3552 1142 struct mqd_manager *mqd_mgr;
64c7f8cf 1143
64c7f8cf
BG		 1144 retval = 0;
1145
efeaed4d 1146 dqm_lock(dqm);
64c7f8cf 1147
b8cbab04 1148 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
79775b62 1149 pr_warn("Can't create new usermode queue because %d queues were already created\n",
b8cbab04
OG		 1150 dqm->total_queue_count);
1151 retval = -EPERM;
72a01d23 1152 goto out_unlock;
b8cbab04
OG		 1153 }
1154
e139cd2a 1155 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
1156 retval = allocate_sdma_queue(dqm, &q->sdma_id);
894a8293 1157 if (retval)
72a01d23 1158 goto out_unlock;
e139cd2a 1159 q->properties.sdma_queue_id =
98bb9222 1160 q->sdma_id / get_num_sdma_engines(dqm);
e139cd2a 1161 q->properties.sdma_engine_id =
98bb9222 1162 q->sdma_id % get_num_sdma_engines(dqm);
e139cd2a 1163 }
ef568db7
FK		 1164
1165 retval = allocate_doorbell(qpd, q);
1166 if (retval)
1167 goto out_deallocate_sdma_queue;
1168
8d5f3552 1169 mqd_mgr = dqm->ops.get_mqd_manager(dqm,
bcea3081
BG		 1170 get_mqd_type_from_queue_type(q->properties.type));
1171
8d5f3552 1172 if (!mqd_mgr) {
ab7c1648 1173 retval = -ENOMEM;
ef568db7 1174 goto out_deallocate_doorbell;
64c7f8cf 1175 }
26103436
FK		 1176 /*
1177 * Eviction state logic: we only mark active queues as evicted
1178 * to avoid the overhead of restoring inactive queues later
1179 */
1180 if (qpd->evicted)
1181 q->properties.is_evicted = (q->properties.queue_size > 0 &&
1182 q->properties.queue_percent > 0 &&
1183 q->properties.queue_address != 0);
64c7f8cf 1184
bfd5e378 1185 dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
373d7080
FK		 1186
1187 q->properties.tba_addr = qpd->tba_addr;
1188 q->properties.tma_addr = qpd->tma_addr;
8d5f3552 1189 retval = mqd_mgr->init_mqd(mqd_mgr, &q->mqd, &q->mqd_mem_obj,
64c7f8cf 1190 &q->gart_mqd_addr, &q->properties);
4eacc26b 1191 if (retval)
ef568db7 1192 goto out_deallocate_doorbell;
64c7f8cf
BG		 1193
1194 list_add(&q->list, &qpd->queues_list);
bc920fd4 1195 qpd->queue_count++;
64c7f8cf
BG		 1196 if (q->properties.is_active) {
1197 dqm->queue_count++;
c4744e24
YZ		 1198 retval = execute_queues_cpsch(dqm,
1199 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
64c7f8cf
BG		 1200 }
1201
bcea3081 1202 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
8eabaf54 1203 dqm->sdma_queue_count++;
b8cbab04
OG		 1204 /*
1205 * Unconditionally increment this counter, regardless of the queue's
1206 * type or whether the queue is active.
1207 */
1208 dqm->total_queue_count++;
1209
1210 pr_debug("Total of %d queues are accountable so far\n",
1211 dqm->total_queue_count);
1212
efeaed4d 1213 dqm_unlock(dqm);
72a01d23
FK		 1214 return retval;
1215
ef568db7
FK		 1216out_deallocate_doorbell:
1217 deallocate_doorbell(qpd, q);
72a01d23
FK		 1218out_deallocate_sdma_queue:
1219 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
1220 deallocate_sdma_queue(dqm, q->sdma_id);
1221out_unlock:
efeaed4d
FK		 1222 dqm_unlock(dqm);
1223
64c7f8cf
BG		 1224 return retval;
1225}
1226
788bf83d 1227int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
d80d19bd 1228 unsigned int fence_value,
8c72c3d7 1229 unsigned int timeout_ms)
64c7f8cf 1230{
8c72c3d7 1231 unsigned long end_jiffies = msecs_to_jiffies(timeout_ms) + jiffies;
64c7f8cf
BG		 1232
1233 while (*fence_addr != fence_value) {
8c72c3d7 1234 if (time_after(jiffies, end_jiffies)) {
79775b62 1235 pr_err("qcm fence wait loop timeout expired\n");
0e9a860c
YZ		 1236 /* In HWS case, this is used to halt the driver thread
1237 * in order not to mess up CP states before doing
1238 * scandumps for FW debugging.
1239 */
1240 while (halt_if_hws_hang)
1241 schedule();
1242
64c7f8cf
BG		 1243 return -ETIME;
1244 }
99331a51 1245 schedule();
64c7f8cf
BG		 1246 }
1247
1248 return 0;
1249}
1250
7da2bcf8 1251static int unmap_sdma_queues(struct device_queue_manager *dqm,
bcea3081
BG		 1252 unsigned int sdma_engine)
1253{
1254 return pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_SDMA,
7da2bcf8 1255 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, false,
bcea3081
BG		 1256 sdma_engine);
1257}
1258
60a00956
FK		 1259/* dqm->lock mutex has to be locked before calling this function */
1260static int map_queues_cpsch(struct device_queue_manager *dqm)
1261{
1262 int retval;
1263
1264 if (dqm->queue_count <= 0 || dqm->processes_count <= 0)
1265 return 0;
1266
1267 if (dqm->active_runlist)
1268 return 0;
1269
1270 retval = pm_send_runlist(&dqm->packets, &dqm->queues);
1271 if (retval) {
1272 pr_err("failed to execute runlist\n");
1273 return retval;
1274 }
1275 dqm->active_runlist = true;
1276
1277 return retval;
1278}
1279
ac30c783 1280/* dqm->lock mutex has to be locked before calling this function */
7da2bcf8 1281static int unmap_queues_cpsch(struct device_queue_manager *dqm,
4465f466
YZ		 1282 enum kfd_unmap_queues_filter filter,
1283 uint32_t filter_param)
64c7f8cf 1284{
9fd3f1bf 1285 int retval = 0;
64c7f8cf 1286
73ea648d
SL		 1287 if (dqm->is_hws_hang)
1288 return -EIO;
991ca8ee 1289 if (!dqm->active_runlist)
ac30c783 1290 return retval;
bcea3081 1291
79775b62 1292 pr_debug("Before destroying queues, sdma queue count is : %u\n",
bcea3081
BG		 1293 dqm->sdma_queue_count);
1294
1295 if (dqm->sdma_queue_count > 0) {
7da2bcf8
YZ		 1296 unmap_sdma_queues(dqm, 0);
1297 unmap_sdma_queues(dqm, 1);
bcea3081
BG		 1298 }
1299
64c7f8cf 1300 retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE,
4465f466 1301 filter, filter_param, false, 0);
4eacc26b 1302 if (retval)
ac30c783 1303 return retval;
64c7f8cf
BG		 1304
1305 *dqm->fence_addr = KFD_FENCE_INIT;
1306 pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr,
1307 KFD_FENCE_COMPLETED);
1308 /* should be timed out */
c3447e81 1309 retval = amdkfd_fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED,
64c7f8cf 1310 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS);
9fd3f1bf 1311 if (retval)
ac30c783 1312 return retval;
9fd3f1bf 1313
64c7f8cf
BG		 1314 pm_release_ib(&dqm->packets);
1315 dqm->active_runlist = false;
1316
64c7f8cf
BG		 1317 return retval;
1318}
1319
ac30c783 1320/* dqm->lock mutex has to be locked before calling this function */
c4744e24
YZ		 1321static int execute_queues_cpsch(struct device_queue_manager *dqm,
1322 enum kfd_unmap_queues_filter filter,
1323 uint32_t filter_param)
64c7f8cf
BG		 1324{
1325 int retval;
1326
73ea648d
SL		 1327 if (dqm->is_hws_hang)
1328 return -EIO;
c4744e24 1329 retval = unmap_queues_cpsch(dqm, filter, filter_param);
4eacc26b 1330 if (retval) {
c4744e24 1331 pr_err("The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n");
73ea648d
SL		 1332 dqm->is_hws_hang = true;
1333 schedule_work(&dqm->hw_exception_work);
ac30c783 1334 return retval;
64c7f8cf
BG		 1335 }
1336
60a00956 1337 return map_queues_cpsch(dqm);
64c7f8cf
BG		 1338}
1339
1340static int destroy_queue_cpsch(struct device_queue_manager *dqm,
1341 struct qcm_process_device *qpd,
1342 struct queue *q)
1343{
1344 int retval;
8d5f3552 1345 struct mqd_manager *mqd_mgr;
992839ad 1346 bool preempt_all_queues;
64c7f8cf 1347
992839ad
YS		 1348 preempt_all_queues = false;
1349
64c7f8cf
BG		 1350 retval = 0;
1351
1352 /* remove queue from list to prevent rescheduling after preemption */
efeaed4d 1353 dqm_lock(dqm);
992839ad
YS		 1354
1355 if (qpd->is_debug) {
1356 /*
1357 * error, currently we do not allow to destroy a queue
1358 * of a currently debugged process
1359 */
1360 retval = -EBUSY;
1361 goto failed_try_destroy_debugged_queue;
1362
1363 }
1364
8d5f3552 1365 mqd_mgr = dqm->ops.get_mqd_manager(dqm,
bcea3081 1366 get_mqd_type_from_queue_type(q->properties.type));
8d5f3552 1367 if (!mqd_mgr) {
64c7f8cf
BG		 1368 retval = -ENOMEM;
1369 goto failed;
1370 }
1371
ef568db7
FK		 1372 deallocate_doorbell(qpd, q);
1373
e139cd2a 1374 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
bcea3081 1375 dqm->sdma_queue_count--;
e139cd2a 1376 deallocate_sdma_queue(dqm, q->sdma_id);
1377 }
bcea3081 1378
64c7f8cf 1379 list_del(&q->list);
bc920fd4 1380 qpd->queue_count--;
40a526dc 1381 if (q->properties.is_active) {
b6819cec 1382 dqm->queue_count--;
40a526dc 1383 retval = execute_queues_cpsch(dqm,
9fd3f1bf 1384 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
40a526dc
YZ		 1385 if (retval == -ETIME)
1386 qpd->reset_wavefronts = true;
1387 }
64c7f8cf 1388
8d5f3552 1389 mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
b8cbab04
OG		 1390
1391 /*
1392 * Unconditionally decrement this counter, regardless of the queue's
1393 * type
1394 */
1395 dqm->total_queue_count--;
1396 pr_debug("Total of %d queues are accountable so far\n",
1397 dqm->total_queue_count);
64c7f8cf 1398
efeaed4d 1399 dqm_unlock(dqm);
64c7f8cf 1400
9e827224 1401 return retval;
64c7f8cf
BG		 1402
1403failed:
992839ad
YS		 1404failed_try_destroy_debugged_queue:
1405
efeaed4d 1406 dqm_unlock(dqm);
64c7f8cf
BG		 1407 return retval;
1408}
1409
1410/*
1411 * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to
1412 * stay in user mode.
1413 */
1414#define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL
1415/* APE1 limit is inclusive and 64K aligned. */
1416#define APE1_LIMIT_ALIGNMENT 0xFFFF
1417
1418static bool set_cache_memory_policy(struct device_queue_manager *dqm,
1419 struct qcm_process_device *qpd,
1420 enum cache_policy default_policy,
1421 enum cache_policy alternate_policy,
1422 void __user *alternate_aperture_base,
1423 uint64_t alternate_aperture_size)
1424{
bed4f110
FK		 1425 bool retval = true;
1426
1427 if (!dqm->asic_ops.set_cache_memory_policy)
1428 return retval;
64c7f8cf 1429
efeaed4d 1430 dqm_lock(dqm);
64c7f8cf
BG		 1431
1432 if (alternate_aperture_size == 0) {
1433 /* base > limit disables APE1 */
1434 qpd->sh_mem_ape1_base = 1;
1435 qpd->sh_mem_ape1_limit = 0;
1436 } else {
1437 /*
1438 * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]},
1439 * SH_MEM_APE1_BASE[31:0], 0x0000 }
1440 * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]},
1441 * SH_MEM_APE1_LIMIT[31:0], 0xFFFF }
1442 * Verify that the base and size parameters can be
1443 * represented in this format and convert them.
1444 * Additionally restrict APE1 to user-mode addresses.
1445 */
1446
1447 uint64_t base = (uintptr_t)alternate_aperture_base;
1448 uint64_t limit = base + alternate_aperture_size - 1;
1449
ab7c1648
KR		 1450 if (limit <= base || (base & APE1_FIXED_BITS_MASK) != 0 ||
1451 (limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT) {
1452 retval = false;
64c7f8cf 1453 goto out;
ab7c1648 1454 }
64c7f8cf
BG		 1455
1456 qpd->sh_mem_ape1_base = base >> 16;
1457 qpd->sh_mem_ape1_limit = limit >> 16;
1458 }
1459
bfd5e378 1460 retval = dqm->asic_ops.set_cache_memory_policy(
a22fc854
BG		 1461 dqm,
1462 qpd,
1463 default_policy,
1464 alternate_policy,
1465 alternate_aperture_base,
1466 alternate_aperture_size);
64c7f8cf 1467
d146c5a7 1468 if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
64c7f8cf
BG		 1469 program_sh_mem_settings(dqm, qpd);
1470
79775b62 1471 pr_debug("sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
64c7f8cf
BG		 1472 qpd->sh_mem_config, qpd->sh_mem_ape1_base,
1473 qpd->sh_mem_ape1_limit);
1474
64c7f8cf 1475out:
efeaed4d 1476 dqm_unlock(dqm);
ab7c1648 1477 return retval;
64c7f8cf
BG		 1478}
1479
d7b9bd22
FK		 1480static int set_trap_handler(struct device_queue_manager *dqm,
1481 struct qcm_process_device *qpd,
1482 uint64_t tba_addr,
1483 uint64_t tma_addr)
1484{
1485 uint64_t *tma;
1486
1487 if (dqm->dev->cwsr_enabled) {
1488 /* Jump from CWSR trap handler to user trap */
1489 tma = (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
1490 tma[0] = tba_addr;
1491 tma[1] = tma_addr;
1492 } else {
1493 qpd->tba_addr = tba_addr;
1494 qpd->tma_addr = tma_addr;
1495 }
1496
1497 return 0;
1498}
1499
9fd3f1bf
FK		 1500static int process_termination_nocpsch(struct device_queue_manager *dqm,
1501 struct qcm_process_device *qpd)
1502{
1503 struct queue *q, *next;
1504 struct device_process_node *cur, *next_dpn;
1505 int retval = 0;
1506
efeaed4d 1507 dqm_lock(dqm);
9fd3f1bf
FK		 1508
1509 /* Clear all user mode queues */
1510 list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
1511 int ret;
1512
1513 ret = destroy_queue_nocpsch_locked(dqm, qpd, q);
1514 if (ret)
1515 retval = ret;
1516 }
1517
1518 /* Unregister process */
1519 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
1520 if (qpd == cur->qpd) {
1521 list_del(&cur->list);
1522 kfree(cur);
1523 dqm->processes_count--;
1524 break;
1525 }
1526 }
1527
efeaed4d 1528 dqm_unlock(dqm);
9fd3f1bf
FK		 1529 return retval;
1530}
1531
5df099e8
JC		 1532static int get_wave_state(struct device_queue_manager *dqm,
1533 struct queue *q,
1534 void __user *ctl_stack,
1535 u32 *ctl_stack_used_size,
1536 u32 *save_area_used_size)
1537{
1538 struct mqd_manager *mqd;
1539 int r;
1540
1541 dqm_lock(dqm);
1542
1543 if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE ||
1544 q->properties.is_active || !q->device->cwsr_enabled) {
1545 r = -EINVAL;
1546 goto dqm_unlock;
1547 }
1548
1549 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
1550 if (!mqd) {
1551 r = -ENOMEM;
1552 goto dqm_unlock;
1553 }
1554
1555 if (!mqd->get_wave_state) {
1556 r = -EINVAL;
1557 goto dqm_unlock;
1558 }
1559
1560 r = mqd->get_wave_state(mqd, q->mqd, ctl_stack, ctl_stack_used_size,
1561 save_area_used_size);
1562
1563dqm_unlock:
1564 dqm_unlock(dqm);
1565 return r;
1566}
9fd3f1bf
FK		 1567
1568static int process_termination_cpsch(struct device_queue_manager *dqm,
1569 struct qcm_process_device *qpd)
1570{
1571 int retval;
1572 struct queue *q, *next;
1573 struct kernel_queue *kq, *kq_next;
8d5f3552 1574 struct mqd_manager *mqd_mgr;
9fd3f1bf
FK		 1575 struct device_process_node *cur, *next_dpn;
1576 enum kfd_unmap_queues_filter filter =
1577 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES;
1578
1579 retval = 0;
1580
efeaed4d 1581 dqm_lock(dqm);
9fd3f1bf
FK		 1582
1583 /* Clean all kernel queues */
1584 list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) {
1585 list_del(&kq->list);
1586 dqm->queue_count--;
1587 qpd->is_debug = false;
1588 dqm->total_queue_count--;
1589 filter = KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES;
1590 }
1591
1592 /* Clear all user mode queues */
1593 list_for_each_entry(q, &qpd->queues_list, list) {
72a01d23 1594 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
9fd3f1bf 1595 dqm->sdma_queue_count--;
72a01d23
FK		 1596 deallocate_sdma_queue(dqm, q->sdma_id);
1597 }
9fd3f1bf
FK		 1598
1599 if (q->properties.is_active)
1600 dqm->queue_count--;
1601
1602 dqm->total_queue_count--;
1603 }
1604
1605 /* Unregister process */
1606 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
1607 if (qpd == cur->qpd) {
1608 list_del(&cur->list);
1609 kfree(cur);
1610 dqm->processes_count--;
1611 break;
1612 }
1613 }
1614
1615 retval = execute_queues_cpsch(dqm, filter, 0);
73ea648d 1616 if ((!dqm->is_hws_hang) && (retval || qpd->reset_wavefronts)) {
9fd3f1bf
FK		 1617 pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev);
1618 dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process);
1619 qpd->reset_wavefronts = false;
1620 }
1621
1622 /* lastly, free mqd resources */
1623 list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
8d5f3552 1624 mqd_mgr = dqm->ops.get_mqd_manager(dqm,
9fd3f1bf 1625 get_mqd_type_from_queue_type(q->properties.type));
8d5f3552 1626 if (!mqd_mgr) {
9fd3f1bf
FK		 1627 retval = -ENOMEM;
1628 goto out;
1629 }
1630 list_del(&q->list);
bc920fd4 1631 qpd->queue_count--;
8d5f3552 1632 mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
9fd3f1bf
FK		 1633 }
1634
1635out:
efeaed4d 1636 dqm_unlock(dqm);
9fd3f1bf
FK		 1637 return retval;
1638}
1639
64c7f8cf
BG		 1640struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
1641{
1642 struct device_queue_manager *dqm;
1643
79775b62 1644 pr_debug("Loading device queue manager\n");
a22fc854 1645
dbf56ab1 1646 dqm = kzalloc(sizeof(*dqm), GFP_KERNEL);
64c7f8cf
BG		 1647 if (!dqm)
1648 return NULL;
1649
d146c5a7
FK		 1650 switch (dev->device_info->asic_family) {
1651 /* HWS is not available on Hawaii. */
1652 case CHIP_HAWAII:
1653 /* HWS depends on CWSR for timely dequeue. CWSR is not
1654 * available on Tonga.
1655 *
1656 * FIXME: This argument also applies to Kaveri.
1657 */
1658 case CHIP_TONGA:
1659 dqm->sched_policy = KFD_SCHED_POLICY_NO_HWS;
1660 break;
1661 default:
1662 dqm->sched_policy = sched_policy;
1663 break;
1664 }
1665
64c7f8cf 1666 dqm->dev = dev;
d146c5a7 1667 switch (dqm->sched_policy) {
64c7f8cf
BG		 1668 case KFD_SCHED_POLICY_HWS:
1669 case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION:
1670 /* initialize dqm for cp scheduling */
45c9a5e4
OG		 1671 dqm->ops.create_queue = create_queue_cpsch;
1672 dqm->ops.initialize = initialize_cpsch;
1673 dqm->ops.start = start_cpsch;
1674 dqm->ops.stop = stop_cpsch;
1675 dqm->ops.destroy_queue = destroy_queue_cpsch;
1676 dqm->ops.update_queue = update_queue;
58dcd5bf
YZ		 1677 dqm->ops.get_mqd_manager = get_mqd_manager;
1678 dqm->ops.register_process = register_process;
1679 dqm->ops.unregister_process = unregister_process;
1680 dqm->ops.uninitialize = uninitialize;
45c9a5e4
OG		 1681 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch;
1682 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch;
1683 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
d7b9bd22 1684 dqm->ops.set_trap_handler = set_trap_handler;
9fd3f1bf 1685 dqm->ops.process_termination = process_termination_cpsch;
26103436
FK		 1686 dqm->ops.evict_process_queues = evict_process_queues_cpsch;
1687 dqm->ops.restore_process_queues = restore_process_queues_cpsch;
5df099e8 1688 dqm->ops.get_wave_state = get_wave_state;
64c7f8cf
BG		 1689 break;
1690 case KFD_SCHED_POLICY_NO_HWS:
1691 /* initialize dqm for no cp scheduling */
45c9a5e4
OG		 1692 dqm->ops.start = start_nocpsch;
1693 dqm->ops.stop = stop_nocpsch;
1694 dqm->ops.create_queue = create_queue_nocpsch;
1695 dqm->ops.destroy_queue = destroy_queue_nocpsch;
1696 dqm->ops.update_queue = update_queue;
58dcd5bf
YZ		 1697 dqm->ops.get_mqd_manager = get_mqd_manager;
1698 dqm->ops.register_process = register_process;
1699 dqm->ops.unregister_process = unregister_process;
45c9a5e4 1700 dqm->ops.initialize = initialize_nocpsch;
58dcd5bf 1701 dqm->ops.uninitialize = uninitialize;
45c9a5e4 1702 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
d7b9bd22 1703 dqm->ops.set_trap_handler = set_trap_handler;
9fd3f1bf 1704 dqm->ops.process_termination = process_termination_nocpsch;
26103436
FK		 1705 dqm->ops.evict_process_queues = evict_process_queues_nocpsch;
1706 dqm->ops.restore_process_queues =
1707 restore_process_queues_nocpsch;
5df099e8 1708 dqm->ops.get_wave_state = get_wave_state;
64c7f8cf
BG		 1709 break;
1710 default:
d146c5a7 1711 pr_err("Invalid scheduling policy %d\n", dqm->sched_policy);
32fa8219 1712 goto out_free;
64c7f8cf
BG		 1713 }
1714
a22fc854
BG		 1715 switch (dev->device_info->asic_family) {
1716 case CHIP_CARRIZO:
bfd5e378 1717 device_queue_manager_init_vi(&dqm->asic_ops);
300dec95
OG		 1718 break;
1719
a22fc854 1720 case CHIP_KAVERI:
bfd5e378 1721 device_queue_manager_init_cik(&dqm->asic_ops);
300dec95 1722 break;
97672cbe
FK		 1723
1724 case CHIP_HAWAII:
1725 device_queue_manager_init_cik_hawaii(&dqm->asic_ops);
1726 break;
1727
1728 case CHIP_TONGA:
1729 case CHIP_FIJI:
1730 case CHIP_POLARIS10:
1731 case CHIP_POLARIS11:
1732 device_queue_manager_init_vi_tonga(&dqm->asic_ops);
1733 break;
bed4f110
FK		 1734
1735 case CHIP_VEGA10:
1736 case CHIP_RAVEN:
1737 device_queue_manager_init_v9(&dqm->asic_ops);
1738 break;
e596b903
YZ		 1739 default:
1740 WARN(1, "Unexpected ASIC family %u",
1741 dev->device_info->asic_family);
1742 goto out_free;
a22fc854
BG		 1743 }
1744
32fa8219
FK		 1745 if (!dqm->ops.initialize(dqm))
1746 return dqm;
64c7f8cf 1747
32fa8219
FK		 1748out_free:
1749 kfree(dqm);
1750 return NULL;
64c7f8cf
BG		 1751}
1752
1753void device_queue_manager_uninit(struct device_queue_manager *dqm)
1754{
45c9a5e4 1755 dqm->ops.uninitialize(dqm);
64c7f8cf
BG		 1756 kfree(dqm);
1757}
851a645e 1758
2640c3fa 1759int kfd_process_vm_fault(struct device_queue_manager *dqm,
1760 unsigned int pasid)
1761{
1762 struct kfd_process_device *pdd;
1763 struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
1764 int ret = 0;
1765
1766 if (!p)
1767 return -EINVAL;
1768 pdd = kfd_get_process_device_data(dqm->dev, p);
1769 if (pdd)
1770 ret = dqm->ops.evict_process_queues(dqm, &pdd->qpd);
1771 kfd_unref_process(p);
1772
1773 return ret;
1774}
1775
73ea648d
SL		 1776static void kfd_process_hw_exception(struct work_struct *work)
1777{
1778 struct device_queue_manager *dqm = container_of(work,
1779 struct device_queue_manager, hw_exception_work);
1780 dqm->dev->kfd2kgd->gpu_recover(dqm->dev->kgd);
1781}
1782
851a645e
FK		 1783#if defined(CONFIG_DEBUG_FS)
1784
1785static void seq_reg_dump(struct seq_file *m,
1786 uint32_t (*dump)[2], uint32_t n_regs)
1787{
1788 uint32_t i, count;
1789
1790 for (i = 0, count = 0; i < n_regs; i++) {
1791 if (count == 0 ||
1792 dump[i-1][0] + sizeof(uint32_t) != dump[i][0]) {
1793 seq_printf(m, "%s %08x: %08x",
1794 i ? "\n" : "",
1795 dump[i][0], dump[i][1]);
1796 count = 7;
1797 } else {
1798 seq_printf(m, " %08x", dump[i][1]);
1799 count--;
1800 }
1801 }
1802
1803 seq_puts(m, "\n");
1804}
1805
1806int dqm_debugfs_hqds(struct seq_file *m, void *data)
1807{
1808 struct device_queue_manager *dqm = data;
1809 uint32_t (*dump)[2], n_regs;
1810 int pipe, queue;
1811 int r = 0;
1812
24f48a42
OZ		 1813 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->kgd,
1814 KFD_CIK_HIQ_PIPE, KFD_CIK_HIQ_QUEUE, &dump, &n_regs);
1815 if (!r) {
1816 seq_printf(m, " HIQ on MEC %d Pipe %d Queue %d\n",
1817 KFD_CIK_HIQ_PIPE/get_pipes_per_mec(dqm)+1,
1818 KFD_CIK_HIQ_PIPE%get_pipes_per_mec(dqm),
1819 KFD_CIK_HIQ_QUEUE);
1820 seq_reg_dump(m, dump, n_regs);
1821
1822 kfree(dump);
1823 }
1824
851a645e
FK		 1825 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
1826 int pipe_offset = pipe * get_queues_per_pipe(dqm);
1827
1828 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) {
1829 if (!test_bit(pipe_offset + queue,
1830 dqm->dev->shared_resources.queue_bitmap))
1831 continue;
1832
1833 r = dqm->dev->kfd2kgd->hqd_dump(
1834 dqm->dev->kgd, pipe, queue, &dump, &n_regs);
1835 if (r)
1836 break;
1837
1838 seq_printf(m, " CP Pipe %d, Queue %d\n",
1839 pipe, queue);
1840 seq_reg_dump(m, dump, n_regs);
1841
1842 kfree(dump);
1843 }
1844 }
1845
98bb9222 1846 for (pipe = 0; pipe < get_num_sdma_engines(dqm); pipe++) {
d5094189
SL		 1847 for (queue = 0;
1848 queue < dqm->dev->device_info->num_sdma_queues_per_engine;
1849 queue++) {
851a645e
FK		 1850 r = dqm->dev->kfd2kgd->hqd_sdma_dump(
1851 dqm->dev->kgd, pipe, queue, &dump, &n_regs);
1852 if (r)
1853 break;
1854
1855 seq_printf(m, " SDMA Engine %d, RLC %d\n",
1856 pipe, queue);
1857 seq_reg_dump(m, dump, n_regs);
1858
1859 kfree(dump);
1860 }
1861 }
1862
1863 return r;
1864}
1865
a29ec470
SL		 1866int dqm_debugfs_execute_queues(struct device_queue_manager *dqm)
1867{
1868 int r = 0;
1869
1870 dqm_lock(dqm);
1871 dqm->active_runlist = true;
1872 r = execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
1873 dqm_unlock(dqm);
1874
1875 return r;
1876}
1877
851a645e 1878#endif
Ubuntu Eoan Linux kernel mirror