]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/amdkfd: Allocate gart memory using new interface
[mirror_ubuntu-artful-kernel.git] / drivers / gpu / drm / amd / amdkfd / kfd_device_queue_manager.c
CommitLineData
64c7f8cf
BG		 1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#include <linux/slab.h>
25#include <linux/list.h>
26#include <linux/types.h>
27#include <linux/printk.h>
28#include <linux/bitops.h>
29#include "kfd_priv.h"
30#include "kfd_device_queue_manager.h"
31#include "kfd_mqd_manager.h"
32#include "cik_regs.h"
33#include "kfd_kernel_queue.h"
34#include "../../radeon/cik_reg.h"
35
36/* Size of the per-pipe EOP queue */
37#define CIK_HPD_EOP_BYTES_LOG2 11
38#define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)
39
40static bool is_mem_initialized;
41
42static int init_memory(struct device_queue_manager *dqm);
43static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
44 unsigned int pasid, unsigned int vmid);
45
46static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
47 struct queue *q,
48 struct qcm_process_device *qpd);
bcea3081 49
64c7f8cf
BG		 50static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock);
51static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock);
52
bcea3081
BG		 53static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
54 struct queue *q,
55 struct qcm_process_device *qpd);
56
57static void deallocate_sdma_queue(struct device_queue_manager *dqm,
58 unsigned int sdma_queue_id);
59
60static inline
61enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
62{
63 if (type == KFD_QUEUE_TYPE_SDMA)
64 return KFD_MQD_TYPE_CIK_SDMA;
65 return KFD_MQD_TYPE_CIK_CP;
66}
64c7f8cf
BG		 67
68static inline unsigned int get_pipes_num(struct device_queue_manager *dqm)
69{
70 BUG_ON(!dqm || !dqm->dev);
71 return dqm->dev->shared_resources.compute_pipe_count;
72}
73
74static inline unsigned int get_first_pipe(struct device_queue_manager *dqm)
75{
76 BUG_ON(!dqm);
77 return dqm->dev->shared_resources.first_compute_pipe;
78}
79
80static inline unsigned int get_pipes_num_cpsch(void)
81{
82 return PIPE_PER_ME_CP_SCHEDULING;
83}
84
52a5fdce
AS		 85static inline unsigned int
86get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd)
64c7f8cf 87{
64c7f8cf
BG		 88 uint32_t nybble;
89
64c7f8cf
BG		 90 nybble = (pdd->lds_base >> 60) & 0x0E;
91
92 return nybble;
64c7f8cf
BG		 93}
94
52a5fdce 95static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
64c7f8cf 96{
64c7f8cf
BG		 97 unsigned int shared_base;
98
64c7f8cf
BG		 99 shared_base = (pdd->lds_base >> 16) & 0xFF;
100
101 return shared_base;
102}
103
104static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble);
105static void init_process_memory(struct device_queue_manager *dqm,
106 struct qcm_process_device *qpd)
107{
52a5fdce 108 struct kfd_process_device *pdd;
64c7f8cf
BG		 109 unsigned int temp;
110
111 BUG_ON(!dqm || !qpd);
112
52a5fdce
AS		 113 pdd = qpd_to_pdd(qpd);
114
64c7f8cf
BG		 115 /* check if sh_mem_config register already configured */
116 if (qpd->sh_mem_config == 0) {
117 qpd->sh_mem_config =
118 ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED) |
119 DEFAULT_MTYPE(MTYPE_NONCACHED) |
120 APE1_MTYPE(MTYPE_NONCACHED);
121 qpd->sh_mem_ape1_limit = 0;
122 qpd->sh_mem_ape1_base = 0;
123 }
124
125 if (qpd->pqm->process->is_32bit_user_mode) {
52a5fdce 126 temp = get_sh_mem_bases_32(pdd);
64c7f8cf
BG		 127 qpd->sh_mem_bases = SHARED_BASE(temp);
128 qpd->sh_mem_config |= PTR32;
129 } else {
52a5fdce 130 temp = get_sh_mem_bases_nybble_64(pdd);
64c7f8cf
BG		 131 qpd->sh_mem_bases = compute_sh_mem_bases_64bit(temp);
132 }
133
134 pr_debug("kfd: is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n",
135 qpd->pqm->process->is_32bit_user_mode, temp, qpd->sh_mem_bases);
136}
137
138static void program_sh_mem_settings(struct device_queue_manager *dqm,
139 struct qcm_process_device *qpd)
140{
141 return kfd2kgd->program_sh_mem_settings(dqm->dev->kgd, qpd->vmid,
142 qpd->sh_mem_config,
143 qpd->sh_mem_ape1_base,
144 qpd->sh_mem_ape1_limit,
145 qpd->sh_mem_bases);
146}
147
148static int allocate_vmid(struct device_queue_manager *dqm,
149 struct qcm_process_device *qpd,
150 struct queue *q)
151{
152 int bit, allocated_vmid;
153
154 if (dqm->vmid_bitmap == 0)
155 return -ENOMEM;
156
157 bit = find_first_bit((unsigned long *)&dqm->vmid_bitmap, CIK_VMID_NUM);
158 clear_bit(bit, (unsigned long *)&dqm->vmid_bitmap);
159
160 /* Kaveri kfd vmid's starts from vmid 8 */
161 allocated_vmid = bit + KFD_VMID_START_OFFSET;
162 pr_debug("kfd: vmid allocation %d\n", allocated_vmid);
163 qpd->vmid = allocated_vmid;
164 q->properties.vmid = allocated_vmid;
165
166 set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid);
167 program_sh_mem_settings(dqm, qpd);
168
169 return 0;
170}
171
172static void deallocate_vmid(struct device_queue_manager *dqm,
173 struct qcm_process_device *qpd,
174 struct queue *q)
175{
176 int bit = qpd->vmid - KFD_VMID_START_OFFSET;
177
178 set_bit(bit, (unsigned long *)&dqm->vmid_bitmap);
179 qpd->vmid = 0;
180 q->properties.vmid = 0;
181}
182
183static int create_queue_nocpsch(struct device_queue_manager *dqm,
184 struct queue *q,
185 struct qcm_process_device *qpd,
186 int *allocated_vmid)
187{
188 int retval;
189
190 BUG_ON(!dqm || !q || !qpd || !allocated_vmid);
191
192 pr_debug("kfd: In func %s\n", __func__);
193 print_queue(q);
194
195 mutex_lock(&dqm->lock);
196
197 if (list_empty(&qpd->queues_list)) {
198 retval = allocate_vmid(dqm, qpd, q);
199 if (retval != 0) {
200 mutex_unlock(&dqm->lock);
201 return retval;
202 }
203 }
204 *allocated_vmid = qpd->vmid;
205 q->properties.vmid = qpd->vmid;
206
bcea3081
BG		 207 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
208 retval = create_compute_queue_nocpsch(dqm, q, qpd);
209 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
210 retval = create_sdma_queue_nocpsch(dqm, q, qpd);
64c7f8cf
BG		 211
212 if (retval != 0) {
213 if (list_empty(&qpd->queues_list)) {
214 deallocate_vmid(dqm, qpd, q);
215 *allocated_vmid = 0;
216 }
217 mutex_unlock(&dqm->lock);
218 return retval;
219 }
220
221 list_add(&q->list, &qpd->queues_list);
222 dqm->queue_count++;
bcea3081
BG		 223 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
224 dqm->sdma_queue_count++;
64c7f8cf
BG		 225 mutex_unlock(&dqm->lock);
226 return 0;
227}
228
229static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
230{
231 bool set;
232 int pipe, bit;
233
234 set = false;
235
236 for (pipe = dqm->next_pipe_to_allocate; pipe < get_pipes_num(dqm);
237 pipe = (pipe + 1) % get_pipes_num(dqm)) {
238 if (dqm->allocated_queues[pipe] != 0) {
239 bit = find_first_bit(
240 (unsigned long *)&dqm->allocated_queues[pipe],
241 QUEUES_PER_PIPE);
242
243 clear_bit(bit,
244 (unsigned long *)&dqm->allocated_queues[pipe]);
245 q->pipe = pipe;
246 q->queue = bit;
247 set = true;
248 break;
249 }
250 }
251
252 if (set == false)
253 return -EBUSY;
254
255 pr_debug("kfd: DQM %s hqd slot - pipe (%d) queue(%d)\n",
256 __func__, q->pipe, q->queue);
257 /* horizontal hqd allocation */
258 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_num(dqm);
259
260 return 0;
261}
262
263static inline void deallocate_hqd(struct device_queue_manager *dqm,
264 struct queue *q)
265{
266 set_bit(q->queue, (unsigned long *)&dqm->allocated_queues[q->pipe]);
267}
268
269static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
270 struct queue *q,
271 struct qcm_process_device *qpd)
272{
273 int retval;
274 struct mqd_manager *mqd;
275
276 BUG_ON(!dqm || !q || !qpd);
277
278 mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE);
279 if (mqd == NULL)
280 return -ENOMEM;
281
282 retval = allocate_hqd(dqm, q);
283 if (retval != 0)
284 return retval;
285
286 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
287 &q->gart_mqd_addr, &q->properties);
288 if (retval != 0) {
289 deallocate_hqd(dqm, q);
290 return retval;
291 }
292
293 return 0;
294}
295
296static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
297 struct qcm_process_device *qpd,
298 struct queue *q)
299{
300 int retval;
bcea3081 301 struct mqd_manager *mqd, *mqd_sdma;
64c7f8cf
BG		 302 BUG_ON(!dqm || !q || !q->mqd || !qpd);
303
304 retval = 0;
305
306 pr_debug("kfd: In Func %s\n", __func__);
307
308 mutex_lock(&dqm->lock);
309 mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE);
310 if (mqd == NULL) {
311 retval = -ENOMEM;
312 goto out;
313 }
314
bcea3081
BG		 315 mqd_sdma = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_SDMA);
316 if (mqd_sdma == NULL) {
317 mutex_unlock(&dqm->lock);
318 return -ENOMEM;
319 }
320
64c7f8cf
BG		 321 retval = mqd->destroy_mqd(mqd, q->mqd,
322 KFD_PREEMPT_TYPE_WAVEFRONT,
323 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS,
324 q->pipe, q->queue);
325
326 if (retval != 0)
327 goto out;
328
bcea3081
BG		 329 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
330 deallocate_hqd(dqm, q);
331 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
332 dqm->sdma_queue_count--;
333 deallocate_sdma_queue(dqm, q->sdma_id);
334 }
64c7f8cf
BG		 335
336 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
337
338 list_del(&q->list);
339 if (list_empty(&qpd->queues_list))
340 deallocate_vmid(dqm, qpd, q);
341 dqm->queue_count--;
342out:
343 mutex_unlock(&dqm->lock);
344 return retval;
345}
346
347static int update_queue(struct device_queue_manager *dqm, struct queue *q)
348{
349 int retval;
350 struct mqd_manager *mqd;
351
352 BUG_ON(!dqm || !q || !q->mqd);
353
354 mutex_lock(&dqm->lock);
bcea3081 355 mqd = dqm->get_mqd_manager(dqm, q->properties.type);
64c7f8cf
BG		 356 if (mqd == NULL) {
357 mutex_unlock(&dqm->lock);
358 return -ENOMEM;
359 }
360
361 retval = mqd->update_mqd(mqd, q->mqd, &q->properties);
362 if (q->properties.is_active == true)
363 dqm->queue_count++;
364 else
365 dqm->queue_count--;
366
367 if (sched_policy != KFD_SCHED_POLICY_NO_HWS)
368 retval = execute_queues_cpsch(dqm, false);
369
370 mutex_unlock(&dqm->lock);
371 return retval;
372}
373
374static struct mqd_manager *get_mqd_manager_nocpsch(
375 struct device_queue_manager *dqm, enum KFD_MQD_TYPE type)
376{
377 struct mqd_manager *mqd;
378
379 BUG_ON(!dqm || type >= KFD_MQD_TYPE_MAX);
380
381 pr_debug("kfd: In func %s mqd type %d\n", __func__, type);
382
383 mqd = dqm->mqds[type];
384 if (!mqd) {
385 mqd = mqd_manager_init(type, dqm->dev);
386 if (mqd == NULL)
387 pr_err("kfd: mqd manager is NULL");
388 dqm->mqds[type] = mqd;
389 }
390
391 return mqd;
392}
393
394static int register_process_nocpsch(struct device_queue_manager *dqm,
395 struct qcm_process_device *qpd)
396{
397 struct device_process_node *n;
398
399 BUG_ON(!dqm || !qpd);
400
401 pr_debug("kfd: In func %s\n", __func__);
402
403 n = kzalloc(sizeof(struct device_process_node), GFP_KERNEL);
404 if (!n)
405 return -ENOMEM;
406
407 n->qpd = qpd;
408
409 mutex_lock(&dqm->lock);
410 list_add(&n->list, &dqm->queues);
411
412 init_process_memory(dqm, qpd);
413 dqm->processes_count++;
414
415 mutex_unlock(&dqm->lock);
416
417 return 0;
418}
419
420static int unregister_process_nocpsch(struct device_queue_manager *dqm,
421 struct qcm_process_device *qpd)
422{
423 int retval;
424 struct device_process_node *cur, *next;
425
426 BUG_ON(!dqm || !qpd);
427
428 BUG_ON(!list_empty(&qpd->queues_list));
429
430 pr_debug("kfd: In func %s\n", __func__);
431
432 retval = 0;
433 mutex_lock(&dqm->lock);
434
435 list_for_each_entry_safe(cur, next, &dqm->queues, list) {
436 if (qpd == cur->qpd) {
437 list_del(&cur->list);
f5d896bb 438 kfree(cur);
64c7f8cf
BG		 439 dqm->processes_count--;
440 goto out;
441 }
442 }
443 /* qpd not found in dqm list */
444 retval = 1;
445out:
446 mutex_unlock(&dqm->lock);
447 return retval;
448}
449
450static int
451set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid,
452 unsigned int vmid)
453{
454 uint32_t pasid_mapping;
455
456 pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid |
457 ATC_VMID_PASID_MAPPING_VALID;
458 return kfd2kgd->set_pasid_vmid_mapping(dqm->dev->kgd, pasid_mapping,
459 vmid);
460}
461
462static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble)
463{
464 /* In 64-bit mode, we can only control the top 3 bits of the LDS,
465 * scratch and GPUVM apertures.
466 * The hardware fills in the remaining 59 bits according to the
467 * following pattern:
468 * LDS: X0000000'00000000 - X0000001'00000000 (4GB)
469 * Scratch: X0000001'00000000 - X0000002'00000000 (4GB)
470 * GPUVM: Y0010000'00000000 - Y0020000'00000000 (1TB)
471 *
472 * (where X/Y is the configurable nybble with the low-bit 0)
473 *
474 * LDS and scratch will have the same top nybble programmed in the
475 * top 3 bits of SH_MEM_BASES.PRIVATE_BASE.
476 * GPUVM can have a different top nybble programmed in the
477 * top 3 bits of SH_MEM_BASES.SHARED_BASE.
478 * We don't bother to support different top nybbles
479 * for LDS/Scratch and GPUVM.
480 */
481
482 BUG_ON((top_address_nybble & 1) || top_address_nybble > 0xE ||
483 top_address_nybble == 0);
484
485 return PRIVATE_BASE(top_address_nybble << 12) |
486 SHARED_BASE(top_address_nybble << 12);
487}
488
489static int init_memory(struct device_queue_manager *dqm)
490{
491 int i, retval;
492
493 for (i = 8; i < 16; i++)
494 set_pasid_vmid_mapping(dqm, 0, i);
495
496 retval = kfd2kgd->init_memory(dqm->dev->kgd);
497 if (retval == 0)
498 is_mem_initialized = true;
499 return retval;
500}
501
502
503static int init_pipelines(struct device_queue_manager *dqm,
504 unsigned int pipes_num, unsigned int first_pipe)
505{
506 void *hpdptr;
507 struct mqd_manager *mqd;
508 unsigned int i, err, inx;
509 uint64_t pipe_hpd_addr;
510
511 BUG_ON(!dqm || !dqm->dev);
512
513 pr_debug("kfd: In func %s\n", __func__);
514
515 /*
516 * Allocate memory for the HPDs. This is hardware-owned per-pipe data.
517 * The driver never accesses this memory after zeroing it.
518 * It doesn't even have to be saved/restored on suspend/resume
519 * because it contains no data when there are no active queues.
520 */
521
522 err = kfd2kgd->allocate_mem(dqm->dev->kgd,
523 CIK_HPD_EOP_BYTES * pipes_num,
524 PAGE_SIZE,
525 KFD_MEMPOOL_SYSTEM_WRITECOMBINE,
526 (struct kgd_mem **) &dqm->pipeline_mem);
527
528 if (err) {
529 pr_err("kfd: error allocate vidmem num pipes: %d\n",
530 pipes_num);
531 return -ENOMEM;
532 }
533
534 hpdptr = dqm->pipeline_mem->cpu_ptr;
535 dqm->pipelines_addr = dqm->pipeline_mem->gpu_addr;
536
537 memset(hpdptr, 0, CIK_HPD_EOP_BYTES * pipes_num);
538
539 mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE);
540 if (mqd == NULL) {
541 kfd2kgd->free_mem(dqm->dev->kgd,
542 (struct kgd_mem *) dqm->pipeline_mem);
543 return -ENOMEM;
544 }
545
546 for (i = 0; i < pipes_num; i++) {
547 inx = i + first_pipe;
548 pipe_hpd_addr = dqm->pipelines_addr + i * CIK_HPD_EOP_BYTES;
549 pr_debug("kfd: pipeline address %llX\n", pipe_hpd_addr);
550 /* = log2(bytes/4)-1 */
551 kfd2kgd->init_pipeline(dqm->dev->kgd, i,
552 CIK_HPD_EOP_BYTES_LOG2 - 3, pipe_hpd_addr);
553 }
554
555 return 0;
556}
557
64c7f8cf
BG		 558static int init_scheduler(struct device_queue_manager *dqm)
559{
560 int retval;
561
562 BUG_ON(!dqm);
563
564 pr_debug("kfd: In %s\n", __func__);
565
566 retval = init_pipelines(dqm, get_pipes_num(dqm), KFD_DQM_FIRST_PIPE);
567 if (retval != 0)
568 return retval;
569
570 retval = init_memory(dqm);
571
572 return retval;
573}
574
575static int initialize_nocpsch(struct device_queue_manager *dqm)
576{
577 int i;
578
579 BUG_ON(!dqm);
580
581 pr_debug("kfd: In func %s num of pipes: %d\n",
582 __func__, get_pipes_num(dqm));
583
584 mutex_init(&dqm->lock);
585 INIT_LIST_HEAD(&dqm->queues);
586 dqm->queue_count = dqm->next_pipe_to_allocate = 0;
bcea3081 587 dqm->sdma_queue_count = 0;
64c7f8cf
BG		 588 dqm->allocated_queues = kcalloc(get_pipes_num(dqm),
589 sizeof(unsigned int), GFP_KERNEL);
590 if (!dqm->allocated_queues) {
591 mutex_destroy(&dqm->lock);
592 return -ENOMEM;
593 }
594
595 for (i = 0; i < get_pipes_num(dqm); i++)
596 dqm->allocated_queues[i] = (1 << QUEUES_PER_PIPE) - 1;
597
598 dqm->vmid_bitmap = (1 << VMID_PER_DEVICE) - 1;
bcea3081 599 dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1;
64c7f8cf
BG		 600
601 init_scheduler(dqm);
602 return 0;
603}
604
605static void uninitialize_nocpsch(struct device_queue_manager *dqm)
606{
6f9d54fd
OG		 607 int i;
608
64c7f8cf
BG		 609 BUG_ON(!dqm);
610
611 BUG_ON(dqm->queue_count > 0 || dqm->processes_count > 0);
612
613 kfree(dqm->allocated_queues);
6f9d54fd
OG		 614 for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
615 kfree(dqm->mqds[i]);
64c7f8cf
BG		 616 mutex_destroy(&dqm->lock);
617 kfd2kgd->free_mem(dqm->dev->kgd,
618 (struct kgd_mem *) dqm->pipeline_mem);
619}
620
621static int start_nocpsch(struct device_queue_manager *dqm)
622{
623 return 0;
624}
625
626static int stop_nocpsch(struct device_queue_manager *dqm)
627{
628 return 0;
629}
630
bcea3081
BG		 631static int allocate_sdma_queue(struct device_queue_manager *dqm,
632 unsigned int *sdma_queue_id)
633{
634 int bit;
635
636 if (dqm->sdma_bitmap == 0)
637 return -ENOMEM;
638
639 bit = find_first_bit((unsigned long *)&dqm->sdma_bitmap,
640 CIK_SDMA_QUEUES);
641
642 clear_bit(bit, (unsigned long *)&dqm->sdma_bitmap);
643 *sdma_queue_id = bit;
644
645 return 0;
646}
647
648static void deallocate_sdma_queue(struct device_queue_manager *dqm,
649 unsigned int sdma_queue_id)
650{
651 if (sdma_queue_id < 0 || sdma_queue_id >= CIK_SDMA_QUEUES)
652 return;
653 set_bit(sdma_queue_id, (unsigned long *)&dqm->sdma_bitmap);
654}
655
656static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q,
657 struct qcm_process_device *qpd)
658{
659 uint32_t value = SDMA_ATC;
660
661 if (q->process->is_32bit_user_mode)
662 value |= SDMA_VA_PTR32 | get_sh_mem_bases_32(qpd_to_pdd(qpd));
663 else
664 value |= SDMA_VA_SHARED_BASE(get_sh_mem_bases_nybble_64(
665 qpd_to_pdd(qpd)));
666 q->properties.sdma_vm_addr = value;
667}
668
669static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
670 struct queue *q,
671 struct qcm_process_device *qpd)
672{
673 struct mqd_manager *mqd;
674 int retval;
675
676 mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_SDMA);
677 if (!mqd)
678 return -ENOMEM;
679
680 retval = allocate_sdma_queue(dqm, &q->sdma_id);
681 if (retval != 0)
682 return retval;
683
684 q->properties.sdma_queue_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
685 q->properties.sdma_engine_id = q->sdma_id / CIK_SDMA_ENGINE_NUM;
686
687 pr_debug("kfd: sdma id is: %d\n", q->sdma_id);
688 pr_debug(" sdma queue id: %d\n", q->properties.sdma_queue_id);
689 pr_debug(" sdma engine id: %d\n", q->properties.sdma_engine_id);
690
691 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
692 &q->gart_mqd_addr, &q->properties);
693 if (retval != 0) {
694 deallocate_sdma_queue(dqm, q->sdma_id);
695 return retval;
696 }
697
698 init_sdma_vm(dqm, q, qpd);
699 return 0;
700}
701
64c7f8cf
BG		 702/*
703 * Device Queue Manager implementation for cp scheduler
704 */
705
706static int set_sched_resources(struct device_queue_manager *dqm)
707{
708 struct scheduling_resources res;
709 unsigned int queue_num, queue_mask;
710
711 BUG_ON(!dqm);
712
713 pr_debug("kfd: In func %s\n", __func__);
714
715 queue_num = get_pipes_num_cpsch() * QUEUES_PER_PIPE;
716 queue_mask = (1 << queue_num) - 1;
717 res.vmid_mask = (1 << VMID_PER_DEVICE) - 1;
718 res.vmid_mask <<= KFD_VMID_START_OFFSET;
719 res.queue_mask = queue_mask << (get_first_pipe(dqm) * QUEUES_PER_PIPE);
720 res.gws_mask = res.oac_mask = res.gds_heap_base =
721 res.gds_heap_size = 0;
722
723 pr_debug("kfd: scheduling resources:\n"
724 " vmid mask: 0x%8X\n"
725 " queue mask: 0x%8llX\n",
726 res.vmid_mask, res.queue_mask);
727
728 return pm_send_set_resources(&dqm->packets, &res);
729}
730
731static int initialize_cpsch(struct device_queue_manager *dqm)
732{
733 int retval;
734
735 BUG_ON(!dqm);
736
737 pr_debug("kfd: In func %s num of pipes: %d\n",
738 __func__, get_pipes_num_cpsch());
739
740 mutex_init(&dqm->lock);
741 INIT_LIST_HEAD(&dqm->queues);
742 dqm->queue_count = dqm->processes_count = 0;
bcea3081 743 dqm->sdma_queue_count = 0;
64c7f8cf
BG		 744 dqm->active_runlist = false;
745 retval = init_pipelines(dqm, get_pipes_num(dqm), 0);
746 if (retval != 0)
747 goto fail_init_pipelines;
748
749 return 0;
750
751fail_init_pipelines:
752 mutex_destroy(&dqm->lock);
753 return retval;
754}
755
756static int start_cpsch(struct device_queue_manager *dqm)
757{
758 struct device_process_node *node;
759 int retval;
760
761 BUG_ON(!dqm);
762
763 retval = 0;
764
765 retval = pm_init(&dqm->packets, dqm);
766 if (retval != 0)
767 goto fail_packet_manager_init;
768
769 retval = set_sched_resources(dqm);
770 if (retval != 0)
771 goto fail_set_sched_resources;
772
773 pr_debug("kfd: allocating fence memory\n");
774
775 /* allocate fence memory on the gart */
776 retval = kfd2kgd->allocate_mem(dqm->dev->kgd,
777 sizeof(*dqm->fence_addr),
778 32,
779 KFD_MEMPOOL_SYSTEM_WRITECOMBINE,
780 (struct kgd_mem **) &dqm->fence_mem);
781
782 if (retval != 0)
783 goto fail_allocate_vidmem;
784
785 dqm->fence_addr = dqm->fence_mem->cpu_ptr;
786 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr;
64c7f8cf
BG		 787 list_for_each_entry(node, &dqm->queues, list)
788 if (node->qpd->pqm->process && dqm->dev)
789 kfd_bind_process_to_device(dqm->dev,
790 node->qpd->pqm->process);
791
792 execute_queues_cpsch(dqm, true);
793
794 return 0;
795fail_allocate_vidmem:
796fail_set_sched_resources:
797 pm_uninit(&dqm->packets);
798fail_packet_manager_init:
799 return retval;
800}
801
802static int stop_cpsch(struct device_queue_manager *dqm)
803{
804 struct device_process_node *node;
805 struct kfd_process_device *pdd;
806
807 BUG_ON(!dqm);
808
809 destroy_queues_cpsch(dqm, true);
810
811 list_for_each_entry(node, &dqm->queues, list) {
52a5fdce 812 pdd = qpd_to_pdd(node->qpd);
64c7f8cf
BG		 813 pdd->bound = false;
814 }
815 kfd2kgd->free_mem(dqm->dev->kgd,
816 (struct kgd_mem *) dqm->fence_mem);
817 pm_uninit(&dqm->packets);
818
819 return 0;
820}
821
822static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
823 struct kernel_queue *kq,
824 struct qcm_process_device *qpd)
825{
826 BUG_ON(!dqm || !kq || !qpd);
827
828 pr_debug("kfd: In func %s\n", __func__);
829
830 mutex_lock(&dqm->lock);
831 list_add(&kq->list, &qpd->priv_queue_list);
832 dqm->queue_count++;
833 qpd->is_debug = true;
834 execute_queues_cpsch(dqm, false);
835 mutex_unlock(&dqm->lock);
836
837 return 0;
838}
839
840static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
841 struct kernel_queue *kq,
842 struct qcm_process_device *qpd)
843{
844 BUG_ON(!dqm || !kq);
845
846 pr_debug("kfd: In %s\n", __func__);
847
848 mutex_lock(&dqm->lock);
849 destroy_queues_cpsch(dqm, false);
850 list_del(&kq->list);
851 dqm->queue_count--;
852 qpd->is_debug = false;
853 execute_queues_cpsch(dqm, false);
854 mutex_unlock(&dqm->lock);
855}
856
bcea3081
BG		 857static void select_sdma_engine_id(struct queue *q)
858{
859 static int sdma_id;
860
861 q->sdma_id = sdma_id;
862 sdma_id = (sdma_id + 1) % 2;
863}
864
64c7f8cf
BG		 865static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
866 struct qcm_process_device *qpd, int *allocate_vmid)
867{
868 int retval;
869 struct mqd_manager *mqd;
870
871 BUG_ON(!dqm || !q || !qpd);
872
873 retval = 0;
874
875 if (allocate_vmid)
876 *allocate_vmid = 0;
877
878 mutex_lock(&dqm->lock);
879
bcea3081
BG		 880 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
881 select_sdma_engine_id(q);
882
883 mqd = dqm->get_mqd_manager(dqm,
884 get_mqd_type_from_queue_type(q->properties.type));
885
64c7f8cf
BG		 886 if (mqd == NULL) {
887 mutex_unlock(&dqm->lock);
888 return -ENOMEM;
889 }
890
891 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
892 &q->gart_mqd_addr, &q->properties);
893 if (retval != 0)
894 goto out;
895
896 list_add(&q->list, &qpd->queues_list);
897 if (q->properties.is_active) {
898 dqm->queue_count++;
899 retval = execute_queues_cpsch(dqm, false);
900 }
901
bcea3081
BG		 902 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
903 dqm->sdma_queue_count++;
904
64c7f8cf
BG		 905out:
906 mutex_unlock(&dqm->lock);
907 return retval;
908}
909
d80d19bd
OG		 910static int fence_wait_timeout(unsigned int *fence_addr,
911 unsigned int fence_value,
912 unsigned long timeout)
64c7f8cf
BG		 913{
914 BUG_ON(!fence_addr);
915 timeout += jiffies;
916
917 while (*fence_addr != fence_value) {
918 if (time_after(jiffies, timeout)) {
919 pr_err("kfd: qcm fence wait loop timeout expired\n");
920 return -ETIME;
921 }
922 cpu_relax();
923 }
924
925 return 0;
926}
927
bcea3081
BG		 928static int destroy_sdma_queues(struct device_queue_manager *dqm,
929 unsigned int sdma_engine)
930{
931 return pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_SDMA,
932 KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false,
933 sdma_engine);
934}
935
64c7f8cf
BG		 936static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock)
937{
938 int retval;
939
940 BUG_ON(!dqm);
941
942 retval = 0;
943
944 if (lock)
945 mutex_lock(&dqm->lock);
946 if (dqm->active_runlist == false)
947 goto out;
bcea3081
BG		 948
949 pr_debug("kfd: Before destroying queues, sdma queue count is : %u\n",
950 dqm->sdma_queue_count);
951
952 if (dqm->sdma_queue_count > 0) {
953 destroy_sdma_queues(dqm, 0);
954 destroy_sdma_queues(dqm, 1);
955 }
956
64c7f8cf
BG		 957 retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE,
958 KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false, 0);
959 if (retval != 0)
960 goto out;
961
962 *dqm->fence_addr = KFD_FENCE_INIT;
963 pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr,
964 KFD_FENCE_COMPLETED);
965 /* should be timed out */
966 fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED,
967 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS);
968 pm_release_ib(&dqm->packets);
969 dqm->active_runlist = false;
970
971out:
972 if (lock)
973 mutex_unlock(&dqm->lock);
974 return retval;
975}
976
977static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock)
978{
979 int retval;
980
981 BUG_ON(!dqm);
982
983 if (lock)
984 mutex_lock(&dqm->lock);
985
986 retval = destroy_queues_cpsch(dqm, false);
987 if (retval != 0) {
988 pr_err("kfd: the cp might be in an unrecoverable state due to an unsuccessful queues preemption");
989 goto out;
990 }
991
992 if (dqm->queue_count <= 0 || dqm->processes_count <= 0) {
993 retval = 0;
994 goto out;
995 }
996
997 if (dqm->active_runlist) {
998 retval = 0;
999 goto out;
1000 }
1001
1002 retval = pm_send_runlist(&dqm->packets, &dqm->queues);
1003 if (retval != 0) {
1004 pr_err("kfd: failed to execute runlist");
1005 goto out;
1006 }
1007 dqm->active_runlist = true;
1008
1009out:
1010 if (lock)
1011 mutex_unlock(&dqm->lock);
1012 return retval;
1013}
1014
1015static int destroy_queue_cpsch(struct device_queue_manager *dqm,
1016 struct qcm_process_device *qpd,
1017 struct queue *q)
1018{
1019 int retval;
1020 struct mqd_manager *mqd;
1021
1022 BUG_ON(!dqm || !qpd || !q);
1023
1024 retval = 0;
1025
1026 /* remove queue from list to prevent rescheduling after preemption */
1027 mutex_lock(&dqm->lock);
bcea3081
BG		 1028 mqd = dqm->get_mqd_manager(dqm,
1029 get_mqd_type_from_queue_type(q->properties.type));
64c7f8cf
BG		 1030 if (!mqd) {
1031 retval = -ENOMEM;
1032 goto failed;
1033 }
1034
bcea3081
BG		 1035 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
1036 dqm->sdma_queue_count--;
1037
64c7f8cf
BG		 1038 list_del(&q->list);
1039 dqm->queue_count--;
1040
1041 execute_queues_cpsch(dqm, false);
1042
1043 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
1044
1045 mutex_unlock(&dqm->lock);
1046
1047 return 0;
1048
1049failed:
1050 mutex_unlock(&dqm->lock);
1051 return retval;
1052}
1053
1054/*
1055 * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to
1056 * stay in user mode.
1057 */
1058#define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL
1059/* APE1 limit is inclusive and 64K aligned. */
1060#define APE1_LIMIT_ALIGNMENT 0xFFFF
1061
1062static bool set_cache_memory_policy(struct device_queue_manager *dqm,
1063 struct qcm_process_device *qpd,
1064 enum cache_policy default_policy,
1065 enum cache_policy alternate_policy,
1066 void __user *alternate_aperture_base,
1067 uint64_t alternate_aperture_size)
1068{
1069 uint32_t default_mtype;
1070 uint32_t ape1_mtype;
1071
1072 pr_debug("kfd: In func %s\n", __func__);
1073
1074 mutex_lock(&dqm->lock);
1075
1076 if (alternate_aperture_size == 0) {
1077 /* base > limit disables APE1 */
1078 qpd->sh_mem_ape1_base = 1;
1079 qpd->sh_mem_ape1_limit = 0;
1080 } else {
1081 /*
1082 * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]},
1083 * SH_MEM_APE1_BASE[31:0], 0x0000 }
1084 * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]},
1085 * SH_MEM_APE1_LIMIT[31:0], 0xFFFF }
1086 * Verify that the base and size parameters can be
1087 * represented in this format and convert them.
1088 * Additionally restrict APE1 to user-mode addresses.
1089 */
1090
1091 uint64_t base = (uintptr_t)alternate_aperture_base;
1092 uint64_t limit = base + alternate_aperture_size - 1;
1093
1094 if (limit <= base)
1095 goto out;
1096
1097 if ((base & APE1_FIXED_BITS_MASK) != 0)
1098 goto out;
1099
1100 if ((limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT)
1101 goto out;
1102
1103 qpd->sh_mem_ape1_base = base >> 16;
1104 qpd->sh_mem_ape1_limit = limit >> 16;
1105 }
1106
1107 default_mtype = (default_policy == cache_policy_coherent) ?
1108 MTYPE_NONCACHED :
1109 MTYPE_CACHED;
1110
1111 ape1_mtype = (alternate_policy == cache_policy_coherent) ?
1112 MTYPE_NONCACHED :
1113 MTYPE_CACHED;
1114
1115 qpd->sh_mem_config = (qpd->sh_mem_config & PTR32)
1116 | ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED)
1117 | DEFAULT_MTYPE(default_mtype)
1118 | APE1_MTYPE(ape1_mtype);
1119
1120 if ((sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
1121 program_sh_mem_settings(dqm, qpd);
1122
1123 pr_debug("kfd: sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
1124 qpd->sh_mem_config, qpd->sh_mem_ape1_base,
1125 qpd->sh_mem_ape1_limit);
1126
1127 mutex_unlock(&dqm->lock);
1128 return true;
1129
1130out:
1131 mutex_unlock(&dqm->lock);
1132 return false;
1133}
1134
1135struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
1136{
1137 struct device_queue_manager *dqm;
1138
1139 BUG_ON(!dev);
1140
1141 dqm = kzalloc(sizeof(struct device_queue_manager), GFP_KERNEL);
1142 if (!dqm)
1143 return NULL;
1144
1145 dqm->dev = dev;
1146 switch (sched_policy) {
1147 case KFD_SCHED_POLICY_HWS:
1148 case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION:
1149 /* initialize dqm for cp scheduling */
1150 dqm->create_queue = create_queue_cpsch;
1151 dqm->initialize = initialize_cpsch;
1152 dqm->start = start_cpsch;
1153 dqm->stop = stop_cpsch;
1154 dqm->destroy_queue = destroy_queue_cpsch;
1155 dqm->update_queue = update_queue;
1156 dqm->get_mqd_manager = get_mqd_manager_nocpsch;
1157 dqm->register_process = register_process_nocpsch;
1158 dqm->unregister_process = unregister_process_nocpsch;
1159 dqm->uninitialize = uninitialize_nocpsch;
1160 dqm->create_kernel_queue = create_kernel_queue_cpsch;
1161 dqm->destroy_kernel_queue = destroy_kernel_queue_cpsch;
1162 dqm->set_cache_memory_policy = set_cache_memory_policy;
1163 break;
1164 case KFD_SCHED_POLICY_NO_HWS:
1165 /* initialize dqm for no cp scheduling */
1166 dqm->start = start_nocpsch;
1167 dqm->stop = stop_nocpsch;
1168 dqm->create_queue = create_queue_nocpsch;
1169 dqm->destroy_queue = destroy_queue_nocpsch;
1170 dqm->update_queue = update_queue;
1171 dqm->get_mqd_manager = get_mqd_manager_nocpsch;
1172 dqm->register_process = register_process_nocpsch;
1173 dqm->unregister_process = unregister_process_nocpsch;
1174 dqm->initialize = initialize_nocpsch;
1175 dqm->uninitialize = uninitialize_nocpsch;
1176 dqm->set_cache_memory_policy = set_cache_memory_policy;
1177 break;
1178 default:
1179 BUG();
1180 break;
1181 }
1182
1183 if (dqm->initialize(dqm) != 0) {
1184 kfree(dqm);
1185 return NULL;
1186 }
1187
1188 return dqm;
1189}
1190
1191void device_queue_manager_uninit(struct device_queue_manager *dqm)
1192{
1193 BUG_ON(!dqm);
1194
1195 dqm->uninitialize(dqm);
1196 kfree(dqm);
1197}
1198
Ubuntu Artful kernel mirror