1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
8 #include <uapi/misc/habanalabs.h>
9 #include "habanalabs.h"
11 #include <linux/uaccess.h>
12 #include <linux/slab.h>
14 static void job_wq_completion(struct work_struct
*work
);
15 static long _hl_cs_wait_ioctl(struct hl_device
*hdev
,
16 struct hl_ctx
*ctx
, u64 timeout_us
, u64 seq
);
17 static void cs_do_release(struct kref
*ref
);
19 static const char *hl_fence_get_driver_name(struct dma_fence
*fence
)
24 static const char *hl_fence_get_timeline_name(struct dma_fence
*fence
)
26 struct hl_dma_fence
*hl_fence
=
27 container_of(fence
, struct hl_dma_fence
, base_fence
);
29 return dev_name(hl_fence
->hdev
->dev
);
32 static bool hl_fence_enable_signaling(struct dma_fence
*fence
)
37 static void hl_fence_release(struct dma_fence
*fence
)
39 struct hl_dma_fence
*hl_fence
=
40 container_of(fence
, struct hl_dma_fence
, base_fence
);
42 kfree_rcu(hl_fence
, base_fence
.rcu
);
45 static const struct dma_fence_ops hl_fence_ops
= {
46 .get_driver_name
= hl_fence_get_driver_name
,
47 .get_timeline_name
= hl_fence_get_timeline_name
,
48 .enable_signaling
= hl_fence_enable_signaling
,
49 .wait
= dma_fence_default_wait
,
50 .release
= hl_fence_release
53 static void cs_get(struct hl_cs
*cs
)
55 kref_get(&cs
->refcount
);
58 static int cs_get_unless_zero(struct hl_cs
*cs
)
60 return kref_get_unless_zero(&cs
->refcount
);
63 static void cs_put(struct hl_cs
*cs
)
65 kref_put(&cs
->refcount
, cs_do_release
);
69 * cs_parser - parse the user command submission
71 * @hpriv : pointer to the private data of the fd
72 * @job : pointer to the job that holds the command submission info
74 * The function parses the command submission of the user. It calls the
75 * ASIC specific parser, which returns a list of memory blocks to send
76 * to the device as different command buffers
79 static int cs_parser(struct hl_fpriv
*hpriv
, struct hl_cs_job
*job
)
81 struct hl_device
*hdev
= hpriv
->hdev
;
82 struct hl_cs_parser parser
;
85 parser
.ctx_id
= job
->cs
->ctx
->asid
;
86 parser
.cs_sequence
= job
->cs
->sequence
;
87 parser
.job_id
= job
->id
;
89 parser
.hw_queue_id
= job
->hw_queue_id
;
90 parser
.job_userptr_list
= &job
->userptr_list
;
91 parser
.patched_cb
= NULL
;
92 parser
.user_cb
= job
->user_cb
;
93 parser
.user_cb_size
= job
->user_cb_size
;
94 parser
.ext_queue
= job
->ext_queue
;
95 job
->patched_cb
= NULL
;
96 parser
.use_virt_addr
= hdev
->mmu_enable
;
98 rc
= hdev
->asic_funcs
->cs_parser(hdev
, &parser
);
101 job
->patched_cb
= parser
.patched_cb
;
102 job
->job_cb_size
= parser
.patched_cb_size
;
104 spin_lock(&job
->patched_cb
->lock
);
105 job
->patched_cb
->cs_cnt
++;
106 spin_unlock(&job
->patched_cb
->lock
);
110 * Whether the parsing worked or not, we don't need the
111 * original CB anymore because it was already parsed and
112 * won't be accessed again for this CS
114 spin_lock(&job
->user_cb
->lock
);
115 job
->user_cb
->cs_cnt
--;
116 spin_unlock(&job
->user_cb
->lock
);
117 hl_cb_put(job
->user_cb
);
124 static void free_job(struct hl_device
*hdev
, struct hl_cs_job
*job
)
126 struct hl_cs
*cs
= job
->cs
;
128 if (job
->ext_queue
) {
129 hl_userptr_delete_list(hdev
, &job
->userptr_list
);
132 * We might arrive here from rollback and patched CB wasn't
133 * created, so we need to check it's not NULL
135 if (job
->patched_cb
) {
136 spin_lock(&job
->patched_cb
->lock
);
137 job
->patched_cb
->cs_cnt
--;
138 spin_unlock(&job
->patched_cb
->lock
);
140 hl_cb_put(job
->patched_cb
);
145 * This is the only place where there can be multiple threads
146 * modifying the list at the same time
148 spin_lock(&cs
->job_lock
);
149 list_del(&job
->cs_node
);
150 spin_unlock(&cs
->job_lock
);
152 hl_debugfs_remove_job(hdev
, job
);
160 static void cs_do_release(struct kref
*ref
)
162 struct hl_cs
*cs
= container_of(ref
, struct hl_cs
,
164 struct hl_device
*hdev
= cs
->ctx
->hdev
;
165 struct hl_cs_job
*job
, *tmp
;
167 cs
->completed
= true;
170 * Although if we reached here it means that all external jobs have
171 * finished, because each one of them took refcnt to CS, we still
172 * need to go over the internal jobs and free them. Otherwise, we
173 * will have leaked memory and what's worse, the CS object (and
174 * potentially the CTX object) could be released, while the JOB
175 * still holds a pointer to them (but no reference).
177 list_for_each_entry_safe(job
, tmp
, &cs
->job_list
, cs_node
)
180 /* We also need to update CI for internal queues */
182 hl_int_hw_queue_update_ci(cs
);
184 spin_lock(&hdev
->hw_queues_mirror_lock
);
185 /* remove CS from hw_queues mirror list */
186 list_del_init(&cs
->mirror_node
);
187 spin_unlock(&hdev
->hw_queues_mirror_lock
);
190 * Don't cancel TDR in case this CS was timedout because we
191 * might be running from the TDR context
193 if ((!cs
->timedout
) &&
194 (hdev
->timeout_jiffies
!= MAX_SCHEDULE_TIMEOUT
)) {
198 cancel_delayed_work_sync(&cs
->work_tdr
);
200 spin_lock(&hdev
->hw_queues_mirror_lock
);
202 /* queue TDR for next CS */
203 next
= list_first_entry_or_null(
204 &hdev
->hw_queues_mirror_list
,
205 struct hl_cs
, mirror_node
);
207 if ((next
) && (!next
->tdr_active
)) {
208 next
->tdr_active
= true;
209 schedule_delayed_work(&next
->work_tdr
,
210 hdev
->timeout_jiffies
);
213 spin_unlock(&hdev
->hw_queues_mirror_lock
);
218 * Must be called before hl_ctx_put because inside we use ctx to get
221 hl_debugfs_remove_cs(cs
);
226 dma_fence_set_error(cs
->fence
, -ETIMEDOUT
);
227 else if (cs
->aborted
)
228 dma_fence_set_error(cs
->fence
, -EIO
);
230 dma_fence_signal(cs
->fence
);
231 dma_fence_put(cs
->fence
);
236 static void cs_timedout(struct work_struct
*work
)
238 struct hl_device
*hdev
;
240 struct hl_cs
*cs
= container_of(work
, struct hl_cs
,
242 rc
= cs_get_unless_zero(cs
);
246 if ((!cs
->submitted
) || (cs
->completed
)) {
251 /* Mark the CS is timed out so we won't try to cancel its TDR */
254 hdev
= cs
->ctx
->hdev
;
255 ctx_asid
= cs
->ctx
->asid
;
257 /* TODO: add information about last signaled seq and last emitted seq */
258 dev_err(hdev
->dev
, "CS %d.%llu got stuck!\n", ctx_asid
, cs
->sequence
);
262 if (hdev
->reset_on_lockup
)
263 hl_device_reset(hdev
, false, false);
266 static int allocate_cs(struct hl_device
*hdev
, struct hl_ctx
*ctx
,
267 struct hl_cs
**cs_new
)
269 struct hl_dma_fence
*fence
;
270 struct dma_fence
*other
= NULL
;
274 cs
= kzalloc(sizeof(*cs
), GFP_ATOMIC
);
279 cs
->submitted
= false;
280 cs
->completed
= false;
281 INIT_LIST_HEAD(&cs
->job_list
);
282 INIT_DELAYED_WORK(&cs
->work_tdr
, cs_timedout
);
283 kref_init(&cs
->refcount
);
284 spin_lock_init(&cs
->job_lock
);
286 fence
= kmalloc(sizeof(*fence
), GFP_ATOMIC
);
293 spin_lock_init(&fence
->lock
);
294 cs
->fence
= &fence
->base_fence
;
296 spin_lock(&ctx
->cs_lock
);
298 fence
->cs_seq
= ctx
->cs_sequence
;
299 other
= ctx
->cs_pending
[fence
->cs_seq
& (HL_MAX_PENDING_CS
- 1)];
300 if ((other
) && (!dma_fence_is_signaled(other
))) {
301 spin_unlock(&ctx
->cs_lock
);
306 dma_fence_init(&fence
->base_fence
, &hl_fence_ops
, &fence
->lock
,
307 ctx
->asid
, ctx
->cs_sequence
);
309 cs
->sequence
= fence
->cs_seq
;
311 ctx
->cs_pending
[fence
->cs_seq
& (HL_MAX_PENDING_CS
- 1)] =
315 dma_fence_get(&fence
->base_fence
);
317 dma_fence_put(other
);
319 spin_unlock(&ctx
->cs_lock
);
332 static void cs_rollback(struct hl_device
*hdev
, struct hl_cs
*cs
)
334 struct hl_cs_job
*job
, *tmp
;
336 list_for_each_entry_safe(job
, tmp
, &cs
->job_list
, cs_node
)
340 void hl_cs_rollback_all(struct hl_device
*hdev
)
342 struct hl_cs
*cs
, *tmp
;
344 /* flush all completions */
345 flush_workqueue(hdev
->cq_wq
);
347 /* Make sure we don't have leftovers in the H/W queues mirror list */
348 list_for_each_entry_safe(cs
, tmp
, &hdev
->hw_queues_mirror_list
,
352 dev_warn_ratelimited(hdev
->dev
, "Killing CS %d.%llu\n",
353 cs
->ctx
->asid
, cs
->sequence
);
354 cs_rollback(hdev
, cs
);
359 static void job_wq_completion(struct work_struct
*work
)
361 struct hl_cs_job
*job
= container_of(work
, struct hl_cs_job
,
363 struct hl_cs
*cs
= job
->cs
;
364 struct hl_device
*hdev
= cs
->ctx
->hdev
;
366 /* job is no longer needed */
370 static struct hl_cb
*validate_queue_index(struct hl_device
*hdev
,
371 struct hl_cb_mgr
*cb_mgr
,
372 struct hl_cs_chunk
*chunk
,
375 struct asic_fixed_properties
*asic
= &hdev
->asic_prop
;
376 struct hw_queue_properties
*hw_queue_prop
;
380 /* Assume external queue */
383 hw_queue_prop
= &asic
->hw_queues_props
[chunk
->queue_index
];
385 if ((chunk
->queue_index
>= HL_MAX_QUEUES
) ||
386 (hw_queue_prop
->type
== QUEUE_TYPE_NA
)) {
387 dev_err(hdev
->dev
, "Queue index %d is invalid\n",
392 if (hw_queue_prop
->kmd_only
) {
393 dev_err(hdev
->dev
, "Queue index %d is restricted for KMD\n",
396 } else if (hw_queue_prop
->type
== QUEUE_TYPE_INT
) {
398 return (struct hl_cb
*) (uintptr_t) chunk
->cb_handle
;
401 /* Retrieve CB object */
402 cb_handle
= (u32
) (chunk
->cb_handle
>> PAGE_SHIFT
);
404 cb
= hl_cb_get(hdev
, cb_mgr
, cb_handle
);
406 dev_err(hdev
->dev
, "CB handle 0x%x invalid\n", cb_handle
);
410 if ((chunk
->cb_size
< 8) || (chunk
->cb_size
> cb
->size
)) {
411 dev_err(hdev
->dev
, "CB size %u invalid\n", chunk
->cb_size
);
415 spin_lock(&cb
->lock
);
417 spin_unlock(&cb
->lock
);
426 struct hl_cs_job
*hl_cs_allocate_job(struct hl_device
*hdev
, bool ext_queue
)
428 struct hl_cs_job
*job
;
430 job
= kzalloc(sizeof(*job
), GFP_ATOMIC
);
434 job
->ext_queue
= ext_queue
;
436 if (job
->ext_queue
) {
437 INIT_LIST_HEAD(&job
->userptr_list
);
438 INIT_WORK(&job
->finish_work
, job_wq_completion
);
444 static int _hl_cs_ioctl(struct hl_fpriv
*hpriv
, void __user
*chunks
,
445 u32 num_chunks
, u64
*cs_seq
)
447 struct hl_device
*hdev
= hpriv
->hdev
;
448 struct hl_cs_chunk
*cs_chunk_array
;
449 struct hl_cs_job
*job
;
452 bool ext_queue_present
= false;
454 int rc
, i
, parse_cnt
;
456 *cs_seq
= ULLONG_MAX
;
458 if (num_chunks
> HL_MAX_JOBS_PER_CS
) {
460 "Number of chunks can NOT be larger than %d\n",
466 cs_chunk_array
= kmalloc_array(num_chunks
, sizeof(*cs_chunk_array
),
468 if (!cs_chunk_array
) {
473 size_to_copy
= num_chunks
* sizeof(struct hl_cs_chunk
);
474 if (copy_from_user(cs_chunk_array
, chunks
, size_to_copy
)) {
475 dev_err(hdev
->dev
, "Failed to copy cs chunk array from user\n");
477 goto free_cs_chunk_array
;
480 /* increment refcnt for context */
481 hl_ctx_get(hdev
, hpriv
->ctx
);
483 rc
= allocate_cs(hdev
, hpriv
->ctx
, &cs
);
485 hl_ctx_put(hpriv
->ctx
);
486 goto free_cs_chunk_array
;
489 *cs_seq
= cs
->sequence
;
491 hl_debugfs_add_cs(cs
);
493 /* Validate ALL the CS chunks before submitting the CS */
494 for (i
= 0, parse_cnt
= 0 ; i
< num_chunks
; i
++, parse_cnt
++) {
495 struct hl_cs_chunk
*chunk
= &cs_chunk_array
[i
];
498 cb
= validate_queue_index(hdev
, &hpriv
->cb_mgr
, chunk
,
501 ext_queue_present
= true;
508 job
= hl_cs_allocate_job(hdev
, ext_queue
);
510 dev_err(hdev
->dev
, "Failed to allocate a new job\n");
521 job
->user_cb_size
= chunk
->cb_size
;
523 job
->job_cb_size
= cb
->size
;
525 job
->job_cb_size
= chunk
->cb_size
;
526 job
->hw_queue_id
= chunk
->queue_index
;
528 cs
->jobs_in_queue_cnt
[job
->hw_queue_id
]++;
530 list_add_tail(&job
->cs_node
, &cs
->job_list
);
533 * Increment CS reference. When CS reference is 0, CS is
534 * done and can be signaled to user and free all its resources
535 * Only increment for JOB on external queues, because only
536 * for those JOBs we get completion
541 hl_debugfs_add_job(hdev
, job
);
543 rc
= cs_parser(hpriv
, job
);
546 "Failed to parse JOB %d.%llu.%d, err %d, rejecting the CS\n",
547 cs
->ctx
->asid
, cs
->sequence
, job
->id
, rc
);
552 if (!ext_queue_present
) {
554 "Reject CS %d.%llu because no external queues jobs\n",
555 cs
->ctx
->asid
, cs
->sequence
);
560 rc
= hl_hw_queue_schedule_cs(cs
);
563 "Failed to submit CS %d.%llu to H/W queues, error %d\n",
564 cs
->ctx
->asid
, cs
->sequence
, rc
);
568 rc
= HL_CS_STATUS_SUCCESS
;
572 spin_lock(&cb
->lock
);
574 spin_unlock(&cb
->lock
);
577 cs_rollback(hdev
, cs
);
578 *cs_seq
= ULLONG_MAX
;
579 /* The path below is both for good and erroneous exits */
581 /* We finished with the CS in this function, so put the ref */
584 kfree(cs_chunk_array
);
589 int hl_cs_ioctl(struct hl_fpriv
*hpriv
, void *data
)
591 struct hl_device
*hdev
= hpriv
->hdev
;
592 union hl_cs_args
*args
= data
;
593 struct hl_ctx
*ctx
= hpriv
->ctx
;
596 u64 cs_seq
= ULONG_MAX
;
598 bool need_soft_reset
= false;
600 if (hl_device_disabled_or_in_reset(hdev
)) {
602 "Device is %s. Can't submit new CS\n",
603 atomic_read(&hdev
->in_reset
) ? "in_reset" : "disabled");
608 do_restore
= atomic_cmpxchg(&ctx
->thread_restore_token
, 1, 0);
610 if (do_restore
|| (args
->in
.cs_flags
& HL_CS_FLAGS_FORCE_RESTORE
)) {
613 chunks
= (void __user
*)(uintptr_t)args
->in
.chunks_restore
;
614 num_chunks
= args
->in
.num_chunks_restore
;
616 mutex_lock(&hpriv
->restore_phase_mutex
);
619 rc
= hdev
->asic_funcs
->context_switch(hdev
, ctx
->asid
);
621 dev_err_ratelimited(hdev
->dev
,
622 "Failed to switch to context %d, rejecting CS! %d\n",
625 * If we timedout, or if the device is not IDLE
626 * while we want to do context-switch (-EBUSY),
627 * we need to soft-reset because QMAN is
628 * probably stuck. However, we can't call to
629 * reset here directly because of deadlock, so
630 * need to do it at the very end of this
633 if ((rc
== -ETIMEDOUT
) || (rc
== -EBUSY
))
634 need_soft_reset
= true;
635 mutex_unlock(&hpriv
->restore_phase_mutex
);
640 hdev
->asic_funcs
->restore_phase_topology(hdev
);
642 if (num_chunks
== 0) {
644 "Need to run restore phase but restore CS is empty\n");
647 rc
= _hl_cs_ioctl(hpriv
, chunks
, num_chunks
,
651 mutex_unlock(&hpriv
->restore_phase_mutex
);
655 "Failed to submit restore CS for context %d (%d)\n",
660 /* Need to wait for restore completion before execution phase */
661 if (num_chunks
> 0) {
662 ret
= _hl_cs_wait_ioctl(hdev
, ctx
,
663 jiffies_to_usecs(hdev
->timeout_jiffies
),
667 "Restore CS for context %d failed to complete %ld\n",
674 ctx
->thread_restore_wait_token
= 1;
675 } else if (!ctx
->thread_restore_wait_token
) {
678 rc
= hl_poll_timeout_memory(hdev
,
679 (u64
) (uintptr_t) &ctx
->thread_restore_wait_token
,
680 jiffies_to_usecs(hdev
->timeout_jiffies
),
685 "restore phase hasn't finished in time\n");
691 chunks
= (void __user
*)(uintptr_t)args
->in
.chunks_execute
;
692 num_chunks
= args
->in
.num_chunks_execute
;
694 if (num_chunks
== 0) {
696 "Got execute CS with 0 chunks, context %d\n",
702 rc
= _hl_cs_ioctl(hpriv
, chunks
, num_chunks
, &cs_seq
);
706 memset(args
, 0, sizeof(*args
));
707 args
->out
.status
= rc
;
708 args
->out
.seq
= cs_seq
;
711 if (((rc
== -ETIMEDOUT
) || (rc
== -EBUSY
)) && (need_soft_reset
))
712 hl_device_reset(hdev
, false, false);
717 static long _hl_cs_wait_ioctl(struct hl_device
*hdev
,
718 struct hl_ctx
*ctx
, u64 timeout_us
, u64 seq
)
720 struct dma_fence
*fence
;
721 unsigned long timeout
;
724 if (timeout_us
== MAX_SCHEDULE_TIMEOUT
)
725 timeout
= timeout_us
;
727 timeout
= usecs_to_jiffies(timeout_us
);
729 hl_ctx_get(hdev
, ctx
);
731 fence
= hl_ctx_get_fence(ctx
, seq
);
735 rc
= dma_fence_wait_timeout(fence
, true, timeout
);
736 if (fence
->error
== -ETIMEDOUT
)
738 else if (fence
->error
== -EIO
)
740 dma_fence_put(fence
);
749 int hl_cs_wait_ioctl(struct hl_fpriv
*hpriv
, void *data
)
751 struct hl_device
*hdev
= hpriv
->hdev
;
752 union hl_wait_cs_args
*args
= data
;
753 u64 seq
= args
->in
.seq
;
756 rc
= _hl_cs_wait_ioctl(hdev
, hpriv
->ctx
, args
->in
.timeout_us
, seq
);
758 memset(args
, 0, sizeof(*args
));
761 dev_err(hdev
->dev
, "Error %ld on waiting for CS handle %llu\n",
763 if (rc
== -ERESTARTSYS
) {
764 args
->out
.status
= HL_WAIT_CS_STATUS_INTERRUPTED
;
766 } else if (rc
== -ETIMEDOUT
) {
767 args
->out
.status
= HL_WAIT_CS_STATUS_TIMEDOUT
;
768 } else if (rc
== -EIO
) {
769 args
->out
.status
= HL_WAIT_CS_STATUS_ABORTED
;
775 args
->out
.status
= HL_WAIT_CS_STATUS_BUSY
;
777 args
->out
.status
= HL_WAIT_CS_STATUS_COMPLETED
;