]>
Commit | Line | Data |
---|---|---|
c4d66343 OG |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | ||
3 | /* | |
4 | * Copyright 2016-2019 HabanaLabs, Ltd. | |
5 | * All Rights Reserved. | |
6 | */ | |
7 | ||
e00dac3d TT |
8 | #define pr_fmt(fmt) "habanalabs: " fmt |
9 | ||
c4d66343 OG |
10 | #include "habanalabs.h" |
11 | ||
12 | #include <linux/pci.h> | |
f8c8c7d5 | 13 | #include <linux/sched/signal.h> |
d91389bc | 14 | #include <linux/hwmon.h> |
aa957088 | 15 | #include <uapi/misc/habanalabs.h> |
c4d66343 | 16 | |
f650a95b OS |
17 | #define HL_PLDM_PENDING_RESET_PER_SEC (HL_PENDING_RESET_PER_SEC * 10) |
18 | ||
f8c8c7d5 OG |
19 | bool hl_device_disabled_or_in_reset(struct hl_device *hdev) |
20 | { | |
21 | if ((hdev->disabled) || (atomic_read(&hdev->in_reset))) | |
22 | return true; | |
23 | else | |
24 | return false; | |
25 | } | |
26 | ||
aa957088 DBZ |
27 | enum hl_device_status hl_device_status(struct hl_device *hdev) |
28 | { | |
29 | enum hl_device_status status; | |
30 | ||
31 | if (hdev->disabled) | |
32 | status = HL_DEVICE_STATUS_MALFUNCTION; | |
33 | else if (atomic_read(&hdev->in_reset)) | |
34 | status = HL_DEVICE_STATUS_IN_RESET; | |
35 | else | |
36 | status = HL_DEVICE_STATUS_OPERATIONAL; | |
37 | ||
38 | return status; | |
39 | }; | |
40 | ||
c4d66343 OG |
41 | static void hpriv_release(struct kref *ref) |
42 | { | |
43 | struct hl_fpriv *hpriv; | |
44 | struct hl_device *hdev; | |
45 | ||
46 | hpriv = container_of(ref, struct hl_fpriv, refcount); | |
47 | ||
48 | hdev = hpriv->hdev; | |
49 | ||
50 | put_pid(hpriv->taskpid); | |
51 | ||
c2164773 OG |
52 | hl_debugfs_remove_file(hpriv); |
53 | ||
eff6f4a0 OG |
54 | mutex_destroy(&hpriv->restore_phase_mutex); |
55 | ||
eb7caf84 OG |
56 | mutex_lock(&hdev->fpriv_list_lock); |
57 | list_del(&hpriv->dev_node); | |
86d5307a | 58 | hdev->compute_ctx = NULL; |
eb7caf84 OG |
59 | mutex_unlock(&hdev->fpriv_list_lock); |
60 | ||
61 | kfree(hpriv); | |
c4d66343 OG |
62 | } |
63 | ||
64 | void hl_hpriv_get(struct hl_fpriv *hpriv) | |
65 | { | |
66 | kref_get(&hpriv->refcount); | |
67 | } | |
68 | ||
69 | void hl_hpriv_put(struct hl_fpriv *hpriv) | |
70 | { | |
71 | kref_put(&hpriv->refcount, hpriv_release); | |
72 | } | |
73 | ||
74 | /* | |
75 | * hl_device_release - release function for habanalabs device | |
76 | * | |
77 | * @inode: pointer to inode structure | |
78 | * @filp: pointer to file structure | |
79 | * | |
80 | * Called when process closes an habanalabs device | |
81 | */ | |
82 | static int hl_device_release(struct inode *inode, struct file *filp) | |
83 | { | |
84 | struct hl_fpriv *hpriv = filp->private_data; | |
85 | ||
be5d926b | 86 | hl_cb_mgr_fini(hpriv->hdev, &hpriv->cb_mgr); |
0861e41d OG |
87 | hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr); |
88 | ||
c4d66343 OG |
89 | filp->private_data = NULL; |
90 | ||
91 | hl_hpriv_put(hpriv); | |
92 | ||
93 | return 0; | |
94 | } | |
95 | ||
4d6a7751 OG |
96 | static int hl_device_release_ctrl(struct inode *inode, struct file *filp) |
97 | { | |
98 | struct hl_fpriv *hpriv = filp->private_data; | |
99 | struct hl_device *hdev; | |
100 | ||
101 | filp->private_data = NULL; | |
102 | ||
103 | hdev = hpriv->hdev; | |
104 | ||
105 | mutex_lock(&hdev->fpriv_list_lock); | |
106 | list_del(&hpriv->dev_node); | |
107 | mutex_unlock(&hdev->fpriv_list_lock); | |
108 | ||
109 | kfree(hpriv); | |
110 | ||
111 | return 0; | |
112 | } | |
113 | ||
be5d926b OG |
114 | /* |
115 | * hl_mmap - mmap function for habanalabs device | |
116 | * | |
117 | * @*filp: pointer to file structure | |
118 | * @*vma: pointer to vm_area_struct of the process | |
119 | * | |
120 | * Called when process does an mmap on habanalabs device. Call the device's mmap | |
121 | * function at the end of the common code. | |
122 | */ | |
123 | static int hl_mmap(struct file *filp, struct vm_area_struct *vma) | |
124 | { | |
125 | struct hl_fpriv *hpriv = filp->private_data; | |
126 | ||
127 | if ((vma->vm_pgoff & HL_MMAP_CB_MASK) == HL_MMAP_CB_MASK) { | |
128 | vma->vm_pgoff ^= HL_MMAP_CB_MASK; | |
129 | return hl_cb_mmap(hpriv, vma); | |
130 | } | |
131 | ||
5e6e0239 | 132 | return -EINVAL; |
be5d926b OG |
133 | } |
134 | ||
c4d66343 OG |
135 | static const struct file_operations hl_ops = { |
136 | .owner = THIS_MODULE, | |
137 | .open = hl_device_open, | |
be5d926b OG |
138 | .release = hl_device_release, |
139 | .mmap = hl_mmap, | |
140 | .unlocked_ioctl = hl_ioctl, | |
141 | .compat_ioctl = hl_ioctl | |
c4d66343 OG |
142 | }; |
143 | ||
4d6a7751 OG |
144 | static const struct file_operations hl_ctrl_ops = { |
145 | .owner = THIS_MODULE, | |
146 | .open = hl_device_open_ctrl, | |
147 | .release = hl_device_release_ctrl, | |
148 | .unlocked_ioctl = hl_ioctl_control, | |
149 | .compat_ioctl = hl_ioctl_control | |
150 | }; | |
151 | ||
ea451f88 TT |
152 | static void device_release_func(struct device *dev) |
153 | { | |
154 | kfree(dev); | |
155 | } | |
156 | ||
c4d66343 | 157 | /* |
ea451f88 | 158 | * device_init_cdev - Initialize cdev and device for habanalabs device |
c4d66343 OG |
159 | * |
160 | * @hdev: pointer to habanalabs device structure | |
161 | * @hclass: pointer to the class object of the device | |
162 | * @minor: minor number of the specific device | |
b968eb1a OG |
163 | * @fpos: file operations to install for this device |
164 | * @name: name of the device as it will appear in the filesystem | |
ea451f88 TT |
165 | * @cdev: pointer to the char device object that will be initialized |
166 | * @dev: pointer to the device object that will be initialized | |
c4d66343 | 167 | * |
ea451f88 | 168 | * Initialize a cdev and a Linux device for habanalabs's device. |
c4d66343 | 169 | */ |
ea451f88 | 170 | static int device_init_cdev(struct hl_device *hdev, struct class *hclass, |
b968eb1a OG |
171 | int minor, const struct file_operations *fops, |
172 | char *name, struct cdev *cdev, | |
173 | struct device **dev) | |
c4d66343 | 174 | { |
b968eb1a OG |
175 | cdev_init(cdev, fops); |
176 | cdev->owner = THIS_MODULE; | |
ea451f88 TT |
177 | |
178 | *dev = kzalloc(sizeof(**dev), GFP_KERNEL); | |
179 | if (!*dev) | |
180 | return -ENOMEM; | |
181 | ||
182 | device_initialize(*dev); | |
183 | (*dev)->devt = MKDEV(hdev->major, minor); | |
184 | (*dev)->class = hclass; | |
185 | (*dev)->release = device_release_func; | |
186 | dev_set_drvdata(*dev, hdev); | |
187 | dev_set_name(*dev, "%s", name); | |
188 | ||
189 | return 0; | |
190 | } | |
191 | ||
192 | static int device_cdev_sysfs_add(struct hl_device *hdev) | |
193 | { | |
194 | int rc; | |
195 | ||
196 | rc = cdev_device_add(&hdev->cdev, hdev->dev); | |
197 | if (rc) { | |
198 | dev_err(hdev->dev, | |
199 | "failed to add a char device to the system\n"); | |
200 | return rc; | |
c4d66343 OG |
201 | } |
202 | ||
ea451f88 TT |
203 | rc = cdev_device_add(&hdev->cdev_ctrl, hdev->dev_ctrl); |
204 | if (rc) { | |
205 | dev_err(hdev->dev, | |
206 | "failed to add a control char device to the system\n"); | |
207 | goto delete_cdev_device; | |
c4d66343 OG |
208 | } |
209 | ||
ea451f88 TT |
210 | /* hl_sysfs_init() must be done after adding the device to the system */ |
211 | rc = hl_sysfs_init(hdev); | |
212 | if (rc) { | |
213 | dev_err(hdev->dev, "failed to initialize sysfs\n"); | |
214 | goto delete_ctrl_cdev_device; | |
215 | } | |
216 | ||
217 | hdev->cdev_sysfs_created = true; | |
c4d66343 OG |
218 | |
219 | return 0; | |
220 | ||
ea451f88 TT |
221 | delete_ctrl_cdev_device: |
222 | cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl); | |
223 | delete_cdev_device: | |
224 | cdev_device_del(&hdev->cdev, hdev->dev); | |
225 | return rc; | |
226 | } | |
227 | ||
228 | static void device_cdev_sysfs_del(struct hl_device *hdev) | |
229 | { | |
230 | /* device_release() won't be called so must free devices explicitly */ | |
231 | if (!hdev->cdev_sysfs_created) { | |
232 | kfree(hdev->dev_ctrl); | |
233 | kfree(hdev->dev); | |
234 | return; | |
235 | } | |
236 | ||
237 | hl_sysfs_fini(hdev); | |
238 | cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl); | |
239 | cdev_device_del(&hdev->cdev, hdev->dev); | |
c4d66343 OG |
240 | } |
241 | ||
242 | /* | |
243 | * device_early_init - do some early initialization for the habanalabs device | |
244 | * | |
245 | * @hdev: pointer to habanalabs device structure | |
246 | * | |
247 | * Install the relevant function pointers and call the early_init function, | |
248 | * if such a function exists | |
249 | */ | |
250 | static int device_early_init(struct hl_device *hdev) | |
251 | { | |
99b9d7b4 OG |
252 | int rc; |
253 | ||
c4d66343 OG |
254 | switch (hdev->asic_type) { |
255 | case ASIC_GOYA: | |
99b9d7b4 | 256 | goya_set_asic_funcs(hdev); |
c4d66343 OG |
257 | strlcpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name)); |
258 | break; | |
259 | default: | |
260 | dev_err(hdev->dev, "Unrecognized ASIC type %d\n", | |
261 | hdev->asic_type); | |
262 | return -EINVAL; | |
263 | } | |
264 | ||
99b9d7b4 OG |
265 | rc = hdev->asic_funcs->early_init(hdev); |
266 | if (rc) | |
267 | return rc; | |
268 | ||
0861e41d OG |
269 | rc = hl_asid_init(hdev); |
270 | if (rc) | |
271 | goto early_fini; | |
272 | ||
9494a8dd OG |
273 | hdev->cq_wq = alloc_workqueue("hl-free-jobs", WQ_UNBOUND, 0); |
274 | if (hdev->cq_wq == NULL) { | |
275 | dev_err(hdev->dev, "Failed to allocate CQ workqueue\n"); | |
276 | rc = -ENOMEM; | |
277 | goto asid_fini; | |
278 | } | |
279 | ||
1251f23a OG |
280 | hdev->eq_wq = alloc_workqueue("hl-events", WQ_UNBOUND, 0); |
281 | if (hdev->eq_wq == NULL) { | |
282 | dev_err(hdev->dev, "Failed to allocate EQ workqueue\n"); | |
283 | rc = -ENOMEM; | |
284 | goto free_cq_wq; | |
285 | } | |
286 | ||
d91389bc OG |
287 | hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info), |
288 | GFP_KERNEL); | |
289 | if (!hdev->hl_chip_info) { | |
290 | rc = -ENOMEM; | |
291 | goto free_eq_wq; | |
292 | } | |
293 | ||
75b3cb2b OG |
294 | hdev->idle_busy_ts_arr = kmalloc_array(HL_IDLE_BUSY_TS_ARR_SIZE, |
295 | sizeof(struct hl_device_idle_busy_ts), | |
296 | (GFP_KERNEL | __GFP_ZERO)); | |
297 | if (!hdev->idle_busy_ts_arr) { | |
298 | rc = -ENOMEM; | |
299 | goto free_chip_info; | |
300 | } | |
301 | ||
be5d926b OG |
302 | hl_cb_mgr_init(&hdev->kernel_cb_mgr); |
303 | ||
9494a8dd | 304 | mutex_init(&hdev->send_cpu_message_lock); |
19734970 | 305 | mutex_init(&hdev->debug_lock); |
8d45f1de | 306 | mutex_init(&hdev->mmu_cache_lock); |
eff6f4a0 OG |
307 | INIT_LIST_HEAD(&hdev->hw_queues_mirror_list); |
308 | spin_lock_init(&hdev->hw_queues_mirror_lock); | |
eb7caf84 OG |
309 | INIT_LIST_HEAD(&hdev->fpriv_list); |
310 | mutex_init(&hdev->fpriv_list_lock); | |
f8c8c7d5 | 311 | atomic_set(&hdev->in_reset, 0); |
0861e41d | 312 | |
c4d66343 | 313 | return 0; |
0861e41d | 314 | |
75b3cb2b OG |
315 | free_chip_info: |
316 | kfree(hdev->hl_chip_info); | |
d91389bc OG |
317 | free_eq_wq: |
318 | destroy_workqueue(hdev->eq_wq); | |
1251f23a OG |
319 | free_cq_wq: |
320 | destroy_workqueue(hdev->cq_wq); | |
9494a8dd OG |
321 | asid_fini: |
322 | hl_asid_fini(hdev); | |
0861e41d OG |
323 | early_fini: |
324 | if (hdev->asic_funcs->early_fini) | |
325 | hdev->asic_funcs->early_fini(hdev); | |
326 | ||
327 | return rc; | |
c4d66343 OG |
328 | } |
329 | ||
330 | /* | |
331 | * device_early_fini - finalize all that was done in device_early_init | |
332 | * | |
333 | * @hdev: pointer to habanalabs device structure | |
334 | * | |
335 | */ | |
336 | static void device_early_fini(struct hl_device *hdev) | |
337 | { | |
8d45f1de | 338 | mutex_destroy(&hdev->mmu_cache_lock); |
19734970 | 339 | mutex_destroy(&hdev->debug_lock); |
9494a8dd | 340 | mutex_destroy(&hdev->send_cpu_message_lock); |
99b9d7b4 | 341 | |
eb7caf84 OG |
342 | mutex_destroy(&hdev->fpriv_list_lock); |
343 | ||
be5d926b OG |
344 | hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr); |
345 | ||
75b3cb2b | 346 | kfree(hdev->idle_busy_ts_arr); |
d91389bc OG |
347 | kfree(hdev->hl_chip_info); |
348 | ||
1251f23a | 349 | destroy_workqueue(hdev->eq_wq); |
9494a8dd OG |
350 | destroy_workqueue(hdev->cq_wq); |
351 | ||
0861e41d OG |
352 | hl_asid_fini(hdev); |
353 | ||
99b9d7b4 OG |
354 | if (hdev->asic_funcs->early_fini) |
355 | hdev->asic_funcs->early_fini(hdev); | |
c4d66343 OG |
356 | } |
357 | ||
d91389bc OG |
358 | static void set_freq_to_low_job(struct work_struct *work) |
359 | { | |
360 | struct hl_device *hdev = container_of(work, struct hl_device, | |
361 | work_freq.work); | |
362 | ||
eb7caf84 OG |
363 | mutex_lock(&hdev->fpriv_list_lock); |
364 | ||
365 | if (!hdev->compute_ctx) | |
d91389bc OG |
366 | hl_device_set_frequency(hdev, PLL_LOW); |
367 | ||
eb7caf84 OG |
368 | mutex_unlock(&hdev->fpriv_list_lock); |
369 | ||
d91389bc OG |
370 | schedule_delayed_work(&hdev->work_freq, |
371 | usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC)); | |
372 | } | |
373 | ||
f8c8c7d5 OG |
374 | static void hl_device_heartbeat(struct work_struct *work) |
375 | { | |
376 | struct hl_device *hdev = container_of(work, struct hl_device, | |
377 | work_heartbeat.work); | |
378 | ||
379 | if (hl_device_disabled_or_in_reset(hdev)) | |
380 | goto reschedule; | |
381 | ||
382 | if (!hdev->asic_funcs->send_heartbeat(hdev)) | |
383 | goto reschedule; | |
384 | ||
385 | dev_err(hdev->dev, "Device heartbeat failed!\n"); | |
386 | hl_device_reset(hdev, true, false); | |
387 | ||
388 | return; | |
389 | ||
390 | reschedule: | |
391 | schedule_delayed_work(&hdev->work_heartbeat, | |
392 | usecs_to_jiffies(HL_HEARTBEAT_PER_USEC)); | |
393 | } | |
394 | ||
d91389bc OG |
395 | /* |
396 | * device_late_init - do late stuff initialization for the habanalabs device | |
397 | * | |
398 | * @hdev: pointer to habanalabs device structure | |
399 | * | |
400 | * Do stuff that either needs the device H/W queues to be active or needs | |
401 | * to happen after all the rest of the initialization is finished | |
402 | */ | |
403 | static int device_late_init(struct hl_device *hdev) | |
404 | { | |
405 | int rc; | |
406 | ||
0b28d26b OG |
407 | if (hdev->asic_funcs->late_init) { |
408 | rc = hdev->asic_funcs->late_init(hdev); | |
409 | if (rc) { | |
410 | dev_err(hdev->dev, | |
411 | "failed late initialization for the H/W\n"); | |
412 | return rc; | |
413 | } | |
414 | } | |
415 | ||
d91389bc OG |
416 | hdev->high_pll = hdev->asic_prop.high_pll; |
417 | ||
418 | /* force setting to low frequency */ | |
eb7caf84 | 419 | hdev->curr_pll_profile = PLL_LOW; |
d91389bc OG |
420 | |
421 | if (hdev->pm_mng_profile == PM_AUTO) | |
422 | hdev->asic_funcs->set_pll_profile(hdev, PLL_LOW); | |
423 | else | |
424 | hdev->asic_funcs->set_pll_profile(hdev, PLL_LAST); | |
425 | ||
0b28d26b | 426 | INIT_DELAYED_WORK(&hdev->work_freq, set_freq_to_low_job); |
d91389bc | 427 | schedule_delayed_work(&hdev->work_freq, |
0b28d26b | 428 | usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC)); |
d91389bc | 429 | |
f8c8c7d5 OG |
430 | if (hdev->heartbeat) { |
431 | INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat); | |
432 | schedule_delayed_work(&hdev->work_heartbeat, | |
433 | usecs_to_jiffies(HL_HEARTBEAT_PER_USEC)); | |
434 | } | |
435 | ||
d91389bc OG |
436 | hdev->late_init_done = true; |
437 | ||
438 | return 0; | |
439 | } | |
440 | ||
441 | /* | |
442 | * device_late_fini - finalize all that was done in device_late_init | |
443 | * | |
444 | * @hdev: pointer to habanalabs device structure | |
445 | * | |
446 | */ | |
447 | static void device_late_fini(struct hl_device *hdev) | |
448 | { | |
449 | if (!hdev->late_init_done) | |
450 | return; | |
451 | ||
452 | cancel_delayed_work_sync(&hdev->work_freq); | |
f8c8c7d5 OG |
453 | if (hdev->heartbeat) |
454 | cancel_delayed_work_sync(&hdev->work_heartbeat); | |
d91389bc OG |
455 | |
456 | if (hdev->asic_funcs->late_fini) | |
457 | hdev->asic_funcs->late_fini(hdev); | |
458 | ||
459 | hdev->late_init_done = false; | |
460 | } | |
461 | ||
75b3cb2b OG |
462 | uint32_t hl_device_utilization(struct hl_device *hdev, uint32_t period_ms) |
463 | { | |
464 | struct hl_device_idle_busy_ts *ts; | |
465 | ktime_t zero_ktime, curr = ktime_get(); | |
466 | u32 overlap_cnt = 0, last_index = hdev->idle_busy_ts_idx; | |
467 | s64 period_us, last_start_us, last_end_us, last_busy_time_us, | |
468 | total_busy_time_us = 0, total_busy_time_ms; | |
469 | ||
470 | zero_ktime = ktime_set(0, 0); | |
471 | period_us = period_ms * USEC_PER_MSEC; | |
472 | ts = &hdev->idle_busy_ts_arr[last_index]; | |
473 | ||
474 | /* check case that device is currently in idle */ | |
475 | if (!ktime_compare(ts->busy_to_idle_ts, zero_ktime) && | |
476 | !ktime_compare(ts->idle_to_busy_ts, zero_ktime)) { | |
477 | ||
478 | last_index--; | |
479 | /* Handle case idle_busy_ts_idx was 0 */ | |
480 | if (last_index > HL_IDLE_BUSY_TS_ARR_SIZE) | |
481 | last_index = HL_IDLE_BUSY_TS_ARR_SIZE - 1; | |
482 | ||
483 | ts = &hdev->idle_busy_ts_arr[last_index]; | |
484 | } | |
485 | ||
486 | while (overlap_cnt < HL_IDLE_BUSY_TS_ARR_SIZE) { | |
487 | /* Check if we are in last sample case. i.e. if the sample | |
488 | * begun before the sampling period. This could be a real | |
489 | * sample or 0 so need to handle both cases | |
490 | */ | |
491 | last_start_us = ktime_to_us( | |
492 | ktime_sub(curr, ts->idle_to_busy_ts)); | |
493 | ||
494 | if (last_start_us > period_us) { | |
495 | ||
496 | /* First check two cases: | |
497 | * 1. If the device is currently busy | |
498 | * 2. If the device was idle during the whole sampling | |
499 | * period | |
500 | */ | |
501 | ||
502 | if (!ktime_compare(ts->busy_to_idle_ts, zero_ktime)) { | |
503 | /* Check if the device is currently busy */ | |
504 | if (ktime_compare(ts->idle_to_busy_ts, | |
505 | zero_ktime)) | |
506 | return 100; | |
507 | ||
508 | /* We either didn't have any activity or we | |
509 | * reached an entry which is 0. Either way, | |
510 | * exit and return what was accumulated so far | |
511 | */ | |
512 | break; | |
513 | } | |
514 | ||
515 | /* If sample has finished, check it is relevant */ | |
516 | last_end_us = ktime_to_us( | |
517 | ktime_sub(curr, ts->busy_to_idle_ts)); | |
518 | ||
519 | if (last_end_us > period_us) | |
520 | break; | |
521 | ||
522 | /* It is relevant so add it but with adjustment */ | |
523 | last_busy_time_us = ktime_to_us( | |
524 | ktime_sub(ts->busy_to_idle_ts, | |
525 | ts->idle_to_busy_ts)); | |
526 | total_busy_time_us += last_busy_time_us - | |
527 | (last_start_us - period_us); | |
528 | break; | |
529 | } | |
530 | ||
531 | /* Check if the sample is finished or still open */ | |
532 | if (ktime_compare(ts->busy_to_idle_ts, zero_ktime)) | |
533 | last_busy_time_us = ktime_to_us( | |
534 | ktime_sub(ts->busy_to_idle_ts, | |
535 | ts->idle_to_busy_ts)); | |
536 | else | |
537 | last_busy_time_us = ktime_to_us( | |
538 | ktime_sub(curr, ts->idle_to_busy_ts)); | |
539 | ||
540 | total_busy_time_us += last_busy_time_us; | |
541 | ||
542 | last_index--; | |
543 | /* Handle case idle_busy_ts_idx was 0 */ | |
544 | if (last_index > HL_IDLE_BUSY_TS_ARR_SIZE) | |
545 | last_index = HL_IDLE_BUSY_TS_ARR_SIZE - 1; | |
546 | ||
547 | ts = &hdev->idle_busy_ts_arr[last_index]; | |
548 | ||
549 | overlap_cnt++; | |
550 | } | |
551 | ||
552 | total_busy_time_ms = DIV_ROUND_UP_ULL(total_busy_time_us, | |
553 | USEC_PER_MSEC); | |
554 | ||
555 | return DIV_ROUND_UP_ULL(total_busy_time_ms * 100, period_ms); | |
556 | } | |
557 | ||
d91389bc OG |
558 | /* |
559 | * hl_device_set_frequency - set the frequency of the device | |
560 | * | |
561 | * @hdev: pointer to habanalabs device structure | |
562 | * @freq: the new frequency value | |
563 | * | |
eb7caf84 OG |
564 | * Change the frequency if needed. This function has no protection against |
565 | * concurrency, therefore it is assumed that the calling function has protected | |
566 | * itself against the case of calling this function from multiple threads with | |
567 | * different values | |
568 | * | |
569 | * Returns 0 if no change was done, otherwise returns 1 | |
d91389bc OG |
570 | */ |
571 | int hl_device_set_frequency(struct hl_device *hdev, enum hl_pll_frequency freq) | |
572 | { | |
eb7caf84 OG |
573 | if ((hdev->pm_mng_profile == PM_MANUAL) || |
574 | (hdev->curr_pll_profile == freq)) | |
d91389bc OG |
575 | return 0; |
576 | ||
d91389bc OG |
577 | dev_dbg(hdev->dev, "Changing device frequency to %s\n", |
578 | freq == PLL_HIGH ? "high" : "low"); | |
579 | ||
580 | hdev->asic_funcs->set_pll_profile(hdev, freq); | |
581 | ||
eb7caf84 OG |
582 | hdev->curr_pll_profile = freq; |
583 | ||
d91389bc OG |
584 | return 1; |
585 | } | |
586 | ||
19734970 OG |
587 | int hl_device_set_debug_mode(struct hl_device *hdev, bool enable) |
588 | { | |
589 | int rc = 0; | |
590 | ||
591 | mutex_lock(&hdev->debug_lock); | |
592 | ||
593 | if (!enable) { | |
594 | if (!hdev->in_debug) { | |
595 | dev_err(hdev->dev, | |
596 | "Failed to disable debug mode because device was not in debug mode\n"); | |
597 | rc = -EFAULT; | |
598 | goto out; | |
599 | } | |
600 | ||
a37e4719 OS |
601 | if (!hdev->hard_reset_pending) |
602 | hdev->asic_funcs->halt_coresight(hdev); | |
603 | ||
19734970 OG |
604 | hdev->in_debug = 0; |
605 | ||
606 | goto out; | |
607 | } | |
608 | ||
609 | if (hdev->in_debug) { | |
610 | dev_err(hdev->dev, | |
611 | "Failed to enable debug mode because device is already in debug mode\n"); | |
612 | rc = -EFAULT; | |
613 | goto out; | |
614 | } | |
615 | ||
19734970 OG |
616 | hdev->in_debug = 1; |
617 | ||
19734970 OG |
618 | out: |
619 | mutex_unlock(&hdev->debug_lock); | |
620 | ||
621 | return rc; | |
622 | } | |
623 | ||
c4d66343 OG |
624 | /* |
625 | * hl_device_suspend - initiate device suspend | |
626 | * | |
627 | * @hdev: pointer to habanalabs device structure | |
628 | * | |
629 | * Puts the hw in the suspend state (all asics). | |
630 | * Returns 0 for success or an error on failure. | |
631 | * Called at driver suspend. | |
632 | */ | |
633 | int hl_device_suspend(struct hl_device *hdev) | |
634 | { | |
99b9d7b4 OG |
635 | int rc; |
636 | ||
c4d66343 OG |
637 | pci_save_state(hdev->pdev); |
638 | ||
7cb5101e OG |
639 | /* Block future CS/VM/JOB completion operations */ |
640 | rc = atomic_cmpxchg(&hdev->in_reset, 0, 1); | |
641 | if (rc) { | |
642 | dev_err(hdev->dev, "Can't suspend while in reset\n"); | |
643 | return -EIO; | |
644 | } | |
645 | ||
646 | /* This blocks all other stuff that is not blocked by in_reset */ | |
647 | hdev->disabled = true; | |
648 | ||
649 | /* | |
650 | * Flush anyone that is inside the critical section of enqueue | |
651 | * jobs to the H/W | |
652 | */ | |
653 | hdev->asic_funcs->hw_queues_lock(hdev); | |
654 | hdev->asic_funcs->hw_queues_unlock(hdev); | |
655 | ||
656 | /* Flush processes that are sending message to CPU */ | |
657 | mutex_lock(&hdev->send_cpu_message_lock); | |
658 | mutex_unlock(&hdev->send_cpu_message_lock); | |
659 | ||
99b9d7b4 OG |
660 | rc = hdev->asic_funcs->suspend(hdev); |
661 | if (rc) | |
662 | dev_err(hdev->dev, | |
663 | "Failed to disable PCI access of device CPU\n"); | |
664 | ||
c4d66343 OG |
665 | /* Shut down the device */ |
666 | pci_disable_device(hdev->pdev); | |
667 | pci_set_power_state(hdev->pdev, PCI_D3hot); | |
668 | ||
669 | return 0; | |
670 | } | |
671 | ||
672 | /* | |
673 | * hl_device_resume - initiate device resume | |
674 | * | |
675 | * @hdev: pointer to habanalabs device structure | |
676 | * | |
677 | * Bring the hw back to operating state (all asics). | |
678 | * Returns 0 for success or an error on failure. | |
679 | * Called at driver resume. | |
680 | */ | |
681 | int hl_device_resume(struct hl_device *hdev) | |
682 | { | |
683 | int rc; | |
684 | ||
685 | pci_set_power_state(hdev->pdev, PCI_D0); | |
686 | pci_restore_state(hdev->pdev); | |
7cb5101e | 687 | rc = pci_enable_device_mem(hdev->pdev); |
c4d66343 OG |
688 | if (rc) { |
689 | dev_err(hdev->dev, | |
690 | "Failed to enable PCI device in resume\n"); | |
691 | return rc; | |
692 | } | |
693 | ||
7cb5101e OG |
694 | pci_set_master(hdev->pdev); |
695 | ||
99b9d7b4 OG |
696 | rc = hdev->asic_funcs->resume(hdev); |
697 | if (rc) { | |
7cb5101e OG |
698 | dev_err(hdev->dev, "Failed to resume device after suspend\n"); |
699 | goto disable_device; | |
700 | } | |
701 | ||
702 | ||
703 | hdev->disabled = false; | |
704 | atomic_set(&hdev->in_reset, 0); | |
705 | ||
706 | rc = hl_device_reset(hdev, true, false); | |
707 | if (rc) { | |
708 | dev_err(hdev->dev, "Failed to reset device during resume\n"); | |
709 | goto disable_device; | |
99b9d7b4 OG |
710 | } |
711 | ||
c4d66343 | 712 | return 0; |
7cb5101e OG |
713 | |
714 | disable_device: | |
715 | pci_clear_master(hdev->pdev); | |
716 | pci_disable_device(hdev->pdev); | |
717 | ||
718 | return rc; | |
c4d66343 OG |
719 | } |
720 | ||
caa3c8e5 | 721 | static void device_kill_open_processes(struct hl_device *hdev) |
f8c8c7d5 | 722 | { |
f650a95b | 723 | u16 pending_total, pending_cnt; |
eb7caf84 | 724 | struct hl_fpriv *hpriv; |
f8c8c7d5 OG |
725 | struct task_struct *task = NULL; |
726 | ||
f650a95b OS |
727 | if (hdev->pldm) |
728 | pending_total = HL_PLDM_PENDING_RESET_PER_SEC; | |
729 | else | |
730 | pending_total = HL_PENDING_RESET_PER_SEC; | |
731 | ||
eb7caf84 OG |
732 | /* Giving time for user to close FD, and for processes that are inside |
733 | * hl_device_open to finish | |
734 | */ | |
735 | if (!list_empty(&hdev->fpriv_list)) | |
f8c8c7d5 | 736 | ssleep(1); |
f8c8c7d5 | 737 | |
eb7caf84 OG |
738 | mutex_lock(&hdev->fpriv_list_lock); |
739 | ||
740 | /* This section must be protected because we are dereferencing | |
741 | * pointers that are freed if the process exits | |
742 | */ | |
743 | list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node) { | |
744 | task = get_pid_task(hpriv->taskpid, PIDTYPE_PID); | |
f8c8c7d5 | 745 | if (task) { |
4d6a7751 OG |
746 | dev_info(hdev->dev, "Killing user process pid=%d\n", |
747 | task_pid_nr(task)); | |
f8c8c7d5 | 748 | send_sig(SIGKILL, task, 1); |
eb7caf84 | 749 | usleep_range(1000, 10000); |
f8c8c7d5 OG |
750 | |
751 | put_task_struct(task); | |
752 | } | |
753 | } | |
754 | ||
eb7caf84 OG |
755 | mutex_unlock(&hdev->fpriv_list_lock); |
756 | ||
caa3c8e5 OG |
757 | /* We killed the open users, but because the driver cleans up after the |
758 | * user contexts are closed (e.g. mmu mappings), we need to wait again | |
759 | * to make sure the cleaning phase is finished before continuing with | |
760 | * the reset | |
761 | */ | |
762 | ||
f650a95b OS |
763 | pending_cnt = pending_total; |
764 | ||
eb7caf84 OG |
765 | while ((!list_empty(&hdev->fpriv_list)) && (pending_cnt)) { |
766 | dev_info(hdev->dev, | |
767 | "Waiting for all unmap operations to finish before hard reset\n"); | |
f650a95b OS |
768 | |
769 | pending_cnt--; | |
770 | ||
771 | ssleep(1); | |
772 | } | |
773 | ||
eb7caf84 | 774 | if (!list_empty(&hdev->fpriv_list)) |
f650a95b OS |
775 | dev_crit(hdev->dev, |
776 | "Going to hard reset with open user contexts\n"); | |
caa3c8e5 OG |
777 | } |
778 | ||
779 | static void device_hard_reset_pending(struct work_struct *work) | |
780 | { | |
781 | struct hl_device_reset_work *device_reset_work = | |
782 | container_of(work, struct hl_device_reset_work, reset_work); | |
783 | struct hl_device *hdev = device_reset_work->hdev; | |
784 | ||
f8c8c7d5 OG |
785 | hl_device_reset(hdev, true, true); |
786 | ||
787 | kfree(device_reset_work); | |
788 | } | |
789 | ||
790 | /* | |
791 | * hl_device_reset - reset the device | |
792 | * | |
793 | * @hdev: pointer to habanalabs device structure | |
794 | * @hard_reset: should we do hard reset to all engines or just reset the | |
795 | * compute/dma engines | |
796 | * | |
797 | * Block future CS and wait for pending CS to be enqueued | |
798 | * Call ASIC H/W fini | |
799 | * Flush all completions | |
800 | * Re-initialize all internal data structures | |
801 | * Call ASIC H/W init, late_init | |
802 | * Test queues | |
803 | * Enable device | |
804 | * | |
805 | * Returns 0 for success or an error on failure. | |
806 | */ | |
807 | int hl_device_reset(struct hl_device *hdev, bool hard_reset, | |
808 | bool from_hard_reset_thread) | |
809 | { | |
810 | int i, rc; | |
811 | ||
812 | if (!hdev->init_done) { | |
813 | dev_err(hdev->dev, | |
814 | "Can't reset before initialization is done\n"); | |
815 | return 0; | |
816 | } | |
817 | ||
818 | /* | |
819 | * Prevent concurrency in this function - only one reset should be | |
820 | * done at any given time. Only need to perform this if we didn't | |
821 | * get from the dedicated hard reset thread | |
822 | */ | |
823 | if (!from_hard_reset_thread) { | |
824 | /* Block future CS/VM/JOB completion operations */ | |
825 | rc = atomic_cmpxchg(&hdev->in_reset, 0, 1); | |
826 | if (rc) | |
827 | return 0; | |
828 | ||
829 | /* This also blocks future CS/VM/JOB completion operations */ | |
830 | hdev->disabled = true; | |
831 | ||
eb7caf84 | 832 | /* Flush anyone that is inside the critical section of enqueue |
f8c8c7d5 OG |
833 | * jobs to the H/W |
834 | */ | |
835 | hdev->asic_funcs->hw_queues_lock(hdev); | |
836 | hdev->asic_funcs->hw_queues_unlock(hdev); | |
837 | ||
eb7caf84 OG |
838 | /* Flush anyone that is inside device open */ |
839 | mutex_lock(&hdev->fpriv_list_lock); | |
840 | mutex_unlock(&hdev->fpriv_list_lock); | |
841 | ||
f8c8c7d5 OG |
842 | dev_err(hdev->dev, "Going to RESET device!\n"); |
843 | } | |
844 | ||
845 | again: | |
846 | if ((hard_reset) && (!from_hard_reset_thread)) { | |
847 | struct hl_device_reset_work *device_reset_work; | |
848 | ||
3f5398cf OG |
849 | hdev->hard_reset_pending = true; |
850 | ||
f8c8c7d5 OG |
851 | device_reset_work = kzalloc(sizeof(*device_reset_work), |
852 | GFP_ATOMIC); | |
853 | if (!device_reset_work) { | |
854 | rc = -ENOMEM; | |
855 | goto out_err; | |
856 | } | |
857 | ||
858 | /* | |
859 | * Because the reset function can't run from interrupt or | |
860 | * from heartbeat work, we need to call the reset function | |
861 | * from a dedicated work | |
862 | */ | |
863 | INIT_WORK(&device_reset_work->reset_work, | |
caa3c8e5 | 864 | device_hard_reset_pending); |
f8c8c7d5 OG |
865 | device_reset_work->hdev = hdev; |
866 | schedule_work(&device_reset_work->reset_work); | |
867 | ||
868 | return 0; | |
869 | } | |
870 | ||
871 | if (hard_reset) { | |
872 | device_late_fini(hdev); | |
873 | ||
874 | /* | |
875 | * Now that the heartbeat thread is closed, flush processes | |
876 | * which are sending messages to CPU | |
877 | */ | |
878 | mutex_lock(&hdev->send_cpu_message_lock); | |
879 | mutex_unlock(&hdev->send_cpu_message_lock); | |
880 | } | |
881 | ||
882 | /* | |
883 | * Halt the engines and disable interrupts so we won't get any more | |
884 | * completions from H/W and we won't have any accesses from the | |
885 | * H/W to the host machine | |
886 | */ | |
887 | hdev->asic_funcs->halt_engines(hdev, hard_reset); | |
888 | ||
eff6f4a0 OG |
889 | /* Go over all the queues, release all CS and their jobs */ |
890 | hl_cs_rollback_all(hdev); | |
891 | ||
55f6d680 OG |
892 | if (hard_reset) { |
893 | /* Kill processes here after CS rollback. This is because the | |
894 | * process can't really exit until all its CSs are done, which | |
895 | * is what we do in cs rollback | |
896 | */ | |
4aecb05e OG |
897 | device_kill_open_processes(hdev); |
898 | ||
55f6d680 OG |
899 | /* Flush the Event queue workers to make sure no other thread is |
900 | * reading or writing to registers during the reset | |
901 | */ | |
902 | flush_workqueue(hdev->eq_wq); | |
903 | } | |
904 | ||
0878a420 OG |
905 | /* Release kernel context */ |
906 | if ((hard_reset) && (hl_ctx_put(hdev->kernel_ctx) == 1)) | |
f8c8c7d5 | 907 | hdev->kernel_ctx = NULL; |
f8c8c7d5 OG |
908 | |
909 | /* Reset the H/W. It will be in idle state after this returns */ | |
910 | hdev->asic_funcs->hw_fini(hdev, hard_reset); | |
911 | ||
0feaf86d OS |
912 | if (hard_reset) { |
913 | hl_vm_fini(hdev); | |
37d68ce5 | 914 | hl_mmu_fini(hdev); |
f8c8c7d5 | 915 | hl_eq_reset(hdev, &hdev->event_queue); |
0feaf86d | 916 | } |
f8c8c7d5 OG |
917 | |
918 | /* Re-initialize PI,CI to 0 in all queues (hw queue, cq) */ | |
919 | hl_hw_queue_reset(hdev, hard_reset); | |
920 | for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) | |
921 | hl_cq_reset(hdev, &hdev->completion_queue[i]); | |
922 | ||
75b3cb2b OG |
923 | hdev->idle_busy_ts_idx = 0; |
924 | hdev->idle_busy_ts_arr[0].busy_to_idle_ts = ktime_set(0, 0); | |
925 | hdev->idle_busy_ts_arr[0].idle_to_busy_ts = ktime_set(0, 0); | |
926 | ||
927 | if (hdev->cs_active_cnt) | |
928 | dev_crit(hdev->dev, "CS active cnt %d is not 0 during reset\n", | |
929 | hdev->cs_active_cnt); | |
930 | ||
eb7caf84 OG |
931 | mutex_lock(&hdev->fpriv_list_lock); |
932 | ||
027d35d0 | 933 | /* Make sure the context switch phase will run again */ |
86d5307a OG |
934 | if (hdev->compute_ctx) { |
935 | atomic_set(&hdev->compute_ctx->thread_ctx_switch_token, 1); | |
936 | hdev->compute_ctx->thread_ctx_switch_wait_token = 0; | |
eff6f4a0 OG |
937 | } |
938 | ||
eb7caf84 OG |
939 | mutex_unlock(&hdev->fpriv_list_lock); |
940 | ||
f8c8c7d5 OG |
941 | /* Finished tear-down, starting to re-initialize */ |
942 | ||
943 | if (hard_reset) { | |
a28ce422 | 944 | hdev->device_cpu_disabled = false; |
3f5398cf | 945 | hdev->hard_reset_pending = false; |
a28ce422 | 946 | |
0878a420 OG |
947 | if (hdev->kernel_ctx) { |
948 | dev_crit(hdev->dev, | |
949 | "kernel ctx was alive during hard reset, something is terribly wrong\n"); | |
950 | rc = -EBUSY; | |
951 | goto out_err; | |
952 | } | |
953 | ||
37d68ce5 OG |
954 | rc = hl_mmu_init(hdev); |
955 | if (rc) { | |
956 | dev_err(hdev->dev, | |
957 | "Failed to initialize MMU S/W after hard reset\n"); | |
958 | goto out_err; | |
959 | } | |
960 | ||
f8c8c7d5 OG |
961 | /* Allocate the kernel context */ |
962 | hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), | |
963 | GFP_KERNEL); | |
964 | if (!hdev->kernel_ctx) { | |
965 | rc = -ENOMEM; | |
966 | goto out_err; | |
967 | } | |
968 | ||
86d5307a | 969 | hdev->compute_ctx = NULL; |
f8c8c7d5 OG |
970 | |
971 | rc = hl_ctx_init(hdev, hdev->kernel_ctx, true); | |
972 | if (rc) { | |
973 | dev_err(hdev->dev, | |
974 | "failed to init kernel ctx in hard reset\n"); | |
975 | kfree(hdev->kernel_ctx); | |
976 | hdev->kernel_ctx = NULL; | |
977 | goto out_err; | |
978 | } | |
979 | } | |
980 | ||
981 | rc = hdev->asic_funcs->hw_init(hdev); | |
982 | if (rc) { | |
983 | dev_err(hdev->dev, | |
984 | "failed to initialize the H/W after reset\n"); | |
985 | goto out_err; | |
986 | } | |
987 | ||
988 | hdev->disabled = false; | |
989 | ||
990 | /* Check that the communication with the device is working */ | |
991 | rc = hdev->asic_funcs->test_queues(hdev); | |
992 | if (rc) { | |
993 | dev_err(hdev->dev, | |
994 | "Failed to detect if device is alive after reset\n"); | |
995 | goto out_err; | |
996 | } | |
997 | ||
998 | if (hard_reset) { | |
999 | rc = device_late_init(hdev); | |
1000 | if (rc) { | |
1001 | dev_err(hdev->dev, | |
1002 | "Failed late init after hard reset\n"); | |
1003 | goto out_err; | |
1004 | } | |
1005 | ||
0feaf86d OS |
1006 | rc = hl_vm_init(hdev); |
1007 | if (rc) { | |
1008 | dev_err(hdev->dev, | |
1009 | "Failed to init memory module after hard reset\n"); | |
1010 | goto out_err; | |
1011 | } | |
1012 | ||
f8c8c7d5 | 1013 | hl_set_max_power(hdev, hdev->max_power); |
f8c8c7d5 OG |
1014 | } else { |
1015 | rc = hdev->asic_funcs->soft_reset_late_init(hdev); | |
1016 | if (rc) { | |
1017 | dev_err(hdev->dev, | |
1018 | "Failed late init after soft reset\n"); | |
1019 | goto out_err; | |
1020 | } | |
1021 | } | |
1022 | ||
1023 | atomic_set(&hdev->in_reset, 0); | |
1024 | ||
1025 | if (hard_reset) | |
1026 | hdev->hard_reset_cnt++; | |
1027 | else | |
1028 | hdev->soft_reset_cnt++; | |
1029 | ||
867b58ac OG |
1030 | dev_warn(hdev->dev, "Successfully finished resetting the device\n"); |
1031 | ||
f8c8c7d5 OG |
1032 | return 0; |
1033 | ||
1034 | out_err: | |
1035 | hdev->disabled = true; | |
1036 | ||
1037 | if (hard_reset) { | |
1038 | dev_err(hdev->dev, | |
1039 | "Failed to reset! Device is NOT usable\n"); | |
1040 | hdev->hard_reset_cnt++; | |
1041 | } else { | |
1042 | dev_err(hdev->dev, | |
1043 | "Failed to do soft-reset, trying hard reset\n"); | |
1044 | hdev->soft_reset_cnt++; | |
1045 | hard_reset = true; | |
1046 | goto again; | |
1047 | } | |
1048 | ||
1049 | atomic_set(&hdev->in_reset, 0); | |
1050 | ||
1051 | return rc; | |
1052 | } | |
1053 | ||
c4d66343 OG |
1054 | /* |
1055 | * hl_device_init - main initialization function for habanalabs device | |
1056 | * | |
1057 | * @hdev: pointer to habanalabs device structure | |
1058 | * | |
1059 | * Allocate an id for the device, do early initialization and then call the | |
1060 | * ASIC specific initialization functions. Finally, create the cdev and the | |
1061 | * Linux device to expose it to the user | |
1062 | */ | |
1063 | int hl_device_init(struct hl_device *hdev, struct class *hclass) | |
1064 | { | |
9494a8dd | 1065 | int i, rc, cq_ready_cnt; |
b968eb1a | 1066 | char *name; |
ea451f88 | 1067 | bool add_cdev_sysfs_on_err = false; |
b968eb1a | 1068 | |
4d6a7751 | 1069 | name = kasprintf(GFP_KERNEL, "hl%d", hdev->id / 2); |
ea451f88 TT |
1070 | if (!name) { |
1071 | rc = -ENOMEM; | |
1072 | goto out_disabled; | |
1073 | } | |
c4d66343 | 1074 | |
ea451f88 TT |
1075 | /* Initialize cdev and device structures */ |
1076 | rc = device_init_cdev(hdev, hclass, hdev->id, &hl_ops, name, | |
b968eb1a OG |
1077 | &hdev->cdev, &hdev->dev); |
1078 | ||
1079 | kfree(name); | |
c4d66343 OG |
1080 | |
1081 | if (rc) | |
1082 | goto out_disabled; | |
1083 | ||
4d6a7751 OG |
1084 | name = kasprintf(GFP_KERNEL, "hl_controlD%d", hdev->id / 2); |
1085 | if (!name) { | |
1086 | rc = -ENOMEM; | |
ea451f88 | 1087 | goto free_dev; |
4d6a7751 OG |
1088 | } |
1089 | ||
ea451f88 TT |
1090 | /* Initialize cdev and device structures for control device */ |
1091 | rc = device_init_cdev(hdev, hclass, hdev->id_control, &hl_ctrl_ops, | |
4d6a7751 OG |
1092 | name, &hdev->cdev_ctrl, &hdev->dev_ctrl); |
1093 | ||
1094 | kfree(name); | |
1095 | ||
1096 | if (rc) | |
ea451f88 | 1097 | goto free_dev; |
4d6a7751 | 1098 | |
c4d66343 OG |
1099 | /* Initialize ASIC function pointers and perform early init */ |
1100 | rc = device_early_init(hdev); | |
1101 | if (rc) | |
ea451f88 | 1102 | goto free_dev_ctrl; |
c4d66343 | 1103 | |
99b9d7b4 OG |
1104 | /* |
1105 | * Start calling ASIC initialization. First S/W then H/W and finally | |
1106 | * late init | |
1107 | */ | |
1108 | rc = hdev->asic_funcs->sw_init(hdev); | |
1109 | if (rc) | |
1110 | goto early_fini; | |
1111 | ||
9494a8dd OG |
1112 | /* |
1113 | * Initialize the H/W queues. Must be done before hw_init, because | |
1114 | * there the addresses of the kernel queue are being written to the | |
1115 | * registers of the device | |
1116 | */ | |
1117 | rc = hl_hw_queues_create(hdev); | |
1118 | if (rc) { | |
1119 | dev_err(hdev->dev, "failed to initialize kernel queues\n"); | |
1120 | goto sw_fini; | |
1121 | } | |
1122 | ||
1123 | /* | |
1124 | * Initialize the completion queues. Must be done before hw_init, | |
1125 | * because there the addresses of the completion queues are being | |
1126 | * passed as arguments to request_irq | |
1127 | */ | |
1128 | hdev->completion_queue = | |
1129 | kcalloc(hdev->asic_prop.completion_queues_count, | |
1130 | sizeof(*hdev->completion_queue), GFP_KERNEL); | |
1131 | ||
1132 | if (!hdev->completion_queue) { | |
1133 | dev_err(hdev->dev, "failed to allocate completion queues\n"); | |
1134 | rc = -ENOMEM; | |
1135 | goto hw_queues_destroy; | |
1136 | } | |
1137 | ||
1138 | for (i = 0, cq_ready_cnt = 0; | |
1139 | i < hdev->asic_prop.completion_queues_count; | |
1140 | i++, cq_ready_cnt++) { | |
1141 | rc = hl_cq_init(hdev, &hdev->completion_queue[i], i); | |
1142 | if (rc) { | |
1143 | dev_err(hdev->dev, | |
1144 | "failed to initialize completion queue\n"); | |
1145 | goto cq_fini; | |
1146 | } | |
1147 | } | |
1148 | ||
1251f23a OG |
1149 | /* |
1150 | * Initialize the event queue. Must be done before hw_init, | |
1151 | * because there the address of the event queue is being | |
1152 | * passed as argument to request_irq | |
1153 | */ | |
1154 | rc = hl_eq_init(hdev, &hdev->event_queue); | |
1155 | if (rc) { | |
1156 | dev_err(hdev->dev, "failed to initialize event queue\n"); | |
1157 | goto cq_fini; | |
1158 | } | |
1159 | ||
37d68ce5 OG |
1160 | /* MMU S/W must be initialized before kernel context is created */ |
1161 | rc = hl_mmu_init(hdev); | |
1162 | if (rc) { | |
1163 | dev_err(hdev->dev, "Failed to initialize MMU S/W structures\n"); | |
1164 | goto eq_fini; | |
1165 | } | |
1166 | ||
0861e41d OG |
1167 | /* Allocate the kernel context */ |
1168 | hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL); | |
1169 | if (!hdev->kernel_ctx) { | |
1170 | rc = -ENOMEM; | |
37d68ce5 | 1171 | goto mmu_fini; |
0861e41d OG |
1172 | } |
1173 | ||
86d5307a | 1174 | hdev->compute_ctx = NULL; |
0861e41d OG |
1175 | |
1176 | rc = hl_ctx_init(hdev, hdev->kernel_ctx, true); | |
1177 | if (rc) { | |
1178 | dev_err(hdev->dev, "failed to initialize kernel context\n"); | |
508c5849 TT |
1179 | kfree(hdev->kernel_ctx); |
1180 | goto mmu_fini; | |
0861e41d OG |
1181 | } |
1182 | ||
be5d926b OG |
1183 | rc = hl_cb_pool_init(hdev); |
1184 | if (rc) { | |
1185 | dev_err(hdev->dev, "failed to initialize CB pool\n"); | |
1186 | goto release_ctx; | |
1187 | } | |
1188 | ||
c2164773 OG |
1189 | hl_debugfs_add_device(hdev); |
1190 | ||
f8c8c7d5 OG |
1191 | if (hdev->asic_funcs->get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) { |
1192 | dev_info(hdev->dev, | |
1193 | "H/W state is dirty, must reset before initializing\n"); | |
908087ff | 1194 | hdev->asic_funcs->halt_engines(hdev, true); |
f8c8c7d5 OG |
1195 | hdev->asic_funcs->hw_fini(hdev, true); |
1196 | } | |
1197 | ||
ea451f88 TT |
1198 | /* |
1199 | * From this point, in case of an error, add char devices and create | |
1200 | * sysfs nodes as part of the error flow, to allow debugging. | |
1201 | */ | |
1202 | add_cdev_sysfs_on_err = true; | |
1203 | ||
839c4803 OG |
1204 | rc = hdev->asic_funcs->hw_init(hdev); |
1205 | if (rc) { | |
1206 | dev_err(hdev->dev, "failed to initialize the H/W\n"); | |
1207 | rc = 0; | |
1208 | goto out_disabled; | |
1209 | } | |
1210 | ||
1211 | hdev->disabled = false; | |
1212 | ||
9494a8dd OG |
1213 | /* Check that the communication with the device is working */ |
1214 | rc = hdev->asic_funcs->test_queues(hdev); | |
1215 | if (rc) { | |
1216 | dev_err(hdev->dev, "Failed to detect if device is alive\n"); | |
1217 | rc = 0; | |
1218 | goto out_disabled; | |
1219 | } | |
1220 | ||
d91389bc OG |
1221 | rc = device_late_init(hdev); |
1222 | if (rc) { | |
1223 | dev_err(hdev->dev, "Failed late initialization\n"); | |
1224 | rc = 0; | |
1225 | goto out_disabled; | |
1226 | } | |
1227 | ||
1228 | dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n", | |
1229 | hdev->asic_name, | |
1230 | hdev->asic_prop.dram_size / 1024 / 1024 / 1024); | |
1231 | ||
0feaf86d OS |
1232 | rc = hl_vm_init(hdev); |
1233 | if (rc) { | |
1234 | dev_err(hdev->dev, "Failed to initialize memory module\n"); | |
1235 | rc = 0; | |
1236 | goto out_disabled; | |
1237 | } | |
1238 | ||
d91389bc | 1239 | /* |
ea451f88 TT |
1240 | * Expose devices and sysfs nodes to user. |
1241 | * From here there is no need to add char devices and create sysfs nodes | |
1242 | * in case of an error. | |
1243 | */ | |
1244 | add_cdev_sysfs_on_err = false; | |
1245 | rc = device_cdev_sysfs_add(hdev); | |
1246 | if (rc) { | |
1247 | dev_err(hdev->dev, | |
1248 | "Failed to add char devices and sysfs nodes\n"); | |
1249 | rc = 0; | |
1250 | goto out_disabled; | |
1251 | } | |
1252 | ||
1253 | /* | |
1254 | * hl_hwmon_init() must be called after device_late_init(), because only | |
d91389bc | 1255 | * there we get the information from the device about which |
ea451f88 TT |
1256 | * hwmon-related sensors the device supports. |
1257 | * Furthermore, it must be done after adding the device to the system. | |
d91389bc OG |
1258 | */ |
1259 | rc = hl_hwmon_init(hdev); | |
1260 | if (rc) { | |
1261 | dev_err(hdev->dev, "Failed to initialize hwmon\n"); | |
1262 | rc = 0; | |
1263 | goto out_disabled; | |
1264 | } | |
1265 | ||
c4d66343 OG |
1266 | dev_notice(hdev->dev, |
1267 | "Successfully added device to habanalabs driver\n"); | |
1268 | ||
f8c8c7d5 OG |
1269 | hdev->init_done = true; |
1270 | ||
c4d66343 OG |
1271 | return 0; |
1272 | ||
be5d926b OG |
1273 | release_ctx: |
1274 | if (hl_ctx_put(hdev->kernel_ctx) != 1) | |
1275 | dev_err(hdev->dev, | |
1276 | "kernel ctx is still alive on initialization failure\n"); | |
37d68ce5 OG |
1277 | mmu_fini: |
1278 | hl_mmu_fini(hdev); | |
1251f23a OG |
1279 | eq_fini: |
1280 | hl_eq_fini(hdev, &hdev->event_queue); | |
9494a8dd OG |
1281 | cq_fini: |
1282 | for (i = 0 ; i < cq_ready_cnt ; i++) | |
1283 | hl_cq_fini(hdev, &hdev->completion_queue[i]); | |
1284 | kfree(hdev->completion_queue); | |
1285 | hw_queues_destroy: | |
1286 | hl_hw_queues_destroy(hdev); | |
0861e41d OG |
1287 | sw_fini: |
1288 | hdev->asic_funcs->sw_fini(hdev); | |
99b9d7b4 OG |
1289 | early_fini: |
1290 | device_early_fini(hdev); | |
ea451f88 TT |
1291 | free_dev_ctrl: |
1292 | kfree(hdev->dev_ctrl); | |
1293 | free_dev: | |
1294 | kfree(hdev->dev); | |
c4d66343 OG |
1295 | out_disabled: |
1296 | hdev->disabled = true; | |
ea451f88 TT |
1297 | if (add_cdev_sysfs_on_err) |
1298 | device_cdev_sysfs_add(hdev); | |
c4d66343 OG |
1299 | if (hdev->pdev) |
1300 | dev_err(&hdev->pdev->dev, | |
1301 | "Failed to initialize hl%d. Device is NOT usable !\n", | |
307eae93 | 1302 | hdev->id / 2); |
c4d66343 OG |
1303 | else |
1304 | pr_err("Failed to initialize hl%d. Device is NOT usable !\n", | |
307eae93 | 1305 | hdev->id / 2); |
c4d66343 OG |
1306 | |
1307 | return rc; | |
1308 | } | |
1309 | ||
1310 | /* | |
1311 | * hl_device_fini - main tear-down function for habanalabs device | |
1312 | * | |
1313 | * @hdev: pointer to habanalabs device structure | |
1314 | * | |
1315 | * Destroy the device, call ASIC fini functions and release the id | |
1316 | */ | |
1317 | void hl_device_fini(struct hl_device *hdev) | |
1318 | { | |
f8c8c7d5 OG |
1319 | int i, rc; |
1320 | ktime_t timeout; | |
1321 | ||
c4d66343 OG |
1322 | dev_info(hdev->dev, "Removing device\n"); |
1323 | ||
f8c8c7d5 OG |
1324 | /* |
1325 | * This function is competing with the reset function, so try to | |
1326 | * take the reset atomic and if we are already in middle of reset, | |
1327 | * wait until reset function is finished. Reset function is designed | |
1328 | * to always finish (could take up to a few seconds in worst case). | |
1329 | */ | |
1330 | ||
1331 | timeout = ktime_add_us(ktime_get(), | |
1332 | HL_PENDING_RESET_PER_SEC * 1000 * 1000 * 4); | |
1333 | rc = atomic_cmpxchg(&hdev->in_reset, 0, 1); | |
1334 | while (rc) { | |
1335 | usleep_range(50, 200); | |
1336 | rc = atomic_cmpxchg(&hdev->in_reset, 0, 1); | |
1337 | if (ktime_compare(ktime_get(), timeout) > 0) { | |
1338 | WARN(1, "Failed to remove device because reset function did not finish\n"); | |
1339 | return; | |
1340 | } | |
a1c92d1c | 1341 | } |
f8c8c7d5 | 1342 | |
c4d66343 OG |
1343 | /* Mark device as disabled */ |
1344 | hdev->disabled = true; | |
1345 | ||
eb7caf84 | 1346 | /* Flush anyone that is inside the critical section of enqueue |
caa3c8e5 OG |
1347 | * jobs to the H/W |
1348 | */ | |
1349 | hdev->asic_funcs->hw_queues_lock(hdev); | |
1350 | hdev->asic_funcs->hw_queues_unlock(hdev); | |
1351 | ||
eb7caf84 OG |
1352 | /* Flush anyone that is inside device open */ |
1353 | mutex_lock(&hdev->fpriv_list_lock); | |
1354 | mutex_unlock(&hdev->fpriv_list_lock); | |
1355 | ||
3f5398cf OG |
1356 | hdev->hard_reset_pending = true; |
1357 | ||
d91389bc OG |
1358 | hl_hwmon_fini(hdev); |
1359 | ||
1360 | device_late_fini(hdev); | |
1361 | ||
c2164773 OG |
1362 | hl_debugfs_remove_device(hdev); |
1363 | ||
1251f23a OG |
1364 | /* |
1365 | * Halt the engines and disable interrupts so we won't get any more | |
1366 | * completions from H/W and we won't have any accesses from the | |
1367 | * H/W to the host machine | |
1368 | */ | |
1369 | hdev->asic_funcs->halt_engines(hdev, true); | |
1370 | ||
eff6f4a0 OG |
1371 | /* Go over all the queues, release all CS and their jobs */ |
1372 | hl_cs_rollback_all(hdev); | |
1373 | ||
4aecb05e OG |
1374 | /* Kill processes here after CS rollback. This is because the process |
1375 | * can't really exit until all its CSs are done, which is what we | |
1376 | * do in cs rollback | |
1377 | */ | |
1378 | device_kill_open_processes(hdev); | |
1379 | ||
be5d926b OG |
1380 | hl_cb_pool_fini(hdev); |
1381 | ||
0861e41d OG |
1382 | /* Release kernel context */ |
1383 | if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1)) | |
1384 | dev_err(hdev->dev, "kernel ctx is still alive\n"); | |
1385 | ||
839c4803 OG |
1386 | /* Reset the H/W. It will be in idle state after this returns */ |
1387 | hdev->asic_funcs->hw_fini(hdev, true); | |
1388 | ||
0feaf86d OS |
1389 | hl_vm_fini(hdev); |
1390 | ||
37d68ce5 OG |
1391 | hl_mmu_fini(hdev); |
1392 | ||
1251f23a OG |
1393 | hl_eq_fini(hdev, &hdev->event_queue); |
1394 | ||
9494a8dd OG |
1395 | for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) |
1396 | hl_cq_fini(hdev, &hdev->completion_queue[i]); | |
1397 | kfree(hdev->completion_queue); | |
1398 | ||
1399 | hl_hw_queues_destroy(hdev); | |
1400 | ||
99b9d7b4 OG |
1401 | /* Call ASIC S/W finalize function */ |
1402 | hdev->asic_funcs->sw_fini(hdev); | |
1403 | ||
c4d66343 OG |
1404 | device_early_fini(hdev); |
1405 | ||
ea451f88 TT |
1406 | /* Hide devices and sysfs nodes from user */ |
1407 | device_cdev_sysfs_del(hdev); | |
c4d66343 OG |
1408 | |
1409 | pr_info("removed device successfully\n"); | |
1410 | } | |
1411 | ||
99b9d7b4 OG |
1412 | /* |
1413 | * MMIO register access helper functions. | |
1414 | */ | |
1415 | ||
1416 | /* | |
1417 | * hl_rreg - Read an MMIO register | |
1418 | * | |
1419 | * @hdev: pointer to habanalabs device structure | |
1420 | * @reg: MMIO register offset (in bytes) | |
1421 | * | |
1422 | * Returns the value of the MMIO register we are asked to read | |
1423 | * | |
1424 | */ | |
1425 | inline u32 hl_rreg(struct hl_device *hdev, u32 reg) | |
1426 | { | |
1427 | return readl(hdev->rmmio + reg); | |
1428 | } | |
1429 | ||
1430 | /* | |
1431 | * hl_wreg - Write to an MMIO register | |
1432 | * | |
1433 | * @hdev: pointer to habanalabs device structure | |
1434 | * @reg: MMIO register offset (in bytes) | |
1435 | * @val: 32-bit value | |
1436 | * | |
1437 | * Writes the 32-bit value into the MMIO register | |
1438 | * | |
1439 | */ | |
1440 | inline void hl_wreg(struct hl_device *hdev, u32 reg, u32 val) | |
1441 | { | |
1442 | writel(val, hdev->rmmio + reg); | |
1443 | } |