1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
8 #define pr_fmt(fmt) "habanalabs: " fmt
10 #include "habanalabs.h"
12 #include <linux/pci.h>
13 #include <linux/sched/signal.h>
14 #include <linux/hwmon.h>
15 #include <uapi/misc/habanalabs.h>
17 #define HL_PLDM_PENDING_RESET_PER_SEC (HL_PENDING_RESET_PER_SEC * 10)
19 bool hl_device_disabled_or_in_reset(struct hl_device *hdev)
21 if ((hdev->disabled) || (atomic_read(&hdev->in_reset)))
27 enum hl_device_status hl_device_status(struct hl_device *hdev)
29 enum hl_device_status status;
32 status = HL_DEVICE_STATUS_MALFUNCTION;
33 else if (atomic_read(&hdev->in_reset))
34 status = HL_DEVICE_STATUS_IN_RESET;
36 status = HL_DEVICE_STATUS_OPERATIONAL;
41 static void hpriv_release(struct kref *ref)
43 struct hl_fpriv *hpriv;
44 struct hl_device *hdev;
46 hpriv = container_of(ref, struct hl_fpriv, refcount);
50 put_pid(hpriv->taskpid);
52 hl_debugfs_remove_file(hpriv);
54 mutex_destroy(&hpriv->restore_phase_mutex);
56 mutex_lock(&hdev->fpriv_list_lock);
57 list_del(&hpriv->dev_node);
58 hdev->compute_ctx = NULL;
59 mutex_unlock(&hdev->fpriv_list_lock);
64 void hl_hpriv_get(struct hl_fpriv *hpriv)
66 kref_get(&hpriv->refcount);
69 void hl_hpriv_put(struct hl_fpriv *hpriv)
71 kref_put(&hpriv->refcount, hpriv_release);
75 * hl_device_release - release function for habanalabs device
77 * @inode: pointer to inode structure
78 * @filp: pointer to file structure
80 * Called when process closes an habanalabs device
82 static int hl_device_release(struct inode *inode, struct file *filp)
84 struct hl_fpriv *hpriv = filp->private_data;
86 hl_cb_mgr_fini(hpriv->hdev, &hpriv->cb_mgr);
87 hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);
89 filp->private_data = NULL;
96 static int hl_device_release_ctrl(struct inode *inode, struct file *filp)
98 struct hl_fpriv *hpriv = filp->private_data;
99 struct hl_device *hdev;
101 filp->private_data = NULL;
105 mutex_lock(&hdev->fpriv_list_lock);
106 list_del(&hpriv->dev_node);
107 mutex_unlock(&hdev->fpriv_list_lock);
109 put_pid(hpriv->taskpid);
117 * hl_mmap - mmap function for habanalabs device
119 * @*filp: pointer to file structure
120 * @*vma: pointer to vm_area_struct of the process
122 * Called when process does an mmap on habanalabs device. Call the device's mmap
123 * function at the end of the common code.
125 static int hl_mmap(struct file *filp, struct vm_area_struct *vma)
127 struct hl_fpriv *hpriv = filp->private_data;
128 unsigned long vm_pgoff;
130 vm_pgoff = vma->vm_pgoff;
131 vma->vm_pgoff = HL_MMAP_OFFSET_VALUE_GET(vm_pgoff);
133 switch (vm_pgoff & HL_MMAP_TYPE_MASK) {
134 case HL_MMAP_TYPE_CB:
135 return hl_cb_mmap(hpriv, vma);
141 static const struct file_operations hl_ops = {
142 .owner = THIS_MODULE,
143 .open = hl_device_open,
144 .release = hl_device_release,
146 .unlocked_ioctl = hl_ioctl,
147 .compat_ioctl = hl_ioctl
150 static const struct file_operations hl_ctrl_ops = {
151 .owner = THIS_MODULE,
152 .open = hl_device_open_ctrl,
153 .release = hl_device_release_ctrl,
154 .unlocked_ioctl = hl_ioctl_control,
155 .compat_ioctl = hl_ioctl_control
158 static void device_release_func(struct device *dev)
164 * device_init_cdev - Initialize cdev and device for habanalabs device
166 * @hdev: pointer to habanalabs device structure
167 * @hclass: pointer to the class object of the device
168 * @minor: minor number of the specific device
169 * @fpos: file operations to install for this device
170 * @name: name of the device as it will appear in the filesystem
171 * @cdev: pointer to the char device object that will be initialized
172 * @dev: pointer to the device object that will be initialized
174 * Initialize a cdev and a Linux device for habanalabs's device.
176 static int device_init_cdev(struct hl_device *hdev, struct class *hclass,
177 int minor, const struct file_operations *fops,
178 char *name, struct cdev *cdev,
181 cdev_init(cdev, fops);
182 cdev->owner = THIS_MODULE;
184 *dev = kzalloc(sizeof(**dev), GFP_KERNEL);
188 device_initialize(*dev);
189 (*dev)->devt = MKDEV(hdev->major, minor);
190 (*dev)->class = hclass;
191 (*dev)->release = device_release_func;
192 dev_set_drvdata(*dev, hdev);
193 dev_set_name(*dev, "%s", name);
198 static int device_cdev_sysfs_add(struct hl_device *hdev)
202 rc = cdev_device_add(&hdev->cdev, hdev->dev);
205 "failed to add a char device to the system\n");
209 rc = cdev_device_add(&hdev->cdev_ctrl, hdev->dev_ctrl);
212 "failed to add a control char device to the system\n");
213 goto delete_cdev_device;
216 /* hl_sysfs_init() must be done after adding the device to the system */
217 rc = hl_sysfs_init(hdev);
219 dev_err(hdev->dev, "failed to initialize sysfs\n");
220 goto delete_ctrl_cdev_device;
223 hdev->cdev_sysfs_created = true;
227 delete_ctrl_cdev_device:
228 cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
230 cdev_device_del(&hdev->cdev, hdev->dev);
234 static void device_cdev_sysfs_del(struct hl_device *hdev)
236 if (!hdev->cdev_sysfs_created)
240 cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
241 cdev_device_del(&hdev->cdev, hdev->dev);
244 put_device(hdev->dev);
245 put_device(hdev->dev_ctrl);
249 * device_early_init - do some early initialization for the habanalabs device
251 * @hdev: pointer to habanalabs device structure
253 * Install the relevant function pointers and call the early_init function,
254 * if such a function exists
256 static int device_early_init(struct hl_device *hdev)
261 switch (hdev->asic_type) {
263 goya_set_asic_funcs(hdev);
264 strlcpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name));
267 gaudi_set_asic_funcs(hdev);
268 sprintf(hdev->asic_name, "GAUDI");
271 dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
276 rc = hdev->asic_funcs->early_init(hdev);
280 rc = hl_asid_init(hdev);
284 if (hdev->asic_prop.completion_queues_count) {
285 hdev->cq_wq = kcalloc(hdev->asic_prop.completion_queues_count,
286 sizeof(*hdev->cq_wq),
294 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) {
295 snprintf(workq_name, 32, "hl-free-jobs-%u", (u32) i);
296 hdev->cq_wq[i] = create_singlethread_workqueue(workq_name);
297 if (hdev->cq_wq[i] == NULL) {
298 dev_err(hdev->dev, "Failed to allocate CQ workqueue\n");
304 hdev->eq_wq = alloc_workqueue("hl-events", WQ_UNBOUND, 0);
305 if (hdev->eq_wq == NULL) {
306 dev_err(hdev->dev, "Failed to allocate EQ workqueue\n");
311 hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info),
313 if (!hdev->hl_chip_info) {
318 hdev->idle_busy_ts_arr = kmalloc_array(HL_IDLE_BUSY_TS_ARR_SIZE,
319 sizeof(struct hl_device_idle_busy_ts),
320 (GFP_KERNEL | __GFP_ZERO));
321 if (!hdev->idle_busy_ts_arr) {
326 rc = hl_mmu_if_set_funcs(hdev);
328 goto free_idle_busy_ts_arr;
330 hl_cb_mgr_init(&hdev->kernel_cb_mgr);
332 mutex_init(&hdev->send_cpu_message_lock);
333 mutex_init(&hdev->debug_lock);
334 mutex_init(&hdev->mmu_cache_lock);
335 INIT_LIST_HEAD(&hdev->hw_queues_mirror_list);
336 spin_lock_init(&hdev->hw_queues_mirror_lock);
337 INIT_LIST_HEAD(&hdev->fpriv_list);
338 mutex_init(&hdev->fpriv_list_lock);
339 atomic_set(&hdev->in_reset, 0);
343 free_idle_busy_ts_arr:
344 kfree(hdev->idle_busy_ts_arr);
346 kfree(hdev->hl_chip_info);
348 destroy_workqueue(hdev->eq_wq);
350 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
352 destroy_workqueue(hdev->cq_wq[i]);
357 if (hdev->asic_funcs->early_fini)
358 hdev->asic_funcs->early_fini(hdev);
364 * device_early_fini - finalize all that was done in device_early_init
366 * @hdev: pointer to habanalabs device structure
369 static void device_early_fini(struct hl_device *hdev)
373 mutex_destroy(&hdev->mmu_cache_lock);
374 mutex_destroy(&hdev->debug_lock);
375 mutex_destroy(&hdev->send_cpu_message_lock);
377 mutex_destroy(&hdev->fpriv_list_lock);
379 hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr);
381 kfree(hdev->idle_busy_ts_arr);
382 kfree(hdev->hl_chip_info);
384 destroy_workqueue(hdev->eq_wq);
386 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
387 destroy_workqueue(hdev->cq_wq[i]);
392 if (hdev->asic_funcs->early_fini)
393 hdev->asic_funcs->early_fini(hdev);
396 static void set_freq_to_low_job(struct work_struct *work)
398 struct hl_device *hdev = container_of(work, struct hl_device,
401 mutex_lock(&hdev->fpriv_list_lock);
403 if (!hdev->compute_ctx)
404 hl_device_set_frequency(hdev, PLL_LOW);
406 mutex_unlock(&hdev->fpriv_list_lock);
408 schedule_delayed_work(&hdev->work_freq,
409 usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC));
412 static void hl_device_heartbeat(struct work_struct *work)
414 struct hl_device *hdev = container_of(work, struct hl_device,
415 work_heartbeat.work);
417 if (hl_device_disabled_or_in_reset(hdev))
420 if (!hdev->asic_funcs->send_heartbeat(hdev))
423 dev_err(hdev->dev, "Device heartbeat failed!\n");
424 hl_device_reset(hdev, true, false);
429 schedule_delayed_work(&hdev->work_heartbeat,
430 usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
434 * device_late_init - do late stuff initialization for the habanalabs device
436 * @hdev: pointer to habanalabs device structure
438 * Do stuff that either needs the device H/W queues to be active or needs
439 * to happen after all the rest of the initialization is finished
441 static int device_late_init(struct hl_device *hdev)
445 if (hdev->asic_funcs->late_init) {
446 rc = hdev->asic_funcs->late_init(hdev);
449 "failed late initialization for the H/W\n");
454 hdev->high_pll = hdev->asic_prop.high_pll;
456 /* force setting to low frequency */
457 hdev->curr_pll_profile = PLL_LOW;
459 if (hdev->pm_mng_profile == PM_AUTO)
460 hdev->asic_funcs->set_pll_profile(hdev, PLL_LOW);
462 hdev->asic_funcs->set_pll_profile(hdev, PLL_LAST);
464 INIT_DELAYED_WORK(&hdev->work_freq, set_freq_to_low_job);
465 schedule_delayed_work(&hdev->work_freq,
466 usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC));
468 if (hdev->heartbeat) {
469 INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat);
470 schedule_delayed_work(&hdev->work_heartbeat,
471 usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
474 hdev->late_init_done = true;
480 * device_late_fini - finalize all that was done in device_late_init
482 * @hdev: pointer to habanalabs device structure
485 static void device_late_fini(struct hl_device *hdev)
487 if (!hdev->late_init_done)
490 cancel_delayed_work_sync(&hdev->work_freq);
492 cancel_delayed_work_sync(&hdev->work_heartbeat);
494 if (hdev->asic_funcs->late_fini)
495 hdev->asic_funcs->late_fini(hdev);
497 hdev->late_init_done = false;
500 uint32_t hl_device_utilization(struct hl_device *hdev, uint32_t period_ms)
502 struct hl_device_idle_busy_ts *ts;
503 ktime_t zero_ktime, curr = ktime_get();
504 u32 overlap_cnt = 0, last_index = hdev->idle_busy_ts_idx;
505 s64 period_us, last_start_us, last_end_us, last_busy_time_us,
506 total_busy_time_us = 0, total_busy_time_ms;
508 zero_ktime = ktime_set(0, 0);
509 period_us = period_ms * USEC_PER_MSEC;
510 ts = &hdev->idle_busy_ts_arr[last_index];
512 /* check case that device is currently in idle */
513 if (!ktime_compare(ts->busy_to_idle_ts, zero_ktime) &&
514 !ktime_compare(ts->idle_to_busy_ts, zero_ktime)) {
517 /* Handle case idle_busy_ts_idx was 0 */
518 if (last_index > HL_IDLE_BUSY_TS_ARR_SIZE)
519 last_index = HL_IDLE_BUSY_TS_ARR_SIZE - 1;
521 ts = &hdev->idle_busy_ts_arr[last_index];
524 while (overlap_cnt < HL_IDLE_BUSY_TS_ARR_SIZE) {
525 /* Check if we are in last sample case. i.e. if the sample
526 * begun before the sampling period. This could be a real
527 * sample or 0 so need to handle both cases
529 last_start_us = ktime_to_us(
530 ktime_sub(curr, ts->idle_to_busy_ts));
532 if (last_start_us > period_us) {
534 /* First check two cases:
535 * 1. If the device is currently busy
536 * 2. If the device was idle during the whole sampling
540 if (!ktime_compare(ts->busy_to_idle_ts, zero_ktime)) {
541 /* Check if the device is currently busy */
542 if (ktime_compare(ts->idle_to_busy_ts,
546 /* We either didn't have any activity or we
547 * reached an entry which is 0. Either way,
548 * exit and return what was accumulated so far
553 /* If sample has finished, check it is relevant */
554 last_end_us = ktime_to_us(
555 ktime_sub(curr, ts->busy_to_idle_ts));
557 if (last_end_us > period_us)
560 /* It is relevant so add it but with adjustment */
561 last_busy_time_us = ktime_to_us(
562 ktime_sub(ts->busy_to_idle_ts,
563 ts->idle_to_busy_ts));
564 total_busy_time_us += last_busy_time_us -
565 (last_start_us - period_us);
569 /* Check if the sample is finished or still open */
570 if (ktime_compare(ts->busy_to_idle_ts, zero_ktime))
571 last_busy_time_us = ktime_to_us(
572 ktime_sub(ts->busy_to_idle_ts,
573 ts->idle_to_busy_ts));
575 last_busy_time_us = ktime_to_us(
576 ktime_sub(curr, ts->idle_to_busy_ts));
578 total_busy_time_us += last_busy_time_us;
581 /* Handle case idle_busy_ts_idx was 0 */
582 if (last_index > HL_IDLE_BUSY_TS_ARR_SIZE)
583 last_index = HL_IDLE_BUSY_TS_ARR_SIZE - 1;
585 ts = &hdev->idle_busy_ts_arr[last_index];
590 total_busy_time_ms = DIV_ROUND_UP_ULL(total_busy_time_us,
593 return DIV_ROUND_UP_ULL(total_busy_time_ms * 100, period_ms);
597 * hl_device_set_frequency - set the frequency of the device
599 * @hdev: pointer to habanalabs device structure
600 * @freq: the new frequency value
602 * Change the frequency if needed. This function has no protection against
603 * concurrency, therefore it is assumed that the calling function has protected
604 * itself against the case of calling this function from multiple threads with
607 * Returns 0 if no change was done, otherwise returns 1
609 int hl_device_set_frequency(struct hl_device *hdev, enum hl_pll_frequency freq)
611 if ((hdev->pm_mng_profile == PM_MANUAL) ||
612 (hdev->curr_pll_profile == freq))
615 dev_dbg(hdev->dev, "Changing device frequency to %s\n",
616 freq == PLL_HIGH ? "high" : "low");
618 hdev->asic_funcs->set_pll_profile(hdev, freq);
620 hdev->curr_pll_profile = freq;
625 int hl_device_set_debug_mode(struct hl_device *hdev, bool enable)
629 mutex_lock(&hdev->debug_lock);
632 if (!hdev->in_debug) {
634 "Failed to disable debug mode because device was not in debug mode\n");
639 if (!hdev->hard_reset_pending)
640 hdev->asic_funcs->halt_coresight(hdev);
644 if (!hdev->hard_reset_pending)
645 hdev->asic_funcs->set_clock_gating(hdev);
650 if (hdev->in_debug) {
652 "Failed to enable debug mode because device is already in debug mode\n");
657 hdev->asic_funcs->disable_clock_gating(hdev);
661 mutex_unlock(&hdev->debug_lock);
667 * hl_device_suspend - initiate device suspend
669 * @hdev: pointer to habanalabs device structure
671 * Puts the hw in the suspend state (all asics).
672 * Returns 0 for success or an error on failure.
673 * Called at driver suspend.
675 int hl_device_suspend(struct hl_device *hdev)
679 pci_save_state(hdev->pdev);
681 /* Block future CS/VM/JOB completion operations */
682 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
684 dev_err(hdev->dev, "Can't suspend while in reset\n");
688 /* This blocks all other stuff that is not blocked by in_reset */
689 hdev->disabled = true;
692 * Flush anyone that is inside the critical section of enqueue
695 hdev->asic_funcs->hw_queues_lock(hdev);
696 hdev->asic_funcs->hw_queues_unlock(hdev);
698 /* Flush processes that are sending message to CPU */
699 mutex_lock(&hdev->send_cpu_message_lock);
700 mutex_unlock(&hdev->send_cpu_message_lock);
702 rc = hdev->asic_funcs->suspend(hdev);
705 "Failed to disable PCI access of device CPU\n");
707 /* Shut down the device */
708 pci_disable_device(hdev->pdev);
709 pci_set_power_state(hdev->pdev, PCI_D3hot);
715 * hl_device_resume - initiate device resume
717 * @hdev: pointer to habanalabs device structure
719 * Bring the hw back to operating state (all asics).
720 * Returns 0 for success or an error on failure.
721 * Called at driver resume.
723 int hl_device_resume(struct hl_device *hdev)
727 pci_set_power_state(hdev->pdev, PCI_D0);
728 pci_restore_state(hdev->pdev);
729 rc = pci_enable_device_mem(hdev->pdev);
732 "Failed to enable PCI device in resume\n");
736 pci_set_master(hdev->pdev);
738 rc = hdev->asic_funcs->resume(hdev);
740 dev_err(hdev->dev, "Failed to resume device after suspend\n");
745 hdev->disabled = false;
746 atomic_set(&hdev->in_reset, 0);
748 rc = hl_device_reset(hdev, true, false);
750 dev_err(hdev->dev, "Failed to reset device during resume\n");
757 pci_clear_master(hdev->pdev);
758 pci_disable_device(hdev->pdev);
763 static int device_kill_open_processes(struct hl_device *hdev)
765 u16 pending_total, pending_cnt;
766 struct hl_fpriv *hpriv;
767 struct task_struct *task = NULL;
770 pending_total = HL_PLDM_PENDING_RESET_PER_SEC;
772 pending_total = HL_PENDING_RESET_PER_SEC;
774 /* Giving time for user to close FD, and for processes that are inside
775 * hl_device_open to finish
777 if (!list_empty(&hdev->fpriv_list))
780 mutex_lock(&hdev->fpriv_list_lock);
782 /* This section must be protected because we are dereferencing
783 * pointers that are freed if the process exits
785 list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node) {
786 task = get_pid_task(hpriv->taskpid, PIDTYPE_PID);
788 dev_info(hdev->dev, "Killing user process pid=%d\n",
790 send_sig(SIGKILL, task, 1);
791 usleep_range(1000, 10000);
793 put_task_struct(task);
797 mutex_unlock(&hdev->fpriv_list_lock);
799 /* We killed the open users, but because the driver cleans up after the
800 * user contexts are closed (e.g. mmu mappings), we need to wait again
801 * to make sure the cleaning phase is finished before continuing with
805 pending_cnt = pending_total;
807 while ((!list_empty(&hdev->fpriv_list)) && (pending_cnt)) {
809 "Waiting for all unmap operations to finish before hard reset\n");
816 return list_empty(&hdev->fpriv_list) ? 0 : -EBUSY;
819 static void device_hard_reset_pending(struct work_struct *work)
821 struct hl_device_reset_work *device_reset_work =
822 container_of(work, struct hl_device_reset_work, reset_work);
823 struct hl_device *hdev = device_reset_work->hdev;
825 hl_device_reset(hdev, true, true);
827 kfree(device_reset_work);
831 * hl_device_reset - reset the device
833 * @hdev: pointer to habanalabs device structure
834 * @hard_reset: should we do hard reset to all engines or just reset the
835 * compute/dma engines
836 * @from_hard_reset_thread: is the caller the hard-reset thread
838 * Block future CS and wait for pending CS to be enqueued
840 * Flush all completions
841 * Re-initialize all internal data structures
842 * Call ASIC H/W init, late_init
846 * Returns 0 for success or an error on failure.
848 int hl_device_reset(struct hl_device *hdev, bool hard_reset,
849 bool from_hard_reset_thread)
853 if (!hdev->init_done) {
855 "Can't reset before initialization is done\n");
859 if ((!hard_reset) && (!hdev->supports_soft_reset)) {
860 dev_dbg(hdev->dev, "Doing hard-reset instead of soft-reset\n");
865 * Prevent concurrency in this function - only one reset should be
866 * done at any given time. Only need to perform this if we didn't
867 * get from the dedicated hard reset thread
869 if (!from_hard_reset_thread) {
870 /* Block future CS/VM/JOB completion operations */
871 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
876 /* Disable PCI access from device F/W so he won't send
877 * us additional interrupts. We disable MSI/MSI-X at
878 * the halt_engines function and we can't have the F/W
879 * sending us interrupts after that. We need to disable
880 * the access here because if the device is marked
881 * disable, the message won't be send. Also, in case
882 * of heartbeat, the device CPU is marked as disable
883 * so this message won't be sent
885 if (hl_fw_send_pci_access_msg(hdev,
886 CPUCP_PACKET_DISABLE_PCI_ACCESS))
888 "Failed to disable PCI access by F/W\n");
891 /* This also blocks future CS/VM/JOB completion operations */
892 hdev->disabled = true;
894 /* Flush anyone that is inside the critical section of enqueue
897 hdev->asic_funcs->hw_queues_lock(hdev);
898 hdev->asic_funcs->hw_queues_unlock(hdev);
900 /* Flush anyone that is inside device open */
901 mutex_lock(&hdev->fpriv_list_lock);
902 mutex_unlock(&hdev->fpriv_list_lock);
904 dev_err(hdev->dev, "Going to RESET device!\n");
908 if ((hard_reset) && (!from_hard_reset_thread)) {
909 struct hl_device_reset_work *device_reset_work;
911 hdev->hard_reset_pending = true;
913 device_reset_work = kzalloc(sizeof(*device_reset_work),
915 if (!device_reset_work) {
921 * Because the reset function can't run from interrupt or
922 * from heartbeat work, we need to call the reset function
923 * from a dedicated work
925 INIT_WORK(&device_reset_work->reset_work,
926 device_hard_reset_pending);
927 device_reset_work->hdev = hdev;
928 schedule_work(&device_reset_work->reset_work);
934 device_late_fini(hdev);
937 * Now that the heartbeat thread is closed, flush processes
938 * which are sending messages to CPU
940 mutex_lock(&hdev->send_cpu_message_lock);
941 mutex_unlock(&hdev->send_cpu_message_lock);
945 * Halt the engines and disable interrupts so we won't get any more
946 * completions from H/W and we won't have any accesses from the
947 * H/W to the host machine
949 hdev->asic_funcs->halt_engines(hdev, hard_reset);
951 /* Go over all the queues, release all CS and their jobs */
952 hl_cs_rollback_all(hdev);
955 /* Kill processes here after CS rollback. This is because the
956 * process can't really exit until all its CSs are done, which
957 * is what we do in cs rollback
959 rc = device_kill_open_processes(hdev);
962 "Failed to kill all open processes, stopping hard reset\n");
966 /* Flush the Event queue workers to make sure no other thread is
967 * reading or writing to registers during the reset
969 flush_workqueue(hdev->eq_wq);
972 /* Reset the H/W. It will be in idle state after this returns */
973 hdev->asic_funcs->hw_fini(hdev, hard_reset);
976 /* Release kernel context */
977 if (hl_ctx_put(hdev->kernel_ctx) == 1)
978 hdev->kernel_ctx = NULL;
981 hl_eq_reset(hdev, &hdev->event_queue);
984 /* Re-initialize PI,CI to 0 in all queues (hw queue, cq) */
985 hl_hw_queue_reset(hdev, hard_reset);
986 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
987 hl_cq_reset(hdev, &hdev->completion_queue[i]);
989 hdev->idle_busy_ts_idx = 0;
990 hdev->idle_busy_ts_arr[0].busy_to_idle_ts = ktime_set(0, 0);
991 hdev->idle_busy_ts_arr[0].idle_to_busy_ts = ktime_set(0, 0);
993 if (hdev->cs_active_cnt)
994 dev_crit(hdev->dev, "CS active cnt %d is not 0 during reset\n",
995 hdev->cs_active_cnt);
997 mutex_lock(&hdev->fpriv_list_lock);
999 /* Make sure the context switch phase will run again */
1000 if (hdev->compute_ctx) {
1001 atomic_set(&hdev->compute_ctx->thread_ctx_switch_token, 1);
1002 hdev->compute_ctx->thread_ctx_switch_wait_token = 0;
1005 mutex_unlock(&hdev->fpriv_list_lock);
1007 /* Finished tear-down, starting to re-initialize */
1010 hdev->device_cpu_disabled = false;
1011 hdev->hard_reset_pending = false;
1013 if (hdev->kernel_ctx) {
1015 "kernel ctx was alive during hard reset, something is terribly wrong\n");
1020 rc = hl_mmu_init(hdev);
1023 "Failed to initialize MMU S/W after hard reset\n");
1027 /* Allocate the kernel context */
1028 hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx),
1030 if (!hdev->kernel_ctx) {
1036 hdev->compute_ctx = NULL;
1038 rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
1041 "failed to init kernel ctx in hard reset\n");
1042 kfree(hdev->kernel_ctx);
1043 hdev->kernel_ctx = NULL;
1049 /* Device is now enabled as part of the initialization requires
1050 * communication with the device firmware to get information that
1051 * is required for the initialization itself
1053 hdev->disabled = false;
1055 rc = hdev->asic_funcs->hw_init(hdev);
1058 "failed to initialize the H/W after reset\n");
1062 /* Check that the communication with the device is working */
1063 rc = hdev->asic_funcs->test_queues(hdev);
1066 "Failed to detect if device is alive after reset\n");
1071 rc = device_late_init(hdev);
1074 "Failed late init after hard reset\n");
1078 rc = hl_vm_init(hdev);
1081 "Failed to init memory module after hard reset\n");
1085 hl_set_max_power(hdev);
1087 rc = hdev->asic_funcs->soft_reset_late_init(hdev);
1090 "Failed late init after soft reset\n");
1095 atomic_set(&hdev->in_reset, 0);
1098 hdev->hard_reset_cnt++;
1100 hdev->soft_reset_cnt++;
1102 dev_warn(hdev->dev, "Successfully finished resetting the device\n");
1107 hdev->disabled = true;
1111 "Failed to reset! Device is NOT usable\n");
1112 hdev->hard_reset_cnt++;
1115 "Failed to do soft-reset, trying hard reset\n");
1116 hdev->soft_reset_cnt++;
1121 atomic_set(&hdev->in_reset, 0);
1127 * hl_device_init - main initialization function for habanalabs device
1129 * @hdev: pointer to habanalabs device structure
1131 * Allocate an id for the device, do early initialization and then call the
1132 * ASIC specific initialization functions. Finally, create the cdev and the
1133 * Linux device to expose it to the user
1135 int hl_device_init(struct hl_device *hdev, struct class *hclass)
1137 int i, rc, cq_cnt, cq_ready_cnt;
1139 bool add_cdev_sysfs_on_err = false;
1141 name = kasprintf(GFP_KERNEL, "hl%d", hdev->id / 2);
1147 /* Initialize cdev and device structures */
1148 rc = device_init_cdev(hdev, hclass, hdev->id, &hl_ops, name,
1149 &hdev->cdev, &hdev->dev);
1156 name = kasprintf(GFP_KERNEL, "hl_controlD%d", hdev->id / 2);
1162 /* Initialize cdev and device structures for control device */
1163 rc = device_init_cdev(hdev, hclass, hdev->id_control, &hl_ctrl_ops,
1164 name, &hdev->cdev_ctrl, &hdev->dev_ctrl);
1171 /* Initialize ASIC function pointers and perform early init */
1172 rc = device_early_init(hdev);
1177 * Start calling ASIC initialization. First S/W then H/W and finally
1180 rc = hdev->asic_funcs->sw_init(hdev);
1185 * Initialize the H/W queues. Must be done before hw_init, because
1186 * there the addresses of the kernel queue are being written to the
1187 * registers of the device
1189 rc = hl_hw_queues_create(hdev);
1191 dev_err(hdev->dev, "failed to initialize kernel queues\n");
1195 cq_cnt = hdev->asic_prop.completion_queues_count;
1198 * Initialize the completion queues. Must be done before hw_init,
1199 * because there the addresses of the completion queues are being
1200 * passed as arguments to request_irq
1203 hdev->completion_queue = kcalloc(cq_cnt,
1204 sizeof(*hdev->completion_queue),
1207 if (!hdev->completion_queue) {
1209 "failed to allocate completion queues\n");
1211 goto hw_queues_destroy;
1215 for (i = 0, cq_ready_cnt = 0 ; i < cq_cnt ; i++, cq_ready_cnt++) {
1216 rc = hl_cq_init(hdev, &hdev->completion_queue[i],
1217 hdev->asic_funcs->get_queue_id_for_cq(hdev, i));
1220 "failed to initialize completion queue\n");
1223 hdev->completion_queue[i].cq_idx = i;
1227 * Initialize the event queue. Must be done before hw_init,
1228 * because there the address of the event queue is being
1229 * passed as argument to request_irq
1231 rc = hl_eq_init(hdev, &hdev->event_queue);
1233 dev_err(hdev->dev, "failed to initialize event queue\n");
1237 /* MMU S/W must be initialized before kernel context is created */
1238 rc = hl_mmu_init(hdev);
1240 dev_err(hdev->dev, "Failed to initialize MMU S/W structures\n");
1244 /* Allocate the kernel context */
1245 hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL);
1246 if (!hdev->kernel_ctx) {
1251 hdev->compute_ctx = NULL;
1253 rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
1255 dev_err(hdev->dev, "failed to initialize kernel context\n");
1256 kfree(hdev->kernel_ctx);
1260 rc = hl_cb_pool_init(hdev);
1262 dev_err(hdev->dev, "failed to initialize CB pool\n");
1266 hl_debugfs_add_device(hdev);
1268 if (hdev->asic_funcs->get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {
1270 "H/W state is dirty, must reset before initializing\n");
1271 hdev->asic_funcs->halt_engines(hdev, true);
1272 hdev->asic_funcs->hw_fini(hdev, true);
1276 * From this point, in case of an error, add char devices and create
1277 * sysfs nodes as part of the error flow, to allow debugging.
1279 add_cdev_sysfs_on_err = true;
1281 /* Device is now enabled as part of the initialization requires
1282 * communication with the device firmware to get information that
1283 * is required for the initialization itself
1285 hdev->disabled = false;
1287 rc = hdev->asic_funcs->hw_init(hdev);
1289 dev_err(hdev->dev, "failed to initialize the H/W\n");
1294 /* Check that the communication with the device is working */
1295 rc = hdev->asic_funcs->test_queues(hdev);
1297 dev_err(hdev->dev, "Failed to detect if device is alive\n");
1302 rc = device_late_init(hdev);
1304 dev_err(hdev->dev, "Failed late initialization\n");
1309 dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n",
1311 hdev->asic_prop.dram_size / 1024 / 1024 / 1024);
1313 rc = hl_vm_init(hdev);
1315 dev_err(hdev->dev, "Failed to initialize memory module\n");
1321 * Expose devices and sysfs nodes to user.
1322 * From here there is no need to add char devices and create sysfs nodes
1323 * in case of an error.
1325 add_cdev_sysfs_on_err = false;
1326 rc = device_cdev_sysfs_add(hdev);
1329 "Failed to add char devices and sysfs nodes\n");
1334 /* Need to call this again because the max power might change,
1335 * depending on card type for certain ASICs
1337 hl_set_max_power(hdev);
1340 * hl_hwmon_init() must be called after device_late_init(), because only
1341 * there we get the information from the device about which
1342 * hwmon-related sensors the device supports.
1343 * Furthermore, it must be done after adding the device to the system.
1345 rc = hl_hwmon_init(hdev);
1347 dev_err(hdev->dev, "Failed to initialize hwmon\n");
1352 dev_notice(hdev->dev,
1353 "Successfully added device to habanalabs driver\n");
1355 hdev->init_done = true;
1360 if (hl_ctx_put(hdev->kernel_ctx) != 1)
1362 "kernel ctx is still alive on initialization failure\n");
1366 hl_eq_fini(hdev, &hdev->event_queue);
1368 for (i = 0 ; i < cq_ready_cnt ; i++)
1369 hl_cq_fini(hdev, &hdev->completion_queue[i]);
1370 kfree(hdev->completion_queue);
1372 hl_hw_queues_destroy(hdev);
1374 hdev->asic_funcs->sw_fini(hdev);
1376 device_early_fini(hdev);
1378 put_device(hdev->dev_ctrl);
1380 put_device(hdev->dev);
1382 hdev->disabled = true;
1383 if (add_cdev_sysfs_on_err)
1384 device_cdev_sysfs_add(hdev);
1386 dev_err(&hdev->pdev->dev,
1387 "Failed to initialize hl%d. Device is NOT usable !\n",
1390 pr_err("Failed to initialize hl%d. Device is NOT usable !\n",
1397 * hl_device_fini - main tear-down function for habanalabs device
1399 * @hdev: pointer to habanalabs device structure
1401 * Destroy the device, call ASIC fini functions and release the id
1403 void hl_device_fini(struct hl_device *hdev)
1408 dev_info(hdev->dev, "Removing device\n");
1411 * This function is competing with the reset function, so try to
1412 * take the reset atomic and if we are already in middle of reset,
1413 * wait until reset function is finished. Reset function is designed
1414 * to always finish. However, in Gaudi, because of all the network
1415 * ports, the hard reset could take between 10-30 seconds
1418 timeout = ktime_add_us(ktime_get(),
1419 HL_HARD_RESET_MAX_TIMEOUT * 1000 * 1000);
1420 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
1422 usleep_range(50, 200);
1423 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
1424 if (ktime_compare(ktime_get(), timeout) > 0) {
1425 WARN(1, "Failed to remove device because reset function did not finish\n");
1430 /* Disable PCI access from device F/W so it won't send us additional
1431 * interrupts. We disable MSI/MSI-X at the halt_engines function and we
1432 * can't have the F/W sending us interrupts after that. We need to
1433 * disable the access here because if the device is marked disable, the
1434 * message won't be send. Also, in case of heartbeat, the device CPU is
1435 * marked as disable so this message won't be sent
1437 hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS);
1439 /* Mark device as disabled */
1440 hdev->disabled = true;
1442 /* Flush anyone that is inside the critical section of enqueue
1445 hdev->asic_funcs->hw_queues_lock(hdev);
1446 hdev->asic_funcs->hw_queues_unlock(hdev);
1448 /* Flush anyone that is inside device open */
1449 mutex_lock(&hdev->fpriv_list_lock);
1450 mutex_unlock(&hdev->fpriv_list_lock);
1452 hdev->hard_reset_pending = true;
1454 hl_hwmon_fini(hdev);
1456 device_late_fini(hdev);
1458 hl_debugfs_remove_device(hdev);
1461 * Halt the engines and disable interrupts so we won't get any more
1462 * completions from H/W and we won't have any accesses from the
1463 * H/W to the host machine
1465 hdev->asic_funcs->halt_engines(hdev, true);
1467 /* Go over all the queues, release all CS and their jobs */
1468 hl_cs_rollback_all(hdev);
1470 /* Kill processes here after CS rollback. This is because the process
1471 * can't really exit until all its CSs are done, which is what we
1474 rc = device_kill_open_processes(hdev);
1476 dev_crit(hdev->dev, "Failed to kill all open processes\n");
1478 hl_cb_pool_fini(hdev);
1480 /* Reset the H/W. It will be in idle state after this returns */
1481 hdev->asic_funcs->hw_fini(hdev, true);
1483 /* Release kernel context */
1484 if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1))
1485 dev_err(hdev->dev, "kernel ctx is still alive\n");
1491 hl_eq_fini(hdev, &hdev->event_queue);
1493 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
1494 hl_cq_fini(hdev, &hdev->completion_queue[i]);
1495 kfree(hdev->completion_queue);
1497 hl_hw_queues_destroy(hdev);
1499 /* Call ASIC S/W finalize function */
1500 hdev->asic_funcs->sw_fini(hdev);
1502 device_early_fini(hdev);
1504 /* Hide devices and sysfs nodes from user */
1505 device_cdev_sysfs_del(hdev);
1507 pr_info("removed device successfully\n");
1511 * MMIO register access helper functions.
1515 * hl_rreg - Read an MMIO register
1517 * @hdev: pointer to habanalabs device structure
1518 * @reg: MMIO register offset (in bytes)
1520 * Returns the value of the MMIO register we are asked to read
1523 inline u32 hl_rreg(struct hl_device *hdev, u32 reg)
1525 return readl(hdev->rmmio + reg);
1529 * hl_wreg - Write to an MMIO register
1531 * @hdev: pointer to habanalabs device structure
1532 * @reg: MMIO register offset (in bytes)
1533 * @val: 32-bit value
1535 * Writes the 32-bit value into the MMIO register
1538 inline void hl_wreg(struct hl_device *hdev, u32 reg, u32 val)
1540 writel(val, hdev->rmmio + reg);