drivers/accel/habanalabs/common/device.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2022 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #define pr_fmt(fmt)                     "habanalabs: " fmt
   9
  10 #include <uapi/drm/habanalabs_accel.h>
  11 #include "habanalabs.h"
  12
  13 #include <linux/pci.h>
  14 #include <linux/hwmon.h>
  15 #include <linux/vmalloc.h>
  16
  17 #include <drm/drm_accel.h>
  18 #include <drm/drm_drv.h>
  19
  20 #include <trace/events/habanalabs.h>
  21
  22 #define HL_RESET_DELAY_USEC                     10000   /* 10ms */
  23
  24 #define HL_DEVICE_RELEASE_WATCHDOG_TIMEOUT_SEC  30
  25
  26 enum dma_alloc_type {
  27         DMA_ALLOC_COHERENT,
  28         DMA_ALLOC_POOL,
  29 };
  30
  31 #define MEM_SCRUB_DEFAULT_VAL 0x1122334455667788
  32
  33 /*
  34  * hl_set_dram_bar- sets the bar to allow later access to address
  35  *
  36  * @hdev: pointer to habanalabs device structure.
  37  * @addr: the address the caller wants to access.
  38  * @region: the PCI region.
  39  * @new_bar_region_base: the new BAR region base address.
  40  *
  41  * @return: the old BAR base address on success, U64_MAX for failure.
  42  *          The caller should set it back to the old address after use.
  43  *
  44  * In case the bar space does not cover the whole address space,
  45  * the bar base address should be set to allow access to a given address.
  46  * This function can be called also if the bar doesn't need to be set,
  47  * in that case it just won't change the base.
  48  */
  49 static u64 hl_set_dram_bar(struct hl_device *hdev, u64 addr, struct pci_mem_region *region,
  50                                 u64 *new_bar_region_base)
  51 {
  52         struct asic_fixed_properties *prop = &hdev->asic_prop;
  53         u64 bar_base_addr, old_base;
  54
  55         if (is_power_of_2(prop->dram_pci_bar_size))
  56                 bar_base_addr = addr & ~(prop->dram_pci_bar_size - 0x1ull);
  57         else
  58                 bar_base_addr = DIV_ROUND_DOWN_ULL(addr, prop->dram_pci_bar_size) *
  59                                 prop->dram_pci_bar_size;
  60
  61         old_base = hdev->asic_funcs->set_dram_bar_base(hdev, bar_base_addr);
  62
  63         /* in case of success we need to update the new BAR base */
  64         if ((old_base != U64_MAX) && new_bar_region_base)
  65                 *new_bar_region_base = bar_base_addr;
  66
  67         return old_base;
  68 }
  69
  70 int hl_access_sram_dram_region(struct hl_device *hdev, u64 addr, u64 *val,
  71         enum debugfs_access_type acc_type, enum pci_region region_type, bool set_dram_bar)
  72 {
  73         struct pci_mem_region *region = &hdev->pci_mem_region[region_type];
  74         u64 old_base = 0, rc, bar_region_base = region->region_base;
  75         void __iomem *acc_addr;
  76
  77         if (set_dram_bar) {
  78                 old_base = hl_set_dram_bar(hdev, addr, region, &bar_region_base);
  79                 if (old_base == U64_MAX)
  80                         return -EIO;
  81         }
  82
  83         acc_addr = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
  84                         (addr - bar_region_base);
  85
  86         switch (acc_type) {
  87         case DEBUGFS_READ8:
  88                 *val = readb(acc_addr);
  89                 break;
  90         case DEBUGFS_WRITE8:
  91                 writeb(*val, acc_addr);
  92                 break;
  93         case DEBUGFS_READ32:
  94                 *val = readl(acc_addr);
  95                 break;
  96         case DEBUGFS_WRITE32:
  97                 writel(*val, acc_addr);
  98                 break;
  99         case DEBUGFS_READ64:
 100                 *val = readq(acc_addr);
 101                 break;
 102         case DEBUGFS_WRITE64:
 103                 writeq(*val, acc_addr);
 104                 break;
 105         }
 106
 107         if (set_dram_bar) {
 108                 rc = hl_set_dram_bar(hdev, old_base, region, NULL);
 109                 if (rc == U64_MAX)
 110                         return -EIO;
 111         }
 112
 113         return 0;
 114 }
 115
 116 static void *hl_dma_alloc_common(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle,
 117                                         gfp_t flag, enum dma_alloc_type alloc_type,
 118                                         const char *caller)
 119 {
 120         void *ptr = NULL;
 121
 122         switch (alloc_type) {
 123         case DMA_ALLOC_COHERENT:
 124                 ptr = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, size, dma_handle, flag);
 125                 break;
 126         case DMA_ALLOC_POOL:
 127                 ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, size, flag, dma_handle);
 128                 break;
 129         }
 130
 131         if (trace_habanalabs_dma_alloc_enabled() && !ZERO_OR_NULL_PTR(ptr))
 132                 trace_habanalabs_dma_alloc(hdev->dev, (u64) (uintptr_t) ptr, *dma_handle, size,
 133                                                 caller);
 134
 135         return ptr;
 136 }
 137
 138 static void hl_asic_dma_free_common(struct hl_device *hdev, size_t size, void *cpu_addr,
 139                                         dma_addr_t dma_handle, enum dma_alloc_type alloc_type,
 140                                         const char *caller)
 141 {
 142         /* this is needed to avoid warning on using freed pointer */
 143         u64 store_cpu_addr = (u64) (uintptr_t) cpu_addr;
 144
 145         switch (alloc_type) {
 146         case DMA_ALLOC_COHERENT:
 147                 hdev->asic_funcs->asic_dma_free_coherent(hdev, size, cpu_addr, dma_handle);
 148                 break;
 149         case DMA_ALLOC_POOL:
 150                 hdev->asic_funcs->asic_dma_pool_free(hdev, cpu_addr, dma_handle);
 151                 break;
 152         }
 153
 154         trace_habanalabs_dma_free(hdev->dev, store_cpu_addr, dma_handle, size, caller);
 155 }
 156
 157 void *hl_asic_dma_alloc_coherent_caller(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle,
 158                                         gfp_t flag, const char *caller)
 159 {
 160         return hl_dma_alloc_common(hdev, size, dma_handle, flag, DMA_ALLOC_COHERENT, caller);
 161 }
 162
 163 void hl_asic_dma_free_coherent_caller(struct hl_device *hdev, size_t size, void *cpu_addr,
 164                                         dma_addr_t dma_handle, const char *caller)
 165 {
 166         hl_asic_dma_free_common(hdev, size, cpu_addr, dma_handle, DMA_ALLOC_COHERENT, caller);
 167 }
 168
 169 void *hl_asic_dma_pool_zalloc_caller(struct hl_device *hdev, size_t size, gfp_t mem_flags,
 170                                         dma_addr_t *dma_handle, const char *caller)
 171 {
 172         return hl_dma_alloc_common(hdev, size, dma_handle, mem_flags, DMA_ALLOC_POOL, caller);
 173 }
 174
 175 void hl_asic_dma_pool_free_caller(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr,
 176                                         const char *caller)
 177 {
 178         hl_asic_dma_free_common(hdev, 0, vaddr, dma_addr, DMA_ALLOC_POOL, caller);
 179 }
 180
 181 void *hl_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle)
 182 {
 183         return hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, size, dma_handle);
 184 }
 185
 186 void hl_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr)
 187 {
 188         hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, size, vaddr);
 189 }
 190
 191 int hl_dma_map_sgtable_caller(struct hl_device *hdev, struct sg_table *sgt,
 192                                 enum dma_data_direction dir, const char *caller)
 193 {
 194         struct asic_fixed_properties *prop = &hdev->asic_prop;
 195         struct scatterlist *sg;
 196         int rc, i;
 197
 198         rc = hdev->asic_funcs->dma_map_sgtable(hdev, sgt, dir);
 199         if (rc)
 200                 return rc;
 201
 202         if (!trace_habanalabs_dma_map_page_enabled())
 203                 return 0;
 204
 205         for_each_sgtable_dma_sg(sgt, sg, i)
 206                 trace_habanalabs_dma_map_page(hdev->dev,
 207                                 page_to_phys(sg_page(sg)),
 208                                 sg->dma_address - prop->device_dma_offset_for_host_access,
 209 #ifdef CONFIG_NEED_SG_DMA_LENGTH
 210                                 sg->dma_length,
 211 #else
 212                                 sg->length,
 213 #endif
 214                                 dir, caller);
 215
 216         return 0;
 217 }
 218
 219 int hl_asic_dma_map_sgtable(struct hl_device *hdev, struct sg_table *sgt,
 220                                 enum dma_data_direction dir)
 221 {
 222         struct asic_fixed_properties *prop = &hdev->asic_prop;
 223         struct scatterlist *sg;
 224         int rc, i;
 225
 226         rc = dma_map_sgtable(&hdev->pdev->dev, sgt, dir, 0);
 227         if (rc)
 228                 return rc;
 229
 230         /* Shift to the device's base physical address of host memory if necessary */
 231         if (prop->device_dma_offset_for_host_access)
 232                 for_each_sgtable_dma_sg(sgt, sg, i)
 233                         sg->dma_address += prop->device_dma_offset_for_host_access;
 234
 235         return 0;
 236 }
 237
 238 void hl_dma_unmap_sgtable_caller(struct hl_device *hdev, struct sg_table *sgt,
 239                                         enum dma_data_direction dir, const char *caller)
 240 {
 241         struct asic_fixed_properties *prop = &hdev->asic_prop;
 242         struct scatterlist *sg;
 243         int i;
 244
 245         hdev->asic_funcs->dma_unmap_sgtable(hdev, sgt, dir);
 246
 247         if (trace_habanalabs_dma_unmap_page_enabled()) {
 248                 for_each_sgtable_dma_sg(sgt, sg, i)
 249                         trace_habanalabs_dma_unmap_page(hdev->dev, page_to_phys(sg_page(sg)),
 250                                         sg->dma_address - prop->device_dma_offset_for_host_access,
 251 #ifdef CONFIG_NEED_SG_DMA_LENGTH
 252                                         sg->dma_length,
 253 #else
 254                                         sg->length,
 255 #endif
 256                                         dir, caller);
 257         }
 258 }
 259
 260 void hl_asic_dma_unmap_sgtable(struct hl_device *hdev, struct sg_table *sgt,
 261                                 enum dma_data_direction dir)
 262 {
 263         struct asic_fixed_properties *prop = &hdev->asic_prop;
 264         struct scatterlist *sg;
 265         int i;
 266
 267         /* Cancel the device's base physical address of host memory if necessary */
 268         if (prop->device_dma_offset_for_host_access)
 269                 for_each_sgtable_dma_sg(sgt, sg, i)
 270                         sg->dma_address -= prop->device_dma_offset_for_host_access;
 271
 272         dma_unmap_sgtable(&hdev->pdev->dev, sgt, dir, 0);
 273 }
 274
 275 /*
 276  * hl_access_cfg_region - access the config region
 277  *
 278  * @hdev: pointer to habanalabs device structure
 279  * @addr: the address to access
 280  * @val: the value to write from or read to
 281  * @acc_type: the type of access (read/write 64/32)
 282  */
 283 int hl_access_cfg_region(struct hl_device *hdev, u64 addr, u64 *val,
 284         enum debugfs_access_type acc_type)
 285 {
 286         struct pci_mem_region *cfg_region = &hdev->pci_mem_region[PCI_REGION_CFG];
 287         u32 val_h, val_l;
 288
 289         if (!IS_ALIGNED(addr, sizeof(u32))) {
 290                 dev_err(hdev->dev, "address %#llx not a multiple of %zu\n", addr, sizeof(u32));
 291                 return -EINVAL;
 292         }
 293
 294         switch (acc_type) {
 295         case DEBUGFS_READ32:
 296                 *val = RREG32(addr - cfg_region->region_base);
 297                 break;
 298         case DEBUGFS_WRITE32:
 299                 WREG32(addr - cfg_region->region_base, *val);
 300                 break;
 301         case DEBUGFS_READ64:
 302                 val_l = RREG32(addr - cfg_region->region_base);
 303                 val_h = RREG32(addr + sizeof(u32) - cfg_region->region_base);
 304
 305                 *val = (((u64) val_h) << 32) | val_l;
 306                 break;
 307         case DEBUGFS_WRITE64:
 308                 WREG32(addr - cfg_region->region_base, lower_32_bits(*val));
 309                 WREG32(addr + sizeof(u32) - cfg_region->region_base, upper_32_bits(*val));
 310                 break;
 311         default:
 312                 dev_err(hdev->dev, "access type %d is not supported\n", acc_type);
 313                 return -EOPNOTSUPP;
 314         }
 315
 316         return 0;
 317 }
 318
 319 /*
 320  * hl_access_dev_mem - access device memory
 321  *
 322  * @hdev: pointer to habanalabs device structure
 323  * @region_type: the type of the region the address belongs to
 324  * @addr: the address to access
 325  * @val: the value to write from or read to
 326  * @acc_type: the type of access (r/w, 32/64)
 327  */
 328 int hl_access_dev_mem(struct hl_device *hdev, enum pci_region region_type,
 329                         u64 addr, u64 *val, enum debugfs_access_type acc_type)
 330 {
 331         switch (region_type) {
 332         case PCI_REGION_CFG:
 333                 return hl_access_cfg_region(hdev, addr, val, acc_type);
 334         case PCI_REGION_SRAM:
 335         case PCI_REGION_DRAM:
 336                 return hl_access_sram_dram_region(hdev, addr, val, acc_type,
 337                                 region_type, (region_type == PCI_REGION_DRAM));
 338         default:
 339                 return -EFAULT;
 340         }
 341
 342         return 0;
 343 }
 344
 345 void hl_engine_data_sprintf(struct engines_data *e, const char *fmt, ...)
 346 {
 347         va_list args;
 348         int str_size;
 349
 350         va_start(args, fmt);
 351         /* Calculate formatted string length. Assuming each string is null terminated, hence
 352          * increment result by 1
 353          */
 354         str_size = vsnprintf(NULL, 0, fmt, args) + 1;
 355         va_end(args);
 356
 357         if ((e->actual_size + str_size) < e->allocated_buf_size) {
 358                 va_start(args, fmt);
 359                 vsnprintf(e->buf + e->actual_size, str_size, fmt, args);
 360                 va_end(args);
 361         }
 362
 363         /* Need to update the size even when not updating destination buffer to get the exact size
 364          * of all input strings
 365          */
 366         e->actual_size += str_size;
 367 }
 368
 369 enum hl_device_status hl_device_status(struct hl_device *hdev)
 370 {
 371         enum hl_device_status status;
 372
 373         if (hdev->device_fini_pending) {
 374                 status = HL_DEVICE_STATUS_MALFUNCTION;
 375         } else if (hdev->reset_info.in_reset) {
 376                 if (hdev->reset_info.in_compute_reset)
 377                         status = HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE;
 378                 else
 379                         status = HL_DEVICE_STATUS_IN_RESET;
 380         } else if (hdev->reset_info.needs_reset) {
 381                 status = HL_DEVICE_STATUS_NEEDS_RESET;
 382         } else if (hdev->disabled) {
 383                 status = HL_DEVICE_STATUS_MALFUNCTION;
 384         } else if (!hdev->init_done) {
 385                 status = HL_DEVICE_STATUS_IN_DEVICE_CREATION;
 386         } else {
 387                 status = HL_DEVICE_STATUS_OPERATIONAL;
 388         }
 389
 390         return status;
 391 }
 392
 393 bool hl_device_operational(struct hl_device *hdev,
 394                 enum hl_device_status *status)
 395 {
 396         enum hl_device_status current_status;
 397
 398         current_status = hl_device_status(hdev);
 399         if (status)
 400                 *status = current_status;
 401
 402         switch (current_status) {
 403         case HL_DEVICE_STATUS_MALFUNCTION:
 404         case HL_DEVICE_STATUS_IN_RESET:
 405         case HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE:
 406         case HL_DEVICE_STATUS_NEEDS_RESET:
 407                 return false;
 408         case HL_DEVICE_STATUS_OPERATIONAL:
 409         case HL_DEVICE_STATUS_IN_DEVICE_CREATION:
 410         default:
 411                 return true;
 412         }
 413 }
 414
 415 bool hl_ctrl_device_operational(struct hl_device *hdev,
 416                 enum hl_device_status *status)
 417 {
 418         enum hl_device_status current_status;
 419
 420         current_status = hl_device_status(hdev);
 421         if (status)
 422                 *status = current_status;
 423
 424         switch (current_status) {
 425         case HL_DEVICE_STATUS_MALFUNCTION:
 426                 return false;
 427         case HL_DEVICE_STATUS_IN_RESET:
 428         case HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE:
 429         case HL_DEVICE_STATUS_NEEDS_RESET:
 430         case HL_DEVICE_STATUS_OPERATIONAL:
 431         case HL_DEVICE_STATUS_IN_DEVICE_CREATION:
 432         default:
 433                 return true;
 434         }
 435 }
 436
 437 static void print_idle_status_mask(struct hl_device *hdev, const char *message,
 438                                         u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE])
 439 {
 440         if (idle_mask[3])
 441                 dev_err(hdev->dev, "%s (mask %#llx_%016llx_%016llx_%016llx)\n",
 442                         message, idle_mask[3], idle_mask[2], idle_mask[1], idle_mask[0]);
 443         else if (idle_mask[2])
 444                 dev_err(hdev->dev, "%s (mask %#llx_%016llx_%016llx)\n",
 445                         message, idle_mask[2], idle_mask[1], idle_mask[0]);
 446         else if (idle_mask[1])
 447                 dev_err(hdev->dev, "%s (mask %#llx_%016llx)\n",
 448                         message, idle_mask[1], idle_mask[0]);
 449         else
 450                 dev_err(hdev->dev, "%s (mask %#llx)\n", message, idle_mask[0]);
 451 }
 452
 453 static void hpriv_release(struct kref *ref)
 454 {
 455         u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0};
 456         bool reset_device, device_is_idle = true;
 457         struct hl_fpriv *hpriv;
 458         struct hl_device *hdev;
 459
 460         hpriv = container_of(ref, struct hl_fpriv, refcount);
 461
 462         hdev = hpriv->hdev;
 463
 464         hdev->asic_funcs->send_device_activity(hdev, false);
 465
 466         hl_debugfs_remove_file(hpriv);
 467
 468         mutex_destroy(&hpriv->ctx_lock);
 469         mutex_destroy(&hpriv->restore_phase_mutex);
 470
 471         /* There should be no memory buffers at this point and handles IDR can be destroyed */
 472         hl_mem_mgr_idr_destroy(&hpriv->mem_mgr);
 473
 474         /* Device should be reset if reset-upon-device-release is enabled, or if there is a pending
 475          * reset that waits for device release.
 476          */
 477         reset_device = hdev->reset_upon_device_release || hdev->reset_info.watchdog_active;
 478
 479         /* Check the device idle status and reset if not idle.
 480          * Skip it if already in reset, or if device is going to be reset in any case.
 481          */
 482         if (!hdev->reset_info.in_reset && !reset_device && !hdev->pldm)
 483                 device_is_idle = hdev->asic_funcs->is_device_idle(hdev, idle_mask,
 484                                                         HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL);
 485         if (!device_is_idle) {
 486                 print_idle_status_mask(hdev, "device is not idle after user context is closed",
 487                                         idle_mask);
 488                 reset_device = true;
 489         }
 490
 491         /* We need to remove the user from the list to make sure the reset process won't
 492          * try to kill the user process. Because, if we got here, it means there are no
 493          * more driver/device resources that the user process is occupying so there is
 494          * no need to kill it
 495          *
 496          * However, we can't set the compute_ctx to NULL at this stage. This is to prevent
 497          * a race between the release and opening the device again. We don't want to let
 498          * a user open the device while there a reset is about to happen.
 499          */
 500         mutex_lock(&hdev->fpriv_list_lock);
 501         list_del(&hpriv->dev_node);
 502         mutex_unlock(&hdev->fpriv_list_lock);
 503
 504         put_pid(hpriv->taskpid);
 505
 506         if (reset_device) {
 507                 hl_device_reset(hdev, HL_DRV_RESET_DEV_RELEASE);
 508         } else {
 509                 /* Scrubbing is handled within hl_device_reset(), so here need to do it directly */
 510                 int rc = hdev->asic_funcs->scrub_device_mem(hdev);
 511
 512                 if (rc) {
 513                         dev_err(hdev->dev, "failed to scrub memory from hpriv release (%d)\n", rc);
 514                         hl_device_reset(hdev, HL_DRV_RESET_HARD);
 515                 }
 516         }
 517
 518         /* Now we can mark the compute_ctx as not active. Even if a reset is running in a different
 519          * thread, we don't care because the in_reset is marked so if a user will try to open
 520          * the device it will fail on that, even if compute_ctx is false.
 521          */
 522         mutex_lock(&hdev->fpriv_list_lock);
 523         hdev->is_compute_ctx_active = false;
 524         mutex_unlock(&hdev->fpriv_list_lock);
 525
 526         hdev->compute_ctx_in_release = 0;
 527
 528         /* release the eventfd */
 529         if (hpriv->notifier_event.eventfd)
 530                 eventfd_ctx_put(hpriv->notifier_event.eventfd);
 531
 532         mutex_destroy(&hpriv->notifier_event.lock);
 533
 534         kfree(hpriv);
 535 }
 536
 537 void hl_hpriv_get(struct hl_fpriv *hpriv)
 538 {
 539         kref_get(&hpriv->refcount);
 540 }
 541
 542 int hl_hpriv_put(struct hl_fpriv *hpriv)
 543 {
 544         return kref_put(&hpriv->refcount, hpriv_release);
 545 }
 546
 547 static void print_device_in_use_info(struct hl_device *hdev, const char *message)
 548 {
 549         u32 active_cs_num, dmabuf_export_cnt;
 550         bool unknown_reason = true;
 551         char buf[128];
 552         size_t size;
 553         int offset;
 554
 555         size = sizeof(buf);
 556         offset = 0;
 557
 558         active_cs_num = hl_get_active_cs_num(hdev);
 559         if (active_cs_num) {
 560                 unknown_reason = false;
 561                 offset += scnprintf(buf + offset, size - offset, " [%u active CS]", active_cs_num);
 562         }
 563
 564         dmabuf_export_cnt = atomic_read(&hdev->dmabuf_export_cnt);
 565         if (dmabuf_export_cnt) {
 566                 unknown_reason = false;
 567                 offset += scnprintf(buf + offset, size - offset, " [%u exported dma-buf]",
 568                                         dmabuf_export_cnt);
 569         }
 570
 571         if (unknown_reason)
 572                 scnprintf(buf + offset, size - offset, " [unknown reason]");
 573
 574         dev_notice(hdev->dev, "%s%s\n", message, buf);
 575 }
 576
 577 /*
 578  * hl_device_release() - release function for habanalabs device.
 579  * @ddev: pointer to DRM device structure.
 580  * @file: pointer to DRM file private data structure.
 581  *
 582  * Called when process closes an habanalabs device
 583  */
 584 void hl_device_release(struct drm_device *ddev, struct drm_file *file_priv)
 585 {
 586         struct hl_fpriv *hpriv = file_priv->driver_priv;
 587         struct hl_device *hdev = to_hl_device(ddev);
 588
 589         if (!hdev) {
 590                 pr_crit("Closing FD after device was removed. Memory leak will occur and it is advised to reboot.\n");
 591                 put_pid(hpriv->taskpid);
 592         }
 593
 594         hl_ctx_mgr_fini(hdev, &hpriv->ctx_mgr);
 595
 596         /* Memory buffers might be still in use at this point and thus the handles IDR destruction
 597          * is postponed to hpriv_release().
 598          */
 599         hl_mem_mgr_fini(&hpriv->mem_mgr);
 600
 601         hdev->compute_ctx_in_release = 1;
 602
 603         if (!hl_hpriv_put(hpriv)) {
 604                 print_device_in_use_info(hdev, "User process closed FD but device still in use");
 605                 hl_device_reset(hdev, HL_DRV_RESET_HARD);
 606         }
 607
 608         hdev->last_open_session_duration_jif = jiffies - hdev->last_successful_open_jif;
 609 }
 610
 611 static int hl_device_release_ctrl(struct inode *inode, struct file *filp)
 612 {
 613         struct hl_fpriv *hpriv = filp->private_data;
 614         struct hl_device *hdev = hpriv->hdev;
 615
 616         filp->private_data = NULL;
 617
 618         if (!hdev) {
 619                 pr_err("Closing FD after device was removed\n");
 620                 goto out;
 621         }
 622
 623         mutex_lock(&hdev->fpriv_ctrl_list_lock);
 624         list_del(&hpriv->dev_node);
 625         mutex_unlock(&hdev->fpriv_ctrl_list_lock);
 626 out:
 627         put_pid(hpriv->taskpid);
 628
 629         kfree(hpriv);
 630
 631         return 0;
 632 }
 633
 634 static int __hl_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma)
 635 {
 636         struct hl_device *hdev = hpriv->hdev;
 637         unsigned long vm_pgoff;
 638
 639         if (!hdev) {
 640                 pr_err_ratelimited("Trying to mmap after device was removed! Please close FD\n");
 641                 return -ENODEV;
 642         }
 643
 644         vm_pgoff = vma->vm_pgoff;
 645
 646         switch (vm_pgoff & HL_MMAP_TYPE_MASK) {
 647         case HL_MMAP_TYPE_BLOCK:
 648                 vma->vm_pgoff = HL_MMAP_OFFSET_VALUE_GET(vm_pgoff);
 649                 return hl_hw_block_mmap(hpriv, vma);
 650
 651         case HL_MMAP_TYPE_CB:
 652         case HL_MMAP_TYPE_TS_BUFF:
 653                 return hl_mem_mgr_mmap(&hpriv->mem_mgr, vma, NULL);
 654         }
 655         return -EINVAL;
 656 }
 657
 658 /*
 659  * hl_mmap - mmap function for habanalabs device
 660  *
 661  * @*filp: pointer to file structure
 662  * @*vma: pointer to vm_area_struct of the process
 663  *
 664  * Called when process does an mmap on habanalabs device. Call the relevant mmap
 665  * function at the end of the common code.
 666  */
 667 int hl_mmap(struct file *filp, struct vm_area_struct *vma)
 668 {
 669         struct drm_file *file_priv = filp->private_data;
 670         struct hl_fpriv *hpriv = file_priv->driver_priv;
 671
 672         return __hl_mmap(hpriv, vma);
 673 }
 674
 675 static const struct file_operations hl_ctrl_ops = {
 676         .owner = THIS_MODULE,
 677         .open = hl_device_open_ctrl,
 678         .release = hl_device_release_ctrl,
 679         .unlocked_ioctl = hl_ioctl_control,
 680         .compat_ioctl = hl_ioctl_control
 681 };
 682
 683 static void device_release_func(struct device *dev)
 684 {
 685         kfree(dev);
 686 }
 687
 688 /*
 689  * device_init_cdev - Initialize cdev and device for habanalabs device
 690  *
 691  * @hdev: pointer to habanalabs device structure
 692  * @class: pointer to the class object of the device
 693  * @minor: minor number of the specific device
 694  * @fops: file operations to install for this device
 695  * @name: name of the device as it will appear in the filesystem
 696  * @cdev: pointer to the char device object that will be initialized
 697  * @dev: pointer to the device object that will be initialized
 698  *
 699  * Initialize a cdev and a Linux device for habanalabs's device.
 700  */
 701 static int device_init_cdev(struct hl_device *hdev, const struct class *class,
 702                                 int minor, const struct file_operations *fops,
 703                                 char *name, struct cdev *cdev,
 704                                 struct device **dev)
 705 {
 706         cdev_init(cdev, fops);
 707         cdev->owner = THIS_MODULE;
 708
 709         *dev = kzalloc(sizeof(**dev), GFP_KERNEL);
 710         if (!*dev)
 711                 return -ENOMEM;
 712
 713         device_initialize(*dev);
 714         (*dev)->devt = MKDEV(hdev->major, minor);
 715         (*dev)->class = class;
 716         (*dev)->release = device_release_func;
 717         dev_set_drvdata(*dev, hdev);
 718         dev_set_name(*dev, "%s", name);
 719
 720         return 0;
 721 }
 722
 723 static int cdev_sysfs_debugfs_add(struct hl_device *hdev)
 724 {
 725         const struct class *accel_class = hdev->drm.accel->kdev->class;
 726         char name[32];
 727         int rc;
 728
 729         hdev->cdev_idx = hdev->drm.accel->index;
 730
 731         /* Initialize cdev and device structures for the control device */
 732         snprintf(name, sizeof(name), "accel_controlD%d", hdev->cdev_idx);
 733         rc = device_init_cdev(hdev, accel_class, hdev->cdev_idx, &hl_ctrl_ops, name,
 734                                 &hdev->cdev_ctrl, &hdev->dev_ctrl);
 735         if (rc)
 736                 return rc;
 737
 738         rc = cdev_device_add(&hdev->cdev_ctrl, hdev->dev_ctrl);
 739         if (rc) {
 740                 dev_err(hdev->dev_ctrl,
 741                         "failed to add an accel control char device to the system\n");
 742                 goto free_ctrl_device;
 743         }
 744
 745         rc = hl_sysfs_init(hdev);
 746         if (rc) {
 747                 dev_err(hdev->dev, "failed to initialize sysfs\n");
 748                 goto delete_ctrl_cdev_device;
 749         }
 750
 751         hl_debugfs_add_device(hdev);
 752
 753         hdev->cdev_sysfs_debugfs_created = true;
 754
 755         return 0;
 756
 757 delete_ctrl_cdev_device:
 758         cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
 759 free_ctrl_device:
 760         put_device(hdev->dev_ctrl);
 761         return rc;
 762 }
 763
 764 static void cdev_sysfs_debugfs_remove(struct hl_device *hdev)
 765 {
 766         if (!hdev->cdev_sysfs_debugfs_created)
 767                 return;
 768
 769         hl_sysfs_fini(hdev);
 770
 771         cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
 772         put_device(hdev->dev_ctrl);
 773 }
 774
 775 static void device_hard_reset_pending(struct work_struct *work)
 776 {
 777         struct hl_device_reset_work *device_reset_work =
 778                 container_of(work, struct hl_device_reset_work, reset_work.work);
 779         struct hl_device *hdev = device_reset_work->hdev;
 780         u32 flags;
 781         int rc;
 782
 783         flags = device_reset_work->flags | HL_DRV_RESET_FROM_RESET_THR;
 784
 785         rc = hl_device_reset(hdev, flags);
 786
 787         if ((rc == -EBUSY) && !hdev->device_fini_pending) {
 788                 struct hl_ctx *ctx = hl_get_compute_ctx(hdev);
 789
 790                 if (ctx) {
 791                         /* The read refcount value should subtracted by one, because the read is
 792                          * protected with hl_get_compute_ctx().
 793                          */
 794                         dev_info(hdev->dev,
 795                                 "Could not reset device (compute_ctx refcount %u). will try again in %u seconds",
 796                                 kref_read(&ctx->refcount) - 1, HL_PENDING_RESET_PER_SEC);
 797                         hl_ctx_put(ctx);
 798                 } else {
 799                         dev_info(hdev->dev, "Could not reset device. will try again in %u seconds",
 800                                 HL_PENDING_RESET_PER_SEC);
 801                 }
 802
 803                 queue_delayed_work(hdev->reset_wq, &device_reset_work->reset_work,
 804                                         msecs_to_jiffies(HL_PENDING_RESET_PER_SEC * 1000));
 805         }
 806 }
 807
 808 static void device_release_watchdog_func(struct work_struct *work)
 809 {
 810         struct hl_device_reset_work *watchdog_work =
 811                         container_of(work, struct hl_device_reset_work, reset_work.work);
 812         struct hl_device *hdev = watchdog_work->hdev;
 813         u32 flags;
 814
 815         dev_dbg(hdev->dev, "Device wasn't released in time. Initiate hard-reset.\n");
 816
 817         flags = watchdog_work->flags | HL_DRV_RESET_HARD | HL_DRV_RESET_FROM_WD_THR;
 818
 819         hl_device_reset(hdev, flags);
 820 }
 821
 822 /*
 823  * device_early_init - do some early initialization for the habanalabs device
 824  *
 825  * @hdev: pointer to habanalabs device structure
 826  *
 827  * Install the relevant function pointers and call the early_init function,
 828  * if such a function exists
 829  */
 830 static int device_early_init(struct hl_device *hdev)
 831 {
 832         int i, rc;
 833         char workq_name[32];
 834
 835         switch (hdev->asic_type) {
 836         case ASIC_GOYA:
 837                 goya_set_asic_funcs(hdev);
 838                 strscpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name));
 839                 break;
 840         case ASIC_GAUDI:
 841                 gaudi_set_asic_funcs(hdev);
 842                 strscpy(hdev->asic_name, "GAUDI", sizeof(hdev->asic_name));
 843                 break;
 844         case ASIC_GAUDI_SEC:
 845                 gaudi_set_asic_funcs(hdev);
 846                 strscpy(hdev->asic_name, "GAUDI SEC", sizeof(hdev->asic_name));
 847                 break;
 848         case ASIC_GAUDI2:
 849                 gaudi2_set_asic_funcs(hdev);
 850                 strscpy(hdev->asic_name, "GAUDI2", sizeof(hdev->asic_name));
 851                 break;
 852         case ASIC_GAUDI2B:
 853                 gaudi2_set_asic_funcs(hdev);
 854                 strscpy(hdev->asic_name, "GAUDI2B", sizeof(hdev->asic_name));
 855                 break;
 856         case ASIC_GAUDI2C:
 857                 gaudi2_set_asic_funcs(hdev);
 858                 strscpy(hdev->asic_name, "GAUDI2C", sizeof(hdev->asic_name));
 859                 break;
 860         default:
 861                 dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
 862                         hdev->asic_type);
 863                 return -EINVAL;
 864         }
 865
 866         rc = hdev->asic_funcs->early_init(hdev);
 867         if (rc)
 868                 return rc;
 869
 870         rc = hl_asid_init(hdev);
 871         if (rc)
 872                 goto early_fini;
 873
 874         if (hdev->asic_prop.completion_queues_count) {
 875                 hdev->cq_wq = kcalloc(hdev->asic_prop.completion_queues_count,
 876                                 sizeof(struct workqueue_struct *),
 877                                 GFP_KERNEL);
 878                 if (!hdev->cq_wq) {
 879                         rc = -ENOMEM;
 880                         goto asid_fini;
 881                 }
 882         }
 883
 884         for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) {
 885                 snprintf(workq_name, 32, "hl%u-free-jobs-%u", hdev->cdev_idx, (u32) i);
 886                 hdev->cq_wq[i] = create_singlethread_workqueue(workq_name);
 887                 if (hdev->cq_wq[i] == NULL) {
 888                         dev_err(hdev->dev, "Failed to allocate CQ workqueue\n");
 889                         rc = -ENOMEM;
 890                         goto free_cq_wq;
 891                 }
 892         }
 893
 894         snprintf(workq_name, 32, "hl%u-events", hdev->cdev_idx);
 895         hdev->eq_wq = create_singlethread_workqueue(workq_name);
 896         if (hdev->eq_wq == NULL) {
 897                 dev_err(hdev->dev, "Failed to allocate EQ workqueue\n");
 898                 rc = -ENOMEM;
 899                 goto free_cq_wq;
 900         }
 901
 902         snprintf(workq_name, 32, "hl%u-cs-completions", hdev->cdev_idx);
 903         hdev->cs_cmplt_wq = alloc_workqueue(workq_name, WQ_UNBOUND, 0);
 904         if (!hdev->cs_cmplt_wq) {
 905                 dev_err(hdev->dev,
 906                         "Failed to allocate CS completions workqueue\n");
 907                 rc = -ENOMEM;
 908                 goto free_eq_wq;
 909         }
 910
 911         snprintf(workq_name, 32, "hl%u-ts-free-obj", hdev->cdev_idx);
 912         hdev->ts_free_obj_wq = alloc_workqueue(workq_name, WQ_UNBOUND, 0);
 913         if (!hdev->ts_free_obj_wq) {
 914                 dev_err(hdev->dev,
 915                         "Failed to allocate Timestamp registration free workqueue\n");
 916                 rc = -ENOMEM;
 917                 goto free_cs_cmplt_wq;
 918         }
 919
 920         snprintf(workq_name, 32, "hl%u-prefetch", hdev->cdev_idx);
 921         hdev->prefetch_wq = alloc_workqueue(workq_name, WQ_UNBOUND, 0);
 922         if (!hdev->prefetch_wq) {
 923                 dev_err(hdev->dev, "Failed to allocate MMU prefetch workqueue\n");
 924                 rc = -ENOMEM;
 925                 goto free_ts_free_wq;
 926         }
 927
 928         hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info), GFP_KERNEL);
 929         if (!hdev->hl_chip_info) {
 930                 rc = -ENOMEM;
 931                 goto free_prefetch_wq;
 932         }
 933
 934         rc = hl_mmu_if_set_funcs(hdev);
 935         if (rc)
 936                 goto free_chip_info;
 937
 938         hl_mem_mgr_init(hdev->dev, &hdev->kernel_mem_mgr);
 939
 940         snprintf(workq_name, 32, "hl%u_device_reset", hdev->cdev_idx);
 941         hdev->reset_wq = create_singlethread_workqueue(workq_name);
 942         if (!hdev->reset_wq) {
 943                 rc = -ENOMEM;
 944                 dev_err(hdev->dev, "Failed to create device reset WQ\n");
 945                 goto free_cb_mgr;
 946         }
 947
 948         INIT_DELAYED_WORK(&hdev->device_reset_work.reset_work, device_hard_reset_pending);
 949         hdev->device_reset_work.hdev = hdev;
 950         hdev->device_fini_pending = 0;
 951
 952         INIT_DELAYED_WORK(&hdev->device_release_watchdog_work.reset_work,
 953                                 device_release_watchdog_func);
 954         hdev->device_release_watchdog_work.hdev = hdev;
 955
 956         mutex_init(&hdev->send_cpu_message_lock);
 957         mutex_init(&hdev->debug_lock);
 958         INIT_LIST_HEAD(&hdev->cs_mirror_list);
 959         spin_lock_init(&hdev->cs_mirror_lock);
 960         spin_lock_init(&hdev->reset_info.lock);
 961         INIT_LIST_HEAD(&hdev->fpriv_list);
 962         INIT_LIST_HEAD(&hdev->fpriv_ctrl_list);
 963         mutex_init(&hdev->fpriv_list_lock);
 964         mutex_init(&hdev->fpriv_ctrl_list_lock);
 965         mutex_init(&hdev->clk_throttling.lock);
 966
 967         return 0;
 968
 969 free_cb_mgr:
 970         hl_mem_mgr_fini(&hdev->kernel_mem_mgr);
 971         hl_mem_mgr_idr_destroy(&hdev->kernel_mem_mgr);
 972 free_chip_info:
 973         kfree(hdev->hl_chip_info);
 974 free_prefetch_wq:
 975         destroy_workqueue(hdev->prefetch_wq);
 976 free_ts_free_wq:
 977         destroy_workqueue(hdev->ts_free_obj_wq);
 978 free_cs_cmplt_wq:
 979         destroy_workqueue(hdev->cs_cmplt_wq);
 980 free_eq_wq:
 981         destroy_workqueue(hdev->eq_wq);
 982 free_cq_wq:
 983         for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
 984                 if (hdev->cq_wq[i])
 985                         destroy_workqueue(hdev->cq_wq[i]);
 986         kfree(hdev->cq_wq);
 987 asid_fini:
 988         hl_asid_fini(hdev);
 989 early_fini:
 990         if (hdev->asic_funcs->early_fini)
 991                 hdev->asic_funcs->early_fini(hdev);
 992
 993         return rc;
 994 }
 995
 996 /*
 997  * device_early_fini - finalize all that was done in device_early_init
 998  *
 999  * @hdev: pointer to habanalabs device structure
1000  *
1001  */
1002 static void device_early_fini(struct hl_device *hdev)
1003 {
1004         int i;
1005
1006         mutex_destroy(&hdev->debug_lock);
1007         mutex_destroy(&hdev->send_cpu_message_lock);
1008
1009         mutex_destroy(&hdev->fpriv_list_lock);
1010         mutex_destroy(&hdev->fpriv_ctrl_list_lock);
1011
1012         mutex_destroy(&hdev->clk_throttling.lock);
1013
1014         hl_mem_mgr_fini(&hdev->kernel_mem_mgr);
1015         hl_mem_mgr_idr_destroy(&hdev->kernel_mem_mgr);
1016
1017         kfree(hdev->hl_chip_info);
1018
1019         destroy_workqueue(hdev->prefetch_wq);
1020         destroy_workqueue(hdev->ts_free_obj_wq);
1021         destroy_workqueue(hdev->cs_cmplt_wq);
1022         destroy_workqueue(hdev->eq_wq);
1023         destroy_workqueue(hdev->reset_wq);
1024
1025         for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
1026                 destroy_workqueue(hdev->cq_wq[i]);
1027         kfree(hdev->cq_wq);
1028
1029         hl_asid_fini(hdev);
1030
1031         if (hdev->asic_funcs->early_fini)
1032                 hdev->asic_funcs->early_fini(hdev);
1033 }
1034
1035 static bool is_pci_link_healthy(struct hl_device *hdev)
1036 {
1037         u16 vendor_id;
1038
1039         if (!hdev->pdev)
1040                 return false;
1041
1042         pci_read_config_word(hdev->pdev, PCI_VENDOR_ID, &vendor_id);
1043
1044         return (vendor_id == PCI_VENDOR_ID_HABANALABS);
1045 }
1046
1047 static int hl_device_eq_heartbeat_check(struct hl_device *hdev)
1048 {
1049         struct asic_fixed_properties *prop = &hdev->asic_prop;
1050
1051         if (!prop->cpucp_info.eq_health_check_supported)
1052                 return 0;
1053
1054         if (hdev->eq_heartbeat_received) {
1055                 hdev->eq_heartbeat_received = false;
1056         } else {
1057                 dev_err(hdev->dev, "EQ heartbeat event was not received!\n");
1058                 return -EIO;
1059         }
1060
1061         return 0;
1062 }
1063
1064 static void hl_device_heartbeat(struct work_struct *work)
1065 {
1066         struct hl_device *hdev = container_of(work, struct hl_device,
1067                                                 work_heartbeat.work);
1068         struct hl_info_fw_err_info info = {0};
1069         u64 event_mask = HL_NOTIFIER_EVENT_DEVICE_RESET | HL_NOTIFIER_EVENT_DEVICE_UNAVAILABLE;
1070
1071         /* Start heartbeat checks only after driver has enabled events from FW */
1072         if (!hl_device_operational(hdev, NULL) || !hdev->init_done)
1073                 goto reschedule;
1074
1075         /*
1076          * For EQ health check need to check if driver received the heartbeat eq event
1077          * in order to validate the eq is working.
1078          * Only if both the EQ is healthy and we managed to send the next heartbeat reschedule.
1079          */
1080         if ((!hl_device_eq_heartbeat_check(hdev)) && (!hdev->asic_funcs->send_heartbeat(hdev)))
1081                 goto reschedule;
1082
1083         if (hl_device_operational(hdev, NULL))
1084                 dev_err(hdev->dev, "Device heartbeat failed! PCI link is %s\n",
1085                         is_pci_link_healthy(hdev) ? "healthy" : "broken");
1086
1087         info.err_type = HL_INFO_FW_HEARTBEAT_ERR;
1088         info.event_mask = &event_mask;
1089         hl_handle_fw_err(hdev, &info);
1090         hl_device_cond_reset(hdev, HL_DRV_RESET_HARD | HL_DRV_RESET_HEARTBEAT, event_mask);
1091
1092         return;
1093
1094 reschedule:
1095         /*
1096          * prev_reset_trigger tracks consecutive fatal h/w errors until first
1097          * heartbeat immediately post reset.
1098          * If control reached here, then at least one heartbeat work has been
1099          * scheduled since last reset/init cycle.
1100          * So if the device is not already in reset cycle, reset the flag
1101          * prev_reset_trigger as no reset occurred with HL_DRV_RESET_FW_FATAL_ERR
1102          * status for at least one heartbeat. From this point driver restarts
1103          * tracking future consecutive fatal errors.
1104          */
1105         if (!hdev->reset_info.in_reset)
1106                 hdev->reset_info.prev_reset_trigger = HL_RESET_TRIGGER_DEFAULT;
1107
1108         schedule_delayed_work(&hdev->work_heartbeat,
1109                         usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
1110 }
1111
1112 /*
1113  * device_late_init - do late stuff initialization for the habanalabs device
1114  *
1115  * @hdev: pointer to habanalabs device structure
1116  *
1117  * Do stuff that either needs the device H/W queues to be active or needs
1118  * to happen after all the rest of the initialization is finished
1119  */
1120 static int device_late_init(struct hl_device *hdev)
1121 {
1122         int rc;
1123
1124         if (hdev->asic_funcs->late_init) {
1125                 rc = hdev->asic_funcs->late_init(hdev);
1126                 if (rc) {
1127                         dev_err(hdev->dev,
1128                                 "failed late initialization for the H/W\n");
1129                         return rc;
1130                 }
1131         }
1132
1133         hdev->high_pll = hdev->asic_prop.high_pll;
1134
1135         if (hdev->heartbeat) {
1136                 /*
1137                  * Before scheduling the heartbeat driver will check if eq event has received.
1138                  * for the first schedule we need to set the indication as true then for the next
1139                  * one this indication will be true only if eq event was sent by FW.
1140                  */
1141                 hdev->eq_heartbeat_received = true;
1142
1143                 INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat);
1144
1145                 schedule_delayed_work(&hdev->work_heartbeat,
1146                                 usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
1147         }
1148
1149         hdev->late_init_done = true;
1150
1151         return 0;
1152 }
1153
1154 /*
1155  * device_late_fini - finalize all that was done in device_late_init
1156  *
1157  * @hdev: pointer to habanalabs device structure
1158  *
1159  */
1160 static void device_late_fini(struct hl_device *hdev)
1161 {
1162         if (!hdev->late_init_done)
1163                 return;
1164
1165         if (hdev->heartbeat)
1166                 cancel_delayed_work_sync(&hdev->work_heartbeat);
1167
1168         if (hdev->asic_funcs->late_fini)
1169                 hdev->asic_funcs->late_fini(hdev);
1170
1171         hdev->late_init_done = false;
1172 }
1173
1174 int hl_device_utilization(struct hl_device *hdev, u32 *utilization)
1175 {
1176         u64 max_power, curr_power, dc_power, dividend, divisor;
1177         int rc;
1178
1179         max_power = hdev->max_power;
1180         dc_power = hdev->asic_prop.dc_power_default;
1181         divisor = max_power - dc_power;
1182         if (!divisor) {
1183                 dev_warn(hdev->dev, "device utilization is not supported\n");
1184                 return -EOPNOTSUPP;
1185         }
1186         rc = hl_fw_cpucp_power_get(hdev, &curr_power);
1187
1188         if (rc)
1189                 return rc;
1190
1191         curr_power = clamp(curr_power, dc_power, max_power);
1192
1193         dividend = (curr_power - dc_power) * 100;
1194         *utilization = (u32) div_u64(dividend, divisor);
1195
1196         return 0;
1197 }
1198
1199 int hl_device_set_debug_mode(struct hl_device *hdev, struct hl_ctx *ctx, bool enable)
1200 {
1201         int rc = 0;
1202
1203         mutex_lock(&hdev->debug_lock);
1204
1205         if (!enable) {
1206                 if (!hdev->in_debug) {
1207                         dev_err(hdev->dev,
1208                                 "Failed to disable debug mode because device was not in debug mode\n");
1209                         rc = -EFAULT;
1210                         goto out;
1211                 }
1212
1213                 if (!hdev->reset_info.hard_reset_pending)
1214                         hdev->asic_funcs->halt_coresight(hdev, ctx);
1215
1216                 hdev->in_debug = 0;
1217
1218                 goto out;
1219         }
1220
1221         if (hdev->in_debug) {
1222                 dev_err(hdev->dev,
1223                         "Failed to enable debug mode because device is already in debug mode\n");
1224                 rc = -EFAULT;
1225                 goto out;
1226         }
1227
1228         hdev->in_debug = 1;
1229
1230 out:
1231         mutex_unlock(&hdev->debug_lock);
1232
1233         return rc;
1234 }
1235
1236 static void take_release_locks(struct hl_device *hdev)
1237 {
1238         /* Flush anyone that is inside the critical section of enqueue
1239          * jobs to the H/W
1240          */
1241         hdev->asic_funcs->hw_queues_lock(hdev);
1242         hdev->asic_funcs->hw_queues_unlock(hdev);
1243
1244         /* Flush processes that are sending message to CPU */
1245         mutex_lock(&hdev->send_cpu_message_lock);
1246         mutex_unlock(&hdev->send_cpu_message_lock);
1247
1248         /* Flush anyone that is inside device open */
1249         mutex_lock(&hdev->fpriv_list_lock);
1250         mutex_unlock(&hdev->fpriv_list_lock);
1251         mutex_lock(&hdev->fpriv_ctrl_list_lock);
1252         mutex_unlock(&hdev->fpriv_ctrl_list_lock);
1253 }
1254
1255 static void hl_abort_waiting_for_completions(struct hl_device *hdev)
1256 {
1257         hl_abort_waiting_for_cs_completions(hdev);
1258
1259         /* Release all pending user interrupts, each pending user interrupt
1260          * holds a reference to a user context.
1261          */
1262         hl_release_pending_user_interrupts(hdev);
1263 }
1264
1265 static void cleanup_resources(struct hl_device *hdev, bool hard_reset, bool fw_reset,
1266                                 bool skip_wq_flush)
1267 {
1268         if (hard_reset)
1269                 device_late_fini(hdev);
1270
1271         /*
1272          * Halt the engines and disable interrupts so we won't get any more
1273          * completions from H/W and we won't have any accesses from the
1274          * H/W to the host machine
1275          */
1276         hdev->asic_funcs->halt_engines(hdev, hard_reset, fw_reset);
1277
1278         /* Go over all the queues, release all CS and their jobs */
1279         hl_cs_rollback_all(hdev, skip_wq_flush);
1280
1281         /* flush the MMU prefetch workqueue */
1282         flush_workqueue(hdev->prefetch_wq);
1283
1284         hl_abort_waiting_for_completions(hdev);
1285 }
1286
1287 /*
1288  * hl_device_suspend - initiate device suspend
1289  *
1290  * @hdev: pointer to habanalabs device structure
1291  *
1292  * Puts the hw in the suspend state (all asics).
1293  * Returns 0 for success or an error on failure.
1294  * Called at driver suspend.
1295  */
1296 int hl_device_suspend(struct hl_device *hdev)
1297 {
1298         int rc;
1299
1300         pci_save_state(hdev->pdev);
1301
1302         /* Block future CS/VM/JOB completion operations */
1303         spin_lock(&hdev->reset_info.lock);
1304         if (hdev->reset_info.in_reset) {
1305                 spin_unlock(&hdev->reset_info.lock);
1306                 dev_err(hdev->dev, "Can't suspend while in reset\n");
1307                 return -EIO;
1308         }
1309         hdev->reset_info.in_reset = 1;
1310         spin_unlock(&hdev->reset_info.lock);
1311
1312         /* This blocks all other stuff that is not blocked by in_reset */
1313         hdev->disabled = true;
1314
1315         take_release_locks(hdev);
1316
1317         rc = hdev->asic_funcs->suspend(hdev);
1318         if (rc)
1319                 dev_err(hdev->dev,
1320                         "Failed to disable PCI access of device CPU\n");
1321
1322         /* Shut down the device */
1323         pci_disable_device(hdev->pdev);
1324         pci_set_power_state(hdev->pdev, PCI_D3hot);
1325
1326         return 0;
1327 }
1328
1329 /*
1330  * hl_device_resume - initiate device resume
1331  *
1332  * @hdev: pointer to habanalabs device structure
1333  *
1334  * Bring the hw back to operating state (all asics).
1335  * Returns 0 for success or an error on failure.
1336  * Called at driver resume.
1337  */
1338 int hl_device_resume(struct hl_device *hdev)
1339 {
1340         int rc;
1341
1342         pci_set_power_state(hdev->pdev, PCI_D0);
1343         pci_restore_state(hdev->pdev);
1344         rc = pci_enable_device_mem(hdev->pdev);
1345         if (rc) {
1346                 dev_err(hdev->dev,
1347                         "Failed to enable PCI device in resume\n");
1348                 return rc;
1349         }
1350
1351         pci_set_master(hdev->pdev);
1352
1353         rc = hdev->asic_funcs->resume(hdev);
1354         if (rc) {
1355                 dev_err(hdev->dev, "Failed to resume device after suspend\n");
1356                 goto disable_device;
1357         }
1358
1359
1360         /* 'in_reset' was set to true during suspend, now we must clear it in order
1361          * for hard reset to be performed
1362          */
1363         spin_lock(&hdev->reset_info.lock);
1364         hdev->reset_info.in_reset = 0;
1365         spin_unlock(&hdev->reset_info.lock);
1366
1367         rc = hl_device_reset(hdev, HL_DRV_RESET_HARD);
1368         if (rc) {
1369                 dev_err(hdev->dev, "Failed to reset device during resume\n");
1370                 goto disable_device;
1371         }
1372
1373         return 0;
1374
1375 disable_device:
1376         pci_disable_device(hdev->pdev);
1377
1378         return rc;
1379 }
1380
1381 static int device_kill_open_processes(struct hl_device *hdev, u32 timeout, bool control_dev)
1382 {
1383         struct task_struct *task = NULL;
1384         struct list_head *hpriv_list;
1385         struct hl_fpriv *hpriv;
1386         struct mutex *hpriv_lock;
1387         u32 pending_cnt;
1388
1389         hpriv_lock = control_dev ? &hdev->fpriv_ctrl_list_lock : &hdev->fpriv_list_lock;
1390         hpriv_list = control_dev ? &hdev->fpriv_ctrl_list : &hdev->fpriv_list;
1391
1392         /* Giving time for user to close FD, and for processes that are inside
1393          * hl_device_open to finish
1394          */
1395         if (!list_empty(hpriv_list))
1396                 ssleep(1);
1397
1398         if (timeout) {
1399                 pending_cnt = timeout;
1400         } else {
1401                 if (hdev->process_kill_trial_cnt) {
1402                         /* Processes have been already killed */
1403                         pending_cnt = 1;
1404                         goto wait_for_processes;
1405                 } else {
1406                         /* Wait a small period after process kill */
1407                         pending_cnt = HL_PENDING_RESET_PER_SEC;
1408                 }
1409         }
1410
1411         mutex_lock(hpriv_lock);
1412
1413         /* This section must be protected because we are dereferencing
1414          * pointers that are freed if the process exits
1415          */
1416         list_for_each_entry(hpriv, hpriv_list, dev_node) {
1417                 task = get_pid_task(hpriv->taskpid, PIDTYPE_PID);
1418                 if (task) {
1419                         dev_info(hdev->dev, "Killing user process pid=%d\n",
1420                                 task_pid_nr(task));
1421                         send_sig(SIGKILL, task, 1);
1422                         usleep_range(1000, 10000);
1423
1424                         put_task_struct(task);
1425                 } else {
1426                         dev_dbg(hdev->dev,
1427                                 "Can't get task struct for user process %d, process was killed from outside the driver\n",
1428                                 pid_nr(hpriv->taskpid));
1429                 }
1430         }
1431
1432         mutex_unlock(hpriv_lock);
1433
1434         /*
1435          * We killed the open users, but that doesn't mean they are closed.
1436          * It could be that they are running a long cleanup phase in the driver
1437          * e.g. MMU unmappings, or running other long teardown flow even before
1438          * our cleanup.
1439          * Therefore we need to wait again to make sure they are closed before
1440          * continuing with the reset.
1441          */
1442
1443 wait_for_processes:
1444         while ((!list_empty(hpriv_list)) && (pending_cnt)) {
1445                 dev_dbg(hdev->dev,
1446                         "Waiting for all unmap operations to finish before hard reset\n");
1447
1448                 pending_cnt--;
1449
1450                 ssleep(1);
1451         }
1452
1453         /* All processes exited successfully */
1454         if (list_empty(hpriv_list))
1455                 return 0;
1456
1457         /* Give up waiting for processes to exit */
1458         if (hdev->process_kill_trial_cnt == HL_PENDING_RESET_MAX_TRIALS)
1459                 return -ETIME;
1460
1461         hdev->process_kill_trial_cnt++;
1462
1463         return -EBUSY;
1464 }
1465
1466 static void device_disable_open_processes(struct hl_device *hdev, bool control_dev)
1467 {
1468         struct list_head *hpriv_list;
1469         struct hl_fpriv *hpriv;
1470         struct mutex *hpriv_lock;
1471
1472         hpriv_lock = control_dev ? &hdev->fpriv_ctrl_list_lock : &hdev->fpriv_list_lock;
1473         hpriv_list = control_dev ? &hdev->fpriv_ctrl_list : &hdev->fpriv_list;
1474
1475         mutex_lock(hpriv_lock);
1476         list_for_each_entry(hpriv, hpriv_list, dev_node)
1477                 hpriv->hdev = NULL;
1478         mutex_unlock(hpriv_lock);
1479 }
1480
1481 static void send_disable_pci_access(struct hl_device *hdev, u32 flags)
1482 {
1483         /* If reset is due to heartbeat, device CPU is no responsive in
1484          * which case no point sending PCI disable message to it.
1485          */
1486         if ((flags & HL_DRV_RESET_HARD) &&
1487                         !(flags & (HL_DRV_RESET_HEARTBEAT | HL_DRV_RESET_BYPASS_REQ_TO_FW))) {
1488                 /* Disable PCI access from device F/W so he won't send
1489                  * us additional interrupts. We disable MSI/MSI-X at
1490                  * the halt_engines function and we can't have the F/W
1491                  * sending us interrupts after that. We need to disable
1492                  * the access here because if the device is marked
1493                  * disable, the message won't be send. Also, in case
1494                  * of heartbeat, the device CPU is marked as disable
1495                  * so this message won't be sent
1496                  */
1497                 if (hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0)) {
1498                         dev_warn(hdev->dev, "Failed to disable FW's PCI access\n");
1499                         return;
1500                 }
1501
1502                 /* verify that last EQs are handled before disabled is set */
1503                 if (hdev->cpu_queues_enable)
1504                         synchronize_irq(pci_irq_vector(hdev->pdev,
1505                                         hdev->asic_prop.eq_interrupt_id));
1506         }
1507 }
1508
1509 static void handle_reset_trigger(struct hl_device *hdev, u32 flags)
1510 {
1511         u32 cur_reset_trigger = HL_RESET_TRIGGER_DEFAULT;
1512
1513         /* No consecutive mechanism when user context exists */
1514         if (hdev->is_compute_ctx_active)
1515                 return;
1516
1517         /*
1518          * 'reset cause' is being updated here, because getting here
1519          * means that it's the 1st time and the last time we're here
1520          * ('in_reset' makes sure of it). This makes sure that
1521          * 'reset_cause' will continue holding its 1st recorded reason!
1522          */
1523         if (flags & HL_DRV_RESET_HEARTBEAT) {
1524                 hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_HEARTBEAT;
1525                 cur_reset_trigger = HL_DRV_RESET_HEARTBEAT;
1526         } else if (flags & HL_DRV_RESET_TDR) {
1527                 hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_TDR;
1528                 cur_reset_trigger = HL_DRV_RESET_TDR;
1529         } else if (flags & HL_DRV_RESET_FW_FATAL_ERR) {
1530                 hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
1531                 cur_reset_trigger = HL_DRV_RESET_FW_FATAL_ERR;
1532         } else {
1533                 hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
1534         }
1535
1536         /*
1537          * If reset cause is same twice, then reset_trigger_repeated
1538          * is set and if this reset is due to a fatal FW error
1539          * device is set to an unstable state.
1540          */
1541         if (hdev->reset_info.prev_reset_trigger != cur_reset_trigger) {
1542                 hdev->reset_info.prev_reset_trigger = cur_reset_trigger;
1543                 hdev->reset_info.reset_trigger_repeated = 0;
1544         } else {
1545                 hdev->reset_info.reset_trigger_repeated = 1;
1546         }
1547 }
1548
1549 /*
1550  * hl_device_reset - reset the device
1551  *
1552  * @hdev: pointer to habanalabs device structure
1553  * @flags: reset flags.
1554  *
1555  * Block future CS and wait for pending CS to be enqueued
1556  * Call ASIC H/W fini
1557  * Flush all completions
1558  * Re-initialize all internal data structures
1559  * Call ASIC H/W init, late_init
1560  * Test queues
1561  * Enable device
1562  *
1563  * Returns 0 for success or an error on failure.
1564  */
1565 int hl_device_reset(struct hl_device *hdev, u32 flags)
1566 {
1567         bool hard_reset, from_hard_reset_thread, fw_reset, reset_upon_device_release,
1568                 schedule_hard_reset = false, delay_reset, from_dev_release, from_watchdog_thread;
1569         u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0};
1570         struct hl_ctx *ctx;
1571         int i, rc, hw_fini_rc;
1572
1573         if (!hdev->init_done) {
1574                 dev_err(hdev->dev, "Can't reset before initialization is done\n");
1575                 return 0;
1576         }
1577
1578         hard_reset = !!(flags & HL_DRV_RESET_HARD);
1579         from_hard_reset_thread = !!(flags & HL_DRV_RESET_FROM_RESET_THR);
1580         fw_reset = !!(flags & HL_DRV_RESET_BYPASS_REQ_TO_FW);
1581         from_dev_release = !!(flags & HL_DRV_RESET_DEV_RELEASE);
1582         delay_reset = !!(flags & HL_DRV_RESET_DELAY);
1583         from_watchdog_thread = !!(flags & HL_DRV_RESET_FROM_WD_THR);
1584         reset_upon_device_release = hdev->reset_upon_device_release && from_dev_release;
1585
1586         if (!hard_reset && (hl_device_status(hdev) == HL_DEVICE_STATUS_MALFUNCTION)) {
1587                 dev_dbg(hdev->dev, "soft-reset isn't supported on a malfunctioning device\n");
1588                 return 0;
1589         }
1590
1591         if (!hard_reset && !hdev->asic_prop.supports_compute_reset) {
1592                 dev_dbg(hdev->dev, "asic doesn't support compute reset - do hard-reset instead\n");
1593                 hard_reset = true;
1594         }
1595
1596         if (reset_upon_device_release) {
1597                 if (hard_reset) {
1598                         dev_crit(hdev->dev,
1599                                 "Aborting reset because hard-reset is mutually exclusive with reset-on-device-release\n");
1600                         return -EINVAL;
1601                 }
1602
1603                 goto do_reset;
1604         }
1605
1606         if (!hard_reset && !hdev->asic_prop.allow_inference_soft_reset) {
1607                 dev_dbg(hdev->dev,
1608                         "asic doesn't allow inference soft reset - do hard-reset instead\n");
1609                 hard_reset = true;
1610         }
1611
1612 do_reset:
1613         /* Re-entry of reset thread */
1614         if (from_hard_reset_thread && hdev->process_kill_trial_cnt)
1615                 goto kill_processes;
1616
1617         /*
1618          * Prevent concurrency in this function - only one reset should be
1619          * done at any given time. We need to perform this only if we didn't
1620          * get here from a dedicated hard reset thread.
1621          */
1622         if (!from_hard_reset_thread) {
1623                 /* Block future CS/VM/JOB completion operations */
1624                 spin_lock(&hdev->reset_info.lock);
1625                 if (hdev->reset_info.in_reset) {
1626                         /* We allow scheduling of a hard reset only during a compute reset */
1627                         if (hard_reset && hdev->reset_info.in_compute_reset)
1628                                 hdev->reset_info.hard_reset_schedule_flags = flags;
1629                         spin_unlock(&hdev->reset_info.lock);
1630                         return 0;
1631                 }
1632
1633                 /* This still allows the completion of some KDMA ops
1634                  * Update this before in_reset because in_compute_reset implies we are in reset
1635                  */
1636                 hdev->reset_info.in_compute_reset = !hard_reset;
1637
1638                 hdev->reset_info.in_reset = 1;
1639
1640                 spin_unlock(&hdev->reset_info.lock);
1641
1642                 /* Cancel the device release watchdog work if required.
1643                  * In case of reset-upon-device-release while the release watchdog work is
1644                  * scheduled due to a hard-reset, do hard-reset instead of compute-reset.
1645                  */
1646                 if ((hard_reset || from_dev_release) && hdev->reset_info.watchdog_active) {
1647                         struct hl_device_reset_work *watchdog_work =
1648                                         &hdev->device_release_watchdog_work;
1649
1650                         hdev->reset_info.watchdog_active = 0;
1651                         if (!from_watchdog_thread)
1652                                 cancel_delayed_work_sync(&watchdog_work->reset_work);
1653
1654                         if (from_dev_release && (watchdog_work->flags & HL_DRV_RESET_HARD)) {
1655                                 hdev->reset_info.in_compute_reset = 0;
1656                                 flags |= HL_DRV_RESET_HARD;
1657                                 flags &= ~HL_DRV_RESET_DEV_RELEASE;
1658                                 hard_reset = true;
1659                         }
1660                 }
1661
1662                 if (delay_reset)
1663                         usleep_range(HL_RESET_DELAY_USEC, HL_RESET_DELAY_USEC << 1);
1664
1665 escalate_reset_flow:
1666                 handle_reset_trigger(hdev, flags);
1667                 send_disable_pci_access(hdev, flags);
1668
1669                 /* This also blocks future CS/VM/JOB completion operations */
1670                 hdev->disabled = true;
1671
1672                 take_release_locks(hdev);
1673
1674                 if (hard_reset)
1675                         dev_info(hdev->dev, "Going to reset device\n");
1676                 else if (reset_upon_device_release)
1677                         dev_dbg(hdev->dev, "Going to reset device after release by user\n");
1678                 else
1679                         dev_dbg(hdev->dev, "Going to reset engines of inference device\n");
1680         }
1681
1682         if ((hard_reset) && (!from_hard_reset_thread)) {
1683                 hdev->reset_info.hard_reset_pending = true;
1684
1685                 hdev->process_kill_trial_cnt = 0;
1686
1687                 hdev->device_reset_work.flags = flags;
1688
1689                 /*
1690                  * Because the reset function can't run from heartbeat work,
1691                  * we need to call the reset function from a dedicated work.
1692                  */
1693                 queue_delayed_work(hdev->reset_wq, &hdev->device_reset_work.reset_work, 0);
1694
1695                 return 0;
1696         }
1697
1698         cleanup_resources(hdev, hard_reset, fw_reset, from_dev_release);
1699
1700 kill_processes:
1701         if (hard_reset) {
1702                 /* Kill processes here after CS rollback. This is because the
1703                  * process can't really exit until all its CSs are done, which
1704                  * is what we do in cs rollback
1705                  */
1706                 rc = device_kill_open_processes(hdev, 0, false);
1707
1708                 if (rc == -EBUSY) {
1709                         if (hdev->device_fini_pending) {
1710                                 dev_crit(hdev->dev,
1711                                         "%s Failed to kill all open processes, stopping hard reset\n",
1712                                         dev_name(&(hdev)->pdev->dev));
1713                                 goto out_err;
1714                         }
1715
1716                         /* signal reset thread to reschedule */
1717                         return rc;
1718                 }
1719
1720                 if (rc) {
1721                         dev_crit(hdev->dev,
1722                                 "%s Failed to kill all open processes, stopping hard reset\n",
1723                                 dev_name(&(hdev)->pdev->dev));
1724                         goto out_err;
1725                 }
1726
1727                 /* Flush the Event queue workers to make sure no other thread is
1728                  * reading or writing to registers during the reset
1729                  */
1730                 flush_workqueue(hdev->eq_wq);
1731         }
1732
1733         /* Reset the H/W. It will be in idle state after this returns */
1734         hw_fini_rc = hdev->asic_funcs->hw_fini(hdev, hard_reset, fw_reset);
1735
1736         if (hard_reset) {
1737                 hdev->fw_loader.fw_comp_loaded = FW_TYPE_NONE;
1738
1739                 /* Release kernel context */
1740                 if (hdev->kernel_ctx && hl_ctx_put(hdev->kernel_ctx) == 1)
1741                         hdev->kernel_ctx = NULL;
1742
1743                 hl_vm_fini(hdev);
1744                 hl_mmu_fini(hdev);
1745                 hl_eq_reset(hdev, &hdev->event_queue);
1746         }
1747
1748         /* Re-initialize PI,CI to 0 in all queues (hw queue, cq) */
1749         hl_hw_queue_reset(hdev, hard_reset);
1750         for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
1751                 hl_cq_reset(hdev, &hdev->completion_queue[i]);
1752
1753         /* Make sure the context switch phase will run again */
1754         ctx = hl_get_compute_ctx(hdev);
1755         if (ctx) {
1756                 atomic_set(&ctx->thread_ctx_switch_token, 1);
1757                 ctx->thread_ctx_switch_wait_token = 0;
1758                 hl_ctx_put(ctx);
1759         }
1760
1761         if (hw_fini_rc) {
1762                 rc = hw_fini_rc;
1763                 goto out_err;
1764         }
1765         /* Finished tear-down, starting to re-initialize */
1766
1767         if (hard_reset) {
1768                 hdev->device_cpu_disabled = false;
1769                 hdev->reset_info.hard_reset_pending = false;
1770
1771                 if (hdev->reset_info.reset_trigger_repeated &&
1772                                 (hdev->reset_info.prev_reset_trigger ==
1773                                                 HL_DRV_RESET_FW_FATAL_ERR)) {
1774                         /* if there 2 back to back resets from FW,
1775                          * ensure driver puts the driver in a unusable state
1776                          */
1777                         dev_crit(hdev->dev,
1778                                 "%s Consecutive FW fatal errors received, stopping hard reset\n",
1779                                 dev_name(&(hdev)->pdev->dev));
1780                         rc = -EIO;
1781                         goto out_err;
1782                 }
1783
1784                 if (hdev->kernel_ctx) {
1785                         dev_crit(hdev->dev,
1786                                 "%s kernel ctx was alive during hard reset, something is terribly wrong\n",
1787                                 dev_name(&(hdev)->pdev->dev));
1788                         rc = -EBUSY;
1789                         goto out_err;
1790                 }
1791
1792                 rc = hl_mmu_init(hdev);
1793                 if (rc) {
1794                         dev_err(hdev->dev,
1795                                 "Failed to initialize MMU S/W after hard reset\n");
1796                         goto out_err;
1797                 }
1798
1799                 /* Allocate the kernel context */
1800                 hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx),
1801                                                 GFP_KERNEL);
1802                 if (!hdev->kernel_ctx) {
1803                         rc = -ENOMEM;
1804                         hl_mmu_fini(hdev);
1805                         goto out_err;
1806                 }
1807
1808                 hdev->is_compute_ctx_active = false;
1809
1810                 rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
1811                 if (rc) {
1812                         dev_err(hdev->dev,
1813                                 "failed to init kernel ctx in hard reset\n");
1814                         kfree(hdev->kernel_ctx);
1815                         hdev->kernel_ctx = NULL;
1816                         hl_mmu_fini(hdev);
1817                         goto out_err;
1818                 }
1819         }
1820
1821         /* Device is now enabled as part of the initialization requires
1822          * communication with the device firmware to get information that
1823          * is required for the initialization itself
1824          */
1825         hdev->disabled = false;
1826
1827         /* F/W security enabled indication might be updated after hard-reset */
1828         if (hard_reset) {
1829                 rc = hl_fw_read_preboot_status(hdev);
1830                 if (rc)
1831                         goto out_err;
1832         }
1833
1834         rc = hdev->asic_funcs->hw_init(hdev);
1835         if (rc) {
1836                 dev_err(hdev->dev, "failed to initialize the H/W after reset\n");
1837                 goto out_err;
1838         }
1839
1840         /* If device is not idle fail the reset process */
1841         if (!hdev->asic_funcs->is_device_idle(hdev, idle_mask,
1842                                                 HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL)) {
1843                 print_idle_status_mask(hdev, "device is not idle after reset", idle_mask);
1844                 rc = -EIO;
1845                 goto out_err;
1846         }
1847
1848         /* Check that the communication with the device is working */
1849         rc = hdev->asic_funcs->test_queues(hdev);
1850         if (rc) {
1851                 dev_err(hdev->dev, "Failed to detect if device is alive after reset\n");
1852                 goto out_err;
1853         }
1854
1855         if (hard_reset) {
1856                 rc = device_late_init(hdev);
1857                 if (rc) {
1858                         dev_err(hdev->dev, "Failed late init after hard reset\n");
1859                         goto out_err;
1860                 }
1861
1862                 rc = hl_vm_init(hdev);
1863                 if (rc) {
1864                         dev_err(hdev->dev, "Failed to init memory module after hard reset\n");
1865                         goto out_err;
1866                 }
1867
1868                 if (!hdev->asic_prop.fw_security_enabled)
1869                         hl_fw_set_max_power(hdev);
1870         } else {
1871                 rc = hdev->asic_funcs->compute_reset_late_init(hdev);
1872                 if (rc) {
1873                         if (reset_upon_device_release)
1874                                 dev_err(hdev->dev,
1875                                         "Failed late init in reset after device release\n");
1876                         else
1877                                 dev_err(hdev->dev, "Failed late init after compute reset\n");
1878                         goto out_err;
1879                 }
1880         }
1881
1882         rc = hdev->asic_funcs->scrub_device_mem(hdev);
1883         if (rc) {
1884                 dev_err(hdev->dev, "scrub mem failed from device reset (%d)\n", rc);
1885                 goto out_err;
1886         }
1887
1888         spin_lock(&hdev->reset_info.lock);
1889         hdev->reset_info.in_compute_reset = 0;
1890
1891         /* Schedule hard reset only if requested and if not already in hard reset.
1892          * We keep 'in_reset' enabled, so no other reset can go in during the hard
1893          * reset schedule
1894          */
1895         if (!hard_reset && hdev->reset_info.hard_reset_schedule_flags)
1896                 schedule_hard_reset = true;
1897         else
1898                 hdev->reset_info.in_reset = 0;
1899
1900         spin_unlock(&hdev->reset_info.lock);
1901
1902         hdev->reset_info.needs_reset = false;
1903
1904         if (hard_reset)
1905                 dev_info(hdev->dev,
1906                          "Successfully finished resetting the %s device\n",
1907                          dev_name(&(hdev)->pdev->dev));
1908         else
1909                 dev_dbg(hdev->dev,
1910                         "Successfully finished resetting the %s device\n",
1911                         dev_name(&(hdev)->pdev->dev));
1912
1913         if (hard_reset) {
1914                 hdev->reset_info.hard_reset_cnt++;
1915
1916                 /* After reset is done, we are ready to receive events from
1917                  * the F/W. We can't do it before because we will ignore events
1918                  * and if those events are fatal, we won't know about it and
1919                  * the device will be operational although it shouldn't be
1920                  */
1921                 hdev->asic_funcs->enable_events_from_fw(hdev);
1922         } else {
1923                 if (!reset_upon_device_release)
1924                         hdev->reset_info.compute_reset_cnt++;
1925
1926                 if (schedule_hard_reset) {
1927                         dev_info(hdev->dev, "Performing hard reset scheduled during compute reset\n");
1928                         flags = hdev->reset_info.hard_reset_schedule_flags;
1929                         hdev->reset_info.hard_reset_schedule_flags = 0;
1930                         hard_reset = true;
1931                         goto escalate_reset_flow;
1932                 }
1933         }
1934
1935         return 0;
1936
1937 out_err:
1938         hdev->disabled = true;
1939
1940         spin_lock(&hdev->reset_info.lock);
1941         hdev->reset_info.in_compute_reset = 0;
1942
1943         if (hard_reset) {
1944                 dev_err(hdev->dev,
1945                         "%s Failed to reset! Device is NOT usable\n",
1946                         dev_name(&(hdev)->pdev->dev));
1947                 hdev->reset_info.hard_reset_cnt++;
1948         } else {
1949                 if (reset_upon_device_release) {
1950                         dev_err(hdev->dev, "Failed to reset device after user release\n");
1951                         flags &= ~HL_DRV_RESET_DEV_RELEASE;
1952                 } else {
1953                         dev_err(hdev->dev, "Failed to do compute reset\n");
1954                         hdev->reset_info.compute_reset_cnt++;
1955                 }
1956
1957                 spin_unlock(&hdev->reset_info.lock);
1958                 flags |= HL_DRV_RESET_HARD;
1959                 hard_reset = true;
1960                 goto escalate_reset_flow;
1961         }
1962
1963         hdev->reset_info.in_reset = 0;
1964
1965         spin_unlock(&hdev->reset_info.lock);
1966
1967         return rc;
1968 }
1969
1970 /*
1971  * hl_device_cond_reset() - conditionally reset the device.
1972  * @hdev: pointer to habanalabs device structure.
1973  * @reset_flags: reset flags.
1974  * @event_mask: events to notify user about.
1975  *
1976  * Conditionally reset the device, or alternatively schedule a watchdog work to reset the device
1977  * unless another reset precedes it.
1978  */
1979 int hl_device_cond_reset(struct hl_device *hdev, u32 flags, u64 event_mask)
1980 {
1981         struct hl_ctx *ctx = NULL;
1982
1983         /* F/W reset cannot be postponed */
1984         if (flags & HL_DRV_RESET_BYPASS_REQ_TO_FW)
1985                 goto device_reset;
1986
1987         /* Device release watchdog is relevant only if user exists and gets a reset notification */
1988         if (!(event_mask & HL_NOTIFIER_EVENT_DEVICE_RESET)) {
1989                 dev_err(hdev->dev, "Resetting device without a reset indication to user\n");
1990                 goto device_reset;
1991         }
1992
1993         ctx = hl_get_compute_ctx(hdev);
1994         if (!ctx)
1995                 goto device_reset;
1996
1997         /*
1998          * There is no point in postponing the reset if user is not registered for events.
1999          * However if no eventfd_ctx exists but the device release watchdog is already scheduled, it
2000          * just implies that user has unregistered as part of handling a previous event. In this
2001          * case an immediate reset is not required.
2002          */
2003         if (!ctx->hpriv->notifier_event.eventfd && !hdev->reset_info.watchdog_active)
2004                 goto device_reset;
2005
2006         /* Schedule the device release watchdog work unless reset is already in progress or if the
2007          * work is already scheduled.
2008          */
2009         spin_lock(&hdev->reset_info.lock);
2010         if (hdev->reset_info.in_reset) {
2011                 spin_unlock(&hdev->reset_info.lock);
2012                 goto device_reset;
2013         }
2014
2015         if (hdev->reset_info.watchdog_active) {
2016                 hdev->device_release_watchdog_work.flags |= flags;
2017                 goto out;
2018         }
2019
2020         hdev->device_release_watchdog_work.flags = flags;
2021         dev_dbg(hdev->dev, "Device is going to be hard-reset in %u sec unless being released\n",
2022                 hdev->device_release_watchdog_timeout_sec);
2023         schedule_delayed_work(&hdev->device_release_watchdog_work.reset_work,
2024                                 msecs_to_jiffies(hdev->device_release_watchdog_timeout_sec * 1000));
2025         hdev->reset_info.watchdog_active = 1;
2026 out:
2027         spin_unlock(&hdev->reset_info.lock);
2028
2029         hl_notifier_event_send_all(hdev, event_mask);
2030
2031         hl_ctx_put(ctx);
2032
2033         hl_abort_waiting_for_completions(hdev);
2034
2035         return 0;
2036
2037 device_reset:
2038         if (event_mask)
2039                 hl_notifier_event_send_all(hdev, event_mask);
2040         if (ctx)
2041                 hl_ctx_put(ctx);
2042
2043         return hl_device_reset(hdev, flags | HL_DRV_RESET_HARD);
2044 }
2045
2046 static void hl_notifier_event_send(struct hl_notifier_event *notifier_event, u64 event_mask)
2047 {
2048         mutex_lock(&notifier_event->lock);
2049         notifier_event->events_mask |= event_mask;
2050
2051         if (notifier_event->eventfd)
2052                 eventfd_signal(notifier_event->eventfd);
2053
2054         mutex_unlock(&notifier_event->lock);
2055 }
2056
2057 /*
2058  * hl_notifier_event_send_all - notify all user processes via eventfd
2059  *
2060  * @hdev: pointer to habanalabs device structure
2061  * @event_mask: the occurred event/s
2062  * Returns 0 for success or an error on failure.
2063  */
2064 void hl_notifier_event_send_all(struct hl_device *hdev, u64 event_mask)
2065 {
2066         struct hl_fpriv *hpriv;
2067
2068         if (!event_mask) {
2069                 dev_warn(hdev->dev, "Skip sending zero event");
2070                 return;
2071         }
2072
2073         mutex_lock(&hdev->fpriv_list_lock);
2074
2075         list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node)
2076                 hl_notifier_event_send(&hpriv->notifier_event, event_mask);
2077
2078         mutex_unlock(&hdev->fpriv_list_lock);
2079 }
2080
2081 /*
2082  * hl_device_init - main initialization function for habanalabs device
2083  *
2084  * @hdev: pointer to habanalabs device structure
2085  *
2086  * Allocate an id for the device, do early initialization and then call the
2087  * ASIC specific initialization functions. Finally, create the cdev and the
2088  * Linux device to expose it to the user
2089  */
2090 int hl_device_init(struct hl_device *hdev)
2091 {
2092         int i, rc, cq_cnt, user_interrupt_cnt, cq_ready_cnt;
2093         struct hl_ts_free_jobs *free_jobs_data;
2094         bool expose_interfaces_on_err = false;
2095         void *p;
2096
2097         /* Initialize ASIC function pointers and perform early init */
2098         rc = device_early_init(hdev);
2099         if (rc)
2100                 goto out_disabled;
2101
2102         user_interrupt_cnt = hdev->asic_prop.user_dec_intr_count +
2103                                 hdev->asic_prop.user_interrupt_count;
2104
2105         if (user_interrupt_cnt) {
2106                 hdev->user_interrupt = kcalloc(user_interrupt_cnt, sizeof(*hdev->user_interrupt),
2107                                                 GFP_KERNEL);
2108                 if (!hdev->user_interrupt) {
2109                         rc = -ENOMEM;
2110                         goto early_fini;
2111                 }
2112
2113                 /* Timestamp records supported only if CQ supported in device */
2114                 if (hdev->asic_prop.first_available_cq[0] != USHRT_MAX) {
2115                         for (i = 0 ; i < user_interrupt_cnt ; i++) {
2116                                 p = vzalloc(TIMESTAMP_FREE_NODES_NUM *
2117                                                 sizeof(struct timestamp_reg_free_node));
2118                                 if (!p) {
2119                                         rc = -ENOMEM;
2120                                         goto free_usr_intr_mem;
2121                                 }
2122                                 free_jobs_data = &hdev->user_interrupt[i].ts_free_jobs_data;
2123                                 free_jobs_data->free_nodes_pool = p;
2124                                 free_jobs_data->free_nodes_length = TIMESTAMP_FREE_NODES_NUM;
2125                                 free_jobs_data->next_avail_free_node_idx = 0;
2126                         }
2127                 }
2128         }
2129
2130         free_jobs_data = &hdev->common_user_cq_interrupt.ts_free_jobs_data;
2131         p = vzalloc(TIMESTAMP_FREE_NODES_NUM *
2132                                 sizeof(struct timestamp_reg_free_node));
2133         if (!p) {
2134                 rc = -ENOMEM;
2135                 goto free_usr_intr_mem;
2136         }
2137
2138         free_jobs_data->free_nodes_pool = p;
2139         free_jobs_data->free_nodes_length = TIMESTAMP_FREE_NODES_NUM;
2140         free_jobs_data->next_avail_free_node_idx = 0;
2141
2142         /*
2143          * Start calling ASIC initialization. First S/W then H/W and finally
2144          * late init
2145          */
2146         rc = hdev->asic_funcs->sw_init(hdev);
2147         if (rc)
2148                 goto free_common_usr_intr_mem;
2149
2150
2151         /* initialize completion structure for multi CS wait */
2152         hl_multi_cs_completion_init(hdev);
2153
2154         /*
2155          * Initialize the H/W queues. Must be done before hw_init, because
2156          * there the addresses of the kernel queue are being written to the
2157          * registers of the device
2158          */
2159         rc = hl_hw_queues_create(hdev);
2160         if (rc) {
2161                 dev_err(hdev->dev, "failed to initialize kernel queues\n");
2162                 goto sw_fini;
2163         }
2164
2165         cq_cnt = hdev->asic_prop.completion_queues_count;
2166
2167         /*
2168          * Initialize the completion queues. Must be done before hw_init,
2169          * because there the addresses of the completion queues are being
2170          * passed as arguments to request_irq
2171          */
2172         if (cq_cnt) {
2173                 hdev->completion_queue = kcalloc(cq_cnt,
2174                                 sizeof(*hdev->completion_queue),
2175                                 GFP_KERNEL);
2176
2177                 if (!hdev->completion_queue) {
2178                         dev_err(hdev->dev,
2179                                 "failed to allocate completion queues\n");
2180                         rc = -ENOMEM;
2181                         goto hw_queues_destroy;
2182                 }
2183         }
2184
2185         for (i = 0, cq_ready_cnt = 0 ; i < cq_cnt ; i++, cq_ready_cnt++) {
2186                 rc = hl_cq_init(hdev, &hdev->completion_queue[i],
2187                                 hdev->asic_funcs->get_queue_id_for_cq(hdev, i));
2188                 if (rc) {
2189                         dev_err(hdev->dev,
2190                                 "failed to initialize completion queue\n");
2191                         goto cq_fini;
2192                 }
2193                 hdev->completion_queue[i].cq_idx = i;
2194         }
2195
2196         hdev->shadow_cs_queue = kcalloc(hdev->asic_prop.max_pending_cs,
2197                                         sizeof(struct hl_cs *), GFP_KERNEL);
2198         if (!hdev->shadow_cs_queue) {
2199                 rc = -ENOMEM;
2200                 goto cq_fini;
2201         }
2202
2203         /*
2204          * Initialize the event queue. Must be done before hw_init,
2205          * because there the address of the event queue is being
2206          * passed as argument to request_irq
2207          */
2208         rc = hl_eq_init(hdev, &hdev->event_queue);
2209         if (rc) {
2210                 dev_err(hdev->dev, "failed to initialize event queue\n");
2211                 goto free_shadow_cs_queue;
2212         }
2213
2214         /* MMU S/W must be initialized before kernel context is created */
2215         rc = hl_mmu_init(hdev);
2216         if (rc) {
2217                 dev_err(hdev->dev, "Failed to initialize MMU S/W structures\n");
2218                 goto eq_fini;
2219         }
2220
2221         /* Allocate the kernel context */
2222         hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL);
2223         if (!hdev->kernel_ctx) {
2224                 rc = -ENOMEM;
2225                 goto mmu_fini;
2226         }
2227
2228         hdev->is_compute_ctx_active = false;
2229
2230         hdev->asic_funcs->state_dump_init(hdev);
2231
2232         hdev->device_release_watchdog_timeout_sec = HL_DEVICE_RELEASE_WATCHDOG_TIMEOUT_SEC;
2233
2234         hdev->memory_scrub_val = MEM_SCRUB_DEFAULT_VAL;
2235
2236         rc = hl_debugfs_device_init(hdev);
2237         if (rc) {
2238                 dev_err(hdev->dev, "failed to initialize debugfs entry structure\n");
2239                 kfree(hdev->kernel_ctx);
2240                 goto mmu_fini;
2241         }
2242
2243         /* The debugfs entry structure is accessed in hl_ctx_init(), so it must be called after
2244          * hl_debugfs_device_init().
2245          */
2246         rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
2247         if (rc) {
2248                 dev_err(hdev->dev, "failed to initialize kernel context\n");
2249                 kfree(hdev->kernel_ctx);
2250                 goto debugfs_device_fini;
2251         }
2252
2253         rc = hl_cb_pool_init(hdev);
2254         if (rc) {
2255                 dev_err(hdev->dev, "failed to initialize CB pool\n");
2256                 goto release_ctx;
2257         }
2258
2259         rc = hl_dec_init(hdev);
2260         if (rc) {
2261                 dev_err(hdev->dev, "Failed to initialize the decoder module\n");
2262                 goto cb_pool_fini;
2263         }
2264
2265         /*
2266          * From this point, override rc (=0) in case of an error to allow debugging
2267          * (by adding char devices and creating sysfs/debugfs files as part of the error flow).
2268          */
2269         expose_interfaces_on_err = true;
2270
2271         /* Device is now enabled as part of the initialization requires
2272          * communication with the device firmware to get information that
2273          * is required for the initialization itself
2274          */
2275         hdev->disabled = false;
2276
2277         rc = hdev->asic_funcs->hw_init(hdev);
2278         if (rc) {
2279                 dev_err(hdev->dev, "failed to initialize the H/W\n");
2280                 rc = 0;
2281                 goto out_disabled;
2282         }
2283
2284         /* Check that the communication with the device is working */
2285         rc = hdev->asic_funcs->test_queues(hdev);
2286         if (rc) {
2287                 dev_err(hdev->dev, "Failed to detect if device is alive\n");
2288                 rc = 0;
2289                 goto out_disabled;
2290         }
2291
2292         rc = device_late_init(hdev);
2293         if (rc) {
2294                 dev_err(hdev->dev, "Failed late initialization\n");
2295                 rc = 0;
2296                 goto out_disabled;
2297         }
2298
2299         dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n",
2300                 hdev->asic_name,
2301                 hdev->asic_prop.dram_size / SZ_1G);
2302
2303         rc = hl_vm_init(hdev);
2304         if (rc) {
2305                 dev_err(hdev->dev, "Failed to initialize memory module\n");
2306                 rc = 0;
2307                 goto out_disabled;
2308         }
2309
2310         /*
2311          * Expose devices and sysfs/debugfs files to user.
2312          * From here there is no need to expose them in case of an error.
2313          */
2314         expose_interfaces_on_err = false;
2315
2316         rc = drm_dev_register(&hdev->drm, 0);
2317         if (rc) {
2318                 dev_err(hdev->dev, "Failed to register DRM device, rc %d\n", rc);
2319                 rc = 0;
2320                 goto out_disabled;
2321         }
2322
2323         rc = cdev_sysfs_debugfs_add(hdev);
2324         if (rc) {
2325                 dev_err(hdev->dev, "Failed to add char devices and sysfs/debugfs files\n");
2326                 rc = 0;
2327                 goto out_disabled;
2328         }
2329
2330         /* Need to call this again because the max power might change,
2331          * depending on card type for certain ASICs
2332          */
2333         if (hdev->asic_prop.set_max_power_on_device_init &&
2334                         !hdev->asic_prop.fw_security_enabled)
2335                 hl_fw_set_max_power(hdev);
2336
2337         /*
2338          * hl_hwmon_init() must be called after device_late_init(), because only
2339          * there we get the information from the device about which
2340          * hwmon-related sensors the device supports.
2341          * Furthermore, it must be done after adding the device to the system.
2342          */
2343         rc = hl_hwmon_init(hdev);
2344         if (rc) {
2345                 dev_err(hdev->dev, "Failed to initialize hwmon\n");
2346                 rc = 0;
2347                 goto out_disabled;
2348         }
2349
2350         dev_notice(hdev->dev,
2351                 "Successfully added device %s to habanalabs driver\n",
2352                 dev_name(&(hdev)->pdev->dev));
2353
2354         /* After initialization is done, we are ready to receive events from
2355          * the F/W. We can't do it before because we will ignore events and if
2356          * those events are fatal, we won't know about it and the device will
2357          * be operational although it shouldn't be
2358          */
2359         hdev->asic_funcs->enable_events_from_fw(hdev);
2360
2361         hdev->init_done = true;
2362
2363         return 0;
2364
2365 cb_pool_fini:
2366         hl_cb_pool_fini(hdev);
2367 release_ctx:
2368         if (hl_ctx_put(hdev->kernel_ctx) != 1)
2369                 dev_err(hdev->dev,
2370                         "kernel ctx is still alive on initialization failure\n");
2371 debugfs_device_fini:
2372         hl_debugfs_device_fini(hdev);
2373 mmu_fini:
2374         hl_mmu_fini(hdev);
2375 eq_fini:
2376         hl_eq_fini(hdev, &hdev->event_queue);
2377 free_shadow_cs_queue:
2378         kfree(hdev->shadow_cs_queue);
2379 cq_fini:
2380         for (i = 0 ; i < cq_ready_cnt ; i++)
2381                 hl_cq_fini(hdev, &hdev->completion_queue[i]);
2382         kfree(hdev->completion_queue);
2383 hw_queues_destroy:
2384         hl_hw_queues_destroy(hdev);
2385 sw_fini:
2386         hdev->asic_funcs->sw_fini(hdev);
2387 free_common_usr_intr_mem:
2388         vfree(hdev->common_user_cq_interrupt.ts_free_jobs_data.free_nodes_pool);
2389 free_usr_intr_mem:
2390         if (user_interrupt_cnt) {
2391                 for (i = 0 ; i < user_interrupt_cnt ; i++) {
2392                         if (!hdev->user_interrupt[i].ts_free_jobs_data.free_nodes_pool)
2393                                 break;
2394                         vfree(hdev->user_interrupt[i].ts_free_jobs_data.free_nodes_pool);
2395                 }
2396                 kfree(hdev->user_interrupt);
2397         }
2398 early_fini:
2399         device_early_fini(hdev);
2400 out_disabled:
2401         hdev->disabled = true;
2402         if (expose_interfaces_on_err) {
2403                 drm_dev_register(&hdev->drm, 0);
2404                 cdev_sysfs_debugfs_add(hdev);
2405         }
2406
2407         pr_err("Failed to initialize accel%d. Device %s is NOT usable!\n",
2408                 hdev->cdev_idx, dev_name(&hdev->pdev->dev));
2409
2410         return rc;
2411 }
2412
2413 /*
2414  * hl_device_fini - main tear-down function for habanalabs device
2415  *
2416  * @hdev: pointer to habanalabs device structure
2417  *
2418  * Destroy the device, call ASIC fini functions and release the id
2419  */
2420 void hl_device_fini(struct hl_device *hdev)
2421 {
2422         u32 user_interrupt_cnt;
2423         bool device_in_reset;
2424         ktime_t timeout;
2425         u64 reset_sec;
2426         int i, rc;
2427
2428         dev_info(hdev->dev, "Removing device %s\n", dev_name(&(hdev)->pdev->dev));
2429
2430         hdev->device_fini_pending = 1;
2431         flush_delayed_work(&hdev->device_reset_work.reset_work);
2432
2433         if (hdev->pldm)
2434                 reset_sec = HL_PLDM_HARD_RESET_MAX_TIMEOUT;
2435         else
2436                 reset_sec = HL_HARD_RESET_MAX_TIMEOUT;
2437
2438         /*
2439          * This function is competing with the reset function, so try to
2440          * take the reset atomic and if we are already in middle of reset,
2441          * wait until reset function is finished. Reset function is designed
2442          * to always finish. However, in Gaudi, because of all the network
2443          * ports, the hard reset could take between 10-30 seconds
2444          */
2445
2446         timeout = ktime_add_us(ktime_get(), reset_sec * 1000 * 1000);
2447
2448         spin_lock(&hdev->reset_info.lock);
2449         device_in_reset = !!hdev->reset_info.in_reset;
2450         if (!device_in_reset)
2451                 hdev->reset_info.in_reset = 1;
2452         spin_unlock(&hdev->reset_info.lock);
2453
2454         while (device_in_reset) {
2455                 usleep_range(50, 200);
2456
2457                 spin_lock(&hdev->reset_info.lock);
2458                 device_in_reset = !!hdev->reset_info.in_reset;
2459                 if (!device_in_reset)
2460                         hdev->reset_info.in_reset = 1;
2461                 spin_unlock(&hdev->reset_info.lock);
2462
2463                 if (ktime_compare(ktime_get(), timeout) > 0) {
2464                         dev_crit(hdev->dev,
2465                                 "%s Failed to remove device because reset function did not finish\n",
2466                                 dev_name(&(hdev)->pdev->dev));
2467                         return;
2468                 }
2469         }
2470
2471         cancel_delayed_work_sync(&hdev->device_release_watchdog_work.reset_work);
2472
2473         /* Disable PCI access from device F/W so it won't send us additional
2474          * interrupts. We disable MSI/MSI-X at the halt_engines function and we
2475          * can't have the F/W sending us interrupts after that. We need to
2476          * disable the access here because if the device is marked disable, the
2477          * message won't be send. Also, in case of heartbeat, the device CPU is
2478          * marked as disable so this message won't be sent
2479          */
2480         hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0);
2481
2482         /* Mark device as disabled */
2483         hdev->disabled = true;
2484
2485         take_release_locks(hdev);
2486
2487         hdev->reset_info.hard_reset_pending = true;
2488
2489         hl_hwmon_fini(hdev);
2490
2491         cleanup_resources(hdev, true, false, false);
2492
2493         /* Kill processes here after CS rollback. This is because the process
2494          * can't really exit until all its CSs are done, which is what we
2495          * do in cs rollback
2496          */
2497         dev_info(hdev->dev,
2498                 "Waiting for all processes to exit (timeout of %u seconds)",
2499                 HL_WAIT_PROCESS_KILL_ON_DEVICE_FINI);
2500
2501         hdev->process_kill_trial_cnt = 0;
2502         rc = device_kill_open_processes(hdev, HL_WAIT_PROCESS_KILL_ON_DEVICE_FINI, false);
2503         if (rc) {
2504                 dev_crit(hdev->dev, "Failed to kill all open processes (%d)\n", rc);
2505                 device_disable_open_processes(hdev, false);
2506         }
2507
2508         hdev->process_kill_trial_cnt = 0;
2509         rc = device_kill_open_processes(hdev, 0, true);
2510         if (rc) {
2511                 dev_crit(hdev->dev, "Failed to kill all control device open processes (%d)\n", rc);
2512                 device_disable_open_processes(hdev, true);
2513         }
2514
2515         hl_cb_pool_fini(hdev);
2516
2517         /* Reset the H/W. It will be in idle state after this returns */
2518         rc = hdev->asic_funcs->hw_fini(hdev, true, false);
2519         if (rc)
2520                 dev_err(hdev->dev, "hw_fini failed in device fini while removing device %d\n", rc);
2521
2522         hdev->fw_loader.fw_comp_loaded = FW_TYPE_NONE;
2523
2524         /* Release kernel context */
2525         if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1))
2526                 dev_err(hdev->dev, "kernel ctx is still alive\n");
2527
2528         hl_dec_fini(hdev);
2529
2530         hl_vm_fini(hdev);
2531
2532         hl_mmu_fini(hdev);
2533
2534         vfree(hdev->captured_err_info.page_fault_info.user_mappings);
2535
2536         hl_eq_fini(hdev, &hdev->event_queue);
2537
2538         kfree(hdev->shadow_cs_queue);
2539
2540         for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
2541                 hl_cq_fini(hdev, &hdev->completion_queue[i]);
2542         kfree(hdev->completion_queue);
2543
2544         user_interrupt_cnt = hdev->asic_prop.user_dec_intr_count +
2545                                         hdev->asic_prop.user_interrupt_count;
2546
2547         if (user_interrupt_cnt) {
2548                 if (hdev->asic_prop.first_available_cq[0] != USHRT_MAX) {
2549                         for (i = 0 ; i < user_interrupt_cnt ; i++)
2550                                 vfree(hdev->user_interrupt[i].ts_free_jobs_data.free_nodes_pool);
2551                 }
2552
2553                 kfree(hdev->user_interrupt);
2554         }
2555
2556         vfree(hdev->common_user_cq_interrupt.ts_free_jobs_data.free_nodes_pool);
2557
2558         hl_hw_queues_destroy(hdev);
2559
2560         /* Call ASIC S/W finalize function */
2561         hdev->asic_funcs->sw_fini(hdev);
2562
2563         device_early_fini(hdev);
2564
2565         /* Hide devices and sysfs/debugfs files from user */
2566         cdev_sysfs_debugfs_remove(hdev);
2567         drm_dev_unregister(&hdev->drm);
2568
2569         hl_debugfs_device_fini(hdev);
2570
2571         pr_info("removed device successfully\n");
2572 }
2573
2574 /*
2575  * MMIO register access helper functions.
2576  */
2577
2578 /*
2579  * hl_rreg - Read an MMIO register
2580  *
2581  * @hdev: pointer to habanalabs device structure
2582  * @reg: MMIO register offset (in bytes)
2583  *
2584  * Returns the value of the MMIO register we are asked to read
2585  *
2586  */
2587 inline u32 hl_rreg(struct hl_device *hdev, u32 reg)
2588 {
2589         u32 val = readl(hdev->rmmio + reg);
2590
2591         if (unlikely(trace_habanalabs_rreg32_enabled()))
2592                 trace_habanalabs_rreg32(hdev->dev, reg, val);
2593
2594         return val;
2595 }
2596
2597 /*
2598  * hl_wreg - Write to an MMIO register
2599  *
2600  * @hdev: pointer to habanalabs device structure
2601  * @reg: MMIO register offset (in bytes)
2602  * @val: 32-bit value
2603  *
2604  * Writes the 32-bit value into the MMIO register
2605  *
2606  */
2607 inline void hl_wreg(struct hl_device *hdev, u32 reg, u32 val)
2608 {
2609         if (unlikely(trace_habanalabs_wreg32_enabled()))
2610                 trace_habanalabs_wreg32(hdev->dev, reg, val);
2611
2612         writel(val, hdev->rmmio + reg);
2613 }
2614
2615 void hl_capture_razwi(struct hl_device *hdev, u64 addr, u16 *engine_id, u16 num_of_engines,
2616                         u8 flags)
2617 {
2618         struct razwi_info *razwi_info = &hdev->captured_err_info.razwi_info;
2619
2620         if (num_of_engines > HL_RAZWI_MAX_NUM_OF_ENGINES_PER_RTR) {
2621                 dev_err(hdev->dev,
2622                                 "Number of possible razwi initiators (%u) exceeded limit (%u)\n",
2623                                 num_of_engines, HL_RAZWI_MAX_NUM_OF_ENGINES_PER_RTR);
2624                 return;
2625         }
2626
2627         /* In case it's the first razwi since the device was opened, capture its parameters */
2628         if (atomic_cmpxchg(&hdev->captured_err_info.razwi_info.razwi_detected, 0, 1))
2629                 return;
2630
2631         razwi_info->razwi.timestamp = ktime_to_ns(ktime_get());
2632         razwi_info->razwi.addr = addr;
2633         razwi_info->razwi.num_of_possible_engines = num_of_engines;
2634         memcpy(&razwi_info->razwi.engine_id[0], &engine_id[0],
2635                         num_of_engines * sizeof(u16));
2636         razwi_info->razwi.flags = flags;
2637
2638         razwi_info->razwi_info_available = true;
2639 }
2640
2641 void hl_handle_razwi(struct hl_device *hdev, u64 addr, u16 *engine_id, u16 num_of_engines,
2642                         u8 flags, u64 *event_mask)
2643 {
2644         hl_capture_razwi(hdev, addr, engine_id, num_of_engines, flags);
2645
2646         if (event_mask)
2647                 *event_mask |= HL_NOTIFIER_EVENT_RAZWI;
2648 }
2649
2650 static void hl_capture_user_mappings(struct hl_device *hdev, bool is_pmmu)
2651 {
2652         struct page_fault_info *pgf_info = &hdev->captured_err_info.page_fault_info;
2653         struct hl_vm_phys_pg_pack *phys_pg_pack = NULL;
2654         struct hl_vm_hash_node *hnode;
2655         struct hl_userptr *userptr;
2656         enum vm_type *vm_type;
2657         struct hl_ctx *ctx;
2658         u32 map_idx = 0;
2659         int i;
2660
2661         /* Reset previous session count*/
2662         pgf_info->num_of_user_mappings = 0;
2663
2664         ctx = hl_get_compute_ctx(hdev);
2665         if (!ctx) {
2666                 dev_err(hdev->dev, "Can't get user context for user mappings\n");
2667                 return;
2668         }
2669
2670         mutex_lock(&ctx->mem_hash_lock);
2671         hash_for_each(ctx->mem_hash, i, hnode, node) {
2672                 vm_type = hnode->ptr;
2673                 if (((*vm_type == VM_TYPE_USERPTR) && is_pmmu) ||
2674                                 ((*vm_type == VM_TYPE_PHYS_PACK) && !is_pmmu))
2675                         pgf_info->num_of_user_mappings++;
2676
2677         }
2678
2679         if (!pgf_info->num_of_user_mappings)
2680                 goto finish;
2681
2682         /* In case we already allocated in previous session, need to release it before
2683          * allocating new buffer.
2684          */
2685         vfree(pgf_info->user_mappings);
2686         pgf_info->user_mappings =
2687                         vzalloc(pgf_info->num_of_user_mappings * sizeof(struct hl_user_mapping));
2688         if (!pgf_info->user_mappings) {
2689                 pgf_info->num_of_user_mappings = 0;
2690                 goto finish;
2691         }
2692
2693         hash_for_each(ctx->mem_hash, i, hnode, node) {
2694                 vm_type = hnode->ptr;
2695                 if ((*vm_type == VM_TYPE_USERPTR) && (is_pmmu)) {
2696                         userptr = hnode->ptr;
2697                         pgf_info->user_mappings[map_idx].dev_va = hnode->vaddr;
2698                         pgf_info->user_mappings[map_idx].size = userptr->size;
2699                         map_idx++;
2700                 } else if ((*vm_type == VM_TYPE_PHYS_PACK) && (!is_pmmu)) {
2701                         phys_pg_pack = hnode->ptr;
2702                         pgf_info->user_mappings[map_idx].dev_va = hnode->vaddr;
2703                         pgf_info->user_mappings[map_idx].size = phys_pg_pack->total_size;
2704                         map_idx++;
2705                 }
2706         }
2707 finish:
2708         mutex_unlock(&ctx->mem_hash_lock);
2709         hl_ctx_put(ctx);
2710 }
2711
2712 void hl_capture_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_pmmu)
2713 {
2714         struct page_fault_info *pgf_info = &hdev->captured_err_info.page_fault_info;
2715
2716         /* Capture only the first page fault */
2717         if (atomic_cmpxchg(&pgf_info->page_fault_detected, 0, 1))
2718                 return;
2719
2720         pgf_info->page_fault.timestamp = ktime_to_ns(ktime_get());
2721         pgf_info->page_fault.addr = addr;
2722         pgf_info->page_fault.engine_id = eng_id;
2723         hl_capture_user_mappings(hdev, is_pmmu);
2724
2725         pgf_info->page_fault_info_available = true;
2726 }
2727
2728 void hl_handle_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_pmmu,
2729                                 u64 *event_mask)
2730 {
2731         hl_capture_page_fault(hdev, addr, eng_id, is_pmmu);
2732
2733         if (event_mask)
2734                 *event_mask |=  HL_NOTIFIER_EVENT_PAGE_FAULT;
2735 }
2736
2737 static void hl_capture_hw_err(struct hl_device *hdev, u16 event_id)
2738 {
2739         struct hw_err_info *info = &hdev->captured_err_info.hw_err;
2740
2741         /* Capture only the first HW err */
2742         if (atomic_cmpxchg(&info->event_detected, 0, 1))
2743                 return;
2744
2745         info->event.timestamp = ktime_to_ns(ktime_get());
2746         info->event.event_id = event_id;
2747
2748         info->event_info_available = true;
2749 }
2750
2751 void hl_handle_critical_hw_err(struct hl_device *hdev, u16 event_id, u64 *event_mask)
2752 {
2753         hl_capture_hw_err(hdev, event_id);
2754
2755         if (event_mask)
2756                 *event_mask |= HL_NOTIFIER_EVENT_CRITICL_HW_ERR;
2757 }
2758
2759 static void hl_capture_fw_err(struct hl_device *hdev, struct hl_info_fw_err_info *fw_info)
2760 {
2761         struct fw_err_info *info = &hdev->captured_err_info.fw_err;
2762
2763         /* Capture only the first FW error */
2764         if (atomic_cmpxchg(&info->event_detected, 0, 1))
2765                 return;
2766
2767         info->event.timestamp = ktime_to_ns(ktime_get());
2768         info->event.err_type = fw_info->err_type;
2769         if (fw_info->err_type == HL_INFO_FW_REPORTED_ERR)
2770                 info->event.event_id = fw_info->event_id;
2771
2772         info->event_info_available = true;
2773 }
2774
2775 void hl_handle_fw_err(struct hl_device *hdev, struct hl_info_fw_err_info *info)
2776 {
2777         hl_capture_fw_err(hdev, info);
2778
2779         if (info->event_mask)
2780                 *info->event_mask |= HL_NOTIFIER_EVENT_CRITICL_FW_ERR;
2781 }
2782
2783 void hl_capture_engine_err(struct hl_device *hdev, u16 engine_id, u16 error_count)
2784 {
2785         struct engine_err_info *info = &hdev->captured_err_info.engine_err;
2786
2787         /* Capture only the first engine error */
2788         if (atomic_cmpxchg(&info->event_detected, 0, 1))
2789                 return;
2790
2791         info->event.timestamp = ktime_to_ns(ktime_get());
2792         info->event.engine_id = engine_id;
2793         info->event.error_count = error_count;
2794         info->event_info_available = true;
2795 }
2796
2797 void hl_enable_err_info_capture(struct hl_error_info *captured_err_info)
2798 {
2799         vfree(captured_err_info->page_fault_info.user_mappings);
2800         memset(captured_err_info, 0, sizeof(struct hl_error_info));
2801         atomic_set(&captured_err_info->cs_timeout.write_enable, 1);
2802         captured_err_info->undef_opcode.write_enable = true;
2803 }