drivers/accel/habanalabs/common/irq.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2022 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #include "habanalabs.h"
   9
  10 #include <linux/slab.h>
  11
  12 /**
  13  * struct hl_eqe_work - This structure is used to schedule work of EQ
  14  *                      entry and cpucp_reset event
  15  *
  16  * @eq_work:          workqueue object to run when EQ entry is received
  17  * @hdev:             pointer to device structure
  18  * @eq_entry:         copy of the EQ entry
  19  */
  20 struct hl_eqe_work {
  21         struct work_struct      eq_work;
  22         struct hl_device        *hdev;
  23         struct hl_eq_entry      eq_entry;
  24 };
  25
  26 /**
  27  * hl_cq_inc_ptr - increment ci or pi of cq
  28  *
  29  * @ptr: the current ci or pi value of the completion queue
  30  *
  31  * Increment ptr by 1. If it reaches the number of completion queue
  32  * entries, set it to 0
  33  */
  34 inline u32 hl_cq_inc_ptr(u32 ptr)
  35 {
  36         ptr++;
  37         if (unlikely(ptr == HL_CQ_LENGTH))
  38                 ptr = 0;
  39         return ptr;
  40 }
  41
  42 /**
  43  * hl_eq_inc_ptr - increment ci of eq
  44  *
  45  * @ptr: the current ci value of the event queue
  46  *
  47  * Increment ptr by 1. If it reaches the number of event queue
  48  * entries, set it to 0
  49  */
  50 static inline u32 hl_eq_inc_ptr(u32 ptr)
  51 {
  52         ptr++;
  53         if (unlikely(ptr == HL_EQ_LENGTH))
  54                 ptr = 0;
  55         return ptr;
  56 }
  57
  58 static void irq_handle_eqe(struct work_struct *work)
  59 {
  60         struct hl_eqe_work *eqe_work = container_of(work, struct hl_eqe_work,
  61                                                         eq_work);
  62         struct hl_device *hdev = eqe_work->hdev;
  63
  64         hdev->asic_funcs->handle_eqe(hdev, &eqe_work->eq_entry);
  65
  66         kfree(eqe_work);
  67 }
  68
  69 /**
  70  * job_finish - queue job finish work
  71  *
  72  * @hdev: pointer to device structure
  73  * @cs_seq: command submission sequence
  74  * @cq: completion queue
  75  * @timestamp: interrupt timestamp
  76  *
  77  */
  78 static void job_finish(struct hl_device *hdev, u32 cs_seq, struct hl_cq *cq, ktime_t timestamp)
  79 {
  80         struct hl_hw_queue *queue;
  81         struct hl_cs_job *job;
  82
  83         queue = &hdev->kernel_queues[cq->hw_queue_id];
  84         job = queue->shadow_queue[hl_pi_2_offset(cs_seq)];
  85         job->timestamp = timestamp;
  86         queue_work(hdev->cq_wq[cq->cq_idx], &job->finish_work);
  87
  88         atomic_inc(&queue->ci);
  89 }
  90
  91 /**
  92  * cs_finish - queue all cs jobs finish work
  93  *
  94  * @hdev: pointer to device structure
  95  * @cs_seq: command submission sequence
  96  * @timestamp: interrupt timestamp
  97  *
  98  */
  99 static void cs_finish(struct hl_device *hdev, u16 cs_seq, ktime_t timestamp)
 100 {
 101         struct asic_fixed_properties *prop = &hdev->asic_prop;
 102         struct hl_hw_queue *queue;
 103         struct hl_cs *cs;
 104         struct hl_cs_job *job;
 105
 106         cs = hdev->shadow_cs_queue[cs_seq & (prop->max_pending_cs - 1)];
 107         if (!cs) {
 108                 dev_warn(hdev->dev,
 109                         "No pointer to CS in shadow array at index %d\n",
 110                         cs_seq);
 111                 return;
 112         }
 113
 114         list_for_each_entry(job, &cs->job_list, cs_node) {
 115                 queue = &hdev->kernel_queues[job->hw_queue_id];
 116                 atomic_inc(&queue->ci);
 117         }
 118
 119         cs->completion_timestamp = timestamp;
 120         queue_work(hdev->cs_cmplt_wq, &cs->finish_work);
 121 }
 122
 123 /**
 124  * hl_irq_handler_cq - irq handler for completion queue
 125  *
 126  * @irq: irq number
 127  * @arg: pointer to completion queue structure
 128  *
 129  */
 130 irqreturn_t hl_irq_handler_cq(int irq, void *arg)
 131 {
 132         struct hl_cq *cq = arg;
 133         struct hl_device *hdev = cq->hdev;
 134         bool shadow_index_valid, entry_ready;
 135         u16 shadow_index;
 136         struct hl_cq_entry *cq_entry, *cq_base;
 137         ktime_t timestamp = ktime_get();
 138
 139         if (hdev->disabled) {
 140                 dev_dbg(hdev->dev,
 141                         "Device disabled but received IRQ %d for CQ %d\n",
 142                         irq, cq->hw_queue_id);
 143                 return IRQ_HANDLED;
 144         }
 145
 146         cq_base = cq->kernel_address;
 147
 148         while (1) {
 149                 cq_entry = (struct hl_cq_entry *) &cq_base[cq->ci];
 150
 151                 entry_ready = !!FIELD_GET(CQ_ENTRY_READY_MASK,
 152                                 le32_to_cpu(cq_entry->data));
 153                 if (!entry_ready)
 154                         break;
 155
 156                 /* Make sure we read CQ entry contents after we've
 157                  * checked the ownership bit.
 158                  */
 159                 dma_rmb();
 160
 161                 shadow_index_valid =
 162                         !!FIELD_GET(CQ_ENTRY_SHADOW_INDEX_VALID_MASK,
 163                                         le32_to_cpu(cq_entry->data));
 164
 165                 shadow_index = FIELD_GET(CQ_ENTRY_SHADOW_INDEX_MASK,
 166                                 le32_to_cpu(cq_entry->data));
 167
 168                 /*
 169                  * CQ interrupt handler has 2 modes of operation:
 170                  * 1. Interrupt per CS completion: (Single CQ for all queues)
 171                  *    CQ entry represents a completed CS
 172                  *
 173                  * 2. Interrupt per CS job completion in queue: (CQ per queue)
 174                  *    CQ entry represents a completed job in a certain queue
 175                  */
 176                 if (shadow_index_valid && !hdev->disabled) {
 177                         if (hdev->asic_prop.completion_mode ==
 178                                         HL_COMPLETION_MODE_CS)
 179                                 cs_finish(hdev, shadow_index, timestamp);
 180                         else
 181                                 job_finish(hdev, shadow_index, cq, timestamp);
 182                 }
 183
 184                 /* Clear CQ entry ready bit */
 185                 cq_entry->data = cpu_to_le32(le32_to_cpu(cq_entry->data) &
 186                                                 ~CQ_ENTRY_READY_MASK);
 187
 188                 cq->ci = hl_cq_inc_ptr(cq->ci);
 189
 190                 /* Increment free slots */
 191                 atomic_inc(&cq->free_slots_cnt);
 192         }
 193
 194         return IRQ_HANDLED;
 195 }
 196
 197 /*
 198  * hl_ts_free_objects - handler of the free objects workqueue.
 199  * This function should put refcount to objects that the registration node
 200  * took refcount to them.
 201  * @work: workqueue object pointer
 202  */
 203 static void hl_ts_free_objects(struct work_struct *work)
 204 {
 205         struct timestamp_reg_work_obj *job =
 206                         container_of(work, struct timestamp_reg_work_obj, free_obj);
 207         struct list_head *dynamic_alloc_free_list_head = job->dynamic_alloc_free_obj_head;
 208         struct timestamp_reg_free_node *free_obj, *temp_free_obj;
 209         struct list_head *free_list_head = job->free_obj_head;
 210
 211         struct hl_device *hdev = job->hdev;
 212
 213         list_for_each_entry_safe(free_obj, temp_free_obj, free_list_head, free_objects_node) {
 214                 dev_dbg(hdev->dev, "About to put refcount to buf (%p) cq_cb(%p)\n",
 215                                         free_obj->buf,
 216                                         free_obj->cq_cb);
 217
 218                 hl_mmap_mem_buf_put(free_obj->buf);
 219                 hl_cb_put(free_obj->cq_cb);
 220                 atomic_set(&free_obj->in_use, 0);
 221         }
 222
 223         kfree(free_list_head);
 224
 225         if (dynamic_alloc_free_list_head) {
 226                 list_for_each_entry_safe(free_obj, temp_free_obj, dynamic_alloc_free_list_head,
 227                                                                 free_objects_node) {
 228                         dev_dbg(hdev->dev,
 229                                 "Dynamic_Alloc list: About to put refcount to buf (%p) cq_cb(%p)\n",
 230                                                 free_obj->buf,
 231                                                 free_obj->cq_cb);
 232
 233                         hl_mmap_mem_buf_put(free_obj->buf);
 234                         hl_cb_put(free_obj->cq_cb);
 235                         list_del(&free_obj->free_objects_node);
 236                         kfree(free_obj);
 237                 }
 238
 239                 kfree(dynamic_alloc_free_list_head);
 240         }
 241
 242         kfree(job);
 243 }
 244
 245 /*
 246  * This function called with spin_lock of wait_list_lock taken
 247  * This function will set timestamp and delete the registration node from the
 248  * wait_list_lock.
 249  * and since we're protected with spin_lock here, so we cannot just put the refcount
 250  * for the objects here, since the release function may be called and it's also a long
 251  * logic (which might sleep also) that cannot be handled in irq context.
 252  * so here we'll be filling a list with nodes of "put" jobs and then will send this
 253  * list to a dedicated workqueue to do the actual put.
 254  */
 255 static int handle_registration_node(struct hl_device *hdev, struct hl_user_pending_interrupt *pend,
 256                                                 struct list_head **free_list,
 257                                                 struct list_head **dynamic_alloc_list,
 258                                                 struct hl_user_interrupt *intr)
 259 {
 260         struct hl_ts_free_jobs *ts_free_jobs_data;
 261         struct timestamp_reg_free_node *free_node;
 262         u32 free_node_index;
 263         u64 timestamp;
 264
 265         ts_free_jobs_data = &intr->ts_free_jobs_data;
 266         free_node_index = ts_free_jobs_data->next_avail_free_node_idx;
 267
 268         if (!(*free_list)) {
 269                 /* Alloc/Init the timestamp registration free objects list */
 270                 *free_list = kmalloc(sizeof(struct list_head), GFP_ATOMIC);
 271                 if (!(*free_list))
 272                         return -ENOMEM;
 273
 274                 INIT_LIST_HEAD(*free_list);
 275         }
 276
 277         free_node = &ts_free_jobs_data->free_nodes_pool[free_node_index];
 278         if (atomic_cmpxchg(&free_node->in_use, 0, 1)) {
 279                 dev_dbg(hdev->dev,
 280                         "Timestamp free node pool is full, buff: %p, record: %p, irq: %u\n",
 281                                 pend->ts_reg_info.buf,
 282                                 pend,
 283                                 intr->interrupt_id);
 284
 285                 if (!(*dynamic_alloc_list)) {
 286                         *dynamic_alloc_list = kmalloc(sizeof(struct list_head), GFP_ATOMIC);
 287                         if (!(*dynamic_alloc_list))
 288                                 return -ENOMEM;
 289
 290                         INIT_LIST_HEAD(*dynamic_alloc_list);
 291                 }
 292
 293                 free_node = kmalloc(sizeof(struct timestamp_reg_free_node), GFP_ATOMIC);
 294                 if (!free_node)
 295                         return -ENOMEM;
 296
 297                 free_node->dynamic_alloc = 1;
 298         }
 299
 300         timestamp = ktime_to_ns(intr->timestamp);
 301
 302         *pend->ts_reg_info.timestamp_kernel_addr = timestamp;
 303
 304         dev_dbg(hdev->dev, "Irq handle: Timestamp record (%p) ts cb address (%p), interrupt_id: %u\n",
 305                         pend, pend->ts_reg_info.timestamp_kernel_addr, intr->interrupt_id);
 306
 307         list_del(&pend->list_node);
 308
 309         /* Putting the refcount for ts_buff and cq_cb objects will be handled
 310          * in workqueue context, just add job to free_list.
 311          */
 312         free_node->buf = pend->ts_reg_info.buf;
 313         free_node->cq_cb = pend->ts_reg_info.cq_cb;
 314
 315         if (free_node->dynamic_alloc) {
 316                 list_add(&free_node->free_objects_node, *dynamic_alloc_list);
 317         } else {
 318                 ts_free_jobs_data->next_avail_free_node_idx =
 319                                 (++free_node_index) % ts_free_jobs_data->free_nodes_length;
 320                 list_add(&free_node->free_objects_node, *free_list);
 321         }
 322
 323         /* Mark TS record as free */
 324         pend->ts_reg_info.in_use = false;
 325
 326         return 0;
 327 }
 328
 329 static void handle_user_interrupt_ts_list(struct hl_device *hdev, struct hl_user_interrupt *intr)
 330 {
 331         struct list_head *ts_reg_free_list_head = NULL, *dynamic_alloc_list_head = NULL;
 332         struct hl_user_pending_interrupt *pend, *temp_pend;
 333         struct timestamp_reg_work_obj *job;
 334         bool reg_node_handle_fail = false;
 335         unsigned long flags;
 336         int rc;
 337
 338         /* For registration nodes:
 339          * As part of handling the registration nodes, we should put refcount to
 340          * some objects. the problem is that we cannot do that under spinlock
 341          * or in irq handler context at all (since release functions are long and
 342          * might sleep), so we will need to handle that part in workqueue context.
 343          * To avoid handling kmalloc failure which compels us rolling back actions
 344          * and move nodes hanged on the free list back to the interrupt ts list
 345          * we always alloc the job of the WQ at the beginning.
 346          */
 347         job = kmalloc(sizeof(*job), GFP_ATOMIC);
 348         if (!job)
 349                 return;
 350
 351         spin_lock_irqsave(&intr->ts_list_lock, flags);
 352         list_for_each_entry_safe(pend, temp_pend, &intr->ts_list_head, list_node) {
 353                 if ((pend->cq_kernel_addr && *(pend->cq_kernel_addr) >= pend->cq_target_value) ||
 354                                 !pend->cq_kernel_addr) {
 355                         if (!reg_node_handle_fail) {
 356                                 rc = handle_registration_node(hdev, pend,
 357                                                 &ts_reg_free_list_head,
 358                                                 &dynamic_alloc_list_head, intr);
 359                                 if (rc)
 360                                         reg_node_handle_fail = true;
 361                         }
 362                 }
 363         }
 364         spin_unlock_irqrestore(&intr->ts_list_lock, flags);
 365
 366         if (ts_reg_free_list_head) {
 367                 INIT_WORK(&job->free_obj, hl_ts_free_objects);
 368                 job->free_obj_head = ts_reg_free_list_head;
 369                 job->dynamic_alloc_free_obj_head = dynamic_alloc_list_head;
 370                 job->hdev = hdev;
 371                 queue_work(hdev->ts_free_obj_wq, &job->free_obj);
 372         } else {
 373                 kfree(job);
 374         }
 375 }
 376
 377 static void handle_user_interrupt_wait_list(struct hl_device *hdev, struct hl_user_interrupt *intr)
 378 {
 379         struct hl_user_pending_interrupt *pend, *temp_pend;
 380         unsigned long flags;
 381
 382         spin_lock_irqsave(&intr->wait_list_lock, flags);
 383         list_for_each_entry_safe(pend, temp_pend, &intr->wait_list_head, list_node) {
 384                 if ((pend->cq_kernel_addr && *(pend->cq_kernel_addr) >= pend->cq_target_value) ||
 385                                 !pend->cq_kernel_addr) {
 386                         /* Handle wait target value node */
 387                         pend->fence.timestamp = intr->timestamp;
 388                         complete_all(&pend->fence.completion);
 389                 }
 390         }
 391         spin_unlock_irqrestore(&intr->wait_list_lock, flags);
 392 }
 393
 394 static void handle_tpc_interrupt(struct hl_device *hdev)
 395 {
 396         u64 event_mask;
 397         u32 flags;
 398
 399         event_mask = HL_NOTIFIER_EVENT_TPC_ASSERT |
 400                 HL_NOTIFIER_EVENT_USER_ENGINE_ERR |
 401                 HL_NOTIFIER_EVENT_DEVICE_RESET;
 402
 403         flags = HL_DRV_RESET_DELAY;
 404
 405         dev_err_ratelimited(hdev->dev, "Received TPC assert\n");
 406         hl_device_cond_reset(hdev, flags, event_mask);
 407 }
 408
 409 static void handle_unexpected_user_interrupt(struct hl_device *hdev)
 410 {
 411         dev_err_ratelimited(hdev->dev, "Received unexpected user error interrupt\n");
 412 }
 413
 414 /**
 415  * hl_irq_user_interrupt_handler - irq handler for user interrupts.
 416  *
 417  * @irq: irq number
 418  * @arg: pointer to user interrupt structure
 419  */
 420 irqreturn_t hl_irq_user_interrupt_handler(int irq, void *arg)
 421 {
 422         struct hl_user_interrupt *user_int = arg;
 423         struct hl_device *hdev = user_int->hdev;
 424
 425         user_int->timestamp = ktime_get();
 426         switch (user_int->type) {
 427         case HL_USR_INTERRUPT_CQ:
 428                 /* First handle user waiters threads */
 429                 handle_user_interrupt_wait_list(hdev, &hdev->common_user_cq_interrupt);
 430                 handle_user_interrupt_wait_list(hdev, user_int);
 431
 432                 /* Second handle user timestamp registrations */
 433                 handle_user_interrupt_ts_list(hdev,  &hdev->common_user_cq_interrupt);
 434                 handle_user_interrupt_ts_list(hdev, user_int);
 435                 break;
 436         case HL_USR_INTERRUPT_DECODER:
 437                 handle_user_interrupt_wait_list(hdev, &hdev->common_decoder_interrupt);
 438
 439                 /* Handle decoder interrupt registered on this specific irq */
 440                 handle_user_interrupt_wait_list(hdev, user_int);
 441                 break;
 442         default:
 443                 break;
 444         }
 445
 446         return IRQ_HANDLED;
 447 }
 448
 449 /**
 450  * hl_irq_user_interrupt_thread_handler - irq thread handler for user interrupts.
 451  * This function is invoked by threaded irq mechanism
 452  *
 453  * @irq: irq number
 454  * @arg: pointer to user interrupt structure
 455  *
 456  */
 457 irqreturn_t hl_irq_user_interrupt_thread_handler(int irq, void *arg)
 458 {
 459         struct hl_user_interrupt *user_int = arg;
 460         struct hl_device *hdev = user_int->hdev;
 461
 462         user_int->timestamp = ktime_get();
 463         switch (user_int->type) {
 464         case HL_USR_INTERRUPT_TPC:
 465                 handle_tpc_interrupt(hdev);
 466                 break;
 467         case HL_USR_INTERRUPT_UNEXPECTED:
 468                 handle_unexpected_user_interrupt(hdev);
 469                 break;
 470         default:
 471                 break;
 472         }
 473
 474         return IRQ_HANDLED;
 475 }
 476
 477 irqreturn_t hl_irq_eq_error_interrupt_thread_handler(int irq, void *arg)
 478 {
 479         u64 event_mask = HL_NOTIFIER_EVENT_DEVICE_RESET | HL_NOTIFIER_EVENT_DEVICE_UNAVAILABLE;
 480         struct hl_device *hdev = arg;
 481
 482         dev_err(hdev->dev, "EQ error interrupt received\n");
 483
 484         hl_device_cond_reset(hdev, HL_DRV_RESET_HARD, event_mask);
 485
 486         return IRQ_HANDLED;
 487 }
 488
 489 /**
 490  * hl_irq_handler_eq - irq handler for event queue
 491  *
 492  * @irq: irq number
 493  * @arg: pointer to event queue structure
 494  *
 495  */
 496 irqreturn_t hl_irq_handler_eq(int irq, void *arg)
 497 {
 498         struct hl_eq *eq = arg;
 499         struct hl_device *hdev = eq->hdev;
 500         struct hl_eq_entry *eq_entry;
 501         struct hl_eq_entry *eq_base;
 502         struct hl_eqe_work *handle_eqe_work;
 503         bool entry_ready;
 504         u32 cur_eqe, ctl;
 505         u16 cur_eqe_index, event_type;
 506
 507         eq_base = eq->kernel_address;
 508
 509         while (1) {
 510                 cur_eqe = le32_to_cpu(eq_base[eq->ci].hdr.ctl);
 511                 entry_ready = !!FIELD_GET(EQ_CTL_READY_MASK, cur_eqe);
 512
 513                 if (!entry_ready)
 514                         break;
 515
 516                 cur_eqe_index = FIELD_GET(EQ_CTL_INDEX_MASK, cur_eqe);
 517                 if ((hdev->event_queue.check_eqe_index) &&
 518                                 (((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK) != cur_eqe_index)) {
 519                         dev_err(hdev->dev,
 520                                 "EQE %#x in queue is ready but index does not match %d!=%d",
 521                                 cur_eqe,
 522                                 ((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK),
 523                                 cur_eqe_index);
 524                         break;
 525                 }
 526
 527                 eq->prev_eqe_index++;
 528
 529                 eq_entry = &eq_base[eq->ci];
 530
 531                 /*
 532                  * Make sure we read EQ entry contents after we've
 533                  * checked the ownership bit.
 534                  */
 535                 dma_rmb();
 536
 537                 if (hdev->disabled && !hdev->reset_info.in_compute_reset) {
 538                         ctl = le32_to_cpu(eq_entry->hdr.ctl);
 539                         event_type = ((ctl & EQ_CTL_EVENT_TYPE_MASK) >> EQ_CTL_EVENT_TYPE_SHIFT);
 540                         dev_warn(hdev->dev,
 541                                 "Device disabled but received an EQ event (%u)\n", event_type);
 542                         goto skip_irq;
 543                 }
 544
 545                 handle_eqe_work = kmalloc(sizeof(*handle_eqe_work), GFP_ATOMIC);
 546                 if (handle_eqe_work) {
 547                         INIT_WORK(&handle_eqe_work->eq_work, irq_handle_eqe);
 548                         handle_eqe_work->hdev = hdev;
 549
 550                         memcpy(&handle_eqe_work->eq_entry, eq_entry,
 551                                         sizeof(*eq_entry));
 552
 553                         queue_work(hdev->eq_wq, &handle_eqe_work->eq_work);
 554                 }
 555 skip_irq:
 556                 /* Clear EQ entry ready bit */
 557                 eq_entry->hdr.ctl =
 558                         cpu_to_le32(le32_to_cpu(eq_entry->hdr.ctl) &
 559                                                         ~EQ_CTL_READY_MASK);
 560
 561                 eq->ci = hl_eq_inc_ptr(eq->ci);
 562
 563                 hdev->asic_funcs->update_eq_ci(hdev, eq->ci);
 564         }
 565
 566         return IRQ_HANDLED;
 567 }
 568
 569 /**
 570  * hl_irq_handler_dec_abnrm - Decoder error interrupt handler
 571  * @irq: IRQ number
 572  * @arg: pointer to decoder structure.
 573  */
 574 irqreturn_t hl_irq_handler_dec_abnrm(int irq, void *arg)
 575 {
 576         struct hl_dec *dec = arg;
 577
 578         schedule_work(&dec->abnrm_intr_work);
 579
 580         return IRQ_HANDLED;
 581 }
 582
 583 /**
 584  * hl_cq_init - main initialization function for an cq object
 585  *
 586  * @hdev: pointer to device structure
 587  * @q: pointer to cq structure
 588  * @hw_queue_id: The H/W queue ID this completion queue belongs to
 589  *               HL_INVALID_QUEUE if cq is not attached to any specific queue
 590  *
 591  * Allocate dma-able memory for the completion queue and initialize fields
 592  * Returns 0 on success
 593  */
 594 int hl_cq_init(struct hl_device *hdev, struct hl_cq *q, u32 hw_queue_id)
 595 {
 596         void *p;
 597
 598         p = hl_asic_dma_alloc_coherent(hdev, HL_CQ_SIZE_IN_BYTES, &q->bus_address,
 599                                         GFP_KERNEL | __GFP_ZERO);
 600         if (!p)
 601                 return -ENOMEM;
 602
 603         q->hdev = hdev;
 604         q->kernel_address = p;
 605         q->hw_queue_id = hw_queue_id;
 606         q->ci = 0;
 607         q->pi = 0;
 608
 609         atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);
 610
 611         return 0;
 612 }
 613
 614 /**
 615  * hl_cq_fini - destroy completion queue
 616  *
 617  * @hdev: pointer to device structure
 618  * @q: pointer to cq structure
 619  *
 620  * Free the completion queue memory
 621  */
 622 void hl_cq_fini(struct hl_device *hdev, struct hl_cq *q)
 623 {
 624         hl_asic_dma_free_coherent(hdev, HL_CQ_SIZE_IN_BYTES, q->kernel_address, q->bus_address);
 625 }
 626
 627 void hl_cq_reset(struct hl_device *hdev, struct hl_cq *q)
 628 {
 629         q->ci = 0;
 630         q->pi = 0;
 631
 632         atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);
 633
 634         /*
 635          * It's not enough to just reset the PI/CI because the H/W may have
 636          * written valid completion entries before it was halted and therefore
 637          * we need to clean the actual queues so we won't process old entries
 638          * when the device is operational again
 639          */
 640
 641         memset(q->kernel_address, 0, HL_CQ_SIZE_IN_BYTES);
 642 }
 643
 644 /**
 645  * hl_eq_init - main initialization function for an event queue object
 646  *
 647  * @hdev: pointer to device structure
 648  * @q: pointer to eq structure
 649  *
 650  * Allocate dma-able memory for the event queue and initialize fields
 651  * Returns 0 on success
 652  */
 653 int hl_eq_init(struct hl_device *hdev, struct hl_eq *q)
 654 {
 655         void *p;
 656
 657         p = hl_cpu_accessible_dma_pool_alloc(hdev, HL_EQ_SIZE_IN_BYTES, &q->bus_address);
 658         if (!p)
 659                 return -ENOMEM;
 660
 661         q->hdev = hdev;
 662         q->kernel_address = p;
 663         q->ci = 0;
 664         q->prev_eqe_index = 0;
 665
 666         return 0;
 667 }
 668
 669 /**
 670  * hl_eq_fini - destroy event queue
 671  *
 672  * @hdev: pointer to device structure
 673  * @q: pointer to eq structure
 674  *
 675  * Free the event queue memory
 676  */
 677 void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q)
 678 {
 679         flush_workqueue(hdev->eq_wq);
 680
 681         hl_cpu_accessible_dma_pool_free(hdev, HL_EQ_SIZE_IN_BYTES, q->kernel_address);
 682 }
 683
 684 void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q)
 685 {
 686         q->ci = 0;
 687         q->prev_eqe_index = 0;
 688
 689         /*
 690          * It's not enough to just reset the PI/CI because the H/W may have
 691          * written valid completion entries before it was halted and therefore
 692          * we need to clean the actual queues so we won't process old entries
 693          * when the device is operational again
 694          */
 695
 696         memset(q->kernel_address, 0, HL_EQ_SIZE_IN_BYTES);
 697 }