drivers/accel/qaic/qaic_drv.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2
   3 /* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */
   4 /* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */
   5
   6 #include <linux/delay.h>
   7 #include <linux/dma-mapping.h>
   8 #include <linux/idr.h>
   9 #include <linux/interrupt.h>
  10 #include <linux/list.h>
  11 #include <linux/kobject.h>
  12 #include <linux/kref.h>
  13 #include <linux/mhi.h>
  14 #include <linux/module.h>
  15 #include <linux/msi.h>
  16 #include <linux/mutex.h>
  17 #include <linux/pci.h>
  18 #include <linux/spinlock.h>
  19 #include <linux/workqueue.h>
  20 #include <linux/wait.h>
  21 #include <drm/drm_accel.h>
  22 #include <drm/drm_drv.h>
  23 #include <drm/drm_file.h>
  24 #include <drm/drm_gem.h>
  25 #include <drm/drm_ioctl.h>
  26 #include <drm/drm_managed.h>
  27 #include <uapi/drm/qaic_accel.h>
  28
  29 #include "mhi_controller.h"
  30 #include "qaic.h"
  31 #include "qaic_timesync.h"
  32
  33 MODULE_IMPORT_NS(DMA_BUF);
  34
  35 #define PCI_DEV_AIC100                  0xa100
  36 #define QAIC_NAME                       "qaic"
  37 #define QAIC_DESC                       "Qualcomm Cloud AI Accelerators"
  38 #define CNTL_MAJOR                      5
  39 #define CNTL_MINOR                      0
  40
  41 bool datapath_polling;
  42 module_param(datapath_polling, bool, 0400);
  43 MODULE_PARM_DESC(datapath_polling, "Operate the datapath in polling mode");
  44 static bool link_up;
  45 static DEFINE_IDA(qaic_usrs);
  46
  47 static void free_usr(struct kref *kref)
  48 {
  49         struct qaic_user *usr = container_of(kref, struct qaic_user, ref_count);
  50
  51         cleanup_srcu_struct(&usr->qddev_lock);
  52         ida_free(&qaic_usrs, usr->handle);
  53         kfree(usr);
  54 }
  55
  56 static int qaic_open(struct drm_device *dev, struct drm_file *file)
  57 {
  58         struct qaic_drm_device *qddev = to_qaic_drm_device(dev);
  59         struct qaic_device *qdev = qddev->qdev;
  60         struct qaic_user *usr;
  61         int rcu_id;
  62         int ret;
  63
  64         rcu_id = srcu_read_lock(&qdev->dev_lock);
  65         if (qdev->dev_state != QAIC_ONLINE) {
  66                 ret = -ENODEV;
  67                 goto dev_unlock;
  68         }
  69
  70         usr = kmalloc(sizeof(*usr), GFP_KERNEL);
  71         if (!usr) {
  72                 ret = -ENOMEM;
  73                 goto dev_unlock;
  74         }
  75
  76         usr->handle = ida_alloc(&qaic_usrs, GFP_KERNEL);
  77         if (usr->handle < 0) {
  78                 ret = usr->handle;
  79                 goto free_usr;
  80         }
  81         usr->qddev = qddev;
  82         atomic_set(&usr->chunk_id, 0);
  83         init_srcu_struct(&usr->qddev_lock);
  84         kref_init(&usr->ref_count);
  85
  86         ret = mutex_lock_interruptible(&qddev->users_mutex);
  87         if (ret)
  88                 goto cleanup_usr;
  89
  90         list_add(&usr->node, &qddev->users);
  91         mutex_unlock(&qddev->users_mutex);
  92
  93         file->driver_priv = usr;
  94
  95         srcu_read_unlock(&qdev->dev_lock, rcu_id);
  96         return 0;
  97
  98 cleanup_usr:
  99         cleanup_srcu_struct(&usr->qddev_lock);
 100         ida_free(&qaic_usrs, usr->handle);
 101 free_usr:
 102         kfree(usr);
 103 dev_unlock:
 104         srcu_read_unlock(&qdev->dev_lock, rcu_id);
 105         return ret;
 106 }
 107
 108 static void qaic_postclose(struct drm_device *dev, struct drm_file *file)
 109 {
 110         struct qaic_user *usr = file->driver_priv;
 111         struct qaic_drm_device *qddev;
 112         struct qaic_device *qdev;
 113         int qdev_rcu_id;
 114         int usr_rcu_id;
 115         int i;
 116
 117         qddev = usr->qddev;
 118         usr_rcu_id = srcu_read_lock(&usr->qddev_lock);
 119         if (qddev) {
 120                 qdev = qddev->qdev;
 121                 qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
 122                 if (qdev->dev_state == QAIC_ONLINE) {
 123                         qaic_release_usr(qdev, usr);
 124                         for (i = 0; i < qdev->num_dbc; ++i)
 125                                 if (qdev->dbc[i].usr && qdev->dbc[i].usr->handle == usr->handle)
 126                                         release_dbc(qdev, i);
 127                 }
 128                 srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
 129
 130                 mutex_lock(&qddev->users_mutex);
 131                 if (!list_empty(&usr->node))
 132                         list_del_init(&usr->node);
 133                 mutex_unlock(&qddev->users_mutex);
 134         }
 135
 136         srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
 137         kref_put(&usr->ref_count, free_usr);
 138
 139         file->driver_priv = NULL;
 140 }
 141
 142 DEFINE_DRM_ACCEL_FOPS(qaic_accel_fops);
 143
 144 static const struct drm_ioctl_desc qaic_drm_ioctls[] = {
 145         DRM_IOCTL_DEF_DRV(QAIC_MANAGE, qaic_manage_ioctl, 0),
 146         DRM_IOCTL_DEF_DRV(QAIC_CREATE_BO, qaic_create_bo_ioctl, 0),
 147         DRM_IOCTL_DEF_DRV(QAIC_MMAP_BO, qaic_mmap_bo_ioctl, 0),
 148         DRM_IOCTL_DEF_DRV(QAIC_ATTACH_SLICE_BO, qaic_attach_slice_bo_ioctl, 0),
 149         DRM_IOCTL_DEF_DRV(QAIC_EXECUTE_BO, qaic_execute_bo_ioctl, 0),
 150         DRM_IOCTL_DEF_DRV(QAIC_PARTIAL_EXECUTE_BO, qaic_partial_execute_bo_ioctl, 0),
 151         DRM_IOCTL_DEF_DRV(QAIC_WAIT_BO, qaic_wait_bo_ioctl, 0),
 152         DRM_IOCTL_DEF_DRV(QAIC_PERF_STATS_BO, qaic_perf_stats_bo_ioctl, 0),
 153         DRM_IOCTL_DEF_DRV(QAIC_DETACH_SLICE_BO, qaic_detach_slice_bo_ioctl, 0),
 154 };
 155
 156 static const struct drm_driver qaic_accel_driver = {
 157         .driver_features        = DRIVER_GEM | DRIVER_COMPUTE_ACCEL,
 158
 159         .name                   = QAIC_NAME,
 160         .desc                   = QAIC_DESC,
 161         .date                   = "20190618",
 162
 163         .fops                   = &qaic_accel_fops,
 164         .open                   = qaic_open,
 165         .postclose              = qaic_postclose,
 166
 167         .ioctls                 = qaic_drm_ioctls,
 168         .num_ioctls             = ARRAY_SIZE(qaic_drm_ioctls),
 169         .gem_prime_import       = qaic_gem_prime_import,
 170 };
 171
 172 static int qaic_create_drm_device(struct qaic_device *qdev, s32 partition_id)
 173 {
 174         struct qaic_drm_device *qddev = qdev->qddev;
 175         struct drm_device *drm = to_drm(qddev);
 176         int ret;
 177
 178         /* Hold off implementing partitions until the uapi is determined */
 179         if (partition_id != QAIC_NO_PARTITION)
 180                 return -EINVAL;
 181
 182         qddev->partition_id = partition_id;
 183
 184         ret = drm_dev_register(drm, 0);
 185         if (ret)
 186                 pci_dbg(qdev->pdev, "drm_dev_register failed %d\n", ret);
 187
 188         return ret;
 189 }
 190
 191 static void qaic_destroy_drm_device(struct qaic_device *qdev, s32 partition_id)
 192 {
 193         struct qaic_drm_device *qddev = qdev->qddev;
 194         struct drm_device *drm = to_drm(qddev);
 195         struct qaic_user *usr;
 196
 197         drm_dev_unregister(drm);
 198         qddev->partition_id = 0;
 199         /*
 200          * Existing users get unresolvable errors till they close FDs.
 201          * Need to sync carefully with users calling close(). The
 202          * list of users can be modified elsewhere when the lock isn't
 203          * held here, but the sync'ing the srcu with the mutex held
 204          * could deadlock. Grab the mutex so that the list will be
 205          * unmodified. The user we get will exist as long as the
 206          * lock is held. Signal that the qcdev is going away, and
 207          * grab a reference to the user so they don't go away for
 208          * synchronize_srcu(). Then release the mutex to avoid
 209          * deadlock and make sure the user has observed the signal.
 210          * With the lock released, we cannot maintain any state of the
 211          * user list.
 212          */
 213         mutex_lock(&qddev->users_mutex);
 214         while (!list_empty(&qddev->users)) {
 215                 usr = list_first_entry(&qddev->users, struct qaic_user, node);
 216                 list_del_init(&usr->node);
 217                 kref_get(&usr->ref_count);
 218                 usr->qddev = NULL;
 219                 mutex_unlock(&qddev->users_mutex);
 220                 synchronize_srcu(&usr->qddev_lock);
 221                 kref_put(&usr->ref_count, free_usr);
 222                 mutex_lock(&qddev->users_mutex);
 223         }
 224         mutex_unlock(&qddev->users_mutex);
 225 }
 226
 227 static int qaic_mhi_probe(struct mhi_device *mhi_dev, const struct mhi_device_id *id)
 228 {
 229         u16 major = -1, minor = -1;
 230         struct qaic_device *qdev;
 231         int ret;
 232
 233         /*
 234          * Invoking this function indicates that the control channel to the
 235          * device is available. We use that as a signal to indicate that
 236          * the device side firmware has booted. The device side firmware
 237          * manages the device resources, so we need to communicate with it
 238          * via the control channel in order to utilize the device. Therefore
 239          * we wait until this signal to create the drm dev that userspace will
 240          * use to control the device, because without the device side firmware,
 241          * userspace can't do anything useful.
 242          */
 243
 244         qdev = pci_get_drvdata(to_pci_dev(mhi_dev->mhi_cntrl->cntrl_dev));
 245
 246         dev_set_drvdata(&mhi_dev->dev, qdev);
 247         qdev->cntl_ch = mhi_dev;
 248
 249         ret = qaic_control_open(qdev);
 250         if (ret) {
 251                 pci_dbg(qdev->pdev, "%s: control_open failed %d\n", __func__, ret);
 252                 return ret;
 253         }
 254
 255         qdev->dev_state = QAIC_BOOT;
 256         ret = get_cntl_version(qdev, NULL, &major, &minor);
 257         if (ret || major != CNTL_MAJOR || minor > CNTL_MINOR) {
 258                 pci_err(qdev->pdev, "%s: Control protocol version (%d.%d) not supported. Supported version is (%d.%d). Ret: %d\n",
 259                         __func__, major, minor, CNTL_MAJOR, CNTL_MINOR, ret);
 260                 ret = -EINVAL;
 261                 goto close_control;
 262         }
 263         qdev->dev_state = QAIC_ONLINE;
 264         kobject_uevent(&(to_accel_kdev(qdev->qddev))->kobj, KOBJ_ONLINE);
 265
 266         return ret;
 267
 268 close_control:
 269         qaic_control_close(qdev);
 270         return ret;
 271 }
 272
 273 static void qaic_mhi_remove(struct mhi_device *mhi_dev)
 274 {
 275 /* This is redundant since we have already observed the device crash */
 276 }
 277
 278 static void qaic_notify_reset(struct qaic_device *qdev)
 279 {
 280         int i;
 281
 282         kobject_uevent(&(to_accel_kdev(qdev->qddev))->kobj, KOBJ_OFFLINE);
 283         qdev->dev_state = QAIC_OFFLINE;
 284         /* wake up any waiters to avoid waiting for timeouts at sync */
 285         wake_all_cntl(qdev);
 286         for (i = 0; i < qdev->num_dbc; ++i)
 287                 wakeup_dbc(qdev, i);
 288         synchronize_srcu(&qdev->dev_lock);
 289 }
 290
 291 void qaic_dev_reset_clean_local_state(struct qaic_device *qdev)
 292 {
 293         int i;
 294
 295         qaic_notify_reset(qdev);
 296
 297         /* start tearing things down */
 298         for (i = 0; i < qdev->num_dbc; ++i)
 299                 release_dbc(qdev, i);
 300 }
 301
 302 static void cleanup_qdev(struct qaic_device *qdev)
 303 {
 304         int i;
 305
 306         for (i = 0; i < qdev->num_dbc; ++i)
 307                 cleanup_srcu_struct(&qdev->dbc[i].ch_lock);
 308         cleanup_srcu_struct(&qdev->dev_lock);
 309         pci_set_drvdata(qdev->pdev, NULL);
 310         destroy_workqueue(qdev->cntl_wq);
 311         destroy_workqueue(qdev->qts_wq);
 312 }
 313
 314 static struct qaic_device *create_qdev(struct pci_dev *pdev, const struct pci_device_id *id)
 315 {
 316         struct qaic_drm_device *qddev;
 317         struct qaic_device *qdev;
 318         int i;
 319
 320         qdev = devm_kzalloc(&pdev->dev, sizeof(*qdev), GFP_KERNEL);
 321         if (!qdev)
 322                 return NULL;
 323
 324         qdev->dev_state = QAIC_OFFLINE;
 325         if (id->device == PCI_DEV_AIC100) {
 326                 qdev->num_dbc = 16;
 327                 qdev->dbc = devm_kcalloc(&pdev->dev, qdev->num_dbc, sizeof(*qdev->dbc), GFP_KERNEL);
 328                 if (!qdev->dbc)
 329                         return NULL;
 330         }
 331
 332         qdev->cntl_wq = alloc_workqueue("qaic_cntl", WQ_UNBOUND, 0);
 333         if (!qdev->cntl_wq)
 334                 return NULL;
 335
 336         qdev->qts_wq = alloc_workqueue("qaic_ts", WQ_UNBOUND, 0);
 337         if (!qdev->qts_wq) {
 338                 destroy_workqueue(qdev->cntl_wq);
 339                 return NULL;
 340         }
 341
 342         pci_set_drvdata(pdev, qdev);
 343         qdev->pdev = pdev;
 344
 345         mutex_init(&qdev->cntl_mutex);
 346         INIT_LIST_HEAD(&qdev->cntl_xfer_list);
 347         init_srcu_struct(&qdev->dev_lock);
 348
 349         for (i = 0; i < qdev->num_dbc; ++i) {
 350                 spin_lock_init(&qdev->dbc[i].xfer_lock);
 351                 qdev->dbc[i].qdev = qdev;
 352                 qdev->dbc[i].id = i;
 353                 INIT_LIST_HEAD(&qdev->dbc[i].xfer_list);
 354                 init_srcu_struct(&qdev->dbc[i].ch_lock);
 355                 init_waitqueue_head(&qdev->dbc[i].dbc_release);
 356                 INIT_LIST_HEAD(&qdev->dbc[i].bo_lists);
 357         }
 358
 359         qddev = devm_drm_dev_alloc(&pdev->dev, &qaic_accel_driver, struct qaic_drm_device, drm);
 360         if (IS_ERR(qddev)) {
 361                 cleanup_qdev(qdev);
 362                 return NULL;
 363         }
 364
 365         drmm_mutex_init(to_drm(qddev), &qddev->users_mutex);
 366         INIT_LIST_HEAD(&qddev->users);
 367         qddev->qdev = qdev;
 368         qdev->qddev = qddev;
 369
 370         return qdev;
 371 }
 372
 373 static int init_pci(struct qaic_device *qdev, struct pci_dev *pdev)
 374 {
 375         int bars;
 376         int ret;
 377
 378         bars = pci_select_bars(pdev, IORESOURCE_MEM);
 379
 380         /* make sure the device has the expected BARs */
 381         if (bars != (BIT(0) | BIT(2) | BIT(4))) {
 382                 pci_dbg(pdev, "%s: expected BARs 0, 2, and 4 not found in device. Found 0x%x\n",
 383                         __func__, bars);
 384                 return -EINVAL;
 385         }
 386
 387         ret = pcim_enable_device(pdev);
 388         if (ret)
 389                 return ret;
 390
 391         ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
 392         if (ret)
 393                 return ret;
 394         ret = dma_set_max_seg_size(&pdev->dev, UINT_MAX);
 395         if (ret)
 396                 return ret;
 397
 398         qdev->bar_0 = devm_ioremap_resource(&pdev->dev, &pdev->resource[0]);
 399         if (IS_ERR(qdev->bar_0))
 400                 return PTR_ERR(qdev->bar_0);
 401
 402         qdev->bar_2 = devm_ioremap_resource(&pdev->dev, &pdev->resource[2]);
 403         if (IS_ERR(qdev->bar_2))
 404                 return PTR_ERR(qdev->bar_2);
 405
 406         /* Managed release since we use pcim_enable_device above */
 407         pci_set_master(pdev);
 408
 409         return 0;
 410 }
 411
 412 static int init_msi(struct qaic_device *qdev, struct pci_dev *pdev)
 413 {
 414         int mhi_irq;
 415         int ret;
 416         int i;
 417
 418         /* Managed release since we use pcim_enable_device */
 419         ret = pci_alloc_irq_vectors(pdev, 32, 32, PCI_IRQ_MSI);
 420         if (ret == -ENOSPC) {
 421                 ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
 422                 if (ret < 0)
 423                         return ret;
 424
 425                 /*
 426                  * Operate in one MSI mode. All interrupts will be directed to
 427                  * MSI0; every interrupt will wake up all the interrupt handlers
 428                  * (MHI and DBC[0-15]). Since the interrupt is now shared, it is
 429                  * not disabled during DBC threaded handler, but only one thread
 430                  * will be allowed to run per DBC, so while it can be
 431                  * interrupted, it shouldn't race with itself.
 432                  */
 433                 qdev->single_msi = true;
 434                 pci_info(pdev, "Allocating 32 MSIs failed, operating in 1 MSI mode. Performance may be impacted.\n");
 435         } else if (ret < 0) {
 436                 return ret;
 437         }
 438
 439         mhi_irq = pci_irq_vector(pdev, 0);
 440         if (mhi_irq < 0)
 441                 return mhi_irq;
 442
 443         for (i = 0; i < qdev->num_dbc; ++i) {
 444                 ret = devm_request_threaded_irq(&pdev->dev,
 445                                                 pci_irq_vector(pdev, qdev->single_msi ? 0 : i + 1),
 446                                                 dbc_irq_handler, dbc_irq_threaded_fn, IRQF_SHARED,
 447                                                 "qaic_dbc", &qdev->dbc[i]);
 448                 if (ret)
 449                         return ret;
 450
 451                 if (datapath_polling) {
 452                         qdev->dbc[i].irq = pci_irq_vector(pdev, qdev->single_msi ? 0 : i + 1);
 453                         if (!qdev->single_msi)
 454                                 disable_irq_nosync(qdev->dbc[i].irq);
 455                         INIT_WORK(&qdev->dbc[i].poll_work, irq_polling_work);
 456                 }
 457         }
 458
 459         return mhi_irq;
 460 }
 461
 462 static int qaic_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 463 {
 464         struct qaic_device *qdev;
 465         int mhi_irq;
 466         int ret;
 467         int i;
 468
 469         qdev = create_qdev(pdev, id);
 470         if (!qdev)
 471                 return -ENOMEM;
 472
 473         ret = init_pci(qdev, pdev);
 474         if (ret)
 475                 goto cleanup_qdev;
 476
 477         for (i = 0; i < qdev->num_dbc; ++i)
 478                 qdev->dbc[i].dbc_base = qdev->bar_2 + QAIC_DBC_OFF(i);
 479
 480         mhi_irq = init_msi(qdev, pdev);
 481         if (mhi_irq < 0) {
 482                 ret = mhi_irq;
 483                 goto cleanup_qdev;
 484         }
 485
 486         ret = qaic_create_drm_device(qdev, QAIC_NO_PARTITION);
 487         if (ret)
 488                 goto cleanup_qdev;
 489
 490         qdev->mhi_cntrl = qaic_mhi_register_controller(pdev, qdev->bar_0, mhi_irq,
 491                                                        qdev->single_msi);
 492         if (IS_ERR(qdev->mhi_cntrl)) {
 493                 ret = PTR_ERR(qdev->mhi_cntrl);
 494                 goto cleanup_drm_dev;
 495         }
 496
 497         return 0;
 498
 499 cleanup_drm_dev:
 500         qaic_destroy_drm_device(qdev, QAIC_NO_PARTITION);
 501 cleanup_qdev:
 502         cleanup_qdev(qdev);
 503         return ret;
 504 }
 505
 506 static void qaic_pci_remove(struct pci_dev *pdev)
 507 {
 508         struct qaic_device *qdev = pci_get_drvdata(pdev);
 509
 510         if (!qdev)
 511                 return;
 512
 513         qaic_dev_reset_clean_local_state(qdev);
 514         qaic_destroy_drm_device(qdev, QAIC_NO_PARTITION);
 515         qaic_mhi_free_controller(qdev->mhi_cntrl, link_up);
 516         cleanup_qdev(qdev);
 517 }
 518
 519 static void qaic_pci_shutdown(struct pci_dev *pdev)
 520 {
 521         /* see qaic_exit for what link_up is doing */
 522         link_up = true;
 523         qaic_pci_remove(pdev);
 524 }
 525
 526 static pci_ers_result_t qaic_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t error)
 527 {
 528         return PCI_ERS_RESULT_NEED_RESET;
 529 }
 530
 531 static void qaic_pci_reset_prepare(struct pci_dev *pdev)
 532 {
 533         struct qaic_device *qdev = pci_get_drvdata(pdev);
 534
 535         qaic_notify_reset(qdev);
 536         qaic_mhi_start_reset(qdev->mhi_cntrl);
 537         qaic_dev_reset_clean_local_state(qdev);
 538 }
 539
 540 static void qaic_pci_reset_done(struct pci_dev *pdev)
 541 {
 542         struct qaic_device *qdev = pci_get_drvdata(pdev);
 543
 544         qaic_mhi_reset_done(qdev->mhi_cntrl);
 545 }
 546
 547 static const struct mhi_device_id qaic_mhi_match_table[] = {
 548         { .chan = "QAIC_CONTROL", },
 549         {},
 550 };
 551
 552 static struct mhi_driver qaic_mhi_driver = {
 553         .id_table = qaic_mhi_match_table,
 554         .remove = qaic_mhi_remove,
 555         .probe = qaic_mhi_probe,
 556         .ul_xfer_cb = qaic_mhi_ul_xfer_cb,
 557         .dl_xfer_cb = qaic_mhi_dl_xfer_cb,
 558         .driver = {
 559                 .name = "qaic_mhi",
 560         },
 561 };
 562
 563 static const struct pci_device_id qaic_ids[] = {
 564         { PCI_DEVICE(PCI_VENDOR_ID_QCOM, PCI_DEV_AIC100), },
 565         { }
 566 };
 567 MODULE_DEVICE_TABLE(pci, qaic_ids);
 568
 569 static const struct pci_error_handlers qaic_pci_err_handler = {
 570         .error_detected = qaic_pci_error_detected,
 571         .reset_prepare = qaic_pci_reset_prepare,
 572         .reset_done = qaic_pci_reset_done,
 573 };
 574
 575 static struct pci_driver qaic_pci_driver = {
 576         .name = QAIC_NAME,
 577         .id_table = qaic_ids,
 578         .probe = qaic_pci_probe,
 579         .remove = qaic_pci_remove,
 580         .shutdown = qaic_pci_shutdown,
 581         .err_handler = &qaic_pci_err_handler,
 582 };
 583
 584 static int __init qaic_init(void)
 585 {
 586         int ret;
 587
 588         ret = pci_register_driver(&qaic_pci_driver);
 589         if (ret) {
 590                 pr_debug("qaic: pci_register_driver failed %d\n", ret);
 591                 return ret;
 592         }
 593
 594         ret = mhi_driver_register(&qaic_mhi_driver);
 595         if (ret) {
 596                 pr_debug("qaic: mhi_driver_register failed %d\n", ret);
 597                 goto free_pci;
 598         }
 599
 600         ret = qaic_timesync_init();
 601         if (ret)
 602                 pr_debug("qaic: qaic_timesync_init failed %d\n", ret);
 603
 604         return 0;
 605
 606 free_pci:
 607         pci_unregister_driver(&qaic_pci_driver);
 608         return ret;
 609 }
 610
 611 static void __exit qaic_exit(void)
 612 {
 613         /*
 614          * We assume that qaic_pci_remove() is called due to a hotplug event
 615          * which would mean that the link is down, and thus
 616          * qaic_mhi_free_controller() should not try to access the device during
 617          * cleanup.
 618          * We call pci_unregister_driver() below, which also triggers
 619          * qaic_pci_remove(), but since this is module exit, we expect the link
 620          * to the device to be up, in which case qaic_mhi_free_controller()
 621          * should try to access the device during cleanup to put the device in
 622          * a sane state.
 623          * For that reason, we set link_up here to let qaic_mhi_free_controller
 624          * know the expected link state. Since the module is going to be
 625          * removed at the end of this, we don't need to worry about
 626          * reinitializing the link_up state after the cleanup is done.
 627          */
 628         link_up = true;
 629         qaic_timesync_deinit();
 630         mhi_driver_unregister(&qaic_mhi_driver);
 631         pci_unregister_driver(&qaic_pci_driver);
 632 }
 633
 634 module_init(qaic_init);
 635 module_exit(qaic_exit);
 636
 637 MODULE_AUTHOR(QAIC_DESC " Kernel Driver Team");
 638 MODULE_DESCRIPTION(QAIC_DESC " Accel Driver");
 639 MODULE_LICENSE("GPL");