drivers/net/ethernet/amd/pds_core/main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2023 Advanced Micro Devices, Inc */
   3
   4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   5
   6 #include <linux/pci.h>
   7
   8 #include <linux/pds/pds_common.h>
   9
  10 #include "core.h"
  11
  12 MODULE_DESCRIPTION(PDSC_DRV_DESCRIPTION);
  13 MODULE_AUTHOR("Advanced Micro Devices, Inc");
  14 MODULE_LICENSE("GPL");
  15
  16 /* Supported devices */
  17 static const struct pci_device_id pdsc_id_table[] = {
  18         { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_CORE_PF) },
  19         { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_VDPA_VF) },
  20         { 0, }  /* end of table */
  21 };
  22 MODULE_DEVICE_TABLE(pci, pdsc_id_table);
  23
  24 static void pdsc_wdtimer_cb(struct timer_list *t)
  25 {
  26         struct pdsc *pdsc = from_timer(pdsc, t, wdtimer);
  27
  28         dev_dbg(pdsc->dev, "%s: jiffies %ld\n", __func__, jiffies);
  29         mod_timer(&pdsc->wdtimer,
  30                   round_jiffies(jiffies + pdsc->wdtimer_period));
  31
  32         queue_work(pdsc->wq, &pdsc->health_work);
  33 }
  34
  35 static void pdsc_unmap_bars(struct pdsc *pdsc)
  36 {
  37         struct pdsc_dev_bar *bars = pdsc->bars;
  38         unsigned int i;
  39
  40         pdsc->info_regs = NULL;
  41         pdsc->cmd_regs = NULL;
  42         pdsc->intr_status = NULL;
  43         pdsc->intr_ctrl = NULL;
  44
  45         for (i = 0; i < PDS_CORE_BARS_MAX; i++) {
  46                 if (bars[i].vaddr)
  47                         pci_iounmap(pdsc->pdev, bars[i].vaddr);
  48         }
  49 }
  50
  51 static int pdsc_map_bars(struct pdsc *pdsc)
  52 {
  53         struct pdsc_dev_bar *bar = pdsc->bars;
  54         struct pci_dev *pdev = pdsc->pdev;
  55         struct device *dev = pdsc->dev;
  56         struct pdsc_dev_bar *bars;
  57         unsigned int i, j;
  58         int num_bars = 0;
  59         int err;
  60         u32 sig;
  61
  62         bars = pdsc->bars;
  63
  64         /* Since the PCI interface in the hardware is configurable,
  65          * we need to poke into all the bars to find the set we're
  66          * expecting.
  67          */
  68         for (i = 0, j = 0; i < PDS_CORE_BARS_MAX; i++) {
  69                 if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM))
  70                         continue;
  71
  72                 bars[j].len = pci_resource_len(pdev, i);
  73                 bars[j].bus_addr = pci_resource_start(pdev, i);
  74                 bars[j].res_index = i;
  75
  76                 /* only map the whole bar 0 */
  77                 if (j > 0) {
  78                         bars[j].vaddr = NULL;
  79                 } else {
  80                         bars[j].vaddr = pci_iomap(pdev, i, bars[j].len);
  81                         if (!bars[j].vaddr) {
  82                                 dev_err(dev, "Cannot map BAR %d, aborting\n", i);
  83                                 return -ENODEV;
  84                         }
  85                 }
  86
  87                 j++;
  88         }
  89         num_bars = j;
  90
  91         /* BAR0: dev_cmd and interrupts */
  92         if (num_bars < 1) {
  93                 dev_err(dev, "No bars found\n");
  94                 err = -EFAULT;
  95                 goto err_out;
  96         }
  97
  98         if (bar->len < PDS_CORE_BAR0_SIZE) {
  99                 dev_err(dev, "Resource bar size %lu too small\n", bar->len);
 100                 err = -EFAULT;
 101                 goto err_out;
 102         }
 103
 104         pdsc->info_regs = bar->vaddr + PDS_CORE_BAR0_DEV_INFO_REGS_OFFSET;
 105         pdsc->cmd_regs = bar->vaddr + PDS_CORE_BAR0_DEV_CMD_REGS_OFFSET;
 106         pdsc->intr_status = bar->vaddr + PDS_CORE_BAR0_INTR_STATUS_OFFSET;
 107         pdsc->intr_ctrl = bar->vaddr + PDS_CORE_BAR0_INTR_CTRL_OFFSET;
 108
 109         sig = ioread32(&pdsc->info_regs->signature);
 110         if (sig != PDS_CORE_DEV_INFO_SIGNATURE) {
 111                 dev_err(dev, "Incompatible firmware signature %x", sig);
 112                 err = -EFAULT;
 113                 goto err_out;
 114         }
 115
 116         /* BAR1: doorbells */
 117         bar++;
 118         if (num_bars < 2) {
 119                 dev_err(dev, "Doorbell bar missing\n");
 120                 err = -EFAULT;
 121                 goto err_out;
 122         }
 123
 124         pdsc->db_pages = bar->vaddr;
 125         pdsc->phy_db_pages = bar->bus_addr;
 126
 127         return 0;
 128
 129 err_out:
 130         pdsc_unmap_bars(pdsc);
 131         return err;
 132 }
 133
 134 void __iomem *pdsc_map_dbpage(struct pdsc *pdsc, int page_num)
 135 {
 136         return pci_iomap_range(pdsc->pdev,
 137                                pdsc->bars[PDS_CORE_PCI_BAR_DBELL].res_index,
 138                                (u64)page_num << PAGE_SHIFT, PAGE_SIZE);
 139 }
 140
 141 static int pdsc_sriov_configure(struct pci_dev *pdev, int num_vfs)
 142 {
 143         struct pdsc *pdsc = pci_get_drvdata(pdev);
 144         struct device *dev = pdsc->dev;
 145         int ret = 0;
 146
 147         if (num_vfs > 0) {
 148                 pdsc->vfs = kcalloc(num_vfs, sizeof(struct pdsc_vf),
 149                                     GFP_KERNEL);
 150                 if (!pdsc->vfs)
 151                         return -ENOMEM;
 152                 pdsc->num_vfs = num_vfs;
 153
 154                 ret = pci_enable_sriov(pdev, num_vfs);
 155                 if (ret) {
 156                         dev_err(dev, "Cannot enable SRIOV: %pe\n",
 157                                 ERR_PTR(ret));
 158                         goto no_vfs;
 159                 }
 160
 161                 return num_vfs;
 162         }
 163
 164 no_vfs:
 165         pci_disable_sriov(pdev);
 166
 167         kfree(pdsc->vfs);
 168         pdsc->vfs = NULL;
 169         pdsc->num_vfs = 0;
 170
 171         return ret;
 172 }
 173
 174 static int pdsc_init_vf(struct pdsc *vf)
 175 {
 176         struct devlink *dl;
 177         struct pdsc *pf;
 178         int err;
 179
 180         pf = pdsc_get_pf_struct(vf->pdev);
 181         if (IS_ERR_OR_NULL(pf))
 182                 return PTR_ERR(pf) ?: -1;
 183
 184         vf->vf_id = pci_iov_vf_id(vf->pdev);
 185
 186         dl = priv_to_devlink(vf);
 187         devl_lock(dl);
 188         devl_register(dl);
 189         devl_unlock(dl);
 190
 191         pf->vfs[vf->vf_id].vf = vf;
 192         err = pdsc_auxbus_dev_add(vf, pf);
 193         if (err) {
 194                 devl_lock(dl);
 195                 devl_unregister(dl);
 196                 devl_unlock(dl);
 197         }
 198
 199         return err;
 200 }
 201
 202 static const struct devlink_health_reporter_ops pdsc_fw_reporter_ops = {
 203         .name = "fw",
 204         .diagnose = pdsc_fw_reporter_diagnose,
 205 };
 206
 207 static const struct devlink_param pdsc_dl_params[] = {
 208         DEVLINK_PARAM_GENERIC(ENABLE_VNET,
 209                               BIT(DEVLINK_PARAM_CMODE_RUNTIME),
 210                               pdsc_dl_enable_get,
 211                               pdsc_dl_enable_set,
 212                               pdsc_dl_enable_validate),
 213 };
 214
 215 #define PDSC_WQ_NAME_LEN 24
 216
 217 static int pdsc_init_pf(struct pdsc *pdsc)
 218 {
 219         struct devlink_health_reporter *hr;
 220         char wq_name[PDSC_WQ_NAME_LEN];
 221         struct devlink *dl;
 222         int err;
 223
 224         pcie_print_link_status(pdsc->pdev);
 225
 226         err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME);
 227         if (err) {
 228                 dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n",
 229                         ERR_PTR(err));
 230                 return err;
 231         }
 232
 233         err = pdsc_map_bars(pdsc);
 234         if (err)
 235                 goto err_out_release_regions;
 236
 237         /* General workqueue and timer, but don't start timer yet */
 238         snprintf(wq_name, sizeof(wq_name), "%s.%d", PDS_CORE_DRV_NAME, pdsc->uid);
 239         pdsc->wq = create_singlethread_workqueue(wq_name);
 240         INIT_WORK(&pdsc->health_work, pdsc_health_thread);
 241         INIT_WORK(&pdsc->pci_reset_work, pdsc_pci_reset_thread);
 242         timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0);
 243         pdsc->wdtimer_period = PDSC_WATCHDOG_SECS * HZ;
 244
 245         mutex_init(&pdsc->devcmd_lock);
 246         mutex_init(&pdsc->config_lock);
 247         spin_lock_init(&pdsc->adminq_lock);
 248
 249         mutex_lock(&pdsc->config_lock);
 250         set_bit(PDSC_S_FW_DEAD, &pdsc->state);
 251
 252         err = pdsc_setup(pdsc, PDSC_SETUP_INIT);
 253         if (err) {
 254                 mutex_unlock(&pdsc->config_lock);
 255                 goto err_out_unmap_bars;
 256         }
 257
 258         err = pdsc_start(pdsc);
 259         if (err) {
 260                 mutex_unlock(&pdsc->config_lock);
 261                 goto err_out_teardown;
 262         }
 263
 264         mutex_unlock(&pdsc->config_lock);
 265
 266         dl = priv_to_devlink(pdsc);
 267         devl_lock(dl);
 268         err = devl_params_register(dl, pdsc_dl_params,
 269                                    ARRAY_SIZE(pdsc_dl_params));
 270         if (err) {
 271                 devl_unlock(dl);
 272                 dev_warn(pdsc->dev, "Failed to register devlink params: %pe\n",
 273                          ERR_PTR(err));
 274                 goto err_out_stop;
 275         }
 276
 277         hr = devl_health_reporter_create(dl, &pdsc_fw_reporter_ops, 0, pdsc);
 278         if (IS_ERR(hr)) {
 279                 devl_unlock(dl);
 280                 dev_warn(pdsc->dev, "Failed to create fw reporter: %pe\n", hr);
 281                 err = PTR_ERR(hr);
 282                 goto err_out_unreg_params;
 283         }
 284         pdsc->fw_reporter = hr;
 285
 286         devl_register(dl);
 287         devl_unlock(dl);
 288
 289         /* Lastly, start the health check timer */
 290         mod_timer(&pdsc->wdtimer, round_jiffies(jiffies + pdsc->wdtimer_period));
 291
 292         return 0;
 293
 294 err_out_unreg_params:
 295         devlink_params_unregister(dl, pdsc_dl_params,
 296                                   ARRAY_SIZE(pdsc_dl_params));
 297 err_out_stop:
 298         pdsc_stop(pdsc);
 299 err_out_teardown:
 300         pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING);
 301 err_out_unmap_bars:
 302         timer_shutdown_sync(&pdsc->wdtimer);
 303         if (pdsc->wq)
 304                 destroy_workqueue(pdsc->wq);
 305         mutex_destroy(&pdsc->config_lock);
 306         mutex_destroy(&pdsc->devcmd_lock);
 307         pci_free_irq_vectors(pdsc->pdev);
 308         pdsc_unmap_bars(pdsc);
 309 err_out_release_regions:
 310         pci_release_regions(pdsc->pdev);
 311
 312         return err;
 313 }
 314
 315 static const struct devlink_ops pdsc_dl_ops = {
 316         .info_get       = pdsc_dl_info_get,
 317         .flash_update   = pdsc_dl_flash_update,
 318 };
 319
 320 static const struct devlink_ops pdsc_dl_vf_ops = {
 321 };
 322
 323 static DEFINE_IDA(pdsc_ida);
 324
 325 static int pdsc_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 326 {
 327         struct device *dev = &pdev->dev;
 328         const struct devlink_ops *ops;
 329         struct devlink *dl;
 330         struct pdsc *pdsc;
 331         bool is_pf;
 332         int err;
 333
 334         is_pf = !pdev->is_virtfn;
 335         ops = is_pf ? &pdsc_dl_ops : &pdsc_dl_vf_ops;
 336         dl = devlink_alloc(ops, sizeof(struct pdsc), dev);
 337         if (!dl)
 338                 return -ENOMEM;
 339         pdsc = devlink_priv(dl);
 340
 341         pdsc->pdev = pdev;
 342         pdsc->dev = &pdev->dev;
 343         set_bit(PDSC_S_INITING_DRIVER, &pdsc->state);
 344         pci_set_drvdata(pdev, pdsc);
 345         pdsc_debugfs_add_dev(pdsc);
 346
 347         err = ida_alloc(&pdsc_ida, GFP_KERNEL);
 348         if (err < 0) {
 349                 dev_err(pdsc->dev, "%s: id alloc failed: %pe\n",
 350                         __func__, ERR_PTR(err));
 351                 goto err_out_free_devlink;
 352         }
 353         pdsc->uid = err;
 354
 355         /* Query system for DMA addressing limitation for the device. */
 356         err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(PDS_CORE_ADDR_LEN));
 357         if (err) {
 358                 dev_err(dev, "Unable to obtain 64-bit DMA for consistent allocations, aborting: %pe\n",
 359                         ERR_PTR(err));
 360                 goto err_out_free_ida;
 361         }
 362
 363         err = pci_enable_device(pdev);
 364         if (err) {
 365                 dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err));
 366                 goto err_out_free_ida;
 367         }
 368         pci_set_master(pdev);
 369
 370         if (is_pf)
 371                 err = pdsc_init_pf(pdsc);
 372         else
 373                 err = pdsc_init_vf(pdsc);
 374         if (err) {
 375                 dev_err(dev, "Cannot init device: %pe\n", ERR_PTR(err));
 376                 goto err_out_disable_device;
 377         }
 378
 379         clear_bit(PDSC_S_INITING_DRIVER, &pdsc->state);
 380         return 0;
 381
 382 err_out_disable_device:
 383         pci_disable_device(pdev);
 384 err_out_free_ida:
 385         ida_free(&pdsc_ida, pdsc->uid);
 386 err_out_free_devlink:
 387         pdsc_debugfs_del_dev(pdsc);
 388         devlink_free(dl);
 389
 390         return err;
 391 }
 392
 393 static void pdsc_remove(struct pci_dev *pdev)
 394 {
 395         struct pdsc *pdsc = pci_get_drvdata(pdev);
 396         struct devlink *dl;
 397
 398         /* Unhook the registrations first to be sure there
 399          * are no requests while we're stopping.
 400          */
 401         dl = priv_to_devlink(pdsc);
 402         devl_lock(dl);
 403         devl_unregister(dl);
 404         if (!pdev->is_virtfn) {
 405                 if (pdsc->fw_reporter) {
 406                         devl_health_reporter_destroy(pdsc->fw_reporter);
 407                         pdsc->fw_reporter = NULL;
 408                 }
 409                 devl_params_unregister(dl, pdsc_dl_params,
 410                                        ARRAY_SIZE(pdsc_dl_params));
 411         }
 412         devl_unlock(dl);
 413
 414         if (pdev->is_virtfn) {
 415                 struct pdsc *pf;
 416
 417                 pf = pdsc_get_pf_struct(pdsc->pdev);
 418                 if (!IS_ERR(pf)) {
 419                         pdsc_auxbus_dev_del(pdsc, pf);
 420                         pf->vfs[pdsc->vf_id].vf = NULL;
 421                 }
 422         } else {
 423                 /* Remove the VFs and their aux_bus connections before other
 424                  * cleanup so that the clients can use the AdminQ to cleanly
 425                  * shut themselves down.
 426                  */
 427                 pdsc_sriov_configure(pdev, 0);
 428
 429                 timer_shutdown_sync(&pdsc->wdtimer);
 430                 if (pdsc->wq)
 431                         destroy_workqueue(pdsc->wq);
 432
 433                 mutex_lock(&pdsc->config_lock);
 434                 set_bit(PDSC_S_STOPPING_DRIVER, &pdsc->state);
 435
 436                 pdsc_stop(pdsc);
 437                 pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING);
 438                 mutex_unlock(&pdsc->config_lock);
 439                 mutex_destroy(&pdsc->config_lock);
 440                 mutex_destroy(&pdsc->devcmd_lock);
 441
 442                 pdsc_unmap_bars(pdsc);
 443                 pci_release_regions(pdev);
 444         }
 445
 446         pci_disable_device(pdev);
 447
 448         ida_free(&pdsc_ida, pdsc->uid);
 449         pdsc_debugfs_del_dev(pdsc);
 450         devlink_free(dl);
 451 }
 452
 453 static void pdsc_stop_health_thread(struct pdsc *pdsc)
 454 {
 455         if (pdsc->pdev->is_virtfn)
 456                 return;
 457
 458         timer_shutdown_sync(&pdsc->wdtimer);
 459         if (pdsc->health_work.func)
 460                 cancel_work_sync(&pdsc->health_work);
 461 }
 462
 463 static void pdsc_restart_health_thread(struct pdsc *pdsc)
 464 {
 465         if (pdsc->pdev->is_virtfn)
 466                 return;
 467
 468         timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0);
 469         mod_timer(&pdsc->wdtimer, jiffies + 1);
 470 }
 471
 472 static void pdsc_reset_prepare(struct pci_dev *pdev)
 473 {
 474         struct pdsc *pdsc = pci_get_drvdata(pdev);
 475
 476         pdsc_stop_health_thread(pdsc);
 477         pdsc_fw_down(pdsc);
 478
 479         pdsc_unmap_bars(pdsc);
 480         pci_release_regions(pdev);
 481         if (pci_is_enabled(pdev))
 482                 pci_disable_device(pdev);
 483 }
 484
 485 static void pdsc_reset_done(struct pci_dev *pdev)
 486 {
 487         struct pdsc *pdsc = pci_get_drvdata(pdev);
 488         struct device *dev = pdsc->dev;
 489         int err;
 490
 491         err = pci_enable_device(pdev);
 492         if (err) {
 493                 dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err));
 494                 return;
 495         }
 496         pci_set_master(pdev);
 497
 498         if (!pdev->is_virtfn) {
 499                 pcie_print_link_status(pdsc->pdev);
 500
 501                 err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME);
 502                 if (err) {
 503                         dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n",
 504                                 ERR_PTR(err));
 505                         return;
 506                 }
 507
 508                 err = pdsc_map_bars(pdsc);
 509                 if (err)
 510                         return;
 511         }
 512
 513         pdsc_fw_up(pdsc);
 514         pdsc_restart_health_thread(pdsc);
 515 }
 516
 517 static const struct pci_error_handlers pdsc_err_handler = {
 518         /* FLR handling */
 519         .reset_prepare      = pdsc_reset_prepare,
 520         .reset_done         = pdsc_reset_done,
 521 };
 522
 523 static struct pci_driver pdsc_driver = {
 524         .name = PDS_CORE_DRV_NAME,
 525         .id_table = pdsc_id_table,
 526         .probe = pdsc_probe,
 527         .remove = pdsc_remove,
 528         .sriov_configure = pdsc_sriov_configure,
 529         .err_handler = &pdsc_err_handler,
 530 };
 531
 532 void *pdsc_get_pf_struct(struct pci_dev *vf_pdev)
 533 {
 534         return pci_iov_get_pf_drvdata(vf_pdev, &pdsc_driver);
 535 }
 536 EXPORT_SYMBOL_GPL(pdsc_get_pf_struct);
 537
 538 static int __init pdsc_init_module(void)
 539 {
 540         if (strcmp(KBUILD_MODNAME, PDS_CORE_DRV_NAME))
 541                 return -EINVAL;
 542
 543         pdsc_debugfs_create();
 544         return pci_register_driver(&pdsc_driver);
 545 }
 546
 547 static void __exit pdsc_cleanup_module(void)
 548 {
 549         pci_unregister_driver(&pdsc_driver);
 550         pdsc_debugfs_destroy();
 551 }
 552
 553 module_init(pdsc_init_module);
 554 module_exit(pdsc_cleanup_module);