drivers/net/wwan/t7xx/t7xx_pci.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2021, MediaTek Inc.
   4  * Copyright (c) 2021-2022, Intel Corporation.
   5  *
   6  * Authors:
   7  *  Haijun Liu <haijun.liu@mediatek.com>
   8  *  Ricardo Martinez <ricardo.martinez@linux.intel.com>
   9  *  Sreehari Kancharla <sreehari.kancharla@intel.com>
  10  *
  11  * Contributors:
  12  *  Amir Hanania <amir.hanania@intel.com>
  13  *  Andy Shevchenko <andriy.shevchenko@linux.intel.com>
  14  *  Chiranjeevi Rapolu <chiranjeevi.rapolu@intel.com>
  15  *  Eliot Lee <eliot.lee@intel.com>
  16  *  Moises Veleta <moises.veleta@intel.com>
  17  */
  18
  19 #include <linux/atomic.h>
  20 #include <linux/bits.h>
  21 #include <linux/completion.h>
  22 #include <linux/device.h>
  23 #include <linux/dma-mapping.h>
  24 #include <linux/gfp.h>
  25 #include <linux/interrupt.h>
  26 #include <linux/io.h>
  27 #include <linux/iopoll.h>
  28 #include <linux/jiffies.h>
  29 #include <linux/list.h>
  30 #include <linux/module.h>
  31 #include <linux/mutex.h>
  32 #include <linux/pci.h>
  33 #include <linux/pm.h>
  34 #include <linux/pm_runtime.h>
  35 #include <linux/pm_wakeup.h>
  36 #include <linux/spinlock.h>
  37
  38 #include "t7xx_mhccif.h"
  39 #include "t7xx_modem_ops.h"
  40 #include "t7xx_pci.h"
  41 #include "t7xx_pcie_mac.h"
  42 #include "t7xx_reg.h"
  43 #include "t7xx_state_monitor.h"
  44
  45 #define T7XX_PCI_IREG_BASE              0
  46 #define T7XX_PCI_EREG_BASE              2
  47
  48 #define T7XX_INIT_TIMEOUT               20
  49 #define PM_SLEEP_DIS_TIMEOUT_MS         20
  50 #define PM_ACK_TIMEOUT_MS               1500
  51 #define PM_AUTOSUSPEND_MS               20000
  52 #define PM_RESOURCE_POLL_TIMEOUT_US     10000
  53 #define PM_RESOURCE_POLL_STEP_US        100
  54
  55 enum t7xx_pm_state {
  56         MTK_PM_EXCEPTION,
  57         MTK_PM_INIT,            /* Device initialized, but handshake not completed */
  58         MTK_PM_SUSPENDED,
  59         MTK_PM_RESUMED,
  60 };
  61
  62 static void t7xx_dev_set_sleep_capability(struct t7xx_pci_dev *t7xx_dev, bool enable)
  63 {
  64         void __iomem *ctrl_reg = IREG_BASE(t7xx_dev) + T7XX_PCIE_MISC_CTRL;
  65         u32 value;
  66
  67         value = ioread32(ctrl_reg);
  68
  69         if (enable)
  70                 value &= ~T7XX_PCIE_MISC_MAC_SLEEP_DIS;
  71         else
  72                 value |= T7XX_PCIE_MISC_MAC_SLEEP_DIS;
  73
  74         iowrite32(value, ctrl_reg);
  75 }
  76
  77 static int t7xx_wait_pm_config(struct t7xx_pci_dev *t7xx_dev)
  78 {
  79         int ret, val;
  80
  81         ret = read_poll_timeout(ioread32, val,
  82                                 (val & T7XX_PCIE_RESOURCE_STS_MSK) == T7XX_PCIE_RESOURCE_STS_MSK,
  83                                 PM_RESOURCE_POLL_STEP_US, PM_RESOURCE_POLL_TIMEOUT_US, true,
  84                                 IREG_BASE(t7xx_dev) + T7XX_PCIE_RESOURCE_STATUS);
  85         if (ret == -ETIMEDOUT)
  86                 dev_err(&t7xx_dev->pdev->dev, "PM configuration timed out\n");
  87
  88         return ret;
  89 }
  90
  91 static int t7xx_pci_pm_init(struct t7xx_pci_dev *t7xx_dev)
  92 {
  93         struct pci_dev *pdev = t7xx_dev->pdev;
  94
  95         INIT_LIST_HEAD(&t7xx_dev->md_pm_entities);
  96         mutex_init(&t7xx_dev->md_pm_entity_mtx);
  97         spin_lock_init(&t7xx_dev->md_pm_lock);
  98         init_completion(&t7xx_dev->sleep_lock_acquire);
  99         init_completion(&t7xx_dev->pm_sr_ack);
 100         init_completion(&t7xx_dev->init_done);
 101         atomic_set(&t7xx_dev->md_pm_state, MTK_PM_INIT);
 102
 103         device_init_wakeup(&pdev->dev, true);
 104         dev_pm_set_driver_flags(&pdev->dev, pdev->dev.power.driver_flags |
 105                                 DPM_FLAG_NO_DIRECT_COMPLETE);
 106
 107         iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR);
 108         pm_runtime_set_autosuspend_delay(&pdev->dev, PM_AUTOSUSPEND_MS);
 109         pm_runtime_use_autosuspend(&pdev->dev);
 110
 111         return t7xx_wait_pm_config(t7xx_dev);
 112 }
 113
 114 void t7xx_pci_pm_init_late(struct t7xx_pci_dev *t7xx_dev)
 115 {
 116         /* Enable the PCIe resource lock only after MD deep sleep is done */
 117         t7xx_mhccif_mask_clr(t7xx_dev,
 118                              D2H_INT_DS_LOCK_ACK |
 119                              D2H_INT_SUSPEND_ACK |
 120                              D2H_INT_RESUME_ACK |
 121                              D2H_INT_SUSPEND_ACK_AP |
 122                              D2H_INT_RESUME_ACK_AP);
 123         iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR);
 124         atomic_set(&t7xx_dev->md_pm_state, MTK_PM_RESUMED);
 125
 126         pm_runtime_mark_last_busy(&t7xx_dev->pdev->dev);
 127         pm_runtime_allow(&t7xx_dev->pdev->dev);
 128         pm_runtime_put_noidle(&t7xx_dev->pdev->dev);
 129         complete_all(&t7xx_dev->init_done);
 130 }
 131
 132 static int t7xx_pci_pm_reinit(struct t7xx_pci_dev *t7xx_dev)
 133 {
 134         /* The device is kept in FSM re-init flow
 135          * so just roll back PM setting to the init setting.
 136          */
 137         atomic_set(&t7xx_dev->md_pm_state, MTK_PM_INIT);
 138
 139         pm_runtime_get_noresume(&t7xx_dev->pdev->dev);
 140
 141         iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR);
 142         return t7xx_wait_pm_config(t7xx_dev);
 143 }
 144
 145 void t7xx_pci_pm_exp_detected(struct t7xx_pci_dev *t7xx_dev)
 146 {
 147         iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR);
 148         t7xx_wait_pm_config(t7xx_dev);
 149         atomic_set(&t7xx_dev->md_pm_state, MTK_PM_EXCEPTION);
 150 }
 151
 152 int t7xx_pci_pm_entity_register(struct t7xx_pci_dev *t7xx_dev, struct md_pm_entity *pm_entity)
 153 {
 154         struct md_pm_entity *entity;
 155
 156         mutex_lock(&t7xx_dev->md_pm_entity_mtx);
 157         list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) {
 158                 if (entity->id == pm_entity->id) {
 159                         mutex_unlock(&t7xx_dev->md_pm_entity_mtx);
 160                         return -EEXIST;
 161                 }
 162         }
 163
 164         list_add_tail(&pm_entity->entity, &t7xx_dev->md_pm_entities);
 165         mutex_unlock(&t7xx_dev->md_pm_entity_mtx);
 166         return 0;
 167 }
 168
 169 int t7xx_pci_pm_entity_unregister(struct t7xx_pci_dev *t7xx_dev, struct md_pm_entity *pm_entity)
 170 {
 171         struct md_pm_entity *entity, *tmp_entity;
 172
 173         mutex_lock(&t7xx_dev->md_pm_entity_mtx);
 174         list_for_each_entry_safe(entity, tmp_entity, &t7xx_dev->md_pm_entities, entity) {
 175                 if (entity->id == pm_entity->id) {
 176                         list_del(&pm_entity->entity);
 177                         mutex_unlock(&t7xx_dev->md_pm_entity_mtx);
 178                         return 0;
 179                 }
 180         }
 181
 182         mutex_unlock(&t7xx_dev->md_pm_entity_mtx);
 183
 184         return -ENXIO;
 185 }
 186
 187 int t7xx_pci_sleep_disable_complete(struct t7xx_pci_dev *t7xx_dev)
 188 {
 189         struct device *dev = &t7xx_dev->pdev->dev;
 190         int ret;
 191
 192         ret = wait_for_completion_timeout(&t7xx_dev->sleep_lock_acquire,
 193                                           msecs_to_jiffies(PM_SLEEP_DIS_TIMEOUT_MS));
 194         if (!ret)
 195                 dev_err_ratelimited(dev, "Resource wait complete timed out\n");
 196
 197         return ret;
 198 }
 199
 200 /**
 201  * t7xx_pci_disable_sleep() - Disable deep sleep capability.
 202  * @t7xx_dev: MTK device.
 203  *
 204  * Lock the deep sleep capability, note that the device can still go into deep sleep
 205  * state while device is in D0 state, from the host's point-of-view.
 206  *
 207  * If device is in deep sleep state, wake up the device and disable deep sleep capability.
 208  */
 209 void t7xx_pci_disable_sleep(struct t7xx_pci_dev *t7xx_dev)
 210 {
 211         unsigned long flags;
 212
 213         spin_lock_irqsave(&t7xx_dev->md_pm_lock, flags);
 214         t7xx_dev->sleep_disable_count++;
 215         if (atomic_read(&t7xx_dev->md_pm_state) < MTK_PM_RESUMED)
 216                 goto unlock_and_complete;
 217
 218         if (t7xx_dev->sleep_disable_count == 1) {
 219                 u32 status;
 220
 221                 reinit_completion(&t7xx_dev->sleep_lock_acquire);
 222                 t7xx_dev_set_sleep_capability(t7xx_dev, false);
 223
 224                 status = ioread32(IREG_BASE(t7xx_dev) + T7XX_PCIE_RESOURCE_STATUS);
 225                 if (status & T7XX_PCIE_RESOURCE_STS_MSK)
 226                         goto unlock_and_complete;
 227
 228                 t7xx_mhccif_h2d_swint_trigger(t7xx_dev, H2D_CH_DS_LOCK);
 229         }
 230         spin_unlock_irqrestore(&t7xx_dev->md_pm_lock, flags);
 231         return;
 232
 233 unlock_and_complete:
 234         spin_unlock_irqrestore(&t7xx_dev->md_pm_lock, flags);
 235         complete_all(&t7xx_dev->sleep_lock_acquire);
 236 }
 237
 238 /**
 239  * t7xx_pci_enable_sleep() - Enable deep sleep capability.
 240  * @t7xx_dev: MTK device.
 241  *
 242  * After enabling deep sleep, device can enter into deep sleep state.
 243  */
 244 void t7xx_pci_enable_sleep(struct t7xx_pci_dev *t7xx_dev)
 245 {
 246         unsigned long flags;
 247
 248         spin_lock_irqsave(&t7xx_dev->md_pm_lock, flags);
 249         t7xx_dev->sleep_disable_count--;
 250         if (atomic_read(&t7xx_dev->md_pm_state) < MTK_PM_RESUMED)
 251                 goto unlock;
 252
 253         if (t7xx_dev->sleep_disable_count == 0)
 254                 t7xx_dev_set_sleep_capability(t7xx_dev, true);
 255
 256 unlock:
 257         spin_unlock_irqrestore(&t7xx_dev->md_pm_lock, flags);
 258 }
 259
 260 static int t7xx_send_pm_request(struct t7xx_pci_dev *t7xx_dev, u32 request)
 261 {
 262         unsigned long wait_ret;
 263
 264         reinit_completion(&t7xx_dev->pm_sr_ack);
 265         t7xx_mhccif_h2d_swint_trigger(t7xx_dev, request);
 266         wait_ret = wait_for_completion_timeout(&t7xx_dev->pm_sr_ack,
 267                                                msecs_to_jiffies(PM_ACK_TIMEOUT_MS));
 268         if (!wait_ret)
 269                 return -ETIMEDOUT;
 270
 271         return 0;
 272 }
 273
 274 static int __t7xx_pci_pm_suspend(struct pci_dev *pdev)
 275 {
 276         enum t7xx_pm_id entity_id = PM_ENTITY_ID_INVALID;
 277         struct t7xx_pci_dev *t7xx_dev;
 278         struct md_pm_entity *entity;
 279         int ret;
 280
 281         t7xx_dev = pci_get_drvdata(pdev);
 282         if (atomic_read(&t7xx_dev->md_pm_state) <= MTK_PM_INIT) {
 283                 dev_err(&pdev->dev, "[PM] Exiting suspend, modem in invalid state\n");
 284                 return -EFAULT;
 285         }
 286
 287         iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR);
 288         ret = t7xx_wait_pm_config(t7xx_dev);
 289         if (ret) {
 290                 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR);
 291                 return ret;
 292         }
 293
 294         atomic_set(&t7xx_dev->md_pm_state, MTK_PM_SUSPENDED);
 295         t7xx_pcie_mac_clear_int(t7xx_dev, SAP_RGU_INT);
 296         t7xx_dev->rgu_pci_irq_en = false;
 297
 298         list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) {
 299                 if (!entity->suspend)
 300                         continue;
 301
 302                 ret = entity->suspend(t7xx_dev, entity->entity_param);
 303                 if (ret) {
 304                         entity_id = entity->id;
 305                         dev_err(&pdev->dev, "[PM] Suspend error: %d, id: %d\n", ret, entity_id);
 306                         goto abort_suspend;
 307                 }
 308         }
 309
 310         ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_SUSPEND_REQ);
 311         if (ret) {
 312                 dev_err(&pdev->dev, "[PM] MD suspend error: %d\n", ret);
 313                 goto abort_suspend;
 314         }
 315
 316         ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_SUSPEND_REQ_AP);
 317         if (ret) {
 318                 t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ);
 319                 dev_err(&pdev->dev, "[PM] SAP suspend error: %d\n", ret);
 320                 goto abort_suspend;
 321         }
 322
 323         list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) {
 324                 if (entity->suspend_late)
 325                         entity->suspend_late(t7xx_dev, entity->entity_param);
 326         }
 327
 328         iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR);
 329         return 0;
 330
 331 abort_suspend:
 332         list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) {
 333                 if (entity_id == entity->id)
 334                         break;
 335
 336                 if (entity->resume)
 337                         entity->resume(t7xx_dev, entity->entity_param);
 338         }
 339
 340         iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR);
 341         atomic_set(&t7xx_dev->md_pm_state, MTK_PM_RESUMED);
 342         t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT);
 343         return ret;
 344 }
 345
 346 static void t7xx_pcie_interrupt_reinit(struct t7xx_pci_dev *t7xx_dev)
 347 {
 348         t7xx_pcie_set_mac_msix_cfg(t7xx_dev, EXT_INT_NUM);
 349
 350         /* Disable interrupt first and let the IPs enable them */
 351         iowrite32(MSIX_MSK_SET_ALL, IREG_BASE(t7xx_dev) + IMASK_HOST_MSIX_CLR_GRP0_0);
 352
 353         /* Device disables PCIe interrupts during resume and
 354          * following function will re-enable PCIe interrupts.
 355          */
 356         t7xx_pcie_mac_interrupts_en(t7xx_dev);
 357         t7xx_pcie_mac_set_int(t7xx_dev, MHCCIF_INT);
 358 }
 359
 360 static int t7xx_pcie_reinit(struct t7xx_pci_dev *t7xx_dev, bool is_d3)
 361 {
 362         int ret;
 363
 364         ret = pcim_enable_device(t7xx_dev->pdev);
 365         if (ret)
 366                 return ret;
 367
 368         t7xx_pcie_mac_atr_init(t7xx_dev);
 369         t7xx_pcie_interrupt_reinit(t7xx_dev);
 370
 371         if (is_d3) {
 372                 t7xx_mhccif_init(t7xx_dev);
 373                 return t7xx_pci_pm_reinit(t7xx_dev);
 374         }
 375
 376         return 0;
 377 }
 378
 379 static int t7xx_send_fsm_command(struct t7xx_pci_dev *t7xx_dev, u32 event)
 380 {
 381         struct t7xx_fsm_ctl *fsm_ctl = t7xx_dev->md->fsm_ctl;
 382         struct device *dev = &t7xx_dev->pdev->dev;
 383         int ret = -EINVAL;
 384
 385         switch (event) {
 386         case FSM_CMD_STOP:
 387                 ret = t7xx_fsm_append_cmd(fsm_ctl, FSM_CMD_STOP, FSM_CMD_FLAG_WAIT_FOR_COMPLETION);
 388                 break;
 389
 390         case FSM_CMD_START:
 391                 t7xx_pcie_mac_clear_int(t7xx_dev, SAP_RGU_INT);
 392                 t7xx_pcie_mac_clear_int_status(t7xx_dev, SAP_RGU_INT);
 393                 t7xx_dev->rgu_pci_irq_en = true;
 394                 t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT);
 395                 ret = t7xx_fsm_append_cmd(fsm_ctl, FSM_CMD_START, 0);
 396                 break;
 397
 398         default:
 399                 break;
 400         }
 401
 402         if (ret)
 403                 dev_err(dev, "Failure handling FSM command %u, %d\n", event, ret);
 404
 405         return ret;
 406 }
 407
 408 static int __t7xx_pci_pm_resume(struct pci_dev *pdev, bool state_check)
 409 {
 410         struct t7xx_pci_dev *t7xx_dev;
 411         struct md_pm_entity *entity;
 412         u32 prev_state;
 413         int ret = 0;
 414
 415         t7xx_dev = pci_get_drvdata(pdev);
 416         if (atomic_read(&t7xx_dev->md_pm_state) <= MTK_PM_INIT) {
 417                 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR);
 418                 return 0;
 419         }
 420
 421         t7xx_pcie_mac_interrupts_en(t7xx_dev);
 422         prev_state = ioread32(IREG_BASE(t7xx_dev) + T7XX_PCIE_PM_RESUME_STATE);
 423
 424         if (state_check) {
 425                 /* For D3/L3 resume, the device could boot so quickly that the
 426                  * initial value of the dummy register might be overwritten.
 427                  * Identify new boots if the ATR source address register is not initialized.
 428                  */
 429                 u32 atr_reg_val = ioread32(IREG_BASE(t7xx_dev) +
 430                                            ATR_PCIE_WIN0_T0_ATR_PARAM_SRC_ADDR);
 431                 if (prev_state == PM_RESUME_REG_STATE_L3 ||
 432                     (prev_state == PM_RESUME_REG_STATE_INIT &&
 433                      atr_reg_val == ATR_SRC_ADDR_INVALID)) {
 434                         ret = t7xx_send_fsm_command(t7xx_dev, FSM_CMD_STOP);
 435                         if (ret)
 436                                 return ret;
 437
 438                         ret = t7xx_pcie_reinit(t7xx_dev, true);
 439                         if (ret)
 440                                 return ret;
 441
 442                         t7xx_clear_rgu_irq(t7xx_dev);
 443                         return t7xx_send_fsm_command(t7xx_dev, FSM_CMD_START);
 444                 }
 445
 446                 if (prev_state == PM_RESUME_REG_STATE_EXP ||
 447                     prev_state == PM_RESUME_REG_STATE_L2_EXP) {
 448                         if (prev_state == PM_RESUME_REG_STATE_L2_EXP) {
 449                                 ret = t7xx_pcie_reinit(t7xx_dev, false);
 450                                 if (ret)
 451                                         return ret;
 452                         }
 453
 454                         atomic_set(&t7xx_dev->md_pm_state, MTK_PM_SUSPENDED);
 455                         t7xx_dev->rgu_pci_irq_en = true;
 456                         t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT);
 457
 458                         t7xx_mhccif_mask_clr(t7xx_dev,
 459                                              D2H_INT_EXCEPTION_INIT |
 460                                              D2H_INT_EXCEPTION_INIT_DONE |
 461                                              D2H_INT_EXCEPTION_CLEARQ_DONE |
 462                                              D2H_INT_EXCEPTION_ALLQ_RESET |
 463                                              D2H_INT_PORT_ENUM);
 464
 465                         return ret;
 466                 }
 467
 468                 if (prev_state == PM_RESUME_REG_STATE_L2) {
 469                         ret = t7xx_pcie_reinit(t7xx_dev, false);
 470                         if (ret)
 471                                 return ret;
 472
 473                 } else if (prev_state != PM_RESUME_REG_STATE_L1 &&
 474                            prev_state != PM_RESUME_REG_STATE_INIT) {
 475                         ret = t7xx_send_fsm_command(t7xx_dev, FSM_CMD_STOP);
 476                         if (ret)
 477                                 return ret;
 478
 479                         t7xx_clear_rgu_irq(t7xx_dev);
 480                         atomic_set(&t7xx_dev->md_pm_state, MTK_PM_SUSPENDED);
 481                         return 0;
 482                 }
 483         }
 484
 485         iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR);
 486         t7xx_wait_pm_config(t7xx_dev);
 487
 488         list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) {
 489                 if (entity->resume_early)
 490                         entity->resume_early(t7xx_dev, entity->entity_param);
 491         }
 492
 493         ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ);
 494         if (ret)
 495                 dev_err(&pdev->dev, "[PM] MD resume error: %d\n", ret);
 496
 497         ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ_AP);
 498         if (ret)
 499                 dev_err(&pdev->dev, "[PM] SAP resume error: %d\n", ret);
 500
 501         list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) {
 502                 if (entity->resume) {
 503                         ret = entity->resume(t7xx_dev, entity->entity_param);
 504                         if (ret)
 505                                 dev_err(&pdev->dev, "[PM] Resume entry ID: %d error: %d\n",
 506                                         entity->id, ret);
 507                 }
 508         }
 509
 510         t7xx_dev->rgu_pci_irq_en = true;
 511         t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT);
 512         iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR);
 513         pm_runtime_mark_last_busy(&pdev->dev);
 514         atomic_set(&t7xx_dev->md_pm_state, MTK_PM_RESUMED);
 515
 516         return ret;
 517 }
 518
 519 static int t7xx_pci_pm_resume_noirq(struct device *dev)
 520 {
 521         struct pci_dev *pdev = to_pci_dev(dev);
 522         struct t7xx_pci_dev *t7xx_dev;
 523
 524         t7xx_dev = pci_get_drvdata(pdev);
 525         t7xx_pcie_mac_interrupts_dis(t7xx_dev);
 526
 527         return 0;
 528 }
 529
 530 static void t7xx_pci_shutdown(struct pci_dev *pdev)
 531 {
 532         __t7xx_pci_pm_suspend(pdev);
 533 }
 534
 535 static int t7xx_pci_pm_prepare(struct device *dev)
 536 {
 537         struct pci_dev *pdev = to_pci_dev(dev);
 538         struct t7xx_pci_dev *t7xx_dev;
 539
 540         t7xx_dev = pci_get_drvdata(pdev);
 541         if (!wait_for_completion_timeout(&t7xx_dev->init_done, T7XX_INIT_TIMEOUT * HZ)) {
 542                 dev_warn(dev, "Not ready for system sleep.\n");
 543                 return -ETIMEDOUT;
 544         }
 545
 546         return 0;
 547 }
 548
 549 static int t7xx_pci_pm_suspend(struct device *dev)
 550 {
 551         return __t7xx_pci_pm_suspend(to_pci_dev(dev));
 552 }
 553
 554 static int t7xx_pci_pm_resume(struct device *dev)
 555 {
 556         return __t7xx_pci_pm_resume(to_pci_dev(dev), true);
 557 }
 558
 559 static int t7xx_pci_pm_thaw(struct device *dev)
 560 {
 561         return __t7xx_pci_pm_resume(to_pci_dev(dev), false);
 562 }
 563
 564 static int t7xx_pci_pm_runtime_suspend(struct device *dev)
 565 {
 566         return __t7xx_pci_pm_suspend(to_pci_dev(dev));
 567 }
 568
 569 static int t7xx_pci_pm_runtime_resume(struct device *dev)
 570 {
 571         return __t7xx_pci_pm_resume(to_pci_dev(dev), true);
 572 }
 573
 574 static const struct dev_pm_ops t7xx_pci_pm_ops = {
 575         .prepare = t7xx_pci_pm_prepare,
 576         .suspend = t7xx_pci_pm_suspend,
 577         .resume = t7xx_pci_pm_resume,
 578         .resume_noirq = t7xx_pci_pm_resume_noirq,
 579         .freeze = t7xx_pci_pm_suspend,
 580         .thaw = t7xx_pci_pm_thaw,
 581         .poweroff = t7xx_pci_pm_suspend,
 582         .restore = t7xx_pci_pm_resume,
 583         .restore_noirq = t7xx_pci_pm_resume_noirq,
 584         .runtime_suspend = t7xx_pci_pm_runtime_suspend,
 585         .runtime_resume = t7xx_pci_pm_runtime_resume
 586 };
 587
 588 static int t7xx_request_irq(struct pci_dev *pdev)
 589 {
 590         struct t7xx_pci_dev *t7xx_dev;
 591         int ret = 0, i;
 592
 593         t7xx_dev = pci_get_drvdata(pdev);
 594
 595         for (i = 0; i < EXT_INT_NUM; i++) {
 596                 const char *irq_descr;
 597                 int irq_vec;
 598
 599                 if (!t7xx_dev->intr_handler[i])
 600                         continue;
 601
 602                 irq_descr = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_%d",
 603                                            dev_driver_string(&pdev->dev), i);
 604                 if (!irq_descr) {
 605                         ret = -ENOMEM;
 606                         break;
 607                 }
 608
 609                 irq_vec = pci_irq_vector(pdev, i);
 610                 ret = request_threaded_irq(irq_vec, t7xx_dev->intr_handler[i],
 611                                            t7xx_dev->intr_thread[i], 0, irq_descr,
 612                                            t7xx_dev->callback_param[i]);
 613                 if (ret) {
 614                         dev_err(&pdev->dev, "Failed to request IRQ: %d\n", ret);
 615                         break;
 616                 }
 617         }
 618
 619         if (ret) {
 620                 while (i--) {
 621                         if (!t7xx_dev->intr_handler[i])
 622                                 continue;
 623
 624                         free_irq(pci_irq_vector(pdev, i), t7xx_dev->callback_param[i]);
 625                 }
 626         }
 627
 628         return ret;
 629 }
 630
 631 static int t7xx_setup_msix(struct t7xx_pci_dev *t7xx_dev)
 632 {
 633         struct pci_dev *pdev = t7xx_dev->pdev;
 634         int ret;
 635
 636         /* Only using 6 interrupts, but HW-design requires power-of-2 IRQs allocation */
 637         ret = pci_alloc_irq_vectors(pdev, EXT_INT_NUM, EXT_INT_NUM, PCI_IRQ_MSIX);
 638         if (ret < 0) {
 639                 dev_err(&pdev->dev, "Failed to allocate MSI-X entry: %d\n", ret);
 640                 return ret;
 641         }
 642
 643         ret = t7xx_request_irq(pdev);
 644         if (ret) {
 645                 pci_free_irq_vectors(pdev);
 646                 return ret;
 647         }
 648
 649         t7xx_pcie_set_mac_msix_cfg(t7xx_dev, EXT_INT_NUM);
 650         return 0;
 651 }
 652
 653 static int t7xx_interrupt_init(struct t7xx_pci_dev *t7xx_dev)
 654 {
 655         int ret, i;
 656
 657         if (!t7xx_dev->pdev->msix_cap)
 658                 return -EINVAL;
 659
 660         ret = t7xx_setup_msix(t7xx_dev);
 661         if (ret)
 662                 return ret;
 663
 664         /* IPs enable interrupts when ready */
 665         for (i = 0; i < EXT_INT_NUM; i++)
 666                 t7xx_pcie_mac_set_int(t7xx_dev, i);
 667
 668         return 0;
 669 }
 670
 671 static void t7xx_pci_infracfg_ao_calc(struct t7xx_pci_dev *t7xx_dev)
 672 {
 673         t7xx_dev->base_addr.infracfg_ao_base = t7xx_dev->base_addr.pcie_ext_reg_base +
 674                                               INFRACFG_AO_DEV_CHIP -
 675                                               t7xx_dev->base_addr.pcie_dev_reg_trsl_addr;
 676 }
 677
 678 static int t7xx_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 679 {
 680         struct t7xx_pci_dev *t7xx_dev;
 681         int ret;
 682
 683         t7xx_dev = devm_kzalloc(&pdev->dev, sizeof(*t7xx_dev), GFP_KERNEL);
 684         if (!t7xx_dev)
 685                 return -ENOMEM;
 686
 687         pci_set_drvdata(pdev, t7xx_dev);
 688         t7xx_dev->pdev = pdev;
 689
 690         ret = pcim_enable_device(pdev);
 691         if (ret)
 692                 return ret;
 693
 694         pci_set_master(pdev);
 695
 696         ret = pcim_iomap_regions(pdev, BIT(T7XX_PCI_IREG_BASE) | BIT(T7XX_PCI_EREG_BASE),
 697                                  pci_name(pdev));
 698         if (ret) {
 699                 dev_err(&pdev->dev, "Could not request BARs: %d\n", ret);
 700                 return -ENOMEM;
 701         }
 702
 703         ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
 704         if (ret) {
 705                 dev_err(&pdev->dev, "Could not set PCI DMA mask: %d\n", ret);
 706                 return ret;
 707         }
 708
 709         ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
 710         if (ret) {
 711                 dev_err(&pdev->dev, "Could not set consistent PCI DMA mask: %d\n", ret);
 712                 return ret;
 713         }
 714
 715         IREG_BASE(t7xx_dev) = pcim_iomap_table(pdev)[T7XX_PCI_IREG_BASE];
 716         t7xx_dev->base_addr.pcie_ext_reg_base = pcim_iomap_table(pdev)[T7XX_PCI_EREG_BASE];
 717
 718         ret = t7xx_pci_pm_init(t7xx_dev);
 719         if (ret)
 720                 return ret;
 721
 722         t7xx_pcie_mac_atr_init(t7xx_dev);
 723         t7xx_pci_infracfg_ao_calc(t7xx_dev);
 724         t7xx_mhccif_init(t7xx_dev);
 725
 726         ret = t7xx_md_init(t7xx_dev);
 727         if (ret)
 728                 return ret;
 729
 730         t7xx_pcie_mac_interrupts_dis(t7xx_dev);
 731
 732         ret = t7xx_interrupt_init(t7xx_dev);
 733         if (ret) {
 734                 t7xx_md_exit(t7xx_dev);
 735                 return ret;
 736         }
 737
 738         t7xx_pcie_mac_set_int(t7xx_dev, MHCCIF_INT);
 739         t7xx_pcie_mac_interrupts_en(t7xx_dev);
 740
 741         return 0;
 742 }
 743
 744 static void t7xx_pci_remove(struct pci_dev *pdev)
 745 {
 746         struct t7xx_pci_dev *t7xx_dev;
 747         int i;
 748
 749         t7xx_dev = pci_get_drvdata(pdev);
 750         t7xx_md_exit(t7xx_dev);
 751
 752         for (i = 0; i < EXT_INT_NUM; i++) {
 753                 if (!t7xx_dev->intr_handler[i])
 754                         continue;
 755
 756                 free_irq(pci_irq_vector(pdev, i), t7xx_dev->callback_param[i]);
 757         }
 758
 759         pci_free_irq_vectors(t7xx_dev->pdev);
 760 }
 761
 762 static const struct pci_device_id t7xx_pci_table[] = {
 763         { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x4d75) },
 764         { }
 765 };
 766 MODULE_DEVICE_TABLE(pci, t7xx_pci_table);
 767
 768 static struct pci_driver t7xx_pci_driver = {
 769         .name = "mtk_t7xx",
 770         .id_table = t7xx_pci_table,
 771         .probe = t7xx_pci_probe,
 772         .remove = t7xx_pci_remove,
 773         .driver.pm = &t7xx_pci_pm_ops,
 774         .shutdown = t7xx_pci_shutdown,
 775 };
 776
 777 module_pci_driver(t7xx_pci_driver);
 778
 779 MODULE_AUTHOR("MediaTek Inc");
 780 MODULE_DESCRIPTION("MediaTek PCIe 5G WWAN modem T7xx driver");
 781 MODULE_LICENSE("GPL");