drivers/bus/mhi/core/pm.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved.
   4  *
   5  */
   6
   7 #include <linux/delay.h>
   8 #include <linux/device.h>
   9 #include <linux/dma-direction.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/list.h>
  13 #include <linux/mhi.h>
  14 #include <linux/module.h>
  15 #include <linux/slab.h>
  16 #include <linux/wait.h>
  17 #include "internal.h"
  18
  19 /*
  20  * Not all MHI state transitions are synchronous. Transitions like Linkdown,
  21  * SYS_ERR, and shutdown can happen anytime asynchronously. This function will
  22  * transition to a new state only if we're allowed to.
  23  *
  24  * Priority increases as we go down. For instance, from any state in L0, the
  25  * transition can be made to states in L1, L2 and L3. A notable exception to
  26  * this rule is state DISABLE.  From DISABLE state we can only transition to
  27  * POR state. Also, while in L2 state, user cannot jump back to previous
  28  * L1 or L0 states.
  29  *
  30  * Valid transitions:
  31  * L0: DISABLE <--> POR
  32  *     POR <--> POR
  33  *     POR -> M0 -> M2 --> M0
  34  *     POR -> FW_DL_ERR
  35  *     FW_DL_ERR <--> FW_DL_ERR
  36  *     M0 <--> M0
  37  *     M0 -> FW_DL_ERR
  38  *     M0 -> M3_ENTER -> M3 -> M3_EXIT --> M0
  39  * L1: SYS_ERR_DETECT -> SYS_ERR_PROCESS --> POR
  40  * L2: SHUTDOWN_PROCESS -> DISABLE
  41  * L3: LD_ERR_FATAL_DETECT <--> LD_ERR_FATAL_DETECT
  42  *     LD_ERR_FATAL_DETECT -> SHUTDOWN_PROCESS
  43  */
  44 static struct mhi_pm_transitions const dev_state_transitions[] = {
  45         /* L0 States */
  46         {
  47                 MHI_PM_DISABLE,
  48                 MHI_PM_POR
  49         },
  50         {
  51                 MHI_PM_POR,
  52                 MHI_PM_POR | MHI_PM_DISABLE | MHI_PM_M0 |
  53                 MHI_PM_SYS_ERR_DETECT | MHI_PM_SHUTDOWN_PROCESS |
  54                 MHI_PM_LD_ERR_FATAL_DETECT | MHI_PM_FW_DL_ERR
  55         },
  56         {
  57                 MHI_PM_M0,
  58                 MHI_PM_M0 | MHI_PM_M2 | MHI_PM_M3_ENTER |
  59                 MHI_PM_SYS_ERR_DETECT | MHI_PM_SHUTDOWN_PROCESS |
  60                 MHI_PM_LD_ERR_FATAL_DETECT | MHI_PM_FW_DL_ERR
  61         },
  62         {
  63                 MHI_PM_M2,
  64                 MHI_PM_M0 | MHI_PM_SYS_ERR_DETECT | MHI_PM_SHUTDOWN_PROCESS |
  65                 MHI_PM_LD_ERR_FATAL_DETECT
  66         },
  67         {
  68                 MHI_PM_M3_ENTER,
  69                 MHI_PM_M3 | MHI_PM_SYS_ERR_DETECT | MHI_PM_SHUTDOWN_PROCESS |
  70                 MHI_PM_LD_ERR_FATAL_DETECT
  71         },
  72         {
  73                 MHI_PM_M3,
  74                 MHI_PM_M3_EXIT | MHI_PM_SYS_ERR_DETECT |
  75                 MHI_PM_SHUTDOWN_PROCESS | MHI_PM_LD_ERR_FATAL_DETECT
  76         },
  77         {
  78                 MHI_PM_M3_EXIT,
  79                 MHI_PM_M0 | MHI_PM_SYS_ERR_DETECT | MHI_PM_SHUTDOWN_PROCESS |
  80                 MHI_PM_LD_ERR_FATAL_DETECT
  81         },
  82         {
  83                 MHI_PM_FW_DL_ERR,
  84                 MHI_PM_FW_DL_ERR | MHI_PM_SYS_ERR_DETECT |
  85                 MHI_PM_SHUTDOWN_PROCESS | MHI_PM_LD_ERR_FATAL_DETECT
  86         },
  87         /* L1 States */
  88         {
  89                 MHI_PM_SYS_ERR_DETECT,
  90                 MHI_PM_SYS_ERR_PROCESS | MHI_PM_SHUTDOWN_PROCESS |
  91                 MHI_PM_LD_ERR_FATAL_DETECT
  92         },
  93         {
  94                 MHI_PM_SYS_ERR_PROCESS,
  95                 MHI_PM_POR | MHI_PM_SHUTDOWN_PROCESS |
  96                 MHI_PM_LD_ERR_FATAL_DETECT
  97         },
  98         /* L2 States */
  99         {
 100                 MHI_PM_SHUTDOWN_PROCESS,
 101                 MHI_PM_DISABLE | MHI_PM_LD_ERR_FATAL_DETECT
 102         },
 103         /* L3 States */
 104         {
 105                 MHI_PM_LD_ERR_FATAL_DETECT,
 106                 MHI_PM_LD_ERR_FATAL_DETECT | MHI_PM_SHUTDOWN_PROCESS
 107         },
 108 };
 109
 110 enum mhi_pm_state __must_check mhi_tryset_pm_state(struct mhi_controller *mhi_cntrl,
 111                                                    enum mhi_pm_state state)
 112 {
 113         unsigned long cur_state = mhi_cntrl->pm_state;
 114         int index = find_last_bit(&cur_state, 32);
 115
 116         if (unlikely(index >= ARRAY_SIZE(dev_state_transitions)))
 117                 return cur_state;
 118
 119         if (unlikely(dev_state_transitions[index].from_state != cur_state))
 120                 return cur_state;
 121
 122         if (unlikely(!(dev_state_transitions[index].to_states & state)))
 123                 return cur_state;
 124
 125         mhi_cntrl->pm_state = state;
 126         return mhi_cntrl->pm_state;
 127 }
 128
 129 void mhi_set_mhi_state(struct mhi_controller *mhi_cntrl, enum mhi_state state)
 130 {
 131         if (state == MHI_STATE_RESET) {
 132                 mhi_write_reg_field(mhi_cntrl, mhi_cntrl->regs, MHICTRL,
 133                                     MHICTRL_RESET_MASK, MHICTRL_RESET_SHIFT, 1);
 134         } else {
 135                 mhi_write_reg_field(mhi_cntrl, mhi_cntrl->regs, MHICTRL,
 136                                     MHICTRL_MHISTATE_MASK,
 137                                     MHICTRL_MHISTATE_SHIFT, state);
 138         }
 139 }
 140
 141 /* NOP for backward compatibility, host allowed to ring DB in M2 state */
 142 static void mhi_toggle_dev_wake_nop(struct mhi_controller *mhi_cntrl)
 143 {
 144 }
 145
 146 static void mhi_toggle_dev_wake(struct mhi_controller *mhi_cntrl)
 147 {
 148         mhi_cntrl->wake_get(mhi_cntrl, false);
 149         mhi_cntrl->wake_put(mhi_cntrl, true);
 150 }
 151
 152 /* Handle device ready state transition */
 153 int mhi_ready_state_transition(struct mhi_controller *mhi_cntrl)
 154 {
 155         void __iomem *base = mhi_cntrl->regs;
 156         struct mhi_event *mhi_event;
 157         enum mhi_pm_state cur_state;
 158         struct device *dev = &mhi_cntrl->mhi_dev->dev;
 159         u32 reset = 1, ready = 0;
 160         int ret, i;
 161
 162         /* Wait for RESET to be cleared and READY bit to be set by the device */
 163         wait_event_timeout(mhi_cntrl->state_event,
 164                            MHI_PM_IN_FATAL_STATE(mhi_cntrl->pm_state) ||
 165                            mhi_read_reg_field(mhi_cntrl, base, MHICTRL,
 166                                               MHICTRL_RESET_MASK,
 167                                               MHICTRL_RESET_SHIFT, &reset) ||
 168                            mhi_read_reg_field(mhi_cntrl, base, MHISTATUS,
 169                                               MHISTATUS_READY_MASK,
 170                                               MHISTATUS_READY_SHIFT, &ready) ||
 171                            (!reset && ready),
 172                            msecs_to_jiffies(mhi_cntrl->timeout_ms));
 173
 174         /* Check if device entered error state */
 175         if (MHI_PM_IN_FATAL_STATE(mhi_cntrl->pm_state)) {
 176                 dev_err(dev, "Device link is not accessible\n");
 177                 return -EIO;
 178         }
 179
 180         /* Timeout if device did not transition to ready state */
 181         if (reset || !ready) {
 182                 dev_err(dev, "Device Ready timeout\n");
 183                 return -ETIMEDOUT;
 184         }
 185
 186         dev_dbg(dev, "Device in READY State\n");
 187         write_lock_irq(&mhi_cntrl->pm_lock);
 188         cur_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_POR);
 189         mhi_cntrl->dev_state = MHI_STATE_READY;
 190         write_unlock_irq(&mhi_cntrl->pm_lock);
 191
 192         if (cur_state != MHI_PM_POR) {
 193                 dev_err(dev, "Error moving to state %s from %s\n",
 194                         to_mhi_pm_state_str(MHI_PM_POR),
 195                         to_mhi_pm_state_str(cur_state));
 196                 return -EIO;
 197         }
 198
 199         read_lock_bh(&mhi_cntrl->pm_lock);
 200         if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) {
 201                 dev_err(dev, "Device registers not accessible\n");
 202                 goto error_mmio;
 203         }
 204
 205         /* Configure MMIO registers */
 206         ret = mhi_init_mmio(mhi_cntrl);
 207         if (ret) {
 208                 dev_err(dev, "Error configuring MMIO registers\n");
 209                 goto error_mmio;
 210         }
 211
 212         /* Add elements to all SW event rings */
 213         mhi_event = mhi_cntrl->mhi_event;
 214         for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) {
 215                 struct mhi_ring *ring = &mhi_event->ring;
 216
 217                 /* Skip if this is an offload or HW event */
 218                 if (mhi_event->offload_ev || mhi_event->hw_ring)
 219                         continue;
 220
 221                 ring->wp = ring->base + ring->len - ring->el_size;
 222                 *ring->ctxt_wp = ring->iommu_base + ring->len - ring->el_size;
 223                 /* Update all cores */
 224                 smp_wmb();
 225
 226                 /* Ring the event ring db */
 227                 spin_lock_irq(&mhi_event->lock);
 228                 mhi_ring_er_db(mhi_event);
 229                 spin_unlock_irq(&mhi_event->lock);
 230         }
 231
 232         /* Set MHI to M0 state */
 233         mhi_set_mhi_state(mhi_cntrl, MHI_STATE_M0);
 234         read_unlock_bh(&mhi_cntrl->pm_lock);
 235
 236         return 0;
 237
 238 error_mmio:
 239         read_unlock_bh(&mhi_cntrl->pm_lock);
 240
 241         return -EIO;
 242 }
 243
 244 int mhi_pm_m0_transition(struct mhi_controller *mhi_cntrl)
 245 {
 246         enum mhi_pm_state cur_state;
 247         struct mhi_chan *mhi_chan;
 248         struct device *dev = &mhi_cntrl->mhi_dev->dev;
 249         int i;
 250
 251         write_lock_irq(&mhi_cntrl->pm_lock);
 252         mhi_cntrl->dev_state = MHI_STATE_M0;
 253         cur_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_M0);
 254         write_unlock_irq(&mhi_cntrl->pm_lock);
 255         if (unlikely(cur_state != MHI_PM_M0)) {
 256                 dev_err(dev, "Unable to transition to M0 state\n");
 257                 return -EIO;
 258         }
 259         mhi_cntrl->M0++;
 260
 261         /* Wake up the device */
 262         read_lock_bh(&mhi_cntrl->pm_lock);
 263         mhi_cntrl->wake_get(mhi_cntrl, true);
 264
 265         /* Ring all event rings and CMD ring only if we're in mission mode */
 266         if (MHI_IN_MISSION_MODE(mhi_cntrl->ee)) {
 267                 struct mhi_event *mhi_event = mhi_cntrl->mhi_event;
 268                 struct mhi_cmd *mhi_cmd =
 269                         &mhi_cntrl->mhi_cmd[PRIMARY_CMD_RING];
 270
 271                 for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) {
 272                         if (mhi_event->offload_ev)
 273                                 continue;
 274
 275                         spin_lock_irq(&mhi_event->lock);
 276                         mhi_ring_er_db(mhi_event);
 277                         spin_unlock_irq(&mhi_event->lock);
 278                 }
 279
 280                 /* Only ring primary cmd ring if ring is not empty */
 281                 spin_lock_irq(&mhi_cmd->lock);
 282                 if (mhi_cmd->ring.rp != mhi_cmd->ring.wp)
 283                         mhi_ring_cmd_db(mhi_cntrl, mhi_cmd);
 284                 spin_unlock_irq(&mhi_cmd->lock);
 285         }
 286
 287         /* Ring channel DB registers */
 288         mhi_chan = mhi_cntrl->mhi_chan;
 289         for (i = 0; i < mhi_cntrl->max_chan; i++, mhi_chan++) {
 290                 struct mhi_ring *tre_ring = &mhi_chan->tre_ring;
 291
 292                 if (mhi_chan->db_cfg.reset_req) {
 293                         write_lock_irq(&mhi_chan->lock);
 294                         mhi_chan->db_cfg.db_mode = true;
 295                         write_unlock_irq(&mhi_chan->lock);
 296                 }
 297
 298                 read_lock_irq(&mhi_chan->lock);
 299
 300                 /* Only ring DB if ring is not empty */
 301                 if (tre_ring->base && tre_ring->wp  != tre_ring->rp)
 302                         mhi_ring_chan_db(mhi_cntrl, mhi_chan);
 303                 read_unlock_irq(&mhi_chan->lock);
 304         }
 305
 306         mhi_cntrl->wake_put(mhi_cntrl, false);
 307         read_unlock_bh(&mhi_cntrl->pm_lock);
 308         wake_up_all(&mhi_cntrl->state_event);
 309
 310         return 0;
 311 }
 312
 313 /*
 314  * After receiving the MHI state change event from the device indicating the
 315  * transition to M1 state, the host can transition the device to M2 state
 316  * for keeping it in low power state.
 317  */
 318 void mhi_pm_m1_transition(struct mhi_controller *mhi_cntrl)
 319 {
 320         enum mhi_pm_state state;
 321         struct device *dev = &mhi_cntrl->mhi_dev->dev;
 322
 323         write_lock_irq(&mhi_cntrl->pm_lock);
 324         state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_M2);
 325         if (state == MHI_PM_M2) {
 326                 mhi_set_mhi_state(mhi_cntrl, MHI_STATE_M2);
 327                 mhi_cntrl->dev_state = MHI_STATE_M2;
 328
 329                 write_unlock_irq(&mhi_cntrl->pm_lock);
 330
 331                 mhi_cntrl->M2++;
 332                 wake_up_all(&mhi_cntrl->state_event);
 333
 334                 /* If there are any pending resources, exit M2 immediately */
 335                 if (unlikely(atomic_read(&mhi_cntrl->pending_pkts) ||
 336                              atomic_read(&mhi_cntrl->dev_wake))) {
 337                         dev_dbg(dev,
 338                                 "Exiting M2, pending_pkts: %d dev_wake: %d\n",
 339                                 atomic_read(&mhi_cntrl->pending_pkts),
 340                                 atomic_read(&mhi_cntrl->dev_wake));
 341                         read_lock_bh(&mhi_cntrl->pm_lock);
 342                         mhi_cntrl->wake_get(mhi_cntrl, true);
 343                         mhi_cntrl->wake_put(mhi_cntrl, true);
 344                         read_unlock_bh(&mhi_cntrl->pm_lock);
 345                 } else {
 346                         mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_IDLE);
 347                 }
 348         } else {
 349                 write_unlock_irq(&mhi_cntrl->pm_lock);
 350         }
 351 }
 352
 353 /* MHI M3 completion handler */
 354 int mhi_pm_m3_transition(struct mhi_controller *mhi_cntrl)
 355 {
 356         enum mhi_pm_state state;
 357         struct device *dev = &mhi_cntrl->mhi_dev->dev;
 358
 359         write_lock_irq(&mhi_cntrl->pm_lock);
 360         mhi_cntrl->dev_state = MHI_STATE_M3;
 361         state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_M3);
 362         write_unlock_irq(&mhi_cntrl->pm_lock);
 363         if (state != MHI_PM_M3) {
 364                 dev_err(dev, "Unable to transition to M3 state\n");
 365                 return -EIO;
 366         }
 367
 368         mhi_cntrl->M3++;
 369         wake_up_all(&mhi_cntrl->state_event);
 370
 371         return 0;
 372 }
 373
 374 /* Handle device Mission Mode transition */
 375 static int mhi_pm_mission_mode_transition(struct mhi_controller *mhi_cntrl)
 376 {
 377         struct mhi_event *mhi_event;
 378         struct device *dev = &mhi_cntrl->mhi_dev->dev;
 379         enum mhi_ee_type current_ee = mhi_cntrl->ee;
 380         int i, ret;
 381
 382         dev_dbg(dev, "Processing Mission Mode transition\n");
 383
 384         write_lock_irq(&mhi_cntrl->pm_lock);
 385         if (MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state))
 386                 mhi_cntrl->ee = mhi_get_exec_env(mhi_cntrl);
 387         write_unlock_irq(&mhi_cntrl->pm_lock);
 388
 389         if (!MHI_IN_MISSION_MODE(mhi_cntrl->ee))
 390                 return -EIO;
 391
 392         wake_up_all(&mhi_cntrl->state_event);
 393
 394         device_for_each_child(&mhi_cntrl->mhi_dev->dev, &current_ee,
 395                               mhi_destroy_device);
 396         mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_EE_MISSION_MODE);
 397
 398         /* Force MHI to be in M0 state before continuing */
 399         ret = __mhi_device_get_sync(mhi_cntrl);
 400         if (ret)
 401                 return ret;
 402
 403         read_lock_bh(&mhi_cntrl->pm_lock);
 404
 405         if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
 406                 ret = -EIO;
 407                 goto error_mission_mode;
 408         }
 409
 410         /* Add elements to all HW event rings */
 411         mhi_event = mhi_cntrl->mhi_event;
 412         for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) {
 413                 struct mhi_ring *ring = &mhi_event->ring;
 414
 415                 if (mhi_event->offload_ev || !mhi_event->hw_ring)
 416                         continue;
 417
 418                 ring->wp = ring->base + ring->len - ring->el_size;
 419                 *ring->ctxt_wp = ring->iommu_base + ring->len - ring->el_size;
 420                 /* Update to all cores */
 421                 smp_wmb();
 422
 423                 spin_lock_irq(&mhi_event->lock);
 424                 if (MHI_DB_ACCESS_VALID(mhi_cntrl))
 425                         mhi_ring_er_db(mhi_event);
 426                 spin_unlock_irq(&mhi_event->lock);
 427         }
 428
 429         read_unlock_bh(&mhi_cntrl->pm_lock);
 430
 431         /*
 432          * The MHI devices are only created when the client device switches its
 433          * Execution Environment (EE) to either SBL or AMSS states
 434          */
 435         mhi_create_devices(mhi_cntrl);
 436
 437         read_lock_bh(&mhi_cntrl->pm_lock);
 438
 439 error_mission_mode:
 440         mhi_cntrl->wake_put(mhi_cntrl, false);
 441         read_unlock_bh(&mhi_cntrl->pm_lock);
 442
 443         return ret;
 444 }
 445
 446 /* Handle SYS_ERR and Shutdown transitions */
 447 static void mhi_pm_disable_transition(struct mhi_controller *mhi_cntrl,
 448                                       enum mhi_pm_state transition_state)
 449 {
 450         enum mhi_pm_state cur_state, prev_state;
 451         struct mhi_event *mhi_event;
 452         struct mhi_cmd_ctxt *cmd_ctxt;
 453         struct mhi_cmd *mhi_cmd;
 454         struct mhi_event_ctxt *er_ctxt;
 455         struct device *dev = &mhi_cntrl->mhi_dev->dev;
 456         int ret, i;
 457
 458         dev_dbg(dev, "Transitioning from PM state: %s to: %s\n",
 459                 to_mhi_pm_state_str(mhi_cntrl->pm_state),
 460                 to_mhi_pm_state_str(transition_state));
 461
 462         /* We must notify MHI control driver so it can clean up first */
 463         if (transition_state == MHI_PM_SYS_ERR_PROCESS)
 464                 mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_SYS_ERROR);
 465
 466         mutex_lock(&mhi_cntrl->pm_mutex);
 467         write_lock_irq(&mhi_cntrl->pm_lock);
 468         prev_state = mhi_cntrl->pm_state;
 469         cur_state = mhi_tryset_pm_state(mhi_cntrl, transition_state);
 470         if (cur_state == transition_state) {
 471                 mhi_cntrl->ee = MHI_EE_DISABLE_TRANSITION;
 472                 mhi_cntrl->dev_state = MHI_STATE_RESET;
 473         }
 474         write_unlock_irq(&mhi_cntrl->pm_lock);
 475
 476         /* Wake up threads waiting for state transition */
 477         wake_up_all(&mhi_cntrl->state_event);
 478
 479         if (cur_state != transition_state) {
 480                 dev_err(dev, "Failed to transition to state: %s from: %s\n",
 481                         to_mhi_pm_state_str(transition_state),
 482                         to_mhi_pm_state_str(cur_state));
 483                 mutex_unlock(&mhi_cntrl->pm_mutex);
 484                 return;
 485         }
 486
 487         /* Trigger MHI RESET so that the device will not access host memory */
 488         if (MHI_REG_ACCESS_VALID(prev_state)) {
 489                 u32 in_reset = -1;
 490                 unsigned long timeout = msecs_to_jiffies(mhi_cntrl->timeout_ms);
 491
 492                 dev_dbg(dev, "Triggering MHI Reset in device\n");
 493                 mhi_set_mhi_state(mhi_cntrl, MHI_STATE_RESET);
 494
 495                 /* Wait for the reset bit to be cleared by the device */
 496                 ret = wait_event_timeout(mhi_cntrl->state_event,
 497                                          mhi_read_reg_field(mhi_cntrl,
 498                                                             mhi_cntrl->regs,
 499                                                             MHICTRL,
 500                                                             MHICTRL_RESET_MASK,
 501                                                             MHICTRL_RESET_SHIFT,
 502                                                             &in_reset) ||
 503                                         !in_reset, timeout);
 504                 if ((!ret || in_reset) && cur_state == MHI_PM_SYS_ERR_PROCESS) {
 505                         dev_err(dev, "Device failed to exit MHI Reset state\n");
 506                         mutex_unlock(&mhi_cntrl->pm_mutex);
 507                         return;
 508                 }
 509
 510                 /*
 511                  * Device will clear BHI_INTVEC as a part of RESET processing,
 512                  * hence re-program it
 513                  */
 514                 mhi_write_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_INTVEC, 0);
 515         }
 516
 517         dev_dbg(dev,
 518                  "Waiting for all pending event ring processing to complete\n");
 519         mhi_event = mhi_cntrl->mhi_event;
 520         for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) {
 521                 if (mhi_event->offload_ev)
 522                         continue;
 523                 tasklet_kill(&mhi_event->task);
 524         }
 525
 526         /* Release lock and wait for all pending threads to complete */
 527         mutex_unlock(&mhi_cntrl->pm_mutex);
 528         dev_dbg(dev, "Waiting for all pending threads to complete\n");
 529         wake_up_all(&mhi_cntrl->state_event);
 530
 531         dev_dbg(dev, "Reset all active channels and remove MHI devices\n");
 532         device_for_each_child(mhi_cntrl->cntrl_dev, NULL, mhi_destroy_device);
 533
 534         mutex_lock(&mhi_cntrl->pm_mutex);
 535
 536         WARN_ON(atomic_read(&mhi_cntrl->dev_wake));
 537         WARN_ON(atomic_read(&mhi_cntrl->pending_pkts));
 538
 539         /* Reset the ev rings and cmd rings */
 540         dev_dbg(dev, "Resetting EV CTXT and CMD CTXT\n");
 541         mhi_cmd = mhi_cntrl->mhi_cmd;
 542         cmd_ctxt = mhi_cntrl->mhi_ctxt->cmd_ctxt;
 543         for (i = 0; i < NR_OF_CMD_RINGS; i++, mhi_cmd++, cmd_ctxt++) {
 544                 struct mhi_ring *ring = &mhi_cmd->ring;
 545
 546                 ring->rp = ring->base;
 547                 ring->wp = ring->base;
 548                 cmd_ctxt->rp = cmd_ctxt->rbase;
 549                 cmd_ctxt->wp = cmd_ctxt->rbase;
 550         }
 551
 552         mhi_event = mhi_cntrl->mhi_event;
 553         er_ctxt = mhi_cntrl->mhi_ctxt->er_ctxt;
 554         for (i = 0; i < mhi_cntrl->total_ev_rings; i++, er_ctxt++,
 555                      mhi_event++) {
 556                 struct mhi_ring *ring = &mhi_event->ring;
 557
 558                 /* Skip offload events */
 559                 if (mhi_event->offload_ev)
 560                         continue;
 561
 562                 ring->rp = ring->base;
 563                 ring->wp = ring->base;
 564                 er_ctxt->rp = er_ctxt->rbase;
 565                 er_ctxt->wp = er_ctxt->rbase;
 566         }
 567
 568         if (cur_state == MHI_PM_SYS_ERR_PROCESS) {
 569                 mhi_ready_state_transition(mhi_cntrl);
 570         } else {
 571                 /* Move to disable state */
 572                 write_lock_irq(&mhi_cntrl->pm_lock);
 573                 cur_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_DISABLE);
 574                 write_unlock_irq(&mhi_cntrl->pm_lock);
 575                 if (unlikely(cur_state != MHI_PM_DISABLE))
 576                         dev_err(dev, "Error moving from PM state: %s to: %s\n",
 577                                 to_mhi_pm_state_str(cur_state),
 578                                 to_mhi_pm_state_str(MHI_PM_DISABLE));
 579         }
 580
 581         dev_dbg(dev, "Exiting with PM state: %s, MHI state: %s\n",
 582                 to_mhi_pm_state_str(mhi_cntrl->pm_state),
 583                 TO_MHI_STATE_STR(mhi_cntrl->dev_state));
 584
 585         mutex_unlock(&mhi_cntrl->pm_mutex);
 586 }
 587
 588 /* Queue a new work item and schedule work */
 589 int mhi_queue_state_transition(struct mhi_controller *mhi_cntrl,
 590                                enum dev_st_transition state)
 591 {
 592         struct state_transition *item = kmalloc(sizeof(*item), GFP_ATOMIC);
 593         unsigned long flags;
 594
 595         if (!item)
 596                 return -ENOMEM;
 597
 598         item->state = state;
 599         spin_lock_irqsave(&mhi_cntrl->transition_lock, flags);
 600         list_add_tail(&item->node, &mhi_cntrl->transition_list);
 601         spin_unlock_irqrestore(&mhi_cntrl->transition_lock, flags);
 602
 603         schedule_work(&mhi_cntrl->st_worker);
 604
 605         return 0;
 606 }
 607
 608 /* SYS_ERR worker */
 609 void mhi_pm_sys_err_handler(struct mhi_controller *mhi_cntrl)
 610 {
 611         struct device *dev = &mhi_cntrl->mhi_dev->dev;
 612
 613         /* skip if controller supports RDDM */
 614         if (mhi_cntrl->rddm_image) {
 615                 dev_dbg(dev, "Controller supports RDDM, skip SYS_ERROR\n");
 616                 return;
 617         }
 618
 619         mhi_queue_state_transition(mhi_cntrl, DEV_ST_TRANSITION_SYS_ERR);
 620 }
 621
 622 /* Device State Transition worker */
 623 void mhi_pm_st_worker(struct work_struct *work)
 624 {
 625         struct state_transition *itr, *tmp;
 626         LIST_HEAD(head);
 627         struct mhi_controller *mhi_cntrl = container_of(work,
 628                                                         struct mhi_controller,
 629                                                         st_worker);
 630         struct device *dev = &mhi_cntrl->mhi_dev->dev;
 631
 632         spin_lock_irq(&mhi_cntrl->transition_lock);
 633         list_splice_tail_init(&mhi_cntrl->transition_list, &head);
 634         spin_unlock_irq(&mhi_cntrl->transition_lock);
 635
 636         list_for_each_entry_safe(itr, tmp, &head, node) {
 637                 list_del(&itr->node);
 638                 dev_dbg(dev, "Handling state transition: %s\n",
 639                         TO_DEV_STATE_TRANS_STR(itr->state));
 640
 641                 switch (itr->state) {
 642                 case DEV_ST_TRANSITION_PBL:
 643                         write_lock_irq(&mhi_cntrl->pm_lock);
 644                         if (MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state))
 645                                 mhi_cntrl->ee = mhi_get_exec_env(mhi_cntrl);
 646                         write_unlock_irq(&mhi_cntrl->pm_lock);
 647                         if (MHI_IN_PBL(mhi_cntrl->ee))
 648                                 mhi_fw_load_handler(mhi_cntrl);
 649                         break;
 650                 case DEV_ST_TRANSITION_SBL:
 651                         write_lock_irq(&mhi_cntrl->pm_lock);
 652                         mhi_cntrl->ee = MHI_EE_SBL;
 653                         write_unlock_irq(&mhi_cntrl->pm_lock);
 654                         /*
 655                          * The MHI devices are only created when the client
 656                          * device switches its Execution Environment (EE) to
 657                          * either SBL or AMSS states
 658                          */
 659                         mhi_create_devices(mhi_cntrl);
 660                         break;
 661                 case DEV_ST_TRANSITION_MISSION_MODE:
 662                         mhi_pm_mission_mode_transition(mhi_cntrl);
 663                         break;
 664                 case DEV_ST_TRANSITION_READY:
 665                         mhi_ready_state_transition(mhi_cntrl);
 666                         break;
 667                 case DEV_ST_TRANSITION_SYS_ERR:
 668                         mhi_pm_disable_transition
 669                                 (mhi_cntrl, MHI_PM_SYS_ERR_PROCESS);
 670                         break;
 671                 case DEV_ST_TRANSITION_DISABLE:
 672                         mhi_pm_disable_transition
 673                                 (mhi_cntrl, MHI_PM_SHUTDOWN_PROCESS);
 674                         break;
 675                 default:
 676                         break;
 677                 }
 678                 kfree(itr);
 679         }
 680 }
 681
 682 int mhi_pm_suspend(struct mhi_controller *mhi_cntrl)
 683 {
 684         struct mhi_chan *itr, *tmp;
 685         struct device *dev = &mhi_cntrl->mhi_dev->dev;
 686         enum mhi_pm_state new_state;
 687         int ret;
 688
 689         if (mhi_cntrl->pm_state == MHI_PM_DISABLE)
 690                 return -EINVAL;
 691
 692         if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))
 693                 return -EIO;
 694
 695         /* Return busy if there are any pending resources */
 696         if (atomic_read(&mhi_cntrl->dev_wake) ||
 697             atomic_read(&mhi_cntrl->pending_pkts))
 698                 return -EBUSY;
 699
 700         /* Take MHI out of M2 state */
 701         read_lock_bh(&mhi_cntrl->pm_lock);
 702         mhi_cntrl->wake_get(mhi_cntrl, false);
 703         read_unlock_bh(&mhi_cntrl->pm_lock);
 704
 705         ret = wait_event_timeout(mhi_cntrl->state_event,
 706                                  mhi_cntrl->dev_state == MHI_STATE_M0 ||
 707                                  mhi_cntrl->dev_state == MHI_STATE_M1 ||
 708                                  MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state),
 709                                  msecs_to_jiffies(mhi_cntrl->timeout_ms));
 710
 711         read_lock_bh(&mhi_cntrl->pm_lock);
 712         mhi_cntrl->wake_put(mhi_cntrl, false);
 713         read_unlock_bh(&mhi_cntrl->pm_lock);
 714
 715         if (!ret || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
 716                 dev_err(dev,
 717                         "Could not enter M0/M1 state");
 718                 return -EIO;
 719         }
 720
 721         write_lock_irq(&mhi_cntrl->pm_lock);
 722
 723         if (atomic_read(&mhi_cntrl->dev_wake) ||
 724             atomic_read(&mhi_cntrl->pending_pkts)) {
 725                 write_unlock_irq(&mhi_cntrl->pm_lock);
 726                 return -EBUSY;
 727         }
 728
 729         dev_info(dev, "Allowing M3 transition\n");
 730         new_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_M3_ENTER);
 731         if (new_state != MHI_PM_M3_ENTER) {
 732                 write_unlock_irq(&mhi_cntrl->pm_lock);
 733                 dev_err(dev,
 734                         "Error setting to PM state: %s from: %s\n",
 735                         to_mhi_pm_state_str(MHI_PM_M3_ENTER),
 736                         to_mhi_pm_state_str(mhi_cntrl->pm_state));
 737                 return -EIO;
 738         }
 739
 740         /* Set MHI to M3 and wait for completion */
 741         mhi_set_mhi_state(mhi_cntrl, MHI_STATE_M3);
 742         write_unlock_irq(&mhi_cntrl->pm_lock);
 743         dev_info(dev, "Wait for M3 completion\n");
 744
 745         ret = wait_event_timeout(mhi_cntrl->state_event,
 746                                  mhi_cntrl->dev_state == MHI_STATE_M3 ||
 747                                  MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state),
 748                                  msecs_to_jiffies(mhi_cntrl->timeout_ms));
 749
 750         if (!ret || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
 751                 dev_err(dev,
 752                         "Did not enter M3 state, MHI state: %s, PM state: %s\n",
 753                         TO_MHI_STATE_STR(mhi_cntrl->dev_state),
 754                         to_mhi_pm_state_str(mhi_cntrl->pm_state));
 755                 return -EIO;
 756         }
 757
 758         /* Notify clients about entering LPM */
 759         list_for_each_entry_safe(itr, tmp, &mhi_cntrl->lpm_chans, node) {
 760                 mutex_lock(&itr->mutex);
 761                 if (itr->mhi_dev)
 762                         mhi_notify(itr->mhi_dev, MHI_CB_LPM_ENTER);
 763                 mutex_unlock(&itr->mutex);
 764         }
 765
 766         return 0;
 767 }
 768 EXPORT_SYMBOL_GPL(mhi_pm_suspend);
 769
 770 int mhi_pm_resume(struct mhi_controller *mhi_cntrl)
 771 {
 772         struct mhi_chan *itr, *tmp;
 773         struct device *dev = &mhi_cntrl->mhi_dev->dev;
 774         enum mhi_pm_state cur_state;
 775         int ret;
 776
 777         dev_info(dev, "Entered with PM state: %s, MHI state: %s\n",
 778                  to_mhi_pm_state_str(mhi_cntrl->pm_state),
 779                  TO_MHI_STATE_STR(mhi_cntrl->dev_state));
 780
 781         if (mhi_cntrl->pm_state == MHI_PM_DISABLE)
 782                 return 0;
 783
 784         if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))
 785                 return -EIO;
 786
 787         /* Notify clients about exiting LPM */
 788         list_for_each_entry_safe(itr, tmp, &mhi_cntrl->lpm_chans, node) {
 789                 mutex_lock(&itr->mutex);
 790                 if (itr->mhi_dev)
 791                         mhi_notify(itr->mhi_dev, MHI_CB_LPM_EXIT);
 792                 mutex_unlock(&itr->mutex);
 793         }
 794
 795         write_lock_irq(&mhi_cntrl->pm_lock);
 796         cur_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_M3_EXIT);
 797         if (cur_state != MHI_PM_M3_EXIT) {
 798                 write_unlock_irq(&mhi_cntrl->pm_lock);
 799                 dev_info(dev,
 800                          "Error setting to PM state: %s from: %s\n",
 801                          to_mhi_pm_state_str(MHI_PM_M3_EXIT),
 802                          to_mhi_pm_state_str(mhi_cntrl->pm_state));
 803                 return -EIO;
 804         }
 805
 806         /* Set MHI to M0 and wait for completion */
 807         mhi_set_mhi_state(mhi_cntrl, MHI_STATE_M0);
 808         write_unlock_irq(&mhi_cntrl->pm_lock);
 809
 810         ret = wait_event_timeout(mhi_cntrl->state_event,
 811                                  mhi_cntrl->dev_state == MHI_STATE_M0 ||
 812                                  mhi_cntrl->dev_state == MHI_STATE_M2 ||
 813                                  MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state),
 814                                  msecs_to_jiffies(mhi_cntrl->timeout_ms));
 815
 816         if (!ret || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
 817                 dev_err(dev,
 818                         "Did not enter M0 state, MHI state: %s, PM state: %s\n",
 819                         TO_MHI_STATE_STR(mhi_cntrl->dev_state),
 820                         to_mhi_pm_state_str(mhi_cntrl->pm_state));
 821                 return -EIO;
 822         }
 823
 824         return 0;
 825 }
 826 EXPORT_SYMBOL_GPL(mhi_pm_resume);
 827
 828 int __mhi_device_get_sync(struct mhi_controller *mhi_cntrl)
 829 {
 830         int ret;
 831
 832         /* Wake up the device */
 833         read_lock_bh(&mhi_cntrl->pm_lock);
 834         mhi_cntrl->wake_get(mhi_cntrl, true);
 835         if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state))
 836                 mhi_trigger_resume(mhi_cntrl);
 837         read_unlock_bh(&mhi_cntrl->pm_lock);
 838
 839         ret = wait_event_timeout(mhi_cntrl->state_event,
 840                                  mhi_cntrl->pm_state == MHI_PM_M0 ||
 841                                  MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state),
 842                                  msecs_to_jiffies(mhi_cntrl->timeout_ms));
 843
 844         if (!ret || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
 845                 read_lock_bh(&mhi_cntrl->pm_lock);
 846                 mhi_cntrl->wake_put(mhi_cntrl, false);
 847                 read_unlock_bh(&mhi_cntrl->pm_lock);
 848                 return -EIO;
 849         }
 850
 851         return 0;
 852 }
 853
 854 /* Assert device wake db */
 855 static void mhi_assert_dev_wake(struct mhi_controller *mhi_cntrl, bool force)
 856 {
 857         unsigned long flags;
 858
 859         /*
 860          * If force flag is set, then increment the wake count value and
 861          * ring wake db
 862          */
 863         if (unlikely(force)) {
 864                 spin_lock_irqsave(&mhi_cntrl->wlock, flags);
 865                 atomic_inc(&mhi_cntrl->dev_wake);
 866                 if (MHI_WAKE_DB_FORCE_SET_VALID(mhi_cntrl->pm_state) &&
 867                     !mhi_cntrl->wake_set) {
 868                         mhi_write_db(mhi_cntrl, mhi_cntrl->wake_db, 1);
 869                         mhi_cntrl->wake_set = true;
 870                 }
 871                 spin_unlock_irqrestore(&mhi_cntrl->wlock, flags);
 872         } else {
 873                 /*
 874                  * If resources are already requested, then just increment
 875                  * the wake count value and return
 876                  */
 877                 if (likely(atomic_add_unless(&mhi_cntrl->dev_wake, 1, 0)))
 878                         return;
 879
 880                 spin_lock_irqsave(&mhi_cntrl->wlock, flags);
 881                 if ((atomic_inc_return(&mhi_cntrl->dev_wake) == 1) &&
 882                     MHI_WAKE_DB_SET_VALID(mhi_cntrl->pm_state) &&
 883                     !mhi_cntrl->wake_set) {
 884                         mhi_write_db(mhi_cntrl, mhi_cntrl->wake_db, 1);
 885                         mhi_cntrl->wake_set = true;
 886                 }
 887                 spin_unlock_irqrestore(&mhi_cntrl->wlock, flags);
 888         }
 889 }
 890
 891 /* De-assert device wake db */
 892 static void mhi_deassert_dev_wake(struct mhi_controller *mhi_cntrl,
 893                                   bool override)
 894 {
 895         unsigned long flags;
 896
 897         /*
 898          * Only continue if there is a single resource, else just decrement
 899          * and return
 900          */
 901         if (likely(atomic_add_unless(&mhi_cntrl->dev_wake, -1, 1)))
 902                 return;
 903
 904         spin_lock_irqsave(&mhi_cntrl->wlock, flags);
 905         if ((atomic_dec_return(&mhi_cntrl->dev_wake) == 0) &&
 906             MHI_WAKE_DB_CLEAR_VALID(mhi_cntrl->pm_state) && !override &&
 907             mhi_cntrl->wake_set) {
 908                 mhi_write_db(mhi_cntrl, mhi_cntrl->wake_db, 0);
 909                 mhi_cntrl->wake_set = false;
 910         }
 911         spin_unlock_irqrestore(&mhi_cntrl->wlock, flags);
 912 }
 913
 914 int mhi_async_power_up(struct mhi_controller *mhi_cntrl)
 915 {
 916         enum mhi_state state;
 917         enum mhi_ee_type current_ee;
 918         enum dev_st_transition next_state;
 919         struct device *dev = &mhi_cntrl->mhi_dev->dev;
 920         u32 val;
 921         int ret;
 922
 923         dev_info(dev, "Requested to power ON\n");
 924
 925         if (mhi_cntrl->nr_irqs < 1)
 926                 return -EINVAL;
 927
 928         /* Supply default wake routines if not provided by controller driver */
 929         if (!mhi_cntrl->wake_get || !mhi_cntrl->wake_put ||
 930             !mhi_cntrl->wake_toggle) {
 931                 mhi_cntrl->wake_get = mhi_assert_dev_wake;
 932                 mhi_cntrl->wake_put = mhi_deassert_dev_wake;
 933                 mhi_cntrl->wake_toggle = (mhi_cntrl->db_access & MHI_PM_M2) ?
 934                         mhi_toggle_dev_wake_nop : mhi_toggle_dev_wake;
 935         }
 936
 937         mutex_lock(&mhi_cntrl->pm_mutex);
 938         mhi_cntrl->pm_state = MHI_PM_DISABLE;
 939
 940         if (!mhi_cntrl->pre_init) {
 941                 /* Setup device context */
 942                 ret = mhi_init_dev_ctxt(mhi_cntrl);
 943                 if (ret)
 944                         goto error_dev_ctxt;
 945         }
 946
 947         ret = mhi_init_irq_setup(mhi_cntrl);
 948         if (ret)
 949                 goto error_setup_irq;
 950
 951         /* Setup BHI offset & INTVEC */
 952         write_lock_irq(&mhi_cntrl->pm_lock);
 953         ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->regs, BHIOFF, &val);
 954         if (ret) {
 955                 write_unlock_irq(&mhi_cntrl->pm_lock);
 956                 goto error_bhi_offset;
 957         }
 958
 959         mhi_cntrl->bhi = mhi_cntrl->regs + val;
 960
 961         /* Setup BHIE offset */
 962         if (mhi_cntrl->fbc_download) {
 963                 ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->regs, BHIEOFF, &val);
 964                 if (ret) {
 965                         write_unlock_irq(&mhi_cntrl->pm_lock);
 966                         dev_err(dev, "Error reading BHIE offset\n");
 967                         goto error_bhi_offset;
 968                 }
 969
 970                 mhi_cntrl->bhie = mhi_cntrl->regs + val;
 971         }
 972
 973         mhi_write_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_INTVEC, 0);
 974         mhi_cntrl->pm_state = MHI_PM_POR;
 975         mhi_cntrl->ee = MHI_EE_MAX;
 976         current_ee = mhi_get_exec_env(mhi_cntrl);
 977         write_unlock_irq(&mhi_cntrl->pm_lock);
 978
 979         /* Confirm that the device is in valid exec env */
 980         if (!MHI_IN_PBL(current_ee) && current_ee != MHI_EE_AMSS) {
 981                 dev_err(dev, "Not a valid EE for power on\n");
 982                 ret = -EIO;
 983                 goto error_bhi_offset;
 984         }
 985
 986         state = mhi_get_mhi_state(mhi_cntrl);
 987         if (state == MHI_STATE_SYS_ERR) {
 988                 mhi_set_mhi_state(mhi_cntrl, MHI_STATE_RESET);
 989                 ret = wait_event_timeout(mhi_cntrl->state_event,
 990                                 MHI_PM_IN_FATAL_STATE(mhi_cntrl->pm_state) ||
 991                                         mhi_read_reg_field(mhi_cntrl,
 992                                                            mhi_cntrl->regs,
 993                                                            MHICTRL,
 994                                                            MHICTRL_RESET_MASK,
 995                                                            MHICTRL_RESET_SHIFT,
 996                                                            &val) ||
 997                                         !val,
 998                                 msecs_to_jiffies(mhi_cntrl->timeout_ms));
 999                 if (!ret) {
1000                         ret = -EIO;
1001                         dev_info(dev, "Failed to reset MHI due to syserr state\n");
1002                         goto error_bhi_offset;
1003                 }
1004
1005                 /*
1006                  * device cleares INTVEC as part of RESET processing,
1007                  * re-program it
1008                  */
1009                 mhi_write_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_INTVEC, 0);
1010         }
1011
1012         /* Transition to next state */
1013         next_state = MHI_IN_PBL(current_ee) ?
1014                 DEV_ST_TRANSITION_PBL : DEV_ST_TRANSITION_READY;
1015
1016         mhi_queue_state_transition(mhi_cntrl, next_state);
1017
1018         mutex_unlock(&mhi_cntrl->pm_mutex);
1019
1020         dev_info(dev, "Power on setup success\n");
1021
1022         return 0;
1023
1024 error_bhi_offset:
1025         mhi_deinit_free_irq(mhi_cntrl);
1026
1027 error_setup_irq:
1028         if (!mhi_cntrl->pre_init)
1029                 mhi_deinit_dev_ctxt(mhi_cntrl);
1030
1031 error_dev_ctxt:
1032         mutex_unlock(&mhi_cntrl->pm_mutex);
1033
1034         return ret;
1035 }
1036 EXPORT_SYMBOL_GPL(mhi_async_power_up);
1037
1038 void mhi_power_down(struct mhi_controller *mhi_cntrl, bool graceful)
1039 {
1040         enum mhi_pm_state cur_state;
1041         struct device *dev = &mhi_cntrl->mhi_dev->dev;
1042
1043         /* If it's not a graceful shutdown, force MHI to linkdown state */
1044         if (!graceful) {
1045                 mutex_lock(&mhi_cntrl->pm_mutex);
1046                 write_lock_irq(&mhi_cntrl->pm_lock);
1047                 cur_state = mhi_tryset_pm_state(mhi_cntrl,
1048                                                 MHI_PM_LD_ERR_FATAL_DETECT);
1049                 write_unlock_irq(&mhi_cntrl->pm_lock);
1050                 mutex_unlock(&mhi_cntrl->pm_mutex);
1051                 if (cur_state != MHI_PM_LD_ERR_FATAL_DETECT)
1052                         dev_dbg(dev, "Failed to move to state: %s from: %s\n",
1053                                 to_mhi_pm_state_str(MHI_PM_LD_ERR_FATAL_DETECT),
1054                                 to_mhi_pm_state_str(mhi_cntrl->pm_state));
1055         }
1056
1057         mhi_queue_state_transition(mhi_cntrl, DEV_ST_TRANSITION_DISABLE);
1058
1059         /* Wait for shutdown to complete */
1060         flush_work(&mhi_cntrl->st_worker);
1061
1062         mhi_deinit_free_irq(mhi_cntrl);
1063
1064         if (!mhi_cntrl->pre_init) {
1065                 /* Free all allocated resources */
1066                 if (mhi_cntrl->fbc_image) {
1067                         mhi_free_bhie_table(mhi_cntrl, mhi_cntrl->fbc_image);
1068                         mhi_cntrl->fbc_image = NULL;
1069                 }
1070                 mhi_deinit_dev_ctxt(mhi_cntrl);
1071         }
1072 }
1073 EXPORT_SYMBOL_GPL(mhi_power_down);
1074
1075 int mhi_sync_power_up(struct mhi_controller *mhi_cntrl)
1076 {
1077         int ret = mhi_async_power_up(mhi_cntrl);
1078
1079         if (ret)
1080                 return ret;
1081
1082         wait_event_timeout(mhi_cntrl->state_event,
1083                            MHI_IN_MISSION_MODE(mhi_cntrl->ee) ||
1084                            MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state),
1085                            msecs_to_jiffies(mhi_cntrl->timeout_ms));
1086
1087         ret = (MHI_IN_MISSION_MODE(mhi_cntrl->ee)) ? 0 : -ETIMEDOUT;
1088         if (ret)
1089                 mhi_power_down(mhi_cntrl, false);
1090
1091         return ret;
1092 }
1093 EXPORT_SYMBOL(mhi_sync_power_up);
1094
1095 int mhi_force_rddm_mode(struct mhi_controller *mhi_cntrl)
1096 {
1097         struct device *dev = &mhi_cntrl->mhi_dev->dev;
1098         int ret;
1099
1100         /* Check if device is already in RDDM */
1101         if (mhi_cntrl->ee == MHI_EE_RDDM)
1102                 return 0;
1103
1104         dev_dbg(dev, "Triggering SYS_ERR to force RDDM state\n");
1105         mhi_set_mhi_state(mhi_cntrl, MHI_STATE_SYS_ERR);
1106
1107         /* Wait for RDDM event */
1108         ret = wait_event_timeout(mhi_cntrl->state_event,
1109                                  mhi_cntrl->ee == MHI_EE_RDDM,
1110                                  msecs_to_jiffies(mhi_cntrl->timeout_ms));
1111         ret = ret ? 0 : -EIO;
1112
1113         return ret;
1114 }
1115 EXPORT_SYMBOL_GPL(mhi_force_rddm_mode);
1116
1117 void mhi_device_get(struct mhi_device *mhi_dev)
1118 {
1119         struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
1120
1121         mhi_dev->dev_wake++;
1122         read_lock_bh(&mhi_cntrl->pm_lock);
1123         if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state))
1124                 mhi_trigger_resume(mhi_cntrl);
1125
1126         mhi_cntrl->wake_get(mhi_cntrl, true);
1127         read_unlock_bh(&mhi_cntrl->pm_lock);
1128 }
1129 EXPORT_SYMBOL_GPL(mhi_device_get);
1130
1131 int mhi_device_get_sync(struct mhi_device *mhi_dev)
1132 {
1133         struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
1134         int ret;
1135
1136         ret = __mhi_device_get_sync(mhi_cntrl);
1137         if (!ret)
1138                 mhi_dev->dev_wake++;
1139
1140         return ret;
1141 }
1142 EXPORT_SYMBOL_GPL(mhi_device_get_sync);
1143
1144 void mhi_device_put(struct mhi_device *mhi_dev)
1145 {
1146         struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
1147
1148         mhi_dev->dev_wake--;
1149         read_lock_bh(&mhi_cntrl->pm_lock);
1150         if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state))
1151                 mhi_trigger_resume(mhi_cntrl);
1152
1153         mhi_cntrl->wake_put(mhi_cntrl, false);
1154         read_unlock_bh(&mhi_cntrl->pm_lock);
1155 }
1156 EXPORT_SYMBOL_GPL(mhi_device_put);