1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved.
7 #include <linux/device.h>
8 #include <linux/dma-direction.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/interrupt.h>
11 #include <linux/list.h>
12 #include <linux/mhi.h>
13 #include <linux/module.h>
14 #include <linux/skbuff.h>
15 #include <linux/slab.h>
18 int __must_check mhi_read_reg(struct mhi_controller *mhi_cntrl,
19 void __iomem *base, u32 offset, u32 *out)
21 return mhi_cntrl->read_reg(mhi_cntrl, base + offset, out);
24 int __must_check mhi_read_reg_field(struct mhi_controller *mhi_cntrl,
25 void __iomem *base, u32 offset,
26 u32 mask, u32 shift, u32 *out)
31 ret = mhi_read_reg(mhi_cntrl, base, offset, &tmp);
35 *out = (tmp & mask) >> shift;
40 void mhi_write_reg(struct mhi_controller *mhi_cntrl, void __iomem *base,
43 mhi_cntrl->write_reg(mhi_cntrl, base + offset, val);
46 void mhi_write_reg_field(struct mhi_controller *mhi_cntrl, void __iomem *base,
47 u32 offset, u32 mask, u32 shift, u32 val)
52 ret = mhi_read_reg(mhi_cntrl, base, offset, &tmp);
57 tmp |= (val << shift);
58 mhi_write_reg(mhi_cntrl, base, offset, tmp);
61 void mhi_write_db(struct mhi_controller *mhi_cntrl, void __iomem *db_addr,
64 mhi_write_reg(mhi_cntrl, db_addr, 4, upper_32_bits(db_val));
65 mhi_write_reg(mhi_cntrl, db_addr, 0, lower_32_bits(db_val));
68 void mhi_db_brstmode(struct mhi_controller *mhi_cntrl,
69 struct db_cfg *db_cfg,
70 void __iomem *db_addr,
73 if (db_cfg->db_mode) {
74 db_cfg->db_val = db_val;
75 mhi_write_db(mhi_cntrl, db_addr, db_val);
80 void mhi_db_brstmode_disable(struct mhi_controller *mhi_cntrl,
81 struct db_cfg *db_cfg,
82 void __iomem *db_addr,
85 db_cfg->db_val = db_val;
86 mhi_write_db(mhi_cntrl, db_addr, db_val);
89 void mhi_ring_er_db(struct mhi_event *mhi_event)
91 struct mhi_ring *ring = &mhi_event->ring;
93 mhi_event->db_cfg.process_db(mhi_event->mhi_cntrl, &mhi_event->db_cfg,
94 ring->db_addr, *ring->ctxt_wp);
97 void mhi_ring_cmd_db(struct mhi_controller *mhi_cntrl, struct mhi_cmd *mhi_cmd)
100 struct mhi_ring *ring = &mhi_cmd->ring;
102 db = ring->iommu_base + (ring->wp - ring->base);
104 mhi_write_db(mhi_cntrl, ring->db_addr, db);
107 void mhi_ring_chan_db(struct mhi_controller *mhi_cntrl,
108 struct mhi_chan *mhi_chan)
110 struct mhi_ring *ring = &mhi_chan->tre_ring;
113 db = ring->iommu_base + (ring->wp - ring->base);
115 mhi_chan->db_cfg.process_db(mhi_cntrl, &mhi_chan->db_cfg,
119 enum mhi_ee_type mhi_get_exec_env(struct mhi_controller *mhi_cntrl)
122 int ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_EXECENV, &exec);
124 return (ret) ? MHI_EE_MAX : exec;
127 enum mhi_state mhi_get_mhi_state(struct mhi_controller *mhi_cntrl)
130 int ret = mhi_read_reg_field(mhi_cntrl, mhi_cntrl->regs, MHISTATUS,
131 MHISTATUS_MHISTATE_MASK,
132 MHISTATUS_MHISTATE_SHIFT, &state);
133 return ret ? MHI_STATE_MAX : state;
136 int mhi_map_single_no_bb(struct mhi_controller *mhi_cntrl,
137 struct mhi_buf_info *buf_info)
139 buf_info->p_addr = dma_map_single(mhi_cntrl->cntrl_dev,
140 buf_info->v_addr, buf_info->len,
142 if (dma_mapping_error(mhi_cntrl->cntrl_dev, buf_info->p_addr))
148 int mhi_map_single_use_bb(struct mhi_controller *mhi_cntrl,
149 struct mhi_buf_info *buf_info)
151 void *buf = mhi_alloc_coherent(mhi_cntrl, buf_info->len,
152 &buf_info->p_addr, GFP_ATOMIC);
157 if (buf_info->dir == DMA_TO_DEVICE)
158 memcpy(buf, buf_info->v_addr, buf_info->len);
160 buf_info->bb_addr = buf;
165 void mhi_unmap_single_no_bb(struct mhi_controller *mhi_cntrl,
166 struct mhi_buf_info *buf_info)
168 dma_unmap_single(mhi_cntrl->cntrl_dev, buf_info->p_addr, buf_info->len,
172 void mhi_unmap_single_use_bb(struct mhi_controller *mhi_cntrl,
173 struct mhi_buf_info *buf_info)
175 if (buf_info->dir == DMA_FROM_DEVICE)
176 memcpy(buf_info->v_addr, buf_info->bb_addr, buf_info->len);
178 mhi_free_coherent(mhi_cntrl, buf_info->len, buf_info->bb_addr,
182 static int get_nr_avail_ring_elements(struct mhi_controller *mhi_cntrl,
183 struct mhi_ring *ring)
187 if (ring->wp < ring->rp) {
188 nr_el = ((ring->rp - ring->wp) / ring->el_size) - 1;
190 nr_el = (ring->rp - ring->base) / ring->el_size;
191 nr_el += ((ring->base + ring->len - ring->wp) /
198 static void *mhi_to_virtual(struct mhi_ring *ring, dma_addr_t addr)
200 return (addr - ring->iommu_base) + ring->base;
203 static void mhi_add_ring_element(struct mhi_controller *mhi_cntrl,
204 struct mhi_ring *ring)
206 ring->wp += ring->el_size;
207 if (ring->wp >= (ring->base + ring->len))
208 ring->wp = ring->base;
213 static void mhi_del_ring_element(struct mhi_controller *mhi_cntrl,
214 struct mhi_ring *ring)
216 ring->rp += ring->el_size;
217 if (ring->rp >= (ring->base + ring->len))
218 ring->rp = ring->base;
223 static bool is_valid_ring_ptr(struct mhi_ring *ring, dma_addr_t addr)
225 return addr >= ring->iommu_base && addr < ring->iommu_base + ring->len;
228 int mhi_destroy_device(struct device *dev, void *data)
230 struct mhi_chan *ul_chan, *dl_chan;
231 struct mhi_device *mhi_dev;
232 struct mhi_controller *mhi_cntrl;
233 enum mhi_ee_type ee = MHI_EE_MAX;
235 if (dev->bus != &mhi_bus_type)
238 mhi_dev = to_mhi_device(dev);
239 mhi_cntrl = mhi_dev->mhi_cntrl;
241 /* Only destroy virtual devices thats attached to bus */
242 if (mhi_dev->dev_type == MHI_DEVICE_CONTROLLER)
245 ul_chan = mhi_dev->ul_chan;
246 dl_chan = mhi_dev->dl_chan;
249 * If execution environment is specified, remove only those devices that
250 * started in them based on ee_mask for the channels as we move on to a
251 * different execution environment
254 ee = *(enum mhi_ee_type *)data;
257 * For the suspend and resume case, this function will get called
258 * without mhi_unregister_controller(). Hence, we need to drop the
259 * references to mhi_dev created for ul and dl channels. We can
260 * be sure that there will be no instances of mhi_dev left after
264 if (ee != MHI_EE_MAX && !(ul_chan->ee_mask & BIT(ee)))
267 put_device(&ul_chan->mhi_dev->dev);
271 if (ee != MHI_EE_MAX && !(dl_chan->ee_mask & BIT(ee)))
274 put_device(&dl_chan->mhi_dev->dev);
277 dev_dbg(&mhi_cntrl->mhi_dev->dev, "destroy device for chan:%s\n",
280 /* Notify the client and remove the device from MHI bus */
287 void mhi_notify(struct mhi_device *mhi_dev, enum mhi_callback cb_reason)
289 struct mhi_driver *mhi_drv;
291 if (!mhi_dev->dev.driver)
294 mhi_drv = to_mhi_driver(mhi_dev->dev.driver);
296 if (mhi_drv->status_cb)
297 mhi_drv->status_cb(mhi_dev, cb_reason);
299 EXPORT_SYMBOL_GPL(mhi_notify);
301 /* Bind MHI channels to MHI devices */
302 void mhi_create_devices(struct mhi_controller *mhi_cntrl)
304 struct mhi_chan *mhi_chan;
305 struct mhi_device *mhi_dev;
306 struct device *dev = &mhi_cntrl->mhi_dev->dev;
309 mhi_chan = mhi_cntrl->mhi_chan;
310 for (i = 0; i < mhi_cntrl->max_chan; i++, mhi_chan++) {
311 if (!mhi_chan->configured || mhi_chan->mhi_dev ||
312 !(mhi_chan->ee_mask & BIT(mhi_cntrl->ee)))
314 mhi_dev = mhi_alloc_device(mhi_cntrl);
318 mhi_dev->dev_type = MHI_DEVICE_XFER;
319 switch (mhi_chan->dir) {
321 mhi_dev->ul_chan = mhi_chan;
322 mhi_dev->ul_chan_id = mhi_chan->chan;
324 case DMA_FROM_DEVICE:
325 /* We use dl_chan as offload channels */
326 mhi_dev->dl_chan = mhi_chan;
327 mhi_dev->dl_chan_id = mhi_chan->chan;
330 dev_err(dev, "Direction not supported\n");
331 put_device(&mhi_dev->dev);
335 get_device(&mhi_dev->dev);
336 mhi_chan->mhi_dev = mhi_dev;
338 /* Check next channel if it matches */
339 if ((i + 1) < mhi_cntrl->max_chan && mhi_chan[1].configured) {
340 if (!strcmp(mhi_chan[1].name, mhi_chan->name)) {
343 if (mhi_chan->dir == DMA_TO_DEVICE) {
344 mhi_dev->ul_chan = mhi_chan;
345 mhi_dev->ul_chan_id = mhi_chan->chan;
347 mhi_dev->dl_chan = mhi_chan;
348 mhi_dev->dl_chan_id = mhi_chan->chan;
350 get_device(&mhi_dev->dev);
351 mhi_chan->mhi_dev = mhi_dev;
355 /* Channel name is same for both UL and DL */
356 mhi_dev->name = mhi_chan->name;
357 dev_set_name(&mhi_dev->dev, "%s_%s",
358 dev_name(mhi_cntrl->cntrl_dev),
361 /* Init wakeup source if available */
362 if (mhi_dev->dl_chan && mhi_dev->dl_chan->wake_capable)
363 device_init_wakeup(&mhi_dev->dev, true);
365 ret = device_add(&mhi_dev->dev);
367 put_device(&mhi_dev->dev);
371 irqreturn_t mhi_irq_handler(int irq_number, void *dev)
373 struct mhi_event *mhi_event = dev;
374 struct mhi_controller *mhi_cntrl = mhi_event->mhi_cntrl;
375 struct mhi_event_ctxt *er_ctxt =
376 &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_event->er_index];
377 struct mhi_ring *ev_ring = &mhi_event->ring;
378 dma_addr_t ptr = er_ctxt->rp;
381 if (!is_valid_ring_ptr(ev_ring, ptr)) {
382 dev_err(&mhi_cntrl->mhi_dev->dev,
383 "Event ring rp points outside of the event ring\n");
387 dev_rp = mhi_to_virtual(ev_ring, ptr);
389 /* Only proceed if event ring has pending events */
390 if (ev_ring->rp == dev_rp)
393 /* For client managed event ring, notify pending data */
394 if (mhi_event->cl_manage) {
395 struct mhi_chan *mhi_chan = mhi_event->mhi_chan;
396 struct mhi_device *mhi_dev = mhi_chan->mhi_dev;
399 mhi_notify(mhi_dev, MHI_CB_PENDING_DATA);
401 tasklet_schedule(&mhi_event->task);
407 irqreturn_t mhi_intvec_threaded_handler(int irq_number, void *priv)
409 struct mhi_controller *mhi_cntrl = priv;
410 struct device *dev = &mhi_cntrl->mhi_dev->dev;
411 enum mhi_state state = MHI_STATE_MAX;
412 enum mhi_pm_state pm_state = 0;
413 enum mhi_ee_type ee = 0;
415 write_lock_irq(&mhi_cntrl->pm_lock);
416 if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) {
417 write_unlock_irq(&mhi_cntrl->pm_lock);
421 state = mhi_get_mhi_state(mhi_cntrl);
423 mhi_cntrl->ee = mhi_get_exec_env(mhi_cntrl);
424 dev_dbg(dev, "local ee:%s device ee:%s dev_state:%s\n",
425 TO_MHI_EXEC_STR(mhi_cntrl->ee), TO_MHI_EXEC_STR(ee),
426 TO_MHI_STATE_STR(state));
428 if (state == MHI_STATE_SYS_ERR) {
429 dev_dbg(dev, "System error detected\n");
430 pm_state = mhi_tryset_pm_state(mhi_cntrl,
431 MHI_PM_SYS_ERR_DETECT);
433 write_unlock_irq(&mhi_cntrl->pm_lock);
435 /* If device supports RDDM don't bother processing SYS error */
436 if (mhi_cntrl->rddm_image) {
437 if (mhi_cntrl->ee == MHI_EE_RDDM && mhi_cntrl->ee != ee) {
438 mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_EE_RDDM);
439 wake_up_all(&mhi_cntrl->state_event);
444 if (pm_state == MHI_PM_SYS_ERR_DETECT) {
445 wake_up_all(&mhi_cntrl->state_event);
447 /* For fatal errors, we let controller decide next step */
449 mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_FATAL_ERROR);
451 mhi_pm_sys_err_handler(mhi_cntrl);
459 irqreturn_t mhi_intvec_handler(int irq_number, void *dev)
461 struct mhi_controller *mhi_cntrl = dev;
463 /* Wake up events waiting for state change */
464 wake_up_all(&mhi_cntrl->state_event);
466 return IRQ_WAKE_THREAD;
469 static void mhi_recycle_ev_ring_element(struct mhi_controller *mhi_cntrl,
470 struct mhi_ring *ring)
475 ring->wp += ring->el_size;
476 ctxt_wp = *ring->ctxt_wp + ring->el_size;
478 if (ring->wp >= (ring->base + ring->len)) {
479 ring->wp = ring->base;
480 ctxt_wp = ring->iommu_base;
483 *ring->ctxt_wp = ctxt_wp;
486 ring->rp += ring->el_size;
487 if (ring->rp >= (ring->base + ring->len))
488 ring->rp = ring->base;
490 /* Update to all cores */
494 static int parse_xfer_event(struct mhi_controller *mhi_cntrl,
495 struct mhi_tre *event,
496 struct mhi_chan *mhi_chan)
498 struct mhi_ring *buf_ring, *tre_ring;
499 struct device *dev = &mhi_cntrl->mhi_dev->dev;
500 struct mhi_result result;
501 unsigned long flags = 0;
504 ev_code = MHI_TRE_GET_EV_CODE(event);
505 buf_ring = &mhi_chan->buf_ring;
506 tre_ring = &mhi_chan->tre_ring;
508 result.transaction_status = (ev_code == MHI_EV_CC_OVERFLOW) ?
512 * If it's a DB Event then we need to grab the lock
513 * with preemption disabled and as a write because we
514 * have to update db register and there are chances that
515 * another thread could be doing the same.
517 if (ev_code >= MHI_EV_CC_OOB)
518 write_lock_irqsave(&mhi_chan->lock, flags);
520 read_lock_bh(&mhi_chan->lock);
522 if (mhi_chan->ch_state != MHI_CH_STATE_ENABLED)
523 goto end_process_tx_event;
526 case MHI_EV_CC_OVERFLOW:
530 dma_addr_t ptr = MHI_TRE_GET_EV_PTR(event);
531 struct mhi_tre *local_rp, *ev_tre;
533 struct mhi_buf_info *buf_info;
536 if (!is_valid_ring_ptr(tre_ring, ptr)) {
537 dev_err(&mhi_cntrl->mhi_dev->dev,
538 "Event element points outside of the tre ring\n");
541 /* Get the TRB this event points to */
542 ev_tre = mhi_to_virtual(tre_ring, ptr);
545 if (dev_rp >= (tre_ring->base + tre_ring->len))
546 dev_rp = tre_ring->base;
548 result.dir = mhi_chan->dir;
550 local_rp = tre_ring->rp;
551 while (local_rp != dev_rp) {
552 buf_info = buf_ring->rp;
553 /* If it's the last TRE, get length from the event */
554 if (local_rp == ev_tre)
555 xfer_len = MHI_TRE_GET_EV_LEN(event);
557 xfer_len = buf_info->len;
559 /* Unmap if it's not pre-mapped by client */
560 if (likely(!buf_info->pre_mapped))
561 mhi_cntrl->unmap_single(mhi_cntrl, buf_info);
563 result.buf_addr = buf_info->cb_buf;
565 /* truncate to buf len if xfer_len is larger */
567 min_t(u16, xfer_len, buf_info->len);
568 mhi_del_ring_element(mhi_cntrl, buf_ring);
569 mhi_del_ring_element(mhi_cntrl, tre_ring);
570 local_rp = tre_ring->rp;
573 mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
575 if (mhi_chan->dir == DMA_TO_DEVICE)
576 atomic_dec(&mhi_cntrl->pending_pkts);
579 * Recycle the buffer if buffer is pre-allocated,
580 * if there is an error, not much we can do apart
581 * from dropping the packet
583 if (mhi_chan->pre_alloc) {
584 if (mhi_queue_buf(mhi_chan->mhi_dev,
587 buf_info->len, MHI_EOT)) {
589 "Error recycling buffer for chan:%d\n",
591 kfree(buf_info->cb_buf);
598 case MHI_EV_CC_DB_MODE:
602 mhi_chan->db_cfg.db_mode = 1;
603 read_lock_irqsave(&mhi_cntrl->pm_lock, flags);
604 if (tre_ring->wp != tre_ring->rp &&
605 MHI_DB_ACCESS_VALID(mhi_cntrl)) {
606 mhi_ring_chan_db(mhi_cntrl, mhi_chan);
608 read_unlock_irqrestore(&mhi_cntrl->pm_lock, flags);
611 case MHI_EV_CC_BAD_TRE:
613 dev_err(dev, "Unknown event 0x%x\n", ev_code);
615 } /* switch(MHI_EV_READ_CODE(EV_TRB_CODE,event)) */
617 end_process_tx_event:
618 if (ev_code >= MHI_EV_CC_OOB)
619 write_unlock_irqrestore(&mhi_chan->lock, flags);
621 read_unlock_bh(&mhi_chan->lock);
626 static int parse_rsc_event(struct mhi_controller *mhi_cntrl,
627 struct mhi_tre *event,
628 struct mhi_chan *mhi_chan)
630 struct mhi_ring *buf_ring, *tre_ring;
631 struct mhi_buf_info *buf_info;
632 struct mhi_result result;
634 u32 cookie; /* offset to local descriptor */
637 buf_ring = &mhi_chan->buf_ring;
638 tre_ring = &mhi_chan->tre_ring;
640 ev_code = MHI_TRE_GET_EV_CODE(event);
641 cookie = MHI_TRE_GET_EV_COOKIE(event);
642 xfer_len = MHI_TRE_GET_EV_LEN(event);
644 /* Received out of bound cookie */
645 WARN_ON(cookie >= buf_ring->len);
647 buf_info = buf_ring->base + cookie;
649 result.transaction_status = (ev_code == MHI_EV_CC_OVERFLOW) ?
652 /* truncate to buf len if xfer_len is larger */
653 result.bytes_xferd = min_t(u16, xfer_len, buf_info->len);
654 result.buf_addr = buf_info->cb_buf;
655 result.dir = mhi_chan->dir;
657 read_lock_bh(&mhi_chan->lock);
659 if (mhi_chan->ch_state != MHI_CH_STATE_ENABLED)
660 goto end_process_rsc_event;
662 WARN_ON(!buf_info->used);
664 /* notify the client */
665 mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
668 * Note: We're arbitrarily incrementing RP even though, completion
669 * packet we processed might not be the same one, reason we can do this
670 * is because device guaranteed to cache descriptors in order it
671 * receive, so even though completion event is different we can re-use
672 * all descriptors in between.
674 * Transfer Ring has descriptors: A, B, C, D
675 * Last descriptor host queue is D (WP) and first descriptor
676 * host queue is A (RP).
677 * The completion event we just serviced is descriptor C.
678 * Then we can safely queue descriptors to replace A, B, and C
679 * even though host did not receive any completions.
681 mhi_del_ring_element(mhi_cntrl, tre_ring);
682 buf_info->used = false;
684 end_process_rsc_event:
685 read_unlock_bh(&mhi_chan->lock);
690 static void mhi_process_cmd_completion(struct mhi_controller *mhi_cntrl,
693 dma_addr_t ptr = MHI_TRE_GET_EV_PTR(tre);
694 struct mhi_cmd *cmd_ring = &mhi_cntrl->mhi_cmd[PRIMARY_CMD_RING];
695 struct mhi_ring *mhi_ring = &cmd_ring->ring;
696 struct mhi_tre *cmd_pkt;
697 struct mhi_chan *mhi_chan;
700 if (!is_valid_ring_ptr(mhi_ring, ptr)) {
701 dev_err(&mhi_cntrl->mhi_dev->dev,
702 "Event element points outside of the cmd ring\n");
706 cmd_pkt = mhi_to_virtual(mhi_ring, ptr);
708 chan = MHI_TRE_GET_CMD_CHID(cmd_pkt);
710 if (chan < mhi_cntrl->max_chan &&
711 mhi_cntrl->mhi_chan[chan].configured) {
712 mhi_chan = &mhi_cntrl->mhi_chan[chan];
713 write_lock_bh(&mhi_chan->lock);
714 mhi_chan->ccs = MHI_TRE_GET_EV_CODE(tre);
715 complete(&mhi_chan->completion);
716 write_unlock_bh(&mhi_chan->lock);
718 dev_err(&mhi_cntrl->mhi_dev->dev,
719 "Completion packet for invalid channel ID: %d\n", chan);
722 mhi_del_ring_element(mhi_cntrl, mhi_ring);
725 int mhi_process_ctrl_ev_ring(struct mhi_controller *mhi_cntrl,
726 struct mhi_event *mhi_event,
729 struct mhi_tre *dev_rp, *local_rp;
730 struct mhi_ring *ev_ring = &mhi_event->ring;
731 struct mhi_event_ctxt *er_ctxt =
732 &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_event->er_index];
733 struct mhi_chan *mhi_chan;
734 struct device *dev = &mhi_cntrl->mhi_dev->dev;
737 dma_addr_t ptr = er_ctxt->rp;
740 * This is a quick check to avoid unnecessary event processing
741 * in case MHI is already in error state, but it's still possible
742 * to transition to error state while processing events
744 if (unlikely(MHI_EVENT_ACCESS_INVALID(mhi_cntrl->pm_state)))
747 if (!is_valid_ring_ptr(ev_ring, ptr)) {
748 dev_err(&mhi_cntrl->mhi_dev->dev,
749 "Event ring rp points outside of the event ring\n");
753 dev_rp = mhi_to_virtual(ev_ring, ptr);
754 local_rp = ev_ring->rp;
756 while (dev_rp != local_rp) {
757 enum mhi_pkt_type type = MHI_TRE_GET_EV_TYPE(local_rp);
760 case MHI_PKT_TYPE_BW_REQ_EVENT:
762 struct mhi_link_info *link_info;
764 link_info = &mhi_cntrl->mhi_link_info;
765 write_lock_irq(&mhi_cntrl->pm_lock);
766 link_info->target_link_speed =
767 MHI_TRE_GET_EV_LINKSPEED(local_rp);
768 link_info->target_link_width =
769 MHI_TRE_GET_EV_LINKWIDTH(local_rp);
770 write_unlock_irq(&mhi_cntrl->pm_lock);
771 dev_dbg(dev, "Received BW_REQ event\n");
772 mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_BW_REQ);
775 case MHI_PKT_TYPE_STATE_CHANGE_EVENT:
777 enum mhi_state new_state;
779 new_state = MHI_TRE_GET_EV_STATE(local_rp);
781 dev_dbg(dev, "State change event to state: %s\n",
782 TO_MHI_STATE_STR(new_state));
786 mhi_pm_m0_transition(mhi_cntrl);
789 mhi_pm_m1_transition(mhi_cntrl);
792 mhi_pm_m3_transition(mhi_cntrl);
794 case MHI_STATE_SYS_ERR:
796 enum mhi_pm_state new_state;
798 /* skip SYS_ERROR handling if RDDM supported */
799 if (mhi_cntrl->ee == MHI_EE_RDDM ||
800 mhi_cntrl->rddm_image)
803 dev_dbg(dev, "System error detected\n");
804 write_lock_irq(&mhi_cntrl->pm_lock);
805 new_state = mhi_tryset_pm_state(mhi_cntrl,
806 MHI_PM_SYS_ERR_DETECT);
807 write_unlock_irq(&mhi_cntrl->pm_lock);
808 if (new_state == MHI_PM_SYS_ERR_DETECT)
809 mhi_pm_sys_err_handler(mhi_cntrl);
813 dev_err(dev, "Invalid state: %s\n",
814 TO_MHI_STATE_STR(new_state));
819 case MHI_PKT_TYPE_CMD_COMPLETION_EVENT:
820 mhi_process_cmd_completion(mhi_cntrl, local_rp);
822 case MHI_PKT_TYPE_EE_EVENT:
824 enum dev_st_transition st = DEV_ST_TRANSITION_MAX;
825 enum mhi_ee_type event = MHI_TRE_GET_EV_EXECENV(local_rp);
827 dev_dbg(dev, "Received EE event: %s\n",
828 TO_MHI_EXEC_STR(event));
831 st = DEV_ST_TRANSITION_SBL;
835 st = DEV_ST_TRANSITION_MISSION_MODE;
838 mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_EE_RDDM);
839 write_lock_irq(&mhi_cntrl->pm_lock);
840 mhi_cntrl->ee = event;
841 write_unlock_irq(&mhi_cntrl->pm_lock);
842 wake_up_all(&mhi_cntrl->state_event);
846 "Unhandled EE event: 0x%x\n", type);
848 if (st != DEV_ST_TRANSITION_MAX)
849 mhi_queue_state_transition(mhi_cntrl, st);
853 case MHI_PKT_TYPE_TX_EVENT:
854 chan = MHI_TRE_GET_EV_CHID(local_rp);
856 WARN_ON(chan >= mhi_cntrl->max_chan);
859 * Only process the event ring elements whose channel
860 * ID is within the maximum supported range.
862 if (chan < mhi_cntrl->max_chan) {
863 mhi_chan = &mhi_cntrl->mhi_chan[chan];
864 if (!mhi_chan->configured)
866 parse_xfer_event(mhi_cntrl, local_rp, mhi_chan);
871 dev_err(dev, "Unhandled event type: %d\n", type);
875 mhi_recycle_ev_ring_element(mhi_cntrl, ev_ring);
876 local_rp = ev_ring->rp;
879 if (!is_valid_ring_ptr(ev_ring, ptr)) {
880 dev_err(&mhi_cntrl->mhi_dev->dev,
881 "Event ring rp points outside of the event ring\n");
885 dev_rp = mhi_to_virtual(ev_ring, ptr);
889 read_lock_bh(&mhi_cntrl->pm_lock);
890 if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl)))
891 mhi_ring_er_db(mhi_event);
892 read_unlock_bh(&mhi_cntrl->pm_lock);
897 int mhi_process_data_event_ring(struct mhi_controller *mhi_cntrl,
898 struct mhi_event *mhi_event,
901 struct mhi_tre *dev_rp, *local_rp;
902 struct mhi_ring *ev_ring = &mhi_event->ring;
903 struct mhi_event_ctxt *er_ctxt =
904 &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_event->er_index];
907 struct mhi_chan *mhi_chan;
908 dma_addr_t ptr = er_ctxt->rp;
910 if (unlikely(MHI_EVENT_ACCESS_INVALID(mhi_cntrl->pm_state)))
913 if (!is_valid_ring_ptr(ev_ring, ptr)) {
914 dev_err(&mhi_cntrl->mhi_dev->dev,
915 "Event ring rp points outside of the event ring\n");
919 dev_rp = mhi_to_virtual(ev_ring, ptr);
920 local_rp = ev_ring->rp;
922 while (dev_rp != local_rp && event_quota > 0) {
923 enum mhi_pkt_type type = MHI_TRE_GET_EV_TYPE(local_rp);
925 chan = MHI_TRE_GET_EV_CHID(local_rp);
927 WARN_ON(chan >= mhi_cntrl->max_chan);
930 * Only process the event ring elements whose channel
931 * ID is within the maximum supported range.
933 if (chan < mhi_cntrl->max_chan &&
934 mhi_cntrl->mhi_chan[chan].configured) {
935 mhi_chan = &mhi_cntrl->mhi_chan[chan];
937 if (likely(type == MHI_PKT_TYPE_TX_EVENT)) {
938 parse_xfer_event(mhi_cntrl, local_rp, mhi_chan);
940 } else if (type == MHI_PKT_TYPE_RSC_TX_EVENT) {
941 parse_rsc_event(mhi_cntrl, local_rp, mhi_chan);
946 mhi_recycle_ev_ring_element(mhi_cntrl, ev_ring);
947 local_rp = ev_ring->rp;
950 if (!is_valid_ring_ptr(ev_ring, ptr)) {
951 dev_err(&mhi_cntrl->mhi_dev->dev,
952 "Event ring rp points outside of the event ring\n");
956 dev_rp = mhi_to_virtual(ev_ring, ptr);
959 read_lock_bh(&mhi_cntrl->pm_lock);
960 if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl)))
961 mhi_ring_er_db(mhi_event);
962 read_unlock_bh(&mhi_cntrl->pm_lock);
967 void mhi_ev_task(unsigned long data)
969 struct mhi_event *mhi_event = (struct mhi_event *)data;
970 struct mhi_controller *mhi_cntrl = mhi_event->mhi_cntrl;
972 /* process all pending events */
973 spin_lock_bh(&mhi_event->lock);
974 mhi_event->process_event(mhi_cntrl, mhi_event, U32_MAX);
975 spin_unlock_bh(&mhi_event->lock);
978 void mhi_ctrl_ev_task(unsigned long data)
980 struct mhi_event *mhi_event = (struct mhi_event *)data;
981 struct mhi_controller *mhi_cntrl = mhi_event->mhi_cntrl;
982 struct device *dev = &mhi_cntrl->mhi_dev->dev;
983 enum mhi_state state;
984 enum mhi_pm_state pm_state = 0;
988 * We can check PM state w/o a lock here because there is no way
989 * PM state can change from reg access valid to no access while this
990 * thread being executed.
992 if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) {
994 * We may have a pending event but not allowed to
995 * process it since we are probably in a suspended state,
996 * so trigger a resume.
998 mhi_trigger_resume(mhi_cntrl);
1003 /* Process ctrl events events */
1004 ret = mhi_event->process_event(mhi_cntrl, mhi_event, U32_MAX);
1007 * We received an IRQ but no events to process, maybe device went to
1008 * SYS_ERR state? Check the state to confirm.
1011 write_lock_irq(&mhi_cntrl->pm_lock);
1012 state = mhi_get_mhi_state(mhi_cntrl);
1013 if (state == MHI_STATE_SYS_ERR) {
1014 dev_dbg(dev, "System error detected\n");
1015 pm_state = mhi_tryset_pm_state(mhi_cntrl,
1016 MHI_PM_SYS_ERR_DETECT);
1018 write_unlock_irq(&mhi_cntrl->pm_lock);
1019 if (pm_state == MHI_PM_SYS_ERR_DETECT)
1020 mhi_pm_sys_err_handler(mhi_cntrl);
1024 static bool mhi_is_ring_full(struct mhi_controller *mhi_cntrl,
1025 struct mhi_ring *ring)
1027 void *tmp = ring->wp + ring->el_size;
1029 if (tmp >= (ring->base + ring->len))
1032 return (tmp == ring->rp);
1035 int mhi_queue_skb(struct mhi_device *mhi_dev, enum dma_data_direction dir,
1036 struct sk_buff *skb, size_t len, enum mhi_flags mflags)
1038 struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
1039 struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan :
1041 struct mhi_ring *tre_ring = &mhi_chan->tre_ring;
1042 struct mhi_buf_info buf_info = { };
1045 /* If MHI host pre-allocates buffers then client drivers cannot queue */
1046 if (mhi_chan->pre_alloc)
1049 if (mhi_is_ring_full(mhi_cntrl, tre_ring))
1052 read_lock_bh(&mhi_cntrl->pm_lock);
1053 if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) {
1054 read_unlock_bh(&mhi_cntrl->pm_lock);
1058 /* we're in M3 or transitioning to M3 */
1059 if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state))
1060 mhi_trigger_resume(mhi_cntrl);
1062 /* Toggle wake to exit out of M2 */
1063 mhi_cntrl->wake_toggle(mhi_cntrl);
1065 buf_info.v_addr = skb->data;
1066 buf_info.cb_buf = skb;
1069 ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &buf_info, mflags);
1070 if (unlikely(ret)) {
1071 read_unlock_bh(&mhi_cntrl->pm_lock);
1075 if (mhi_chan->dir == DMA_TO_DEVICE)
1076 atomic_inc(&mhi_cntrl->pending_pkts);
1078 if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) {
1079 read_lock_bh(&mhi_chan->lock);
1080 mhi_ring_chan_db(mhi_cntrl, mhi_chan);
1081 read_unlock_bh(&mhi_chan->lock);
1084 read_unlock_bh(&mhi_cntrl->pm_lock);
1088 EXPORT_SYMBOL_GPL(mhi_queue_skb);
1090 int mhi_queue_dma(struct mhi_device *mhi_dev, enum dma_data_direction dir,
1091 struct mhi_buf *mhi_buf, size_t len, enum mhi_flags mflags)
1093 struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
1094 struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan :
1096 struct device *dev = &mhi_cntrl->mhi_dev->dev;
1097 struct mhi_ring *tre_ring = &mhi_chan->tre_ring;
1098 struct mhi_buf_info buf_info = { };
1101 /* If MHI host pre-allocates buffers then client drivers cannot queue */
1102 if (mhi_chan->pre_alloc)
1105 if (mhi_is_ring_full(mhi_cntrl, tre_ring))
1108 read_lock_bh(&mhi_cntrl->pm_lock);
1109 if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) {
1110 dev_err(dev, "MHI is not in activate state, PM state: %s\n",
1111 to_mhi_pm_state_str(mhi_cntrl->pm_state));
1112 read_unlock_bh(&mhi_cntrl->pm_lock);
1117 /* we're in M3 or transitioning to M3 */
1118 if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state))
1119 mhi_trigger_resume(mhi_cntrl);
1121 /* Toggle wake to exit out of M2 */
1122 mhi_cntrl->wake_toggle(mhi_cntrl);
1124 buf_info.p_addr = mhi_buf->dma_addr;
1125 buf_info.cb_buf = mhi_buf;
1126 buf_info.pre_mapped = true;
1129 ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &buf_info, mflags);
1130 if (unlikely(ret)) {
1131 read_unlock_bh(&mhi_cntrl->pm_lock);
1135 if (mhi_chan->dir == DMA_TO_DEVICE)
1136 atomic_inc(&mhi_cntrl->pending_pkts);
1138 if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) {
1139 read_lock_bh(&mhi_chan->lock);
1140 mhi_ring_chan_db(mhi_cntrl, mhi_chan);
1141 read_unlock_bh(&mhi_chan->lock);
1144 read_unlock_bh(&mhi_cntrl->pm_lock);
1148 EXPORT_SYMBOL_GPL(mhi_queue_dma);
1150 int mhi_gen_tre(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan,
1151 struct mhi_buf_info *info, enum mhi_flags flags)
1153 struct mhi_ring *buf_ring, *tre_ring;
1154 struct mhi_tre *mhi_tre;
1155 struct mhi_buf_info *buf_info;
1156 int eot, eob, chain, bei;
1159 buf_ring = &mhi_chan->buf_ring;
1160 tre_ring = &mhi_chan->tre_ring;
1162 buf_info = buf_ring->wp;
1163 WARN_ON(buf_info->used);
1164 buf_info->pre_mapped = info->pre_mapped;
1165 if (info->pre_mapped)
1166 buf_info->p_addr = info->p_addr;
1168 buf_info->v_addr = info->v_addr;
1169 buf_info->cb_buf = info->cb_buf;
1170 buf_info->wp = tre_ring->wp;
1171 buf_info->dir = mhi_chan->dir;
1172 buf_info->len = info->len;
1174 if (!info->pre_mapped) {
1175 ret = mhi_cntrl->map_single(mhi_cntrl, buf_info);
1180 eob = !!(flags & MHI_EOB);
1181 eot = !!(flags & MHI_EOT);
1182 chain = !!(flags & MHI_CHAIN);
1183 bei = !!(mhi_chan->intmod);
1185 mhi_tre = tre_ring->wp;
1186 mhi_tre->ptr = MHI_TRE_DATA_PTR(buf_info->p_addr);
1187 mhi_tre->dword[0] = MHI_TRE_DATA_DWORD0(info->len);
1188 mhi_tre->dword[1] = MHI_TRE_DATA_DWORD1(bei, eot, eob, chain);
1191 mhi_add_ring_element(mhi_cntrl, tre_ring);
1192 mhi_add_ring_element(mhi_cntrl, buf_ring);
1197 int mhi_queue_buf(struct mhi_device *mhi_dev, enum dma_data_direction dir,
1198 void *buf, size_t len, enum mhi_flags mflags)
1200 struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
1201 struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan :
1203 struct mhi_ring *tre_ring;
1204 struct mhi_buf_info buf_info = { };
1205 unsigned long flags;
1209 * this check here only as a guard, it's always
1210 * possible mhi can enter error while executing rest of function,
1211 * which is not fatal so we do not need to hold pm_lock
1213 if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)))
1216 tre_ring = &mhi_chan->tre_ring;
1217 if (mhi_is_ring_full(mhi_cntrl, tre_ring))
1220 buf_info.v_addr = buf;
1221 buf_info.cb_buf = buf;
1224 ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &buf_info, mflags);
1228 read_lock_irqsave(&mhi_cntrl->pm_lock, flags);
1230 /* we're in M3 or transitioning to M3 */
1231 if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state))
1232 mhi_trigger_resume(mhi_cntrl);
1234 /* Toggle wake to exit out of M2 */
1235 mhi_cntrl->wake_toggle(mhi_cntrl);
1237 if (mhi_chan->dir == DMA_TO_DEVICE)
1238 atomic_inc(&mhi_cntrl->pending_pkts);
1240 if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) {
1241 unsigned long flags;
1243 read_lock_irqsave(&mhi_chan->lock, flags);
1244 mhi_ring_chan_db(mhi_cntrl, mhi_chan);
1245 read_unlock_irqrestore(&mhi_chan->lock, flags);
1248 read_unlock_irqrestore(&mhi_cntrl->pm_lock, flags);
1252 EXPORT_SYMBOL_GPL(mhi_queue_buf);
1254 int mhi_send_cmd(struct mhi_controller *mhi_cntrl,
1255 struct mhi_chan *mhi_chan,
1256 enum mhi_cmd_type cmd)
1258 struct mhi_tre *cmd_tre = NULL;
1259 struct mhi_cmd *mhi_cmd = &mhi_cntrl->mhi_cmd[PRIMARY_CMD_RING];
1260 struct mhi_ring *ring = &mhi_cmd->ring;
1261 struct device *dev = &mhi_cntrl->mhi_dev->dev;
1265 chan = mhi_chan->chan;
1267 spin_lock_bh(&mhi_cmd->lock);
1268 if (!get_nr_avail_ring_elements(mhi_cntrl, ring)) {
1269 spin_unlock_bh(&mhi_cmd->lock);
1273 /* prepare the cmd tre */
1276 case MHI_CMD_RESET_CHAN:
1277 cmd_tre->ptr = MHI_TRE_CMD_RESET_PTR;
1278 cmd_tre->dword[0] = MHI_TRE_CMD_RESET_DWORD0;
1279 cmd_tre->dword[1] = MHI_TRE_CMD_RESET_DWORD1(chan);
1281 case MHI_CMD_START_CHAN:
1282 cmd_tre->ptr = MHI_TRE_CMD_START_PTR;
1283 cmd_tre->dword[0] = MHI_TRE_CMD_START_DWORD0;
1284 cmd_tre->dword[1] = MHI_TRE_CMD_START_DWORD1(chan);
1287 dev_err(dev, "Command not supported\n");
1291 /* queue to hardware */
1292 mhi_add_ring_element(mhi_cntrl, ring);
1293 read_lock_bh(&mhi_cntrl->pm_lock);
1294 if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl)))
1295 mhi_ring_cmd_db(mhi_cntrl, mhi_cmd);
1296 read_unlock_bh(&mhi_cntrl->pm_lock);
1297 spin_unlock_bh(&mhi_cmd->lock);
1302 static void __mhi_unprepare_channel(struct mhi_controller *mhi_cntrl,
1303 struct mhi_chan *mhi_chan)
1306 struct device *dev = &mhi_cntrl->mhi_dev->dev;
1308 dev_dbg(dev, "Entered: unprepare channel:%d\n", mhi_chan->chan);
1310 /* no more processing events for this channel */
1311 mutex_lock(&mhi_chan->mutex);
1312 write_lock_irq(&mhi_chan->lock);
1313 if (mhi_chan->ch_state != MHI_CH_STATE_ENABLED) {
1314 write_unlock_irq(&mhi_chan->lock);
1315 mutex_unlock(&mhi_chan->mutex);
1319 mhi_chan->ch_state = MHI_CH_STATE_DISABLED;
1320 write_unlock_irq(&mhi_chan->lock);
1322 reinit_completion(&mhi_chan->completion);
1323 read_lock_bh(&mhi_cntrl->pm_lock);
1324 if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
1325 read_unlock_bh(&mhi_cntrl->pm_lock);
1326 goto error_invalid_state;
1329 mhi_cntrl->wake_toggle(mhi_cntrl);
1330 read_unlock_bh(&mhi_cntrl->pm_lock);
1332 mhi_cntrl->runtime_get(mhi_cntrl);
1333 mhi_cntrl->runtime_put(mhi_cntrl);
1334 ret = mhi_send_cmd(mhi_cntrl, mhi_chan, MHI_CMD_RESET_CHAN);
1336 goto error_invalid_state;
1338 /* even if it fails we will still reset */
1339 ret = wait_for_completion_timeout(&mhi_chan->completion,
1340 msecs_to_jiffies(mhi_cntrl->timeout_ms));
1341 if (!ret || mhi_chan->ccs != MHI_EV_CC_SUCCESS)
1343 "Failed to receive cmd completion, still resetting\n");
1345 error_invalid_state:
1346 if (!mhi_chan->offload_ch) {
1347 mhi_reset_chan(mhi_cntrl, mhi_chan);
1348 mhi_deinit_chan_ctxt(mhi_cntrl, mhi_chan);
1350 dev_dbg(dev, "chan:%d successfully resetted\n", mhi_chan->chan);
1351 mutex_unlock(&mhi_chan->mutex);
1354 int mhi_prepare_channel(struct mhi_controller *mhi_cntrl,
1355 struct mhi_chan *mhi_chan)
1358 struct device *dev = &mhi_cntrl->mhi_dev->dev;
1360 dev_dbg(dev, "Preparing channel: %d\n", mhi_chan->chan);
1362 if (!(BIT(mhi_cntrl->ee) & mhi_chan->ee_mask)) {
1364 "Current EE: %s Required EE Mask: 0x%x for chan: %s\n",
1365 TO_MHI_EXEC_STR(mhi_cntrl->ee), mhi_chan->ee_mask,
1370 mutex_lock(&mhi_chan->mutex);
1372 /* If channel is not in disable state, do not allow it to start */
1373 if (mhi_chan->ch_state != MHI_CH_STATE_DISABLED) {
1375 dev_dbg(dev, "channel: %d is not in disabled state\n",
1377 goto error_init_chan;
1380 /* Check of client manages channel context for offload channels */
1381 if (!mhi_chan->offload_ch) {
1382 ret = mhi_init_chan_ctxt(mhi_cntrl, mhi_chan);
1384 goto error_init_chan;
1387 reinit_completion(&mhi_chan->completion);
1388 read_lock_bh(&mhi_cntrl->pm_lock);
1389 if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
1390 read_unlock_bh(&mhi_cntrl->pm_lock);
1392 goto error_pm_state;
1395 mhi_cntrl->wake_toggle(mhi_cntrl);
1396 read_unlock_bh(&mhi_cntrl->pm_lock);
1397 mhi_cntrl->runtime_get(mhi_cntrl);
1398 mhi_cntrl->runtime_put(mhi_cntrl);
1400 ret = mhi_send_cmd(mhi_cntrl, mhi_chan, MHI_CMD_START_CHAN);
1402 goto error_pm_state;
1404 ret = wait_for_completion_timeout(&mhi_chan->completion,
1405 msecs_to_jiffies(mhi_cntrl->timeout_ms));
1406 if (!ret || mhi_chan->ccs != MHI_EV_CC_SUCCESS) {
1408 goto error_pm_state;
1411 write_lock_irq(&mhi_chan->lock);
1412 mhi_chan->ch_state = MHI_CH_STATE_ENABLED;
1413 write_unlock_irq(&mhi_chan->lock);
1415 /* Pre-allocate buffer for xfer ring */
1416 if (mhi_chan->pre_alloc) {
1417 int nr_el = get_nr_avail_ring_elements(mhi_cntrl,
1418 &mhi_chan->tre_ring);
1419 size_t len = mhi_cntrl->buffer_len;
1423 struct mhi_buf_info info = { };
1424 buf = kmalloc(len, GFP_KERNEL);
1427 goto error_pre_alloc;
1430 /* Prepare transfer descriptors */
1434 ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &info, MHI_EOT);
1437 goto error_pre_alloc;
1441 read_lock_bh(&mhi_cntrl->pm_lock);
1442 if (MHI_DB_ACCESS_VALID(mhi_cntrl)) {
1443 read_lock_irq(&mhi_chan->lock);
1444 mhi_ring_chan_db(mhi_cntrl, mhi_chan);
1445 read_unlock_irq(&mhi_chan->lock);
1447 read_unlock_bh(&mhi_cntrl->pm_lock);
1450 mutex_unlock(&mhi_chan->mutex);
1452 dev_dbg(dev, "Chan: %d successfully moved to start state\n",
1458 if (!mhi_chan->offload_ch)
1459 mhi_deinit_chan_ctxt(mhi_cntrl, mhi_chan);
1462 mutex_unlock(&mhi_chan->mutex);
1467 mutex_unlock(&mhi_chan->mutex);
1468 __mhi_unprepare_channel(mhi_cntrl, mhi_chan);
1473 static void mhi_mark_stale_events(struct mhi_controller *mhi_cntrl,
1474 struct mhi_event *mhi_event,
1475 struct mhi_event_ctxt *er_ctxt,
1479 struct mhi_tre *dev_rp, *local_rp;
1480 struct mhi_ring *ev_ring;
1481 struct device *dev = &mhi_cntrl->mhi_dev->dev;
1482 unsigned long flags;
1485 dev_dbg(dev, "Marking all events for chan: %d as stale\n", chan);
1487 ev_ring = &mhi_event->ring;
1489 /* mark all stale events related to channel as STALE event */
1490 spin_lock_irqsave(&mhi_event->lock, flags);
1493 if (!is_valid_ring_ptr(ev_ring, ptr)) {
1494 dev_err(&mhi_cntrl->mhi_dev->dev,
1495 "Event ring rp points outside of the event ring\n");
1496 dev_rp = ev_ring->rp;
1498 dev_rp = mhi_to_virtual(ev_ring, ptr);
1501 local_rp = ev_ring->rp;
1502 while (dev_rp != local_rp) {
1503 if (MHI_TRE_GET_EV_TYPE(local_rp) == MHI_PKT_TYPE_TX_EVENT &&
1504 chan == MHI_TRE_GET_EV_CHID(local_rp))
1505 local_rp->dword[1] = MHI_TRE_EV_DWORD1(chan,
1506 MHI_PKT_TYPE_STALE_EVENT);
1508 if (local_rp == (ev_ring->base + ev_ring->len))
1509 local_rp = ev_ring->base;
1512 dev_dbg(dev, "Finished marking events as stale events\n");
1513 spin_unlock_irqrestore(&mhi_event->lock, flags);
1516 static void mhi_reset_data_chan(struct mhi_controller *mhi_cntrl,
1517 struct mhi_chan *mhi_chan)
1519 struct mhi_ring *buf_ring, *tre_ring;
1520 struct mhi_result result;
1522 /* Reset any pending buffers */
1523 buf_ring = &mhi_chan->buf_ring;
1524 tre_ring = &mhi_chan->tre_ring;
1525 result.transaction_status = -ENOTCONN;
1526 result.bytes_xferd = 0;
1527 while (tre_ring->rp != tre_ring->wp) {
1528 struct mhi_buf_info *buf_info = buf_ring->rp;
1530 if (mhi_chan->dir == DMA_TO_DEVICE)
1531 atomic_dec(&mhi_cntrl->pending_pkts);
1533 if (!buf_info->pre_mapped)
1534 mhi_cntrl->unmap_single(mhi_cntrl, buf_info);
1536 mhi_del_ring_element(mhi_cntrl, buf_ring);
1537 mhi_del_ring_element(mhi_cntrl, tre_ring);
1539 if (mhi_chan->pre_alloc) {
1540 kfree(buf_info->cb_buf);
1542 result.buf_addr = buf_info->cb_buf;
1543 mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
1548 void mhi_reset_chan(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan)
1550 struct mhi_event *mhi_event;
1551 struct mhi_event_ctxt *er_ctxt;
1552 int chan = mhi_chan->chan;
1554 /* Nothing to reset, client doesn't queue buffers */
1555 if (mhi_chan->offload_ch)
1558 read_lock_bh(&mhi_cntrl->pm_lock);
1559 mhi_event = &mhi_cntrl->mhi_event[mhi_chan->er_index];
1560 er_ctxt = &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_chan->er_index];
1562 mhi_mark_stale_events(mhi_cntrl, mhi_event, er_ctxt, chan);
1564 mhi_reset_data_chan(mhi_cntrl, mhi_chan);
1566 read_unlock_bh(&mhi_cntrl->pm_lock);
1569 /* Move channel to start state */
1570 int mhi_prepare_for_transfer(struct mhi_device *mhi_dev)
1573 struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
1574 struct mhi_chan *mhi_chan;
1576 for (dir = 0; dir < 2; dir++) {
1577 mhi_chan = dir ? mhi_dev->dl_chan : mhi_dev->ul_chan;
1581 ret = mhi_prepare_channel(mhi_cntrl, mhi_chan);
1583 goto error_open_chan;
1589 for (--dir; dir >= 0; dir--) {
1590 mhi_chan = dir ? mhi_dev->dl_chan : mhi_dev->ul_chan;
1594 __mhi_unprepare_channel(mhi_cntrl, mhi_chan);
1599 EXPORT_SYMBOL_GPL(mhi_prepare_for_transfer);
1601 void mhi_unprepare_from_transfer(struct mhi_device *mhi_dev)
1603 struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
1604 struct mhi_chan *mhi_chan;
1607 for (dir = 0; dir < 2; dir++) {
1608 mhi_chan = dir ? mhi_dev->ul_chan : mhi_dev->dl_chan;
1612 __mhi_unprepare_channel(mhi_cntrl, mhi_chan);
1615 EXPORT_SYMBOL_GPL(mhi_unprepare_from_transfer);
1617 int mhi_poll(struct mhi_device *mhi_dev, u32 budget)
1619 struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
1620 struct mhi_chan *mhi_chan = mhi_dev->dl_chan;
1621 struct mhi_event *mhi_event = &mhi_cntrl->mhi_event[mhi_chan->er_index];
1624 spin_lock_bh(&mhi_event->lock);
1625 ret = mhi_event->process_event(mhi_cntrl, mhi_event, budget);
1626 spin_unlock_bh(&mhi_event->lock);
1630 EXPORT_SYMBOL_GPL(mhi_poll);