1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2019 MediaTek Inc.
5 * Stanley Chu <stanley.chu@mediatek.com>
6 * Peter Wang <peter.wang@mediatek.com>
9 #include <linux/arm-smccc.h>
10 #include <linux/bitfield.h>
11 #include <linux/clk.h>
12 #include <linux/delay.h>
13 #include <linux/module.h>
15 #include <linux/of_address.h>
16 #include <linux/of_device.h>
17 #include <linux/of_platform.h>
18 #include <linux/phy/phy.h>
19 #include <linux/platform_device.h>
20 #include <linux/pm_qos.h>
21 #include <linux/regulator/consumer.h>
22 #include <linux/reset.h>
23 #include <linux/soc/mediatek/mtk_sip_svc.h>
25 #include <ufs/ufshcd.h>
26 #include "ufshcd-pltfrm.h"
27 #include <ufs/ufs_quirks.h>
28 #include <ufs/unipro.h>
29 #include "ufs-mediatek.h"
31 static int ufs_mtk_config_mcq(struct ufs_hba *hba, bool irq);
33 #define CREATE_TRACE_POINTS
34 #include "ufs-mediatek-trace.h"
35 #undef CREATE_TRACE_POINTS
37 #define MAX_SUPP_MAC 64
38 #define MCQ_QUEUE_OFFSET(c) ((((c) >> 16) & 0xFF) * 0x200)
40 static const struct ufs_dev_quirk ufs_mtk_dev_fixups[] = {
41 { .wmanufacturerid = UFS_ANY_VENDOR,
42 .model = UFS_ANY_MODEL,
43 .quirk = UFS_DEVICE_QUIRK_DELAY_AFTER_LPM |
44 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM },
45 { .wmanufacturerid = UFS_VENDOR_SKHYNIX,
46 .model = "H9HQ21AFAMZDAR",
47 .quirk = UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES },
51 static const struct of_device_id ufs_mtk_of_match[] = {
52 { .compatible = "mediatek,mt8183-ufshci" },
57 * Details of UIC Errors
59 static const char *const ufs_uic_err_str[] = {
63 "Transport Link Layer",
67 static const char *const ufs_uic_pa_err_str[] = {
68 "PHY error on Lane 0",
69 "PHY error on Lane 1",
70 "PHY error on Lane 2",
71 "PHY error on Lane 3",
72 "Generic PHY Adapter Error. This should be the LINERESET indication"
75 static const char *const ufs_uic_dl_err_str[] = {
77 "TCx_REPLAY_TIMER_EXPIRED",
78 "AFCx_REQUEST_TIMER_EXPIRED",
79 "FCx_PROTECTION_TIMER_EXPIRED",
82 "MAX_FRAME_LENGTH_EXCEEDED",
83 "WRONG_SEQUENCE_NUMBER",
84 "AFC_FRAME_SYNTAX_ERROR",
85 "NAC_FRAME_SYNTAX_ERROR",
88 "BAD_CTRL_SYMBOL_TYPE",
90 "PA_ERROR_IND_RECEIVED",
94 static bool ufs_mtk_is_boost_crypt_enabled(struct ufs_hba *hba)
96 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
98 return !!(host->caps & UFS_MTK_CAP_BOOST_CRYPT_ENGINE);
101 static bool ufs_mtk_is_va09_supported(struct ufs_hba *hba)
103 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
105 return !!(host->caps & UFS_MTK_CAP_VA09_PWR_CTRL);
108 static bool ufs_mtk_is_broken_vcc(struct ufs_hba *hba)
110 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
112 return !!(host->caps & UFS_MTK_CAP_BROKEN_VCC);
115 static bool ufs_mtk_is_pmc_via_fastauto(struct ufs_hba *hba)
117 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
119 return !!(host->caps & UFS_MTK_CAP_PMC_VIA_FASTAUTO);
122 static void ufs_mtk_cfg_unipro_cg(struct ufs_hba *hba, bool enable)
128 UIC_ARG_MIB(VS_SAVEPOWERCONTROL), &tmp);
130 (1 << RX_SYMBOL_CLK_GATE_EN) |
131 (1 << SYS_CLK_GATE_EN) |
132 (1 << TX_CLK_GATE_EN);
134 UIC_ARG_MIB(VS_SAVEPOWERCONTROL), tmp);
137 UIC_ARG_MIB(VS_DEBUGCLOCKENABLE), &tmp);
138 tmp = tmp & ~(1 << TX_SYMBOL_CLK_REQ_FORCE);
140 UIC_ARG_MIB(VS_DEBUGCLOCKENABLE), tmp);
143 UIC_ARG_MIB(VS_SAVEPOWERCONTROL), &tmp);
144 tmp = tmp & ~((1 << RX_SYMBOL_CLK_GATE_EN) |
145 (1 << SYS_CLK_GATE_EN) |
146 (1 << TX_CLK_GATE_EN));
148 UIC_ARG_MIB(VS_SAVEPOWERCONTROL), tmp);
151 UIC_ARG_MIB(VS_DEBUGCLOCKENABLE), &tmp);
152 tmp = tmp | (1 << TX_SYMBOL_CLK_REQ_FORCE);
154 UIC_ARG_MIB(VS_DEBUGCLOCKENABLE), tmp);
158 static void ufs_mtk_crypto_enable(struct ufs_hba *hba)
160 struct arm_smccc_res res;
162 ufs_mtk_crypto_ctrl(res, 1);
164 dev_info(hba->dev, "%s: crypto enable failed, err: %lu\n",
166 hba->caps &= ~UFSHCD_CAP_CRYPTO;
170 static void ufs_mtk_host_reset(struct ufs_hba *hba)
172 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
174 reset_control_assert(host->hci_reset);
175 reset_control_assert(host->crypto_reset);
176 reset_control_assert(host->unipro_reset);
178 usleep_range(100, 110);
180 reset_control_deassert(host->unipro_reset);
181 reset_control_deassert(host->crypto_reset);
182 reset_control_deassert(host->hci_reset);
185 static void ufs_mtk_init_reset_control(struct ufs_hba *hba,
186 struct reset_control **rc,
189 *rc = devm_reset_control_get(hba->dev, str);
191 dev_info(hba->dev, "Failed to get reset control %s: %ld\n",
197 static void ufs_mtk_init_reset(struct ufs_hba *hba)
199 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
201 ufs_mtk_init_reset_control(hba, &host->hci_reset,
203 ufs_mtk_init_reset_control(hba, &host->unipro_reset,
205 ufs_mtk_init_reset_control(hba, &host->crypto_reset,
209 static int ufs_mtk_hce_enable_notify(struct ufs_hba *hba,
210 enum ufs_notify_change_status status)
212 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
214 if (status == PRE_CHANGE) {
215 if (host->unipro_lpm) {
216 hba->vps->hba_enable_delay_us = 0;
218 hba->vps->hba_enable_delay_us = 600;
219 ufs_mtk_host_reset(hba);
222 if (hba->caps & UFSHCD_CAP_CRYPTO)
223 ufs_mtk_crypto_enable(hba);
225 if (host->caps & UFS_MTK_CAP_DISABLE_AH8) {
226 ufshcd_writel(hba, 0,
227 REG_AUTO_HIBERNATE_IDLE_TIMER);
228 hba->capabilities &= ~MASK_AUTO_HIBERN8_SUPPORT;
233 * Turn on CLK_CG early to bypass abnormal ERR_CHK signal
234 * to prevent host hang issue
237 ufshcd_readl(hba, REG_UFS_XOUFS_CTRL) | 0x80,
244 static int ufs_mtk_bind_mphy(struct ufs_hba *hba)
246 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
247 struct device *dev = hba->dev;
248 struct device_node *np = dev->of_node;
251 host->mphy = devm_of_phy_get_by_index(dev, np, 0);
253 if (host->mphy == ERR_PTR(-EPROBE_DEFER)) {
255 * UFS driver might be probed before the phy driver does.
256 * In that case we would like to return EPROBE_DEFER code.
260 "%s: required phy hasn't probed yet. err = %d\n",
262 } else if (IS_ERR(host->mphy)) {
263 err = PTR_ERR(host->mphy);
264 if (err != -ENODEV) {
265 dev_info(dev, "%s: PHY get failed %d\n", __func__,
273 * Allow unbound mphy because not every platform needs specific
282 static int ufs_mtk_setup_ref_clk(struct ufs_hba *hba, bool on)
284 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
285 struct arm_smccc_res res;
286 ktime_t timeout, time_checked;
289 if (host->ref_clk_enabled == on)
292 ufs_mtk_ref_clk_notify(on, PRE_CHANGE, res);
295 ufshcd_writel(hba, REFCLK_REQUEST, REG_UFS_REFCLK_CTRL);
297 ufshcd_delay_us(host->ref_clk_gating_wait_us, 10);
298 ufshcd_writel(hba, REFCLK_RELEASE, REG_UFS_REFCLK_CTRL);
302 timeout = ktime_add_us(ktime_get(), REFCLK_REQ_TIMEOUT_US);
304 time_checked = ktime_get();
305 value = ufshcd_readl(hba, REG_UFS_REFCLK_CTRL);
307 /* Wait until ack bit equals to req bit */
308 if (((value & REFCLK_ACK) >> 1) == (value & REFCLK_REQUEST))
311 usleep_range(100, 200);
312 } while (ktime_before(time_checked, timeout));
314 dev_err(hba->dev, "missing ack of refclk req, reg: 0x%x\n", value);
316 ufs_mtk_ref_clk_notify(host->ref_clk_enabled, POST_CHANGE, res);
321 host->ref_clk_enabled = on;
323 ufshcd_delay_us(host->ref_clk_ungating_wait_us, 10);
325 ufs_mtk_ref_clk_notify(on, POST_CHANGE, res);
330 static void ufs_mtk_setup_ref_clk_wait_us(struct ufs_hba *hba,
333 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
335 if (hba->dev_info.clk_gating_wait_us) {
336 host->ref_clk_gating_wait_us =
337 hba->dev_info.clk_gating_wait_us;
339 host->ref_clk_gating_wait_us = gating_us;
342 host->ref_clk_ungating_wait_us = REFCLK_DEFAULT_WAIT_US;
345 static void ufs_mtk_dbg_sel(struct ufs_hba *hba)
347 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
349 if (((host->ip_ver >> 16) & 0xFF) >= 0x36) {
350 ufshcd_writel(hba, 0x820820, REG_UFS_DEBUG_SEL);
351 ufshcd_writel(hba, 0x0, REG_UFS_DEBUG_SEL_B0);
352 ufshcd_writel(hba, 0x55555555, REG_UFS_DEBUG_SEL_B1);
353 ufshcd_writel(hba, 0xaaaaaaaa, REG_UFS_DEBUG_SEL_B2);
354 ufshcd_writel(hba, 0xffffffff, REG_UFS_DEBUG_SEL_B3);
356 ufshcd_writel(hba, 0x20, REG_UFS_DEBUG_SEL);
360 static void ufs_mtk_wait_idle_state(struct ufs_hba *hba,
361 unsigned long retry_ms)
363 u64 timeout, time_checked;
367 /* cannot use plain ktime_get() in suspend */
368 timeout = ktime_get_mono_fast_ns() + retry_ms * 1000000UL;
370 /* wait a specific time after check base */
375 time_checked = ktime_get_mono_fast_ns();
376 ufs_mtk_dbg_sel(hba);
377 val = ufshcd_readl(hba, REG_UFS_PROBE);
382 * if state is in H8 enter and H8 enter confirm
383 * wait until return to idle state.
385 if ((sm >= VS_HIB_ENTER) && (sm <= VS_HIB_EXIT)) {
389 } else if (!wait_idle)
392 if (wait_idle && (sm == VS_HCE_BASE))
394 } while (time_checked < timeout);
396 if (wait_idle && sm != VS_HCE_BASE)
397 dev_info(hba->dev, "wait idle tmo: 0x%x\n", val);
400 static int ufs_mtk_wait_link_state(struct ufs_hba *hba, u32 state,
401 unsigned long max_wait_ms)
403 ktime_t timeout, time_checked;
406 timeout = ktime_add_ms(ktime_get(), max_wait_ms);
408 time_checked = ktime_get();
409 ufs_mtk_dbg_sel(hba);
410 val = ufshcd_readl(hba, REG_UFS_PROBE);
416 /* Sleep for max. 200us */
417 usleep_range(100, 200);
418 } while (ktime_before(time_checked, timeout));
423 static int ufs_mtk_mphy_power_on(struct ufs_hba *hba, bool on)
425 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
426 struct phy *mphy = host->mphy;
427 struct arm_smccc_res res;
430 if (!mphy || !(on ^ host->mphy_powered_on))
434 if (ufs_mtk_is_va09_supported(hba)) {
435 ret = regulator_enable(host->reg_va09);
438 /* wait 200 us to stablize VA09 */
439 usleep_range(200, 210);
440 ufs_mtk_va09_pwr_ctrl(res, 1);
445 if (ufs_mtk_is_va09_supported(hba)) {
446 ufs_mtk_va09_pwr_ctrl(res, 0);
447 ret = regulator_disable(host->reg_va09);
453 "failed to %s va09: %d\n",
454 on ? "enable" : "disable",
457 host->mphy_powered_on = on;
463 static int ufs_mtk_get_host_clk(struct device *dev, const char *name,
464 struct clk **clk_out)
469 clk = devm_clk_get(dev, name);
478 static void ufs_mtk_boost_crypt(struct ufs_hba *hba, bool boost)
480 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
481 struct ufs_mtk_crypt_cfg *cfg;
482 struct regulator *reg;
485 if (!ufs_mtk_is_boost_crypt_enabled(hba))
489 volt = cfg->vcore_volt;
490 reg = cfg->reg_vcore;
492 ret = clk_prepare_enable(cfg->clk_crypt_mux);
494 dev_info(hba->dev, "clk_prepare_enable(): %d\n",
500 ret = regulator_set_voltage(reg, volt, INT_MAX);
503 "failed to set vcore to %d\n", volt);
507 ret = clk_set_parent(cfg->clk_crypt_mux,
508 cfg->clk_crypt_perf);
511 "failed to set clk_crypt_perf\n");
512 regulator_set_voltage(reg, 0, INT_MAX);
516 ret = clk_set_parent(cfg->clk_crypt_mux,
520 "failed to set clk_crypt_lp\n");
524 ret = regulator_set_voltage(reg, 0, INT_MAX);
527 "failed to set vcore to MIN\n");
531 clk_disable_unprepare(cfg->clk_crypt_mux);
534 static int ufs_mtk_init_host_clk(struct ufs_hba *hba, const char *name,
539 ret = ufs_mtk_get_host_clk(hba->dev, name, clk);
541 dev_info(hba->dev, "%s: failed to get %s: %d", __func__,
548 static void ufs_mtk_init_boost_crypt(struct ufs_hba *hba)
550 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
551 struct ufs_mtk_crypt_cfg *cfg;
552 struct device *dev = hba->dev;
553 struct regulator *reg;
556 host->crypt = devm_kzalloc(dev, sizeof(*(host->crypt)),
561 reg = devm_regulator_get_optional(dev, "dvfsrc-vcore");
563 dev_info(dev, "failed to get dvfsrc-vcore: %ld",
568 if (of_property_read_u32(dev->of_node, "boost-crypt-vcore-min",
570 dev_info(dev, "failed to get boost-crypt-vcore-min");
575 if (ufs_mtk_init_host_clk(hba, "crypt_mux",
576 &cfg->clk_crypt_mux))
579 if (ufs_mtk_init_host_clk(hba, "crypt_lp",
583 if (ufs_mtk_init_host_clk(hba, "crypt_perf",
584 &cfg->clk_crypt_perf))
587 cfg->reg_vcore = reg;
588 cfg->vcore_volt = volt;
589 host->caps |= UFS_MTK_CAP_BOOST_CRYPT_ENGINE;
595 static void ufs_mtk_init_va09_pwr_ctrl(struct ufs_hba *hba)
597 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
599 host->reg_va09 = regulator_get(hba->dev, "va09");
600 if (IS_ERR(host->reg_va09))
601 dev_info(hba->dev, "failed to get va09");
603 host->caps |= UFS_MTK_CAP_VA09_PWR_CTRL;
606 static void ufs_mtk_init_host_caps(struct ufs_hba *hba)
608 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
609 struct device_node *np = hba->dev->of_node;
611 if (of_property_read_bool(np, "mediatek,ufs-boost-crypt"))
612 ufs_mtk_init_boost_crypt(hba);
614 if (of_property_read_bool(np, "mediatek,ufs-support-va09"))
615 ufs_mtk_init_va09_pwr_ctrl(hba);
617 if (of_property_read_bool(np, "mediatek,ufs-disable-ah8"))
618 host->caps |= UFS_MTK_CAP_DISABLE_AH8;
620 if (of_property_read_bool(np, "mediatek,ufs-broken-vcc"))
621 host->caps |= UFS_MTK_CAP_BROKEN_VCC;
623 if (of_property_read_bool(np, "mediatek,ufs-pmc-via-fastauto"))
624 host->caps |= UFS_MTK_CAP_PMC_VIA_FASTAUTO;
626 dev_info(hba->dev, "caps: 0x%x", host->caps);
629 static void ufs_mtk_boost_pm_qos(struct ufs_hba *hba, bool boost)
631 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
633 if (!host || !host->pm_qos_init)
636 cpu_latency_qos_update_request(&host->pm_qos_req,
637 boost ? 0 : PM_QOS_DEFAULT_VALUE);
640 static void ufs_mtk_scale_perf(struct ufs_hba *hba, bool scale_up)
642 ufs_mtk_boost_crypt(hba, scale_up);
643 ufs_mtk_boost_pm_qos(hba, scale_up);
646 static void ufs_mtk_pwr_ctrl(struct ufs_hba *hba, bool on)
648 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
651 phy_power_on(host->mphy);
652 ufs_mtk_setup_ref_clk(hba, on);
653 if (!ufshcd_is_clkscaling_supported(hba))
654 ufs_mtk_scale_perf(hba, on);
656 if (!ufshcd_is_clkscaling_supported(hba))
657 ufs_mtk_scale_perf(hba, on);
658 ufs_mtk_setup_ref_clk(hba, on);
659 phy_power_off(host->mphy);
664 * ufs_mtk_setup_clocks - enables/disable clocks
665 * @hba: host controller instance
666 * @on: If true, enable clocks else disable them.
667 * @status: PRE_CHANGE or POST_CHANGE notify
669 * Return: 0 on success, non-zero on failure.
671 static int ufs_mtk_setup_clocks(struct ufs_hba *hba, bool on,
672 enum ufs_notify_change_status status)
674 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
675 bool clk_pwr_off = false;
679 * In case ufs_mtk_init() is not yet done, simply ignore.
680 * This ufs_mtk_setup_clocks() shall be called from
681 * ufs_mtk_init() after init is done.
686 if (!on && status == PRE_CHANGE) {
687 if (ufshcd_is_link_off(hba)) {
689 } else if (ufshcd_is_link_hibern8(hba) ||
690 (!ufshcd_can_hibern8_during_gating(hba) &&
691 ufshcd_is_auto_hibern8_enabled(hba))) {
693 * Gate ref-clk and poweroff mphy if link state is in
694 * OFF or Hibern8 by either Auto-Hibern8 or
695 * ufshcd_link_state_transition().
697 ret = ufs_mtk_wait_link_state(hba,
705 ufs_mtk_pwr_ctrl(hba, false);
706 } else if (on && status == POST_CHANGE) {
707 ufs_mtk_pwr_ctrl(hba, true);
713 static void ufs_mtk_get_controller_version(struct ufs_hba *hba)
715 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
718 if (host->hw_ver.major)
721 /* Set default (minimum) version anyway */
722 host->hw_ver.major = 2;
724 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_LOCALVERINFO), &ver);
726 if (ver >= UFS_UNIPRO_VER_1_8) {
727 host->hw_ver.major = 3;
729 * Fix HCI version for some platforms with
732 if (hba->ufs_version < ufshci_version(3, 0))
733 hba->ufs_version = ufshci_version(3, 0);
738 static u32 ufs_mtk_get_ufs_hci_version(struct ufs_hba *hba)
740 return hba->ufs_version;
744 * ufs_mtk_init_clocks - Init mtk driver private clocks
746 * @hba: per adapter instance
748 static void ufs_mtk_init_clocks(struct ufs_hba *hba)
750 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
751 struct list_head *head = &hba->clk_list_head;
752 struct ufs_mtk_clk *mclk = &host->mclk;
753 struct ufs_clk_info *clki, *clki_tmp;
756 * Find private clocks and store them in struct ufs_mtk_clk.
757 * Remove "ufs_sel_min_src" and "ufs_sel_min_src" from list to avoid
758 * being switched on/off in clock gating.
760 list_for_each_entry_safe(clki, clki_tmp, head, list) {
761 if (!strcmp(clki->name, "ufs_sel")) {
762 host->mclk.ufs_sel_clki = clki;
763 } else if (!strcmp(clki->name, "ufs_sel_max_src")) {
764 host->mclk.ufs_sel_max_clki = clki;
765 clk_disable_unprepare(clki->clk);
766 list_del(&clki->list);
767 } else if (!strcmp(clki->name, "ufs_sel_min_src")) {
768 host->mclk.ufs_sel_min_clki = clki;
769 clk_disable_unprepare(clki->clk);
770 list_del(&clki->list);
774 if (!mclk->ufs_sel_clki || !mclk->ufs_sel_max_clki ||
775 !mclk->ufs_sel_min_clki) {
776 hba->caps &= ~UFSHCD_CAP_CLK_SCALING;
778 "%s: Clk-scaling not ready. Feature disabled.",
783 #define MAX_VCC_NAME 30
784 static int ufs_mtk_vreg_fix_vcc(struct ufs_hba *hba)
786 struct ufs_vreg_info *info = &hba->vreg_info;
787 struct device_node *np = hba->dev->of_node;
788 struct device *dev = hba->dev;
789 char vcc_name[MAX_VCC_NAME];
790 struct arm_smccc_res res;
793 if (hba->vreg_info.vcc)
796 if (of_property_read_bool(np, "mediatek,ufs-vcc-by-num")) {
797 ufs_mtk_get_vcc_num(res);
798 if (res.a1 > UFS_VCC_NONE && res.a1 < UFS_VCC_MAX)
799 snprintf(vcc_name, MAX_VCC_NAME, "vcc-opt%lu", res.a1);
802 } else if (of_property_read_bool(np, "mediatek,ufs-vcc-by-ver")) {
803 ver = (hba->dev_info.wspecversion & 0xF00) >> 8;
804 snprintf(vcc_name, MAX_VCC_NAME, "vcc-ufs%u", ver);
809 err = ufshcd_populate_vreg(dev, vcc_name, &info->vcc, false);
813 err = ufshcd_get_vreg(dev, info->vcc);
817 err = regulator_enable(info->vcc->reg);
819 info->vcc->enabled = true;
820 dev_info(dev, "%s: %s enabled\n", __func__, vcc_name);
826 static void ufs_mtk_vreg_fix_vccqx(struct ufs_hba *hba)
828 struct ufs_vreg_info *info = &hba->vreg_info;
829 struct ufs_vreg **vreg_on, **vreg_off;
831 if (hba->dev_info.wspecversion >= 0x0300) {
832 vreg_on = &info->vccq;
833 vreg_off = &info->vccq2;
835 vreg_on = &info->vccq2;
836 vreg_off = &info->vccq;
840 (*vreg_on)->always_on = true;
843 regulator_disable((*vreg_off)->reg);
844 devm_kfree(hba->dev, (*vreg_off)->name);
845 devm_kfree(hba->dev, *vreg_off);
850 static void ufs_mtk_init_mcq_irq(struct ufs_hba *hba)
852 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
853 struct platform_device *pdev;
857 host->mcq_nr_intr = UFSHCD_MAX_Q_NR;
858 pdev = container_of(hba->dev, struct platform_device, dev);
860 for (i = 0; i < host->mcq_nr_intr; i++) {
861 /* irq index 0 is legacy irq, sq/cq irq start from index 1 */
862 irq = platform_get_irq(pdev, i + 1);
864 host->mcq_intr_info[i].irq = MTK_MCQ_INVALID_IRQ;
867 host->mcq_intr_info[i].hba = hba;
868 host->mcq_intr_info[i].irq = irq;
869 dev_info(hba->dev, "get platform mcq irq: %d, %d\n", i, irq);
874 /* invalidate irq info */
875 for (i = 0; i < host->mcq_nr_intr; i++)
876 host->mcq_intr_info[i].irq = MTK_MCQ_INVALID_IRQ;
878 host->mcq_nr_intr = 0;
882 * ufs_mtk_init - find other essential mmio bases
883 * @hba: host controller instance
885 * Binds PHY with controller and powers up PHY enabling clocks
888 * Return: -EPROBE_DEFER if binding fails, returns negative error
889 * on phy power up failure and returns zero on success.
891 static int ufs_mtk_init(struct ufs_hba *hba)
893 const struct of_device_id *id;
894 struct device *dev = hba->dev;
895 struct ufs_mtk_host *host;
898 host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
901 dev_info(dev, "%s: no memory for mtk ufs host\n", __func__);
906 ufshcd_set_variant(hba, host);
908 id = of_match_device(ufs_mtk_of_match, dev);
914 /* Initialize host capability */
915 ufs_mtk_init_host_caps(hba);
917 ufs_mtk_init_mcq_irq(hba);
919 err = ufs_mtk_bind_mphy(hba);
921 goto out_variant_clear;
923 ufs_mtk_init_reset(hba);
925 /* Enable runtime autosuspend */
926 hba->caps |= UFSHCD_CAP_RPM_AUTOSUSPEND;
928 /* Enable clock-gating */
929 hba->caps |= UFSHCD_CAP_CLK_GATING;
931 /* Enable inline encryption */
932 hba->caps |= UFSHCD_CAP_CRYPTO;
934 /* Enable WriteBooster */
935 hba->caps |= UFSHCD_CAP_WB_EN;
937 /* Enable clk scaling*/
938 hba->caps |= UFSHCD_CAP_CLK_SCALING;
940 hba->quirks |= UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL;
941 hba->quirks |= UFSHCD_QUIRK_MCQ_BROKEN_INTR;
942 hba->quirks |= UFSHCD_QUIRK_MCQ_BROKEN_RTC;
943 hba->vps->wb_flush_threshold = UFS_WB_BUF_REMAIN_PERCENT(80);
945 if (host->caps & UFS_MTK_CAP_DISABLE_AH8)
946 hba->caps |= UFSHCD_CAP_HIBERN8_WITH_CLK_GATING;
948 ufs_mtk_init_clocks(hba);
951 * ufshcd_vops_init() is invoked after
952 * ufshcd_setup_clock(true) in ufshcd_hba_init() thus
953 * phy clock setup is skipped.
955 * Enable phy clocks specifically here.
957 ufs_mtk_mphy_power_on(hba, true);
958 ufs_mtk_setup_clocks(hba, true, POST_CHANGE);
960 host->ip_ver = ufshcd_readl(hba, REG_UFS_MTK_IP_VER);
962 /* Initialize pm-qos request */
963 cpu_latency_qos_add_request(&host->pm_qos_req, PM_QOS_DEFAULT_VALUE);
964 host->pm_qos_init = true;
969 ufshcd_set_variant(hba, NULL);
974 static bool ufs_mtk_pmc_via_fastauto(struct ufs_hba *hba,
975 struct ufs_pa_layer_attr *dev_req_params)
977 if (!ufs_mtk_is_pmc_via_fastauto(hba))
980 if (dev_req_params->hs_rate == hba->pwr_info.hs_rate)
983 if (dev_req_params->pwr_tx != FAST_MODE &&
984 dev_req_params->gear_tx < UFS_HS_G4)
987 if (dev_req_params->pwr_rx != FAST_MODE &&
988 dev_req_params->gear_rx < UFS_HS_G4)
994 static int ufs_mtk_pre_pwr_change(struct ufs_hba *hba,
995 struct ufs_pa_layer_attr *dev_max_params,
996 struct ufs_pa_layer_attr *dev_req_params)
998 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
999 struct ufs_host_params host_params;
1002 ufshcd_init_host_params(&host_params);
1003 host_params.hs_rx_gear = UFS_HS_G5;
1004 host_params.hs_tx_gear = UFS_HS_G5;
1006 ret = ufshcd_negotiate_pwr_params(&host_params, dev_max_params, dev_req_params);
1008 pr_info("%s: failed to determine capabilities\n",
1012 if (ufs_mtk_pmc_via_fastauto(hba, dev_req_params)) {
1013 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), true);
1014 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), UFS_HS_G1);
1016 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), true);
1017 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), UFS_HS_G1);
1019 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
1020 dev_req_params->lane_tx);
1021 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
1022 dev_req_params->lane_rx);
1023 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
1024 dev_req_params->hs_rate);
1026 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXHSADAPTTYPE),
1029 ret = ufshcd_uic_change_pwr_mode(hba,
1030 FASTAUTO_MODE << 4 | FASTAUTO_MODE);
1033 dev_err(hba->dev, "%s: HSG1B FASTAUTO failed ret=%d\n",
1038 if (host->hw_ver.major >= 3) {
1039 ret = ufshcd_dme_configure_adapt(hba,
1040 dev_req_params->gear_tx,
1047 static int ufs_mtk_pwr_change_notify(struct ufs_hba *hba,
1048 enum ufs_notify_change_status stage,
1049 struct ufs_pa_layer_attr *dev_max_params,
1050 struct ufs_pa_layer_attr *dev_req_params)
1056 ret = ufs_mtk_pre_pwr_change(hba, dev_max_params,
1069 static int ufs_mtk_unipro_set_lpm(struct ufs_hba *hba, bool lpm)
1072 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
1074 ret = ufshcd_dme_set(hba,
1075 UIC_ARG_MIB_SEL(VS_UNIPROPOWERDOWNCONTROL, 0),
1079 * Forcibly set as non-LPM mode if UIC commands is failed
1080 * to use default hba_enable_delay_us value for re-enabling
1083 host->unipro_lpm = lpm;
1089 static int ufs_mtk_pre_link(struct ufs_hba *hba)
1094 ufs_mtk_get_controller_version(hba);
1096 ret = ufs_mtk_unipro_set_lpm(hba, false);
1101 * Setting PA_Local_TX_LCC_Enable to 0 before link startup
1102 * to make sure that both host and device TX LCC are disabled
1103 * once link startup is completed.
1105 ret = ufshcd_disable_host_tx_lcc(hba);
1109 /* disable deep stall */
1110 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(VS_SAVEPOWERCONTROL), &tmp);
1116 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(VS_SAVEPOWERCONTROL), tmp);
1121 static void ufs_mtk_setup_clk_gating(struct ufs_hba *hba)
1125 if (ufshcd_is_clkgating_allowed(hba)) {
1126 if (ufshcd_is_auto_hibern8_supported(hba) && hba->ahit)
1127 ah_ms = FIELD_GET(UFSHCI_AHIBERN8_TIMER_MASK,
1131 ufshcd_clkgate_delay_set(hba->dev, ah_ms + 5);
1135 static void ufs_mtk_post_link(struct ufs_hba *hba)
1137 /* enable unipro clock gating feature */
1138 ufs_mtk_cfg_unipro_cg(hba, true);
1140 /* will be configured during probe hba */
1141 if (ufshcd_is_auto_hibern8_supported(hba))
1142 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 10) |
1143 FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
1145 ufs_mtk_setup_clk_gating(hba);
1148 static int ufs_mtk_link_startup_notify(struct ufs_hba *hba,
1149 enum ufs_notify_change_status stage)
1155 ret = ufs_mtk_pre_link(hba);
1158 ufs_mtk_post_link(hba);
1168 static int ufs_mtk_device_reset(struct ufs_hba *hba)
1170 struct arm_smccc_res res;
1172 /* disable hba before device reset */
1173 ufshcd_hba_stop(hba);
1175 ufs_mtk_device_reset_ctrl(0, res);
1178 * The reset signal is active low. UFS devices shall detect
1179 * more than or equal to 1us of positive or negative RST_n
1182 * To be on safe side, keep the reset low for at least 10us.
1184 usleep_range(10, 15);
1186 ufs_mtk_device_reset_ctrl(1, res);
1188 /* Some devices may need time to respond to rst_n */
1189 usleep_range(10000, 15000);
1191 dev_info(hba->dev, "device reset done\n");
1196 static int ufs_mtk_link_set_hpm(struct ufs_hba *hba)
1200 err = ufshcd_hba_enable(hba);
1204 err = ufs_mtk_unipro_set_lpm(hba, false);
1208 err = ufshcd_uic_hibern8_exit(hba);
1210 ufshcd_set_link_active(hba);
1214 if (!hba->mcq_enabled) {
1215 err = ufshcd_make_hba_operational(hba);
1217 ufs_mtk_config_mcq(hba, false);
1218 ufshcd_mcq_make_queues_operational(hba);
1219 ufshcd_mcq_config_mac(hba, hba->nutrs);
1220 /* Enable MCQ mode */
1221 ufshcd_writel(hba, ufshcd_readl(hba, REG_UFS_MEM_CFG) | 0x1,
1231 static int ufs_mtk_link_set_lpm(struct ufs_hba *hba)
1235 /* Disable reset confirm feature by UniPro */
1237 (ufshcd_readl(hba, REG_UFS_XOUFS_CTRL) & ~0x100),
1238 REG_UFS_XOUFS_CTRL);
1240 err = ufs_mtk_unipro_set_lpm(hba, true);
1242 /* Resume UniPro state for following error recovery */
1243 ufs_mtk_unipro_set_lpm(hba, false);
1250 static void ufs_mtk_vccqx_set_lpm(struct ufs_hba *hba, bool lpm)
1252 struct ufs_vreg *vccqx = NULL;
1254 if (hba->vreg_info.vccq)
1255 vccqx = hba->vreg_info.vccq;
1257 vccqx = hba->vreg_info.vccq2;
1259 regulator_set_mode(vccqx->reg,
1260 lpm ? REGULATOR_MODE_IDLE : REGULATOR_MODE_NORMAL);
1263 static void ufs_mtk_vsx_set_lpm(struct ufs_hba *hba, bool lpm)
1265 struct arm_smccc_res res;
1267 ufs_mtk_device_pwr_ctrl(!lpm,
1268 (unsigned long)hba->dev_info.wspecversion,
1272 static void ufs_mtk_dev_vreg_set_lpm(struct ufs_hba *hba, bool lpm)
1274 if (!hba->vreg_info.vccq && !hba->vreg_info.vccq2)
1277 /* Skip if VCC is assumed always-on */
1278 if (!hba->vreg_info.vcc)
1281 /* Bypass LPM when device is still active */
1282 if (lpm && ufshcd_is_ufs_dev_active(hba))
1285 /* Bypass LPM if VCC is enabled */
1286 if (lpm && hba->vreg_info.vcc->enabled)
1290 ufs_mtk_vccqx_set_lpm(hba, lpm);
1291 ufs_mtk_vsx_set_lpm(hba, lpm);
1293 ufs_mtk_vsx_set_lpm(hba, lpm);
1294 ufs_mtk_vccqx_set_lpm(hba, lpm);
1298 static void ufs_mtk_auto_hibern8_disable(struct ufs_hba *hba)
1302 /* disable auto-hibern8 */
1303 ufshcd_writel(hba, 0, REG_AUTO_HIBERNATE_IDLE_TIMER);
1305 /* wait host return to idle state when auto-hibern8 off */
1306 ufs_mtk_wait_idle_state(hba, 5);
1308 ret = ufs_mtk_wait_link_state(hba, VS_LINK_UP, 100);
1310 dev_warn(hba->dev, "exit h8 state fail, ret=%d\n", ret);
1313 static int ufs_mtk_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op,
1314 enum ufs_notify_change_status status)
1317 struct arm_smccc_res res;
1319 if (status == PRE_CHANGE) {
1320 if (ufshcd_is_auto_hibern8_supported(hba))
1321 ufs_mtk_auto_hibern8_disable(hba);
1325 if (ufshcd_is_link_hibern8(hba)) {
1326 err = ufs_mtk_link_set_lpm(hba);
1331 if (!ufshcd_is_link_active(hba)) {
1333 * Make sure no error will be returned to prevent
1334 * ufshcd_suspend() re-enabling regulators while vreg is still
1335 * in low-power mode.
1337 err = ufs_mtk_mphy_power_on(hba, false);
1342 if (ufshcd_is_link_off(hba))
1343 ufs_mtk_device_reset_ctrl(0, res);
1345 ufs_mtk_host_pwr_ctrl(HOST_PWR_HCI, false, res);
1350 * Set link as off state enforcedly to trigger
1351 * ufshcd_host_reset_and_restore() in ufshcd_suspend()
1352 * for completed host reset.
1354 ufshcd_set_link_off(hba);
1358 static int ufs_mtk_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
1361 struct arm_smccc_res res;
1363 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL)
1364 ufs_mtk_dev_vreg_set_lpm(hba, false);
1366 ufs_mtk_host_pwr_ctrl(HOST_PWR_HCI, true, res);
1368 err = ufs_mtk_mphy_power_on(hba, true);
1372 if (ufshcd_is_link_hibern8(hba)) {
1373 err = ufs_mtk_link_set_hpm(hba);
1380 return ufshcd_link_recovery(hba);
1383 static void ufs_mtk_dbg_register_dump(struct ufs_hba *hba)
1385 /* Dump ufshci register 0x140 ~ 0x14C */
1386 ufshcd_dump_regs(hba, REG_UFS_XOUFS_CTRL, 0x10,
1387 "XOUFS Ctrl (0x140): ");
1389 ufshcd_dump_regs(hba, REG_UFS_EXTREG, 0x4, "Ext Reg ");
1391 /* Dump ufshci register 0x2200 ~ 0x22AC */
1392 ufshcd_dump_regs(hba, REG_UFS_MPHYCTRL,
1393 REG_UFS_REJECT_MON - REG_UFS_MPHYCTRL + 4,
1394 "MPHY Ctrl (0x2200): ");
1396 /* Direct debugging information to REG_MTK_PROBE */
1397 ufs_mtk_dbg_sel(hba);
1398 ufshcd_dump_regs(hba, REG_UFS_PROBE, 0x4, "Debug Probe ");
1401 static int ufs_mtk_apply_dev_quirks(struct ufs_hba *hba)
1403 struct ufs_dev_info *dev_info = &hba->dev_info;
1404 u16 mid = dev_info->wmanufacturerid;
1406 if (mid == UFS_VENDOR_SAMSUNG) {
1407 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 6);
1408 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME), 10);
1412 * Decide waiting time before gating reference clock and
1413 * after ungating reference clock according to vendors'
1416 if (mid == UFS_VENDOR_SAMSUNG)
1417 ufs_mtk_setup_ref_clk_wait_us(hba, 1);
1418 else if (mid == UFS_VENDOR_SKHYNIX)
1419 ufs_mtk_setup_ref_clk_wait_us(hba, 30);
1420 else if (mid == UFS_VENDOR_TOSHIBA)
1421 ufs_mtk_setup_ref_clk_wait_us(hba, 100);
1423 ufs_mtk_setup_ref_clk_wait_us(hba,
1424 REFCLK_DEFAULT_WAIT_US);
1428 static void ufs_mtk_fixup_dev_quirks(struct ufs_hba *hba)
1430 ufshcd_fixup_dev_quirks(hba, ufs_mtk_dev_fixups);
1432 if (ufs_mtk_is_broken_vcc(hba) && hba->vreg_info.vcc &&
1433 (hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM)) {
1434 hba->vreg_info.vcc->always_on = true;
1436 * VCC will be kept always-on thus we don't
1437 * need any delay during regulator operations
1439 hba->dev_quirks &= ~(UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM |
1440 UFS_DEVICE_QUIRK_DELAY_AFTER_LPM);
1443 ufs_mtk_vreg_fix_vcc(hba);
1444 ufs_mtk_vreg_fix_vccqx(hba);
1447 static void ufs_mtk_event_notify(struct ufs_hba *hba,
1448 enum ufs_event_type evt, void *data)
1450 unsigned int val = *(u32 *)data;
1454 trace_ufs_mtk_event(evt, val);
1456 /* Print details of UIC Errors */
1457 if (evt <= UFS_EVT_DME_ERR) {
1459 "Host UIC Error Code (%s): %08x\n",
1460 ufs_uic_err_str[evt], val);
1464 if (evt == UFS_EVT_PA_ERR) {
1465 for_each_set_bit(bit, ®, ARRAY_SIZE(ufs_uic_pa_err_str))
1466 dev_info(hba->dev, "%s\n", ufs_uic_pa_err_str[bit]);
1469 if (evt == UFS_EVT_DL_ERR) {
1470 for_each_set_bit(bit, ®, ARRAY_SIZE(ufs_uic_dl_err_str))
1471 dev_info(hba->dev, "%s\n", ufs_uic_dl_err_str[bit]);
1475 static void ufs_mtk_config_scaling_param(struct ufs_hba *hba,
1476 struct devfreq_dev_profile *profile,
1477 struct devfreq_simple_ondemand_data *data)
1479 /* Customize min gear in clk scaling */
1480 hba->clk_scaling.min_gear = UFS_HS_G4;
1482 hba->vps->devfreq_profile.polling_ms = 200;
1483 hba->vps->ondemand_data.upthreshold = 50;
1484 hba->vps->ondemand_data.downdifferential = 20;
1488 * ufs_mtk_clk_scale - Internal clk scaling operation
1490 * MTK platform supports clk scaling by switching parent of ufs_sel(mux).
1491 * The ufs_sel downstream to ufs_ck which feeds directly to UFS hardware.
1492 * Max and min clocks rate of ufs_sel defined in dts should match rate of
1493 * "ufs_sel_max_src" and "ufs_sel_min_src" respectively.
1494 * This prevent changing rate of pll clock that is shared between modules.
1496 * @hba: per adapter instance
1497 * @scale_up: True for scaling up and false for scaling down
1499 static void ufs_mtk_clk_scale(struct ufs_hba *hba, bool scale_up)
1501 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
1502 struct ufs_mtk_clk *mclk = &host->mclk;
1503 struct ufs_clk_info *clki = mclk->ufs_sel_clki;
1506 ret = clk_prepare_enable(clki->clk);
1509 "clk_prepare_enable() fail, ret: %d\n", ret);
1514 ret = clk_set_parent(clki->clk, mclk->ufs_sel_max_clki->clk);
1515 clki->curr_freq = clki->max_freq;
1517 ret = clk_set_parent(clki->clk, mclk->ufs_sel_min_clki->clk);
1518 clki->curr_freq = clki->min_freq;
1523 "Failed to set ufs_sel_clki, ret: %d\n", ret);
1526 clk_disable_unprepare(clki->clk);
1528 trace_ufs_mtk_clk_scale(clki->name, scale_up, clk_get_rate(clki->clk));
1531 static int ufs_mtk_clk_scale_notify(struct ufs_hba *hba, bool scale_up,
1532 enum ufs_notify_change_status status)
1534 if (!ufshcd_is_clkscaling_supported(hba))
1537 if (status == PRE_CHANGE) {
1538 /* Switch parent before clk_set_rate() */
1539 ufs_mtk_clk_scale(hba, scale_up);
1541 /* Request interrupt latency QoS accordingly */
1542 ufs_mtk_scale_perf(hba, scale_up);
1548 static int ufs_mtk_get_hba_mac(struct ufs_hba *hba)
1550 return MAX_SUPP_MAC;
1553 static int ufs_mtk_op_runtime_config(struct ufs_hba *hba)
1555 struct ufshcd_mcq_opr_info_t *opr;
1558 hba->mcq_opr[OPR_SQD].offset = REG_UFS_MTK_SQD;
1559 hba->mcq_opr[OPR_SQIS].offset = REG_UFS_MTK_SQIS;
1560 hba->mcq_opr[OPR_CQD].offset = REG_UFS_MTK_CQD;
1561 hba->mcq_opr[OPR_CQIS].offset = REG_UFS_MTK_CQIS;
1563 for (i = 0; i < OPR_MAX; i++) {
1564 opr = &hba->mcq_opr[i];
1565 opr->stride = REG_UFS_MCQ_STRIDE;
1566 opr->base = hba->mmio_base + opr->offset;
1572 static int ufs_mtk_mcq_config_resource(struct ufs_hba *hba)
1574 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
1576 /* fail mcq initialization if interrupt is not filled properly */
1577 if (!host->mcq_nr_intr) {
1578 dev_info(hba->dev, "IRQs not ready. MCQ disabled.");
1582 hba->mcq_base = hba->mmio_base + MCQ_QUEUE_OFFSET(hba->mcq_capabilities);
1586 static irqreturn_t ufs_mtk_mcq_intr(int irq, void *__intr_info)
1588 struct ufs_mtk_mcq_intr_info *mcq_intr_info = __intr_info;
1589 struct ufs_hba *hba = mcq_intr_info->hba;
1590 struct ufs_hw_queue *hwq;
1592 int qid = mcq_intr_info->qid;
1594 hwq = &hba->uhq[qid];
1596 events = ufshcd_mcq_read_cqis(hba, qid);
1598 ufshcd_mcq_write_cqis(hba, events, qid);
1600 if (events & UFSHCD_MCQ_CQIS_TAIL_ENT_PUSH_STS)
1601 ufshcd_mcq_poll_cqe_lock(hba, hwq);
1606 static int ufs_mtk_config_mcq_irq(struct ufs_hba *hba)
1608 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
1612 for (i = 0; i < host->mcq_nr_intr; i++) {
1613 irq = host->mcq_intr_info[i].irq;
1614 if (irq == MTK_MCQ_INVALID_IRQ) {
1615 dev_err(hba->dev, "invalid irq. %d\n", i);
1619 host->mcq_intr_info[i].qid = i;
1620 ret = devm_request_irq(hba->dev, irq, ufs_mtk_mcq_intr, 0, UFSHCD,
1621 &host->mcq_intr_info[i]);
1623 dev_dbg(hba->dev, "request irq %d intr %s\n", irq, ret ? "failed" : "");
1626 dev_err(hba->dev, "Cannot request irq %d\n", ret);
1634 static int ufs_mtk_config_mcq(struct ufs_hba *hba, bool irq)
1636 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
1639 if (!host->mcq_set_intr) {
1640 /* Disable irq option register */
1641 ufshcd_rmwl(hba, MCQ_INTR_EN_MSK, 0, REG_UFS_MMIO_OPT_CTRL_0);
1644 ret = ufs_mtk_config_mcq_irq(hba);
1649 host->mcq_set_intr = true;
1652 ufshcd_rmwl(hba, MCQ_AH8, MCQ_AH8, REG_UFS_MMIO_OPT_CTRL_0);
1653 ufshcd_rmwl(hba, MCQ_INTR_EN_MSK, MCQ_MULTI_INTR_EN, REG_UFS_MMIO_OPT_CTRL_0);
1658 static int ufs_mtk_config_esi(struct ufs_hba *hba)
1660 return ufs_mtk_config_mcq(hba, true);
1664 * struct ufs_hba_mtk_vops - UFS MTK specific variant operations
1666 * The variant operations configure the necessary controller and PHY
1667 * handshake during initialization.
1669 static const struct ufs_hba_variant_ops ufs_hba_mtk_vops = {
1670 .name = "mediatek.ufshci",
1671 .init = ufs_mtk_init,
1672 .get_ufs_hci_version = ufs_mtk_get_ufs_hci_version,
1673 .setup_clocks = ufs_mtk_setup_clocks,
1674 .hce_enable_notify = ufs_mtk_hce_enable_notify,
1675 .link_startup_notify = ufs_mtk_link_startup_notify,
1676 .pwr_change_notify = ufs_mtk_pwr_change_notify,
1677 .apply_dev_quirks = ufs_mtk_apply_dev_quirks,
1678 .fixup_dev_quirks = ufs_mtk_fixup_dev_quirks,
1679 .suspend = ufs_mtk_suspend,
1680 .resume = ufs_mtk_resume,
1681 .dbg_register_dump = ufs_mtk_dbg_register_dump,
1682 .device_reset = ufs_mtk_device_reset,
1683 .event_notify = ufs_mtk_event_notify,
1684 .config_scaling_param = ufs_mtk_config_scaling_param,
1685 .clk_scale_notify = ufs_mtk_clk_scale_notify,
1687 .get_hba_mac = ufs_mtk_get_hba_mac,
1688 .op_runtime_config = ufs_mtk_op_runtime_config,
1689 .mcq_config_resource = ufs_mtk_mcq_config_resource,
1690 .config_esi = ufs_mtk_config_esi,
1694 * ufs_mtk_probe - probe routine of the driver
1695 * @pdev: pointer to Platform device handle
1697 * Return: zero for success and non-zero for failure.
1699 static int ufs_mtk_probe(struct platform_device *pdev)
1702 struct device *dev = &pdev->dev;
1703 struct device_node *reset_node;
1704 struct platform_device *reset_pdev;
1705 struct device_link *link;
1707 reset_node = of_find_compatible_node(NULL, NULL,
1710 dev_notice(dev, "find ti,syscon-reset fail\n");
1713 reset_pdev = of_find_device_by_node(reset_node);
1715 dev_notice(dev, "find reset_pdev fail\n");
1718 link = device_link_add(dev, &reset_pdev->dev,
1719 DL_FLAG_AUTOPROBE_CONSUMER);
1720 put_device(&reset_pdev->dev);
1722 dev_notice(dev, "add reset device_link fail\n");
1725 /* supplier is not probed */
1726 if (link->status == DL_STATE_DORMANT) {
1727 err = -EPROBE_DEFER;
1732 /* perform generic probe */
1733 err = ufshcd_pltfrm_init(pdev, &ufs_hba_mtk_vops);
1737 dev_err(dev, "probe failed %d\n", err);
1739 of_node_put(reset_node);
1744 * ufs_mtk_remove - set driver_data of the device to NULL
1745 * @pdev: pointer to platform device handle
1749 static void ufs_mtk_remove(struct platform_device *pdev)
1751 struct ufs_hba *hba = platform_get_drvdata(pdev);
1753 pm_runtime_get_sync(&(pdev)->dev);
1757 #ifdef CONFIG_PM_SLEEP
1758 static int ufs_mtk_system_suspend(struct device *dev)
1760 struct ufs_hba *hba = dev_get_drvdata(dev);
1763 ret = ufshcd_system_suspend(dev);
1767 ufs_mtk_dev_vreg_set_lpm(hba, true);
1772 static int ufs_mtk_system_resume(struct device *dev)
1774 struct ufs_hba *hba = dev_get_drvdata(dev);
1776 ufs_mtk_dev_vreg_set_lpm(hba, false);
1778 return ufshcd_system_resume(dev);
1783 static int ufs_mtk_runtime_suspend(struct device *dev)
1785 struct ufs_hba *hba = dev_get_drvdata(dev);
1788 ret = ufshcd_runtime_suspend(dev);
1792 ufs_mtk_dev_vreg_set_lpm(hba, true);
1797 static int ufs_mtk_runtime_resume(struct device *dev)
1799 struct ufs_hba *hba = dev_get_drvdata(dev);
1801 ufs_mtk_dev_vreg_set_lpm(hba, false);
1803 return ufshcd_runtime_resume(dev);
1807 static const struct dev_pm_ops ufs_mtk_pm_ops = {
1808 SET_SYSTEM_SLEEP_PM_OPS(ufs_mtk_system_suspend,
1809 ufs_mtk_system_resume)
1810 SET_RUNTIME_PM_OPS(ufs_mtk_runtime_suspend,
1811 ufs_mtk_runtime_resume, NULL)
1812 .prepare = ufshcd_suspend_prepare,
1813 .complete = ufshcd_resume_complete,
1816 static struct platform_driver ufs_mtk_pltform = {
1817 .probe = ufs_mtk_probe,
1818 .remove_new = ufs_mtk_remove,
1820 .name = "ufshcd-mtk",
1821 .pm = &ufs_mtk_pm_ops,
1822 .of_match_table = ufs_mtk_of_match,
1826 MODULE_AUTHOR("Stanley Chu <stanley.chu@mediatek.com>");
1827 MODULE_AUTHOR("Peter Wang <peter.wang@mediatek.com>");
1828 MODULE_DESCRIPTION("MediaTek UFS Host Driver");
1829 MODULE_LICENSE("GPL v2");
1831 module_platform_driver(ufs_mtk_pltform);