1 // SPDX-License-Identifier: GPL-2.0+
3 * PCIe host controller driver for the following SoCs
7 * Copyright (C) 2019-2022 NVIDIA Corporation.
9 * Author: Vidya Sagar <vidyas@nvidia.com>
12 #include <linux/bitfield.h>
13 #include <linux/clk.h>
14 #include <linux/debugfs.h>
15 #include <linux/delay.h>
16 #include <linux/gpio.h>
17 #include <linux/gpio/consumer.h>
18 #include <linux/interconnect.h>
19 #include <linux/interrupt.h>
20 #include <linux/iopoll.h>
21 #include <linux/kernel.h>
22 #include <linux/module.h>
24 #include <linux/of_gpio.h>
25 #include <linux/of_pci.h>
26 #include <linux/pci.h>
27 #include <linux/phy/phy.h>
28 #include <linux/pinctrl/consumer.h>
29 #include <linux/platform_device.h>
30 #include <linux/pm_runtime.h>
31 #include <linux/random.h>
32 #include <linux/reset.h>
33 #include <linux/resource.h>
34 #include <linux/types.h>
35 #include "pcie-designware.h"
36 #include <soc/tegra/bpmp.h>
37 #include <soc/tegra/bpmp-abi.h>
38 #include "../../pci.h"
40 #define TEGRA194_DWC_IP_VER 0x490A
41 #define TEGRA234_DWC_IP_VER 0x562A
43 #define APPL_PINMUX 0x0
44 #define APPL_PINMUX_PEX_RST BIT(0)
45 #define APPL_PINMUX_CLKREQ_OVERRIDE_EN BIT(2)
46 #define APPL_PINMUX_CLKREQ_OVERRIDE BIT(3)
47 #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN BIT(4)
48 #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE BIT(5)
51 #define APPL_CTRL_SYS_PRE_DET_STATE BIT(6)
52 #define APPL_CTRL_LTSSM_EN BIT(7)
53 #define APPL_CTRL_HW_HOT_RST_EN BIT(20)
54 #define APPL_CTRL_HW_HOT_RST_MODE_MASK GENMASK(1, 0)
55 #define APPL_CTRL_HW_HOT_RST_MODE_SHIFT 22
56 #define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST 0x1
57 #define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST_LTSSM_EN 0x2
59 #define APPL_INTR_EN_L0_0 0x8
60 #define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN BIT(0)
61 #define APPL_INTR_EN_L0_0_MSI_RCV_INT_EN BIT(4)
62 #define APPL_INTR_EN_L0_0_INT_INT_EN BIT(8)
63 #define APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN BIT(15)
64 #define APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN BIT(19)
65 #define APPL_INTR_EN_L0_0_SYS_INTR_EN BIT(30)
66 #define APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN BIT(31)
68 #define APPL_INTR_STATUS_L0 0xC
69 #define APPL_INTR_STATUS_L0_LINK_STATE_INT BIT(0)
70 #define APPL_INTR_STATUS_L0_INT_INT BIT(8)
71 #define APPL_INTR_STATUS_L0_PCI_CMD_EN_INT BIT(15)
72 #define APPL_INTR_STATUS_L0_PEX_RST_INT BIT(16)
73 #define APPL_INTR_STATUS_L0_CDM_REG_CHK_INT BIT(18)
75 #define APPL_INTR_EN_L1_0_0 0x1C
76 #define APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN BIT(1)
77 #define APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN BIT(3)
78 #define APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN BIT(30)
80 #define APPL_INTR_STATUS_L1_0_0 0x20
81 #define APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED BIT(1)
82 #define APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED BIT(3)
83 #define APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE BIT(30)
85 #define APPL_INTR_STATUS_L1_1 0x2C
86 #define APPL_INTR_STATUS_L1_2 0x30
87 #define APPL_INTR_STATUS_L1_3 0x34
88 #define APPL_INTR_STATUS_L1_6 0x3C
89 #define APPL_INTR_STATUS_L1_7 0x40
90 #define APPL_INTR_STATUS_L1_15_CFG_BME_CHGED BIT(1)
92 #define APPL_INTR_EN_L1_8_0 0x44
93 #define APPL_INTR_EN_L1_8_BW_MGT_INT_EN BIT(2)
94 #define APPL_INTR_EN_L1_8_AUTO_BW_INT_EN BIT(3)
95 #define APPL_INTR_EN_L1_8_INTX_EN BIT(11)
96 #define APPL_INTR_EN_L1_8_AER_INT_EN BIT(15)
98 #define APPL_INTR_STATUS_L1_8_0 0x4C
99 #define APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK GENMASK(11, 6)
100 #define APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS BIT(2)
101 #define APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS BIT(3)
103 #define APPL_INTR_STATUS_L1_9 0x54
104 #define APPL_INTR_STATUS_L1_10 0x58
105 #define APPL_INTR_STATUS_L1_11 0x64
106 #define APPL_INTR_STATUS_L1_13 0x74
107 #define APPL_INTR_STATUS_L1_14 0x78
108 #define APPL_INTR_STATUS_L1_15 0x7C
109 #define APPL_INTR_STATUS_L1_17 0x88
111 #define APPL_INTR_EN_L1_18 0x90
112 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMPLT BIT(2)
113 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR BIT(1)
114 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0)
116 #define APPL_INTR_STATUS_L1_18 0x94
117 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT BIT(2)
118 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR BIT(1)
119 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0)
121 #define APPL_MSI_CTRL_1 0xAC
123 #define APPL_MSI_CTRL_2 0xB0
125 #define APPL_LEGACY_INTX 0xB8
127 #define APPL_LTR_MSG_1 0xC4
128 #define LTR_MSG_REQ BIT(15)
129 #define LTR_NOSNOOP_MSG_REQ BIT(31)
131 #define APPL_LTR_MSG_2 0xC8
132 #define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE BIT(3)
134 #define APPL_LINK_STATUS 0xCC
135 #define APPL_LINK_STATUS_RDLH_LINK_UP BIT(0)
137 #define APPL_DEBUG 0xD0
138 #define APPL_DEBUG_PM_LINKST_IN_L2_LAT BIT(21)
139 #define APPL_DEBUG_PM_LINKST_IN_L0 0x11
140 #define APPL_DEBUG_LTSSM_STATE_MASK GENMASK(8, 3)
141 #define APPL_DEBUG_LTSSM_STATE_SHIFT 3
142 #define LTSSM_STATE_PRE_DETECT 5
144 #define APPL_RADM_STATUS 0xE4
145 #define APPL_PM_XMT_TURNOFF_STATE BIT(0)
147 #define APPL_DM_TYPE 0x100
148 #define APPL_DM_TYPE_MASK GENMASK(3, 0)
149 #define APPL_DM_TYPE_RP 0x4
150 #define APPL_DM_TYPE_EP 0x0
152 #define APPL_CFG_BASE_ADDR 0x104
153 #define APPL_CFG_BASE_ADDR_MASK GENMASK(31, 12)
155 #define APPL_CFG_IATU_DMA_BASE_ADDR 0x108
156 #define APPL_CFG_IATU_DMA_BASE_ADDR_MASK GENMASK(31, 18)
158 #define APPL_CFG_MISC 0x110
159 #define APPL_CFG_MISC_SLV_EP_MODE BIT(14)
160 #define APPL_CFG_MISC_ARCACHE_MASK GENMASK(13, 10)
161 #define APPL_CFG_MISC_ARCACHE_SHIFT 10
162 #define APPL_CFG_MISC_ARCACHE_VAL 3
164 #define APPL_CFG_SLCG_OVERRIDE 0x114
165 #define APPL_CFG_SLCG_OVERRIDE_SLCG_EN_MASTER BIT(0)
167 #define APPL_CAR_RESET_OVRD 0x12C
168 #define APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N BIT(0)
170 #define IO_BASE_IO_DECODE BIT(0)
171 #define IO_BASE_IO_DECODE_BIT8 BIT(8)
173 #define CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE BIT(0)
174 #define CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE BIT(16)
176 #define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF 0x718
177 #define CFG_TIMER_CTRL_ACK_NAK_SHIFT (19)
182 #define GEN3_EQ_CONTROL_OFF 0x8a8
183 #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT 8
184 #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK GENMASK(23, 8)
185 #define GEN3_EQ_CONTROL_OFF_FB_MODE_MASK GENMASK(3, 0)
187 #define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT 0x8D0
188 #define AMBA_ERROR_RESPONSE_CRS_SHIFT 3
189 #define AMBA_ERROR_RESPONSE_CRS_MASK GENMASK(1, 0)
190 #define AMBA_ERROR_RESPONSE_CRS_OKAY 0
191 #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFFFFFF 1
192 #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 2
194 #define MSIX_ADDR_MATCH_LOW_OFF 0x940
195 #define MSIX_ADDR_MATCH_LOW_OFF_EN BIT(0)
196 #define MSIX_ADDR_MATCH_LOW_OFF_MASK GENMASK(31, 2)
198 #define MSIX_ADDR_MATCH_HIGH_OFF 0x944
199 #define MSIX_ADDR_MATCH_HIGH_OFF_MASK GENMASK(31, 0)
201 #define PORT_LOGIC_MSIX_DOORBELL 0x948
203 #define CAP_SPCIE_CAP_OFF 0x154
204 #define CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK GENMASK(3, 0)
205 #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK GENMASK(11, 8)
206 #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT 8
208 #define PME_ACK_TIMEOUT 10000
210 #define LTSSM_TIMEOUT 50000 /* 50ms */
212 #define GEN3_GEN4_EQ_PRESET_INIT 5
214 #define GEN1_CORE_CLK_FREQ 62500000
215 #define GEN2_CORE_CLK_FREQ 125000000
216 #define GEN3_CORE_CLK_FREQ 250000000
217 #define GEN4_CORE_CLK_FREQ 500000000
219 #define LTR_MSG_TIMEOUT (100 * 1000)
221 #define PERST_DEBOUNCE_TIME (5 * 1000)
223 #define EP_STATE_DISABLED 0
224 #define EP_STATE_ENABLED 1
226 static const unsigned int pcie_gen_freq[] = {
227 GEN1_CORE_CLK_FREQ, /* PCI_EXP_LNKSTA_CLS == 0; undefined */
234 struct tegra_pcie_dw_of_data {
236 enum dw_pcie_device_mode mode;
237 bool has_msix_doorbell_access_fix;
238 bool has_sbr_reset_fix;
239 bool has_l1ss_exit_fix;
240 bool has_ltr_req_fix;
241 u32 cdm_chk_int_en_bit;
246 struct tegra_pcie_dw {
248 struct resource *appl_res;
249 struct resource *dbi_res;
250 struct resource *atu_dma_res;
251 void __iomem *appl_base;
252 struct clk *core_clk;
253 struct reset_control *core_apb_rst;
254 struct reset_control *core_rst;
256 struct tegra_bpmp *bpmp;
258 struct tegra_pcie_dw_of_data *of_data;
260 bool supports_clkreq;
261 bool enable_cdm_check;
264 bool update_fc_fixup;
265 bool enable_ext_refclk;
270 u32 cfg_link_cap_l1sub;
275 u32 aspm_l0s_enter_lat;
277 struct regulator *pex_ctl_supply;
278 struct regulator *slot_ctl_3v3;
279 struct regulator *slot_ctl_12v;
281 unsigned int phy_count;
284 struct dentry *debugfs;
286 /* Endpoint mode specific */
287 struct gpio_desc *pex_rst_gpiod;
288 struct gpio_desc *pex_refclk_sel_gpiod;
289 unsigned int pex_rst_irq;
292 struct icc_path *icc_path;
295 static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci)
297 return container_of(pci, struct tegra_pcie_dw, pci);
300 static inline void appl_writel(struct tegra_pcie_dw *pcie, const u32 value,
303 writel_relaxed(value, pcie->appl_base + reg);
306 static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg)
308 return readl_relaxed(pcie->appl_base + reg);
311 struct tegra_pcie_soc {
312 enum dw_pcie_device_mode mode;
315 static void tegra_pcie_icc_set(struct tegra_pcie_dw *pcie)
317 struct dw_pcie *pci = &pcie->pci;
318 u32 val, speed, width;
320 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
322 speed = FIELD_GET(PCI_EXP_LNKSTA_CLS, val);
323 width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val);
325 val = width * PCIE_SPEED2MBS_ENC(pcie_link_speed[speed]);
327 if (icc_set_bw(pcie->icc_path, Mbps_to_icc(val), 0))
328 dev_err(pcie->dev, "can't set bw[%u]\n", val);
330 if (speed >= ARRAY_SIZE(pcie_gen_freq))
333 clk_set_rate(pcie->core_clk, pcie_gen_freq[speed]);
336 static void apply_bad_link_workaround(struct dw_pcie_rp *pp)
338 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
339 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
340 u32 current_link_width;
344 * NOTE:- Since this scenario is uncommon and link as such is not
345 * stable anyway, not waiting to confirm if link is really
346 * transitioning to Gen-2 speed
348 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
349 if (val & PCI_EXP_LNKSTA_LBMS) {
350 current_link_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val);
351 if (pcie->init_link_width > current_link_width) {
352 dev_warn(pci->dev, "PCIe link is bad, width reduced\n");
353 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
355 val &= ~PCI_EXP_LNKCTL2_TLS;
356 val |= PCI_EXP_LNKCTL2_TLS_2_5GT;
357 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
358 PCI_EXP_LNKCTL2, val);
360 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
362 val |= PCI_EXP_LNKCTL_RL;
363 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
364 PCI_EXP_LNKCTL, val);
369 static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg)
371 struct tegra_pcie_dw *pcie = arg;
372 struct dw_pcie *pci = &pcie->pci;
373 struct dw_pcie_rp *pp = &pci->pp;
374 u32 val, status_l0, status_l1;
377 status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0);
378 if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
379 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
380 appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_0_0);
381 if (!pcie->of_data->has_sbr_reset_fix &&
382 status_l1 & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) {
383 /* SBR & Surprise Link Down WAR */
384 val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
385 val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
386 appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
388 val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
389 val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
390 appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
392 val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
393 val |= PORT_LOGIC_SPEED_CHANGE;
394 dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val);
398 if (status_l0 & APPL_INTR_STATUS_L0_INT_INT) {
399 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0);
400 if (status_l1 & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) {
402 APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS,
403 APPL_INTR_STATUS_L1_8_0);
404 apply_bad_link_workaround(pp);
406 if (status_l1 & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) {
407 val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
409 val_w |= PCI_EXP_LNKSTA_LBMS;
410 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
411 PCI_EXP_LNKSTA, val_w);
414 APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS,
415 APPL_INTR_STATUS_L1_8_0);
417 val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
419 dev_dbg(pci->dev, "Link Speed : Gen-%u\n", val_w &
424 if (status_l0 & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) {
425 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_18);
426 val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS);
427 if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) {
428 dev_info(pci->dev, "CDM check complete\n");
429 val |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE;
431 if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) {
432 dev_err(pci->dev, "CDM comparison mismatch\n");
433 val |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR;
435 if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) {
436 dev_err(pci->dev, "CDM Logic error\n");
437 val |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR;
439 dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, val);
440 val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR);
441 dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", val);
447 static void pex_ep_event_hot_rst_done(struct tegra_pcie_dw *pcie)
451 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
452 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
453 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
454 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
455 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
456 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
457 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
458 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
459 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
460 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
461 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
462 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
463 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
464 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
465 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
466 appl_writel(pcie, 0xFFFFFFFF, APPL_MSI_CTRL_2);
468 val = appl_readl(pcie, APPL_CTRL);
469 val |= APPL_CTRL_LTSSM_EN;
470 appl_writel(pcie, val, APPL_CTRL);
473 static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg)
475 struct tegra_pcie_dw *pcie = arg;
476 struct dw_pcie_ep *ep = &pcie->pci.ep;
477 struct dw_pcie *pci = &pcie->pci;
480 if (test_and_clear_bit(0, &pcie->link_status))
481 dw_pcie_ep_linkup(ep);
483 tegra_pcie_icc_set(pcie);
485 if (pcie->of_data->has_ltr_req_fix)
488 /* If EP doesn't advertise L1SS, just return */
489 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
490 if (!(val & (PCI_L1SS_CAP_ASPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_2)))
493 /* Check if BME is set to '1' */
494 val = dw_pcie_readl_dbi(pci, PCI_COMMAND);
495 if (val & PCI_COMMAND_MASTER) {
498 /* 110us for both snoop and no-snoop */
499 val = FIELD_PREP(PCI_LTR_VALUE_MASK, 110) |
500 FIELD_PREP(PCI_LTR_SCALE_MASK, 2) |
502 FIELD_PREP(PCI_LTR_NOSNOOP_VALUE, 110) |
503 FIELD_PREP(PCI_LTR_NOSNOOP_SCALE, 2) |
505 appl_writel(pcie, val, APPL_LTR_MSG_1);
507 /* Send LTR upstream */
508 val = appl_readl(pcie, APPL_LTR_MSG_2);
509 val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE;
510 appl_writel(pcie, val, APPL_LTR_MSG_2);
512 timeout = ktime_add_us(ktime_get(), LTR_MSG_TIMEOUT);
514 val = appl_readl(pcie, APPL_LTR_MSG_2);
515 if (!(val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE))
517 if (ktime_after(ktime_get(), timeout))
519 usleep_range(1000, 1100);
521 if (val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE)
522 dev_err(pcie->dev, "Failed to send LTR message\n");
528 static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg)
530 struct tegra_pcie_dw *pcie = arg;
532 u32 status_l0, status_l1, link_status;
534 status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0);
535 if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
536 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
537 appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_0_0);
539 if (status_l1 & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE)
540 pex_ep_event_hot_rst_done(pcie);
542 if (status_l1 & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) {
543 link_status = appl_readl(pcie, APPL_LINK_STATUS);
544 if (link_status & APPL_LINK_STATUS_RDLH_LINK_UP) {
545 dev_dbg(pcie->dev, "Link is up with Host\n");
546 set_bit(0, &pcie->link_status);
547 return IRQ_WAKE_THREAD;
554 if (status_l0 & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) {
555 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_15);
556 appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_15);
558 if (status_l1 & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED)
559 return IRQ_WAKE_THREAD;
565 dev_warn(pcie->dev, "Random interrupt (STATUS = 0x%08X)\n",
567 appl_writel(pcie, status_l0, APPL_INTR_STATUS_L0);
573 static int tegra_pcie_dw_rd_own_conf(struct pci_bus *bus, u32 devfn, int where,
576 struct dw_pcie_rp *pp = bus->sysdata;
577 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
578 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
581 * This is an endpoint mode specific register happen to appear even
582 * when controller is operating in root port mode and system hangs
583 * when it is accessed with link being in ASPM-L1 state.
584 * So skip accessing it altogether
586 if (!pcie->of_data->has_msix_doorbell_access_fix &&
587 !PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL) {
589 return PCIBIOS_SUCCESSFUL;
592 return pci_generic_config_read(bus, devfn, where, size, val);
595 static int tegra_pcie_dw_wr_own_conf(struct pci_bus *bus, u32 devfn, int where,
598 struct dw_pcie_rp *pp = bus->sysdata;
599 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
600 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
603 * This is an endpoint mode specific register happen to appear even
604 * when controller is operating in root port mode and system hangs
605 * when it is accessed with link being in ASPM-L1 state.
606 * So skip accessing it altogether
608 if (!pcie->of_data->has_msix_doorbell_access_fix &&
609 !PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL)
610 return PCIBIOS_SUCCESSFUL;
612 return pci_generic_config_write(bus, devfn, where, size, val);
615 static struct pci_ops tegra_pci_ops = {
616 .map_bus = dw_pcie_own_conf_map_bus,
617 .read = tegra_pcie_dw_rd_own_conf,
618 .write = tegra_pcie_dw_wr_own_conf,
621 #if defined(CONFIG_PCIEASPM)
622 static void disable_aspm_l11(struct tegra_pcie_dw *pcie)
626 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
627 val &= ~PCI_L1SS_CAP_ASPM_L1_1;
628 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
631 static void disable_aspm_l12(struct tegra_pcie_dw *pcie)
635 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
636 val &= ~PCI_L1SS_CAP_ASPM_L1_2;
637 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
640 static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event)
644 val = dw_pcie_readl_dbi(&pcie->pci, pcie->ras_des_cap +
645 PCIE_RAS_DES_EVENT_COUNTER_CONTROL);
646 val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT);
647 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
648 val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT;
649 val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
650 dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap +
651 PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val);
652 val = dw_pcie_readl_dbi(&pcie->pci, pcie->ras_des_cap +
653 PCIE_RAS_DES_EVENT_COUNTER_DATA);
658 static int aspm_state_cnt(struct seq_file *s, void *data)
660 struct tegra_pcie_dw *pcie = (struct tegra_pcie_dw *)
661 dev_get_drvdata(s->private);
664 seq_printf(s, "Tx L0s entry count : %u\n",
665 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Tx_L0S));
667 seq_printf(s, "Rx L0s entry count : %u\n",
668 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Rx_L0S));
670 seq_printf(s, "Link L1 entry count : %u\n",
671 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1));
673 seq_printf(s, "Link L1.1 entry count : %u\n",
674 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_1));
676 seq_printf(s, "Link L1.2 entry count : %u\n",
677 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2));
679 /* Clear all counters */
680 dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap +
681 PCIE_RAS_DES_EVENT_COUNTER_CONTROL,
682 EVENT_COUNTER_ALL_CLEAR);
684 /* Re-enable counting */
685 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
686 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
687 dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap +
688 PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val);
693 static void init_host_aspm(struct tegra_pcie_dw *pcie)
695 struct dw_pcie *pci = &pcie->pci;
698 val = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS);
699 pcie->cfg_link_cap_l1sub = val + PCI_L1SS_CAP;
701 pcie->ras_des_cap = dw_pcie_find_ext_capability(&pcie->pci,
702 PCI_EXT_CAP_ID_VNDR);
704 /* Enable ASPM counters */
705 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
706 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
707 dw_pcie_writel_dbi(pci, pcie->ras_des_cap +
708 PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val);
710 /* Program T_cmrt and T_pwr_on values */
711 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
712 val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE);
713 val |= (pcie->aspm_cmrt << 8);
714 val |= (pcie->aspm_pwr_on_t << 19);
715 dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val);
717 /* Program L0s and L1 entrance latencies */
718 val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR);
719 val &= ~PORT_AFR_L0S_ENTRANCE_LAT_MASK;
720 val |= (pcie->aspm_l0s_enter_lat << PORT_AFR_L0S_ENTRANCE_LAT_SHIFT);
721 val |= PORT_AFR_ENTER_ASPM;
722 dw_pcie_writel_dbi(pci, PCIE_PORT_AFR, val);
725 static void init_debugfs(struct tegra_pcie_dw *pcie)
727 debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt", pcie->debugfs,
731 static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; }
732 static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; }
733 static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; }
734 static inline void init_debugfs(struct tegra_pcie_dw *pcie) { return; }
737 static void tegra_pcie_enable_system_interrupts(struct dw_pcie_rp *pp)
739 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
740 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
744 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
745 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
746 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
748 if (!pcie->of_data->has_sbr_reset_fix) {
749 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
750 val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN;
751 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
754 if (pcie->enable_cdm_check) {
755 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
756 val |= pcie->of_data->cdm_chk_int_en_bit;
757 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
759 val = appl_readl(pcie, APPL_INTR_EN_L1_18);
760 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR;
761 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR;
762 appl_writel(pcie, val, APPL_INTR_EN_L1_18);
765 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
767 pcie->init_link_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val_w);
769 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
771 val_w |= PCI_EXP_LNKCTL_LBMIE;
772 dw_pcie_writew_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL,
776 static void tegra_pcie_enable_legacy_interrupts(struct dw_pcie_rp *pp)
778 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
779 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
782 /* Enable legacy interrupt generation */
783 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
784 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
785 val |= APPL_INTR_EN_L0_0_INT_INT_EN;
786 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
788 val = appl_readl(pcie, APPL_INTR_EN_L1_8_0);
789 val |= APPL_INTR_EN_L1_8_INTX_EN;
790 val |= APPL_INTR_EN_L1_8_AUTO_BW_INT_EN;
791 val |= APPL_INTR_EN_L1_8_BW_MGT_INT_EN;
792 if (IS_ENABLED(CONFIG_PCIEAER))
793 val |= APPL_INTR_EN_L1_8_AER_INT_EN;
794 appl_writel(pcie, val, APPL_INTR_EN_L1_8_0);
797 static void tegra_pcie_enable_msi_interrupts(struct dw_pcie_rp *pp)
799 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
800 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
803 /* Enable MSI interrupt generation */
804 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
805 val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN;
806 val |= APPL_INTR_EN_L0_0_MSI_RCV_INT_EN;
807 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
810 static void tegra_pcie_enable_interrupts(struct dw_pcie_rp *pp)
812 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
813 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
815 /* Clear interrupt statuses before enabling interrupts */
816 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
817 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
818 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
819 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
820 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
821 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
822 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
823 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
824 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
825 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
826 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
827 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
828 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
829 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
830 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
832 tegra_pcie_enable_system_interrupts(pp);
833 tegra_pcie_enable_legacy_interrupts(pp);
834 if (IS_ENABLED(CONFIG_PCI_MSI))
835 tegra_pcie_enable_msi_interrupts(pp);
838 static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie)
840 struct dw_pcie *pci = &pcie->pci;
843 /* Program init preset */
844 for (i = 0; i < pcie->num_lanes; i++) {
845 val = dw_pcie_readw_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2));
846 val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK;
847 val |= GEN3_GEN4_EQ_PRESET_INIT;
848 val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK;
849 val |= (GEN3_GEN4_EQ_PRESET_INIT <<
850 CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT);
851 dw_pcie_writew_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2), val);
853 offset = dw_pcie_find_ext_capability(pci,
854 PCI_EXT_CAP_ID_PL_16GT) +
856 val = dw_pcie_readb_dbi(pci, offset + i);
857 val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK;
858 val |= GEN3_GEN4_EQ_PRESET_INIT;
859 val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK;
860 val |= (GEN3_GEN4_EQ_PRESET_INIT <<
861 PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT);
862 dw_pcie_writeb_dbi(pci, offset + i, val);
865 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
866 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
867 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
869 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
870 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
871 val |= (0x3ff << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
872 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
873 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
875 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
876 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
877 val |= (0x1 << GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT);
878 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
880 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
881 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
882 val |= (pcie->of_data->gen4_preset_vec <<
883 GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
884 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
885 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
887 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
888 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
889 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
892 static int tegra_pcie_dw_host_init(struct dw_pcie_rp *pp)
894 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
895 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
899 pp->bridge->ops = &tegra_pci_ops;
901 if (!pcie->pcie_cap_base)
902 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
905 val = dw_pcie_readl_dbi(pci, PCI_IO_BASE);
906 val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8);
907 dw_pcie_writel_dbi(pci, PCI_IO_BASE, val);
909 val = dw_pcie_readl_dbi(pci, PCI_PREF_MEMORY_BASE);
910 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE;
911 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE;
912 dw_pcie_writel_dbi(pci, PCI_PREF_MEMORY_BASE, val);
914 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0);
916 /* Enable as 0xFFFF0001 response for CRS */
917 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT);
918 val &= ~(AMBA_ERROR_RESPONSE_CRS_MASK << AMBA_ERROR_RESPONSE_CRS_SHIFT);
919 val |= (AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 <<
920 AMBA_ERROR_RESPONSE_CRS_SHIFT);
921 dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val);
923 /* Clear Slot Clock Configuration bit if SRNS configuration */
924 if (pcie->enable_srns) {
925 val_16 = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
927 val_16 &= ~PCI_EXP_LNKSTA_SLC;
928 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA,
932 config_gen3_gen4_eq_presets(pcie);
934 init_host_aspm(pcie);
936 /* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */
937 if (!pcie->supports_clkreq) {
938 disable_aspm_l11(pcie);
939 disable_aspm_l12(pcie);
942 if (!pcie->of_data->has_l1ss_exit_fix) {
943 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
944 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
945 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
948 if (pcie->update_fc_fixup) {
949 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
950 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
951 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
954 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
959 static int tegra_pcie_dw_start_link(struct dw_pcie *pci)
961 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
962 struct dw_pcie_rp *pp = &pci->pp;
963 u32 val, offset, tmp;
966 if (pcie->of_data->mode == DW_PCIE_EP_TYPE) {
967 enable_irq(pcie->pex_rst_irq);
973 val = appl_readl(pcie, APPL_PINMUX);
974 val &= ~APPL_PINMUX_PEX_RST;
975 appl_writel(pcie, val, APPL_PINMUX);
977 usleep_range(100, 200);
980 val = appl_readl(pcie, APPL_CTRL);
981 val |= APPL_CTRL_LTSSM_EN;
982 appl_writel(pcie, val, APPL_CTRL);
985 val = appl_readl(pcie, APPL_PINMUX);
986 val |= APPL_PINMUX_PEX_RST;
987 appl_writel(pcie, val, APPL_PINMUX);
991 if (dw_pcie_wait_for_link(pci)) {
995 * There are some endpoints which can't get the link up if
996 * root port has Data Link Feature (DLF) enabled.
997 * Refer Spec rev 4.0 ver 1.0 sec 3.4.2 & 7.7.4 for more info
998 * on Scaled Flow Control and DLF.
999 * So, need to confirm that is indeed the case here and attempt
1000 * link up once again with DLF disabled.
1002 val = appl_readl(pcie, APPL_DEBUG);
1003 val &= APPL_DEBUG_LTSSM_STATE_MASK;
1004 val >>= APPL_DEBUG_LTSSM_STATE_SHIFT;
1005 tmp = appl_readl(pcie, APPL_LINK_STATUS);
1006 tmp &= APPL_LINK_STATUS_RDLH_LINK_UP;
1007 if (!(val == 0x11 && !tmp)) {
1008 /* Link is down for all good reasons */
1012 dev_info(pci->dev, "Link is down in DLL");
1013 dev_info(pci->dev, "Trying again with DLFE disabled\n");
1015 val = appl_readl(pcie, APPL_CTRL);
1016 val &= ~APPL_CTRL_LTSSM_EN;
1017 appl_writel(pcie, val, APPL_CTRL);
1019 reset_control_assert(pcie->core_rst);
1020 reset_control_deassert(pcie->core_rst);
1022 offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF);
1023 val = dw_pcie_readl_dbi(pci, offset + PCI_DLF_CAP);
1024 val &= ~PCI_DLF_EXCHANGE_ENABLE;
1025 dw_pcie_writel_dbi(pci, offset + PCI_DLF_CAP, val);
1027 tegra_pcie_dw_host_init(pp);
1028 dw_pcie_setup_rc(pp);
1034 tegra_pcie_icc_set(pcie);
1036 tegra_pcie_enable_interrupts(pp);
1041 static int tegra_pcie_dw_link_up(struct dw_pcie *pci)
1043 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1044 u32 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
1046 return !!(val & PCI_EXP_LNKSTA_DLLLA);
1049 static void tegra_pcie_dw_stop_link(struct dw_pcie *pci)
1051 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1053 disable_irq(pcie->pex_rst_irq);
1056 static const struct dw_pcie_ops tegra_dw_pcie_ops = {
1057 .link_up = tegra_pcie_dw_link_up,
1058 .start_link = tegra_pcie_dw_start_link,
1059 .stop_link = tegra_pcie_dw_stop_link,
1062 static const struct dw_pcie_host_ops tegra_pcie_dw_host_ops = {
1063 .host_init = tegra_pcie_dw_host_init,
1066 static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie)
1068 unsigned int phy_count = pcie->phy_count;
1070 while (phy_count--) {
1071 phy_power_off(pcie->phys[phy_count]);
1072 phy_exit(pcie->phys[phy_count]);
1076 static int tegra_pcie_enable_phy(struct tegra_pcie_dw *pcie)
1081 for (i = 0; i < pcie->phy_count; i++) {
1082 ret = phy_init(pcie->phys[i]);
1086 ret = phy_power_on(pcie->phys[i]);
1095 phy_power_off(pcie->phys[i]);
1097 phy_exit(pcie->phys[i]);
1103 static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie)
1105 struct platform_device *pdev = to_platform_device(pcie->dev);
1106 struct device_node *np = pcie->dev->of_node;
1109 pcie->dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi");
1110 if (!pcie->dbi_res) {
1111 dev_err(pcie->dev, "Failed to find \"dbi\" region\n");
1115 ret = of_property_read_u32(np, "nvidia,aspm-cmrt-us", &pcie->aspm_cmrt);
1117 dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret);
1121 ret = of_property_read_u32(np, "nvidia,aspm-pwr-on-t-us",
1122 &pcie->aspm_pwr_on_t);
1124 dev_info(pcie->dev, "Failed to read ASPM Power On time: %d\n",
1127 ret = of_property_read_u32(np, "nvidia,aspm-l0s-entrance-latency-us",
1128 &pcie->aspm_l0s_enter_lat);
1131 "Failed to read ASPM L0s Entrance latency: %d\n", ret);
1133 ret = of_property_read_u32(np, "num-lanes", &pcie->num_lanes);
1135 dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret);
1139 ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid);
1141 dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret);
1145 ret = of_property_count_strings(np, "phy-names");
1147 dev_err(pcie->dev, "Failed to find PHY entries: %d\n",
1151 pcie->phy_count = ret;
1153 if (of_property_read_bool(np, "nvidia,update-fc-fixup"))
1154 pcie->update_fc_fixup = true;
1156 /* RP using an external REFCLK is supported only in Tegra234 */
1157 if (pcie->of_data->version == TEGRA194_DWC_IP_VER) {
1158 if (pcie->of_data->mode == DW_PCIE_EP_TYPE)
1159 pcie->enable_ext_refclk = true;
1161 pcie->enable_ext_refclk =
1162 of_property_read_bool(pcie->dev->of_node,
1163 "nvidia,enable-ext-refclk");
1166 pcie->supports_clkreq =
1167 of_property_read_bool(pcie->dev->of_node, "supports-clkreq");
1169 pcie->enable_cdm_check =
1170 of_property_read_bool(np, "snps,enable-cdm-check");
1172 if (pcie->of_data->version == TEGRA234_DWC_IP_VER)
1174 of_property_read_bool(np, "nvidia,enable-srns");
1176 if (pcie->of_data->mode == DW_PCIE_RC_TYPE)
1179 /* Endpoint mode specific DT entries */
1180 pcie->pex_rst_gpiod = devm_gpiod_get(pcie->dev, "reset", GPIOD_IN);
1181 if (IS_ERR(pcie->pex_rst_gpiod)) {
1182 int err = PTR_ERR(pcie->pex_rst_gpiod);
1183 const char *level = KERN_ERR;
1185 if (err == -EPROBE_DEFER)
1188 dev_printk(level, pcie->dev,
1189 dev_fmt("Failed to get PERST GPIO: %d\n"),
1194 pcie->pex_refclk_sel_gpiod = devm_gpiod_get(pcie->dev,
1195 "nvidia,refclk-select",
1197 if (IS_ERR(pcie->pex_refclk_sel_gpiod)) {
1198 int err = PTR_ERR(pcie->pex_refclk_sel_gpiod);
1199 const char *level = KERN_ERR;
1201 if (err == -EPROBE_DEFER)
1204 dev_printk(level, pcie->dev,
1205 dev_fmt("Failed to get REFCLK select GPIOs: %d\n"),
1207 pcie->pex_refclk_sel_gpiod = NULL;
1213 static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie,
1216 struct mrq_uphy_response resp;
1217 struct tegra_bpmp_message msg;
1218 struct mrq_uphy_request req;
1221 * Controller-5 doesn't need to have its state set by BPMP-FW in
1224 if (pcie->of_data->version == TEGRA194_DWC_IP_VER && pcie->cid == 5)
1227 memset(&req, 0, sizeof(req));
1228 memset(&resp, 0, sizeof(resp));
1230 req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE;
1231 req.controller_state.pcie_controller = pcie->cid;
1232 req.controller_state.enable = enable;
1234 memset(&msg, 0, sizeof(msg));
1237 msg.tx.size = sizeof(req);
1238 msg.rx.data = &resp;
1239 msg.rx.size = sizeof(resp);
1241 return tegra_bpmp_transfer(pcie->bpmp, &msg);
1244 static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie,
1247 struct mrq_uphy_response resp;
1248 struct tegra_bpmp_message msg;
1249 struct mrq_uphy_request req;
1251 memset(&req, 0, sizeof(req));
1252 memset(&resp, 0, sizeof(resp));
1255 req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_INIT;
1256 req.ep_ctrlr_pll_init.ep_controller = pcie->cid;
1258 req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_OFF;
1259 req.ep_ctrlr_pll_off.ep_controller = pcie->cid;
1262 memset(&msg, 0, sizeof(msg));
1265 msg.tx.size = sizeof(req);
1266 msg.rx.data = &resp;
1267 msg.rx.size = sizeof(resp);
1269 return tegra_bpmp_transfer(pcie->bpmp, &msg);
1272 static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie)
1274 struct dw_pcie_rp *pp = &pcie->pci.pp;
1275 struct pci_bus *child, *root_bus = NULL;
1276 struct pci_dev *pdev;
1279 * link doesn't go into L2 state with some of the endpoints with Tegra
1280 * if they are not in D0 state. So, need to make sure that immediate
1281 * downstream devices are in D0 state before sending PME_TurnOff to put
1282 * link into L2 state.
1283 * This is as per PCI Express Base r4.0 v1.0 September 27-2017,
1284 * 5.2 Link State Power Management (Page #428).
1287 list_for_each_entry(child, &pp->bridge->bus->children, node) {
1288 /* Bring downstream devices to D0 if they are not already in */
1289 if (child->parent == pp->bridge->bus) {
1296 dev_err(pcie->dev, "Failed to find downstream devices\n");
1300 list_for_each_entry(pdev, &root_bus->devices, bus_list) {
1301 if (PCI_SLOT(pdev->devfn) == 0) {
1302 if (pci_set_power_state(pdev, PCI_D0))
1304 "Failed to transition %s to D0 state\n",
1305 dev_name(&pdev->dev));
1310 static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie)
1312 pcie->slot_ctl_3v3 = devm_regulator_get_optional(pcie->dev, "vpcie3v3");
1313 if (IS_ERR(pcie->slot_ctl_3v3)) {
1314 if (PTR_ERR(pcie->slot_ctl_3v3) != -ENODEV)
1315 return PTR_ERR(pcie->slot_ctl_3v3);
1317 pcie->slot_ctl_3v3 = NULL;
1320 pcie->slot_ctl_12v = devm_regulator_get_optional(pcie->dev, "vpcie12v");
1321 if (IS_ERR(pcie->slot_ctl_12v)) {
1322 if (PTR_ERR(pcie->slot_ctl_12v) != -ENODEV)
1323 return PTR_ERR(pcie->slot_ctl_12v);
1325 pcie->slot_ctl_12v = NULL;
1331 static int tegra_pcie_enable_slot_regulators(struct tegra_pcie_dw *pcie)
1335 if (pcie->slot_ctl_3v3) {
1336 ret = regulator_enable(pcie->slot_ctl_3v3);
1339 "Failed to enable 3.3V slot supply: %d\n", ret);
1344 if (pcie->slot_ctl_12v) {
1345 ret = regulator_enable(pcie->slot_ctl_12v);
1348 "Failed to enable 12V slot supply: %d\n", ret);
1349 goto fail_12v_enable;
1354 * According to PCI Express Card Electromechanical Specification
1355 * Revision 1.1, Table-2.4, T_PVPERL (Power stable to PERST# inactive)
1356 * should be a minimum of 100ms.
1358 if (pcie->slot_ctl_3v3 || pcie->slot_ctl_12v)
1364 if (pcie->slot_ctl_3v3)
1365 regulator_disable(pcie->slot_ctl_3v3);
1369 static void tegra_pcie_disable_slot_regulators(struct tegra_pcie_dw *pcie)
1371 if (pcie->slot_ctl_12v)
1372 regulator_disable(pcie->slot_ctl_12v);
1373 if (pcie->slot_ctl_3v3)
1374 regulator_disable(pcie->slot_ctl_3v3);
1377 static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie,
1383 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true);
1386 "Failed to enable controller %u: %d\n", pcie->cid, ret);
1390 if (pcie->enable_ext_refclk) {
1391 ret = tegra_pcie_bpmp_set_pll_state(pcie, true);
1393 dev_err(pcie->dev, "Failed to init UPHY: %d\n", ret);
1398 ret = tegra_pcie_enable_slot_regulators(pcie);
1400 goto fail_slot_reg_en;
1402 ret = regulator_enable(pcie->pex_ctl_supply);
1404 dev_err(pcie->dev, "Failed to enable regulator: %d\n", ret);
1408 ret = clk_prepare_enable(pcie->core_clk);
1410 dev_err(pcie->dev, "Failed to enable core clock: %d\n", ret);
1414 ret = reset_control_deassert(pcie->core_apb_rst);
1416 dev_err(pcie->dev, "Failed to deassert core APB reset: %d\n",
1418 goto fail_core_apb_rst;
1421 if (en_hw_hot_rst || pcie->of_data->has_sbr_reset_fix) {
1422 /* Enable HW_HOT_RST mode */
1423 val = appl_readl(pcie, APPL_CTRL);
1424 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
1425 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
1426 val |= (APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST_LTSSM_EN <<
1427 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
1428 val |= APPL_CTRL_HW_HOT_RST_EN;
1429 appl_writel(pcie, val, APPL_CTRL);
1432 ret = tegra_pcie_enable_phy(pcie);
1434 dev_err(pcie->dev, "Failed to enable PHY: %d\n", ret);
1438 /* Update CFG base address */
1439 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1440 APPL_CFG_BASE_ADDR);
1442 /* Configure this core for RP mode operation */
1443 appl_writel(pcie, APPL_DM_TYPE_RP, APPL_DM_TYPE);
1445 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1447 val = appl_readl(pcie, APPL_CTRL);
1448 appl_writel(pcie, val | APPL_CTRL_SYS_PRE_DET_STATE, APPL_CTRL);
1450 val = appl_readl(pcie, APPL_CFG_MISC);
1451 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1452 appl_writel(pcie, val, APPL_CFG_MISC);
1454 if (pcie->enable_srns || pcie->enable_ext_refclk) {
1456 * When Tegra PCIe RP is using external clock, it cannot supply
1457 * same clock to its downstream hierarchy. Hence, gate PCIe RP
1458 * REFCLK out pads when RP & EP are using separate clocks or RP
1459 * is using an external REFCLK.
1461 val = appl_readl(pcie, APPL_PINMUX);
1462 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1463 val &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1464 appl_writel(pcie, val, APPL_PINMUX);
1467 if (!pcie->supports_clkreq) {
1468 val = appl_readl(pcie, APPL_PINMUX);
1469 val |= APPL_PINMUX_CLKREQ_OVERRIDE_EN;
1470 val &= ~APPL_PINMUX_CLKREQ_OVERRIDE;
1471 appl_writel(pcie, val, APPL_PINMUX);
1474 /* Update iATU_DMA base address */
1476 pcie->atu_dma_res->start & APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1477 APPL_CFG_IATU_DMA_BASE_ADDR);
1479 reset_control_deassert(pcie->core_rst);
1484 reset_control_assert(pcie->core_apb_rst);
1486 clk_disable_unprepare(pcie->core_clk);
1488 regulator_disable(pcie->pex_ctl_supply);
1490 tegra_pcie_disable_slot_regulators(pcie);
1492 if (pcie->enable_ext_refclk)
1493 tegra_pcie_bpmp_set_pll_state(pcie, false);
1495 tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1500 static void tegra_pcie_unconfig_controller(struct tegra_pcie_dw *pcie)
1504 ret = reset_control_assert(pcie->core_rst);
1506 dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n", ret);
1508 tegra_pcie_disable_phy(pcie);
1510 ret = reset_control_assert(pcie->core_apb_rst);
1512 dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret);
1514 clk_disable_unprepare(pcie->core_clk);
1516 ret = regulator_disable(pcie->pex_ctl_supply);
1518 dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret);
1520 tegra_pcie_disable_slot_regulators(pcie);
1522 if (pcie->enable_ext_refclk) {
1523 ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1525 dev_err(pcie->dev, "Failed to deinit UPHY: %d\n", ret);
1528 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1530 dev_err(pcie->dev, "Failed to disable controller %d: %d\n",
1534 static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie)
1536 struct dw_pcie *pci = &pcie->pci;
1537 struct dw_pcie_rp *pp = &pci->pp;
1540 ret = tegra_pcie_config_controller(pcie, false);
1544 pp->ops = &tegra_pcie_dw_host_ops;
1546 ret = dw_pcie_host_init(pp);
1548 dev_err(pcie->dev, "Failed to add PCIe port: %d\n", ret);
1549 goto fail_host_init;
1555 tegra_pcie_unconfig_controller(pcie);
1559 static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie)
1563 if (!tegra_pcie_dw_link_up(&pcie->pci))
1566 val = appl_readl(pcie, APPL_RADM_STATUS);
1567 val |= APPL_PM_XMT_TURNOFF_STATE;
1568 appl_writel(pcie, val, APPL_RADM_STATUS);
1570 return readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, val,
1571 val & APPL_DEBUG_PM_LINKST_IN_L2_LAT,
1572 1, PME_ACK_TIMEOUT);
1575 static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie)
1580 if (!tegra_pcie_dw_link_up(&pcie->pci)) {
1581 dev_dbg(pcie->dev, "PCIe link is not up...!\n");
1586 * PCIe controller exits from L2 only if reset is applied, so
1587 * controller doesn't handle interrupts. But in cases where
1588 * L2 entry fails, PERST# is asserted which can trigger surprise
1589 * link down AER. However this function call happens in
1590 * suspend_noirq(), so AER interrupt will not be processed.
1591 * Disable all interrupts to avoid such a scenario.
1593 appl_writel(pcie, 0x0, APPL_INTR_EN_L0_0);
1595 if (tegra_pcie_try_link_l2(pcie)) {
1596 dev_info(pcie->dev, "Link didn't transition to L2 state\n");
1598 * TX lane clock freq will reset to Gen1 only if link is in L2
1600 * So apply pex_rst to end point to force RP to go into detect
1603 data = appl_readl(pcie, APPL_PINMUX);
1604 data &= ~APPL_PINMUX_PEX_RST;
1605 appl_writel(pcie, data, APPL_PINMUX);
1608 * Some cards do not go to detect state even after de-asserting
1609 * PERST#. So, de-assert LTSSM to bring link to detect state.
1611 data = readl(pcie->appl_base + APPL_CTRL);
1612 data &= ~APPL_CTRL_LTSSM_EN;
1613 writel(data, pcie->appl_base + APPL_CTRL);
1615 err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG,
1618 APPL_DEBUG_LTSSM_STATE_MASK) >>
1619 APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1620 LTSSM_STATE_PRE_DETECT,
1623 dev_info(pcie->dev, "Link didn't go to detect state\n");
1626 * DBI registers may not be accessible after this as PLL-E would be
1627 * down depending on how CLKREQ is pulled by end point
1629 data = appl_readl(pcie, APPL_PINMUX);
1630 data |= (APPL_PINMUX_CLKREQ_OVERRIDE_EN | APPL_PINMUX_CLKREQ_OVERRIDE);
1631 /* Cut REFCLK to slot */
1632 data |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1633 data &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1634 appl_writel(pcie, data, APPL_PINMUX);
1637 static void tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie)
1639 tegra_pcie_downstream_dev_to_D0(pcie);
1640 dw_pcie_host_deinit(&pcie->pci.pp);
1641 tegra_pcie_dw_pme_turnoff(pcie);
1642 tegra_pcie_unconfig_controller(pcie);
1645 static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie)
1647 struct device *dev = pcie->dev;
1651 pm_runtime_enable(dev);
1653 ret = pm_runtime_get_sync(dev);
1655 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1657 goto fail_pm_get_sync;
1660 ret = pinctrl_pm_select_default_state(dev);
1662 dev_err(dev, "Failed to configure sideband pins: %d\n", ret);
1663 goto fail_pm_get_sync;
1666 ret = tegra_pcie_init_controller(pcie);
1668 dev_err(dev, "Failed to initialize controller: %d\n", ret);
1669 goto fail_pm_get_sync;
1672 pcie->link_state = tegra_pcie_dw_link_up(&pcie->pci);
1673 if (!pcie->link_state) {
1675 goto fail_host_init;
1678 name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node);
1681 goto fail_host_init;
1684 pcie->debugfs = debugfs_create_dir(name, NULL);
1690 tegra_pcie_deinit_controller(pcie);
1692 pm_runtime_put_sync(dev);
1693 pm_runtime_disable(dev);
1697 static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie)
1702 if (pcie->ep_state == EP_STATE_DISABLED)
1706 val = appl_readl(pcie, APPL_CTRL);
1707 val &= ~APPL_CTRL_LTSSM_EN;
1708 appl_writel(pcie, val, APPL_CTRL);
1710 ret = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val,
1711 ((val & APPL_DEBUG_LTSSM_STATE_MASK) >>
1712 APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1713 LTSSM_STATE_PRE_DETECT,
1716 dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret);
1718 reset_control_assert(pcie->core_rst);
1720 tegra_pcie_disable_phy(pcie);
1722 reset_control_assert(pcie->core_apb_rst);
1724 clk_disable_unprepare(pcie->core_clk);
1726 pm_runtime_put_sync(pcie->dev);
1728 if (pcie->enable_ext_refclk) {
1729 ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1731 dev_err(pcie->dev, "Failed to turn off UPHY: %d\n",
1735 ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1737 dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", ret);
1739 pcie->ep_state = EP_STATE_DISABLED;
1740 dev_dbg(pcie->dev, "Uninitialization of endpoint is completed\n");
1743 static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie)
1745 struct dw_pcie *pci = &pcie->pci;
1746 struct dw_pcie_ep *ep = &pci->ep;
1747 struct device *dev = pcie->dev;
1752 if (pcie->ep_state == EP_STATE_ENABLED)
1755 ret = pm_runtime_resume_and_get(dev);
1757 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1762 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true);
1764 dev_err(pcie->dev, "Failed to enable controller %u: %d\n",
1766 goto fail_set_ctrl_state;
1769 if (pcie->enable_ext_refclk) {
1770 ret = tegra_pcie_bpmp_set_pll_state(pcie, true);
1772 dev_err(dev, "Failed to init UPHY for PCIe EP: %d\n",
1778 ret = clk_prepare_enable(pcie->core_clk);
1780 dev_err(dev, "Failed to enable core clock: %d\n", ret);
1781 goto fail_core_clk_enable;
1784 ret = reset_control_deassert(pcie->core_apb_rst);
1786 dev_err(dev, "Failed to deassert core APB reset: %d\n", ret);
1787 goto fail_core_apb_rst;
1790 ret = tegra_pcie_enable_phy(pcie);
1792 dev_err(dev, "Failed to enable PHY: %d\n", ret);
1796 /* Clear any stale interrupt statuses */
1797 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
1798 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
1799 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
1800 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
1801 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
1802 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
1803 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
1804 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
1805 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
1806 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
1807 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
1808 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
1809 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
1810 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
1811 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
1813 /* configure this core for EP mode operation */
1814 val = appl_readl(pcie, APPL_DM_TYPE);
1815 val &= ~APPL_DM_TYPE_MASK;
1816 val |= APPL_DM_TYPE_EP;
1817 appl_writel(pcie, val, APPL_DM_TYPE);
1819 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1821 val = appl_readl(pcie, APPL_CTRL);
1822 val |= APPL_CTRL_SYS_PRE_DET_STATE;
1823 val |= APPL_CTRL_HW_HOT_RST_EN;
1824 appl_writel(pcie, val, APPL_CTRL);
1826 val = appl_readl(pcie, APPL_CFG_MISC);
1827 val |= APPL_CFG_MISC_SLV_EP_MODE;
1828 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1829 appl_writel(pcie, val, APPL_CFG_MISC);
1831 val = appl_readl(pcie, APPL_PINMUX);
1832 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1833 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1834 appl_writel(pcie, val, APPL_PINMUX);
1836 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1837 APPL_CFG_BASE_ADDR);
1839 appl_writel(pcie, pcie->atu_dma_res->start &
1840 APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1841 APPL_CFG_IATU_DMA_BASE_ADDR);
1843 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
1844 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
1845 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
1846 val |= APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN;
1847 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
1849 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
1850 val |= APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN;
1851 val |= APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN;
1852 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
1854 reset_control_deassert(pcie->core_rst);
1856 if (pcie->update_fc_fixup) {
1857 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
1858 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
1859 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
1862 config_gen3_gen4_eq_presets(pcie);
1864 init_host_aspm(pcie);
1866 /* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */
1867 if (!pcie->supports_clkreq) {
1868 disable_aspm_l11(pcie);
1869 disable_aspm_l12(pcie);
1872 if (!pcie->of_data->has_l1ss_exit_fix) {
1873 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
1874 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
1875 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
1878 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
1881 /* Clear Slot Clock Configuration bit if SRNS configuration */
1882 if (pcie->enable_srns) {
1883 val_16 = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
1885 val_16 &= ~PCI_EXP_LNKSTA_SLC;
1886 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA,
1890 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
1892 val = (ep->msi_mem_phys & MSIX_ADDR_MATCH_LOW_OFF_MASK);
1893 val |= MSIX_ADDR_MATCH_LOW_OFF_EN;
1894 dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_LOW_OFF, val);
1895 val = (upper_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK);
1896 dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val);
1898 ret = dw_pcie_ep_init_complete(ep);
1900 dev_err(dev, "Failed to complete initialization: %d\n", ret);
1901 goto fail_init_complete;
1904 dw_pcie_ep_init_notify(ep);
1906 /* Program the private control to allow sending LTR upstream */
1907 if (pcie->of_data->has_ltr_req_fix) {
1908 val = appl_readl(pcie, APPL_LTR_MSG_2);
1909 val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE;
1910 appl_writel(pcie, val, APPL_LTR_MSG_2);
1914 val = appl_readl(pcie, APPL_CTRL);
1915 val |= APPL_CTRL_LTSSM_EN;
1916 appl_writel(pcie, val, APPL_CTRL);
1918 pcie->ep_state = EP_STATE_ENABLED;
1919 dev_dbg(dev, "Initialization of endpoint is completed\n");
1924 reset_control_assert(pcie->core_rst);
1925 tegra_pcie_disable_phy(pcie);
1927 reset_control_assert(pcie->core_apb_rst);
1929 clk_disable_unprepare(pcie->core_clk);
1930 fail_core_clk_enable:
1931 tegra_pcie_bpmp_set_pll_state(pcie, false);
1933 tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1934 fail_set_ctrl_state:
1935 pm_runtime_put_sync(dev);
1938 static irqreturn_t tegra_pcie_ep_pex_rst_irq(int irq, void *arg)
1940 struct tegra_pcie_dw *pcie = arg;
1942 if (gpiod_get_value(pcie->pex_rst_gpiod))
1943 pex_ep_event_pex_rst_assert(pcie);
1945 pex_ep_event_pex_rst_deassert(pcie);
1950 static int tegra_pcie_ep_raise_legacy_irq(struct tegra_pcie_dw *pcie, u16 irq)
1952 /* Tegra194 supports only INTA */
1956 appl_writel(pcie, 1, APPL_LEGACY_INTX);
1957 usleep_range(1000, 2000);
1958 appl_writel(pcie, 0, APPL_LEGACY_INTX);
1962 static int tegra_pcie_ep_raise_msi_irq(struct tegra_pcie_dw *pcie, u16 irq)
1964 if (unlikely(irq > 31))
1967 appl_writel(pcie, BIT(irq), APPL_MSI_CTRL_1);
1972 static int tegra_pcie_ep_raise_msix_irq(struct tegra_pcie_dw *pcie, u16 irq)
1974 struct dw_pcie_ep *ep = &pcie->pci.ep;
1976 writel(irq, ep->msi_mem);
1981 static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
1982 enum pci_epc_irq_type type,
1985 struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
1986 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1989 case PCI_EPC_IRQ_LEGACY:
1990 return tegra_pcie_ep_raise_legacy_irq(pcie, interrupt_num);
1992 case PCI_EPC_IRQ_MSI:
1993 return tegra_pcie_ep_raise_msi_irq(pcie, interrupt_num);
1995 case PCI_EPC_IRQ_MSIX:
1996 return tegra_pcie_ep_raise_msix_irq(pcie, interrupt_num);
1999 dev_err(pci->dev, "Unknown IRQ type\n");
2006 static const struct pci_epc_features tegra_pcie_epc_features = {
2007 .linkup_notifier = true,
2008 .core_init_notifier = true,
2009 .msi_capable = false,
2010 .msix_capable = false,
2011 .reserved_bar = 1 << BAR_2 | 1 << BAR_3 | 1 << BAR_4 | 1 << BAR_5,
2012 .bar_fixed_64bit = 1 << BAR_0,
2013 .bar_fixed_size[0] = SZ_1M,
2016 static const struct pci_epc_features*
2017 tegra_pcie_ep_get_features(struct dw_pcie_ep *ep)
2019 return &tegra_pcie_epc_features;
2022 static const struct dw_pcie_ep_ops pcie_ep_ops = {
2023 .raise_irq = tegra_pcie_ep_raise_irq,
2024 .get_features = tegra_pcie_ep_get_features,
2027 static int tegra_pcie_config_ep(struct tegra_pcie_dw *pcie,
2028 struct platform_device *pdev)
2030 struct dw_pcie *pci = &pcie->pci;
2031 struct device *dev = pcie->dev;
2032 struct dw_pcie_ep *ep;
2037 ep->ops = &pcie_ep_ops;
2039 ep->page_size = SZ_64K;
2041 ret = gpiod_set_debounce(pcie->pex_rst_gpiod, PERST_DEBOUNCE_TIME);
2043 dev_err(dev, "Failed to set PERST GPIO debounce time: %d\n",
2048 ret = gpiod_to_irq(pcie->pex_rst_gpiod);
2050 dev_err(dev, "Failed to get IRQ for PERST GPIO: %d\n", ret);
2053 pcie->pex_rst_irq = (unsigned int)ret;
2055 name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_pex_rst_irq",
2058 dev_err(dev, "Failed to create PERST IRQ string\n");
2062 irq_set_status_flags(pcie->pex_rst_irq, IRQ_NOAUTOEN);
2064 pcie->ep_state = EP_STATE_DISABLED;
2066 ret = devm_request_threaded_irq(dev, pcie->pex_rst_irq, NULL,
2067 tegra_pcie_ep_pex_rst_irq,
2068 IRQF_TRIGGER_RISING |
2069 IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
2070 name, (void *)pcie);
2072 dev_err(dev, "Failed to request IRQ for PERST: %d\n", ret);
2076 pm_runtime_enable(dev);
2078 ret = dw_pcie_ep_init(ep);
2080 dev_err(dev, "Failed to initialize DWC Endpoint subsystem: %d\n",
2082 pm_runtime_disable(dev);
2089 static int tegra_pcie_dw_probe(struct platform_device *pdev)
2091 const struct tegra_pcie_dw_of_data *data;
2092 struct device *dev = &pdev->dev;
2093 struct resource *atu_dma_res;
2094 struct tegra_pcie_dw *pcie;
2095 struct dw_pcie_rp *pp;
2096 struct dw_pcie *pci;
2102 data = of_device_get_match_data(dev);
2104 pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
2109 pci->dev = &pdev->dev;
2110 pci->ops = &tegra_dw_pcie_ops;
2111 pcie->dev = &pdev->dev;
2112 pcie->of_data = (struct tegra_pcie_dw_of_data *)data;
2113 pci->n_fts[0] = pcie->of_data->n_fts[0];
2114 pci->n_fts[1] = pcie->of_data->n_fts[1];
2116 pp->num_vectors = MAX_MSI_IRQS;
2118 ret = tegra_pcie_dw_parse_dt(pcie);
2120 const char *level = KERN_ERR;
2122 if (ret == -EPROBE_DEFER)
2125 dev_printk(level, dev,
2126 dev_fmt("Failed to parse device tree: %d\n"),
2131 ret = tegra_pcie_get_slot_regulators(pcie);
2133 const char *level = KERN_ERR;
2135 if (ret == -EPROBE_DEFER)
2138 dev_printk(level, dev,
2139 dev_fmt("Failed to get slot regulators: %d\n"),
2144 if (pcie->pex_refclk_sel_gpiod)
2145 gpiod_set_value(pcie->pex_refclk_sel_gpiod, 1);
2147 pcie->pex_ctl_supply = devm_regulator_get(dev, "vddio-pex-ctl");
2148 if (IS_ERR(pcie->pex_ctl_supply)) {
2149 ret = PTR_ERR(pcie->pex_ctl_supply);
2150 if (ret != -EPROBE_DEFER)
2151 dev_err(dev, "Failed to get regulator: %ld\n",
2152 PTR_ERR(pcie->pex_ctl_supply));
2156 pcie->core_clk = devm_clk_get(dev, "core");
2157 if (IS_ERR(pcie->core_clk)) {
2158 dev_err(dev, "Failed to get core clock: %ld\n",
2159 PTR_ERR(pcie->core_clk));
2160 return PTR_ERR(pcie->core_clk);
2163 pcie->appl_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2165 if (!pcie->appl_res) {
2166 dev_err(dev, "Failed to find \"appl\" region\n");
2170 pcie->appl_base = devm_ioremap_resource(dev, pcie->appl_res);
2171 if (IS_ERR(pcie->appl_base))
2172 return PTR_ERR(pcie->appl_base);
2174 pcie->core_apb_rst = devm_reset_control_get(dev, "apb");
2175 if (IS_ERR(pcie->core_apb_rst)) {
2176 dev_err(dev, "Failed to get APB reset: %ld\n",
2177 PTR_ERR(pcie->core_apb_rst));
2178 return PTR_ERR(pcie->core_apb_rst);
2181 phys = devm_kcalloc(dev, pcie->phy_count, sizeof(*phys), GFP_KERNEL);
2185 for (i = 0; i < pcie->phy_count; i++) {
2186 name = kasprintf(GFP_KERNEL, "p2u-%u", i);
2188 dev_err(dev, "Failed to create P2U string\n");
2191 phys[i] = devm_phy_get(dev, name);
2193 if (IS_ERR(phys[i])) {
2194 ret = PTR_ERR(phys[i]);
2195 if (ret != -EPROBE_DEFER)
2196 dev_err(dev, "Failed to get PHY: %d\n", ret);
2203 atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2206 dev_err(dev, "Failed to find \"atu_dma\" region\n");
2209 pcie->atu_dma_res = atu_dma_res;
2211 pci->atu_size = resource_size(atu_dma_res);
2212 pci->atu_base = devm_ioremap_resource(dev, atu_dma_res);
2213 if (IS_ERR(pci->atu_base))
2214 return PTR_ERR(pci->atu_base);
2216 pcie->core_rst = devm_reset_control_get(dev, "core");
2217 if (IS_ERR(pcie->core_rst)) {
2218 dev_err(dev, "Failed to get core reset: %ld\n",
2219 PTR_ERR(pcie->core_rst));
2220 return PTR_ERR(pcie->core_rst);
2223 pp->irq = platform_get_irq_byname(pdev, "intr");
2227 pcie->bpmp = tegra_bpmp_get(dev);
2228 if (IS_ERR(pcie->bpmp))
2229 return PTR_ERR(pcie->bpmp);
2231 platform_set_drvdata(pdev, pcie);
2233 pcie->icc_path = devm_of_icc_get(&pdev->dev, "write");
2234 ret = PTR_ERR_OR_ZERO(pcie->icc_path);
2236 tegra_bpmp_put(pcie->bpmp);
2237 dev_err_probe(&pdev->dev, ret, "failed to get write interconnect\n");
2241 switch (pcie->of_data->mode) {
2242 case DW_PCIE_RC_TYPE:
2243 ret = devm_request_irq(dev, pp->irq, tegra_pcie_rp_irq_handler,
2244 IRQF_SHARED, "tegra-pcie-intr", pcie);
2246 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2251 ret = tegra_pcie_config_rp(pcie);
2252 if (ret && ret != -ENOMEDIUM)
2258 case DW_PCIE_EP_TYPE:
2259 ret = devm_request_threaded_irq(dev, pp->irq,
2260 tegra_pcie_ep_hard_irq,
2261 tegra_pcie_ep_irq_thread,
2262 IRQF_SHARED | IRQF_ONESHOT,
2263 "tegra-pcie-ep-intr", pcie);
2265 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2270 ret = tegra_pcie_config_ep(pcie, pdev);
2276 dev_err(dev, "Invalid PCIe device type %d\n",
2277 pcie->of_data->mode);
2281 tegra_bpmp_put(pcie->bpmp);
2285 static void tegra_pcie_dw_remove(struct platform_device *pdev)
2287 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2289 if (pcie->of_data->mode == DW_PCIE_RC_TYPE) {
2290 if (!pcie->link_state)
2293 debugfs_remove_recursive(pcie->debugfs);
2294 tegra_pcie_deinit_controller(pcie);
2295 pm_runtime_put_sync(pcie->dev);
2297 disable_irq(pcie->pex_rst_irq);
2298 pex_ep_event_pex_rst_assert(pcie);
2301 pm_runtime_disable(pcie->dev);
2302 tegra_bpmp_put(pcie->bpmp);
2303 if (pcie->pex_refclk_sel_gpiod)
2304 gpiod_set_value(pcie->pex_refclk_sel_gpiod, 0);
2307 static int tegra_pcie_dw_suspend_late(struct device *dev)
2309 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2312 if (pcie->of_data->mode == DW_PCIE_EP_TYPE) {
2313 dev_err(dev, "Failed to Suspend as Tegra PCIe is in EP mode\n");
2317 if (!pcie->link_state)
2320 /* Enable HW_HOT_RST mode */
2321 if (!pcie->of_data->has_sbr_reset_fix) {
2322 val = appl_readl(pcie, APPL_CTRL);
2323 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2324 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2325 val |= APPL_CTRL_HW_HOT_RST_EN;
2326 appl_writel(pcie, val, APPL_CTRL);
2332 static int tegra_pcie_dw_suspend_noirq(struct device *dev)
2334 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2336 if (!pcie->link_state)
2339 tegra_pcie_downstream_dev_to_D0(pcie);
2340 tegra_pcie_dw_pme_turnoff(pcie);
2341 tegra_pcie_unconfig_controller(pcie);
2346 static int tegra_pcie_dw_resume_noirq(struct device *dev)
2348 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2351 if (!pcie->link_state)
2354 ret = tegra_pcie_config_controller(pcie, true);
2358 ret = tegra_pcie_dw_host_init(&pcie->pci.pp);
2360 dev_err(dev, "Failed to init host: %d\n", ret);
2361 goto fail_host_init;
2364 dw_pcie_setup_rc(&pcie->pci.pp);
2366 ret = tegra_pcie_dw_start_link(&pcie->pci);
2368 goto fail_host_init;
2373 tegra_pcie_unconfig_controller(pcie);
2377 static int tegra_pcie_dw_resume_early(struct device *dev)
2379 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2382 if (pcie->of_data->mode == DW_PCIE_EP_TYPE) {
2383 dev_err(dev, "Suspend is not supported in EP mode");
2387 if (!pcie->link_state)
2390 /* Disable HW_HOT_RST mode */
2391 if (!pcie->of_data->has_sbr_reset_fix) {
2392 val = appl_readl(pcie, APPL_CTRL);
2393 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2394 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2395 val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST <<
2396 APPL_CTRL_HW_HOT_RST_MODE_SHIFT;
2397 val &= ~APPL_CTRL_HW_HOT_RST_EN;
2398 appl_writel(pcie, val, APPL_CTRL);
2404 static void tegra_pcie_dw_shutdown(struct platform_device *pdev)
2406 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2408 if (pcie->of_data->mode == DW_PCIE_RC_TYPE) {
2409 if (!pcie->link_state)
2412 debugfs_remove_recursive(pcie->debugfs);
2413 tegra_pcie_downstream_dev_to_D0(pcie);
2415 disable_irq(pcie->pci.pp.irq);
2416 if (IS_ENABLED(CONFIG_PCI_MSI))
2417 disable_irq(pcie->pci.pp.msi_irq[0]);
2419 tegra_pcie_dw_pme_turnoff(pcie);
2420 tegra_pcie_unconfig_controller(pcie);
2421 pm_runtime_put_sync(pcie->dev);
2423 disable_irq(pcie->pex_rst_irq);
2424 pex_ep_event_pex_rst_assert(pcie);
2428 static const struct tegra_pcie_dw_of_data tegra194_pcie_dw_rc_of_data = {
2429 .version = TEGRA194_DWC_IP_VER,
2430 .mode = DW_PCIE_RC_TYPE,
2431 .cdm_chk_int_en_bit = BIT(19),
2432 /* Gen4 - 5, 6, 8 and 9 presets enabled */
2433 .gen4_preset_vec = 0x360,
2434 .n_fts = { 52, 52 },
2437 static const struct tegra_pcie_dw_of_data tegra194_pcie_dw_ep_of_data = {
2438 .version = TEGRA194_DWC_IP_VER,
2439 .mode = DW_PCIE_EP_TYPE,
2440 .cdm_chk_int_en_bit = BIT(19),
2441 /* Gen4 - 5, 6, 8 and 9 presets enabled */
2442 .gen4_preset_vec = 0x360,
2443 .n_fts = { 52, 52 },
2446 static const struct tegra_pcie_dw_of_data tegra234_pcie_dw_rc_of_data = {
2447 .version = TEGRA234_DWC_IP_VER,
2448 .mode = DW_PCIE_RC_TYPE,
2449 .has_msix_doorbell_access_fix = true,
2450 .has_sbr_reset_fix = true,
2451 .has_l1ss_exit_fix = true,
2452 .cdm_chk_int_en_bit = BIT(18),
2453 /* Gen4 - 6, 8 and 9 presets enabled */
2454 .gen4_preset_vec = 0x340,
2455 .n_fts = { 52, 80 },
2458 static const struct tegra_pcie_dw_of_data tegra234_pcie_dw_ep_of_data = {
2459 .version = TEGRA234_DWC_IP_VER,
2460 .mode = DW_PCIE_EP_TYPE,
2461 .has_l1ss_exit_fix = true,
2462 .has_ltr_req_fix = true,
2463 .cdm_chk_int_en_bit = BIT(18),
2464 /* Gen4 - 6, 8 and 9 presets enabled */
2465 .gen4_preset_vec = 0x340,
2466 .n_fts = { 52, 80 },
2469 static const struct of_device_id tegra_pcie_dw_of_match[] = {
2471 .compatible = "nvidia,tegra194-pcie",
2472 .data = &tegra194_pcie_dw_rc_of_data,
2475 .compatible = "nvidia,tegra194-pcie-ep",
2476 .data = &tegra194_pcie_dw_ep_of_data,
2479 .compatible = "nvidia,tegra234-pcie",
2480 .data = &tegra234_pcie_dw_rc_of_data,
2483 .compatible = "nvidia,tegra234-pcie-ep",
2484 .data = &tegra234_pcie_dw_ep_of_data,
2489 static const struct dev_pm_ops tegra_pcie_dw_pm_ops = {
2490 .suspend_late = tegra_pcie_dw_suspend_late,
2491 .suspend_noirq = tegra_pcie_dw_suspend_noirq,
2492 .resume_noirq = tegra_pcie_dw_resume_noirq,
2493 .resume_early = tegra_pcie_dw_resume_early,
2496 static struct platform_driver tegra_pcie_dw_driver = {
2497 .probe = tegra_pcie_dw_probe,
2498 .remove_new = tegra_pcie_dw_remove,
2499 .shutdown = tegra_pcie_dw_shutdown,
2501 .name = "tegra194-pcie",
2502 .pm = &tegra_pcie_dw_pm_ops,
2503 .of_match_table = tegra_pcie_dw_of_match,
2506 module_platform_driver(tegra_pcie_dw_driver);
2508 MODULE_DEVICE_TABLE(of, tegra_pcie_dw_of_match);
2510 MODULE_AUTHOR("Vidya Sagar <vidyas@nvidia.com>");
2511 MODULE_DESCRIPTION("NVIDIA PCIe host controller driver");
2512 MODULE_LICENSE("GPL v2");