1 // SPDX-License-Identifier: GPL-2.0+
3 * PCIe host controller driver for Tegra SoCs
5 * Copyright (c) 2010, CompuLab, Ltd.
6 * Author: Mike Rapoport <mike@compulab.co.il>
8 * Based on NVIDIA PCIe driver
9 * Copyright (c) 2008-2009, NVIDIA Corporation.
11 * Bits taken from arch/arm/mach-dove/pcie.c
13 * Author: Thierry Reding <treding@nvidia.com>
16 #include <linux/clk.h>
17 #include <linux/debugfs.h>
18 #include <linux/delay.h>
19 #include <linux/export.h>
20 #include <linux/gpio/consumer.h>
21 #include <linux/interrupt.h>
22 #include <linux/iopoll.h>
23 #include <linux/irq.h>
24 #include <linux/irqchip/chained_irq.h>
25 #include <linux/irqdomain.h>
26 #include <linux/kernel.h>
27 #include <linux/init.h>
28 #include <linux/module.h>
29 #include <linux/msi.h>
30 #include <linux/of_address.h>
31 #include <linux/of_pci.h>
32 #include <linux/of_platform.h>
33 #include <linux/pci.h>
34 #include <linux/phy/phy.h>
35 #include <linux/pinctrl/consumer.h>
36 #include <linux/platform_device.h>
37 #include <linux/reset.h>
38 #include <linux/sizes.h>
39 #include <linux/slab.h>
40 #include <linux/vmalloc.h>
41 #include <linux/regulator/consumer.h>
43 #include <soc/tegra/cpuidle.h>
44 #include <soc/tegra/pmc.h>
48 #define INT_PCI_MSI_NR (8 * 32)
50 /* register definitions */
52 #define AFI_AXI_BAR0_SZ 0x00
53 #define AFI_AXI_BAR1_SZ 0x04
54 #define AFI_AXI_BAR2_SZ 0x08
55 #define AFI_AXI_BAR3_SZ 0x0c
56 #define AFI_AXI_BAR4_SZ 0x10
57 #define AFI_AXI_BAR5_SZ 0x14
59 #define AFI_AXI_BAR0_START 0x18
60 #define AFI_AXI_BAR1_START 0x1c
61 #define AFI_AXI_BAR2_START 0x20
62 #define AFI_AXI_BAR3_START 0x24
63 #define AFI_AXI_BAR4_START 0x28
64 #define AFI_AXI_BAR5_START 0x2c
66 #define AFI_FPCI_BAR0 0x30
67 #define AFI_FPCI_BAR1 0x34
68 #define AFI_FPCI_BAR2 0x38
69 #define AFI_FPCI_BAR3 0x3c
70 #define AFI_FPCI_BAR4 0x40
71 #define AFI_FPCI_BAR5 0x44
73 #define AFI_CACHE_BAR0_SZ 0x48
74 #define AFI_CACHE_BAR0_ST 0x4c
75 #define AFI_CACHE_BAR1_SZ 0x50
76 #define AFI_CACHE_BAR1_ST 0x54
78 #define AFI_MSI_BAR_SZ 0x60
79 #define AFI_MSI_FPCI_BAR_ST 0x64
80 #define AFI_MSI_AXI_BAR_ST 0x68
82 #define AFI_MSI_VEC(x) (0x6c + ((x) * 4))
83 #define AFI_MSI_EN_VEC(x) (0x8c + ((x) * 4))
85 #define AFI_CONFIGURATION 0xac
86 #define AFI_CONFIGURATION_EN_FPCI (1 << 0)
87 #define AFI_CONFIGURATION_CLKEN_OVERRIDE (1 << 31)
89 #define AFI_FPCI_ERROR_MASKS 0xb0
91 #define AFI_INTR_MASK 0xb4
92 #define AFI_INTR_MASK_INT_MASK (1 << 0)
93 #define AFI_INTR_MASK_MSI_MASK (1 << 8)
95 #define AFI_INTR_CODE 0xb8
96 #define AFI_INTR_CODE_MASK 0xf
97 #define AFI_INTR_INI_SLAVE_ERROR 1
98 #define AFI_INTR_INI_DECODE_ERROR 2
99 #define AFI_INTR_TARGET_ABORT 3
100 #define AFI_INTR_MASTER_ABORT 4
101 #define AFI_INTR_INVALID_WRITE 5
102 #define AFI_INTR_LEGACY 6
103 #define AFI_INTR_FPCI_DECODE_ERROR 7
104 #define AFI_INTR_AXI_DECODE_ERROR 8
105 #define AFI_INTR_FPCI_TIMEOUT 9
106 #define AFI_INTR_PE_PRSNT_SENSE 10
107 #define AFI_INTR_PE_CLKREQ_SENSE 11
108 #define AFI_INTR_CLKCLAMP_SENSE 12
109 #define AFI_INTR_RDY4PD_SENSE 13
110 #define AFI_INTR_P2P_ERROR 14
112 #define AFI_INTR_SIGNATURE 0xbc
113 #define AFI_UPPER_FPCI_ADDRESS 0xc0
114 #define AFI_SM_INTR_ENABLE 0xc4
115 #define AFI_SM_INTR_INTA_ASSERT (1 << 0)
116 #define AFI_SM_INTR_INTB_ASSERT (1 << 1)
117 #define AFI_SM_INTR_INTC_ASSERT (1 << 2)
118 #define AFI_SM_INTR_INTD_ASSERT (1 << 3)
119 #define AFI_SM_INTR_INTA_DEASSERT (1 << 4)
120 #define AFI_SM_INTR_INTB_DEASSERT (1 << 5)
121 #define AFI_SM_INTR_INTC_DEASSERT (1 << 6)
122 #define AFI_SM_INTR_INTD_DEASSERT (1 << 7)
124 #define AFI_AFI_INTR_ENABLE 0xc8
125 #define AFI_INTR_EN_INI_SLVERR (1 << 0)
126 #define AFI_INTR_EN_INI_DECERR (1 << 1)
127 #define AFI_INTR_EN_TGT_SLVERR (1 << 2)
128 #define AFI_INTR_EN_TGT_DECERR (1 << 3)
129 #define AFI_INTR_EN_TGT_WRERR (1 << 4)
130 #define AFI_INTR_EN_DFPCI_DECERR (1 << 5)
131 #define AFI_INTR_EN_AXI_DECERR (1 << 6)
132 #define AFI_INTR_EN_FPCI_TIMEOUT (1 << 7)
133 #define AFI_INTR_EN_PRSNT_SENSE (1 << 8)
135 #define AFI_PCIE_PME 0xf0
137 #define AFI_PCIE_CONFIG 0x0f8
138 #define AFI_PCIE_CONFIG_PCIE_DISABLE(x) (1 << ((x) + 1))
139 #define AFI_PCIE_CONFIG_PCIE_DISABLE_ALL 0xe
140 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_MASK (0xf << 20)
141 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_SINGLE (0x0 << 20)
142 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_420 (0x0 << 20)
143 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X2_X1 (0x0 << 20)
144 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_401 (0x0 << 20)
145 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_DUAL (0x1 << 20)
146 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_222 (0x1 << 20)
147 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X4_X1 (0x1 << 20)
148 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_211 (0x1 << 20)
149 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_411 (0x2 << 20)
150 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_111 (0x2 << 20)
151 #define AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO(x) (1 << ((x) + 29))
152 #define AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO_ALL (0x7 << 29)
154 #define AFI_FUSE 0x104
155 #define AFI_FUSE_PCIE_T0_GEN2_DIS (1 << 2)
157 #define AFI_PEX0_CTRL 0x110
158 #define AFI_PEX1_CTRL 0x118
159 #define AFI_PEX_CTRL_RST (1 << 0)
160 #define AFI_PEX_CTRL_CLKREQ_EN (1 << 1)
161 #define AFI_PEX_CTRL_REFCLK_EN (1 << 3)
162 #define AFI_PEX_CTRL_OVERRIDE_EN (1 << 4)
164 #define AFI_PLLE_CONTROL 0x160
165 #define AFI_PLLE_CONTROL_BYPASS_PADS2PLLE_CONTROL (1 << 9)
166 #define AFI_PLLE_CONTROL_PADS2PLLE_CONTROL_EN (1 << 1)
168 #define AFI_PEXBIAS_CTRL_0 0x168
170 #define RP_ECTL_2_R1 0x00000e84
171 #define RP_ECTL_2_R1_RX_CTLE_1C_MASK 0xffff
173 #define RP_ECTL_4_R1 0x00000e8c
174 #define RP_ECTL_4_R1_RX_CDR_CTRL_1C_MASK (0xffff << 16)
175 #define RP_ECTL_4_R1_RX_CDR_CTRL_1C_SHIFT 16
177 #define RP_ECTL_5_R1 0x00000e90
178 #define RP_ECTL_5_R1_RX_EQ_CTRL_L_1C_MASK 0xffffffff
180 #define RP_ECTL_6_R1 0x00000e94
181 #define RP_ECTL_6_R1_RX_EQ_CTRL_H_1C_MASK 0xffffffff
183 #define RP_ECTL_2_R2 0x00000ea4
184 #define RP_ECTL_2_R2_RX_CTLE_1C_MASK 0xffff
186 #define RP_ECTL_4_R2 0x00000eac
187 #define RP_ECTL_4_R2_RX_CDR_CTRL_1C_MASK (0xffff << 16)
188 #define RP_ECTL_4_R2_RX_CDR_CTRL_1C_SHIFT 16
190 #define RP_ECTL_5_R2 0x00000eb0
191 #define RP_ECTL_5_R2_RX_EQ_CTRL_L_1C_MASK 0xffffffff
193 #define RP_ECTL_6_R2 0x00000eb4
194 #define RP_ECTL_6_R2_RX_EQ_CTRL_H_1C_MASK 0xffffffff
196 #define RP_VEND_XP 0x00000f00
197 #define RP_VEND_XP_DL_UP (1 << 30)
198 #define RP_VEND_XP_OPPORTUNISTIC_ACK (1 << 27)
199 #define RP_VEND_XP_OPPORTUNISTIC_UPDATEFC (1 << 28)
200 #define RP_VEND_XP_UPDATE_FC_THRESHOLD_MASK (0xff << 18)
202 #define RP_VEND_CTL0 0x00000f44
203 #define RP_VEND_CTL0_DSK_RST_PULSE_WIDTH_MASK (0xf << 12)
204 #define RP_VEND_CTL0_DSK_RST_PULSE_WIDTH (0x9 << 12)
206 #define RP_VEND_CTL1 0x00000f48
207 #define RP_VEND_CTL1_ERPT (1 << 13)
209 #define RP_VEND_XP_BIST 0x00000f4c
210 #define RP_VEND_XP_BIST_GOTO_L1_L2_AFTER_DLLP_DONE (1 << 28)
212 #define RP_VEND_CTL2 0x00000fa8
213 #define RP_VEND_CTL2_PCA_ENABLE (1 << 7)
215 #define RP_PRIV_MISC 0x00000fe0
216 #define RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT (0xe << 0)
217 #define RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT (0xf << 0)
218 #define RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD_MASK (0x7f << 16)
219 #define RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD (0xf << 16)
220 #define RP_PRIV_MISC_CTLR_CLK_CLAMP_ENABLE (1 << 23)
221 #define RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD_MASK (0x7f << 24)
222 #define RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD (0xf << 24)
223 #define RP_PRIV_MISC_TMS_CLK_CLAMP_ENABLE (1 << 31)
225 #define RP_LINK_CONTROL_STATUS 0x00000090
226 #define RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE 0x20000000
227 #define RP_LINK_CONTROL_STATUS_LINKSTAT_MASK 0x3fff0000
229 #define RP_LINK_CONTROL_STATUS_2 0x000000b0
231 #define PADS_CTL_SEL 0x0000009c
233 #define PADS_CTL 0x000000a0
234 #define PADS_CTL_IDDQ_1L (1 << 0)
235 #define PADS_CTL_TX_DATA_EN_1L (1 << 6)
236 #define PADS_CTL_RX_DATA_EN_1L (1 << 10)
238 #define PADS_PLL_CTL_TEGRA20 0x000000b8
239 #define PADS_PLL_CTL_TEGRA30 0x000000b4
240 #define PADS_PLL_CTL_RST_B4SM (1 << 1)
241 #define PADS_PLL_CTL_LOCKDET (1 << 8)
242 #define PADS_PLL_CTL_REFCLK_MASK (0x3 << 16)
243 #define PADS_PLL_CTL_REFCLK_INTERNAL_CML (0 << 16)
244 #define PADS_PLL_CTL_REFCLK_INTERNAL_CMOS (1 << 16)
245 #define PADS_PLL_CTL_REFCLK_EXTERNAL (2 << 16)
246 #define PADS_PLL_CTL_TXCLKREF_MASK (0x1 << 20)
247 #define PADS_PLL_CTL_TXCLKREF_DIV10 (0 << 20)
248 #define PADS_PLL_CTL_TXCLKREF_DIV5 (1 << 20)
249 #define PADS_PLL_CTL_TXCLKREF_BUF_EN (1 << 22)
251 #define PADS_REFCLK_CFG0 0x000000c8
252 #define PADS_REFCLK_CFG1 0x000000cc
253 #define PADS_REFCLK_BIAS 0x000000d0
256 * Fields in PADS_REFCLK_CFG*. Those registers form an array of 16-bit
257 * entries, one entry per PCIe port. These field definitions and desired
258 * values aren't in the TRM, but do come from NVIDIA.
260 #define PADS_REFCLK_CFG_TERM_SHIFT 2 /* 6:2 */
261 #define PADS_REFCLK_CFG_E_TERM_SHIFT 7
262 #define PADS_REFCLK_CFG_PREDI_SHIFT 8 /* 11:8 */
263 #define PADS_REFCLK_CFG_DRVI_SHIFT 12 /* 15:12 */
265 #define PME_ACK_TIMEOUT 10000
266 #define LINK_RETRAIN_TIMEOUT 100000 /* in usec */
269 DECLARE_BITMAP(used, INT_PCI_MSI_NR);
270 struct irq_domain *domain;
271 struct mutex map_lock;
272 spinlock_t mask_lock;
278 /* used to differentiate between Tegra SoC generations */
279 struct tegra_pcie_port_soc {
286 struct tegra_pcie_soc {
287 unsigned int num_ports;
288 const struct tegra_pcie_port_soc *ports;
289 unsigned int msi_base_shift;
290 unsigned long afi_pex2_ctrl;
293 u32 pads_refclk_cfg0;
294 u32 pads_refclk_cfg1;
295 u32 update_fc_threshold;
296 bool has_pex_clkreq_en;
297 bool has_pex_bias_ctrl;
298 bool has_intr_prsnt_sense;
301 bool force_pca_enable;
303 bool update_clamp_threshold;
304 bool program_deskew_time;
305 bool update_fc_timer;
337 struct reset_control *pex_rst;
338 struct reset_control *afi_rst;
339 struct reset_control *pcie_xrst;
344 struct tegra_msi msi;
346 struct list_head ports;
349 struct regulator_bulk_data *supplies;
350 unsigned int num_supplies;
352 const struct tegra_pcie_soc *soc;
353 struct dentry *debugfs;
356 static inline struct tegra_pcie *msi_to_pcie(struct tegra_msi *msi)
358 return container_of(msi, struct tegra_pcie, msi);
361 struct tegra_pcie_port {
362 struct tegra_pcie *pcie;
363 struct device_node *np;
364 struct list_head list;
365 struct resource regs;
372 struct gpio_desc *reset_gpio;
375 static inline void afi_writel(struct tegra_pcie *pcie, u32 value,
376 unsigned long offset)
378 writel(value, pcie->afi + offset);
381 static inline u32 afi_readl(struct tegra_pcie *pcie, unsigned long offset)
383 return readl(pcie->afi + offset);
386 static inline void pads_writel(struct tegra_pcie *pcie, u32 value,
387 unsigned long offset)
389 writel(value, pcie->pads + offset);
392 static inline u32 pads_readl(struct tegra_pcie *pcie, unsigned long offset)
394 return readl(pcie->pads + offset);
398 * The configuration space mapping on Tegra is somewhat similar to the ECAM
399 * defined by PCIe. However it deviates a bit in how the 4 bits for extended
400 * register accesses are mapped:
402 * [27:24] extended register number
404 * [15:11] device number
405 * [10: 8] function number
406 * [ 7: 0] register number
408 * Mapping the whole extended configuration space would require 256 MiB of
409 * virtual address space, only a small part of which will actually be used.
411 * To work around this, a 4 KiB region is used to generate the required
412 * configuration transaction with relevant B:D:F and register offset values.
413 * This is achieved by dynamically programming base address and size of
414 * AFI_AXI_BAR used for end point config space mapping to make sure that the
415 * address (access to which generates correct config transaction) falls in
418 static unsigned int tegra_pcie_conf_offset(u8 bus, unsigned int devfn,
421 return ((where & 0xf00) << 16) | (bus << 16) | (PCI_SLOT(devfn) << 11) |
422 (PCI_FUNC(devfn) << 8) | (where & 0xff);
425 static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus,
429 struct tegra_pcie *pcie = bus->sysdata;
430 void __iomem *addr = NULL;
432 if (bus->number == 0) {
433 unsigned int slot = PCI_SLOT(devfn);
434 struct tegra_pcie_port *port;
436 list_for_each_entry(port, &pcie->ports, list) {
437 if (port->index + 1 == slot) {
438 addr = port->base + (where & ~3);
446 offset = tegra_pcie_conf_offset(bus->number, devfn, where);
448 /* move 4 KiB window to offset within the FPCI region */
449 base = 0xfe100000 + ((offset & ~(SZ_4K - 1)) >> 8);
450 afi_writel(pcie, base, AFI_FPCI_BAR0);
452 /* move to correct offset within the 4 KiB page */
453 addr = pcie->cfg + (offset & (SZ_4K - 1));
459 static int tegra_pcie_config_read(struct pci_bus *bus, unsigned int devfn,
460 int where, int size, u32 *value)
462 if (bus->number == 0)
463 return pci_generic_config_read32(bus, devfn, where, size,
466 return pci_generic_config_read(bus, devfn, where, size, value);
469 static int tegra_pcie_config_write(struct pci_bus *bus, unsigned int devfn,
470 int where, int size, u32 value)
472 if (bus->number == 0)
473 return pci_generic_config_write32(bus, devfn, where, size,
476 return pci_generic_config_write(bus, devfn, where, size, value);
479 static struct pci_ops tegra_pcie_ops = {
480 .map_bus = tegra_pcie_map_bus,
481 .read = tegra_pcie_config_read,
482 .write = tegra_pcie_config_write,
485 static unsigned long tegra_pcie_port_get_pex_ctrl(struct tegra_pcie_port *port)
487 const struct tegra_pcie_soc *soc = port->pcie->soc;
488 unsigned long ret = 0;
490 switch (port->index) {
500 ret = soc->afi_pex2_ctrl;
507 static void tegra_pcie_port_reset(struct tegra_pcie_port *port)
509 unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port);
512 /* pulse reset signal */
513 if (port->reset_gpio) {
514 gpiod_set_value(port->reset_gpio, 1);
516 value = afi_readl(port->pcie, ctrl);
517 value &= ~AFI_PEX_CTRL_RST;
518 afi_writel(port->pcie, value, ctrl);
521 usleep_range(1000, 2000);
523 if (port->reset_gpio) {
524 gpiod_set_value(port->reset_gpio, 0);
526 value = afi_readl(port->pcie, ctrl);
527 value |= AFI_PEX_CTRL_RST;
528 afi_writel(port->pcie, value, ctrl);
532 static void tegra_pcie_enable_rp_features(struct tegra_pcie_port *port)
534 const struct tegra_pcie_soc *soc = port->pcie->soc;
537 /* Enable AER capability */
538 value = readl(port->base + RP_VEND_CTL1);
539 value |= RP_VEND_CTL1_ERPT;
540 writel(value, port->base + RP_VEND_CTL1);
542 /* Optimal settings to enhance bandwidth */
543 value = readl(port->base + RP_VEND_XP);
544 value |= RP_VEND_XP_OPPORTUNISTIC_ACK;
545 value |= RP_VEND_XP_OPPORTUNISTIC_UPDATEFC;
546 writel(value, port->base + RP_VEND_XP);
549 * LTSSM will wait for DLLP to finish before entering L1 or L2,
550 * to avoid truncation of PM messages which results in receiver errors
552 value = readl(port->base + RP_VEND_XP_BIST);
553 value |= RP_VEND_XP_BIST_GOTO_L1_L2_AFTER_DLLP_DONE;
554 writel(value, port->base + RP_VEND_XP_BIST);
556 value = readl(port->base + RP_PRIV_MISC);
557 value |= RP_PRIV_MISC_CTLR_CLK_CLAMP_ENABLE;
558 value |= RP_PRIV_MISC_TMS_CLK_CLAMP_ENABLE;
560 if (soc->update_clamp_threshold) {
561 value &= ~(RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD_MASK |
562 RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD_MASK);
563 value |= RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD |
564 RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD;
567 writel(value, port->base + RP_PRIV_MISC);
570 static void tegra_pcie_program_ectl_settings(struct tegra_pcie_port *port)
572 const struct tegra_pcie_soc *soc = port->pcie->soc;
575 value = readl(port->base + RP_ECTL_2_R1);
576 value &= ~RP_ECTL_2_R1_RX_CTLE_1C_MASK;
577 value |= soc->ectl.regs.rp_ectl_2_r1;
578 writel(value, port->base + RP_ECTL_2_R1);
580 value = readl(port->base + RP_ECTL_4_R1);
581 value &= ~RP_ECTL_4_R1_RX_CDR_CTRL_1C_MASK;
582 value |= soc->ectl.regs.rp_ectl_4_r1 <<
583 RP_ECTL_4_R1_RX_CDR_CTRL_1C_SHIFT;
584 writel(value, port->base + RP_ECTL_4_R1);
586 value = readl(port->base + RP_ECTL_5_R1);
587 value &= ~RP_ECTL_5_R1_RX_EQ_CTRL_L_1C_MASK;
588 value |= soc->ectl.regs.rp_ectl_5_r1;
589 writel(value, port->base + RP_ECTL_5_R1);
591 value = readl(port->base + RP_ECTL_6_R1);
592 value &= ~RP_ECTL_6_R1_RX_EQ_CTRL_H_1C_MASK;
593 value |= soc->ectl.regs.rp_ectl_6_r1;
594 writel(value, port->base + RP_ECTL_6_R1);
596 value = readl(port->base + RP_ECTL_2_R2);
597 value &= ~RP_ECTL_2_R2_RX_CTLE_1C_MASK;
598 value |= soc->ectl.regs.rp_ectl_2_r2;
599 writel(value, port->base + RP_ECTL_2_R2);
601 value = readl(port->base + RP_ECTL_4_R2);
602 value &= ~RP_ECTL_4_R2_RX_CDR_CTRL_1C_MASK;
603 value |= soc->ectl.regs.rp_ectl_4_r2 <<
604 RP_ECTL_4_R2_RX_CDR_CTRL_1C_SHIFT;
605 writel(value, port->base + RP_ECTL_4_R2);
607 value = readl(port->base + RP_ECTL_5_R2);
608 value &= ~RP_ECTL_5_R2_RX_EQ_CTRL_L_1C_MASK;
609 value |= soc->ectl.regs.rp_ectl_5_r2;
610 writel(value, port->base + RP_ECTL_5_R2);
612 value = readl(port->base + RP_ECTL_6_R2);
613 value &= ~RP_ECTL_6_R2_RX_EQ_CTRL_H_1C_MASK;
614 value |= soc->ectl.regs.rp_ectl_6_r2;
615 writel(value, port->base + RP_ECTL_6_R2);
618 static void tegra_pcie_apply_sw_fixup(struct tegra_pcie_port *port)
620 const struct tegra_pcie_soc *soc = port->pcie->soc;
624 * Sometimes link speed change from Gen2 to Gen1 fails due to
625 * instability in deskew logic on lane-0. Increase the deskew
626 * retry time to resolve this issue.
628 if (soc->program_deskew_time) {
629 value = readl(port->base + RP_VEND_CTL0);
630 value &= ~RP_VEND_CTL0_DSK_RST_PULSE_WIDTH_MASK;
631 value |= RP_VEND_CTL0_DSK_RST_PULSE_WIDTH;
632 writel(value, port->base + RP_VEND_CTL0);
635 if (soc->update_fc_timer) {
636 value = readl(port->base + RP_VEND_XP);
637 value &= ~RP_VEND_XP_UPDATE_FC_THRESHOLD_MASK;
638 value |= soc->update_fc_threshold;
639 writel(value, port->base + RP_VEND_XP);
643 * PCIe link doesn't come up with few legacy PCIe endpoints if
644 * root port advertises both Gen-1 and Gen-2 speeds in Tegra.
645 * Hence, the strategy followed here is to initially advertise
646 * only Gen-1 and after link is up, retrain link to Gen-2 speed
648 value = readl(port->base + RP_LINK_CONTROL_STATUS_2);
649 value &= ~PCI_EXP_LNKSTA_CLS;
650 value |= PCI_EXP_LNKSTA_CLS_2_5GB;
651 writel(value, port->base + RP_LINK_CONTROL_STATUS_2);
654 static void tegra_pcie_port_enable(struct tegra_pcie_port *port)
656 unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port);
657 const struct tegra_pcie_soc *soc = port->pcie->soc;
660 /* enable reference clock */
661 value = afi_readl(port->pcie, ctrl);
662 value |= AFI_PEX_CTRL_REFCLK_EN;
664 if (soc->has_pex_clkreq_en)
665 value |= AFI_PEX_CTRL_CLKREQ_EN;
667 value |= AFI_PEX_CTRL_OVERRIDE_EN;
669 afi_writel(port->pcie, value, ctrl);
671 tegra_pcie_port_reset(port);
673 if (soc->force_pca_enable) {
674 value = readl(port->base + RP_VEND_CTL2);
675 value |= RP_VEND_CTL2_PCA_ENABLE;
676 writel(value, port->base + RP_VEND_CTL2);
679 tegra_pcie_enable_rp_features(port);
681 if (soc->ectl.enable)
682 tegra_pcie_program_ectl_settings(port);
684 tegra_pcie_apply_sw_fixup(port);
687 static void tegra_pcie_port_disable(struct tegra_pcie_port *port)
689 unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port);
690 const struct tegra_pcie_soc *soc = port->pcie->soc;
693 /* assert port reset */
694 value = afi_readl(port->pcie, ctrl);
695 value &= ~AFI_PEX_CTRL_RST;
696 afi_writel(port->pcie, value, ctrl);
698 /* disable reference clock */
699 value = afi_readl(port->pcie, ctrl);
701 if (soc->has_pex_clkreq_en)
702 value &= ~AFI_PEX_CTRL_CLKREQ_EN;
704 value &= ~AFI_PEX_CTRL_REFCLK_EN;
705 afi_writel(port->pcie, value, ctrl);
707 /* disable PCIe port and set CLKREQ# as GPIO to allow PLLE power down */
708 value = afi_readl(port->pcie, AFI_PCIE_CONFIG);
709 value |= AFI_PCIE_CONFIG_PCIE_DISABLE(port->index);
710 value |= AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO(port->index);
711 afi_writel(port->pcie, value, AFI_PCIE_CONFIG);
714 static void tegra_pcie_port_free(struct tegra_pcie_port *port)
716 struct tegra_pcie *pcie = port->pcie;
717 struct device *dev = pcie->dev;
719 devm_iounmap(dev, port->base);
720 devm_release_mem_region(dev, port->regs.start,
721 resource_size(&port->regs));
722 list_del(&port->list);
723 devm_kfree(dev, port);
726 /* Tegra PCIE root complex wrongly reports device class */
727 static void tegra_pcie_fixup_class(struct pci_dev *dev)
729 dev->class = PCI_CLASS_BRIDGE_PCI_NORMAL;
731 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0bf0, tegra_pcie_fixup_class);
732 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0bf1, tegra_pcie_fixup_class);
733 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0e1c, tegra_pcie_fixup_class);
734 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0e1d, tegra_pcie_fixup_class);
736 /* Tegra20 and Tegra30 PCIE requires relaxed ordering */
737 static void tegra_pcie_relax_enable(struct pci_dev *dev)
739 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
741 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0bf0, tegra_pcie_relax_enable);
742 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0bf1, tegra_pcie_relax_enable);
743 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0e1c, tegra_pcie_relax_enable);
744 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0e1d, tegra_pcie_relax_enable);
746 static int tegra_pcie_map_irq(const struct pci_dev *pdev, u8 slot, u8 pin)
748 struct tegra_pcie *pcie = pdev->bus->sysdata;
751 tegra_cpuidle_pcie_irqs_in_use();
753 irq = of_irq_parse_and_map_pci(pdev, slot, pin);
760 static irqreturn_t tegra_pcie_isr(int irq, void *arg)
762 static const char * const err_msg[] = {
770 "Response decoding error",
771 "AXI response decoding error",
772 "Transaction timeout",
773 "Slot present pin change",
774 "Slot clock request change",
775 "TMS clock ramp change",
776 "TMS ready for power down",
779 struct tegra_pcie *pcie = arg;
780 struct device *dev = pcie->dev;
783 code = afi_readl(pcie, AFI_INTR_CODE) & AFI_INTR_CODE_MASK;
784 signature = afi_readl(pcie, AFI_INTR_SIGNATURE);
785 afi_writel(pcie, 0, AFI_INTR_CODE);
787 if (code == AFI_INTR_LEGACY)
790 if (code >= ARRAY_SIZE(err_msg))
794 * do not pollute kernel log with master abort reports since they
795 * happen a lot during enumeration
797 if (code == AFI_INTR_MASTER_ABORT || code == AFI_INTR_PE_PRSNT_SENSE)
798 dev_dbg(dev, "%s, signature: %08x\n", err_msg[code], signature);
800 dev_err(dev, "%s, signature: %08x\n", err_msg[code], signature);
802 if (code == AFI_INTR_TARGET_ABORT || code == AFI_INTR_MASTER_ABORT ||
803 code == AFI_INTR_FPCI_DECODE_ERROR) {
804 u32 fpci = afi_readl(pcie, AFI_UPPER_FPCI_ADDRESS) & 0xff;
805 u64 address = (u64)fpci << 32 | (signature & 0xfffffffc);
807 if (code == AFI_INTR_MASTER_ABORT)
808 dev_dbg(dev, " FPCI address: %10llx\n", address);
810 dev_err(dev, " FPCI address: %10llx\n", address);
817 * FPCI map is as follows:
818 * - 0xfdfc000000: I/O space
819 * - 0xfdfe000000: type 0 configuration space
820 * - 0xfdff000000: type 1 configuration space
821 * - 0xfe00000000: type 0 extended configuration space
822 * - 0xfe10000000: type 1 extended configuration space
824 static void tegra_pcie_setup_translations(struct tegra_pcie *pcie)
827 struct resource_entry *entry;
828 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
830 /* Bar 0: type 1 extended configuration space */
831 size = resource_size(&pcie->cs);
832 afi_writel(pcie, pcie->cs.start, AFI_AXI_BAR0_START);
833 afi_writel(pcie, size >> 12, AFI_AXI_BAR0_SZ);
835 resource_list_for_each_entry(entry, &bridge->windows) {
836 u32 fpci_bar, axi_address;
837 struct resource *res = entry->res;
839 size = resource_size(res);
841 switch (resource_type(res)) {
843 /* Bar 1: downstream IO bar */
844 fpci_bar = 0xfdfc0000;
845 axi_address = pci_pio_to_address(res->start);
846 afi_writel(pcie, axi_address, AFI_AXI_BAR1_START);
847 afi_writel(pcie, size >> 12, AFI_AXI_BAR1_SZ);
848 afi_writel(pcie, fpci_bar, AFI_FPCI_BAR1);
851 fpci_bar = (((res->start >> 12) & 0x0fffffff) << 4) | 0x1;
852 axi_address = res->start;
854 if (res->flags & IORESOURCE_PREFETCH) {
855 /* Bar 2: prefetchable memory BAR */
856 afi_writel(pcie, axi_address, AFI_AXI_BAR2_START);
857 afi_writel(pcie, size >> 12, AFI_AXI_BAR2_SZ);
858 afi_writel(pcie, fpci_bar, AFI_FPCI_BAR2);
861 /* Bar 3: non prefetchable memory BAR */
862 afi_writel(pcie, axi_address, AFI_AXI_BAR3_START);
863 afi_writel(pcie, size >> 12, AFI_AXI_BAR3_SZ);
864 afi_writel(pcie, fpci_bar, AFI_FPCI_BAR3);
870 /* NULL out the remaining BARs as they are not used */
871 afi_writel(pcie, 0, AFI_AXI_BAR4_START);
872 afi_writel(pcie, 0, AFI_AXI_BAR4_SZ);
873 afi_writel(pcie, 0, AFI_FPCI_BAR4);
875 afi_writel(pcie, 0, AFI_AXI_BAR5_START);
876 afi_writel(pcie, 0, AFI_AXI_BAR5_SZ);
877 afi_writel(pcie, 0, AFI_FPCI_BAR5);
879 if (pcie->soc->has_cache_bars) {
880 /* map all upstream transactions as uncached */
881 afi_writel(pcie, 0, AFI_CACHE_BAR0_ST);
882 afi_writel(pcie, 0, AFI_CACHE_BAR0_SZ);
883 afi_writel(pcie, 0, AFI_CACHE_BAR1_ST);
884 afi_writel(pcie, 0, AFI_CACHE_BAR1_SZ);
887 /* MSI translations are setup only when needed */
888 afi_writel(pcie, 0, AFI_MSI_FPCI_BAR_ST);
889 afi_writel(pcie, 0, AFI_MSI_BAR_SZ);
890 afi_writel(pcie, 0, AFI_MSI_AXI_BAR_ST);
891 afi_writel(pcie, 0, AFI_MSI_BAR_SZ);
894 static int tegra_pcie_pll_wait(struct tegra_pcie *pcie, unsigned long timeout)
896 const struct tegra_pcie_soc *soc = pcie->soc;
899 timeout = jiffies + msecs_to_jiffies(timeout);
901 while (time_before(jiffies, timeout)) {
902 value = pads_readl(pcie, soc->pads_pll_ctl);
903 if (value & PADS_PLL_CTL_LOCKDET)
910 static int tegra_pcie_phy_enable(struct tegra_pcie *pcie)
912 struct device *dev = pcie->dev;
913 const struct tegra_pcie_soc *soc = pcie->soc;
917 /* initialize internal PHY, enable up to 16 PCIE lanes */
918 pads_writel(pcie, 0x0, PADS_CTL_SEL);
920 /* override IDDQ to 1 on all 4 lanes */
921 value = pads_readl(pcie, PADS_CTL);
922 value |= PADS_CTL_IDDQ_1L;
923 pads_writel(pcie, value, PADS_CTL);
926 * Set up PHY PLL inputs select PLLE output as refclock,
927 * set TX ref sel to div10 (not div5).
929 value = pads_readl(pcie, soc->pads_pll_ctl);
930 value &= ~(PADS_PLL_CTL_REFCLK_MASK | PADS_PLL_CTL_TXCLKREF_MASK);
931 value |= PADS_PLL_CTL_REFCLK_INTERNAL_CML | soc->tx_ref_sel;
932 pads_writel(pcie, value, soc->pads_pll_ctl);
935 value = pads_readl(pcie, soc->pads_pll_ctl);
936 value &= ~PADS_PLL_CTL_RST_B4SM;
937 pads_writel(pcie, value, soc->pads_pll_ctl);
939 usleep_range(20, 100);
941 /* take PLL out of reset */
942 value = pads_readl(pcie, soc->pads_pll_ctl);
943 value |= PADS_PLL_CTL_RST_B4SM;
944 pads_writel(pcie, value, soc->pads_pll_ctl);
946 /* wait for the PLL to lock */
947 err = tegra_pcie_pll_wait(pcie, 500);
949 dev_err(dev, "PLL failed to lock: %d\n", err);
953 /* turn off IDDQ override */
954 value = pads_readl(pcie, PADS_CTL);
955 value &= ~PADS_CTL_IDDQ_1L;
956 pads_writel(pcie, value, PADS_CTL);
958 /* enable TX/RX data */
959 value = pads_readl(pcie, PADS_CTL);
960 value |= PADS_CTL_TX_DATA_EN_1L | PADS_CTL_RX_DATA_EN_1L;
961 pads_writel(pcie, value, PADS_CTL);
966 static int tegra_pcie_phy_disable(struct tegra_pcie *pcie)
968 const struct tegra_pcie_soc *soc = pcie->soc;
971 /* disable TX/RX data */
972 value = pads_readl(pcie, PADS_CTL);
973 value &= ~(PADS_CTL_TX_DATA_EN_1L | PADS_CTL_RX_DATA_EN_1L);
974 pads_writel(pcie, value, PADS_CTL);
977 value = pads_readl(pcie, PADS_CTL);
978 value |= PADS_CTL_IDDQ_1L;
979 pads_writel(pcie, value, PADS_CTL);
982 value = pads_readl(pcie, soc->pads_pll_ctl);
983 value &= ~PADS_PLL_CTL_RST_B4SM;
984 pads_writel(pcie, value, soc->pads_pll_ctl);
986 usleep_range(20, 100);
991 static int tegra_pcie_port_phy_power_on(struct tegra_pcie_port *port)
993 struct device *dev = port->pcie->dev;
997 for (i = 0; i < port->lanes; i++) {
998 err = phy_power_on(port->phys[i]);
1000 dev_err(dev, "failed to power on PHY#%u: %d\n", i, err);
1008 static int tegra_pcie_port_phy_power_off(struct tegra_pcie_port *port)
1010 struct device *dev = port->pcie->dev;
1014 for (i = 0; i < port->lanes; i++) {
1015 err = phy_power_off(port->phys[i]);
1017 dev_err(dev, "failed to power off PHY#%u: %d\n", i,
1026 static int tegra_pcie_phy_power_on(struct tegra_pcie *pcie)
1028 struct device *dev = pcie->dev;
1029 struct tegra_pcie_port *port;
1032 if (pcie->legacy_phy) {
1034 err = phy_power_on(pcie->phy);
1036 err = tegra_pcie_phy_enable(pcie);
1039 dev_err(dev, "failed to power on PHY: %d\n", err);
1044 list_for_each_entry(port, &pcie->ports, list) {
1045 err = tegra_pcie_port_phy_power_on(port);
1048 "failed to power on PCIe port %u PHY: %d\n",
1057 static int tegra_pcie_phy_power_off(struct tegra_pcie *pcie)
1059 struct device *dev = pcie->dev;
1060 struct tegra_pcie_port *port;
1063 if (pcie->legacy_phy) {
1065 err = phy_power_off(pcie->phy);
1067 err = tegra_pcie_phy_disable(pcie);
1070 dev_err(dev, "failed to power off PHY: %d\n", err);
1075 list_for_each_entry(port, &pcie->ports, list) {
1076 err = tegra_pcie_port_phy_power_off(port);
1079 "failed to power off PCIe port %u PHY: %d\n",
1088 static void tegra_pcie_enable_controller(struct tegra_pcie *pcie)
1090 const struct tegra_pcie_soc *soc = pcie->soc;
1091 struct tegra_pcie_port *port;
1092 unsigned long value;
1094 /* enable PLL power down */
1096 value = afi_readl(pcie, AFI_PLLE_CONTROL);
1097 value &= ~AFI_PLLE_CONTROL_BYPASS_PADS2PLLE_CONTROL;
1098 value |= AFI_PLLE_CONTROL_PADS2PLLE_CONTROL_EN;
1099 afi_writel(pcie, value, AFI_PLLE_CONTROL);
1102 /* power down PCIe slot clock bias pad */
1103 if (soc->has_pex_bias_ctrl)
1104 afi_writel(pcie, 0, AFI_PEXBIAS_CTRL_0);
1106 /* configure mode and disable all ports */
1107 value = afi_readl(pcie, AFI_PCIE_CONFIG);
1108 value &= ~AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_MASK;
1109 value |= AFI_PCIE_CONFIG_PCIE_DISABLE_ALL | pcie->xbar_config;
1110 value |= AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO_ALL;
1112 list_for_each_entry(port, &pcie->ports, list) {
1113 value &= ~AFI_PCIE_CONFIG_PCIE_DISABLE(port->index);
1114 value &= ~AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO(port->index);
1117 afi_writel(pcie, value, AFI_PCIE_CONFIG);
1119 if (soc->has_gen2) {
1120 value = afi_readl(pcie, AFI_FUSE);
1121 value &= ~AFI_FUSE_PCIE_T0_GEN2_DIS;
1122 afi_writel(pcie, value, AFI_FUSE);
1124 value = afi_readl(pcie, AFI_FUSE);
1125 value |= AFI_FUSE_PCIE_T0_GEN2_DIS;
1126 afi_writel(pcie, value, AFI_FUSE);
1129 /* Disable AFI dynamic clock gating and enable PCIe */
1130 value = afi_readl(pcie, AFI_CONFIGURATION);
1131 value |= AFI_CONFIGURATION_EN_FPCI;
1132 value |= AFI_CONFIGURATION_CLKEN_OVERRIDE;
1133 afi_writel(pcie, value, AFI_CONFIGURATION);
1135 value = AFI_INTR_EN_INI_SLVERR | AFI_INTR_EN_INI_DECERR |
1136 AFI_INTR_EN_TGT_SLVERR | AFI_INTR_EN_TGT_DECERR |
1137 AFI_INTR_EN_TGT_WRERR | AFI_INTR_EN_DFPCI_DECERR;
1139 if (soc->has_intr_prsnt_sense)
1140 value |= AFI_INTR_EN_PRSNT_SENSE;
1142 afi_writel(pcie, value, AFI_AFI_INTR_ENABLE);
1143 afi_writel(pcie, 0xffffffff, AFI_SM_INTR_ENABLE);
1145 /* don't enable MSI for now, only when needed */
1146 afi_writel(pcie, AFI_INTR_MASK_INT_MASK, AFI_INTR_MASK);
1148 /* disable all exceptions */
1149 afi_writel(pcie, 0, AFI_FPCI_ERROR_MASKS);
1152 static void tegra_pcie_power_off(struct tegra_pcie *pcie)
1154 struct device *dev = pcie->dev;
1155 const struct tegra_pcie_soc *soc = pcie->soc;
1158 reset_control_assert(pcie->afi_rst);
1160 clk_disable_unprepare(pcie->pll_e);
1161 if (soc->has_cml_clk)
1162 clk_disable_unprepare(pcie->cml_clk);
1163 clk_disable_unprepare(pcie->afi_clk);
1165 if (!dev->pm_domain)
1166 tegra_powergate_power_off(TEGRA_POWERGATE_PCIE);
1168 err = regulator_bulk_disable(pcie->num_supplies, pcie->supplies);
1170 dev_warn(dev, "failed to disable regulators: %d\n", err);
1173 static int tegra_pcie_power_on(struct tegra_pcie *pcie)
1175 struct device *dev = pcie->dev;
1176 const struct tegra_pcie_soc *soc = pcie->soc;
1179 reset_control_assert(pcie->pcie_xrst);
1180 reset_control_assert(pcie->afi_rst);
1181 reset_control_assert(pcie->pex_rst);
1183 if (!dev->pm_domain)
1184 tegra_powergate_power_off(TEGRA_POWERGATE_PCIE);
1186 /* enable regulators */
1187 err = regulator_bulk_enable(pcie->num_supplies, pcie->supplies);
1189 dev_err(dev, "failed to enable regulators: %d\n", err);
1191 if (!dev->pm_domain) {
1192 err = tegra_powergate_power_on(TEGRA_POWERGATE_PCIE);
1194 dev_err(dev, "failed to power ungate: %d\n", err);
1195 goto regulator_disable;
1197 err = tegra_powergate_remove_clamping(TEGRA_POWERGATE_PCIE);
1199 dev_err(dev, "failed to remove clamp: %d\n", err);
1204 err = clk_prepare_enable(pcie->afi_clk);
1206 dev_err(dev, "failed to enable AFI clock: %d\n", err);
1210 if (soc->has_cml_clk) {
1211 err = clk_prepare_enable(pcie->cml_clk);
1213 dev_err(dev, "failed to enable CML clock: %d\n", err);
1214 goto disable_afi_clk;
1218 err = clk_prepare_enable(pcie->pll_e);
1220 dev_err(dev, "failed to enable PLLE clock: %d\n", err);
1221 goto disable_cml_clk;
1224 reset_control_deassert(pcie->afi_rst);
1229 if (soc->has_cml_clk)
1230 clk_disable_unprepare(pcie->cml_clk);
1232 clk_disable_unprepare(pcie->afi_clk);
1234 if (!dev->pm_domain)
1235 tegra_powergate_power_off(TEGRA_POWERGATE_PCIE);
1237 regulator_bulk_disable(pcie->num_supplies, pcie->supplies);
1242 static void tegra_pcie_apply_pad_settings(struct tegra_pcie *pcie)
1244 const struct tegra_pcie_soc *soc = pcie->soc;
1246 /* Configure the reference clock driver */
1247 pads_writel(pcie, soc->pads_refclk_cfg0, PADS_REFCLK_CFG0);
1249 if (soc->num_ports > 2)
1250 pads_writel(pcie, soc->pads_refclk_cfg1, PADS_REFCLK_CFG1);
1253 static int tegra_pcie_clocks_get(struct tegra_pcie *pcie)
1255 struct device *dev = pcie->dev;
1256 const struct tegra_pcie_soc *soc = pcie->soc;
1258 pcie->pex_clk = devm_clk_get(dev, "pex");
1259 if (IS_ERR(pcie->pex_clk))
1260 return PTR_ERR(pcie->pex_clk);
1262 pcie->afi_clk = devm_clk_get(dev, "afi");
1263 if (IS_ERR(pcie->afi_clk))
1264 return PTR_ERR(pcie->afi_clk);
1266 pcie->pll_e = devm_clk_get(dev, "pll_e");
1267 if (IS_ERR(pcie->pll_e))
1268 return PTR_ERR(pcie->pll_e);
1270 if (soc->has_cml_clk) {
1271 pcie->cml_clk = devm_clk_get(dev, "cml");
1272 if (IS_ERR(pcie->cml_clk))
1273 return PTR_ERR(pcie->cml_clk);
1279 static int tegra_pcie_resets_get(struct tegra_pcie *pcie)
1281 struct device *dev = pcie->dev;
1283 pcie->pex_rst = devm_reset_control_get_exclusive(dev, "pex");
1284 if (IS_ERR(pcie->pex_rst))
1285 return PTR_ERR(pcie->pex_rst);
1287 pcie->afi_rst = devm_reset_control_get_exclusive(dev, "afi");
1288 if (IS_ERR(pcie->afi_rst))
1289 return PTR_ERR(pcie->afi_rst);
1291 pcie->pcie_xrst = devm_reset_control_get_exclusive(dev, "pcie_x");
1292 if (IS_ERR(pcie->pcie_xrst))
1293 return PTR_ERR(pcie->pcie_xrst);
1298 static int tegra_pcie_phys_get_legacy(struct tegra_pcie *pcie)
1300 struct device *dev = pcie->dev;
1303 pcie->phy = devm_phy_optional_get(dev, "pcie");
1304 if (IS_ERR(pcie->phy)) {
1305 err = PTR_ERR(pcie->phy);
1306 dev_err(dev, "failed to get PHY: %d\n", err);
1310 err = phy_init(pcie->phy);
1312 dev_err(dev, "failed to initialize PHY: %d\n", err);
1316 pcie->legacy_phy = true;
1321 static struct phy *devm_of_phy_optional_get_index(struct device *dev,
1322 struct device_node *np,
1323 const char *consumer,
1329 name = kasprintf(GFP_KERNEL, "%s-%u", consumer, index);
1331 return ERR_PTR(-ENOMEM);
1333 phy = devm_of_phy_optional_get(dev, np, name);
1339 static int tegra_pcie_port_get_phys(struct tegra_pcie_port *port)
1341 struct device *dev = port->pcie->dev;
1346 port->phys = devm_kcalloc(dev, sizeof(phy), port->lanes, GFP_KERNEL);
1350 for (i = 0; i < port->lanes; i++) {
1351 phy = devm_of_phy_optional_get_index(dev, port->np, "pcie", i);
1353 dev_err(dev, "failed to get PHY#%u: %ld\n", i,
1355 return PTR_ERR(phy);
1358 err = phy_init(phy);
1360 dev_err(dev, "failed to initialize PHY#%u: %d\n", i,
1365 port->phys[i] = phy;
1371 static int tegra_pcie_phys_get(struct tegra_pcie *pcie)
1373 const struct tegra_pcie_soc *soc = pcie->soc;
1374 struct device_node *np = pcie->dev->of_node;
1375 struct tegra_pcie_port *port;
1378 if (!soc->has_gen2 || of_property_present(np, "phys"))
1379 return tegra_pcie_phys_get_legacy(pcie);
1381 list_for_each_entry(port, &pcie->ports, list) {
1382 err = tegra_pcie_port_get_phys(port);
1390 static void tegra_pcie_phys_put(struct tegra_pcie *pcie)
1392 struct tegra_pcie_port *port;
1393 struct device *dev = pcie->dev;
1396 if (pcie->legacy_phy) {
1397 err = phy_exit(pcie->phy);
1399 dev_err(dev, "failed to teardown PHY: %d\n", err);
1403 list_for_each_entry(port, &pcie->ports, list) {
1404 for (i = 0; i < port->lanes; i++) {
1405 err = phy_exit(port->phys[i]);
1407 dev_err(dev, "failed to teardown PHY#%u: %d\n",
1413 static int tegra_pcie_get_resources(struct tegra_pcie *pcie)
1415 struct device *dev = pcie->dev;
1416 struct platform_device *pdev = to_platform_device(dev);
1417 struct resource *res;
1418 const struct tegra_pcie_soc *soc = pcie->soc;
1421 err = tegra_pcie_clocks_get(pcie);
1423 dev_err(dev, "failed to get clocks: %d\n", err);
1427 err = tegra_pcie_resets_get(pcie);
1429 dev_err(dev, "failed to get resets: %d\n", err);
1433 if (soc->program_uphy) {
1434 err = tegra_pcie_phys_get(pcie);
1436 dev_err(dev, "failed to get PHYs: %d\n", err);
1441 pcie->pads = devm_platform_ioremap_resource_byname(pdev, "pads");
1442 if (IS_ERR(pcie->pads)) {
1443 err = PTR_ERR(pcie->pads);
1447 pcie->afi = devm_platform_ioremap_resource_byname(pdev, "afi");
1448 if (IS_ERR(pcie->afi)) {
1449 err = PTR_ERR(pcie->afi);
1453 /* request configuration space, but remap later, on demand */
1454 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cs");
1456 err = -EADDRNOTAVAIL;
1462 /* constrain configuration space to 4 KiB */
1463 pcie->cs.end = pcie->cs.start + SZ_4K - 1;
1465 pcie->cfg = devm_ioremap_resource(dev, &pcie->cs);
1466 if (IS_ERR(pcie->cfg)) {
1467 err = PTR_ERR(pcie->cfg);
1471 /* request interrupt */
1472 err = platform_get_irq_byname(pdev, "intr");
1478 err = request_irq(pcie->irq, tegra_pcie_isr, IRQF_SHARED, "PCIE", pcie);
1480 dev_err(dev, "failed to register IRQ: %d\n", err);
1487 if (soc->program_uphy)
1488 tegra_pcie_phys_put(pcie);
1493 static int tegra_pcie_put_resources(struct tegra_pcie *pcie)
1495 const struct tegra_pcie_soc *soc = pcie->soc;
1498 free_irq(pcie->irq, pcie);
1500 if (soc->program_uphy)
1501 tegra_pcie_phys_put(pcie);
1506 static void tegra_pcie_pme_turnoff(struct tegra_pcie_port *port)
1508 struct tegra_pcie *pcie = port->pcie;
1509 const struct tegra_pcie_soc *soc = pcie->soc;
1514 val = afi_readl(pcie, AFI_PCIE_PME);
1515 val |= (0x1 << soc->ports[port->index].pme.turnoff_bit);
1516 afi_writel(pcie, val, AFI_PCIE_PME);
1518 ack_bit = soc->ports[port->index].pme.ack_bit;
1519 err = readl_poll_timeout(pcie->afi + AFI_PCIE_PME, val,
1520 val & (0x1 << ack_bit), 1, PME_ACK_TIMEOUT);
1522 dev_err(pcie->dev, "PME Ack is not received on port: %d\n",
1525 usleep_range(10000, 11000);
1527 val = afi_readl(pcie, AFI_PCIE_PME);
1528 val &= ~(0x1 << soc->ports[port->index].pme.turnoff_bit);
1529 afi_writel(pcie, val, AFI_PCIE_PME);
1532 static void tegra_pcie_msi_irq(struct irq_desc *desc)
1534 struct tegra_pcie *pcie = irq_desc_get_handler_data(desc);
1535 struct irq_chip *chip = irq_desc_get_chip(desc);
1536 struct tegra_msi *msi = &pcie->msi;
1537 struct device *dev = pcie->dev;
1540 chained_irq_enter(chip, desc);
1542 for (i = 0; i < 8; i++) {
1543 unsigned long reg = afi_readl(pcie, AFI_MSI_VEC(i));
1546 unsigned int offset = find_first_bit(®, 32);
1547 unsigned int index = i * 32 + offset;
1550 ret = generic_handle_domain_irq(msi->domain->parent, index);
1553 * that's weird who triggered this?
1556 dev_info(dev, "unexpected MSI\n");
1557 afi_writel(pcie, BIT(index % 32), AFI_MSI_VEC(index));
1560 /* see if there's any more pending in this vector */
1561 reg = afi_readl(pcie, AFI_MSI_VEC(i));
1565 chained_irq_exit(chip, desc);
1568 static void tegra_msi_top_irq_ack(struct irq_data *d)
1570 irq_chip_ack_parent(d);
1573 static void tegra_msi_top_irq_mask(struct irq_data *d)
1575 pci_msi_mask_irq(d);
1576 irq_chip_mask_parent(d);
1579 static void tegra_msi_top_irq_unmask(struct irq_data *d)
1581 pci_msi_unmask_irq(d);
1582 irq_chip_unmask_parent(d);
1585 static struct irq_chip tegra_msi_top_chip = {
1586 .name = "Tegra PCIe MSI",
1587 .irq_ack = tegra_msi_top_irq_ack,
1588 .irq_mask = tegra_msi_top_irq_mask,
1589 .irq_unmask = tegra_msi_top_irq_unmask,
1592 static void tegra_msi_irq_ack(struct irq_data *d)
1594 struct tegra_msi *msi = irq_data_get_irq_chip_data(d);
1595 struct tegra_pcie *pcie = msi_to_pcie(msi);
1596 unsigned int index = d->hwirq / 32;
1598 /* clear the interrupt */
1599 afi_writel(pcie, BIT(d->hwirq % 32), AFI_MSI_VEC(index));
1602 static void tegra_msi_irq_mask(struct irq_data *d)
1604 struct tegra_msi *msi = irq_data_get_irq_chip_data(d);
1605 struct tegra_pcie *pcie = msi_to_pcie(msi);
1606 unsigned int index = d->hwirq / 32;
1607 unsigned long flags;
1610 spin_lock_irqsave(&msi->mask_lock, flags);
1611 value = afi_readl(pcie, AFI_MSI_EN_VEC(index));
1612 value &= ~BIT(d->hwirq % 32);
1613 afi_writel(pcie, value, AFI_MSI_EN_VEC(index));
1614 spin_unlock_irqrestore(&msi->mask_lock, flags);
1617 static void tegra_msi_irq_unmask(struct irq_data *d)
1619 struct tegra_msi *msi = irq_data_get_irq_chip_data(d);
1620 struct tegra_pcie *pcie = msi_to_pcie(msi);
1621 unsigned int index = d->hwirq / 32;
1622 unsigned long flags;
1625 spin_lock_irqsave(&msi->mask_lock, flags);
1626 value = afi_readl(pcie, AFI_MSI_EN_VEC(index));
1627 value |= BIT(d->hwirq % 32);
1628 afi_writel(pcie, value, AFI_MSI_EN_VEC(index));
1629 spin_unlock_irqrestore(&msi->mask_lock, flags);
1632 static int tegra_msi_set_affinity(struct irq_data *d, const struct cpumask *mask, bool force)
1637 static void tegra_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
1639 struct tegra_msi *msi = irq_data_get_irq_chip_data(data);
1641 msg->address_lo = lower_32_bits(msi->phys);
1642 msg->address_hi = upper_32_bits(msi->phys);
1643 msg->data = data->hwirq;
1646 static struct irq_chip tegra_msi_bottom_chip = {
1647 .name = "Tegra MSI",
1648 .irq_ack = tegra_msi_irq_ack,
1649 .irq_mask = tegra_msi_irq_mask,
1650 .irq_unmask = tegra_msi_irq_unmask,
1651 .irq_set_affinity = tegra_msi_set_affinity,
1652 .irq_compose_msi_msg = tegra_compose_msi_msg,
1655 static int tegra_msi_domain_alloc(struct irq_domain *domain, unsigned int virq,
1656 unsigned int nr_irqs, void *args)
1658 struct tegra_msi *msi = domain->host_data;
1662 mutex_lock(&msi->map_lock);
1664 hwirq = bitmap_find_free_region(msi->used, INT_PCI_MSI_NR, order_base_2(nr_irqs));
1666 mutex_unlock(&msi->map_lock);
1671 for (i = 0; i < nr_irqs; i++)
1672 irq_domain_set_info(domain, virq + i, hwirq + i,
1673 &tegra_msi_bottom_chip, domain->host_data,
1674 handle_edge_irq, NULL, NULL);
1676 tegra_cpuidle_pcie_irqs_in_use();
1681 static void tegra_msi_domain_free(struct irq_domain *domain, unsigned int virq,
1682 unsigned int nr_irqs)
1684 struct irq_data *d = irq_domain_get_irq_data(domain, virq);
1685 struct tegra_msi *msi = domain->host_data;
1687 mutex_lock(&msi->map_lock);
1689 bitmap_release_region(msi->used, d->hwirq, order_base_2(nr_irqs));
1691 mutex_unlock(&msi->map_lock);
1694 static const struct irq_domain_ops tegra_msi_domain_ops = {
1695 .alloc = tegra_msi_domain_alloc,
1696 .free = tegra_msi_domain_free,
1699 static struct msi_domain_info tegra_msi_info = {
1700 .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
1702 .chip = &tegra_msi_top_chip,
1705 static int tegra_allocate_domains(struct tegra_msi *msi)
1707 struct tegra_pcie *pcie = msi_to_pcie(msi);
1708 struct fwnode_handle *fwnode = dev_fwnode(pcie->dev);
1709 struct irq_domain *parent;
1711 parent = irq_domain_create_linear(fwnode, INT_PCI_MSI_NR,
1712 &tegra_msi_domain_ops, msi);
1714 dev_err(pcie->dev, "failed to create IRQ domain\n");
1717 irq_domain_update_bus_token(parent, DOMAIN_BUS_NEXUS);
1719 msi->domain = pci_msi_create_irq_domain(fwnode, &tegra_msi_info, parent);
1721 dev_err(pcie->dev, "failed to create MSI domain\n");
1722 irq_domain_remove(parent);
1729 static void tegra_free_domains(struct tegra_msi *msi)
1731 struct irq_domain *parent = msi->domain->parent;
1733 irq_domain_remove(msi->domain);
1734 irq_domain_remove(parent);
1737 static int tegra_pcie_msi_setup(struct tegra_pcie *pcie)
1739 struct platform_device *pdev = to_platform_device(pcie->dev);
1740 struct tegra_msi *msi = &pcie->msi;
1741 struct device *dev = pcie->dev;
1744 mutex_init(&msi->map_lock);
1745 spin_lock_init(&msi->mask_lock);
1747 if (IS_ENABLED(CONFIG_PCI_MSI)) {
1748 err = tegra_allocate_domains(msi);
1753 err = platform_get_irq_byname(pdev, "msi");
1755 goto free_irq_domain;
1759 irq_set_chained_handler_and_data(msi->irq, tegra_pcie_msi_irq, pcie);
1761 /* Though the PCIe controller can address >32-bit address space, to
1762 * facilitate endpoints that support only 32-bit MSI target address,
1763 * the mask is set to 32-bit to make sure that MSI target address is
1764 * always a 32-bit address
1766 err = dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
1768 dev_err(dev, "failed to set DMA coherent mask: %d\n", err);
1772 msi->virt = dma_alloc_attrs(dev, PAGE_SIZE, &msi->phys, GFP_KERNEL,
1773 DMA_ATTR_NO_KERNEL_MAPPING);
1775 dev_err(dev, "failed to allocate DMA memory for MSI\n");
1783 irq_set_chained_handler_and_data(msi->irq, NULL, NULL);
1785 if (IS_ENABLED(CONFIG_PCI_MSI))
1786 tegra_free_domains(msi);
1791 static void tegra_pcie_enable_msi(struct tegra_pcie *pcie)
1793 const struct tegra_pcie_soc *soc = pcie->soc;
1794 struct tegra_msi *msi = &pcie->msi;
1795 u32 reg, msi_state[INT_PCI_MSI_NR / 32];
1798 afi_writel(pcie, msi->phys >> soc->msi_base_shift, AFI_MSI_FPCI_BAR_ST);
1799 afi_writel(pcie, msi->phys, AFI_MSI_AXI_BAR_ST);
1800 /* this register is in 4K increments */
1801 afi_writel(pcie, 1, AFI_MSI_BAR_SZ);
1803 /* Restore the MSI allocation state */
1804 bitmap_to_arr32(msi_state, msi->used, INT_PCI_MSI_NR);
1805 for (i = 0; i < ARRAY_SIZE(msi_state); i++)
1806 afi_writel(pcie, msi_state[i], AFI_MSI_EN_VEC(i));
1808 /* and unmask the MSI interrupt */
1809 reg = afi_readl(pcie, AFI_INTR_MASK);
1810 reg |= AFI_INTR_MASK_MSI_MASK;
1811 afi_writel(pcie, reg, AFI_INTR_MASK);
1814 static void tegra_pcie_msi_teardown(struct tegra_pcie *pcie)
1816 struct tegra_msi *msi = &pcie->msi;
1817 unsigned int i, irq;
1819 dma_free_attrs(pcie->dev, PAGE_SIZE, msi->virt, msi->phys,
1820 DMA_ATTR_NO_KERNEL_MAPPING);
1822 for (i = 0; i < INT_PCI_MSI_NR; i++) {
1823 irq = irq_find_mapping(msi->domain, i);
1825 irq_domain_free_irqs(irq, 1);
1828 irq_set_chained_handler_and_data(msi->irq, NULL, NULL);
1830 if (IS_ENABLED(CONFIG_PCI_MSI))
1831 tegra_free_domains(msi);
1834 static int tegra_pcie_disable_msi(struct tegra_pcie *pcie)
1838 /* mask the MSI interrupt */
1839 value = afi_readl(pcie, AFI_INTR_MASK);
1840 value &= ~AFI_INTR_MASK_MSI_MASK;
1841 afi_writel(pcie, value, AFI_INTR_MASK);
1846 static void tegra_pcie_disable_interrupts(struct tegra_pcie *pcie)
1850 value = afi_readl(pcie, AFI_INTR_MASK);
1851 value &= ~AFI_INTR_MASK_INT_MASK;
1852 afi_writel(pcie, value, AFI_INTR_MASK);
1855 static int tegra_pcie_get_xbar_config(struct tegra_pcie *pcie, u32 lanes,
1858 struct device *dev = pcie->dev;
1859 struct device_node *np = dev->of_node;
1861 if (of_device_is_compatible(np, "nvidia,tegra186-pcie")) {
1864 dev_info(dev, "4x1, 1x1 configuration\n");
1865 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_401;
1869 dev_info(dev, "2x1, 1X1, 1x1 configuration\n");
1870 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_211;
1874 dev_info(dev, "1x1, 1x1, 1x1 configuration\n");
1875 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_111;
1879 dev_info(dev, "wrong configuration updated in DT, "
1880 "switching to default 2x1, 1x1, 1x1 "
1882 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_211;
1885 } else if (of_device_is_compatible(np, "nvidia,tegra124-pcie") ||
1886 of_device_is_compatible(np, "nvidia,tegra210-pcie")) {
1889 dev_info(dev, "4x1, 1x1 configuration\n");
1890 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X4_X1;
1894 dev_info(dev, "2x1, 1x1 configuration\n");
1895 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X2_X1;
1898 } else if (of_device_is_compatible(np, "nvidia,tegra30-pcie")) {
1901 dev_info(dev, "4x1, 2x1 configuration\n");
1902 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_420;
1906 dev_info(dev, "2x3 configuration\n");
1907 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_222;
1911 dev_info(dev, "4x1, 1x2 configuration\n");
1912 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_411;
1915 } else if (of_device_is_compatible(np, "nvidia,tegra20-pcie")) {
1918 dev_info(dev, "single-mode configuration\n");
1919 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_SINGLE;
1923 dev_info(dev, "dual-mode configuration\n");
1924 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_DUAL;
1933 * Check whether a given set of supplies is available in a device tree node.
1934 * This is used to check whether the new or the legacy device tree bindings
1937 static bool of_regulator_bulk_available(struct device_node *np,
1938 struct regulator_bulk_data *supplies,
1939 unsigned int num_supplies)
1944 for (i = 0; i < num_supplies; i++) {
1945 snprintf(property, 32, "%s-supply", supplies[i].supply);
1947 if (!of_property_present(np, property))
1955 * Old versions of the device tree binding for this device used a set of power
1956 * supplies that didn't match the hardware inputs. This happened to work for a
1957 * number of cases but is not future proof. However to preserve backwards-
1958 * compatibility with old device trees, this function will try to use the old
1961 static int tegra_pcie_get_legacy_regulators(struct tegra_pcie *pcie)
1963 struct device *dev = pcie->dev;
1964 struct device_node *np = dev->of_node;
1966 if (of_device_is_compatible(np, "nvidia,tegra30-pcie"))
1967 pcie->num_supplies = 3;
1968 else if (of_device_is_compatible(np, "nvidia,tegra20-pcie"))
1969 pcie->num_supplies = 2;
1971 if (pcie->num_supplies == 0) {
1972 dev_err(dev, "device %pOF not supported in legacy mode\n", np);
1976 pcie->supplies = devm_kcalloc(dev, pcie->num_supplies,
1977 sizeof(*pcie->supplies),
1979 if (!pcie->supplies)
1982 pcie->supplies[0].supply = "pex-clk";
1983 pcie->supplies[1].supply = "vdd";
1985 if (pcie->num_supplies > 2)
1986 pcie->supplies[2].supply = "avdd";
1988 return devm_regulator_bulk_get(dev, pcie->num_supplies, pcie->supplies);
1992 * Obtains the list of regulators required for a particular generation of the
1995 * This would've been nice to do simply by providing static tables for use
1996 * with the regulator_bulk_*() API, but unfortunately Tegra30 is a bit quirky
1997 * in that it has two pairs or AVDD_PEX and VDD_PEX supplies (PEXA and PEXB)
1998 * and either seems to be optional depending on which ports are being used.
2000 static int tegra_pcie_get_regulators(struct tegra_pcie *pcie, u32 lane_mask)
2002 struct device *dev = pcie->dev;
2003 struct device_node *np = dev->of_node;
2006 if (of_device_is_compatible(np, "nvidia,tegra186-pcie")) {
2007 pcie->num_supplies = 4;
2009 pcie->supplies = devm_kcalloc(pcie->dev, pcie->num_supplies,
2010 sizeof(*pcie->supplies),
2012 if (!pcie->supplies)
2015 pcie->supplies[i++].supply = "dvdd-pex";
2016 pcie->supplies[i++].supply = "hvdd-pex-pll";
2017 pcie->supplies[i++].supply = "hvdd-pex";
2018 pcie->supplies[i++].supply = "vddio-pexctl-aud";
2019 } else if (of_device_is_compatible(np, "nvidia,tegra210-pcie")) {
2020 pcie->num_supplies = 3;
2022 pcie->supplies = devm_kcalloc(pcie->dev, pcie->num_supplies,
2023 sizeof(*pcie->supplies),
2025 if (!pcie->supplies)
2028 pcie->supplies[i++].supply = "hvddio-pex";
2029 pcie->supplies[i++].supply = "dvddio-pex";
2030 pcie->supplies[i++].supply = "vddio-pex-ctl";
2031 } else if (of_device_is_compatible(np, "nvidia,tegra124-pcie")) {
2032 pcie->num_supplies = 4;
2034 pcie->supplies = devm_kcalloc(dev, pcie->num_supplies,
2035 sizeof(*pcie->supplies),
2037 if (!pcie->supplies)
2040 pcie->supplies[i++].supply = "avddio-pex";
2041 pcie->supplies[i++].supply = "dvddio-pex";
2042 pcie->supplies[i++].supply = "hvdd-pex";
2043 pcie->supplies[i++].supply = "vddio-pex-ctl";
2044 } else if (of_device_is_compatible(np, "nvidia,tegra30-pcie")) {
2045 bool need_pexa = false, need_pexb = false;
2047 /* VDD_PEXA and AVDD_PEXA supply lanes 0 to 3 */
2048 if (lane_mask & 0x0f)
2051 /* VDD_PEXB and AVDD_PEXB supply lanes 4 to 5 */
2052 if (lane_mask & 0x30)
2055 pcie->num_supplies = 4 + (need_pexa ? 2 : 0) +
2056 (need_pexb ? 2 : 0);
2058 pcie->supplies = devm_kcalloc(dev, pcie->num_supplies,
2059 sizeof(*pcie->supplies),
2061 if (!pcie->supplies)
2064 pcie->supplies[i++].supply = "avdd-pex-pll";
2065 pcie->supplies[i++].supply = "hvdd-pex";
2066 pcie->supplies[i++].supply = "vddio-pex-ctl";
2067 pcie->supplies[i++].supply = "avdd-plle";
2070 pcie->supplies[i++].supply = "avdd-pexa";
2071 pcie->supplies[i++].supply = "vdd-pexa";
2075 pcie->supplies[i++].supply = "avdd-pexb";
2076 pcie->supplies[i++].supply = "vdd-pexb";
2078 } else if (of_device_is_compatible(np, "nvidia,tegra20-pcie")) {
2079 pcie->num_supplies = 5;
2081 pcie->supplies = devm_kcalloc(dev, pcie->num_supplies,
2082 sizeof(*pcie->supplies),
2084 if (!pcie->supplies)
2087 pcie->supplies[0].supply = "avdd-pex";
2088 pcie->supplies[1].supply = "vdd-pex";
2089 pcie->supplies[2].supply = "avdd-pex-pll";
2090 pcie->supplies[3].supply = "avdd-plle";
2091 pcie->supplies[4].supply = "vddio-pex-clk";
2094 if (of_regulator_bulk_available(dev->of_node, pcie->supplies,
2095 pcie->num_supplies))
2096 return devm_regulator_bulk_get(dev, pcie->num_supplies,
2100 * If not all regulators are available for this new scheme, assume
2101 * that the device tree complies with an older version of the device
2104 dev_info(dev, "using legacy DT binding for power supplies\n");
2106 devm_kfree(dev, pcie->supplies);
2107 pcie->num_supplies = 0;
2109 return tegra_pcie_get_legacy_regulators(pcie);
2112 static int tegra_pcie_parse_dt(struct tegra_pcie *pcie)
2114 struct device *dev = pcie->dev;
2115 struct device_node *np = dev->of_node, *port;
2116 const struct tegra_pcie_soc *soc = pcie->soc;
2117 u32 lanes = 0, mask = 0;
2118 unsigned int lane = 0;
2121 /* parse root ports */
2122 for_each_child_of_node(np, port) {
2123 struct tegra_pcie_port *rp;
2128 err = of_pci_get_devfn(port);
2130 dev_err(dev, "failed to parse address: %d\n", err);
2134 index = PCI_SLOT(err);
2136 if (index < 1 || index > soc->num_ports) {
2137 dev_err(dev, "invalid port number: %d\n", index);
2144 err = of_property_read_u32(port, "nvidia,num-lanes", &value);
2146 dev_err(dev, "failed to parse # of lanes: %d\n",
2152 dev_err(dev, "invalid # of lanes: %u\n", value);
2157 lanes |= value << (index << 3);
2159 if (!of_device_is_available(port)) {
2164 mask |= ((1 << value) - 1) << lane;
2167 rp = devm_kzalloc(dev, sizeof(*rp), GFP_KERNEL);
2173 err = of_address_to_resource(port, 0, &rp->regs);
2175 dev_err(dev, "failed to parse address: %d\n", err);
2179 INIT_LIST_HEAD(&rp->list);
2185 rp->base = devm_pci_remap_cfg_resource(dev, &rp->regs);
2186 if (IS_ERR(rp->base)) {
2187 err = PTR_ERR(rp->base);
2191 label = devm_kasprintf(dev, GFP_KERNEL, "pex-reset-%u", index);
2198 * Returns -ENOENT if reset-gpios property is not populated
2199 * and in this case fall back to using AFI per port register
2200 * to toggle PERST# SFIO line.
2202 rp->reset_gpio = devm_fwnode_gpiod_get(dev,
2203 of_fwnode_handle(port),
2207 if (IS_ERR(rp->reset_gpio)) {
2208 if (PTR_ERR(rp->reset_gpio) == -ENOENT) {
2209 rp->reset_gpio = NULL;
2211 dev_err(dev, "failed to get reset GPIO: %ld\n",
2212 PTR_ERR(rp->reset_gpio));
2213 err = PTR_ERR(rp->reset_gpio);
2218 list_add_tail(&rp->list, &pcie->ports);
2221 err = tegra_pcie_get_xbar_config(pcie, lanes, &pcie->xbar_config);
2223 dev_err(dev, "invalid lane configuration\n");
2227 err = tegra_pcie_get_regulators(pcie, mask);
2239 * FIXME: If there are no PCIe cards attached, then calling this function
2240 * can result in the increase of the bootup time as there are big timeout
2243 #define TEGRA_PCIE_LINKUP_TIMEOUT 200 /* up to 1.2 seconds */
2244 static bool tegra_pcie_port_check_link(struct tegra_pcie_port *port)
2246 struct device *dev = port->pcie->dev;
2247 unsigned int retries = 3;
2248 unsigned long value;
2250 /* override presence detection */
2251 value = readl(port->base + RP_PRIV_MISC);
2252 value &= ~RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT;
2253 value |= RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT;
2254 writel(value, port->base + RP_PRIV_MISC);
2257 unsigned int timeout = TEGRA_PCIE_LINKUP_TIMEOUT;
2260 value = readl(port->base + RP_VEND_XP);
2262 if (value & RP_VEND_XP_DL_UP)
2265 usleep_range(1000, 2000);
2266 } while (--timeout);
2269 dev_dbg(dev, "link %u down, retrying\n", port->index);
2273 timeout = TEGRA_PCIE_LINKUP_TIMEOUT;
2276 value = readl(port->base + RP_LINK_CONTROL_STATUS);
2278 if (value & RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE)
2281 usleep_range(1000, 2000);
2282 } while (--timeout);
2285 tegra_pcie_port_reset(port);
2286 } while (--retries);
2291 static void tegra_pcie_change_link_speed(struct tegra_pcie *pcie)
2293 struct device *dev = pcie->dev;
2294 struct tegra_pcie_port *port;
2298 list_for_each_entry(port, &pcie->ports, list) {
2300 * "Supported Link Speeds Vector" in "Link Capabilities 2"
2301 * is not supported by Tegra. tegra_pcie_change_link_speed()
2302 * is called only for Tegra chips which support Gen2.
2303 * So there no harm if supported link speed is not verified.
2305 value = readl(port->base + RP_LINK_CONTROL_STATUS_2);
2306 value &= ~PCI_EXP_LNKSTA_CLS;
2307 value |= PCI_EXP_LNKSTA_CLS_5_0GB;
2308 writel(value, port->base + RP_LINK_CONTROL_STATUS_2);
2311 * Poll until link comes back from recovery to avoid race
2314 deadline = ktime_add_us(ktime_get(), LINK_RETRAIN_TIMEOUT);
2316 while (ktime_before(ktime_get(), deadline)) {
2317 value = readl(port->base + RP_LINK_CONTROL_STATUS);
2318 if ((value & PCI_EXP_LNKSTA_LT) == 0)
2321 usleep_range(2000, 3000);
2324 if (value & PCI_EXP_LNKSTA_LT)
2325 dev_warn(dev, "PCIe port %u link is in recovery\n",
2328 /* Retrain the link */
2329 value = readl(port->base + RP_LINK_CONTROL_STATUS);
2330 value |= PCI_EXP_LNKCTL_RL;
2331 writel(value, port->base + RP_LINK_CONTROL_STATUS);
2333 deadline = ktime_add_us(ktime_get(), LINK_RETRAIN_TIMEOUT);
2335 while (ktime_before(ktime_get(), deadline)) {
2336 value = readl(port->base + RP_LINK_CONTROL_STATUS);
2337 if ((value & PCI_EXP_LNKSTA_LT) == 0)
2340 usleep_range(2000, 3000);
2343 if (value & PCI_EXP_LNKSTA_LT)
2344 dev_err(dev, "failed to retrain link of port %u\n",
2349 static void tegra_pcie_enable_ports(struct tegra_pcie *pcie)
2351 struct device *dev = pcie->dev;
2352 struct tegra_pcie_port *port, *tmp;
2354 list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
2355 dev_info(dev, "probing port %u, using %u lanes\n",
2356 port->index, port->lanes);
2358 tegra_pcie_port_enable(port);
2361 /* Start LTSSM from Tegra side */
2362 reset_control_deassert(pcie->pcie_xrst);
2364 list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
2365 if (tegra_pcie_port_check_link(port))
2368 dev_info(dev, "link %u down, ignoring\n", port->index);
2370 tegra_pcie_port_disable(port);
2371 tegra_pcie_port_free(port);
2374 if (pcie->soc->has_gen2)
2375 tegra_pcie_change_link_speed(pcie);
2378 static void tegra_pcie_disable_ports(struct tegra_pcie *pcie)
2380 struct tegra_pcie_port *port, *tmp;
2382 reset_control_assert(pcie->pcie_xrst);
2384 list_for_each_entry_safe(port, tmp, &pcie->ports, list)
2385 tegra_pcie_port_disable(port);
2388 static const struct tegra_pcie_port_soc tegra20_pcie_ports[] = {
2389 { .pme.turnoff_bit = 0, .pme.ack_bit = 5 },
2390 { .pme.turnoff_bit = 8, .pme.ack_bit = 10 },
2393 static const struct tegra_pcie_soc tegra20_pcie = {
2395 .ports = tegra20_pcie_ports,
2396 .msi_base_shift = 0,
2397 .pads_pll_ctl = PADS_PLL_CTL_TEGRA20,
2398 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_DIV10,
2399 .pads_refclk_cfg0 = 0xfa5cfa5c,
2400 .has_pex_clkreq_en = false,
2401 .has_pex_bias_ctrl = false,
2402 .has_intr_prsnt_sense = false,
2403 .has_cml_clk = false,
2405 .force_pca_enable = false,
2406 .program_uphy = true,
2407 .update_clamp_threshold = false,
2408 .program_deskew_time = false,
2409 .update_fc_timer = false,
2410 .has_cache_bars = true,
2411 .ectl.enable = false,
2414 static const struct tegra_pcie_port_soc tegra30_pcie_ports[] = {
2415 { .pme.turnoff_bit = 0, .pme.ack_bit = 5 },
2416 { .pme.turnoff_bit = 8, .pme.ack_bit = 10 },
2417 { .pme.turnoff_bit = 16, .pme.ack_bit = 18 },
2420 static const struct tegra_pcie_soc tegra30_pcie = {
2422 .ports = tegra30_pcie_ports,
2423 .msi_base_shift = 8,
2424 .afi_pex2_ctrl = 0x128,
2425 .pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
2426 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
2427 .pads_refclk_cfg0 = 0xfa5cfa5c,
2428 .pads_refclk_cfg1 = 0xfa5cfa5c,
2429 .has_pex_clkreq_en = true,
2430 .has_pex_bias_ctrl = true,
2431 .has_intr_prsnt_sense = true,
2432 .has_cml_clk = true,
2434 .force_pca_enable = false,
2435 .program_uphy = true,
2436 .update_clamp_threshold = false,
2437 .program_deskew_time = false,
2438 .update_fc_timer = false,
2439 .has_cache_bars = false,
2440 .ectl.enable = false,
2443 static const struct tegra_pcie_soc tegra124_pcie = {
2445 .ports = tegra20_pcie_ports,
2446 .msi_base_shift = 8,
2447 .pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
2448 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
2449 .pads_refclk_cfg0 = 0x44ac44ac,
2450 .has_pex_clkreq_en = true,
2451 .has_pex_bias_ctrl = true,
2452 .has_intr_prsnt_sense = true,
2453 .has_cml_clk = true,
2455 .force_pca_enable = false,
2456 .program_uphy = true,
2457 .update_clamp_threshold = true,
2458 .program_deskew_time = false,
2459 .update_fc_timer = false,
2460 .has_cache_bars = false,
2461 .ectl.enable = false,
2464 static const struct tegra_pcie_soc tegra210_pcie = {
2466 .ports = tegra20_pcie_ports,
2467 .msi_base_shift = 8,
2468 .pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
2469 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
2470 .pads_refclk_cfg0 = 0x90b890b8,
2471 /* FC threshold is bit[25:18] */
2472 .update_fc_threshold = 0x01800000,
2473 .has_pex_clkreq_en = true,
2474 .has_pex_bias_ctrl = true,
2475 .has_intr_prsnt_sense = true,
2476 .has_cml_clk = true,
2478 .force_pca_enable = true,
2479 .program_uphy = true,
2480 .update_clamp_threshold = true,
2481 .program_deskew_time = true,
2482 .update_fc_timer = true,
2483 .has_cache_bars = false,
2486 .rp_ectl_2_r1 = 0x0000000f,
2487 .rp_ectl_4_r1 = 0x00000067,
2488 .rp_ectl_5_r1 = 0x55010000,
2489 .rp_ectl_6_r1 = 0x00000001,
2490 .rp_ectl_2_r2 = 0x0000008f,
2491 .rp_ectl_4_r2 = 0x000000c7,
2492 .rp_ectl_5_r2 = 0x55010000,
2493 .rp_ectl_6_r2 = 0x00000001,
2499 static const struct tegra_pcie_port_soc tegra186_pcie_ports[] = {
2500 { .pme.turnoff_bit = 0, .pme.ack_bit = 5 },
2501 { .pme.turnoff_bit = 8, .pme.ack_bit = 10 },
2502 { .pme.turnoff_bit = 12, .pme.ack_bit = 14 },
2505 static const struct tegra_pcie_soc tegra186_pcie = {
2507 .ports = tegra186_pcie_ports,
2508 .msi_base_shift = 8,
2509 .afi_pex2_ctrl = 0x19c,
2510 .pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
2511 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
2512 .pads_refclk_cfg0 = 0x80b880b8,
2513 .pads_refclk_cfg1 = 0x000480b8,
2514 .has_pex_clkreq_en = true,
2515 .has_pex_bias_ctrl = true,
2516 .has_intr_prsnt_sense = true,
2517 .has_cml_clk = false,
2519 .force_pca_enable = false,
2520 .program_uphy = false,
2521 .update_clamp_threshold = false,
2522 .program_deskew_time = false,
2523 .update_fc_timer = false,
2524 .has_cache_bars = false,
2525 .ectl.enable = false,
2528 static const struct of_device_id tegra_pcie_of_match[] = {
2529 { .compatible = "nvidia,tegra186-pcie", .data = &tegra186_pcie },
2530 { .compatible = "nvidia,tegra210-pcie", .data = &tegra210_pcie },
2531 { .compatible = "nvidia,tegra124-pcie", .data = &tegra124_pcie },
2532 { .compatible = "nvidia,tegra30-pcie", .data = &tegra30_pcie },
2533 { .compatible = "nvidia,tegra20-pcie", .data = &tegra20_pcie },
2536 MODULE_DEVICE_TABLE(of, tegra_pcie_of_match);
2538 static void *tegra_pcie_ports_seq_start(struct seq_file *s, loff_t *pos)
2540 struct tegra_pcie *pcie = s->private;
2542 if (list_empty(&pcie->ports))
2545 seq_puts(s, "Index Status\n");
2547 return seq_list_start(&pcie->ports, *pos);
2550 static void *tegra_pcie_ports_seq_next(struct seq_file *s, void *v, loff_t *pos)
2552 struct tegra_pcie *pcie = s->private;
2554 return seq_list_next(v, &pcie->ports, pos);
2557 static void tegra_pcie_ports_seq_stop(struct seq_file *s, void *v)
2561 static int tegra_pcie_ports_seq_show(struct seq_file *s, void *v)
2563 bool up = false, active = false;
2564 struct tegra_pcie_port *port;
2567 port = list_entry(v, struct tegra_pcie_port, list);
2569 value = readl(port->base + RP_VEND_XP);
2571 if (value & RP_VEND_XP_DL_UP)
2574 value = readl(port->base + RP_LINK_CONTROL_STATUS);
2576 if (value & RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE)
2579 seq_printf(s, "%2u ", port->index);
2588 seq_puts(s, "active");
2595 static const struct seq_operations tegra_pcie_ports_sops = {
2596 .start = tegra_pcie_ports_seq_start,
2597 .next = tegra_pcie_ports_seq_next,
2598 .stop = tegra_pcie_ports_seq_stop,
2599 .show = tegra_pcie_ports_seq_show,
2602 DEFINE_SEQ_ATTRIBUTE(tegra_pcie_ports);
2604 static void tegra_pcie_debugfs_exit(struct tegra_pcie *pcie)
2606 debugfs_remove_recursive(pcie->debugfs);
2607 pcie->debugfs = NULL;
2610 static void tegra_pcie_debugfs_init(struct tegra_pcie *pcie)
2612 pcie->debugfs = debugfs_create_dir("pcie", NULL);
2614 debugfs_create_file("ports", S_IFREG | S_IRUGO, pcie->debugfs, pcie,
2615 &tegra_pcie_ports_fops);
2618 static int tegra_pcie_probe(struct platform_device *pdev)
2620 struct device *dev = &pdev->dev;
2621 struct pci_host_bridge *host;
2622 struct tegra_pcie *pcie;
2625 host = devm_pci_alloc_host_bridge(dev, sizeof(*pcie));
2629 pcie = pci_host_bridge_priv(host);
2630 host->sysdata = pcie;
2631 platform_set_drvdata(pdev, pcie);
2633 pcie->soc = of_device_get_match_data(dev);
2634 INIT_LIST_HEAD(&pcie->ports);
2637 err = tegra_pcie_parse_dt(pcie);
2641 err = tegra_pcie_get_resources(pcie);
2643 dev_err(dev, "failed to request resources: %d\n", err);
2647 err = tegra_pcie_msi_setup(pcie);
2649 dev_err(dev, "failed to enable MSI support: %d\n", err);
2653 pm_runtime_enable(pcie->dev);
2654 err = pm_runtime_get_sync(pcie->dev);
2656 dev_err(dev, "fail to enable pcie controller: %d\n", err);
2657 goto pm_runtime_put;
2660 host->ops = &tegra_pcie_ops;
2661 host->map_irq = tegra_pcie_map_irq;
2663 err = pci_host_probe(host);
2665 dev_err(dev, "failed to register host: %d\n", err);
2666 goto pm_runtime_put;
2669 if (IS_ENABLED(CONFIG_DEBUG_FS))
2670 tegra_pcie_debugfs_init(pcie);
2675 pm_runtime_put_sync(pcie->dev);
2676 pm_runtime_disable(pcie->dev);
2677 tegra_pcie_msi_teardown(pcie);
2679 tegra_pcie_put_resources(pcie);
2683 static void tegra_pcie_remove(struct platform_device *pdev)
2685 struct tegra_pcie *pcie = platform_get_drvdata(pdev);
2686 struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
2687 struct tegra_pcie_port *port, *tmp;
2689 if (IS_ENABLED(CONFIG_DEBUG_FS))
2690 tegra_pcie_debugfs_exit(pcie);
2692 pci_stop_root_bus(host->bus);
2693 pci_remove_root_bus(host->bus);
2694 pm_runtime_put_sync(pcie->dev);
2695 pm_runtime_disable(pcie->dev);
2697 if (IS_ENABLED(CONFIG_PCI_MSI))
2698 tegra_pcie_msi_teardown(pcie);
2700 tegra_pcie_put_resources(pcie);
2702 list_for_each_entry_safe(port, tmp, &pcie->ports, list)
2703 tegra_pcie_port_free(port);
2706 static int tegra_pcie_pm_suspend(struct device *dev)
2708 struct tegra_pcie *pcie = dev_get_drvdata(dev);
2709 struct tegra_pcie_port *port;
2712 list_for_each_entry(port, &pcie->ports, list)
2713 tegra_pcie_pme_turnoff(port);
2715 tegra_pcie_disable_ports(pcie);
2718 * AFI_INTR is unmasked in tegra_pcie_enable_controller(), mask it to
2719 * avoid unwanted interrupts raised by AFI after pex_rst is asserted.
2721 tegra_pcie_disable_interrupts(pcie);
2723 if (pcie->soc->program_uphy) {
2724 err = tegra_pcie_phy_power_off(pcie);
2726 dev_err(dev, "failed to power off PHY(s): %d\n", err);
2729 reset_control_assert(pcie->pex_rst);
2730 clk_disable_unprepare(pcie->pex_clk);
2732 if (IS_ENABLED(CONFIG_PCI_MSI))
2733 tegra_pcie_disable_msi(pcie);
2735 pinctrl_pm_select_idle_state(dev);
2736 tegra_pcie_power_off(pcie);
2741 static int tegra_pcie_pm_resume(struct device *dev)
2743 struct tegra_pcie *pcie = dev_get_drvdata(dev);
2746 err = tegra_pcie_power_on(pcie);
2748 dev_err(dev, "tegra pcie power on fail: %d\n", err);
2752 err = pinctrl_pm_select_default_state(dev);
2754 dev_err(dev, "failed to disable PCIe IO DPD: %d\n", err);
2758 tegra_pcie_enable_controller(pcie);
2759 tegra_pcie_setup_translations(pcie);
2761 if (IS_ENABLED(CONFIG_PCI_MSI))
2762 tegra_pcie_enable_msi(pcie);
2764 err = clk_prepare_enable(pcie->pex_clk);
2766 dev_err(dev, "failed to enable PEX clock: %d\n", err);
2767 goto pex_dpd_enable;
2770 reset_control_deassert(pcie->pex_rst);
2772 if (pcie->soc->program_uphy) {
2773 err = tegra_pcie_phy_power_on(pcie);
2775 dev_err(dev, "failed to power on PHY(s): %d\n", err);
2776 goto disable_pex_clk;
2780 tegra_pcie_apply_pad_settings(pcie);
2781 tegra_pcie_enable_ports(pcie);
2786 reset_control_assert(pcie->pex_rst);
2787 clk_disable_unprepare(pcie->pex_clk);
2789 pinctrl_pm_select_idle_state(dev);
2791 tegra_pcie_power_off(pcie);
2796 static const struct dev_pm_ops tegra_pcie_pm_ops = {
2797 RUNTIME_PM_OPS(tegra_pcie_pm_suspend, tegra_pcie_pm_resume, NULL)
2798 NOIRQ_SYSTEM_SLEEP_PM_OPS(tegra_pcie_pm_suspend, tegra_pcie_pm_resume)
2801 static struct platform_driver tegra_pcie_driver = {
2803 .name = "tegra-pcie",
2804 .of_match_table = tegra_pcie_of_match,
2805 .suppress_bind_attrs = true,
2806 .pm = &tegra_pcie_pm_ops,
2808 .probe = tegra_pcie_probe,
2809 .remove_new = tegra_pcie_remove,
2811 module_platform_driver(tegra_pcie_driver);