]>
Commit | Line | Data |
---|---|---|
56e15a23 VS |
1 | // SPDX-License-Identifier: GPL-2.0+ |
2 | /* | |
3 | * PCIe host controller driver for Tegra194 SoC | |
4 | * | |
5 | * Copyright (C) 2019 NVIDIA Corporation. | |
6 | * | |
7 | * Author: Vidya Sagar <vidyas@nvidia.com> | |
8 | */ | |
9 | ||
10 | #include <linux/clk.h> | |
11 | #include <linux/debugfs.h> | |
12 | #include <linux/delay.h> | |
13 | #include <linux/gpio.h> | |
c57247f9 | 14 | #include <linux/gpio/consumer.h> |
56e15a23 VS |
15 | #include <linux/interrupt.h> |
16 | #include <linux/iopoll.h> | |
17 | #include <linux/kernel.h> | |
18 | #include <linux/module.h> | |
19 | #include <linux/of.h> | |
20 | #include <linux/of_device.h> | |
21 | #include <linux/of_gpio.h> | |
22 | #include <linux/of_irq.h> | |
23 | #include <linux/of_pci.h> | |
24 | #include <linux/pci.h> | |
25 | #include <linux/phy/phy.h> | |
f4ff4faf | 26 | #include <linux/pinctrl/consumer.h> |
56e15a23 VS |
27 | #include <linux/platform_device.h> |
28 | #include <linux/pm_runtime.h> | |
29 | #include <linux/random.h> | |
30 | #include <linux/reset.h> | |
31 | #include <linux/resource.h> | |
32 | #include <linux/types.h> | |
33 | #include "pcie-designware.h" | |
34 | #include <soc/tegra/bpmp.h> | |
35 | #include <soc/tegra/bpmp-abi.h> | |
36 | #include "../../pci.h" | |
37 | ||
38 | #define APPL_PINMUX 0x0 | |
39 | #define APPL_PINMUX_PEX_RST BIT(0) | |
40 | #define APPL_PINMUX_CLKREQ_OVERRIDE_EN BIT(2) | |
41 | #define APPL_PINMUX_CLKREQ_OVERRIDE BIT(3) | |
42 | #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN BIT(4) | |
43 | #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE BIT(5) | |
56e15a23 VS |
44 | |
45 | #define APPL_CTRL 0x4 | |
46 | #define APPL_CTRL_SYS_PRE_DET_STATE BIT(6) | |
47 | #define APPL_CTRL_LTSSM_EN BIT(7) | |
48 | #define APPL_CTRL_HW_HOT_RST_EN BIT(20) | |
49 | #define APPL_CTRL_HW_HOT_RST_MODE_MASK GENMASK(1, 0) | |
50 | #define APPL_CTRL_HW_HOT_RST_MODE_SHIFT 22 | |
51 | #define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST 0x1 | |
52 | ||
53 | #define APPL_INTR_EN_L0_0 0x8 | |
54 | #define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN BIT(0) | |
55 | #define APPL_INTR_EN_L0_0_MSI_RCV_INT_EN BIT(4) | |
56 | #define APPL_INTR_EN_L0_0_INT_INT_EN BIT(8) | |
c57247f9 | 57 | #define APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN BIT(15) |
56e15a23 VS |
58 | #define APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN BIT(19) |
59 | #define APPL_INTR_EN_L0_0_SYS_INTR_EN BIT(30) | |
60 | #define APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN BIT(31) | |
61 | ||
62 | #define APPL_INTR_STATUS_L0 0xC | |
63 | #define APPL_INTR_STATUS_L0_LINK_STATE_INT BIT(0) | |
64 | #define APPL_INTR_STATUS_L0_INT_INT BIT(8) | |
c57247f9 VS |
65 | #define APPL_INTR_STATUS_L0_PCI_CMD_EN_INT BIT(15) |
66 | #define APPL_INTR_STATUS_L0_PEX_RST_INT BIT(16) | |
56e15a23 VS |
67 | #define APPL_INTR_STATUS_L0_CDM_REG_CHK_INT BIT(18) |
68 | ||
69 | #define APPL_INTR_EN_L1_0_0 0x1C | |
70 | #define APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN BIT(1) | |
c57247f9 VS |
71 | #define APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN BIT(3) |
72 | #define APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN BIT(30) | |
56e15a23 VS |
73 | |
74 | #define APPL_INTR_STATUS_L1_0_0 0x20 | |
75 | #define APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED BIT(1) | |
c57247f9 VS |
76 | #define APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED BIT(3) |
77 | #define APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE BIT(30) | |
56e15a23 VS |
78 | |
79 | #define APPL_INTR_STATUS_L1_1 0x2C | |
80 | #define APPL_INTR_STATUS_L1_2 0x30 | |
81 | #define APPL_INTR_STATUS_L1_3 0x34 | |
82 | #define APPL_INTR_STATUS_L1_6 0x3C | |
83 | #define APPL_INTR_STATUS_L1_7 0x40 | |
c57247f9 | 84 | #define APPL_INTR_STATUS_L1_15_CFG_BME_CHGED BIT(1) |
56e15a23 VS |
85 | |
86 | #define APPL_INTR_EN_L1_8_0 0x44 | |
87 | #define APPL_INTR_EN_L1_8_BW_MGT_INT_EN BIT(2) | |
88 | #define APPL_INTR_EN_L1_8_AUTO_BW_INT_EN BIT(3) | |
89 | #define APPL_INTR_EN_L1_8_INTX_EN BIT(11) | |
90 | #define APPL_INTR_EN_L1_8_AER_INT_EN BIT(15) | |
91 | ||
92 | #define APPL_INTR_STATUS_L1_8_0 0x4C | |
93 | #define APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK GENMASK(11, 6) | |
94 | #define APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS BIT(2) | |
95 | #define APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS BIT(3) | |
96 | ||
97 | #define APPL_INTR_STATUS_L1_9 0x54 | |
98 | #define APPL_INTR_STATUS_L1_10 0x58 | |
99 | #define APPL_INTR_STATUS_L1_11 0x64 | |
100 | #define APPL_INTR_STATUS_L1_13 0x74 | |
101 | #define APPL_INTR_STATUS_L1_14 0x78 | |
102 | #define APPL_INTR_STATUS_L1_15 0x7C | |
103 | #define APPL_INTR_STATUS_L1_17 0x88 | |
104 | ||
105 | #define APPL_INTR_EN_L1_18 0x90 | |
106 | #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMPLT BIT(2) | |
107 | #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR BIT(1) | |
108 | #define APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0) | |
109 | ||
110 | #define APPL_INTR_STATUS_L1_18 0x94 | |
111 | #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT BIT(2) | |
112 | #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR BIT(1) | |
113 | #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0) | |
114 | ||
c57247f9 VS |
115 | #define APPL_MSI_CTRL_1 0xAC |
116 | ||
56e15a23 VS |
117 | #define APPL_MSI_CTRL_2 0xB0 |
118 | ||
c57247f9 VS |
119 | #define APPL_LEGACY_INTX 0xB8 |
120 | ||
56e15a23 VS |
121 | #define APPL_LTR_MSG_1 0xC4 |
122 | #define LTR_MSG_REQ BIT(15) | |
123 | #define LTR_MST_NO_SNOOP_SHIFT 16 | |
124 | ||
125 | #define APPL_LTR_MSG_2 0xC8 | |
126 | #define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE BIT(3) | |
127 | ||
128 | #define APPL_LINK_STATUS 0xCC | |
129 | #define APPL_LINK_STATUS_RDLH_LINK_UP BIT(0) | |
130 | ||
131 | #define APPL_DEBUG 0xD0 | |
132 | #define APPL_DEBUG_PM_LINKST_IN_L2_LAT BIT(21) | |
133 | #define APPL_DEBUG_PM_LINKST_IN_L0 0x11 | |
134 | #define APPL_DEBUG_LTSSM_STATE_MASK GENMASK(8, 3) | |
135 | #define APPL_DEBUG_LTSSM_STATE_SHIFT 3 | |
136 | #define LTSSM_STATE_PRE_DETECT 5 | |
137 | ||
138 | #define APPL_RADM_STATUS 0xE4 | |
139 | #define APPL_PM_XMT_TURNOFF_STATE BIT(0) | |
140 | ||
141 | #define APPL_DM_TYPE 0x100 | |
142 | #define APPL_DM_TYPE_MASK GENMASK(3, 0) | |
143 | #define APPL_DM_TYPE_RP 0x4 | |
144 | #define APPL_DM_TYPE_EP 0x0 | |
145 | ||
146 | #define APPL_CFG_BASE_ADDR 0x104 | |
147 | #define APPL_CFG_BASE_ADDR_MASK GENMASK(31, 12) | |
148 | ||
149 | #define APPL_CFG_IATU_DMA_BASE_ADDR 0x108 | |
150 | #define APPL_CFG_IATU_DMA_BASE_ADDR_MASK GENMASK(31, 18) | |
151 | ||
152 | #define APPL_CFG_MISC 0x110 | |
153 | #define APPL_CFG_MISC_SLV_EP_MODE BIT(14) | |
154 | #define APPL_CFG_MISC_ARCACHE_MASK GENMASK(13, 10) | |
155 | #define APPL_CFG_MISC_ARCACHE_SHIFT 10 | |
156 | #define APPL_CFG_MISC_ARCACHE_VAL 3 | |
157 | ||
158 | #define APPL_CFG_SLCG_OVERRIDE 0x114 | |
159 | #define APPL_CFG_SLCG_OVERRIDE_SLCG_EN_MASTER BIT(0) | |
160 | ||
161 | #define APPL_CAR_RESET_OVRD 0x12C | |
162 | #define APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N BIT(0) | |
163 | ||
164 | #define IO_BASE_IO_DECODE BIT(0) | |
165 | #define IO_BASE_IO_DECODE_BIT8 BIT(8) | |
166 | ||
167 | #define CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE BIT(0) | |
168 | #define CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE BIT(16) | |
169 | ||
170 | #define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF 0x718 | |
171 | #define CFG_TIMER_CTRL_ACK_NAK_SHIFT (19) | |
172 | ||
173 | #define EVENT_COUNTER_ALL_CLEAR 0x3 | |
174 | #define EVENT_COUNTER_ENABLE_ALL 0x7 | |
175 | #define EVENT_COUNTER_ENABLE_SHIFT 2 | |
176 | #define EVENT_COUNTER_EVENT_SEL_MASK GENMASK(7, 0) | |
177 | #define EVENT_COUNTER_EVENT_SEL_SHIFT 16 | |
178 | #define EVENT_COUNTER_EVENT_Tx_L0S 0x2 | |
179 | #define EVENT_COUNTER_EVENT_Rx_L0S 0x3 | |
180 | #define EVENT_COUNTER_EVENT_L1 0x5 | |
181 | #define EVENT_COUNTER_EVENT_L1_1 0x7 | |
182 | #define EVENT_COUNTER_EVENT_L1_2 0x8 | |
183 | #define EVENT_COUNTER_GROUP_SEL_SHIFT 24 | |
184 | #define EVENT_COUNTER_GROUP_5 0x5 | |
185 | ||
186 | #define PORT_LOGIC_ACK_F_ASPM_CTRL 0x70C | |
187 | #define ENTER_ASPM BIT(30) | |
188 | #define L0S_ENTRANCE_LAT_SHIFT 24 | |
189 | #define L0S_ENTRANCE_LAT_MASK GENMASK(26, 24) | |
190 | #define L1_ENTRANCE_LAT_SHIFT 27 | |
191 | #define L1_ENTRANCE_LAT_MASK GENMASK(29, 27) | |
192 | #define N_FTS_SHIFT 8 | |
193 | #define N_FTS_MASK GENMASK(7, 0) | |
194 | #define N_FTS_VAL 52 | |
195 | ||
196 | #define PORT_LOGIC_GEN2_CTRL 0x80C | |
197 | #define PORT_LOGIC_GEN2_CTRL_DIRECT_SPEED_CHANGE BIT(17) | |
198 | #define FTS_MASK GENMASK(7, 0) | |
199 | #define FTS_VAL 52 | |
200 | ||
201 | #define PORT_LOGIC_MSI_CTRL_INT_0_EN 0x828 | |
202 | ||
203 | #define GEN3_EQ_CONTROL_OFF 0x8a8 | |
204 | #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT 8 | |
205 | #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK GENMASK(23, 8) | |
206 | #define GEN3_EQ_CONTROL_OFF_FB_MODE_MASK GENMASK(3, 0) | |
207 | ||
208 | #define GEN3_RELATED_OFF 0x890 | |
209 | #define GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL BIT(0) | |
210 | #define GEN3_RELATED_OFF_GEN3_EQ_DISABLE BIT(16) | |
211 | #define GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT 24 | |
212 | #define GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK GENMASK(25, 24) | |
213 | ||
214 | #define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT 0x8D0 | |
215 | #define AMBA_ERROR_RESPONSE_CRS_SHIFT 3 | |
216 | #define AMBA_ERROR_RESPONSE_CRS_MASK GENMASK(1, 0) | |
217 | #define AMBA_ERROR_RESPONSE_CRS_OKAY 0 | |
218 | #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFFFFFF 1 | |
219 | #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 2 | |
220 | ||
c57247f9 VS |
221 | #define MSIX_ADDR_MATCH_LOW_OFF 0x940 |
222 | #define MSIX_ADDR_MATCH_LOW_OFF_EN BIT(0) | |
223 | #define MSIX_ADDR_MATCH_LOW_OFF_MASK GENMASK(31, 2) | |
224 | ||
225 | #define MSIX_ADDR_MATCH_HIGH_OFF 0x944 | |
226 | #define MSIX_ADDR_MATCH_HIGH_OFF_MASK GENMASK(31, 0) | |
227 | ||
56e15a23 VS |
228 | #define PORT_LOGIC_MSIX_DOORBELL 0x948 |
229 | ||
230 | #define CAP_SPCIE_CAP_OFF 0x154 | |
231 | #define CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK GENMASK(3, 0) | |
232 | #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK GENMASK(11, 8) | |
233 | #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT 8 | |
234 | ||
235 | #define PME_ACK_TIMEOUT 10000 | |
236 | ||
237 | #define LTSSM_TIMEOUT 50000 /* 50ms */ | |
238 | ||
239 | #define GEN3_GEN4_EQ_PRESET_INIT 5 | |
240 | ||
241 | #define GEN1_CORE_CLK_FREQ 62500000 | |
242 | #define GEN2_CORE_CLK_FREQ 125000000 | |
243 | #define GEN3_CORE_CLK_FREQ 250000000 | |
244 | #define GEN4_CORE_CLK_FREQ 500000000 | |
245 | ||
c57247f9 VS |
246 | #define LTR_MSG_TIMEOUT (100 * 1000) |
247 | ||
248 | #define PERST_DEBOUNCE_TIME (5 * 1000) | |
249 | ||
250 | #define EP_STATE_DISABLED 0 | |
251 | #define EP_STATE_ENABLED 1 | |
252 | ||
56e15a23 VS |
253 | static const unsigned int pcie_gen_freq[] = { |
254 | GEN1_CORE_CLK_FREQ, | |
255 | GEN2_CORE_CLK_FREQ, | |
256 | GEN3_CORE_CLK_FREQ, | |
257 | GEN4_CORE_CLK_FREQ | |
258 | }; | |
259 | ||
260 | static const u32 event_cntr_ctrl_offset[] = { | |
261 | 0x1d8, | |
262 | 0x1a8, | |
263 | 0x1a8, | |
264 | 0x1a8, | |
265 | 0x1c4, | |
266 | 0x1d8 | |
267 | }; | |
268 | ||
269 | static const u32 event_cntr_data_offset[] = { | |
270 | 0x1dc, | |
271 | 0x1ac, | |
272 | 0x1ac, | |
273 | 0x1ac, | |
274 | 0x1c8, | |
275 | 0x1dc | |
276 | }; | |
277 | ||
278 | struct tegra_pcie_dw { | |
279 | struct device *dev; | |
280 | struct resource *appl_res; | |
281 | struct resource *dbi_res; | |
282 | struct resource *atu_dma_res; | |
283 | void __iomem *appl_base; | |
284 | struct clk *core_clk; | |
285 | struct reset_control *core_apb_rst; | |
286 | struct reset_control *core_rst; | |
287 | struct dw_pcie pci; | |
288 | struct tegra_bpmp *bpmp; | |
289 | ||
c57247f9 VS |
290 | enum dw_pcie_device_mode mode; |
291 | ||
56e15a23 VS |
292 | bool supports_clkreq; |
293 | bool enable_cdm_check; | |
294 | bool link_state; | |
295 | bool update_fc_fixup; | |
296 | u8 init_link_width; | |
297 | u32 msi_ctrl_int; | |
298 | u32 num_lanes; | |
299 | u32 max_speed; | |
300 | u32 cid; | |
301 | u32 cfg_link_cap_l1sub; | |
302 | u32 pcie_cap_base; | |
303 | u32 aspm_cmrt; | |
304 | u32 aspm_pwr_on_t; | |
305 | u32 aspm_l0s_enter_lat; | |
306 | ||
307 | struct regulator *pex_ctl_supply; | |
0a901f21 VS |
308 | struct regulator *slot_ctl_3v3; |
309 | struct regulator *slot_ctl_12v; | |
56e15a23 VS |
310 | |
311 | unsigned int phy_count; | |
312 | struct phy **phys; | |
313 | ||
314 | struct dentry *debugfs; | |
c57247f9 VS |
315 | |
316 | /* Endpoint mode specific */ | |
317 | struct gpio_desc *pex_rst_gpiod; | |
318 | struct gpio_desc *pex_refclk_sel_gpiod; | |
319 | unsigned int pex_rst_irq; | |
320 | int ep_state; | |
321 | }; | |
322 | ||
323 | struct tegra_pcie_dw_of_data { | |
324 | enum dw_pcie_device_mode mode; | |
56e15a23 VS |
325 | }; |
326 | ||
327 | static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci) | |
328 | { | |
329 | return container_of(pci, struct tegra_pcie_dw, pci); | |
330 | } | |
331 | ||
332 | static inline void appl_writel(struct tegra_pcie_dw *pcie, const u32 value, | |
333 | const u32 reg) | |
334 | { | |
335 | writel_relaxed(value, pcie->appl_base + reg); | |
336 | } | |
337 | ||
338 | static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg) | |
339 | { | |
340 | return readl_relaxed(pcie->appl_base + reg); | |
341 | } | |
342 | ||
343 | struct tegra_pcie_soc { | |
344 | enum dw_pcie_device_mode mode; | |
345 | }; | |
346 | ||
347 | static void apply_bad_link_workaround(struct pcie_port *pp) | |
348 | { | |
349 | struct dw_pcie *pci = to_dw_pcie_from_pp(pp); | |
350 | struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); | |
351 | u32 current_link_width; | |
352 | u16 val; | |
353 | ||
354 | /* | |
355 | * NOTE:- Since this scenario is uncommon and link as such is not | |
356 | * stable anyway, not waiting to confirm if link is really | |
357 | * transitioning to Gen-2 speed | |
358 | */ | |
359 | val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA); | |
360 | if (val & PCI_EXP_LNKSTA_LBMS) { | |
361 | current_link_width = (val & PCI_EXP_LNKSTA_NLW) >> | |
362 | PCI_EXP_LNKSTA_NLW_SHIFT; | |
363 | if (pcie->init_link_width > current_link_width) { | |
364 | dev_warn(pci->dev, "PCIe link is bad, width reduced\n"); | |
365 | val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + | |
366 | PCI_EXP_LNKCTL2); | |
367 | val &= ~PCI_EXP_LNKCTL2_TLS; | |
368 | val |= PCI_EXP_LNKCTL2_TLS_2_5GT; | |
369 | dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + | |
370 | PCI_EXP_LNKCTL2, val); | |
371 | ||
372 | val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + | |
373 | PCI_EXP_LNKCTL); | |
374 | val |= PCI_EXP_LNKCTL_RL; | |
375 | dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + | |
376 | PCI_EXP_LNKCTL, val); | |
377 | } | |
378 | } | |
379 | } | |
380 | ||
c57247f9 | 381 | static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg) |
56e15a23 | 382 | { |
c57247f9 | 383 | struct tegra_pcie_dw *pcie = arg; |
56e15a23 VS |
384 | struct dw_pcie *pci = &pcie->pci; |
385 | struct pcie_port *pp = &pci->pp; | |
386 | u32 val, tmp; | |
387 | u16 val_w; | |
388 | ||
389 | val = appl_readl(pcie, APPL_INTR_STATUS_L0); | |
390 | if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) { | |
391 | val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0); | |
392 | if (val & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) { | |
393 | appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0); | |
394 | ||
395 | /* SBR & Surprise Link Down WAR */ | |
396 | val = appl_readl(pcie, APPL_CAR_RESET_OVRD); | |
397 | val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N; | |
398 | appl_writel(pcie, val, APPL_CAR_RESET_OVRD); | |
399 | udelay(1); | |
400 | val = appl_readl(pcie, APPL_CAR_RESET_OVRD); | |
401 | val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N; | |
402 | appl_writel(pcie, val, APPL_CAR_RESET_OVRD); | |
403 | ||
404 | val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL); | |
405 | val |= PORT_LOGIC_GEN2_CTRL_DIRECT_SPEED_CHANGE; | |
406 | dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val); | |
407 | } | |
408 | } | |
409 | ||
410 | if (val & APPL_INTR_STATUS_L0_INT_INT) { | |
411 | val = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0); | |
412 | if (val & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) { | |
413 | appl_writel(pcie, | |
414 | APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS, | |
415 | APPL_INTR_STATUS_L1_8_0); | |
416 | apply_bad_link_workaround(pp); | |
417 | } | |
418 | if (val & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) { | |
419 | appl_writel(pcie, | |
420 | APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS, | |
421 | APPL_INTR_STATUS_L1_8_0); | |
422 | ||
423 | val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + | |
424 | PCI_EXP_LNKSTA); | |
425 | dev_dbg(pci->dev, "Link Speed : Gen-%u\n", val_w & | |
426 | PCI_EXP_LNKSTA_CLS); | |
427 | } | |
428 | } | |
429 | ||
430 | val = appl_readl(pcie, APPL_INTR_STATUS_L0); | |
431 | if (val & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) { | |
432 | val = appl_readl(pcie, APPL_INTR_STATUS_L1_18); | |
433 | tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS); | |
434 | if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) { | |
435 | dev_info(pci->dev, "CDM check complete\n"); | |
436 | tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE; | |
437 | } | |
438 | if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) { | |
439 | dev_err(pci->dev, "CDM comparison mismatch\n"); | |
440 | tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR; | |
441 | } | |
442 | if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) { | |
443 | dev_err(pci->dev, "CDM Logic error\n"); | |
444 | tmp |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR; | |
445 | } | |
446 | dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, tmp); | |
447 | tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR); | |
448 | dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", tmp); | |
449 | } | |
450 | ||
451 | return IRQ_HANDLED; | |
452 | } | |
453 | ||
c57247f9 VS |
454 | static void pex_ep_event_hot_rst_done(struct tegra_pcie_dw *pcie) |
455 | { | |
456 | u32 val; | |
457 | ||
458 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0); | |
459 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0); | |
460 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1); | |
461 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2); | |
462 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3); | |
463 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6); | |
464 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7); | |
465 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0); | |
466 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9); | |
467 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10); | |
468 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11); | |
469 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13); | |
470 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14); | |
471 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15); | |
472 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17); | |
473 | appl_writel(pcie, 0xFFFFFFFF, APPL_MSI_CTRL_2); | |
474 | ||
475 | val = appl_readl(pcie, APPL_CTRL); | |
476 | val |= APPL_CTRL_LTSSM_EN; | |
477 | appl_writel(pcie, val, APPL_CTRL); | |
478 | } | |
479 | ||
480 | static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg) | |
56e15a23 VS |
481 | { |
482 | struct tegra_pcie_dw *pcie = arg; | |
c57247f9 VS |
483 | struct dw_pcie *pci = &pcie->pci; |
484 | u32 val, speed; | |
485 | ||
486 | speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) & | |
487 | PCI_EXP_LNKSTA_CLS; | |
488 | clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]); | |
489 | ||
490 | /* If EP doesn't advertise L1SS, just return */ | |
491 | val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub); | |
492 | if (!(val & (PCI_L1SS_CAP_ASPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_2))) | |
493 | return IRQ_HANDLED; | |
494 | ||
495 | /* Check if BME is set to '1' */ | |
496 | val = dw_pcie_readl_dbi(pci, PCI_COMMAND); | |
497 | if (val & PCI_COMMAND_MASTER) { | |
498 | ktime_t timeout; | |
499 | ||
500 | /* 110us for both snoop and no-snoop */ | |
501 | val = 110 | (2 << PCI_LTR_SCALE_SHIFT) | LTR_MSG_REQ; | |
502 | val |= (val << LTR_MST_NO_SNOOP_SHIFT); | |
503 | appl_writel(pcie, val, APPL_LTR_MSG_1); | |
504 | ||
505 | /* Send LTR upstream */ | |
506 | val = appl_readl(pcie, APPL_LTR_MSG_2); | |
507 | val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE; | |
508 | appl_writel(pcie, val, APPL_LTR_MSG_2); | |
509 | ||
510 | timeout = ktime_add_us(ktime_get(), LTR_MSG_TIMEOUT); | |
511 | for (;;) { | |
512 | val = appl_readl(pcie, APPL_LTR_MSG_2); | |
513 | if (!(val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE)) | |
514 | break; | |
515 | if (ktime_after(ktime_get(), timeout)) | |
516 | break; | |
517 | usleep_range(1000, 1100); | |
518 | } | |
519 | if (val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE) | |
520 | dev_err(pcie->dev, "Failed to send LTR message\n"); | |
521 | } | |
56e15a23 | 522 | |
c57247f9 VS |
523 | return IRQ_HANDLED; |
524 | } | |
525 | ||
526 | static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg) | |
527 | { | |
528 | struct tegra_pcie_dw *pcie = arg; | |
529 | struct dw_pcie_ep *ep = &pcie->pci.ep; | |
530 | int spurious = 1; | |
531 | u32 val, tmp; | |
532 | ||
533 | val = appl_readl(pcie, APPL_INTR_STATUS_L0); | |
534 | if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) { | |
535 | val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0); | |
536 | appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0); | |
537 | ||
538 | if (val & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE) | |
539 | pex_ep_event_hot_rst_done(pcie); | |
540 | ||
541 | if (val & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) { | |
542 | tmp = appl_readl(pcie, APPL_LINK_STATUS); | |
543 | if (tmp & APPL_LINK_STATUS_RDLH_LINK_UP) { | |
544 | dev_dbg(pcie->dev, "Link is up with Host\n"); | |
545 | dw_pcie_ep_linkup(ep); | |
546 | } | |
547 | } | |
548 | ||
549 | spurious = 0; | |
550 | } | |
551 | ||
552 | if (val & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) { | |
553 | val = appl_readl(pcie, APPL_INTR_STATUS_L1_15); | |
554 | appl_writel(pcie, val, APPL_INTR_STATUS_L1_15); | |
555 | ||
556 | if (val & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED) | |
557 | return IRQ_WAKE_THREAD; | |
558 | ||
559 | spurious = 0; | |
560 | } | |
561 | ||
562 | if (spurious) { | |
563 | dev_warn(pcie->dev, "Random interrupt (STATUS = 0x%08X)\n", | |
564 | val); | |
565 | appl_writel(pcie, val, APPL_INTR_STATUS_L0); | |
566 | } | |
567 | ||
568 | return IRQ_HANDLED; | |
56e15a23 VS |
569 | } |
570 | ||
571 | static int tegra_pcie_dw_rd_own_conf(struct pcie_port *pp, int where, int size, | |
572 | u32 *val) | |
573 | { | |
574 | struct dw_pcie *pci = to_dw_pcie_from_pp(pp); | |
575 | ||
576 | /* | |
577 | * This is an endpoint mode specific register happen to appear even | |
578 | * when controller is operating in root port mode and system hangs | |
579 | * when it is accessed with link being in ASPM-L1 state. | |
580 | * So skip accessing it altogether | |
581 | */ | |
582 | if (where == PORT_LOGIC_MSIX_DOORBELL) { | |
583 | *val = 0x00000000; | |
584 | return PCIBIOS_SUCCESSFUL; | |
585 | } | |
586 | ||
587 | return dw_pcie_read(pci->dbi_base + where, size, val); | |
588 | } | |
589 | ||
590 | static int tegra_pcie_dw_wr_own_conf(struct pcie_port *pp, int where, int size, | |
591 | u32 val) | |
592 | { | |
593 | struct dw_pcie *pci = to_dw_pcie_from_pp(pp); | |
594 | ||
595 | /* | |
596 | * This is an endpoint mode specific register happen to appear even | |
597 | * when controller is operating in root port mode and system hangs | |
598 | * when it is accessed with link being in ASPM-L1 state. | |
599 | * So skip accessing it altogether | |
600 | */ | |
601 | if (where == PORT_LOGIC_MSIX_DOORBELL) | |
602 | return PCIBIOS_SUCCESSFUL; | |
603 | ||
604 | return dw_pcie_write(pci->dbi_base + where, size, val); | |
605 | } | |
606 | ||
607 | #if defined(CONFIG_PCIEASPM) | |
608 | static void disable_aspm_l11(struct tegra_pcie_dw *pcie) | |
609 | { | |
610 | u32 val; | |
611 | ||
612 | val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub); | |
613 | val &= ~PCI_L1SS_CAP_ASPM_L1_1; | |
614 | dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val); | |
615 | } | |
616 | ||
617 | static void disable_aspm_l12(struct tegra_pcie_dw *pcie) | |
618 | { | |
619 | u32 val; | |
620 | ||
621 | val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub); | |
622 | val &= ~PCI_L1SS_CAP_ASPM_L1_2; | |
623 | dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val); | |
624 | } | |
625 | ||
626 | static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event) | |
627 | { | |
628 | u32 val; | |
629 | ||
630 | val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid]); | |
631 | val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT); | |
632 | val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; | |
633 | val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT; | |
634 | val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; | |
635 | dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val); | |
636 | val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_data_offset[pcie->cid]); | |
637 | ||
638 | return val; | |
639 | } | |
640 | ||
641 | static int aspm_state_cnt(struct seq_file *s, void *data) | |
642 | { | |
643 | struct tegra_pcie_dw *pcie = (struct tegra_pcie_dw *) | |
644 | dev_get_drvdata(s->private); | |
645 | u32 val; | |
646 | ||
647 | seq_printf(s, "Tx L0s entry count : %u\n", | |
648 | event_counter_prog(pcie, EVENT_COUNTER_EVENT_Tx_L0S)); | |
649 | ||
650 | seq_printf(s, "Rx L0s entry count : %u\n", | |
651 | event_counter_prog(pcie, EVENT_COUNTER_EVENT_Rx_L0S)); | |
652 | ||
653 | seq_printf(s, "Link L1 entry count : %u\n", | |
654 | event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1)); | |
655 | ||
656 | seq_printf(s, "Link L1.1 entry count : %u\n", | |
657 | event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_1)); | |
658 | ||
659 | seq_printf(s, "Link L1.2 entry count : %u\n", | |
660 | event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2)); | |
661 | ||
662 | /* Clear all counters */ | |
663 | dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], | |
664 | EVENT_COUNTER_ALL_CLEAR); | |
665 | ||
666 | /* Re-enable counting */ | |
667 | val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; | |
668 | val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; | |
669 | dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val); | |
670 | ||
671 | return 0; | |
672 | } | |
673 | ||
674 | static void init_host_aspm(struct tegra_pcie_dw *pcie) | |
675 | { | |
676 | struct dw_pcie *pci = &pcie->pci; | |
677 | u32 val; | |
678 | ||
679 | val = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS); | |
680 | pcie->cfg_link_cap_l1sub = val + PCI_L1SS_CAP; | |
681 | ||
682 | /* Enable ASPM counters */ | |
683 | val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; | |
684 | val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; | |
685 | dw_pcie_writel_dbi(pci, event_cntr_ctrl_offset[pcie->cid], val); | |
686 | ||
687 | /* Program T_cmrt and T_pwr_on values */ | |
688 | val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub); | |
689 | val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE); | |
690 | val |= (pcie->aspm_cmrt << 8); | |
691 | val |= (pcie->aspm_pwr_on_t << 19); | |
692 | dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val); | |
693 | ||
694 | /* Program L0s and L1 entrance latencies */ | |
695 | val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL); | |
696 | val &= ~L0S_ENTRANCE_LAT_MASK; | |
697 | val |= (pcie->aspm_l0s_enter_lat << L0S_ENTRANCE_LAT_SHIFT); | |
698 | val |= ENTER_ASPM; | |
699 | dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val); | |
700 | } | |
701 | ||
702 | static int init_debugfs(struct tegra_pcie_dw *pcie) | |
703 | { | |
704 | struct dentry *d; | |
705 | ||
706 | d = debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt", | |
707 | pcie->debugfs, aspm_state_cnt); | |
708 | if (IS_ERR_OR_NULL(d)) | |
709 | dev_err(pcie->dev, | |
710 | "Failed to create debugfs file \"aspm_state_cnt\"\n"); | |
711 | ||
712 | return 0; | |
713 | } | |
714 | #else | |
715 | static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; } | |
716 | static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; } | |
717 | static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; } | |
718 | static inline int init_debugfs(struct tegra_pcie_dw *pcie) { return 0; } | |
719 | #endif | |
720 | ||
721 | static void tegra_pcie_enable_system_interrupts(struct pcie_port *pp) | |
722 | { | |
723 | struct dw_pcie *pci = to_dw_pcie_from_pp(pp); | |
724 | struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); | |
725 | u32 val; | |
726 | u16 val_w; | |
727 | ||
728 | val = appl_readl(pcie, APPL_INTR_EN_L0_0); | |
729 | val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN; | |
730 | appl_writel(pcie, val, APPL_INTR_EN_L0_0); | |
731 | ||
732 | val = appl_readl(pcie, APPL_INTR_EN_L1_0_0); | |
733 | val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN; | |
734 | appl_writel(pcie, val, APPL_INTR_EN_L1_0_0); | |
735 | ||
736 | if (pcie->enable_cdm_check) { | |
737 | val = appl_readl(pcie, APPL_INTR_EN_L0_0); | |
738 | val |= APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN; | |
739 | appl_writel(pcie, val, APPL_INTR_EN_L0_0); | |
740 | ||
741 | val = appl_readl(pcie, APPL_INTR_EN_L1_18); | |
742 | val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR; | |
743 | val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR; | |
744 | appl_writel(pcie, val, APPL_INTR_EN_L1_18); | |
745 | } | |
746 | ||
747 | val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base + | |
748 | PCI_EXP_LNKSTA); | |
749 | pcie->init_link_width = (val_w & PCI_EXP_LNKSTA_NLW) >> | |
750 | PCI_EXP_LNKSTA_NLW_SHIFT; | |
751 | ||
752 | val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base + | |
753 | PCI_EXP_LNKCTL); | |
754 | val_w |= PCI_EXP_LNKCTL_LBMIE; | |
755 | dw_pcie_writew_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL, | |
756 | val_w); | |
757 | } | |
758 | ||
759 | static void tegra_pcie_enable_legacy_interrupts(struct pcie_port *pp) | |
760 | { | |
761 | struct dw_pcie *pci = to_dw_pcie_from_pp(pp); | |
762 | struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); | |
763 | u32 val; | |
764 | ||
765 | /* Enable legacy interrupt generation */ | |
766 | val = appl_readl(pcie, APPL_INTR_EN_L0_0); | |
767 | val |= APPL_INTR_EN_L0_0_SYS_INTR_EN; | |
768 | val |= APPL_INTR_EN_L0_0_INT_INT_EN; | |
769 | appl_writel(pcie, val, APPL_INTR_EN_L0_0); | |
770 | ||
771 | val = appl_readl(pcie, APPL_INTR_EN_L1_8_0); | |
772 | val |= APPL_INTR_EN_L1_8_INTX_EN; | |
773 | val |= APPL_INTR_EN_L1_8_AUTO_BW_INT_EN; | |
774 | val |= APPL_INTR_EN_L1_8_BW_MGT_INT_EN; | |
775 | if (IS_ENABLED(CONFIG_PCIEAER)) | |
776 | val |= APPL_INTR_EN_L1_8_AER_INT_EN; | |
777 | appl_writel(pcie, val, APPL_INTR_EN_L1_8_0); | |
778 | } | |
779 | ||
780 | static void tegra_pcie_enable_msi_interrupts(struct pcie_port *pp) | |
781 | { | |
782 | struct dw_pcie *pci = to_dw_pcie_from_pp(pp); | |
783 | struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); | |
784 | u32 val; | |
785 | ||
786 | dw_pcie_msi_init(pp); | |
787 | ||
788 | /* Enable MSI interrupt generation */ | |
789 | val = appl_readl(pcie, APPL_INTR_EN_L0_0); | |
790 | val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN; | |
791 | val |= APPL_INTR_EN_L0_0_MSI_RCV_INT_EN; | |
792 | appl_writel(pcie, val, APPL_INTR_EN_L0_0); | |
793 | } | |
794 | ||
795 | static void tegra_pcie_enable_interrupts(struct pcie_port *pp) | |
796 | { | |
797 | struct dw_pcie *pci = to_dw_pcie_from_pp(pp); | |
798 | struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); | |
799 | ||
800 | /* Clear interrupt statuses before enabling interrupts */ | |
801 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0); | |
802 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0); | |
803 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1); | |
804 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2); | |
805 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3); | |
806 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6); | |
807 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7); | |
808 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0); | |
809 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9); | |
810 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10); | |
811 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11); | |
812 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13); | |
813 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14); | |
814 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15); | |
815 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17); | |
816 | ||
817 | tegra_pcie_enable_system_interrupts(pp); | |
818 | tegra_pcie_enable_legacy_interrupts(pp); | |
819 | if (IS_ENABLED(CONFIG_PCI_MSI)) | |
820 | tegra_pcie_enable_msi_interrupts(pp); | |
821 | } | |
822 | ||
823 | static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie) | |
824 | { | |
825 | struct dw_pcie *pci = &pcie->pci; | |
826 | u32 val, offset, i; | |
827 | ||
828 | /* Program init preset */ | |
829 | for (i = 0; i < pcie->num_lanes; i++) { | |
830 | dw_pcie_read(pci->dbi_base + CAP_SPCIE_CAP_OFF | |
831 | + (i * 2), 2, &val); | |
832 | val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK; | |
833 | val |= GEN3_GEN4_EQ_PRESET_INIT; | |
834 | val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK; | |
835 | val |= (GEN3_GEN4_EQ_PRESET_INIT << | |
836 | CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT); | |
837 | dw_pcie_write(pci->dbi_base + CAP_SPCIE_CAP_OFF | |
838 | + (i * 2), 2, val); | |
839 | ||
840 | offset = dw_pcie_find_ext_capability(pci, | |
841 | PCI_EXT_CAP_ID_PL_16GT) + | |
842 | PCI_PL_16GT_LE_CTRL; | |
843 | dw_pcie_read(pci->dbi_base + offset + i, 1, &val); | |
844 | val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK; | |
845 | val |= GEN3_GEN4_EQ_PRESET_INIT; | |
846 | val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK; | |
847 | val |= (GEN3_GEN4_EQ_PRESET_INIT << | |
848 | PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT); | |
849 | dw_pcie_write(pci->dbi_base + offset + i, 1, val); | |
850 | } | |
851 | ||
852 | val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); | |
853 | val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; | |
854 | dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); | |
855 | ||
856 | val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF); | |
857 | val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK; | |
858 | val |= (0x3ff << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT); | |
859 | val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK; | |
860 | dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val); | |
861 | ||
862 | val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); | |
863 | val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; | |
864 | val |= (0x1 << GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT); | |
865 | dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); | |
866 | ||
867 | val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF); | |
868 | val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK; | |
869 | val |= (0x360 << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT); | |
870 | val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK; | |
871 | dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val); | |
872 | ||
873 | val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); | |
874 | val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; | |
875 | dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); | |
876 | } | |
877 | ||
878 | static void tegra_pcie_prepare_host(struct pcie_port *pp) | |
879 | { | |
880 | struct dw_pcie *pci = to_dw_pcie_from_pp(pp); | |
881 | struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); | |
882 | u32 val; | |
883 | ||
884 | val = dw_pcie_readl_dbi(pci, PCI_IO_BASE); | |
885 | val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8); | |
886 | dw_pcie_writel_dbi(pci, PCI_IO_BASE, val); | |
887 | ||
888 | val = dw_pcie_readl_dbi(pci, PCI_PREF_MEMORY_BASE); | |
889 | val |= CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE; | |
890 | val |= CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE; | |
891 | dw_pcie_writel_dbi(pci, PCI_PREF_MEMORY_BASE, val); | |
892 | ||
893 | dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0); | |
894 | ||
895 | /* Configure FTS */ | |
896 | val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL); | |
897 | val &= ~(N_FTS_MASK << N_FTS_SHIFT); | |
898 | val |= N_FTS_VAL << N_FTS_SHIFT; | |
899 | dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val); | |
900 | ||
901 | val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL); | |
902 | val &= ~FTS_MASK; | |
903 | val |= FTS_VAL; | |
904 | dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val); | |
905 | ||
906 | /* Enable as 0xFFFF0001 response for CRS */ | |
907 | val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT); | |
908 | val &= ~(AMBA_ERROR_RESPONSE_CRS_MASK << AMBA_ERROR_RESPONSE_CRS_SHIFT); | |
909 | val |= (AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 << | |
910 | AMBA_ERROR_RESPONSE_CRS_SHIFT); | |
911 | dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val); | |
912 | ||
913 | /* Configure Max Speed from DT */ | |
914 | if (pcie->max_speed && pcie->max_speed != -EINVAL) { | |
915 | val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + | |
916 | PCI_EXP_LNKCAP); | |
917 | val &= ~PCI_EXP_LNKCAP_SLS; | |
918 | val |= pcie->max_speed; | |
919 | dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, | |
920 | val); | |
921 | } | |
922 | ||
923 | /* Configure Max lane width from DT */ | |
924 | val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP); | |
925 | val &= ~PCI_EXP_LNKCAP_MLW; | |
926 | val |= (pcie->num_lanes << PCI_EXP_LNKSTA_NLW_SHIFT); | |
927 | dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, val); | |
928 | ||
929 | config_gen3_gen4_eq_presets(pcie); | |
930 | ||
931 | init_host_aspm(pcie); | |
932 | ||
933 | val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); | |
934 | val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL; | |
935 | dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); | |
936 | ||
937 | if (pcie->update_fc_fixup) { | |
938 | val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF); | |
939 | val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT; | |
940 | dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val); | |
941 | } | |
942 | ||
943 | dw_pcie_setup_rc(pp); | |
944 | ||
945 | clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ); | |
946 | ||
947 | /* Assert RST */ | |
948 | val = appl_readl(pcie, APPL_PINMUX); | |
949 | val &= ~APPL_PINMUX_PEX_RST; | |
950 | appl_writel(pcie, val, APPL_PINMUX); | |
951 | ||
952 | usleep_range(100, 200); | |
953 | ||
954 | /* Enable LTSSM */ | |
955 | val = appl_readl(pcie, APPL_CTRL); | |
956 | val |= APPL_CTRL_LTSSM_EN; | |
957 | appl_writel(pcie, val, APPL_CTRL); | |
958 | ||
959 | /* De-assert RST */ | |
960 | val = appl_readl(pcie, APPL_PINMUX); | |
961 | val |= APPL_PINMUX_PEX_RST; | |
962 | appl_writel(pcie, val, APPL_PINMUX); | |
963 | ||
964 | msleep(100); | |
965 | } | |
966 | ||
967 | static int tegra_pcie_dw_host_init(struct pcie_port *pp) | |
968 | { | |
969 | struct dw_pcie *pci = to_dw_pcie_from_pp(pp); | |
970 | struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); | |
971 | u32 val, tmp, offset, speed; | |
972 | ||
973 | tegra_pcie_prepare_host(pp); | |
974 | ||
975 | if (dw_pcie_wait_for_link(pci)) { | |
976 | /* | |
977 | * There are some endpoints which can't get the link up if | |
978 | * root port has Data Link Feature (DLF) enabled. | |
979 | * Refer Spec rev 4.0 ver 1.0 sec 3.4.2 & 7.7.4 for more info | |
980 | * on Scaled Flow Control and DLF. | |
981 | * So, need to confirm that is indeed the case here and attempt | |
982 | * link up once again with DLF disabled. | |
983 | */ | |
984 | val = appl_readl(pcie, APPL_DEBUG); | |
985 | val &= APPL_DEBUG_LTSSM_STATE_MASK; | |
986 | val >>= APPL_DEBUG_LTSSM_STATE_SHIFT; | |
987 | tmp = appl_readl(pcie, APPL_LINK_STATUS); | |
988 | tmp &= APPL_LINK_STATUS_RDLH_LINK_UP; | |
989 | if (!(val == 0x11 && !tmp)) { | |
990 | /* Link is down for all good reasons */ | |
991 | return 0; | |
992 | } | |
993 | ||
994 | dev_info(pci->dev, "Link is down in DLL"); | |
995 | dev_info(pci->dev, "Trying again with DLFE disabled\n"); | |
996 | /* Disable LTSSM */ | |
997 | val = appl_readl(pcie, APPL_CTRL); | |
998 | val &= ~APPL_CTRL_LTSSM_EN; | |
999 | appl_writel(pcie, val, APPL_CTRL); | |
1000 | ||
1001 | reset_control_assert(pcie->core_rst); | |
1002 | reset_control_deassert(pcie->core_rst); | |
1003 | ||
1004 | offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF); | |
1005 | val = dw_pcie_readl_dbi(pci, offset + PCI_DLF_CAP); | |
1006 | val &= ~PCI_DLF_EXCHANGE_ENABLE; | |
1007 | dw_pcie_writel_dbi(pci, offset, val); | |
1008 | ||
1009 | tegra_pcie_prepare_host(pp); | |
1010 | ||
1011 | if (dw_pcie_wait_for_link(pci)) | |
1012 | return 0; | |
1013 | } | |
1014 | ||
1015 | speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) & | |
1016 | PCI_EXP_LNKSTA_CLS; | |
1017 | clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]); | |
1018 | ||
1019 | tegra_pcie_enable_interrupts(pp); | |
1020 | ||
1021 | return 0; | |
1022 | } | |
1023 | ||
1024 | static int tegra_pcie_dw_link_up(struct dw_pcie *pci) | |
1025 | { | |
1026 | struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); | |
1027 | u32 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA); | |
1028 | ||
1029 | return !!(val & PCI_EXP_LNKSTA_DLLLA); | |
1030 | } | |
1031 | ||
1032 | static void tegra_pcie_set_msi_vec_num(struct pcie_port *pp) | |
1033 | { | |
1034 | pp->num_vectors = MAX_MSI_IRQS; | |
1035 | } | |
1036 | ||
c57247f9 VS |
1037 | static int tegra_pcie_dw_start_link(struct dw_pcie *pci) |
1038 | { | |
1039 | struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); | |
1040 | ||
1041 | enable_irq(pcie->pex_rst_irq); | |
1042 | ||
1043 | return 0; | |
1044 | } | |
1045 | ||
1046 | static void tegra_pcie_dw_stop_link(struct dw_pcie *pci) | |
1047 | { | |
1048 | struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); | |
1049 | ||
1050 | disable_irq(pcie->pex_rst_irq); | |
1051 | } | |
1052 | ||
56e15a23 VS |
1053 | static const struct dw_pcie_ops tegra_dw_pcie_ops = { |
1054 | .link_up = tegra_pcie_dw_link_up, | |
c57247f9 VS |
1055 | .start_link = tegra_pcie_dw_start_link, |
1056 | .stop_link = tegra_pcie_dw_stop_link, | |
56e15a23 VS |
1057 | }; |
1058 | ||
1059 | static struct dw_pcie_host_ops tegra_pcie_dw_host_ops = { | |
1060 | .rd_own_conf = tegra_pcie_dw_rd_own_conf, | |
1061 | .wr_own_conf = tegra_pcie_dw_wr_own_conf, | |
1062 | .host_init = tegra_pcie_dw_host_init, | |
1063 | .set_num_vectors = tegra_pcie_set_msi_vec_num, | |
1064 | }; | |
1065 | ||
1066 | static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie) | |
1067 | { | |
1068 | unsigned int phy_count = pcie->phy_count; | |
1069 | ||
1070 | while (phy_count--) { | |
1071 | phy_power_off(pcie->phys[phy_count]); | |
1072 | phy_exit(pcie->phys[phy_count]); | |
1073 | } | |
1074 | } | |
1075 | ||
1076 | static int tegra_pcie_enable_phy(struct tegra_pcie_dw *pcie) | |
1077 | { | |
1078 | unsigned int i; | |
1079 | int ret; | |
1080 | ||
1081 | for (i = 0; i < pcie->phy_count; i++) { | |
1082 | ret = phy_init(pcie->phys[i]); | |
1083 | if (ret < 0) | |
1084 | goto phy_power_off; | |
1085 | ||
1086 | ret = phy_power_on(pcie->phys[i]); | |
1087 | if (ret < 0) | |
1088 | goto phy_exit; | |
1089 | } | |
1090 | ||
1091 | return 0; | |
1092 | ||
1093 | phy_power_off: | |
1094 | while (i--) { | |
1095 | phy_power_off(pcie->phys[i]); | |
1096 | phy_exit: | |
1097 | phy_exit(pcie->phys[i]); | |
1098 | } | |
1099 | ||
1100 | return ret; | |
1101 | } | |
1102 | ||
1103 | static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie) | |
1104 | { | |
1105 | struct device_node *np = pcie->dev->of_node; | |
1106 | int ret; | |
1107 | ||
1108 | ret = of_property_read_u32(np, "nvidia,aspm-cmrt-us", &pcie->aspm_cmrt); | |
1109 | if (ret < 0) { | |
1110 | dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret); | |
1111 | return ret; | |
1112 | } | |
1113 | ||
1114 | ret = of_property_read_u32(np, "nvidia,aspm-pwr-on-t-us", | |
1115 | &pcie->aspm_pwr_on_t); | |
1116 | if (ret < 0) | |
1117 | dev_info(pcie->dev, "Failed to read ASPM Power On time: %d\n", | |
1118 | ret); | |
1119 | ||
1120 | ret = of_property_read_u32(np, "nvidia,aspm-l0s-entrance-latency-us", | |
1121 | &pcie->aspm_l0s_enter_lat); | |
1122 | if (ret < 0) | |
1123 | dev_info(pcie->dev, | |
1124 | "Failed to read ASPM L0s Entrance latency: %d\n", ret); | |
1125 | ||
1126 | ret = of_property_read_u32(np, "num-lanes", &pcie->num_lanes); | |
1127 | if (ret < 0) { | |
1128 | dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret); | |
1129 | return ret; | |
1130 | } | |
1131 | ||
1132 | pcie->max_speed = of_pci_get_max_link_speed(np); | |
1133 | ||
1134 | ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid); | |
1135 | if (ret) { | |
1136 | dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret); | |
1137 | return ret; | |
1138 | } | |
1139 | ||
1140 | ret = of_property_count_strings(np, "phy-names"); | |
1141 | if (ret < 0) { | |
1142 | dev_err(pcie->dev, "Failed to find PHY entries: %d\n", | |
1143 | ret); | |
1144 | return ret; | |
1145 | } | |
1146 | pcie->phy_count = ret; | |
1147 | ||
1148 | if (of_property_read_bool(np, "nvidia,update-fc-fixup")) | |
1149 | pcie->update_fc_fixup = true; | |
1150 | ||
1151 | pcie->supports_clkreq = | |
1152 | of_property_read_bool(pcie->dev->of_node, "supports-clkreq"); | |
1153 | ||
1154 | pcie->enable_cdm_check = | |
1155 | of_property_read_bool(np, "snps,enable-cdm-check"); | |
1156 | ||
c57247f9 VS |
1157 | if (pcie->mode == DW_PCIE_RC_TYPE) |
1158 | return 0; | |
1159 | ||
1160 | /* Endpoint mode specific DT entries */ | |
1161 | pcie->pex_rst_gpiod = devm_gpiod_get(pcie->dev, "reset", GPIOD_IN); | |
1162 | if (IS_ERR(pcie->pex_rst_gpiod)) { | |
54455238 TR |
1163 | int err = PTR_ERR(pcie->pex_rst_gpiod); |
1164 | const char *level = KERN_ERR; | |
1165 | ||
1166 | if (err == -EPROBE_DEFER) | |
1167 | level = KERN_DEBUG; | |
1168 | ||
1169 | dev_printk(level, pcie->dev, | |
1170 | dev_fmt("Failed to get PERST GPIO: %d\n"), | |
1171 | err); | |
1172 | return err; | |
c57247f9 VS |
1173 | } |
1174 | ||
1175 | pcie->pex_refclk_sel_gpiod = devm_gpiod_get(pcie->dev, | |
1176 | "nvidia,refclk-select", | |
1177 | GPIOD_OUT_HIGH); | |
1178 | if (IS_ERR(pcie->pex_refclk_sel_gpiod)) { | |
54455238 TR |
1179 | int err = PTR_ERR(pcie->pex_refclk_sel_gpiod); |
1180 | const char *level = KERN_ERR; | |
1181 | ||
1182 | if (err == -EPROBE_DEFER) | |
1183 | level = KERN_DEBUG; | |
1184 | ||
1185 | dev_printk(level, pcie->dev, | |
1186 | dev_fmt("Failed to get REFCLK select GPIOs: %d\n"), | |
1187 | err); | |
c57247f9 VS |
1188 | pcie->pex_refclk_sel_gpiod = NULL; |
1189 | } | |
1190 | ||
56e15a23 VS |
1191 | return 0; |
1192 | } | |
1193 | ||
1194 | static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie, | |
1195 | bool enable) | |
1196 | { | |
1197 | struct mrq_uphy_response resp; | |
1198 | struct tegra_bpmp_message msg; | |
1199 | struct mrq_uphy_request req; | |
1200 | ||
1201 | /* Controller-5 doesn't need to have its state set by BPMP-FW */ | |
1202 | if (pcie->cid == 5) | |
1203 | return 0; | |
1204 | ||
1205 | memset(&req, 0, sizeof(req)); | |
1206 | memset(&resp, 0, sizeof(resp)); | |
1207 | ||
1208 | req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE; | |
1209 | req.controller_state.pcie_controller = pcie->cid; | |
1210 | req.controller_state.enable = enable; | |
1211 | ||
1212 | memset(&msg, 0, sizeof(msg)); | |
1213 | msg.mrq = MRQ_UPHY; | |
1214 | msg.tx.data = &req; | |
1215 | msg.tx.size = sizeof(req); | |
1216 | msg.rx.data = &resp; | |
1217 | msg.rx.size = sizeof(resp); | |
1218 | ||
1219 | return tegra_bpmp_transfer(pcie->bpmp, &msg); | |
1220 | } | |
1221 | ||
c57247f9 VS |
1222 | static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie, |
1223 | bool enable) | |
1224 | { | |
1225 | struct mrq_uphy_response resp; | |
1226 | struct tegra_bpmp_message msg; | |
1227 | struct mrq_uphy_request req; | |
1228 | ||
1229 | memset(&req, 0, sizeof(req)); | |
1230 | memset(&resp, 0, sizeof(resp)); | |
1231 | ||
1232 | if (enable) { | |
1233 | req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_INIT; | |
1234 | req.ep_ctrlr_pll_init.ep_controller = pcie->cid; | |
1235 | } else { | |
1236 | req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_OFF; | |
1237 | req.ep_ctrlr_pll_off.ep_controller = pcie->cid; | |
1238 | } | |
1239 | ||
1240 | memset(&msg, 0, sizeof(msg)); | |
1241 | msg.mrq = MRQ_UPHY; | |
1242 | msg.tx.data = &req; | |
1243 | msg.tx.size = sizeof(req); | |
1244 | msg.rx.data = &resp; | |
1245 | msg.rx.size = sizeof(resp); | |
1246 | ||
1247 | return tegra_bpmp_transfer(pcie->bpmp, &msg); | |
1248 | } | |
1249 | ||
56e15a23 VS |
1250 | static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie) |
1251 | { | |
1252 | struct pcie_port *pp = &pcie->pci.pp; | |
1253 | struct pci_bus *child, *root_bus = NULL; | |
1254 | struct pci_dev *pdev; | |
1255 | ||
1256 | /* | |
1257 | * link doesn't go into L2 state with some of the endpoints with Tegra | |
1258 | * if they are not in D0 state. So, need to make sure that immediate | |
1259 | * downstream devices are in D0 state before sending PME_TurnOff to put | |
1260 | * link into L2 state. | |
1261 | * This is as per PCI Express Base r4.0 v1.0 September 27-2017, | |
1262 | * 5.2 Link State Power Management (Page #428). | |
1263 | */ | |
1264 | ||
1265 | list_for_each_entry(child, &pp->root_bus->children, node) { | |
1266 | /* Bring downstream devices to D0 if they are not already in */ | |
1267 | if (child->parent == pp->root_bus) { | |
1268 | root_bus = child; | |
1269 | break; | |
1270 | } | |
1271 | } | |
1272 | ||
1273 | if (!root_bus) { | |
1274 | dev_err(pcie->dev, "Failed to find downstream devices\n"); | |
1275 | return; | |
1276 | } | |
1277 | ||
1278 | list_for_each_entry(pdev, &root_bus->devices, bus_list) { | |
1279 | if (PCI_SLOT(pdev->devfn) == 0) { | |
1280 | if (pci_set_power_state(pdev, PCI_D0)) | |
1281 | dev_err(pcie->dev, | |
1282 | "Failed to transition %s to D0 state\n", | |
1283 | dev_name(&pdev->dev)); | |
1284 | } | |
1285 | } | |
1286 | } | |
1287 | ||
0a901f21 VS |
1288 | static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie) |
1289 | { | |
1290 | pcie->slot_ctl_3v3 = devm_regulator_get_optional(pcie->dev, "vpcie3v3"); | |
1291 | if (IS_ERR(pcie->slot_ctl_3v3)) { | |
1292 | if (PTR_ERR(pcie->slot_ctl_3v3) != -ENODEV) | |
1293 | return PTR_ERR(pcie->slot_ctl_3v3); | |
1294 | ||
1295 | pcie->slot_ctl_3v3 = NULL; | |
1296 | } | |
1297 | ||
1298 | pcie->slot_ctl_12v = devm_regulator_get_optional(pcie->dev, "vpcie12v"); | |
1299 | if (IS_ERR(pcie->slot_ctl_12v)) { | |
1300 | if (PTR_ERR(pcie->slot_ctl_12v) != -ENODEV) | |
1301 | return PTR_ERR(pcie->slot_ctl_12v); | |
1302 | ||
1303 | pcie->slot_ctl_12v = NULL; | |
1304 | } | |
1305 | ||
1306 | return 0; | |
1307 | } | |
1308 | ||
1309 | static int tegra_pcie_enable_slot_regulators(struct tegra_pcie_dw *pcie) | |
1310 | { | |
1311 | int ret; | |
1312 | ||
1313 | if (pcie->slot_ctl_3v3) { | |
1314 | ret = regulator_enable(pcie->slot_ctl_3v3); | |
1315 | if (ret < 0) { | |
1316 | dev_err(pcie->dev, | |
1317 | "Failed to enable 3.3V slot supply: %d\n", ret); | |
1318 | return ret; | |
1319 | } | |
1320 | } | |
1321 | ||
1322 | if (pcie->slot_ctl_12v) { | |
1323 | ret = regulator_enable(pcie->slot_ctl_12v); | |
1324 | if (ret < 0) { | |
1325 | dev_err(pcie->dev, | |
1326 | "Failed to enable 12V slot supply: %d\n", ret); | |
1327 | goto fail_12v_enable; | |
1328 | } | |
1329 | } | |
1330 | ||
1331 | /* | |
1332 | * According to PCI Express Card Electromechanical Specification | |
1333 | * Revision 1.1, Table-2.4, T_PVPERL (Power stable to PERST# inactive) | |
1334 | * should be a minimum of 100ms. | |
1335 | */ | |
1336 | if (pcie->slot_ctl_3v3 || pcie->slot_ctl_12v) | |
1337 | msleep(100); | |
1338 | ||
1339 | return 0; | |
1340 | ||
1341 | fail_12v_enable: | |
1342 | if (pcie->slot_ctl_3v3) | |
1343 | regulator_disable(pcie->slot_ctl_3v3); | |
1344 | return ret; | |
1345 | } | |
1346 | ||
1347 | static void tegra_pcie_disable_slot_regulators(struct tegra_pcie_dw *pcie) | |
1348 | { | |
1349 | if (pcie->slot_ctl_12v) | |
1350 | regulator_disable(pcie->slot_ctl_12v); | |
1351 | if (pcie->slot_ctl_3v3) | |
1352 | regulator_disable(pcie->slot_ctl_3v3); | |
1353 | } | |
1354 | ||
56e15a23 VS |
1355 | static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie, |
1356 | bool en_hw_hot_rst) | |
1357 | { | |
1358 | int ret; | |
1359 | u32 val; | |
1360 | ||
1361 | ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true); | |
1362 | if (ret) { | |
1363 | dev_err(pcie->dev, | |
1364 | "Failed to enable controller %u: %d\n", pcie->cid, ret); | |
1365 | return ret; | |
1366 | } | |
1367 | ||
0a901f21 VS |
1368 | ret = tegra_pcie_enable_slot_regulators(pcie); |
1369 | if (ret < 0) | |
1370 | goto fail_slot_reg_en; | |
1371 | ||
56e15a23 VS |
1372 | ret = regulator_enable(pcie->pex_ctl_supply); |
1373 | if (ret < 0) { | |
1374 | dev_err(pcie->dev, "Failed to enable regulator: %d\n", ret); | |
1375 | goto fail_reg_en; | |
1376 | } | |
1377 | ||
1378 | ret = clk_prepare_enable(pcie->core_clk); | |
1379 | if (ret) { | |
1380 | dev_err(pcie->dev, "Failed to enable core clock: %d\n", ret); | |
1381 | goto fail_core_clk; | |
1382 | } | |
1383 | ||
1384 | ret = reset_control_deassert(pcie->core_apb_rst); | |
1385 | if (ret) { | |
1386 | dev_err(pcie->dev, "Failed to deassert core APB reset: %d\n", | |
1387 | ret); | |
1388 | goto fail_core_apb_rst; | |
1389 | } | |
1390 | ||
1391 | if (en_hw_hot_rst) { | |
1392 | /* Enable HW_HOT_RST mode */ | |
1393 | val = appl_readl(pcie, APPL_CTRL); | |
1394 | val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << | |
1395 | APPL_CTRL_HW_HOT_RST_MODE_SHIFT); | |
1396 | val |= APPL_CTRL_HW_HOT_RST_EN; | |
1397 | appl_writel(pcie, val, APPL_CTRL); | |
1398 | } | |
1399 | ||
1400 | ret = tegra_pcie_enable_phy(pcie); | |
1401 | if (ret) { | |
1402 | dev_err(pcie->dev, "Failed to enable PHY: %d\n", ret); | |
1403 | goto fail_phy; | |
1404 | } | |
1405 | ||
1406 | /* Update CFG base address */ | |
1407 | appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK, | |
1408 | APPL_CFG_BASE_ADDR); | |
1409 | ||
1410 | /* Configure this core for RP mode operation */ | |
1411 | appl_writel(pcie, APPL_DM_TYPE_RP, APPL_DM_TYPE); | |
1412 | ||
1413 | appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE); | |
1414 | ||
1415 | val = appl_readl(pcie, APPL_CTRL); | |
1416 | appl_writel(pcie, val | APPL_CTRL_SYS_PRE_DET_STATE, APPL_CTRL); | |
1417 | ||
1418 | val = appl_readl(pcie, APPL_CFG_MISC); | |
1419 | val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT); | |
1420 | appl_writel(pcie, val, APPL_CFG_MISC); | |
1421 | ||
1422 | if (!pcie->supports_clkreq) { | |
1423 | val = appl_readl(pcie, APPL_PINMUX); | |
ff5c2bb9 VS |
1424 | val |= APPL_PINMUX_CLKREQ_OVERRIDE_EN; |
1425 | val &= ~APPL_PINMUX_CLKREQ_OVERRIDE; | |
56e15a23 VS |
1426 | appl_writel(pcie, val, APPL_PINMUX); |
1427 | } | |
1428 | ||
1429 | /* Update iATU_DMA base address */ | |
1430 | appl_writel(pcie, | |
1431 | pcie->atu_dma_res->start & APPL_CFG_IATU_DMA_BASE_ADDR_MASK, | |
1432 | APPL_CFG_IATU_DMA_BASE_ADDR); | |
1433 | ||
1434 | reset_control_deassert(pcie->core_rst); | |
1435 | ||
1436 | pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci, | |
1437 | PCI_CAP_ID_EXP); | |
1438 | ||
1439 | /* Disable ASPM-L1SS advertisement as there is no CLKREQ routing */ | |
1440 | if (!pcie->supports_clkreq) { | |
1441 | disable_aspm_l11(pcie); | |
1442 | disable_aspm_l12(pcie); | |
1443 | } | |
1444 | ||
1445 | return ret; | |
1446 | ||
1447 | fail_phy: | |
1448 | reset_control_assert(pcie->core_apb_rst); | |
1449 | fail_core_apb_rst: | |
1450 | clk_disable_unprepare(pcie->core_clk); | |
1451 | fail_core_clk: | |
1452 | regulator_disable(pcie->pex_ctl_supply); | |
1453 | fail_reg_en: | |
0a901f21 VS |
1454 | tegra_pcie_disable_slot_regulators(pcie); |
1455 | fail_slot_reg_en: | |
56e15a23 VS |
1456 | tegra_pcie_bpmp_set_ctrl_state(pcie, false); |
1457 | ||
1458 | return ret; | |
1459 | } | |
1460 | ||
1461 | static int __deinit_controller(struct tegra_pcie_dw *pcie) | |
1462 | { | |
1463 | int ret; | |
1464 | ||
1465 | ret = reset_control_assert(pcie->core_rst); | |
1466 | if (ret) { | |
1467 | dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n", | |
1468 | ret); | |
1469 | return ret; | |
1470 | } | |
1471 | ||
1472 | tegra_pcie_disable_phy(pcie); | |
1473 | ||
1474 | ret = reset_control_assert(pcie->core_apb_rst); | |
1475 | if (ret) { | |
1476 | dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret); | |
1477 | return ret; | |
1478 | } | |
1479 | ||
1480 | clk_disable_unprepare(pcie->core_clk); | |
1481 | ||
1482 | ret = regulator_disable(pcie->pex_ctl_supply); | |
1483 | if (ret) { | |
1484 | dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret); | |
1485 | return ret; | |
1486 | } | |
1487 | ||
0a901f21 VS |
1488 | tegra_pcie_disable_slot_regulators(pcie); |
1489 | ||
56e15a23 VS |
1490 | ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false); |
1491 | if (ret) { | |
1492 | dev_err(pcie->dev, "Failed to disable controller %d: %d\n", | |
1493 | pcie->cid, ret); | |
1494 | return ret; | |
1495 | } | |
1496 | ||
1497 | return ret; | |
1498 | } | |
1499 | ||
1500 | static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie) | |
1501 | { | |
1502 | struct dw_pcie *pci = &pcie->pci; | |
1503 | struct pcie_port *pp = &pci->pp; | |
1504 | int ret; | |
1505 | ||
1506 | ret = tegra_pcie_config_controller(pcie, false); | |
1507 | if (ret < 0) | |
1508 | return ret; | |
1509 | ||
1510 | pp->ops = &tegra_pcie_dw_host_ops; | |
1511 | ||
1512 | ret = dw_pcie_host_init(pp); | |
1513 | if (ret < 0) { | |
1514 | dev_err(pcie->dev, "Failed to add PCIe port: %d\n", ret); | |
1515 | goto fail_host_init; | |
1516 | } | |
1517 | ||
1518 | return 0; | |
1519 | ||
1520 | fail_host_init: | |
1521 | return __deinit_controller(pcie); | |
1522 | } | |
1523 | ||
1524 | static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie) | |
1525 | { | |
1526 | u32 val; | |
1527 | ||
1528 | if (!tegra_pcie_dw_link_up(&pcie->pci)) | |
1529 | return 0; | |
1530 | ||
1531 | val = appl_readl(pcie, APPL_RADM_STATUS); | |
1532 | val |= APPL_PM_XMT_TURNOFF_STATE; | |
1533 | appl_writel(pcie, val, APPL_RADM_STATUS); | |
1534 | ||
1535 | return readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, val, | |
1536 | val & APPL_DEBUG_PM_LINKST_IN_L2_LAT, | |
1537 | 1, PME_ACK_TIMEOUT); | |
1538 | } | |
1539 | ||
1540 | static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie) | |
1541 | { | |
1542 | u32 data; | |
1543 | int err; | |
1544 | ||
1545 | if (!tegra_pcie_dw_link_up(&pcie->pci)) { | |
1546 | dev_dbg(pcie->dev, "PCIe link is not up...!\n"); | |
1547 | return; | |
1548 | } | |
1549 | ||
1550 | if (tegra_pcie_try_link_l2(pcie)) { | |
1551 | dev_info(pcie->dev, "Link didn't transition to L2 state\n"); | |
1552 | /* | |
1553 | * TX lane clock freq will reset to Gen1 only if link is in L2 | |
1554 | * or detect state. | |
1555 | * So apply pex_rst to end point to force RP to go into detect | |
1556 | * state | |
1557 | */ | |
1558 | data = appl_readl(pcie, APPL_PINMUX); | |
1559 | data &= ~APPL_PINMUX_PEX_RST; | |
1560 | appl_writel(pcie, data, APPL_PINMUX); | |
1561 | ||
1562 | err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, | |
1563 | data, | |
1564 | ((data & | |
1565 | APPL_DEBUG_LTSSM_STATE_MASK) >> | |
1566 | APPL_DEBUG_LTSSM_STATE_SHIFT) == | |
1567 | LTSSM_STATE_PRE_DETECT, | |
1568 | 1, LTSSM_TIMEOUT); | |
1569 | if (err) { | |
1570 | dev_info(pcie->dev, "Link didn't go to detect state\n"); | |
1571 | } else { | |
1572 | /* Disable LTSSM after link is in detect state */ | |
1573 | data = appl_readl(pcie, APPL_CTRL); | |
1574 | data &= ~APPL_CTRL_LTSSM_EN; | |
1575 | appl_writel(pcie, data, APPL_CTRL); | |
1576 | } | |
1577 | } | |
1578 | /* | |
1579 | * DBI registers may not be accessible after this as PLL-E would be | |
1580 | * down depending on how CLKREQ is pulled by end point | |
1581 | */ | |
1582 | data = appl_readl(pcie, APPL_PINMUX); | |
1583 | data |= (APPL_PINMUX_CLKREQ_OVERRIDE_EN | APPL_PINMUX_CLKREQ_OVERRIDE); | |
1584 | /* Cut REFCLK to slot */ | |
1585 | data |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN; | |
1586 | data &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE; | |
1587 | appl_writel(pcie, data, APPL_PINMUX); | |
1588 | } | |
1589 | ||
1590 | static int tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie) | |
1591 | { | |
1592 | tegra_pcie_downstream_dev_to_D0(pcie); | |
1593 | dw_pcie_host_deinit(&pcie->pci.pp); | |
1594 | tegra_pcie_dw_pme_turnoff(pcie); | |
1595 | ||
1596 | return __deinit_controller(pcie); | |
1597 | } | |
1598 | ||
1599 | static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie) | |
1600 | { | |
1601 | struct pcie_port *pp = &pcie->pci.pp; | |
1602 | struct device *dev = pcie->dev; | |
1603 | char *name; | |
1604 | int ret; | |
1605 | ||
1606 | if (IS_ENABLED(CONFIG_PCI_MSI)) { | |
1607 | pp->msi_irq = of_irq_get_byname(dev->of_node, "msi"); | |
1608 | if (!pp->msi_irq) { | |
1609 | dev_err(dev, "Failed to get MSI interrupt\n"); | |
1610 | return -ENODEV; | |
1611 | } | |
1612 | } | |
1613 | ||
1614 | pm_runtime_enable(dev); | |
1615 | ||
1616 | ret = pm_runtime_get_sync(dev); | |
1617 | if (ret < 0) { | |
1618 | dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n", | |
1619 | ret); | |
f4ff4faf VS |
1620 | goto fail_pm_get_sync; |
1621 | } | |
1622 | ||
1623 | ret = pinctrl_pm_select_default_state(dev); | |
1624 | if (ret < 0) { | |
1625 | dev_err(dev, "Failed to configure sideband pins: %d\n", ret); | |
1626 | goto fail_pinctrl; | |
56e15a23 VS |
1627 | } |
1628 | ||
1629 | tegra_pcie_init_controller(pcie); | |
1630 | ||
1631 | pcie->link_state = tegra_pcie_dw_link_up(&pcie->pci); | |
1632 | if (!pcie->link_state) { | |
1633 | ret = -ENOMEDIUM; | |
1634 | goto fail_host_init; | |
1635 | } | |
1636 | ||
1637 | name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node); | |
1638 | if (!name) { | |
1639 | ret = -ENOMEM; | |
1640 | goto fail_host_init; | |
1641 | } | |
1642 | ||
1643 | pcie->debugfs = debugfs_create_dir(name, NULL); | |
1644 | if (!pcie->debugfs) | |
1645 | dev_err(dev, "Failed to create debugfs\n"); | |
1646 | else | |
1647 | init_debugfs(pcie); | |
1648 | ||
1649 | return ret; | |
1650 | ||
1651 | fail_host_init: | |
1652 | tegra_pcie_deinit_controller(pcie); | |
f4ff4faf | 1653 | fail_pinctrl: |
56e15a23 | 1654 | pm_runtime_put_sync(dev); |
f4ff4faf | 1655 | fail_pm_get_sync: |
56e15a23 VS |
1656 | pm_runtime_disable(dev); |
1657 | return ret; | |
1658 | } | |
1659 | ||
c57247f9 VS |
1660 | static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie) |
1661 | { | |
1662 | u32 val; | |
1663 | int ret; | |
1664 | ||
1665 | if (pcie->ep_state == EP_STATE_DISABLED) | |
1666 | return; | |
1667 | ||
1668 | /* Disable LTSSM */ | |
1669 | val = appl_readl(pcie, APPL_CTRL); | |
1670 | val &= ~APPL_CTRL_LTSSM_EN; | |
1671 | appl_writel(pcie, val, APPL_CTRL); | |
1672 | ||
1673 | ret = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val, | |
1674 | ((val & APPL_DEBUG_LTSSM_STATE_MASK) >> | |
1675 | APPL_DEBUG_LTSSM_STATE_SHIFT) == | |
1676 | LTSSM_STATE_PRE_DETECT, | |
1677 | 1, LTSSM_TIMEOUT); | |
1678 | if (ret) | |
1679 | dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret); | |
1680 | ||
1681 | reset_control_assert(pcie->core_rst); | |
1682 | ||
1683 | tegra_pcie_disable_phy(pcie); | |
1684 | ||
1685 | reset_control_assert(pcie->core_apb_rst); | |
1686 | ||
1687 | clk_disable_unprepare(pcie->core_clk); | |
1688 | ||
1689 | pm_runtime_put_sync(pcie->dev); | |
1690 | ||
1691 | ret = tegra_pcie_bpmp_set_pll_state(pcie, false); | |
1692 | if (ret) | |
1693 | dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", ret); | |
1694 | ||
1695 | pcie->ep_state = EP_STATE_DISABLED; | |
1696 | dev_dbg(pcie->dev, "Uninitialization of endpoint is completed\n"); | |
1697 | } | |
1698 | ||
1699 | static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie) | |
1700 | { | |
1701 | struct dw_pcie *pci = &pcie->pci; | |
1702 | struct dw_pcie_ep *ep = &pci->ep; | |
1703 | struct device *dev = pcie->dev; | |
1704 | u32 val; | |
1705 | int ret; | |
1706 | ||
1707 | if (pcie->ep_state == EP_STATE_ENABLED) | |
1708 | return; | |
1709 | ||
1710 | ret = pm_runtime_get_sync(dev); | |
1711 | if (ret < 0) { | |
1712 | dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n", | |
1713 | ret); | |
1714 | return; | |
1715 | } | |
1716 | ||
1717 | ret = tegra_pcie_bpmp_set_pll_state(pcie, true); | |
1718 | if (ret) { | |
1719 | dev_err(dev, "Failed to init UPHY for PCIe EP: %d\n", ret); | |
1720 | goto fail_pll_init; | |
1721 | } | |
1722 | ||
1723 | ret = clk_prepare_enable(pcie->core_clk); | |
1724 | if (ret) { | |
1725 | dev_err(dev, "Failed to enable core clock: %d\n", ret); | |
1726 | goto fail_core_clk_enable; | |
1727 | } | |
1728 | ||
1729 | ret = reset_control_deassert(pcie->core_apb_rst); | |
1730 | if (ret) { | |
1731 | dev_err(dev, "Failed to deassert core APB reset: %d\n", ret); | |
1732 | goto fail_core_apb_rst; | |
1733 | } | |
1734 | ||
1735 | ret = tegra_pcie_enable_phy(pcie); | |
1736 | if (ret) { | |
1737 | dev_err(dev, "Failed to enable PHY: %d\n", ret); | |
1738 | goto fail_phy; | |
1739 | } | |
1740 | ||
1741 | /* Clear any stale interrupt statuses */ | |
1742 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0); | |
1743 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0); | |
1744 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1); | |
1745 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2); | |
1746 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3); | |
1747 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6); | |
1748 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7); | |
1749 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0); | |
1750 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9); | |
1751 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10); | |
1752 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11); | |
1753 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13); | |
1754 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14); | |
1755 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15); | |
1756 | appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17); | |
1757 | ||
1758 | /* configure this core for EP mode operation */ | |
1759 | val = appl_readl(pcie, APPL_DM_TYPE); | |
1760 | val &= ~APPL_DM_TYPE_MASK; | |
1761 | val |= APPL_DM_TYPE_EP; | |
1762 | appl_writel(pcie, val, APPL_DM_TYPE); | |
1763 | ||
1764 | appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE); | |
1765 | ||
1766 | val = appl_readl(pcie, APPL_CTRL); | |
1767 | val |= APPL_CTRL_SYS_PRE_DET_STATE; | |
1768 | val |= APPL_CTRL_HW_HOT_RST_EN; | |
1769 | appl_writel(pcie, val, APPL_CTRL); | |
1770 | ||
1771 | val = appl_readl(pcie, APPL_CFG_MISC); | |
1772 | val |= APPL_CFG_MISC_SLV_EP_MODE; | |
1773 | val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT); | |
1774 | appl_writel(pcie, val, APPL_CFG_MISC); | |
1775 | ||
1776 | val = appl_readl(pcie, APPL_PINMUX); | |
1777 | val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN; | |
1778 | val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE; | |
1779 | appl_writel(pcie, val, APPL_PINMUX); | |
1780 | ||
1781 | appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK, | |
1782 | APPL_CFG_BASE_ADDR); | |
1783 | ||
1784 | appl_writel(pcie, pcie->atu_dma_res->start & | |
1785 | APPL_CFG_IATU_DMA_BASE_ADDR_MASK, | |
1786 | APPL_CFG_IATU_DMA_BASE_ADDR); | |
1787 | ||
1788 | val = appl_readl(pcie, APPL_INTR_EN_L0_0); | |
1789 | val |= APPL_INTR_EN_L0_0_SYS_INTR_EN; | |
1790 | val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN; | |
1791 | val |= APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN; | |
1792 | appl_writel(pcie, val, APPL_INTR_EN_L0_0); | |
1793 | ||
1794 | val = appl_readl(pcie, APPL_INTR_EN_L1_0_0); | |
1795 | val |= APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN; | |
1796 | val |= APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN; | |
1797 | appl_writel(pcie, val, APPL_INTR_EN_L1_0_0); | |
1798 | ||
1799 | reset_control_deassert(pcie->core_rst); | |
1800 | ||
1801 | if (pcie->update_fc_fixup) { | |
1802 | val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF); | |
1803 | val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT; | |
1804 | dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val); | |
1805 | } | |
1806 | ||
1807 | config_gen3_gen4_eq_presets(pcie); | |
1808 | ||
1809 | init_host_aspm(pcie); | |
1810 | ||
1811 | /* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */ | |
1812 | if (!pcie->supports_clkreq) { | |
1813 | disable_aspm_l11(pcie); | |
1814 | disable_aspm_l12(pcie); | |
1815 | } | |
1816 | ||
1817 | val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); | |
1818 | val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL; | |
1819 | dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); | |
1820 | ||
1821 | /* Configure N_FTS & FTS */ | |
1822 | val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL); | |
1823 | val &= ~(N_FTS_MASK << N_FTS_SHIFT); | |
1824 | val |= N_FTS_VAL << N_FTS_SHIFT; | |
1825 | dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val); | |
1826 | ||
1827 | val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL); | |
1828 | val &= ~FTS_MASK; | |
1829 | val |= FTS_VAL; | |
1830 | dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val); | |
1831 | ||
1832 | /* Configure Max Speed from DT */ | |
1833 | if (pcie->max_speed && pcie->max_speed != -EINVAL) { | |
1834 | val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + | |
1835 | PCI_EXP_LNKCAP); | |
1836 | val &= ~PCI_EXP_LNKCAP_SLS; | |
1837 | val |= pcie->max_speed; | |
1838 | dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, | |
1839 | val); | |
1840 | } | |
1841 | ||
1842 | pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci, | |
1843 | PCI_CAP_ID_EXP); | |
1844 | clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ); | |
1845 | ||
1846 | val = (ep->msi_mem_phys & MSIX_ADDR_MATCH_LOW_OFF_MASK); | |
1847 | val |= MSIX_ADDR_MATCH_LOW_OFF_EN; | |
1848 | dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_LOW_OFF, val); | |
1849 | val = (lower_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK); | |
1850 | dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val); | |
1851 | ||
1852 | ret = dw_pcie_ep_init_complete(ep); | |
1853 | if (ret) { | |
1854 | dev_err(dev, "Failed to complete initialization: %d\n", ret); | |
1855 | goto fail_init_complete; | |
1856 | } | |
1857 | ||
1858 | dw_pcie_ep_init_notify(ep); | |
1859 | ||
1860 | /* Enable LTSSM */ | |
1861 | val = appl_readl(pcie, APPL_CTRL); | |
1862 | val |= APPL_CTRL_LTSSM_EN; | |
1863 | appl_writel(pcie, val, APPL_CTRL); | |
1864 | ||
1865 | pcie->ep_state = EP_STATE_ENABLED; | |
1866 | dev_dbg(dev, "Initialization of endpoint is completed\n"); | |
1867 | ||
1868 | return; | |
1869 | ||
1870 | fail_init_complete: | |
1871 | reset_control_assert(pcie->core_rst); | |
1872 | tegra_pcie_disable_phy(pcie); | |
1873 | fail_phy: | |
1874 | reset_control_assert(pcie->core_apb_rst); | |
1875 | fail_core_apb_rst: | |
1876 | clk_disable_unprepare(pcie->core_clk); | |
1877 | fail_core_clk_enable: | |
1878 | tegra_pcie_bpmp_set_pll_state(pcie, false); | |
1879 | fail_pll_init: | |
1880 | pm_runtime_put_sync(dev); | |
1881 | } | |
1882 | ||
1883 | static irqreturn_t tegra_pcie_ep_pex_rst_irq(int irq, void *arg) | |
1884 | { | |
1885 | struct tegra_pcie_dw *pcie = arg; | |
1886 | ||
1887 | if (gpiod_get_value(pcie->pex_rst_gpiod)) | |
1888 | pex_ep_event_pex_rst_assert(pcie); | |
1889 | else | |
1890 | pex_ep_event_pex_rst_deassert(pcie); | |
1891 | ||
1892 | return IRQ_HANDLED; | |
1893 | } | |
1894 | ||
1895 | static int tegra_pcie_ep_raise_legacy_irq(struct tegra_pcie_dw *pcie, u16 irq) | |
1896 | { | |
1897 | /* Tegra194 supports only INTA */ | |
1898 | if (irq > 1) | |
1899 | return -EINVAL; | |
1900 | ||
1901 | appl_writel(pcie, 1, APPL_LEGACY_INTX); | |
1902 | usleep_range(1000, 2000); | |
1903 | appl_writel(pcie, 0, APPL_LEGACY_INTX); | |
1904 | return 0; | |
1905 | } | |
1906 | ||
1907 | static int tegra_pcie_ep_raise_msi_irq(struct tegra_pcie_dw *pcie, u16 irq) | |
1908 | { | |
1909 | if (unlikely(irq > 31)) | |
1910 | return -EINVAL; | |
1911 | ||
1912 | appl_writel(pcie, (1 << irq), APPL_MSI_CTRL_1); | |
1913 | ||
1914 | return 0; | |
1915 | } | |
1916 | ||
1917 | static int tegra_pcie_ep_raise_msix_irq(struct tegra_pcie_dw *pcie, u16 irq) | |
1918 | { | |
1919 | struct dw_pcie_ep *ep = &pcie->pci.ep; | |
1920 | ||
1921 | writel(irq, ep->msi_mem); | |
1922 | ||
1923 | return 0; | |
1924 | } | |
1925 | ||
1926 | static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, | |
1927 | enum pci_epc_irq_type type, | |
1928 | u16 interrupt_num) | |
1929 | { | |
1930 | struct dw_pcie *pci = to_dw_pcie_from_ep(ep); | |
1931 | struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); | |
1932 | ||
1933 | switch (type) { | |
1934 | case PCI_EPC_IRQ_LEGACY: | |
1935 | return tegra_pcie_ep_raise_legacy_irq(pcie, interrupt_num); | |
1936 | ||
1937 | case PCI_EPC_IRQ_MSI: | |
1938 | return tegra_pcie_ep_raise_msi_irq(pcie, interrupt_num); | |
1939 | ||
1940 | case PCI_EPC_IRQ_MSIX: | |
1941 | return tegra_pcie_ep_raise_msix_irq(pcie, interrupt_num); | |
1942 | ||
1943 | default: | |
1944 | dev_err(pci->dev, "Unknown IRQ type\n"); | |
1945 | return -EPERM; | |
1946 | } | |
1947 | ||
1948 | return 0; | |
1949 | } | |
1950 | ||
1951 | static const struct pci_epc_features tegra_pcie_epc_features = { | |
1952 | .linkup_notifier = true, | |
1953 | .core_init_notifier = true, | |
1954 | .msi_capable = false, | |
1955 | .msix_capable = false, | |
1956 | .reserved_bar = 1 << BAR_2 | 1 << BAR_3 | 1 << BAR_4 | 1 << BAR_5, | |
1957 | .bar_fixed_64bit = 1 << BAR_0, | |
1958 | .bar_fixed_size[0] = SZ_1M, | |
1959 | }; | |
1960 | ||
1961 | static const struct pci_epc_features* | |
1962 | tegra_pcie_ep_get_features(struct dw_pcie_ep *ep) | |
1963 | { | |
1964 | return &tegra_pcie_epc_features; | |
1965 | } | |
1966 | ||
1967 | static struct dw_pcie_ep_ops pcie_ep_ops = { | |
1968 | .raise_irq = tegra_pcie_ep_raise_irq, | |
1969 | .get_features = tegra_pcie_ep_get_features, | |
1970 | }; | |
1971 | ||
1972 | static int tegra_pcie_config_ep(struct tegra_pcie_dw *pcie, | |
1973 | struct platform_device *pdev) | |
1974 | { | |
1975 | struct dw_pcie *pci = &pcie->pci; | |
1976 | struct device *dev = pcie->dev; | |
1977 | struct dw_pcie_ep *ep; | |
1978 | struct resource *res; | |
1979 | char *name; | |
1980 | int ret; | |
1981 | ||
1982 | ep = &pci->ep; | |
1983 | ep->ops = &pcie_ep_ops; | |
1984 | ||
1985 | res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space"); | |
1986 | if (!res) | |
1987 | return -EINVAL; | |
1988 | ||
1989 | ep->phys_base = res->start; | |
1990 | ep->addr_size = resource_size(res); | |
1991 | ep->page_size = SZ_64K; | |
1992 | ||
1993 | ret = gpiod_set_debounce(pcie->pex_rst_gpiod, PERST_DEBOUNCE_TIME); | |
1994 | if (ret < 0) { | |
1995 | dev_err(dev, "Failed to set PERST GPIO debounce time: %d\n", | |
1996 | ret); | |
1997 | return ret; | |
1998 | } | |
1999 | ||
2000 | ret = gpiod_to_irq(pcie->pex_rst_gpiod); | |
2001 | if (ret < 0) { | |
2002 | dev_err(dev, "Failed to get IRQ for PERST GPIO: %d\n", ret); | |
2003 | return ret; | |
2004 | } | |
2005 | pcie->pex_rst_irq = (unsigned int)ret; | |
2006 | ||
2007 | name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_pex_rst_irq", | |
2008 | pcie->cid); | |
2009 | if (!name) { | |
2010 | dev_err(dev, "Failed to create PERST IRQ string\n"); | |
2011 | return -ENOMEM; | |
2012 | } | |
2013 | ||
2014 | irq_set_status_flags(pcie->pex_rst_irq, IRQ_NOAUTOEN); | |
2015 | ||
2016 | pcie->ep_state = EP_STATE_DISABLED; | |
2017 | ||
2018 | ret = devm_request_threaded_irq(dev, pcie->pex_rst_irq, NULL, | |
2019 | tegra_pcie_ep_pex_rst_irq, | |
2020 | IRQF_TRIGGER_RISING | | |
2021 | IRQF_TRIGGER_FALLING | IRQF_ONESHOT, | |
2022 | name, (void *)pcie); | |
2023 | if (ret < 0) { | |
2024 | dev_err(dev, "Failed to request IRQ for PERST: %d\n", ret); | |
2025 | return ret; | |
2026 | } | |
2027 | ||
2028 | name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_ep_work", | |
2029 | pcie->cid); | |
2030 | if (!name) { | |
2031 | dev_err(dev, "Failed to create PCIe EP work thread string\n"); | |
2032 | return -ENOMEM; | |
2033 | } | |
2034 | ||
2035 | pm_runtime_enable(dev); | |
2036 | ||
2037 | ret = dw_pcie_ep_init(ep); | |
2038 | if (ret) { | |
2039 | dev_err(dev, "Failed to initialize DWC Endpoint subsystem: %d\n", | |
2040 | ret); | |
2041 | return ret; | |
2042 | } | |
2043 | ||
2044 | return 0; | |
2045 | } | |
2046 | ||
56e15a23 VS |
2047 | static int tegra_pcie_dw_probe(struct platform_device *pdev) |
2048 | { | |
c57247f9 | 2049 | const struct tegra_pcie_dw_of_data *data; |
56e15a23 VS |
2050 | struct device *dev = &pdev->dev; |
2051 | struct resource *atu_dma_res; | |
2052 | struct tegra_pcie_dw *pcie; | |
2053 | struct resource *dbi_res; | |
2054 | struct pcie_port *pp; | |
2055 | struct dw_pcie *pci; | |
2056 | struct phy **phys; | |
2057 | char *name; | |
2058 | int ret; | |
2059 | u32 i; | |
2060 | ||
c57247f9 VS |
2061 | data = of_device_get_match_data(dev); |
2062 | ||
56e15a23 VS |
2063 | pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); |
2064 | if (!pcie) | |
2065 | return -ENOMEM; | |
2066 | ||
2067 | pci = &pcie->pci; | |
2068 | pci->dev = &pdev->dev; | |
2069 | pci->ops = &tegra_dw_pcie_ops; | |
2070 | pp = &pci->pp; | |
2071 | pcie->dev = &pdev->dev; | |
c57247f9 | 2072 | pcie->mode = (enum dw_pcie_device_mode)data->mode; |
56e15a23 VS |
2073 | |
2074 | ret = tegra_pcie_dw_parse_dt(pcie); | |
2075 | if (ret < 0) { | |
54455238 TR |
2076 | const char *level = KERN_ERR; |
2077 | ||
2078 | if (ret == -EPROBE_DEFER) | |
2079 | level = KERN_DEBUG; | |
2080 | ||
2081 | dev_printk(level, dev, | |
2082 | dev_fmt("Failed to parse device tree: %d\n"), | |
2083 | ret); | |
56e15a23 VS |
2084 | return ret; |
2085 | } | |
2086 | ||
0a901f21 VS |
2087 | ret = tegra_pcie_get_slot_regulators(pcie); |
2088 | if (ret < 0) { | |
54455238 TR |
2089 | const char *level = KERN_ERR; |
2090 | ||
2091 | if (ret == -EPROBE_DEFER) | |
2092 | level = KERN_DEBUG; | |
2093 | ||
2094 | dev_printk(level, dev, | |
2095 | dev_fmt("Failed to get slot regulators: %d\n"), | |
2096 | ret); | |
0a901f21 VS |
2097 | return ret; |
2098 | } | |
2099 | ||
c57247f9 VS |
2100 | if (pcie->pex_refclk_sel_gpiod) |
2101 | gpiod_set_value(pcie->pex_refclk_sel_gpiod, 1); | |
2102 | ||
56e15a23 VS |
2103 | pcie->pex_ctl_supply = devm_regulator_get(dev, "vddio-pex-ctl"); |
2104 | if (IS_ERR(pcie->pex_ctl_supply)) { | |
2105 | ret = PTR_ERR(pcie->pex_ctl_supply); | |
2106 | if (ret != -EPROBE_DEFER) | |
2107 | dev_err(dev, "Failed to get regulator: %ld\n", | |
2108 | PTR_ERR(pcie->pex_ctl_supply)); | |
2109 | return ret; | |
2110 | } | |
2111 | ||
2112 | pcie->core_clk = devm_clk_get(dev, "core"); | |
2113 | if (IS_ERR(pcie->core_clk)) { | |
2114 | dev_err(dev, "Failed to get core clock: %ld\n", | |
2115 | PTR_ERR(pcie->core_clk)); | |
2116 | return PTR_ERR(pcie->core_clk); | |
2117 | } | |
2118 | ||
2119 | pcie->appl_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, | |
2120 | "appl"); | |
2121 | if (!pcie->appl_res) { | |
2122 | dev_err(dev, "Failed to find \"appl\" region\n"); | |
2123 | return -ENODEV; | |
2124 | } | |
2125 | ||
2126 | pcie->appl_base = devm_ioremap_resource(dev, pcie->appl_res); | |
2127 | if (IS_ERR(pcie->appl_base)) | |
2128 | return PTR_ERR(pcie->appl_base); | |
2129 | ||
2130 | pcie->core_apb_rst = devm_reset_control_get(dev, "apb"); | |
2131 | if (IS_ERR(pcie->core_apb_rst)) { | |
2132 | dev_err(dev, "Failed to get APB reset: %ld\n", | |
2133 | PTR_ERR(pcie->core_apb_rst)); | |
2134 | return PTR_ERR(pcie->core_apb_rst); | |
2135 | } | |
2136 | ||
2137 | phys = devm_kcalloc(dev, pcie->phy_count, sizeof(*phys), GFP_KERNEL); | |
2138 | if (!phys) | |
2139 | return -ENOMEM; | |
2140 | ||
2141 | for (i = 0; i < pcie->phy_count; i++) { | |
2142 | name = kasprintf(GFP_KERNEL, "p2u-%u", i); | |
2143 | if (!name) { | |
2144 | dev_err(dev, "Failed to create P2U string\n"); | |
2145 | return -ENOMEM; | |
2146 | } | |
2147 | phys[i] = devm_phy_get(dev, name); | |
2148 | kfree(name); | |
2149 | if (IS_ERR(phys[i])) { | |
2150 | ret = PTR_ERR(phys[i]); | |
2151 | if (ret != -EPROBE_DEFER) | |
2152 | dev_err(dev, "Failed to get PHY: %d\n", ret); | |
2153 | return ret; | |
2154 | } | |
2155 | } | |
2156 | ||
2157 | pcie->phys = phys; | |
2158 | ||
2159 | dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi"); | |
2160 | if (!dbi_res) { | |
2161 | dev_err(dev, "Failed to find \"dbi\" region\n"); | |
2162 | return -ENODEV; | |
2163 | } | |
2164 | pcie->dbi_res = dbi_res; | |
2165 | ||
2166 | pci->dbi_base = devm_ioremap_resource(dev, dbi_res); | |
2167 | if (IS_ERR(pci->dbi_base)) | |
2168 | return PTR_ERR(pci->dbi_base); | |
2169 | ||
2170 | /* Tegra HW locates DBI2 at a fixed offset from DBI */ | |
2171 | pci->dbi_base2 = pci->dbi_base + 0x1000; | |
2172 | ||
2173 | atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, | |
2174 | "atu_dma"); | |
2175 | if (!atu_dma_res) { | |
2176 | dev_err(dev, "Failed to find \"atu_dma\" region\n"); | |
2177 | return -ENODEV; | |
2178 | } | |
2179 | pcie->atu_dma_res = atu_dma_res; | |
2180 | ||
2181 | pci->atu_base = devm_ioremap_resource(dev, atu_dma_res); | |
2182 | if (IS_ERR(pci->atu_base)) | |
2183 | return PTR_ERR(pci->atu_base); | |
2184 | ||
2185 | pcie->core_rst = devm_reset_control_get(dev, "core"); | |
2186 | if (IS_ERR(pcie->core_rst)) { | |
2187 | dev_err(dev, "Failed to get core reset: %ld\n", | |
2188 | PTR_ERR(pcie->core_rst)); | |
2189 | return PTR_ERR(pcie->core_rst); | |
2190 | } | |
2191 | ||
2192 | pp->irq = platform_get_irq_byname(pdev, "intr"); | |
2193 | if (!pp->irq) { | |
2194 | dev_err(dev, "Failed to get \"intr\" interrupt\n"); | |
2195 | return -ENODEV; | |
2196 | } | |
2197 | ||
56e15a23 VS |
2198 | pcie->bpmp = tegra_bpmp_get(dev); |
2199 | if (IS_ERR(pcie->bpmp)) | |
2200 | return PTR_ERR(pcie->bpmp); | |
2201 | ||
2202 | platform_set_drvdata(pdev, pcie); | |
2203 | ||
c57247f9 VS |
2204 | switch (pcie->mode) { |
2205 | case DW_PCIE_RC_TYPE: | |
2206 | ret = devm_request_irq(dev, pp->irq, tegra_pcie_rp_irq_handler, | |
2207 | IRQF_SHARED, "tegra-pcie-intr", pcie); | |
2208 | if (ret) { | |
2209 | dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq, | |
2210 | ret); | |
2211 | goto fail; | |
2212 | } | |
2213 | ||
2214 | ret = tegra_pcie_config_rp(pcie); | |
2215 | if (ret && ret != -ENOMEDIUM) | |
2216 | goto fail; | |
2217 | else | |
2218 | return 0; | |
2219 | break; | |
2220 | ||
2221 | case DW_PCIE_EP_TYPE: | |
2222 | ret = devm_request_threaded_irq(dev, pp->irq, | |
2223 | tegra_pcie_ep_hard_irq, | |
2224 | tegra_pcie_ep_irq_thread, | |
2225 | IRQF_SHARED | IRQF_ONESHOT, | |
2226 | "tegra-pcie-ep-intr", pcie); | |
2227 | if (ret) { | |
2228 | dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq, | |
2229 | ret); | |
2230 | goto fail; | |
2231 | } | |
2232 | ||
2233 | ret = tegra_pcie_config_ep(pcie, pdev); | |
2234 | if (ret < 0) | |
2235 | goto fail; | |
2236 | break; | |
2237 | ||
2238 | default: | |
2239 | dev_err(dev, "Invalid PCIe device type %d\n", pcie->mode); | |
2240 | } | |
56e15a23 VS |
2241 | |
2242 | fail: | |
2243 | tegra_bpmp_put(pcie->bpmp); | |
2244 | return ret; | |
2245 | } | |
2246 | ||
2247 | static int tegra_pcie_dw_remove(struct platform_device *pdev) | |
2248 | { | |
2249 | struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev); | |
2250 | ||
2251 | if (!pcie->link_state) | |
2252 | return 0; | |
2253 | ||
2254 | debugfs_remove_recursive(pcie->debugfs); | |
2255 | tegra_pcie_deinit_controller(pcie); | |
2256 | pm_runtime_put_sync(pcie->dev); | |
2257 | pm_runtime_disable(pcie->dev); | |
2258 | tegra_bpmp_put(pcie->bpmp); | |
c57247f9 VS |
2259 | if (pcie->pex_refclk_sel_gpiod) |
2260 | gpiod_set_value(pcie->pex_refclk_sel_gpiod, 0); | |
56e15a23 VS |
2261 | |
2262 | return 0; | |
2263 | } | |
2264 | ||
2265 | static int tegra_pcie_dw_suspend_late(struct device *dev) | |
2266 | { | |
2267 | struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); | |
2268 | u32 val; | |
2269 | ||
2270 | if (!pcie->link_state) | |
2271 | return 0; | |
2272 | ||
2273 | /* Enable HW_HOT_RST mode */ | |
2274 | val = appl_readl(pcie, APPL_CTRL); | |
2275 | val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << | |
2276 | APPL_CTRL_HW_HOT_RST_MODE_SHIFT); | |
2277 | val |= APPL_CTRL_HW_HOT_RST_EN; | |
2278 | appl_writel(pcie, val, APPL_CTRL); | |
2279 | ||
2280 | return 0; | |
2281 | } | |
2282 | ||
2283 | static int tegra_pcie_dw_suspend_noirq(struct device *dev) | |
2284 | { | |
2285 | struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); | |
2286 | ||
2287 | if (!pcie->link_state) | |
2288 | return 0; | |
2289 | ||
2290 | /* Save MSI interrupt vector */ | |
2291 | pcie->msi_ctrl_int = dw_pcie_readl_dbi(&pcie->pci, | |
2292 | PORT_LOGIC_MSI_CTRL_INT_0_EN); | |
2293 | tegra_pcie_downstream_dev_to_D0(pcie); | |
2294 | tegra_pcie_dw_pme_turnoff(pcie); | |
2295 | ||
2296 | return __deinit_controller(pcie); | |
2297 | } | |
2298 | ||
2299 | static int tegra_pcie_dw_resume_noirq(struct device *dev) | |
2300 | { | |
2301 | struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); | |
2302 | int ret; | |
2303 | ||
2304 | if (!pcie->link_state) | |
2305 | return 0; | |
2306 | ||
2307 | ret = tegra_pcie_config_controller(pcie, true); | |
2308 | if (ret < 0) | |
2309 | return ret; | |
2310 | ||
2311 | ret = tegra_pcie_dw_host_init(&pcie->pci.pp); | |
2312 | if (ret < 0) { | |
2313 | dev_err(dev, "Failed to init host: %d\n", ret); | |
2314 | goto fail_host_init; | |
2315 | } | |
2316 | ||
2317 | /* Restore MSI interrupt vector */ | |
2318 | dw_pcie_writel_dbi(&pcie->pci, PORT_LOGIC_MSI_CTRL_INT_0_EN, | |
2319 | pcie->msi_ctrl_int); | |
2320 | ||
2321 | return 0; | |
2322 | ||
2323 | fail_host_init: | |
2324 | return __deinit_controller(pcie); | |
2325 | } | |
2326 | ||
2327 | static int tegra_pcie_dw_resume_early(struct device *dev) | |
2328 | { | |
2329 | struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); | |
2330 | u32 val; | |
2331 | ||
2332 | if (!pcie->link_state) | |
2333 | return 0; | |
2334 | ||
2335 | /* Disable HW_HOT_RST mode */ | |
2336 | val = appl_readl(pcie, APPL_CTRL); | |
2337 | val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << | |
2338 | APPL_CTRL_HW_HOT_RST_MODE_SHIFT); | |
2339 | val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST << | |
2340 | APPL_CTRL_HW_HOT_RST_MODE_SHIFT; | |
2341 | val &= ~APPL_CTRL_HW_HOT_RST_EN; | |
2342 | appl_writel(pcie, val, APPL_CTRL); | |
2343 | ||
2344 | return 0; | |
2345 | } | |
2346 | ||
2347 | static void tegra_pcie_dw_shutdown(struct platform_device *pdev) | |
2348 | { | |
2349 | struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev); | |
2350 | ||
2351 | if (!pcie->link_state) | |
2352 | return; | |
2353 | ||
2354 | debugfs_remove_recursive(pcie->debugfs); | |
2355 | tegra_pcie_downstream_dev_to_D0(pcie); | |
2356 | ||
2357 | disable_irq(pcie->pci.pp.irq); | |
2358 | if (IS_ENABLED(CONFIG_PCI_MSI)) | |
2359 | disable_irq(pcie->pci.pp.msi_irq); | |
2360 | ||
2361 | tegra_pcie_dw_pme_turnoff(pcie); | |
2362 | __deinit_controller(pcie); | |
2363 | } | |
2364 | ||
c57247f9 VS |
2365 | static const struct tegra_pcie_dw_of_data tegra_pcie_dw_rc_of_data = { |
2366 | .mode = DW_PCIE_RC_TYPE, | |
2367 | }; | |
2368 | ||
2369 | static const struct tegra_pcie_dw_of_data tegra_pcie_dw_ep_of_data = { | |
2370 | .mode = DW_PCIE_EP_TYPE, | |
2371 | }; | |
2372 | ||
56e15a23 VS |
2373 | static const struct of_device_id tegra_pcie_dw_of_match[] = { |
2374 | { | |
2375 | .compatible = "nvidia,tegra194-pcie", | |
c57247f9 VS |
2376 | .data = &tegra_pcie_dw_rc_of_data, |
2377 | }, | |
2378 | { | |
2379 | .compatible = "nvidia,tegra194-pcie-ep", | |
2380 | .data = &tegra_pcie_dw_ep_of_data, | |
56e15a23 VS |
2381 | }, |
2382 | {}, | |
2383 | }; | |
2384 | ||
2385 | static const struct dev_pm_ops tegra_pcie_dw_pm_ops = { | |
2386 | .suspend_late = tegra_pcie_dw_suspend_late, | |
2387 | .suspend_noirq = tegra_pcie_dw_suspend_noirq, | |
2388 | .resume_noirq = tegra_pcie_dw_resume_noirq, | |
2389 | .resume_early = tegra_pcie_dw_resume_early, | |
2390 | }; | |
2391 | ||
2392 | static struct platform_driver tegra_pcie_dw_driver = { | |
2393 | .probe = tegra_pcie_dw_probe, | |
2394 | .remove = tegra_pcie_dw_remove, | |
2395 | .shutdown = tegra_pcie_dw_shutdown, | |
2396 | .driver = { | |
2397 | .name = "tegra194-pcie", | |
2398 | .pm = &tegra_pcie_dw_pm_ops, | |
2399 | .of_match_table = tegra_pcie_dw_of_match, | |
2400 | }, | |
2401 | }; | |
2402 | module_platform_driver(tegra_pcie_dw_driver); | |
2403 | ||
2404 | MODULE_DEVICE_TABLE(of, tegra_pcie_dw_of_match); | |
2405 | ||
2406 | MODULE_AUTHOR("Vidya Sagar <vidyas@nvidia.com>"); | |
2407 | MODULE_DESCRIPTION("NVIDIA PCIe host controller driver"); | |
2408 | MODULE_LICENSE("GPL v2"); |