1 // SPDX-License-Identifier: GPL-2.0
3 * MediaTek PCIe host controller driver.
5 * Copyright (c) 2017 MediaTek Inc.
6 * Author: Ryder Lee <ryder.lee@mediatek.com>
7 * Honghui Zhang <honghui.zhang@mediatek.com>
10 #include <linux/clk.h>
11 #include <linux/delay.h>
12 #include <linux/iopoll.h>
13 #include <linux/irq.h>
14 #include <linux/irqchip/chained_irq.h>
15 #include <linux/irqdomain.h>
16 #include <linux/kernel.h>
17 #include <linux/msi.h>
18 #include <linux/of_address.h>
19 #include <linux/of_pci.h>
20 #include <linux/of_platform.h>
21 #include <linux/pci.h>
22 #include <linux/phy/phy.h>
23 #include <linux/platform_device.h>
24 #include <linux/pm_runtime.h>
25 #include <linux/reset.h>
29 /* PCIe shared registers */
30 #define PCIE_SYS_CFG 0x00
31 #define PCIE_INT_ENABLE 0x0c
32 #define PCIE_CFG_ADDR 0x20
33 #define PCIE_CFG_DATA 0x24
35 /* PCIe per port registers */
36 #define PCIE_BAR0_SETUP 0x10
37 #define PCIE_CLASS 0x34
38 #define PCIE_LINK_STATUS 0x50
40 #define PCIE_PORT_INT_EN(x) BIT(20 + (x))
41 #define PCIE_PORT_PERST(x) BIT(1 + (x))
42 #define PCIE_PORT_LINKUP BIT(0)
43 #define PCIE_BAR_MAP_MAX GENMASK(31, 16)
45 #define PCIE_BAR_ENABLE BIT(0)
46 #define PCIE_REVISION_ID BIT(0)
47 #define PCIE_CLASS_CODE (0x60400 << 8)
48 #define PCIE_CONF_REG(regn) (((regn) & GENMASK(7, 2)) | \
49 ((((regn) >> 8) & GENMASK(3, 0)) << 24))
50 #define PCIE_CONF_FUN(fun) (((fun) << 8) & GENMASK(10, 8))
51 #define PCIE_CONF_DEV(dev) (((dev) << 11) & GENMASK(15, 11))
52 #define PCIE_CONF_BUS(bus) (((bus) << 16) & GENMASK(23, 16))
53 #define PCIE_CONF_ADDR(regn, fun, dev, bus) \
54 (PCIE_CONF_REG(regn) | PCIE_CONF_FUN(fun) | \
55 PCIE_CONF_DEV(dev) | PCIE_CONF_BUS(bus))
57 /* MediaTek specific configuration registers */
58 #define PCIE_FTS_NUM 0x70c
59 #define PCIE_FTS_NUM_MASK GENMASK(15, 8)
60 #define PCIE_FTS_NUM_L0(x) ((x) & 0xff << 8)
62 #define PCIE_FC_CREDIT 0x73c
63 #define PCIE_FC_CREDIT_MASK (GENMASK(31, 31) | GENMASK(28, 16))
64 #define PCIE_FC_CREDIT_VAL(x) ((x) << 16)
66 /* PCIe V2 share registers */
67 #define PCIE_SYS_CFG_V2 0x0
68 #define PCIE_CSR_LTSSM_EN(x) BIT(0 + (x) * 8)
69 #define PCIE_CSR_ASPM_L1_EN(x) BIT(1 + (x) * 8)
71 /* PCIe V2 per-port registers */
72 #define PCIE_MSI_VECTOR 0x0c0
74 #define PCIE_CONF_VEND_ID 0x100
75 #define PCIE_CONF_CLASS_ID 0x106
77 #define PCIE_INT_MASK 0x420
78 #define INTX_MASK GENMASK(19, 16)
80 #define PCIE_INT_STATUS 0x424
81 #define MSI_STATUS BIT(23)
82 #define PCIE_IMSI_STATUS 0x42c
83 #define PCIE_IMSI_ADDR 0x430
84 #define MSI_MASK BIT(23)
85 #define MTK_MSI_IRQS_NUM 32
87 #define PCIE_AHB_TRANS_BASE0_L 0x438
88 #define PCIE_AHB_TRANS_BASE0_H 0x43c
89 #define AHB2PCIE_SIZE(x) ((x) & GENMASK(4, 0))
90 #define PCIE_AXI_WINDOW0 0x448
91 #define WIN_ENABLE BIT(7)
93 /* PCIe V2 configuration transaction header */
94 #define PCIE_CFG_HEADER0 0x460
95 #define PCIE_CFG_HEADER1 0x464
96 #define PCIE_CFG_HEADER2 0x468
97 #define PCIE_CFG_WDATA 0x470
98 #define PCIE_APP_TLP_REQ 0x488
99 #define PCIE_CFG_RDATA 0x48c
100 #define APP_CFG_REQ BIT(0)
101 #define APP_CPL_STATUS GENMASK(7, 5)
103 #define CFG_WRRD_TYPE_0 4
107 #define CFG_DW0_LENGTH(length) ((length) & GENMASK(9, 0))
108 #define CFG_DW0_TYPE(type) (((type) << 24) & GENMASK(28, 24))
109 #define CFG_DW0_FMT(fmt) (((fmt) << 29) & GENMASK(31, 29))
110 #define CFG_DW2_REGN(regn) ((regn) & GENMASK(11, 2))
111 #define CFG_DW2_FUN(fun) (((fun) << 16) & GENMASK(18, 16))
112 #define CFG_DW2_DEV(dev) (((dev) << 19) & GENMASK(23, 19))
113 #define CFG_DW2_BUS(bus) (((bus) << 24) & GENMASK(31, 24))
114 #define CFG_HEADER_DW0(type, fmt) \
115 (CFG_DW0_LENGTH(1) | CFG_DW0_TYPE(type) | CFG_DW0_FMT(fmt))
116 #define CFG_HEADER_DW1(where, size) \
117 (GENMASK(((size) - 1), 0) << ((where) & 0x3))
118 #define CFG_HEADER_DW2(regn, fun, dev, bus) \
119 (CFG_DW2_REGN(regn) | CFG_DW2_FUN(fun) | \
120 CFG_DW2_DEV(dev) | CFG_DW2_BUS(bus))
122 #define PCIE_RST_CTRL 0x510
123 #define PCIE_PHY_RSTB BIT(0)
124 #define PCIE_PIPE_SRSTB BIT(1)
125 #define PCIE_MAC_SRSTB BIT(2)
126 #define PCIE_CRSTB BIT(3)
127 #define PCIE_PERSTB BIT(8)
128 #define PCIE_LINKDOWN_RST_EN GENMASK(15, 13)
129 #define PCIE_LINK_STATUS_V2 0x804
130 #define PCIE_PORT_LINKUP_V2 BIT(10)
132 struct mtk_pcie_port
;
135 * struct mtk_pcie_soc - differentiate between host generations
136 * @need_fix_class_id: whether this host's class ID needed to be fixed or not
137 * @ops: pointer to configuration access functions
138 * @startup: pointer to controller setting functions
139 * @setup_irq: pointer to initialize IRQ functions
141 struct mtk_pcie_soc
{
142 bool need_fix_class_id
;
144 int (*startup
)(struct mtk_pcie_port
*port
);
145 int (*setup_irq
)(struct mtk_pcie_port
*port
, struct device_node
*node
);
149 * struct mtk_pcie_port - PCIe port information
150 * @base: IO mapped register base
152 * @pcie: pointer to PCIe host info
153 * @reset: pointer to port reset control
154 * @sys_ck: pointer to transaction/data link layer clock
155 * @ahb_ck: pointer to AHB slave interface operating clock for CSR access
156 * and RC initiated MMIO access
157 * @axi_ck: pointer to application layer MMIO channel operating clock
158 * @aux_ck: pointer to pe2_mac_bridge and pe2_mac_core operating clock
159 * when pcie_mac_ck/pcie_pipe_ck is turned off
160 * @obff_ck: pointer to OBFF functional block operating clock
161 * @pipe_ck: pointer to LTSSM and PHY/MAC layer operating clock
162 * @phy: pointer to PHY control block
165 * @irq_domain: legacy INTx IRQ domain
166 * @inner_domain: inner IRQ domain
167 * @msi_domain: MSI IRQ domain
168 * @lock: protect the msi_irq_in_use bitmap
169 * @msi_irq_in_use: bit map for assigned MSI IRQ
171 struct mtk_pcie_port
{
173 struct list_head list
;
174 struct mtk_pcie
*pcie
;
175 struct reset_control
*reset
;
185 struct irq_domain
*irq_domain
;
186 struct irq_domain
*inner_domain
;
187 struct irq_domain
*msi_domain
;
189 DECLARE_BITMAP(msi_irq_in_use
, MTK_MSI_IRQS_NUM
);
193 * struct mtk_pcie - PCIe host information
194 * @dev: pointer to PCIe device
195 * @base: IO mapped register base
196 * @free_ck: free-run reference clock
199 * @mem: non-prefetchable memory resource
201 * @offset: IO / Memory offset
202 * @ports: pointer to PCIe port information
203 * @soc: pointer to SoC-dependent operations
213 struct resource busn
;
218 struct list_head ports
;
219 const struct mtk_pcie_soc
*soc
;
222 static void mtk_pcie_subsys_powerdown(struct mtk_pcie
*pcie
)
224 struct device
*dev
= pcie
->dev
;
226 clk_disable_unprepare(pcie
->free_ck
);
228 if (dev
->pm_domain
) {
229 pm_runtime_put_sync(dev
);
230 pm_runtime_disable(dev
);
234 static void mtk_pcie_port_free(struct mtk_pcie_port
*port
)
236 struct mtk_pcie
*pcie
= port
->pcie
;
237 struct device
*dev
= pcie
->dev
;
239 devm_iounmap(dev
, port
->base
);
240 list_del(&port
->list
);
241 devm_kfree(dev
, port
);
244 static void mtk_pcie_put_resources(struct mtk_pcie
*pcie
)
246 struct mtk_pcie_port
*port
, *tmp
;
248 list_for_each_entry_safe(port
, tmp
, &pcie
->ports
, list
) {
249 phy_power_off(port
->phy
);
251 clk_disable_unprepare(port
->pipe_ck
);
252 clk_disable_unprepare(port
->obff_ck
);
253 clk_disable_unprepare(port
->axi_ck
);
254 clk_disable_unprepare(port
->aux_ck
);
255 clk_disable_unprepare(port
->ahb_ck
);
256 clk_disable_unprepare(port
->sys_ck
);
257 mtk_pcie_port_free(port
);
260 mtk_pcie_subsys_powerdown(pcie
);
263 static int mtk_pcie_check_cfg_cpld(struct mtk_pcie_port
*port
)
268 err
= readl_poll_timeout_atomic(port
->base
+ PCIE_APP_TLP_REQ
, val
,
269 !(val
& APP_CFG_REQ
), 10,
270 100 * USEC_PER_MSEC
);
272 return PCIBIOS_SET_FAILED
;
274 if (readl(port
->base
+ PCIE_APP_TLP_REQ
) & APP_CPL_STATUS
)
275 return PCIBIOS_SET_FAILED
;
277 return PCIBIOS_SUCCESSFUL
;
280 static int mtk_pcie_hw_rd_cfg(struct mtk_pcie_port
*port
, u32 bus
, u32 devfn
,
281 int where
, int size
, u32
*val
)
285 /* Write PCIe configuration transaction header for Cfgrd */
286 writel(CFG_HEADER_DW0(CFG_WRRD_TYPE_0
, CFG_RD_FMT
),
287 port
->base
+ PCIE_CFG_HEADER0
);
288 writel(CFG_HEADER_DW1(where
, size
), port
->base
+ PCIE_CFG_HEADER1
);
289 writel(CFG_HEADER_DW2(where
, PCI_FUNC(devfn
), PCI_SLOT(devfn
), bus
),
290 port
->base
+ PCIE_CFG_HEADER2
);
292 /* Trigger h/w to transmit Cfgrd TLP */
293 tmp
= readl(port
->base
+ PCIE_APP_TLP_REQ
);
295 writel(tmp
, port
->base
+ PCIE_APP_TLP_REQ
);
297 /* Check completion status */
298 if (mtk_pcie_check_cfg_cpld(port
))
299 return PCIBIOS_SET_FAILED
;
301 /* Read cpld payload of Cfgrd */
302 *val
= readl(port
->base
+ PCIE_CFG_RDATA
);
305 *val
= (*val
>> (8 * (where
& 3))) & 0xff;
307 *val
= (*val
>> (8 * (where
& 3))) & 0xffff;
309 return PCIBIOS_SUCCESSFUL
;
312 static int mtk_pcie_hw_wr_cfg(struct mtk_pcie_port
*port
, u32 bus
, u32 devfn
,
313 int where
, int size
, u32 val
)
315 /* Write PCIe configuration transaction header for Cfgwr */
316 writel(CFG_HEADER_DW0(CFG_WRRD_TYPE_0
, CFG_WR_FMT
),
317 port
->base
+ PCIE_CFG_HEADER0
);
318 writel(CFG_HEADER_DW1(where
, size
), port
->base
+ PCIE_CFG_HEADER1
);
319 writel(CFG_HEADER_DW2(where
, PCI_FUNC(devfn
), PCI_SLOT(devfn
), bus
),
320 port
->base
+ PCIE_CFG_HEADER2
);
322 /* Write Cfgwr data */
323 val
= val
<< 8 * (where
& 3);
324 writel(val
, port
->base
+ PCIE_CFG_WDATA
);
326 /* Trigger h/w to transmit Cfgwr TLP */
327 val
= readl(port
->base
+ PCIE_APP_TLP_REQ
);
329 writel(val
, port
->base
+ PCIE_APP_TLP_REQ
);
331 /* Check completion status */
332 return mtk_pcie_check_cfg_cpld(port
);
335 static struct mtk_pcie_port
*mtk_pcie_find_port(struct pci_bus
*bus
,
338 struct mtk_pcie
*pcie
= bus
->sysdata
;
339 struct mtk_pcie_port
*port
;
341 list_for_each_entry(port
, &pcie
->ports
, list
)
342 if (port
->slot
== PCI_SLOT(devfn
))
348 static int mtk_pcie_config_read(struct pci_bus
*bus
, unsigned int devfn
,
349 int where
, int size
, u32
*val
)
351 struct mtk_pcie_port
*port
;
352 u32 bn
= bus
->number
;
355 port
= mtk_pcie_find_port(bus
, devfn
);
358 return PCIBIOS_DEVICE_NOT_FOUND
;
361 ret
= mtk_pcie_hw_rd_cfg(port
, bn
, devfn
, where
, size
, val
);
368 static int mtk_pcie_config_write(struct pci_bus
*bus
, unsigned int devfn
,
369 int where
, int size
, u32 val
)
371 struct mtk_pcie_port
*port
;
372 u32 bn
= bus
->number
;
374 port
= mtk_pcie_find_port(bus
, devfn
);
376 return PCIBIOS_DEVICE_NOT_FOUND
;
378 return mtk_pcie_hw_wr_cfg(port
, bn
, devfn
, where
, size
, val
);
381 static struct pci_ops mtk_pcie_ops_v2
= {
382 .read
= mtk_pcie_config_read
,
383 .write
= mtk_pcie_config_write
,
386 static int mtk_pcie_startup_port_v2(struct mtk_pcie_port
*port
)
388 struct mtk_pcie
*pcie
= port
->pcie
;
389 struct resource
*mem
= &pcie
->mem
;
390 const struct mtk_pcie_soc
*soc
= port
->pcie
->soc
;
395 /* MT7622 platforms need to enable LTSSM and ASPM from PCIe subsys */
397 val
= readl(pcie
->base
+ PCIE_SYS_CFG_V2
);
398 val
|= PCIE_CSR_LTSSM_EN(port
->slot
) |
399 PCIE_CSR_ASPM_L1_EN(port
->slot
);
400 writel(val
, pcie
->base
+ PCIE_SYS_CFG_V2
);
403 /* Assert all reset signals */
404 writel(0, port
->base
+ PCIE_RST_CTRL
);
407 * Enable PCIe link down reset, if link status changed from link up to
408 * link down, this will reset MAC control registers and configuration
411 writel(PCIE_LINKDOWN_RST_EN
, port
->base
+ PCIE_RST_CTRL
);
413 /* De-assert PHY, PE, PIPE, MAC and configuration reset */
414 val
= readl(port
->base
+ PCIE_RST_CTRL
);
415 val
|= PCIE_PHY_RSTB
| PCIE_PERSTB
| PCIE_PIPE_SRSTB
|
416 PCIE_MAC_SRSTB
| PCIE_CRSTB
;
417 writel(val
, port
->base
+ PCIE_RST_CTRL
);
419 /* Set up vendor ID and class code */
420 if (soc
->need_fix_class_id
) {
421 val
= PCI_VENDOR_ID_MEDIATEK
;
422 writew(val
, port
->base
+ PCIE_CONF_VEND_ID
);
424 val
= PCI_CLASS_BRIDGE_HOST
;
425 writew(val
, port
->base
+ PCIE_CONF_CLASS_ID
);
428 /* 100ms timeout value should be enough for Gen1/2 training */
429 err
= readl_poll_timeout(port
->base
+ PCIE_LINK_STATUS_V2
, val
,
430 !!(val
& PCIE_PORT_LINKUP_V2
), 20,
431 100 * USEC_PER_MSEC
);
436 val
= readl(port
->base
+ PCIE_INT_MASK
);
438 writel(val
, port
->base
+ PCIE_INT_MASK
);
440 /* Set AHB to PCIe translation windows */
441 size
= mem
->end
- mem
->start
;
442 val
= lower_32_bits(mem
->start
) | AHB2PCIE_SIZE(fls(size
));
443 writel(val
, port
->base
+ PCIE_AHB_TRANS_BASE0_L
);
445 val
= upper_32_bits(mem
->start
);
446 writel(val
, port
->base
+ PCIE_AHB_TRANS_BASE0_H
);
448 /* Set PCIe to AXI translation memory space.*/
449 val
= fls(0xffffffff) | WIN_ENABLE
;
450 writel(val
, port
->base
+ PCIE_AXI_WINDOW0
);
455 static void mtk_compose_msi_msg(struct irq_data
*data
, struct msi_msg
*msg
)
457 struct mtk_pcie_port
*port
= irq_data_get_irq_chip_data(data
);
460 /* MT2712/MT7622 only support 32-bit MSI addresses */
461 addr
= virt_to_phys(port
->base
+ PCIE_MSI_VECTOR
);
463 msg
->address_lo
= lower_32_bits(addr
);
465 msg
->data
= data
->hwirq
;
467 dev_dbg(port
->pcie
->dev
, "msi#%d address_hi %#x address_lo %#x\n",
468 (int)data
->hwirq
, msg
->address_hi
, msg
->address_lo
);
471 static int mtk_msi_set_affinity(struct irq_data
*irq_data
,
472 const struct cpumask
*mask
, bool force
)
477 static void mtk_msi_ack_irq(struct irq_data
*data
)
479 struct mtk_pcie_port
*port
= irq_data_get_irq_chip_data(data
);
480 u32 hwirq
= data
->hwirq
;
482 writel(1 << hwirq
, port
->base
+ PCIE_IMSI_STATUS
);
485 static struct irq_chip mtk_msi_bottom_irq_chip
= {
487 .irq_compose_msi_msg
= mtk_compose_msi_msg
,
488 .irq_set_affinity
= mtk_msi_set_affinity
,
489 .irq_ack
= mtk_msi_ack_irq
,
492 static int mtk_pcie_irq_domain_alloc(struct irq_domain
*domain
, unsigned int virq
,
493 unsigned int nr_irqs
, void *args
)
495 struct mtk_pcie_port
*port
= domain
->host_data
;
498 WARN_ON(nr_irqs
!= 1);
499 mutex_lock(&port
->lock
);
501 bit
= find_first_zero_bit(port
->msi_irq_in_use
, MTK_MSI_IRQS_NUM
);
502 if (bit
>= MTK_MSI_IRQS_NUM
) {
503 mutex_unlock(&port
->lock
);
507 __set_bit(bit
, port
->msi_irq_in_use
);
509 mutex_unlock(&port
->lock
);
511 irq_domain_set_info(domain
, virq
, bit
, &mtk_msi_bottom_irq_chip
,
512 domain
->host_data
, handle_edge_irq
,
518 static void mtk_pcie_irq_domain_free(struct irq_domain
*domain
,
519 unsigned int virq
, unsigned int nr_irqs
)
521 struct irq_data
*d
= irq_domain_get_irq_data(domain
, virq
);
522 struct mtk_pcie_port
*port
= irq_data_get_irq_chip_data(d
);
524 mutex_lock(&port
->lock
);
526 if (!test_bit(d
->hwirq
, port
->msi_irq_in_use
))
527 dev_err(port
->pcie
->dev
, "trying to free unused MSI#%lu\n",
530 __clear_bit(d
->hwirq
, port
->msi_irq_in_use
);
532 mutex_unlock(&port
->lock
);
534 irq_domain_free_irqs_parent(domain
, virq
, nr_irqs
);
537 static const struct irq_domain_ops msi_domain_ops
= {
538 .alloc
= mtk_pcie_irq_domain_alloc
,
539 .free
= mtk_pcie_irq_domain_free
,
542 static struct irq_chip mtk_msi_irq_chip
= {
543 .name
= "MTK PCIe MSI",
544 .irq_ack
= irq_chip_ack_parent
,
545 .irq_mask
= pci_msi_mask_irq
,
546 .irq_unmask
= pci_msi_unmask_irq
,
549 static struct msi_domain_info mtk_msi_domain_info
= {
550 .flags
= (MSI_FLAG_USE_DEF_DOM_OPS
| MSI_FLAG_USE_DEF_CHIP_OPS
|
552 .chip
= &mtk_msi_irq_chip
,
555 static int mtk_pcie_allocate_msi_domains(struct mtk_pcie_port
*port
)
557 struct fwnode_handle
*fwnode
= of_node_to_fwnode(port
->pcie
->dev
->of_node
);
559 mutex_init(&port
->lock
);
561 port
->inner_domain
= irq_domain_create_linear(fwnode
, MTK_MSI_IRQS_NUM
,
562 &msi_domain_ops
, port
);
563 if (!port
->inner_domain
) {
564 dev_err(port
->pcie
->dev
, "failed to create IRQ domain\n");
568 port
->msi_domain
= pci_msi_create_irq_domain(fwnode
, &mtk_msi_domain_info
,
570 if (!port
->msi_domain
) {
571 dev_err(port
->pcie
->dev
, "failed to create MSI domain\n");
572 irq_domain_remove(port
->inner_domain
);
579 static void mtk_pcie_enable_msi(struct mtk_pcie_port
*port
)
582 phys_addr_t msg_addr
;
584 msg_addr
= virt_to_phys(port
->base
+ PCIE_MSI_VECTOR
);
585 val
= lower_32_bits(msg_addr
);
586 writel(val
, port
->base
+ PCIE_IMSI_ADDR
);
588 val
= readl(port
->base
+ PCIE_INT_MASK
);
590 writel(val
, port
->base
+ PCIE_INT_MASK
);
593 static int mtk_pcie_intx_map(struct irq_domain
*domain
, unsigned int irq
,
594 irq_hw_number_t hwirq
)
596 irq_set_chip_and_handler(irq
, &dummy_irq_chip
, handle_simple_irq
);
597 irq_set_chip_data(irq
, domain
->host_data
);
602 static const struct irq_domain_ops intx_domain_ops
= {
603 .map
= mtk_pcie_intx_map
,
606 static int mtk_pcie_init_irq_domain(struct mtk_pcie_port
*port
,
607 struct device_node
*node
)
609 struct device
*dev
= port
->pcie
->dev
;
610 struct device_node
*pcie_intc_node
;
614 pcie_intc_node
= of_get_next_child(node
, NULL
);
615 if (!pcie_intc_node
) {
616 dev_err(dev
, "no PCIe Intc node found\n");
620 port
->irq_domain
= irq_domain_add_linear(pcie_intc_node
, PCI_NUM_INTX
,
621 &intx_domain_ops
, port
);
622 if (!port
->irq_domain
) {
623 dev_err(dev
, "failed to get INTx IRQ domain\n");
627 if (IS_ENABLED(CONFIG_PCI_MSI
)) {
628 ret
= mtk_pcie_allocate_msi_domains(port
);
632 mtk_pcie_enable_msi(port
);
638 static void mtk_pcie_intr_handler(struct irq_desc
*desc
)
640 struct mtk_pcie_port
*port
= irq_desc_get_handler_data(desc
);
641 struct irq_chip
*irqchip
= irq_desc_get_chip(desc
);
642 unsigned long status
;
644 u32 bit
= INTX_SHIFT
;
646 chained_irq_enter(irqchip
, desc
);
648 status
= readl(port
->base
+ PCIE_INT_STATUS
);
649 if (status
& INTX_MASK
) {
650 for_each_set_bit_from(bit
, &status
, PCI_NUM_INTX
+ INTX_SHIFT
) {
652 writel(1 << bit
, port
->base
+ PCIE_INT_STATUS
);
653 virq
= irq_find_mapping(port
->irq_domain
,
655 generic_handle_irq(virq
);
659 if (IS_ENABLED(CONFIG_PCI_MSI
)) {
660 if (status
& MSI_STATUS
){
661 unsigned long imsi_status
;
663 while ((imsi_status
= readl(port
->base
+ PCIE_IMSI_STATUS
))) {
664 for_each_set_bit(bit
, &imsi_status
, MTK_MSI_IRQS_NUM
) {
665 virq
= irq_find_mapping(port
->inner_domain
, bit
);
666 generic_handle_irq(virq
);
669 /* Clear MSI interrupt status */
670 writel(MSI_STATUS
, port
->base
+ PCIE_INT_STATUS
);
674 chained_irq_exit(irqchip
, desc
);
679 static int mtk_pcie_setup_irq(struct mtk_pcie_port
*port
,
680 struct device_node
*node
)
682 struct mtk_pcie
*pcie
= port
->pcie
;
683 struct device
*dev
= pcie
->dev
;
684 struct platform_device
*pdev
= to_platform_device(dev
);
687 err
= mtk_pcie_init_irq_domain(port
, node
);
689 dev_err(dev
, "failed to init PCIe IRQ domain\n");
693 irq
= platform_get_irq(pdev
, port
->slot
);
694 irq_set_chained_handler_and_data(irq
, mtk_pcie_intr_handler
, port
);
699 static void __iomem
*mtk_pcie_map_bus(struct pci_bus
*bus
,
700 unsigned int devfn
, int where
)
702 struct mtk_pcie
*pcie
= bus
->sysdata
;
704 writel(PCIE_CONF_ADDR(where
, PCI_FUNC(devfn
), PCI_SLOT(devfn
),
705 bus
->number
), pcie
->base
+ PCIE_CFG_ADDR
);
707 return pcie
->base
+ PCIE_CFG_DATA
+ (where
& 3);
710 static struct pci_ops mtk_pcie_ops
= {
711 .map_bus
= mtk_pcie_map_bus
,
712 .read
= pci_generic_config_read
,
713 .write
= pci_generic_config_write
,
716 static int mtk_pcie_startup_port(struct mtk_pcie_port
*port
)
718 struct mtk_pcie
*pcie
= port
->pcie
;
719 u32 func
= PCI_FUNC(port
->slot
<< 3);
720 u32 slot
= PCI_SLOT(port
->slot
<< 3);
724 /* assert port PERST_N */
725 val
= readl(pcie
->base
+ PCIE_SYS_CFG
);
726 val
|= PCIE_PORT_PERST(port
->slot
);
727 writel(val
, pcie
->base
+ PCIE_SYS_CFG
);
729 /* de-assert port PERST_N */
730 val
= readl(pcie
->base
+ PCIE_SYS_CFG
);
731 val
&= ~PCIE_PORT_PERST(port
->slot
);
732 writel(val
, pcie
->base
+ PCIE_SYS_CFG
);
734 /* 100ms timeout value should be enough for Gen1/2 training */
735 err
= readl_poll_timeout(port
->base
+ PCIE_LINK_STATUS
, val
,
736 !!(val
& PCIE_PORT_LINKUP
), 20,
737 100 * USEC_PER_MSEC
);
741 /* enable interrupt */
742 val
= readl(pcie
->base
+ PCIE_INT_ENABLE
);
743 val
|= PCIE_PORT_INT_EN(port
->slot
);
744 writel(val
, pcie
->base
+ PCIE_INT_ENABLE
);
746 /* map to all DDR region. We need to set it before cfg operation. */
747 writel(PCIE_BAR_MAP_MAX
| PCIE_BAR_ENABLE
,
748 port
->base
+ PCIE_BAR0_SETUP
);
750 /* configure class code and revision ID */
751 writel(PCIE_CLASS_CODE
| PCIE_REVISION_ID
, port
->base
+ PCIE_CLASS
);
753 /* configure FC credit */
754 writel(PCIE_CONF_ADDR(PCIE_FC_CREDIT
, func
, slot
, 0),
755 pcie
->base
+ PCIE_CFG_ADDR
);
756 val
= readl(pcie
->base
+ PCIE_CFG_DATA
);
757 val
&= ~PCIE_FC_CREDIT_MASK
;
758 val
|= PCIE_FC_CREDIT_VAL(0x806c);
759 writel(PCIE_CONF_ADDR(PCIE_FC_CREDIT
, func
, slot
, 0),
760 pcie
->base
+ PCIE_CFG_ADDR
);
761 writel(val
, pcie
->base
+ PCIE_CFG_DATA
);
763 /* configure RC FTS number to 250 when it leaves L0s */
764 writel(PCIE_CONF_ADDR(PCIE_FTS_NUM
, func
, slot
, 0),
765 pcie
->base
+ PCIE_CFG_ADDR
);
766 val
= readl(pcie
->base
+ PCIE_CFG_DATA
);
767 val
&= ~PCIE_FTS_NUM_MASK
;
768 val
|= PCIE_FTS_NUM_L0(0x50);
769 writel(PCIE_CONF_ADDR(PCIE_FTS_NUM
, func
, slot
, 0),
770 pcie
->base
+ PCIE_CFG_ADDR
);
771 writel(val
, pcie
->base
+ PCIE_CFG_DATA
);
776 static void mtk_pcie_enable_port(struct mtk_pcie_port
*port
)
778 struct mtk_pcie
*pcie
= port
->pcie
;
779 struct device
*dev
= pcie
->dev
;
782 err
= clk_prepare_enable(port
->sys_ck
);
784 dev_err(dev
, "failed to enable sys_ck%d clock\n", port
->slot
);
788 err
= clk_prepare_enable(port
->ahb_ck
);
790 dev_err(dev
, "failed to enable ahb_ck%d\n", port
->slot
);
794 err
= clk_prepare_enable(port
->aux_ck
);
796 dev_err(dev
, "failed to enable aux_ck%d\n", port
->slot
);
800 err
= clk_prepare_enable(port
->axi_ck
);
802 dev_err(dev
, "failed to enable axi_ck%d\n", port
->slot
);
806 err
= clk_prepare_enable(port
->obff_ck
);
808 dev_err(dev
, "failed to enable obff_ck%d\n", port
->slot
);
812 err
= clk_prepare_enable(port
->pipe_ck
);
814 dev_err(dev
, "failed to enable pipe_ck%d\n", port
->slot
);
818 reset_control_assert(port
->reset
);
819 reset_control_deassert(port
->reset
);
821 err
= phy_init(port
->phy
);
823 dev_err(dev
, "failed to initialize port%d phy\n", port
->slot
);
827 err
= phy_power_on(port
->phy
);
829 dev_err(dev
, "failed to power on port%d phy\n", port
->slot
);
833 if (!pcie
->soc
->startup(port
))
836 dev_info(dev
, "Port%d link down\n", port
->slot
);
838 phy_power_off(port
->phy
);
842 clk_disable_unprepare(port
->pipe_ck
);
844 clk_disable_unprepare(port
->obff_ck
);
846 clk_disable_unprepare(port
->axi_ck
);
848 clk_disable_unprepare(port
->aux_ck
);
850 clk_disable_unprepare(port
->ahb_ck
);
852 clk_disable_unprepare(port
->sys_ck
);
854 mtk_pcie_port_free(port
);
857 static int mtk_pcie_parse_port(struct mtk_pcie
*pcie
,
858 struct device_node
*node
,
861 struct mtk_pcie_port
*port
;
862 struct resource
*regs
;
863 struct device
*dev
= pcie
->dev
;
864 struct platform_device
*pdev
= to_platform_device(dev
);
868 port
= devm_kzalloc(dev
, sizeof(*port
), GFP_KERNEL
);
872 err
= of_property_read_u32(node
, "num-lanes", &port
->lane
);
874 dev_err(dev
, "missing num-lanes property\n");
878 snprintf(name
, sizeof(name
), "port%d", slot
);
879 regs
= platform_get_resource_byname(pdev
, IORESOURCE_MEM
, name
);
880 port
->base
= devm_ioremap_resource(dev
, regs
);
881 if (IS_ERR(port
->base
)) {
882 dev_err(dev
, "failed to map port%d base\n", slot
);
883 return PTR_ERR(port
->base
);
886 snprintf(name
, sizeof(name
), "sys_ck%d", slot
);
887 port
->sys_ck
= devm_clk_get(dev
, name
);
888 if (IS_ERR(port
->sys_ck
)) {
889 dev_err(dev
, "failed to get sys_ck%d clock\n", slot
);
890 return PTR_ERR(port
->sys_ck
);
893 /* sys_ck might be divided into the following parts in some chips */
894 snprintf(name
, sizeof(name
), "ahb_ck%d", slot
);
895 port
->ahb_ck
= devm_clk_get(dev
, name
);
896 if (IS_ERR(port
->ahb_ck
)) {
897 if (PTR_ERR(port
->ahb_ck
) == -EPROBE_DEFER
)
898 return -EPROBE_DEFER
;
903 snprintf(name
, sizeof(name
), "axi_ck%d", slot
);
904 port
->axi_ck
= devm_clk_get(dev
, name
);
905 if (IS_ERR(port
->axi_ck
)) {
906 if (PTR_ERR(port
->axi_ck
) == -EPROBE_DEFER
)
907 return -EPROBE_DEFER
;
912 snprintf(name
, sizeof(name
), "aux_ck%d", slot
);
913 port
->aux_ck
= devm_clk_get(dev
, name
);
914 if (IS_ERR(port
->aux_ck
)) {
915 if (PTR_ERR(port
->aux_ck
) == -EPROBE_DEFER
)
916 return -EPROBE_DEFER
;
921 snprintf(name
, sizeof(name
), "obff_ck%d", slot
);
922 port
->obff_ck
= devm_clk_get(dev
, name
);
923 if (IS_ERR(port
->obff_ck
)) {
924 if (PTR_ERR(port
->obff_ck
) == -EPROBE_DEFER
)
925 return -EPROBE_DEFER
;
927 port
->obff_ck
= NULL
;
930 snprintf(name
, sizeof(name
), "pipe_ck%d", slot
);
931 port
->pipe_ck
= devm_clk_get(dev
, name
);
932 if (IS_ERR(port
->pipe_ck
)) {
933 if (PTR_ERR(port
->pipe_ck
) == -EPROBE_DEFER
)
934 return -EPROBE_DEFER
;
936 port
->pipe_ck
= NULL
;
939 snprintf(name
, sizeof(name
), "pcie-rst%d", slot
);
940 port
->reset
= devm_reset_control_get_optional_exclusive(dev
, name
);
941 if (PTR_ERR(port
->reset
) == -EPROBE_DEFER
)
942 return PTR_ERR(port
->reset
);
944 /* some platforms may use default PHY setting */
945 snprintf(name
, sizeof(name
), "pcie-phy%d", slot
);
946 port
->phy
= devm_phy_optional_get(dev
, name
);
947 if (IS_ERR(port
->phy
))
948 return PTR_ERR(port
->phy
);
953 if (pcie
->soc
->setup_irq
) {
954 err
= pcie
->soc
->setup_irq(port
, node
);
959 INIT_LIST_HEAD(&port
->list
);
960 list_add_tail(&port
->list
, &pcie
->ports
);
965 static int mtk_pcie_subsys_powerup(struct mtk_pcie
*pcie
)
967 struct device
*dev
= pcie
->dev
;
968 struct platform_device
*pdev
= to_platform_device(dev
);
969 struct resource
*regs
;
972 /* get shared registers, which are optional */
973 regs
= platform_get_resource_byname(pdev
, IORESOURCE_MEM
, "subsys");
975 pcie
->base
= devm_ioremap_resource(dev
, regs
);
976 if (IS_ERR(pcie
->base
)) {
977 dev_err(dev
, "failed to map shared register\n");
978 return PTR_ERR(pcie
->base
);
982 pcie
->free_ck
= devm_clk_get(dev
, "free_ck");
983 if (IS_ERR(pcie
->free_ck
)) {
984 if (PTR_ERR(pcie
->free_ck
) == -EPROBE_DEFER
)
985 return -EPROBE_DEFER
;
987 pcie
->free_ck
= NULL
;
990 if (dev
->pm_domain
) {
991 pm_runtime_enable(dev
);
992 pm_runtime_get_sync(dev
);
995 /* enable top level clock */
996 err
= clk_prepare_enable(pcie
->free_ck
);
998 dev_err(dev
, "failed to enable free_ck\n");
1005 if (dev
->pm_domain
) {
1006 pm_runtime_put_sync(dev
);
1007 pm_runtime_disable(dev
);
1013 static int mtk_pcie_setup(struct mtk_pcie
*pcie
)
1015 struct device
*dev
= pcie
->dev
;
1016 struct device_node
*node
= dev
->of_node
, *child
;
1017 struct of_pci_range_parser parser
;
1018 struct of_pci_range range
;
1019 struct resource res
;
1020 struct mtk_pcie_port
*port
, *tmp
;
1023 if (of_pci_range_parser_init(&parser
, node
)) {
1024 dev_err(dev
, "missing \"ranges\" property\n");
1028 for_each_of_pci_range(&parser
, &range
) {
1029 err
= of_pci_range_to_resource(&range
, node
, &res
);
1033 switch (res
.flags
& IORESOURCE_TYPE_BITS
) {
1035 pcie
->offset
.io
= res
.start
- range
.pci_addr
;
1037 memcpy(&pcie
->pio
, &res
, sizeof(res
));
1038 pcie
->pio
.name
= node
->full_name
;
1040 pcie
->io
.start
= range
.cpu_addr
;
1041 pcie
->io
.end
= range
.cpu_addr
+ range
.size
- 1;
1042 pcie
->io
.flags
= IORESOURCE_MEM
;
1043 pcie
->io
.name
= "I/O";
1045 memcpy(&res
, &pcie
->io
, sizeof(res
));
1048 case IORESOURCE_MEM
:
1049 pcie
->offset
.mem
= res
.start
- range
.pci_addr
;
1051 memcpy(&pcie
->mem
, &res
, sizeof(res
));
1052 pcie
->mem
.name
= "non-prefetchable";
1057 err
= of_pci_parse_bus_range(node
, &pcie
->busn
);
1059 dev_err(dev
, "failed to parse bus ranges property: %d\n", err
);
1060 pcie
->busn
.name
= node
->name
;
1061 pcie
->busn
.start
= 0;
1062 pcie
->busn
.end
= 0xff;
1063 pcie
->busn
.flags
= IORESOURCE_BUS
;
1066 for_each_available_child_of_node(node
, child
) {
1069 err
= of_pci_get_devfn(child
);
1071 dev_err(dev
, "failed to parse devfn: %d\n", err
);
1075 slot
= PCI_SLOT(err
);
1077 err
= mtk_pcie_parse_port(pcie
, child
, slot
);
1082 err
= mtk_pcie_subsys_powerup(pcie
);
1086 /* enable each port, and then check link status */
1087 list_for_each_entry_safe(port
, tmp
, &pcie
->ports
, list
)
1088 mtk_pcie_enable_port(port
);
1090 /* power down PCIe subsys if slots are all empty (link down) */
1091 if (list_empty(&pcie
->ports
))
1092 mtk_pcie_subsys_powerdown(pcie
);
1097 static int mtk_pcie_request_resources(struct mtk_pcie
*pcie
)
1099 struct pci_host_bridge
*host
= pci_host_bridge_from_priv(pcie
);
1100 struct list_head
*windows
= &host
->windows
;
1101 struct device
*dev
= pcie
->dev
;
1104 pci_add_resource_offset(windows
, &pcie
->pio
, pcie
->offset
.io
);
1105 pci_add_resource_offset(windows
, &pcie
->mem
, pcie
->offset
.mem
);
1106 pci_add_resource(windows
, &pcie
->busn
);
1108 err
= devm_request_pci_bus_resources(dev
, windows
);
1112 pci_remap_iospace(&pcie
->pio
, pcie
->io
.start
);
1117 static int mtk_pcie_register_host(struct pci_host_bridge
*host
)
1119 struct mtk_pcie
*pcie
= pci_host_bridge_priv(host
);
1120 struct pci_bus
*child
;
1123 host
->busnr
= pcie
->busn
.start
;
1124 host
->dev
.parent
= pcie
->dev
;
1125 host
->ops
= pcie
->soc
->ops
;
1126 host
->map_irq
= of_irq_parse_and_map_pci
;
1127 host
->swizzle_irq
= pci_common_swizzle
;
1128 host
->sysdata
= pcie
;
1130 err
= pci_scan_root_bus_bridge(host
);
1134 pci_bus_size_bridges(host
->bus
);
1135 pci_bus_assign_resources(host
->bus
);
1137 list_for_each_entry(child
, &host
->bus
->children
, node
)
1138 pcie_bus_configure_settings(child
);
1140 pci_bus_add_devices(host
->bus
);
1145 static int mtk_pcie_probe(struct platform_device
*pdev
)
1147 struct device
*dev
= &pdev
->dev
;
1148 struct mtk_pcie
*pcie
;
1149 struct pci_host_bridge
*host
;
1152 host
= devm_pci_alloc_host_bridge(dev
, sizeof(*pcie
));
1156 pcie
= pci_host_bridge_priv(host
);
1159 pcie
->soc
= of_device_get_match_data(dev
);
1160 platform_set_drvdata(pdev
, pcie
);
1161 INIT_LIST_HEAD(&pcie
->ports
);
1163 err
= mtk_pcie_setup(pcie
);
1167 err
= mtk_pcie_request_resources(pcie
);
1171 err
= mtk_pcie_register_host(host
);
1178 if (!list_empty(&pcie
->ports
))
1179 mtk_pcie_put_resources(pcie
);
1184 static const struct mtk_pcie_soc mtk_pcie_soc_v1
= {
1185 .ops
= &mtk_pcie_ops
,
1186 .startup
= mtk_pcie_startup_port
,
1189 static const struct mtk_pcie_soc mtk_pcie_soc_mt2712
= {
1190 .ops
= &mtk_pcie_ops_v2
,
1191 .startup
= mtk_pcie_startup_port_v2
,
1192 .setup_irq
= mtk_pcie_setup_irq
,
1195 static const struct mtk_pcie_soc mtk_pcie_soc_mt7622
= {
1196 .need_fix_class_id
= true,
1197 .ops
= &mtk_pcie_ops_v2
,
1198 .startup
= mtk_pcie_startup_port_v2
,
1199 .setup_irq
= mtk_pcie_setup_irq
,
1202 static const struct of_device_id mtk_pcie_ids
[] = {
1203 { .compatible
= "mediatek,mt2701-pcie", .data
= &mtk_pcie_soc_v1
},
1204 { .compatible
= "mediatek,mt7623-pcie", .data
= &mtk_pcie_soc_v1
},
1205 { .compatible
= "mediatek,mt2712-pcie", .data
= &mtk_pcie_soc_mt2712
},
1206 { .compatible
= "mediatek,mt7622-pcie", .data
= &mtk_pcie_soc_mt7622
},
1210 static struct platform_driver mtk_pcie_driver
= {
1211 .probe
= mtk_pcie_probe
,
1214 .of_match_table
= mtk_pcie_ids
,
1215 .suppress_bind_attrs
= true,
1218 builtin_platform_driver(mtk_pcie_driver
);