[mirror_ubuntu-jammy-kernel.git] / drivers / pci / dwc / pcie-designware-ep.c

// SPDX-License-Identifier: GPL-2.0
/**
 * Synopsys DesignWare PCIe Endpoint controller driver
 *
 * Copyright (C) 2017 Texas Instruments
 * Author: Kishon Vijay Abraham I <kishon@ti.com>
 */

#include <linux/of.h>

#include "pcie-designware.h"
#include <linux/pci-epc.h>
#include <linux/pci-epf.h>

void dw_pcie_ep_linkup(struct dw_pcie_ep *ep)
{
	struct pci_epc *epc = ep->epc;

	pci_epc_linkup(epc);
}

static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar,
				   int flags)
{
	u32 reg;

	reg = PCI_BASE_ADDRESS_0 + (4 * bar);
	dw_pcie_dbi_ro_wr_en(pci);
	dw_pcie_writel_dbi2(pci, reg, 0x0);
	dw_pcie_writel_dbi(pci, reg, 0x0);
	if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) {
		dw_pcie_writel_dbi2(pci, reg + 4, 0x0);
		dw_pcie_writel_dbi(pci, reg + 4, 0x0);
	}
	dw_pcie_dbi_ro_wr_dis(pci);
}

void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar)
{
	__dw_pcie_ep_reset_bar(pci, bar, 0);
}

static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no,
				   struct pci_epf_header *hdr)
{
	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);

	dw_pcie_dbi_ro_wr_en(pci);
	dw_pcie_writew_dbi(pci, PCI_VENDOR_ID, hdr->vendorid);
	dw_pcie_writew_dbi(pci, PCI_DEVICE_ID, hdr->deviceid);
	dw_pcie_writeb_dbi(pci, PCI_REVISION_ID, hdr->revid);
	dw_pcie_writeb_dbi(pci, PCI_CLASS_PROG, hdr->progif_code);
	dw_pcie_writew_dbi(pci, PCI_CLASS_DEVICE,
			   hdr->subclass_code | hdr->baseclass_code << 8);
	dw_pcie_writeb_dbi(pci, PCI_CACHE_LINE_SIZE,
			   hdr->cache_line_size);
	dw_pcie_writew_dbi(pci, PCI_SUBSYSTEM_VENDOR_ID,
			   hdr->subsys_vendor_id);
	dw_pcie_writew_dbi(pci, PCI_SUBSYSTEM_ID, hdr->subsys_id);
	dw_pcie_writeb_dbi(pci, PCI_INTERRUPT_PIN,
			   hdr->interrupt_pin);
	dw_pcie_dbi_ro_wr_dis(pci);

	return 0;
}

static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, enum pci_barno bar,
				  dma_addr_t cpu_addr,
				  enum dw_pcie_as_type as_type)
{
	int ret;
	u32 free_win;
	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);

	free_win = find_first_zero_bit(ep->ib_window_map, ep->num_ib_windows);
	if (free_win >= ep->num_ib_windows) {
		dev_err(pci->dev, "No free inbound window\n");
		return -EINVAL;
	}

	ret = dw_pcie_prog_inbound_atu(pci, free_win, bar, cpu_addr,
				       as_type);
	if (ret < 0) {
		dev_err(pci->dev, "Failed to program IB window\n");
		return ret;
	}

	ep->bar_to_atu[bar] = free_win;
	set_bit(free_win, ep->ib_window_map);

	return 0;
}

static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, phys_addr_t phys_addr,
				   u64 pci_addr, size_t size)
{
	u32 free_win;
	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);

	free_win = find_first_zero_bit(ep->ob_window_map, ep->num_ob_windows);
	if (free_win >= ep->num_ob_windows) {
		dev_err(pci->dev, "No free outbound window\n");
		return -EINVAL;
	}

	dw_pcie_prog_outbound_atu(pci, free_win, PCIE_ATU_TYPE_MEM,
				  phys_addr, pci_addr, size);

	set_bit(free_win, ep->ob_window_map);
	ep->outbound_addr[free_win] = phys_addr;

	return 0;
}

static void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no,
				 struct pci_epf_bar *epf_bar)
{
	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
	enum pci_barno bar = epf_bar->barno;
	u32 atu_index = ep->bar_to_atu[bar];

	__dw_pcie_ep_reset_bar(pci, bar, epf_bar->flags);

	dw_pcie_disable_atu(pci, atu_index, DW_PCIE_REGION_INBOUND);
	clear_bit(atu_index, ep->ib_window_map);
}

static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no,
			      struct pci_epf_bar *epf_bar)
{
	int ret;
	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
	enum pci_barno bar = epf_bar->barno;
	size_t size = epf_bar->size;
	int flags = epf_bar->flags;
	enum dw_pcie_as_type as_type;
	u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar);

	if (!(flags & PCI_BASE_ADDRESS_SPACE))
		as_type = DW_PCIE_AS_MEM;
	else
		as_type = DW_PCIE_AS_IO;

	ret = dw_pcie_ep_inbound_atu(ep, bar, epf_bar->phys_addr, as_type);
	if (ret)
		return ret;

	dw_pcie_dbi_ro_wr_en(pci);

	dw_pcie_writel_dbi2(pci, reg, lower_32_bits(size - 1));
	dw_pcie_writel_dbi(pci, reg, flags);

	if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) {
		dw_pcie_writel_dbi2(pci, reg + 4, upper_32_bits(size - 1));
		dw_pcie_writel_dbi(pci, reg + 4, 0);
	}

	dw_pcie_dbi_ro_wr_dis(pci);

	return 0;
}

static int dw_pcie_find_index(struct dw_pcie_ep *ep, phys_addr_t addr,
			      u32 *atu_index)
{
	u32 index;

	for (index = 0; index < ep->num_ob_windows; index++) {
		if (ep->outbound_addr[index] != addr)
			continue;
		*atu_index = index;
		return 0;
	}

	return -EINVAL;
}

static void dw_pcie_ep_unmap_addr(struct pci_epc *epc, u8 func_no,
				  phys_addr_t addr)
{
	int ret;
	u32 atu_index;
	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);

	ret = dw_pcie_find_index(ep, addr, &atu_index);
	if (ret < 0)
		return;

	dw_pcie_disable_atu(pci, atu_index, DW_PCIE_REGION_OUTBOUND);
	clear_bit(atu_index, ep->ob_window_map);
}

static int dw_pcie_ep_map_addr(struct pci_epc *epc, u8 func_no,
			       phys_addr_t addr,
			       u64 pci_addr, size_t size)
{
	int ret;
	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);

	ret = dw_pcie_ep_outbound_atu(ep, addr, pci_addr, size);
	if (ret) {
		dev_err(pci->dev, "Failed to enable address\n");
		return ret;
	}

	return 0;
}

static int dw_pcie_ep_get_msi(struct pci_epc *epc, u8 func_no)
{
	int val;
	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);

	val = dw_pcie_readw_dbi(pci, MSI_MESSAGE_CONTROL);
	if (!(val & MSI_CAP_MSI_EN_MASK))
		return -EINVAL;

	val = (val & MSI_CAP_MME_MASK) >> MSI_CAP_MME_SHIFT;
	return val;
}

static int dw_pcie_ep_set_msi(struct pci_epc *epc, u8 func_no, u8 encode_int)
{
	int val;
	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);

	val = dw_pcie_readw_dbi(pci, MSI_MESSAGE_CONTROL);
	val &= ~MSI_CAP_MMC_MASK;
	val |= (encode_int << MSI_CAP_MMC_SHIFT) & MSI_CAP_MMC_MASK;
	dw_pcie_dbi_ro_wr_en(pci);
	dw_pcie_writew_dbi(pci, MSI_MESSAGE_CONTROL, val);
	dw_pcie_dbi_ro_wr_dis(pci);

	return 0;
}

static int dw_pcie_ep_raise_irq(struct pci_epc *epc, u8 func_no,
				enum pci_epc_irq_type type, u8 interrupt_num)
{
	struct dw_pcie_ep *ep = epc_get_drvdata(epc);

	if (!ep->ops->raise_irq)
		return -EINVAL;

	return ep->ops->raise_irq(ep, func_no, type, interrupt_num);
}

static void dw_pcie_ep_stop(struct pci_epc *epc)
{
	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);

	if (!pci->ops->stop_link)
		return;

	pci->ops->stop_link(pci);
}

static int dw_pcie_ep_start(struct pci_epc *epc)
{
	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);

	if (!pci->ops->start_link)
		return -EINVAL;

	return pci->ops->start_link(pci);
}

static const struct pci_epc_ops epc_ops = {
	.write_header		= dw_pcie_ep_write_header,
	.set_bar		= dw_pcie_ep_set_bar,
	.clear_bar		= dw_pcie_ep_clear_bar,
	.map_addr		= dw_pcie_ep_map_addr,
	.unmap_addr		= dw_pcie_ep_unmap_addr,
	.set_msi		= dw_pcie_ep_set_msi,
	.get_msi		= dw_pcie_ep_get_msi,
	.raise_irq		= dw_pcie_ep_raise_irq,
	.start			= dw_pcie_ep_start,
	.stop			= dw_pcie_ep_stop,
};

int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no,
			     u8 interrupt_num)
{
	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
	struct pci_epc *epc = ep->epc;
	u16 msg_ctrl, msg_data;
	u32 msg_addr_lower, msg_addr_upper;
	u64 msg_addr;
	bool has_upper;
	int ret;

	/* Raise MSI per the PCI Local Bus Specification Revision 3.0, 6.8.1. */
	msg_ctrl = dw_pcie_readw_dbi(pci, MSI_MESSAGE_CONTROL);
	has_upper = !!(msg_ctrl & PCI_MSI_FLAGS_64BIT);
	msg_addr_lower = dw_pcie_readl_dbi(pci, MSI_MESSAGE_ADDR_L32);
	if (has_upper) {
		msg_addr_upper = dw_pcie_readl_dbi(pci, MSI_MESSAGE_ADDR_U32);
		msg_data = dw_pcie_readw_dbi(pci, MSI_MESSAGE_DATA_64);
	} else {
		msg_addr_upper = 0;
		msg_data = dw_pcie_readw_dbi(pci, MSI_MESSAGE_DATA_32);
	}
	msg_addr = ((u64) msg_addr_upper) << 32 | msg_addr_lower;
	ret = dw_pcie_ep_map_addr(epc, func_no, ep->msi_mem_phys, msg_addr,
				  epc->mem->page_size);
	if (ret)
		return ret;

	writel(msg_data | (interrupt_num - 1), ep->msi_mem);

	dw_pcie_ep_unmap_addr(epc, func_no, ep->msi_mem_phys);

	return 0;
}

void dw_pcie_ep_exit(struct dw_pcie_ep *ep)
{
	struct pci_epc *epc = ep->epc;

	pci_epc_mem_free_addr(epc, ep->msi_mem_phys, ep->msi_mem,
			      epc->mem->page_size);

	pci_epc_mem_exit(epc);
}

int dw_pcie_ep_init(struct dw_pcie_ep *ep)
{
	int ret;
	void *addr;
	struct pci_epc *epc;
	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
	struct device *dev = pci->dev;
	struct device_node *np = dev->of_node;

	if (!pci->dbi_base || !pci->dbi_base2) {
		dev_err(dev, "dbi_base/dbi_base2 is not populated\n");
		return -EINVAL;
	}

	ret = of_property_read_u32(np, "num-ib-windows", &ep->num_ib_windows);
	if (ret < 0) {
		dev_err(dev, "Unable to read *num-ib-windows* property\n");
		return ret;
	}
	if (ep->num_ib_windows > MAX_IATU_IN) {
		dev_err(dev, "Invalid *num-ib-windows*\n");
		return -EINVAL;
	}

	ret = of_property_read_u32(np, "num-ob-windows", &ep->num_ob_windows);
	if (ret < 0) {
		dev_err(dev, "Unable to read *num-ob-windows* property\n");
		return ret;
	}
	if (ep->num_ob_windows > MAX_IATU_OUT) {
		dev_err(dev, "Invalid *num-ob-windows*\n");
		return -EINVAL;
	}

	ep->ib_window_map = devm_kcalloc(dev,
					 BITS_TO_LONGS(ep->num_ib_windows),
					 sizeof(long),
					 GFP_KERNEL);
	if (!ep->ib_window_map)
		return -ENOMEM;

	ep->ob_window_map = devm_kcalloc(dev,
					 BITS_TO_LONGS(ep->num_ob_windows),
					 sizeof(long),
					 GFP_KERNEL);
	if (!ep->ob_window_map)
		return -ENOMEM;

	addr = devm_kcalloc(dev, ep->num_ob_windows, sizeof(phys_addr_t),
			    GFP_KERNEL);
	if (!addr)
		return -ENOMEM;
	ep->outbound_addr = addr;

	if (ep->ops->ep_init)
		ep->ops->ep_init(ep);

	epc = devm_pci_epc_create(dev, &epc_ops);
	if (IS_ERR(epc)) {
		dev_err(dev, "Failed to create epc device\n");
		return PTR_ERR(epc);
	}

	ret = of_property_read_u8(np, "max-functions", &epc->max_functions);
	if (ret < 0)
		epc->max_functions = 1;

	ret = __pci_epc_mem_init(epc, ep->phys_base, ep->addr_size,
				 ep->page_size);
	if (ret < 0) {
		dev_err(dev, "Failed to initialize address space\n");
		return ret;
	}

	ep->msi_mem = pci_epc_mem_alloc_addr(epc, &ep->msi_mem_phys,
					     epc->mem->page_size);
	if (!ep->msi_mem) {
		dev_err(dev, "Failed to reserve memory for MSI\n");
		return -ENOMEM;
	}

	epc->features = EPC_FEATURE_NO_LINKUP_NOTIFIER;
	EPC_FEATURE_SET_BAR(epc->features, BAR_0);

	ep->epc = epc;
	epc_set_drvdata(epc, ep);
	dw_pcie_setup(pci);

	return 0;
}
Commit	Line	Data
8cfab3cf	1	// SPDX-License-Identifier: GPL-2.0
f8aed6ec	2	/**
96291d56	3	* Synopsys DesignWare PCIe Endpoint controller driver
f8aed6ec KVA	4	*
	5	* Copyright (C) 2017 Texas Instruments
	6	* Author: Kishon Vijay Abraham I <kishon@ti.com>
f8aed6ec KVA	7	*/
	8
	9	#include <linux/of.h>
	10
	11	#include "pcie-designware.h"
	12	#include <linux/pci-epc.h>
	13	#include <linux/pci-epf.h>
	14
	15	void dw_pcie_ep_linkup(struct dw_pcie_ep *ep)
	16	{
	17	struct pci_epc *epc = ep->epc;
	18
	19	pci_epc_linkup(epc);
	20	}
	21
77d08dbd NC	22	static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar,
77d08dbd NC	23	int flags)
f8aed6ec KVA	24	{
	25	u32 reg;
	26
	27	reg = PCI_BASE_ADDRESS_0 + (4 * bar);
1cab826b	28	dw_pcie_dbi_ro_wr_en(pci);
f8aed6ec KVA	29	dw_pcie_writel_dbi2(pci, reg, 0x0);
f8aed6ec KVA	30	dw_pcie_writel_dbi(pci, reg, 0x0);
96a3be43 NC	31	if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) {
	32	dw_pcie_writel_dbi2(pci, reg + 4, 0x0);
	33	dw_pcie_writel_dbi(pci, reg + 4, 0x0);
	34	}
1cab826b	35	dw_pcie_dbi_ro_wr_dis(pci);
f8aed6ec KVA	36	}
f8aed6ec KVA	37
77d08dbd NC	38	void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar)
	39	{
	40	__dw_pcie_ep_reset_bar(pci, bar, 0);
	41	}
	42
4494738d	43	static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no,
f8aed6ec KVA	44	struct pci_epf_header *hdr)
	45	{
	46	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
	47	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
	48
1cab826b	49	dw_pcie_dbi_ro_wr_en(pci);
f8aed6ec KVA	50	dw_pcie_writew_dbi(pci, PCI_VENDOR_ID, hdr->vendorid);
	51	dw_pcie_writew_dbi(pci, PCI_DEVICE_ID, hdr->deviceid);
	52	dw_pcie_writeb_dbi(pci, PCI_REVISION_ID, hdr->revid);
	53	dw_pcie_writeb_dbi(pci, PCI_CLASS_PROG, hdr->progif_code);
	54	dw_pcie_writew_dbi(pci, PCI_CLASS_DEVICE,
	55	hdr->subclass_code \| hdr->baseclass_code << 8);
	56	dw_pcie_writeb_dbi(pci, PCI_CACHE_LINE_SIZE,
	57	hdr->cache_line_size);
	58	dw_pcie_writew_dbi(pci, PCI_SUBSYSTEM_VENDOR_ID,
	59	hdr->subsys_vendor_id);
	60	dw_pcie_writew_dbi(pci, PCI_SUBSYSTEM_ID, hdr->subsys_id);
	61	dw_pcie_writeb_dbi(pci, PCI_INTERRUPT_PIN,
	62	hdr->interrupt_pin);
1cab826b	63	dw_pcie_dbi_ro_wr_dis(pci);
f8aed6ec KVA	64
	65	return 0;
	66	}
	67
	68	static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, enum pci_barno bar,
	69	dma_addr_t cpu_addr,
	70	enum dw_pcie_as_type as_type)
	71	{
	72	int ret;
	73	u32 free_win;
	74	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
	75
ad4a5bec	76	free_win = find_first_zero_bit(ep->ib_window_map, ep->num_ib_windows);
f8aed6ec	77	if (free_win >= ep->num_ib_windows) {
b4a8a51c	78	dev_err(pci->dev, "No free inbound window\n");
f8aed6ec KVA	79	return -EINVAL;
	80	}
	81
	82	ret = dw_pcie_prog_inbound_atu(pci, free_win, bar, cpu_addr,
	83	as_type);
	84	if (ret < 0) {
	85	dev_err(pci->dev, "Failed to program IB window\n");
	86	return ret;
	87	}
	88
	89	ep->bar_to_atu[bar] = free_win;
ad4a5bec	90	set_bit(free_win, ep->ib_window_map);
f8aed6ec KVA	91
	92	return 0;
	93	}
	94
	95	static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, phys_addr_t phys_addr,
	96	u64 pci_addr, size_t size)
	97	{
	98	u32 free_win;
	99	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
	100
ad4a5bec	101	free_win = find_first_zero_bit(ep->ob_window_map, ep->num_ob_windows);
f8aed6ec	102	if (free_win >= ep->num_ob_windows) {
b4a8a51c	103	dev_err(pci->dev, "No free outbound window\n");
f8aed6ec KVA	104	return -EINVAL;
	105	}
	106
	107	dw_pcie_prog_outbound_atu(pci, free_win, PCIE_ATU_TYPE_MEM,
	108	phys_addr, pci_addr, size);
	109
ad4a5bec	110	set_bit(free_win, ep->ob_window_map);
f8aed6ec KVA	111	ep->outbound_addr[free_win] = phys_addr;
	112
	113	return 0;
	114	}
	115
4494738d	116	static void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no,
77d08dbd	117	struct pci_epf_bar *epf_bar)
f8aed6ec KVA	118	{
	119	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
	120	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
77d08dbd	121	enum pci_barno bar = epf_bar->barno;
f8aed6ec KVA	122	u32 atu_index = ep->bar_to_atu[bar];
f8aed6ec KVA	123
77d08dbd	124	__dw_pcie_ep_reset_bar(pci, bar, epf_bar->flags);
f8aed6ec KVA	125
f8aed6ec KVA	126	dw_pcie_disable_atu(pci, atu_index, DW_PCIE_REGION_INBOUND);
ad4a5bec	127	clear_bit(atu_index, ep->ib_window_map);
f8aed6ec KVA	128	}
f8aed6ec KVA	129
4494738d	130	static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no,
bc4a4897	131	struct pci_epf_bar *epf_bar)
f8aed6ec KVA	132	{
	133	int ret;
	134	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
	135	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
bc4a4897 NC	136	enum pci_barno bar = epf_bar->barno;
	137	size_t size = epf_bar->size;
	138	int flags = epf_bar->flags;
f8aed6ec KVA	139	enum dw_pcie_as_type as_type;
	140	u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar);
	141
	142	if (!(flags & PCI_BASE_ADDRESS_SPACE))
	143	as_type = DW_PCIE_AS_MEM;
	144	else
	145	as_type = DW_PCIE_AS_IO;
	146
bc4a4897	147	ret = dw_pcie_ep_inbound_atu(ep, bar, epf_bar->phys_addr, as_type);
f8aed6ec KVA	148	if (ret)
	149	return ret;
	150
1cab826b	151	dw_pcie_dbi_ro_wr_en(pci);
d28810ba NC	152
d28810ba NC	153	dw_pcie_writel_dbi2(pci, reg, lower_32_bits(size - 1));
f8aed6ec	154	dw_pcie_writel_dbi(pci, reg, flags);
d28810ba NC	155
	156	if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) {
	157	dw_pcie_writel_dbi2(pci, reg + 4, upper_32_bits(size - 1));
	158	dw_pcie_writel_dbi(pci, reg + 4, 0);
	159	}
	160
1cab826b	161	dw_pcie_dbi_ro_wr_dis(pci);
f8aed6ec KVA	162
	163	return 0;
	164	}
	165
	166	static int dw_pcie_find_index(struct dw_pcie_ep *ep, phys_addr_t addr,
	167	u32 *atu_index)
	168	{
	169	u32 index;
	170
	171	for (index = 0; index < ep->num_ob_windows; index++) {
	172	if (ep->outbound_addr[index] != addr)
	173	continue;
	174	*atu_index = index;
	175	return 0;
	176	}
	177
	178	return -EINVAL;
	179	}
	180
4494738d CP	181	static void dw_pcie_ep_unmap_addr(struct pci_epc *epc, u8 func_no,
4494738d CP	182	phys_addr_t addr)
f8aed6ec KVA	183	{
	184	int ret;
	185	u32 atu_index;
	186	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
	187	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
	188
	189	ret = dw_pcie_find_index(ep, addr, &atu_index);
	190	if (ret < 0)
	191	return;
	192
	193	dw_pcie_disable_atu(pci, atu_index, DW_PCIE_REGION_OUTBOUND);
ad4a5bec	194	clear_bit(atu_index, ep->ob_window_map);
f8aed6ec KVA	195	}
f8aed6ec KVA	196
4494738d CP	197	static int dw_pcie_ep_map_addr(struct pci_epc *epc, u8 func_no,
4494738d CP	198	phys_addr_t addr,
f8aed6ec KVA	199	u64 pci_addr, size_t size)
	200	{
	201	int ret;
	202	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
	203	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
	204
	205	ret = dw_pcie_ep_outbound_atu(ep, addr, pci_addr, size);
	206	if (ret) {
b4a8a51c	207	dev_err(pci->dev, "Failed to enable address\n");
f8aed6ec KVA	208	return ret;
	209	}
	210
	211	return 0;
	212	}
	213
4494738d	214	static int dw_pcie_ep_get_msi(struct pci_epc *epc, u8 func_no)
f8aed6ec KVA	215	{
f8aed6ec KVA	216	int val;
f8aed6ec KVA	217	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
	218	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
	219
a134a457 KVA	220	val = dw_pcie_readw_dbi(pci, MSI_MESSAGE_CONTROL);
a134a457 KVA	221	if (!(val & MSI_CAP_MSI_EN_MASK))
f8aed6ec KVA	222	return -EINVAL;
f8aed6ec KVA	223
a134a457	224	val = (val & MSI_CAP_MME_MASK) >> MSI_CAP_MME_SHIFT;
f8aed6ec KVA	225	return val;
	226	}
	227
4494738d	228	static int dw_pcie_ep_set_msi(struct pci_epc *epc, u8 func_no, u8 encode_int)
f8aed6ec KVA	229	{
	230	int val;
	231	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
	232	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
	233
099a95f3 NC	234	val = dw_pcie_readw_dbi(pci, MSI_MESSAGE_CONTROL);
	235	val &= ~MSI_CAP_MMC_MASK;
	236	val \|= (encode_int << MSI_CAP_MMC_SHIFT) & MSI_CAP_MMC_MASK;
1cab826b	237	dw_pcie_dbi_ro_wr_en(pci);
f8aed6ec	238	dw_pcie_writew_dbi(pci, MSI_MESSAGE_CONTROL, val);
1cab826b	239	dw_pcie_dbi_ro_wr_dis(pci);
f8aed6ec KVA	240
	241	return 0;
	242	}
	243
4494738d	244	static int dw_pcie_ep_raise_irq(struct pci_epc *epc, u8 func_no,
f8aed6ec KVA	245	enum pci_epc_irq_type type, u8 interrupt_num)
	246	{
	247	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
	248
	249	if (!ep->ops->raise_irq)
	250	return -EINVAL;
	251
16093362	252	return ep->ops->raise_irq(ep, func_no, type, interrupt_num);
f8aed6ec KVA	253	}
	254
	255	static void dw_pcie_ep_stop(struct pci_epc *epc)
	256	{
	257	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
	258	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
	259
	260	if (!pci->ops->stop_link)
	261	return;
	262
	263	pci->ops->stop_link(pci);
	264	}
	265
	266	static int dw_pcie_ep_start(struct pci_epc *epc)
	267	{
	268	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
	269	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
	270
	271	if (!pci->ops->start_link)
	272	return -EINVAL;
	273
	274	return pci->ops->start_link(pci);
	275	}
	276
	277	static const struct pci_epc_ops epc_ops = {
	278	.write_header = dw_pcie_ep_write_header,
	279	.set_bar = dw_pcie_ep_set_bar,
	280	.clear_bar = dw_pcie_ep_clear_bar,
	281	.map_addr = dw_pcie_ep_map_addr,
	282	.unmap_addr = dw_pcie_ep_unmap_addr,
	283	.set_msi = dw_pcie_ep_set_msi,
	284	.get_msi = dw_pcie_ep_get_msi,
	285	.raise_irq = dw_pcie_ep_raise_irq,
	286	.start = dw_pcie_ep_start,
	287	.stop = dw_pcie_ep_stop,
	288	};
	289
16093362	290	int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no,
6f6d7873 NC	291	u8 interrupt_num)
	292	{
	293	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
	294	struct pci_epc *epc = ep->epc;
	295	u16 msg_ctrl, msg_data;
	296	u32 msg_addr_lower, msg_addr_upper;
	297	u64 msg_addr;
	298	bool has_upper;
	299	int ret;
	300
	301	/* Raise MSI per the PCI Local Bus Specification Revision 3.0, 6.8.1. */
	302	msg_ctrl = dw_pcie_readw_dbi(pci, MSI_MESSAGE_CONTROL);
	303	has_upper = !!(msg_ctrl & PCI_MSI_FLAGS_64BIT);
	304	msg_addr_lower = dw_pcie_readl_dbi(pci, MSI_MESSAGE_ADDR_L32);
	305	if (has_upper) {
	306	msg_addr_upper = dw_pcie_readl_dbi(pci, MSI_MESSAGE_ADDR_U32);
	307	msg_data = dw_pcie_readw_dbi(pci, MSI_MESSAGE_DATA_64);
	308	} else {
	309	msg_addr_upper = 0;
	310	msg_data = dw_pcie_readw_dbi(pci, MSI_MESSAGE_DATA_32);
	311	}
	312	msg_addr = ((u64) msg_addr_upper) << 32 \| msg_addr_lower;
16093362	313	ret = dw_pcie_ep_map_addr(epc, func_no, ep->msi_mem_phys, msg_addr,
6f6d7873 NC	314	epc->mem->page_size);
	315	if (ret)
	316	return ret;
	317
	318	writel(msg_data \| (interrupt_num - 1), ep->msi_mem);
	319
16093362	320	dw_pcie_ep_unmap_addr(epc, func_no, ep->msi_mem_phys);
6f6d7873 NC	321
	322	return 0;
	323	}
	324
f8aed6ec KVA	325	void dw_pcie_ep_exit(struct dw_pcie_ep *ep)
	326	{
	327	struct pci_epc *epc = ep->epc;
	328
2fd0c9d9 NC	329	pci_epc_mem_free_addr(epc, ep->msi_mem_phys, ep->msi_mem,
	330	epc->mem->page_size);
	331
f8aed6ec KVA	332	pci_epc_mem_exit(epc);
	333	}
	334
	335	int dw_pcie_ep_init(struct dw_pcie_ep *ep)
	336	{
	337	int ret;
	338	void *addr;
f8aed6ec KVA	339	struct pci_epc *epc;
	340	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
	341	struct device *dev = pci->dev;
	342	struct device_node *np = dev->of_node;
	343
	344	if (!pci->dbi_base \|\| !pci->dbi_base2) {
ae15d863	345	dev_err(dev, "dbi_base/dbi_base2 is not populated\n");
f8aed6ec KVA	346	return -EINVAL;
	347	}
	348
	349	ret = of_property_read_u32(np, "num-ib-windows", &ep->num_ib_windows);
	350	if (ret < 0) {
b4a8a51c	351	dev_err(dev, "Unable to read num-ib-windows property\n");
f8aed6ec KVA	352	return ret;
f8aed6ec KVA	353	}
ad4a5bec	354	if (ep->num_ib_windows > MAX_IATU_IN) {
b4a8a51c	355	dev_err(dev, "Invalid num-ib-windows\n");
ad4a5bec NC	356	return -EINVAL;
ad4a5bec NC	357	}
f8aed6ec KVA	358
	359	ret = of_property_read_u32(np, "num-ob-windows", &ep->num_ob_windows);
	360	if (ret < 0) {
b4a8a51c	361	dev_err(dev, "Unable to read num-ob-windows property\n");
f8aed6ec KVA	362	return ret;
f8aed6ec KVA	363	}
ad4a5bec	364	if (ep->num_ob_windows > MAX_IATU_OUT) {
b4a8a51c	365	dev_err(dev, "Invalid num-ob-windows\n");
ad4a5bec NC	366	return -EINVAL;
	367	}
	368
a86854d0	369	ep->ib_window_map = devm_kcalloc(dev,
ad4a5bec	370	BITS_TO_LONGS(ep->num_ib_windows),
a86854d0	371	sizeof(long),
ad4a5bec NC	372	GFP_KERNEL);
	373	if (!ep->ib_window_map)
	374	return -ENOMEM;
	375
a86854d0	376	ep->ob_window_map = devm_kcalloc(dev,
ad4a5bec	377	BITS_TO_LONGS(ep->num_ob_windows),
a86854d0	378	sizeof(long),
ad4a5bec NC	379	GFP_KERNEL);
	380	if (!ep->ob_window_map)
	381	return -ENOMEM;
f8aed6ec	382
a86854d0	383	addr = devm_kcalloc(dev, ep->num_ob_windows, sizeof(phys_addr_t),
f8aed6ec KVA	384	GFP_KERNEL);
	385	if (!addr)
	386	return -ENOMEM;
	387	ep->outbound_addr = addr;
	388
f8aed6ec KVA	389	if (ep->ops->ep_init)
	390	ep->ops->ep_init(ep);
	391
	392	epc = devm_pci_epc_create(dev, &epc_ops);
	393	if (IS_ERR(epc)) {
b4a8a51c	394	dev_err(dev, "Failed to create epc device\n");
f8aed6ec KVA	395	return PTR_ERR(epc);
	396	}
	397
	398	ret = of_property_read_u8(np, "max-functions", &epc->max_functions);
	399	if (ret < 0)
	400	epc->max_functions = 1;
	401
a937fe08 KVA	402	ret = __pci_epc_mem_init(epc, ep->phys_base, ep->addr_size,
a937fe08 KVA	403	ep->page_size);
f8aed6ec KVA	404	if (ret < 0) {
	405	dev_err(dev, "Failed to initialize address space\n");
	406	return ret;
	407	}
	408
2fd0c9d9 NC	409	ep->msi_mem = pci_epc_mem_alloc_addr(epc, &ep->msi_mem_phys,
	410	epc->mem->page_size);
	411	if (!ep->msi_mem) {
	412	dev_err(dev, "Failed to reserve memory for MSI\n");
	413	return -ENOMEM;
	414	}
	415
1d906b22 GP	416	epc->features = EPC_FEATURE_NO_LINKUP_NOTIFIER;
	417	EPC_FEATURE_SET_BAR(epc->features, BAR_0);
	418
f8aed6ec KVA	419	ep->epc = epc;
	420	epc_set_drvdata(epc, ep);
	421	dw_pcie_setup(pci);
	422
	423	return 0;
	424	}