[mirror_ubuntu-bionic-kernel.git] / arch / powerpc / platforms / powernv / ocxl.c

// SPDX-License-Identifier: GPL-2.0+
// Copyright 2017 IBM Corp.
#include <asm/pnv-ocxl.h>
#include <asm/opal.h>
#include <asm/xive.h>
#include <misc/ocxl-config.h>
#include "pci.h"

#define PNV_OCXL_TL_P9_RECV_CAP		0x000000000000000Full
#define PNV_OCXL_ACTAG_MAX		64
/* PASIDs are 20-bit, but on P9, NPU can only handle 15 bits */
#define PNV_OCXL_PASID_BITS		15
#define PNV_OCXL_PASID_MAX		((1 << PNV_OCXL_PASID_BITS) - 1)

#define AFU_PRESENT (1 << 31)
#define AFU_INDEX_MASK 0x3F000000
#define AFU_INDEX_SHIFT 24
#define ACTAG_MASK 0xFFF


struct actag_range {
	u16 start;
	u16 count;
};

struct npu_link {
	struct list_head list;
	int domain;
	int bus;
	int dev;
	u16 fn_desired_actags[8];
	struct actag_range fn_actags[8];
	bool assignment_done;
};
static struct list_head links_list = LIST_HEAD_INIT(links_list);
static DEFINE_MUTEX(links_list_lock);


/*
 * opencapi actags handling:
 *
 * When sending commands, the opencapi device references the memory
 * context it's targeting with an 'actag', which is really an alias
 * for a (BDF, pasid) combination. When it receives a command, the NPU
 * must do a lookup of the actag to identify the memory context. The
 * hardware supports a finite number of actags per link (64 for
 * POWER9).
 *
 * The device can carry multiple functions, and each function can have
 * multiple AFUs. Each AFU advertises in its config space the number
 * of desired actags. The host must configure in the config space of
 * the AFU how many actags the AFU is really allowed to use (which can
 * be less than what the AFU desires).
 *
 * When a PCI function is probed by the driver, it has no visibility
 * about the other PCI functions and how many actags they'd like,
 * which makes it impossible to distribute actags fairly among AFUs.
 *
 * Unfortunately, the only way to know how many actags a function
 * desires is by looking at the data for each AFU in the config space
 * and add them up. Similarly, the only way to know how many actags
 * all the functions of the physical device desire is by adding the
 * previously computed function counts. Then we can match that against
 * what the hardware supports.
 *
 * To get a comprehensive view, we use a 'pci fixup': at the end of
 * PCI enumeration, each function counts how many actags its AFUs
 * desire and we save it in a 'npu_link' structure, shared between all
 * the PCI functions of a same device. Therefore, when the first
 * function is probed by the driver, we can get an idea of the total
 * count of desired actags for the device, and assign the actags to
 * the AFUs, by pro-rating if needed.
 */

static int find_dvsec_from_pos(struct pci_dev *dev, int dvsec_id, int pos)
{
	int vsec = pos;
	u16 vendor, id;

	while ((vsec = pci_find_next_ext_capability(dev, vsec,
						    OCXL_EXT_CAP_ID_DVSEC))) {
		pci_read_config_word(dev, vsec + OCXL_DVSEC_VENDOR_OFFSET,
				&vendor);
		pci_read_config_word(dev, vsec + OCXL_DVSEC_ID_OFFSET, &id);
		if (vendor == PCI_VENDOR_ID_IBM && id == dvsec_id)
			return vsec;
	}
	return 0;
}

static int find_dvsec_afu_ctrl(struct pci_dev *dev, u8 afu_idx)
{
	int vsec = 0;
	u8 idx;

	while ((vsec = find_dvsec_from_pos(dev, OCXL_DVSEC_AFU_CTRL_ID,
					   vsec))) {
		pci_read_config_byte(dev, vsec + OCXL_DVSEC_AFU_CTRL_AFU_IDX,
				&idx);
		if (idx == afu_idx)
			return vsec;
	}
	return 0;
}

static int get_max_afu_index(struct pci_dev *dev, int *afu_idx)
{
	int pos;
	u32 val;

	pos = find_dvsec_from_pos(dev, OCXL_DVSEC_FUNC_ID, 0);
	if (!pos)
		return -ESRCH;

	pci_read_config_dword(dev, pos + OCXL_DVSEC_FUNC_OFF_INDEX, &val);
	if (val & AFU_PRESENT)
		*afu_idx = (val & AFU_INDEX_MASK) >> AFU_INDEX_SHIFT;
	else
		*afu_idx = -1;
	return 0;
}

static int get_actag_count(struct pci_dev *dev, int afu_idx, int *actag)
{
	int pos;
	u16 actag_sup;

	pos = find_dvsec_afu_ctrl(dev, afu_idx);
	if (!pos)
		return -ESRCH;

	pci_read_config_word(dev, pos + OCXL_DVSEC_AFU_CTRL_ACTAG_SUP,
			&actag_sup);
	*actag = actag_sup & ACTAG_MASK;
	return 0;
}

static struct npu_link *find_link(struct pci_dev *dev)
{
	struct npu_link *link;

	list_for_each_entry(link, &links_list, list) {
		/* The functions of a device all share the same link */
		if (link->domain == pci_domain_nr(dev->bus) &&
			link->bus == dev->bus->number &&
			link->dev == PCI_SLOT(dev->devfn)) {
			return link;
		}
	}

	/* link doesn't exist yet. Allocate one */
	link = kzalloc(sizeof(struct npu_link), GFP_KERNEL);
	if (!link)
		return NULL;
	link->domain = pci_domain_nr(dev->bus);
	link->bus = dev->bus->number;
	link->dev = PCI_SLOT(dev->devfn);
	list_add(&link->list, &links_list);
	return link;
}

static void pnv_ocxl_fixup_actag(struct pci_dev *dev)
{
	struct pci_controller *hose = pci_bus_to_host(dev->bus);
	struct pnv_phb *phb = hose->private_data;
	struct npu_link *link;
	int rc, afu_idx = -1, i, actag;

	if (!machine_is(powernv))
		return;

	if (phb->type != PNV_PHB_NPU_OCAPI)
		return;

	mutex_lock(&links_list_lock);

	link = find_link(dev);
	if (!link) {
		dev_warn(&dev->dev, "couldn't update actag information\n");
		mutex_unlock(&links_list_lock);
		return;
	}

	/*
	 * Check how many actags are desired for the AFUs under that
	 * function and add it to the count for the link
	 */
	rc = get_max_afu_index(dev, &afu_idx);
	if (rc) {
		/* Most likely an invalid config space */
		dev_dbg(&dev->dev, "couldn't find AFU information\n");
		afu_idx = -1;
	}

	link->fn_desired_actags[PCI_FUNC(dev->devfn)] = 0;
	for (i = 0; i <= afu_idx; i++) {
		/*
		 * AFU index 'holes' are allowed. So don't fail if we
		 * can't read the actag info for an index
		 */
		rc = get_actag_count(dev, i, &actag);
		if (rc)
			continue;
		link->fn_desired_actags[PCI_FUNC(dev->devfn)] += actag;
	}
	dev_dbg(&dev->dev, "total actags for function: %d\n",
		link->fn_desired_actags[PCI_FUNC(dev->devfn)]);

	mutex_unlock(&links_list_lock);
}
DECLARE_PCI_FIXUP_HEADER(PCI_ANY_ID, PCI_ANY_ID, pnv_ocxl_fixup_actag);

static u16 assign_fn_actags(u16 desired, u16 total)
{
	u16 count;

	if (total <= PNV_OCXL_ACTAG_MAX)
		count = desired;
	else
		count = PNV_OCXL_ACTAG_MAX * desired / total;

	return count;
}

static void assign_actags(struct npu_link *link)
{
	u16 actag_count, range_start = 0, total_desired = 0;
	int i;

	for (i = 0; i < 8; i++)
		total_desired += link->fn_desired_actags[i];

	for (i = 0; i < 8; i++) {
		if (link->fn_desired_actags[i]) {
			actag_count = assign_fn_actags(
				link->fn_desired_actags[i],
				total_desired);
			link->fn_actags[i].start = range_start;
			link->fn_actags[i].count = actag_count;
			range_start += actag_count;
			WARN_ON(range_start >= PNV_OCXL_ACTAG_MAX);
		}
		pr_debug("link %x:%x:%x fct %d actags: start=%d count=%d (desired=%d)\n",
			link->domain, link->bus, link->dev, i,
			link->fn_actags[i].start, link->fn_actags[i].count,
			link->fn_desired_actags[i]);
	}
	link->assignment_done = true;
}

int pnv_ocxl_get_actag(struct pci_dev *dev, u16 *base, u16 *enabled,
		u16 *supported)
{
	struct npu_link *link;

	mutex_lock(&links_list_lock);

	link = find_link(dev);
	if (!link) {
		dev_err(&dev->dev, "actag information not found\n");
		mutex_unlock(&links_list_lock);
		return -ENODEV;
	}
	/*
	 * On p9, we only have 64 actags per link, so they must be
	 * shared by all the functions of the same adapter. We counted
	 * the desired actag counts during PCI enumeration, so that we
	 * can allocate a pro-rated number of actags to each function.
	 */
	if (!link->assignment_done)
		assign_actags(link);

	*base      = link->fn_actags[PCI_FUNC(dev->devfn)].start;
	*enabled   = link->fn_actags[PCI_FUNC(dev->devfn)].count;
	*supported = link->fn_desired_actags[PCI_FUNC(dev->devfn)];

	mutex_unlock(&links_list_lock);
	return 0;
}
EXPORT_SYMBOL_GPL(pnv_ocxl_get_actag);

int pnv_ocxl_get_pasid_count(struct pci_dev *dev, int *count)
{
	struct npu_link *link;
	int i, rc = -EINVAL;

	/*
	 * The number of PASIDs (process address space ID) which can
	 * be used by a function depends on how many functions exist
	 * on the device. The NPU needs to be configured to know how
	 * many bits are available to PASIDs and how many are to be
	 * used by the function BDF indentifier.
	 *
	 * We only support one AFU-carrying function for now.
	 */
	mutex_lock(&links_list_lock);

	link = find_link(dev);
	if (!link) {
		dev_err(&dev->dev, "actag information not found\n");
		mutex_unlock(&links_list_lock);
		return -ENODEV;
	}

	for (i = 0; i < 8; i++)
		if (link->fn_desired_actags[i] && (i == PCI_FUNC(dev->devfn))) {
			*count = PNV_OCXL_PASID_MAX;
			rc = 0;
			break;
		}

	mutex_unlock(&links_list_lock);
	dev_dbg(&dev->dev, "%d PASIDs available for function\n",
		rc ? 0 : *count);
	return rc;
}
EXPORT_SYMBOL_GPL(pnv_ocxl_get_pasid_count);

static void set_templ_rate(unsigned int templ, unsigned int rate, char *buf)
{
	int shift, idx;

	WARN_ON(templ > PNV_OCXL_TL_MAX_TEMPLATE);
	idx = (PNV_OCXL_TL_MAX_TEMPLATE - templ) / 2;
	shift = 4 * (1 - ((PNV_OCXL_TL_MAX_TEMPLATE - templ) % 2));
	buf[idx] |= rate << shift;
}

int pnv_ocxl_get_tl_cap(struct pci_dev *dev, long *cap,
			char *rate_buf, int rate_buf_size)
{
	if (rate_buf_size != PNV_OCXL_TL_RATE_BUF_SIZE)
		return -EINVAL;
	/*
	 * The TL capabilities are a characteristic of the NPU, so
	 * we go with hard-coded values.
	 *
	 * The receiving rate of each template is encoded on 4 bits.
	 *
	 * On P9:
	 * - templates 0 -> 3 are supported
	 * - templates 0, 1 and 3 have a 0 receiving rate
	 * - template 2 has receiving rate of 1 (extra cycle)
	 */
	memset(rate_buf, 0, rate_buf_size);
	set_templ_rate(2, 1, rate_buf);
	*cap = PNV_OCXL_TL_P9_RECV_CAP;
	return 0;
}
EXPORT_SYMBOL_GPL(pnv_ocxl_get_tl_cap);

int pnv_ocxl_set_tl_conf(struct pci_dev *dev, long cap,
			uint64_t rate_buf_phys, int rate_buf_size)
{
	struct pci_controller *hose = pci_bus_to_host(dev->bus);
	struct pnv_phb *phb = hose->private_data;
	int rc;

	if (rate_buf_size != PNV_OCXL_TL_RATE_BUF_SIZE)
		return -EINVAL;

	rc = opal_npu_tl_set(phb->opal_id, dev->devfn, cap,
			rate_buf_phys, rate_buf_size);
	if (rc) {
		dev_err(&dev->dev, "Can't configure host TL: %d\n", rc);
		return -EINVAL;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(pnv_ocxl_set_tl_conf);

int pnv_ocxl_get_xsl_irq(struct pci_dev *dev, int *hwirq)
{
	int rc;

	rc = of_property_read_u32(dev->dev.of_node, "ibm,opal-xsl-irq", hwirq);
	if (rc) {
		dev_err(&dev->dev,
			"Can't get translation interrupt for device\n");
		return rc;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(pnv_ocxl_get_xsl_irq);

void pnv_ocxl_unmap_xsl_regs(void __iomem *dsisr, void __iomem *dar,
			void __iomem *tfc, void __iomem *pe_handle)
{
	iounmap(dsisr);
	iounmap(dar);
	iounmap(tfc);
	iounmap(pe_handle);
}
EXPORT_SYMBOL_GPL(pnv_ocxl_unmap_xsl_regs);

int pnv_ocxl_map_xsl_regs(struct pci_dev *dev, void __iomem **dsisr,
			void __iomem **dar, void __iomem **tfc,
			void __iomem **pe_handle)
{
	u64 reg;
	int i, j, rc = 0;
	void __iomem *regs[4];

	/*
	 * opal stores the mmio addresses of the DSISR, DAR, TFC and
	 * PE_HANDLE registers in a device tree property, in that
	 * order
	 */
	for (i = 0; i < 4; i++) {
		rc = of_property_read_u64_index(dev->dev.of_node,
						"ibm,opal-xsl-mmio", i, &reg);
		if (rc)
			break;
		regs[i] = ioremap(reg, 8);
		if (!regs[i]) {
			rc = -EINVAL;
			break;
		}
	}
	if (rc) {
		dev_err(&dev->dev, "Can't map translation mmio registers\n");
		for (j = i - 1; j >= 0; j--)
			iounmap(regs[j]);
	} else {
		*dsisr = regs[0];
		*dar = regs[1];
		*tfc = regs[2];
		*pe_handle = regs[3];
	}
	return rc;
}
EXPORT_SYMBOL_GPL(pnv_ocxl_map_xsl_regs);

struct spa_data {
	u64 phb_opal_id;
	u32 bdfn;
};

int pnv_ocxl_spa_setup(struct pci_dev *dev, void *spa_mem, int PE_mask,
		void **platform_data)
{
	struct pci_controller *hose = pci_bus_to_host(dev->bus);
	struct pnv_phb *phb = hose->private_data;
	struct spa_data *data;
	u32 bdfn;
	int rc;

	data = kzalloc(sizeof(*data), GFP_KERNEL);
	if (!data)
		return -ENOMEM;

	bdfn = (dev->bus->number << 8) | dev->devfn;
	rc = opal_npu_spa_setup(phb->opal_id, bdfn, virt_to_phys(spa_mem),
				PE_mask);
	if (rc) {
		dev_err(&dev->dev, "Can't setup Shared Process Area: %d\n", rc);
		kfree(data);
		return rc;
	}
	data->phb_opal_id = phb->opal_id;
	data->bdfn = bdfn;
	*platform_data = (void *) data;
	return 0;
}
EXPORT_SYMBOL_GPL(pnv_ocxl_spa_setup);

void pnv_ocxl_spa_release(void *platform_data)
{
	struct spa_data *data = (struct spa_data *) platform_data;
	int rc;

	rc = opal_npu_spa_setup(data->phb_opal_id, data->bdfn, 0, 0);
	WARN_ON(rc);
	kfree(data);
}
EXPORT_SYMBOL_GPL(pnv_ocxl_spa_release);

int pnv_ocxl_spa_remove_pe(void *platform_data, int pe_handle)
{
	struct spa_data *data = (struct spa_data *) platform_data;
	int rc;

	rc = opal_npu_spa_clear_cache(data->phb_opal_id, data->bdfn, pe_handle);
	return rc;
}
EXPORT_SYMBOL_GPL(pnv_ocxl_spa_remove_pe);

int pnv_ocxl_alloc_xive_irq(u32 *irq, u64 *trigger_addr)
{
	__be64 flags, trigger_page;
	s64 rc;
	u32 hwirq;

	hwirq = xive_native_alloc_irq();
	if (!hwirq)
		return -ENOENT;

	rc = opal_xive_get_irq_info(hwirq, &flags, NULL, &trigger_page, NULL,
				NULL);
	if (rc || !trigger_page) {
		xive_native_free_irq(hwirq);
		return -ENOENT;
	}
	*irq = hwirq;
	*trigger_addr = be64_to_cpu(trigger_page);
	return 0;

}
EXPORT_SYMBOL_GPL(pnv_ocxl_alloc_xive_irq);

void pnv_ocxl_free_xive_irq(u32 irq)
{
	xive_native_free_irq(irq);
}
EXPORT_SYMBOL_GPL(pnv_ocxl_free_xive_irq);
Commit	Line	Data
bad9d1e8 FB	1	// SPDX-License-Identifier: GPL-2.0+
	2	// Copyright 2017 IBM Corp.
	3	#include <asm/pnv-ocxl.h>
	4	#include <asm/opal.h>
5bc71b82	5	#include <asm/xive.h>
1558b51a	6	#include <misc/ocxl-config.h>
bad9d1e8 FB	7	#include "pci.h"
	8
	9	#define PNV_OCXL_TL_P9_RECV_CAP 0x000000000000000Full
1558b51a	10	#define PNV_OCXL_ACTAG_MAX 64
bad9d1e8 FB	11	/* PASIDs are 20-bit, but on P9, NPU can only handle 15 bits */
	12	#define PNV_OCXL_PASID_BITS 15
	13	#define PNV_OCXL_PASID_MAX ((1 << PNV_OCXL_PASID_BITS) - 1)
	14
1558b51a FB	15	#define AFU_PRESENT (1 << 31)
	16	#define AFU_INDEX_MASK 0x3F000000
	17	#define AFU_INDEX_SHIFT 24
	18	#define ACTAG_MASK 0xFFF
	19
	20
	21	struct actag_range {
	22	u16 start;
	23	u16 count;
	24	};
	25
	26	struct npu_link {
	27	struct list_head list;
	28	int domain;
	29	int bus;
	30	int dev;
	31	u16 fn_desired_actags[8];
	32	struct actag_range fn_actags[8];
	33	bool assignment_done;
	34	};
	35	static struct list_head links_list = LIST_HEAD_INIT(links_list);
	36	static DEFINE_MUTEX(links_list_lock);
	37
	38
	39	/*
	40	* opencapi actags handling:
	41	*
	42	* When sending commands, the opencapi device references the memory
	43	* context it's targeting with an 'actag', which is really an alias
	44	* for a (BDF, pasid) combination. When it receives a command, the NPU
	45	* must do a lookup of the actag to identify the memory context. The
	46	* hardware supports a finite number of actags per link (64 for
	47	* POWER9).
	48	*
	49	* The device can carry multiple functions, and each function can have
	50	* multiple AFUs. Each AFU advertises in its config space the number
	51	* of desired actags. The host must configure in the config space of
	52	* the AFU how many actags the AFU is really allowed to use (which can
	53	* be less than what the AFU desires).
	54	*
	55	* When a PCI function is probed by the driver, it has no visibility
	56	* about the other PCI functions and how many actags they'd like,
	57	* which makes it impossible to distribute actags fairly among AFUs.
	58	*
	59	* Unfortunately, the only way to know how many actags a function
	60	* desires is by looking at the data for each AFU in the config space
	61	* and add them up. Similarly, the only way to know how many actags
	62	* all the functions of the physical device desire is by adding the
	63	* previously computed function counts. Then we can match that against
	64	* what the hardware supports.
	65	*
	66	* To get a comprehensive view, we use a 'pci fixup': at the end of
	67	* PCI enumeration, each function counts how many actags its AFUs
	68	* desire and we save it in a 'npu_link' structure, shared between all
	69	* the PCI functions of a same device. Therefore, when the first
	70	* function is probed by the driver, we can get an idea of the total
	71	* count of desired actags for the device, and assign the actags to
	72	* the AFUs, by pro-rating if needed.
	73	*/
	74
	75	static int find_dvsec_from_pos(struct pci_dev *dev, int dvsec_id, int pos)
	76	{
	77	int vsec = pos;
	78	u16 vendor, id;
79
80	while ((vsec = pci_find_next_ext_capability(dev, vsec,
81	OCXL_EXT_CAP_ID_DVSEC))) {
82	pci_read_config_word(dev, vsec + OCXL_DVSEC_VENDOR_OFFSET,
83	&vendor);
84	pci_read_config_word(dev, vsec + OCXL_DVSEC_ID_OFFSET, &id);
85	if (vendor == PCI_VENDOR_ID_IBM && id == dvsec_id)
86	return vsec;
87	}
88	return 0;
89	}
90
91	static int find_dvsec_afu_ctrl(struct pci_dev *dev, u8 afu_idx)
92	{
93	int vsec = 0;
94	u8 idx;
95
96	while ((vsec = find_dvsec_from_pos(dev, OCXL_DVSEC_AFU_CTRL_ID,
97	vsec))) {
98	pci_read_config_byte(dev, vsec + OCXL_DVSEC_AFU_CTRL_AFU_IDX,
99	&idx);
100	if (idx == afu_idx)
101	return vsec;
102	}
103	return 0;
104	}
105
106	static int get_max_afu_index(struct pci_dev dev, int afu_idx)
107	{
108	int pos;
109	u32 val;
110
111	pos = find_dvsec_from_pos(dev, OCXL_DVSEC_FUNC_ID, 0);
112	if (!pos)
113	return -ESRCH;
114
115	pci_read_config_dword(dev, pos + OCXL_DVSEC_FUNC_OFF_INDEX, &val);
116	if (val & AFU_PRESENT)
117	*afu_idx = (val & AFU_INDEX_MASK) >> AFU_INDEX_SHIFT;
118	else
119	*afu_idx = -1;
120	return 0;
121	}
122
123	static int get_actag_count(struct pci_dev dev, int afu_idx, int actag)
124	{
125	int pos;
126	u16 actag_sup;
127
128	pos = find_dvsec_afu_ctrl(dev, afu_idx);
129	if (!pos)
130	return -ESRCH;
131
132	pci_read_config_word(dev, pos + OCXL_DVSEC_AFU_CTRL_ACTAG_SUP,
133	&actag_sup);
134	*actag = actag_sup & ACTAG_MASK;
135	return 0;
136	}
137
138	static struct npu_link find_link(struct pci_dev dev)
139	{
140	struct npu_link *link;
141
142	list_for_each_entry(link, &links_list, list) {
143	/* The functions of a device all share the same link */
144	if (link->domain == pci_domain_nr(dev->bus) &&
145	link->bus == dev->bus->number &&
146	link->dev == PCI_SLOT(dev->devfn)) {
147	return link;
148	}
149	}
150
151	/* link doesn't exist yet. Allocate one */
152	link = kzalloc(sizeof(struct npu_link), GFP_KERNEL);
153	if (!link)
154	return NULL;
155	link->domain = pci_domain_nr(dev->bus);
156	link->bus = dev->bus->number;
157	link->dev = PCI_SLOT(dev->devfn);
158	list_add(&link->list, &links_list);
159	return link;
160	}
161
162	static void pnv_ocxl_fixup_actag(struct pci_dev *dev)
163	{
164	struct pci_controller *hose = pci_bus_to_host(dev->bus);
165	struct pnv_phb *phb = hose->private_data;
166	struct npu_link *link;
167	int rc, afu_idx = -1, i, actag;
168
169	if (!machine_is(powernv))
170	return;
171
172	if (phb->type != PNV_PHB_NPU_OCAPI)
173	return;
174
175	mutex_lock(&links_list_lock);
176
177	link = find_link(dev);
178	if (!link) {
179	dev_warn(&dev->dev, "couldn't update actag information\n");
180	mutex_unlock(&links_list_lock);
181	return;
182	}
183
184	/*
185	* Check how many actags are desired for the AFUs under that
186	* function and add it to the count for the link
187	*/
188	rc = get_max_afu_index(dev, &afu_idx);
189	if (rc) {
190	/* Most likely an invalid config space */
191	dev_dbg(&dev->dev, "couldn't find AFU information\n");
192	afu_idx = -1;
193	}
194
195	link->fn_desired_actags[PCI_FUNC(dev->devfn)] = 0;
196	for (i = 0; i <= afu_idx; i++) {
197	/*
198	* AFU index 'holes' are allowed. So don't fail if we
199	* can't read the actag info for an index
200	*/
201	rc = get_actag_count(dev, i, &actag);
202	if (rc)
203	continue;
204	link->fn_desired_actags[PCI_FUNC(dev->devfn)] += actag;
205	}
206	dev_dbg(&dev->dev, "total actags for function: %d\n",
207	link->fn_desired_actags[PCI_FUNC(dev->devfn)]);
208
209	mutex_unlock(&links_list_lock);
210	}
211	DECLARE_PCI_FIXUP_HEADER(PCI_ANY_ID, PCI_ANY_ID, pnv_ocxl_fixup_actag);
212
213	static u16 assign_fn_actags(u16 desired, u16 total)
214	{
215	u16 count;
216
217	if (total <= PNV_OCXL_ACTAG_MAX)
218	count = desired;
219	else
220	count = PNV_OCXL_ACTAG_MAX * desired / total;
221
222	return count;
223	}
224
225	static void assign_actags(struct npu_link *link)
226	{
227	u16 actag_count, range_start = 0, total_desired = 0;
228	int i;
229
230	for (i = 0; i < 8; i++)
231	total_desired += link->fn_desired_actags[i];
232
233	for (i = 0; i < 8; i++) {
234	if (link->fn_desired_actags[i]) {
235	actag_count = assign_fn_actags(
236	link->fn_desired_actags[i],
237	total_desired);
238	link->fn_actags[i].start = range_start;
239	link->fn_actags[i].count = actag_count;
240	range_start += actag_count;
241	WARN_ON(range_start >= PNV_OCXL_ACTAG_MAX);
242	}
243	pr_debug("link %x:%x:%x fct %d actags: start=%d count=%d (desired=%d)\n",
244	link->domain, link->bus, link->dev, i,
245	link->fn_actags[i].start, link->fn_actags[i].count,
246	link->fn_desired_actags[i]);
247	}
248	link->assignment_done = true;
249	}
250
251	int pnv_ocxl_get_actag(struct pci_dev dev, u16 base, u16 *enabled,
252	u16 *supported)
253	{
254	struct npu_link *link;
255
256	mutex_lock(&links_list_lock);
257
258	link = find_link(dev);
259	if (!link) {
260	dev_err(&dev->dev, "actag information not found\n");
261	mutex_unlock(&links_list_lock);
262	return -ENODEV;
263	}
264	/*
265	* On p9, we only have 64 actags per link, so they must be
266	* shared by all the functions of the same adapter. We counted
267	* the desired actag counts during PCI enumeration, so that we
268	* can allocate a pro-rated number of actags to each function.
269	*/
270	if (!link->assignment_done)
271	assign_actags(link);
272
273	*base = link->fn_actags[PCI_FUNC(dev->devfn)].start;
274	*enabled = link->fn_actags[PCI_FUNC(dev->devfn)].count;
275	*supported = link->fn_desired_actags[PCI_FUNC(dev->devfn)];
276
277	mutex_unlock(&links_list_lock);
278	return 0;
279	}
280	EXPORT_SYMBOL_GPL(pnv_ocxl_get_actag);
281
282	int pnv_ocxl_get_pasid_count(struct pci_dev dev, int count)
283	{
284	struct npu_link *link;
285	int i, rc = -EINVAL;
286
287	/*
288	* The number of PASIDs (process address space ID) which can
289	* be used by a function depends on how many functions exist
290	* on the device. The NPU needs to be configured to know how
291	* many bits are available to PASIDs and how many are to be
292	* used by the function BDF indentifier.
293	*
294	* We only support one AFU-carrying function for now.
295	*/
296	mutex_lock(&links_list_lock);
297
298	link = find_link(dev);
299	if (!link) {
300	dev_err(&dev->dev, "actag information not found\n");
301	mutex_unlock(&links_list_lock);
302	return -ENODEV;
303	}
304
305	for (i = 0; i < 8; i++)
306	if (link->fn_desired_actags[i] && (i == PCI_FUNC(dev->devfn))) {
307	*count = PNV_OCXL_PASID_MAX;
308	rc = 0;
309	break;
310	}
311
312	mutex_unlock(&links_list_lock);
313	dev_dbg(&dev->dev, "%d PASIDs available for function\n",
314	rc ? 0 : *count);
315	return rc;
316	}
317	EXPORT_SYMBOL_GPL(pnv_ocxl_get_pasid_count);
bad9d1e8 FB	318
	319	static void set_templ_rate(unsigned int templ, unsigned int rate, char *buf)
	320	{
	321	int shift, idx;
	322
	323	WARN_ON(templ > PNV_OCXL_TL_MAX_TEMPLATE);
	324	idx = (PNV_OCXL_TL_MAX_TEMPLATE - templ) / 2;
	325	shift = 4 * (1 - ((PNV_OCXL_TL_MAX_TEMPLATE - templ) % 2));
	326	buf[idx] \|= rate << shift;
	327	}
	328
	329	int pnv_ocxl_get_tl_cap(struct pci_dev dev, long cap,
	330	char *rate_buf, int rate_buf_size)
	331	{
	332	if (rate_buf_size != PNV_OCXL_TL_RATE_BUF_SIZE)
	333	return -EINVAL;
	334	/*
	335	* The TL capabilities are a characteristic of the NPU, so
	336	* we go with hard-coded values.
	337	*
	338	* The receiving rate of each template is encoded on 4 bits.
	339	*
	340	* On P9:
	341	* - templates 0 -> 3 are supported
	342	* - templates 0, 1 and 3 have a 0 receiving rate
	343	* - template 2 has receiving rate of 1 (extra cycle)
	344	*/
	345	memset(rate_buf, 0, rate_buf_size);
	346	set_templ_rate(2, 1, rate_buf);
	347	*cap = PNV_OCXL_TL_P9_RECV_CAP;
	348	return 0;
	349	}
	350	EXPORT_SYMBOL_GPL(pnv_ocxl_get_tl_cap);
	351
	352	int pnv_ocxl_set_tl_conf(struct pci_dev *dev, long cap,
	353	uint64_t rate_buf_phys, int rate_buf_size)
	354	{
	355	struct pci_controller *hose = pci_bus_to_host(dev->bus);
	356	struct pnv_phb *phb = hose->private_data;
	357	int rc;
	358
	359	if (rate_buf_size != PNV_OCXL_TL_RATE_BUF_SIZE)
	360	return -EINVAL;
	361
	362	rc = opal_npu_tl_set(phb->opal_id, dev->devfn, cap,
	363	rate_buf_phys, rate_buf_size);
	364	if (rc) {
	365	dev_err(&dev->dev, "Can't configure host TL: %d\n", rc);
	366	return -EINVAL;
	367	}
	368	return 0;
	369	}
	370	EXPORT_SYMBOL_GPL(pnv_ocxl_set_tl_conf);
	371
	372	int pnv_ocxl_get_xsl_irq(struct pci_dev dev, int hwirq)
	373	{
	374	int rc;
	375
	376	rc = of_property_read_u32(dev->dev.of_node, "ibm,opal-xsl-irq", hwirq);
	377	if (rc) {
	378	dev_err(&dev->dev,
	379	"Can't get translation interrupt for device\n");
	380	return rc;
	381	}
382	return 0;
383	}
384	EXPORT_SYMBOL_GPL(pnv_ocxl_get_xsl_irq);
385
386	void pnv_ocxl_unmap_xsl_regs(void __iomem dsisr, void __iomem dar,
387	void __iomem tfc, void __iomem pe_handle)
388	{
389	iounmap(dsisr);
390	iounmap(dar);
391	iounmap(tfc);
392	iounmap(pe_handle);
393	}
394	EXPORT_SYMBOL_GPL(pnv_ocxl_unmap_xsl_regs);
395
396	int pnv_ocxl_map_xsl_regs(struct pci_dev dev, void __iomem *dsisr,
397	void __iomem dar, void __iomem tfc,
398	void __iomem **pe_handle)
399	{
400	u64 reg;
401	int i, j, rc = 0;
402	void __iomem *regs[4];
403
404	/*
405	* opal stores the mmio addresses of the DSISR, DAR, TFC and
406	* PE_HANDLE registers in a device tree property, in that
407	* order
408	*/
409	for (i = 0; i < 4; i++) {
410	rc = of_property_read_u64_index(dev->dev.of_node,
411	"ibm,opal-xsl-mmio", i, &reg);
412	if (rc)
413	break;
414	regs[i] = ioremap(reg, 8);
415	if (!regs[i]) {
416	rc = -EINVAL;
417	break;
418	}
419	}
420	if (rc) {
421	dev_err(&dev->dev, "Can't map translation mmio registers\n");
422	for (j = i - 1; j >= 0; j--)
423	iounmap(regs[j]);
424	} else {
425	*dsisr = regs[0];
426	*dar = regs[1];
427	*tfc = regs[2];
428	*pe_handle = regs[3];
429	}
430	return rc;
431	}
432	EXPORT_SYMBOL_GPL(pnv_ocxl_map_xsl_regs);
433
434	struct spa_data {
435	u64 phb_opal_id;
436	u32 bdfn;
437	};
438
439	int pnv_ocxl_spa_setup(struct pci_dev dev, void spa_mem, int PE_mask,
440	void **platform_data)
441	{
442	struct pci_controller *hose = pci_bus_to_host(dev->bus);
443	struct pnv_phb *phb = hose->private_data;
444	struct spa_data *data;
445	u32 bdfn;
446	int rc;
447
448	data = kzalloc(sizeof(*data), GFP_KERNEL);
449	if (!data)
450	return -ENOMEM;
451
452	bdfn = (dev->bus->number << 8) \| dev->devfn;
453	rc = opal_npu_spa_setup(phb->opal_id, bdfn, virt_to_phys(spa_mem),
454	PE_mask);
455	if (rc) {
456	dev_err(&dev->dev, "Can't setup Shared Process Area: %d\n", rc);
457	kfree(data);
458	return rc;
459	}
460	data->phb_opal_id = phb->opal_id;
461	data->bdfn = bdfn;
462	platform_data = (void ) data;
463	return 0;
464	}
465	EXPORT_SYMBOL_GPL(pnv_ocxl_spa_setup);
466
467	void pnv_ocxl_spa_release(void *platform_data)
468	{
469	struct spa_data data = (struct spa_data ) platform_data;
470	int rc;
471
472	rc = opal_npu_spa_setup(data->phb_opal_id, data->bdfn, 0, 0);
473	WARN_ON(rc);
474	kfree(data);
475	}
476	EXPORT_SYMBOL_GPL(pnv_ocxl_spa_release);
477
478	int pnv_ocxl_spa_remove_pe(void *platform_data, int pe_handle)
479	{
480	struct spa_data data = (struct spa_data ) platform_data;
481	int rc;
482
483	rc = opal_npu_spa_clear_cache(data->phb_opal_id, data->bdfn, pe_handle);
484	return rc;
485	}
486	EXPORT_SYMBOL_GPL(pnv_ocxl_spa_remove_pe);
5bc71b82 FB	487
	488	int pnv_ocxl_alloc_xive_irq(u32 irq, u64 trigger_addr)
	489	{
	490	__be64 flags, trigger_page;
	491	s64 rc;
	492	u32 hwirq;
	493
	494	hwirq = xive_native_alloc_irq();
	495	if (!hwirq)
	496	return -ENOENT;
	497
	498	rc = opal_xive_get_irq_info(hwirq, &flags, NULL, &trigger_page, NULL,
	499	NULL);
	500	if (rc \|\| !trigger_page) {
	501	xive_native_free_irq(hwirq);
	502	return -ENOENT;
	503	}
	504	*irq = hwirq;
	505	*trigger_addr = be64_to_cpu(trigger_page);
	506	return 0;
	507
	508	}
	509	EXPORT_SYMBOL_GPL(pnv_ocxl_alloc_xive_irq);
	510
	511	void pnv_ocxl_free_xive_irq(u32 irq)
	512	{
	513	xive_native_free_irq(irq);
	514	}
	515	EXPORT_SYMBOL_GPL(pnv_ocxl_free_xive_irq);