[ceph.git] / ceph / src / spdk / dpdk / drivers / raw / ifpga_rawdev / base / ifpga_enumerate.c

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2010-2018 Intel Corporation
 */

#include "opae_hw_api.h"
#include "ifpga_api.h"

#include "ifpga_hw.h"
#include "ifpga_enumerate.h"
#include "ifpga_feature_dev.h"

struct build_feature_devs_info {
	struct opae_adapter_data_pci *pci_data;

	struct ifpga_afu_info *acc_info;

	void *fiu;
	enum fpga_id_type current_type;
	int current_port_id;

	void *ioaddr;
	void *ioend;
	uint64_t phys_addr;
	int current_bar;

	void *pfme_hdr;

	struct ifpga_hw *hw;
};

static int feature_revision(void __iomem *start)
{
	struct feature_header header;

	header.csr = readq(start);

	return header.revision;
}

static u32 feature_size(void __iomem *start)
{
	struct feature_header header;

	header.csr = readq(start);

	/*the size of private feature is 4KB aligned*/
	return header.next_header_offset ? header.next_header_offset:4096;
}

static u64 feature_id(void __iomem *start)
{
	struct feature_header header;

	header.csr = readq(start);

	switch (header.type) {
	case FEATURE_TYPE_FIU:
		return FEATURE_ID_FIU_HEADER;
	case FEATURE_TYPE_PRIVATE:
		return header.id;
	case FEATURE_TYPE_AFU:
		return FEATURE_ID_AFU;
	}

	WARN_ON(1);
	return 0;
}

static int
build_info_add_sub_feature(struct build_feature_devs_info *binfo,
		void __iomem *start, u64 fid, unsigned int size,
		unsigned int vec_start,
		unsigned int vec_cnt)
{
	struct ifpga_hw *hw = binfo->hw;
	struct ifpga_feature *feature = NULL;
	struct feature_irq_ctx *ctx = NULL;
	int port_id, ret = 0;
	unsigned int i;

	fid = fid?fid:feature_id(start);
	size = size?size:feature_size(start);

	feature = opae_malloc(sizeof(struct ifpga_feature));
	if (!feature)
		return -ENOMEM;

	feature->state = IFPGA_FEATURE_ATTACHED;
	feature->addr = start;
	feature->id = fid;
	feature->size = size;
	feature->revision = feature_revision(start);
	feature->phys_addr = binfo->phys_addr +
				((u8 *)start - (u8 *)binfo->ioaddr);
	feature->vec_start = vec_start;
	feature->vec_cnt = vec_cnt;

	dev_debug(binfo, "%s: id=0x%llx, phys_addr=0x%llx, size=%u\n",
			__func__, (unsigned long long)feature->id,
			(unsigned long long)feature->phys_addr, size);

	if (vec_cnt) {
		if (vec_start + vec_cnt <= vec_start)
			return -EINVAL;

		ctx = zmalloc(sizeof(*ctx) * vec_cnt);
		if (!ctx)
			return -ENOMEM;

		for (i = 0; i < vec_cnt; i++) {
			ctx[i].eventfd = -1;
			ctx[i].idx = vec_start + i;
		}
	}

	feature->ctx = ctx;
	feature->ctx_num = vec_cnt;
	feature->vfio_dev_fd = binfo->pci_data->vfio_dev_fd;

	if (binfo->current_type == FME_ID) {
		feature->parent = &hw->fme;
		feature->type = FEATURE_FME_TYPE;
		feature->name = get_fme_feature_name(fid);
		TAILQ_INSERT_TAIL(&hw->fme.feature_list, feature, next);
	} else if (binfo->current_type == PORT_ID) {
		port_id = binfo->current_port_id;
		feature->parent = &hw->port[port_id];
		feature->type = FEATURE_PORT_TYPE;
		feature->name = get_port_feature_name(fid);
		TAILQ_INSERT_TAIL(&hw->port[port_id].feature_list,
				feature, next);
	} else {
		return -EFAULT;
	}
	return ret;
}

static int
create_feature_instance(struct build_feature_devs_info *binfo,
			void __iomem *start, u64 fid,
			unsigned int size, unsigned int vec_start,
			unsigned int vec_cnt)
{
	return build_info_add_sub_feature(binfo, start, fid, size, vec_start,
			vec_cnt);
}

/*
 * UAFU GUID is dynamic as it can be changed after FME downloads different
 * Green Bitstream to the port, so we treat the unknown GUIDs which are
 * attached on port's feature list as UAFU.
 */
static bool feature_is_UAFU(struct build_feature_devs_info *binfo)
{
	if (binfo->current_type != PORT_ID)
		return false;

	return true;
}

static int parse_feature_port_uafu(struct build_feature_devs_info *binfo,
				   struct feature_header *hdr)
{
	u64 id = PORT_FEATURE_ID_UAFU;
	struct ifpga_afu_info *info;
	void *start = (void *)hdr;
	struct feature_port_header *port_hdr = binfo->ioaddr;
	struct feature_port_capability capability;
	int ret;
	int size;

	capability.csr = readq(&port_hdr->capability);

	size = capability.mmio_size << 10;

	ret = create_feature_instance(binfo, hdr, id, size, 0, 0);
	if (ret)
		return ret;

	info = opae_malloc(sizeof(*info));
	if (!info)
		return -ENOMEM;

	info->region[0].addr = start;
	info->region[0].phys_addr = binfo->phys_addr +
			(uint8_t *)start - (uint8_t *)binfo->ioaddr;
	info->region[0].len = size;
	info->num_regions = 1;

	binfo->acc_info = info;

	return ret;
}

static int parse_feature_afus(struct build_feature_devs_info *binfo,
			      struct feature_header *hdr)
{
	int ret;
	struct feature_afu_header *afu_hdr, header;
	u8 __iomem *start;
	u8 __iomem *end = binfo->ioend;

	start = (u8 __iomem *)hdr;
	for (; start < end; start += header.next_afu) {
		if ((unsigned int)(end - start) <
			(unsigned int)(sizeof(*afu_hdr) + sizeof(*hdr)))
			return -EINVAL;

		hdr = (struct feature_header *)start;
		afu_hdr = (struct feature_afu_header *)(hdr + 1);
		header.csr = readq(&afu_hdr->csr);

		if (feature_is_UAFU(binfo)) {
			ret = parse_feature_port_uafu(binfo, hdr);
			if (ret)
				return ret;
		}

		if (!header.next_afu)
			break;
	}

	return 0;
}

/* create and register proper private data */
static int build_info_commit_dev(struct build_feature_devs_info *binfo)
{
	struct ifpga_afu_info *info = binfo->acc_info;
	struct ifpga_hw *hw = binfo->hw;
	struct opae_manager *mgr;
	struct opae_bridge *br;
	struct opae_accelerator *acc;
	struct ifpga_port_hw *port;
	struct ifpga_fme_hw *fme;
	struct ifpga_feature *feature;

	if (!binfo->fiu)
		return 0;

	if (binfo->current_type == PORT_ID) {
		/* return error if no valid acc info data structure */
		if (!info)
			return -EFAULT;

		br = opae_bridge_alloc(hw->adapter->name, &ifpga_br_ops,
				       binfo->fiu);
		if (!br)
			return -ENOMEM;

		br->id = binfo->current_port_id;

		/* update irq info */
		port = &hw->port[binfo->current_port_id];
		feature = get_feature_by_id(&port->feature_list,
				PORT_FEATURE_ID_UINT);
		if (feature)
			info->num_irqs = feature->vec_cnt;

		acc = opae_accelerator_alloc(hw->adapter->name,
					     &ifpga_acc_ops, info);
		if (!acc) {
			opae_bridge_free(br);
			return -ENOMEM;
		}

		acc->br = br;
		if (hw->adapter->mgr)
			acc->mgr = hw->adapter->mgr;
		acc->index = br->id;

		fme = &hw->fme;
		fme->nums_acc_region = info->num_regions;

		opae_adapter_add_acc(hw->adapter, acc);

	} else if (binfo->current_type == FME_ID) {
		mgr = opae_manager_alloc(hw->adapter->name, &ifpga_mgr_ops,
				&ifpga_mgr_network_ops, binfo->fiu);
		if (!mgr)
			return -ENOMEM;

		mgr->adapter = hw->adapter;
		hw->adapter->mgr = mgr;
	}

	binfo->fiu = NULL;

	return 0;
}

static int
build_info_create_dev(struct build_feature_devs_info *binfo,
		      enum fpga_id_type type, unsigned int index)
{
	int ret;

	ret = build_info_commit_dev(binfo);
	if (ret)
		return ret;

	binfo->current_type = type;

	if (type == FME_ID) {
		binfo->fiu = &binfo->hw->fme;
	} else if (type == PORT_ID) {
		binfo->fiu = &binfo->hw->port[index];
		binfo->current_port_id = index;
	}

	return 0;
}

static int parse_feature_fme(struct build_feature_devs_info *binfo,
			     struct feature_header *start)
{
	struct ifpga_hw *hw = binfo->hw;
	struct ifpga_fme_hw *fme = &hw->fme;
	int ret;

	ret = build_info_create_dev(binfo, FME_ID, 0);
	if (ret)
		return ret;

	/* Update FME states */
	fme->state = IFPGA_FME_IMPLEMENTED;
	fme->parent = hw;
	TAILQ_INIT(&fme->feature_list);
	spinlock_init(&fme->lock);

	return create_feature_instance(binfo, start, 0, 0, 0, 0);
}

static int parse_feature_port(struct build_feature_devs_info *binfo,
			      void __iomem *start)
{
	struct feature_port_header *port_hdr;
	struct feature_port_capability capability;
	struct ifpga_hw *hw = binfo->hw;
	struct ifpga_port_hw *port;
	unsigned int port_id;
	int ret;

	/* Get current port's id */
	port_hdr = (struct feature_port_header *)start;
	capability.csr = readq(&port_hdr->capability);
	port_id = capability.port_number;

	ret = build_info_create_dev(binfo, PORT_ID, port_id);
	if (ret)
		return ret;

	/*found a Port device*/
	port = &hw->port[port_id];
	port->port_id = binfo->current_port_id;
	port->parent = hw;
	port->state = IFPGA_PORT_ATTACHED;
	spinlock_init(&port->lock);
	TAILQ_INIT(&port->feature_list);

	return create_feature_instance(binfo, start, 0, 0, 0, 0);
}

static void enable_port_uafu(struct build_feature_devs_info *binfo,
			     void __iomem *start)
{
	struct ifpga_port_hw *port = &binfo->hw->port[binfo->current_port_id];

	UNUSED(start);

	fpga_port_reset(port);
}

static int parse_feature_fiu(struct build_feature_devs_info *binfo,
			     struct feature_header *hdr)
{
	struct feature_header header;
	struct feature_fiu_header *fiu_hdr, fiu_header;
	u8 __iomem *start = (u8 __iomem *)hdr;
	int ret;

	header.csr = readq(hdr);

	switch (header.id) {
	case FEATURE_FIU_ID_FME:
		ret = parse_feature_fme(binfo, hdr);
		binfo->pfme_hdr = hdr;
		if (ret)
			return ret;
		break;
	case FEATURE_FIU_ID_PORT:
		ret = parse_feature_port(binfo, hdr);
		enable_port_uafu(binfo, hdr);
		if (ret)
			return ret;

		/* Check Port FIU's next_afu pointer to User AFU DFH */
		fiu_hdr = (struct feature_fiu_header *)(hdr + 1);
		fiu_header.csr = readq(&fiu_hdr->csr);

		if (fiu_header.next_afu) {
			start += fiu_header.next_afu;
			ret = parse_feature_afus(binfo,
						(struct feature_header *)start);
			if (ret)
				return ret;
		} else {
			dev_info(binfo, "No AFUs detected on Port\n");
		}

		break;
	default:
		dev_info(binfo, "FIU TYPE %d is not supported yet.\n",
			 header.id);
	}

	return 0;
}

static void parse_feature_irqs(struct build_feature_devs_info *binfo,
		void __iomem *start, unsigned int *vec_start,
		unsigned int *vec_cnt)
{
	UNUSED(binfo);
	u64 id;

	id = feature_id(start);

	if (id == PORT_FEATURE_ID_UINT) {
		struct feature_port_uint *port_uint = start;
		struct feature_port_uint_cap uint_cap;

		uint_cap.csr = readq(&port_uint->capability);
		if (uint_cap.intr_num) {
			*vec_start = uint_cap.first_vec_num;
			*vec_cnt = uint_cap.intr_num;
		} else {
			dev_debug(binfo, "UAFU doesn't support interrupt\n");
		}
	} else if (id == PORT_FEATURE_ID_ERROR) {
		struct feature_port_error *port_err = start;
		struct feature_port_err_capability port_err_cap;

		port_err_cap.csr = readq(&port_err->error_capability);
		if (port_err_cap.support_intr) {
			*vec_start = port_err_cap.intr_vector_num;
			*vec_cnt = 1;
		} else {
			dev_debug(&binfo, "Port error doesn't support interrupt\n");
		}

	} else if (id == FME_FEATURE_ID_GLOBAL_ERR) {
		struct feature_fme_err *fme_err = start;
		struct feature_fme_error_capability fme_err_cap;

		fme_err_cap.csr = readq(&fme_err->fme_err_capability);
		if (fme_err_cap.support_intr) {
			*vec_start = fme_err_cap.intr_vector_num;
			*vec_cnt = 1;
		} else {
			dev_debug(&binfo, "FME error doesn't support interrupt\n");
		}
	}
}

static int parse_feature_fme_private(struct build_feature_devs_info *binfo,
				     struct feature_header *hdr)
{
	unsigned int vec_start = 0;
	unsigned int vec_cnt = 0;

	parse_feature_irqs(binfo, hdr, &vec_start, &vec_cnt);

	return create_feature_instance(binfo, hdr, 0, 0, vec_start, vec_cnt);
}

static int parse_feature_port_private(struct build_feature_devs_info *binfo,
				      struct feature_header *hdr)
{
	unsigned int vec_start = 0;
	unsigned int vec_cnt = 0;

	parse_feature_irqs(binfo, hdr, &vec_start, &vec_cnt);

	return create_feature_instance(binfo, hdr, 0, 0, vec_start, vec_cnt);
}

static int parse_feature_private(struct build_feature_devs_info *binfo,
				 struct feature_header *hdr)
{
	struct feature_header header;

	header.csr = readq(hdr);

	switch (binfo->current_type) {
	case FME_ID:
		return parse_feature_fme_private(binfo, hdr);
	case PORT_ID:
		return parse_feature_port_private(binfo, hdr);
	default:
		dev_err(binfo, "private feature %x belonging to AFU %d (unknown_type) is not supported yet.\n",
			header.id, binfo->current_type);
	}
	return 0;
}

static int parse_feature(struct build_feature_devs_info *binfo,
			 struct feature_header *hdr)
{
	struct feature_header header;
	int ret = 0;

	header.csr = readq(hdr);

	switch (header.type) {
	case FEATURE_TYPE_AFU:
		ret = parse_feature_afus(binfo, hdr);
		break;
	case FEATURE_TYPE_PRIVATE:
		ret = parse_feature_private(binfo, hdr);
		break;
	case FEATURE_TYPE_FIU:
		ret = parse_feature_fiu(binfo, hdr);
		break;
	default:
		dev_err(binfo, "Feature Type %x is not supported.\n",
			hdr->type);
	};

	return ret;
}

static int
parse_feature_list(struct build_feature_devs_info *binfo, u8 __iomem *start)
{
	struct feature_header *hdr, header;
	u8 __iomem *end = (u8 __iomem *)binfo->ioend;
	int ret = 0;

	for (; start < end; start += header.next_header_offset) {
		if ((unsigned int)(end - start) < (unsigned int)sizeof(*hdr)) {
			dev_err(binfo, "The region is too small to contain a feature.\n");
			ret =  -EINVAL;
			break;
		}

		hdr = (struct feature_header *)start;
		header.csr = readq(hdr);

		dev_debug(binfo, "%s: address=0x%p, val=0x%llx, header.id=0x%x, header.next_offset=0x%x, header.eol=0x%x, header.type=0x%x\n",
			__func__, hdr, (unsigned long long)header.csr,
			header.id, header.next_header_offset,
			header.end_of_list, header.type);

		ret = parse_feature(binfo, hdr);
		if (ret)
			return ret;

		if (header.end_of_list || !header.next_header_offset)
			break;
	}

	return build_info_commit_dev(binfo);
}

/* switch the memory mapping to BAR# @bar */
static int parse_switch_to(struct build_feature_devs_info *binfo, int bar)
{
	struct opae_adapter_data_pci *pci_data = binfo->pci_data;

	if (!pci_data->region[bar].addr)
		return -ENOMEM;

	binfo->ioaddr = pci_data->region[bar].addr;
	binfo->ioend = (u8 __iomem *)binfo->ioaddr + pci_data->region[bar].len;
	binfo->phys_addr = pci_data->region[bar].phys_addr;
	binfo->current_bar = bar;

	return 0;
}

static int parse_ports_from_fme(struct build_feature_devs_info *binfo)
{
	struct feature_fme_header *fme_hdr;
	struct feature_fme_port port;
	int i = 0, ret = 0;

	if (!binfo->pfme_hdr) {
		dev_info(binfo,  "VF is detected.\n");
		return ret;
	}

	fme_hdr = binfo->pfme_hdr;

	do {
		port.csr = readq(&fme_hdr->port[i]);
		if (!port.port_implemented)
			break;

		/* skip port which only could be accessed via VF */
		if (port.afu_access_control == FME_AFU_ACCESS_VF)
			continue;

		ret = parse_switch_to(binfo, port.port_bar);
		if (ret)
			break;

		ret = parse_feature_list(binfo,
					 (u8 __iomem *)binfo->ioaddr +
					  port.port_offset);
		if (ret)
			break;
	} while (++i < MAX_FPGA_PORT_NUM);

	return ret;
}

static struct build_feature_devs_info *
build_info_alloc_and_init(struct ifpga_hw *hw)
{
	struct build_feature_devs_info *binfo;

	binfo = zmalloc(sizeof(*binfo));
	if (!binfo)
		return binfo;

	binfo->hw = hw;
	binfo->pci_data = hw->pci_data;

	/* fpga feature list starts from BAR 0 */
	if (parse_switch_to(binfo, 0)) {
		free(binfo);
		return NULL;
	}

	return binfo;
}

static void build_info_free(struct build_feature_devs_info *binfo)
{
	free(binfo);
}

static void ifpga_print_device_feature_list(struct ifpga_hw *hw)
{
	struct ifpga_fme_hw *fme = &hw->fme;
	struct ifpga_port_hw *port;
	struct ifpga_feature *feature;
	int i;

	dev_info(hw, "found fme_device, is in PF: %s\n",
		 is_ifpga_hw_pf(hw) ? "yes" : "no");

	ifpga_for_each_fme_feature(fme, feature) {
		if (feature->state != IFPGA_FEATURE_ATTACHED)
			continue;

		dev_info(hw, "%12s:	%p - %p  - paddr: 0x%lx\n",
			 feature->name, feature->addr,
			 feature->addr + feature->size - 1,
			 (unsigned long)feature->phys_addr);

	}

	for (i = 0; i < MAX_FPGA_PORT_NUM; i++) {
		port = &hw->port[i];

		if (port->state != IFPGA_PORT_ATTACHED)
			continue;

		dev_info(hw, "port device: %d\n", port->port_id);

		ifpga_for_each_port_feature(port, feature) {
			if (feature->state != IFPGA_FEATURE_ATTACHED)
				continue;

			dev_info(hw, "%12s:	%p - %p  - paddr:0x%lx\n",
				 feature->name,
				 feature->addr,
				 feature->addr +
				 feature->size - 1,
				 (unsigned long)feature->phys_addr);
		}

	}
}

int ifpga_bus_enumerate(struct ifpga_hw *hw)
{
	struct build_feature_devs_info *binfo;
	int ret;

	binfo = build_info_alloc_and_init(hw);
	if (!binfo)
		return -ENOMEM;

	ret = parse_feature_list(binfo, binfo->ioaddr);
	if (ret)
		goto exit;

	ret = parse_ports_from_fme(binfo);
	if (ret)
		goto exit;

	ifpga_print_device_feature_list(hw);

exit:
	build_info_free(binfo);
	return ret;
}

int ifpga_bus_init(struct ifpga_hw *hw)
{
	int i;
	struct ifpga_port_hw *port;

	fme_hw_init(&hw->fme);
	for (i = 0; i < MAX_FPGA_PORT_NUM; i++) {
		port = &hw->port[i];
		port_hw_init(port);
	}

	return 0;
}
Commit	Line	Data
11fdf7f2 TL	1	/* SPDX-License-Identifier: BSD-3-Clause
	2	* Copyright(c) 2010-2018 Intel Corporation
	3	*/
	4
	5	#include "opae_hw_api.h"
	6	#include "ifpga_api.h"
	7
	8	#include "ifpga_hw.h"
	9	#include "ifpga_enumerate.h"
	10	#include "ifpga_feature_dev.h"
	11
	12	struct build_feature_devs_info {
	13	struct opae_adapter_data_pci *pci_data;
	14
	15	struct ifpga_afu_info *acc_info;
	16
	17	void *fiu;
	18	enum fpga_id_type current_type;
	19	int current_port_id;
	20
	21	void *ioaddr;
	22	void *ioend;
	23	uint64_t phys_addr;
	24	int current_bar;
	25
	26	void *pfme_hdr;
	27
	28	struct ifpga_hw *hw;
	29	};
	30
9f95a23c TL	31	static int feature_revision(void __iomem *start)
	32	{
	33	struct feature_header header;
11fdf7f2	34
9f95a23c	35	header.csr = readq(start);
11fdf7f2	36
9f95a23c TL	37	return header.revision;
	38	}
	39
	40	static u32 feature_size(void __iomem *start)
	41	{
	42	struct feature_header header;
	43
	44	header.csr = readq(start);
	45
	46	/the size of private feature is 4KB aligned/
	47	return header.next_header_offset ? header.next_header_offset:4096;
	48	}
11fdf7f2 TL	49
	50	static u64 feature_id(void __iomem *start)
	51	{
	52	struct feature_header header;
	53
	54	header.csr = readq(start);
	55
	56	switch (header.type) {
	57	case FEATURE_TYPE_FIU:
9f95a23c	58	return FEATURE_ID_FIU_HEADER;
11fdf7f2 TL	59	case FEATURE_TYPE_PRIVATE:
	60	return header.id;
	61	case FEATURE_TYPE_AFU:
	62	return FEATURE_ID_AFU;
	63	}
	64
	65	WARN_ON(1);
	66	return 0;
	67	}
	68
	69	static int
	70	build_info_add_sub_feature(struct build_feature_devs_info *binfo,
9f95a23c TL	71	void __iomem *start, u64 fid, unsigned int size,
	72	unsigned int vec_start,
	73	unsigned int vec_cnt)
11fdf7f2 TL	74	{
11fdf7f2 TL	75	struct ifpga_hw *hw = binfo->hw;
9f95a23c	76	struct ifpga_feature *feature = NULL;
11fdf7f2 TL	77	struct feature_irq_ctx *ctx = NULL;
	78	int port_id, ret = 0;
	79	unsigned int i;
	80
9f95a23c TL	81	fid = fid?fid:feature_id(start);
	82	size = size?size:feature_size(start);
	83
	84	feature = opae_malloc(sizeof(struct ifpga_feature));
	85	if (!feature)
	86	return -ENOMEM;
11fdf7f2 TL	87
	88	feature->state = IFPGA_FEATURE_ATTACHED;
	89	feature->addr = start;
9f95a23c TL	90	feature->id = fid;
	91	feature->size = size;
	92	feature->revision = feature_revision(start);
11fdf7f2 TL	93	feature->phys_addr = binfo->phys_addr +
11fdf7f2 TL	94	((u8 )start - (u8 )binfo->ioaddr);
9f95a23c TL	95	feature->vec_start = vec_start;
	96	feature->vec_cnt = vec_cnt;
	97
	98	dev_debug(binfo, "%s: id=0x%llx, phys_addr=0x%llx, size=%u\n",
	99	__func__, (unsigned long long)feature->id,
	100	(unsigned long long)feature->phys_addr, size);
11fdf7f2 TL	101
	102	if (vec_cnt) {
	103	if (vec_start + vec_cnt <= vec_start)
	104	return -EINVAL;
	105
	106	ctx = zmalloc(sizeof(ctx) vec_cnt);
	107	if (!ctx)
	108	return -ENOMEM;
	109
	110	for (i = 0; i < vec_cnt; i++) {
	111	ctx[i].eventfd = -1;
	112	ctx[i].idx = vec_start + i;
	113	}
	114	}
	115
	116	feature->ctx = ctx;
	117	feature->ctx_num = vec_cnt;
	118	feature->vfio_dev_fd = binfo->pci_data->vfio_dev_fd;
	119
9f95a23c TL	120	if (binfo->current_type == FME_ID) {
	121	feature->parent = &hw->fme;
	122	feature->type = FEATURE_FME_TYPE;
	123	feature->name = get_fme_feature_name(fid);
	124	TAILQ_INSERT_TAIL(&hw->fme.feature_list, feature, next);
	125	} else if (binfo->current_type == PORT_ID) {
	126	port_id = binfo->current_port_id;
	127	feature->parent = &hw->port[port_id];
	128	feature->type = FEATURE_PORT_TYPE;
	129	feature->name = get_port_feature_name(fid);
	130	TAILQ_INSERT_TAIL(&hw->port[port_id].feature_list,
	131	feature, next);
	132	} else {
	133	return -EFAULT;
	134	}
11fdf7f2 TL	135	return ret;
	136	}
	137
	138	static int
	139	create_feature_instance(struct build_feature_devs_info *binfo,
9f95a23c TL	140	void __iomem *start, u64 fid,
	141	unsigned int size, unsigned int vec_start,
	142	unsigned int vec_cnt)
11fdf7f2	143	{
9f95a23c TL	144	return build_info_add_sub_feature(binfo, start, fid, size, vec_start,
9f95a23c TL	145	vec_cnt);
11fdf7f2 TL	146	}
	147
	148	/*
	149	* UAFU GUID is dynamic as it can be changed after FME downloads different
	150	* Green Bitstream to the port, so we treat the unknown GUIDs which are
	151	* attached on port's feature list as UAFU.
	152	*/
	153	static bool feature_is_UAFU(struct build_feature_devs_info *binfo)
	154	{
	155	if (binfo->current_type != PORT_ID)
	156	return false;
	157
	158	return true;
	159	}
	160
	161	static int parse_feature_port_uafu(struct build_feature_devs_info *binfo,
	162	struct feature_header *hdr)
	163	{
9f95a23c	164	u64 id = PORT_FEATURE_ID_UAFU;
11fdf7f2 TL	165	struct ifpga_afu_info *info;
11fdf7f2 TL	166	void start = (void )hdr;
9f95a23c TL	167	struct feature_port_header *port_hdr = binfo->ioaddr;
9f95a23c TL	168	struct feature_port_capability capability;
11fdf7f2	169	int ret;
9f95a23c	170	int size;
11fdf7f2	171
9f95a23c TL	172	capability.csr = readq(&port_hdr->capability);
	173
	174	size = capability.mmio_size << 10;
11fdf7f2	175
9f95a23c	176	ret = create_feature_instance(binfo, hdr, id, size, 0, 0);
11fdf7f2 TL	177	if (ret)
	178	return ret;
	179
9f95a23c	180	info = opae_malloc(sizeof(*info));
11fdf7f2 TL	181	if (!info)
	182	return -ENOMEM;
	183
	184	info->region[0].addr = start;
	185	info->region[0].phys_addr = binfo->phys_addr +
	186	(uint8_t )start - (uint8_t )binfo->ioaddr;
9f95a23c	187	info->region[0].len = size;
11fdf7f2 TL	188	info->num_regions = 1;
	189
	190	binfo->acc_info = info;
	191
	192	return ret;
	193	}
	194
	195	static int parse_feature_afus(struct build_feature_devs_info *binfo,
	196	struct feature_header *hdr)
	197	{
	198	int ret;
	199	struct feature_afu_header *afu_hdr, header;
	200	u8 __iomem *start;
	201	u8 __iomem *end = binfo->ioend;
	202
	203	start = (u8 __iomem *)hdr;
	204	for (; start < end; start += header.next_afu) {
	205	if ((unsigned int)(end - start) <
	206	(unsigned int)(sizeof(afu_hdr) + sizeof(hdr)))
	207	return -EINVAL;
	208
	209	hdr = (struct feature_header *)start;
	210	afu_hdr = (struct feature_afu_header *)(hdr + 1);
	211	header.csr = readq(&afu_hdr->csr);
	212
	213	if (feature_is_UAFU(binfo)) {
	214	ret = parse_feature_port_uafu(binfo, hdr);
	215	if (ret)
	216	return ret;
	217	}
	218
	219	if (!header.next_afu)
	220	break;
	221	}
	222
	223	return 0;
	224	}
	225
	226	/* create and register proper private data */
	227	static int build_info_commit_dev(struct build_feature_devs_info *binfo)
	228	{
	229	struct ifpga_afu_info *info = binfo->acc_info;
	230	struct ifpga_hw *hw = binfo->hw;
	231	struct opae_manager *mgr;
	232	struct opae_bridge *br;
	233	struct opae_accelerator *acc;
9f95a23c TL	234	struct ifpga_port_hw *port;
	235	struct ifpga_fme_hw *fme;
	236	struct ifpga_feature *feature;
11fdf7f2 TL	237
	238	if (!binfo->fiu)
	239	return 0;
	240
	241	if (binfo->current_type == PORT_ID) {
	242	/* return error if no valid acc info data structure */
	243	if (!info)
	244	return -EFAULT;
	245
	246	br = opae_bridge_alloc(hw->adapter->name, &ifpga_br_ops,
	247	binfo->fiu);
	248	if (!br)
	249	return -ENOMEM;
	250
	251	br->id = binfo->current_port_id;
	252
	253	/* update irq info */
9f95a23c TL	254	port = &hw->port[binfo->current_port_id];
	255	feature = get_feature_by_id(&port->feature_list,
	256	PORT_FEATURE_ID_UINT);
	257	if (feature)
	258	info->num_irqs = feature->vec_cnt;
11fdf7f2 TL	259
	260	acc = opae_accelerator_alloc(hw->adapter->name,
	261	&ifpga_acc_ops, info);
	262	if (!acc) {
	263	opae_bridge_free(br);
	264	return -ENOMEM;
	265	}
	266
	267	acc->br = br;
9f95a23c TL	268	if (hw->adapter->mgr)
9f95a23c TL	269	acc->mgr = hw->adapter->mgr;
11fdf7f2 TL	270	acc->index = br->id;
11fdf7f2 TL	271
9f95a23c TL	272	fme = &hw->fme;
	273	fme->nums_acc_region = info->num_regions;
	274
11fdf7f2 TL	275	opae_adapter_add_acc(hw->adapter, acc);
	276
	277	} else if (binfo->current_type == FME_ID) {
	278	mgr = opae_manager_alloc(hw->adapter->name, &ifpga_mgr_ops,
9f95a23c	279	&ifpga_mgr_network_ops, binfo->fiu);
11fdf7f2 TL	280	if (!mgr)
	281	return -ENOMEM;
	282
	283	mgr->adapter = hw->adapter;
	284	hw->adapter->mgr = mgr;
	285	}
	286
	287	binfo->fiu = NULL;
	288
	289	return 0;
	290	}
	291
	292	static int
	293	build_info_create_dev(struct build_feature_devs_info *binfo,
	294	enum fpga_id_type type, unsigned int index)
	295	{
	296	int ret;
	297
	298	ret = build_info_commit_dev(binfo);
	299	if (ret)
	300	return ret;
	301
	302	binfo->current_type = type;
	303
	304	if (type == FME_ID) {
	305	binfo->fiu = &binfo->hw->fme;
	306	} else if (type == PORT_ID) {
	307	binfo->fiu = &binfo->hw->port[index];
	308	binfo->current_port_id = index;
	309	}
	310
	311	return 0;
	312	}
	313
	314	static int parse_feature_fme(struct build_feature_devs_info *binfo,
	315	struct feature_header *start)
	316	{
	317	struct ifpga_hw *hw = binfo->hw;
	318	struct ifpga_fme_hw *fme = &hw->fme;
	319	int ret;
	320
	321	ret = build_info_create_dev(binfo, FME_ID, 0);
	322	if (ret)
	323	return ret;
	324
	325	/* Update FME states */
	326	fme->state = IFPGA_FME_IMPLEMENTED;
	327	fme->parent = hw;
9f95a23c	328	TAILQ_INIT(&fme->feature_list);
11fdf7f2 TL	329	spinlock_init(&fme->lock);
11fdf7f2 TL	330
9f95a23c	331	return create_feature_instance(binfo, start, 0, 0, 0, 0);
11fdf7f2 TL	332	}
	333
	334	static int parse_feature_port(struct build_feature_devs_info *binfo,
	335	void __iomem *start)
	336	{
	337	struct feature_port_header *port_hdr;
	338	struct feature_port_capability capability;
	339	struct ifpga_hw *hw = binfo->hw;
	340	struct ifpga_port_hw *port;
	341	unsigned int port_id;
	342	int ret;
	343
	344	/* Get current port's id */
	345	port_hdr = (struct feature_port_header *)start;
	346	capability.csr = readq(&port_hdr->capability);
	347	port_id = capability.port_number;
	348
	349	ret = build_info_create_dev(binfo, PORT_ID, port_id);
	350	if (ret)
	351	return ret;
	352
	353	/found a Port device/
	354	port = &hw->port[port_id];
	355	port->port_id = binfo->current_port_id;
	356	port->parent = hw;
	357	port->state = IFPGA_PORT_ATTACHED;
	358	spinlock_init(&port->lock);
9f95a23c	359	TAILQ_INIT(&port->feature_list);
11fdf7f2	360
9f95a23c	361	return create_feature_instance(binfo, start, 0, 0, 0, 0);
11fdf7f2 TL	362	}
	363
	364	static void enable_port_uafu(struct build_feature_devs_info *binfo,
	365	void __iomem *start)
	366	{
11fdf7f2 TL	367	struct ifpga_port_hw *port = &binfo->hw->port[binfo->current_port_id];
11fdf7f2 TL	368
9f95a23c TL	369	UNUSED(start);
	370
	371	fpga_port_reset(port);
11fdf7f2 TL	372	}
	373
	374	static int parse_feature_fiu(struct build_feature_devs_info *binfo,
	375	struct feature_header *hdr)
	376	{
	377	struct feature_header header;
	378	struct feature_fiu_header *fiu_hdr, fiu_header;
	379	u8 __iomem start = (u8 __iomem )hdr;
	380	int ret;
	381
	382	header.csr = readq(hdr);
	383
	384	switch (header.id) {
	385	case FEATURE_FIU_ID_FME:
	386	ret = parse_feature_fme(binfo, hdr);
	387	binfo->pfme_hdr = hdr;
	388	if (ret)
	389	return ret;
	390	break;
	391	case FEATURE_FIU_ID_PORT:
	392	ret = parse_feature_port(binfo, hdr);
	393	enable_port_uafu(binfo, hdr);
	394	if (ret)
	395	return ret;
	396
	397	/* Check Port FIU's next_afu pointer to User AFU DFH */
	398	fiu_hdr = (struct feature_fiu_header *)(hdr + 1);
	399	fiu_header.csr = readq(&fiu_hdr->csr);
	400
	401	if (fiu_header.next_afu) {
	402	start += fiu_header.next_afu;
	403	ret = parse_feature_afus(binfo,
	404	(struct feature_header *)start);
	405	if (ret)
	406	return ret;
	407	} else {
	408	dev_info(binfo, "No AFUs detected on Port\n");
	409	}
	410
	411	break;
	412	default:
	413	dev_info(binfo, "FIU TYPE %d is not supported yet.\n",
	414	header.id);
	415	}
	416
	417	return 0;
	418	}
	419
	420	static void parse_feature_irqs(struct build_feature_devs_info *binfo,
9f95a23c TL	421	void __iomem start, unsigned int vec_start,
9f95a23c TL	422	unsigned int *vec_cnt)
11fdf7f2	423	{
11fdf7f2	424	UNUSED(binfo);
9f95a23c TL	425	u64 id;
	426
	427	id = feature_id(start);
11fdf7f2	428
9f95a23c	429	if (id == PORT_FEATURE_ID_UINT) {
11fdf7f2 TL	430	struct feature_port_uint *port_uint = start;
	431	struct feature_port_uint_cap uint_cap;
	432
	433	uint_cap.csr = readq(&port_uint->capability);
	434	if (uint_cap.intr_num) {
9f95a23c TL	435	*vec_start = uint_cap.first_vec_num;
9f95a23c TL	436	*vec_cnt = uint_cap.intr_num;
11fdf7f2 TL	437	} else {
	438	dev_debug(binfo, "UAFU doesn't support interrupt\n");
	439	}
9f95a23c	440	} else if (id == PORT_FEATURE_ID_ERROR) {
11fdf7f2 TL	441	struct feature_port_error *port_err = start;
	442	struct feature_port_err_capability port_err_cap;
	443
	444	port_err_cap.csr = readq(&port_err->error_capability);
	445	if (port_err_cap.support_intr) {
9f95a23c TL	446	*vec_start = port_err_cap.intr_vector_num;
9f95a23c TL	447	*vec_cnt = 1;
11fdf7f2 TL	448	} else {
	449	dev_debug(&binfo, "Port error doesn't support interrupt\n");
	450	}
	451
9f95a23c	452	} else if (id == FME_FEATURE_ID_GLOBAL_ERR) {
11fdf7f2 TL	453	struct feature_fme_err *fme_err = start;
	454	struct feature_fme_error_capability fme_err_cap;
	455
	456	fme_err_cap.csr = readq(&fme_err->fme_err_capability);
	457	if (fme_err_cap.support_intr) {
9f95a23c TL	458	*vec_start = fme_err_cap.intr_vector_num;
9f95a23c TL	459	*vec_cnt = 1;
11fdf7f2 TL	460	} else {
	461	dev_debug(&binfo, "FME error doesn't support interrupt\n");
	462	}
	463	}
	464	}
	465
	466	static int parse_feature_fme_private(struct build_feature_devs_info *binfo,
	467	struct feature_header *hdr)
	468	{
9f95a23c TL	469	unsigned int vec_start = 0;
9f95a23c TL	470	unsigned int vec_cnt = 0;
11fdf7f2	471
9f95a23c	472	parse_feature_irqs(binfo, hdr, &vec_start, &vec_cnt);
11fdf7f2	473
9f95a23c	474	return create_feature_instance(binfo, hdr, 0, 0, vec_start, vec_cnt);
11fdf7f2 TL	475	}
	476
	477	static int parse_feature_port_private(struct build_feature_devs_info *binfo,
	478	struct feature_header *hdr)
	479	{
9f95a23c TL	480	unsigned int vec_start = 0;
9f95a23c TL	481	unsigned int vec_cnt = 0;
11fdf7f2	482
9f95a23c	483	parse_feature_irqs(binfo, hdr, &vec_start, &vec_cnt);
11fdf7f2	484
9f95a23c	485	return create_feature_instance(binfo, hdr, 0, 0, vec_start, vec_cnt);
11fdf7f2 TL	486	}
	487
	488	static int parse_feature_private(struct build_feature_devs_info *binfo,
	489	struct feature_header *hdr)
	490	{
	491	struct feature_header header;
	492
	493	header.csr = readq(hdr);
	494
	495	switch (binfo->current_type) {
	496	case FME_ID:
	497	return parse_feature_fme_private(binfo, hdr);
	498	case PORT_ID:
	499	return parse_feature_port_private(binfo, hdr);
	500	default:
	501	dev_err(binfo, "private feature %x belonging to AFU %d (unknown_type) is not supported yet.\n",
	502	header.id, binfo->current_type);
	503	}
	504	return 0;
	505	}
	506
	507	static int parse_feature(struct build_feature_devs_info *binfo,
	508	struct feature_header *hdr)
	509	{
	510	struct feature_header header;
	511	int ret = 0;
	512
	513	header.csr = readq(hdr);
	514
	515	switch (header.type) {
	516	case FEATURE_TYPE_AFU:
	517	ret = parse_feature_afus(binfo, hdr);
	518	break;
	519	case FEATURE_TYPE_PRIVATE:
	520	ret = parse_feature_private(binfo, hdr);
	521	break;
	522	case FEATURE_TYPE_FIU:
	523	ret = parse_feature_fiu(binfo, hdr);
	524	break;
	525	default:
	526	dev_err(binfo, "Feature Type %x is not supported.\n",
	527	hdr->type);
	528	};
	529
	530	return ret;
	531	}
	532
	533	static int
	534	parse_feature_list(struct build_feature_devs_info binfo, u8 __iomem start)
	535	{
	536	struct feature_header *hdr, header;
	537	u8 __iomem end = (u8 __iomem )binfo->ioend;
	538	int ret = 0;
	539
	540	for (; start < end; start += header.next_header_offset) {
	541	if ((unsigned int)(end - start) < (unsigned int)sizeof(*hdr)) {
	542	dev_err(binfo, "The region is too small to contain a feature.\n");
	543	ret = -EINVAL;
	544	break;
	545	}
	546
	547	hdr = (struct feature_header *)start;
9f95a23c TL	548	header.csr = readq(hdr);
	549
	550	dev_debug(binfo, "%s: address=0x%p, val=0x%llx, header.id=0x%x, header.next_offset=0x%x, header.eol=0x%x, header.type=0x%x\n",
	551	__func__, hdr, (unsigned long long)header.csr,
	552	header.id, header.next_header_offset,
	553	header.end_of_list, header.type);
	554
11fdf7f2 TL	555	ret = parse_feature(binfo, hdr);
	556	if (ret)
	557	return ret;
	558
9f95a23c	559	if (header.end_of_list \|\| !header.next_header_offset)
11fdf7f2 TL	560	break;
	561	}
	562
	563	return build_info_commit_dev(binfo);
	564	}
	565
	566	/* switch the memory mapping to BAR# @bar */
	567	static int parse_switch_to(struct build_feature_devs_info *binfo, int bar)
	568	{
	569	struct opae_adapter_data_pci *pci_data = binfo->pci_data;
	570
	571	if (!pci_data->region[bar].addr)
	572	return -ENOMEM;
	573
	574	binfo->ioaddr = pci_data->region[bar].addr;
	575	binfo->ioend = (u8 __iomem *)binfo->ioaddr + pci_data->region[bar].len;
	576	binfo->phys_addr = pci_data->region[bar].phys_addr;
	577	binfo->current_bar = bar;
	578
	579	return 0;
	580	}
	581
	582	static int parse_ports_from_fme(struct build_feature_devs_info *binfo)
	583	{
	584	struct feature_fme_header *fme_hdr;
	585	struct feature_fme_port port;
	586	int i = 0, ret = 0;
	587
	588	if (!binfo->pfme_hdr) {
	589	dev_info(binfo, "VF is detected.\n");
	590	return ret;
	591	}
	592
	593	fme_hdr = binfo->pfme_hdr;
	594
	595	do {
	596	port.csr = readq(&fme_hdr->port[i]);
	597	if (!port.port_implemented)
	598	break;
	599
	600	/* skip port which only could be accessed via VF */
	601	if (port.afu_access_control == FME_AFU_ACCESS_VF)
	602	continue;
	603
	604	ret = parse_switch_to(binfo, port.port_bar);
	605	if (ret)
	606	break;
	607
	608	ret = parse_feature_list(binfo,
	609	(u8 __iomem *)binfo->ioaddr +
	610	port.port_offset);
	611	if (ret)
	612	break;
	613	} while (++i < MAX_FPGA_PORT_NUM);
	614
	615	return ret;
	616	}
	617
	618	static struct build_feature_devs_info *
	619	build_info_alloc_and_init(struct ifpga_hw *hw)
	620	{
	621	struct build_feature_devs_info *binfo;
	622
	623	binfo = zmalloc(sizeof(*binfo));
624	if (!binfo)
625	return binfo;
626
627	binfo->hw = hw;
628	binfo->pci_data = hw->pci_data;
629
630	/* fpga feature list starts from BAR 0 */
631	if (parse_switch_to(binfo, 0)) {
632	free(binfo);
633	return NULL;
634	}
635
636	return binfo;
637	}
638
639	static void build_info_free(struct build_feature_devs_info *binfo)
640	{
641	free(binfo);
642	}
643
644	static void ifpga_print_device_feature_list(struct ifpga_hw *hw)
645	{
646	struct ifpga_fme_hw *fme = &hw->fme;
647	struct ifpga_port_hw *port;
9f95a23c TL	648	struct ifpga_feature *feature;
9f95a23c TL	649	int i;
11fdf7f2 TL	650
	651	dev_info(hw, "found fme_device, is in PF: %s\n",
	652	is_ifpga_hw_pf(hw) ? "yes" : "no");
	653
9f95a23c	654	ifpga_for_each_fme_feature(fme, feature) {
11fdf7f2 TL	655	if (feature->state != IFPGA_FEATURE_ATTACHED)
	656	continue;
	657
9f95a23c	658	dev_info(hw, "%12s: %p - %p - paddr: 0x%lx\n",
11fdf7f2 TL	659	feature->name, feature->addr,
	660	feature->addr + feature->size - 1,
	661	(unsigned long)feature->phys_addr);
9f95a23c	662
11fdf7f2 TL	663	}
	664
	665	for (i = 0; i < MAX_FPGA_PORT_NUM; i++) {
	666	port = &hw->port[i];
	667
	668	if (port->state != IFPGA_PORT_ATTACHED)
	669	continue;
	670
	671	dev_info(hw, "port device: %d\n", port->port_id);
	672
9f95a23c	673	ifpga_for_each_port_feature(port, feature) {
11fdf7f2 TL	674	if (feature->state != IFPGA_FEATURE_ATTACHED)
	675	continue;
	676
9f95a23c	677	dev_info(hw, "%12s: %p - %p - paddr:0x%lx\n",
11fdf7f2 TL	678	feature->name,
	679	feature->addr,
	680	feature->addr +
	681	feature->size - 1,
	682	(unsigned long)feature->phys_addr);
	683	}
9f95a23c	684
11fdf7f2 TL	685	}
	686	}
	687
	688	int ifpga_bus_enumerate(struct ifpga_hw *hw)
	689	{
	690	struct build_feature_devs_info *binfo;
	691	int ret;
	692
	693	binfo = build_info_alloc_and_init(hw);
	694	if (!binfo)
	695	return -ENOMEM;
	696
	697	ret = parse_feature_list(binfo, binfo->ioaddr);
	698	if (ret)
	699	goto exit;
	700
	701	ret = parse_ports_from_fme(binfo);
	702	if (ret)
	703	goto exit;
	704
	705	ifpga_print_device_feature_list(hw);
	706
	707	exit:
	708	build_info_free(binfo);
	709	return ret;
	710	}
	711
	712	int ifpga_bus_init(struct ifpga_hw *hw)
	713	{
	714	int i;
9f95a23c	715	struct ifpga_port_hw *port;
11fdf7f2 TL	716
11fdf7f2 TL	717	fme_hw_init(&hw->fme);
9f95a23c TL	718	for (i = 0; i < MAX_FPGA_PORT_NUM; i++) {
	719	port = &hw->port[i];
	720	port_hw_init(port);
	721	}
11fdf7f2 TL	722
	723	return 0;
	724	}