[mirror_ubuntu-jammy-kernel.git] / drivers / dma / idxd / idxd.h

/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
#ifndef _IDXD_H_
#define _IDXD_H_

#include <linux/sbitmap.h>
#include <linux/dmaengine.h>
#include <linux/percpu-rwsem.h>
#include <linux/wait.h>
#include <linux/cdev.h>
#include <linux/idr.h>
#include "registers.h"

#define IDXD_DRIVER_VERSION	"1.00"

extern struct kmem_cache *idxd_desc_pool;

struct idxd_device;
struct idxd_wq;

#define IDXD_REG_TIMEOUT	50
#define IDXD_DRAIN_TIMEOUT	5000

enum idxd_type {
	IDXD_TYPE_UNKNOWN = -1,
	IDXD_TYPE_DSA = 0,
	IDXD_TYPE_IAX,
	IDXD_TYPE_MAX,
};

#define IDXD_NAME_SIZE		128

struct idxd_device_driver {
	struct device_driver drv;
};

struct idxd_irq_entry {
	struct idxd_device *idxd;
	int id;
	int vector;
	struct llist_head pending_llist;
	struct list_head work_list;
	/*
	 * Lock to protect access between irq thread process descriptor
	 * and irq thread processing error descriptor.
	 */
	spinlock_t list_lock;
};

struct idxd_group {
	struct device conf_dev;
	struct idxd_device *idxd;
	struct grpcfg grpcfg;
	int id;
	int num_engines;
	int num_wqs;
	bool use_token_limit;
	u8 tokens_allowed;
	u8 tokens_reserved;
	int tc_a;
	int tc_b;
};

#define IDXD_MAX_PRIORITY	0xf

enum idxd_wq_state {
	IDXD_WQ_DISABLED = 0,
	IDXD_WQ_ENABLED,
};

enum idxd_wq_flag {
	WQ_FLAG_DEDICATED = 0,
	WQ_FLAG_BLOCK_ON_FAULT,
};

enum idxd_wq_type {
	IDXD_WQT_NONE = 0,
	IDXD_WQT_KERNEL,
	IDXD_WQT_USER,
};

struct idxd_cdev {
	struct idxd_wq *wq;
	struct cdev cdev;
	struct device dev;
	int minor;
};

#define IDXD_ALLOCATED_BATCH_SIZE	128U
#define WQ_NAME_SIZE   1024
#define WQ_TYPE_SIZE   10

enum idxd_op_type {
	IDXD_OP_BLOCK = 0,
	IDXD_OP_NONBLOCK = 1,
};

enum idxd_complete_type {
	IDXD_COMPLETE_NORMAL = 0,
	IDXD_COMPLETE_ABORT,
	IDXD_COMPLETE_DEV_FAIL,
};

struct idxd_dma_chan {
	struct dma_chan chan;
	struct idxd_wq *wq;
};

struct idxd_wq {
	void __iomem *portal;
	struct percpu_ref wq_active;
	struct completion wq_dead;
	struct device conf_dev;
	struct idxd_cdev *idxd_cdev;
	struct wait_queue_head err_queue;
	struct idxd_device *idxd;
	int id;
	enum idxd_wq_type type;
	struct idxd_group *group;
	int client_count;
	struct mutex wq_lock;	/* mutex for workqueue */
	u32 size;
	u32 threshold;
	u32 priority;
	enum idxd_wq_state state;
	unsigned long flags;
	union wqcfg *wqcfg;
	u32 vec_ptr;		/* interrupt steering */
	struct dsa_hw_desc **hw_descs;
	int num_descs;
	union {
		struct dsa_completion_record *compls;
		struct iax_completion_record *iax_compls;
	};
	void *compls_raw;
	dma_addr_t compls_addr;
	dma_addr_t compls_addr_raw;
	int compls_size;
	struct idxd_desc **descs;
	struct sbitmap_queue sbq;
	struct idxd_dma_chan *idxd_chan;
	char name[WQ_NAME_SIZE + 1];
	u64 max_xfer_bytes;
	u32 max_batch_size;
	bool ats_dis;
};

struct idxd_engine {
	struct device conf_dev;
	int id;
	struct idxd_group *group;
	struct idxd_device *idxd;
};

/* shadow registers */
struct idxd_hw {
	u32 version;
	union gen_cap_reg gen_cap;
	union wq_cap_reg wq_cap;
	union group_cap_reg group_cap;
	union engine_cap_reg engine_cap;
	struct opcap opcap;
	u32 cmd_cap;
};

enum idxd_device_state {
	IDXD_DEV_HALTED = -1,
	IDXD_DEV_DISABLED = 0,
	IDXD_DEV_CONF_READY,
	IDXD_DEV_ENABLED,
};

enum idxd_device_flag {
	IDXD_FLAG_CONFIGURABLE = 0,
	IDXD_FLAG_CMD_RUNNING,
	IDXD_FLAG_PASID_ENABLED,
};

struct idxd_dma_dev {
	struct idxd_device *idxd;
	struct dma_device dma;
};

struct idxd_driver_data {
	const char *name_prefix;
	enum idxd_type type;
	struct device_type *dev_type;
	int compl_size;
	int align;
};

struct idxd_device {
	struct device conf_dev;
	struct idxd_driver_data *data;
	struct list_head list;
	struct idxd_hw hw;
	enum idxd_device_state state;
	unsigned long flags;
	int id;
	int major;
	u8 cmd_status;

	struct pci_dev *pdev;
	void __iomem *reg_base;

	spinlock_t dev_lock;	/* spinlock for device */
	spinlock_t cmd_lock;	/* spinlock for device commands */
	struct completion *cmd_done;
	struct idxd_group **groups;
	struct idxd_wq **wqs;
	struct idxd_engine **engines;

	struct iommu_sva *sva;
	unsigned int pasid;

	int num_groups;

	u32 msix_perm_offset;
	u32 wqcfg_offset;
	u32 grpcfg_offset;
	u32 perfmon_offset;

	u64 max_xfer_bytes;
	u32 max_batch_size;
	int max_groups;
	int max_engines;
	int max_tokens;
	int max_wqs;
	int max_wq_size;
	int token_limit;
	int nr_tokens;		/* non-reserved tokens */
	unsigned int wqcfg_size;

	union sw_err_reg sw_err;
	wait_queue_head_t cmd_waitq;
	int num_wq_irqs;
	struct idxd_irq_entry *irq_entries;

	struct idxd_dma_dev *idxd_dma;
	struct workqueue_struct *wq;
	struct work_struct work;

	int *int_handles;
};

/* IDXD software descriptor */
struct idxd_desc {
	union {
		struct dsa_hw_desc *hw;
		struct iax_hw_desc *iax_hw;
	};
	dma_addr_t desc_dma;
	union {
		struct dsa_completion_record *completion;
		struct iax_completion_record *iax_completion;
	};
	dma_addr_t compl_dma;
	struct dma_async_tx_descriptor txd;
	struct llist_node llnode;
	struct list_head list;
	int id;
	int cpu;
	unsigned int vector;
	struct idxd_wq *wq;
};

#define confdev_to_idxd(dev) container_of(dev, struct idxd_device, conf_dev)
#define confdev_to_wq(dev) container_of(dev, struct idxd_wq, conf_dev)

extern struct bus_type dsa_bus_type;
extern struct bus_type iax_bus_type;

extern bool support_enqcmd;
extern struct ida idxd_ida;
extern struct device_type dsa_device_type;
extern struct device_type iax_device_type;
extern struct device_type idxd_wq_device_type;
extern struct device_type idxd_engine_device_type;
extern struct device_type idxd_group_device_type;

static inline bool is_dsa_dev(struct device *dev)
{
	return dev->type == &dsa_device_type;
}

static inline bool is_iax_dev(struct device *dev)
{
	return dev->type == &iax_device_type;
}

static inline bool is_idxd_dev(struct device *dev)
{
	return is_dsa_dev(dev) || is_iax_dev(dev);
}

static inline bool is_idxd_wq_dev(struct device *dev)
{
	return dev->type == &idxd_wq_device_type;
}

static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq)
{
	if (wq->type == IDXD_WQT_KERNEL && strcmp(wq->name, "dmaengine") == 0)
		return true;
	return false;
}

static inline bool is_idxd_wq_cdev(struct idxd_wq *wq)
{
	return wq->type == IDXD_WQT_USER;
}

static inline bool wq_dedicated(struct idxd_wq *wq)
{
	return test_bit(WQ_FLAG_DEDICATED, &wq->flags);
}

static inline bool wq_shared(struct idxd_wq *wq)
{
	return !test_bit(WQ_FLAG_DEDICATED, &wq->flags);
}

static inline bool device_pasid_enabled(struct idxd_device *idxd)
{
	return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
}

static inline bool device_swq_supported(struct idxd_device *idxd)
{
	return (support_enqcmd && device_pasid_enabled(idxd));
}

enum idxd_portal_prot {
	IDXD_PORTAL_UNLIMITED = 0,
	IDXD_PORTAL_LIMITED,
};

enum idxd_interrupt_type {
	IDXD_IRQ_MSIX = 0,
	IDXD_IRQ_IMS,
};

static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot)
{
	return prot * 0x1000;
}

static inline int idxd_get_wq_portal_full_offset(int wq_id,
						 enum idxd_portal_prot prot)
{
	return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot);
}

static inline void idxd_wq_get(struct idxd_wq *wq)
{
	wq->client_count++;
}

static inline void idxd_wq_put(struct idxd_wq *wq)
{
	wq->client_count--;
}

static inline int idxd_wq_refcount(struct idxd_wq *wq)
{
	return wq->client_count;
};

int idxd_register_bus_type(void);
void idxd_unregister_bus_type(void);
int idxd_register_devices(struct idxd_device *idxd);
void idxd_unregister_devices(struct idxd_device *idxd);
int idxd_register_driver(void);
void idxd_unregister_driver(void);
void idxd_wqs_quiesce(struct idxd_device *idxd);

/* device interrupt control */
void idxd_msix_perm_setup(struct idxd_device *idxd);
void idxd_msix_perm_clear(struct idxd_device *idxd);
irqreturn_t idxd_misc_thread(int vec, void *data);
irqreturn_t idxd_wq_thread(int irq, void *data);
void idxd_mask_error_interrupts(struct idxd_device *idxd);
void idxd_unmask_error_interrupts(struct idxd_device *idxd);
void idxd_mask_msix_vectors(struct idxd_device *idxd);
void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id);
void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id);

/* device control */
int idxd_device_init_reset(struct idxd_device *idxd);
int idxd_device_enable(struct idxd_device *idxd);
int idxd_device_disable(struct idxd_device *idxd);
void idxd_device_reset(struct idxd_device *idxd);
void idxd_device_cleanup(struct idxd_device *idxd);
int idxd_device_config(struct idxd_device *idxd);
void idxd_device_wqs_clear_state(struct idxd_device *idxd);
void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid);
int idxd_device_load_config(struct idxd_device *idxd);
int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle,
				   enum idxd_interrupt_type irq_type);
int idxd_device_release_int_handle(struct idxd_device *idxd, int handle,
				   enum idxd_interrupt_type irq_type);

/* work queue control */
void idxd_wqs_unmap_portal(struct idxd_device *idxd);
int idxd_wq_alloc_resources(struct idxd_wq *wq);
void idxd_wq_free_resources(struct idxd_wq *wq);
int idxd_wq_enable(struct idxd_wq *wq);
int idxd_wq_disable(struct idxd_wq *wq);
void idxd_wq_drain(struct idxd_wq *wq);
void idxd_wq_reset(struct idxd_wq *wq);
int idxd_wq_map_portal(struct idxd_wq *wq);
void idxd_wq_unmap_portal(struct idxd_wq *wq);
void idxd_wq_disable_cleanup(struct idxd_wq *wq);
int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid);
int idxd_wq_disable_pasid(struct idxd_wq *wq);
void idxd_wq_quiesce(struct idxd_wq *wq);
int idxd_wq_init_percpu_ref(struct idxd_wq *wq);

/* submission */
int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc);
struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype);
void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc);

/* dmaengine */
int idxd_register_dma_device(struct idxd_device *idxd);
void idxd_unregister_dma_device(struct idxd_device *idxd);
int idxd_register_dma_channel(struct idxd_wq *wq);
void idxd_unregister_dma_channel(struct idxd_wq *wq);
void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res);
void idxd_dma_complete_txd(struct idxd_desc *desc,
			   enum idxd_complete_type comp_type);

/* cdev */
int idxd_cdev_register(void);
void idxd_cdev_remove(void);
int idxd_cdev_get_major(struct idxd_device *idxd);
int idxd_wq_add_cdev(struct idxd_wq *wq);
void idxd_wq_del_cdev(struct idxd_wq *wq);

#endif
Commit	Line	Data
bfe1d560 DJ	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
	3	#ifndef _IDXD_H_
	4	#define _IDXD_H_
	5
	6	#include <linux/sbitmap.h>
8f47d1a5	7	#include <linux/dmaengine.h>
bfe1d560 DJ	8	#include <linux/percpu-rwsem.h>
bfe1d560 DJ	9	#include <linux/wait.h>
42d279f9	10	#include <linux/cdev.h>
47c16ac2	11	#include <linux/idr.h>
bfe1d560 DJ	12	#include "registers.h"
	13
	14	#define IDXD_DRIVER_VERSION "1.00"
	15
	16	extern struct kmem_cache *idxd_desc_pool;
	17
39786285 DJ	18	struct idxd_device;
	19	struct idxd_wq;
	20
bfe1d560 DJ	21	#define IDXD_REG_TIMEOUT 50
	22	#define IDXD_DRAIN_TIMEOUT 5000
	23
	24	enum idxd_type {
	25	IDXD_TYPE_UNKNOWN = -1,
	26	IDXD_TYPE_DSA = 0,
f25b4638 DJ	27	IDXD_TYPE_IAX,
f25b4638 DJ	28	IDXD_TYPE_MAX,
bfe1d560 DJ	29	};
	30
	31	#define IDXD_NAME_SIZE 128
	32
	33	struct idxd_device_driver {
	34	struct device_driver drv;
	35	};
	36
	37	struct idxd_irq_entry {
	38	struct idxd_device *idxd;
	39	int id;
5fc8e85f	40	int vector;
bfe1d560 DJ	41	struct llist_head pending_llist;
bfe1d560 DJ	42	struct list_head work_list;
e4f4d8cd DJ	43	/*
	44	* Lock to protect access between irq thread process descriptor
	45	* and irq thread processing error descriptor.
	46	*/
	47	spinlock_t list_lock;
bfe1d560 DJ	48	};
	49
	50	struct idxd_group {
	51	struct device conf_dev;
	52	struct idxd_device *idxd;
	53	struct grpcfg grpcfg;
	54	int id;
	55	int num_engines;
	56	int num_wqs;
	57	bool use_token_limit;
	58	u8 tokens_allowed;
	59	u8 tokens_reserved;
	60	int tc_a;
	61	int tc_b;
	62	};
	63
	64	#define IDXD_MAX_PRIORITY 0xf
	65
	66	enum idxd_wq_state {
	67	IDXD_WQ_DISABLED = 0,
	68	IDXD_WQ_ENABLED,
	69	};
	70
	71	enum idxd_wq_flag {
	72	WQ_FLAG_DEDICATED = 0,
8e50d392	73	WQ_FLAG_BLOCK_ON_FAULT,
bfe1d560 DJ	74	};
	75
	76	enum idxd_wq_type {
	77	IDXD_WQT_NONE = 0,
	78	IDXD_WQT_KERNEL,
42d279f9 DJ	79	IDXD_WQT_USER,
	80	};
	81
	82	struct idxd_cdev {
04922b74	83	struct idxd_wq *wq;
42d279f9	84	struct cdev cdev;
04922b74	85	struct device dev;
42d279f9	86	int minor;
bfe1d560 DJ	87	};
	88
	89	#define IDXD_ALLOCATED_BATCH_SIZE 128U
	90	#define WQ_NAME_SIZE 1024
	91	#define WQ_TYPE_SIZE 10
	92
d1dfe5b8 DJ	93	enum idxd_op_type {
	94	IDXD_OP_BLOCK = 0,
	95	IDXD_OP_NONBLOCK = 1,
	96	};
	97
8f47d1a5 DJ	98	enum idxd_complete_type {
	99	IDXD_COMPLETE_NORMAL = 0,
	100	IDXD_COMPLETE_ABORT,
8e50d392	101	IDXD_COMPLETE_DEV_FAIL,
8f47d1a5 DJ	102	};
8f47d1a5 DJ	103
39786285 DJ	104	struct idxd_dma_chan {
	105	struct dma_chan chan;
	106	struct idxd_wq *wq;
	107	};
	108
bfe1d560	109	struct idxd_wq {
8e50d392	110	void __iomem *portal;
93a40a6d DJ	111	struct percpu_ref wq_active;
93a40a6d DJ	112	struct completion wq_dead;
bfe1d560	113	struct device conf_dev;
04922b74 DJ	114	struct idxd_cdev *idxd_cdev;
04922b74 DJ	115	struct wait_queue_head err_queue;
bfe1d560 DJ	116	struct idxd_device *idxd;
	117	int id;
	118	enum idxd_wq_type type;
	119	struct idxd_group *group;
	120	int client_count;
	121	struct mutex wq_lock; /* mutex for workqueue */
	122	u32 size;
	123	u32 threshold;
	124	u32 priority;
	125	enum idxd_wq_state state;
	126	unsigned long flags;
d98793b5	127	union wqcfg *wqcfg;
bfe1d560 DJ	128	u32 vec_ptr; /* interrupt steering */
	129	struct dsa_hw_desc **hw_descs;
	130	int num_descs;
f25b4638 DJ	131	union {
	132	struct dsa_completion_record *compls;
	133	struct iax_completion_record *iax_compls;
	134	};
	135	void *compls_raw;
bfe1d560	136	dma_addr_t compls_addr;
f25b4638	137	dma_addr_t compls_addr_raw;
bfe1d560 DJ	138	int compls_size;
bfe1d560 DJ	139	struct idxd_desc **descs;
0705107f	140	struct sbitmap_queue sbq;
39786285	141	struct idxd_dma_chan *idxd_chan;
bfe1d560	142	char name[WQ_NAME_SIZE + 1];
d7aad555	143	u64 max_xfer_bytes;
e7184b15	144	u32 max_batch_size;
92de5fa2	145	bool ats_dis;
bfe1d560 DJ	146	};
	147
	148	struct idxd_engine {
	149	struct device conf_dev;
	150	int id;
	151	struct idxd_group *group;
	152	struct idxd_device *idxd;
	153	};
	154
	155	/* shadow registers */
	156	struct idxd_hw {
	157	u32 version;
	158	union gen_cap_reg gen_cap;
	159	union wq_cap_reg wq_cap;
	160	union group_cap_reg group_cap;
	161	union engine_cap_reg engine_cap;
	162	struct opcap opcap;
eb15e715	163	u32 cmd_cap;
bfe1d560 DJ	164	};
	165
	166	enum idxd_device_state {
	167	IDXD_DEV_HALTED = -1,
	168	IDXD_DEV_DISABLED = 0,
	169	IDXD_DEV_CONF_READY,
	170	IDXD_DEV_ENABLED,
	171	};
	172
	173	enum idxd_device_flag {
	174	IDXD_FLAG_CONFIGURABLE = 0,
0d5c10b4	175	IDXD_FLAG_CMD_RUNNING,
8e50d392	176	IDXD_FLAG_PASID_ENABLED,
bfe1d560 DJ	177	};
bfe1d560 DJ	178
39786285 DJ	179	struct idxd_dma_dev {
	180	struct idxd_device *idxd;
	181	struct dma_device dma;
	182	};
	183
435b512d DJ	184	struct idxd_driver_data {
435b512d DJ	185	const char *name_prefix;
bfe1d560	186	enum idxd_type type;
435b512d DJ	187	struct device_type *dev_type;
	188	int compl_size;
	189	int align;
	190	};
	191
	192	struct idxd_device {
bfe1d560	193	struct device conf_dev;
435b512d	194	struct idxd_driver_data *data;
bfe1d560 DJ	195	struct list_head list;
	196	struct idxd_hw hw;
	197	enum idxd_device_state state;
	198	unsigned long flags;
	199	int id;
42d279f9	200	int major;
ff18de55	201	u8 cmd_status;
bfe1d560 DJ	202
	203	struct pci_dev *pdev;
	204	void __iomem *reg_base;
	205
	206	spinlock_t dev_lock; /* spinlock for device */
53b2ee7f	207	spinlock_t cmd_lock; /* spinlock for device commands */
0d5c10b4	208	struct completion *cmd_done;
defe49f9	209	struct idxd_group **groups;
7c5dd23e	210	struct idxd_wq **wqs;
75b91130	211	struct idxd_engine **engines;
bfe1d560	212
8e50d392 DJ	213	struct iommu_sva *sva;
	214	unsigned int pasid;
	215
bfe1d560 DJ	216	int num_groups;
	217
	218	u32 msix_perm_offset;
	219	u32 wqcfg_offset;
	220	u32 grpcfg_offset;
	221	u32 perfmon_offset;
	222
	223	u64 max_xfer_bytes;
	224	u32 max_batch_size;
	225	int max_groups;
	226	int max_engines;
	227	int max_tokens;
	228	int max_wqs;
	229	int max_wq_size;
	230	int token_limit;
c52ca478	231	int nr_tokens; /* non-reserved tokens */
d98793b5	232	unsigned int wqcfg_size;
bfe1d560 DJ	233
bfe1d560 DJ	234	union sw_err_reg sw_err;
0d5c10b4	235	wait_queue_head_t cmd_waitq;
bfe1d560 DJ	236	int num_wq_irqs;
bfe1d560 DJ	237	struct idxd_irq_entry *irq_entries;
8f47d1a5	238
39786285	239	struct idxd_dma_dev *idxd_dma;
0d5c10b4 DJ	240	struct workqueue_struct *wq;
0d5c10b4 DJ	241	struct work_struct work;
eb15e715 DJ	242
eb15e715 DJ	243	int *int_handles;
bfe1d560 DJ	244	};
	245
	246	/* IDXD software descriptor */
	247	struct idxd_desc {
f25b4638 DJ	248	union {
	249	struct dsa_hw_desc *hw;
	250	struct iax_hw_desc *iax_hw;
	251	};
bfe1d560	252	dma_addr_t desc_dma;
f25b4638 DJ	253	union {
	254	struct dsa_completion_record *completion;
	255	struct iax_completion_record *iax_completion;
	256	};
bfe1d560	257	dma_addr_t compl_dma;
8f47d1a5	258	struct dma_async_tx_descriptor txd;
bfe1d560 DJ	259	struct llist_node llnode;
	260	struct list_head list;
	261	int id;
0705107f	262	int cpu;
eb15e715	263	unsigned int vector;
bfe1d560 DJ	264	struct idxd_wq *wq;
	265	};
	266
	267	#define confdev_to_idxd(dev) container_of(dev, struct idxd_device, conf_dev)
	268	#define confdev_to_wq(dev) container_of(dev, struct idxd_wq, conf_dev)
	269
42d279f9	270	extern struct bus_type dsa_bus_type;
f25b4638	271	extern struct bus_type iax_bus_type;
42d279f9	272
8e50d392	273	extern bool support_enqcmd;
4b73e4eb	274	extern struct ida idxd_ida;
47c16ac2 DJ	275	extern struct device_type dsa_device_type;
47c16ac2 DJ	276	extern struct device_type iax_device_type;
7c5dd23e	277	extern struct device_type idxd_wq_device_type;
75b91130	278	extern struct device_type idxd_engine_device_type;
defe49f9	279	extern struct device_type idxd_group_device_type;
47c16ac2 DJ	280
	281	static inline bool is_dsa_dev(struct device *dev)
	282	{
	283	return dev->type == &dsa_device_type;
	284	}
	285
	286	static inline bool is_iax_dev(struct device *dev)
	287	{
	288	return dev->type == &iax_device_type;
	289	}
	290
	291	static inline bool is_idxd_dev(struct device *dev)
	292	{
	293	return is_dsa_dev(dev) \|\| is_iax_dev(dev);
	294	}
8e50d392	295
7c5dd23e DJ	296	static inline bool is_idxd_wq_dev(struct device *dev)
	297	{
	298	return dev->type == &idxd_wq_device_type;
	299	}
	300
	301	static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq)
	302	{
	303	if (wq->type == IDXD_WQT_KERNEL && strcmp(wq->name, "dmaengine") == 0)
	304	return true;
	305	return false;
	306	}
	307
	308	static inline bool is_idxd_wq_cdev(struct idxd_wq *wq)
	309	{
	310	return wq->type == IDXD_WQT_USER;
	311	}
	312
bfe1d560 DJ	313	static inline bool wq_dedicated(struct idxd_wq *wq)
	314	{
	315	return test_bit(WQ_FLAG_DEDICATED, &wq->flags);
	316	}
	317
8e50d392 DJ	318	static inline bool wq_shared(struct idxd_wq *wq)
	319	{
	320	return !test_bit(WQ_FLAG_DEDICATED, &wq->flags);
	321	}
	322
	323	static inline bool device_pasid_enabled(struct idxd_device *idxd)
	324	{
	325	return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
	326	}
	327
	328	static inline bool device_swq_supported(struct idxd_device *idxd)
	329	{
	330	return (support_enqcmd && device_pasid_enabled(idxd));
	331	}
	332
42d279f9 DJ	333	enum idxd_portal_prot {
	334	IDXD_PORTAL_UNLIMITED = 0,
	335	IDXD_PORTAL_LIMITED,
	336	};
	337
eb15e715 DJ	338	enum idxd_interrupt_type {
	339	IDXD_IRQ_MSIX = 0,
	340	IDXD_IRQ_IMS,
	341	};
	342
42d279f9 DJ	343	static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot)
	344	{
	345	return prot * 0x1000;
	346	}
	347
	348	static inline int idxd_get_wq_portal_full_offset(int wq_id,
	349	enum idxd_portal_prot prot)
	350	{
	351	return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot);
	352	}
	353
c52ca478 DJ	354	static inline void idxd_wq_get(struct idxd_wq *wq)
	355	{
	356	wq->client_count++;
	357	}
	358
	359	static inline void idxd_wq_put(struct idxd_wq *wq)
	360	{
	361	wq->client_count--;
	362	}
	363
	364	static inline int idxd_wq_refcount(struct idxd_wq *wq)
	365	{
	366	return wq->client_count;
	367	};
	368
c52ca478 DJ	369	int idxd_register_bus_type(void);
c52ca478 DJ	370	void idxd_unregister_bus_type(void);
47c16ac2 DJ	371	int idxd_register_devices(struct idxd_device *idxd);
47c16ac2 DJ	372	void idxd_unregister_devices(struct idxd_device *idxd);
c52ca478 DJ	373	int idxd_register_driver(void);
c52ca478 DJ	374	void idxd_unregister_driver(void);
5b0c68c4	375	void idxd_wqs_quiesce(struct idxd_device *idxd);
bfe1d560 DJ	376
bfe1d560 DJ	377	/* device interrupt control */
6df0e6c5 DJ	378	void idxd_msix_perm_setup(struct idxd_device *idxd);
6df0e6c5 DJ	379	void idxd_msix_perm_clear(struct idxd_device *idxd);
bfe1d560 DJ	380	irqreturn_t idxd_misc_thread(int vec, void *data);
	381	irqreturn_t idxd_wq_thread(int irq, void *data);
	382	void idxd_mask_error_interrupts(struct idxd_device *idxd);
	383	void idxd_unmask_error_interrupts(struct idxd_device *idxd);
	384	void idxd_mask_msix_vectors(struct idxd_device *idxd);
4548a6ad DJ	385	void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id);
4548a6ad DJ	386	void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id);
bfe1d560 DJ	387
bfe1d560 DJ	388	/* device control */
89e3becd	389	int idxd_device_init_reset(struct idxd_device *idxd);
bfe1d560 DJ	390	int idxd_device_enable(struct idxd_device *idxd);
bfe1d560 DJ	391	int idxd_device_disable(struct idxd_device *idxd);
0d5c10b4	392	void idxd_device_reset(struct idxd_device *idxd);
bfe1d560 DJ	393	void idxd_device_cleanup(struct idxd_device *idxd);
	394	int idxd_device_config(struct idxd_device *idxd);
	395	void idxd_device_wqs_clear_state(struct idxd_device *idxd);
8e50d392	396	void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid);
8c66bbdc	397	int idxd_device_load_config(struct idxd_device *idxd);
eb15e715 DJ	398	int idxd_device_request_int_handle(struct idxd_device idxd, int idx, int handle,
	399	enum idxd_interrupt_type irq_type);
	400	int idxd_device_release_int_handle(struct idxd_device *idxd, int handle,
	401	enum idxd_interrupt_type irq_type);
bfe1d560 DJ	402
bfe1d560 DJ	403	/* work queue control */
5b0c68c4	404	void idxd_wqs_unmap_portal(struct idxd_device *idxd);
bfe1d560 DJ	405	int idxd_wq_alloc_resources(struct idxd_wq *wq);
	406	void idxd_wq_free_resources(struct idxd_wq *wq);
	407	int idxd_wq_enable(struct idxd_wq *wq);
	408	int idxd_wq_disable(struct idxd_wq *wq);
0d5c10b4	409	void idxd_wq_drain(struct idxd_wq *wq);
ea9aadc0	410	void idxd_wq_reset(struct idxd_wq *wq);
c52ca478 DJ	411	int idxd_wq_map_portal(struct idxd_wq *wq);
c52ca478 DJ	412	void idxd_wq_unmap_portal(struct idxd_wq *wq);
da32b28c	413	void idxd_wq_disable_cleanup(struct idxd_wq *wq);
8e50d392 DJ	414	int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid);
8e50d392 DJ	415	int idxd_wq_disable_pasid(struct idxd_wq *wq);
93a40a6d DJ	416	void idxd_wq_quiesce(struct idxd_wq *wq);
93a40a6d DJ	417	int idxd_wq_init_percpu_ref(struct idxd_wq *wq);
bfe1d560	418
d1dfe5b8 DJ	419	/* submission */
	420	int idxd_submit_desc(struct idxd_wq wq, struct idxd_desc desc);
	421	struct idxd_desc idxd_alloc_desc(struct idxd_wq wq, enum idxd_op_type optype);
	422	void idxd_free_desc(struct idxd_wq wq, struct idxd_desc desc);
	423
8f47d1a5 DJ	424	/* dmaengine */
	425	int idxd_register_dma_device(struct idxd_device *idxd);
	426	void idxd_unregister_dma_device(struct idxd_device *idxd);
	427	int idxd_register_dma_channel(struct idxd_wq *wq);
	428	void idxd_unregister_dma_channel(struct idxd_wq *wq);
	429	void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res);
	430	void idxd_dma_complete_txd(struct idxd_desc *desc,
	431	enum idxd_complete_type comp_type);
8f47d1a5	432
42d279f9 DJ	433	/* cdev */
	434	int idxd_cdev_register(void);
	435	void idxd_cdev_remove(void);
	436	int idxd_cdev_get_major(struct idxd_device *idxd);
	437	int idxd_wq_add_cdev(struct idxd_wq *wq);
	438	void idxd_wq_del_cdev(struct idxd_wq *wq);
	439
bfe1d560	440	#endif