[mirror_ubuntu-hirsute-kernel.git] / drivers / dma / idxd / cdev.c

// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/device.h>
#include <linux/sched/task.h>
#include <linux/intel-svm.h>
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/cdev.h>
#include <linux/fs.h>
#include <linux/poll.h>
#include <uapi/linux/idxd.h>
#include "registers.h"
#include "idxd.h"

struct idxd_cdev_context {
	const char *name;
	dev_t devt;
	struct ida minor_ida;
};

/*
 * ictx is an array based off of accelerator types. enum idxd_type
 * is used as index
 */
static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = {
	{ .name = "dsa" },
};

struct idxd_user_context {
	struct idxd_wq *wq;
	struct task_struct *task;
	unsigned int flags;
};

enum idxd_cdev_cleanup {
	CDEV_NORMAL = 0,
	CDEV_FAILED,
};

static void idxd_cdev_dev_release(struct device *dev)
{
	dev_dbg(dev, "releasing cdev device\n");
	kfree(dev);
}

static struct device_type idxd_cdev_device_type = {
	.name = "idxd_cdev",
	.release = idxd_cdev_dev_release,
};

static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode)
{
	struct cdev *cdev = inode->i_cdev;

	return container_of(cdev, struct idxd_cdev, cdev);
}

static inline struct idxd_wq *idxd_cdev_wq(struct idxd_cdev *idxd_cdev)
{
	return container_of(idxd_cdev, struct idxd_wq, idxd_cdev);
}

static inline struct idxd_wq *inode_wq(struct inode *inode)
{
	return idxd_cdev_wq(inode_idxd_cdev(inode));
}

static int idxd_cdev_open(struct inode *inode, struct file *filp)
{
	struct idxd_user_context *ctx;
	struct idxd_device *idxd;
	struct idxd_wq *wq;
	struct device *dev;
	struct idxd_cdev *idxd_cdev;

	wq = inode_wq(inode);
	idxd = wq->idxd;
	dev = &idxd->pdev->dev;
	idxd_cdev = &wq->idxd_cdev;

	dev_dbg(dev, "%s called\n", __func__);

	if (idxd_wq_refcount(wq) > 1 && wq_dedicated(wq))
		return -EBUSY;

	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
	if (!ctx)
		return -ENOMEM;

	ctx->wq = wq;
	filp->private_data = ctx;
	idxd_wq_get(wq);
	return 0;
}

static int idxd_cdev_release(struct inode *node, struct file *filep)
{
	struct idxd_user_context *ctx = filep->private_data;
	struct idxd_wq *wq = ctx->wq;
	struct idxd_device *idxd = wq->idxd;
	struct device *dev = &idxd->pdev->dev;

	dev_dbg(dev, "%s called\n", __func__);
	filep->private_data = NULL;

	kfree(ctx);
	idxd_wq_put(wq);
	return 0;
}

static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma,
		     const char *func)
{
	struct device *dev = &wq->idxd->pdev->dev;

	if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
		dev_info_ratelimited(dev,
				     "%s: %s: mapping too large: %lu\n",
				     current->comm, func,
				     vma->vm_end - vma->vm_start);
		return -EINVAL;
	}

	return 0;
}

static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma)
{
	struct idxd_user_context *ctx = filp->private_data;
	struct idxd_wq *wq = ctx->wq;
	struct idxd_device *idxd = wq->idxd;
	struct pci_dev *pdev = idxd->pdev;
	phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR);
	unsigned long pfn;
	int rc;

	dev_dbg(&pdev->dev, "%s called\n", __func__);
	rc = check_vma(wq, vma, __func__);

	vma->vm_flags |= VM_DONTCOPY;
	pfn = (base + idxd_get_wq_portal_full_offset(wq->id,
				IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT;
	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
	vma->vm_private_data = ctx;

	return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE,
			vma->vm_page_prot);
}

static __poll_t idxd_cdev_poll(struct file *filp,
			       struct poll_table_struct *wait)
{
	struct idxd_user_context *ctx = filp->private_data;
	struct idxd_wq *wq = ctx->wq;
	struct idxd_device *idxd = wq->idxd;
	struct idxd_cdev *idxd_cdev = &wq->idxd_cdev;
	unsigned long flags;
	__poll_t out = 0;

	poll_wait(filp, &idxd_cdev->err_queue, wait);
	spin_lock_irqsave(&idxd->dev_lock, flags);
	if (idxd->sw_err.valid)
		out = EPOLLIN | EPOLLRDNORM;
	spin_unlock_irqrestore(&idxd->dev_lock, flags);

	return out;
}

static const struct file_operations idxd_cdev_fops = {
	.owner = THIS_MODULE,
	.open = idxd_cdev_open,
	.release = idxd_cdev_release,
	.mmap = idxd_cdev_mmap,
	.poll = idxd_cdev_poll,
};

int idxd_cdev_get_major(struct idxd_device *idxd)
{
	return MAJOR(ictx[idxd->type].devt);
}

static int idxd_wq_cdev_dev_setup(struct idxd_wq *wq)
{
	struct idxd_device *idxd = wq->idxd;
	struct idxd_cdev *idxd_cdev = &wq->idxd_cdev;
	struct idxd_cdev_context *cdev_ctx;
	struct device *dev;
	int minor, rc;

	idxd_cdev->dev = kzalloc(sizeof(*idxd_cdev->dev), GFP_KERNEL);
	if (!idxd_cdev->dev)
		return -ENOMEM;

	dev = idxd_cdev->dev;
	dev->parent = &idxd->pdev->dev;
	dev_set_name(dev, "%s/wq%u.%u", idxd_get_dev_name(idxd),
		     idxd->id, wq->id);
	dev->bus = idxd_get_bus_type(idxd);

	cdev_ctx = &ictx[wq->idxd->type];
	minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL);
	if (minor < 0) {
		rc = minor;
		kfree(dev);
		goto ida_err;
	}

	dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor);
	dev->type = &idxd_cdev_device_type;
	rc = device_register(dev);
	if (rc < 0) {
		dev_err(&idxd->pdev->dev, "device register failed\n");
		goto dev_reg_err;
	}
	idxd_cdev->minor = minor;

	return 0;

 dev_reg_err:
	ida_simple_remove(&cdev_ctx->minor_ida, MINOR(dev->devt));
	put_device(dev);
 ida_err:
	idxd_cdev->dev = NULL;
	return rc;
}

static void idxd_wq_cdev_cleanup(struct idxd_wq *wq,
				 enum idxd_cdev_cleanup cdev_state)
{
	struct idxd_cdev *idxd_cdev = &wq->idxd_cdev;
	struct idxd_cdev_context *cdev_ctx;

	cdev_ctx = &ictx[wq->idxd->type];
	if (cdev_state == CDEV_NORMAL)
		cdev_del(&idxd_cdev->cdev);
	device_unregister(idxd_cdev->dev);
	/*
	 * The device_type->release() will be called on the device and free
	 * the allocated struct device. We can just forget it.
	 */
	ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor);
	idxd_cdev->dev = NULL;
	idxd_cdev->minor = -1;
}

int idxd_wq_add_cdev(struct idxd_wq *wq)
{
	struct idxd_cdev *idxd_cdev = &wq->idxd_cdev;
	struct cdev *cdev = &idxd_cdev->cdev;
	struct device *dev;
	int rc;

	rc = idxd_wq_cdev_dev_setup(wq);
	if (rc < 0)
		return rc;

	dev = idxd_cdev->dev;
	cdev_init(cdev, &idxd_cdev_fops);
	cdev_set_parent(cdev, &dev->kobj);
	rc = cdev_add(cdev, dev->devt, 1);
	if (rc) {
		dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc);
		idxd_wq_cdev_cleanup(wq, CDEV_FAILED);
		return rc;
	}

	init_waitqueue_head(&idxd_cdev->err_queue);
	return 0;
}

void idxd_wq_del_cdev(struct idxd_wq *wq)
{
	idxd_wq_cdev_cleanup(wq, CDEV_NORMAL);
}

int idxd_cdev_register(void)
{
	int rc, i;

	for (i = 0; i < IDXD_TYPE_MAX; i++) {
		ida_init(&ictx[i].minor_ida);
		rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK,
					 ictx[i].name);
		if (rc)
			return rc;
	}

	return 0;
}

void idxd_cdev_remove(void)
{
	int i;

	for (i = 0; i < IDXD_TYPE_MAX; i++) {
		unregister_chrdev_region(ictx[i].devt, MINORMASK);
		ida_destroy(&ictx[i].minor_ida);
	}
}
Commit	Line	Data
42d279f9 DJ	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
	3	#include <linux/init.h>
	4	#include <linux/kernel.h>
	5	#include <linux/module.h>
	6	#include <linux/pci.h>
	7	#include <linux/device.h>
	8	#include <linux/sched/task.h>
	9	#include <linux/intel-svm.h>
	10	#include <linux/io-64-nonatomic-lo-hi.h>
	11	#include <linux/cdev.h>
	12	#include <linux/fs.h>
	13	#include <linux/poll.h>
	14	#include <uapi/linux/idxd.h>
	15	#include "registers.h"
	16	#include "idxd.h"
	17
	18	struct idxd_cdev_context {
	19	const char *name;
	20	dev_t devt;
	21	struct ida minor_ida;
	22	};
	23
	24	/*
	25	* ictx is an array based off of accelerator types. enum idxd_type
	26	* is used as index
	27	*/
	28	static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = {
	29	{ .name = "dsa" },
	30	};
	31
	32	struct idxd_user_context {
	33	struct idxd_wq *wq;
	34	struct task_struct *task;
	35	unsigned int flags;
	36	};
	37
	38	enum idxd_cdev_cleanup {
	39	CDEV_NORMAL = 0,
	40	CDEV_FAILED,
	41	};
	42
	43	static void idxd_cdev_dev_release(struct device *dev)
	44	{
	45	dev_dbg(dev, "releasing cdev device\n");
	46	kfree(dev);
	47	}
	48
	49	static struct device_type idxd_cdev_device_type = {
	50	.name = "idxd_cdev",
	51	.release = idxd_cdev_dev_release,
	52	};
	53
	54	static inline struct idxd_cdev inode_idxd_cdev(struct inode inode)
	55	{
	56	struct cdev *cdev = inode->i_cdev;
	57
	58	return container_of(cdev, struct idxd_cdev, cdev);
	59	}
	60
	61	static inline struct idxd_wq idxd_cdev_wq(struct idxd_cdev idxd_cdev)
	62	{
	63	return container_of(idxd_cdev, struct idxd_wq, idxd_cdev);
	64	}
65
66	static inline struct idxd_wq inode_wq(struct inode inode)
67	{
68	return idxd_cdev_wq(inode_idxd_cdev(inode));
69	}
70
71	static int idxd_cdev_open(struct inode inode, struct file filp)
72	{
73	struct idxd_user_context *ctx;
74	struct idxd_device *idxd;
75	struct idxd_wq *wq;
76	struct device *dev;
77	struct idxd_cdev *idxd_cdev;
78
79	wq = inode_wq(inode);
80	idxd = wq->idxd;
81	dev = &idxd->pdev->dev;
82	idxd_cdev = &wq->idxd_cdev;
83
84	dev_dbg(dev, "%s called\n", __func__);
85
86	if (idxd_wq_refcount(wq) > 1 && wq_dedicated(wq))
87	return -EBUSY;
88
89	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
90	if (!ctx)
91	return -ENOMEM;
92
93	ctx->wq = wq;
94	filp->private_data = ctx;
95	idxd_wq_get(wq);
96	return 0;
97	}
98
99	static int idxd_cdev_release(struct inode node, struct file filep)
100	{
101	struct idxd_user_context *ctx = filep->private_data;
102	struct idxd_wq *wq = ctx->wq;
103	struct idxd_device *idxd = wq->idxd;
104	struct device *dev = &idxd->pdev->dev;
105
106	dev_dbg(dev, "%s called\n", __func__);
107	filep->private_data = NULL;
108
109	kfree(ctx);
110	idxd_wq_put(wq);
111	return 0;
112	}
113
114	static int check_vma(struct idxd_wq wq, struct vm_area_struct vma,
115	const char *func)
116	{
117	struct device *dev = &wq->idxd->pdev->dev;
118
119	if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
120	dev_info_ratelimited(dev,
121	"%s: %s: mapping too large: %lu\n",
122	current->comm, func,
123	vma->vm_end - vma->vm_start);
124	return -EINVAL;
125	}
126
127	return 0;
128	}
129
130	static int idxd_cdev_mmap(struct file filp, struct vm_area_struct vma)
131	{
132	struct idxd_user_context *ctx = filp->private_data;
133	struct idxd_wq *wq = ctx->wq;
134	struct idxd_device *idxd = wq->idxd;
135	struct pci_dev *pdev = idxd->pdev;
136	phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR);
137	unsigned long pfn;
138	int rc;
139
140	dev_dbg(&pdev->dev, "%s called\n", __func__);
141	rc = check_vma(wq, vma, __func__);
142
143	vma->vm_flags \|= VM_DONTCOPY;
144	pfn = (base + idxd_get_wq_portal_full_offset(wq->id,
145	IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT;
146	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
147	vma->vm_private_data = ctx;
148
149	return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE,
150	vma->vm_page_prot);
151	}
152
153	static __poll_t idxd_cdev_poll(struct file *filp,
154	struct poll_table_struct *wait)
155	{
156	struct idxd_user_context *ctx = filp->private_data;
157	struct idxd_wq *wq = ctx->wq;
158	struct idxd_device *idxd = wq->idxd;
159	struct idxd_cdev *idxd_cdev = &wq->idxd_cdev;
160	unsigned long flags;
161	__poll_t out = 0;
162
163	poll_wait(filp, &idxd_cdev->err_queue, wait);
164	spin_lock_irqsave(&idxd->dev_lock, flags);
165	if (idxd->sw_err.valid)
166	out = EPOLLIN \| EPOLLRDNORM;
167	spin_unlock_irqrestore(&idxd->dev_lock, flags);
168
169	return out;
170	}
171
172	static const struct file_operations idxd_cdev_fops = {
173	.owner = THIS_MODULE,
174	.open = idxd_cdev_open,
175	.release = idxd_cdev_release,
176	.mmap = idxd_cdev_mmap,
177	.poll = idxd_cdev_poll,
178	};
179
180	int idxd_cdev_get_major(struct idxd_device *idxd)
181	{
182	return MAJOR(ictx[idxd->type].devt);
183	}
184
185	static int idxd_wq_cdev_dev_setup(struct idxd_wq *wq)
186	{
187	struct idxd_device *idxd = wq->idxd;
188	struct idxd_cdev *idxd_cdev = &wq->idxd_cdev;
189	struct idxd_cdev_context *cdev_ctx;
190	struct device *dev;
191	int minor, rc;
192
193	idxd_cdev->dev = kzalloc(sizeof(*idxd_cdev->dev), GFP_KERNEL);
194	if (!idxd_cdev->dev)
195	return -ENOMEM;
196
197	dev = idxd_cdev->dev;
198	dev->parent = &idxd->pdev->dev;
199	dev_set_name(dev, "%s/wq%u.%u", idxd_get_dev_name(idxd),
200	idxd->id, wq->id);
201	dev->bus = idxd_get_bus_type(idxd);
202
203	cdev_ctx = &ictx[wq->idxd->type];
204	minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL);
205	if (minor < 0) {
206	rc = minor;
2227ab42	207	kfree(dev);
42d279f9 DJ	208	goto ida_err;
	209	}
	210
	211	dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor);
	212	dev->type = &idxd_cdev_device_type;
	213	rc = device_register(dev);
	214	if (rc < 0) {
	215	dev_err(&idxd->pdev->dev, "device register failed\n");
42d279f9 DJ	216	goto dev_reg_err;
	217	}
	218	idxd_cdev->minor = minor;
	219
	220	return 0;
	221
	222	dev_reg_err:
	223	ida_simple_remove(&cdev_ctx->minor_ida, MINOR(dev->devt));
2227ab42	224	put_device(dev);
42d279f9	225	ida_err:
42d279f9 DJ	226	idxd_cdev->dev = NULL;
	227	return rc;
	228	}
	229
	230	static void idxd_wq_cdev_cleanup(struct idxd_wq *wq,
	231	enum idxd_cdev_cleanup cdev_state)
	232	{
	233	struct idxd_cdev *idxd_cdev = &wq->idxd_cdev;
	234	struct idxd_cdev_context *cdev_ctx;
	235
	236	cdev_ctx = &ictx[wq->idxd->type];
	237	if (cdev_state == CDEV_NORMAL)
	238	cdev_del(&idxd_cdev->cdev);
	239	device_unregister(idxd_cdev->dev);
	240	/*
	241	* The device_type->release() will be called on the device and free
	242	* the allocated struct device. We can just forget it.
	243	*/
	244	ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor);
	245	idxd_cdev->dev = NULL;
	246	idxd_cdev->minor = -1;
	247	}
	248
	249	int idxd_wq_add_cdev(struct idxd_wq *wq)
	250	{
	251	struct idxd_cdev *idxd_cdev = &wq->idxd_cdev;
	252	struct cdev *cdev = &idxd_cdev->cdev;
	253	struct device *dev;
	254	int rc;
	255
	256	rc = idxd_wq_cdev_dev_setup(wq);
	257	if (rc < 0)
	258	return rc;
	259
	260	dev = idxd_cdev->dev;
	261	cdev_init(cdev, &idxd_cdev_fops);
	262	cdev_set_parent(cdev, &dev->kobj);
	263	rc = cdev_add(cdev, dev->devt, 1);
	264	if (rc) {
	265	dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc);
	266	idxd_wq_cdev_cleanup(wq, CDEV_FAILED);
	267	return rc;
	268	}
	269
	270	init_waitqueue_head(&idxd_cdev->err_queue);
	271	return 0;
	272	}
	273
	274	void idxd_wq_del_cdev(struct idxd_wq *wq)
	275	{
	276	idxd_wq_cdev_cleanup(wq, CDEV_NORMAL);
	277	}
	278
	279	int idxd_cdev_register(void)
	280	{
	281	int rc, i;
	282
	283	for (i = 0; i < IDXD_TYPE_MAX; i++) {
	284	ida_init(&ictx[i].minor_ida);
	285	rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK,
	286	ictx[i].name);
	287	if (rc)
	288	return rc;
	289	}
290
291	return 0;
292	}
293
294	void idxd_cdev_remove(void)
295	{
296	int i;
297
298	for (i = 0; i < IDXD_TYPE_MAX; i++) {
299	unregister_chrdev_region(ictx[i].devt, MINORMASK);
300	ida_destroy(&ictx[i].minor_ida);
301	}
302	}