[mirror_ubuntu-jammy-kernel.git] / drivers / dax / device.c

// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2016-2018 Intel Corporation. All rights reserved. */
#include <linux/memremap.h>
#include <linux/pagemap.h>
#include <linux/module.h>
#include <linux/device.h>
#include <linux/pfn_t.h>
#include <linux/cdev.h>
#include <linux/slab.h>
#include <linux/dax.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/mman.h>
#include "dax-private.h"
#include "bus.h"

static struct dev_dax *ref_to_dev_dax(struct percpu_ref *ref)
{
	return container_of(ref, struct dev_dax, ref);
}

static void dev_dax_percpu_release(struct percpu_ref *ref)
{
	struct dev_dax *dev_dax = ref_to_dev_dax(ref);

	dev_dbg(&dev_dax->dev, "%s\n", __func__);
	complete(&dev_dax->cmp);
}

static void dev_dax_percpu_exit(struct percpu_ref *ref)
{
	struct dev_dax *dev_dax = ref_to_dev_dax(ref);

	dev_dbg(&dev_dax->dev, "%s\n", __func__);
	wait_for_completion(&dev_dax->cmp);
	percpu_ref_exit(ref);
}

static void dev_dax_percpu_kill(struct percpu_ref *data)
{
	struct percpu_ref *ref = data;
	struct dev_dax *dev_dax = ref_to_dev_dax(ref);

	dev_dbg(&dev_dax->dev, "%s\n", __func__);
	percpu_ref_kill(ref);
}

static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma,
		const char *func)
{
	struct dax_region *dax_region = dev_dax->region;
	struct device *dev = &dev_dax->dev;
	unsigned long mask;

	if (!dax_alive(dev_dax->dax_dev))
		return -ENXIO;

	/* prevent private mappings from being established */
	if ((vma->vm_flags & VM_MAYSHARE) != VM_MAYSHARE) {
		dev_info_ratelimited(dev,
				"%s: %s: fail, attempted private mapping\n",
				current->comm, func);
		return -EINVAL;
	}

	mask = dax_region->align - 1;
	if (vma->vm_start & mask || vma->vm_end & mask) {
		dev_info_ratelimited(dev,
				"%s: %s: fail, unaligned vma (%#lx - %#lx, %#lx)\n",
				current->comm, func, vma->vm_start, vma->vm_end,
				mask);
		return -EINVAL;
	}

	if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) == PFN_DEV
			&& (vma->vm_flags & VM_DONTCOPY) == 0) {
		dev_info_ratelimited(dev,
				"%s: %s: fail, dax range requires MADV_DONTFORK\n",
				current->comm, func);
		return -EINVAL;
	}

	if (!vma_is_dax(vma)) {
		dev_info_ratelimited(dev,
				"%s: %s: fail, vma is not DAX capable\n",
				current->comm, func);
		return -EINVAL;
	}

	return 0;
}

/* see "strong" declaration in tools/testing/nvdimm/dax-dev.c */
__weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff,
		unsigned long size)
{
	struct resource *res = &dev_dax->region->res;
	phys_addr_t phys;

	phys = pgoff * PAGE_SIZE + res->start;
	if (phys >= res->start && phys <= res->end) {
		if (phys + size - 1 <= res->end)
			return phys;
	}

	return -1;
}

static vm_fault_t __dev_dax_pte_fault(struct dev_dax *dev_dax,
				struct vm_fault *vmf, pfn_t *pfn)
{
	struct device *dev = &dev_dax->dev;
	struct dax_region *dax_region;
	phys_addr_t phys;
	unsigned int fault_size = PAGE_SIZE;

	if (check_vma(dev_dax, vmf->vma, __func__))
		return VM_FAULT_SIGBUS;

	dax_region = dev_dax->region;
	if (dax_region->align > PAGE_SIZE) {
		dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n",
			dax_region->align, fault_size);
		return VM_FAULT_SIGBUS;
	}

	if (fault_size != dax_region->align)
		return VM_FAULT_SIGBUS;

	phys = dax_pgoff_to_phys(dev_dax, vmf->pgoff, PAGE_SIZE);
	if (phys == -1) {
		dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", vmf->pgoff);
		return VM_FAULT_SIGBUS;
	}

	*pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);

	return vmf_insert_mixed(vmf->vma, vmf->address, *pfn);
}

static vm_fault_t __dev_dax_pmd_fault(struct dev_dax *dev_dax,
				struct vm_fault *vmf, pfn_t *pfn)
{
	unsigned long pmd_addr = vmf->address & PMD_MASK;
	struct device *dev = &dev_dax->dev;
	struct dax_region *dax_region;
	phys_addr_t phys;
	pgoff_t pgoff;
	unsigned int fault_size = PMD_SIZE;

	if (check_vma(dev_dax, vmf->vma, __func__))
		return VM_FAULT_SIGBUS;

	dax_region = dev_dax->region;
	if (dax_region->align > PMD_SIZE) {
		dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n",
			dax_region->align, fault_size);
		return VM_FAULT_SIGBUS;
	}

	/* dax pmd mappings require pfn_t_devmap() */
	if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) != (PFN_DEV|PFN_MAP)) {
		dev_dbg(dev, "region lacks devmap flags\n");
		return VM_FAULT_SIGBUS;
	}

	if (fault_size < dax_region->align)
		return VM_FAULT_SIGBUS;
	else if (fault_size > dax_region->align)
		return VM_FAULT_FALLBACK;

	/* if we are outside of the VMA */
	if (pmd_addr < vmf->vma->vm_start ||
			(pmd_addr + PMD_SIZE) > vmf->vma->vm_end)
		return VM_FAULT_SIGBUS;

	pgoff = linear_page_index(vmf->vma, pmd_addr);
	phys = dax_pgoff_to_phys(dev_dax, pgoff, PMD_SIZE);
	if (phys == -1) {
		dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff);
		return VM_FAULT_SIGBUS;
	}

	*pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);

	return vmf_insert_pfn_pmd(vmf, *pfn, vmf->flags & FAULT_FLAG_WRITE);
}

#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax,
				struct vm_fault *vmf, pfn_t *pfn)
{
	unsigned long pud_addr = vmf->address & PUD_MASK;
	struct device *dev = &dev_dax->dev;
	struct dax_region *dax_region;
	phys_addr_t phys;
	pgoff_t pgoff;
	unsigned int fault_size = PUD_SIZE;


	if (check_vma(dev_dax, vmf->vma, __func__))
		return VM_FAULT_SIGBUS;

	dax_region = dev_dax->region;
	if (dax_region->align > PUD_SIZE) {
		dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n",
			dax_region->align, fault_size);
		return VM_FAULT_SIGBUS;
	}

	/* dax pud mappings require pfn_t_devmap() */
	if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) != (PFN_DEV|PFN_MAP)) {
		dev_dbg(dev, "region lacks devmap flags\n");
		return VM_FAULT_SIGBUS;
	}

	if (fault_size < dax_region->align)
		return VM_FAULT_SIGBUS;
	else if (fault_size > dax_region->align)
		return VM_FAULT_FALLBACK;

	/* if we are outside of the VMA */
	if (pud_addr < vmf->vma->vm_start ||
			(pud_addr + PUD_SIZE) > vmf->vma->vm_end)
		return VM_FAULT_SIGBUS;

	pgoff = linear_page_index(vmf->vma, pud_addr);
	phys = dax_pgoff_to_phys(dev_dax, pgoff, PUD_SIZE);
	if (phys == -1) {
		dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff);
		return VM_FAULT_SIGBUS;
	}

	*pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);

	return vmf_insert_pfn_pud(vmf, *pfn, vmf->flags & FAULT_FLAG_WRITE);
}
#else
static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax,
				struct vm_fault *vmf, pfn_t *pfn)
{
	return VM_FAULT_FALLBACK;
}
#endif /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */

static vm_fault_t dev_dax_huge_fault(struct vm_fault *vmf,
		enum page_entry_size pe_size)
{
	struct file *filp = vmf->vma->vm_file;
	unsigned long fault_size;
	vm_fault_t rc = VM_FAULT_SIGBUS;
	int id;
	pfn_t pfn;
	struct dev_dax *dev_dax = filp->private_data;

	dev_dbg(&dev_dax->dev, "%s: %s (%#lx - %#lx) size = %d\n", current->comm,
			(vmf->flags & FAULT_FLAG_WRITE) ? "write" : "read",
			vmf->vma->vm_start, vmf->vma->vm_end, pe_size);

	id = dax_read_lock();
	switch (pe_size) {
	case PE_SIZE_PTE:
		fault_size = PAGE_SIZE;
		rc = __dev_dax_pte_fault(dev_dax, vmf, &pfn);
		break;
	case PE_SIZE_PMD:
		fault_size = PMD_SIZE;
		rc = __dev_dax_pmd_fault(dev_dax, vmf, &pfn);
		break;
	case PE_SIZE_PUD:
		fault_size = PUD_SIZE;
		rc = __dev_dax_pud_fault(dev_dax, vmf, &pfn);
		break;
	default:
		rc = VM_FAULT_SIGBUS;
	}

	if (rc == VM_FAULT_NOPAGE) {
		unsigned long i;
		pgoff_t pgoff;

		/*
		 * In the device-dax case the only possibility for a
		 * VM_FAULT_NOPAGE result is when device-dax capacity is
		 * mapped. No need to consider the zero page, or racing
		 * conflicting mappings.
		 */
		pgoff = linear_page_index(vmf->vma, vmf->address
				& ~(fault_size - 1));
		for (i = 0; i < fault_size / PAGE_SIZE; i++) {
			struct page *page;

			page = pfn_to_page(pfn_t_to_pfn(pfn) + i);
			if (page->mapping)
				continue;
			page->mapping = filp->f_mapping;
			page->index = pgoff + i;
		}
	}
	dax_read_unlock(id);

	return rc;
}

static vm_fault_t dev_dax_fault(struct vm_fault *vmf)
{
	return dev_dax_huge_fault(vmf, PE_SIZE_PTE);
}

static int dev_dax_split(struct vm_area_struct *vma, unsigned long addr)
{
	struct file *filp = vma->vm_file;
	struct dev_dax *dev_dax = filp->private_data;
	struct dax_region *dax_region = dev_dax->region;

	if (!IS_ALIGNED(addr, dax_region->align))
		return -EINVAL;
	return 0;
}

static unsigned long dev_dax_pagesize(struct vm_area_struct *vma)
{
	struct file *filp = vma->vm_file;
	struct dev_dax *dev_dax = filp->private_data;
	struct dax_region *dax_region = dev_dax->region;

	return dax_region->align;
}

static const struct vm_operations_struct dax_vm_ops = {
	.fault = dev_dax_fault,
	.huge_fault = dev_dax_huge_fault,
	.split = dev_dax_split,
	.pagesize = dev_dax_pagesize,
};

static int dax_mmap(struct file *filp, struct vm_area_struct *vma)
{
	struct dev_dax *dev_dax = filp->private_data;
	int rc, id;

	dev_dbg(&dev_dax->dev, "trace\n");

	/*
	 * We lock to check dax_dev liveness and will re-check at
	 * fault time.
	 */
	id = dax_read_lock();
	rc = check_vma(dev_dax, vma, __func__);
	dax_read_unlock(id);
	if (rc)
		return rc;

	vma->vm_ops = &dax_vm_ops;
	vma->vm_flags |= VM_HUGEPAGE;
	return 0;
}

/* return an unmapped area aligned to the dax region specified alignment */
static unsigned long dax_get_unmapped_area(struct file *filp,
		unsigned long addr, unsigned long len, unsigned long pgoff,
		unsigned long flags)
{
	unsigned long off, off_end, off_align, len_align, addr_align, align;
	struct dev_dax *dev_dax = filp ? filp->private_data : NULL;
	struct dax_region *dax_region;

	if (!dev_dax || addr)
		goto out;

	dax_region = dev_dax->region;
	align = dax_region->align;
	off = pgoff << PAGE_SHIFT;
	off_end = off + len;
	off_align = round_up(off, align);

	if ((off_end <= off_align) || ((off_end - off_align) < align))
		goto out;

	len_align = len + align;
	if ((off + len_align) < off)
		goto out;

	addr_align = current->mm->get_unmapped_area(filp, addr, len_align,
			pgoff, flags);
	if (!IS_ERR_VALUE(addr_align)) {
		addr_align += (off - addr_align) & (align - 1);
		return addr_align;
	}
 out:
	return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);
}

static const struct address_space_operations dev_dax_aops = {
	.set_page_dirty		= noop_set_page_dirty,
	.invalidatepage		= noop_invalidatepage,
};

static int dax_open(struct inode *inode, struct file *filp)
{
	struct dax_device *dax_dev = inode_dax(inode);
	struct inode *__dax_inode = dax_inode(dax_dev);
	struct dev_dax *dev_dax = dax_get_private(dax_dev);

	dev_dbg(&dev_dax->dev, "trace\n");
	inode->i_mapping = __dax_inode->i_mapping;
	inode->i_mapping->host = __dax_inode;
	inode->i_mapping->a_ops = &dev_dax_aops;
	filp->f_mapping = inode->i_mapping;
	filp->f_wb_err = filemap_sample_wb_err(filp->f_mapping);
	filp->private_data = dev_dax;
	inode->i_flags = S_DAX;

	return 0;
}

static int dax_release(struct inode *inode, struct file *filp)
{
	struct dev_dax *dev_dax = filp->private_data;

	dev_dbg(&dev_dax->dev, "trace\n");
	return 0;
}

static const struct file_operations dax_fops = {
	.llseek = noop_llseek,
	.owner = THIS_MODULE,
	.open = dax_open,
	.release = dax_release,
	.get_unmapped_area = dax_get_unmapped_area,
	.mmap = dax_mmap,
	.mmap_supported_flags = MAP_SYNC,
};

static void dev_dax_cdev_del(void *cdev)
{
	cdev_del(cdev);
}

static void dev_dax_kill(void *dev_dax)
{
	kill_dev_dax(dev_dax);
}

int dev_dax_probe(struct device *dev)
{
	struct dev_dax *dev_dax = to_dev_dax(dev);
	struct dax_device *dax_dev = dev_dax->dax_dev;
	struct resource *res = &dev_dax->region->res;
	struct inode *inode;
	struct cdev *cdev;
	void *addr;
	int rc;

	/* 1:1 map region resource range to device-dax instance range */
	if (!devm_request_mem_region(dev, res->start, resource_size(res),
				dev_name(dev))) {
		dev_warn(dev, "could not reserve region %pR\n", res);
		return -EBUSY;
	}

	init_completion(&dev_dax->cmp);
	rc = percpu_ref_init(&dev_dax->ref, dev_dax_percpu_release, 0,
			GFP_KERNEL);
	if (rc)
		return rc;

	dev_dax->pgmap.ref = &dev_dax->ref;
	dev_dax->pgmap.kill = dev_dax_percpu_kill;
	dev_dax->pgmap.cleanup = dev_dax_percpu_exit;
	addr = devm_memremap_pages(dev, &dev_dax->pgmap);
	if (IS_ERR(addr))
		return PTR_ERR(addr);

	inode = dax_inode(dax_dev);
	cdev = inode->i_cdev;
	cdev_init(cdev, &dax_fops);
	if (dev->class) {
		/* for the CONFIG_DEV_DAX_PMEM_COMPAT case */
		cdev->owner = dev->parent->driver->owner;
	} else
		cdev->owner = dev->driver->owner;
	cdev_set_parent(cdev, &dev->kobj);
	rc = cdev_add(cdev, dev->devt, 1);
	if (rc)
		return rc;

	rc = devm_add_action_or_reset(dev, dev_dax_cdev_del, cdev);
	if (rc)
		return rc;

	run_dax(dax_dev);
	return devm_add_action_or_reset(dev, dev_dax_kill, dev_dax);
}
EXPORT_SYMBOL_GPL(dev_dax_probe);

static int dev_dax_remove(struct device *dev)
{
	/* all probe actions are unwound by devm */
	return 0;
}

static struct dax_device_driver device_dax_driver = {
	.drv = {
		.probe = dev_dax_probe,
		.remove = dev_dax_remove,
	},
	.match_always = 1,
};

static int __init dax_init(void)
{
	return dax_driver_register(&device_dax_driver);
}

static void __exit dax_exit(void)
{
	dax_driver_unregister(&device_dax_driver);
}

MODULE_AUTHOR("Intel Corporation");
MODULE_LICENSE("GPL v2");
module_init(dax_init);
module_exit(dax_exit);
MODULE_ALIAS_DAX_DEVICE(0);
Commit	Line	Data
51cf784c DW	1	// SPDX-License-Identifier: GPL-2.0
51cf784c DW	2	/* Copyright(c) 2016-2018 Intel Corporation. All rights reserved. */
89ec9f2c	3	#include <linux/memremap.h>
ab68f262 DW	4	#include <linux/pagemap.h>
	5	#include <linux/module.h>
	6	#include <linux/device.h>
	7	#include <linux/pfn_t.h>
ba09c01d	8	#include <linux/cdev.h>
ab68f262 DW	9	#include <linux/slab.h>
	10	#include <linux/dax.h>
	11	#include <linux/fs.h>
	12	#include <linux/mm.h>
ef842302	13	#include <linux/mman.h>
efebc711	14	#include "dax-private.h"
51cf784c	15	#include "bus.h"
ab68f262	16
89ec9f2c DW	17	static struct dev_dax ref_to_dev_dax(struct percpu_ref ref)
	18	{
	19	return container_of(ref, struct dev_dax, ref);
	20	}
	21
	22	static void dev_dax_percpu_release(struct percpu_ref *ref)
	23	{
	24	struct dev_dax *dev_dax = ref_to_dev_dax(ref);
	25
	26	dev_dbg(&dev_dax->dev, "%s\n", __func__);
	27	complete(&dev_dax->cmp);
	28	}
	29
50f44ee7	30	static void dev_dax_percpu_exit(struct percpu_ref *ref)
89ec9f2c	31	{
89ec9f2c DW	32	struct dev_dax *dev_dax = ref_to_dev_dax(ref);
	33
	34	dev_dbg(&dev_dax->dev, "%s\n", __func__);
	35	wait_for_completion(&dev_dax->cmp);
	36	percpu_ref_exit(ref);
	37	}
	38
	39	static void dev_dax_percpu_kill(struct percpu_ref *data)
	40	{
	41	struct percpu_ref *ref = data;
	42	struct dev_dax *dev_dax = ref_to_dev_dax(ref);
	43
	44	dev_dbg(&dev_dax->dev, "%s\n", __func__);
	45	percpu_ref_kill(ref);
	46	}
	47
5f0694b3	48	static int check_vma(struct dev_dax dev_dax, struct vm_area_struct vma,
dee41079 DW	49	const char *func)
dee41079 DW	50	{
5f0694b3 DW	51	struct dax_region *dax_region = dev_dax->region;
5f0694b3 DW	52	struct device *dev = &dev_dax->dev;
dee41079 DW	53	unsigned long mask;
dee41079 DW	54
7b6be844	55	if (!dax_alive(dev_dax->dax_dev))
dee41079 DW	56	return -ENXIO;
dee41079 DW	57
4cb19355	58	/* prevent private mappings from being established */
325896ff	59	if ((vma->vm_flags & VM_MAYSHARE) != VM_MAYSHARE) {
5a14e91d JM	60	dev_info_ratelimited(dev,
5a14e91d JM	61	"%s: %s: fail, attempted private mapping\n",
dee41079 DW	62	current->comm, func);
	63	return -EINVAL;
	64	}
	65
	66	mask = dax_region->align - 1;
	67	if (vma->vm_start & mask \|\| vma->vm_end & mask) {
5a14e91d JM	68	dev_info_ratelimited(dev,
5a14e91d JM	69	"%s: %s: fail, unaligned vma (%#lx - %#lx, %#lx)\n",
dee41079 DW	70	current->comm, func, vma->vm_start, vma->vm_end,
	71	mask);
	72	return -EINVAL;
	73	}
	74
	75	if ((dax_region->pfn_flags & (PFN_DEV\|PFN_MAP)) == PFN_DEV
	76	&& (vma->vm_flags & VM_DONTCOPY) == 0) {
5a14e91d JM	77	dev_info_ratelimited(dev,
5a14e91d JM	78	"%s: %s: fail, dax range requires MADV_DONTFORK\n",
dee41079 DW	79	current->comm, func);
	80	return -EINVAL;
	81	}
	82
	83	if (!vma_is_dax(vma)) {
5a14e91d JM	84	dev_info_ratelimited(dev,
5a14e91d JM	85	"%s: %s: fail, vma is not DAX capable\n",
dee41079 DW	86	current->comm, func);
	87	return -EINVAL;
	88	}
	89
	90	return 0;
	91	}
	92
efebc711	93	/* see "strong" declaration in tools/testing/nvdimm/dax-dev.c */
73616367	94	__weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff,
dee41079 DW	95	unsigned long size)
dee41079 DW	96	{
753a0850 DW	97	struct resource *res = &dev_dax->region->res;
753a0850 DW	98	phys_addr_t phys;
dee41079	99
753a0850 DW	100	phys = pgoff * PAGE_SIZE + res->start;
753a0850 DW	101	if (phys >= res->start && phys <= res->end) {
dee41079 DW	102	if (phys + size - 1 <= res->end)
	103	return phys;
	104	}
	105
	106	return -1;
	107	}
	108
226ab561	109	static vm_fault_t __dev_dax_pte_fault(struct dev_dax *dev_dax,
2232c638	110	struct vm_fault vmf, pfn_t pfn)
dee41079	111	{
5f0694b3	112	struct device *dev = &dev_dax->dev;
dee41079	113	struct dax_region *dax_region;
dee41079	114	phys_addr_t phys;
0134ed4f	115	unsigned int fault_size = PAGE_SIZE;
dee41079	116
5f0694b3	117	if (check_vma(dev_dax, vmf->vma, __func__))
dee41079 DW	118	return VM_FAULT_SIGBUS;
dee41079 DW	119
5f0694b3	120	dax_region = dev_dax->region;
dee41079	121	if (dax_region->align > PAGE_SIZE) {
6daaca52 DW	122	dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n",
6daaca52 DW	123	dax_region->align, fault_size);
dee41079 DW	124	return VM_FAULT_SIGBUS;
	125	}
	126
0134ed4f DJ	127	if (fault_size != dax_region->align)
	128	return VM_FAULT_SIGBUS;
	129
73616367	130	phys = dax_pgoff_to_phys(dev_dax, vmf->pgoff, PAGE_SIZE);
dee41079	131	if (phys == -1) {
6daaca52	132	dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", vmf->pgoff);
dee41079 DW	133	return VM_FAULT_SIGBUS;
	134	}
	135
2232c638	136	*pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
dee41079	137
2232c638	138	return vmf_insert_mixed(vmf->vma, vmf->address, *pfn);
dee41079 DW	139	}
dee41079 DW	140
226ab561	141	static vm_fault_t __dev_dax_pmd_fault(struct dev_dax *dev_dax,
2232c638	142	struct vm_fault vmf, pfn_t pfn)
dee41079	143	{
d8a849e1	144	unsigned long pmd_addr = vmf->address & PMD_MASK;
5f0694b3	145	struct device *dev = &dev_dax->dev;
dee41079 DW	146	struct dax_region *dax_region;
	147	phys_addr_t phys;
	148	pgoff_t pgoff;
0134ed4f	149	unsigned int fault_size = PMD_SIZE;
dee41079	150
5f0694b3	151	if (check_vma(dev_dax, vmf->vma, __func__))
dee41079 DW	152	return VM_FAULT_SIGBUS;
dee41079 DW	153
5f0694b3	154	dax_region = dev_dax->region;
dee41079	155	if (dax_region->align > PMD_SIZE) {
6daaca52 DW	156	dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n",
6daaca52 DW	157	dax_region->align, fault_size);
dee41079 DW	158	return VM_FAULT_SIGBUS;
	159	}
	160
	161	/* dax pmd mappings require pfn_t_devmap() */
	162	if ((dax_region->pfn_flags & (PFN_DEV\|PFN_MAP)) != (PFN_DEV\|PFN_MAP)) {
6daaca52	163	dev_dbg(dev, "region lacks devmap flags\n");
dee41079 DW	164	return VM_FAULT_SIGBUS;
	165	}
	166
0134ed4f DJ	167	if (fault_size < dax_region->align)
	168	return VM_FAULT_SIGBUS;
	169	else if (fault_size > dax_region->align)
	170	return VM_FAULT_FALLBACK;
	171
	172	/* if we are outside of the VMA */
	173	if (pmd_addr < vmf->vma->vm_start \|\|
	174	(pmd_addr + PMD_SIZE) > vmf->vma->vm_end)
	175	return VM_FAULT_SIGBUS;
	176
f4200391	177	pgoff = linear_page_index(vmf->vma, pmd_addr);
73616367	178	phys = dax_pgoff_to_phys(dev_dax, pgoff, PMD_SIZE);
dee41079	179	if (phys == -1) {
6daaca52	180	dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff);
dee41079 DW	181	return VM_FAULT_SIGBUS;
	182	}
	183
2232c638	184	*pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
dee41079	185
fce86ff5	186	return vmf_insert_pfn_pmd(vmf, *pfn, vmf->flags & FAULT_FLAG_WRITE);
dee41079 DW	187	}
dee41079 DW	188
9557feee	189	#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
226ab561	190	static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax,
2232c638	191	struct vm_fault vmf, pfn_t pfn)
9557feee DJ	192	{
9557feee DJ	193	unsigned long pud_addr = vmf->address & PUD_MASK;
5f0694b3	194	struct device *dev = &dev_dax->dev;
9557feee DJ	195	struct dax_region *dax_region;
	196	phys_addr_t phys;
	197	pgoff_t pgoff;
70b085b0 DJ	198	unsigned int fault_size = PUD_SIZE;
70b085b0 DJ	199
9557feee	200
5f0694b3	201	if (check_vma(dev_dax, vmf->vma, __func__))
9557feee DJ	202	return VM_FAULT_SIGBUS;
9557feee DJ	203
5f0694b3	204	dax_region = dev_dax->region;
9557feee	205	if (dax_region->align > PUD_SIZE) {
6daaca52 DW	206	dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n",
6daaca52 DW	207	dax_region->align, fault_size);
9557feee DJ	208	return VM_FAULT_SIGBUS;
	209	}
	210
	211	/* dax pud mappings require pfn_t_devmap() */
	212	if ((dax_region->pfn_flags & (PFN_DEV\|PFN_MAP)) != (PFN_DEV\|PFN_MAP)) {
6daaca52	213	dev_dbg(dev, "region lacks devmap flags\n");
9557feee DJ	214	return VM_FAULT_SIGBUS;
	215	}
	216
70b085b0 DJ	217	if (fault_size < dax_region->align)
	218	return VM_FAULT_SIGBUS;
	219	else if (fault_size > dax_region->align)
	220	return VM_FAULT_FALLBACK;
	221
	222	/* if we are outside of the VMA */
	223	if (pud_addr < vmf->vma->vm_start \|\|
	224	(pud_addr + PUD_SIZE) > vmf->vma->vm_end)
	225	return VM_FAULT_SIGBUS;
	226
9557feee	227	pgoff = linear_page_index(vmf->vma, pud_addr);
73616367	228	phys = dax_pgoff_to_phys(dev_dax, pgoff, PUD_SIZE);
9557feee	229	if (phys == -1) {
6daaca52	230	dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff);
9557feee DJ	231	return VM_FAULT_SIGBUS;
	232	}
	233
2232c638	234	*pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
9557feee	235
fce86ff5	236	return vmf_insert_pfn_pud(vmf, *pfn, vmf->flags & FAULT_FLAG_WRITE);
9557feee DJ	237	}
9557feee DJ	238	#else
226ab561	239	static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax,
2232c638	240	struct vm_fault vmf, pfn_t pfn)
9557feee DJ	241	{
	242	return VM_FAULT_FALLBACK;
	243	}
	244	#endif /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
	245
226ab561	246	static vm_fault_t dev_dax_huge_fault(struct vm_fault *vmf,
c791ace1	247	enum page_entry_size pe_size)
dee41079	248	{
f4200391	249	struct file *filp = vmf->vma->vm_file;
2232c638	250	unsigned long fault_size;
36bdac1e SJ	251	vm_fault_t rc = VM_FAULT_SIGBUS;
36bdac1e SJ	252	int id;
2232c638	253	pfn_t pfn;
5f0694b3	254	struct dev_dax *dev_dax = filp->private_data;
dee41079	255
6daaca52 DW	256	dev_dbg(&dev_dax->dev, "%s: %s (%#lx - %#lx) size = %d\n", current->comm,
6daaca52 DW	257	(vmf->flags & FAULT_FLAG_WRITE) ? "write" : "read",
76202620	258	vmf->vma->vm_start, vmf->vma->vm_end, pe_size);
dee41079	259
7b6be844	260	id = dax_read_lock();
c791ace1 DJ	261	switch (pe_size) {
c791ace1 DJ	262	case PE_SIZE_PTE:
2232c638 DW	263	fault_size = PAGE_SIZE;
2232c638 DW	264	rc = __dev_dax_pte_fault(dev_dax, vmf, &pfn);
a2d58167	265	break;
c791ace1	266	case PE_SIZE_PMD:
2232c638 DW	267	fault_size = PMD_SIZE;
2232c638 DW	268	rc = __dev_dax_pmd_fault(dev_dax, vmf, &pfn);
9557feee	269	break;
c791ace1	270	case PE_SIZE_PUD:
2232c638 DW	271	fault_size = PUD_SIZE;
2232c638 DW	272	rc = __dev_dax_pud_fault(dev_dax, vmf, &pfn);
a2d58167 DJ	273	break;
a2d58167 DJ	274	default:
54eafcc9	275	rc = VM_FAULT_SIGBUS;
a2d58167	276	}
2232c638 DW	277
	278	if (rc == VM_FAULT_NOPAGE) {
	279	unsigned long i;
35de2995	280	pgoff_t pgoff;
2232c638 DW	281
	282	/*
	283	* In the device-dax case the only possibility for a
	284	* VM_FAULT_NOPAGE result is when device-dax capacity is
	285	* mapped. No need to consider the zero page, or racing
	286	* conflicting mappings.
	287	*/
35de2995 DW	288	pgoff = linear_page_index(vmf->vma, vmf->address
35de2995 DW	289	& ~(fault_size - 1));
2232c638 DW	290	for (i = 0; i < fault_size / PAGE_SIZE; i++) {
	291	struct page *page;
	292
	293	page = pfn_to_page(pfn_t_to_pfn(pfn) + i);
	294	if (page->mapping)
	295	continue;
	296	page->mapping = filp->f_mapping;
35de2995	297	page->index = pgoff + i;
2232c638 DW	298	}
2232c638 DW	299	}
7b6be844	300	dax_read_unlock(id);
dee41079 DW	301
	302	return rc;
	303	}
	304
226ab561	305	static vm_fault_t dev_dax_fault(struct vm_fault *vmf)
c791ace1	306	{
5f0694b3	307	return dev_dax_huge_fault(vmf, PE_SIZE_PTE);
c791ace1 DJ	308	}
c791ace1 DJ	309
9702cffd DW	310	static int dev_dax_split(struct vm_area_struct *vma, unsigned long addr)
	311	{
	312	struct file *filp = vma->vm_file;
	313	struct dev_dax *dev_dax = filp->private_data;
	314	struct dax_region *dax_region = dev_dax->region;
	315
	316	if (!IS_ALIGNED(addr, dax_region->align))
	317	return -EINVAL;
	318	return 0;
	319	}
	320
c1d53b92 DW	321	static unsigned long dev_dax_pagesize(struct vm_area_struct *vma)
	322	{
	323	struct file *filp = vma->vm_file;
	324	struct dev_dax *dev_dax = filp->private_data;
	325	struct dax_region *dax_region = dev_dax->region;
	326
	327	return dax_region->align;
	328	}
	329
5f0694b3 DW	330	static const struct vm_operations_struct dax_vm_ops = {
	331	.fault = dev_dax_fault,
	332	.huge_fault = dev_dax_huge_fault,
9702cffd	333	.split = dev_dax_split,
c1d53b92	334	.pagesize = dev_dax_pagesize,
dee41079 DW	335	};
dee41079 DW	336
af69f51e	337	static int dax_mmap(struct file filp, struct vm_area_struct vma)
dee41079	338	{
5f0694b3	339	struct dev_dax *dev_dax = filp->private_data;
7b6be844	340	int rc, id;
dee41079	341
6daaca52	342	dev_dbg(&dev_dax->dev, "trace\n");
dee41079	343
7b6be844 DW	344	/*
	345	* We lock to check dax_dev liveness and will re-check at
	346	* fault time.
	347	*/
	348	id = dax_read_lock();
5f0694b3	349	rc = check_vma(dev_dax, vma, __func__);
7b6be844	350	dax_read_unlock(id);
dee41079 DW	351	if (rc)
	352	return rc;
	353
5f0694b3	354	vma->vm_ops = &dax_vm_ops;
e1fb4a08	355	vma->vm_flags \|= VM_HUGEPAGE;
dee41079	356	return 0;
043a9255 DW	357	}
	358
	359	/* return an unmapped area aligned to the dax region specified alignment */
af69f51e	360	static unsigned long dax_get_unmapped_area(struct file *filp,
043a9255 DW	361	unsigned long addr, unsigned long len, unsigned long pgoff,
	362	unsigned long flags)
	363	{
	364	unsigned long off, off_end, off_align, len_align, addr_align, align;
5f0694b3	365	struct dev_dax *dev_dax = filp ? filp->private_data : NULL;
043a9255 DW	366	struct dax_region *dax_region;
043a9255 DW	367
5f0694b3	368	if (!dev_dax \|\| addr)
043a9255 DW	369	goto out;
043a9255 DW	370
5f0694b3	371	dax_region = dev_dax->region;
043a9255 DW	372	align = dax_region->align;
	373	off = pgoff << PAGE_SHIFT;
	374	off_end = off + len;
	375	off_align = round_up(off, align);
	376
	377	if ((off_end <= off_align) \|\| ((off_end - off_align) < align))
	378	goto out;
	379
	380	len_align = len + align;
	381	if ((off + len_align) < off)
	382	goto out;
dee41079	383
043a9255 DW	384	addr_align = current->mm->get_unmapped_area(filp, addr, len_align,
	385	pgoff, flags);
	386	if (!IS_ERR_VALUE(addr_align)) {
	387	addr_align += (off - addr_align) & (align - 1);
	388	return addr_align;
	389	}
	390	out:
	391	return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);
	392	}
	393
41c9b1be DJ	394	static const struct address_space_operations dev_dax_aops = {
	395	.set_page_dirty = noop_set_page_dirty,
	396	.invalidatepage = noop_invalidatepage,
	397	};
	398
af69f51e	399	static int dax_open(struct inode inode, struct file filp)
043a9255	400	{
7b6be844 DW	401	struct dax_device *dax_dev = inode_dax(inode);
	402	struct inode *__dax_inode = dax_inode(dax_dev);
	403	struct dev_dax *dev_dax = dax_get_private(dax_dev);
043a9255	404
6daaca52	405	dev_dbg(&dev_dax->dev, "trace\n");
7b6be844 DW	406	inode->i_mapping = __dax_inode->i_mapping;
7b6be844 DW	407	inode->i_mapping->host = __dax_inode;
41c9b1be	408	inode->i_mapping->a_ops = &dev_dax_aops;
3bc52c45	409	filp->f_mapping = inode->i_mapping;
5660e13d	410	filp->f_wb_err = filemap_sample_wb_err(filp->f_mapping);
5f0694b3	411	filp->private_data = dev_dax;
ebd84d72	412	inode->i_flags = S_DAX;
043a9255	413
043a9255 DW	414	return 0;
043a9255 DW	415	}
dee41079	416
af69f51e	417	static int dax_release(struct inode inode, struct file filp)
043a9255	418	{
5f0694b3	419	struct dev_dax *dev_dax = filp->private_data;
043a9255	420
6daaca52	421	dev_dbg(&dev_dax->dev, "trace\n");
043a9255	422	return 0;
dee41079 DW	423	}
dee41079 DW	424
ab68f262 DW	425	static const struct file_operations dax_fops = {
	426	.llseek = noop_llseek,
	427	.owner = THIS_MODULE,
af69f51e DW	428	.open = dax_open,
	429	.release = dax_release,
	430	.get_unmapped_area = dax_get_unmapped_area,
	431	.mmap = dax_mmap,
ef842302	432	.mmap_supported_flags = MAP_SYNC,
ab68f262 DW	433	};
ab68f262 DW	434
9567da0b	435	static void dev_dax_cdev_del(void *cdev)
043a9255	436	{
9567da0b DW	437	cdev_del(cdev);
9567da0b DW	438	}
043a9255	439
9567da0b DW	440	static void dev_dax_kill(void *dev_dax)
	441	{
	442	kill_dev_dax(dev_dax);
ebd84d72 DW	443	}
ebd84d72 DW	444
730926c3	445	int dev_dax_probe(struct device *dev)
043a9255	446	{
9567da0b DW	447	struct dev_dax *dev_dax = to_dev_dax(dev);
9567da0b DW	448	struct dax_device *dax_dev = dev_dax->dax_dev;
89ec9f2c	449	struct resource *res = &dev_dax->region->res;
7b6be844	450	struct inode *inode;
ba09c01d	451	struct cdev *cdev;
89ec9f2c	452	void *addr;
753a0850	453	int rc;
43fe51e1	454
89ec9f2c DW	455	/* 1:1 map region resource range to device-dax instance range */
	456	if (!devm_request_mem_region(dev, res->start, resource_size(res),
	457	dev_name(dev))) {
	458	dev_warn(dev, "could not reserve region %pR\n", res);
	459	return -EBUSY;
	460	}
	461
	462	init_completion(&dev_dax->cmp);
	463	rc = percpu_ref_init(&dev_dax->ref, dev_dax_percpu_release, 0,
	464	GFP_KERNEL);
	465	if (rc)
	466	return rc;
	467
89ec9f2c DW	468	dev_dax->pgmap.ref = &dev_dax->ref;
89ec9f2c DW	469	dev_dax->pgmap.kill = dev_dax_percpu_kill;
50f44ee7	470	dev_dax->pgmap.cleanup = dev_dax_percpu_exit;
89ec9f2c	471	addr = devm_memremap_pages(dev, &dev_dax->pgmap);
50f44ee7	472	if (IS_ERR(addr))
89ec9f2c	473	return PTR_ERR(addr);
89ec9f2c	474
7b6be844 DW	475	inode = dax_inode(dax_dev);
7b6be844 DW	476	cdev = inode->i_cdev;
ba09c01d	477	cdev_init(cdev, &dax_fops);
730926c3 DW	478	if (dev->class) {
	479	/* for the CONFIG_DEV_DAX_PMEM_COMPAT case */
	480	cdev->owner = dev->parent->driver->owner;
	481	} else
	482	cdev->owner = dev->driver->owner;
9567da0b DW	483	cdev_set_parent(cdev, &dev->kobj);
9567da0b DW	484	rc = cdev_add(cdev, dev->devt, 1);
d76911ee	485	if (rc)
9567da0b	486	return rc;
d76911ee	487
9567da0b DW	488	rc = devm_add_action_or_reset(dev, dev_dax_cdev_del, cdev);
	489	if (rc)
	490	return rc;
043a9255	491
9567da0b DW	492	run_dax(dax_dev);
	493	return devm_add_action_or_reset(dev, dev_dax_kill, dev_dax);
	494	}
730926c3	495	EXPORT_SYMBOL_GPL(dev_dax_probe);
043a9255	496
9567da0b DW	497	static int dev_dax_remove(struct device *dev)
	498	{
	499	/* all probe actions are unwound by devm */
	500	return 0;
043a9255	501	}
9567da0b	502
d200781e DW	503	static struct dax_device_driver device_dax_driver = {
	504	.drv = {
	505	.probe = dev_dax_probe,
	506	.remove = dev_dax_remove,
	507	},
	508	.match_always = 1,
9567da0b	509	};
043a9255	510
ab68f262 DW	511	static int __init dax_init(void)
ab68f262 DW	512	{
9567da0b	513	return dax_driver_register(&device_dax_driver);
ab68f262 DW	514	}
	515
	516	static void __exit dax_exit(void)
	517	{
d200781e	518	dax_driver_unregister(&device_dax_driver);
ab68f262 DW	519	}
	520
	521	MODULE_AUTHOR("Intel Corporation");
	522	MODULE_LICENSE("GPL v2");
9567da0b	523	module_init(dax_init);
ab68f262	524	module_exit(dax_exit);
9567da0b	525	MODULE_ALIAS_DAX_DEVICE(0);