2 * Copyright(c) 2016 - 2017 Intel Corporation. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
13 #include <linux/pagemap.h>
14 #include <linux/module.h>
15 #include <linux/device.h>
16 #include <linux/pfn_t.h>
17 #include <linux/cdev.h>
18 #include <linux/slab.h>
19 #include <linux/dax.h>
24 static struct class *dax_class
;
27 * struct dax_region - mapping infrastructure for dax devices
28 * @id: kernel-wide unique region for a memory range
29 * @base: linear address corresponding to @res
30 * @kref: to pin while other agents have a need to do lookups
31 * @dev: parent device backing this region
32 * @align: allocation and mapping alignment for child dax devices
33 * @res: physical address range of the region
34 * @pfn_flags: identify whether the pfns are paged back or not
44 unsigned long pfn_flags
;
48 * struct dev_dax - instance data for a subdivision of a dax region
49 * @region - parent region
50 * @dax_dev - core dax functionality
52 * @id - child id in the region
53 * @num_resources - number of physical address extents in this device
54 * @res - array of physical address ranges
57 struct dax_region
*region
;
58 struct dax_device
*dax_dev
;
62 struct resource res
[0];
65 static ssize_t
id_show(struct device
*dev
,
66 struct device_attribute
*attr
, char *buf
)
68 struct dax_region
*dax_region
;
72 dax_region
= dev_get_drvdata(dev
);
74 rc
= sprintf(buf
, "%d\n", dax_region
->id
);
79 static DEVICE_ATTR_RO(id
);
81 static ssize_t
region_size_show(struct device
*dev
,
82 struct device_attribute
*attr
, char *buf
)
84 struct dax_region
*dax_region
;
88 dax_region
= dev_get_drvdata(dev
);
90 rc
= sprintf(buf
, "%llu\n", (unsigned long long)
91 resource_size(&dax_region
->res
));
96 static struct device_attribute dev_attr_region_size
= __ATTR(size
, 0444,
97 region_size_show
, NULL
);
99 static ssize_t
align_show(struct device
*dev
,
100 struct device_attribute
*attr
, char *buf
)
102 struct dax_region
*dax_region
;
106 dax_region
= dev_get_drvdata(dev
);
108 rc
= sprintf(buf
, "%u\n", dax_region
->align
);
113 static DEVICE_ATTR_RO(align
);
115 static struct attribute
*dax_region_attributes
[] = {
116 &dev_attr_region_size
.attr
,
117 &dev_attr_align
.attr
,
122 static const struct attribute_group dax_region_attribute_group
= {
123 .name
= "dax_region",
124 .attrs
= dax_region_attributes
,
127 static const struct attribute_group
*dax_region_attribute_groups
[] = {
128 &dax_region_attribute_group
,
132 static void dax_region_free(struct kref
*kref
)
134 struct dax_region
*dax_region
;
136 dax_region
= container_of(kref
, struct dax_region
, kref
);
140 void dax_region_put(struct dax_region
*dax_region
)
142 kref_put(&dax_region
->kref
, dax_region_free
);
144 EXPORT_SYMBOL_GPL(dax_region_put
);
146 static void dax_region_unregister(void *region
)
148 struct dax_region
*dax_region
= region
;
150 sysfs_remove_groups(&dax_region
->dev
->kobj
,
151 dax_region_attribute_groups
);
152 dax_region_put(dax_region
);
155 struct dax_region
*alloc_dax_region(struct device
*parent
, int region_id
,
156 struct resource
*res
, unsigned int align
, void *addr
,
157 unsigned long pfn_flags
)
159 struct dax_region
*dax_region
;
162 * The DAX core assumes that it can store its private data in
163 * parent->driver_data. This WARN is a reminder / safeguard for
164 * developers of device-dax drivers.
166 if (dev_get_drvdata(parent
)) {
167 dev_WARN(parent
, "dax core failed to setup private data\n");
171 if (!IS_ALIGNED(res
->start
, align
)
172 || !IS_ALIGNED(resource_size(res
), align
))
175 dax_region
= kzalloc(sizeof(*dax_region
), GFP_KERNEL
);
179 dev_set_drvdata(parent
, dax_region
);
180 memcpy(&dax_region
->res
, res
, sizeof(*res
));
181 dax_region
->pfn_flags
= pfn_flags
;
182 kref_init(&dax_region
->kref
);
183 dax_region
->id
= region_id
;
184 ida_init(&dax_region
->ida
);
185 dax_region
->align
= align
;
186 dax_region
->dev
= parent
;
187 dax_region
->base
= addr
;
188 if (sysfs_create_groups(&parent
->kobj
, dax_region_attribute_groups
)) {
193 kref_get(&dax_region
->kref
);
194 if (devm_add_action_or_reset(parent
, dax_region_unregister
, dax_region
))
198 EXPORT_SYMBOL_GPL(alloc_dax_region
);
200 static struct dev_dax
*to_dev_dax(struct device
*dev
)
202 return container_of(dev
, struct dev_dax
, dev
);
205 static ssize_t
size_show(struct device
*dev
,
206 struct device_attribute
*attr
, char *buf
)
208 struct dev_dax
*dev_dax
= to_dev_dax(dev
);
209 unsigned long long size
= 0;
212 for (i
= 0; i
< dev_dax
->num_resources
; i
++)
213 size
+= resource_size(&dev_dax
->res
[i
]);
215 return sprintf(buf
, "%llu\n", size
);
217 static DEVICE_ATTR_RO(size
);
219 static struct attribute
*dev_dax_attributes
[] = {
224 static const struct attribute_group dev_dax_attribute_group
= {
225 .attrs
= dev_dax_attributes
,
228 static const struct attribute_group
*dax_attribute_groups
[] = {
229 &dev_dax_attribute_group
,
233 static int check_vma(struct dev_dax
*dev_dax
, struct vm_area_struct
*vma
,
236 struct dax_region
*dax_region
= dev_dax
->region
;
237 struct device
*dev
= &dev_dax
->dev
;
240 if (!dax_alive(dev_dax
->dax_dev
))
243 /* prevent private mappings from being established */
244 if ((vma
->vm_flags
& VM_MAYSHARE
) != VM_MAYSHARE
) {
245 dev_info(dev
, "%s: %s: fail, attempted private mapping\n",
246 current
->comm
, func
);
250 mask
= dax_region
->align
- 1;
251 if (vma
->vm_start
& mask
|| vma
->vm_end
& mask
) {
252 dev_info(dev
, "%s: %s: fail, unaligned vma (%#lx - %#lx, %#lx)\n",
253 current
->comm
, func
, vma
->vm_start
, vma
->vm_end
,
258 if ((dax_region
->pfn_flags
& (PFN_DEV
|PFN_MAP
)) == PFN_DEV
259 && (vma
->vm_flags
& VM_DONTCOPY
) == 0) {
260 dev_info(dev
, "%s: %s: fail, dax range requires MADV_DONTFORK\n",
261 current
->comm
, func
);
265 if (!vma_is_dax(vma
)) {
266 dev_info(dev
, "%s: %s: fail, vma is not DAX capable\n",
267 current
->comm
, func
);
274 static phys_addr_t
pgoff_to_phys(struct dev_dax
*dev_dax
, pgoff_t pgoff
,
277 struct resource
*res
;
281 for (i
= 0; i
< dev_dax
->num_resources
; i
++) {
282 res
= &dev_dax
->res
[i
];
283 phys
= pgoff
* PAGE_SIZE
+ res
->start
;
284 if (phys
>= res
->start
&& phys
<= res
->end
)
286 pgoff
-= PHYS_PFN(resource_size(res
));
289 if (i
< dev_dax
->num_resources
) {
290 res
= &dev_dax
->res
[i
];
291 if (phys
+ size
- 1 <= res
->end
)
298 static int __dev_dax_pte_fault(struct dev_dax
*dev_dax
, struct vm_fault
*vmf
)
300 struct device
*dev
= &dev_dax
->dev
;
301 struct dax_region
*dax_region
;
302 int rc
= VM_FAULT_SIGBUS
;
305 unsigned int fault_size
= PAGE_SIZE
;
307 if (check_vma(dev_dax
, vmf
->vma
, __func__
))
308 return VM_FAULT_SIGBUS
;
310 dax_region
= dev_dax
->region
;
311 if (dax_region
->align
> PAGE_SIZE
) {
312 dev_dbg(dev
, "%s: alignment (%#x) > fault size (%#x)\n",
313 __func__
, dax_region
->align
, fault_size
);
314 return VM_FAULT_SIGBUS
;
317 if (fault_size
!= dax_region
->align
)
318 return VM_FAULT_SIGBUS
;
320 phys
= pgoff_to_phys(dev_dax
, vmf
->pgoff
, PAGE_SIZE
);
322 dev_dbg(dev
, "%s: pgoff_to_phys(%#lx) failed\n", __func__
,
324 return VM_FAULT_SIGBUS
;
327 pfn
= phys_to_pfn_t(phys
, dax_region
->pfn_flags
);
329 rc
= vm_insert_mixed(vmf
->vma
, vmf
->address
, pfn
);
333 if (rc
< 0 && rc
!= -EBUSY
)
334 return VM_FAULT_SIGBUS
;
336 return VM_FAULT_NOPAGE
;
339 static int __dev_dax_pmd_fault(struct dev_dax
*dev_dax
, struct vm_fault
*vmf
)
341 unsigned long pmd_addr
= vmf
->address
& PMD_MASK
;
342 struct device
*dev
= &dev_dax
->dev
;
343 struct dax_region
*dax_region
;
347 unsigned int fault_size
= PMD_SIZE
;
349 if (check_vma(dev_dax
, vmf
->vma
, __func__
))
350 return VM_FAULT_SIGBUS
;
352 dax_region
= dev_dax
->region
;
353 if (dax_region
->align
> PMD_SIZE
) {
354 dev_dbg(dev
, "%s: alignment (%#x) > fault size (%#x)\n",
355 __func__
, dax_region
->align
, fault_size
);
356 return VM_FAULT_SIGBUS
;
359 /* dax pmd mappings require pfn_t_devmap() */
360 if ((dax_region
->pfn_flags
& (PFN_DEV
|PFN_MAP
)) != (PFN_DEV
|PFN_MAP
)) {
361 dev_dbg(dev
, "%s: region lacks devmap flags\n", __func__
);
362 return VM_FAULT_SIGBUS
;
365 if (fault_size
< dax_region
->align
)
366 return VM_FAULT_SIGBUS
;
367 else if (fault_size
> dax_region
->align
)
368 return VM_FAULT_FALLBACK
;
370 /* if we are outside of the VMA */
371 if (pmd_addr
< vmf
->vma
->vm_start
||
372 (pmd_addr
+ PMD_SIZE
) > vmf
->vma
->vm_end
)
373 return VM_FAULT_SIGBUS
;
375 pgoff
= linear_page_index(vmf
->vma
, pmd_addr
);
376 phys
= pgoff_to_phys(dev_dax
, pgoff
, PMD_SIZE
);
378 dev_dbg(dev
, "%s: pgoff_to_phys(%#lx) failed\n", __func__
,
380 return VM_FAULT_SIGBUS
;
383 pfn
= phys_to_pfn_t(phys
, dax_region
->pfn_flags
);
385 return vmf_insert_pfn_pmd(vmf
->vma
, vmf
->address
, vmf
->pmd
, pfn
,
386 vmf
->flags
& FAULT_FLAG_WRITE
);
389 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
390 static int __dev_dax_pud_fault(struct dev_dax
*dev_dax
, struct vm_fault
*vmf
)
392 unsigned long pud_addr
= vmf
->address
& PUD_MASK
;
393 struct device
*dev
= &dev_dax
->dev
;
394 struct dax_region
*dax_region
;
398 unsigned int fault_size
= PUD_SIZE
;
401 if (check_vma(dev_dax
, vmf
->vma
, __func__
))
402 return VM_FAULT_SIGBUS
;
404 dax_region
= dev_dax
->region
;
405 if (dax_region
->align
> PUD_SIZE
) {
406 dev_dbg(dev
, "%s: alignment (%#x) > fault size (%#x)\n",
407 __func__
, dax_region
->align
, fault_size
);
408 return VM_FAULT_SIGBUS
;
411 /* dax pud mappings require pfn_t_devmap() */
412 if ((dax_region
->pfn_flags
& (PFN_DEV
|PFN_MAP
)) != (PFN_DEV
|PFN_MAP
)) {
413 dev_dbg(dev
, "%s: region lacks devmap flags\n", __func__
);
414 return VM_FAULT_SIGBUS
;
417 if (fault_size
< dax_region
->align
)
418 return VM_FAULT_SIGBUS
;
419 else if (fault_size
> dax_region
->align
)
420 return VM_FAULT_FALLBACK
;
422 /* if we are outside of the VMA */
423 if (pud_addr
< vmf
->vma
->vm_start
||
424 (pud_addr
+ PUD_SIZE
) > vmf
->vma
->vm_end
)
425 return VM_FAULT_SIGBUS
;
427 pgoff
= linear_page_index(vmf
->vma
, pud_addr
);
428 phys
= pgoff_to_phys(dev_dax
, pgoff
, PUD_SIZE
);
430 dev_dbg(dev
, "%s: pgoff_to_phys(%#lx) failed\n", __func__
,
432 return VM_FAULT_SIGBUS
;
435 pfn
= phys_to_pfn_t(phys
, dax_region
->pfn_flags
);
437 return vmf_insert_pfn_pud(vmf
->vma
, vmf
->address
, vmf
->pud
, pfn
,
438 vmf
->flags
& FAULT_FLAG_WRITE
);
441 static int __dev_dax_pud_fault(struct dev_dax
*dev_dax
, struct vm_fault
*vmf
)
443 return VM_FAULT_FALLBACK
;
445 #endif /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
447 static int dev_dax_huge_fault(struct vm_fault
*vmf
,
448 enum page_entry_size pe_size
)
451 struct file
*filp
= vmf
->vma
->vm_file
;
452 struct dev_dax
*dev_dax
= filp
->private_data
;
454 dev_dbg(&dev_dax
->dev
, "%s: %s: %s (%#lx - %#lx) size = %d\n", __func__
,
455 current
->comm
, (vmf
->flags
& FAULT_FLAG_WRITE
)
457 vmf
->vma
->vm_start
, vmf
->vma
->vm_end
, pe_size
);
459 id
= dax_read_lock();
462 rc
= __dev_dax_pte_fault(dev_dax
, vmf
);
465 rc
= __dev_dax_pmd_fault(dev_dax
, vmf
);
468 rc
= __dev_dax_pud_fault(dev_dax
, vmf
);
471 rc
= VM_FAULT_SIGBUS
;
478 static int dev_dax_fault(struct vm_fault
*vmf
)
480 return dev_dax_huge_fault(vmf
, PE_SIZE_PTE
);
483 static const struct vm_operations_struct dax_vm_ops
= {
484 .fault
= dev_dax_fault
,
485 .huge_fault
= dev_dax_huge_fault
,
488 static int dax_mmap(struct file
*filp
, struct vm_area_struct
*vma
)
490 struct dev_dax
*dev_dax
= filp
->private_data
;
493 dev_dbg(&dev_dax
->dev
, "%s\n", __func__
);
496 * We lock to check dax_dev liveness and will re-check at
499 id
= dax_read_lock();
500 rc
= check_vma(dev_dax
, vma
, __func__
);
505 vma
->vm_ops
= &dax_vm_ops
;
506 vma
->vm_flags
|= VM_MIXEDMAP
| VM_HUGEPAGE
;
510 /* return an unmapped area aligned to the dax region specified alignment */
511 static unsigned long dax_get_unmapped_area(struct file
*filp
,
512 unsigned long addr
, unsigned long len
, unsigned long pgoff
,
515 unsigned long off
, off_end
, off_align
, len_align
, addr_align
, align
;
516 struct dev_dax
*dev_dax
= filp
? filp
->private_data
: NULL
;
517 struct dax_region
*dax_region
;
519 if (!dev_dax
|| addr
)
522 dax_region
= dev_dax
->region
;
523 align
= dax_region
->align
;
524 off
= pgoff
<< PAGE_SHIFT
;
526 off_align
= round_up(off
, align
);
528 if ((off_end
<= off_align
) || ((off_end
- off_align
) < align
))
531 len_align
= len
+ align
;
532 if ((off
+ len_align
) < off
)
535 addr_align
= current
->mm
->get_unmapped_area(filp
, addr
, len_align
,
537 if (!IS_ERR_VALUE(addr_align
)) {
538 addr_align
+= (off
- addr_align
) & (align
- 1);
542 return current
->mm
->get_unmapped_area(filp
, addr
, len
, pgoff
, flags
);
545 static int dax_open(struct inode
*inode
, struct file
*filp
)
547 struct dax_device
*dax_dev
= inode_dax(inode
);
548 struct inode
*__dax_inode
= dax_inode(dax_dev
);
549 struct dev_dax
*dev_dax
= dax_get_private(dax_dev
);
551 dev_dbg(&dev_dax
->dev
, "%s\n", __func__
);
552 inode
->i_mapping
= __dax_inode
->i_mapping
;
553 inode
->i_mapping
->host
= __dax_inode
;
554 filp
->f_mapping
= inode
->i_mapping
;
555 filp
->private_data
= dev_dax
;
556 inode
->i_flags
= S_DAX
;
561 static int dax_release(struct inode
*inode
, struct file
*filp
)
563 struct dev_dax
*dev_dax
= filp
->private_data
;
565 dev_dbg(&dev_dax
->dev
, "%s\n", __func__
);
569 static const struct file_operations dax_fops
= {
570 .llseek
= noop_llseek
,
571 .owner
= THIS_MODULE
,
573 .release
= dax_release
,
574 .get_unmapped_area
= dax_get_unmapped_area
,
578 static void dev_dax_release(struct device
*dev
)
580 struct dev_dax
*dev_dax
= to_dev_dax(dev
);
581 struct dax_region
*dax_region
= dev_dax
->region
;
582 struct dax_device
*dax_dev
= dev_dax
->dax_dev
;
584 ida_simple_remove(&dax_region
->ida
, dev_dax
->id
);
585 dax_region_put(dax_region
);
590 static void kill_dev_dax(struct dev_dax
*dev_dax
)
592 struct dax_device
*dax_dev
= dev_dax
->dax_dev
;
593 struct inode
*inode
= dax_inode(dax_dev
);
596 unmap_mapping_range(inode
->i_mapping
, 0, 0, 1);
599 static void unregister_dev_dax(void *dev
)
601 struct dev_dax
*dev_dax
= to_dev_dax(dev
);
602 struct dax_device
*dax_dev
= dev_dax
->dax_dev
;
603 struct inode
*inode
= dax_inode(dax_dev
);
604 struct cdev
*cdev
= inode
->i_cdev
;
606 dev_dbg(dev
, "%s\n", __func__
);
608 kill_dev_dax(dev_dax
);
609 cdev_device_del(cdev
, dev
);
613 struct dev_dax
*devm_create_dev_dax(struct dax_region
*dax_region
,
614 struct resource
*res
, int count
)
616 struct device
*parent
= dax_region
->dev
;
617 struct dax_device
*dax_dev
;
618 struct dev_dax
*dev_dax
;
624 dev_dax
= kzalloc(sizeof(*dev_dax
) + sizeof(*res
) * count
, GFP_KERNEL
);
626 return ERR_PTR(-ENOMEM
);
628 for (i
= 0; i
< count
; i
++) {
629 if (!IS_ALIGNED(res
[i
].start
, dax_region
->align
)
630 || !IS_ALIGNED(resource_size(&res
[i
]),
631 dax_region
->align
)) {
635 dev_dax
->res
[i
].start
= res
[i
].start
;
636 dev_dax
->res
[i
].end
= res
[i
].end
;
642 dev_dax
->id
= ida_simple_get(&dax_region
->ida
, 0, 0, GFP_KERNEL
);
643 if (dev_dax
->id
< 0) {
648 dax_dev
= alloc_dax(dev_dax
, NULL
);
652 /* from here on we're committed to teardown via dax_dev_release() */
654 device_initialize(dev
);
656 inode
= dax_inode(dax_dev
);
657 cdev
= inode
->i_cdev
;
658 cdev_init(cdev
, &dax_fops
);
659 cdev
->owner
= parent
->driver
->owner
;
661 dev_dax
->num_resources
= count
;
662 dev_dax
->dax_dev
= dax_dev
;
663 dev_dax
->region
= dax_region
;
664 kref_get(&dax_region
->kref
);
666 dev
->devt
= inode
->i_rdev
;
667 dev
->class = dax_class
;
668 dev
->parent
= parent
;
669 dev
->groups
= dax_attribute_groups
;
670 dev
->release
= dev_dax_release
;
671 dev_set_name(dev
, "dax%d.%d", dax_region
->id
, dev_dax
->id
);
673 rc
= cdev_device_add(cdev
, dev
);
675 kill_dev_dax(dev_dax
);
680 rc
= devm_add_action_or_reset(dax_region
->dev
, unregister_dev_dax
, dev
);
687 ida_simple_remove(&dax_region
->ida
, dev_dax
->id
);
693 EXPORT_SYMBOL_GPL(devm_create_dev_dax
);
695 static int __init
dax_init(void)
697 dax_class
= class_create(THIS_MODULE
, "dax");
698 return PTR_ERR_OR_ZERO(dax_class
);
701 static void __exit
dax_exit(void)
703 class_destroy(dax_class
);
706 MODULE_AUTHOR("Intel Corporation");
707 MODULE_LICENSE("GPL v2");
708 subsys_initcall(dax_init
);
709 module_exit(dax_exit
);