1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2017-2018 Intel Corporation. All rights reserved. */
3 #include <linux/memremap.h>
4 #include <linux/device.h>
5 #include <linux/mutex.h>
6 #include <linux/list.h>
7 #include <linux/slab.h>
10 #include "dax-private.h"
13 static struct class *dax_class
;
15 static DEFINE_MUTEX(dax_bus_lock
);
17 #define DAX_NAME_LEN 30
19 struct list_head list
;
20 char dev_name
[DAX_NAME_LEN
];
23 static int dax_bus_uevent(struct device
*dev
, struct kobj_uevent_env
*env
)
26 * We only ever expect to handle device-dax instances, i.e. the
27 * @type argument to MODULE_ALIAS_DAX_DEVICE() is always zero
29 return add_uevent_var(env
, "MODALIAS=" DAX_DEVICE_MODALIAS_FMT
, 0);
32 static struct dax_device_driver
*to_dax_drv(struct device_driver
*drv
)
34 return container_of(drv
, struct dax_device_driver
, drv
);
37 static struct dax_id
*__dax_match_id(struct dax_device_driver
*dax_drv
,
40 struct dax_id
*dax_id
;
42 lockdep_assert_held(&dax_bus_lock
);
44 list_for_each_entry(dax_id
, &dax_drv
->ids
, list
)
45 if (sysfs_streq(dax_id
->dev_name
, dev_name
))
50 static int dax_match_id(struct dax_device_driver
*dax_drv
, struct device
*dev
)
54 mutex_lock(&dax_bus_lock
);
55 match
= !!__dax_match_id(dax_drv
, dev_name(dev
));
56 mutex_unlock(&dax_bus_lock
);
66 static ssize_t
do_id_store(struct device_driver
*drv
, const char *buf
,
67 size_t count
, enum id_action action
)
69 struct dax_device_driver
*dax_drv
= to_dax_drv(drv
);
70 unsigned int region_id
, id
;
71 char devname
[DAX_NAME_LEN
];
72 struct dax_id
*dax_id
;
76 fields
= sscanf(buf
, "dax%d.%d", ®ion_id
, &id
);
79 sprintf(devname
, "dax%d.%d", region_id
, id
);
80 if (!sysfs_streq(buf
, devname
))
83 mutex_lock(&dax_bus_lock
);
84 dax_id
= __dax_match_id(dax_drv
, buf
);
86 if (action
== ID_ADD
) {
87 dax_id
= kzalloc(sizeof(*dax_id
), GFP_KERNEL
);
89 strncpy(dax_id
->dev_name
, buf
, DAX_NAME_LEN
);
90 list_add(&dax_id
->list
, &dax_drv
->ids
);
94 } else if (action
== ID_REMOVE
) {
95 list_del(&dax_id
->list
);
98 mutex_unlock(&dax_bus_lock
);
102 if (action
== ID_ADD
)
103 rc
= driver_attach(drv
);
109 static ssize_t
new_id_store(struct device_driver
*drv
, const char *buf
,
112 return do_id_store(drv
, buf
, count
, ID_ADD
);
114 static DRIVER_ATTR_WO(new_id
);
116 static ssize_t
remove_id_store(struct device_driver
*drv
, const char *buf
,
119 return do_id_store(drv
, buf
, count
, ID_REMOVE
);
121 static DRIVER_ATTR_WO(remove_id
);
123 static struct attribute
*dax_drv_attrs
[] = {
124 &driver_attr_new_id
.attr
,
125 &driver_attr_remove_id
.attr
,
128 ATTRIBUTE_GROUPS(dax_drv
);
130 static int dax_bus_match(struct device
*dev
, struct device_driver
*drv
);
132 static bool is_static(struct dax_region
*dax_region
)
134 return (dax_region
->res
.flags
& IORESOURCE_DAX_STATIC
) != 0;
137 static u64
dev_dax_size(struct dev_dax
*dev_dax
)
142 device_lock_assert(&dev_dax
->dev
);
144 for (i
= 0; i
< dev_dax
->nr_range
; i
++)
145 size
+= range_len(&dev_dax
->ranges
[i
].range
);
150 static int dax_bus_probe(struct device
*dev
)
152 struct dax_device_driver
*dax_drv
= to_dax_drv(dev
->driver
);
153 struct dev_dax
*dev_dax
= to_dev_dax(dev
);
154 struct dax_region
*dax_region
= dev_dax
->region
;
157 if (dev_dax_size(dev_dax
) == 0 || dev_dax
->id
< 0)
160 rc
= dax_drv
->probe(dev_dax
);
162 if (rc
|| is_static(dax_region
))
166 * Track new seed creation only after successful probe of the
169 if (dax_region
->seed
== dev
)
170 dax_region
->seed
= NULL
;
175 static void dax_bus_remove(struct device
*dev
)
177 struct dax_device_driver
*dax_drv
= to_dax_drv(dev
->driver
);
178 struct dev_dax
*dev_dax
= to_dev_dax(dev
);
181 dax_drv
->remove(dev_dax
);
184 static struct bus_type dax_bus_type
= {
186 .uevent
= dax_bus_uevent
,
187 .match
= dax_bus_match
,
188 .probe
= dax_bus_probe
,
189 .remove
= dax_bus_remove
,
190 .drv_groups
= dax_drv_groups
,
193 static int dax_bus_match(struct device
*dev
, struct device_driver
*drv
)
195 struct dax_device_driver
*dax_drv
= to_dax_drv(drv
);
198 * All but the 'device-dax' driver, which has 'match_always'
199 * set, requires an exact id match.
201 if (dax_drv
->match_always
)
204 return dax_match_id(dax_drv
, dev
);
208 * Rely on the fact that drvdata is set before the attributes are
209 * registered, and that the attributes are unregistered before drvdata
210 * is cleared to assume that drvdata is always valid.
212 static ssize_t
id_show(struct device
*dev
,
213 struct device_attribute
*attr
, char *buf
)
215 struct dax_region
*dax_region
= dev_get_drvdata(dev
);
217 return sprintf(buf
, "%d\n", dax_region
->id
);
219 static DEVICE_ATTR_RO(id
);
221 static ssize_t
region_size_show(struct device
*dev
,
222 struct device_attribute
*attr
, char *buf
)
224 struct dax_region
*dax_region
= dev_get_drvdata(dev
);
226 return sprintf(buf
, "%llu\n", (unsigned long long)
227 resource_size(&dax_region
->res
));
229 static struct device_attribute dev_attr_region_size
= __ATTR(size
, 0444,
230 region_size_show
, NULL
);
232 static ssize_t
region_align_show(struct device
*dev
,
233 struct device_attribute
*attr
, char *buf
)
235 struct dax_region
*dax_region
= dev_get_drvdata(dev
);
237 return sprintf(buf
, "%u\n", dax_region
->align
);
239 static struct device_attribute dev_attr_region_align
=
240 __ATTR(align
, 0400, region_align_show
, NULL
);
242 #define for_each_dax_region_resource(dax_region, res) \
243 for (res = (dax_region)->res.child; res; res = res->sibling)
245 static unsigned long long dax_region_avail_size(struct dax_region
*dax_region
)
247 resource_size_t size
= resource_size(&dax_region
->res
);
248 struct resource
*res
;
250 device_lock_assert(dax_region
->dev
);
252 for_each_dax_region_resource(dax_region
, res
)
253 size
-= resource_size(res
);
257 static ssize_t
available_size_show(struct device
*dev
,
258 struct device_attribute
*attr
, char *buf
)
260 struct dax_region
*dax_region
= dev_get_drvdata(dev
);
261 unsigned long long size
;
264 size
= dax_region_avail_size(dax_region
);
267 return sprintf(buf
, "%llu\n", size
);
269 static DEVICE_ATTR_RO(available_size
);
271 static ssize_t
seed_show(struct device
*dev
,
272 struct device_attribute
*attr
, char *buf
)
274 struct dax_region
*dax_region
= dev_get_drvdata(dev
);
278 if (is_static(dax_region
))
282 seed
= dax_region
->seed
;
283 rc
= sprintf(buf
, "%s\n", seed
? dev_name(seed
) : "");
288 static DEVICE_ATTR_RO(seed
);
290 static ssize_t
create_show(struct device
*dev
,
291 struct device_attribute
*attr
, char *buf
)
293 struct dax_region
*dax_region
= dev_get_drvdata(dev
);
294 struct device
*youngest
;
297 if (is_static(dax_region
))
301 youngest
= dax_region
->youngest
;
302 rc
= sprintf(buf
, "%s\n", youngest
? dev_name(youngest
) : "");
308 static ssize_t
create_store(struct device
*dev
, struct device_attribute
*attr
,
309 const char *buf
, size_t len
)
311 struct dax_region
*dax_region
= dev_get_drvdata(dev
);
312 unsigned long long avail
;
316 if (is_static(dax_region
))
319 rc
= kstrtoint(buf
, 0, &val
);
326 avail
= dax_region_avail_size(dax_region
);
330 struct dev_dax_data data
= {
331 .dax_region
= dax_region
,
335 struct dev_dax
*dev_dax
= devm_create_dev_dax(&data
);
338 rc
= PTR_ERR(dev_dax
);
341 * In support of crafting multiple new devices
342 * simultaneously multiple seeds can be created,
343 * but only the first one that has not been
344 * successfully bound is tracked as the region
347 if (!dax_region
->seed
)
348 dax_region
->seed
= &dev_dax
->dev
;
349 dax_region
->youngest
= &dev_dax
->dev
;
357 static DEVICE_ATTR_RW(create
);
359 void kill_dev_dax(struct dev_dax
*dev_dax
)
361 struct dax_device
*dax_dev
= dev_dax
->dax_dev
;
362 struct inode
*inode
= dax_inode(dax_dev
);
365 unmap_mapping_range(inode
->i_mapping
, 0, 0, 1);
367 EXPORT_SYMBOL_GPL(kill_dev_dax
);
369 static void trim_dev_dax_range(struct dev_dax
*dev_dax
)
371 int i
= dev_dax
->nr_range
- 1;
372 struct range
*range
= &dev_dax
->ranges
[i
].range
;
373 struct dax_region
*dax_region
= dev_dax
->region
;
375 device_lock_assert(dax_region
->dev
);
376 dev_dbg(&dev_dax
->dev
, "delete range[%d]: %#llx:%#llx\n", i
,
377 (unsigned long long)range
->start
,
378 (unsigned long long)range
->end
);
380 __release_region(&dax_region
->res
, range
->start
, range_len(range
));
381 if (--dev_dax
->nr_range
== 0) {
382 kfree(dev_dax
->ranges
);
383 dev_dax
->ranges
= NULL
;
387 static void free_dev_dax_ranges(struct dev_dax
*dev_dax
)
389 while (dev_dax
->nr_range
)
390 trim_dev_dax_range(dev_dax
);
393 static void unregister_dev_dax(void *dev
)
395 struct dev_dax
*dev_dax
= to_dev_dax(dev
);
397 dev_dbg(dev
, "%s\n", __func__
);
399 kill_dev_dax(dev_dax
);
400 free_dev_dax_ranges(dev_dax
);
405 /* a return value >= 0 indicates this invocation invalidated the id */
406 static int __free_dev_dax_id(struct dev_dax
*dev_dax
)
408 struct dax_region
*dax_region
= dev_dax
->region
;
409 struct device
*dev
= &dev_dax
->dev
;
410 int rc
= dev_dax
->id
;
412 device_lock_assert(dev
);
414 if (is_static(dax_region
) || dev_dax
->id
< 0)
416 ida_free(&dax_region
->ida
, dev_dax
->id
);
421 static int free_dev_dax_id(struct dev_dax
*dev_dax
)
423 struct device
*dev
= &dev_dax
->dev
;
427 rc
= __free_dev_dax_id(dev_dax
);
432 static ssize_t
delete_store(struct device
*dev
, struct device_attribute
*attr
,
433 const char *buf
, size_t len
)
435 struct dax_region
*dax_region
= dev_get_drvdata(dev
);
436 struct dev_dax
*dev_dax
;
437 struct device
*victim
;
441 if (is_static(dax_region
))
444 victim
= device_find_child_by_name(dax_region
->dev
, buf
);
450 dev_dax
= to_dev_dax(victim
);
451 if (victim
->driver
|| dev_dax_size(dev_dax
))
455 * Invalidate the device so it does not become active
456 * again, but always preserve device-id-0 so that
457 * /sys/bus/dax/ is guaranteed to be populated while any
458 * dax_region is registered.
460 if (dev_dax
->id
> 0) {
461 do_del
= __free_dev_dax_id(dev_dax
) >= 0;
463 if (dax_region
->seed
== victim
)
464 dax_region
->seed
= NULL
;
465 if (dax_region
->youngest
== victim
)
466 dax_region
->youngest
= NULL
;
470 device_unlock(victim
);
472 /* won the race to invalidate the device, clean it up */
474 devm_release_action(dev
, unregister_dev_dax
, victim
);
480 static DEVICE_ATTR_WO(delete);
482 static umode_t
dax_region_visible(struct kobject
*kobj
, struct attribute
*a
,
485 struct device
*dev
= container_of(kobj
, struct device
, kobj
);
486 struct dax_region
*dax_region
= dev_get_drvdata(dev
);
488 if (is_static(dax_region
))
489 if (a
== &dev_attr_available_size
.attr
490 || a
== &dev_attr_create
.attr
491 || a
== &dev_attr_seed
.attr
492 || a
== &dev_attr_delete
.attr
)
497 static struct attribute
*dax_region_attributes
[] = {
498 &dev_attr_available_size
.attr
,
499 &dev_attr_region_size
.attr
,
500 &dev_attr_region_align
.attr
,
501 &dev_attr_create
.attr
,
503 &dev_attr_delete
.attr
,
508 static const struct attribute_group dax_region_attribute_group
= {
509 .name
= "dax_region",
510 .attrs
= dax_region_attributes
,
511 .is_visible
= dax_region_visible
,
514 static const struct attribute_group
*dax_region_attribute_groups
[] = {
515 &dax_region_attribute_group
,
519 static void dax_region_free(struct kref
*kref
)
521 struct dax_region
*dax_region
;
523 dax_region
= container_of(kref
, struct dax_region
, kref
);
527 void dax_region_put(struct dax_region
*dax_region
)
529 kref_put(&dax_region
->kref
, dax_region_free
);
531 EXPORT_SYMBOL_GPL(dax_region_put
);
533 static void dax_region_unregister(void *region
)
535 struct dax_region
*dax_region
= region
;
537 sysfs_remove_groups(&dax_region
->dev
->kobj
,
538 dax_region_attribute_groups
);
539 dax_region_put(dax_region
);
542 struct dax_region
*alloc_dax_region(struct device
*parent
, int region_id
,
543 struct range
*range
, int target_node
, unsigned int align
,
546 struct dax_region
*dax_region
;
549 * The DAX core assumes that it can store its private data in
550 * parent->driver_data. This WARN is a reminder / safeguard for
551 * developers of device-dax drivers.
553 if (dev_get_drvdata(parent
)) {
554 dev_WARN(parent
, "dax core failed to setup private data\n");
558 if (!IS_ALIGNED(range
->start
, align
)
559 || !IS_ALIGNED(range_len(range
), align
))
562 dax_region
= kzalloc(sizeof(*dax_region
), GFP_KERNEL
);
566 dev_set_drvdata(parent
, dax_region
);
567 kref_init(&dax_region
->kref
);
568 dax_region
->id
= region_id
;
569 dax_region
->align
= align
;
570 dax_region
->dev
= parent
;
571 dax_region
->target_node
= target_node
;
572 ida_init(&dax_region
->ida
);
573 dax_region
->res
= (struct resource
) {
574 .start
= range
->start
,
576 .flags
= IORESOURCE_MEM
| flags
,
579 if (sysfs_create_groups(&parent
->kobj
, dax_region_attribute_groups
)) {
584 kref_get(&dax_region
->kref
);
585 if (devm_add_action_or_reset(parent
, dax_region_unregister
, dax_region
))
589 EXPORT_SYMBOL_GPL(alloc_dax_region
);
591 static void dax_mapping_release(struct device
*dev
)
593 struct dax_mapping
*mapping
= to_dax_mapping(dev
);
594 struct dev_dax
*dev_dax
= to_dev_dax(dev
->parent
);
596 ida_free(&dev_dax
->ida
, mapping
->id
);
600 static void unregister_dax_mapping(void *data
)
602 struct device
*dev
= data
;
603 struct dax_mapping
*mapping
= to_dax_mapping(dev
);
604 struct dev_dax
*dev_dax
= to_dev_dax(dev
->parent
);
605 struct dax_region
*dax_region
= dev_dax
->region
;
607 dev_dbg(dev
, "%s\n", __func__
);
609 device_lock_assert(dax_region
->dev
);
611 dev_dax
->ranges
[mapping
->range_id
].mapping
= NULL
;
612 mapping
->range_id
= -1;
618 static struct dev_dax_range
*get_dax_range(struct device
*dev
)
620 struct dax_mapping
*mapping
= to_dax_mapping(dev
);
621 struct dev_dax
*dev_dax
= to_dev_dax(dev
->parent
);
622 struct dax_region
*dax_region
= dev_dax
->region
;
624 device_lock(dax_region
->dev
);
625 if (mapping
->range_id
< 0) {
626 device_unlock(dax_region
->dev
);
630 return &dev_dax
->ranges
[mapping
->range_id
];
633 static void put_dax_range(struct dev_dax_range
*dax_range
)
635 struct dax_mapping
*mapping
= dax_range
->mapping
;
636 struct dev_dax
*dev_dax
= to_dev_dax(mapping
->dev
.parent
);
637 struct dax_region
*dax_region
= dev_dax
->region
;
639 device_unlock(dax_region
->dev
);
642 static ssize_t
start_show(struct device
*dev
,
643 struct device_attribute
*attr
, char *buf
)
645 struct dev_dax_range
*dax_range
;
648 dax_range
= get_dax_range(dev
);
651 rc
= sprintf(buf
, "%#llx\n", dax_range
->range
.start
);
652 put_dax_range(dax_range
);
656 static DEVICE_ATTR(start
, 0400, start_show
, NULL
);
658 static ssize_t
end_show(struct device
*dev
,
659 struct device_attribute
*attr
, char *buf
)
661 struct dev_dax_range
*dax_range
;
664 dax_range
= get_dax_range(dev
);
667 rc
= sprintf(buf
, "%#llx\n", dax_range
->range
.end
);
668 put_dax_range(dax_range
);
672 static DEVICE_ATTR(end
, 0400, end_show
, NULL
);
674 static ssize_t
pgoff_show(struct device
*dev
,
675 struct device_attribute
*attr
, char *buf
)
677 struct dev_dax_range
*dax_range
;
680 dax_range
= get_dax_range(dev
);
683 rc
= sprintf(buf
, "%#lx\n", dax_range
->pgoff
);
684 put_dax_range(dax_range
);
688 static DEVICE_ATTR(page_offset
, 0400, pgoff_show
, NULL
);
690 static struct attribute
*dax_mapping_attributes
[] = {
691 &dev_attr_start
.attr
,
693 &dev_attr_page_offset
.attr
,
697 static const struct attribute_group dax_mapping_attribute_group
= {
698 .attrs
= dax_mapping_attributes
,
701 static const struct attribute_group
*dax_mapping_attribute_groups
[] = {
702 &dax_mapping_attribute_group
,
706 static struct device_type dax_mapping_type
= {
707 .release
= dax_mapping_release
,
708 .groups
= dax_mapping_attribute_groups
,
711 static int devm_register_dax_mapping(struct dev_dax
*dev_dax
, int range_id
)
713 struct dax_region
*dax_region
= dev_dax
->region
;
714 struct dax_mapping
*mapping
;
718 device_lock_assert(dax_region
->dev
);
720 if (dev_WARN_ONCE(&dev_dax
->dev
, !dax_region
->dev
->driver
,
721 "region disabled\n"))
724 mapping
= kzalloc(sizeof(*mapping
), GFP_KERNEL
);
727 mapping
->range_id
= range_id
;
728 mapping
->id
= ida_alloc(&dev_dax
->ida
, GFP_KERNEL
);
729 if (mapping
->id
< 0) {
733 dev_dax
->ranges
[range_id
].mapping
= mapping
;
735 device_initialize(dev
);
736 dev
->parent
= &dev_dax
->dev
;
737 dev
->type
= &dax_mapping_type
;
738 dev_set_name(dev
, "mapping%d", mapping
->id
);
739 rc
= device_add(dev
);
745 rc
= devm_add_action_or_reset(dax_region
->dev
, unregister_dax_mapping
,
752 static int alloc_dev_dax_range(struct dev_dax
*dev_dax
, u64 start
,
753 resource_size_t size
)
755 struct dax_region
*dax_region
= dev_dax
->region
;
756 struct resource
*res
= &dax_region
->res
;
757 struct device
*dev
= &dev_dax
->dev
;
758 struct dev_dax_range
*ranges
;
759 unsigned long pgoff
= 0;
760 struct resource
*alloc
;
763 device_lock_assert(dax_region
->dev
);
765 /* handle the seed alloc special case */
767 if (dev_WARN_ONCE(dev
, dev_dax
->nr_range
,
768 "0-size allocation must be first\n"))
770 /* nr_range == 0 is elsewhere special cased as 0-size device */
774 alloc
= __request_region(res
, start
, size
, dev_name(dev
), 0);
778 ranges
= krealloc(dev_dax
->ranges
, sizeof(*ranges
)
779 * (dev_dax
->nr_range
+ 1), GFP_KERNEL
);
781 __release_region(res
, alloc
->start
, resource_size(alloc
));
785 for (i
= 0; i
< dev_dax
->nr_range
; i
++)
786 pgoff
+= PHYS_PFN(range_len(&ranges
[i
].range
));
787 dev_dax
->ranges
= ranges
;
788 ranges
[dev_dax
->nr_range
++] = (struct dev_dax_range
) {
791 .start
= alloc
->start
,
796 dev_dbg(dev
, "alloc range[%d]: %pa:%pa\n", dev_dax
->nr_range
- 1,
797 &alloc
->start
, &alloc
->end
);
799 * A dev_dax instance must be registered before mapping device
800 * children can be added. Defer to devm_create_dev_dax() to add
801 * the initial mapping device.
803 if (!device_is_registered(&dev_dax
->dev
))
806 rc
= devm_register_dax_mapping(dev_dax
, dev_dax
->nr_range
- 1);
808 trim_dev_dax_range(dev_dax
);
813 static int adjust_dev_dax_range(struct dev_dax
*dev_dax
, struct resource
*res
, resource_size_t size
)
815 int last_range
= dev_dax
->nr_range
- 1;
816 struct dev_dax_range
*dax_range
= &dev_dax
->ranges
[last_range
];
817 struct dax_region
*dax_region
= dev_dax
->region
;
818 bool is_shrink
= resource_size(res
) > size
;
819 struct range
*range
= &dax_range
->range
;
820 struct device
*dev
= &dev_dax
->dev
;
823 device_lock_assert(dax_region
->dev
);
825 if (dev_WARN_ONCE(dev
, !size
, "deletion is handled by dev_dax_shrink\n"))
828 rc
= adjust_resource(res
, range
->start
, size
);
832 *range
= (struct range
) {
833 .start
= range
->start
,
834 .end
= range
->start
+ size
- 1,
837 dev_dbg(dev
, "%s range[%d]: %#llx:%#llx\n", is_shrink
? "shrink" : "extend",
838 last_range
, (unsigned long long) range
->start
,
839 (unsigned long long) range
->end
);
844 static ssize_t
size_show(struct device
*dev
,
845 struct device_attribute
*attr
, char *buf
)
847 struct dev_dax
*dev_dax
= to_dev_dax(dev
);
848 unsigned long long size
;
851 size
= dev_dax_size(dev_dax
);
854 return sprintf(buf
, "%llu\n", size
);
857 static bool alloc_is_aligned(struct dev_dax
*dev_dax
, resource_size_t size
)
860 * The minimum mapping granularity for a device instance is a
861 * single subsection, unless the arch says otherwise.
863 return IS_ALIGNED(size
, max_t(unsigned long, dev_dax
->align
, memremap_compat_align()));
866 static int dev_dax_shrink(struct dev_dax
*dev_dax
, resource_size_t size
)
868 resource_size_t to_shrink
= dev_dax_size(dev_dax
) - size
;
869 struct dax_region
*dax_region
= dev_dax
->region
;
870 struct device
*dev
= &dev_dax
->dev
;
873 for (i
= dev_dax
->nr_range
- 1; i
>= 0; i
--) {
874 struct range
*range
= &dev_dax
->ranges
[i
].range
;
875 struct dax_mapping
*mapping
= dev_dax
->ranges
[i
].mapping
;
876 struct resource
*adjust
= NULL
, *res
;
877 resource_size_t shrink
;
879 shrink
= min_t(u64
, to_shrink
, range_len(range
));
880 if (shrink
>= range_len(range
)) {
881 devm_release_action(dax_region
->dev
,
882 unregister_dax_mapping
, &mapping
->dev
);
883 trim_dev_dax_range(dev_dax
);
890 for_each_dax_region_resource(dax_region
, res
)
891 if (strcmp(res
->name
, dev_name(dev
)) == 0
892 && res
->start
== range
->start
) {
897 if (dev_WARN_ONCE(dev
, !adjust
|| i
!= dev_dax
->nr_range
- 1,
898 "failed to find matching resource\n"))
900 return adjust_dev_dax_range(dev_dax
, adjust
, range_len(range
)
907 * Only allow adjustments that preserve the relative pgoff of existing
908 * allocations. I.e. the dev_dax->ranges array is ordered by increasing pgoff.
910 static bool adjust_ok(struct dev_dax
*dev_dax
, struct resource
*res
)
912 struct dev_dax_range
*last
;
915 if (dev_dax
->nr_range
== 0)
917 if (strcmp(res
->name
, dev_name(&dev_dax
->dev
)) != 0)
919 last
= &dev_dax
->ranges
[dev_dax
->nr_range
- 1];
920 if (last
->range
.start
!= res
->start
|| last
->range
.end
!= res
->end
)
922 for (i
= 0; i
< dev_dax
->nr_range
- 1; i
++) {
923 struct dev_dax_range
*dax_range
= &dev_dax
->ranges
[i
];
925 if (dax_range
->pgoff
> last
->pgoff
)
932 static ssize_t
dev_dax_resize(struct dax_region
*dax_region
,
933 struct dev_dax
*dev_dax
, resource_size_t size
)
935 resource_size_t avail
= dax_region_avail_size(dax_region
), to_alloc
;
936 resource_size_t dev_size
= dev_dax_size(dev_dax
);
937 struct resource
*region_res
= &dax_region
->res
;
938 struct device
*dev
= &dev_dax
->dev
;
939 struct resource
*res
, *first
;
940 resource_size_t alloc
= 0;
945 if (size
== dev_size
)
947 if (size
> dev_size
&& size
- dev_size
> avail
)
950 return dev_dax_shrink(dev_dax
, size
);
952 to_alloc
= size
- dev_size
;
953 if (dev_WARN_ONCE(dev
, !alloc_is_aligned(dev_dax
, to_alloc
),
954 "resize of %pa misaligned\n", &to_alloc
))
958 * Expand the device into the unused portion of the region. This
959 * may involve adjusting the end of an existing resource, or
960 * allocating a new resource.
963 first
= region_res
->child
;
965 return alloc_dev_dax_range(dev_dax
, dax_region
->res
.start
, to_alloc
);
968 for (res
= first
; res
; res
= res
->sibling
) {
969 struct resource
*next
= res
->sibling
;
971 /* space at the beginning of the region */
972 if (res
== first
&& res
->start
> dax_region
->res
.start
) {
973 alloc
= min(res
->start
- dax_region
->res
.start
, to_alloc
);
974 rc
= alloc_dev_dax_range(dev_dax
, dax_region
->res
.start
, alloc
);
979 /* space between allocations */
980 if (next
&& next
->start
> res
->end
+ 1)
981 alloc
= min(next
->start
- (res
->end
+ 1), to_alloc
);
983 /* space at the end of the region */
984 if (!alloc
&& !next
&& res
->end
< region_res
->end
)
985 alloc
= min(region_res
->end
- res
->end
, to_alloc
);
990 if (adjust_ok(dev_dax
, res
)) {
991 rc
= adjust_dev_dax_range(dev_dax
, res
, resource_size(res
) + alloc
);
994 rc
= alloc_dev_dax_range(dev_dax
, res
->end
+ 1, alloc
);
1005 static ssize_t
size_store(struct device
*dev
, struct device_attribute
*attr
,
1006 const char *buf
, size_t len
)
1009 unsigned long long val
;
1010 struct dev_dax
*dev_dax
= to_dev_dax(dev
);
1011 struct dax_region
*dax_region
= dev_dax
->region
;
1013 rc
= kstrtoull(buf
, 0, &val
);
1017 if (!alloc_is_aligned(dev_dax
, val
)) {
1018 dev_dbg(dev
, "%s: size: %lld misaligned\n", __func__
, val
);
1022 device_lock(dax_region
->dev
);
1023 if (!dax_region
->dev
->driver
) {
1024 device_unlock(dax_region
->dev
);
1028 rc
= dev_dax_resize(dax_region
, dev_dax
, val
);
1030 device_unlock(dax_region
->dev
);
1032 return rc
== 0 ? len
: rc
;
1034 static DEVICE_ATTR_RW(size
);
1036 static ssize_t
range_parse(const char *opt
, size_t len
, struct range
*range
)
1038 unsigned long long addr
= 0;
1039 char *start
, *end
, *str
;
1040 ssize_t rc
= -EINVAL
;
1042 str
= kstrdup(opt
, GFP_KERNEL
);
1047 start
= strsep(&end
, "-");
1051 rc
= kstrtoull(start
, 16, &addr
);
1054 range
->start
= addr
;
1056 rc
= kstrtoull(end
, 16, &addr
);
1066 static ssize_t
mapping_store(struct device
*dev
, struct device_attribute
*attr
,
1067 const char *buf
, size_t len
)
1069 struct dev_dax
*dev_dax
= to_dev_dax(dev
);
1070 struct dax_region
*dax_region
= dev_dax
->region
;
1075 rc
= range_parse(buf
, len
, &r
);
1080 device_lock(dax_region
->dev
);
1081 if (!dax_region
->dev
->driver
) {
1082 device_unlock(dax_region
->dev
);
1087 to_alloc
= range_len(&r
);
1088 if (alloc_is_aligned(dev_dax
, to_alloc
))
1089 rc
= alloc_dev_dax_range(dev_dax
, r
.start
, to_alloc
);
1091 device_unlock(dax_region
->dev
);
1093 return rc
== 0 ? len
: rc
;
1095 static DEVICE_ATTR_WO(mapping
);
1097 static ssize_t
align_show(struct device
*dev
,
1098 struct device_attribute
*attr
, char *buf
)
1100 struct dev_dax
*dev_dax
= to_dev_dax(dev
);
1102 return sprintf(buf
, "%d\n", dev_dax
->align
);
1105 static ssize_t
dev_dax_validate_align(struct dev_dax
*dev_dax
)
1107 struct device
*dev
= &dev_dax
->dev
;
1110 for (i
= 0; i
< dev_dax
->nr_range
; i
++) {
1111 size_t len
= range_len(&dev_dax
->ranges
[i
].range
);
1113 if (!alloc_is_aligned(dev_dax
, len
)) {
1114 dev_dbg(dev
, "%s: align %u invalid for range %d\n",
1115 __func__
, dev_dax
->align
, i
);
1123 static ssize_t
align_store(struct device
*dev
, struct device_attribute
*attr
,
1124 const char *buf
, size_t len
)
1126 struct dev_dax
*dev_dax
= to_dev_dax(dev
);
1127 struct dax_region
*dax_region
= dev_dax
->region
;
1128 unsigned long val
, align_save
;
1131 rc
= kstrtoul(buf
, 0, &val
);
1135 if (!dax_align_valid(val
))
1138 device_lock(dax_region
->dev
);
1139 if (!dax_region
->dev
->driver
) {
1140 device_unlock(dax_region
->dev
);
1150 align_save
= dev_dax
->align
;
1151 dev_dax
->align
= val
;
1152 rc
= dev_dax_validate_align(dev_dax
);
1154 dev_dax
->align
= align_save
;
1157 device_unlock(dax_region
->dev
);
1158 return rc
== 0 ? len
: rc
;
1160 static DEVICE_ATTR_RW(align
);
1162 static int dev_dax_target_node(struct dev_dax
*dev_dax
)
1164 struct dax_region
*dax_region
= dev_dax
->region
;
1166 return dax_region
->target_node
;
1169 static ssize_t
target_node_show(struct device
*dev
,
1170 struct device_attribute
*attr
, char *buf
)
1172 struct dev_dax
*dev_dax
= to_dev_dax(dev
);
1174 return sprintf(buf
, "%d\n", dev_dax_target_node(dev_dax
));
1176 static DEVICE_ATTR_RO(target_node
);
1178 static ssize_t
resource_show(struct device
*dev
,
1179 struct device_attribute
*attr
, char *buf
)
1181 struct dev_dax
*dev_dax
= to_dev_dax(dev
);
1182 struct dax_region
*dax_region
= dev_dax
->region
;
1183 unsigned long long start
;
1185 if (dev_dax
->nr_range
< 1)
1186 start
= dax_region
->res
.start
;
1188 start
= dev_dax
->ranges
[0].range
.start
;
1190 return sprintf(buf
, "%#llx\n", start
);
1192 static DEVICE_ATTR(resource
, 0400, resource_show
, NULL
);
1194 static ssize_t
modalias_show(struct device
*dev
, struct device_attribute
*attr
,
1198 * We only ever expect to handle device-dax instances, i.e. the
1199 * @type argument to MODULE_ALIAS_DAX_DEVICE() is always zero
1201 return sprintf(buf
, DAX_DEVICE_MODALIAS_FMT
"\n", 0);
1203 static DEVICE_ATTR_RO(modalias
);
1205 static ssize_t
numa_node_show(struct device
*dev
,
1206 struct device_attribute
*attr
, char *buf
)
1208 return sprintf(buf
, "%d\n", dev_to_node(dev
));
1210 static DEVICE_ATTR_RO(numa_node
);
1212 static umode_t
dev_dax_visible(struct kobject
*kobj
, struct attribute
*a
, int n
)
1214 struct device
*dev
= container_of(kobj
, struct device
, kobj
);
1215 struct dev_dax
*dev_dax
= to_dev_dax(dev
);
1216 struct dax_region
*dax_region
= dev_dax
->region
;
1218 if (a
== &dev_attr_target_node
.attr
&& dev_dax_target_node(dev_dax
) < 0)
1220 if (a
== &dev_attr_numa_node
.attr
&& !IS_ENABLED(CONFIG_NUMA
))
1222 if (a
== &dev_attr_mapping
.attr
&& is_static(dax_region
))
1224 if ((a
== &dev_attr_align
.attr
||
1225 a
== &dev_attr_size
.attr
) && is_static(dax_region
))
1230 static struct attribute
*dev_dax_attributes
[] = {
1231 &dev_attr_modalias
.attr
,
1232 &dev_attr_size
.attr
,
1233 &dev_attr_mapping
.attr
,
1234 &dev_attr_target_node
.attr
,
1235 &dev_attr_align
.attr
,
1236 &dev_attr_resource
.attr
,
1237 &dev_attr_numa_node
.attr
,
1241 static const struct attribute_group dev_dax_attribute_group
= {
1242 .attrs
= dev_dax_attributes
,
1243 .is_visible
= dev_dax_visible
,
1246 static const struct attribute_group
*dax_attribute_groups
[] = {
1247 &dev_dax_attribute_group
,
1251 static void dev_dax_release(struct device
*dev
)
1253 struct dev_dax
*dev_dax
= to_dev_dax(dev
);
1254 struct dax_region
*dax_region
= dev_dax
->region
;
1255 struct dax_device
*dax_dev
= dev_dax
->dax_dev
;
1258 free_dev_dax_id(dev_dax
);
1259 dax_region_put(dax_region
);
1260 kfree(dev_dax
->pgmap
);
1264 static const struct device_type dev_dax_type
= {
1265 .release
= dev_dax_release
,
1266 .groups
= dax_attribute_groups
,
1269 struct dev_dax
*devm_create_dev_dax(struct dev_dax_data
*data
)
1271 struct dax_region
*dax_region
= data
->dax_region
;
1272 struct device
*parent
= dax_region
->dev
;
1273 struct dax_device
*dax_dev
;
1274 struct dev_dax
*dev_dax
;
1275 struct inode
*inode
;
1279 dev_dax
= kzalloc(sizeof(*dev_dax
), GFP_KERNEL
);
1281 return ERR_PTR(-ENOMEM
);
1283 if (is_static(dax_region
)) {
1284 if (dev_WARN_ONCE(parent
, data
->id
< 0,
1285 "dynamic id specified to static region\n")) {
1290 dev_dax
->id
= data
->id
;
1292 if (dev_WARN_ONCE(parent
, data
->id
>= 0,
1293 "static id specified to dynamic region\n")) {
1298 rc
= ida_alloc(&dax_region
->ida
, GFP_KERNEL
);
1304 dev_dax
->region
= dax_region
;
1305 dev
= &dev_dax
->dev
;
1306 device_initialize(dev
);
1307 dev_set_name(dev
, "dax%d.%d", dax_region
->id
, dev_dax
->id
);
1309 rc
= alloc_dev_dax_range(dev_dax
, dax_region
->res
.start
, data
->size
);
1314 dev_WARN_ONCE(parent
, !is_static(dax_region
),
1315 "custom dev_pagemap requires a static dax_region\n");
1317 dev_dax
->pgmap
= kmemdup(data
->pgmap
,
1318 sizeof(struct dev_pagemap
), GFP_KERNEL
);
1319 if (!dev_dax
->pgmap
) {
1326 * No 'host' or dax_operations since there is no access to this
1327 * device outside of mmap of the resulting character device.
1329 dax_dev
= alloc_dax(dev_dax
, NULL
, NULL
, DAXDEV_F_SYNC
);
1330 if (IS_ERR(dax_dev
)) {
1331 rc
= PTR_ERR(dax_dev
);
1335 /* a device_dax instance is dead while the driver is not attached */
1338 dev_dax
->dax_dev
= dax_dev
;
1339 dev_dax
->target_node
= dax_region
->target_node
;
1340 dev_dax
->align
= dax_region
->align
;
1341 ida_init(&dev_dax
->ida
);
1342 kref_get(&dax_region
->kref
);
1344 inode
= dax_inode(dax_dev
);
1345 dev
->devt
= inode
->i_rdev
;
1346 if (data
->subsys
== DEV_DAX_BUS
)
1347 dev
->bus
= &dax_bus_type
;
1349 dev
->class = dax_class
;
1350 dev
->parent
= parent
;
1351 dev
->type
= &dev_dax_type
;
1353 rc
= device_add(dev
);
1355 kill_dev_dax(dev_dax
);
1360 rc
= devm_add_action_or_reset(dax_region
->dev
, unregister_dev_dax
, dev
);
1364 /* register mapping device for the initial allocation range */
1365 if (dev_dax
->nr_range
&& range_len(&dev_dax
->ranges
[0].range
)) {
1366 rc
= devm_register_dax_mapping(dev_dax
, 0);
1374 kfree(dev_dax
->pgmap
);
1376 free_dev_dax_ranges(dev_dax
);
1378 free_dev_dax_id(dev_dax
);
1384 EXPORT_SYMBOL_GPL(devm_create_dev_dax
);
1386 static int match_always_count
;
1388 int __dax_driver_register(struct dax_device_driver
*dax_drv
,
1389 struct module
*module
, const char *mod_name
)
1391 struct device_driver
*drv
= &dax_drv
->drv
;
1395 * dax_bus_probe() calls dax_drv->probe() unconditionally.
1396 * So better be safe than sorry and ensure it is provided.
1398 if (!dax_drv
->probe
)
1401 INIT_LIST_HEAD(&dax_drv
->ids
);
1402 drv
->owner
= module
;
1403 drv
->name
= mod_name
;
1404 drv
->mod_name
= mod_name
;
1405 drv
->bus
= &dax_bus_type
;
1407 /* there can only be one default driver */
1408 mutex_lock(&dax_bus_lock
);
1409 match_always_count
+= dax_drv
->match_always
;
1410 if (match_always_count
> 1) {
1411 match_always_count
--;
1415 mutex_unlock(&dax_bus_lock
);
1419 rc
= driver_register(drv
);
1420 if (rc
&& dax_drv
->match_always
) {
1421 mutex_lock(&dax_bus_lock
);
1422 match_always_count
-= dax_drv
->match_always
;
1423 mutex_unlock(&dax_bus_lock
);
1428 EXPORT_SYMBOL_GPL(__dax_driver_register
);
1430 void dax_driver_unregister(struct dax_device_driver
*dax_drv
)
1432 struct device_driver
*drv
= &dax_drv
->drv
;
1433 struct dax_id
*dax_id
, *_id
;
1435 mutex_lock(&dax_bus_lock
);
1436 match_always_count
-= dax_drv
->match_always
;
1437 list_for_each_entry_safe(dax_id
, _id
, &dax_drv
->ids
, list
) {
1438 list_del(&dax_id
->list
);
1441 mutex_unlock(&dax_bus_lock
);
1442 driver_unregister(drv
);
1444 EXPORT_SYMBOL_GPL(dax_driver_unregister
);
1446 int __init
dax_bus_init(void)
1450 if (IS_ENABLED(CONFIG_DEV_DAX_PMEM_COMPAT
)) {
1451 dax_class
= class_create(THIS_MODULE
, "dax");
1452 if (IS_ERR(dax_class
))
1453 return PTR_ERR(dax_class
);
1456 rc
= bus_register(&dax_bus_type
);
1458 class_destroy(dax_class
);
1462 void __exit
dax_bus_exit(void)
1464 bus_unregister(&dax_bus_type
);
1465 class_destroy(dax_class
);