2 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
14 #include <linux/vmalloc.h>
15 #include <linux/uaccess.h>
16 #include <linux/module.h>
17 #include <linux/blkdev.h>
18 #include <linux/fcntl.h>
19 #include <linux/async.h>
20 #include <linux/genhd.h>
21 #include <linux/ndctl.h>
22 #include <linux/sched.h>
23 #include <linux/slab.h>
33 static int nvdimm_bus_major
;
34 static struct class *nd_class
;
35 static DEFINE_IDA(nd_ida
);
37 static int to_nd_device_type(struct device
*dev
)
40 return ND_DEVICE_DIMM
;
41 else if (is_memory(dev
))
42 return ND_DEVICE_REGION_PMEM
;
43 else if (is_nd_blk(dev
))
44 return ND_DEVICE_REGION_BLK
;
45 else if (is_nd_dax(dev
))
46 return ND_DEVICE_DAX_PMEM
;
47 else if (is_nd_region(dev
->parent
))
48 return nd_region_to_nstype(to_nd_region(dev
->parent
));
53 static int nvdimm_bus_uevent(struct device
*dev
, struct kobj_uevent_env
*env
)
56 * Ensure that region devices always have their numa node set as
59 if (is_nd_region(dev
))
60 set_dev_node(dev
, to_nd_region(dev
)->numa_node
);
61 return add_uevent_var(env
, "MODALIAS=" ND_DEVICE_MODALIAS_FMT
,
62 to_nd_device_type(dev
));
65 static struct module
*to_bus_provider(struct device
*dev
)
67 /* pin bus providers while regions are enabled */
68 if (is_nd_region(dev
)) {
69 struct nvdimm_bus
*nvdimm_bus
= walk_to_nvdimm_bus(dev
);
71 return nvdimm_bus
->nd_desc
->module
;
76 static void nvdimm_bus_probe_start(struct nvdimm_bus
*nvdimm_bus
)
78 nvdimm_bus_lock(&nvdimm_bus
->dev
);
79 nvdimm_bus
->probe_active
++;
80 nvdimm_bus_unlock(&nvdimm_bus
->dev
);
83 static void nvdimm_bus_probe_end(struct nvdimm_bus
*nvdimm_bus
)
85 nvdimm_bus_lock(&nvdimm_bus
->dev
);
86 if (--nvdimm_bus
->probe_active
== 0)
87 wake_up(&nvdimm_bus
->probe_wait
);
88 nvdimm_bus_unlock(&nvdimm_bus
->dev
);
91 static int nvdimm_bus_probe(struct device
*dev
)
93 struct nd_device_driver
*nd_drv
= to_nd_device_driver(dev
->driver
);
94 struct module
*provider
= to_bus_provider(dev
);
95 struct nvdimm_bus
*nvdimm_bus
= walk_to_nvdimm_bus(dev
);
98 if (!try_module_get(provider
))
101 nvdimm_bus_probe_start(nvdimm_bus
);
102 rc
= nd_drv
->probe(dev
);
104 nd_region_probe_success(nvdimm_bus
, dev
);
106 nd_region_disable(nvdimm_bus
, dev
);
107 nvdimm_bus_probe_end(nvdimm_bus
);
109 dev_dbg(&nvdimm_bus
->dev
, "%s.probe(%s) = %d\n", dev
->driver
->name
,
113 module_put(provider
);
117 static int nvdimm_bus_remove(struct device
*dev
)
119 struct nd_device_driver
*nd_drv
= to_nd_device_driver(dev
->driver
);
120 struct module
*provider
= to_bus_provider(dev
);
121 struct nvdimm_bus
*nvdimm_bus
= walk_to_nvdimm_bus(dev
);
125 rc
= nd_drv
->remove(dev
);
126 nd_region_disable(nvdimm_bus
, dev
);
128 dev_dbg(&nvdimm_bus
->dev
, "%s.remove(%s) = %d\n", dev
->driver
->name
,
130 module_put(provider
);
134 static void nvdimm_bus_shutdown(struct device
*dev
)
136 struct nvdimm_bus
*nvdimm_bus
= walk_to_nvdimm_bus(dev
);
137 struct nd_device_driver
*nd_drv
= NULL
;
140 nd_drv
= to_nd_device_driver(dev
->driver
);
142 if (nd_drv
&& nd_drv
->shutdown
) {
143 nd_drv
->shutdown(dev
);
144 dev_dbg(&nvdimm_bus
->dev
, "%s.shutdown(%s)\n",
145 dev
->driver
->name
, dev_name(dev
));
149 void nd_device_notify(struct device
*dev
, enum nvdimm_event event
)
153 struct nd_device_driver
*nd_drv
;
155 nd_drv
= to_nd_device_driver(dev
->driver
);
157 nd_drv
->notify(dev
, event
);
161 EXPORT_SYMBOL(nd_device_notify
);
163 void nvdimm_region_notify(struct nd_region
*nd_region
, enum nvdimm_event event
)
165 struct nvdimm_bus
*nvdimm_bus
= walk_to_nvdimm_bus(&nd_region
->dev
);
170 /* caller is responsible for holding a reference on the device */
171 nd_device_notify(&nd_region
->dev
, event
);
173 EXPORT_SYMBOL_GPL(nvdimm_region_notify
);
175 struct clear_badblocks_context
{
176 resource_size_t phys
, cleared
;
179 static int nvdimm_clear_badblocks_region(struct device
*dev
, void *data
)
181 struct clear_badblocks_context
*ctx
= data
;
182 struct nd_region
*nd_region
;
183 resource_size_t ndr_end
;
186 /* make sure device is a region */
187 if (!is_nd_pmem(dev
))
190 nd_region
= to_nd_region(dev
);
191 ndr_end
= nd_region
->ndr_start
+ nd_region
->ndr_size
- 1;
193 /* make sure we are in the region */
194 if (ctx
->phys
< nd_region
->ndr_start
195 || (ctx
->phys
+ ctx
->cleared
) > ndr_end
)
198 sector
= (ctx
->phys
- nd_region
->ndr_start
) / 512;
199 badblocks_clear(&nd_region
->bb
, sector
, ctx
->cleared
/ 512);
201 if (nd_region
->bb_state
)
202 sysfs_notify_dirent(nd_region
->bb_state
);
207 static void nvdimm_clear_badblocks_regions(struct nvdimm_bus
*nvdimm_bus
,
208 phys_addr_t phys
, u64 cleared
)
210 struct clear_badblocks_context ctx
= {
215 device_for_each_child(&nvdimm_bus
->dev
, &ctx
,
216 nvdimm_clear_badblocks_region
);
219 static void nvdimm_account_cleared_poison(struct nvdimm_bus
*nvdimm_bus
,
220 phys_addr_t phys
, u64 cleared
)
223 nvdimm_forget_poison(nvdimm_bus
, phys
, cleared
);
225 if (cleared
> 0 && cleared
/ 512)
226 nvdimm_clear_badblocks_regions(nvdimm_bus
, phys
, cleared
);
229 long nvdimm_clear_poison(struct device
*dev
, phys_addr_t phys
,
232 struct nvdimm_bus
*nvdimm_bus
= walk_to_nvdimm_bus(dev
);
233 struct nvdimm_bus_descriptor
*nd_desc
;
234 struct nd_cmd_clear_error clear_err
;
235 struct nd_cmd_ars_cap ars_cap
;
236 u32 clear_err_unit
, mask
;
242 nd_desc
= nvdimm_bus
->nd_desc
;
244 * if ndctl does not exist, it's PMEM_LEGACY and
245 * we want to just pretend everything is handled.
250 memset(&ars_cap
, 0, sizeof(ars_cap
));
251 ars_cap
.address
= phys
;
252 ars_cap
.length
= len
;
253 rc
= nd_desc
->ndctl(nd_desc
, NULL
, ND_CMD_ARS_CAP
, &ars_cap
,
254 sizeof(ars_cap
), &cmd_rc
);
259 clear_err_unit
= ars_cap
.clear_err_unit
;
260 if (!clear_err_unit
|| !is_power_of_2(clear_err_unit
))
263 mask
= clear_err_unit
- 1;
264 if ((phys
| len
) & mask
)
266 memset(&clear_err
, 0, sizeof(clear_err
));
267 clear_err
.address
= phys
;
268 clear_err
.length
= len
;
269 rc
= nd_desc
->ndctl(nd_desc
, NULL
, ND_CMD_CLEAR_ERROR
, &clear_err
,
270 sizeof(clear_err
), &cmd_rc
);
276 nvdimm_account_cleared_poison(nvdimm_bus
, phys
, clear_err
.cleared
);
278 return clear_err
.cleared
;
280 EXPORT_SYMBOL_GPL(nvdimm_clear_poison
);
282 static int nvdimm_bus_match(struct device
*dev
, struct device_driver
*drv
);
284 static struct bus_type nvdimm_bus_type
= {
286 .uevent
= nvdimm_bus_uevent
,
287 .match
= nvdimm_bus_match
,
288 .probe
= nvdimm_bus_probe
,
289 .remove
= nvdimm_bus_remove
,
290 .shutdown
= nvdimm_bus_shutdown
,
293 static void nvdimm_bus_release(struct device
*dev
)
295 struct nvdimm_bus
*nvdimm_bus
;
297 nvdimm_bus
= container_of(dev
, struct nvdimm_bus
, dev
);
298 ida_simple_remove(&nd_ida
, nvdimm_bus
->id
);
302 static bool is_nvdimm_bus(struct device
*dev
)
304 return dev
->release
== nvdimm_bus_release
;
307 struct nvdimm_bus
*walk_to_nvdimm_bus(struct device
*nd_dev
)
311 for (dev
= nd_dev
; dev
; dev
= dev
->parent
)
312 if (is_nvdimm_bus(dev
))
314 dev_WARN_ONCE(nd_dev
, !dev
, "invalid dev, not on nd bus\n");
316 return to_nvdimm_bus(dev
);
320 struct nvdimm_bus
*to_nvdimm_bus(struct device
*dev
)
322 struct nvdimm_bus
*nvdimm_bus
;
324 nvdimm_bus
= container_of(dev
, struct nvdimm_bus
, dev
);
325 WARN_ON(!is_nvdimm_bus(dev
));
328 EXPORT_SYMBOL_GPL(to_nvdimm_bus
);
330 struct nvdimm_bus
*nvdimm_bus_register(struct device
*parent
,
331 struct nvdimm_bus_descriptor
*nd_desc
)
333 struct nvdimm_bus
*nvdimm_bus
;
336 nvdimm_bus
= kzalloc(sizeof(*nvdimm_bus
), GFP_KERNEL
);
339 INIT_LIST_HEAD(&nvdimm_bus
->list
);
340 INIT_LIST_HEAD(&nvdimm_bus
->mapping_list
);
341 INIT_LIST_HEAD(&nvdimm_bus
->poison_list
);
342 init_waitqueue_head(&nvdimm_bus
->probe_wait
);
343 nvdimm_bus
->id
= ida_simple_get(&nd_ida
, 0, 0, GFP_KERNEL
);
344 mutex_init(&nvdimm_bus
->reconfig_mutex
);
345 spin_lock_init(&nvdimm_bus
->poison_lock
);
346 if (nvdimm_bus
->id
< 0) {
350 nvdimm_bus
->nd_desc
= nd_desc
;
351 nvdimm_bus
->dev
.parent
= parent
;
352 nvdimm_bus
->dev
.release
= nvdimm_bus_release
;
353 nvdimm_bus
->dev
.groups
= nd_desc
->attr_groups
;
354 nvdimm_bus
->dev
.bus
= &nvdimm_bus_type
;
355 dev_set_name(&nvdimm_bus
->dev
, "ndbus%d", nvdimm_bus
->id
);
356 rc
= device_register(&nvdimm_bus
->dev
);
358 dev_dbg(&nvdimm_bus
->dev
, "registration failed: %d\n", rc
);
364 put_device(&nvdimm_bus
->dev
);
367 EXPORT_SYMBOL_GPL(nvdimm_bus_register
);
369 void nvdimm_bus_unregister(struct nvdimm_bus
*nvdimm_bus
)
373 device_unregister(&nvdimm_bus
->dev
);
375 EXPORT_SYMBOL_GPL(nvdimm_bus_unregister
);
377 static int child_unregister(struct device
*dev
, void *data
)
380 * the singular ndctl class device per bus needs to be
381 * "device_destroy"ed, so skip it here
383 * i.e. remove classless children
388 nd_device_unregister(dev
, ND_SYNC
);
392 static void free_poison_list(struct list_head
*poison_list
)
394 struct nd_poison
*pl
, *next
;
396 list_for_each_entry_safe(pl
, next
, poison_list
, list
) {
400 list_del_init(poison_list
);
403 static int nd_bus_remove(struct device
*dev
)
405 struct nvdimm_bus
*nvdimm_bus
= to_nvdimm_bus(dev
);
407 mutex_lock(&nvdimm_bus_list_mutex
);
408 list_del_init(&nvdimm_bus
->list
);
409 mutex_unlock(&nvdimm_bus_list_mutex
);
412 device_for_each_child(&nvdimm_bus
->dev
, NULL
, child_unregister
);
414 spin_lock(&nvdimm_bus
->poison_lock
);
415 free_poison_list(&nvdimm_bus
->poison_list
);
416 spin_unlock(&nvdimm_bus
->poison_lock
);
418 nvdimm_bus_destroy_ndctl(nvdimm_bus
);
423 static int nd_bus_probe(struct device
*dev
)
425 struct nvdimm_bus
*nvdimm_bus
= to_nvdimm_bus(dev
);
428 rc
= nvdimm_bus_create_ndctl(nvdimm_bus
);
432 mutex_lock(&nvdimm_bus_list_mutex
);
433 list_add_tail(&nvdimm_bus
->list
, &nvdimm_bus_list
);
434 mutex_unlock(&nvdimm_bus_list_mutex
);
436 /* enable bus provider attributes to look up their local context */
437 dev_set_drvdata(dev
, nvdimm_bus
->nd_desc
);
442 static struct nd_device_driver nd_bus_driver
= {
443 .probe
= nd_bus_probe
,
444 .remove
= nd_bus_remove
,
447 .suppress_bind_attrs
= true,
448 .bus
= &nvdimm_bus_type
,
449 .owner
= THIS_MODULE
,
450 .mod_name
= KBUILD_MODNAME
,
454 static int nvdimm_bus_match(struct device
*dev
, struct device_driver
*drv
)
456 struct nd_device_driver
*nd_drv
= to_nd_device_driver(drv
);
458 if (is_nvdimm_bus(dev
) && nd_drv
== &nd_bus_driver
)
461 return !!test_bit(to_nd_device_type(dev
), &nd_drv
->type
);
464 static ASYNC_DOMAIN_EXCLUSIVE(nd_async_domain
);
466 void nd_synchronize(void)
468 async_synchronize_full_domain(&nd_async_domain
);
470 EXPORT_SYMBOL_GPL(nd_synchronize
);
472 static void nd_async_device_register(void *d
, async_cookie_t cookie
)
474 struct device
*dev
= d
;
476 if (device_add(dev
) != 0) {
477 dev_err(dev
, "%s: failed\n", __func__
);
483 static void nd_async_device_unregister(void *d
, async_cookie_t cookie
)
485 struct device
*dev
= d
;
487 /* flush bus operations before delete */
488 nvdimm_bus_lock(dev
);
489 nvdimm_bus_unlock(dev
);
491 device_unregister(dev
);
495 void __nd_device_register(struct device
*dev
)
499 dev
->bus
= &nvdimm_bus_type
;
501 async_schedule_domain(nd_async_device_register
, dev
,
505 void nd_device_register(struct device
*dev
)
507 device_initialize(dev
);
508 __nd_device_register(dev
);
510 EXPORT_SYMBOL(nd_device_register
);
512 void nd_device_unregister(struct device
*dev
, enum nd_async_mode mode
)
517 async_schedule_domain(nd_async_device_unregister
, dev
,
522 device_unregister(dev
);
526 EXPORT_SYMBOL(nd_device_unregister
);
529 * __nd_driver_register() - register a region or a namespace driver
530 * @nd_drv: driver to register
531 * @owner: automatically set by nd_driver_register() macro
532 * @mod_name: automatically set by nd_driver_register() macro
534 int __nd_driver_register(struct nd_device_driver
*nd_drv
, struct module
*owner
,
535 const char *mod_name
)
537 struct device_driver
*drv
= &nd_drv
->drv
;
540 pr_debug("driver type bitmask not set (%pf)\n",
541 __builtin_return_address(0));
545 if (!nd_drv
->probe
) {
546 pr_debug("%s ->probe() must be specified\n", mod_name
);
550 drv
->bus
= &nvdimm_bus_type
;
552 drv
->mod_name
= mod_name
;
554 return driver_register(drv
);
556 EXPORT_SYMBOL(__nd_driver_register
);
558 int nvdimm_revalidate_disk(struct gendisk
*disk
)
560 struct device
*dev
= disk_to_dev(disk
)->parent
;
561 struct nd_region
*nd_region
= to_nd_region(dev
->parent
);
562 const char *pol
= nd_region
->ro
? "only" : "write";
564 if (nd_region
->ro
== get_disk_ro(disk
))
567 dev_info(dev
, "%s read-%s, marking %s read-%s\n",
568 dev_name(&nd_region
->dev
), pol
, disk
->disk_name
, pol
);
569 set_disk_ro(disk
, nd_region
->ro
);
574 EXPORT_SYMBOL(nvdimm_revalidate_disk
);
576 static ssize_t
modalias_show(struct device
*dev
, struct device_attribute
*attr
,
579 return sprintf(buf
, ND_DEVICE_MODALIAS_FMT
"\n",
580 to_nd_device_type(dev
));
582 static DEVICE_ATTR_RO(modalias
);
584 static ssize_t
devtype_show(struct device
*dev
, struct device_attribute
*attr
,
587 return sprintf(buf
, "%s\n", dev
->type
->name
);
589 static DEVICE_ATTR_RO(devtype
);
591 static struct attribute
*nd_device_attributes
[] = {
592 &dev_attr_modalias
.attr
,
593 &dev_attr_devtype
.attr
,
598 * nd_device_attribute_group - generic attributes for all devices on an nd bus
600 struct attribute_group nd_device_attribute_group
= {
601 .attrs
= nd_device_attributes
,
603 EXPORT_SYMBOL_GPL(nd_device_attribute_group
);
605 static ssize_t
numa_node_show(struct device
*dev
,
606 struct device_attribute
*attr
, char *buf
)
608 return sprintf(buf
, "%d\n", dev_to_node(dev
));
610 static DEVICE_ATTR_RO(numa_node
);
612 static struct attribute
*nd_numa_attributes
[] = {
613 &dev_attr_numa_node
.attr
,
617 static umode_t
nd_numa_attr_visible(struct kobject
*kobj
, struct attribute
*a
,
620 if (!IS_ENABLED(CONFIG_NUMA
))
627 * nd_numa_attribute_group - NUMA attributes for all devices on an nd bus
629 struct attribute_group nd_numa_attribute_group
= {
630 .attrs
= nd_numa_attributes
,
631 .is_visible
= nd_numa_attr_visible
,
633 EXPORT_SYMBOL_GPL(nd_numa_attribute_group
);
635 int nvdimm_bus_create_ndctl(struct nvdimm_bus
*nvdimm_bus
)
637 dev_t devt
= MKDEV(nvdimm_bus_major
, nvdimm_bus
->id
);
640 dev
= device_create(nd_class
, &nvdimm_bus
->dev
, devt
, nvdimm_bus
,
641 "ndctl%d", nvdimm_bus
->id
);
644 dev_dbg(&nvdimm_bus
->dev
, "failed to register ndctl%d: %ld\n",
645 nvdimm_bus
->id
, PTR_ERR(dev
));
646 return PTR_ERR_OR_ZERO(dev
);
649 void nvdimm_bus_destroy_ndctl(struct nvdimm_bus
*nvdimm_bus
)
651 device_destroy(nd_class
, MKDEV(nvdimm_bus_major
, nvdimm_bus
->id
));
654 static const struct nd_cmd_desc __nd_cmd_dimm_descs
[] = {
655 [ND_CMD_IMPLEMENTED
] = { },
658 .out_sizes
= { 4, 128, },
660 [ND_CMD_SMART_THRESHOLD
] = {
662 .out_sizes
= { 4, 8, },
664 [ND_CMD_DIMM_FLAGS
] = {
666 .out_sizes
= { 4, 4 },
668 [ND_CMD_GET_CONFIG_SIZE
] = {
670 .out_sizes
= { 4, 4, 4, },
672 [ND_CMD_GET_CONFIG_DATA
] = {
674 .in_sizes
= { 4, 4, },
676 .out_sizes
= { 4, UINT_MAX
, },
678 [ND_CMD_SET_CONFIG_DATA
] = {
680 .in_sizes
= { 4, 4, UINT_MAX
, },
686 .in_sizes
= { 4, 4, UINT_MAX
, },
688 .out_sizes
= { 4, 4, UINT_MAX
, },
692 .in_sizes
= { sizeof(struct nd_cmd_pkg
), UINT_MAX
, },
694 .out_sizes
= { UINT_MAX
, },
698 const struct nd_cmd_desc
*nd_cmd_dimm_desc(int cmd
)
700 if (cmd
< ARRAY_SIZE(__nd_cmd_dimm_descs
))
701 return &__nd_cmd_dimm_descs
[cmd
];
704 EXPORT_SYMBOL_GPL(nd_cmd_dimm_desc
);
706 static const struct nd_cmd_desc __nd_cmd_bus_descs
[] = {
707 [ND_CMD_IMPLEMENTED
] = { },
710 .in_sizes
= { 8, 8, },
712 .out_sizes
= { 4, 4, 4, 4, },
714 [ND_CMD_ARS_START
] = {
716 .in_sizes
= { 8, 8, 2, 1, 5, },
718 .out_sizes
= { 4, 4, },
720 [ND_CMD_ARS_STATUS
] = {
722 .out_sizes
= { 4, 4, UINT_MAX
, },
724 [ND_CMD_CLEAR_ERROR
] = {
726 .in_sizes
= { 8, 8, },
728 .out_sizes
= { 4, 4, 8, },
732 .in_sizes
= { sizeof(struct nd_cmd_pkg
), UINT_MAX
, },
734 .out_sizes
= { UINT_MAX
, },
738 const struct nd_cmd_desc
*nd_cmd_bus_desc(int cmd
)
740 if (cmd
< ARRAY_SIZE(__nd_cmd_bus_descs
))
741 return &__nd_cmd_bus_descs
[cmd
];
744 EXPORT_SYMBOL_GPL(nd_cmd_bus_desc
);
746 u32
nd_cmd_in_size(struct nvdimm
*nvdimm
, int cmd
,
747 const struct nd_cmd_desc
*desc
, int idx
, void *buf
)
749 if (idx
>= desc
->in_num
)
752 if (desc
->in_sizes
[idx
] < UINT_MAX
)
753 return desc
->in_sizes
[idx
];
755 if (nvdimm
&& cmd
== ND_CMD_SET_CONFIG_DATA
&& idx
== 2) {
756 struct nd_cmd_set_config_hdr
*hdr
= buf
;
758 return hdr
->in_length
;
759 } else if (nvdimm
&& cmd
== ND_CMD_VENDOR
&& idx
== 2) {
760 struct nd_cmd_vendor_hdr
*hdr
= buf
;
762 return hdr
->in_length
;
763 } else if (cmd
== ND_CMD_CALL
) {
764 struct nd_cmd_pkg
*pkg
= buf
;
766 return pkg
->nd_size_in
;
771 EXPORT_SYMBOL_GPL(nd_cmd_in_size
);
773 u32
nd_cmd_out_size(struct nvdimm
*nvdimm
, int cmd
,
774 const struct nd_cmd_desc
*desc
, int idx
, const u32
*in_field
,
775 const u32
*out_field
, unsigned long remainder
)
777 if (idx
>= desc
->out_num
)
780 if (desc
->out_sizes
[idx
] < UINT_MAX
)
781 return desc
->out_sizes
[idx
];
783 if (nvdimm
&& cmd
== ND_CMD_GET_CONFIG_DATA
&& idx
== 1)
785 else if (nvdimm
&& cmd
== ND_CMD_VENDOR
&& idx
== 2)
787 else if (!nvdimm
&& cmd
== ND_CMD_ARS_STATUS
&& idx
== 2) {
789 * Per table 9-276 ARS Data in ACPI 6.1, out_field[1] is
790 * "Size of Output Buffer in bytes, including this
793 if (out_field
[1] < 4)
796 * ACPI 6.1 is ambiguous if 'status' is included in the
797 * output size. If we encounter an output size that
798 * overshoots the remainder by 4 bytes, assume it was
799 * including 'status'.
801 if (out_field
[1] - 8 == remainder
)
803 return out_field
[1] - 4;
804 } else if (cmd
== ND_CMD_CALL
) {
805 struct nd_cmd_pkg
*pkg
= (struct nd_cmd_pkg
*) in_field
;
807 return pkg
->nd_size_out
;
813 EXPORT_SYMBOL_GPL(nd_cmd_out_size
);
815 void wait_nvdimm_bus_probe_idle(struct device
*dev
)
817 struct nvdimm_bus
*nvdimm_bus
= walk_to_nvdimm_bus(dev
);
820 if (nvdimm_bus
->probe_active
== 0)
822 nvdimm_bus_unlock(&nvdimm_bus
->dev
);
823 wait_event(nvdimm_bus
->probe_wait
,
824 nvdimm_bus
->probe_active
== 0);
825 nvdimm_bus_lock(&nvdimm_bus
->dev
);
829 static int nd_pmem_forget_poison_check(struct device
*dev
, void *data
)
831 struct nd_cmd_clear_error
*clear_err
=
832 (struct nd_cmd_clear_error
*)data
;
833 struct nd_btt
*nd_btt
= is_nd_btt(dev
) ? to_nd_btt(dev
) : NULL
;
834 struct nd_pfn
*nd_pfn
= is_nd_pfn(dev
) ? to_nd_pfn(dev
) : NULL
;
835 struct nd_dax
*nd_dax
= is_nd_dax(dev
) ? to_nd_dax(dev
) : NULL
;
836 struct nd_namespace_common
*ndns
= NULL
;
837 struct nd_namespace_io
*nsio
;
838 resource_size_t offset
= 0, end_trunc
= 0, start
, end
, pstart
, pend
;
840 if (nd_dax
|| !dev
->driver
)
843 start
= clear_err
->address
;
844 end
= clear_err
->address
+ clear_err
->cleared
- 1;
846 if (nd_btt
|| nd_pfn
|| nd_dax
) {
852 ndns
= nd_dax
->nd_pfn
.ndns
;
859 nsio
= to_nd_namespace_io(&ndns
->dev
);
860 pstart
= nsio
->res
.start
+ offset
;
861 pend
= nsio
->res
.end
- end_trunc
;
863 if ((pstart
>= start
) && (pend
<= end
))
870 static int nd_ns_forget_poison_check(struct device
*dev
, void *data
)
872 return device_for_each_child(dev
, data
, nd_pmem_forget_poison_check
);
875 /* set_config requires an idle interleave set */
876 static int nd_cmd_clear_to_send(struct nvdimm_bus
*nvdimm_bus
,
877 struct nvdimm
*nvdimm
, unsigned int cmd
, void *data
)
879 struct nvdimm_bus_descriptor
*nd_desc
= nvdimm_bus
->nd_desc
;
881 /* ask the bus provider if it would like to block this request */
882 if (nd_desc
->clear_to_send
) {
883 int rc
= nd_desc
->clear_to_send(nd_desc
, nvdimm
, cmd
);
889 /* require clear error to go through the pmem driver */
890 if (!nvdimm
&& cmd
== ND_CMD_CLEAR_ERROR
)
891 return device_for_each_child(&nvdimm_bus
->dev
, data
,
892 nd_ns_forget_poison_check
);
894 if (!nvdimm
|| cmd
!= ND_CMD_SET_CONFIG_DATA
)
897 /* prevent label manipulation while the kernel owns label updates */
898 wait_nvdimm_bus_probe_idle(&nvdimm_bus
->dev
);
899 if (atomic_read(&nvdimm
->busy
))
904 static int __nd_ioctl(struct nvdimm_bus
*nvdimm_bus
, struct nvdimm
*nvdimm
,
905 int read_only
, unsigned int ioctl_cmd
, unsigned long arg
)
907 struct nvdimm_bus_descriptor
*nd_desc
= nvdimm_bus
->nd_desc
;
908 size_t buf_len
= 0, in_len
= 0, out_len
= 0;
909 static char out_env
[ND_CMD_MAX_ENVELOPE
];
910 static char in_env
[ND_CMD_MAX_ENVELOPE
];
911 const struct nd_cmd_desc
*desc
= NULL
;
912 unsigned int cmd
= _IOC_NR(ioctl_cmd
);
913 unsigned int func
= cmd
;
914 void __user
*p
= (void __user
*) arg
;
915 struct device
*dev
= &nvdimm_bus
->dev
;
916 struct nd_cmd_pkg pkg
;
917 const char *cmd_name
, *dimm_name
;
918 unsigned long cmd_mask
;
923 desc
= nd_cmd_dimm_desc(cmd
);
924 cmd_name
= nvdimm_cmd_name(cmd
);
925 cmd_mask
= nvdimm
->cmd_mask
;
926 dimm_name
= dev_name(&nvdimm
->dev
);
928 desc
= nd_cmd_bus_desc(cmd
);
929 cmd_name
= nvdimm_bus_cmd_name(cmd
);
930 cmd_mask
= nd_desc
->cmd_mask
;
934 if (cmd
== ND_CMD_CALL
) {
935 if (copy_from_user(&pkg
, p
, sizeof(pkg
)))
939 if (!desc
|| (desc
->out_num
+ desc
->in_num
== 0) ||
940 !test_bit(cmd
, &cmd_mask
))
943 /* fail write commands (when read-only) */
947 case ND_CMD_SET_CONFIG_DATA
:
948 case ND_CMD_ARS_START
:
949 case ND_CMD_CLEAR_ERROR
:
951 dev_dbg(&nvdimm_bus
->dev
, "'%s' command while read-only.\n",
952 nvdimm
? nvdimm_cmd_name(cmd
)
953 : nvdimm_bus_cmd_name(cmd
));
959 /* process an input envelope */
960 for (i
= 0; i
< desc
->in_num
; i
++) {
963 in_size
= nd_cmd_in_size(nvdimm
, cmd
, desc
, i
, in_env
);
964 if (in_size
== UINT_MAX
) {
965 dev_err(dev
, "%s:%s unknown input size cmd: %s field: %d\n",
966 __func__
, dimm_name
, cmd_name
, i
);
969 if (in_len
< sizeof(in_env
))
970 copy
= min_t(u32
, sizeof(in_env
) - in_len
, in_size
);
973 if (copy
&& copy_from_user(&in_env
[in_len
], p
+ in_len
, copy
))
978 if (cmd
== ND_CMD_CALL
) {
979 func
= pkg
.nd_command
;
980 dev_dbg(dev
, "%s:%s, idx: %llu, in: %zu, out: %zu, len %zu\n",
981 __func__
, dimm_name
, pkg
.nd_command
,
982 in_len
, out_len
, buf_len
);
984 for (i
= 0; i
< ARRAY_SIZE(pkg
.nd_reserved2
); i
++)
985 if (pkg
.nd_reserved2
[i
])
989 /* process an output envelope */
990 for (i
= 0; i
< desc
->out_num
; i
++) {
991 u32 out_size
= nd_cmd_out_size(nvdimm
, cmd
, desc
, i
,
992 (u32
*) in_env
, (u32
*) out_env
, 0);
995 if (out_size
== UINT_MAX
) {
996 dev_dbg(dev
, "%s:%s unknown output size cmd: %s field: %d\n",
997 __func__
, dimm_name
, cmd_name
, i
);
1000 if (out_len
< sizeof(out_env
))
1001 copy
= min_t(u32
, sizeof(out_env
) - out_len
, out_size
);
1004 if (copy
&& copy_from_user(&out_env
[out_len
],
1005 p
+ in_len
+ out_len
, copy
))
1007 out_len
+= out_size
;
1010 buf_len
= out_len
+ in_len
;
1011 if (buf_len
> ND_IOCTL_MAX_BUFLEN
) {
1012 dev_dbg(dev
, "%s:%s cmd: %s buf_len: %zu > %d\n", __func__
,
1013 dimm_name
, cmd_name
, buf_len
,
1014 ND_IOCTL_MAX_BUFLEN
);
1018 buf
= vmalloc(buf_len
);
1022 if (copy_from_user(buf
, p
, buf_len
)) {
1027 nvdimm_bus_lock(&nvdimm_bus
->dev
);
1028 rc
= nd_cmd_clear_to_send(nvdimm_bus
, nvdimm
, func
, buf
);
1032 rc
= nd_desc
->ndctl(nd_desc
, nvdimm
, cmd
, buf
, buf_len
, &cmd_rc
);
1036 if (!nvdimm
&& cmd
== ND_CMD_CLEAR_ERROR
&& cmd_rc
>= 0) {
1037 struct nd_cmd_clear_error
*clear_err
= buf
;
1039 nvdimm_account_cleared_poison(nvdimm_bus
, clear_err
->address
,
1040 clear_err
->cleared
);
1042 nvdimm_bus_unlock(&nvdimm_bus
->dev
);
1044 if (copy_to_user(p
, buf
, buf_len
))
1051 nvdimm_bus_unlock(&nvdimm_bus
->dev
);
1057 static long nd_ioctl(struct file
*file
, unsigned int cmd
, unsigned long arg
)
1059 long id
= (long) file
->private_data
;
1060 int rc
= -ENXIO
, ro
;
1061 struct nvdimm_bus
*nvdimm_bus
;
1063 ro
= ((file
->f_flags
& O_ACCMODE
) == O_RDONLY
);
1064 mutex_lock(&nvdimm_bus_list_mutex
);
1065 list_for_each_entry(nvdimm_bus
, &nvdimm_bus_list
, list
) {
1066 if (nvdimm_bus
->id
== id
) {
1067 rc
= __nd_ioctl(nvdimm_bus
, NULL
, ro
, cmd
, arg
);
1071 mutex_unlock(&nvdimm_bus_list_mutex
);
1076 static int match_dimm(struct device
*dev
, void *data
)
1078 long id
= (long) data
;
1080 if (is_nvdimm(dev
)) {
1081 struct nvdimm
*nvdimm
= to_nvdimm(dev
);
1083 return nvdimm
->id
== id
;
1089 static long nvdimm_ioctl(struct file
*file
, unsigned int cmd
, unsigned long arg
)
1091 int rc
= -ENXIO
, ro
;
1092 struct nvdimm_bus
*nvdimm_bus
;
1094 ro
= ((file
->f_flags
& O_ACCMODE
) == O_RDONLY
);
1095 mutex_lock(&nvdimm_bus_list_mutex
);
1096 list_for_each_entry(nvdimm_bus
, &nvdimm_bus_list
, list
) {
1097 struct device
*dev
= device_find_child(&nvdimm_bus
->dev
,
1098 file
->private_data
, match_dimm
);
1099 struct nvdimm
*nvdimm
;
1104 nvdimm
= to_nvdimm(dev
);
1105 rc
= __nd_ioctl(nvdimm_bus
, nvdimm
, ro
, cmd
, arg
);
1109 mutex_unlock(&nvdimm_bus_list_mutex
);
1114 static int nd_open(struct inode
*inode
, struct file
*file
)
1116 long minor
= iminor(inode
);
1118 file
->private_data
= (void *) minor
;
1122 static const struct file_operations nvdimm_bus_fops
= {
1123 .owner
= THIS_MODULE
,
1125 .unlocked_ioctl
= nd_ioctl
,
1126 .compat_ioctl
= nd_ioctl
,
1127 .llseek
= noop_llseek
,
1130 static const struct file_operations nvdimm_fops
= {
1131 .owner
= THIS_MODULE
,
1133 .unlocked_ioctl
= nvdimm_ioctl
,
1134 .compat_ioctl
= nvdimm_ioctl
,
1135 .llseek
= noop_llseek
,
1138 int __init
nvdimm_bus_init(void)
1142 BUILD_BUG_ON(sizeof(struct nd_smart_payload
) != 128);
1143 BUILD_BUG_ON(sizeof(struct nd_smart_threshold_payload
) != 8);
1145 rc
= bus_register(&nvdimm_bus_type
);
1149 rc
= register_chrdev(0, "ndctl", &nvdimm_bus_fops
);
1151 goto err_bus_chrdev
;
1152 nvdimm_bus_major
= rc
;
1154 rc
= register_chrdev(0, "dimmctl", &nvdimm_fops
);
1156 goto err_dimm_chrdev
;
1159 nd_class
= class_create(THIS_MODULE
, "nd");
1160 if (IS_ERR(nd_class
)) {
1161 rc
= PTR_ERR(nd_class
);
1165 rc
= driver_register(&nd_bus_driver
.drv
);
1172 class_destroy(nd_class
);
1174 unregister_chrdev(nvdimm_major
, "dimmctl");
1176 unregister_chrdev(nvdimm_bus_major
, "ndctl");
1178 bus_unregister(&nvdimm_bus_type
);
1183 void nvdimm_bus_exit(void)
1185 driver_unregister(&nd_bus_driver
.drv
);
1186 class_destroy(nd_class
);
1187 unregister_chrdev(nvdimm_bus_major
, "ndctl");
1188 unregister_chrdev(nvdimm_major
, "dimmctl");
1189 bus_unregister(&nvdimm_bus_type
);
1190 ida_destroy(&nd_ida
);