1 // SPDX-License-Identifier: GPL-2.0
3 * Memory subsystem support
5 * Written by Matt Tolentino <matthew.e.tolentino@intel.com>
6 * Dave Hansen <haveblue@us.ibm.com>
8 * This file provides the necessary infrastructure to represent
9 * a SPARSEMEM-memory-model system's physical memory in /sysfs.
10 * All arch-independent code that assumes MEMORY_HOTPLUG requires
11 * SPARSEMEM should be contained here, or in mm/memory_hotplug.c.
14 #include <linux/module.h>
15 #include <linux/init.h>
16 #include <linux/topology.h>
17 #include <linux/capability.h>
18 #include <linux/device.h>
19 #include <linux/memory.h>
20 #include <linux/memory_hotplug.h>
22 #include <linux/mutex.h>
23 #include <linux/stat.h>
24 #include <linux/slab.h>
26 #include <linux/atomic.h>
27 #include <linux/uaccess.h>
29 static DEFINE_MUTEX(mem_sysfs_mutex
);
31 #define MEMORY_CLASS_NAME "memory"
33 #define to_memory_block(dev) container_of(dev, struct memory_block, dev)
35 static int sections_per_block
;
37 static inline int base_memory_block_id(int section_nr
)
39 return section_nr
/ sections_per_block
;
42 static int memory_subsys_online(struct device
*dev
);
43 static int memory_subsys_offline(struct device
*dev
);
45 static struct bus_type memory_subsys
= {
46 .name
= MEMORY_CLASS_NAME
,
47 .dev_name
= MEMORY_CLASS_NAME
,
48 .online
= memory_subsys_online
,
49 .offline
= memory_subsys_offline
,
52 static BLOCKING_NOTIFIER_HEAD(memory_chain
);
54 int register_memory_notifier(struct notifier_block
*nb
)
56 return blocking_notifier_chain_register(&memory_chain
, nb
);
58 EXPORT_SYMBOL(register_memory_notifier
);
60 void unregister_memory_notifier(struct notifier_block
*nb
)
62 blocking_notifier_chain_unregister(&memory_chain
, nb
);
64 EXPORT_SYMBOL(unregister_memory_notifier
);
66 static ATOMIC_NOTIFIER_HEAD(memory_isolate_chain
);
68 int register_memory_isolate_notifier(struct notifier_block
*nb
)
70 return atomic_notifier_chain_register(&memory_isolate_chain
, nb
);
72 EXPORT_SYMBOL(register_memory_isolate_notifier
);
74 void unregister_memory_isolate_notifier(struct notifier_block
*nb
)
76 atomic_notifier_chain_unregister(&memory_isolate_chain
, nb
);
78 EXPORT_SYMBOL(unregister_memory_isolate_notifier
);
80 static void memory_block_release(struct device
*dev
)
82 struct memory_block
*mem
= to_memory_block(dev
);
87 unsigned long __weak
memory_block_size_bytes(void)
89 return MIN_MEMORY_BLOCK_SIZE
;
92 static unsigned long get_memory_block_size(void)
94 unsigned long block_sz
;
96 block_sz
= memory_block_size_bytes();
98 /* Validate blk_sz is a power of 2 and not less than section size */
99 if ((block_sz
& (block_sz
- 1)) || (block_sz
< MIN_MEMORY_BLOCK_SIZE
)) {
101 block_sz
= MIN_MEMORY_BLOCK_SIZE
;
108 * use this as the physical section index that this memsection
112 static ssize_t
show_mem_start_phys_index(struct device
*dev
,
113 struct device_attribute
*attr
, char *buf
)
115 struct memory_block
*mem
= to_memory_block(dev
);
116 unsigned long phys_index
;
118 phys_index
= mem
->start_section_nr
/ sections_per_block
;
119 return sprintf(buf
, "%08lx\n", phys_index
);
123 * Show whether the section of memory is likely to be hot-removable
125 static ssize_t
show_mem_removable(struct device
*dev
,
126 struct device_attribute
*attr
, char *buf
)
128 unsigned long i
, pfn
;
130 struct memory_block
*mem
= to_memory_block(dev
);
132 if (mem
->state
!= MEM_ONLINE
)
135 for (i
= 0; i
< sections_per_block
; i
++) {
136 if (!present_section_nr(mem
->start_section_nr
+ i
))
138 pfn
= section_nr_to_pfn(mem
->start_section_nr
+ i
);
139 ret
&= is_mem_section_removable(pfn
, PAGES_PER_SECTION
);
143 return sprintf(buf
, "%d\n", ret
);
147 * online, offline, going offline, etc.
149 static ssize_t
show_mem_state(struct device
*dev
,
150 struct device_attribute
*attr
, char *buf
)
152 struct memory_block
*mem
= to_memory_block(dev
);
156 * We can probably put these states in a nice little array
157 * so that they're not open-coded
159 switch (mem
->state
) {
161 len
= sprintf(buf
, "online\n");
164 len
= sprintf(buf
, "offline\n");
166 case MEM_GOING_OFFLINE
:
167 len
= sprintf(buf
, "going-offline\n");
170 len
= sprintf(buf
, "ERROR-UNKNOWN-%ld\n",
179 int memory_notify(unsigned long val
, void *v
)
181 return blocking_notifier_call_chain(&memory_chain
, val
, v
);
184 int memory_isolate_notify(unsigned long val
, void *v
)
186 return atomic_notifier_call_chain(&memory_isolate_chain
, val
, v
);
190 * The probe routines leave the pages reserved, just as the bootmem code does.
191 * Make sure they're still that way.
193 static bool pages_correctly_reserved(unsigned long start_pfn
)
197 unsigned long pfn
= start_pfn
;
200 * memmap between sections is not contiguous except with
201 * SPARSEMEM_VMEMMAP. We lookup the page once per section
202 * and assume memmap is contiguous within each section
204 for (i
= 0; i
< sections_per_block
; i
++, pfn
+= PAGES_PER_SECTION
) {
205 if (WARN_ON_ONCE(!pfn_valid(pfn
)))
207 page
= pfn_to_page(pfn
);
209 for (j
= 0; j
< PAGES_PER_SECTION
; j
++) {
210 if (PageReserved(page
+ j
))
213 printk(KERN_WARNING
"section number %ld page number %d "
214 "not reserved, was it already online?\n",
215 pfn_to_section_nr(pfn
), j
);
225 * MEMORY_HOTPLUG depends on SPARSEMEM in mm/Kconfig, so it is
226 * OK to have direct references to sparsemem variables in here.
227 * Must already be protected by mem_hotplug_begin().
230 memory_block_action(unsigned long phys_index
, unsigned long action
, int online_type
)
232 unsigned long start_pfn
;
233 unsigned long nr_pages
= PAGES_PER_SECTION
* sections_per_block
;
236 start_pfn
= section_nr_to_pfn(phys_index
);
240 if (!pages_correctly_reserved(start_pfn
))
243 ret
= online_pages(start_pfn
, nr_pages
, online_type
);
246 ret
= offline_pages(start_pfn
, nr_pages
);
249 WARN(1, KERN_WARNING
"%s(%ld, %ld) unknown action: "
250 "%ld\n", __func__
, phys_index
, action
, action
);
257 static int memory_block_change_state(struct memory_block
*mem
,
258 unsigned long to_state
, unsigned long from_state_req
)
262 if (mem
->state
!= from_state_req
)
265 if (to_state
== MEM_OFFLINE
)
266 mem
->state
= MEM_GOING_OFFLINE
;
268 ret
= memory_block_action(mem
->start_section_nr
, to_state
,
271 mem
->state
= ret
? from_state_req
: to_state
;
276 /* The device lock serializes operations on memory_subsys_[online|offline] */
277 static int memory_subsys_online(struct device
*dev
)
279 struct memory_block
*mem
= to_memory_block(dev
);
282 if (mem
->state
== MEM_ONLINE
)
286 * If we are called from store_mem_state(), online_type will be
287 * set >= 0 Otherwise we were called from the device online
288 * attribute and need to set the online_type.
290 if (mem
->online_type
< 0)
291 mem
->online_type
= MMOP_ONLINE_KEEP
;
293 /* Already under protection of mem_hotplug_begin() */
294 ret
= memory_block_change_state(mem
, MEM_ONLINE
, MEM_OFFLINE
);
296 /* clear online_type */
297 mem
->online_type
= -1;
302 static int memory_subsys_offline(struct device
*dev
)
304 struct memory_block
*mem
= to_memory_block(dev
);
306 if (mem
->state
== MEM_OFFLINE
)
309 /* Can't offline block with non-present sections */
310 if (mem
->section_count
!= sections_per_block
)
313 return memory_block_change_state(mem
, MEM_OFFLINE
, MEM_ONLINE
);
317 store_mem_state(struct device
*dev
,
318 struct device_attribute
*attr
, const char *buf
, size_t count
)
320 struct memory_block
*mem
= to_memory_block(dev
);
321 int ret
, online_type
;
323 ret
= lock_device_hotplug_sysfs();
327 if (sysfs_streq(buf
, "online_kernel"))
328 online_type
= MMOP_ONLINE_KERNEL
;
329 else if (sysfs_streq(buf
, "online_movable"))
330 online_type
= MMOP_ONLINE_MOVABLE
;
331 else if (sysfs_streq(buf
, "online"))
332 online_type
= MMOP_ONLINE_KEEP
;
333 else if (sysfs_streq(buf
, "offline"))
334 online_type
= MMOP_OFFLINE
;
341 * Memory hotplug needs to hold mem_hotplug_begin() for probe to find
342 * the correct memory block to online before doing device_online(dev),
343 * which will take dev->mutex. Take the lock early to prevent an
344 * inversion, memory_subsys_online() callbacks will be implemented by
345 * assuming it's already protected.
349 switch (online_type
) {
350 case MMOP_ONLINE_KERNEL
:
351 case MMOP_ONLINE_MOVABLE
:
352 case MMOP_ONLINE_KEEP
:
353 mem
->online_type
= online_type
;
354 ret
= device_online(&mem
->dev
);
357 ret
= device_offline(&mem
->dev
);
360 ret
= -EINVAL
; /* should never happen */
365 unlock_device_hotplug();
376 * phys_device is a bad name for this. What I really want
377 * is a way to differentiate between memory ranges that
378 * are part of physical devices that constitute
379 * a complete removable unit or fru.
380 * i.e. do these ranges belong to the same physical device,
381 * s.t. if I offline all of these sections I can then
382 * remove the physical device?
384 static ssize_t
show_phys_device(struct device
*dev
,
385 struct device_attribute
*attr
, char *buf
)
387 struct memory_block
*mem
= to_memory_block(dev
);
388 return sprintf(buf
, "%d\n", mem
->phys_device
);
391 #ifdef CONFIG_MEMORY_HOTREMOVE
392 static void print_allowed_zone(char *buf
, int nid
, unsigned long start_pfn
,
393 unsigned long nr_pages
, int online_type
,
394 struct zone
*default_zone
)
398 zone
= zone_for_pfn_range(online_type
, nid
, start_pfn
, nr_pages
);
399 if (zone
!= default_zone
) {
401 strcat(buf
, zone
->name
);
405 static ssize_t
show_valid_zones(struct device
*dev
,
406 struct device_attribute
*attr
, char *buf
)
408 struct memory_block
*mem
= to_memory_block(dev
);
409 unsigned long start_pfn
= section_nr_to_pfn(mem
->start_section_nr
);
410 unsigned long nr_pages
= PAGES_PER_SECTION
* sections_per_block
;
411 unsigned long valid_start_pfn
, valid_end_pfn
;
412 struct zone
*default_zone
;
416 * The block contains more than one zone can not be offlined.
417 * This can happen e.g. for ZONE_DMA and ZONE_DMA32
419 if (!test_pages_in_a_zone(start_pfn
, start_pfn
+ nr_pages
, &valid_start_pfn
, &valid_end_pfn
))
420 return sprintf(buf
, "none\n");
422 start_pfn
= valid_start_pfn
;
423 nr_pages
= valid_end_pfn
- start_pfn
;
426 * Check the existing zone. Make sure that we do that only on the
427 * online nodes otherwise the page_zone is not reliable
429 if (mem
->state
== MEM_ONLINE
) {
430 strcat(buf
, page_zone(pfn_to_page(start_pfn
))->name
);
434 nid
= pfn_to_nid(start_pfn
);
435 default_zone
= zone_for_pfn_range(MMOP_ONLINE_KEEP
, nid
, start_pfn
, nr_pages
);
436 strcat(buf
, default_zone
->name
);
438 print_allowed_zone(buf
, nid
, start_pfn
, nr_pages
, MMOP_ONLINE_KERNEL
,
440 print_allowed_zone(buf
, nid
, start_pfn
, nr_pages
, MMOP_ONLINE_MOVABLE
,
447 static DEVICE_ATTR(valid_zones
, 0444, show_valid_zones
, NULL
);
450 static DEVICE_ATTR(phys_index
, 0444, show_mem_start_phys_index
, NULL
);
451 static DEVICE_ATTR(state
, 0644, show_mem_state
, store_mem_state
);
452 static DEVICE_ATTR(phys_device
, 0444, show_phys_device
, NULL
);
453 static DEVICE_ATTR(removable
, 0444, show_mem_removable
, NULL
);
456 * Block size attribute stuff
459 print_block_size(struct device
*dev
, struct device_attribute
*attr
,
462 return sprintf(buf
, "%lx\n", get_memory_block_size());
465 static DEVICE_ATTR(block_size_bytes
, 0444, print_block_size
, NULL
);
468 * Memory auto online policy.
472 show_auto_online_blocks(struct device
*dev
, struct device_attribute
*attr
,
475 if (memhp_auto_online
)
476 return sprintf(buf
, "online\n");
478 return sprintf(buf
, "offline\n");
482 store_auto_online_blocks(struct device
*dev
, struct device_attribute
*attr
,
483 const char *buf
, size_t count
)
485 if (sysfs_streq(buf
, "online"))
486 memhp_auto_online
= true;
487 else if (sysfs_streq(buf
, "offline"))
488 memhp_auto_online
= false;
495 static DEVICE_ATTR(auto_online_blocks
, 0644, show_auto_online_blocks
,
496 store_auto_online_blocks
);
499 * Some architectures will have custom drivers to do this, and
500 * will not need to do it from userspace. The fake hot-add code
501 * as well as ppc64 will do all of their discovery in userspace
502 * and will require this interface.
504 #ifdef CONFIG_ARCH_MEMORY_PROBE
506 memory_probe_store(struct device
*dev
, struct device_attribute
*attr
,
507 const char *buf
, size_t count
)
511 unsigned long pages_per_block
= PAGES_PER_SECTION
* sections_per_block
;
513 ret
= kstrtoull(buf
, 0, &phys_addr
);
517 if (phys_addr
& ((pages_per_block
<< PAGE_SHIFT
) - 1))
520 ret
= lock_device_hotplug_sysfs();
524 nid
= memory_add_physaddr_to_nid(phys_addr
);
525 ret
= __add_memory(nid
, phys_addr
,
526 MIN_MEMORY_BLOCK_SIZE
* sections_per_block
);
533 unlock_device_hotplug();
537 static DEVICE_ATTR(probe
, S_IWUSR
, NULL
, memory_probe_store
);
540 #ifdef CONFIG_MEMORY_FAILURE
542 * Support for offlining pages of memory
545 /* Soft offline a page */
547 store_soft_offline_page(struct device
*dev
,
548 struct device_attribute
*attr
,
549 const char *buf
, size_t count
)
553 if (!capable(CAP_SYS_ADMIN
))
555 if (kstrtoull(buf
, 0, &pfn
) < 0)
560 /* Only online pages can be soft-offlined (esp., not ZONE_DEVICE). */
561 if (!pfn_to_online_page(pfn
))
563 ret
= soft_offline_page(pfn_to_page(pfn
), 0);
564 return ret
== 0 ? count
: ret
;
567 /* Forcibly offline a page, including killing processes. */
569 store_hard_offline_page(struct device
*dev
,
570 struct device_attribute
*attr
,
571 const char *buf
, size_t count
)
575 if (!capable(CAP_SYS_ADMIN
))
577 if (kstrtoull(buf
, 0, &pfn
) < 0)
580 ret
= memory_failure(pfn
, 0, 0);
581 return ret
? ret
: count
;
584 static DEVICE_ATTR(soft_offline_page
, S_IWUSR
, NULL
, store_soft_offline_page
);
585 static DEVICE_ATTR(hard_offline_page
, S_IWUSR
, NULL
, store_hard_offline_page
);
589 * Note that phys_device is optional. It is here to allow for
590 * differentiation between which *physical* devices each
591 * section belongs to...
593 int __weak
arch_get_memory_phys_device(unsigned long start_pfn
)
599 * A reference for the returned object is held and the reference for the
600 * hinted object is released.
602 struct memory_block
*find_memory_block_hinted(struct mem_section
*section
,
603 struct memory_block
*hint
)
605 int block_id
= base_memory_block_id(__section_nr(section
));
606 struct device
*hintdev
= hint
? &hint
->dev
: NULL
;
609 dev
= subsys_find_device_by_id(&memory_subsys
, block_id
, hintdev
);
611 put_device(&hint
->dev
);
614 return to_memory_block(dev
);
618 * For now, we have a linear search to go find the appropriate
619 * memory_block corresponding to a particular phys_index. If
620 * this gets to be a real problem, we can always use a radix
621 * tree or something here.
623 * This could be made generic for all device subsystems.
625 struct memory_block
*find_memory_block(struct mem_section
*section
)
627 return find_memory_block_hinted(section
, NULL
);
630 static struct attribute
*memory_memblk_attrs
[] = {
631 &dev_attr_phys_index
.attr
,
632 &dev_attr_state
.attr
,
633 &dev_attr_phys_device
.attr
,
634 &dev_attr_removable
.attr
,
635 #ifdef CONFIG_MEMORY_HOTREMOVE
636 &dev_attr_valid_zones
.attr
,
641 static struct attribute_group memory_memblk_attr_group
= {
642 .attrs
= memory_memblk_attrs
,
645 static const struct attribute_group
*memory_memblk_attr_groups
[] = {
646 &memory_memblk_attr_group
,
651 * register_memory - Setup a sysfs device for a memory block
654 int register_memory(struct memory_block
*memory
)
656 memory
->dev
.bus
= &memory_subsys
;
657 memory
->dev
.id
= memory
->start_section_nr
/ sections_per_block
;
658 memory
->dev
.release
= memory_block_release
;
659 memory
->dev
.groups
= memory_memblk_attr_groups
;
660 memory
->dev
.offline
= memory
->state
== MEM_OFFLINE
;
662 return device_register(&memory
->dev
);
665 static int init_memory_block(struct memory_block
**memory
,
666 struct mem_section
*section
, unsigned long state
)
668 struct memory_block
*mem
;
669 unsigned long start_pfn
;
673 mem
= kzalloc(sizeof(*mem
), GFP_KERNEL
);
677 scn_nr
= __section_nr(section
);
678 mem
->start_section_nr
=
679 base_memory_block_id(scn_nr
) * sections_per_block
;
680 mem
->end_section_nr
= mem
->start_section_nr
+ sections_per_block
- 1;
682 start_pfn
= section_nr_to_pfn(mem
->start_section_nr
);
683 mem
->phys_device
= arch_get_memory_phys_device(start_pfn
);
685 ret
= register_memory(mem
);
691 static int add_memory_block(int base_section_nr
)
693 struct memory_block
*mem
;
694 int i
, ret
, section_count
= 0, section_nr
;
696 for (i
= base_section_nr
;
697 (i
< base_section_nr
+ sections_per_block
) && i
< NR_MEM_SECTIONS
;
699 if (!present_section_nr(i
))
701 if (section_count
== 0)
706 if (section_count
== 0)
708 ret
= init_memory_block(&mem
, __nr_to_section(section_nr
), MEM_ONLINE
);
711 mem
->section_count
= section_count
;
716 * need an interface for the VM to add new memory regions,
717 * but without onlining it.
719 int register_new_memory(int nid
, struct mem_section
*section
)
722 struct memory_block
*mem
;
724 mutex_lock(&mem_sysfs_mutex
);
726 mem
= find_memory_block(section
);
728 mem
->section_count
++;
729 put_device(&mem
->dev
);
731 ret
= init_memory_block(&mem
, section
, MEM_OFFLINE
);
734 mem
->section_count
++;
737 if (mem
->section_count
== sections_per_block
)
738 ret
= register_mem_sect_under_node(mem
, nid
);
740 mutex_unlock(&mem_sysfs_mutex
);
744 #ifdef CONFIG_MEMORY_HOTREMOVE
746 unregister_memory(struct memory_block
*memory
)
748 BUG_ON(memory
->dev
.bus
!= &memory_subsys
);
750 /* drop the ref. we got in remove_memory_block() */
751 put_device(&memory
->dev
);
752 device_unregister(&memory
->dev
);
755 static int remove_memory_section(unsigned long node_id
,
756 struct mem_section
*section
, int phys_device
)
758 struct memory_block
*mem
;
760 mutex_lock(&mem_sysfs_mutex
);
763 * Some users of the memory hotplug do not want/need memblock to
764 * track all sections. Skip over those.
766 mem
= find_memory_block(section
);
770 unregister_mem_sect_under_nodes(mem
, __section_nr(section
));
772 mem
->section_count
--;
773 if (mem
->section_count
== 0)
774 unregister_memory(mem
);
776 put_device(&mem
->dev
);
779 mutex_unlock(&mem_sysfs_mutex
);
783 int unregister_memory_section(struct mem_section
*section
)
785 if (!present_section(section
))
788 return remove_memory_section(0, section
, 0);
790 #endif /* CONFIG_MEMORY_HOTREMOVE */
792 /* return true if the memory block is offlined, otherwise, return false */
793 bool is_memblock_offlined(struct memory_block
*mem
)
795 return mem
->state
== MEM_OFFLINE
;
798 static struct attribute
*memory_root_attrs
[] = {
799 #ifdef CONFIG_ARCH_MEMORY_PROBE
800 &dev_attr_probe
.attr
,
803 #ifdef CONFIG_MEMORY_FAILURE
804 &dev_attr_soft_offline_page
.attr
,
805 &dev_attr_hard_offline_page
.attr
,
808 &dev_attr_block_size_bytes
.attr
,
809 &dev_attr_auto_online_blocks
.attr
,
813 static struct attribute_group memory_root_attr_group
= {
814 .attrs
= memory_root_attrs
,
817 static const struct attribute_group
*memory_root_attr_groups
[] = {
818 &memory_root_attr_group
,
823 * Initialize the sysfs support for memory devices...
825 int __init
memory_dev_init(void)
830 unsigned long block_sz
;
832 ret
= subsys_system_register(&memory_subsys
, memory_root_attr_groups
);
836 block_sz
= get_memory_block_size();
837 sections_per_block
= block_sz
/ MIN_MEMORY_BLOCK_SIZE
;
840 * Create entries for memory sections that were found
841 * during boot and have been initialized
843 mutex_lock(&mem_sysfs_mutex
);
844 for (i
= 0; i
< NR_MEM_SECTIONS
; i
+= sections_per_block
) {
845 /* Don't iterate over sections we know are !present: */
846 if (i
> __highest_present_section_nr
)
849 err
= add_memory_block(i
);
853 mutex_unlock(&mem_sysfs_mutex
);
857 printk(KERN_ERR
"%s() failed: %d\n", __func__
, ret
);