2 * Memory subsystem support
4 * Written by Matt Tolentino <matthew.e.tolentino@intel.com>
5 * Dave Hansen <haveblue@us.ibm.com>
7 * This file provides the necessary infrastructure to represent
8 * a SPARSEMEM-memory-model system's physical memory in /sysfs.
9 * All arch-independent code that assumes MEMORY_HOTPLUG requires
10 * SPARSEMEM should be contained here, or in mm/memory_hotplug.c.
13 #include <linux/module.h>
14 #include <linux/init.h>
15 #include <linux/topology.h>
16 #include <linux/capability.h>
17 #include <linux/device.h>
18 #include <linux/memory.h>
19 #include <linux/kobject.h>
20 #include <linux/memory_hotplug.h>
22 #include <linux/mutex.h>
23 #include <linux/stat.h>
24 #include <linux/slab.h>
26 #include <linux/atomic.h>
27 #include <asm/uaccess.h>
29 static DEFINE_MUTEX(mem_sysfs_mutex
);
31 #define MEMORY_CLASS_NAME "memory"
33 static int sections_per_block
;
35 static inline int base_memory_block_id(int section_nr
)
37 return section_nr
/ sections_per_block
;
40 static struct bus_type memory_subsys
= {
41 .name
= MEMORY_CLASS_NAME
,
42 .dev_name
= MEMORY_CLASS_NAME
,
45 static BLOCKING_NOTIFIER_HEAD(memory_chain
);
47 int register_memory_notifier(struct notifier_block
*nb
)
49 return blocking_notifier_chain_register(&memory_chain
, nb
);
51 EXPORT_SYMBOL(register_memory_notifier
);
53 void unregister_memory_notifier(struct notifier_block
*nb
)
55 blocking_notifier_chain_unregister(&memory_chain
, nb
);
57 EXPORT_SYMBOL(unregister_memory_notifier
);
59 static ATOMIC_NOTIFIER_HEAD(memory_isolate_chain
);
61 int register_memory_isolate_notifier(struct notifier_block
*nb
)
63 return atomic_notifier_chain_register(&memory_isolate_chain
, nb
);
65 EXPORT_SYMBOL(register_memory_isolate_notifier
);
67 void unregister_memory_isolate_notifier(struct notifier_block
*nb
)
69 atomic_notifier_chain_unregister(&memory_isolate_chain
, nb
);
71 EXPORT_SYMBOL(unregister_memory_isolate_notifier
);
73 static void memory_block_release(struct device
*dev
)
75 struct memory_block
*mem
= container_of(dev
, struct memory_block
, dev
);
81 * register_memory - Setup a sysfs device for a memory block
84 int register_memory(struct memory_block
*memory
)
88 memory
->dev
.bus
= &memory_subsys
;
89 memory
->dev
.id
= memory
->start_section_nr
/ sections_per_block
;
90 memory
->dev
.release
= memory_block_release
;
92 error
= device_register(&memory
->dev
);
96 unsigned long __weak
memory_block_size_bytes(void)
98 return MIN_MEMORY_BLOCK_SIZE
;
101 static unsigned long get_memory_block_size(void)
103 unsigned long block_sz
;
105 block_sz
= memory_block_size_bytes();
107 /* Validate blk_sz is a power of 2 and not less than section size */
108 if ((block_sz
& (block_sz
- 1)) || (block_sz
< MIN_MEMORY_BLOCK_SIZE
)) {
110 block_sz
= MIN_MEMORY_BLOCK_SIZE
;
117 * use this as the physical section index that this memsection
121 static ssize_t
show_mem_start_phys_index(struct device
*dev
,
122 struct device_attribute
*attr
, char *buf
)
124 struct memory_block
*mem
=
125 container_of(dev
, struct memory_block
, dev
);
126 unsigned long phys_index
;
128 phys_index
= mem
->start_section_nr
/ sections_per_block
;
129 return sprintf(buf
, "%08lx\n", phys_index
);
132 static ssize_t
show_mem_end_phys_index(struct device
*dev
,
133 struct device_attribute
*attr
, char *buf
)
135 struct memory_block
*mem
=
136 container_of(dev
, struct memory_block
, dev
);
137 unsigned long phys_index
;
139 phys_index
= mem
->end_section_nr
/ sections_per_block
;
140 return sprintf(buf
, "%08lx\n", phys_index
);
144 * Show whether the section of memory is likely to be hot-removable
146 static ssize_t
show_mem_removable(struct device
*dev
,
147 struct device_attribute
*attr
, char *buf
)
149 unsigned long i
, pfn
;
151 struct memory_block
*mem
=
152 container_of(dev
, struct memory_block
, dev
);
154 for (i
= 0; i
< sections_per_block
; i
++) {
155 pfn
= section_nr_to_pfn(mem
->start_section_nr
+ i
);
156 ret
&= is_mem_section_removable(pfn
, PAGES_PER_SECTION
);
159 return sprintf(buf
, "%d\n", ret
);
163 * online, offline, going offline, etc.
165 static ssize_t
show_mem_state(struct device
*dev
,
166 struct device_attribute
*attr
, char *buf
)
168 struct memory_block
*mem
=
169 container_of(dev
, struct memory_block
, dev
);
173 * We can probably put these states in a nice little array
174 * so that they're not open-coded
176 switch (mem
->state
) {
178 len
= sprintf(buf
, "online\n");
181 len
= sprintf(buf
, "offline\n");
183 case MEM_GOING_OFFLINE
:
184 len
= sprintf(buf
, "going-offline\n");
187 len
= sprintf(buf
, "ERROR-UNKNOWN-%ld\n",
196 int memory_notify(unsigned long val
, void *v
)
198 return blocking_notifier_call_chain(&memory_chain
, val
, v
);
201 int memory_isolate_notify(unsigned long val
, void *v
)
203 return atomic_notifier_call_chain(&memory_isolate_chain
, val
, v
);
207 * The probe routines leave the pages reserved, just as the bootmem code does.
208 * Make sure they're still that way.
210 static bool pages_correctly_reserved(unsigned long start_pfn
)
214 unsigned long pfn
= start_pfn
;
217 * memmap between sections is not contiguous except with
218 * SPARSEMEM_VMEMMAP. We lookup the page once per section
219 * and assume memmap is contiguous within each section
221 for (i
= 0; i
< sections_per_block
; i
++, pfn
+= PAGES_PER_SECTION
) {
222 if (WARN_ON_ONCE(!pfn_valid(pfn
)))
224 page
= pfn_to_page(pfn
);
226 for (j
= 0; j
< PAGES_PER_SECTION
; j
++) {
227 if (PageReserved(page
+ j
))
230 printk(KERN_WARNING
"section number %ld page number %d "
231 "not reserved, was it already online?\n",
232 pfn_to_section_nr(pfn
), j
);
242 * MEMORY_HOTPLUG depends on SPARSEMEM in mm/Kconfig, so it is
243 * OK to have direct references to sparsemem variables in here.
246 memory_block_action(unsigned long phys_index
, unsigned long action
, int online_type
)
248 unsigned long start_pfn
;
249 unsigned long nr_pages
= PAGES_PER_SECTION
* sections_per_block
;
250 struct page
*first_page
;
253 first_page
= pfn_to_page(phys_index
<< PFN_SECTION_SHIFT
);
254 start_pfn
= page_to_pfn(first_page
);
258 if (!pages_correctly_reserved(start_pfn
))
261 ret
= online_pages(start_pfn
, nr_pages
, online_type
);
264 ret
= offline_pages(start_pfn
, nr_pages
);
267 WARN(1, KERN_WARNING
"%s(%ld, %ld) unknown action: "
268 "%ld\n", __func__
, phys_index
, action
, action
);
275 static int __memory_block_change_state(struct memory_block
*mem
,
276 unsigned long to_state
, unsigned long from_state_req
,
281 if (mem
->state
!= from_state_req
) {
286 if (to_state
== MEM_OFFLINE
)
287 mem
->state
= MEM_GOING_OFFLINE
;
289 ret
= memory_block_action(mem
->start_section_nr
, to_state
, online_type
);
292 mem
->state
= from_state_req
;
296 mem
->state
= to_state
;
297 switch (mem
->state
) {
299 kobject_uevent(&mem
->dev
.kobj
, KOBJ_OFFLINE
);
302 kobject_uevent(&mem
->dev
.kobj
, KOBJ_ONLINE
);
311 static int memory_block_change_state(struct memory_block
*mem
,
312 unsigned long to_state
, unsigned long from_state_req
,
317 mutex_lock(&mem
->state_mutex
);
318 ret
= __memory_block_change_state(mem
, to_state
, from_state_req
,
320 mutex_unlock(&mem
->state_mutex
);
325 store_mem_state(struct device
*dev
,
326 struct device_attribute
*attr
, const char *buf
, size_t count
)
328 struct memory_block
*mem
;
331 mem
= container_of(dev
, struct memory_block
, dev
);
333 if (!strncmp(buf
, "online_kernel", min_t(int, count
, 13)))
334 ret
= memory_block_change_state(mem
, MEM_ONLINE
,
335 MEM_OFFLINE
, ONLINE_KERNEL
);
336 else if (!strncmp(buf
, "online_movable", min_t(int, count
, 14)))
337 ret
= memory_block_change_state(mem
, MEM_ONLINE
,
338 MEM_OFFLINE
, ONLINE_MOVABLE
);
339 else if (!strncmp(buf
, "online", min_t(int, count
, 6)))
340 ret
= memory_block_change_state(mem
, MEM_ONLINE
,
341 MEM_OFFLINE
, ONLINE_KEEP
);
342 else if(!strncmp(buf
, "offline", min_t(int, count
, 7)))
343 ret
= memory_block_change_state(mem
, MEM_OFFLINE
,
352 * phys_device is a bad name for this. What I really want
353 * is a way to differentiate between memory ranges that
354 * are part of physical devices that constitute
355 * a complete removable unit or fru.
356 * i.e. do these ranges belong to the same physical device,
357 * s.t. if I offline all of these sections I can then
358 * remove the physical device?
360 static ssize_t
show_phys_device(struct device
*dev
,
361 struct device_attribute
*attr
, char *buf
)
363 struct memory_block
*mem
=
364 container_of(dev
, struct memory_block
, dev
);
365 return sprintf(buf
, "%d\n", mem
->phys_device
);
368 static DEVICE_ATTR(phys_index
, 0444, show_mem_start_phys_index
, NULL
);
369 static DEVICE_ATTR(end_phys_index
, 0444, show_mem_end_phys_index
, NULL
);
370 static DEVICE_ATTR(state
, 0644, show_mem_state
, store_mem_state
);
371 static DEVICE_ATTR(phys_device
, 0444, show_phys_device
, NULL
);
372 static DEVICE_ATTR(removable
, 0444, show_mem_removable
, NULL
);
374 #define mem_create_simple_file(mem, attr_name) \
375 device_create_file(&mem->dev, &dev_attr_##attr_name)
376 #define mem_remove_simple_file(mem, attr_name) \
377 device_remove_file(&mem->dev, &dev_attr_##attr_name)
380 * Block size attribute stuff
383 print_block_size(struct device
*dev
, struct device_attribute
*attr
,
386 return sprintf(buf
, "%lx\n", get_memory_block_size());
389 static DEVICE_ATTR(block_size_bytes
, 0444, print_block_size
, NULL
);
391 static int block_size_init(void)
393 return device_create_file(memory_subsys
.dev_root
,
394 &dev_attr_block_size_bytes
);
398 * Some architectures will have custom drivers to do this, and
399 * will not need to do it from userspace. The fake hot-add code
400 * as well as ppc64 will do all of their discovery in userspace
401 * and will require this interface.
403 #ifdef CONFIG_ARCH_MEMORY_PROBE
405 memory_probe_store(struct device
*dev
, struct device_attribute
*attr
,
406 const char *buf
, size_t count
)
411 unsigned long pages_per_block
= PAGES_PER_SECTION
* sections_per_block
;
413 phys_addr
= simple_strtoull(buf
, NULL
, 0);
415 if (phys_addr
& ((pages_per_block
<< PAGE_SHIFT
) - 1))
418 for (i
= 0; i
< sections_per_block
; i
++) {
419 nid
= memory_add_physaddr_to_nid(phys_addr
);
420 ret
= add_memory(nid
, phys_addr
,
421 PAGES_PER_SECTION
<< PAGE_SHIFT
);
425 phys_addr
+= MIN_MEMORY_BLOCK_SIZE
;
432 static DEVICE_ATTR(probe
, S_IWUSR
, NULL
, memory_probe_store
);
434 static int memory_probe_init(void)
436 return device_create_file(memory_subsys
.dev_root
, &dev_attr_probe
);
439 static inline int memory_probe_init(void)
445 #ifdef CONFIG_MEMORY_FAILURE
447 * Support for offlining pages of memory
450 /* Soft offline a page */
452 store_soft_offline_page(struct device
*dev
,
453 struct device_attribute
*attr
,
454 const char *buf
, size_t count
)
458 if (!capable(CAP_SYS_ADMIN
))
460 if (strict_strtoull(buf
, 0, &pfn
) < 0)
465 ret
= soft_offline_page(pfn_to_page(pfn
), 0);
466 return ret
== 0 ? count
: ret
;
469 /* Forcibly offline a page, including killing processes. */
471 store_hard_offline_page(struct device
*dev
,
472 struct device_attribute
*attr
,
473 const char *buf
, size_t count
)
477 if (!capable(CAP_SYS_ADMIN
))
479 if (strict_strtoull(buf
, 0, &pfn
) < 0)
482 ret
= memory_failure(pfn
, 0, 0);
483 return ret
? ret
: count
;
486 static DEVICE_ATTR(soft_offline_page
, S_IWUSR
, NULL
, store_soft_offline_page
);
487 static DEVICE_ATTR(hard_offline_page
, S_IWUSR
, NULL
, store_hard_offline_page
);
489 static __init
int memory_fail_init(void)
493 err
= device_create_file(memory_subsys
.dev_root
,
494 &dev_attr_soft_offline_page
);
496 err
= device_create_file(memory_subsys
.dev_root
,
497 &dev_attr_hard_offline_page
);
501 static inline int memory_fail_init(void)
508 * Note that phys_device is optional. It is here to allow for
509 * differentiation between which *physical* devices each
510 * section belongs to...
512 int __weak
arch_get_memory_phys_device(unsigned long start_pfn
)
518 * A reference for the returned object is held and the reference for the
519 * hinted object is released.
521 struct memory_block
*find_memory_block_hinted(struct mem_section
*section
,
522 struct memory_block
*hint
)
524 int block_id
= base_memory_block_id(__section_nr(section
));
525 struct device
*hintdev
= hint
? &hint
->dev
: NULL
;
528 dev
= subsys_find_device_by_id(&memory_subsys
, block_id
, hintdev
);
530 put_device(&hint
->dev
);
533 return container_of(dev
, struct memory_block
, dev
);
537 * For now, we have a linear search to go find the appropriate
538 * memory_block corresponding to a particular phys_index. If
539 * this gets to be a real problem, we can always use a radix
540 * tree or something here.
542 * This could be made generic for all device subsystems.
544 struct memory_block
*find_memory_block(struct mem_section
*section
)
546 return find_memory_block_hinted(section
, NULL
);
549 static int init_memory_block(struct memory_block
**memory
,
550 struct mem_section
*section
, unsigned long state
)
552 struct memory_block
*mem
;
553 unsigned long start_pfn
;
557 mem
= kzalloc(sizeof(*mem
), GFP_KERNEL
);
561 scn_nr
= __section_nr(section
);
562 mem
->start_section_nr
=
563 base_memory_block_id(scn_nr
) * sections_per_block
;
564 mem
->end_section_nr
= mem
->start_section_nr
+ sections_per_block
- 1;
566 mem
->section_count
++;
567 mutex_init(&mem
->state_mutex
);
568 start_pfn
= section_nr_to_pfn(mem
->start_section_nr
);
569 mem
->phys_device
= arch_get_memory_phys_device(start_pfn
);
571 ret
= register_memory(mem
);
573 ret
= mem_create_simple_file(mem
, phys_index
);
575 ret
= mem_create_simple_file(mem
, end_phys_index
);
577 ret
= mem_create_simple_file(mem
, state
);
579 ret
= mem_create_simple_file(mem
, phys_device
);
581 ret
= mem_create_simple_file(mem
, removable
);
587 static int add_memory_section(int nid
, struct mem_section
*section
,
588 struct memory_block
**mem_p
,
589 unsigned long state
, enum mem_add_context context
)
591 struct memory_block
*mem
= NULL
;
592 int scn_nr
= __section_nr(section
);
595 mutex_lock(&mem_sysfs_mutex
);
597 if (context
== BOOT
) {
598 /* same memory block ? */
600 if (scn_nr
>= (*mem_p
)->start_section_nr
&&
601 scn_nr
<= (*mem_p
)->end_section_nr
) {
603 kobject_get(&mem
->dev
.kobj
);
606 mem
= find_memory_block(section
);
609 mem
->section_count
++;
610 kobject_put(&mem
->dev
.kobj
);
612 ret
= init_memory_block(&mem
, section
, state
);
613 /* store memory_block pointer for next loop */
614 if (!ret
&& context
== BOOT
)
620 if (context
== HOTPLUG
&&
621 mem
->section_count
== sections_per_block
)
622 ret
= register_mem_sect_under_node(mem
, nid
);
625 mutex_unlock(&mem_sysfs_mutex
);
630 * need an interface for the VM to add new memory regions,
631 * but without onlining it.
633 int register_new_memory(int nid
, struct mem_section
*section
)
635 return add_memory_section(nid
, section
, NULL
, MEM_OFFLINE
, HOTPLUG
);
638 #ifdef CONFIG_MEMORY_HOTREMOVE
640 unregister_memory(struct memory_block
*memory
)
642 BUG_ON(memory
->dev
.bus
!= &memory_subsys
);
644 /* drop the ref. we got in remove_memory_block() */
645 kobject_put(&memory
->dev
.kobj
);
646 device_unregister(&memory
->dev
);
649 static int remove_memory_block(unsigned long node_id
,
650 struct mem_section
*section
, int phys_device
)
652 struct memory_block
*mem
;
654 mutex_lock(&mem_sysfs_mutex
);
655 mem
= find_memory_block(section
);
656 unregister_mem_sect_under_nodes(mem
, __section_nr(section
));
658 mem
->section_count
--;
659 if (mem
->section_count
== 0) {
660 mem_remove_simple_file(mem
, phys_index
);
661 mem_remove_simple_file(mem
, end_phys_index
);
662 mem_remove_simple_file(mem
, state
);
663 mem_remove_simple_file(mem
, phys_device
);
664 mem_remove_simple_file(mem
, removable
);
665 unregister_memory(mem
);
667 kobject_put(&mem
->dev
.kobj
);
669 mutex_unlock(&mem_sysfs_mutex
);
673 int unregister_memory_section(struct mem_section
*section
)
675 if (!present_section(section
))
678 return remove_memory_block(0, section
, 0);
680 #endif /* CONFIG_MEMORY_HOTREMOVE */
683 * offline one memory block. If the memory block has been offlined, do nothing.
685 int offline_memory_block(struct memory_block
*mem
)
689 mutex_lock(&mem
->state_mutex
);
690 if (mem
->state
!= MEM_OFFLINE
)
691 ret
= __memory_block_change_state(mem
, MEM_OFFLINE
, MEM_ONLINE
, -1);
692 mutex_unlock(&mem
->state_mutex
);
697 /* return true if the memory block is offlined, otherwise, return false */
698 bool is_memblock_offlined(struct memory_block
*mem
)
700 return mem
->state
== MEM_OFFLINE
;
704 * Initialize the sysfs support for memory devices...
706 int __init
memory_dev_init(void)
711 unsigned long block_sz
;
712 struct memory_block
*mem
= NULL
;
714 ret
= subsys_system_register(&memory_subsys
, NULL
);
718 block_sz
= get_memory_block_size();
719 sections_per_block
= block_sz
/ MIN_MEMORY_BLOCK_SIZE
;
722 * Create entries for memory sections that were found
723 * during boot and have been initialized
725 for (i
= 0; i
< NR_MEM_SECTIONS
; i
++) {
726 if (!present_section_nr(i
))
728 /* don't need to reuse memory_block if only one per block */
729 err
= add_memory_section(0, __nr_to_section(i
),
730 (sections_per_block
== 1) ? NULL
: &mem
,
737 err
= memory_probe_init();
740 err
= memory_fail_init();
743 err
= block_size_init();
748 printk(KERN_ERR
"%s() failed: %d\n", __func__
, ret
);