1 // SPDX-License-Identifier: GPL-2.0
3 * Basic Node interface support
6 #include <linux/module.h>
7 #include <linux/init.h>
9 #include <linux/memory.h>
10 #include <linux/vmstat.h>
11 #include <linux/notifier.h>
12 #include <linux/node.h>
13 #include <linux/hugetlb.h>
14 #include <linux/compaction.h>
15 #include <linux/cpumask.h>
16 #include <linux/topology.h>
17 #include <linux/nodemask.h>
18 #include <linux/cpu.h>
19 #include <linux/device.h>
20 #include <linux/pm_runtime.h>
21 #include <linux/swap.h>
22 #include <linux/slab.h>
24 static struct bus_type node_subsys
= {
30 static ssize_t
node_read_cpumap(struct device
*dev
, bool list
, char *buf
)
34 struct node
*node_dev
= to_node(dev
);
36 /* 2008/04/07: buf currently PAGE_SIZE, need 9 chars per 32 bits. */
37 BUILD_BUG_ON((NR_CPUS
/32 * 9) > (PAGE_SIZE
-1));
39 if (!alloc_cpumask_var(&mask
, GFP_KERNEL
))
42 cpumask_and(mask
, cpumask_of_node(node_dev
->dev
.id
), cpu_online_mask
);
43 n
= cpumap_print_to_pagebuf(list
, buf
, mask
);
44 free_cpumask_var(mask
);
49 static inline ssize_t
cpumap_show(struct device
*dev
,
50 struct device_attribute
*attr
,
53 return node_read_cpumap(dev
, false, buf
);
56 static DEVICE_ATTR_RO(cpumap
);
58 static inline ssize_t
cpulist_show(struct device
*dev
,
59 struct device_attribute
*attr
,
62 return node_read_cpumap(dev
, true, buf
);
65 static DEVICE_ATTR_RO(cpulist
);
68 * struct node_access_nodes - Access class device to hold user visible
69 * relationships to other nodes.
70 * @dev: Device for this memory access class
71 * @list_node: List element in the node's access list
72 * @access: The access class rank
73 * @hmem_attrs: Heterogeneous memory performance attributes
75 struct node_access_nodes
{
77 struct list_head list_node
;
79 #ifdef CONFIG_HMEM_REPORTING
80 struct node_hmem_attrs hmem_attrs
;
83 #define to_access_nodes(dev) container_of(dev, struct node_access_nodes, dev)
85 static struct attribute
*node_init_access_node_attrs
[] = {
89 static struct attribute
*node_targ_access_node_attrs
[] = {
93 static const struct attribute_group initiators
= {
95 .attrs
= node_init_access_node_attrs
,
98 static const struct attribute_group targets
= {
100 .attrs
= node_targ_access_node_attrs
,
103 static const struct attribute_group
*node_access_node_groups
[] = {
109 static void node_remove_accesses(struct node
*node
)
111 struct node_access_nodes
*c
, *cnext
;
113 list_for_each_entry_safe(c
, cnext
, &node
->access_list
, list_node
) {
114 list_del(&c
->list_node
);
115 device_unregister(&c
->dev
);
119 static void node_access_release(struct device
*dev
)
121 kfree(to_access_nodes(dev
));
124 static struct node_access_nodes
*node_init_node_access(struct node
*node
,
127 struct node_access_nodes
*access_node
;
130 list_for_each_entry(access_node
, &node
->access_list
, list_node
)
131 if (access_node
->access
== access
)
134 access_node
= kzalloc(sizeof(*access_node
), GFP_KERNEL
);
138 access_node
->access
= access
;
139 dev
= &access_node
->dev
;
140 dev
->parent
= &node
->dev
;
141 dev
->release
= node_access_release
;
142 dev
->groups
= node_access_node_groups
;
143 if (dev_set_name(dev
, "access%u", access
))
146 if (device_register(dev
))
149 pm_runtime_no_callbacks(dev
);
150 list_add_tail(&access_node
->list_node
, &node
->access_list
);
153 kfree_const(dev
->kobj
.name
);
159 #ifdef CONFIG_HMEM_REPORTING
160 #define ACCESS_ATTR(name) \
161 static ssize_t name##_show(struct device *dev, \
162 struct device_attribute *attr, \
165 return sysfs_emit(buf, "%u\n", \
166 to_access_nodes(dev)->hmem_attrs.name); \
168 static DEVICE_ATTR_RO(name)
170 ACCESS_ATTR(read_bandwidth
);
171 ACCESS_ATTR(read_latency
);
172 ACCESS_ATTR(write_bandwidth
);
173 ACCESS_ATTR(write_latency
);
175 static struct attribute
*access_attrs
[] = {
176 &dev_attr_read_bandwidth
.attr
,
177 &dev_attr_read_latency
.attr
,
178 &dev_attr_write_bandwidth
.attr
,
179 &dev_attr_write_latency
.attr
,
184 * node_set_perf_attrs - Set the performance values for given access class
185 * @nid: Node identifier to be set
186 * @hmem_attrs: Heterogeneous memory performance attributes
187 * @access: The access class the for the given attributes
189 void node_set_perf_attrs(unsigned int nid
, struct node_hmem_attrs
*hmem_attrs
,
192 struct node_access_nodes
*c
;
196 if (WARN_ON_ONCE(!node_online(nid
)))
199 node
= node_devices
[nid
];
200 c
= node_init_node_access(node
, access
);
204 c
->hmem_attrs
= *hmem_attrs
;
205 for (i
= 0; access_attrs
[i
] != NULL
; i
++) {
206 if (sysfs_add_file_to_group(&c
->dev
.kobj
, access_attrs
[i
],
208 pr_info("failed to add performance attribute to node %d\n",
216 * struct node_cache_info - Internal tracking for memory node caches
217 * @dev: Device represeting the cache level
218 * @node: List element for tracking in the node
219 * @cache_attrs:Attributes for this cache level
221 struct node_cache_info
{
223 struct list_head node
;
224 struct node_cache_attrs cache_attrs
;
226 #define to_cache_info(device) container_of(device, struct node_cache_info, dev)
228 #define CACHE_ATTR(name, fmt) \
229 static ssize_t name##_show(struct device *dev, \
230 struct device_attribute *attr, \
233 return sysfs_emit(buf, fmt "\n", \
234 to_cache_info(dev)->cache_attrs.name); \
236 static DEVICE_ATTR_RO(name);
238 CACHE_ATTR(size
, "%llu")
239 CACHE_ATTR(line_size
, "%u")
240 CACHE_ATTR(indexing
, "%u")
241 CACHE_ATTR(write_policy
, "%u")
243 static struct attribute
*cache_attrs
[] = {
244 &dev_attr_indexing
.attr
,
246 &dev_attr_line_size
.attr
,
247 &dev_attr_write_policy
.attr
,
250 ATTRIBUTE_GROUPS(cache
);
252 static void node_cache_release(struct device
*dev
)
257 static void node_cacheinfo_release(struct device
*dev
)
259 struct node_cache_info
*info
= to_cache_info(dev
);
263 static void node_init_cache_dev(struct node
*node
)
267 dev
= kzalloc(sizeof(*dev
), GFP_KERNEL
);
271 device_initialize(dev
);
272 dev
->parent
= &node
->dev
;
273 dev
->release
= node_cache_release
;
274 if (dev_set_name(dev
, "memory_side_cache"))
280 pm_runtime_no_callbacks(dev
);
281 node
->cache_dev
= dev
;
288 * node_add_cache() - add cache attribute to a memory node
289 * @nid: Node identifier that has new cache attributes
290 * @cache_attrs: Attributes for the cache being added
292 void node_add_cache(unsigned int nid
, struct node_cache_attrs
*cache_attrs
)
294 struct node_cache_info
*info
;
298 if (!node_online(nid
) || !node_devices
[nid
])
301 node
= node_devices
[nid
];
302 list_for_each_entry(info
, &node
->cache_attrs
, node
) {
303 if (info
->cache_attrs
.level
== cache_attrs
->level
) {
305 "attempt to add duplicate cache level:%d\n",
311 if (!node
->cache_dev
)
312 node_init_cache_dev(node
);
313 if (!node
->cache_dev
)
316 info
= kzalloc(sizeof(*info
), GFP_KERNEL
);
321 device_initialize(dev
);
322 dev
->parent
= node
->cache_dev
;
323 dev
->release
= node_cacheinfo_release
;
324 dev
->groups
= cache_groups
;
325 if (dev_set_name(dev
, "index%d", cache_attrs
->level
))
328 info
->cache_attrs
= *cache_attrs
;
329 if (device_add(dev
)) {
330 dev_warn(&node
->dev
, "failed to add cache level:%d\n",
334 pm_runtime_no_callbacks(dev
);
335 list_add_tail(&info
->node
, &node
->cache_attrs
);
341 static void node_remove_caches(struct node
*node
)
343 struct node_cache_info
*info
, *next
;
345 if (!node
->cache_dev
)
348 list_for_each_entry_safe(info
, next
, &node
->cache_attrs
, node
) {
349 list_del(&info
->node
);
350 device_unregister(&info
->dev
);
352 device_unregister(node
->cache_dev
);
355 static void node_init_caches(unsigned int nid
)
357 INIT_LIST_HEAD(&node_devices
[nid
]->cache_attrs
);
360 static void node_init_caches(unsigned int nid
) { }
361 static void node_remove_caches(struct node
*node
) { }
364 #define K(x) ((x) << (PAGE_SHIFT - 10))
365 static ssize_t
node_read_meminfo(struct device
*dev
,
366 struct device_attribute
*attr
, char *buf
)
370 struct pglist_data
*pgdat
= NODE_DATA(nid
);
372 unsigned long sreclaimable
, sunreclaimable
;
373 unsigned long swapcached
= 0;
375 si_meminfo_node(&i
, nid
);
376 sreclaimable
= node_page_state_pages(pgdat
, NR_SLAB_RECLAIMABLE_B
);
377 sunreclaimable
= node_page_state_pages(pgdat
, NR_SLAB_UNRECLAIMABLE_B
);
379 swapcached
= node_page_state_pages(pgdat
, NR_SWAPCACHE
);
381 len
= sysfs_emit_at(buf
, len
,
382 "Node %d MemTotal: %8lu kB\n"
383 "Node %d MemFree: %8lu kB\n"
384 "Node %d MemUsed: %8lu kB\n"
385 "Node %d SwapCached: %8lu kB\n"
386 "Node %d Active: %8lu kB\n"
387 "Node %d Inactive: %8lu kB\n"
388 "Node %d Active(anon): %8lu kB\n"
389 "Node %d Inactive(anon): %8lu kB\n"
390 "Node %d Active(file): %8lu kB\n"
391 "Node %d Inactive(file): %8lu kB\n"
392 "Node %d Unevictable: %8lu kB\n"
393 "Node %d Mlocked: %8lu kB\n",
396 nid
, K(i
.totalram
- i
.freeram
),
398 nid
, K(node_page_state(pgdat
, NR_ACTIVE_ANON
) +
399 node_page_state(pgdat
, NR_ACTIVE_FILE
)),
400 nid
, K(node_page_state(pgdat
, NR_INACTIVE_ANON
) +
401 node_page_state(pgdat
, NR_INACTIVE_FILE
)),
402 nid
, K(node_page_state(pgdat
, NR_ACTIVE_ANON
)),
403 nid
, K(node_page_state(pgdat
, NR_INACTIVE_ANON
)),
404 nid
, K(node_page_state(pgdat
, NR_ACTIVE_FILE
)),
405 nid
, K(node_page_state(pgdat
, NR_INACTIVE_FILE
)),
406 nid
, K(node_page_state(pgdat
, NR_UNEVICTABLE
)),
407 nid
, K(sum_zone_node_page_state(nid
, NR_MLOCK
)));
409 #ifdef CONFIG_HIGHMEM
410 len
+= sysfs_emit_at(buf
, len
,
411 "Node %d HighTotal: %8lu kB\n"
412 "Node %d HighFree: %8lu kB\n"
413 "Node %d LowTotal: %8lu kB\n"
414 "Node %d LowFree: %8lu kB\n",
417 nid
, K(i
.totalram
- i
.totalhigh
),
418 nid
, K(i
.freeram
- i
.freehigh
));
420 len
+= sysfs_emit_at(buf
, len
,
421 "Node %d Dirty: %8lu kB\n"
422 "Node %d Writeback: %8lu kB\n"
423 "Node %d FilePages: %8lu kB\n"
424 "Node %d Mapped: %8lu kB\n"
425 "Node %d AnonPages: %8lu kB\n"
426 "Node %d Shmem: %8lu kB\n"
427 "Node %d KernelStack: %8lu kB\n"
428 #ifdef CONFIG_SHADOW_CALL_STACK
429 "Node %d ShadowCallStack:%8lu kB\n"
431 "Node %d PageTables: %8lu kB\n"
432 "Node %d NFS_Unstable: %8lu kB\n"
433 "Node %d Bounce: %8lu kB\n"
434 "Node %d WritebackTmp: %8lu kB\n"
435 "Node %d KReclaimable: %8lu kB\n"
436 "Node %d Slab: %8lu kB\n"
437 "Node %d SReclaimable: %8lu kB\n"
438 "Node %d SUnreclaim: %8lu kB\n"
439 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
440 "Node %d AnonHugePages: %8lu kB\n"
441 "Node %d ShmemHugePages: %8lu kB\n"
442 "Node %d ShmemPmdMapped: %8lu kB\n"
443 "Node %d FileHugePages: %8lu kB\n"
444 "Node %d FilePmdMapped: %8lu kB\n"
447 nid
, K(node_page_state(pgdat
, NR_FILE_DIRTY
)),
448 nid
, K(node_page_state(pgdat
, NR_WRITEBACK
)),
449 nid
, K(node_page_state(pgdat
, NR_FILE_PAGES
)),
450 nid
, K(node_page_state(pgdat
, NR_FILE_MAPPED
)),
451 nid
, K(node_page_state(pgdat
, NR_ANON_MAPPED
)),
453 nid
, node_page_state(pgdat
, NR_KERNEL_STACK_KB
),
454 #ifdef CONFIG_SHADOW_CALL_STACK
455 nid
, node_page_state(pgdat
, NR_KERNEL_SCS_KB
),
457 nid
, K(node_page_state(pgdat
, NR_PAGETABLE
)),
459 nid
, K(sum_zone_node_page_state(nid
, NR_BOUNCE
)),
460 nid
, K(node_page_state(pgdat
, NR_WRITEBACK_TEMP
)),
461 nid
, K(sreclaimable
+
462 node_page_state(pgdat
, NR_KERNEL_MISC_RECLAIMABLE
)),
463 nid
, K(sreclaimable
+ sunreclaimable
),
464 nid
, K(sreclaimable
),
465 nid
, K(sunreclaimable
)
466 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
468 nid
, K(node_page_state(pgdat
, NR_ANON_THPS
)),
469 nid
, K(node_page_state(pgdat
, NR_SHMEM_THPS
)),
470 nid
, K(node_page_state(pgdat
, NR_SHMEM_PMDMAPPED
)),
471 nid
, K(node_page_state(pgdat
, NR_FILE_THPS
)),
472 nid
, K(node_page_state(pgdat
, NR_FILE_PMDMAPPED
))
475 len
+= hugetlb_report_node_meminfo(buf
, len
, nid
);
480 static DEVICE_ATTR(meminfo
, 0444, node_read_meminfo
, NULL
);
482 static ssize_t
node_read_numastat(struct device
*dev
,
483 struct device_attribute
*attr
, char *buf
)
485 fold_vm_numa_events();
486 return sysfs_emit(buf
,
490 "interleave_hit %lu\n"
493 sum_zone_numa_event_state(dev
->id
, NUMA_HIT
),
494 sum_zone_numa_event_state(dev
->id
, NUMA_MISS
),
495 sum_zone_numa_event_state(dev
->id
, NUMA_FOREIGN
),
496 sum_zone_numa_event_state(dev
->id
, NUMA_INTERLEAVE_HIT
),
497 sum_zone_numa_event_state(dev
->id
, NUMA_LOCAL
),
498 sum_zone_numa_event_state(dev
->id
, NUMA_OTHER
));
500 static DEVICE_ATTR(numastat
, 0444, node_read_numastat
, NULL
);
502 static ssize_t
node_read_vmstat(struct device
*dev
,
503 struct device_attribute
*attr
, char *buf
)
506 struct pglist_data
*pgdat
= NODE_DATA(nid
);
510 for (i
= 0; i
< NR_VM_ZONE_STAT_ITEMS
; i
++)
511 len
+= sysfs_emit_at(buf
, len
, "%s %lu\n",
513 sum_zone_node_page_state(nid
, i
));
516 fold_vm_numa_events();
517 for (i
= 0; i
< NR_VM_NUMA_EVENT_ITEMS
; i
++)
518 len
+= sysfs_emit_at(buf
, len
, "%s %lu\n",
520 sum_zone_numa_event_state(nid
, i
));
523 for (i
= 0; i
< NR_VM_NODE_STAT_ITEMS
; i
++) {
524 unsigned long pages
= node_page_state_pages(pgdat
, i
);
526 if (vmstat_item_print_in_thp(i
))
527 pages
/= HPAGE_PMD_NR
;
528 len
+= sysfs_emit_at(buf
, len
, "%s %lu\n", node_stat_name(i
),
534 static DEVICE_ATTR(vmstat
, 0444, node_read_vmstat
, NULL
);
536 static ssize_t
node_read_distance(struct device
*dev
,
537 struct device_attribute
*attr
, char *buf
)
544 * buf is currently PAGE_SIZE in length and each node needs 4 chars
545 * at the most (distance + space or newline).
547 BUILD_BUG_ON(MAX_NUMNODES
* 4 > PAGE_SIZE
);
549 for_each_online_node(i
) {
550 len
+= sysfs_emit_at(buf
, len
, "%s%d",
551 i
? " " : "", node_distance(nid
, i
));
554 len
+= sysfs_emit_at(buf
, len
, "\n");
557 static DEVICE_ATTR(distance
, 0444, node_read_distance
, NULL
);
559 static struct attribute
*node_dev_attrs
[] = {
560 &dev_attr_cpumap
.attr
,
561 &dev_attr_cpulist
.attr
,
562 &dev_attr_meminfo
.attr
,
563 &dev_attr_numastat
.attr
,
564 &dev_attr_distance
.attr
,
565 &dev_attr_vmstat
.attr
,
568 ATTRIBUTE_GROUPS(node_dev
);
570 #ifdef CONFIG_HUGETLBFS
572 * hugetlbfs per node attributes registration interface:
573 * When/if hugetlb[fs] subsystem initializes [sometime after this module],
574 * it will register its per node attributes for all online nodes with
575 * memory. It will also call register_hugetlbfs_with_node(), below, to
576 * register its attribute registration functions with this node driver.
577 * Once these hooks have been initialized, the node driver will call into
578 * the hugetlb module to [un]register attributes for hot-plugged nodes.
580 static node_registration_func_t __hugetlb_register_node
;
581 static node_registration_func_t __hugetlb_unregister_node
;
583 static inline bool hugetlb_register_node(struct node
*node
)
585 if (__hugetlb_register_node
&&
586 node_state(node
->dev
.id
, N_MEMORY
)) {
587 __hugetlb_register_node(node
);
593 static inline void hugetlb_unregister_node(struct node
*node
)
595 if (__hugetlb_unregister_node
)
596 __hugetlb_unregister_node(node
);
599 void register_hugetlbfs_with_node(node_registration_func_t doregister
,
600 node_registration_func_t unregister
)
602 __hugetlb_register_node
= doregister
;
603 __hugetlb_unregister_node
= unregister
;
606 static inline void hugetlb_register_node(struct node
*node
) {}
608 static inline void hugetlb_unregister_node(struct node
*node
) {}
611 static void node_device_release(struct device
*dev
)
613 struct node
*node
= to_node(dev
);
615 #if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) && defined(CONFIG_HUGETLBFS)
617 * We schedule the work only when a memory section is
618 * onlined/offlined on this node. When we come here,
619 * all the memory on this node has been offlined,
620 * so we won't enqueue new work to this work.
622 * The work is using node->node_work, so we should
623 * flush work before freeing the memory.
625 flush_work(&node
->node_work
);
631 * register_node - Setup a sysfs device for a node.
632 * @num - Node number to use when creating the device.
634 * Initialize and register the node device.
636 static int register_node(struct node
*node
, int num
)
641 node
->dev
.bus
= &node_subsys
;
642 node
->dev
.release
= node_device_release
;
643 node
->dev
.groups
= node_dev_groups
;
644 error
= device_register(&node
->dev
);
647 put_device(&node
->dev
);
649 hugetlb_register_node(node
);
651 compaction_register_node(node
);
657 * unregister_node - unregister a node device
658 * @node: node going away
660 * Unregisters a node device @node. All the devices on the node must be
661 * unregistered before calling this function.
663 void unregister_node(struct node
*node
)
665 hugetlb_unregister_node(node
); /* no-op, if memoryless node */
666 node_remove_accesses(node
);
667 node_remove_caches(node
);
668 device_unregister(&node
->dev
);
671 struct node
*node_devices
[MAX_NUMNODES
];
674 * register cpu under node
676 int register_cpu_under_node(unsigned int cpu
, unsigned int nid
)
681 if (!node_online(nid
))
684 obj
= get_cpu_device(cpu
);
688 ret
= sysfs_create_link(&node_devices
[nid
]->dev
.kobj
,
690 kobject_name(&obj
->kobj
));
694 return sysfs_create_link(&obj
->kobj
,
695 &node_devices
[nid
]->dev
.kobj
,
696 kobject_name(&node_devices
[nid
]->dev
.kobj
));
700 * register_memory_node_under_compute_node - link memory node to its compute
701 * node for a given access class.
702 * @mem_nid: Memory node number
703 * @cpu_nid: Cpu node number
704 * @access: Access class to register
707 * For use with platforms that may have separate memory and compute nodes.
708 * This function will export node relationships linking which memory
709 * initiator nodes can access memory targets at a given ranked access
712 int register_memory_node_under_compute_node(unsigned int mem_nid
,
713 unsigned int cpu_nid
,
716 struct node
*init_node
, *targ_node
;
717 struct node_access_nodes
*initiator
, *target
;
720 if (!node_online(cpu_nid
) || !node_online(mem_nid
))
723 init_node
= node_devices
[cpu_nid
];
724 targ_node
= node_devices
[mem_nid
];
725 initiator
= node_init_node_access(init_node
, access
);
726 target
= node_init_node_access(targ_node
, access
);
727 if (!initiator
|| !target
)
730 ret
= sysfs_add_link_to_group(&initiator
->dev
.kobj
, "targets",
731 &targ_node
->dev
.kobj
,
732 dev_name(&targ_node
->dev
));
736 ret
= sysfs_add_link_to_group(&target
->dev
.kobj
, "initiators",
737 &init_node
->dev
.kobj
,
738 dev_name(&init_node
->dev
));
744 sysfs_remove_link_from_group(&initiator
->dev
.kobj
, "targets",
745 dev_name(&targ_node
->dev
));
749 int unregister_cpu_under_node(unsigned int cpu
, unsigned int nid
)
753 if (!node_online(nid
))
756 obj
= get_cpu_device(cpu
);
760 sysfs_remove_link(&node_devices
[nid
]->dev
.kobj
,
761 kobject_name(&obj
->kobj
));
762 sysfs_remove_link(&obj
->kobj
,
763 kobject_name(&node_devices
[nid
]->dev
.kobj
));
768 #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
769 static int __ref
get_nid_for_pfn(unsigned long pfn
)
771 if (!pfn_valid_within(pfn
))
773 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
774 if (system_state
< SYSTEM_RUNNING
)
775 return early_pfn_to_nid(pfn
);
777 return pfn_to_nid(pfn
);
780 static void do_register_memory_block_under_node(int nid
,
781 struct memory_block
*mem_blk
)
786 * If this memory block spans multiple nodes, we only indicate
787 * the last processed node.
791 ret
= sysfs_create_link_nowarn(&node_devices
[nid
]->dev
.kobj
,
793 kobject_name(&mem_blk
->dev
.kobj
));
794 if (ret
&& ret
!= -EEXIST
)
795 dev_err_ratelimited(&node_devices
[nid
]->dev
,
796 "can't create link to %s in sysfs (%d)\n",
797 kobject_name(&mem_blk
->dev
.kobj
), ret
);
799 ret
= sysfs_create_link_nowarn(&mem_blk
->dev
.kobj
,
800 &node_devices
[nid
]->dev
.kobj
,
801 kobject_name(&node_devices
[nid
]->dev
.kobj
));
802 if (ret
&& ret
!= -EEXIST
)
803 dev_err_ratelimited(&mem_blk
->dev
,
804 "can't create link to %s in sysfs (%d)\n",
805 kobject_name(&node_devices
[nid
]->dev
.kobj
),
809 /* register memory section under specified node if it spans that node */
810 static int register_mem_block_under_node_early(struct memory_block
*mem_blk
,
813 unsigned long memory_block_pfns
= memory_block_size_bytes() / PAGE_SIZE
;
814 unsigned long start_pfn
= section_nr_to_pfn(mem_blk
->start_section_nr
);
815 unsigned long end_pfn
= start_pfn
+ memory_block_pfns
- 1;
816 int nid
= *(int *)arg
;
819 for (pfn
= start_pfn
; pfn
<= end_pfn
; pfn
++) {
823 * memory block could have several absent sections from start.
824 * skip pfn range from absent section
826 if (!pfn_in_present_section(pfn
)) {
827 pfn
= round_down(pfn
+ PAGES_PER_SECTION
,
828 PAGES_PER_SECTION
) - 1;
833 * We need to check if page belongs to nid only at the boot
834 * case because node's ranges can be interleaved.
836 page_nid
= get_nid_for_pfn(pfn
);
842 do_register_memory_block_under_node(nid
, mem_blk
);
845 /* mem section does not span the specified node */
850 * During hotplug we know that all pages in the memory block belong to the same
853 static int register_mem_block_under_node_hotplug(struct memory_block
*mem_blk
,
856 int nid
= *(int *)arg
;
858 do_register_memory_block_under_node(nid
, mem_blk
);
863 * Unregister a memory block device under the node it spans. Memory blocks
864 * with multiple nodes cannot be offlined and therefore also never be removed.
866 void unregister_memory_block_under_nodes(struct memory_block
*mem_blk
)
868 if (mem_blk
->nid
== NUMA_NO_NODE
)
871 sysfs_remove_link(&node_devices
[mem_blk
->nid
]->dev
.kobj
,
872 kobject_name(&mem_blk
->dev
.kobj
));
873 sysfs_remove_link(&mem_blk
->dev
.kobj
,
874 kobject_name(&node_devices
[mem_blk
->nid
]->dev
.kobj
));
877 void link_mem_sections(int nid
, unsigned long start_pfn
, unsigned long end_pfn
,
878 enum meminit_context context
)
880 walk_memory_blocks_func_t func
;
882 if (context
== MEMINIT_HOTPLUG
)
883 func
= register_mem_block_under_node_hotplug
;
885 func
= register_mem_block_under_node_early
;
887 walk_memory_blocks(PFN_PHYS(start_pfn
), PFN_PHYS(end_pfn
- start_pfn
),
892 #ifdef CONFIG_HUGETLBFS
894 * Handle per node hstate attribute [un]registration on transistions
895 * to/from memoryless state.
897 static void node_hugetlb_work(struct work_struct
*work
)
899 struct node
*node
= container_of(work
, struct node
, node_work
);
902 * We only get here when a node transitions to/from memoryless state.
903 * We can detect which transition occurred by examining whether the
904 * node has memory now. hugetlb_register_node() already check this
905 * so we try to register the attributes. If that fails, then the
906 * node has transitioned to memoryless, try to unregister the
909 if (!hugetlb_register_node(node
))
910 hugetlb_unregister_node(node
);
913 static void init_node_hugetlb_work(int nid
)
915 INIT_WORK(&node_devices
[nid
]->node_work
, node_hugetlb_work
);
918 static int node_memory_callback(struct notifier_block
*self
,
919 unsigned long action
, void *arg
)
921 struct memory_notify
*mnb
= arg
;
922 int nid
= mnb
->status_change_nid
;
928 * offload per node hstate [un]registration to a work thread
929 * when transitioning to/from memoryless state.
931 if (nid
!= NUMA_NO_NODE
)
932 schedule_work(&node_devices
[nid
]->node_work
);
935 case MEM_GOING_ONLINE
:
936 case MEM_GOING_OFFLINE
:
937 case MEM_CANCEL_ONLINE
:
938 case MEM_CANCEL_OFFLINE
:
945 #endif /* CONFIG_HUGETLBFS */
946 #endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */
948 #if !defined(CONFIG_MEMORY_HOTPLUG_SPARSE) || \
949 !defined(CONFIG_HUGETLBFS)
950 static inline int node_memory_callback(struct notifier_block
*self
,
951 unsigned long action
, void *arg
)
956 static void init_node_hugetlb_work(int nid
) { }
960 int __register_one_node(int nid
)
965 node_devices
[nid
] = kzalloc(sizeof(struct node
), GFP_KERNEL
);
966 if (!node_devices
[nid
])
969 error
= register_node(node_devices
[nid
], nid
);
971 /* link cpu under this node */
972 for_each_present_cpu(cpu
) {
973 if (cpu_to_node(cpu
) == nid
)
974 register_cpu_under_node(cpu
, nid
);
977 INIT_LIST_HEAD(&node_devices
[nid
]->access_list
);
978 /* initialize work queue for memory hot plug */
979 init_node_hugetlb_work(nid
);
980 node_init_caches(nid
);
985 void unregister_one_node(int nid
)
987 if (!node_devices
[nid
])
990 unregister_node(node_devices
[nid
]);
991 node_devices
[nid
] = NULL
;
995 * node states attributes
999 struct device_attribute attr
;
1000 enum node_states state
;
1003 static ssize_t
show_node_state(struct device
*dev
,
1004 struct device_attribute
*attr
, char *buf
)
1006 struct node_attr
*na
= container_of(attr
, struct node_attr
, attr
);
1008 return sysfs_emit(buf
, "%*pbl\n",
1009 nodemask_pr_args(&node_states
[na
->state
]));
1012 #define _NODE_ATTR(name, state) \
1013 { __ATTR(name, 0444, show_node_state, NULL), state }
1015 static struct node_attr node_state_attr
[] = {
1016 [N_POSSIBLE
] = _NODE_ATTR(possible
, N_POSSIBLE
),
1017 [N_ONLINE
] = _NODE_ATTR(online
, N_ONLINE
),
1018 [N_NORMAL_MEMORY
] = _NODE_ATTR(has_normal_memory
, N_NORMAL_MEMORY
),
1019 #ifdef CONFIG_HIGHMEM
1020 [N_HIGH_MEMORY
] = _NODE_ATTR(has_high_memory
, N_HIGH_MEMORY
),
1022 [N_MEMORY
] = _NODE_ATTR(has_memory
, N_MEMORY
),
1023 [N_CPU
] = _NODE_ATTR(has_cpu
, N_CPU
),
1024 [N_GENERIC_INITIATOR
] = _NODE_ATTR(has_generic_initiator
,
1025 N_GENERIC_INITIATOR
),
1028 static struct attribute
*node_state_attrs
[] = {
1029 &node_state_attr
[N_POSSIBLE
].attr
.attr
,
1030 &node_state_attr
[N_ONLINE
].attr
.attr
,
1031 &node_state_attr
[N_NORMAL_MEMORY
].attr
.attr
,
1032 #ifdef CONFIG_HIGHMEM
1033 &node_state_attr
[N_HIGH_MEMORY
].attr
.attr
,
1035 &node_state_attr
[N_MEMORY
].attr
.attr
,
1036 &node_state_attr
[N_CPU
].attr
.attr
,
1037 &node_state_attr
[N_GENERIC_INITIATOR
].attr
.attr
,
1041 static const struct attribute_group memory_root_attr_group
= {
1042 .attrs
= node_state_attrs
,
1045 static const struct attribute_group
*cpu_root_attr_groups
[] = {
1046 &memory_root_attr_group
,
1050 #define NODE_CALLBACK_PRI 2 /* lower than SLAB */
1051 static int __init
register_node_type(void)
1055 BUILD_BUG_ON(ARRAY_SIZE(node_state_attr
) != NR_NODE_STATES
);
1056 BUILD_BUG_ON(ARRAY_SIZE(node_state_attrs
)-1 != NR_NODE_STATES
);
1058 ret
= subsys_system_register(&node_subsys
, cpu_root_attr_groups
);
1060 static struct notifier_block node_memory_callback_nb
= {
1061 .notifier_call
= node_memory_callback
,
1062 .priority
= NODE_CALLBACK_PRI
,
1064 register_hotmemory_notifier(&node_memory_callback_nb
);
1068 * Note: we're not going to unregister the node class if we fail
1069 * to register the node state class attribute files.
1073 postcore_initcall(register_node_type
);