]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - arch/ia64/mm/discontig.c
[PATCH] Have x86_64 use add_active_range() and free_area_init_nodes
[mirror_ubuntu-bionic-kernel.git] / arch / ia64 / mm / discontig.c
CommitLineData
1da177e4
LT
1/*
2 * Copyright (c) 2000, 2003 Silicon Graphics, Inc. All rights reserved.
3 * Copyright (c) 2001 Intel Corp.
4 * Copyright (c) 2001 Tony Luck <tony.luck@intel.com>
5 * Copyright (c) 2002 NEC Corp.
6 * Copyright (c) 2002 Kimio Suganuma <k-suganuma@da.jp.nec.com>
7 * Copyright (c) 2004 Silicon Graphics, Inc
8 * Russ Anderson <rja@sgi.com>
9 * Jesse Barnes <jbarnes@sgi.com>
10 * Jack Steiner <steiner@sgi.com>
11 */
12
13/*
14 * Platform initialization for Discontig Memory
15 */
16
17#include <linux/kernel.h>
18#include <linux/mm.h>
19#include <linux/swap.h>
20#include <linux/bootmem.h>
21#include <linux/acpi.h>
22#include <linux/efi.h>
23#include <linux/nodemask.h>
24#include <asm/pgalloc.h>
25#include <asm/tlb.h>
26#include <asm/meminit.h>
27#include <asm/numa.h>
28#include <asm/sections.h>
29
30/*
31 * Track per-node information needed to setup the boot memory allocator, the
32 * per-node areas, and the real VM.
33 */
34struct early_node_data {
35 struct ia64_node_data *node_data;
1da177e4
LT
36 unsigned long pernode_addr;
37 unsigned long pernode_size;
38 struct bootmem_data bootmem_data;
39 unsigned long num_physpages;
40 unsigned long num_dma_physpages;
41 unsigned long min_pfn;
42 unsigned long max_pfn;
43};
44
45static struct early_node_data mem_data[MAX_NUMNODES] __initdata;
564601a5 46static nodemask_t memory_less_mask __initdata;
1da177e4 47
ae5a2c1c
YG
48static pg_data_t *pgdat_list[MAX_NUMNODES];
49
1da177e4
LT
50/*
51 * To prevent cache aliasing effects, align per-node structures so that they
52 * start at addresses that are strided by node number.
53 */
acb7f672 54#define MAX_NODE_ALIGN_OFFSET (32 * 1024 * 1024)
1da177e4 55#define NODEDATA_ALIGN(addr, node) \
acb7f672
JS
56 ((((addr) + 1024*1024-1) & ~(1024*1024-1)) + \
57 (((node)*PERCPU_PAGE_SIZE) & (MAX_NODE_ALIGN_OFFSET - 1)))
1da177e4
LT
58
59/**
60 * build_node_maps - callback to setup bootmem structs for each node
61 * @start: physical start of range
62 * @len: length of range
63 * @node: node where this range resides
64 *
65 * We allocate a struct bootmem_data for each piece of memory that we wish to
66 * treat as a virtually contiguous block (i.e. each node). Each such block
67 * must start on an %IA64_GRANULE_SIZE boundary, so we round the address down
68 * if necessary. Any non-existent pages will simply be part of the virtual
69 * memmap. We also update min_low_pfn and max_low_pfn here as we receive
70 * memory ranges from the caller.
71 */
72static int __init build_node_maps(unsigned long start, unsigned long len,
73 int node)
74{
75 unsigned long cstart, epfn, end = start + len;
76 struct bootmem_data *bdp = &mem_data[node].bootmem_data;
77
78 epfn = GRANULEROUNDUP(end) >> PAGE_SHIFT;
79 cstart = GRANULEROUNDDOWN(start);
80
81 if (!bdp->node_low_pfn) {
82 bdp->node_boot_start = cstart;
83 bdp->node_low_pfn = epfn;
84 } else {
85 bdp->node_boot_start = min(cstart, bdp->node_boot_start);
86 bdp->node_low_pfn = max(epfn, bdp->node_low_pfn);
87 }
88
89 min_low_pfn = min(min_low_pfn, bdp->node_boot_start>>PAGE_SHIFT);
90 max_low_pfn = max(max_low_pfn, bdp->node_low_pfn);
91
92 return 0;
93}
94
95/**
564601a5 96 * early_nr_cpus_node - return number of cpus on a given node
1da177e4
LT
97 * @node: node to check
98 *
564601a5 99 * Count the number of cpus on @node. We can't use nr_cpus_node() yet because
1da177e4 100 * acpi_boot_init() (which builds the node_to_cpu_mask array) hasn't been
564601a5 101 * called yet. Note that node 0 will also count all non-existent cpus.
1da177e4 102 */
dd0932d9 103static int __meminit early_nr_cpus_node(int node)
1da177e4
LT
104{
105 int cpu, n = 0;
106
107 for (cpu = 0; cpu < NR_CPUS; cpu++)
108 if (node == node_cpuid[cpu].nid)
564601a5 109 n++;
1da177e4
LT
110
111 return n;
112}
113
564601a5 114/**
115 * compute_pernodesize - compute size of pernode data
116 * @node: the node id.
117 */
dd0932d9 118static unsigned long __meminit compute_pernodesize(int node)
564601a5 119{
120 unsigned long pernodesize = 0, cpus;
121
122 cpus = early_nr_cpus_node(node);
123 pernodesize += PERCPU_PAGE_SIZE * cpus;
124 pernodesize += node * L1_CACHE_BYTES;
125 pernodesize += L1_CACHE_ALIGN(sizeof(pg_data_t));
126 pernodesize += L1_CACHE_ALIGN(sizeof(struct ia64_node_data));
127 pernodesize = PAGE_ALIGN(pernodesize);
128 return pernodesize;
129}
1da177e4 130
8d7e3517
TL
131/**
132 * per_cpu_node_setup - setup per-cpu areas on each node
133 * @cpu_data: per-cpu area on this node
134 * @node: node to setup
135 *
136 * Copy the static per-cpu data into the region we just set aside and then
137 * setup __per_cpu_offset for each CPU on this node. Return a pointer to
138 * the end of the area.
139 */
140static void *per_cpu_node_setup(void *cpu_data, int node)
141{
142#ifdef CONFIG_SMP
143 int cpu;
144
145 for (cpu = 0; cpu < NR_CPUS; cpu++) {
146 if (node == node_cpuid[cpu].nid) {
147 memcpy(__va(cpu_data), __phys_per_cpu_start,
148 __per_cpu_end - __per_cpu_start);
149 __per_cpu_offset[cpu] = (char*)__va(cpu_data) -
150 __per_cpu_start;
151 cpu_data += PERCPU_PAGE_SIZE;
152 }
153 }
154#endif
155 return cpu_data;
156}
157
1da177e4 158/**
564601a5 159 * fill_pernode - initialize pernode data.
160 * @node: the node id.
161 * @pernode: physical address of pernode data
162 * @pernodesize: size of the pernode data
1da177e4 163 */
564601a5 164static void __init fill_pernode(int node, unsigned long pernode,
165 unsigned long pernodesize)
1da177e4 166{
564601a5 167 void *cpu_data;
8d7e3517 168 int cpus = early_nr_cpus_node(node);
564601a5 169 struct bootmem_data *bdp = &mem_data[node].bootmem_data;
1da177e4 170
564601a5 171 mem_data[node].pernode_addr = pernode;
172 mem_data[node].pernode_size = pernodesize;
173 memset(__va(pernode), 0, pernodesize);
1da177e4 174
564601a5 175 cpu_data = (void *)pernode;
176 pernode += PERCPU_PAGE_SIZE * cpus;
177 pernode += node * L1_CACHE_BYTES;
178
ae5a2c1c 179 pgdat_list[node] = __va(pernode);
564601a5 180 pernode += L1_CACHE_ALIGN(sizeof(pg_data_t));
181
182 mem_data[node].node_data = __va(pernode);
183 pernode += L1_CACHE_ALIGN(sizeof(struct ia64_node_data));
184
ae5a2c1c 185 pgdat_list[node]->bdata = bdp;
564601a5 186 pernode += L1_CACHE_ALIGN(sizeof(pg_data_t));
187
8d7e3517 188 cpu_data = per_cpu_node_setup(cpu_data, node);
1da177e4 189
564601a5 190 return;
191}
8d7e3517 192
1da177e4
LT
193/**
194 * find_pernode_space - allocate memory for memory map and per-node structures
195 * @start: physical start of range
196 * @len: length of range
197 * @node: node where this range resides
198 *
199 * This routine reserves space for the per-cpu data struct, the list of
200 * pg_data_ts and the per-node data struct. Each node will have something like
201 * the following in the first chunk of addr. space large enough to hold it.
202 *
203 * ________________________
204 * | |
205 * |~~~~~~~~~~~~~~~~~~~~~~~~| <-- NODEDATA_ALIGN(start, node) for the first
206 * | PERCPU_PAGE_SIZE * | start and length big enough
207 * | cpus_on_this_node | Node 0 will also have entries for all non-existent cpus.
208 * |------------------------|
209 * | local pg_data_t * |
210 * |------------------------|
211 * | local ia64_node_data |
212 * |------------------------|
213 * | ??? |
214 * |________________________|
215 *
216 * Once this space has been set aside, the bootmem maps are initialized. We
217 * could probably move the allocation of the per-cpu and ia64_node_data space
218 * outside of this function and use alloc_bootmem_node(), but doing it here
219 * is straightforward and we get the alignments we want so...
220 */
221static int __init find_pernode_space(unsigned long start, unsigned long len,
222 int node)
223{
564601a5 224 unsigned long epfn;
1da177e4 225 unsigned long pernodesize = 0, pernode, pages, mapsize;
1da177e4
LT
226 struct bootmem_data *bdp = &mem_data[node].bootmem_data;
227
228 epfn = (start + len) >> PAGE_SHIFT;
229
230 pages = bdp->node_low_pfn - (bdp->node_boot_start >> PAGE_SHIFT);
231 mapsize = bootmem_bootmap_pages(pages) << PAGE_SHIFT;
232
233 /*
234 * Make sure this memory falls within this node's usable memory
235 * since we may have thrown some away in build_maps().
236 */
237 if (start < bdp->node_boot_start || epfn > bdp->node_low_pfn)
238 return 0;
239
240 /* Don't setup this node's local space twice... */
241 if (mem_data[node].pernode_addr)
242 return 0;
243
244 /*
245 * Calculate total size needed, incl. what's necessary
246 * for good alignment and alias prevention.
247 */
564601a5 248 pernodesize = compute_pernodesize(node);
1da177e4
LT
249 pernode = NODEDATA_ALIGN(start, node);
250
251 /* Is this range big enough for what we want to store here? */
564601a5 252 if (start + len > (pernode + pernodesize + mapsize))
253 fill_pernode(node, pernode, pernodesize);
1da177e4
LT
254
255 return 0;
256}
257
258/**
259 * free_node_bootmem - free bootmem allocator memory for use
260 * @start: physical start of range
261 * @len: length of range
262 * @node: node where this range resides
263 *
264 * Simply calls the bootmem allocator to free the specified ranged from
265 * the given pg_data_t's bdata struct. After this function has been called
266 * for all the entries in the EFI memory map, the bootmem allocator will
267 * be ready to service allocation requests.
268 */
269static int __init free_node_bootmem(unsigned long start, unsigned long len,
270 int node)
271{
ae5a2c1c 272 free_bootmem_node(pgdat_list[node], start, len);
1da177e4
LT
273
274 return 0;
275}
276
277/**
278 * reserve_pernode_space - reserve memory for per-node space
279 *
280 * Reserve the space used by the bootmem maps & per-node space in the boot
281 * allocator so that when we actually create the real mem maps we don't
282 * use their memory.
283 */
284static void __init reserve_pernode_space(void)
285{
286 unsigned long base, size, pages;
287 struct bootmem_data *bdp;
288 int node;
289
290 for_each_online_node(node) {
ae5a2c1c 291 pg_data_t *pdp = pgdat_list[node];
1da177e4 292
564601a5 293 if (node_isset(node, memory_less_mask))
294 continue;
295
1da177e4
LT
296 bdp = pdp->bdata;
297
298 /* First the bootmem_map itself */
299 pages = bdp->node_low_pfn - (bdp->node_boot_start>>PAGE_SHIFT);
300 size = bootmem_bootmap_pages(pages) << PAGE_SHIFT;
301 base = __pa(bdp->node_bootmem_map);
302 reserve_bootmem_node(pdp, base, size);
303
304 /* Now the per-node space */
305 size = mem_data[node].pernode_size;
306 base = __pa(mem_data[node].pernode_addr);
307 reserve_bootmem_node(pdp, base, size);
308 }
309}
310
7049027c
YG
311static void __meminit scatter_node_data(void)
312{
313 pg_data_t **dst;
314 int node;
315
dd8041f1
YG
316 /*
317 * for_each_online_node() can't be used at here.
318 * node_online_map is not set for hot-added nodes at this time,
319 * because we are halfway through initialization of the new node's
320 * structures. If for_each_online_node() is used, a new node's
321 * pg_data_ptrs will be not initialized. Insted of using it,
322 * pgdat_list[] is checked.
323 */
324 for_each_node(node) {
325 if (pgdat_list[node]) {
326 dst = LOCAL_DATA_ADDR(pgdat_list[node])->pg_data_ptrs;
327 memcpy(dst, pgdat_list, sizeof(pgdat_list));
328 }
7049027c
YG
329 }
330}
331
1da177e4
LT
332/**
333 * initialize_pernode_data - fixup per-cpu & per-node pointers
334 *
335 * Each node's per-node area has a copy of the global pg_data_t list, so
336 * we copy that to each node here, as well as setting the per-cpu pointer
337 * to the local node data structure. The active_cpus field of the per-node
338 * structure gets setup by the platform_cpu_init() function later.
339 */
340static void __init initialize_pernode_data(void)
341{
8d7e3517 342 int cpu, node;
1da177e4 343
7049027c
YG
344 scatter_node_data();
345
8d7e3517 346#ifdef CONFIG_SMP
1da177e4
LT
347 /* Set the node_data pointer for each per-cpu struct */
348 for (cpu = 0; cpu < NR_CPUS; cpu++) {
349 node = node_cpuid[cpu].nid;
350 per_cpu(cpu_info, cpu).node_data = mem_data[node].node_data;
351 }
8d7e3517
TL
352#else
353 {
354 struct cpuinfo_ia64 *cpu0_cpu_info;
355 cpu = 0;
356 node = node_cpuid[cpu].nid;
357 cpu0_cpu_info = (struct cpuinfo_ia64 *)(__phys_per_cpu_start +
358 ((char *)&per_cpu__cpu_info - __per_cpu_start));
359 cpu0_cpu_info->node_data = mem_data[node].node_data;
360 }
361#endif /* CONFIG_SMP */
1da177e4
LT
362}
363
564601a5 364/**
365 * memory_less_node_alloc - * attempt to allocate memory on the best NUMA slit
366 * node but fall back to any other node when __alloc_bootmem_node fails
367 * for best.
368 * @nid: node id
369 * @pernodesize: size of this node's pernode data
564601a5 370 */
97835245 371static void __init *memory_less_node_alloc(int nid, unsigned long pernodesize)
564601a5 372{
373 void *ptr = NULL;
374 u8 best = 0xff;
97835245 375 int bestnode = -1, node, anynode = 0;
564601a5 376
377 for_each_online_node(node) {
378 if (node_isset(node, memory_less_mask))
379 continue;
380 else if (node_distance(nid, node) < best) {
381 best = node_distance(nid, node);
382 bestnode = node;
383 }
97835245 384 anynode = node;
564601a5 385 }
386
97835245
BP
387 if (bestnode == -1)
388 bestnode = anynode;
389
ae5a2c1c 390 ptr = __alloc_bootmem_node(pgdat_list[bestnode], pernodesize,
97835245 391 PERCPU_PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
564601a5 392
564601a5 393 return ptr;
394}
395
564601a5 396/**
397 * memory_less_nodes - allocate and initialize CPU only nodes pernode
398 * information.
399 */
400static void __init memory_less_nodes(void)
401{
402 unsigned long pernodesize;
403 void *pernode;
404 int node;
405
406 for_each_node_mask(node, memory_less_mask) {
407 pernodesize = compute_pernodesize(node);
97835245 408 pernode = memory_less_node_alloc(node, pernodesize);
564601a5 409 fill_pernode(node, __pa(pernode), pernodesize);
410 }
411
412 return;
413}
414
2d4b1fa2
BP
415#ifdef CONFIG_SPARSEMEM
416/**
417 * register_sparse_mem - notify SPARSEMEM that this memory range exists.
418 * @start: physical start of range
419 * @end: physical end of range
420 * @arg: unused
421 *
422 * Simply calls SPARSEMEM to register memory section(s).
423 */
424static int __init register_sparse_mem(unsigned long start, unsigned long end,
425 void *arg)
426{
427 int nid;
428
429 start = __pa(start) >> PAGE_SHIFT;
430 end = __pa(end) >> PAGE_SHIFT;
431 nid = early_pfn_to_nid(start);
432 memory_present(nid, start, end);
433
434 return 0;
435}
436
437static void __init arch_sparse_init(void)
438{
439 efi_memmap_walk(register_sparse_mem, NULL);
440 sparse_init();
441}
442#else
443#define arch_sparse_init() do {} while (0)
444#endif
445
1da177e4
LT
446/**
447 * find_memory - walk the EFI memory map and setup the bootmem allocator
448 *
449 * Called early in boot to setup the bootmem allocator, and to
450 * allocate the per-cpu and per-node structures.
451 */
452void __init find_memory(void)
453{
454 int node;
455
456 reserve_memory();
457
458 if (num_online_nodes() == 0) {
459 printk(KERN_ERR "node info missing!\n");
460 node_set_online(0);
461 }
462
564601a5 463 nodes_or(memory_less_mask, memory_less_mask, node_online_map);
1da177e4
LT
464 min_low_pfn = -1;
465 max_low_pfn = 0;
466
1da177e4
LT
467 /* These actually end up getting called by call_pernode_memory() */
468 efi_memmap_walk(filter_rsvd_memory, build_node_maps);
469 efi_memmap_walk(filter_rsvd_memory, find_pernode_space);
470
564601a5 471 for_each_online_node(node)
472 if (mem_data[node].bootmem_data.node_low_pfn) {
473 node_clear(node, memory_less_mask);
474 mem_data[node].min_pfn = ~0UL;
475 }
1da177e4
LT
476 /*
477 * Initialize the boot memory maps in reverse order since that's
478 * what the bootmem allocator expects
479 */
480 for (node = MAX_NUMNODES - 1; node >= 0; node--) {
481 unsigned long pernode, pernodesize, map;
482 struct bootmem_data *bdp;
483
484 if (!node_online(node))
485 continue;
564601a5 486 else if (node_isset(node, memory_less_mask))
487 continue;
1da177e4
LT
488
489 bdp = &mem_data[node].bootmem_data;
490 pernode = mem_data[node].pernode_addr;
491 pernodesize = mem_data[node].pernode_size;
492 map = pernode + pernodesize;
493
ae5a2c1c 494 init_bootmem_node(pgdat_list[node],
1da177e4
LT
495 map>>PAGE_SHIFT,
496 bdp->node_boot_start>>PAGE_SHIFT,
497 bdp->node_low_pfn);
498 }
499
500 efi_memmap_walk(filter_rsvd_memory, free_node_bootmem);
501
502 reserve_pernode_space();
564601a5 503 memory_less_nodes();
1da177e4
LT
504 initialize_pernode_data();
505
506 max_pfn = max_low_pfn;
507
508 find_initrd();
509}
510
8d7e3517 511#ifdef CONFIG_SMP
1da177e4
LT
512/**
513 * per_cpu_init - setup per-cpu variables
514 *
515 * find_pernode_space() does most of this already, we just need to set
516 * local_per_cpu_offset
517 */
244fd545 518void __cpuinit *per_cpu_init(void)
1da177e4
LT
519{
520 int cpu;
ff741906
AR
521 static int first_time = 1;
522
1da177e4 523
8d7e3517
TL
524 if (smp_processor_id() != 0)
525 return __per_cpu_start + __per_cpu_offset[smp_processor_id()];
526
ff741906
AR
527 if (first_time) {
528 first_time = 0;
529 for (cpu = 0; cpu < NR_CPUS; cpu++)
530 per_cpu(local_per_cpu_offset, cpu) = __per_cpu_offset[cpu];
531 }
1da177e4
LT
532
533 return __per_cpu_start + __per_cpu_offset[smp_processor_id()];
534}
8d7e3517 535#endif /* CONFIG_SMP */
1da177e4
LT
536
537/**
538 * show_mem - give short summary of memory stats
539 *
540 * Shows a simple page count of reserved and used pages in the system.
541 * For discontig machines, it does this on a per-pgdat basis.
542 */
543void show_mem(void)
544{
545 int i, total_reserved = 0;
546 int total_shared = 0, total_cached = 0;
547 unsigned long total_present = 0;
548 pg_data_t *pgdat;
549
550 printk("Mem-info:\n");
551 show_free_areas();
552 printk("Free swap: %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10));
ec936fc5 553 for_each_online_pgdat(pgdat) {
208d54e5
DH
554 unsigned long present;
555 unsigned long flags;
1da177e4 556 int shared = 0, cached = 0, reserved = 0;
208d54e5 557
1da177e4 558 printk("Node ID: %d\n", pgdat->node_id);
208d54e5
DH
559 pgdat_resize_lock(pgdat, &flags);
560 present = pgdat->node_present_pages;
1da177e4 561 for(i = 0; i < pgdat->node_spanned_pages; i++) {
2d4b1fa2
BP
562 struct page *page;
563 if (pfn_valid(pgdat->node_start_pfn + i))
564 page = pfn_to_page(pgdat->node_start_pfn + i);
ace1d816 565 else {
e44e41d0
BP
566 i = vmemmap_find_next_valid_pfn(pgdat->node_id,
567 i) - 1;
1da177e4 568 continue;
ace1d816 569 }
408fde81 570 if (PageReserved(page))
1da177e4 571 reserved++;
408fde81 572 else if (PageSwapCache(page))
1da177e4 573 cached++;
408fde81
DH
574 else if (page_count(page))
575 shared += page_count(page)-1;
1da177e4 576 }
208d54e5 577 pgdat_resize_unlock(pgdat, &flags);
1da177e4
LT
578 total_present += present;
579 total_reserved += reserved;
580 total_cached += cached;
581 total_shared += shared;
582 printk("\t%ld pages of RAM\n", present);
583 printk("\t%d reserved pages\n", reserved);
584 printk("\t%d pages shared\n", shared);
585 printk("\t%d pages swap cached\n", cached);
586 }
587 printk("%ld pages of RAM\n", total_present);
588 printk("%d reserved pages\n", total_reserved);
589 printk("%d pages shared\n", total_shared);
590 printk("%d pages swap cached\n", total_cached);
fde740e4
RH
591 printk("Total of %ld pages in page table cache\n",
592 pgtable_quicklist_total_size());
1da177e4
LT
593 printk("%d free buffer pages\n", nr_free_buffer_pages());
594}
595
596/**
597 * call_pernode_memory - use SRAT to call callback functions with node info
598 * @start: physical start of range
599 * @len: length of range
600 * @arg: function to call for each range
601 *
602 * efi_memmap_walk() knows nothing about layout of memory across nodes. Find
603 * out to which node a block of memory belongs. Ignore memory that we cannot
604 * identify, and split blocks that run across multiple nodes.
605 *
606 * Take this opportunity to round the start address up and the end address
607 * down to page boundaries.
608 */
609void call_pernode_memory(unsigned long start, unsigned long len, void *arg)
610{
611 unsigned long rs, re, end = start + len;
612 void (*func)(unsigned long, unsigned long, int);
613 int i;
614
615 start = PAGE_ALIGN(start);
616 end &= PAGE_MASK;
617 if (start >= end)
618 return;
619
620 func = arg;
621
622 if (!num_node_memblks) {
623 /* No SRAT table, so assume one node (node 0) */
624 if (start < end)
625 (*func)(start, end - start, 0);
626 return;
627 }
628
629 for (i = 0; i < num_node_memblks; i++) {
630 rs = max(start, node_memblk[i].start_paddr);
631 re = min(end, node_memblk[i].start_paddr +
632 node_memblk[i].size);
633
634 if (rs < re)
635 (*func)(rs, re - rs, node_memblk[i].nid);
636
637 if (re == end)
638 break;
639 }
640}
641
642/**
643 * count_node_pages - callback to build per-node memory info structures
644 * @start: physical start of range
645 * @len: length of range
646 * @node: node where this range resides
647 *
648 * Each node has it's own number of physical pages, DMAable pages, start, and
649 * end page frame number. This routine will be called by call_pernode_memory()
650 * for each piece of usable memory and will setup these values for each node.
651 * Very similar to build_maps().
652 */
653static __init int count_node_pages(unsigned long start, unsigned long len, int node)
654{
655 unsigned long end = start + len;
656
657 mem_data[node].num_physpages += len >> PAGE_SHIFT;
658 if (start <= __pa(MAX_DMA_ADDRESS))
659 mem_data[node].num_dma_physpages +=
660 (min(end, __pa(MAX_DMA_ADDRESS)) - start) >>PAGE_SHIFT;
661 start = GRANULEROUNDDOWN(start);
662 start = ORDERROUNDDOWN(start);
663 end = GRANULEROUNDUP(end);
664 mem_data[node].max_pfn = max(mem_data[node].max_pfn,
665 end >> PAGE_SHIFT);
666 mem_data[node].min_pfn = min(mem_data[node].min_pfn,
667 start >> PAGE_SHIFT);
668
669 return 0;
670}
671
672/**
673 * paging_init - setup page tables
674 *
675 * paging_init() sets up the page tables for each node of the system and frees
676 * the bootmem allocator memory for general use.
677 */
678void __init paging_init(void)
679{
680 unsigned long max_dma;
681 unsigned long zones_size[MAX_NR_ZONES];
682 unsigned long zholes_size[MAX_NR_ZONES];
683 unsigned long pfn_offset = 0;
684 int node;
685
686 max_dma = virt_to_phys((void *) MAX_DMA_ADDRESS) >> PAGE_SHIFT;
687
2d4b1fa2
BP
688 arch_sparse_init();
689
1da177e4
LT
690 efi_memmap_walk(filter_rsvd_memory, count_node_pages);
691
2d4b1fa2 692#ifdef CONFIG_VIRTUAL_MEM_MAP
921eea1c
BP
693 vmalloc_end -= PAGE_ALIGN(ALIGN(max_low_pfn, MAX_ORDER_NR_PAGES) *
694 sizeof(struct page));
564601a5 695 vmem_map = (struct page *) vmalloc_end;
696 efi_memmap_walk(create_mem_map_page_table, NULL);
697 printk("Virtual mem_map starts at 0x%p\n", vmem_map);
2d4b1fa2 698#endif
564601a5 699
1da177e4
LT
700 for_each_online_node(node) {
701 memset(zones_size, 0, sizeof(zones_size));
702 memset(zholes_size, 0, sizeof(zholes_size));
703
704 num_physpages += mem_data[node].num_physpages;
705
706 if (mem_data[node].min_pfn >= max_dma) {
707 /* All of this node's memory is above ZONE_DMA */
708 zones_size[ZONE_NORMAL] = mem_data[node].max_pfn -
709 mem_data[node].min_pfn;
710 zholes_size[ZONE_NORMAL] = mem_data[node].max_pfn -
711 mem_data[node].min_pfn -
712 mem_data[node].num_physpages;
713 } else if (mem_data[node].max_pfn < max_dma) {
714 /* All of this node's memory is in ZONE_DMA */
715 zones_size[ZONE_DMA] = mem_data[node].max_pfn -
716 mem_data[node].min_pfn;
717 zholes_size[ZONE_DMA] = mem_data[node].max_pfn -
718 mem_data[node].min_pfn -
719 mem_data[node].num_dma_physpages;
720 } else {
721 /* This node has memory in both zones */
722 zones_size[ZONE_DMA] = max_dma -
723 mem_data[node].min_pfn;
724 zholes_size[ZONE_DMA] = zones_size[ZONE_DMA] -
725 mem_data[node].num_dma_physpages;
726 zones_size[ZONE_NORMAL] = mem_data[node].max_pfn -
727 max_dma;
728 zholes_size[ZONE_NORMAL] = zones_size[ZONE_NORMAL] -
729 (mem_data[node].num_physpages -
730 mem_data[node].num_dma_physpages);
731 }
732
1da177e4
LT
733 pfn_offset = mem_data[node].min_pfn;
734
2d4b1fa2 735#ifdef CONFIG_VIRTUAL_MEM_MAP
1da177e4 736 NODE_DATA(node)->node_mem_map = vmem_map + pfn_offset;
2d4b1fa2 737#endif
1da177e4
LT
738 free_area_init_node(node, NODE_DATA(node), zones_size,
739 pfn_offset, zholes_size);
740 }
741
742 zero_page_memmap_ptr = virt_to_page(ia64_imva(empty_zero_page));
743}
7049027c 744
dd0932d9
YG
745pg_data_t *arch_alloc_nodedata(int nid)
746{
747 unsigned long size = compute_pernodesize(nid);
748
749 return kzalloc(size, GFP_KERNEL);
750}
751
752void arch_free_nodedata(pg_data_t *pgdat)
753{
754 kfree(pgdat);
755}
756
7049027c
YG
757void arch_refresh_nodedata(int update_node, pg_data_t *update_pgdat)
758{
759 pgdat_list[update_node] = update_pgdat;
760 scatter_node_data();
761}