]>
Commit | Line | Data |
---|---|---|
d41dee36 AW |
1 | /* |
2 | * sparse memory mappings. | |
3 | */ | |
d41dee36 AW |
4 | #include <linux/mm.h> |
5 | #include <linux/mmzone.h> | |
6 | #include <linux/bootmem.h> | |
0b0acbec | 7 | #include <linux/highmem.h> |
d41dee36 | 8 | #include <linux/module.h> |
28ae55c9 | 9 | #include <linux/spinlock.h> |
0b0acbec | 10 | #include <linux/vmalloc.h> |
d41dee36 AW |
11 | #include <asm/dma.h> |
12 | ||
13 | /* | |
14 | * Permanent SPARSEMEM data: | |
15 | * | |
16 | * 1) mem_section - memory sections, mem_map's for valid memory | |
17 | */ | |
3e347261 | 18 | #ifdef CONFIG_SPARSEMEM_EXTREME |
802f192e | 19 | struct mem_section *mem_section[NR_SECTION_ROOTS] |
22fc6ecc | 20 | ____cacheline_internodealigned_in_smp; |
3e347261 BP |
21 | #else |
22 | struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT] | |
22fc6ecc | 23 | ____cacheline_internodealigned_in_smp; |
3e347261 BP |
24 | #endif |
25 | EXPORT_SYMBOL(mem_section); | |
26 | ||
89689ae7 CL |
27 | #ifdef NODE_NOT_IN_PAGE_FLAGS |
28 | /* | |
29 | * If we did not store the node number in the page then we have to | |
30 | * do a lookup in the section_to_node_table in order to find which | |
31 | * node the page belongs to. | |
32 | */ | |
33 | #if MAX_NUMNODES <= 256 | |
34 | static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; | |
35 | #else | |
36 | static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; | |
37 | #endif | |
38 | ||
25ba77c1 | 39 | int page_to_nid(struct page *page) |
89689ae7 CL |
40 | { |
41 | return section_to_node_table[page_to_section(page)]; | |
42 | } | |
43 | EXPORT_SYMBOL(page_to_nid); | |
85770ffe AW |
44 | |
45 | static void set_section_nid(unsigned long section_nr, int nid) | |
46 | { | |
47 | section_to_node_table[section_nr] = nid; | |
48 | } | |
49 | #else /* !NODE_NOT_IN_PAGE_FLAGS */ | |
50 | static inline void set_section_nid(unsigned long section_nr, int nid) | |
51 | { | |
52 | } | |
89689ae7 CL |
53 | #endif |
54 | ||
3e347261 | 55 | #ifdef CONFIG_SPARSEMEM_EXTREME |
577a32f6 | 56 | static struct mem_section noinline __init_refok *sparse_index_alloc(int nid) |
28ae55c9 DH |
57 | { |
58 | struct mem_section *section = NULL; | |
59 | unsigned long array_size = SECTIONS_PER_ROOT * | |
60 | sizeof(struct mem_section); | |
61 | ||
39d24e64 | 62 | if (slab_is_available()) |
46a66eec MK |
63 | section = kmalloc_node(array_size, GFP_KERNEL, nid); |
64 | else | |
65 | section = alloc_bootmem_node(NODE_DATA(nid), array_size); | |
28ae55c9 DH |
66 | |
67 | if (section) | |
68 | memset(section, 0, array_size); | |
69 | ||
70 | return section; | |
3e347261 | 71 | } |
802f192e | 72 | |
a3142c8e | 73 | static int __meminit sparse_index_init(unsigned long section_nr, int nid) |
802f192e | 74 | { |
34af946a | 75 | static DEFINE_SPINLOCK(index_init_lock); |
28ae55c9 DH |
76 | unsigned long root = SECTION_NR_TO_ROOT(section_nr); |
77 | struct mem_section *section; | |
78 | int ret = 0; | |
802f192e BP |
79 | |
80 | if (mem_section[root]) | |
28ae55c9 | 81 | return -EEXIST; |
3e347261 | 82 | |
28ae55c9 DH |
83 | section = sparse_index_alloc(nid); |
84 | /* | |
85 | * This lock keeps two different sections from | |
86 | * reallocating for the same index | |
87 | */ | |
88 | spin_lock(&index_init_lock); | |
3e347261 | 89 | |
28ae55c9 DH |
90 | if (mem_section[root]) { |
91 | ret = -EEXIST; | |
92 | goto out; | |
93 | } | |
94 | ||
95 | mem_section[root] = section; | |
96 | out: | |
97 | spin_unlock(&index_init_lock); | |
98 | return ret; | |
99 | } | |
100 | #else /* !SPARSEMEM_EXTREME */ | |
101 | static inline int sparse_index_init(unsigned long section_nr, int nid) | |
102 | { | |
103 | return 0; | |
802f192e | 104 | } |
28ae55c9 DH |
105 | #endif |
106 | ||
4ca644d9 DH |
107 | /* |
108 | * Although written for the SPARSEMEM_EXTREME case, this happens | |
109 | * to also work for the flat array case becase | |
110 | * NR_SECTION_ROOTS==NR_MEM_SECTIONS. | |
111 | */ | |
112 | int __section_nr(struct mem_section* ms) | |
113 | { | |
114 | unsigned long root_nr; | |
115 | struct mem_section* root; | |
116 | ||
12783b00 MK |
117 | for (root_nr = 0; root_nr < NR_SECTION_ROOTS; root_nr++) { |
118 | root = __nr_to_section(root_nr * SECTIONS_PER_ROOT); | |
4ca644d9 DH |
119 | if (!root) |
120 | continue; | |
121 | ||
122 | if ((ms >= root) && (ms < (root + SECTIONS_PER_ROOT))) | |
123 | break; | |
124 | } | |
125 | ||
126 | return (root_nr * SECTIONS_PER_ROOT) + (ms - root); | |
127 | } | |
128 | ||
30c253e6 AW |
129 | /* |
130 | * During early boot, before section_mem_map is used for an actual | |
131 | * mem_map, we use section_mem_map to store the section's NUMA | |
132 | * node. This keeps us from having to use another data structure. The | |
133 | * node information is cleared just before we store the real mem_map. | |
134 | */ | |
135 | static inline unsigned long sparse_encode_early_nid(int nid) | |
136 | { | |
137 | return (nid << SECTION_NID_SHIFT); | |
138 | } | |
139 | ||
140 | static inline int sparse_early_nid(struct mem_section *section) | |
141 | { | |
142 | return (section->section_mem_map >> SECTION_NID_SHIFT); | |
143 | } | |
144 | ||
d41dee36 | 145 | /* Record a memory area against a node. */ |
a3142c8e | 146 | void __init memory_present(int nid, unsigned long start, unsigned long end) |
d41dee36 AW |
147 | { |
148 | unsigned long pfn; | |
149 | ||
150 | start &= PAGE_SECTION_MASK; | |
151 | for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) { | |
152 | unsigned long section = pfn_to_section_nr(pfn); | |
802f192e BP |
153 | struct mem_section *ms; |
154 | ||
155 | sparse_index_init(section, nid); | |
85770ffe | 156 | set_section_nid(section, nid); |
802f192e BP |
157 | |
158 | ms = __nr_to_section(section); | |
159 | if (!ms->section_mem_map) | |
30c253e6 AW |
160 | ms->section_mem_map = sparse_encode_early_nid(nid) | |
161 | SECTION_MARKED_PRESENT; | |
d41dee36 AW |
162 | } |
163 | } | |
164 | ||
165 | /* | |
166 | * Only used by the i386 NUMA architecures, but relatively | |
167 | * generic code. | |
168 | */ | |
169 | unsigned long __init node_memmap_size_bytes(int nid, unsigned long start_pfn, | |
170 | unsigned long end_pfn) | |
171 | { | |
172 | unsigned long pfn; | |
173 | unsigned long nr_pages = 0; | |
174 | ||
175 | for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { | |
176 | if (nid != early_pfn_to_nid(pfn)) | |
177 | continue; | |
178 | ||
179 | if (pfn_valid(pfn)) | |
180 | nr_pages += PAGES_PER_SECTION; | |
181 | } | |
182 | ||
183 | return nr_pages * sizeof(struct page); | |
184 | } | |
185 | ||
29751f69 AW |
186 | /* |
187 | * Subtle, we encode the real pfn into the mem_map such that | |
188 | * the identity pfn - section_mem_map will return the actual | |
189 | * physical page frame number. | |
190 | */ | |
191 | static unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum) | |
192 | { | |
193 | return (unsigned long)(mem_map - (section_nr_to_pfn(pnum))); | |
194 | } | |
195 | ||
196 | /* | |
197 | * We need this if we ever free the mem_maps. While not implemented yet, | |
198 | * this function is included for parity with its sibling. | |
199 | */ | |
200 | static __attribute((unused)) | |
201 | struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum) | |
202 | { | |
203 | return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum); | |
204 | } | |
205 | ||
a3142c8e | 206 | static int __meminit sparse_init_one_section(struct mem_section *ms, |
29751f69 AW |
207 | unsigned long pnum, struct page *mem_map) |
208 | { | |
209 | if (!valid_section(ms)) | |
210 | return -EINVAL; | |
211 | ||
30c253e6 | 212 | ms->section_mem_map &= ~SECTION_MAP_MASK; |
29751f69 AW |
213 | ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum); |
214 | ||
215 | return 1; | |
216 | } | |
217 | ||
dec2e6b7 | 218 | __attribute__((weak)) __init |
2e1c49db ZN |
219 | void *alloc_bootmem_high_node(pg_data_t *pgdat, unsigned long size) |
220 | { | |
221 | return NULL; | |
222 | } | |
223 | ||
a3142c8e | 224 | static struct page __init *sparse_early_mem_map_alloc(unsigned long pnum) |
29751f69 AW |
225 | { |
226 | struct page *map; | |
802f192e | 227 | struct mem_section *ms = __nr_to_section(pnum); |
30c253e6 | 228 | int nid = sparse_early_nid(ms); |
29751f69 AW |
229 | |
230 | map = alloc_remap(nid, sizeof(struct page) * PAGES_PER_SECTION); | |
231 | if (map) | |
232 | return map; | |
233 | ||
2e1c49db ZN |
234 | map = alloc_bootmem_high_node(NODE_DATA(nid), |
235 | sizeof(struct page) * PAGES_PER_SECTION); | |
236 | if (map) | |
237 | return map; | |
238 | ||
29751f69 AW |
239 | map = alloc_bootmem_node(NODE_DATA(nid), |
240 | sizeof(struct page) * PAGES_PER_SECTION); | |
241 | if (map) | |
242 | return map; | |
243 | ||
244 | printk(KERN_WARNING "%s: allocation failed\n", __FUNCTION__); | |
802f192e | 245 | ms->section_mem_map = 0; |
29751f69 AW |
246 | return NULL; |
247 | } | |
248 | ||
193faea9 SR |
249 | /* |
250 | * Allocate the accumulated non-linear sections, allocate a mem_map | |
251 | * for each and record the physical to section mapping. | |
252 | */ | |
253 | void __init sparse_init(void) | |
254 | { | |
255 | unsigned long pnum; | |
256 | struct page *map; | |
257 | ||
258 | for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) { | |
259 | if (!valid_section_nr(pnum)) | |
260 | continue; | |
261 | ||
262 | map = sparse_early_mem_map_alloc(pnum); | |
263 | if (!map) | |
264 | continue; | |
265 | sparse_init_one_section(__nr_to_section(pnum), pnum, map); | |
266 | } | |
267 | } | |
268 | ||
269 | #ifdef CONFIG_MEMORY_HOTPLUG | |
0b0acbec DH |
270 | static struct page *__kmalloc_section_memmap(unsigned long nr_pages) |
271 | { | |
272 | struct page *page, *ret; | |
273 | unsigned long memmap_size = sizeof(struct page) * nr_pages; | |
274 | ||
f2d0aa5b | 275 | page = alloc_pages(GFP_KERNEL|__GFP_NOWARN, get_order(memmap_size)); |
0b0acbec DH |
276 | if (page) |
277 | goto got_map_page; | |
278 | ||
279 | ret = vmalloc(memmap_size); | |
280 | if (ret) | |
281 | goto got_map_ptr; | |
282 | ||
283 | return NULL; | |
284 | got_map_page: | |
285 | ret = (struct page *)pfn_to_kaddr(page_to_pfn(page)); | |
286 | got_map_ptr: | |
287 | memset(ret, 0, memmap_size); | |
288 | ||
289 | return ret; | |
290 | } | |
291 | ||
292 | static int vaddr_in_vmalloc_area(void *addr) | |
293 | { | |
294 | if (addr >= (void *)VMALLOC_START && | |
295 | addr < (void *)VMALLOC_END) | |
296 | return 1; | |
297 | return 0; | |
298 | } | |
299 | ||
300 | static void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages) | |
301 | { | |
302 | if (vaddr_in_vmalloc_area(memmap)) | |
303 | vfree(memmap); | |
304 | else | |
305 | free_pages((unsigned long)memmap, | |
306 | get_order(sizeof(struct page) * nr_pages)); | |
307 | } | |
308 | ||
29751f69 AW |
309 | /* |
310 | * returns the number of sections whose mem_maps were properly | |
311 | * set. If this is <=0, then that means that the passed-in | |
312 | * map was not consumed and must be freed. | |
313 | */ | |
0b0acbec DH |
314 | int sparse_add_one_section(struct zone *zone, unsigned long start_pfn, |
315 | int nr_pages) | |
29751f69 | 316 | { |
0b0acbec DH |
317 | unsigned long section_nr = pfn_to_section_nr(start_pfn); |
318 | struct pglist_data *pgdat = zone->zone_pgdat; | |
319 | struct mem_section *ms; | |
320 | struct page *memmap; | |
321 | unsigned long flags; | |
322 | int ret; | |
29751f69 | 323 | |
0b0acbec DH |
324 | /* |
325 | * no locking for this, because it does its own | |
326 | * plus, it does a kmalloc | |
327 | */ | |
328 | sparse_index_init(section_nr, pgdat->node_id); | |
329 | memmap = __kmalloc_section_memmap(nr_pages); | |
330 | ||
331 | pgdat_resize_lock(pgdat, &flags); | |
29751f69 | 332 | |
0b0acbec DH |
333 | ms = __pfn_to_section(start_pfn); |
334 | if (ms->section_mem_map & SECTION_MARKED_PRESENT) { | |
335 | ret = -EEXIST; | |
336 | goto out; | |
337 | } | |
29751f69 AW |
338 | ms->section_mem_map |= SECTION_MARKED_PRESENT; |
339 | ||
0b0acbec DH |
340 | ret = sparse_init_one_section(ms, section_nr, memmap); |
341 | ||
0b0acbec DH |
342 | out: |
343 | pgdat_resize_unlock(pgdat, &flags); | |
46a66eec MK |
344 | if (ret <= 0) |
345 | __kfree_section_memmap(memmap, nr_pages); | |
0b0acbec | 346 | return ret; |
29751f69 | 347 | } |
a3142c8e | 348 | #endif |