]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - mm/cma.c
staging: fsl-dpaa2/eth: Only store bpid in priv struct
[mirror_ubuntu-bionic-kernel.git] / mm / cma.c
CommitLineData
a254129e
JK
1/*
2 * Contiguous Memory Allocator
3 *
4 * Copyright (c) 2010-2011 by Samsung Electronics.
5 * Copyright IBM Corporation, 2013
6 * Copyright LG Electronics Inc., 2014
7 * Written by:
8 * Marek Szyprowski <m.szyprowski@samsung.com>
9 * Michal Nazarewicz <mina86@mina86.com>
10 * Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
11 * Joonsoo Kim <iamjoonsoo.kim@lge.com>
12 *
13 * This program is free software; you can redistribute it and/or
14 * modify it under the terms of the GNU General Public License as
15 * published by the Free Software Foundation; either version 2 of the
16 * License or (at your optional) any later version of the license.
17 */
18
19#define pr_fmt(fmt) "cma: " fmt
20
21#ifdef CONFIG_CMA_DEBUG
22#ifndef DEBUG
23# define DEBUG
24#endif
25#endif
99e8ea6c 26#define CREATE_TRACE_POINTS
a254129e
JK
27
28#include <linux/memblock.h>
29#include <linux/err.h>
30#include <linux/mm.h>
31#include <linux/mutex.h>
32#include <linux/sizes.h>
33#include <linux/slab.h>
34#include <linux/log2.h>
35#include <linux/cma.h>
f7426b98 36#include <linux/highmem.h>
620951e2 37#include <linux/io.h>
99e8ea6c 38#include <trace/events/cma.h>
a254129e 39
28b24c1f
SL
40#include "cma.h"
41
42struct cma cma_areas[MAX_CMA_AREAS];
43unsigned cma_area_count;
a254129e
JK
44static DEFINE_MUTEX(cma_mutex);
45
ac173824 46phys_addr_t cma_get_base(const struct cma *cma)
a254129e
JK
47{
48 return PFN_PHYS(cma->base_pfn);
49}
50
ac173824 51unsigned long cma_get_size(const struct cma *cma)
a254129e
JK
52{
53 return cma->count << PAGE_SHIFT;
54}
55
f318dd08
LA
56const char *cma_get_name(const struct cma *cma)
57{
58 return cma->name ? cma->name : "(undefined)";
59}
60
ac173824
SL
61static unsigned long cma_bitmap_aligned_mask(const struct cma *cma,
62 int align_order)
a254129e 63{
68faed63
WY
64 if (align_order <= cma->order_per_bit)
65 return 0;
66 return (1UL << (align_order - cma->order_per_bit)) - 1;
a254129e
JK
67}
68
850fc430
DP
69/*
70 * Find a PFN aligned to the specified order and return an offset represented in
71 * order_per_bits.
72 */
ac173824
SL
73static unsigned long cma_bitmap_aligned_offset(const struct cma *cma,
74 int align_order)
b5be83e3 75{
b5be83e3
GF
76 if (align_order <= cma->order_per_bit)
77 return 0;
850fc430
DP
78
79 return (ALIGN(cma->base_pfn, (1UL << align_order))
80 - cma->base_pfn) >> cma->order_per_bit;
b5be83e3
GF
81}
82
ac173824
SL
83static unsigned long cma_bitmap_pages_to_bits(const struct cma *cma,
84 unsigned long pages)
a254129e
JK
85{
86 return ALIGN(pages, 1UL << cma->order_per_bit) >> cma->order_per_bit;
87}
88
ac173824
SL
89static void cma_clear_bitmap(struct cma *cma, unsigned long pfn,
90 unsigned int count)
a254129e
JK
91{
92 unsigned long bitmap_no, bitmap_count;
93
94 bitmap_no = (pfn - cma->base_pfn) >> cma->order_per_bit;
95 bitmap_count = cma_bitmap_pages_to_bits(cma, count);
96
97 mutex_lock(&cma->lock);
98 bitmap_clear(cma->bitmap, bitmap_no, bitmap_count);
99 mutex_unlock(&cma->lock);
100}
101
102static int __init cma_activate_area(struct cma *cma)
103{
104 int bitmap_size = BITS_TO_LONGS(cma_bitmap_maxno(cma)) * sizeof(long);
105 unsigned long base_pfn = cma->base_pfn, pfn = base_pfn;
106 unsigned i = cma->count >> pageblock_order;
107 struct zone *zone;
108
109 cma->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
110
111 if (!cma->bitmap)
112 return -ENOMEM;
113
114 WARN_ON_ONCE(!pfn_valid(pfn));
115 zone = page_zone(pfn_to_page(pfn));
116
117 do {
118 unsigned j;
119
120 base_pfn = pfn;
121 for (j = pageblock_nr_pages; j; --j, pfn++) {
122 WARN_ON_ONCE(!pfn_valid(pfn));
123 /*
124 * alloc_contig_range requires the pfn range
125 * specified to be in the same zone. Make this
126 * simple by forcing the entire CMA resv range
127 * to be in the same zone.
128 */
129 if (page_zone(pfn_to_page(pfn)) != zone)
130 goto err;
131 }
132 init_cma_reserved_pageblock(pfn_to_page(base_pfn));
133 } while (--i);
134
135 mutex_init(&cma->lock);
26b02a1f
SL
136
137#ifdef CONFIG_CMA_DEBUGFS
138 INIT_HLIST_HEAD(&cma->mem_head);
139 spin_lock_init(&cma->mem_head_lock);
140#endif
141
a254129e
JK
142 return 0;
143
144err:
145 kfree(cma->bitmap);
f022d8cb 146 cma->count = 0;
a254129e
JK
147 return -EINVAL;
148}
149
150static int __init cma_init_reserved_areas(void)
151{
152 int i;
153
154 for (i = 0; i < cma_area_count; i++) {
155 int ret = cma_activate_area(&cma_areas[i]);
156
157 if (ret)
158 return ret;
159 }
160
161 return 0;
162}
163core_initcall(cma_init_reserved_areas);
164
de9e14ee
MS
165/**
166 * cma_init_reserved_mem() - create custom contiguous area from reserved memory
167 * @base: Base address of the reserved area
168 * @size: Size of the reserved area (in bytes),
169 * @order_per_bit: Order of pages represented by one bit on bitmap.
170 * @res_cma: Pointer to store the created cma region.
171 *
172 * This function creates custom contiguous area from already reserved memory.
173 */
174int __init cma_init_reserved_mem(phys_addr_t base, phys_addr_t size,
ac173824 175 unsigned int order_per_bit,
f318dd08 176 const char *name,
ac173824 177 struct cma **res_cma)
de9e14ee
MS
178{
179 struct cma *cma;
180 phys_addr_t alignment;
181
182 /* Sanity checks */
183 if (cma_area_count == ARRAY_SIZE(cma_areas)) {
184 pr_err("Not enough slots for CMA reserved regions!\n");
185 return -ENOSPC;
186 }
187
188 if (!size || !memblock_is_region_reserved(base, size))
189 return -EINVAL;
190
0f96ae29 191 /* ensure minimal alignment required by mm core */
badbda53
SR
192 alignment = PAGE_SIZE <<
193 max_t(unsigned long, MAX_ORDER - 1, pageblock_order);
de9e14ee
MS
194
195 /* alignment should be aligned with order_per_bit */
196 if (!IS_ALIGNED(alignment >> PAGE_SHIFT, 1 << order_per_bit))
197 return -EINVAL;
198
199 if (ALIGN(base, alignment) != base || ALIGN(size, alignment) != size)
200 return -EINVAL;
201
202 /*
203 * Each reserved area must be initialised later, when more kernel
204 * subsystems (like slab allocator) are available.
205 */
206 cma = &cma_areas[cma_area_count];
f318dd08
LA
207 if (name) {
208 cma->name = name;
209 } else {
210 cma->name = kasprintf(GFP_KERNEL, "cma%d\n", cma_area_count);
211 if (!cma->name)
212 return -ENOMEM;
213 }
de9e14ee
MS
214 cma->base_pfn = PFN_DOWN(base);
215 cma->count = size >> PAGE_SHIFT;
216 cma->order_per_bit = order_per_bit;
217 *res_cma = cma;
218 cma_area_count++;
94737a85 219 totalcma_pages += (size / PAGE_SIZE);
de9e14ee
MS
220
221 return 0;
222}
223
a254129e
JK
224/**
225 * cma_declare_contiguous() - reserve custom contiguous area
a254129e 226 * @base: Base address of the reserved area optional, use 0 for any
c1f733aa 227 * @size: Size of the reserved area (in bytes),
a254129e
JK
228 * @limit: End address of the reserved memory (optional, 0 for any).
229 * @alignment: Alignment for the CMA area, should be power of 2 or zero
230 * @order_per_bit: Order of pages represented by one bit on bitmap.
a254129e 231 * @fixed: hint about where to place the reserved area
c1f733aa 232 * @res_cma: Pointer to store the created cma region.
a254129e
JK
233 *
234 * This function reserves memory from early allocator. It should be
235 * called by arch specific code once the early allocator (memblock or bootmem)
236 * has been activated and all other subsystems have already allocated/reserved
237 * memory. This function allows to create custom reserved areas.
238 *
239 * If @fixed is true, reserve contiguous area at exactly @base. If false,
240 * reserve in range from @base to @limit.
241 */
c1f733aa
JK
242int __init cma_declare_contiguous(phys_addr_t base,
243 phys_addr_t size, phys_addr_t limit,
a254129e 244 phys_addr_t alignment, unsigned int order_per_bit,
f318dd08 245 bool fixed, const char *name, struct cma **res_cma)
a254129e 246{
f7426b98 247 phys_addr_t memblock_end = memblock_end_of_DRAM();
6b101e2a 248 phys_addr_t highmem_start;
a254129e
JK
249 int ret = 0;
250
6b101e2a 251 /*
2dece445
LA
252 * We can't use __pa(high_memory) directly, since high_memory
253 * isn't a valid direct map VA, and DEBUG_VIRTUAL will (validly)
254 * complain. Find the boundary by adding one to the last valid
255 * address.
6b101e2a 256 */
2dece445 257 highmem_start = __pa(high_memory - 1) + 1;
56fa4f60
LP
258 pr_debug("%s(size %pa, base %pa, limit %pa alignment %pa)\n",
259 __func__, &size, &base, &limit, &alignment);
a254129e
JK
260
261 if (cma_area_count == ARRAY_SIZE(cma_areas)) {
262 pr_err("Not enough slots for CMA reserved regions!\n");
263 return -ENOSPC;
264 }
265
266 if (!size)
267 return -EINVAL;
268
269 if (alignment && !is_power_of_2(alignment))
270 return -EINVAL;
271
272 /*
273 * Sanitise input arguments.
274 * Pages both ends in CMA area could be merged into adjacent unmovable
275 * migratetype page by page allocator's buddy algorithm. In the case,
276 * you couldn't get a contiguous memory, which is not what we want.
277 */
badbda53
SR
278 alignment = max(alignment, (phys_addr_t)PAGE_SIZE <<
279 max_t(unsigned long, MAX_ORDER - 1, pageblock_order));
a254129e
JK
280 base = ALIGN(base, alignment);
281 size = ALIGN(size, alignment);
282 limit &= ~(alignment - 1);
283
800a85d3
LP
284 if (!base)
285 fixed = false;
286
a254129e
JK
287 /* size should be aligned with order_per_bit */
288 if (!IS_ALIGNED(size >> PAGE_SHIFT, 1 << order_per_bit))
289 return -EINVAL;
290
f7426b98 291 /*
16195ddd
LP
292 * If allocating at a fixed base the request region must not cross the
293 * low/high memory boundary.
f7426b98 294 */
16195ddd 295 if (fixed && base < highmem_start && base + size > highmem_start) {
f7426b98 296 ret = -EINVAL;
56fa4f60
LP
297 pr_err("Region at %pa defined on low/high memory boundary (%pa)\n",
298 &base, &highmem_start);
f7426b98
MS
299 goto err;
300 }
301
16195ddd
LP
302 /*
303 * If the limit is unspecified or above the memblock end, its effective
304 * value will be the memblock end. Set it explicitly to simplify further
305 * checks.
306 */
307 if (limit == 0 || limit > memblock_end)
308 limit = memblock_end;
309
a254129e 310 /* Reserve memory */
800a85d3 311 if (fixed) {
a254129e
JK
312 if (memblock_is_region_reserved(base, size) ||
313 memblock_reserve(base, size) < 0) {
314 ret = -EBUSY;
315 goto err;
316 }
317 } else {
16195ddd
LP
318 phys_addr_t addr = 0;
319
320 /*
321 * All pages in the reserved area must come from the same zone.
322 * If the requested region crosses the low/high memory boundary,
323 * try allocating from high memory first and fall back to low
324 * memory in case of failure.
325 */
326 if (base < highmem_start && limit > highmem_start) {
327 addr = memblock_alloc_range(size, alignment,
fc6daaf9
TL
328 highmem_start, limit,
329 MEMBLOCK_NONE);
16195ddd
LP
330 limit = highmem_start;
331 }
332
a254129e 333 if (!addr) {
16195ddd 334 addr = memblock_alloc_range(size, alignment, base,
fc6daaf9
TL
335 limit,
336 MEMBLOCK_NONE);
16195ddd
LP
337 if (!addr) {
338 ret = -ENOMEM;
339 goto err;
340 }
a254129e 341 }
16195ddd 342
620951e2
TR
343 /*
344 * kmemleak scans/reads tracked objects for pointers to other
345 * objects but this address isn't mapped and accessible
346 */
9099daed 347 kmemleak_ignore_phys(addr);
16195ddd 348 base = addr;
a254129e
JK
349 }
350
f318dd08 351 ret = cma_init_reserved_mem(base, size, order_per_bit, name, res_cma);
de9e14ee
MS
352 if (ret)
353 goto err;
a254129e 354
56fa4f60
LP
355 pr_info("Reserved %ld MiB at %pa\n", (unsigned long)size / SZ_1M,
356 &base);
a254129e
JK
357 return 0;
358
359err:
0de9d2eb 360 pr_err("Failed to reserve %ld MiB\n", (unsigned long)size / SZ_1M);
a254129e
JK
361 return ret;
362}
363
dbe43d4d
JK
364#ifdef CONFIG_CMA_DEBUG
365static void cma_debug_show_areas(struct cma *cma)
366{
367 unsigned long next_zero_bit, next_set_bit;
368 unsigned long start = 0;
369 unsigned int nr_zero, nr_total = 0;
370
371 mutex_lock(&cma->lock);
372 pr_info("number of available pages: ");
373 for (;;) {
374 next_zero_bit = find_next_zero_bit(cma->bitmap, cma->count, start);
375 if (next_zero_bit >= cma->count)
376 break;
377 next_set_bit = find_next_bit(cma->bitmap, cma->count, next_zero_bit);
378 nr_zero = next_set_bit - next_zero_bit;
379 pr_cont("%s%u@%lu", nr_total ? "+" : "", nr_zero, next_zero_bit);
380 nr_total += nr_zero;
381 start = next_zero_bit + nr_zero;
382 }
383 pr_cont("=> %u free of %lu total pages\n", nr_total, cma->count);
384 mutex_unlock(&cma->lock);
385}
386#else
387static inline void cma_debug_show_areas(struct cma *cma) { }
388#endif
389
a254129e
JK
390/**
391 * cma_alloc() - allocate pages from contiguous area
392 * @cma: Contiguous memory region for which the allocation is performed.
393 * @count: Requested number of pages.
394 * @align: Requested alignment of pages (in PAGE_SIZE order).
395 *
396 * This function allocates part of contiguous memory on specific
397 * contiguous memory area.
398 */
e2f466e3
LS
399struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align,
400 gfp_t gfp_mask)
a254129e 401{
3acaea68
AM
402 unsigned long mask, offset;
403 unsigned long pfn = -1;
404 unsigned long start = 0;
a254129e
JK
405 unsigned long bitmap_maxno, bitmap_no, bitmap_count;
406 struct page *page = NULL;
dbe43d4d 407 int ret = -ENOMEM;
a254129e
JK
408
409 if (!cma || !cma->count)
410 return NULL;
411
67a2e213 412 pr_debug("%s(cma %p, count %zu, align %d)\n", __func__, (void *)cma,
a254129e
JK
413 count, align);
414
415 if (!count)
416 return NULL;
417
418 mask = cma_bitmap_aligned_mask(cma, align);
b5be83e3 419 offset = cma_bitmap_aligned_offset(cma, align);
a254129e
JK
420 bitmap_maxno = cma_bitmap_maxno(cma);
421 bitmap_count = cma_bitmap_pages_to_bits(cma, count);
422
6b36ba59
SH
423 if (bitmap_count > bitmap_maxno)
424 return NULL;
425
a254129e
JK
426 for (;;) {
427 mutex_lock(&cma->lock);
b5be83e3
GF
428 bitmap_no = bitmap_find_next_zero_area_off(cma->bitmap,
429 bitmap_maxno, start, bitmap_count, mask,
430 offset);
a254129e
JK
431 if (bitmap_no >= bitmap_maxno) {
432 mutex_unlock(&cma->lock);
433 break;
434 }
435 bitmap_set(cma->bitmap, bitmap_no, bitmap_count);
436 /*
437 * It's safe to drop the lock here. We've marked this region for
438 * our exclusive use. If the migration fails we will take the
439 * lock again and unmark it.
440 */
441 mutex_unlock(&cma->lock);
442
443 pfn = cma->base_pfn + (bitmap_no << cma->order_per_bit);
444 mutex_lock(&cma_mutex);
ca96b625 445 ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA,
e2f466e3 446 gfp_mask);
a254129e
JK
447 mutex_unlock(&cma_mutex);
448 if (ret == 0) {
449 page = pfn_to_page(pfn);
450 break;
a254129e 451 }
b7155e76 452
a254129e 453 cma_clear_bitmap(cma, pfn, count);
b7155e76
JK
454 if (ret != -EBUSY)
455 break;
456
a254129e
JK
457 pr_debug("%s(): memory range at %p is busy, retrying\n",
458 __func__, pfn_to_page(pfn));
459 /* try again with a bit different memory target */
460 start = bitmap_no + mask + 1;
461 }
462
3acaea68 463 trace_cma_alloc(pfn, page, count, align);
99e8ea6c 464
dbe43d4d
JK
465 if (ret) {
466 pr_info("%s: alloc failed, req-size: %zu pages, ret: %d\n",
467 __func__, count, ret);
468 cma_debug_show_areas(cma);
469 }
470
a254129e
JK
471 pr_debug("%s(): returned %p\n", __func__, page);
472 return page;
473}
474
475/**
476 * cma_release() - release allocated pages
477 * @cma: Contiguous memory region for which the allocation is performed.
478 * @pages: Allocated pages.
479 * @count: Number of allocated pages.
480 *
481 * This function releases memory allocated by alloc_cma().
482 * It returns false when provided pages do not belong to contiguous area and
483 * true otherwise.
484 */
ac173824 485bool cma_release(struct cma *cma, const struct page *pages, unsigned int count)
a254129e
JK
486{
487 unsigned long pfn;
488
489 if (!cma || !pages)
490 return false;
491
492 pr_debug("%s(page %p)\n", __func__, (void *)pages);
493
494 pfn = page_to_pfn(pages);
495
496 if (pfn < cma->base_pfn || pfn >= cma->base_pfn + cma->count)
497 return false;
498
499 VM_BUG_ON(pfn + count > cma->base_pfn + cma->count);
500
501 free_contig_range(pfn, count);
502 cma_clear_bitmap(cma, pfn, count);
99e8ea6c 503 trace_cma_release(pfn, pages, count);
a254129e
JK
504
505 return true;
506}
e4231bcd
LA
507
508int cma_for_each_area(int (*it)(struct cma *cma, void *data), void *data)
509{
510 int i;
511
512 for (i = 0; i < cma_area_count; i++) {
513 int ret = it(&cma_areas[i], data);
514
515 if (ret)
516 return ret;
517 }
518
519 return 0;
520}