1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Contiguous Memory Allocator
5 * Copyright (c) 2010-2011 by Samsung Electronics.
6 * Copyright IBM Corporation, 2013
7 * Copyright LG Electronics Inc., 2014
9 * Marek Szyprowski <m.szyprowski@samsung.com>
10 * Michal Nazarewicz <mina86@mina86.com>
11 * Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
12 * Joonsoo Kim <iamjoonsoo.kim@lge.com>
15 #define pr_fmt(fmt) "cma: " fmt
17 #ifdef CONFIG_CMA_DEBUG
22 #define CREATE_TRACE_POINTS
24 #include <linux/memblock.h>
25 #include <linux/err.h>
27 #include <linux/sizes.h>
28 #include <linux/slab.h>
29 #include <linux/log2.h>
30 #include <linux/cma.h>
31 #include <linux/highmem.h>
33 #include <linux/kmemleak.h>
34 #include <trace/events/cma.h>
38 struct cma cma_areas
[MAX_CMA_AREAS
];
39 unsigned cma_area_count
;
40 static DEFINE_MUTEX(cma_mutex
);
42 phys_addr_t
cma_get_base(const struct cma
*cma
)
44 return PFN_PHYS(cma
->base_pfn
);
47 unsigned long cma_get_size(const struct cma
*cma
)
49 return cma
->count
<< PAGE_SHIFT
;
52 const char *cma_get_name(const struct cma
*cma
)
57 static unsigned long cma_bitmap_aligned_mask(const struct cma
*cma
,
58 unsigned int align_order
)
60 if (align_order
<= cma
->order_per_bit
)
62 return (1UL << (align_order
- cma
->order_per_bit
)) - 1;
66 * Find the offset of the base PFN from the specified align_order.
67 * The value returned is represented in order_per_bits.
69 static unsigned long cma_bitmap_aligned_offset(const struct cma
*cma
,
70 unsigned int align_order
)
72 return (cma
->base_pfn
& ((1UL << align_order
) - 1))
73 >> cma
->order_per_bit
;
76 static unsigned long cma_bitmap_pages_to_bits(const struct cma
*cma
,
79 return ALIGN(pages
, 1UL << cma
->order_per_bit
) >> cma
->order_per_bit
;
82 static void cma_clear_bitmap(struct cma
*cma
, unsigned long pfn
,
85 unsigned long bitmap_no
, bitmap_count
;
88 bitmap_no
= (pfn
- cma
->base_pfn
) >> cma
->order_per_bit
;
89 bitmap_count
= cma_bitmap_pages_to_bits(cma
, count
);
91 spin_lock_irqsave(&cma
->lock
, flags
);
92 bitmap_clear(cma
->bitmap
, bitmap_no
, bitmap_count
);
93 spin_unlock_irqrestore(&cma
->lock
, flags
);
96 static void __init
cma_activate_area(struct cma
*cma
)
98 unsigned long base_pfn
= cma
->base_pfn
, pfn
;
101 cma
->bitmap
= bitmap_zalloc(cma_bitmap_maxno(cma
), GFP_KERNEL
);
106 * alloc_contig_range() requires the pfn range specified to be in the
107 * same zone. Simplify by forcing the entire CMA resv range to be in the
110 WARN_ON_ONCE(!pfn_valid(base_pfn
));
111 zone
= page_zone(pfn_to_page(base_pfn
));
112 for (pfn
= base_pfn
+ 1; pfn
< base_pfn
+ cma
->count
; pfn
++) {
113 WARN_ON_ONCE(!pfn_valid(pfn
));
114 if (page_zone(pfn_to_page(pfn
)) != zone
)
118 for (pfn
= base_pfn
; pfn
< base_pfn
+ cma
->count
;
119 pfn
+= pageblock_nr_pages
)
120 init_cma_reserved_pageblock(pfn_to_page(pfn
));
122 spin_lock_init(&cma
->lock
);
124 #ifdef CONFIG_CMA_DEBUGFS
125 INIT_HLIST_HEAD(&cma
->mem_head
);
126 spin_lock_init(&cma
->mem_head_lock
);
132 bitmap_free(cma
->bitmap
);
134 /* Expose all pages to the buddy, they are useless for CMA. */
135 for (pfn
= base_pfn
; pfn
< base_pfn
+ cma
->count
; pfn
++)
136 free_reserved_page(pfn_to_page(pfn
));
137 totalcma_pages
-= cma
->count
;
139 pr_err("CMA area %s could not be activated\n", cma
->name
);
143 static int __init
cma_init_reserved_areas(void)
147 for (i
= 0; i
< cma_area_count
; i
++)
148 cma_activate_area(&cma_areas
[i
]);
152 core_initcall(cma_init_reserved_areas
);
155 * cma_init_reserved_mem() - create custom contiguous area from reserved memory
156 * @base: Base address of the reserved area
157 * @size: Size of the reserved area (in bytes),
158 * @order_per_bit: Order of pages represented by one bit on bitmap.
159 * @name: The name of the area. If this parameter is NULL, the name of
160 * the area will be set to "cmaN", where N is a running counter of
162 * @res_cma: Pointer to store the created cma region.
164 * This function creates custom contiguous area from already reserved memory.
166 int __init
cma_init_reserved_mem(phys_addr_t base
, phys_addr_t size
,
167 unsigned int order_per_bit
,
169 struct cma
**res_cma
)
172 phys_addr_t alignment
;
175 if (cma_area_count
== ARRAY_SIZE(cma_areas
)) {
176 pr_err("Not enough slots for CMA reserved regions!\n");
180 if (!size
|| !memblock_is_region_reserved(base
, size
))
183 /* ensure minimal alignment required by mm core */
184 alignment
= PAGE_SIZE
<<
185 max_t(unsigned long, MAX_ORDER
- 1, pageblock_order
);
187 /* alignment should be aligned with order_per_bit */
188 if (!IS_ALIGNED(alignment
>> PAGE_SHIFT
, 1 << order_per_bit
))
191 if (ALIGN(base
, alignment
) != base
|| ALIGN(size
, alignment
) != size
)
195 * Each reserved area must be initialised later, when more kernel
196 * subsystems (like slab allocator) are available.
198 cma
= &cma_areas
[cma_area_count
];
201 snprintf(cma
->name
, CMA_MAX_NAME
, name
);
203 snprintf(cma
->name
, CMA_MAX_NAME
, "cma%d\n", cma_area_count
);
205 cma
->base_pfn
= PFN_DOWN(base
);
206 cma
->count
= size
>> PAGE_SHIFT
;
207 cma
->order_per_bit
= order_per_bit
;
210 totalcma_pages
+= (size
/ PAGE_SIZE
);
216 * cma_declare_contiguous_nid() - reserve custom contiguous area
217 * @base: Base address of the reserved area optional, use 0 for any
218 * @size: Size of the reserved area (in bytes),
219 * @limit: End address of the reserved memory (optional, 0 for any).
220 * @alignment: Alignment for the CMA area, should be power of 2 or zero
221 * @order_per_bit: Order of pages represented by one bit on bitmap.
222 * @fixed: hint about where to place the reserved area
223 * @name: The name of the area. See function cma_init_reserved_mem()
224 * @res_cma: Pointer to store the created cma region.
225 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
227 * This function reserves memory from early allocator. It should be
228 * called by arch specific code once the early allocator (memblock or bootmem)
229 * has been activated and all other subsystems have already allocated/reserved
230 * memory. This function allows to create custom reserved areas.
232 * If @fixed is true, reserve contiguous area at exactly @base. If false,
233 * reserve in range from @base to @limit.
235 int __init
cma_declare_contiguous_nid(phys_addr_t base
,
236 phys_addr_t size
, phys_addr_t limit
,
237 phys_addr_t alignment
, unsigned int order_per_bit
,
238 bool fixed
, const char *name
, struct cma
**res_cma
,
241 phys_addr_t memblock_end
= memblock_end_of_DRAM();
242 phys_addr_t highmem_start
;
246 * We can't use __pa(high_memory) directly, since high_memory
247 * isn't a valid direct map VA, and DEBUG_VIRTUAL will (validly)
248 * complain. Find the boundary by adding one to the last valid
251 highmem_start
= __pa(high_memory
- 1) + 1;
252 pr_debug("%s(size %pa, base %pa, limit %pa alignment %pa)\n",
253 __func__
, &size
, &base
, &limit
, &alignment
);
255 if (cma_area_count
== ARRAY_SIZE(cma_areas
)) {
256 pr_err("Not enough slots for CMA reserved regions!\n");
263 if (alignment
&& !is_power_of_2(alignment
))
267 * Sanitise input arguments.
268 * Pages both ends in CMA area could be merged into adjacent unmovable
269 * migratetype page by page allocator's buddy algorithm. In the case,
270 * you couldn't get a contiguous memory, which is not what we want.
272 alignment
= max(alignment
, (phys_addr_t
)PAGE_SIZE
<<
273 max_t(unsigned long, MAX_ORDER
- 1, pageblock_order
));
274 if (fixed
&& base
& (alignment
- 1)) {
276 pr_err("Region at %pa must be aligned to %pa bytes\n",
280 base
= ALIGN(base
, alignment
);
281 size
= ALIGN(size
, alignment
);
282 limit
&= ~(alignment
- 1);
287 /* size should be aligned with order_per_bit */
288 if (!IS_ALIGNED(size
>> PAGE_SHIFT
, 1 << order_per_bit
))
292 * If allocating at a fixed base the request region must not cross the
293 * low/high memory boundary.
295 if (fixed
&& base
< highmem_start
&& base
+ size
> highmem_start
) {
297 pr_err("Region at %pa defined on low/high memory boundary (%pa)\n",
298 &base
, &highmem_start
);
303 * If the limit is unspecified or above the memblock end, its effective
304 * value will be the memblock end. Set it explicitly to simplify further
307 if (limit
== 0 || limit
> memblock_end
)
308 limit
= memblock_end
;
310 if (base
+ size
> limit
) {
312 pr_err("Size (%pa) of region at %pa exceeds limit (%pa)\n",
313 &size
, &base
, &limit
);
319 if (memblock_is_region_reserved(base
, size
) ||
320 memblock_reserve(base
, size
) < 0) {
325 phys_addr_t addr
= 0;
328 * All pages in the reserved area must come from the same zone.
329 * If the requested region crosses the low/high memory boundary,
330 * try allocating from high memory first and fall back to low
331 * memory in case of failure.
333 if (base
< highmem_start
&& limit
> highmem_start
) {
334 addr
= memblock_alloc_range_nid(size
, alignment
,
335 highmem_start
, limit
, nid
, true);
336 limit
= highmem_start
;
340 * If there is enough memory, try a bottom-up allocation first.
341 * It will place the new cma area close to the start of the node
342 * and guarantee that the compaction is moving pages out of the
343 * cma area and not into it.
344 * Avoid using first 4GB to not interfere with constrained zones
347 #ifdef CONFIG_PHYS_ADDR_T_64BIT
348 if (!memblock_bottom_up() && memblock_end
>= SZ_4G
+ size
) {
349 memblock_set_bottom_up(true);
350 addr
= memblock_alloc_range_nid(size
, alignment
, SZ_4G
,
352 memblock_set_bottom_up(false);
357 addr
= memblock_alloc_range_nid(size
, alignment
, base
,
366 * kmemleak scans/reads tracked objects for pointers to other
367 * objects but this address isn't mapped and accessible
369 kmemleak_ignore_phys(addr
);
373 ret
= cma_init_reserved_mem(base
, size
, order_per_bit
, name
, res_cma
);
377 pr_info("Reserved %ld MiB at %pa\n", (unsigned long)size
/ SZ_1M
,
382 memblock_free(base
, size
);
384 pr_err("Failed to reserve %ld MiB\n", (unsigned long)size
/ SZ_1M
);
388 #ifdef CONFIG_CMA_DEBUG
389 static void cma_debug_show_areas(struct cma
*cma
)
391 unsigned long next_zero_bit
, next_set_bit
, nr_zero
;
392 unsigned long start
= 0;
393 unsigned long nr_part
, nr_total
= 0;
394 unsigned long nbits
= cma_bitmap_maxno(cma
);
396 spin_lock_irq(&cma
->lock
);
397 pr_info("number of available pages: ");
399 next_zero_bit
= find_next_zero_bit(cma
->bitmap
, nbits
, start
);
400 if (next_zero_bit
>= nbits
)
402 next_set_bit
= find_next_bit(cma
->bitmap
, nbits
, next_zero_bit
);
403 nr_zero
= next_set_bit
- next_zero_bit
;
404 nr_part
= nr_zero
<< cma
->order_per_bit
;
405 pr_cont("%s%lu@%lu", nr_total
? "+" : "", nr_part
,
408 start
= next_zero_bit
+ nr_zero
;
410 pr_cont("=> %lu free of %lu total pages\n", nr_total
, cma
->count
);
411 spin_unlock_irq(&cma
->lock
);
414 static inline void cma_debug_show_areas(struct cma
*cma
) { }
418 * cma_alloc() - allocate pages from contiguous area
419 * @cma: Contiguous memory region for which the allocation is performed.
420 * @count: Requested number of pages.
421 * @align: Requested alignment of pages (in PAGE_SIZE order).
422 * @no_warn: Avoid printing message about failed allocation
424 * This function allocates part of contiguous memory on specific
425 * contiguous memory area.
427 struct page
*cma_alloc(struct cma
*cma
, unsigned long count
,
428 unsigned int align
, bool no_warn
)
430 unsigned long mask
, offset
;
431 unsigned long pfn
= -1;
432 unsigned long start
= 0;
433 unsigned long bitmap_maxno
, bitmap_no
, bitmap_count
;
435 struct page
*page
= NULL
;
438 if (!cma
|| !cma
->count
|| !cma
->bitmap
)
441 pr_debug("%s(cma %p, count %lu, align %d)\n", __func__
, (void *)cma
,
447 trace_cma_alloc_start(cma
->name
, count
, align
);
449 mask
= cma_bitmap_aligned_mask(cma
, align
);
450 offset
= cma_bitmap_aligned_offset(cma
, align
);
451 bitmap_maxno
= cma_bitmap_maxno(cma
);
452 bitmap_count
= cma_bitmap_pages_to_bits(cma
, count
);
454 if (bitmap_count
> bitmap_maxno
)
458 spin_lock_irq(&cma
->lock
);
459 bitmap_no
= bitmap_find_next_zero_area_off(cma
->bitmap
,
460 bitmap_maxno
, start
, bitmap_count
, mask
,
462 if (bitmap_no
>= bitmap_maxno
) {
463 spin_unlock_irq(&cma
->lock
);
466 bitmap_set(cma
->bitmap
, bitmap_no
, bitmap_count
);
468 * It's safe to drop the lock here. We've marked this region for
469 * our exclusive use. If the migration fails we will take the
470 * lock again and unmark it.
472 spin_unlock_irq(&cma
->lock
);
474 pfn
= cma
->base_pfn
+ (bitmap_no
<< cma
->order_per_bit
);
475 mutex_lock(&cma_mutex
);
476 ret
= alloc_contig_range(pfn
, pfn
+ count
, MIGRATE_CMA
,
477 GFP_KERNEL
| (no_warn
? __GFP_NOWARN
: 0));
478 mutex_unlock(&cma_mutex
);
480 page
= pfn_to_page(pfn
);
484 cma_clear_bitmap(cma
, pfn
, count
);
488 pr_debug("%s(): memory range at %p is busy, retrying\n",
489 __func__
, pfn_to_page(pfn
));
491 trace_cma_alloc_busy_retry(cma
->name
, pfn
, pfn_to_page(pfn
),
493 /* try again with a bit different memory target */
494 start
= bitmap_no
+ mask
+ 1;
497 trace_cma_alloc_finish(cma
->name
, pfn
, page
, count
, align
);
500 * CMA can allocate multiple page blocks, which results in different
501 * blocks being marked with different tags. Reset the tags to ignore
505 for (i
= 0; i
< count
; i
++)
506 page_kasan_tag_reset(page
+ i
);
509 if (ret
&& !no_warn
) {
510 pr_err_ratelimited("%s: %s: alloc failed, req-size: %lu pages, ret: %d\n",
511 __func__
, cma
->name
, count
, ret
);
512 cma_debug_show_areas(cma
);
515 pr_debug("%s(): returned %p\n", __func__
, page
);
518 count_vm_event(CMA_ALLOC_SUCCESS
);
519 cma_sysfs_account_success_pages(cma
, count
);
521 count_vm_event(CMA_ALLOC_FAIL
);
523 cma_sysfs_account_fail_pages(cma
, count
);
530 * cma_release() - release allocated pages
531 * @cma: Contiguous memory region for which the allocation is performed.
532 * @pages: Allocated pages.
533 * @count: Number of allocated pages.
535 * This function releases memory allocated by cma_alloc().
536 * It returns false when provided pages do not belong to contiguous area and
539 bool cma_release(struct cma
*cma
, const struct page
*pages
,
547 pr_debug("%s(page %p, count %lu)\n", __func__
, (void *)pages
, count
);
549 pfn
= page_to_pfn(pages
);
551 if (pfn
< cma
->base_pfn
|| pfn
>= cma
->base_pfn
+ cma
->count
)
554 VM_BUG_ON(pfn
+ count
> cma
->base_pfn
+ cma
->count
);
556 free_contig_range(pfn
, count
);
557 cma_clear_bitmap(cma
, pfn
, count
);
558 trace_cma_release(cma
->name
, pfn
, pages
, count
);
563 int cma_for_each_area(int (*it
)(struct cma
*cma
, void *data
), void *data
)
567 for (i
= 0; i
< cma_area_count
; i
++) {
568 int ret
= it(&cma_areas
[i
], data
);