1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Contiguous Memory Allocator
5 * Copyright (c) 2010-2011 by Samsung Electronics.
6 * Copyright IBM Corporation, 2013
7 * Copyright LG Electronics Inc., 2014
9 * Marek Szyprowski <m.szyprowski@samsung.com>
10 * Michal Nazarewicz <mina86@mina86.com>
11 * Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
12 * Joonsoo Kim <iamjoonsoo.kim@lge.com>
15 #define pr_fmt(fmt) "cma: " fmt
17 #ifdef CONFIG_CMA_DEBUG
22 #define CREATE_TRACE_POINTS
24 #include <linux/memblock.h>
25 #include <linux/err.h>
27 #include <linux/sizes.h>
28 #include <linux/slab.h>
29 #include <linux/log2.h>
30 #include <linux/cma.h>
31 #include <linux/highmem.h>
33 #include <linux/kmemleak.h>
34 #include <trace/events/cma.h>
38 struct cma cma_areas
[MAX_CMA_AREAS
];
39 unsigned cma_area_count
;
40 static DEFINE_MUTEX(cma_mutex
);
42 phys_addr_t
cma_get_base(const struct cma
*cma
)
44 return PFN_PHYS(cma
->base_pfn
);
47 unsigned long cma_get_size(const struct cma
*cma
)
49 return cma
->count
<< PAGE_SHIFT
;
52 const char *cma_get_name(const struct cma
*cma
)
57 static unsigned long cma_bitmap_aligned_mask(const struct cma
*cma
,
58 unsigned int align_order
)
60 if (align_order
<= cma
->order_per_bit
)
62 return (1UL << (align_order
- cma
->order_per_bit
)) - 1;
66 * Find the offset of the base PFN from the specified align_order.
67 * The value returned is represented in order_per_bits.
69 static unsigned long cma_bitmap_aligned_offset(const struct cma
*cma
,
70 unsigned int align_order
)
72 return (cma
->base_pfn
& ((1UL << align_order
) - 1))
73 >> cma
->order_per_bit
;
76 static unsigned long cma_bitmap_pages_to_bits(const struct cma
*cma
,
79 return ALIGN(pages
, 1UL << cma
->order_per_bit
) >> cma
->order_per_bit
;
82 static void cma_clear_bitmap(struct cma
*cma
, unsigned long pfn
,
85 unsigned long bitmap_no
, bitmap_count
;
88 bitmap_no
= (pfn
- cma
->base_pfn
) >> cma
->order_per_bit
;
89 bitmap_count
= cma_bitmap_pages_to_bits(cma
, count
);
91 spin_lock_irqsave(&cma
->lock
, flags
);
92 bitmap_clear(cma
->bitmap
, bitmap_no
, bitmap_count
);
93 spin_unlock_irqrestore(&cma
->lock
, flags
);
96 static void __init
cma_activate_area(struct cma
*cma
)
98 unsigned long base_pfn
= cma
->base_pfn
, pfn
;
101 cma
->bitmap
= bitmap_zalloc(cma_bitmap_maxno(cma
), GFP_KERNEL
);
106 * alloc_contig_range() requires the pfn range specified to be in the
107 * same zone. Simplify by forcing the entire CMA resv range to be in the
110 WARN_ON_ONCE(!pfn_valid(base_pfn
));
111 zone
= page_zone(pfn_to_page(base_pfn
));
112 for (pfn
= base_pfn
+ 1; pfn
< base_pfn
+ cma
->count
; pfn
++) {
113 WARN_ON_ONCE(!pfn_valid(pfn
));
114 if (page_zone(pfn_to_page(pfn
)) != zone
)
118 for (pfn
= base_pfn
; pfn
< base_pfn
+ cma
->count
;
119 pfn
+= pageblock_nr_pages
)
120 init_cma_reserved_pageblock(pfn_to_page(pfn
));
122 spin_lock_init(&cma
->lock
);
124 #ifdef CONFIG_CMA_DEBUGFS
125 INIT_HLIST_HEAD(&cma
->mem_head
);
126 spin_lock_init(&cma
->mem_head_lock
);
132 bitmap_free(cma
->bitmap
);
134 /* Expose all pages to the buddy, they are useless for CMA. */
135 if (!cma
->reserve_pages_on_error
) {
136 for (pfn
= base_pfn
; pfn
< base_pfn
+ cma
->count
; pfn
++)
137 free_reserved_page(pfn_to_page(pfn
));
139 totalcma_pages
-= cma
->count
;
141 pr_err("CMA area %s could not be activated\n", cma
->name
);
145 static int __init
cma_init_reserved_areas(void)
149 for (i
= 0; i
< cma_area_count
; i
++)
150 cma_activate_area(&cma_areas
[i
]);
154 core_initcall(cma_init_reserved_areas
);
156 void __init
cma_reserve_pages_on_error(struct cma
*cma
)
158 cma
->reserve_pages_on_error
= true;
162 * cma_init_reserved_mem() - create custom contiguous area from reserved memory
163 * @base: Base address of the reserved area
164 * @size: Size of the reserved area (in bytes),
165 * @order_per_bit: Order of pages represented by one bit on bitmap.
166 * @name: The name of the area. If this parameter is NULL, the name of
167 * the area will be set to "cmaN", where N is a running counter of
169 * @res_cma: Pointer to store the created cma region.
171 * This function creates custom contiguous area from already reserved memory.
173 int __init
cma_init_reserved_mem(phys_addr_t base
, phys_addr_t size
,
174 unsigned int order_per_bit
,
176 struct cma
**res_cma
)
181 if (cma_area_count
== ARRAY_SIZE(cma_areas
)) {
182 pr_err("Not enough slots for CMA reserved regions!\n");
186 if (!size
|| !memblock_is_region_reserved(base
, size
))
189 /* alignment should be aligned with order_per_bit */
190 if (!IS_ALIGNED(CMA_MIN_ALIGNMENT_PAGES
, 1 << order_per_bit
))
193 /* ensure minimal alignment required by mm core */
194 if (!IS_ALIGNED(base
| size
, CMA_MIN_ALIGNMENT_BYTES
))
198 * Each reserved area must be initialised later, when more kernel
199 * subsystems (like slab allocator) are available.
201 cma
= &cma_areas
[cma_area_count
];
204 snprintf(cma
->name
, CMA_MAX_NAME
, name
);
206 snprintf(cma
->name
, CMA_MAX_NAME
, "cma%d\n", cma_area_count
);
208 cma
->base_pfn
= PFN_DOWN(base
);
209 cma
->count
= size
>> PAGE_SHIFT
;
210 cma
->order_per_bit
= order_per_bit
;
213 totalcma_pages
+= (size
/ PAGE_SIZE
);
219 * cma_declare_contiguous_nid() - reserve custom contiguous area
220 * @base: Base address of the reserved area optional, use 0 for any
221 * @size: Size of the reserved area (in bytes),
222 * @limit: End address of the reserved memory (optional, 0 for any).
223 * @alignment: Alignment for the CMA area, should be power of 2 or zero
224 * @order_per_bit: Order of pages represented by one bit on bitmap.
225 * @fixed: hint about where to place the reserved area
226 * @name: The name of the area. See function cma_init_reserved_mem()
227 * @res_cma: Pointer to store the created cma region.
228 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
230 * This function reserves memory from early allocator. It should be
231 * called by arch specific code once the early allocator (memblock or bootmem)
232 * has been activated and all other subsystems have already allocated/reserved
233 * memory. This function allows to create custom reserved areas.
235 * If @fixed is true, reserve contiguous area at exactly @base. If false,
236 * reserve in range from @base to @limit.
238 int __init
cma_declare_contiguous_nid(phys_addr_t base
,
239 phys_addr_t size
, phys_addr_t limit
,
240 phys_addr_t alignment
, unsigned int order_per_bit
,
241 bool fixed
, const char *name
, struct cma
**res_cma
,
244 phys_addr_t memblock_end
= memblock_end_of_DRAM();
245 phys_addr_t highmem_start
;
249 * We can't use __pa(high_memory) directly, since high_memory
250 * isn't a valid direct map VA, and DEBUG_VIRTUAL will (validly)
251 * complain. Find the boundary by adding one to the last valid
254 highmem_start
= __pa(high_memory
- 1) + 1;
255 pr_debug("%s(size %pa, base %pa, limit %pa alignment %pa)\n",
256 __func__
, &size
, &base
, &limit
, &alignment
);
258 if (cma_area_count
== ARRAY_SIZE(cma_areas
)) {
259 pr_err("Not enough slots for CMA reserved regions!\n");
266 if (alignment
&& !is_power_of_2(alignment
))
269 /* Sanitise input arguments. */
270 alignment
= max_t(phys_addr_t
, alignment
, CMA_MIN_ALIGNMENT_BYTES
);
271 if (fixed
&& base
& (alignment
- 1)) {
273 pr_err("Region at %pa must be aligned to %pa bytes\n",
277 base
= ALIGN(base
, alignment
);
278 size
= ALIGN(size
, alignment
);
279 limit
&= ~(alignment
- 1);
284 /* size should be aligned with order_per_bit */
285 if (!IS_ALIGNED(size
>> PAGE_SHIFT
, 1 << order_per_bit
))
289 * If allocating at a fixed base the request region must not cross the
290 * low/high memory boundary.
292 if (fixed
&& base
< highmem_start
&& base
+ size
> highmem_start
) {
294 pr_err("Region at %pa defined on low/high memory boundary (%pa)\n",
295 &base
, &highmem_start
);
300 * If the limit is unspecified or above the memblock end, its effective
301 * value will be the memblock end. Set it explicitly to simplify further
304 if (limit
== 0 || limit
> memblock_end
)
305 limit
= memblock_end
;
307 if (base
+ size
> limit
) {
309 pr_err("Size (%pa) of region at %pa exceeds limit (%pa)\n",
310 &size
, &base
, &limit
);
316 if (memblock_is_region_reserved(base
, size
) ||
317 memblock_reserve(base
, size
) < 0) {
322 phys_addr_t addr
= 0;
325 * All pages in the reserved area must come from the same zone.
326 * If the requested region crosses the low/high memory boundary,
327 * try allocating from high memory first and fall back to low
328 * memory in case of failure.
330 if (base
< highmem_start
&& limit
> highmem_start
) {
331 addr
= memblock_alloc_range_nid(size
, alignment
,
332 highmem_start
, limit
, nid
, true);
333 limit
= highmem_start
;
337 * If there is enough memory, try a bottom-up allocation first.
338 * It will place the new cma area close to the start of the node
339 * and guarantee that the compaction is moving pages out of the
340 * cma area and not into it.
341 * Avoid using first 4GB to not interfere with constrained zones
344 #ifdef CONFIG_PHYS_ADDR_T_64BIT
345 if (!memblock_bottom_up() && memblock_end
>= SZ_4G
+ size
) {
346 memblock_set_bottom_up(true);
347 addr
= memblock_alloc_range_nid(size
, alignment
, SZ_4G
,
349 memblock_set_bottom_up(false);
354 addr
= memblock_alloc_range_nid(size
, alignment
, base
,
363 * kmemleak scans/reads tracked objects for pointers to other
364 * objects but this address isn't mapped and accessible
366 kmemleak_ignore_phys(addr
);
370 ret
= cma_init_reserved_mem(base
, size
, order_per_bit
, name
, res_cma
);
374 pr_info("Reserved %ld MiB at %pa\n", (unsigned long)size
/ SZ_1M
,
379 memblock_phys_free(base
, size
);
381 pr_err("Failed to reserve %ld MiB\n", (unsigned long)size
/ SZ_1M
);
385 #ifdef CONFIG_CMA_DEBUG
386 static void cma_debug_show_areas(struct cma
*cma
)
388 unsigned long next_zero_bit
, next_set_bit
, nr_zero
;
389 unsigned long start
= 0;
390 unsigned long nr_part
, nr_total
= 0;
391 unsigned long nbits
= cma_bitmap_maxno(cma
);
393 spin_lock_irq(&cma
->lock
);
394 pr_info("number of available pages: ");
396 next_zero_bit
= find_next_zero_bit(cma
->bitmap
, nbits
, start
);
397 if (next_zero_bit
>= nbits
)
399 next_set_bit
= find_next_bit(cma
->bitmap
, nbits
, next_zero_bit
);
400 nr_zero
= next_set_bit
- next_zero_bit
;
401 nr_part
= nr_zero
<< cma
->order_per_bit
;
402 pr_cont("%s%lu@%lu", nr_total
? "+" : "", nr_part
,
405 start
= next_zero_bit
+ nr_zero
;
407 pr_cont("=> %lu free of %lu total pages\n", nr_total
, cma
->count
);
408 spin_unlock_irq(&cma
->lock
);
411 static inline void cma_debug_show_areas(struct cma
*cma
) { }
415 * cma_alloc() - allocate pages from contiguous area
416 * @cma: Contiguous memory region for which the allocation is performed.
417 * @count: Requested number of pages.
418 * @align: Requested alignment of pages (in PAGE_SIZE order).
419 * @no_warn: Avoid printing message about failed allocation
421 * This function allocates part of contiguous memory on specific
422 * contiguous memory area.
424 struct page
*cma_alloc(struct cma
*cma
, unsigned long count
,
425 unsigned int align
, bool no_warn
)
427 unsigned long mask
, offset
;
428 unsigned long pfn
= -1;
429 unsigned long start
= 0;
430 unsigned long bitmap_maxno
, bitmap_no
, bitmap_count
;
432 struct page
*page
= NULL
;
435 if (!cma
|| !cma
->count
|| !cma
->bitmap
)
438 pr_debug("%s(cma %p, count %lu, align %d)\n", __func__
, (void *)cma
,
444 trace_cma_alloc_start(cma
->name
, count
, align
);
446 mask
= cma_bitmap_aligned_mask(cma
, align
);
447 offset
= cma_bitmap_aligned_offset(cma
, align
);
448 bitmap_maxno
= cma_bitmap_maxno(cma
);
449 bitmap_count
= cma_bitmap_pages_to_bits(cma
, count
);
451 if (bitmap_count
> bitmap_maxno
)
455 spin_lock_irq(&cma
->lock
);
456 bitmap_no
= bitmap_find_next_zero_area_off(cma
->bitmap
,
457 bitmap_maxno
, start
, bitmap_count
, mask
,
459 if (bitmap_no
>= bitmap_maxno
) {
460 spin_unlock_irq(&cma
->lock
);
463 bitmap_set(cma
->bitmap
, bitmap_no
, bitmap_count
);
465 * It's safe to drop the lock here. We've marked this region for
466 * our exclusive use. If the migration fails we will take the
467 * lock again and unmark it.
469 spin_unlock_irq(&cma
->lock
);
471 pfn
= cma
->base_pfn
+ (bitmap_no
<< cma
->order_per_bit
);
472 mutex_lock(&cma_mutex
);
473 ret
= alloc_contig_range(pfn
, pfn
+ count
, MIGRATE_CMA
,
474 GFP_KERNEL
| (no_warn
? __GFP_NOWARN
: 0));
475 mutex_unlock(&cma_mutex
);
477 page
= pfn_to_page(pfn
);
481 cma_clear_bitmap(cma
, pfn
, count
);
485 pr_debug("%s(): memory range at %p is busy, retrying\n",
486 __func__
, pfn_to_page(pfn
));
488 trace_cma_alloc_busy_retry(cma
->name
, pfn
, pfn_to_page(pfn
),
490 /* try again with a bit different memory target */
491 start
= bitmap_no
+ mask
+ 1;
494 trace_cma_alloc_finish(cma
->name
, pfn
, page
, count
, align
);
497 * CMA can allocate multiple page blocks, which results in different
498 * blocks being marked with different tags. Reset the tags to ignore
502 for (i
= 0; i
< count
; i
++)
503 page_kasan_tag_reset(page
+ i
);
506 if (ret
&& !no_warn
) {
507 pr_err_ratelimited("%s: %s: alloc failed, req-size: %lu pages, ret: %d\n",
508 __func__
, cma
->name
, count
, ret
);
509 cma_debug_show_areas(cma
);
512 pr_debug("%s(): returned %p\n", __func__
, page
);
515 count_vm_event(CMA_ALLOC_SUCCESS
);
516 cma_sysfs_account_success_pages(cma
, count
);
518 count_vm_event(CMA_ALLOC_FAIL
);
520 cma_sysfs_account_fail_pages(cma
, count
);
526 bool cma_pages_valid(struct cma
*cma
, const struct page
*pages
,
534 pfn
= page_to_pfn(pages
);
536 if (pfn
< cma
->base_pfn
|| pfn
>= cma
->base_pfn
+ cma
->count
) {
537 pr_debug("%s(page %p, count %lu)\n", __func__
,
538 (void *)pages
, count
);
546 * cma_release() - release allocated pages
547 * @cma: Contiguous memory region for which the allocation is performed.
548 * @pages: Allocated pages.
549 * @count: Number of allocated pages.
551 * This function releases memory allocated by cma_alloc().
552 * It returns false when provided pages do not belong to contiguous area and
555 bool cma_release(struct cma
*cma
, const struct page
*pages
,
560 if (!cma_pages_valid(cma
, pages
, count
))
563 pr_debug("%s(page %p, count %lu)\n", __func__
, (void *)pages
, count
);
565 pfn
= page_to_pfn(pages
);
567 VM_BUG_ON(pfn
+ count
> cma
->base_pfn
+ cma
->count
);
569 free_contig_range(pfn
, count
);
570 cma_clear_bitmap(cma
, pfn
, count
);
571 trace_cma_release(cma
->name
, pfn
, pages
, count
);
576 int cma_for_each_area(int (*it
)(struct cma
*cma
, void *data
), void *data
)
580 for (i
= 0; i
< cma_area_count
; i
++) {
581 int ret
= it(&cma_areas
[i
], data
);