1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _MM_PERCPU_INTERNAL_H
3 #define _MM_PERCPU_INTERNAL_H
5 #include <linux/types.h>
6 #include <linux/percpu.h>
9 * pcpu_block_md is the metadata block struct.
10 * Each chunk's bitmap is split into a number of full blocks.
11 * All units are in terms of bits.
13 * The scan hint is the largest known contiguous area before the contig hint.
14 * It is not necessarily the actual largest contig hint though. There is an
15 * invariant that the scan_hint_start > contig_hint_start iff
16 * scan_hint == contig_hint. This is necessary because when scanning forward,
17 * we don't know if a new contig hint would be better than the current one.
19 struct pcpu_block_md
{
20 int scan_hint
; /* scan hint for block */
21 int scan_hint_start
; /* block relative starting
22 position of the scan hint */
23 int contig_hint
; /* contig hint for block */
24 int contig_hint_start
; /* block relative starting
25 position of the contig hint */
26 int left_free
; /* size of free space along
27 the left side of the block */
28 int right_free
; /* size of free space along
29 the right side of the block */
30 int first_free
; /* block position of first free */
31 int nr_bits
; /* total bits responsible for */
35 #ifdef CONFIG_PERCPU_STATS
36 int nr_alloc
; /* # of allocations */
37 size_t max_alloc_size
; /* largest allocation size */
40 struct list_head list
; /* linked to pcpu_slot lists */
41 int free_bytes
; /* free bytes in the chunk */
42 struct pcpu_block_md chunk_md
;
43 void *base_addr
; /* base address of this chunk */
45 unsigned long *alloc_map
; /* allocation map */
46 unsigned long *bound_map
; /* boundary map */
47 struct pcpu_block_md
*md_blocks
; /* metadata blocks */
49 void *data
; /* chunk data */
50 bool immutable
; /* no [de]population allowed */
51 bool isolated
; /* isolated from active chunk
53 int start_offset
; /* the overlap with the previous
54 region to have a page aligned
56 int end_offset
; /* additional area required to
57 have the region end page
59 #ifdef CONFIG_MEMCG_KMEM
60 struct obj_cgroup
**obj_cgroups
; /* vector of object cgroups */
63 int nr_pages
; /* # of pages served by this chunk */
64 int nr_populated
; /* # of populated pages */
65 int nr_empty_pop_pages
; /* # of empty populated pages */
66 unsigned long populated
[]; /* populated bitmap */
69 extern spinlock_t pcpu_lock
;
71 extern struct list_head
*pcpu_chunk_lists
;
72 extern int pcpu_nr_slots
;
73 extern int pcpu_sidelined_slot
;
74 extern int pcpu_to_depopulate_slot
;
75 extern int pcpu_nr_empty_pop_pages
;
77 extern struct pcpu_chunk
*pcpu_first_chunk
;
78 extern struct pcpu_chunk
*pcpu_reserved_chunk
;
81 * pcpu_chunk_nr_blocks - converts nr_pages to # of md_blocks
82 * @chunk: chunk of interest
84 * This conversion is from the number of physical pages that the chunk
85 * serves to the number of bitmap blocks used.
87 static inline int pcpu_chunk_nr_blocks(struct pcpu_chunk
*chunk
)
89 return chunk
->nr_pages
* PAGE_SIZE
/ PCPU_BITMAP_BLOCK_SIZE
;
93 * pcpu_nr_pages_to_map_bits - converts the pages to size of bitmap
94 * @pages: number of physical pages
96 * This conversion is from physical pages to the number of bits
97 * required in the bitmap.
99 static inline int pcpu_nr_pages_to_map_bits(int pages
)
101 return pages
* PAGE_SIZE
/ PCPU_MIN_ALLOC_SIZE
;
105 * pcpu_chunk_map_bits - helper to convert nr_pages to size of bitmap
106 * @chunk: chunk of interest
108 * This conversion is from the number of physical pages that the chunk
109 * serves to the number of bits in the bitmap.
111 static inline int pcpu_chunk_map_bits(struct pcpu_chunk
*chunk
)
113 return pcpu_nr_pages_to_map_bits(chunk
->nr_pages
);
116 #ifdef CONFIG_PERCPU_STATS
118 #include <linux/spinlock.h>
120 struct percpu_stats
{
121 u64 nr_alloc
; /* lifetime # of allocations */
122 u64 nr_dealloc
; /* lifetime # of deallocations */
123 u64 nr_cur_alloc
; /* current # of allocations */
124 u64 nr_max_alloc
; /* max # of live allocations */
125 u32 nr_chunks
; /* current # of live chunks */
126 u32 nr_max_chunks
; /* max # of live chunks */
127 size_t min_alloc_size
; /* min allocation size */
128 size_t max_alloc_size
; /* max allocation size */
131 extern struct percpu_stats pcpu_stats
;
132 extern struct pcpu_alloc_info pcpu_stats_ai
;
135 * For debug purposes. We don't care about the flexible array.
137 static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info
*ai
)
139 memcpy(&pcpu_stats_ai
, ai
, sizeof(struct pcpu_alloc_info
));
141 /* initialize min_alloc_size to unit_size */
142 pcpu_stats
.min_alloc_size
= pcpu_stats_ai
.unit_size
;
146 * pcpu_stats_area_alloc - increment area allocation stats
147 * @chunk: the location of the area being allocated
148 * @size: size of area to allocate in bytes
153 static inline void pcpu_stats_area_alloc(struct pcpu_chunk
*chunk
, size_t size
)
155 lockdep_assert_held(&pcpu_lock
);
157 pcpu_stats
.nr_alloc
++;
158 pcpu_stats
.nr_cur_alloc
++;
159 pcpu_stats
.nr_max_alloc
=
160 max(pcpu_stats
.nr_max_alloc
, pcpu_stats
.nr_cur_alloc
);
161 pcpu_stats
.min_alloc_size
=
162 min(pcpu_stats
.min_alloc_size
, size
);
163 pcpu_stats
.max_alloc_size
=
164 max(pcpu_stats
.max_alloc_size
, size
);
167 chunk
->max_alloc_size
= max(chunk
->max_alloc_size
, size
);
171 * pcpu_stats_area_dealloc - decrement allocation stats
172 * @chunk: the location of the area being deallocated
177 static inline void pcpu_stats_area_dealloc(struct pcpu_chunk
*chunk
)
179 lockdep_assert_held(&pcpu_lock
);
181 pcpu_stats
.nr_dealloc
++;
182 pcpu_stats
.nr_cur_alloc
--;
188 * pcpu_stats_chunk_alloc - increment chunk stats
190 static inline void pcpu_stats_chunk_alloc(void)
193 spin_lock_irqsave(&pcpu_lock
, flags
);
195 pcpu_stats
.nr_chunks
++;
196 pcpu_stats
.nr_max_chunks
=
197 max(pcpu_stats
.nr_max_chunks
, pcpu_stats
.nr_chunks
);
199 spin_unlock_irqrestore(&pcpu_lock
, flags
);
203 * pcpu_stats_chunk_dealloc - decrement chunk stats
205 static inline void pcpu_stats_chunk_dealloc(void)
208 spin_lock_irqsave(&pcpu_lock
, flags
);
210 pcpu_stats
.nr_chunks
--;
212 spin_unlock_irqrestore(&pcpu_lock
, flags
);
217 static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info
*ai
)
221 static inline void pcpu_stats_area_alloc(struct pcpu_chunk
*chunk
, size_t size
)
225 static inline void pcpu_stats_area_dealloc(struct pcpu_chunk
*chunk
)
229 static inline void pcpu_stats_chunk_alloc(void)
233 static inline void pcpu_stats_chunk_dealloc(void)
237 #endif /* !CONFIG_PERCPU_STATS */