]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - include/linux/memcontrol.h
rhashtable: Mark internal/private inline functions as such
[mirror_ubuntu-bionic-kernel.git] / include / linux / memcontrol.h
1 /* memcontrol.h - Memory Controller
2 *
3 * Copyright IBM Corporation, 2007
4 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
5 *
6 * Copyright 2007 OpenVZ SWsoft Inc
7 * Author: Pavel Emelianov <xemul@openvz.org>
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 */
19
20 #ifndef _LINUX_MEMCONTROL_H
21 #define _LINUX_MEMCONTROL_H
22 #include <linux/cgroup.h>
23 #include <linux/vm_event_item.h>
24 #include <linux/hardirq.h>
25 #include <linux/jump_label.h>
26
27 struct mem_cgroup;
28 struct page;
29 struct mm_struct;
30 struct kmem_cache;
31
32 /*
33 * The corresponding mem_cgroup_stat_names is defined in mm/memcontrol.c,
34 * These two lists should keep in accord with each other.
35 */
36 enum mem_cgroup_stat_index {
37 /*
38 * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
39 */
40 MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */
41 MEM_CGROUP_STAT_RSS, /* # of pages charged as anon rss */
42 MEM_CGROUP_STAT_RSS_HUGE, /* # of pages charged as anon huge */
43 MEM_CGROUP_STAT_FILE_MAPPED, /* # of pages charged as file rss */
44 MEM_CGROUP_STAT_WRITEBACK, /* # of pages under writeback */
45 MEM_CGROUP_STAT_SWAP, /* # of pages, swapped out */
46 MEM_CGROUP_STAT_NSTATS,
47 };
48
49 struct mem_cgroup_reclaim_cookie {
50 struct zone *zone;
51 int priority;
52 unsigned int generation;
53 };
54
55 enum mem_cgroup_events_index {
56 MEM_CGROUP_EVENTS_PGPGIN, /* # of pages paged in */
57 MEM_CGROUP_EVENTS_PGPGOUT, /* # of pages paged out */
58 MEM_CGROUP_EVENTS_PGFAULT, /* # of page-faults */
59 MEM_CGROUP_EVENTS_PGMAJFAULT, /* # of major page-faults */
60 MEM_CGROUP_EVENTS_NSTATS,
61 /* default hierarchy events */
62 MEMCG_LOW = MEM_CGROUP_EVENTS_NSTATS,
63 MEMCG_HIGH,
64 MEMCG_MAX,
65 MEMCG_OOM,
66 MEMCG_NR_EVENTS,
67 };
68
69 #ifdef CONFIG_MEMCG
70 void mem_cgroup_events(struct mem_cgroup *memcg,
71 enum mem_cgroup_events_index idx,
72 unsigned int nr);
73
74 bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg);
75
76 int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
77 gfp_t gfp_mask, struct mem_cgroup **memcgp);
78 void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
79 bool lrucare);
80 void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg);
81 void mem_cgroup_uncharge(struct page *page);
82 void mem_cgroup_uncharge_list(struct list_head *page_list);
83
84 void mem_cgroup_migrate(struct page *oldpage, struct page *newpage,
85 bool lrucare);
86
87 struct lruvec *mem_cgroup_zone_lruvec(struct zone *, struct mem_cgroup *);
88 struct lruvec *mem_cgroup_page_lruvec(struct page *, struct zone *);
89
90 bool mem_cgroup_is_descendant(struct mem_cgroup *memcg,
91 struct mem_cgroup *root);
92 bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg);
93
94 extern struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page);
95 extern struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p);
96
97 extern struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg);
98 extern struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css);
99
100 static inline bool mm_match_cgroup(struct mm_struct *mm,
101 struct mem_cgroup *memcg)
102 {
103 struct mem_cgroup *task_memcg;
104 bool match = false;
105
106 rcu_read_lock();
107 task_memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
108 if (task_memcg)
109 match = mem_cgroup_is_descendant(task_memcg, memcg);
110 rcu_read_unlock();
111 return match;
112 }
113
114 extern struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg);
115
116 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *,
117 struct mem_cgroup *,
118 struct mem_cgroup_reclaim_cookie *);
119 void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *);
120
121 /*
122 * For memory reclaim.
123 */
124 int mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec);
125 bool mem_cgroup_lruvec_online(struct lruvec *lruvec);
126 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg);
127 unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list);
128 void mem_cgroup_update_lru_size(struct lruvec *, enum lru_list, int);
129 extern void mem_cgroup_print_oom_info(struct mem_cgroup *memcg,
130 struct task_struct *p);
131
132 static inline void mem_cgroup_oom_enable(void)
133 {
134 WARN_ON(current->memcg_oom.may_oom);
135 current->memcg_oom.may_oom = 1;
136 }
137
138 static inline void mem_cgroup_oom_disable(void)
139 {
140 WARN_ON(!current->memcg_oom.may_oom);
141 current->memcg_oom.may_oom = 0;
142 }
143
144 static inline bool task_in_memcg_oom(struct task_struct *p)
145 {
146 return p->memcg_oom.memcg;
147 }
148
149 bool mem_cgroup_oom_synchronize(bool wait);
150
151 #ifdef CONFIG_MEMCG_SWAP
152 extern int do_swap_account;
153 #endif
154
155 static inline bool mem_cgroup_disabled(void)
156 {
157 if (memory_cgrp_subsys.disabled)
158 return true;
159 return false;
160 }
161
162 struct mem_cgroup *mem_cgroup_begin_page_stat(struct page *page);
163 void mem_cgroup_update_page_stat(struct mem_cgroup *memcg,
164 enum mem_cgroup_stat_index idx, int val);
165 void mem_cgroup_end_page_stat(struct mem_cgroup *memcg);
166
167 static inline void mem_cgroup_inc_page_stat(struct mem_cgroup *memcg,
168 enum mem_cgroup_stat_index idx)
169 {
170 mem_cgroup_update_page_stat(memcg, idx, 1);
171 }
172
173 static inline void mem_cgroup_dec_page_stat(struct mem_cgroup *memcg,
174 enum mem_cgroup_stat_index idx)
175 {
176 mem_cgroup_update_page_stat(memcg, idx, -1);
177 }
178
179 unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
180 gfp_t gfp_mask,
181 unsigned long *total_scanned);
182
183 void __mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx);
184 static inline void mem_cgroup_count_vm_event(struct mm_struct *mm,
185 enum vm_event_item idx)
186 {
187 if (mem_cgroup_disabled())
188 return;
189 __mem_cgroup_count_vm_event(mm, idx);
190 }
191 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
192 void mem_cgroup_split_huge_fixup(struct page *head);
193 #endif
194
195 #else /* CONFIG_MEMCG */
196 struct mem_cgroup;
197
198 static inline void mem_cgroup_events(struct mem_cgroup *memcg,
199 enum mem_cgroup_events_index idx,
200 unsigned int nr)
201 {
202 }
203
204 static inline bool mem_cgroup_low(struct mem_cgroup *root,
205 struct mem_cgroup *memcg)
206 {
207 return false;
208 }
209
210 static inline int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
211 gfp_t gfp_mask,
212 struct mem_cgroup **memcgp)
213 {
214 *memcgp = NULL;
215 return 0;
216 }
217
218 static inline void mem_cgroup_commit_charge(struct page *page,
219 struct mem_cgroup *memcg,
220 bool lrucare)
221 {
222 }
223
224 static inline void mem_cgroup_cancel_charge(struct page *page,
225 struct mem_cgroup *memcg)
226 {
227 }
228
229 static inline void mem_cgroup_uncharge(struct page *page)
230 {
231 }
232
233 static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
234 {
235 }
236
237 static inline void mem_cgroup_migrate(struct page *oldpage,
238 struct page *newpage,
239 bool lrucare)
240 {
241 }
242
243 static inline struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
244 struct mem_cgroup *memcg)
245 {
246 return &zone->lruvec;
247 }
248
249 static inline struct lruvec *mem_cgroup_page_lruvec(struct page *page,
250 struct zone *zone)
251 {
252 return &zone->lruvec;
253 }
254
255 static inline struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
256 {
257 return NULL;
258 }
259
260 static inline bool mm_match_cgroup(struct mm_struct *mm,
261 struct mem_cgroup *memcg)
262 {
263 return true;
264 }
265
266 static inline bool task_in_mem_cgroup(struct task_struct *task,
267 const struct mem_cgroup *memcg)
268 {
269 return true;
270 }
271
272 static inline struct cgroup_subsys_state
273 *mem_cgroup_css(struct mem_cgroup *memcg)
274 {
275 return NULL;
276 }
277
278 static inline struct mem_cgroup *
279 mem_cgroup_iter(struct mem_cgroup *root,
280 struct mem_cgroup *prev,
281 struct mem_cgroup_reclaim_cookie *reclaim)
282 {
283 return NULL;
284 }
285
286 static inline void mem_cgroup_iter_break(struct mem_cgroup *root,
287 struct mem_cgroup *prev)
288 {
289 }
290
291 static inline bool mem_cgroup_disabled(void)
292 {
293 return true;
294 }
295
296 static inline int
297 mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec)
298 {
299 return 1;
300 }
301
302 static inline bool mem_cgroup_lruvec_online(struct lruvec *lruvec)
303 {
304 return true;
305 }
306
307 static inline unsigned long
308 mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
309 {
310 return 0;
311 }
312
313 static inline void
314 mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
315 int increment)
316 {
317 }
318
319 static inline void
320 mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
321 {
322 }
323
324 static inline struct mem_cgroup *mem_cgroup_begin_page_stat(struct page *page)
325 {
326 return NULL;
327 }
328
329 static inline void mem_cgroup_end_page_stat(struct mem_cgroup *memcg)
330 {
331 }
332
333 static inline void mem_cgroup_oom_enable(void)
334 {
335 }
336
337 static inline void mem_cgroup_oom_disable(void)
338 {
339 }
340
341 static inline bool task_in_memcg_oom(struct task_struct *p)
342 {
343 return false;
344 }
345
346 static inline bool mem_cgroup_oom_synchronize(bool wait)
347 {
348 return false;
349 }
350
351 static inline void mem_cgroup_inc_page_stat(struct mem_cgroup *memcg,
352 enum mem_cgroup_stat_index idx)
353 {
354 }
355
356 static inline void mem_cgroup_dec_page_stat(struct mem_cgroup *memcg,
357 enum mem_cgroup_stat_index idx)
358 {
359 }
360
361 static inline
362 unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
363 gfp_t gfp_mask,
364 unsigned long *total_scanned)
365 {
366 return 0;
367 }
368
369 static inline void mem_cgroup_split_huge_fixup(struct page *head)
370 {
371 }
372
373 static inline
374 void mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx)
375 {
376 }
377 #endif /* CONFIG_MEMCG */
378
379 enum {
380 UNDER_LIMIT,
381 SOFT_LIMIT,
382 OVER_LIMIT,
383 };
384
385 struct sock;
386 #if defined(CONFIG_INET) && defined(CONFIG_MEMCG_KMEM)
387 void sock_update_memcg(struct sock *sk);
388 void sock_release_memcg(struct sock *sk);
389 #else
390 static inline void sock_update_memcg(struct sock *sk)
391 {
392 }
393 static inline void sock_release_memcg(struct sock *sk)
394 {
395 }
396 #endif /* CONFIG_INET && CONFIG_MEMCG_KMEM */
397
398 #ifdef CONFIG_MEMCG_KMEM
399 extern struct static_key memcg_kmem_enabled_key;
400
401 extern int memcg_nr_cache_ids;
402 extern void memcg_get_cache_ids(void);
403 extern void memcg_put_cache_ids(void);
404
405 /*
406 * Helper macro to loop through all memcg-specific caches. Callers must still
407 * check if the cache is valid (it is either valid or NULL).
408 * the slab_mutex must be held when looping through those caches
409 */
410 #define for_each_memcg_cache_index(_idx) \
411 for ((_idx) = 0; (_idx) < memcg_nr_cache_ids; (_idx)++)
412
413 static inline bool memcg_kmem_enabled(void)
414 {
415 return static_key_false(&memcg_kmem_enabled_key);
416 }
417
418 bool memcg_kmem_is_active(struct mem_cgroup *memcg);
419
420 /*
421 * In general, we'll do everything in our power to not incur in any overhead
422 * for non-memcg users for the kmem functions. Not even a function call, if we
423 * can avoid it.
424 *
425 * Therefore, we'll inline all those functions so that in the best case, we'll
426 * see that kmemcg is off for everybody and proceed quickly. If it is on,
427 * we'll still do most of the flag checking inline. We check a lot of
428 * conditions, but because they are pretty simple, they are expected to be
429 * fast.
430 */
431 bool __memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **memcg,
432 int order);
433 void __memcg_kmem_commit_charge(struct page *page,
434 struct mem_cgroup *memcg, int order);
435 void __memcg_kmem_uncharge_pages(struct page *page, int order);
436
437 int memcg_cache_id(struct mem_cgroup *memcg);
438
439 struct kmem_cache *__memcg_kmem_get_cache(struct kmem_cache *cachep);
440 void __memcg_kmem_put_cache(struct kmem_cache *cachep);
441
442 struct mem_cgroup *__mem_cgroup_from_kmem(void *ptr);
443
444 int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp,
445 unsigned long nr_pages);
446 void memcg_uncharge_kmem(struct mem_cgroup *memcg, unsigned long nr_pages);
447
448 /**
449 * memcg_kmem_newpage_charge: verify if a new kmem allocation is allowed.
450 * @gfp: the gfp allocation flags.
451 * @memcg: a pointer to the memcg this was charged against.
452 * @order: allocation order.
453 *
454 * returns true if the memcg where the current task belongs can hold this
455 * allocation.
456 *
457 * We return true automatically if this allocation is not to be accounted to
458 * any memcg.
459 */
460 static inline bool
461 memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **memcg, int order)
462 {
463 if (!memcg_kmem_enabled())
464 return true;
465
466 /*
467 * __GFP_NOFAIL allocations will move on even if charging is not
468 * possible. Therefore we don't even try, and have this allocation
469 * unaccounted. We could in theory charge it forcibly, but we hope
470 * those allocations are rare, and won't be worth the trouble.
471 */
472 if (gfp & __GFP_NOFAIL)
473 return true;
474 if (in_interrupt() || (!current->mm) || (current->flags & PF_KTHREAD))
475 return true;
476
477 /* If the test is dying, just let it go. */
478 if (unlikely(fatal_signal_pending(current)))
479 return true;
480
481 return __memcg_kmem_newpage_charge(gfp, memcg, order);
482 }
483
484 /**
485 * memcg_kmem_uncharge_pages: uncharge pages from memcg
486 * @page: pointer to struct page being freed
487 * @order: allocation order.
488 */
489 static inline void
490 memcg_kmem_uncharge_pages(struct page *page, int order)
491 {
492 if (memcg_kmem_enabled())
493 __memcg_kmem_uncharge_pages(page, order);
494 }
495
496 /**
497 * memcg_kmem_commit_charge: embeds correct memcg in a page
498 * @page: pointer to struct page recently allocated
499 * @memcg: the memcg structure we charged against
500 * @order: allocation order.
501 *
502 * Needs to be called after memcg_kmem_newpage_charge, regardless of success or
503 * failure of the allocation. if @page is NULL, this function will revert the
504 * charges. Otherwise, it will commit @page to @memcg.
505 */
506 static inline void
507 memcg_kmem_commit_charge(struct page *page, struct mem_cgroup *memcg, int order)
508 {
509 if (memcg_kmem_enabled() && memcg)
510 __memcg_kmem_commit_charge(page, memcg, order);
511 }
512
513 /**
514 * memcg_kmem_get_cache: selects the correct per-memcg cache for allocation
515 * @cachep: the original global kmem cache
516 * @gfp: allocation flags.
517 *
518 * All memory allocated from a per-memcg cache is charged to the owner memcg.
519 */
520 static __always_inline struct kmem_cache *
521 memcg_kmem_get_cache(struct kmem_cache *cachep, gfp_t gfp)
522 {
523 if (!memcg_kmem_enabled())
524 return cachep;
525 if (gfp & __GFP_NOFAIL)
526 return cachep;
527 if (in_interrupt() || (!current->mm) || (current->flags & PF_KTHREAD))
528 return cachep;
529 if (unlikely(fatal_signal_pending(current)))
530 return cachep;
531
532 return __memcg_kmem_get_cache(cachep);
533 }
534
535 static __always_inline void memcg_kmem_put_cache(struct kmem_cache *cachep)
536 {
537 if (memcg_kmem_enabled())
538 __memcg_kmem_put_cache(cachep);
539 }
540
541 static __always_inline struct mem_cgroup *mem_cgroup_from_kmem(void *ptr)
542 {
543 if (!memcg_kmem_enabled())
544 return NULL;
545 return __mem_cgroup_from_kmem(ptr);
546 }
547 #else
548 #define for_each_memcg_cache_index(_idx) \
549 for (; NULL; )
550
551 static inline bool memcg_kmem_enabled(void)
552 {
553 return false;
554 }
555
556 static inline bool memcg_kmem_is_active(struct mem_cgroup *memcg)
557 {
558 return false;
559 }
560
561 static inline bool
562 memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **memcg, int order)
563 {
564 return true;
565 }
566
567 static inline void memcg_kmem_uncharge_pages(struct page *page, int order)
568 {
569 }
570
571 static inline void
572 memcg_kmem_commit_charge(struct page *page, struct mem_cgroup *memcg, int order)
573 {
574 }
575
576 static inline int memcg_cache_id(struct mem_cgroup *memcg)
577 {
578 return -1;
579 }
580
581 static inline void memcg_get_cache_ids(void)
582 {
583 }
584
585 static inline void memcg_put_cache_ids(void)
586 {
587 }
588
589 static inline struct kmem_cache *
590 memcg_kmem_get_cache(struct kmem_cache *cachep, gfp_t gfp)
591 {
592 return cachep;
593 }
594
595 static inline void memcg_kmem_put_cache(struct kmem_cache *cachep)
596 {
597 }
598
599 static inline struct mem_cgroup *mem_cgroup_from_kmem(void *ptr)
600 {
601 return NULL;
602 }
603 #endif /* CONFIG_MEMCG_KMEM */
604 #endif /* _LINUX_MEMCONTROL_H */
605