]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - mm/memcontrol.c
Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/nab/target...
[mirror_ubuntu-bionic-kernel.git] / mm / memcontrol.c
CommitLineData
8cdea7c0
BS
1/* memcontrol.c - Memory Controller
2 *
3 * Copyright IBM Corporation, 2007
4 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
5 *
78fb7466
PE
6 * Copyright 2007 OpenVZ SWsoft Inc
7 * Author: Pavel Emelianov <xemul@openvz.org>
8 *
2e72b634
KS
9 * Memory thresholds
10 * Copyright (C) 2009 Nokia Corporation
11 * Author: Kirill A. Shutemov
12 *
7ae1e1d0
GC
13 * Kernel Memory Controller
14 * Copyright (C) 2012 Parallels Inc. and Google Inc.
15 * Authors: Glauber Costa and Suleiman Souhlal
16 *
1575e68b
JW
17 * Native page reclaim
18 * Charge lifetime sanitation
19 * Lockless page tracking & accounting
20 * Unified hierarchy configuration model
21 * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner
22 *
8cdea7c0
BS
23 * This program is free software; you can redistribute it and/or modify
24 * it under the terms of the GNU General Public License as published by
25 * the Free Software Foundation; either version 2 of the License, or
26 * (at your option) any later version.
27 *
28 * This program is distributed in the hope that it will be useful,
29 * but WITHOUT ANY WARRANTY; without even the implied warranty of
30 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
31 * GNU General Public License for more details.
32 */
33
3e32cb2e 34#include <linux/page_counter.h>
8cdea7c0
BS
35#include <linux/memcontrol.h>
36#include <linux/cgroup.h>
78fb7466 37#include <linux/mm.h>
6e84f315 38#include <linux/sched/mm.h>
3a4f8a0b 39#include <linux/shmem_fs.h>
4ffef5fe 40#include <linux/hugetlb.h>
d13d1443 41#include <linux/pagemap.h>
d52aa412 42#include <linux/smp.h>
8a9f3ccd 43#include <linux/page-flags.h>
66e1707b 44#include <linux/backing-dev.h>
8a9f3ccd
BS
45#include <linux/bit_spinlock.h>
46#include <linux/rcupdate.h>
e222432b 47#include <linux/limits.h>
b9e15baf 48#include <linux/export.h>
8c7c6e34 49#include <linux/mutex.h>
bb4cc1a8 50#include <linux/rbtree.h>
b6ac57d5 51#include <linux/slab.h>
66e1707b 52#include <linux/swap.h>
02491447 53#include <linux/swapops.h>
66e1707b 54#include <linux/spinlock.h>
2e72b634 55#include <linux/eventfd.h>
79bd9814 56#include <linux/poll.h>
2e72b634 57#include <linux/sort.h>
66e1707b 58#include <linux/fs.h>
d2ceb9b7 59#include <linux/seq_file.h>
70ddf637 60#include <linux/vmpressure.h>
b69408e8 61#include <linux/mm_inline.h>
5d1ea48b 62#include <linux/swap_cgroup.h>
cdec2e42 63#include <linux/cpu.h>
158e0a2d 64#include <linux/oom.h>
0056f4e6 65#include <linux/lockdep.h>
79bd9814 66#include <linux/file.h>
b23afb93 67#include <linux/tracehook.h>
08e552c6 68#include "internal.h"
d1a4c0b3 69#include <net/sock.h>
4bd2c1ee 70#include <net/ip.h>
f35c3a8e 71#include "slab.h"
8cdea7c0 72
7c0f6ba6 73#include <linux/uaccess.h>
8697d331 74
cc8e970c
KM
75#include <trace/events/vmscan.h>
76
073219e9
TH
77struct cgroup_subsys memory_cgrp_subsys __read_mostly;
78EXPORT_SYMBOL(memory_cgrp_subsys);
68ae564b 79
7d828602
JW
80struct mem_cgroup *root_mem_cgroup __read_mostly;
81
a181b0e8 82#define MEM_CGROUP_RECLAIM_RETRIES 5
8cdea7c0 83
f7e1cb6e
JW
84/* Socket memory accounting disabled? */
85static bool cgroup_memory_nosocket;
86
04823c83
VD
87/* Kernel memory accounting disabled? */
88static bool cgroup_memory_nokmem;
89
21afa38e 90/* Whether the swap controller is active */
c255a458 91#ifdef CONFIG_MEMCG_SWAP
c077719b 92int do_swap_account __read_mostly;
c077719b 93#else
a0db00fc 94#define do_swap_account 0
c077719b
KH
95#endif
96
7941d214
JW
97/* Whether legacy memory+swap accounting is active */
98static bool do_memsw_account(void)
99{
100 return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && do_swap_account;
101}
102
71cd3113 103static const char *const mem_cgroup_lru_names[] = {
58cf188e
SZ
104 "inactive_anon",
105 "active_anon",
106 "inactive_file",
107 "active_file",
108 "unevictable",
109};
110
a0db00fc
KS
111#define THRESHOLDS_EVENTS_TARGET 128
112#define SOFTLIMIT_EVENTS_TARGET 1024
113#define NUMAINFO_EVENTS_TARGET 1024
e9f8974f 114
bb4cc1a8
AM
115/*
116 * Cgroups above their limits are maintained in a RB-Tree, independent of
117 * their hierarchy representation
118 */
119
ef8f2327 120struct mem_cgroup_tree_per_node {
bb4cc1a8
AM
121 struct rb_root rb_root;
122 spinlock_t lock;
123};
124
bb4cc1a8
AM
125struct mem_cgroup_tree {
126 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
127};
128
129static struct mem_cgroup_tree soft_limit_tree __read_mostly;
130
9490ff27
KH
131/* for OOM */
132struct mem_cgroup_eventfd_list {
133 struct list_head list;
134 struct eventfd_ctx *eventfd;
135};
2e72b634 136
79bd9814
TH
137/*
138 * cgroup_event represents events which userspace want to receive.
139 */
3bc942f3 140struct mem_cgroup_event {
79bd9814 141 /*
59b6f873 142 * memcg which the event belongs to.
79bd9814 143 */
59b6f873 144 struct mem_cgroup *memcg;
79bd9814
TH
145 /*
146 * eventfd to signal userspace about the event.
147 */
148 struct eventfd_ctx *eventfd;
149 /*
150 * Each of these stored in a list by the cgroup.
151 */
152 struct list_head list;
fba94807
TH
153 /*
154 * register_event() callback will be used to add new userspace
155 * waiter for changes related to this event. Use eventfd_signal()
156 * on eventfd to send notification to userspace.
157 */
59b6f873 158 int (*register_event)(struct mem_cgroup *memcg,
347c4a87 159 struct eventfd_ctx *eventfd, const char *args);
fba94807
TH
160 /*
161 * unregister_event() callback will be called when userspace closes
162 * the eventfd or on cgroup removing. This callback must be set,
163 * if you want provide notification functionality.
164 */
59b6f873 165 void (*unregister_event)(struct mem_cgroup *memcg,
fba94807 166 struct eventfd_ctx *eventfd);
79bd9814
TH
167 /*
168 * All fields below needed to unregister event when
169 * userspace closes eventfd.
170 */
171 poll_table pt;
172 wait_queue_head_t *wqh;
173 wait_queue_t wait;
174 struct work_struct remove;
175};
176
c0ff4b85
R
177static void mem_cgroup_threshold(struct mem_cgroup *memcg);
178static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
2e72b634 179
7dc74be0
DN
180/* Stuffs for move charges at task migration. */
181/*
1dfab5ab 182 * Types of charges to be moved.
7dc74be0 183 */
1dfab5ab
JW
184#define MOVE_ANON 0x1U
185#define MOVE_FILE 0x2U
186#define MOVE_MASK (MOVE_ANON | MOVE_FILE)
7dc74be0 187
4ffef5fe
DN
188/* "mc" and its members are protected by cgroup_mutex */
189static struct move_charge_struct {
b1dd693e 190 spinlock_t lock; /* for from, to */
264a0ae1 191 struct mm_struct *mm;
4ffef5fe
DN
192 struct mem_cgroup *from;
193 struct mem_cgroup *to;
1dfab5ab 194 unsigned long flags;
4ffef5fe 195 unsigned long precharge;
854ffa8d 196 unsigned long moved_charge;
483c30b5 197 unsigned long moved_swap;
8033b97c
DN
198 struct task_struct *moving_task; /* a task moving charges */
199 wait_queue_head_t waitq; /* a waitq for other context */
200} mc = {
2bd9bb20 201 .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
8033b97c
DN
202 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
203};
4ffef5fe 204
4e416953
BS
205/*
206 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
207 * limit reclaim to prevent infinite loops, if they ever occur.
208 */
a0db00fc 209#define MEM_CGROUP_MAX_RECLAIM_LOOPS 100
bb4cc1a8 210#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
4e416953 211
217bc319
KH
212enum charge_type {
213 MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
41326c17 214 MEM_CGROUP_CHARGE_TYPE_ANON,
d13d1443 215 MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
8a9478ca 216 MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */
c05555b5
KH
217 NR_CHARGE_TYPE,
218};
219
8c7c6e34 220/* for encoding cft->private value on file */
86ae53e1
GC
221enum res_type {
222 _MEM,
223 _MEMSWAP,
224 _OOM_TYPE,
510fc4e1 225 _KMEM,
d55f90bf 226 _TCP,
86ae53e1
GC
227};
228
a0db00fc
KS
229#define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))
230#define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff)
8c7c6e34 231#define MEMFILE_ATTR(val) ((val) & 0xffff)
9490ff27
KH
232/* Used for OOM nofiier */
233#define OOM_CONTROL (0)
8c7c6e34 234
70ddf637
AV
235/* Some nice accessors for the vmpressure. */
236struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
237{
238 if (!memcg)
239 memcg = root_mem_cgroup;
240 return &memcg->vmpressure;
241}
242
243struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
244{
245 return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
246}
247
7ffc0edc
MH
248static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
249{
250 return (memcg == root_mem_cgroup);
251}
252
127424c8 253#ifndef CONFIG_SLOB
55007d84 254/*
f7ce3190 255 * This will be the memcg's index in each cache's ->memcg_params.memcg_caches.
b8627835
LZ
256 * The main reason for not using cgroup id for this:
257 * this works better in sparse environments, where we have a lot of memcgs,
258 * but only a few kmem-limited. Or also, if we have, for instance, 200
259 * memcgs, and none but the 200th is kmem-limited, we'd have to have a
260 * 200 entry array for that.
55007d84 261 *
dbcf73e2
VD
262 * The current size of the caches array is stored in memcg_nr_cache_ids. It
263 * will double each time we have to increase it.
55007d84 264 */
dbcf73e2
VD
265static DEFINE_IDA(memcg_cache_ida);
266int memcg_nr_cache_ids;
749c5415 267
05257a1a
VD
268/* Protects memcg_nr_cache_ids */
269static DECLARE_RWSEM(memcg_cache_ids_sem);
270
271void memcg_get_cache_ids(void)
272{
273 down_read(&memcg_cache_ids_sem);
274}
275
276void memcg_put_cache_ids(void)
277{
278 up_read(&memcg_cache_ids_sem);
279}
280
55007d84
GC
281/*
282 * MIN_SIZE is different than 1, because we would like to avoid going through
283 * the alloc/free process all the time. In a small machine, 4 kmem-limited
284 * cgroups is a reasonable guess. In the future, it could be a parameter or
285 * tunable, but that is strictly not necessary.
286 *
b8627835 287 * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get
55007d84
GC
288 * this constant directly from cgroup, but it is understandable that this is
289 * better kept as an internal representation in cgroup.c. In any case, the
b8627835 290 * cgrp_id space is not getting any smaller, and we don't have to necessarily
55007d84
GC
291 * increase ours as well if it increases.
292 */
293#define MEMCG_CACHES_MIN_SIZE 4
b8627835 294#define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX
55007d84 295
d7f25f8a
GC
296/*
297 * A lot of the calls to the cache allocation functions are expected to be
298 * inlined by the compiler. Since the calls to memcg_kmem_get_cache are
299 * conditional to this static branch, we'll have to allow modules that does
300 * kmem_cache_alloc and the such to see this symbol as well
301 */
ef12947c 302DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key);
d7f25f8a 303EXPORT_SYMBOL(memcg_kmem_enabled_key);
a8964b9b 304
17cc4dfe
TH
305struct workqueue_struct *memcg_kmem_cache_wq;
306
127424c8 307#endif /* !CONFIG_SLOB */
a8964b9b 308
ad7fa852
TH
309/**
310 * mem_cgroup_css_from_page - css of the memcg associated with a page
311 * @page: page of interest
312 *
313 * If memcg is bound to the default hierarchy, css of the memcg associated
314 * with @page is returned. The returned css remains associated with @page
315 * until it is released.
316 *
317 * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
318 * is returned.
ad7fa852
TH
319 */
320struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
321{
322 struct mem_cgroup *memcg;
323
ad7fa852
TH
324 memcg = page->mem_cgroup;
325
9e10a130 326 if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
ad7fa852
TH
327 memcg = root_mem_cgroup;
328
ad7fa852
TH
329 return &memcg->css;
330}
331
2fc04524
VD
332/**
333 * page_cgroup_ino - return inode number of the memcg a page is charged to
334 * @page: the page
335 *
336 * Look up the closest online ancestor of the memory cgroup @page is charged to
337 * and return its inode number or 0 if @page is not charged to any cgroup. It
338 * is safe to call this function without holding a reference to @page.
339 *
340 * Note, this function is inherently racy, because there is nothing to prevent
341 * the cgroup inode from getting torn down and potentially reallocated a moment
342 * after page_cgroup_ino() returns, so it only should be used by callers that
343 * do not care (such as procfs interfaces).
344 */
345ino_t page_cgroup_ino(struct page *page)
346{
347 struct mem_cgroup *memcg;
348 unsigned long ino = 0;
349
350 rcu_read_lock();
351 memcg = READ_ONCE(page->mem_cgroup);
352 while (memcg && !(memcg->css.flags & CSS_ONLINE))
353 memcg = parent_mem_cgroup(memcg);
354 if (memcg)
355 ino = cgroup_ino(memcg->css.cgroup);
356 rcu_read_unlock();
357 return ino;
358}
359
ef8f2327
MG
360static struct mem_cgroup_per_node *
361mem_cgroup_page_nodeinfo(struct mem_cgroup *memcg, struct page *page)
f64c3f54 362{
97a6c37b 363 int nid = page_to_nid(page);
f64c3f54 364
ef8f2327 365 return memcg->nodeinfo[nid];
f64c3f54
BS
366}
367
ef8f2327
MG
368static struct mem_cgroup_tree_per_node *
369soft_limit_tree_node(int nid)
bb4cc1a8 370{
ef8f2327 371 return soft_limit_tree.rb_tree_per_node[nid];
bb4cc1a8
AM
372}
373
ef8f2327 374static struct mem_cgroup_tree_per_node *
bb4cc1a8
AM
375soft_limit_tree_from_page(struct page *page)
376{
377 int nid = page_to_nid(page);
bb4cc1a8 378
ef8f2327 379 return soft_limit_tree.rb_tree_per_node[nid];
bb4cc1a8
AM
380}
381
ef8f2327
MG
382static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz,
383 struct mem_cgroup_tree_per_node *mctz,
3e32cb2e 384 unsigned long new_usage_in_excess)
bb4cc1a8
AM
385{
386 struct rb_node **p = &mctz->rb_root.rb_node;
387 struct rb_node *parent = NULL;
ef8f2327 388 struct mem_cgroup_per_node *mz_node;
bb4cc1a8
AM
389
390 if (mz->on_tree)
391 return;
392
393 mz->usage_in_excess = new_usage_in_excess;
394 if (!mz->usage_in_excess)
395 return;
396 while (*p) {
397 parent = *p;
ef8f2327 398 mz_node = rb_entry(parent, struct mem_cgroup_per_node,
bb4cc1a8
AM
399 tree_node);
400 if (mz->usage_in_excess < mz_node->usage_in_excess)
401 p = &(*p)->rb_left;
402 /*
403 * We can't avoid mem cgroups that are over their soft
404 * limit by the same amount
405 */
406 else if (mz->usage_in_excess >= mz_node->usage_in_excess)
407 p = &(*p)->rb_right;
408 }
409 rb_link_node(&mz->tree_node, parent, p);
410 rb_insert_color(&mz->tree_node, &mctz->rb_root);
411 mz->on_tree = true;
412}
413
ef8f2327
MG
414static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
415 struct mem_cgroup_tree_per_node *mctz)
bb4cc1a8
AM
416{
417 if (!mz->on_tree)
418 return;
419 rb_erase(&mz->tree_node, &mctz->rb_root);
420 mz->on_tree = false;
421}
422
ef8f2327
MG
423static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
424 struct mem_cgroup_tree_per_node *mctz)
bb4cc1a8 425{
0a31bc97
JW
426 unsigned long flags;
427
428 spin_lock_irqsave(&mctz->lock, flags);
cf2c8127 429 __mem_cgroup_remove_exceeded(mz, mctz);
0a31bc97 430 spin_unlock_irqrestore(&mctz->lock, flags);
bb4cc1a8
AM
431}
432
3e32cb2e
JW
433static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
434{
435 unsigned long nr_pages = page_counter_read(&memcg->memory);
4db0c3c2 436 unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
3e32cb2e
JW
437 unsigned long excess = 0;
438
439 if (nr_pages > soft_limit)
440 excess = nr_pages - soft_limit;
441
442 return excess;
443}
bb4cc1a8
AM
444
445static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
446{
3e32cb2e 447 unsigned long excess;
ef8f2327
MG
448 struct mem_cgroup_per_node *mz;
449 struct mem_cgroup_tree_per_node *mctz;
bb4cc1a8 450
e231875b 451 mctz = soft_limit_tree_from_page(page);
bfc7228b
LD
452 if (!mctz)
453 return;
bb4cc1a8
AM
454 /*
455 * Necessary to update all ancestors when hierarchy is used.
456 * because their event counter is not touched.
457 */
458 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
ef8f2327 459 mz = mem_cgroup_page_nodeinfo(memcg, page);
3e32cb2e 460 excess = soft_limit_excess(memcg);
bb4cc1a8
AM
461 /*
462 * We have to update the tree if mz is on RB-tree or
463 * mem is over its softlimit.
464 */
465 if (excess || mz->on_tree) {
0a31bc97
JW
466 unsigned long flags;
467
468 spin_lock_irqsave(&mctz->lock, flags);
bb4cc1a8
AM
469 /* if on-tree, remove it */
470 if (mz->on_tree)
cf2c8127 471 __mem_cgroup_remove_exceeded(mz, mctz);
bb4cc1a8
AM
472 /*
473 * Insert again. mz->usage_in_excess will be updated.
474 * If excess is 0, no tree ops.
475 */
cf2c8127 476 __mem_cgroup_insert_exceeded(mz, mctz, excess);
0a31bc97 477 spin_unlock_irqrestore(&mctz->lock, flags);
bb4cc1a8
AM
478 }
479 }
480}
481
482static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
483{
ef8f2327
MG
484 struct mem_cgroup_tree_per_node *mctz;
485 struct mem_cgroup_per_node *mz;
486 int nid;
bb4cc1a8 487
e231875b 488 for_each_node(nid) {
ef8f2327
MG
489 mz = mem_cgroup_nodeinfo(memcg, nid);
490 mctz = soft_limit_tree_node(nid);
bfc7228b
LD
491 if (mctz)
492 mem_cgroup_remove_exceeded(mz, mctz);
bb4cc1a8
AM
493 }
494}
495
ef8f2327
MG
496static struct mem_cgroup_per_node *
497__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
bb4cc1a8
AM
498{
499 struct rb_node *rightmost = NULL;
ef8f2327 500 struct mem_cgroup_per_node *mz;
bb4cc1a8
AM
501
502retry:
503 mz = NULL;
504 rightmost = rb_last(&mctz->rb_root);
505 if (!rightmost)
506 goto done; /* Nothing to reclaim from */
507
ef8f2327 508 mz = rb_entry(rightmost, struct mem_cgroup_per_node, tree_node);
bb4cc1a8
AM
509 /*
510 * Remove the node now but someone else can add it back,
511 * we will to add it back at the end of reclaim to its correct
512 * position in the tree.
513 */
cf2c8127 514 __mem_cgroup_remove_exceeded(mz, mctz);
3e32cb2e 515 if (!soft_limit_excess(mz->memcg) ||
ec903c0c 516 !css_tryget_online(&mz->memcg->css))
bb4cc1a8
AM
517 goto retry;
518done:
519 return mz;
520}
521
ef8f2327
MG
522static struct mem_cgroup_per_node *
523mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
bb4cc1a8 524{
ef8f2327 525 struct mem_cgroup_per_node *mz;
bb4cc1a8 526
0a31bc97 527 spin_lock_irq(&mctz->lock);
bb4cc1a8 528 mz = __mem_cgroup_largest_soft_limit_node(mctz);
0a31bc97 529 spin_unlock_irq(&mctz->lock);
bb4cc1a8
AM
530 return mz;
531}
532
711d3d2c 533/*
484ebb3b
GT
534 * Return page count for single (non recursive) @memcg.
535 *
711d3d2c
KH
536 * Implementation Note: reading percpu statistics for memcg.
537 *
538 * Both of vmstat[] and percpu_counter has threshold and do periodic
539 * synchronization to implement "quick" read. There are trade-off between
540 * reading cost and precision of value. Then, we may have a chance to implement
484ebb3b 541 * a periodic synchronization of counter in memcg's counter.
711d3d2c
KH
542 *
543 * But this _read() function is used for user interface now. The user accounts
544 * memory usage by memory cgroup and he _always_ requires exact value because
545 * he accounts memory. Even if we provide quick-and-fuzzy read, we always
546 * have to visit all online cpus and make sum. So, for now, unnecessary
547 * synchronization is not implemented. (just implemented for cpu hotplug)
548 *
549 * If there are kernel internal actions which can make use of some not-exact
550 * value, and reading all cpu value can be performance bottleneck in some
484ebb3b 551 * common workload, threshold and synchronization as vmstat[] should be
711d3d2c
KH
552 * implemented.
553 */
c62b1a3b 554
ccda7f43
JW
555static unsigned long memcg_sum_events(struct mem_cgroup *memcg,
556 enum memcg_event_item event)
e9f8974f
JW
557{
558 unsigned long val = 0;
559 int cpu;
560
733a572e 561 for_each_possible_cpu(cpu)
df0e53d0 562 val += per_cpu(memcg->stat->events[event], cpu);
e9f8974f
JW
563 return val;
564}
565
c0ff4b85 566static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
b070e65c 567 struct page *page,
f627c2f5 568 bool compound, int nr_pages)
d52aa412 569{
b2402857
KH
570 /*
571 * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
572 * counted as CACHE even if it's on ANON LRU.
573 */
0a31bc97 574 if (PageAnon(page))
71cd3113 575 __this_cpu_add(memcg->stat->count[MEMCG_RSS], nr_pages);
9a4caf1e 576 else {
71cd3113 577 __this_cpu_add(memcg->stat->count[MEMCG_CACHE], nr_pages);
9a4caf1e 578 if (PageSwapBacked(page))
71cd3113 579 __this_cpu_add(memcg->stat->count[NR_SHMEM], nr_pages);
9a4caf1e 580 }
55e462b0 581
f627c2f5
KS
582 if (compound) {
583 VM_BUG_ON_PAGE(!PageTransHuge(page), page);
71cd3113 584 __this_cpu_add(memcg->stat->count[MEMCG_RSS_HUGE], nr_pages);
f627c2f5 585 }
b070e65c 586
e401f176
KH
587 /* pagein of a big page is an event. So, ignore page size */
588 if (nr_pages > 0)
df0e53d0 589 __this_cpu_inc(memcg->stat->events[PGPGIN]);
3751d604 590 else {
df0e53d0 591 __this_cpu_inc(memcg->stat->events[PGPGOUT]);
3751d604
KH
592 nr_pages = -nr_pages; /* for event */
593 }
e401f176 594
13114716 595 __this_cpu_add(memcg->stat->nr_page_events, nr_pages);
6d12e2d8
KH
596}
597
0a6b76dd
VD
598unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
599 int nid, unsigned int lru_mask)
bb2a0de9 600{
b4536f0c 601 struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg);
e231875b 602 unsigned long nr = 0;
ef8f2327 603 enum lru_list lru;
889976db 604
e231875b 605 VM_BUG_ON((unsigned)nid >= nr_node_ids);
bb2a0de9 606
ef8f2327
MG
607 for_each_lru(lru) {
608 if (!(BIT(lru) & lru_mask))
609 continue;
b4536f0c 610 nr += mem_cgroup_get_lru_size(lruvec, lru);
e231875b
JZ
611 }
612 return nr;
889976db 613}
bb2a0de9 614
c0ff4b85 615static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
bb2a0de9 616 unsigned int lru_mask)
6d12e2d8 617{
e231875b 618 unsigned long nr = 0;
889976db 619 int nid;
6d12e2d8 620
31aaea4a 621 for_each_node_state(nid, N_MEMORY)
e231875b
JZ
622 nr += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
623 return nr;
d52aa412
KH
624}
625
f53d7ce3
JW
626static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
627 enum mem_cgroup_events_target target)
7a159cc9
JW
628{
629 unsigned long val, next;
630
13114716 631 val = __this_cpu_read(memcg->stat->nr_page_events);
4799401f 632 next = __this_cpu_read(memcg->stat->targets[target]);
7a159cc9 633 /* from time_after() in jiffies.h */
f53d7ce3
JW
634 if ((long)next - (long)val < 0) {
635 switch (target) {
636 case MEM_CGROUP_TARGET_THRESH:
637 next = val + THRESHOLDS_EVENTS_TARGET;
638 break;
bb4cc1a8
AM
639 case MEM_CGROUP_TARGET_SOFTLIMIT:
640 next = val + SOFTLIMIT_EVENTS_TARGET;
641 break;
f53d7ce3
JW
642 case MEM_CGROUP_TARGET_NUMAINFO:
643 next = val + NUMAINFO_EVENTS_TARGET;
644 break;
645 default:
646 break;
647 }
648 __this_cpu_write(memcg->stat->targets[target], next);
649 return true;
7a159cc9 650 }
f53d7ce3 651 return false;
d2265e6f
KH
652}
653
654/*
655 * Check events in order.
656 *
657 */
c0ff4b85 658static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
d2265e6f
KH
659{
660 /* threshold event is triggered in finer grain than soft limit */
f53d7ce3
JW
661 if (unlikely(mem_cgroup_event_ratelimit(memcg,
662 MEM_CGROUP_TARGET_THRESH))) {
bb4cc1a8 663 bool do_softlimit;
82b3f2a7 664 bool do_numainfo __maybe_unused;
f53d7ce3 665
bb4cc1a8
AM
666 do_softlimit = mem_cgroup_event_ratelimit(memcg,
667 MEM_CGROUP_TARGET_SOFTLIMIT);
f53d7ce3
JW
668#if MAX_NUMNODES > 1
669 do_numainfo = mem_cgroup_event_ratelimit(memcg,
670 MEM_CGROUP_TARGET_NUMAINFO);
671#endif
c0ff4b85 672 mem_cgroup_threshold(memcg);
bb4cc1a8
AM
673 if (unlikely(do_softlimit))
674 mem_cgroup_update_tree(memcg, page);
453a9bf3 675#if MAX_NUMNODES > 1
f53d7ce3 676 if (unlikely(do_numainfo))
c0ff4b85 677 atomic_inc(&memcg->numainfo_events);
453a9bf3 678#endif
0a31bc97 679 }
d2265e6f
KH
680}
681
cf475ad2 682struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
78fb7466 683{
31a78f23
BS
684 /*
685 * mm_update_next_owner() may clear mm->owner to NULL
686 * if it races with swapoff, page migration, etc.
687 * So this can be called with p == NULL.
688 */
689 if (unlikely(!p))
690 return NULL;
691
073219e9 692 return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
78fb7466 693}
33398cf2 694EXPORT_SYMBOL(mem_cgroup_from_task);
78fb7466 695
df381975 696static struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
54595fe2 697{
c0ff4b85 698 struct mem_cgroup *memcg = NULL;
0b7f569e 699
54595fe2
KH
700 rcu_read_lock();
701 do {
6f6acb00
MH
702 /*
703 * Page cache insertions can happen withou an
704 * actual mm context, e.g. during disk probing
705 * on boot, loopback IO, acct() writes etc.
706 */
707 if (unlikely(!mm))
df381975 708 memcg = root_mem_cgroup;
6f6acb00
MH
709 else {
710 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
711 if (unlikely(!memcg))
712 memcg = root_mem_cgroup;
713 }
ec903c0c 714 } while (!css_tryget_online(&memcg->css));
54595fe2 715 rcu_read_unlock();
c0ff4b85 716 return memcg;
54595fe2
KH
717}
718
5660048c
JW
719/**
720 * mem_cgroup_iter - iterate over memory cgroup hierarchy
721 * @root: hierarchy root
722 * @prev: previously returned memcg, NULL on first invocation
723 * @reclaim: cookie for shared reclaim walks, NULL for full walks
724 *
725 * Returns references to children of the hierarchy below @root, or
726 * @root itself, or %NULL after a full round-trip.
727 *
728 * Caller must pass the return value in @prev on subsequent
729 * invocations for reference counting, or use mem_cgroup_iter_break()
730 * to cancel a hierarchy walk before the round-trip is complete.
731 *
732 * Reclaimers can specify a zone and a priority level in @reclaim to
733 * divide up the memcgs in the hierarchy among all concurrent
734 * reclaimers operating on the same zone and priority.
735 */
694fbc0f 736struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
5660048c 737 struct mem_cgroup *prev,
694fbc0f 738 struct mem_cgroup_reclaim_cookie *reclaim)
14067bb3 739{
33398cf2 740 struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
5ac8fb31 741 struct cgroup_subsys_state *css = NULL;
9f3a0d09 742 struct mem_cgroup *memcg = NULL;
5ac8fb31 743 struct mem_cgroup *pos = NULL;
711d3d2c 744
694fbc0f
AM
745 if (mem_cgroup_disabled())
746 return NULL;
5660048c 747
9f3a0d09
JW
748 if (!root)
749 root = root_mem_cgroup;
7d74b06f 750
9f3a0d09 751 if (prev && !reclaim)
5ac8fb31 752 pos = prev;
14067bb3 753
9f3a0d09
JW
754 if (!root->use_hierarchy && root != root_mem_cgroup) {
755 if (prev)
5ac8fb31 756 goto out;
694fbc0f 757 return root;
9f3a0d09 758 }
14067bb3 759
542f85f9 760 rcu_read_lock();
5f578161 761
5ac8fb31 762 if (reclaim) {
ef8f2327 763 struct mem_cgroup_per_node *mz;
5ac8fb31 764
ef8f2327 765 mz = mem_cgroup_nodeinfo(root, reclaim->pgdat->node_id);
5ac8fb31
JW
766 iter = &mz->iter[reclaim->priority];
767
768 if (prev && reclaim->generation != iter->generation)
769 goto out_unlock;
770
6df38689 771 while (1) {
4db0c3c2 772 pos = READ_ONCE(iter->position);
6df38689
VD
773 if (!pos || css_tryget(&pos->css))
774 break;
5ac8fb31 775 /*
6df38689
VD
776 * css reference reached zero, so iter->position will
777 * be cleared by ->css_released. However, we should not
778 * rely on this happening soon, because ->css_released
779 * is called from a work queue, and by busy-waiting we
780 * might block it. So we clear iter->position right
781 * away.
5ac8fb31 782 */
6df38689
VD
783 (void)cmpxchg(&iter->position, pos, NULL);
784 }
5ac8fb31
JW
785 }
786
787 if (pos)
788 css = &pos->css;
789
790 for (;;) {
791 css = css_next_descendant_pre(css, &root->css);
792 if (!css) {
793 /*
794 * Reclaimers share the hierarchy walk, and a
795 * new one might jump in right at the end of
796 * the hierarchy - make sure they see at least
797 * one group and restart from the beginning.
798 */
799 if (!prev)
800 continue;
801 break;
527a5ec9 802 }
7d74b06f 803
5ac8fb31
JW
804 /*
805 * Verify the css and acquire a reference. The root
806 * is provided by the caller, so we know it's alive
807 * and kicking, and don't take an extra reference.
808 */
809 memcg = mem_cgroup_from_css(css);
14067bb3 810
5ac8fb31
JW
811 if (css == &root->css)
812 break;
14067bb3 813
0b8f73e1
JW
814 if (css_tryget(css))
815 break;
9f3a0d09 816
5ac8fb31 817 memcg = NULL;
9f3a0d09 818 }
5ac8fb31
JW
819
820 if (reclaim) {
5ac8fb31 821 /*
6df38689
VD
822 * The position could have already been updated by a competing
823 * thread, so check that the value hasn't changed since we read
824 * it to avoid reclaiming from the same cgroup twice.
5ac8fb31 825 */
6df38689
VD
826 (void)cmpxchg(&iter->position, pos, memcg);
827
5ac8fb31
JW
828 if (pos)
829 css_put(&pos->css);
830
831 if (!memcg)
832 iter->generation++;
833 else if (!prev)
834 reclaim->generation = iter->generation;
9f3a0d09 835 }
5ac8fb31 836
542f85f9
MH
837out_unlock:
838 rcu_read_unlock();
5ac8fb31 839out:
c40046f3
MH
840 if (prev && prev != root)
841 css_put(&prev->css);
842
9f3a0d09 843 return memcg;
14067bb3 844}
7d74b06f 845
5660048c
JW
846/**
847 * mem_cgroup_iter_break - abort a hierarchy walk prematurely
848 * @root: hierarchy root
849 * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
850 */
851void mem_cgroup_iter_break(struct mem_cgroup *root,
852 struct mem_cgroup *prev)
9f3a0d09
JW
853{
854 if (!root)
855 root = root_mem_cgroup;
856 if (prev && prev != root)
857 css_put(&prev->css);
858}
7d74b06f 859
6df38689
VD
860static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
861{
862 struct mem_cgroup *memcg = dead_memcg;
863 struct mem_cgroup_reclaim_iter *iter;
ef8f2327
MG
864 struct mem_cgroup_per_node *mz;
865 int nid;
6df38689
VD
866 int i;
867
868 while ((memcg = parent_mem_cgroup(memcg))) {
869 for_each_node(nid) {
ef8f2327
MG
870 mz = mem_cgroup_nodeinfo(memcg, nid);
871 for (i = 0; i <= DEF_PRIORITY; i++) {
872 iter = &mz->iter[i];
873 cmpxchg(&iter->position,
874 dead_memcg, NULL);
6df38689
VD
875 }
876 }
877 }
878}
879
9f3a0d09
JW
880/*
881 * Iteration constructs for visiting all cgroups (under a tree). If
882 * loops are exited prematurely (break), mem_cgroup_iter_break() must
883 * be used for reference counting.
884 */
885#define for_each_mem_cgroup_tree(iter, root) \
527a5ec9 886 for (iter = mem_cgroup_iter(root, NULL, NULL); \
9f3a0d09 887 iter != NULL; \
527a5ec9 888 iter = mem_cgroup_iter(root, iter, NULL))
711d3d2c 889
9f3a0d09 890#define for_each_mem_cgroup(iter) \
527a5ec9 891 for (iter = mem_cgroup_iter(NULL, NULL, NULL); \
9f3a0d09 892 iter != NULL; \
527a5ec9 893 iter = mem_cgroup_iter(NULL, iter, NULL))
14067bb3 894
7c5f64f8
VD
895/**
896 * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy
897 * @memcg: hierarchy root
898 * @fn: function to call for each task
899 * @arg: argument passed to @fn
900 *
901 * This function iterates over tasks attached to @memcg or to any of its
902 * descendants and calls @fn for each task. If @fn returns a non-zero
903 * value, the function breaks the iteration loop and returns the value.
904 * Otherwise, it will iterate over all tasks and return 0.
905 *
906 * This function must not be called for the root memory cgroup.
907 */
908int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
909 int (*fn)(struct task_struct *, void *), void *arg)
910{
911 struct mem_cgroup *iter;
912 int ret = 0;
913
914 BUG_ON(memcg == root_mem_cgroup);
915
916 for_each_mem_cgroup_tree(iter, memcg) {
917 struct css_task_iter it;
918 struct task_struct *task;
919
920 css_task_iter_start(&iter->css, &it);
921 while (!ret && (task = css_task_iter_next(&it)))
922 ret = fn(task, arg);
923 css_task_iter_end(&it);
924 if (ret) {
925 mem_cgroup_iter_break(memcg, iter);
926 break;
927 }
928 }
929 return ret;
930}
931
925b7673 932/**
dfe0e773 933 * mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page
925b7673 934 * @page: the page
fa9add64 935 * @zone: zone of the page
dfe0e773
JW
936 *
937 * This function is only safe when following the LRU page isolation
938 * and putback protocol: the LRU lock must be held, and the page must
939 * either be PageLRU() or the caller must have isolated/allocated it.
925b7673 940 */
599d0c95 941struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgdat)
08e552c6 942{
ef8f2327 943 struct mem_cgroup_per_node *mz;
925b7673 944 struct mem_cgroup *memcg;
bea8c150 945 struct lruvec *lruvec;
6d12e2d8 946
bea8c150 947 if (mem_cgroup_disabled()) {
599d0c95 948 lruvec = &pgdat->lruvec;
bea8c150
HD
949 goto out;
950 }
925b7673 951
1306a85a 952 memcg = page->mem_cgroup;
7512102c 953 /*
dfe0e773 954 * Swapcache readahead pages are added to the LRU - and
29833315 955 * possibly migrated - before they are charged.
7512102c 956 */
29833315
JW
957 if (!memcg)
958 memcg = root_mem_cgroup;
7512102c 959
ef8f2327 960 mz = mem_cgroup_page_nodeinfo(memcg, page);
bea8c150
HD
961 lruvec = &mz->lruvec;
962out:
963 /*
964 * Since a node can be onlined after the mem_cgroup was created,
965 * we have to be prepared to initialize lruvec->zone here;
966 * and if offlined then reonlined, we need to reinitialize it.
967 */
599d0c95
MG
968 if (unlikely(lruvec->pgdat != pgdat))
969 lruvec->pgdat = pgdat;
bea8c150 970 return lruvec;
08e552c6 971}
b69408e8 972
925b7673 973/**
fa9add64
HD
974 * mem_cgroup_update_lru_size - account for adding or removing an lru page
975 * @lruvec: mem_cgroup per zone lru vector
976 * @lru: index of lru list the page is sitting on
b4536f0c 977 * @zid: zone id of the accounted pages
fa9add64 978 * @nr_pages: positive when adding or negative when removing
925b7673 979 *
ca707239
HD
980 * This function must be called under lru_lock, just before a page is added
981 * to or just after a page is removed from an lru list (that ordering being
982 * so as to allow it to check that lru_size 0 is consistent with list_empty).
3f58a829 983 */
fa9add64 984void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
b4536f0c 985 int zid, int nr_pages)
3f58a829 986{
ef8f2327 987 struct mem_cgroup_per_node *mz;
fa9add64 988 unsigned long *lru_size;
ca707239 989 long size;
3f58a829
MK
990
991 if (mem_cgroup_disabled())
992 return;
993
ef8f2327 994 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
b4536f0c 995 lru_size = &mz->lru_zone_size[zid][lru];
ca707239
HD
996
997 if (nr_pages < 0)
998 *lru_size += nr_pages;
999
1000 size = *lru_size;
b4536f0c
MH
1001 if (WARN_ONCE(size < 0,
1002 "%s(%p, %d, %d): lru_size %ld\n",
1003 __func__, lruvec, lru, nr_pages, size)) {
ca707239
HD
1004 VM_BUG_ON(1);
1005 *lru_size = 0;
1006 }
1007
1008 if (nr_pages > 0)
1009 *lru_size += nr_pages;
08e552c6 1010}
544122e5 1011
2314b42d 1012bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg)
c3ac9a8a 1013{
2314b42d 1014 struct mem_cgroup *task_memcg;
158e0a2d 1015 struct task_struct *p;
ffbdccf5 1016 bool ret;
4c4a2214 1017
158e0a2d 1018 p = find_lock_task_mm(task);
de077d22 1019 if (p) {
2314b42d 1020 task_memcg = get_mem_cgroup_from_mm(p->mm);
de077d22
DR
1021 task_unlock(p);
1022 } else {
1023 /*
1024 * All threads may have already detached their mm's, but the oom
1025 * killer still needs to detect if they have already been oom
1026 * killed to prevent needlessly killing additional tasks.
1027 */
ffbdccf5 1028 rcu_read_lock();
2314b42d
JW
1029 task_memcg = mem_cgroup_from_task(task);
1030 css_get(&task_memcg->css);
ffbdccf5 1031 rcu_read_unlock();
de077d22 1032 }
2314b42d
JW
1033 ret = mem_cgroup_is_descendant(task_memcg, memcg);
1034 css_put(&task_memcg->css);
4c4a2214
DR
1035 return ret;
1036}
1037
19942822 1038/**
9d11ea9f 1039 * mem_cgroup_margin - calculate chargeable space of a memory cgroup
dad7557e 1040 * @memcg: the memory cgroup
19942822 1041 *
9d11ea9f 1042 * Returns the maximum amount of memory @mem can be charged with, in
7ec99d62 1043 * pages.
19942822 1044 */
c0ff4b85 1045static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
19942822 1046{
3e32cb2e
JW
1047 unsigned long margin = 0;
1048 unsigned long count;
1049 unsigned long limit;
9d11ea9f 1050
3e32cb2e 1051 count = page_counter_read(&memcg->memory);
4db0c3c2 1052 limit = READ_ONCE(memcg->memory.limit);
3e32cb2e
JW
1053 if (count < limit)
1054 margin = limit - count;
1055
7941d214 1056 if (do_memsw_account()) {
3e32cb2e 1057 count = page_counter_read(&memcg->memsw);
4db0c3c2 1058 limit = READ_ONCE(memcg->memsw.limit);
3e32cb2e
JW
1059 if (count <= limit)
1060 margin = min(margin, limit - count);
cbedbac3
LR
1061 else
1062 margin = 0;
3e32cb2e
JW
1063 }
1064
1065 return margin;
19942822
JW
1066}
1067
32047e2a 1068/*
bdcbb659 1069 * A routine for checking "mem" is under move_account() or not.
32047e2a 1070 *
bdcbb659
QH
1071 * Checking a cgroup is mc.from or mc.to or under hierarchy of
1072 * moving cgroups. This is for waiting at high-memory pressure
1073 * caused by "move".
32047e2a 1074 */
c0ff4b85 1075static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
4b534334 1076{
2bd9bb20
KH
1077 struct mem_cgroup *from;
1078 struct mem_cgroup *to;
4b534334 1079 bool ret = false;
2bd9bb20
KH
1080 /*
1081 * Unlike task_move routines, we access mc.to, mc.from not under
1082 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1083 */
1084 spin_lock(&mc.lock);
1085 from = mc.from;
1086 to = mc.to;
1087 if (!from)
1088 goto unlock;
3e92041d 1089
2314b42d
JW
1090 ret = mem_cgroup_is_descendant(from, memcg) ||
1091 mem_cgroup_is_descendant(to, memcg);
2bd9bb20
KH
1092unlock:
1093 spin_unlock(&mc.lock);
4b534334
KH
1094 return ret;
1095}
1096
c0ff4b85 1097static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
4b534334
KH
1098{
1099 if (mc.moving_task && current != mc.moving_task) {
c0ff4b85 1100 if (mem_cgroup_under_move(memcg)) {
4b534334
KH
1101 DEFINE_WAIT(wait);
1102 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1103 /* moving charge context might have finished. */
1104 if (mc.moving_task)
1105 schedule();
1106 finish_wait(&mc.waitq, &wait);
1107 return true;
1108 }
1109 }
1110 return false;
1111}
1112
71cd3113
JW
1113unsigned int memcg1_stats[] = {
1114 MEMCG_CACHE,
1115 MEMCG_RSS,
1116 MEMCG_RSS_HUGE,
1117 NR_SHMEM,
1118 NR_FILE_MAPPED,
1119 NR_FILE_DIRTY,
1120 NR_WRITEBACK,
1121 MEMCG_SWAP,
1122};
1123
1124static const char *const memcg1_stat_names[] = {
1125 "cache",
1126 "rss",
1127 "rss_huge",
1128 "shmem",
1129 "mapped_file",
1130 "dirty",
1131 "writeback",
1132 "swap",
1133};
1134
58cf188e 1135#define K(x) ((x) << (PAGE_SHIFT-10))
e222432b 1136/**
58cf188e 1137 * mem_cgroup_print_oom_info: Print OOM information relevant to memory controller.
e222432b
BS
1138 * @memcg: The memory cgroup that went over limit
1139 * @p: Task that is going to be killed
1140 *
1141 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1142 * enabled
1143 */
1144void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
1145{
58cf188e
SZ
1146 struct mem_cgroup *iter;
1147 unsigned int i;
e222432b 1148
e222432b
BS
1149 rcu_read_lock();
1150
2415b9f5
BV
1151 if (p) {
1152 pr_info("Task in ");
1153 pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
1154 pr_cont(" killed as a result of limit of ");
1155 } else {
1156 pr_info("Memory limit reached of cgroup ");
1157 }
1158
e61734c5 1159 pr_cont_cgroup_path(memcg->css.cgroup);
0346dadb 1160 pr_cont("\n");
e222432b 1161
e222432b
BS
1162 rcu_read_unlock();
1163
3e32cb2e
JW
1164 pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
1165 K((u64)page_counter_read(&memcg->memory)),
1166 K((u64)memcg->memory.limit), memcg->memory.failcnt);
1167 pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
1168 K((u64)page_counter_read(&memcg->memsw)),
1169 K((u64)memcg->memsw.limit), memcg->memsw.failcnt);
1170 pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
1171 K((u64)page_counter_read(&memcg->kmem)),
1172 K((u64)memcg->kmem.limit), memcg->kmem.failcnt);
58cf188e
SZ
1173
1174 for_each_mem_cgroup_tree(iter, memcg) {
e61734c5
TH
1175 pr_info("Memory cgroup stats for ");
1176 pr_cont_cgroup_path(iter->css.cgroup);
58cf188e
SZ
1177 pr_cont(":");
1178
71cd3113
JW
1179 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
1180 if (memcg1_stats[i] == MEMCG_SWAP && !do_swap_account)
58cf188e 1181 continue;
71cd3113 1182 pr_cont(" %s:%luKB", memcg1_stat_names[i],
ccda7f43 1183 K(memcg_page_state(iter, memcg1_stats[i])));
58cf188e
SZ
1184 }
1185
1186 for (i = 0; i < NR_LRU_LISTS; i++)
1187 pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
1188 K(mem_cgroup_nr_lru_pages(iter, BIT(i))));
1189
1190 pr_cont("\n");
1191 }
e222432b
BS
1192}
1193
81d39c20
KH
1194/*
1195 * This function returns the number of memcg under hierarchy tree. Returns
1196 * 1(self count) if no children.
1197 */
c0ff4b85 1198static int mem_cgroup_count_children(struct mem_cgroup *memcg)
81d39c20
KH
1199{
1200 int num = 0;
7d74b06f
KH
1201 struct mem_cgroup *iter;
1202
c0ff4b85 1203 for_each_mem_cgroup_tree(iter, memcg)
7d74b06f 1204 num++;
81d39c20
KH
1205 return num;
1206}
1207
a63d83f4
DR
1208/*
1209 * Return the memory (and swap, if configured) limit for a memcg.
1210 */
7c5f64f8 1211unsigned long mem_cgroup_get_limit(struct mem_cgroup *memcg)
a63d83f4 1212{
3e32cb2e 1213 unsigned long limit;
f3e8eb70 1214
3e32cb2e 1215 limit = memcg->memory.limit;
9a5a8f19 1216 if (mem_cgroup_swappiness(memcg)) {
3e32cb2e 1217 unsigned long memsw_limit;
37e84351 1218 unsigned long swap_limit;
9a5a8f19 1219
3e32cb2e 1220 memsw_limit = memcg->memsw.limit;
37e84351
VD
1221 swap_limit = memcg->swap.limit;
1222 swap_limit = min(swap_limit, (unsigned long)total_swap_pages);
1223 limit = min(limit + swap_limit, memsw_limit);
9a5a8f19 1224 }
9a5a8f19 1225 return limit;
a63d83f4
DR
1226}
1227
b6e6edcf 1228static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
19965460 1229 int order)
9cbb78bb 1230{
6e0fc46d
DR
1231 struct oom_control oc = {
1232 .zonelist = NULL,
1233 .nodemask = NULL,
2a966b77 1234 .memcg = memcg,
6e0fc46d
DR
1235 .gfp_mask = gfp_mask,
1236 .order = order,
6e0fc46d 1237 };
7c5f64f8 1238 bool ret;
9cbb78bb 1239
dc56401f 1240 mutex_lock(&oom_lock);
7c5f64f8 1241 ret = out_of_memory(&oc);
dc56401f 1242 mutex_unlock(&oom_lock);
7c5f64f8 1243 return ret;
9cbb78bb
DR
1244}
1245
ae6e71d3
MC
1246#if MAX_NUMNODES > 1
1247
4d0c066d
KH
1248/**
1249 * test_mem_cgroup_node_reclaimable
dad7557e 1250 * @memcg: the target memcg
4d0c066d
KH
1251 * @nid: the node ID to be checked.
1252 * @noswap : specify true here if the user wants flle only information.
1253 *
1254 * This function returns whether the specified memcg contains any
1255 * reclaimable pages on a node. Returns true if there are any reclaimable
1256 * pages in the node.
1257 */
c0ff4b85 1258static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
4d0c066d
KH
1259 int nid, bool noswap)
1260{
c0ff4b85 1261 if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
4d0c066d
KH
1262 return true;
1263 if (noswap || !total_swap_pages)
1264 return false;
c0ff4b85 1265 if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
4d0c066d
KH
1266 return true;
1267 return false;
1268
1269}
889976db
YH
1270
1271/*
1272 * Always updating the nodemask is not very good - even if we have an empty
1273 * list or the wrong list here, we can start from some node and traverse all
1274 * nodes based on the zonelist. So update the list loosely once per 10 secs.
1275 *
1276 */
c0ff4b85 1277static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
889976db
YH
1278{
1279 int nid;
453a9bf3
KH
1280 /*
1281 * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
1282 * pagein/pageout changes since the last update.
1283 */
c0ff4b85 1284 if (!atomic_read(&memcg->numainfo_events))
453a9bf3 1285 return;
c0ff4b85 1286 if (atomic_inc_return(&memcg->numainfo_updating) > 1)
889976db
YH
1287 return;
1288
889976db 1289 /* make a nodemask where this memcg uses memory from */
31aaea4a 1290 memcg->scan_nodes = node_states[N_MEMORY];
889976db 1291
31aaea4a 1292 for_each_node_mask(nid, node_states[N_MEMORY]) {
889976db 1293
c0ff4b85
R
1294 if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
1295 node_clear(nid, memcg->scan_nodes);
889976db 1296 }
453a9bf3 1297
c0ff4b85
R
1298 atomic_set(&memcg->numainfo_events, 0);
1299 atomic_set(&memcg->numainfo_updating, 0);
889976db
YH
1300}
1301
1302/*
1303 * Selecting a node where we start reclaim from. Because what we need is just
1304 * reducing usage counter, start from anywhere is O,K. Considering
1305 * memory reclaim from current node, there are pros. and cons.
1306 *
1307 * Freeing memory from current node means freeing memory from a node which
1308 * we'll use or we've used. So, it may make LRU bad. And if several threads
1309 * hit limits, it will see a contention on a node. But freeing from remote
1310 * node means more costs for memory reclaim because of memory latency.
1311 *
1312 * Now, we use round-robin. Better algorithm is welcomed.
1313 */
c0ff4b85 1314int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
889976db
YH
1315{
1316 int node;
1317
c0ff4b85
R
1318 mem_cgroup_may_update_nodemask(memcg);
1319 node = memcg->last_scanned_node;
889976db 1320
0edaf86c 1321 node = next_node_in(node, memcg->scan_nodes);
889976db 1322 /*
fda3d69b
MH
1323 * mem_cgroup_may_update_nodemask might have seen no reclaimmable pages
1324 * last time it really checked all the LRUs due to rate limiting.
1325 * Fallback to the current node in that case for simplicity.
889976db
YH
1326 */
1327 if (unlikely(node == MAX_NUMNODES))
1328 node = numa_node_id();
1329
c0ff4b85 1330 memcg->last_scanned_node = node;
889976db
YH
1331 return node;
1332}
889976db 1333#else
c0ff4b85 1334int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
889976db
YH
1335{
1336 return 0;
1337}
1338#endif
1339
0608f43d 1340static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
ef8f2327 1341 pg_data_t *pgdat,
0608f43d
AM
1342 gfp_t gfp_mask,
1343 unsigned long *total_scanned)
1344{
1345 struct mem_cgroup *victim = NULL;
1346 int total = 0;
1347 int loop = 0;
1348 unsigned long excess;
1349 unsigned long nr_scanned;
1350 struct mem_cgroup_reclaim_cookie reclaim = {
ef8f2327 1351 .pgdat = pgdat,
0608f43d
AM
1352 .priority = 0,
1353 };
1354
3e32cb2e 1355 excess = soft_limit_excess(root_memcg);
0608f43d
AM
1356
1357 while (1) {
1358 victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
1359 if (!victim) {
1360 loop++;
1361 if (loop >= 2) {
1362 /*
1363 * If we have not been able to reclaim
1364 * anything, it might because there are
1365 * no reclaimable pages under this hierarchy
1366 */
1367 if (!total)
1368 break;
1369 /*
1370 * We want to do more targeted reclaim.
1371 * excess >> 2 is not to excessive so as to
1372 * reclaim too much, nor too less that we keep
1373 * coming back to reclaim from this cgroup
1374 */
1375 if (total >= (excess >> 2) ||
1376 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
1377 break;
1378 }
1379 continue;
1380 }
a9dd0a83 1381 total += mem_cgroup_shrink_node(victim, gfp_mask, false,
ef8f2327 1382 pgdat, &nr_scanned);
0608f43d 1383 *total_scanned += nr_scanned;
3e32cb2e 1384 if (!soft_limit_excess(root_memcg))
0608f43d 1385 break;
6d61ef40 1386 }
0608f43d
AM
1387 mem_cgroup_iter_break(root_memcg, victim);
1388 return total;
6d61ef40
BS
1389}
1390
0056f4e6
JW
1391#ifdef CONFIG_LOCKDEP
1392static struct lockdep_map memcg_oom_lock_dep_map = {
1393 .name = "memcg_oom_lock",
1394};
1395#endif
1396
fb2a6fc5
JW
1397static DEFINE_SPINLOCK(memcg_oom_lock);
1398
867578cb
KH
1399/*
1400 * Check OOM-Killer is already running under our hierarchy.
1401 * If someone is running, return false.
1402 */
fb2a6fc5 1403static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
867578cb 1404{
79dfdacc 1405 struct mem_cgroup *iter, *failed = NULL;
a636b327 1406
fb2a6fc5
JW
1407 spin_lock(&memcg_oom_lock);
1408
9f3a0d09 1409 for_each_mem_cgroup_tree(iter, memcg) {
23751be0 1410 if (iter->oom_lock) {
79dfdacc
MH
1411 /*
1412 * this subtree of our hierarchy is already locked
1413 * so we cannot give a lock.
1414 */
79dfdacc 1415 failed = iter;
9f3a0d09
JW
1416 mem_cgroup_iter_break(memcg, iter);
1417 break;
23751be0
JW
1418 } else
1419 iter->oom_lock = true;
7d74b06f 1420 }
867578cb 1421
fb2a6fc5
JW
1422 if (failed) {
1423 /*
1424 * OK, we failed to lock the whole subtree so we have
1425 * to clean up what we set up to the failing subtree
1426 */
1427 for_each_mem_cgroup_tree(iter, memcg) {
1428 if (iter == failed) {
1429 mem_cgroup_iter_break(memcg, iter);
1430 break;
1431 }
1432 iter->oom_lock = false;
79dfdacc 1433 }
0056f4e6
JW
1434 } else
1435 mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
fb2a6fc5
JW
1436
1437 spin_unlock(&memcg_oom_lock);
1438
1439 return !failed;
a636b327 1440}
0b7f569e 1441
fb2a6fc5 1442static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
0b7f569e 1443{
7d74b06f
KH
1444 struct mem_cgroup *iter;
1445
fb2a6fc5 1446 spin_lock(&memcg_oom_lock);
0056f4e6 1447 mutex_release(&memcg_oom_lock_dep_map, 1, _RET_IP_);
c0ff4b85 1448 for_each_mem_cgroup_tree(iter, memcg)
79dfdacc 1449 iter->oom_lock = false;
fb2a6fc5 1450 spin_unlock(&memcg_oom_lock);
79dfdacc
MH
1451}
1452
c0ff4b85 1453static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
79dfdacc
MH
1454{
1455 struct mem_cgroup *iter;
1456
c2b42d3c 1457 spin_lock(&memcg_oom_lock);
c0ff4b85 1458 for_each_mem_cgroup_tree(iter, memcg)
c2b42d3c
TH
1459 iter->under_oom++;
1460 spin_unlock(&memcg_oom_lock);
79dfdacc
MH
1461}
1462
c0ff4b85 1463static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
79dfdacc
MH
1464{
1465 struct mem_cgroup *iter;
1466
867578cb
KH
1467 /*
1468 * When a new child is created while the hierarchy is under oom,
c2b42d3c 1469 * mem_cgroup_oom_lock() may not be called. Watch for underflow.
867578cb 1470 */
c2b42d3c 1471 spin_lock(&memcg_oom_lock);
c0ff4b85 1472 for_each_mem_cgroup_tree(iter, memcg)
c2b42d3c
TH
1473 if (iter->under_oom > 0)
1474 iter->under_oom--;
1475 spin_unlock(&memcg_oom_lock);
0b7f569e
KH
1476}
1477
867578cb
KH
1478static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1479
dc98df5a 1480struct oom_wait_info {
d79154bb 1481 struct mem_cgroup *memcg;
dc98df5a
KH
1482 wait_queue_t wait;
1483};
1484
1485static int memcg_oom_wake_function(wait_queue_t *wait,
1486 unsigned mode, int sync, void *arg)
1487{
d79154bb
HD
1488 struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
1489 struct mem_cgroup *oom_wait_memcg;
dc98df5a
KH
1490 struct oom_wait_info *oom_wait_info;
1491
1492 oom_wait_info = container_of(wait, struct oom_wait_info, wait);
d79154bb 1493 oom_wait_memcg = oom_wait_info->memcg;
dc98df5a 1494
2314b42d
JW
1495 if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
1496 !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
dc98df5a 1497 return 0;
dc98df5a
KH
1498 return autoremove_wake_function(wait, mode, sync, arg);
1499}
1500
c0ff4b85 1501static void memcg_oom_recover(struct mem_cgroup *memcg)
3c11ecf4 1502{
c2b42d3c
TH
1503 /*
1504 * For the following lockless ->under_oom test, the only required
1505 * guarantee is that it must see the state asserted by an OOM when
1506 * this function is called as a result of userland actions
1507 * triggered by the notification of the OOM. This is trivially
1508 * achieved by invoking mem_cgroup_mark_under_oom() before
1509 * triggering notification.
1510 */
1511 if (memcg && memcg->under_oom)
f4b90b70 1512 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
3c11ecf4
KH
1513}
1514
3812c8c8 1515static void mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
0b7f569e 1516{
d0db7afa 1517 if (!current->memcg_may_oom)
3812c8c8 1518 return;
867578cb 1519 /*
49426420
JW
1520 * We are in the middle of the charge context here, so we
1521 * don't want to block when potentially sitting on a callstack
1522 * that holds all kinds of filesystem and mm locks.
1523 *
1524 * Also, the caller may handle a failed allocation gracefully
1525 * (like optional page cache readahead) and so an OOM killer
1526 * invocation might not even be necessary.
1527 *
1528 * That's why we don't do anything here except remember the
1529 * OOM context and then deal with it at the end of the page
1530 * fault when the stack is unwound, the locks are released,
1531 * and when we know whether the fault was overall successful.
867578cb 1532 */
49426420 1533 css_get(&memcg->css);
626ebc41
TH
1534 current->memcg_in_oom = memcg;
1535 current->memcg_oom_gfp_mask = mask;
1536 current->memcg_oom_order = order;
3812c8c8
JW
1537}
1538
1539/**
1540 * mem_cgroup_oom_synchronize - complete memcg OOM handling
49426420 1541 * @handle: actually kill/wait or just clean up the OOM state
3812c8c8 1542 *
49426420
JW
1543 * This has to be called at the end of a page fault if the memcg OOM
1544 * handler was enabled.
3812c8c8 1545 *
49426420 1546 * Memcg supports userspace OOM handling where failed allocations must
3812c8c8
JW
1547 * sleep on a waitqueue until the userspace task resolves the
1548 * situation. Sleeping directly in the charge context with all kinds
1549 * of locks held is not a good idea, instead we remember an OOM state
1550 * in the task and mem_cgroup_oom_synchronize() has to be called at
49426420 1551 * the end of the page fault to complete the OOM handling.
3812c8c8
JW
1552 *
1553 * Returns %true if an ongoing memcg OOM situation was detected and
49426420 1554 * completed, %false otherwise.
3812c8c8 1555 */
49426420 1556bool mem_cgroup_oom_synchronize(bool handle)
3812c8c8 1557{
626ebc41 1558 struct mem_cgroup *memcg = current->memcg_in_oom;
3812c8c8 1559 struct oom_wait_info owait;
49426420 1560 bool locked;
3812c8c8
JW
1561
1562 /* OOM is global, do not handle */
3812c8c8 1563 if (!memcg)
49426420 1564 return false;
3812c8c8 1565
7c5f64f8 1566 if (!handle)
49426420 1567 goto cleanup;
3812c8c8
JW
1568
1569 owait.memcg = memcg;
1570 owait.wait.flags = 0;
1571 owait.wait.func = memcg_oom_wake_function;
1572 owait.wait.private = current;
1573 INIT_LIST_HEAD(&owait.wait.task_list);
867578cb 1574
3812c8c8 1575 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
49426420
JW
1576 mem_cgroup_mark_under_oom(memcg);
1577
1578 locked = mem_cgroup_oom_trylock(memcg);
1579
1580 if (locked)
1581 mem_cgroup_oom_notify(memcg);
1582
1583 if (locked && !memcg->oom_kill_disable) {
1584 mem_cgroup_unmark_under_oom(memcg);
1585 finish_wait(&memcg_oom_waitq, &owait.wait);
626ebc41
TH
1586 mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask,
1587 current->memcg_oom_order);
49426420 1588 } else {
3812c8c8 1589 schedule();
49426420
JW
1590 mem_cgroup_unmark_under_oom(memcg);
1591 finish_wait(&memcg_oom_waitq, &owait.wait);
1592 }
1593
1594 if (locked) {
fb2a6fc5
JW
1595 mem_cgroup_oom_unlock(memcg);
1596 /*
1597 * There is no guarantee that an OOM-lock contender
1598 * sees the wakeups triggered by the OOM kill
1599 * uncharges. Wake any sleepers explicitely.
1600 */
1601 memcg_oom_recover(memcg);
1602 }
49426420 1603cleanup:
626ebc41 1604 current->memcg_in_oom = NULL;
3812c8c8 1605 css_put(&memcg->css);
867578cb 1606 return true;
0b7f569e
KH
1607}
1608
d7365e78 1609/**
81f8c3a4
JW
1610 * lock_page_memcg - lock a page->mem_cgroup binding
1611 * @page: the page
32047e2a 1612 *
81f8c3a4
JW
1613 * This function protects unlocked LRU pages from being moved to
1614 * another cgroup and stabilizes their page->mem_cgroup binding.
d69b042f 1615 */
62cccb8c 1616void lock_page_memcg(struct page *page)
89c06bd5
KH
1617{
1618 struct mem_cgroup *memcg;
6de22619 1619 unsigned long flags;
89c06bd5 1620
6de22619
JW
1621 /*
1622 * The RCU lock is held throughout the transaction. The fast
1623 * path can get away without acquiring the memcg->move_lock
1624 * because page moving starts with an RCU grace period.
6de22619 1625 */
d7365e78
JW
1626 rcu_read_lock();
1627
1628 if (mem_cgroup_disabled())
62cccb8c 1629 return;
89c06bd5 1630again:
1306a85a 1631 memcg = page->mem_cgroup;
29833315 1632 if (unlikely(!memcg))
62cccb8c 1633 return;
d7365e78 1634
bdcbb659 1635 if (atomic_read(&memcg->moving_account) <= 0)
62cccb8c 1636 return;
89c06bd5 1637
6de22619 1638 spin_lock_irqsave(&memcg->move_lock, flags);
1306a85a 1639 if (memcg != page->mem_cgroup) {
6de22619 1640 spin_unlock_irqrestore(&memcg->move_lock, flags);
89c06bd5
KH
1641 goto again;
1642 }
6de22619
JW
1643
1644 /*
1645 * When charge migration first begins, we can have locked and
1646 * unlocked page stat updates happening concurrently. Track
81f8c3a4 1647 * the task who has the lock for unlock_page_memcg().
6de22619
JW
1648 */
1649 memcg->move_lock_task = current;
1650 memcg->move_lock_flags = flags;
d7365e78 1651
62cccb8c 1652 return;
89c06bd5 1653}
81f8c3a4 1654EXPORT_SYMBOL(lock_page_memcg);
89c06bd5 1655
d7365e78 1656/**
81f8c3a4 1657 * unlock_page_memcg - unlock a page->mem_cgroup binding
62cccb8c 1658 * @page: the page
d7365e78 1659 */
62cccb8c 1660void unlock_page_memcg(struct page *page)
89c06bd5 1661{
62cccb8c
JW
1662 struct mem_cgroup *memcg = page->mem_cgroup;
1663
6de22619
JW
1664 if (memcg && memcg->move_lock_task == current) {
1665 unsigned long flags = memcg->move_lock_flags;
1666
1667 memcg->move_lock_task = NULL;
1668 memcg->move_lock_flags = 0;
1669
1670 spin_unlock_irqrestore(&memcg->move_lock, flags);
1671 }
89c06bd5 1672
d7365e78 1673 rcu_read_unlock();
89c06bd5 1674}
81f8c3a4 1675EXPORT_SYMBOL(unlock_page_memcg);
89c06bd5 1676
cdec2e42
KH
1677/*
1678 * size of first charge trial. "32" comes from vmscan.c's magic value.
1679 * TODO: maybe necessary to use big numbers in big irons.
1680 */
7ec99d62 1681#define CHARGE_BATCH 32U
cdec2e42
KH
1682struct memcg_stock_pcp {
1683 struct mem_cgroup *cached; /* this never be root cgroup */
11c9ea4e 1684 unsigned int nr_pages;
cdec2e42 1685 struct work_struct work;
26fe6168 1686 unsigned long flags;
a0db00fc 1687#define FLUSHING_CACHED_CHARGE 0
cdec2e42
KH
1688};
1689static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
9f50fad6 1690static DEFINE_MUTEX(percpu_charge_mutex);
cdec2e42 1691
a0956d54
SS
1692/**
1693 * consume_stock: Try to consume stocked charge on this cpu.
1694 * @memcg: memcg to consume from.
1695 * @nr_pages: how many pages to charge.
1696 *
1697 * The charges will only happen if @memcg matches the current cpu's memcg
1698 * stock, and at least @nr_pages are available in that stock. Failure to
1699 * service an allocation will refill the stock.
1700 *
1701 * returns true if successful, false otherwise.
cdec2e42 1702 */
a0956d54 1703static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
cdec2e42
KH
1704{
1705 struct memcg_stock_pcp *stock;
db2ba40c 1706 unsigned long flags;
3e32cb2e 1707 bool ret = false;
cdec2e42 1708
a0956d54 1709 if (nr_pages > CHARGE_BATCH)
3e32cb2e 1710 return ret;
a0956d54 1711
db2ba40c
JW
1712 local_irq_save(flags);
1713
1714 stock = this_cpu_ptr(&memcg_stock);
3e32cb2e 1715 if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
a0956d54 1716 stock->nr_pages -= nr_pages;
3e32cb2e
JW
1717 ret = true;
1718 }
db2ba40c
JW
1719
1720 local_irq_restore(flags);
1721
cdec2e42
KH
1722 return ret;
1723}
1724
1725/*
3e32cb2e 1726 * Returns stocks cached in percpu and reset cached information.
cdec2e42
KH
1727 */
1728static void drain_stock(struct memcg_stock_pcp *stock)
1729{
1730 struct mem_cgroup *old = stock->cached;
1731
11c9ea4e 1732 if (stock->nr_pages) {
3e32cb2e 1733 page_counter_uncharge(&old->memory, stock->nr_pages);
7941d214 1734 if (do_memsw_account())
3e32cb2e 1735 page_counter_uncharge(&old->memsw, stock->nr_pages);
e8ea14cc 1736 css_put_many(&old->css, stock->nr_pages);
11c9ea4e 1737 stock->nr_pages = 0;
cdec2e42
KH
1738 }
1739 stock->cached = NULL;
cdec2e42
KH
1740}
1741
cdec2e42
KH
1742static void drain_local_stock(struct work_struct *dummy)
1743{
db2ba40c
JW
1744 struct memcg_stock_pcp *stock;
1745 unsigned long flags;
1746
1747 local_irq_save(flags);
1748
1749 stock = this_cpu_ptr(&memcg_stock);
cdec2e42 1750 drain_stock(stock);
26fe6168 1751 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
db2ba40c
JW
1752
1753 local_irq_restore(flags);
cdec2e42
KH
1754}
1755
1756/*
3e32cb2e 1757 * Cache charges(val) to local per_cpu area.
320cc51d 1758 * This will be consumed by consume_stock() function, later.
cdec2e42 1759 */
c0ff4b85 1760static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
cdec2e42 1761{
db2ba40c
JW
1762 struct memcg_stock_pcp *stock;
1763 unsigned long flags;
1764
1765 local_irq_save(flags);
cdec2e42 1766
db2ba40c 1767 stock = this_cpu_ptr(&memcg_stock);
c0ff4b85 1768 if (stock->cached != memcg) { /* reset if necessary */
cdec2e42 1769 drain_stock(stock);
c0ff4b85 1770 stock->cached = memcg;
cdec2e42 1771 }
11c9ea4e 1772 stock->nr_pages += nr_pages;
db2ba40c
JW
1773
1774 local_irq_restore(flags);
cdec2e42
KH
1775}
1776
1777/*
c0ff4b85 1778 * Drains all per-CPU charge caches for given root_memcg resp. subtree
6d3d6aa2 1779 * of the hierarchy under it.
cdec2e42 1780 */
6d3d6aa2 1781static void drain_all_stock(struct mem_cgroup *root_memcg)
cdec2e42 1782{
26fe6168 1783 int cpu, curcpu;
d38144b7 1784
6d3d6aa2
JW
1785 /* If someone's already draining, avoid adding running more workers. */
1786 if (!mutex_trylock(&percpu_charge_mutex))
1787 return;
cdec2e42 1788 /* Notify other cpus that system-wide "drain" is running */
cdec2e42 1789 get_online_cpus();
5af12d0e 1790 curcpu = get_cpu();
cdec2e42
KH
1791 for_each_online_cpu(cpu) {
1792 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
c0ff4b85 1793 struct mem_cgroup *memcg;
26fe6168 1794
c0ff4b85
R
1795 memcg = stock->cached;
1796 if (!memcg || !stock->nr_pages)
26fe6168 1797 continue;
2314b42d 1798 if (!mem_cgroup_is_descendant(memcg, root_memcg))
3e92041d 1799 continue;
d1a05b69
MH
1800 if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
1801 if (cpu == curcpu)
1802 drain_local_stock(&stock->work);
1803 else
1804 schedule_work_on(cpu, &stock->work);
1805 }
cdec2e42 1806 }
5af12d0e 1807 put_cpu();
f894ffa8 1808 put_online_cpus();
9f50fad6 1809 mutex_unlock(&percpu_charge_mutex);
cdec2e42
KH
1810}
1811
308167fc 1812static int memcg_hotplug_cpu_dead(unsigned int cpu)
cdec2e42 1813{
cdec2e42
KH
1814 struct memcg_stock_pcp *stock;
1815
cdec2e42
KH
1816 stock = &per_cpu(memcg_stock, cpu);
1817 drain_stock(stock);
308167fc 1818 return 0;
cdec2e42
KH
1819}
1820
f7e1cb6e
JW
1821static void reclaim_high(struct mem_cgroup *memcg,
1822 unsigned int nr_pages,
1823 gfp_t gfp_mask)
1824{
1825 do {
1826 if (page_counter_read(&memcg->memory) <= memcg->high)
1827 continue;
31176c78 1828 mem_cgroup_event(memcg, MEMCG_HIGH);
f7e1cb6e
JW
1829 try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask, true);
1830 } while ((memcg = parent_mem_cgroup(memcg)));
1831}
1832
1833static void high_work_func(struct work_struct *work)
1834{
1835 struct mem_cgroup *memcg;
1836
1837 memcg = container_of(work, struct mem_cgroup, high_work);
1838 reclaim_high(memcg, CHARGE_BATCH, GFP_KERNEL);
1839}
1840
b23afb93
TH
1841/*
1842 * Scheduled by try_charge() to be executed from the userland return path
1843 * and reclaims memory over the high limit.
1844 */
1845void mem_cgroup_handle_over_high(void)
1846{
1847 unsigned int nr_pages = current->memcg_nr_pages_over_high;
f7e1cb6e 1848 struct mem_cgroup *memcg;
b23afb93
TH
1849
1850 if (likely(!nr_pages))
1851 return;
1852
f7e1cb6e
JW
1853 memcg = get_mem_cgroup_from_mm(current->mm);
1854 reclaim_high(memcg, nr_pages, GFP_KERNEL);
b23afb93
TH
1855 css_put(&memcg->css);
1856 current->memcg_nr_pages_over_high = 0;
1857}
1858
00501b53
JW
1859static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
1860 unsigned int nr_pages)
8a9f3ccd 1861{
7ec99d62 1862 unsigned int batch = max(CHARGE_BATCH, nr_pages);
9b130619 1863 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
6539cc05 1864 struct mem_cgroup *mem_over_limit;
3e32cb2e 1865 struct page_counter *counter;
6539cc05 1866 unsigned long nr_reclaimed;
b70a2a21
JW
1867 bool may_swap = true;
1868 bool drained = false;
a636b327 1869
ce00a967 1870 if (mem_cgroup_is_root(memcg))
10d53c74 1871 return 0;
6539cc05 1872retry:
b6b6cc72 1873 if (consume_stock(memcg, nr_pages))
10d53c74 1874 return 0;
8a9f3ccd 1875
7941d214 1876 if (!do_memsw_account() ||
6071ca52
JW
1877 page_counter_try_charge(&memcg->memsw, batch, &counter)) {
1878 if (page_counter_try_charge(&memcg->memory, batch, &counter))
6539cc05 1879 goto done_restock;
7941d214 1880 if (do_memsw_account())
3e32cb2e
JW
1881 page_counter_uncharge(&memcg->memsw, batch);
1882 mem_over_limit = mem_cgroup_from_counter(counter, memory);
3fbe7244 1883 } else {
3e32cb2e 1884 mem_over_limit = mem_cgroup_from_counter(counter, memsw);
b70a2a21 1885 may_swap = false;
3fbe7244 1886 }
7a81b88c 1887
6539cc05
JW
1888 if (batch > nr_pages) {
1889 batch = nr_pages;
1890 goto retry;
1891 }
6d61ef40 1892
06b078fc
JW
1893 /*
1894 * Unlike in global OOM situations, memcg is not in a physical
1895 * memory shortage. Allow dying and OOM-killed tasks to
1896 * bypass the last charges so that they can exit quickly and
1897 * free their memory.
1898 */
1899 if (unlikely(test_thread_flag(TIF_MEMDIE) ||
1900 fatal_signal_pending(current) ||
1901 current->flags & PF_EXITING))
10d53c74 1902 goto force;
06b078fc 1903
89a28483
JW
1904 /*
1905 * Prevent unbounded recursion when reclaim operations need to
1906 * allocate memory. This might exceed the limits temporarily,
1907 * but we prefer facilitating memory reclaim and getting back
1908 * under the limit over triggering OOM kills in these cases.
1909 */
1910 if (unlikely(current->flags & PF_MEMALLOC))
1911 goto force;
1912
06b078fc
JW
1913 if (unlikely(task_in_memcg_oom(current)))
1914 goto nomem;
1915
d0164adc 1916 if (!gfpflags_allow_blocking(gfp_mask))
6539cc05 1917 goto nomem;
4b534334 1918
31176c78 1919 mem_cgroup_event(mem_over_limit, MEMCG_MAX);
241994ed 1920
b70a2a21
JW
1921 nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
1922 gfp_mask, may_swap);
6539cc05 1923
61e02c74 1924 if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
6539cc05 1925 goto retry;
28c34c29 1926
b70a2a21 1927 if (!drained) {
6d3d6aa2 1928 drain_all_stock(mem_over_limit);
b70a2a21
JW
1929 drained = true;
1930 goto retry;
1931 }
1932
28c34c29
JW
1933 if (gfp_mask & __GFP_NORETRY)
1934 goto nomem;
6539cc05
JW
1935 /*
1936 * Even though the limit is exceeded at this point, reclaim
1937 * may have been able to free some pages. Retry the charge
1938 * before killing the task.
1939 *
1940 * Only for regular pages, though: huge pages are rather
1941 * unlikely to succeed so close to the limit, and we fall back
1942 * to regular pages anyway in case of failure.
1943 */
61e02c74 1944 if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
6539cc05
JW
1945 goto retry;
1946 /*
1947 * At task move, charge accounts can be doubly counted. So, it's
1948 * better to wait until the end of task_move if something is going on.
1949 */
1950 if (mem_cgroup_wait_acct_move(mem_over_limit))
1951 goto retry;
1952
9b130619
JW
1953 if (nr_retries--)
1954 goto retry;
1955
06b078fc 1956 if (gfp_mask & __GFP_NOFAIL)
10d53c74 1957 goto force;
06b078fc 1958
6539cc05 1959 if (fatal_signal_pending(current))
10d53c74 1960 goto force;
6539cc05 1961
31176c78 1962 mem_cgroup_event(mem_over_limit, MEMCG_OOM);
241994ed 1963
3608de07
JM
1964 mem_cgroup_oom(mem_over_limit, gfp_mask,
1965 get_order(nr_pages * PAGE_SIZE));
7a81b88c 1966nomem:
6d1fdc48 1967 if (!(gfp_mask & __GFP_NOFAIL))
3168ecbe 1968 return -ENOMEM;
10d53c74
TH
1969force:
1970 /*
1971 * The allocation either can't fail or will lead to more memory
1972 * being freed very soon. Allow memory usage go over the limit
1973 * temporarily by force charging it.
1974 */
1975 page_counter_charge(&memcg->memory, nr_pages);
7941d214 1976 if (do_memsw_account())
10d53c74
TH
1977 page_counter_charge(&memcg->memsw, nr_pages);
1978 css_get_many(&memcg->css, nr_pages);
1979
1980 return 0;
6539cc05
JW
1981
1982done_restock:
e8ea14cc 1983 css_get_many(&memcg->css, batch);
6539cc05
JW
1984 if (batch > nr_pages)
1985 refill_stock(memcg, batch - nr_pages);
b23afb93 1986
241994ed 1987 /*
b23afb93
TH
1988 * If the hierarchy is above the normal consumption range, schedule
1989 * reclaim on returning to userland. We can perform reclaim here
71baba4b 1990 * if __GFP_RECLAIM but let's always punt for simplicity and so that
b23afb93
TH
1991 * GFP_KERNEL can consistently be used during reclaim. @memcg is
1992 * not recorded as it most likely matches current's and won't
1993 * change in the meantime. As high limit is checked again before
1994 * reclaim, the cost of mismatch is negligible.
241994ed
JW
1995 */
1996 do {
b23afb93 1997 if (page_counter_read(&memcg->memory) > memcg->high) {
f7e1cb6e
JW
1998 /* Don't bother a random interrupted task */
1999 if (in_interrupt()) {
2000 schedule_work(&memcg->high_work);
2001 break;
2002 }
9516a18a 2003 current->memcg_nr_pages_over_high += batch;
b23afb93
TH
2004 set_notify_resume(current);
2005 break;
2006 }
241994ed 2007 } while ((memcg = parent_mem_cgroup(memcg)));
10d53c74
TH
2008
2009 return 0;
7a81b88c 2010}
8a9f3ccd 2011
00501b53 2012static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
a3032a2c 2013{
ce00a967
JW
2014 if (mem_cgroup_is_root(memcg))
2015 return;
2016
3e32cb2e 2017 page_counter_uncharge(&memcg->memory, nr_pages);
7941d214 2018 if (do_memsw_account())
3e32cb2e 2019 page_counter_uncharge(&memcg->memsw, nr_pages);
ce00a967 2020
e8ea14cc 2021 css_put_many(&memcg->css, nr_pages);
d01dd17f
KH
2022}
2023
0a31bc97
JW
2024static void lock_page_lru(struct page *page, int *isolated)
2025{
2026 struct zone *zone = page_zone(page);
2027
a52633d8 2028 spin_lock_irq(zone_lru_lock(zone));
0a31bc97
JW
2029 if (PageLRU(page)) {
2030 struct lruvec *lruvec;
2031
599d0c95 2032 lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
0a31bc97
JW
2033 ClearPageLRU(page);
2034 del_page_from_lru_list(page, lruvec, page_lru(page));
2035 *isolated = 1;
2036 } else
2037 *isolated = 0;
2038}
2039
2040static void unlock_page_lru(struct page *page, int isolated)
2041{
2042 struct zone *zone = page_zone(page);
2043
2044 if (isolated) {
2045 struct lruvec *lruvec;
2046
599d0c95 2047 lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
0a31bc97
JW
2048 VM_BUG_ON_PAGE(PageLRU(page), page);
2049 SetPageLRU(page);
2050 add_page_to_lru_list(page, lruvec, page_lru(page));
2051 }
a52633d8 2052 spin_unlock_irq(zone_lru_lock(zone));
0a31bc97
JW
2053}
2054
00501b53 2055static void commit_charge(struct page *page, struct mem_cgroup *memcg,
6abb5a86 2056 bool lrucare)
7a81b88c 2057{
0a31bc97 2058 int isolated;
9ce70c02 2059
1306a85a 2060 VM_BUG_ON_PAGE(page->mem_cgroup, page);
9ce70c02
HD
2061
2062 /*
2063 * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
2064 * may already be on some other mem_cgroup's LRU. Take care of it.
2065 */
0a31bc97
JW
2066 if (lrucare)
2067 lock_page_lru(page, &isolated);
9ce70c02 2068
0a31bc97
JW
2069 /*
2070 * Nobody should be changing or seriously looking at
1306a85a 2071 * page->mem_cgroup at this point:
0a31bc97
JW
2072 *
2073 * - the page is uncharged
2074 *
2075 * - the page is off-LRU
2076 *
2077 * - an anonymous fault has exclusive page access, except for
2078 * a locked page table
2079 *
2080 * - a page cache insertion, a swapin fault, or a migration
2081 * have the page locked
2082 */
1306a85a 2083 page->mem_cgroup = memcg;
9ce70c02 2084
0a31bc97
JW
2085 if (lrucare)
2086 unlock_page_lru(page, isolated);
7a81b88c 2087}
66e1707b 2088
127424c8 2089#ifndef CONFIG_SLOB
f3bb3043 2090static int memcg_alloc_cache_id(void)
55007d84 2091{
f3bb3043
VD
2092 int id, size;
2093 int err;
2094
dbcf73e2 2095 id = ida_simple_get(&memcg_cache_ida,
f3bb3043
VD
2096 0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
2097 if (id < 0)
2098 return id;
55007d84 2099
dbcf73e2 2100 if (id < memcg_nr_cache_ids)
f3bb3043
VD
2101 return id;
2102
2103 /*
2104 * There's no space for the new id in memcg_caches arrays,
2105 * so we have to grow them.
2106 */
05257a1a 2107 down_write(&memcg_cache_ids_sem);
f3bb3043
VD
2108
2109 size = 2 * (id + 1);
55007d84
GC
2110 if (size < MEMCG_CACHES_MIN_SIZE)
2111 size = MEMCG_CACHES_MIN_SIZE;
2112 else if (size > MEMCG_CACHES_MAX_SIZE)
2113 size = MEMCG_CACHES_MAX_SIZE;
2114
f3bb3043 2115 err = memcg_update_all_caches(size);
60d3fd32
VD
2116 if (!err)
2117 err = memcg_update_all_list_lrus(size);
05257a1a
VD
2118 if (!err)
2119 memcg_nr_cache_ids = size;
2120
2121 up_write(&memcg_cache_ids_sem);
2122
f3bb3043 2123 if (err) {
dbcf73e2 2124 ida_simple_remove(&memcg_cache_ida, id);
f3bb3043
VD
2125 return err;
2126 }
2127 return id;
2128}
2129
2130static void memcg_free_cache_id(int id)
2131{
dbcf73e2 2132 ida_simple_remove(&memcg_cache_ida, id);
55007d84
GC
2133}
2134
d5b3cf71 2135struct memcg_kmem_cache_create_work {
5722d094
VD
2136 struct mem_cgroup *memcg;
2137 struct kmem_cache *cachep;
2138 struct work_struct work;
2139};
2140
d5b3cf71 2141static void memcg_kmem_cache_create_func(struct work_struct *w)
d7f25f8a 2142{
d5b3cf71
VD
2143 struct memcg_kmem_cache_create_work *cw =
2144 container_of(w, struct memcg_kmem_cache_create_work, work);
5722d094
VD
2145 struct mem_cgroup *memcg = cw->memcg;
2146 struct kmem_cache *cachep = cw->cachep;
d7f25f8a 2147
d5b3cf71 2148 memcg_create_kmem_cache(memcg, cachep);
bd673145 2149
5722d094 2150 css_put(&memcg->css);
d7f25f8a
GC
2151 kfree(cw);
2152}
2153
2154/*
2155 * Enqueue the creation of a per-memcg kmem_cache.
d7f25f8a 2156 */
d5b3cf71
VD
2157static void __memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
2158 struct kmem_cache *cachep)
d7f25f8a 2159{
d5b3cf71 2160 struct memcg_kmem_cache_create_work *cw;
d7f25f8a 2161
776ed0f0 2162 cw = kmalloc(sizeof(*cw), GFP_NOWAIT);
8135be5a 2163 if (!cw)
d7f25f8a 2164 return;
8135be5a
VD
2165
2166 css_get(&memcg->css);
d7f25f8a
GC
2167
2168 cw->memcg = memcg;
2169 cw->cachep = cachep;
d5b3cf71 2170 INIT_WORK(&cw->work, memcg_kmem_cache_create_func);
d7f25f8a 2171
17cc4dfe 2172 queue_work(memcg_kmem_cache_wq, &cw->work);
d7f25f8a
GC
2173}
2174
d5b3cf71
VD
2175static void memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
2176 struct kmem_cache *cachep)
0e9d92f2
GC
2177{
2178 /*
2179 * We need to stop accounting when we kmalloc, because if the
2180 * corresponding kmalloc cache is not yet created, the first allocation
d5b3cf71 2181 * in __memcg_schedule_kmem_cache_create will recurse.
0e9d92f2
GC
2182 *
2183 * However, it is better to enclose the whole function. Depending on
2184 * the debugging options enabled, INIT_WORK(), for instance, can
2185 * trigger an allocation. This too, will make us recurse. Because at
2186 * this point we can't allow ourselves back into memcg_kmem_get_cache,
2187 * the safest choice is to do it like this, wrapping the whole function.
2188 */
6f185c29 2189 current->memcg_kmem_skip_account = 1;
d5b3cf71 2190 __memcg_schedule_kmem_cache_create(memcg, cachep);
6f185c29 2191 current->memcg_kmem_skip_account = 0;
0e9d92f2 2192}
c67a8a68 2193
45264778
VD
2194static inline bool memcg_kmem_bypass(void)
2195{
2196 if (in_interrupt() || !current->mm || (current->flags & PF_KTHREAD))
2197 return true;
2198 return false;
2199}
2200
2201/**
2202 * memcg_kmem_get_cache: select the correct per-memcg cache for allocation
2203 * @cachep: the original global kmem cache
2204 *
d7f25f8a
GC
2205 * Return the kmem_cache we're supposed to use for a slab allocation.
2206 * We try to use the current memcg's version of the cache.
2207 *
45264778
VD
2208 * If the cache does not exist yet, if we are the first user of it, we
2209 * create it asynchronously in a workqueue and let the current allocation
2210 * go through with the original cache.
d7f25f8a 2211 *
45264778
VD
2212 * This function takes a reference to the cache it returns to assure it
2213 * won't get destroyed while we are working with it. Once the caller is
2214 * done with it, memcg_kmem_put_cache() must be called to release the
2215 * reference.
d7f25f8a 2216 */
45264778 2217struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep)
d7f25f8a
GC
2218{
2219 struct mem_cgroup *memcg;
959c8963 2220 struct kmem_cache *memcg_cachep;
2a4db7eb 2221 int kmemcg_id;
d7f25f8a 2222
f7ce3190 2223 VM_BUG_ON(!is_root_cache(cachep));
d7f25f8a 2224
45264778 2225 if (memcg_kmem_bypass())
230e9fc2
VD
2226 return cachep;
2227
9d100c5e 2228 if (current->memcg_kmem_skip_account)
0e9d92f2
GC
2229 return cachep;
2230
8135be5a 2231 memcg = get_mem_cgroup_from_mm(current->mm);
4db0c3c2 2232 kmemcg_id = READ_ONCE(memcg->kmemcg_id);
2a4db7eb 2233 if (kmemcg_id < 0)
ca0dde97 2234 goto out;
d7f25f8a 2235
2a4db7eb 2236 memcg_cachep = cache_from_memcg_idx(cachep, kmemcg_id);
8135be5a
VD
2237 if (likely(memcg_cachep))
2238 return memcg_cachep;
ca0dde97
LZ
2239
2240 /*
2241 * If we are in a safe context (can wait, and not in interrupt
2242 * context), we could be be predictable and return right away.
2243 * This would guarantee that the allocation being performed
2244 * already belongs in the new cache.
2245 *
2246 * However, there are some clashes that can arrive from locking.
2247 * For instance, because we acquire the slab_mutex while doing
776ed0f0
VD
2248 * memcg_create_kmem_cache, this means no further allocation
2249 * could happen with the slab_mutex held. So it's better to
2250 * defer everything.
ca0dde97 2251 */
d5b3cf71 2252 memcg_schedule_kmem_cache_create(memcg, cachep);
ca0dde97 2253out:
8135be5a 2254 css_put(&memcg->css);
ca0dde97 2255 return cachep;
d7f25f8a 2256}
d7f25f8a 2257
45264778
VD
2258/**
2259 * memcg_kmem_put_cache: drop reference taken by memcg_kmem_get_cache
2260 * @cachep: the cache returned by memcg_kmem_get_cache
2261 */
2262void memcg_kmem_put_cache(struct kmem_cache *cachep)
8135be5a
VD
2263{
2264 if (!is_root_cache(cachep))
f7ce3190 2265 css_put(&cachep->memcg_params.memcg->css);
8135be5a
VD
2266}
2267
45264778
VD
2268/**
2269 * memcg_kmem_charge: charge a kmem page
2270 * @page: page to charge
2271 * @gfp: reclaim mode
2272 * @order: allocation order
2273 * @memcg: memory cgroup to charge
2274 *
2275 * Returns 0 on success, an error code on failure.
2276 */
2277int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
2278 struct mem_cgroup *memcg)
7ae1e1d0 2279{
f3ccb2c4
VD
2280 unsigned int nr_pages = 1 << order;
2281 struct page_counter *counter;
7ae1e1d0
GC
2282 int ret;
2283
f3ccb2c4 2284 ret = try_charge(memcg, gfp, nr_pages);
52c29b04 2285 if (ret)
f3ccb2c4 2286 return ret;
52c29b04
JW
2287
2288 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
2289 !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
2290 cancel_charge(memcg, nr_pages);
2291 return -ENOMEM;
7ae1e1d0
GC
2292 }
2293
f3ccb2c4 2294 page->mem_cgroup = memcg;
7ae1e1d0 2295
f3ccb2c4 2296 return 0;
7ae1e1d0
GC
2297}
2298
45264778
VD
2299/**
2300 * memcg_kmem_charge: charge a kmem page to the current memory cgroup
2301 * @page: page to charge
2302 * @gfp: reclaim mode
2303 * @order: allocation order
2304 *
2305 * Returns 0 on success, an error code on failure.
2306 */
2307int memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
7ae1e1d0 2308{
f3ccb2c4 2309 struct mem_cgroup *memcg;
fcff7d7e 2310 int ret = 0;
7ae1e1d0 2311
45264778
VD
2312 if (memcg_kmem_bypass())
2313 return 0;
2314
f3ccb2c4 2315 memcg = get_mem_cgroup_from_mm(current->mm);
c4159a75 2316 if (!mem_cgroup_is_root(memcg)) {
45264778 2317 ret = memcg_kmem_charge_memcg(page, gfp, order, memcg);
c4159a75
VD
2318 if (!ret)
2319 __SetPageKmemcg(page);
2320 }
7ae1e1d0 2321 css_put(&memcg->css);
d05e83a6 2322 return ret;
7ae1e1d0 2323}
45264778
VD
2324/**
2325 * memcg_kmem_uncharge: uncharge a kmem page
2326 * @page: page to uncharge
2327 * @order: allocation order
2328 */
2329void memcg_kmem_uncharge(struct page *page, int order)
7ae1e1d0 2330{
1306a85a 2331 struct mem_cgroup *memcg = page->mem_cgroup;
f3ccb2c4 2332 unsigned int nr_pages = 1 << order;
7ae1e1d0 2333
7ae1e1d0
GC
2334 if (!memcg)
2335 return;
2336
309381fe 2337 VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
29833315 2338
52c29b04
JW
2339 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
2340 page_counter_uncharge(&memcg->kmem, nr_pages);
2341
f3ccb2c4 2342 page_counter_uncharge(&memcg->memory, nr_pages);
7941d214 2343 if (do_memsw_account())
f3ccb2c4 2344 page_counter_uncharge(&memcg->memsw, nr_pages);
60d3fd32 2345
1306a85a 2346 page->mem_cgroup = NULL;
c4159a75
VD
2347
2348 /* slab pages do not have PageKmemcg flag set */
2349 if (PageKmemcg(page))
2350 __ClearPageKmemcg(page);
2351
f3ccb2c4 2352 css_put_many(&memcg->css, nr_pages);
60d3fd32 2353}
127424c8 2354#endif /* !CONFIG_SLOB */
7ae1e1d0 2355
ca3e0214
KH
2356#ifdef CONFIG_TRANSPARENT_HUGEPAGE
2357
ca3e0214
KH
2358/*
2359 * Because tail pages are not marked as "used", set it. We're under
a52633d8 2360 * zone_lru_lock and migration entries setup in all page mappings.
ca3e0214 2361 */
e94c8a9c 2362void mem_cgroup_split_huge_fixup(struct page *head)
ca3e0214 2363{
e94c8a9c 2364 int i;
ca3e0214 2365
3d37c4a9
KH
2366 if (mem_cgroup_disabled())
2367 return;
b070e65c 2368
29833315 2369 for (i = 1; i < HPAGE_PMD_NR; i++)
1306a85a 2370 head[i].mem_cgroup = head->mem_cgroup;
b9982f8d 2371
71cd3113 2372 __this_cpu_sub(head->mem_cgroup->stat->count[MEMCG_RSS_HUGE],
b070e65c 2373 HPAGE_PMD_NR);
ca3e0214 2374}
12d27107 2375#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
ca3e0214 2376
c255a458 2377#ifdef CONFIG_MEMCG_SWAP
0a31bc97
JW
2378static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,
2379 bool charge)
d13d1443 2380{
0a31bc97 2381 int val = (charge) ? 1 : -1;
71cd3113 2382 this_cpu_add(memcg->stat->count[MEMCG_SWAP], val);
d13d1443 2383}
02491447
DN
2384
2385/**
2386 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
2387 * @entry: swap entry to be moved
2388 * @from: mem_cgroup which the entry is moved from
2389 * @to: mem_cgroup which the entry is moved to
2390 *
2391 * It succeeds only when the swap_cgroup's record for this entry is the same
2392 * as the mem_cgroup's id of @from.
2393 *
2394 * Returns 0 on success, -EINVAL on failure.
2395 *
3e32cb2e 2396 * The caller must have charged to @to, IOW, called page_counter_charge() about
02491447
DN
2397 * both res and memsw, and called css_get().
2398 */
2399static int mem_cgroup_move_swap_account(swp_entry_t entry,
e91cbb42 2400 struct mem_cgroup *from, struct mem_cgroup *to)
02491447
DN
2401{
2402 unsigned short old_id, new_id;
2403
34c00c31
LZ
2404 old_id = mem_cgroup_id(from);
2405 new_id = mem_cgroup_id(to);
02491447
DN
2406
2407 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
02491447 2408 mem_cgroup_swap_statistics(from, false);
483c30b5 2409 mem_cgroup_swap_statistics(to, true);
02491447
DN
2410 return 0;
2411 }
2412 return -EINVAL;
2413}
2414#else
2415static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
e91cbb42 2416 struct mem_cgroup *from, struct mem_cgroup *to)
02491447
DN
2417{
2418 return -EINVAL;
2419}
8c7c6e34 2420#endif
d13d1443 2421
3e32cb2e 2422static DEFINE_MUTEX(memcg_limit_mutex);
f212ad7c 2423
d38d2a75 2424static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
3e32cb2e 2425 unsigned long limit)
628f4235 2426{
3e32cb2e
JW
2427 unsigned long curusage;
2428 unsigned long oldusage;
2429 bool enlarge = false;
81d39c20 2430 int retry_count;
3e32cb2e 2431 int ret;
81d39c20
KH
2432
2433 /*
2434 * For keeping hierarchical_reclaim simple, how long we should retry
2435 * is depends on callers. We set our retry-count to be function
2436 * of # of children which we should visit in this loop.
2437 */
3e32cb2e
JW
2438 retry_count = MEM_CGROUP_RECLAIM_RETRIES *
2439 mem_cgroup_count_children(memcg);
81d39c20 2440
3e32cb2e 2441 oldusage = page_counter_read(&memcg->memory);
628f4235 2442
3e32cb2e 2443 do {
628f4235
KH
2444 if (signal_pending(current)) {
2445 ret = -EINTR;
2446 break;
2447 }
3e32cb2e
JW
2448
2449 mutex_lock(&memcg_limit_mutex);
2450 if (limit > memcg->memsw.limit) {
2451 mutex_unlock(&memcg_limit_mutex);
8c7c6e34 2452 ret = -EINVAL;
628f4235
KH
2453 break;
2454 }
3e32cb2e
JW
2455 if (limit > memcg->memory.limit)
2456 enlarge = true;
2457 ret = page_counter_limit(&memcg->memory, limit);
2458 mutex_unlock(&memcg_limit_mutex);
8c7c6e34
KH
2459
2460 if (!ret)
2461 break;
2462
b70a2a21
JW
2463 try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, true);
2464
3e32cb2e 2465 curusage = page_counter_read(&memcg->memory);
81d39c20 2466 /* Usage is reduced ? */
f894ffa8 2467 if (curusage >= oldusage)
81d39c20
KH
2468 retry_count--;
2469 else
2470 oldusage = curusage;
3e32cb2e
JW
2471 } while (retry_count);
2472
3c11ecf4
KH
2473 if (!ret && enlarge)
2474 memcg_oom_recover(memcg);
14797e23 2475
8c7c6e34
KH
2476 return ret;
2477}
2478
338c8431 2479static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
3e32cb2e 2480 unsigned long limit)
8c7c6e34 2481{
3e32cb2e
JW
2482 unsigned long curusage;
2483 unsigned long oldusage;
2484 bool enlarge = false;
81d39c20 2485 int retry_count;
3e32cb2e 2486 int ret;
8c7c6e34 2487
81d39c20 2488 /* see mem_cgroup_resize_res_limit */
3e32cb2e
JW
2489 retry_count = MEM_CGROUP_RECLAIM_RETRIES *
2490 mem_cgroup_count_children(memcg);
2491
2492 oldusage = page_counter_read(&memcg->memsw);
2493
2494 do {
8c7c6e34
KH
2495 if (signal_pending(current)) {
2496 ret = -EINTR;
2497 break;
2498 }
3e32cb2e
JW
2499
2500 mutex_lock(&memcg_limit_mutex);
2501 if (limit < memcg->memory.limit) {
2502 mutex_unlock(&memcg_limit_mutex);
8c7c6e34 2503 ret = -EINVAL;
8c7c6e34
KH
2504 break;
2505 }
3e32cb2e
JW
2506 if (limit > memcg->memsw.limit)
2507 enlarge = true;
2508 ret = page_counter_limit(&memcg->memsw, limit);
2509 mutex_unlock(&memcg_limit_mutex);
8c7c6e34
KH
2510
2511 if (!ret)
2512 break;
2513
b70a2a21
JW
2514 try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, false);
2515
3e32cb2e 2516 curusage = page_counter_read(&memcg->memsw);
81d39c20 2517 /* Usage is reduced ? */
8c7c6e34 2518 if (curusage >= oldusage)
628f4235 2519 retry_count--;
81d39c20
KH
2520 else
2521 oldusage = curusage;
3e32cb2e
JW
2522 } while (retry_count);
2523
3c11ecf4
KH
2524 if (!ret && enlarge)
2525 memcg_oom_recover(memcg);
3e32cb2e 2526
628f4235
KH
2527 return ret;
2528}
2529
ef8f2327 2530unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
0608f43d
AM
2531 gfp_t gfp_mask,
2532 unsigned long *total_scanned)
2533{
2534 unsigned long nr_reclaimed = 0;
ef8f2327 2535 struct mem_cgroup_per_node *mz, *next_mz = NULL;
0608f43d
AM
2536 unsigned long reclaimed;
2537 int loop = 0;
ef8f2327 2538 struct mem_cgroup_tree_per_node *mctz;
3e32cb2e 2539 unsigned long excess;
0608f43d
AM
2540 unsigned long nr_scanned;
2541
2542 if (order > 0)
2543 return 0;
2544
ef8f2327 2545 mctz = soft_limit_tree_node(pgdat->node_id);
d6507ff5
MH
2546
2547 /*
2548 * Do not even bother to check the largest node if the root
2549 * is empty. Do it lockless to prevent lock bouncing. Races
2550 * are acceptable as soft limit is best effort anyway.
2551 */
bfc7228b 2552 if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root))
d6507ff5
MH
2553 return 0;
2554
0608f43d
AM
2555 /*
2556 * This loop can run a while, specially if mem_cgroup's continuously
2557 * keep exceeding their soft limit and putting the system under
2558 * pressure
2559 */
2560 do {
2561 if (next_mz)
2562 mz = next_mz;
2563 else
2564 mz = mem_cgroup_largest_soft_limit_node(mctz);
2565 if (!mz)
2566 break;
2567
2568 nr_scanned = 0;
ef8f2327 2569 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
0608f43d
AM
2570 gfp_mask, &nr_scanned);
2571 nr_reclaimed += reclaimed;
2572 *total_scanned += nr_scanned;
0a31bc97 2573 spin_lock_irq(&mctz->lock);
bc2f2e7f 2574 __mem_cgroup_remove_exceeded(mz, mctz);
0608f43d
AM
2575
2576 /*
2577 * If we failed to reclaim anything from this memory cgroup
2578 * it is time to move on to the next cgroup
2579 */
2580 next_mz = NULL;
bc2f2e7f
VD
2581 if (!reclaimed)
2582 next_mz = __mem_cgroup_largest_soft_limit_node(mctz);
2583
3e32cb2e 2584 excess = soft_limit_excess(mz->memcg);
0608f43d
AM
2585 /*
2586 * One school of thought says that we should not add
2587 * back the node to the tree if reclaim returns 0.
2588 * But our reclaim could return 0, simply because due
2589 * to priority we are exposing a smaller subset of
2590 * memory to reclaim from. Consider this as a longer
2591 * term TODO.
2592 */
2593 /* If excess == 0, no tree ops */
cf2c8127 2594 __mem_cgroup_insert_exceeded(mz, mctz, excess);
0a31bc97 2595 spin_unlock_irq(&mctz->lock);
0608f43d
AM
2596 css_put(&mz->memcg->css);
2597 loop++;
2598 /*
2599 * Could not reclaim anything and there are no more
2600 * mem cgroups to try or we seem to be looping without
2601 * reclaiming anything.
2602 */
2603 if (!nr_reclaimed &&
2604 (next_mz == NULL ||
2605 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
2606 break;
2607 } while (!nr_reclaimed);
2608 if (next_mz)
2609 css_put(&next_mz->memcg->css);
2610 return nr_reclaimed;
2611}
2612
ea280e7b
TH
2613/*
2614 * Test whether @memcg has children, dead or alive. Note that this
2615 * function doesn't care whether @memcg has use_hierarchy enabled and
2616 * returns %true if there are child csses according to the cgroup
2617 * hierarchy. Testing use_hierarchy is the caller's responsiblity.
2618 */
b5f99b53
GC
2619static inline bool memcg_has_children(struct mem_cgroup *memcg)
2620{
ea280e7b
TH
2621 bool ret;
2622
ea280e7b
TH
2623 rcu_read_lock();
2624 ret = css_next_child(NULL, &memcg->css);
2625 rcu_read_unlock();
2626 return ret;
b5f99b53
GC
2627}
2628
c26251f9 2629/*
51038171 2630 * Reclaims as many pages from the given memcg as possible.
c26251f9
MH
2631 *
2632 * Caller is responsible for holding css reference for memcg.
2633 */
2634static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
2635{
2636 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
c26251f9 2637
c1e862c1
KH
2638 /* we call try-to-free pages for make this cgroup empty */
2639 lru_add_drain_all();
f817ed48 2640 /* try to free all pages in this cgroup */
3e32cb2e 2641 while (nr_retries && page_counter_read(&memcg->memory)) {
f817ed48 2642 int progress;
c1e862c1 2643
c26251f9
MH
2644 if (signal_pending(current))
2645 return -EINTR;
2646
b70a2a21
JW
2647 progress = try_to_free_mem_cgroup_pages(memcg, 1,
2648 GFP_KERNEL, true);
c1e862c1 2649 if (!progress) {
f817ed48 2650 nr_retries--;
c1e862c1 2651 /* maybe some writeback is necessary */
8aa7e847 2652 congestion_wait(BLK_RW_ASYNC, HZ/10);
c1e862c1 2653 }
f817ed48
KH
2654
2655 }
ab5196c2
MH
2656
2657 return 0;
cc847582
KH
2658}
2659
6770c64e
TH
2660static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
2661 char *buf, size_t nbytes,
2662 loff_t off)
c1e862c1 2663{
6770c64e 2664 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
c26251f9 2665
d8423011
MH
2666 if (mem_cgroup_is_root(memcg))
2667 return -EINVAL;
6770c64e 2668 return mem_cgroup_force_empty(memcg) ?: nbytes;
c1e862c1
KH
2669}
2670
182446d0
TH
2671static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
2672 struct cftype *cft)
18f59ea7 2673{
182446d0 2674 return mem_cgroup_from_css(css)->use_hierarchy;
18f59ea7
BS
2675}
2676
182446d0
TH
2677static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
2678 struct cftype *cft, u64 val)
18f59ea7
BS
2679{
2680 int retval = 0;
182446d0 2681 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5c9d535b 2682 struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent);
18f59ea7 2683
567fb435 2684 if (memcg->use_hierarchy == val)
0b8f73e1 2685 return 0;
567fb435 2686
18f59ea7 2687 /*
af901ca1 2688 * If parent's use_hierarchy is set, we can't make any modifications
18f59ea7
BS
2689 * in the child subtrees. If it is unset, then the change can
2690 * occur, provided the current cgroup has no children.
2691 *
2692 * For the root cgroup, parent_mem is NULL, we allow value to be
2693 * set if there are no children.
2694 */
c0ff4b85 2695 if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
18f59ea7 2696 (val == 1 || val == 0)) {
ea280e7b 2697 if (!memcg_has_children(memcg))
c0ff4b85 2698 memcg->use_hierarchy = val;
18f59ea7
BS
2699 else
2700 retval = -EBUSY;
2701 } else
2702 retval = -EINVAL;
567fb435 2703
18f59ea7
BS
2704 return retval;
2705}
2706
72b54e73 2707static void tree_stat(struct mem_cgroup *memcg, unsigned long *stat)
ce00a967
JW
2708{
2709 struct mem_cgroup *iter;
72b54e73 2710 int i;
ce00a967 2711
72b54e73 2712 memset(stat, 0, sizeof(*stat) * MEMCG_NR_STAT);
ce00a967 2713
72b54e73
VD
2714 for_each_mem_cgroup_tree(iter, memcg) {
2715 for (i = 0; i < MEMCG_NR_STAT; i++)
ccda7f43 2716 stat[i] += memcg_page_state(iter, i);
72b54e73 2717 }
ce00a967
JW
2718}
2719
72b54e73 2720static void tree_events(struct mem_cgroup *memcg, unsigned long *events)
587d9f72
JW
2721{
2722 struct mem_cgroup *iter;
72b54e73 2723 int i;
587d9f72 2724
72b54e73 2725 memset(events, 0, sizeof(*events) * MEMCG_NR_EVENTS);
587d9f72 2726
72b54e73
VD
2727 for_each_mem_cgroup_tree(iter, memcg) {
2728 for (i = 0; i < MEMCG_NR_EVENTS; i++)
ccda7f43 2729 events[i] += memcg_sum_events(iter, i);
72b54e73 2730 }
587d9f72
JW
2731}
2732
6f646156 2733static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
ce00a967 2734{
72b54e73 2735 unsigned long val = 0;
ce00a967 2736
3e32cb2e 2737 if (mem_cgroup_is_root(memcg)) {
72b54e73
VD
2738 struct mem_cgroup *iter;
2739
2740 for_each_mem_cgroup_tree(iter, memcg) {
ccda7f43
JW
2741 val += memcg_page_state(iter, MEMCG_CACHE);
2742 val += memcg_page_state(iter, MEMCG_RSS);
72b54e73 2743 if (swap)
ccda7f43 2744 val += memcg_page_state(iter, MEMCG_SWAP);
72b54e73 2745 }
3e32cb2e 2746 } else {
ce00a967 2747 if (!swap)
3e32cb2e 2748 val = page_counter_read(&memcg->memory);
ce00a967 2749 else
3e32cb2e 2750 val = page_counter_read(&memcg->memsw);
ce00a967 2751 }
c12176d3 2752 return val;
ce00a967
JW
2753}
2754
3e32cb2e
JW
2755enum {
2756 RES_USAGE,
2757 RES_LIMIT,
2758 RES_MAX_USAGE,
2759 RES_FAILCNT,
2760 RES_SOFT_LIMIT,
2761};
ce00a967 2762
791badbd 2763static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
05b84301 2764 struct cftype *cft)
8cdea7c0 2765{
182446d0 2766 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3e32cb2e 2767 struct page_counter *counter;
af36f906 2768
3e32cb2e 2769 switch (MEMFILE_TYPE(cft->private)) {
8c7c6e34 2770 case _MEM:
3e32cb2e
JW
2771 counter = &memcg->memory;
2772 break;
8c7c6e34 2773 case _MEMSWAP:
3e32cb2e
JW
2774 counter = &memcg->memsw;
2775 break;
510fc4e1 2776 case _KMEM:
3e32cb2e 2777 counter = &memcg->kmem;
510fc4e1 2778 break;
d55f90bf 2779 case _TCP:
0db15298 2780 counter = &memcg->tcpmem;
d55f90bf 2781 break;
8c7c6e34
KH
2782 default:
2783 BUG();
8c7c6e34 2784 }
3e32cb2e
JW
2785
2786 switch (MEMFILE_ATTR(cft->private)) {
2787 case RES_USAGE:
2788 if (counter == &memcg->memory)
c12176d3 2789 return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
3e32cb2e 2790 if (counter == &memcg->memsw)
c12176d3 2791 return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
3e32cb2e
JW
2792 return (u64)page_counter_read(counter) * PAGE_SIZE;
2793 case RES_LIMIT:
2794 return (u64)counter->limit * PAGE_SIZE;
2795 case RES_MAX_USAGE:
2796 return (u64)counter->watermark * PAGE_SIZE;
2797 case RES_FAILCNT:
2798 return counter->failcnt;
2799 case RES_SOFT_LIMIT:
2800 return (u64)memcg->soft_limit * PAGE_SIZE;
2801 default:
2802 BUG();
2803 }
8cdea7c0 2804}
510fc4e1 2805
127424c8 2806#ifndef CONFIG_SLOB
567e9ab2 2807static int memcg_online_kmem(struct mem_cgroup *memcg)
d6441637 2808{
d6441637
VD
2809 int memcg_id;
2810
b313aeee
VD
2811 if (cgroup_memory_nokmem)
2812 return 0;
2813
2a4db7eb 2814 BUG_ON(memcg->kmemcg_id >= 0);
567e9ab2 2815 BUG_ON(memcg->kmem_state);
d6441637 2816
f3bb3043 2817 memcg_id = memcg_alloc_cache_id();
0b8f73e1
JW
2818 if (memcg_id < 0)
2819 return memcg_id;
d6441637 2820
ef12947c 2821 static_branch_inc(&memcg_kmem_enabled_key);
d6441637 2822 /*
567e9ab2 2823 * A memory cgroup is considered kmem-online as soon as it gets
900a38f0 2824 * kmemcg_id. Setting the id after enabling static branching will
d6441637
VD
2825 * guarantee no one starts accounting before all call sites are
2826 * patched.
2827 */
900a38f0 2828 memcg->kmemcg_id = memcg_id;
567e9ab2 2829 memcg->kmem_state = KMEM_ONLINE;
bc2791f8 2830 INIT_LIST_HEAD(&memcg->kmem_caches);
0b8f73e1
JW
2831
2832 return 0;
d6441637
VD
2833}
2834
8e0a8912
JW
2835static void memcg_offline_kmem(struct mem_cgroup *memcg)
2836{
2837 struct cgroup_subsys_state *css;
2838 struct mem_cgroup *parent, *child;
2839 int kmemcg_id;
2840
2841 if (memcg->kmem_state != KMEM_ONLINE)
2842 return;
2843 /*
2844 * Clear the online state before clearing memcg_caches array
2845 * entries. The slab_mutex in memcg_deactivate_kmem_caches()
2846 * guarantees that no cache will be created for this cgroup
2847 * after we are done (see memcg_create_kmem_cache()).
2848 */
2849 memcg->kmem_state = KMEM_ALLOCATED;
2850
2851 memcg_deactivate_kmem_caches(memcg);
2852
2853 kmemcg_id = memcg->kmemcg_id;
2854 BUG_ON(kmemcg_id < 0);
2855
2856 parent = parent_mem_cgroup(memcg);
2857 if (!parent)
2858 parent = root_mem_cgroup;
2859
2860 /*
2861 * Change kmemcg_id of this cgroup and all its descendants to the
2862 * parent's id, and then move all entries from this cgroup's list_lrus
2863 * to ones of the parent. After we have finished, all list_lrus
2864 * corresponding to this cgroup are guaranteed to remain empty. The
2865 * ordering is imposed by list_lru_node->lock taken by
2866 * memcg_drain_all_list_lrus().
2867 */
3a06bb78 2868 rcu_read_lock(); /* can be called from css_free w/o cgroup_mutex */
8e0a8912
JW
2869 css_for_each_descendant_pre(css, &memcg->css) {
2870 child = mem_cgroup_from_css(css);
2871 BUG_ON(child->kmemcg_id != kmemcg_id);
2872 child->kmemcg_id = parent->kmemcg_id;
2873 if (!memcg->use_hierarchy)
2874 break;
2875 }
3a06bb78
TH
2876 rcu_read_unlock();
2877
8e0a8912
JW
2878 memcg_drain_all_list_lrus(kmemcg_id, parent->kmemcg_id);
2879
2880 memcg_free_cache_id(kmemcg_id);
2881}
2882
2883static void memcg_free_kmem(struct mem_cgroup *memcg)
2884{
0b8f73e1
JW
2885 /* css_alloc() failed, offlining didn't happen */
2886 if (unlikely(memcg->kmem_state == KMEM_ONLINE))
2887 memcg_offline_kmem(memcg);
2888
8e0a8912
JW
2889 if (memcg->kmem_state == KMEM_ALLOCATED) {
2890 memcg_destroy_kmem_caches(memcg);
2891 static_branch_dec(&memcg_kmem_enabled_key);
2892 WARN_ON(page_counter_read(&memcg->kmem));
2893 }
8e0a8912 2894}
d6441637 2895#else
0b8f73e1 2896static int memcg_online_kmem(struct mem_cgroup *memcg)
127424c8
JW
2897{
2898 return 0;
2899}
2900static void memcg_offline_kmem(struct mem_cgroup *memcg)
2901{
2902}
2903static void memcg_free_kmem(struct mem_cgroup *memcg)
2904{
2905}
2906#endif /* !CONFIG_SLOB */
2907
d6441637 2908static int memcg_update_kmem_limit(struct mem_cgroup *memcg,
3e32cb2e 2909 unsigned long limit)
d6441637 2910{
b313aeee 2911 int ret;
127424c8
JW
2912
2913 mutex_lock(&memcg_limit_mutex);
127424c8 2914 ret = page_counter_limit(&memcg->kmem, limit);
127424c8
JW
2915 mutex_unlock(&memcg_limit_mutex);
2916 return ret;
d6441637 2917}
510fc4e1 2918
d55f90bf
VD
2919static int memcg_update_tcp_limit(struct mem_cgroup *memcg, unsigned long limit)
2920{
2921 int ret;
2922
2923 mutex_lock(&memcg_limit_mutex);
2924
0db15298 2925 ret = page_counter_limit(&memcg->tcpmem, limit);
d55f90bf
VD
2926 if (ret)
2927 goto out;
2928
0db15298 2929 if (!memcg->tcpmem_active) {
d55f90bf
VD
2930 /*
2931 * The active flag needs to be written after the static_key
2932 * update. This is what guarantees that the socket activation
2d758073
JW
2933 * function is the last one to run. See mem_cgroup_sk_alloc()
2934 * for details, and note that we don't mark any socket as
2935 * belonging to this memcg until that flag is up.
d55f90bf
VD
2936 *
2937 * We need to do this, because static_keys will span multiple
2938 * sites, but we can't control their order. If we mark a socket
2939 * as accounted, but the accounting functions are not patched in
2940 * yet, we'll lose accounting.
2941 *
2d758073 2942 * We never race with the readers in mem_cgroup_sk_alloc(),
d55f90bf
VD
2943 * because when this value change, the code to process it is not
2944 * patched in yet.
2945 */
2946 static_branch_inc(&memcg_sockets_enabled_key);
0db15298 2947 memcg->tcpmem_active = true;
d55f90bf
VD
2948 }
2949out:
2950 mutex_unlock(&memcg_limit_mutex);
2951 return ret;
2952}
d55f90bf 2953
628f4235
KH
2954/*
2955 * The user of this function is...
2956 * RES_LIMIT.
2957 */
451af504
TH
2958static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
2959 char *buf, size_t nbytes, loff_t off)
8cdea7c0 2960{
451af504 2961 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3e32cb2e 2962 unsigned long nr_pages;
628f4235
KH
2963 int ret;
2964
451af504 2965 buf = strstrip(buf);
650c5e56 2966 ret = page_counter_memparse(buf, "-1", &nr_pages);
3e32cb2e
JW
2967 if (ret)
2968 return ret;
af36f906 2969
3e32cb2e 2970 switch (MEMFILE_ATTR(of_cft(of)->private)) {
628f4235 2971 case RES_LIMIT:
4b3bde4c
BS
2972 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
2973 ret = -EINVAL;
2974 break;
2975 }
3e32cb2e
JW
2976 switch (MEMFILE_TYPE(of_cft(of)->private)) {
2977 case _MEM:
2978 ret = mem_cgroup_resize_limit(memcg, nr_pages);
8c7c6e34 2979 break;
3e32cb2e
JW
2980 case _MEMSWAP:
2981 ret = mem_cgroup_resize_memsw_limit(memcg, nr_pages);
296c81d8 2982 break;
3e32cb2e
JW
2983 case _KMEM:
2984 ret = memcg_update_kmem_limit(memcg, nr_pages);
2985 break;
d55f90bf
VD
2986 case _TCP:
2987 ret = memcg_update_tcp_limit(memcg, nr_pages);
2988 break;
3e32cb2e 2989 }
296c81d8 2990 break;
3e32cb2e
JW
2991 case RES_SOFT_LIMIT:
2992 memcg->soft_limit = nr_pages;
2993 ret = 0;
628f4235
KH
2994 break;
2995 }
451af504 2996 return ret ?: nbytes;
8cdea7c0
BS
2997}
2998
6770c64e
TH
2999static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
3000 size_t nbytes, loff_t off)
c84872e1 3001{
6770c64e 3002 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3e32cb2e 3003 struct page_counter *counter;
c84872e1 3004
3e32cb2e
JW
3005 switch (MEMFILE_TYPE(of_cft(of)->private)) {
3006 case _MEM:
3007 counter = &memcg->memory;
3008 break;
3009 case _MEMSWAP:
3010 counter = &memcg->memsw;
3011 break;
3012 case _KMEM:
3013 counter = &memcg->kmem;
3014 break;
d55f90bf 3015 case _TCP:
0db15298 3016 counter = &memcg->tcpmem;
d55f90bf 3017 break;
3e32cb2e
JW
3018 default:
3019 BUG();
3020 }
af36f906 3021
3e32cb2e 3022 switch (MEMFILE_ATTR(of_cft(of)->private)) {
29f2a4da 3023 case RES_MAX_USAGE:
3e32cb2e 3024 page_counter_reset_watermark(counter);
29f2a4da
PE
3025 break;
3026 case RES_FAILCNT:
3e32cb2e 3027 counter->failcnt = 0;
29f2a4da 3028 break;
3e32cb2e
JW
3029 default:
3030 BUG();
29f2a4da 3031 }
f64c3f54 3032
6770c64e 3033 return nbytes;
c84872e1
PE
3034}
3035
182446d0 3036static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
7dc74be0
DN
3037 struct cftype *cft)
3038{
182446d0 3039 return mem_cgroup_from_css(css)->move_charge_at_immigrate;
7dc74be0
DN
3040}
3041
02491447 3042#ifdef CONFIG_MMU
182446d0 3043static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
7dc74be0
DN
3044 struct cftype *cft, u64 val)
3045{
182446d0 3046 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
7dc74be0 3047
1dfab5ab 3048 if (val & ~MOVE_MASK)
7dc74be0 3049 return -EINVAL;
ee5e8472 3050
7dc74be0 3051 /*
ee5e8472
GC
3052 * No kind of locking is needed in here, because ->can_attach() will
3053 * check this value once in the beginning of the process, and then carry
3054 * on with stale data. This means that changes to this value will only
3055 * affect task migrations starting after the change.
7dc74be0 3056 */
c0ff4b85 3057 memcg->move_charge_at_immigrate = val;
7dc74be0
DN
3058 return 0;
3059}
02491447 3060#else
182446d0 3061static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
02491447
DN
3062 struct cftype *cft, u64 val)
3063{
3064 return -ENOSYS;
3065}
3066#endif
7dc74be0 3067
406eb0c9 3068#ifdef CONFIG_NUMA
2da8ca82 3069static int memcg_numa_stat_show(struct seq_file *m, void *v)
406eb0c9 3070{
25485de6
GT
3071 struct numa_stat {
3072 const char *name;
3073 unsigned int lru_mask;
3074 };
3075
3076 static const struct numa_stat stats[] = {
3077 { "total", LRU_ALL },
3078 { "file", LRU_ALL_FILE },
3079 { "anon", LRU_ALL_ANON },
3080 { "unevictable", BIT(LRU_UNEVICTABLE) },
3081 };
3082 const struct numa_stat *stat;
406eb0c9 3083 int nid;
25485de6 3084 unsigned long nr;
2da8ca82 3085 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
406eb0c9 3086
25485de6
GT
3087 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3088 nr = mem_cgroup_nr_lru_pages(memcg, stat->lru_mask);
3089 seq_printf(m, "%s=%lu", stat->name, nr);
3090 for_each_node_state(nid, N_MEMORY) {
3091 nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
3092 stat->lru_mask);
3093 seq_printf(m, " N%d=%lu", nid, nr);
3094 }
3095 seq_putc(m, '\n');
406eb0c9 3096 }
406eb0c9 3097
071aee13
YH
3098 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3099 struct mem_cgroup *iter;
3100
3101 nr = 0;
3102 for_each_mem_cgroup_tree(iter, memcg)
3103 nr += mem_cgroup_nr_lru_pages(iter, stat->lru_mask);
3104 seq_printf(m, "hierarchical_%s=%lu", stat->name, nr);
3105 for_each_node_state(nid, N_MEMORY) {
3106 nr = 0;
3107 for_each_mem_cgroup_tree(iter, memcg)
3108 nr += mem_cgroup_node_nr_lru_pages(
3109 iter, nid, stat->lru_mask);
3110 seq_printf(m, " N%d=%lu", nid, nr);
3111 }
3112 seq_putc(m, '\n');
406eb0c9 3113 }
406eb0c9 3114
406eb0c9
YH
3115 return 0;
3116}
3117#endif /* CONFIG_NUMA */
3118
df0e53d0
JW
3119/* Universal VM events cgroup1 shows, original sort order */
3120unsigned int memcg1_events[] = {
3121 PGPGIN,
3122 PGPGOUT,
3123 PGFAULT,
3124 PGMAJFAULT,
3125};
3126
3127static const char *const memcg1_event_names[] = {
3128 "pgpgin",
3129 "pgpgout",
3130 "pgfault",
3131 "pgmajfault",
3132};
3133
2da8ca82 3134static int memcg_stat_show(struct seq_file *m, void *v)
d2ceb9b7 3135{
2da8ca82 3136 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3e32cb2e 3137 unsigned long memory, memsw;
af7c4b0e
JW
3138 struct mem_cgroup *mi;
3139 unsigned int i;
406eb0c9 3140
71cd3113 3141 BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names) != ARRAY_SIZE(memcg1_stats));
70bc068c
RS
3142 BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);
3143
71cd3113
JW
3144 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
3145 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
1dd3a273 3146 continue;
71cd3113 3147 seq_printf(m, "%s %lu\n", memcg1_stat_names[i],
ccda7f43 3148 memcg_page_state(memcg, memcg1_stats[i]) *
71cd3113 3149 PAGE_SIZE);
1dd3a273 3150 }
7b854121 3151
df0e53d0
JW
3152 for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
3153 seq_printf(m, "%s %lu\n", memcg1_event_names[i],
ccda7f43 3154 memcg_sum_events(memcg, memcg1_events[i]));
af7c4b0e
JW
3155
3156 for (i = 0; i < NR_LRU_LISTS; i++)
3157 seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
3158 mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);
3159
14067bb3 3160 /* Hierarchical information */
3e32cb2e
JW
3161 memory = memsw = PAGE_COUNTER_MAX;
3162 for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
3163 memory = min(memory, mi->memory.limit);
3164 memsw = min(memsw, mi->memsw.limit);
fee7b548 3165 }
3e32cb2e
JW
3166 seq_printf(m, "hierarchical_memory_limit %llu\n",
3167 (u64)memory * PAGE_SIZE);
7941d214 3168 if (do_memsw_account())
3e32cb2e
JW
3169 seq_printf(m, "hierarchical_memsw_limit %llu\n",
3170 (u64)memsw * PAGE_SIZE);
7f016ee8 3171
71cd3113 3172 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
484ebb3b 3173 unsigned long long val = 0;
af7c4b0e 3174
71cd3113 3175 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
1dd3a273 3176 continue;
af7c4b0e 3177 for_each_mem_cgroup_tree(mi, memcg)
ccda7f43 3178 val += memcg_page_state(mi, memcg1_stats[i]) *
71cd3113
JW
3179 PAGE_SIZE;
3180 seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i], val);
af7c4b0e
JW
3181 }
3182
df0e53d0 3183 for (i = 0; i < ARRAY_SIZE(memcg1_events); i++) {
af7c4b0e
JW
3184 unsigned long long val = 0;
3185
3186 for_each_mem_cgroup_tree(mi, memcg)
ccda7f43 3187 val += memcg_sum_events(mi, memcg1_events[i]);
df0e53d0 3188 seq_printf(m, "total_%s %llu\n", memcg1_event_names[i], val);
af7c4b0e
JW
3189 }
3190
3191 for (i = 0; i < NR_LRU_LISTS; i++) {
3192 unsigned long long val = 0;
3193
3194 for_each_mem_cgroup_tree(mi, memcg)
3195 val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE;
3196 seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val);
1dd3a273 3197 }
14067bb3 3198
7f016ee8 3199#ifdef CONFIG_DEBUG_VM
7f016ee8 3200 {
ef8f2327
MG
3201 pg_data_t *pgdat;
3202 struct mem_cgroup_per_node *mz;
89abfab1 3203 struct zone_reclaim_stat *rstat;
7f016ee8
KM
3204 unsigned long recent_rotated[2] = {0, 0};
3205 unsigned long recent_scanned[2] = {0, 0};
3206
ef8f2327
MG
3207 for_each_online_pgdat(pgdat) {
3208 mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
3209 rstat = &mz->lruvec.reclaim_stat;
7f016ee8 3210
ef8f2327
MG
3211 recent_rotated[0] += rstat->recent_rotated[0];
3212 recent_rotated[1] += rstat->recent_rotated[1];
3213 recent_scanned[0] += rstat->recent_scanned[0];
3214 recent_scanned[1] += rstat->recent_scanned[1];
3215 }
78ccf5b5
JW
3216 seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
3217 seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
3218 seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
3219 seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
7f016ee8
KM
3220 }
3221#endif
3222
d2ceb9b7
KH
3223 return 0;
3224}
3225
182446d0
TH
3226static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
3227 struct cftype *cft)
a7885eb8 3228{
182446d0 3229 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
a7885eb8 3230
1f4c025b 3231 return mem_cgroup_swappiness(memcg);
a7885eb8
KM
3232}
3233
182446d0
TH
3234static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
3235 struct cftype *cft, u64 val)
a7885eb8 3236{
182446d0 3237 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
a7885eb8 3238
3dae7fec 3239 if (val > 100)
a7885eb8
KM
3240 return -EINVAL;
3241
14208b0e 3242 if (css->parent)
3dae7fec
JW
3243 memcg->swappiness = val;
3244 else
3245 vm_swappiness = val;
068b38c1 3246
a7885eb8
KM
3247 return 0;
3248}
3249
2e72b634
KS
3250static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
3251{
3252 struct mem_cgroup_threshold_ary *t;
3e32cb2e 3253 unsigned long usage;
2e72b634
KS
3254 int i;
3255
3256 rcu_read_lock();
3257 if (!swap)
2c488db2 3258 t = rcu_dereference(memcg->thresholds.primary);
2e72b634 3259 else
2c488db2 3260 t = rcu_dereference(memcg->memsw_thresholds.primary);
2e72b634
KS
3261
3262 if (!t)
3263 goto unlock;
3264
ce00a967 3265 usage = mem_cgroup_usage(memcg, swap);
2e72b634
KS
3266
3267 /*
748dad36 3268 * current_threshold points to threshold just below or equal to usage.
2e72b634
KS
3269 * If it's not true, a threshold was crossed after last
3270 * call of __mem_cgroup_threshold().
3271 */
5407a562 3272 i = t->current_threshold;
2e72b634
KS
3273
3274 /*
3275 * Iterate backward over array of thresholds starting from
3276 * current_threshold and check if a threshold is crossed.
3277 * If none of thresholds below usage is crossed, we read
3278 * only one element of the array here.
3279 */
3280 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
3281 eventfd_signal(t->entries[i].eventfd, 1);
3282
3283 /* i = current_threshold + 1 */
3284 i++;
3285
3286 /*
3287 * Iterate forward over array of thresholds starting from
3288 * current_threshold+1 and check if a threshold is crossed.
3289 * If none of thresholds above usage is crossed, we read
3290 * only one element of the array here.
3291 */
3292 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
3293 eventfd_signal(t->entries[i].eventfd, 1);
3294
3295 /* Update current_threshold */
5407a562 3296 t->current_threshold = i - 1;
2e72b634
KS
3297unlock:
3298 rcu_read_unlock();
3299}
3300
3301static void mem_cgroup_threshold(struct mem_cgroup *memcg)
3302{
ad4ca5f4
KS
3303 while (memcg) {
3304 __mem_cgroup_threshold(memcg, false);
7941d214 3305 if (do_memsw_account())
ad4ca5f4
KS
3306 __mem_cgroup_threshold(memcg, true);
3307
3308 memcg = parent_mem_cgroup(memcg);
3309 }
2e72b634
KS
3310}
3311
3312static int compare_thresholds(const void *a, const void *b)
3313{
3314 const struct mem_cgroup_threshold *_a = a;
3315 const struct mem_cgroup_threshold *_b = b;
3316
2bff24a3
GT
3317 if (_a->threshold > _b->threshold)
3318 return 1;
3319
3320 if (_a->threshold < _b->threshold)
3321 return -1;
3322
3323 return 0;
2e72b634
KS
3324}
3325
c0ff4b85 3326static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
9490ff27
KH
3327{
3328 struct mem_cgroup_eventfd_list *ev;
3329
2bcf2e92
MH
3330 spin_lock(&memcg_oom_lock);
3331
c0ff4b85 3332 list_for_each_entry(ev, &memcg->oom_notify, list)
9490ff27 3333 eventfd_signal(ev->eventfd, 1);
2bcf2e92
MH
3334
3335 spin_unlock(&memcg_oom_lock);
9490ff27
KH
3336 return 0;
3337}
3338
c0ff4b85 3339static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
9490ff27 3340{
7d74b06f
KH
3341 struct mem_cgroup *iter;
3342
c0ff4b85 3343 for_each_mem_cgroup_tree(iter, memcg)
7d74b06f 3344 mem_cgroup_oom_notify_cb(iter);
9490ff27
KH
3345}
3346
59b6f873 3347static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
347c4a87 3348 struct eventfd_ctx *eventfd, const char *args, enum res_type type)
2e72b634 3349{
2c488db2
KS
3350 struct mem_cgroup_thresholds *thresholds;
3351 struct mem_cgroup_threshold_ary *new;
3e32cb2e
JW
3352 unsigned long threshold;
3353 unsigned long usage;
2c488db2 3354 int i, size, ret;
2e72b634 3355
650c5e56 3356 ret = page_counter_memparse(args, "-1", &threshold);
2e72b634
KS
3357 if (ret)
3358 return ret;
3359
3360 mutex_lock(&memcg->thresholds_lock);
2c488db2 3361
05b84301 3362 if (type == _MEM) {
2c488db2 3363 thresholds = &memcg->thresholds;
ce00a967 3364 usage = mem_cgroup_usage(memcg, false);
05b84301 3365 } else if (type == _MEMSWAP) {
2c488db2 3366 thresholds = &memcg->memsw_thresholds;
ce00a967 3367 usage = mem_cgroup_usage(memcg, true);
05b84301 3368 } else
2e72b634
KS
3369 BUG();
3370
2e72b634 3371 /* Check if a threshold crossed before adding a new one */
2c488db2 3372 if (thresholds->primary)
2e72b634
KS
3373 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3374
2c488db2 3375 size = thresholds->primary ? thresholds->primary->size + 1 : 1;
2e72b634
KS
3376
3377 /* Allocate memory for new array of thresholds */
2c488db2 3378 new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
2e72b634 3379 GFP_KERNEL);
2c488db2 3380 if (!new) {
2e72b634
KS
3381 ret = -ENOMEM;
3382 goto unlock;
3383 }
2c488db2 3384 new->size = size;
2e72b634
KS
3385
3386 /* Copy thresholds (if any) to new array */
2c488db2
KS
3387 if (thresholds->primary) {
3388 memcpy(new->entries, thresholds->primary->entries, (size - 1) *
2e72b634 3389 sizeof(struct mem_cgroup_threshold));
2c488db2
KS
3390 }
3391
2e72b634 3392 /* Add new threshold */
2c488db2
KS
3393 new->entries[size - 1].eventfd = eventfd;
3394 new->entries[size - 1].threshold = threshold;
2e72b634
KS
3395
3396 /* Sort thresholds. Registering of new threshold isn't time-critical */
2c488db2 3397 sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
2e72b634
KS
3398 compare_thresholds, NULL);
3399
3400 /* Find current threshold */
2c488db2 3401 new->current_threshold = -1;
2e72b634 3402 for (i = 0; i < size; i++) {
748dad36 3403 if (new->entries[i].threshold <= usage) {
2e72b634 3404 /*
2c488db2
KS
3405 * new->current_threshold will not be used until
3406 * rcu_assign_pointer(), so it's safe to increment
2e72b634
KS
3407 * it here.
3408 */
2c488db2 3409 ++new->current_threshold;
748dad36
SZ
3410 } else
3411 break;
2e72b634
KS
3412 }
3413
2c488db2
KS
3414 /* Free old spare buffer and save old primary buffer as spare */
3415 kfree(thresholds->spare);
3416 thresholds->spare = thresholds->primary;
3417
3418 rcu_assign_pointer(thresholds->primary, new);
2e72b634 3419
907860ed 3420 /* To be sure that nobody uses thresholds */
2e72b634
KS
3421 synchronize_rcu();
3422
2e72b634
KS
3423unlock:
3424 mutex_unlock(&memcg->thresholds_lock);
3425
3426 return ret;
3427}
3428
59b6f873 3429static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
347c4a87
TH
3430 struct eventfd_ctx *eventfd, const char *args)
3431{
59b6f873 3432 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
347c4a87
TH
3433}
3434
59b6f873 3435static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
347c4a87
TH
3436 struct eventfd_ctx *eventfd, const char *args)
3437{
59b6f873 3438 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
347c4a87
TH
3439}
3440
59b6f873 3441static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
347c4a87 3442 struct eventfd_ctx *eventfd, enum res_type type)
2e72b634 3443{
2c488db2
KS
3444 struct mem_cgroup_thresholds *thresholds;
3445 struct mem_cgroup_threshold_ary *new;
3e32cb2e 3446 unsigned long usage;
2c488db2 3447 int i, j, size;
2e72b634
KS
3448
3449 mutex_lock(&memcg->thresholds_lock);
05b84301
JW
3450
3451 if (type == _MEM) {
2c488db2 3452 thresholds = &memcg->thresholds;
ce00a967 3453 usage = mem_cgroup_usage(memcg, false);
05b84301 3454 } else if (type == _MEMSWAP) {
2c488db2 3455 thresholds = &memcg->memsw_thresholds;
ce00a967 3456 usage = mem_cgroup_usage(memcg, true);
05b84301 3457 } else
2e72b634
KS
3458 BUG();
3459
371528ca
AV
3460 if (!thresholds->primary)
3461 goto unlock;
3462
2e72b634
KS
3463 /* Check if a threshold crossed before removing */
3464 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3465
3466 /* Calculate new number of threshold */
2c488db2
KS
3467 size = 0;
3468 for (i = 0; i < thresholds->primary->size; i++) {
3469 if (thresholds->primary->entries[i].eventfd != eventfd)
2e72b634
KS
3470 size++;
3471 }
3472
2c488db2 3473 new = thresholds->spare;
907860ed 3474
2e72b634
KS
3475 /* Set thresholds array to NULL if we don't have thresholds */
3476 if (!size) {
2c488db2
KS
3477 kfree(new);
3478 new = NULL;
907860ed 3479 goto swap_buffers;
2e72b634
KS
3480 }
3481
2c488db2 3482 new->size = size;
2e72b634
KS
3483
3484 /* Copy thresholds and find current threshold */
2c488db2
KS
3485 new->current_threshold = -1;
3486 for (i = 0, j = 0; i < thresholds->primary->size; i++) {
3487 if (thresholds->primary->entries[i].eventfd == eventfd)
2e72b634
KS
3488 continue;
3489
2c488db2 3490 new->entries[j] = thresholds->primary->entries[i];
748dad36 3491 if (new->entries[j].threshold <= usage) {
2e72b634 3492 /*
2c488db2 3493 * new->current_threshold will not be used
2e72b634
KS
3494 * until rcu_assign_pointer(), so it's safe to increment
3495 * it here.
3496 */
2c488db2 3497 ++new->current_threshold;
2e72b634
KS
3498 }
3499 j++;
3500 }
3501
907860ed 3502swap_buffers:
2c488db2
KS
3503 /* Swap primary and spare array */
3504 thresholds->spare = thresholds->primary;
8c757763 3505
2c488db2 3506 rcu_assign_pointer(thresholds->primary, new);
2e72b634 3507
907860ed 3508 /* To be sure that nobody uses thresholds */
2e72b634 3509 synchronize_rcu();
6611d8d7
MC
3510
3511 /* If all events are unregistered, free the spare array */
3512 if (!new) {
3513 kfree(thresholds->spare);
3514 thresholds->spare = NULL;
3515 }
371528ca 3516unlock:
2e72b634 3517 mutex_unlock(&memcg->thresholds_lock);
2e72b634 3518}
c1e862c1 3519
59b6f873 3520static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
347c4a87
TH
3521 struct eventfd_ctx *eventfd)
3522{
59b6f873 3523 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
347c4a87
TH
3524}
3525
59b6f873 3526static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
347c4a87
TH
3527 struct eventfd_ctx *eventfd)
3528{
59b6f873 3529 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
347c4a87
TH
3530}
3531
59b6f873 3532static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
347c4a87 3533 struct eventfd_ctx *eventfd, const char *args)
9490ff27 3534{
9490ff27 3535 struct mem_cgroup_eventfd_list *event;
9490ff27 3536
9490ff27
KH
3537 event = kmalloc(sizeof(*event), GFP_KERNEL);
3538 if (!event)
3539 return -ENOMEM;
3540
1af8efe9 3541 spin_lock(&memcg_oom_lock);
9490ff27
KH
3542
3543 event->eventfd = eventfd;
3544 list_add(&event->list, &memcg->oom_notify);
3545
3546 /* already in OOM ? */
c2b42d3c 3547 if (memcg->under_oom)
9490ff27 3548 eventfd_signal(eventfd, 1);
1af8efe9 3549 spin_unlock(&memcg_oom_lock);
9490ff27
KH
3550
3551 return 0;
3552}
3553
59b6f873 3554static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
347c4a87 3555 struct eventfd_ctx *eventfd)
9490ff27 3556{
9490ff27 3557 struct mem_cgroup_eventfd_list *ev, *tmp;
9490ff27 3558
1af8efe9 3559 spin_lock(&memcg_oom_lock);
9490ff27 3560
c0ff4b85 3561 list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
9490ff27
KH
3562 if (ev->eventfd == eventfd) {
3563 list_del(&ev->list);
3564 kfree(ev);
3565 }
3566 }
3567
1af8efe9 3568 spin_unlock(&memcg_oom_lock);
9490ff27
KH
3569}
3570
2da8ca82 3571static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v)
3c11ecf4 3572{
2da8ca82 3573 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf));
3c11ecf4 3574
791badbd 3575 seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable);
c2b42d3c 3576 seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
3c11ecf4
KH
3577 return 0;
3578}
3579
182446d0 3580static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
3c11ecf4
KH
3581 struct cftype *cft, u64 val)
3582{
182446d0 3583 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3c11ecf4
KH
3584
3585 /* cannot set to root cgroup and only 0 and 1 are allowed */
14208b0e 3586 if (!css->parent || !((val == 0) || (val == 1)))
3c11ecf4
KH
3587 return -EINVAL;
3588
c0ff4b85 3589 memcg->oom_kill_disable = val;
4d845ebf 3590 if (!val)
c0ff4b85 3591 memcg_oom_recover(memcg);
3dae7fec 3592
3c11ecf4
KH
3593 return 0;
3594}
3595
52ebea74
TH
3596#ifdef CONFIG_CGROUP_WRITEBACK
3597
3598struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg)
3599{
3600 return &memcg->cgwb_list;
3601}
3602
841710aa
TH
3603static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
3604{
3605 return wb_domain_init(&memcg->cgwb_domain, gfp);
3606}
3607
3608static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
3609{
3610 wb_domain_exit(&memcg->cgwb_domain);
3611}
3612
2529bb3a
TH
3613static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
3614{
3615 wb_domain_size_changed(&memcg->cgwb_domain);
3616}
3617
841710aa
TH
3618struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
3619{
3620 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
3621
3622 if (!memcg->css.parent)
3623 return NULL;
3624
3625 return &memcg->cgwb_domain;
3626}
3627
c2aa723a
TH
3628/**
3629 * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
3630 * @wb: bdi_writeback in question
c5edf9cd
TH
3631 * @pfilepages: out parameter for number of file pages
3632 * @pheadroom: out parameter for number of allocatable pages according to memcg
c2aa723a
TH
3633 * @pdirty: out parameter for number of dirty pages
3634 * @pwriteback: out parameter for number of pages under writeback
3635 *
c5edf9cd
TH
3636 * Determine the numbers of file, headroom, dirty, and writeback pages in
3637 * @wb's memcg. File, dirty and writeback are self-explanatory. Headroom
3638 * is a bit more involved.
c2aa723a 3639 *
c5edf9cd
TH
3640 * A memcg's headroom is "min(max, high) - used". In the hierarchy, the
3641 * headroom is calculated as the lowest headroom of itself and the
3642 * ancestors. Note that this doesn't consider the actual amount of
3643 * available memory in the system. The caller should further cap
3644 * *@pheadroom accordingly.
c2aa723a 3645 */
c5edf9cd
TH
3646void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
3647 unsigned long *pheadroom, unsigned long *pdirty,
3648 unsigned long *pwriteback)
c2aa723a
TH
3649{
3650 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
3651 struct mem_cgroup *parent;
c2aa723a 3652
ccda7f43 3653 *pdirty = memcg_page_state(memcg, NR_FILE_DIRTY);
c2aa723a
TH
3654
3655 /* this should eventually include NR_UNSTABLE_NFS */
ccda7f43 3656 *pwriteback = memcg_page_state(memcg, NR_WRITEBACK);
c5edf9cd
TH
3657 *pfilepages = mem_cgroup_nr_lru_pages(memcg, (1 << LRU_INACTIVE_FILE) |
3658 (1 << LRU_ACTIVE_FILE));
3659 *pheadroom = PAGE_COUNTER_MAX;
c2aa723a 3660
c2aa723a
TH
3661 while ((parent = parent_mem_cgroup(memcg))) {
3662 unsigned long ceiling = min(memcg->memory.limit, memcg->high);
3663 unsigned long used = page_counter_read(&memcg->memory);
3664
c5edf9cd 3665 *pheadroom = min(*pheadroom, ceiling - min(ceiling, used));
c2aa723a
TH
3666 memcg = parent;
3667 }
c2aa723a
TH
3668}
3669
841710aa
TH
3670#else /* CONFIG_CGROUP_WRITEBACK */
3671
3672static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
3673{
3674 return 0;
3675}
3676
3677static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
3678{
3679}
3680
2529bb3a
TH
3681static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
3682{
3683}
3684
52ebea74
TH
3685#endif /* CONFIG_CGROUP_WRITEBACK */
3686
3bc942f3
TH
3687/*
3688 * DO NOT USE IN NEW FILES.
3689 *
3690 * "cgroup.event_control" implementation.
3691 *
3692 * This is way over-engineered. It tries to support fully configurable
3693 * events for each user. Such level of flexibility is completely
3694 * unnecessary especially in the light of the planned unified hierarchy.
3695 *
3696 * Please deprecate this and replace with something simpler if at all
3697 * possible.
3698 */
3699
79bd9814
TH
3700/*
3701 * Unregister event and free resources.
3702 *
3703 * Gets called from workqueue.
3704 */
3bc942f3 3705static void memcg_event_remove(struct work_struct *work)
79bd9814 3706{
3bc942f3
TH
3707 struct mem_cgroup_event *event =
3708 container_of(work, struct mem_cgroup_event, remove);
59b6f873 3709 struct mem_cgroup *memcg = event->memcg;
79bd9814
TH
3710
3711 remove_wait_queue(event->wqh, &event->wait);
3712
59b6f873 3713 event->unregister_event(memcg, event->eventfd);
79bd9814
TH
3714
3715 /* Notify userspace the event is going away. */
3716 eventfd_signal(event->eventfd, 1);
3717
3718 eventfd_ctx_put(event->eventfd);
3719 kfree(event);
59b6f873 3720 css_put(&memcg->css);
79bd9814
TH
3721}
3722
3723/*
3724 * Gets called on POLLHUP on eventfd when user closes it.
3725 *
3726 * Called with wqh->lock held and interrupts disabled.
3727 */
3bc942f3
TH
3728static int memcg_event_wake(wait_queue_t *wait, unsigned mode,
3729 int sync, void *key)
79bd9814 3730{
3bc942f3
TH
3731 struct mem_cgroup_event *event =
3732 container_of(wait, struct mem_cgroup_event, wait);
59b6f873 3733 struct mem_cgroup *memcg = event->memcg;
79bd9814
TH
3734 unsigned long flags = (unsigned long)key;
3735
3736 if (flags & POLLHUP) {
3737 /*
3738 * If the event has been detached at cgroup removal, we
3739 * can simply return knowing the other side will cleanup
3740 * for us.
3741 *
3742 * We can't race against event freeing since the other
3743 * side will require wqh->lock via remove_wait_queue(),
3744 * which we hold.
3745 */
fba94807 3746 spin_lock(&memcg->event_list_lock);
79bd9814
TH
3747 if (!list_empty(&event->list)) {
3748 list_del_init(&event->list);
3749 /*
3750 * We are in atomic context, but cgroup_event_remove()
3751 * may sleep, so we have to call it in workqueue.
3752 */
3753 schedule_work(&event->remove);
3754 }
fba94807 3755 spin_unlock(&memcg->event_list_lock);
79bd9814
TH
3756 }
3757
3758 return 0;
3759}
3760
3bc942f3 3761static void memcg_event_ptable_queue_proc(struct file *file,
79bd9814
TH
3762 wait_queue_head_t *wqh, poll_table *pt)
3763{
3bc942f3
TH
3764 struct mem_cgroup_event *event =
3765 container_of(pt, struct mem_cgroup_event, pt);
79bd9814
TH
3766
3767 event->wqh = wqh;
3768 add_wait_queue(wqh, &event->wait);
3769}
3770
3771/*
3bc942f3
TH
3772 * DO NOT USE IN NEW FILES.
3773 *
79bd9814
TH
3774 * Parse input and register new cgroup event handler.
3775 *
3776 * Input must be in format '<event_fd> <control_fd> <args>'.
3777 * Interpretation of args is defined by control file implementation.
3778 */
451af504
TH
3779static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
3780 char *buf, size_t nbytes, loff_t off)
79bd9814 3781{
451af504 3782 struct cgroup_subsys_state *css = of_css(of);
fba94807 3783 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3bc942f3 3784 struct mem_cgroup_event *event;
79bd9814
TH
3785 struct cgroup_subsys_state *cfile_css;
3786 unsigned int efd, cfd;
3787 struct fd efile;
3788 struct fd cfile;
fba94807 3789 const char *name;
79bd9814
TH
3790 char *endp;
3791 int ret;
3792
451af504
TH
3793 buf = strstrip(buf);
3794
3795 efd = simple_strtoul(buf, &endp, 10);
79bd9814
TH
3796 if (*endp != ' ')
3797 return -EINVAL;
451af504 3798 buf = endp + 1;
79bd9814 3799
451af504 3800 cfd = simple_strtoul(buf, &endp, 10);
79bd9814
TH
3801 if ((*endp != ' ') && (*endp != '\0'))
3802 return -EINVAL;
451af504 3803 buf = endp + 1;
79bd9814
TH
3804
3805 event = kzalloc(sizeof(*event), GFP_KERNEL);
3806 if (!event)
3807 return -ENOMEM;
3808
59b6f873 3809 event->memcg = memcg;
79bd9814 3810 INIT_LIST_HEAD(&event->list);
3bc942f3
TH
3811 init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);
3812 init_waitqueue_func_entry(&event->wait, memcg_event_wake);
3813 INIT_WORK(&event->remove, memcg_event_remove);
79bd9814
TH
3814
3815 efile = fdget(efd);
3816 if (!efile.file) {
3817 ret = -EBADF;
3818 goto out_kfree;
3819 }
3820
3821 event->eventfd = eventfd_ctx_fileget(efile.file);
3822 if (IS_ERR(event->eventfd)) {
3823 ret = PTR_ERR(event->eventfd);
3824 goto out_put_efile;
3825 }
3826
3827 cfile = fdget(cfd);
3828 if (!cfile.file) {
3829 ret = -EBADF;
3830 goto out_put_eventfd;
3831 }
3832
3833 /* the process need read permission on control file */
3834 /* AV: shouldn't we check that it's been opened for read instead? */
3835 ret = inode_permission(file_inode(cfile.file), MAY_READ);
3836 if (ret < 0)
3837 goto out_put_cfile;
3838
fba94807
TH
3839 /*
3840 * Determine the event callbacks and set them in @event. This used
3841 * to be done via struct cftype but cgroup core no longer knows
3842 * about these events. The following is crude but the whole thing
3843 * is for compatibility anyway.
3bc942f3
TH
3844 *
3845 * DO NOT ADD NEW FILES.
fba94807 3846 */
b583043e 3847 name = cfile.file->f_path.dentry->d_name.name;
fba94807
TH
3848
3849 if (!strcmp(name, "memory.usage_in_bytes")) {
3850 event->register_event = mem_cgroup_usage_register_event;
3851 event->unregister_event = mem_cgroup_usage_unregister_event;
3852 } else if (!strcmp(name, "memory.oom_control")) {
3853 event->register_event = mem_cgroup_oom_register_event;
3854 event->unregister_event = mem_cgroup_oom_unregister_event;
3855 } else if (!strcmp(name, "memory.pressure_level")) {
3856 event->register_event = vmpressure_register_event;
3857 event->unregister_event = vmpressure_unregister_event;
3858 } else if (!strcmp(name, "memory.memsw.usage_in_bytes")) {
347c4a87
TH
3859 event->register_event = memsw_cgroup_usage_register_event;
3860 event->unregister_event = memsw_cgroup_usage_unregister_event;
fba94807
TH
3861 } else {
3862 ret = -EINVAL;
3863 goto out_put_cfile;
3864 }
3865
79bd9814 3866 /*
b5557c4c
TH
3867 * Verify @cfile should belong to @css. Also, remaining events are
3868 * automatically removed on cgroup destruction but the removal is
3869 * asynchronous, so take an extra ref on @css.
79bd9814 3870 */
b583043e 3871 cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent,
ec903c0c 3872 &memory_cgrp_subsys);
79bd9814 3873 ret = -EINVAL;
5a17f543 3874 if (IS_ERR(cfile_css))
79bd9814 3875 goto out_put_cfile;
5a17f543
TH
3876 if (cfile_css != css) {
3877 css_put(cfile_css);
79bd9814 3878 goto out_put_cfile;
5a17f543 3879 }
79bd9814 3880
451af504 3881 ret = event->register_event(memcg, event->eventfd, buf);
79bd9814
TH
3882 if (ret)
3883 goto out_put_css;
3884
3885 efile.file->f_op->poll(efile.file, &event->pt);
3886
fba94807
TH
3887 spin_lock(&memcg->event_list_lock);
3888 list_add(&event->list, &memcg->event_list);
3889 spin_unlock(&memcg->event_list_lock);
79bd9814
TH
3890
3891 fdput(cfile);
3892 fdput(efile);
3893
451af504 3894 return nbytes;
79bd9814
TH
3895
3896out_put_css:
b5557c4c 3897 css_put(css);
79bd9814
TH
3898out_put_cfile:
3899 fdput(cfile);
3900out_put_eventfd:
3901 eventfd_ctx_put(event->eventfd);
3902out_put_efile:
3903 fdput(efile);
3904out_kfree:
3905 kfree(event);
3906
3907 return ret;
3908}
3909
241994ed 3910static struct cftype mem_cgroup_legacy_files[] = {
8cdea7c0 3911 {
0eea1030 3912 .name = "usage_in_bytes",
8c7c6e34 3913 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
791badbd 3914 .read_u64 = mem_cgroup_read_u64,
8cdea7c0 3915 },
c84872e1
PE
3916 {
3917 .name = "max_usage_in_bytes",
8c7c6e34 3918 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
6770c64e 3919 .write = mem_cgroup_reset,
791badbd 3920 .read_u64 = mem_cgroup_read_u64,
c84872e1 3921 },
8cdea7c0 3922 {
0eea1030 3923 .name = "limit_in_bytes",
8c7c6e34 3924 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
451af504 3925 .write = mem_cgroup_write,
791badbd 3926 .read_u64 = mem_cgroup_read_u64,
8cdea7c0 3927 },
296c81d8
BS
3928 {
3929 .name = "soft_limit_in_bytes",
3930 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
451af504 3931 .write = mem_cgroup_write,
791badbd 3932 .read_u64 = mem_cgroup_read_u64,
296c81d8 3933 },
8cdea7c0
BS
3934 {
3935 .name = "failcnt",
8c7c6e34 3936 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
6770c64e 3937 .write = mem_cgroup_reset,
791badbd 3938 .read_u64 = mem_cgroup_read_u64,
8cdea7c0 3939 },
d2ceb9b7
KH
3940 {
3941 .name = "stat",
2da8ca82 3942 .seq_show = memcg_stat_show,
d2ceb9b7 3943 },
c1e862c1
KH
3944 {
3945 .name = "force_empty",
6770c64e 3946 .write = mem_cgroup_force_empty_write,
c1e862c1 3947 },
18f59ea7
BS
3948 {
3949 .name = "use_hierarchy",
3950 .write_u64 = mem_cgroup_hierarchy_write,
3951 .read_u64 = mem_cgroup_hierarchy_read,
3952 },
79bd9814 3953 {
3bc942f3 3954 .name = "cgroup.event_control", /* XXX: for compat */
451af504 3955 .write = memcg_write_event_control,
7dbdb199 3956 .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE,
79bd9814 3957 },
a7885eb8
KM
3958 {
3959 .name = "swappiness",
3960 .read_u64 = mem_cgroup_swappiness_read,
3961 .write_u64 = mem_cgroup_swappiness_write,
3962 },
7dc74be0
DN
3963 {
3964 .name = "move_charge_at_immigrate",
3965 .read_u64 = mem_cgroup_move_charge_read,
3966 .write_u64 = mem_cgroup_move_charge_write,
3967 },
9490ff27
KH
3968 {
3969 .name = "oom_control",
2da8ca82 3970 .seq_show = mem_cgroup_oom_control_read,
3c11ecf4 3971 .write_u64 = mem_cgroup_oom_control_write,
9490ff27
KH
3972 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
3973 },
70ddf637
AV
3974 {
3975 .name = "pressure_level",
70ddf637 3976 },
406eb0c9
YH
3977#ifdef CONFIG_NUMA
3978 {
3979 .name = "numa_stat",
2da8ca82 3980 .seq_show = memcg_numa_stat_show,
406eb0c9
YH
3981 },
3982#endif
510fc4e1
GC
3983 {
3984 .name = "kmem.limit_in_bytes",
3985 .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
451af504 3986 .write = mem_cgroup_write,
791badbd 3987 .read_u64 = mem_cgroup_read_u64,
510fc4e1
GC
3988 },
3989 {
3990 .name = "kmem.usage_in_bytes",
3991 .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
791badbd 3992 .read_u64 = mem_cgroup_read_u64,
510fc4e1
GC
3993 },
3994 {
3995 .name = "kmem.failcnt",
3996 .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
6770c64e 3997 .write = mem_cgroup_reset,
791badbd 3998 .read_u64 = mem_cgroup_read_u64,
510fc4e1
GC
3999 },
4000 {
4001 .name = "kmem.max_usage_in_bytes",
4002 .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
6770c64e 4003 .write = mem_cgroup_reset,
791badbd 4004 .read_u64 = mem_cgroup_read_u64,
510fc4e1 4005 },
749c5415
GC
4006#ifdef CONFIG_SLABINFO
4007 {
4008 .name = "kmem.slabinfo",
bc2791f8
TH
4009 .seq_start = memcg_slab_start,
4010 .seq_next = memcg_slab_next,
4011 .seq_stop = memcg_slab_stop,
b047501c 4012 .seq_show = memcg_slab_show,
749c5415
GC
4013 },
4014#endif
d55f90bf
VD
4015 {
4016 .name = "kmem.tcp.limit_in_bytes",
4017 .private = MEMFILE_PRIVATE(_TCP, RES_LIMIT),
4018 .write = mem_cgroup_write,
4019 .read_u64 = mem_cgroup_read_u64,
4020 },
4021 {
4022 .name = "kmem.tcp.usage_in_bytes",
4023 .private = MEMFILE_PRIVATE(_TCP, RES_USAGE),
4024 .read_u64 = mem_cgroup_read_u64,
4025 },
4026 {
4027 .name = "kmem.tcp.failcnt",
4028 .private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT),
4029 .write = mem_cgroup_reset,
4030 .read_u64 = mem_cgroup_read_u64,
4031 },
4032 {
4033 .name = "kmem.tcp.max_usage_in_bytes",
4034 .private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE),
4035 .write = mem_cgroup_reset,
4036 .read_u64 = mem_cgroup_read_u64,
4037 },
6bc10349 4038 { }, /* terminate */
af36f906 4039};
8c7c6e34 4040
73f576c0
JW
4041/*
4042 * Private memory cgroup IDR
4043 *
4044 * Swap-out records and page cache shadow entries need to store memcg
4045 * references in constrained space, so we maintain an ID space that is
4046 * limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of
4047 * memory-controlled cgroups to 64k.
4048 *
4049 * However, there usually are many references to the oflline CSS after
4050 * the cgroup has been destroyed, such as page cache or reclaimable
4051 * slab objects, that don't need to hang on to the ID. We want to keep
4052 * those dead CSS from occupying IDs, or we might quickly exhaust the
4053 * relatively small ID space and prevent the creation of new cgroups
4054 * even when there are much fewer than 64k cgroups - possibly none.
4055 *
4056 * Maintain a private 16-bit ID space for memcg, and allow the ID to
4057 * be freed and recycled when it's no longer needed, which is usually
4058 * when the CSS is offlined.
4059 *
4060 * The only exception to that are records of swapped out tmpfs/shmem
4061 * pages that need to be attributed to live ancestors on swapin. But
4062 * those references are manageable from userspace.
4063 */
4064
4065static DEFINE_IDR(mem_cgroup_idr);
4066
615d66c3 4067static void mem_cgroup_id_get_many(struct mem_cgroup *memcg, unsigned int n)
73f576c0 4068{
58fa2a55 4069 VM_BUG_ON(atomic_read(&memcg->id.ref) <= 0);
615d66c3 4070 atomic_add(n, &memcg->id.ref);
73f576c0
JW
4071}
4072
615d66c3 4073static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
73f576c0 4074{
58fa2a55 4075 VM_BUG_ON(atomic_read(&memcg->id.ref) < n);
615d66c3 4076 if (atomic_sub_and_test(n, &memcg->id.ref)) {
73f576c0
JW
4077 idr_remove(&mem_cgroup_idr, memcg->id.id);
4078 memcg->id.id = 0;
4079
4080 /* Memcg ID pins CSS */
4081 css_put(&memcg->css);
4082 }
4083}
4084
615d66c3
VD
4085static inline void mem_cgroup_id_get(struct mem_cgroup *memcg)
4086{
4087 mem_cgroup_id_get_many(memcg, 1);
4088}
4089
4090static inline void mem_cgroup_id_put(struct mem_cgroup *memcg)
4091{
4092 mem_cgroup_id_put_many(memcg, 1);
4093}
4094
73f576c0
JW
4095/**
4096 * mem_cgroup_from_id - look up a memcg from a memcg id
4097 * @id: the memcg id to look up
4098 *
4099 * Caller must hold rcu_read_lock().
4100 */
4101struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
4102{
4103 WARN_ON_ONCE(!rcu_read_lock_held());
4104 return idr_find(&mem_cgroup_idr, id);
4105}
4106
ef8f2327 4107static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
6d12e2d8
KH
4108{
4109 struct mem_cgroup_per_node *pn;
ef8f2327 4110 int tmp = node;
1ecaab2b
KH
4111 /*
4112 * This routine is called against possible nodes.
4113 * But it's BUG to call kmalloc() against offline node.
4114 *
4115 * TODO: this routine can waste much memory for nodes which will
4116 * never be onlined. It's better to use memory hotplug callback
4117 * function.
4118 */
41e3355d
KH
4119 if (!node_state(node, N_NORMAL_MEMORY))
4120 tmp = -1;
17295c88 4121 pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
6d12e2d8
KH
4122 if (!pn)
4123 return 1;
1ecaab2b 4124
ef8f2327
MG
4125 lruvec_init(&pn->lruvec);
4126 pn->usage_in_excess = 0;
4127 pn->on_tree = false;
4128 pn->memcg = memcg;
4129
54f72fe0 4130 memcg->nodeinfo[node] = pn;
6d12e2d8
KH
4131 return 0;
4132}
4133
ef8f2327 4134static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
1ecaab2b 4135{
54f72fe0 4136 kfree(memcg->nodeinfo[node]);
1ecaab2b
KH
4137}
4138
40e952f9 4139static void __mem_cgroup_free(struct mem_cgroup *memcg)
59927fb9 4140{
c8b2a36f 4141 int node;
59927fb9 4142
c8b2a36f 4143 for_each_node(node)
ef8f2327 4144 free_mem_cgroup_per_node_info(memcg, node);
c8b2a36f 4145 free_percpu(memcg->stat);
8ff69e2c 4146 kfree(memcg);
59927fb9 4147}
3afe36b1 4148
40e952f9
TE
4149static void mem_cgroup_free(struct mem_cgroup *memcg)
4150{
4151 memcg_wb_domain_exit(memcg);
4152 __mem_cgroup_free(memcg);
4153}
4154
0b8f73e1 4155static struct mem_cgroup *mem_cgroup_alloc(void)
8cdea7c0 4156{
d142e3e6 4157 struct mem_cgroup *memcg;
0b8f73e1 4158 size_t size;
6d12e2d8 4159 int node;
8cdea7c0 4160
0b8f73e1
JW
4161 size = sizeof(struct mem_cgroup);
4162 size += nr_node_ids * sizeof(struct mem_cgroup_per_node *);
4163
4164 memcg = kzalloc(size, GFP_KERNEL);
c0ff4b85 4165 if (!memcg)
0b8f73e1
JW
4166 return NULL;
4167
73f576c0
JW
4168 memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL,
4169 1, MEM_CGROUP_ID_MAX,
4170 GFP_KERNEL);
4171 if (memcg->id.id < 0)
4172 goto fail;
4173
0b8f73e1
JW
4174 memcg->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
4175 if (!memcg->stat)
4176 goto fail;
78fb7466 4177
3ed28fa1 4178 for_each_node(node)
ef8f2327 4179 if (alloc_mem_cgroup_per_node_info(memcg, node))
0b8f73e1 4180 goto fail;
f64c3f54 4181
0b8f73e1
JW
4182 if (memcg_wb_domain_init(memcg, GFP_KERNEL))
4183 goto fail;
28dbc4b6 4184
f7e1cb6e 4185 INIT_WORK(&memcg->high_work, high_work_func);
d142e3e6
GC
4186 memcg->last_scanned_node = MAX_NUMNODES;
4187 INIT_LIST_HEAD(&memcg->oom_notify);
d142e3e6
GC
4188 mutex_init(&memcg->thresholds_lock);
4189 spin_lock_init(&memcg->move_lock);
70ddf637 4190 vmpressure_init(&memcg->vmpressure);
fba94807
TH
4191 INIT_LIST_HEAD(&memcg->event_list);
4192 spin_lock_init(&memcg->event_list_lock);
d886f4e4 4193 memcg->socket_pressure = jiffies;
127424c8 4194#ifndef CONFIG_SLOB
900a38f0 4195 memcg->kmemcg_id = -1;
900a38f0 4196#endif
52ebea74
TH
4197#ifdef CONFIG_CGROUP_WRITEBACK
4198 INIT_LIST_HEAD(&memcg->cgwb_list);
4199#endif
73f576c0 4200 idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
0b8f73e1
JW
4201 return memcg;
4202fail:
73f576c0
JW
4203 if (memcg->id.id > 0)
4204 idr_remove(&mem_cgroup_idr, memcg->id.id);
40e952f9 4205 __mem_cgroup_free(memcg);
0b8f73e1 4206 return NULL;
d142e3e6
GC
4207}
4208
0b8f73e1
JW
4209static struct cgroup_subsys_state * __ref
4210mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
d142e3e6 4211{
0b8f73e1
JW
4212 struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
4213 struct mem_cgroup *memcg;
4214 long error = -ENOMEM;
d142e3e6 4215
0b8f73e1
JW
4216 memcg = mem_cgroup_alloc();
4217 if (!memcg)
4218 return ERR_PTR(error);
d142e3e6 4219
0b8f73e1
JW
4220 memcg->high = PAGE_COUNTER_MAX;
4221 memcg->soft_limit = PAGE_COUNTER_MAX;
4222 if (parent) {
4223 memcg->swappiness = mem_cgroup_swappiness(parent);
4224 memcg->oom_kill_disable = parent->oom_kill_disable;
4225 }
4226 if (parent && parent->use_hierarchy) {
4227 memcg->use_hierarchy = true;
3e32cb2e 4228 page_counter_init(&memcg->memory, &parent->memory);
37e84351 4229 page_counter_init(&memcg->swap, &parent->swap);
3e32cb2e
JW
4230 page_counter_init(&memcg->memsw, &parent->memsw);
4231 page_counter_init(&memcg->kmem, &parent->kmem);
0db15298 4232 page_counter_init(&memcg->tcpmem, &parent->tcpmem);
18f59ea7 4233 } else {
3e32cb2e 4234 page_counter_init(&memcg->memory, NULL);
37e84351 4235 page_counter_init(&memcg->swap, NULL);
3e32cb2e
JW
4236 page_counter_init(&memcg->memsw, NULL);
4237 page_counter_init(&memcg->kmem, NULL);
0db15298 4238 page_counter_init(&memcg->tcpmem, NULL);
8c7f6edb
TH
4239 /*
4240 * Deeper hierachy with use_hierarchy == false doesn't make
4241 * much sense so let cgroup subsystem know about this
4242 * unfortunate state in our controller.
4243 */
d142e3e6 4244 if (parent != root_mem_cgroup)
073219e9 4245 memory_cgrp_subsys.broken_hierarchy = true;
18f59ea7 4246 }
d6441637 4247
0b8f73e1
JW
4248 /* The following stuff does not apply to the root */
4249 if (!parent) {
4250 root_mem_cgroup = memcg;
4251 return &memcg->css;
4252 }
4253
b313aeee 4254 error = memcg_online_kmem(memcg);
0b8f73e1
JW
4255 if (error)
4256 goto fail;
127424c8 4257
f7e1cb6e 4258 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
ef12947c 4259 static_branch_inc(&memcg_sockets_enabled_key);
f7e1cb6e 4260
0b8f73e1
JW
4261 return &memcg->css;
4262fail:
4263 mem_cgroup_free(memcg);
ea3a9645 4264 return ERR_PTR(-ENOMEM);
0b8f73e1
JW
4265}
4266
73f576c0 4267static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
0b8f73e1 4268{
58fa2a55
VD
4269 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4270
73f576c0 4271 /* Online state pins memcg ID, memcg ID pins CSS */
58fa2a55 4272 atomic_set(&memcg->id.ref, 1);
73f576c0 4273 css_get(css);
2f7dd7a4 4274 return 0;
8cdea7c0
BS
4275}
4276
eb95419b 4277static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
df878fb0 4278{
eb95419b 4279 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3bc942f3 4280 struct mem_cgroup_event *event, *tmp;
79bd9814
TH
4281
4282 /*
4283 * Unregister events and notify userspace.
4284 * Notify userspace about cgroup removing only after rmdir of cgroup
4285 * directory to avoid race between userspace and kernelspace.
4286 */
fba94807
TH
4287 spin_lock(&memcg->event_list_lock);
4288 list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
79bd9814
TH
4289 list_del_init(&event->list);
4290 schedule_work(&event->remove);
4291 }
fba94807 4292 spin_unlock(&memcg->event_list_lock);
ec64f515 4293
567e9ab2 4294 memcg_offline_kmem(memcg);
52ebea74 4295 wb_memcg_offline(memcg);
73f576c0
JW
4296
4297 mem_cgroup_id_put(memcg);
df878fb0
KH
4298}
4299
6df38689
VD
4300static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
4301{
4302 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4303
4304 invalidate_reclaim_iterators(memcg);
4305}
4306
eb95419b 4307static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
8cdea7c0 4308{
eb95419b 4309 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
c268e994 4310
f7e1cb6e 4311 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
ef12947c 4312 static_branch_dec(&memcg_sockets_enabled_key);
127424c8 4313
0db15298 4314 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)
d55f90bf 4315 static_branch_dec(&memcg_sockets_enabled_key);
3893e302 4316
0b8f73e1
JW
4317 vmpressure_cleanup(&memcg->vmpressure);
4318 cancel_work_sync(&memcg->high_work);
4319 mem_cgroup_remove_from_trees(memcg);
d886f4e4 4320 memcg_free_kmem(memcg);
0b8f73e1 4321 mem_cgroup_free(memcg);
8cdea7c0
BS
4322}
4323
1ced953b
TH
4324/**
4325 * mem_cgroup_css_reset - reset the states of a mem_cgroup
4326 * @css: the target css
4327 *
4328 * Reset the states of the mem_cgroup associated with @css. This is
4329 * invoked when the userland requests disabling on the default hierarchy
4330 * but the memcg is pinned through dependency. The memcg should stop
4331 * applying policies and should revert to the vanilla state as it may be
4332 * made visible again.
4333 *
4334 * The current implementation only resets the essential configurations.
4335 * This needs to be expanded to cover all the visible parts.
4336 */
4337static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
4338{
4339 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4340
d334c9bc
VD
4341 page_counter_limit(&memcg->memory, PAGE_COUNTER_MAX);
4342 page_counter_limit(&memcg->swap, PAGE_COUNTER_MAX);
4343 page_counter_limit(&memcg->memsw, PAGE_COUNTER_MAX);
4344 page_counter_limit(&memcg->kmem, PAGE_COUNTER_MAX);
4345 page_counter_limit(&memcg->tcpmem, PAGE_COUNTER_MAX);
241994ed
JW
4346 memcg->low = 0;
4347 memcg->high = PAGE_COUNTER_MAX;
24d404dc 4348 memcg->soft_limit = PAGE_COUNTER_MAX;
2529bb3a 4349 memcg_wb_domain_size_changed(memcg);
1ced953b
TH
4350}
4351
02491447 4352#ifdef CONFIG_MMU
7dc74be0 4353/* Handlers for move charge at task migration. */
854ffa8d 4354static int mem_cgroup_do_precharge(unsigned long count)
7dc74be0 4355{
05b84301 4356 int ret;
9476db97 4357
d0164adc
MG
4358 /* Try a single bulk charge without reclaim first, kswapd may wake */
4359 ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count);
9476db97 4360 if (!ret) {
854ffa8d 4361 mc.precharge += count;
854ffa8d
DN
4362 return ret;
4363 }
9476db97 4364
3674534b 4365 /* Try charges one by one with reclaim, but do not retry */
854ffa8d 4366 while (count--) {
3674534b 4367 ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1);
38c5d72f 4368 if (ret)
38c5d72f 4369 return ret;
854ffa8d 4370 mc.precharge++;
9476db97 4371 cond_resched();
854ffa8d 4372 }
9476db97 4373 return 0;
4ffef5fe
DN
4374}
4375
4ffef5fe
DN
4376union mc_target {
4377 struct page *page;
02491447 4378 swp_entry_t ent;
4ffef5fe
DN
4379};
4380
4ffef5fe 4381enum mc_target_type {
8d32ff84 4382 MC_TARGET_NONE = 0,
4ffef5fe 4383 MC_TARGET_PAGE,
02491447 4384 MC_TARGET_SWAP,
4ffef5fe
DN
4385};
4386
90254a65
DN
4387static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
4388 unsigned long addr, pte_t ptent)
4ffef5fe 4389{
90254a65 4390 struct page *page = vm_normal_page(vma, addr, ptent);
4ffef5fe 4391
90254a65
DN
4392 if (!page || !page_mapped(page))
4393 return NULL;
4394 if (PageAnon(page)) {
1dfab5ab 4395 if (!(mc.flags & MOVE_ANON))
90254a65 4396 return NULL;
1dfab5ab
JW
4397 } else {
4398 if (!(mc.flags & MOVE_FILE))
4399 return NULL;
4400 }
90254a65
DN
4401 if (!get_page_unless_zero(page))
4402 return NULL;
4403
4404 return page;
4405}
4406
4b91355e 4407#ifdef CONFIG_SWAP
90254a65 4408static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
48406ef8 4409 pte_t ptent, swp_entry_t *entry)
90254a65 4410{
90254a65
DN
4411 struct page *page = NULL;
4412 swp_entry_t ent = pte_to_swp_entry(ptent);
4413
1dfab5ab 4414 if (!(mc.flags & MOVE_ANON) || non_swap_entry(ent))
90254a65 4415 return NULL;
4b91355e
KH
4416 /*
4417 * Because lookup_swap_cache() updates some statistics counter,
4418 * we call find_get_page() with swapper_space directly.
4419 */
f6ab1f7f 4420 page = find_get_page(swap_address_space(ent), swp_offset(ent));
7941d214 4421 if (do_memsw_account())
90254a65
DN
4422 entry->val = ent.val;
4423
4424 return page;
4425}
4b91355e
KH
4426#else
4427static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
48406ef8 4428 pte_t ptent, swp_entry_t *entry)
4b91355e
KH
4429{
4430 return NULL;
4431}
4432#endif
90254a65 4433
87946a72
DN
4434static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
4435 unsigned long addr, pte_t ptent, swp_entry_t *entry)
4436{
4437 struct page *page = NULL;
87946a72
DN
4438 struct address_space *mapping;
4439 pgoff_t pgoff;
4440
4441 if (!vma->vm_file) /* anonymous vma */
4442 return NULL;
1dfab5ab 4443 if (!(mc.flags & MOVE_FILE))
87946a72
DN
4444 return NULL;
4445
87946a72 4446 mapping = vma->vm_file->f_mapping;
0661a336 4447 pgoff = linear_page_index(vma, addr);
87946a72
DN
4448
4449 /* page is moved even if it's not RSS of this task(page-faulted). */
aa3b1895
HD
4450#ifdef CONFIG_SWAP
4451 /* shmem/tmpfs may report page out on swap: account for that too. */
139b6a6f
JW
4452 if (shmem_mapping(mapping)) {
4453 page = find_get_entry(mapping, pgoff);
4454 if (radix_tree_exceptional_entry(page)) {
4455 swp_entry_t swp = radix_to_swp_entry(page);
7941d214 4456 if (do_memsw_account())
139b6a6f 4457 *entry = swp;
f6ab1f7f
HY
4458 page = find_get_page(swap_address_space(swp),
4459 swp_offset(swp));
139b6a6f
JW
4460 }
4461 } else
4462 page = find_get_page(mapping, pgoff);
4463#else
4464 page = find_get_page(mapping, pgoff);
aa3b1895 4465#endif
87946a72
DN
4466 return page;
4467}
4468
b1b0deab
CG
4469/**
4470 * mem_cgroup_move_account - move account of the page
4471 * @page: the page
25843c2b 4472 * @compound: charge the page as compound or small page
b1b0deab
CG
4473 * @from: mem_cgroup which the page is moved from.
4474 * @to: mem_cgroup which the page is moved to. @from != @to.
4475 *
3ac808fd 4476 * The caller must make sure the page is not on LRU (isolate_page() is useful.)
b1b0deab
CG
4477 *
4478 * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
4479 * from old cgroup.
4480 */
4481static int mem_cgroup_move_account(struct page *page,
f627c2f5 4482 bool compound,
b1b0deab
CG
4483 struct mem_cgroup *from,
4484 struct mem_cgroup *to)
4485{
4486 unsigned long flags;
f627c2f5 4487 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
b1b0deab 4488 int ret;
c4843a75 4489 bool anon;
b1b0deab
CG
4490
4491 VM_BUG_ON(from == to);
4492 VM_BUG_ON_PAGE(PageLRU(page), page);
f627c2f5 4493 VM_BUG_ON(compound && !PageTransHuge(page));
b1b0deab
CG
4494
4495 /*
6a93ca8f 4496 * Prevent mem_cgroup_migrate() from looking at
45637bab 4497 * page->mem_cgroup of its source page while we change it.
b1b0deab 4498 */
f627c2f5 4499 ret = -EBUSY;
b1b0deab
CG
4500 if (!trylock_page(page))
4501 goto out;
4502
4503 ret = -EINVAL;
4504 if (page->mem_cgroup != from)
4505 goto out_unlock;
4506
c4843a75
GT
4507 anon = PageAnon(page);
4508
b1b0deab
CG
4509 spin_lock_irqsave(&from->move_lock, flags);
4510
c4843a75 4511 if (!anon && page_mapped(page)) {
71cd3113
JW
4512 __this_cpu_sub(from->stat->count[NR_FILE_MAPPED], nr_pages);
4513 __this_cpu_add(to->stat->count[NR_FILE_MAPPED], nr_pages);
b1b0deab
CG
4514 }
4515
c4843a75
GT
4516 /*
4517 * move_lock grabbed above and caller set from->moving_account, so
ccda7f43 4518 * mod_memcg_page_state will serialize updates to PageDirty.
c4843a75
GT
4519 * So mapping should be stable for dirty pages.
4520 */
4521 if (!anon && PageDirty(page)) {
4522 struct address_space *mapping = page_mapping(page);
4523
4524 if (mapping_cap_account_dirty(mapping)) {
71cd3113 4525 __this_cpu_sub(from->stat->count[NR_FILE_DIRTY],
c4843a75 4526 nr_pages);
71cd3113 4527 __this_cpu_add(to->stat->count[NR_FILE_DIRTY],
c4843a75
GT
4528 nr_pages);
4529 }
4530 }
4531
b1b0deab 4532 if (PageWriteback(page)) {
71cd3113
JW
4533 __this_cpu_sub(from->stat->count[NR_WRITEBACK], nr_pages);
4534 __this_cpu_add(to->stat->count[NR_WRITEBACK], nr_pages);
b1b0deab
CG
4535 }
4536
4537 /*
4538 * It is safe to change page->mem_cgroup here because the page
4539 * is referenced, charged, and isolated - we can't race with
4540 * uncharging, charging, migration, or LRU putback.
4541 */
4542
4543 /* caller should have done css_get */
4544 page->mem_cgroup = to;
4545 spin_unlock_irqrestore(&from->move_lock, flags);
4546
4547 ret = 0;
4548
4549 local_irq_disable();
f627c2f5 4550 mem_cgroup_charge_statistics(to, page, compound, nr_pages);
b1b0deab 4551 memcg_check_events(to, page);
f627c2f5 4552 mem_cgroup_charge_statistics(from, page, compound, -nr_pages);
b1b0deab
CG
4553 memcg_check_events(from, page);
4554 local_irq_enable();
4555out_unlock:
4556 unlock_page(page);
4557out:
4558 return ret;
4559}
4560
7cf7806c
LR
4561/**
4562 * get_mctgt_type - get target type of moving charge
4563 * @vma: the vma the pte to be checked belongs
4564 * @addr: the address corresponding to the pte to be checked
4565 * @ptent: the pte to be checked
4566 * @target: the pointer the target page or swap ent will be stored(can be NULL)
4567 *
4568 * Returns
4569 * 0(MC_TARGET_NONE): if the pte is not a target for move charge.
4570 * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
4571 * move charge. if @target is not NULL, the page is stored in target->page
4572 * with extra refcnt got(Callers should handle it).
4573 * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
4574 * target for charge migration. if @target is not NULL, the entry is stored
4575 * in target->ent.
4576 *
4577 * Called with pte lock held.
4578 */
4579
8d32ff84 4580static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
90254a65
DN
4581 unsigned long addr, pte_t ptent, union mc_target *target)
4582{
4583 struct page *page = NULL;
8d32ff84 4584 enum mc_target_type ret = MC_TARGET_NONE;
90254a65
DN
4585 swp_entry_t ent = { .val = 0 };
4586
4587 if (pte_present(ptent))
4588 page = mc_handle_present_pte(vma, addr, ptent);
4589 else if (is_swap_pte(ptent))
48406ef8 4590 page = mc_handle_swap_pte(vma, ptent, &ent);
0661a336 4591 else if (pte_none(ptent))
87946a72 4592 page = mc_handle_file_pte(vma, addr, ptent, &ent);
90254a65
DN
4593
4594 if (!page && !ent.val)
8d32ff84 4595 return ret;
02491447 4596 if (page) {
02491447 4597 /*
0a31bc97 4598 * Do only loose check w/o serialization.
1306a85a 4599 * mem_cgroup_move_account() checks the page is valid or
0a31bc97 4600 * not under LRU exclusion.
02491447 4601 */
1306a85a 4602 if (page->mem_cgroup == mc.from) {
02491447
DN
4603 ret = MC_TARGET_PAGE;
4604 if (target)
4605 target->page = page;
4606 }
4607 if (!ret || !target)
4608 put_page(page);
4609 }
90254a65
DN
4610 /* There is a swap entry and a page doesn't exist or isn't charged */
4611 if (ent.val && !ret &&
34c00c31 4612 mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
7f0f1546
KH
4613 ret = MC_TARGET_SWAP;
4614 if (target)
4615 target->ent = ent;
4ffef5fe 4616 }
4ffef5fe
DN
4617 return ret;
4618}
4619
12724850
NH
4620#ifdef CONFIG_TRANSPARENT_HUGEPAGE
4621/*
4622 * We don't consider swapping or file mapped pages because THP does not
4623 * support them for now.
4624 * Caller should make sure that pmd_trans_huge(pmd) is true.
4625 */
4626static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
4627 unsigned long addr, pmd_t pmd, union mc_target *target)
4628{
4629 struct page *page = NULL;
12724850
NH
4630 enum mc_target_type ret = MC_TARGET_NONE;
4631
4632 page = pmd_page(pmd);
309381fe 4633 VM_BUG_ON_PAGE(!page || !PageHead(page), page);
1dfab5ab 4634 if (!(mc.flags & MOVE_ANON))
12724850 4635 return ret;
1306a85a 4636 if (page->mem_cgroup == mc.from) {
12724850
NH
4637 ret = MC_TARGET_PAGE;
4638 if (target) {
4639 get_page(page);
4640 target->page = page;
4641 }
4642 }
4643 return ret;
4644}
4645#else
4646static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
4647 unsigned long addr, pmd_t pmd, union mc_target *target)
4648{
4649 return MC_TARGET_NONE;
4650}
4651#endif
4652
4ffef5fe
DN
4653static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
4654 unsigned long addr, unsigned long end,
4655 struct mm_walk *walk)
4656{
26bcd64a 4657 struct vm_area_struct *vma = walk->vma;
4ffef5fe
DN
4658 pte_t *pte;
4659 spinlock_t *ptl;
4660
b6ec57f4
KS
4661 ptl = pmd_trans_huge_lock(pmd, vma);
4662 if (ptl) {
12724850
NH
4663 if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
4664 mc.precharge += HPAGE_PMD_NR;
bf929152 4665 spin_unlock(ptl);
1a5a9906 4666 return 0;
12724850 4667 }
03319327 4668
45f83cef
AA
4669 if (pmd_trans_unstable(pmd))
4670 return 0;
4ffef5fe
DN
4671 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4672 for (; addr != end; pte++, addr += PAGE_SIZE)
8d32ff84 4673 if (get_mctgt_type(vma, addr, *pte, NULL))
4ffef5fe
DN
4674 mc.precharge++; /* increment precharge temporarily */
4675 pte_unmap_unlock(pte - 1, ptl);
4676 cond_resched();
4677
7dc74be0
DN
4678 return 0;
4679}
4680
4ffef5fe
DN
4681static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
4682{
4683 unsigned long precharge;
4ffef5fe 4684
26bcd64a
NH
4685 struct mm_walk mem_cgroup_count_precharge_walk = {
4686 .pmd_entry = mem_cgroup_count_precharge_pte_range,
4687 .mm = mm,
4688 };
dfe076b0 4689 down_read(&mm->mmap_sem);
0247f3f4
JM
4690 walk_page_range(0, mm->highest_vm_end,
4691 &mem_cgroup_count_precharge_walk);
dfe076b0 4692 up_read(&mm->mmap_sem);
4ffef5fe
DN
4693
4694 precharge = mc.precharge;
4695 mc.precharge = 0;
4696
4697 return precharge;
4698}
4699
4ffef5fe
DN
4700static int mem_cgroup_precharge_mc(struct mm_struct *mm)
4701{
dfe076b0
DN
4702 unsigned long precharge = mem_cgroup_count_precharge(mm);
4703
4704 VM_BUG_ON(mc.moving_task);
4705 mc.moving_task = current;
4706 return mem_cgroup_do_precharge(precharge);
4ffef5fe
DN
4707}
4708
dfe076b0
DN
4709/* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
4710static void __mem_cgroup_clear_mc(void)
4ffef5fe 4711{
2bd9bb20
KH
4712 struct mem_cgroup *from = mc.from;
4713 struct mem_cgroup *to = mc.to;
4714
4ffef5fe 4715 /* we must uncharge all the leftover precharges from mc.to */
854ffa8d 4716 if (mc.precharge) {
00501b53 4717 cancel_charge(mc.to, mc.precharge);
854ffa8d
DN
4718 mc.precharge = 0;
4719 }
4720 /*
4721 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
4722 * we must uncharge here.
4723 */
4724 if (mc.moved_charge) {
00501b53 4725 cancel_charge(mc.from, mc.moved_charge);
854ffa8d 4726 mc.moved_charge = 0;
4ffef5fe 4727 }
483c30b5
DN
4728 /* we must fixup refcnts and charges */
4729 if (mc.moved_swap) {
483c30b5 4730 /* uncharge swap account from the old cgroup */
ce00a967 4731 if (!mem_cgroup_is_root(mc.from))
3e32cb2e 4732 page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
483c30b5 4733
615d66c3
VD
4734 mem_cgroup_id_put_many(mc.from, mc.moved_swap);
4735
05b84301 4736 /*
3e32cb2e
JW
4737 * we charged both to->memory and to->memsw, so we
4738 * should uncharge to->memory.
05b84301 4739 */
ce00a967 4740 if (!mem_cgroup_is_root(mc.to))
3e32cb2e
JW
4741 page_counter_uncharge(&mc.to->memory, mc.moved_swap);
4742
615d66c3
VD
4743 mem_cgroup_id_get_many(mc.to, mc.moved_swap);
4744 css_put_many(&mc.to->css, mc.moved_swap);
3e32cb2e 4745
483c30b5
DN
4746 mc.moved_swap = 0;
4747 }
dfe076b0
DN
4748 memcg_oom_recover(from);
4749 memcg_oom_recover(to);
4750 wake_up_all(&mc.waitq);
4751}
4752
4753static void mem_cgroup_clear_mc(void)
4754{
264a0ae1
TH
4755 struct mm_struct *mm = mc.mm;
4756
dfe076b0
DN
4757 /*
4758 * we must clear moving_task before waking up waiters at the end of
4759 * task migration.
4760 */
4761 mc.moving_task = NULL;
4762 __mem_cgroup_clear_mc();
2bd9bb20 4763 spin_lock(&mc.lock);
4ffef5fe
DN
4764 mc.from = NULL;
4765 mc.to = NULL;
264a0ae1 4766 mc.mm = NULL;
2bd9bb20 4767 spin_unlock(&mc.lock);
264a0ae1
TH
4768
4769 mmput(mm);
4ffef5fe
DN
4770}
4771
1f7dd3e5 4772static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
7dc74be0 4773{
1f7dd3e5 4774 struct cgroup_subsys_state *css;
eed67d75 4775 struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
9f2115f9 4776 struct mem_cgroup *from;
4530eddb 4777 struct task_struct *leader, *p;
9f2115f9 4778 struct mm_struct *mm;
1dfab5ab 4779 unsigned long move_flags;
9f2115f9 4780 int ret = 0;
7dc74be0 4781
1f7dd3e5
TH
4782 /* charge immigration isn't supported on the default hierarchy */
4783 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
9f2115f9
TH
4784 return 0;
4785
4530eddb
TH
4786 /*
4787 * Multi-process migrations only happen on the default hierarchy
4788 * where charge immigration is not used. Perform charge
4789 * immigration if @tset contains a leader and whine if there are
4790 * multiple.
4791 */
4792 p = NULL;
1f7dd3e5 4793 cgroup_taskset_for_each_leader(leader, css, tset) {
4530eddb
TH
4794 WARN_ON_ONCE(p);
4795 p = leader;
1f7dd3e5 4796 memcg = mem_cgroup_from_css(css);
4530eddb
TH
4797 }
4798 if (!p)
4799 return 0;
4800
1f7dd3e5
TH
4801 /*
4802 * We are now commited to this value whatever it is. Changes in this
4803 * tunable will only affect upcoming migrations, not the current one.
4804 * So we need to save it, and keep it going.
4805 */
4806 move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
4807 if (!move_flags)
4808 return 0;
4809
9f2115f9
TH
4810 from = mem_cgroup_from_task(p);
4811
4812 VM_BUG_ON(from == memcg);
4813
4814 mm = get_task_mm(p);
4815 if (!mm)
4816 return 0;
4817 /* We move charges only when we move a owner of the mm */
4818 if (mm->owner == p) {
4819 VM_BUG_ON(mc.from);
4820 VM_BUG_ON(mc.to);
4821 VM_BUG_ON(mc.precharge);
4822 VM_BUG_ON(mc.moved_charge);
4823 VM_BUG_ON(mc.moved_swap);
4824
4825 spin_lock(&mc.lock);
264a0ae1 4826 mc.mm = mm;
9f2115f9
TH
4827 mc.from = from;
4828 mc.to = memcg;
4829 mc.flags = move_flags;
4830 spin_unlock(&mc.lock);
4831 /* We set mc.moving_task later */
4832
4833 ret = mem_cgroup_precharge_mc(mm);
4834 if (ret)
4835 mem_cgroup_clear_mc();
264a0ae1
TH
4836 } else {
4837 mmput(mm);
7dc74be0
DN
4838 }
4839 return ret;
4840}
4841
1f7dd3e5 4842static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
7dc74be0 4843{
4e2f245d
JW
4844 if (mc.to)
4845 mem_cgroup_clear_mc();
7dc74be0
DN
4846}
4847
4ffef5fe
DN
4848static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
4849 unsigned long addr, unsigned long end,
4850 struct mm_walk *walk)
7dc74be0 4851{
4ffef5fe 4852 int ret = 0;
26bcd64a 4853 struct vm_area_struct *vma = walk->vma;
4ffef5fe
DN
4854 pte_t *pte;
4855 spinlock_t *ptl;
12724850
NH
4856 enum mc_target_type target_type;
4857 union mc_target target;
4858 struct page *page;
4ffef5fe 4859
b6ec57f4
KS
4860 ptl = pmd_trans_huge_lock(pmd, vma);
4861 if (ptl) {
62ade86a 4862 if (mc.precharge < HPAGE_PMD_NR) {
bf929152 4863 spin_unlock(ptl);
12724850
NH
4864 return 0;
4865 }
4866 target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
4867 if (target_type == MC_TARGET_PAGE) {
4868 page = target.page;
4869 if (!isolate_lru_page(page)) {
f627c2f5 4870 if (!mem_cgroup_move_account(page, true,
1306a85a 4871 mc.from, mc.to)) {
12724850
NH
4872 mc.precharge -= HPAGE_PMD_NR;
4873 mc.moved_charge += HPAGE_PMD_NR;
4874 }
4875 putback_lru_page(page);
4876 }
4877 put_page(page);
4878 }
bf929152 4879 spin_unlock(ptl);
1a5a9906 4880 return 0;
12724850
NH
4881 }
4882
45f83cef
AA
4883 if (pmd_trans_unstable(pmd))
4884 return 0;
4ffef5fe
DN
4885retry:
4886 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4887 for (; addr != end; addr += PAGE_SIZE) {
4888 pte_t ptent = *(pte++);
02491447 4889 swp_entry_t ent;
4ffef5fe
DN
4890
4891 if (!mc.precharge)
4892 break;
4893
8d32ff84 4894 switch (get_mctgt_type(vma, addr, ptent, &target)) {
4ffef5fe
DN
4895 case MC_TARGET_PAGE:
4896 page = target.page;
53f9263b
KS
4897 /*
4898 * We can have a part of the split pmd here. Moving it
4899 * can be done but it would be too convoluted so simply
4900 * ignore such a partial THP and keep it in original
4901 * memcg. There should be somebody mapping the head.
4902 */
4903 if (PageTransCompound(page))
4904 goto put;
4ffef5fe
DN
4905 if (isolate_lru_page(page))
4906 goto put;
f627c2f5
KS
4907 if (!mem_cgroup_move_account(page, false,
4908 mc.from, mc.to)) {
4ffef5fe 4909 mc.precharge--;
854ffa8d
DN
4910 /* we uncharge from mc.from later. */
4911 mc.moved_charge++;
4ffef5fe
DN
4912 }
4913 putback_lru_page(page);
8d32ff84 4914put: /* get_mctgt_type() gets the page */
4ffef5fe
DN
4915 put_page(page);
4916 break;
02491447
DN
4917 case MC_TARGET_SWAP:
4918 ent = target.ent;
e91cbb42 4919 if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
02491447 4920 mc.precharge--;
483c30b5
DN
4921 /* we fixup refcnts and charges later. */
4922 mc.moved_swap++;
4923 }
02491447 4924 break;
4ffef5fe
DN
4925 default:
4926 break;
4927 }
4928 }
4929 pte_unmap_unlock(pte - 1, ptl);
4930 cond_resched();
4931
4932 if (addr != end) {
4933 /*
4934 * We have consumed all precharges we got in can_attach().
4935 * We try charge one by one, but don't do any additional
4936 * charges to mc.to if we have failed in charge once in attach()
4937 * phase.
4938 */
854ffa8d 4939 ret = mem_cgroup_do_precharge(1);
4ffef5fe
DN
4940 if (!ret)
4941 goto retry;
4942 }
4943
4944 return ret;
4945}
4946
264a0ae1 4947static void mem_cgroup_move_charge(void)
4ffef5fe 4948{
26bcd64a
NH
4949 struct mm_walk mem_cgroup_move_charge_walk = {
4950 .pmd_entry = mem_cgroup_move_charge_pte_range,
264a0ae1 4951 .mm = mc.mm,
26bcd64a 4952 };
4ffef5fe
DN
4953
4954 lru_add_drain_all();
312722cb 4955 /*
81f8c3a4
JW
4956 * Signal lock_page_memcg() to take the memcg's move_lock
4957 * while we're moving its pages to another memcg. Then wait
4958 * for already started RCU-only updates to finish.
312722cb
JW
4959 */
4960 atomic_inc(&mc.from->moving_account);
4961 synchronize_rcu();
dfe076b0 4962retry:
264a0ae1 4963 if (unlikely(!down_read_trylock(&mc.mm->mmap_sem))) {
dfe076b0
DN
4964 /*
4965 * Someone who are holding the mmap_sem might be waiting in
4966 * waitq. So we cancel all extra charges, wake up all waiters,
4967 * and retry. Because we cancel precharges, we might not be able
4968 * to move enough charges, but moving charge is a best-effort
4969 * feature anyway, so it wouldn't be a big problem.
4970 */
4971 __mem_cgroup_clear_mc();
4972 cond_resched();
4973 goto retry;
4974 }
26bcd64a
NH
4975 /*
4976 * When we have consumed all precharges and failed in doing
4977 * additional charge, the page walk just aborts.
4978 */
0247f3f4
JM
4979 walk_page_range(0, mc.mm->highest_vm_end, &mem_cgroup_move_charge_walk);
4980
264a0ae1 4981 up_read(&mc.mm->mmap_sem);
312722cb 4982 atomic_dec(&mc.from->moving_account);
7dc74be0
DN
4983}
4984
264a0ae1 4985static void mem_cgroup_move_task(void)
67e465a7 4986{
264a0ae1
TH
4987 if (mc.to) {
4988 mem_cgroup_move_charge();
a433658c 4989 mem_cgroup_clear_mc();
264a0ae1 4990 }
67e465a7 4991}
5cfb80a7 4992#else /* !CONFIG_MMU */
1f7dd3e5 4993static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
5cfb80a7
DN
4994{
4995 return 0;
4996}
1f7dd3e5 4997static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
5cfb80a7
DN
4998{
4999}
264a0ae1 5000static void mem_cgroup_move_task(void)
5cfb80a7
DN
5001{
5002}
5003#endif
67e465a7 5004
f00baae7
TH
5005/*
5006 * Cgroup retains root cgroups across [un]mount cycles making it necessary
aa6ec29b
TH
5007 * to verify whether we're attached to the default hierarchy on each mount
5008 * attempt.
f00baae7 5009 */
eb95419b 5010static void mem_cgroup_bind(struct cgroup_subsys_state *root_css)
f00baae7
TH
5011{
5012 /*
aa6ec29b 5013 * use_hierarchy is forced on the default hierarchy. cgroup core
f00baae7
TH
5014 * guarantees that @root doesn't have any children, so turning it
5015 * on for the root memcg is enough.
5016 */
9e10a130 5017 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
7feee590
VD
5018 root_mem_cgroup->use_hierarchy = true;
5019 else
5020 root_mem_cgroup->use_hierarchy = false;
f00baae7
TH
5021}
5022
241994ed
JW
5023static u64 memory_current_read(struct cgroup_subsys_state *css,
5024 struct cftype *cft)
5025{
f5fc3c5d
JW
5026 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5027
5028 return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
241994ed
JW
5029}
5030
5031static int memory_low_show(struct seq_file *m, void *v)
5032{
5033 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
4db0c3c2 5034 unsigned long low = READ_ONCE(memcg->low);
241994ed
JW
5035
5036 if (low == PAGE_COUNTER_MAX)
d2973697 5037 seq_puts(m, "max\n");
241994ed
JW
5038 else
5039 seq_printf(m, "%llu\n", (u64)low * PAGE_SIZE);
5040
5041 return 0;
5042}
5043
5044static ssize_t memory_low_write(struct kernfs_open_file *of,
5045 char *buf, size_t nbytes, loff_t off)
5046{
5047 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5048 unsigned long low;
5049 int err;
5050
5051 buf = strstrip(buf);
d2973697 5052 err = page_counter_memparse(buf, "max", &low);
241994ed
JW
5053 if (err)
5054 return err;
5055
5056 memcg->low = low;
5057
5058 return nbytes;
5059}
5060
5061static int memory_high_show(struct seq_file *m, void *v)
5062{
5063 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
4db0c3c2 5064 unsigned long high = READ_ONCE(memcg->high);
241994ed
JW
5065
5066 if (high == PAGE_COUNTER_MAX)
d2973697 5067 seq_puts(m, "max\n");
241994ed
JW
5068 else
5069 seq_printf(m, "%llu\n", (u64)high * PAGE_SIZE);
5070
5071 return 0;
5072}
5073
5074static ssize_t memory_high_write(struct kernfs_open_file *of,
5075 char *buf, size_t nbytes, loff_t off)
5076{
5077 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
588083bb 5078 unsigned long nr_pages;
241994ed
JW
5079 unsigned long high;
5080 int err;
5081
5082 buf = strstrip(buf);
d2973697 5083 err = page_counter_memparse(buf, "max", &high);
241994ed
JW
5084 if (err)
5085 return err;
5086
5087 memcg->high = high;
5088
588083bb
JW
5089 nr_pages = page_counter_read(&memcg->memory);
5090 if (nr_pages > high)
5091 try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
5092 GFP_KERNEL, true);
5093
2529bb3a 5094 memcg_wb_domain_size_changed(memcg);
241994ed
JW
5095 return nbytes;
5096}
5097
5098static int memory_max_show(struct seq_file *m, void *v)
5099{
5100 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
4db0c3c2 5101 unsigned long max = READ_ONCE(memcg->memory.limit);
241994ed
JW
5102
5103 if (max == PAGE_COUNTER_MAX)
d2973697 5104 seq_puts(m, "max\n");
241994ed
JW
5105 else
5106 seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE);
5107
5108 return 0;
5109}
5110
5111static ssize_t memory_max_write(struct kernfs_open_file *of,
5112 char *buf, size_t nbytes, loff_t off)
5113{
5114 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
b6e6edcf
JW
5115 unsigned int nr_reclaims = MEM_CGROUP_RECLAIM_RETRIES;
5116 bool drained = false;
241994ed
JW
5117 unsigned long max;
5118 int err;
5119
5120 buf = strstrip(buf);
d2973697 5121 err = page_counter_memparse(buf, "max", &max);
241994ed
JW
5122 if (err)
5123 return err;
5124
b6e6edcf
JW
5125 xchg(&memcg->memory.limit, max);
5126
5127 for (;;) {
5128 unsigned long nr_pages = page_counter_read(&memcg->memory);
5129
5130 if (nr_pages <= max)
5131 break;
5132
5133 if (signal_pending(current)) {
5134 err = -EINTR;
5135 break;
5136 }
5137
5138 if (!drained) {
5139 drain_all_stock(memcg);
5140 drained = true;
5141 continue;
5142 }
5143
5144 if (nr_reclaims) {
5145 if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
5146 GFP_KERNEL, true))
5147 nr_reclaims--;
5148 continue;
5149 }
5150
31176c78 5151 mem_cgroup_event(memcg, MEMCG_OOM);
b6e6edcf
JW
5152 if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
5153 break;
5154 }
241994ed 5155
2529bb3a 5156 memcg_wb_domain_size_changed(memcg);
241994ed
JW
5157 return nbytes;
5158}
5159
5160static int memory_events_show(struct seq_file *m, void *v)
5161{
5162 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5163
ccda7f43
JW
5164 seq_printf(m, "low %lu\n", memcg_sum_events(memcg, MEMCG_LOW));
5165 seq_printf(m, "high %lu\n", memcg_sum_events(memcg, MEMCG_HIGH));
5166 seq_printf(m, "max %lu\n", memcg_sum_events(memcg, MEMCG_MAX));
5167 seq_printf(m, "oom %lu\n", memcg_sum_events(memcg, MEMCG_OOM));
241994ed
JW
5168
5169 return 0;
5170}
5171
587d9f72
JW
5172static int memory_stat_show(struct seq_file *m, void *v)
5173{
5174 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
72b54e73
VD
5175 unsigned long stat[MEMCG_NR_STAT];
5176 unsigned long events[MEMCG_NR_EVENTS];
587d9f72
JW
5177 int i;
5178
5179 /*
5180 * Provide statistics on the state of the memory subsystem as
5181 * well as cumulative event counters that show past behavior.
5182 *
5183 * This list is ordered following a combination of these gradients:
5184 * 1) generic big picture -> specifics and details
5185 * 2) reflecting userspace activity -> reflecting kernel heuristics
5186 *
5187 * Current memory state:
5188 */
5189
72b54e73
VD
5190 tree_stat(memcg, stat);
5191 tree_events(memcg, events);
5192
587d9f72 5193 seq_printf(m, "anon %llu\n",
71cd3113 5194 (u64)stat[MEMCG_RSS] * PAGE_SIZE);
587d9f72 5195 seq_printf(m, "file %llu\n",
71cd3113 5196 (u64)stat[MEMCG_CACHE] * PAGE_SIZE);
12580e4b 5197 seq_printf(m, "kernel_stack %llu\n",
efdc9490 5198 (u64)stat[MEMCG_KERNEL_STACK_KB] * 1024);
27ee57c9
VD
5199 seq_printf(m, "slab %llu\n",
5200 (u64)(stat[MEMCG_SLAB_RECLAIMABLE] +
5201 stat[MEMCG_SLAB_UNRECLAIMABLE]) * PAGE_SIZE);
b2807f07 5202 seq_printf(m, "sock %llu\n",
72b54e73 5203 (u64)stat[MEMCG_SOCK] * PAGE_SIZE);
587d9f72 5204
9a4caf1e 5205 seq_printf(m, "shmem %llu\n",
71cd3113 5206 (u64)stat[NR_SHMEM] * PAGE_SIZE);
587d9f72 5207 seq_printf(m, "file_mapped %llu\n",
71cd3113 5208 (u64)stat[NR_FILE_MAPPED] * PAGE_SIZE);
587d9f72 5209 seq_printf(m, "file_dirty %llu\n",
71cd3113 5210 (u64)stat[NR_FILE_DIRTY] * PAGE_SIZE);
587d9f72 5211 seq_printf(m, "file_writeback %llu\n",
71cd3113 5212 (u64)stat[NR_WRITEBACK] * PAGE_SIZE);
587d9f72
JW
5213
5214 for (i = 0; i < NR_LRU_LISTS; i++) {
5215 struct mem_cgroup *mi;
5216 unsigned long val = 0;
5217
5218 for_each_mem_cgroup_tree(mi, memcg)
5219 val += mem_cgroup_nr_lru_pages(mi, BIT(i));
5220 seq_printf(m, "%s %llu\n",
5221 mem_cgroup_lru_names[i], (u64)val * PAGE_SIZE);
5222 }
5223
27ee57c9
VD
5224 seq_printf(m, "slab_reclaimable %llu\n",
5225 (u64)stat[MEMCG_SLAB_RECLAIMABLE] * PAGE_SIZE);
5226 seq_printf(m, "slab_unreclaimable %llu\n",
5227 (u64)stat[MEMCG_SLAB_UNRECLAIMABLE] * PAGE_SIZE);
5228
587d9f72
JW
5229 /* Accumulated memory events */
5230
df0e53d0
JW
5231 seq_printf(m, "pgfault %lu\n", events[PGFAULT]);
5232 seq_printf(m, "pgmajfault %lu\n", events[PGMAJFAULT]);
587d9f72 5233
2a2e4885 5234 seq_printf(m, "workingset_refault %lu\n",
71cd3113 5235 stat[WORKINGSET_REFAULT]);
2a2e4885 5236 seq_printf(m, "workingset_activate %lu\n",
71cd3113 5237 stat[WORKINGSET_ACTIVATE]);
2a2e4885 5238 seq_printf(m, "workingset_nodereclaim %lu\n",
71cd3113 5239 stat[WORKINGSET_NODERECLAIM]);
2a2e4885 5240
587d9f72
JW
5241 return 0;
5242}
5243
241994ed
JW
5244static struct cftype memory_files[] = {
5245 {
5246 .name = "current",
f5fc3c5d 5247 .flags = CFTYPE_NOT_ON_ROOT,
241994ed
JW
5248 .read_u64 = memory_current_read,
5249 },
5250 {
5251 .name = "low",
5252 .flags = CFTYPE_NOT_ON_ROOT,
5253 .seq_show = memory_low_show,
5254 .write = memory_low_write,
5255 },
5256 {
5257 .name = "high",
5258 .flags = CFTYPE_NOT_ON_ROOT,
5259 .seq_show = memory_high_show,
5260 .write = memory_high_write,
5261 },
5262 {
5263 .name = "max",
5264 .flags = CFTYPE_NOT_ON_ROOT,
5265 .seq_show = memory_max_show,
5266 .write = memory_max_write,
5267 },
5268 {
5269 .name = "events",
5270 .flags = CFTYPE_NOT_ON_ROOT,
472912a2 5271 .file_offset = offsetof(struct mem_cgroup, events_file),
241994ed
JW
5272 .seq_show = memory_events_show,
5273 },
587d9f72
JW
5274 {
5275 .name = "stat",
5276 .flags = CFTYPE_NOT_ON_ROOT,
5277 .seq_show = memory_stat_show,
5278 },
241994ed
JW
5279 { } /* terminate */
5280};
5281
073219e9 5282struct cgroup_subsys memory_cgrp_subsys = {
92fb9748 5283 .css_alloc = mem_cgroup_css_alloc,
d142e3e6 5284 .css_online = mem_cgroup_css_online,
92fb9748 5285 .css_offline = mem_cgroup_css_offline,
6df38689 5286 .css_released = mem_cgroup_css_released,
92fb9748 5287 .css_free = mem_cgroup_css_free,
1ced953b 5288 .css_reset = mem_cgroup_css_reset,
7dc74be0
DN
5289 .can_attach = mem_cgroup_can_attach,
5290 .cancel_attach = mem_cgroup_cancel_attach,
264a0ae1 5291 .post_attach = mem_cgroup_move_task,
f00baae7 5292 .bind = mem_cgroup_bind,
241994ed
JW
5293 .dfl_cftypes = memory_files,
5294 .legacy_cftypes = mem_cgroup_legacy_files,
6d12e2d8 5295 .early_init = 0,
8cdea7c0 5296};
c077719b 5297
241994ed
JW
5298/**
5299 * mem_cgroup_low - check if memory consumption is below the normal range
5300 * @root: the highest ancestor to consider
5301 * @memcg: the memory cgroup to check
5302 *
5303 * Returns %true if memory consumption of @memcg, and that of all
5304 * configurable ancestors up to @root, is below the normal range.
5305 */
5306bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg)
5307{
5308 if (mem_cgroup_disabled())
5309 return false;
5310
5311 /*
5312 * The toplevel group doesn't have a configurable range, so
5313 * it's never low when looked at directly, and it is not
5314 * considered an ancestor when assessing the hierarchy.
5315 */
5316
5317 if (memcg == root_mem_cgroup)
5318 return false;
5319
4e54dede 5320 if (page_counter_read(&memcg->memory) >= memcg->low)
241994ed
JW
5321 return false;
5322
5323 while (memcg != root) {
5324 memcg = parent_mem_cgroup(memcg);
5325
5326 if (memcg == root_mem_cgroup)
5327 break;
5328
4e54dede 5329 if (page_counter_read(&memcg->memory) >= memcg->low)
241994ed
JW
5330 return false;
5331 }
5332 return true;
5333}
5334
00501b53
JW
5335/**
5336 * mem_cgroup_try_charge - try charging a page
5337 * @page: page to charge
5338 * @mm: mm context of the victim
5339 * @gfp_mask: reclaim mode
5340 * @memcgp: charged memcg return
25843c2b 5341 * @compound: charge the page as compound or small page
00501b53
JW
5342 *
5343 * Try to charge @page to the memcg that @mm belongs to, reclaiming
5344 * pages according to @gfp_mask if necessary.
5345 *
5346 * Returns 0 on success, with *@memcgp pointing to the charged memcg.
5347 * Otherwise, an error code is returned.
5348 *
5349 * After page->mapping has been set up, the caller must finalize the
5350 * charge with mem_cgroup_commit_charge(). Or abort the transaction
5351 * with mem_cgroup_cancel_charge() in case page instantiation fails.
5352 */
5353int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
f627c2f5
KS
5354 gfp_t gfp_mask, struct mem_cgroup **memcgp,
5355 bool compound)
00501b53
JW
5356{
5357 struct mem_cgroup *memcg = NULL;
f627c2f5 5358 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
00501b53
JW
5359 int ret = 0;
5360
5361 if (mem_cgroup_disabled())
5362 goto out;
5363
5364 if (PageSwapCache(page)) {
00501b53
JW
5365 /*
5366 * Every swap fault against a single page tries to charge the
5367 * page, bail as early as possible. shmem_unuse() encounters
5368 * already charged pages, too. The USED bit is protected by
5369 * the page lock, which serializes swap cache removal, which
5370 * in turn serializes uncharging.
5371 */
e993d905 5372 VM_BUG_ON_PAGE(!PageLocked(page), page);
1306a85a 5373 if (page->mem_cgroup)
00501b53 5374 goto out;
e993d905 5375
37e84351 5376 if (do_swap_account) {
e993d905
VD
5377 swp_entry_t ent = { .val = page_private(page), };
5378 unsigned short id = lookup_swap_cgroup_id(ent);
5379
5380 rcu_read_lock();
5381 memcg = mem_cgroup_from_id(id);
5382 if (memcg && !css_tryget_online(&memcg->css))
5383 memcg = NULL;
5384 rcu_read_unlock();
5385 }
00501b53
JW
5386 }
5387
00501b53
JW
5388 if (!memcg)
5389 memcg = get_mem_cgroup_from_mm(mm);
5390
5391 ret = try_charge(memcg, gfp_mask, nr_pages);
5392
5393 css_put(&memcg->css);
00501b53
JW
5394out:
5395 *memcgp = memcg;
5396 return ret;
5397}
5398
5399/**
5400 * mem_cgroup_commit_charge - commit a page charge
5401 * @page: page to charge
5402 * @memcg: memcg to charge the page to
5403 * @lrucare: page might be on LRU already
25843c2b 5404 * @compound: charge the page as compound or small page
00501b53
JW
5405 *
5406 * Finalize a charge transaction started by mem_cgroup_try_charge(),
5407 * after page->mapping has been set up. This must happen atomically
5408 * as part of the page instantiation, i.e. under the page table lock
5409 * for anonymous pages, under the page lock for page and swap cache.
5410 *
5411 * In addition, the page must not be on the LRU during the commit, to
5412 * prevent racing with task migration. If it might be, use @lrucare.
5413 *
5414 * Use mem_cgroup_cancel_charge() to cancel the transaction instead.
5415 */
5416void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
f627c2f5 5417 bool lrucare, bool compound)
00501b53 5418{
f627c2f5 5419 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
00501b53
JW
5420
5421 VM_BUG_ON_PAGE(!page->mapping, page);
5422 VM_BUG_ON_PAGE(PageLRU(page) && !lrucare, page);
5423
5424 if (mem_cgroup_disabled())
5425 return;
5426 /*
5427 * Swap faults will attempt to charge the same page multiple
5428 * times. But reuse_swap_page() might have removed the page
5429 * from swapcache already, so we can't check PageSwapCache().
5430 */
5431 if (!memcg)
5432 return;
5433
6abb5a86
JW
5434 commit_charge(page, memcg, lrucare);
5435
6abb5a86 5436 local_irq_disable();
f627c2f5 5437 mem_cgroup_charge_statistics(memcg, page, compound, nr_pages);
6abb5a86
JW
5438 memcg_check_events(memcg, page);
5439 local_irq_enable();
00501b53 5440
7941d214 5441 if (do_memsw_account() && PageSwapCache(page)) {
00501b53
JW
5442 swp_entry_t entry = { .val = page_private(page) };
5443 /*
5444 * The swap entry might not get freed for a long time,
5445 * let's not wait for it. The page already received a
5446 * memory+swap charge, drop the swap entry duplicate.
5447 */
5448 mem_cgroup_uncharge_swap(entry);
5449 }
5450}
5451
5452/**
5453 * mem_cgroup_cancel_charge - cancel a page charge
5454 * @page: page to charge
5455 * @memcg: memcg to charge the page to
25843c2b 5456 * @compound: charge the page as compound or small page
00501b53
JW
5457 *
5458 * Cancel a charge transaction started by mem_cgroup_try_charge().
5459 */
f627c2f5
KS
5460void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
5461 bool compound)
00501b53 5462{
f627c2f5 5463 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
00501b53
JW
5464
5465 if (mem_cgroup_disabled())
5466 return;
5467 /*
5468 * Swap faults will attempt to charge the same page multiple
5469 * times. But reuse_swap_page() might have removed the page
5470 * from swapcache already, so we can't check PageSwapCache().
5471 */
5472 if (!memcg)
5473 return;
5474
00501b53
JW
5475 cancel_charge(memcg, nr_pages);
5476}
5477
747db954 5478static void uncharge_batch(struct mem_cgroup *memcg, unsigned long pgpgout,
747db954 5479 unsigned long nr_anon, unsigned long nr_file,
9a4caf1e
JW
5480 unsigned long nr_kmem, unsigned long nr_huge,
5481 unsigned long nr_shmem, struct page *dummy_page)
747db954 5482{
5e8d35f8 5483 unsigned long nr_pages = nr_anon + nr_file + nr_kmem;
747db954
JW
5484 unsigned long flags;
5485
ce00a967 5486 if (!mem_cgroup_is_root(memcg)) {
18eca2e6 5487 page_counter_uncharge(&memcg->memory, nr_pages);
7941d214 5488 if (do_memsw_account())
18eca2e6 5489 page_counter_uncharge(&memcg->memsw, nr_pages);
5e8d35f8
VD
5490 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && nr_kmem)
5491 page_counter_uncharge(&memcg->kmem, nr_kmem);
ce00a967
JW
5492 memcg_oom_recover(memcg);
5493 }
747db954
JW
5494
5495 local_irq_save(flags);
71cd3113
JW
5496 __this_cpu_sub(memcg->stat->count[MEMCG_RSS], nr_anon);
5497 __this_cpu_sub(memcg->stat->count[MEMCG_CACHE], nr_file);
5498 __this_cpu_sub(memcg->stat->count[MEMCG_RSS_HUGE], nr_huge);
5499 __this_cpu_sub(memcg->stat->count[NR_SHMEM], nr_shmem);
df0e53d0 5500 __this_cpu_add(memcg->stat->events[PGPGOUT], pgpgout);
18eca2e6 5501 __this_cpu_add(memcg->stat->nr_page_events, nr_pages);
747db954
JW
5502 memcg_check_events(memcg, dummy_page);
5503 local_irq_restore(flags);
e8ea14cc
JW
5504
5505 if (!mem_cgroup_is_root(memcg))
18eca2e6 5506 css_put_many(&memcg->css, nr_pages);
747db954
JW
5507}
5508
5509static void uncharge_list(struct list_head *page_list)
5510{
5511 struct mem_cgroup *memcg = NULL;
9a4caf1e 5512 unsigned long nr_shmem = 0;
747db954
JW
5513 unsigned long nr_anon = 0;
5514 unsigned long nr_file = 0;
5515 unsigned long nr_huge = 0;
5e8d35f8 5516 unsigned long nr_kmem = 0;
747db954 5517 unsigned long pgpgout = 0;
747db954
JW
5518 struct list_head *next;
5519 struct page *page;
5520
8b592656
JW
5521 /*
5522 * Note that the list can be a single page->lru; hence the
5523 * do-while loop instead of a simple list_for_each_entry().
5524 */
747db954
JW
5525 next = page_list->next;
5526 do {
747db954
JW
5527 page = list_entry(next, struct page, lru);
5528 next = page->lru.next;
5529
5530 VM_BUG_ON_PAGE(PageLRU(page), page);
5531 VM_BUG_ON_PAGE(page_count(page), page);
5532
1306a85a 5533 if (!page->mem_cgroup)
747db954
JW
5534 continue;
5535
5536 /*
5537 * Nobody should be changing or seriously looking at
1306a85a 5538 * page->mem_cgroup at this point, we have fully
29833315 5539 * exclusive access to the page.
747db954
JW
5540 */
5541
1306a85a 5542 if (memcg != page->mem_cgroup) {
747db954 5543 if (memcg) {
18eca2e6 5544 uncharge_batch(memcg, pgpgout, nr_anon, nr_file,
9a4caf1e
JW
5545 nr_kmem, nr_huge, nr_shmem, page);
5546 pgpgout = nr_anon = nr_file = nr_kmem = 0;
5547 nr_huge = nr_shmem = 0;
747db954 5548 }
1306a85a 5549 memcg = page->mem_cgroup;
747db954
JW
5550 }
5551
5e8d35f8
VD
5552 if (!PageKmemcg(page)) {
5553 unsigned int nr_pages = 1;
747db954 5554
5e8d35f8
VD
5555 if (PageTransHuge(page)) {
5556 nr_pages <<= compound_order(page);
5e8d35f8
VD
5557 nr_huge += nr_pages;
5558 }
5559 if (PageAnon(page))
5560 nr_anon += nr_pages;
9a4caf1e 5561 else {
5e8d35f8 5562 nr_file += nr_pages;
9a4caf1e
JW
5563 if (PageSwapBacked(page))
5564 nr_shmem += nr_pages;
5565 }
5e8d35f8 5566 pgpgout++;
c4159a75 5567 } else {
5e8d35f8 5568 nr_kmem += 1 << compound_order(page);
c4159a75
VD
5569 __ClearPageKmemcg(page);
5570 }
747db954 5571
1306a85a 5572 page->mem_cgroup = NULL;
747db954
JW
5573 } while (next != page_list);
5574
5575 if (memcg)
18eca2e6 5576 uncharge_batch(memcg, pgpgout, nr_anon, nr_file,
9a4caf1e 5577 nr_kmem, nr_huge, nr_shmem, page);
747db954
JW
5578}
5579
0a31bc97
JW
5580/**
5581 * mem_cgroup_uncharge - uncharge a page
5582 * @page: page to uncharge
5583 *
5584 * Uncharge a page previously charged with mem_cgroup_try_charge() and
5585 * mem_cgroup_commit_charge().
5586 */
5587void mem_cgroup_uncharge(struct page *page)
5588{
0a31bc97
JW
5589 if (mem_cgroup_disabled())
5590 return;
5591
747db954 5592 /* Don't touch page->lru of any random page, pre-check: */
1306a85a 5593 if (!page->mem_cgroup)
0a31bc97
JW
5594 return;
5595
747db954
JW
5596 INIT_LIST_HEAD(&page->lru);
5597 uncharge_list(&page->lru);
5598}
0a31bc97 5599
747db954
JW
5600/**
5601 * mem_cgroup_uncharge_list - uncharge a list of page
5602 * @page_list: list of pages to uncharge
5603 *
5604 * Uncharge a list of pages previously charged with
5605 * mem_cgroup_try_charge() and mem_cgroup_commit_charge().
5606 */
5607void mem_cgroup_uncharge_list(struct list_head *page_list)
5608{
5609 if (mem_cgroup_disabled())
5610 return;
0a31bc97 5611
747db954
JW
5612 if (!list_empty(page_list))
5613 uncharge_list(page_list);
0a31bc97
JW
5614}
5615
5616/**
6a93ca8f
JW
5617 * mem_cgroup_migrate - charge a page's replacement
5618 * @oldpage: currently circulating page
5619 * @newpage: replacement page
0a31bc97 5620 *
6a93ca8f
JW
5621 * Charge @newpage as a replacement page for @oldpage. @oldpage will
5622 * be uncharged upon free.
0a31bc97
JW
5623 *
5624 * Both pages must be locked, @newpage->mapping must be set up.
5625 */
6a93ca8f 5626void mem_cgroup_migrate(struct page *oldpage, struct page *newpage)
0a31bc97 5627{
29833315 5628 struct mem_cgroup *memcg;
44b7a8d3
JW
5629 unsigned int nr_pages;
5630 bool compound;
d93c4130 5631 unsigned long flags;
0a31bc97
JW
5632
5633 VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
5634 VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
0a31bc97 5635 VM_BUG_ON_PAGE(PageAnon(oldpage) != PageAnon(newpage), newpage);
6abb5a86
JW
5636 VM_BUG_ON_PAGE(PageTransHuge(oldpage) != PageTransHuge(newpage),
5637 newpage);
0a31bc97
JW
5638
5639 if (mem_cgroup_disabled())
5640 return;
5641
5642 /* Page cache replacement: new page already charged? */
1306a85a 5643 if (newpage->mem_cgroup)
0a31bc97
JW
5644 return;
5645
45637bab 5646 /* Swapcache readahead pages can get replaced before being charged */
1306a85a 5647 memcg = oldpage->mem_cgroup;
29833315 5648 if (!memcg)
0a31bc97
JW
5649 return;
5650
44b7a8d3
JW
5651 /* Force-charge the new page. The old one will be freed soon */
5652 compound = PageTransHuge(newpage);
5653 nr_pages = compound ? hpage_nr_pages(newpage) : 1;
5654
5655 page_counter_charge(&memcg->memory, nr_pages);
5656 if (do_memsw_account())
5657 page_counter_charge(&memcg->memsw, nr_pages);
5658 css_get_many(&memcg->css, nr_pages);
0a31bc97 5659
9cf7666a 5660 commit_charge(newpage, memcg, false);
44b7a8d3 5661
d93c4130 5662 local_irq_save(flags);
44b7a8d3
JW
5663 mem_cgroup_charge_statistics(memcg, newpage, compound, nr_pages);
5664 memcg_check_events(memcg, newpage);
d93c4130 5665 local_irq_restore(flags);
0a31bc97
JW
5666}
5667
ef12947c 5668DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);
11092087
JW
5669EXPORT_SYMBOL(memcg_sockets_enabled_key);
5670
2d758073 5671void mem_cgroup_sk_alloc(struct sock *sk)
11092087
JW
5672{
5673 struct mem_cgroup *memcg;
5674
2d758073
JW
5675 if (!mem_cgroup_sockets_enabled)
5676 return;
5677
5678 /*
5679 * Socket cloning can throw us here with sk_memcg already
11092087
JW
5680 * filled. It won't however, necessarily happen from
5681 * process context. So the test for root memcg given
5682 * the current task's memcg won't help us in this case.
5683 *
5684 * Respecting the original socket's memcg is a better
5685 * decision in this case.
5686 */
5687 if (sk->sk_memcg) {
5688 BUG_ON(mem_cgroup_is_root(sk->sk_memcg));
5689 css_get(&sk->sk_memcg->css);
5690 return;
5691 }
5692
5693 rcu_read_lock();
5694 memcg = mem_cgroup_from_task(current);
f7e1cb6e
JW
5695 if (memcg == root_mem_cgroup)
5696 goto out;
0db15298 5697 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)
f7e1cb6e 5698 goto out;
f7e1cb6e 5699 if (css_tryget_online(&memcg->css))
11092087 5700 sk->sk_memcg = memcg;
f7e1cb6e 5701out:
11092087
JW
5702 rcu_read_unlock();
5703}
11092087 5704
2d758073 5705void mem_cgroup_sk_free(struct sock *sk)
11092087 5706{
2d758073
JW
5707 if (sk->sk_memcg)
5708 css_put(&sk->sk_memcg->css);
11092087
JW
5709}
5710
5711/**
5712 * mem_cgroup_charge_skmem - charge socket memory
5713 * @memcg: memcg to charge
5714 * @nr_pages: number of pages to charge
5715 *
5716 * Charges @nr_pages to @memcg. Returns %true if the charge fit within
5717 * @memcg's configured limit, %false if the charge had to be forced.
5718 */
5719bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
5720{
f7e1cb6e 5721 gfp_t gfp_mask = GFP_KERNEL;
11092087 5722
f7e1cb6e 5723 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
0db15298 5724 struct page_counter *fail;
f7e1cb6e 5725
0db15298
JW
5726 if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
5727 memcg->tcpmem_pressure = 0;
f7e1cb6e
JW
5728 return true;
5729 }
0db15298
JW
5730 page_counter_charge(&memcg->tcpmem, nr_pages);
5731 memcg->tcpmem_pressure = 1;
f7e1cb6e 5732 return false;
11092087 5733 }
d886f4e4 5734
f7e1cb6e
JW
5735 /* Don't block in the packet receive path */
5736 if (in_softirq())
5737 gfp_mask = GFP_NOWAIT;
5738
b2807f07
JW
5739 this_cpu_add(memcg->stat->count[MEMCG_SOCK], nr_pages);
5740
f7e1cb6e
JW
5741 if (try_charge(memcg, gfp_mask, nr_pages) == 0)
5742 return true;
5743
5744 try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages);
11092087
JW
5745 return false;
5746}
5747
5748/**
5749 * mem_cgroup_uncharge_skmem - uncharge socket memory
5750 * @memcg - memcg to uncharge
5751 * @nr_pages - number of pages to uncharge
5752 */
5753void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
5754{
f7e1cb6e 5755 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
0db15298 5756 page_counter_uncharge(&memcg->tcpmem, nr_pages);
f7e1cb6e
JW
5757 return;
5758 }
d886f4e4 5759
b2807f07
JW
5760 this_cpu_sub(memcg->stat->count[MEMCG_SOCK], nr_pages);
5761
f7e1cb6e
JW
5762 page_counter_uncharge(&memcg->memory, nr_pages);
5763 css_put_many(&memcg->css, nr_pages);
11092087
JW
5764}
5765
f7e1cb6e
JW
5766static int __init cgroup_memory(char *s)
5767{
5768 char *token;
5769
5770 while ((token = strsep(&s, ",")) != NULL) {
5771 if (!*token)
5772 continue;
5773 if (!strcmp(token, "nosocket"))
5774 cgroup_memory_nosocket = true;
04823c83
VD
5775 if (!strcmp(token, "nokmem"))
5776 cgroup_memory_nokmem = true;
f7e1cb6e
JW
5777 }
5778 return 0;
5779}
5780__setup("cgroup.memory=", cgroup_memory);
11092087 5781
2d11085e 5782/*
1081312f
MH
5783 * subsys_initcall() for memory controller.
5784 *
308167fc
SAS
5785 * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this
5786 * context because of lock dependencies (cgroup_lock -> cpu hotplug) but
5787 * basically everything that doesn't depend on a specific mem_cgroup structure
5788 * should be initialized from here.
2d11085e
MH
5789 */
5790static int __init mem_cgroup_init(void)
5791{
95a045f6
JW
5792 int cpu, node;
5793
13583c3d
VD
5794#ifndef CONFIG_SLOB
5795 /*
5796 * Kmem cache creation is mostly done with the slab_mutex held,
17cc4dfe
TH
5797 * so use a workqueue with limited concurrency to avoid stalling
5798 * all worker threads in case lots of cgroups are created and
5799 * destroyed simultaneously.
13583c3d 5800 */
17cc4dfe
TH
5801 memcg_kmem_cache_wq = alloc_workqueue("memcg_kmem_cache", 0, 1);
5802 BUG_ON(!memcg_kmem_cache_wq);
13583c3d
VD
5803#endif
5804
308167fc
SAS
5805 cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL,
5806 memcg_hotplug_cpu_dead);
95a045f6
JW
5807
5808 for_each_possible_cpu(cpu)
5809 INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
5810 drain_local_stock);
5811
5812 for_each_node(node) {
5813 struct mem_cgroup_tree_per_node *rtpn;
95a045f6
JW
5814
5815 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL,
5816 node_online(node) ? node : NUMA_NO_NODE);
5817
ef8f2327
MG
5818 rtpn->rb_root = RB_ROOT;
5819 spin_lock_init(&rtpn->lock);
95a045f6
JW
5820 soft_limit_tree.rb_tree_per_node[node] = rtpn;
5821 }
5822
2d11085e
MH
5823 return 0;
5824}
5825subsys_initcall(mem_cgroup_init);
21afa38e
JW
5826
5827#ifdef CONFIG_MEMCG_SWAP
358c07fc
AB
5828static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg)
5829{
5830 while (!atomic_inc_not_zero(&memcg->id.ref)) {
5831 /*
5832 * The root cgroup cannot be destroyed, so it's refcount must
5833 * always be >= 1.
5834 */
5835 if (WARN_ON_ONCE(memcg == root_mem_cgroup)) {
5836 VM_BUG_ON(1);
5837 break;
5838 }
5839 memcg = parent_mem_cgroup(memcg);
5840 if (!memcg)
5841 memcg = root_mem_cgroup;
5842 }
5843 return memcg;
5844}
5845
21afa38e
JW
5846/**
5847 * mem_cgroup_swapout - transfer a memsw charge to swap
5848 * @page: page whose memsw charge to transfer
5849 * @entry: swap entry to move the charge to
5850 *
5851 * Transfer the memsw charge of @page to @entry.
5852 */
5853void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
5854{
1f47b61f 5855 struct mem_cgroup *memcg, *swap_memcg;
21afa38e
JW
5856 unsigned short oldid;
5857
5858 VM_BUG_ON_PAGE(PageLRU(page), page);
5859 VM_BUG_ON_PAGE(page_count(page), page);
5860
7941d214 5861 if (!do_memsw_account())
21afa38e
JW
5862 return;
5863
5864 memcg = page->mem_cgroup;
5865
5866 /* Readahead page, never charged */
5867 if (!memcg)
5868 return;
5869
1f47b61f
VD
5870 /*
5871 * In case the memcg owning these pages has been offlined and doesn't
5872 * have an ID allocated to it anymore, charge the closest online
5873 * ancestor for the swap instead and transfer the memory+swap charge.
5874 */
5875 swap_memcg = mem_cgroup_id_get_online(memcg);
5876 oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg));
21afa38e 5877 VM_BUG_ON_PAGE(oldid, page);
1f47b61f 5878 mem_cgroup_swap_statistics(swap_memcg, true);
21afa38e
JW
5879
5880 page->mem_cgroup = NULL;
5881
5882 if (!mem_cgroup_is_root(memcg))
5883 page_counter_uncharge(&memcg->memory, 1);
5884
1f47b61f
VD
5885 if (memcg != swap_memcg) {
5886 if (!mem_cgroup_is_root(swap_memcg))
5887 page_counter_charge(&swap_memcg->memsw, 1);
5888 page_counter_uncharge(&memcg->memsw, 1);
5889 }
5890
ce9ce665
SAS
5891 /*
5892 * Interrupts should be disabled here because the caller holds the
5893 * mapping->tree_lock lock which is taken with interrupts-off. It is
5894 * important here to have the interrupts disabled because it is the
5895 * only synchronisation we have for udpating the per-CPU variables.
5896 */
5897 VM_BUG_ON(!irqs_disabled());
f627c2f5 5898 mem_cgroup_charge_statistics(memcg, page, false, -1);
21afa38e 5899 memcg_check_events(memcg, page);
73f576c0
JW
5900
5901 if (!mem_cgroup_is_root(memcg))
5902 css_put(&memcg->css);
21afa38e
JW
5903}
5904
37e84351
VD
5905/*
5906 * mem_cgroup_try_charge_swap - try charging a swap entry
5907 * @page: page being added to swap
5908 * @entry: swap entry to charge
5909 *
5910 * Try to charge @entry to the memcg that @page belongs to.
5911 *
5912 * Returns 0 on success, -ENOMEM on failure.
5913 */
5914int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
5915{
5916 struct mem_cgroup *memcg;
5917 struct page_counter *counter;
5918 unsigned short oldid;
5919
5920 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) || !do_swap_account)
5921 return 0;
5922
5923 memcg = page->mem_cgroup;
5924
5925 /* Readahead page, never charged */
5926 if (!memcg)
5927 return 0;
5928
1f47b61f
VD
5929 memcg = mem_cgroup_id_get_online(memcg);
5930
37e84351 5931 if (!mem_cgroup_is_root(memcg) &&
1f47b61f
VD
5932 !page_counter_try_charge(&memcg->swap, 1, &counter)) {
5933 mem_cgroup_id_put(memcg);
37e84351 5934 return -ENOMEM;
1f47b61f 5935 }
37e84351
VD
5936
5937 oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg));
5938 VM_BUG_ON_PAGE(oldid, page);
5939 mem_cgroup_swap_statistics(memcg, true);
5940
37e84351
VD
5941 return 0;
5942}
5943
21afa38e
JW
5944/**
5945 * mem_cgroup_uncharge_swap - uncharge a swap entry
5946 * @entry: swap entry to uncharge
5947 *
37e84351 5948 * Drop the swap charge associated with @entry.
21afa38e
JW
5949 */
5950void mem_cgroup_uncharge_swap(swp_entry_t entry)
5951{
5952 struct mem_cgroup *memcg;
5953 unsigned short id;
5954
37e84351 5955 if (!do_swap_account)
21afa38e
JW
5956 return;
5957
5958 id = swap_cgroup_record(entry, 0);
5959 rcu_read_lock();
adbe427b 5960 memcg = mem_cgroup_from_id(id);
21afa38e 5961 if (memcg) {
37e84351
VD
5962 if (!mem_cgroup_is_root(memcg)) {
5963 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
5964 page_counter_uncharge(&memcg->swap, 1);
5965 else
5966 page_counter_uncharge(&memcg->memsw, 1);
5967 }
21afa38e 5968 mem_cgroup_swap_statistics(memcg, false);
73f576c0 5969 mem_cgroup_id_put(memcg);
21afa38e
JW
5970 }
5971 rcu_read_unlock();
5972}
5973
d8b38438
VD
5974long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
5975{
5976 long nr_swap_pages = get_nr_swap_pages();
5977
5978 if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
5979 return nr_swap_pages;
5980 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
5981 nr_swap_pages = min_t(long, nr_swap_pages,
5982 READ_ONCE(memcg->swap.limit) -
5983 page_counter_read(&memcg->swap));
5984 return nr_swap_pages;
5985}
5986
5ccc5aba
VD
5987bool mem_cgroup_swap_full(struct page *page)
5988{
5989 struct mem_cgroup *memcg;
5990
5991 VM_BUG_ON_PAGE(!PageLocked(page), page);
5992
5993 if (vm_swap_full())
5994 return true;
5995 if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
5996 return false;
5997
5998 memcg = page->mem_cgroup;
5999 if (!memcg)
6000 return false;
6001
6002 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
6003 if (page_counter_read(&memcg->swap) * 2 >= memcg->swap.limit)
6004 return true;
6005
6006 return false;
6007}
6008
21afa38e
JW
6009/* for remember boot option*/
6010#ifdef CONFIG_MEMCG_SWAP_ENABLED
6011static int really_do_swap_account __initdata = 1;
6012#else
6013static int really_do_swap_account __initdata;
6014#endif
6015
6016static int __init enable_swap_account(char *s)
6017{
6018 if (!strcmp(s, "1"))
6019 really_do_swap_account = 1;
6020 else if (!strcmp(s, "0"))
6021 really_do_swap_account = 0;
6022 return 1;
6023}
6024__setup("swapaccount=", enable_swap_account);
6025
37e84351
VD
6026static u64 swap_current_read(struct cgroup_subsys_state *css,
6027 struct cftype *cft)
6028{
6029 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6030
6031 return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
6032}
6033
6034static int swap_max_show(struct seq_file *m, void *v)
6035{
6036 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
6037 unsigned long max = READ_ONCE(memcg->swap.limit);
6038
6039 if (max == PAGE_COUNTER_MAX)
6040 seq_puts(m, "max\n");
6041 else
6042 seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE);
6043
6044 return 0;
6045}
6046
6047static ssize_t swap_max_write(struct kernfs_open_file *of,
6048 char *buf, size_t nbytes, loff_t off)
6049{
6050 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6051 unsigned long max;
6052 int err;
6053
6054 buf = strstrip(buf);
6055 err = page_counter_memparse(buf, "max", &max);
6056 if (err)
6057 return err;
6058
6059 mutex_lock(&memcg_limit_mutex);
6060 err = page_counter_limit(&memcg->swap, max);
6061 mutex_unlock(&memcg_limit_mutex);
6062 if (err)
6063 return err;
6064
6065 return nbytes;
6066}
6067
6068static struct cftype swap_files[] = {
6069 {
6070 .name = "swap.current",
6071 .flags = CFTYPE_NOT_ON_ROOT,
6072 .read_u64 = swap_current_read,
6073 },
6074 {
6075 .name = "swap.max",
6076 .flags = CFTYPE_NOT_ON_ROOT,
6077 .seq_show = swap_max_show,
6078 .write = swap_max_write,
6079 },
6080 { } /* terminate */
6081};
6082
21afa38e
JW
6083static struct cftype memsw_cgroup_files[] = {
6084 {
6085 .name = "memsw.usage_in_bytes",
6086 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
6087 .read_u64 = mem_cgroup_read_u64,
6088 },
6089 {
6090 .name = "memsw.max_usage_in_bytes",
6091 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
6092 .write = mem_cgroup_reset,
6093 .read_u64 = mem_cgroup_read_u64,
6094 },
6095 {
6096 .name = "memsw.limit_in_bytes",
6097 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
6098 .write = mem_cgroup_write,
6099 .read_u64 = mem_cgroup_read_u64,
6100 },
6101 {
6102 .name = "memsw.failcnt",
6103 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
6104 .write = mem_cgroup_reset,
6105 .read_u64 = mem_cgroup_read_u64,
6106 },
6107 { }, /* terminate */
6108};
6109
6110static int __init mem_cgroup_swap_init(void)
6111{
6112 if (!mem_cgroup_disabled() && really_do_swap_account) {
6113 do_swap_account = 1;
37e84351
VD
6114 WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys,
6115 swap_files));
21afa38e
JW
6116 WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys,
6117 memsw_cgroup_files));
6118 }
6119 return 0;
6120}
6121subsys_initcall(mem_cgroup_swap_init);
6122
6123#endif /* CONFIG_MEMCG_SWAP */