]>
git.proxmox.com Git - mirror_ubuntu-kernels.git/blob - mm/page_counter.c
1 // SPDX-License-Identifier: GPL-2.0
3 * Lockless hierarchical page accounting & limiting
5 * Copyright (C) 2014 Red Hat, Inc., Johannes Weiner
8 #include <linux/page_counter.h>
9 #include <linux/atomic.h>
10 #include <linux/kernel.h>
11 #include <linux/string.h>
12 #include <linux/sched.h>
13 #include <linux/bug.h>
16 static void propagate_protected_usage(struct page_counter
*c
,
19 unsigned long protected, old_protected
;
20 unsigned long low
, min
;
26 min
= READ_ONCE(c
->min
);
27 if (min
|| atomic_long_read(&c
->min_usage
)) {
28 protected = min(usage
, min
);
29 old_protected
= atomic_long_xchg(&c
->min_usage
, protected);
30 delta
= protected - old_protected
;
32 atomic_long_add(delta
, &c
->parent
->children_min_usage
);
35 low
= READ_ONCE(c
->low
);
36 if (low
|| atomic_long_read(&c
->low_usage
)) {
37 protected = min(usage
, low
);
38 old_protected
= atomic_long_xchg(&c
->low_usage
, protected);
39 delta
= protected - old_protected
;
41 atomic_long_add(delta
, &c
->parent
->children_low_usage
);
46 * page_counter_cancel - take pages out of the local counter
48 * @nr_pages: number of pages to cancel
50 void page_counter_cancel(struct page_counter
*counter
, unsigned long nr_pages
)
54 new = atomic_long_sub_return(nr_pages
, &counter
->usage
);
55 /* More uncharges than charges? */
56 if (WARN_ONCE(new < 0, "page_counter underflow: %ld nr_pages=%lu\n",
59 atomic_long_set(&counter
->usage
, new);
61 propagate_protected_usage(counter
, new);
65 * page_counter_charge - hierarchically charge pages
67 * @nr_pages: number of pages to charge
69 * NOTE: This does not consider any configured counter limits.
71 void page_counter_charge(struct page_counter
*counter
, unsigned long nr_pages
)
73 struct page_counter
*c
;
75 for (c
= counter
; c
; c
= c
->parent
) {
78 new = atomic_long_add_return(nr_pages
, &c
->usage
);
79 propagate_protected_usage(c
, new);
81 * This is indeed racy, but we can live with some
82 * inaccuracy in the watermark.
84 if (new > READ_ONCE(c
->watermark
))
85 WRITE_ONCE(c
->watermark
, new);
90 * page_counter_try_charge - try to hierarchically charge pages
92 * @nr_pages: number of pages to charge
93 * @fail: points first counter to hit its limit, if any
95 * Returns %true on success, or %false and @fail if the counter or one
96 * of its ancestors has hit its configured limit.
98 bool page_counter_try_charge(struct page_counter
*counter
,
99 unsigned long nr_pages
,
100 struct page_counter
**fail
)
102 struct page_counter
*c
;
104 for (c
= counter
; c
; c
= c
->parent
) {
107 * Charge speculatively to avoid an expensive CAS. If
108 * a bigger charge fails, it might falsely lock out a
109 * racing smaller charge and send it into reclaim
110 * early, but the error is limited to the difference
111 * between the two sizes, which is less than 2M/4M in
112 * case of a THP locking out a regular page charge.
114 * The atomic_long_add_return() implies a full memory
115 * barrier between incrementing the count and reading
116 * the limit. When racing with page_counter_set_max(),
117 * we either see the new limit or the setter sees the
118 * counter has changed and retries.
120 new = atomic_long_add_return(nr_pages
, &c
->usage
);
122 atomic_long_sub(nr_pages
, &c
->usage
);
124 * This is racy, but we can live with some
125 * inaccuracy in the failcnt which is only used
128 data_race(c
->failcnt
++);
132 propagate_protected_usage(c
, new);
134 * Just like with failcnt, we can live with some
135 * inaccuracy in the watermark.
137 if (new > READ_ONCE(c
->watermark
))
138 WRITE_ONCE(c
->watermark
, new);
143 for (c
= counter
; c
!= *fail
; c
= c
->parent
)
144 page_counter_cancel(c
, nr_pages
);
150 * page_counter_uncharge - hierarchically uncharge pages
152 * @nr_pages: number of pages to uncharge
154 void page_counter_uncharge(struct page_counter
*counter
, unsigned long nr_pages
)
156 struct page_counter
*c
;
158 for (c
= counter
; c
; c
= c
->parent
)
159 page_counter_cancel(c
, nr_pages
);
163 * page_counter_set_max - set the maximum number of pages allowed
165 * @nr_pages: limit to set
167 * Returns 0 on success, -EBUSY if the current number of pages on the
168 * counter already exceeds the specified limit.
170 * The caller must serialize invocations on the same counter.
172 int page_counter_set_max(struct page_counter
*counter
, unsigned long nr_pages
)
179 * Update the limit while making sure that it's not
180 * below the concurrently-changing counter value.
182 * The xchg implies two full memory barriers before
183 * and after, so the read-swap-read is ordered and
184 * ensures coherency with page_counter_try_charge():
185 * that function modifies the count before checking
186 * the limit, so if it sees the old limit, we see the
187 * modified counter and retry.
189 usage
= page_counter_read(counter
);
191 if (usage
> nr_pages
)
194 old
= xchg(&counter
->max
, nr_pages
);
196 if (page_counter_read(counter
) <= usage
)
205 * page_counter_set_min - set the amount of protected memory
207 * @nr_pages: value to set
209 * The caller must serialize invocations on the same counter.
211 void page_counter_set_min(struct page_counter
*counter
, unsigned long nr_pages
)
213 struct page_counter
*c
;
215 WRITE_ONCE(counter
->min
, nr_pages
);
217 for (c
= counter
; c
; c
= c
->parent
)
218 propagate_protected_usage(c
, atomic_long_read(&c
->usage
));
222 * page_counter_set_low - set the amount of protected memory
224 * @nr_pages: value to set
226 * The caller must serialize invocations on the same counter.
228 void page_counter_set_low(struct page_counter
*counter
, unsigned long nr_pages
)
230 struct page_counter
*c
;
232 WRITE_ONCE(counter
->low
, nr_pages
);
234 for (c
= counter
; c
; c
= c
->parent
)
235 propagate_protected_usage(c
, atomic_long_read(&c
->usage
));
239 * page_counter_memparse - memparse() for page counter limits
240 * @buf: string to parse
241 * @max: string meaning maximum possible value
242 * @nr_pages: returns the result in number of pages
244 * Returns -EINVAL, or 0 and @nr_pages on success. @nr_pages will be
245 * limited to %PAGE_COUNTER_MAX.
247 int page_counter_memparse(const char *buf
, const char *max
,
248 unsigned long *nr_pages
)
253 if (!strcmp(buf
, max
)) {
254 *nr_pages
= PAGE_COUNTER_MAX
;
258 bytes
= memparse(buf
, &end
);
262 *nr_pages
= min(bytes
/ PAGE_SIZE
, (u64
)PAGE_COUNTER_MAX
);