]>
Commit | Line | Data |
---|---|---|
3e32cb2e JW |
1 | /* |
2 | * Lockless hierarchical page accounting & limiting | |
3 | * | |
4 | * Copyright (C) 2014 Red Hat, Inc., Johannes Weiner | |
5 | */ | |
6 | ||
7 | #include <linux/page_counter.h> | |
8 | #include <linux/atomic.h> | |
9 | #include <linux/kernel.h> | |
10 | #include <linux/string.h> | |
11 | #include <linux/sched.h> | |
12 | #include <linux/bug.h> | |
13 | #include <asm/page.h> | |
14 | ||
15 | /** | |
16 | * page_counter_cancel - take pages out of the local counter | |
17 | * @counter: counter | |
18 | * @nr_pages: number of pages to cancel | |
19 | * | |
20 | * Returns whether there are remaining pages in the counter. | |
21 | */ | |
22 | int page_counter_cancel(struct page_counter *counter, unsigned long nr_pages) | |
23 | { | |
24 | long new; | |
25 | ||
26 | new = atomic_long_sub_return(nr_pages, &counter->count); | |
27 | ||
28 | /* More uncharges than charges? */ | |
29 | WARN_ON_ONCE(new < 0); | |
30 | ||
31 | return new > 0; | |
32 | } | |
33 | ||
34 | /** | |
35 | * page_counter_charge - hierarchically charge pages | |
36 | * @counter: counter | |
37 | * @nr_pages: number of pages to charge | |
38 | * | |
39 | * NOTE: This does not consider any configured counter limits. | |
40 | */ | |
41 | void page_counter_charge(struct page_counter *counter, unsigned long nr_pages) | |
42 | { | |
43 | struct page_counter *c; | |
44 | ||
45 | for (c = counter; c; c = c->parent) { | |
46 | long new; | |
47 | ||
48 | new = atomic_long_add_return(nr_pages, &c->count); | |
49 | /* | |
50 | * This is indeed racy, but we can live with some | |
51 | * inaccuracy in the watermark. | |
52 | */ | |
53 | if (new > c->watermark) | |
54 | c->watermark = new; | |
55 | } | |
56 | } | |
57 | ||
58 | /** | |
59 | * page_counter_try_charge - try to hierarchically charge pages | |
60 | * @counter: counter | |
61 | * @nr_pages: number of pages to charge | |
62 | * @fail: points first counter to hit its limit, if any | |
63 | * | |
64 | * Returns 0 on success, or -ENOMEM and @fail if the counter or one of | |
65 | * its ancestors has hit its configured limit. | |
66 | */ | |
67 | int page_counter_try_charge(struct page_counter *counter, | |
68 | unsigned long nr_pages, | |
69 | struct page_counter **fail) | |
70 | { | |
71 | struct page_counter *c; | |
72 | ||
73 | for (c = counter; c; c = c->parent) { | |
74 | long new; | |
75 | /* | |
76 | * Charge speculatively to avoid an expensive CAS. If | |
77 | * a bigger charge fails, it might falsely lock out a | |
78 | * racing smaller charge and send it into reclaim | |
79 | * early, but the error is limited to the difference | |
80 | * between the two sizes, which is less than 2M/4M in | |
81 | * case of a THP locking out a regular page charge. | |
82 | * | |
83 | * The atomic_long_add_return() implies a full memory | |
84 | * barrier between incrementing the count and reading | |
85 | * the limit. When racing with page_counter_limit(), | |
86 | * we either see the new limit or the setter sees the | |
87 | * counter has changed and retries. | |
88 | */ | |
89 | new = atomic_long_add_return(nr_pages, &c->count); | |
90 | if (new > c->limit) { | |
91 | atomic_long_sub(nr_pages, &c->count); | |
92 | /* | |
93 | * This is racy, but we can live with some | |
94 | * inaccuracy in the failcnt. | |
95 | */ | |
96 | c->failcnt++; | |
97 | *fail = c; | |
98 | goto failed; | |
99 | } | |
100 | /* | |
101 | * Just like with failcnt, we can live with some | |
102 | * inaccuracy in the watermark. | |
103 | */ | |
104 | if (new > c->watermark) | |
105 | c->watermark = new; | |
106 | } | |
107 | return 0; | |
108 | ||
109 | failed: | |
110 | for (c = counter; c != *fail; c = c->parent) | |
111 | page_counter_cancel(c, nr_pages); | |
112 | ||
113 | return -ENOMEM; | |
114 | } | |
115 | ||
116 | /** | |
117 | * page_counter_uncharge - hierarchically uncharge pages | |
118 | * @counter: counter | |
119 | * @nr_pages: number of pages to uncharge | |
120 | * | |
121 | * Returns whether there are remaining charges in @counter. | |
122 | */ | |
123 | int page_counter_uncharge(struct page_counter *counter, unsigned long nr_pages) | |
124 | { | |
125 | struct page_counter *c; | |
126 | int ret = 1; | |
127 | ||
128 | for (c = counter; c; c = c->parent) { | |
129 | int remainder; | |
130 | ||
131 | remainder = page_counter_cancel(c, nr_pages); | |
132 | if (c == counter && !remainder) | |
133 | ret = 0; | |
134 | } | |
135 | ||
136 | return ret; | |
137 | } | |
138 | ||
139 | /** | |
140 | * page_counter_limit - limit the number of pages allowed | |
141 | * @counter: counter | |
142 | * @limit: limit to set | |
143 | * | |
144 | * Returns 0 on success, -EBUSY if the current number of pages on the | |
145 | * counter already exceeds the specified limit. | |
146 | * | |
147 | * The caller must serialize invocations on the same counter. | |
148 | */ | |
149 | int page_counter_limit(struct page_counter *counter, unsigned long limit) | |
150 | { | |
151 | for (;;) { | |
152 | unsigned long old; | |
153 | long count; | |
154 | ||
155 | /* | |
156 | * Update the limit while making sure that it's not | |
157 | * below the concurrently-changing counter value. | |
158 | * | |
159 | * The xchg implies two full memory barriers before | |
160 | * and after, so the read-swap-read is ordered and | |
161 | * ensures coherency with page_counter_try_charge(): | |
162 | * that function modifies the count before checking | |
163 | * the limit, so if it sees the old limit, we see the | |
164 | * modified counter and retry. | |
165 | */ | |
166 | count = atomic_long_read(&counter->count); | |
167 | ||
168 | if (count > limit) | |
169 | return -EBUSY; | |
170 | ||
171 | old = xchg(&counter->limit, limit); | |
172 | ||
173 | if (atomic_long_read(&counter->count) <= count) | |
174 | return 0; | |
175 | ||
176 | counter->limit = old; | |
177 | cond_resched(); | |
178 | } | |
179 | } | |
180 | ||
181 | /** | |
182 | * page_counter_memparse - memparse() for page counter limits | |
183 | * @buf: string to parse | |
184 | * @nr_pages: returns the result in number of pages | |
185 | * | |
186 | * Returns -EINVAL, or 0 and @nr_pages on success. @nr_pages will be | |
187 | * limited to %PAGE_COUNTER_MAX. | |
188 | */ | |
189 | int page_counter_memparse(const char *buf, unsigned long *nr_pages) | |
190 | { | |
191 | char unlimited[] = "-1"; | |
192 | char *end; | |
193 | u64 bytes; | |
194 | ||
195 | if (!strncmp(buf, unlimited, sizeof(unlimited))) { | |
196 | *nr_pages = PAGE_COUNTER_MAX; | |
197 | return 0; | |
198 | } | |
199 | ||
200 | bytes = memparse(buf, &end); | |
201 | if (*end != '\0') | |
202 | return -EINVAL; | |
203 | ||
204 | *nr_pages = min(bytes / PAGE_SIZE, (u64)PAGE_COUNTER_MAX); | |
205 | ||
206 | return 0; | |
207 | } |