]>
Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | #ifndef _LINUX_CPUSET_H |
2 | #define _LINUX_CPUSET_H | |
3 | /* | |
4 | * cpuset interface | |
5 | * | |
6 | * Copyright (C) 2003 BULL SA | |
825a46af | 7 | * Copyright (C) 2004-2006 Silicon Graphics, Inc. |
1da177e4 LT |
8 | * |
9 | */ | |
10 | ||
11 | #include <linux/sched.h> | |
105ab3d8 | 12 | #include <linux/sched/topology.h> |
f719ff9b | 13 | #include <linux/sched/task.h> |
1da177e4 LT |
14 | #include <linux/cpumask.h> |
15 | #include <linux/nodemask.h> | |
a1bc5a4e | 16 | #include <linux/mm.h> |
664eedde | 17 | #include <linux/jump_label.h> |
1da177e4 LT |
18 | |
19 | #ifdef CONFIG_CPUSETS | |
20 | ||
89affbf5 DZ |
21 | /* |
22 | * Static branch rewrites can happen in an arbitrary order for a given | |
23 | * key. In code paths where we need to loop with read_mems_allowed_begin() and | |
24 | * read_mems_allowed_retry() to get a consistent view of mems_allowed, we need | |
25 | * to ensure that begin() always gets rewritten before retry() in the | |
26 | * disabled -> enabled transition. If not, then if local irqs are disabled | |
27 | * around the loop, we can deadlock since retry() would always be | |
28 | * comparing the latest value of the mems_allowed seqcount against 0 as | |
29 | * begin() still would see cpusets_enabled() as false. The enabled -> disabled | |
30 | * transition should happen in reverse order for the same reasons (want to stop | |
31 | * looking at real value of mems_allowed.sequence in retry() first). | |
32 | */ | |
33 | extern struct static_key_false cpusets_pre_enable_key; | |
002f2906 | 34 | extern struct static_key_false cpusets_enabled_key; |
664eedde MG |
35 | static inline bool cpusets_enabled(void) |
36 | { | |
002f2906 | 37 | return static_branch_unlikely(&cpusets_enabled_key); |
664eedde MG |
38 | } |
39 | ||
40 | static inline int nr_cpusets(void) | |
41 | { | |
42 | /* jump label reference count + the top-level cpuset */ | |
002f2906 | 43 | return static_key_count(&cpusets_enabled_key.key) + 1; |
664eedde MG |
44 | } |
45 | ||
46 | static inline void cpuset_inc(void) | |
47 | { | |
89affbf5 | 48 | static_branch_inc(&cpusets_pre_enable_key); |
002f2906 | 49 | static_branch_inc(&cpusets_enabled_key); |
664eedde MG |
50 | } |
51 | ||
52 | static inline void cpuset_dec(void) | |
53 | { | |
002f2906 | 54 | static_branch_dec(&cpusets_enabled_key); |
89affbf5 | 55 | static_branch_dec(&cpusets_pre_enable_key); |
664eedde | 56 | } |
202f72d5 | 57 | |
1da177e4 LT |
58 | extern int cpuset_init(void); |
59 | extern void cpuset_init_smp(void); | |
30e03acd | 60 | extern void cpuset_update_active_cpus(void); |
6af866af | 61 | extern void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask); |
2baab4e9 | 62 | extern void cpuset_cpus_allowed_fallback(struct task_struct *p); |
909d75a3 | 63 | extern nodemask_t cpuset_mems_allowed(struct task_struct *p); |
9276b1bc | 64 | #define cpuset_current_mems_allowed (current->mems_allowed) |
1da177e4 | 65 | void cpuset_init_current_mems_allowed(void); |
19770b32 | 66 | int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask); |
202f72d5 | 67 | |
002f2906 | 68 | extern bool __cpuset_node_allowed(int node, gfp_t gfp_mask); |
02a0e53d | 69 | |
002f2906 | 70 | static inline bool cpuset_node_allowed(int node, gfp_t gfp_mask) |
02a0e53d | 71 | { |
002f2906 VB |
72 | if (cpusets_enabled()) |
73 | return __cpuset_node_allowed(node, gfp_mask); | |
74 | return true; | |
02a0e53d PJ |
75 | } |
76 | ||
002f2906 | 77 | static inline bool __cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask) |
202f72d5 | 78 | { |
002f2906 VB |
79 | return __cpuset_node_allowed(zone_to_nid(z), gfp_mask); |
80 | } | |
81 | ||
82 | static inline bool cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask) | |
83 | { | |
84 | if (cpusets_enabled()) | |
85 | return __cpuset_zone_allowed(z, gfp_mask); | |
86 | return true; | |
202f72d5 PJ |
87 | } |
88 | ||
bbe373f2 DR |
89 | extern int cpuset_mems_allowed_intersects(const struct task_struct *tsk1, |
90 | const struct task_struct *tsk2); | |
3e0d98b9 PJ |
91 | |
92 | #define cpuset_memory_pressure_bump() \ | |
93 | do { \ | |
94 | if (cpuset_memory_pressure_enabled) \ | |
95 | __cpuset_memory_pressure_bump(); \ | |
96 | } while (0) | |
97 | extern int cpuset_memory_pressure_enabled; | |
98 | extern void __cpuset_memory_pressure_bump(void); | |
99 | ||
df5f8314 EB |
100 | extern void cpuset_task_status_allowed(struct seq_file *m, |
101 | struct task_struct *task); | |
52de4779 ZL |
102 | extern int proc_cpuset_show(struct seq_file *m, struct pid_namespace *ns, |
103 | struct pid *pid, struct task_struct *tsk); | |
1da177e4 | 104 | |
825a46af | 105 | extern int cpuset_mem_spread_node(void); |
6adef3eb | 106 | extern int cpuset_slab_spread_node(void); |
825a46af PJ |
107 | |
108 | static inline int cpuset_do_page_mem_spread(void) | |
109 | { | |
2ad654bc | 110 | return task_spread_page(current); |
825a46af PJ |
111 | } |
112 | ||
113 | static inline int cpuset_do_slab_mem_spread(void) | |
114 | { | |
2ad654bc | 115 | return task_spread_slab(current); |
825a46af PJ |
116 | } |
117 | ||
8793d854 PM |
118 | extern int current_cpuset_is_being_rebound(void); |
119 | ||
e761b772 MK |
120 | extern void rebuild_sched_domains(void); |
121 | ||
da39da3a | 122 | extern void cpuset_print_current_mems_allowed(void); |
75aa1994 | 123 | |
c0ff7453 | 124 | /* |
d26914d1 MG |
125 | * read_mems_allowed_begin is required when making decisions involving |
126 | * mems_allowed such as during page allocation. mems_allowed can be updated in | |
127 | * parallel and depending on the new value an operation can fail potentially | |
128 | * causing process failure. A retry loop with read_mems_allowed_begin and | |
129 | * read_mems_allowed_retry prevents these artificial failures. | |
c0ff7453 | 130 | */ |
d26914d1 | 131 | static inline unsigned int read_mems_allowed_begin(void) |
c0ff7453 | 132 | { |
89affbf5 | 133 | if (!static_branch_unlikely(&cpusets_pre_enable_key)) |
46e700ab MG |
134 | return 0; |
135 | ||
cc9a6c87 | 136 | return read_seqcount_begin(¤t->mems_allowed_seq); |
c0ff7453 MX |
137 | } |
138 | ||
cc9a6c87 | 139 | /* |
d26914d1 MG |
140 | * If this returns true, the operation that took place after |
141 | * read_mems_allowed_begin may have failed artificially due to a concurrent | |
142 | * update of mems_allowed. It is up to the caller to retry the operation if | |
cc9a6c87 MG |
143 | * appropriate. |
144 | */ | |
d26914d1 | 145 | static inline bool read_mems_allowed_retry(unsigned int seq) |
c0ff7453 | 146 | { |
89affbf5 | 147 | if (!static_branch_unlikely(&cpusets_enabled_key)) |
46e700ab MG |
148 | return false; |
149 | ||
d26914d1 | 150 | return read_seqcount_retry(¤t->mems_allowed_seq, seq); |
c0ff7453 MX |
151 | } |
152 | ||
58568d2a MX |
153 | static inline void set_mems_allowed(nodemask_t nodemask) |
154 | { | |
db751fe3 JS |
155 | unsigned long flags; |
156 | ||
c0ff7453 | 157 | task_lock(current); |
db751fe3 | 158 | local_irq_save(flags); |
cc9a6c87 | 159 | write_seqcount_begin(¤t->mems_allowed_seq); |
58568d2a | 160 | current->mems_allowed = nodemask; |
cc9a6c87 | 161 | write_seqcount_end(¤t->mems_allowed_seq); |
db751fe3 | 162 | local_irq_restore(flags); |
c0ff7453 | 163 | task_unlock(current); |
58568d2a MX |
164 | } |
165 | ||
1da177e4 LT |
166 | #else /* !CONFIG_CPUSETS */ |
167 | ||
664eedde MG |
168 | static inline bool cpusets_enabled(void) { return false; } |
169 | ||
1da177e4 LT |
170 | static inline int cpuset_init(void) { return 0; } |
171 | static inline void cpuset_init_smp(void) {} | |
1da177e4 | 172 | |
30e03acd | 173 | static inline void cpuset_update_active_cpus(void) |
3a101d05 TH |
174 | { |
175 | partition_sched_domains(1, NULL, NULL); | |
176 | } | |
177 | ||
6af866af LZ |
178 | static inline void cpuset_cpus_allowed(struct task_struct *p, |
179 | struct cpumask *mask) | |
1da177e4 | 180 | { |
aa85ea5b | 181 | cpumask_copy(mask, cpu_possible_mask); |
1da177e4 LT |
182 | } |
183 | ||
2baab4e9 | 184 | static inline void cpuset_cpus_allowed_fallback(struct task_struct *p) |
9084bb82 | 185 | { |
9084bb82 ON |
186 | } |
187 | ||
909d75a3 PJ |
188 | static inline nodemask_t cpuset_mems_allowed(struct task_struct *p) |
189 | { | |
190 | return node_possible_map; | |
191 | } | |
192 | ||
38d7bee9 | 193 | #define cpuset_current_mems_allowed (node_states[N_MEMORY]) |
1da177e4 | 194 | static inline void cpuset_init_current_mems_allowed(void) {} |
1da177e4 | 195 | |
19770b32 | 196 | static inline int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask) |
1da177e4 LT |
197 | { |
198 | return 1; | |
199 | } | |
200 | ||
002f2906 | 201 | static inline bool cpuset_node_allowed(int node, gfp_t gfp_mask) |
02a0e53d | 202 | { |
002f2906 | 203 | return true; |
02a0e53d PJ |
204 | } |
205 | ||
002f2906 | 206 | static inline bool __cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask) |
1da177e4 | 207 | { |
002f2906 VB |
208 | return true; |
209 | } | |
210 | ||
211 | static inline bool cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask) | |
212 | { | |
213 | return true; | |
1da177e4 LT |
214 | } |
215 | ||
bbe373f2 DR |
216 | static inline int cpuset_mems_allowed_intersects(const struct task_struct *tsk1, |
217 | const struct task_struct *tsk2) | |
ef08e3b4 PJ |
218 | { |
219 | return 1; | |
220 | } | |
221 | ||
3e0d98b9 PJ |
222 | static inline void cpuset_memory_pressure_bump(void) {} |
223 | ||
df5f8314 EB |
224 | static inline void cpuset_task_status_allowed(struct seq_file *m, |
225 | struct task_struct *task) | |
1da177e4 | 226 | { |
1da177e4 LT |
227 | } |
228 | ||
825a46af PJ |
229 | static inline int cpuset_mem_spread_node(void) |
230 | { | |
231 | return 0; | |
232 | } | |
233 | ||
6adef3eb JS |
234 | static inline int cpuset_slab_spread_node(void) |
235 | { | |
236 | return 0; | |
237 | } | |
238 | ||
825a46af PJ |
239 | static inline int cpuset_do_page_mem_spread(void) |
240 | { | |
241 | return 0; | |
242 | } | |
243 | ||
244 | static inline int cpuset_do_slab_mem_spread(void) | |
245 | { | |
246 | return 0; | |
247 | } | |
248 | ||
8793d854 PM |
249 | static inline int current_cpuset_is_being_rebound(void) |
250 | { | |
251 | return 0; | |
252 | } | |
253 | ||
e761b772 MK |
254 | static inline void rebuild_sched_domains(void) |
255 | { | |
dfb512ec | 256 | partition_sched_domains(1, NULL, NULL); |
e761b772 MK |
257 | } |
258 | ||
da39da3a | 259 | static inline void cpuset_print_current_mems_allowed(void) |
75aa1994 DR |
260 | { |
261 | } | |
262 | ||
58568d2a MX |
263 | static inline void set_mems_allowed(nodemask_t nodemask) |
264 | { | |
265 | } | |
266 | ||
d26914d1 | 267 | static inline unsigned int read_mems_allowed_begin(void) |
c0ff7453 | 268 | { |
cc9a6c87 | 269 | return 0; |
c0ff7453 MX |
270 | } |
271 | ||
d26914d1 | 272 | static inline bool read_mems_allowed_retry(unsigned int seq) |
c0ff7453 | 273 | { |
d26914d1 | 274 | return false; |
c0ff7453 MX |
275 | } |
276 | ||
1da177e4 LT |
277 | #endif /* !CONFIG_CPUSETS */ |
278 | ||
279 | #endif /* _LINUX_CPUSET_H */ |