]>
Commit | Line | Data |
---|---|---|
78634061 FW |
1 | /* |
2 | * Housekeeping management. Manage the targets for routine code that can run on | |
3 | * any CPU: unbound workqueues, timers, kthreads and any offloadable work. | |
4 | * | |
5 | * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker | |
6 | * | |
7 | */ | |
8 | ||
9 | #include <linux/sched/isolation.h> | |
10 | #include <linux/tick.h> | |
11 | #include <linux/init.h> | |
12 | #include <linux/kernel.h> | |
e179f5a0 | 13 | #include <linux/static_key.h> |
78634061 | 14 | |
e179f5a0 FW |
15 | DEFINE_STATIC_KEY_FALSE(housekeeping_overriden); |
16 | EXPORT_SYMBOL_GPL(housekeeping_overriden); | |
7e56a1cf | 17 | static cpumask_var_t housekeeping_mask; |
de201559 | 18 | static unsigned int housekeeping_flags; |
7e56a1cf | 19 | |
de201559 | 20 | int housekeeping_any_cpu(enum hk_flags flags) |
7e56a1cf | 21 | { |
e179f5a0 | 22 | if (static_branch_unlikely(&housekeeping_overriden)) |
de201559 FW |
23 | if (housekeeping_flags & flags) |
24 | return cpumask_any_and(housekeeping_mask, cpu_online_mask); | |
7e56a1cf FW |
25 | return smp_processor_id(); |
26 | } | |
27 | EXPORT_SYMBOL_GPL(housekeeping_any_cpu); | |
28 | ||
de201559 | 29 | const struct cpumask *housekeeping_cpumask(enum hk_flags flags) |
7e56a1cf | 30 | { |
e179f5a0 | 31 | if (static_branch_unlikely(&housekeeping_overriden)) |
de201559 FW |
32 | if (housekeeping_flags & flags) |
33 | return housekeeping_mask; | |
7e56a1cf FW |
34 | return cpu_possible_mask; |
35 | } | |
36 | EXPORT_SYMBOL_GPL(housekeeping_cpumask); | |
37 | ||
de201559 | 38 | void housekeeping_affine(struct task_struct *t, enum hk_flags flags) |
7e56a1cf | 39 | { |
e179f5a0 | 40 | if (static_branch_unlikely(&housekeeping_overriden)) |
de201559 FW |
41 | if (housekeeping_flags & flags) |
42 | set_cpus_allowed_ptr(t, housekeeping_mask); | |
7e56a1cf FW |
43 | } |
44 | EXPORT_SYMBOL_GPL(housekeeping_affine); | |
45 | ||
de201559 | 46 | bool housekeeping_test_cpu(int cpu, enum hk_flags flags) |
7e56a1cf | 47 | { |
e179f5a0 | 48 | if (static_branch_unlikely(&housekeeping_overriden)) |
de201559 FW |
49 | if (housekeeping_flags & flags) |
50 | return cpumask_test_cpu(cpu, housekeeping_mask); | |
7e56a1cf FW |
51 | return true; |
52 | } | |
53 | EXPORT_SYMBOL_GPL(housekeeping_test_cpu); | |
78634061 FW |
54 | |
55 | void __init housekeeping_init(void) | |
56 | { | |
6f1982fe | 57 | if (!housekeeping_flags) |
78634061 FW |
58 | return; |
59 | ||
6f1982fe FW |
60 | static_branch_enable(&housekeeping_overriden); |
61 | ||
62 | /* We need at least one CPU to handle housekeeping work */ | |
63 | WARN_ON_ONCE(cpumask_empty(housekeeping_mask)); | |
64 | } | |
65 | ||
edb93821 | 66 | static int __init housekeeping_setup(char *str, enum hk_flags flags) |
6f1982fe FW |
67 | { |
68 | cpumask_var_t non_housekeeping_mask; | |
edb93821 | 69 | int err; |
6f1982fe FW |
70 | |
71 | alloc_bootmem_cpumask_var(&non_housekeeping_mask); | |
edb93821 FW |
72 | err = cpulist_parse(str, non_housekeeping_mask); |
73 | if (err < 0 || cpumask_last(non_housekeeping_mask) >= nr_cpu_ids) { | |
74 | pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n"); | |
6f1982fe FW |
75 | free_bootmem_cpumask_var(non_housekeeping_mask); |
76 | return 0; | |
78634061 FW |
77 | } |
78 | ||
edb93821 FW |
79 | if (!housekeeping_flags) { |
80 | alloc_bootmem_cpumask_var(&housekeeping_mask); | |
81 | cpumask_andnot(housekeeping_mask, | |
82 | cpu_possible_mask, non_housekeeping_mask); | |
83 | if (cpumask_empty(housekeeping_mask)) | |
84 | cpumask_set_cpu(smp_processor_id(), housekeeping_mask); | |
85 | } else { | |
86 | cpumask_var_t tmp; | |
87 | ||
88 | alloc_bootmem_cpumask_var(&tmp); | |
89 | cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask); | |
90 | if (!cpumask_equal(tmp, housekeeping_mask)) { | |
91 | pr_warn("Housekeeping: nohz_full= must match isolcpus=\n"); | |
92 | free_bootmem_cpumask_var(tmp); | |
93 | free_bootmem_cpumask_var(non_housekeeping_mask); | |
94 | return 0; | |
95 | } | |
96 | free_bootmem_cpumask_var(tmp); | |
97 | } | |
de201559 | 98 | |
edb93821 FW |
99 | if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) { |
100 | if (IS_ENABLED(CONFIG_NO_HZ_FULL)) { | |
101 | tick_nohz_full_setup(non_housekeeping_mask); | |
102 | } else { | |
103 | pr_warn("Housekeeping: nohz unsupported." | |
104 | " Build with CONFIG_NO_HZ_FULL\n"); | |
105 | free_bootmem_cpumask_var(non_housekeeping_mask); | |
106 | return 0; | |
107 | } | |
108 | } | |
e179f5a0 | 109 | |
edb93821 | 110 | housekeeping_flags |= flags; |
6f1982fe FW |
111 | |
112 | free_bootmem_cpumask_var(non_housekeeping_mask); | |
113 | ||
114 | return 1; | |
78634061 | 115 | } |
edb93821 FW |
116 | |
117 | static int __init housekeeping_nohz_full_setup(char *str) | |
118 | { | |
119 | unsigned int flags; | |
120 | ||
121 | flags = HK_FLAG_TICK | HK_FLAG_TIMER | HK_FLAG_RCU | HK_FLAG_MISC; | |
122 | ||
123 | return housekeeping_setup(str, flags); | |
124 | } | |
6f1982fe | 125 | __setup("nohz_full=", housekeeping_nohz_full_setup); |
edb93821 FW |
126 | |
127 | static int __init housekeeping_isolcpus_setup(char *str) | |
128 | { | |
129 | return housekeeping_setup(str, HK_FLAG_DOMAIN); | |
130 | } | |
131 | __setup("isolcpus=", housekeeping_isolcpus_setup); |