kernel/sched/isolation.c

   1 /*
   2  *  Housekeeping management. Manage the targets for routine code that can run on
   3  *  any CPU: unbound workqueues, timers, kthreads and any offloadable work.
   4  *
   5  * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
   6  * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
   7  *
   8  */
   9 #include <linux/sched/isolation.h>
  10 #include <linux/tick.h>
  11 #include <linux/init.h>
  12 #include <linux/kernel.h>
  13 #include <linux/static_key.h>
  14 #include <linux/ctype.h>
  15
  16 #include "sched.h"
  17
  18 DEFINE_STATIC_KEY_FALSE(housekeeping_overriden);
  19 EXPORT_SYMBOL_GPL(housekeeping_overriden);
  20 static cpumask_var_t housekeeping_mask;
  21 static unsigned int housekeeping_flags;
  22
  23 int housekeeping_any_cpu(enum hk_flags flags)
  24 {
  25         if (static_branch_unlikely(&housekeeping_overriden))
  26                 if (housekeeping_flags & flags)
  27                         return cpumask_any_and(housekeeping_mask, cpu_online_mask);
  28         return smp_processor_id();
  29 }
  30 EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
  31
  32 const struct cpumask *housekeeping_cpumask(enum hk_flags flags)
  33 {
  34         if (static_branch_unlikely(&housekeeping_overriden))
  35                 if (housekeeping_flags & flags)
  36                         return housekeeping_mask;
  37         return cpu_possible_mask;
  38 }
  39 EXPORT_SYMBOL_GPL(housekeeping_cpumask);
  40
  41 void housekeeping_affine(struct task_struct *t, enum hk_flags flags)
  42 {
  43         if (static_branch_unlikely(&housekeeping_overriden))
  44                 if (housekeeping_flags & flags)
  45                         set_cpus_allowed_ptr(t, housekeeping_mask);
  46 }
  47 EXPORT_SYMBOL_GPL(housekeeping_affine);
  48
  49 bool housekeeping_test_cpu(int cpu, enum hk_flags flags)
  50 {
  51         if (static_branch_unlikely(&housekeeping_overriden))
  52                 if (housekeeping_flags & flags)
  53                         return cpumask_test_cpu(cpu, housekeeping_mask);
  54         return true;
  55 }
  56 EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
  57
  58 void __init housekeeping_init(void)
  59 {
  60         if (!housekeeping_flags)
  61                 return;
  62
  63         static_branch_enable(&housekeeping_overriden);
  64
  65         if (housekeeping_flags & HK_FLAG_TICK)
  66                 sched_tick_offload_init();
  67
  68         /* We need at least one CPU to handle housekeeping work */
  69         WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
  70 }
  71
  72 static int __init housekeeping_setup(char *str, enum hk_flags flags)
  73 {
  74         cpumask_var_t non_housekeeping_mask;
  75         int err;
  76
  77         alloc_bootmem_cpumask_var(&non_housekeeping_mask);
  78         err = cpulist_parse(str, non_housekeeping_mask);
  79         if (err < 0 || cpumask_last(non_housekeeping_mask) >= nr_cpu_ids) {
  80                 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
  81                 free_bootmem_cpumask_var(non_housekeeping_mask);
  82                 return 0;
  83         }
  84
  85         if (!housekeeping_flags) {
  86                 alloc_bootmem_cpumask_var(&housekeeping_mask);
  87                 cpumask_andnot(housekeeping_mask,
  88                                cpu_possible_mask, non_housekeeping_mask);
  89                 if (cpumask_empty(housekeeping_mask))
  90                         cpumask_set_cpu(smp_processor_id(), housekeeping_mask);
  91         } else {
  92                 cpumask_var_t tmp;
  93
  94                 alloc_bootmem_cpumask_var(&tmp);
  95                 cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask);
  96                 if (!cpumask_equal(tmp, housekeeping_mask)) {
  97                         pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
  98                         free_bootmem_cpumask_var(tmp);
  99                         free_bootmem_cpumask_var(non_housekeeping_mask);
 100                         return 0;
 101                 }
 102                 free_bootmem_cpumask_var(tmp);
 103         }
 104
 105         if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) {
 106                 if (IS_ENABLED(CONFIG_NO_HZ_FULL)) {
 107                         tick_nohz_full_setup(non_housekeeping_mask);
 108                 } else {
 109                         pr_warn("Housekeeping: nohz unsupported."
 110                                 " Build with CONFIG_NO_HZ_FULL\n");
 111                         free_bootmem_cpumask_var(non_housekeeping_mask);
 112                         return 0;
 113                 }
 114         }
 115
 116         housekeeping_flags |= flags;
 117
 118         free_bootmem_cpumask_var(non_housekeeping_mask);
 119
 120         return 1;
 121 }
 122
 123 static int __init housekeeping_nohz_full_setup(char *str)
 124 {
 125         unsigned int flags;
 126
 127         flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | HK_FLAG_MISC;
 128
 129         return housekeeping_setup(str, flags);
 130 }
 131 __setup("nohz_full=", housekeeping_nohz_full_setup);
 132
 133 static int __init housekeeping_isolcpus_setup(char *str)
 134 {
 135         unsigned int flags = 0;
 136
 137         while (isalpha(*str)) {
 138                 if (!strncmp(str, "nohz,", 5)) {
 139                         str += 5;
 140                         flags |= HK_FLAG_TICK;
 141                         continue;
 142                 }
 143
 144                 if (!strncmp(str, "domain,", 7)) {
 145                         str += 7;
 146                         flags |= HK_FLAG_DOMAIN;
 147                         continue;
 148                 }
 149
 150                 pr_warn("isolcpus: Error, unknown flag\n");
 151                 return 0;
 152         }
 153
 154         /* Default behaviour for isolcpus without flags */
 155         if (!flags)
 156                 flags |= HK_FLAG_DOMAIN;
 157
 158         return housekeeping_setup(str, flags);
 159 }
 160 __setup("isolcpus=", housekeeping_isolcpus_setup);