drivers/cpuidle/cpuidle-powernv.c

   1 /*
   2  *  cpuidle-powernv - idle state cpuidle driver.
   3  *  Adapted from drivers/cpuidle/cpuidle-pseries
   4  *
   5  */
   6
   7 #include <linux/kernel.h>
   8 #include <linux/module.h>
   9 #include <linux/init.h>
  10 #include <linux/moduleparam.h>
  11 #include <linux/cpuidle.h>
  12 #include <linux/cpu.h>
  13 #include <linux/notifier.h>
  14 #include <linux/clockchips.h>
  15 #include <linux/of.h>
  16 #include <linux/slab.h>
  17
  18 #include <asm/machdep.h>
  19 #include <asm/firmware.h>
  20 #include <asm/opal.h>
  21 #include <asm/runlatch.h>
  22
  23 /*
  24  * Expose only those Hardware idle states via the cpuidle framework
  25  * that have latency value below POWERNV_THRESHOLD_LATENCY_NS.
  26  */
  27 #define POWERNV_THRESHOLD_LATENCY_NS 200000
  28
  29 static struct cpuidle_driver powernv_idle_driver = {
  30         .name             = "powernv_idle",
  31         .owner            = THIS_MODULE,
  32 };
  33
  34 static int max_idle_state;
  35 static struct cpuidle_state *cpuidle_state_table;
  36
  37 static u64 stop_psscr_table[CPUIDLE_STATE_MAX];
  38
  39 static u64 snooze_timeout;
  40 static bool snooze_timeout_en;
  41
  42 static int snooze_loop(struct cpuidle_device *dev,
  43                         struct cpuidle_driver *drv,
  44                         int index)
  45 {
  46         u64 snooze_exit_time;
  47
  48         local_irq_enable();
  49         set_thread_flag(TIF_POLLING_NRFLAG);
  50
  51         snooze_exit_time = get_tb() + snooze_timeout;
  52         ppc64_runlatch_off();
  53         while (!need_resched()) {
  54                 HMT_low();
  55                 HMT_very_low();
  56                 if (snooze_timeout_en && get_tb() > snooze_exit_time)
  57                         break;
  58         }
  59
  60         HMT_medium();
  61         ppc64_runlatch_on();
  62         clear_thread_flag(TIF_POLLING_NRFLAG);
  63         smp_mb();
  64         return index;
  65 }
  66
  67 static int nap_loop(struct cpuidle_device *dev,
  68                         struct cpuidle_driver *drv,
  69                         int index)
  70 {
  71         ppc64_runlatch_off();
  72         power7_idle();
  73         ppc64_runlatch_on();
  74         return index;
  75 }
  76
  77 /* Register for fastsleep only in oneshot mode of broadcast */
  78 #ifdef CONFIG_TICK_ONESHOT
  79 static int fastsleep_loop(struct cpuidle_device *dev,
  80                                 struct cpuidle_driver *drv,
  81                                 int index)
  82 {
  83         unsigned long old_lpcr = mfspr(SPRN_LPCR);
  84         unsigned long new_lpcr;
  85
  86         if (unlikely(system_state < SYSTEM_RUNNING))
  87                 return index;
  88
  89         new_lpcr = old_lpcr;
  90         /* Do not exit powersave upon decrementer as we've setup the timer
  91          * offload.
  92          */
  93         new_lpcr &= ~LPCR_PECE1;
  94
  95         mtspr(SPRN_LPCR, new_lpcr);
  96         power7_sleep();
  97
  98         mtspr(SPRN_LPCR, old_lpcr);
  99
 100         return index;
 101 }
 102 #endif
 103
 104 static int stop_loop(struct cpuidle_device *dev,
 105                      struct cpuidle_driver *drv,
 106                      int index)
 107 {
 108         ppc64_runlatch_off();
 109         power9_idle_stop(stop_psscr_table[index]);
 110         ppc64_runlatch_on();
 111         return index;
 112 }
 113
 114 /*
 115  * States for dedicated partition case.
 116  */
 117 static struct cpuidle_state powernv_states[CPUIDLE_STATE_MAX] = {
 118         { /* Snooze */
 119                 .name = "snooze",
 120                 .desc = "snooze",
 121                 .exit_latency = 0,
 122                 .target_residency = 0,
 123                 .enter = snooze_loop },
 124 };
 125
 126 static int powernv_cpuidle_cpu_online(unsigned int cpu)
 127 {
 128         struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu);
 129
 130         if (dev && cpuidle_get_driver()) {
 131                 cpuidle_pause_and_lock();
 132                 cpuidle_enable_device(dev);
 133                 cpuidle_resume_and_unlock();
 134         }
 135         return 0;
 136 }
 137
 138 static int powernv_cpuidle_cpu_dead(unsigned int cpu)
 139 {
 140         struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu);
 141
 142         if (dev && cpuidle_get_driver()) {
 143                 cpuidle_pause_and_lock();
 144                 cpuidle_disable_device(dev);
 145                 cpuidle_resume_and_unlock();
 146         }
 147         return 0;
 148 }
 149
 150 /*
 151  * powernv_cpuidle_driver_init()
 152  */
 153 static int powernv_cpuidle_driver_init(void)
 154 {
 155         int idle_state;
 156         struct cpuidle_driver *drv = &powernv_idle_driver;
 157
 158         drv->state_count = 0;
 159
 160         for (idle_state = 0; idle_state < max_idle_state; ++idle_state) {
 161                 /* Is the state not enabled? */
 162                 if (cpuidle_state_table[idle_state].enter == NULL)
 163                         continue;
 164
 165                 drv->states[drv->state_count] = /* structure copy */
 166                         cpuidle_state_table[idle_state];
 167
 168                 drv->state_count += 1;
 169         }
 170
 171         return 0;
 172 }
 173
 174 static inline void add_powernv_state(int index, const char *name,
 175                                      unsigned int flags,
 176                                      int (*idle_fn)(struct cpuidle_device *,
 177                                                     struct cpuidle_driver *,
 178                                                     int),
 179                                      unsigned int target_residency,
 180                                      unsigned int exit_latency,
 181                                      u64 psscr_val)
 182 {
 183         strlcpy(powernv_states[index].name, name, CPUIDLE_NAME_LEN);
 184         strlcpy(powernv_states[index].desc, name, CPUIDLE_NAME_LEN);
 185         powernv_states[index].flags = flags;
 186         powernv_states[index].target_residency = target_residency;
 187         powernv_states[index].exit_latency = exit_latency;
 188         powernv_states[index].enter = idle_fn;
 189         stop_psscr_table[index] = psscr_val;
 190 }
 191
 192 static int powernv_add_idle_states(void)
 193 {
 194         struct device_node *power_mgt;
 195         int nr_idle_states = 1; /* Snooze */
 196         int dt_idle_states;
 197         u32 latency_ns[CPUIDLE_STATE_MAX];
 198         u32 residency_ns[CPUIDLE_STATE_MAX];
 199         u32 flags[CPUIDLE_STATE_MAX];
 200         u64 psscr_val[CPUIDLE_STATE_MAX];
 201         const char *names[CPUIDLE_STATE_MAX];
 202         int i, rc;
 203
 204         /* Currently we have snooze statically defined */
 205
 206         power_mgt = of_find_node_by_path("/ibm,opal/power-mgt");
 207         if (!power_mgt) {
 208                 pr_warn("opal: PowerMgmt Node not found\n");
 209                 goto out;
 210         }
 211
 212         /* Read values of any property to determine the num of idle states */
 213         dt_idle_states = of_property_count_u32_elems(power_mgt, "ibm,cpu-idle-state-flags");
 214         if (dt_idle_states < 0) {
 215                 pr_warn("cpuidle-powernv: no idle states found in the DT\n");
 216                 goto out;
 217         }
 218
 219         /*
 220          * Since snooze is used as first idle state, max idle states allowed is
 221          * CPUIDLE_STATE_MAX -1
 222          */
 223         if (dt_idle_states > CPUIDLE_STATE_MAX - 1) {
 224                 pr_warn("cpuidle-powernv: discovered idle states more than allowed");
 225                 dt_idle_states = CPUIDLE_STATE_MAX - 1;
 226         }
 227
 228         if (of_property_read_u32_array(power_mgt,
 229                         "ibm,cpu-idle-state-flags", flags, dt_idle_states)) {
 230                 pr_warn("cpuidle-powernv : missing ibm,cpu-idle-state-flags in DT\n");
 231                 goto out;
 232         }
 233
 234         if (of_property_read_u32_array(power_mgt,
 235                 "ibm,cpu-idle-state-latencies-ns", latency_ns,
 236                 dt_idle_states)) {
 237                 pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-latencies-ns in DT\n");
 238                 goto out;
 239         }
 240         if (of_property_read_string_array(power_mgt,
 241                 "ibm,cpu-idle-state-names", names, dt_idle_states) < 0) {
 242                 pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-names in DT\n");
 243                 goto out;
 244         }
 245
 246         /*
 247          * If the idle states use stop instruction, probe for psscr values
 248          * which are necessary to specify required stop level.
 249          */
 250         if (flags[0] & (OPAL_PM_STOP_INST_FAST | OPAL_PM_STOP_INST_DEEP))
 251                 if (of_property_read_u64_array(power_mgt,
 252                     "ibm,cpu-idle-state-psscr", psscr_val, dt_idle_states)) {
 253                         pr_warn("cpuidle-powernv: missing ibm,cpu-idle-states-psscr in DT\n");
 254                         goto out;
 255                 }
 256
 257         rc = of_property_read_u32_array(power_mgt,
 258                 "ibm,cpu-idle-state-residency-ns", residency_ns, dt_idle_states);
 259
 260         for (i = 0; i < dt_idle_states; i++) {
 261                 unsigned int exit_latency, target_residency;
 262                 /*
 263                  * If an idle state has exit latency beyond
 264                  * POWERNV_THRESHOLD_LATENCY_NS then don't use it
 265                  * in cpu-idle.
 266                  */
 267                 if (latency_ns[i] > POWERNV_THRESHOLD_LATENCY_NS)
 268                         continue;
 269                 /*
 270                  * Firmware passes residency and latency values in ns.
 271                  * cpuidle expects it in us.
 272                  */
 273                 exit_latency = latency_ns[i] / 1000;
 274                 if (!rc)
 275                         target_residency = residency_ns[i] / 1000;
 276                 else
 277                         target_residency = 0;
 278
 279                 /*
 280                  * For nap and fastsleep, use default target_residency
 281                  * values if f/w does not expose it.
 282                  */
 283                 if (flags[i] & OPAL_PM_NAP_ENABLED) {
 284                         if (!rc)
 285                                 target_residency = 100;
 286                         /* Add NAP state */
 287                         add_powernv_state(nr_idle_states, "Nap",
 288                                           CPUIDLE_FLAG_NONE, nap_loop,
 289                                           target_residency, exit_latency, 0);
 290                 } else if ((flags[i] & OPAL_PM_STOP_INST_FAST) &&
 291                                 !(flags[i] & OPAL_PM_TIMEBASE_STOP)) {
 292                         add_powernv_state(nr_idle_states, names[i],
 293                                           CPUIDLE_FLAG_NONE, stop_loop,
 294                                           target_residency, exit_latency,
 295                                           psscr_val[i]);
 296                 }
 297
 298                 /*
 299                  * All cpuidle states with CPUIDLE_FLAG_TIMER_STOP set must come
 300                  * within this config dependency check.
 301                  */
 302 #ifdef CONFIG_TICK_ONESHOT
 303                 if (flags[i] & OPAL_PM_SLEEP_ENABLED ||
 304                         flags[i] & OPAL_PM_SLEEP_ENABLED_ER1) {
 305                         if (!rc)
 306                                 target_residency = 300000;
 307                         /* Add FASTSLEEP state */
 308                         add_powernv_state(nr_idle_states, "FastSleep",
 309                                           CPUIDLE_FLAG_TIMER_STOP,
 310                                           fastsleep_loop,
 311                                           target_residency, exit_latency, 0);
 312                 } else if ((flags[i] & OPAL_PM_STOP_INST_DEEP) &&
 313                                 (flags[i] & OPAL_PM_TIMEBASE_STOP)) {
 314                         add_powernv_state(nr_idle_states, names[i],
 315                                           CPUIDLE_FLAG_TIMER_STOP, stop_loop,
 316                                           target_residency, exit_latency,
 317                                           psscr_val[i]);
 318                 }
 319 #endif
 320                 nr_idle_states++;
 321         }
 322 out:
 323         return nr_idle_states;
 324 }
 325
 326 /*
 327  * powernv_idle_probe()
 328  * Choose state table for shared versus dedicated partition
 329  */
 330 static int powernv_idle_probe(void)
 331 {
 332         if (cpuidle_disable != IDLE_NO_OVERRIDE)
 333                 return -ENODEV;
 334
 335         if (firmware_has_feature(FW_FEATURE_OPAL)) {
 336                 cpuidle_state_table = powernv_states;
 337                 /* Device tree can indicate more idle states */
 338                 max_idle_state = powernv_add_idle_states();
 339                 if (max_idle_state > 1) {
 340                         snooze_timeout_en = true;
 341                         snooze_timeout = powernv_states[1].target_residency *
 342                                          tb_ticks_per_usec;
 343                 }
 344         } else
 345                 return -ENODEV;
 346
 347         return 0;
 348 }
 349
 350 static int __init powernv_processor_idle_init(void)
 351 {
 352         int retval;
 353
 354         retval = powernv_idle_probe();
 355         if (retval)
 356                 return retval;
 357
 358         powernv_cpuidle_driver_init();
 359         retval = cpuidle_register(&powernv_idle_driver, NULL);
 360         if (retval) {
 361                 printk(KERN_DEBUG "Registration of powernv driver failed.\n");
 362                 return retval;
 363         }
 364
 365         retval = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
 366                                            "cpuidle/powernv:online",
 367                                            powernv_cpuidle_cpu_online, NULL);
 368         WARN_ON(retval < 0);
 369         retval = cpuhp_setup_state_nocalls(CPUHP_CPUIDLE_DEAD,
 370                                            "cpuidle/powernv:dead", NULL,
 371                                            powernv_cpuidle_cpu_dead);
 372         WARN_ON(retval < 0);
 373         printk(KERN_DEBUG "powernv_idle_driver registered\n");
 374         return 0;
 375 }
 376
 377 device_initcall(powernv_processor_idle_init);