]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/cpufreq/cpufreq_ondemand.c
Merge branch 'topic/lx6464es' into for-linus
[mirror_ubuntu-artful-kernel.git] / drivers / cpufreq / cpufreq_ondemand.c
CommitLineData
1da177e4
LT
1/*
2 * drivers/cpufreq/cpufreq_ondemand.c
3 *
4 * Copyright (C) 2001 Russell King
5 * (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
6 * Jun Nakajima <jun.nakajima@intel.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13#include <linux/kernel.h>
14#include <linux/module.h>
1da177e4 15#include <linux/init.h>
1da177e4 16#include <linux/cpufreq.h>
138a0128 17#include <linux/cpu.h>
1da177e4
LT
18#include <linux/jiffies.h>
19#include <linux/kernel_stat.h>
3fc54d37 20#include <linux/mutex.h>
80800913 21#include <linux/hrtimer.h>
22#include <linux/tick.h>
23#include <linux/ktime.h>
9411b4ef 24#include <linux/sched.h>
1da177e4
LT
25
26/*
27 * dbs is used in this file as a shortform for demandbased switching
28 * It helps to keep variable names smaller, simpler
29 */
30
e9d95bf7 31#define DEF_FREQUENCY_DOWN_DIFFERENTIAL (10)
1da177e4 32#define DEF_FREQUENCY_UP_THRESHOLD (80)
80800913 33#define MICRO_FREQUENCY_DOWN_DIFFERENTIAL (3)
34#define MICRO_FREQUENCY_UP_THRESHOLD (95)
c29f1403 35#define MIN_FREQUENCY_UP_THRESHOLD (11)
1da177e4
LT
36#define MAX_FREQUENCY_UP_THRESHOLD (100)
37
32ee8c3e
DJ
38/*
39 * The polling frequency of this governor depends on the capability of
1da177e4 40 * the processor. Default polling frequency is 1000 times the transition
32ee8c3e
DJ
41 * latency of the processor. The governor will work on any processor with
42 * transition latency <= 10mS, using appropriate sampling
1da177e4
LT
43 * rate.
44 * For CPUs with transition latency > 10mS (mostly drivers with CPUFREQ_ETERNAL)
45 * this governor will not work.
46 * All times here are in uS.
47 */
32ee8c3e 48static unsigned int def_sampling_rate;
df8b59be
DJ
49#define MIN_SAMPLING_RATE_RATIO (2)
50/* for correct statistics, we need at least 10 ticks between each measure */
e08f5f5b
GS
51#define MIN_STAT_SAMPLING_RATE \
52 (MIN_SAMPLING_RATE_RATIO * jiffies_to_usecs(10))
53#define MIN_SAMPLING_RATE \
54 (def_sampling_rate / MIN_SAMPLING_RATE_RATIO)
112124ab
TR
55/* Above MIN_SAMPLING_RATE will vanish with its sysfs file soon
56 * Define the minimal settable sampling rate to the greater of:
57 * - "HW transition latency" * 100 (same as default sampling / 10)
58 * - MIN_STAT_SAMPLING_RATE
59 * To avoid that userspace shoots itself.
60*/
61static unsigned int minimum_sampling_rate(void)
62{
63 return max(def_sampling_rate / 10, MIN_STAT_SAMPLING_RATE);
64}
65
66/* This will also vanish soon with removing sampling_rate_max */
1da177e4 67#define MAX_SAMPLING_RATE (500 * def_sampling_rate)
112124ab 68#define LATENCY_MULTIPLIER (1000)
1c256245 69#define TRANSITION_LATENCY_LIMIT (10 * 1000 * 1000)
1da177e4 70
c4028958
DH
71static void do_dbs_timer(struct work_struct *work);
72
73/* Sampling types */
529af7a1 74enum {DBS_NORMAL_SAMPLE, DBS_SUB_SAMPLE};
1da177e4
LT
75
76struct cpu_dbs_info_s {
ccb2fe20
VP
77 cputime64_t prev_cpu_idle;
78 cputime64_t prev_cpu_wall;
80800913 79 cputime64_t prev_cpu_nice;
32ee8c3e 80 struct cpufreq_policy *cur_policy;
2b03f891 81 struct delayed_work work;
05ca0350
AS
82 struct cpufreq_frequency_table *freq_table;
83 unsigned int freq_lo;
84 unsigned int freq_lo_jiffies;
85 unsigned int freq_hi_jiffies;
529af7a1
VP
86 int cpu;
87 unsigned int enable:1,
2b03f891 88 sample_type:1;
1da177e4
LT
89};
90static DEFINE_PER_CPU(struct cpu_dbs_info_s, cpu_dbs_info);
91
92static unsigned int dbs_enable; /* number of CPUs using this policy */
93
4ec223d0
VP
94/*
95 * DEADLOCK ALERT! There is a ordering requirement between cpu_hotplug
96 * lock and dbs_mutex. cpu_hotplug lock should always be held before
97 * dbs_mutex. If any function that can potentially take cpu_hotplug lock
98 * (like __cpufreq_driver_target()) is being called with dbs_mutex taken, then
99 * cpu_hotplug lock should be taken before that. Note that cpu_hotplug lock
100 * is recursive for the same process. -Venki
b14893a6
MD
101 * DEADLOCK ALERT! (2) : do_dbs_timer() must not take the dbs_mutex, because it
102 * would deadlock with cancel_delayed_work_sync(), which is needed for proper
103 * raceless workqueue teardown.
4ec223d0 104 */
ffac80e9 105static DEFINE_MUTEX(dbs_mutex);
1da177e4 106
2f8a835c 107static struct workqueue_struct *kondemand_wq;
6810b548 108
05ca0350 109static struct dbs_tuners {
32ee8c3e 110 unsigned int sampling_rate;
32ee8c3e 111 unsigned int up_threshold;
e9d95bf7 112 unsigned int down_differential;
32ee8c3e 113 unsigned int ignore_nice;
05ca0350
AS
114 unsigned int powersave_bias;
115} dbs_tuners_ins = {
32ee8c3e 116 .up_threshold = DEF_FREQUENCY_UP_THRESHOLD,
e9d95bf7 117 .down_differential = DEF_FREQUENCY_DOWN_DIFFERENTIAL,
9cbad61b 118 .ignore_nice = 0,
05ca0350 119 .powersave_bias = 0,
1da177e4
LT
120};
121
80800913 122static inline cputime64_t get_cpu_idle_time_jiffy(unsigned int cpu,
123 cputime64_t *wall)
dac1c1a5 124{
ea487615 125 cputime64_t idle_time;
3430502d 126 cputime64_t cur_wall_time;
ea487615 127 cputime64_t busy_time;
ccb2fe20 128
3430502d 129 cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
ea487615
VP
130 busy_time = cputime64_add(kstat_cpu(cpu).cpustat.user,
131 kstat_cpu(cpu).cpustat.system);
ccb2fe20 132
ea487615
VP
133 busy_time = cputime64_add(busy_time, kstat_cpu(cpu).cpustat.irq);
134 busy_time = cputime64_add(busy_time, kstat_cpu(cpu).cpustat.softirq);
135 busy_time = cputime64_add(busy_time, kstat_cpu(cpu).cpustat.steal);
1ca3abdb 136 busy_time = cputime64_add(busy_time, kstat_cpu(cpu).cpustat.nice);
ea487615 137
3430502d 138 idle_time = cputime64_sub(cur_wall_time, busy_time);
139 if (wall)
140 *wall = cur_wall_time;
141
ea487615 142 return idle_time;
dac1c1a5
DJ
143}
144
80800913 145static inline cputime64_t get_cpu_idle_time(unsigned int cpu, cputime64_t *wall)
146{
147 u64 idle_time = get_cpu_idle_time_us(cpu, wall);
148
149 if (idle_time == -1ULL)
150 return get_cpu_idle_time_jiffy(cpu, wall);
151
80800913 152 return idle_time;
153}
154
05ca0350
AS
155/*
156 * Find right freq to be set now with powersave_bias on.
157 * Returns the freq_hi to be used right now and will set freq_hi_jiffies,
158 * freq_lo, and freq_lo_jiffies in percpu area for averaging freqs.
159 */
b5ecf60f
AB
160static unsigned int powersave_bias_target(struct cpufreq_policy *policy,
161 unsigned int freq_next,
162 unsigned int relation)
05ca0350
AS
163{
164 unsigned int freq_req, freq_reduc, freq_avg;
165 unsigned int freq_hi, freq_lo;
166 unsigned int index = 0;
167 unsigned int jiffies_total, jiffies_hi, jiffies_lo;
168 struct cpu_dbs_info_s *dbs_info = &per_cpu(cpu_dbs_info, policy->cpu);
169
170 if (!dbs_info->freq_table) {
171 dbs_info->freq_lo = 0;
172 dbs_info->freq_lo_jiffies = 0;
173 return freq_next;
174 }
175
176 cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_next,
177 relation, &index);
178 freq_req = dbs_info->freq_table[index].frequency;
179 freq_reduc = freq_req * dbs_tuners_ins.powersave_bias / 1000;
180 freq_avg = freq_req - freq_reduc;
181
182 /* Find freq bounds for freq_avg in freq_table */
183 index = 0;
184 cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg,
185 CPUFREQ_RELATION_H, &index);
186 freq_lo = dbs_info->freq_table[index].frequency;
187 index = 0;
188 cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg,
189 CPUFREQ_RELATION_L, &index);
190 freq_hi = dbs_info->freq_table[index].frequency;
191
192 /* Find out how long we have to be in hi and lo freqs */
193 if (freq_hi == freq_lo) {
194 dbs_info->freq_lo = 0;
195 dbs_info->freq_lo_jiffies = 0;
196 return freq_lo;
197 }
198 jiffies_total = usecs_to_jiffies(dbs_tuners_ins.sampling_rate);
199 jiffies_hi = (freq_avg - freq_lo) * jiffies_total;
200 jiffies_hi += ((freq_hi - freq_lo) / 2);
201 jiffies_hi /= (freq_hi - freq_lo);
202 jiffies_lo = jiffies_total - jiffies_hi;
203 dbs_info->freq_lo = freq_lo;
204 dbs_info->freq_lo_jiffies = jiffies_lo;
205 dbs_info->freq_hi_jiffies = jiffies_hi;
206 return freq_hi;
207}
208
209static void ondemand_powersave_bias_init(void)
210{
211 int i;
212 for_each_online_cpu(i) {
213 struct cpu_dbs_info_s *dbs_info = &per_cpu(cpu_dbs_info, i);
214 dbs_info->freq_table = cpufreq_frequency_get_table(i);
215 dbs_info->freq_lo = 0;
216 }
217}
218
1da177e4
LT
219/************************** sysfs interface ************************/
220static ssize_t show_sampling_rate_max(struct cpufreq_policy *policy, char *buf)
221{
9411b4ef
TR
222 static int print_once;
223
224 if (!print_once) {
225 printk(KERN_INFO "CPUFREQ: ondemand sampling_rate_max "
226 "sysfs file is deprecated - used by: %s\n",
227 current->comm);
228 print_once = 1;
229 }
2b03f891 230 return sprintf(buf, "%u\n", MAX_SAMPLING_RATE);
1da177e4
LT
231}
232
233static ssize_t show_sampling_rate_min(struct cpufreq_policy *policy, char *buf)
234{
9411b4ef
TR
235 static int print_once;
236
237 if (!print_once) {
238 printk(KERN_INFO "CPUFREQ: ondemand sampling_rate_min "
239 "sysfs file is deprecated - used by: %s\n",
240 current->comm);
241 print_once = 1;
242 }
2b03f891 243 return sprintf(buf, "%u\n", MIN_SAMPLING_RATE);
1da177e4
LT
244}
245
32ee8c3e
DJ
246#define define_one_ro(_name) \
247static struct freq_attr _name = \
1da177e4
LT
248__ATTR(_name, 0444, show_##_name, NULL)
249
250define_one_ro(sampling_rate_max);
251define_one_ro(sampling_rate_min);
252
253/* cpufreq_ondemand Governor Tunables */
254#define show_one(file_name, object) \
255static ssize_t show_##file_name \
256(struct cpufreq_policy *unused, char *buf) \
257{ \
258 return sprintf(buf, "%u\n", dbs_tuners_ins.object); \
259}
260show_one(sampling_rate, sampling_rate);
1da177e4 261show_one(up_threshold, up_threshold);
001893cd 262show_one(ignore_nice_load, ignore_nice);
05ca0350 263show_one(powersave_bias, powersave_bias);
1da177e4 264
32ee8c3e 265static ssize_t store_sampling_rate(struct cpufreq_policy *unused,
1da177e4
LT
266 const char *buf, size_t count)
267{
268 unsigned int input;
269 int ret;
ffac80e9 270 ret = sscanf(buf, "%u", &input);
1da177e4 271
3fc54d37 272 mutex_lock(&dbs_mutex);
112124ab 273 if (ret != 1) {
3fc54d37 274 mutex_unlock(&dbs_mutex);
1da177e4
LT
275 return -EINVAL;
276 }
112124ab 277 dbs_tuners_ins.sampling_rate = max(input, minimum_sampling_rate());
3fc54d37 278 mutex_unlock(&dbs_mutex);
1da177e4
LT
279
280 return count;
281}
282
32ee8c3e 283static ssize_t store_up_threshold(struct cpufreq_policy *unused,
1da177e4
LT
284 const char *buf, size_t count)
285{
286 unsigned int input;
287 int ret;
ffac80e9 288 ret = sscanf(buf, "%u", &input);
1da177e4 289
3fc54d37 290 mutex_lock(&dbs_mutex);
32ee8c3e 291 if (ret != 1 || input > MAX_FREQUENCY_UP_THRESHOLD ||
c29f1403 292 input < MIN_FREQUENCY_UP_THRESHOLD) {
3fc54d37 293 mutex_unlock(&dbs_mutex);
1da177e4
LT
294 return -EINVAL;
295 }
296
297 dbs_tuners_ins.up_threshold = input;
3fc54d37 298 mutex_unlock(&dbs_mutex);
1da177e4
LT
299
300 return count;
301}
302
001893cd 303static ssize_t store_ignore_nice_load(struct cpufreq_policy *policy,
3d5ee9e5
DJ
304 const char *buf, size_t count)
305{
306 unsigned int input;
307 int ret;
308
309 unsigned int j;
32ee8c3e 310
ffac80e9 311 ret = sscanf(buf, "%u", &input);
2b03f891 312 if (ret != 1)
3d5ee9e5
DJ
313 return -EINVAL;
314
2b03f891 315 if (input > 1)
3d5ee9e5 316 input = 1;
32ee8c3e 317
3fc54d37 318 mutex_lock(&dbs_mutex);
2b03f891 319 if (input == dbs_tuners_ins.ignore_nice) { /* nothing to do */
3fc54d37 320 mutex_unlock(&dbs_mutex);
3d5ee9e5
DJ
321 return count;
322 }
323 dbs_tuners_ins.ignore_nice = input;
324
ccb2fe20 325 /* we need to re-evaluate prev_cpu_idle */
dac1c1a5 326 for_each_online_cpu(j) {
ccb2fe20
VP
327 struct cpu_dbs_info_s *dbs_info;
328 dbs_info = &per_cpu(cpu_dbs_info, j);
3430502d 329 dbs_info->prev_cpu_idle = get_cpu_idle_time(j,
330 &dbs_info->prev_cpu_wall);
1ca3abdb
VP
331 if (dbs_tuners_ins.ignore_nice)
332 dbs_info->prev_cpu_nice = kstat_cpu(j).cpustat.nice;
333
3d5ee9e5 334 }
3fc54d37 335 mutex_unlock(&dbs_mutex);
3d5ee9e5
DJ
336
337 return count;
338}
339
05ca0350
AS
340static ssize_t store_powersave_bias(struct cpufreq_policy *unused,
341 const char *buf, size_t count)
342{
343 unsigned int input;
344 int ret;
345 ret = sscanf(buf, "%u", &input);
346
347 if (ret != 1)
348 return -EINVAL;
349
350 if (input > 1000)
351 input = 1000;
352
353 mutex_lock(&dbs_mutex);
354 dbs_tuners_ins.powersave_bias = input;
355 ondemand_powersave_bias_init();
356 mutex_unlock(&dbs_mutex);
357
358 return count;
359}
360
1da177e4
LT
361#define define_one_rw(_name) \
362static struct freq_attr _name = \
363__ATTR(_name, 0644, show_##_name, store_##_name)
364
365define_one_rw(sampling_rate);
1da177e4 366define_one_rw(up_threshold);
001893cd 367define_one_rw(ignore_nice_load);
05ca0350 368define_one_rw(powersave_bias);
1da177e4 369
2b03f891 370static struct attribute *dbs_attributes[] = {
1da177e4
LT
371 &sampling_rate_max.attr,
372 &sampling_rate_min.attr,
373 &sampling_rate.attr,
1da177e4 374 &up_threshold.attr,
001893cd 375 &ignore_nice_load.attr,
05ca0350 376 &powersave_bias.attr,
1da177e4
LT
377 NULL
378};
379
380static struct attribute_group dbs_attr_group = {
381 .attrs = dbs_attributes,
382 .name = "ondemand",
383};
384
385/************************** sysfs end ************************/
386
2f8a835c 387static void dbs_check_cpu(struct cpu_dbs_info_s *this_dbs_info)
1da177e4 388{
c43aa3bd 389 unsigned int max_load_freq;
1da177e4
LT
390
391 struct cpufreq_policy *policy;
392 unsigned int j;
393
1da177e4
LT
394 if (!this_dbs_info->enable)
395 return;
396
05ca0350 397 this_dbs_info->freq_lo = 0;
1da177e4 398 policy = this_dbs_info->cur_policy;
ea487615 399
32ee8c3e 400 /*
c29f1403
DJ
401 * Every sampling_rate, we check, if current idle time is less
402 * than 20% (default), then we try to increase frequency
ccb2fe20 403 * Every sampling_rate, we look for a the lowest
c29f1403
DJ
404 * frequency which can sustain the load while keeping idle time over
405 * 30%. If such a frequency exist, we try to decrease to this frequency.
1da177e4 406 *
32ee8c3e
DJ
407 * Any frequency increase takes it to the maximum frequency.
408 * Frequency reduction happens at minimum steps of
409 * 5% (default) of current frequency
1da177e4
LT
410 */
411
c43aa3bd 412 /* Get Absolute Load - in terms of freq */
413 max_load_freq = 0;
414
835481d9 415 for_each_cpu(j, policy->cpus) {
1da177e4 416 struct cpu_dbs_info_s *j_dbs_info;
c43aa3bd 417 cputime64_t cur_wall_time, cur_idle_time;
418 unsigned int idle_time, wall_time;
419 unsigned int load, load_freq;
420 int freq_avg;
1da177e4 421
1da177e4 422 j_dbs_info = &per_cpu(cpu_dbs_info, j);
3430502d 423
424 cur_idle_time = get_cpu_idle_time(j, &cur_wall_time);
425
c43aa3bd 426 wall_time = (unsigned int) cputime64_sub(cur_wall_time,
427 j_dbs_info->prev_cpu_wall);
428 j_dbs_info->prev_cpu_wall = cur_wall_time;
429
c43aa3bd 430 idle_time = (unsigned int) cputime64_sub(cur_idle_time,
ccb2fe20 431 j_dbs_info->prev_cpu_idle);
c43aa3bd 432 j_dbs_info->prev_cpu_idle = cur_idle_time;
1da177e4 433
1ca3abdb
VP
434 if (dbs_tuners_ins.ignore_nice) {
435 cputime64_t cur_nice;
436 unsigned long cur_nice_jiffies;
437
438 cur_nice = cputime64_sub(kstat_cpu(j).cpustat.nice,
439 j_dbs_info->prev_cpu_nice);
440 /*
441 * Assumption: nice time between sampling periods will
442 * be less than 2^32 jiffies for 32 bit sys
443 */
444 cur_nice_jiffies = (unsigned long)
445 cputime64_to_jiffies64(cur_nice);
446
447 j_dbs_info->prev_cpu_nice = kstat_cpu(j).cpustat.nice;
448 idle_time += jiffies_to_usecs(cur_nice_jiffies);
449 }
450
3430502d 451 if (unlikely(!wall_time || wall_time < idle_time))
c43aa3bd 452 continue;
c43aa3bd 453
454 load = 100 * (wall_time - idle_time) / wall_time;
455
456 freq_avg = __cpufreq_driver_getavg(policy, j);
457 if (freq_avg <= 0)
458 freq_avg = policy->cur;
459
460 load_freq = load * freq_avg;
461 if (load_freq > max_load_freq)
462 max_load_freq = load_freq;
1da177e4
LT
463 }
464
ccb2fe20 465 /* Check for frequency increase */
c43aa3bd 466 if (max_load_freq > dbs_tuners_ins.up_threshold * policy->cur) {
c11420a6 467 /* if we are already at full speed then break out early */
05ca0350
AS
468 if (!dbs_tuners_ins.powersave_bias) {
469 if (policy->cur == policy->max)
470 return;
471
472 __cpufreq_driver_target(policy, policy->max,
473 CPUFREQ_RELATION_H);
474 } else {
475 int freq = powersave_bias_target(policy, policy->max,
476 CPUFREQ_RELATION_H);
477 __cpufreq_driver_target(policy, freq,
478 CPUFREQ_RELATION_L);
479 }
1da177e4
LT
480 return;
481 }
482
483 /* Check for frequency decrease */
c29f1403
DJ
484 /* if we cannot reduce the frequency anymore, break out early */
485 if (policy->cur == policy->min)
486 return;
1da177e4 487
c29f1403
DJ
488 /*
489 * The optimal frequency is the frequency that is the lowest that
490 * can support the current CPU usage without triggering the up
491 * policy. To be safe, we focus 10 points under the threshold.
492 */
e9d95bf7 493 if (max_load_freq <
494 (dbs_tuners_ins.up_threshold - dbs_tuners_ins.down_differential) *
495 policy->cur) {
c43aa3bd 496 unsigned int freq_next;
e9d95bf7 497 freq_next = max_load_freq /
498 (dbs_tuners_ins.up_threshold -
499 dbs_tuners_ins.down_differential);
dfde5d62 500
05ca0350
AS
501 if (!dbs_tuners_ins.powersave_bias) {
502 __cpufreq_driver_target(policy, freq_next,
503 CPUFREQ_RELATION_L);
504 } else {
505 int freq = powersave_bias_target(policy, freq_next,
506 CPUFREQ_RELATION_L);
507 __cpufreq_driver_target(policy, freq,
508 CPUFREQ_RELATION_L);
509 }
ccb2fe20 510 }
1da177e4
LT
511}
512
c4028958 513static void do_dbs_timer(struct work_struct *work)
32ee8c3e 514{
529af7a1
VP
515 struct cpu_dbs_info_s *dbs_info =
516 container_of(work, struct cpu_dbs_info_s, work.work);
517 unsigned int cpu = dbs_info->cpu;
518 int sample_type = dbs_info->sample_type;
519
1ce28d6b
AS
520 /* We want all CPUs to do sampling nearly on same jiffy */
521 int delay = usecs_to_jiffies(dbs_tuners_ins.sampling_rate);
c4028958 522
1ce28d6b 523 delay -= jiffies % delay;
2f8a835c 524
56463b78 525 if (lock_policy_rwsem_write(cpu) < 0)
2cd7cbdf 526 return;
56463b78
VP
527
528 if (!dbs_info->enable) {
529 unlock_policy_rwsem_write(cpu);
530 return;
531 }
532
05ca0350 533 /* Common NORMAL_SAMPLE setup */
c4028958 534 dbs_info->sample_type = DBS_NORMAL_SAMPLE;
05ca0350 535 if (!dbs_tuners_ins.powersave_bias ||
c4028958 536 sample_type == DBS_NORMAL_SAMPLE) {
05ca0350 537 dbs_check_cpu(dbs_info);
05ca0350
AS
538 if (dbs_info->freq_lo) {
539 /* Setup timer for SUB_SAMPLE */
c4028958 540 dbs_info->sample_type = DBS_SUB_SAMPLE;
05ca0350
AS
541 delay = dbs_info->freq_hi_jiffies;
542 }
543 } else {
544 __cpufreq_driver_target(dbs_info->cur_policy,
2b03f891 545 dbs_info->freq_lo, CPUFREQ_RELATION_H);
05ca0350 546 }
1ce28d6b 547 queue_delayed_work_on(cpu, kondemand_wq, &dbs_info->work, delay);
56463b78 548 unlock_policy_rwsem_write(cpu);
32ee8c3e 549}
1da177e4 550
529af7a1 551static inline void dbs_timer_init(struct cpu_dbs_info_s *dbs_info)
1da177e4 552{
1ce28d6b
AS
553 /* We want all CPUs to do sampling nearly on same jiffy */
554 int delay = usecs_to_jiffies(dbs_tuners_ins.sampling_rate);
555 delay -= jiffies % delay;
2f8a835c 556
c18a1483 557 dbs_info->enable = 1;
05ca0350 558 ondemand_powersave_bias_init();
c4028958 559 dbs_info->sample_type = DBS_NORMAL_SAMPLE;
28287033 560 INIT_DELAYED_WORK_DEFERRABLE(&dbs_info->work, do_dbs_timer);
529af7a1 561 queue_delayed_work_on(dbs_info->cpu, kondemand_wq, &dbs_info->work,
2b03f891 562 delay);
1da177e4
LT
563}
564
2cd7cbdf 565static inline void dbs_timer_exit(struct cpu_dbs_info_s *dbs_info)
1da177e4 566{
2cd7cbdf 567 dbs_info->enable = 0;
b14893a6 568 cancel_delayed_work_sync(&dbs_info->work);
1da177e4
LT
569}
570
571static int cpufreq_governor_dbs(struct cpufreq_policy *policy,
572 unsigned int event)
573{
574 unsigned int cpu = policy->cpu;
575 struct cpu_dbs_info_s *this_dbs_info;
576 unsigned int j;
914f7c31 577 int rc;
1da177e4
LT
578
579 this_dbs_info = &per_cpu(cpu_dbs_info, cpu);
580
581 switch (event) {
582 case CPUFREQ_GOV_START:
ffac80e9 583 if ((!cpu_online(cpu)) || (!policy->cur))
1da177e4
LT
584 return -EINVAL;
585
1da177e4
LT
586 if (this_dbs_info->enable) /* Already enabled */
587 break;
32ee8c3e 588
3fc54d37 589 mutex_lock(&dbs_mutex);
2f8a835c 590 dbs_enable++;
914f7c31
JG
591
592 rc = sysfs_create_group(&policy->kobj, &dbs_attr_group);
593 if (rc) {
914f7c31
JG
594 dbs_enable--;
595 mutex_unlock(&dbs_mutex);
596 return rc;
597 }
598
835481d9 599 for_each_cpu(j, policy->cpus) {
1da177e4
LT
600 struct cpu_dbs_info_s *j_dbs_info;
601 j_dbs_info = &per_cpu(cpu_dbs_info, j);
602 j_dbs_info->cur_policy = policy;
32ee8c3e 603
3430502d 604 j_dbs_info->prev_cpu_idle = get_cpu_idle_time(j,
605 &j_dbs_info->prev_cpu_wall);
1ca3abdb
VP
606 if (dbs_tuners_ins.ignore_nice) {
607 j_dbs_info->prev_cpu_nice =
608 kstat_cpu(j).cpustat.nice;
609 }
1da177e4 610 }
529af7a1 611 this_dbs_info->cpu = cpu;
1da177e4
LT
612 /*
613 * Start the timerschedule work, when this governor
614 * is used for first time
615 */
616 if (dbs_enable == 1) {
617 unsigned int latency;
618 /* policy latency is in nS. Convert it to uS first */
df8b59be
DJ
619 latency = policy->cpuinfo.transition_latency / 1000;
620 if (latency == 0)
621 latency = 1;
1da177e4 622
112124ab
TR
623 def_sampling_rate =
624 max(latency * LATENCY_MULTIPLIER,
625 MIN_STAT_SAMPLING_RATE);
df8b59be 626
1da177e4 627 dbs_tuners_ins.sampling_rate = def_sampling_rate;
1da177e4 628 }
529af7a1 629 dbs_timer_init(this_dbs_info);
32ee8c3e 630
3fc54d37 631 mutex_unlock(&dbs_mutex);
1da177e4
LT
632 break;
633
634 case CPUFREQ_GOV_STOP:
3fc54d37 635 mutex_lock(&dbs_mutex);
2cd7cbdf 636 dbs_timer_exit(this_dbs_info);
1da177e4
LT
637 sysfs_remove_group(&policy->kobj, &dbs_attr_group);
638 dbs_enable--;
3fc54d37 639 mutex_unlock(&dbs_mutex);
1da177e4
LT
640
641 break;
642
643 case CPUFREQ_GOV_LIMITS:
3fc54d37 644 mutex_lock(&dbs_mutex);
1da177e4 645 if (policy->max < this_dbs_info->cur_policy->cur)
ffac80e9 646 __cpufreq_driver_target(this_dbs_info->cur_policy,
2b03f891 647 policy->max, CPUFREQ_RELATION_H);
1da177e4 648 else if (policy->min > this_dbs_info->cur_policy->cur)
ffac80e9 649 __cpufreq_driver_target(this_dbs_info->cur_policy,
2b03f891 650 policy->min, CPUFREQ_RELATION_L);
3fc54d37 651 mutex_unlock(&dbs_mutex);
1da177e4
LT
652 break;
653 }
654 return 0;
655}
656
c4d14bc0
SW
657#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND
658static
659#endif
1c256245
TR
660struct cpufreq_governor cpufreq_gov_ondemand = {
661 .name = "ondemand",
662 .governor = cpufreq_governor_dbs,
663 .max_transition_latency = TRANSITION_LATENCY_LIMIT,
664 .owner = THIS_MODULE,
1da177e4 665};
1da177e4
LT
666
667static int __init cpufreq_gov_dbs_init(void)
668{
888a794c 669 int err;
80800913 670 cputime64_t wall;
4f6e6b9f
AR
671 u64 idle_time;
672 int cpu = get_cpu();
80800913 673
4f6e6b9f
AR
674 idle_time = get_cpu_idle_time_us(cpu, &wall);
675 put_cpu();
80800913 676 if (idle_time != -1ULL) {
677 /* Idle micro accounting is supported. Use finer thresholds */
678 dbs_tuners_ins.up_threshold = MICRO_FREQUENCY_UP_THRESHOLD;
679 dbs_tuners_ins.down_differential =
680 MICRO_FREQUENCY_DOWN_DIFFERENTIAL;
681 }
888a794c 682
56463b78
VP
683 kondemand_wq = create_workqueue("kondemand");
684 if (!kondemand_wq) {
685 printk(KERN_ERR "Creation of kondemand failed\n");
686 return -EFAULT;
687 }
888a794c
AM
688 err = cpufreq_register_governor(&cpufreq_gov_ondemand);
689 if (err)
690 destroy_workqueue(kondemand_wq);
691
692 return err;
1da177e4
LT
693}
694
695static void __exit cpufreq_gov_dbs_exit(void)
696{
1c256245 697 cpufreq_unregister_governor(&cpufreq_gov_ondemand);
56463b78 698 destroy_workqueue(kondemand_wq);
1da177e4
LT
699}
700
701
ffac80e9
VP
702MODULE_AUTHOR("Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>");
703MODULE_AUTHOR("Alexey Starikovskiy <alexey.y.starikovskiy@intel.com>");
704MODULE_DESCRIPTION("'cpufreq_ondemand' - A dynamic cpufreq governor for "
2b03f891 705 "Low Latency Frequency Transition capable processors");
ffac80e9 706MODULE_LICENSE("GPL");
1da177e4 707
6915719b
JW
708#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND
709fs_initcall(cpufreq_gov_dbs_init);
710#else
1da177e4 711module_init(cpufreq_gov_dbs_init);
6915719b 712#endif
1da177e4 713module_exit(cpufreq_gov_dbs_exit);