[mirror_ubuntu-artful-kernel.git] / drivers / cpufreq / cpufreq_ondemand.c

/*
 *  drivers/cpufreq/cpufreq_ondemand.c
 *
 *  Copyright (C)  2001 Russell King
 *            (C)  2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
 *                      Jun Nakajima <jun.nakajima@intel.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/cpu.h>
#include <linux/percpu-defs.h>
#include <linux/slab.h>
#include <linux/tick.h>
#include "cpufreq_governor.h"

/* On-demand governor macros */
#define DEF_FREQUENCY_UP_THRESHOLD		(80)
#define DEF_SAMPLING_DOWN_FACTOR		(1)
#define MAX_SAMPLING_DOWN_FACTOR		(100000)
#define MICRO_FREQUENCY_UP_THRESHOLD		(95)
#define MICRO_FREQUENCY_MIN_SAMPLE_RATE		(10000)
#define MIN_FREQUENCY_UP_THRESHOLD		(11)
#define MAX_FREQUENCY_UP_THRESHOLD		(100)

static DEFINE_PER_CPU(struct od_cpu_dbs_info_s, od_cpu_dbs_info);

static struct dbs_governor od_dbs_gov;
static struct od_ops od_ops;

static unsigned int default_powersave_bias;

/*
 * Not all CPUs want IO time to be accounted as busy; this depends on how
 * efficient idling at a higher frequency/voltage is.
 * Pavel Machek says this is not so for various generations of AMD and old
 * Intel systems.
 * Mike Chan (android.com) claims this is also not true for ARM.
 * Because of this, whitelist specific known (series) of CPUs by default, and
 * leave all others up to the user.
 */
static int should_io_be_busy(void)
{
#if defined(CONFIG_X86)
	/*
	 * For Intel, Core 2 (model 15) and later have an efficient idle.
	 */
	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
			boot_cpu_data.x86 == 6 &&
			boot_cpu_data.x86_model >= 15)
		return 1;
#endif
	return 0;
}

/*
 * Find right freq to be set now with powersave_bias on.
 * Returns the freq_hi to be used right now and will set freq_hi_delay_us,
 * freq_lo, and freq_lo_delay_us in percpu area for averaging freqs.
 */
static unsigned int generic_powersave_bias_target(struct cpufreq_policy *policy,
		unsigned int freq_next, unsigned int relation)
{
	unsigned int freq_req, freq_reduc, freq_avg;
	unsigned int freq_hi, freq_lo;
	unsigned int index = 0;
	unsigned int delay_hi_us;
	struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info,
						   policy->cpu);
	struct policy_dbs_info *policy_dbs = policy->governor_data;
	struct dbs_data *dbs_data = policy_dbs->dbs_data;
	struct od_dbs_tuners *od_tuners = dbs_data->tuners;

	if (!dbs_info->freq_table) {
		dbs_info->freq_lo = 0;
		dbs_info->freq_lo_delay_us = 0;
		return freq_next;
	}

	cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_next,
			relation, &index);
	freq_req = dbs_info->freq_table[index].frequency;
	freq_reduc = freq_req * od_tuners->powersave_bias / 1000;
	freq_avg = freq_req - freq_reduc;

	/* Find freq bounds for freq_avg in freq_table */
	index = 0;
	cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg,
			CPUFREQ_RELATION_H, &index);
	freq_lo = dbs_info->freq_table[index].frequency;
	index = 0;
	cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg,
			CPUFREQ_RELATION_L, &index);
	freq_hi = dbs_info->freq_table[index].frequency;

	/* Find out how long we have to be in hi and lo freqs */
	if (freq_hi == freq_lo) {
		dbs_info->freq_lo = 0;
		dbs_info->freq_lo_delay_us = 0;
		return freq_lo;
	}
	delay_hi_us = (freq_avg - freq_lo) * dbs_data->sampling_rate;
	delay_hi_us += (freq_hi - freq_lo) / 2;
	delay_hi_us /= freq_hi - freq_lo;
	dbs_info->freq_hi_delay_us = delay_hi_us;
	dbs_info->freq_lo = freq_lo;
	dbs_info->freq_lo_delay_us = dbs_data->sampling_rate - delay_hi_us;
	return freq_hi;
}

static void ondemand_powersave_bias_init(struct cpufreq_policy *policy)
{
	unsigned int cpu = policy->cpu;
	struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu);

	dbs_info->freq_table = cpufreq_frequency_get_table(cpu);
	dbs_info->freq_lo = 0;
}

static void dbs_freq_increase(struct cpufreq_policy *policy, unsigned int freq)
{
	struct policy_dbs_info *policy_dbs = policy->governor_data;
	struct dbs_data *dbs_data = policy_dbs->dbs_data;
	struct od_dbs_tuners *od_tuners = dbs_data->tuners;

	if (od_tuners->powersave_bias)
		freq = od_ops.powersave_bias_target(policy, freq,
				CPUFREQ_RELATION_H);
	else if (policy->cur == policy->max)
		return;

	__cpufreq_driver_target(policy, freq, od_tuners->powersave_bias ?
			CPUFREQ_RELATION_L : CPUFREQ_RELATION_H);
}

/*
 * Every sampling_rate, we check, if current idle time is less than 20%
 * (default), then we try to increase frequency. Else, we adjust the frequency
 * proportional to load.
 */
static void od_update(struct cpufreq_policy *policy)
{
	struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, policy->cpu);
	struct policy_dbs_info *policy_dbs = dbs_info->cdbs.policy_dbs;
	struct dbs_data *dbs_data = policy_dbs->dbs_data;
	struct od_dbs_tuners *od_tuners = dbs_data->tuners;
	unsigned int load = dbs_update(policy);

	dbs_info->freq_lo = 0;

	/* Check for frequency increase */
	if (load > dbs_data->up_threshold) {
		/* If switching to max speed, apply sampling_down_factor */
		if (policy->cur < policy->max)
			policy_dbs->rate_mult = dbs_data->sampling_down_factor;
		dbs_freq_increase(policy, policy->max);
	} else {
		/* Calculate the next frequency proportional to load */
		unsigned int freq_next, min_f, max_f;

		min_f = policy->cpuinfo.min_freq;
		max_f = policy->cpuinfo.max_freq;
		freq_next = min_f + load * (max_f - min_f) / 100;

		/* No longer fully busy, reset rate_mult */
		policy_dbs->rate_mult = 1;

		if (od_tuners->powersave_bias)
			freq_next = od_ops.powersave_bias_target(policy,
								 freq_next,
								 CPUFREQ_RELATION_L);

		__cpufreq_driver_target(policy, freq_next, CPUFREQ_RELATION_C);
	}
}

static unsigned int od_dbs_timer(struct cpufreq_policy *policy)
{
	struct policy_dbs_info *policy_dbs = policy->governor_data;
	struct dbs_data *dbs_data = policy_dbs->dbs_data;
	struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, policy->cpu);
	int sample_type = dbs_info->sample_type;

	/* Common NORMAL_SAMPLE setup */
	dbs_info->sample_type = OD_NORMAL_SAMPLE;
	/*
	 * OD_SUB_SAMPLE doesn't make sense if sample_delay_ns is 0, so ignore
	 * it then.
	 */
	if (sample_type == OD_SUB_SAMPLE && policy_dbs->sample_delay_ns > 0) {
		__cpufreq_driver_target(policy, dbs_info->freq_lo,
					CPUFREQ_RELATION_H);
		return dbs_info->freq_lo_delay_us;
	}

	od_update(policy);

	if (dbs_info->freq_lo) {
		/* Setup timer for SUB_SAMPLE */
		dbs_info->sample_type = OD_SUB_SAMPLE;
		return dbs_info->freq_hi_delay_us;
	}

	return dbs_data->sampling_rate * policy_dbs->rate_mult;
}

/************************** sysfs interface ************************/
static struct dbs_governor od_dbs_gov;

static ssize_t store_io_is_busy(struct dbs_data *dbs_data, const char *buf,
		size_t count)
{
	unsigned int input;
	int ret;

	ret = sscanf(buf, "%u", &input);
	if (ret != 1)
		return -EINVAL;
	dbs_data->io_is_busy = !!input;

	/* we need to re-evaluate prev_cpu_idle */
	gov_update_cpu_data(&od_dbs_gov, dbs_data);

	return count;
}

static ssize_t store_up_threshold(struct dbs_data *dbs_data, const char *buf,
		size_t count)
{
	unsigned int input;
	int ret;
	ret = sscanf(buf, "%u", &input);

	if (ret != 1 || input > MAX_FREQUENCY_UP_THRESHOLD ||
			input < MIN_FREQUENCY_UP_THRESHOLD) {
		return -EINVAL;
	}

	dbs_data->up_threshold = input;
	return count;
}

static ssize_t store_sampling_down_factor(struct dbs_data *dbs_data,
		const char *buf, size_t count)
{
	struct policy_dbs_info *policy_dbs;
	unsigned int input;
	int ret;
	ret = sscanf(buf, "%u", &input);

	if (ret != 1 || input > MAX_SAMPLING_DOWN_FACTOR || input < 1)
		return -EINVAL;

	dbs_data->sampling_down_factor = input;

	/* Reset down sampling multiplier in case it was active */
	list_for_each_entry(policy_dbs, &dbs_data->policy_dbs_list, list) {
		/*
		 * Doing this without locking might lead to using different
		 * rate_mult values in od_update() and od_dbs_timer().
		 */
		mutex_lock(&policy_dbs->timer_mutex);
		policy_dbs->rate_mult = 1;
		mutex_unlock(&policy_dbs->timer_mutex);
	}

	return count;
}

static ssize_t store_ignore_nice_load(struct dbs_data *dbs_data,
		const char *buf, size_t count)
{
	unsigned int input;
	int ret;

	ret = sscanf(buf, "%u", &input);
	if (ret != 1)
		return -EINVAL;

	if (input > 1)
		input = 1;

	if (input == dbs_data->ignore_nice_load) { /* nothing to do */
		return count;
	}
	dbs_data->ignore_nice_load = input;

	/* we need to re-evaluate prev_cpu_idle */
	gov_update_cpu_data(&od_dbs_gov, dbs_data);

	return count;
}

static ssize_t store_powersave_bias(struct dbs_data *dbs_data, const char *buf,
		size_t count)
{
	struct od_dbs_tuners *od_tuners = dbs_data->tuners;
	struct policy_dbs_info *policy_dbs;
	unsigned int input;
	int ret;
	ret = sscanf(buf, "%u", &input);

	if (ret != 1)
		return -EINVAL;

	if (input > 1000)
		input = 1000;

	od_tuners->powersave_bias = input;

	list_for_each_entry(policy_dbs, &dbs_data->policy_dbs_list, list)
		ondemand_powersave_bias_init(policy_dbs->policy);

	return count;
}

gov_show_one_common(sampling_rate);
gov_show_one_common(up_threshold);
gov_show_one_common(sampling_down_factor);
gov_show_one_common(ignore_nice_load);
gov_show_one_common(min_sampling_rate);
gov_show_one_common(io_is_busy);
gov_show_one(od, powersave_bias);

gov_attr_rw(sampling_rate);
gov_attr_rw(io_is_busy);
gov_attr_rw(up_threshold);
gov_attr_rw(sampling_down_factor);
gov_attr_rw(ignore_nice_load);
gov_attr_rw(powersave_bias);
gov_attr_ro(min_sampling_rate);

static struct attribute *od_attributes[] = {
	&min_sampling_rate.attr,
	&sampling_rate.attr,
	&up_threshold.attr,
	&sampling_down_factor.attr,
	&ignore_nice_load.attr,
	&powersave_bias.attr,
	&io_is_busy.attr,
	NULL
};

/************************** sysfs end ************************/

static int od_init(struct dbs_data *dbs_data, bool notify)
{
	struct od_dbs_tuners *tuners;
	u64 idle_time;
	int cpu;

	tuners = kzalloc(sizeof(*tuners), GFP_KERNEL);
	if (!tuners) {
		pr_err("%s: kzalloc failed\n", __func__);
		return -ENOMEM;
	}

	cpu = get_cpu();
	idle_time = get_cpu_idle_time_us(cpu, NULL);
	put_cpu();
	if (idle_time != -1ULL) {
		/* Idle micro accounting is supported. Use finer thresholds */
		dbs_data->up_threshold = MICRO_FREQUENCY_UP_THRESHOLD;
		/*
		 * In nohz/micro accounting case we set the minimum frequency
		 * not depending on HZ, but fixed (very low). The deferred
		 * timer might skip some samples if idle/sleeping as needed.
		*/
		dbs_data->min_sampling_rate = MICRO_FREQUENCY_MIN_SAMPLE_RATE;
	} else {
		dbs_data->up_threshold = DEF_FREQUENCY_UP_THRESHOLD;

		/* For correct statistics, we need 10 ticks for each measure */
		dbs_data->min_sampling_rate = MIN_SAMPLING_RATE_RATIO *
			jiffies_to_usecs(10);
	}

	dbs_data->sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR;
	dbs_data->ignore_nice_load = 0;
	tuners->powersave_bias = default_powersave_bias;
	dbs_data->io_is_busy = should_io_be_busy();

	dbs_data->tuners = tuners;
	return 0;
}

static void od_exit(struct dbs_data *dbs_data, bool notify)
{
	kfree(dbs_data->tuners);
}

static void od_start(struct cpufreq_policy *policy)
{
	struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, policy->cpu);

	dbs_info->sample_type = OD_NORMAL_SAMPLE;
	ondemand_powersave_bias_init(policy);
}

define_get_cpu_dbs_routines(od_cpu_dbs_info);

static struct od_ops od_ops = {
	.powersave_bias_target = generic_powersave_bias_target,
};

static struct dbs_governor od_dbs_gov = {
	.gov = {
		.name = "ondemand",
		.governor = cpufreq_governor_dbs,
		.max_transition_latency	= TRANSITION_LATENCY_LIMIT,
		.owner = THIS_MODULE,
	},
	.kobj_type = { .default_attrs = od_attributes },
	.get_cpu_cdbs = get_cpu_cdbs,
	.gov_dbs_timer = od_dbs_timer,
	.init = od_init,
	.exit = od_exit,
	.start = od_start,
};

#define CPU_FREQ_GOV_ONDEMAND	(&od_dbs_gov.gov)

static void od_set_powersave_bias(unsigned int powersave_bias)
{
	struct cpufreq_policy *policy;
	struct dbs_data *dbs_data;
	struct od_dbs_tuners *od_tuners;
	unsigned int cpu;
	cpumask_t done;

	default_powersave_bias = powersave_bias;
	cpumask_clear(&done);

	get_online_cpus();
	for_each_online_cpu(cpu) {
		struct policy_dbs_info *policy_dbs;

		if (cpumask_test_cpu(cpu, &done))
			continue;

		policy_dbs = per_cpu(od_cpu_dbs_info, cpu).cdbs.policy_dbs;
		if (!policy_dbs)
			continue;

		policy = policy_dbs->policy;
		cpumask_or(&done, &done, policy->cpus);

		if (policy->governor != CPU_FREQ_GOV_ONDEMAND)
			continue;

		dbs_data = policy_dbs->dbs_data;
		od_tuners = dbs_data->tuners;
		od_tuners->powersave_bias = default_powersave_bias;
	}
	put_online_cpus();
}

void od_register_powersave_bias_handler(unsigned int (*f)
		(struct cpufreq_policy *, unsigned int, unsigned int),
		unsigned int powersave_bias)
{
	od_ops.powersave_bias_target = f;
	od_set_powersave_bias(powersave_bias);
}
EXPORT_SYMBOL_GPL(od_register_powersave_bias_handler);

void od_unregister_powersave_bias_handler(void)
{
	od_ops.powersave_bias_target = generic_powersave_bias_target;
	od_set_powersave_bias(0);
}
EXPORT_SYMBOL_GPL(od_unregister_powersave_bias_handler);

static int __init cpufreq_gov_dbs_init(void)
{
	return cpufreq_register_governor(CPU_FREQ_GOV_ONDEMAND);
}

static void __exit cpufreq_gov_dbs_exit(void)
{
	cpufreq_unregister_governor(CPU_FREQ_GOV_ONDEMAND);
}

MODULE_AUTHOR("Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>");
MODULE_AUTHOR("Alexey Starikovskiy <alexey.y.starikovskiy@intel.com>");
MODULE_DESCRIPTION("'cpufreq_ondemand' - A dynamic cpufreq governor for "
	"Low Latency Frequency Transition capable processors");
MODULE_LICENSE("GPL");

#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND
struct cpufreq_governor *cpufreq_default_governor(void)
{
	return CPU_FREQ_GOV_ONDEMAND;
}

fs_initcall(cpufreq_gov_dbs_init);
#else
module_init(cpufreq_gov_dbs_init);
#endif
module_exit(cpufreq_gov_dbs_exit);
Commit	Line	Data
1da177e4 LT	1	/*
	2	* drivers/cpufreq/cpufreq_ondemand.c
	3	*
	4	* Copyright (C) 2001 Russell King
	5	* (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
	6	* Jun Nakajima <jun.nakajima@intel.com>
	7	*
	8	* This program is free software; you can redistribute it and/or modify
	9	* it under the terms of the GNU General Public License version 2 as
	10	* published by the Free Software Foundation.
	11	*/
	12
4471a34f VK	13	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
4471a34f VK	14
5ff0a268	15	#include <linux/cpu.h>
4471a34f	16	#include <linux/percpu-defs.h>
4d5dcc42	17	#include <linux/slab.h>
80800913	18	#include <linux/tick.h>
4471a34f	19	#include "cpufreq_governor.h"
1da177e4	20
06eb09d1	21	/* On-demand governor macros */
1da177e4	22	#define DEF_FREQUENCY_UP_THRESHOLD (80)
3f78a9f7 DN	23	#define DEF_SAMPLING_DOWN_FACTOR (1)
3f78a9f7 DN	24	#define MAX_SAMPLING_DOWN_FACTOR (100000)
80800913	25	#define MICRO_FREQUENCY_UP_THRESHOLD (95)
cef9615a	26	#define MICRO_FREQUENCY_MIN_SAMPLE_RATE (10000)
c29f1403	27	#define MIN_FREQUENCY_UP_THRESHOLD (11)
1da177e4 LT	28	#define MAX_FREQUENCY_UP_THRESHOLD (100)
1da177e4 LT	29
4471a34f	30	static DEFINE_PER_CPU(struct od_cpu_dbs_info_s, od_cpu_dbs_info);
1da177e4	31
a33cce1c	32	static struct dbs_governor od_dbs_gov;
fb30809e JS	33	static struct od_ops od_ops;
fb30809e JS	34
c2837558 JS	35	static unsigned int default_powersave_bias;
c2837558 JS	36
4471a34f VK	37	/*
	38	* Not all CPUs want IO time to be accounted as busy; this depends on how
	39	* efficient idling at a higher frequency/voltage is.
	40	* Pavel Machek says this is not so for various generations of AMD and old
	41	* Intel systems.
06eb09d1	42	* Mike Chan (android.com) claims this is also not true for ARM.
4471a34f VK	43	* Because of this, whitelist specific known (series) of CPUs by default, and
	44	* leave all others up to the user.
	45	*/
	46	static int should_io_be_busy(void)
	47	{
	48	#if defined(CONFIG_X86)
	49	/*
06eb09d1	50	* For Intel, Core 2 (model 15) and later have an efficient idle.
4471a34f VK	51	*/
	52	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
	53	boot_cpu_data.x86 == 6 &&
	54	boot_cpu_data.x86_model >= 15)
	55	return 1;
	56	#endif
	57	return 0;
6b8fcd90 AV	58	}
6b8fcd90 AV	59
05ca0350 AS	60	/*
05ca0350 AS	61	* Find right freq to be set now with powersave_bias on.
07aa4402 RW	62	* Returns the freq_hi to be used right now and will set freq_hi_delay_us,
07aa4402 RW	63	* freq_lo, and freq_lo_delay_us in percpu area for averaging freqs.
05ca0350	64	*/
fb30809e	65	static unsigned int generic_powersave_bias_target(struct cpufreq_policy *policy,
4471a34f	66	unsigned int freq_next, unsigned int relation)
05ca0350 AS	67	{
	68	unsigned int freq_req, freq_reduc, freq_avg;
	69	unsigned int freq_hi, freq_lo;
	70	unsigned int index = 0;
07aa4402	71	unsigned int delay_hi_us;
4471a34f	72	struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info,
245b2e70	73	policy->cpu);
bc505475 RW	74	struct policy_dbs_info *policy_dbs = policy->governor_data;
bc505475 RW	75	struct dbs_data *dbs_data = policy_dbs->dbs_data;
4d5dcc42	76	struct od_dbs_tuners *od_tuners = dbs_data->tuners;
05ca0350 AS	77
	78	if (!dbs_info->freq_table) {
	79	dbs_info->freq_lo = 0;
07aa4402	80	dbs_info->freq_lo_delay_us = 0;
05ca0350 AS	81	return freq_next;
	82	}
	83
	84	cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_next,
	85	relation, &index);
	86	freq_req = dbs_info->freq_table[index].frequency;
4d5dcc42	87	freq_reduc = freq_req * od_tuners->powersave_bias / 1000;
05ca0350 AS	88	freq_avg = freq_req - freq_reduc;
	89
	90	/* Find freq bounds for freq_avg in freq_table */
	91	index = 0;
	92	cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg,
	93	CPUFREQ_RELATION_H, &index);
	94	freq_lo = dbs_info->freq_table[index].frequency;
	95	index = 0;
	96	cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg,
	97	CPUFREQ_RELATION_L, &index);
	98	freq_hi = dbs_info->freq_table[index].frequency;
	99
	100	/* Find out how long we have to be in hi and lo freqs */
	101	if (freq_hi == freq_lo) {
	102	dbs_info->freq_lo = 0;
07aa4402	103	dbs_info->freq_lo_delay_us = 0;
05ca0350 AS	104	return freq_lo;
05ca0350 AS	105	}
07aa4402 RW	106	delay_hi_us = (freq_avg - freq_lo) * dbs_data->sampling_rate;
	107	delay_hi_us += (freq_hi - freq_lo) / 2;
	108	delay_hi_us /= freq_hi - freq_lo;
	109	dbs_info->freq_hi_delay_us = delay_hi_us;
05ca0350	110	dbs_info->freq_lo = freq_lo;
07aa4402	111	dbs_info->freq_lo_delay_us = dbs_data->sampling_rate - delay_hi_us;
05ca0350 AS	112	return freq_hi;
	113	}
	114
d1db75ff	115	static void ondemand_powersave_bias_init(struct cpufreq_policy *policy)
05ca0350	116	{
d1db75ff RW	117	unsigned int cpu = policy->cpu;
	118	struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu);
	119
	120	dbs_info->freq_table = cpufreq_frequency_get_table(cpu);
	121	dbs_info->freq_lo = 0;
05ca0350 AS	122	}
05ca0350 AS	123
3a3e9e06	124	static void dbs_freq_increase(struct cpufreq_policy *policy, unsigned int freq)
4471a34f	125	{
bc505475 RW	126	struct policy_dbs_info *policy_dbs = policy->governor_data;
bc505475 RW	127	struct dbs_data *dbs_data = policy_dbs->dbs_data;
4d5dcc42 VK	128	struct od_dbs_tuners *od_tuners = dbs_data->tuners;
	129
	130	if (od_tuners->powersave_bias)
3a3e9e06	131	freq = od_ops.powersave_bias_target(policy, freq,
fb30809e	132	CPUFREQ_RELATION_H);
3a3e9e06	133	else if (policy->cur == policy->max)
4471a34f	134	return;
0e625ac1	135
3a3e9e06	136	__cpufreq_driver_target(policy, freq, od_tuners->powersave_bias ?
4471a34f VK	137	CPUFREQ_RELATION_L : CPUFREQ_RELATION_H);
	138	}
	139
	140	/*
	141	* Every sampling_rate, we check, if current idle time is less than 20%
dfa5bb62 SK	142	* (default), then we try to increase frequency. Else, we adjust the frequency
dfa5bb62 SK	143	* proportional to load.
4471a34f	144	*/
4cccf755	145	static void od_update(struct cpufreq_policy *policy)
1da177e4	146	{
4cccf755	147	struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, policy->cpu);
bc505475	148	struct policy_dbs_info *policy_dbs = dbs_info->cdbs.policy_dbs;
bc505475	149	struct dbs_data *dbs_data = policy_dbs->dbs_data;
4d5dcc42	150	struct od_dbs_tuners *od_tuners = dbs_data->tuners;
4cccf755	151	unsigned int load = dbs_update(policy);
4471a34f VK	152
	153	dbs_info->freq_lo = 0;
	154
	155	/* Check for frequency increase */
ff4b1789	156	if (load > dbs_data->up_threshold) {
4471a34f VK	157	/* If switching to max speed, apply sampling_down_factor */
4471a34f VK	158	if (policy->cur < policy->max)
57dc3bcd	159	policy_dbs->rate_mult = dbs_data->sampling_down_factor;
4471a34f	160	dbs_freq_increase(policy, policy->max);
dfa5bb62 SK	161	} else {
dfa5bb62 SK	162	/* Calculate the next frequency proportional to load */
6393d6a1 SK	163	unsigned int freq_next, min_f, max_f;
	164
	165	min_f = policy->cpuinfo.min_freq;
	166	max_f = policy->cpuinfo.max_freq;
	167	freq_next = min_f + load * (max_f - min_f) / 100;
4471a34f VK	168
4471a34f VK	169	/* No longer fully busy, reset rate_mult */
57dc3bcd	170	policy_dbs->rate_mult = 1;
4471a34f	171
a7f35cff RW	172	if (od_tuners->powersave_bias)
	173	freq_next = od_ops.powersave_bias_target(policy,
	174	freq_next,
	175	CPUFREQ_RELATION_L);
	176
6393d6a1	177	__cpufreq_driver_target(policy, freq_next, CPUFREQ_RELATION_C);
4471a34f	178	}
1da177e4 LT	179	}
1da177e4 LT	180
9be4fd2c	181	static unsigned int od_dbs_timer(struct cpufreq_policy *policy)
4471a34f	182	{
bc505475 RW	183	struct policy_dbs_info *policy_dbs = policy->governor_data;
bc505475 RW	184	struct dbs_data *dbs_data = policy_dbs->dbs_data;
d10b5eb5	185	struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, policy->cpu);
6e96c5b3	186	int sample_type = dbs_info->sample_type;
4447266b	187
4471a34f	188	/* Common NORMAL_SAMPLE setup */
43e0ee36	189	dbs_info->sample_type = OD_NORMAL_SAMPLE;
4cccf755 RW	190	/*
	191	* OD_SUB_SAMPLE doesn't make sense if sample_delay_ns is 0, so ignore
	192	* it then.
	193	*/
	194	if (sample_type == OD_SUB_SAMPLE && policy_dbs->sample_delay_ns > 0) {
43e0ee36	195	__cpufreq_driver_target(policy, dbs_info->freq_lo,
42994af6	196	CPUFREQ_RELATION_H);
07aa4402	197	return dbs_info->freq_lo_delay_us;
6e96c5b3 RW	198	}
	199
	200	od_update(policy);
	201
	202	if (dbs_info->freq_lo) {
	203	/* Setup timer for SUB_SAMPLE */
	204	dbs_info->sample_type = OD_SUB_SAMPLE;
07aa4402	205	return dbs_info->freq_hi_delay_us;
4471a34f VK	206	}
4471a34f VK	207
07aa4402	208	return dbs_data->sampling_rate * policy_dbs->rate_mult;
da53d61e FB	209	}
da53d61e FB	210
4471a34f	211	/************************ sysfs interface **********************/
7bdad34d	212	static struct dbs_governor od_dbs_gov;
1da177e4	213
4d5dcc42 VK	214	static ssize_t store_io_is_busy(struct dbs_data dbs_data, const char buf,
4d5dcc42 VK	215	size_t count)
19379b11 AV	216	{
	217	unsigned int input;
	218	int ret;
	219
	220	ret = sscanf(buf, "%u", &input);
	221	if (ret != 1)
	222	return -EINVAL;
8847e038	223	dbs_data->io_is_busy = !!input;
9366d840 SK	224
9366d840 SK	225	/* we need to re-evaluate prev_cpu_idle */
a33cce1c RW	226	gov_update_cpu_data(&od_dbs_gov, dbs_data);
a33cce1c RW	227
19379b11 AV	228	return count;
	229	}
	230
4d5dcc42 VK	231	static ssize_t store_up_threshold(struct dbs_data dbs_data, const char buf,
4d5dcc42 VK	232	size_t count)
1da177e4 LT	233	{
	234	unsigned int input;
	235	int ret;
ffac80e9	236	ret = sscanf(buf, "%u", &input);
1da177e4	237
32ee8c3e	238	if (ret != 1 \|\| input > MAX_FREQUENCY_UP_THRESHOLD \|\|
c29f1403	239	input < MIN_FREQUENCY_UP_THRESHOLD) {
1da177e4 LT	240	return -EINVAL;
1da177e4 LT	241	}
4bd4e428	242
ff4b1789	243	dbs_data->up_threshold = input;
1da177e4 LT	244	return count;
	245	}
	246
4d5dcc42 VK	247	static ssize_t store_sampling_down_factor(struct dbs_data *dbs_data,
4d5dcc42 VK	248	const char *buf, size_t count)
3f78a9f7	249	{
57dc3bcd RW	250	struct policy_dbs_info *policy_dbs;
57dc3bcd RW	251	unsigned int input;
3f78a9f7 DN	252	int ret;
	253	ret = sscanf(buf, "%u", &input);
	254
	255	if (ret != 1 \|\| input > MAX_SAMPLING_DOWN_FACTOR \|\| input < 1)
	256	return -EINVAL;
57dc3bcd	257
ff4b1789	258	dbs_data->sampling_down_factor = input;
3f78a9f7 DN	259
3f78a9f7 DN	260	/* Reset down sampling multiplier in case it was active */
57dc3bcd RW	261	list_for_each_entry(policy_dbs, &dbs_data->policy_dbs_list, list) {
	262	/*
	263	* Doing this without locking might lead to using different
	264	* rate_mult values in od_update() and od_dbs_timer().
	265	*/
	266	mutex_lock(&policy_dbs->timer_mutex);
	267	policy_dbs->rate_mult = 1;
	268	mutex_unlock(&policy_dbs->timer_mutex);
3f78a9f7	269	}
57dc3bcd	270
3f78a9f7 DN	271	return count;
	272	}
	273
6c4640c3 VK	274	static ssize_t store_ignore_nice_load(struct dbs_data *dbs_data,
6c4640c3 VK	275	const char *buf, size_t count)
3d5ee9e5 DJ	276	{
	277	unsigned int input;
	278	int ret;
	279
ffac80e9	280	ret = sscanf(buf, "%u", &input);
2b03f891	281	if (ret != 1)
3d5ee9e5 DJ	282	return -EINVAL;
3d5ee9e5 DJ	283
2b03f891	284	if (input > 1)
3d5ee9e5	285	input = 1;
32ee8c3e	286
ff4b1789	287	if (input == dbs_data->ignore_nice_load) { /* nothing to do */
3d5ee9e5 DJ	288	return count;
3d5ee9e5 DJ	289	}
ff4b1789	290	dbs_data->ignore_nice_load = input;
3d5ee9e5	291
ccb2fe20	292	/* we need to re-evaluate prev_cpu_idle */
a33cce1c	293	gov_update_cpu_data(&od_dbs_gov, dbs_data);
1ca3abdb	294
3d5ee9e5 DJ	295	return count;
	296	}
	297
4d5dcc42 VK	298	static ssize_t store_powersave_bias(struct dbs_data dbs_data, const char buf,
4d5dcc42 VK	299	size_t count)
05ca0350	300	{
4d5dcc42	301	struct od_dbs_tuners *od_tuners = dbs_data->tuners;
d1db75ff	302	struct policy_dbs_info *policy_dbs;
05ca0350 AS	303	unsigned int input;
	304	int ret;
	305	ret = sscanf(buf, "%u", &input);
	306
	307	if (ret != 1)
	308	return -EINVAL;
	309
	310	if (input > 1000)
	311	input = 1000;
	312
4d5dcc42	313	od_tuners->powersave_bias = input;
d1db75ff RW	314
	315	list_for_each_entry(policy_dbs, &dbs_data->policy_dbs_list, list)
	316	ondemand_powersave_bias_init(policy_dbs->policy);
	317
05ca0350 AS	318	return count;
	319	}
	320
c4435630 VK	321	gov_show_one_common(sampling_rate);
	322	gov_show_one_common(up_threshold);
	323	gov_show_one_common(sampling_down_factor);
	324	gov_show_one_common(ignore_nice_load);
	325	gov_show_one_common(min_sampling_rate);
8847e038	326	gov_show_one_common(io_is_busy);
c4435630 VK	327	gov_show_one(od, powersave_bias);
	328
	329	gov_attr_rw(sampling_rate);
	330	gov_attr_rw(io_is_busy);
	331	gov_attr_rw(up_threshold);
	332	gov_attr_rw(sampling_down_factor);
	333	gov_attr_rw(ignore_nice_load);
	334	gov_attr_rw(powersave_bias);
	335	gov_attr_ro(min_sampling_rate);
	336
	337	static struct attribute *od_attributes[] = {
	338	&min_sampling_rate.attr,
	339	&sampling_rate.attr,
	340	&up_threshold.attr,
	341	&sampling_down_factor.attr,
	342	&ignore_nice_load.attr,
	343	&powersave_bias.attr,
	344	&io_is_busy.attr,
1da177e4 LT	345	NULL
	346	};
	347
1da177e4 LT	348	/************************ sysfs end **********************/
1da177e4 LT	349
8e0484d2	350	static int od_init(struct dbs_data *dbs_data, bool notify)
4d5dcc42 VK	351	{
	352	struct od_dbs_tuners *tuners;
	353	u64 idle_time;
	354	int cpu;
	355
d5b73cd8	356	tuners = kzalloc(sizeof(*tuners), GFP_KERNEL);
4d5dcc42 VK	357	if (!tuners) {
	358	pr_err("%s: kzalloc failed\n", __func__);
	359	return -ENOMEM;
	360	}
	361
	362	cpu = get_cpu();
	363	idle_time = get_cpu_idle_time_us(cpu, NULL);
	364	put_cpu();
	365	if (idle_time != -1ULL) {
	366	/* Idle micro accounting is supported. Use finer thresholds */
ff4b1789	367	dbs_data->up_threshold = MICRO_FREQUENCY_UP_THRESHOLD;
4d5dcc42 VK	368	/*
	369	* In nohz/micro accounting case we set the minimum frequency
	370	* not depending on HZ, but fixed (very low). The deferred
	371	* timer might skip some samples if idle/sleeping as needed.
	372	*/
	373	dbs_data->min_sampling_rate = MICRO_FREQUENCY_MIN_SAMPLE_RATE;
	374	} else {
ff4b1789	375	dbs_data->up_threshold = DEF_FREQUENCY_UP_THRESHOLD;
4d5dcc42 VK	376
	377	/* For correct statistics, we need 10 ticks for each measure */
	378	dbs_data->min_sampling_rate = MIN_SAMPLING_RATE_RATIO *
	379	jiffies_to_usecs(10);
	380	}
	381
ff4b1789 VK	382	dbs_data->sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR;
ff4b1789 VK	383	dbs_data->ignore_nice_load = 0;
c2837558	384	tuners->powersave_bias = default_powersave_bias;
8847e038	385	dbs_data->io_is_busy = should_io_be_busy();
4d5dcc42 VK	386
4d5dcc42 VK	387	dbs_data->tuners = tuners;
4d5dcc42 VK	388	return 0;
	389	}
	390
8e0484d2	391	static void od_exit(struct dbs_data *dbs_data, bool notify)
4d5dcc42 VK	392	{
	393	kfree(dbs_data->tuners);
	394	}
	395
702c9e54 RW	396	static void od_start(struct cpufreq_policy *policy)
702c9e54 RW	397	{
d1db75ff	398	struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, policy->cpu);
702c9e54 RW	399
702c9e54 RW	400	dbs_info->sample_type = OD_NORMAL_SAMPLE;
d1db75ff	401	ondemand_powersave_bias_init(policy);
702c9e54 RW	402	}
702c9e54 RW	403
4471a34f	404	define_get_cpu_dbs_routines(od_cpu_dbs_info);
6b8fcd90	405
4471a34f	406	static struct od_ops od_ops = {
fb30809e	407	.powersave_bias_target = generic_powersave_bias_target,
4471a34f	408	};
2f8a835c	409
7bdad34d	410	static struct dbs_governor od_dbs_gov = {
af926185 RW	411	.gov = {
af926185 RW	412	.name = "ondemand",
906a6e5a	413	.governor = cpufreq_governor_dbs,
af926185 RW	414	.max_transition_latency = TRANSITION_LATENCY_LIMIT,
	415	.owner = THIS_MODULE,
	416	},
c4435630	417	.kobj_type = { .default_attrs = od_attributes },
4471a34f	418	.get_cpu_cdbs = get_cpu_cdbs,
4471a34f	419	.gov_dbs_timer = od_dbs_timer,
4d5dcc42 VK	420	.init = od_init,
4d5dcc42 VK	421	.exit = od_exit,
702c9e54	422	.start = od_start,
4471a34f	423	};
1da177e4	424
7bdad34d	425	#define CPU_FREQ_GOV_ONDEMAND (&od_dbs_gov.gov)
af926185	426
fb30809e JS	427	static void od_set_powersave_bias(unsigned int powersave_bias)
	428	{
	429	struct cpufreq_policy *policy;
	430	struct dbs_data *dbs_data;
	431	struct od_dbs_tuners *od_tuners;
	432	unsigned int cpu;
	433	cpumask_t done;
	434
c2837558	435	default_powersave_bias = powersave_bias;
fb30809e JS	436	cpumask_clear(&done);
	437
	438	get_online_cpus();
	439	for_each_online_cpu(cpu) {
e40e7b25	440	struct policy_dbs_info *policy_dbs;
44152cb8	441
fb30809e JS	442	if (cpumask_test_cpu(cpu, &done))
	443	continue;
	444
e40e7b25 RW	445	policy_dbs = per_cpu(od_cpu_dbs_info, cpu).cdbs.policy_dbs;
e40e7b25 RW	446	if (!policy_dbs)
c2837558	447	continue;
fb30809e	448
e40e7b25	449	policy = policy_dbs->policy;
fb30809e	450	cpumask_or(&done, &done, policy->cpus);
c2837558	451
af926185	452	if (policy->governor != CPU_FREQ_GOV_ONDEMAND)
c2837558 JS	453	continue;
c2837558 JS	454
bc505475	455	dbs_data = policy_dbs->dbs_data;
c2837558 JS	456	od_tuners = dbs_data->tuners;
c2837558 JS	457	od_tuners->powersave_bias = default_powersave_bias;
fb30809e JS	458	}
	459	put_online_cpus();
	460	}
	461
	462	void od_register_powersave_bias_handler(unsigned int (*f)
	463	(struct cpufreq_policy *, unsigned int, unsigned int),
	464	unsigned int powersave_bias)
	465	{
	466	od_ops.powersave_bias_target = f;
	467	od_set_powersave_bias(powersave_bias);
	468	}
	469	EXPORT_SYMBOL_GPL(od_register_powersave_bias_handler);
	470
	471	void od_unregister_powersave_bias_handler(void)
	472	{
	473	od_ops.powersave_bias_target = generic_powersave_bias_target;
	474	od_set_powersave_bias(0);
	475	}
	476	EXPORT_SYMBOL_GPL(od_unregister_powersave_bias_handler);
	477
1da177e4 LT	478	static int __init cpufreq_gov_dbs_init(void)
1da177e4 LT	479	{
af926185	480	return cpufreq_register_governor(CPU_FREQ_GOV_ONDEMAND);
1da177e4 LT	481	}
	482
	483	static void __exit cpufreq_gov_dbs_exit(void)
	484	{
af926185	485	cpufreq_unregister_governor(CPU_FREQ_GOV_ONDEMAND);
1da177e4 LT	486	}
1da177e4 LT	487
ffac80e9 VP	488	MODULE_AUTHOR("Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>");
	489	MODULE_AUTHOR("Alexey Starikovskiy <alexey.y.starikovskiy@intel.com>");
	490	MODULE_DESCRIPTION("'cpufreq_ondemand' - A dynamic cpufreq governor for "
2b03f891	491	"Low Latency Frequency Transition capable processors");
ffac80e9	492	MODULE_LICENSE("GPL");
1da177e4	493
6915719b	494	#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND
de1df26b RW	495	struct cpufreq_governor *cpufreq_default_governor(void)
de1df26b RW	496	{
af926185	497	return CPU_FREQ_GOV_ONDEMAND;
de1df26b RW	498	}
de1df26b RW	499
6915719b JW	500	fs_initcall(cpufreq_gov_dbs_init);
6915719b JW	501	#else
1da177e4	502	module_init(cpufreq_gov_dbs_init);
6915719b	503	#endif
1da177e4	504	module_exit(cpufreq_gov_dbs_exit);