2 * intel_pstate.c: Native P state management for Intel processors
4 * (C) Copyright 2012 Intel Corporation
5 * Author: Dirk Brandewie <dirk.j.brandewie@intel.com>
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; version 2
13 #include <linux/kernel.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/module.h>
16 #include <linux/ktime.h>
17 #include <linux/hrtimer.h>
18 #include <linux/tick.h>
19 #include <linux/slab.h>
20 #include <linux/sched.h>
21 #include <linux/list.h>
22 #include <linux/cpu.h>
23 #include <linux/cpufreq.h>
24 #include <linux/sysfs.h>
25 #include <linux/types.h>
27 #include <linux/debugfs.h>
28 #include <trace/events/power.h>
30 #include <asm/div64.h>
32 #include <asm/cpu_device_id.h>
34 #define SAMPLE_COUNT 3
37 #define int_tofp(X) ((int64_t)(X) << FRAC_BITS)
38 #define fp_toint(X) ((X) >> FRAC_BITS)
40 static inline int32_t mul_fp(int32_t x
, int32_t y
)
42 return ((int64_t)x
* (int64_t)y
) >> FRAC_BITS
;
45 static inline int32_t div_fp(int32_t x
, int32_t y
)
47 return div_s64((int64_t)x
<< FRAC_BITS
, (int64_t)y
);
79 struct timer_list timer
;
81 struct pstate_adjust_policy
*pstate_policy
;
82 struct pstate_data pstate
;
92 struct sample samples
[SAMPLE_COUNT
];
95 static struct cpudata
**all_cpu_data
;
96 struct pstate_adjust_policy
{
105 static struct pstate_adjust_policy default_policy
= {
106 .sample_rate_ms
= 10,
124 static struct perf_limits limits
= {
127 .max_perf
= int_tofp(1),
130 .max_policy_pct
= 100,
131 .max_sysfs_pct
= 100,
134 static inline void pid_reset(struct _pid
*pid
, int setpoint
, int busy
,
135 int deadband
, int integral
) {
136 pid
->setpoint
= setpoint
;
137 pid
->deadband
= deadband
;
138 pid
->integral
= int_tofp(integral
);
139 pid
->last_err
= setpoint
- busy
;
142 static inline void pid_p_gain_set(struct _pid
*pid
, int percent
)
144 pid
->p_gain
= div_fp(int_tofp(percent
), int_tofp(100));
147 static inline void pid_i_gain_set(struct _pid
*pid
, int percent
)
149 pid
->i_gain
= div_fp(int_tofp(percent
), int_tofp(100));
152 static inline void pid_d_gain_set(struct _pid
*pid
, int percent
)
155 pid
->d_gain
= div_fp(int_tofp(percent
), int_tofp(100));
158 static signed int pid_calc(struct _pid
*pid
, int busy
)
160 signed int err
, result
;
161 int32_t pterm
, dterm
, fp_error
;
162 int32_t integral_limit
;
164 err
= pid
->setpoint
- busy
;
165 fp_error
= int_tofp(err
);
167 if (abs(err
) <= pid
->deadband
)
170 pterm
= mul_fp(pid
->p_gain
, fp_error
);
172 pid
->integral
+= fp_error
;
174 /* limit the integral term */
175 integral_limit
= int_tofp(30);
176 if (pid
->integral
> integral_limit
)
177 pid
->integral
= integral_limit
;
178 if (pid
->integral
< -integral_limit
)
179 pid
->integral
= -integral_limit
;
181 dterm
= mul_fp(pid
->d_gain
, (err
- pid
->last_err
));
184 result
= pterm
+ mul_fp(pid
->integral
, pid
->i_gain
) + dterm
;
186 return (signed int)fp_toint(result
);
189 static inline void intel_pstate_busy_pid_reset(struct cpudata
*cpu
)
191 pid_p_gain_set(&cpu
->pid
, cpu
->pstate_policy
->p_gain_pct
);
192 pid_d_gain_set(&cpu
->pid
, cpu
->pstate_policy
->d_gain_pct
);
193 pid_i_gain_set(&cpu
->pid
, cpu
->pstate_policy
->i_gain_pct
);
196 cpu
->pstate_policy
->setpoint
,
198 cpu
->pstate_policy
->deadband
,
202 static inline void intel_pstate_idle_pid_reset(struct cpudata
*cpu
)
204 pid_p_gain_set(&cpu
->idle_pid
, cpu
->pstate_policy
->p_gain_pct
);
205 pid_d_gain_set(&cpu
->idle_pid
, cpu
->pstate_policy
->d_gain_pct
);
206 pid_i_gain_set(&cpu
->idle_pid
, cpu
->pstate_policy
->i_gain_pct
);
208 pid_reset(&cpu
->idle_pid
,
211 cpu
->pstate_policy
->deadband
,
215 static inline void intel_pstate_reset_all_pid(void)
218 for_each_online_cpu(cpu
) {
219 if (all_cpu_data
[cpu
])
220 intel_pstate_busy_pid_reset(all_cpu_data
[cpu
]);
224 /************************** debugfs begin ************************/
225 static int pid_param_set(void *data
, u64 val
)
228 intel_pstate_reset_all_pid();
231 static int pid_param_get(void *data
, u64
*val
)
236 DEFINE_SIMPLE_ATTRIBUTE(fops_pid_param
, pid_param_get
,
237 pid_param_set
, "%llu\n");
244 static struct pid_param pid_files
[] = {
245 {"sample_rate_ms", &default_policy
.sample_rate_ms
},
246 {"d_gain_pct", &default_policy
.d_gain_pct
},
247 {"i_gain_pct", &default_policy
.i_gain_pct
},
248 {"deadband", &default_policy
.deadband
},
249 {"setpoint", &default_policy
.setpoint
},
250 {"p_gain_pct", &default_policy
.p_gain_pct
},
254 static struct dentry
*debugfs_parent
;
255 static void intel_pstate_debug_expose_params(void)
259 debugfs_parent
= debugfs_create_dir("pstate_snb", NULL
);
260 if (IS_ERR_OR_NULL(debugfs_parent
))
262 while (pid_files
[i
].name
) {
263 debugfs_create_file(pid_files
[i
].name
, 0660,
264 debugfs_parent
, pid_files
[i
].value
,
270 /************************** debugfs end ************************/
272 /************************** sysfs begin ************************/
273 #define show_one(file_name, object) \
274 static ssize_t show_##file_name \
275 (struct kobject *kobj, struct attribute *attr, char *buf) \
277 return sprintf(buf, "%u\n", limits.object); \
280 static ssize_t
store_no_turbo(struct kobject
*a
, struct attribute
*b
,
281 const char *buf
, size_t count
)
285 ret
= sscanf(buf
, "%u", &input
);
288 limits
.no_turbo
= clamp_t(int, input
, 0 , 1);
293 static ssize_t
store_max_perf_pct(struct kobject
*a
, struct attribute
*b
,
294 const char *buf
, size_t count
)
298 ret
= sscanf(buf
, "%u", &input
);
302 limits
.max_sysfs_pct
= clamp_t(int, input
, 0 , 100);
303 limits
.max_perf_pct
= min(limits
.max_policy_pct
, limits
.max_sysfs_pct
);
304 limits
.max_perf
= div_fp(int_tofp(limits
.max_perf_pct
), int_tofp(100));
308 static ssize_t
store_min_perf_pct(struct kobject
*a
, struct attribute
*b
,
309 const char *buf
, size_t count
)
313 ret
= sscanf(buf
, "%u", &input
);
316 limits
.min_perf_pct
= clamp_t(int, input
, 0 , 100);
317 limits
.min_perf
= div_fp(int_tofp(limits
.min_perf_pct
), int_tofp(100));
322 show_one(no_turbo
, no_turbo
);
323 show_one(max_perf_pct
, max_perf_pct
);
324 show_one(min_perf_pct
, min_perf_pct
);
326 define_one_global_rw(no_turbo
);
327 define_one_global_rw(max_perf_pct
);
328 define_one_global_rw(min_perf_pct
);
330 static struct attribute
*intel_pstate_attributes
[] = {
337 static struct attribute_group intel_pstate_attr_group
= {
338 .attrs
= intel_pstate_attributes
,
340 static struct kobject
*intel_pstate_kobject
;
342 static void intel_pstate_sysfs_expose_params(void)
346 intel_pstate_kobject
= kobject_create_and_add("intel_pstate",
347 &cpu_subsys
.dev_root
->kobj
);
348 BUG_ON(!intel_pstate_kobject
);
349 rc
= sysfs_create_group(intel_pstate_kobject
,
350 &intel_pstate_attr_group
);
354 /************************** sysfs end ************************/
356 static int intel_pstate_min_pstate(void)
359 rdmsrl(MSR_PLATFORM_INFO
, value
);
360 return (value
>> 40) & 0xFF;
363 static int intel_pstate_max_pstate(void)
366 rdmsrl(MSR_PLATFORM_INFO
, value
);
367 return (value
>> 8) & 0xFF;
370 static int intel_pstate_turbo_pstate(void)
374 rdmsrl(MSR_NHM_TURBO_RATIO_LIMIT
, value
);
375 nont
= intel_pstate_max_pstate();
376 ret
= ((value
) & 255);
382 static void intel_pstate_get_min_max(struct cpudata
*cpu
, int *min
, int *max
)
384 int max_perf
= cpu
->pstate
.turbo_pstate
;
387 max_perf
= cpu
->pstate
.max_pstate
;
389 max_perf
= fp_toint(mul_fp(int_tofp(max_perf
), limits
.max_perf
));
390 *max
= clamp_t(int, max_perf
,
391 cpu
->pstate
.min_pstate
, cpu
->pstate
.turbo_pstate
);
393 min_perf
= fp_toint(mul_fp(int_tofp(max_perf
), limits
.min_perf
));
394 *min
= clamp_t(int, min_perf
,
395 cpu
->pstate
.min_pstate
, max_perf
);
398 static void intel_pstate_set_pstate(struct cpudata
*cpu
, int pstate
)
400 int max_perf
, min_perf
;
402 intel_pstate_get_min_max(cpu
, &min_perf
, &max_perf
);
404 pstate
= clamp_t(int, pstate
, min_perf
, max_perf
);
406 if (pstate
== cpu
->pstate
.current_pstate
)
410 trace_cpu_frequency(pstate
* 100000, cpu
->cpu
);
412 cpu
->pstate
.current_pstate
= pstate
;
413 wrmsrl(MSR_IA32_PERF_CTL
, pstate
<< 8);
417 static inline void intel_pstate_pstate_increase(struct cpudata
*cpu
, int steps
)
420 target
= cpu
->pstate
.current_pstate
+ steps
;
422 intel_pstate_set_pstate(cpu
, target
);
425 static inline void intel_pstate_pstate_decrease(struct cpudata
*cpu
, int steps
)
428 target
= cpu
->pstate
.current_pstate
- steps
;
429 intel_pstate_set_pstate(cpu
, target
);
432 static void intel_pstate_get_cpu_pstates(struct cpudata
*cpu
)
434 sprintf(cpu
->name
, "Intel 2nd generation core");
436 cpu
->pstate
.min_pstate
= intel_pstate_min_pstate();
437 cpu
->pstate
.max_pstate
= intel_pstate_max_pstate();
438 cpu
->pstate
.turbo_pstate
= intel_pstate_turbo_pstate();
441 * goto max pstate so we don't slow up boot if we are built-in if we are
442 * a module we will take care of it during normal operation
444 intel_pstate_set_pstate(cpu
, cpu
->pstate
.max_pstate
);
447 static inline void intel_pstate_calc_busy(struct cpudata
*cpu
,
448 struct sample
*sample
)
451 core_pct
= div64_u64(sample
->aperf
* 100, sample
->mperf
);
452 sample
->freq
= cpu
->pstate
.max_pstate
* core_pct
* 1000;
454 sample
->core_pct_busy
= core_pct
;
457 static inline void intel_pstate_sample(struct cpudata
*cpu
)
461 rdmsrl(MSR_IA32_APERF
, aperf
);
462 rdmsrl(MSR_IA32_MPERF
, mperf
);
463 cpu
->sample_ptr
= (cpu
->sample_ptr
+ 1) % SAMPLE_COUNT
;
464 cpu
->samples
[cpu
->sample_ptr
].aperf
= aperf
;
465 cpu
->samples
[cpu
->sample_ptr
].mperf
= mperf
;
466 cpu
->samples
[cpu
->sample_ptr
].aperf
-= cpu
->prev_aperf
;
467 cpu
->samples
[cpu
->sample_ptr
].mperf
-= cpu
->prev_mperf
;
469 intel_pstate_calc_busy(cpu
, &cpu
->samples
[cpu
->sample_ptr
]);
471 cpu
->prev_aperf
= aperf
;
472 cpu
->prev_mperf
= mperf
;
475 static inline void intel_pstate_set_sample_time(struct cpudata
*cpu
)
477 int sample_time
, delay
;
479 sample_time
= cpu
->pstate_policy
->sample_rate_ms
;
480 delay
= msecs_to_jiffies(sample_time
);
481 mod_timer_pinned(&cpu
->timer
, jiffies
+ delay
);
484 static inline void intel_pstate_idle_mode(struct cpudata
*cpu
)
489 static inline void intel_pstate_normal_mode(struct cpudata
*cpu
)
494 static inline int intel_pstate_get_scaled_busy(struct cpudata
*cpu
)
497 int32_t core_busy
, turbo_pstate
, current_pstate
;
499 core_busy
= int_tofp(cpu
->samples
[cpu
->sample_ptr
].core_pct_busy
);
500 turbo_pstate
= int_tofp(cpu
->pstate
.turbo_pstate
);
501 current_pstate
= int_tofp(cpu
->pstate
.current_pstate
);
502 busy_scaled
= mul_fp(core_busy
, div_fp(turbo_pstate
, current_pstate
));
504 return fp_toint(busy_scaled
);
507 static inline void intel_pstate_adjust_busy_pstate(struct cpudata
*cpu
)
515 busy_scaled
= intel_pstate_get_scaled_busy(cpu
);
517 ctl
= pid_calc(pid
, busy_scaled
);
521 intel_pstate_pstate_increase(cpu
, steps
);
523 intel_pstate_pstate_decrease(cpu
, steps
);
526 static inline void intel_pstate_adjust_idle_pstate(struct cpudata
*cpu
)
533 pid
= &cpu
->idle_pid
;
535 busy_scaled
= intel_pstate_get_scaled_busy(cpu
);
537 ctl
= pid_calc(pid
, 100 - busy_scaled
);
541 intel_pstate_pstate_decrease(cpu
, steps
);
543 intel_pstate_pstate_increase(cpu
, steps
);
545 if (cpu
->pstate
.current_pstate
== cpu
->pstate
.min_pstate
)
546 intel_pstate_normal_mode(cpu
);
549 static void intel_pstate_timer_func(unsigned long __data
)
551 struct cpudata
*cpu
= (struct cpudata
*) __data
;
553 intel_pstate_sample(cpu
);
556 intel_pstate_adjust_busy_pstate(cpu
);
558 intel_pstate_adjust_idle_pstate(cpu
);
560 #if defined(XPERF_FIX)
561 if (cpu
->pstate
.current_pstate
== cpu
->pstate
.min_pstate
) {
562 cpu
->min_pstate_count
++;
563 if (!(cpu
->min_pstate_count
% 5)) {
564 intel_pstate_set_pstate(cpu
, cpu
->pstate
.max_pstate
);
565 intel_pstate_idle_mode(cpu
);
568 cpu
->min_pstate_count
= 0;
570 intel_pstate_set_sample_time(cpu
);
573 #define ICPU(model, policy) \
574 { X86_VENDOR_INTEL, 6, model, X86_FEATURE_ANY, (unsigned long)&policy }
576 static const struct x86_cpu_id intel_pstate_cpu_ids
[] = {
577 ICPU(0x2a, default_policy
),
578 ICPU(0x2d, default_policy
),
581 MODULE_DEVICE_TABLE(x86cpu
, intel_pstate_cpu_ids
);
583 static int intel_pstate_init_cpu(unsigned int cpunum
)
586 const struct x86_cpu_id
*id
;
589 id
= x86_match_cpu(intel_pstate_cpu_ids
);
593 all_cpu_data
[cpunum
] = kzalloc(sizeof(struct cpudata
), GFP_KERNEL
);
594 if (!all_cpu_data
[cpunum
])
597 cpu
= all_cpu_data
[cpunum
];
599 intel_pstate_get_cpu_pstates(cpu
);
603 (struct pstate_adjust_policy
*)id
->driver_data
;
604 init_timer_deferrable(&cpu
->timer
);
605 cpu
->timer
.function
= intel_pstate_timer_func
;
608 cpu
->timer
.expires
= jiffies
+ HZ
/100;
609 intel_pstate_busy_pid_reset(cpu
);
610 intel_pstate_idle_pid_reset(cpu
);
611 intel_pstate_sample(cpu
);
612 intel_pstate_set_pstate(cpu
, cpu
->pstate
.max_pstate
);
614 add_timer_on(&cpu
->timer
, cpunum
);
616 pr_info("Intel pstate controlling: cpu %d\n", cpunum
);
621 static unsigned int intel_pstate_get(unsigned int cpu_num
)
623 struct sample
*sample
;
626 cpu
= all_cpu_data
[cpu_num
];
629 sample
= &cpu
->samples
[cpu
->sample_ptr
];
633 static int intel_pstate_set_policy(struct cpufreq_policy
*policy
)
637 cpu
= all_cpu_data
[policy
->cpu
];
639 if (!policy
->cpuinfo
.max_freq
)
642 if (policy
->policy
== CPUFREQ_POLICY_PERFORMANCE
) {
643 limits
.min_perf_pct
= 100;
644 limits
.min_perf
= int_tofp(1);
645 limits
.max_perf_pct
= 100;
646 limits
.max_perf
= int_tofp(1);
650 limits
.min_perf_pct
= (policy
->min
* 100) / policy
->cpuinfo
.max_freq
;
651 limits
.min_perf_pct
= clamp_t(int, limits
.min_perf_pct
, 0 , 100);
652 limits
.min_perf
= div_fp(int_tofp(limits
.min_perf_pct
), int_tofp(100));
654 limits
.max_policy_pct
= policy
->max
* 100 / policy
->cpuinfo
.max_freq
;
655 limits
.max_policy_pct
= clamp_t(int, limits
.max_policy_pct
, 0 , 100);
656 limits
.max_perf_pct
= min(limits
.max_policy_pct
, limits
.max_sysfs_pct
);
657 limits
.max_perf
= div_fp(int_tofp(limits
.max_perf_pct
), int_tofp(100));
662 static int intel_pstate_verify_policy(struct cpufreq_policy
*policy
)
664 cpufreq_verify_within_limits(policy
,
665 policy
->cpuinfo
.min_freq
,
666 policy
->cpuinfo
.max_freq
);
668 if ((policy
->policy
!= CPUFREQ_POLICY_POWERSAVE
) &&
669 (policy
->policy
!= CPUFREQ_POLICY_PERFORMANCE
))
675 static int __cpuinit
intel_pstate_cpu_exit(struct cpufreq_policy
*policy
)
677 int cpu
= policy
->cpu
;
679 del_timer(&all_cpu_data
[cpu
]->timer
);
680 kfree(all_cpu_data
[cpu
]);
681 all_cpu_data
[cpu
] = NULL
;
685 static int __cpuinit
intel_pstate_cpu_init(struct cpufreq_policy
*policy
)
687 int rc
, min_pstate
, max_pstate
;
690 rc
= intel_pstate_init_cpu(policy
->cpu
);
694 cpu
= all_cpu_data
[policy
->cpu
];
696 if (!limits
.no_turbo
&&
697 limits
.min_perf_pct
== 100 && limits
.max_perf_pct
== 100)
698 policy
->policy
= CPUFREQ_POLICY_PERFORMANCE
;
700 policy
->policy
= CPUFREQ_POLICY_POWERSAVE
;
702 intel_pstate_get_min_max(cpu
, &min_pstate
, &max_pstate
);
703 policy
->min
= min_pstate
* 100000;
704 policy
->max
= max_pstate
* 100000;
706 /* cpuinfo and default policy values */
707 policy
->cpuinfo
.min_freq
= cpu
->pstate
.min_pstate
* 100000;
708 policy
->cpuinfo
.max_freq
= cpu
->pstate
.turbo_pstate
* 100000;
709 policy
->cpuinfo
.transition_latency
= CPUFREQ_ETERNAL
;
710 cpumask_set_cpu(policy
->cpu
, policy
->cpus
);
715 static struct cpufreq_driver intel_pstate_driver
= {
716 .flags
= CPUFREQ_CONST_LOOPS
,
717 .verify
= intel_pstate_verify_policy
,
718 .setpolicy
= intel_pstate_set_policy
,
719 .get
= intel_pstate_get
,
720 .init
= intel_pstate_cpu_init
,
721 .exit
= intel_pstate_cpu_exit
,
722 .name
= "intel_pstate",
723 .owner
= THIS_MODULE
,
726 static int __initdata no_load
;
728 static int intel_pstate_msrs_not_valid(void)
730 /* Check that all the msr's we are using are valid. */
731 u64 aperf
, mperf
, tmp
;
733 rdmsrl(MSR_IA32_APERF
, aperf
);
734 rdmsrl(MSR_IA32_MPERF
, mperf
);
736 if (!intel_pstate_min_pstate() ||
737 !intel_pstate_max_pstate() ||
738 !intel_pstate_turbo_pstate())
741 rdmsrl(MSR_IA32_APERF
, tmp
);
745 rdmsrl(MSR_IA32_MPERF
, tmp
);
751 static int __init
intel_pstate_init(void)
754 const struct x86_cpu_id
*id
;
759 id
= x86_match_cpu(intel_pstate_cpu_ids
);
763 if (intel_pstate_msrs_not_valid())
766 pr_info("Intel P-state driver initializing.\n");
768 all_cpu_data
= vmalloc(sizeof(void *) * num_possible_cpus());
771 memset(all_cpu_data
, 0, sizeof(void *) * num_possible_cpus());
773 rc
= cpufreq_register_driver(&intel_pstate_driver
);
777 intel_pstate_debug_expose_params();
778 intel_pstate_sysfs_expose_params();
782 for_each_online_cpu(cpu
) {
783 if (all_cpu_data
[cpu
]) {
784 del_timer_sync(&all_cpu_data
[cpu
]->timer
);
785 kfree(all_cpu_data
[cpu
]);
793 device_initcall(intel_pstate_init
);
795 static int __init
intel_pstate_setup(char *str
)
800 if (!strcmp(str
, "disable"))
804 early_param("intel_pstate", intel_pstate_setup
);
806 MODULE_AUTHOR("Dirk Brandewie <dirk.j.brandewie@intel.com>");
807 MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors");
808 MODULE_LICENSE("GPL");