1 // SPDX-License-Identifier: GPL-2.0-only
3 * intel_pstate.c: Native P state management for Intel processors
5 * (C) Copyright 2012 Intel Corporation
6 * Author: Dirk Brandewie <dirk.j.brandewie@intel.com>
9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
11 #include <linux/kernel.h>
12 #include <linux/kernel_stat.h>
13 #include <linux/module.h>
14 #include <linux/ktime.h>
15 #include <linux/hrtimer.h>
16 #include <linux/tick.h>
17 #include <linux/slab.h>
18 #include <linux/sched/cpufreq.h>
19 #include <linux/list.h>
20 #include <linux/cpu.h>
21 #include <linux/cpufreq.h>
22 #include <linux/sysfs.h>
23 #include <linux/types.h>
25 #include <linux/acpi.h>
26 #include <linux/vmalloc.h>
27 #include <linux/pm_qos.h>
28 #include <trace/events/power.h>
30 #include <asm/div64.h>
32 #include <asm/cpu_device_id.h>
33 #include <asm/cpufeature.h>
34 #include <asm/intel-family.h>
36 #define INTEL_PSTATE_SAMPLING_INTERVAL (10 * NSEC_PER_MSEC)
38 #define INTEL_CPUFREQ_TRANSITION_LATENCY 20000
39 #define INTEL_CPUFREQ_TRANSITION_DELAY_HWP 5000
40 #define INTEL_CPUFREQ_TRANSITION_DELAY 500
43 #include <acpi/processor.h>
44 #include <acpi/cppc_acpi.h>
48 #define int_tofp(X) ((int64_t)(X) << FRAC_BITS)
49 #define fp_toint(X) ((X) >> FRAC_BITS)
51 #define ONE_EIGHTH_FP ((int64_t)1 << (FRAC_BITS - 3))
54 #define EXT_FRAC_BITS (EXT_BITS + FRAC_BITS)
55 #define fp_ext_toint(X) ((X) >> EXT_FRAC_BITS)
56 #define int_ext_tofp(X) ((int64_t)(X) << EXT_FRAC_BITS)
58 static inline int32_t mul_fp(int32_t x
, int32_t y
)
60 return ((int64_t)x
* (int64_t)y
) >> FRAC_BITS
;
63 static inline int32_t div_fp(s64 x
, s64 y
)
65 return div64_s64((int64_t)x
<< FRAC_BITS
, y
);
68 static inline int ceiling_fp(int32_t x
)
73 mask
= (1 << FRAC_BITS
) - 1;
79 static inline int32_t percent_fp(int percent
)
81 return div_fp(percent
, 100);
84 static inline u64
mul_ext_fp(u64 x
, u64 y
)
86 return (x
* y
) >> EXT_FRAC_BITS
;
89 static inline u64
div_ext_fp(u64 x
, u64 y
)
91 return div64_u64(x
<< EXT_FRAC_BITS
, y
);
94 static inline int32_t percent_ext_fp(int percent
)
96 return div_ext_fp(percent
, 100);
100 * struct sample - Store performance sample
101 * @core_avg_perf: Ratio of APERF/MPERF which is the actual average
102 * performance during last sample period
103 * @busy_scaled: Scaled busy value which is used to calculate next
104 * P state. This can be different than core_avg_perf
105 * to account for cpu idle period
106 * @aperf: Difference of actual performance frequency clock count
107 * read from APERF MSR between last and current sample
108 * @mperf: Difference of maximum performance frequency clock count
109 * read from MPERF MSR between last and current sample
110 * @tsc: Difference of time stamp counter between last and
112 * @time: Current time from scheduler
114 * This structure is used in the cpudata structure to store performance sample
115 * data for choosing next P State.
118 int32_t core_avg_perf
;
127 * struct pstate_data - Store P state data
128 * @current_pstate: Current requested P state
129 * @min_pstate: Min P state possible for this platform
130 * @max_pstate: Max P state possible for this platform
131 * @max_pstate_physical:This is physical Max P state for a processor
132 * This can be higher than the max_pstate which can
133 * be limited by platform thermal design power limits
134 * @scaling: Scaling factor to convert frequency to cpufreq
136 * @turbo_pstate: Max Turbo P state possible for this platform
137 * @max_freq: @max_pstate frequency in cpufreq units
138 * @turbo_freq: @turbo_pstate frequency in cpufreq units
140 * Stores the per cpu model P state limits and current P state.
146 int max_pstate_physical
;
149 unsigned int max_freq
;
150 unsigned int turbo_freq
;
154 * struct vid_data - Stores voltage information data
155 * @min: VID data for this platform corresponding to
157 * @max: VID data corresponding to the highest P State.
158 * @turbo: VID data for turbo P state
159 * @ratio: Ratio of (vid max - vid min) /
160 * (max P state - Min P State)
162 * Stores the voltage data for DVFS (Dynamic Voltage and Frequency Scaling)
163 * This data is used in Atom platforms, where in addition to target P state,
164 * the voltage data needs to be specified to select next P State.
174 * struct global_params - Global parameters, mostly tunable via sysfs.
175 * @no_turbo: Whether or not to use turbo P-states.
176 * @turbo_disabled: Whether or not turbo P-states are available at all,
177 * based on the MSR_IA32_MISC_ENABLE value and whether or
178 * not the maximum reported turbo P-state is different from
179 * the maximum reported non-turbo one.
180 * @turbo_disabled_mf: The @turbo_disabled value reflected by cpuinfo.max_freq.
181 * @min_perf_pct: Minimum capacity limit in percent of the maximum turbo
183 * @max_perf_pct: Maximum capacity limit in percent of the maximum turbo
186 struct global_params
{
189 bool turbo_disabled_mf
;
195 * struct cpudata - Per CPU instance data storage
196 * @cpu: CPU number for this instance data
197 * @policy: CPUFreq policy value
198 * @update_util: CPUFreq utility callback information
199 * @update_util_set: CPUFreq utility callback is set
200 * @iowait_boost: iowait-related boost fraction
201 * @last_update: Time of the last update.
202 * @pstate: Stores P state limits for this CPU
203 * @vid: Stores VID limits for this CPU
204 * @last_sample_time: Last Sample time
205 * @aperf_mperf_shift: APERF vs MPERF counting frequency difference
206 * @prev_aperf: Last APERF value read from APERF MSR
207 * @prev_mperf: Last MPERF value read from MPERF MSR
208 * @prev_tsc: Last timestamp counter (TSC) value
209 * @prev_cummulative_iowait: IO Wait time difference from last and
211 * @sample: Storage for storing last Sample data
212 * @min_perf_ratio: Minimum capacity in terms of PERF or HWP ratios
213 * @max_perf_ratio: Maximum capacity in terms of PERF or HWP ratios
214 * @acpi_perf_data: Stores ACPI perf information read from _PSS
215 * @valid_pss_table: Set to true for valid ACPI _PSS entries found
216 * @epp_powersave: Last saved HWP energy performance preference
217 * (EPP) or energy performance bias (EPB),
218 * when policy switched to performance
219 * @epp_policy: Last saved policy used to set EPP/EPB
220 * @epp_default: Power on default HWP energy performance
222 * @epp_cached Cached HWP energy-performance preference value
223 * @hwp_req_cached: Cached value of the last HWP Request MSR
224 * @hwp_cap_cached: Cached value of the last HWP Capabilities MSR
225 * @last_io_update: Last time when IO wake flag was set
226 * @sched_flags: Store scheduler flags for possible cross CPU update
227 * @hwp_boost_min: Last HWP boosted min performance
228 * @suspended: Whether or not the driver has been suspended.
230 * This structure stores per CPU instance data for all CPUs.
236 struct update_util_data update_util
;
237 bool update_util_set
;
239 struct pstate_data pstate
;
243 u64 last_sample_time
;
244 u64 aperf_mperf_shift
;
248 u64 prev_cummulative_iowait
;
249 struct sample sample
;
250 int32_t min_perf_ratio
;
251 int32_t max_perf_ratio
;
253 struct acpi_processor_performance acpi_perf_data
;
254 bool valid_pss_table
;
256 unsigned int iowait_boost
;
264 unsigned int sched_flags
;
269 static struct cpudata
**all_cpu_data
;
272 * struct pstate_funcs - Per CPU model specific callbacks
273 * @get_max: Callback to get maximum non turbo effective P state
274 * @get_max_physical: Callback to get maximum non turbo physical P state
275 * @get_min: Callback to get minimum P state
276 * @get_turbo: Callback to get turbo P state
277 * @get_scaling: Callback to get frequency scaling factor
278 * @get_aperf_mperf_shift: Callback to get the APERF vs MPERF frequency difference
279 * @get_val: Callback to convert P state to actual MSR write value
280 * @get_vid: Callback to get VID data for Atom platforms
282 * Core and Atom CPU models have different way to get P State limits. This
283 * structure is used to store those callbacks.
285 struct pstate_funcs
{
286 int (*get_max
)(void);
287 int (*get_max_physical
)(void);
288 int (*get_min
)(void);
289 int (*get_turbo
)(void);
290 int (*get_scaling
)(void);
291 int (*get_aperf_mperf_shift
)(void);
292 u64 (*get_val
)(struct cpudata
*, int pstate
);
293 void (*get_vid
)(struct cpudata
*);
296 static struct pstate_funcs pstate_funcs __read_mostly
;
298 static int hwp_active __read_mostly
;
299 static int hwp_mode_bdw __read_mostly
;
300 static bool per_cpu_limits __read_mostly
;
301 static bool hwp_boost __read_mostly
;
303 static struct cpufreq_driver
*intel_pstate_driver __read_mostly
;
306 static bool acpi_ppc
;
309 static struct global_params global
;
311 static DEFINE_MUTEX(intel_pstate_driver_lock
);
312 static DEFINE_MUTEX(intel_pstate_limits_lock
);
316 static bool intel_pstate_acpi_pm_profile_server(void)
318 if (acpi_gbl_FADT
.preferred_profile
== PM_ENTERPRISE_SERVER
||
319 acpi_gbl_FADT
.preferred_profile
== PM_PERFORMANCE_SERVER
)
325 static bool intel_pstate_get_ppc_enable_status(void)
327 if (intel_pstate_acpi_pm_profile_server())
333 #ifdef CONFIG_ACPI_CPPC_LIB
335 /* The work item is needed to avoid CPU hotplug locking issues */
336 static void intel_pstste_sched_itmt_work_fn(struct work_struct
*work
)
338 sched_set_itmt_support();
341 static DECLARE_WORK(sched_itmt_work
, intel_pstste_sched_itmt_work_fn
);
343 static void intel_pstate_set_itmt_prio(int cpu
)
345 struct cppc_perf_caps cppc_perf
;
346 static u32 max_highest_perf
= 0, min_highest_perf
= U32_MAX
;
349 ret
= cppc_get_perf_caps(cpu
, &cppc_perf
);
354 * The priorities can be set regardless of whether or not
355 * sched_set_itmt_support(true) has been called and it is valid to
356 * update them at any time after it has been called.
358 sched_set_itmt_core_prio(cppc_perf
.highest_perf
, cpu
);
360 if (max_highest_perf
<= min_highest_perf
) {
361 if (cppc_perf
.highest_perf
> max_highest_perf
)
362 max_highest_perf
= cppc_perf
.highest_perf
;
364 if (cppc_perf
.highest_perf
< min_highest_perf
)
365 min_highest_perf
= cppc_perf
.highest_perf
;
367 if (max_highest_perf
> min_highest_perf
) {
369 * This code can be run during CPU online under the
370 * CPU hotplug locks, so sched_set_itmt_support()
371 * cannot be called from here. Queue up a work item
374 schedule_work(&sched_itmt_work
);
379 static int intel_pstate_get_cppc_guranteed(int cpu
)
381 struct cppc_perf_caps cppc_perf
;
384 ret
= cppc_get_perf_caps(cpu
, &cppc_perf
);
388 if (cppc_perf
.guaranteed_perf
)
389 return cppc_perf
.guaranteed_perf
;
391 return cppc_perf
.nominal_perf
;
394 #else /* CONFIG_ACPI_CPPC_LIB */
395 static void intel_pstate_set_itmt_prio(int cpu
)
398 #endif /* CONFIG_ACPI_CPPC_LIB */
400 static void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy
*policy
)
407 intel_pstate_set_itmt_prio(policy
->cpu
);
411 if (!intel_pstate_get_ppc_enable_status())
414 cpu
= all_cpu_data
[policy
->cpu
];
416 ret
= acpi_processor_register_performance(&cpu
->acpi_perf_data
,
422 * Check if the control value in _PSS is for PERF_CTL MSR, which should
423 * guarantee that the states returned by it map to the states in our
426 if (cpu
->acpi_perf_data
.control_register
.space_id
!=
427 ACPI_ADR_SPACE_FIXED_HARDWARE
)
431 * If there is only one entry _PSS, simply ignore _PSS and continue as
432 * usual without taking _PSS into account
434 if (cpu
->acpi_perf_data
.state_count
< 2)
437 pr_debug("CPU%u - ACPI _PSS perf data\n", policy
->cpu
);
438 for (i
= 0; i
< cpu
->acpi_perf_data
.state_count
; i
++) {
439 pr_debug(" %cP%d: %u MHz, %u mW, 0x%x\n",
440 (i
== cpu
->acpi_perf_data
.state
? '*' : ' '), i
,
441 (u32
) cpu
->acpi_perf_data
.states
[i
].core_frequency
,
442 (u32
) cpu
->acpi_perf_data
.states
[i
].power
,
443 (u32
) cpu
->acpi_perf_data
.states
[i
].control
);
447 * The _PSS table doesn't contain whole turbo frequency range.
448 * This just contains +1 MHZ above the max non turbo frequency,
449 * with control value corresponding to max turbo ratio. But
450 * when cpufreq set policy is called, it will call with this
451 * max frequency, which will cause a reduced performance as
452 * this driver uses real max turbo frequency as the max
453 * frequency. So correct this frequency in _PSS table to
454 * correct max turbo frequency based on the turbo state.
455 * Also need to convert to MHz as _PSS freq is in MHz.
457 if (!global
.turbo_disabled
)
458 cpu
->acpi_perf_data
.states
[0].core_frequency
=
459 policy
->cpuinfo
.max_freq
/ 1000;
460 cpu
->valid_pss_table
= true;
461 pr_debug("_PPC limits will be enforced\n");
466 cpu
->valid_pss_table
= false;
467 acpi_processor_unregister_performance(policy
->cpu
);
470 static void intel_pstate_exit_perf_limits(struct cpufreq_policy
*policy
)
474 cpu
= all_cpu_data
[policy
->cpu
];
475 if (!cpu
->valid_pss_table
)
478 acpi_processor_unregister_performance(policy
->cpu
);
480 #else /* CONFIG_ACPI */
481 static inline void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy
*policy
)
485 static inline void intel_pstate_exit_perf_limits(struct cpufreq_policy
*policy
)
489 static inline bool intel_pstate_acpi_pm_profile_server(void)
493 #endif /* CONFIG_ACPI */
495 #ifndef CONFIG_ACPI_CPPC_LIB
496 static int intel_pstate_get_cppc_guranteed(int cpu
)
500 #endif /* CONFIG_ACPI_CPPC_LIB */
502 static inline void update_turbo_state(void)
507 cpu
= all_cpu_data
[0];
508 rdmsrl(MSR_IA32_MISC_ENABLE
, misc_en
);
509 global
.turbo_disabled
=
510 (misc_en
& MSR_IA32_MISC_ENABLE_TURBO_DISABLE
||
511 cpu
->pstate
.max_pstate
== cpu
->pstate
.turbo_pstate
);
514 static int min_perf_pct_min(void)
516 struct cpudata
*cpu
= all_cpu_data
[0];
517 int turbo_pstate
= cpu
->pstate
.turbo_pstate
;
519 return turbo_pstate
?
520 (cpu
->pstate
.min_pstate
* 100 / turbo_pstate
) : 0;
523 static s16
intel_pstate_get_epb(struct cpudata
*cpu_data
)
528 if (!boot_cpu_has(X86_FEATURE_EPB
))
531 ret
= rdmsrl_on_cpu(cpu_data
->cpu
, MSR_IA32_ENERGY_PERF_BIAS
, &epb
);
535 return (s16
)(epb
& 0x0f);
538 static s16
intel_pstate_get_epp(struct cpudata
*cpu_data
, u64 hwp_req_data
)
542 if (boot_cpu_has(X86_FEATURE_HWP_EPP
)) {
544 * When hwp_req_data is 0, means that caller didn't read
545 * MSR_HWP_REQUEST, so need to read and get EPP.
548 epp
= rdmsrl_on_cpu(cpu_data
->cpu
, MSR_HWP_REQUEST
,
553 epp
= (hwp_req_data
>> 24) & 0xff;
555 /* When there is no EPP present, HWP uses EPB settings */
556 epp
= intel_pstate_get_epb(cpu_data
);
562 static int intel_pstate_set_epb(int cpu
, s16 pref
)
567 if (!boot_cpu_has(X86_FEATURE_EPB
))
570 ret
= rdmsrl_on_cpu(cpu
, MSR_IA32_ENERGY_PERF_BIAS
, &epb
);
574 epb
= (epb
& ~0x0f) | pref
;
575 wrmsrl_on_cpu(cpu
, MSR_IA32_ENERGY_PERF_BIAS
, epb
);
581 * EPP/EPB display strings corresponding to EPP index in the
582 * energy_perf_strings[]
584 *-------------------------------------
587 * 2 balance_performance
591 static const char * const energy_perf_strings
[] = {
594 "balance_performance",
599 static const unsigned int epp_values
[] = {
601 HWP_EPP_BALANCE_PERFORMANCE
,
602 HWP_EPP_BALANCE_POWERSAVE
,
606 static int intel_pstate_get_energy_pref_index(struct cpudata
*cpu_data
, int *raw_epp
)
612 epp
= intel_pstate_get_epp(cpu_data
, 0);
616 if (boot_cpu_has(X86_FEATURE_HWP_EPP
)) {
617 if (epp
== HWP_EPP_PERFORMANCE
)
619 if (epp
== HWP_EPP_BALANCE_PERFORMANCE
)
621 if (epp
== HWP_EPP_BALANCE_POWERSAVE
)
623 if (epp
== HWP_EPP_POWERSAVE
)
627 } else if (boot_cpu_has(X86_FEATURE_EPB
)) {
630 * 0x00-0x03 : Performance
631 * 0x04-0x07 : Balance performance
632 * 0x08-0x0B : Balance power
634 * The EPB is a 4 bit value, but our ranges restrict the
635 * value which can be set. Here only using top two bits
638 index
= (epp
>> 2) + 1;
644 static int intel_pstate_set_epp(struct cpudata
*cpu
, u32 epp
)
649 * Use the cached HWP Request MSR value, because in the active mode the
650 * register itself may be updated by intel_pstate_hwp_boost_up() or
651 * intel_pstate_hwp_boost_down() at any time.
653 u64 value
= READ_ONCE(cpu
->hwp_req_cached
);
655 value
&= ~GENMASK_ULL(31, 24);
656 value
|= (u64
)epp
<< 24;
658 * The only other updater of hwp_req_cached in the active mode,
659 * intel_pstate_hwp_set(), is called under the same lock as this
660 * function, so it cannot run in parallel with the update below.
662 WRITE_ONCE(cpu
->hwp_req_cached
, value
);
663 ret
= wrmsrl_on_cpu(cpu
->cpu
, MSR_HWP_REQUEST
, value
);
665 cpu
->epp_cached
= epp
;
670 static int intel_pstate_set_energy_pref_index(struct cpudata
*cpu_data
,
671 int pref_index
, bool use_raw
,
678 epp
= cpu_data
->epp_default
;
680 if (boot_cpu_has(X86_FEATURE_HWP_EPP
)) {
683 else if (epp
== -EINVAL
)
684 epp
= epp_values
[pref_index
- 1];
687 * To avoid confusion, refuse to set EPP to any values different
688 * from 0 (performance) if the current policy is "performance",
689 * because those values would be overridden.
691 if (epp
> 0 && cpu_data
->policy
== CPUFREQ_POLICY_PERFORMANCE
)
694 ret
= intel_pstate_set_epp(cpu_data
, epp
);
697 epp
= (pref_index
- 1) << 2;
698 ret
= intel_pstate_set_epb(cpu_data
->cpu
, epp
);
704 static ssize_t
show_energy_performance_available_preferences(
705 struct cpufreq_policy
*policy
, char *buf
)
710 while (energy_perf_strings
[i
] != NULL
)
711 ret
+= sprintf(&buf
[ret
], "%s ", energy_perf_strings
[i
++]);
713 ret
+= sprintf(&buf
[ret
], "\n");
718 cpufreq_freq_attr_ro(energy_performance_available_preferences
);
720 static struct cpufreq_driver intel_pstate
;
722 static ssize_t
store_energy_performance_preference(
723 struct cpufreq_policy
*policy
, const char *buf
, size_t count
)
725 struct cpudata
*cpu
= all_cpu_data
[policy
->cpu
];
726 char str_preference
[21];
731 ret
= sscanf(buf
, "%20s", str_preference
);
735 ret
= match_string(energy_perf_strings
, -1, str_preference
);
737 if (!boot_cpu_has(X86_FEATURE_HWP_EPP
))
740 ret
= kstrtouint(buf
, 10, &epp
);
751 * This function runs with the policy R/W semaphore held, which
752 * guarantees that the driver pointer will not change while it is
755 if (!intel_pstate_driver
)
758 mutex_lock(&intel_pstate_limits_lock
);
760 if (intel_pstate_driver
== &intel_pstate
) {
761 ret
= intel_pstate_set_energy_pref_index(cpu
, ret
, raw
, epp
);
764 * In the passive mode the governor needs to be stopped on the
765 * target CPU before the EPP update and restarted after it,
766 * which is super-heavy-weight, so make sure it is worth doing
770 epp
= ret
? epp_values
[ret
- 1] : cpu
->epp_default
;
772 if (cpu
->epp_cached
!= epp
) {
775 cpufreq_stop_governor(policy
);
776 ret
= intel_pstate_set_epp(cpu
, epp
);
777 err
= cpufreq_start_governor(policy
);
783 mutex_unlock(&intel_pstate_limits_lock
);
788 static ssize_t
show_energy_performance_preference(
789 struct cpufreq_policy
*policy
, char *buf
)
791 struct cpudata
*cpu_data
= all_cpu_data
[policy
->cpu
];
792 int preference
, raw_epp
;
794 preference
= intel_pstate_get_energy_pref_index(cpu_data
, &raw_epp
);
799 return sprintf(buf
, "%d\n", raw_epp
);
801 return sprintf(buf
, "%s\n", energy_perf_strings
[preference
]);
804 cpufreq_freq_attr_rw(energy_performance_preference
);
806 static ssize_t
show_base_frequency(struct cpufreq_policy
*policy
, char *buf
)
812 ratio
= intel_pstate_get_cppc_guranteed(policy
->cpu
);
814 rdmsrl_on_cpu(policy
->cpu
, MSR_HWP_CAPABILITIES
, &cap
);
815 ratio
= HWP_GUARANTEED_PERF(cap
);
818 cpu
= all_cpu_data
[policy
->cpu
];
820 return sprintf(buf
, "%d\n", ratio
* cpu
->pstate
.scaling
);
823 cpufreq_freq_attr_ro(base_frequency
);
825 static struct freq_attr
*hwp_cpufreq_attrs
[] = {
826 &energy_performance_preference
,
827 &energy_performance_available_preferences
,
832 static void intel_pstate_get_hwp_max(unsigned int cpu
, int *phy_max
,
837 rdmsrl_on_cpu(cpu
, MSR_HWP_CAPABILITIES
, &cap
);
838 WRITE_ONCE(all_cpu_data
[cpu
]->hwp_cap_cached
, cap
);
839 if (global
.no_turbo
|| global
.turbo_disabled
)
840 *current_max
= HWP_GUARANTEED_PERF(cap
);
842 *current_max
= HWP_HIGHEST_PERF(cap
);
844 *phy_max
= HWP_HIGHEST_PERF(cap
);
847 static void intel_pstate_hwp_set(unsigned int cpu
)
849 struct cpudata
*cpu_data
= all_cpu_data
[cpu
];
854 max
= cpu_data
->max_perf_ratio
;
855 min
= cpu_data
->min_perf_ratio
;
857 if (cpu_data
->policy
== CPUFREQ_POLICY_PERFORMANCE
)
860 rdmsrl_on_cpu(cpu
, MSR_HWP_REQUEST
, &value
);
862 value
&= ~HWP_MIN_PERF(~0L);
863 value
|= HWP_MIN_PERF(min
);
865 value
&= ~HWP_MAX_PERF(~0L);
866 value
|= HWP_MAX_PERF(max
);
868 if (cpu_data
->epp_policy
== cpu_data
->policy
)
871 cpu_data
->epp_policy
= cpu_data
->policy
;
873 if (cpu_data
->policy
== CPUFREQ_POLICY_PERFORMANCE
) {
874 epp
= intel_pstate_get_epp(cpu_data
, value
);
875 cpu_data
->epp_powersave
= epp
;
876 /* If EPP read was failed, then don't try to write */
882 /* skip setting EPP, when saved value is invalid */
883 if (cpu_data
->epp_powersave
< 0)
887 * No need to restore EPP when it is not zero. This
889 * - Policy is not changed
890 * - user has manually changed
891 * - Error reading EPB
893 epp
= intel_pstate_get_epp(cpu_data
, value
);
897 epp
= cpu_data
->epp_powersave
;
899 if (boot_cpu_has(X86_FEATURE_HWP_EPP
)) {
900 value
&= ~GENMASK_ULL(31, 24);
901 value
|= (u64
)epp
<< 24;
903 intel_pstate_set_epb(cpu
, epp
);
906 WRITE_ONCE(cpu_data
->hwp_req_cached
, value
);
907 wrmsrl_on_cpu(cpu
, MSR_HWP_REQUEST
, value
);
910 static void intel_pstate_hwp_offline(struct cpudata
*cpu
)
912 u64 value
= READ_ONCE(cpu
->hwp_req_cached
);
915 if (boot_cpu_has(X86_FEATURE_HWP_EPP
)) {
917 * In case the EPP has been set to "performance" by the
918 * active mode "performance" scaling algorithm, replace that
919 * temporary value with the cached EPP one.
921 value
&= ~GENMASK_ULL(31, 24);
922 value
|= HWP_ENERGY_PERF_PREFERENCE(cpu
->epp_cached
);
923 WRITE_ONCE(cpu
->hwp_req_cached
, value
);
926 value
&= ~GENMASK_ULL(31, 0);
927 min_perf
= HWP_LOWEST_PERF(cpu
->hwp_cap_cached
);
929 /* Set hwp_max = hwp_min */
930 value
|= HWP_MAX_PERF(min_perf
);
931 value
|= HWP_MIN_PERF(min_perf
);
934 if (boot_cpu_has(X86_FEATURE_HWP_EPP
))
935 value
|= HWP_ENERGY_PERF_PREFERENCE(HWP_EPP_POWERSAVE
);
937 wrmsrl_on_cpu(cpu
->cpu
, MSR_HWP_REQUEST
, value
);
940 #define POWER_CTL_EE_ENABLE 1
941 #define POWER_CTL_EE_DISABLE 2
943 static int power_ctl_ee_state
;
945 static void set_power_ctl_ee_state(bool input
)
949 mutex_lock(&intel_pstate_driver_lock
);
950 rdmsrl(MSR_IA32_POWER_CTL
, power_ctl
);
952 power_ctl
&= ~BIT(MSR_IA32_POWER_CTL_BIT_EE
);
953 power_ctl_ee_state
= POWER_CTL_EE_ENABLE
;
955 power_ctl
|= BIT(MSR_IA32_POWER_CTL_BIT_EE
);
956 power_ctl_ee_state
= POWER_CTL_EE_DISABLE
;
958 wrmsrl(MSR_IA32_POWER_CTL
, power_ctl
);
959 mutex_unlock(&intel_pstate_driver_lock
);
962 static void intel_pstate_hwp_enable(struct cpudata
*cpudata
);
964 static void intel_pstate_hwp_reenable(struct cpudata
*cpu
)
966 intel_pstate_hwp_enable(cpu
);
967 wrmsrl_on_cpu(cpu
->cpu
, MSR_HWP_REQUEST
, READ_ONCE(cpu
->hwp_req_cached
));
970 static int intel_pstate_suspend(struct cpufreq_policy
*policy
)
972 struct cpudata
*cpu
= all_cpu_data
[policy
->cpu
];
974 pr_debug("CPU %d suspending\n", cpu
->cpu
);
976 cpu
->suspended
= true;
981 static int intel_pstate_resume(struct cpufreq_policy
*policy
)
983 struct cpudata
*cpu
= all_cpu_data
[policy
->cpu
];
985 pr_debug("CPU %d resuming\n", cpu
->cpu
);
987 /* Only restore if the system default is changed */
988 if (power_ctl_ee_state
== POWER_CTL_EE_ENABLE
)
989 set_power_ctl_ee_state(true);
990 else if (power_ctl_ee_state
== POWER_CTL_EE_DISABLE
)
991 set_power_ctl_ee_state(false);
993 if (cpu
->suspended
&& hwp_active
) {
994 mutex_lock(&intel_pstate_limits_lock
);
996 /* Re-enable HWP, because "online" has not done that. */
997 intel_pstate_hwp_reenable(cpu
);
999 mutex_unlock(&intel_pstate_limits_lock
);
1002 cpu
->suspended
= false;
1007 static void intel_pstate_update_policies(void)
1011 for_each_possible_cpu(cpu
)
1012 cpufreq_update_policy(cpu
);
1015 static void intel_pstate_update_max_freq(unsigned int cpu
)
1017 struct cpufreq_policy
*policy
= cpufreq_cpu_acquire(cpu
);
1018 struct cpudata
*cpudata
;
1023 cpudata
= all_cpu_data
[cpu
];
1024 policy
->cpuinfo
.max_freq
= global
.turbo_disabled_mf
?
1025 cpudata
->pstate
.max_freq
: cpudata
->pstate
.turbo_freq
;
1027 refresh_frequency_limits(policy
);
1029 cpufreq_cpu_release(policy
);
1032 static void intel_pstate_update_limits(unsigned int cpu
)
1034 mutex_lock(&intel_pstate_driver_lock
);
1036 update_turbo_state();
1038 * If turbo has been turned on or off globally, policy limits for
1039 * all CPUs need to be updated to reflect that.
1041 if (global
.turbo_disabled_mf
!= global
.turbo_disabled
) {
1042 global
.turbo_disabled_mf
= global
.turbo_disabled
;
1043 arch_set_max_freq_ratio(global
.turbo_disabled
);
1044 for_each_possible_cpu(cpu
)
1045 intel_pstate_update_max_freq(cpu
);
1047 cpufreq_update_policy(cpu
);
1050 mutex_unlock(&intel_pstate_driver_lock
);
1053 /************************** sysfs begin ************************/
1054 #define show_one(file_name, object) \
1055 static ssize_t show_##file_name \
1056 (struct kobject *kobj, struct kobj_attribute *attr, char *buf) \
1058 return sprintf(buf, "%u\n", global.object); \
1061 static ssize_t
intel_pstate_show_status(char *buf
);
1062 static int intel_pstate_update_status(const char *buf
, size_t size
);
1064 static ssize_t
show_status(struct kobject
*kobj
,
1065 struct kobj_attribute
*attr
, char *buf
)
1069 mutex_lock(&intel_pstate_driver_lock
);
1070 ret
= intel_pstate_show_status(buf
);
1071 mutex_unlock(&intel_pstate_driver_lock
);
1076 static ssize_t
store_status(struct kobject
*a
, struct kobj_attribute
*b
,
1077 const char *buf
, size_t count
)
1079 char *p
= memchr(buf
, '\n', count
);
1082 mutex_lock(&intel_pstate_driver_lock
);
1083 ret
= intel_pstate_update_status(buf
, p
? p
- buf
: count
);
1084 mutex_unlock(&intel_pstate_driver_lock
);
1086 return ret
< 0 ? ret
: count
;
1089 static ssize_t
show_turbo_pct(struct kobject
*kobj
,
1090 struct kobj_attribute
*attr
, char *buf
)
1092 struct cpudata
*cpu
;
1093 int total
, no_turbo
, turbo_pct
;
1096 mutex_lock(&intel_pstate_driver_lock
);
1098 if (!intel_pstate_driver
) {
1099 mutex_unlock(&intel_pstate_driver_lock
);
1103 cpu
= all_cpu_data
[0];
1105 total
= cpu
->pstate
.turbo_pstate
- cpu
->pstate
.min_pstate
+ 1;
1106 no_turbo
= cpu
->pstate
.max_pstate
- cpu
->pstate
.min_pstate
+ 1;
1107 turbo_fp
= div_fp(no_turbo
, total
);
1108 turbo_pct
= 100 - fp_toint(mul_fp(turbo_fp
, int_tofp(100)));
1110 mutex_unlock(&intel_pstate_driver_lock
);
1112 return sprintf(buf
, "%u\n", turbo_pct
);
1115 static ssize_t
show_num_pstates(struct kobject
*kobj
,
1116 struct kobj_attribute
*attr
, char *buf
)
1118 struct cpudata
*cpu
;
1121 mutex_lock(&intel_pstate_driver_lock
);
1123 if (!intel_pstate_driver
) {
1124 mutex_unlock(&intel_pstate_driver_lock
);
1128 cpu
= all_cpu_data
[0];
1129 total
= cpu
->pstate
.turbo_pstate
- cpu
->pstate
.min_pstate
+ 1;
1131 mutex_unlock(&intel_pstate_driver_lock
);
1133 return sprintf(buf
, "%u\n", total
);
1136 static ssize_t
show_no_turbo(struct kobject
*kobj
,
1137 struct kobj_attribute
*attr
, char *buf
)
1141 mutex_lock(&intel_pstate_driver_lock
);
1143 if (!intel_pstate_driver
) {
1144 mutex_unlock(&intel_pstate_driver_lock
);
1148 update_turbo_state();
1149 if (global
.turbo_disabled
)
1150 ret
= sprintf(buf
, "%u\n", global
.turbo_disabled
);
1152 ret
= sprintf(buf
, "%u\n", global
.no_turbo
);
1154 mutex_unlock(&intel_pstate_driver_lock
);
1159 static ssize_t
store_no_turbo(struct kobject
*a
, struct kobj_attribute
*b
,
1160 const char *buf
, size_t count
)
1165 ret
= sscanf(buf
, "%u", &input
);
1169 mutex_lock(&intel_pstate_driver_lock
);
1171 if (!intel_pstate_driver
) {
1172 mutex_unlock(&intel_pstate_driver_lock
);
1176 mutex_lock(&intel_pstate_limits_lock
);
1178 update_turbo_state();
1179 if (global
.turbo_disabled
) {
1180 pr_notice_once("Turbo disabled by BIOS or unavailable on processor\n");
1181 mutex_unlock(&intel_pstate_limits_lock
);
1182 mutex_unlock(&intel_pstate_driver_lock
);
1186 global
.no_turbo
= clamp_t(int, input
, 0, 1);
1188 if (global
.no_turbo
) {
1189 struct cpudata
*cpu
= all_cpu_data
[0];
1190 int pct
= cpu
->pstate
.max_pstate
* 100 / cpu
->pstate
.turbo_pstate
;
1192 /* Squash the global minimum into the permitted range. */
1193 if (global
.min_perf_pct
> pct
)
1194 global
.min_perf_pct
= pct
;
1197 mutex_unlock(&intel_pstate_limits_lock
);
1199 intel_pstate_update_policies();
1201 mutex_unlock(&intel_pstate_driver_lock
);
1206 static void update_qos_request(enum freq_qos_req_type type
)
1208 int max_state
, turbo_max
, freq
, i
, perf_pct
;
1209 struct freq_qos_request
*req
;
1210 struct cpufreq_policy
*policy
;
1212 for_each_possible_cpu(i
) {
1213 struct cpudata
*cpu
= all_cpu_data
[i
];
1215 policy
= cpufreq_cpu_get(i
);
1219 req
= policy
->driver_data
;
1220 cpufreq_cpu_put(policy
);
1226 intel_pstate_get_hwp_max(i
, &turbo_max
, &max_state
);
1228 turbo_max
= cpu
->pstate
.turbo_pstate
;
1230 if (type
== FREQ_QOS_MIN
) {
1231 perf_pct
= global
.min_perf_pct
;
1234 perf_pct
= global
.max_perf_pct
;
1237 freq
= DIV_ROUND_UP(turbo_max
* perf_pct
, 100);
1238 freq
*= cpu
->pstate
.scaling
;
1240 if (freq_qos_update_request(req
, freq
) < 0)
1241 pr_warn("Failed to update freq constraint: CPU%d\n", i
);
1245 static ssize_t
store_max_perf_pct(struct kobject
*a
, struct kobj_attribute
*b
,
1246 const char *buf
, size_t count
)
1251 ret
= sscanf(buf
, "%u", &input
);
1255 mutex_lock(&intel_pstate_driver_lock
);
1257 if (!intel_pstate_driver
) {
1258 mutex_unlock(&intel_pstate_driver_lock
);
1262 mutex_lock(&intel_pstate_limits_lock
);
1264 global
.max_perf_pct
= clamp_t(int, input
, global
.min_perf_pct
, 100);
1266 mutex_unlock(&intel_pstate_limits_lock
);
1268 if (intel_pstate_driver
== &intel_pstate
)
1269 intel_pstate_update_policies();
1271 update_qos_request(FREQ_QOS_MAX
);
1273 mutex_unlock(&intel_pstate_driver_lock
);
1278 static ssize_t
store_min_perf_pct(struct kobject
*a
, struct kobj_attribute
*b
,
1279 const char *buf
, size_t count
)
1284 ret
= sscanf(buf
, "%u", &input
);
1288 mutex_lock(&intel_pstate_driver_lock
);
1290 if (!intel_pstate_driver
) {
1291 mutex_unlock(&intel_pstate_driver_lock
);
1295 mutex_lock(&intel_pstate_limits_lock
);
1297 global
.min_perf_pct
= clamp_t(int, input
,
1298 min_perf_pct_min(), global
.max_perf_pct
);
1300 mutex_unlock(&intel_pstate_limits_lock
);
1302 if (intel_pstate_driver
== &intel_pstate
)
1303 intel_pstate_update_policies();
1305 update_qos_request(FREQ_QOS_MIN
);
1307 mutex_unlock(&intel_pstate_driver_lock
);
1312 static ssize_t
show_hwp_dynamic_boost(struct kobject
*kobj
,
1313 struct kobj_attribute
*attr
, char *buf
)
1315 return sprintf(buf
, "%u\n", hwp_boost
);
1318 static ssize_t
store_hwp_dynamic_boost(struct kobject
*a
,
1319 struct kobj_attribute
*b
,
1320 const char *buf
, size_t count
)
1325 ret
= kstrtouint(buf
, 10, &input
);
1329 mutex_lock(&intel_pstate_driver_lock
);
1330 hwp_boost
= !!input
;
1331 intel_pstate_update_policies();
1332 mutex_unlock(&intel_pstate_driver_lock
);
1337 static ssize_t
show_energy_efficiency(struct kobject
*kobj
, struct kobj_attribute
*attr
,
1343 rdmsrl(MSR_IA32_POWER_CTL
, power_ctl
);
1344 enable
= !!(power_ctl
& BIT(MSR_IA32_POWER_CTL_BIT_EE
));
1345 return sprintf(buf
, "%d\n", !enable
);
1348 static ssize_t
store_energy_efficiency(struct kobject
*a
, struct kobj_attribute
*b
,
1349 const char *buf
, size_t count
)
1354 ret
= kstrtobool(buf
, &input
);
1358 set_power_ctl_ee_state(input
);
1363 show_one(max_perf_pct
, max_perf_pct
);
1364 show_one(min_perf_pct
, min_perf_pct
);
1366 define_one_global_rw(status
);
1367 define_one_global_rw(no_turbo
);
1368 define_one_global_rw(max_perf_pct
);
1369 define_one_global_rw(min_perf_pct
);
1370 define_one_global_ro(turbo_pct
);
1371 define_one_global_ro(num_pstates
);
1372 define_one_global_rw(hwp_dynamic_boost
);
1373 define_one_global_rw(energy_efficiency
);
1375 static struct attribute
*intel_pstate_attributes
[] = {
1383 static const struct attribute_group intel_pstate_attr_group
= {
1384 .attrs
= intel_pstate_attributes
,
1387 static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids
[];
1389 static struct kobject
*intel_pstate_kobject
;
1391 static void __init
intel_pstate_sysfs_expose_params(void)
1395 intel_pstate_kobject
= kobject_create_and_add("intel_pstate",
1396 &cpu_subsys
.dev_root
->kobj
);
1397 if (WARN_ON(!intel_pstate_kobject
))
1400 rc
= sysfs_create_group(intel_pstate_kobject
, &intel_pstate_attr_group
);
1405 * If per cpu limits are enforced there are no global limits, so
1406 * return without creating max/min_perf_pct attributes
1411 rc
= sysfs_create_file(intel_pstate_kobject
, &max_perf_pct
.attr
);
1414 rc
= sysfs_create_file(intel_pstate_kobject
, &min_perf_pct
.attr
);
1417 if (x86_match_cpu(intel_pstate_cpu_ee_disable_ids
)) {
1418 rc
= sysfs_create_file(intel_pstate_kobject
, &energy_efficiency
.attr
);
1423 static void __init
intel_pstate_sysfs_remove(void)
1425 if (!intel_pstate_kobject
)
1428 sysfs_remove_group(intel_pstate_kobject
, &intel_pstate_attr_group
);
1430 if (!per_cpu_limits
) {
1431 sysfs_remove_file(intel_pstate_kobject
, &max_perf_pct
.attr
);
1432 sysfs_remove_file(intel_pstate_kobject
, &min_perf_pct
.attr
);
1434 if (x86_match_cpu(intel_pstate_cpu_ee_disable_ids
))
1435 sysfs_remove_file(intel_pstate_kobject
, &energy_efficiency
.attr
);
1438 kobject_put(intel_pstate_kobject
);
1441 static void intel_pstate_sysfs_expose_hwp_dynamic_boost(void)
1448 rc
= sysfs_create_file(intel_pstate_kobject
, &hwp_dynamic_boost
.attr
);
1452 static void intel_pstate_sysfs_hide_hwp_dynamic_boost(void)
1457 sysfs_remove_file(intel_pstate_kobject
, &hwp_dynamic_boost
.attr
);
1460 /************************** sysfs end ************************/
1462 static void intel_pstate_hwp_enable(struct cpudata
*cpudata
)
1464 /* First disable HWP notification interrupt as we don't process them */
1465 if (boot_cpu_has(X86_FEATURE_HWP_NOTIFY
))
1466 wrmsrl_on_cpu(cpudata
->cpu
, MSR_HWP_INTERRUPT
, 0x00);
1468 wrmsrl_on_cpu(cpudata
->cpu
, MSR_PM_ENABLE
, 0x1);
1469 if (cpudata
->epp_default
== -EINVAL
)
1470 cpudata
->epp_default
= intel_pstate_get_epp(cpudata
, 0);
1473 static int atom_get_min_pstate(void)
1477 rdmsrl(MSR_ATOM_CORE_RATIOS
, value
);
1478 return (value
>> 8) & 0x7F;
1481 static int atom_get_max_pstate(void)
1485 rdmsrl(MSR_ATOM_CORE_RATIOS
, value
);
1486 return (value
>> 16) & 0x7F;
1489 static int atom_get_turbo_pstate(void)
1493 rdmsrl(MSR_ATOM_CORE_TURBO_RATIOS
, value
);
1494 return value
& 0x7F;
1497 static u64
atom_get_val(struct cpudata
*cpudata
, int pstate
)
1503 val
= (u64
)pstate
<< 8;
1504 if (global
.no_turbo
&& !global
.turbo_disabled
)
1505 val
|= (u64
)1 << 32;
1507 vid_fp
= cpudata
->vid
.min
+ mul_fp(
1508 int_tofp(pstate
- cpudata
->pstate
.min_pstate
),
1509 cpudata
->vid
.ratio
);
1511 vid_fp
= clamp_t(int32_t, vid_fp
, cpudata
->vid
.min
, cpudata
->vid
.max
);
1512 vid
= ceiling_fp(vid_fp
);
1514 if (pstate
> cpudata
->pstate
.max_pstate
)
1515 vid
= cpudata
->vid
.turbo
;
1520 static int silvermont_get_scaling(void)
1524 /* Defined in Table 35-6 from SDM (Sept 2015) */
1525 static int silvermont_freq_table
[] = {
1526 83300, 100000, 133300, 116700, 80000};
1528 rdmsrl(MSR_FSB_FREQ
, value
);
1532 return silvermont_freq_table
[i
];
1535 static int airmont_get_scaling(void)
1539 /* Defined in Table 35-10 from SDM (Sept 2015) */
1540 static int airmont_freq_table
[] = {
1541 83300, 100000, 133300, 116700, 80000,
1542 93300, 90000, 88900, 87500};
1544 rdmsrl(MSR_FSB_FREQ
, value
);
1548 return airmont_freq_table
[i
];
1551 static void atom_get_vid(struct cpudata
*cpudata
)
1555 rdmsrl(MSR_ATOM_CORE_VIDS
, value
);
1556 cpudata
->vid
.min
= int_tofp((value
>> 8) & 0x7f);
1557 cpudata
->vid
.max
= int_tofp((value
>> 16) & 0x7f);
1558 cpudata
->vid
.ratio
= div_fp(
1559 cpudata
->vid
.max
- cpudata
->vid
.min
,
1560 int_tofp(cpudata
->pstate
.max_pstate
-
1561 cpudata
->pstate
.min_pstate
));
1563 rdmsrl(MSR_ATOM_CORE_TURBO_VIDS
, value
);
1564 cpudata
->vid
.turbo
= value
& 0x7f;
1567 static int core_get_min_pstate(void)
1571 rdmsrl(MSR_PLATFORM_INFO
, value
);
1572 return (value
>> 40) & 0xFF;
1575 static int core_get_max_pstate_physical(void)
1579 rdmsrl(MSR_PLATFORM_INFO
, value
);
1580 return (value
>> 8) & 0xFF;
1583 static int core_get_tdp_ratio(u64 plat_info
)
1585 /* Check how many TDP levels present */
1586 if (plat_info
& 0x600000000) {
1592 /* Get the TDP level (0, 1, 2) to get ratios */
1593 err
= rdmsrl_safe(MSR_CONFIG_TDP_CONTROL
, &tdp_ctrl
);
1597 /* TDP MSR are continuous starting at 0x648 */
1598 tdp_msr
= MSR_CONFIG_TDP_NOMINAL
+ (tdp_ctrl
& 0x03);
1599 err
= rdmsrl_safe(tdp_msr
, &tdp_ratio
);
1603 /* For level 1 and 2, bits[23:16] contain the ratio */
1604 if (tdp_ctrl
& 0x03)
1607 tdp_ratio
&= 0xff; /* ratios are only 8 bits long */
1608 pr_debug("tdp_ratio %x\n", (int)tdp_ratio
);
1610 return (int)tdp_ratio
;
1616 static int core_get_max_pstate(void)
1624 rdmsrl(MSR_PLATFORM_INFO
, plat_info
);
1625 max_pstate
= (plat_info
>> 8) & 0xFF;
1627 tdp_ratio
= core_get_tdp_ratio(plat_info
);
1632 /* Turbo activation ratio is not used on HWP platforms */
1636 err
= rdmsrl_safe(MSR_TURBO_ACTIVATION_RATIO
, &tar
);
1640 /* Do some sanity checking for safety */
1641 tar_levels
= tar
& 0xff;
1642 if (tdp_ratio
- 1 == tar_levels
) {
1643 max_pstate
= tar_levels
;
1644 pr_debug("max_pstate=TAC %x\n", max_pstate
);
1651 static int core_get_turbo_pstate(void)
1656 rdmsrl(MSR_TURBO_RATIO_LIMIT
, value
);
1657 nont
= core_get_max_pstate();
1658 ret
= (value
) & 255;
1664 static inline int core_get_scaling(void)
1669 static u64
core_get_val(struct cpudata
*cpudata
, int pstate
)
1673 val
= (u64
)pstate
<< 8;
1674 if (global
.no_turbo
&& !global
.turbo_disabled
)
1675 val
|= (u64
)1 << 32;
1680 static int knl_get_aperf_mperf_shift(void)
1685 static int knl_get_turbo_pstate(void)
1690 rdmsrl(MSR_TURBO_RATIO_LIMIT
, value
);
1691 nont
= core_get_max_pstate();
1692 ret
= (((value
) >> 8) & 0xFF);
1698 static void intel_pstate_set_pstate(struct cpudata
*cpu
, int pstate
)
1700 trace_cpu_frequency(pstate
* cpu
->pstate
.scaling
, cpu
->cpu
);
1701 cpu
->pstate
.current_pstate
= pstate
;
1703 * Generally, there is no guarantee that this code will always run on
1704 * the CPU being updated, so force the register update to run on the
1707 wrmsrl_on_cpu(cpu
->cpu
, MSR_IA32_PERF_CTL
,
1708 pstate_funcs
.get_val(cpu
, pstate
));
1711 static void intel_pstate_set_min_pstate(struct cpudata
*cpu
)
1713 intel_pstate_set_pstate(cpu
, cpu
->pstate
.min_pstate
);
1716 static void intel_pstate_max_within_limits(struct cpudata
*cpu
)
1718 int pstate
= max(cpu
->pstate
.min_pstate
, cpu
->max_perf_ratio
);
1720 update_turbo_state();
1721 intel_pstate_set_pstate(cpu
, pstate
);
1724 static void intel_pstate_get_cpu_pstates(struct cpudata
*cpu
)
1726 cpu
->pstate
.min_pstate
= pstate_funcs
.get_min();
1727 cpu
->pstate
.max_pstate
= pstate_funcs
.get_max();
1728 cpu
->pstate
.max_pstate_physical
= pstate_funcs
.get_max_physical();
1729 cpu
->pstate
.turbo_pstate
= pstate_funcs
.get_turbo();
1730 cpu
->pstate
.scaling
= pstate_funcs
.get_scaling();
1731 cpu
->pstate
.max_freq
= cpu
->pstate
.max_pstate
* cpu
->pstate
.scaling
;
1733 if (hwp_active
&& !hwp_mode_bdw
) {
1734 unsigned int phy_max
, current_max
;
1736 intel_pstate_get_hwp_max(cpu
->cpu
, &phy_max
, ¤t_max
);
1737 cpu
->pstate
.turbo_freq
= phy_max
* cpu
->pstate
.scaling
;
1738 cpu
->pstate
.turbo_pstate
= phy_max
;
1740 cpu
->pstate
.turbo_freq
= cpu
->pstate
.turbo_pstate
* cpu
->pstate
.scaling
;
1743 if (pstate_funcs
.get_aperf_mperf_shift
)
1744 cpu
->aperf_mperf_shift
= pstate_funcs
.get_aperf_mperf_shift();
1746 if (pstate_funcs
.get_vid
)
1747 pstate_funcs
.get_vid(cpu
);
1749 intel_pstate_set_min_pstate(cpu
);
1753 * Long hold time will keep high perf limits for long time,
1754 * which negatively impacts perf/watt for some workloads,
1755 * like specpower. 3ms is based on experiements on some
1758 static int hwp_boost_hold_time_ns
= 3 * NSEC_PER_MSEC
;
1760 static inline void intel_pstate_hwp_boost_up(struct cpudata
*cpu
)
1762 u64 hwp_req
= READ_ONCE(cpu
->hwp_req_cached
);
1763 u32 max_limit
= (hwp_req
& 0xff00) >> 8;
1764 u32 min_limit
= (hwp_req
& 0xff);
1768 * Cases to consider (User changes via sysfs or boot time):
1769 * If, P0 (Turbo max) = P1 (Guaranteed max) = min:
1771 * If, P0 (Turbo max) > P1 (Guaranteed max) = min:
1772 * Should result in one level boost only for P0.
1773 * If, P0 (Turbo max) = P1 (Guaranteed max) > min:
1774 * Should result in two level boost:
1775 * (min + p1)/2 and P1.
1776 * If, P0 (Turbo max) > P1 (Guaranteed max) > min:
1777 * Should result in three level boost:
1778 * (min + p1)/2, P1 and P0.
1781 /* If max and min are equal or already at max, nothing to boost */
1782 if (max_limit
== min_limit
|| cpu
->hwp_boost_min
>= max_limit
)
1785 if (!cpu
->hwp_boost_min
)
1786 cpu
->hwp_boost_min
= min_limit
;
1788 /* level at half way mark between min and guranteed */
1789 boost_level1
= (HWP_GUARANTEED_PERF(cpu
->hwp_cap_cached
) + min_limit
) >> 1;
1791 if (cpu
->hwp_boost_min
< boost_level1
)
1792 cpu
->hwp_boost_min
= boost_level1
;
1793 else if (cpu
->hwp_boost_min
< HWP_GUARANTEED_PERF(cpu
->hwp_cap_cached
))
1794 cpu
->hwp_boost_min
= HWP_GUARANTEED_PERF(cpu
->hwp_cap_cached
);
1795 else if (cpu
->hwp_boost_min
== HWP_GUARANTEED_PERF(cpu
->hwp_cap_cached
) &&
1796 max_limit
!= HWP_GUARANTEED_PERF(cpu
->hwp_cap_cached
))
1797 cpu
->hwp_boost_min
= max_limit
;
1801 hwp_req
= (hwp_req
& ~GENMASK_ULL(7, 0)) | cpu
->hwp_boost_min
;
1802 wrmsrl(MSR_HWP_REQUEST
, hwp_req
);
1803 cpu
->last_update
= cpu
->sample
.time
;
1806 static inline void intel_pstate_hwp_boost_down(struct cpudata
*cpu
)
1808 if (cpu
->hwp_boost_min
) {
1811 /* Check if we are idle for hold time to boost down */
1812 expired
= time_after64(cpu
->sample
.time
, cpu
->last_update
+
1813 hwp_boost_hold_time_ns
);
1815 wrmsrl(MSR_HWP_REQUEST
, cpu
->hwp_req_cached
);
1816 cpu
->hwp_boost_min
= 0;
1819 cpu
->last_update
= cpu
->sample
.time
;
1822 static inline void intel_pstate_update_util_hwp_local(struct cpudata
*cpu
,
1825 cpu
->sample
.time
= time
;
1827 if (cpu
->sched_flags
& SCHED_CPUFREQ_IOWAIT
) {
1830 cpu
->sched_flags
= 0;
1832 * Set iowait_boost flag and update time. Since IO WAIT flag
1833 * is set all the time, we can't just conclude that there is
1834 * some IO bound activity is scheduled on this CPU with just
1835 * one occurrence. If we receive at least two in two
1836 * consecutive ticks, then we treat as boost candidate.
1838 if (time_before64(time
, cpu
->last_io_update
+ 2 * TICK_NSEC
))
1841 cpu
->last_io_update
= time
;
1844 intel_pstate_hwp_boost_up(cpu
);
1847 intel_pstate_hwp_boost_down(cpu
);
1851 static inline void intel_pstate_update_util_hwp(struct update_util_data
*data
,
1852 u64 time
, unsigned int flags
)
1854 struct cpudata
*cpu
= container_of(data
, struct cpudata
, update_util
);
1856 cpu
->sched_flags
|= flags
;
1858 if (smp_processor_id() == cpu
->cpu
)
1859 intel_pstate_update_util_hwp_local(cpu
, time
);
1862 static inline void intel_pstate_calc_avg_perf(struct cpudata
*cpu
)
1864 struct sample
*sample
= &cpu
->sample
;
1866 sample
->core_avg_perf
= div_ext_fp(sample
->aperf
, sample
->mperf
);
1869 static inline bool intel_pstate_sample(struct cpudata
*cpu
, u64 time
)
1872 unsigned long flags
;
1875 local_irq_save(flags
);
1876 rdmsrl(MSR_IA32_APERF
, aperf
);
1877 rdmsrl(MSR_IA32_MPERF
, mperf
);
1879 if (cpu
->prev_mperf
== mperf
|| cpu
->prev_tsc
== tsc
) {
1880 local_irq_restore(flags
);
1883 local_irq_restore(flags
);
1885 cpu
->last_sample_time
= cpu
->sample
.time
;
1886 cpu
->sample
.time
= time
;
1887 cpu
->sample
.aperf
= aperf
;
1888 cpu
->sample
.mperf
= mperf
;
1889 cpu
->sample
.tsc
= tsc
;
1890 cpu
->sample
.aperf
-= cpu
->prev_aperf
;
1891 cpu
->sample
.mperf
-= cpu
->prev_mperf
;
1892 cpu
->sample
.tsc
-= cpu
->prev_tsc
;
1894 cpu
->prev_aperf
= aperf
;
1895 cpu
->prev_mperf
= mperf
;
1896 cpu
->prev_tsc
= tsc
;
1898 * First time this function is invoked in a given cycle, all of the
1899 * previous sample data fields are equal to zero or stale and they must
1900 * be populated with meaningful numbers for things to work, so assume
1901 * that sample.time will always be reset before setting the utilization
1902 * update hook and make the caller skip the sample then.
1904 if (cpu
->last_sample_time
) {
1905 intel_pstate_calc_avg_perf(cpu
);
1911 static inline int32_t get_avg_frequency(struct cpudata
*cpu
)
1913 return mul_ext_fp(cpu
->sample
.core_avg_perf
, cpu_khz
);
1916 static inline int32_t get_avg_pstate(struct cpudata
*cpu
)
1918 return mul_ext_fp(cpu
->pstate
.max_pstate_physical
,
1919 cpu
->sample
.core_avg_perf
);
1922 static inline int32_t get_target_pstate(struct cpudata
*cpu
)
1924 struct sample
*sample
= &cpu
->sample
;
1926 int target
, avg_pstate
;
1928 busy_frac
= div_fp(sample
->mperf
<< cpu
->aperf_mperf_shift
,
1931 if (busy_frac
< cpu
->iowait_boost
)
1932 busy_frac
= cpu
->iowait_boost
;
1934 sample
->busy_scaled
= busy_frac
* 100;
1936 target
= global
.no_turbo
|| global
.turbo_disabled
?
1937 cpu
->pstate
.max_pstate
: cpu
->pstate
.turbo_pstate
;
1938 target
+= target
>> 2;
1939 target
= mul_fp(target
, busy_frac
);
1940 if (target
< cpu
->pstate
.min_pstate
)
1941 target
= cpu
->pstate
.min_pstate
;
1944 * If the average P-state during the previous cycle was higher than the
1945 * current target, add 50% of the difference to the target to reduce
1946 * possible performance oscillations and offset possible performance
1947 * loss related to moving the workload from one CPU to another within
1950 avg_pstate
= get_avg_pstate(cpu
);
1951 if (avg_pstate
> target
)
1952 target
+= (avg_pstate
- target
) >> 1;
1957 static int intel_pstate_prepare_request(struct cpudata
*cpu
, int pstate
)
1959 int min_pstate
= max(cpu
->pstate
.min_pstate
, cpu
->min_perf_ratio
);
1960 int max_pstate
= max(min_pstate
, cpu
->max_perf_ratio
);
1962 return clamp_t(int, pstate
, min_pstate
, max_pstate
);
1965 static void intel_pstate_update_pstate(struct cpudata
*cpu
, int pstate
)
1967 if (pstate
== cpu
->pstate
.current_pstate
)
1970 cpu
->pstate
.current_pstate
= pstate
;
1971 wrmsrl(MSR_IA32_PERF_CTL
, pstate_funcs
.get_val(cpu
, pstate
));
1974 static void intel_pstate_adjust_pstate(struct cpudata
*cpu
)
1976 int from
= cpu
->pstate
.current_pstate
;
1977 struct sample
*sample
;
1980 update_turbo_state();
1982 target_pstate
= get_target_pstate(cpu
);
1983 target_pstate
= intel_pstate_prepare_request(cpu
, target_pstate
);
1984 trace_cpu_frequency(target_pstate
* cpu
->pstate
.scaling
, cpu
->cpu
);
1985 intel_pstate_update_pstate(cpu
, target_pstate
);
1987 sample
= &cpu
->sample
;
1988 trace_pstate_sample(mul_ext_fp(100, sample
->core_avg_perf
),
1989 fp_toint(sample
->busy_scaled
),
1991 cpu
->pstate
.current_pstate
,
1995 get_avg_frequency(cpu
),
1996 fp_toint(cpu
->iowait_boost
* 100));
1999 static void intel_pstate_update_util(struct update_util_data
*data
, u64 time
,
2002 struct cpudata
*cpu
= container_of(data
, struct cpudata
, update_util
);
2005 /* Don't allow remote callbacks */
2006 if (smp_processor_id() != cpu
->cpu
)
2009 delta_ns
= time
- cpu
->last_update
;
2010 if (flags
& SCHED_CPUFREQ_IOWAIT
) {
2011 /* Start over if the CPU may have been idle. */
2012 if (delta_ns
> TICK_NSEC
) {
2013 cpu
->iowait_boost
= ONE_EIGHTH_FP
;
2014 } else if (cpu
->iowait_boost
>= ONE_EIGHTH_FP
) {
2015 cpu
->iowait_boost
<<= 1;
2016 if (cpu
->iowait_boost
> int_tofp(1))
2017 cpu
->iowait_boost
= int_tofp(1);
2019 cpu
->iowait_boost
= ONE_EIGHTH_FP
;
2021 } else if (cpu
->iowait_boost
) {
2022 /* Clear iowait_boost if the CPU may have been idle. */
2023 if (delta_ns
> TICK_NSEC
)
2024 cpu
->iowait_boost
= 0;
2026 cpu
->iowait_boost
>>= 1;
2028 cpu
->last_update
= time
;
2029 delta_ns
= time
- cpu
->sample
.time
;
2030 if ((s64
)delta_ns
< INTEL_PSTATE_SAMPLING_INTERVAL
)
2033 if (intel_pstate_sample(cpu
, time
))
2034 intel_pstate_adjust_pstate(cpu
);
2037 static struct pstate_funcs core_funcs
= {
2038 .get_max
= core_get_max_pstate
,
2039 .get_max_physical
= core_get_max_pstate_physical
,
2040 .get_min
= core_get_min_pstate
,
2041 .get_turbo
= core_get_turbo_pstate
,
2042 .get_scaling
= core_get_scaling
,
2043 .get_val
= core_get_val
,
2046 static const struct pstate_funcs silvermont_funcs
= {
2047 .get_max
= atom_get_max_pstate
,
2048 .get_max_physical
= atom_get_max_pstate
,
2049 .get_min
= atom_get_min_pstate
,
2050 .get_turbo
= atom_get_turbo_pstate
,
2051 .get_val
= atom_get_val
,
2052 .get_scaling
= silvermont_get_scaling
,
2053 .get_vid
= atom_get_vid
,
2056 static const struct pstate_funcs airmont_funcs
= {
2057 .get_max
= atom_get_max_pstate
,
2058 .get_max_physical
= atom_get_max_pstate
,
2059 .get_min
= atom_get_min_pstate
,
2060 .get_turbo
= atom_get_turbo_pstate
,
2061 .get_val
= atom_get_val
,
2062 .get_scaling
= airmont_get_scaling
,
2063 .get_vid
= atom_get_vid
,
2066 static const struct pstate_funcs knl_funcs
= {
2067 .get_max
= core_get_max_pstate
,
2068 .get_max_physical
= core_get_max_pstate_physical
,
2069 .get_min
= core_get_min_pstate
,
2070 .get_turbo
= knl_get_turbo_pstate
,
2071 .get_aperf_mperf_shift
= knl_get_aperf_mperf_shift
,
2072 .get_scaling
= core_get_scaling
,
2073 .get_val
= core_get_val
,
2076 #define X86_MATCH(model, policy) \
2077 X86_MATCH_VENDOR_FAM_MODEL_FEATURE(INTEL, 6, INTEL_FAM6_##model, \
2078 X86_FEATURE_APERFMPERF, &policy)
2080 static const struct x86_cpu_id intel_pstate_cpu_ids
[] = {
2081 X86_MATCH(SANDYBRIDGE
, core_funcs
),
2082 X86_MATCH(SANDYBRIDGE_X
, core_funcs
),
2083 X86_MATCH(ATOM_SILVERMONT
, silvermont_funcs
),
2084 X86_MATCH(IVYBRIDGE
, core_funcs
),
2085 X86_MATCH(HASWELL
, core_funcs
),
2086 X86_MATCH(BROADWELL
, core_funcs
),
2087 X86_MATCH(IVYBRIDGE_X
, core_funcs
),
2088 X86_MATCH(HASWELL_X
, core_funcs
),
2089 X86_MATCH(HASWELL_L
, core_funcs
),
2090 X86_MATCH(HASWELL_G
, core_funcs
),
2091 X86_MATCH(BROADWELL_G
, core_funcs
),
2092 X86_MATCH(ATOM_AIRMONT
, airmont_funcs
),
2093 X86_MATCH(SKYLAKE_L
, core_funcs
),
2094 X86_MATCH(BROADWELL_X
, core_funcs
),
2095 X86_MATCH(SKYLAKE
, core_funcs
),
2096 X86_MATCH(BROADWELL_D
, core_funcs
),
2097 X86_MATCH(XEON_PHI_KNL
, knl_funcs
),
2098 X86_MATCH(XEON_PHI_KNM
, knl_funcs
),
2099 X86_MATCH(ATOM_GOLDMONT
, core_funcs
),
2100 X86_MATCH(ATOM_GOLDMONT_PLUS
, core_funcs
),
2101 X86_MATCH(SKYLAKE_X
, core_funcs
),
2104 MODULE_DEVICE_TABLE(x86cpu
, intel_pstate_cpu_ids
);
2106 static const struct x86_cpu_id intel_pstate_cpu_oob_ids
[] __initconst
= {
2107 X86_MATCH(BROADWELL_D
, core_funcs
),
2108 X86_MATCH(BROADWELL_X
, core_funcs
),
2109 X86_MATCH(SKYLAKE_X
, core_funcs
),
2113 static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids
[] = {
2114 X86_MATCH(KABYLAKE
, core_funcs
),
2118 static const struct x86_cpu_id intel_pstate_hwp_boost_ids
[] = {
2119 X86_MATCH(SKYLAKE_X
, core_funcs
),
2120 X86_MATCH(SKYLAKE
, core_funcs
),
2124 static int intel_pstate_init_cpu(unsigned int cpunum
)
2126 struct cpudata
*cpu
;
2128 cpu
= all_cpu_data
[cpunum
];
2131 cpu
= kzalloc(sizeof(*cpu
), GFP_KERNEL
);
2135 all_cpu_data
[cpunum
] = cpu
;
2139 cpu
->epp_default
= -EINVAL
;
2142 const struct x86_cpu_id
*id
;
2144 intel_pstate_hwp_enable(cpu
);
2146 id
= x86_match_cpu(intel_pstate_hwp_boost_ids
);
2147 if (id
&& intel_pstate_acpi_pm_profile_server())
2150 } else if (hwp_active
) {
2152 * Re-enable HWP in case this happens after a resume from ACPI
2153 * S3 if the CPU was offline during the whole system/resume
2156 intel_pstate_hwp_reenable(cpu
);
2159 cpu
->epp_powersave
= -EINVAL
;
2160 cpu
->epp_policy
= 0;
2162 intel_pstate_get_cpu_pstates(cpu
);
2164 pr_debug("controlling: cpu %d\n", cpunum
);
2169 static void intel_pstate_set_update_util_hook(unsigned int cpu_num
)
2171 struct cpudata
*cpu
= all_cpu_data
[cpu_num
];
2173 if (hwp_active
&& !hwp_boost
)
2176 if (cpu
->update_util_set
)
2179 /* Prevent intel_pstate_update_util() from using stale data. */
2180 cpu
->sample
.time
= 0;
2181 cpufreq_add_update_util_hook(cpu_num
, &cpu
->update_util
,
2183 intel_pstate_update_util_hwp
:
2184 intel_pstate_update_util
));
2185 cpu
->update_util_set
= true;
2188 static void intel_pstate_clear_update_util_hook(unsigned int cpu
)
2190 struct cpudata
*cpu_data
= all_cpu_data
[cpu
];
2192 if (!cpu_data
->update_util_set
)
2195 cpufreq_remove_update_util_hook(cpu
);
2196 cpu_data
->update_util_set
= false;
2200 static int intel_pstate_get_max_freq(struct cpudata
*cpu
)
2202 return global
.turbo_disabled
|| global
.no_turbo
?
2203 cpu
->pstate
.max_freq
: cpu
->pstate
.turbo_freq
;
2206 static void intel_pstate_update_perf_limits(struct cpudata
*cpu
,
2207 unsigned int policy_min
,
2208 unsigned int policy_max
)
2210 int max_freq
= intel_pstate_get_max_freq(cpu
);
2211 int32_t max_policy_perf
, min_policy_perf
;
2212 int max_state
, turbo_max
;
2215 * HWP needs some special consideration, because on BDX the
2216 * HWP_REQUEST uses abstract value to represent performance
2217 * rather than pure ratios.
2220 intel_pstate_get_hwp_max(cpu
->cpu
, &turbo_max
, &max_state
);
2222 max_state
= global
.no_turbo
|| global
.turbo_disabled
?
2223 cpu
->pstate
.max_pstate
: cpu
->pstate
.turbo_pstate
;
2224 turbo_max
= cpu
->pstate
.turbo_pstate
;
2227 max_policy_perf
= max_state
* policy_max
/ max_freq
;
2228 if (policy_max
== policy_min
) {
2229 min_policy_perf
= max_policy_perf
;
2231 min_policy_perf
= max_state
* policy_min
/ max_freq
;
2232 min_policy_perf
= clamp_t(int32_t, min_policy_perf
,
2233 0, max_policy_perf
);
2236 pr_debug("cpu:%d max_state %d min_policy_perf:%d max_policy_perf:%d\n",
2237 cpu
->cpu
, max_state
, min_policy_perf
, max_policy_perf
);
2239 /* Normalize user input to [min_perf, max_perf] */
2240 if (per_cpu_limits
) {
2241 cpu
->min_perf_ratio
= min_policy_perf
;
2242 cpu
->max_perf_ratio
= max_policy_perf
;
2244 int32_t global_min
, global_max
;
2246 /* Global limits are in percent of the maximum turbo P-state. */
2247 global_max
= DIV_ROUND_UP(turbo_max
* global
.max_perf_pct
, 100);
2248 global_min
= DIV_ROUND_UP(turbo_max
* global
.min_perf_pct
, 100);
2249 global_min
= clamp_t(int32_t, global_min
, 0, global_max
);
2251 pr_debug("cpu:%d global_min:%d global_max:%d\n", cpu
->cpu
,
2252 global_min
, global_max
);
2254 cpu
->min_perf_ratio
= max(min_policy_perf
, global_min
);
2255 cpu
->min_perf_ratio
= min(cpu
->min_perf_ratio
, max_policy_perf
);
2256 cpu
->max_perf_ratio
= min(max_policy_perf
, global_max
);
2257 cpu
->max_perf_ratio
= max(min_policy_perf
, cpu
->max_perf_ratio
);
2259 /* Make sure min_perf <= max_perf */
2260 cpu
->min_perf_ratio
= min(cpu
->min_perf_ratio
,
2261 cpu
->max_perf_ratio
);
2264 pr_debug("cpu:%d max_perf_ratio:%d min_perf_ratio:%d\n", cpu
->cpu
,
2265 cpu
->max_perf_ratio
,
2266 cpu
->min_perf_ratio
);
2269 static int intel_pstate_set_policy(struct cpufreq_policy
*policy
)
2271 struct cpudata
*cpu
;
2273 if (!policy
->cpuinfo
.max_freq
)
2276 pr_debug("set_policy cpuinfo.max %u policy->max %u\n",
2277 policy
->cpuinfo
.max_freq
, policy
->max
);
2279 cpu
= all_cpu_data
[policy
->cpu
];
2280 cpu
->policy
= policy
->policy
;
2282 mutex_lock(&intel_pstate_limits_lock
);
2284 intel_pstate_update_perf_limits(cpu
, policy
->min
, policy
->max
);
2286 if (cpu
->policy
== CPUFREQ_POLICY_PERFORMANCE
) {
2288 * NOHZ_FULL CPUs need this as the governor callback may not
2289 * be invoked on them.
2291 intel_pstate_clear_update_util_hook(policy
->cpu
);
2292 intel_pstate_max_within_limits(cpu
);
2294 intel_pstate_set_update_util_hook(policy
->cpu
);
2299 * When hwp_boost was active before and dynamically it
2300 * was turned off, in that case we need to clear the
2304 intel_pstate_clear_update_util_hook(policy
->cpu
);
2305 intel_pstate_hwp_set(policy
->cpu
);
2308 mutex_unlock(&intel_pstate_limits_lock
);
2313 static void intel_pstate_adjust_policy_max(struct cpudata
*cpu
,
2314 struct cpufreq_policy_data
*policy
)
2317 cpu
->pstate
.max_pstate_physical
> cpu
->pstate
.max_pstate
&&
2318 policy
->max
< policy
->cpuinfo
.max_freq
&&
2319 policy
->max
> cpu
->pstate
.max_freq
) {
2320 pr_debug("policy->max > max non turbo frequency\n");
2321 policy
->max
= policy
->cpuinfo
.max_freq
;
2325 static void intel_pstate_verify_cpu_policy(struct cpudata
*cpu
,
2326 struct cpufreq_policy_data
*policy
)
2328 update_turbo_state();
2329 cpufreq_verify_within_limits(policy
, policy
->cpuinfo
.min_freq
,
2330 intel_pstate_get_max_freq(cpu
));
2332 intel_pstate_adjust_policy_max(cpu
, policy
);
2335 static int intel_pstate_verify_policy(struct cpufreq_policy_data
*policy
)
2337 intel_pstate_verify_cpu_policy(all_cpu_data
[policy
->cpu
], policy
);
2342 static int intel_pstate_cpu_offline(struct cpufreq_policy
*policy
)
2344 struct cpudata
*cpu
= all_cpu_data
[policy
->cpu
];
2346 pr_debug("CPU %d going offline\n", cpu
->cpu
);
2352 * If the CPU is an SMT thread and it goes offline with the performance
2353 * settings different from the minimum, it will prevent its sibling
2354 * from getting to lower performance levels, so force the minimum
2355 * performance on CPU offline to prevent that from happening.
2358 intel_pstate_hwp_offline(cpu
);
2360 intel_pstate_set_min_pstate(cpu
);
2362 intel_pstate_exit_perf_limits(policy
);
2367 static int intel_pstate_cpu_online(struct cpufreq_policy
*policy
)
2369 struct cpudata
*cpu
= all_cpu_data
[policy
->cpu
];
2371 pr_debug("CPU %d going online\n", cpu
->cpu
);
2373 intel_pstate_init_acpi_perf_limits(policy
);
2377 * Re-enable HWP and clear the "suspended" flag to let "resume"
2378 * know that it need not do that.
2380 intel_pstate_hwp_reenable(cpu
);
2381 cpu
->suspended
= false;
2387 static void intel_pstate_stop_cpu(struct cpufreq_policy
*policy
)
2389 pr_debug("CPU %d stopping\n", policy
->cpu
);
2391 intel_pstate_clear_update_util_hook(policy
->cpu
);
2394 static int intel_pstate_cpu_exit(struct cpufreq_policy
*policy
)
2396 pr_debug("CPU %d exiting\n", policy
->cpu
);
2398 policy
->fast_switch_possible
= false;
2403 static int __intel_pstate_cpu_init(struct cpufreq_policy
*policy
)
2405 struct cpudata
*cpu
;
2408 rc
= intel_pstate_init_cpu(policy
->cpu
);
2412 cpu
= all_cpu_data
[policy
->cpu
];
2414 cpu
->max_perf_ratio
= 0xFF;
2415 cpu
->min_perf_ratio
= 0;
2417 policy
->min
= cpu
->pstate
.min_pstate
* cpu
->pstate
.scaling
;
2418 policy
->max
= cpu
->pstate
.turbo_pstate
* cpu
->pstate
.scaling
;
2420 /* cpuinfo and default policy values */
2421 policy
->cpuinfo
.min_freq
= cpu
->pstate
.min_pstate
* cpu
->pstate
.scaling
;
2422 update_turbo_state();
2423 global
.turbo_disabled_mf
= global
.turbo_disabled
;
2424 policy
->cpuinfo
.max_freq
= global
.turbo_disabled
?
2425 cpu
->pstate
.max_pstate
: cpu
->pstate
.turbo_pstate
;
2426 policy
->cpuinfo
.max_freq
*= cpu
->pstate
.scaling
;
2429 unsigned int max_freq
;
2431 max_freq
= global
.turbo_disabled
?
2432 cpu
->pstate
.max_freq
: cpu
->pstate
.turbo_freq
;
2433 if (max_freq
< policy
->cpuinfo
.max_freq
)
2434 policy
->cpuinfo
.max_freq
= max_freq
;
2437 intel_pstate_init_acpi_perf_limits(policy
);
2439 policy
->fast_switch_possible
= true;
2444 static int intel_pstate_cpu_init(struct cpufreq_policy
*policy
)
2446 int ret
= __intel_pstate_cpu_init(policy
);
2452 * Set the policy to powersave to provide a valid fallback value in case
2453 * the default cpufreq governor is neither powersave nor performance.
2455 policy
->policy
= CPUFREQ_POLICY_POWERSAVE
;
2458 struct cpudata
*cpu
= all_cpu_data
[policy
->cpu
];
2460 cpu
->epp_cached
= intel_pstate_get_epp(cpu
, 0);
2466 static struct cpufreq_driver intel_pstate
= {
2467 .flags
= CPUFREQ_CONST_LOOPS
,
2468 .verify
= intel_pstate_verify_policy
,
2469 .setpolicy
= intel_pstate_set_policy
,
2470 .suspend
= intel_pstate_suspend
,
2471 .resume
= intel_pstate_resume
,
2472 .init
= intel_pstate_cpu_init
,
2473 .exit
= intel_pstate_cpu_exit
,
2474 .stop_cpu
= intel_pstate_stop_cpu
,
2475 .offline
= intel_pstate_cpu_offline
,
2476 .online
= intel_pstate_cpu_online
,
2477 .update_limits
= intel_pstate_update_limits
,
2478 .name
= "intel_pstate",
2481 static int intel_cpufreq_verify_policy(struct cpufreq_policy_data
*policy
)
2483 struct cpudata
*cpu
= all_cpu_data
[policy
->cpu
];
2485 intel_pstate_verify_cpu_policy(cpu
, policy
);
2486 intel_pstate_update_perf_limits(cpu
, policy
->min
, policy
->max
);
2491 /* Use of trace in passive mode:
2493 * In passive mode the trace core_busy field (also known as the
2494 * performance field, and lablelled as such on the graphs; also known as
2495 * core_avg_perf) is not needed and so is re-assigned to indicate if the
2496 * driver call was via the normal or fast switch path. Various graphs
2497 * output from the intel_pstate_tracer.py utility that include core_busy
2498 * (or performance or core_avg_perf) have a fixed y-axis from 0 to 100%,
2499 * so we use 10 to indicate the the normal path through the driver, and
2500 * 90 to indicate the fast switch path through the driver.
2501 * The scaled_busy field is not used, and is set to 0.
2504 #define INTEL_PSTATE_TRACE_TARGET 10
2505 #define INTEL_PSTATE_TRACE_FAST_SWITCH 90
2507 static void intel_cpufreq_trace(struct cpudata
*cpu
, unsigned int trace_type
, int old_pstate
)
2509 struct sample
*sample
;
2511 if (!trace_pstate_sample_enabled())
2514 if (!intel_pstate_sample(cpu
, ktime_get()))
2517 sample
= &cpu
->sample
;
2518 trace_pstate_sample(trace_type
,
2521 cpu
->pstate
.current_pstate
,
2525 get_avg_frequency(cpu
),
2526 fp_toint(cpu
->iowait_boost
* 100));
2529 static void intel_cpufreq_adjust_hwp(struct cpudata
*cpu
, u32 target_pstate
,
2530 bool strict
, bool fast_switch
)
2532 u64 prev
= READ_ONCE(cpu
->hwp_req_cached
), value
= prev
;
2534 value
&= ~HWP_MIN_PERF(~0L);
2535 value
|= HWP_MIN_PERF(target_pstate
);
2538 * The entire MSR needs to be updated in order to update the HWP min
2539 * field in it, so opportunistically update the max too if needed.
2541 value
&= ~HWP_MAX_PERF(~0L);
2542 value
|= HWP_MAX_PERF(strict
? target_pstate
: cpu
->max_perf_ratio
);
2547 WRITE_ONCE(cpu
->hwp_req_cached
, value
);
2549 wrmsrl(MSR_HWP_REQUEST
, value
);
2551 wrmsrl_on_cpu(cpu
->cpu
, MSR_HWP_REQUEST
, value
);
2554 static void intel_cpufreq_adjust_perf_ctl(struct cpudata
*cpu
,
2555 u32 target_pstate
, bool fast_switch
)
2558 wrmsrl(MSR_IA32_PERF_CTL
,
2559 pstate_funcs
.get_val(cpu
, target_pstate
));
2561 wrmsrl_on_cpu(cpu
->cpu
, MSR_IA32_PERF_CTL
,
2562 pstate_funcs
.get_val(cpu
, target_pstate
));
2565 static int intel_cpufreq_update_pstate(struct cpufreq_policy
*policy
,
2566 int target_pstate
, bool fast_switch
)
2568 struct cpudata
*cpu
= all_cpu_data
[policy
->cpu
];
2569 int old_pstate
= cpu
->pstate
.current_pstate
;
2571 target_pstate
= intel_pstate_prepare_request(cpu
, target_pstate
);
2573 intel_cpufreq_adjust_hwp(cpu
, target_pstate
,
2574 policy
->strict_target
, fast_switch
);
2575 cpu
->pstate
.current_pstate
= target_pstate
;
2576 } else if (target_pstate
!= old_pstate
) {
2577 intel_cpufreq_adjust_perf_ctl(cpu
, target_pstate
, fast_switch
);
2578 cpu
->pstate
.current_pstate
= target_pstate
;
2581 intel_cpufreq_trace(cpu
, fast_switch
? INTEL_PSTATE_TRACE_FAST_SWITCH
:
2582 INTEL_PSTATE_TRACE_TARGET
, old_pstate
);
2584 return target_pstate
;
2587 static int intel_cpufreq_target(struct cpufreq_policy
*policy
,
2588 unsigned int target_freq
,
2589 unsigned int relation
)
2591 struct cpudata
*cpu
= all_cpu_data
[policy
->cpu
];
2592 struct cpufreq_freqs freqs
;
2595 update_turbo_state();
2597 freqs
.old
= policy
->cur
;
2598 freqs
.new = target_freq
;
2600 cpufreq_freq_transition_begin(policy
, &freqs
);
2603 case CPUFREQ_RELATION_L
:
2604 target_pstate
= DIV_ROUND_UP(freqs
.new, cpu
->pstate
.scaling
);
2606 case CPUFREQ_RELATION_H
:
2607 target_pstate
= freqs
.new / cpu
->pstate
.scaling
;
2610 target_pstate
= DIV_ROUND_CLOSEST(freqs
.new, cpu
->pstate
.scaling
);
2614 target_pstate
= intel_cpufreq_update_pstate(policy
, target_pstate
, false);
2616 freqs
.new = target_pstate
* cpu
->pstate
.scaling
;
2618 cpufreq_freq_transition_end(policy
, &freqs
, false);
2623 static unsigned int intel_cpufreq_fast_switch(struct cpufreq_policy
*policy
,
2624 unsigned int target_freq
)
2626 struct cpudata
*cpu
= all_cpu_data
[policy
->cpu
];
2629 update_turbo_state();
2631 target_pstate
= DIV_ROUND_UP(target_freq
, cpu
->pstate
.scaling
);
2633 target_pstate
= intel_cpufreq_update_pstate(policy
, target_pstate
, true);
2635 return target_pstate
* cpu
->pstate
.scaling
;
2638 static int intel_cpufreq_cpu_init(struct cpufreq_policy
*policy
)
2640 int max_state
, turbo_max
, min_freq
, max_freq
, ret
;
2641 struct freq_qos_request
*req
;
2642 struct cpudata
*cpu
;
2645 dev
= get_cpu_device(policy
->cpu
);
2649 ret
= __intel_pstate_cpu_init(policy
);
2653 policy
->cpuinfo
.transition_latency
= INTEL_CPUFREQ_TRANSITION_LATENCY
;
2654 /* This reflects the intel_pstate_get_cpu_pstates() setting. */
2655 policy
->cur
= policy
->cpuinfo
.min_freq
;
2657 req
= kcalloc(2, sizeof(*req
), GFP_KERNEL
);
2663 cpu
= all_cpu_data
[policy
->cpu
];
2668 intel_pstate_get_hwp_max(policy
->cpu
, &turbo_max
, &max_state
);
2669 policy
->transition_delay_us
= INTEL_CPUFREQ_TRANSITION_DELAY_HWP
;
2670 rdmsrl_on_cpu(cpu
->cpu
, MSR_HWP_REQUEST
, &value
);
2671 WRITE_ONCE(cpu
->hwp_req_cached
, value
);
2672 cpu
->epp_cached
= intel_pstate_get_epp(cpu
, value
);
2674 turbo_max
= cpu
->pstate
.turbo_pstate
;
2675 policy
->transition_delay_us
= INTEL_CPUFREQ_TRANSITION_DELAY
;
2678 min_freq
= DIV_ROUND_UP(turbo_max
* global
.min_perf_pct
, 100);
2679 min_freq
*= cpu
->pstate
.scaling
;
2680 max_freq
= DIV_ROUND_UP(turbo_max
* global
.max_perf_pct
, 100);
2681 max_freq
*= cpu
->pstate
.scaling
;
2683 ret
= freq_qos_add_request(&policy
->constraints
, req
, FREQ_QOS_MIN
,
2686 dev_err(dev
, "Failed to add min-freq constraint (%d)\n", ret
);
2690 ret
= freq_qos_add_request(&policy
->constraints
, req
+ 1, FREQ_QOS_MAX
,
2693 dev_err(dev
, "Failed to add max-freq constraint (%d)\n", ret
);
2694 goto remove_min_req
;
2697 policy
->driver_data
= req
;
2702 freq_qos_remove_request(req
);
2706 intel_pstate_exit_perf_limits(policy
);
2711 static int intel_cpufreq_cpu_exit(struct cpufreq_policy
*policy
)
2713 struct freq_qos_request
*req
;
2715 req
= policy
->driver_data
;
2717 freq_qos_remove_request(req
+ 1);
2718 freq_qos_remove_request(req
);
2721 return intel_pstate_cpu_exit(policy
);
2724 static struct cpufreq_driver intel_cpufreq
= {
2725 .flags
= CPUFREQ_CONST_LOOPS
,
2726 .verify
= intel_cpufreq_verify_policy
,
2727 .target
= intel_cpufreq_target
,
2728 .fast_switch
= intel_cpufreq_fast_switch
,
2729 .init
= intel_cpufreq_cpu_init
,
2730 .exit
= intel_cpufreq_cpu_exit
,
2731 .offline
= intel_pstate_cpu_offline
,
2732 .online
= intel_pstate_cpu_online
,
2733 .suspend
= intel_pstate_suspend
,
2734 .resume
= intel_pstate_resume
,
2735 .update_limits
= intel_pstate_update_limits
,
2736 .name
= "intel_cpufreq",
2739 static struct cpufreq_driver
*default_driver
;
2741 static void intel_pstate_driver_cleanup(void)
2746 for_each_online_cpu(cpu
) {
2747 if (all_cpu_data
[cpu
]) {
2748 if (intel_pstate_driver
== &intel_pstate
)
2749 intel_pstate_clear_update_util_hook(cpu
);
2751 kfree(all_cpu_data
[cpu
]);
2752 all_cpu_data
[cpu
] = NULL
;
2757 intel_pstate_driver
= NULL
;
2760 static int intel_pstate_register_driver(struct cpufreq_driver
*driver
)
2764 if (driver
== &intel_pstate
)
2765 intel_pstate_sysfs_expose_hwp_dynamic_boost();
2767 memset(&global
, 0, sizeof(global
));
2768 global
.max_perf_pct
= 100;
2770 intel_pstate_driver
= driver
;
2771 ret
= cpufreq_register_driver(intel_pstate_driver
);
2773 intel_pstate_driver_cleanup();
2777 global
.min_perf_pct
= min_perf_pct_min();
2782 static ssize_t
intel_pstate_show_status(char *buf
)
2784 if (!intel_pstate_driver
)
2785 return sprintf(buf
, "off\n");
2787 return sprintf(buf
, "%s\n", intel_pstate_driver
== &intel_pstate
?
2788 "active" : "passive");
2791 static int intel_pstate_update_status(const char *buf
, size_t size
)
2793 if (size
== 3 && !strncmp(buf
, "off", size
)) {
2794 if (!intel_pstate_driver
)
2800 cpufreq_unregister_driver(intel_pstate_driver
);
2801 intel_pstate_driver_cleanup();
2805 if (size
== 6 && !strncmp(buf
, "active", size
)) {
2806 if (intel_pstate_driver
) {
2807 if (intel_pstate_driver
== &intel_pstate
)
2810 cpufreq_unregister_driver(intel_pstate_driver
);
2813 return intel_pstate_register_driver(&intel_pstate
);
2816 if (size
== 7 && !strncmp(buf
, "passive", size
)) {
2817 if (intel_pstate_driver
) {
2818 if (intel_pstate_driver
== &intel_cpufreq
)
2821 cpufreq_unregister_driver(intel_pstate_driver
);
2822 intel_pstate_sysfs_hide_hwp_dynamic_boost();
2825 return intel_pstate_register_driver(&intel_cpufreq
);
2831 static int no_load __initdata
;
2832 static int no_hwp __initdata
;
2833 static int hwp_only __initdata
;
2834 static unsigned int force_load __initdata
;
2836 static int __init
intel_pstate_msrs_not_valid(void)
2838 if (!pstate_funcs
.get_max() ||
2839 !pstate_funcs
.get_min() ||
2840 !pstate_funcs
.get_turbo())
2846 static void __init
copy_cpu_funcs(struct pstate_funcs
*funcs
)
2848 pstate_funcs
.get_max
= funcs
->get_max
;
2849 pstate_funcs
.get_max_physical
= funcs
->get_max_physical
;
2850 pstate_funcs
.get_min
= funcs
->get_min
;
2851 pstate_funcs
.get_turbo
= funcs
->get_turbo
;
2852 pstate_funcs
.get_scaling
= funcs
->get_scaling
;
2853 pstate_funcs
.get_val
= funcs
->get_val
;
2854 pstate_funcs
.get_vid
= funcs
->get_vid
;
2855 pstate_funcs
.get_aperf_mperf_shift
= funcs
->get_aperf_mperf_shift
;
2860 static bool __init
intel_pstate_no_acpi_pss(void)
2864 for_each_possible_cpu(i
) {
2866 union acpi_object
*pss
;
2867 struct acpi_buffer buffer
= { ACPI_ALLOCATE_BUFFER
, NULL
};
2868 struct acpi_processor
*pr
= per_cpu(processors
, i
);
2873 status
= acpi_evaluate_object(pr
->handle
, "_PSS", NULL
, &buffer
);
2874 if (ACPI_FAILURE(status
))
2877 pss
= buffer
.pointer
;
2878 if (pss
&& pss
->type
== ACPI_TYPE_PACKAGE
) {
2886 pr_debug("ACPI _PSS not found\n");
2890 static bool __init
intel_pstate_no_acpi_pcch(void)
2895 status
= acpi_get_handle(NULL
, "\\_SB", &handle
);
2896 if (ACPI_FAILURE(status
))
2899 if (acpi_has_method(handle
, "PCCH"))
2903 pr_debug("ACPI PCCH not found\n");
2907 static bool __init
intel_pstate_has_acpi_ppc(void)
2911 for_each_possible_cpu(i
) {
2912 struct acpi_processor
*pr
= per_cpu(processors
, i
);
2916 if (acpi_has_method(pr
->handle
, "_PPC"))
2919 pr_debug("ACPI _PPC not found\n");
2928 /* Hardware vendor-specific info that has its own power management modes */
2929 static struct acpi_platform_list plat_info
[] __initdata
= {
2930 {"HP ", "ProLiant", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PSS
},
2931 {"ORACLE", "X4-2 ", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
2932 {"ORACLE", "X4-2L ", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
2933 {"ORACLE", "X4-2B ", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
2934 {"ORACLE", "X3-2 ", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
2935 {"ORACLE", "X3-2L ", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
2936 {"ORACLE", "X3-2B ", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
2937 {"ORACLE", "X4470M2 ", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
2938 {"ORACLE", "X4270M3 ", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
2939 {"ORACLE", "X4270M2 ", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
2940 {"ORACLE", "X4170M2 ", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
2941 {"ORACLE", "X4170 M3", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
2942 {"ORACLE", "X4275 M3", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
2943 {"ORACLE", "X6-2 ", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
2944 {"ORACLE", "Sudbury ", 0, ACPI_SIG_FADT
, all_versions
, NULL
, PPC
},
2948 #define BITMASK_OOB (BIT(8) | BIT(18))
2950 static bool __init
intel_pstate_platform_pwr_mgmt_exists(void)
2952 const struct x86_cpu_id
*id
;
2956 id
= x86_match_cpu(intel_pstate_cpu_oob_ids
);
2958 rdmsrl(MSR_MISC_PWR_MGMT
, misc_pwr
);
2959 if (misc_pwr
& BITMASK_OOB
) {
2960 pr_debug("Bit 8 or 18 in the MISC_PWR_MGMT MSR set\n");
2961 pr_debug("P states are controlled in Out of Band mode by the firmware/hardware\n");
2966 idx
= acpi_match_platform_list(plat_info
);
2970 switch (plat_info
[idx
].data
) {
2972 if (!intel_pstate_no_acpi_pss())
2975 return intel_pstate_no_acpi_pcch();
2977 return intel_pstate_has_acpi_ppc() && !force_load
;
2983 static void intel_pstate_request_control_from_smm(void)
2986 * It may be unsafe to request P-states control from SMM if _PPC support
2987 * has not been enabled.
2990 acpi_processor_pstate_control();
2992 #else /* CONFIG_ACPI not enabled */
2993 static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; }
2994 static inline bool intel_pstate_has_acpi_ppc(void) { return false; }
2995 static inline void intel_pstate_request_control_from_smm(void) {}
2996 #endif /* CONFIG_ACPI */
2998 #define INTEL_PSTATE_HWP_BROADWELL 0x01
3000 #define X86_MATCH_HWP(model, hwp_mode) \
3001 X86_MATCH_VENDOR_FAM_MODEL_FEATURE(INTEL, 6, INTEL_FAM6_##model, \
3002 X86_FEATURE_HWP, hwp_mode)
3004 static const struct x86_cpu_id hwp_support_ids
[] __initconst
= {
3005 X86_MATCH_HWP(BROADWELL_X
, INTEL_PSTATE_HWP_BROADWELL
),
3006 X86_MATCH_HWP(BROADWELL_D
, INTEL_PSTATE_HWP_BROADWELL
),
3007 X86_MATCH_HWP(ANY
, 0),
3011 static int __init
intel_pstate_init(void)
3013 const struct x86_cpu_id
*id
;
3016 if (boot_cpu_data
.x86_vendor
!= X86_VENDOR_INTEL
)
3022 id
= x86_match_cpu(hwp_support_ids
);
3024 copy_cpu_funcs(&core_funcs
);
3026 * Avoid enabling HWP for processors without EPP support,
3027 * because that means incomplete HWP implementation which is a
3028 * corner case and supporting it is generally problematic.
3030 if (!no_hwp
&& boot_cpu_has(X86_FEATURE_HWP_EPP
)) {
3032 hwp_mode_bdw
= id
->driver_data
;
3033 intel_pstate
.attr
= hwp_cpufreq_attrs
;
3034 intel_cpufreq
.attr
= hwp_cpufreq_attrs
;
3035 intel_cpufreq
.flags
|= CPUFREQ_NEED_UPDATE_LIMITS
;
3036 if (!default_driver
)
3037 default_driver
= &intel_pstate
;
3039 goto hwp_cpu_matched
;
3042 id
= x86_match_cpu(intel_pstate_cpu_ids
);
3044 pr_info("CPU model not supported\n");
3048 copy_cpu_funcs((struct pstate_funcs
*)id
->driver_data
);
3051 if (intel_pstate_msrs_not_valid()) {
3052 pr_info("Invalid MSRs\n");
3055 /* Without HWP start in the passive mode. */
3056 if (!default_driver
)
3057 default_driver
= &intel_cpufreq
;
3061 * The Intel pstate driver will be ignored if the platform
3062 * firmware has its own power management modes.
3064 if (intel_pstate_platform_pwr_mgmt_exists()) {
3065 pr_info("P-states controlled by the platform\n");
3069 if (!hwp_active
&& hwp_only
)
3072 pr_info("Intel P-state driver initializing\n");
3074 all_cpu_data
= vzalloc(array_size(sizeof(void *), num_possible_cpus()));
3078 intel_pstate_request_control_from_smm();
3080 intel_pstate_sysfs_expose_params();
3082 mutex_lock(&intel_pstate_driver_lock
);
3083 rc
= intel_pstate_register_driver(default_driver
);
3084 mutex_unlock(&intel_pstate_driver_lock
);
3086 intel_pstate_sysfs_remove();
3091 const struct x86_cpu_id
*id
;
3093 id
= x86_match_cpu(intel_pstate_cpu_ee_disable_ids
);
3095 set_power_ctl_ee_state(false);
3096 pr_info("Disabling energy efficiency optimization\n");
3099 pr_info("HWP enabled\n");
3104 device_initcall(intel_pstate_init
);
3106 static int __init
intel_pstate_setup(char *str
)
3111 if (!strcmp(str
, "disable"))
3113 else if (!strcmp(str
, "active"))
3114 default_driver
= &intel_pstate
;
3115 else if (!strcmp(str
, "passive"))
3116 default_driver
= &intel_cpufreq
;
3118 if (!strcmp(str
, "no_hwp")) {
3119 pr_info("HWP disabled\n");
3122 if (!strcmp(str
, "force"))
3124 if (!strcmp(str
, "hwp_only"))
3126 if (!strcmp(str
, "per_cpu_perf_limits"))
3127 per_cpu_limits
= true;
3130 if (!strcmp(str
, "support_acpi_ppc"))
3136 early_param("intel_pstate", intel_pstate_setup
);
3138 MODULE_AUTHOR("Dirk Brandewie <dirk.j.brandewie@intel.com>");
3139 MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors");
3140 MODULE_LICENSE("GPL");