]>
Commit | Line | Data |
---|---|---|
b886d83c | 1 | // SPDX-License-Identifier: GPL-2.0-only |
5477fb3b AC |
2 | /* |
3 | * CPPC (Collaborative Processor Performance Control) driver for | |
4 | * interfacing with the CPUfreq layer and governors. See | |
5 | * cppc_acpi.c for CPPC specific methods. | |
6 | * | |
7 | * (C) Copyright 2014, 2015 Linaro Ltd. | |
8 | * Author: Ashwin Chaugule <ashwin.chaugule@linaro.org> | |
5477fb3b AC |
9 | */ |
10 | ||
11 | #define pr_fmt(fmt) "CPPC Cpufreq:" fmt | |
12 | ||
1eb5dde6 | 13 | #include <linux/arch_topology.h> |
5477fb3b AC |
14 | #include <linux/kernel.h> |
15 | #include <linux/module.h> | |
16 | #include <linux/delay.h> | |
17 | #include <linux/cpu.h> | |
18 | #include <linux/cpufreq.h> | |
ad38677d | 19 | #include <linux/dmi.h> |
1eb5dde6 VK |
20 | #include <linux/irq_work.h> |
21 | #include <linux/kthread.h> | |
3d41386d | 22 | #include <linux/time.h> |
5477fb3b | 23 | #include <linux/vmalloc.h> |
1eb5dde6 | 24 | #include <uapi/linux/sched/types.h> |
5477fb3b | 25 | |
ad38677d AS |
26 | #include <asm/unaligned.h> |
27 | ||
5477fb3b AC |
28 | #include <acpi/cppc_acpi.h> |
29 | ||
ad38677d AS |
30 | /* Minimum struct length needed for the DMI processor entry we want */ |
31 | #define DMI_ENTRY_PROCESSOR_MIN_LENGTH 48 | |
32 | ||
63087265 IV |
33 | /* Offset in the DMI processor structure for the max frequency */ |
34 | #define DMI_PROCESSOR_MAX_SPEED 0x14 | |
ad38677d | 35 | |
5477fb3b | 36 | /* |
a28b2bfc IV |
37 | * This list contains information parsed from per CPU ACPI _CPC and _PSD |
38 | * structures: e.g. the highest and lowest supported performance, capabilities, | |
39 | * desired performance, level requested etc. Depending on the share_type, not | |
40 | * all CPUs will have an entry in the list. | |
5477fb3b | 41 | */ |
a28b2bfc IV |
42 | static LIST_HEAD(cpu_data_list); |
43 | ||
54e74df5 | 44 | static bool boost_supported; |
5477fb3b | 45 | |
6c8d750f | 46 | struct cppc_workaround_oem_info { |
c7402379 | 47 | char oem_id[ACPI_OEM_ID_SIZE + 1]; |
6c8d750f XW |
48 | char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1]; |
49 | u32 oem_revision; | |
50 | }; | |
51 | ||
6c8d750f XW |
52 | static struct cppc_workaround_oem_info wa_info[] = { |
53 | { | |
54 | .oem_id = "HISI ", | |
55 | .oem_table_id = "HIP07 ", | |
56 | .oem_revision = 0, | |
57 | }, { | |
58 | .oem_id = "HISI ", | |
59 | .oem_table_id = "HIP08 ", | |
60 | .oem_revision = 0, | |
61 | } | |
62 | }; | |
63 | ||
1eb5dde6 VK |
64 | #ifdef CONFIG_ACPI_CPPC_CPUFREQ_FIE |
65 | ||
66 | /* Frequency invariance support */ | |
67 | struct cppc_freq_invariance { | |
68 | int cpu; | |
69 | struct irq_work irq_work; | |
70 | struct kthread_work work; | |
71 | struct cppc_perf_fb_ctrs prev_perf_fb_ctrs; | |
72 | struct cppc_cpudata *cpu_data; | |
73 | }; | |
74 | ||
75 | static DEFINE_PER_CPU(struct cppc_freq_invariance, cppc_freq_inv); | |
76 | static struct kthread_worker *kworker_fie; | |
77 | ||
78 | static struct cpufreq_driver cppc_cpufreq_driver; | |
79 | static unsigned int hisi_cppc_cpufreq_get_rate(unsigned int cpu); | |
80 | static int cppc_perf_from_fbctrs(struct cppc_cpudata *cpu_data, | |
81 | struct cppc_perf_fb_ctrs *fb_ctrs_t0, | |
82 | struct cppc_perf_fb_ctrs *fb_ctrs_t1); | |
83 | ||
84 | /** | |
85 | * cppc_scale_freq_workfn - CPPC arch_freq_scale updater for frequency invariance | |
86 | * @work: The work item. | |
87 | * | |
88 | * The CPPC driver register itself with the topology core to provide its own | |
89 | * implementation (cppc_scale_freq_tick()) of topology_scale_freq_tick() which | |
90 | * gets called by the scheduler on every tick. | |
91 | * | |
92 | * Note that the arch specific counters have higher priority than CPPC counters, | |
93 | * if available, though the CPPC driver doesn't need to have any special | |
94 | * handling for that. | |
95 | * | |
96 | * On an invocation of cppc_scale_freq_tick(), we schedule an irq work (since we | |
97 | * reach here from hard-irq context), which then schedules a normal work item | |
98 | * and cppc_scale_freq_workfn() updates the per_cpu arch_freq_scale variable | |
99 | * based on the counter updates since the last tick. | |
100 | */ | |
101 | static void cppc_scale_freq_workfn(struct kthread_work *work) | |
102 | { | |
103 | struct cppc_freq_invariance *cppc_fi; | |
104 | struct cppc_perf_fb_ctrs fb_ctrs = {0}; | |
105 | struct cppc_cpudata *cpu_data; | |
106 | unsigned long local_freq_scale; | |
107 | u64 perf; | |
108 | ||
109 | cppc_fi = container_of(work, struct cppc_freq_invariance, work); | |
110 | cpu_data = cppc_fi->cpu_data; | |
111 | ||
112 | if (cppc_get_perf_ctrs(cppc_fi->cpu, &fb_ctrs)) { | |
113 | pr_warn("%s: failed to read perf counters\n", __func__); | |
114 | return; | |
115 | } | |
116 | ||
117 | perf = cppc_perf_from_fbctrs(cpu_data, &cppc_fi->prev_perf_fb_ctrs, | |
118 | &fb_ctrs); | |
119 | cppc_fi->prev_perf_fb_ctrs = fb_ctrs; | |
120 | ||
121 | perf <<= SCHED_CAPACITY_SHIFT; | |
122 | local_freq_scale = div64_u64(perf, cpu_data->perf_caps.highest_perf); | |
123 | ||
124 | /* This can happen due to counter's overflow */ | |
125 | if (unlikely(local_freq_scale > 1024)) | |
126 | local_freq_scale = 1024; | |
127 | ||
128 | per_cpu(arch_freq_scale, cppc_fi->cpu) = local_freq_scale; | |
129 | } | |
130 | ||
131 | static void cppc_irq_work(struct irq_work *irq_work) | |
132 | { | |
133 | struct cppc_freq_invariance *cppc_fi; | |
134 | ||
135 | cppc_fi = container_of(irq_work, struct cppc_freq_invariance, irq_work); | |
136 | kthread_queue_work(kworker_fie, &cppc_fi->work); | |
137 | } | |
138 | ||
139 | static void cppc_scale_freq_tick(void) | |
140 | { | |
141 | struct cppc_freq_invariance *cppc_fi = &per_cpu(cppc_freq_inv, smp_processor_id()); | |
142 | ||
143 | /* | |
144 | * cppc_get_perf_ctrs() can potentially sleep, call that from the right | |
145 | * context. | |
146 | */ | |
147 | irq_work_queue(&cppc_fi->irq_work); | |
148 | } | |
149 | ||
150 | static struct scale_freq_data cppc_sftd = { | |
151 | .source = SCALE_FREQ_SOURCE_CPPC, | |
152 | .set_freq_scale = cppc_scale_freq_tick, | |
153 | }; | |
154 | ||
155 | static void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy) | |
156 | { | |
157 | struct cppc_freq_invariance *cppc_fi; | |
158 | int cpu, ret; | |
159 | ||
160 | if (cppc_cpufreq_driver.get == hisi_cppc_cpufreq_get_rate) | |
161 | return; | |
162 | ||
163 | for_each_cpu(cpu, policy->cpus) { | |
164 | cppc_fi = &per_cpu(cppc_freq_inv, cpu); | |
165 | cppc_fi->cpu = cpu; | |
166 | cppc_fi->cpu_data = policy->driver_data; | |
167 | kthread_init_work(&cppc_fi->work, cppc_scale_freq_workfn); | |
168 | init_irq_work(&cppc_fi->irq_work, cppc_irq_work); | |
169 | ||
170 | ret = cppc_get_perf_ctrs(cpu, &cppc_fi->prev_perf_fb_ctrs); | |
171 | if (ret) { | |
172 | pr_warn("%s: failed to read perf counters for cpu:%d: %d\n", | |
173 | __func__, cpu, ret); | |
174 | ||
175 | /* | |
176 | * Don't abort if the CPU was offline while the driver | |
177 | * was getting registered. | |
178 | */ | |
179 | if (cpu_online(cpu)) | |
180 | return; | |
181 | } | |
182 | } | |
183 | ||
184 | /* Register for freq-invariance */ | |
185 | topology_set_scale_freq_source(&cppc_sftd, policy->cpus); | |
186 | } | |
187 | ||
188 | /* | |
189 | * We free all the resources on policy's removal and not on CPU removal as the | |
190 | * irq-work are per-cpu and the hotplug core takes care of flushing the pending | |
191 | * irq-works (hint: smpcfd_dying_cpu()) on CPU hotplug. Even if the kthread-work | |
192 | * fires on another CPU after the concerned CPU is removed, it won't harm. | |
193 | * | |
194 | * We just need to make sure to remove them all on policy->exit(). | |
195 | */ | |
196 | static void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy) | |
197 | { | |
198 | struct cppc_freq_invariance *cppc_fi; | |
199 | int cpu; | |
200 | ||
201 | if (cppc_cpufreq_driver.get == hisi_cppc_cpufreq_get_rate) | |
202 | return; | |
203 | ||
204 | /* policy->cpus will be empty here, use related_cpus instead */ | |
205 | topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_CPPC, policy->related_cpus); | |
206 | ||
207 | for_each_cpu(cpu, policy->related_cpus) { | |
208 | cppc_fi = &per_cpu(cppc_freq_inv, cpu); | |
209 | irq_work_sync(&cppc_fi->irq_work); | |
210 | kthread_cancel_work_sync(&cppc_fi->work); | |
211 | } | |
212 | } | |
213 | ||
214 | static void __init cppc_freq_invariance_init(void) | |
215 | { | |
216 | struct sched_attr attr = { | |
217 | .size = sizeof(struct sched_attr), | |
218 | .sched_policy = SCHED_DEADLINE, | |
219 | .sched_nice = 0, | |
220 | .sched_priority = 0, | |
221 | /* | |
222 | * Fake (unused) bandwidth; workaround to "fix" | |
223 | * priority inheritance. | |
224 | */ | |
225 | .sched_runtime = 1000000, | |
226 | .sched_deadline = 10000000, | |
227 | .sched_period = 10000000, | |
228 | }; | |
229 | int ret; | |
230 | ||
231 | if (cppc_cpufreq_driver.get == hisi_cppc_cpufreq_get_rate) | |
232 | return; | |
233 | ||
234 | kworker_fie = kthread_create_worker(0, "cppc_fie"); | |
235 | if (IS_ERR(kworker_fie)) | |
236 | return; | |
237 | ||
238 | ret = sched_setattr_nocheck(kworker_fie->task, &attr); | |
239 | if (ret) { | |
240 | pr_warn("%s: failed to set SCHED_DEADLINE: %d\n", __func__, | |
241 | ret); | |
242 | kthread_destroy_worker(kworker_fie); | |
243 | return; | |
244 | } | |
245 | } | |
246 | ||
247 | static void cppc_freq_invariance_exit(void) | |
248 | { | |
249 | if (cppc_cpufreq_driver.get == hisi_cppc_cpufreq_get_rate) | |
250 | return; | |
251 | ||
252 | kthread_destroy_worker(kworker_fie); | |
253 | kworker_fie = NULL; | |
254 | } | |
255 | ||
256 | #else | |
257 | static inline void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy) | |
258 | { | |
259 | } | |
260 | ||
261 | static inline void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy) | |
262 | { | |
263 | } | |
264 | ||
265 | static inline void cppc_freq_invariance_init(void) | |
266 | { | |
267 | } | |
268 | ||
269 | static inline void cppc_freq_invariance_exit(void) | |
270 | { | |
271 | } | |
272 | #endif /* CONFIG_ACPI_CPPC_CPUFREQ_FIE */ | |
273 | ||
ad38677d AS |
274 | /* Callback function used to retrieve the max frequency from DMI */ |
275 | static void cppc_find_dmi_mhz(const struct dmi_header *dm, void *private) | |
276 | { | |
277 | const u8 *dmi_data = (const u8 *)dm; | |
278 | u16 *mhz = (u16 *)private; | |
279 | ||
280 | if (dm->type == DMI_ENTRY_PROCESSOR && | |
281 | dm->length >= DMI_ENTRY_PROCESSOR_MIN_LENGTH) { | |
282 | u16 val = (u16)get_unaligned((const u16 *) | |
283 | (dmi_data + DMI_PROCESSOR_MAX_SPEED)); | |
284 | *mhz = val > *mhz ? val : *mhz; | |
285 | } | |
286 | } | |
287 | ||
288 | /* Look up the max frequency in DMI */ | |
289 | static u64 cppc_get_dmi_max_khz(void) | |
290 | { | |
291 | u16 mhz = 0; | |
292 | ||
293 | dmi_walk(cppc_find_dmi_mhz, &mhz); | |
294 | ||
295 | /* | |
296 | * Real stupid fallback value, just in case there is no | |
297 | * actual value set. | |
298 | */ | |
299 | mhz = mhz ? mhz : 1; | |
300 | ||
301 | return (1000 * mhz); | |
302 | } | |
303 | ||
256f19d2 PP |
304 | /* |
305 | * If CPPC lowest_freq and nominal_freq registers are exposed then we can | |
8b8bde7f PG |
306 | * use them to convert perf to freq and vice versa. The conversion is |
307 | * extrapolated as an affine function passing by the 2 points: | |
308 | * - (Low perf, Low freq) | |
309 | * - (Nominal perf, Nominal perf) | |
256f19d2 | 310 | */ |
48ad8dc9 | 311 | static unsigned int cppc_cpufreq_perf_to_khz(struct cppc_cpudata *cpu_data, |
63087265 | 312 | unsigned int perf) |
256f19d2 | 313 | { |
48ad8dc9 | 314 | struct cppc_perf_caps *caps = &cpu_data->perf_caps; |
8b8bde7f | 315 | s64 retval, offset = 0; |
63087265 | 316 | static u64 max_khz; |
256f19d2 PP |
317 | u64 mul, div; |
318 | ||
319 | if (caps->lowest_freq && caps->nominal_freq) { | |
8b8bde7f PG |
320 | mul = caps->nominal_freq - caps->lowest_freq; |
321 | div = caps->nominal_perf - caps->lowest_perf; | |
322 | offset = caps->nominal_freq - div64_u64(caps->nominal_perf * mul, div); | |
256f19d2 PP |
323 | } else { |
324 | if (!max_khz) | |
325 | max_khz = cppc_get_dmi_max_khz(); | |
326 | mul = max_khz; | |
4264e02d | 327 | div = caps->highest_perf; |
256f19d2 | 328 | } |
8b8bde7f PG |
329 | |
330 | retval = offset + div64_u64(perf * mul, div); | |
331 | if (retval >= 0) | |
332 | return retval; | |
333 | return 0; | |
256f19d2 PP |
334 | } |
335 | ||
48ad8dc9 | 336 | static unsigned int cppc_cpufreq_khz_to_perf(struct cppc_cpudata *cpu_data, |
63087265 | 337 | unsigned int freq) |
256f19d2 | 338 | { |
48ad8dc9 | 339 | struct cppc_perf_caps *caps = &cpu_data->perf_caps; |
8b8bde7f | 340 | s64 retval, offset = 0; |
63087265 | 341 | static u64 max_khz; |
256f19d2 PP |
342 | u64 mul, div; |
343 | ||
344 | if (caps->lowest_freq && caps->nominal_freq) { | |
8b8bde7f PG |
345 | mul = caps->nominal_perf - caps->lowest_perf; |
346 | div = caps->nominal_freq - caps->lowest_freq; | |
347 | offset = caps->nominal_perf - div64_u64(caps->nominal_freq * mul, div); | |
256f19d2 PP |
348 | } else { |
349 | if (!max_khz) | |
350 | max_khz = cppc_get_dmi_max_khz(); | |
4264e02d | 351 | mul = caps->highest_perf; |
256f19d2 PP |
352 | div = max_khz; |
353 | } | |
354 | ||
8b8bde7f PG |
355 | retval = offset + div64_u64(freq * mul, div); |
356 | if (retval >= 0) | |
357 | return retval; | |
358 | return 0; | |
256f19d2 PP |
359 | } |
360 | ||
5477fb3b | 361 | static int cppc_cpufreq_set_target(struct cpufreq_policy *policy, |
63087265 IV |
362 | unsigned int target_freq, |
363 | unsigned int relation) | |
a28b2bfc | 364 | |
5477fb3b | 365 | { |
a28b2bfc | 366 | struct cppc_cpudata *cpu_data = policy->driver_data; |
d2641a5c | 367 | unsigned int cpu = policy->cpu; |
5477fb3b | 368 | struct cpufreq_freqs freqs; |
c197d758 | 369 | u32 desired_perf; |
5477fb3b AC |
370 | int ret = 0; |
371 | ||
48ad8dc9 | 372 | desired_perf = cppc_cpufreq_khz_to_perf(cpu_data, target_freq); |
c197d758 | 373 | /* Return if it is exactly the same perf */ |
48ad8dc9 | 374 | if (desired_perf == cpu_data->perf_ctrls.desired_perf) |
c197d758 HT |
375 | return ret; |
376 | ||
48ad8dc9 | 377 | cpu_data->perf_ctrls.desired_perf = desired_perf; |
5477fb3b AC |
378 | freqs.old = policy->cur; |
379 | freqs.new = target_freq; | |
380 | ||
381 | cpufreq_freq_transition_begin(policy, &freqs); | |
d2641a5c | 382 | ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); |
5477fb3b AC |
383 | cpufreq_freq_transition_end(policy, &freqs, ret != 0); |
384 | ||
385 | if (ret) | |
386 | pr_debug("Failed to set target on CPU:%d. ret:%d\n", | |
d2641a5c | 387 | cpu, ret); |
5477fb3b AC |
388 | |
389 | return ret; | |
390 | } | |
391 | ||
1e4f63ae | 392 | static int cppc_verify_policy(struct cpufreq_policy_data *policy) |
5477fb3b AC |
393 | { |
394 | cpufreq_verify_within_cpu_limits(policy); | |
395 | return 0; | |
396 | } | |
397 | ||
d4f3388a PP |
398 | /* |
399 | * The PCC subspace describes the rate at which platform can accept commands | |
400 | * on the shared PCC channel (including READs which do not count towards freq | |
63087265 | 401 | * transition requests), so ideally we need to use the PCC values as a fallback |
d4f3388a PP |
402 | * if we don't have a platform specific transition_delay_us |
403 | */ | |
404 | #ifdef CONFIG_ARM64 | |
405 | #include <asm/cputype.h> | |
406 | ||
48ad8dc9 | 407 | static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu) |
d4f3388a PP |
408 | { |
409 | unsigned long implementor = read_cpuid_implementor(); | |
410 | unsigned long part_num = read_cpuid_part_number(); | |
d4f3388a PP |
411 | |
412 | switch (implementor) { | |
413 | case ARM_CPU_IMP_QCOM: | |
414 | switch (part_num) { | |
415 | case QCOM_CPU_PART_FALKOR_V1: | |
416 | case QCOM_CPU_PART_FALKOR: | |
2b53d1bd | 417 | return 10000; |
d4f3388a | 418 | } |
d4f3388a | 419 | } |
2b53d1bd | 420 | return cppc_get_transition_latency(cpu) / NSEC_PER_USEC; |
d4f3388a PP |
421 | } |
422 | ||
423 | #else | |
424 | ||
48ad8dc9 | 425 | static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu) |
d4f3388a PP |
426 | { |
427 | return cppc_get_transition_latency(cpu) / NSEC_PER_USEC; | |
428 | } | |
429 | #endif | |
430 | ||
a28b2bfc IV |
431 | |
432 | static struct cppc_cpudata *cppc_cpufreq_get_cpu_data(unsigned int cpu) | |
5477fb3b | 433 | { |
a28b2bfc IV |
434 | struct cppc_cpudata *cpu_data; |
435 | int ret; | |
436 | ||
437 | cpu_data = kzalloc(sizeof(struct cppc_cpudata), GFP_KERNEL); | |
438 | if (!cpu_data) | |
439 | goto out; | |
5477fb3b | 440 | |
a28b2bfc IV |
441 | if (!zalloc_cpumask_var(&cpu_data->shared_cpu_map, GFP_KERNEL)) |
442 | goto free_cpu; | |
5477fb3b | 443 | |
a28b2bfc | 444 | ret = acpi_get_psd_map(cpu, cpu_data); |
5477fb3b | 445 | if (ret) { |
a28b2bfc IV |
446 | pr_debug("Err parsing CPU%d PSD data: ret:%d\n", cpu, ret); |
447 | goto free_mask; | |
448 | } | |
449 | ||
450 | ret = cppc_get_perf_caps(cpu, &cpu_data->perf_caps); | |
451 | if (ret) { | |
452 | pr_debug("Err reading CPU%d perf caps: ret:%d\n", cpu, ret); | |
453 | goto free_mask; | |
5477fb3b AC |
454 | } |
455 | ||
256f19d2 | 456 | /* Convert the lowest and nominal freq from MHz to KHz */ |
a28b2bfc IV |
457 | cpu_data->perf_caps.lowest_freq *= 1000; |
458 | cpu_data->perf_caps.nominal_freq *= 1000; | |
459 | ||
460 | list_add(&cpu_data->node, &cpu_data_list); | |
461 | ||
462 | return cpu_data; | |
463 | ||
464 | free_mask: | |
465 | free_cpumask_var(cpu_data->shared_cpu_map); | |
466 | free_cpu: | |
467 | kfree(cpu_data); | |
468 | out: | |
469 | return NULL; | |
470 | } | |
471 | ||
fe2535a4 VK |
472 | static void cppc_cpufreq_put_cpu_data(struct cpufreq_policy *policy) |
473 | { | |
474 | struct cppc_cpudata *cpu_data = policy->driver_data; | |
475 | ||
476 | list_del(&cpu_data->node); | |
477 | free_cpumask_var(cpu_data->shared_cpu_map); | |
478 | kfree(cpu_data); | |
479 | policy->driver_data = NULL; | |
480 | } | |
481 | ||
a28b2bfc IV |
482 | static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy) |
483 | { | |
484 | unsigned int cpu = policy->cpu; | |
485 | struct cppc_cpudata *cpu_data; | |
486 | struct cppc_perf_caps *caps; | |
487 | int ret; | |
488 | ||
489 | cpu_data = cppc_cpufreq_get_cpu_data(cpu); | |
490 | if (!cpu_data) { | |
491 | pr_err("Error in acquiring _CPC/_PSD data for CPU%d.\n", cpu); | |
492 | return -ENODEV; | |
493 | } | |
494 | caps = &cpu_data->perf_caps; | |
495 | policy->driver_data = cpu_data; | |
ad38677d | 496 | |
73808d0f PP |
497 | /* |
498 | * Set min to lowest nonlinear perf to avoid any efficiency penalty (see | |
499 | * Section 8.4.7.1.1.5 of ACPI 6.1 spec) | |
500 | */ | |
bb025fb6 IV |
501 | policy->min = cppc_cpufreq_perf_to_khz(cpu_data, |
502 | caps->lowest_nonlinear_perf); | |
503 | policy->max = cppc_cpufreq_perf_to_khz(cpu_data, | |
504 | caps->nominal_perf); | |
73808d0f PP |
505 | |
506 | /* | |
507 | * Set cpuinfo.min_freq to Lowest to make the full range of performance | |
508 | * available if userspace wants to use any perf between lowest & lowest | |
509 | * nonlinear perf | |
510 | */ | |
bb025fb6 IV |
511 | policy->cpuinfo.min_freq = cppc_cpufreq_perf_to_khz(cpu_data, |
512 | caps->lowest_perf); | |
513 | policy->cpuinfo.max_freq = cppc_cpufreq_perf_to_khz(cpu_data, | |
514 | caps->nominal_perf); | |
73808d0f | 515 | |
48ad8dc9 IV |
516 | policy->transition_delay_us = cppc_cpufreq_get_transition_delay_us(cpu); |
517 | policy->shared_type = cpu_data->shared_type; | |
5477fb3b | 518 | |
bf76bb20 IV |
519 | switch (policy->shared_type) { |
520 | case CPUFREQ_SHARED_TYPE_HW: | |
521 | case CPUFREQ_SHARED_TYPE_NONE: | |
522 | /* Nothing to be done - we'll have a policy for each CPU */ | |
523 | break; | |
524 | case CPUFREQ_SHARED_TYPE_ANY: | |
a28b2bfc IV |
525 | /* |
526 | * All CPUs in the domain will share a policy and all cpufreq | |
527 | * operations will use a single cppc_cpudata structure stored | |
528 | * in policy->driver_data. | |
529 | */ | |
48ad8dc9 | 530 | cpumask_copy(policy->cpus, cpu_data->shared_cpu_map); |
bf76bb20 IV |
531 | break; |
532 | default: | |
533 | pr_debug("Unsupported CPU co-ord type: %d\n", | |
534 | policy->shared_type); | |
fe2535a4 VK |
535 | ret = -EFAULT; |
536 | goto out; | |
5477fb3b AC |
537 | } |
538 | ||
54e74df5 XW |
539 | /* |
540 | * If 'highest_perf' is greater than 'nominal_perf', we assume CPU Boost | |
541 | * is supported. | |
542 | */ | |
bb025fb6 | 543 | if (caps->highest_perf > caps->nominal_perf) |
54e74df5 XW |
544 | boost_supported = true; |
545 | ||
5477fb3b | 546 | /* Set policy->cur to max now. The governors will adjust later. */ |
bb025fb6 IV |
547 | policy->cur = cppc_cpufreq_perf_to_khz(cpu_data, caps->highest_perf); |
548 | cpu_data->perf_ctrls.desired_perf = caps->highest_perf; | |
5477fb3b | 549 | |
48ad8dc9 | 550 | ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); |
fe2535a4 | 551 | if (ret) { |
5477fb3b | 552 | pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n", |
bb025fb6 | 553 | caps->highest_perf, cpu, ret); |
fe2535a4 VK |
554 | goto out; |
555 | } | |
556 | ||
1eb5dde6 | 557 | cppc_cpufreq_cpu_fie_init(policy); |
fe2535a4 | 558 | return 0; |
5477fb3b | 559 | |
fe2535a4 VK |
560 | out: |
561 | cppc_cpufreq_put_cpu_data(policy); | |
5477fb3b AC |
562 | return ret; |
563 | } | |
564 | ||
9357a380 VK |
565 | static int cppc_cpufreq_cpu_exit(struct cpufreq_policy *policy) |
566 | { | |
567 | struct cppc_cpudata *cpu_data = policy->driver_data; | |
568 | struct cppc_perf_caps *caps = &cpu_data->perf_caps; | |
569 | unsigned int cpu = policy->cpu; | |
570 | int ret; | |
571 | ||
1eb5dde6 VK |
572 | cppc_cpufreq_cpu_fie_exit(policy); |
573 | ||
9357a380 VK |
574 | cpu_data->perf_ctrls.desired_perf = caps->lowest_perf; |
575 | ||
576 | ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); | |
577 | if (ret) | |
578 | pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n", | |
579 | caps->lowest_perf, cpu, ret); | |
580 | ||
fe2535a4 | 581 | cppc_cpufreq_put_cpu_data(policy); |
9357a380 VK |
582 | return 0; |
583 | } | |
584 | ||
33477d84 GC |
585 | static inline u64 get_delta(u64 t1, u64 t0) |
586 | { | |
587 | if (t1 > t0 || t0 > ~(u32)0) | |
588 | return t1 - t0; | |
589 | ||
590 | return (u32)t1 - (u32)t0; | |
591 | } | |
592 | ||
1eb5dde6 VK |
593 | static int cppc_perf_from_fbctrs(struct cppc_cpudata *cpu_data, |
594 | struct cppc_perf_fb_ctrs *fb_ctrs_t0, | |
595 | struct cppc_perf_fb_ctrs *fb_ctrs_t1) | |
33477d84 GC |
596 | { |
597 | u64 delta_reference, delta_delivered; | |
1eb5dde6 | 598 | u64 reference_perf; |
33477d84 | 599 | |
eead1840 | 600 | reference_perf = fb_ctrs_t0->reference_perf; |
33477d84 | 601 | |
eead1840 VK |
602 | delta_reference = get_delta(fb_ctrs_t1->reference, |
603 | fb_ctrs_t0->reference); | |
604 | delta_delivered = get_delta(fb_ctrs_t1->delivered, | |
605 | fb_ctrs_t0->delivered); | |
33477d84 | 606 | |
1eb5dde6 VK |
607 | /* Check to avoid divide-by zero and invalid delivered_perf */ |
608 | if (!delta_reference || !delta_delivered) | |
609 | return cpu_data->perf_ctrls.desired_perf; | |
33477d84 | 610 | |
1eb5dde6 | 611 | return (reference_perf * delta_delivered) / delta_reference; |
33477d84 GC |
612 | } |
613 | ||
48ad8dc9 | 614 | static unsigned int cppc_cpufreq_get_rate(unsigned int cpu) |
33477d84 GC |
615 | { |
616 | struct cppc_perf_fb_ctrs fb_ctrs_t0 = {0}, fb_ctrs_t1 = {0}; | |
a28b2bfc IV |
617 | struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); |
618 | struct cppc_cpudata *cpu_data = policy->driver_data; | |
1eb5dde6 | 619 | u64 delivered_perf; |
33477d84 GC |
620 | int ret; |
621 | ||
a28b2bfc IV |
622 | cpufreq_cpu_put(policy); |
623 | ||
48ad8dc9 | 624 | ret = cppc_get_perf_ctrs(cpu, &fb_ctrs_t0); |
33477d84 GC |
625 | if (ret) |
626 | return ret; | |
627 | ||
628 | udelay(2); /* 2usec delay between sampling */ | |
629 | ||
48ad8dc9 | 630 | ret = cppc_get_perf_ctrs(cpu, &fb_ctrs_t1); |
33477d84 GC |
631 | if (ret) |
632 | return ret; | |
633 | ||
1eb5dde6 VK |
634 | delivered_perf = cppc_perf_from_fbctrs(cpu_data, &fb_ctrs_t0, |
635 | &fb_ctrs_t1); | |
636 | ||
637 | return cppc_cpufreq_perf_to_khz(cpu_data, delivered_perf); | |
33477d84 GC |
638 | } |
639 | ||
54e74df5 XW |
640 | static int cppc_cpufreq_set_boost(struct cpufreq_policy *policy, int state) |
641 | { | |
a28b2bfc | 642 | struct cppc_cpudata *cpu_data = policy->driver_data; |
bb025fb6 | 643 | struct cppc_perf_caps *caps = &cpu_data->perf_caps; |
54e74df5 XW |
644 | int ret; |
645 | ||
646 | if (!boost_supported) { | |
647 | pr_err("BOOST not supported by CPU or firmware\n"); | |
648 | return -EINVAL; | |
649 | } | |
650 | ||
54e74df5 | 651 | if (state) |
48ad8dc9 | 652 | policy->max = cppc_cpufreq_perf_to_khz(cpu_data, |
bb025fb6 | 653 | caps->highest_perf); |
54e74df5 | 654 | else |
48ad8dc9 | 655 | policy->max = cppc_cpufreq_perf_to_khz(cpu_data, |
bb025fb6 | 656 | caps->nominal_perf); |
54e74df5 XW |
657 | policy->cpuinfo.max_freq = policy->max; |
658 | ||
659 | ret = freq_qos_update_request(policy->max_freq_req, policy->max); | |
660 | if (ret < 0) | |
661 | return ret; | |
662 | ||
663 | return 0; | |
664 | } | |
665 | ||
cfdc589f IV |
666 | static ssize_t show_freqdomain_cpus(struct cpufreq_policy *policy, char *buf) |
667 | { | |
a28b2bfc | 668 | struct cppc_cpudata *cpu_data = policy->driver_data; |
cfdc589f | 669 | |
a28b2bfc | 670 | return cpufreq_show_cpus(cpu_data->shared_cpu_map, buf); |
cfdc589f IV |
671 | } |
672 | cpufreq_freq_attr_ro(freqdomain_cpus); | |
673 | ||
674 | static struct freq_attr *cppc_cpufreq_attr[] = { | |
675 | &freqdomain_cpus, | |
676 | NULL, | |
677 | }; | |
678 | ||
5477fb3b AC |
679 | static struct cpufreq_driver cppc_cpufreq_driver = { |
680 | .flags = CPUFREQ_CONST_LOOPS, | |
681 | .verify = cppc_verify_policy, | |
682 | .target = cppc_cpufreq_set_target, | |
33477d84 | 683 | .get = cppc_cpufreq_get_rate, |
5477fb3b | 684 | .init = cppc_cpufreq_cpu_init, |
9357a380 | 685 | .exit = cppc_cpufreq_cpu_exit, |
54e74df5 | 686 | .set_boost = cppc_cpufreq_set_boost, |
cfdc589f | 687 | .attr = cppc_cpufreq_attr, |
5477fb3b AC |
688 | .name = "cppc_cpufreq", |
689 | }; | |
690 | ||
d88b0f0e VK |
691 | /* |
692 | * HISI platform does not support delivered performance counter and | |
693 | * reference performance counter. It can calculate the performance using the | |
694 | * platform specific mechanism. We reuse the desired performance register to | |
695 | * store the real performance calculated by the platform. | |
696 | */ | |
48ad8dc9 | 697 | static unsigned int hisi_cppc_cpufreq_get_rate(unsigned int cpu) |
d88b0f0e | 698 | { |
a28b2bfc IV |
699 | struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); |
700 | struct cppc_cpudata *cpu_data = policy->driver_data; | |
d88b0f0e VK |
701 | u64 desired_perf; |
702 | int ret; | |
703 | ||
a28b2bfc IV |
704 | cpufreq_cpu_put(policy); |
705 | ||
48ad8dc9 | 706 | ret = cppc_get_desired_perf(cpu, &desired_perf); |
d88b0f0e VK |
707 | if (ret < 0) |
708 | return -EIO; | |
709 | ||
48ad8dc9 | 710 | return cppc_cpufreq_perf_to_khz(cpu_data, desired_perf); |
d88b0f0e VK |
711 | } |
712 | ||
713 | static void cppc_check_hisi_workaround(void) | |
714 | { | |
715 | struct acpi_table_header *tbl; | |
716 | acpi_status status = AE_OK; | |
717 | int i; | |
718 | ||
719 | status = acpi_get_table(ACPI_SIG_PCCT, 0, &tbl); | |
720 | if (ACPI_FAILURE(status) || !tbl) | |
721 | return; | |
722 | ||
723 | for (i = 0; i < ARRAY_SIZE(wa_info); i++) { | |
724 | if (!memcmp(wa_info[i].oem_id, tbl->oem_id, ACPI_OEM_ID_SIZE) && | |
725 | !memcmp(wa_info[i].oem_table_id, tbl->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) && | |
726 | wa_info[i].oem_revision == tbl->oem_revision) { | |
727 | /* Overwrite the get() callback */ | |
728 | cppc_cpufreq_driver.get = hisi_cppc_cpufreq_get_rate; | |
729 | break; | |
730 | } | |
731 | } | |
732 | ||
733 | acpi_put_table(tbl); | |
734 | } | |
735 | ||
5477fb3b AC |
736 | static int __init cppc_cpufreq_init(void) |
737 | { | |
1eb5dde6 VK |
738 | int ret; |
739 | ||
a28b2bfc | 740 | if ((acpi_disabled) || !acpi_cpc_valid()) |
5477fb3b AC |
741 | return -ENODEV; |
742 | ||
a28b2bfc | 743 | INIT_LIST_HEAD(&cpu_data_list); |
5477fb3b | 744 | |
6c8d750f | 745 | cppc_check_hisi_workaround(); |
1eb5dde6 | 746 | cppc_freq_invariance_init(); |
6c8d750f | 747 | |
1eb5dde6 VK |
748 | ret = cpufreq_register_driver(&cppc_cpufreq_driver); |
749 | if (ret) | |
750 | cppc_freq_invariance_exit(); | |
751 | ||
752 | return ret; | |
a28b2bfc | 753 | } |
5477fb3b | 754 | |
a28b2bfc IV |
755 | static inline void free_cpu_data(void) |
756 | { | |
757 | struct cppc_cpudata *iter, *tmp; | |
5477fb3b | 758 | |
a28b2bfc IV |
759 | list_for_each_entry_safe(iter, tmp, &cpu_data_list, node) { |
760 | free_cpumask_var(iter->shared_cpu_map); | |
761 | list_del(&iter->node); | |
762 | kfree(iter); | |
55b55abc | 763 | } |
5477fb3b | 764 | |
5477fb3b AC |
765 | } |
766 | ||
a29a1e76 AC |
767 | static void __exit cppc_cpufreq_exit(void) |
768 | { | |
a29a1e76 | 769 | cpufreq_unregister_driver(&cppc_cpufreq_driver); |
1eb5dde6 | 770 | cppc_freq_invariance_exit(); |
a29a1e76 | 771 | |
a28b2bfc | 772 | free_cpu_data(); |
a29a1e76 AC |
773 | } |
774 | ||
775 | module_exit(cppc_cpufreq_exit); | |
776 | MODULE_AUTHOR("Ashwin Chaugule"); | |
777 | MODULE_DESCRIPTION("CPUFreq driver based on the ACPI CPPC v5.0+ spec"); | |
778 | MODULE_LICENSE("GPL"); | |
779 | ||
5477fb3b | 780 | late_initcall(cppc_cpufreq_init); |
974f8649 | 781 | |
8ff3c226 | 782 | static const struct acpi_device_id cppc_acpi_ids[] __used = { |
974f8649 PP |
783 | {ACPI_PROCESSOR_DEVICE_HID, }, |
784 | {} | |
785 | }; | |
786 | ||
787 | MODULE_DEVICE_TABLE(acpi, cppc_acpi_ids); |