]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - drivers/powercap/intel_rapl.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
powercap/rapl: add package reference per domain
[mirror_ubuntu-bionic-kernel.git] / drivers / powercap / intel_rapl.c
CommitLineData
2d281d81
JP		 1/*
2 * Intel Running Average Power Limit (RAPL) Driver
3 * Copyright (c) 2013, Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc.
16 *
17 */
18#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
19
20#include <linux/kernel.h>
21#include <linux/module.h>
22#include <linux/list.h>
23#include <linux/types.h>
24#include <linux/device.h>
25#include <linux/slab.h>
26#include <linux/log2.h>
27#include <linux/bitmap.h>
28#include <linux/delay.h>
29#include <linux/sysfs.h>
30#include <linux/cpu.h>
31#include <linux/powercap.h>
3c2c0845 32#include <asm/iosf_mbi.h>
2d281d81
JP		 33
34#include <asm/processor.h>
35#include <asm/cpu_device_id.h>
36
37/* bitmasks for RAPL MSRs, used by primitive access functions */
38#define ENERGY_STATUS_MASK 0xffffffff
39
40#define POWER_LIMIT1_MASK 0x7FFF
41#define POWER_LIMIT1_ENABLE BIT(15)
42#define POWER_LIMIT1_CLAMP BIT(16)
43
44#define POWER_LIMIT2_MASK (0x7FFFULL<<32)
45#define POWER_LIMIT2_ENABLE BIT_ULL(47)
46#define POWER_LIMIT2_CLAMP BIT_ULL(48)
47#define POWER_PACKAGE_LOCK BIT_ULL(63)
48#define POWER_PP_LOCK BIT(31)
49
50#define TIME_WINDOW1_MASK (0x7FULL<<17)
51#define TIME_WINDOW2_MASK (0x7FULL<<49)
52
53#define POWER_UNIT_OFFSET 0
54#define POWER_UNIT_MASK 0x0F
55
56#define ENERGY_UNIT_OFFSET 0x08
57#define ENERGY_UNIT_MASK 0x1F00
58
59#define TIME_UNIT_OFFSET 0x10
60#define TIME_UNIT_MASK 0xF0000
61
62#define POWER_INFO_MAX_MASK (0x7fffULL<<32)
63#define POWER_INFO_MIN_MASK (0x7fffULL<<16)
64#define POWER_INFO_MAX_TIME_WIN_MASK (0x3fULL<<48)
65#define POWER_INFO_THERMAL_SPEC_MASK 0x7fff
66
67#define PERF_STATUS_THROTTLE_TIME_MASK 0xffffffff
68#define PP_POLICY_MASK 0x1F
69
70/* Non HW constants */
71#define RAPL_PRIMITIVE_DERIVED BIT(1) /* not from raw data */
72#define RAPL_PRIMITIVE_DUMMY BIT(2)
73
2d281d81
JP		 74#define TIME_WINDOW_MAX_MSEC 40000
75#define TIME_WINDOW_MIN_MSEC 250
d474a4d3 76#define ENERGY_UNIT_SCALE 1000 /* scale from driver unit to powercap unit */
2d281d81
JP		 77enum unit_type {
78 ARBITRARY_UNIT, /* no translation */
79 POWER_UNIT,
80 ENERGY_UNIT,
81 TIME_UNIT,
82};
83
84enum rapl_domain_type {
85 RAPL_DOMAIN_PACKAGE, /* entire package/socket */
86 RAPL_DOMAIN_PP0, /* core power plane */
87 RAPL_DOMAIN_PP1, /* graphics uncore */
88 RAPL_DOMAIN_DRAM,/* DRAM control_type */
89 RAPL_DOMAIN_MAX,
90};
91
92enum rapl_domain_msr_id {
93 RAPL_DOMAIN_MSR_LIMIT,
94 RAPL_DOMAIN_MSR_STATUS,
95 RAPL_DOMAIN_MSR_PERF,
96 RAPL_DOMAIN_MSR_POLICY,
97 RAPL_DOMAIN_MSR_INFO,
98 RAPL_DOMAIN_MSR_MAX,
99};
100
101/* per domain data, some are optional */
102enum rapl_primitives {
103 ENERGY_COUNTER,
104 POWER_LIMIT1,
105 POWER_LIMIT2,
106 FW_LOCK,
107
108 PL1_ENABLE, /* power limit 1, aka long term */
109 PL1_CLAMP, /* allow frequency to go below OS request */
110 PL2_ENABLE, /* power limit 2, aka short term, instantaneous */
111 PL2_CLAMP,
112
113 TIME_WINDOW1, /* long term */
114 TIME_WINDOW2, /* short term */
115 THERMAL_SPEC_POWER,
116 MAX_POWER,
117
118 MIN_POWER,
119 MAX_TIME_WINDOW,
120 THROTTLED_TIME,
121 PRIORITY_LEVEL,
122
123 /* below are not raw primitive data */
124 AVERAGE_POWER,
125 NR_RAPL_PRIMITIVES,
126};
127
128#define NR_RAW_PRIMITIVES (NR_RAPL_PRIMITIVES - 2)
129
130/* Can be expanded to include events, etc.*/
131struct rapl_domain_data {
132 u64 primitives[NR_RAPL_PRIMITIVES];
133 unsigned long timestamp;
134};
135
f14a1396
JP		 136struct msrl_action {
137 u32 msr_no;
138 u64 clear_mask;
139 u64 set_mask;
140 int err;
141};
2d281d81
JP		 142
143#define DOMAIN_STATE_INACTIVE BIT(0)
144#define DOMAIN_STATE_POWER_LIMIT_SET BIT(1)
145#define DOMAIN_STATE_BIOS_LOCKED BIT(2)
146
147#define NR_POWER_LIMITS (2)
148struct rapl_power_limit {
149 struct powercap_zone_constraint *constraint;
150 int prim_id; /* primitive ID used to enable */
151 struct rapl_domain *domain;
152 const char *name;
153};
154
155static const char pl1_name[] = "long_term";
156static const char pl2_name[] = "short_term";
157
309557f5 158struct rapl_package;
2d281d81
JP		 159struct rapl_domain {
160 const char *name;
161 enum rapl_domain_type id;
162 int msrs[RAPL_DOMAIN_MSR_MAX];
163 struct powercap_zone power_zone;
164 struct rapl_domain_data rdd;
165 struct rapl_power_limit rpl[NR_POWER_LIMITS];
166 u64 attr_map; /* track capabilities */
167 unsigned int state;
d474a4d3 168 unsigned int domain_energy_unit;
309557f5 169 struct rapl_package *rp;
2d281d81
JP		 170};
171#define power_zone_to_rapl_domain(_zone) \
172 container_of(_zone, struct rapl_domain, power_zone)
173
174
175/* Each physical package contains multiple domains, these are the common
176 * data across RAPL domains within a package.
177 */
178struct rapl_package {
179 unsigned int id; /* physical package/socket id */
180 unsigned int nr_domains;
181 unsigned long domain_map; /* bit map of active domains */
3c2c0845
JP		 182 unsigned int power_unit;
183 unsigned int energy_unit;
184 unsigned int time_unit;
2d281d81
JP		 185 struct rapl_domain *domains; /* array of domains, sized at runtime */
186 struct powercap_zone *power_zone; /* keep track of parent zone */
187 int nr_cpus; /* active cpus on the package, topology info is lost during
188 * cpu hotplug. so we have to track ourselves.
189 */
190 unsigned long power_limit_irq; /* keep track of package power limit
191 * notify interrupt enable status.
192 */
193 struct list_head plist;
194};
087e9cba
JP		 195
196struct rapl_defaults {
51b63409 197 u8 floor_freq_reg_addr;
087e9cba
JP		 198 int (*check_unit)(struct rapl_package *rp, int cpu);
199 void (*set_floor_freq)(struct rapl_domain *rd, bool mode);
200 u64 (*compute_time_window)(struct rapl_package *rp, u64 val,
201 bool to_raw);
d474a4d3 202 unsigned int dram_domain_energy_unit;
087e9cba
JP		 203};
204static struct rapl_defaults *rapl_defaults;
205
3c2c0845 206/* Sideband MBI registers */
51b63409
AT		 207#define IOSF_CPU_POWER_BUDGET_CTL_BYT (0x2)
208#define IOSF_CPU_POWER_BUDGET_CTL_TNG (0xdf)
3c2c0845 209
2d281d81
JP		 210#define PACKAGE_PLN_INT_SAVED BIT(0)
211#define MAX_PRIM_NAME (32)
212
213/* per domain data. used to describe individual knobs such that access function
214 * can be consolidated into one instead of many inline functions.
215 */
216struct rapl_primitive_info {
217 const char *name;
218 u64 mask;
219 int shift;
220 enum rapl_domain_msr_id id;
221 enum unit_type unit;
222 u32 flag;
223};
224
225#define PRIMITIVE_INFO_INIT(p, m, s, i, u, f) { \
226 .name = #p, \
227 .mask = m, \
228 .shift = s, \
229 .id = i, \
230 .unit = u, \
231 .flag = f \
232 }
233
234static void rapl_init_domains(struct rapl_package *rp);
235static int rapl_read_data_raw(struct rapl_domain *rd,
236 enum rapl_primitives prim,
237 bool xlate, u64 *data);
238static int rapl_write_data_raw(struct rapl_domain *rd,
239 enum rapl_primitives prim,
240 unsigned long long value);
309557f5 241static u64 rapl_unit_xlate(struct rapl_domain *rd,
d474a4d3 242 enum unit_type type, u64 value,
2d281d81 243 int to_raw);
309557f5 244static void package_power_limit_irq_save(struct rapl_package *rp);
2d281d81
JP		 245
246static LIST_HEAD(rapl_packages); /* guarded by CPU hotplug lock */
247
248static const char * const rapl_domain_names[] = {
249 "package",
250 "core",
251 "uncore",
252 "dram",
253};
254
255static struct powercap_control_type *control_type; /* PowerCap Controller */
256
257/* caller to ensure CPU hotplug lock is held */
258static struct rapl_package *find_package_by_id(int id)
259{
260 struct rapl_package *rp;
261
262 list_for_each_entry(rp, &rapl_packages, plist) {
263 if (rp->id == id)
264 return rp;
265 }
266
267 return NULL;
268}
269
270/* caller to ensure CPU hotplug lock is held */
271static int find_active_cpu_on_package(int package_id)
272{
273 int i;
274
275 for_each_online_cpu(i) {
276 if (topology_physical_package_id(i) == package_id)
277 return i;
278 }
279 /* all CPUs on this package are offline */
280
281 return -ENODEV;
282}
283
284/* caller must hold cpu hotplug lock */
285static void rapl_cleanup_data(void)
286{
287 struct rapl_package *p, *tmp;
288
289 list_for_each_entry_safe(p, tmp, &rapl_packages, plist) {
290 kfree(p->domains);
291 list_del(&p->plist);
292 kfree(p);
293 }
294}
295
296static int get_energy_counter(struct powercap_zone *power_zone, u64 *energy_raw)
297{
298 struct rapl_domain *rd;
299 u64 energy_now;
300
301 /* prevent CPU hotplug, make sure the RAPL domain does not go
302 * away while reading the counter.
303 */
304 get_online_cpus();
305 rd = power_zone_to_rapl_domain(power_zone);
306
307 if (!rapl_read_data_raw(rd, ENERGY_COUNTER, true, &energy_now)) {
308 *energy_raw = energy_now;
309 put_online_cpus();
310
311 return 0;
312 }
313 put_online_cpus();
314
315 return -EIO;
316}
317
318static int get_max_energy_counter(struct powercap_zone *pcd_dev, u64 *energy)
319{
d474a4d3
JP		 320 struct rapl_domain *rd = power_zone_to_rapl_domain(pcd_dev);
321
309557f5 322 *energy = rapl_unit_xlate(rd, ENERGY_UNIT, ENERGY_STATUS_MASK, 0);
2d281d81
JP		 323 return 0;
324}
325
326static int release_zone(struct powercap_zone *power_zone)
327{
328 struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone);
309557f5 329 struct rapl_package *rp = rd->rp;
2d281d81
JP		 330
331 /* package zone is the last zone of a package, we can free
332 * memory here since all children has been unregistered.
333 */
334 if (rd->id == RAPL_DOMAIN_PACKAGE) {
2d281d81
JP		 335 kfree(rd);
336 rp->domains = NULL;
337 }
338
339 return 0;
340
341}
342
343static int find_nr_power_limit(struct rapl_domain *rd)
344{
345 int i;
346
347 for (i = 0; i < NR_POWER_LIMITS; i++) {
348 if (rd->rpl[i].name == NULL)
349 break;
350 }
351
352 return i;
353}
354
355static int set_domain_enable(struct powercap_zone *power_zone, bool mode)
356{
357 struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone);
2d281d81
JP		 358
359 if (rd->state & DOMAIN_STATE_BIOS_LOCKED)
360 return -EACCES;
3c2c0845 361
2d281d81 362 get_online_cpus();
2d281d81 363 rapl_write_data_raw(rd, PL1_ENABLE, mode);
51b63409
AT		 364 if (rapl_defaults->set_floor_freq)
365 rapl_defaults->set_floor_freq(rd, mode);
2d281d81
JP		 366 put_online_cpus();
367
368 return 0;
369}
370
371static int get_domain_enable(struct powercap_zone *power_zone, bool *mode)
372{
373 struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone);
374 u64 val;
375
376 if (rd->state & DOMAIN_STATE_BIOS_LOCKED) {
377 *mode = false;
378 return 0;
379 }
380 get_online_cpus();
381 if (rapl_read_data_raw(rd, PL1_ENABLE, true, &val)) {
382 put_online_cpus();
383 return -EIO;
384 }
385 *mode = val;
386 put_online_cpus();
387
388 return 0;
389}
390
391/* per RAPL domain ops, in the order of rapl_domain_type */
600c395b 392static const struct powercap_zone_ops zone_ops[] = {
2d281d81
JP		 393 /* RAPL_DOMAIN_PACKAGE */
394 {
395 .get_energy_uj = get_energy_counter,
396 .get_max_energy_range_uj = get_max_energy_counter,
397 .release = release_zone,
398 .set_enable = set_domain_enable,
399 .get_enable = get_domain_enable,
400 },
401 /* RAPL_DOMAIN_PP0 */
402 {
403 .get_energy_uj = get_energy_counter,
404 .get_max_energy_range_uj = get_max_energy_counter,
405 .release = release_zone,
406 .set_enable = set_domain_enable,
407 .get_enable = get_domain_enable,
408 },
409 /* RAPL_DOMAIN_PP1 */
410 {
411 .get_energy_uj = get_energy_counter,
412 .get_max_energy_range_uj = get_max_energy_counter,
413 .release = release_zone,
414 .set_enable = set_domain_enable,
415 .get_enable = get_domain_enable,
416 },
417 /* RAPL_DOMAIN_DRAM */
418 {
419 .get_energy_uj = get_energy_counter,
420 .get_max_energy_range_uj = get_max_energy_counter,
421 .release = release_zone,
422 .set_enable = set_domain_enable,
423 .get_enable = get_domain_enable,
424 },
425};
426
427static int set_power_limit(struct powercap_zone *power_zone, int id,
428 u64 power_limit)
429{
430 struct rapl_domain *rd;
431 struct rapl_package *rp;
432 int ret = 0;
433
434 get_online_cpus();
435 rd = power_zone_to_rapl_domain(power_zone);
309557f5 436 rp = rd->rp;
2d281d81
JP		 437
438 if (rd->state & DOMAIN_STATE_BIOS_LOCKED) {
439 dev_warn(&power_zone->dev, "%s locked by BIOS, monitoring only\n",
440 rd->name);
441 ret = -EACCES;
442 goto set_exit;
443 }
444
445 switch (rd->rpl[id].prim_id) {
446 case PL1_ENABLE:
447 rapl_write_data_raw(rd, POWER_LIMIT1, power_limit);
448 break;
449 case PL2_ENABLE:
450 rapl_write_data_raw(rd, POWER_LIMIT2, power_limit);
451 break;
452 default:
453 ret = -EINVAL;
454 }
455 if (!ret)
309557f5 456 package_power_limit_irq_save(rp);
2d281d81
JP		 457set_exit:
458 put_online_cpus();
459 return ret;
460}
461
462static int get_current_power_limit(struct powercap_zone *power_zone, int id,
463 u64 *data)
464{
465 struct rapl_domain *rd;
466 u64 val;
467 int prim;
468 int ret = 0;
469
470 get_online_cpus();
471 rd = power_zone_to_rapl_domain(power_zone);
472 switch (rd->rpl[id].prim_id) {
473 case PL1_ENABLE:
474 prim = POWER_LIMIT1;
475 break;
476 case PL2_ENABLE:
477 prim = POWER_LIMIT2;
478 break;
479 default:
480 put_online_cpus();
481 return -EINVAL;
482 }
483 if (rapl_read_data_raw(rd, prim, true, &val))
484 ret = -EIO;
485 else
486 *data = val;
487
488 put_online_cpus();
489
490 return ret;
491}
492
493static int set_time_window(struct powercap_zone *power_zone, int id,
494 u64 window)
495{
496 struct rapl_domain *rd;
497 int ret = 0;
498
499 get_online_cpus();
500 rd = power_zone_to_rapl_domain(power_zone);
501 switch (rd->rpl[id].prim_id) {
502 case PL1_ENABLE:
503 rapl_write_data_raw(rd, TIME_WINDOW1, window);
504 break;
505 case PL2_ENABLE:
506 rapl_write_data_raw(rd, TIME_WINDOW2, window);
507 break;
508 default:
509 ret = -EINVAL;
510 }
511 put_online_cpus();
512 return ret;
513}
514
515static int get_time_window(struct powercap_zone *power_zone, int id, u64 *data)
516{
517 struct rapl_domain *rd;
518 u64 val;
519 int ret = 0;
520
521 get_online_cpus();
522 rd = power_zone_to_rapl_domain(power_zone);
523 switch (rd->rpl[id].prim_id) {
524 case PL1_ENABLE:
525 ret = rapl_read_data_raw(rd, TIME_WINDOW1, true, &val);
526 break;
527 case PL2_ENABLE:
528 ret = rapl_read_data_raw(rd, TIME_WINDOW2, true, &val);
529 break;
530 default:
531 put_online_cpus();
532 return -EINVAL;
533 }
534 if (!ret)
535 *data = val;
536 put_online_cpus();
537
538 return ret;
539}
540
541static const char *get_constraint_name(struct powercap_zone *power_zone, int id)
542{
543 struct rapl_power_limit *rpl;
544 struct rapl_domain *rd;
545
546 rd = power_zone_to_rapl_domain(power_zone);
547 rpl = (struct rapl_power_limit *) &rd->rpl[id];
548
549 return rpl->name;
550}
551
552
553static int get_max_power(struct powercap_zone *power_zone, int id,
554 u64 *data)
555{
556 struct rapl_domain *rd;
557 u64 val;
558 int prim;
559 int ret = 0;
560
561 get_online_cpus();
562 rd = power_zone_to_rapl_domain(power_zone);
563 switch (rd->rpl[id].prim_id) {
564 case PL1_ENABLE:
565 prim = THERMAL_SPEC_POWER;
566 break;
567 case PL2_ENABLE:
568 prim = MAX_POWER;
569 break;
570 default:
571 put_online_cpus();
572 return -EINVAL;
573 }
574 if (rapl_read_data_raw(rd, prim, true, &val))
575 ret = -EIO;
576 else
577 *data = val;
578
579 put_online_cpus();
580
581 return ret;
582}
583
600c395b 584static const struct powercap_zone_constraint_ops constraint_ops = {
2d281d81
JP		 585 .set_power_limit_uw = set_power_limit,
586 .get_power_limit_uw = get_current_power_limit,
587 .set_time_window_us = set_time_window,
588 .get_time_window_us = get_time_window,
589 .get_max_power_uw = get_max_power,
590 .get_name = get_constraint_name,
591};
592
593/* called after domain detection and package level data are set */
594static void rapl_init_domains(struct rapl_package *rp)
595{
596 int i;
597 struct rapl_domain *rd = rp->domains;
598
599 for (i = 0; i < RAPL_DOMAIN_MAX; i++) {
600 unsigned int mask = rp->domain_map & (1 << i);
601 switch (mask) {
602 case BIT(RAPL_DOMAIN_PACKAGE):
603 rd->name = rapl_domain_names[RAPL_DOMAIN_PACKAGE];
604 rd->id = RAPL_DOMAIN_PACKAGE;
605 rd->msrs[0] = MSR_PKG_POWER_LIMIT;
606 rd->msrs[1] = MSR_PKG_ENERGY_STATUS;
607 rd->msrs[2] = MSR_PKG_PERF_STATUS;
608 rd->msrs[3] = 0;
609 rd->msrs[4] = MSR_PKG_POWER_INFO;
610 rd->rpl[0].prim_id = PL1_ENABLE;
611 rd->rpl[0].name = pl1_name;
612 rd->rpl[1].prim_id = PL2_ENABLE;
613 rd->rpl[1].name = pl2_name;
614 break;
615 case BIT(RAPL_DOMAIN_PP0):
616 rd->name = rapl_domain_names[RAPL_DOMAIN_PP0];
617 rd->id = RAPL_DOMAIN_PP0;
618 rd->msrs[0] = MSR_PP0_POWER_LIMIT;
619 rd->msrs[1] = MSR_PP0_ENERGY_STATUS;
620 rd->msrs[2] = 0;
621 rd->msrs[3] = MSR_PP0_POLICY;
622 rd->msrs[4] = 0;
623 rd->rpl[0].prim_id = PL1_ENABLE;
624 rd->rpl[0].name = pl1_name;
625 break;
626 case BIT(RAPL_DOMAIN_PP1):
627 rd->name = rapl_domain_names[RAPL_DOMAIN_PP1];
628 rd->id = RAPL_DOMAIN_PP1;
629 rd->msrs[0] = MSR_PP1_POWER_LIMIT;
630 rd->msrs[1] = MSR_PP1_ENERGY_STATUS;
631 rd->msrs[2] = 0;
632 rd->msrs[3] = MSR_PP1_POLICY;
633 rd->msrs[4] = 0;
634 rd->rpl[0].prim_id = PL1_ENABLE;
635 rd->rpl[0].name = pl1_name;
636 break;
637 case BIT(RAPL_DOMAIN_DRAM):
638 rd->name = rapl_domain_names[RAPL_DOMAIN_DRAM];
639 rd->id = RAPL_DOMAIN_DRAM;
640 rd->msrs[0] = MSR_DRAM_POWER_LIMIT;
641 rd->msrs[1] = MSR_DRAM_ENERGY_STATUS;
642 rd->msrs[2] = MSR_DRAM_PERF_STATUS;
643 rd->msrs[3] = 0;
644 rd->msrs[4] = MSR_DRAM_POWER_INFO;
645 rd->rpl[0].prim_id = PL1_ENABLE;
646 rd->rpl[0].name = pl1_name;
d474a4d3
JP		 647 rd->domain_energy_unit =
648 rapl_defaults->dram_domain_energy_unit;
649 if (rd->domain_energy_unit)
650 pr_info("DRAM domain energy unit %dpj\n",
651 rd->domain_energy_unit);
2d281d81
JP		 652 break;
653 }
654 if (mask) {
309557f5 655 rd->rp = rp;
2d281d81
JP		 656 rd++;
657 }
658 }
659}
660
309557f5
JP		 661static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type,
662 u64 value, int to_raw)
2d281d81 663{
3c2c0845 664 u64 units = 1;
309557f5 665 struct rapl_package *rp = rd->rp;
d474a4d3 666 u64 scale = 1;
2d281d81 667
2d281d81
JP		 668 switch (type) {
669 case POWER_UNIT:
3c2c0845 670 units = rp->power_unit;
2d281d81
JP		 671 break;
672 case ENERGY_UNIT:
d474a4d3
JP		 673 scale = ENERGY_UNIT_SCALE;
674 /* per domain unit takes precedence */
675 if (rd && rd->domain_energy_unit)
676 units = rd->domain_energy_unit;
677 else
678 units = rp->energy_unit;
2d281d81
JP		 679 break;
680 case TIME_UNIT:
3c2c0845 681 return rapl_defaults->compute_time_window(rp, value, to_raw);
2d281d81
JP		 682 case ARBITRARY_UNIT:
683 default:
684 return value;
685 };
686
687 if (to_raw)
d474a4d3 688 return div64_u64(value, units) * scale;
3c2c0845
JP		 689
690 value *= units;
691
d474a4d3 692 return div64_u64(value, scale);
2d281d81
JP		 693}
694
695/* in the order of enum rapl_primitives */
696static struct rapl_primitive_info rpi[] = {
697 /* name, mask, shift, msr index, unit divisor */
698 PRIMITIVE_INFO_INIT(ENERGY_COUNTER, ENERGY_STATUS_MASK, 0,
699 RAPL_DOMAIN_MSR_STATUS, ENERGY_UNIT, 0),
700 PRIMITIVE_INFO_INIT(POWER_LIMIT1, POWER_LIMIT1_MASK, 0,
701 RAPL_DOMAIN_MSR_LIMIT, POWER_UNIT, 0),
702 PRIMITIVE_INFO_INIT(POWER_LIMIT2, POWER_LIMIT2_MASK, 32,
703 RAPL_DOMAIN_MSR_LIMIT, POWER_UNIT, 0),
704 PRIMITIVE_INFO_INIT(FW_LOCK, POWER_PP_LOCK, 31,
705 RAPL_DOMAIN_MSR_LIMIT, ARBITRARY_UNIT, 0),
706 PRIMITIVE_INFO_INIT(PL1_ENABLE, POWER_LIMIT1_ENABLE, 15,
707 RAPL_DOMAIN_MSR_LIMIT, ARBITRARY_UNIT, 0),
708 PRIMITIVE_INFO_INIT(PL1_CLAMP, POWER_LIMIT1_CLAMP, 16,
709 RAPL_DOMAIN_MSR_LIMIT, ARBITRARY_UNIT, 0),
710 PRIMITIVE_INFO_INIT(PL2_ENABLE, POWER_LIMIT2_ENABLE, 47,
711 RAPL_DOMAIN_MSR_LIMIT, ARBITRARY_UNIT, 0),
712 PRIMITIVE_INFO_INIT(PL2_CLAMP, POWER_LIMIT2_CLAMP, 48,
713 RAPL_DOMAIN_MSR_LIMIT, ARBITRARY_UNIT, 0),
714 PRIMITIVE_INFO_INIT(TIME_WINDOW1, TIME_WINDOW1_MASK, 17,
715 RAPL_DOMAIN_MSR_LIMIT, TIME_UNIT, 0),
716 PRIMITIVE_INFO_INIT(TIME_WINDOW2, TIME_WINDOW2_MASK, 49,
717 RAPL_DOMAIN_MSR_LIMIT, TIME_UNIT, 0),
718 PRIMITIVE_INFO_INIT(THERMAL_SPEC_POWER, POWER_INFO_THERMAL_SPEC_MASK,
719 0, RAPL_DOMAIN_MSR_INFO, POWER_UNIT, 0),
720 PRIMITIVE_INFO_INIT(MAX_POWER, POWER_INFO_MAX_MASK, 32,
721 RAPL_DOMAIN_MSR_INFO, POWER_UNIT, 0),
722 PRIMITIVE_INFO_INIT(MIN_POWER, POWER_INFO_MIN_MASK, 16,
723 RAPL_DOMAIN_MSR_INFO, POWER_UNIT, 0),
724 PRIMITIVE_INFO_INIT(MAX_TIME_WINDOW, POWER_INFO_MAX_TIME_WIN_MASK, 48,
725 RAPL_DOMAIN_MSR_INFO, TIME_UNIT, 0),
726 PRIMITIVE_INFO_INIT(THROTTLED_TIME, PERF_STATUS_THROTTLE_TIME_MASK, 0,
727 RAPL_DOMAIN_MSR_PERF, TIME_UNIT, 0),
728 PRIMITIVE_INFO_INIT(PRIORITY_LEVEL, PP_POLICY_MASK, 0,
729 RAPL_DOMAIN_MSR_POLICY, ARBITRARY_UNIT, 0),
730 /* non-hardware */
731 PRIMITIVE_INFO_INIT(AVERAGE_POWER, 0, 0, 0, POWER_UNIT,
732 RAPL_PRIMITIVE_DERIVED),
733 {NULL, 0, 0, 0},
734};
735
736/* Read primitive data based on its related struct rapl_primitive_info.
737 * if xlate flag is set, return translated data based on data units, i.e.
738 * time, energy, and power.
739 * RAPL MSRs are non-architectual and are laid out not consistently across
740 * domains. Here we use primitive info to allow writing consolidated access
741 * functions.
742 * For a given primitive, it is processed by MSR mask and shift. Unit conversion
743 * is pre-assigned based on RAPL unit MSRs read at init time.
744 * 63-------------------------- 31--------------------------- 0
745 * | xxxxx (mask) |
746 * | |<- shift ----------------|
747 * 63-------------------------- 31--------------------------- 0
748 */
749static int rapl_read_data_raw(struct rapl_domain *rd,
750 enum rapl_primitives prim,
751 bool xlate, u64 *data)
752{
753 u64 value, final;
754 u32 msr;
755 struct rapl_primitive_info *rp = &rpi[prim];
756 int cpu;
757
758 if (!rp->name || rp->flag & RAPL_PRIMITIVE_DUMMY)
759 return -EINVAL;
760
761 msr = rd->msrs[rp->id];
762 if (!msr)
763 return -EINVAL;
764 /* use physical package id to look up active cpus */
309557f5 765 cpu = find_active_cpu_on_package(rd->rp->id);
2d281d81
JP		 766 if (cpu < 0)
767 return cpu;
768
769 /* special-case package domain, which uses a different bit*/
770 if (prim == FW_LOCK && rd->id == RAPL_DOMAIN_PACKAGE) {
771 rp->mask = POWER_PACKAGE_LOCK;
772 rp->shift = 63;
773 }
774 /* non-hardware data are collected by the polling thread */
775 if (rp->flag & RAPL_PRIMITIVE_DERIVED) {
776 *data = rd->rdd.primitives[prim];
777 return 0;
778 }
779
780 if (rdmsrl_safe_on_cpu(cpu, msr, &value)) {
781 pr_debug("failed to read msr 0x%x on cpu %d\n", msr, cpu);
782 return -EIO;
783 }
784
785 final = value & rp->mask;
786 final = final >> rp->shift;
787 if (xlate)
309557f5 788 *data = rapl_unit_xlate(rd, rp->unit, final, 0);
2d281d81
JP		 789 else
790 *data = final;
791
792 return 0;
793}
794
f14a1396
JP		 795
796static int msrl_update_safe(u32 msr_no, u64 clear_mask, u64 set_mask)
797{
798 int err;
799 u64 val;
800
801 err = rdmsrl_safe(msr_no, &val);
802 if (err)
803 goto out;
804
805 val &= ~clear_mask;
806 val |= set_mask;
807
808 err = wrmsrl_safe(msr_no, val);
809
810out:
811 return err;
812}
813
814static void msrl_update_func(void *info)
815{
816 struct msrl_action *ma = info;
817
818 ma->err = msrl_update_safe(ma->msr_no, ma->clear_mask, ma->set_mask);
819}
820
2d281d81
JP		 821/* Similar use of primitive info in the read counterpart */
822static int rapl_write_data_raw(struct rapl_domain *rd,
823 enum rapl_primitives prim,
824 unsigned long long value)
825{
2d281d81
JP		 826 struct rapl_primitive_info *rp = &rpi[prim];
827 int cpu;
f14a1396
JP		 828 u64 bits;
829 struct msrl_action ma;
830 int ret;
2d281d81 831
309557f5 832 cpu = find_active_cpu_on_package(rd->rp->id);
2d281d81
JP		 833 if (cpu < 0)
834 return cpu;
2d281d81 835
309557f5 836 bits = rapl_unit_xlate(rd, rp->unit, value, 1);
f14a1396
JP		 837 bits |= bits << rp->shift;
838 memset(&ma, 0, sizeof(ma));
839
840 ma.msr_no = rd->msrs[rp->id];
841 ma.clear_mask = rp->mask;
842 ma.set_mask = bits;
843
844 ret = smp_call_function_single(cpu, msrl_update_func, &ma, 1);
845 if (ret)
846 WARN_ON_ONCE(ret);
847 else
848 ret = ma.err;
849
850 return ret;
2d281d81
JP		 851}
852
3c2c0845
JP		 853/*
854 * Raw RAPL data stored in MSRs are in certain scales. We need to
855 * convert them into standard units based on the units reported in
856 * the RAPL unit MSRs. This is specific to CPUs as the method to
857 * calculate units differ on different CPUs.
858 * We convert the units to below format based on CPUs.
859 * i.e.
d474a4d3 860 * energy unit: picoJoules : Represented in picoJoules by default
3c2c0845
JP		 861 * power unit : microWatts : Represented in milliWatts by default
862 * time unit : microseconds: Represented in seconds by default
863 */
864static int rapl_check_unit_core(struct rapl_package *rp, int cpu)
2d281d81
JP		 865{
866 u64 msr_val;
867 u32 value;
868
869 if (rdmsrl_safe_on_cpu(cpu, MSR_RAPL_POWER_UNIT, &msr_val)) {
870 pr_err("Failed to read power unit MSR 0x%x on CPU %d, exit.\n",
871 MSR_RAPL_POWER_UNIT, cpu);
872 return -ENODEV;
873 }
874
2d281d81 875 value = (msr_val & ENERGY_UNIT_MASK) >> ENERGY_UNIT_OFFSET;
d474a4d3 876 rp->energy_unit = ENERGY_UNIT_SCALE * 1000000 / (1 << value);
2d281d81
JP		 877
878 value = (msr_val & POWER_UNIT_MASK) >> POWER_UNIT_OFFSET;
3c2c0845 879 rp->power_unit = 1000000 / (1 << value);
2d281d81
JP		 880
881 value = (msr_val & TIME_UNIT_MASK) >> TIME_UNIT_OFFSET;
3c2c0845 882 rp->time_unit = 1000000 / (1 << value);
2d281d81 883
d474a4d3 884 pr_debug("Core CPU package %d energy=%dpJ, time=%dus, power=%duW\n",
3c2c0845 885 rp->id, rp->energy_unit, rp->time_unit, rp->power_unit);
2d281d81
JP		 886
887 return 0;
888}
889
3c2c0845
JP		 890static int rapl_check_unit_atom(struct rapl_package *rp, int cpu)
891{
892 u64 msr_val;
893 u32 value;
894
895 if (rdmsrl_safe_on_cpu(cpu, MSR_RAPL_POWER_UNIT, &msr_val)) {
896 pr_err("Failed to read power unit MSR 0x%x on CPU %d, exit.\n",
897 MSR_RAPL_POWER_UNIT, cpu);
898 return -ENODEV;
899 }
900 value = (msr_val & ENERGY_UNIT_MASK) >> ENERGY_UNIT_OFFSET;
d474a4d3 901 rp->energy_unit = ENERGY_UNIT_SCALE * 1 << value;
3c2c0845
JP		 902
903 value = (msr_val & POWER_UNIT_MASK) >> POWER_UNIT_OFFSET;
904 rp->power_unit = (1 << value) * 1000;
905
906 value = (msr_val & TIME_UNIT_MASK) >> TIME_UNIT_OFFSET;
907 rp->time_unit = 1000000 / (1 << value);
908
d474a4d3 909 pr_debug("Atom package %d energy=%dpJ, time=%dus, power=%duW\n",
3c2c0845
JP		 910 rp->id, rp->energy_unit, rp->time_unit, rp->power_unit);
911
912 return 0;
913}
914
f14a1396
JP		 915static void power_limit_irq_save_cpu(void *info)
916{
917 u32 l, h = 0;
918 struct rapl_package *rp = (struct rapl_package *)info;
919
920 /* save the state of PLN irq mask bit before disabling it */
921 rdmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, &l, &h);
922 if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED)) {
923 rp->power_limit_irq = l & PACKAGE_THERM_INT_PLN_ENABLE;
924 rp->power_limit_irq |= PACKAGE_PLN_INT_SAVED;
925 }
926 l &= ~PACKAGE_THERM_INT_PLN_ENABLE;
927 wrmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
928}
929
3c2c0845 930
2d281d81
JP		 931/* REVISIT:
932 * When package power limit is set artificially low by RAPL, LVT
933 * thermal interrupt for package power limit should be ignored
934 * since we are not really exceeding the real limit. The intention
935 * is to avoid excessive interrupts while we are trying to save power.
936 * A useful feature might be routing the package_power_limit interrupt
937 * to userspace via eventfd. once we have a usecase, this is simple
938 * to do by adding an atomic notifier.
939 */
940
309557f5 941static void package_power_limit_irq_save(struct rapl_package *rp)
2d281d81 942{
2d281d81 943 int cpu;
2d281d81
JP		 944
945 if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN))
946 return;
947
309557f5 948 cpu = find_active_cpu_on_package(rp->id);
2d281d81
JP		 949 if (cpu < 0)
950 return;
f14a1396
JP		 951 if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN))
952 return;
953
954 smp_call_function_single(cpu, power_limit_irq_save_cpu, rp, 1);
955}
956
957static void power_limit_irq_restore_cpu(void *info)
958{
959 u32 l, h = 0;
960 struct rapl_package *rp = (struct rapl_package *)info;
961
962 rdmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, &l, &h);
963
964 if (rp->power_limit_irq & PACKAGE_THERM_INT_PLN_ENABLE)
965 l |= PACKAGE_THERM_INT_PLN_ENABLE;
966 else
967 l &= ~PACKAGE_THERM_INT_PLN_ENABLE;
968
969 wrmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
2d281d81
JP		 970}
971
972/* restore per package power limit interrupt enable state */
309557f5 973static void package_power_limit_irq_restore(struct rapl_package *rp)
2d281d81 974{
2d281d81 975 int cpu;
2d281d81
JP		 976
977 if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN))
978 return;
979
309557f5 980 cpu = find_active_cpu_on_package(rp->id);
2d281d81
JP		 981 if (cpu < 0)
982 return;
983
984 /* irq enable state not saved, nothing to restore */
985 if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED))
986 return;
2d281d81 987
f14a1396 988 smp_call_function_single(cpu, power_limit_irq_restore_cpu, rp, 1);
2d281d81
JP		 989}
990
3c2c0845
JP		 991static void set_floor_freq_default(struct rapl_domain *rd, bool mode)
992{
993 int nr_powerlimit = find_nr_power_limit(rd);
994
995 /* always enable clamp such that p-state can go below OS requested
996 * range. power capping priority over guranteed frequency.
997 */
998 rapl_write_data_raw(rd, PL1_CLAMP, mode);
999
1000 /* some domains have pl2 */
1001 if (nr_powerlimit > 1) {
1002 rapl_write_data_raw(rd, PL2_ENABLE, mode);
1003 rapl_write_data_raw(rd, PL2_CLAMP, mode);
1004 }
1005}
1006
1007static void set_floor_freq_atom(struct rapl_domain *rd, bool enable)
1008{
1009 static u32 power_ctrl_orig_val;
1010 u32 mdata;
1011
51b63409
AT		 1012 if (!rapl_defaults->floor_freq_reg_addr) {
1013 pr_err("Invalid floor frequency config register\n");
1014 return;
1015 }
1016
3c2c0845 1017 if (!power_ctrl_orig_val)
4077a387
AS		 1018 iosf_mbi_read(BT_MBI_UNIT_PMC, MBI_CR_READ,
1019 rapl_defaults->floor_freq_reg_addr,
1020 &power_ctrl_orig_val);
3c2c0845
JP		 1021 mdata = power_ctrl_orig_val;
1022 if (enable) {
1023 mdata &= ~(0x7f << 8);
1024 mdata |= 1 << 8;
1025 }
4077a387
AS		 1026 iosf_mbi_write(BT_MBI_UNIT_PMC, MBI_CR_WRITE,
1027 rapl_defaults->floor_freq_reg_addr, mdata);
3c2c0845
JP		 1028}
1029
1030static u64 rapl_compute_time_window_core(struct rapl_package *rp, u64 value,
1031 bool to_raw)
1032{
1033 u64 f, y; /* fraction and exp. used for time unit */
1034
1035 /*
1036 * Special processing based on 2^Y*(1+F/4), refer
1037 * to Intel Software Developer's manual Vol.3B: CH 14.9.3.
1038 */
1039 if (!to_raw) {
1040 f = (value & 0x60) >> 5;
1041 y = value & 0x1f;
1042 value = (1 << y) * (4 + f) * rp->time_unit / 4;
1043 } else {
1044 do_div(value, rp->time_unit);
1045 y = ilog2(value);
1046 f = div64_u64(4 * (value - (1 << y)), 1 << y);
1047 value = (y & 0x1f) | ((f & 0x3) << 5);
1048 }
1049 return value;
1050}
1051
1052static u64 rapl_compute_time_window_atom(struct rapl_package *rp, u64 value,
1053 bool to_raw)
1054{
1055 /*
1056 * Atom time unit encoding is straight forward val * time_unit,
1057 * where time_unit is default to 1 sec. Never 0.
1058 */
1059 if (!to_raw)
1060 return (value) ? value *= rp->time_unit : rp->time_unit;
1061 else
1062 value = div64_u64(value, rp->time_unit);
1063
1064 return value;
1065}
1066
087e9cba 1067static const struct rapl_defaults rapl_defaults_core = {
51b63409 1068 .floor_freq_reg_addr = 0,
3c2c0845
JP		 1069 .check_unit = rapl_check_unit_core,
1070 .set_floor_freq = set_floor_freq_default,
1071 .compute_time_window = rapl_compute_time_window_core,
087e9cba
JP		 1072};
1073
d474a4d3
JP		 1074static const struct rapl_defaults rapl_defaults_hsw_server = {
1075 .check_unit = rapl_check_unit_core,
1076 .set_floor_freq = set_floor_freq_default,
1077 .compute_time_window = rapl_compute_time_window_core,
1078 .dram_domain_energy_unit = 15300,
1079};
1080
51b63409
AT		 1081static const struct rapl_defaults rapl_defaults_byt = {
1082 .floor_freq_reg_addr = IOSF_CPU_POWER_BUDGET_CTL_BYT,
1083 .check_unit = rapl_check_unit_atom,
1084 .set_floor_freq = set_floor_freq_atom,
1085 .compute_time_window = rapl_compute_time_window_atom,
1086};
1087
1088static const struct rapl_defaults rapl_defaults_tng = {
1089 .floor_freq_reg_addr = IOSF_CPU_POWER_BUDGET_CTL_TNG,
3c2c0845
JP		 1090 .check_unit = rapl_check_unit_atom,
1091 .set_floor_freq = set_floor_freq_atom,
1092 .compute_time_window = rapl_compute_time_window_atom,
087e9cba
JP		 1093};
1094
51b63409
AT		 1095static const struct rapl_defaults rapl_defaults_ann = {
1096 .floor_freq_reg_addr = 0,
1097 .check_unit = rapl_check_unit_atom,
1098 .set_floor_freq = NULL,
1099 .compute_time_window = rapl_compute_time_window_atom,
1100};
1101
1102static const struct rapl_defaults rapl_defaults_cht = {
1103 .floor_freq_reg_addr = 0,
1104 .check_unit = rapl_check_unit_atom,
1105 .set_floor_freq = NULL,
1106 .compute_time_window = rapl_compute_time_window_atom,
1107};
1108
087e9cba
JP		 1109#define RAPL_CPU(_model, _ops) { \
1110 .vendor = X86_VENDOR_INTEL, \
1111 .family = 6, \
1112 .model = _model, \
1113 .driver_data = (kernel_ulong_t)&_ops, \
1114 }
1115
ea85dbca 1116static const struct x86_cpu_id rapl_ids[] __initconst = {
087e9cba
JP		 1117 RAPL_CPU(0x2a, rapl_defaults_core),/* Sandy Bridge */
1118 RAPL_CPU(0x2d, rapl_defaults_core),/* Sandy Bridge EP */
51b63409 1119 RAPL_CPU(0x37, rapl_defaults_byt),/* Valleyview */
087e9cba
JP		 1120 RAPL_CPU(0x3a, rapl_defaults_core),/* Ivy Bridge */
1121 RAPL_CPU(0x3c, rapl_defaults_core),/* Haswell */
1122 RAPL_CPU(0x3d, rapl_defaults_core),/* Broadwell */
d474a4d3 1123 RAPL_CPU(0x3f, rapl_defaults_hsw_server),/* Haswell servers */
34dfa36c 1124 RAPL_CPU(0x4f, rapl_defaults_hsw_server),/* Broadwell servers */
087e9cba 1125 RAPL_CPU(0x45, rapl_defaults_core),/* Haswell ULT */
4e0bec9e 1126 RAPL_CPU(0x47, rapl_defaults_core),/* Broadwell-H */
5fa0fa4b 1127 RAPL_CPU(0x4E, rapl_defaults_core),/* Skylake */
51b63409
AT		 1128 RAPL_CPU(0x4C, rapl_defaults_cht),/* Braswell/Cherryview */
1129 RAPL_CPU(0x4A, rapl_defaults_tng),/* Tangier */
d72be771 1130 RAPL_CPU(0x56, rapl_defaults_core),/* Future Xeon */
51b63409 1131 RAPL_CPU(0x5A, rapl_defaults_ann),/* Annidale */
89e7b255 1132 RAPL_CPU(0X5C, rapl_defaults_core),/* Broxton */
2cac1f70 1133 RAPL_CPU(0x5E, rapl_defaults_core),/* Skylake-H/S */
6f066d4d 1134 RAPL_CPU(0x57, rapl_defaults_hsw_server),/* Knights Landing */
2d281d81
JP		 1135 {}
1136};
1137MODULE_DEVICE_TABLE(x86cpu, rapl_ids);
1138
1139/* read once for all raw primitive data for all packages, domains */
1140static void rapl_update_domain_data(void)
1141{
1142 int dmn, prim;
1143 u64 val;
1144 struct rapl_package *rp;
1145
1146 list_for_each_entry(rp, &rapl_packages, plist) {
1147 for (dmn = 0; dmn < rp->nr_domains; dmn++) {
1148 pr_debug("update package %d domain %s data\n", rp->id,
1149 rp->domains[dmn].name);
1150 /* exclude non-raw primitives */
1151 for (prim = 0; prim < NR_RAW_PRIMITIVES; prim++)
1152 if (!rapl_read_data_raw(&rp->domains[dmn], prim,
1153 rpi[prim].unit,
1154 &val))
1155 rp->domains[dmn].rdd.primitives[prim] =
1156 val;
1157 }
1158 }
1159
1160}
1161
1162static int rapl_unregister_powercap(void)
1163{
1164 struct rapl_package *rp;
1165 struct rapl_domain *rd, *rd_package = NULL;
1166
1167 /* unregister all active rapl packages from the powercap layer,
1168 * hotplug lock held
1169 */
1170 list_for_each_entry(rp, &rapl_packages, plist) {
309557f5 1171 package_power_limit_irq_restore(rp);
2d281d81
JP		 1172
1173 for (rd = rp->domains; rd < rp->domains + rp->nr_domains;
1174 rd++) {
1175 pr_debug("remove package, undo power limit on %d: %s\n",
1176 rp->id, rd->name);
1177 rapl_write_data_raw(rd, PL1_ENABLE, 0);
2d281d81 1178 rapl_write_data_raw(rd, PL1_CLAMP, 0);
5021282c
SI		 1179 if (find_nr_power_limit(rd) > 1) {
1180 rapl_write_data_raw(rd, PL2_ENABLE, 0);
1181 rapl_write_data_raw(rd, PL2_CLAMP, 0);
1182 }
2d281d81
JP		 1183 if (rd->id == RAPL_DOMAIN_PACKAGE) {
1184 rd_package = rd;
1185 continue;
1186 }
1187 powercap_unregister_zone(control_type, &rd->power_zone);
1188 }
1189 /* do the package zone last */
1190 if (rd_package)
1191 powercap_unregister_zone(control_type,
1192 &rd_package->power_zone);
1193 }
1194 powercap_unregister_control_type(control_type);
1195
1196 return 0;
1197}
1198
1199static int rapl_package_register_powercap(struct rapl_package *rp)
1200{
1201 struct rapl_domain *rd;
1202 int ret = 0;
1203 char dev_name[17]; /* max domain name = 7 + 1 + 8 for int + 1 for null*/
1204 struct powercap_zone *power_zone = NULL;
1205 int nr_pl;
1206
1207 /* first we register package domain as the parent zone*/
1208 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
1209 if (rd->id == RAPL_DOMAIN_PACKAGE) {
1210 nr_pl = find_nr_power_limit(rd);
1211 pr_debug("register socket %d package domain %s\n",
1212 rp->id, rd->name);
1213 memset(dev_name, 0, sizeof(dev_name));
1214 snprintf(dev_name, sizeof(dev_name), "%s-%d",
1215 rd->name, rp->id);
1216 power_zone = powercap_register_zone(&rd->power_zone,
1217 control_type,
1218 dev_name, NULL,
1219 &zone_ops[rd->id],
1220 nr_pl,
1221 &constraint_ops);
1222 if (IS_ERR(power_zone)) {
1223 pr_debug("failed to register package, %d\n",
1224 rp->id);
1225 ret = PTR_ERR(power_zone);
1226 goto exit_package;
1227 }
1228 /* track parent zone in per package/socket data */
1229 rp->power_zone = power_zone;
1230 /* done, only one package domain per socket */
1231 break;
1232 }
1233 }
1234 if (!power_zone) {
1235 pr_err("no package domain found, unknown topology!\n");
1236 ret = -ENODEV;
1237 goto exit_package;
1238 }
1239 /* now register domains as children of the socket/package*/
1240 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
1241 if (rd->id == RAPL_DOMAIN_PACKAGE)
1242 continue;
1243 /* number of power limits per domain varies */
1244 nr_pl = find_nr_power_limit(rd);
1245 power_zone = powercap_register_zone(&rd->power_zone,
1246 control_type, rd->name,
1247 rp->power_zone,
1248 &zone_ops[rd->id], nr_pl,
1249 &constraint_ops);
1250
1251 if (IS_ERR(power_zone)) {
1252 pr_debug("failed to register power_zone, %d:%s:%s\n",
1253 rp->id, rd->name, dev_name);
1254 ret = PTR_ERR(power_zone);
1255 goto err_cleanup;
1256 }
1257 }
1258
1259exit_package:
1260 return ret;
1261err_cleanup:
1262 /* clean up previously initialized domains within the package if we
1263 * failed after the first domain setup.
1264 */
1265 while (--rd >= rp->domains) {
1266 pr_debug("unregister package %d domain %s\n", rp->id, rd->name);
1267 powercap_unregister_zone(control_type, &rd->power_zone);
1268 }
1269
1270 return ret;
1271}
1272
1273static int rapl_register_powercap(void)
1274{
1275 struct rapl_domain *rd;
1276 struct rapl_package *rp;
1277 int ret = 0;
1278
1279 control_type = powercap_register_control_type(NULL, "intel-rapl", NULL);
1280 if (IS_ERR(control_type)) {
1281 pr_debug("failed to register powercap control_type.\n");
1282 return PTR_ERR(control_type);
1283 }
1284 /* read the initial data */
1285 rapl_update_domain_data();
1286 list_for_each_entry(rp, &rapl_packages, plist)
1287 if (rapl_package_register_powercap(rp))
1288 goto err_cleanup_package;
1289 return ret;
1290
1291err_cleanup_package:
1292 /* clean up previously initialized packages */
1293 list_for_each_entry_continue_reverse(rp, &rapl_packages, plist) {
1294 for (rd = rp->domains; rd < rp->domains + rp->nr_domains;
1295 rd++) {
1296 pr_debug("unregister zone/package %d, %s domain\n",
1297 rp->id, rd->name);
1298 powercap_unregister_zone(control_type, &rd->power_zone);
1299 }
1300 }
1301
1302 return ret;
1303}
1304
1305static int rapl_check_domain(int cpu, int domain)
1306{
1307 unsigned msr;
9d31c676 1308 u64 val = 0;
2d281d81
JP		 1309
1310 switch (domain) {
1311 case RAPL_DOMAIN_PACKAGE:
1312 msr = MSR_PKG_ENERGY_STATUS;
1313 break;
1314 case RAPL_DOMAIN_PP0:
1315 msr = MSR_PP0_ENERGY_STATUS;
1316 break;
1317 case RAPL_DOMAIN_PP1:
1318 msr = MSR_PP1_ENERGY_STATUS;
1319 break;
1320 case RAPL_DOMAIN_DRAM:
1321 msr = MSR_DRAM_ENERGY_STATUS;
1322 break;
1323 default:
1324 pr_err("invalid domain id %d\n", domain);
1325 return -EINVAL;
1326 }
9d31c676
JP		 1327 /* make sure domain counters are available and contains non-zero
1328 * values, otherwise skip it.
7b874772 1329 */
9d31c676
JP		 1330 if (rdmsrl_safe_on_cpu(cpu, msr, &val) || !val)
1331 return -ENODEV;
2d281d81 1332
9d31c676 1333 return 0;
2d281d81
JP		 1334}
1335
1336/* Detect active and valid domains for the given CPU, caller must
1337 * ensure the CPU belongs to the targeted package and CPU hotlug is disabled.
1338 */
1339static int rapl_detect_domains(struct rapl_package *rp, int cpu)
1340{
1341 int i;
1342 int ret = 0;
1343 struct rapl_domain *rd;
1344 u64 locked;
1345
1346 for (i = 0; i < RAPL_DOMAIN_MAX; i++) {
1347 /* use physical package id to read counters */
fcdf1797 1348 if (!rapl_check_domain(cpu, i)) {
2d281d81 1349 rp->domain_map |= 1 << i;
fcdf1797
JP		 1350 pr_info("Found RAPL domain %s\n", rapl_domain_names[i]);
1351 }
2d281d81
JP		 1352 }
1353 rp->nr_domains = bitmap_weight(&rp->domain_map, RAPL_DOMAIN_MAX);
1354 if (!rp->nr_domains) {
1355 pr_err("no valid rapl domains found in package %d\n", rp->id);
1356 ret = -ENODEV;
1357 goto done;
1358 }
1359 pr_debug("found %d domains on package %d\n", rp->nr_domains, rp->id);
1360
1361 rp->domains = kcalloc(rp->nr_domains + 1, sizeof(struct rapl_domain),
1362 GFP_KERNEL);
1363 if (!rp->domains) {
1364 ret = -ENOMEM;
1365 goto done;
1366 }
1367 rapl_init_domains(rp);
1368
1369 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
1370 /* check if the domain is locked by BIOS */
79a21dbf
PB		 1371 ret = rapl_read_data_raw(rd, FW_LOCK, false, &locked);
1372 if (ret)
1373 return ret;
1374 if (locked) {
2d281d81
JP		 1375 pr_info("RAPL package %d domain %s locked by BIOS\n",
1376 rp->id, rd->name);
79a21dbf 1377 rd->state |= DOMAIN_STATE_BIOS_LOCKED;
2d281d81
JP		 1378 }
1379 }
1380
1381
1382done:
1383 return ret;
1384}
1385
1386static bool is_package_new(int package)
1387{
1388 struct rapl_package *rp;
1389
1390 /* caller prevents cpu hotplug, there will be no new packages added
1391 * or deleted while traversing the package list, no need for locking.
1392 */
1393 list_for_each_entry(rp, &rapl_packages, plist)
1394 if (package == rp->id)
1395 return false;
1396
1397 return true;
1398}
1399
1400/* RAPL interface can be made of a two-level hierarchy: package level and domain
1401 * level. We first detect the number of packages then domains of each package.
1402 * We have to consider the possiblity of CPU online/offline due to hotplug and
1403 * other scenarios.
1404 */
1405static int rapl_detect_topology(void)
1406{
1407 int i;
1408 int phy_package_id;
1409 struct rapl_package *new_package, *rp;
1410
1411 for_each_online_cpu(i) {
1412 phy_package_id = topology_physical_package_id(i);
1413 if (is_package_new(phy_package_id)) {
1414 new_package = kzalloc(sizeof(*rp), GFP_KERNEL);
1415 if (!new_package) {
1416 rapl_cleanup_data();
1417 return -ENOMEM;
1418 }
1419 /* add the new package to the list */
1420 new_package->id = phy_package_id;
1421 new_package->nr_cpus = 1;
1422
1423 /* check if the package contains valid domains */
1424 if (rapl_detect_domains(new_package, i) ||
3c2c0845 1425 rapl_defaults->check_unit(new_package, i)) {
2d281d81
JP		 1426 kfree(new_package->domains);
1427 kfree(new_package);
1428 /* free up the packages already initialized */
1429 rapl_cleanup_data();
1430 return -ENODEV;
1431 }
1432 INIT_LIST_HEAD(&new_package->plist);
1433 list_add(&new_package->plist, &rapl_packages);
1434 } else {
1435 rp = find_package_by_id(phy_package_id);
1436 if (rp)
1437 ++rp->nr_cpus;
1438 }
1439 }
1440
1441 return 0;
1442}
1443
1444/* called from CPU hotplug notifier, hotplug lock held */
1445static void rapl_remove_package(struct rapl_package *rp)
1446{
1447 struct rapl_domain *rd, *rd_package = NULL;
1448
1449 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
1450 if (rd->id == RAPL_DOMAIN_PACKAGE) {
1451 rd_package = rd;
1452 continue;
1453 }
1454 pr_debug("remove package %d, %s domain\n", rp->id, rd->name);
1455 powercap_unregister_zone(control_type, &rd->power_zone);
1456 }
1457 /* do parent zone last */
1458 powercap_unregister_zone(control_type, &rd_package->power_zone);
1459 list_del(&rp->plist);
1460 kfree(rp);
1461}
1462
1463/* called from CPU hotplug notifier, hotplug lock held */
1464static int rapl_add_package(int cpu)
1465{
1466 int ret = 0;
1467 int phy_package_id;
1468 struct rapl_package *rp;
1469
1470 phy_package_id = topology_physical_package_id(cpu);
1471 rp = kzalloc(sizeof(struct rapl_package), GFP_KERNEL);
1472 if (!rp)
1473 return -ENOMEM;
1474
1475 /* add the new package to the list */
1476 rp->id = phy_package_id;
1477 rp->nr_cpus = 1;
1478 /* check if the package contains valid domains */
1479 if (rapl_detect_domains(rp, cpu) ||
3c2c0845 1480 rapl_defaults->check_unit(rp, cpu)) {
2d281d81
JP		 1481 ret = -ENODEV;
1482 goto err_free_package;
1483 }
1484 if (!rapl_package_register_powercap(rp)) {
1485 INIT_LIST_HEAD(&rp->plist);
1486 list_add(&rp->plist, &rapl_packages);
1487 return ret;
1488 }
1489
1490err_free_package:
1491 kfree(rp->domains);
1492 kfree(rp);
1493
1494 return ret;
1495}
1496
1497/* Handles CPU hotplug on multi-socket systems.
1498 * If a CPU goes online as the first CPU of the physical package
1499 * we add the RAPL package to the system. Similarly, when the last
1500 * CPU of the package is removed, we remove the RAPL package and its
1501 * associated domains. Cooling devices are handled accordingly at
1502 * per-domain level.
1503 */
1504static int rapl_cpu_callback(struct notifier_block *nfb,
1505 unsigned long action, void *hcpu)
1506{
1507 unsigned long cpu = (unsigned long)hcpu;
1508 int phy_package_id;
1509 struct rapl_package *rp;
1510
1511 phy_package_id = topology_physical_package_id(cpu);
1512 switch (action) {
1513 case CPU_ONLINE:
1514 case CPU_ONLINE_FROZEN:
1515 case CPU_DOWN_FAILED:
1516 case CPU_DOWN_FAILED_FROZEN:
1517 rp = find_package_by_id(phy_package_id);
1518 if (rp)
1519 ++rp->nr_cpus;
1520 else
1521 rapl_add_package(cpu);
1522 break;
1523 case CPU_DOWN_PREPARE:
1524 case CPU_DOWN_PREPARE_FROZEN:
1525 rp = find_package_by_id(phy_package_id);
1526 if (!rp)
1527 break;
1528 if (--rp->nr_cpus == 0)
1529 rapl_remove_package(rp);
1530 }
1531
1532 return NOTIFY_OK;
1533}
1534
1535static struct notifier_block rapl_cpu_notifier = {
1536 .notifier_call = rapl_cpu_callback,
1537};
1538
1539static int __init rapl_init(void)
1540{
1541 int ret = 0;
087e9cba 1542 const struct x86_cpu_id *id;
2d281d81 1543
087e9cba
JP		 1544 id = x86_match_cpu(rapl_ids);
1545 if (!id) {
2d281d81
JP		 1546 pr_err("driver does not support CPU family %d model %d\n",
1547 boot_cpu_data.x86, boot_cpu_data.x86_model);
1548
1549 return -ENODEV;
1550 }
009f225e 1551
087e9cba
JP		 1552 rapl_defaults = (struct rapl_defaults *)id->driver_data;
1553
009f225e
SB		 1554 cpu_notifier_register_begin();
1555
2d281d81
JP		 1556 /* prevent CPU hotplug during detection */
1557 get_online_cpus();
1558 ret = rapl_detect_topology();
1559 if (ret)
1560 goto done;
1561
1562 if (rapl_register_powercap()) {
1563 rapl_cleanup_data();
1564 ret = -ENODEV;
1565 goto done;
1566 }
009f225e 1567 __register_hotcpu_notifier(&rapl_cpu_notifier);
2d281d81
JP		 1568done:
1569 put_online_cpus();
009f225e 1570 cpu_notifier_register_done();
2d281d81
JP		 1571
1572 return ret;
1573}
1574
1575static void __exit rapl_exit(void)
1576{
009f225e 1577 cpu_notifier_register_begin();
2d281d81 1578 get_online_cpus();
009f225e 1579 __unregister_hotcpu_notifier(&rapl_cpu_notifier);
2d281d81
JP		 1580 rapl_unregister_powercap();
1581 rapl_cleanup_data();
1582 put_online_cpus();
009f225e 1583 cpu_notifier_register_done();
2d281d81
JP		 1584}
1585
1586module_init(rapl_init);
1587module_exit(rapl_exit);
1588
1589MODULE_DESCRIPTION("Driver for Intel RAPL (Running Average Power Limit)");
1590MODULE_AUTHOR("Jacob Pan <jacob.jun.pan@intel.com>");
1591MODULE_LICENSE("GPL v2");
ubuntu-bionic-kernel mirror