]>
Commit | Line | Data |
---|---|---|
7ce1346a | 1 | /* |
940b2f2f | 2 | * Support cstate residency counters |
7ce1346a KL |
3 | * |
4 | * Copyright (C) 2015, Intel Corp. | |
5 | * Author: Kan Liang (kan.liang@intel.com) | |
6 | * | |
7 | * This library is free software; you can redistribute it and/or | |
8 | * modify it under the terms of the GNU Library General Public | |
9 | * License as published by the Free Software Foundation; either | |
10 | * version 2 of the License, or (at your option) any later version. | |
11 | * | |
12 | * This library is distributed in the hope that it will be useful, | |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
15 | * Library General Public License for more details. | |
16 | * | |
17 | */ | |
18 | ||
19 | /* | |
20 | * This file export cstate related free running (read-only) counters | |
21 | * for perf. These counters may be use simultaneously by other tools, | |
22 | * such as turbostat. However, it still make sense to implement them | |
23 | * in perf. Because we can conveniently collect them together with | |
24 | * other events, and allow to use them from tools without special MSR | |
25 | * access code. | |
26 | * | |
27 | * The events only support system-wide mode counting. There is no | |
28 | * sampling support because it is not supported by the hardware. | |
29 | * | |
30 | * According to counters' scope and category, two PMUs are registered | |
31 | * with the perf_event core subsystem. | |
32 | * - 'cstate_core': The counter is available for each physical core. | |
33 | * The counters include CORE_C*_RESIDENCY. | |
34 | * - 'cstate_pkg': The counter is available for each physical package. | |
35 | * The counters include PKG_C*_RESIDENCY. | |
36 | * | |
37 | * All of these counters are specified in the IntelĀ® 64 and IA-32 | |
38 | * Architectures Software Developer.s Manual Vol3b. | |
39 | * | |
40 | * Model specific counters: | |
41 | * MSR_CORE_C1_RES: CORE C1 Residency Counter | |
42 | * perf code: 0x00 | |
2da202aa | 43 | * Available model: SLM,AMT,GLM,CNL,ICX,TNT,ADL,RPL |
7ce1346a KL |
44 | * Scope: Core (each processor core has a MSR) |
45 | * MSR_CORE_C3_RESIDENCY: CORE C3 Residency Counter | |
46 | * perf code: 0x01 | |
1159e094 | 47 | * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,GLM, |
ecf71fbc | 48 | * CNL,KBL,CML,TNT |
7ce1346a KL |
49 | * Scope: Core |
50 | * MSR_CORE_C6_RESIDENCY: CORE C6 Residency Counter | |
51 | * perf code: 0x02 | |
1159e094 | 52 | * Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW, |
87bf399f | 53 | * SKL,KNL,GLM,CNL,KBL,CML,ICL,ICX, |
528c9f1d | 54 | * TGL,TNT,RKL,ADL,RPL,SPR |
7ce1346a KL |
55 | * Scope: Core |
56 | * MSR_CORE_C7_RESIDENCY: CORE C7 Residency Counter | |
57 | * perf code: 0x03 | |
f1857a24 | 58 | * Available model: SNB,IVB,HSW,BDW,SKL,CNL,KBL,CML, |
2da202aa | 59 | * ICL,TGL,RKL,ADL,RPL |
7ce1346a KL |
60 | * Scope: Core |
61 | * MSR_PKG_C2_RESIDENCY: Package C2 Residency Counter. | |
62 | * perf code: 0x00 | |
1ffa6c04 | 63 | * Available model: SNB,IVB,HSW,BDW,SKL,KNL,GLM,CNL, |
2da202aa | 64 | * KBL,CML,ICL,ICX,TGL,TNT,RKL,ADL, |
528c9f1d | 65 | * RPL,SPR |
7ce1346a KL |
66 | * Scope: Package (physical package) |
67 | * MSR_PKG_C3_RESIDENCY: Package C3 Residency Counter. | |
68 | * perf code: 0x01 | |
1159e094 | 69 | * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,KNL, |
d0ca946b | 70 | * GLM,CNL,KBL,CML,ICL,TGL,TNT,RKL, |
2da202aa | 71 | * ADL,RPL |
7ce1346a KL |
72 | * Scope: Package (physical package) |
73 | * MSR_PKG_C6_RESIDENCY: Package C6 Residency Counter. | |
74 | * perf code: 0x02 | |
ecf71fbc | 75 | * Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW, |
87bf399f | 76 | * SKL,KNL,GLM,CNL,KBL,CML,ICL,ICX, |
528c9f1d | 77 | * TGL,TNT,RKL,ADL,RPL,SPR |
7ce1346a KL |
78 | * Scope: Package (physical package) |
79 | * MSR_PKG_C7_RESIDENCY: Package C7 Residency Counter. | |
80 | * perf code: 0x03 | |
1ffa6c04 | 81 | * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,CNL, |
2da202aa | 82 | * KBL,CML,ICL,TGL,RKL,ADL,RPL |
7ce1346a KL |
83 | * Scope: Package (physical package) |
84 | * MSR_PKG_C8_RESIDENCY: Package C8 Residency Counter. | |
85 | * perf code: 0x04 | |
d0ca946b | 86 | * Available model: HSW ULT,KBL,CNL,CML,ICL,TGL,RKL, |
2da202aa | 87 | * ADL,RPL |
7ce1346a KL |
88 | * Scope: Package (physical package) |
89 | * MSR_PKG_C9_RESIDENCY: Package C9 Residency Counter. | |
90 | * perf code: 0x05 | |
d0ca946b | 91 | * Available model: HSW ULT,KBL,CNL,CML,ICL,TGL,RKL, |
2da202aa | 92 | * ADL,RPL |
7ce1346a KL |
93 | * Scope: Package (physical package) |
94 | * MSR_PKG_C10_RESIDENCY: Package C10 Residency Counter. | |
95 | * perf code: 0x06 | |
ecf71fbc | 96 | * Available model: HSW ULT,KBL,GLM,CNL,CML,ICL,TGL, |
2da202aa | 97 | * TNT,RKL,ADL,RPL |
7ce1346a KL |
98 | * Scope: Package (physical package) |
99 | * | |
100 | */ | |
101 | ||
102 | #include <linux/module.h> | |
103 | #include <linux/slab.h> | |
104 | #include <linux/perf_event.h> | |
a5f81290 | 105 | #include <linux/nospec.h> |
7ce1346a | 106 | #include <asm/cpu_device_id.h> |
bf4ad541 | 107 | #include <asm/intel-family.h> |
27f6d22b | 108 | #include "../perf_event.h" |
8f2a28c5 | 109 | #include "../probe.h" |
7ce1346a | 110 | |
c7afba32 TG |
111 | MODULE_LICENSE("GPL"); |
112 | ||
7ce1346a | 113 | #define DEFINE_CSTATE_FORMAT_ATTR(_var, _name, _format) \ |
ebd19fc3 ST |
114 | static ssize_t __cstate_##_var##_show(struct device *dev, \ |
115 | struct device_attribute *attr, \ | |
7ce1346a KL |
116 | char *page) \ |
117 | { \ | |
118 | BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \ | |
119 | return sprintf(page, _format "\n"); \ | |
120 | } \ | |
ebd19fc3 | 121 | static struct device_attribute format_attr_##_var = \ |
7ce1346a KL |
122 | __ATTR(_name, 0444, __cstate_##_var##_show, NULL) |
123 | ||
124 | static ssize_t cstate_get_attr_cpumask(struct device *dev, | |
125 | struct device_attribute *attr, | |
126 | char *buf); | |
127 | ||
424646ee TG |
128 | /* Model -> events mapping */ |
129 | struct cstate_model { | |
130 | unsigned long core_events; | |
131 | unsigned long pkg_events; | |
132 | unsigned long quirks; | |
133 | }; | |
134 | ||
135 | /* Quirk flags */ | |
136 | #define SLM_PKG_C6_USE_C7_MSR (1UL << 0) | |
889882bc | 137 | #define KNL_CORE_C6_MSR (1UL << 1) |
424646ee | 138 | |
7ce1346a KL |
139 | struct perf_cstate_msr { |
140 | u64 msr; | |
141 | struct perf_pmu_events_attr *attr; | |
7ce1346a KL |
142 | }; |
143 | ||
144 | ||
145 | /* cstate_core PMU */ | |
7ce1346a KL |
146 | static struct pmu cstate_core_pmu; |
147 | static bool has_cstate_core; | |
148 | ||
424646ee | 149 | enum perf_cstate_core_events { |
7ce1346a KL |
150 | PERF_CSTATE_CORE_C1_RES = 0, |
151 | PERF_CSTATE_CORE_C3_RES, | |
152 | PERF_CSTATE_CORE_C6_RES, | |
153 | PERF_CSTATE_CORE_C7_RES, | |
154 | ||
155 | PERF_CSTATE_CORE_EVENT_MAX, | |
156 | }; | |
157 | ||
8f2a28c5 JO |
158 | PMU_EVENT_ATTR_STRING(c1-residency, attr_cstate_core_c1, "event=0x00"); |
159 | PMU_EVENT_ATTR_STRING(c3-residency, attr_cstate_core_c3, "event=0x01"); | |
160 | PMU_EVENT_ATTR_STRING(c6-residency, attr_cstate_core_c6, "event=0x02"); | |
161 | PMU_EVENT_ATTR_STRING(c7-residency, attr_cstate_core_c7, "event=0x03"); | |
7ce1346a | 162 | |
8f2a28c5 JO |
163 | static unsigned long core_msr_mask; |
164 | ||
165 | PMU_EVENT_GROUP(events, cstate_core_c1); | |
166 | PMU_EVENT_GROUP(events, cstate_core_c3); | |
167 | PMU_EVENT_GROUP(events, cstate_core_c6); | |
168 | PMU_EVENT_GROUP(events, cstate_core_c7); | |
169 | ||
170 | static bool test_msr(int idx, void *data) | |
171 | { | |
172 | return test_bit(idx, (unsigned long *) data); | |
173 | } | |
174 | ||
175 | static struct perf_msr core_msr[] = { | |
176 | [PERF_CSTATE_CORE_C1_RES] = { MSR_CORE_C1_RES, &group_cstate_core_c1, test_msr }, | |
177 | [PERF_CSTATE_CORE_C3_RES] = { MSR_CORE_C3_RESIDENCY, &group_cstate_core_c3, test_msr }, | |
178 | [PERF_CSTATE_CORE_C6_RES] = { MSR_CORE_C6_RESIDENCY, &group_cstate_core_c6, test_msr }, | |
179 | [PERF_CSTATE_CORE_C7_RES] = { MSR_CORE_C7_RESIDENCY, &group_cstate_core_c7, test_msr }, | |
7ce1346a KL |
180 | }; |
181 | ||
8f2a28c5 | 182 | static struct attribute *attrs_empty[] = { |
7ce1346a KL |
183 | NULL, |
184 | }; | |
185 | ||
8f2a28c5 JO |
186 | /* |
187 | * There are no default events, but we need to create | |
188 | * "events" group (with empty attrs) before updating | |
189 | * it with detected events. | |
190 | */ | |
7ce1346a KL |
191 | static struct attribute_group core_events_attr_group = { |
192 | .name = "events", | |
8f2a28c5 | 193 | .attrs = attrs_empty, |
7ce1346a KL |
194 | }; |
195 | ||
196 | DEFINE_CSTATE_FORMAT_ATTR(core_event, event, "config:0-63"); | |
197 | static struct attribute *core_format_attrs[] = { | |
198 | &format_attr_core_event.attr, | |
199 | NULL, | |
200 | }; | |
201 | ||
202 | static struct attribute_group core_format_attr_group = { | |
203 | .name = "format", | |
204 | .attrs = core_format_attrs, | |
205 | }; | |
206 | ||
207 | static cpumask_t cstate_core_cpu_mask; | |
208 | static DEVICE_ATTR(cpumask, S_IRUGO, cstate_get_attr_cpumask, NULL); | |
209 | ||
210 | static struct attribute *cstate_cpumask_attrs[] = { | |
211 | &dev_attr_cpumask.attr, | |
212 | NULL, | |
213 | }; | |
214 | ||
215 | static struct attribute_group cpumask_attr_group = { | |
216 | .attrs = cstate_cpumask_attrs, | |
217 | }; | |
218 | ||
219 | static const struct attribute_group *core_attr_groups[] = { | |
220 | &core_events_attr_group, | |
221 | &core_format_attr_group, | |
222 | &cpumask_attr_group, | |
223 | NULL, | |
224 | }; | |
225 | ||
7ce1346a | 226 | /* cstate_pkg PMU */ |
7ce1346a KL |
227 | static struct pmu cstate_pkg_pmu; |
228 | static bool has_cstate_pkg; | |
229 | ||
424646ee | 230 | enum perf_cstate_pkg_events { |
7ce1346a KL |
231 | PERF_CSTATE_PKG_C2_RES = 0, |
232 | PERF_CSTATE_PKG_C3_RES, | |
233 | PERF_CSTATE_PKG_C6_RES, | |
234 | PERF_CSTATE_PKG_C7_RES, | |
235 | PERF_CSTATE_PKG_C8_RES, | |
236 | PERF_CSTATE_PKG_C9_RES, | |
237 | PERF_CSTATE_PKG_C10_RES, | |
238 | ||
239 | PERF_CSTATE_PKG_EVENT_MAX, | |
240 | }; | |
241 | ||
8f2a28c5 JO |
242 | PMU_EVENT_ATTR_STRING(c2-residency, attr_cstate_pkg_c2, "event=0x00"); |
243 | PMU_EVENT_ATTR_STRING(c3-residency, attr_cstate_pkg_c3, "event=0x01"); | |
244 | PMU_EVENT_ATTR_STRING(c6-residency, attr_cstate_pkg_c6, "event=0x02"); | |
245 | PMU_EVENT_ATTR_STRING(c7-residency, attr_cstate_pkg_c7, "event=0x03"); | |
246 | PMU_EVENT_ATTR_STRING(c8-residency, attr_cstate_pkg_c8, "event=0x04"); | |
247 | PMU_EVENT_ATTR_STRING(c9-residency, attr_cstate_pkg_c9, "event=0x05"); | |
248 | PMU_EVENT_ATTR_STRING(c10-residency, attr_cstate_pkg_c10, "event=0x06"); | |
249 | ||
250 | static unsigned long pkg_msr_mask; | |
251 | ||
252 | PMU_EVENT_GROUP(events, cstate_pkg_c2); | |
253 | PMU_EVENT_GROUP(events, cstate_pkg_c3); | |
254 | PMU_EVENT_GROUP(events, cstate_pkg_c6); | |
255 | PMU_EVENT_GROUP(events, cstate_pkg_c7); | |
256 | PMU_EVENT_GROUP(events, cstate_pkg_c8); | |
257 | PMU_EVENT_GROUP(events, cstate_pkg_c9); | |
258 | PMU_EVENT_GROUP(events, cstate_pkg_c10); | |
259 | ||
260 | static struct perf_msr pkg_msr[] = { | |
261 | [PERF_CSTATE_PKG_C2_RES] = { MSR_PKG_C2_RESIDENCY, &group_cstate_pkg_c2, test_msr }, | |
262 | [PERF_CSTATE_PKG_C3_RES] = { MSR_PKG_C3_RESIDENCY, &group_cstate_pkg_c3, test_msr }, | |
263 | [PERF_CSTATE_PKG_C6_RES] = { MSR_PKG_C6_RESIDENCY, &group_cstate_pkg_c6, test_msr }, | |
264 | [PERF_CSTATE_PKG_C7_RES] = { MSR_PKG_C7_RESIDENCY, &group_cstate_pkg_c7, test_msr }, | |
265 | [PERF_CSTATE_PKG_C8_RES] = { MSR_PKG_C8_RESIDENCY, &group_cstate_pkg_c8, test_msr }, | |
266 | [PERF_CSTATE_PKG_C9_RES] = { MSR_PKG_C9_RESIDENCY, &group_cstate_pkg_c9, test_msr }, | |
267 | [PERF_CSTATE_PKG_C10_RES] = { MSR_PKG_C10_RESIDENCY, &group_cstate_pkg_c10, test_msr }, | |
7ce1346a KL |
268 | }; |
269 | ||
270 | static struct attribute_group pkg_events_attr_group = { | |
271 | .name = "events", | |
8f2a28c5 | 272 | .attrs = attrs_empty, |
7ce1346a KL |
273 | }; |
274 | ||
275 | DEFINE_CSTATE_FORMAT_ATTR(pkg_event, event, "config:0-63"); | |
276 | static struct attribute *pkg_format_attrs[] = { | |
277 | &format_attr_pkg_event.attr, | |
278 | NULL, | |
279 | }; | |
280 | static struct attribute_group pkg_format_attr_group = { | |
281 | .name = "format", | |
282 | .attrs = pkg_format_attrs, | |
283 | }; | |
284 | ||
285 | static cpumask_t cstate_pkg_cpu_mask; | |
286 | ||
287 | static const struct attribute_group *pkg_attr_groups[] = { | |
288 | &pkg_events_attr_group, | |
289 | &pkg_format_attr_group, | |
290 | &cpumask_attr_group, | |
291 | NULL, | |
292 | }; | |
293 | ||
7ce1346a KL |
294 | static ssize_t cstate_get_attr_cpumask(struct device *dev, |
295 | struct device_attribute *attr, | |
296 | char *buf) | |
297 | { | |
298 | struct pmu *pmu = dev_get_drvdata(dev); | |
299 | ||
300 | if (pmu == &cstate_core_pmu) | |
301 | return cpumap_print_to_pagebuf(true, buf, &cstate_core_cpu_mask); | |
302 | else if (pmu == &cstate_pkg_pmu) | |
303 | return cpumap_print_to_pagebuf(true, buf, &cstate_pkg_cpu_mask); | |
304 | else | |
305 | return 0; | |
306 | } | |
307 | ||
308 | static int cstate_pmu_event_init(struct perf_event *event) | |
309 | { | |
310 | u64 cfg = event->attr.config; | |
49de0493 | 311 | int cpu; |
7ce1346a KL |
312 | |
313 | if (event->attr.type != event->pmu->type) | |
314 | return -ENOENT; | |
315 | ||
316 | /* unsupported modes and filters */ | |
2ff40250 | 317 | if (event->attr.sample_period) /* no sampling */ |
7ce1346a KL |
318 | return -EINVAL; |
319 | ||
49de0493 TG |
320 | if (event->cpu < 0) |
321 | return -EINVAL; | |
322 | ||
7ce1346a KL |
323 | if (event->pmu == &cstate_core_pmu) { |
324 | if (cfg >= PERF_CSTATE_CORE_EVENT_MAX) | |
325 | return -EINVAL; | |
8f2a28c5 JO |
326 | cfg = array_index_nospec((unsigned long)cfg, PERF_CSTATE_CORE_EVENT_MAX); |
327 | if (!(core_msr_mask & (1 << cfg))) | |
7ce1346a KL |
328 | return -EINVAL; |
329 | event->hw.event_base = core_msr[cfg].msr; | |
49de0493 TG |
330 | cpu = cpumask_any_and(&cstate_core_cpu_mask, |
331 | topology_sibling_cpumask(event->cpu)); | |
7ce1346a KL |
332 | } else if (event->pmu == &cstate_pkg_pmu) { |
333 | if (cfg >= PERF_CSTATE_PKG_EVENT_MAX) | |
334 | return -EINVAL; | |
a5f81290 | 335 | cfg = array_index_nospec((unsigned long)cfg, PERF_CSTATE_PKG_EVENT_MAX); |
8f2a28c5 | 336 | if (!(pkg_msr_mask & (1 << cfg))) |
7ce1346a KL |
337 | return -EINVAL; |
338 | event->hw.event_base = pkg_msr[cfg].msr; | |
49de0493 | 339 | cpu = cpumask_any_and(&cstate_pkg_cpu_mask, |
cb63ba0f | 340 | topology_die_cpumask(event->cpu)); |
49de0493 | 341 | } else { |
7ce1346a | 342 | return -ENOENT; |
49de0493 TG |
343 | } |
344 | ||
345 | if (cpu >= nr_cpu_ids) | |
346 | return -ENODEV; | |
7ce1346a | 347 | |
49de0493 | 348 | event->cpu = cpu; |
7ce1346a KL |
349 | event->hw.config = cfg; |
350 | event->hw.idx = -1; | |
49de0493 | 351 | return 0; |
7ce1346a KL |
352 | } |
353 | ||
354 | static inline u64 cstate_pmu_read_counter(struct perf_event *event) | |
355 | { | |
356 | u64 val; | |
357 | ||
358 | rdmsrl(event->hw.event_base, val); | |
359 | return val; | |
360 | } | |
361 | ||
362 | static void cstate_pmu_event_update(struct perf_event *event) | |
363 | { | |
364 | struct hw_perf_event *hwc = &event->hw; | |
365 | u64 prev_raw_count, new_raw_count; | |
366 | ||
367 | again: | |
368 | prev_raw_count = local64_read(&hwc->prev_count); | |
369 | new_raw_count = cstate_pmu_read_counter(event); | |
370 | ||
371 | if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, | |
372 | new_raw_count) != prev_raw_count) | |
373 | goto again; | |
374 | ||
375 | local64_add(new_raw_count - prev_raw_count, &event->count); | |
376 | } | |
377 | ||
378 | static void cstate_pmu_event_start(struct perf_event *event, int mode) | |
379 | { | |
380 | local64_set(&event->hw.prev_count, cstate_pmu_read_counter(event)); | |
381 | } | |
382 | ||
383 | static void cstate_pmu_event_stop(struct perf_event *event, int mode) | |
384 | { | |
385 | cstate_pmu_event_update(event); | |
386 | } | |
387 | ||
388 | static void cstate_pmu_event_del(struct perf_event *event, int mode) | |
389 | { | |
390 | cstate_pmu_event_stop(event, PERF_EF_UPDATE); | |
391 | } | |
392 | ||
393 | static int cstate_pmu_event_add(struct perf_event *event, int mode) | |
394 | { | |
395 | if (mode & PERF_EF_START) | |
396 | cstate_pmu_event_start(event, mode); | |
397 | ||
398 | return 0; | |
399 | } | |
400 | ||
49de0493 TG |
401 | /* |
402 | * Check if exiting cpu is the designated reader. If so migrate the | |
403 | * events when there is a valid target available | |
404 | */ | |
77c34ef1 | 405 | static int cstate_cpu_exit(unsigned int cpu) |
7ce1346a | 406 | { |
49de0493 TG |
407 | unsigned int target; |
408 | ||
409 | if (has_cstate_core && | |
410 | cpumask_test_and_clear_cpu(cpu, &cstate_core_cpu_mask)) { | |
411 | ||
412 | target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu); | |
413 | /* Migrate events if there is a valid target */ | |
414 | if (target < nr_cpu_ids) { | |
7ce1346a | 415 | cpumask_set_cpu(target, &cstate_core_cpu_mask); |
7ce1346a | 416 | perf_pmu_migrate_context(&cstate_core_pmu, cpu, target); |
49de0493 | 417 | } |
7ce1346a KL |
418 | } |
419 | ||
49de0493 TG |
420 | if (has_cstate_pkg && |
421 | cpumask_test_and_clear_cpu(cpu, &cstate_pkg_cpu_mask)) { | |
422 | ||
cb63ba0f | 423 | target = cpumask_any_but(topology_die_cpumask(cpu), cpu); |
49de0493 TG |
424 | /* Migrate events if there is a valid target */ |
425 | if (target < nr_cpu_ids) { | |
7ce1346a | 426 | cpumask_set_cpu(target, &cstate_pkg_cpu_mask); |
7ce1346a | 427 | perf_pmu_migrate_context(&cstate_pkg_pmu, cpu, target); |
49de0493 | 428 | } |
7ce1346a | 429 | } |
77c34ef1 | 430 | return 0; |
7ce1346a KL |
431 | } |
432 | ||
77c34ef1 | 433 | static int cstate_cpu_init(unsigned int cpu) |
7ce1346a | 434 | { |
49de0493 | 435 | unsigned int target; |
7ce1346a | 436 | |
49de0493 TG |
437 | /* |
438 | * If this is the first online thread of that core, set it in | |
439 | * the core cpu mask as the designated reader. | |
440 | */ | |
441 | target = cpumask_any_and(&cstate_core_cpu_mask, | |
442 | topology_sibling_cpumask(cpu)); | |
443 | ||
444 | if (has_cstate_core && target >= nr_cpu_ids) | |
445 | cpumask_set_cpu(cpu, &cstate_core_cpu_mask); | |
446 | ||
447 | /* | |
448 | * If this is the first online thread of that package, set it | |
449 | * in the package cpu mask as the designated reader. | |
450 | */ | |
451 | target = cpumask_any_and(&cstate_pkg_cpu_mask, | |
cb63ba0f | 452 | topology_die_cpumask(cpu)); |
49de0493 TG |
453 | if (has_cstate_pkg && target >= nr_cpu_ids) |
454 | cpumask_set_cpu(cpu, &cstate_pkg_cpu_mask); | |
7ce1346a | 455 | |
77c34ef1 | 456 | return 0; |
7ce1346a KL |
457 | } |
458 | ||
d9f3b450 | 459 | static const struct attribute_group *core_attr_update[] = { |
8f2a28c5 JO |
460 | &group_cstate_core_c1, |
461 | &group_cstate_core_c3, | |
462 | &group_cstate_core_c6, | |
463 | &group_cstate_core_c7, | |
464 | NULL, | |
465 | }; | |
466 | ||
d9f3b450 | 467 | static const struct attribute_group *pkg_attr_update[] = { |
8f2a28c5 JO |
468 | &group_cstate_pkg_c2, |
469 | &group_cstate_pkg_c3, | |
470 | &group_cstate_pkg_c6, | |
471 | &group_cstate_pkg_c7, | |
472 | &group_cstate_pkg_c8, | |
473 | &group_cstate_pkg_c9, | |
474 | &group_cstate_pkg_c10, | |
475 | NULL, | |
476 | }; | |
477 | ||
424646ee TG |
478 | static struct pmu cstate_core_pmu = { |
479 | .attr_groups = core_attr_groups, | |
8f2a28c5 | 480 | .attr_update = core_attr_update, |
424646ee TG |
481 | .name = "cstate_core", |
482 | .task_ctx_nr = perf_invalid_context, | |
483 | .event_init = cstate_pmu_event_init, | |
484 | .add = cstate_pmu_event_add, | |
485 | .del = cstate_pmu_event_del, | |
486 | .start = cstate_pmu_event_start, | |
487 | .stop = cstate_pmu_event_stop, | |
488 | .read = cstate_pmu_event_update, | |
2ff40250 | 489 | .capabilities = PERF_PMU_CAP_NO_INTERRUPT | PERF_PMU_CAP_NO_EXCLUDE, |
74545f63 | 490 | .module = THIS_MODULE, |
424646ee TG |
491 | }; |
492 | ||
493 | static struct pmu cstate_pkg_pmu = { | |
494 | .attr_groups = pkg_attr_groups, | |
8f2a28c5 | 495 | .attr_update = pkg_attr_update, |
424646ee TG |
496 | .name = "cstate_pkg", |
497 | .task_ctx_nr = perf_invalid_context, | |
498 | .event_init = cstate_pmu_event_init, | |
499 | .add = cstate_pmu_event_add, | |
500 | .del = cstate_pmu_event_del, | |
501 | .start = cstate_pmu_event_start, | |
502 | .stop = cstate_pmu_event_stop, | |
503 | .read = cstate_pmu_event_update, | |
2ff40250 | 504 | .capabilities = PERF_PMU_CAP_NO_INTERRUPT | PERF_PMU_CAP_NO_EXCLUDE, |
74545f63 | 505 | .module = THIS_MODULE, |
424646ee TG |
506 | }; |
507 | ||
508 | static const struct cstate_model nhm_cstates __initconst = { | |
509 | .core_events = BIT(PERF_CSTATE_CORE_C3_RES) | | |
510 | BIT(PERF_CSTATE_CORE_C6_RES), | |
511 | ||
512 | .pkg_events = BIT(PERF_CSTATE_PKG_C3_RES) | | |
513 | BIT(PERF_CSTATE_PKG_C6_RES) | | |
514 | BIT(PERF_CSTATE_PKG_C7_RES), | |
515 | }; | |
516 | ||
517 | static const struct cstate_model snb_cstates __initconst = { | |
518 | .core_events = BIT(PERF_CSTATE_CORE_C3_RES) | | |
519 | BIT(PERF_CSTATE_CORE_C6_RES) | | |
520 | BIT(PERF_CSTATE_CORE_C7_RES), | |
521 | ||
522 | .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) | | |
523 | BIT(PERF_CSTATE_PKG_C3_RES) | | |
524 | BIT(PERF_CSTATE_PKG_C6_RES) | | |
525 | BIT(PERF_CSTATE_PKG_C7_RES), | |
526 | }; | |
527 | ||
528 | static const struct cstate_model hswult_cstates __initconst = { | |
529 | .core_events = BIT(PERF_CSTATE_CORE_C3_RES) | | |
530 | BIT(PERF_CSTATE_CORE_C6_RES) | | |
531 | BIT(PERF_CSTATE_CORE_C7_RES), | |
532 | ||
533 | .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) | | |
534 | BIT(PERF_CSTATE_PKG_C3_RES) | | |
535 | BIT(PERF_CSTATE_PKG_C6_RES) | | |
536 | BIT(PERF_CSTATE_PKG_C7_RES) | | |
537 | BIT(PERF_CSTATE_PKG_C8_RES) | | |
538 | BIT(PERF_CSTATE_PKG_C9_RES) | | |
539 | BIT(PERF_CSTATE_PKG_C10_RES), | |
540 | }; | |
541 | ||
1159e094 HP |
542 | static const struct cstate_model cnl_cstates __initconst = { |
543 | .core_events = BIT(PERF_CSTATE_CORE_C1_RES) | | |
544 | BIT(PERF_CSTATE_CORE_C3_RES) | | |
545 | BIT(PERF_CSTATE_CORE_C6_RES) | | |
546 | BIT(PERF_CSTATE_CORE_C7_RES), | |
547 | ||
548 | .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) | | |
549 | BIT(PERF_CSTATE_PKG_C3_RES) | | |
550 | BIT(PERF_CSTATE_PKG_C6_RES) | | |
551 | BIT(PERF_CSTATE_PKG_C7_RES) | | |
552 | BIT(PERF_CSTATE_PKG_C8_RES) | | |
553 | BIT(PERF_CSTATE_PKG_C9_RES) | | |
554 | BIT(PERF_CSTATE_PKG_C10_RES), | |
555 | }; | |
556 | ||
f1857a24 KL |
557 | static const struct cstate_model icl_cstates __initconst = { |
558 | .core_events = BIT(PERF_CSTATE_CORE_C6_RES) | | |
559 | BIT(PERF_CSTATE_CORE_C7_RES), | |
560 | ||
561 | .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) | | |
562 | BIT(PERF_CSTATE_PKG_C3_RES) | | |
563 | BIT(PERF_CSTATE_PKG_C6_RES) | | |
564 | BIT(PERF_CSTATE_PKG_C7_RES) | | |
565 | BIT(PERF_CSTATE_PKG_C8_RES) | | |
566 | BIT(PERF_CSTATE_PKG_C9_RES) | | |
567 | BIT(PERF_CSTATE_PKG_C10_RES), | |
568 | }; | |
569 | ||
87bf399f ZR |
570 | static const struct cstate_model icx_cstates __initconst = { |
571 | .core_events = BIT(PERF_CSTATE_CORE_C1_RES) | | |
572 | BIT(PERF_CSTATE_CORE_C6_RES), | |
573 | ||
574 | .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) | | |
575 | BIT(PERF_CSTATE_PKG_C6_RES), | |
576 | }; | |
577 | ||
d0ca946b KL |
578 | static const struct cstate_model adl_cstates __initconst = { |
579 | .core_events = BIT(PERF_CSTATE_CORE_C1_RES) | | |
580 | BIT(PERF_CSTATE_CORE_C6_RES) | | |
581 | BIT(PERF_CSTATE_CORE_C7_RES), | |
582 | ||
583 | .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) | | |
584 | BIT(PERF_CSTATE_PKG_C3_RES) | | |
585 | BIT(PERF_CSTATE_PKG_C6_RES) | | |
586 | BIT(PERF_CSTATE_PKG_C7_RES) | | |
587 | BIT(PERF_CSTATE_PKG_C8_RES) | | |
588 | BIT(PERF_CSTATE_PKG_C9_RES) | | |
589 | BIT(PERF_CSTATE_PKG_C10_RES), | |
590 | }; | |
591 | ||
424646ee TG |
592 | static const struct cstate_model slm_cstates __initconst = { |
593 | .core_events = BIT(PERF_CSTATE_CORE_C1_RES) | | |
594 | BIT(PERF_CSTATE_CORE_C6_RES), | |
595 | ||
596 | .pkg_events = BIT(PERF_CSTATE_PKG_C6_RES), | |
597 | .quirks = SLM_PKG_C6_USE_C7_MSR, | |
598 | }; | |
599 | ||
889882bc LO |
600 | |
601 | static const struct cstate_model knl_cstates __initconst = { | |
602 | .core_events = BIT(PERF_CSTATE_CORE_C6_RES), | |
603 | ||
604 | .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) | | |
605 | BIT(PERF_CSTATE_PKG_C3_RES) | | |
606 | BIT(PERF_CSTATE_PKG_C6_RES), | |
607 | .quirks = KNL_CORE_C6_MSR, | |
608 | }; | |
609 | ||
610 | ||
5c10b048 HP |
611 | static const struct cstate_model glm_cstates __initconst = { |
612 | .core_events = BIT(PERF_CSTATE_CORE_C1_RES) | | |
613 | BIT(PERF_CSTATE_CORE_C3_RES) | | |
614 | BIT(PERF_CSTATE_CORE_C6_RES), | |
615 | ||
616 | .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) | | |
617 | BIT(PERF_CSTATE_PKG_C3_RES) | | |
618 | BIT(PERF_CSTATE_PKG_C6_RES) | | |
619 | BIT(PERF_CSTATE_PKG_C10_RES), | |
620 | }; | |
621 | ||
889882bc | 622 | |
424646ee | 623 | static const struct x86_cpu_id intel_cstates_match[] __initconst = { |
ef37219a TG |
624 | X86_MATCH_INTEL_FAM6_MODEL(NEHALEM, &nhm_cstates), |
625 | X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EP, &nhm_cstates), | |
626 | X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EX, &nhm_cstates), | |
424646ee | 627 | |
ef37219a TG |
628 | X86_MATCH_INTEL_FAM6_MODEL(WESTMERE, &nhm_cstates), |
629 | X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EP, &nhm_cstates), | |
630 | X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EX, &nhm_cstates), | |
424646ee | 631 | |
ef37219a TG |
632 | X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE, &snb_cstates), |
633 | X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE_X, &snb_cstates), | |
424646ee | 634 | |
ef37219a TG |
635 | X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE, &snb_cstates), |
636 | X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &snb_cstates), | |
424646ee | 637 | |
ef37219a TG |
638 | X86_MATCH_INTEL_FAM6_MODEL(HASWELL, &snb_cstates), |
639 | X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &snb_cstates), | |
640 | X86_MATCH_INTEL_FAM6_MODEL(HASWELL_G, &snb_cstates), | |
424646ee | 641 | |
ef37219a | 642 | X86_MATCH_INTEL_FAM6_MODEL(HASWELL_L, &hswult_cstates), |
424646ee | 643 | |
ef37219a TG |
644 | X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT, &slm_cstates), |
645 | X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_D, &slm_cstates), | |
646 | X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT, &slm_cstates), | |
424646ee | 647 | |
ef37219a TG |
648 | X86_MATCH_INTEL_FAM6_MODEL(BROADWELL, &snb_cstates), |
649 | X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &snb_cstates), | |
650 | X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, &snb_cstates), | |
651 | X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &snb_cstates), | |
424646ee | 652 | |
ef37219a TG |
653 | X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_L, &snb_cstates), |
654 | X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE, &snb_cstates), | |
655 | X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &snb_cstates), | |
889882bc | 656 | |
ef37219a TG |
657 | X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE_L, &hswult_cstates), |
658 | X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE, &hswult_cstates), | |
659 | X86_MATCH_INTEL_FAM6_MODEL(COMETLAKE_L, &hswult_cstates), | |
660 | X86_MATCH_INTEL_FAM6_MODEL(COMETLAKE, &hswult_cstates), | |
f2029b1e | 661 | |
ef37219a | 662 | X86_MATCH_INTEL_FAM6_MODEL(CANNONLAKE_L, &cnl_cstates), |
1159e094 | 663 | |
ef37219a TG |
664 | X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &knl_cstates), |
665 | X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &knl_cstates), | |
5c10b048 | 666 | |
ef37219a TG |
667 | X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT, &glm_cstates), |
668 | X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_D, &glm_cstates), | |
669 | X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_PLUS, &glm_cstates), | |
670 | X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, &glm_cstates), | |
671 | X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT, &glm_cstates), | |
5b16ef2e | 672 | X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_L, &glm_cstates), |
f08c47d1 | 673 | |
ef37219a TG |
674 | X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_L, &icl_cstates), |
675 | X86_MATCH_INTEL_FAM6_MODEL(ICELAKE, &icl_cstates), | |
87bf399f ZR |
676 | X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, &icx_cstates), |
677 | X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, &icx_cstates), | |
528c9f1d | 678 | X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &icx_cstates), |
87bf399f | 679 | |
ef37219a TG |
680 | X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE_L, &icl_cstates), |
681 | X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE, &icl_cstates), | |
cbea5639 | 682 | X86_MATCH_INTEL_FAM6_MODEL(ROCKETLAKE, &icl_cstates), |
d0ca946b KL |
683 | X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE, &adl_cstates), |
684 | X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, &adl_cstates), | |
2da202aa | 685 | X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE, &adl_cstates), |
424646ee TG |
686 | { }, |
687 | }; | |
688 | MODULE_DEVICE_TABLE(x86cpu, intel_cstates_match); | |
689 | ||
424646ee | 690 | static int __init cstate_probe(const struct cstate_model *cm) |
7ce1346a KL |
691 | { |
692 | /* SLM has different MSR for PKG C6 */ | |
424646ee | 693 | if (cm->quirks & SLM_PKG_C6_USE_C7_MSR) |
7ce1346a | 694 | pkg_msr[PERF_CSTATE_PKG_C6_RES].msr = MSR_PKG_C7_RESIDENCY; |
7ce1346a | 695 | |
889882bc LO |
696 | /* KNL has different MSR for CORE C6 */ |
697 | if (cm->quirks & KNL_CORE_C6_MSR) | |
698 | pkg_msr[PERF_CSTATE_CORE_C6_RES].msr = MSR_KNL_CORE_C6_RESIDENCY; | |
699 | ||
700 | ||
8f2a28c5 JO |
701 | core_msr_mask = perf_msr_probe(core_msr, PERF_CSTATE_CORE_EVENT_MAX, |
702 | true, (void *) &cm->core_events); | |
703 | ||
704 | pkg_msr_mask = perf_msr_probe(pkg_msr, PERF_CSTATE_PKG_EVENT_MAX, | |
705 | true, (void *) &cm->pkg_events); | |
7ce1346a | 706 | |
8f2a28c5 JO |
707 | has_cstate_core = !!core_msr_mask; |
708 | has_cstate_pkg = !!pkg_msr_mask; | |
7ce1346a KL |
709 | |
710 | return (has_cstate_core || has_cstate_pkg) ? 0 : -ENODEV; | |
711 | } | |
712 | ||
c7afba32 | 713 | static inline void cstate_cleanup(void) |
7ce1346a | 714 | { |
834fcd29 TG |
715 | cpuhp_remove_state_nocalls(CPUHP_AP_PERF_X86_CSTATE_ONLINE); |
716 | cpuhp_remove_state_nocalls(CPUHP_AP_PERF_X86_CSTATE_STARTING); | |
717 | ||
d29859e7 TG |
718 | if (has_cstate_core) |
719 | perf_pmu_unregister(&cstate_core_pmu); | |
7ce1346a | 720 | |
d29859e7 TG |
721 | if (has_cstate_pkg) |
722 | perf_pmu_unregister(&cstate_pkg_pmu); | |
7ce1346a KL |
723 | } |
724 | ||
d29859e7 | 725 | static int __init cstate_init(void) |
7ce1346a | 726 | { |
77c34ef1 | 727 | int err; |
d29859e7 | 728 | |
77c34ef1 | 729 | cpuhp_setup_state(CPUHP_AP_PERF_X86_CSTATE_STARTING, |
834fcd29 | 730 | "perf/x86/cstate:starting", cstate_cpu_init, NULL); |
77c34ef1 | 731 | cpuhp_setup_state(CPUHP_AP_PERF_X86_CSTATE_ONLINE, |
834fcd29 | 732 | "perf/x86/cstate:online", NULL, cstate_cpu_exit); |
7ce1346a KL |
733 | |
734 | if (has_cstate_core) { | |
735 | err = perf_pmu_register(&cstate_core_pmu, cstate_core_pmu.name, -1); | |
d29859e7 TG |
736 | if (err) { |
737 | has_cstate_core = false; | |
738 | pr_info("Failed to register cstate core pmu\n"); | |
834fcd29 | 739 | cstate_cleanup(); |
77c34ef1 | 740 | return err; |
d29859e7 | 741 | } |
7ce1346a KL |
742 | } |
743 | ||
744 | if (has_cstate_pkg) { | |
cb63ba0f KL |
745 | if (topology_max_die_per_package() > 1) { |
746 | err = perf_pmu_register(&cstate_pkg_pmu, | |
747 | "cstate_die", -1); | |
748 | } else { | |
749 | err = perf_pmu_register(&cstate_pkg_pmu, | |
750 | cstate_pkg_pmu.name, -1); | |
751 | } | |
d29859e7 TG |
752 | if (err) { |
753 | has_cstate_pkg = false; | |
754 | pr_info("Failed to register cstate pkg pmu\n"); | |
755 | cstate_cleanup(); | |
77c34ef1 | 756 | return err; |
d29859e7 | 757 | } |
7ce1346a | 758 | } |
834fcd29 | 759 | return 0; |
7ce1346a KL |
760 | } |
761 | ||
762 | static int __init cstate_pmu_init(void) | |
763 | { | |
424646ee | 764 | const struct x86_cpu_id *id; |
7ce1346a KL |
765 | int err; |
766 | ||
424646ee | 767 | if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) |
7ce1346a KL |
768 | return -ENODEV; |
769 | ||
424646ee TG |
770 | id = x86_match_cpu(intel_cstates_match); |
771 | if (!id) | |
772 | return -ENODEV; | |
773 | ||
774 | err = cstate_probe((const struct cstate_model *) id->driver_data); | |
7ce1346a KL |
775 | if (err) |
776 | return err; | |
777 | ||
d29859e7 | 778 | return cstate_init(); |
7ce1346a | 779 | } |
c7afba32 TG |
780 | module_init(cstate_pmu_init); |
781 | ||
782 | static void __exit cstate_pmu_exit(void) | |
783 | { | |
c7afba32 | 784 | cstate_cleanup(); |
c7afba32 TG |
785 | } |
786 | module_exit(cstate_pmu_exit); |