]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
b7b7c782 | 2 | #include <linux/perf_event.h> |
dde5e720 | 3 | #include <linux/sysfs.h> |
06ce6e9b | 4 | #include <linux/nospec.h> |
353bf605 | 5 | #include <asm/intel-family.h> |
dde5e720 | 6 | #include "probe.h" |
b7b7c782 AL |
7 | |
8 | enum perf_msr_id { | |
9 | PERF_MSR_TSC = 0, | |
10 | PERF_MSR_APERF = 1, | |
11 | PERF_MSR_MPERF = 2, | |
12 | PERF_MSR_PPERF = 3, | |
13 | PERF_MSR_SMI = 4, | |
8a224261 | 14 | PERF_MSR_PTSC = 5, |
aaf24884 | 15 | PERF_MSR_IRPERF = 6, |
9ae21dd6 | 16 | PERF_MSR_THERM = 7, |
b7b7c782 AL |
17 | PERF_MSR_EVENT_MAX, |
18 | }; | |
19 | ||
dde5e720 | 20 | static bool test_aperfmperf(int idx, void *data) |
19b3340c PZ |
21 | { |
22 | return boot_cpu_has(X86_FEATURE_APERFMPERF); | |
23 | } | |
24 | ||
dde5e720 | 25 | static bool test_ptsc(int idx, void *data) |
8a224261 HR |
26 | { |
27 | return boot_cpu_has(X86_FEATURE_PTSC); | |
28 | } | |
29 | ||
dde5e720 | 30 | static bool test_irperf(int idx, void *data) |
aaf24884 HR |
31 | { |
32 | return boot_cpu_has(X86_FEATURE_IRPERF); | |
33 | } | |
34 | ||
dde5e720 | 35 | static bool test_therm_status(int idx, void *data) |
9ae21dd6 SE |
36 | { |
37 | return boot_cpu_has(X86_FEATURE_DTHERM); | |
38 | } | |
39 | ||
dde5e720 | 40 | static bool test_intel(int idx, void *data) |
19b3340c PZ |
41 | { |
42 | if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL || | |
43 | boot_cpu_data.x86 != 6) | |
44 | return false; | |
45 | ||
46 | switch (boot_cpu_data.x86_model) { | |
353bf605 | 47 | case INTEL_FAM6_NEHALEM: |
b325e04e | 48 | case INTEL_FAM6_NEHALEM_G: |
353bf605 DH |
49 | case INTEL_FAM6_NEHALEM_EP: |
50 | case INTEL_FAM6_NEHALEM_EX: | |
19b3340c | 51 | |
353bf605 DH |
52 | case INTEL_FAM6_WESTMERE: |
53 | case INTEL_FAM6_WESTMERE_EP: | |
54 | case INTEL_FAM6_WESTMERE_EX: | |
19b3340c | 55 | |
353bf605 DH |
56 | case INTEL_FAM6_SANDYBRIDGE: |
57 | case INTEL_FAM6_SANDYBRIDGE_X: | |
19b3340c | 58 | |
353bf605 DH |
59 | case INTEL_FAM6_IVYBRIDGE: |
60 | case INTEL_FAM6_IVYBRIDGE_X: | |
19b3340c | 61 | |
353bf605 DH |
62 | case INTEL_FAM6_HASWELL_CORE: |
63 | case INTEL_FAM6_HASWELL_X: | |
64 | case INTEL_FAM6_HASWELL_ULT: | |
65 | case INTEL_FAM6_HASWELL_GT3E: | |
19b3340c | 66 | |
353bf605 DH |
67 | case INTEL_FAM6_BROADWELL_CORE: |
68 | case INTEL_FAM6_BROADWELL_XEON_D: | |
69 | case INTEL_FAM6_BROADWELL_GT3E: | |
70 | case INTEL_FAM6_BROADWELL_X: | |
19b3340c | 71 | |
f2c4db1b PZ |
72 | case INTEL_FAM6_ATOM_SILVERMONT: |
73 | case INTEL_FAM6_ATOM_SILVERMONT_X: | |
353bf605 | 74 | case INTEL_FAM6_ATOM_AIRMONT: |
1aaccc40 KL |
75 | |
76 | case INTEL_FAM6_ATOM_GOLDMONT: | |
f2c4db1b | 77 | case INTEL_FAM6_ATOM_GOLDMONT_X: |
1aaccc40 | 78 | |
f2c4db1b | 79 | case INTEL_FAM6_ATOM_GOLDMONT_PLUS: |
1aaccc40 KL |
80 | |
81 | case INTEL_FAM6_XEON_PHI_KNL: | |
82 | case INTEL_FAM6_XEON_PHI_KNM: | |
19b3340c PZ |
83 | if (idx == PERF_MSR_SMI) |
84 | return true; | |
85 | break; | |
86 | ||
353bf605 DH |
87 | case INTEL_FAM6_SKYLAKE_MOBILE: |
88 | case INTEL_FAM6_SKYLAKE_DESKTOP: | |
5134596c DH |
89 | case INTEL_FAM6_SKYLAKE_X: |
90 | case INTEL_FAM6_KABYLAKE_MOBILE: | |
91 | case INTEL_FAM6_KABYLAKE_DESKTOP: | |
cf50d79a | 92 | case INTEL_FAM6_ICELAKE_MOBILE: |
19b3340c PZ |
93 | if (idx == PERF_MSR_SMI || idx == PERF_MSR_PPERF) |
94 | return true; | |
95 | break; | |
96 | } | |
97 | ||
98 | return false; | |
99 | } | |
100 | ||
dde5e720 JO |
101 | PMU_EVENT_ATTR_STRING(tsc, attr_tsc, "event=0x00" ); |
102 | PMU_EVENT_ATTR_STRING(aperf, attr_aperf, "event=0x01" ); | |
103 | PMU_EVENT_ATTR_STRING(mperf, attr_mperf, "event=0x02" ); | |
104 | PMU_EVENT_ATTR_STRING(pperf, attr_pperf, "event=0x03" ); | |
105 | PMU_EVENT_ATTR_STRING(smi, attr_smi, "event=0x04" ); | |
106 | PMU_EVENT_ATTR_STRING(ptsc, attr_ptsc, "event=0x05" ); | |
107 | PMU_EVENT_ATTR_STRING(irperf, attr_irperf, "event=0x06" ); | |
108 | PMU_EVENT_ATTR_STRING(cpu_thermal_margin, attr_therm, "event=0x07" ); | |
109 | PMU_EVENT_ATTR_STRING(cpu_thermal_margin.snapshot, attr_therm_snap, "1" ); | |
110 | PMU_EVENT_ATTR_STRING(cpu_thermal_margin.unit, attr_therm_unit, "C" ); | |
111 | ||
112 | static unsigned long msr_mask; | |
113 | ||
114 | PMU_EVENT_GROUP(events, aperf); | |
115 | PMU_EVENT_GROUP(events, mperf); | |
116 | PMU_EVENT_GROUP(events, pperf); | |
117 | PMU_EVENT_GROUP(events, smi); | |
118 | PMU_EVENT_GROUP(events, ptsc); | |
119 | PMU_EVENT_GROUP(events, irperf); | |
120 | ||
121 | static struct attribute *attrs_therm[] = { | |
122 | &attr_therm.attr.attr, | |
123 | &attr_therm_snap.attr.attr, | |
124 | &attr_therm_unit.attr.attr, | |
125 | NULL, | |
b7b7c782 AL |
126 | }; |
127 | ||
dde5e720 JO |
128 | static struct attribute_group group_therm = { |
129 | .name = "events", | |
130 | .attrs = attrs_therm, | |
131 | }; | |
b7b7c782 | 132 | |
19b3340c | 133 | static struct perf_msr msr[] = { |
dde5e720 JO |
134 | [PERF_MSR_TSC] = { .no_check = true, }, |
135 | [PERF_MSR_APERF] = { MSR_IA32_APERF, &group_aperf, test_aperfmperf, }, | |
136 | [PERF_MSR_MPERF] = { MSR_IA32_MPERF, &group_mperf, test_aperfmperf, }, | |
137 | [PERF_MSR_PPERF] = { MSR_PPERF, &group_pperf, test_intel, }, | |
138 | [PERF_MSR_SMI] = { MSR_SMI_COUNT, &group_smi, test_intel, }, | |
139 | [PERF_MSR_PTSC] = { MSR_F15H_PTSC, &group_ptsc, test_ptsc, }, | |
140 | [PERF_MSR_IRPERF] = { MSR_F17H_IRPERF, &group_irperf, test_irperf, }, | |
141 | [PERF_MSR_THERM] = { MSR_IA32_THERM_STATUS, &group_therm, test_therm_status, }, | |
19b3340c PZ |
142 | }; |
143 | ||
dde5e720 JO |
144 | static struct attribute *events_attrs[] = { |
145 | &attr_tsc.attr.attr, | |
19b3340c | 146 | NULL, |
b7b7c782 AL |
147 | }; |
148 | ||
149 | static struct attribute_group events_attr_group = { | |
150 | .name = "events", | |
151 | .attrs = events_attrs, | |
152 | }; | |
153 | ||
154 | PMU_FORMAT_ATTR(event, "config:0-63"); | |
155 | static struct attribute *format_attrs[] = { | |
156 | &format_attr_event.attr, | |
157 | NULL, | |
158 | }; | |
159 | static struct attribute_group format_attr_group = { | |
160 | .name = "format", | |
161 | .attrs = format_attrs, | |
162 | }; | |
163 | ||
164 | static const struct attribute_group *attr_groups[] = { | |
165 | &events_attr_group, | |
166 | &format_attr_group, | |
167 | NULL, | |
168 | }; | |
169 | ||
dde5e720 JO |
170 | const struct attribute_group *attr_update[] = { |
171 | &group_aperf, | |
172 | &group_mperf, | |
173 | &group_pperf, | |
174 | &group_smi, | |
175 | &group_ptsc, | |
176 | &group_irperf, | |
177 | &group_therm, | |
178 | NULL, | |
179 | }; | |
180 | ||
b7b7c782 AL |
181 | static int msr_event_init(struct perf_event *event) |
182 | { | |
183 | u64 cfg = event->attr.config; | |
184 | ||
185 | if (event->attr.type != event->pmu->type) | |
186 | return -ENOENT; | |
187 | ||
b7b7c782 | 188 | /* unsupported modes and filters */ |
2ff40250 | 189 | if (event->attr.sample_period) /* no sampling */ |
b7b7c782 AL |
190 | return -EINVAL; |
191 | ||
06ce6e9b PZ |
192 | if (cfg >= PERF_MSR_EVENT_MAX) |
193 | return -EINVAL; | |
194 | ||
195 | cfg = array_index_nospec((unsigned long)cfg, PERF_MSR_EVENT_MAX); | |
196 | ||
dde5e720 | 197 | if (!(msr_mask & (1 << cfg))) |
19b3340c PZ |
198 | return -EINVAL; |
199 | ||
9128d3ed IM |
200 | event->hw.idx = -1; |
201 | event->hw.event_base = msr[cfg].msr; | |
202 | event->hw.config = cfg; | |
b7b7c782 AL |
203 | |
204 | return 0; | |
205 | } | |
206 | ||
207 | static inline u64 msr_read_counter(struct perf_event *event) | |
208 | { | |
209 | u64 now; | |
210 | ||
211 | if (event->hw.event_base) | |
212 | rdmsrl(event->hw.event_base, now); | |
213 | else | |
ea89c065 | 214 | now = rdtsc_ordered(); |
b7b7c782 AL |
215 | |
216 | return now; | |
217 | } | |
ea89c065 | 218 | |
b7b7c782 AL |
219 | static void msr_event_update(struct perf_event *event) |
220 | { | |
221 | u64 prev, now; | |
222 | s64 delta; | |
223 | ||
9128d3ed | 224 | /* Careful, an NMI might modify the previous event value: */ |
b7b7c782 AL |
225 | again: |
226 | prev = local64_read(&event->hw.prev_count); | |
227 | now = msr_read_counter(event); | |
228 | ||
229 | if (local64_cmpxchg(&event->hw.prev_count, prev, now) != prev) | |
230 | goto again; | |
231 | ||
232 | delta = now - prev; | |
9ae21dd6 | 233 | if (unlikely(event->hw.event_base == MSR_SMI_COUNT)) { |
78e3c795 | 234 | delta = sign_extend64(delta, 31); |
9ae21dd6 SE |
235 | local64_add(delta, &event->count); |
236 | } else if (unlikely(event->hw.event_base == MSR_IA32_THERM_STATUS)) { | |
9128d3ed | 237 | /* If valid, extract digital readout, otherwise set to -1: */ |
9ae21dd6 SE |
238 | now = now & (1ULL << 31) ? (now >> 16) & 0x3f : -1; |
239 | local64_set(&event->count, now); | |
9128d3ed | 240 | } else { |
9ae21dd6 | 241 | local64_add(delta, &event->count); |
9128d3ed | 242 | } |
b7b7c782 AL |
243 | } |
244 | ||
245 | static void msr_event_start(struct perf_event *event, int flags) | |
246 | { | |
9128d3ed | 247 | u64 now = msr_read_counter(event); |
b7b7c782 | 248 | |
b7b7c782 AL |
249 | local64_set(&event->hw.prev_count, now); |
250 | } | |
251 | ||
252 | static void msr_event_stop(struct perf_event *event, int flags) | |
253 | { | |
254 | msr_event_update(event); | |
255 | } | |
256 | ||
257 | static void msr_event_del(struct perf_event *event, int flags) | |
258 | { | |
259 | msr_event_stop(event, PERF_EF_UPDATE); | |
260 | } | |
261 | ||
262 | static int msr_event_add(struct perf_event *event, int flags) | |
263 | { | |
264 | if (flags & PERF_EF_START) | |
265 | msr_event_start(event, flags); | |
266 | ||
267 | return 0; | |
268 | } | |
269 | ||
270 | static struct pmu pmu_msr = { | |
271 | .task_ctx_nr = perf_sw_context, | |
272 | .attr_groups = attr_groups, | |
273 | .event_init = msr_event_init, | |
274 | .add = msr_event_add, | |
275 | .del = msr_event_del, | |
276 | .start = msr_event_start, | |
277 | .stop = msr_event_stop, | |
278 | .read = msr_event_update, | |
2ff40250 | 279 | .capabilities = PERF_PMU_CAP_NO_INTERRUPT | PERF_PMU_CAP_NO_EXCLUDE, |
dde5e720 | 280 | .attr_update = attr_update, |
b7b7c782 AL |
281 | }; |
282 | ||
b7b7c782 AL |
283 | static int __init msr_init(void) |
284 | { | |
19b3340c PZ |
285 | if (!boot_cpu_has(X86_FEATURE_TSC)) { |
286 | pr_cont("no MSR PMU driver.\n"); | |
287 | return 0; | |
b7b7c782 AL |
288 | } |
289 | ||
dde5e720 | 290 | msr_mask = perf_msr_probe(msr, PERF_MSR_EVENT_MAX, true, NULL); |
b7b7c782 AL |
291 | |
292 | perf_pmu_register(&pmu_msr, "msr", -1); | |
293 | ||
294 | return 0; | |
295 | } | |
296 | device_initcall(msr_init); |