]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - arch/x86/kernel/cpu/perf_event_intel_uncore.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'perf-core-for-mingo' of git://git.kernel.org/pub/scm/linux/kernel/git...
[mirror_ubuntu-bionic-kernel.git] / arch / x86 / kernel / cpu / perf_event_intel_uncore.c
CommitLineData
087bfbb0
YZ		 1#include "perf_event_intel_uncore.h"
2
3static struct intel_uncore_type *empty_uncore[] = { NULL, };
4static struct intel_uncore_type **msr_uncores = empty_uncore;
14371cce
YZ		 5static struct intel_uncore_type **pci_uncores = empty_uncore;
6/* pci bus to socket mapping */
7static int pcibus_to_physid[256] = { [0 ... 255] = -1, };
8
9static DEFINE_RAW_SPINLOCK(uncore_box_lock);
087bfbb0
YZ		 10
11/* mask of cpus that collect uncore events */
12static cpumask_t uncore_cpu_mask;
13
14/* constraint for the fixed counter */
15static struct event_constraint constraint_fixed =
16 EVENT_CONSTRAINT(~0ULL, 1 << UNCORE_PMC_IDX_FIXED, ~0ULL);
17
fcde10e9
YZ		 18DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7");
19DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15");
20DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18");
21DEFINE_UNCORE_FORMAT_ATTR(inv, inv, "config:23");
22DEFINE_UNCORE_FORMAT_ATTR(cmask5, cmask, "config:24-28");
23DEFINE_UNCORE_FORMAT_ATTR(cmask8, cmask, "config:24-31");
7c94ee2e
YZ		 24DEFINE_UNCORE_FORMAT_ATTR(thresh8, thresh, "config:24-31");
25DEFINE_UNCORE_FORMAT_ATTR(thresh5, thresh, "config:24-28");
26DEFINE_UNCORE_FORMAT_ATTR(occ_sel, occ_sel, "config:14-15");
27DEFINE_UNCORE_FORMAT_ATTR(occ_invert, occ_invert, "config:30");
28DEFINE_UNCORE_FORMAT_ATTR(occ_edge, occ_edge, "config:14-51");
29
30/* Sandy Bridge-EP uncore support */
31static void snbep_uncore_pci_disable_box(struct intel_uncore_box *box)
32{
33 struct pci_dev *pdev = box->pci_dev;
34 int box_ctl = uncore_pci_box_ctl(box);
35 u32 config;
36
37 pci_read_config_dword(pdev, box_ctl, &config);
38 config |= SNBEP_PMON_BOX_CTL_FRZ;
39 pci_write_config_dword(pdev, box_ctl, config);
40}
41
42static void snbep_uncore_pci_enable_box(struct intel_uncore_box *box)
43{
44 struct pci_dev *pdev = box->pci_dev;
45 int box_ctl = uncore_pci_box_ctl(box);
46 u32 config;
47
48 pci_read_config_dword(pdev, box_ctl, &config);
49 config &= ~SNBEP_PMON_BOX_CTL_FRZ;
50 pci_write_config_dword(pdev, box_ctl, config);
51}
52
53static void snbep_uncore_pci_enable_event(struct intel_uncore_box *box,
54 struct perf_event *event)
55{
56 struct pci_dev *pdev = box->pci_dev;
57 struct hw_perf_event *hwc = &event->hw;
58
59 pci_write_config_dword(pdev, hwc->config_base, hwc->config |
60 SNBEP_PMON_CTL_EN);
61}
62
63static void snbep_uncore_pci_disable_event(struct intel_uncore_box *box,
64 struct perf_event *event)
65{
66 struct pci_dev *pdev = box->pci_dev;
67 struct hw_perf_event *hwc = &event->hw;
68
69 pci_write_config_dword(pdev, hwc->config_base, hwc->config);
70}
71
72static u64 snbep_uncore_pci_read_counter(struct intel_uncore_box *box,
73 struct perf_event *event)
74{
75 struct pci_dev *pdev = box->pci_dev;
76 struct hw_perf_event *hwc = &event->hw;
77 u64 count;
78
79 pci_read_config_dword(pdev, hwc->event_base, (u32 *)&count);
80 pci_read_config_dword(pdev, hwc->event_base + 4, (u32 *)&count + 1);
81 return count;
82}
83
84static void snbep_uncore_pci_init_box(struct intel_uncore_box *box)
85{
86 struct pci_dev *pdev = box->pci_dev;
87 pci_write_config_dword(pdev, SNBEP_PCI_PMON_BOX_CTL,
88 SNBEP_PMON_BOX_CTL_INT);
89}
90
91static void snbep_uncore_msr_disable_box(struct intel_uncore_box *box)
92{
93 u64 config;
94 unsigned msr;
95
96 msr = uncore_msr_box_ctl(box);
97 if (msr) {
98 rdmsrl(msr, config);
99 config |= SNBEP_PMON_BOX_CTL_FRZ;
100 wrmsrl(msr, config);
101 return;
102 }
103}
104
105static void snbep_uncore_msr_enable_box(struct intel_uncore_box *box)
106{
107 u64 config;
108 unsigned msr;
109
110 msr = uncore_msr_box_ctl(box);
111 if (msr) {
112 rdmsrl(msr, config);
113 config &= ~SNBEP_PMON_BOX_CTL_FRZ;
114 wrmsrl(msr, config);
115 return;
116 }
117}
118
119static void snbep_uncore_msr_enable_event(struct intel_uncore_box *box,
120 struct perf_event *event)
121{
122 struct hw_perf_event *hwc = &event->hw;
123
124 wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
125}
126
127static void snbep_uncore_msr_disable_event(struct intel_uncore_box *box,
128 struct perf_event *event)
129{
130 struct hw_perf_event *hwc = &event->hw;
131
132 wrmsrl(hwc->config_base, hwc->config);
133}
134
135static u64 snbep_uncore_msr_read_counter(struct intel_uncore_box *box,
136 struct perf_event *event)
137{
138 struct hw_perf_event *hwc = &event->hw;
139 u64 count;
140
141 rdmsrl(hwc->event_base, count);
142 return count;
143}
144
145static void snbep_uncore_msr_init_box(struct intel_uncore_box *box)
146{
147 unsigned msr = uncore_msr_box_ctl(box);
148 if (msr)
149 wrmsrl(msr, SNBEP_PMON_BOX_CTL_INT);
150}
151
152static struct attribute *snbep_uncore_formats_attr[] = {
153 &format_attr_event.attr,
154 &format_attr_umask.attr,
155 &format_attr_edge.attr,
156 &format_attr_inv.attr,
157 &format_attr_thresh8.attr,
158 NULL,
159};
160
161static struct attribute *snbep_uncore_ubox_formats_attr[] = {
162 &format_attr_event.attr,
163 &format_attr_umask.attr,
164 &format_attr_edge.attr,
165 &format_attr_inv.attr,
166 &format_attr_thresh5.attr,
167 NULL,
168};
169
170static struct attribute *snbep_uncore_pcu_formats_attr[] = {
171 &format_attr_event.attr,
172 &format_attr_occ_sel.attr,
173 &format_attr_edge.attr,
174 &format_attr_inv.attr,
175 &format_attr_thresh5.attr,
176 &format_attr_occ_invert.attr,
177 &format_attr_occ_edge.attr,
178 NULL,
179};
180
181static struct uncore_event_desc snbep_uncore_imc_events[] = {
2992c542
PZ		 182 INTEL_UNCORE_EVENT_DESC(clockticks, "event=0xff,umask=0xff"),
183 INTEL_UNCORE_EVENT_DESC(cas_count_read, "event=0x04,umask=0x03"),
184 INTEL_UNCORE_EVENT_DESC(cas_count_write, "event=0x04,umask=0x0c"),
7c94ee2e
YZ		 185 { /* end: all zeroes */ },
186};
187
188static struct uncore_event_desc snbep_uncore_qpi_events[] = {
2992c542
PZ		 189 INTEL_UNCORE_EVENT_DESC(clockticks, "event=0x14"),
190 INTEL_UNCORE_EVENT_DESC(txl_flits_active, "event=0x00,umask=0x06"),
191 INTEL_UNCORE_EVENT_DESC(drs_data, "event=0x02,umask=0x08"),
192 INTEL_UNCORE_EVENT_DESC(ncb_data, "event=0x03,umask=0x04"),
7c94ee2e
YZ		 193 { /* end: all zeroes */ },
194};
195
196static struct attribute_group snbep_uncore_format_group = {
197 .name = "format",
198 .attrs = snbep_uncore_formats_attr,
199};
200
201static struct attribute_group snbep_uncore_ubox_format_group = {
202 .name = "format",
203 .attrs = snbep_uncore_ubox_formats_attr,
204};
205
206static struct attribute_group snbep_uncore_pcu_format_group = {
207 .name = "format",
208 .attrs = snbep_uncore_pcu_formats_attr,
209};
210
211static struct intel_uncore_ops snbep_uncore_msr_ops = {
212 .init_box = snbep_uncore_msr_init_box,
213 .disable_box = snbep_uncore_msr_disable_box,
214 .enable_box = snbep_uncore_msr_enable_box,
215 .disable_event = snbep_uncore_msr_disable_event,
216 .enable_event = snbep_uncore_msr_enable_event,
217 .read_counter = snbep_uncore_msr_read_counter,
218};
219
220static struct intel_uncore_ops snbep_uncore_pci_ops = {
221 .init_box = snbep_uncore_pci_init_box,
222 .disable_box = snbep_uncore_pci_disable_box,
223 .enable_box = snbep_uncore_pci_enable_box,
224 .disable_event = snbep_uncore_pci_disable_event,
225 .enable_event = snbep_uncore_pci_enable_event,
226 .read_counter = snbep_uncore_pci_read_counter,
227};
228
229static struct event_constraint snbep_uncore_cbox_constraints[] = {
230 UNCORE_EVENT_CONSTRAINT(0x01, 0x1),
231 UNCORE_EVENT_CONSTRAINT(0x02, 0x3),
232 UNCORE_EVENT_CONSTRAINT(0x04, 0x3),
233 UNCORE_EVENT_CONSTRAINT(0x05, 0x3),
234 UNCORE_EVENT_CONSTRAINT(0x07, 0x3),
235 UNCORE_EVENT_CONSTRAINT(0x11, 0x1),
236 UNCORE_EVENT_CONSTRAINT(0x12, 0x3),
237 UNCORE_EVENT_CONSTRAINT(0x13, 0x3),
238 UNCORE_EVENT_CONSTRAINT(0x1b, 0xc),
239 UNCORE_EVENT_CONSTRAINT(0x1c, 0xc),
240 UNCORE_EVENT_CONSTRAINT(0x1d, 0xc),
241 UNCORE_EVENT_CONSTRAINT(0x1e, 0xc),
242 UNCORE_EVENT_CONSTRAINT(0x1f, 0xe),
243 UNCORE_EVENT_CONSTRAINT(0x21, 0x3),
244 UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
245 UNCORE_EVENT_CONSTRAINT(0x31, 0x3),
246 UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
247 UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
248 UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
249 UNCORE_EVENT_CONSTRAINT(0x35, 0x3),
250 UNCORE_EVENT_CONSTRAINT(0x36, 0x1),
251 UNCORE_EVENT_CONSTRAINT(0x37, 0x3),
252 UNCORE_EVENT_CONSTRAINT(0x38, 0x3),
253 UNCORE_EVENT_CONSTRAINT(0x39, 0x3),
254 UNCORE_EVENT_CONSTRAINT(0x3b, 0x1),
255 EVENT_CONSTRAINT_END
256};
257
258static struct event_constraint snbep_uncore_r2pcie_constraints[] = {
259 UNCORE_EVENT_CONSTRAINT(0x10, 0x3),
260 UNCORE_EVENT_CONSTRAINT(0x11, 0x3),
261 UNCORE_EVENT_CONSTRAINT(0x12, 0x1),
262 UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
263 UNCORE_EVENT_CONSTRAINT(0x24, 0x3),
264 UNCORE_EVENT_CONSTRAINT(0x25, 0x3),
265 UNCORE_EVENT_CONSTRAINT(0x26, 0x3),
266 UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
267 UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
268 UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
269 EVENT_CONSTRAINT_END
270};
271
272static struct event_constraint snbep_uncore_r3qpi_constraints[] = {
273 UNCORE_EVENT_CONSTRAINT(0x10, 0x3),
274 UNCORE_EVENT_CONSTRAINT(0x11, 0x3),
275 UNCORE_EVENT_CONSTRAINT(0x12, 0x3),
276 UNCORE_EVENT_CONSTRAINT(0x13, 0x1),
277 UNCORE_EVENT_CONSTRAINT(0x20, 0x3),
278 UNCORE_EVENT_CONSTRAINT(0x21, 0x3),
279 UNCORE_EVENT_CONSTRAINT(0x22, 0x3),
280 UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
281 UNCORE_EVENT_CONSTRAINT(0x24, 0x3),
282 UNCORE_EVENT_CONSTRAINT(0x25, 0x3),
283 UNCORE_EVENT_CONSTRAINT(0x26, 0x3),
284 UNCORE_EVENT_CONSTRAINT(0x30, 0x3),
285 UNCORE_EVENT_CONSTRAINT(0x31, 0x3),
286 UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
287 UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
288 UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
289 UNCORE_EVENT_CONSTRAINT(0x36, 0x3),
290 UNCORE_EVENT_CONSTRAINT(0x37, 0x3),
291 EVENT_CONSTRAINT_END
292};
293
294static struct intel_uncore_type snbep_uncore_ubox = {
295 .name = "ubox",
296 .num_counters = 2,
297 .num_boxes = 1,
298 .perf_ctr_bits = 44,
299 .fixed_ctr_bits = 48,
300 .perf_ctr = SNBEP_U_MSR_PMON_CTR0,
301 .event_ctl = SNBEP_U_MSR_PMON_CTL0,
302 .event_mask = SNBEP_U_MSR_PMON_RAW_EVENT_MASK,
303 .fixed_ctr = SNBEP_U_MSR_PMON_UCLK_FIXED_CTR,
304 .fixed_ctl = SNBEP_U_MSR_PMON_UCLK_FIXED_CTL,
305 .ops = &snbep_uncore_msr_ops,
306 .format_group = &snbep_uncore_ubox_format_group,
307};
308
309static struct intel_uncore_type snbep_uncore_cbox = {
310 .name = "cbox",
311 .num_counters = 4,
312 .num_boxes = 8,
313 .perf_ctr_bits = 44,
314 .event_ctl = SNBEP_C0_MSR_PMON_CTL0,
315 .perf_ctr = SNBEP_C0_MSR_PMON_CTR0,
316 .event_mask = SNBEP_PMON_RAW_EVENT_MASK,
317 .box_ctl = SNBEP_C0_MSR_PMON_BOX_CTL,
318 .msr_offset = SNBEP_CBO_MSR_OFFSET,
319 .constraints = snbep_uncore_cbox_constraints,
320 .ops = &snbep_uncore_msr_ops,
321 .format_group = &snbep_uncore_format_group,
322};
323
324static struct intel_uncore_type snbep_uncore_pcu = {
325 .name = "pcu",
326 .num_counters = 4,
327 .num_boxes = 1,
328 .perf_ctr_bits = 48,
329 .perf_ctr = SNBEP_PCU_MSR_PMON_CTR0,
330 .event_ctl = SNBEP_PCU_MSR_PMON_CTL0,
331 .event_mask = SNBEP_PCU_MSR_PMON_RAW_EVENT_MASK,
332 .box_ctl = SNBEP_PCU_MSR_PMON_BOX_CTL,
333 .ops = &snbep_uncore_msr_ops,
334 .format_group = &snbep_uncore_pcu_format_group,
335};
336
337static struct intel_uncore_type *snbep_msr_uncores[] = {
338 &snbep_uncore_ubox,
339 &snbep_uncore_cbox,
340 &snbep_uncore_pcu,
341 NULL,
342};
343
344#define SNBEP_UNCORE_PCI_COMMON_INIT() \
345 .perf_ctr = SNBEP_PCI_PMON_CTR0, \
346 .event_ctl = SNBEP_PCI_PMON_CTL0, \
347 .event_mask = SNBEP_PMON_RAW_EVENT_MASK, \
348 .box_ctl = SNBEP_PCI_PMON_BOX_CTL, \
349 .ops = &snbep_uncore_pci_ops, \
350 .format_group = &snbep_uncore_format_group
351
352static struct intel_uncore_type snbep_uncore_ha = {
353 .name = "ha",
354 .num_counters = 4,
355 .num_boxes = 1,
356 .perf_ctr_bits = 48,
357 SNBEP_UNCORE_PCI_COMMON_INIT(),
358};
359
360static struct intel_uncore_type snbep_uncore_imc = {
361 .name = "imc",
362 .num_counters = 4,
363 .num_boxes = 4,
364 .perf_ctr_bits = 48,
365 .fixed_ctr_bits = 48,
366 .fixed_ctr = SNBEP_MC_CHy_PCI_PMON_FIXED_CTR,
367 .fixed_ctl = SNBEP_MC_CHy_PCI_PMON_FIXED_CTL,
368 .event_descs = snbep_uncore_imc_events,
369 SNBEP_UNCORE_PCI_COMMON_INIT(),
370};
371
372static struct intel_uncore_type snbep_uncore_qpi = {
373 .name = "qpi",
374 .num_counters = 4,
375 .num_boxes = 2,
376 .perf_ctr_bits = 48,
377 .event_descs = snbep_uncore_qpi_events,
378 SNBEP_UNCORE_PCI_COMMON_INIT(),
379};
380
381
382static struct intel_uncore_type snbep_uncore_r2pcie = {
383 .name = "r2pcie",
384 .num_counters = 4,
385 .num_boxes = 1,
386 .perf_ctr_bits = 44,
387 .constraints = snbep_uncore_r2pcie_constraints,
388 SNBEP_UNCORE_PCI_COMMON_INIT(),
389};
390
391static struct intel_uncore_type snbep_uncore_r3qpi = {
392 .name = "r3qpi",
393 .num_counters = 3,
394 .num_boxes = 2,
395 .perf_ctr_bits = 44,
396 .constraints = snbep_uncore_r3qpi_constraints,
397 SNBEP_UNCORE_PCI_COMMON_INIT(),
398};
399
400static struct intel_uncore_type *snbep_pci_uncores[] = {
401 &snbep_uncore_ha,
402 &snbep_uncore_imc,
403 &snbep_uncore_qpi,
404 &snbep_uncore_r2pcie,
405 &snbep_uncore_r3qpi,
406 NULL,
407};
408
409static DEFINE_PCI_DEVICE_TABLE(snbep_uncore_pci_ids) = {
410 { /* Home Agent */
411 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_HA),
412 .driver_data = (unsigned long)&snbep_uncore_ha,
413 },
414 { /* MC Channel 0 */
415 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC0),
416 .driver_data = (unsigned long)&snbep_uncore_imc,
417 },
418 { /* MC Channel 1 */
419 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC1),
420 .driver_data = (unsigned long)&snbep_uncore_imc,
421 },
422 { /* MC Channel 2 */
423 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC2),
424 .driver_data = (unsigned long)&snbep_uncore_imc,
425 },
426 { /* MC Channel 3 */
427 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC3),
428 .driver_data = (unsigned long)&snbep_uncore_imc,
429 },
430 { /* QPI Port 0 */
431 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_QPI0),
432 .driver_data = (unsigned long)&snbep_uncore_qpi,
433 },
434 { /* QPI Port 1 */
435 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_QPI1),
436 .driver_data = (unsigned long)&snbep_uncore_qpi,
437 },
438 { /* P2PCIe */
439 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R2PCIE),
440 .driver_data = (unsigned long)&snbep_uncore_r2pcie,
441 },
442 { /* R3QPI Link 0 */
443 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R3QPI0),
444 .driver_data = (unsigned long)&snbep_uncore_r3qpi,
445 },
446 { /* R3QPI Link 1 */
447 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R3QPI1),
448 .driver_data = (unsigned long)&snbep_uncore_r3qpi,
449 },
450 { /* end: all zeroes */ }
451};
452
453static struct pci_driver snbep_uncore_pci_driver = {
454 .name = "snbep_uncore",
455 .id_table = snbep_uncore_pci_ids,
456};
457
458/*
459 * build pci bus to socket mapping
460 */
461static void snbep_pci2phy_map_init(void)
462{
463 struct pci_dev *ubox_dev = NULL;
464 int i, bus, nodeid;
465 u32 config;
466
467 while (1) {
468 /* find the UBOX device */
469 ubox_dev = pci_get_device(PCI_VENDOR_ID_INTEL,
470 PCI_DEVICE_ID_INTEL_JAKETOWN_UBOX,
471 ubox_dev);
472 if (!ubox_dev)
473 break;
474 bus = ubox_dev->bus->number;
475 /* get the Node ID of the local register */
476 pci_read_config_dword(ubox_dev, 0x40, &config);
477 nodeid = config;
478 /* get the Node ID mapping */
479 pci_read_config_dword(ubox_dev, 0x54, &config);
480 /*
481 * every three bits in the Node ID mapping register maps
482 * to a particular node.
483 */
484 for (i = 0; i < 8; i++) {
485 if (nodeid == ((config >> (3 * i)) & 0x7)) {
486 pcibus_to_physid[bus] = i;
487 break;
488 }
489 }
490 };
491 return;
492}
493/* end of Sandy Bridge-EP uncore support */
494
fcde10e9
YZ		 495
496/* Sandy Bridge uncore support */
497static void snb_uncore_msr_enable_event(struct intel_uncore_box *box,
498 struct perf_event *event)
499{
500 struct hw_perf_event *hwc = &event->hw;
501
502 if (hwc->idx < UNCORE_PMC_IDX_FIXED)
503 wrmsrl(hwc->config_base, hwc->config | SNB_UNC_CTL_EN);
504 else
505 wrmsrl(hwc->config_base, SNB_UNC_CTL_EN);
506}
507
508static void snb_uncore_msr_disable_event(struct intel_uncore_box *box,
509 struct perf_event *event)
510{
511 wrmsrl(event->hw.config_base, 0);
512}
513
514static u64 snb_uncore_msr_read_counter(struct intel_uncore_box *box,
515 struct perf_event *event)
516{
517 u64 count;
518 rdmsrl(event->hw.event_base, count);
519 return count;
520}
521
522static void snb_uncore_msr_init_box(struct intel_uncore_box *box)
523{
524 if (box->pmu->pmu_idx == 0) {
525 wrmsrl(SNB_UNC_PERF_GLOBAL_CTL,
526 SNB_UNC_GLOBAL_CTL_EN | SNB_UNC_GLOBAL_CTL_CORE_ALL);
527 }
528}
529
530static struct attribute *snb_uncore_formats_attr[] = {
531 &format_attr_event.attr,
532 &format_attr_umask.attr,
533 &format_attr_edge.attr,
534 &format_attr_inv.attr,
535 &format_attr_cmask5.attr,
536 NULL,
537};
538
539static struct attribute_group snb_uncore_format_group = {
540 .name = "format",
541 .attrs = snb_uncore_formats_attr,
542};
543
544static struct intel_uncore_ops snb_uncore_msr_ops = {
545 .init_box = snb_uncore_msr_init_box,
546 .disable_event = snb_uncore_msr_disable_event,
547 .enable_event = snb_uncore_msr_enable_event,
548 .read_counter = snb_uncore_msr_read_counter,
549};
550
551static struct event_constraint snb_uncore_cbox_constraints[] = {
552 UNCORE_EVENT_CONSTRAINT(0x80, 0x1),
553 UNCORE_EVENT_CONSTRAINT(0x83, 0x1),
554 EVENT_CONSTRAINT_END
555};
556
557static struct intel_uncore_type snb_uncore_cbox = {
558 .name = "cbox",
559 .num_counters = 2,
560 .num_boxes = 4,
561 .perf_ctr_bits = 44,
562 .fixed_ctr_bits = 48,
563 .perf_ctr = SNB_UNC_CBO_0_PER_CTR0,
564 .event_ctl = SNB_UNC_CBO_0_PERFEVTSEL0,
565 .fixed_ctr = SNB_UNC_FIXED_CTR,
566 .fixed_ctl = SNB_UNC_FIXED_CTR_CTRL,
567 .single_fixed = 1,
568 .event_mask = SNB_UNC_RAW_EVENT_MASK,
569 .msr_offset = SNB_UNC_CBO_MSR_OFFSET,
570 .constraints = snb_uncore_cbox_constraints,
571 .ops = &snb_uncore_msr_ops,
572 .format_group = &snb_uncore_format_group,
573};
574
575static struct intel_uncore_type *snb_msr_uncores[] = {
576 &snb_uncore_cbox,
577 NULL,
578};
579/* end of Sandy Bridge uncore support */
580
581/* Nehalem uncore support */
582static void nhm_uncore_msr_disable_box(struct intel_uncore_box *box)
583{
584 wrmsrl(NHM_UNC_PERF_GLOBAL_CTL, 0);
585}
586
587static void nhm_uncore_msr_enable_box(struct intel_uncore_box *box)
588{
589 wrmsrl(NHM_UNC_PERF_GLOBAL_CTL,
590 NHM_UNC_GLOBAL_CTL_EN_PC_ALL | NHM_UNC_GLOBAL_CTL_EN_FC);
591}
592
593static void nhm_uncore_msr_enable_event(struct intel_uncore_box *box,
594 struct perf_event *event)
595{
596 struct hw_perf_event *hwc = &event->hw;
597
598 if (hwc->idx < UNCORE_PMC_IDX_FIXED)
599 wrmsrl(hwc->config_base, hwc->config | SNB_UNC_CTL_EN);
600 else
601 wrmsrl(hwc->config_base, NHM_UNC_FIXED_CTR_CTL_EN);
602}
603
604static struct attribute *nhm_uncore_formats_attr[] = {
605 &format_attr_event.attr,
606 &format_attr_umask.attr,
607 &format_attr_edge.attr,
608 &format_attr_inv.attr,
609 &format_attr_cmask8.attr,
610 NULL,
611};
612
613static struct attribute_group nhm_uncore_format_group = {
614 .name = "format",
615 .attrs = nhm_uncore_formats_attr,
616};
617
618static struct uncore_event_desc nhm_uncore_events[] = {
2992c542
PZ		 619 INTEL_UNCORE_EVENT_DESC(clockticks, "event=0xff,umask=0xff"),
620 INTEL_UNCORE_EVENT_DESC(qmc_writes_full_any, "event=0x2f,umask=0x0f"),
621 INTEL_UNCORE_EVENT_DESC(qmc_normal_reads_any, "event=0x2c,umask=0x0f"),
622 INTEL_UNCORE_EVENT_DESC(qhl_request_ioh_reads, "event=0x20,umask=0x01"),
623 INTEL_UNCORE_EVENT_DESC(qhl_request_ioh_writes, "event=0x20,umask=0x02"),
624 INTEL_UNCORE_EVENT_DESC(qhl_request_remote_reads, "event=0x20,umask=0x04"),
625 INTEL_UNCORE_EVENT_DESC(qhl_request_remote_writes, "event=0x20,umask=0x08"),
626 INTEL_UNCORE_EVENT_DESC(qhl_request_local_reads, "event=0x20,umask=0x10"),
627 INTEL_UNCORE_EVENT_DESC(qhl_request_local_writes, "event=0x20,umask=0x20"),
fcde10e9
YZ		 628 { /* end: all zeroes */ },
629};
630
631static struct intel_uncore_ops nhm_uncore_msr_ops = {
632 .disable_box = nhm_uncore_msr_disable_box,
633 .enable_box = nhm_uncore_msr_enable_box,
634 .disable_event = snb_uncore_msr_disable_event,
635 .enable_event = nhm_uncore_msr_enable_event,
636 .read_counter = snb_uncore_msr_read_counter,
637};
638
639static struct intel_uncore_type nhm_uncore = {
640 .name = "",
641 .num_counters = 8,
642 .num_boxes = 1,
643 .perf_ctr_bits = 48,
644 .fixed_ctr_bits = 48,
645 .event_ctl = NHM_UNC_PERFEVTSEL0,
646 .perf_ctr = NHM_UNC_UNCORE_PMC0,
647 .fixed_ctr = NHM_UNC_FIXED_CTR,
648 .fixed_ctl = NHM_UNC_FIXED_CTR_CTRL,
649 .event_mask = NHM_UNC_RAW_EVENT_MASK,
650 .event_descs = nhm_uncore_events,
651 .ops = &nhm_uncore_msr_ops,
652 .format_group = &nhm_uncore_format_group,
653};
654
655static struct intel_uncore_type *nhm_msr_uncores[] = {
656 &nhm_uncore,
657 NULL,
658};
659/* end of Nehalem uncore support */
660
087bfbb0
YZ		 661static void uncore_assign_hw_event(struct intel_uncore_box *box,
662 struct perf_event *event, int idx)
663{
664 struct hw_perf_event *hwc = &event->hw;
665
666 hwc->idx = idx;
667 hwc->last_tag = ++box->tags[idx];
668
669 if (hwc->idx == UNCORE_PMC_IDX_FIXED) {
14371cce
YZ		 670 hwc->event_base = uncore_fixed_ctr(box);
671 hwc->config_base = uncore_fixed_ctl(box);
087bfbb0
YZ		 672 return;
673 }
674
14371cce
YZ		 675 hwc->config_base = uncore_event_ctl(box, hwc->idx);
676 hwc->event_base = uncore_perf_ctr(box, hwc->idx);
087bfbb0
YZ		 677}
678
679static void uncore_perf_event_update(struct intel_uncore_box *box,
680 struct perf_event *event)
681{
682 u64 prev_count, new_count, delta;
683 int shift;
684
685 if (event->hw.idx >= UNCORE_PMC_IDX_FIXED)
686 shift = 64 - uncore_fixed_ctr_bits(box);
687 else
688 shift = 64 - uncore_perf_ctr_bits(box);
689
690 /* the hrtimer might modify the previous event value */
691again:
692 prev_count = local64_read(&event->hw.prev_count);
693 new_count = uncore_read_counter(box, event);
694 if (local64_xchg(&event->hw.prev_count, new_count) != prev_count)
695 goto again;
696
697 delta = (new_count << shift) - (prev_count << shift);
698 delta >>= shift;
699
700 local64_add(delta, &event->count);
701}
702
703/*
704 * The overflow interrupt is unavailable for SandyBridge-EP, is broken
705 * for SandyBridge. So we use hrtimer to periodically poll the counter
706 * to avoid overflow.
707 */
708static enum hrtimer_restart uncore_pmu_hrtimer(struct hrtimer *hrtimer)
709{
710 struct intel_uncore_box *box;
711 unsigned long flags;
712 int bit;
713
714 box = container_of(hrtimer, struct intel_uncore_box, hrtimer);
715 if (!box->n_active || box->cpu != smp_processor_id())
716 return HRTIMER_NORESTART;
717 /*
718 * disable local interrupt to prevent uncore_pmu_event_start/stop
719 * to interrupt the update process
720 */
721 local_irq_save(flags);
722
723 for_each_set_bit(bit, box->active_mask, UNCORE_PMC_IDX_MAX)
724 uncore_perf_event_update(box, box->events[bit]);
725
726 local_irq_restore(flags);
727
728 hrtimer_forward_now(hrtimer, ns_to_ktime(UNCORE_PMU_HRTIMER_INTERVAL));
729 return HRTIMER_RESTART;
730}
731
732static void uncore_pmu_start_hrtimer(struct intel_uncore_box *box)
733{
734 __hrtimer_start_range_ns(&box->hrtimer,
735 ns_to_ktime(UNCORE_PMU_HRTIMER_INTERVAL), 0,
736 HRTIMER_MODE_REL_PINNED, 0);
737}
738
739static void uncore_pmu_cancel_hrtimer(struct intel_uncore_box *box)
740{
741 hrtimer_cancel(&box->hrtimer);
742}
743
744static void uncore_pmu_init_hrtimer(struct intel_uncore_box *box)
745{
746 hrtimer_init(&box->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
747 box->hrtimer.function = uncore_pmu_hrtimer;
748}
749
750struct intel_uncore_box *uncore_alloc_box(int cpu)
751{
752 struct intel_uncore_box *box;
753
754 box = kmalloc_node(sizeof(*box), GFP_KERNEL | __GFP_ZERO,
755 cpu_to_node(cpu));
756 if (!box)
757 return NULL;
758
759 uncore_pmu_init_hrtimer(box);
760 atomic_set(&box->refcnt, 1);
761 box->cpu = -1;
762 box->phys_id = -1;
763
764 return box;
765}
766
767static struct intel_uncore_box *
768uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu)
769{
14371cce
YZ		 770 static struct intel_uncore_box *box;
771
772 box = *per_cpu_ptr(pmu->box, cpu);
773 if (box)
774 return box;
775
776 raw_spin_lock(&uncore_box_lock);
777 list_for_each_entry(box, &pmu->box_list, list) {
778 if (box->phys_id == topology_physical_package_id(cpu)) {
779 atomic_inc(&box->refcnt);
780 *per_cpu_ptr(pmu->box, cpu) = box;
781 break;
782 }
783 }
784 raw_spin_unlock(&uncore_box_lock);
785
087bfbb0
YZ		 786 return *per_cpu_ptr(pmu->box, cpu);
787}
788
789static struct intel_uncore_pmu *uncore_event_to_pmu(struct perf_event *event)
790{
791 return container_of(event->pmu, struct intel_uncore_pmu, pmu);
792}
793
794static struct intel_uncore_box *uncore_event_to_box(struct perf_event *event)
795{
796 /*
797 * perf core schedules event on the basis of cpu, uncore events are
798 * collected by one of the cpus inside a physical package.
799 */
800 return uncore_pmu_to_box(uncore_event_to_pmu(event),
801 smp_processor_id());
802}
803
804static int uncore_collect_events(struct intel_uncore_box *box,
805 struct perf_event *leader, bool dogrp)
806{
807 struct perf_event *event;
808 int n, max_count;
809
810 max_count = box->pmu->type->num_counters;
811 if (box->pmu->type->fixed_ctl)
812 max_count++;
813
814 if (box->n_events >= max_count)
815 return -EINVAL;
816
817 n = box->n_events;
818 box->event_list[n] = leader;
819 n++;
820 if (!dogrp)
821 return n;
822
823 list_for_each_entry(event, &leader->sibling_list, group_entry) {
824 if (event->state <= PERF_EVENT_STATE_OFF)
825 continue;
826
827 if (n >= max_count)
828 return -EINVAL;
829
830 box->event_list[n] = event;
831 n++;
832 }
833 return n;
834}
835
836static struct event_constraint *
837uncore_event_constraint(struct intel_uncore_type *type,
838 struct perf_event *event)
839{
840 struct event_constraint *c;
841
842 if (event->hw.config == ~0ULL)
843 return &constraint_fixed;
844
845 if (type->constraints) {
846 for_each_event_constraint(c, type->constraints) {
847 if ((event->hw.config & c->cmask) == c->code)
848 return c;
849 }
850 }
851
852 return &type->unconstrainted;
853}
854
855static int uncore_assign_events(struct intel_uncore_box *box,
856 int assign[], int n)
857{
858 unsigned long used_mask[BITS_TO_LONGS(UNCORE_PMC_IDX_MAX)];
859 struct event_constraint *c, *constraints[UNCORE_PMC_IDX_MAX];
860 int i, ret, wmin, wmax;
861 struct hw_perf_event *hwc;
862
863 bitmap_zero(used_mask, UNCORE_PMC_IDX_MAX);
864
865 for (i = 0, wmin = UNCORE_PMC_IDX_MAX, wmax = 0; i < n; i++) {
866 c = uncore_event_constraint(box->pmu->type,
867 box->event_list[i]);
868 constraints[i] = c;
869 wmin = min(wmin, c->weight);
870 wmax = max(wmax, c->weight);
871 }
872
873 /* fastpath, try to reuse previous register */
874 for (i = 0; i < n; i++) {
875 hwc = &box->event_list[i]->hw;
876 c = constraints[i];
877
878 /* never assigned */
879 if (hwc->idx == -1)
880 break;
881
882 /* constraint still honored */
883 if (!test_bit(hwc->idx, c->idxmsk))
884 break;
885
886 /* not already used */
887 if (test_bit(hwc->idx, used_mask))
888 break;
889
890 __set_bit(hwc->idx, used_mask);
891 assign[i] = hwc->idx;
892 }
893 if (i == n)
894 return 0;
895
896 /* slow path */
897 ret = perf_assign_events(constraints, n, wmin, wmax, assign);
898 return ret ? -EINVAL : 0;
899}
900
901static void uncore_pmu_event_start(struct perf_event *event, int flags)
902{
903 struct intel_uncore_box *box = uncore_event_to_box(event);
904 int idx = event->hw.idx;
905
906 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
907 return;
908
909 if (WARN_ON_ONCE(idx == -1 || idx >= UNCORE_PMC_IDX_MAX))
910 return;
911
912 event->hw.state = 0;
913 box->events[idx] = event;
914 box->n_active++;
915 __set_bit(idx, box->active_mask);
916
917 local64_set(&event->hw.prev_count, uncore_read_counter(box, event));
918 uncore_enable_event(box, event);
919
920 if (box->n_active == 1) {
921 uncore_enable_box(box);
922 uncore_pmu_start_hrtimer(box);
923 }
924}
925
926static void uncore_pmu_event_stop(struct perf_event *event, int flags)
927{
928 struct intel_uncore_box *box = uncore_event_to_box(event);
929 struct hw_perf_event *hwc = &event->hw;
930
931 if (__test_and_clear_bit(hwc->idx, box->active_mask)) {
932 uncore_disable_event(box, event);
933 box->n_active--;
934 box->events[hwc->idx] = NULL;
935 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
936 hwc->state |= PERF_HES_STOPPED;
937
938 if (box->n_active == 0) {
939 uncore_disable_box(box);
940 uncore_pmu_cancel_hrtimer(box);
941 }
942 }
943
944 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
945 /*
946 * Drain the remaining delta count out of a event
947 * that we are disabling:
948 */
949 uncore_perf_event_update(box, event);
950 hwc->state |= PERF_HES_UPTODATE;
951 }
952}
953
954static int uncore_pmu_event_add(struct perf_event *event, int flags)
955{
956 struct intel_uncore_box *box = uncore_event_to_box(event);
957 struct hw_perf_event *hwc = &event->hw;
958 int assign[UNCORE_PMC_IDX_MAX];
959 int i, n, ret;
960
961 if (!box)
962 return -ENODEV;
963
964 ret = n = uncore_collect_events(box, event, false);
965 if (ret < 0)
966 return ret;
967
968 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
969 if (!(flags & PERF_EF_START))
970 hwc->state |= PERF_HES_ARCH;
971
972 ret = uncore_assign_events(box, assign, n);
973 if (ret)
974 return ret;
975
976 /* save events moving to new counters */
977 for (i = 0; i < box->n_events; i++) {
978 event = box->event_list[i];
979 hwc = &event->hw;
980
981 if (hwc->idx == assign[i] &&
982 hwc->last_tag == box->tags[assign[i]])
983 continue;
984 /*
985 * Ensure we don't accidentally enable a stopped
986 * counter simply because we rescheduled.
987 */
988 if (hwc->state & PERF_HES_STOPPED)
989 hwc->state |= PERF_HES_ARCH;
990
991 uncore_pmu_event_stop(event, PERF_EF_UPDATE);
992 }
993
994 /* reprogram moved events into new counters */
995 for (i = 0; i < n; i++) {
996 event = box->event_list[i];
997 hwc = &event->hw;
998
999 if (hwc->idx != assign[i] ||
1000 hwc->last_tag != box->tags[assign[i]])
1001 uncore_assign_hw_event(box, event, assign[i]);
1002 else if (i < box->n_events)
1003 continue;
1004
1005 if (hwc->state & PERF_HES_ARCH)
1006 continue;
1007
1008 uncore_pmu_event_start(event, 0);
1009 }
1010 box->n_events = n;
1011
1012 return 0;
1013}
1014
1015static void uncore_pmu_event_del(struct perf_event *event, int flags)
1016{
1017 struct intel_uncore_box *box = uncore_event_to_box(event);
1018 int i;
1019
1020 uncore_pmu_event_stop(event, PERF_EF_UPDATE);
1021
1022 for (i = 0; i < box->n_events; i++) {
1023 if (event == box->event_list[i]) {
1024 while (++i < box->n_events)
1025 box->event_list[i - 1] = box->event_list[i];
1026
1027 --box->n_events;
1028 break;
1029 }
1030 }
1031
1032 event->hw.idx = -1;
1033 event->hw.last_tag = ~0ULL;
1034}
1035
1036static void uncore_pmu_event_read(struct perf_event *event)
1037{
1038 struct intel_uncore_box *box = uncore_event_to_box(event);
1039 uncore_perf_event_update(box, event);
1040}
1041
1042/*
1043 * validation ensures the group can be loaded onto the
1044 * PMU if it was the only group available.
1045 */
1046static int uncore_validate_group(struct intel_uncore_pmu *pmu,
1047 struct perf_event *event)
1048{
1049 struct perf_event *leader = event->group_leader;
1050 struct intel_uncore_box *fake_box;
1051 int assign[UNCORE_PMC_IDX_MAX];
1052 int ret = -EINVAL, n;
1053
1054 fake_box = uncore_alloc_box(smp_processor_id());
1055 if (!fake_box)
1056 return -ENOMEM;
1057
1058 fake_box->pmu = pmu;
1059 /*
1060 * the event is not yet connected with its
1061 * siblings therefore we must first collect
1062 * existing siblings, then add the new event
1063 * before we can simulate the scheduling
1064 */
1065 n = uncore_collect_events(fake_box, leader, true);
1066 if (n < 0)
1067 goto out;
1068
1069 fake_box->n_events = n;
1070 n = uncore_collect_events(fake_box, event, false);
1071 if (n < 0)
1072 goto out;
1073
1074 fake_box->n_events = n;
1075
1076 ret = uncore_assign_events(fake_box, assign, n);
1077out:
1078 kfree(fake_box);
1079 return ret;
1080}
1081
1082int uncore_pmu_event_init(struct perf_event *event)
1083{
1084 struct intel_uncore_pmu *pmu;
1085 struct intel_uncore_box *box;
1086 struct hw_perf_event *hwc = &event->hw;
1087 int ret;
1088
1089 if (event->attr.type != event->pmu->type)
1090 return -ENOENT;
1091
1092 pmu = uncore_event_to_pmu(event);
1093 /* no device found for this pmu */
1094 if (pmu->func_id < 0)
1095 return -ENOENT;
1096
1097 /*
1098 * Uncore PMU does measure at all privilege level all the time.
1099 * So it doesn't make sense to specify any exclude bits.
1100 */
1101 if (event->attr.exclude_user || event->attr.exclude_kernel ||
1102 event->attr.exclude_hv || event->attr.exclude_idle)
1103 return -EINVAL;
1104
1105 /* Sampling not supported yet */
1106 if (hwc->sample_period)
1107 return -EINVAL;
1108
1109 /*
1110 * Place all uncore events for a particular physical package
1111 * onto a single cpu
1112 */
1113 if (event->cpu < 0)
1114 return -EINVAL;
1115 box = uncore_pmu_to_box(pmu, event->cpu);
1116 if (!box || box->cpu < 0)
1117 return -EINVAL;
1118 event->cpu = box->cpu;
1119
1120 if (event->attr.config == UNCORE_FIXED_EVENT) {
1121 /* no fixed counter */
1122 if (!pmu->type->fixed_ctl)
1123 return -EINVAL;
1124 /*
1125 * if there is only one fixed counter, only the first pmu
1126 * can access the fixed counter
1127 */
1128 if (pmu->type->single_fixed && pmu->pmu_idx > 0)
1129 return -EINVAL;
1130 hwc->config = ~0ULL;
1131 } else {
1132 hwc->config = event->attr.config & pmu->type->event_mask;
1133 }
1134
1135 event->hw.idx = -1;
1136 event->hw.last_tag = ~0ULL;
1137
1138 if (event->group_leader != event)
1139 ret = uncore_validate_group(pmu, event);
1140 else
1141 ret = 0;
1142
1143 return ret;
1144}
1145
1146static int __init uncore_pmu_register(struct intel_uncore_pmu *pmu)
1147{
1148 int ret;
1149
1150 pmu->pmu = (struct pmu) {
1151 .attr_groups = pmu->type->attr_groups,
1152 .task_ctx_nr = perf_invalid_context,
1153 .event_init = uncore_pmu_event_init,
1154 .add = uncore_pmu_event_add,
1155 .del = uncore_pmu_event_del,
1156 .start = uncore_pmu_event_start,
1157 .stop = uncore_pmu_event_stop,
1158 .read = uncore_pmu_event_read,
1159 };
1160
1161 if (pmu->type->num_boxes == 1) {
1162 if (strlen(pmu->type->name) > 0)
1163 sprintf(pmu->name, "uncore_%s", pmu->type->name);
1164 else
1165 sprintf(pmu->name, "uncore");
1166 } else {
1167 sprintf(pmu->name, "uncore_%s_%d", pmu->type->name,
1168 pmu->pmu_idx);
1169 }
1170
1171 ret = perf_pmu_register(&pmu->pmu, pmu->name, -1);
1172 return ret;
1173}
1174
1175static void __init uncore_type_exit(struct intel_uncore_type *type)
1176{
1177 int i;
1178
1179 for (i = 0; i < type->num_boxes; i++)
1180 free_percpu(type->pmus[i].box);
1181 kfree(type->pmus);
1182 type->pmus = NULL;
1183 kfree(type->attr_groups[1]);
1184 type->attr_groups[1] = NULL;
1185}
1186
14371cce
YZ		 1187static void uncore_types_exit(struct intel_uncore_type **types)
1188{
1189 int i;
1190 for (i = 0; types[i]; i++)
1191 uncore_type_exit(types[i]);
1192}
1193
087bfbb0
YZ		 1194static int __init uncore_type_init(struct intel_uncore_type *type)
1195{
1196 struct intel_uncore_pmu *pmus;
1197 struct attribute_group *events_group;
1198 struct attribute **attrs;
1199 int i, j;
1200
1201 pmus = kzalloc(sizeof(*pmus) * type->num_boxes, GFP_KERNEL);
1202 if (!pmus)
1203 return -ENOMEM;
1204
1205 type->unconstrainted = (struct event_constraint)
1206 __EVENT_CONSTRAINT(0, (1ULL << type->num_counters) - 1,
1207 0, type->num_counters, 0);
1208
1209 for (i = 0; i < type->num_boxes; i++) {
1210 pmus[i].func_id = -1;
1211 pmus[i].pmu_idx = i;
1212 pmus[i].type = type;
14371cce 1213 INIT_LIST_HEAD(&pmus[i].box_list);
087bfbb0
YZ		 1214 pmus[i].box = alloc_percpu(struct intel_uncore_box *);
1215 if (!pmus[i].box)
1216 goto fail;
1217 }
1218
1219 if (type->event_descs) {
1220 i = 0;
1221 while (type->event_descs[i].attr.attr.name)
1222 i++;
1223
1224 events_group = kzalloc(sizeof(struct attribute *) * (i + 1) +
1225 sizeof(*events_group), GFP_KERNEL);
1226 if (!events_group)
1227 goto fail;
1228
1229 attrs = (struct attribute **)(events_group + 1);
1230 events_group->name = "events";
1231 events_group->attrs = attrs;
1232
1233 for (j = 0; j < i; j++)
1234 attrs[j] = &type->event_descs[j].attr.attr;
1235
1236 type->attr_groups[1] = events_group;
1237 }
1238
1239 type->pmus = pmus;
1240 return 0;
1241fail:
1242 uncore_type_exit(type);
1243 return -ENOMEM;
1244}
1245
1246static int __init uncore_types_init(struct intel_uncore_type **types)
1247{
1248 int i, ret;
1249
1250 for (i = 0; types[i]; i++) {
1251 ret = uncore_type_init(types[i]);
1252 if (ret)
1253 goto fail;
1254 }
1255 return 0;
1256fail:
1257 while (--i >= 0)
1258 uncore_type_exit(types[i]);
1259 return ret;
1260}
1261
14371cce
YZ		 1262static struct pci_driver *uncore_pci_driver;
1263static bool pcidrv_registered;
1264
1265/*
1266 * add a pci uncore device
1267 */
1268static int __devinit uncore_pci_add(struct intel_uncore_type *type,
1269 struct pci_dev *pdev)
1270{
1271 struct intel_uncore_pmu *pmu;
1272 struct intel_uncore_box *box;
1273 int i, phys_id;
1274
1275 phys_id = pcibus_to_physid[pdev->bus->number];
1276 if (phys_id < 0)
1277 return -ENODEV;
1278
1279 box = uncore_alloc_box(0);
1280 if (!box)
1281 return -ENOMEM;
1282
1283 /*
1284 * for performance monitoring unit with multiple boxes,
1285 * each box has a different function id.
1286 */
1287 for (i = 0; i < type->num_boxes; i++) {
1288 pmu = &type->pmus[i];
1289 if (pmu->func_id == pdev->devfn)
1290 break;
1291 if (pmu->func_id < 0) {
1292 pmu->func_id = pdev->devfn;
1293 break;
1294 }
1295 pmu = NULL;
1296 }
1297
1298 if (!pmu) {
1299 kfree(box);
1300 return -EINVAL;
1301 }
1302
1303 box->phys_id = phys_id;
1304 box->pci_dev = pdev;
1305 box->pmu = pmu;
1306 uncore_box_init(box);
1307 pci_set_drvdata(pdev, box);
1308
1309 raw_spin_lock(&uncore_box_lock);
1310 list_add_tail(&box->list, &pmu->box_list);
1311 raw_spin_unlock(&uncore_box_lock);
1312
1313 return 0;
1314}
1315
1316static void __devexit uncore_pci_remove(struct pci_dev *pdev)
1317{
1318 struct intel_uncore_box *box = pci_get_drvdata(pdev);
1319 struct intel_uncore_pmu *pmu = box->pmu;
1320 int cpu, phys_id = pcibus_to_physid[pdev->bus->number];
1321
1322 if (WARN_ON_ONCE(phys_id != box->phys_id))
1323 return;
1324
1325 raw_spin_lock(&uncore_box_lock);
1326 list_del(&box->list);
1327 raw_spin_unlock(&uncore_box_lock);
1328
1329 for_each_possible_cpu(cpu) {
1330 if (*per_cpu_ptr(pmu->box, cpu) == box) {
1331 *per_cpu_ptr(pmu->box, cpu) = NULL;
1332 atomic_dec(&box->refcnt);
1333 }
1334 }
1335
1336 WARN_ON_ONCE(atomic_read(&box->refcnt) != 1);
1337 kfree(box);
1338}
1339
1340static int __devinit uncore_pci_probe(struct pci_dev *pdev,
1341 const struct pci_device_id *id)
1342{
1343 struct intel_uncore_type *type;
1344
1345 type = (struct intel_uncore_type *)id->driver_data;
1346 return uncore_pci_add(type, pdev);
1347}
1348
1349static int __init uncore_pci_init(void)
1350{
1351 int ret;
1352
1353 switch (boot_cpu_data.x86_model) {
7c94ee2e
YZ		 1354 case 45: /* Sandy Bridge-EP */
1355 pci_uncores = snbep_pci_uncores;
1356 uncore_pci_driver = &snbep_uncore_pci_driver;
1357 snbep_pci2phy_map_init();
1358 break;
14371cce
YZ		 1359 default:
1360 return 0;
1361 }
1362
1363 ret = uncore_types_init(pci_uncores);
1364 if (ret)
1365 return ret;
1366
1367 uncore_pci_driver->probe = uncore_pci_probe;
1368 uncore_pci_driver->remove = uncore_pci_remove;
1369
1370 ret = pci_register_driver(uncore_pci_driver);
1371 if (ret == 0)
1372 pcidrv_registered = true;
1373 else
1374 uncore_types_exit(pci_uncores);
1375
1376 return ret;
1377}
1378
1379static void __init uncore_pci_exit(void)
1380{
1381 if (pcidrv_registered) {
1382 pcidrv_registered = false;
1383 pci_unregister_driver(uncore_pci_driver);
1384 uncore_types_exit(pci_uncores);
1385 }
1386}
1387
087bfbb0
YZ		 1388static void __cpuinit uncore_cpu_dying(int cpu)
1389{
1390 struct intel_uncore_type *type;
1391 struct intel_uncore_pmu *pmu;
1392 struct intel_uncore_box *box;
1393 int i, j;
1394
1395 for (i = 0; msr_uncores[i]; i++) {
1396 type = msr_uncores[i];
1397 for (j = 0; j < type->num_boxes; j++) {
1398 pmu = &type->pmus[j];
1399 box = *per_cpu_ptr(pmu->box, cpu);
1400 *per_cpu_ptr(pmu->box, cpu) = NULL;
1401 if (box && atomic_dec_and_test(&box->refcnt))
1402 kfree(box);
1403 }
1404 }
1405}
1406
1407static int __cpuinit uncore_cpu_starting(int cpu)
1408{
1409 struct intel_uncore_type *type;
1410 struct intel_uncore_pmu *pmu;
1411 struct intel_uncore_box *box, *exist;
1412 int i, j, k, phys_id;
1413
1414 phys_id = topology_physical_package_id(cpu);
1415
1416 for (i = 0; msr_uncores[i]; i++) {
1417 type = msr_uncores[i];
1418 for (j = 0; j < type->num_boxes; j++) {
1419 pmu = &type->pmus[j];
1420 box = *per_cpu_ptr(pmu->box, cpu);
1421 /* called by uncore_cpu_init? */
1422 if (box && box->phys_id >= 0) {
1423 uncore_box_init(box);
1424 continue;
1425 }
1426
1427 for_each_online_cpu(k) {
1428 exist = *per_cpu_ptr(pmu->box, k);
1429 if (exist && exist->phys_id == phys_id) {
1430 atomic_inc(&exist->refcnt);
1431 *per_cpu_ptr(pmu->box, cpu) = exist;
1432 kfree(box);
1433 box = NULL;
1434 break;
1435 }
1436 }
1437
1438 if (box) {
1439 box->phys_id = phys_id;
1440 uncore_box_init(box);
1441 }
1442 }
1443 }
1444 return 0;
1445}
1446
1447static int __cpuinit uncore_cpu_prepare(int cpu, int phys_id)
1448{
1449 struct intel_uncore_type *type;
1450 struct intel_uncore_pmu *pmu;
1451 struct intel_uncore_box *box;
1452 int i, j;
1453
1454 for (i = 0; msr_uncores[i]; i++) {
1455 type = msr_uncores[i];
1456 for (j = 0; j < type->num_boxes; j++) {
1457 pmu = &type->pmus[j];
1458 if (pmu->func_id < 0)
1459 pmu->func_id = j;
1460
1461 box = uncore_alloc_box(cpu);
1462 if (!box)
1463 return -ENOMEM;
1464
1465 box->pmu = pmu;
1466 box->phys_id = phys_id;
1467 *per_cpu_ptr(pmu->box, cpu) = box;
1468 }
1469 }
1470 return 0;
1471}
1472
1473static void __cpuinit uncore_change_context(struct intel_uncore_type **uncores,
1474 int old_cpu, int new_cpu)
1475{
1476 struct intel_uncore_type *type;
1477 struct intel_uncore_pmu *pmu;
1478 struct intel_uncore_box *box;
1479 int i, j;
1480
1481 for (i = 0; uncores[i]; i++) {
1482 type = uncores[i];
1483 for (j = 0; j < type->num_boxes; j++) {
1484 pmu = &type->pmus[j];
1485 if (old_cpu < 0)
1486 box = uncore_pmu_to_box(pmu, new_cpu);
1487 else
1488 box = uncore_pmu_to_box(pmu, old_cpu);
1489 if (!box)
1490 continue;
1491
1492 if (old_cpu < 0) {
1493 WARN_ON_ONCE(box->cpu != -1);
1494 box->cpu = new_cpu;
1495 continue;
1496 }
1497
1498 WARN_ON_ONCE(box->cpu != old_cpu);
1499 if (new_cpu >= 0) {
1500 uncore_pmu_cancel_hrtimer(box);
1501 perf_pmu_migrate_context(&pmu->pmu,
1502 old_cpu, new_cpu);
1503 box->cpu = new_cpu;
1504 } else {
1505 box->cpu = -1;
1506 }
1507 }
1508 }
1509}
1510
1511static void __cpuinit uncore_event_exit_cpu(int cpu)
1512{
1513 int i, phys_id, target;
1514
1515 /* if exiting cpu is used for collecting uncore events */
1516 if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask))
1517 return;
1518
1519 /* find a new cpu to collect uncore events */
1520 phys_id = topology_physical_package_id(cpu);
1521 target = -1;
1522 for_each_online_cpu(i) {
1523 if (i == cpu)
1524 continue;
1525 if (phys_id == topology_physical_package_id(i)) {
1526 target = i;
1527 break;
1528 }
1529 }
1530
1531 /* migrate uncore events to the new cpu */
1532 if (target >= 0)
1533 cpumask_set_cpu(target, &uncore_cpu_mask);
1534
1535 uncore_change_context(msr_uncores, cpu, target);
14371cce 1536 uncore_change_context(pci_uncores, cpu, target);
087bfbb0
YZ		 1537}
1538
1539static void __cpuinit uncore_event_init_cpu(int cpu)
1540{
1541 int i, phys_id;
1542
1543 phys_id = topology_physical_package_id(cpu);
1544 for_each_cpu(i, &uncore_cpu_mask) {
1545 if (phys_id == topology_physical_package_id(i))
1546 return;
1547 }
1548
1549 cpumask_set_cpu(cpu, &uncore_cpu_mask);
1550
1551 uncore_change_context(msr_uncores, -1, cpu);
14371cce 1552 uncore_change_context(pci_uncores, -1, cpu);
087bfbb0
YZ		 1553}
1554
1555static int __cpuinit uncore_cpu_notifier(struct notifier_block *self,
1556 unsigned long action, void *hcpu)
1557{
1558 unsigned int cpu = (long)hcpu;
1559
1560 /* allocate/free data structure for uncore box */
1561 switch (action & ~CPU_TASKS_FROZEN) {
1562 case CPU_UP_PREPARE:
1563 uncore_cpu_prepare(cpu, -1);
1564 break;
1565 case CPU_STARTING:
1566 uncore_cpu_starting(cpu);
1567 break;
1568 case CPU_UP_CANCELED:
1569 case CPU_DYING:
1570 uncore_cpu_dying(cpu);
1571 break;
1572 default:
1573 break;
1574 }
1575
1576 /* select the cpu that collects uncore events */
1577 switch (action & ~CPU_TASKS_FROZEN) {
1578 case CPU_DOWN_FAILED:
1579 case CPU_STARTING:
1580 uncore_event_init_cpu(cpu);
1581 break;
1582 case CPU_DOWN_PREPARE:
1583 uncore_event_exit_cpu(cpu);
1584 break;
1585 default:
1586 break;
1587 }
1588
1589 return NOTIFY_OK;
1590}
1591
1592static struct notifier_block uncore_cpu_nb __cpuinitdata = {
1593 .notifier_call = uncore_cpu_notifier,
1594 /*
1595 * to migrate uncore events, our notifier should be executed
1596 * before perf core's notifier.
1597 */
1598 .priority = CPU_PRI_PERF + 1,
1599};
1600
1601static void __init uncore_cpu_setup(void *dummy)
1602{
1603 uncore_cpu_starting(smp_processor_id());
1604}
1605
1606static int __init uncore_cpu_init(void)
1607{
1608 int ret, cpu;
1609
1610 switch (boot_cpu_data.x86_model) {
fcde10e9
YZ		 1611 case 26: /* Nehalem */
1612 case 30:
1613 case 37: /* Westmere */
1614 case 44:
1615 msr_uncores = nhm_msr_uncores;
1616 break;
1617 case 42: /* Sandy Bridge */
1618 msr_uncores = snb_msr_uncores;
1619 break;
7c94ee2e
YZ		 1620 case 45: /* Sandy Birdge-EP */
1621 msr_uncores = snbep_msr_uncores;
1622 break;
087bfbb0
YZ		 1623 default:
1624 return 0;
1625 }
1626
1627 ret = uncore_types_init(msr_uncores);
1628 if (ret)
1629 return ret;
1630
1631 get_online_cpus();
1632
1633 for_each_online_cpu(cpu) {
1634 int i, phys_id = topology_physical_package_id(cpu);
1635
1636 for_each_cpu(i, &uncore_cpu_mask) {
1637 if (phys_id == topology_physical_package_id(i)) {
1638 phys_id = -1;
1639 break;
1640 }
1641 }
1642 if (phys_id < 0)
1643 continue;
1644
1645 uncore_cpu_prepare(cpu, phys_id);
1646 uncore_event_init_cpu(cpu);
1647 }
1648 on_each_cpu(uncore_cpu_setup, NULL, 1);
1649
1650 register_cpu_notifier(&uncore_cpu_nb);
1651
1652 put_online_cpus();
1653
1654 return 0;
1655}
1656
1657static int __init uncore_pmus_register(void)
1658{
1659 struct intel_uncore_pmu *pmu;
1660 struct intel_uncore_type *type;
1661 int i, j;
1662
1663 for (i = 0; msr_uncores[i]; i++) {
1664 type = msr_uncores[i];
1665 for (j = 0; j < type->num_boxes; j++) {
1666 pmu = &type->pmus[j];
1667 uncore_pmu_register(pmu);
1668 }
1669 }
1670
14371cce
YZ		 1671 for (i = 0; pci_uncores[i]; i++) {
1672 type = pci_uncores[i];
1673 for (j = 0; j < type->num_boxes; j++) {
1674 pmu = &type->pmus[j];
1675 uncore_pmu_register(pmu);
1676 }
1677 }
1678
087bfbb0
YZ		 1679 return 0;
1680}
1681
1682static int __init intel_uncore_init(void)
1683{
1684 int ret;
1685
1686 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
1687 return -ENODEV;
1688
14371cce 1689 ret = uncore_pci_init();
087bfbb0
YZ		 1690 if (ret)
1691 goto fail;
14371cce
YZ		 1692 ret = uncore_cpu_init();
1693 if (ret) {
1694 uncore_pci_exit();
1695 goto fail;
1696 }
087bfbb0
YZ		 1697
1698 uncore_pmus_register();
1699 return 0;
1700fail:
1701 return ret;
1702}
1703device_initcall(intel_uncore_init);
ubuntu-bionic-kernel mirror