arch/x86/kernel/cpu/perf_event_intel_ds.c

   1 #ifdef CONFIG_CPU_SUP_INTEL
   2
   3 /* The maximal number of PEBS events: */
   4 #define MAX_PEBS_EVENTS         4
   5
   6 /* The size of a BTS record in bytes: */
   7 #define BTS_RECORD_SIZE         24
   8
   9 #define BTS_BUFFER_SIZE         (PAGE_SIZE << 4)
  10 #define PEBS_BUFFER_SIZE        PAGE_SIZE
  11
  12 /*
  13  * pebs_record_32 for p4 and core not supported
  14
  15 struct pebs_record_32 {
  16         u32 flags, ip;
  17         u32 ax, bc, cx, dx;
  18         u32 si, di, bp, sp;
  19 };
  20
  21  */
  22
  23 struct pebs_record_core {
  24         u64 flags, ip;
  25         u64 ax, bx, cx, dx;
  26         u64 si, di, bp, sp;
  27         u64 r8,  r9,  r10, r11;
  28         u64 r12, r13, r14, r15;
  29 };
  30
  31 struct pebs_record_nhm {
  32         u64 flags, ip;
  33         u64 ax, bx, cx, dx;
  34         u64 si, di, bp, sp;
  35         u64 r8,  r9,  r10, r11;
  36         u64 r12, r13, r14, r15;
  37         u64 status, dla, dse, lat;
  38 };
  39
  40 /*
  41  * A debug store configuration.
  42  *
  43  * We only support architectures that use 64bit fields.
  44  */
  45 struct debug_store {
  46         u64     bts_buffer_base;
  47         u64     bts_index;
  48         u64     bts_absolute_maximum;
  49         u64     bts_interrupt_threshold;
  50         u64     pebs_buffer_base;
  51         u64     pebs_index;
  52         u64     pebs_absolute_maximum;
  53         u64     pebs_interrupt_threshold;
  54         u64     pebs_event_reset[MAX_PEBS_EVENTS];
  55 };
  56
  57 static void init_debug_store_on_cpu(int cpu)
  58 {
  59         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
  60
  61         if (!ds)
  62                 return;
  63
  64         wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA,
  65                      (u32)((u64)(unsigned long)ds),
  66                      (u32)((u64)(unsigned long)ds >> 32));
  67 }
  68
  69 static void fini_debug_store_on_cpu(int cpu)
  70 {
  71         if (!per_cpu(cpu_hw_events, cpu).ds)
  72                 return;
  73
  74         wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA, 0, 0);
  75 }
  76
  77 static void release_ds_buffers(void)
  78 {
  79         int cpu;
  80
  81         if (!x86_pmu.bts && !x86_pmu.pebs)
  82                 return;
  83
  84         get_online_cpus();
  85
  86         for_each_online_cpu(cpu)
  87                 fini_debug_store_on_cpu(cpu);
  88
  89         for_each_possible_cpu(cpu) {
  90                 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
  91
  92                 if (!ds)
  93                         continue;
  94
  95                 per_cpu(cpu_hw_events, cpu).ds = NULL;
  96
  97                 kfree((void *)(unsigned long)ds->pebs_buffer_base);
  98                 kfree((void *)(unsigned long)ds->bts_buffer_base);
  99                 kfree(ds);
 100         }
 101
 102         put_online_cpus();
 103 }
 104
 105 static int reserve_ds_buffers(void)
 106 {
 107         int cpu, err = 0;
 108
 109         if (!x86_pmu.bts && !x86_pmu.pebs)
 110                 return 0;
 111
 112         get_online_cpus();
 113
 114         for_each_possible_cpu(cpu) {
 115                 struct debug_store *ds;
 116                 void *buffer;
 117                 int max, thresh;
 118
 119                 err = -ENOMEM;
 120                 ds = kzalloc(sizeof(*ds), GFP_KERNEL);
 121                 if (unlikely(!ds))
 122                         break;
 123                 per_cpu(cpu_hw_events, cpu).ds = ds;
 124
 125                 if (x86_pmu.bts) {
 126                         buffer = kzalloc(BTS_BUFFER_SIZE, GFP_KERNEL);
 127                         if (unlikely(!buffer))
 128                                 break;
 129
 130                         max = BTS_BUFFER_SIZE / BTS_RECORD_SIZE;
 131                         thresh = max / 16;
 132
 133                         ds->bts_buffer_base = (u64)(unsigned long)buffer;
 134                         ds->bts_index = ds->bts_buffer_base;
 135                         ds->bts_absolute_maximum = ds->bts_buffer_base +
 136                                 max * BTS_RECORD_SIZE;
 137                         ds->bts_interrupt_threshold = ds->bts_absolute_maximum -
 138                                 thresh * BTS_RECORD_SIZE;
 139                 }
 140
 141                 if (x86_pmu.pebs) {
 142                         buffer = kzalloc(PEBS_BUFFER_SIZE, GFP_KERNEL);
 143                         if (unlikely(!buffer))
 144                                 break;
 145
 146                         max = PEBS_BUFFER_SIZE / x86_pmu.pebs_record_size;
 147
 148                         ds->pebs_buffer_base = (u64)(unsigned long)buffer;
 149                         ds->pebs_index = ds->pebs_buffer_base;
 150                         ds->pebs_absolute_maximum = ds->pebs_buffer_base +
 151                                 max * x86_pmu.pebs_record_size;
 152                         /*
 153                          * Always use single record PEBS
 154                          */
 155                         ds->pebs_interrupt_threshold = ds->pebs_buffer_base +
 156                                 x86_pmu.pebs_record_size;
 157                 }
 158
 159                 err = 0;
 160         }
 161
 162         if (err)
 163                 release_ds_buffers();
 164         else {
 165                 for_each_online_cpu(cpu)
 166                         init_debug_store_on_cpu(cpu);
 167         }
 168
 169         put_online_cpus();
 170
 171         return err;
 172 }
 173
 174 /*
 175  * BTS
 176  */
 177
 178 static struct event_constraint bts_constraint =
 179         EVENT_CONSTRAINT(0, 1ULL << X86_PMC_IDX_FIXED_BTS, 0);
 180
 181 static void intel_pmu_enable_bts(u64 config)
 182 {
 183         unsigned long debugctlmsr;
 184
 185         debugctlmsr = get_debugctlmsr();
 186
 187         debugctlmsr |= DEBUGCTLMSR_TR;
 188         debugctlmsr |= DEBUGCTLMSR_BTS;
 189         debugctlmsr |= DEBUGCTLMSR_BTINT;
 190
 191         if (!(config & ARCH_PERFMON_EVENTSEL_OS))
 192                 debugctlmsr |= DEBUGCTLMSR_BTS_OFF_OS;
 193
 194         if (!(config & ARCH_PERFMON_EVENTSEL_USR))
 195                 debugctlmsr |= DEBUGCTLMSR_BTS_OFF_USR;
 196
 197         update_debugctlmsr(debugctlmsr);
 198 }
 199
 200 static void intel_pmu_disable_bts(void)
 201 {
 202         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 203         unsigned long debugctlmsr;
 204
 205         if (!cpuc->ds)
 206                 return;
 207
 208         debugctlmsr = get_debugctlmsr();
 209
 210         debugctlmsr &=
 211                 ~(DEBUGCTLMSR_TR | DEBUGCTLMSR_BTS | DEBUGCTLMSR_BTINT |
 212                   DEBUGCTLMSR_BTS_OFF_OS | DEBUGCTLMSR_BTS_OFF_USR);
 213
 214         update_debugctlmsr(debugctlmsr);
 215 }
 216
 217 static int intel_pmu_drain_bts_buffer(void)
 218 {
 219         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 220         struct debug_store *ds = cpuc->ds;
 221         struct bts_record {
 222                 u64     from;
 223                 u64     to;
 224                 u64     flags;
 225         };
 226         struct perf_event *event = cpuc->events[X86_PMC_IDX_FIXED_BTS];
 227         struct bts_record *at, *top;
 228         struct perf_output_handle handle;
 229         struct perf_event_header header;
 230         struct perf_sample_data data;
 231         struct pt_regs regs;
 232
 233         if (!event)
 234                 return 0;
 235
 236         if (!ds)
 237                 return 0;
 238
 239         at  = (struct bts_record *)(unsigned long)ds->bts_buffer_base;
 240         top = (struct bts_record *)(unsigned long)ds->bts_index;
 241
 242         if (top <= at)
 243                 return 0;
 244
 245         ds->bts_index = ds->bts_buffer_base;
 246
 247         perf_sample_data_init(&data, 0);
 248         data.period = event->hw.last_period;
 249         regs.ip     = 0;
 250
 251         /*
 252          * Prepare a generic sample, i.e. fill in the invariant fields.
 253          * We will overwrite the from and to address before we output
 254          * the sample.
 255          */
 256         perf_prepare_sample(&header, &data, event, &regs);
 257
 258         if (perf_output_begin(&handle, event, header.size * (top - at), 1, 1))
 259                 return 1;
 260
 261         for (; at < top; at++) {
 262                 data.ip         = at->from;
 263                 data.addr       = at->to;
 264
 265                 perf_output_sample(&handle, &header, &data, event);
 266         }
 267
 268         perf_output_end(&handle);
 269
 270         /* There's new data available. */
 271         event->hw.interrupts++;
 272         event->pending_kill = POLL_IN;
 273         return 1;
 274 }
 275
 276 /*
 277  * PEBS
 278  */
 279
 280 static struct event_constraint intel_core_pebs_events[] = {
 281         PEBS_EVENT_CONSTRAINT(0x00c0, 0x1), /* INSTR_RETIRED.ANY */
 282         PEBS_EVENT_CONSTRAINT(0xfec1, 0x1), /* X87_OPS_RETIRED.ANY */
 283         PEBS_EVENT_CONSTRAINT(0x00c5, 0x1), /* BR_INST_RETIRED.MISPRED */
 284         PEBS_EVENT_CONSTRAINT(0x1fc7, 0x1), /* SIMD_INST_RETURED.ANY */
 285         PEBS_EVENT_CONSTRAINT(0x01cb, 0x1), /* MEM_LOAD_RETIRED.L1D_MISS */
 286         PEBS_EVENT_CONSTRAINT(0x02cb, 0x1), /* MEM_LOAD_RETIRED.L1D_LINE_MISS */
 287         PEBS_EVENT_CONSTRAINT(0x04cb, 0x1), /* MEM_LOAD_RETIRED.L2_MISS */
 288         PEBS_EVENT_CONSTRAINT(0x08cb, 0x1), /* MEM_LOAD_RETIRED.L2_LINE_MISS */
 289         PEBS_EVENT_CONSTRAINT(0x10cb, 0x1), /* MEM_LOAD_RETIRED.DTLB_MISS */
 290         EVENT_CONSTRAINT_END
 291 };
 292
 293 static struct event_constraint intel_nehalem_pebs_events[] = {
 294         PEBS_EVENT_CONSTRAINT(0x00c0, 0xf), /* INSTR_RETIRED.ANY */
 295         PEBS_EVENT_CONSTRAINT(0xfec1, 0xf), /* X87_OPS_RETIRED.ANY */
 296         PEBS_EVENT_CONSTRAINT(0x00c5, 0xf), /* BR_INST_RETIRED.MISPRED */
 297         PEBS_EVENT_CONSTRAINT(0x1fc7, 0xf), /* SIMD_INST_RETURED.ANY */
 298         PEBS_EVENT_CONSTRAINT(0x01cb, 0xf), /* MEM_LOAD_RETIRED.L1D_MISS */
 299         PEBS_EVENT_CONSTRAINT(0x02cb, 0xf), /* MEM_LOAD_RETIRED.L1D_LINE_MISS */
 300         PEBS_EVENT_CONSTRAINT(0x04cb, 0xf), /* MEM_LOAD_RETIRED.L2_MISS */
 301         PEBS_EVENT_CONSTRAINT(0x08cb, 0xf), /* MEM_LOAD_RETIRED.L2_LINE_MISS */
 302         PEBS_EVENT_CONSTRAINT(0x10cb, 0xf), /* MEM_LOAD_RETIRED.DTLB_MISS */
 303         EVENT_CONSTRAINT_END
 304 };
 305
 306 static struct event_constraint *
 307 intel_pebs_constraints(struct perf_event *event)
 308 {
 309         struct event_constraint *c;
 310
 311         if (!event->attr.precise_ip)
 312                 return NULL;
 313
 314         if (x86_pmu.pebs_constraints) {
 315                 for_each_event_constraint(c, x86_pmu.pebs_constraints) {
 316                         if ((event->hw.config & c->cmask) == c->code)
 317                                 return c;
 318                 }
 319         }
 320
 321         return &emptyconstraint;
 322 }
 323
 324 static void intel_pmu_pebs_enable(struct perf_event *event)
 325 {
 326         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 327         struct hw_perf_event *hwc = &event->hw;
 328
 329         hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
 330
 331         cpuc->pebs_enabled |= 1ULL << hwc->idx;
 332         WARN_ON_ONCE(cpuc->enabled);
 333
 334         if (x86_pmu.intel_cap.pebs_trap && event->attr.precise_ip > 1)
 335                 intel_pmu_lbr_enable(event);
 336 }
 337
 338 static void intel_pmu_pebs_disable(struct perf_event *event)
 339 {
 340         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 341         struct hw_perf_event *hwc = &event->hw;
 342
 343         cpuc->pebs_enabled &= ~(1ULL << hwc->idx);
 344         if (cpuc->enabled)
 345                 wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
 346
 347         hwc->config |= ARCH_PERFMON_EVENTSEL_INT;
 348
 349         if (x86_pmu.intel_cap.pebs_trap && event->attr.precise_ip > 1)
 350                 intel_pmu_lbr_disable(event);
 351 }
 352
 353 static void intel_pmu_pebs_enable_all(void)
 354 {
 355         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 356
 357         if (cpuc->pebs_enabled)
 358                 wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
 359 }
 360
 361 static void intel_pmu_pebs_disable_all(void)
 362 {
 363         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 364
 365         if (cpuc->pebs_enabled)
 366                 wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
 367 }
 368
 369 #include <asm/insn.h>
 370
 371 static inline bool kernel_ip(unsigned long ip)
 372 {
 373 #ifdef CONFIG_X86_32
 374         return ip > PAGE_OFFSET;
 375 #else
 376         return (long)ip < 0;
 377 #endif
 378 }
 379
 380 static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs)
 381 {
 382         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 383         unsigned long from = cpuc->lbr_entries[0].from;
 384         unsigned long old_to, to = cpuc->lbr_entries[0].to;
 385         unsigned long ip = regs->ip;
 386
 387         /*
 388          * We don't need to fixup if the PEBS assist is fault like
 389          */
 390         if (!x86_pmu.intel_cap.pebs_trap)
 391                 return 1;
 392
 393         /*
 394          * No LBR entry, no basic block, no rewinding
 395          */
 396         if (!cpuc->lbr_stack.nr || !from || !to)
 397                 return 0;
 398
 399         /*
 400          * Basic blocks should never cross user/kernel boundaries
 401          */
 402         if (kernel_ip(ip) != kernel_ip(to))
 403                 return 0;
 404
 405         /*
 406          * unsigned math, either ip is before the start (impossible) or
 407          * the basic block is larger than 1 page (sanity)
 408          */
 409         if ((ip - to) > PAGE_SIZE)
 410                 return 0;
 411
 412         /*
 413          * We sampled a branch insn, rewind using the LBR stack
 414          */
 415         if (ip == to) {
 416                 regs->ip = from;
 417                 return 1;
 418         }
 419
 420         do {
 421                 struct insn insn;
 422                 u8 buf[MAX_INSN_SIZE];
 423                 void *kaddr;
 424
 425                 old_to = to;
 426                 if (!kernel_ip(ip)) {
 427                         int bytes, size = MAX_INSN_SIZE;
 428
 429                         bytes = copy_from_user_nmi(buf, (void __user *)to, size);
 430                         if (bytes != size)
 431                                 return 0;
 432
 433                         kaddr = buf;
 434                 } else
 435                         kaddr = (void *)to;
 436
 437                 kernel_insn_init(&insn, kaddr);
 438                 insn_get_length(&insn);
 439                 to += insn.length;
 440         } while (to < ip);
 441
 442         if (to == ip) {
 443                 regs->ip = old_to;
 444                 return 1;
 445         }
 446
 447         /*
 448          * Even though we decoded the basic block, the instruction stream
 449          * never matched the given IP, either the TO or the IP got corrupted.
 450          */
 451         return 0;
 452 }
 453
 454 static int intel_pmu_save_and_restart(struct perf_event *event);
 455
 456 static void __intel_pmu_pebs_event(struct perf_event *event,
 457                                    struct pt_regs *iregs, void *__pebs)
 458 {
 459         /*
 460          * We cast to pebs_record_core since that is a subset of
 461          * both formats and we don't use the other fields in this
 462          * routine.
 463          */
 464         struct pebs_record_core *pebs = __pebs;
 465         struct perf_sample_data data;
 466         struct pt_regs regs;
 467
 468         if (!intel_pmu_save_and_restart(event))
 469                 return;
 470
 471         perf_sample_data_init(&data, 0);
 472         data.period = event->hw.last_period;
 473
 474         /*
 475          * We use the interrupt regs as a base because the PEBS record
 476          * does not contain a full regs set, specifically it seems to
 477          * lack segment descriptors, which get used by things like
 478          * user_mode().
 479          *
 480          * In the simple case fix up only the IP and BP,SP regs, for
 481          * PERF_SAMPLE_IP and PERF_SAMPLE_CALLCHAIN to function properly.
 482          * A possible PERF_SAMPLE_REGS will have to transfer all regs.
 483          */
 484         regs = *iregs;
 485         regs.ip = pebs->ip;
 486         regs.bp = pebs->bp;
 487         regs.sp = pebs->sp;
 488
 489         if (event->attr.precise_ip > 1 && intel_pmu_pebs_fixup_ip(&regs))
 490                 regs.flags |= PERF_EFLAGS_EXACT;
 491         else
 492                 regs.flags &= ~PERF_EFLAGS_EXACT;
 493
 494         if (perf_event_overflow(event, 1, &data, &regs))
 495                 x86_pmu_stop(event, 0);
 496 }
 497
 498 static void intel_pmu_drain_pebs_core(struct pt_regs *iregs)
 499 {
 500         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 501         struct debug_store *ds = cpuc->ds;
 502         struct perf_event *event = cpuc->events[0]; /* PMC0 only */
 503         struct pebs_record_core *at, *top;
 504         int n;
 505
 506         if (!ds || !x86_pmu.pebs)
 507                 return;
 508
 509         at  = (struct pebs_record_core *)(unsigned long)ds->pebs_buffer_base;
 510         top = (struct pebs_record_core *)(unsigned long)ds->pebs_index;
 511
 512         /*
 513          * Whatever else happens, drain the thing
 514          */
 515         ds->pebs_index = ds->pebs_buffer_base;
 516
 517         if (!test_bit(0, cpuc->active_mask))
 518                 return;
 519
 520         WARN_ON_ONCE(!event);
 521
 522         if (!event->attr.precise_ip)
 523                 return;
 524
 525         n = top - at;
 526         if (n <= 0)
 527                 return;
 528
 529         /*
 530          * Should not happen, we program the threshold at 1 and do not
 531          * set a reset value.
 532          */
 533         WARN_ON_ONCE(n > 1);
 534         at += n - 1;
 535
 536         __intel_pmu_pebs_event(event, iregs, at);
 537 }
 538
 539 static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs)
 540 {
 541         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 542         struct debug_store *ds = cpuc->ds;
 543         struct pebs_record_nhm *at, *top;
 544         struct perf_event *event = NULL;
 545         u64 status = 0;
 546         int bit, n;
 547
 548         if (!ds || !x86_pmu.pebs)
 549                 return;
 550
 551         at  = (struct pebs_record_nhm *)(unsigned long)ds->pebs_buffer_base;
 552         top = (struct pebs_record_nhm *)(unsigned long)ds->pebs_index;
 553
 554         ds->pebs_index = ds->pebs_buffer_base;
 555
 556         n = top - at;
 557         if (n <= 0)
 558                 return;
 559
 560         /*
 561          * Should not happen, we program the threshold at 1 and do not
 562          * set a reset value.
 563          */
 564         WARN_ON_ONCE(n > MAX_PEBS_EVENTS);
 565
 566         for ( ; at < top; at++) {
 567                 for_each_set_bit(bit, (unsigned long *)&at->status, MAX_PEBS_EVENTS) {
 568                         event = cpuc->events[bit];
 569                         if (!test_bit(bit, cpuc->active_mask))
 570                                 continue;
 571
 572                         WARN_ON_ONCE(!event);
 573
 574                         if (!event->attr.precise_ip)
 575                                 continue;
 576
 577                         if (__test_and_set_bit(bit, (unsigned long *)&status))
 578                                 continue;
 579
 580                         break;
 581                 }
 582
 583                 if (!event || bit >= MAX_PEBS_EVENTS)
 584                         continue;
 585
 586                 __intel_pmu_pebs_event(event, iregs, at);
 587         }
 588 }
 589
 590 /*
 591  * BTS, PEBS probe and setup
 592  */
 593
 594 static void intel_ds_init(void)
 595 {
 596         /*
 597          * No support for 32bit formats
 598          */
 599         if (!boot_cpu_has(X86_FEATURE_DTES64))
 600                 return;
 601
 602         x86_pmu.bts  = boot_cpu_has(X86_FEATURE_BTS);
 603         x86_pmu.pebs = boot_cpu_has(X86_FEATURE_PEBS);
 604         if (x86_pmu.pebs) {
 605                 char pebs_type = x86_pmu.intel_cap.pebs_trap ?  '+' : '-';
 606                 int format = x86_pmu.intel_cap.pebs_format;
 607
 608                 switch (format) {
 609                 case 0:
 610                         printk(KERN_CONT "PEBS fmt0%c, ", pebs_type);
 611                         x86_pmu.pebs_record_size = sizeof(struct pebs_record_core);
 612                         x86_pmu.drain_pebs = intel_pmu_drain_pebs_core;
 613                         x86_pmu.pebs_constraints = intel_core_pebs_events;
 614                         break;
 615
 616                 case 1:
 617                         printk(KERN_CONT "PEBS fmt1%c, ", pebs_type);
 618                         x86_pmu.pebs_record_size = sizeof(struct pebs_record_nhm);
 619                         x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm;
 620                         x86_pmu.pebs_constraints = intel_nehalem_pebs_events;
 621                         break;
 622
 623                 default:
 624                         printk(KERN_CONT "no PEBS fmt%d%c, ", format, pebs_type);
 625                         x86_pmu.pebs = 0;
 626                         break;
 627                 }
 628         }
 629 }
 630
 631 #else /* CONFIG_CPU_SUP_INTEL */
 632
 633 static int reserve_ds_buffers(void)
 634 {
 635         return 0;
 636 }
 637
 638 static void release_ds_buffers(void)
 639 {
 640 }
 641
 642 #endif /* CONFIG_CPU_SUP_INTEL */