arch/x86/events/intel/bts.c

   1 /*
   2  * BTS PMU driver for perf
   3  * Copyright (c) 2013-2014, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  */
  14
  15 #undef DEBUG
  16
  17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  18
  19 #include <linux/bitops.h>
  20 #include <linux/types.h>
  21 #include <linux/slab.h>
  22 #include <linux/debugfs.h>
  23 #include <linux/device.h>
  24 #include <linux/coredump.h>
  25
  26 #include <asm-generic/sizes.h>
  27 #include <asm/perf_event.h>
  28
  29 #include "../perf_event.h"
  30
  31 struct bts_ctx {
  32         struct perf_output_handle       handle;
  33         struct debug_store              ds_back;
  34         int                             state;
  35 };
  36
  37 /* BTS context states: */
  38 enum {
  39         /* no ongoing AUX transactions */
  40         BTS_STATE_STOPPED = 0,
  41         /* AUX transaction is on, BTS tracing is disabled */
  42         BTS_STATE_INACTIVE,
  43         /* AUX transaction is on, BTS tracing is running */
  44         BTS_STATE_ACTIVE,
  45 };
  46
  47 static DEFINE_PER_CPU(struct bts_ctx, bts_ctx);
  48
  49 #define BTS_RECORD_SIZE         24
  50 #define BTS_SAFETY_MARGIN       4080
  51
  52 struct bts_phys {
  53         struct page     *page;
  54         unsigned long   size;
  55         unsigned long   offset;
  56         unsigned long   displacement;
  57 };
  58
  59 struct bts_buffer {
  60         size_t          real_size;      /* multiple of BTS_RECORD_SIZE */
  61         unsigned int    nr_pages;
  62         unsigned int    nr_bufs;
  63         unsigned int    cur_buf;
  64         bool            snapshot;
  65         local_t         data_size;
  66         local_t         head;
  67         unsigned long   end;
  68         void            **data_pages;
  69         struct bts_phys buf[0];
  70 };
  71
  72 struct pmu bts_pmu;
  73
  74 static size_t buf_size(struct page *page)
  75 {
  76         return 1 << (PAGE_SHIFT + page_private(page));
  77 }
  78
  79 static void *
  80 bts_buffer_setup_aux(int cpu, void **pages, int nr_pages, bool overwrite)
  81 {
  82         struct bts_buffer *buf;
  83         struct page *page;
  84         int node = (cpu == -1) ? cpu : cpu_to_node(cpu);
  85         unsigned long offset;
  86         size_t size = nr_pages << PAGE_SHIFT;
  87         int pg, nbuf, pad;
  88
  89         /* count all the high order buffers */
  90         for (pg = 0, nbuf = 0; pg < nr_pages;) {
  91                 page = virt_to_page(pages[pg]);
  92                 if (WARN_ON_ONCE(!PagePrivate(page) && nr_pages > 1))
  93                         return NULL;
  94                 pg += 1 << page_private(page);
  95                 nbuf++;
  96         }
  97
  98         /*
  99          * to avoid interrupts in overwrite mode, only allow one physical
 100          */
 101         if (overwrite && nbuf > 1)
 102                 return NULL;
 103
 104         buf = kzalloc_node(offsetof(struct bts_buffer, buf[nbuf]), GFP_KERNEL, node);
 105         if (!buf)
 106                 return NULL;
 107
 108         buf->nr_pages = nr_pages;
 109         buf->nr_bufs = nbuf;
 110         buf->snapshot = overwrite;
 111         buf->data_pages = pages;
 112         buf->real_size = size - size % BTS_RECORD_SIZE;
 113
 114         for (pg = 0, nbuf = 0, offset = 0, pad = 0; nbuf < buf->nr_bufs; nbuf++) {
 115                 unsigned int __nr_pages;
 116
 117                 page = virt_to_page(pages[pg]);
 118                 __nr_pages = PagePrivate(page) ? 1 << page_private(page) : 1;
 119                 buf->buf[nbuf].page = page;
 120                 buf->buf[nbuf].offset = offset;
 121                 buf->buf[nbuf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0);
 122                 buf->buf[nbuf].size = buf_size(page) - buf->buf[nbuf].displacement;
 123                 pad = buf->buf[nbuf].size % BTS_RECORD_SIZE;
 124                 buf->buf[nbuf].size -= pad;
 125
 126                 pg += __nr_pages;
 127                 offset += __nr_pages << PAGE_SHIFT;
 128         }
 129
 130         return buf;
 131 }
 132
 133 static void bts_buffer_free_aux(void *data)
 134 {
 135         kfree(data);
 136 }
 137
 138 static unsigned long bts_buffer_offset(struct bts_buffer *buf, unsigned int idx)
 139 {
 140         return buf->buf[idx].offset + buf->buf[idx].displacement;
 141 }
 142
 143 static void
 144 bts_config_buffer(struct bts_buffer *buf)
 145 {
 146         int cpu = raw_smp_processor_id();
 147         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
 148         struct bts_phys *phys = &buf->buf[buf->cur_buf];
 149         unsigned long index, thresh = 0, end = phys->size;
 150         struct page *page = phys->page;
 151
 152         index = local_read(&buf->head);
 153
 154         if (!buf->snapshot) {
 155                 if (buf->end < phys->offset + buf_size(page))
 156                         end = buf->end - phys->offset - phys->displacement;
 157
 158                 index -= phys->offset + phys->displacement;
 159
 160                 if (end - index > BTS_SAFETY_MARGIN)
 161                         thresh = end - BTS_SAFETY_MARGIN;
 162                 else if (end - index > BTS_RECORD_SIZE)
 163                         thresh = end - BTS_RECORD_SIZE;
 164                 else
 165                         thresh = end;
 166         }
 167
 168         ds->bts_buffer_base = (u64)(long)page_address(page) + phys->displacement;
 169         ds->bts_index = ds->bts_buffer_base + index;
 170         ds->bts_absolute_maximum = ds->bts_buffer_base + end;
 171         ds->bts_interrupt_threshold = !buf->snapshot
 172                 ? ds->bts_buffer_base + thresh
 173                 : ds->bts_absolute_maximum + BTS_RECORD_SIZE;
 174 }
 175
 176 static void bts_buffer_pad_out(struct bts_phys *phys, unsigned long head)
 177 {
 178         unsigned long index = head - phys->offset;
 179
 180         memset(page_address(phys->page) + index, 0, phys->size - index);
 181 }
 182
 183 static void bts_update(struct bts_ctx *bts)
 184 {
 185         int cpu = raw_smp_processor_id();
 186         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
 187         struct bts_buffer *buf = perf_get_aux(&bts->handle);
 188         unsigned long index = ds->bts_index - ds->bts_buffer_base, old, head;
 189
 190         if (!buf)
 191                 return;
 192
 193         head = index + bts_buffer_offset(buf, buf->cur_buf);
 194         old = local_xchg(&buf->head, head);
 195
 196         if (!buf->snapshot) {
 197                 if (old == head)
 198                         return;
 199
 200                 if (ds->bts_index >= ds->bts_absolute_maximum)
 201                         perf_aux_output_flag(&bts->handle,
 202                                              PERF_AUX_FLAG_TRUNCATED);
 203
 204                 /*
 205                  * old and head are always in the same physical buffer, so we
 206                  * can subtract them to get the data size.
 207                  */
 208                 local_add(head - old, &buf->data_size);
 209         } else {
 210                 local_set(&buf->data_size, head);
 211         }
 212 }
 213
 214 static int
 215 bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle);
 216
 217 /*
 218  * Ordering PMU callbacks wrt themselves and the PMI is done by means
 219  * of bts::state, which:
 220  *  - is set when bts::handle::event is valid, that is, between
 221  *    perf_aux_output_begin() and perf_aux_output_end();
 222  *  - is zero otherwise;
 223  *  - is ordered against bts::handle::event with a compiler barrier.
 224  */
 225
 226 static void __bts_event_start(struct perf_event *event)
 227 {
 228         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 229         struct bts_buffer *buf = perf_get_aux(&bts->handle);
 230         u64 config = 0;
 231
 232         if (!buf->snapshot)
 233                 config |= ARCH_PERFMON_EVENTSEL_INT;
 234         if (!event->attr.exclude_kernel)
 235                 config |= ARCH_PERFMON_EVENTSEL_OS;
 236         if (!event->attr.exclude_user)
 237                 config |= ARCH_PERFMON_EVENTSEL_USR;
 238
 239         bts_config_buffer(buf);
 240
 241         /*
 242          * local barrier to make sure that ds configuration made it
 243          * before we enable BTS and bts::state goes ACTIVE
 244          */
 245         wmb();
 246
 247         /* INACTIVE/STOPPED -> ACTIVE */
 248         WRITE_ONCE(bts->state, BTS_STATE_ACTIVE);
 249
 250         intel_pmu_enable_bts(config);
 251
 252 }
 253
 254 static void bts_event_start(struct perf_event *event, int flags)
 255 {
 256         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 257         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 258         struct bts_buffer *buf;
 259
 260         buf = perf_aux_output_begin(&bts->handle, event);
 261         if (!buf)
 262                 goto fail_stop;
 263
 264         if (bts_buffer_reset(buf, &bts->handle))
 265                 goto fail_end_stop;
 266
 267         bts->ds_back.bts_buffer_base = cpuc->ds->bts_buffer_base;
 268         bts->ds_back.bts_absolute_maximum = cpuc->ds->bts_absolute_maximum;
 269         bts->ds_back.bts_interrupt_threshold = cpuc->ds->bts_interrupt_threshold;
 270
 271         event->hw.itrace_started = 1;
 272         event->hw.state = 0;
 273
 274         __bts_event_start(event);
 275
 276         return;
 277
 278 fail_end_stop:
 279         perf_aux_output_end(&bts->handle, 0);
 280
 281 fail_stop:
 282         event->hw.state = PERF_HES_STOPPED;
 283 }
 284
 285 static void __bts_event_stop(struct perf_event *event, int state)
 286 {
 287         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 288
 289         /* ACTIVE -> INACTIVE(PMI)/STOPPED(->stop()) */
 290         WRITE_ONCE(bts->state, state);
 291
 292         /*
 293          * No extra synchronization is mandated by the documentation to have
 294          * BTS data stores globally visible.
 295          */
 296         intel_pmu_disable_bts();
 297 }
 298
 299 static void bts_event_stop(struct perf_event *event, int flags)
 300 {
 301         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 302         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 303         struct bts_buffer *buf = NULL;
 304         int state = READ_ONCE(bts->state);
 305
 306         if (state == BTS_STATE_ACTIVE)
 307                 __bts_event_stop(event, BTS_STATE_STOPPED);
 308
 309         if (state != BTS_STATE_STOPPED)
 310                 buf = perf_get_aux(&bts->handle);
 311
 312         event->hw.state |= PERF_HES_STOPPED;
 313
 314         if (flags & PERF_EF_UPDATE) {
 315                 bts_update(bts);
 316
 317                 if (buf) {
 318                         if (buf->snapshot)
 319                                 bts->handle.head =
 320                                         local_xchg(&buf->data_size,
 321                                                    buf->nr_pages << PAGE_SHIFT);
 322                         perf_aux_output_end(&bts->handle,
 323                                             local_xchg(&buf->data_size, 0));
 324                 }
 325
 326                 cpuc->ds->bts_index = bts->ds_back.bts_buffer_base;
 327                 cpuc->ds->bts_buffer_base = bts->ds_back.bts_buffer_base;
 328                 cpuc->ds->bts_absolute_maximum = bts->ds_back.bts_absolute_maximum;
 329                 cpuc->ds->bts_interrupt_threshold = bts->ds_back.bts_interrupt_threshold;
 330         }
 331 }
 332
 333 void intel_bts_enable_local(void)
 334 {
 335         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 336         int state = READ_ONCE(bts->state);
 337
 338         /*
 339          * Here we transition from INACTIVE to ACTIVE;
 340          * if we instead are STOPPED from the interrupt handler,
 341          * stay that way. Can't be ACTIVE here though.
 342          */
 343         if (WARN_ON_ONCE(state == BTS_STATE_ACTIVE))
 344                 return;
 345
 346         if (state == BTS_STATE_STOPPED)
 347                 return;
 348
 349         if (bts->handle.event)
 350                 __bts_event_start(bts->handle.event);
 351 }
 352
 353 void intel_bts_disable_local(void)
 354 {
 355         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 356
 357         /*
 358          * Here we transition from ACTIVE to INACTIVE;
 359          * do nothing for STOPPED or INACTIVE.
 360          */
 361         if (READ_ONCE(bts->state) != BTS_STATE_ACTIVE)
 362                 return;
 363
 364         if (bts->handle.event)
 365                 __bts_event_stop(bts->handle.event, BTS_STATE_INACTIVE);
 366 }
 367
 368 static int
 369 bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle)
 370 {
 371         unsigned long head, space, next_space, pad, gap, skip, wakeup;
 372         unsigned int next_buf;
 373         struct bts_phys *phys, *next_phys;
 374         int ret;
 375
 376         if (buf->snapshot)
 377                 return 0;
 378
 379         head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
 380
 381         phys = &buf->buf[buf->cur_buf];
 382         space = phys->offset + phys->displacement + phys->size - head;
 383         pad = space;
 384         if (space > handle->size) {
 385                 space = handle->size;
 386                 space -= space % BTS_RECORD_SIZE;
 387         }
 388         if (space <= BTS_SAFETY_MARGIN) {
 389                 /* See if next phys buffer has more space */
 390                 next_buf = buf->cur_buf + 1;
 391                 if (next_buf >= buf->nr_bufs)
 392                         next_buf = 0;
 393                 next_phys = &buf->buf[next_buf];
 394                 gap = buf_size(phys->page) - phys->displacement - phys->size +
 395                       next_phys->displacement;
 396                 skip = pad + gap;
 397                 if (handle->size >= skip) {
 398                         next_space = next_phys->size;
 399                         if (next_space + skip > handle->size) {
 400                                 next_space = handle->size - skip;
 401                                 next_space -= next_space % BTS_RECORD_SIZE;
 402                         }
 403                         if (next_space > space || !space) {
 404                                 if (pad)
 405                                         bts_buffer_pad_out(phys, head);
 406                                 ret = perf_aux_output_skip(handle, skip);
 407                                 if (ret)
 408                                         return ret;
 409                                 /* Advance to next phys buffer */
 410                                 phys = next_phys;
 411                                 space = next_space;
 412                                 head = phys->offset + phys->displacement;
 413                                 /*
 414                                  * After this, cur_buf and head won't match ds
 415                                  * anymore, so we must not be racing with
 416                                  * bts_update().
 417                                  */
 418                                 buf->cur_buf = next_buf;
 419                                 local_set(&buf->head, head);
 420                         }
 421                 }
 422         }
 423
 424         /* Don't go far beyond wakeup watermark */
 425         wakeup = BTS_SAFETY_MARGIN + BTS_RECORD_SIZE + handle->wakeup -
 426                  handle->head;
 427         if (space > wakeup) {
 428                 space = wakeup;
 429                 space -= space % BTS_RECORD_SIZE;
 430         }
 431
 432         buf->end = head + space;
 433
 434         /*
 435          * If we have no space, the lost notification would have been sent when
 436          * we hit absolute_maximum - see bts_update()
 437          */
 438         if (!space)
 439                 return -ENOSPC;
 440
 441         return 0;
 442 }
 443
 444 int intel_bts_interrupt(void)
 445 {
 446         struct debug_store *ds = this_cpu_ptr(&cpu_hw_events)->ds;
 447         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 448         struct perf_event *event = bts->handle.event;
 449         struct bts_buffer *buf;
 450         s64 old_head;
 451         int err = -ENOSPC, handled = 0;
 452
 453         /*
 454          * The only surefire way of knowing if this NMI is ours is by checking
 455          * the write ptr against the PMI threshold.
 456          */
 457         if (ds && (ds->bts_index >= ds->bts_interrupt_threshold))
 458                 handled = 1;
 459
 460         /*
 461          * this is wrapped in intel_bts_enable_local/intel_bts_disable_local,
 462          * so we can only be INACTIVE or STOPPED
 463          */
 464         if (READ_ONCE(bts->state) == BTS_STATE_STOPPED)
 465                 return handled;
 466
 467         buf = perf_get_aux(&bts->handle);
 468         if (!buf)
 469                 return handled;
 470
 471         /*
 472          * Skip snapshot counters: they don't use the interrupt, but
 473          * there's no other way of telling, because the pointer will
 474          * keep moving
 475          */
 476         if (buf->snapshot)
 477                 return 0;
 478
 479         old_head = local_read(&buf->head);
 480         bts_update(bts);
 481
 482         /* no new data */
 483         if (old_head == local_read(&buf->head))
 484                 return handled;
 485
 486         perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0));
 487
 488         buf = perf_aux_output_begin(&bts->handle, event);
 489         if (buf)
 490                 err = bts_buffer_reset(buf, &bts->handle);
 491
 492         if (err) {
 493                 WRITE_ONCE(bts->state, BTS_STATE_STOPPED);
 494
 495                 if (buf) {
 496                         /*
 497                          * BTS_STATE_STOPPED should be visible before
 498                          * cleared handle::event
 499                          */
 500                         barrier();
 501                         perf_aux_output_end(&bts->handle, 0);
 502                 }
 503         }
 504
 505         return 1;
 506 }
 507
 508 static void bts_event_del(struct perf_event *event, int mode)
 509 {
 510         bts_event_stop(event, PERF_EF_UPDATE);
 511 }
 512
 513 static int bts_event_add(struct perf_event *event, int mode)
 514 {
 515         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 516         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 517         struct hw_perf_event *hwc = &event->hw;
 518
 519         event->hw.state = PERF_HES_STOPPED;
 520
 521         if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))
 522                 return -EBUSY;
 523
 524         if (bts->handle.event)
 525                 return -EBUSY;
 526
 527         if (mode & PERF_EF_START) {
 528                 bts_event_start(event, 0);
 529                 if (hwc->state & PERF_HES_STOPPED)
 530                         return -EINVAL;
 531         }
 532
 533         return 0;
 534 }
 535
 536 static void bts_event_destroy(struct perf_event *event)
 537 {
 538         x86_release_hardware();
 539         x86_del_exclusive(x86_lbr_exclusive_bts);
 540 }
 541
 542 static int bts_event_init(struct perf_event *event)
 543 {
 544         int ret;
 545
 546         if (event->attr.type != bts_pmu.type)
 547                 return -ENOENT;
 548
 549         if (x86_add_exclusive(x86_lbr_exclusive_bts))
 550                 return -EBUSY;
 551
 552         /*
 553          * BTS leaks kernel addresses even when CPL0 tracing is
 554          * disabled, so disallow intel_bts driver for unprivileged
 555          * users on paranoid systems since it provides trace data
 556          * to the user in a zero-copy fashion.
 557          *
 558          * Note that the default paranoia setting permits unprivileged
 559          * users to profile the kernel.
 560          */
 561         if (event->attr.exclude_kernel && perf_paranoid_kernel() &&
 562             !capable(CAP_SYS_ADMIN))
 563                 return -EACCES;
 564
 565         ret = x86_reserve_hardware();
 566         if (ret) {
 567                 x86_del_exclusive(x86_lbr_exclusive_bts);
 568                 return ret;
 569         }
 570
 571         event->destroy = bts_event_destroy;
 572
 573         return 0;
 574 }
 575
 576 static void bts_event_read(struct perf_event *event)
 577 {
 578 }
 579
 580 static __init int bts_init(void)
 581 {
 582         if (!boot_cpu_has(X86_FEATURE_DTES64) || !x86_pmu.bts)
 583                 return -ENODEV;
 584
 585         bts_pmu.capabilities    = PERF_PMU_CAP_AUX_NO_SG | PERF_PMU_CAP_ITRACE |
 586                                   PERF_PMU_CAP_EXCLUSIVE;
 587         bts_pmu.task_ctx_nr     = perf_sw_context;
 588         bts_pmu.event_init      = bts_event_init;
 589         bts_pmu.add             = bts_event_add;
 590         bts_pmu.del             = bts_event_del;
 591         bts_pmu.start           = bts_event_start;
 592         bts_pmu.stop            = bts_event_stop;
 593         bts_pmu.read            = bts_event_read;
 594         bts_pmu.setup_aux       = bts_buffer_setup_aux;
 595         bts_pmu.free_aux        = bts_buffer_free_aux;
 596
 597         return perf_pmu_register(&bts_pmu, "intel_bts", -1);
 598 }
 599 arch_initcall(bts_init);