arch/x86/events/amd/iommu.c

   1 /*
   2  * Copyright (C) 2013 Advanced Micro Devices, Inc.
   3  *
   4  * Author: Steven Kinney <Steven.Kinney@amd.com>
   5  * Author: Suravee Suthikulpanit <Suraveee.Suthikulpanit@amd.com>
   6  *
   7  * Perf: amd_iommu - AMD IOMMU Performance Counter PMU implementation
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2 as
  11  * published by the Free Software Foundation.
  12  */
  13
  14 #define pr_fmt(fmt)     "perf/amd_iommu: " fmt
  15
  16 #include <linux/perf_event.h>
  17 #include <linux/init.h>
  18 #include <linux/cpumask.h>
  19 #include <linux/slab.h>
  20
  21 #include "../perf_event.h"
  22 #include "iommu.h"
  23
  24 #define COUNTER_SHIFT           16
  25
  26 /* iommu pmu conf masks */
  27 #define GET_CSOURCE(x)     ((x)->conf & 0xFFULL)
  28 #define GET_DEVID(x)       (((x)->conf >> 8)  & 0xFFFFULL)
  29 #define GET_DOMID(x)       (((x)->conf >> 24) & 0xFFFFULL)
  30 #define GET_PASID(x)       (((x)->conf >> 40) & 0xFFFFFULL)
  31
  32 /* iommu pmu conf1 masks */
  33 #define GET_DEVID_MASK(x)  ((x)->conf1  & 0xFFFFULL)
  34 #define GET_DOMID_MASK(x)  (((x)->conf1 >> 16) & 0xFFFFULL)
  35 #define GET_PASID_MASK(x)  (((x)->conf1 >> 32) & 0xFFFFFULL)
  36
  37 #define IOMMU_NAME_SIZE 16
  38
  39 struct perf_amd_iommu {
  40         struct list_head list;
  41         struct pmu pmu;
  42         struct amd_iommu *iommu;
  43         char name[IOMMU_NAME_SIZE];
  44         u8 max_banks;
  45         u8 max_counters;
  46         u64 cntr_assign_mask;
  47         raw_spinlock_t lock;
  48 };
  49
  50 static LIST_HEAD(perf_amd_iommu_list);
  51
  52 /*---------------------------------------------
  53  * sysfs format attributes
  54  *---------------------------------------------*/
  55 PMU_FORMAT_ATTR(csource,    "config:0-7");
  56 PMU_FORMAT_ATTR(devid,      "config:8-23");
  57 PMU_FORMAT_ATTR(domid,      "config:24-39");
  58 PMU_FORMAT_ATTR(pasid,      "config:40-59");
  59 PMU_FORMAT_ATTR(devid_mask, "config1:0-15");
  60 PMU_FORMAT_ATTR(domid_mask, "config1:16-31");
  61 PMU_FORMAT_ATTR(pasid_mask, "config1:32-51");
  62
  63 static struct attribute *iommu_format_attrs[] = {
  64         &format_attr_csource.attr,
  65         &format_attr_devid.attr,
  66         &format_attr_pasid.attr,
  67         &format_attr_domid.attr,
  68         &format_attr_devid_mask.attr,
  69         &format_attr_pasid_mask.attr,
  70         &format_attr_domid_mask.attr,
  71         NULL,
  72 };
  73
  74 static struct attribute_group amd_iommu_format_group = {
  75         .name = "format",
  76         .attrs = iommu_format_attrs,
  77 };
  78
  79 /*---------------------------------------------
  80  * sysfs events attributes
  81  *---------------------------------------------*/
  82 static struct attribute_group amd_iommu_events_group = {
  83         .name = "events",
  84 };
  85
  86 struct amd_iommu_event_desc {
  87         struct kobj_attribute attr;
  88         const char *event;
  89 };
  90
  91 static ssize_t _iommu_event_show(struct kobject *kobj,
  92                                 struct kobj_attribute *attr, char *buf)
  93 {
  94         struct amd_iommu_event_desc *event =
  95                 container_of(attr, struct amd_iommu_event_desc, attr);
  96         return sprintf(buf, "%s\n", event->event);
  97 }
  98
  99 #define AMD_IOMMU_EVENT_DESC(_name, _event)                     \
 100 {                                                               \
 101         .attr  = __ATTR(_name, 0444, _iommu_event_show, NULL),  \
 102         .event = _event,                                        \
 103 }
 104
 105 static struct amd_iommu_event_desc amd_iommu_v2_event_descs[] = {
 106         AMD_IOMMU_EVENT_DESC(mem_pass_untrans,        "csource=0x01"),
 107         AMD_IOMMU_EVENT_DESC(mem_pass_pretrans,       "csource=0x02"),
 108         AMD_IOMMU_EVENT_DESC(mem_pass_excl,           "csource=0x03"),
 109         AMD_IOMMU_EVENT_DESC(mem_target_abort,        "csource=0x04"),
 110         AMD_IOMMU_EVENT_DESC(mem_trans_total,         "csource=0x05"),
 111         AMD_IOMMU_EVENT_DESC(mem_iommu_tlb_pte_hit,   "csource=0x06"),
 112         AMD_IOMMU_EVENT_DESC(mem_iommu_tlb_pte_mis,   "csource=0x07"),
 113         AMD_IOMMU_EVENT_DESC(mem_iommu_tlb_pde_hit,   "csource=0x08"),
 114         AMD_IOMMU_EVENT_DESC(mem_iommu_tlb_pde_mis,   "csource=0x09"),
 115         AMD_IOMMU_EVENT_DESC(mem_dte_hit,             "csource=0x0a"),
 116         AMD_IOMMU_EVENT_DESC(mem_dte_mis,             "csource=0x0b"),
 117         AMD_IOMMU_EVENT_DESC(page_tbl_read_tot,       "csource=0x0c"),
 118         AMD_IOMMU_EVENT_DESC(page_tbl_read_nst,       "csource=0x0d"),
 119         AMD_IOMMU_EVENT_DESC(page_tbl_read_gst,       "csource=0x0e"),
 120         AMD_IOMMU_EVENT_DESC(int_dte_hit,             "csource=0x0f"),
 121         AMD_IOMMU_EVENT_DESC(int_dte_mis,             "csource=0x10"),
 122         AMD_IOMMU_EVENT_DESC(cmd_processed,           "csource=0x11"),
 123         AMD_IOMMU_EVENT_DESC(cmd_processed_inv,       "csource=0x12"),
 124         AMD_IOMMU_EVENT_DESC(tlb_inv,                 "csource=0x13"),
 125         AMD_IOMMU_EVENT_DESC(ign_rd_wr_mmio_1ff8h,    "csource=0x14"),
 126         AMD_IOMMU_EVENT_DESC(vapic_int_non_guest,     "csource=0x15"),
 127         AMD_IOMMU_EVENT_DESC(vapic_int_guest,         "csource=0x16"),
 128         AMD_IOMMU_EVENT_DESC(smi_recv,                "csource=0x17"),
 129         AMD_IOMMU_EVENT_DESC(smi_blk,                 "csource=0x18"),
 130         { /* end: all zeroes */ },
 131 };
 132
 133 /*---------------------------------------------
 134  * sysfs cpumask attributes
 135  *---------------------------------------------*/
 136 static cpumask_t iommu_cpumask;
 137
 138 static ssize_t _iommu_cpumask_show(struct device *dev,
 139                                    struct device_attribute *attr,
 140                                    char *buf)
 141 {
 142         return cpumap_print_to_pagebuf(true, buf, &iommu_cpumask);
 143 }
 144 static DEVICE_ATTR(cpumask, S_IRUGO, _iommu_cpumask_show, NULL);
 145
 146 static struct attribute *iommu_cpumask_attrs[] = {
 147         &dev_attr_cpumask.attr,
 148         NULL,
 149 };
 150
 151 static struct attribute_group amd_iommu_cpumask_group = {
 152         .attrs = iommu_cpumask_attrs,
 153 };
 154
 155 /*---------------------------------------------*/
 156
 157 static int get_next_avail_iommu_bnk_cntr(struct perf_event *event)
 158 {
 159         struct perf_amd_iommu *piommu = container_of(event->pmu, struct perf_amd_iommu, pmu);
 160         int max_cntrs = piommu->max_counters;
 161         int max_banks = piommu->max_banks;
 162         u32 shift, bank, cntr;
 163         unsigned long flags;
 164         int retval;
 165
 166         raw_spin_lock_irqsave(&piommu->lock, flags);
 167
 168         for (bank = 0, shift = 0; bank < max_banks; bank++) {
 169                 for (cntr = 0; cntr < max_cntrs; cntr++) {
 170                         shift = bank + (bank*3) + cntr;
 171                         if (piommu->cntr_assign_mask & BIT_ULL(shift)) {
 172                                 continue;
 173                         } else {
 174                                 piommu->cntr_assign_mask |= BIT_ULL(shift);
 175                                 event->hw.iommu_bank = bank;
 176                                 event->hw.iommu_cntr = cntr;
 177                                 retval = 0;
 178                                 goto out;
 179                         }
 180                 }
 181         }
 182         retval = -ENOSPC;
 183 out:
 184         raw_spin_unlock_irqrestore(&piommu->lock, flags);
 185         return retval;
 186 }
 187
 188 static int clear_avail_iommu_bnk_cntr(struct perf_amd_iommu *perf_iommu,
 189                                         u8 bank, u8 cntr)
 190 {
 191         unsigned long flags;
 192         int max_banks, max_cntrs;
 193         int shift = 0;
 194
 195         max_banks = perf_iommu->max_banks;
 196         max_cntrs = perf_iommu->max_counters;
 197
 198         if ((bank > max_banks) || (cntr > max_cntrs))
 199                 return -EINVAL;
 200
 201         shift = bank + cntr + (bank*3);
 202
 203         raw_spin_lock_irqsave(&perf_iommu->lock, flags);
 204         perf_iommu->cntr_assign_mask &= ~(1ULL<<shift);
 205         raw_spin_unlock_irqrestore(&perf_iommu->lock, flags);
 206
 207         return 0;
 208 }
 209
 210 static int perf_iommu_event_init(struct perf_event *event)
 211 {
 212         struct hw_perf_event *hwc = &event->hw;
 213
 214         /* test the event attr type check for PMU enumeration */
 215         if (event->attr.type != event->pmu->type)
 216                 return -ENOENT;
 217
 218         /*
 219          * IOMMU counters are shared across all cores.
 220          * Therefore, it does not support per-process mode.
 221          * Also, it does not support event sampling mode.
 222          */
 223         if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
 224                 return -EINVAL;
 225
 226         if (event->cpu < 0)
 227                 return -EINVAL;
 228
 229         /* update the hw_perf_event struct with the iommu config data */
 230         hwc->conf  = event->attr.config;
 231         hwc->conf1 = event->attr.config1;
 232
 233         return 0;
 234 }
 235
 236 static inline struct amd_iommu *perf_event_2_iommu(struct perf_event *ev)
 237 {
 238         return (container_of(ev->pmu, struct perf_amd_iommu, pmu))->iommu;
 239 }
 240
 241 static void perf_iommu_enable_event(struct perf_event *ev)
 242 {
 243         struct amd_iommu *iommu = perf_event_2_iommu(ev);
 244         struct hw_perf_event *hwc = &ev->hw;
 245         u8 bank = hwc->iommu_bank;
 246         u8 cntr = hwc->iommu_cntr;
 247         u64 reg = 0ULL;
 248
 249         reg = GET_CSOURCE(hwc);
 250         amd_iommu_pc_set_reg(iommu, bank, cntr, IOMMU_PC_COUNTER_SRC_REG, &reg);
 251
 252         reg = GET_DEVID_MASK(hwc);
 253         reg = GET_DEVID(hwc) | (reg << 32);
 254         if (reg)
 255                 reg |= BIT(31);
 256         amd_iommu_pc_set_reg(iommu, bank, cntr, IOMMU_PC_DEVID_MATCH_REG, &reg);
 257
 258         reg = GET_PASID_MASK(hwc);
 259         reg = GET_PASID(hwc) | (reg << 32);
 260         if (reg)
 261                 reg |= BIT(31);
 262         amd_iommu_pc_set_reg(iommu, bank, cntr, IOMMU_PC_PASID_MATCH_REG, &reg);
 263
 264         reg = GET_DOMID_MASK(hwc);
 265         reg = GET_DOMID(hwc) | (reg << 32);
 266         if (reg)
 267                 reg |= BIT(31);
 268         amd_iommu_pc_set_reg(iommu, bank, cntr, IOMMU_PC_DOMID_MATCH_REG, &reg);
 269 }
 270
 271 static void perf_iommu_disable_event(struct perf_event *event)
 272 {
 273         struct amd_iommu *iommu = perf_event_2_iommu(event);
 274         struct hw_perf_event *hwc = &event->hw;
 275         u64 reg = 0ULL;
 276
 277         amd_iommu_pc_set_reg(iommu, hwc->iommu_bank, hwc->iommu_cntr,
 278                              IOMMU_PC_COUNTER_SRC_REG, &reg);
 279 }
 280
 281 static void perf_iommu_start(struct perf_event *event, int flags)
 282 {
 283         struct hw_perf_event *hwc = &event->hw;
 284
 285         if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
 286                 return;
 287
 288         WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
 289         hwc->state = 0;
 290
 291         if (flags & PERF_EF_RELOAD) {
 292                 u64 prev_raw_count = local64_read(&hwc->prev_count);
 293                 struct amd_iommu *iommu = perf_event_2_iommu(event);
 294
 295                 amd_iommu_pc_set_reg(iommu, hwc->iommu_bank, hwc->iommu_cntr,
 296                                      IOMMU_PC_COUNTER_REG, &prev_raw_count);
 297         }
 298
 299         perf_iommu_enable_event(event);
 300         perf_event_update_userpage(event);
 301
 302 }
 303
 304 static void perf_iommu_read(struct perf_event *event)
 305 {
 306         u64 count, prev, delta;
 307         struct hw_perf_event *hwc = &event->hw;
 308         struct amd_iommu *iommu = perf_event_2_iommu(event);
 309
 310         if (amd_iommu_pc_get_reg(iommu, hwc->iommu_bank, hwc->iommu_cntr,
 311                                  IOMMU_PC_COUNTER_REG, &count))
 312                 return;
 313
 314         /* IOMMU pc counter register is only 48 bits */
 315         count &= GENMASK_ULL(47, 0);
 316
 317         prev = local64_read(&hwc->prev_count);
 318         if (local64_cmpxchg(&hwc->prev_count, prev, count) != prev)
 319                 return;
 320
 321         /* Handle 48-bit counter overflow */
 322         delta = (count << COUNTER_SHIFT) - (prev << COUNTER_SHIFT);
 323         delta >>= COUNTER_SHIFT;
 324         local64_add(delta, &event->count);
 325 }
 326
 327 static void perf_iommu_stop(struct perf_event *event, int flags)
 328 {
 329         struct hw_perf_event *hwc = &event->hw;
 330
 331         if (hwc->state & PERF_HES_UPTODATE)
 332                 return;
 333
 334         perf_iommu_disable_event(event);
 335         WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
 336         hwc->state |= PERF_HES_STOPPED;
 337
 338         if (hwc->state & PERF_HES_UPTODATE)
 339                 return;
 340
 341         perf_iommu_read(event);
 342         hwc->state |= PERF_HES_UPTODATE;
 343 }
 344
 345 static int perf_iommu_add(struct perf_event *event, int flags)
 346 {
 347         int retval;
 348
 349         event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
 350
 351         /* request an iommu bank/counter */
 352         retval = get_next_avail_iommu_bnk_cntr(event);
 353         if (retval)
 354                 return retval;
 355
 356         if (flags & PERF_EF_START)
 357                 perf_iommu_start(event, PERF_EF_RELOAD);
 358
 359         return 0;
 360 }
 361
 362 static void perf_iommu_del(struct perf_event *event, int flags)
 363 {
 364         struct hw_perf_event *hwc = &event->hw;
 365         struct perf_amd_iommu *perf_iommu =
 366                         container_of(event->pmu, struct perf_amd_iommu, pmu);
 367
 368         perf_iommu_stop(event, PERF_EF_UPDATE);
 369
 370         /* clear the assigned iommu bank/counter */
 371         clear_avail_iommu_bnk_cntr(perf_iommu,
 372                                    hwc->iommu_bank, hwc->iommu_cntr);
 373
 374         perf_event_update_userpage(event);
 375 }
 376
 377 static __init int _init_events_attrs(void)
 378 {
 379         int i = 0, j;
 380         struct attribute **attrs;
 381
 382         while (amd_iommu_v2_event_descs[i].attr.attr.name)
 383                 i++;
 384
 385         attrs = kcalloc(i + 1, sizeof(struct attribute **), GFP_KERNEL);
 386         if (!attrs)
 387                 return -ENOMEM;
 388
 389         for (j = 0; j < i; j++)
 390                 attrs[j] = &amd_iommu_v2_event_descs[j].attr.attr;
 391
 392         amd_iommu_events_group.attrs = attrs;
 393         return 0;
 394 }
 395
 396 static const struct attribute_group *amd_iommu_attr_groups[] = {
 397         &amd_iommu_format_group,
 398         &amd_iommu_cpumask_group,
 399         &amd_iommu_events_group,
 400         NULL,
 401 };
 402
 403 static const struct pmu iommu_pmu __initconst = {
 404         .event_init     = perf_iommu_event_init,
 405         .add            = perf_iommu_add,
 406         .del            = perf_iommu_del,
 407         .start          = perf_iommu_start,
 408         .stop           = perf_iommu_stop,
 409         .read           = perf_iommu_read,
 410         .task_ctx_nr    = perf_invalid_context,
 411         .attr_groups    = amd_iommu_attr_groups,
 412         .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
 413 };
 414
 415 static __init int init_one_iommu(unsigned int idx)
 416 {
 417         struct perf_amd_iommu *perf_iommu;
 418         int ret;
 419
 420         perf_iommu = kzalloc(sizeof(struct perf_amd_iommu), GFP_KERNEL);
 421         if (!perf_iommu)
 422                 return -ENOMEM;
 423
 424         raw_spin_lock_init(&perf_iommu->lock);
 425
 426         perf_iommu->pmu          = iommu_pmu;
 427         perf_iommu->iommu        = get_amd_iommu(idx);
 428         perf_iommu->max_banks    = amd_iommu_pc_get_max_banks(idx);
 429         perf_iommu->max_counters = amd_iommu_pc_get_max_counters(idx);
 430
 431         if (!perf_iommu->iommu ||
 432             !perf_iommu->max_banks ||
 433             !perf_iommu->max_counters) {
 434                 kfree(perf_iommu);
 435                 return -EINVAL;
 436         }
 437
 438         snprintf(perf_iommu->name, IOMMU_NAME_SIZE, "amd_iommu_%u", idx);
 439
 440         ret = perf_pmu_register(&perf_iommu->pmu, perf_iommu->name, -1);
 441         if (!ret) {
 442                 pr_info("Detected AMD IOMMU #%d (%d banks, %d counters/bank).\n",
 443                         idx, perf_iommu->max_banks, perf_iommu->max_counters);
 444                 list_add_tail(&perf_iommu->list, &perf_amd_iommu_list);
 445         } else {
 446                 pr_warn("Error initializing IOMMU %d.\n", idx);
 447                 kfree(perf_iommu);
 448         }
 449         return ret;
 450 }
 451
 452 static __init int amd_iommu_pc_init(void)
 453 {
 454         unsigned int i, cnt = 0;
 455         int ret;
 456
 457         /* Make sure the IOMMU PC resource is available */
 458         if (!amd_iommu_pc_supported())
 459                 return -ENODEV;
 460
 461         ret = _init_events_attrs();
 462         if (ret)
 463                 return ret;
 464
 465         /*
 466          * An IOMMU PMU is specific to an IOMMU, and can function independently.
 467          * So we go through all IOMMUs and ignore the one that fails init
 468          * unless all IOMMU are failing.
 469          */
 470         for (i = 0; i < amd_iommu_get_num_iommus(); i++) {
 471                 ret = init_one_iommu(i);
 472                 if (!ret)
 473                         cnt++;
 474         }
 475
 476         if (!cnt) {
 477                 kfree(amd_iommu_events_group.attrs);
 478                 return -ENODEV;
 479         }
 480
 481         /* Init cpumask attributes to only core 0 */
 482         cpumask_set_cpu(0, &iommu_cpumask);
 483         return 0;
 484 }
 485
 486 device_initcall(amd_iommu_pc_init);