[mirror_ubuntu-zesty-kernel.git] / arch / x86 / events / amd / uncore.c

/*
 * Copyright (C) 2013 Advanced Micro Devices, Inc.
 *
 * Author: Jacob Shin <jacob.shin@amd.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

#include <linux/perf_event.h>
#include <linux/percpu.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/cpu.h>
#include <linux/cpumask.h>

#include <asm/cpufeature.h>
#include <asm/perf_event.h>
#include <asm/msr.h>

#define NUM_COUNTERS_NB		4
#define NUM_COUNTERS_L2		4
#define MAX_COUNTERS		NUM_COUNTERS_NB

#define RDPMC_BASE_NB		6
#define RDPMC_BASE_L2		10

#define COUNTER_SHIFT		16

static HLIST_HEAD(uncore_unused_list);

struct amd_uncore {
	int id;
	int refcnt;
	int cpu;
	int num_counters;
	int rdpmc_base;
	u32 msr_base;
	cpumask_t *active_mask;
	struct pmu *pmu;
	struct perf_event *events[MAX_COUNTERS];
	struct hlist_node node;
};

static struct amd_uncore * __percpu *amd_uncore_nb;
static struct amd_uncore * __percpu *amd_uncore_l2;

static struct pmu amd_nb_pmu;
static struct pmu amd_l2_pmu;

static cpumask_t amd_nb_active_mask;
static cpumask_t amd_l2_active_mask;

static bool is_nb_event(struct perf_event *event)
{
	return event->pmu->type == amd_nb_pmu.type;
}

static bool is_l2_event(struct perf_event *event)
{
	return event->pmu->type == amd_l2_pmu.type;
}

static struct amd_uncore *event_to_amd_uncore(struct perf_event *event)
{
	if (is_nb_event(event) && amd_uncore_nb)
		return *per_cpu_ptr(amd_uncore_nb, event->cpu);
	else if (is_l2_event(event) && amd_uncore_l2)
		return *per_cpu_ptr(amd_uncore_l2, event->cpu);

	return NULL;
}

static void amd_uncore_read(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;
	u64 prev, new;
	s64 delta;

	/*
	 * since we do not enable counter overflow interrupts,
	 * we do not have to worry about prev_count changing on us
	 */

	prev = local64_read(&hwc->prev_count);
	rdpmcl(hwc->event_base_rdpmc, new);
	local64_set(&hwc->prev_count, new);
	delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT);
	delta >>= COUNTER_SHIFT;
	local64_add(delta, &event->count);
}

static void amd_uncore_start(struct perf_event *event, int flags)
{
	struct hw_perf_event *hwc = &event->hw;

	if (flags & PERF_EF_RELOAD)
		wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count));

	hwc->state = 0;
	wrmsrl(hwc->config_base, (hwc->config | ARCH_PERFMON_EVENTSEL_ENABLE));
	perf_event_update_userpage(event);
}

static void amd_uncore_stop(struct perf_event *event, int flags)
{
	struct hw_perf_event *hwc = &event->hw;

	wrmsrl(hwc->config_base, hwc->config);
	hwc->state |= PERF_HES_STOPPED;

	if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
		amd_uncore_read(event);
		hwc->state |= PERF_HES_UPTODATE;
	}
}

static int amd_uncore_add(struct perf_event *event, int flags)
{
	int i;
	struct amd_uncore *uncore = event_to_amd_uncore(event);
	struct hw_perf_event *hwc = &event->hw;

	/* are we already assigned? */
	if (hwc->idx != -1 && uncore->events[hwc->idx] == event)
		goto out;

	for (i = 0; i < uncore->num_counters; i++) {
		if (uncore->events[i] == event) {
			hwc->idx = i;
			goto out;
		}
	}

	/* if not, take the first available counter */
	hwc->idx = -1;
	for (i = 0; i < uncore->num_counters; i++) {
		if (cmpxchg(&uncore->events[i], NULL, event) == NULL) {
			hwc->idx = i;
			break;
		}
	}

out:
	if (hwc->idx == -1)
		return -EBUSY;

	hwc->config_base = uncore->msr_base + (2 * hwc->idx);
	hwc->event_base = uncore->msr_base + 1 + (2 * hwc->idx);
	hwc->event_base_rdpmc = uncore->rdpmc_base + hwc->idx;
	hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;

	if (flags & PERF_EF_START)
		amd_uncore_start(event, PERF_EF_RELOAD);

	return 0;
}

static void amd_uncore_del(struct perf_event *event, int flags)
{
	int i;
	struct amd_uncore *uncore = event_to_amd_uncore(event);
	struct hw_perf_event *hwc = &event->hw;

	amd_uncore_stop(event, PERF_EF_UPDATE);

	for (i = 0; i < uncore->num_counters; i++) {
		if (cmpxchg(&uncore->events[i], event, NULL) == event)
			break;
	}

	hwc->idx = -1;
}

static int amd_uncore_event_init(struct perf_event *event)
{
	struct amd_uncore *uncore;
	struct hw_perf_event *hwc = &event->hw;

	if (event->attr.type != event->pmu->type)
		return -ENOENT;

	/*
	 * NB and L2 counters (MSRs) are shared across all cores that share the
	 * same NB / L2 cache. Interrupts can be directed to a single target
	 * core, however, event counts generated by processes running on other
	 * cores cannot be masked out. So we do not support sampling and
	 * per-thread events.
	 */
	if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
		return -EINVAL;

	/* NB and L2 counters do not have usr/os/guest/host bits */
	if (event->attr.exclude_user || event->attr.exclude_kernel ||
	    event->attr.exclude_host || event->attr.exclude_guest)
		return -EINVAL;

	/* and we do not enable counter overflow interrupts */
	hwc->config = event->attr.config & AMD64_RAW_EVENT_MASK_NB;
	hwc->idx = -1;

	if (event->cpu < 0)
		return -EINVAL;

	uncore = event_to_amd_uncore(event);
	if (!uncore)
		return -ENODEV;

	/*
	 * since request can come in to any of the shared cores, we will remap
	 * to a single common cpu.
	 */
	event->cpu = uncore->cpu;

	return 0;
}

static ssize_t amd_uncore_attr_show_cpumask(struct device *dev,
					    struct device_attribute *attr,
					    char *buf)
{
	cpumask_t *active_mask;
	struct pmu *pmu = dev_get_drvdata(dev);

	if (pmu->type == amd_nb_pmu.type)
		active_mask = &amd_nb_active_mask;
	else if (pmu->type == amd_l2_pmu.type)
		active_mask = &amd_l2_active_mask;
	else
		return 0;

	return cpumap_print_to_pagebuf(true, buf, active_mask);
}
static DEVICE_ATTR(cpumask, S_IRUGO, amd_uncore_attr_show_cpumask, NULL);

static struct attribute *amd_uncore_attrs[] = {
	&dev_attr_cpumask.attr,
	NULL,
};

static struct attribute_group amd_uncore_attr_group = {
	.attrs = amd_uncore_attrs,
};

PMU_FORMAT_ATTR(event, "config:0-7,32-35");
PMU_FORMAT_ATTR(umask, "config:8-15");

static struct attribute *amd_uncore_format_attr[] = {
	&format_attr_event.attr,
	&format_attr_umask.attr,
	NULL,
};

static struct attribute_group amd_uncore_format_group = {
	.name = "format",
	.attrs = amd_uncore_format_attr,
};

static const struct attribute_group *amd_uncore_attr_groups[] = {
	&amd_uncore_attr_group,
	&amd_uncore_format_group,
	NULL,
};

static struct pmu amd_nb_pmu = {
	.task_ctx_nr	= perf_invalid_context,
	.attr_groups	= amd_uncore_attr_groups,
	.name		= "amd_nb",
	.event_init	= amd_uncore_event_init,
	.add		= amd_uncore_add,
	.del		= amd_uncore_del,
	.start		= amd_uncore_start,
	.stop		= amd_uncore_stop,
	.read		= amd_uncore_read,
};

static struct pmu amd_l2_pmu = {
	.task_ctx_nr	= perf_invalid_context,
	.attr_groups	= amd_uncore_attr_groups,
	.name		= "amd_l2",
	.event_init	= amd_uncore_event_init,
	.add		= amd_uncore_add,
	.del		= amd_uncore_del,
	.start		= amd_uncore_start,
	.stop		= amd_uncore_stop,
	.read		= amd_uncore_read,
};

static struct amd_uncore *amd_uncore_alloc(unsigned int cpu)
{
	return kzalloc_node(sizeof(struct amd_uncore), GFP_KERNEL,
			cpu_to_node(cpu));
}

static int amd_uncore_cpu_up_prepare(unsigned int cpu)
{
	struct amd_uncore *uncore_nb = NULL, *uncore_l2;

	if (amd_uncore_nb) {
		uncore_nb = amd_uncore_alloc(cpu);
		if (!uncore_nb)
			goto fail;
		uncore_nb->cpu = cpu;
		uncore_nb->num_counters = NUM_COUNTERS_NB;
		uncore_nb->rdpmc_base = RDPMC_BASE_NB;
		uncore_nb->msr_base = MSR_F15H_NB_PERF_CTL;
		uncore_nb->active_mask = &amd_nb_active_mask;
		uncore_nb->pmu = &amd_nb_pmu;
		uncore_nb->id = -1;
		*per_cpu_ptr(amd_uncore_nb, cpu) = uncore_nb;
	}

	if (amd_uncore_l2) {
		uncore_l2 = amd_uncore_alloc(cpu);
		if (!uncore_l2)
			goto fail;
		uncore_l2->cpu = cpu;
		uncore_l2->num_counters = NUM_COUNTERS_L2;
		uncore_l2->rdpmc_base = RDPMC_BASE_L2;
		uncore_l2->msr_base = MSR_F16H_L2I_PERF_CTL;
		uncore_l2->active_mask = &amd_l2_active_mask;
		uncore_l2->pmu = &amd_l2_pmu;
		uncore_l2->id = -1;
		*per_cpu_ptr(amd_uncore_l2, cpu) = uncore_l2;
	}

	return 0;

fail:
	if (amd_uncore_nb)
		*per_cpu_ptr(amd_uncore_nb, cpu) = NULL;
	kfree(uncore_nb);
	return -ENOMEM;
}

static struct amd_uncore *
amd_uncore_find_online_sibling(struct amd_uncore *this,
			       struct amd_uncore * __percpu *uncores)
{
	unsigned int cpu;
	struct amd_uncore *that;

	for_each_online_cpu(cpu) {
		that = *per_cpu_ptr(uncores, cpu);

		if (!that)
			continue;

		if (this == that)
			continue;

		if (this->id == that->id) {
			hlist_add_head(&this->node, &uncore_unused_list);
			this = that;
			break;
		}
	}

	this->refcnt++;
	return this;
}

static int amd_uncore_cpu_starting(unsigned int cpu)
{
	unsigned int eax, ebx, ecx, edx;
	struct amd_uncore *uncore;

	if (amd_uncore_nb) {
		uncore = *per_cpu_ptr(amd_uncore_nb, cpu);
		cpuid(0x8000001e, &eax, &ebx, &ecx, &edx);
		uncore->id = ecx & 0xff;

		uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_nb);
		*per_cpu_ptr(amd_uncore_nb, cpu) = uncore;
	}

	if (amd_uncore_l2) {
		unsigned int apicid = cpu_data(cpu).apicid;
		unsigned int nshared;

		uncore = *per_cpu_ptr(amd_uncore_l2, cpu);
		cpuid_count(0x8000001d, 2, &eax, &ebx, &ecx, &edx);
		nshared = ((eax >> 14) & 0xfff) + 1;
		uncore->id = apicid - (apicid % nshared);

		uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_l2);
		*per_cpu_ptr(amd_uncore_l2, cpu) = uncore;
	}

	return 0;
}

static void uncore_clean_online(void)
{
	struct amd_uncore *uncore;
	struct hlist_node *n;

	hlist_for_each_entry_safe(uncore, n, &uncore_unused_list, node) {
		hlist_del(&uncore->node);
		kfree(uncore);
	}
}

static void uncore_online(unsigned int cpu,
			  struct amd_uncore * __percpu *uncores)
{
	struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu);

	uncore_clean_online();

	if (cpu == uncore->cpu)
		cpumask_set_cpu(cpu, uncore->active_mask);
}

static int amd_uncore_cpu_online(unsigned int cpu)
{
	if (amd_uncore_nb)
		uncore_online(cpu, amd_uncore_nb);

	if (amd_uncore_l2)
		uncore_online(cpu, amd_uncore_l2);

	return 0;
}

static void uncore_down_prepare(unsigned int cpu,
				struct amd_uncore * __percpu *uncores)
{
	unsigned int i;
	struct amd_uncore *this = *per_cpu_ptr(uncores, cpu);

	if (this->cpu != cpu)
		return;

	/* this cpu is going down, migrate to a shared sibling if possible */
	for_each_online_cpu(i) {
		struct amd_uncore *that = *per_cpu_ptr(uncores, i);

		if (cpu == i)
			continue;

		if (this == that) {
			perf_pmu_migrate_context(this->pmu, cpu, i);
			cpumask_clear_cpu(cpu, that->active_mask);
			cpumask_set_cpu(i, that->active_mask);
			that->cpu = i;
			break;
		}
	}
}

static int amd_uncore_cpu_down_prepare(unsigned int cpu)
{
	if (amd_uncore_nb)
		uncore_down_prepare(cpu, amd_uncore_nb);

	if (amd_uncore_l2)
		uncore_down_prepare(cpu, amd_uncore_l2);

	return 0;
}

static void uncore_dead(unsigned int cpu, struct amd_uncore * __percpu *uncores)
{
	struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu);

	if (cpu == uncore->cpu)
		cpumask_clear_cpu(cpu, uncore->active_mask);

	if (!--uncore->refcnt)
		kfree(uncore);
	*per_cpu_ptr(uncores, cpu) = NULL;
}

static int amd_uncore_cpu_dead(unsigned int cpu)
{
	if (amd_uncore_nb)
		uncore_dead(cpu, amd_uncore_nb);

	if (amd_uncore_l2)
		uncore_dead(cpu, amd_uncore_l2);

	return 0;
}

static int __init amd_uncore_init(void)
{
	int ret = -ENODEV;

	if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
		goto fail_nodev;

	if (!boot_cpu_has(X86_FEATURE_TOPOEXT))
		goto fail_nodev;

	if (boot_cpu_has(X86_FEATURE_PERFCTR_NB)) {
		amd_uncore_nb = alloc_percpu(struct amd_uncore *);
		if (!amd_uncore_nb) {
			ret = -ENOMEM;
			goto fail_nb;
		}
		ret = perf_pmu_register(&amd_nb_pmu, amd_nb_pmu.name, -1);
		if (ret)
			goto fail_nb;

		pr_info("perf: AMD NB counters detected\n");
		ret = 0;
	}

	if (boot_cpu_has(X86_FEATURE_PERFCTR_L2)) {
		amd_uncore_l2 = alloc_percpu(struct amd_uncore *);
		if (!amd_uncore_l2) {
			ret = -ENOMEM;
			goto fail_l2;
		}
		ret = perf_pmu_register(&amd_l2_pmu, amd_l2_pmu.name, -1);
		if (ret)
			goto fail_l2;

		pr_info("perf: AMD L2I counters detected\n");
		ret = 0;
	}

	/*
	 * Install callbacks. Core will call them for each online cpu.
	 */
	if (cpuhp_setup_state(CPUHP_PERF_X86_AMD_UNCORE_PREP,
			      "perf/x86/amd/uncore:prepare",
			      amd_uncore_cpu_up_prepare, amd_uncore_cpu_dead))
		goto fail_l2;

	if (cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING,
			      "perf/x86/amd/uncore:starting",
			      amd_uncore_cpu_starting, NULL))
		goto fail_prep;
	if (cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE,
			      "perf/x86/amd/uncore:online",
			      amd_uncore_cpu_online,
			      amd_uncore_cpu_down_prepare))
		goto fail_start;
	return 0;

fail_start:
	cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING);
fail_prep:
	cpuhp_remove_state(CPUHP_PERF_X86_AMD_UNCORE_PREP);
fail_l2:
	if (boot_cpu_has(X86_FEATURE_PERFCTR_NB))
		perf_pmu_unregister(&amd_nb_pmu);
	if (amd_uncore_l2)
		free_percpu(amd_uncore_l2);
fail_nb:
	if (amd_uncore_nb)
		free_percpu(amd_uncore_nb);

fail_nodev:
	return ret;
}
device_initcall(amd_uncore_init);
Commit	Line	Data
c43ca509 JS	1	/*
	2	* Copyright (C) 2013 Advanced Micro Devices, Inc.
	3	*
	4	* Author: Jacob Shin <jacob.shin@amd.com>
	5	*
	6	* This program is free software; you can redistribute it and/or modify
	7	* it under the terms of the GNU General Public License version 2 as
	8	* published by the Free Software Foundation.
	9	*/
	10
	11	#include <linux/perf_event.h>
	12	#include <linux/percpu.h>
	13	#include <linux/types.h>
	14	#include <linux/slab.h>
	15	#include <linux/init.h>
	16	#include <linux/cpu.h>
	17	#include <linux/cpumask.h>
	18
	19	#include <asm/cpufeature.h>
	20	#include <asm/perf_event.h>
	21	#include <asm/msr.h>
	22
	23	#define NUM_COUNTERS_NB 4
	24	#define NUM_COUNTERS_L2 4
	25	#define MAX_COUNTERS NUM_COUNTERS_NB
	26
	27	#define RDPMC_BASE_NB 6
	28	#define RDPMC_BASE_L2 10
	29
	30	#define COUNTER_SHIFT 16
	31
7d762e49 SAS	32	static HLIST_HEAD(uncore_unused_list);
7d762e49 SAS	33
c43ca509 JS	34	struct amd_uncore {
	35	int id;
	36	int refcnt;
	37	int cpu;
	38	int num_counters;
	39	int rdpmc_base;
	40	u32 msr_base;
	41	cpumask_t *active_mask;
	42	struct pmu *pmu;
	43	struct perf_event *events[MAX_COUNTERS];
7d762e49	44	struct hlist_node node;
c43ca509 JS	45	};
	46
	47	static struct amd_uncore * __percpu *amd_uncore_nb;
	48	static struct amd_uncore * __percpu *amd_uncore_l2;
	49
	50	static struct pmu amd_nb_pmu;
	51	static struct pmu amd_l2_pmu;
	52
	53	static cpumask_t amd_nb_active_mask;
	54	static cpumask_t amd_l2_active_mask;
	55
	56	static bool is_nb_event(struct perf_event *event)
	57	{
	58	return event->pmu->type == amd_nb_pmu.type;
	59	}
	60
	61	static bool is_l2_event(struct perf_event *event)
	62	{
	63	return event->pmu->type == amd_l2_pmu.type;
	64	}
	65
	66	static struct amd_uncore event_to_amd_uncore(struct perf_event event)
	67	{
	68	if (is_nb_event(event) && amd_uncore_nb)
	69	return *per_cpu_ptr(amd_uncore_nb, event->cpu);
	70	else if (is_l2_event(event) && amd_uncore_l2)
	71	return *per_cpu_ptr(amd_uncore_l2, event->cpu);
	72
	73	return NULL;
	74	}
	75
	76	static void amd_uncore_read(struct perf_event *event)
	77	{
	78	struct hw_perf_event *hwc = &event->hw;
	79	u64 prev, new;
	80	s64 delta;
	81
	82	/*
	83	* since we do not enable counter overflow interrupts,
	84	* we do not have to worry about prev_count changing on us
	85	*/
	86
	87	prev = local64_read(&hwc->prev_count);
	88	rdpmcl(hwc->event_base_rdpmc, new);
	89	local64_set(&hwc->prev_count, new);
	90	delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT);
	91	delta >>= COUNTER_SHIFT;
	92	local64_add(delta, &event->count);
	93	}
	94
	95	static void amd_uncore_start(struct perf_event *event, int flags)
	96	{
	97	struct hw_perf_event *hwc = &event->hw;
	98
	99	if (flags & PERF_EF_RELOAD)
	100	wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count));
	101
	102	hwc->state = 0;
	103	wrmsrl(hwc->config_base, (hwc->config \| ARCH_PERFMON_EVENTSEL_ENABLE));
	104	perf_event_update_userpage(event);
	105	}
	106
	107	static void amd_uncore_stop(struct perf_event *event, int flags)
	108	{
109	struct hw_perf_event *hwc = &event->hw;
110
111	wrmsrl(hwc->config_base, hwc->config);
112	hwc->state \|= PERF_HES_STOPPED;
113
114	if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
115	amd_uncore_read(event);
116	hwc->state \|= PERF_HES_UPTODATE;
117	}
118	}
119
120	static int amd_uncore_add(struct perf_event *event, int flags)
121	{
122	int i;
123	struct amd_uncore *uncore = event_to_amd_uncore(event);
124	struct hw_perf_event *hwc = &event->hw;
125
126	/* are we already assigned? */
127	if (hwc->idx != -1 && uncore->events[hwc->idx] == event)
128	goto out;
129
130	for (i = 0; i < uncore->num_counters; i++) {
131	if (uncore->events[i] == event) {
132	hwc->idx = i;
133	goto out;
134	}
135	}
136
137	/* if not, take the first available counter */
138	hwc->idx = -1;
139	for (i = 0; i < uncore->num_counters; i++) {
140	if (cmpxchg(&uncore->events[i], NULL, event) == NULL) {
141	hwc->idx = i;
142	break;
143	}
144	}
145
146	out:
147	if (hwc->idx == -1)
148	return -EBUSY;
149
150	hwc->config_base = uncore->msr_base + (2 * hwc->idx);
151	hwc->event_base = uncore->msr_base + 1 + (2 * hwc->idx);
152	hwc->event_base_rdpmc = uncore->rdpmc_base + hwc->idx;
153	hwc->state = PERF_HES_UPTODATE \| PERF_HES_STOPPED;
154
155	if (flags & PERF_EF_START)
156	amd_uncore_start(event, PERF_EF_RELOAD);
157
158	return 0;
159	}
160
161	static void amd_uncore_del(struct perf_event *event, int flags)
162	{
163	int i;
164	struct amd_uncore *uncore = event_to_amd_uncore(event);
165	struct hw_perf_event *hwc = &event->hw;
166
167	amd_uncore_stop(event, PERF_EF_UPDATE);
168
169	for (i = 0; i < uncore->num_counters; i++) {
170	if (cmpxchg(&uncore->events[i], event, NULL) == event)
171	break;
172	}
173
174	hwc->idx = -1;
175	}
176
177	static int amd_uncore_event_init(struct perf_event *event)
178	{
179	struct amd_uncore *uncore;
180	struct hw_perf_event *hwc = &event->hw;
181
182	if (event->attr.type != event->pmu->type)
183	return -ENOENT;
184
185	/*
186	* NB and L2 counters (MSRs) are shared across all cores that share the
187	* same NB / L2 cache. Interrupts can be directed to a single target
188	* core, however, event counts generated by processes running on other
189	* cores cannot be masked out. So we do not support sampling and
190	* per-thread events.
191	*/
192	if (is_sampling_event(event) \|\| event->attach_state & PERF_ATTACH_TASK)
193	return -EINVAL;
194
195	/* NB and L2 counters do not have usr/os/guest/host bits */
196	if (event->attr.exclude_user \|\| event->attr.exclude_kernel \|\|
197	event->attr.exclude_host \|\| event->attr.exclude_guest)
198	return -EINVAL;
199
200	/* and we do not enable counter overflow interrupts */
201	hwc->config = event->attr.config & AMD64_RAW_EVENT_MASK_NB;
202	hwc->idx = -1;
203
204	if (event->cpu < 0)
205	return -EINVAL;
206
207	uncore = event_to_amd_uncore(event);
208	if (!uncore)
209	return -ENODEV;
210
211	/*
212	* since request can come in to any of the shared cores, we will remap
213	* to a single common cpu.
214	*/
215	event->cpu = uncore->cpu;
216
217	return 0;
218	}
219
220	static ssize_t amd_uncore_attr_show_cpumask(struct device *dev,
221	struct device_attribute *attr,
222	char *buf)
223	{
c43ca509 JS	224	cpumask_t *active_mask;
	225	struct pmu *pmu = dev_get_drvdata(dev);
	226
	227	if (pmu->type == amd_nb_pmu.type)
	228	active_mask = &amd_nb_active_mask;
	229	else if (pmu->type == amd_l2_pmu.type)
	230	active_mask = &amd_l2_active_mask;
	231	else
	232	return 0;
	233
5aaba363	234	return cpumap_print_to_pagebuf(true, buf, active_mask);
c43ca509 JS	235	}
	236	static DEVICE_ATTR(cpumask, S_IRUGO, amd_uncore_attr_show_cpumask, NULL);
	237
	238	static struct attribute *amd_uncore_attrs[] = {
	239	&dev_attr_cpumask.attr,
	240	NULL,
	241	};
	242
	243	static struct attribute_group amd_uncore_attr_group = {
	244	.attrs = amd_uncore_attrs,
	245	};
	246
	247	PMU_FORMAT_ATTR(event, "config:0-7,32-35");
	248	PMU_FORMAT_ATTR(umask, "config:8-15");
	249
	250	static struct attribute *amd_uncore_format_attr[] = {
	251	&format_attr_event.attr,
	252	&format_attr_umask.attr,
	253	NULL,
	254	};
	255
	256	static struct attribute_group amd_uncore_format_group = {
	257	.name = "format",
	258	.attrs = amd_uncore_format_attr,
	259	};
	260
	261	static const struct attribute_group *amd_uncore_attr_groups[] = {
	262	&amd_uncore_attr_group,
	263	&amd_uncore_format_group,
	264	NULL,
	265	};
	266
	267	static struct pmu amd_nb_pmu = {
31d50c55	268	.task_ctx_nr = perf_invalid_context,
c43ca509 JS	269	.attr_groups = amd_uncore_attr_groups,
	270	.name = "amd_nb",
	271	.event_init = amd_uncore_event_init,
	272	.add = amd_uncore_add,
	273	.del = amd_uncore_del,
	274	.start = amd_uncore_start,
	275	.stop = amd_uncore_stop,
	276	.read = amd_uncore_read,
	277	};
	278
	279	static struct pmu amd_l2_pmu = {
31d50c55	280	.task_ctx_nr = perf_invalid_context,
c43ca509 JS	281	.attr_groups = amd_uncore_attr_groups,
	282	.name = "amd_l2",
	283	.event_init = amd_uncore_event_init,
	284	.add = amd_uncore_add,
	285	.del = amd_uncore_del,
	286	.start = amd_uncore_start,
	287	.stop = amd_uncore_stop,
	288	.read = amd_uncore_read,
	289	};
	290
148f9bb8	291	static struct amd_uncore *amd_uncore_alloc(unsigned int cpu)
c43ca509 JS	292	{
	293	return kzalloc_node(sizeof(struct amd_uncore), GFP_KERNEL,
	294	cpu_to_node(cpu));
	295	}
	296
503d3291	297	static int amd_uncore_cpu_up_prepare(unsigned int cpu)
c43ca509	298	{
503d3291	299	struct amd_uncore uncore_nb = NULL, uncore_l2;
c43ca509 JS	300
c43ca509 JS	301	if (amd_uncore_nb) {
503d3291 ZZ	302	uncore_nb = amd_uncore_alloc(cpu);
	303	if (!uncore_nb)
	304	goto fail;
	305	uncore_nb->cpu = cpu;
	306	uncore_nb->num_counters = NUM_COUNTERS_NB;
	307	uncore_nb->rdpmc_base = RDPMC_BASE_NB;
	308	uncore_nb->msr_base = MSR_F15H_NB_PERF_CTL;
	309	uncore_nb->active_mask = &amd_nb_active_mask;
	310	uncore_nb->pmu = &amd_nb_pmu;
7d762e49	311	uncore_nb->id = -1;
503d3291	312	*per_cpu_ptr(amd_uncore_nb, cpu) = uncore_nb;
c43ca509 JS	313	}
	314
	315	if (amd_uncore_l2) {
503d3291 ZZ	316	uncore_l2 = amd_uncore_alloc(cpu);
	317	if (!uncore_l2)
	318	goto fail;
	319	uncore_l2->cpu = cpu;
	320	uncore_l2->num_counters = NUM_COUNTERS_L2;
	321	uncore_l2->rdpmc_base = RDPMC_BASE_L2;
	322	uncore_l2->msr_base = MSR_F16H_L2I_PERF_CTL;
	323	uncore_l2->active_mask = &amd_l2_active_mask;
	324	uncore_l2->pmu = &amd_l2_pmu;
7d762e49	325	uncore_l2->id = -1;
503d3291	326	*per_cpu_ptr(amd_uncore_l2, cpu) = uncore_l2;
c43ca509	327	}
503d3291 ZZ	328
	329	return 0;
	330
	331	fail:
8bc9162c TG	332	if (amd_uncore_nb)
8bc9162c TG	333	*per_cpu_ptr(amd_uncore_nb, cpu) = NULL;
503d3291 ZZ	334	kfree(uncore_nb);
503d3291 ZZ	335	return -ENOMEM;
c43ca509 JS	336	}
	337
	338	static struct amd_uncore *
148f9bb8 PG	339	amd_uncore_find_online_sibling(struct amd_uncore *this,
148f9bb8 PG	340	struct amd_uncore * __percpu *uncores)
c43ca509 JS	341	{
	342	unsigned int cpu;
	343	struct amd_uncore *that;
	344
	345	for_each_online_cpu(cpu) {
	346	that = *per_cpu_ptr(uncores, cpu);
	347
	348	if (!that)
	349	continue;
	350
	351	if (this == that)
	352	continue;
	353
	354	if (this->id == that->id) {
7d762e49	355	hlist_add_head(&this->node, &uncore_unused_list);
c43ca509 JS	356	this = that;
	357	break;
	358	}
	359	}
	360
	361	this->refcnt++;
	362	return this;
	363	}
	364
96b2bd38	365	static int amd_uncore_cpu_starting(unsigned int cpu)
c43ca509 JS	366	{
	367	unsigned int eax, ebx, ecx, edx;
	368	struct amd_uncore *uncore;
	369
	370	if (amd_uncore_nb) {
	371	uncore = *per_cpu_ptr(amd_uncore_nb, cpu);
	372	cpuid(0x8000001e, &eax, &ebx, &ecx, &edx);
	373	uncore->id = ecx & 0xff;
	374
	375	uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_nb);
	376	*per_cpu_ptr(amd_uncore_nb, cpu) = uncore;
	377	}
	378
	379	if (amd_uncore_l2) {
	380	unsigned int apicid = cpu_data(cpu).apicid;
	381	unsigned int nshared;
	382
	383	uncore = *per_cpu_ptr(amd_uncore_l2, cpu);
	384	cpuid_count(0x8000001d, 2, &eax, &ebx, &ecx, &edx);
	385	nshared = ((eax >> 14) & 0xfff) + 1;
	386	uncore->id = apicid - (apicid % nshared);
	387
	388	uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_l2);
	389	*per_cpu_ptr(amd_uncore_l2, cpu) = uncore;
	390	}
96b2bd38 RC	391
96b2bd38 RC	392	return 0;
c43ca509 JS	393	}
c43ca509 JS	394
7d762e49 SAS	395	static void uncore_clean_online(void)
	396	{
	397	struct amd_uncore *uncore;
	398	struct hlist_node *n;
	399
	400	hlist_for_each_entry_safe(uncore, n, &uncore_unused_list, node) {
	401	hlist_del(&uncore->node);
	402	kfree(uncore);
	403	}
	404	}
	405
148f9bb8 PG	406	static void uncore_online(unsigned int cpu,
148f9bb8 PG	407	struct amd_uncore * __percpu *uncores)
c43ca509 JS	408	{
	409	struct amd_uncore uncore = per_cpu_ptr(uncores, cpu);
	410
7d762e49	411	uncore_clean_online();
c43ca509 JS	412
	413	if (cpu == uncore->cpu)
	414	cpumask_set_cpu(cpu, uncore->active_mask);
	415	}
	416
96b2bd38	417	static int amd_uncore_cpu_online(unsigned int cpu)
c43ca509 JS	418	{
	419	if (amd_uncore_nb)
	420	uncore_online(cpu, amd_uncore_nb);
	421
	422	if (amd_uncore_l2)
	423	uncore_online(cpu, amd_uncore_l2);
96b2bd38 RC	424
96b2bd38 RC	425	return 0;
c43ca509 JS	426	}
c43ca509 JS	427
148f9bb8 PG	428	static void uncore_down_prepare(unsigned int cpu,
148f9bb8 PG	429	struct amd_uncore * __percpu *uncores)
c43ca509 JS	430	{
	431	unsigned int i;
	432	struct amd_uncore this = per_cpu_ptr(uncores, cpu);
	433
	434	if (this->cpu != cpu)
	435	return;
	436
	437	/* this cpu is going down, migrate to a shared sibling if possible */
	438	for_each_online_cpu(i) {
	439	struct amd_uncore that = per_cpu_ptr(uncores, i);
	440
	441	if (cpu == i)
	442	continue;
	443
	444	if (this == that) {
	445	perf_pmu_migrate_context(this->pmu, cpu, i);
	446	cpumask_clear_cpu(cpu, that->active_mask);
	447	cpumask_set_cpu(i, that->active_mask);
	448	that->cpu = i;
	449	break;
	450	}
	451	}
	452	}
	453
96b2bd38	454	static int amd_uncore_cpu_down_prepare(unsigned int cpu)
c43ca509 JS	455	{
	456	if (amd_uncore_nb)
	457	uncore_down_prepare(cpu, amd_uncore_nb);
	458
	459	if (amd_uncore_l2)
	460	uncore_down_prepare(cpu, amd_uncore_l2);
96b2bd38 RC	461
96b2bd38 RC	462	return 0;
c43ca509 JS	463	}
c43ca509 JS	464
148f9bb8	465	static void uncore_dead(unsigned int cpu, struct amd_uncore * __percpu *uncores)
c43ca509 JS	466	{
	467	struct amd_uncore uncore = per_cpu_ptr(uncores, cpu);
	468
	469	if (cpu == uncore->cpu)
	470	cpumask_clear_cpu(cpu, uncore->active_mask);
	471
	472	if (!--uncore->refcnt)
	473	kfree(uncore);
503d3291	474	*per_cpu_ptr(uncores, cpu) = NULL;
c43ca509 JS	475	}
c43ca509 JS	476
96b2bd38	477	static int amd_uncore_cpu_dead(unsigned int cpu)
c43ca509 JS	478	{
	479	if (amd_uncore_nb)
	480	uncore_dead(cpu, amd_uncore_nb);
	481
	482	if (amd_uncore_l2)
	483	uncore_dead(cpu, amd_uncore_l2);
c43ca509	484
96b2bd38	485	return 0;
503d3291 ZZ	486	}
503d3291 ZZ	487
c43ca509 JS	488	static int __init amd_uncore_init(void)
c43ca509 JS	489	{
c43ca509 JS	490	int ret = -ENODEV;
	491
	492	if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
503d3291	493	goto fail_nodev;
c43ca509	494
362f924b	495	if (!boot_cpu_has(X86_FEATURE_TOPOEXT))
503d3291	496	goto fail_nodev;
c43ca509	497
362f924b	498	if (boot_cpu_has(X86_FEATURE_PERFCTR_NB)) {
c43ca509	499	amd_uncore_nb = alloc_percpu(struct amd_uncore *);
503d3291 ZZ	500	if (!amd_uncore_nb) {
	501	ret = -ENOMEM;
	502	goto fail_nb;
	503	}
	504	ret = perf_pmu_register(&amd_nb_pmu, amd_nb_pmu.name, -1);
	505	if (ret)
	506	goto fail_nb;
c43ca509	507
1b74dde7	508	pr_info("perf: AMD NB counters detected\n");
c43ca509 JS	509	ret = 0;
	510	}
	511
362f924b	512	if (boot_cpu_has(X86_FEATURE_PERFCTR_L2)) {
c43ca509	513	amd_uncore_l2 = alloc_percpu(struct amd_uncore *);
503d3291 ZZ	514	if (!amd_uncore_l2) {
	515	ret = -ENOMEM;
	516	goto fail_l2;
	517	}
	518	ret = perf_pmu_register(&amd_l2_pmu, amd_l2_pmu.name, -1);
	519	if (ret)
	520	goto fail_l2;
c43ca509	521
1b74dde7	522	pr_info("perf: AMD L2I counters detected\n");
c43ca509 JS	523	ret = 0;
	524	}
	525
96b2bd38 RC	526	/*
	527	* Install callbacks. Core will call them for each online cpu.
	528	*/
	529	if (cpuhp_setup_state(CPUHP_PERF_X86_AMD_UNCORE_PREP,
73c1b41e	530	"perf/x86/amd/uncore:prepare",
96b2bd38 RC	531	amd_uncore_cpu_up_prepare, amd_uncore_cpu_dead))
	532	goto fail_l2;
	533
	534	if (cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING,
73c1b41e	535	"perf/x86/amd/uncore:starting",
96b2bd38 RC	536	amd_uncore_cpu_starting, NULL))
	537	goto fail_prep;
	538	if (cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE,
73c1b41e	539	"perf/x86/amd/uncore:online",
96b2bd38 RC	540	amd_uncore_cpu_online,
	541	amd_uncore_cpu_down_prepare))
	542	goto fail_start;
c43ca509	543	return 0;
503d3291	544
96b2bd38 RC	545	fail_start:
	546	cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING);
	547	fail_prep:
	548	cpuhp_remove_state(CPUHP_PERF_X86_AMD_UNCORE_PREP);
503d3291	549	fail_l2:
362f924b	550	if (boot_cpu_has(X86_FEATURE_PERFCTR_NB))
503d3291 ZZ	551	perf_pmu_unregister(&amd_nb_pmu);
	552	if (amd_uncore_l2)
	553	free_percpu(amd_uncore_l2);
	554	fail_nb:
	555	if (amd_uncore_nb)
	556	free_percpu(amd_uncore_nb);
	557
	558	fail_nodev:
	559	return ret;
c43ca509 JS	560	}
c43ca509 JS	561	device_initcall(amd_uncore_init);