[mirror_ubuntu-hirsute-kernel.git] / arch / x86 / events / intel / cstate.c

/*
 * perf_event_intel_cstate.c: support cstate residency counters
 *
 * Copyright (C) 2015, Intel Corp.
 * Author: Kan Liang (kan.liang@intel.com)
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 */

/*
 * This file export cstate related free running (read-only) counters
 * for perf. These counters may be use simultaneously by other tools,
 * such as turbostat. However, it still make sense to implement them
 * in perf. Because we can conveniently collect them together with
 * other events, and allow to use them from tools without special MSR
 * access code.
 *
 * The events only support system-wide mode counting. There is no
 * sampling support because it is not supported by the hardware.
 *
 * According to counters' scope and category, two PMUs are registered
 * with the perf_event core subsystem.
 *  - 'cstate_core': The counter is available for each physical core.
 *    The counters include CORE_C*_RESIDENCY.
 *  - 'cstate_pkg': The counter is available for each physical package.
 *    The counters include PKG_C*_RESIDENCY.
 *
 * All of these counters are specified in the Intel® 64 and IA-32
 * Architectures Software Developer.s Manual Vol3b.
 *
 * Model specific counters:
 *	MSR_CORE_C1_RES: CORE C1 Residency Counter
 *			 perf code: 0x00
 *			 Available model: SLM,AMT
 *			 Scope: Core (each processor core has a MSR)
 *	MSR_CORE_C3_RESIDENCY: CORE C3 Residency Counter
 *			       perf code: 0x01
 *			       Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL
 *			       Scope: Core
 *	MSR_CORE_C6_RESIDENCY: CORE C6 Residency Counter
 *			       perf code: 0x02
 *			       Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,SKL
 *			       Scope: Core
 *	MSR_CORE_C7_RESIDENCY: CORE C7 Residency Counter
 *			       perf code: 0x03
 *			       Available model: SNB,IVB,HSW,BDW,SKL
 *			       Scope: Core
 *	MSR_PKG_C2_RESIDENCY:  Package C2 Residency Counter.
 *			       perf code: 0x00
 *			       Available model: SNB,IVB,HSW,BDW,SKL
 *			       Scope: Package (physical package)
 *	MSR_PKG_C3_RESIDENCY:  Package C3 Residency Counter.
 *			       perf code: 0x01
 *			       Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL
 *			       Scope: Package (physical package)
 *	MSR_PKG_C6_RESIDENCY:  Package C6 Residency Counter.
 *			       perf code: 0x02
 *			       Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,SKL
 *			       Scope: Package (physical package)
 *	MSR_PKG_C7_RESIDENCY:  Package C7 Residency Counter.
 *			       perf code: 0x03
 *			       Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL
 *			       Scope: Package (physical package)
 *	MSR_PKG_C8_RESIDENCY:  Package C8 Residency Counter.
 *			       perf code: 0x04
 *			       Available model: HSW ULT only
 *			       Scope: Package (physical package)
 *	MSR_PKG_C9_RESIDENCY:  Package C9 Residency Counter.
 *			       perf code: 0x05
 *			       Available model: HSW ULT only
 *			       Scope: Package (physical package)
 *	MSR_PKG_C10_RESIDENCY: Package C10 Residency Counter.
 *			       perf code: 0x06
 *			       Available model: HSW ULT only
 *			       Scope: Package (physical package)
 *
 */

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/perf_event.h>
#include <asm/cpu_device_id.h>
#include "../perf_event.h"

#define DEFINE_CSTATE_FORMAT_ATTR(_var, _name, _format)		\
static ssize_t __cstate_##_var##_show(struct kobject *kobj,	\
				struct kobj_attribute *attr,	\
				char *page)			\
{								\
	BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE);		\
	return sprintf(page, _format "\n");			\
}								\
static struct kobj_attribute format_attr_##_var =		\
	__ATTR(_name, 0444, __cstate_##_var##_show, NULL)

static ssize_t cstate_get_attr_cpumask(struct device *dev,
				       struct device_attribute *attr,
				       char *buf);

/* Model -> events mapping */
struct cstate_model {
	unsigned long		core_events;
	unsigned long		pkg_events;
	unsigned long		quirks;
};

/* Quirk flags */
#define SLM_PKG_C6_USE_C7_MSR	(1UL << 0)

struct perf_cstate_msr {
	u64	msr;
	struct	perf_pmu_events_attr *attr;
};


/* cstate_core PMU */
static struct pmu cstate_core_pmu;
static bool has_cstate_core;

enum perf_cstate_core_events {
	PERF_CSTATE_CORE_C1_RES = 0,
	PERF_CSTATE_CORE_C3_RES,
	PERF_CSTATE_CORE_C6_RES,
	PERF_CSTATE_CORE_C7_RES,

	PERF_CSTATE_CORE_EVENT_MAX,
};

PMU_EVENT_ATTR_STRING(c1-residency, evattr_cstate_core_c1, "event=0x00");
PMU_EVENT_ATTR_STRING(c3-residency, evattr_cstate_core_c3, "event=0x01");
PMU_EVENT_ATTR_STRING(c6-residency, evattr_cstate_core_c6, "event=0x02");
PMU_EVENT_ATTR_STRING(c7-residency, evattr_cstate_core_c7, "event=0x03");

static struct perf_cstate_msr core_msr[] = {
	[PERF_CSTATE_CORE_C1_RES] = { MSR_CORE_C1_RES,		&evattr_cstate_core_c1 },
	[PERF_CSTATE_CORE_C3_RES] = { MSR_CORE_C3_RESIDENCY,	&evattr_cstate_core_c3 },
	[PERF_CSTATE_CORE_C6_RES] = { MSR_CORE_C6_RESIDENCY,	&evattr_cstate_core_c6 },
	[PERF_CSTATE_CORE_C7_RES] = { MSR_CORE_C7_RESIDENCY,	&evattr_cstate_core_c7 },
};

static struct attribute *core_events_attrs[PERF_CSTATE_CORE_EVENT_MAX + 1] = {
	NULL,
};

static struct attribute_group core_events_attr_group = {
	.name = "events",
	.attrs = core_events_attrs,
};

DEFINE_CSTATE_FORMAT_ATTR(core_event, event, "config:0-63");
static struct attribute *core_format_attrs[] = {
	&format_attr_core_event.attr,
	NULL,
};

static struct attribute_group core_format_attr_group = {
	.name = "format",
	.attrs = core_format_attrs,
};

static cpumask_t cstate_core_cpu_mask;
static DEVICE_ATTR(cpumask, S_IRUGO, cstate_get_attr_cpumask, NULL);

static struct attribute *cstate_cpumask_attrs[] = {
	&dev_attr_cpumask.attr,
	NULL,
};

static struct attribute_group cpumask_attr_group = {
	.attrs = cstate_cpumask_attrs,
};

static const struct attribute_group *core_attr_groups[] = {
	&core_events_attr_group,
	&core_format_attr_group,
	&cpumask_attr_group,
	NULL,
};

/* cstate_pkg PMU */
static struct pmu cstate_pkg_pmu;
static bool has_cstate_pkg;

enum perf_cstate_pkg_events {
	PERF_CSTATE_PKG_C2_RES = 0,
	PERF_CSTATE_PKG_C3_RES,
	PERF_CSTATE_PKG_C6_RES,
	PERF_CSTATE_PKG_C7_RES,
	PERF_CSTATE_PKG_C8_RES,
	PERF_CSTATE_PKG_C9_RES,
	PERF_CSTATE_PKG_C10_RES,

	PERF_CSTATE_PKG_EVENT_MAX,
};

PMU_EVENT_ATTR_STRING(c2-residency, evattr_cstate_pkg_c2, "event=0x00");
PMU_EVENT_ATTR_STRING(c3-residency, evattr_cstate_pkg_c3, "event=0x01");
PMU_EVENT_ATTR_STRING(c6-residency, evattr_cstate_pkg_c6, "event=0x02");
PMU_EVENT_ATTR_STRING(c7-residency, evattr_cstate_pkg_c7, "event=0x03");
PMU_EVENT_ATTR_STRING(c8-residency, evattr_cstate_pkg_c8, "event=0x04");
PMU_EVENT_ATTR_STRING(c9-residency, evattr_cstate_pkg_c9, "event=0x05");
PMU_EVENT_ATTR_STRING(c10-residency, evattr_cstate_pkg_c10, "event=0x06");

static struct perf_cstate_msr pkg_msr[] = {
	[PERF_CSTATE_PKG_C2_RES] = { MSR_PKG_C2_RESIDENCY,	&evattr_cstate_pkg_c2 },
	[PERF_CSTATE_PKG_C3_RES] = { MSR_PKG_C3_RESIDENCY,	&evattr_cstate_pkg_c3 },
	[PERF_CSTATE_PKG_C6_RES] = { MSR_PKG_C6_RESIDENCY,	&evattr_cstate_pkg_c6 },
	[PERF_CSTATE_PKG_C7_RES] = { MSR_PKG_C7_RESIDENCY,	&evattr_cstate_pkg_c7 },
	[PERF_CSTATE_PKG_C8_RES] = { MSR_PKG_C8_RESIDENCY,	&evattr_cstate_pkg_c8 },
	[PERF_CSTATE_PKG_C9_RES] = { MSR_PKG_C9_RESIDENCY,	&evattr_cstate_pkg_c9 },
	[PERF_CSTATE_PKG_C10_RES] = { MSR_PKG_C10_RESIDENCY,	&evattr_cstate_pkg_c10 },
};

static struct attribute *pkg_events_attrs[PERF_CSTATE_PKG_EVENT_MAX + 1] = {
	NULL,
};

static struct attribute_group pkg_events_attr_group = {
	.name = "events",
	.attrs = pkg_events_attrs,
};

DEFINE_CSTATE_FORMAT_ATTR(pkg_event, event, "config:0-63");
static struct attribute *pkg_format_attrs[] = {
	&format_attr_pkg_event.attr,
	NULL,
};
static struct attribute_group pkg_format_attr_group = {
	.name = "format",
	.attrs = pkg_format_attrs,
};

static cpumask_t cstate_pkg_cpu_mask;

static const struct attribute_group *pkg_attr_groups[] = {
	&pkg_events_attr_group,
	&pkg_format_attr_group,
	&cpumask_attr_group,
	NULL,
};

static ssize_t cstate_get_attr_cpumask(struct device *dev,
				       struct device_attribute *attr,
				       char *buf)
{
	struct pmu *pmu = dev_get_drvdata(dev);

	if (pmu == &cstate_core_pmu)
		return cpumap_print_to_pagebuf(true, buf, &cstate_core_cpu_mask);
	else if (pmu == &cstate_pkg_pmu)
		return cpumap_print_to_pagebuf(true, buf, &cstate_pkg_cpu_mask);
	else
		return 0;
}

static int cstate_pmu_event_init(struct perf_event *event)
{
	u64 cfg = event->attr.config;
	int cpu;

	if (event->attr.type != event->pmu->type)
		return -ENOENT;

	/* unsupported modes and filters */
	if (event->attr.exclude_user   ||
	    event->attr.exclude_kernel ||
	    event->attr.exclude_hv     ||
	    event->attr.exclude_idle   ||
	    event->attr.exclude_host   ||
	    event->attr.exclude_guest  ||
	    event->attr.sample_period) /* no sampling */
		return -EINVAL;

	if (event->cpu < 0)
		return -EINVAL;

	if (event->pmu == &cstate_core_pmu) {
		if (cfg >= PERF_CSTATE_CORE_EVENT_MAX)
			return -EINVAL;
		if (!core_msr[cfg].attr)
			return -EINVAL;
		event->hw.event_base = core_msr[cfg].msr;
		cpu = cpumask_any_and(&cstate_core_cpu_mask,
				      topology_sibling_cpumask(event->cpu));
	} else if (event->pmu == &cstate_pkg_pmu) {
		if (cfg >= PERF_CSTATE_PKG_EVENT_MAX)
			return -EINVAL;
		if (!pkg_msr[cfg].attr)
			return -EINVAL;
		event->hw.event_base = pkg_msr[cfg].msr;
		cpu = cpumask_any_and(&cstate_pkg_cpu_mask,
				      topology_core_cpumask(event->cpu));
	} else {
		return -ENOENT;
	}

	if (cpu >= nr_cpu_ids)
		return -ENODEV;

	event->cpu = cpu;
	event->hw.config = cfg;
	event->hw.idx = -1;
	return 0;
}

static inline u64 cstate_pmu_read_counter(struct perf_event *event)
{
	u64 val;

	rdmsrl(event->hw.event_base, val);
	return val;
}

static void cstate_pmu_event_update(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;
	u64 prev_raw_count, new_raw_count;

again:
	prev_raw_count = local64_read(&hwc->prev_count);
	new_raw_count = cstate_pmu_read_counter(event);

	if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
			    new_raw_count) != prev_raw_count)
		goto again;

	local64_add(new_raw_count - prev_raw_count, &event->count);
}

static void cstate_pmu_event_start(struct perf_event *event, int mode)
{
	local64_set(&event->hw.prev_count, cstate_pmu_read_counter(event));
}

static void cstate_pmu_event_stop(struct perf_event *event, int mode)
{
	cstate_pmu_event_update(event);
}

static void cstate_pmu_event_del(struct perf_event *event, int mode)
{
	cstate_pmu_event_stop(event, PERF_EF_UPDATE);
}

static int cstate_pmu_event_add(struct perf_event *event, int mode)
{
	if (mode & PERF_EF_START)
		cstate_pmu_event_start(event, mode);

	return 0;
}

/*
 * Check if exiting cpu is the designated reader. If so migrate the
 * events when there is a valid target available
 */
static void cstate_cpu_exit(int cpu)
{
	unsigned int target;

	if (has_cstate_core &&
	    cpumask_test_and_clear_cpu(cpu, &cstate_core_cpu_mask)) {

		target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu);
		/* Migrate events if there is a valid target */
		if (target < nr_cpu_ids) {
			cpumask_set_cpu(target, &cstate_core_cpu_mask);
			perf_pmu_migrate_context(&cstate_core_pmu, cpu, target);
		}
	}

	if (has_cstate_pkg &&
	    cpumask_test_and_clear_cpu(cpu, &cstate_pkg_cpu_mask)) {

		target = cpumask_any_but(topology_core_cpumask(cpu), cpu);
		/* Migrate events if there is a valid target */
		if (target < nr_cpu_ids) {
			cpumask_set_cpu(target, &cstate_pkg_cpu_mask);
			perf_pmu_migrate_context(&cstate_pkg_pmu, cpu, target);
		}
	}
}

static void cstate_cpu_init(int cpu)
{
	unsigned int target;

	/*
	 * If this is the first online thread of that core, set it in
	 * the core cpu mask as the designated reader.
	 */
	target = cpumask_any_and(&cstate_core_cpu_mask,
				 topology_sibling_cpumask(cpu));

	if (has_cstate_core && target >= nr_cpu_ids)
		cpumask_set_cpu(cpu, &cstate_core_cpu_mask);

	/*
	 * If this is the first online thread of that package, set it
	 * in the package cpu mask as the designated reader.
	 */
	target = cpumask_any_and(&cstate_pkg_cpu_mask,
				 topology_core_cpumask(cpu));
	if (has_cstate_pkg && target >= nr_cpu_ids)
		cpumask_set_cpu(cpu, &cstate_pkg_cpu_mask);
}

static int cstate_cpu_notifier(struct notifier_block *self,
			       unsigned long action, void *hcpu)
{
	unsigned int cpu = (long)hcpu;

	switch (action & ~CPU_TASKS_FROZEN) {
	case CPU_STARTING:
		cstate_cpu_init(cpu);
		break;
	case CPU_DOWN_PREPARE:
		cstate_cpu_exit(cpu);
		break;
	default:
		break;
	}
	return NOTIFY_OK;
}

static struct pmu cstate_core_pmu = {
	.attr_groups	= core_attr_groups,
	.name		= "cstate_core",
	.task_ctx_nr	= perf_invalid_context,
	.event_init	= cstate_pmu_event_init,
	.add		= cstate_pmu_event_add,
	.del		= cstate_pmu_event_del,
	.start		= cstate_pmu_event_start,
	.stop		= cstate_pmu_event_stop,
	.read		= cstate_pmu_event_update,
	.capabilities	= PERF_PMU_CAP_NO_INTERRUPT,
};

static struct pmu cstate_pkg_pmu = {
	.attr_groups	= pkg_attr_groups,
	.name		= "cstate_pkg",
	.task_ctx_nr	= perf_invalid_context,
	.event_init	= cstate_pmu_event_init,
	.add		= cstate_pmu_event_add,
	.del		= cstate_pmu_event_del,
	.start		= cstate_pmu_event_start,
	.stop		= cstate_pmu_event_stop,
	.read		= cstate_pmu_event_update,
	.capabilities	= PERF_PMU_CAP_NO_INTERRUPT,
};

static const struct cstate_model nhm_cstates __initconst = {
	.core_events		= BIT(PERF_CSTATE_CORE_C3_RES) |
				  BIT(PERF_CSTATE_CORE_C6_RES),

	.pkg_events		= BIT(PERF_CSTATE_PKG_C3_RES) |
				  BIT(PERF_CSTATE_PKG_C6_RES) |
				  BIT(PERF_CSTATE_PKG_C7_RES),
};

static const struct cstate_model snb_cstates __initconst = {
	.core_events		= BIT(PERF_CSTATE_CORE_C3_RES) |
				  BIT(PERF_CSTATE_CORE_C6_RES) |
				  BIT(PERF_CSTATE_CORE_C7_RES),

	.pkg_events		= BIT(PERF_CSTATE_PKG_C2_RES) |
				  BIT(PERF_CSTATE_PKG_C3_RES) |
				  BIT(PERF_CSTATE_PKG_C6_RES) |
				  BIT(PERF_CSTATE_PKG_C7_RES),
};

static const struct cstate_model hswult_cstates __initconst = {
	.core_events		= BIT(PERF_CSTATE_CORE_C3_RES) |
				  BIT(PERF_CSTATE_CORE_C6_RES) |
				  BIT(PERF_CSTATE_CORE_C7_RES),

	.pkg_events		= BIT(PERF_CSTATE_PKG_C2_RES) |
				  BIT(PERF_CSTATE_PKG_C3_RES) |
				  BIT(PERF_CSTATE_PKG_C6_RES) |
				  BIT(PERF_CSTATE_PKG_C7_RES) |
				  BIT(PERF_CSTATE_PKG_C8_RES) |
				  BIT(PERF_CSTATE_PKG_C9_RES) |
				  BIT(PERF_CSTATE_PKG_C10_RES),
};

static const struct cstate_model slm_cstates __initconst = {
	.core_events		= BIT(PERF_CSTATE_CORE_C1_RES) |
				  BIT(PERF_CSTATE_CORE_C6_RES),

	.pkg_events		= BIT(PERF_CSTATE_PKG_C6_RES),
	.quirks			= SLM_PKG_C6_USE_C7_MSR,
};

#define X86_CSTATES_MODEL(model, states)				\
	{ X86_VENDOR_INTEL, 6, model, X86_FEATURE_ANY, (unsigned long) &(states) }

static const struct x86_cpu_id intel_cstates_match[] __initconst = {
	X86_CSTATES_MODEL(30, nhm_cstates),    /* 45nm Nehalem              */
	X86_CSTATES_MODEL(26, nhm_cstates),    /* 45nm Nehalem-EP           */
	X86_CSTATES_MODEL(46, nhm_cstates),    /* 45nm Nehalem-EX           */

	X86_CSTATES_MODEL(37, nhm_cstates),    /* 32nm Westmere             */
	X86_CSTATES_MODEL(44, nhm_cstates),    /* 32nm Westmere-EP          */
	X86_CSTATES_MODEL(47, nhm_cstates),    /* 32nm Westmere-EX          */

	X86_CSTATES_MODEL(42, snb_cstates),    /* 32nm SandyBridge          */
	X86_CSTATES_MODEL(45, snb_cstates),    /* 32nm SandyBridge-E/EN/EP  */

	X86_CSTATES_MODEL(58, snb_cstates),    /* 22nm IvyBridge            */
	X86_CSTATES_MODEL(62, snb_cstates),    /* 22nm IvyBridge-EP/EX      */

	X86_CSTATES_MODEL(60, snb_cstates),    /* 22nm Haswell Core         */
	X86_CSTATES_MODEL(63, snb_cstates),    /* 22nm Haswell Server       */
	X86_CSTATES_MODEL(70, snb_cstates),    /* 22nm Haswell + GT3e       */

	X86_CSTATES_MODEL(69, hswult_cstates), /* 22nm Haswell ULT          */

	X86_CSTATES_MODEL(55, slm_cstates),    /* 22nm Atom Silvermont      */
	X86_CSTATES_MODEL(77, slm_cstates),    /* 22nm Atom Avoton/Rangely  */
	X86_CSTATES_MODEL(76, slm_cstates),    /* 22nm Atom Airmont         */

	X86_CSTATES_MODEL(61, snb_cstates),    /* 14nm Broadwell Core-M     */
	X86_CSTATES_MODEL(86, snb_cstates),    /* 14nm Broadwell Xeon D     */
	X86_CSTATES_MODEL(71, snb_cstates),    /* 14nm Broadwell + GT3e     */
	X86_CSTATES_MODEL(79, snb_cstates),    /* 14nm Broadwell Server     */

	X86_CSTATES_MODEL(78, snb_cstates),    /* 14nm Skylake Mobile       */
	X86_CSTATES_MODEL(94, snb_cstates),    /* 14nm Skylake Desktop      */
	{ },
};
MODULE_DEVICE_TABLE(x86cpu, intel_cstates_match);

/*
 * Probe the cstate events and insert the available one into sysfs attrs
 * Return false if there are no available events.
 */
static bool __init cstate_probe_msr(const unsigned long evmsk, int max,
                                   struct perf_cstate_msr *msr,
                                   struct attribute **attrs)
{
	bool found = false;
	unsigned int bit;
	u64 val;

	for (bit = 0; bit < max; bit++) {
		if (test_bit(bit, &evmsk) && !rdmsrl_safe(msr[bit].msr, &val)) {
			*attrs++ = &msr[bit].attr->attr.attr;
			found = true;
		} else {
			msr[bit].attr = NULL;
		}
	}
	*attrs = NULL;

	return found;
}

static int __init cstate_probe(const struct cstate_model *cm)
{
	/* SLM has different MSR for PKG C6 */
	if (cm->quirks & SLM_PKG_C6_USE_C7_MSR)
		pkg_msr[PERF_CSTATE_PKG_C6_RES].msr = MSR_PKG_C7_RESIDENCY;

	has_cstate_core = cstate_probe_msr(cm->core_events,
					   PERF_CSTATE_CORE_EVENT_MAX,
					   core_msr, core_events_attrs);

	has_cstate_pkg = cstate_probe_msr(cm->pkg_events,
					  PERF_CSTATE_PKG_EVENT_MAX,
					  pkg_msr, pkg_events_attrs);

	return (has_cstate_core || has_cstate_pkg) ? 0 : -ENODEV;
}

static void __init cstate_cpumask_init(void)
{
	int cpu;

	cpu_notifier_register_begin();

	for_each_online_cpu(cpu)
		cstate_cpu_init(cpu);

	__perf_cpu_notifier(cstate_cpu_notifier);

	cpu_notifier_register_done();
}

static void __init cstate_pmus_register(void)
{
	int err;

	if (has_cstate_core) {
		err = perf_pmu_register(&cstate_core_pmu, cstate_core_pmu.name, -1);
		if (WARN_ON(err))
			pr_info("Failed to register PMU %s error %d\n",
				cstate_core_pmu.name, err);
	}

	if (has_cstate_pkg) {
		err = perf_pmu_register(&cstate_pkg_pmu, cstate_pkg_pmu.name, -1);
		if (WARN_ON(err))
			pr_info("Failed to register PMU %s error %d\n",
				cstate_pkg_pmu.name, err);
	}
}

static int __init cstate_pmu_init(void)
{
	const struct x86_cpu_id *id;
	int err;

	if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
		return -ENODEV;

	id = x86_match_cpu(intel_cstates_match);
	if (!id)
		return -ENODEV;

	err = cstate_probe((const struct cstate_model *) id->driver_data);
	if (err)
		return err;

	cstate_cpumask_init();

	cstate_pmus_register();

	return 0;
}
device_initcall(cstate_pmu_init);
Commit	Line	Data
7ce1346a KL	1	/*
	2	* perf_event_intel_cstate.c: support cstate residency counters
	3	*
	4	* Copyright (C) 2015, Intel Corp.
	5	* Author: Kan Liang (kan.liang@intel.com)
	6	*
	7	* This library is free software; you can redistribute it and/or
	8	* modify it under the terms of the GNU Library General Public
	9	* License as published by the Free Software Foundation; either
	10	* version 2 of the License, or (at your option) any later version.
	11	*
	12	* This library is distributed in the hope that it will be useful,
	13	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	15	* Library General Public License for more details.
	16	*
	17	*/
	18
	19	/*
	20	* This file export cstate related free running (read-only) counters
	21	* for perf. These counters may be use simultaneously by other tools,
	22	* such as turbostat. However, it still make sense to implement them
	23	* in perf. Because we can conveniently collect them together with
	24	* other events, and allow to use them from tools without special MSR
	25	* access code.
	26	*
	27	* The events only support system-wide mode counting. There is no
	28	* sampling support because it is not supported by the hardware.
	29	*
	30	* According to counters' scope and category, two PMUs are registered
	31	* with the perf_event core subsystem.
	32	* - 'cstate_core': The counter is available for each physical core.
	33	* The counters include CORE_C*_RESIDENCY.
	34	* - 'cstate_pkg': The counter is available for each physical package.
	35	* The counters include PKG_C*_RESIDENCY.
	36	*
	37	* All of these counters are specified in the Intel® 64 and IA-32
	38	* Architectures Software Developer.s Manual Vol3b.
	39	*
	40	* Model specific counters:
	41	* MSR_CORE_C1_RES: CORE C1 Residency Counter
	42	* perf code: 0x00
	43	* Available model: SLM,AMT
	44	* Scope: Core (each processor core has a MSR)
	45	* MSR_CORE_C3_RESIDENCY: CORE C3 Residency Counter
	46	* perf code: 0x01
	47	* Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL
	48	* Scope: Core
	49	* MSR_CORE_C6_RESIDENCY: CORE C6 Residency Counter
	50	* perf code: 0x02
	51	* Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,SKL
	52	* Scope: Core
	53	* MSR_CORE_C7_RESIDENCY: CORE C7 Residency Counter
	54	* perf code: 0x03
	55	* Available model: SNB,IVB,HSW,BDW,SKL
	56	* Scope: Core
	57	* MSR_PKG_C2_RESIDENCY: Package C2 Residency Counter.
	58	* perf code: 0x00
	59	* Available model: SNB,IVB,HSW,BDW,SKL
	60	* Scope: Package (physical package)
	61	* MSR_PKG_C3_RESIDENCY: Package C3 Residency Counter.
	62	* perf code: 0x01
	63	* Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL
	64	* Scope: Package (physical package)
65	* MSR_PKG_C6_RESIDENCY: Package C6 Residency Counter.
66	* perf code: 0x02
67	* Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,SKL
68	* Scope: Package (physical package)
69	* MSR_PKG_C7_RESIDENCY: Package C7 Residency Counter.
70	* perf code: 0x03
71	* Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL
72	* Scope: Package (physical package)
73	* MSR_PKG_C8_RESIDENCY: Package C8 Residency Counter.
74	* perf code: 0x04
75	* Available model: HSW ULT only
76	* Scope: Package (physical package)
77	* MSR_PKG_C9_RESIDENCY: Package C9 Residency Counter.
78	* perf code: 0x05
79	* Available model: HSW ULT only
80	* Scope: Package (physical package)
81	* MSR_PKG_C10_RESIDENCY: Package C10 Residency Counter.
82	* perf code: 0x06
83	* Available model: HSW ULT only
84	* Scope: Package (physical package)
85	*
86	*/
87
88	#include <linux/module.h>
89	#include <linux/slab.h>
90	#include <linux/perf_event.h>
91	#include <asm/cpu_device_id.h>
27f6d22b	92	#include "../perf_event.h"
7ce1346a KL	93
	94	#define DEFINE_CSTATE_FORMAT_ATTR(_var, _name, _format) \
	95	static ssize_t __cstate_##_var##_show(struct kobject *kobj, \
	96	struct kobj_attribute *attr, \
	97	char *page) \
	98	{ \
	99	BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
	100	return sprintf(page, _format "\n"); \
	101	} \
	102	static struct kobj_attribute format_attr_##_var = \
	103	__ATTR(_name, 0444, __cstate_##_var##_show, NULL)
	104
	105	static ssize_t cstate_get_attr_cpumask(struct device *dev,
	106	struct device_attribute *attr,
	107	char *buf);
	108
424646ee TG	109	/* Model -> events mapping */
	110	struct cstate_model {
	111	unsigned long core_events;
	112	unsigned long pkg_events;
	113	unsigned long quirks;
	114	};
	115
	116	/* Quirk flags */
	117	#define SLM_PKG_C6_USE_C7_MSR (1UL << 0)
	118
7ce1346a KL	119	struct perf_cstate_msr {
	120	u64 msr;
	121	struct perf_pmu_events_attr *attr;
7ce1346a KL	122	};
	123
	124
	125	/* cstate_core PMU */
7ce1346a KL	126	static struct pmu cstate_core_pmu;
	127	static bool has_cstate_core;
	128
424646ee	129	enum perf_cstate_core_events {
7ce1346a KL	130	PERF_CSTATE_CORE_C1_RES = 0,
	131	PERF_CSTATE_CORE_C3_RES,
	132	PERF_CSTATE_CORE_C6_RES,
	133	PERF_CSTATE_CORE_C7_RES,
	134
	135	PERF_CSTATE_CORE_EVENT_MAX,
	136	};
	137
7ce1346a KL	138	PMU_EVENT_ATTR_STRING(c1-residency, evattr_cstate_core_c1, "event=0x00");
	139	PMU_EVENT_ATTR_STRING(c3-residency, evattr_cstate_core_c3, "event=0x01");
	140	PMU_EVENT_ATTR_STRING(c6-residency, evattr_cstate_core_c6, "event=0x02");
	141	PMU_EVENT_ATTR_STRING(c7-residency, evattr_cstate_core_c7, "event=0x03");
	142
	143	static struct perf_cstate_msr core_msr[] = {
424646ee TG	144	[PERF_CSTATE_CORE_C1_RES] = { MSR_CORE_C1_RES, &evattr_cstate_core_c1 },
	145	[PERF_CSTATE_CORE_C3_RES] = { MSR_CORE_C3_RESIDENCY, &evattr_cstate_core_c3 },
	146	[PERF_CSTATE_CORE_C6_RES] = { MSR_CORE_C6_RESIDENCY, &evattr_cstate_core_c6 },
	147	[PERF_CSTATE_CORE_C7_RES] = { MSR_CORE_C7_RESIDENCY, &evattr_cstate_core_c7 },
7ce1346a KL	148	};
	149
	150	static struct attribute *core_events_attrs[PERF_CSTATE_CORE_EVENT_MAX + 1] = {
	151	NULL,
	152	};
	153
	154	static struct attribute_group core_events_attr_group = {
	155	.name = "events",
	156	.attrs = core_events_attrs,
	157	};
	158
	159	DEFINE_CSTATE_FORMAT_ATTR(core_event, event, "config:0-63");
	160	static struct attribute *core_format_attrs[] = {
	161	&format_attr_core_event.attr,
	162	NULL,
	163	};
	164
	165	static struct attribute_group core_format_attr_group = {
	166	.name = "format",
	167	.attrs = core_format_attrs,
	168	};
	169
	170	static cpumask_t cstate_core_cpu_mask;
	171	static DEVICE_ATTR(cpumask, S_IRUGO, cstate_get_attr_cpumask, NULL);
	172
	173	static struct attribute *cstate_cpumask_attrs[] = {
	174	&dev_attr_cpumask.attr,
	175	NULL,
	176	};
	177
	178	static struct attribute_group cpumask_attr_group = {
	179	.attrs = cstate_cpumask_attrs,
	180	};
	181
	182	static const struct attribute_group *core_attr_groups[] = {
	183	&core_events_attr_group,
	184	&core_format_attr_group,
	185	&cpumask_attr_group,
	186	NULL,
	187	};
	188
7ce1346a	189	/* cstate_pkg PMU */
7ce1346a KL	190	static struct pmu cstate_pkg_pmu;
	191	static bool has_cstate_pkg;
	192
424646ee	193	enum perf_cstate_pkg_events {
7ce1346a KL	194	PERF_CSTATE_PKG_C2_RES = 0,
	195	PERF_CSTATE_PKG_C3_RES,
	196	PERF_CSTATE_PKG_C6_RES,
	197	PERF_CSTATE_PKG_C7_RES,
	198	PERF_CSTATE_PKG_C8_RES,
	199	PERF_CSTATE_PKG_C9_RES,
	200	PERF_CSTATE_PKG_C10_RES,
	201
	202	PERF_CSTATE_PKG_EVENT_MAX,
	203	};
	204
7ce1346a KL	205	PMU_EVENT_ATTR_STRING(c2-residency, evattr_cstate_pkg_c2, "event=0x00");
	206	PMU_EVENT_ATTR_STRING(c3-residency, evattr_cstate_pkg_c3, "event=0x01");
	207	PMU_EVENT_ATTR_STRING(c6-residency, evattr_cstate_pkg_c6, "event=0x02");
	208	PMU_EVENT_ATTR_STRING(c7-residency, evattr_cstate_pkg_c7, "event=0x03");
	209	PMU_EVENT_ATTR_STRING(c8-residency, evattr_cstate_pkg_c8, "event=0x04");
	210	PMU_EVENT_ATTR_STRING(c9-residency, evattr_cstate_pkg_c9, "event=0x05");
	211	PMU_EVENT_ATTR_STRING(c10-residency, evattr_cstate_pkg_c10, "event=0x06");
	212
	213	static struct perf_cstate_msr pkg_msr[] = {
424646ee TG	214	[PERF_CSTATE_PKG_C2_RES] = { MSR_PKG_C2_RESIDENCY, &evattr_cstate_pkg_c2 },
	215	[PERF_CSTATE_PKG_C3_RES] = { MSR_PKG_C3_RESIDENCY, &evattr_cstate_pkg_c3 },
	216	[PERF_CSTATE_PKG_C6_RES] = { MSR_PKG_C6_RESIDENCY, &evattr_cstate_pkg_c6 },
	217	[PERF_CSTATE_PKG_C7_RES] = { MSR_PKG_C7_RESIDENCY, &evattr_cstate_pkg_c7 },
	218	[PERF_CSTATE_PKG_C8_RES] = { MSR_PKG_C8_RESIDENCY, &evattr_cstate_pkg_c8 },
	219	[PERF_CSTATE_PKG_C9_RES] = { MSR_PKG_C9_RESIDENCY, &evattr_cstate_pkg_c9 },
	220	[PERF_CSTATE_PKG_C10_RES] = { MSR_PKG_C10_RESIDENCY, &evattr_cstate_pkg_c10 },
7ce1346a KL	221	};
	222
	223	static struct attribute *pkg_events_attrs[PERF_CSTATE_PKG_EVENT_MAX + 1] = {
	224	NULL,
	225	};
	226
	227	static struct attribute_group pkg_events_attr_group = {
	228	.name = "events",
	229	.attrs = pkg_events_attrs,
	230	};
	231
	232	DEFINE_CSTATE_FORMAT_ATTR(pkg_event, event, "config:0-63");
	233	static struct attribute *pkg_format_attrs[] = {
	234	&format_attr_pkg_event.attr,
	235	NULL,
	236	};
	237	static struct attribute_group pkg_format_attr_group = {
	238	.name = "format",
	239	.attrs = pkg_format_attrs,
	240	};
	241
	242	static cpumask_t cstate_pkg_cpu_mask;
	243
	244	static const struct attribute_group *pkg_attr_groups[] = {
	245	&pkg_events_attr_group,
	246	&pkg_format_attr_group,
	247	&cpumask_attr_group,
	248	NULL,
	249	};
	250
7ce1346a KL	251	static ssize_t cstate_get_attr_cpumask(struct device *dev,
	252	struct device_attribute *attr,
	253	char *buf)
	254	{
	255	struct pmu *pmu = dev_get_drvdata(dev);
	256
	257	if (pmu == &cstate_core_pmu)
	258	return cpumap_print_to_pagebuf(true, buf, &cstate_core_cpu_mask);
	259	else if (pmu == &cstate_pkg_pmu)
	260	return cpumap_print_to_pagebuf(true, buf, &cstate_pkg_cpu_mask);
	261	else
	262	return 0;
	263	}
	264
	265	static int cstate_pmu_event_init(struct perf_event *event)
	266	{
	267	u64 cfg = event->attr.config;
49de0493	268	int cpu;
7ce1346a KL	269
	270	if (event->attr.type != event->pmu->type)
	271	return -ENOENT;
	272
	273	/* unsupported modes and filters */
	274	if (event->attr.exclude_user \|\|
	275	event->attr.exclude_kernel \|\|
	276	event->attr.exclude_hv \|\|
	277	event->attr.exclude_idle \|\|
	278	event->attr.exclude_host \|\|
	279	event->attr.exclude_guest \|\|
	280	event->attr.sample_period) /* no sampling */
	281	return -EINVAL;
	282
49de0493 TG	283	if (event->cpu < 0)
	284	return -EINVAL;
	285
7ce1346a KL	286	if (event->pmu == &cstate_core_pmu) {
	287	if (cfg >= PERF_CSTATE_CORE_EVENT_MAX)
	288	return -EINVAL;
	289	if (!core_msr[cfg].attr)
	290	return -EINVAL;
	291	event->hw.event_base = core_msr[cfg].msr;
49de0493 TG	292	cpu = cpumask_any_and(&cstate_core_cpu_mask,
49de0493 TG	293	topology_sibling_cpumask(event->cpu));
7ce1346a KL	294	} else if (event->pmu == &cstate_pkg_pmu) {
	295	if (cfg >= PERF_CSTATE_PKG_EVENT_MAX)
	296	return -EINVAL;
	297	if (!pkg_msr[cfg].attr)
	298	return -EINVAL;
	299	event->hw.event_base = pkg_msr[cfg].msr;
49de0493 TG	300	cpu = cpumask_any_and(&cstate_pkg_cpu_mask,
	301	topology_core_cpumask(event->cpu));
	302	} else {
7ce1346a	303	return -ENOENT;
49de0493 TG	304	}
	305
	306	if (cpu >= nr_cpu_ids)
	307	return -ENODEV;
7ce1346a	308
49de0493	309	event->cpu = cpu;
7ce1346a KL	310	event->hw.config = cfg;
7ce1346a KL	311	event->hw.idx = -1;
49de0493	312	return 0;
7ce1346a KL	313	}
	314
	315	static inline u64 cstate_pmu_read_counter(struct perf_event *event)
	316	{
	317	u64 val;
	318
	319	rdmsrl(event->hw.event_base, val);
	320	return val;
	321	}
	322
	323	static void cstate_pmu_event_update(struct perf_event *event)
	324	{
	325	struct hw_perf_event *hwc = &event->hw;
	326	u64 prev_raw_count, new_raw_count;
	327
	328	again:
	329	prev_raw_count = local64_read(&hwc->prev_count);
	330	new_raw_count = cstate_pmu_read_counter(event);
	331
	332	if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
	333	new_raw_count) != prev_raw_count)
	334	goto again;
	335
	336	local64_add(new_raw_count - prev_raw_count, &event->count);
	337	}
	338
	339	static void cstate_pmu_event_start(struct perf_event *event, int mode)
	340	{
	341	local64_set(&event->hw.prev_count, cstate_pmu_read_counter(event));
	342	}
	343
	344	static void cstate_pmu_event_stop(struct perf_event *event, int mode)
	345	{
	346	cstate_pmu_event_update(event);
	347	}
	348
	349	static void cstate_pmu_event_del(struct perf_event *event, int mode)
	350	{
	351	cstate_pmu_event_stop(event, PERF_EF_UPDATE);
	352	}
	353
	354	static int cstate_pmu_event_add(struct perf_event *event, int mode)
	355	{
	356	if (mode & PERF_EF_START)
	357	cstate_pmu_event_start(event, mode);
	358
	359	return 0;
	360	}
	361
49de0493 TG	362	/*
	363	* Check if exiting cpu is the designated reader. If so migrate the
	364	* events when there is a valid target available
	365	*/
7ce1346a KL	366	static void cstate_cpu_exit(int cpu)
7ce1346a KL	367	{
49de0493 TG	368	unsigned int target;
	369
	370	if (has_cstate_core &&
	371	cpumask_test_and_clear_cpu(cpu, &cstate_core_cpu_mask)) {
	372
	373	target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu);
	374	/* Migrate events if there is a valid target */
	375	if (target < nr_cpu_ids) {
7ce1346a	376	cpumask_set_cpu(target, &cstate_core_cpu_mask);
7ce1346a	377	perf_pmu_migrate_context(&cstate_core_pmu, cpu, target);
49de0493	378	}
7ce1346a KL	379	}
7ce1346a KL	380
49de0493 TG	381	if (has_cstate_pkg &&
	382	cpumask_test_and_clear_cpu(cpu, &cstate_pkg_cpu_mask)) {
	383
	384	target = cpumask_any_but(topology_core_cpumask(cpu), cpu);
	385	/* Migrate events if there is a valid target */
	386	if (target < nr_cpu_ids) {
7ce1346a	387	cpumask_set_cpu(target, &cstate_pkg_cpu_mask);
7ce1346a	388	perf_pmu_migrate_context(&cstate_pkg_pmu, cpu, target);
49de0493	389	}
7ce1346a KL	390	}
	391	}
	392
	393	static void cstate_cpu_init(int cpu)
	394	{
49de0493	395	unsigned int target;
7ce1346a	396
49de0493 TG	397	/*
	398	* If this is the first online thread of that core, set it in
	399	* the core cpu mask as the designated reader.
	400	*/
	401	target = cpumask_any_and(&cstate_core_cpu_mask,
	402	topology_sibling_cpumask(cpu));
	403
	404	if (has_cstate_core && target >= nr_cpu_ids)
	405	cpumask_set_cpu(cpu, &cstate_core_cpu_mask);
	406
	407	/*
	408	* If this is the first online thread of that package, set it
	409	* in the package cpu mask as the designated reader.
	410	*/
	411	target = cpumask_any_and(&cstate_pkg_cpu_mask,
	412	topology_core_cpumask(cpu));
	413	if (has_cstate_pkg && target >= nr_cpu_ids)
	414	cpumask_set_cpu(cpu, &cstate_pkg_cpu_mask);
7ce1346a KL	415	}
	416
	417	static int cstate_cpu_notifier(struct notifier_block *self,
49de0493	418	unsigned long action, void *hcpu)
7ce1346a KL	419	{
	420	unsigned int cpu = (long)hcpu;
	421
	422	switch (action & ~CPU_TASKS_FROZEN) {
7ce1346a KL	423	case CPU_STARTING:
	424	cstate_cpu_init(cpu);
	425	break;
7ce1346a KL	426	case CPU_DOWN_PREPARE:
	427	cstate_cpu_exit(cpu);
	428	break;
	429	default:
	430	break;
	431	}
7ce1346a KL	432	return NOTIFY_OK;
	433	}
	434
424646ee TG	435	static struct pmu cstate_core_pmu = {
	436	.attr_groups = core_attr_groups,
	437	.name = "cstate_core",
	438	.task_ctx_nr = perf_invalid_context,
	439	.event_init = cstate_pmu_event_init,
	440	.add = cstate_pmu_event_add,
	441	.del = cstate_pmu_event_del,
	442	.start = cstate_pmu_event_start,
	443	.stop = cstate_pmu_event_stop,
	444	.read = cstate_pmu_event_update,
	445	.capabilities = PERF_PMU_CAP_NO_INTERRUPT,
	446	};
	447
	448	static struct pmu cstate_pkg_pmu = {
	449	.attr_groups = pkg_attr_groups,
	450	.name = "cstate_pkg",
	451	.task_ctx_nr = perf_invalid_context,
	452	.event_init = cstate_pmu_event_init,
	453	.add = cstate_pmu_event_add,
	454	.del = cstate_pmu_event_del,
	455	.start = cstate_pmu_event_start,
	456	.stop = cstate_pmu_event_stop,
	457	.read = cstate_pmu_event_update,
	458	.capabilities = PERF_PMU_CAP_NO_INTERRUPT,
	459	};
	460
	461	static const struct cstate_model nhm_cstates __initconst = {
	462	.core_events = BIT(PERF_CSTATE_CORE_C3_RES) \|
	463	BIT(PERF_CSTATE_CORE_C6_RES),
	464
	465	.pkg_events = BIT(PERF_CSTATE_PKG_C3_RES) \|
	466	BIT(PERF_CSTATE_PKG_C6_RES) \|
	467	BIT(PERF_CSTATE_PKG_C7_RES),
	468	};
	469
	470	static const struct cstate_model snb_cstates __initconst = {
	471	.core_events = BIT(PERF_CSTATE_CORE_C3_RES) \|
	472	BIT(PERF_CSTATE_CORE_C6_RES) \|
	473	BIT(PERF_CSTATE_CORE_C7_RES),
	474
	475	.pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) \|
	476	BIT(PERF_CSTATE_PKG_C3_RES) \|
	477	BIT(PERF_CSTATE_PKG_C6_RES) \|
	478	BIT(PERF_CSTATE_PKG_C7_RES),
	479	};
	480
	481	static const struct cstate_model hswult_cstates __initconst = {
	482	.core_events = BIT(PERF_CSTATE_CORE_C3_RES) \|
	483	BIT(PERF_CSTATE_CORE_C6_RES) \|
	484	BIT(PERF_CSTATE_CORE_C7_RES),
	485
	486	.pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) \|
	487	BIT(PERF_CSTATE_PKG_C3_RES) \|
	488	BIT(PERF_CSTATE_PKG_C6_RES) \|
	489	BIT(PERF_CSTATE_PKG_C7_RES) \|
	490	BIT(PERF_CSTATE_PKG_C8_RES) \|
	491	BIT(PERF_CSTATE_PKG_C9_RES) \|
	492	BIT(PERF_CSTATE_PKG_C10_RES),
	493	};
	494
	495	static const struct cstate_model slm_cstates __initconst = {
	496	.core_events = BIT(PERF_CSTATE_CORE_C1_RES) \|
	497	BIT(PERF_CSTATE_CORE_C6_RES),
	498
499	.pkg_events = BIT(PERF_CSTATE_PKG_C6_RES),
500	.quirks = SLM_PKG_C6_USE_C7_MSR,
501	};
502
503	#define X86_CSTATES_MODEL(model, states) \
504	{ X86_VENDOR_INTEL, 6, model, X86_FEATURE_ANY, (unsigned long) &(states) }
505
506	static const struct x86_cpu_id intel_cstates_match[] __initconst = {
507	X86_CSTATES_MODEL(30, nhm_cstates), /* 45nm Nehalem */
508	X86_CSTATES_MODEL(26, nhm_cstates), /* 45nm Nehalem-EP */
509	X86_CSTATES_MODEL(46, nhm_cstates), /* 45nm Nehalem-EX */
510
511	X86_CSTATES_MODEL(37, nhm_cstates), /* 32nm Westmere */
512	X86_CSTATES_MODEL(44, nhm_cstates), /* 32nm Westmere-EP */
513	X86_CSTATES_MODEL(47, nhm_cstates), /* 32nm Westmere-EX */
514
515	X86_CSTATES_MODEL(42, snb_cstates), /* 32nm SandyBridge */
516	X86_CSTATES_MODEL(45, snb_cstates), /* 32nm SandyBridge-E/EN/EP */
517
518	X86_CSTATES_MODEL(58, snb_cstates), /* 22nm IvyBridge */
519	X86_CSTATES_MODEL(62, snb_cstates), /* 22nm IvyBridge-EP/EX */
520
521	X86_CSTATES_MODEL(60, snb_cstates), /* 22nm Haswell Core */
522	X86_CSTATES_MODEL(63, snb_cstates), /* 22nm Haswell Server */
523	X86_CSTATES_MODEL(70, snb_cstates), /* 22nm Haswell + GT3e */
524
525	X86_CSTATES_MODEL(69, hswult_cstates), /* 22nm Haswell ULT */
526
527	X86_CSTATES_MODEL(55, slm_cstates), /* 22nm Atom Silvermont */
528	X86_CSTATES_MODEL(77, slm_cstates), /* 22nm Atom Avoton/Rangely */
529	X86_CSTATES_MODEL(76, slm_cstates), /* 22nm Atom Airmont */
530
531	X86_CSTATES_MODEL(61, snb_cstates), /* 14nm Broadwell Core-M */
532	X86_CSTATES_MODEL(86, snb_cstates), /* 14nm Broadwell Xeon D */
533	X86_CSTATES_MODEL(71, snb_cstates), /* 14nm Broadwell + GT3e */
534	X86_CSTATES_MODEL(79, snb_cstates), /* 14nm Broadwell Server */
535
536	X86_CSTATES_MODEL(78, snb_cstates), /* 14nm Skylake Mobile */
537	X86_CSTATES_MODEL(94, snb_cstates), /* 14nm Skylake Desktop */
538	{ },
539	};
540	MODULE_DEVICE_TABLE(x86cpu, intel_cstates_match);
541
7ce1346a KL	542	/*
7ce1346a KL	543	* Probe the cstate events and insert the available one into sysfs attrs
424646ee	544	* Return false if there are no available events.
7ce1346a	545	*/
424646ee TG	546	static bool __init cstate_probe_msr(const unsigned long evmsk, int max,
	547	struct perf_cstate_msr *msr,
	548	struct attribute **attrs)
7ce1346a	549	{
424646ee TG	550	bool found = false;
424646ee TG	551	unsigned int bit;
7ce1346a KL	552	u64 val;
7ce1346a KL	553
424646ee TG	554	for (bit = 0; bit < max; bit++) {
	555	if (test_bit(bit, &evmsk) && !rdmsrl_safe(msr[bit].msr, &val)) {
	556	*attrs++ = &msr[bit].attr->attr.attr;
	557	found = true;
	558	} else {
	559	msr[bit].attr = NULL;
	560	}
7ce1346a	561	}
424646ee	562	*attrs = NULL;
7ce1346a	563
424646ee	564	return found;
7ce1346a KL	565	}
7ce1346a KL	566
424646ee	567	static int __init cstate_probe(const struct cstate_model *cm)
7ce1346a KL	568	{
7ce1346a KL	569	/* SLM has different MSR for PKG C6 */
424646ee	570	if (cm->quirks & SLM_PKG_C6_USE_C7_MSR)
7ce1346a	571	pkg_msr[PERF_CSTATE_PKG_C6_RES].msr = MSR_PKG_C7_RESIDENCY;
7ce1346a	572
424646ee TG	573	has_cstate_core = cstate_probe_msr(cm->core_events,
	574	PERF_CSTATE_CORE_EVENT_MAX,
	575	core_msr, core_events_attrs);
7ce1346a	576
424646ee TG	577	has_cstate_pkg = cstate_probe_msr(cm->pkg_events,
	578	PERF_CSTATE_PKG_EVENT_MAX,
	579	pkg_msr, pkg_events_attrs);
7ce1346a KL	580
	581	return (has_cstate_core \|\| has_cstate_pkg) ? 0 : -ENODEV;
	582	}
	583
	584	static void __init cstate_cpumask_init(void)
	585	{
	586	int cpu;
	587
	588	cpu_notifier_register_begin();
	589
	590	for_each_online_cpu(cpu)
	591	cstate_cpu_init(cpu);
	592
	593	__perf_cpu_notifier(cstate_cpu_notifier);
	594
	595	cpu_notifier_register_done();
	596	}
	597
7ce1346a KL	598	static void __init cstate_pmus_register(void)
	599	{
	600	int err;
	601
	602	if (has_cstate_core) {
	603	err = perf_pmu_register(&cstate_core_pmu, cstate_core_pmu.name, -1);
	604	if (WARN_ON(err))
	605	pr_info("Failed to register PMU %s error %d\n",
	606	cstate_core_pmu.name, err);
	607	}
	608
	609	if (has_cstate_pkg) {
	610	err = perf_pmu_register(&cstate_pkg_pmu, cstate_pkg_pmu.name, -1);
	611	if (WARN_ON(err))
	612	pr_info("Failed to register PMU %s error %d\n",
	613	cstate_pkg_pmu.name, err);
	614	}
	615	}
	616
	617	static int __init cstate_pmu_init(void)
	618	{
424646ee	619	const struct x86_cpu_id *id;
7ce1346a KL	620	int err;
7ce1346a KL	621
424646ee	622	if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
7ce1346a KL	623	return -ENODEV;
7ce1346a KL	624
424646ee TG	625	id = x86_match_cpu(intel_cstates_match);
	626	if (!id)
	627	return -ENODEV;
	628
	629	err = cstate_probe((const struct cstate_model *) id->driver_data);
7ce1346a KL	630	if (err)
	631	return err;
	632
	633	cstate_cpumask_init();
	634
	635	cstate_pmus_register();
	636
	637	return 0;
	638	}
7ce1346a	639	device_initcall(cstate_pmu_init);