[mirror_ubuntu-artful-kernel.git] / virt / kvm / arm / pmu.c

/*
 * Copyright (C) 2015 Linaro Ltd.
 * Author: Shannon Zhao <shannon.zhao@linaro.org>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <linux/cpu.h>
#include <linux/kvm.h>
#include <linux/kvm_host.h>
#include <linux/perf_event.h>
#include <linux/uaccess.h>
#include <asm/kvm_emulate.h>
#include <kvm/arm_pmu.h>
#include <kvm/arm_vgic.h>

/**
 * kvm_pmu_get_counter_value - get PMU counter value
 * @vcpu: The vcpu pointer
 * @select_idx: The counter index
 */
u64 kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu, u64 select_idx)
{
	u64 counter, reg, enabled, running;
	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	struct kvm_pmc *pmc = &pmu->pmc[select_idx];

	reg = (select_idx == ARMV8_PMU_CYCLE_IDX)
	      ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + select_idx;
	counter = vcpu_sys_reg(vcpu, reg);

	/* The real counter value is equal to the value of counter register plus
	 * the value perf event counts.
	 */
	if (pmc->perf_event)
		counter += perf_event_read_value(pmc->perf_event, &enabled,
						 &running);

	return counter & pmc->bitmask;
}

/**
 * kvm_pmu_set_counter_value - set PMU counter value
 * @vcpu: The vcpu pointer
 * @select_idx: The counter index
 * @val: The counter value
 */
void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, u64 select_idx, u64 val)
{
	u64 reg;

	reg = (select_idx == ARMV8_PMU_CYCLE_IDX)
	      ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + select_idx;
	vcpu_sys_reg(vcpu, reg) += (s64)val - kvm_pmu_get_counter_value(vcpu, select_idx);
}

/**
 * kvm_pmu_stop_counter - stop PMU counter
 * @pmc: The PMU counter pointer
 *
 * If this counter has been configured to monitor some event, release it here.
 */
static void kvm_pmu_stop_counter(struct kvm_vcpu *vcpu, struct kvm_pmc *pmc)
{
	u64 counter, reg;

	if (pmc->perf_event) {
		counter = kvm_pmu_get_counter_value(vcpu, pmc->idx);
		reg = (pmc->idx == ARMV8_PMU_CYCLE_IDX)
		       ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + pmc->idx;
		vcpu_sys_reg(vcpu, reg) = counter;
		perf_event_disable(pmc->perf_event);
		perf_event_release_kernel(pmc->perf_event);
		pmc->perf_event = NULL;
	}
}

/**
 * kvm_pmu_vcpu_reset - reset pmu state for cpu
 * @vcpu: The vcpu pointer
 *
 */
void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu)
{
	int i;
	struct kvm_pmu *pmu = &vcpu->arch.pmu;

	for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
		kvm_pmu_stop_counter(vcpu, &pmu->pmc[i]);
		pmu->pmc[i].idx = i;
		pmu->pmc[i].bitmask = 0xffffffffUL;
	}
}

/**
 * kvm_pmu_vcpu_destroy - free perf event of PMU for cpu
 * @vcpu: The vcpu pointer
 *
 */
void kvm_pmu_vcpu_destroy(struct kvm_vcpu *vcpu)
{
	int i;
	struct kvm_pmu *pmu = &vcpu->arch.pmu;

	for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
		struct kvm_pmc *pmc = &pmu->pmc[i];

		if (pmc->perf_event) {
			perf_event_disable(pmc->perf_event);
			perf_event_release_kernel(pmc->perf_event);
			pmc->perf_event = NULL;
		}
	}
}

u64 kvm_pmu_valid_counter_mask(struct kvm_vcpu *vcpu)
{
	u64 val = vcpu_sys_reg(vcpu, PMCR_EL0) >> ARMV8_PMU_PMCR_N_SHIFT;

	val &= ARMV8_PMU_PMCR_N_MASK;
	if (val == 0)
		return BIT(ARMV8_PMU_CYCLE_IDX);
	else
		return GENMASK(val - 1, 0) | BIT(ARMV8_PMU_CYCLE_IDX);
}

/**
 * kvm_pmu_enable_counter - enable selected PMU counter
 * @vcpu: The vcpu pointer
 * @val: the value guest writes to PMCNTENSET register
 *
 * Call perf_event_enable to start counting the perf event
 */
void kvm_pmu_enable_counter(struct kvm_vcpu *vcpu, u64 val)
{
	int i;
	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	struct kvm_pmc *pmc;

	if (!(vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) || !val)
		return;

	for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
		if (!(val & BIT(i)))
			continue;

		pmc = &pmu->pmc[i];
		if (pmc->perf_event) {
			perf_event_enable(pmc->perf_event);
			if (pmc->perf_event->state != PERF_EVENT_STATE_ACTIVE)
				kvm_debug("fail to enable perf event\n");
		}
	}
}

/**
 * kvm_pmu_disable_counter - disable selected PMU counter
 * @vcpu: The vcpu pointer
 * @val: the value guest writes to PMCNTENCLR register
 *
 * Call perf_event_disable to stop counting the perf event
 */
void kvm_pmu_disable_counter(struct kvm_vcpu *vcpu, u64 val)
{
	int i;
	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	struct kvm_pmc *pmc;

	if (!val)
		return;

	for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
		if (!(val & BIT(i)))
			continue;

		pmc = &pmu->pmc[i];
		if (pmc->perf_event)
			perf_event_disable(pmc->perf_event);
	}
}

static u64 kvm_pmu_overflow_status(struct kvm_vcpu *vcpu)
{
	u64 reg = 0;

	if ((vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E))
		reg = vcpu_sys_reg(vcpu, PMOVSSET_EL0);
		reg &= vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
		reg &= vcpu_sys_reg(vcpu, PMINTENSET_EL1);
		reg &= kvm_pmu_valid_counter_mask(vcpu);

	return reg;
}

/**
 * kvm_pmu_overflow_set - set PMU overflow interrupt
 * @vcpu: The vcpu pointer
 * @val: the value guest writes to PMOVSSET register
 */
void kvm_pmu_overflow_set(struct kvm_vcpu *vcpu, u64 val)
{
	u64 reg;

	if (val == 0)
		return;

	vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= val;
	reg = kvm_pmu_overflow_status(vcpu);
	if (reg != 0)
		kvm_vcpu_kick(vcpu);
}

static void kvm_pmu_update_state(struct kvm_vcpu *vcpu)
{
	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	bool overflow;

	if (!kvm_arm_pmu_v3_ready(vcpu))
		return;

	overflow = !!kvm_pmu_overflow_status(vcpu);
	if (pmu->irq_level != overflow) {
		pmu->irq_level = overflow;
		kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
				    pmu->irq_num, overflow);
	}
}

/**
 * kvm_pmu_flush_hwstate - flush pmu state to cpu
 * @vcpu: The vcpu pointer
 *
 * Check if the PMU has overflowed while we were running in the host, and inject
 * an interrupt if that was the case.
 */
void kvm_pmu_flush_hwstate(struct kvm_vcpu *vcpu)
{
	kvm_pmu_update_state(vcpu);
}

/**
 * kvm_pmu_sync_hwstate - sync pmu state from cpu
 * @vcpu: The vcpu pointer
 *
 * Check if the PMU has overflowed while we were running in the guest, and
 * inject an interrupt if that was the case.
 */
void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu)
{
	kvm_pmu_update_state(vcpu);
}

static inline struct kvm_vcpu *kvm_pmc_to_vcpu(struct kvm_pmc *pmc)
{
	struct kvm_pmu *pmu;
	struct kvm_vcpu_arch *vcpu_arch;

	pmc -= pmc->idx;
	pmu = container_of(pmc, struct kvm_pmu, pmc[0]);
	vcpu_arch = container_of(pmu, struct kvm_vcpu_arch, pmu);
	return container_of(vcpu_arch, struct kvm_vcpu, arch);
}

/**
 * When perf event overflows, call kvm_pmu_overflow_set to set overflow status.
 */
static void kvm_pmu_perf_overflow(struct perf_event *perf_event,
				  struct perf_sample_data *data,
				  struct pt_regs *regs)
{
	struct kvm_pmc *pmc = perf_event->overflow_handler_context;
	struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
	int idx = pmc->idx;

	kvm_pmu_overflow_set(vcpu, BIT(idx));
}

/**
 * kvm_pmu_software_increment - do software increment
 * @vcpu: The vcpu pointer
 * @val: the value guest writes to PMSWINC register
 */
void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val)
{
	int i;
	u64 type, enable, reg;

	if (val == 0)
		return;

	enable = vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
	for (i = 0; i < ARMV8_PMU_CYCLE_IDX; i++) {
		if (!(val & BIT(i)))
			continue;
		type = vcpu_sys_reg(vcpu, PMEVTYPER0_EL0 + i)
		       & ARMV8_PMU_EVTYPE_EVENT;
		if ((type == ARMV8_PMU_EVTYPE_EVENT_SW_INCR)
		    && (enable & BIT(i))) {
			reg = vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i) + 1;
			reg = lower_32_bits(reg);
			vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i) = reg;
			if (!reg)
				kvm_pmu_overflow_set(vcpu, BIT(i));
		}
	}
}

/**
 * kvm_pmu_handle_pmcr - handle PMCR register
 * @vcpu: The vcpu pointer
 * @val: the value guest writes to PMCR register
 */
void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val)
{
	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	struct kvm_pmc *pmc;
	u64 mask;
	int i;

	mask = kvm_pmu_valid_counter_mask(vcpu);
	if (val & ARMV8_PMU_PMCR_E) {
		kvm_pmu_enable_counter(vcpu,
				vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & mask);
	} else {
		kvm_pmu_disable_counter(vcpu, mask);
	}

	if (val & ARMV8_PMU_PMCR_C)
		kvm_pmu_set_counter_value(vcpu, ARMV8_PMU_CYCLE_IDX, 0);

	if (val & ARMV8_PMU_PMCR_P) {
		for (i = 0; i < ARMV8_PMU_CYCLE_IDX; i++)
			kvm_pmu_set_counter_value(vcpu, i, 0);
	}

	if (val & ARMV8_PMU_PMCR_LC) {
		pmc = &pmu->pmc[ARMV8_PMU_CYCLE_IDX];
		pmc->bitmask = 0xffffffffffffffffUL;
	}
}

static bool kvm_pmu_counter_is_enabled(struct kvm_vcpu *vcpu, u64 select_idx)
{
	return (vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) &&
	       (vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & BIT(select_idx));
}

/**
 * kvm_pmu_set_counter_event_type - set selected counter to monitor some event
 * @vcpu: The vcpu pointer
 * @data: The data guest writes to PMXEVTYPER_EL0
 * @select_idx: The number of selected counter
 *
 * When OS accesses PMXEVTYPER_EL0, that means it wants to set a PMC to count an
 * event with given hardware event number. Here we call perf_event API to
 * emulate this action and create a kernel perf event for it.
 */
void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data,
				    u64 select_idx)
{
	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
	struct perf_event *event;
	struct perf_event_attr attr;
	u64 eventsel, counter;

	kvm_pmu_stop_counter(vcpu, pmc);
	eventsel = data & ARMV8_PMU_EVTYPE_EVENT;

	/* Software increment event does't need to be backed by a perf event */
	if (eventsel == ARMV8_PMU_EVTYPE_EVENT_SW_INCR)
		return;

	memset(&attr, 0, sizeof(struct perf_event_attr));
	attr.type = PERF_TYPE_RAW;
	attr.size = sizeof(attr);
	attr.pinned = 1;
	attr.disabled = !kvm_pmu_counter_is_enabled(vcpu, select_idx);
	attr.exclude_user = data & ARMV8_PMU_EXCLUDE_EL0 ? 1 : 0;
	attr.exclude_kernel = data & ARMV8_PMU_EXCLUDE_EL1 ? 1 : 0;
	attr.exclude_hv = 1; /* Don't count EL2 events */
	attr.exclude_host = 1; /* Don't count host events */
	attr.config = eventsel;

	counter = kvm_pmu_get_counter_value(vcpu, select_idx);
	/* The initial sample period (overflow count) of an event. */
	attr.sample_period = (-counter) & pmc->bitmask;

	event = perf_event_create_kernel_counter(&attr, -1, current,
						 kvm_pmu_perf_overflow, pmc);
	if (IS_ERR(event)) {
		pr_err_once("kvm: pmu event creation failed %ld\n",
			    PTR_ERR(event));
		return;
	}

	pmc->perf_event = event;
}

bool kvm_arm_support_pmu_v3(void)
{
	/*
	 * Check if HW_PERF_EVENTS are supported by checking the number of
	 * hardware performance counters. This could ensure the presence of
	 * a physical PMU and CONFIG_PERF_EVENT is selected.
	 */
	return (perf_num_counters() > 0);
}

static int kvm_arm_pmu_v3_init(struct kvm_vcpu *vcpu)
{
	if (!kvm_arm_support_pmu_v3())
		return -ENODEV;

	if (!test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features) ||
	    !kvm_arm_pmu_irq_initialized(vcpu))
		return -ENXIO;

	if (kvm_arm_pmu_v3_ready(vcpu))
		return -EBUSY;

	kvm_pmu_vcpu_reset(vcpu);
	vcpu->arch.pmu.ready = true;

	return 0;
}

static bool irq_is_valid(struct kvm *kvm, int irq, bool is_ppi)
{
	int i;
	struct kvm_vcpu *vcpu;

	kvm_for_each_vcpu(i, vcpu, kvm) {
		if (!kvm_arm_pmu_irq_initialized(vcpu))
			continue;

		if (is_ppi) {
			if (vcpu->arch.pmu.irq_num != irq)
				return false;
		} else {
			if (vcpu->arch.pmu.irq_num == irq)
				return false;
		}
	}

	return true;
}


int kvm_arm_pmu_v3_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
{
	switch (attr->attr) {
	case KVM_ARM_VCPU_PMU_V3_IRQ: {
		int __user *uaddr = (int __user *)(long)attr->addr;
		int irq;

		if (!test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))
			return -ENODEV;

		if (get_user(irq, uaddr))
			return -EFAULT;

		/*
		 * The PMU overflow interrupt could be a PPI or SPI, but for one
		 * VM the interrupt type must be same for each vcpu. As a PPI,
		 * the interrupt number is the same for all vcpus, while as an
		 * SPI it must be a separate number per vcpu.
		 */
		if (irq < VGIC_NR_SGIS || irq >= vcpu->kvm->arch.vgic.nr_irqs ||
		    !irq_is_valid(vcpu->kvm, irq, irq < VGIC_NR_PRIVATE_IRQS))
			return -EINVAL;

		if (kvm_arm_pmu_irq_initialized(vcpu))
			return -EBUSY;

		kvm_debug("Set kvm ARM PMU irq: %d\n", irq);
		vcpu->arch.pmu.irq_num = irq;
		return 0;
	}
	case KVM_ARM_VCPU_PMU_V3_INIT:
		return kvm_arm_pmu_v3_init(vcpu);
	}

	return -ENXIO;
}

int kvm_arm_pmu_v3_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
{
	switch (attr->attr) {
	case KVM_ARM_VCPU_PMU_V3_IRQ: {
		int __user *uaddr = (int __user *)(long)attr->addr;
		int irq;

		if (!test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))
			return -ENODEV;

		if (!kvm_arm_pmu_irq_initialized(vcpu))
			return -ENXIO;

		irq = vcpu->arch.pmu.irq_num;
		return put_user(irq, uaddr);
	}
	}

	return -ENXIO;
}

int kvm_arm_pmu_v3_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
{
	switch (attr->attr) {
	case KVM_ARM_VCPU_PMU_V3_IRQ:
	case KVM_ARM_VCPU_PMU_V3_INIT:
		if (kvm_arm_support_pmu_v3() &&
		    test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))
			return 0;
	}

	return -ENXIO;
}
Commit	Line	Data
051ff581 SZ	1	/*
	2	* Copyright (C) 2015 Linaro Ltd.
	3	* Author: Shannon Zhao <shannon.zhao@linaro.org>
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License version 2 as
	7	* published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	* GNU General Public License for more details.
	13	*
	14	* You should have received a copy of the GNU General Public License
	15	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	16	*/
	17
	18	#include <linux/cpu.h>
	19	#include <linux/kvm.h>
	20	#include <linux/kvm_host.h>
	21	#include <linux/perf_event.h>
bb0c70bc	22	#include <linux/uaccess.h>
051ff581 SZ	23	#include <asm/kvm_emulate.h>
051ff581 SZ	24	#include <kvm/arm_pmu.h>
b02386eb	25	#include <kvm/arm_vgic.h>
051ff581 SZ	26
	27	/**
	28	* kvm_pmu_get_counter_value - get PMU counter value
	29	* @vcpu: The vcpu pointer
	30	* @select_idx: The counter index
	31	*/
	32	u64 kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu, u64 select_idx)
	33	{
	34	u64 counter, reg, enabled, running;
	35	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	36	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
	37
	38	reg = (select_idx == ARMV8_PMU_CYCLE_IDX)
	39	? PMCCNTR_EL0 : PMEVCNTR0_EL0 + select_idx;
	40	counter = vcpu_sys_reg(vcpu, reg);
	41
	42	/* The real counter value is equal to the value of counter register plus
	43	* the value perf event counts.
	44	*/
	45	if (pmc->perf_event)
	46	counter += perf_event_read_value(pmc->perf_event, &enabled,
	47	&running);
	48
	49	return counter & pmc->bitmask;
	50	}
	51
	52	/**
	53	* kvm_pmu_set_counter_value - set PMU counter value
	54	* @vcpu: The vcpu pointer
	55	* @select_idx: The counter index
	56	* @val: The counter value
	57	*/
	58	void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, u64 select_idx, u64 val)
	59	{
	60	u64 reg;
	61
	62	reg = (select_idx == ARMV8_PMU_CYCLE_IDX)
	63	? PMCCNTR_EL0 : PMEVCNTR0_EL0 + select_idx;
	64	vcpu_sys_reg(vcpu, reg) += (s64)val - kvm_pmu_get_counter_value(vcpu, select_idx);
	65	}
96b0eebc	66
7f766358 SZ	67	/**
	68	* kvm_pmu_stop_counter - stop PMU counter
	69	* @pmc: The PMU counter pointer
	70	*
	71	* If this counter has been configured to monitor some event, release it here.
	72	*/
	73	static void kvm_pmu_stop_counter(struct kvm_vcpu vcpu, struct kvm_pmc pmc)
	74	{
	75	u64 counter, reg;
	76
	77	if (pmc->perf_event) {
	78	counter = kvm_pmu_get_counter_value(vcpu, pmc->idx);
	79	reg = (pmc->idx == ARMV8_PMU_CYCLE_IDX)
	80	? PMCCNTR_EL0 : PMEVCNTR0_EL0 + pmc->idx;
	81	vcpu_sys_reg(vcpu, reg) = counter;
	82	perf_event_disable(pmc->perf_event);
	83	perf_event_release_kernel(pmc->perf_event);
	84	pmc->perf_event = NULL;
	85	}
	86	}
	87
2aa36e98 SZ	88	/**
	89	* kvm_pmu_vcpu_reset - reset pmu state for cpu
	90	* @vcpu: The vcpu pointer
	91	*
	92	*/
	93	void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu)
	94	{
	95	int i;
	96	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	97
	98	for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
	99	kvm_pmu_stop_counter(vcpu, &pmu->pmc[i]);
	100	pmu->pmc[i].idx = i;
	101	pmu->pmc[i].bitmask = 0xffffffffUL;
	102	}
	103	}
	104
5f0a714a SZ	105	/**
	106	* kvm_pmu_vcpu_destroy - free perf event of PMU for cpu
	107	* @vcpu: The vcpu pointer
	108	*
	109	*/
	110	void kvm_pmu_vcpu_destroy(struct kvm_vcpu *vcpu)
	111	{
	112	int i;
	113	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	114
	115	for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
	116	struct kvm_pmc *pmc = &pmu->pmc[i];
	117
	118	if (pmc->perf_event) {
	119	perf_event_disable(pmc->perf_event);
	120	perf_event_release_kernel(pmc->perf_event);
	121	pmc->perf_event = NULL;
	122	}
	123	}
	124	}
	125
96b0eebc SZ	126	u64 kvm_pmu_valid_counter_mask(struct kvm_vcpu *vcpu)
	127	{
	128	u64 val = vcpu_sys_reg(vcpu, PMCR_EL0) >> ARMV8_PMU_PMCR_N_SHIFT;
	129
	130	val &= ARMV8_PMU_PMCR_N_MASK;
	131	if (val == 0)
	132	return BIT(ARMV8_PMU_CYCLE_IDX);
	133	else
	134	return GENMASK(val - 1, 0) \| BIT(ARMV8_PMU_CYCLE_IDX);
	135	}
	136
	137	/**
	138	* kvm_pmu_enable_counter - enable selected PMU counter
	139	* @vcpu: The vcpu pointer
	140	* @val: the value guest writes to PMCNTENSET register
	141	*
	142	* Call perf_event_enable to start counting the perf event
	143	*/
	144	void kvm_pmu_enable_counter(struct kvm_vcpu *vcpu, u64 val)
	145	{
	146	int i;
	147	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	148	struct kvm_pmc *pmc;
	149
	150	if (!(vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) \|\| !val)
	151	return;
	152
	153	for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
	154	if (!(val & BIT(i)))
	155	continue;
	156
	157	pmc = &pmu->pmc[i];
	158	if (pmc->perf_event) {
	159	perf_event_enable(pmc->perf_event);
	160	if (pmc->perf_event->state != PERF_EVENT_STATE_ACTIVE)
	161	kvm_debug("fail to enable perf event\n");
	162	}
	163	}
	164	}
	165
	166	/**
	167	* kvm_pmu_disable_counter - disable selected PMU counter
	168	* @vcpu: The vcpu pointer
	169	* @val: the value guest writes to PMCNTENCLR register
	170	*
	171	* Call perf_event_disable to stop counting the perf event
	172	*/
	173	void kvm_pmu_disable_counter(struct kvm_vcpu *vcpu, u64 val)
	174	{
	175	int i;
	176	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	177	struct kvm_pmc *pmc;
	178
	179	if (!val)
	180	return;
	181
	182	for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
	183	if (!(val & BIT(i)))
	184	continue;
	185
	186	pmc = &pmu->pmc[i];
	187	if (pmc->perf_event)
	188	perf_event_disable(pmc->perf_event);
	189	}
190	}
7f766358	191
76d883c4 SZ	192	static u64 kvm_pmu_overflow_status(struct kvm_vcpu *vcpu)
	193	{
	194	u64 reg = 0;
	195
	196	if ((vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E))
	197	reg = vcpu_sys_reg(vcpu, PMOVSSET_EL0);
	198	reg &= vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
	199	reg &= vcpu_sys_reg(vcpu, PMINTENSET_EL1);
	200	reg &= kvm_pmu_valid_counter_mask(vcpu);
	201
	202	return reg;
	203	}
	204
	205	/**
	206	* kvm_pmu_overflow_set - set PMU overflow interrupt
	207	* @vcpu: The vcpu pointer
	208	* @val: the value guest writes to PMOVSSET register
	209	*/
	210	void kvm_pmu_overflow_set(struct kvm_vcpu *vcpu, u64 val)
	211	{
	212	u64 reg;
	213
	214	if (val == 0)
	215	return;
	216
	217	vcpu_sys_reg(vcpu, PMOVSSET_EL0) \|= val;
	218	reg = kvm_pmu_overflow_status(vcpu);
	219	if (reg != 0)
	220	kvm_vcpu_kick(vcpu);
	221	}
	222
b02386eb SZ	223	static void kvm_pmu_update_state(struct kvm_vcpu *vcpu)
	224	{
	225	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	226	bool overflow;
	227
	228	if (!kvm_arm_pmu_v3_ready(vcpu))
	229	return;
	230
	231	overflow = !!kvm_pmu_overflow_status(vcpu);
	232	if (pmu->irq_level != overflow) {
	233	pmu->irq_level = overflow;
	234	kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
	235	pmu->irq_num, overflow);
	236	}
	237	}
	238
	239	/**
	240	* kvm_pmu_flush_hwstate - flush pmu state to cpu
	241	* @vcpu: The vcpu pointer
	242	*
	243	* Check if the PMU has overflowed while we were running in the host, and inject
	244	* an interrupt if that was the case.
	245	*/
	246	void kvm_pmu_flush_hwstate(struct kvm_vcpu *vcpu)
	247	{
	248	kvm_pmu_update_state(vcpu);
	249	}
	250
	251	/**
	252	* kvm_pmu_sync_hwstate - sync pmu state from cpu
	253	* @vcpu: The vcpu pointer
	254	*
	255	* Check if the PMU has overflowed while we were running in the guest, and
	256	* inject an interrupt if that was the case.
	257	*/
	258	void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu)
	259	{
	260	kvm_pmu_update_state(vcpu);
	261	}
	262
	263	static inline struct kvm_vcpu kvm_pmc_to_vcpu(struct kvm_pmc pmc)
	264	{
	265	struct kvm_pmu *pmu;
	266	struct kvm_vcpu_arch *vcpu_arch;
	267
	268	pmc -= pmc->idx;
	269	pmu = container_of(pmc, struct kvm_pmu, pmc[0]);
	270	vcpu_arch = container_of(pmu, struct kvm_vcpu_arch, pmu);
	271	return container_of(vcpu_arch, struct kvm_vcpu, arch);
	272	}
	273
	274	/**
	275	* When perf event overflows, call kvm_pmu_overflow_set to set overflow status.
	276	*/
	277	static void kvm_pmu_perf_overflow(struct perf_event *perf_event,
	278	struct perf_sample_data *data,
	279	struct pt_regs *regs)
	280	{
	281	struct kvm_pmc *pmc = perf_event->overflow_handler_context;
	282	struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
	283	int idx = pmc->idx;
	284
	285	kvm_pmu_overflow_set(vcpu, BIT(idx));
	286	}
287
7a0adc70 SZ	288	/**
	289	* kvm_pmu_software_increment - do software increment
	290	* @vcpu: The vcpu pointer
	291	* @val: the value guest writes to PMSWINC register
	292	*/
	293	void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val)
	294	{
	295	int i;
	296	u64 type, enable, reg;
	297
	298	if (val == 0)
	299	return;
	300
	301	enable = vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
	302	for (i = 0; i < ARMV8_PMU_CYCLE_IDX; i++) {
	303	if (!(val & BIT(i)))
	304	continue;
	305	type = vcpu_sys_reg(vcpu, PMEVTYPER0_EL0 + i)
	306	& ARMV8_PMU_EVTYPE_EVENT;
	307	if ((type == ARMV8_PMU_EVTYPE_EVENT_SW_INCR)
	308	&& (enable & BIT(i))) {
	309	reg = vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i) + 1;
	310	reg = lower_32_bits(reg);
	311	vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i) = reg;
	312	if (!reg)
	313	kvm_pmu_overflow_set(vcpu, BIT(i));
	314	}
	315	}
	316	}
	317
76993739 SZ	318	/**
	319	* kvm_pmu_handle_pmcr - handle PMCR register
	320	* @vcpu: The vcpu pointer
	321	* @val: the value guest writes to PMCR register
	322	*/
	323	void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val)
	324	{
	325	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	326	struct kvm_pmc *pmc;
	327	u64 mask;
	328	int i;
	329
	330	mask = kvm_pmu_valid_counter_mask(vcpu);
	331	if (val & ARMV8_PMU_PMCR_E) {
	332	kvm_pmu_enable_counter(vcpu,
	333	vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & mask);
	334	} else {
	335	kvm_pmu_disable_counter(vcpu, mask);
	336	}
	337
	338	if (val & ARMV8_PMU_PMCR_C)
	339	kvm_pmu_set_counter_value(vcpu, ARMV8_PMU_CYCLE_IDX, 0);
	340
	341	if (val & ARMV8_PMU_PMCR_P) {
	342	for (i = 0; i < ARMV8_PMU_CYCLE_IDX; i++)
	343	kvm_pmu_set_counter_value(vcpu, i, 0);
	344	}
	345
	346	if (val & ARMV8_PMU_PMCR_LC) {
	347	pmc = &pmu->pmc[ARMV8_PMU_CYCLE_IDX];
	348	pmc->bitmask = 0xffffffffffffffffUL;
	349	}
	350	}
	351
7f766358 SZ	352	static bool kvm_pmu_counter_is_enabled(struct kvm_vcpu *vcpu, u64 select_idx)
	353	{
	354	return (vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) &&
	355	(vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & BIT(select_idx));
	356	}
	357
	358	/**
	359	* kvm_pmu_set_counter_event_type - set selected counter to monitor some event
	360	* @vcpu: The vcpu pointer
	361	* @data: The data guest writes to PMXEVTYPER_EL0
	362	* @select_idx: The number of selected counter
	363	*
	364	* When OS accesses PMXEVTYPER_EL0, that means it wants to set a PMC to count an
	365	* event with given hardware event number. Here we call perf_event API to
	366	* emulate this action and create a kernel perf event for it.
	367	*/
	368	void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data,
	369	u64 select_idx)
	370	{
	371	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	372	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
	373	struct perf_event *event;
	374	struct perf_event_attr attr;
	375	u64 eventsel, counter;
	376
	377	kvm_pmu_stop_counter(vcpu, pmc);
	378	eventsel = data & ARMV8_PMU_EVTYPE_EVENT;
	379
7a0adc70 SZ	380	/* Software increment event does't need to be backed by a perf event */
	381	if (eventsel == ARMV8_PMU_EVTYPE_EVENT_SW_INCR)
	382	return;
	383
7f766358 SZ	384	memset(&attr, 0, sizeof(struct perf_event_attr));
	385	attr.type = PERF_TYPE_RAW;
	386	attr.size = sizeof(attr);
	387	attr.pinned = 1;
	388	attr.disabled = !kvm_pmu_counter_is_enabled(vcpu, select_idx);
	389	attr.exclude_user = data & ARMV8_PMU_EXCLUDE_EL0 ? 1 : 0;
	390	attr.exclude_kernel = data & ARMV8_PMU_EXCLUDE_EL1 ? 1 : 0;
	391	attr.exclude_hv = 1; /* Don't count EL2 events */
	392	attr.exclude_host = 1; /* Don't count host events */
	393	attr.config = eventsel;
	394
	395	counter = kvm_pmu_get_counter_value(vcpu, select_idx);
	396	/* The initial sample period (overflow count) of an event. */
	397	attr.sample_period = (-counter) & pmc->bitmask;
	398
b02386eb SZ	399	event = perf_event_create_kernel_counter(&attr, -1, current,
b02386eb SZ	400	kvm_pmu_perf_overflow, pmc);
7f766358 SZ	401	if (IS_ERR(event)) {
	402	pr_err_once("kvm: pmu event creation failed %ld\n",
	403	PTR_ERR(event));
	404	return;
	405	}
	406
	407	pmc->perf_event = event;
	408	}
808e7381 SZ	409
	410	bool kvm_arm_support_pmu_v3(void)
	411	{
	412	/*
	413	* Check if HW_PERF_EVENTS are supported by checking the number of
	414	* hardware performance counters. This could ensure the presence of
	415	* a physical PMU and CONFIG_PERF_EVENT is selected.
	416	*/
	417	return (perf_num_counters() > 0);
	418	}
bb0c70bc SZ	419
	420	static int kvm_arm_pmu_v3_init(struct kvm_vcpu *vcpu)
	421	{
	422	if (!kvm_arm_support_pmu_v3())
	423	return -ENODEV;
	424
	425	if (!test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features) \|\|
	426	!kvm_arm_pmu_irq_initialized(vcpu))
	427	return -ENXIO;
	428
	429	if (kvm_arm_pmu_v3_ready(vcpu))
	430	return -EBUSY;
	431
	432	kvm_pmu_vcpu_reset(vcpu);
	433	vcpu->arch.pmu.ready = true;
	434
	435	return 0;
	436	}
	437
	438	static bool irq_is_valid(struct kvm *kvm, int irq, bool is_ppi)
	439	{
	440	int i;
	441	struct kvm_vcpu *vcpu;
	442
	443	kvm_for_each_vcpu(i, vcpu, kvm) {
	444	if (!kvm_arm_pmu_irq_initialized(vcpu))
	445	continue;
	446
	447	if (is_ppi) {
	448	if (vcpu->arch.pmu.irq_num != irq)
	449	return false;
	450	} else {
	451	if (vcpu->arch.pmu.irq_num == irq)
	452	return false;
	453	}
	454	}
	455
	456	return true;
	457	}
	458
	459
	460	int kvm_arm_pmu_v3_set_attr(struct kvm_vcpu vcpu, struct kvm_device_attr attr)
	461	{
	462	switch (attr->attr) {
	463	case KVM_ARM_VCPU_PMU_V3_IRQ: {
	464	int __user uaddr = (int __user )(long)attr->addr;
	465	int irq;
	466
	467	if (!test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))
	468	return -ENODEV;
	469
	470	if (get_user(irq, uaddr))
	471	return -EFAULT;
	472
	473	/*
	474	* The PMU overflow interrupt could be a PPI or SPI, but for one
	475	* VM the interrupt type must be same for each vcpu. As a PPI,
	476	* the interrupt number is the same for all vcpus, while as an
	477	* SPI it must be a separate number per vcpu.
	478	*/
	479	if (irq < VGIC_NR_SGIS \|\| irq >= vcpu->kvm->arch.vgic.nr_irqs \|\|
	480	!irq_is_valid(vcpu->kvm, irq, irq < VGIC_NR_PRIVATE_IRQS))
	481	return -EINVAL;
	482
483	if (kvm_arm_pmu_irq_initialized(vcpu))
484	return -EBUSY;
485
486	kvm_debug("Set kvm ARM PMU irq: %d\n", irq);
487	vcpu->arch.pmu.irq_num = irq;
488	return 0;
489	}
490	case KVM_ARM_VCPU_PMU_V3_INIT:
491	return kvm_arm_pmu_v3_init(vcpu);
492	}
493
494	return -ENXIO;
495	}
496
497	int kvm_arm_pmu_v3_get_attr(struct kvm_vcpu vcpu, struct kvm_device_attr attr)
498	{
499	switch (attr->attr) {
500	case KVM_ARM_VCPU_PMU_V3_IRQ: {
501	int __user uaddr = (int __user )(long)attr->addr;
502	int irq;
503
504	if (!test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))
505	return -ENODEV;
506
507	if (!kvm_arm_pmu_irq_initialized(vcpu))
508	return -ENXIO;
509
510	irq = vcpu->arch.pmu.irq_num;
511	return put_user(irq, uaddr);
512	}
513	}
514
515	return -ENXIO;
516	}
517
518	int kvm_arm_pmu_v3_has_attr(struct kvm_vcpu vcpu, struct kvm_device_attr attr)
519	{
520	switch (attr->attr) {
521	case KVM_ARM_VCPU_PMU_V3_IRQ:
522	case KVM_ARM_VCPU_PMU_V3_INIT:
523	if (kvm_arm_support_pmu_v3() &&
524	test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))
525	return 0;
526	}
527
528	return -ENXIO;
529	}