[mirror_ubuntu-hirsute-kernel.git] / virt / kvm / arm / pmu.c

/*
 * Copyright (C) 2015 Linaro Ltd.
 * Author: Shannon Zhao <shannon.zhao@linaro.org>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <linux/cpu.h>
#include <linux/kvm.h>
#include <linux/kvm_host.h>
#include <linux/perf_event.h>
#include <asm/kvm_emulate.h>
#include <kvm/arm_pmu.h>
#include <kvm/arm_vgic.h>

/**
 * kvm_pmu_get_counter_value - get PMU counter value
 * @vcpu: The vcpu pointer
 * @select_idx: The counter index
 */
u64 kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu, u64 select_idx)
{
	u64 counter, reg, enabled, running;
	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	struct kvm_pmc *pmc = &pmu->pmc[select_idx];

	reg = (select_idx == ARMV8_PMU_CYCLE_IDX)
	      ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + select_idx;
	counter = vcpu_sys_reg(vcpu, reg);

	/* The real counter value is equal to the value of counter register plus
	 * the value perf event counts.
	 */
	if (pmc->perf_event)
		counter += perf_event_read_value(pmc->perf_event, &enabled,
						 &running);

	return counter & pmc->bitmask;
}

/**
 * kvm_pmu_set_counter_value - set PMU counter value
 * @vcpu: The vcpu pointer
 * @select_idx: The counter index
 * @val: The counter value
 */
void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, u64 select_idx, u64 val)
{
	u64 reg;

	reg = (select_idx == ARMV8_PMU_CYCLE_IDX)
	      ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + select_idx;
	vcpu_sys_reg(vcpu, reg) += (s64)val - kvm_pmu_get_counter_value(vcpu, select_idx);
}

/**
 * kvm_pmu_stop_counter - stop PMU counter
 * @pmc: The PMU counter pointer
 *
 * If this counter has been configured to monitor some event, release it here.
 */
static void kvm_pmu_stop_counter(struct kvm_vcpu *vcpu, struct kvm_pmc *pmc)
{
	u64 counter, reg;

	if (pmc->perf_event) {
		counter = kvm_pmu_get_counter_value(vcpu, pmc->idx);
		reg = (pmc->idx == ARMV8_PMU_CYCLE_IDX)
		       ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + pmc->idx;
		vcpu_sys_reg(vcpu, reg) = counter;
		perf_event_disable(pmc->perf_event);
		perf_event_release_kernel(pmc->perf_event);
		pmc->perf_event = NULL;
	}
}

/**
 * kvm_pmu_vcpu_reset - reset pmu state for cpu
 * @vcpu: The vcpu pointer
 *
 */
void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu)
{
	int i;
	struct kvm_pmu *pmu = &vcpu->arch.pmu;

	for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
		kvm_pmu_stop_counter(vcpu, &pmu->pmc[i]);
		pmu->pmc[i].idx = i;
		pmu->pmc[i].bitmask = 0xffffffffUL;
	}
}

u64 kvm_pmu_valid_counter_mask(struct kvm_vcpu *vcpu)
{
	u64 val = vcpu_sys_reg(vcpu, PMCR_EL0) >> ARMV8_PMU_PMCR_N_SHIFT;

	val &= ARMV8_PMU_PMCR_N_MASK;
	if (val == 0)
		return BIT(ARMV8_PMU_CYCLE_IDX);
	else
		return GENMASK(val - 1, 0) | BIT(ARMV8_PMU_CYCLE_IDX);
}

/**
 * kvm_pmu_enable_counter - enable selected PMU counter
 * @vcpu: The vcpu pointer
 * @val: the value guest writes to PMCNTENSET register
 *
 * Call perf_event_enable to start counting the perf event
 */
void kvm_pmu_enable_counter(struct kvm_vcpu *vcpu, u64 val)
{
	int i;
	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	struct kvm_pmc *pmc;

	if (!(vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) || !val)
		return;

	for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
		if (!(val & BIT(i)))
			continue;

		pmc = &pmu->pmc[i];
		if (pmc->perf_event) {
			perf_event_enable(pmc->perf_event);
			if (pmc->perf_event->state != PERF_EVENT_STATE_ACTIVE)
				kvm_debug("fail to enable perf event\n");
		}
	}
}

/**
 * kvm_pmu_disable_counter - disable selected PMU counter
 * @vcpu: The vcpu pointer
 * @val: the value guest writes to PMCNTENCLR register
 *
 * Call perf_event_disable to stop counting the perf event
 */
void kvm_pmu_disable_counter(struct kvm_vcpu *vcpu, u64 val)
{
	int i;
	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	struct kvm_pmc *pmc;

	if (!val)
		return;

	for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
		if (!(val & BIT(i)))
			continue;

		pmc = &pmu->pmc[i];
		if (pmc->perf_event)
			perf_event_disable(pmc->perf_event);
	}
}

static u64 kvm_pmu_overflow_status(struct kvm_vcpu *vcpu)
{
	u64 reg = 0;

	if ((vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E))
		reg = vcpu_sys_reg(vcpu, PMOVSSET_EL0);
		reg &= vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
		reg &= vcpu_sys_reg(vcpu, PMINTENSET_EL1);
		reg &= kvm_pmu_valid_counter_mask(vcpu);

	return reg;
}

/**
 * kvm_pmu_overflow_set - set PMU overflow interrupt
 * @vcpu: The vcpu pointer
 * @val: the value guest writes to PMOVSSET register
 */
void kvm_pmu_overflow_set(struct kvm_vcpu *vcpu, u64 val)
{
	u64 reg;

	if (val == 0)
		return;

	vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= val;
	reg = kvm_pmu_overflow_status(vcpu);
	if (reg != 0)
		kvm_vcpu_kick(vcpu);
}

static void kvm_pmu_update_state(struct kvm_vcpu *vcpu)
{
	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	bool overflow;

	if (!kvm_arm_pmu_v3_ready(vcpu))
		return;

	overflow = !!kvm_pmu_overflow_status(vcpu);
	if (pmu->irq_level != overflow) {
		pmu->irq_level = overflow;
		kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
				    pmu->irq_num, overflow);
	}
}

/**
 * kvm_pmu_flush_hwstate - flush pmu state to cpu
 * @vcpu: The vcpu pointer
 *
 * Check if the PMU has overflowed while we were running in the host, and inject
 * an interrupt if that was the case.
 */
void kvm_pmu_flush_hwstate(struct kvm_vcpu *vcpu)
{
	kvm_pmu_update_state(vcpu);
}

/**
 * kvm_pmu_sync_hwstate - sync pmu state from cpu
 * @vcpu: The vcpu pointer
 *
 * Check if the PMU has overflowed while we were running in the guest, and
 * inject an interrupt if that was the case.
 */
void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu)
{
	kvm_pmu_update_state(vcpu);
}

static inline struct kvm_vcpu *kvm_pmc_to_vcpu(struct kvm_pmc *pmc)
{
	struct kvm_pmu *pmu;
	struct kvm_vcpu_arch *vcpu_arch;

	pmc -= pmc->idx;
	pmu = container_of(pmc, struct kvm_pmu, pmc[0]);
	vcpu_arch = container_of(pmu, struct kvm_vcpu_arch, pmu);
	return container_of(vcpu_arch, struct kvm_vcpu, arch);
}

/**
 * When perf event overflows, call kvm_pmu_overflow_set to set overflow status.
 */
static void kvm_pmu_perf_overflow(struct perf_event *perf_event,
				  struct perf_sample_data *data,
				  struct pt_regs *regs)
{
	struct kvm_pmc *pmc = perf_event->overflow_handler_context;
	struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
	int idx = pmc->idx;

	kvm_pmu_overflow_set(vcpu, BIT(idx));
}

/**
 * kvm_pmu_software_increment - do software increment
 * @vcpu: The vcpu pointer
 * @val: the value guest writes to PMSWINC register
 */
void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val)
{
	int i;
	u64 type, enable, reg;

	if (val == 0)
		return;

	enable = vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
	for (i = 0; i < ARMV8_PMU_CYCLE_IDX; i++) {
		if (!(val & BIT(i)))
			continue;
		type = vcpu_sys_reg(vcpu, PMEVTYPER0_EL0 + i)
		       & ARMV8_PMU_EVTYPE_EVENT;
		if ((type == ARMV8_PMU_EVTYPE_EVENT_SW_INCR)
		    && (enable & BIT(i))) {
			reg = vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i) + 1;
			reg = lower_32_bits(reg);
			vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i) = reg;
			if (!reg)
				kvm_pmu_overflow_set(vcpu, BIT(i));
		}
	}
}

/**
 * kvm_pmu_handle_pmcr - handle PMCR register
 * @vcpu: The vcpu pointer
 * @val: the value guest writes to PMCR register
 */
void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val)
{
	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	struct kvm_pmc *pmc;
	u64 mask;
	int i;

	mask = kvm_pmu_valid_counter_mask(vcpu);
	if (val & ARMV8_PMU_PMCR_E) {
		kvm_pmu_enable_counter(vcpu,
				vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & mask);
	} else {
		kvm_pmu_disable_counter(vcpu, mask);
	}

	if (val & ARMV8_PMU_PMCR_C)
		kvm_pmu_set_counter_value(vcpu, ARMV8_PMU_CYCLE_IDX, 0);

	if (val & ARMV8_PMU_PMCR_P) {
		for (i = 0; i < ARMV8_PMU_CYCLE_IDX; i++)
			kvm_pmu_set_counter_value(vcpu, i, 0);
	}

	if (val & ARMV8_PMU_PMCR_LC) {
		pmc = &pmu->pmc[ARMV8_PMU_CYCLE_IDX];
		pmc->bitmask = 0xffffffffffffffffUL;
	}
}

static bool kvm_pmu_counter_is_enabled(struct kvm_vcpu *vcpu, u64 select_idx)
{
	return (vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) &&
	       (vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & BIT(select_idx));
}

/**
 * kvm_pmu_set_counter_event_type - set selected counter to monitor some event
 * @vcpu: The vcpu pointer
 * @data: The data guest writes to PMXEVTYPER_EL0
 * @select_idx: The number of selected counter
 *
 * When OS accesses PMXEVTYPER_EL0, that means it wants to set a PMC to count an
 * event with given hardware event number. Here we call perf_event API to
 * emulate this action and create a kernel perf event for it.
 */
void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data,
				    u64 select_idx)
{
	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
	struct perf_event *event;
	struct perf_event_attr attr;
	u64 eventsel, counter;

	kvm_pmu_stop_counter(vcpu, pmc);
	eventsel = data & ARMV8_PMU_EVTYPE_EVENT;

	/* Software increment event does't need to be backed by a perf event */
	if (eventsel == ARMV8_PMU_EVTYPE_EVENT_SW_INCR)
		return;

	memset(&attr, 0, sizeof(struct perf_event_attr));
	attr.type = PERF_TYPE_RAW;
	attr.size = sizeof(attr);
	attr.pinned = 1;
	attr.disabled = !kvm_pmu_counter_is_enabled(vcpu, select_idx);
	attr.exclude_user = data & ARMV8_PMU_EXCLUDE_EL0 ? 1 : 0;
	attr.exclude_kernel = data & ARMV8_PMU_EXCLUDE_EL1 ? 1 : 0;
	attr.exclude_hv = 1; /* Don't count EL2 events */
	attr.exclude_host = 1; /* Don't count host events */
	attr.config = eventsel;

	counter = kvm_pmu_get_counter_value(vcpu, select_idx);
	/* The initial sample period (overflow count) of an event. */
	attr.sample_period = (-counter) & pmc->bitmask;

	event = perf_event_create_kernel_counter(&attr, -1, current,
						 kvm_pmu_perf_overflow, pmc);
	if (IS_ERR(event)) {
		pr_err_once("kvm: pmu event creation failed %ld\n",
			    PTR_ERR(event));
		return;
	}

	pmc->perf_event = event;
}
Commit	Line	Data
051ff581 SZ	1	/*
	2	* Copyright (C) 2015 Linaro Ltd.
	3	* Author: Shannon Zhao <shannon.zhao@linaro.org>
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License version 2 as
	7	* published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	* GNU General Public License for more details.
	13	*
	14	* You should have received a copy of the GNU General Public License
	15	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	16	*/
	17
	18	#include <linux/cpu.h>
	19	#include <linux/kvm.h>
	20	#include <linux/kvm_host.h>
	21	#include <linux/perf_event.h>
	22	#include <asm/kvm_emulate.h>
	23	#include <kvm/arm_pmu.h>
b02386eb	24	#include <kvm/arm_vgic.h>
051ff581 SZ	25
	26	/**
	27	* kvm_pmu_get_counter_value - get PMU counter value
	28	* @vcpu: The vcpu pointer
	29	* @select_idx: The counter index
	30	*/
	31	u64 kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu, u64 select_idx)
	32	{
	33	u64 counter, reg, enabled, running;
	34	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	35	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
	36
	37	reg = (select_idx == ARMV8_PMU_CYCLE_IDX)
	38	? PMCCNTR_EL0 : PMEVCNTR0_EL0 + select_idx;
	39	counter = vcpu_sys_reg(vcpu, reg);
	40
	41	/* The real counter value is equal to the value of counter register plus
	42	* the value perf event counts.
	43	*/
	44	if (pmc->perf_event)
	45	counter += perf_event_read_value(pmc->perf_event, &enabled,
	46	&running);
	47
	48	return counter & pmc->bitmask;
	49	}
	50
	51	/**
	52	* kvm_pmu_set_counter_value - set PMU counter value
	53	* @vcpu: The vcpu pointer
	54	* @select_idx: The counter index
	55	* @val: The counter value
	56	*/
	57	void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, u64 select_idx, u64 val)
	58	{
	59	u64 reg;
	60
	61	reg = (select_idx == ARMV8_PMU_CYCLE_IDX)
	62	? PMCCNTR_EL0 : PMEVCNTR0_EL0 + select_idx;
	63	vcpu_sys_reg(vcpu, reg) += (s64)val - kvm_pmu_get_counter_value(vcpu, select_idx);
	64	}
96b0eebc	65
7f766358 SZ	66	/**
	67	* kvm_pmu_stop_counter - stop PMU counter
	68	* @pmc: The PMU counter pointer
	69	*
	70	* If this counter has been configured to monitor some event, release it here.
	71	*/
	72	static void kvm_pmu_stop_counter(struct kvm_vcpu vcpu, struct kvm_pmc pmc)
	73	{
	74	u64 counter, reg;
	75
	76	if (pmc->perf_event) {
	77	counter = kvm_pmu_get_counter_value(vcpu, pmc->idx);
	78	reg = (pmc->idx == ARMV8_PMU_CYCLE_IDX)
	79	? PMCCNTR_EL0 : PMEVCNTR0_EL0 + pmc->idx;
	80	vcpu_sys_reg(vcpu, reg) = counter;
	81	perf_event_disable(pmc->perf_event);
	82	perf_event_release_kernel(pmc->perf_event);
	83	pmc->perf_event = NULL;
	84	}
	85	}
	86
2aa36e98 SZ	87	/**
	88	* kvm_pmu_vcpu_reset - reset pmu state for cpu
	89	* @vcpu: The vcpu pointer
	90	*
	91	*/
	92	void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu)
	93	{
	94	int i;
	95	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	96
	97	for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
	98	kvm_pmu_stop_counter(vcpu, &pmu->pmc[i]);
	99	pmu->pmc[i].idx = i;
	100	pmu->pmc[i].bitmask = 0xffffffffUL;
	101	}
	102	}
	103
96b0eebc SZ	104	u64 kvm_pmu_valid_counter_mask(struct kvm_vcpu *vcpu)
	105	{
	106	u64 val = vcpu_sys_reg(vcpu, PMCR_EL0) >> ARMV8_PMU_PMCR_N_SHIFT;
	107
	108	val &= ARMV8_PMU_PMCR_N_MASK;
	109	if (val == 0)
	110	return BIT(ARMV8_PMU_CYCLE_IDX);
	111	else
	112	return GENMASK(val - 1, 0) \| BIT(ARMV8_PMU_CYCLE_IDX);
	113	}
	114
	115	/**
	116	* kvm_pmu_enable_counter - enable selected PMU counter
	117	* @vcpu: The vcpu pointer
	118	* @val: the value guest writes to PMCNTENSET register
	119	*
	120	* Call perf_event_enable to start counting the perf event
	121	*/
	122	void kvm_pmu_enable_counter(struct kvm_vcpu *vcpu, u64 val)
	123	{
	124	int i;
	125	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	126	struct kvm_pmc *pmc;
	127
	128	if (!(vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) \|\| !val)
	129	return;
	130
	131	for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
	132	if (!(val & BIT(i)))
	133	continue;
	134
	135	pmc = &pmu->pmc[i];
	136	if (pmc->perf_event) {
	137	perf_event_enable(pmc->perf_event);
	138	if (pmc->perf_event->state != PERF_EVENT_STATE_ACTIVE)
	139	kvm_debug("fail to enable perf event\n");
	140	}
	141	}
	142	}
	143
	144	/**
	145	* kvm_pmu_disable_counter - disable selected PMU counter
	146	* @vcpu: The vcpu pointer
	147	* @val: the value guest writes to PMCNTENCLR register
	148	*
	149	* Call perf_event_disable to stop counting the perf event
	150	*/
	151	void kvm_pmu_disable_counter(struct kvm_vcpu *vcpu, u64 val)
	152	{
	153	int i;
	154	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	155	struct kvm_pmc *pmc;
	156
	157	if (!val)
	158	return;
	159
	160	for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
	161	if (!(val & BIT(i)))
	162	continue;
	163
	164	pmc = &pmu->pmc[i];
	165	if (pmc->perf_event)
	166	perf_event_disable(pmc->perf_event);
	167	}
168	}
7f766358	169
76d883c4 SZ	170	static u64 kvm_pmu_overflow_status(struct kvm_vcpu *vcpu)
	171	{
	172	u64 reg = 0;
	173
	174	if ((vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E))
	175	reg = vcpu_sys_reg(vcpu, PMOVSSET_EL0);
	176	reg &= vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
	177	reg &= vcpu_sys_reg(vcpu, PMINTENSET_EL1);
	178	reg &= kvm_pmu_valid_counter_mask(vcpu);
	179
	180	return reg;
	181	}
	182
	183	/**
	184	* kvm_pmu_overflow_set - set PMU overflow interrupt
	185	* @vcpu: The vcpu pointer
	186	* @val: the value guest writes to PMOVSSET register
	187	*/
	188	void kvm_pmu_overflow_set(struct kvm_vcpu *vcpu, u64 val)
	189	{
	190	u64 reg;
	191
	192	if (val == 0)
	193	return;
	194
	195	vcpu_sys_reg(vcpu, PMOVSSET_EL0) \|= val;
	196	reg = kvm_pmu_overflow_status(vcpu);
	197	if (reg != 0)
	198	kvm_vcpu_kick(vcpu);
	199	}
	200
b02386eb SZ	201	static void kvm_pmu_update_state(struct kvm_vcpu *vcpu)
	202	{
	203	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	204	bool overflow;
	205
	206	if (!kvm_arm_pmu_v3_ready(vcpu))
	207	return;
	208
	209	overflow = !!kvm_pmu_overflow_status(vcpu);
	210	if (pmu->irq_level != overflow) {
	211	pmu->irq_level = overflow;
	212	kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
	213	pmu->irq_num, overflow);
	214	}
	215	}
	216
	217	/**
	218	* kvm_pmu_flush_hwstate - flush pmu state to cpu
	219	* @vcpu: The vcpu pointer
	220	*
	221	* Check if the PMU has overflowed while we were running in the host, and inject
	222	* an interrupt if that was the case.
	223	*/
	224	void kvm_pmu_flush_hwstate(struct kvm_vcpu *vcpu)
	225	{
	226	kvm_pmu_update_state(vcpu);
	227	}
	228
	229	/**
	230	* kvm_pmu_sync_hwstate - sync pmu state from cpu
	231	* @vcpu: The vcpu pointer
	232	*
	233	* Check if the PMU has overflowed while we were running in the guest, and
	234	* inject an interrupt if that was the case.
	235	*/
	236	void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu)
	237	{
	238	kvm_pmu_update_state(vcpu);
	239	}
	240
	241	static inline struct kvm_vcpu kvm_pmc_to_vcpu(struct kvm_pmc pmc)
	242	{
	243	struct kvm_pmu *pmu;
	244	struct kvm_vcpu_arch *vcpu_arch;
	245
	246	pmc -= pmc->idx;
	247	pmu = container_of(pmc, struct kvm_pmu, pmc[0]);
	248	vcpu_arch = container_of(pmu, struct kvm_vcpu_arch, pmu);
	249	return container_of(vcpu_arch, struct kvm_vcpu, arch);
	250	}
	251
	252	/**
	253	* When perf event overflows, call kvm_pmu_overflow_set to set overflow status.
	254	*/
	255	static void kvm_pmu_perf_overflow(struct perf_event *perf_event,
	256	struct perf_sample_data *data,
	257	struct pt_regs *regs)
	258	{
	259	struct kvm_pmc *pmc = perf_event->overflow_handler_context;
	260	struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
	261	int idx = pmc->idx;
	262
	263	kvm_pmu_overflow_set(vcpu, BIT(idx));
	264	}
265
7a0adc70 SZ	266	/**
	267	* kvm_pmu_software_increment - do software increment
	268	* @vcpu: The vcpu pointer
	269	* @val: the value guest writes to PMSWINC register
	270	*/
	271	void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val)
	272	{
	273	int i;
	274	u64 type, enable, reg;
	275
	276	if (val == 0)
	277	return;
	278
	279	enable = vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
	280	for (i = 0; i < ARMV8_PMU_CYCLE_IDX; i++) {
	281	if (!(val & BIT(i)))
	282	continue;
	283	type = vcpu_sys_reg(vcpu, PMEVTYPER0_EL0 + i)
	284	& ARMV8_PMU_EVTYPE_EVENT;
	285	if ((type == ARMV8_PMU_EVTYPE_EVENT_SW_INCR)
	286	&& (enable & BIT(i))) {
	287	reg = vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i) + 1;
	288	reg = lower_32_bits(reg);
	289	vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i) = reg;
	290	if (!reg)
	291	kvm_pmu_overflow_set(vcpu, BIT(i));
	292	}
	293	}
	294	}
	295
76993739 SZ	296	/**
	297	* kvm_pmu_handle_pmcr - handle PMCR register
	298	* @vcpu: The vcpu pointer
	299	* @val: the value guest writes to PMCR register
	300	*/
	301	void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val)
	302	{
	303	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	304	struct kvm_pmc *pmc;
	305	u64 mask;
	306	int i;
	307
	308	mask = kvm_pmu_valid_counter_mask(vcpu);
	309	if (val & ARMV8_PMU_PMCR_E) {
	310	kvm_pmu_enable_counter(vcpu,
	311	vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & mask);
	312	} else {
	313	kvm_pmu_disable_counter(vcpu, mask);
	314	}
	315
	316	if (val & ARMV8_PMU_PMCR_C)
	317	kvm_pmu_set_counter_value(vcpu, ARMV8_PMU_CYCLE_IDX, 0);
	318
	319	if (val & ARMV8_PMU_PMCR_P) {
	320	for (i = 0; i < ARMV8_PMU_CYCLE_IDX; i++)
	321	kvm_pmu_set_counter_value(vcpu, i, 0);
	322	}
	323
	324	if (val & ARMV8_PMU_PMCR_LC) {
	325	pmc = &pmu->pmc[ARMV8_PMU_CYCLE_IDX];
	326	pmc->bitmask = 0xffffffffffffffffUL;
	327	}
	328	}
	329
7f766358 SZ	330	static bool kvm_pmu_counter_is_enabled(struct kvm_vcpu *vcpu, u64 select_idx)
	331	{
	332	return (vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) &&
	333	(vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & BIT(select_idx));
	334	}
	335
	336	/**
	337	* kvm_pmu_set_counter_event_type - set selected counter to monitor some event
	338	* @vcpu: The vcpu pointer
	339	* @data: The data guest writes to PMXEVTYPER_EL0
	340	* @select_idx: The number of selected counter
	341	*
	342	* When OS accesses PMXEVTYPER_EL0, that means it wants to set a PMC to count an
	343	* event with given hardware event number. Here we call perf_event API to
	344	* emulate this action and create a kernel perf event for it.
	345	*/
	346	void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data,
	347	u64 select_idx)
	348	{
	349	struct kvm_pmu *pmu = &vcpu->arch.pmu;
	350	struct kvm_pmc *pmc = &pmu->pmc[select_idx];
	351	struct perf_event *event;
	352	struct perf_event_attr attr;
	353	u64 eventsel, counter;
	354
	355	kvm_pmu_stop_counter(vcpu, pmc);
	356	eventsel = data & ARMV8_PMU_EVTYPE_EVENT;
	357
7a0adc70 SZ	358	/* Software increment event does't need to be backed by a perf event */
	359	if (eventsel == ARMV8_PMU_EVTYPE_EVENT_SW_INCR)
	360	return;
	361
7f766358 SZ	362	memset(&attr, 0, sizeof(struct perf_event_attr));
	363	attr.type = PERF_TYPE_RAW;
	364	attr.size = sizeof(attr);
	365	attr.pinned = 1;
	366	attr.disabled = !kvm_pmu_counter_is_enabled(vcpu, select_idx);
	367	attr.exclude_user = data & ARMV8_PMU_EXCLUDE_EL0 ? 1 : 0;
	368	attr.exclude_kernel = data & ARMV8_PMU_EXCLUDE_EL1 ? 1 : 0;
	369	attr.exclude_hv = 1; /* Don't count EL2 events */
	370	attr.exclude_host = 1; /* Don't count host events */
	371	attr.config = eventsel;
	372
	373	counter = kvm_pmu_get_counter_value(vcpu, select_idx);
	374	/* The initial sample period (overflow count) of an event. */
	375	attr.sample_period = (-counter) & pmc->bitmask;
	376
b02386eb SZ	377	event = perf_event_create_kernel_counter(&attr, -1, current,
b02386eb SZ	378	kvm_pmu_perf_overflow, pmc);
7f766358 SZ	379	if (IS_ERR(event)) {
	380	pr_err_once("kvm: pmu event creation failed %ld\n",
	381	PTR_ERR(event));
	382	return;
	383	}
	384
	385	pmc->perf_event = event;
	386	}