[mirror_ubuntu-artful-kernel.git] / arch / x86 / kernel / cpu / aperfmperf.c

/*
 * x86 APERF/MPERF KHz calculation for
 * /sys/.../cpufreq/scaling_cur_freq
 *
 * Copyright (C) 2017 Intel Corp.
 * Author: Len Brown <len.brown@intel.com>
 *
 * This file is licensed under GPLv2.
 */

#include <linux/delay.h>
#include <linux/ktime.h>
#include <linux/math64.h>
#include <linux/percpu.h>
#include <linux/smp.h>

#include "cpu.h"

struct aperfmperf_sample {
	unsigned int	khz;
	ktime_t	time;
	u64	aperf;
	u64	mperf;
};

static DEFINE_PER_CPU(struct aperfmperf_sample, samples);

#define APERFMPERF_CACHE_THRESHOLD_MS	10
#define APERFMPERF_REFRESH_DELAY_MS	10
#define APERFMPERF_STALE_THRESHOLD_MS	1000

/*
 * aperfmperf_snapshot_khz()
 * On the current CPU, snapshot APERF, MPERF, and jiffies
 * unless we already did it within 10ms
 * calculate kHz, save snapshot
 */
static void aperfmperf_snapshot_khz(void *dummy)
{
	u64 aperf, aperf_delta;
	u64 mperf, mperf_delta;
	struct aperfmperf_sample *s = this_cpu_ptr(&samples);
	unsigned long flags;

	local_irq_save(flags);
	rdmsrl(MSR_IA32_APERF, aperf);
	rdmsrl(MSR_IA32_MPERF, mperf);
	local_irq_restore(flags);

	aperf_delta = aperf - s->aperf;
	mperf_delta = mperf - s->mperf;

	/*
	 * There is no architectural guarantee that MPERF
	 * increments faster than we can read it.
	 */
	if (mperf_delta == 0)
		return;

	s->time = ktime_get();
	s->aperf = aperf;
	s->mperf = mperf;
	s->khz = div64_u64((cpu_khz * aperf_delta), mperf_delta);
}

static bool aperfmperf_snapshot_cpu(int cpu, ktime_t now, bool wait)
{
	s64 time_delta = ktime_ms_delta(now, per_cpu(samples.time, cpu));

	/* Don't bother re-computing within the cache threshold time. */
	if (time_delta < APERFMPERF_CACHE_THRESHOLD_MS)
		return true;

	smp_call_function_single(cpu, aperfmperf_snapshot_khz, NULL, wait);

	/* Return false if the previous iteration was too long ago. */
	return time_delta <= APERFMPERF_STALE_THRESHOLD_MS;
}

unsigned int aperfmperf_get_khz(int cpu)
{
	if (!cpu_khz)
		return 0;

	if (!static_cpu_has(X86_FEATURE_APERFMPERF))
		return 0;

	aperfmperf_snapshot_cpu(cpu, ktime_get(), true);
	return per_cpu(samples.khz, cpu);
}

void arch_freq_prepare_all(void)
{
	ktime_t now = ktime_get();
	bool wait = false;
	int cpu;

	if (!cpu_khz)
		return;

	if (!static_cpu_has(X86_FEATURE_APERFMPERF))
		return;

	for_each_online_cpu(cpu)
		if (!aperfmperf_snapshot_cpu(cpu, now, false))
			wait = true;

	if (wait)
		msleep(APERFMPERF_REFRESH_DELAY_MS);
}

unsigned int arch_freq_get_on_cpu(int cpu)
{
	if (!cpu_khz)
		return 0;

	if (!static_cpu_has(X86_FEATURE_APERFMPERF))
		return 0;

	if (aperfmperf_snapshot_cpu(cpu, ktime_get(), true))
		return per_cpu(samples.khz, cpu);

	msleep(APERFMPERF_REFRESH_DELAY_MS);
	smp_call_function_single(cpu, aperfmperf_snapshot_khz, NULL, 1);

	return per_cpu(samples.khz, cpu);
}
Commit	Line	Data
f8475cef LB	1	/*
	2	* x86 APERF/MPERF KHz calculation for
	3	* /sys/.../cpufreq/scaling_cur_freq
	4	*
	5	* Copyright (C) 2017 Intel Corp.
	6	* Author: Len Brown <len.brown@intel.com>
	7	*
	8	* This file is licensed under GPLv2.
	9	*/
	10
4815d3c5 RW	11	#include <linux/delay.h>
4815d3c5 RW	12	#include <linux/ktime.h>
f8475cef LB	13	#include <linux/math64.h>
	14	#include <linux/percpu.h>
	15	#include <linux/smp.h>
	16
3246f29e RW	17	#include "cpu.h"
3246f29e RW	18
f8475cef LB	19	struct aperfmperf_sample {
f8475cef LB	20	unsigned int khz;
4815d3c5	21	ktime_t time;
f8475cef LB	22	u64 aperf;
	23	u64 mperf;
	24	};
	25
	26	static DEFINE_PER_CPU(struct aperfmperf_sample, samples);
	27
4815d3c5	28	#define APERFMPERF_CACHE_THRESHOLD_MS 10
3246f29e	29	#define APERFMPERF_REFRESH_DELAY_MS 10
4815d3c5 RW	30	#define APERFMPERF_STALE_THRESHOLD_MS 1000
4815d3c5 RW	31
f8475cef LB	32	/*
	33	* aperfmperf_snapshot_khz()
	34	* On the current CPU, snapshot APERF, MPERF, and jiffies
	35	* unless we already did it within 10ms
	36	* calculate kHz, save snapshot
	37	*/
	38	static void aperfmperf_snapshot_khz(void *dummy)
	39	{
	40	u64 aperf, aperf_delta;
	41	u64 mperf, mperf_delta;
	42	struct aperfmperf_sample *s = this_cpu_ptr(&samples);
8e2f3bce	43	unsigned long flags;
f8475cef	44
8e2f3bce	45	local_irq_save(flags);
f8475cef LB	46	rdmsrl(MSR_IA32_APERF, aperf);
f8475cef LB	47	rdmsrl(MSR_IA32_MPERF, mperf);
8e2f3bce	48	local_irq_restore(flags);
f8475cef LB	49
	50	aperf_delta = aperf - s->aperf;
	51	mperf_delta = mperf - s->mperf;
	52
	53	/*
	54	* There is no architectural guarantee that MPERF
	55	* increments faster than we can read it.
	56	*/
	57	if (mperf_delta == 0)
	58	return;
	59
3246f29e	60	s->time = ktime_get();
f8475cef LB	61	s->aperf = aperf;
f8475cef LB	62	s->mperf = mperf;
3246f29e	63	s->khz = div64_u64((cpu_khz * aperf_delta), mperf_delta);
f8475cef LB	64	}
f8475cef LB	65
3246f29e	66	static bool aperfmperf_snapshot_cpu(int cpu, ktime_t now, bool wait)
f8475cef	67	{
3246f29e RW	68	s64 time_delta = ktime_ms_delta(now, per_cpu(samples.time, cpu));
	69
	70	/* Don't bother re-computing within the cache threshold time. */
	71	if (time_delta < APERFMPERF_CACHE_THRESHOLD_MS)
	72	return true;
	73
	74	smp_call_function_single(cpu, aperfmperf_snapshot_khz, NULL, wait);
	75
	76	/* Return false if the previous iteration was too long ago. */
	77	return time_delta <= APERFMPERF_STALE_THRESHOLD_MS;
	78	}
4815d3c5	79
3246f29e RW	80	unsigned int aperfmperf_get_khz(int cpu)
3246f29e RW	81	{
f8475cef LB	82	if (!cpu_khz)
	83	return 0;
	84
	85	if (!static_cpu_has(X86_FEATURE_APERFMPERF))
	86	return 0;
	87
3246f29e RW	88	aperfmperf_snapshot_cpu(cpu, ktime_get(), true);
	89	return per_cpu(samples.khz, cpu);
	90	}
41d9b2fc	91
3246f29e RW	92	void arch_freq_prepare_all(void)
	93	{
	94	ktime_t now = ktime_get();
	95	bool wait = false;
	96	int cpu;
	97
	98	if (!cpu_khz)
	99	return;
	100
	101	if (!static_cpu_has(X86_FEATURE_APERFMPERF))
	102	return;
	103
	104	for_each_online_cpu(cpu)
	105	if (!aperfmperf_snapshot_cpu(cpu, now, false))
	106	wait = true;
	107
	108	if (wait)
	109	msleep(APERFMPERF_REFRESH_DELAY_MS);
	110	}
	111
	112	unsigned int arch_freq_get_on_cpu(int cpu)
	113	{
	114	if (!cpu_khz)
	115	return 0;
	116
	117	if (!static_cpu_has(X86_FEATURE_APERFMPERF))
	118	return 0;
	119
	120	if (aperfmperf_snapshot_cpu(cpu, ktime_get(), true))
	121	return per_cpu(samples.khz, cpu);
4815d3c5 RW	122
	123	msleep(APERFMPERF_REFRESH_DELAY_MS);
	124	smp_call_function_single(cpu, aperfmperf_snapshot_khz, NULL, 1);
f8475cef LB	125
	126	return per_cpu(samples.khz, cpu);
	127	}