[mirror_ubuntu-jammy-kernel.git] / arch / x86 / kernel / cpu / aperfmperf.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * x86 APERF/MPERF KHz calculation for
 * /sys/.../cpufreq/scaling_cur_freq
 *
 * Copyright (C) 2017 Intel Corp.
 * Author: Len Brown <len.brown@intel.com>
 */

#include <linux/delay.h>
#include <linux/ktime.h>
#include <linux/math64.h>
#include <linux/percpu.h>
#include <linux/cpufreq.h>
#include <linux/smp.h>
#include <linux/sched/isolation.h>
#include <linux/rcupdate.h>

#include "cpu.h"

struct aperfmperf_sample {
	unsigned int	khz;
	atomic_t	scfpending;
	ktime_t	time;
	u64	aperf;
	u64	mperf;
};

static DEFINE_PER_CPU(struct aperfmperf_sample, samples);

#define APERFMPERF_CACHE_THRESHOLD_MS	10
#define APERFMPERF_REFRESH_DELAY_MS	10
#define APERFMPERF_STALE_THRESHOLD_MS	1000

/*
 * aperfmperf_snapshot_khz()
 * On the current CPU, snapshot APERF, MPERF, and jiffies
 * unless we already did it within 10ms
 * calculate kHz, save snapshot
 */
static void aperfmperf_snapshot_khz(void *dummy)
{
	u64 aperf, aperf_delta;
	u64 mperf, mperf_delta;
	struct aperfmperf_sample *s = this_cpu_ptr(&samples);
	unsigned long flags;

	local_irq_save(flags);
	rdmsrl(MSR_IA32_APERF, aperf);
	rdmsrl(MSR_IA32_MPERF, mperf);
	local_irq_restore(flags);

	aperf_delta = aperf - s->aperf;
	mperf_delta = mperf - s->mperf;

	/*
	 * There is no architectural guarantee that MPERF
	 * increments faster than we can read it.
	 */
	if (mperf_delta == 0)
		return;

	s->time = ktime_get();
	s->aperf = aperf;
	s->mperf = mperf;
	s->khz = div64_u64((cpu_khz * aperf_delta), mperf_delta);
	atomic_set_release(&s->scfpending, 0);
}

static bool aperfmperf_snapshot_cpu(int cpu, ktime_t now, bool wait)
{
	s64 time_delta = ktime_ms_delta(now, per_cpu(samples.time, cpu));
	struct aperfmperf_sample *s = per_cpu_ptr(&samples, cpu);

	/* Don't bother re-computing within the cache threshold time. */
	if (time_delta < APERFMPERF_CACHE_THRESHOLD_MS)
		return true;

	if (!atomic_xchg(&s->scfpending, 1) || wait)
		smp_call_function_single(cpu, aperfmperf_snapshot_khz, NULL, wait);

	/* Return false if the previous iteration was too long ago. */
	return time_delta <= APERFMPERF_STALE_THRESHOLD_MS;
}

unsigned int aperfmperf_get_khz(int cpu)
{
	if (!cpu_khz)
		return 0;

	if (!boot_cpu_has(X86_FEATURE_APERFMPERF))
		return 0;

	if (!housekeeping_cpu(cpu, HK_FLAG_MISC))
		return 0;

	if (rcu_is_idle_cpu(cpu))
		return 0; /* Idle CPUs are completely uninteresting. */

	aperfmperf_snapshot_cpu(cpu, ktime_get(), true);
	return per_cpu(samples.khz, cpu);
}

void arch_freq_prepare_all(void)
{
	ktime_t now = ktime_get();
	bool wait = false;
	int cpu;

	if (!cpu_khz)
		return;

	if (!boot_cpu_has(X86_FEATURE_APERFMPERF))
		return;

	for_each_online_cpu(cpu) {
		if (!housekeeping_cpu(cpu, HK_FLAG_MISC))
			continue;
		if (rcu_is_idle_cpu(cpu))
			continue; /* Idle CPUs are completely uninteresting. */
		if (!aperfmperf_snapshot_cpu(cpu, now, false))
			wait = true;
	}

	if (wait)
		msleep(APERFMPERF_REFRESH_DELAY_MS);
}

unsigned int arch_freq_get_on_cpu(int cpu)
{
	struct aperfmperf_sample *s = per_cpu_ptr(&samples, cpu);

	if (!cpu_khz)
		return 0;

	if (!boot_cpu_has(X86_FEATURE_APERFMPERF))
		return 0;

	if (!housekeeping_cpu(cpu, HK_FLAG_MISC))
		return 0;

	if (aperfmperf_snapshot_cpu(cpu, ktime_get(), true))
		return per_cpu(samples.khz, cpu);

	msleep(APERFMPERF_REFRESH_DELAY_MS);
	atomic_set(&s->scfpending, 1);
	smp_mb(); /* ->scfpending before smp_call_function_single(). */
	smp_call_function_single(cpu, aperfmperf_snapshot_khz, NULL, 1);

	return per_cpu(samples.khz, cpu);
}
Commit	Line	Data
82664963	1	// SPDX-License-Identifier: GPL-2.0-only
f8475cef LB	2	/*
	3	* x86 APERF/MPERF KHz calculation for
	4	* /sys/.../cpufreq/scaling_cur_freq
	5	*
	6	* Copyright (C) 2017 Intel Corp.
	7	* Author: Len Brown <len.brown@intel.com>
f8475cef LB	8	*/
f8475cef LB	9
4815d3c5 RW	10	#include <linux/delay.h>
4815d3c5 RW	11	#include <linux/ktime.h>
f8475cef LB	12	#include <linux/math64.h>
f8475cef LB	13	#include <linux/percpu.h>
ad3bc25a	14	#include <linux/cpufreq.h>
f8475cef	15	#include <linux/smp.h>
cc9e303c	16	#include <linux/sched/isolation.h>
3fcd6a23	17	#include <linux/rcupdate.h>
f8475cef	18
7d5905dc RW	19	#include "cpu.h"
7d5905dc RW	20
f8475cef LB	21	struct aperfmperf_sample {
f8475cef LB	22	unsigned int khz;
f4deaf90	23	atomic_t scfpending;
4815d3c5	24	ktime_t time;
f8475cef LB	25	u64 aperf;
	26	u64 mperf;
	27	};
	28
	29	static DEFINE_PER_CPU(struct aperfmperf_sample, samples);
	30
4815d3c5	31	#define APERFMPERF_CACHE_THRESHOLD_MS 10
7d5905dc	32	#define APERFMPERF_REFRESH_DELAY_MS 10
4815d3c5 RW	33	#define APERFMPERF_STALE_THRESHOLD_MS 1000
4815d3c5 RW	34
f8475cef LB	35	/*
	36	* aperfmperf_snapshot_khz()
	37	* On the current CPU, snapshot APERF, MPERF, and jiffies
	38	* unless we already did it within 10ms
	39	* calculate kHz, save snapshot
	40	*/
	41	static void aperfmperf_snapshot_khz(void *dummy)
	42	{
	43	u64 aperf, aperf_delta;
	44	u64 mperf, mperf_delta;
	45	struct aperfmperf_sample *s = this_cpu_ptr(&samples);
8e2f3bce	46	unsigned long flags;
f8475cef	47
8e2f3bce	48	local_irq_save(flags);
f8475cef LB	49	rdmsrl(MSR_IA32_APERF, aperf);
f8475cef LB	50	rdmsrl(MSR_IA32_MPERF, mperf);
8e2f3bce	51	local_irq_restore(flags);
f8475cef LB	52
	53	aperf_delta = aperf - s->aperf;
	54	mperf_delta = mperf - s->mperf;
	55
	56	/*
	57	* There is no architectural guarantee that MPERF
	58	* increments faster than we can read it.
	59	*/
	60	if (mperf_delta == 0)
	61	return;
	62
7d5905dc	63	s->time = ktime_get();
f8475cef LB	64	s->aperf = aperf;
f8475cef LB	65	s->mperf = mperf;
7d5905dc	66	s->khz = div64_u64((cpu_khz * aperf_delta), mperf_delta);
f4deaf90	67	atomic_set_release(&s->scfpending, 0);
f8475cef LB	68	}
f8475cef LB	69
7d5905dc	70	static bool aperfmperf_snapshot_cpu(int cpu, ktime_t now, bool wait)
f8475cef	71	{
7d5905dc	72	s64 time_delta = ktime_ms_delta(now, per_cpu(samples.time, cpu));
f4deaf90	73	struct aperfmperf_sample *s = per_cpu_ptr(&samples, cpu);
7d5905dc RW	74
	75	/* Don't bother re-computing within the cache threshold time. */
	76	if (time_delta < APERFMPERF_CACHE_THRESHOLD_MS)
	77	return true;
	78
f4deaf90 PM	79	if (!atomic_xchg(&s->scfpending, 1) \|\| wait)
f4deaf90 PM	80	smp_call_function_single(cpu, aperfmperf_snapshot_khz, NULL, wait);
7d5905dc RW	81
	82	/* Return false if the previous iteration was too long ago. */
	83	return time_delta <= APERFMPERF_STALE_THRESHOLD_MS;
	84	}
4815d3c5	85
7d5905dc RW	86	unsigned int aperfmperf_get_khz(int cpu)
7d5905dc RW	87	{
f8475cef LB	88	if (!cpu_khz)
	89	return 0;
	90
67e87d43	91	if (!boot_cpu_has(X86_FEATURE_APERFMPERF))
f8475cef LB	92	return 0;
f8475cef LB	93
cc9e303c KK	94	if (!housekeeping_cpu(cpu, HK_FLAG_MISC))
	95	return 0;
	96
3fcd6a23 PM	97	if (rcu_is_idle_cpu(cpu))
	98	return 0; /* Idle CPUs are completely uninteresting. */
	99
7d5905dc RW	100	aperfmperf_snapshot_cpu(cpu, ktime_get(), true);
	101	return per_cpu(samples.khz, cpu);
	102	}
b29c6ef7	103
7d5905dc RW	104	void arch_freq_prepare_all(void)
	105	{
	106	ktime_t now = ktime_get();
	107	bool wait = false;
	108	int cpu;
	109
	110	if (!cpu_khz)
	111	return;
	112
67e87d43	113	if (!boot_cpu_has(X86_FEATURE_APERFMPERF))
7d5905dc RW	114	return;
7d5905dc RW	115
cc9e303c KK	116	for_each_online_cpu(cpu) {
	117	if (!housekeeping_cpu(cpu, HK_FLAG_MISC))
	118	continue;
3fcd6a23 PM	119	if (rcu_is_idle_cpu(cpu))
3fcd6a23 PM	120	continue; /* Idle CPUs are completely uninteresting. */
7d5905dc RW	121	if (!aperfmperf_snapshot_cpu(cpu, now, false))
7d5905dc RW	122	wait = true;
cc9e303c	123	}
7d5905dc RW	124
	125	if (wait)
	126	msleep(APERFMPERF_REFRESH_DELAY_MS);
	127	}
	128
	129	unsigned int arch_freq_get_on_cpu(int cpu)
	130	{
f4deaf90 PM	131	struct aperfmperf_sample *s = per_cpu_ptr(&samples, cpu);
f4deaf90 PM	132
7d5905dc RW	133	if (!cpu_khz)
	134	return 0;
	135
67e87d43	136	if (!boot_cpu_has(X86_FEATURE_APERFMPERF))
7d5905dc RW	137	return 0;
7d5905dc RW	138
cc9e303c KK	139	if (!housekeeping_cpu(cpu, HK_FLAG_MISC))
	140	return 0;
	141
7d5905dc RW	142	if (aperfmperf_snapshot_cpu(cpu, ktime_get(), true))
7d5905dc RW	143	return per_cpu(samples.khz, cpu);
4815d3c5 RW	144
4815d3c5 RW	145	msleep(APERFMPERF_REFRESH_DELAY_MS);
f4deaf90 PM	146	atomic_set(&s->scfpending, 1);
f4deaf90 PM	147	smp_mb(); /* ->scfpending before smp_call_function_single(). */
4815d3c5	148	smp_call_function_single(cpu, aperfmperf_snapshot_khz, NULL, 1);
f8475cef LB	149
	150	return per_cpu(samples.khz, cpu);
	151	}