[mirror_ubuntu-bionic-kernel.git] / arch / x86 / kernel / tsc_32.c

#include <linux/sched.h>
#include <linux/clocksource.h>
#include <linux/workqueue.h>
#include <linux/cpufreq.h>
#include <linux/jiffies.h>
#include <linux/init.h>
#include <linux/dmi.h>
#include <linux/percpu.h>

#include <asm/delay.h>
#include <asm/tsc.h>
#include <asm/io.h>
#include <asm/timer.h>

#include "mach_timer.h"

static int tsc_enabled;

/*
 * On some systems the TSC frequency does not
 * change with the cpu frequency. So we need
 * an extra value to store the TSC freq
 */
unsigned int tsc_khz;
EXPORT_SYMBOL_GPL(tsc_khz);

#ifdef CONFIG_X86_TSC
static int __init tsc_setup(char *str)
{
	printk(KERN_WARNING "notsc: Kernel compiled with CONFIG_X86_TSC, "
				"cannot disable TSC completely.\n");
	mark_tsc_unstable("user disabled TSC");
	return 1;
}
#else
/*
 * disable flag for tsc. Takes effect by clearing the TSC cpu flag
 * in cpu/common.c
 */
static int __init tsc_setup(char *str)
{
	setup_clear_cpu_cap(X86_FEATURE_TSC);
	return 1;
}
#endif

__setup("notsc", tsc_setup);

/*
 * code to mark and check if the TSC is unstable
 * due to cpufreq or due to unsynced TSCs
 */
static int tsc_unstable;

int check_tsc_unstable(void)
{
	return tsc_unstable;
}
EXPORT_SYMBOL_GPL(check_tsc_unstable);

/* Accelerators for sched_clock()
 * convert from cycles(64bits) => nanoseconds (64bits)
 *  basic equation:
 *		ns = cycles / (freq / ns_per_sec)
 *		ns = cycles * (ns_per_sec / freq)
 *		ns = cycles * (10^9 / (cpu_khz * 10^3))
 *		ns = cycles * (10^6 / cpu_khz)
 *
 *	Then we use scaling math (suggested by george@mvista.com) to get:
 *		ns = cycles * (10^6 * SC / cpu_khz) / SC
 *		ns = cycles * cyc2ns_scale / SC
 *
 *	And since SC is a constant power of two, we can convert the div
 *  into a shift.
 *
 *  We can use khz divisor instead of mhz to keep a better precision, since
 *  cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits.
 *  (mathieu.desnoyers@polymtl.ca)
 *
 *			-johnstul@us.ibm.com "math is hard, lets go shopping!"
 */

DEFINE_PER_CPU(unsigned long, cyc2ns);

static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)
{
	unsigned long long tsc_now, ns_now;
	unsigned long flags, *scale;

	local_irq_save(flags);
	sched_clock_idle_sleep_event();

	scale = &per_cpu(cyc2ns, cpu);

	rdtscll(tsc_now);
	ns_now = __cycles_2_ns(tsc_now);

	if (cpu_khz)
		*scale = (NSEC_PER_MSEC << CYC2NS_SCALE_FACTOR)/cpu_khz;

	/*
	 * Start smoothly with the new frequency:
	 */
	sched_clock_idle_wakeup_event(0);
	local_irq_restore(flags);
}

/*
 * Scheduler clock - returns current time in nanosec units.
 */
unsigned long long native_sched_clock(void)
{
	unsigned long long this_offset;

	/*
	 * Fall back to jiffies if there's no TSC available:
	 * ( But note that we still use it if the TSC is marked
	 *   unstable. We do this because unlike Time Of Day,
	 *   the scheduler clock tolerates small errors and it's
	 *   very important for it to be as fast as the platform
	 *   can achive it. )
	 */
	if (unlikely(!tsc_enabled && !tsc_unstable))
		/* No locking but a rare wrong value is not a big deal: */
		return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ);

	/* read the Time Stamp Counter: */
	rdtscll(this_offset);

	/* return the value in ns */
	return cycles_2_ns(this_offset);
}

/* We need to define a real function for sched_clock, to override the
   weak default version */
#ifdef CONFIG_PARAVIRT
unsigned long long sched_clock(void)
{
	return paravirt_sched_clock();
}
#else
unsigned long long sched_clock(void)
	__attribute__((alias("native_sched_clock")));
#endif

unsigned long native_calculate_cpu_khz(void)
{
	unsigned long long start, end;
	unsigned long count;
	u64 delta64 = (u64)ULLONG_MAX;
	int i;
	unsigned long flags;

	local_irq_save(flags);

	/* run 3 times to ensure the cache is warm and to get an accurate reading */
	for (i = 0; i < 3; i++) {
		mach_prepare_counter();
		rdtscll(start);
		mach_countup(&count);
		rdtscll(end);

		/*
		 * Error: ECTCNEVERSET
		 * The CTC wasn't reliable: we got a hit on the very first read,
		 * or the CPU was so fast/slow that the quotient wouldn't fit in
		 * 32 bits..
		 */
		if (count <= 1)
			continue;

		/* cpu freq too slow: */
		if ((end - start) <= CALIBRATE_TIME_MSEC)
			continue;

		/*
		 * We want the minimum time of all runs in case one of them
		 * is inaccurate due to SMI or other delay
		 */
		delta64 = min(delta64, (end - start));
	}

	/* cpu freq too fast (or every run was bad): */
	if (delta64 > (1ULL<<32))
		goto err;

	delta64 += CALIBRATE_TIME_MSEC/2; /* round for do_div */
	do_div(delta64,CALIBRATE_TIME_MSEC);

	local_irq_restore(flags);
	return (unsigned long)delta64;
err:
	local_irq_restore(flags);
	return 0;
}

int recalibrate_cpu_khz(void)
{
#ifndef CONFIG_SMP
	unsigned long cpu_khz_old = cpu_khz;

	if (cpu_has_tsc) {
		cpu_khz = calculate_cpu_khz();
		tsc_khz = cpu_khz;
		cpu_data(0).loops_per_jiffy =
			cpufreq_scale(cpu_data(0).loops_per_jiffy,
					cpu_khz_old, cpu_khz);
		return 0;
	} else
		return -ENODEV;
#else
	return -ENODEV;
#endif
}

EXPORT_SYMBOL(recalibrate_cpu_khz);

#ifdef CONFIG_CPU_FREQ

/*
 * if the CPU frequency is scaled, TSC-based delays will need a different
 * loops_per_jiffy value to function properly.
 */
static unsigned int ref_freq;
static unsigned long loops_per_jiffy_ref;
static unsigned long cpu_khz_ref;

static int
time_cpufreq_notifier(struct notifier_block *nb, unsigned long val, void *data)
{
	struct cpufreq_freqs *freq = data;

	if (!ref_freq) {
		if (!freq->old){
			ref_freq = freq->new;
			return 0;
		}
		ref_freq = freq->old;
		loops_per_jiffy_ref = cpu_data(freq->cpu).loops_per_jiffy;
		cpu_khz_ref = cpu_khz;
	}

	if ((val == CPUFREQ_PRECHANGE  && freq->old < freq->new) ||
	    (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||
	    (val == CPUFREQ_RESUMECHANGE)) {
		if (!(freq->flags & CPUFREQ_CONST_LOOPS))
			cpu_data(freq->cpu).loops_per_jiffy =
				cpufreq_scale(loops_per_jiffy_ref,
						ref_freq, freq->new);

		if (cpu_khz) {

			if (num_online_cpus() == 1)
				cpu_khz = cpufreq_scale(cpu_khz_ref,
						ref_freq, freq->new);
			if (!(freq->flags & CPUFREQ_CONST_LOOPS)) {
				tsc_khz = cpu_khz;
				set_cyc2ns_scale(cpu_khz, freq->cpu);
				/*
				 * TSC based sched_clock turns
				 * to junk w/ cpufreq
				 */
				mark_tsc_unstable("cpufreq changes");
			}
		}
	}

	return 0;
}

static struct notifier_block time_cpufreq_notifier_block = {
	.notifier_call	= time_cpufreq_notifier
};

static int __init cpufreq_tsc(void)
{
	return cpufreq_register_notifier(&time_cpufreq_notifier_block,
					 CPUFREQ_TRANSITION_NOTIFIER);
}
core_initcall(cpufreq_tsc);

#endif

/* clock source code */

static unsigned long current_tsc_khz;
static struct clocksource clocksource_tsc;

/*
 * We compare the TSC to the cycle_last value in the clocksource
 * structure to avoid a nasty time-warp issue. This can be observed in
 * a very small window right after one CPU updated cycle_last under
 * xtime lock and the other CPU reads a TSC value which is smaller
 * than the cycle_last reference value due to a TSC which is slighty
 * behind. This delta is nowhere else observable, but in that case it
 * results in a forward time jump in the range of hours due to the
 * unsigned delta calculation of the time keeping core code, which is
 * necessary to support wrapping clocksources like pm timer.
 */
static cycle_t read_tsc(void)
{
	cycle_t ret;

	rdtscll(ret);

	return ret >= clocksource_tsc.cycle_last ?
		ret : clocksource_tsc.cycle_last;
}

static struct clocksource clocksource_tsc = {
	.name			= "tsc",
	.rating			= 300,
	.read			= read_tsc,
	.mask			= CLOCKSOURCE_MASK(64),
	.mult			= 0, /* to be set */
	.shift			= 22,
	.flags			= CLOCK_SOURCE_IS_CONTINUOUS |
				  CLOCK_SOURCE_MUST_VERIFY,
};

void mark_tsc_unstable(char *reason)
{
	if (!tsc_unstable) {
		tsc_unstable = 1;
		tsc_enabled = 0;
		printk("Marking TSC unstable due to: %s.\n", reason);
		/* Can be called before registration */
		if (clocksource_tsc.mult)
			clocksource_change_rating(&clocksource_tsc, 0);
		else
			clocksource_tsc.rating = 0;
	}
}
EXPORT_SYMBOL_GPL(mark_tsc_unstable);

static int __init dmi_mark_tsc_unstable(const struct dmi_system_id *d)
{
	printk(KERN_NOTICE "%s detected: marking TSC unstable.\n",
		       d->ident);
	tsc_unstable = 1;
	return 0;
}

/* List of systems that have known TSC problems */
static struct dmi_system_id __initdata bad_tsc_dmi_table[] = {
	{
	 .callback = dmi_mark_tsc_unstable,
	 .ident = "IBM Thinkpad 380XD",
	 .matches = {
		     DMI_MATCH(DMI_BOARD_VENDOR, "IBM"),
		     DMI_MATCH(DMI_BOARD_NAME, "2635FA0"),
		     },
	 },
	 {}
};

/*
 * Make an educated guess if the TSC is trustworthy and synchronized
 * over all CPUs.
 */
__cpuinit int unsynchronized_tsc(void)
{
	if (!cpu_has_tsc || tsc_unstable)
		return 1;

	/* Anything with constant TSC should be synchronized */
	if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
		return 0;

	/*
	 * Intel systems are normally all synchronized.
	 * Exceptions must mark TSC as unstable:
	 */
	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
		/* assume multi socket systems are not synchronized: */
		if (num_possible_cpus() > 1)
			tsc_unstable = 1;
	}
	return tsc_unstable;
}

/*
 * Geode_LX - the OLPC CPU has a possibly a very reliable TSC
 */
#ifdef CONFIG_MGEODE_LX
/* RTSC counts during suspend */
#define RTSC_SUSP 0x100

static void __init check_geode_tsc_reliable(void)
{
	unsigned long res_low, res_high;

	rdmsr_safe(MSR_GEODE_BUSCONT_CONF0, &res_low, &res_high);
	if (res_low & RTSC_SUSP)
		clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
}
#else
static inline void check_geode_tsc_reliable(void) { }
#endif


void __init tsc_init(void)
{
	int cpu;

	if (!cpu_has_tsc)
		return;

	cpu_khz = calculate_cpu_khz();
	tsc_khz = cpu_khz;

	if (!cpu_khz) {
		mark_tsc_unstable("could not calculate TSC khz");
		return;
	}

	printk("Detected %lu.%03lu MHz processor.\n",
				(unsigned long)cpu_khz / 1000,
				(unsigned long)cpu_khz % 1000);

	/*
	 * Secondary CPUs do not run through tsc_init(), so set up
	 * all the scale factors for all CPUs, assuming the same
	 * speed as the bootup CPU. (cpufreq notifiers will fix this
	 * up if their speed diverges)
	 */
	for_each_possible_cpu(cpu)
		set_cyc2ns_scale(cpu_khz, cpu);

	use_tsc_delay();

	/* Check and install the TSC clocksource */
	dmi_check_system(bad_tsc_dmi_table);

	unsynchronized_tsc();
	check_geode_tsc_reliable();
	current_tsc_khz = tsc_khz;
	clocksource_tsc.mult = clocksource_khz2mult(current_tsc_khz,
							clocksource_tsc.shift);
	/* lower the rating if we already know its unstable: */
	if (check_tsc_unstable()) {
		clocksource_tsc.rating = 0;
		clocksource_tsc.flags &= ~CLOCK_SOURCE_IS_CONTINUOUS;
	} else
		tsc_enabled = 1;

	clocksource_register(&clocksource_tsc);
}
Commit	Line	Data
bb29ab26	1	#include <linux/sched.h>
5d0cf410	2	#include <linux/clocksource.h>
539eb11e JS	3	#include <linux/workqueue.h>
	4	#include <linux/cpufreq.h>
	5	#include <linux/jiffies.h>
	6	#include <linux/init.h>
5d0cf410	7	#include <linux/dmi.h>
53d517cd	8	#include <linux/percpu.h>
539eb11e	9
5d0cf410	10	#include <asm/delay.h>
539eb11e JS	11	#include <asm/tsc.h>
539eb11e JS	12	#include <asm/io.h>
6cb9a835	13	#include <asm/timer.h>
539eb11e JS	14
	15	#include "mach_timer.h"
	16
d9a5c0a4 TG	17	static int tsc_enabled;
d9a5c0a4 TG	18
539eb11e JS	19	/*
	20	* On some systems the TSC frequency does not
	21	* change with the cpu frequency. So we need
	22	* an extra value to store the TSC freq
	23	*/
	24	unsigned int tsc_khz;
d7e28ffe	25	EXPORT_SYMBOL_GPL(tsc_khz);
539eb11e	26
539eb11e JS	27	#ifdef CONFIG_X86_TSC
	28	static int __init tsc_setup(char *str)
	29	{
	30	printk(KERN_WARNING "notsc: Kernel compiled with CONFIG_X86_TSC, "
7265b6f1 PM	31	"cannot disable TSC completely.\n");
7265b6f1 PM	32	mark_tsc_unstable("user disabled TSC");
539eb11e JS	33	return 1;
	34	}
	35	#else
	36	/*
	37	* disable flag for tsc. Takes effect by clearing the TSC cpu flag
	38	* in cpu/common.c
	39	*/
	40	static int __init tsc_setup(char *str)
	41	{
404ee5b1	42	setup_clear_cpu_cap(X86_FEATURE_TSC);
539eb11e JS	43	return 1;
	44	}
	45	#endif
	46
	47	__setup("notsc", tsc_setup);
	48
539eb11e JS	49	/*
	50	* code to mark and check if the TSC is unstable
	51	* due to cpufreq or due to unsynced TSCs
	52	*/
	53	static int tsc_unstable;
	54
d7e28ffe	55	int check_tsc_unstable(void)
539eb11e JS	56	{
	57	return tsc_unstable;
	58	}
d7e28ffe	59	EXPORT_SYMBOL_GPL(check_tsc_unstable);
539eb11e	60
27b46d76	61	/* Accelerators for sched_clock()
539eb11e JS	62	* convert from cycles(64bits) => nanoseconds (64bits)
	63	* basic equation:
	64	* ns = cycles / (freq / ns_per_sec)
	65	* ns = cycles * (ns_per_sec / freq)
	66	* ns = cycles * (10^9 / (cpu_khz * 10^3))
	67	* ns = cycles * (10^6 / cpu_khz)
	68	*
	69	* Then we use scaling math (suggested by george@mvista.com) to get:
	70	* ns = cycles * (10^6 * SC / cpu_khz) / SC
	71	* ns = cycles * cyc2ns_scale / SC
	72	*
	73	* And since SC is a constant power of two, we can convert the div
	74	* into a shift.
	75	*
96315129	76	* We can use khz divisor instead of mhz to keep a better precision, since
539eb11e JS	77	* cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits.
	78	* (mathieu.desnoyers@polymtl.ca)
	79	*
	80	* -johnstul@us.ibm.com "math is hard, lets go shopping!"
	81	*/
539eb11e	82
53d517cd	83	DEFINE_PER_CPU(unsigned long, cyc2ns);
539eb11e	84
53d517cd	85	static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)
539eb11e	86	{
53d517cd	87	unsigned long long tsc_now, ns_now;
1725037f	88	unsigned long flags, *scale;
53d517cd GC	89
	90	local_irq_save(flags);
	91	sched_clock_idle_sleep_event();
	92
	93	scale = &per_cpu(cyc2ns, cpu);
	94
	95	rdtscll(tsc_now);
	96	ns_now = __cycles_2_ns(tsc_now);
	97
53d517cd GC	98	if (cpu_khz)
	99	*scale = (NSEC_PER_MSEC << CYC2NS_SCALE_FACTOR)/cpu_khz;
	100
	101	/*
	102	* Start smoothly with the new frequency:
	103	*/
	104	sched_clock_idle_wakeup_event(0);
	105	local_irq_restore(flags);
539eb11e JS	106	}
539eb11e JS	107
539eb11e JS	108	/*
	109	* Scheduler clock - returns current time in nanosec units.
	110	*/
688340ea	111	unsigned long long native_sched_clock(void)
539eb11e JS	112	{
	113	unsigned long long this_offset;
	114
	115	/*
f9690982	116	* Fall back to jiffies if there's no TSC available:
bb29ab26 IM	117	* ( But note that we still use it if the TSC is marked
	118	* unstable. We do this because unlike Time Of Day,
	119	* the scheduler clock tolerates small errors and it's
	120	* very important for it to be as fast as the platform
	121	* can achive it. )
539eb11e	122	*/
bb29ab26	123	if (unlikely(!tsc_enabled && !tsc_unstable))
f9690982	124	/* No locking but a rare wrong value is not a big deal: */
539eb11e JS	125	return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ);
	126
	127	/* read the Time Stamp Counter: */
688340ea	128	rdtscll(this_offset);
539eb11e JS	129
	130	/* return the value in ns */
	131	return cycles_2_ns(this_offset);
	132	}
	133
688340ea JF	134	/* We need to define a real function for sched_clock, to override the
	135	weak default version */
	136	#ifdef CONFIG_PARAVIRT
	137	unsigned long long sched_clock(void)
	138	{
	139	return paravirt_sched_clock();
	140	}
	141	#else
	142	unsigned long long sched_clock(void)
	143	__attribute__((alias("native_sched_clock")));
	144	#endif
	145
1182d852	146	unsigned long native_calculate_cpu_khz(void)
539eb11e JS	147	{
	148	unsigned long long start, end;
	149	unsigned long count;
edaf420f	150	u64 delta64 = (u64)ULLONG_MAX;
539eb11e JS	151	int i;
	152	unsigned long flags;
	153
	154	local_irq_save(flags);
	155
8c660065	156	/* run 3 times to ensure the cache is warm and to get an accurate reading */
539eb11e JS	157	for (i = 0; i < 3; i++) {
	158	mach_prepare_counter();
	159	rdtscll(start);
	160	mach_countup(&count);
	161	rdtscll(end);
8c660065 DJ	162
	163	/*
	164	* Error: ECTCNEVERSET
	165	* The CTC wasn't reliable: we got a hit on the very first read,
	166	* or the CPU was so fast/slow that the quotient wouldn't fit in
	167	* 32 bits..
	168	*/
	169	if (count <= 1)
	170	continue;
	171
	172	/* cpu freq too slow: */
	173	if ((end - start) <= CALIBRATE_TIME_MSEC)
	174	continue;
	175
	176	/*
	177	* We want the minimum time of all runs in case one of them
	178	* is inaccurate due to SMI or other delay
	179	*/
edaf420f	180	delta64 = min(delta64, (end - start));
539eb11e	181	}
539eb11e	182
8c660065	183	/* cpu freq too fast (or every run was bad): */
539eb11e JS	184	if (delta64 > (1ULL<<32))
	185	goto err;
	186
539eb11e JS	187	delta64 += CALIBRATE_TIME_MSEC/2; /* round for do_div */
	188	do_div(delta64,CALIBRATE_TIME_MSEC);
	189
	190	local_irq_restore(flags);
	191	return (unsigned long)delta64;
	192	err:
	193	local_irq_restore(flags);
	194	return 0;
	195	}
	196
	197	int recalibrate_cpu_khz(void)
	198	{
	199	#ifndef CONFIG_SMP
	200	unsigned long cpu_khz_old = cpu_khz;
	201
	202	if (cpu_has_tsc) {
	203	cpu_khz = calculate_cpu_khz();
	204	tsc_khz = cpu_khz;
92cb7612 MT	205	cpu_data(0).loops_per_jiffy =
92cb7612 MT	206	cpufreq_scale(cpu_data(0).loops_per_jiffy,
539eb11e JS	207	cpu_khz_old, cpu_khz);
	208	return 0;
	209	} else
	210	return -ENODEV;
	211	#else
	212	return -ENODEV;
	213	#endif
	214	}
	215
	216	EXPORT_SYMBOL(recalibrate_cpu_khz);
	217
539eb11e JS	218	#ifdef CONFIG_CPU_FREQ
539eb11e JS	219
539eb11e JS	220	/*
	221	* if the CPU frequency is scaled, TSC-based delays will need a different
	222	* loops_per_jiffy value to function properly.
	223	*/
4bd01600 PM	224	static unsigned int ref_freq;
	225	static unsigned long loops_per_jiffy_ref;
	226	static unsigned long cpu_khz_ref;
539eb11e JS	227
	228	static int
	229	time_cpufreq_notifier(struct notifier_block nb, unsigned long val, void data)
	230	{
	231	struct cpufreq_freqs *freq = data;
	232
539eb11e JS	233	if (!ref_freq) {
	234	if (!freq->old){
	235	ref_freq = freq->new;
df3624aa	236	return 0;
539eb11e JS	237	}
539eb11e JS	238	ref_freq = freq->old;
92cb7612	239	loops_per_jiffy_ref = cpu_data(freq->cpu).loops_per_jiffy;
539eb11e JS	240	cpu_khz_ref = cpu_khz;
	241	}
	242
	243	if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) \|\|
	244	(val == CPUFREQ_POSTCHANGE && freq->old > freq->new) \|\|
	245	(val == CPUFREQ_RESUMECHANGE)) {
	246	if (!(freq->flags & CPUFREQ_CONST_LOOPS))
92cb7612	247	cpu_data(freq->cpu).loops_per_jiffy =
539eb11e JS	248	cpufreq_scale(loops_per_jiffy_ref,
	249	ref_freq, freq->new);
	250
	251	if (cpu_khz) {
	252
	253	if (num_online_cpus() == 1)
	254	cpu_khz = cpufreq_scale(cpu_khz_ref,
	255	ref_freq, freq->new);
	256	if (!(freq->flags & CPUFREQ_CONST_LOOPS)) {
	257	tsc_khz = cpu_khz;
4f41c94d	258	set_cyc2ns_scale(cpu_khz, freq->cpu);
539eb11e JS	259	/*
	260	* TSC based sched_clock turns
	261	* to junk w/ cpufreq
	262	*/
5a90cf20	263	mark_tsc_unstable("cpufreq changes");
539eb11e JS	264	}
	265	}
	266	}
539eb11e JS	267
	268	return 0;
	269	}
	270
	271	static struct notifier_block time_cpufreq_notifier_block = {
	272	.notifier_call = time_cpufreq_notifier
	273	};
	274
	275	static int __init cpufreq_tsc(void)
	276	{
26a08eb3 TG	277	return cpufreq_register_notifier(&time_cpufreq_notifier_block,
26a08eb3 TG	278	CPUFREQ_TRANSITION_NOTIFIER);
539eb11e	279	}
539eb11e JS	280	core_initcall(cpufreq_tsc);
	281
	282	#endif
5d0cf410 JS	283
	284	/* clock source code */
	285
4bd01600	286	static unsigned long current_tsc_khz;
d8bb6f4c	287	static struct clocksource clocksource_tsc;
5d0cf410	288
d8bb6f4c TG	289	/*
	290	* We compare the TSC to the cycle_last value in the clocksource
	291	* structure to avoid a nasty time-warp issue. This can be observed in
	292	* a very small window right after one CPU updated cycle_last under
	293	* xtime lock and the other CPU reads a TSC value which is smaller
	294	* than the cycle_last reference value due to a TSC which is slighty
	295	* behind. This delta is nowhere else observable, but in that case it
	296	* results in a forward time jump in the range of hours due to the
	297	* unsigned delta calculation of the time keeping core code, which is
	298	* necessary to support wrapping clocksources like pm timer.
	299	*/
5d0cf410 JS	300	static cycle_t read_tsc(void)
	301	{
	302	cycle_t ret;
	303
	304	rdtscll(ret);
	305
d8bb6f4c TG	306	return ret >= clocksource_tsc.cycle_last ?
d8bb6f4c TG	307	ret : clocksource_tsc.cycle_last;
5d0cf410 JS	308	}
	309
	310	static struct clocksource clocksource_tsc = {
	311	.name = "tsc",
	312	.rating = 300,
	313	.read = read_tsc,
7f9f303a	314	.mask = CLOCKSOURCE_MASK(64),
5d0cf410 JS	315	.mult = 0, /* to be set */
5d0cf410 JS	316	.shift = 22,
73b08d2a TG	317	.flags = CLOCK_SOURCE_IS_CONTINUOUS \|
73b08d2a TG	318	CLOCK_SOURCE_MUST_VERIFY,
5d0cf410 JS	319	};
5d0cf410 JS	320
5a90cf20	321	void mark_tsc_unstable(char *reason)
5d0cf410	322	{
7e69f2b1 TG	323	if (!tsc_unstable) {
7e69f2b1 TG	324	tsc_unstable = 1;
d9a5c0a4	325	tsc_enabled = 0;
5a90cf20	326	printk("Marking TSC unstable due to: %s.\n", reason);
7e69f2b1 TG	327	/* Can be called before registration */
	328	if (clocksource_tsc.mult)
	329	clocksource_change_rating(&clocksource_tsc, 0);
	330	else
	331	clocksource_tsc.rating = 0;
5d0cf410	332	}
5d0cf410	333	}
7e69f2b1	334	EXPORT_SYMBOL_GPL(mark_tsc_unstable);
5d0cf410	335
1855256c	336	static int __init dmi_mark_tsc_unstable(const struct dmi_system_id *d)
5d0cf410 JS	337	{
	338	printk(KERN_NOTICE "%s detected: marking TSC unstable.\n",
	339	d->ident);
7e69f2b1	340	tsc_unstable = 1;
5d0cf410 JS	341	return 0;
	342	}
	343
	344	/* List of systems that have known TSC problems */
	345	static struct dmi_system_id __initdata bad_tsc_dmi_table[] = {
	346	{
	347	.callback = dmi_mark_tsc_unstable,
	348	.ident = "IBM Thinkpad 380XD",
	349	.matches = {
	350	DMI_MATCH(DMI_BOARD_VENDOR, "IBM"),
	351	DMI_MATCH(DMI_BOARD_NAME, "2635FA0"),
	352	},
	353	},
	354	{}
	355	};
	356
5d0cf410 JS	357	/*
	358	* Make an educated guess if the TSC is trustworthy and synchronized
	359	* over all CPUs.
	360	*/
95492e46	361	__cpuinit int unsynchronized_tsc(void)
5d0cf410	362	{
95492e46 IM	363	if (!cpu_has_tsc \|\| tsc_unstable)
95492e46 IM	364	return 1;
51fc97b9 AK	365
	366	/* Anything with constant TSC should be synchronized */
	367	if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
	368	return 0;
	369
5d0cf410 JS	370	/*
	371	* Intel systems are normally all synchronized.
	372	* Exceptions must mark TSC as unstable:
	373	*/
7e69f2b1 TG	374	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
	375	/* assume multi socket systems are not synchronized: */
	376	if (num_possible_cpus() > 1)
	377	tsc_unstable = 1;
	378	}
	379	return tsc_unstable;
5d0cf410 JS	380	}
5d0cf410 JS	381
07190a08 MT	382	/*
	383	* Geode_LX - the OLPC CPU has a possibly a very reliable TSC
	384	*/
	385	#ifdef CONFIG_MGEODE_LX
	386	/* RTSC counts during suspend */
	387	#define RTSC_SUSP 0x100
	388
	389	static void __init check_geode_tsc_reliable(void)
	390	{
f97586b6	391	unsigned long res_low, res_high;
07190a08	392
f97586b6 IM	393	rdmsr_safe(MSR_GEODE_BUSCONT_CONF0, &res_low, &res_high);
f97586b6 IM	394	if (res_low & RTSC_SUSP)
07190a08 MT	395	clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
	396	}
	397	#else
	398	static inline void check_geode_tsc_reliable(void) { }
	399	#endif
	400
6bb74df4 JS	401
6bb74df4 JS	402	void __init tsc_init(void)
5d0cf410	403	{
53d517cd GC	404	int cpu;
53d517cd GC	405
404ee5b1	406	if (!cpu_has_tsc)
3c2047cd	407	return;
5d0cf410	408
6bb74df4 JS	409	cpu_khz = calculate_cpu_khz();
6bb74df4 JS	410	tsc_khz = cpu_khz;
5d0cf410	411
3c2047cd RR	412	if (!cpu_khz) {
	413	mark_tsc_unstable("could not calculate TSC khz");
	414	return;
	415	}
5d0cf410	416
6bb74df4 JS	417	printk("Detected %lu.%03lu MHz processor.\n",
	418	(unsigned long)cpu_khz / 1000,
	419	(unsigned long)cpu_khz % 1000);
	420
53d517cd GC	421	/*
	422	* Secondary CPUs do not run through tsc_init(), so set up
	423	* all the scale factors for all CPUs, assuming the same
	424	* speed as the bootup CPU. (cpufreq notifiers will fix this
	425	* up if their speed diverges)
	426	*/
	427	for_each_possible_cpu(cpu)
	428	set_cyc2ns_scale(cpu_khz, cpu);
	429
6bb74df4 JS	430	use_tsc_delay();
	431
	432	/* Check and install the TSC clocksource */
	433	dmi_check_system(bad_tsc_dmi_table);
	434
	435	unsynchronized_tsc();
	436	check_geode_tsc_reliable();
	437	current_tsc_khz = tsc_khz;
	438	clocksource_tsc.mult = clocksource_khz2mult(current_tsc_khz,
	439	clocksource_tsc.shift);
	440	/* lower the rating if we already know its unstable: */
	441	if (check_tsc_unstable()) {
	442	clocksource_tsc.rating = 0;
	443	clocksource_tsc.flags &= ~CLOCK_SOURCE_IS_CONTINUOUS;
d9a5c0a4 TG	444	} else
	445	tsc_enabled = 1;
	446
6bb74df4	447	clocksource_register(&clocksource_tsc);
6bb74df4	448	}