[mirror_ubuntu-artful-kernel.git] / arch / x86 / kernel / tsc_32.c

#include <linux/sched.h>
#include <linux/clocksource.h>
#include <linux/workqueue.h>
#include <linux/cpufreq.h>
#include <linux/jiffies.h>
#include <linux/init.h>
#include <linux/dmi.h>
#include <linux/percpu.h>

#include <asm/delay.h>
#include <asm/tsc.h>
#include <asm/io.h>
#include <asm/timer.h>

#include "mach_timer.h"

static int tsc_enabled;

/*
 * On some systems the TSC frequency does not
 * change with the cpu frequency. So we need
 * an extra value to store the TSC freq
 */
unsigned int tsc_khz;
EXPORT_SYMBOL_GPL(tsc_khz);

#ifdef CONFIG_X86_TSC
static int __init tsc_setup(char *str)
{
	printk(KERN_WARNING "notsc: Kernel compiled with CONFIG_X86_TSC, "
				"cannot disable TSC completely.\n");
	mark_tsc_unstable("user disabled TSC");
	return 1;
}
#else
/*
 * disable flag for tsc. Takes effect by clearing the TSC cpu flag
 * in cpu/common.c
 */
static int __init tsc_setup(char *str)
{
	setup_clear_cpu_cap(X86_FEATURE_TSC);
	return 1;
}
#endif

__setup("notsc", tsc_setup);

/*
 * code to mark and check if the TSC is unstable
 * due to cpufreq or due to unsynced TSCs
 */
static int tsc_unstable;

int check_tsc_unstable(void)
{
	return tsc_unstable;
}
EXPORT_SYMBOL_GPL(check_tsc_unstable);

/* Accelerators for sched_clock()
 * convert from cycles(64bits) => nanoseconds (64bits)
 *  basic equation:
 *		ns = cycles / (freq / ns_per_sec)
 *		ns = cycles * (ns_per_sec / freq)
 *		ns = cycles * (10^9 / (cpu_khz * 10^3))
 *		ns = cycles * (10^6 / cpu_khz)
 *
 *	Then we use scaling math (suggested by george@mvista.com) to get:
 *		ns = cycles * (10^6 * SC / cpu_khz) / SC
 *		ns = cycles * cyc2ns_scale / SC
 *
 *	And since SC is a constant power of two, we can convert the div
 *  into a shift.
 *
 *  We can use khz divisor instead of mhz to keep a better precision, since
 *  cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits.
 *  (mathieu.desnoyers@polymtl.ca)
 *
 *			-johnstul@us.ibm.com "math is hard, lets go shopping!"
 */

DEFINE_PER_CPU(unsigned long, cyc2ns);

static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)
{
	unsigned long flags, prev_scale, *scale;
	unsigned long long tsc_now, ns_now;

	local_irq_save(flags);
	sched_clock_idle_sleep_event();

	scale = &per_cpu(cyc2ns, cpu);

	rdtscll(tsc_now);
	ns_now = __cycles_2_ns(tsc_now);

	prev_scale = *scale;
	if (cpu_khz)
		*scale = (NSEC_PER_MSEC << CYC2NS_SCALE_FACTOR)/cpu_khz;

	/*
	 * Start smoothly with the new frequency:
	 */
	sched_clock_idle_wakeup_event(0);
	local_irq_restore(flags);
}

/*
 * Scheduler clock - returns current time in nanosec units.
 */
unsigned long long native_sched_clock(void)
{
	unsigned long long this_offset;

	/*
	 * Fall back to jiffies if there's no TSC available:
	 * ( But note that we still use it if the TSC is marked
	 *   unstable. We do this because unlike Time Of Day,
	 *   the scheduler clock tolerates small errors and it's
	 *   very important for it to be as fast as the platform
	 *   can achive it. )
	 */
	if (unlikely(!tsc_enabled && !tsc_unstable))
		/* No locking but a rare wrong value is not a big deal: */
		return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ);

	/* read the Time Stamp Counter: */
	rdtscll(this_offset);

	/* return the value in ns */
	return cycles_2_ns(this_offset);
}

/* We need to define a real function for sched_clock, to override the
   weak default version */
#ifdef CONFIG_PARAVIRT
unsigned long long sched_clock(void)
{
	return paravirt_sched_clock();
}
#else
unsigned long long sched_clock(void)
	__attribute__((alias("native_sched_clock")));
#endif

unsigned long native_calculate_cpu_khz(void)
{
	unsigned long long start, end;
	unsigned long count;
	u64 delta64 = (u64)ULLONG_MAX;
	int i;
	unsigned long flags;

	local_irq_save(flags);

	/* run 3 times to ensure the cache is warm and to get an accurate reading */
	for (i = 0; i < 3; i++) {
		mach_prepare_counter();
		rdtscll(start);
		mach_countup(&count);
		rdtscll(end);

		/*
		 * Error: ECTCNEVERSET
		 * The CTC wasn't reliable: we got a hit on the very first read,
		 * or the CPU was so fast/slow that the quotient wouldn't fit in
		 * 32 bits..
		 */
		if (count <= 1)
			continue;

		/* cpu freq too slow: */
		if ((end - start) <= CALIBRATE_TIME_MSEC)
			continue;

		/*
		 * We want the minimum time of all runs in case one of them
		 * is inaccurate due to SMI or other delay
		 */
		delta64 = min(delta64, (end - start));
	}

	/* cpu freq too fast (or every run was bad): */
	if (delta64 > (1ULL<<32))
		goto err;

	delta64 += CALIBRATE_TIME_MSEC/2; /* round for do_div */
	do_div(delta64,CALIBRATE_TIME_MSEC);

	local_irq_restore(flags);
	return (unsigned long)delta64;
err:
	local_irq_restore(flags);
	return 0;
}

int recalibrate_cpu_khz(void)
{
#ifndef CONFIG_SMP
	unsigned long cpu_khz_old = cpu_khz;

	if (cpu_has_tsc) {
		cpu_khz = calculate_cpu_khz();
		tsc_khz = cpu_khz;
		cpu_data(0).loops_per_jiffy =
			cpufreq_scale(cpu_data(0).loops_per_jiffy,
					cpu_khz_old, cpu_khz);
		return 0;
	} else
		return -ENODEV;
#else
	return -ENODEV;
#endif
}

EXPORT_SYMBOL(recalibrate_cpu_khz);

#ifdef CONFIG_CPU_FREQ

/*
 * if the CPU frequency is scaled, TSC-based delays will need a different
 * loops_per_jiffy value to function properly.
 */
static unsigned int ref_freq = 0;
static unsigned long loops_per_jiffy_ref = 0;
static unsigned long cpu_khz_ref = 0;

static int
time_cpufreq_notifier(struct notifier_block *nb, unsigned long val, void *data)
{
	struct cpufreq_freqs *freq = data;

	if (!ref_freq) {
		if (!freq->old){
			ref_freq = freq->new;
			return 0;
		}
		ref_freq = freq->old;
		loops_per_jiffy_ref = cpu_data(freq->cpu).loops_per_jiffy;
		cpu_khz_ref = cpu_khz;
	}

	if ((val == CPUFREQ_PRECHANGE  && freq->old < freq->new) ||
	    (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||
	    (val == CPUFREQ_RESUMECHANGE)) {
		if (!(freq->flags & CPUFREQ_CONST_LOOPS))
			cpu_data(freq->cpu).loops_per_jiffy =
				cpufreq_scale(loops_per_jiffy_ref,
						ref_freq, freq->new);

		if (cpu_khz) {

			if (num_online_cpus() == 1)
				cpu_khz = cpufreq_scale(cpu_khz_ref,
						ref_freq, freq->new);
			if (!(freq->flags & CPUFREQ_CONST_LOOPS)) {
				tsc_khz = cpu_khz;
				set_cyc2ns_scale(cpu_khz, freq->cpu);
				/*
				 * TSC based sched_clock turns
				 * to junk w/ cpufreq
				 */
				mark_tsc_unstable("cpufreq changes");
			}
		}
	}

	return 0;
}

static struct notifier_block time_cpufreq_notifier_block = {
	.notifier_call	= time_cpufreq_notifier
};

static int __init cpufreq_tsc(void)
{
	return cpufreq_register_notifier(&time_cpufreq_notifier_block,
					 CPUFREQ_TRANSITION_NOTIFIER);
}
core_initcall(cpufreq_tsc);

#endif

/* clock source code */

static unsigned long current_tsc_khz = 0;

static cycle_t read_tsc(void)
{
	cycle_t ret;

	rdtscll(ret);

	return ret;
}

static struct clocksource clocksource_tsc = {
	.name			= "tsc",
	.rating			= 300,
	.read			= read_tsc,
	.mask			= CLOCKSOURCE_MASK(64),
	.mult			= 0, /* to be set */
	.shift			= 22,
	.flags			= CLOCK_SOURCE_IS_CONTINUOUS |
				  CLOCK_SOURCE_MUST_VERIFY,
};

void mark_tsc_unstable(char *reason)
{
	if (!tsc_unstable) {
		tsc_unstable = 1;
		tsc_enabled = 0;
		printk("Marking TSC unstable due to: %s.\n", reason);
		/* Can be called before registration */
		if (clocksource_tsc.mult)
			clocksource_change_rating(&clocksource_tsc, 0);
		else
			clocksource_tsc.rating = 0;
	}
}
EXPORT_SYMBOL_GPL(mark_tsc_unstable);

static int __init dmi_mark_tsc_unstable(const struct dmi_system_id *d)
{
	printk(KERN_NOTICE "%s detected: marking TSC unstable.\n",
		       d->ident);
	tsc_unstable = 1;
	return 0;
}

/* List of systems that have known TSC problems */
static struct dmi_system_id __initdata bad_tsc_dmi_table[] = {
	{
	 .callback = dmi_mark_tsc_unstable,
	 .ident = "IBM Thinkpad 380XD",
	 .matches = {
		     DMI_MATCH(DMI_BOARD_VENDOR, "IBM"),
		     DMI_MATCH(DMI_BOARD_NAME, "2635FA0"),
		     },
	 },
	 {}
};

/*
 * Make an educated guess if the TSC is trustworthy and synchronized
 * over all CPUs.
 */
__cpuinit int unsynchronized_tsc(void)
{
	if (!cpu_has_tsc || tsc_unstable)
		return 1;

	/* Anything with constant TSC should be synchronized */
	if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
		return 0;

	/*
	 * Intel systems are normally all synchronized.
	 * Exceptions must mark TSC as unstable:
	 */
	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
		/* assume multi socket systems are not synchronized: */
		if (num_possible_cpus() > 1)
			tsc_unstable = 1;
	}
	return tsc_unstable;
}

/*
 * Geode_LX - the OLPC CPU has a possibly a very reliable TSC
 */
#ifdef CONFIG_MGEODE_LX
/* RTSC counts during suspend */
#define RTSC_SUSP 0x100

static void __init check_geode_tsc_reliable(void)
{
	unsigned long res_low, res_high;

	rdmsr_safe(MSR_GEODE_BUSCONT_CONF0, &res_low, &res_high);
	if (res_low & RTSC_SUSP)
		clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
}
#else
static inline void check_geode_tsc_reliable(void) { }
#endif


void __init tsc_init(void)
{
	int cpu;

	if (!cpu_has_tsc)
		goto out_no_tsc;

	cpu_khz = calculate_cpu_khz();
	tsc_khz = cpu_khz;

	if (!cpu_khz)
		goto out_no_tsc;

	printk("Detected %lu.%03lu MHz processor.\n",
				(unsigned long)cpu_khz / 1000,
				(unsigned long)cpu_khz % 1000);

	/*
	 * Secondary CPUs do not run through tsc_init(), so set up
	 * all the scale factors for all CPUs, assuming the same
	 * speed as the bootup CPU. (cpufreq notifiers will fix this
	 * up if their speed diverges)
	 */
	for_each_possible_cpu(cpu)
		set_cyc2ns_scale(cpu_khz, cpu);

	use_tsc_delay();

	/* Check and install the TSC clocksource */
	dmi_check_system(bad_tsc_dmi_table);

	unsynchronized_tsc();
	check_geode_tsc_reliable();
	current_tsc_khz = tsc_khz;
	clocksource_tsc.mult = clocksource_khz2mult(current_tsc_khz,
							clocksource_tsc.shift);
	/* lower the rating if we already know its unstable: */
	if (check_tsc_unstable()) {
		clocksource_tsc.rating = 0;
		clocksource_tsc.flags &= ~CLOCK_SOURCE_IS_CONTINUOUS;
	} else
		tsc_enabled = 1;

	clocksource_register(&clocksource_tsc);

	return;

out_no_tsc:
	setup_clear_cpu_cap(X86_FEATURE_TSC);
}
Commit	Line	Data
bb29ab26	1	#include <linux/sched.h>
5d0cf410	2	#include <linux/clocksource.h>
539eb11e JS	3	#include <linux/workqueue.h>
	4	#include <linux/cpufreq.h>
	5	#include <linux/jiffies.h>
	6	#include <linux/init.h>
5d0cf410	7	#include <linux/dmi.h>
53d517cd	8	#include <linux/percpu.h>
539eb11e	9
5d0cf410	10	#include <asm/delay.h>
539eb11e JS	11	#include <asm/tsc.h>
539eb11e JS	12	#include <asm/io.h>
6cb9a835	13	#include <asm/timer.h>
539eb11e JS	14
	15	#include "mach_timer.h"
	16
d9a5c0a4 TG	17	static int tsc_enabled;
d9a5c0a4 TG	18
539eb11e JS	19	/*
	20	* On some systems the TSC frequency does not
	21	* change with the cpu frequency. So we need
	22	* an extra value to store the TSC freq
	23	*/
	24	unsigned int tsc_khz;
d7e28ffe	25	EXPORT_SYMBOL_GPL(tsc_khz);
539eb11e	26
539eb11e JS	27	#ifdef CONFIG_X86_TSC
	28	static int __init tsc_setup(char *str)
	29	{
	30	printk(KERN_WARNING "notsc: Kernel compiled with CONFIG_X86_TSC, "
7265b6f1 PM	31	"cannot disable TSC completely.\n");
7265b6f1 PM	32	mark_tsc_unstable("user disabled TSC");
539eb11e JS	33	return 1;
	34	}
	35	#else
	36	/*
	37	* disable flag for tsc. Takes effect by clearing the TSC cpu flag
	38	* in cpu/common.c
	39	*/
	40	static int __init tsc_setup(char *str)
	41	{
404ee5b1	42	setup_clear_cpu_cap(X86_FEATURE_TSC);
539eb11e JS	43	return 1;
	44	}
	45	#endif
	46
	47	__setup("notsc", tsc_setup);
	48
539eb11e JS	49	/*
	50	* code to mark and check if the TSC is unstable
	51	* due to cpufreq or due to unsynced TSCs
	52	*/
	53	static int tsc_unstable;
	54
d7e28ffe	55	int check_tsc_unstable(void)
539eb11e JS	56	{
	57	return tsc_unstable;
	58	}
d7e28ffe	59	EXPORT_SYMBOL_GPL(check_tsc_unstable);
539eb11e	60
27b46d76	61	/* Accelerators for sched_clock()
539eb11e JS	62	* convert from cycles(64bits) => nanoseconds (64bits)
	63	* basic equation:
	64	* ns = cycles / (freq / ns_per_sec)
	65	* ns = cycles * (ns_per_sec / freq)
	66	* ns = cycles * (10^9 / (cpu_khz * 10^3))
	67	* ns = cycles * (10^6 / cpu_khz)
	68	*
	69	* Then we use scaling math (suggested by george@mvista.com) to get:
	70	* ns = cycles * (10^6 * SC / cpu_khz) / SC
	71	* ns = cycles * cyc2ns_scale / SC
	72	*
	73	* And since SC is a constant power of two, we can convert the div
	74	* into a shift.
	75	*
96315129	76	* We can use khz divisor instead of mhz to keep a better precision, since
539eb11e JS	77	* cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits.
	78	* (mathieu.desnoyers@polymtl.ca)
	79	*
	80	* -johnstul@us.ibm.com "math is hard, lets go shopping!"
	81	*/
539eb11e	82
53d517cd	83	DEFINE_PER_CPU(unsigned long, cyc2ns);
539eb11e	84
53d517cd	85	static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)
539eb11e	86	{
53d517cd GC	87	unsigned long flags, prev_scale, *scale;
	88	unsigned long long tsc_now, ns_now;
	89
	90	local_irq_save(flags);
	91	sched_clock_idle_sleep_event();
	92
	93	scale = &per_cpu(cyc2ns, cpu);
	94
	95	rdtscll(tsc_now);
	96	ns_now = __cycles_2_ns(tsc_now);
	97
	98	prev_scale = *scale;
	99	if (cpu_khz)
	100	*scale = (NSEC_PER_MSEC << CYC2NS_SCALE_FACTOR)/cpu_khz;
	101
	102	/*
	103	* Start smoothly with the new frequency:
	104	*/
	105	sched_clock_idle_wakeup_event(0);
	106	local_irq_restore(flags);
539eb11e JS	107	}
539eb11e JS	108
539eb11e JS	109	/*
	110	* Scheduler clock - returns current time in nanosec units.
	111	*/
688340ea	112	unsigned long long native_sched_clock(void)
539eb11e JS	113	{
	114	unsigned long long this_offset;
	115
	116	/*
f9690982	117	* Fall back to jiffies if there's no TSC available:
bb29ab26 IM	118	* ( But note that we still use it if the TSC is marked
	119	* unstable. We do this because unlike Time Of Day,
	120	* the scheduler clock tolerates small errors and it's
	121	* very important for it to be as fast as the platform
	122	* can achive it. )
539eb11e	123	*/
bb29ab26	124	if (unlikely(!tsc_enabled && !tsc_unstable))
f9690982	125	/* No locking but a rare wrong value is not a big deal: */
539eb11e JS	126	return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ);
	127
	128	/* read the Time Stamp Counter: */
688340ea	129	rdtscll(this_offset);
539eb11e JS	130
	131	/* return the value in ns */
	132	return cycles_2_ns(this_offset);
	133	}
	134
688340ea JF	135	/* We need to define a real function for sched_clock, to override the
	136	weak default version */
	137	#ifdef CONFIG_PARAVIRT
	138	unsigned long long sched_clock(void)
	139	{
	140	return paravirt_sched_clock();
	141	}
	142	#else
	143	unsigned long long sched_clock(void)
	144	__attribute__((alias("native_sched_clock")));
	145	#endif
	146
1182d852	147	unsigned long native_calculate_cpu_khz(void)
539eb11e JS	148	{
	149	unsigned long long start, end;
	150	unsigned long count;
edaf420f	151	u64 delta64 = (u64)ULLONG_MAX;
539eb11e JS	152	int i;
	153	unsigned long flags;
	154
	155	local_irq_save(flags);
	156
8c660065	157	/* run 3 times to ensure the cache is warm and to get an accurate reading */
539eb11e JS	158	for (i = 0; i < 3; i++) {
	159	mach_prepare_counter();
	160	rdtscll(start);
	161	mach_countup(&count);
	162	rdtscll(end);
8c660065 DJ	163
	164	/*
	165	* Error: ECTCNEVERSET
	166	* The CTC wasn't reliable: we got a hit on the very first read,
	167	* or the CPU was so fast/slow that the quotient wouldn't fit in
	168	* 32 bits..
	169	*/
	170	if (count <= 1)
	171	continue;
	172
	173	/* cpu freq too slow: */
	174	if ((end - start) <= CALIBRATE_TIME_MSEC)
	175	continue;
	176
	177	/*
	178	* We want the minimum time of all runs in case one of them
	179	* is inaccurate due to SMI or other delay
	180	*/
edaf420f	181	delta64 = min(delta64, (end - start));
539eb11e	182	}
539eb11e	183
8c660065	184	/* cpu freq too fast (or every run was bad): */
539eb11e JS	185	if (delta64 > (1ULL<<32))
	186	goto err;
	187
539eb11e JS	188	delta64 += CALIBRATE_TIME_MSEC/2; /* round for do_div */
	189	do_div(delta64,CALIBRATE_TIME_MSEC);
	190
	191	local_irq_restore(flags);
	192	return (unsigned long)delta64;
	193	err:
	194	local_irq_restore(flags);
	195	return 0;
	196	}
	197
	198	int recalibrate_cpu_khz(void)
	199	{
	200	#ifndef CONFIG_SMP
	201	unsigned long cpu_khz_old = cpu_khz;
	202
	203	if (cpu_has_tsc) {
	204	cpu_khz = calculate_cpu_khz();
	205	tsc_khz = cpu_khz;
92cb7612 MT	206	cpu_data(0).loops_per_jiffy =
92cb7612 MT	207	cpufreq_scale(cpu_data(0).loops_per_jiffy,
539eb11e JS	208	cpu_khz_old, cpu_khz);
	209	return 0;
	210	} else
	211	return -ENODEV;
	212	#else
	213	return -ENODEV;
	214	#endif
	215	}
	216
	217	EXPORT_SYMBOL(recalibrate_cpu_khz);
	218
539eb11e JS	219	#ifdef CONFIG_CPU_FREQ
539eb11e JS	220
539eb11e JS	221	/*
	222	* if the CPU frequency is scaled, TSC-based delays will need a different
	223	* loops_per_jiffy value to function properly.
	224	*/
	225	static unsigned int ref_freq = 0;
	226	static unsigned long loops_per_jiffy_ref = 0;
	227	static unsigned long cpu_khz_ref = 0;
	228
	229	static int
	230	time_cpufreq_notifier(struct notifier_block nb, unsigned long val, void data)
	231	{
	232	struct cpufreq_freqs *freq = data;
	233
539eb11e JS	234	if (!ref_freq) {
	235	if (!freq->old){
	236	ref_freq = freq->new;
df3624aa	237	return 0;
539eb11e JS	238	}
539eb11e JS	239	ref_freq = freq->old;
92cb7612	240	loops_per_jiffy_ref = cpu_data(freq->cpu).loops_per_jiffy;
539eb11e JS	241	cpu_khz_ref = cpu_khz;
	242	}
	243
	244	if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) \|\|
	245	(val == CPUFREQ_POSTCHANGE && freq->old > freq->new) \|\|
	246	(val == CPUFREQ_RESUMECHANGE)) {
	247	if (!(freq->flags & CPUFREQ_CONST_LOOPS))
92cb7612	248	cpu_data(freq->cpu).loops_per_jiffy =
539eb11e JS	249	cpufreq_scale(loops_per_jiffy_ref,
	250	ref_freq, freq->new);
	251
	252	if (cpu_khz) {
	253
	254	if (num_online_cpus() == 1)
	255	cpu_khz = cpufreq_scale(cpu_khz_ref,
	256	ref_freq, freq->new);
	257	if (!(freq->flags & CPUFREQ_CONST_LOOPS)) {
	258	tsc_khz = cpu_khz;
4f41c94d	259	set_cyc2ns_scale(cpu_khz, freq->cpu);
539eb11e JS	260	/*
	261	* TSC based sched_clock turns
	262	* to junk w/ cpufreq
	263	*/
5a90cf20	264	mark_tsc_unstable("cpufreq changes");
539eb11e JS	265	}
	266	}
	267	}
539eb11e JS	268
	269	return 0;
	270	}
	271
	272	static struct notifier_block time_cpufreq_notifier_block = {
	273	.notifier_call = time_cpufreq_notifier
	274	};
	275
	276	static int __init cpufreq_tsc(void)
	277	{
26a08eb3 TG	278	return cpufreq_register_notifier(&time_cpufreq_notifier_block,
26a08eb3 TG	279	CPUFREQ_TRANSITION_NOTIFIER);
539eb11e	280	}
539eb11e JS	281	core_initcall(cpufreq_tsc);
	282
	283	#endif
5d0cf410 JS	284
	285	/* clock source code */
	286
	287	static unsigned long current_tsc_khz = 0;
5d0cf410 JS	288
	289	static cycle_t read_tsc(void)
	290	{
	291	cycle_t ret;
	292
	293	rdtscll(ret);
	294
5b13d863	295	return ret;
5d0cf410 JS	296	}
	297
	298	static struct clocksource clocksource_tsc = {
	299	.name = "tsc",
	300	.rating = 300,
	301	.read = read_tsc,
7f9f303a	302	.mask = CLOCKSOURCE_MASK(64),
5d0cf410 JS	303	.mult = 0, /* to be set */
5d0cf410 JS	304	.shift = 22,
73b08d2a TG	305	.flags = CLOCK_SOURCE_IS_CONTINUOUS \|
73b08d2a TG	306	CLOCK_SOURCE_MUST_VERIFY,
5d0cf410 JS	307	};
5d0cf410 JS	308
5a90cf20	309	void mark_tsc_unstable(char *reason)
5d0cf410	310	{
7e69f2b1 TG	311	if (!tsc_unstable) {
7e69f2b1 TG	312	tsc_unstable = 1;
d9a5c0a4	313	tsc_enabled = 0;
5a90cf20	314	printk("Marking TSC unstable due to: %s.\n", reason);
7e69f2b1 TG	315	/* Can be called before registration */
	316	if (clocksource_tsc.mult)
	317	clocksource_change_rating(&clocksource_tsc, 0);
	318	else
	319	clocksource_tsc.rating = 0;
5d0cf410	320	}
5d0cf410	321	}
7e69f2b1	322	EXPORT_SYMBOL_GPL(mark_tsc_unstable);
5d0cf410	323
1855256c	324	static int __init dmi_mark_tsc_unstable(const struct dmi_system_id *d)
5d0cf410 JS	325	{
	326	printk(KERN_NOTICE "%s detected: marking TSC unstable.\n",
	327	d->ident);
7e69f2b1	328	tsc_unstable = 1;
5d0cf410 JS	329	return 0;
	330	}
	331
	332	/* List of systems that have known TSC problems */
	333	static struct dmi_system_id __initdata bad_tsc_dmi_table[] = {
	334	{
	335	.callback = dmi_mark_tsc_unstable,
	336	.ident = "IBM Thinkpad 380XD",
	337	.matches = {
	338	DMI_MATCH(DMI_BOARD_VENDOR, "IBM"),
	339	DMI_MATCH(DMI_BOARD_NAME, "2635FA0"),
	340	},
	341	},
	342	{}
	343	};
	344
5d0cf410 JS	345	/*
	346	* Make an educated guess if the TSC is trustworthy and synchronized
	347	* over all CPUs.
	348	*/
95492e46	349	__cpuinit int unsynchronized_tsc(void)
5d0cf410	350	{
95492e46 IM	351	if (!cpu_has_tsc \|\| tsc_unstable)
95492e46 IM	352	return 1;
51fc97b9 AK	353
	354	/* Anything with constant TSC should be synchronized */
	355	if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
	356	return 0;
	357
5d0cf410 JS	358	/*
	359	* Intel systems are normally all synchronized.
	360	* Exceptions must mark TSC as unstable:
	361	*/
7e69f2b1 TG	362	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
	363	/* assume multi socket systems are not synchronized: */
	364	if (num_possible_cpus() > 1)
	365	tsc_unstable = 1;
	366	}
	367	return tsc_unstable;
5d0cf410 JS	368	}
5d0cf410 JS	369
07190a08 MT	370	/*
	371	* Geode_LX - the OLPC CPU has a possibly a very reliable TSC
	372	*/
	373	#ifdef CONFIG_MGEODE_LX
	374	/* RTSC counts during suspend */
	375	#define RTSC_SUSP 0x100
	376
	377	static void __init check_geode_tsc_reliable(void)
	378	{
f97586b6	379	unsigned long res_low, res_high;
07190a08	380
f97586b6 IM	381	rdmsr_safe(MSR_GEODE_BUSCONT_CONF0, &res_low, &res_high);
f97586b6 IM	382	if (res_low & RTSC_SUSP)
07190a08 MT	383	clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
	384	}
	385	#else
	386	static inline void check_geode_tsc_reliable(void) { }
	387	#endif
	388
6bb74df4 JS	389
6bb74df4 JS	390	void __init tsc_init(void)
5d0cf410	391	{
53d517cd GC	392	int cpu;
53d517cd GC	393
404ee5b1	394	if (!cpu_has_tsc)
6bb74df4	395	goto out_no_tsc;
5d0cf410	396
6bb74df4 JS	397	cpu_khz = calculate_cpu_khz();
6bb74df4 JS	398	tsc_khz = cpu_khz;
5d0cf410	399
6bb74df4 JS	400	if (!cpu_khz)
6bb74df4 JS	401	goto out_no_tsc;
5d0cf410	402
6bb74df4 JS	403	printk("Detected %lu.%03lu MHz processor.\n",
	404	(unsigned long)cpu_khz / 1000,
	405	(unsigned long)cpu_khz % 1000);
	406
53d517cd GC	407	/*
	408	* Secondary CPUs do not run through tsc_init(), so set up
	409	* all the scale factors for all CPUs, assuming the same
	410	* speed as the bootup CPU. (cpufreq notifiers will fix this
	411	* up if their speed diverges)
	412	*/
	413	for_each_possible_cpu(cpu)
	414	set_cyc2ns_scale(cpu_khz, cpu);
	415
6bb74df4 JS	416	use_tsc_delay();
	417
	418	/* Check and install the TSC clocksource */
	419	dmi_check_system(bad_tsc_dmi_table);
	420
	421	unsynchronized_tsc();
	422	check_geode_tsc_reliable();
	423	current_tsc_khz = tsc_khz;
	424	clocksource_tsc.mult = clocksource_khz2mult(current_tsc_khz,
	425	clocksource_tsc.shift);
	426	/* lower the rating if we already know its unstable: */
	427	if (check_tsc_unstable()) {
	428	clocksource_tsc.rating = 0;
	429	clocksource_tsc.flags &= ~CLOCK_SOURCE_IS_CONTINUOUS;
d9a5c0a4 TG	430	} else
	431	tsc_enabled = 1;
	432
6bb74df4	433	clocksource_register(&clocksource_tsc);
5d0cf410	434
6bb74df4	435	return;
5d0cf410	436
6bb74df4	437	out_no_tsc:
404ee5b1	438	setup_clear_cpu_cap(X86_FEATURE_TSC);
6bb74df4	439	}