[mirror_ubuntu-jammy-kernel.git] / arch / x86 / kernel / vsyscall_64.c

/*
 *  Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
 *  Copyright 2003 Andi Kleen, SuSE Labs.
 *
 *  Thanks to hpa@transmeta.com for some useful hint.
 *  Special thanks to Ingo Molnar for his early experience with
 *  a different vsyscall implementation for Linux/IA32 and for the name.
 *
 *  vsyscall 1 is located at -10Mbyte, vsyscall 2 is located
 *  at virtual address -10Mbyte+1024bytes etc... There are at max 4
 *  vsyscalls. One vsyscall can reserve more than 1 slot to avoid
 *  jumping out of line if necessary. We cannot add more with this
 *  mechanism because older kernels won't return -ENOSYS.
 *  If we want more than four we need a vDSO.
 *
 *  Note: the concept clashes with user mode linux. If you use UML and
 *  want per guest time just set the kernel.vsyscall64 sysctl to 0.
 */

#include <linux/time.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/timer.h>
#include <linux/seqlock.h>
#include <linux/jiffies.h>
#include <linux/sysctl.h>
#include <linux/clocksource.h>
#include <linux/getcpu.h>
#include <linux/cpu.h>
#include <linux/smp.h>
#include <linux/notifier.h>

#include <asm/vsyscall.h>
#include <asm/pgtable.h>
#include <asm/page.h>
#include <asm/unistd.h>
#include <asm/fixmap.h>
#include <asm/errno.h>
#include <asm/io.h>
#include <asm/segment.h>
#include <asm/desc.h>
#include <asm/topology.h>
#include <asm/vgtod.h>

#define __vsyscall(nr) __attribute__ ((unused,__section__(".vsyscall_" #nr)))
#define __syscall_clobber "r11","rcx","memory"
#define __pa_vsymbol(x)			\
	({unsigned long v;  		\
	extern char __vsyscall_0; 	\
	  asm("" : "=r" (v) : "0" (x)); \
	  ((v - VSYSCALL_FIRST_PAGE) + __pa_symbol(&__vsyscall_0)); })

/*
 * vsyscall_gtod_data contains data that is :
 * - readonly from vsyscalls
 * - writen by timer interrupt or systcl (/proc/sys/kernel/vsyscall64)
 * Try to keep this structure as small as possible to avoid cache line ping pongs
 */
int __vgetcpu_mode __section_vgetcpu_mode;

struct vsyscall_gtod_data __vsyscall_gtod_data __section_vsyscall_gtod_data =
{
	.lock = SEQLOCK_UNLOCKED,
	.sysctl_enabled = 1,
};

void update_vsyscall(struct timespec *wall_time, struct clocksource *clock)
{
	unsigned long flags;

	write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
	/* copy vsyscall data */
	vsyscall_gtod_data.clock.vread = clock->vread;
	vsyscall_gtod_data.clock.cycle_last = clock->cycle_last;
	vsyscall_gtod_data.clock.mask = clock->mask;
	vsyscall_gtod_data.clock.mult = clock->mult;
	vsyscall_gtod_data.clock.shift = clock->shift;
	vsyscall_gtod_data.wall_time_sec = wall_time->tv_sec;
	vsyscall_gtod_data.wall_time_nsec = wall_time->tv_nsec;
	vsyscall_gtod_data.sys_tz = sys_tz;
	vsyscall_gtod_data.wall_time_nsec = wall_time->tv_nsec;
	vsyscall_gtod_data.wall_to_monotonic = wall_to_monotonic;
	write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
}

/* RED-PEN may want to readd seq locking, but then the variable should be
 * write-once.
 */
static __always_inline void do_get_tz(struct timezone * tz)
{
	*tz = __vsyscall_gtod_data.sys_tz;
}

static __always_inline int gettimeofday(struct timeval *tv, struct timezone *tz)
{
	int ret;
	asm volatile("vsysc2: syscall"
		: "=a" (ret)
		: "0" (__NR_gettimeofday),"D" (tv),"S" (tz)
		: __syscall_clobber );
	return ret;
}

static __always_inline long time_syscall(long *t)
{
	long secs;
	asm volatile("vsysc1: syscall"
		: "=a" (secs)
		: "0" (__NR_time),"D" (t) : __syscall_clobber);
	return secs;
}

static __always_inline void do_vgettimeofday(struct timeval * tv)
{
	cycle_t now, base, mask, cycle_delta;
	unsigned seq;
	unsigned long mult, shift, nsec;
	cycle_t (*vread)(void);
	do {
		seq = read_seqbegin(&__vsyscall_gtod_data.lock);

		vread = __vsyscall_gtod_data.clock.vread;
		if (unlikely(!__vsyscall_gtod_data.sysctl_enabled || !vread)) {
			gettimeofday(tv,NULL);
			return;
		}
		now = vread();
		base = __vsyscall_gtod_data.clock.cycle_last;
		mask = __vsyscall_gtod_data.clock.mask;
		mult = __vsyscall_gtod_data.clock.mult;
		shift = __vsyscall_gtod_data.clock.shift;

		tv->tv_sec = __vsyscall_gtod_data.wall_time_sec;
		nsec = __vsyscall_gtod_data.wall_time_nsec;
	} while (read_seqretry(&__vsyscall_gtod_data.lock, seq));

	/* calculate interval: */
	cycle_delta = (now - base) & mask;
	/* convert to nsecs: */
	nsec += (cycle_delta * mult) >> shift;

	while (nsec >= NSEC_PER_SEC) {
		tv->tv_sec += 1;
		nsec -= NSEC_PER_SEC;
	}
	tv->tv_usec = nsec / NSEC_PER_USEC;
}

int __vsyscall(0) vgettimeofday(struct timeval * tv, struct timezone * tz)
{
	if (tv)
		do_vgettimeofday(tv);
	if (tz)
		do_get_tz(tz);
	return 0;
}

/* This will break when the xtime seconds get inaccurate, but that is
 * unlikely */
time_t __vsyscall(1) vtime(time_t *t)
{
	struct timeval tv;
	time_t result;
	if (unlikely(!__vsyscall_gtod_data.sysctl_enabled))
		return time_syscall(t);

	vgettimeofday(&tv, 0);
	result = tv.tv_sec;
	if (t)
		*t = result;
	return result;
}

/* Fast way to get current CPU and node.
   This helps to do per node and per CPU caches in user space.
   The result is not guaranteed without CPU affinity, but usually
   works out because the scheduler tries to keep a thread on the same
   CPU.

   tcache must point to a two element sized long array.
   All arguments can be NULL. */
long __vsyscall(2)
vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
{
	unsigned int dummy, p;
	unsigned long j = 0;

	/* Fast cache - only recompute value once per jiffies and avoid
	   relatively costly rdtscp/cpuid otherwise.
	   This works because the scheduler usually keeps the process
	   on the same CPU and this syscall doesn't guarantee its
	   results anyways.
	   We do this here because otherwise user space would do it on
	   its own in a likely inferior way (no access to jiffies).
	   If you don't like it pass NULL. */
	if (tcache && tcache->blob[0] == (j = __jiffies)) {
		p = tcache->blob[1];
	} else if (__vgetcpu_mode == VGETCPU_RDTSCP) {
		/* Load per CPU data from RDTSCP */
		rdtscp(dummy, dummy, p);
	} else {
		/* Load per CPU data from GDT */
		asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
	}
	if (tcache) {
		tcache->blob[0] = j;
		tcache->blob[1] = p;
	}
	if (cpu)
		*cpu = p & 0xfff;
	if (node)
		*node = p >> 12;
	return 0;
}

long __vsyscall(3) venosys_1(void)
{
	return -ENOSYS;
}

#ifdef CONFIG_SYSCTL

#define SYSCALL 0x050f
#define NOP2    0x9090

/*
 * NOP out syscall in vsyscall page when not needed.
 */
static int vsyscall_sysctl_change(ctl_table *ctl, int write, struct file * filp,
                        void __user *buffer, size_t *lenp, loff_t *ppos)
{
	extern u16 vsysc1, vsysc2;
	u16 __iomem *map1;
	u16 __iomem *map2;
	int ret = proc_dointvec(ctl, write, filp, buffer, lenp, ppos);
	if (!write)
		return ret;
	/* gcc has some trouble with __va(__pa()), so just do it this
	   way. */
	map1 = ioremap(__pa_vsymbol(&vsysc1), 2);
	if (!map1)
		return -ENOMEM;
	map2 = ioremap(__pa_vsymbol(&vsysc2), 2);
	if (!map2) {
		ret = -ENOMEM;
		goto out;
	}
	if (!vsyscall_gtod_data.sysctl_enabled) {
		writew(SYSCALL, map1);
		writew(SYSCALL, map2);
	} else {
		writew(NOP2, map1);
		writew(NOP2, map2);
	}
	iounmap(map2);
out:
	iounmap(map1);
	return ret;
}

static int vsyscall_sysctl_nostrat(ctl_table *t, int __user *name, int nlen,
				void __user *oldval, size_t __user *oldlenp,
				void __user *newval, size_t newlen)
{
	return -ENOSYS;
}

static ctl_table kernel_table2[] = {
	{ .ctl_name = 99, .procname = "vsyscall64",
	  .data = &vsyscall_gtod_data.sysctl_enabled, .maxlen = sizeof(int),
	  .mode = 0644,
	  .strategy = vsyscall_sysctl_nostrat,
	  .proc_handler = vsyscall_sysctl_change },
	{}
};

static ctl_table kernel_root_table2[] = {
	{ .ctl_name = CTL_KERN, .procname = "kernel", .mode = 0555,
	  .child = kernel_table2 },
	{}
};

#endif

/* Assume __initcall executes before all user space. Hopefully kmod
   doesn't violate that. We'll find out if it does. */
static void __cpuinit vsyscall_set_cpu(int cpu)
{
	unsigned long *d;
	unsigned long node = 0;
#ifdef CONFIG_NUMA
	node = cpu_to_node[cpu];
#endif
	if (cpu_has(&cpu_data[cpu], X86_FEATURE_RDTSCP))
		write_rdtscp_aux((node << 12) | cpu);

	/* Store cpu number in limit so that it can be loaded quickly
	   in user space in vgetcpu.
	   12 bits for the CPU and 8 bits for the node. */
	d = (unsigned long *)(cpu_gdt(cpu) + GDT_ENTRY_PER_CPU);
	*d = 0x0f40000000000ULL;
	*d |= cpu;
	*d |= (node & 0xf) << 12;
	*d |= (node >> 4) << 48;
}

static void __cpuinit cpu_vsyscall_init(void *arg)
{
	/* preemption should be already off */
	vsyscall_set_cpu(raw_smp_processor_id());
}

static int __cpuinit
cpu_vsyscall_notifier(struct notifier_block *n, unsigned long action, void *arg)
{
	long cpu = (long)arg;
	if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
		smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 0, 1);
	return NOTIFY_DONE;
}

static void __init map_vsyscall(void)
{
	extern char __vsyscall_0;
	unsigned long physaddr_page0 = __pa_symbol(&__vsyscall_0);

	/* Note that VSYSCALL_MAPPED_PAGES must agree with the code below. */
	__set_fixmap(VSYSCALL_FIRST_PAGE, physaddr_page0, PAGE_KERNEL_VSYSCALL);
}

static int __init vsyscall_init(void)
{
	BUG_ON(((unsigned long) &vgettimeofday !=
			VSYSCALL_ADDR(__NR_vgettimeofday)));
	BUG_ON((unsigned long) &vtime != VSYSCALL_ADDR(__NR_vtime));
	BUG_ON((VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE)));
	BUG_ON((unsigned long) &vgetcpu != VSYSCALL_ADDR(__NR_vgetcpu));
	map_vsyscall();
#ifdef CONFIG_SYSCTL
	register_sysctl_table(kernel_root_table2);
#endif
	on_each_cpu(cpu_vsyscall_init, NULL, 0, 1);
	hotcpu_notifier(cpu_vsyscall_notifier, 0);
	return 0;
}

__initcall(vsyscall_init);
Commit	Line	Data
1da177e4	1	/*
1da177e4 LT	2	* Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
	3	* Copyright 2003 Andi Kleen, SuSE Labs.
	4	*
	5	* Thanks to hpa@transmeta.com for some useful hint.
	6	* Special thanks to Ingo Molnar for his early experience with
	7	* a different vsyscall implementation for Linux/IA32 and for the name.
	8	*
	9	* vsyscall 1 is located at -10Mbyte, vsyscall 2 is located
	10	* at virtual address -10Mbyte+1024bytes etc... There are at max 4
	11	* vsyscalls. One vsyscall can reserve more than 1 slot to avoid
	12	* jumping out of line if necessary. We cannot add more with this
	13	* mechanism because older kernels won't return -ENOSYS.
	14	* If we want more than four we need a vDSO.
	15	*
	16	* Note: the concept clashes with user mode linux. If you use UML and
	17	* want per guest time just set the kernel.vsyscall64 sysctl to 0.
	18	*/
	19
	20	#include <linux/time.h>
	21	#include <linux/init.h>
	22	#include <linux/kernel.h>
	23	#include <linux/timer.h>
	24	#include <linux/seqlock.h>
	25	#include <linux/jiffies.h>
	26	#include <linux/sysctl.h>
7460ed28	27	#include <linux/clocksource.h>
c08c8205	28	#include <linux/getcpu.h>
8c131af1 AK	29	#include <linux/cpu.h>
	30	#include <linux/smp.h>
	31	#include <linux/notifier.h>
1da177e4 LT	32
	33	#include <asm/vsyscall.h>
	34	#include <asm/pgtable.h>
	35	#include <asm/page.h>
7460ed28	36	#include <asm/unistd.h>
1da177e4 LT	37	#include <asm/fixmap.h>
	38	#include <asm/errno.h>
	39	#include <asm/io.h>
c08c8205 VP	40	#include <asm/segment.h>
	41	#include <asm/desc.h>
	42	#include <asm/topology.h>
2aae950b	43	#include <asm/vgtod.h>
1da177e4 LT	44
1da177e4 LT	45	#define __vsyscall(nr) __attribute__ ((unused,__section__(".vsyscall_" #nr)))
f5738cee	46	#define __syscall_clobber "r11","rcx","memory"
0dbf7028 VG	47	#define __pa_vsymbol(x) \
	48	({unsigned long v; \
	49	extern char __vsyscall_0; \
	50	asm("" : "=r" (v) : "0" (x)); \
	51	((v - VSYSCALL_FIRST_PAGE) + __pa_symbol(&__vsyscall_0)); })
1da177e4	52
c8118c6c ED	53	/*
	54	* vsyscall_gtod_data contains data that is :
	55	* - readonly from vsyscalls
	56	* - writen by timer interrupt or systcl (/proc/sys/kernel/vsyscall64)
	57	* Try to keep this structure as small as possible to avoid cache line ping pongs
	58	*/
c08c8205	59	int __vgetcpu_mode __section_vgetcpu_mode;
1da177e4	60
2aae950b	61	struct vsyscall_gtod_data __vsyscall_gtod_data __section_vsyscall_gtod_data =
1da177e4	62	{
7460ed28 JS	63	.lock = SEQLOCK_UNLOCKED,
	64	.sysctl_enabled = 1,
	65	};
1da177e4	66
7460ed28	67	void update_vsyscall(struct timespec wall_time, struct clocksource clock)
1da177e4	68	{
7460ed28	69	unsigned long flags;
1da177e4	70
7460ed28 JS	71	write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
7460ed28 JS	72	/* copy vsyscall data */
c8118c6c ED	73	vsyscall_gtod_data.clock.vread = clock->vread;
	74	vsyscall_gtod_data.clock.cycle_last = clock->cycle_last;
	75	vsyscall_gtod_data.clock.mask = clock->mask;
	76	vsyscall_gtod_data.clock.mult = clock->mult;
	77	vsyscall_gtod_data.clock.shift = clock->shift;
	78	vsyscall_gtod_data.wall_time_sec = wall_time->tv_sec;
	79	vsyscall_gtod_data.wall_time_nsec = wall_time->tv_nsec;
7460ed28	80	vsyscall_gtod_data.sys_tz = sys_tz;
2aae950b AK	81	vsyscall_gtod_data.wall_time_nsec = wall_time->tv_nsec;
2aae950b AK	82	vsyscall_gtod_data.wall_to_monotonic = wall_to_monotonic;
7460ed28	83	write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
1da177e4 LT	84	}
1da177e4 LT	85
7460ed28 JS	86	/* RED-PEN may want to readd seq locking, but then the variable should be
	87	* write-once.
	88	*/
2c8bc944	89	static __always_inline void do_get_tz(struct timezone * tz)
1da177e4	90	{
7460ed28	91	*tz = __vsyscall_gtod_data.sys_tz;
1da177e4 LT	92	}
1da177e4 LT	93
2c8bc944	94	static __always_inline int gettimeofday(struct timeval tv, struct timezone tz)
1da177e4 LT	95	{
	96	int ret;
	97	asm volatile("vsysc2: syscall"
	98	: "=a" (ret)
7460ed28 JS	99	: "0" (__NR_gettimeofday),"D" (tv),"S" (tz)
7460ed28 JS	100	: __syscall_clobber );
1da177e4 LT	101	return ret;
	102	}
	103
2c8bc944	104	static __always_inline long time_syscall(long *t)
1da177e4 LT	105	{
	106	long secs;
	107	asm volatile("vsysc1: syscall"
	108	: "=a" (secs)
	109	: "0" (__NR_time),"D" (t) : __syscall_clobber);
	110	return secs;
	111	}
	112
7460ed28 JS	113	static __always_inline void do_vgettimeofday(struct timeval * tv)
	114	{
	115	cycle_t now, base, mask, cycle_delta;
c8118c6c ED	116	unsigned seq;
c8118c6c ED	117	unsigned long mult, shift, nsec;
7460ed28 JS	118	cycle_t (*vread)(void);
	119	do {
	120	seq = read_seqbegin(&__vsyscall_gtod_data.lock);
	121
	122	vread = __vsyscall_gtod_data.clock.vread;
	123	if (unlikely(!__vsyscall_gtod_data.sysctl_enabled \|\| !vread)) {
89952d13	124	gettimeofday(tv,NULL);
7460ed28 JS	125	return;
	126	}
	127	now = vread();
	128	base = __vsyscall_gtod_data.clock.cycle_last;
	129	mask = __vsyscall_gtod_data.clock.mask;
	130	mult = __vsyscall_gtod_data.clock.mult;
	131	shift = __vsyscall_gtod_data.clock.shift;
	132
c8118c6c ED	133	tv->tv_sec = __vsyscall_gtod_data.wall_time_sec;
c8118c6c ED	134	nsec = __vsyscall_gtod_data.wall_time_nsec;
7460ed28 JS	135	} while (read_seqretry(&__vsyscall_gtod_data.lock, seq));
	136
	137	/* calculate interval: */
	138	cycle_delta = (now - base) & mask;
	139	/* convert to nsecs: */
c8118c6c	140	nsec += (cycle_delta * mult) >> shift;
7460ed28	141
c8118c6c	142	while (nsec >= NSEC_PER_SEC) {
7460ed28	143	tv->tv_sec += 1;
c8118c6c	144	nsec -= NSEC_PER_SEC;
7460ed28	145	}
c8118c6c	146	tv->tv_usec = nsec / NSEC_PER_USEC;
7460ed28 JS	147	}
7460ed28 JS	148
2e8ad43e	149	int __vsyscall(0) vgettimeofday(struct timeval * tv, struct timezone * tz)
1da177e4	150	{
1da177e4 LT	151	if (tv)
	152	do_vgettimeofday(tv);
	153	if (tz)
	154	do_get_tz(tz);
	155	return 0;
	156	}
	157
	158	/* This will break when the xtime seconds get inaccurate, but that is
	159	* unlikely */
2e8ad43e	160	time_t __vsyscall(1) vtime(time_t *t)
1da177e4	161	{
d0aff6e6	162	struct timeval tv;
272a3713	163	time_t result;
7460ed28	164	if (unlikely(!__vsyscall_gtod_data.sysctl_enabled))
1da177e4	165	return time_syscall(t);
d0aff6e6 JS	166
	167	vgettimeofday(&tv, 0);
	168	result = tv.tv_sec;
272a3713 ED	169	if (t)
	170	*t = result;
	171	return result;
1da177e4 LT	172	}
1da177e4 LT	173
c08c8205 VP	174	/* Fast way to get current CPU and node.
	175	This helps to do per node and per CPU caches in user space.
	176	The result is not guaranteed without CPU affinity, but usually
	177	works out because the scheduler tries to keep a thread on the same
	178	CPU.
	179
	180	tcache must point to a two element sized long array.
	181	All arguments can be NULL. */
	182	long __vsyscall(2)
	183	vgetcpu(unsigned cpu, unsigned node, struct getcpu_cache *tcache)
1da177e4	184	{
c08c8205 VP	185	unsigned int dummy, p;
	186	unsigned long j = 0;
	187
	188	/* Fast cache - only recompute value once per jiffies and avoid
	189	relatively costly rdtscp/cpuid otherwise.
	190	This works because the scheduler usually keeps the process
	191	on the same CPU and this syscall doesn't guarantee its
	192	results anyways.
	193	We do this here because otherwise user space would do it on
	194	its own in a likely inferior way (no access to jiffies).
	195	If you don't like it pass NULL. */
34596dc9 AK	196	if (tcache && tcache->blob[0] == (j = __jiffies)) {
34596dc9 AK	197	p = tcache->blob[1];
c08c8205 VP	198	} else if (__vgetcpu_mode == VGETCPU_RDTSCP) {
	199	/* Load per CPU data from RDTSCP */
	200	rdtscp(dummy, dummy, p);
	201	} else {
	202	/* Load per CPU data from GDT */
	203	asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
	204	}
	205	if (tcache) {
34596dc9 AK	206	tcache->blob[0] = j;
34596dc9 AK	207	tcache->blob[1] = p;
c08c8205 VP	208	}
	209	if (cpu)
	210	*cpu = p & 0xfff;
	211	if (node)
	212	*node = p >> 12;
	213	return 0;
1da177e4 LT	214	}
1da177e4 LT	215
2e8ad43e	216	long __vsyscall(3) venosys_1(void)
1da177e4 LT	217	{
	218	return -ENOSYS;
	219	}
	220
	221	#ifdef CONFIG_SYSCTL
	222
	223	#define SYSCALL 0x050f
	224	#define NOP2 0x9090
	225
	226	/*
	227	* NOP out syscall in vsyscall page when not needed.
	228	*/
	229	static int vsyscall_sysctl_change(ctl_table ctl, int write, struct file filp,
	230	void __user buffer, size_t lenp, loff_t *ppos)
	231	{
	232	extern u16 vsysc1, vsysc2;
131cfd7b AK	233	u16 __iomem *map1;
131cfd7b AK	234	u16 __iomem *map2;
1da177e4 LT	235	int ret = proc_dointvec(ctl, write, filp, buffer, lenp, ppos);
	236	if (!write)
	237	return ret;
	238	/* gcc has some trouble with __va(__pa()), so just do it this
	239	way. */
0dbf7028	240	map1 = ioremap(__pa_vsymbol(&vsysc1), 2);
1da177e4 LT	241	if (!map1)
1da177e4 LT	242	return -ENOMEM;
0dbf7028	243	map2 = ioremap(__pa_vsymbol(&vsysc2), 2);
1da177e4 LT	244	if (!map2) {
	245	ret = -ENOMEM;
	246	goto out;
	247	}
7460ed28	248	if (!vsyscall_gtod_data.sysctl_enabled) {
131cfd7b AK	249	writew(SYSCALL, map1);
131cfd7b AK	250	writew(SYSCALL, map2);
1da177e4	251	} else {
131cfd7b AK	252	writew(NOP2, map1);
131cfd7b AK	253	writew(NOP2, map2);
1da177e4 LT	254	}
	255	iounmap(map2);
	256	out:
	257	iounmap(map1);
	258	return ret;
	259	}
	260
	261	static int vsyscall_sysctl_nostrat(ctl_table t, int __user name, int nlen,
	262	void __user oldval, size_t __user oldlenp,
1f29bcd7	263	void __user *newval, size_t newlen)
1da177e4 LT	264	{
	265	return -ENOSYS;
	266	}
	267
	268	static ctl_table kernel_table2[] = {
	269	{ .ctl_name = 99, .procname = "vsyscall64",
7460ed28 JS	270	.data = &vsyscall_gtod_data.sysctl_enabled, .maxlen = sizeof(int),
7460ed28 JS	271	.mode = 0644,
1da177e4 LT	272	.strategy = vsyscall_sysctl_nostrat,
1da177e4 LT	273	.proc_handler = vsyscall_sysctl_change },
7a44d37d	274	{}
1da177e4 LT	275	};
	276
	277	static ctl_table kernel_root_table2[] = {
	278	{ .ctl_name = CTL_KERN, .procname = "kernel", .mode = 0555,
	279	.child = kernel_table2 },
7a44d37d	280	{}
1da177e4 LT	281	};
	282
	283	#endif
	284
8c131af1 AK	285	/* Assume __initcall executes before all user space. Hopefully kmod
	286	doesn't violate that. We'll find out if it does. */
	287	static void __cpuinit vsyscall_set_cpu(int cpu)
c08c8205 VP	288	{
	289	unsigned long *d;
	290	unsigned long node = 0;
	291	#ifdef CONFIG_NUMA
	292	node = cpu_to_node[cpu];
	293	#endif
8c131af1 AK	294	if (cpu_has(&cpu_data[cpu], X86_FEATURE_RDTSCP))
8c131af1 AK	295	write_rdtscp_aux((node << 12) \| cpu);
c08c8205 VP	296
	297	/* Store cpu number in limit so that it can be loaded quickly
	298	in user space in vgetcpu.
	299	12 bits for the CPU and 8 bits for the node. */
	300	d = (unsigned long *)(cpu_gdt(cpu) + GDT_ENTRY_PER_CPU);
	301	*d = 0x0f40000000000ULL;
	302	*d \|= cpu;
	303	*d \|= (node & 0xf) << 12;
	304	*d \|= (node >> 4) << 48;
	305	}
	306
8c131af1 AK	307	static void __cpuinit cpu_vsyscall_init(void *arg)
	308	{
	309	/* preemption should be already off */
	310	vsyscall_set_cpu(raw_smp_processor_id());
	311	}
	312
	313	static int __cpuinit
	314	cpu_vsyscall_notifier(struct notifier_block n, unsigned long action, void arg)
	315	{
	316	long cpu = (long)arg;
8bb78442	317	if (action == CPU_ONLINE \|\| action == CPU_ONLINE_FROZEN)
8c131af1 AK	318	smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 0, 1);
	319	return NOTIFY_DONE;
	320	}
	321
1da177e4 LT	322	static void __init map_vsyscall(void)
	323	{
	324	extern char __vsyscall_0;
	325	unsigned long physaddr_page0 = __pa_symbol(&__vsyscall_0);
	326
103efcd9	327	/* Note that VSYSCALL_MAPPED_PAGES must agree with the code below. */
1da177e4 LT	328	__set_fixmap(VSYSCALL_FIRST_PAGE, physaddr_page0, PAGE_KERNEL_VSYSCALL);
	329	}
	330
	331	static int __init vsyscall_init(void)
	332	{
	333	BUG_ON(((unsigned long) &vgettimeofday !=
	334	VSYSCALL_ADDR(__NR_vgettimeofday)));
	335	BUG_ON((unsigned long) &vtime != VSYSCALL_ADDR(__NR_vtime));
	336	BUG_ON((VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE)));
c08c8205	337	BUG_ON((unsigned long) &vgetcpu != VSYSCALL_ADDR(__NR_vgetcpu));
1da177e4	338	map_vsyscall();
f3c5f5e7	339	#ifdef CONFIG_SYSCTL
0b4d4147	340	register_sysctl_table(kernel_root_table2);
f3c5f5e7	341	#endif
8c131af1 AK	342	on_each_cpu(cpu_vsyscall_init, NULL, 0, 1);
8c131af1 AK	343	hotcpu_notifier(cpu_vsyscall_notifier, 0);
1da177e4 LT	344	return 0;
	345	}
	346
	347	__initcall(vsyscall_init);