]>
Commit | Line | Data |
---|---|---|
2aae950b AK |
1 | /* |
2 | * Copyright 2006 Andi Kleen, SUSE Labs. | |
3 | * Subject to the GNU Public License, v.2 | |
4 | * | |
5 | * Fast user context implementation of clock_gettime and gettimeofday. | |
6 | * | |
7 | * The code should have no internal unresolved relocations. | |
8 | * Check with readelf after changing. | |
9 | * Also alternative() doesn't work. | |
10 | */ | |
11 | ||
2b7d0390 | 12 | /* Disable profiling for userspace code: */ |
2ed84eeb | 13 | #define DISABLE_BRANCH_PROFILING |
2b7d0390 | 14 | |
2aae950b AK |
15 | #include <linux/kernel.h> |
16 | #include <linux/posix-timers.h> | |
17 | #include <linux/time.h> | |
18 | #include <linux/string.h> | |
19 | #include <asm/vsyscall.h> | |
20 | #include <asm/vgtod.h> | |
21 | #include <asm/timex.h> | |
22 | #include <asm/hpet.h> | |
23 | #include <asm/unistd.h> | |
24 | #include <asm/io.h> | |
2aae950b AK |
25 | #include "vextern.h" |
26 | ||
27 | #define gtod vdso_vsyscall_gtod_data | |
28 | ||
23adec55 | 29 | notrace static long vdso_fallback_gettime(long clock, struct timespec *ts) |
2aae950b AK |
30 | { |
31 | long ret; | |
32 | asm("syscall" : "=a" (ret) : | |
33 | "0" (__NR_clock_gettime),"D" (clock), "S" (ts) : "memory"); | |
34 | return ret; | |
35 | } | |
36 | ||
23adec55 | 37 | notrace static inline long vgetns(void) |
2aae950b | 38 | { |
95b08679 | 39 | long v; |
2aae950b AK |
40 | cycles_t (*vread)(void); |
41 | vread = gtod->clock.vread; | |
95b08679 AK |
42 | v = (vread() - gtod->clock.cycle_last) & gtod->clock.mask; |
43 | return (v * gtod->clock.mult) >> gtod->clock.shift; | |
2aae950b AK |
44 | } |
45 | ||
23adec55 | 46 | notrace static noinline int do_realtime(struct timespec *ts) |
2aae950b AK |
47 | { |
48 | unsigned long seq, ns; | |
49 | do { | |
50 | seq = read_seqbegin(>od->lock); | |
51 | ts->tv_sec = gtod->wall_time_sec; | |
52 | ts->tv_nsec = gtod->wall_time_nsec; | |
53 | ns = vgetns(); | |
54 | } while (unlikely(read_seqretry(>od->lock, seq))); | |
55 | timespec_add_ns(ts, ns); | |
56 | return 0; | |
57 | } | |
58 | ||
59 | /* Copy of the version in kernel/time.c which we cannot directly access */ | |
23adec55 SR |
60 | notrace static void |
61 | vset_normalized_timespec(struct timespec *ts, long sec, long nsec) | |
2aae950b AK |
62 | { |
63 | while (nsec >= NSEC_PER_SEC) { | |
64 | nsec -= NSEC_PER_SEC; | |
65 | ++sec; | |
66 | } | |
67 | while (nsec < 0) { | |
68 | nsec += NSEC_PER_SEC; | |
69 | --sec; | |
70 | } | |
71 | ts->tv_sec = sec; | |
72 | ts->tv_nsec = nsec; | |
73 | } | |
74 | ||
23adec55 | 75 | notrace static noinline int do_monotonic(struct timespec *ts) |
2aae950b AK |
76 | { |
77 | unsigned long seq, ns, secs; | |
78 | do { | |
79 | seq = read_seqbegin(>od->lock); | |
80 | secs = gtod->wall_time_sec; | |
81 | ns = gtod->wall_time_nsec + vgetns(); | |
82 | secs += gtod->wall_to_monotonic.tv_sec; | |
83 | ns += gtod->wall_to_monotonic.tv_nsec; | |
84 | } while (unlikely(read_seqretry(>od->lock, seq))); | |
85 | vset_normalized_timespec(ts, secs, ns); | |
86 | return 0; | |
87 | } | |
88 | ||
23adec55 | 89 | notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts) |
2aae950b AK |
90 | { |
91 | if (likely(gtod->sysctl_enabled && gtod->clock.vread)) | |
92 | switch (clock) { | |
93 | case CLOCK_REALTIME: | |
94 | return do_realtime(ts); | |
95 | case CLOCK_MONOTONIC: | |
96 | return do_monotonic(ts); | |
97 | } | |
98 | return vdso_fallback_gettime(clock, ts); | |
99 | } | |
100 | int clock_gettime(clockid_t, struct timespec *) | |
101 | __attribute__((weak, alias("__vdso_clock_gettime"))); | |
102 | ||
23adec55 | 103 | notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz) |
2aae950b AK |
104 | { |
105 | long ret; | |
106 | if (likely(gtod->sysctl_enabled && gtod->clock.vread)) { | |
2f65dd47 JW |
107 | if (likely(tv != NULL)) { |
108 | BUILD_BUG_ON(offsetof(struct timeval, tv_usec) != | |
109 | offsetof(struct timespec, tv_nsec) || | |
110 | sizeof(*tv) != sizeof(struct timespec)); | |
111 | do_realtime((struct timespec *)tv); | |
112 | tv->tv_usec /= 1000; | |
113 | } | |
2aae950b | 114 | if (unlikely(tz != NULL)) { |
a1289643 AK |
115 | /* Avoid memcpy. Some old compilers fail to inline it */ |
116 | tz->tz_minuteswest = gtod->sys_tz.tz_minuteswest; | |
117 | tz->tz_dsttime = gtod->sys_tz.tz_dsttime; | |
2aae950b AK |
118 | } |
119 | return 0; | |
120 | } | |
121 | asm("syscall" : "=a" (ret) : | |
122 | "0" (__NR_gettimeofday), "D" (tv), "S" (tz) : "memory"); | |
123 | return ret; | |
124 | } | |
125 | int gettimeofday(struct timeval *, struct timezone *) | |
126 | __attribute__((weak, alias("__vdso_gettimeofday"))); |