]>
Commit | Line | Data |
---|---|---|
2aae950b AK |
1 | /* |
2 | * Copyright 2006 Andi Kleen, SUSE Labs. | |
3 | * Subject to the GNU Public License, v.2 | |
4 | * | |
f144a6b4 | 5 | * Fast user context implementation of clock_gettime, gettimeofday, and time. |
2aae950b AK |
6 | * |
7 | * The code should have no internal unresolved relocations. | |
8 | * Check with readelf after changing. | |
2aae950b AK |
9 | */ |
10 | ||
2b7d0390 | 11 | /* Disable profiling for userspace code: */ |
2ed84eeb | 12 | #define DISABLE_BRANCH_PROFILING |
2b7d0390 | 13 | |
2aae950b AK |
14 | #include <linux/kernel.h> |
15 | #include <linux/posix-timers.h> | |
16 | #include <linux/time.h> | |
17 | #include <linux/string.h> | |
18 | #include <asm/vsyscall.h> | |
98d0ac38 | 19 | #include <asm/fixmap.h> |
2aae950b AK |
20 | #include <asm/vgtod.h> |
21 | #include <asm/timex.h> | |
22 | #include <asm/hpet.h> | |
23 | #include <asm/unistd.h> | |
24 | #include <asm/io.h> | |
2aae950b | 25 | |
8c49d9a7 | 26 | #define gtod (&VVAR(vsyscall_gtod_data)) |
2aae950b | 27 | |
98d0ac38 AL |
28 | notrace static cycle_t vread_tsc(void) |
29 | { | |
30 | cycle_t ret; | |
31 | u64 last; | |
32 | ||
33 | /* | |
34 | * Empirically, a fence (of type that depends on the CPU) | |
35 | * before rdtsc is enough to ensure that rdtsc is ordered | |
36 | * with respect to loads. The various CPU manuals are unclear | |
37 | * as to whether rdtsc can be reordered with later loads, | |
38 | * but no one has ever seen it happen. | |
39 | */ | |
40 | rdtsc_barrier(); | |
41 | ret = (cycle_t)vget_cycles(); | |
42 | ||
43 | last = VVAR(vsyscall_gtod_data).clock.cycle_last; | |
44 | ||
45 | if (likely(ret >= last)) | |
46 | return ret; | |
47 | ||
48 | /* | |
49 | * GCC likes to generate cmov here, but this branch is extremely | |
50 | * predictable (it's just a funciton of time and the likely is | |
51 | * very likely) and there's a data dependence, so force GCC | |
52 | * to generate a branch instead. I don't barrier() because | |
53 | * we don't actually need a barrier, and if this function | |
54 | * ever gets inlined it will generate worse code. | |
55 | */ | |
56 | asm volatile (""); | |
57 | return last; | |
58 | } | |
59 | ||
60 | static notrace cycle_t vread_hpet(void) | |
61 | { | |
62 | return readl((const void __iomem *)fix_to_virt(VSYSCALL_HPET) + 0xf0); | |
63 | } | |
64 | ||
23adec55 | 65 | notrace static long vdso_fallback_gettime(long clock, struct timespec *ts) |
2aae950b AK |
66 | { |
67 | long ret; | |
68 | asm("syscall" : "=a" (ret) : | |
69 | "0" (__NR_clock_gettime),"D" (clock), "S" (ts) : "memory"); | |
70 | return ret; | |
71 | } | |
72 | ||
23adec55 | 73 | notrace static inline long vgetns(void) |
2aae950b | 74 | { |
95b08679 | 75 | long v; |
98d0ac38 AL |
76 | cycles_t cycles; |
77 | if (gtod->clock.vclock_mode == VCLOCK_TSC) | |
78 | cycles = vread_tsc(); | |
79 | else | |
80 | cycles = vread_hpet(); | |
81 | v = (cycles - gtod->clock.cycle_last) & gtod->clock.mask; | |
95b08679 | 82 | return (v * gtod->clock.mult) >> gtod->clock.shift; |
2aae950b AK |
83 | } |
84 | ||
23adec55 | 85 | notrace static noinline int do_realtime(struct timespec *ts) |
2aae950b AK |
86 | { |
87 | unsigned long seq, ns; | |
88 | do { | |
89 | seq = read_seqbegin(>od->lock); | |
90 | ts->tv_sec = gtod->wall_time_sec; | |
91 | ts->tv_nsec = gtod->wall_time_nsec; | |
92 | ns = vgetns(); | |
93 | } while (unlikely(read_seqretry(>od->lock, seq))); | |
94 | timespec_add_ns(ts, ns); | |
95 | return 0; | |
96 | } | |
97 | ||
23adec55 | 98 | notrace static noinline int do_monotonic(struct timespec *ts) |
2aae950b AK |
99 | { |
100 | unsigned long seq, ns, secs; | |
101 | do { | |
102 | seq = read_seqbegin(>od->lock); | |
103 | secs = gtod->wall_time_sec; | |
104 | ns = gtod->wall_time_nsec + vgetns(); | |
105 | secs += gtod->wall_to_monotonic.tv_sec; | |
106 | ns += gtod->wall_to_monotonic.tv_nsec; | |
107 | } while (unlikely(read_seqretry(>od->lock, seq))); | |
0f51f285 AL |
108 | |
109 | /* wall_time_nsec, vgetns(), and wall_to_monotonic.tv_nsec | |
110 | * are all guaranteed to be nonnegative. | |
111 | */ | |
112 | while (ns >= NSEC_PER_SEC) { | |
113 | ns -= NSEC_PER_SEC; | |
114 | ++secs; | |
115 | } | |
116 | ts->tv_sec = secs; | |
117 | ts->tv_nsec = ns; | |
118 | ||
2aae950b AK |
119 | return 0; |
120 | } | |
121 | ||
da15cfda JS |
122 | notrace static noinline int do_realtime_coarse(struct timespec *ts) |
123 | { | |
124 | unsigned long seq; | |
125 | do { | |
126 | seq = read_seqbegin(>od->lock); | |
127 | ts->tv_sec = gtod->wall_time_coarse.tv_sec; | |
128 | ts->tv_nsec = gtod->wall_time_coarse.tv_nsec; | |
129 | } while (unlikely(read_seqretry(>od->lock, seq))); | |
130 | return 0; | |
131 | } | |
132 | ||
133 | notrace static noinline int do_monotonic_coarse(struct timespec *ts) | |
134 | { | |
135 | unsigned long seq, ns, secs; | |
136 | do { | |
137 | seq = read_seqbegin(>od->lock); | |
138 | secs = gtod->wall_time_coarse.tv_sec; | |
139 | ns = gtod->wall_time_coarse.tv_nsec; | |
140 | secs += gtod->wall_to_monotonic.tv_sec; | |
141 | ns += gtod->wall_to_monotonic.tv_nsec; | |
142 | } while (unlikely(read_seqretry(>od->lock, seq))); | |
0f51f285 AL |
143 | |
144 | /* wall_time_nsec and wall_to_monotonic.tv_nsec are | |
145 | * guaranteed to be between 0 and NSEC_PER_SEC. | |
146 | */ | |
147 | if (ns >= NSEC_PER_SEC) { | |
148 | ns -= NSEC_PER_SEC; | |
149 | ++secs; | |
150 | } | |
151 | ts->tv_sec = secs; | |
152 | ts->tv_nsec = ns; | |
153 | ||
da15cfda JS |
154 | return 0; |
155 | } | |
156 | ||
23adec55 | 157 | notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts) |
2aae950b | 158 | { |
0d7b8547 AL |
159 | switch (clock) { |
160 | case CLOCK_REALTIME: | |
98d0ac38 | 161 | if (likely(gtod->clock.vclock_mode != VCLOCK_NONE)) |
0d7b8547 AL |
162 | return do_realtime(ts); |
163 | break; | |
164 | case CLOCK_MONOTONIC: | |
98d0ac38 | 165 | if (likely(gtod->clock.vclock_mode != VCLOCK_NONE)) |
0d7b8547 AL |
166 | return do_monotonic(ts); |
167 | break; | |
168 | case CLOCK_REALTIME_COARSE: | |
169 | return do_realtime_coarse(ts); | |
170 | case CLOCK_MONOTONIC_COARSE: | |
171 | return do_monotonic_coarse(ts); | |
172 | } | |
173 | ||
2aae950b AK |
174 | return vdso_fallback_gettime(clock, ts); |
175 | } | |
176 | int clock_gettime(clockid_t, struct timespec *) | |
177 | __attribute__((weak, alias("__vdso_clock_gettime"))); | |
178 | ||
23adec55 | 179 | notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz) |
2aae950b AK |
180 | { |
181 | long ret; | |
98d0ac38 | 182 | if (likely(gtod->clock.vclock_mode != VCLOCK_NONE)) { |
2f65dd47 JW |
183 | if (likely(tv != NULL)) { |
184 | BUILD_BUG_ON(offsetof(struct timeval, tv_usec) != | |
185 | offsetof(struct timespec, tv_nsec) || | |
186 | sizeof(*tv) != sizeof(struct timespec)); | |
187 | do_realtime((struct timespec *)tv); | |
188 | tv->tv_usec /= 1000; | |
189 | } | |
2aae950b | 190 | if (unlikely(tz != NULL)) { |
a1289643 AK |
191 | /* Avoid memcpy. Some old compilers fail to inline it */ |
192 | tz->tz_minuteswest = gtod->sys_tz.tz_minuteswest; | |
193 | tz->tz_dsttime = gtod->sys_tz.tz_dsttime; | |
2aae950b AK |
194 | } |
195 | return 0; | |
196 | } | |
197 | asm("syscall" : "=a" (ret) : | |
198 | "0" (__NR_gettimeofday), "D" (tv), "S" (tz) : "memory"); | |
199 | return ret; | |
200 | } | |
201 | int gettimeofday(struct timeval *, struct timezone *) | |
202 | __attribute__((weak, alias("__vdso_gettimeofday"))); | |
f144a6b4 | 203 | |
0d7b8547 AL |
204 | /* |
205 | * This will break when the xtime seconds get inaccurate, but that is | |
206 | * unlikely | |
207 | */ | |
f144a6b4 AL |
208 | notrace time_t __vdso_time(time_t *t) |
209 | { | |
973aa818 | 210 | /* This is atomic on x86_64 so we don't need any locks. */ |
0d7b8547 | 211 | time_t result = ACCESS_ONCE(VVAR(vsyscall_gtod_data).wall_time_sec); |
f144a6b4 AL |
212 | |
213 | if (t) | |
214 | *t = result; | |
215 | return result; | |
216 | } | |
217 | int time(time_t *t) | |
218 | __attribute__((weak, alias("__vdso_time"))); |