2 * Copyright 2006 Andi Kleen, SUSE Labs.
3 * Subject to the GNU Public License, v.2
5 * Fast user context implementation of clock_gettime, gettimeofday, and time.
7 * 32 Bit compat layer by Stefani Seibold <stefani@seibold.net>
8 * sponsored by Rohde & Schwarz GmbH & Co. KG Munich/Germany
10 * The code should have no internal unresolved relocations.
11 * Check with readelf after changing.
14 #include <uapi/linux/time.h>
15 #include <asm/vgtod.h>
18 #include <asm/unistd.h>
20 #include <linux/math64.h>
21 #include <linux/time.h>
23 #define gtod (&VVAR(vsyscall_gtod_data))
25 extern int __vdso_clock_gettime(clockid_t clock
, struct timespec
*ts
);
26 extern int __vdso_gettimeofday(struct timeval
*tv
, struct timezone
*tz
);
27 extern time_t __vdso_time(time_t *t
);
29 #ifdef CONFIG_HPET_TIMER
31 __attribute__((visibility("hidden")));
33 static notrace cycle_t
vread_hpet(void)
35 return *(const volatile u32
*)(&hpet_page
+ HPET_COUNTER
);
41 #include <linux/kernel.h>
42 #include <asm/vsyscall.h>
43 #include <asm/fixmap.h>
44 #include <asm/pvclock.h>
46 notrace
static long vdso_fallback_gettime(long clock
, struct timespec
*ts
)
49 asm("syscall" : "=a" (ret
) :
50 "0" (__NR_clock_gettime
), "D" (clock
), "S" (ts
) : "memory");
54 notrace
static long vdso_fallback_gtod(struct timeval
*tv
, struct timezone
*tz
)
58 asm("syscall" : "=a" (ret
) :
59 "0" (__NR_gettimeofday
), "D" (tv
), "S" (tz
) : "memory");
63 #ifdef CONFIG_PARAVIRT_CLOCK
65 static notrace
const struct pvclock_vsyscall_time_info
*get_pvti(int cpu
)
67 const struct pvclock_vsyscall_time_info
*pvti_base
;
68 int idx
= cpu
/ (PAGE_SIZE
/PVTI_SIZE
);
69 int offset
= cpu
% (PAGE_SIZE
/PVTI_SIZE
);
71 BUG_ON(PVCLOCK_FIXMAP_BEGIN
+ idx
> PVCLOCK_FIXMAP_END
);
73 pvti_base
= (struct pvclock_vsyscall_time_info
*)
74 __fix_to_virt(PVCLOCK_FIXMAP_BEGIN
+idx
);
76 return &pvti_base
[offset
];
79 static notrace cycle_t
vread_pvclock(int *mode
)
81 const struct pvclock_vsyscall_time_info
*pvti
;
91 * When looping to get a consistent (time-info, tsc) pair, we
92 * also need to deal with the possibility we can switch vcpus,
93 * so make sure we always re-fetch time-info for the current vcpu.
96 cpu
= __getcpu() & VGETCPU_CPU_MASK
;
97 /* TODO: We can put vcpu id into higher bits of pvti.version.
98 * This will save a couple of cycles by getting rid of
99 * __getcpu() calls (Gleb).
102 /* Make sure migrate_count will change if we leave the VCPU. */
104 pvti
= get_pvti(cpu
);
105 migrate_count
= pvti
->migrate_count
;
108 cpu
= __getcpu() & VGETCPU_CPU_MASK
;
109 } while (unlikely(cpu
!= cpu1
));
111 version
= __pvclock_read_cycles(&pvti
->pvti
, &ret
, &flags
);
114 * Test we're still on the cpu as well as the version.
115 * - We must read TSC of pvti's VCPU.
116 * - KVM doesn't follow the versioning protocol, so data could
117 * change before version if we left the VCPU.
120 } while (unlikely((pvti
->pvti
.version
& 1) ||
121 pvti
->pvti
.version
!= version
||
122 pvti
->migrate_count
!= migrate_count
));
124 if (unlikely(!(flags
& PVCLOCK_TSC_STABLE_BIT
)))
127 /* refer to tsc.c read_tsc() comment for rationale */
128 last
= gtod
->cycle_last
;
130 if (likely(ret
>= last
))
139 notrace
static long vdso_fallback_gettime(long clock
, struct timespec
*ts
)
144 "mov %%ebx, %%edx \n"
146 "call __kernel_vsyscall \n"
147 "mov %%edx, %%ebx \n"
149 : "0" (__NR_clock_gettime
), "g" (clock
), "c" (ts
)
154 notrace
static long vdso_fallback_gtod(struct timeval
*tv
, struct timezone
*tz
)
159 "mov %%ebx, %%edx \n"
161 "call __kernel_vsyscall \n"
162 "mov %%edx, %%ebx \n"
164 : "0" (__NR_gettimeofday
), "g" (tv
), "c" (tz
)
169 #ifdef CONFIG_PARAVIRT_CLOCK
171 static notrace cycle_t
vread_pvclock(int *mode
)
180 notrace
static cycle_t
vread_tsc(void)
186 * Empirically, a fence (of type that depends on the CPU)
187 * before rdtsc is enough to ensure that rdtsc is ordered
188 * with respect to loads. The various CPU manuals are unclear
189 * as to whether rdtsc can be reordered with later loads,
190 * but no one has ever seen it happen.
193 ret
= (cycle_t
)__native_read_tsc();
195 last
= gtod
->cycle_last
;
197 if (likely(ret
>= last
))
201 * GCC likes to generate cmov here, but this branch is extremely
202 * predictable (it's just a funciton of time and the likely is
203 * very likely) and there's a data dependence, so force GCC
204 * to generate a branch instead. I don't barrier() because
205 * we don't actually need a barrier, and if this function
206 * ever gets inlined it will generate worse code.
212 notrace
static inline u64
vgetsns(int *mode
)
217 if (gtod
->vclock_mode
== VCLOCK_TSC
)
218 cycles
= vread_tsc();
219 #ifdef CONFIG_HPET_TIMER
220 else if (gtod
->vclock_mode
== VCLOCK_HPET
)
221 cycles
= vread_hpet();
223 #ifdef CONFIG_PARAVIRT_CLOCK
224 else if (gtod
->vclock_mode
== VCLOCK_PVCLOCK
)
225 cycles
= vread_pvclock(mode
);
229 v
= (cycles
- gtod
->cycle_last
) & gtod
->mask
;
230 return v
* gtod
->mult
;
233 /* Code size doesn't matter (vdso is 4k anyway) and this is faster. */
234 notrace
static int __always_inline
do_realtime(struct timespec
*ts
)
241 seq
= gtod_read_begin(gtod
);
242 mode
= gtod
->vclock_mode
;
243 ts
->tv_sec
= gtod
->wall_time_sec
;
244 ns
= gtod
->wall_time_snsec
;
245 ns
+= vgetsns(&mode
);
247 } while (unlikely(gtod_read_retry(gtod
, seq
)));
249 ts
->tv_sec
+= __iter_div_u64_rem(ns
, NSEC_PER_SEC
, &ns
);
255 notrace
static int __always_inline
do_monotonic(struct timespec
*ts
)
262 seq
= gtod_read_begin(gtod
);
263 mode
= gtod
->vclock_mode
;
264 ts
->tv_sec
= gtod
->monotonic_time_sec
;
265 ns
= gtod
->monotonic_time_snsec
;
266 ns
+= vgetsns(&mode
);
268 } while (unlikely(gtod_read_retry(gtod
, seq
)));
270 ts
->tv_sec
+= __iter_div_u64_rem(ns
, NSEC_PER_SEC
, &ns
);
276 notrace
static void do_realtime_coarse(struct timespec
*ts
)
280 seq
= gtod_read_begin(gtod
);
281 ts
->tv_sec
= gtod
->wall_time_coarse_sec
;
282 ts
->tv_nsec
= gtod
->wall_time_coarse_nsec
;
283 } while (unlikely(gtod_read_retry(gtod
, seq
)));
286 notrace
static void do_monotonic_coarse(struct timespec
*ts
)
290 seq
= gtod_read_begin(gtod
);
291 ts
->tv_sec
= gtod
->monotonic_time_coarse_sec
;
292 ts
->tv_nsec
= gtod
->monotonic_time_coarse_nsec
;
293 } while (unlikely(gtod_read_retry(gtod
, seq
)));
296 notrace
int __vdso_clock_gettime(clockid_t clock
, struct timespec
*ts
)
300 if (do_realtime(ts
) == VCLOCK_NONE
)
303 case CLOCK_MONOTONIC
:
304 if (do_monotonic(ts
) == VCLOCK_NONE
)
307 case CLOCK_REALTIME_COARSE
:
308 do_realtime_coarse(ts
);
310 case CLOCK_MONOTONIC_COARSE
:
311 do_monotonic_coarse(ts
);
319 return vdso_fallback_gettime(clock
, ts
);
321 int clock_gettime(clockid_t
, struct timespec
*)
322 __attribute__((weak
, alias("__vdso_clock_gettime")));
324 notrace
int __vdso_gettimeofday(struct timeval
*tv
, struct timezone
*tz
)
326 if (likely(tv
!= NULL
)) {
327 if (unlikely(do_realtime((struct timespec
*)tv
) == VCLOCK_NONE
))
328 return vdso_fallback_gtod(tv
, tz
);
331 if (unlikely(tz
!= NULL
)) {
332 tz
->tz_minuteswest
= gtod
->tz_minuteswest
;
333 tz
->tz_dsttime
= gtod
->tz_dsttime
;
338 int gettimeofday(struct timeval
*, struct timezone
*)
339 __attribute__((weak
, alias("__vdso_gettimeofday")));
342 * This will break when the xtime seconds get inaccurate, but that is
345 notrace
time_t __vdso_time(time_t *t
)
347 /* This is atomic on x86 so we don't need any locks. */
348 time_t result
= ACCESS_ONCE(gtod
->wall_time_sec
);
355 __attribute__((weak
, alias("__vdso_time")));