]>
Commit | Line | Data |
---|---|---|
00b26474 VF |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Generic userspace implementations of gettimeofday() and similar. | |
4 | */ | |
00b26474 VF |
5 | #include <vdso/datapage.h> |
6 | #include <vdso/helpers.h> | |
7 | ||
9d90b93b TG |
8 | #ifndef vdso_calc_delta |
9 | /* | |
10 | * Default implementation which works for all sane clocksources. That | |
11 | * obviously excludes x86/TSC. | |
12 | */ | |
13 | static __always_inline | |
14 | u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult) | |
15 | { | |
16 | return ((cycles - last) & mask) * mult; | |
17 | } | |
18 | #endif | |
19 | ||
8345228c CL |
20 | #ifndef vdso_shift_ns |
21 | static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) | |
22 | { | |
23 | return ns >> shift; | |
24 | } | |
25 | #endif | |
26 | ||
1dff4156 TG |
27 | #ifndef __arch_vdso_hres_capable |
28 | static inline bool __arch_vdso_hres_capable(void) | |
29 | { | |
30 | return true; | |
31 | } | |
32 | #endif | |
33 | ||
ae12e085 CL |
34 | #ifndef vdso_clocksource_ok |
35 | static inline bool vdso_clocksource_ok(const struct vdso_data *vd) | |
36 | { | |
37 | return vd->clock_mode != VDSO_CLOCKMODE_NONE; | |
38 | } | |
39 | #endif | |
40 | ||
72ce7780 TG |
41 | #ifndef vdso_cycles_ok |
42 | static inline bool vdso_cycles_ok(u64 cycles) | |
43 | { | |
44 | return true; | |
45 | } | |
46 | #endif | |
47 | ||
660fd04f TG |
48 | #ifdef CONFIG_TIME_NS |
49 | static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, | |
50 | struct __kernel_timespec *ts) | |
51 | { | |
52 | const struct vdso_data *vd = __arch_get_timens_vdso_data(); | |
53 | const struct timens_offset *offs = &vdns->offset[clk]; | |
54 | const struct vdso_timestamp *vdso_ts; | |
55 | u64 cycles, last, ns; | |
56 | u32 seq; | |
57 | s64 sec; | |
58 | ||
59 | if (clk != CLOCK_MONOTONIC_RAW) | |
60 | vd = &vd[CS_HRES_COARSE]; | |
61 | else | |
62 | vd = &vd[CS_RAW]; | |
63 | vdso_ts = &vd->basetime[clk]; | |
64 | ||
65 | do { | |
66 | seq = vdso_read_begin(vd); | |
f86fd32d | 67 | |
ae12e085 | 68 | if (unlikely(!vdso_clocksource_ok(vd))) |
5d51bee7 | 69 | return -1; |
f86fd32d | 70 | |
4c5a116a | 71 | cycles = __arch_get_hw_counter(vd->clock_mode, vd); |
72ce7780 TG |
72 | if (unlikely(!vdso_cycles_ok(cycles))) |
73 | return -1; | |
660fd04f TG |
74 | ns = vdso_ts->nsec; |
75 | last = vd->cycle_last; | |
660fd04f | 76 | ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); |
8345228c | 77 | ns = vdso_shift_ns(ns, vd->shift); |
660fd04f TG |
78 | sec = vdso_ts->sec; |
79 | } while (unlikely(vdso_read_retry(vd, seq))); | |
80 | ||
81 | /* Add the namespace offset */ | |
82 | sec += offs->sec; | |
83 | ns += offs->nsec; | |
84 | ||
85 | /* | |
86 | * Do this outside the loop: a race inside the loop could result | |
87 | * in __iter_div_u64_rem() being extremely slow. | |
88 | */ | |
89 | ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); | |
90 | ts->tv_nsec = ns; | |
91 | ||
92 | return 0; | |
93 | } | |
94 | #else | |
95 | static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void) | |
96 | { | |
97 | return NULL; | |
98 | } | |
99 | ||
100 | static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, | |
101 | struct __kernel_timespec *ts) | |
102 | { | |
103 | return -EINVAL; | |
104 | } | |
105 | #endif | |
106 | ||
c966533f | 107 | static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, |
660fd04f | 108 | struct __kernel_timespec *ts) |
00b26474 VF |
109 | { |
110 | const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; | |
111 | u64 cycles, last, sec, ns; | |
112 | u32 seq; | |
113 | ||
1dff4156 TG |
114 | /* Allows to compile the high resolution parts out */ |
115 | if (!__arch_vdso_hres_capable()) | |
116 | return -1; | |
117 | ||
00b26474 | 118 | do { |
660fd04f | 119 | /* |
2d6b01bd | 120 | * Open coded to handle VDSO_CLOCKMODE_TIMENS. Time namespace |
660fd04f TG |
121 | * enabled tasks have a special VVAR page installed which |
122 | * has vd->seq set to 1 and vd->clock_mode set to | |
2d6b01bd | 123 | * VDSO_CLOCKMODE_TIMENS. For non time namespace affected tasks |
660fd04f TG |
124 | * this does not affect performance because if vd->seq is |
125 | * odd, i.e. a concurrent update is in progress the extra | |
126 | * check for vd->clock_mode is just a few extra | |
127 | * instructions while spin waiting for vd->seq to become | |
128 | * even again. | |
129 | */ | |
130 | while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) { | |
131 | if (IS_ENABLED(CONFIG_TIME_NS) && | |
2d6b01bd | 132 | vd->clock_mode == VDSO_CLOCKMODE_TIMENS) |
660fd04f TG |
133 | return do_hres_timens(vd, clk, ts); |
134 | cpu_relax(); | |
135 | } | |
136 | smp_rmb(); | |
137 | ||
ae12e085 | 138 | if (unlikely(!vdso_clocksource_ok(vd))) |
5d51bee7 | 139 | return -1; |
f86fd32d | 140 | |
4c5a116a | 141 | cycles = __arch_get_hw_counter(vd->clock_mode, vd); |
72ce7780 TG |
142 | if (unlikely(!vdso_cycles_ok(cycles))) |
143 | return -1; | |
00b26474 VF |
144 | ns = vdso_ts->nsec; |
145 | last = vd->cycle_last; | |
9d90b93b | 146 | ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); |
8345228c | 147 | ns = vdso_shift_ns(ns, vd->shift); |
00b26474 VF |
148 | sec = vdso_ts->sec; |
149 | } while (unlikely(vdso_read_retry(vd, seq))); | |
150 | ||
151 | /* | |
152 | * Do this outside the loop: a race inside the loop could result | |
153 | * in __iter_div_u64_rem() being extremely slow. | |
154 | */ | |
155 | ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); | |
156 | ts->tv_nsec = ns; | |
157 | ||
158 | return 0; | |
159 | } | |
160 | ||
660fd04f TG |
161 | #ifdef CONFIG_TIME_NS |
162 | static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, | |
163 | struct __kernel_timespec *ts) | |
164 | { | |
165 | const struct vdso_data *vd = __arch_get_timens_vdso_data(); | |
166 | const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; | |
167 | const struct timens_offset *offs = &vdns->offset[clk]; | |
168 | u64 nsec; | |
169 | s64 sec; | |
170 | s32 seq; | |
171 | ||
172 | do { | |
173 | seq = vdso_read_begin(vd); | |
174 | sec = vdso_ts->sec; | |
175 | nsec = vdso_ts->nsec; | |
176 | } while (unlikely(vdso_read_retry(vd, seq))); | |
177 | ||
178 | /* Add the namespace offset */ | |
179 | sec += offs->sec; | |
180 | nsec += offs->nsec; | |
181 | ||
182 | /* | |
183 | * Do this outside the loop: a race inside the loop could result | |
184 | * in __iter_div_u64_rem() being extremely slow. | |
185 | */ | |
186 | ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec); | |
187 | ts->tv_nsec = nsec; | |
188 | return 0; | |
189 | } | |
190 | #else | |
191 | static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, | |
192 | struct __kernel_timespec *ts) | |
193 | { | |
194 | return -1; | |
195 | } | |
196 | #endif | |
197 | ||
c966533f AV |
198 | static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk, |
199 | struct __kernel_timespec *ts) | |
00b26474 VF |
200 | { |
201 | const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; | |
202 | u32 seq; | |
203 | ||
204 | do { | |
660fd04f | 205 | /* |
2d6b01bd | 206 | * Open coded to handle VDSO_CLOCK_TIMENS. See comment in |
660fd04f TG |
207 | * do_hres(). |
208 | */ | |
209 | while ((seq = READ_ONCE(vd->seq)) & 1) { | |
210 | if (IS_ENABLED(CONFIG_TIME_NS) && | |
2d6b01bd | 211 | vd->clock_mode == VDSO_CLOCKMODE_TIMENS) |
660fd04f TG |
212 | return do_coarse_timens(vd, clk, ts); |
213 | cpu_relax(); | |
214 | } | |
215 | smp_rmb(); | |
216 | ||
00b26474 VF |
217 | ts->tv_sec = vdso_ts->sec; |
218 | ts->tv_nsec = vdso_ts->nsec; | |
219 | } while (unlikely(vdso_read_retry(vd, seq))); | |
8463cf80 CL |
220 | |
221 | return 0; | |
00b26474 VF |
222 | } |
223 | ||
b91c8c42 | 224 | static __always_inline int |
e876f0b6 CL |
225 | __cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock, |
226 | struct __kernel_timespec *ts) | |
00b26474 | 227 | { |
00b26474 VF |
228 | u32 msk; |
229 | ||
230 | /* Check for negative values or invalid clocks */ | |
231 | if (unlikely((u32) clock >= MAX_CLOCKS)) | |
502a590a | 232 | return -1; |
00b26474 VF |
233 | |
234 | /* | |
235 | * Convert the clockid to a bitmask and use it to check which | |
236 | * clocks are handled in the VDSO directly. | |
237 | */ | |
238 | msk = 1U << clock; | |
8463cf80 | 239 | if (likely(msk & VDSO_HRES)) |
c966533f | 240 | vd = &vd[CS_HRES_COARSE]; |
8463cf80 CL |
241 | else if (msk & VDSO_COARSE) |
242 | return do_coarse(&vd[CS_HRES_COARSE], clock, ts); | |
243 | else if (msk & VDSO_RAW) | |
c966533f AV |
244 | vd = &vd[CS_RAW]; |
245 | else | |
246 | return -1; | |
8463cf80 | 247 | |
c966533f | 248 | return do_hres(vd, clock, ts); |
502a590a | 249 | } |
00b26474 | 250 | |
502a590a | 251 | static __maybe_unused int |
e876f0b6 CL |
252 | __cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock, |
253 | struct __kernel_timespec *ts) | |
502a590a | 254 | { |
e876f0b6 | 255 | int ret = __cvdso_clock_gettime_common(vd, clock, ts); |
502a590a TG |
256 | |
257 | if (unlikely(ret)) | |
258 | return clock_gettime_fallback(clock, ts); | |
259 | return 0; | |
00b26474 VF |
260 | } |
261 | ||
e876f0b6 CL |
262 | static __maybe_unused int |
263 | __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) | |
264 | { | |
265 | return __cvdso_clock_gettime_data(__arch_get_vdso_data(), clock, ts); | |
266 | } | |
267 | ||
bf279849 | 268 | #ifdef BUILD_VDSO32 |
00b26474 | 269 | static __maybe_unused int |
e876f0b6 CL |
270 | __cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock, |
271 | struct old_timespec32 *res) | |
00b26474 VF |
272 | { |
273 | struct __kernel_timespec ts; | |
274 | int ret; | |
275 | ||
e876f0b6 | 276 | ret = __cvdso_clock_gettime_common(vd, clock, &ts); |
00b26474 | 277 | |
c60a32ea TG |
278 | if (unlikely(ret)) |
279 | return clock_gettime32_fallback(clock, res); | |
502a590a | 280 | |
a279235d VF |
281 | /* For ret == 0 */ |
282 | res->tv_sec = ts.tv_sec; | |
283 | res->tv_nsec = ts.tv_nsec; | |
284 | ||
00b26474 | 285 | return ret; |
00b26474 | 286 | } |
e876f0b6 CL |
287 | |
288 | static __maybe_unused int | |
289 | __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) | |
290 | { | |
291 | return __cvdso_clock_gettime32_data(__arch_get_vdso_data(), clock, res); | |
292 | } | |
bf279849 | 293 | #endif /* BUILD_VDSO32 */ |
00b26474 VF |
294 | |
295 | static __maybe_unused int | |
e876f0b6 CL |
296 | __cvdso_gettimeofday_data(const struct vdso_data *vd, |
297 | struct __kernel_old_timeval *tv, struct timezone *tz) | |
00b26474 | 298 | { |
00b26474 VF |
299 | |
300 | if (likely(tv != NULL)) { | |
301 | struct __kernel_timespec ts; | |
302 | ||
303 | if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts)) | |
304 | return gettimeofday_fallback(tv, tz); | |
305 | ||
306 | tv->tv_sec = ts.tv_sec; | |
307 | tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC; | |
308 | } | |
309 | ||
310 | if (unlikely(tz != NULL)) { | |
660fd04f | 311 | if (IS_ENABLED(CONFIG_TIME_NS) && |
2d6b01bd | 312 | vd->clock_mode == VDSO_CLOCKMODE_TIMENS) |
660fd04f TG |
313 | vd = __arch_get_timens_vdso_data(); |
314 | ||
00b26474 VF |
315 | tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; |
316 | tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; | |
317 | } | |
318 | ||
319 | return 0; | |
320 | } | |
321 | ||
e876f0b6 CL |
322 | static __maybe_unused int |
323 | __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) | |
324 | { | |
325 | return __cvdso_gettimeofday_data(__arch_get_vdso_data(), tv, tz); | |
326 | } | |
327 | ||
00b26474 | 328 | #ifdef VDSO_HAS_TIME |
e876f0b6 CL |
329 | static __maybe_unused __kernel_old_time_t |
330 | __cvdso_time_data(const struct vdso_data *vd, __kernel_old_time_t *time) | |
00b26474 | 331 | { |
660fd04f TG |
332 | __kernel_old_time_t t; |
333 | ||
2d6b01bd TG |
334 | if (IS_ENABLED(CONFIG_TIME_NS) && |
335 | vd->clock_mode == VDSO_CLOCKMODE_TIMENS) | |
660fd04f TG |
336 | vd = __arch_get_timens_vdso_data(); |
337 | ||
338 | t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); | |
00b26474 VF |
339 | |
340 | if (time) | |
341 | *time = t; | |
342 | ||
343 | return t; | |
344 | } | |
e876f0b6 CL |
345 | |
346 | static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) | |
347 | { | |
348 | return __cvdso_time_data(__arch_get_vdso_data(), time); | |
349 | } | |
00b26474 VF |
350 | #endif /* VDSO_HAS_TIME */ |
351 | ||
352 | #ifdef VDSO_HAS_CLOCK_GETRES | |
353 | static __maybe_unused | |
e876f0b6 CL |
354 | int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock, |
355 | struct __kernel_timespec *res) | |
00b26474 | 356 | { |
00b26474 | 357 | u32 msk; |
502a590a | 358 | u64 ns; |
00b26474 VF |
359 | |
360 | /* Check for negative values or invalid clocks */ | |
361 | if (unlikely((u32) clock >= MAX_CLOCKS)) | |
502a590a | 362 | return -1; |
00b26474 | 363 | |
2d6b01bd TG |
364 | if (IS_ENABLED(CONFIG_TIME_NS) && |
365 | vd->clock_mode == VDSO_CLOCKMODE_TIMENS) | |
660fd04f TG |
366 | vd = __arch_get_timens_vdso_data(); |
367 | ||
00b26474 VF |
368 | /* |
369 | * Convert the clockid to a bitmask and use it to check which | |
370 | * clocks are handled in the VDSO directly. | |
371 | */ | |
372 | msk = 1U << clock; | |
cdb7c5a9 | 373 | if (msk & (VDSO_HRES | VDSO_RAW)) { |
00b26474 VF |
374 | /* |
375 | * Preserves the behaviour of posix_get_hrtimer_res(). | |
376 | */ | |
49a101d7 | 377 | ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res); |
00b26474 VF |
378 | } else if (msk & VDSO_COARSE) { |
379 | /* | |
380 | * Preserves the behaviour of posix_get_coarse_res(). | |
381 | */ | |
382 | ns = LOW_RES_NSEC; | |
00b26474 | 383 | } else { |
502a590a | 384 | return -1; |
00b26474 VF |
385 | } |
386 | ||
1638b8f0 TG |
387 | if (likely(res)) { |
388 | res->tv_sec = 0; | |
389 | res->tv_nsec = ns; | |
390 | } | |
00b26474 | 391 | return 0; |
502a590a TG |
392 | } |
393 | ||
ffd08731 | 394 | static __maybe_unused |
e876f0b6 CL |
395 | int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock, |
396 | struct __kernel_timespec *res) | |
502a590a | 397 | { |
e876f0b6 | 398 | int ret = __cvdso_clock_getres_common(vd, clock, res); |
00b26474 | 399 | |
502a590a TG |
400 | if (unlikely(ret)) |
401 | return clock_getres_fallback(clock, res); | |
402 | return 0; | |
00b26474 VF |
403 | } |
404 | ||
e876f0b6 CL |
405 | static __maybe_unused |
406 | int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) | |
407 | { | |
408 | return __cvdso_clock_getres_data(__arch_get_vdso_data(), clock, res); | |
409 | } | |
410 | ||
bf279849 | 411 | #ifdef BUILD_VDSO32 |
00b26474 | 412 | static __maybe_unused int |
e876f0b6 CL |
413 | __cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock, |
414 | struct old_timespec32 *res) | |
00b26474 VF |
415 | { |
416 | struct __kernel_timespec ts; | |
417 | int ret; | |
418 | ||
e876f0b6 | 419 | ret = __cvdso_clock_getres_common(vd, clock, &ts); |
c60a32ea | 420 | |
c60a32ea TG |
421 | if (unlikely(ret)) |
422 | return clock_getres32_fallback(clock, res); | |
00b26474 | 423 | |
a279235d | 424 | if (likely(res)) { |
00b26474 VF |
425 | res->tv_sec = ts.tv_sec; |
426 | res->tv_nsec = ts.tv_nsec; | |
427 | } | |
00b26474 | 428 | return ret; |
00b26474 | 429 | } |
e876f0b6 CL |
430 | |
431 | static __maybe_unused int | |
432 | __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) | |
433 | { | |
434 | return __cvdso_clock_getres_time32_data(__arch_get_vdso_data(), | |
435 | clock, res); | |
436 | } | |
bf279849 | 437 | #endif /* BUILD_VDSO32 */ |
00b26474 | 438 | #endif /* VDSO_HAS_CLOCK_GETRES */ |