[mirror_ubuntu-jammy-kernel.git] / lib / vdso / gettimeofday.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Generic userspace implementations of gettimeofday() and similar.
 */
#include <vdso/datapage.h>
#include <vdso/helpers.h>

#ifndef vdso_calc_delta
/*
 * Default implementation which works for all sane clocksources. That
 * obviously excludes x86/TSC.
 */
static __always_inline
u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
{
	return ((cycles - last) & mask) * mult;
}
#endif

#ifndef vdso_shift_ns
static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift)
{
	return ns >> shift;
}
#endif

#ifndef __arch_vdso_hres_capable
static inline bool __arch_vdso_hres_capable(void)
{
	return true;
}
#endif

#ifndef vdso_clocksource_ok
static inline bool vdso_clocksource_ok(const struct vdso_data *vd)
{
	return vd->clock_mode != VDSO_CLOCKMODE_NONE;
}
#endif

#ifndef vdso_cycles_ok
static inline bool vdso_cycles_ok(u64 cycles)
{
	return true;
}
#endif

#ifdef CONFIG_TIME_NS
static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
			  struct __kernel_timespec *ts)
{
	const struct vdso_data *vd = __arch_get_timens_vdso_data();
	const struct timens_offset *offs = &vdns->offset[clk];
	const struct vdso_timestamp *vdso_ts;
	u64 cycles, last, ns;
	u32 seq;
	s64 sec;

	if (clk != CLOCK_MONOTONIC_RAW)
		vd = &vd[CS_HRES_COARSE];
	else
		vd = &vd[CS_RAW];
	vdso_ts = &vd->basetime[clk];

	do {
		seq = vdso_read_begin(vd);

		if (unlikely(!vdso_clocksource_ok(vd)))
			return -1;

		cycles = __arch_get_hw_counter(vd->clock_mode, vd);
		if (unlikely(!vdso_cycles_ok(cycles)))
			return -1;
		ns = vdso_ts->nsec;
		last = vd->cycle_last;
		ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
		ns = vdso_shift_ns(ns, vd->shift);
		sec = vdso_ts->sec;
	} while (unlikely(vdso_read_retry(vd, seq)));

	/* Add the namespace offset */
	sec += offs->sec;
	ns += offs->nsec;

	/*
	 * Do this outside the loop: a race inside the loop could result
	 * in __iter_div_u64_rem() being extremely slow.
	 */
	ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
	ts->tv_nsec = ns;

	return 0;
}
#else
static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void)
{
	return NULL;
}

static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
			  struct __kernel_timespec *ts)
{
	return -EINVAL;
}
#endif

static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk,
				   struct __kernel_timespec *ts)
{
	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
	u64 cycles, last, sec, ns;
	u32 seq;

	/* Allows to compile the high resolution parts out */
	if (!__arch_vdso_hres_capable())
		return -1;

	do {
		/*
		 * Open coded to handle VDSO_CLOCKMODE_TIMENS. Time namespace
		 * enabled tasks have a special VVAR page installed which
		 * has vd->seq set to 1 and vd->clock_mode set to
		 * VDSO_CLOCKMODE_TIMENS. For non time namespace affected tasks
		 * this does not affect performance because if vd->seq is
		 * odd, i.e. a concurrent update is in progress the extra
		 * check for vd->clock_mode is just a few extra
		 * instructions while spin waiting for vd->seq to become
		 * even again.
		 */
		while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) {
			if (IS_ENABLED(CONFIG_TIME_NS) &&
			    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
				return do_hres_timens(vd, clk, ts);
			cpu_relax();
		}
		smp_rmb();

		if (unlikely(!vdso_clocksource_ok(vd)))
			return -1;

		cycles = __arch_get_hw_counter(vd->clock_mode, vd);
		if (unlikely(!vdso_cycles_ok(cycles)))
			return -1;
		ns = vdso_ts->nsec;
		last = vd->cycle_last;
		ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
		ns = vdso_shift_ns(ns, vd->shift);
		sec = vdso_ts->sec;
	} while (unlikely(vdso_read_retry(vd, seq)));

	/*
	 * Do this outside the loop: a race inside the loop could result
	 * in __iter_div_u64_rem() being extremely slow.
	 */
	ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
	ts->tv_nsec = ns;

	return 0;
}

#ifdef CONFIG_TIME_NS
static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
			    struct __kernel_timespec *ts)
{
	const struct vdso_data *vd = __arch_get_timens_vdso_data();
	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
	const struct timens_offset *offs = &vdns->offset[clk];
	u64 nsec;
	s64 sec;
	s32 seq;

	do {
		seq = vdso_read_begin(vd);
		sec = vdso_ts->sec;
		nsec = vdso_ts->nsec;
	} while (unlikely(vdso_read_retry(vd, seq)));

	/* Add the namespace offset */
	sec += offs->sec;
	nsec += offs->nsec;

	/*
	 * Do this outside the loop: a race inside the loop could result
	 * in __iter_div_u64_rem() being extremely slow.
	 */
	ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
	ts->tv_nsec = nsec;
	return 0;
}
#else
static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
			    struct __kernel_timespec *ts)
{
	return -1;
}
#endif

static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk,
				     struct __kernel_timespec *ts)
{
	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
	u32 seq;

	do {
		/*
		 * Open coded to handle VDSO_CLOCK_TIMENS. See comment in
		 * do_hres().
		 */
		while ((seq = READ_ONCE(vd->seq)) & 1) {
			if (IS_ENABLED(CONFIG_TIME_NS) &&
			    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
				return do_coarse_timens(vd, clk, ts);
			cpu_relax();
		}
		smp_rmb();

		ts->tv_sec = vdso_ts->sec;
		ts->tv_nsec = vdso_ts->nsec;
	} while (unlikely(vdso_read_retry(vd, seq)));

	return 0;
}

static __always_inline int
__cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock,
			     struct __kernel_timespec *ts)
{
	u32 msk;

	/* Check for negative values or invalid clocks */
	if (unlikely((u32) clock >= MAX_CLOCKS))
		return -1;

	/*
	 * Convert the clockid to a bitmask and use it to check which
	 * clocks are handled in the VDSO directly.
	 */
	msk = 1U << clock;
	if (likely(msk & VDSO_HRES))
		vd = &vd[CS_HRES_COARSE];
	else if (msk & VDSO_COARSE)
		return do_coarse(&vd[CS_HRES_COARSE], clock, ts);
	else if (msk & VDSO_RAW)
		vd = &vd[CS_RAW];
	else
		return -1;

	return do_hres(vd, clock, ts);
}

static __maybe_unused int
__cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock,
			   struct __kernel_timespec *ts)
{
	int ret = __cvdso_clock_gettime_common(vd, clock, ts);

	if (unlikely(ret))
		return clock_gettime_fallback(clock, ts);
	return 0;
}

static __maybe_unused int
__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
{
	return __cvdso_clock_gettime_data(__arch_get_vdso_data(), clock, ts);
}

#ifdef BUILD_VDSO32
static __maybe_unused int
__cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock,
			     struct old_timespec32 *res)
{
	struct __kernel_timespec ts;
	int ret;

	ret = __cvdso_clock_gettime_common(vd, clock, &ts);

	if (unlikely(ret))
		return clock_gettime32_fallback(clock, res);

	/* For ret == 0 */
	res->tv_sec = ts.tv_sec;
	res->tv_nsec = ts.tv_nsec;

	return ret;
}

static __maybe_unused int
__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
{
	return __cvdso_clock_gettime32_data(__arch_get_vdso_data(), clock, res);
}
#endif /* BUILD_VDSO32 */

static __maybe_unused int
__cvdso_gettimeofday_data(const struct vdso_data *vd,
			  struct __kernel_old_timeval *tv, struct timezone *tz)
{

	if (likely(tv != NULL)) {
		struct __kernel_timespec ts;

		if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
			return gettimeofday_fallback(tv, tz);

		tv->tv_sec = ts.tv_sec;
		tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
	}

	if (unlikely(tz != NULL)) {
		if (IS_ENABLED(CONFIG_TIME_NS) &&
		    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
			vd = __arch_get_timens_vdso_data();

		tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
		tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
	}

	return 0;
}

static __maybe_unused int
__cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
{
	return __cvdso_gettimeofday_data(__arch_get_vdso_data(), tv, tz);
}

#ifdef VDSO_HAS_TIME
static __maybe_unused __kernel_old_time_t
__cvdso_time_data(const struct vdso_data *vd, __kernel_old_time_t *time)
{
	__kernel_old_time_t t;

	if (IS_ENABLED(CONFIG_TIME_NS) &&
	    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
		vd = __arch_get_timens_vdso_data();

	t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);

	if (time)
		*time = t;

	return t;
}

static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time)
{
	return __cvdso_time_data(__arch_get_vdso_data(), time);
}
#endif /* VDSO_HAS_TIME */

#ifdef VDSO_HAS_CLOCK_GETRES
static __maybe_unused
int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock,
				struct __kernel_timespec *res)
{
	u32 msk;
	u64 ns;

	/* Check for negative values or invalid clocks */
	if (unlikely((u32) clock >= MAX_CLOCKS))
		return -1;

	if (IS_ENABLED(CONFIG_TIME_NS) &&
	    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
		vd = __arch_get_timens_vdso_data();

	/*
	 * Convert the clockid to a bitmask and use it to check which
	 * clocks are handled in the VDSO directly.
	 */
	msk = 1U << clock;
	if (msk & (VDSO_HRES | VDSO_RAW)) {
		/*
		 * Preserves the behaviour of posix_get_hrtimer_res().
		 */
		ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);
	} else if (msk & VDSO_COARSE) {
		/*
		 * Preserves the behaviour of posix_get_coarse_res().
		 */
		ns = LOW_RES_NSEC;
	} else {
		return -1;
	}

	if (likely(res)) {
		res->tv_sec = 0;
		res->tv_nsec = ns;
	}
	return 0;
}

static __maybe_unused
int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock,
			      struct __kernel_timespec *res)
{
	int ret = __cvdso_clock_getres_common(vd, clock, res);

	if (unlikely(ret))
		return clock_getres_fallback(clock, res);
	return 0;
}

static __maybe_unused
int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
{
	return __cvdso_clock_getres_data(__arch_get_vdso_data(), clock, res);
}

#ifdef BUILD_VDSO32
static __maybe_unused int
__cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock,
				 struct old_timespec32 *res)
{
	struct __kernel_timespec ts;
	int ret;

	ret = __cvdso_clock_getres_common(vd, clock, &ts);

	if (unlikely(ret))
		return clock_getres32_fallback(clock, res);

	if (likely(res)) {
		res->tv_sec = ts.tv_sec;
		res->tv_nsec = ts.tv_nsec;
	}
	return ret;
}

static __maybe_unused int
__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
{
	return __cvdso_clock_getres_time32_data(__arch_get_vdso_data(),
						clock, res);
}
#endif /* BUILD_VDSO32 */
#endif /* VDSO_HAS_CLOCK_GETRES */
Commit	Line	Data
00b26474 VF	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Generic userspace implementations of gettimeofday() and similar.
	4	*/
00b26474 VF	5	#include <vdso/datapage.h>
	6	#include <vdso/helpers.h>
	7
9d90b93b TG	8	#ifndef vdso_calc_delta
	9	/*
	10	* Default implementation which works for all sane clocksources. That
	11	* obviously excludes x86/TSC.
	12	*/
	13	static __always_inline
	14	u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
	15	{
	16	return ((cycles - last) & mask) * mult;
	17	}
	18	#endif
	19
8345228c CL	20	#ifndef vdso_shift_ns
	21	static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift)
	22	{
	23	return ns >> shift;
	24	}
	25	#endif
	26
1dff4156 TG	27	#ifndef __arch_vdso_hres_capable
	28	static inline bool __arch_vdso_hres_capable(void)
	29	{
	30	return true;
	31	}
	32	#endif
	33
ae12e085 CL	34	#ifndef vdso_clocksource_ok
	35	static inline bool vdso_clocksource_ok(const struct vdso_data *vd)
	36	{
	37	return vd->clock_mode != VDSO_CLOCKMODE_NONE;
	38	}
	39	#endif
	40
72ce7780 TG	41	#ifndef vdso_cycles_ok
	42	static inline bool vdso_cycles_ok(u64 cycles)
	43	{
	44	return true;
	45	}
	46	#endif
	47
660fd04f TG	48	#ifdef CONFIG_TIME_NS
	49	static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
	50	struct __kernel_timespec *ts)
	51	{
	52	const struct vdso_data *vd = __arch_get_timens_vdso_data();
	53	const struct timens_offset *offs = &vdns->offset[clk];
	54	const struct vdso_timestamp *vdso_ts;
	55	u64 cycles, last, ns;
	56	u32 seq;
	57	s64 sec;
	58
	59	if (clk != CLOCK_MONOTONIC_RAW)
	60	vd = &vd[CS_HRES_COARSE];
	61	else
	62	vd = &vd[CS_RAW];
	63	vdso_ts = &vd->basetime[clk];
	64
	65	do {
	66	seq = vdso_read_begin(vd);
f86fd32d	67
ae12e085	68	if (unlikely(!vdso_clocksource_ok(vd)))
5d51bee7	69	return -1;
f86fd32d	70
4c5a116a	71	cycles = __arch_get_hw_counter(vd->clock_mode, vd);
72ce7780 TG	72	if (unlikely(!vdso_cycles_ok(cycles)))
72ce7780 TG	73	return -1;
660fd04f TG	74	ns = vdso_ts->nsec;
660fd04f TG	75	last = vd->cycle_last;
660fd04f	76	ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
8345228c	77	ns = vdso_shift_ns(ns, vd->shift);
660fd04f TG	78	sec = vdso_ts->sec;
	79	} while (unlikely(vdso_read_retry(vd, seq)));
	80
	81	/* Add the namespace offset */
	82	sec += offs->sec;
	83	ns += offs->nsec;
	84
	85	/*
	86	* Do this outside the loop: a race inside the loop could result
	87	* in __iter_div_u64_rem() being extremely slow.
	88	*/
	89	ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
	90	ts->tv_nsec = ns;
	91
	92	return 0;
	93	}
	94	#else
	95	static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void)
	96	{
	97	return NULL;
	98	}
	99
	100	static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
	101	struct __kernel_timespec *ts)
	102	{
	103	return -EINVAL;
	104	}
	105	#endif
	106
c966533f	107	static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk,
660fd04f	108	struct __kernel_timespec *ts)
00b26474 VF	109	{
	110	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
	111	u64 cycles, last, sec, ns;
	112	u32 seq;
	113
1dff4156 TG	114	/* Allows to compile the high resolution parts out */
	115	if (!__arch_vdso_hres_capable())
	116	return -1;
	117
00b26474	118	do {
660fd04f	119	/*
2d6b01bd	120	* Open coded to handle VDSO_CLOCKMODE_TIMENS. Time namespace
660fd04f TG	121	* enabled tasks have a special VVAR page installed which
660fd04f TG	122	* has vd->seq set to 1 and vd->clock_mode set to
2d6b01bd	123	* VDSO_CLOCKMODE_TIMENS. For non time namespace affected tasks
660fd04f TG	124	* this does not affect performance because if vd->seq is
	125	* odd, i.e. a concurrent update is in progress the extra
	126	* check for vd->clock_mode is just a few extra
	127	* instructions while spin waiting for vd->seq to become
	128	* even again.
	129	*/
	130	while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) {
	131	if (IS_ENABLED(CONFIG_TIME_NS) &&
2d6b01bd	132	vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
660fd04f TG	133	return do_hres_timens(vd, clk, ts);
	134	cpu_relax();
	135	}
	136	smp_rmb();
	137
ae12e085	138	if (unlikely(!vdso_clocksource_ok(vd)))
5d51bee7	139	return -1;
f86fd32d	140
4c5a116a	141	cycles = __arch_get_hw_counter(vd->clock_mode, vd);
72ce7780 TG	142	if (unlikely(!vdso_cycles_ok(cycles)))
72ce7780 TG	143	return -1;
00b26474 VF	144	ns = vdso_ts->nsec;
00b26474 VF	145	last = vd->cycle_last;
9d90b93b	146	ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
8345228c	147	ns = vdso_shift_ns(ns, vd->shift);
00b26474 VF	148	sec = vdso_ts->sec;
	149	} while (unlikely(vdso_read_retry(vd, seq)));
	150
	151	/*
	152	* Do this outside the loop: a race inside the loop could result
	153	* in __iter_div_u64_rem() being extremely slow.
	154	*/
	155	ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
	156	ts->tv_nsec = ns;
	157
	158	return 0;
	159	}
	160
660fd04f TG	161	#ifdef CONFIG_TIME_NS
	162	static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
	163	struct __kernel_timespec *ts)
	164	{
	165	const struct vdso_data *vd = __arch_get_timens_vdso_data();
	166	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
	167	const struct timens_offset *offs = &vdns->offset[clk];
	168	u64 nsec;
	169	s64 sec;
	170	s32 seq;
	171
	172	do {
	173	seq = vdso_read_begin(vd);
	174	sec = vdso_ts->sec;
	175	nsec = vdso_ts->nsec;
	176	} while (unlikely(vdso_read_retry(vd, seq)));
	177
	178	/* Add the namespace offset */
	179	sec += offs->sec;
	180	nsec += offs->nsec;
	181
	182	/*
	183	* Do this outside the loop: a race inside the loop could result
	184	* in __iter_div_u64_rem() being extremely slow.
	185	*/
	186	ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
	187	ts->tv_nsec = nsec;
	188	return 0;
	189	}
	190	#else
	191	static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
	192	struct __kernel_timespec *ts)
	193	{
	194	return -1;
	195	}
	196	#endif
	197
c966533f AV	198	static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk,
c966533f AV	199	struct __kernel_timespec *ts)
00b26474 VF	200	{
	201	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
	202	u32 seq;
	203
	204	do {
660fd04f	205	/*
2d6b01bd	206	* Open coded to handle VDSO_CLOCK_TIMENS. See comment in
660fd04f TG	207	* do_hres().
	208	*/
	209	while ((seq = READ_ONCE(vd->seq)) & 1) {
	210	if (IS_ENABLED(CONFIG_TIME_NS) &&
2d6b01bd	211	vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
660fd04f TG	212	return do_coarse_timens(vd, clk, ts);
	213	cpu_relax();
	214	}
	215	smp_rmb();
	216
00b26474 VF	217	ts->tv_sec = vdso_ts->sec;
	218	ts->tv_nsec = vdso_ts->nsec;
	219	} while (unlikely(vdso_read_retry(vd, seq)));
8463cf80 CL	220
8463cf80 CL	221	return 0;
00b26474 VF	222	}
00b26474 VF	223
b91c8c42	224	static __always_inline int
e876f0b6 CL	225	__cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock,
e876f0b6 CL	226	struct __kernel_timespec *ts)
00b26474	227	{
00b26474 VF	228	u32 msk;
	229
	230	/* Check for negative values or invalid clocks */
	231	if (unlikely((u32) clock >= MAX_CLOCKS))
502a590a	232	return -1;
00b26474 VF	233
	234	/*
	235	* Convert the clockid to a bitmask and use it to check which
	236	* clocks are handled in the VDSO directly.
	237	*/
	238	msk = 1U << clock;
8463cf80	239	if (likely(msk & VDSO_HRES))
c966533f	240	vd = &vd[CS_HRES_COARSE];
8463cf80 CL	241	else if (msk & VDSO_COARSE)
	242	return do_coarse(&vd[CS_HRES_COARSE], clock, ts);
	243	else if (msk & VDSO_RAW)
c966533f AV	244	vd = &vd[CS_RAW];
	245	else
	246	return -1;
8463cf80	247
c966533f	248	return do_hres(vd, clock, ts);
502a590a	249	}
00b26474	250
502a590a	251	static __maybe_unused int
e876f0b6 CL	252	__cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock,
e876f0b6 CL	253	struct __kernel_timespec *ts)
502a590a	254	{
e876f0b6	255	int ret = __cvdso_clock_gettime_common(vd, clock, ts);
502a590a TG	256
	257	if (unlikely(ret))
	258	return clock_gettime_fallback(clock, ts);
	259	return 0;
00b26474 VF	260	}
00b26474 VF	261
e876f0b6 CL	262	static __maybe_unused int
	263	__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
	264	{
	265	return __cvdso_clock_gettime_data(__arch_get_vdso_data(), clock, ts);
	266	}
	267
bf279849	268	#ifdef BUILD_VDSO32
00b26474	269	static __maybe_unused int
e876f0b6 CL	270	__cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock,
e876f0b6 CL	271	struct old_timespec32 *res)
00b26474 VF	272	{
	273	struct __kernel_timespec ts;
	274	int ret;
	275
e876f0b6	276	ret = __cvdso_clock_gettime_common(vd, clock, &ts);
00b26474	277
c60a32ea TG	278	if (unlikely(ret))
c60a32ea TG	279	return clock_gettime32_fallback(clock, res);
502a590a	280
a279235d VF	281	/* For ret == 0 */
	282	res->tv_sec = ts.tv_sec;
	283	res->tv_nsec = ts.tv_nsec;
	284
00b26474	285	return ret;
00b26474	286	}
e876f0b6 CL	287
	288	static __maybe_unused int
	289	__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
	290	{
	291	return __cvdso_clock_gettime32_data(__arch_get_vdso_data(), clock, res);
	292	}
bf279849	293	#endif /* BUILD_VDSO32 */
00b26474 VF	294
00b26474 VF	295	static __maybe_unused int
e876f0b6 CL	296	__cvdso_gettimeofday_data(const struct vdso_data *vd,
e876f0b6 CL	297	struct __kernel_old_timeval tv, struct timezone tz)
00b26474	298	{
00b26474 VF	299
	300	if (likely(tv != NULL)) {
	301	struct __kernel_timespec ts;
	302
	303	if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
	304	return gettimeofday_fallback(tv, tz);
	305
	306	tv->tv_sec = ts.tv_sec;
	307	tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
	308	}
	309
	310	if (unlikely(tz != NULL)) {
660fd04f	311	if (IS_ENABLED(CONFIG_TIME_NS) &&
2d6b01bd	312	vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
660fd04f TG	313	vd = __arch_get_timens_vdso_data();
660fd04f TG	314
00b26474 VF	315	tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
	316	tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
	317	}
	318
	319	return 0;
	320	}
	321
e876f0b6 CL	322	static __maybe_unused int
	323	__cvdso_gettimeofday(struct __kernel_old_timeval tv, struct timezone tz)
	324	{
	325	return __cvdso_gettimeofday_data(__arch_get_vdso_data(), tv, tz);
	326	}
	327
00b26474	328	#ifdef VDSO_HAS_TIME
e876f0b6 CL	329	static __maybe_unused __kernel_old_time_t
e876f0b6 CL	330	__cvdso_time_data(const struct vdso_data vd, __kernel_old_time_t time)
00b26474	331	{
660fd04f TG	332	__kernel_old_time_t t;
660fd04f TG	333
2d6b01bd TG	334	if (IS_ENABLED(CONFIG_TIME_NS) &&
2d6b01bd TG	335	vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
660fd04f TG	336	vd = __arch_get_timens_vdso_data();
	337
	338	t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
00b26474 VF	339
	340	if (time)
	341	*time = t;
	342
	343	return t;
	344	}
e876f0b6 CL	345
	346	static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time)
	347	{
	348	return __cvdso_time_data(__arch_get_vdso_data(), time);
	349	}
00b26474 VF	350	#endif /* VDSO_HAS_TIME */
	351
	352	#ifdef VDSO_HAS_CLOCK_GETRES
	353	static __maybe_unused
e876f0b6 CL	354	int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock,
e876f0b6 CL	355	struct __kernel_timespec *res)
00b26474	356	{
00b26474	357	u32 msk;
502a590a	358	u64 ns;
00b26474 VF	359
	360	/* Check for negative values or invalid clocks */
	361	if (unlikely((u32) clock >= MAX_CLOCKS))
502a590a	362	return -1;
00b26474	363
2d6b01bd TG	364	if (IS_ENABLED(CONFIG_TIME_NS) &&
2d6b01bd TG	365	vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
660fd04f TG	366	vd = __arch_get_timens_vdso_data();
660fd04f TG	367
00b26474 VF	368	/*
	369	* Convert the clockid to a bitmask and use it to check which
	370	* clocks are handled in the VDSO directly.
	371	*/
	372	msk = 1U << clock;
cdb7c5a9	373	if (msk & (VDSO_HRES \| VDSO_RAW)) {
00b26474 VF	374	/*
	375	* Preserves the behaviour of posix_get_hrtimer_res().
	376	*/
49a101d7	377	ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);
00b26474 VF	378	} else if (msk & VDSO_COARSE) {
	379	/*
	380	* Preserves the behaviour of posix_get_coarse_res().
	381	*/
	382	ns = LOW_RES_NSEC;
00b26474	383	} else {
502a590a	384	return -1;
00b26474 VF	385	}
00b26474 VF	386
1638b8f0 TG	387	if (likely(res)) {
	388	res->tv_sec = 0;
	389	res->tv_nsec = ns;
	390	}
00b26474	391	return 0;
502a590a TG	392	}
502a590a TG	393
ffd08731	394	static __maybe_unused
e876f0b6 CL	395	int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock,
e876f0b6 CL	396	struct __kernel_timespec *res)
502a590a	397	{
e876f0b6	398	int ret = __cvdso_clock_getres_common(vd, clock, res);
00b26474	399
502a590a TG	400	if (unlikely(ret))
	401	return clock_getres_fallback(clock, res);
	402	return 0;
00b26474 VF	403	}
00b26474 VF	404
e876f0b6 CL	405	static __maybe_unused
	406	int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
	407	{
	408	return __cvdso_clock_getres_data(__arch_get_vdso_data(), clock, res);
	409	}
	410
bf279849	411	#ifdef BUILD_VDSO32
00b26474	412	static __maybe_unused int
e876f0b6 CL	413	__cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock,
e876f0b6 CL	414	struct old_timespec32 *res)
00b26474 VF	415	{
	416	struct __kernel_timespec ts;
	417	int ret;
	418
e876f0b6	419	ret = __cvdso_clock_getres_common(vd, clock, &ts);
c60a32ea	420
c60a32ea TG	421	if (unlikely(ret))
c60a32ea TG	422	return clock_getres32_fallback(clock, res);
00b26474	423
a279235d	424	if (likely(res)) {
00b26474 VF	425	res->tv_sec = ts.tv_sec;
	426	res->tv_nsec = ts.tv_nsec;
	427	}
00b26474	428	return ret;
00b26474	429	}
e876f0b6 CL	430
	431	static __maybe_unused int
	432	__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
	433	{
	434	return __cvdso_clock_getres_time32_data(__arch_get_vdso_data(),
	435	clock, res);
	436	}
bf279849	437	#endif /* BUILD_VDSO32 */
00b26474	438	#endif /* VDSO_HAS_CLOCK_GETRES */