+++ /dev/null
-/* $Id: timer-r0drv-linux.c $ */
-/** @file
- * IPRT - Timers, Ring-0 Driver, Linux.
- */
-
-/*
- * Copyright (C) 2006-2016 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- *
- * The contents of this file may alternatively be used under the terms
- * of the Common Development and Distribution License Version 1.0
- * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
- * VirtualBox OSE distribution, in which case the provisions of the
- * CDDL are applicable instead of those of the GPL.
- *
- * You may elect to license modified versions of this file under the
- * terms and conditions of either the GPL or the CDDL or both.
- */
-
-
-/*********************************************************************************************************************************
-* Header Files *
-*********************************************************************************************************************************/
-#include "the-linux-kernel.h"
-#include "internal/iprt.h"
-
-#include <iprt/timer.h>
-#include <iprt/time.h>
-#include <iprt/mp.h>
-#include <iprt/cpuset.h>
-#include <iprt/spinlock.h>
-#include <iprt/err.h>
-#include <iprt/asm.h>
-#include <iprt/assert.h>
-#include <iprt/alloc.h>
-
-#include "internal/magics.h"
-
-/** @def RTTIMER_LINUX_WITH_HRTIMER
- * Whether to use high resolution timers. */
-#if !defined(RTTIMER_LINUX_WITH_HRTIMER) \
- && defined(IPRT_LINUX_HAS_HRTIMER)
-# define RTTIMER_LINUX_WITH_HRTIMER
-#endif
-
-#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 31)
-# define mod_timer_pinned mod_timer
-# define HRTIMER_MODE_ABS_PINNED HRTIMER_MODE_ABS
-#endif
-
-
-/*********************************************************************************************************************************
-* Structures and Typedefs *
-*********************************************************************************************************************************/
-/**
- * Timer state machine.
- *
- * This is used to try handle the issues with MP events and
- * timers that runs on all CPUs. It's relatively nasty :-/
- */
-typedef enum RTTIMERLNXSTATE
-{
- /** Stopped. */
- RTTIMERLNXSTATE_STOPPED = 0,
- /** Transient state; next ACTIVE. */
- RTTIMERLNXSTATE_STARTING,
- /** Transient state; next ACTIVE. (not really necessary) */
- RTTIMERLNXSTATE_MP_STARTING,
- /** Active. */
- RTTIMERLNXSTATE_ACTIVE,
- /** Active and in callback; next ACTIVE, STOPPED or CALLBACK_DESTROYING. */
- RTTIMERLNXSTATE_CALLBACK,
- /** Stopped while in the callback; next STOPPED. */
- RTTIMERLNXSTATE_CB_STOPPING,
- /** Restarted while in the callback; next ACTIVE, STOPPED, DESTROYING. */
- RTTIMERLNXSTATE_CB_RESTARTING,
- /** The callback shall destroy the timer; next STOPPED. */
- RTTIMERLNXSTATE_CB_DESTROYING,
- /** Transient state; next STOPPED. */
- RTTIMERLNXSTATE_STOPPING,
- /** Transient state; next STOPPED. */
- RTTIMERLNXSTATE_MP_STOPPING,
- /** The usual 32-bit hack. */
- RTTIMERLNXSTATE_32BIT_HACK = 0x7fffffff
-} RTTIMERLNXSTATE;
-
-
-/**
- * A Linux sub-timer.
- */
-typedef struct RTTIMERLNXSUBTIMER
-{
- /** Timer specific data. */
- union
- {
-#if defined(RTTIMER_LINUX_WITH_HRTIMER)
- /** High resolution timer. */
- struct
- {
- /** The linux timer structure. */
- struct hrtimer LnxTimer;
- } Hr;
-#endif
- /** Standard timer. */
- struct
- {
- /** The linux timer structure. */
- struct timer_list LnxTimer;
- /** The start of the current run (ns).
- * This is used to calculate when the timer ought to fire the next time. */
- uint64_t u64NextTS;
- /** The u64NextTS in jiffies. */
- unsigned long ulNextJiffies;
- /** Set when starting or changing the timer so that u64StartTs
- * and u64NextTS gets reinitialized (eliminating some jitter). */
- bool volatile fFirstAfterChg;
- } Std;
- } u;
- /** The current tick number. */
- uint64_t iTick;
- /** Restart the single shot timer at this specific time.
- * Used when a single shot timer is restarted from the callback. */
- uint64_t volatile uNsRestartAt;
- /** Pointer to the parent timer. */
- PRTTIMER pParent;
- /** The current sub-timer state. */
- RTTIMERLNXSTATE volatile enmState;
-} RTTIMERLNXSUBTIMER;
-/** Pointer to a linux sub-timer. */
-typedef RTTIMERLNXSUBTIMER *PRTTIMERLNXSUBTIMER;
-
-
-/**
- * The internal representation of an Linux timer handle.
- */
-typedef struct RTTIMER
-{
- /** Magic.
- * This is RTTIMER_MAGIC, but changes to something else before the timer
- * is destroyed to indicate clearly that thread should exit. */
- uint32_t volatile u32Magic;
- /** Spinlock synchronizing the fSuspended and MP event handling.
- * This is NIL_RTSPINLOCK if cCpus == 1. */
- RTSPINLOCK hSpinlock;
- /** Flag indicating that the timer is suspended. */
- bool volatile fSuspended;
- /** Whether the timer must run on one specific CPU or not. */
- bool fSpecificCpu;
-#ifdef CONFIG_SMP
- /** Whether the timer must run on all CPUs or not. */
- bool fAllCpus;
-#endif /* else: All -> specific on non-SMP kernels */
- /** Whether it is a high resolution timer or a standard one. */
- bool fHighRes;
- /** The id of the CPU it must run on if fSpecificCpu is set. */
- RTCPUID idCpu;
- /** The number of CPUs this timer should run on. */
- RTCPUID cCpus;
- /** Callback. */
- PFNRTTIMER pfnTimer;
- /** User argument. */
- void *pvUser;
- /** The timer interval. 0 if one-shot. */
- uint64_t volatile u64NanoInterval;
- /** This is set to the number of jiffies between ticks if the interval is
- * an exact number of jiffies. (Standard timers only.) */
- unsigned long volatile cJiffies;
- /** The change interval spinlock for standard timers only. */
- spinlock_t ChgIntLock;
- /** Workqueue item for delayed destruction. */
- RTR0LNXWORKQUEUEITEM DtorWorkqueueItem;
- /** Sub-timers.
- * Normally there is just one, but for RTTIMER_FLAGS_CPU_ALL this will contain
- * an entry for all possible cpus. In that case the index will be the same as
- * for the RTCpuSet. */
- RTTIMERLNXSUBTIMER aSubTimers[1];
-} RTTIMER;
-
-
-/**
- * A rtTimerLinuxStartOnCpu and rtTimerLinuxStartOnCpu argument package.
- */
-typedef struct RTTIMERLINUXSTARTONCPUARGS
-{
- /** The current time (RTTimeSystemNanoTS). */
- uint64_t u64Now;
- /** When to start firing (delta). */
- uint64_t u64First;
-} RTTIMERLINUXSTARTONCPUARGS;
-/** Pointer to a rtTimerLinuxStartOnCpu argument package. */
-typedef RTTIMERLINUXSTARTONCPUARGS *PRTTIMERLINUXSTARTONCPUARGS;
-
-
-/*********************************************************************************************************************************
-* Internal Functions *
-*********************************************************************************************************************************/
-#ifdef CONFIG_SMP
-static DECLCALLBACK(void) rtTimerLinuxMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser);
-#endif
-
-#if 0
-#define DEBUG_HACKING
-#include <iprt/string.h>
-#include <iprt/asm-amd64-x86.h>
-static void myLogBackdoorPrintf(const char *pszFormat, ...)
-{
- char szTmp[256];
- va_list args;
- size_t cb;
-
- cb = RTStrPrintf(szTmp, sizeof(szTmp) - 10, "%d: ", RTMpCpuId());
- va_start(args, pszFormat);
- cb += RTStrPrintfV(&szTmp[cb], sizeof(szTmp) - cb, pszFormat, args);
- va_end(args);
-
- ASMOutStrU8(0x504, (uint8_t *)&szTmp[0], cb);
-}
-# define RTAssertMsg1Weak(pszExpr, uLine, pszFile, pszFunction) \
- myLogBackdoorPrintf("\n!!Guest Assertion failed!!\n%s(%d) %s\n%s\n", uLine, pszFile, pszFunction, (pszExpr))
-# define RTAssertMsg2Weak myLogBackdoorPrintf
-# define RTTIMERLNX_LOG(a) myLogBackdoorPrintf a
-#else
-# define RTTIMERLNX_LOG(a) do { } while (0)
-#endif
-
-/**
- * Sets the state.
- */
-DECLINLINE(void) rtTimerLnxSetState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState)
-{
-#ifdef DEBUG_HACKING
- RTTIMERLNX_LOG(("set %d -> %d\n", *penmState, enmNewState));
-#endif
- ASMAtomicWriteU32((uint32_t volatile *)penmState, enmNewState);
-}
-
-
-/**
- * Sets the state if it has a certain value.
- *
- * @return true if xchg was done.
- * @return false if xchg wasn't done.
- */
-#ifdef DEBUG_HACKING
-#define rtTimerLnxCmpXchgState(penmState, enmNewState, enmCurState) rtTimerLnxCmpXchgStateDebug(penmState, enmNewState, enmCurState, __LINE__)
-static bool rtTimerLnxCmpXchgStateDebug(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState,
- RTTIMERLNXSTATE enmCurState, uint32_t uLine)
-{
- RTTIMERLNXSTATE enmOldState = enmCurState;
- bool fRc = ASMAtomicCmpXchgExU32((uint32_t volatile *)penmState, enmNewState, enmCurState, (uint32_t *)&enmOldState);
- RTTIMERLNX_LOG(("cxg %d -> %d - %d at %u\n", enmOldState, enmNewState, fRc, uLine));
- return fRc;
-}
-#else
-DECLINLINE(bool) rtTimerLnxCmpXchgState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState,
- RTTIMERLNXSTATE enmCurState)
-{
- return ASMAtomicCmpXchgU32((uint32_t volatile *)penmState, enmNewState, enmCurState);
-}
-#endif
-
-
-/**
- * Gets the state.
- */
-DECLINLINE(RTTIMERLNXSTATE) rtTimerLnxGetState(RTTIMERLNXSTATE volatile *penmState)
-{
- return (RTTIMERLNXSTATE)ASMAtomicUoReadU32((uint32_t volatile *)penmState);
-}
-
-#ifdef RTTIMER_LINUX_WITH_HRTIMER
-
-/**
- * Converts a nano second time stamp to ktime_t.
- *
- * ASSUMES RTTimeSystemNanoTS() is implemented using ktime_get_ts().
- *
- * @returns ktime_t.
- * @param cNanoSecs Nanoseconds.
- */
-DECLINLINE(ktime_t) rtTimerLnxNanoToKt(uint64_t cNanoSecs)
-{
- /* With some luck the compiler optimizes the division out of this... (Bet it doesn't.) */
- return ktime_set(cNanoSecs / 1000000000, cNanoSecs % 1000000000);
-}
-
-/**
- * Converts ktime_t to a nano second time stamp.
- *
- * ASSUMES RTTimeSystemNanoTS() is implemented using ktime_get_ts().
- *
- * @returns nano second time stamp.
- * @param Kt ktime_t.
- */
-DECLINLINE(uint64_t) rtTimerLnxKtToNano(ktime_t Kt)
-{
- return ktime_to_ns(Kt);
-}
-
-#endif /* RTTIMER_LINUX_WITH_HRTIMER */
-
-/**
- * Converts a nano second interval to jiffies.
- *
- * @returns Jiffies.
- * @param cNanoSecs Nanoseconds.
- */
-DECLINLINE(unsigned long) rtTimerLnxNanoToJiffies(uint64_t cNanoSecs)
-{
- /* this can be made even better... */
- if (cNanoSecs > (uint64_t)TICK_NSEC * MAX_JIFFY_OFFSET)
- return MAX_JIFFY_OFFSET;
-# if ARCH_BITS == 32
- if (RT_LIKELY(cNanoSecs <= UINT32_MAX))
- return ((uint32_t)cNanoSecs + (TICK_NSEC-1)) / TICK_NSEC;
-# endif
- return (cNanoSecs + (TICK_NSEC-1)) / TICK_NSEC;
-}
-
-
-/**
- * Starts a sub-timer (RTTimerStart).
- *
- * @param pSubTimer The sub-timer to start.
- * @param u64Now The current timestamp (RTTimeSystemNanoTS()).
- * @param u64First The interval from u64Now to the first time the timer should fire.
- * @param fPinned true = timer pinned to a specific CPU,
- * false = timer can migrate between CPUs
- * @param fHighRes Whether the user requested a high resolution timer or not.
- * @param enmOldState The old timer state.
- */
-static void rtTimerLnxStartSubTimer(PRTTIMERLNXSUBTIMER pSubTimer, uint64_t u64Now, uint64_t u64First,
- bool fPinned, bool fHighRes)
-{
- /*
- * Calc when it should start firing.
- */
- uint64_t u64NextTS = u64Now + u64First;
- if (!fHighRes)
- pSubTimer->u.Std.u64NextTS = u64NextTS;
- RTTIMERLNX_LOG(("startsubtimer %p\n", pSubTimer->pParent));
-
- pSubTimer->iTick = 0;
-
-#ifdef RTTIMER_LINUX_WITH_HRTIMER
- if (fHighRes)
- hrtimer_start(&pSubTimer->u.Hr.LnxTimer, rtTimerLnxNanoToKt(u64NextTS),
- fPinned ? HRTIMER_MODE_ABS_PINNED : HRTIMER_MODE_ABS);
- else
-#endif
- {
- unsigned long cJiffies = !u64First ? 0 : rtTimerLnxNanoToJiffies(u64First);
- pSubTimer->u.Std.ulNextJiffies = jiffies + cJiffies;
- pSubTimer->u.Std.fFirstAfterChg = true;
-#ifdef CONFIG_SMP
- if (fPinned)
- {
-# if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)
- mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
-# else
- mod_timer_pinned(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
-# endif
- }
- else
-#endif
- mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
- }
-
- /* Be a bit careful here since we could be racing the callback. */
- if (!rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_STARTING))
- rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_MP_STARTING);
-}
-
-
-/**
- * Stops a sub-timer (RTTimerStart and rtTimerLinuxMpEvent()).
- *
- * The caller has already changed the state, so we will not be in a callback
- * situation wrt to the calling thread.
- *
- * @param pSubTimer The sub-timer.
- * @param fHighRes Whether the user requested a high resolution timer or not.
- */
-static void rtTimerLnxStopSubTimer(PRTTIMERLNXSUBTIMER pSubTimer, bool fHighRes)
-{
- RTTIMERLNX_LOG(("stopsubtimer %p %d\n", pSubTimer->pParent, fHighRes));
-#ifdef RTTIMER_LINUX_WITH_HRTIMER
- if (fHighRes)
- {
- /* There is no equivalent to del_timer in the hrtimer API,
- hrtimer_cancel() == del_timer_sync(). Just like the WARN_ON in
- del_timer_sync() asserts, waiting for a timer callback to complete
- is deadlock prone, so don't do it. */
- int rc = hrtimer_try_to_cancel(&pSubTimer->u.Hr.LnxTimer);
- if (rc < 0)
- {
- hrtimer_start(&pSubTimer->u.Hr.LnxTimer, ktime_set(KTIME_SEC_MAX, 0), HRTIMER_MODE_ABS);
- hrtimer_try_to_cancel(&pSubTimer->u.Hr.LnxTimer);
- }
- }
- else
-#endif
- del_timer(&pSubTimer->u.Std.LnxTimer);
-
- rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED);
-}
-
-
-/**
- * Used by RTTimerDestroy and rtTimerLnxCallbackDestroy to do the actual work.
- *
- * @param pTimer The timer in question.
- */
-static void rtTimerLnxDestroyIt(PRTTIMER pTimer)
-{
- RTSPINLOCK hSpinlock = pTimer->hSpinlock;
- RTCPUID iCpu;
- Assert(pTimer->fSuspended);
- RTTIMERLNX_LOG(("destroyit %p\n", pTimer));
-
- /*
- * Remove the MP notifications first because it'll reduce the risk of
- * us overtaking any MP event that might theoretically be racing us here.
- */
-#ifdef CONFIG_SMP
- if ( pTimer->cCpus > 1
- && hSpinlock != NIL_RTSPINLOCK)
- {
- int rc = RTMpNotificationDeregister(rtTimerLinuxMpEvent, pTimer);
- AssertRC(rc);
- }
-#endif /* CONFIG_SMP */
-
- /*
- * Invalidate the handle.
- */
- ASMAtomicWriteU32(&pTimer->u32Magic, ~RTTIMER_MAGIC);
-
- /*
- * Make sure all timers have stopped executing since we're stopping them in
- * an asynchronous manner up in rtTimerLnxStopSubTimer.
- */
- iCpu = pTimer->cCpus;
- while (iCpu-- > 0)
- {
-#ifdef RTTIMER_LINUX_WITH_HRTIMER
- if (pTimer->fHighRes)
- hrtimer_cancel(&pTimer->aSubTimers[iCpu].u.Hr.LnxTimer);
- else
-#endif
- del_timer_sync(&pTimer->aSubTimers[iCpu].u.Std.LnxTimer);
- }
-
- /*
- * Finally, free the resources.
- */
- RTMemFreeEx(pTimer, RT_OFFSETOF(RTTIMER, aSubTimers[pTimer->cCpus]));
- if (hSpinlock != NIL_RTSPINLOCK)
- RTSpinlockDestroy(hSpinlock);
-}
-
-
-/**
- * Workqueue callback (no DECLCALLBACK!) for deferred destruction.
- *
- * @param pWork Pointer to the DtorWorkqueueItem member of our timer
- * structure.
- */
-static void rtTimerLnxDestroyDeferred(RTR0LNXWORKQUEUEITEM *pWork)
-{
- PRTTIMER pTimer = RT_FROM_MEMBER(pWork, RTTIMER, DtorWorkqueueItem);
- rtTimerLnxDestroyIt(pTimer);
-}
-
-
-/**
- * Called when the timer was destroyed by the callback function.
- *
- * @param pTimer The timer.
- * @param pSubTimer The sub-timer which we're handling, the state of this
- * will be RTTIMERLNXSTATE_CALLBACK_DESTROYING.
- */
-static void rtTimerLnxCallbackDestroy(PRTTIMER pTimer, PRTTIMERLNXSUBTIMER pSubTimer)
-{
- /*
- * If it's an omni timer, the last dude does the destroying.
- */
- if (pTimer->cCpus > 1)
- {
- uint32_t iCpu = pTimer->cCpus;
- RTSpinlockAcquire(pTimer->hSpinlock);
-
- Assert(pSubTimer->enmState == RTTIMERLNXSTATE_CB_DESTROYING);
- rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED);
-
- while (iCpu-- > 0)
- if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) != RTTIMERLNXSTATE_STOPPED)
- {
- RTSpinlockRelease(pTimer->hSpinlock);
- return;
- }
-
- RTSpinlockRelease(pTimer->hSpinlock);
- }
-
- /*
- * Destroying a timer from the callback is unsafe since the callout code
- * might be touching the timer structure upon return (hrtimer does!). So,
- * we have to defer the actual destruction to the IRPT workqueue.
- */
- rtR0LnxWorkqueuePush(&pTimer->DtorWorkqueueItem, rtTimerLnxDestroyDeferred);
-}
-
-
-#ifdef CONFIG_SMP
-/**
- * Deal with a sub-timer that has migrated.
- *
- * @param pTimer The timer.
- * @param pSubTimer The sub-timer.
- */
-static void rtTimerLnxCallbackHandleMigration(PRTTIMER pTimer, PRTTIMERLNXSUBTIMER pSubTimer)
-{
- RTTIMERLNXSTATE enmState;
- if (pTimer->cCpus > 1)
- RTSpinlockAcquire(pTimer->hSpinlock);
-
- do
- {
- enmState = rtTimerLnxGetState(&pSubTimer->enmState);
- switch (enmState)
- {
- case RTTIMERLNXSTATE_STOPPING:
- case RTTIMERLNXSTATE_MP_STOPPING:
- enmState = RTTIMERLNXSTATE_STOPPED;
- case RTTIMERLNXSTATE_STOPPED:
- break;
-
- default:
- AssertMsgFailed(("%d\n", enmState));
- case RTTIMERLNXSTATE_STARTING:
- case RTTIMERLNXSTATE_MP_STARTING:
- case RTTIMERLNXSTATE_ACTIVE:
- case RTTIMERLNXSTATE_CALLBACK:
- case RTTIMERLNXSTATE_CB_STOPPING:
- case RTTIMERLNXSTATE_CB_RESTARTING:
- if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, enmState))
- enmState = RTTIMERLNXSTATE_STOPPED;
- break;
-
- case RTTIMERLNXSTATE_CB_DESTROYING:
- {
- if (pTimer->cCpus > 1)
- RTSpinlockRelease(pTimer->hSpinlock);
-
- rtTimerLnxCallbackDestroy(pTimer, pSubTimer);
- return;
- }
- }
- } while (enmState != RTTIMERLNXSTATE_STOPPED);
-
- if (pTimer->cCpus > 1)
- RTSpinlockRelease(pTimer->hSpinlock);
-}
-#endif /* CONFIG_SMP */
-
-
-/**
- * The slow path of rtTimerLnxChangeToCallbackState.
- *
- * @returns true if changed successfully, false if not.
- * @param pSubTimer The sub-timer.
- */
-static bool rtTimerLnxChangeToCallbackStateSlow(PRTTIMERLNXSUBTIMER pSubTimer)
-{
- for (;;)
- {
- RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pSubTimer->enmState);
- switch (enmState)
- {
- case RTTIMERLNXSTATE_ACTIVE:
- case RTTIMERLNXSTATE_STARTING:
- case RTTIMERLNXSTATE_MP_STARTING:
- if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_CALLBACK, enmState))
- return true;
- break;
-
- case RTTIMERLNXSTATE_CALLBACK:
- case RTTIMERLNXSTATE_CB_STOPPING:
- case RTTIMERLNXSTATE_CB_RESTARTING:
- case RTTIMERLNXSTATE_CB_DESTROYING:
- AssertMsgFailed(("%d\n", enmState));
- default:
- return false;
- }
- ASMNopPause();
- }
-}
-
-
-/**
- * Tries to change the sub-timer state to 'callback'.
- *
- * @returns true if changed successfully, false if not.
- * @param pSubTimer The sub-timer.
- */
-DECLINLINE(bool) rtTimerLnxChangeToCallbackState(PRTTIMERLNXSUBTIMER pSubTimer)
-{
- if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_CALLBACK, RTTIMERLNXSTATE_ACTIVE)))
- return true;
- return rtTimerLnxChangeToCallbackStateSlow(pSubTimer);
-}
-
-
-#ifdef RTTIMER_LINUX_WITH_HRTIMER
-/**
- * Timer callback function for high resolution timers.
- *
- * @returns HRTIMER_NORESTART or HRTIMER_RESTART depending on whether it's a
- * one-shot or interval timer.
- * @param pHrTimer Pointer to the sub-timer structure.
- */
-static enum hrtimer_restart rtTimerLinuxHrCallback(struct hrtimer *pHrTimer)
-{
- PRTTIMERLNXSUBTIMER pSubTimer = RT_FROM_MEMBER(pHrTimer, RTTIMERLNXSUBTIMER, u.Hr.LnxTimer);
- PRTTIMER pTimer = pSubTimer->pParent;
-
-
- RTTIMERLNX_LOG(("hrcallback %p\n", pTimer));
- if (RT_UNLIKELY(!rtTimerLnxChangeToCallbackState(pSubTimer)))
- return HRTIMER_NORESTART;
-
-#ifdef CONFIG_SMP
- /*
- * Check for unwanted migration.
- */
- if (pTimer->fAllCpus || pTimer->fSpecificCpu)
- {
- RTCPUID idCpu = RTMpCpuId();
- if (RT_UNLIKELY( pTimer->fAllCpus
- ? (RTCPUID)(pSubTimer - &pTimer->aSubTimers[0]) != idCpu
- : pTimer->idCpu != idCpu))
- {
- rtTimerLnxCallbackHandleMigration(pTimer, pSubTimer);
- return HRTIMER_NORESTART;
- }
- }
-#endif
-
- if (pTimer->u64NanoInterval)
- {
- /*
- * Periodic timer, run it and update the native timer afterwards so
- * we can handle RTTimerStop and RTTimerChangeInterval from the
- * callback as well as a racing control thread.
- */
- pTimer->pfnTimer(pTimer, pTimer->pvUser, ++pSubTimer->iTick);
- hrtimer_add_expires_ns(&pSubTimer->u.Hr.LnxTimer, ASMAtomicReadU64(&pTimer->u64NanoInterval));
- if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_CALLBACK)))
- return HRTIMER_RESTART;
- }
- else
- {
- /*
- * One shot timer (no omni), stop it before dispatching it.
- * Allow RTTimerStart as well as RTTimerDestroy to be called from
- * the callback.
- */
- ASMAtomicWriteBool(&pTimer->fSuspended, true);
- pTimer->pfnTimer(pTimer, pTimer->pvUser, ++pSubTimer->iTick);
- if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_CALLBACK)))
- return HRTIMER_NORESTART;
- }
-
- /*
- * Some state change occurred while we were in the callback routine.
- */
- for (;;)
- {
- RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pSubTimer->enmState);
- switch (enmState)
- {
- case RTTIMERLNXSTATE_CB_DESTROYING:
- rtTimerLnxCallbackDestroy(pTimer, pSubTimer);
- return HRTIMER_NORESTART;
-
- case RTTIMERLNXSTATE_CB_STOPPING:
- if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_CB_STOPPING))
- return HRTIMER_NORESTART;
- break;
-
- case RTTIMERLNXSTATE_CB_RESTARTING:
- if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_CB_RESTARTING))
- {
- pSubTimer->iTick = 0;
- hrtimer_set_expires(&pSubTimer->u.Hr.LnxTimer, rtTimerLnxNanoToKt(pSubTimer->uNsRestartAt));
- return HRTIMER_RESTART;
- }
- break;
-
- default:
- AssertMsgFailed(("%d\n", enmState));
- return HRTIMER_NORESTART;
- }
- ASMNopPause();
- }
-}
-#endif /* RTTIMER_LINUX_WITH_HRTIMER */
-
-
-/**
- * Timer callback function for standard timers.
- *
- * @param ulUser Address of the sub-timer structure.
- */
-static void rtTimerLinuxStdCallback(unsigned long ulUser)
-{
- PRTTIMERLNXSUBTIMER pSubTimer = (PRTTIMERLNXSUBTIMER)ulUser;
- PRTTIMER pTimer = pSubTimer->pParent;
-
- RTTIMERLNX_LOG(("stdcallback %p\n", pTimer));
- if (RT_UNLIKELY(!rtTimerLnxChangeToCallbackState(pSubTimer)))
- return;
-
-#ifdef CONFIG_SMP
- /*
- * Check for unwanted migration.
- */
- if (pTimer->fAllCpus || pTimer->fSpecificCpu)
- {
- RTCPUID idCpu = RTMpCpuId();
- if (RT_UNLIKELY( pTimer->fAllCpus
- ? (RTCPUID)(pSubTimer - &pTimer->aSubTimers[0]) != idCpu
- : pTimer->idCpu != idCpu))
- {
- rtTimerLnxCallbackHandleMigration(pTimer, pSubTimer);
- return;
- }
- }
-#endif
-
- if (pTimer->u64NanoInterval)
- {
- /*
- * Interval timer, calculate the next timeout.
- *
- * The first time around, we'll re-adjust the u.Std.u64NextTS to
- * try prevent some jittering if we were started at a bad time.
- */
- const uint64_t iTick = ++pSubTimer->iTick;
- uint64_t u64NanoInterval;
- unsigned long cJiffies;
- unsigned long flFlags;
-
- spin_lock_irqsave(&pTimer->ChgIntLock, flFlags);
- u64NanoInterval = pTimer->u64NanoInterval;
- cJiffies = pTimer->cJiffies;
- if (RT_UNLIKELY(pSubTimer->u.Std.fFirstAfterChg))
- {
- pSubTimer->u.Std.fFirstAfterChg = false;
- pSubTimer->u.Std.u64NextTS = RTTimeSystemNanoTS();
- pSubTimer->u.Std.ulNextJiffies = jiffies;
- }
- spin_unlock_irqrestore(&pTimer->ChgIntLock, flFlags);
-
- pSubTimer->u.Std.u64NextTS += u64NanoInterval;
- if (cJiffies)
- {
- pSubTimer->u.Std.ulNextJiffies += cJiffies;
- /* Prevent overflows when the jiffies counter wraps around.
- * Special thanks to Ken Preslan for helping debugging! */
- while (time_before(pSubTimer->u.Std.ulNextJiffies, jiffies))
- {
- pSubTimer->u.Std.ulNextJiffies += cJiffies;
- pSubTimer->u.Std.u64NextTS += u64NanoInterval;
- }
- }
- else
- {
- const uint64_t u64NanoTS = RTTimeSystemNanoTS();
- while (pSubTimer->u.Std.u64NextTS < u64NanoTS)
- pSubTimer->u.Std.u64NextTS += u64NanoInterval;
- pSubTimer->u.Std.ulNextJiffies = jiffies + rtTimerLnxNanoToJiffies(pSubTimer->u.Std.u64NextTS - u64NanoTS);
- }
-
- /*
- * Run the timer and re-arm it unless the state changed .
- * .
- * We must re-arm it afterwards as we're not in a position to undo this .
- * operation if for instance someone stopped or destroyed us while we .
- * were in the callback. (Linux takes care of any races here.)
- */
- pTimer->pfnTimer(pTimer, pTimer->pvUser, iTick);
- if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_CALLBACK)))
- {
-#ifdef CONFIG_SMP
- if (pTimer->fSpecificCpu || pTimer->fAllCpus)
- {
-# if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)
- mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
-# else
- mod_timer_pinned(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
-# endif
- }
- else
-#endif
- mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
- return;
- }
- }
- else
- {
- /*
- * One shot timer, stop it before dispatching it.
- * Allow RTTimerStart as well as RTTimerDestroy to be called from
- * the callback.
- */
- ASMAtomicWriteBool(&pTimer->fSuspended, true);
- pTimer->pfnTimer(pTimer, pTimer->pvUser, ++pSubTimer->iTick);
- if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_CALLBACK)))
- return;
- }
-
- /*
- * Some state change occurred while we were in the callback routine.
- */
- for (;;)
- {
- RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pSubTimer->enmState);
- switch (enmState)
- {
- case RTTIMERLNXSTATE_CB_DESTROYING:
- rtTimerLnxCallbackDestroy(pTimer, pSubTimer);
- return;
-
- case RTTIMERLNXSTATE_CB_STOPPING:
- if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_CB_STOPPING))
- return;
- break;
-
- case RTTIMERLNXSTATE_CB_RESTARTING:
- if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_CB_RESTARTING))
- {
- uint64_t u64NanoTS;
- uint64_t u64NextTS;
- unsigned long flFlags;
-
- spin_lock_irqsave(&pTimer->ChgIntLock, flFlags);
- u64NextTS = pSubTimer->uNsRestartAt;
- u64NanoTS = RTTimeSystemNanoTS();
- pSubTimer->iTick = 0;
- pSubTimer->u.Std.u64NextTS = u64NextTS;
- pSubTimer->u.Std.fFirstAfterChg = true;
- pSubTimer->u.Std.ulNextJiffies = u64NextTS > u64NanoTS
- ? jiffies + rtTimerLnxNanoToJiffies(u64NextTS - u64NanoTS)
- : jiffies;
- spin_unlock_irqrestore(&pTimer->ChgIntLock, flFlags);
-
-#ifdef CONFIG_SMP
- if (pTimer->fSpecificCpu || pTimer->fAllCpus)
- {
-# if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)
- mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
-# else
- mod_timer_pinned(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
-# endif
- }
- else
-#endif
- mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
- return;
- }
- break;
-
- default:
- AssertMsgFailed(("%d\n", enmState));
- return;
- }
- ASMNopPause();
- }
-}
-
-
-#ifdef CONFIG_SMP
-
-/**
- * Per-cpu callback function (RTMpOnAll/RTMpOnSpecific).
- *
- * @param idCpu The current CPU.
- * @param pvUser1 Pointer to the timer.
- * @param pvUser2 Pointer to the argument structure.
- */
-static DECLCALLBACK(void) rtTimerLnxStartAllOnCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2)
-{
- PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
- PRTTIMER pTimer = (PRTTIMER)pvUser1;
- Assert(idCpu < pTimer->cCpus);
- rtTimerLnxStartSubTimer(&pTimer->aSubTimers[idCpu], pArgs->u64Now, pArgs->u64First, true /*fPinned*/, pTimer->fHighRes);
-}
-
-
-/**
- * Worker for RTTimerStart() that takes care of the ugly bits.
- *
- * @returns RTTimerStart() return value.
- * @param pTimer The timer.
- * @param pArgs The argument structure.
- */
-static int rtTimerLnxOmniStart(PRTTIMER pTimer, PRTTIMERLINUXSTARTONCPUARGS pArgs)
-{
- RTCPUID iCpu;
- RTCPUSET OnlineSet;
- RTCPUSET OnlineSet2;
- int rc2;
-
- /*
- * Prepare all the sub-timers for the startup and then flag the timer
- * as a whole as non-suspended, make sure we get them all before
- * clearing fSuspended as the MP handler will be waiting on this
- * should something happen while we're looping.
- */
- RTSpinlockAcquire(pTimer->hSpinlock);
-
- /* Just make it a omni timer restriction that no stop/start races are allowed. */
- for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
- if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) != RTTIMERLNXSTATE_STOPPED)
- {
- RTSpinlockRelease(pTimer->hSpinlock);
- return VERR_TIMER_BUSY;
- }
-
- do
- {
- RTMpGetOnlineSet(&OnlineSet);
- for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
- {
- Assert(pTimer->aSubTimers[iCpu].enmState != RTTIMERLNXSTATE_MP_STOPPING);
- rtTimerLnxSetState(&pTimer->aSubTimers[iCpu].enmState,
- RTCpuSetIsMember(&OnlineSet, iCpu)
- ? RTTIMERLNXSTATE_STARTING
- : RTTIMERLNXSTATE_STOPPED);
- }
- } while (!RTCpuSetIsEqual(&OnlineSet, RTMpGetOnlineSet(&OnlineSet2)));
-
- ASMAtomicWriteBool(&pTimer->fSuspended, false);
-
- RTSpinlockRelease(pTimer->hSpinlock);
-
- /*
- * Start them (can't find any exported function that allows me to
- * do this without the cross calls).
- */
- pArgs->u64Now = RTTimeSystemNanoTS();
- rc2 = RTMpOnAll(rtTimerLnxStartAllOnCpu, pTimer, pArgs);
- AssertRC(rc2); /* screw this if it fails. */
-
- /*
- * Reset the sub-timers who didn't start up (ALL CPUs case).
- */
- RTSpinlockAcquire(pTimer->hSpinlock);
-
- for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
- if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_STARTING))
- {
- /** @todo very odd case for a rainy day. Cpus that temporarily went offline while
- * we were between calls needs to nudged as the MP handler will ignore events for
- * them because of the STARTING state. This is an extremely unlikely case - not that
- * that means anything in my experience... ;-) */
- RTTIMERLNX_LOG(("what!? iCpu=%u -> didn't start\n", iCpu));
- }
-
- RTSpinlockRelease(pTimer->hSpinlock);
-
- return VINF_SUCCESS;
-}
-
-
-/**
- * Worker for RTTimerStop() that takes care of the ugly SMP bits.
- *
- * @returns true if there was any active callbacks, false if not.
- * @param pTimer The timer (valid).
- * @param fForDestroy Whether this is for RTTimerDestroy or not.
- */
-static bool rtTimerLnxOmniStop(PRTTIMER pTimer, bool fForDestroy)
-{
- bool fActiveCallbacks = false;
- RTCPUID iCpu;
- RTTIMERLNXSTATE enmState;
-
-
- /*
- * Mark the timer as suspended and flag all timers as stopping, except
- * for those being stopped by an MP event.
- */
- RTSpinlockAcquire(pTimer->hSpinlock);
-
- ASMAtomicWriteBool(&pTimer->fSuspended, true);
- for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
- {
- for (;;)
- {
- enmState = rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState);
- if ( enmState == RTTIMERLNXSTATE_STOPPED
- || enmState == RTTIMERLNXSTATE_MP_STOPPING)
- break;
- if ( enmState == RTTIMERLNXSTATE_CALLBACK
- || enmState == RTTIMERLNXSTATE_CB_STOPPING
- || enmState == RTTIMERLNXSTATE_CB_RESTARTING)
- {
- Assert(enmState != RTTIMERLNXSTATE_CB_STOPPING || fForDestroy);
- if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState,
- !fForDestroy ? RTTIMERLNXSTATE_CB_STOPPING : RTTIMERLNXSTATE_CB_DESTROYING,
- enmState))
- {
- fActiveCallbacks = true;
- break;
- }
- }
- else
- {
- Assert(enmState == RTTIMERLNXSTATE_ACTIVE);
- if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPING, enmState))
- break;
- }
- ASMNopPause();
- }
- }
-
- RTSpinlockRelease(pTimer->hSpinlock);
-
- /*
- * Do the actual stopping. Fortunately, this doesn't require any IPIs.
- * Unfortunately it cannot be done synchronously.
- */
- for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
- if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) == RTTIMERLNXSTATE_STOPPING)
- rtTimerLnxStopSubTimer(&pTimer->aSubTimers[iCpu], pTimer->fHighRes);
-
- return fActiveCallbacks;
-}
-
-
-/**
- * Per-cpu callback function (RTMpOnSpecific) used by rtTimerLinuxMpEvent()
- * to start a sub-timer on a cpu that just have come online.
- *
- * @param idCpu The current CPU.
- * @param pvUser1 Pointer to the timer.
- * @param pvUser2 Pointer to the argument structure.
- */
-static DECLCALLBACK(void) rtTimerLinuxMpStartOnCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2)
-{
- PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
- PRTTIMER pTimer = (PRTTIMER)pvUser1;
- RTSPINLOCK hSpinlock;
- Assert(idCpu < pTimer->cCpus);
-
- /*
- * We have to be kind of careful here as we might be racing RTTimerStop
- * (and/or RTTimerDestroy, thus the paranoia.
- */
- hSpinlock = pTimer->hSpinlock;
- if ( hSpinlock != NIL_RTSPINLOCK
- && pTimer->u32Magic == RTTIMER_MAGIC)
- {
- RTSpinlockAcquire(hSpinlock);
-
- if ( !ASMAtomicUoReadBool(&pTimer->fSuspended)
- && pTimer->u32Magic == RTTIMER_MAGIC)
- {
- /* We're sane and the timer is not suspended yet. */
- PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
- if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
- rtTimerLnxStartSubTimer(pSubTimer, pArgs->u64Now, pArgs->u64First, true /*fPinned*/, pTimer->fHighRes);
- }
-
- RTSpinlockRelease(hSpinlock);
- }
-}
-
-
-/**
- * MP event notification callback.
- *
- * @param enmEvent The event.
- * @param idCpu The cpu it applies to.
- * @param pvUser The timer.
- */
-static DECLCALLBACK(void) rtTimerLinuxMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser)
-{
- PRTTIMER pTimer = (PRTTIMER)pvUser;
- PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
- RTSPINLOCK hSpinlock;
-
- Assert(idCpu < pTimer->cCpus);
-
- /*
- * Some initial paranoia.
- */
- if (pTimer->u32Magic != RTTIMER_MAGIC)
- return;
- hSpinlock = pTimer->hSpinlock;
- if (hSpinlock == NIL_RTSPINLOCK)
- return;
-
- RTSpinlockAcquire(hSpinlock);
-
- /* Is it active? */
- if ( !ASMAtomicUoReadBool(&pTimer->fSuspended)
- && pTimer->u32Magic == RTTIMER_MAGIC)
- {
- switch (enmEvent)
- {
- /*
- * Try do it without leaving the spin lock, but if we have to, retake it
- * when we're on the right cpu.
- */
- case RTMPEVENT_ONLINE:
- if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
- {
- RTTIMERLINUXSTARTONCPUARGS Args;
- Args.u64Now = RTTimeSystemNanoTS();
- Args.u64First = 0;
-
- if (RTMpCpuId() == idCpu)
- rtTimerLnxStartSubTimer(pSubTimer, Args.u64Now, Args.u64First, true /*fPinned*/, pTimer->fHighRes);
- else
- {
- rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED); /* we'll recheck it. */
- RTSpinlockRelease(hSpinlock);
-
- RTMpOnSpecific(idCpu, rtTimerLinuxMpStartOnCpu, pTimer, &Args);
- return; /* we've left the spinlock */
- }
- }
- break;
-
- /*
- * The CPU is (going) offline, make sure the sub-timer is stopped.
- *
- * Linux will migrate it to a different CPU, but we don't want this. The
- * timer function is checking for this.
- */
- case RTMPEVENT_OFFLINE:
- {
- RTTIMERLNXSTATE enmState;
- while ( (enmState = rtTimerLnxGetState(&pSubTimer->enmState)) == RTTIMERLNXSTATE_ACTIVE
- || enmState == RTTIMERLNXSTATE_CALLBACK
- || enmState == RTTIMERLNXSTATE_CB_RESTARTING)
- {
- if (enmState == RTTIMERLNXSTATE_ACTIVE)
- {
- if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STOPPING, RTTIMERLNXSTATE_ACTIVE))
- {
- RTSpinlockRelease(hSpinlock);
-
- rtTimerLnxStopSubTimer(pSubTimer, pTimer->fHighRes);
- return; /* we've left the spinlock */
- }
- }
- else if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_CB_STOPPING, enmState))
- break;
-
- /* State not stable, try again. */
- ASMNopPause();
- }
- break;
- }
- }
- }
-
- RTSpinlockRelease(hSpinlock);
-}
-
-#endif /* CONFIG_SMP */
-
-
-/**
- * Callback function use by RTTimerStart via RTMpOnSpecific to start a timer
- * running on a specific CPU.
- *
- * @param idCpu The current CPU.
- * @param pvUser1 Pointer to the timer.
- * @param pvUser2 Pointer to the argument structure.
- */
-static DECLCALLBACK(void) rtTimerLnxStartOnSpecificCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2)
-{
- PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
- PRTTIMER pTimer = (PRTTIMER)pvUser1;
- RT_NOREF_PV(idCpu);
- rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], pArgs->u64Now, pArgs->u64First, true /*fPinned*/, pTimer->fHighRes);
-}
-
-
-RTDECL(int) RTTimerStart(PRTTIMER pTimer, uint64_t u64First)
-{
- RTTIMERLINUXSTARTONCPUARGS Args;
- int rc2;
- IPRT_LINUX_SAVE_EFL_AC();
-
- /*
- * Validate.
- */
- AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
- AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
-
- if (!ASMAtomicUoReadBool(&pTimer->fSuspended))
- return VERR_TIMER_ACTIVE;
- RTTIMERLNX_LOG(("start %p cCpus=%d\n", pTimer, pTimer->cCpus));
-
- Args.u64First = u64First;
-#ifdef CONFIG_SMP
- /*
- * Omni timer?
- */
- if (pTimer->fAllCpus)
- {
- rc2 = rtTimerLnxOmniStart(pTimer, &Args);
- IPRT_LINUX_RESTORE_EFL_AC();
- return rc2;
- }
-#endif
-
- /*
- * Simple timer - Pretty straight forward if it wasn't for restarting.
- */
- Args.u64Now = RTTimeSystemNanoTS();
- ASMAtomicWriteU64(&pTimer->aSubTimers[0].uNsRestartAt, Args.u64Now + u64First);
- for (;;)
- {
- RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pTimer->aSubTimers[0].enmState);
- switch (enmState)
- {
- case RTTIMERLNXSTATE_STOPPED:
- if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STARTING, RTTIMERLNXSTATE_STOPPED))
- {
- ASMAtomicWriteBool(&pTimer->fSuspended, false);
- if (!pTimer->fSpecificCpu)
- rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], Args.u64Now, Args.u64First,
- false /*fPinned*/, pTimer->fHighRes);
- else
- {
- rc2 = RTMpOnSpecific(pTimer->idCpu, rtTimerLnxStartOnSpecificCpu, pTimer, &Args);
- if (RT_FAILURE(rc2))
- {
- /* Suspend it, the cpu id is probably invalid or offline. */
- ASMAtomicWriteBool(&pTimer->fSuspended, true);
- rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPED);
- return rc2;
- }
- }
- IPRT_LINUX_RESTORE_EFL_AC();
- return VINF_SUCCESS;
- }
- break;
-
- case RTTIMERLNXSTATE_CALLBACK:
- case RTTIMERLNXSTATE_CB_STOPPING:
- if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_CB_RESTARTING, enmState))
- {
- ASMAtomicWriteBool(&pTimer->fSuspended, false);
- IPRT_LINUX_RESTORE_EFL_AC();
- return VINF_SUCCESS;
- }
- break;
-
- default:
- AssertMsgFailed(("%d\n", enmState));
- IPRT_LINUX_RESTORE_EFL_AC();
- return VERR_INTERNAL_ERROR_4;
- }
- ASMNopPause();
- }
-}
-RT_EXPORT_SYMBOL(RTTimerStart);
-
-
-/**
- * Common worker for RTTimerStop and RTTimerDestroy.
- *
- * @returns true if there was any active callbacks, false if not.
- * @param pTimer The timer to stop.
- * @param fForDestroy Whether it's RTTimerDestroy calling or not.
- */
-static bool rtTimerLnxStop(PRTTIMER pTimer, bool fForDestroy)
-{
- RTTIMERLNX_LOG(("lnxstop %p %d\n", pTimer, fForDestroy));
-#ifdef CONFIG_SMP
- /*
- * Omni timer?
- */
- if (pTimer->fAllCpus)
- return rtTimerLnxOmniStop(pTimer, fForDestroy);
-#endif
-
- /*
- * Simple timer.
- */
- ASMAtomicWriteBool(&pTimer->fSuspended, true);
- for (;;)
- {
- RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pTimer->aSubTimers[0].enmState);
- switch (enmState)
- {
- case RTTIMERLNXSTATE_ACTIVE:
- if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPING, RTTIMERLNXSTATE_ACTIVE))
- {
- rtTimerLnxStopSubTimer(&pTimer->aSubTimers[0], pTimer->fHighRes);
- return false;
- }
- break;
-
- case RTTIMERLNXSTATE_CALLBACK:
- case RTTIMERLNXSTATE_CB_RESTARTING:
- case RTTIMERLNXSTATE_CB_STOPPING:
- Assert(enmState != RTTIMERLNXSTATE_CB_STOPPING || fForDestroy);
- if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[0].enmState,
- !fForDestroy ? RTTIMERLNXSTATE_CB_STOPPING : RTTIMERLNXSTATE_CB_DESTROYING,
- enmState))
- return true;
- break;
-
- case RTTIMERLNXSTATE_STOPPED:
- return VINF_SUCCESS;
-
- case RTTIMERLNXSTATE_CB_DESTROYING:
- AssertMsgFailed(("enmState=%d pTimer=%p\n", enmState, pTimer));
- return true;
-
- default:
- case RTTIMERLNXSTATE_STARTING:
- case RTTIMERLNXSTATE_MP_STARTING:
- case RTTIMERLNXSTATE_STOPPING:
- case RTTIMERLNXSTATE_MP_STOPPING:
- AssertMsgFailed(("enmState=%d pTimer=%p\n", enmState, pTimer));
- return false;
- }
-
- /* State not stable, try again. */
- ASMNopPause();
- }
-}
-
-
-RTDECL(int) RTTimerStop(PRTTIMER pTimer)
-{
- /*
- * Validate.
- */
- IPRT_LINUX_SAVE_EFL_AC();
- AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
- AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
- RTTIMERLNX_LOG(("stop %p\n", pTimer));
-
- if (ASMAtomicUoReadBool(&pTimer->fSuspended))
- return VERR_TIMER_SUSPENDED;
-
- rtTimerLnxStop(pTimer, false /*fForDestroy*/);
-
- IPRT_LINUX_RESTORE_EFL_AC();
- return VINF_SUCCESS;
-}
-RT_EXPORT_SYMBOL(RTTimerStop);
-
-
-RTDECL(int) RTTimerChangeInterval(PRTTIMER pTimer, uint64_t u64NanoInterval)
-{
- unsigned long cJiffies;
- unsigned long flFlags;
- IPRT_LINUX_SAVE_EFL_AC();
-
- /*
- * Validate.
- */
- AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
- AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
- AssertReturn(u64NanoInterval, VERR_INVALID_PARAMETER);
- AssertReturn(u64NanoInterval < UINT64_MAX / 8, VERR_INVALID_PARAMETER);
- AssertReturn(pTimer->u64NanoInterval, VERR_INVALID_STATE);
- RTTIMERLNX_LOG(("change %p %llu\n", pTimer, u64NanoInterval));
-
-#ifdef RTTIMER_LINUX_WITH_HRTIMER
- /*
- * For the high resolution timers it is easy since we don't care so much
- * about when it is applied to the sub-timers.
- */
- if (pTimer->fHighRes)
- {
- ASMAtomicWriteU64(&pTimer->u64NanoInterval, u64NanoInterval);
- IPRT_LINUX_RESTORE_EFL_AC();
- return VINF_SUCCESS;
- }
-#endif
-
- /*
- * Standard timers have a bit more complicated way of calculating
- * their interval and such. So, forget omni timers for now.
- */
- if (pTimer->cCpus > 1)
- return VERR_NOT_SUPPORTED;
-
- cJiffies = u64NanoInterval / RTTimerGetSystemGranularity();
- if (cJiffies * RTTimerGetSystemGranularity() != u64NanoInterval)
- cJiffies = 0;
-
- spin_lock_irqsave(&pTimer->ChgIntLock, flFlags);
- pTimer->aSubTimers[0].u.Std.fFirstAfterChg = true;
- pTimer->cJiffies = cJiffies;
- ASMAtomicWriteU64(&pTimer->u64NanoInterval, u64NanoInterval);
- spin_unlock_irqrestore(&pTimer->ChgIntLock, flFlags);
- IPRT_LINUX_RESTORE_EFL_AC();
- return VINF_SUCCESS;
-}
-RT_EXPORT_SYMBOL(RTTimerChangeInterval);
-
-
-RTDECL(int) RTTimerDestroy(PRTTIMER pTimer)
-{
- bool fCanDestroy;
- IPRT_LINUX_SAVE_EFL_AC();
-
- /*
- * Validate. It's ok to pass NULL pointer.
- */
- if (pTimer == /*NIL_RTTIMER*/ NULL)
- return VINF_SUCCESS;
- AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
- AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
- RTTIMERLNX_LOG(("destroy %p\n", pTimer));
-/** @todo We should invalidate the magic here! */
-
- /*
- * Stop the timer if it's still active, then destroy it if we can.
- */
- if (!ASMAtomicUoReadBool(&pTimer->fSuspended))
- fCanDestroy = rtTimerLnxStop(pTimer, true /*fForDestroy*/);
- else
- {
- uint32_t iCpu = pTimer->cCpus;
- if (pTimer->cCpus > 1)
- RTSpinlockAcquire(pTimer->hSpinlock);
-
- fCanDestroy = true;
- while (iCpu-- > 0)
- {
- for (;;)
- {
- RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState);
- switch (enmState)
- {
- case RTTIMERLNXSTATE_CALLBACK:
- case RTTIMERLNXSTATE_CB_RESTARTING:
- case RTTIMERLNXSTATE_CB_STOPPING:
- if (!rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_CB_DESTROYING, enmState))
- continue;
- fCanDestroy = false;
- break;
-
- case RTTIMERLNXSTATE_CB_DESTROYING:
- AssertMsgFailed(("%d\n", enmState));
- fCanDestroy = false;
- break;
- default:
- break;
- }
- break;
- }
- }
-
- if (pTimer->cCpus > 1)
- RTSpinlockRelease(pTimer->hSpinlock);
- }
-
- if (fCanDestroy)
- {
- /* For paranoid reasons, defer actually destroying the semaphore when
- in atomic or interrupt context. */
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 32)
- if (in_atomic() || in_interrupt())
-#else
- if (in_interrupt())
-#endif
- rtR0LnxWorkqueuePush(&pTimer->DtorWorkqueueItem, rtTimerLnxDestroyDeferred);
- else
- rtTimerLnxDestroyIt(pTimer);
- }
-
- IPRT_LINUX_RESTORE_EFL_AC();
- return VINF_SUCCESS;
-}
-RT_EXPORT_SYMBOL(RTTimerDestroy);
-
-
-RTDECL(int) RTTimerCreateEx(PRTTIMER *ppTimer, uint64_t u64NanoInterval, uint32_t fFlags, PFNRTTIMER pfnTimer, void *pvUser)
-{
- PRTTIMER pTimer;
- RTCPUID iCpu;
- unsigned cCpus;
- int rc;
- IPRT_LINUX_SAVE_EFL_AC();
-
- rtR0LnxWorkqueueFlush(); /* for 2.4 */
- *ppTimer = NULL;
-
- /*
- * Validate flags.
- */
- if (!RTTIMER_FLAGS_ARE_VALID(fFlags))
- {
- IPRT_LINUX_RESTORE_EFL_AC();
- return VERR_INVALID_PARAMETER;
- }
- if ( (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC)
- && (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL
- && !RTMpIsCpuPossible(RTMpCpuIdFromSetIndex(fFlags & RTTIMER_FLAGS_CPU_MASK)))
- {
- IPRT_LINUX_RESTORE_EFL_AC();
- return VERR_CPU_NOT_FOUND;
- }
-
- /*
- * Allocate the timer handler.
- */
- cCpus = 1;
-#ifdef CONFIG_SMP
- if ((fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL)
- {
- cCpus = RTMpGetMaxCpuId() + 1;
- Assert(cCpus <= RTCPUSET_MAX_CPUS); /* On linux we have a 1:1 relationship between cpuid and set index. */
- AssertReturnStmt(u64NanoInterval, IPRT_LINUX_RESTORE_EFL_AC(), VERR_NOT_IMPLEMENTED); /* We don't implement single shot on all cpus, sorry. */
- }
-#endif
-
- rc = RTMemAllocEx(RT_OFFSETOF(RTTIMER, aSubTimers[cCpus]), 0,
- RTMEMALLOCEX_FLAGS_ZEROED | RTMEMALLOCEX_FLAGS_ANY_CTX_FREE, (void **)&pTimer);
- if (RT_FAILURE(rc))
- {
- IPRT_LINUX_RESTORE_EFL_AC();
- return rc;
- }
-
- /*
- * Initialize it.
- */
- pTimer->u32Magic = RTTIMER_MAGIC;
- pTimer->hSpinlock = NIL_RTSPINLOCK;
- pTimer->fSuspended = true;
- pTimer->fHighRes = !!(fFlags & RTTIMER_FLAGS_HIGH_RES);
-#ifdef CONFIG_SMP
- pTimer->fSpecificCpu = (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC) && (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL;
- pTimer->fAllCpus = (fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL;
- pTimer->idCpu = pTimer->fSpecificCpu
- ? RTMpCpuIdFromSetIndex(fFlags & RTTIMER_FLAGS_CPU_MASK)
- : NIL_RTCPUID;
-#else
- pTimer->fSpecificCpu = !!(fFlags & RTTIMER_FLAGS_CPU_SPECIFIC);
- pTimer->idCpu = RTMpCpuId();
-#endif
- pTimer->cCpus = cCpus;
- pTimer->pfnTimer = pfnTimer;
- pTimer->pvUser = pvUser;
- pTimer->u64NanoInterval = u64NanoInterval;
- pTimer->cJiffies = u64NanoInterval / RTTimerGetSystemGranularity();
- if (pTimer->cJiffies * RTTimerGetSystemGranularity() != u64NanoInterval)
- pTimer->cJiffies = 0;
- spin_lock_init(&pTimer->ChgIntLock);
-
- for (iCpu = 0; iCpu < cCpus; iCpu++)
- {
-#ifdef RTTIMER_LINUX_WITH_HRTIMER
- if (pTimer->fHighRes)
- {
- hrtimer_init(&pTimer->aSubTimers[iCpu].u.Hr.LnxTimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
- pTimer->aSubTimers[iCpu].u.Hr.LnxTimer.function = rtTimerLinuxHrCallback;
- }
- else
-#endif
- {
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)
- init_timer_pinned(&pTimer->aSubTimers[iCpu].u.Std.LnxTimer);
-#else
- init_timer(&pTimer->aSubTimers[iCpu].u.Std.LnxTimer);
-#endif
- pTimer->aSubTimers[iCpu].u.Std.LnxTimer.data = (unsigned long)&pTimer->aSubTimers[iCpu];
- pTimer->aSubTimers[iCpu].u.Std.LnxTimer.function = rtTimerLinuxStdCallback;
- pTimer->aSubTimers[iCpu].u.Std.LnxTimer.expires = jiffies;
- pTimer->aSubTimers[iCpu].u.Std.u64NextTS = 0;
- }
- pTimer->aSubTimers[iCpu].iTick = 0;
- pTimer->aSubTimers[iCpu].pParent = pTimer;
- pTimer->aSubTimers[iCpu].enmState = RTTIMERLNXSTATE_STOPPED;
- }
-
-#ifdef CONFIG_SMP
- /*
- * If this is running on ALL cpus, we'll have to register a callback
- * for MP events (so timers can be started/stopped on cpus going
- * online/offline). We also create the spinlock for synchronizing
- * stop/start/mp-event.
- */
- if (cCpus > 1)
- {
- int rc = RTSpinlockCreate(&pTimer->hSpinlock, RTSPINLOCK_FLAGS_INTERRUPT_SAFE, "RTTimerLnx");
- if (RT_SUCCESS(rc))
- rc = RTMpNotificationRegister(rtTimerLinuxMpEvent, pTimer);
- else
- pTimer->hSpinlock = NIL_RTSPINLOCK;
- if (RT_FAILURE(rc))
- {
- RTTimerDestroy(pTimer);
- IPRT_LINUX_RESTORE_EFL_AC();
- return rc;
- }
- }
-#endif /* CONFIG_SMP */
-
- RTTIMERLNX_LOG(("create %p hires=%d fFlags=%#x cCpus=%u\n", pTimer, pTimer->fHighRes, fFlags, cCpus));
- *ppTimer = pTimer;
- IPRT_LINUX_RESTORE_EFL_AC();
- return VINF_SUCCESS;
-}
-RT_EXPORT_SYMBOL(RTTimerCreateEx);
-
-
-RTDECL(uint32_t) RTTimerGetSystemGranularity(void)
-{
-#if 0 /** @todo Not sure if this is what we want or not... Add new API for
- * querying the resolution of the high res timers? */
- struct timespec Ts;
- int rc;
- IPRT_LINUX_SAVE_EFL_AC();
- rc = hrtimer_get_res(CLOCK_MONOTONIC, &Ts);
- IPRT_LINUX_RESTORE_EFL_AC();
- if (!rc)
- {
- Assert(!Ts.tv_sec);
- return Ts.tv_nsec;
- }
-#endif
- return RT_NS_1SEC / HZ; /* ns */
-}
-RT_EXPORT_SYMBOL(RTTimerGetSystemGranularity);
-
-
-RTDECL(int) RTTimerRequestSystemGranularity(uint32_t u32Request, uint32_t *pu32Granted)
-{
- RT_NOREF_PV(u32Request); RT_NOREF_PV(*pu32Granted);
- return VERR_NOT_SUPPORTED;
-}
-RT_EXPORT_SYMBOL(RTTimerRequestSystemGranularity);
-
-
-RTDECL(int) RTTimerReleaseSystemGranularity(uint32_t u32Granted)
-{
- RT_NOREF_PV(u32Granted);
- return VERR_NOT_SUPPORTED;
-}
-RT_EXPORT_SYMBOL(RTTimerReleaseSystemGranularity);
-
-
-RTDECL(bool) RTTimerCanDoHighResolution(void)
-{
-#ifdef RTTIMER_LINUX_WITH_HRTIMER
- return true;
-#else
- return false;
-#endif
-}
-RT_EXPORT_SYMBOL(RTTimerCanDoHighResolution);
-
projects / mirror_ubuntu-bionic-kernel.git / blobdiff