[mirror_ubuntu-bionic-kernel.git] / kernel / time / tick-broadcast.c

/*
 * linux/kernel/time/tick-broadcast.c
 *
 * This file contains functions which emulate a local clock-event
 * device via a broadcast event source.
 *
 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
 * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
 *
 * This code is licenced under the GPL version 2. For details see
 * kernel-base/COPYING.
 */
#include <linux/cpu.h>
#include <linux/err.h>
#include <linux/hrtimer.h>
#include <linux/interrupt.h>
#include <linux/percpu.h>
#include <linux/profile.h>
#include <linux/sched.h>

#include "tick-internal.h"

/*
 * Broadcast support for broken x86 hardware, where the local apic
 * timer stops in C3 state.
 */

static struct tick_device tick_broadcast_device;
/* FIXME: Use cpumask_var_t. */
static DECLARE_BITMAP(tick_broadcast_mask, NR_CPUS);
static DECLARE_BITMAP(tmpmask, NR_CPUS);
static DEFINE_RAW_SPINLOCK(tick_broadcast_lock);
static int tick_broadcast_force;

#ifdef CONFIG_TICK_ONESHOT
static void tick_broadcast_clear_oneshot(int cpu);
#else
static inline void tick_broadcast_clear_oneshot(int cpu) { }
#endif

/*
 * Debugging: see timer_list.c
 */
struct tick_device *tick_get_broadcast_device(void)
{
	return &tick_broadcast_device;
}

struct cpumask *tick_get_broadcast_mask(void)
{
	return to_cpumask(tick_broadcast_mask);
}

/*
 * Start the device in periodic mode
 */
static void tick_broadcast_start_periodic(struct clock_event_device *bc)
{
	if (bc)
		tick_setup_periodic(bc, 1);
}

/*
 * Check, if the device can be utilized as broadcast device:
 */
int tick_check_broadcast_device(struct clock_event_device *dev)
{
	if ((tick_broadcast_device.evtdev &&
	     tick_broadcast_device.evtdev->rating >= dev->rating) ||
	     (dev->features & CLOCK_EVT_FEAT_C3STOP))
		return 0;

	clockevents_exchange_device(NULL, dev);
	tick_broadcast_device.evtdev = dev;
	if (!cpumask_empty(tick_get_broadcast_mask()))
		tick_broadcast_start_periodic(dev);
	return 1;
}

/*
 * Check, if the device is the broadcast device
 */
int tick_is_broadcast_device(struct clock_event_device *dev)
{
	return (dev && tick_broadcast_device.evtdev == dev);
}

/*
 * Check, if the device is disfunctional and a place holder, which
 * needs to be handled by the broadcast device.
 */
int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
{
	unsigned long flags;
	int ret = 0;

	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);

	/*
	 * Devices might be registered with both periodic and oneshot
	 * mode disabled. This signals, that the device needs to be
	 * operated from the broadcast device and is a placeholder for
	 * the cpu local device.
	 */
	if (!tick_device_is_functional(dev)) {
		dev->event_handler = tick_handle_periodic;
		cpumask_set_cpu(cpu, tick_get_broadcast_mask());
		tick_broadcast_start_periodic(tick_broadcast_device.evtdev);
		ret = 1;
	} else {
		/*
		 * When the new device is not affected by the stop
		 * feature and the cpu is marked in the broadcast mask
		 * then clear the broadcast bit.
		 */
		if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) {
			int cpu = smp_processor_id();

			cpumask_clear_cpu(cpu, tick_get_broadcast_mask());
			tick_broadcast_clear_oneshot(cpu);
		}
	}
	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
	return ret;
}

/*
 * Broadcast the event to the cpus, which are set in the mask (mangled).
 */
static void tick_do_broadcast(struct cpumask *mask)
{
	int cpu = smp_processor_id();
	struct tick_device *td;

	/*
	 * Check, if the current cpu is in the mask
	 */
	if (cpumask_test_cpu(cpu, mask)) {
		cpumask_clear_cpu(cpu, mask);
		td = &per_cpu(tick_cpu_device, cpu);
		td->evtdev->event_handler(td->evtdev);
	}

	if (!cpumask_empty(mask)) {
		/*
		 * It might be necessary to actually check whether the devices
		 * have different broadcast functions. For now, just use the
		 * one of the first device. This works as long as we have this
		 * misfeature only on x86 (lapic)
		 */
		td = &per_cpu(tick_cpu_device, cpumask_first(mask));
		td->evtdev->broadcast(mask);
	}
}

/*
 * Periodic broadcast:
 * - invoke the broadcast handlers
 */
static void tick_do_periodic_broadcast(void)
{
	raw_spin_lock(&tick_broadcast_lock);

	cpumask_and(to_cpumask(tmpmask),
		    cpu_online_mask, tick_get_broadcast_mask());
	tick_do_broadcast(to_cpumask(tmpmask));

	raw_spin_unlock(&tick_broadcast_lock);
}

/*
 * Event handler for periodic broadcast ticks
 */
static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
{
	ktime_t next;

	tick_do_periodic_broadcast();

	/*
	 * The device is in periodic mode. No reprogramming necessary:
	 */
	if (dev->mode == CLOCK_EVT_MODE_PERIODIC)
		return;

	/*
	 * Setup the next period for devices, which do not have
	 * periodic mode. We read dev->next_event first and add to it
	 * when the event already expired. clockevents_program_event()
	 * sets dev->next_event only when the event is really
	 * programmed to the device.
	 */
	for (next = dev->next_event; ;) {
		next = ktime_add(next, tick_period);

		if (!clockevents_program_event(dev, next, ktime_get()))
			return;
		tick_do_periodic_broadcast();
	}
}

/*
 * Powerstate information: The system enters/leaves a state, where
 * affected devices might stop
 */
static void tick_do_broadcast_on_off(unsigned long *reason)
{
	struct clock_event_device *bc, *dev;
	struct tick_device *td;
	unsigned long flags;
	int cpu, bc_stopped;

	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);

	cpu = smp_processor_id();
	td = &per_cpu(tick_cpu_device, cpu);
	dev = td->evtdev;
	bc = tick_broadcast_device.evtdev;

	/*
	 * Is the device not affected by the powerstate ?
	 */
	if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP))
		goto out;

	if (!tick_device_is_functional(dev))
		goto out;

	bc_stopped = cpumask_empty(tick_get_broadcast_mask());

	switch (*reason) {
	case CLOCK_EVT_NOTIFY_BROADCAST_ON:
	case CLOCK_EVT_NOTIFY_BROADCAST_FORCE:
		if (!cpumask_test_cpu(cpu, tick_get_broadcast_mask())) {
			cpumask_set_cpu(cpu, tick_get_broadcast_mask());
			if (tick_broadcast_device.mode ==
			    TICKDEV_MODE_PERIODIC)
				clockevents_shutdown(dev);
		}
		if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_FORCE)
			tick_broadcast_force = 1;
		break;
	case CLOCK_EVT_NOTIFY_BROADCAST_OFF:
		if (!tick_broadcast_force &&
		    cpumask_test_cpu(cpu, tick_get_broadcast_mask())) {
			cpumask_clear_cpu(cpu, tick_get_broadcast_mask());
			if (tick_broadcast_device.mode ==
			    TICKDEV_MODE_PERIODIC)
				tick_setup_periodic(dev, 0);
		}
		break;
	}

	if (cpumask_empty(tick_get_broadcast_mask())) {
		if (!bc_stopped)
			clockevents_shutdown(bc);
	} else if (bc_stopped) {
		if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
			tick_broadcast_start_periodic(bc);
		else
			tick_broadcast_setup_oneshot(bc);
	}
out:
	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
}

/*
 * Powerstate information: The system enters/leaves a state, where
 * affected devices might stop.
 */
void tick_broadcast_on_off(unsigned long reason, int *oncpu)
{
	if (!cpumask_test_cpu(*oncpu, cpu_online_mask))
		printk(KERN_ERR "tick-broadcast: ignoring broadcast for "
		       "offline CPU #%d\n", *oncpu);
	else
		tick_do_broadcast_on_off(&reason);
}

/*
 * Set the periodic handler depending on broadcast on/off
 */
void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast)
{
	if (!broadcast)
		dev->event_handler = tick_handle_periodic;
	else
		dev->event_handler = tick_handle_periodic_broadcast;
}

/*
 * Remove a CPU from broadcasting
 */
void tick_shutdown_broadcast(unsigned int *cpup)
{
	struct clock_event_device *bc;
	unsigned long flags;
	unsigned int cpu = *cpup;

	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);

	bc = tick_broadcast_device.evtdev;
	cpumask_clear_cpu(cpu, tick_get_broadcast_mask());

	if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
		if (bc && cpumask_empty(tick_get_broadcast_mask()))
			clockevents_shutdown(bc);
	}

	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
}

void tick_suspend_broadcast(void)
{
	struct clock_event_device *bc;
	unsigned long flags;

	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);

	bc = tick_broadcast_device.evtdev;
	if (bc)
		clockevents_shutdown(bc);

	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
}

int tick_resume_broadcast(void)
{
	struct clock_event_device *bc;
	unsigned long flags;
	int broadcast = 0;

	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);

	bc = tick_broadcast_device.evtdev;

	if (bc) {
		clockevents_set_mode(bc, CLOCK_EVT_MODE_RESUME);

		switch (tick_broadcast_device.mode) {
		case TICKDEV_MODE_PERIODIC:
			if (!cpumask_empty(tick_get_broadcast_mask()))
				tick_broadcast_start_periodic(bc);
			broadcast = cpumask_test_cpu(smp_processor_id(),
						     tick_get_broadcast_mask());
			break;
		case TICKDEV_MODE_ONESHOT:
			broadcast = tick_resume_broadcast_oneshot(bc);
			break;
		}
	}
	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);

	return broadcast;
}


#ifdef CONFIG_TICK_ONESHOT

/* FIXME: use cpumask_var_t. */
static DECLARE_BITMAP(tick_broadcast_oneshot_mask, NR_CPUS);

/*
 * Exposed for debugging: see timer_list.c
 */
struct cpumask *tick_get_broadcast_oneshot_mask(void)
{
	return to_cpumask(tick_broadcast_oneshot_mask);
}

static int tick_broadcast_set_event(ktime_t expires, int force)
{
	struct clock_event_device *bc = tick_broadcast_device.evtdev;

	return tick_dev_program_event(bc, expires, force);
}

int tick_resume_broadcast_oneshot(struct clock_event_device *bc)
{
	clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
	return 0;
}

/*
 * Called from irq_enter() when idle was interrupted to reenable the
 * per cpu device.
 */
void tick_check_oneshot_broadcast(int cpu)
{
	if (cpumask_test_cpu(cpu, to_cpumask(tick_broadcast_oneshot_mask))) {
		struct tick_device *td = &per_cpu(tick_cpu_device, cpu);

		clockevents_set_mode(td->evtdev, CLOCK_EVT_MODE_ONESHOT);
	}
}

/*
 * Handle oneshot mode broadcasting
 */
static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
{
	struct tick_device *td;
	ktime_t now, next_event;
	int cpu;

	raw_spin_lock(&tick_broadcast_lock);
again:
	dev->next_event.tv64 = KTIME_MAX;
	next_event.tv64 = KTIME_MAX;
	cpumask_clear(to_cpumask(tmpmask));
	now = ktime_get();
	/* Find all expired events */
	for_each_cpu(cpu, tick_get_broadcast_oneshot_mask()) {
		td = &per_cpu(tick_cpu_device, cpu);
		if (td->evtdev->next_event.tv64 <= now.tv64)
			cpumask_set_cpu(cpu, to_cpumask(tmpmask));
		else if (td->evtdev->next_event.tv64 < next_event.tv64)
			next_event.tv64 = td->evtdev->next_event.tv64;
	}

	/*
	 * Wakeup the cpus which have an expired event.
	 */
	tick_do_broadcast(to_cpumask(tmpmask));

	/*
	 * Two reasons for reprogram:
	 *
	 * - The global event did not expire any CPU local
	 * events. This happens in dyntick mode, as the maximum PIT
	 * delta is quite small.
	 *
	 * - There are pending events on sleeping CPUs which were not
	 * in the event mask
	 */
	if (next_event.tv64 != KTIME_MAX) {
		/*
		 * Rearm the broadcast device. If event expired,
		 * repeat the above
		 */
		if (tick_broadcast_set_event(next_event, 0))
			goto again;
	}
	raw_spin_unlock(&tick_broadcast_lock);
}

/*
 * Powerstate information: The system enters/leaves a state, where
 * affected devices might stop
 */
void tick_broadcast_oneshot_control(unsigned long reason)
{
	struct clock_event_device *bc, *dev;
	struct tick_device *td;
	unsigned long flags;
	int cpu;

	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);

	/*
	 * Periodic mode does not care about the enter/exit of power
	 * states
	 */
	if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
		goto out;

	bc = tick_broadcast_device.evtdev;
	cpu = smp_processor_id();
	td = &per_cpu(tick_cpu_device, cpu);
	dev = td->evtdev;

	if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
		goto out;

	if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) {
		if (!cpumask_test_cpu(cpu, tick_get_broadcast_oneshot_mask())) {
			cpumask_set_cpu(cpu, tick_get_broadcast_oneshot_mask());
			clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN);
			if (dev->next_event.tv64 < bc->next_event.tv64)
				tick_broadcast_set_event(dev->next_event, 1);
		}
	} else {
		if (cpumask_test_cpu(cpu, tick_get_broadcast_oneshot_mask())) {
			cpumask_clear_cpu(cpu,
					  tick_get_broadcast_oneshot_mask());
			clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT);
			if (dev->next_event.tv64 != KTIME_MAX)
				tick_program_event(dev->next_event, 1);
		}
	}

out:
	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
}

/*
 * Reset the one shot broadcast for a cpu
 *
 * Called with tick_broadcast_lock held
 */
static void tick_broadcast_clear_oneshot(int cpu)
{
	cpumask_clear_cpu(cpu, tick_get_broadcast_oneshot_mask());
}

static void tick_broadcast_init_next_event(struct cpumask *mask,
					   ktime_t expires)
{
	struct tick_device *td;
	int cpu;

	for_each_cpu(cpu, mask) {
		td = &per_cpu(tick_cpu_device, cpu);
		if (td->evtdev)
			td->evtdev->next_event = expires;
	}
}

/**
 * tick_broadcast_setup_oneshot - setup the broadcast device
 */
void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
{
	/* Set it up only once ! */
	if (bc->event_handler != tick_handle_oneshot_broadcast) {
		int was_periodic = bc->mode == CLOCK_EVT_MODE_PERIODIC;
		int cpu = smp_processor_id();

		bc->event_handler = tick_handle_oneshot_broadcast;
		clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);

		/* Take the do_timer update */
		tick_do_timer_cpu = cpu;

		/*
		 * We must be careful here. There might be other CPUs
		 * waiting for periodic broadcast. We need to set the
		 * oneshot_mask bits for those and program the
		 * broadcast device to fire.
		 */
		cpumask_copy(to_cpumask(tmpmask), tick_get_broadcast_mask());
		cpumask_clear_cpu(cpu, to_cpumask(tmpmask));
		cpumask_or(tick_get_broadcast_oneshot_mask(),
			   tick_get_broadcast_oneshot_mask(),
			   to_cpumask(tmpmask));

		if (was_periodic && !cpumask_empty(to_cpumask(tmpmask))) {
			tick_broadcast_init_next_event(to_cpumask(tmpmask),
						       tick_next_period);
			tick_broadcast_set_event(tick_next_period, 1);
		} else
			bc->next_event.tv64 = KTIME_MAX;
	}
}

/*
 * Select oneshot operating mode for the broadcast device
 */
void tick_broadcast_switch_to_oneshot(void)
{
	struct clock_event_device *bc;
	unsigned long flags;

	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);

	tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
	bc = tick_broadcast_device.evtdev;
	if (bc)
		tick_broadcast_setup_oneshot(bc);
	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
}


/*
 * Remove a dead CPU from broadcasting
 */
void tick_shutdown_broadcast_oneshot(unsigned int *cpup)
{
	unsigned long flags;
	unsigned int cpu = *cpup;

	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);

	/*
	 * Clear the broadcast mask flag for the dead cpu, but do not
	 * stop the broadcast device!
	 */
	cpumask_clear_cpu(cpu, tick_get_broadcast_oneshot_mask());

	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
}

/*
 * Check, whether the broadcast device is in one shot mode
 */
int tick_broadcast_oneshot_active(void)
{
	return tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT;
}

/*
 * Check whether the broadcast device supports oneshot.
 */
bool tick_broadcast_oneshot_available(void)
{
	struct clock_event_device *bc = tick_broadcast_device.evtdev;

	return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false;
}

#endif
Commit	Line	Data
f8381cba TG	1	/*
	2	* linux/kernel/time/tick-broadcast.c
	3	*
	4	* This file contains functions which emulate a local clock-event
	5	* device via a broadcast event source.
	6	*
	7	* Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
	8	* Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
	9	* Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
	10	*
	11	* This code is licenced under the GPL version 2. For details see
	12	* kernel-base/COPYING.
	13	*/
	14	#include <linux/cpu.h>
	15	#include <linux/err.h>
	16	#include <linux/hrtimer.h>
d7b90689	17	#include <linux/interrupt.h>
f8381cba TG	18	#include <linux/percpu.h>
	19	#include <linux/profile.h>
	20	#include <linux/sched.h>
f8381cba TG	21
	22	#include "tick-internal.h"
	23
	24	/*
	25	* Broadcast support for broken x86 hardware, where the local apic
	26	* timer stops in C3 state.
	27	*/
	28
a52f5c56	29	static struct tick_device tick_broadcast_device;
6b954823 RR	30	/* FIXME: Use cpumask_var_t. */
	31	static DECLARE_BITMAP(tick_broadcast_mask, NR_CPUS);
	32	static DECLARE_BITMAP(tmpmask, NR_CPUS);
b5f91da0	33	static DEFINE_RAW_SPINLOCK(tick_broadcast_lock);
aa276e1c	34	static int tick_broadcast_force;
f8381cba	35
5590a536 TG	36	#ifdef CONFIG_TICK_ONESHOT
	37	static void tick_broadcast_clear_oneshot(int cpu);
	38	#else
	39	static inline void tick_broadcast_clear_oneshot(int cpu) { }
	40	#endif
	41
289f480a IM	42	/*
	43	* Debugging: see timer_list.c
	44	*/
	45	struct tick_device *tick_get_broadcast_device(void)
	46	{
	47	return &tick_broadcast_device;
	48	}
	49
6b954823	50	struct cpumask *tick_get_broadcast_mask(void)
289f480a	51	{
6b954823	52	return to_cpumask(tick_broadcast_mask);
289f480a IM	53	}
289f480a IM	54
f8381cba TG	55	/*
	56	* Start the device in periodic mode
	57	*/
	58	static void tick_broadcast_start_periodic(struct clock_event_device *bc)
	59	{
18de5bc4	60	if (bc)
f8381cba TG	61	tick_setup_periodic(bc, 1);
	62	}
	63
	64	/*
	65	* Check, if the device can be utilized as broadcast device:
	66	*/
	67	int tick_check_broadcast_device(struct clock_event_device *dev)
	68	{
4a93232d VP	69	if ((tick_broadcast_device.evtdev &&
	70	tick_broadcast_device.evtdev->rating >= dev->rating) \|\|
	71	(dev->features & CLOCK_EVT_FEAT_C3STOP))
f8381cba TG	72	return 0;
	73
	74	clockevents_exchange_device(NULL, dev);
	75	tick_broadcast_device.evtdev = dev;
6b954823	76	if (!cpumask_empty(tick_get_broadcast_mask()))
f8381cba TG	77	tick_broadcast_start_periodic(dev);
	78	return 1;
	79	}
	80
	81	/*
	82	* Check, if the device is the broadcast device
	83	*/
	84	int tick_is_broadcast_device(struct clock_event_device *dev)
	85	{
	86	return (dev && tick_broadcast_device.evtdev == dev);
	87	}
	88
	89	/*
	90	* Check, if the device is disfunctional and a place holder, which
	91	* needs to be handled by the broadcast device.
	92	*/
	93	int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
	94	{
	95	unsigned long flags;
	96	int ret = 0;
	97
b5f91da0	98	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
f8381cba TG	99
	100	/*
	101	* Devices might be registered with both periodic and oneshot
	102	* mode disabled. This signals, that the device needs to be
	103	* operated from the broadcast device and is a placeholder for
	104	* the cpu local device.
	105	*/
	106	if (!tick_device_is_functional(dev)) {
	107	dev->event_handler = tick_handle_periodic;
6b954823	108	cpumask_set_cpu(cpu, tick_get_broadcast_mask());
f8381cba TG	109	tick_broadcast_start_periodic(tick_broadcast_device.evtdev);
f8381cba TG	110	ret = 1;
5590a536 TG	111	} else {
	112	/*
	113	* When the new device is not affected by the stop
	114	* feature and the cpu is marked in the broadcast mask
	115	* then clear the broadcast bit.
	116	*/
	117	if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) {
	118	int cpu = smp_processor_id();
f8381cba	119
6b954823	120	cpumask_clear_cpu(cpu, tick_get_broadcast_mask());
5590a536 TG	121	tick_broadcast_clear_oneshot(cpu);
	122	}
	123	}
b5f91da0	124	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
f8381cba TG	125	return ret;
	126	}
	127
	128	/*
6b954823	129	* Broadcast the event to the cpus, which are set in the mask (mangled).
f8381cba	130	*/
6b954823	131	static void tick_do_broadcast(struct cpumask *mask)
f8381cba	132	{
186e3cb8	133	int cpu = smp_processor_id();
f8381cba TG	134	struct tick_device *td;
	135
	136	/*
	137	* Check, if the current cpu is in the mask
	138	*/
6b954823 RR	139	if (cpumask_test_cpu(cpu, mask)) {
6b954823 RR	140	cpumask_clear_cpu(cpu, mask);
f8381cba TG	141	td = &per_cpu(tick_cpu_device, cpu);
f8381cba TG	142	td->evtdev->event_handler(td->evtdev);
f8381cba TG	143	}
f8381cba TG	144
6b954823	145	if (!cpumask_empty(mask)) {
f8381cba TG	146	/*
	147	* It might be necessary to actually check whether the devices
	148	* have different broadcast functions. For now, just use the
	149	* one of the first device. This works as long as we have this
	150	* misfeature only on x86 (lapic)
	151	*/
6b954823 RR	152	td = &per_cpu(tick_cpu_device, cpumask_first(mask));
6b954823 RR	153	td->evtdev->broadcast(mask);
f8381cba	154	}
f8381cba TG	155	}
	156
	157	/*
	158	* Periodic broadcast:
	159	* - invoke the broadcast handlers
	160	*/
	161	static void tick_do_periodic_broadcast(void)
	162	{
b5f91da0	163	raw_spin_lock(&tick_broadcast_lock);
f8381cba	164
6b954823 RR	165	cpumask_and(to_cpumask(tmpmask),
	166	cpu_online_mask, tick_get_broadcast_mask());
	167	tick_do_broadcast(to_cpumask(tmpmask));
f8381cba	168
b5f91da0	169	raw_spin_unlock(&tick_broadcast_lock);
f8381cba TG	170	}
	171
	172	/*
	173	* Event handler for periodic broadcast ticks
	174	*/
	175	static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
	176	{
d4496b39 TG	177	ktime_t next;
d4496b39 TG	178
f8381cba TG	179	tick_do_periodic_broadcast();
	180
	181	/*
	182	* The device is in periodic mode. No reprogramming necessary:
	183	*/
	184	if (dev->mode == CLOCK_EVT_MODE_PERIODIC)
	185	return;
	186
	187	/*
	188	* Setup the next period for devices, which do not have
d4496b39	189	* periodic mode. We read dev->next_event first and add to it
698f9315	190	* when the event already expired. clockevents_program_event()
d4496b39 TG	191	* sets dev->next_event only when the event is really
d4496b39 TG	192	* programmed to the device.
f8381cba	193	*/
d4496b39 TG	194	for (next = dev->next_event; ;) {
d4496b39 TG	195	next = ktime_add(next, tick_period);
f8381cba TG	196
	197	if (!clockevents_program_event(dev, next, ktime_get()))
	198	return;
	199	tick_do_periodic_broadcast();
	200	}
	201	}
	202
	203	/*
	204	* Powerstate information: The system enters/leaves a state, where
	205	* affected devices might stop
	206	*/
f833bab8	207	static void tick_do_broadcast_on_off(unsigned long *reason)
f8381cba TG	208	{
	209	struct clock_event_device bc, dev;
	210	struct tick_device *td;
f833bab8	211	unsigned long flags;
9c17bcda	212	int cpu, bc_stopped;
f8381cba	213
b5f91da0	214	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
f8381cba TG	215
	216	cpu = smp_processor_id();
	217	td = &per_cpu(tick_cpu_device, cpu);
	218	dev = td->evtdev;
	219	bc = tick_broadcast_device.evtdev;
	220
	221	/*
1595f452	222	* Is the device not affected by the powerstate ?
f8381cba	223	*/
1595f452	224	if (!dev \|\| !(dev->features & CLOCK_EVT_FEAT_C3STOP))
f8381cba TG	225	goto out;
f8381cba TG	226
3dfbc884 TG	227	if (!tick_device_is_functional(dev))
3dfbc884 TG	228	goto out;
1595f452	229
6b954823	230	bc_stopped = cpumask_empty(tick_get_broadcast_mask());
9c17bcda	231
1595f452 TG	232	switch (*reason) {
	233	case CLOCK_EVT_NOTIFY_BROADCAST_ON:
	234	case CLOCK_EVT_NOTIFY_BROADCAST_FORCE:
6b954823 RR	235	if (!cpumask_test_cpu(cpu, tick_get_broadcast_mask())) {
6b954823 RR	236	cpumask_set_cpu(cpu, tick_get_broadcast_mask());
07454bff TG	237	if (tick_broadcast_device.mode ==
07454bff TG	238	TICKDEV_MODE_PERIODIC)
2344abbc	239	clockevents_shutdown(dev);
f8381cba	240	}
3dfbc884	241	if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_FORCE)
aa276e1c	242	tick_broadcast_force = 1;
1595f452 TG	243	break;
1595f452 TG	244	case CLOCK_EVT_NOTIFY_BROADCAST_OFF:
aa276e1c	245	if (!tick_broadcast_force &&
6b954823 RR	246	cpumask_test_cpu(cpu, tick_get_broadcast_mask())) {
6b954823 RR	247	cpumask_clear_cpu(cpu, tick_get_broadcast_mask());
07454bff TG	248	if (tick_broadcast_device.mode ==
07454bff TG	249	TICKDEV_MODE_PERIODIC)
f8381cba TG	250	tick_setup_periodic(dev, 0);
f8381cba TG	251	}
1595f452	252	break;
f8381cba TG	253	}
f8381cba TG	254
6b954823	255	if (cpumask_empty(tick_get_broadcast_mask())) {
9c17bcda	256	if (!bc_stopped)
2344abbc	257	clockevents_shutdown(bc);
9c17bcda	258	} else if (bc_stopped) {
f8381cba TG	259	if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
f8381cba TG	260	tick_broadcast_start_periodic(bc);
79bf2bb3 TG	261	else
79bf2bb3 TG	262	tick_broadcast_setup_oneshot(bc);
f8381cba TG	263	}
f8381cba TG	264	out:
b5f91da0	265	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
f8381cba TG	266	}
	267
	268	/*
	269	* Powerstate information: The system enters/leaves a state, where
	270	* affected devices might stop.
	271	*/
	272	void tick_broadcast_on_off(unsigned long reason, int *oncpu)
	273	{
6b954823	274	if (!cpumask_test_cpu(*oncpu, cpu_online_mask))
833df317	275	printk(KERN_ERR "tick-broadcast: ignoring broadcast for "
72fcde96	276	"offline CPU #%d\n", *oncpu);
bf020cb7	277	else
f833bab8	278	tick_do_broadcast_on_off(&reason);
f8381cba TG	279	}
	280
	281	/*
	282	* Set the periodic handler depending on broadcast on/off
	283	*/
	284	void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast)
	285	{
	286	if (!broadcast)
	287	dev->event_handler = tick_handle_periodic;
	288	else
	289	dev->event_handler = tick_handle_periodic_broadcast;
	290	}
	291
	292	/*
	293	* Remove a CPU from broadcasting
	294	*/
	295	void tick_shutdown_broadcast(unsigned int *cpup)
	296	{
	297	struct clock_event_device *bc;
	298	unsigned long flags;
	299	unsigned int cpu = *cpup;
	300
b5f91da0	301	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
f8381cba TG	302
f8381cba TG	303	bc = tick_broadcast_device.evtdev;
6b954823	304	cpumask_clear_cpu(cpu, tick_get_broadcast_mask());
f8381cba TG	305
f8381cba TG	306	if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
6b954823	307	if (bc && cpumask_empty(tick_get_broadcast_mask()))
2344abbc	308	clockevents_shutdown(bc);
f8381cba TG	309	}
f8381cba TG	310
b5f91da0	311	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
f8381cba	312	}
79bf2bb3	313
6321dd60 TG	314	void tick_suspend_broadcast(void)
	315	{
	316	struct clock_event_device *bc;
	317	unsigned long flags;
	318
b5f91da0	319	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
6321dd60 TG	320
6321dd60 TG	321	bc = tick_broadcast_device.evtdev;
18de5bc4	322	if (bc)
2344abbc	323	clockevents_shutdown(bc);
6321dd60	324
b5f91da0	325	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
6321dd60 TG	326	}
	327
	328	int tick_resume_broadcast(void)
	329	{
	330	struct clock_event_device *bc;
	331	unsigned long flags;
	332	int broadcast = 0;
	333
b5f91da0	334	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
6321dd60 TG	335
6321dd60 TG	336	bc = tick_broadcast_device.evtdev;
6321dd60	337
cd05a1f8	338	if (bc) {
18de5bc4 TG	339	clockevents_set_mode(bc, CLOCK_EVT_MODE_RESUME);
18de5bc4 TG	340
cd05a1f8 TG	341	switch (tick_broadcast_device.mode) {
cd05a1f8 TG	342	case TICKDEV_MODE_PERIODIC:
6b954823	343	if (!cpumask_empty(tick_get_broadcast_mask()))
cd05a1f8	344	tick_broadcast_start_periodic(bc);
6b954823 RR	345	broadcast = cpumask_test_cpu(smp_processor_id(),
6b954823 RR	346	tick_get_broadcast_mask());
cd05a1f8 TG	347	break;
	348	case TICKDEV_MODE_ONESHOT:
	349	broadcast = tick_resume_broadcast_oneshot(bc);
	350	break;
	351	}
6321dd60	352	}
b5f91da0	353	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
6321dd60 TG	354
	355	return broadcast;
	356	}
	357
	358
79bf2bb3 TG	359	#ifdef CONFIG_TICK_ONESHOT
79bf2bb3 TG	360
6b954823 RR	361	/* FIXME: use cpumask_var_t. */
6b954823 RR	362	static DECLARE_BITMAP(tick_broadcast_oneshot_mask, NR_CPUS);
79bf2bb3	363
289f480a	364	/*
6b954823	365	* Exposed for debugging: see timer_list.c
289f480a	366	*/
6b954823	367	struct cpumask *tick_get_broadcast_oneshot_mask(void)
289f480a	368	{
6b954823	369	return to_cpumask(tick_broadcast_oneshot_mask);
289f480a IM	370	}
289f480a IM	371
79bf2bb3 TG	372	static int tick_broadcast_set_event(ktime_t expires, int force)
	373	{
	374	struct clock_event_device *bc = tick_broadcast_device.evtdev;
1fb9b7d2 TG	375
1fb9b7d2 TG	376	return tick_dev_program_event(bc, expires, force);
79bf2bb3 TG	377	}
79bf2bb3 TG	378
cd05a1f8 TG	379	int tick_resume_broadcast_oneshot(struct clock_event_device *bc)
	380	{
	381	clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
b7e113dc	382	return 0;
cd05a1f8 TG	383	}
cd05a1f8 TG	384
fb02fbc1 TG	385	/*
	386	* Called from irq_enter() when idle was interrupted to reenable the
	387	* per cpu device.
	388	*/
	389	void tick_check_oneshot_broadcast(int cpu)
	390	{
6b954823	391	if (cpumask_test_cpu(cpu, to_cpumask(tick_broadcast_oneshot_mask))) {
fb02fbc1 TG	392	struct tick_device *td = &per_cpu(tick_cpu_device, cpu);
	393
	394	clockevents_set_mode(td->evtdev, CLOCK_EVT_MODE_ONESHOT);
	395	}
	396	}
	397
79bf2bb3 TG	398	/*
	399	* Handle oneshot mode broadcasting
	400	*/
	401	static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
	402	{
	403	struct tick_device *td;
cdc6f27d	404	ktime_t now, next_event;
79bf2bb3 TG	405	int cpu;
79bf2bb3 TG	406
b5f91da0	407	raw_spin_lock(&tick_broadcast_lock);
79bf2bb3 TG	408	again:
79bf2bb3 TG	409	dev->next_event.tv64 = KTIME_MAX;
cdc6f27d	410	next_event.tv64 = KTIME_MAX;
6b954823	411	cpumask_clear(to_cpumask(tmpmask));
79bf2bb3 TG	412	now = ktime_get();
79bf2bb3 TG	413	/* Find all expired events */
6b954823	414	for_each_cpu(cpu, tick_get_broadcast_oneshot_mask()) {
79bf2bb3 TG	415	td = &per_cpu(tick_cpu_device, cpu);
79bf2bb3 TG	416	if (td->evtdev->next_event.tv64 <= now.tv64)
6b954823	417	cpumask_set_cpu(cpu, to_cpumask(tmpmask));
cdc6f27d TG	418	else if (td->evtdev->next_event.tv64 < next_event.tv64)
cdc6f27d TG	419	next_event.tv64 = td->evtdev->next_event.tv64;
79bf2bb3 TG	420	}
	421
	422	/*
cdc6f27d TG	423	* Wakeup the cpus which have an expired event.
cdc6f27d TG	424	*/
6b954823	425	tick_do_broadcast(to_cpumask(tmpmask));
cdc6f27d TG	426
	427	/*
	428	* Two reasons for reprogram:
	429	*
	430	* - The global event did not expire any CPU local
	431	* events. This happens in dyntick mode, as the maximum PIT
	432	* delta is quite small.
	433	*
	434	* - There are pending events on sleeping CPUs which were not
	435	* in the event mask
79bf2bb3	436	*/
cdc6f27d	437	if (next_event.tv64 != KTIME_MAX) {
79bf2bb3	438	/*
cdc6f27d TG	439	* Rearm the broadcast device. If event expired,
cdc6f27d TG	440	* repeat the above
79bf2bb3	441	*/
cdc6f27d	442	if (tick_broadcast_set_event(next_event, 0))
79bf2bb3 TG	443	goto again;
79bf2bb3 TG	444	}
b5f91da0	445	raw_spin_unlock(&tick_broadcast_lock);
79bf2bb3 TG	446	}
	447
	448	/*
	449	* Powerstate information: The system enters/leaves a state, where
	450	* affected devices might stop
	451	*/
	452	void tick_broadcast_oneshot_control(unsigned long reason)
	453	{
	454	struct clock_event_device bc, dev;
	455	struct tick_device *td;
	456	unsigned long flags;
	457	int cpu;
	458
b5f91da0	459	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
79bf2bb3 TG	460
	461	/*
	462	* Periodic mode does not care about the enter/exit of power
	463	* states
	464	*/
	465	if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
	466	goto out;
	467
	468	bc = tick_broadcast_device.evtdev;
	469	cpu = smp_processor_id();
	470	td = &per_cpu(tick_cpu_device, cpu);
	471	dev = td->evtdev;
	472
	473	if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
	474	goto out;
	475
	476	if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) {
6b954823 RR	477	if (!cpumask_test_cpu(cpu, tick_get_broadcast_oneshot_mask())) {
6b954823 RR	478	cpumask_set_cpu(cpu, tick_get_broadcast_oneshot_mask());
79bf2bb3 TG	479	clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN);
	480	if (dev->next_event.tv64 < bc->next_event.tv64)
	481	tick_broadcast_set_event(dev->next_event, 1);
	482	}
	483	} else {
6b954823 RR	484	if (cpumask_test_cpu(cpu, tick_get_broadcast_oneshot_mask())) {
	485	cpumask_clear_cpu(cpu,
	486	tick_get_broadcast_oneshot_mask());
79bf2bb3 TG	487	clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT);
	488	if (dev->next_event.tv64 != KTIME_MAX)
	489	tick_program_event(dev->next_event, 1);
	490	}
	491	}
	492
	493	out:
b5f91da0	494	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
79bf2bb3 TG	495	}
79bf2bb3 TG	496
5590a536 TG	497	/*
	498	* Reset the one shot broadcast for a cpu
	499	*
	500	* Called with tick_broadcast_lock held
	501	*/
	502	static void tick_broadcast_clear_oneshot(int cpu)
	503	{
6b954823	504	cpumask_clear_cpu(cpu, tick_get_broadcast_oneshot_mask());
5590a536 TG	505	}
5590a536 TG	506
6b954823 RR	507	static void tick_broadcast_init_next_event(struct cpumask *mask,
6b954823 RR	508	ktime_t expires)
7300711e TG	509	{
	510	struct tick_device *td;
	511	int cpu;
	512
5db0e1e9	513	for_each_cpu(cpu, mask) {
7300711e TG	514	td = &per_cpu(tick_cpu_device, cpu);
	515	if (td->evtdev)
	516	td->evtdev->next_event = expires;
	517	}
	518	}
	519
79bf2bb3	520	/**
8dce39c2	521	* tick_broadcast_setup_oneshot - setup the broadcast device
79bf2bb3 TG	522	*/
	523	void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
	524	{
9c17bcda TG	525	/* Set it up only once ! */
9c17bcda TG	526	if (bc->event_handler != tick_handle_oneshot_broadcast) {
7300711e TG	527	int was_periodic = bc->mode == CLOCK_EVT_MODE_PERIODIC;
7300711e TG	528	int cpu = smp_processor_id();
7300711e	529
9c17bcda TG	530	bc->event_handler = tick_handle_oneshot_broadcast;
9c17bcda TG	531	clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
7300711e TG	532
	533	/* Take the do_timer update */
	534	tick_do_timer_cpu = cpu;
	535
	536	/*
	537	* We must be careful here. There might be other CPUs
	538	* waiting for periodic broadcast. We need to set the
	539	* oneshot_mask bits for those and program the
	540	* broadcast device to fire.
	541	*/
6b954823 RR	542	cpumask_copy(to_cpumask(tmpmask), tick_get_broadcast_mask());
	543	cpumask_clear_cpu(cpu, to_cpumask(tmpmask));
	544	cpumask_or(tick_get_broadcast_oneshot_mask(),
	545	tick_get_broadcast_oneshot_mask(),
	546	to_cpumask(tmpmask));
	547
	548	if (was_periodic && !cpumask_empty(to_cpumask(tmpmask))) {
	549	tick_broadcast_init_next_event(to_cpumask(tmpmask),
	550	tick_next_period);
7300711e TG	551	tick_broadcast_set_event(tick_next_period, 1);
	552	} else
	553	bc->next_event.tv64 = KTIME_MAX;
9c17bcda	554	}
79bf2bb3 TG	555	}
	556
	557	/*
	558	* Select oneshot operating mode for the broadcast device
	559	*/
	560	void tick_broadcast_switch_to_oneshot(void)
	561	{
	562	struct clock_event_device *bc;
	563	unsigned long flags;
	564
b5f91da0	565	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
79bf2bb3 TG	566
	567	tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
	568	bc = tick_broadcast_device.evtdev;
	569	if (bc)
	570	tick_broadcast_setup_oneshot(bc);
b5f91da0	571	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
79bf2bb3 TG	572	}
	573
	574
	575	/*
	576	* Remove a dead CPU from broadcasting
	577	*/
	578	void tick_shutdown_broadcast_oneshot(unsigned int *cpup)
	579	{
79bf2bb3 TG	580	unsigned long flags;
	581	unsigned int cpu = *cpup;
	582
b5f91da0	583	raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
79bf2bb3	584
31d9b393 TG	585	/*
	586	* Clear the broadcast mask flag for the dead cpu, but do not
	587	* stop the broadcast device!
	588	*/
6b954823	589	cpumask_clear_cpu(cpu, tick_get_broadcast_oneshot_mask());
79bf2bb3	590
b5f91da0	591	raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
79bf2bb3 TG	592	}
79bf2bb3 TG	593
27ce4cb4 TG	594	/*
	595	* Check, whether the broadcast device is in one shot mode
	596	*/
	597	int tick_broadcast_oneshot_active(void)
	598	{
	599	return tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT;
	600	}
	601
3a142a06 TG	602	/*
	603	* Check whether the broadcast device supports oneshot.
	604	*/
	605	bool tick_broadcast_oneshot_available(void)
	606	{
	607	struct clock_event_device *bc = tick_broadcast_device.evtdev;
	608
	609	return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false;
	610	}
	611
79bf2bb3	612	#endif