[mirror_ubuntu-zesty-kernel.git] / arch / arm / kernel / smp.c

/*
 *  linux/arch/arm/kernel/smp.c
 *
 *  Copyright (C) 2002 ARM Limited, All Rights Reserved.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */
#include <linux/config.h>
#include <linux/delay.h>
#include <linux/init.h>
#include <linux/spinlock.h>
#include <linux/sched.h>
#include <linux/interrupt.h>
#include <linux/cache.h>
#include <linux/profile.h>
#include <linux/errno.h>
#include <linux/mm.h>
#include <linux/cpu.h>
#include <linux/smp.h>
#include <linux/seq_file.h>

#include <asm/atomic.h>
#include <asm/cacheflush.h>
#include <asm/cpu.h>
#include <asm/mmu_context.h>
#include <asm/pgtable.h>
#include <asm/pgalloc.h>
#include <asm/processor.h>
#include <asm/tlbflush.h>
#include <asm/ptrace.h>

/*
 * bitmask of present and online CPUs.
 * The present bitmask indicates that the CPU is physically present.
 * The online bitmask indicates that the CPU is up and running.
 */
cpumask_t cpu_possible_map;
cpumask_t cpu_online_map;

/*
 * as from 2.5, kernels no longer have an init_tasks structure
 * so we need some other way of telling a new secondary core
 * where to place its SVC stack
 */
struct secondary_data secondary_data;

/*
 * structures for inter-processor calls
 * - A collection of single bit ipi messages.
 */
struct ipi_data {
	spinlock_t lock;
	unsigned long ipi_count;
	unsigned long bits;
};

static DEFINE_PER_CPU(struct ipi_data, ipi_data) = {
	.lock	= SPIN_LOCK_UNLOCKED,
};

enum ipi_msg_type {
	IPI_TIMER,
	IPI_RESCHEDULE,
	IPI_CALL_FUNC,
	IPI_CPU_STOP,
};

struct smp_call_struct {
	void (*func)(void *info);
	void *info;
	int wait;
	cpumask_t pending;
	cpumask_t unfinished;
};

static struct smp_call_struct * volatile smp_call_function_data;
static DEFINE_SPINLOCK(smp_call_function_lock);

int __init __cpu_up(unsigned int cpu)
{
	struct task_struct *idle;
	pgd_t *pgd;
	pmd_t *pmd;
	int ret;

	/*
	 * Spawn a new process manually.  Grab a pointer to
	 * its task struct so we can mess with it
	 */
	idle = fork_idle(cpu);
	if (IS_ERR(idle)) {
		printk(KERN_ERR "CPU%u: fork() failed\n", cpu);
		return PTR_ERR(idle);
	}

	/*
	 * Allocate initial page tables to allow the new CPU to
	 * enable the MMU safely.  This essentially means a set
	 * of our "standard" page tables, with the addition of
	 * a 1:1 mapping for the physical address of the kernel.
	 */
	pgd = pgd_alloc(&init_mm);
	pmd = pmd_offset(pgd, PHYS_OFFSET);
	*pmd = __pmd((PHYS_OFFSET & PGDIR_MASK) |
		     PMD_TYPE_SECT | PMD_SECT_AP_WRITE);

	/*
	 * We need to tell the secondary core where to find
	 * its stack and the page tables.
	 */
	secondary_data.stack = (void *)idle->thread_info + THREAD_SIZE - 8;
	secondary_data.pgdir = virt_to_phys(pgd);
	wmb();

	/*
	 * Now bring the CPU into our world.
	 */
	ret = boot_secondary(cpu, idle);
	if (ret == 0) {
		unsigned long timeout;

		/*
		 * CPU was successfully started, wait for it
		 * to come online or time out.
		 */
		timeout = jiffies + HZ;
		while (time_before(jiffies, timeout)) {
			if (cpu_online(cpu))
				break;

			udelay(10);
			barrier();
		}

		if (!cpu_online(cpu))
			ret = -EIO;
	}

	secondary_data.stack = 0;
	secondary_data.pgdir = 0;

	*pmd_offset(pgd, PHYS_OFFSET) = __pmd(0);
	pgd_free(pgd);

	if (ret) {
		printk(KERN_CRIT "CPU%u: processor failed to boot\n", cpu);

		/*
		 * FIXME: We need to clean up the new idle thread. --rmk
		 */
	}

	return ret;
}

/*
 * This is the secondary CPU boot entry.  We're using this CPUs
 * idle thread stack, but a set of temporary page tables.
 */
asmlinkage void __init secondary_start_kernel(void)
{
	struct mm_struct *mm = &init_mm;
	unsigned int cpu = smp_processor_id();

	printk("CPU%u: Booted secondary processor\n", cpu);

	/*
	 * All kernel threads share the same mm context; grab a
	 * reference and switch to it.
	 */
	atomic_inc(&mm->mm_users);
	atomic_inc(&mm->mm_count);
	current->active_mm = mm;
	cpu_set(cpu, mm->cpu_vm_mask);
	cpu_switch_mm(mm->pgd, mm);
	enter_lazy_tlb(mm, current);

	cpu_init();

	/*
	 * Give the platform a chance to do its own initialisation.
	 */
	platform_secondary_init(cpu);

	/*
	 * Enable local interrupts.
	 */
	local_irq_enable();
	local_fiq_enable();

	calibrate_delay();

	smp_store_cpu_info(cpu);

	/*
	 * OK, now it's safe to let the boot CPU continue
	 */
	cpu_set(cpu, cpu_online_map);

	/*
	 * OK, it's off to the idle thread for us
	 */
	cpu_idle();
}

/*
 * Called by both boot and secondaries to move global data into
 * per-processor storage.
 */
void __init smp_store_cpu_info(unsigned int cpuid)
{
	struct cpuinfo_arm *cpu_info = &per_cpu(cpu_data, cpuid);

	cpu_info->loops_per_jiffy = loops_per_jiffy;
}

void __init smp_cpus_done(unsigned int max_cpus)
{
	int cpu;
	unsigned long bogosum = 0;

	for_each_online_cpu(cpu)
		bogosum += per_cpu(cpu_data, cpu).loops_per_jiffy;

	printk(KERN_INFO "SMP: Total of %d processors activated "
	       "(%lu.%02lu BogoMIPS).\n",
	       num_online_cpus(),
	       bogosum / (500000/HZ),
	       (bogosum / (5000/HZ)) % 100);
}

void __init smp_prepare_boot_cpu(void)
{
	unsigned int cpu = smp_processor_id();

	cpu_set(cpu, cpu_possible_map);
	cpu_set(cpu, cpu_online_map);
}

static void send_ipi_message(cpumask_t callmap, enum ipi_msg_type msg)
{
	unsigned long flags;
	unsigned int cpu;

	local_irq_save(flags);

	for_each_cpu_mask(cpu, callmap) {
		struct ipi_data *ipi = &per_cpu(ipi_data, cpu);

		spin_lock(&ipi->lock);
		ipi->bits |= 1 << msg;
		spin_unlock(&ipi->lock);
	}

	/*
	 * Call the platform specific cross-CPU call function.
	 */
	smp_cross_call(callmap);

	local_irq_restore(flags);
}

/*
 * You must not call this function with disabled interrupts, from a
 * hardware interrupt handler, nor from a bottom half handler.
 */
int smp_call_function_on_cpu(void (*func)(void *info), void *info, int retry,
                             int wait, cpumask_t callmap)
{
	struct smp_call_struct data;
	unsigned long timeout;
	int ret = 0;

	data.func = func;
	data.info = info;
	data.wait = wait;

	cpu_clear(smp_processor_id(), callmap);
	if (cpus_empty(callmap))
		goto out;

	data.pending = callmap;
	if (wait)
		data.unfinished = callmap;

	/*
	 * try to get the mutex on smp_call_function_data
	 */
	spin_lock(&smp_call_function_lock);
	smp_call_function_data = &data;

	send_ipi_message(callmap, IPI_CALL_FUNC);

	timeout = jiffies + HZ;
	while (!cpus_empty(data.pending) && time_before(jiffies, timeout))
		barrier();

	/*
	 * did we time out?
	 */
	if (!cpus_empty(data.pending)) {
		/*
		 * this may be causing our panic - report it
		 */
		printk(KERN_CRIT
		       "CPU%u: smp_call_function timeout for %p(%p)\n"
		       "      callmap %lx pending %lx, %swait\n",
		       smp_processor_id(), func, info, callmap, data.pending,
		       wait ? "" : "no ");

		/*
		 * TRACE
		 */
		timeout = jiffies + (5 * HZ);
		while (!cpus_empty(data.pending) && time_before(jiffies, timeout))
			barrier();

		if (cpus_empty(data.pending))
			printk(KERN_CRIT "     RESOLVED\n");
		else
			printk(KERN_CRIT "     STILL STUCK\n");
	}

	/*
	 * whatever happened, we're done with the data, so release it
	 */
	smp_call_function_data = NULL;
	spin_unlock(&smp_call_function_lock);

	if (!cpus_empty(data.pending)) {
		ret = -ETIMEDOUT;
		goto out;
	}

	if (wait)
		while (!cpus_empty(data.unfinished))
			barrier();
 out:

	return 0;
}

int smp_call_function(void (*func)(void *info), void *info, int retry,
                      int wait)
{
	return smp_call_function_on_cpu(func, info, retry, wait,
					cpu_online_map);
}

void show_ipi_list(struct seq_file *p)
{
	unsigned int cpu;

	seq_puts(p, "IPI:");

	for_each_online_cpu(cpu)
		seq_printf(p, " %10lu", per_cpu(ipi_data, cpu).ipi_count);

	seq_putc(p, '\n');
}

static void ipi_timer(struct pt_regs *regs)
{
	int user = user_mode(regs);

	irq_enter();
	profile_tick(CPU_PROFILING, regs);
	update_process_times(user);
	irq_exit();
}

/*
 * ipi_call_function - handle IPI from smp_call_function()
 *
 * Note that we copy data out of the cross-call structure and then
 * let the caller know that we're here and have done with their data
 */
static void ipi_call_function(unsigned int cpu)
{
	struct smp_call_struct *data = smp_call_function_data;
	void (*func)(void *info) = data->func;
	void *info = data->info;
	int wait = data->wait;

	cpu_clear(cpu, data->pending);

	func(info);

	if (wait)
		cpu_clear(cpu, data->unfinished);
}

static DEFINE_SPINLOCK(stop_lock);

/*
 * ipi_cpu_stop - handle IPI from smp_send_stop()
 */
static void ipi_cpu_stop(unsigned int cpu)
{
	spin_lock(&stop_lock);
	printk(KERN_CRIT "CPU%u: stopping\n", cpu);
	dump_stack();
	spin_unlock(&stop_lock);

	cpu_clear(cpu, cpu_online_map);

	local_fiq_disable();
	local_irq_disable();

	while (1)
		cpu_relax();
}

/*
 * Main handler for inter-processor interrupts
 *
 * For ARM, the ipimask now only identifies a single
 * category of IPI (Bit 1 IPIs have been replaced by a
 * different mechanism):
 *
 *  Bit 0 - Inter-processor function call
 */
void do_IPI(struct pt_regs *regs)
{
	unsigned int cpu = smp_processor_id();
	struct ipi_data *ipi = &per_cpu(ipi_data, cpu);

	ipi->ipi_count++;

	for (;;) {
		unsigned long msgs;

		spin_lock(&ipi->lock);
		msgs = ipi->bits;
		ipi->bits = 0;
		spin_unlock(&ipi->lock);

		if (!msgs)
			break;

		do {
			unsigned nextmsg;

			nextmsg = msgs & -msgs;
			msgs &= ~nextmsg;
			nextmsg = ffz(~nextmsg);

			switch (nextmsg) {
			case IPI_TIMER:
				ipi_timer(regs);
				break;

			case IPI_RESCHEDULE:
				/*
				 * nothing more to do - eveything is
				 * done on the interrupt return path
				 */
				break;

			case IPI_CALL_FUNC:
				ipi_call_function(cpu);
				break;

			case IPI_CPU_STOP:
				ipi_cpu_stop(cpu);
				break;

			default:
				printk(KERN_CRIT "CPU%u: Unknown IPI message 0x%x\n",
				       cpu, nextmsg);
				break;
			}
		} while (msgs);
	}
}

void smp_send_reschedule(int cpu)
{
	send_ipi_message(cpumask_of_cpu(cpu), IPI_RESCHEDULE);
}

void smp_send_timer(void)
{
	cpumask_t mask = cpu_online_map;
	cpu_clear(smp_processor_id(), mask);
	send_ipi_message(mask, IPI_TIMER);
}

void smp_send_stop(void)
{
	cpumask_t mask = cpu_online_map;
	cpu_clear(smp_processor_id(), mask);
	send_ipi_message(mask, IPI_CPU_STOP);
}

/*
 * not supported here
 */
int __init setup_profiling_timer(unsigned int multiplier)
{
	return -EINVAL;
}

static int
on_each_cpu_mask(void (*func)(void *), void *info, int retry, int wait,
		 cpumask_t mask)
{
	int ret = 0;

	preempt_disable();

	ret = smp_call_function_on_cpu(func, info, retry, wait, mask);
	if (cpu_isset(smp_processor_id(), mask))
		func(info);

	preempt_enable();

	return ret;
}

/**********************************************************************/

/*
 * TLB operations
 */
struct tlb_args {
	struct vm_area_struct *ta_vma;
	unsigned long ta_start;
	unsigned long ta_end;
};

static inline void ipi_flush_tlb_all(void *ignored)
{
	local_flush_tlb_all();
}

static inline void ipi_flush_tlb_mm(void *arg)
{
	struct mm_struct *mm = (struct mm_struct *)arg;

	local_flush_tlb_mm(mm);
}

static inline void ipi_flush_tlb_page(void *arg)
{
	struct tlb_args *ta = (struct tlb_args *)arg;

	local_flush_tlb_page(ta->ta_vma, ta->ta_start);
}

static inline void ipi_flush_tlb_kernel_page(void *arg)
{
	struct tlb_args *ta = (struct tlb_args *)arg;

	local_flush_tlb_kernel_page(ta->ta_start);
}

static inline void ipi_flush_tlb_range(void *arg)
{
	struct tlb_args *ta = (struct tlb_args *)arg;

	local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);
}

static inline void ipi_flush_tlb_kernel_range(void *arg)
{
	struct tlb_args *ta = (struct tlb_args *)arg;

	local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end);
}

void flush_tlb_all(void)
{
	on_each_cpu(ipi_flush_tlb_all, NULL, 1, 1);
}

void flush_tlb_mm(struct mm_struct *mm)
{
	cpumask_t mask = mm->cpu_vm_mask;

	on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, 1, mask);
}

void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
{
	cpumask_t mask = vma->vm_mm->cpu_vm_mask;
	struct tlb_args ta;

	ta.ta_vma = vma;
	ta.ta_start = uaddr;

	on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, 1, mask);
}

void flush_tlb_kernel_page(unsigned long kaddr)
{
	struct tlb_args ta;

	ta.ta_start = kaddr;

	on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1, 1);
}

void flush_tlb_range(struct vm_area_struct *vma,
                     unsigned long start, unsigned long end)
{
	cpumask_t mask = vma->vm_mm->cpu_vm_mask;
	struct tlb_args ta;

	ta.ta_vma = vma;
	ta.ta_start = start;
	ta.ta_end = end;

	on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, 1, mask);
}

void flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
	struct tlb_args ta;

	ta.ta_start = start;
	ta.ta_end = end;

	on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1, 1);
}
Commit	Line	Data
1da177e4 LT	1	/*
	2	* linux/arch/arm/kernel/smp.c
	3	*
	4	* Copyright (C) 2002 ARM Limited, All Rights Reserved.
	5	*
	6	* This program is free software; you can redistribute it and/or modify
	7	* it under the terms of the GNU General Public License version 2 as
	8	* published by the Free Software Foundation.
	9	*/
	10	#include <linux/config.h>
	11	#include <linux/delay.h>
	12	#include <linux/init.h>
	13	#include <linux/spinlock.h>
	14	#include <linux/sched.h>
	15	#include <linux/interrupt.h>
	16	#include <linux/cache.h>
	17	#include <linux/profile.h>
	18	#include <linux/errno.h>
	19	#include <linux/mm.h>
	20	#include <linux/cpu.h>
	21	#include <linux/smp.h>
	22	#include <linux/seq_file.h>
	23
	24	#include <asm/atomic.h>
	25	#include <asm/cacheflush.h>
	26	#include <asm/cpu.h>
e65f38ed RK	27	#include <asm/mmu_context.h>
	28	#include <asm/pgtable.h>
	29	#include <asm/pgalloc.h>
1da177e4 LT	30	#include <asm/processor.h>
	31	#include <asm/tlbflush.h>
	32	#include <asm/ptrace.h>
	33
	34	/*
	35	* bitmask of present and online CPUs.
	36	* The present bitmask indicates that the CPU is physically present.
	37	* The online bitmask indicates that the CPU is up and running.
	38	*/
d12734d1	39	cpumask_t cpu_possible_map;
1da177e4 LT	40	cpumask_t cpu_online_map;
1da177e4 LT	41
e65f38ed RK	42	/*
	43	* as from 2.5, kernels no longer have an init_tasks structure
	44	* so we need some other way of telling a new secondary core
	45	* where to place its SVC stack
	46	*/
	47	struct secondary_data secondary_data;
	48
1da177e4 LT	49	/*
	50	* structures for inter-processor calls
	51	* - A collection of single bit ipi messages.
	52	*/
	53	struct ipi_data {
	54	spinlock_t lock;
	55	unsigned long ipi_count;
	56	unsigned long bits;
	57	};
	58
	59	static DEFINE_PER_CPU(struct ipi_data, ipi_data) = {
	60	.lock = SPIN_LOCK_UNLOCKED,
	61	};
	62
	63	enum ipi_msg_type {
	64	IPI_TIMER,
	65	IPI_RESCHEDULE,
	66	IPI_CALL_FUNC,
	67	IPI_CPU_STOP,
	68	};
	69
	70	struct smp_call_struct {
	71	void (func)(void info);
	72	void *info;
	73	int wait;
	74	cpumask_t pending;
	75	cpumask_t unfinished;
	76	};
	77
	78	static struct smp_call_struct * volatile smp_call_function_data;
	79	static DEFINE_SPINLOCK(smp_call_function_lock);
	80
	81	int __init __cpu_up(unsigned int cpu)
	82	{
	83	struct task_struct *idle;
e65f38ed RK	84	pgd_t *pgd;
e65f38ed RK	85	pmd_t *pmd;
1da177e4 LT	86	int ret;
	87
	88	/*
	89	* Spawn a new process manually. Grab a pointer to
	90	* its task struct so we can mess with it
	91	*/
	92	idle = fork_idle(cpu);
	93	if (IS_ERR(idle)) {
	94	printk(KERN_ERR "CPU%u: fork() failed\n", cpu);
	95	return PTR_ERR(idle);
	96	}
	97
e65f38ed RK	98	/*
	99	* Allocate initial page tables to allow the new CPU to
	100	* enable the MMU safely. This essentially means a set
	101	* of our "standard" page tables, with the addition of
	102	* a 1:1 mapping for the physical address of the kernel.
	103	*/
	104	pgd = pgd_alloc(&init_mm);
	105	pmd = pmd_offset(pgd, PHYS_OFFSET);
	106	*pmd = __pmd((PHYS_OFFSET & PGDIR_MASK) \|
	107	PMD_TYPE_SECT \| PMD_SECT_AP_WRITE);
	108
	109	/*
	110	* We need to tell the secondary core where to find
	111	* its stack and the page tables.
	112	*/
	113	secondary_data.stack = (void *)idle->thread_info + THREAD_SIZE - 8;
	114	secondary_data.pgdir = virt_to_phys(pgd);
	115	wmb();
	116
1da177e4 LT	117	/*
	118	* Now bring the CPU into our world.
	119	*/
	120	ret = boot_secondary(cpu, idle);
e65f38ed RK	121	if (ret == 0) {
	122	unsigned long timeout;
	123
	124	/*
	125	* CPU was successfully started, wait for it
	126	* to come online or time out.
	127	*/
	128	timeout = jiffies + HZ;
	129	while (time_before(jiffies, timeout)) {
	130	if (cpu_online(cpu))
	131	break;
	132
	133	udelay(10);
	134	barrier();
	135	}
	136
	137	if (!cpu_online(cpu))
	138	ret = -EIO;
	139	}
	140
	141	secondary_data.stack = 0;
	142	secondary_data.pgdir = 0;
	143
	144	*pmd_offset(pgd, PHYS_OFFSET) = __pmd(0);
	145	pgd_free(pgd);
	146
1da177e4	147	if (ret) {
0908db22 RK	148	printk(KERN_CRIT "CPU%u: processor failed to boot\n", cpu);
0908db22 RK	149
1da177e4 LT	150	/*
	151	* FIXME: We need to clean up the new idle thread. --rmk
	152	*/
	153	}
	154
	155	return ret;
	156	}
	157
e65f38ed RK	158	/*
	159	* This is the secondary CPU boot entry. We're using this CPUs
	160	* idle thread stack, but a set of temporary page tables.
	161	*/
	162	asmlinkage void __init secondary_start_kernel(void)
	163	{
	164	struct mm_struct *mm = &init_mm;
	165	unsigned int cpu = smp_processor_id();
	166
	167	printk("CPU%u: Booted secondary processor\n", cpu);
	168
	169	/*
	170	* All kernel threads share the same mm context; grab a
	171	* reference and switch to it.
	172	*/
	173	atomic_inc(&mm->mm_users);
	174	atomic_inc(&mm->mm_count);
	175	current->active_mm = mm;
	176	cpu_set(cpu, mm->cpu_vm_mask);
	177	cpu_switch_mm(mm->pgd, mm);
	178	enter_lazy_tlb(mm, current);
	179
	180	cpu_init();
	181
	182	/*
	183	* Give the platform a chance to do its own initialisation.
	184	*/
	185	platform_secondary_init(cpu);
	186
	187	/*
	188	* Enable local interrupts.
	189	*/
	190	local_irq_enable();
	191	local_fiq_enable();
	192
	193	calibrate_delay();
	194
	195	smp_store_cpu_info(cpu);
	196
	197	/*
	198	* OK, now it's safe to let the boot CPU continue
	199	*/
	200	cpu_set(cpu, cpu_online_map);
	201
	202	/*
	203	* OK, it's off to the idle thread for us
	204	*/
	205	cpu_idle();
	206	}
	207
1da177e4 LT	208	/*
	209	* Called by both boot and secondaries to move global data into
	210	* per-processor storage.
	211	*/
	212	void __init smp_store_cpu_info(unsigned int cpuid)
	213	{
	214	struct cpuinfo_arm *cpu_info = &per_cpu(cpu_data, cpuid);
	215
	216	cpu_info->loops_per_jiffy = loops_per_jiffy;
	217	}
	218
	219	void __init smp_cpus_done(unsigned int max_cpus)
	220	{
	221	int cpu;
	222	unsigned long bogosum = 0;
	223
	224	for_each_online_cpu(cpu)
	225	bogosum += per_cpu(cpu_data, cpu).loops_per_jiffy;
	226
	227	printk(KERN_INFO "SMP: Total of %d processors activated "
	228	"(%lu.%02lu BogoMIPS).\n",
	229	num_online_cpus(),
	230	bogosum / (500000/HZ),
	231	(bogosum / (5000/HZ)) % 100);
	232	}
	233
	234	void __init smp_prepare_boot_cpu(void)
	235	{
	236	unsigned int cpu = smp_processor_id();
	237
d12734d1	238	cpu_set(cpu, cpu_possible_map);
1da177e4 LT	239	cpu_set(cpu, cpu_online_map);
	240	}
	241
	242	static void send_ipi_message(cpumask_t callmap, enum ipi_msg_type msg)
	243	{
	244	unsigned long flags;
	245	unsigned int cpu;
	246
	247	local_irq_save(flags);
	248
	249	for_each_cpu_mask(cpu, callmap) {
	250	struct ipi_data *ipi = &per_cpu(ipi_data, cpu);
	251
	252	spin_lock(&ipi->lock);
	253	ipi->bits \|= 1 << msg;
	254	spin_unlock(&ipi->lock);
	255	}
	256
	257	/*
	258	* Call the platform specific cross-CPU call function.
	259	*/
	260	smp_cross_call(callmap);
	261
	262	local_irq_restore(flags);
	263	}
	264
	265	/*
	266	* You must not call this function with disabled interrupts, from a
	267	* hardware interrupt handler, nor from a bottom half handler.
	268	*/
	269	int smp_call_function_on_cpu(void (func)(void info), void *info, int retry,
	270	int wait, cpumask_t callmap)
	271	{
	272	struct smp_call_struct data;
	273	unsigned long timeout;
	274	int ret = 0;
	275
	276	data.func = func;
	277	data.info = info;
	278	data.wait = wait;
	279
	280	cpu_clear(smp_processor_id(), callmap);
	281	if (cpus_empty(callmap))
	282	goto out;
	283
	284	data.pending = callmap;
	285	if (wait)
	286	data.unfinished = callmap;
	287
	288	/*
	289	* try to get the mutex on smp_call_function_data
	290	*/
	291	spin_lock(&smp_call_function_lock);
	292	smp_call_function_data = &data;
	293
	294	send_ipi_message(callmap, IPI_CALL_FUNC);
	295
	296	timeout = jiffies + HZ;
	297	while (!cpus_empty(data.pending) && time_before(jiffies, timeout))
	298	barrier();
	299
	300	/*
	301	* did we time out?
	302	*/
303	if (!cpus_empty(data.pending)) {
304	/*
305	* this may be causing our panic - report it
306	*/
307	printk(KERN_CRIT
308	"CPU%u: smp_call_function timeout for %p(%p)\n"
309	" callmap %lx pending %lx, %swait\n",
310	smp_processor_id(), func, info, callmap, data.pending,
311	wait ? "" : "no ");
312
313	/*
314	* TRACE
315	*/
316	timeout = jiffies + (5 * HZ);
317	while (!cpus_empty(data.pending) && time_before(jiffies, timeout))
318	barrier();
319
320	if (cpus_empty(data.pending))
321	printk(KERN_CRIT " RESOLVED\n");
322	else
323	printk(KERN_CRIT " STILL STUCK\n");
324	}
325
326	/*
327	* whatever happened, we're done with the data, so release it
328	*/
329	smp_call_function_data = NULL;
330	spin_unlock(&smp_call_function_lock);
331
332	if (!cpus_empty(data.pending)) {
333	ret = -ETIMEDOUT;
334	goto out;
335	}
336
337	if (wait)
338	while (!cpus_empty(data.unfinished))
339	barrier();
340	out:
341
342	return 0;
343	}
344
345	int smp_call_function(void (func)(void info), void *info, int retry,
346	int wait)
347	{
348	return smp_call_function_on_cpu(func, info, retry, wait,
349	cpu_online_map);
350	}
351
352	void show_ipi_list(struct seq_file *p)
353	{
354	unsigned int cpu;
355
356	seq_puts(p, "IPI:");
357
358	for_each_online_cpu(cpu)
359	seq_printf(p, " %10lu", per_cpu(ipi_data, cpu).ipi_count);
360
361	seq_putc(p, '\n');
362	}
363
364	static void ipi_timer(struct pt_regs *regs)
365	{
366	int user = user_mode(regs);
367
368	irq_enter();
369	profile_tick(CPU_PROFILING, regs);
370	update_process_times(user);
371	irq_exit();
372	}
373
374	/*
375	* ipi_call_function - handle IPI from smp_call_function()
376	*
377	* Note that we copy data out of the cross-call structure and then
378	* let the caller know that we're here and have done with their data
379	*/
380	static void ipi_call_function(unsigned int cpu)
381	{
382	struct smp_call_struct *data = smp_call_function_data;
383	void (func)(void info) = data->func;
384	void *info = data->info;
385	int wait = data->wait;
386
387	cpu_clear(cpu, data->pending);
388
389	func(info);
390
391	if (wait)
392	cpu_clear(cpu, data->unfinished);
393	}
394
395	static DEFINE_SPINLOCK(stop_lock);
396
397	/*
398	* ipi_cpu_stop - handle IPI from smp_send_stop()
399	*/
400	static void ipi_cpu_stop(unsigned int cpu)
401	{
402	spin_lock(&stop_lock);
403	printk(KERN_CRIT "CPU%u: stopping\n", cpu);
404	dump_stack();
405	spin_unlock(&stop_lock);
406
407	cpu_clear(cpu, cpu_online_map);
408
409	local_fiq_disable();
410	local_irq_disable();
411
412	while (1)
413	cpu_relax();
414	}
415
416	/*
417	* Main handler for inter-processor interrupts
418	*
419	* For ARM, the ipimask now only identifies a single
420	* category of IPI (Bit 1 IPIs have been replaced by a
421	* different mechanism):
422	*
423	* Bit 0 - Inter-processor function call
424	*/
425	void do_IPI(struct pt_regs *regs)
426	{
427	unsigned int cpu = smp_processor_id();
428	struct ipi_data *ipi = &per_cpu(ipi_data, cpu);
429
430	ipi->ipi_count++;
431
432	for (;;) {
433	unsigned long msgs;
434
435	spin_lock(&ipi->lock);
436	msgs = ipi->bits;
437	ipi->bits = 0;
438	spin_unlock(&ipi->lock);
439
440	if (!msgs)
441	break;
442
443	do {
444	unsigned nextmsg;
445
446	nextmsg = msgs & -msgs;
447	msgs &= ~nextmsg;
448	nextmsg = ffz(~nextmsg);
449
450	switch (nextmsg) {
451	case IPI_TIMER:
452	ipi_timer(regs);
453	break;
454
455	case IPI_RESCHEDULE:
456	/*
457	* nothing more to do - eveything is
458	* done on the interrupt return path
459	*/
460	break;
461
462	case IPI_CALL_FUNC:
463	ipi_call_function(cpu);
464	break;
465
466	case IPI_CPU_STOP:
467	ipi_cpu_stop(cpu);
468	break;
469
470	default:
471	printk(KERN_CRIT "CPU%u: Unknown IPI message 0x%x\n",
472	cpu, nextmsg);
473	break;
474	}
475	} while (msgs);
476	}
477	}
478
479	void smp_send_reschedule(int cpu)
480	{
481	send_ipi_message(cpumask_of_cpu(cpu), IPI_RESCHEDULE);
482	}
483
484	void smp_send_timer(void)
485	{
486	cpumask_t mask = cpu_online_map;
487	cpu_clear(smp_processor_id(), mask);
488	send_ipi_message(mask, IPI_TIMER);
489	}
490
491	void smp_send_stop(void)
492	{
493	cpumask_t mask = cpu_online_map;
494	cpu_clear(smp_processor_id(), mask);
495	send_ipi_message(mask, IPI_CPU_STOP);
496	}
497
498	/*
499	* not supported here
500	*/
501	int __init setup_profiling_timer(unsigned int multiplier)
502	{
503	return -EINVAL;
504	}
4b0ef3b1 RK	505
	506	static int
	507	on_each_cpu_mask(void (func)(void ), void *info, int retry, int wait,
	508	cpumask_t mask)
	509	{
	510	int ret = 0;
	511
	512	preempt_disable();
	513
	514	ret = smp_call_function_on_cpu(func, info, retry, wait, mask);
	515	if (cpu_isset(smp_processor_id(), mask))
	516	func(info);
	517
	518	preempt_enable();
	519
	520	return ret;
	521	}
	522
	523	/**********************************************************************/
	524
	525	/*
	526	* TLB operations
	527	*/
	528	struct tlb_args {
	529	struct vm_area_struct *ta_vma;
	530	unsigned long ta_start;
	531	unsigned long ta_end;
	532	};
	533
	534	static inline void ipi_flush_tlb_all(void *ignored)
	535	{
	536	local_flush_tlb_all();
	537	}
	538
	539	static inline void ipi_flush_tlb_mm(void *arg)
	540	{
	541	struct mm_struct mm = (struct mm_struct )arg;
	542
	543	local_flush_tlb_mm(mm);
	544	}
	545
	546	static inline void ipi_flush_tlb_page(void *arg)
	547	{
	548	struct tlb_args ta = (struct tlb_args )arg;
	549
	550	local_flush_tlb_page(ta->ta_vma, ta->ta_start);
	551	}
	552
	553	static inline void ipi_flush_tlb_kernel_page(void *arg)
	554	{
	555	struct tlb_args ta = (struct tlb_args )arg;
	556
	557	local_flush_tlb_kernel_page(ta->ta_start);
	558	}
	559
	560	static inline void ipi_flush_tlb_range(void *arg)
	561	{
	562	struct tlb_args ta = (struct tlb_args )arg;
	563
	564	local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);
	565	}
	566
	567	static inline void ipi_flush_tlb_kernel_range(void *arg)
	568	{
569	struct tlb_args ta = (struct tlb_args )arg;
570
571	local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end);
572	}
573
574	void flush_tlb_all(void)
575	{
576	on_each_cpu(ipi_flush_tlb_all, NULL, 1, 1);
577	}
578
579	void flush_tlb_mm(struct mm_struct *mm)
580	{
581	cpumask_t mask = mm->cpu_vm_mask;
582
583	on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, 1, mask);
584	}
585
586	void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
587	{
588	cpumask_t mask = vma->vm_mm->cpu_vm_mask;
589	struct tlb_args ta;
590
591	ta.ta_vma = vma;
592	ta.ta_start = uaddr;
593
594	on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, 1, mask);
595	}
596
597	void flush_tlb_kernel_page(unsigned long kaddr)
598	{
599	struct tlb_args ta;
600
601	ta.ta_start = kaddr;
602
603	on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1, 1);
604	}
605
606	void flush_tlb_range(struct vm_area_struct *vma,
607	unsigned long start, unsigned long end)
608	{
609	cpumask_t mask = vma->vm_mm->cpu_vm_mask;
610	struct tlb_args ta;
611
612	ta.ta_vma = vma;
613	ta.ta_start = start;
614	ta.ta_end = end;
615
616	on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, 1, mask);
617	}
618
619	void flush_tlb_kernel_range(unsigned long start, unsigned long end)
620	{
621	struct tlb_args ta;
622
623	ta.ta_start = start;
624	ta.ta_end = end;
625
626	on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1, 1);
627	}