[mirror_ubuntu-bionic-kernel.git] / include / asm-x86_64 / mmu_context.h

#ifndef __X86_64_MMU_CONTEXT_H
#define __X86_64_MMU_CONTEXT_H

#include <asm/desc.h>
#include <asm/atomic.h>
#include <asm/pgalloc.h>
#include <asm/pda.h>
#include <asm/pgtable.h>
#include <asm/tlbflush.h>

/*
 * possibly do the LDT unload here?
 */
int init_new_context(struct task_struct *tsk, struct mm_struct *mm);
void destroy_context(struct mm_struct *mm);

static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
{
#ifdef CONFIG_SMP
	if (read_pda(mmu_state) == TLBSTATE_OK) 
		write_pda(mmu_state, TLBSTATE_LAZY);
#endif
}

static inline void load_cr3(pgd_t *pgd)
{
	asm volatile("movq %0,%%cr3" :: "r" (__pa(pgd)) : "memory");
}

static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, 
			     struct task_struct *tsk)
{
	unsigned cpu = smp_processor_id();
	if (likely(prev != next)) {
		/* stop flush ipis for the previous mm */
		cpu_clear(cpu, prev->cpu_vm_mask);
#ifdef CONFIG_SMP
		write_pda(mmu_state, TLBSTATE_OK);
		write_pda(active_mm, next);
#endif
		cpu_set(cpu, next->cpu_vm_mask);
		load_cr3(next->pgd);

		if (unlikely(next->context.ldt != prev->context.ldt)) 
			load_LDT_nolock(&next->context, cpu);
	}
#ifdef CONFIG_SMP
	else {
		write_pda(mmu_state, TLBSTATE_OK);
		if (read_pda(active_mm) != next)
			out_of_line_bug();
		if (!cpu_test_and_set(cpu, next->cpu_vm_mask)) {
			/* We were in lazy tlb mode and leave_mm disabled 
			 * tlb flush IPI delivery. We must reload CR3
			 * to make sure to use no freed page tables.
			 */
			load_cr3(next->pgd);
			load_LDT_nolock(&next->context, cpu);
		}
	}
#endif
}

#define deactivate_mm(tsk,mm)	do { \
	load_gs_index(0); \
	asm volatile("movl %0,%%fs"::"r"(0));  \
} while(0)

#define activate_mm(prev, next) \
	switch_mm((prev),(next),NULL)


#endif
Commit	Line	Data
1da177e4 LT	1	#ifndef __X86_64_MMU_CONTEXT_H
	2	#define __X86_64_MMU_CONTEXT_H
	3
1da177e4 LT	4	#include <asm/desc.h>
	5	#include <asm/atomic.h>
	6	#include <asm/pgalloc.h>
	7	#include <asm/pda.h>
	8	#include <asm/pgtable.h>
	9	#include <asm/tlbflush.h>
	10
	11	/*
	12	* possibly do the LDT unload here?
	13	*/
	14	int init_new_context(struct task_struct tsk, struct mm_struct mm);
	15	void destroy_context(struct mm_struct *mm);
	16
1da177e4 LT	17	static inline void enter_lazy_tlb(struct mm_struct mm, struct task_struct tsk)
1da177e4 LT	18	{
e4b5939a	19	#ifdef CONFIG_SMP
1da177e4 LT	20	if (read_pda(mmu_state) == TLBSTATE_OK)
1da177e4 LT	21	write_pda(mmu_state, TLBSTATE_LAZY);
1da177e4	22	#endif
e4b5939a	23	}
1da177e4 LT	24
	25	static inline void load_cr3(pgd_t *pgd)
	26	{
	27	asm volatile("movq %0,%%cr3" :: "r" (__pa(pgd)) : "memory");
	28	}
	29
	30	static inline void switch_mm(struct mm_struct prev, struct mm_struct next,
	31	struct task_struct *tsk)
	32	{
	33	unsigned cpu = smp_processor_id();
	34	if (likely(prev != next)) {
	35	/* stop flush ipis for the previous mm */
3d1712c9	36	cpu_clear(cpu, prev->cpu_vm_mask);
1da177e4 LT	37	#ifdef CONFIG_SMP
	38	write_pda(mmu_state, TLBSTATE_OK);
	39	write_pda(active_mm, next);
	40	#endif
3d1712c9	41	cpu_set(cpu, next->cpu_vm_mask);
1da177e4 LT	42	load_cr3(next->pgd);
	43
	44	if (unlikely(next->context.ldt != prev->context.ldt))
	45	load_LDT_nolock(&next->context, cpu);
	46	}
	47	#ifdef CONFIG_SMP
	48	else {
	49	write_pda(mmu_state, TLBSTATE_OK);
	50	if (read_pda(active_mm) != next)
	51	out_of_line_bug();
3d1712c9	52	if (!cpu_test_and_set(cpu, next->cpu_vm_mask)) {
1da177e4 LT	53	/* We were in lazy tlb mode and leave_mm disabled
	54	* tlb flush IPI delivery. We must reload CR3
	55	* to make sure to use no freed page tables.
	56	*/
	57	load_cr3(next->pgd);
	58	load_LDT_nolock(&next->context, cpu);
	59	}
	60	}
	61	#endif
	62	}
	63
	64	#define deactivate_mm(tsk,mm) do { \
	65	load_gs_index(0); \
	66	asm volatile("movl %0,%%fs"::"r"(0)); \
	67	} while(0)
	68
	69	#define activate_mm(prev, next) \
	70	switch_mm((prev),(next),NULL)
	71
	72
	73	#endif