]>
Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | #ifndef _I386_TLBFLUSH_H |
2 | #define _I386_TLBFLUSH_H | |
3 | ||
1da177e4 LT |
4 | #include <linux/mm.h> |
5 | #include <asm/processor.h> | |
6 | ||
da181a8b RR |
7 | #ifdef CONFIG_PARAVIRT |
8 | #include <asm/paravirt.h> | |
9 | #else | |
10 | #define __flush_tlb() __native_flush_tlb() | |
11 | #define __flush_tlb_global() __native_flush_tlb_global() | |
12 | #define __flush_tlb_single(addr) __native_flush_tlb_single(addr) | |
13 | #endif | |
14 | ||
15 | #define __native_flush_tlb() \ | |
1da177e4 LT |
16 | do { \ |
17 | unsigned int tmpreg; \ | |
18 | \ | |
19 | __asm__ __volatile__( \ | |
20 | "movl %%cr3, %0; \n" \ | |
21 | "movl %0, %%cr3; # flush TLB \n" \ | |
22 | : "=r" (tmpreg) \ | |
23 | :: "memory"); \ | |
24 | } while (0) | |
25 | ||
26 | /* | |
27 | * Global pages have to be flushed a bit differently. Not a real | |
28 | * performance problem because this does not happen often. | |
29 | */ | |
da181a8b | 30 | #define __native_flush_tlb_global() \ |
1da177e4 | 31 | do { \ |
ffaa8bd6 | 32 | unsigned int tmpreg, cr4, cr4_orig; \ |
1da177e4 LT |
33 | \ |
34 | __asm__ __volatile__( \ | |
ffaa8bd6 AA |
35 | "movl %%cr4, %2; # turn off PGE \n" \ |
36 | "movl %2, %1; \n" \ | |
37 | "andl %3, %1; \n" \ | |
38 | "movl %1, %%cr4; \n" \ | |
1da177e4 LT |
39 | "movl %%cr3, %0; \n" \ |
40 | "movl %0, %%cr3; # flush TLB \n" \ | |
41 | "movl %2, %%cr4; # turn PGE back on \n" \ | |
ffaa8bd6 AA |
42 | : "=&r" (tmpreg), "=&r" (cr4), "=&r" (cr4_orig) \ |
43 | : "i" (~X86_CR4_PGE) \ | |
1da177e4 LT |
44 | : "memory"); \ |
45 | } while (0) | |
46 | ||
da181a8b RR |
47 | #define __native_flush_tlb_single(addr) \ |
48 | __asm__ __volatile__("invlpg (%0)" ::"r" (addr) : "memory") | |
49 | ||
1da177e4 LT |
50 | # define __flush_tlb_all() \ |
51 | do { \ | |
52 | if (cpu_has_pge) \ | |
53 | __flush_tlb_global(); \ | |
54 | else \ | |
55 | __flush_tlb(); \ | |
56 | } while (0) | |
57 | ||
58 | #define cpu_has_invlpg (boot_cpu_data.x86 > 3) | |
59 | ||
1da177e4 LT |
60 | #ifdef CONFIG_X86_INVLPG |
61 | # define __flush_tlb_one(addr) __flush_tlb_single(addr) | |
62 | #else | |
63 | # define __flush_tlb_one(addr) \ | |
64 | do { \ | |
65 | if (cpu_has_invlpg) \ | |
66 | __flush_tlb_single(addr); \ | |
67 | else \ | |
68 | __flush_tlb(); \ | |
69 | } while (0) | |
70 | #endif | |
71 | ||
72 | /* | |
73 | * TLB flushing: | |
74 | * | |
75 | * - flush_tlb() flushes the current mm struct TLBs | |
76 | * - flush_tlb_all() flushes all processes TLBs | |
77 | * - flush_tlb_mm(mm) flushes the specified mm context TLB's | |
78 | * - flush_tlb_page(vma, vmaddr) flushes one page | |
79 | * - flush_tlb_range(vma, start, end) flushes a range of pages | |
80 | * - flush_tlb_kernel_range(start, end) flushes a range of kernel pages | |
81 | * - flush_tlb_pgtables(mm, start, end) flushes a range of page tables | |
d4c10477 | 82 | * - flush_tlb_others(cpumask, mm, va) flushes a TLBs on other cpus |
1da177e4 LT |
83 | * |
84 | * ..but the i386 has somewhat limited tlb flushing capabilities, | |
85 | * and page-granular flushes are available only on i486 and up. | |
86 | */ | |
87 | ||
d4c10477 JF |
88 | #define TLB_FLUSH_ALL 0xffffffff |
89 | ||
90 | ||
1da177e4 LT |
91 | #ifndef CONFIG_SMP |
92 | ||
e8edc6e0 AD |
93 | #include <linux/sched.h> |
94 | ||
1da177e4 LT |
95 | #define flush_tlb() __flush_tlb() |
96 | #define flush_tlb_all() __flush_tlb_all() | |
97 | #define local_flush_tlb() __flush_tlb() | |
98 | ||
99 | static inline void flush_tlb_mm(struct mm_struct *mm) | |
100 | { | |
101 | if (mm == current->active_mm) | |
102 | __flush_tlb(); | |
103 | } | |
104 | ||
105 | static inline void flush_tlb_page(struct vm_area_struct *vma, | |
106 | unsigned long addr) | |
107 | { | |
108 | if (vma->vm_mm == current->active_mm) | |
109 | __flush_tlb_one(addr); | |
110 | } | |
111 | ||
112 | static inline void flush_tlb_range(struct vm_area_struct *vma, | |
113 | unsigned long start, unsigned long end) | |
114 | { | |
115 | if (vma->vm_mm == current->active_mm) | |
116 | __flush_tlb(); | |
117 | } | |
118 | ||
d4c10477 JF |
119 | static inline void native_flush_tlb_others(const cpumask_t *cpumask, |
120 | struct mm_struct *mm, unsigned long va) | |
121 | { | |
122 | } | |
123 | ||
124 | #else /* SMP */ | |
1da177e4 LT |
125 | |
126 | #include <asm/smp.h> | |
127 | ||
128 | #define local_flush_tlb() \ | |
129 | __flush_tlb() | |
130 | ||
131 | extern void flush_tlb_all(void); | |
132 | extern void flush_tlb_current_task(void); | |
133 | extern void flush_tlb_mm(struct mm_struct *); | |
134 | extern void flush_tlb_page(struct vm_area_struct *, unsigned long); | |
135 | ||
136 | #define flush_tlb() flush_tlb_current_task() | |
137 | ||
138 | static inline void flush_tlb_range(struct vm_area_struct * vma, unsigned long start, unsigned long end) | |
139 | { | |
140 | flush_tlb_mm(vma->vm_mm); | |
141 | } | |
142 | ||
d4c10477 JF |
143 | void native_flush_tlb_others(const cpumask_t *cpumask, struct mm_struct *mm, |
144 | unsigned long va); | |
145 | ||
1da177e4 LT |
146 | #define TLBSTATE_OK 1 |
147 | #define TLBSTATE_LAZY 2 | |
148 | ||
149 | struct tlb_state | |
150 | { | |
151 | struct mm_struct *active_mm; | |
152 | int state; | |
153 | char __cacheline_padding[L1_CACHE_BYTES-8]; | |
154 | }; | |
155 | DECLARE_PER_CPU(struct tlb_state, cpu_tlbstate); | |
d4c10477 | 156 | #endif /* SMP */ |
1da177e4 | 157 | |
d4c10477 JF |
158 | #ifndef CONFIG_PARAVIRT |
159 | #define flush_tlb_others(mask, mm, va) \ | |
160 | native_flush_tlb_others(&mask, mm, va) | |
1da177e4 LT |
161 | #endif |
162 | ||
45902954 AM |
163 | static inline void flush_tlb_kernel_range(unsigned long start, |
164 | unsigned long end) | |
165 | { | |
166 | flush_tlb_all(); | |
167 | } | |
1da177e4 LT |
168 | |
169 | static inline void flush_tlb_pgtables(struct mm_struct *mm, | |
170 | unsigned long start, unsigned long end) | |
171 | { | |
172 | /* i386 does not keep any page table caches in TLB */ | |
173 | } | |
174 | ||
175 | #endif /* _I386_TLBFLUSH_H */ |