]>
Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * Copyright (C) 2000, 2001, 2002 Jeff Dike (jdike@karaya.com) | |
3 | * Licensed under the GPL | |
4 | */ | |
5 | ||
6 | #include "linux/mm.h" | |
7 | #include "asm/page.h" | |
8 | #include "asm/pgalloc.h" | |
9 | #include "asm/tlbflush.h" | |
10 | #include "choose-mode.h" | |
11 | #include "mode_kern.h" | |
12 | #include "user_util.h" | |
13 | #include "tlb.h" | |
14 | #include "mem.h" | |
15 | #include "mem_user.h" | |
16 | #include "os.h" | |
17 | ||
18 | #define ADD_ROUND(n, inc) (((n) + (inc)) & ~((inc) - 1)) | |
19 | ||
20 | void fix_range_common(struct mm_struct *mm, unsigned long start_addr, | |
21 | unsigned long end_addr, int force, int data, | |
22 | void (*do_ops)(int, struct host_vm_op *, int)) | |
23 | { | |
24 | pgd_t *npgd; | |
25 | pud_t *npud; | |
26 | pmd_t *npmd; | |
27 | pte_t *npte; | |
28 | unsigned long addr, end; | |
29 | int r, w, x; | |
30 | struct host_vm_op ops[16]; | |
31 | int op_index = -1, last_op = sizeof(ops) / sizeof(ops[0]) - 1; | |
32 | ||
33 | if(mm == NULL) return; | |
34 | ||
35 | for(addr = start_addr; addr < end_addr;){ | |
36 | npgd = pgd_offset(mm, addr); | |
37 | if(!pgd_present(*npgd)){ | |
38 | end = ADD_ROUND(addr, PGDIR_SIZE); | |
39 | if(end > end_addr) | |
40 | end = end_addr; | |
41 | if(force || pgd_newpage(*npgd)){ | |
42 | op_index = add_munmap(addr, end - addr, ops, | |
43 | op_index, last_op, data, | |
44 | do_ops); | |
45 | pgd_mkuptodate(*npgd); | |
46 | } | |
47 | addr = end; | |
48 | continue; | |
49 | } | |
50 | ||
51 | npud = pud_offset(npgd, addr); | |
52 | if(!pud_present(*npud)){ | |
53 | end = ADD_ROUND(addr, PUD_SIZE); | |
54 | if(end > end_addr) | |
55 | end = end_addr; | |
56 | if(force || pud_newpage(*npud)){ | |
57 | op_index = add_munmap(addr, end - addr, ops, | |
58 | op_index, last_op, data, | |
59 | do_ops); | |
60 | pud_mkuptodate(*npud); | |
61 | } | |
62 | addr = end; | |
63 | continue; | |
64 | } | |
65 | ||
66 | npmd = pmd_offset(npud, addr); | |
67 | if(!pmd_present(*npmd)){ | |
68 | end = ADD_ROUND(addr, PMD_SIZE); | |
69 | if(end > end_addr) | |
70 | end = end_addr; | |
71 | if(force || pmd_newpage(*npmd)){ | |
72 | op_index = add_munmap(addr, end - addr, ops, | |
73 | op_index, last_op, data, | |
74 | do_ops); | |
75 | pmd_mkuptodate(*npmd); | |
76 | } | |
77 | addr = end; | |
78 | continue; | |
79 | } | |
80 | ||
81 | npte = pte_offset_kernel(npmd, addr); | |
82 | r = pte_read(*npte); | |
83 | w = pte_write(*npte); | |
84 | x = pte_exec(*npte); | |
85 | if(!pte_dirty(*npte)) | |
86 | w = 0; | |
87 | if(!pte_young(*npte)){ | |
88 | r = 0; | |
89 | w = 0; | |
90 | } | |
91 | if(force || pte_newpage(*npte)){ | |
92 | if(pte_present(*npte)) | |
93 | op_index = add_mmap(addr, | |
94 | pte_val(*npte) & PAGE_MASK, | |
95 | PAGE_SIZE, r, w, x, ops, | |
96 | op_index, last_op, data, | |
97 | do_ops); | |
98 | else op_index = add_munmap(addr, PAGE_SIZE, ops, | |
99 | op_index, last_op, data, | |
100 | do_ops); | |
101 | } | |
102 | else if(pte_newprot(*npte)) | |
103 | op_index = add_mprotect(addr, PAGE_SIZE, r, w, x, ops, | |
104 | op_index, last_op, data, | |
105 | do_ops); | |
106 | ||
107 | *npte = pte_mkuptodate(*npte); | |
108 | addr += PAGE_SIZE; | |
109 | } | |
110 | (*do_ops)(data, ops, op_index); | |
111 | } | |
112 | ||
113 | int flush_tlb_kernel_range_common(unsigned long start, unsigned long end) | |
114 | { | |
115 | struct mm_struct *mm; | |
116 | pgd_t *pgd; | |
117 | pud_t *pud; | |
118 | pmd_t *pmd; | |
119 | pte_t *pte; | |
120 | unsigned long addr, last; | |
121 | int updated = 0, err; | |
122 | ||
123 | mm = &init_mm; | |
124 | for(addr = start; addr < end;){ | |
125 | pgd = pgd_offset(mm, addr); | |
126 | if(!pgd_present(*pgd)){ | |
127 | last = ADD_ROUND(addr, PGDIR_SIZE); | |
128 | if(last > end) | |
129 | last = end; | |
130 | if(pgd_newpage(*pgd)){ | |
131 | updated = 1; | |
132 | err = os_unmap_memory((void *) addr, | |
133 | last - addr); | |
134 | if(err < 0) | |
135 | panic("munmap failed, errno = %d\n", | |
136 | -err); | |
137 | } | |
138 | addr = last; | |
139 | continue; | |
140 | } | |
141 | ||
142 | pud = pud_offset(pgd, addr); | |
143 | if(!pud_present(*pud)){ | |
144 | last = ADD_ROUND(addr, PUD_SIZE); | |
145 | if(last > end) | |
146 | last = end; | |
147 | if(pud_newpage(*pud)){ | |
148 | updated = 1; | |
149 | err = os_unmap_memory((void *) addr, | |
150 | last - addr); | |
151 | if(err < 0) | |
152 | panic("munmap failed, errno = %d\n", | |
153 | -err); | |
154 | } | |
155 | addr = last; | |
156 | continue; | |
157 | } | |
158 | ||
159 | pmd = pmd_offset(pud, addr); | |
160 | if(!pmd_present(*pmd)){ | |
161 | last = ADD_ROUND(addr, PMD_SIZE); | |
162 | if(last > end) | |
163 | last = end; | |
164 | if(pmd_newpage(*pmd)){ | |
165 | updated = 1; | |
166 | err = os_unmap_memory((void *) addr, | |
167 | last - addr); | |
168 | if(err < 0) | |
169 | panic("munmap failed, errno = %d\n", | |
170 | -err); | |
171 | } | |
172 | addr = last; | |
173 | continue; | |
174 | } | |
175 | ||
176 | pte = pte_offset_kernel(pmd, addr); | |
177 | if(!pte_present(*pte) || pte_newpage(*pte)){ | |
178 | updated = 1; | |
179 | err = os_unmap_memory((void *) addr, | |
180 | PAGE_SIZE); | |
181 | if(err < 0) | |
182 | panic("munmap failed, errno = %d\n", | |
183 | -err); | |
184 | if(pte_present(*pte)) | |
185 | map_memory(addr, | |
186 | pte_val(*pte) & PAGE_MASK, | |
187 | PAGE_SIZE, 1, 1, 1); | |
188 | } | |
189 | else if(pte_newprot(*pte)){ | |
190 | updated = 1; | |
191 | protect_memory(addr, PAGE_SIZE, 1, 1, 1, 1); | |
192 | } | |
193 | addr += PAGE_SIZE; | |
194 | } | |
195 | return(updated); | |
196 | } | |
197 | ||
198 | void flush_tlb_page(struct vm_area_struct *vma, unsigned long address) | |
199 | { | |
200 | address &= PAGE_MASK; | |
201 | flush_tlb_range(vma, address, address + PAGE_SIZE); | |
202 | } | |
203 | ||
204 | void flush_tlb_all(void) | |
205 | { | |
206 | flush_tlb_mm(current->mm); | |
207 | } | |
208 | ||
209 | void flush_tlb_kernel_range(unsigned long start, unsigned long end) | |
210 | { | |
211 | CHOOSE_MODE_PROC(flush_tlb_kernel_range_tt, | |
212 | flush_tlb_kernel_range_common, start, end); | |
213 | } | |
214 | ||
215 | void flush_tlb_kernel_vm(void) | |
216 | { | |
217 | CHOOSE_MODE(flush_tlb_kernel_vm_tt(), | |
218 | flush_tlb_kernel_range_common(start_vm, end_vm)); | |
219 | } | |
220 | ||
221 | void __flush_tlb_one(unsigned long addr) | |
222 | { | |
223 | CHOOSE_MODE_PROC(__flush_tlb_one_tt, __flush_tlb_one_skas, addr); | |
224 | } | |
225 | ||
226 | void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, | |
227 | unsigned long end) | |
228 | { | |
229 | CHOOSE_MODE_PROC(flush_tlb_range_tt, flush_tlb_range_skas, vma, start, | |
230 | end); | |
231 | } | |
232 | ||
233 | void flush_tlb_mm(struct mm_struct *mm) | |
234 | { | |
235 | CHOOSE_MODE_PROC(flush_tlb_mm_tt, flush_tlb_mm_skas, mm); | |
236 | } | |
237 | ||
238 | void force_flush_all(void) | |
239 | { | |
240 | CHOOSE_MODE(force_flush_all_tt(), force_flush_all_skas()); | |
241 | } | |
242 | ||
243 | pgd_t *pgd_offset_proc(struct mm_struct *mm, unsigned long address) | |
244 | { | |
245 | return(pgd_offset(mm, address)); | |
246 | } | |
247 | ||
248 | pud_t *pud_offset_proc(pgd_t *pgd, unsigned long address) | |
249 | { | |
250 | return(pud_offset(pgd, address)); | |
251 | } | |
252 | ||
253 | pmd_t *pmd_offset_proc(pud_t *pud, unsigned long address) | |
254 | { | |
255 | return(pmd_offset(pud, address)); | |
256 | } | |
257 | ||
258 | pte_t *pte_offset_proc(pmd_t *pmd, unsigned long address) | |
259 | { | |
260 | return(pte_offset_kernel(pmd, address)); | |
261 | } | |
262 | ||
263 | pte_t *addr_pte(struct task_struct *task, unsigned long addr) | |
264 | { | |
265 | pgd_t *pgd = pgd_offset(task->mm, addr); | |
266 | pud_t *pud = pud_offset(pgd, addr); | |
267 | pmd_t *pmd = pmd_offset(pud, addr); | |
268 | ||
269 | return(pte_offset_map(pmd, addr)); | |
270 | } | |
271 | ||
272 | int add_mmap(unsigned long virt, unsigned long phys, unsigned long len, | |
273 | int r, int w, int x, struct host_vm_op *ops, int index, | |
274 | int last_filled, int data, | |
275 | void (*do_ops)(int, struct host_vm_op *, int)) | |
276 | { | |
277 | __u64 offset; | |
278 | struct host_vm_op *last; | |
279 | int fd; | |
280 | ||
281 | fd = phys_mapping(phys, &offset); | |
282 | if(index != -1){ | |
283 | last = &ops[index]; | |
284 | if((last->type == MMAP) && | |
285 | (last->u.mmap.addr + last->u.mmap.len == virt) && | |
286 | (last->u.mmap.r == r) && (last->u.mmap.w == w) && | |
287 | (last->u.mmap.x == x) && (last->u.mmap.fd == fd) && | |
288 | (last->u.mmap.offset + last->u.mmap.len == offset)){ | |
289 | last->u.mmap.len += len; | |
290 | return(index); | |
291 | } | |
292 | } | |
293 | ||
294 | if(index == last_filled){ | |
295 | (*do_ops)(data, ops, last_filled); | |
296 | index = -1; | |
297 | } | |
298 | ||
299 | ops[++index] = ((struct host_vm_op) { .type = MMAP, | |
300 | .u = { .mmap = { | |
301 | .addr = virt, | |
302 | .len = len, | |
303 | .r = r, | |
304 | .w = w, | |
305 | .x = x, | |
306 | .fd = fd, | |
307 | .offset = offset } | |
308 | } }); | |
309 | return(index); | |
310 | } | |
311 | ||
312 | int add_munmap(unsigned long addr, unsigned long len, struct host_vm_op *ops, | |
313 | int index, int last_filled, int data, | |
314 | void (*do_ops)(int, struct host_vm_op *, int)) | |
315 | { | |
316 | struct host_vm_op *last; | |
317 | ||
318 | if(index != -1){ | |
319 | last = &ops[index]; | |
320 | if((last->type == MUNMAP) && | |
321 | (last->u.munmap.addr + last->u.mmap.len == addr)){ | |
322 | last->u.munmap.len += len; | |
323 | return(index); | |
324 | } | |
325 | } | |
326 | ||
327 | if(index == last_filled){ | |
328 | (*do_ops)(data, ops, last_filled); | |
329 | index = -1; | |
330 | } | |
331 | ||
332 | ops[++index] = ((struct host_vm_op) { .type = MUNMAP, | |
333 | .u = { .munmap = { | |
334 | .addr = addr, | |
335 | .len = len } } }); | |
336 | return(index); | |
337 | } | |
338 | ||
339 | int add_mprotect(unsigned long addr, unsigned long len, int r, int w, int x, | |
340 | struct host_vm_op *ops, int index, int last_filled, int data, | |
341 | void (*do_ops)(int, struct host_vm_op *, int)) | |
342 | { | |
343 | struct host_vm_op *last; | |
344 | ||
345 | if(index != -1){ | |
346 | last = &ops[index]; | |
347 | if((last->type == MPROTECT) && | |
348 | (last->u.mprotect.addr + last->u.mprotect.len == addr) && | |
349 | (last->u.mprotect.r == r) && (last->u.mprotect.w == w) && | |
350 | (last->u.mprotect.x == x)){ | |
351 | last->u.mprotect.len += len; | |
352 | return(index); | |
353 | } | |
354 | } | |
355 | ||
356 | if(index == last_filled){ | |
357 | (*do_ops)(data, ops, last_filled); | |
358 | index = -1; | |
359 | } | |
360 | ||
361 | ops[++index] = ((struct host_vm_op) { .type = MPROTECT, | |
362 | .u = { .mprotect = { | |
363 | .addr = addr, | |
364 | .len = len, | |
365 | .r = r, | |
366 | .w = w, | |
367 | .x = x } } }); | |
368 | return(index); | |
369 | } |