]>
git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blob - mm/mremap.c
4 * (C) Copyright 1996 Linus Torvalds
6 * Address space accounting code <alan@redhat.com>
7 * (C) Copyright 2002 Red Hat Inc, All Rights Reserved
11 #include <linux/hugetlb.h>
12 #include <linux/slab.h>
13 #include <linux/shm.h>
14 #include <linux/mman.h>
15 #include <linux/swap.h>
16 #include <linux/capability.h>
18 #include <linux/highmem.h>
19 #include <linux/security.h>
20 #include <linux/syscalls.h>
22 #include <asm/uaccess.h>
23 #include <asm/cacheflush.h>
24 #include <asm/tlbflush.h>
26 static pmd_t
*get_old_pmd(struct mm_struct
*mm
, unsigned long addr
)
32 pgd
= pgd_offset(mm
, addr
);
33 if (pgd_none_or_clear_bad(pgd
))
36 pud
= pud_offset(pgd
, addr
);
37 if (pud_none_or_clear_bad(pud
))
40 pmd
= pmd_offset(pud
, addr
);
41 if (pmd_none_or_clear_bad(pmd
))
47 static pmd_t
*alloc_new_pmd(struct mm_struct
*mm
, unsigned long addr
)
53 pgd
= pgd_offset(mm
, addr
);
54 pud
= pud_alloc(mm
, pgd
, addr
);
58 pmd
= pmd_alloc(mm
, pud
, addr
);
62 if (!pmd_present(*pmd
) && __pte_alloc(mm
, pmd
, addr
))
68 static void move_ptes(struct vm_area_struct
*vma
, pmd_t
*old_pmd
,
69 unsigned long old_addr
, unsigned long old_end
,
70 struct vm_area_struct
*new_vma
, pmd_t
*new_pmd
,
71 unsigned long new_addr
)
73 struct address_space
*mapping
= NULL
;
74 struct mm_struct
*mm
= vma
->vm_mm
;
75 pte_t
*old_pte
, *new_pte
, pte
;
76 spinlock_t
*old_ptl
, *new_ptl
;
80 * Subtle point from Rajesh Venkatasubramanian: before
81 * moving file-based ptes, we must lock vmtruncate out,
82 * since it might clean the dst vma before the src vma,
83 * and we propagate stale pages into the dst afterward.
85 mapping
= vma
->vm_file
->f_mapping
;
86 spin_lock(&mapping
->i_mmap_lock
);
87 if (new_vma
->vm_truncate_count
&&
88 new_vma
->vm_truncate_count
!= vma
->vm_truncate_count
)
89 new_vma
->vm_truncate_count
= 0;
93 * We don't have to worry about the ordering of src and dst
94 * pte locks because exclusive mmap_sem prevents deadlock.
96 old_pte
= pte_offset_map_lock(mm
, old_pmd
, old_addr
, &old_ptl
);
97 new_pte
= pte_offset_map_nested(new_pmd
, new_addr
);
98 new_ptl
= pte_lockptr(mm
, new_pmd
);
99 if (new_ptl
!= old_ptl
)
102 for (; old_addr
< old_end
; old_pte
++, old_addr
+= PAGE_SIZE
,
103 new_pte
++, new_addr
+= PAGE_SIZE
) {
104 if (pte_none(*old_pte
))
106 pte
= ptep_clear_flush(vma
, old_addr
, old_pte
);
107 /* ZERO_PAGE can be dependant on virtual addr */
108 pte
= move_pte(pte
, new_vma
->vm_page_prot
, old_addr
, new_addr
);
109 set_pte_at(mm
, new_addr
, new_pte
, pte
);
112 if (new_ptl
!= old_ptl
)
113 spin_unlock(new_ptl
);
114 pte_unmap_nested(new_pte
- 1);
115 pte_unmap_unlock(old_pte
- 1, old_ptl
);
117 spin_unlock(&mapping
->i_mmap_lock
);
120 #define LATENCY_LIMIT (64 * PAGE_SIZE)
122 static unsigned long move_page_tables(struct vm_area_struct
*vma
,
123 unsigned long old_addr
, struct vm_area_struct
*new_vma
,
124 unsigned long new_addr
, unsigned long len
)
126 unsigned long extent
, next
, old_end
;
127 pmd_t
*old_pmd
, *new_pmd
;
129 old_end
= old_addr
+ len
;
130 flush_cache_range(vma
, old_addr
, old_end
);
132 for (; old_addr
< old_end
; old_addr
+= extent
, new_addr
+= extent
) {
134 next
= (old_addr
+ PMD_SIZE
) & PMD_MASK
;
135 if (next
- 1 > old_end
)
137 extent
= next
- old_addr
;
138 old_pmd
= get_old_pmd(vma
->vm_mm
, old_addr
);
141 new_pmd
= alloc_new_pmd(vma
->vm_mm
, new_addr
);
144 next
= (new_addr
+ PMD_SIZE
) & PMD_MASK
;
145 if (extent
> next
- new_addr
)
146 extent
= next
- new_addr
;
147 if (extent
> LATENCY_LIMIT
)
148 extent
= LATENCY_LIMIT
;
149 move_ptes(vma
, old_pmd
, old_addr
, old_addr
+ extent
,
150 new_vma
, new_pmd
, new_addr
);
153 return len
+ old_addr
- old_end
; /* how much done */
156 static unsigned long move_vma(struct vm_area_struct
*vma
,
157 unsigned long old_addr
, unsigned long old_len
,
158 unsigned long new_len
, unsigned long new_addr
)
160 struct mm_struct
*mm
= vma
->vm_mm
;
161 struct vm_area_struct
*new_vma
;
162 unsigned long vm_flags
= vma
->vm_flags
;
163 unsigned long new_pgoff
;
164 unsigned long moved_len
;
165 unsigned long excess
= 0;
166 unsigned long hiwater_vm
;
170 * We'd prefer to avoid failure later on in do_munmap:
171 * which may split one vma into three before unmapping.
173 if (mm
->map_count
>= sysctl_max_map_count
- 3)
176 new_pgoff
= vma
->vm_pgoff
+ ((old_addr
- vma
->vm_start
) >> PAGE_SHIFT
);
177 new_vma
= copy_vma(&vma
, new_addr
, new_len
, new_pgoff
);
181 moved_len
= move_page_tables(vma
, old_addr
, new_vma
, new_addr
, old_len
);
182 if (moved_len
< old_len
) {
184 * On error, move entries back from new area to old,
185 * which will succeed since page tables still there,
186 * and then proceed to unmap new area instead of old.
188 move_page_tables(new_vma
, new_addr
, vma
, old_addr
, moved_len
);
195 /* Conceal VM_ACCOUNT so old reservation is not undone */
196 if (vm_flags
& VM_ACCOUNT
) {
197 vma
->vm_flags
&= ~VM_ACCOUNT
;
198 excess
= vma
->vm_end
- vma
->vm_start
- old_len
;
199 if (old_addr
> vma
->vm_start
&&
200 old_addr
+ old_len
< vma
->vm_end
)
205 * If we failed to move page tables we still do total_vm increment
206 * since do_munmap() will decrement it by old_len == new_len.
208 * Since total_vm is about to be raised artificially high for a
209 * moment, we need to restore high watermark afterwards: if stats
210 * are taken meanwhile, total_vm and hiwater_vm appear too high.
211 * If this were a serious issue, we'd add a flag to do_munmap().
213 hiwater_vm
= mm
->hiwater_vm
;
214 mm
->total_vm
+= new_len
>> PAGE_SHIFT
;
215 vm_stat_account(mm
, vma
->vm_flags
, vma
->vm_file
, new_len
>>PAGE_SHIFT
);
217 if (do_munmap(mm
, old_addr
, old_len
) < 0) {
218 /* OOM: unable to split vma, just get accounts right */
219 vm_unacct_memory(excess
>> PAGE_SHIFT
);
222 mm
->hiwater_vm
= hiwater_vm
;
224 /* Restore VM_ACCOUNT if one or two pieces of vma left */
226 vma
->vm_flags
|= VM_ACCOUNT
;
228 vma
->vm_next
->vm_flags
|= VM_ACCOUNT
;
231 if (vm_flags
& VM_LOCKED
) {
232 mm
->locked_vm
+= new_len
>> PAGE_SHIFT
;
233 if (new_len
> old_len
)
234 make_pages_present(new_addr
+ old_len
,
242 * Expand (or shrink) an existing mapping, potentially moving it at the
243 * same time (controlled by the MREMAP_MAYMOVE flag and available VM space)
245 * MREMAP_FIXED option added 5-Dec-1999 by Benjamin LaHaise
246 * This option implies MREMAP_MAYMOVE.
248 unsigned long do_mremap(unsigned long addr
,
249 unsigned long old_len
, unsigned long new_len
,
250 unsigned long flags
, unsigned long new_addr
)
252 struct mm_struct
*mm
= current
->mm
;
253 struct vm_area_struct
*vma
;
254 unsigned long ret
= -EINVAL
;
255 unsigned long charged
= 0;
257 if (flags
& ~(MREMAP_FIXED
| MREMAP_MAYMOVE
))
260 if (addr
& ~PAGE_MASK
)
263 old_len
= PAGE_ALIGN(old_len
);
264 new_len
= PAGE_ALIGN(new_len
);
267 * We allow a zero old-len as a special case
268 * for DOS-emu "duplicate shm area" thing. But
269 * a zero new-len is nonsensical.
274 /* new_addr is only valid if MREMAP_FIXED is specified */
275 if (flags
& MREMAP_FIXED
) {
276 if (new_addr
& ~PAGE_MASK
)
278 if (!(flags
& MREMAP_MAYMOVE
))
281 if (new_len
> TASK_SIZE
|| new_addr
> TASK_SIZE
- new_len
)
284 /* Check if the location we're moving into overlaps the
285 * old location at all, and fail if it does.
287 if ((new_addr
<= addr
) && (new_addr
+new_len
) > addr
)
290 if ((addr
<= new_addr
) && (addr
+old_len
) > new_addr
)
293 ret
= do_munmap(mm
, new_addr
, new_len
);
299 * Always allow a shrinking remap: that just unmaps
300 * the unnecessary pages..
301 * do_munmap does all the needed commit accounting
303 if (old_len
>= new_len
) {
304 ret
= do_munmap(mm
, addr
+new_len
, old_len
- new_len
);
305 if (ret
&& old_len
!= new_len
)
308 if (!(flags
& MREMAP_FIXED
) || (new_addr
== addr
))
314 * Ok, we need to grow.. or relocate.
317 vma
= find_vma(mm
, addr
);
318 if (!vma
|| vma
->vm_start
> addr
)
320 if (is_vm_hugetlb_page(vma
)) {
324 /* We can't remap across vm area boundaries */
325 if (old_len
> vma
->vm_end
- addr
)
327 if (vma
->vm_flags
& (VM_DONTEXPAND
| VM_PFNMAP
)) {
328 if (new_len
> old_len
)
331 if (vma
->vm_flags
& VM_LOCKED
) {
332 unsigned long locked
, lock_limit
;
333 locked
= mm
->locked_vm
<< PAGE_SHIFT
;
334 lock_limit
= current
->signal
->rlim
[RLIMIT_MEMLOCK
].rlim_cur
;
335 locked
+= new_len
- old_len
;
337 if (locked
> lock_limit
&& !capable(CAP_IPC_LOCK
))
340 if (!may_expand_vm(mm
, (new_len
- old_len
) >> PAGE_SHIFT
)) {
345 if (vma
->vm_flags
& VM_ACCOUNT
) {
346 charged
= (new_len
- old_len
) >> PAGE_SHIFT
;
347 if (security_vm_enough_memory(charged
))
351 /* old_len exactly to the end of the area..
352 * And we're not relocating the area.
354 if (old_len
== vma
->vm_end
- addr
&&
355 !((flags
& MREMAP_FIXED
) && (addr
!= new_addr
)) &&
356 (old_len
!= new_len
|| !(flags
& MREMAP_MAYMOVE
))) {
357 unsigned long max_addr
= TASK_SIZE
;
359 max_addr
= vma
->vm_next
->vm_start
;
360 /* can we just expand the current mapping? */
361 if (max_addr
- addr
>= new_len
) {
362 int pages
= (new_len
- old_len
) >> PAGE_SHIFT
;
364 vma_adjust(vma
, vma
->vm_start
,
365 addr
+ new_len
, vma
->vm_pgoff
, NULL
);
367 mm
->total_vm
+= pages
;
368 vm_stat_account(mm
, vma
->vm_flags
, vma
->vm_file
, pages
);
369 if (vma
->vm_flags
& VM_LOCKED
) {
370 mm
->locked_vm
+= pages
;
371 make_pages_present(addr
+ old_len
,
380 * We weren't able to just expand or shrink the area,
381 * we need to create a new one and move it..
384 if (flags
& MREMAP_MAYMOVE
) {
385 if (!(flags
& MREMAP_FIXED
)) {
386 unsigned long map_flags
= 0;
387 if (vma
->vm_flags
& VM_MAYSHARE
)
388 map_flags
|= MAP_SHARED
;
390 new_addr
= get_unmapped_area(vma
->vm_file
, 0, new_len
,
391 vma
->vm_pgoff
, map_flags
);
393 if (new_addr
& ~PAGE_MASK
)
396 ret
= move_vma(vma
, addr
, old_len
, new_len
, new_addr
);
399 if (ret
& ~PAGE_MASK
)
400 vm_unacct_memory(charged
);
405 asmlinkage
unsigned long sys_mremap(unsigned long addr
,
406 unsigned long old_len
, unsigned long new_len
,
407 unsigned long flags
, unsigned long new_addr
)
411 down_write(¤t
->mm
->mmap_sem
);
412 ret
= do_mremap(addr
, old_len
, new_len
, flags
, new_addr
);
413 up_write(¤t
->mm
->mmap_sem
);