4 * Copyright IBM Corp. 2008
5 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License (version 2 only)
9 * as published by the Free Software Foundation.
12 #include <linux/init.h>
13 #include <linux/errno.h>
14 #include <linux/sched.h>
15 #include <linux/kernel.h>
17 #include <linux/smp.h>
18 #include <linux/stddef.h>
19 #include <linux/unistd.h>
20 #include <linux/slab.h>
21 #include <linux/user.h>
22 #include <linux/elf.h>
23 #include <linux/security.h>
24 #include <linux/bootmem.h>
25 #include <linux/compat.h>
26 #include <asm/asm-offsets.h>
27 #include <asm/pgtable.h>
28 #include <asm/processor.h>
30 #include <asm/mmu_context.h>
31 #include <asm/sections.h>
33 #include <asm/facility.h>
36 extern char vdso32_start
, vdso32_end
;
37 static void *vdso32_kbase
= &vdso32_start
;
38 static unsigned int vdso32_pages
;
39 static struct page
**vdso32_pagelist
;
42 extern char vdso64_start
, vdso64_end
;
43 static void *vdso64_kbase
= &vdso64_start
;
44 static unsigned int vdso64_pages
;
45 static struct page
**vdso64_pagelist
;
48 * Should the kernel map a VDSO page into processes and pass its
49 * address down to glibc upon exec()?
51 unsigned int __read_mostly vdso_enabled
= 1;
53 static int vdso_fault(const struct vm_special_mapping
*sm
,
54 struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
56 struct page
**vdso_pagelist
;
57 unsigned long vdso_pages
;
59 vdso_pagelist
= vdso64_pagelist
;
60 vdso_pages
= vdso64_pages
;
62 if (is_compat_task()) {
63 vdso_pagelist
= vdso32_pagelist
;
64 vdso_pages
= vdso32_pages
;
68 if (vmf
->pgoff
>= vdso_pages
)
69 return VM_FAULT_SIGBUS
;
71 vmf
->page
= vdso_pagelist
[vmf
->pgoff
];
76 static int vdso_mremap(const struct vm_special_mapping
*sm
,
77 struct vm_area_struct
*vma
)
79 unsigned long vdso_pages
;
81 vdso_pages
= vdso64_pages
;
84 vdso_pages
= vdso32_pages
;
87 if ((vdso_pages
<< PAGE_SHIFT
) != vma
->vm_end
- vma
->vm_start
)
90 if (WARN_ON_ONCE(current
->mm
!= vma
->vm_mm
))
93 current
->mm
->context
.vdso_base
= vma
->vm_start
;
97 static const struct vm_special_mapping vdso_mapping
= {
100 .mremap
= vdso_mremap
,
103 static int __init
vdso_setup(char *s
)
109 if (strncmp(s
, "on", 3) == 0)
111 else if (strncmp(s
, "off", 4) == 0)
114 rc
= kstrtoul(s
, 0, &val
);
115 vdso_enabled
= rc
? 0 : !!val
;
119 __setup("vdso=", vdso_setup
);
125 struct vdso_data data
;
127 } vdso_data_store __page_aligned_data
;
128 struct vdso_data
*vdso_data
= &vdso_data_store
.data
;
131 * Setup vdso data page.
133 static void __init
vdso_init_data(struct vdso_data
*vd
)
135 vd
->ectg_available
= test_facility(31);
139 * Allocate/free per cpu vdso data.
141 #define SEGMENT_ORDER 2
144 * The initial vdso_data structure for the boot CPU. Eventually
145 * it is replaced with a properly allocated structure in vdso_init.
146 * This is necessary because a valid S390_lowcore.vdso_per_cpu_data
147 * pointer is required to be able to return from an interrupt or
148 * program check. See the exit paths in entry.S.
150 struct vdso_data boot_vdso_data __initdata
;
152 void __init
vdso_alloc_boot_cpu(struct lowcore
*lowcore
)
154 lowcore
->vdso_per_cpu_data
= (unsigned long) &boot_vdso_data
;
157 int vdso_alloc_per_cpu(struct lowcore
*lowcore
)
159 unsigned long segment_table
, page_table
, page_frame
;
160 struct vdso_per_cpu_data
*vd
;
162 segment_table
= __get_free_pages(GFP_KERNEL
, SEGMENT_ORDER
);
163 page_table
= get_zeroed_page(GFP_KERNEL
);
164 page_frame
= get_zeroed_page(GFP_KERNEL
);
165 if (!segment_table
|| !page_table
|| !page_frame
)
167 arch_set_page_dat(virt_to_page(segment_table
), SEGMENT_ORDER
);
168 arch_set_page_dat(virt_to_page(page_table
), 0);
170 /* Initialize per-cpu vdso data page */
171 vd
= (struct vdso_per_cpu_data
*) page_frame
;
172 vd
->cpu_nr
= lowcore
->cpu_nr
;
173 vd
->node_id
= cpu_to_node(vd
->cpu_nr
);
175 /* Set up page table for the vdso address space */
176 memset64((u64
*)segment_table
, _SEGMENT_ENTRY_EMPTY
, _CRST_ENTRIES
);
177 memset64((u64
*)page_table
, _PAGE_INVALID
, PTRS_PER_PTE
);
179 *(unsigned long *) segment_table
= _SEGMENT_ENTRY
+ page_table
;
180 *(unsigned long *) page_table
= _PAGE_PROTECT
+ page_frame
;
182 lowcore
->vdso_asce
= segment_table
+
183 _ASCE_TABLE_LENGTH
+ _ASCE_USER_BITS
+ _ASCE_TYPE_SEGMENT
;
184 lowcore
->vdso_per_cpu_data
= page_frame
;
189 free_page(page_frame
);
190 free_page(page_table
);
191 free_pages(segment_table
, SEGMENT_ORDER
);
195 void vdso_free_per_cpu(struct lowcore
*lowcore
)
197 unsigned long segment_table
, page_table
, page_frame
;
199 segment_table
= lowcore
->vdso_asce
& PAGE_MASK
;
200 page_table
= *(unsigned long *) segment_table
;
201 page_frame
= *(unsigned long *) page_table
;
203 free_page(page_frame
);
204 free_page(page_table
);
205 free_pages(segment_table
, SEGMENT_ORDER
);
209 * This is called from binfmt_elf, we create the special vma for the
210 * vDSO and insert it into the mm struct tree
212 int arch_setup_additional_pages(struct linux_binprm
*bprm
, int uses_interp
)
214 struct mm_struct
*mm
= current
->mm
;
215 struct vm_area_struct
*vma
;
216 unsigned long vdso_pages
;
217 unsigned long vdso_base
;
223 * Only map the vdso for dynamically linked elf binaries.
228 vdso_pages
= vdso64_pages
;
230 if (is_compat_task())
231 vdso_pages
= vdso32_pages
;
234 * vDSO has a problem and was disabled, just don't "enable" it for
241 * pick a base address for the vDSO in process space. We try to put
242 * it at vdso_base which is the "natural" base for it, but we might
243 * fail and end up putting it elsewhere.
245 if (down_write_killable(&mm
->mmap_sem
))
247 vdso_base
= get_unmapped_area(NULL
, 0, vdso_pages
<< PAGE_SHIFT
, 0, 0);
248 if (IS_ERR_VALUE(vdso_base
)) {
254 * our vma flags don't have VM_WRITE so by default, the process
255 * isn't allowed to write those pages.
256 * gdb can break that with ptrace interface, and thus trigger COW
257 * on those pages but it's then your responsibility to never do that
258 * on the "data" page of the vDSO or you'll stop getting kernel
259 * updates and your nice userland gettimeofday will be totally dead.
260 * It's fine to use that for setting breakpoints in the vDSO code
263 vma
= _install_special_mapping(mm
, vdso_base
, vdso_pages
<< PAGE_SHIFT
,
265 VM_MAYREAD
|VM_MAYWRITE
|VM_MAYEXEC
,
272 current
->mm
->context
.vdso_base
= vdso_base
;
276 up_write(&mm
->mmap_sem
);
280 static int __init
vdso_init(void)
284 vdso_init_data(vdso_data
);
286 /* Calculate the size of the 32 bit vDSO */
287 vdso32_pages
= ((&vdso32_end
- &vdso32_start
288 + PAGE_SIZE
- 1) >> PAGE_SHIFT
) + 1;
290 /* Make sure pages are in the correct state */
291 vdso32_pagelist
= kzalloc(sizeof(struct page
*) * (vdso32_pages
+ 1),
293 BUG_ON(vdso32_pagelist
== NULL
);
294 for (i
= 0; i
< vdso32_pages
- 1; i
++) {
295 struct page
*pg
= virt_to_page(vdso32_kbase
+ i
*PAGE_SIZE
);
296 ClearPageReserved(pg
);
298 vdso32_pagelist
[i
] = pg
;
300 vdso32_pagelist
[vdso32_pages
- 1] = virt_to_page(vdso_data
);
301 vdso32_pagelist
[vdso32_pages
] = NULL
;
304 /* Calculate the size of the 64 bit vDSO */
305 vdso64_pages
= ((&vdso64_end
- &vdso64_start
306 + PAGE_SIZE
- 1) >> PAGE_SHIFT
) + 1;
308 /* Make sure pages are in the correct state */
309 vdso64_pagelist
= kzalloc(sizeof(struct page
*) * (vdso64_pages
+ 1),
311 BUG_ON(vdso64_pagelist
== NULL
);
312 for (i
= 0; i
< vdso64_pages
- 1; i
++) {
313 struct page
*pg
= virt_to_page(vdso64_kbase
+ i
*PAGE_SIZE
);
314 ClearPageReserved(pg
);
316 vdso64_pagelist
[i
] = pg
;
318 vdso64_pagelist
[vdso64_pages
- 1] = virt_to_page(vdso_data
);
319 vdso64_pagelist
[vdso64_pages
] = NULL
;
320 if (vdso_alloc_per_cpu(&S390_lowcore
))
323 get_page(virt_to_page(vdso_data
));
327 early_initcall(vdso_init
);