1 // SPDX-License-Identifier: GPL-2.0
5 * Copyright IBM Corp. 2008
6 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
9 #include <linux/init.h>
10 #include <linux/errno.h>
11 #include <linux/sched.h>
12 #include <linux/kernel.h>
14 #include <linux/smp.h>
15 #include <linux/stddef.h>
16 #include <linux/unistd.h>
17 #include <linux/slab.h>
18 #include <linux/user.h>
19 #include <linux/elf.h>
20 #include <linux/security.h>
21 #include <linux/memblock.h>
22 #include <linux/compat.h>
23 #include <asm/asm-offsets.h>
24 #include <asm/processor.h>
26 #include <asm/mmu_context.h>
27 #include <asm/sections.h>
29 #include <asm/facility.h>
31 extern char vdso64_start
, vdso64_end
;
32 static void *vdso64_kbase
= &vdso64_start
;
33 static unsigned int vdso64_pages
;
34 static struct page
**vdso64_pagelist
;
37 * Should the kernel map a VDSO page into processes and pass its
38 * address down to glibc upon exec()?
40 unsigned int __read_mostly vdso_enabled
= 1;
42 static vm_fault_t
vdso_fault(const struct vm_special_mapping
*sm
,
43 struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
45 struct page
**vdso_pagelist
;
46 unsigned long vdso_pages
;
48 vdso_pagelist
= vdso64_pagelist
;
49 vdso_pages
= vdso64_pages
;
51 if (vmf
->pgoff
>= vdso_pages
)
52 return VM_FAULT_SIGBUS
;
54 vmf
->page
= vdso_pagelist
[vmf
->pgoff
];
59 static int vdso_mremap(const struct vm_special_mapping
*sm
,
60 struct vm_area_struct
*vma
)
62 unsigned long vdso_pages
;
64 vdso_pages
= vdso64_pages
;
66 if ((vdso_pages
<< PAGE_SHIFT
) != vma
->vm_end
- vma
->vm_start
)
69 if (WARN_ON_ONCE(current
->mm
!= vma
->vm_mm
))
72 current
->mm
->context
.vdso_base
= vma
->vm_start
;
76 static const struct vm_special_mapping vdso_mapping
= {
79 .mremap
= vdso_mremap
,
82 static int __init
vdso_setup(char *str
)
86 if (!kstrtobool(str
, &enabled
))
87 vdso_enabled
= enabled
;
90 __setup("vdso=", vdso_setup
);
96 struct vdso_data data
;
98 } vdso_data_store __page_aligned_data
;
99 struct vdso_data
*vdso_data
= &vdso_data_store
.data
;
102 * Setup vdso data page.
104 static void __init
vdso_init_data(struct vdso_data
*vd
)
106 vd
->ectg_available
= test_facility(31);
110 * Allocate/free per cpu vdso data.
112 #define SEGMENT_ORDER 2
115 * The initial vdso_data structure for the boot CPU. Eventually
116 * it is replaced with a properly allocated structure in vdso_init.
117 * This is necessary because a valid S390_lowcore.vdso_per_cpu_data
118 * pointer is required to be able to return from an interrupt or
119 * program check. See the exit paths in entry.S.
121 struct vdso_data boot_vdso_data __initdata
;
123 void __init
vdso_alloc_boot_cpu(struct lowcore
*lowcore
)
125 lowcore
->vdso_per_cpu_data
= (unsigned long) &boot_vdso_data
;
128 int vdso_alloc_per_cpu(struct lowcore
*lowcore
)
130 unsigned long segment_table
, page_table
, page_frame
;
131 struct vdso_per_cpu_data
*vd
;
133 segment_table
= __get_free_pages(GFP_KERNEL
, SEGMENT_ORDER
);
134 page_table
= get_zeroed_page(GFP_KERNEL
);
135 page_frame
= get_zeroed_page(GFP_KERNEL
);
136 if (!segment_table
|| !page_table
|| !page_frame
)
138 arch_set_page_dat(virt_to_page(segment_table
), SEGMENT_ORDER
);
139 arch_set_page_dat(virt_to_page(page_table
), 0);
141 /* Initialize per-cpu vdso data page */
142 vd
= (struct vdso_per_cpu_data
*) page_frame
;
143 vd
->cpu_nr
= lowcore
->cpu_nr
;
144 vd
->node_id
= cpu_to_node(vd
->cpu_nr
);
146 /* Set up page table for the vdso address space */
147 memset64((u64
*)segment_table
, _SEGMENT_ENTRY_EMPTY
, _CRST_ENTRIES
);
148 memset64((u64
*)page_table
, _PAGE_INVALID
, PTRS_PER_PTE
);
150 *(unsigned long *) segment_table
= _SEGMENT_ENTRY
+ page_table
;
151 *(unsigned long *) page_table
= _PAGE_PROTECT
+ page_frame
;
153 lowcore
->vdso_asce
= segment_table
+
154 _ASCE_TABLE_LENGTH
+ _ASCE_USER_BITS
+ _ASCE_TYPE_SEGMENT
;
155 lowcore
->vdso_per_cpu_data
= page_frame
;
160 free_page(page_frame
);
161 free_page(page_table
);
162 free_pages(segment_table
, SEGMENT_ORDER
);
166 void vdso_free_per_cpu(struct lowcore
*lowcore
)
168 unsigned long segment_table
, page_table
, page_frame
;
170 segment_table
= lowcore
->vdso_asce
& PAGE_MASK
;
171 page_table
= *(unsigned long *) segment_table
;
172 page_frame
= *(unsigned long *) page_table
;
174 free_page(page_frame
);
175 free_page(page_table
);
176 free_pages(segment_table
, SEGMENT_ORDER
);
180 * This is called from binfmt_elf, we create the special vma for the
181 * vDSO and insert it into the mm struct tree
183 int arch_setup_additional_pages(struct linux_binprm
*bprm
, int uses_interp
)
185 struct mm_struct
*mm
= current
->mm
;
186 struct vm_area_struct
*vma
;
187 unsigned long vdso_pages
;
188 unsigned long vdso_base
;
194 if (is_compat_task())
197 vdso_pages
= vdso64_pages
;
199 * vDSO has a problem and was disabled, just don't "enable" it for
206 * pick a base address for the vDSO in process space. We try to put
207 * it at vdso_base which is the "natural" base for it, but we might
208 * fail and end up putting it elsewhere.
210 if (mmap_write_lock_killable(mm
))
212 vdso_base
= get_unmapped_area(NULL
, 0, vdso_pages
<< PAGE_SHIFT
, 0, 0);
213 if (IS_ERR_VALUE(vdso_base
)) {
219 * our vma flags don't have VM_WRITE so by default, the process
220 * isn't allowed to write those pages.
221 * gdb can break that with ptrace interface, and thus trigger COW
222 * on those pages but it's then your responsibility to never do that
223 * on the "data" page of the vDSO or you'll stop getting kernel
224 * updates and your nice userland gettimeofday will be totally dead.
225 * It's fine to use that for setting breakpoints in the vDSO code
228 vma
= _install_special_mapping(mm
, vdso_base
, vdso_pages
<< PAGE_SHIFT
,
230 VM_MAYREAD
|VM_MAYWRITE
|VM_MAYEXEC
,
237 current
->mm
->context
.vdso_base
= vdso_base
;
241 mmap_write_unlock(mm
);
245 static int __init
vdso_init(void)
249 vdso_init_data(vdso_data
);
251 /* Calculate the size of the 64 bit vDSO */
252 vdso64_pages
= ((&vdso64_end
- &vdso64_start
253 + PAGE_SIZE
- 1) >> PAGE_SHIFT
) + 1;
255 /* Make sure pages are in the correct state */
256 vdso64_pagelist
= kcalloc(vdso64_pages
+ 1, sizeof(struct page
*),
258 BUG_ON(vdso64_pagelist
== NULL
);
259 for (i
= 0; i
< vdso64_pages
- 1; i
++) {
260 struct page
*pg
= virt_to_page(vdso64_kbase
+ i
*PAGE_SIZE
);
262 vdso64_pagelist
[i
] = pg
;
264 vdso64_pagelist
[vdso64_pages
- 1] = virt_to_page(vdso_data
);
265 vdso64_pagelist
[vdso64_pages
] = NULL
;
266 if (vdso_alloc_per_cpu(&S390_lowcore
))
269 get_page(virt_to_page(vdso_data
));
273 early_initcall(vdso_init
);