]>
Commit | Line | Data |
---|---|---|
a00bf321 | 1 | #ifdef __wasilibc_unmodified_upstream |
d4db3fa2 | 2 | #define SYSCALL_NO_TLS 1 |
320054e8 | 3 | #include <elf.h> |
a00bf321 | 4 | #endif |
320054e8 | 5 | #include <limits.h> |
a00bf321 | 6 | #ifdef __wasilibc_unmodified_upstream |
320054e8 | 7 | #include <sys/mman.h> |
a00bf321 | 8 | #endif |
320054e8 DG |
9 | #include <string.h> |
10 | #include <stddef.h> | |
11 | #include "pthread_impl.h" | |
12 | #include "libc.h" | |
13 | #include "atomic.h" | |
14 | #include "syscall.h" | |
15 | ||
f41256b6 DG |
16 | volatile int __thread_list_lock; |
17 | ||
a00bf321 | 18 | #ifndef __wasilibc_unmodified_upstream |
957c7113 MK |
19 | |
20 | /* These symbols are generated by wasm-ld. __stack_high/__stack_low | |
21 | * symbols are only available in LLVM v16 and higher, therefore they're | |
22 | * defined as weak symbols and if not available, __heap_base/__data_end | |
23 | * is used instead. | |
24 | * | |
25 | * TODO: remove usage of __heap_base/__data_end for stack size calculation | |
26 | * once we drop support for LLVM v15 and older. | |
27 | */ | |
28 | extern unsigned char __heap_base; | |
29 | extern unsigned char __data_end; | |
30 | extern unsigned char __global_base; | |
31 | extern weak unsigned char __stack_high; | |
32 | extern weak unsigned char __stack_low; | |
33 | ||
34 | static inline void setup_default_stack_size() | |
35 | { | |
36 | ptrdiff_t stack_size; | |
37 | ||
38 | if (&__stack_high) | |
39 | stack_size = &__stack_high - &__stack_low; | |
40 | else { | |
41 | unsigned char *sp; | |
42 | __asm__( | |
43 | ".globaltype __stack_pointer, i32\n" | |
44 | "global.get __stack_pointer\n" | |
45 | "local.set %0\n" | |
46 | : "=r"(sp)); | |
47 | stack_size = sp > &__global_base ? &__heap_base - &__data_end : (ptrdiff_t)&__global_base; | |
48 | } | |
49 | ||
b67d6b26 YT |
50 | __default_stacksize = |
51 | stack_size < DEFAULT_STACK_MAX ? | |
52 | stack_size : DEFAULT_STACK_MAX; | |
957c7113 MK |
53 | } |
54 | ||
a00bf321 | 55 | void __wasi_init_tp() { |
56 | __init_tp((void *)__get_tp()); | |
57 | } | |
58 | #endif | |
59 | ||
320054e8 DG |
60 | int __init_tp(void *p) |
61 | { | |
62 | pthread_t td = p; | |
63 | td->self = td; | |
a00bf321 | 64 | #ifdef __wasilibc_unmodified_upstream |
320054e8 DG |
65 | int r = __set_thread_area(TP_ADJ(p)); |
66 | if (r < 0) return -1; | |
67 | if (!r) libc.can_do_threads = 1; | |
68 | td->detach_state = DT_JOINABLE; | |
f41256b6 | 69 | td->tid = __syscall(SYS_set_tid_address, &__thread_list_lock); |
957c7113 MK |
70 | #else |
71 | setup_default_stack_size(); | |
5a255d5a YT |
72 | td->detach_state = DT_JOINABLE; |
73 | /* | |
74 | * Initialize the TID to a value which doesn't conflict with | |
75 | * host-allocated TIDs, so that TID-based locks can work. | |
76 | * | |
77 | * Note: | |
78 | * - Host-allocated TIDs range from 1 to 0x1fffffff. (inclusive) | |
79 | * - __tl_lock and __lockfile uses TID 0 as "unlocked". | |
80 | * - __lockfile relies on the fact the most significant two bits | |
81 | * of TIDs are 0. | |
82 | */ | |
83 | td->tid = 0x3fffffff; | |
a00bf321 | 84 | #endif |
320054e8 DG |
85 | td->locale = &libc.global_locale; |
86 | td->robust_list.head = &td->robust_list.head; | |
d4db3fa2 | 87 | td->sysinfo = __sysinfo; |
f41256b6 | 88 | td->next = td->prev = td; |
320054e8 DG |
89 | return 0; |
90 | } | |
91 | ||
a00bf321 | 92 | #ifdef __wasilibc_unmodified_upstream |
93 | ||
320054e8 DG |
94 | static struct builtin_tls { |
95 | char c; | |
96 | struct pthread pt; | |
97 | void *space[16]; | |
98 | } builtin_tls[1]; | |
99 | #define MIN_TLS_ALIGN offsetof(struct builtin_tls, pt) | |
100 | ||
101 | static struct tls_module main_tls; | |
a00bf321 | 102 | #endif |
103 | ||
104 | #ifndef __wasilibc_unmodified_upstream | |
105 | extern void __wasm_init_tls(void*); | |
106 | #endif | |
320054e8 DG |
107 | |
108 | void *__copy_tls(unsigned char *mem) | |
109 | { | |
a00bf321 | 110 | #ifdef __wasilibc_unmodified_upstream |
320054e8 DG |
111 | pthread_t td; |
112 | struct tls_module *p; | |
113 | size_t i; | |
114 | uintptr_t *dtv; | |
115 | ||
116 | #ifdef TLS_ABOVE_TP | |
117 | dtv = (uintptr_t*)(mem + libc.tls_size) - (libc.tls_cnt + 1); | |
118 | ||
119 | mem += -((uintptr_t)mem + sizeof(struct pthread)) & (libc.tls_align-1); | |
120 | td = (pthread_t)mem; | |
121 | mem += sizeof(struct pthread); | |
122 | ||
123 | for (i=1, p=libc.tls_head; p; i++, p=p->next) { | |
124 | dtv[i] = (uintptr_t)(mem + p->offset) + DTP_OFFSET; | |
125 | memcpy(mem + p->offset, p->image, p->len); | |
126 | } | |
127 | #else | |
128 | dtv = (uintptr_t *)mem; | |
129 | ||
130 | mem += libc.tls_size - sizeof(struct pthread); | |
131 | mem -= (uintptr_t)mem & (libc.tls_align-1); | |
132 | td = (pthread_t)mem; | |
133 | ||
134 | for (i=1, p=libc.tls_head; p; i++, p=p->next) { | |
135 | dtv[i] = (uintptr_t)(mem - p->offset) + DTP_OFFSET; | |
136 | memcpy(mem - p->offset, p->image, p->len); | |
137 | } | |
138 | #endif | |
139 | dtv[0] = libc.tls_cnt; | |
322bd4ff | 140 | td->dtv = dtv; |
320054e8 | 141 | return td; |
a00bf321 | 142 | #else |
143 | size_t tls_align = __builtin_wasm_tls_align(); | |
144 | volatile void* tls_base = __builtin_wasm_tls_base(); | |
145 | mem += tls_align; | |
146 | mem -= (uintptr_t)mem & (tls_align-1); | |
147 | __wasm_init_tls(mem); | |
148 | __asm__("local.get %0\n" | |
149 | "global.set __tls_base\n" | |
150 | :: "r"(tls_base)); | |
151 | return mem; | |
152 | #endif | |
320054e8 DG |
153 | } |
154 | ||
a00bf321 | 155 | #ifdef __wasilibc_unmodified_upstream |
320054e8 DG |
156 | #if ULONG_MAX == 0xffffffff |
157 | typedef Elf32_Phdr Phdr; | |
158 | #else | |
159 | typedef Elf64_Phdr Phdr; | |
160 | #endif | |
161 | ||
162 | extern weak hidden const size_t _DYNAMIC[]; | |
163 | ||
164 | static void static_init_tls(size_t *aux) | |
165 | { | |
166 | unsigned char *p; | |
167 | size_t n; | |
168 | Phdr *phdr, *tls_phdr=0; | |
169 | size_t base = 0; | |
170 | void *mem; | |
171 | ||
172 | for (p=(void *)aux[AT_PHDR],n=aux[AT_PHNUM]; n; n--,p+=aux[AT_PHENT]) { | |
173 | phdr = (void *)p; | |
174 | if (phdr->p_type == PT_PHDR) | |
175 | base = aux[AT_PHDR] - phdr->p_vaddr; | |
176 | if (phdr->p_type == PT_DYNAMIC && _DYNAMIC) | |
177 | base = (size_t)_DYNAMIC - phdr->p_vaddr; | |
178 | if (phdr->p_type == PT_TLS) | |
179 | tls_phdr = phdr; | |
180 | if (phdr->p_type == PT_GNU_STACK && | |
181 | phdr->p_memsz > __default_stacksize) | |
182 | __default_stacksize = | |
183 | phdr->p_memsz < DEFAULT_STACK_MAX ? | |
184 | phdr->p_memsz : DEFAULT_STACK_MAX; | |
185 | } | |
186 | ||
187 | if (tls_phdr) { | |
188 | main_tls.image = (void *)(base + tls_phdr->p_vaddr); | |
189 | main_tls.len = tls_phdr->p_filesz; | |
190 | main_tls.size = tls_phdr->p_memsz; | |
191 | main_tls.align = tls_phdr->p_align; | |
192 | libc.tls_cnt = 1; | |
193 | libc.tls_head = &main_tls; | |
194 | } | |
195 | ||
196 | main_tls.size += (-main_tls.size - (uintptr_t)main_tls.image) | |
197 | & (main_tls.align-1); | |
198 | #ifdef TLS_ABOVE_TP | |
199 | main_tls.offset = GAP_ABOVE_TP; | |
d4db3fa2 DG |
200 | main_tls.offset += (-GAP_ABOVE_TP + (uintptr_t)main_tls.image) |
201 | & (main_tls.align-1); | |
320054e8 DG |
202 | #else |
203 | main_tls.offset = main_tls.size; | |
204 | #endif | |
205 | if (main_tls.align < MIN_TLS_ALIGN) main_tls.align = MIN_TLS_ALIGN; | |
206 | ||
207 | libc.tls_align = main_tls.align; | |
208 | libc.tls_size = 2*sizeof(void *) + sizeof(struct pthread) | |
209 | #ifdef TLS_ABOVE_TP | |
210 | + main_tls.offset | |
211 | #endif | |
212 | + main_tls.size + main_tls.align | |
213 | + MIN_TLS_ALIGN-1 & -MIN_TLS_ALIGN; | |
214 | ||
215 | if (libc.tls_size > sizeof builtin_tls) { | |
216 | #ifndef SYS_mmap2 | |
217 | #define SYS_mmap2 SYS_mmap | |
218 | #endif | |
219 | mem = (void *)__syscall( | |
220 | SYS_mmap2, | |
221 | 0, libc.tls_size, PROT_READ|PROT_WRITE, | |
222 | MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); | |
223 | /* -4095...-1 cast to void * will crash on dereference anyway, | |
224 | * so don't bloat the init code checking for error codes and | |
225 | * explicitly calling a_crash(). */ | |
226 | } else { | |
227 | mem = builtin_tls; | |
228 | } | |
229 | ||
230 | /* Failure to initialize thread pointer is always fatal. */ | |
231 | if (__init_tp(__copy_tls(mem)) < 0) | |
232 | a_crash(); | |
233 | } | |
234 | ||
235 | weak_alias(static_init_tls, __init_tls); | |
a00bf321 | 236 | #endif |