libc-top-half/musl/src/env/__init_tls.c

   1 #ifdef __wasilibc_unmodified_upstream
   2 #define SYSCALL_NO_TLS 1
   3 #include <elf.h>
   4 #endif
   5 #include <limits.h>
   6 #ifdef __wasilibc_unmodified_upstream
   7 #include <sys/mman.h>
   8 #endif
   9 #include <string.h>
  10 #include <stddef.h>
  11 #include "pthread_impl.h"
  12 #include "libc.h"
  13 #include "atomic.h"
  14 #include "syscall.h"
  15
  16 volatile int __thread_list_lock;
  17
  18 #ifndef __wasilibc_unmodified_upstream
  19
  20 /* These symbols are generated by wasm-ld. __stack_high/__stack_low
  21  * symbols are only available in LLVM v16 and higher, therefore they're
  22  * defined as weak symbols and if not available, __heap_base/__data_end
  23  * is used instead.
  24  *
  25  * TODO: remove usage of __heap_base/__data_end for stack size calculation
  26  * once we drop support for LLVM v15 and older.
  27  */
  28 extern unsigned char __heap_base;
  29 extern unsigned char __data_end;
  30 extern unsigned char __global_base;
  31 extern weak unsigned char __stack_high;
  32 extern weak unsigned char __stack_low;
  33
  34 static inline void setup_default_stack_size()
  35 {
  36         ptrdiff_t stack_size;
  37
  38         if (&__stack_high)
  39                 stack_size = &__stack_high - &__stack_low;
  40         else {
  41                 unsigned char *sp;
  42                 __asm__(
  43                         ".globaltype __stack_pointer, i32\n"
  44                         "global.get __stack_pointer\n"
  45                         "local.set %0\n"
  46                         : "=r"(sp));
  47                 stack_size = sp > &__global_base ? &__heap_base - &__data_end : (ptrdiff_t)&__global_base;
  48         }
  49
  50         if (stack_size > __default_stacksize)
  51                 __default_stacksize =
  52                         stack_size < DEFAULT_STACK_MAX ?
  53                         stack_size : DEFAULT_STACK_MAX;
  54 }
  55
  56 void __wasi_init_tp() {
  57         __init_tp((void *)__get_tp());
  58 }
  59 #endif
  60
  61 int __init_tp(void *p)
  62 {
  63         pthread_t td = p;
  64         td->self = td;
  65 #ifdef __wasilibc_unmodified_upstream
  66         int r = __set_thread_area(TP_ADJ(p));
  67         if (r < 0) return -1;
  68         if (!r) libc.can_do_threads = 1;
  69         td->detach_state = DT_JOINABLE;
  70         td->tid = __syscall(SYS_set_tid_address, &__thread_list_lock);
  71 #else
  72         setup_default_stack_size();
  73 #endif
  74         td->locale = &libc.global_locale;
  75         td->robust_list.head = &td->robust_list.head;
  76         td->sysinfo = __sysinfo;
  77         td->next = td->prev = td;
  78         return 0;
  79 }
  80
  81 #ifdef __wasilibc_unmodified_upstream
  82
  83 static struct builtin_tls {
  84         char c;
  85         struct pthread pt;
  86         void *space[16];
  87 } builtin_tls[1];
  88 #define MIN_TLS_ALIGN offsetof(struct builtin_tls, pt)
  89
  90 static struct tls_module main_tls;
  91 #endif
  92
  93 #ifndef __wasilibc_unmodified_upstream
  94 extern void __wasm_init_tls(void*);
  95 #endif
  96
  97 void *__copy_tls(unsigned char *mem)
  98 {
  99 #ifdef __wasilibc_unmodified_upstream
 100         pthread_t td;
 101         struct tls_module *p;
 102         size_t i;
 103         uintptr_t *dtv;
 104
 105 #ifdef TLS_ABOVE_TP
 106         dtv = (uintptr_t*)(mem + libc.tls_size) - (libc.tls_cnt + 1);
 107
 108         mem += -((uintptr_t)mem + sizeof(struct pthread)) & (libc.tls_align-1);
 109         td = (pthread_t)mem;
 110         mem += sizeof(struct pthread);
 111
 112         for (i=1, p=libc.tls_head; p; i++, p=p->next) {
 113                 dtv[i] = (uintptr_t)(mem + p->offset) + DTP_OFFSET;
 114                 memcpy(mem + p->offset, p->image, p->len);
 115         }
 116 #else
 117         dtv = (uintptr_t *)mem;
 118
 119         mem += libc.tls_size - sizeof(struct pthread);
 120         mem -= (uintptr_t)mem & (libc.tls_align-1);
 121         td = (pthread_t)mem;
 122
 123         for (i=1, p=libc.tls_head; p; i++, p=p->next) {
 124                 dtv[i] = (uintptr_t)(mem - p->offset) + DTP_OFFSET;
 125                 memcpy(mem - p->offset, p->image, p->len);
 126         }
 127 #endif
 128         dtv[0] = libc.tls_cnt;
 129         td->dtv = dtv;
 130         return td;
 131 #else
 132         size_t tls_align = __builtin_wasm_tls_align();
 133         volatile void* tls_base = __builtin_wasm_tls_base();
 134         mem += tls_align;
 135         mem -= (uintptr_t)mem & (tls_align-1);
 136         __wasm_init_tls(mem);
 137         __asm__("local.get %0\n"
 138                         "global.set __tls_base\n"
 139                         :: "r"(tls_base));
 140         return mem;
 141 #endif
 142 }
 143
 144 #ifdef __wasilibc_unmodified_upstream
 145 #if ULONG_MAX == 0xffffffff
 146 typedef Elf32_Phdr Phdr;
 147 #else
 148 typedef Elf64_Phdr Phdr;
 149 #endif
 150
 151 extern weak hidden const size_t _DYNAMIC[];
 152
 153 static void static_init_tls(size_t *aux)
 154 {
 155         unsigned char *p;
 156         size_t n;
 157         Phdr *phdr, *tls_phdr=0;
 158         size_t base = 0;
 159         void *mem;
 160
 161         for (p=(void *)aux[AT_PHDR],n=aux[AT_PHNUM]; n; n--,p+=aux[AT_PHENT]) {
 162                 phdr = (void *)p;
 163                 if (phdr->p_type == PT_PHDR)
 164                         base = aux[AT_PHDR] - phdr->p_vaddr;
 165                 if (phdr->p_type == PT_DYNAMIC && _DYNAMIC)
 166                         base = (size_t)_DYNAMIC - phdr->p_vaddr;
 167                 if (phdr->p_type == PT_TLS)
 168                         tls_phdr = phdr;
 169                 if (phdr->p_type == PT_GNU_STACK &&
 170                     phdr->p_memsz > __default_stacksize)
 171                         __default_stacksize =
 172                                 phdr->p_memsz < DEFAULT_STACK_MAX ?
 173                                 phdr->p_memsz : DEFAULT_STACK_MAX;
 174         }
 175
 176         if (tls_phdr) {
 177                 main_tls.image = (void *)(base + tls_phdr->p_vaddr);
 178                 main_tls.len = tls_phdr->p_filesz;
 179                 main_tls.size = tls_phdr->p_memsz;
 180                 main_tls.align = tls_phdr->p_align;
 181                 libc.tls_cnt = 1;
 182                 libc.tls_head = &main_tls;
 183         }
 184
 185         main_tls.size += (-main_tls.size - (uintptr_t)main_tls.image)
 186                 & (main_tls.align-1);
 187 #ifdef TLS_ABOVE_TP
 188         main_tls.offset = GAP_ABOVE_TP;
 189         main_tls.offset += (-GAP_ABOVE_TP + (uintptr_t)main_tls.image)
 190                 & (main_tls.align-1);
 191 #else
 192         main_tls.offset = main_tls.size;
 193 #endif
 194         if (main_tls.align < MIN_TLS_ALIGN) main_tls.align = MIN_TLS_ALIGN;
 195
 196         libc.tls_align = main_tls.align;
 197         libc.tls_size = 2*sizeof(void *) + sizeof(struct pthread)
 198 #ifdef TLS_ABOVE_TP
 199                 + main_tls.offset
 200 #endif
 201                 + main_tls.size + main_tls.align
 202                 + MIN_TLS_ALIGN-1 & -MIN_TLS_ALIGN;
 203
 204         if (libc.tls_size > sizeof builtin_tls) {
 205 #ifndef SYS_mmap2
 206 #define SYS_mmap2 SYS_mmap
 207 #endif
 208                 mem = (void *)__syscall(
 209                         SYS_mmap2,
 210                         0, libc.tls_size, PROT_READ|PROT_WRITE,
 211                         MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
 212                 /* -4095...-1 cast to void * will crash on dereference anyway,
 213                  * so don't bloat the init code checking for error codes and
 214                  * explicitly calling a_crash(). */
 215         } else {
 216                 mem = builtin_tls;
 217         }
 218
 219         /* Failure to initialize thread pointer is always fatal. */
 220         if (__init_tp(__copy_tls(mem)) < 0)
 221                 a_crash();
 222 }
 223
 224 weak_alias(static_init_tls, __init_tls);
 225 #endif