]>
Commit | Line | Data |
---|---|---|
92a42be0 SL |
1 | /* ===---------- emutls.c - Implements __emutls_get_address ---------------=== |
2 | * | |
3 | * The LLVM Compiler Infrastructure | |
4 | * | |
5 | * This file is dual licensed under the MIT and the University of Illinois Open | |
6 | * Source Licenses. See LICENSE.TXT for details. | |
7 | * | |
8 | * ===----------------------------------------------------------------------=== | |
9 | */ | |
10 | #include <pthread.h> | |
92a42be0 SL |
11 | #include <stdint.h> |
12 | #include <stdlib.h> | |
13 | #include <string.h> | |
14 | ||
15 | #include "int_lib.h" | |
16 | #include "int_util.h" | |
17 | ||
18 | /* Default is not to use posix_memalign, so systems like Android | |
19 | * can use thread local data without heavier POSIX memory allocators. | |
20 | */ | |
21 | #ifndef EMUTLS_USE_POSIX_MEMALIGN | |
22 | #define EMUTLS_USE_POSIX_MEMALIGN 0 | |
23 | #endif | |
24 | ||
25 | /* For every TLS variable xyz, | |
26 | * there is one __emutls_control variable named __emutls_v.xyz. | |
27 | * If xyz has non-zero initial value, __emutls_v.xyz's "value" | |
28 | * will point to __emutls_t.xyz, which has the initial value. | |
29 | */ | |
5bcae85e | 30 | typedef unsigned int gcc_word __attribute__((mode(word))); |
92a42be0 | 31 | typedef struct __emutls_control { |
5bcae85e SL |
32 | /* Must use gcc_word here, instead of size_t, to match GCC. When |
33 | gcc_word is larger than size_t, the upper extra bits are all | |
34 | zeros. We can use variables of size_t to operate on size and | |
35 | align. */ | |
36 | gcc_word size; /* size of the object in bytes */ | |
37 | gcc_word align; /* alignment of the object in bytes */ | |
92a42be0 SL |
38 | union { |
39 | uintptr_t index; /* data[index-1] is the object address */ | |
40 | void* address; /* object address, when in single thread env */ | |
41 | } object; | |
42 | void* value; /* null or non-zero initial value for the object */ | |
43 | } __emutls_control; | |
44 | ||
45 | static __inline void *emutls_memalign_alloc(size_t align, size_t size) { | |
46 | void *base; | |
47 | #if EMUTLS_USE_POSIX_MEMALIGN | |
48 | if (posix_memalign(&base, align, size) != 0) | |
49 | abort(); | |
50 | #else | |
51 | #define EXTRA_ALIGN_PTR_BYTES (align - 1 + sizeof(void*)) | |
52 | char* object; | |
53 | if ((object = malloc(EXTRA_ALIGN_PTR_BYTES + size)) == NULL) | |
54 | abort(); | |
55 | base = (void*)(((uintptr_t)(object + EXTRA_ALIGN_PTR_BYTES)) | |
56 | & ~(uintptr_t)(align - 1)); | |
57 | ||
58 | ((void**)base)[-1] = object; | |
59 | #endif | |
60 | return base; | |
61 | } | |
62 | ||
63 | static __inline void emutls_memalign_free(void *base) { | |
64 | #if EMUTLS_USE_POSIX_MEMALIGN | |
65 | free(base); | |
66 | #else | |
67 | /* The mallocated address is in ((void**)base)[-1] */ | |
68 | free(((void**)base)[-1]); | |
69 | #endif | |
70 | } | |
71 | ||
72 | /* Emulated TLS objects are always allocated at run-time. */ | |
73 | static __inline void *emutls_allocate_object(__emutls_control *control) { | |
74 | /* Use standard C types, check with gcc's emutls.o. */ | |
92a42be0 | 75 | typedef unsigned int gcc_pointer __attribute__((mode(pointer))); |
92a42be0 SL |
76 | COMPILE_TIME_ASSERT(sizeof(uintptr_t) == sizeof(gcc_pointer)); |
77 | COMPILE_TIME_ASSERT(sizeof(uintptr_t) == sizeof(void*)); | |
78 | ||
79 | size_t size = control->size; | |
80 | size_t align = control->align; | |
5bcae85e | 81 | void* base; |
92a42be0 SL |
82 | if (align < sizeof(void*)) |
83 | align = sizeof(void*); | |
84 | /* Make sure that align is power of 2. */ | |
85 | if ((align & (align - 1)) != 0) | |
86 | abort(); | |
87 | ||
5bcae85e | 88 | base = emutls_memalign_alloc(align, size); |
92a42be0 SL |
89 | if (control->value) |
90 | memcpy(base, control->value, size); | |
91 | else | |
92 | memset(base, 0, size); | |
93 | return base; | |
94 | } | |
95 | ||
96 | static pthread_mutex_t emutls_mutex = PTHREAD_MUTEX_INITIALIZER; | |
97 | ||
98 | static size_t emutls_num_object = 0; /* number of allocated TLS objects */ | |
99 | ||
100 | typedef struct emutls_address_array { | |
101 | uintptr_t size; /* number of elements in the 'data' array */ | |
102 | void* data[]; | |
103 | } emutls_address_array; | |
104 | ||
105 | static pthread_key_t emutls_pthread_key; | |
106 | ||
107 | static void emutls_key_destructor(void* ptr) { | |
108 | emutls_address_array* array = (emutls_address_array*)ptr; | |
109 | uintptr_t i; | |
110 | for (i = 0; i < array->size; ++i) { | |
111 | if (array->data[i]) | |
112 | emutls_memalign_free(array->data[i]); | |
113 | } | |
114 | free(ptr); | |
115 | } | |
116 | ||
117 | static void emutls_init(void) { | |
118 | if (pthread_key_create(&emutls_pthread_key, emutls_key_destructor) != 0) | |
119 | abort(); | |
120 | } | |
121 | ||
122 | /* Returns control->object.index; set index if not allocated yet. */ | |
123 | static __inline uintptr_t emutls_get_index(__emutls_control *control) { | |
124 | uintptr_t index = __atomic_load_n(&control->object.index, __ATOMIC_ACQUIRE); | |
125 | if (!index) { | |
126 | static pthread_once_t once = PTHREAD_ONCE_INIT; | |
127 | pthread_once(&once, emutls_init); | |
128 | pthread_mutex_lock(&emutls_mutex); | |
129 | index = control->object.index; | |
130 | if (!index) { | |
131 | index = ++emutls_num_object; | |
132 | __atomic_store_n(&control->object.index, index, __ATOMIC_RELEASE); | |
133 | } | |
134 | pthread_mutex_unlock(&emutls_mutex); | |
135 | } | |
136 | return index; | |
137 | } | |
138 | ||
139 | /* Updates newly allocated thread local emutls_address_array. */ | |
140 | static __inline void emutls_check_array_set_size(emutls_address_array *array, | |
141 | uintptr_t size) { | |
142 | if (array == NULL) | |
143 | abort(); | |
144 | array->size = size; | |
145 | pthread_setspecific(emutls_pthread_key, (void*)array); | |
146 | } | |
147 | ||
148 | /* Returns the new 'data' array size, number of elements, | |
149 | * which must be no smaller than the given index. | |
150 | */ | |
151 | static __inline uintptr_t emutls_new_data_array_size(uintptr_t index) { | |
152 | /* Need to allocate emutls_address_array with one extra slot | |
153 | * to store the data array size. | |
154 | * Round up the emutls_address_array size to multiple of 16. | |
155 | */ | |
156 | return ((index + 1 + 15) & ~((uintptr_t)15)) - 1; | |
157 | } | |
158 | ||
159 | /* Returns the thread local emutls_address_array. | |
160 | * Extends its size if necessary to hold address at index. | |
161 | */ | |
162 | static __inline emutls_address_array * | |
163 | emutls_get_address_array(uintptr_t index) { | |
164 | emutls_address_array* array = pthread_getspecific(emutls_pthread_key); | |
165 | if (array == NULL) { | |
166 | uintptr_t new_size = emutls_new_data_array_size(index); | |
5bcae85e SL |
167 | array = malloc(new_size * sizeof(void *) + sizeof(emutls_address_array)); |
168 | if (array) | |
169 | memset(array->data, 0, new_size * sizeof(void*)); | |
92a42be0 SL |
170 | emutls_check_array_set_size(array, new_size); |
171 | } else if (index > array->size) { | |
172 | uintptr_t orig_size = array->size; | |
173 | uintptr_t new_size = emutls_new_data_array_size(index); | |
5bcae85e | 174 | array = realloc(array, new_size * sizeof(void *) + sizeof(emutls_address_array)); |
92a42be0 SL |
175 | if (array) |
176 | memset(array->data + orig_size, 0, | |
177 | (new_size - orig_size) * sizeof(void*)); | |
178 | emutls_check_array_set_size(array, new_size); | |
179 | } | |
180 | return array; | |
181 | } | |
182 | ||
183 | void* __emutls_get_address(__emutls_control* control) { | |
184 | uintptr_t index = emutls_get_index(control); | |
185 | emutls_address_array* array = emutls_get_address_array(index); | |
186 | if (array->data[index - 1] == NULL) | |
187 | array->data[index - 1] = emutls_allocate_object(control); | |
188 | return array->data[index - 1]; | |
189 | } |