]>
Commit | Line | Data |
---|---|---|
5e7ed085 | 1 | use gccjit::{GlobalKind, LValue, RValue, ToRValue, Type}; |
c295e0f8 XL |
2 | use rustc_codegen_ssa::traits::{BaseTypeMethods, ConstMethods, DerivedTypeMethods, StaticMethods}; |
3 | use rustc_hir as hir; | |
4 | use rustc_hir::Node; | |
5 | use rustc_middle::{bug, span_bug}; | |
6 | use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; | |
7 | use rustc_middle::mir::mono::MonoItem; | |
8 | use rustc_middle::ty::{self, Instance, Ty}; | |
9 | use rustc_middle::ty::layout::LayoutOf; | |
5e7ed085 | 10 | use rustc_middle::mir::interpret::{self, ConstAllocation, ErrorHandled, Scalar as InterpScalar, read_target_uint}; |
c295e0f8 XL |
11 | use rustc_span::def_id::DefId; |
12 | use rustc_target::abi::{self, Align, HasDataLayout, Primitive, Size, WrappingRange}; | |
13 | ||
14 | use crate::base; | |
15 | use crate::context::CodegenCx; | |
16 | use crate::type_of::LayoutGccExt; | |
17 | ||
18 | impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { | |
19 | pub fn const_bitcast(&self, value: RValue<'gcc>, typ: Type<'gcc>) -> RValue<'gcc> { | |
20 | if value.get_type() == self.bool_type.make_pointer() { | |
21 | if let Some(pointee) = typ.get_pointee() { | |
a2a8927a | 22 | if pointee.dyncast_vector().is_some() { |
c295e0f8 XL |
23 | panic!() |
24 | } | |
25 | } | |
26 | } | |
923072b8 FG |
27 | // NOTE: since bitcast makes a value non-constant, don't bitcast if not necessary as some |
28 | // SIMD builtins require a constant value. | |
29 | self.bitcast_if_needed(value, typ) | |
c295e0f8 XL |
30 | } |
31 | } | |
32 | ||
33 | impl<'gcc, 'tcx> StaticMethods for CodegenCx<'gcc, 'tcx> { | |
34 | fn static_addr_of(&self, cv: RValue<'gcc>, align: Align, kind: Option<&str>) -> RValue<'gcc> { | |
a2a8927a XL |
35 | // TODO(antoyo): implement a proper rvalue comparison in libgccjit instead of doing the |
36 | // following: | |
37 | for (value, variable) in &*self.const_globals.borrow() { | |
38 | if format!("{:?}", value) == format!("{:?}", cv) { | |
5e7ed085 FG |
39 | if let Some(global_variable) = self.global_lvalues.borrow().get(variable) { |
40 | let alignment = align.bits() as i32; | |
41 | if alignment > global_variable.get_alignment() { | |
42 | global_variable.set_alignment(alignment); | |
43 | } | |
44 | } | |
a2a8927a XL |
45 | return *variable; |
46 | } | |
c295e0f8 XL |
47 | } |
48 | let global_value = self.static_addr_of_mut(cv, align, kind); | |
923072b8 FG |
49 | #[cfg(feature = "master")] |
50 | self.global_lvalues.borrow().get(&global_value) | |
51 | .expect("`static_addr_of_mut` did not add the global to `self.global_lvalues`") | |
52 | .global_set_readonly(); | |
c295e0f8 XL |
53 | self.const_globals.borrow_mut().insert(cv, global_value); |
54 | global_value | |
55 | } | |
56 | ||
57 | fn codegen_static(&self, def_id: DefId, is_mutable: bool) { | |
58 | let attrs = self.tcx.codegen_fn_attrs(def_id); | |
59 | ||
60 | let value = | |
61 | match codegen_static_initializer(&self, def_id) { | |
62 | Ok((value, _)) => value, | |
63 | // Error has already been reported | |
64 | Err(_) => return, | |
65 | }; | |
66 | ||
67 | let global = self.get_static(def_id); | |
68 | ||
69 | // boolean SSA values are i1, but they have to be stored in i8 slots, | |
70 | // otherwise some LLVM optimization passes don't work as expected | |
71 | let val_llty = self.val_ty(value); | |
72 | let value = | |
73 | if val_llty == self.type_i1() { | |
74 | unimplemented!(); | |
75 | } | |
76 | else { | |
77 | value | |
78 | }; | |
79 | ||
80 | let instance = Instance::mono(self.tcx, def_id); | |
81 | let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); | |
82 | let gcc_type = self.layout_of(ty).gcc_type(self, true); | |
83 | ||
84 | // TODO(antoyo): set alignment. | |
85 | ||
923072b8 | 86 | let value = self.bitcast_if_needed(value, gcc_type); |
a2a8927a | 87 | global.global_set_initializer_rvalue(value); |
c295e0f8 XL |
88 | |
89 | // As an optimization, all shared statics which do not have interior | |
90 | // mutability are placed into read-only memory. | |
91 | if !is_mutable { | |
92 | if self.type_is_freeze(ty) { | |
923072b8 FG |
93 | #[cfg(feature = "master")] |
94 | global.global_set_readonly(); | |
c295e0f8 XL |
95 | } |
96 | } | |
97 | ||
98 | if attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) { | |
99 | // Do not allow LLVM to change the alignment of a TLS on macOS. | |
100 | // | |
101 | // By default a global's alignment can be freely increased. | |
102 | // This allows LLVM to generate more performant instructions | |
103 | // e.g., using load-aligned into a SIMD register. | |
104 | // | |
105 | // However, on macOS 10.10 or below, the dynamic linker does not | |
106 | // respect any alignment given on the TLS (radar 24221680). | |
107 | // This will violate the alignment assumption, and causing segfault at runtime. | |
108 | // | |
109 | // This bug is very easy to trigger. In `println!` and `panic!`, | |
110 | // the `LOCAL_STDOUT`/`LOCAL_STDERR` handles are stored in a TLS, | |
111 | // which the values would be `mem::replace`d on initialization. | |
112 | // The implementation of `mem::replace` will use SIMD | |
113 | // whenever the size is 32 bytes or higher. LLVM notices SIMD is used | |
114 | // and tries to align `LOCAL_STDOUT`/`LOCAL_STDERR` to a 32-byte boundary, | |
115 | // which macOS's dyld disregarded and causing crashes | |
116 | // (see issues #51794, #51758, #50867, #48866 and #44056). | |
117 | // | |
118 | // To workaround the bug, we trick LLVM into not increasing | |
119 | // the global's alignment by explicitly assigning a section to it | |
120 | // (equivalent to automatically generating a `#[link_section]` attribute). | |
121 | // See the comment in the `GlobalValue::canIncreaseAlignment()` function | |
122 | // of `lib/IR/Globals.cpp` for why this works. | |
123 | // | |
124 | // When the alignment is not increased, the optimized `mem::replace` | |
125 | // will use load-unaligned instructions instead, and thus avoiding the crash. | |
126 | // | |
127 | // We could remove this hack whenever we decide to drop macOS 10.10 support. | |
128 | if self.tcx.sess.target.options.is_like_osx { | |
f2b60f7d | 129 | // The `inspect` method is okay here because we checked for provenance, and |
c295e0f8 XL |
130 | // because we are doing this access to inspect the final interpreter state |
131 | // (not as part of the interpreter execution). | |
132 | // | |
133 | // FIXME: This check requires that the (arbitrary) value of undefined bytes | |
134 | // happens to be zero. Instead, we should only check the value of defined bytes | |
135 | // and set all undefined bytes to zero if this allocation is headed for the | |
136 | // BSS. | |
137 | unimplemented!(); | |
138 | } | |
139 | } | |
140 | ||
141 | // Wasm statics with custom link sections get special treatment as they | |
142 | // go into custom sections of the wasm executable. | |
143 | if self.tcx.sess.opts.target_triple.triple().starts_with("wasm32") { | |
144 | if let Some(_section) = attrs.link_section { | |
145 | unimplemented!(); | |
146 | } | |
147 | } else { | |
148 | // TODO(antoyo): set link section. | |
149 | } | |
150 | ||
5099ac24 | 151 | if attrs.flags.contains(CodegenFnAttrFlags::USED) || attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER) { |
c295e0f8 XL |
152 | self.add_used_global(global.to_rvalue()); |
153 | } | |
154 | } | |
155 | ||
156 | /// Add a global value to a list to be stored in the `llvm.used` variable, an array of i8*. | |
157 | fn add_used_global(&self, _global: RValue<'gcc>) { | |
158 | // TODO(antoyo) | |
159 | } | |
160 | ||
161 | fn add_compiler_used_global(&self, _global: RValue<'gcc>) { | |
162 | // TODO(antoyo) | |
163 | } | |
164 | } | |
165 | ||
166 | impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { | |
167 | pub fn static_addr_of_mut(&self, cv: RValue<'gcc>, align: Align, kind: Option<&str>) -> RValue<'gcc> { | |
168 | let global = | |
169 | match kind { | |
170 | Some(kind) if !self.tcx.sess.fewer_names() => { | |
171 | let name = self.generate_local_symbol_name(kind); | |
5e7ed085 | 172 | // TODO(antoyo): check if it's okay that no link_section is set. |
923072b8 FG |
173 | |
174 | let typ = self.val_ty(cv).get_aligned(align.bytes()); | |
175 | let global = self.declare_private_global(&name[..], typ); | |
c295e0f8 XL |
176 | global |
177 | } | |
178 | _ => { | |
179 | let typ = self.val_ty(cv).get_aligned(align.bytes()); | |
180 | let global = self.declare_unnamed_global(typ); | |
181 | global | |
182 | }, | |
183 | }; | |
a2a8927a | 184 | global.global_set_initializer_rvalue(cv); |
c295e0f8 | 185 | // TODO(antoyo): set unnamed address. |
5e7ed085 FG |
186 | let rvalue = global.get_address(None); |
187 | self.global_lvalues.borrow_mut().insert(rvalue, global); | |
188 | rvalue | |
c295e0f8 XL |
189 | } |
190 | ||
191 | pub fn get_static(&self, def_id: DefId) -> LValue<'gcc> { | |
192 | let instance = Instance::mono(self.tcx, def_id); | |
193 | let fn_attrs = self.tcx.codegen_fn_attrs(def_id); | |
194 | if let Some(&global) = self.instances.borrow().get(&instance) { | |
195 | return global; | |
196 | } | |
197 | ||
198 | let defined_in_current_codegen_unit = | |
199 | self.codegen_unit.items().contains_key(&MonoItem::Static(def_id)); | |
200 | assert!( | |
201 | !defined_in_current_codegen_unit, | |
202 | "consts::get_static() should always hit the cache for \ | |
203 | statics defined in the same CGU, but did not for `{:?}`", | |
204 | def_id | |
205 | ); | |
206 | ||
207 | let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); | |
208 | let sym = self.tcx.symbol_name(instance).name; | |
209 | ||
210 | let global = | |
211 | if let Some(def_id) = def_id.as_local() { | |
212 | let id = self.tcx.hir().local_def_id_to_hir_id(def_id); | |
213 | let llty = self.layout_of(ty).gcc_type(self, true); | |
214 | // FIXME: refactor this to work without accessing the HIR | |
215 | let global = match self.tcx.hir().get(id) { | |
216 | Node::Item(&hir::Item { span, kind: hir::ItemKind::Static(..), .. }) => { | |
217 | if let Some(global) = self.get_declared_value(&sym) { | |
218 | if self.val_ty(global) != self.type_ptr_to(llty) { | |
219 | span_bug!(span, "Conflicting types for static"); | |
220 | } | |
221 | } | |
222 | ||
223 | let is_tls = fn_attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL); | |
5e7ed085 FG |
224 | let global = self.declare_global( |
225 | &sym, | |
226 | llty, | |
227 | GlobalKind::Exported, | |
228 | is_tls, | |
229 | fn_attrs.link_section, | |
230 | ); | |
c295e0f8 XL |
231 | |
232 | if !self.tcx.is_reachable_non_generic(def_id) { | |
233 | // TODO(antoyo): set visibility. | |
234 | } | |
235 | ||
236 | global | |
237 | } | |
238 | ||
239 | Node::ForeignItem(&hir::ForeignItem { | |
487cf647 | 240 | span: _, |
c295e0f8 XL |
241 | kind: hir::ForeignItemKind::Static(..), |
242 | .. | |
243 | }) => { | |
244 | let fn_attrs = self.tcx.codegen_fn_attrs(def_id); | |
487cf647 | 245 | check_and_apply_linkage(&self, &fn_attrs, ty, sym) |
c295e0f8 XL |
246 | } |
247 | ||
248 | item => bug!("get_static: expected static, found {:?}", item), | |
249 | }; | |
250 | ||
251 | global | |
252 | } | |
253 | else { | |
254 | // FIXME(nagisa): perhaps the map of externs could be offloaded to llvm somehow? | |
255 | //debug!("get_static: sym={} item_attr={:?}", sym, self.tcx.item_attrs(def_id)); | |
256 | ||
257 | let attrs = self.tcx.codegen_fn_attrs(def_id); | |
487cf647 | 258 | let global = check_and_apply_linkage(&self, &attrs, ty, sym); |
c295e0f8 XL |
259 | |
260 | let needs_dll_storage_attr = false; // TODO(antoyo) | |
261 | ||
262 | // If this assertion triggers, there's something wrong with commandline | |
263 | // argument validation. | |
264 | debug_assert!( | |
265 | !(self.tcx.sess.opts.cg.linker_plugin_lto.enabled() | |
266 | && self.tcx.sess.target.options.is_like_msvc | |
267 | && self.tcx.sess.opts.cg.prefer_dynamic) | |
268 | ); | |
269 | ||
270 | if needs_dll_storage_attr { | |
271 | // This item is external but not foreign, i.e., it originates from an external Rust | |
272 | // crate. Since we don't know whether this crate will be linked dynamically or | |
273 | // statically in the final application, we always mark such symbols as 'dllimport'. | |
274 | // If final linkage happens to be static, we rely on compiler-emitted __imp_ stubs | |
275 | // to make things work. | |
276 | // | |
277 | // However, in some scenarios we defer emission of statics to downstream | |
278 | // crates, so there are cases where a static with an upstream DefId | |
279 | // is actually present in the current crate. We can find out via the | |
280 | // is_codegened_item query. | |
281 | if !self.tcx.is_codegened_item(def_id) { | |
282 | unimplemented!(); | |
283 | } | |
284 | } | |
285 | global | |
286 | }; | |
287 | ||
288 | // TODO(antoyo): set dll storage class. | |
289 | ||
290 | self.instances.borrow_mut().insert(instance, global); | |
291 | global | |
292 | } | |
293 | } | |
294 | ||
5e7ed085 FG |
295 | pub fn const_alloc_to_gcc<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, alloc: ConstAllocation<'tcx>) -> RValue<'gcc> { |
296 | let alloc = alloc.inner(); | |
487cf647 | 297 | let mut llvals = Vec::with_capacity(alloc.provenance().ptrs().len() + 1); |
c295e0f8 XL |
298 | let dl = cx.data_layout(); |
299 | let pointer_size = dl.pointer_size.bytes() as usize; | |
300 | ||
301 | let mut next_offset = 0; | |
487cf647 | 302 | for &(offset, alloc_id) in alloc.provenance().ptrs().iter() { |
c295e0f8 XL |
303 | let offset = offset.bytes(); |
304 | assert_eq!(offset as usize as u64, offset); | |
305 | let offset = offset as usize; | |
306 | if offset > next_offset { | |
f2b60f7d | 307 | // This `inspect` is okay since we have checked that it is not within a pointer with provenance, it |
c295e0f8 XL |
308 | // is within the bounds of the allocation, and it doesn't affect interpreter execution |
309 | // (we inspect the result after interpreter execution). Any undef byte is replaced with | |
310 | // some arbitrary byte value. | |
311 | // | |
312 | // FIXME: relay undef bytes to codegen as undef const bytes | |
313 | let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(next_offset..offset); | |
314 | llvals.push(cx.const_bytes(bytes)); | |
315 | } | |
316 | let ptr_offset = | |
317 | read_target_uint( dl.endian, | |
318 | // This `inspect` is okay since it is within the bounds of the allocation, it doesn't | |
319 | // affect interpreter execution (we inspect the result after interpreter execution), | |
f2b60f7d | 320 | // and we properly interpret the provenance as a relocation pointer offset. |
c295e0f8 XL |
321 | alloc.inspect_with_uninit_and_ptr_outside_interpreter(offset..(offset + pointer_size)), |
322 | ) | |
323 | .expect("const_alloc_to_llvm: could not read relocation pointer") | |
324 | as u64; | |
9ffffee4 FG |
325 | |
326 | let address_space = cx.tcx.global_alloc(alloc_id).address_space(cx); | |
327 | ||
c295e0f8 XL |
328 | llvals.push(cx.scalar_to_backend( |
329 | InterpScalar::from_pointer( | |
330 | interpret::Pointer::new(alloc_id, Size::from_bytes(ptr_offset)), | |
331 | &cx.tcx, | |
332 | ), | |
9ffffee4 FG |
333 | abi::Scalar::Initialized { value: Primitive::Pointer(address_space), valid_range: WrappingRange::full(dl.pointer_size) }, |
334 | cx.type_i8p_ext(address_space), | |
c295e0f8 XL |
335 | )); |
336 | next_offset = offset + pointer_size; | |
337 | } | |
338 | if alloc.len() >= next_offset { | |
339 | let range = next_offset..alloc.len(); | |
f2b60f7d | 340 | // This `inspect` is okay since we have check that it is after all provenance, it is |
c295e0f8 XL |
341 | // within the bounds of the allocation, and it doesn't affect interpreter execution (we |
342 | // inspect the result after interpreter execution). Any undef byte is replaced with some | |
343 | // arbitrary byte value. | |
344 | // | |
345 | // FIXME: relay undef bytes to codegen as undef const bytes | |
346 | let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(range); | |
347 | llvals.push(cx.const_bytes(bytes)); | |
348 | } | |
349 | ||
350 | cx.const_struct(&llvals, true) | |
351 | } | |
352 | ||
5e7ed085 | 353 | pub fn codegen_static_initializer<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, def_id: DefId) -> Result<(RValue<'gcc>, ConstAllocation<'tcx>), ErrorHandled> { |
c295e0f8 XL |
354 | let alloc = cx.tcx.eval_static_initializer(def_id)?; |
355 | Ok((const_alloc_to_gcc(cx, alloc), alloc)) | |
356 | } | |
357 | ||
487cf647 | 358 | fn check_and_apply_linkage<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, attrs: &CodegenFnAttrs, ty: Ty<'tcx>, sym: &str) -> LValue<'gcc> { |
c295e0f8 XL |
359 | let is_tls = attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL); |
360 | let llty = cx.layout_of(ty).gcc_type(cx, true); | |
487cf647 | 361 | if let Some(linkage) = attrs.import_linkage { |
c295e0f8 | 362 | // Declare a symbol `foo` with the desired linkage. |
487cf647 | 363 | let global1 = cx.declare_global_with_linkage(&sym, cx.type_i8(), base::global_linkage_to_gcc(linkage)); |
c295e0f8 XL |
364 | |
365 | // Declare an internal global `extern_with_linkage_foo` which | |
366 | // is initialized with the address of `foo`. If `foo` is | |
367 | // discarded during linking (for example, if `foo` has weak | |
368 | // linkage and there are no definitions), then | |
369 | // `extern_with_linkage_foo` will instead be initialized to | |
370 | // zero. | |
371 | let mut real_name = "_rust_extern_with_linkage_".to_string(); | |
372 | real_name.push_str(&sym); | |
373 | let global2 = cx.define_global(&real_name, llty, is_tls, attrs.link_section); | |
374 | // TODO(antoyo): set linkage. | |
a2a8927a | 375 | global2.global_set_initializer_rvalue(global1.get_address(None)); |
c295e0f8 XL |
376 | // TODO(antoyo): use global_set_initializer() when it will work. |
377 | global2 | |
378 | } | |
379 | else { | |
380 | // Generate an external declaration. | |
381 | // FIXME(nagisa): investigate whether it can be changed into define_global | |
382 | ||
383 | // Thread-local statics in some other crate need to *always* be linked | |
384 | // against in a thread-local fashion, so we need to be sure to apply the | |
385 | // thread-local attribute locally if it was present remotely. If we | |
386 | // don't do this then linker errors can be generated where the linker | |
387 | // complains that one object files has a thread local version of the | |
388 | // symbol and another one doesn't. | |
5e7ed085 | 389 | cx.declare_global(&sym, llty, GlobalKind::Imported, is_tls, attrs.link_section) |
c295e0f8 XL |
390 | } |
391 | } |