]>
Commit | Line | Data |
---|---|---|
5e7ed085 | 1 | use gccjit::{GlobalKind, LValue, RValue, ToRValue, Type}; |
c295e0f8 XL |
2 | use rustc_codegen_ssa::traits::{BaseTypeMethods, ConstMethods, DerivedTypeMethods, StaticMethods}; |
3 | use rustc_hir as hir; | |
4 | use rustc_hir::Node; | |
5 | use rustc_middle::{bug, span_bug}; | |
6 | use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; | |
7 | use rustc_middle::mir::mono::MonoItem; | |
8 | use rustc_middle::ty::{self, Instance, Ty}; | |
9 | use rustc_middle::ty::layout::LayoutOf; | |
5e7ed085 | 10 | use rustc_middle::mir::interpret::{self, ConstAllocation, ErrorHandled, Scalar as InterpScalar, read_target_uint}; |
c295e0f8 XL |
11 | use rustc_span::Span; |
12 | use rustc_span::def_id::DefId; | |
13 | use rustc_target::abi::{self, Align, HasDataLayout, Primitive, Size, WrappingRange}; | |
14 | ||
15 | use crate::base; | |
16 | use crate::context::CodegenCx; | |
17 | use crate::type_of::LayoutGccExt; | |
18 | ||
19 | impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { | |
20 | pub fn const_bitcast(&self, value: RValue<'gcc>, typ: Type<'gcc>) -> RValue<'gcc> { | |
21 | if value.get_type() == self.bool_type.make_pointer() { | |
22 | if let Some(pointee) = typ.get_pointee() { | |
a2a8927a | 23 | if pointee.dyncast_vector().is_some() { |
c295e0f8 XL |
24 | panic!() |
25 | } | |
26 | } | |
27 | } | |
923072b8 FG |
28 | // NOTE: since bitcast makes a value non-constant, don't bitcast if not necessary as some |
29 | // SIMD builtins require a constant value. | |
30 | self.bitcast_if_needed(value, typ) | |
c295e0f8 XL |
31 | } |
32 | } | |
33 | ||
34 | impl<'gcc, 'tcx> StaticMethods for CodegenCx<'gcc, 'tcx> { | |
35 | fn static_addr_of(&self, cv: RValue<'gcc>, align: Align, kind: Option<&str>) -> RValue<'gcc> { | |
a2a8927a XL |
36 | // TODO(antoyo): implement a proper rvalue comparison in libgccjit instead of doing the |
37 | // following: | |
38 | for (value, variable) in &*self.const_globals.borrow() { | |
39 | if format!("{:?}", value) == format!("{:?}", cv) { | |
5e7ed085 FG |
40 | if let Some(global_variable) = self.global_lvalues.borrow().get(variable) { |
41 | let alignment = align.bits() as i32; | |
42 | if alignment > global_variable.get_alignment() { | |
43 | global_variable.set_alignment(alignment); | |
44 | } | |
45 | } | |
a2a8927a XL |
46 | return *variable; |
47 | } | |
c295e0f8 XL |
48 | } |
49 | let global_value = self.static_addr_of_mut(cv, align, kind); | |
923072b8 FG |
50 | #[cfg(feature = "master")] |
51 | self.global_lvalues.borrow().get(&global_value) | |
52 | .expect("`static_addr_of_mut` did not add the global to `self.global_lvalues`") | |
53 | .global_set_readonly(); | |
c295e0f8 XL |
54 | self.const_globals.borrow_mut().insert(cv, global_value); |
55 | global_value | |
56 | } | |
57 | ||
58 | fn codegen_static(&self, def_id: DefId, is_mutable: bool) { | |
59 | let attrs = self.tcx.codegen_fn_attrs(def_id); | |
60 | ||
61 | let value = | |
62 | match codegen_static_initializer(&self, def_id) { | |
63 | Ok((value, _)) => value, | |
64 | // Error has already been reported | |
65 | Err(_) => return, | |
66 | }; | |
67 | ||
68 | let global = self.get_static(def_id); | |
69 | ||
70 | // boolean SSA values are i1, but they have to be stored in i8 slots, | |
71 | // otherwise some LLVM optimization passes don't work as expected | |
72 | let val_llty = self.val_ty(value); | |
73 | let value = | |
74 | if val_llty == self.type_i1() { | |
75 | unimplemented!(); | |
76 | } | |
77 | else { | |
78 | value | |
79 | }; | |
80 | ||
81 | let instance = Instance::mono(self.tcx, def_id); | |
82 | let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); | |
83 | let gcc_type = self.layout_of(ty).gcc_type(self, true); | |
84 | ||
85 | // TODO(antoyo): set alignment. | |
86 | ||
923072b8 | 87 | let value = self.bitcast_if_needed(value, gcc_type); |
a2a8927a | 88 | global.global_set_initializer_rvalue(value); |
c295e0f8 XL |
89 | |
90 | // As an optimization, all shared statics which do not have interior | |
91 | // mutability are placed into read-only memory. | |
92 | if !is_mutable { | |
93 | if self.type_is_freeze(ty) { | |
923072b8 FG |
94 | #[cfg(feature = "master")] |
95 | global.global_set_readonly(); | |
c295e0f8 XL |
96 | } |
97 | } | |
98 | ||
99 | if attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) { | |
100 | // Do not allow LLVM to change the alignment of a TLS on macOS. | |
101 | // | |
102 | // By default a global's alignment can be freely increased. | |
103 | // This allows LLVM to generate more performant instructions | |
104 | // e.g., using load-aligned into a SIMD register. | |
105 | // | |
106 | // However, on macOS 10.10 or below, the dynamic linker does not | |
107 | // respect any alignment given on the TLS (radar 24221680). | |
108 | // This will violate the alignment assumption, and causing segfault at runtime. | |
109 | // | |
110 | // This bug is very easy to trigger. In `println!` and `panic!`, | |
111 | // the `LOCAL_STDOUT`/`LOCAL_STDERR` handles are stored in a TLS, | |
112 | // which the values would be `mem::replace`d on initialization. | |
113 | // The implementation of `mem::replace` will use SIMD | |
114 | // whenever the size is 32 bytes or higher. LLVM notices SIMD is used | |
115 | // and tries to align `LOCAL_STDOUT`/`LOCAL_STDERR` to a 32-byte boundary, | |
116 | // which macOS's dyld disregarded and causing crashes | |
117 | // (see issues #51794, #51758, #50867, #48866 and #44056). | |
118 | // | |
119 | // To workaround the bug, we trick LLVM into not increasing | |
120 | // the global's alignment by explicitly assigning a section to it | |
121 | // (equivalent to automatically generating a `#[link_section]` attribute). | |
122 | // See the comment in the `GlobalValue::canIncreaseAlignment()` function | |
123 | // of `lib/IR/Globals.cpp` for why this works. | |
124 | // | |
125 | // When the alignment is not increased, the optimized `mem::replace` | |
126 | // will use load-unaligned instructions instead, and thus avoiding the crash. | |
127 | // | |
128 | // We could remove this hack whenever we decide to drop macOS 10.10 support. | |
129 | if self.tcx.sess.target.options.is_like_osx { | |
f2b60f7d | 130 | // The `inspect` method is okay here because we checked for provenance, and |
c295e0f8 XL |
131 | // because we are doing this access to inspect the final interpreter state |
132 | // (not as part of the interpreter execution). | |
133 | // | |
134 | // FIXME: This check requires that the (arbitrary) value of undefined bytes | |
135 | // happens to be zero. Instead, we should only check the value of defined bytes | |
136 | // and set all undefined bytes to zero if this allocation is headed for the | |
137 | // BSS. | |
138 | unimplemented!(); | |
139 | } | |
140 | } | |
141 | ||
142 | // Wasm statics with custom link sections get special treatment as they | |
143 | // go into custom sections of the wasm executable. | |
144 | if self.tcx.sess.opts.target_triple.triple().starts_with("wasm32") { | |
145 | if let Some(_section) = attrs.link_section { | |
146 | unimplemented!(); | |
147 | } | |
148 | } else { | |
149 | // TODO(antoyo): set link section. | |
150 | } | |
151 | ||
5099ac24 | 152 | if attrs.flags.contains(CodegenFnAttrFlags::USED) || attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER) { |
c295e0f8 XL |
153 | self.add_used_global(global.to_rvalue()); |
154 | } | |
155 | } | |
156 | ||
157 | /// Add a global value to a list to be stored in the `llvm.used` variable, an array of i8*. | |
158 | fn add_used_global(&self, _global: RValue<'gcc>) { | |
159 | // TODO(antoyo) | |
160 | } | |
161 | ||
162 | fn add_compiler_used_global(&self, _global: RValue<'gcc>) { | |
163 | // TODO(antoyo) | |
164 | } | |
165 | } | |
166 | ||
167 | impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { | |
168 | pub fn static_addr_of_mut(&self, cv: RValue<'gcc>, align: Align, kind: Option<&str>) -> RValue<'gcc> { | |
169 | let global = | |
170 | match kind { | |
171 | Some(kind) if !self.tcx.sess.fewer_names() => { | |
172 | let name = self.generate_local_symbol_name(kind); | |
5e7ed085 | 173 | // TODO(antoyo): check if it's okay that no link_section is set. |
923072b8 FG |
174 | |
175 | let typ = self.val_ty(cv).get_aligned(align.bytes()); | |
176 | let global = self.declare_private_global(&name[..], typ); | |
c295e0f8 XL |
177 | global |
178 | } | |
179 | _ => { | |
180 | let typ = self.val_ty(cv).get_aligned(align.bytes()); | |
181 | let global = self.declare_unnamed_global(typ); | |
182 | global | |
183 | }, | |
184 | }; | |
a2a8927a | 185 | global.global_set_initializer_rvalue(cv); |
c295e0f8 | 186 | // TODO(antoyo): set unnamed address. |
5e7ed085 FG |
187 | let rvalue = global.get_address(None); |
188 | self.global_lvalues.borrow_mut().insert(rvalue, global); | |
189 | rvalue | |
c295e0f8 XL |
190 | } |
191 | ||
192 | pub fn get_static(&self, def_id: DefId) -> LValue<'gcc> { | |
193 | let instance = Instance::mono(self.tcx, def_id); | |
194 | let fn_attrs = self.tcx.codegen_fn_attrs(def_id); | |
195 | if let Some(&global) = self.instances.borrow().get(&instance) { | |
196 | return global; | |
197 | } | |
198 | ||
199 | let defined_in_current_codegen_unit = | |
200 | self.codegen_unit.items().contains_key(&MonoItem::Static(def_id)); | |
201 | assert!( | |
202 | !defined_in_current_codegen_unit, | |
203 | "consts::get_static() should always hit the cache for \ | |
204 | statics defined in the same CGU, but did not for `{:?}`", | |
205 | def_id | |
206 | ); | |
207 | ||
208 | let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); | |
209 | let sym = self.tcx.symbol_name(instance).name; | |
210 | ||
211 | let global = | |
212 | if let Some(def_id) = def_id.as_local() { | |
213 | let id = self.tcx.hir().local_def_id_to_hir_id(def_id); | |
214 | let llty = self.layout_of(ty).gcc_type(self, true); | |
215 | // FIXME: refactor this to work without accessing the HIR | |
216 | let global = match self.tcx.hir().get(id) { | |
217 | Node::Item(&hir::Item { span, kind: hir::ItemKind::Static(..), .. }) => { | |
218 | if let Some(global) = self.get_declared_value(&sym) { | |
219 | if self.val_ty(global) != self.type_ptr_to(llty) { | |
220 | span_bug!(span, "Conflicting types for static"); | |
221 | } | |
222 | } | |
223 | ||
224 | let is_tls = fn_attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL); | |
5e7ed085 FG |
225 | let global = self.declare_global( |
226 | &sym, | |
227 | llty, | |
228 | GlobalKind::Exported, | |
229 | is_tls, | |
230 | fn_attrs.link_section, | |
231 | ); | |
c295e0f8 XL |
232 | |
233 | if !self.tcx.is_reachable_non_generic(def_id) { | |
234 | // TODO(antoyo): set visibility. | |
235 | } | |
236 | ||
237 | global | |
238 | } | |
239 | ||
240 | Node::ForeignItem(&hir::ForeignItem { | |
241 | span, | |
242 | kind: hir::ForeignItemKind::Static(..), | |
243 | .. | |
244 | }) => { | |
245 | let fn_attrs = self.tcx.codegen_fn_attrs(def_id); | |
246 | check_and_apply_linkage(&self, &fn_attrs, ty, sym, span) | |
247 | } | |
248 | ||
249 | item => bug!("get_static: expected static, found {:?}", item), | |
250 | }; | |
251 | ||
252 | global | |
253 | } | |
254 | else { | |
255 | // FIXME(nagisa): perhaps the map of externs could be offloaded to llvm somehow? | |
256 | //debug!("get_static: sym={} item_attr={:?}", sym, self.tcx.item_attrs(def_id)); | |
257 | ||
258 | let attrs = self.tcx.codegen_fn_attrs(def_id); | |
259 | let span = self.tcx.def_span(def_id); | |
260 | let global = check_and_apply_linkage(&self, &attrs, ty, sym, span); | |
261 | ||
262 | let needs_dll_storage_attr = false; // TODO(antoyo) | |
263 | ||
264 | // If this assertion triggers, there's something wrong with commandline | |
265 | // argument validation. | |
266 | debug_assert!( | |
267 | !(self.tcx.sess.opts.cg.linker_plugin_lto.enabled() | |
268 | && self.tcx.sess.target.options.is_like_msvc | |
269 | && self.tcx.sess.opts.cg.prefer_dynamic) | |
270 | ); | |
271 | ||
272 | if needs_dll_storage_attr { | |
273 | // This item is external but not foreign, i.e., it originates from an external Rust | |
274 | // crate. Since we don't know whether this crate will be linked dynamically or | |
275 | // statically in the final application, we always mark such symbols as 'dllimport'. | |
276 | // If final linkage happens to be static, we rely on compiler-emitted __imp_ stubs | |
277 | // to make things work. | |
278 | // | |
279 | // However, in some scenarios we defer emission of statics to downstream | |
280 | // crates, so there are cases where a static with an upstream DefId | |
281 | // is actually present in the current crate. We can find out via the | |
282 | // is_codegened_item query. | |
283 | if !self.tcx.is_codegened_item(def_id) { | |
284 | unimplemented!(); | |
285 | } | |
286 | } | |
287 | global | |
288 | }; | |
289 | ||
290 | // TODO(antoyo): set dll storage class. | |
291 | ||
292 | self.instances.borrow_mut().insert(instance, global); | |
293 | global | |
294 | } | |
295 | } | |
296 | ||
5e7ed085 FG |
297 | pub fn const_alloc_to_gcc<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, alloc: ConstAllocation<'tcx>) -> RValue<'gcc> { |
298 | let alloc = alloc.inner(); | |
f2b60f7d | 299 | let mut llvals = Vec::with_capacity(alloc.provenance().len() + 1); |
c295e0f8 XL |
300 | let dl = cx.data_layout(); |
301 | let pointer_size = dl.pointer_size.bytes() as usize; | |
302 | ||
303 | let mut next_offset = 0; | |
f2b60f7d | 304 | for &(offset, alloc_id) in alloc.provenance().iter() { |
c295e0f8 XL |
305 | let offset = offset.bytes(); |
306 | assert_eq!(offset as usize as u64, offset); | |
307 | let offset = offset as usize; | |
308 | if offset > next_offset { | |
f2b60f7d | 309 | // This `inspect` is okay since we have checked that it is not within a pointer with provenance, it |
c295e0f8 XL |
310 | // is within the bounds of the allocation, and it doesn't affect interpreter execution |
311 | // (we inspect the result after interpreter execution). Any undef byte is replaced with | |
312 | // some arbitrary byte value. | |
313 | // | |
314 | // FIXME: relay undef bytes to codegen as undef const bytes | |
315 | let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(next_offset..offset); | |
316 | llvals.push(cx.const_bytes(bytes)); | |
317 | } | |
318 | let ptr_offset = | |
319 | read_target_uint( dl.endian, | |
320 | // This `inspect` is okay since it is within the bounds of the allocation, it doesn't | |
321 | // affect interpreter execution (we inspect the result after interpreter execution), | |
f2b60f7d | 322 | // and we properly interpret the provenance as a relocation pointer offset. |
c295e0f8 XL |
323 | alloc.inspect_with_uninit_and_ptr_outside_interpreter(offset..(offset + pointer_size)), |
324 | ) | |
325 | .expect("const_alloc_to_llvm: could not read relocation pointer") | |
326 | as u64; | |
327 | llvals.push(cx.scalar_to_backend( | |
328 | InterpScalar::from_pointer( | |
329 | interpret::Pointer::new(alloc_id, Size::from_bytes(ptr_offset)), | |
330 | &cx.tcx, | |
331 | ), | |
04454e1e | 332 | abi::Scalar::Initialized { value: Primitive::Pointer, valid_range: WrappingRange::full(dl.pointer_size) }, |
c295e0f8 XL |
333 | cx.type_i8p(), |
334 | )); | |
335 | next_offset = offset + pointer_size; | |
336 | } | |
337 | if alloc.len() >= next_offset { | |
338 | let range = next_offset..alloc.len(); | |
f2b60f7d | 339 | // This `inspect` is okay since we have check that it is after all provenance, it is |
c295e0f8 XL |
340 | // within the bounds of the allocation, and it doesn't affect interpreter execution (we |
341 | // inspect the result after interpreter execution). Any undef byte is replaced with some | |
342 | // arbitrary byte value. | |
343 | // | |
344 | // FIXME: relay undef bytes to codegen as undef const bytes | |
345 | let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(range); | |
346 | llvals.push(cx.const_bytes(bytes)); | |
347 | } | |
348 | ||
349 | cx.const_struct(&llvals, true) | |
350 | } | |
351 | ||
5e7ed085 | 352 | pub fn codegen_static_initializer<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, def_id: DefId) -> Result<(RValue<'gcc>, ConstAllocation<'tcx>), ErrorHandled> { |
c295e0f8 XL |
353 | let alloc = cx.tcx.eval_static_initializer(def_id)?; |
354 | Ok((const_alloc_to_gcc(cx, alloc), alloc)) | |
355 | } | |
356 | ||
357 | fn check_and_apply_linkage<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, attrs: &CodegenFnAttrs, ty: Ty<'tcx>, sym: &str, span: Span) -> LValue<'gcc> { | |
358 | let is_tls = attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL); | |
359 | let llty = cx.layout_of(ty).gcc_type(cx, true); | |
360 | if let Some(linkage) = attrs.linkage { | |
361 | // If this is a static with a linkage specified, then we need to handle | |
362 | // it a little specially. The typesystem prevents things like &T and | |
363 | // extern "C" fn() from being non-null, so we can't just declare a | |
364 | // static and call it a day. Some linkages (like weak) will make it such | |
365 | // that the static actually has a null value. | |
366 | let llty2 = | |
367 | if let ty::RawPtr(ref mt) = ty.kind() { | |
368 | cx.layout_of(mt.ty).gcc_type(cx, true) | |
369 | } | |
370 | else { | |
371 | cx.sess().span_fatal( | |
372 | span, | |
373 | "must have type `*const T` or `*mut T` due to `#[linkage]` attribute", | |
374 | ) | |
375 | }; | |
376 | // Declare a symbol `foo` with the desired linkage. | |
377 | let global1 = cx.declare_global_with_linkage(&sym, llty2, base::global_linkage_to_gcc(linkage)); | |
378 | ||
379 | // Declare an internal global `extern_with_linkage_foo` which | |
380 | // is initialized with the address of `foo`. If `foo` is | |
381 | // discarded during linking (for example, if `foo` has weak | |
382 | // linkage and there are no definitions), then | |
383 | // `extern_with_linkage_foo` will instead be initialized to | |
384 | // zero. | |
385 | let mut real_name = "_rust_extern_with_linkage_".to_string(); | |
386 | real_name.push_str(&sym); | |
387 | let global2 = cx.define_global(&real_name, llty, is_tls, attrs.link_section); | |
388 | // TODO(antoyo): set linkage. | |
a2a8927a | 389 | global2.global_set_initializer_rvalue(global1.get_address(None)); |
c295e0f8 XL |
390 | // TODO(antoyo): use global_set_initializer() when it will work. |
391 | global2 | |
392 | } | |
393 | else { | |
394 | // Generate an external declaration. | |
395 | // FIXME(nagisa): investigate whether it can be changed into define_global | |
396 | ||
397 | // Thread-local statics in some other crate need to *always* be linked | |
398 | // against in a thread-local fashion, so we need to be sure to apply the | |
399 | // thread-local attribute locally if it was present remotely. If we | |
400 | // don't do this then linker errors can be generated where the linker | |
401 | // complains that one object files has a thread local version of the | |
402 | // symbol and another one doesn't. | |
5e7ed085 | 403 | cx.declare_global(&sym, llty, GlobalKind::Imported, is_tls, attrs.link_section) |
c295e0f8 XL |
404 | } |
405 | } |