]>
Commit | Line | Data |
---|---|---|
9fa01778 | 1 | use crate::base; |
f2b60f7d | 2 | use crate::common::{self, CodegenCx}; |
dfeec247 | 3 | use crate::debuginfo; |
fe692bf9 FG |
4 | use crate::errors::{ |
5 | InvalidMinimumAlignmentNotPowerOfTwo, InvalidMinimumAlignmentTooLarge, SymbolAlreadyDefined, | |
6 | }; | |
ba9703b0 | 7 | use crate::llvm::{self, True}; |
9fa01778 XL |
8 | use crate::type_::Type; |
9 | use crate::type_of::LayoutLlvmExt; | |
10 | use crate::value::Value; | |
dfeec247 | 11 | use rustc_codegen_ssa::traits::*; |
c620b35d | 12 | use rustc_hir::def::DefKind; |
dfeec247 | 13 | use rustc_hir::def_id::DefId; |
ba9703b0 XL |
14 | use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; |
15 | use rustc_middle::mir::interpret::{ | |
9ffffee4 | 16 | read_target_uint, Allocation, ConstAllocation, ErrorHandled, InitChunk, Pointer, |
94222f64 | 17 | Scalar as InterpScalar, |
ba9703b0 XL |
18 | }; |
19 | use rustc_middle::mir::mono::MonoItem; | |
c295e0f8 | 20 | use rustc_middle::ty::layout::LayoutOf; |
c620b35d | 21 | use rustc_middle::ty::{self, Instance}; |
ba9703b0 | 22 | use rustc_middle::{bug, span_bug}; |
487cf647 | 23 | use rustc_session::config::Lto; |
fe692bf9 FG |
24 | use rustc_target::abi::{ |
25 | Align, AlignFromBytesError, HasDataLayout, Primitive, Scalar, Size, WrappingRange, | |
26 | }; | |
94222f64 | 27 | use std::ops::Range; |
e9174d1e | 28 | |
e8be2606 FG |
29 | pub fn const_alloc_to_llvm<'ll>( |
30 | cx: &CodegenCx<'ll, '_>, | |
31 | alloc: ConstAllocation<'_>, | |
32 | is_static: bool, | |
33 | ) -> &'ll Value { | |
5e7ed085 | 34 | let alloc = alloc.inner(); |
e8be2606 FG |
35 | // We expect that callers of const_alloc_to_llvm will instead directly codegen a pointer or |
36 | // integer for any &ZST where the ZST is a constant (i.e. not a static). We should never be | |
37 | // producing empty LLVM allocations as they're just adding noise to binaries and forcing less | |
38 | // optimal codegen. | |
39 | // | |
40 | // Statics have a guaranteed meaningful address so it's less clear that we want to do | |
41 | // something like this; it's also harder. | |
42 | if !is_static { | |
43 | assert!(alloc.len() != 0); | |
44 | } | |
487cf647 | 45 | let mut llvals = Vec::with_capacity(alloc.provenance().ptrs().len() + 1); |
a1dfa0c6 XL |
46 | let dl = cx.data_layout(); |
47 | let pointer_size = dl.pointer_size.bytes() as usize; | |
48 | ||
f2b60f7d FG |
49 | // Note: this function may call `inspect_with_uninit_and_ptr_outside_interpreter`, so `range` |
50 | // must be within the bounds of `alloc` and not contain or overlap a pointer provenance. | |
94222f64 XL |
51 | fn append_chunks_of_init_and_uninit_bytes<'ll, 'a, 'b>( |
52 | llvals: &mut Vec<&'ll Value>, | |
53 | cx: &'a CodegenCx<'ll, 'b>, | |
54 | alloc: &'a Allocation, | |
55 | range: Range<usize>, | |
56 | ) { | |
487cf647 | 57 | let chunks = alloc.init_mask().range_as_init_chunks(range.clone().into()); |
94222f64 XL |
58 | |
59 | let chunk_to_llval = move |chunk| match chunk { | |
60 | InitChunk::Init(range) => { | |
61 | let range = (range.start.bytes() as usize)..(range.end.bytes() as usize); | |
62 | let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(range); | |
63 | cx.const_bytes(bytes) | |
64 | } | |
65 | InitChunk::Uninit(range) => { | |
66 | let len = range.end.bytes() - range.start.bytes(); | |
67 | cx.const_undef(cx.type_array(cx.type_i8(), len)) | |
68 | } | |
69 | }; | |
70 | ||
5e7ed085 FG |
71 | // Generating partially-uninit consts is limited to small numbers of chunks, |
72 | // to avoid the cost of generating large complex const expressions. | |
73 | // For example, `[(u32, u8); 1024 * 1024]` contains uninit padding in each element, | |
74 | // and would result in `{ [5 x i8] zeroinitializer, [3 x i8] undef, ...repeat 1M times... }`. | |
9ffffee4 | 75 | let max = cx.sess().opts.unstable_opts.uninit_const_chunk_threshold; |
5e7ed085 | 76 | let allow_uninit_chunks = chunks.clone().take(max.saturating_add(1)).count() <= max; |
94222f64 | 77 | |
5e7ed085 | 78 | if allow_uninit_chunks { |
94222f64 XL |
79 | llvals.extend(chunks.map(chunk_to_llval)); |
80 | } else { | |
5e7ed085 FG |
81 | // If this allocation contains any uninit bytes, codegen as if it was initialized |
82 | // (using some arbitrary value for uninit bytes). | |
83 | let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(range); | |
84 | llvals.push(cx.const_bytes(bytes)); | |
94222f64 XL |
85 | } |
86 | } | |
87 | ||
a1dfa0c6 | 88 | let mut next_offset = 0; |
4b012472 | 89 | for &(offset, prov) in alloc.provenance().ptrs().iter() { |
a1dfa0c6 XL |
90 | let offset = offset.bytes(); |
91 | assert_eq!(offset as usize as u64, offset); | |
92 | let offset = offset as usize; | |
93 | if offset > next_offset { | |
f2b60f7d | 94 | // This `inspect` is okay since we have checked that there is no provenance, it |
e1599b0c | 95 | // is within the bounds of the allocation, and it doesn't affect interpreter execution |
94222f64 XL |
96 | // (we inspect the result after interpreter execution). |
97 | append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, next_offset..offset); | |
a1dfa0c6 XL |
98 | } |
99 | let ptr_offset = read_target_uint( | |
100 | dl.endian, | |
e1599b0c XL |
101 | // This `inspect` is okay since it is within the bounds of the allocation, it doesn't |
102 | // affect interpreter execution (we inspect the result after interpreter execution), | |
f2b60f7d | 103 | // and we properly interpret the provenance as a relocation pointer offset. |
3dfed10e | 104 | alloc.inspect_with_uninit_and_ptr_outside_interpreter(offset..(offset + pointer_size)), |
dfeec247 XL |
105 | ) |
106 | .expect("const_alloc_to_llvm: could not read relocation pointer") | |
107 | as u64; | |
3dfed10e | 108 | |
4b012472 | 109 | let address_space = cx.tcx.global_alloc(prov.alloc_id()).address_space(cx); |
3dfed10e | 110 | |
a1dfa0c6 | 111 | llvals.push(cx.scalar_to_backend( |
4b012472 | 112 | InterpScalar::from_pointer(Pointer::new(prov, Size::from_bytes(ptr_offset)), &cx.tcx), |
04454e1e | 113 | Scalar::Initialized { |
9ffffee4 | 114 | value: Primitive::Pointer(address_space), |
04454e1e FG |
115 | valid_range: WrappingRange::full(dl.pointer_size), |
116 | }, | |
add651ee | 117 | cx.type_ptr_ext(address_space), |
a1dfa0c6 XL |
118 | )); |
119 | next_offset = offset + pointer_size; | |
1a4d82fc | 120 | } |
e1599b0c XL |
121 | if alloc.len() >= next_offset { |
122 | let range = next_offset..alloc.len(); | |
f2b60f7d | 123 | // This `inspect` is okay since we have check that it is after all provenance, it is |
e1599b0c | 124 | // within the bounds of the allocation, and it doesn't affect interpreter execution (we |
94222f64 XL |
125 | // inspect the result after interpreter execution). |
126 | append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, range); | |
a1dfa0c6 XL |
127 | } |
128 | ||
129 | cx.const_struct(&llvals, true) | |
1a4d82fc JJ |
130 | } |
131 | ||
c620b35d | 132 | fn codegen_static_initializer<'ll, 'tcx>( |
a1dfa0c6 XL |
133 | cx: &CodegenCx<'ll, 'tcx>, |
134 | def_id: DefId, | |
5e7ed085 | 135 | ) -> Result<(&'ll Value, ConstAllocation<'tcx>), ErrorHandled> { |
1b1a35ee | 136 | let alloc = cx.tcx.eval_static_initializer(def_id)?; |
e8be2606 | 137 | Ok((const_alloc_to_llvm(cx, alloc, /*static*/ true), alloc)) |
3b2f2976 XL |
138 | } |
139 | ||
a2a8927a | 140 | fn set_global_alignment<'ll>(cx: &CodegenCx<'ll, '_>, gv: &'ll Value, mut align: Align) { |
ea8adc8c XL |
141 | // The target may require greater alignment for globals than the type does. |
142 | // Note: GCC and Clang also allow `__attribute__((aligned))` on variables, | |
9c376795 | 143 | // which can force it to be smaller. Rust doesn't support this yet. |
29967ef6 | 144 | if let Some(min) = cx.sess().target.min_global_align { |
a1dfa0c6 | 145 | match Align::from_bits(min) { |
ff7c6d11 | 146 | Ok(min) => align = align.max(min), |
fe692bf9 FG |
147 | Err(err) => match err { |
148 | AlignFromBytesError::NotPowerOfTwo(align) => { | |
c0240ec0 | 149 | cx.sess().dcx().emit_err(InvalidMinimumAlignmentNotPowerOfTwo { align }); |
fe692bf9 FG |
150 | } |
151 | AlignFromBytesError::TooLarge(align) => { | |
c0240ec0 | 152 | cx.sess().dcx().emit_err(InvalidMinimumAlignmentTooLarge { align }); |
fe692bf9 FG |
153 | } |
154 | }, | |
ea8adc8c XL |
155 | } |
156 | } | |
157 | unsafe { | |
a1dfa0c6 | 158 | llvm::LLVMSetAlignment(gv, align.bytes() as u32); |
ea8adc8c XL |
159 | } |
160 | } | |
161 | ||
a2a8927a | 162 | fn check_and_apply_linkage<'ll, 'tcx>( |
b7449926 | 163 | cx: &CodegenCx<'ll, 'tcx>, |
8faf50e0 | 164 | attrs: &CodegenFnAttrs, |
c620b35d | 165 | llty: &'ll Type, |
3dfed10e | 166 | sym: &str, |
f2b60f7d | 167 | def_id: DefId, |
b7449926 | 168 | ) -> &'ll Value { |
487cf647 | 169 | if let Some(linkage) = attrs.import_linkage { |
8faf50e0 XL |
170 | debug!("get_static: sym={} linkage={:?}", sym, linkage); |
171 | ||
8faf50e0 XL |
172 | unsafe { |
173 | // Declare a symbol `foo` with the desired linkage. | |
487cf647 | 174 | let g1 = cx.declare_global(sym, cx.type_i8()); |
8faf50e0 XL |
175 | llvm::LLVMRustSetLinkage(g1, base::linkage_to_llvm(linkage)); |
176 | ||
177 | // Declare an internal global `extern_with_linkage_foo` which | |
9c376795 | 178 | // is initialized with the address of `foo`. If `foo` is |
8faf50e0 XL |
179 | // discarded during linking (for example, if `foo` has weak |
180 | // linkage and there are no definitions), then | |
181 | // `extern_with_linkage_foo` will instead be initialized to | |
182 | // zero. | |
183 | let mut real_name = "_rust_extern_with_linkage_".to_string(); | |
c295e0f8 | 184 | real_name.push_str(sym); |
dfeec247 | 185 | let g2 = cx.define_global(&real_name, llty).unwrap_or_else(|| { |
c0240ec0 | 186 | cx.sess().dcx().emit_fatal(SymbolAlreadyDefined { |
487cf647 FG |
187 | span: cx.tcx.def_span(def_id), |
188 | symbol_name: sym, | |
189 | }) | |
8faf50e0 XL |
190 | }); |
191 | llvm::LLVMRustSetLinkage(g2, llvm::Linkage::InternalLinkage); | |
add651ee | 192 | llvm::LLVMSetInitializer(g2, g1); |
8faf50e0 XL |
193 | g2 |
194 | } | |
ed00b5ec FG |
195 | } else if cx.tcx.sess.target.arch == "x86" |
196 | && let Some(dllimport) = common::get_dllimport(cx.tcx, def_id, sym) | |
f2b60f7d | 197 | { |
ed00b5ec FG |
198 | cx.declare_global( |
199 | &common::i686_decorated_name( | |
4b012472 | 200 | dllimport, |
ed00b5ec FG |
201 | common::is_mingw_gnu_toolchain(&cx.tcx.sess.target), |
202 | true, | |
203 | ), | |
204 | llty, | |
205 | ) | |
8faf50e0 XL |
206 | } else { |
207 | // Generate an external declaration. | |
208 | // FIXME(nagisa): investigate whether it can be changed into define_global | |
c295e0f8 | 209 | cx.declare_global(sym, llty) |
8faf50e0 XL |
210 | } |
211 | } | |
212 | ||
a2a8927a | 213 | impl<'ll> CodegenCx<'ll, '_> { |
923072b8 | 214 | pub(crate) fn const_bitcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { |
dfeec247 | 215 | unsafe { llvm::LLVMConstBitCast(val, ty) } |
a1dfa0c6 | 216 | } |
c1a9b12d | 217 | |
923072b8 | 218 | pub(crate) fn static_addr_of_mut( |
a1dfa0c6 XL |
219 | &self, |
220 | cv: &'ll Value, | |
221 | align: Align, | |
222 | kind: Option<&str>, | |
223 | ) -> &'ll Value { | |
224 | unsafe { | |
225 | let gv = match kind { | |
226 | Some(kind) if !self.tcx.sess.fewer_names() => { | |
227 | let name = self.generate_local_symbol_name(kind); | |
a2a8927a | 228 | let gv = self.define_global(&name, self.val_ty(cv)).unwrap_or_else(|| { |
dfeec247 | 229 | bug!("symbol `{}` is already defined", name); |
a1dfa0c6 XL |
230 | }); |
231 | llvm::LLVMRustSetLinkage(gv, llvm::Linkage::PrivateLinkage); | |
232 | gv | |
dfeec247 | 233 | } |
a1dfa0c6 XL |
234 | _ => self.define_private_global(self.val_ty(cv)), |
235 | }; | |
236 | llvm::LLVMSetInitializer(gv, cv); | |
c295e0f8 | 237 | set_global_alignment(self, gv, align); |
ba9703b0 | 238 | llvm::SetUnnamedAddress(gv, llvm::UnnamedAddr::Global); |
a1dfa0c6 XL |
239 | gv |
240 | } | |
241 | } | |
8faf50e0 | 242 | |
c620b35d | 243 | #[instrument(level = "debug", skip(self))] |
923072b8 | 244 | pub(crate) fn get_static(&self, def_id: DefId) -> &'ll Value { |
a1dfa0c6 | 245 | let instance = Instance::mono(self.tcx, def_id); |
c620b35d FG |
246 | trace!(?instance); |
247 | ||
248 | let DefKind::Static { nested, .. } = self.tcx.def_kind(def_id) else { bug!() }; | |
249 | // Nested statics do not have a type, so pick a dummy type and let `codegen_static` figure out | |
250 | // the llvm type from the actual evaluated initializer. | |
251 | let llty = if nested { | |
252 | self.type_i8() | |
253 | } else { | |
254 | let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); | |
255 | trace!(?ty); | |
256 | self.layout_of(ty).llvm_type(self) | |
257 | }; | |
258 | self.get_static_inner(def_id, llty) | |
259 | } | |
260 | ||
261 | #[instrument(level = "debug", skip(self, llty))] | |
262 | pub(crate) fn get_static_inner(&self, def_id: DefId, llty: &'ll Type) -> &'ll Value { | |
e8be2606 FG |
263 | let instance = Instance::mono(self.tcx, def_id); |
264 | if let Some(&g) = self.instances.borrow().get(&instance) { | |
c620b35d | 265 | trace!("used cached value"); |
a1dfa0c6 XL |
266 | return g; |
267 | } | |
268 | ||
dfeec247 XL |
269 | let defined_in_current_codegen_unit = |
270 | self.codegen_unit.items().contains_key(&MonoItem::Static(def_id)); | |
271 | assert!( | |
272 | !defined_in_current_codegen_unit, | |
273 | "consts::get_static() should always hit the cache for \ | |
add651ee | 274 | statics defined in the same CGU, but did not for `{def_id:?}`" |
dfeec247 | 275 | ); |
a1dfa0c6 | 276 | |
e8be2606 | 277 | let sym = self.tcx.symbol_name(instance).name; |
5869c6ff | 278 | let fn_attrs = self.tcx.codegen_fn_attrs(def_id); |
a1dfa0c6 | 279 | |
c620b35d | 280 | debug!(?sym, ?fn_attrs); |
a1dfa0c6 | 281 | |
5869c6ff | 282 | let g = if def_id.is_local() && !self.tcx.is_foreign_item(def_id) { |
5869c6ff | 283 | if let Some(g) = self.get_declared_value(sym) { |
add651ee | 284 | if self.val_ty(g) != self.type_ptr() { |
5869c6ff | 285 | span_bug!(self.tcx.def_span(def_id), "Conflicting types for static"); |
a1dfa0c6 | 286 | } |
5869c6ff | 287 | } |
a1dfa0c6 | 288 | |
5869c6ff | 289 | let g = self.declare_global(sym, llty); |
a1dfa0c6 | 290 | |
5869c6ff XL |
291 | if !self.tcx.is_reachable_non_generic(def_id) { |
292 | unsafe { | |
293 | llvm::LLVMRustSetVisibility(g, llvm::Visibility::Hidden); | |
a1dfa0c6 XL |
294 | } |
295 | } | |
c1a9b12d | 296 | |
9e0c209e | 297 | g |
c1a9b12d | 298 | } else { |
c620b35d | 299 | check_and_apply_linkage(self, fn_attrs, llty, sym, def_id) |
5869c6ff | 300 | }; |
a1dfa0c6 | 301 | |
5869c6ff XL |
302 | // Thread-local statics in some other crate need to *always* be linked |
303 | // against in a thread-local fashion, so we need to be sure to apply the | |
304 | // thread-local attribute locally if it was present remotely. If we | |
305 | // don't do this then linker errors can be generated where the linker | |
306 | // complains that one object files has a thread local version of the | |
307 | // symbol and another one doesn't. | |
308 | if fn_attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) { | |
309 | llvm::set_thread_local_mode(g, self.tls_model); | |
310 | } | |
a1dfa0c6 | 311 | |
487cf647 FG |
312 | let dso_local = unsafe { self.should_assume_dso_local(g, true) }; |
313 | if dso_local { | |
314 | unsafe { | |
315 | llvm::LLVMRustSetDSOLocal(g, true); | |
316 | } | |
317 | } | |
318 | ||
5869c6ff | 319 | if !def_id.is_local() { |
dfeec247 | 320 | let needs_dll_storage_attr = self.use_dll_storage_attrs && !self.tcx.is_foreign_item(def_id) && |
487cf647 FG |
321 | // Local definitions can never be imported, so we must not apply |
322 | // the DLLImport annotation. | |
323 | !dso_local && | |
a1dfa0c6 XL |
324 | // ThinLTO can't handle this workaround in all cases, so we don't |
325 | // emit the attrs. Instead we make them unnecessary by disallowing | |
9fa01778 | 326 | // dynamic linking when linker plugin based LTO is enabled. |
487cf647 FG |
327 | !self.tcx.sess.opts.cg.linker_plugin_lto.enabled() && |
328 | self.tcx.sess.lto() != Lto::Thin; | |
a1dfa0c6 XL |
329 | |
330 | // If this assertion triggers, there's something wrong with commandline | |
331 | // argument validation. | |
dfeec247 XL |
332 | debug_assert!( |
333 | !(self.tcx.sess.opts.cg.linker_plugin_lto.enabled() | |
29967ef6 | 334 | && self.tcx.sess.target.is_like_windows |
dfeec247 XL |
335 | && self.tcx.sess.opts.cg.prefer_dynamic) |
336 | ); | |
a1dfa0c6 XL |
337 | |
338 | if needs_dll_storage_attr { | |
0731742a | 339 | // This item is external but not foreign, i.e., it originates from an external Rust |
a1dfa0c6 XL |
340 | // crate. Since we don't know whether this crate will be linked dynamically or |
341 | // statically in the final application, we always mark such symbols as 'dllimport'. | |
342 | // If final linkage happens to be static, we rely on compiler-emitted __imp_ stubs | |
343 | // to make things work. | |
344 | // | |
345 | // However, in some scenarios we defer emission of statics to downstream | |
346 | // crates, so there are cases where a static with an upstream DefId | |
347 | // is actually present in the current crate. We can find out via the | |
348 | // is_codegened_item query. | |
349 | if !self.tcx.is_codegened_item(def_id) { | |
350 | unsafe { | |
351 | llvm::LLVMSetDLLStorageClass(g, llvm::DLLStorageClass::DllImport); | |
352 | } | |
353 | } | |
354 | } | |
5869c6ff | 355 | } |
a1dfa0c6 | 356 | |
2b03887a FG |
357 | if self.use_dll_storage_attrs |
358 | && let Some(library) = self.tcx.native_library(def_id) | |
359 | && library.kind.is_dllimport() | |
360 | { | |
a1dfa0c6 XL |
361 | // For foreign (native) libs we know the exact storage type to use. |
362 | unsafe { | |
363 | llvm::LLVMSetDLLStorageClass(g, llvm::DLLStorageClass::DllImport); | |
1a4d82fc JJ |
364 | } |
365 | } | |
c1a9b12d | 366 | |
e8be2606 | 367 | self.instances.borrow_mut().insert(instance, g); |
a1dfa0c6 XL |
368 | g |
369 | } | |
5bcae85e | 370 | |
c620b35d | 371 | fn codegen_static_item(&self, def_id: DefId) { |
a1dfa0c6 | 372 | unsafe { |
c620b35d FG |
373 | assert!( |
374 | llvm::LLVMGetInitializer( | |
375 | self.instances.borrow().get(&Instance::mono(self.tcx, def_id)).unwrap() | |
376 | ) | |
377 | .is_none() | |
378 | ); | |
a1dfa0c6 XL |
379 | let attrs = self.tcx.codegen_fn_attrs(def_id); |
380 | ||
5e7ed085 | 381 | let Ok((v, alloc)) = codegen_static_initializer(self, def_id) else { |
a1dfa0c6 | 382 | // Error has already been reported |
5e7ed085 | 383 | return; |
a1dfa0c6 | 384 | }; |
5e7ed085 | 385 | let alloc = alloc.inner(); |
a1dfa0c6 | 386 | |
ed00b5ec | 387 | let val_llty = self.val_ty(v); |
a1dfa0c6 | 388 | |
c620b35d FG |
389 | let g = self.get_static_inner(def_id, val_llty); |
390 | let llty = self.val_ty(g); | |
391 | ||
a1dfa0c6 XL |
392 | let g = if val_llty == llty { |
393 | g | |
394 | } else { | |
395 | // If we created the global with the wrong type, | |
396 | // correct the type. | |
60c5eb7d XL |
397 | let name = llvm::get_value_name(g).to_vec(); |
398 | llvm::set_value_name(g, b""); | |
a1dfa0c6 XL |
399 | |
400 | let linkage = llvm::LLVMRustGetLinkage(g); | |
401 | let visibility = llvm::LLVMRustGetVisibility(g); | |
402 | ||
403 | let new_g = llvm::LLVMRustGetOrInsertGlobal( | |
dfeec247 XL |
404 | self.llmod, |
405 | name.as_ptr().cast(), | |
406 | name.len(), | |
407 | val_llty, | |
408 | ); | |
a1dfa0c6 XL |
409 | |
410 | llvm::LLVMRustSetLinkage(new_g, linkage); | |
411 | llvm::LLVMRustSetVisibility(new_g, visibility); | |
412 | ||
04454e1e FG |
413 | // The old global has had its name removed but is returned by |
414 | // get_static since it is in the instance cache. Provide an | |
415 | // alternative lookup that points to the new global so that | |
416 | // global_asm! can compute the correct mangled symbol name | |
417 | // for the global. | |
418 | self.renamed_statics.borrow_mut().insert(def_id, new_g); | |
419 | ||
a1dfa0c6 XL |
420 | // To avoid breaking any invariants, we leave around the old |
421 | // global for the moment; we'll replace all references to it | |
422 | // with the new global later. (See base::codegen_backend.) | |
423 | self.statics_to_rauw.borrow_mut().push((g, new_g)); | |
424 | new_g | |
425 | }; | |
c620b35d | 426 | set_global_alignment(self, g, alloc.align); |
a1dfa0c6 XL |
427 | llvm::LLVMSetInitializer(g, v); |
428 | ||
17df50a5 XL |
429 | if self.should_assume_dso_local(g, true) { |
430 | llvm::LLVMRustSetDSOLocal(g, true); | |
431 | } | |
432 | ||
c620b35d FG |
433 | // Forward the allocation's mutability (picked by the const interner) to LLVM. |
434 | if alloc.mutability.is_not() { | |
29967ef6 | 435 | llvm::LLVMSetGlobalConstant(g, llvm::True); |
a1dfa0c6 | 436 | } |
5bcae85e | 437 | |
5e7ed085 | 438 | debuginfo::build_global_var_di_node(self, def_id, g); |
a1dfa0c6 XL |
439 | |
440 | if attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) { | |
441 | llvm::set_thread_local_mode(g, self.tls_model); | |
442 | ||
443 | // Do not allow LLVM to change the alignment of a TLS on macOS. | |
444 | // | |
445 | // By default a global's alignment can be freely increased. | |
446 | // This allows LLVM to generate more performant instructions | |
0731742a | 447 | // e.g., using load-aligned into a SIMD register. |
a1dfa0c6 XL |
448 | // |
449 | // However, on macOS 10.10 or below, the dynamic linker does not | |
450 | // respect any alignment given on the TLS (radar 24221680). | |
451 | // This will violate the alignment assumption, and causing segfault at runtime. | |
452 | // | |
453 | // This bug is very easy to trigger. In `println!` and `panic!`, | |
454 | // the `LOCAL_STDOUT`/`LOCAL_STDERR` handles are stored in a TLS, | |
455 | // which the values would be `mem::replace`d on initialization. | |
456 | // The implementation of `mem::replace` will use SIMD | |
457 | // whenever the size is 32 bytes or higher. LLVM notices SIMD is used | |
458 | // and tries to align `LOCAL_STDOUT`/`LOCAL_STDERR` to a 32-byte boundary, | |
459 | // which macOS's dyld disregarded and causing crashes | |
460 | // (see issues #51794, #51758, #50867, #48866 and #44056). | |
461 | // | |
462 | // To workaround the bug, we trick LLVM into not increasing | |
463 | // the global's alignment by explicitly assigning a section to it | |
464 | // (equivalent to automatically generating a `#[link_section]` attribute). | |
465 | // See the comment in the `GlobalValue::canIncreaseAlignment()` function | |
466 | // of `lib/IR/Globals.cpp` for why this works. | |
467 | // | |
468 | // When the alignment is not increased, the optimized `mem::replace` | |
469 | // will use load-unaligned instructions instead, and thus avoiding the crash. | |
470 | // | |
471 | // We could remove this hack whenever we decide to drop macOS 10.10 support. | |
29967ef6 | 472 | if self.tcx.sess.target.is_like_osx { |
f2b60f7d | 473 | // The `inspect` method is okay here because we checked for provenance, and |
ba9703b0 XL |
474 | // because we are doing this access to inspect the final interpreter state |
475 | // (not as part of the interpreter execution). | |
476 | // | |
477 | // FIXME: This check requires that the (arbitrary) value of undefined bytes | |
478 | // happens to be zero. Instead, we should only check the value of defined bytes | |
479 | // and set all undefined bytes to zero if this allocation is headed for the | |
480 | // BSS. | |
487cf647 | 481 | let all_bytes_are_zero = alloc.provenance().ptrs().is_empty() |
ba9703b0 | 482 | && alloc |
3dfed10e | 483 | .inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()) |
e1599b0c | 484 | .iter() |
ba9703b0 XL |
485 | .all(|&byte| byte == 0); |
486 | ||
487 | let sect_name = if all_bytes_are_zero { | |
4b012472 | 488 | c"__DATA,__thread_bss" |
a1dfa0c6 | 489 | } else { |
4b012472 | 490 | c"__DATA,__thread_data" |
a1dfa0c6 XL |
491 | }; |
492 | llvm::LLVMSetSection(g, sect_name.as_ptr()); | |
493 | } | |
494 | } | |
495 | ||
a1dfa0c6 XL |
496 | // Wasm statics with custom link sections get special treatment as they |
497 | // go into custom sections of the wasm executable. | |
3c0e092e | 498 | if self.tcx.sess.target.is_like_wasm { |
a1dfa0c6 | 499 | if let Some(section) = attrs.link_section { |
353b0b11 | 500 | let section = llvm::LLVMMDStringInContext2( |
a1dfa0c6 | 501 | self.llcx, |
e74abb32 | 502 | section.as_str().as_ptr().cast(), |
353b0b11 | 503 | section.as_str().len(), |
a1dfa0c6 | 504 | ); |
487cf647 | 505 | assert!(alloc.provenance().ptrs().is_empty()); |
e1599b0c | 506 | |
f2b60f7d | 507 | // The `inspect` method is okay here because we checked for provenance, and |
e1599b0c XL |
508 | // because we are doing this access to inspect the final interpreter state (not |
509 | // as part of the interpreter execution). | |
dfeec247 | 510 | let bytes = |
3dfed10e | 511 | alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()); |
353b0b11 FG |
512 | let alloc = |
513 | llvm::LLVMMDStringInContext2(self.llcx, bytes.as_ptr().cast(), bytes.len()); | |
a1dfa0c6 | 514 | let data = [section, alloc]; |
353b0b11 FG |
515 | let meta = llvm::LLVMMDNodeInContext2(self.llcx, data.as_ptr(), data.len()); |
516 | let val = llvm::LLVMMetadataAsValue(self.llcx, meta); | |
a1dfa0c6 XL |
517 | llvm::LLVMAddNamedMetadataOperand( |
518 | self.llmod, | |
4b012472 | 519 | c"wasm.custom_sections".as_ptr().cast(), |
353b0b11 | 520 | val, |
a1dfa0c6 XL |
521 | ); |
522 | } | |
523 | } else { | |
c295e0f8 | 524 | base::set_link_section(g, attrs); |
8faf50e0 | 525 | } |
8faf50e0 | 526 | |
a1dfa0c6 | 527 | if attrs.flags.contains(CodegenFnAttrFlags::USED) { |
5099ac24 FG |
528 | // `USED` and `USED_LINKER` can't be used together. |
529 | assert!(!attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER)); | |
530 | ||
94222f64 XL |
531 | // The semantics of #[used] in Rust only require the symbol to make it into the |
532 | // object file. It is explicitly allowed for the linker to strip the symbol if it | |
9ffffee4 | 533 | // is dead, which means we are allowed to use `llvm.compiler.used` instead of |
064997fb FG |
534 | // `llvm.used` here. |
535 | // | |
94222f64 XL |
536 | // Additionally, https://reviews.llvm.org/D97448 in LLVM 13 started emitting unique |
537 | // sections with SHF_GNU_RETAIN flag for llvm.used symbols, which may trigger bugs | |
064997fb FG |
538 | // in the handling of `.init_array` (the static constructor list) in versions of |
539 | // the gold linker (prior to the one released with binutils 2.36). | |
540 | // | |
541 | // That said, we only ever emit these when compiling for ELF targets, unless | |
542 | // `#[used(compiler)]` is explicitly requested. This is to avoid similar breakage | |
543 | // on other targets, in particular MachO targets have *their* static constructor | |
9ffffee4 | 544 | // lists broken if `llvm.compiler.used` is emitted rather than `llvm.used`. However, |
2b03887a | 545 | // that check happens when assigning the `CodegenFnAttrFlags` in `rustc_hir_analysis`, |
064997fb | 546 | // so we don't need to take care of it here. |
94222f64 | 547 | self.add_compiler_used_global(g); |
a1dfa0c6 | 548 | } |
5099ac24 FG |
549 | if attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER) { |
550 | // `USED` and `USED_LINKER` can't be used together. | |
551 | assert!(!attrs.flags.contains(CodegenFnAttrFlags::USED)); | |
552 | ||
553 | self.add_used_global(g); | |
554 | } | |
cc61c64b | 555 | } |
1a4d82fc | 556 | } |
c620b35d FG |
557 | } |
558 | ||
559 | impl<'ll> StaticMethods for CodegenCx<'ll, '_> { | |
560 | fn static_addr_of(&self, cv: &'ll Value, align: Align, kind: Option<&str>) -> &'ll Value { | |
561 | if let Some(&gv) = self.const_globals.borrow().get(&cv) { | |
562 | unsafe { | |
563 | // Upgrade the alignment in cases where the same constant is used with different | |
564 | // alignment requirements | |
565 | let llalign = align.bytes() as u32; | |
566 | if llalign > llvm::LLVMGetAlignment(gv) { | |
567 | llvm::LLVMSetAlignment(gv, llalign); | |
568 | } | |
569 | } | |
570 | return gv; | |
571 | } | |
572 | let gv = self.static_addr_of_mut(cv, align, kind); | |
573 | unsafe { | |
574 | llvm::LLVMSetGlobalConstant(gv, True); | |
575 | } | |
576 | self.const_globals.borrow_mut().insert(cv, gv); | |
577 | gv | |
578 | } | |
579 | ||
580 | fn codegen_static(&self, def_id: DefId) { | |
581 | self.codegen_static_item(def_id) | |
582 | } | |
3dfed10e | 583 | |
add651ee | 584 | /// Add a global value to a list to be stored in the `llvm.used` variable, an array of ptr. |
3dfed10e | 585 | fn add_used_global(&self, global: &'ll Value) { |
add651ee | 586 | self.used_statics.borrow_mut().push(global); |
3dfed10e | 587 | } |
94222f64 XL |
588 | |
589 | /// Add a global value to a list to be stored in the `llvm.compiler.used` variable, | |
add651ee | 590 | /// an array of ptr. |
94222f64 | 591 | fn add_compiler_used_global(&self, global: &'ll Value) { |
add651ee | 592 | self.compiler_used_statics.borrow_mut().push(global); |
94222f64 | 593 | } |
1a4d82fc | 594 | } |