1 //! Handling of `static`s, `const`s and promoted allocations
3 use rustc_span
::DUMMY_SP
;
5 use rustc_data_structures
::fx
::FxHashSet
;
6 use rustc_errors
::ErrorReported
;
7 use rustc_middle
::middle
::codegen_fn_attrs
::CodegenFnAttrFlags
;
8 use rustc_middle
::mir
::interpret
::{
9 read_target_uint
, AllocId
, Allocation
, ConstValue
, ErrorHandled
, GlobalAlloc
, Pointer
, Scalar
,
11 use rustc_middle
::ty
::{Const, ConstKind}
;
13 use cranelift_codegen
::ir
::GlobalValueData
;
14 use cranelift_module
::*;
16 use crate::prelude
::*;
19 pub(crate) struct ConstantCx
{
21 done
: FxHashSet
<DataId
>,
24 #[derive(Copy, Clone, Debug)]
31 pub(crate) fn finalize(mut self, tcx
: TyCtxt
<'_
>, module
: &mut impl Module
) {
32 //println!("todo {:?}", self.todo);
33 define_all_allocs(tcx
, module
, &mut self);
34 //println!("done {:?}", self.done);
39 pub(crate) fn check_constants(fx
: &mut FunctionCx
<'_
, '_
, impl Module
>) {
40 for constant
in &fx
.mir
.required_consts
{
41 let const_
= fx
.monomorphize(&constant
.literal
);
43 ConstKind
::Value(_
) => {}
44 ConstKind
::Unevaluated(def
, ref substs
, promoted
) => {
47 .const_eval_resolve(ParamEnv
::reveal_all(), def
, substs
, promoted
, None
)
50 ErrorHandled
::Reported(ErrorReported
) | ErrorHandled
::Linted
=> {
53 .span_err(constant
.span
, "erroneous constant encountered");
55 ErrorHandled
::TooGeneric
=> {
58 "codgen encountered polymorphic constant: {:?}",
67 | ConstKind
::Bound(_
, _
)
68 | ConstKind
::Placeholder(_
)
69 | ConstKind
::Error(_
) => unreachable
!("{:?}", const_
),
74 pub(crate) fn codegen_static(constants_cx
: &mut ConstantCx
, def_id
: DefId
) {
75 constants_cx
.todo
.push(TodoItem
::Static(def_id
));
78 pub(crate) fn codegen_tls_ref
<'tcx
>(
79 fx
: &mut FunctionCx
<'_
, 'tcx
, impl Module
>,
81 layout
: TyAndLayout
<'tcx
>,
83 let data_id
= data_id_for_static(fx
.tcx
, &mut fx
.cx
.module
, def_id
, false);
84 let local_data_id
= fx
.cx
.module
.declare_data_in_func(data_id
, &mut fx
.bcx
.func
);
85 #[cfg(debug_assertions)]
86 fx
.add_comment(local_data_id
, format
!("tls {:?}", def_id
));
87 let tls_ptr
= fx
.bcx
.ins().tls_value(fx
.pointer_type
, local_data_id
);
88 CValue
::by_val(tls_ptr
, layout
)
91 fn codegen_static_ref
<'tcx
>(
92 fx
: &mut FunctionCx
<'_
, 'tcx
, impl Module
>,
94 layout
: TyAndLayout
<'tcx
>,
96 let data_id
= data_id_for_static(fx
.tcx
, &mut fx
.cx
.module
, def_id
, false);
97 let local_data_id
= fx
.cx
.module
.declare_data_in_func(data_id
, &mut fx
.bcx
.func
);
98 #[cfg(debug_assertions)]
99 fx
.add_comment(local_data_id
, format
!("{:?}", def_id
));
100 let global_ptr
= fx
.bcx
.ins().global_value(fx
.pointer_type
, local_data_id
);
101 assert
!(!layout
.is_unsized(), "unsized statics aren't supported");
103 matches
!(fx
.bcx
.func
.global_values
[local_data_id
], GlobalValueData
::Symbol { tls: false, ..}
),
104 "tls static referenced without Rvalue::ThreadLocalRef"
106 CPlace
::for_ptr(crate::pointer
::Pointer
::new(global_ptr
), layout
)
109 pub(crate) fn codegen_constant
<'tcx
>(
110 fx
: &mut FunctionCx
<'_
, 'tcx
, impl Module
>,
111 constant
: &Constant
<'tcx
>,
113 let const_
= fx
.monomorphize(&constant
.literal
);
114 let const_val
= match const_
.val
{
115 ConstKind
::Value(const_val
) => const_val
,
116 ConstKind
::Unevaluated(def
, ref substs
, promoted
) if fx
.tcx
.is_static(def
.did
) => {
117 assert
!(substs
.is_empty());
118 assert
!(promoted
.is_none());
120 return codegen_static_ref(
123 fx
.layout_of(fx
.monomorphize(&constant
.literal
.ty
)),
127 ConstKind
::Unevaluated(def
, ref substs
, promoted
) => {
130 .const_eval_resolve(ParamEnv
::reveal_all(), def
, substs
, promoted
, None
)
132 Ok(const_val
) => const_val
,
134 if promoted
.is_none() {
137 .span_err(constant
.span
, "erroneous constant encountered");
139 return crate::trap
::trap_unreachable_ret_value(
141 fx
.layout_of(const_
.ty
),
142 "erroneous constant encountered",
148 | ConstKind
::Infer(_
)
149 | ConstKind
::Bound(_
, _
)
150 | ConstKind
::Placeholder(_
)
151 | ConstKind
::Error(_
) => unreachable
!("{:?}", const_
),
154 codegen_const_value(fx
, const_val
, const_
.ty
)
157 pub(crate) fn codegen_const_value
<'tcx
>(
158 fx
: &mut FunctionCx
<'_
, 'tcx
, impl Module
>,
159 const_val
: ConstValue
<'tcx
>,
162 let layout
= fx
.layout_of(ty
);
163 assert
!(!layout
.is_unsized(), "sized const value");
166 return CValue
::by_ref(
167 crate::Pointer
::dangling(layout
.align
.pref
),
173 ConstValue
::Scalar(x
) => {
174 if fx
.clif_type(layout
.ty
).is_none() {
175 let (size
, align
) = (layout
.size
, layout
.align
.pref
);
176 let mut alloc
= Allocation
::from_bytes(
178 .take(size
.bytes_usize())
179 .collect
::<Vec
<u8>>(),
182 let ptr
= Pointer
::new(AllocId(!0), Size
::ZERO
); // The alloc id is never used
183 alloc
.write_scalar(fx
, ptr
, x
.into(), size
).unwrap();
184 let alloc
= fx
.tcx
.intern_const_alloc(alloc
);
185 return CValue
::by_ref(pointer_for_allocation(fx
, alloc
), layout
);
189 Scalar
::Int(int
) => {
190 CValue
::const_val(fx
, layout
, int
)
192 Scalar
::Ptr(ptr
) => {
193 let alloc_kind
= fx
.tcx
.get_global_alloc(ptr
.alloc_id
);
194 let base_addr
= match alloc_kind
{
195 Some(GlobalAlloc
::Memory(alloc
)) => {
196 fx
.cx
.constants_cx
.todo
.push(TodoItem
::Alloc(ptr
.alloc_id
));
197 let data_id
= data_id_for_alloc_id(
203 fx
.cx
.module
.declare_data_in_func(data_id
, &mut fx
.bcx
.func
);
204 #[cfg(debug_assertions)]
205 fx
.add_comment(local_data_id
, format
!("{:?}", ptr
.alloc_id
));
206 fx
.bcx
.ins().global_value(fx
.pointer_type
, local_data_id
)
208 Some(GlobalAlloc
::Function(instance
)) => {
210 crate::abi
::import_function(fx
.tcx
, &mut fx
.cx
.module
, instance
);
212 fx
.cx
.module
.declare_func_in_func(func_id
, &mut fx
.bcx
.func
);
213 fx
.bcx
.ins().func_addr(fx
.pointer_type
, local_func_id
)
215 Some(GlobalAlloc
::Static(def_id
)) => {
216 assert
!(fx
.tcx
.is_static(def_id
));
218 data_id_for_static(fx
.tcx
, &mut fx
.cx
.module
, def_id
, false);
220 fx
.cx
.module
.declare_data_in_func(data_id
, &mut fx
.bcx
.func
);
221 #[cfg(debug_assertions)]
222 fx
.add_comment(local_data_id
, format
!("{:?}", def_id
));
223 fx
.bcx
.ins().global_value(fx
.pointer_type
, local_data_id
)
225 None
=> bug
!("missing allocation {:?}", ptr
.alloc_id
),
227 let val
= if ptr
.offset
.bytes() != 0 {
230 .iadd_imm(base_addr
, i64::try_from(ptr
.offset
.bytes()).unwrap())
234 CValue
::by_val(val
, layout
)
238 ConstValue
::ByRef { alloc, offset }
=> CValue
::by_ref(
239 pointer_for_allocation(fx
, alloc
)
240 .offset_i64(fx
, i64::try_from(offset
.bytes()).unwrap()),
243 ConstValue
::Slice { data, start, end }
=> {
244 let ptr
= pointer_for_allocation(fx
, data
)
245 .offset_i64(fx
, i64::try_from(start
).unwrap())
247 let len
= fx
.bcx
.ins().iconst(
249 i64::try_from(end
.checked_sub(start
).unwrap()).unwrap(),
251 CValue
::by_val_pair(ptr
, len
, layout
)
256 fn pointer_for_allocation
<'tcx
>(
257 fx
: &mut FunctionCx
<'_
, 'tcx
, impl Module
>,
258 alloc
: &'tcx Allocation
,
259 ) -> crate::pointer
::Pointer
{
260 let alloc_id
= fx
.tcx
.create_memory_alloc(alloc
);
261 fx
.cx
.constants_cx
.todo
.push(TodoItem
::Alloc(alloc_id
));
262 let data_id
= data_id_for_alloc_id(&mut fx
.cx
.module
, alloc_id
, alloc
.mutability
);
264 let local_data_id
= fx
.cx
.module
.declare_data_in_func(data_id
, &mut fx
.bcx
.func
);
265 #[cfg(debug_assertions)]
266 fx
.add_comment(local_data_id
, format
!("{:?}", alloc_id
));
267 let global_ptr
= fx
.bcx
.ins().global_value(fx
.pointer_type
, local_data_id
);
268 crate::pointer
::Pointer
::new(global_ptr
)
271 fn data_id_for_alloc_id(
272 module
: &mut impl Module
,
274 mutability
: rustc_hir
::Mutability
,
278 &format
!(".L__alloc_{:x}", alloc_id
.0),
280 mutability
== rustc_hir
::Mutability
::Mut
,
286 fn data_id_for_static(
288 module
: &mut impl Module
,
292 let rlinkage
= tcx
.codegen_fn_attrs(def_id
).linkage
;
293 let linkage
= if definition
{
294 crate::linkage
::get_static_linkage(tcx
, def_id
)
295 } else if rlinkage
== Some(rustc_middle
::mir
::mono
::Linkage
::ExternalWeak
)
296 || rlinkage
== Some(rustc_middle
::mir
::mono
::Linkage
::WeakAny
)
303 let instance
= Instance
::mono(tcx
, def_id
).polymorphize(tcx
);
304 let symbol_name
= tcx
.symbol_name(instance
).name
;
305 let ty
= instance
.ty(tcx
, ParamEnv
::reveal_all());
306 let is_mutable
= if tcx
.is_mutable_static(def_id
) {
309 !ty
.is_freeze(tcx
.at(DUMMY_SP
), ParamEnv
::reveal_all())
312 .layout_of(ParamEnv
::reveal_all().and(ty
))
318 let attrs
= tcx
.codegen_fn_attrs(def_id
);
325 attrs
.flags
.contains(CodegenFnAttrFlags
::THREAD_LOCAL
),
329 if rlinkage
.is_some() {
330 // Comment copied from https://github.com/rust-lang/rust/blob/45060c2a66dfd667f88bd8b94261b28a58d85bd5/src/librustc_codegen_llvm/consts.rs#L141
331 // Declare an internal global `extern_with_linkage_foo` which
332 // is initialized with the address of `foo`. If `foo` is
333 // discarded during linking (for example, if `foo` has weak
334 // linkage and there are no definitions), then
335 // `extern_with_linkage_foo` will instead be initialized to
338 let ref_name
= format
!("_rust_extern_with_linkage_{}", symbol_name
);
339 let ref_data_id
= module
340 .declare_data(&ref_name
, Linkage
::Local
, false, false)
342 let mut data_ctx
= DataContext
::new();
343 data_ctx
.set_align(align
);
344 let data
= module
.declare_data_in_data(data_id
, &mut data_ctx
);
347 .take(pointer_ty(tcx
).bytes() as usize)
350 data_ctx
.write_data_addr(0, data
, 0);
351 match module
.define_data(ref_data_id
, &data_ctx
) {
352 // Every time the static is referenced there will be another definition of this global,
353 // so duplicate definitions are expected and allowed.
354 Err(ModuleError
::DuplicateDefinition(_
)) => {}
363 fn define_all_allocs(tcx
: TyCtxt
<'_
>, module
: &mut impl Module
, cx
: &mut ConstantCx
) {
364 while let Some(todo_item
) = cx
.todo
.pop() {
365 let (data_id
, alloc
, section_name
) = match todo_item
{
366 TodoItem
::Alloc(alloc_id
) => {
367 //println!("alloc_id {}", alloc_id);
368 let alloc
= match tcx
.get_global_alloc(alloc_id
).unwrap() {
369 GlobalAlloc
::Memory(alloc
) => alloc
,
370 GlobalAlloc
::Function(_
) | GlobalAlloc
::Static(_
) => unreachable
!(),
372 let data_id
= data_id_for_alloc_id(module
, alloc_id
, alloc
.mutability
);
373 (data_id
, alloc
, None
)
375 TodoItem
::Static(def_id
) => {
376 //println!("static {:?}", def_id);
378 let section_name
= tcx
379 .codegen_fn_attrs(def_id
)
381 .map(|s
| s
.as_str());
383 let alloc
= tcx
.eval_static_initializer(def_id
).unwrap();
385 let data_id
= data_id_for_static(tcx
, module
, def_id
, true);
386 (data_id
, alloc
, section_name
)
390 //("data_id {}", data_id);
391 if cx
.done
.contains(&data_id
) {
395 let mut data_ctx
= DataContext
::new();
396 data_ctx
.set_align(alloc
.align
.bytes());
398 if let Some(section_name
) = section_name
{
399 // FIXME set correct segment for Mach-O files
400 data_ctx
.set_segment_section("", &*section_name
);
404 .inspect_with_uninit_and_ptr_outside_interpreter(0..alloc
.len())
406 data_ctx
.define(bytes
.into_boxed_slice());
408 for &(offset
, (_tag
, reloc
)) in alloc
.relocations().iter() {
410 let endianness
= tcx
.data_layout
.endian
;
411 let offset
= offset
.bytes() as usize;
412 let ptr_size
= tcx
.data_layout
.pointer_size
;
413 let bytes
= &alloc
.inspect_with_uninit_and_ptr_outside_interpreter(
414 offset
..offset
+ ptr_size
.bytes() as usize,
416 read_target_uint(endianness
, bytes
).unwrap()
419 let reloc_target_alloc
= tcx
.get_global_alloc(reloc
).unwrap();
420 let data_id
= match reloc_target_alloc
{
421 GlobalAlloc
::Function(instance
) => {
422 assert_eq
!(addend
, 0);
423 let func_id
= crate::abi
::import_function(tcx
, module
, instance
);
424 let local_func_id
= module
.declare_func_in_data(func_id
, &mut data_ctx
);
425 data_ctx
.write_function_addr(offset
.bytes() as u32, local_func_id
);
428 GlobalAlloc
::Memory(target_alloc
) => {
429 cx
.todo
.push(TodoItem
::Alloc(reloc
));
430 data_id_for_alloc_id(module
, reloc
, target_alloc
.mutability
)
432 GlobalAlloc
::Static(def_id
) => {
434 .codegen_fn_attrs(def_id
)
436 .contains(CodegenFnAttrFlags
::THREAD_LOCAL
)
438 tcx
.sess
.fatal(&format
!(
439 "Allocation {:?} contains reference to TLS value {:?}",
444 // Don't push a `TodoItem::Static` here, as it will cause statics used by
445 // multiple crates to be duplicated between them. It isn't necessary anyway,
446 // as it will get pushed by `codegen_static` when necessary.
447 data_id_for_static(tcx
, module
, def_id
, false)
451 let global_value
= module
.declare_data_in_data(data_id
, &mut data_ctx
);
452 data_ctx
.write_data_addr(offset
.bytes() as u32, global_value
, addend
as i64);
455 module
.define_data(data_id
, &data_ctx
).unwrap();
456 cx
.done
.insert(data_id
);
459 assert
!(cx
.todo
.is_empty(), "{:?}", cx
.todo
);
462 pub(crate) fn mir_operand_get_const_val
<'tcx
>(
463 fx
: &FunctionCx
<'_
, 'tcx
, impl Module
>,
464 operand
: &Operand
<'tcx
>,
465 ) -> Option
<&'tcx Const
<'tcx
>> {
467 Operand
::Copy(_
) | Operand
::Move(_
) => None
,
468 Operand
::Constant(const_
) => Some(
469 fx
.monomorphize(&const_
.literal
)
470 .eval(fx
.tcx
, ParamEnv
::reveal_all()),