1 //! Handling of `static`s, `const`s and promoted allocations
3 use rustc_span
::DUMMY_SP
;
5 use rustc_data_structures
::fx
::FxHashSet
;
6 use rustc_errors
::ErrorReported
;
7 use rustc_middle
::middle
::codegen_fn_attrs
::CodegenFnAttrFlags
;
8 use rustc_middle
::mir
::interpret
::{
9 read_target_uint
, AllocId
, Allocation
, ConstValue
, ErrorHandled
, GlobalAlloc
, Pointer
, Scalar
,
11 use rustc_middle
::ty
::ConstKind
;
13 use cranelift_codegen
::ir
::GlobalValueData
;
14 use cranelift_module
::*;
16 use crate::prelude
::*;
19 pub(crate) struct ConstantCx
{
21 done
: FxHashSet
<DataId
>,
24 #[derive(Copy, Clone, Debug)]
31 pub(crate) fn finalize(mut self, tcx
: TyCtxt
<'_
>, module
: &mut dyn Module
) {
32 //println!("todo {:?}", self.todo);
33 define_all_allocs(tcx
, module
, &mut self);
34 //println!("done {:?}", self.done);
39 pub(crate) fn check_constants(fx
: &mut FunctionCx
<'_
, '_
, '_
>) -> bool
{
40 let mut all_constants_ok
= true;
41 for constant
in &fx
.mir
.required_consts
{
42 let const_
= match fx
.monomorphize(constant
.literal
) {
43 ConstantKind
::Ty(ct
) => ct
,
44 ConstantKind
::Val(..) => continue,
47 ConstKind
::Value(_
) => {}
48 ConstKind
::Unevaluated(def
, ref substs
, promoted
) => {
50 fx
.tcx
.const_eval_resolve(ParamEnv
::reveal_all(), def
, substs
, promoted
, None
)
52 all_constants_ok
= false;
54 ErrorHandled
::Reported(ErrorReported
) | ErrorHandled
::Linted
=> {
55 fx
.tcx
.sess
.span_err(constant
.span
, "erroneous constant encountered");
57 ErrorHandled
::TooGeneric
=> {
60 "codgen encountered polymorphic constant: {:?}",
69 | ConstKind
::Bound(_
, _
)
70 | ConstKind
::Placeholder(_
)
71 | ConstKind
::Error(_
) => unreachable
!("{:?}", const_
),
77 pub(crate) fn codegen_static(constants_cx
: &mut ConstantCx
, def_id
: DefId
) {
78 constants_cx
.todo
.push(TodoItem
::Static(def_id
));
81 pub(crate) fn codegen_tls_ref
<'tcx
>(
82 fx
: &mut FunctionCx
<'_
, '_
, 'tcx
>,
84 layout
: TyAndLayout
<'tcx
>,
86 let data_id
= data_id_for_static(fx
.tcx
, fx
.cx
.module
, def_id
, false);
87 let local_data_id
= fx
.cx
.module
.declare_data_in_func(data_id
, &mut fx
.bcx
.func
);
88 #[cfg(debug_assertions)]
89 fx
.add_comment(local_data_id
, format
!("tls {:?}", def_id
));
90 let tls_ptr
= fx
.bcx
.ins().tls_value(fx
.pointer_type
, local_data_id
);
91 CValue
::by_val(tls_ptr
, layout
)
94 fn codegen_static_ref
<'tcx
>(
95 fx
: &mut FunctionCx
<'_
, '_
, 'tcx
>,
97 layout
: TyAndLayout
<'tcx
>,
99 let data_id
= data_id_for_static(fx
.tcx
, fx
.cx
.module
, def_id
, false);
100 let local_data_id
= fx
.cx
.module
.declare_data_in_func(data_id
, &mut fx
.bcx
.func
);
101 #[cfg(debug_assertions)]
102 fx
.add_comment(local_data_id
, format
!("{:?}", def_id
));
103 let global_ptr
= fx
.bcx
.ins().global_value(fx
.pointer_type
, local_data_id
);
104 assert
!(!layout
.is_unsized(), "unsized statics aren't supported");
107 fx
.bcx
.func
.global_values
[local_data_id
],
108 GlobalValueData
::Symbol { tls: false, .. }
110 "tls static referenced without Rvalue::ThreadLocalRef"
112 CPlace
::for_ptr(crate::pointer
::Pointer
::new(global_ptr
), layout
)
115 pub(crate) fn codegen_constant
<'tcx
>(
116 fx
: &mut FunctionCx
<'_
, '_
, 'tcx
>,
117 constant
: &Constant
<'tcx
>,
119 let const_
= match fx
.monomorphize(constant
.literal
) {
120 ConstantKind
::Ty(ct
) => ct
,
121 ConstantKind
::Val(val
, ty
) => return codegen_const_value(fx
, val
, ty
),
123 let const_val
= match const_
.val
{
124 ConstKind
::Value(const_val
) => const_val
,
125 ConstKind
::Unevaluated(def
, ref substs
, promoted
) if fx
.tcx
.is_static(def
.did
) => {
126 assert
!(substs
.is_empty());
127 assert
!(promoted
.is_none());
129 return codegen_static_ref(fx
, def
.did
, fx
.layout_of(const_
.ty
)).to_cvalue(fx
);
131 ConstKind
::Unevaluated(def
, ref substs
, promoted
) => {
132 match fx
.tcx
.const_eval_resolve(ParamEnv
::reveal_all(), def
, substs
, promoted
, None
) {
133 Ok(const_val
) => const_val
,
135 span_bug
!(constant
.span
, "erroneous constant not captured by required_consts");
140 | ConstKind
::Infer(_
)
141 | ConstKind
::Bound(_
, _
)
142 | ConstKind
::Placeholder(_
)
143 | ConstKind
::Error(_
) => unreachable
!("{:?}", const_
),
146 codegen_const_value(fx
, const_val
, const_
.ty
)
149 pub(crate) fn codegen_const_value
<'tcx
>(
150 fx
: &mut FunctionCx
<'_
, '_
, 'tcx
>,
151 const_val
: ConstValue
<'tcx
>,
154 let layout
= fx
.layout_of(ty
);
155 assert
!(!layout
.is_unsized(), "sized const value");
158 return CValue
::by_ref(crate::Pointer
::dangling(layout
.align
.pref
), layout
);
162 ConstValue
::Scalar(x
) => {
163 if fx
.clif_type(layout
.ty
).is_none() {
164 let (size
, align
) = (layout
.size
, layout
.align
.pref
);
165 let mut alloc
= Allocation
::from_bytes(
166 std
::iter
::repeat(0).take(size
.bytes_usize()).collect
::<Vec
<u8>>(),
169 let ptr
= Pointer
::new(AllocId(!0), Size
::ZERO
); // The alloc id is never used
170 alloc
.write_scalar(fx
, ptr
, x
.into(), size
).unwrap();
171 let alloc
= fx
.tcx
.intern_const_alloc(alloc
);
172 return CValue
::by_ref(pointer_for_allocation(fx
, alloc
), layout
);
176 Scalar
::Int(int
) => CValue
::const_val(fx
, layout
, int
),
177 Scalar
::Ptr(ptr
) => {
178 let alloc_kind
= fx
.tcx
.get_global_alloc(ptr
.alloc_id
);
179 let base_addr
= match alloc_kind
{
180 Some(GlobalAlloc
::Memory(alloc
)) => {
181 fx
.cx
.constants_cx
.todo
.push(TodoItem
::Alloc(ptr
.alloc_id
));
183 data_id_for_alloc_id(fx
.cx
.module
, ptr
.alloc_id
, alloc
.mutability
);
185 fx
.cx
.module
.declare_data_in_func(data_id
, &mut fx
.bcx
.func
);
186 #[cfg(debug_assertions)]
187 fx
.add_comment(local_data_id
, format
!("{:?}", ptr
.alloc_id
));
188 fx
.bcx
.ins().global_value(fx
.pointer_type
, local_data_id
)
190 Some(GlobalAlloc
::Function(instance
)) => {
192 crate::abi
::import_function(fx
.tcx
, fx
.cx
.module
, instance
);
194 fx
.cx
.module
.declare_func_in_func(func_id
, &mut fx
.bcx
.func
);
195 fx
.bcx
.ins().func_addr(fx
.pointer_type
, local_func_id
)
197 Some(GlobalAlloc
::Static(def_id
)) => {
198 assert
!(fx
.tcx
.is_static(def_id
));
199 let data_id
= data_id_for_static(fx
.tcx
, fx
.cx
.module
, def_id
, false);
201 fx
.cx
.module
.declare_data_in_func(data_id
, &mut fx
.bcx
.func
);
202 #[cfg(debug_assertions)]
203 fx
.add_comment(local_data_id
, format
!("{:?}", def_id
));
204 fx
.bcx
.ins().global_value(fx
.pointer_type
, local_data_id
)
206 None
=> bug
!("missing allocation {:?}", ptr
.alloc_id
),
208 let val
= if ptr
.offset
.bytes() != 0 {
209 fx
.bcx
.ins().iadd_imm(base_addr
, i64::try_from(ptr
.offset
.bytes()).unwrap())
213 CValue
::by_val(val
, layout
)
217 ConstValue
::ByRef { alloc, offset }
=> CValue
::by_ref(
218 pointer_for_allocation(fx
, alloc
)
219 .offset_i64(fx
, i64::try_from(offset
.bytes()).unwrap()),
222 ConstValue
::Slice { data, start, end }
=> {
223 let ptr
= pointer_for_allocation(fx
, data
)
224 .offset_i64(fx
, i64::try_from(start
).unwrap())
229 .iconst(fx
.pointer_type
, i64::try_from(end
.checked_sub(start
).unwrap()).unwrap());
230 CValue
::by_val_pair(ptr
, len
, layout
)
235 fn pointer_for_allocation
<'tcx
>(
236 fx
: &mut FunctionCx
<'_
, '_
, 'tcx
>,
237 alloc
: &'tcx Allocation
,
238 ) -> crate::pointer
::Pointer
{
239 let alloc_id
= fx
.tcx
.create_memory_alloc(alloc
);
240 fx
.cx
.constants_cx
.todo
.push(TodoItem
::Alloc(alloc_id
));
241 let data_id
= data_id_for_alloc_id(fx
.cx
.module
, alloc_id
, alloc
.mutability
);
243 let local_data_id
= fx
.cx
.module
.declare_data_in_func(data_id
, &mut fx
.bcx
.func
);
244 #[cfg(debug_assertions)]
245 fx
.add_comment(local_data_id
, format
!("{:?}", alloc_id
));
246 let global_ptr
= fx
.bcx
.ins().global_value(fx
.pointer_type
, local_data_id
);
247 crate::pointer
::Pointer
::new(global_ptr
)
250 fn data_id_for_alloc_id(
251 module
: &mut dyn Module
,
253 mutability
: rustc_hir
::Mutability
,
257 &format
!(".L__alloc_{:x}", alloc_id
.0),
259 mutability
== rustc_hir
::Mutability
::Mut
,
265 fn data_id_for_static(
267 module
: &mut dyn Module
,
271 let rlinkage
= tcx
.codegen_fn_attrs(def_id
).linkage
;
272 let linkage
= if definition
{
273 crate::linkage
::get_static_linkage(tcx
, def_id
)
274 } else if rlinkage
== Some(rustc_middle
::mir
::mono
::Linkage
::ExternalWeak
)
275 || rlinkage
== Some(rustc_middle
::mir
::mono
::Linkage
::WeakAny
)
282 let instance
= Instance
::mono(tcx
, def_id
).polymorphize(tcx
);
283 let symbol_name
= tcx
.symbol_name(instance
).name
;
284 let ty
= instance
.ty(tcx
, ParamEnv
::reveal_all());
285 let is_mutable
= if tcx
.is_mutable_static(def_id
) {
288 !ty
.is_freeze(tcx
.at(DUMMY_SP
), ParamEnv
::reveal_all())
290 let align
= tcx
.layout_of(ParamEnv
::reveal_all().and(ty
)).unwrap().align
.pref
.bytes();
292 let attrs
= tcx
.codegen_fn_attrs(def_id
);
299 attrs
.flags
.contains(CodegenFnAttrFlags
::THREAD_LOCAL
),
303 if rlinkage
.is_some() {
304 // Comment copied from https://github.com/rust-lang/rust/blob/45060c2a66dfd667f88bd8b94261b28a58d85bd5/src/librustc_codegen_llvm/consts.rs#L141
305 // Declare an internal global `extern_with_linkage_foo` which
306 // is initialized with the address of `foo`. If `foo` is
307 // discarded during linking (for example, if `foo` has weak
308 // linkage and there are no definitions), then
309 // `extern_with_linkage_foo` will instead be initialized to
312 let ref_name
= format
!("_rust_extern_with_linkage_{}", symbol_name
);
313 let ref_data_id
= module
.declare_data(&ref_name
, Linkage
::Local
, false, false).unwrap();
314 let mut data_ctx
= DataContext
::new();
315 data_ctx
.set_align(align
);
316 let data
= module
.declare_data_in_data(data_id
, &mut data_ctx
);
317 data_ctx
.define(std
::iter
::repeat(0).take(pointer_ty(tcx
).bytes() as usize).collect());
318 data_ctx
.write_data_addr(0, data
, 0);
319 match module
.define_data(ref_data_id
, &data_ctx
) {
320 // Every time the static is referenced there will be another definition of this global,
321 // so duplicate definitions are expected and allowed.
322 Err(ModuleError
::DuplicateDefinition(_
)) => {}
331 fn define_all_allocs(tcx
: TyCtxt
<'_
>, module
: &mut dyn Module
, cx
: &mut ConstantCx
) {
332 while let Some(todo_item
) = cx
.todo
.pop() {
333 let (data_id
, alloc
, section_name
) = match todo_item
{
334 TodoItem
::Alloc(alloc_id
) => {
335 //println!("alloc_id {}", alloc_id);
336 let alloc
= match tcx
.get_global_alloc(alloc_id
).unwrap() {
337 GlobalAlloc
::Memory(alloc
) => alloc
,
338 GlobalAlloc
::Function(_
) | GlobalAlloc
::Static(_
) => unreachable
!(),
340 let data_id
= data_id_for_alloc_id(module
, alloc_id
, alloc
.mutability
);
341 (data_id
, alloc
, None
)
343 TodoItem
::Static(def_id
) => {
344 //println!("static {:?}", def_id);
346 let section_name
= tcx
.codegen_fn_attrs(def_id
).link_section
.map(|s
| s
.as_str());
348 let alloc
= tcx
.eval_static_initializer(def_id
).unwrap();
350 let data_id
= data_id_for_static(tcx
, module
, def_id
, true);
351 (data_id
, alloc
, section_name
)
355 //("data_id {}", data_id);
356 if cx
.done
.contains(&data_id
) {
360 let mut data_ctx
= DataContext
::new();
361 data_ctx
.set_align(alloc
.align
.bytes());
363 if let Some(section_name
) = section_name
{
364 // FIXME set correct segment for Mach-O files
365 data_ctx
.set_segment_section("", &*section_name
);
368 let bytes
= alloc
.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc
.len()).to_vec();
369 data_ctx
.define(bytes
.into_boxed_slice());
371 for &(offset
, (_tag
, reloc
)) in alloc
.relocations().iter() {
373 let endianness
= tcx
.data_layout
.endian
;
374 let offset
= offset
.bytes() as usize;
375 let ptr_size
= tcx
.data_layout
.pointer_size
;
376 let bytes
= &alloc
.inspect_with_uninit_and_ptr_outside_interpreter(
377 offset
..offset
+ ptr_size
.bytes() as usize,
379 read_target_uint(endianness
, bytes
).unwrap()
382 let reloc_target_alloc
= tcx
.get_global_alloc(reloc
).unwrap();
383 let data_id
= match reloc_target_alloc
{
384 GlobalAlloc
::Function(instance
) => {
385 assert_eq
!(addend
, 0);
386 let func_id
= crate::abi
::import_function(tcx
, module
, instance
);
387 let local_func_id
= module
.declare_func_in_data(func_id
, &mut data_ctx
);
388 data_ctx
.write_function_addr(offset
.bytes() as u32, local_func_id
);
391 GlobalAlloc
::Memory(target_alloc
) => {
392 cx
.todo
.push(TodoItem
::Alloc(reloc
));
393 data_id_for_alloc_id(module
, reloc
, target_alloc
.mutability
)
395 GlobalAlloc
::Static(def_id
) => {
396 if tcx
.codegen_fn_attrs(def_id
).flags
.contains(CodegenFnAttrFlags
::THREAD_LOCAL
)
398 tcx
.sess
.fatal(&format
!(
399 "Allocation {:?} contains reference to TLS value {:?}",
404 // Don't push a `TodoItem::Static` here, as it will cause statics used by
405 // multiple crates to be duplicated between them. It isn't necessary anyway,
406 // as it will get pushed by `codegen_static` when necessary.
407 data_id_for_static(tcx
, module
, def_id
, false)
411 let global_value
= module
.declare_data_in_data(data_id
, &mut data_ctx
);
412 data_ctx
.write_data_addr(offset
.bytes() as u32, global_value
, addend
as i64);
415 // FIXME don't duplicate definitions in lazy jit mode
416 let _
= module
.define_data(data_id
, &data_ctx
);
417 cx
.done
.insert(data_id
);
420 assert
!(cx
.todo
.is_empty(), "{:?}", cx
.todo
);
423 pub(crate) fn mir_operand_get_const_val
<'tcx
>(
424 fx
: &FunctionCx
<'_
, '_
, 'tcx
>,
425 operand
: &Operand
<'tcx
>,
426 ) -> Option
<ConstValue
<'tcx
>> {
428 Operand
::Copy(_
) | Operand
::Move(_
) => None
,
429 Operand
::Constant(const_
) => match const_
.literal
{
430 ConstantKind
::Ty(const_
) => {
431 fx
.monomorphize(const_
).eval(fx
.tcx
, ParamEnv
::reveal_all()).val
.try_to_value()
433 ConstantKind
::Val(val
, _
) => Some(val
),