1 //! Handling of `static`s, `const`s and promoted allocations
3 use rustc_data_structures
::fx
::{FxHashMap, FxHashSet}
;
4 use rustc_middle
::middle
::codegen_fn_attrs
::CodegenFnAttrFlags
;
5 use rustc_middle
::mir
::interpret
::{
6 read_target_uint
, AllocId
, ConstAllocation
, ConstValue
, ErrorHandled
, GlobalAlloc
, Scalar
,
8 use rustc_middle
::ty
::ConstKind
;
9 use rustc_span
::DUMMY_SP
;
11 use cranelift_codegen
::ir
::GlobalValueData
;
12 use cranelift_module
::*;
14 use crate::prelude
::*;
16 pub(crate) struct ConstantCx
{
18 done
: FxHashSet
<DataId
>,
19 anon_allocs
: FxHashMap
<AllocId
, DataId
>,
22 #[derive(Copy, Clone, Debug)]
29 pub(crate) fn new() -> Self {
30 ConstantCx { todo: vec![], done: FxHashSet::default(), anon_allocs: FxHashMap::default() }
33 pub(crate) fn finalize(mut self, tcx
: TyCtxt
<'_
>, module
: &mut dyn Module
) {
34 //println!("todo {:?}", self.todo);
35 define_all_allocs(tcx
, module
, &mut self);
36 //println!("done {:?}", self.done);
41 pub(crate) fn check_constants(fx
: &mut FunctionCx
<'_
, '_
, '_
>) -> bool
{
42 let mut all_constants_ok
= true;
43 for constant
in &fx
.mir
.required_consts
{
44 let const_
= match fx
.monomorphize(constant
.literal
) {
45 ConstantKind
::Ty(ct
) => ct
,
46 ConstantKind
::Val(..) => continue,
49 ConstKind
::Value(_
) => {}
50 ConstKind
::Unevaluated(unevaluated
) => {
52 fx
.tcx
.const_eval_resolve(ParamEnv
::reveal_all(), unevaluated
, None
)
54 all_constants_ok
= false;
56 ErrorHandled
::Reported(_
) | ErrorHandled
::Linted
=> {
57 fx
.tcx
.sess
.span_err(constant
.span
, "erroneous constant encountered");
59 ErrorHandled
::TooGeneric
=> {
62 "codgen encountered polymorphic constant: {:?}",
71 | ConstKind
::Bound(_
, _
)
72 | ConstKind
::Placeholder(_
)
73 | ConstKind
::Error(_
) => unreachable
!("{:?}", const_
),
79 pub(crate) fn codegen_static(tcx
: TyCtxt
<'_
>, module
: &mut dyn Module
, def_id
: DefId
) {
80 let mut constants_cx
= ConstantCx
::new();
81 constants_cx
.todo
.push(TodoItem
::Static(def_id
));
82 constants_cx
.finalize(tcx
, module
);
85 pub(crate) fn codegen_tls_ref
<'tcx
>(
86 fx
: &mut FunctionCx
<'_
, '_
, 'tcx
>,
88 layout
: TyAndLayout
<'tcx
>,
90 let data_id
= data_id_for_static(fx
.tcx
, fx
.module
, def_id
, false);
91 let local_data_id
= fx
.module
.declare_data_in_func(data_id
, &mut fx
.bcx
.func
);
92 if fx
.clif_comments
.enabled() {
93 fx
.add_comment(local_data_id
, format
!("tls {:?}", def_id
));
95 let tls_ptr
= fx
.bcx
.ins().tls_value(fx
.pointer_type
, local_data_id
);
96 CValue
::by_val(tls_ptr
, layout
)
99 fn codegen_static_ref
<'tcx
>(
100 fx
: &mut FunctionCx
<'_
, '_
, 'tcx
>,
102 layout
: TyAndLayout
<'tcx
>,
104 let data_id
= data_id_for_static(fx
.tcx
, fx
.module
, def_id
, false);
105 let local_data_id
= fx
.module
.declare_data_in_func(data_id
, &mut fx
.bcx
.func
);
106 if fx
.clif_comments
.enabled() {
107 fx
.add_comment(local_data_id
, format
!("{:?}", def_id
));
109 let global_ptr
= fx
.bcx
.ins().global_value(fx
.pointer_type
, local_data_id
);
110 assert
!(!layout
.is_unsized(), "unsized statics aren't supported");
113 fx
.bcx
.func
.global_values
[local_data_id
],
114 GlobalValueData
::Symbol { tls: false, .. }
116 "tls static referenced without Rvalue::ThreadLocalRef"
118 CPlace
::for_ptr(crate::pointer
::Pointer
::new(global_ptr
), layout
)
121 pub(crate) fn codegen_constant
<'tcx
>(
122 fx
: &mut FunctionCx
<'_
, '_
, 'tcx
>,
123 constant
: &Constant
<'tcx
>,
125 let const_
= match fx
.monomorphize(constant
.literal
) {
126 ConstantKind
::Ty(ct
) => ct
,
127 ConstantKind
::Val(val
, ty
) => return codegen_const_value(fx
, val
, ty
),
129 let const_val
= match const_
.kind() {
130 ConstKind
::Value(valtree
) => fx
.tcx
.valtree_to_const_val((const_
.ty(), valtree
)),
131 ConstKind
::Unevaluated(ty
::Unevaluated { def, substs, promoted }
)
132 if fx
.tcx
.is_static(def
.did
) =>
134 assert
!(substs
.is_empty());
135 assert
!(promoted
.is_none());
137 return codegen_static_ref(fx
, def
.did
, fx
.layout_of(const_
.ty())).to_cvalue(fx
);
139 ConstKind
::Unevaluated(unevaluated
) => {
140 match fx
.tcx
.const_eval_resolve(ParamEnv
::reveal_all(), unevaluated
, None
) {
141 Ok(const_val
) => const_val
,
143 span_bug
!(constant
.span
, "erroneous constant not captured by required_consts");
148 | ConstKind
::Infer(_
)
149 | ConstKind
::Bound(_
, _
)
150 | ConstKind
::Placeholder(_
)
151 | ConstKind
::Error(_
) => unreachable
!("{:?}", const_
),
154 codegen_const_value(fx
, const_val
, const_
.ty())
157 pub(crate) fn codegen_const_value
<'tcx
>(
158 fx
: &mut FunctionCx
<'_
, '_
, 'tcx
>,
159 const_val
: ConstValue
<'tcx
>,
162 let layout
= fx
.layout_of(ty
);
163 assert
!(!layout
.is_unsized(), "sized const value");
166 return CValue
::by_ref(crate::Pointer
::dangling(layout
.align
.pref
), layout
);
170 ConstValue
::Scalar(x
) => match x
{
171 Scalar
::Int(int
) => {
172 if fx
.clif_type(layout
.ty
).is_some() {
173 return CValue
::const_val(fx
, layout
, int
);
175 let raw_val
= int
.to_bits(int
.size()).unwrap();
176 let val
= match int
.size().bytes() {
177 1 => fx
.bcx
.ins().iconst(types
::I8
, raw_val
as i64),
178 2 => fx
.bcx
.ins().iconst(types
::I16
, raw_val
as i64),
179 4 => fx
.bcx
.ins().iconst(types
::I32
, raw_val
as i64),
180 8 => fx
.bcx
.ins().iconst(types
::I64
, raw_val
as i64),
182 let lsb
= fx
.bcx
.ins().iconst(types
::I64
, raw_val
as u64 as i64);
184 fx
.bcx
.ins().iconst(types
::I64
, (raw_val
>> 64) as u64 as i64);
185 fx
.bcx
.ins().iconcat(lsb
, msb
)
190 let place
= CPlace
::new_stack_slot(fx
, layout
);
191 place
.to_ptr().store(fx
, val
, MemFlags
::trusted());
195 Scalar
::Ptr(ptr
, _size
) => {
196 let (alloc_id
, offset
) = ptr
.into_parts(); // we know the `offset` is relative
197 let alloc_kind
= fx
.tcx
.get_global_alloc(alloc_id
);
198 let base_addr
= match alloc_kind
{
199 Some(GlobalAlloc
::Memory(alloc
)) => {
200 let data_id
= data_id_for_alloc_id(
201 &mut fx
.constants_cx
,
204 alloc
.inner().mutability
,
207 fx
.module
.declare_data_in_func(data_id
, &mut fx
.bcx
.func
);
208 if fx
.clif_comments
.enabled() {
209 fx
.add_comment(local_data_id
, format
!("{:?}", alloc_id
));
211 fx
.bcx
.ins().global_value(fx
.pointer_type
, local_data_id
)
213 Some(GlobalAlloc
::Function(instance
)) => {
214 let func_id
= crate::abi
::import_function(fx
.tcx
, fx
.module
, instance
);
216 fx
.module
.declare_func_in_func(func_id
, &mut fx
.bcx
.func
);
217 fx
.bcx
.ins().func_addr(fx
.pointer_type
, local_func_id
)
219 Some(GlobalAlloc
::Static(def_id
)) => {
220 assert
!(fx
.tcx
.is_static(def_id
));
221 let data_id
= data_id_for_static(fx
.tcx
, fx
.module
, def_id
, false);
223 fx
.module
.declare_data_in_func(data_id
, &mut fx
.bcx
.func
);
224 if fx
.clif_comments
.enabled() {
225 fx
.add_comment(local_data_id
, format
!("{:?}", def_id
));
227 fx
.bcx
.ins().global_value(fx
.pointer_type
, local_data_id
)
229 None
=> bug
!("missing allocation {:?}", alloc_id
),
231 let val
= if offset
.bytes() != 0 {
232 fx
.bcx
.ins().iadd_imm(base_addr
, i64::try_from(offset
.bytes()).unwrap())
236 CValue
::by_val(val
, layout
)
239 ConstValue
::ByRef { alloc, offset }
=> CValue
::by_ref(
240 pointer_for_allocation(fx
, alloc
)
241 .offset_i64(fx
, i64::try_from(offset
.bytes()).unwrap()),
244 ConstValue
::Slice { data, start, end }
=> {
245 let ptr
= pointer_for_allocation(fx
, data
)
246 .offset_i64(fx
, i64::try_from(start
).unwrap())
251 .iconst(fx
.pointer_type
, i64::try_from(end
.checked_sub(start
).unwrap()).unwrap());
252 CValue
::by_val_pair(ptr
, len
, layout
)
257 pub(crate) fn pointer_for_allocation
<'tcx
>(
258 fx
: &mut FunctionCx
<'_
, '_
, 'tcx
>,
259 alloc
: ConstAllocation
<'tcx
>,
260 ) -> crate::pointer
::Pointer
{
261 let alloc_id
= fx
.tcx
.create_memory_alloc(alloc
);
262 let data_id
= data_id_for_alloc_id(
263 &mut fx
.constants_cx
,
266 alloc
.inner().mutability
,
269 let local_data_id
= fx
.module
.declare_data_in_func(data_id
, &mut fx
.bcx
.func
);
270 if fx
.clif_comments
.enabled() {
271 fx
.add_comment(local_data_id
, format
!("{:?}", alloc_id
));
273 let global_ptr
= fx
.bcx
.ins().global_value(fx
.pointer_type
, local_data_id
);
274 crate::pointer
::Pointer
::new(global_ptr
)
277 pub(crate) fn data_id_for_alloc_id(
279 module
: &mut dyn Module
,
281 mutability
: rustc_hir
::Mutability
,
283 cx
.todo
.push(TodoItem
::Alloc(alloc_id
));
284 *cx
.anon_allocs
.entry(alloc_id
).or_insert_with(|| {
285 module
.declare_anonymous_data(mutability
== rustc_hir
::Mutability
::Mut
, false).unwrap()
289 fn data_id_for_static(
291 module
: &mut dyn Module
,
295 let rlinkage
= tcx
.codegen_fn_attrs(def_id
).linkage
;
296 let linkage
= if definition
{
297 crate::linkage
::get_static_linkage(tcx
, def_id
)
298 } else if rlinkage
== Some(rustc_middle
::mir
::mono
::Linkage
::ExternalWeak
)
299 || rlinkage
== Some(rustc_middle
::mir
::mono
::Linkage
::WeakAny
)
306 let instance
= Instance
::mono(tcx
, def_id
).polymorphize(tcx
);
307 let symbol_name
= tcx
.symbol_name(instance
).name
;
308 let ty
= instance
.ty(tcx
, ParamEnv
::reveal_all());
309 let is_mutable
= if tcx
.is_mutable_static(def_id
) {
312 !ty
.is_freeze(tcx
.at(DUMMY_SP
), ParamEnv
::reveal_all())
314 let align
= tcx
.layout_of(ParamEnv
::reveal_all().and(ty
)).unwrap().align
.pref
.bytes();
316 let attrs
= tcx
.codegen_fn_attrs(def_id
);
323 attrs
.flags
.contains(CodegenFnAttrFlags
::THREAD_LOCAL
),
327 if rlinkage
.is_some() {
328 // Comment copied from https://github.com/rust-lang/rust/blob/45060c2a66dfd667f88bd8b94261b28a58d85bd5/src/librustc_codegen_llvm/consts.rs#L141
329 // Declare an internal global `extern_with_linkage_foo` which
330 // is initialized with the address of `foo`. If `foo` is
331 // discarded during linking (for example, if `foo` has weak
332 // linkage and there are no definitions), then
333 // `extern_with_linkage_foo` will instead be initialized to
336 let ref_name
= format
!("_rust_extern_with_linkage_{}", symbol_name
);
337 let ref_data_id
= module
.declare_data(&ref_name
, Linkage
::Local
, false, false).unwrap();
338 let mut data_ctx
= DataContext
::new();
339 data_ctx
.set_align(align
);
340 let data
= module
.declare_data_in_data(data_id
, &mut data_ctx
);
341 data_ctx
.define(std
::iter
::repeat(0).take(pointer_ty(tcx
).bytes() as usize).collect());
342 data_ctx
.write_data_addr(0, data
, 0);
343 match module
.define_data(ref_data_id
, &data_ctx
) {
344 // Every time the static is referenced there will be another definition of this global,
345 // so duplicate definitions are expected and allowed.
346 Err(ModuleError
::DuplicateDefinition(_
)) => {}
355 fn define_all_allocs(tcx
: TyCtxt
<'_
>, module
: &mut dyn Module
, cx
: &mut ConstantCx
) {
356 while let Some(todo_item
) = cx
.todo
.pop() {
357 let (data_id
, alloc
, section_name
) = match todo_item
{
358 TodoItem
::Alloc(alloc_id
) => {
359 //println!("alloc_id {}", alloc_id);
360 let alloc
= match tcx
.get_global_alloc(alloc_id
).unwrap() {
361 GlobalAlloc
::Memory(alloc
) => alloc
,
362 GlobalAlloc
::Function(_
) | GlobalAlloc
::Static(_
) => unreachable
!(),
364 let data_id
= *cx
.anon_allocs
.entry(alloc_id
).or_insert_with(|| {
366 .declare_anonymous_data(
367 alloc
.inner().mutability
== rustc_hir
::Mutability
::Mut
,
372 (data_id
, alloc
, None
)
374 TodoItem
::Static(def_id
) => {
375 //println!("static {:?}", def_id);
377 let section_name
= tcx
.codegen_fn_attrs(def_id
).link_section
;
379 let alloc
= tcx
.eval_static_initializer(def_id
).unwrap();
381 let data_id
= data_id_for_static(tcx
, module
, def_id
, true);
382 (data_id
, alloc
, section_name
)
386 //("data_id {}", data_id);
387 if cx
.done
.contains(&data_id
) {
391 let mut data_ctx
= DataContext
::new();
392 let alloc
= alloc
.inner();
393 data_ctx
.set_align(alloc
.align
.bytes());
395 if let Some(section_name
) = section_name
{
396 let (segment_name
, section_name
) = if tcx
.sess
.target
.is_like_osx
{
397 let section_name
= section_name
.as_str();
398 if let Some(names
) = section_name
.split_once('
,'
) {
401 tcx
.sess
.fatal(&format
!(
402 "#[link_section = \"{}\"] is not valid for macos target: must be segment and section separated by comma",
407 ("", section_name
.as_str())
409 data_ctx
.set_segment_section(segment_name
, section_name
);
412 let bytes
= alloc
.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc
.len()).to_vec();
413 data_ctx
.define(bytes
.into_boxed_slice());
415 for &(offset
, alloc_id
) in alloc
.relocations().iter() {
417 let endianness
= tcx
.data_layout
.endian
;
418 let offset
= offset
.bytes() as usize;
419 let ptr_size
= tcx
.data_layout
.pointer_size
;
420 let bytes
= &alloc
.inspect_with_uninit_and_ptr_outside_interpreter(
421 offset
..offset
+ ptr_size
.bytes() as usize,
423 read_target_uint(endianness
, bytes
).unwrap()
426 let reloc_target_alloc
= tcx
.get_global_alloc(alloc_id
).unwrap();
427 let data_id
= match reloc_target_alloc
{
428 GlobalAlloc
::Function(instance
) => {
429 assert_eq
!(addend
, 0);
430 let func_id
= crate::abi
::import_function(tcx
, module
, instance
);
431 let local_func_id
= module
.declare_func_in_data(func_id
, &mut data_ctx
);
432 data_ctx
.write_function_addr(offset
.bytes() as u32, local_func_id
);
435 GlobalAlloc
::Memory(target_alloc
) => {
436 data_id_for_alloc_id(cx
, module
, alloc_id
, target_alloc
.inner().mutability
)
438 GlobalAlloc
::Static(def_id
) => {
439 if tcx
.codegen_fn_attrs(def_id
).flags
.contains(CodegenFnAttrFlags
::THREAD_LOCAL
)
441 tcx
.sess
.fatal(&format
!(
442 "Allocation {:?} contains reference to TLS value {:?}",
447 // Don't push a `TodoItem::Static` here, as it will cause statics used by
448 // multiple crates to be duplicated between them. It isn't necessary anyway,
449 // as it will get pushed by `codegen_static` when necessary.
450 data_id_for_static(tcx
, module
, def_id
, false)
454 let global_value
= module
.declare_data_in_data(data_id
, &mut data_ctx
);
455 data_ctx
.write_data_addr(offset
.bytes() as u32, global_value
, addend
as i64);
458 module
.define_data(data_id
, &data_ctx
).unwrap();
459 cx
.done
.insert(data_id
);
462 assert
!(cx
.todo
.is_empty(), "{:?}", cx
.todo
);
465 pub(crate) fn mir_operand_get_const_val
<'tcx
>(
466 fx
: &FunctionCx
<'_
, '_
, 'tcx
>,
467 operand
: &Operand
<'tcx
>,
468 ) -> Option
<ConstValue
<'tcx
>> {
470 Operand
::Constant(const_
) => match const_
.literal
{
471 ConstantKind
::Ty(const_
) => fx
472 .monomorphize(const_
)
473 .eval_for_mir(fx
.tcx
, ParamEnv
::reveal_all())
474 .try_to_value(fx
.tcx
),
475 ConstantKind
::Val(val
, _
) => Some(val
),
477 // FIXME(rust-lang/rust#85105): Casts like `IMM8 as u32` result in the const being stored
478 // inside a temporary before being passed to the intrinsic requiring the const argument.
479 // This code tries to find a single constant defining definition of the referenced local.
480 Operand
::Copy(place
) | Operand
::Move(place
) => {
481 if !place
.projection
.is_empty() {
484 let mut computed_const_val
= None
;
485 for bb_data
in fx
.mir
.basic_blocks() {
486 for stmt
in &bb_data
.statements
{
488 StatementKind
::Assign(local_and_rvalue
) if &local_and_rvalue
.0 == place
=> {
489 match &local_and_rvalue
.1 {
490 Rvalue
::Cast(CastKind
::Misc
, operand
, ty
) => {
491 if computed_const_val
.is_some() {
492 return None
; // local assigned twice
494 if !matches
!(ty
.kind(), ty
::Uint(_
) | ty
::Int(_
)) {
497 let const_val
= mir_operand_get_const_val(fx
, operand
)?
;
498 if fx
.layout_of(*ty
).size
499 != const_val
.try_to_scalar_int()?
.size()
503 computed_const_val
= Some(const_val
);
505 Rvalue
::Use(operand
) => {
506 computed_const_val
= mir_operand_get_const_val(fx
, operand
)
511 StatementKind
::SetDiscriminant { place: stmt_place, variant_index: _ }
512 if &**stmt_place
== place
=>
516 StatementKind
::CopyNonOverlapping(_
) => {
518 } // conservative handling
519 StatementKind
::Assign(_
)
520 | StatementKind
::FakeRead(_
)
521 | StatementKind
::SetDiscriminant { .. }
522 | StatementKind
::Deinit(_
)
523 | StatementKind
::StorageLive(_
)
524 | StatementKind
::StorageDead(_
)
525 | StatementKind
::Retag(_
, _
)
526 | StatementKind
::AscribeUserType(_
, _
)
527 | StatementKind
::Coverage(_
)
528 | StatementKind
::Nop
=> {}
531 match &bb_data
.terminator().kind
{
532 TerminatorKind
::Goto { .. }
533 | TerminatorKind
::SwitchInt { .. }
534 | TerminatorKind
::Resume
535 | TerminatorKind
::Abort
536 | TerminatorKind
::Return
537 | TerminatorKind
::Unreachable
538 | TerminatorKind
::Drop { .. }
539 | TerminatorKind
::Assert { .. }
=> {}
540 TerminatorKind
::DropAndReplace { .. }
541 | TerminatorKind
::Yield { .. }
542 | TerminatorKind
::GeneratorDrop
543 | TerminatorKind
::FalseEdge { .. }
544 | TerminatorKind
::FalseUnwind { .. }
=> unreachable
!(),
545 TerminatorKind
::InlineAsm { .. }
=> return None
,
546 TerminatorKind
::Call { destination, target: Some(_), .. }
547 if destination
== place
=>
551 TerminatorKind
::Call { .. }
=> {}