1 //! The JIT driver uses [`cranelift_jit`] to JIT execute programs without writing any object
4 use std
::cell
::RefCell
;
6 use std
::lazy
::SyncOnceCell
;
7 use std
::os
::raw
::{c_char, c_int}
;
8 use std
::sync
::{mpsc, Mutex}
;
10 use cranelift_codegen
::binemit
::{NullStackMapSink, NullTrapSink}
;
11 use rustc_codegen_ssa
::CrateInfo
;
12 use rustc_middle
::mir
::mono
::MonoItem
;
13 use rustc_session
::Session
;
14 use rustc_span
::Symbol
;
16 use cranelift_jit
::{JITBuilder, JITModule}
;
18 use crate::{prelude::*, BackendConfig}
;
19 use crate::{CodegenCx, CodegenMode}
;
22 backend_config
: BackendConfig
,
23 jit_module
: JITModule
,
27 static LAZY_JIT_STATE
: RefCell
<Option
<JitState
>> = const { RefCell::new(None) }
;
30 /// The Sender owned by the rustc thread
31 static GLOBAL_MESSAGE_SENDER
: SyncOnceCell
<Mutex
<mpsc
::Sender
<UnsafeMessage
>>> =
34 /// A message that is sent from the jitted runtime to the rustc thread.
35 /// Senders are responsible for upholding `Send` semantics.
37 /// Request that the specified `Instance` be lazily jitted.
39 /// Nothing accessible through `instance_ptr` may be moved or mutated by the sender after
40 /// this message is sent.
42 instance_ptr
: *const Instance
<'
static>,
43 trampoline_ptr
: *const u8,
44 tx
: mpsc
::Sender
<*const u8>,
47 unsafe impl Send
for UnsafeMessage {}
51 fn send(self) -> Result
<(), mpsc
::SendError
<UnsafeMessage
>> {
53 /// The Sender owned by the local thread
54 static LOCAL_MESSAGE_SENDER
: mpsc
::Sender
<UnsafeMessage
> =
60 LOCAL_MESSAGE_SENDER
.with(|sender
| sender
.send(self))
64 fn create_jit_module
<'tcx
>(
66 backend_config
: &BackendConfig
,
68 ) -> (JITModule
, CodegenCx
<'tcx
>) {
69 let crate_info
= CrateInfo
::new(tcx
, "dummy_target_cpu".to_string());
70 let imported_symbols
= load_imported_symbols_for_jit(tcx
.sess
, crate_info
);
72 let isa
= crate::build_isa(tcx
.sess
, backend_config
);
73 let mut jit_builder
= JITBuilder
::with_isa(isa
, cranelift_module
::default_libcall_names());
74 jit_builder
.hotswap(hotswap
);
75 crate::compiler_builtins
::register_functions_for_jit(&mut jit_builder
);
76 jit_builder
.symbols(imported_symbols
);
77 let mut jit_module
= JITModule
::new(jit_builder
);
79 let mut cx
= crate::CodegenCx
::new(
81 backend_config
.clone(),
84 Symbol
::intern("dummy_cgu_name"),
87 crate::allocator
::codegen(tcx
, &mut jit_module
, &mut cx
.unwind_context
);
88 crate::main_shim
::maybe_create_entry_wrapper(
91 &mut cx
.unwind_context
,
99 pub(crate) fn run_jit(tcx
: TyCtxt
<'_
>, backend_config
: BackendConfig
) -> ! {
100 if !tcx
.sess
.opts
.output_types
.should_codegen() {
101 tcx
.sess
.fatal("JIT mode doesn't work with `cargo check`");
104 if !tcx
.sess
.crate_types().contains(&rustc_session
::config
::CrateType
::Executable
) {
105 tcx
.sess
.fatal("can't jit non-executable crate");
108 let (mut jit_module
, mut cx
) = create_jit_module(
111 matches
!(backend_config
.codegen_mode
, CodegenMode
::JitLazy
),
114 let (_
, cgus
) = tcx
.collect_and_partition_mono_items(());
115 let mono_items
= cgus
117 .map(|cgu
| cgu
.items_in_deterministic_order(tcx
).into_iter())
119 .collect
::<FxHashMap
<_
, (_
, _
)>>()
121 .collect
::<Vec
<(_
, (_
, _
))>>();
123 super::time(tcx
, backend_config
.display_cg_time
, "codegen mono items", || {
124 super::predefine_mono_items(tcx
, &mut jit_module
, &mono_items
);
125 for (mono_item
, _
) in mono_items
{
127 MonoItem
::Fn(inst
) => match backend_config
.codegen_mode
{
128 CodegenMode
::Aot
=> unreachable
!(),
129 CodegenMode
::Jit
=> {
130 cx
.tcx
.sess
.time("codegen fn", || {
131 crate::base
::codegen_fn(&mut cx
, &mut jit_module
, inst
)
134 CodegenMode
::JitLazy
=> codegen_shim(&mut cx
, &mut jit_module
, inst
),
136 MonoItem
::Static(def_id
) => {
137 crate::constant
::codegen_static(tcx
, &mut jit_module
, def_id
);
139 MonoItem
::GlobalAsm(item_id
) => {
140 let item
= tcx
.hir().item(item_id
);
141 tcx
.sess
.span_fatal(item
.span
, "Global asm is not supported in JIT mode");
147 if !cx
.global_asm
.is_empty() {
148 tcx
.sess
.fatal("Inline asm is not supported in JIT mode");
151 tcx
.sess
.abort_if_errors();
153 jit_module
.finalize_definitions();
154 unsafe { cx.unwind_context.register_jit(&jit_module) }
;
157 "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed"
160 let args
= std
::iter
::once(&*tcx
.crate_name(LOCAL_CRATE
).as_str().to_string())
161 .chain(backend_config
.jit_args
.iter().map(|arg
| &**arg
))
162 .map(|arg
| CString
::new(arg
).unwrap())
163 .collect
::<Vec
<_
>>();
165 let start_sig
= Signature
{
167 AbiParam
::new(jit_module
.target_config().pointer_type()),
168 AbiParam
::new(jit_module
.target_config().pointer_type()),
170 returns
: vec
![AbiParam
::new(jit_module
.target_config().pointer_type() /*isize*/)],
171 call_conv
: jit_module
.target_config().default_call_conv
,
173 let start_func_id
= jit_module
.declare_function("main", Linkage
::Import
, &start_sig
).unwrap();
174 let finalized_start
: *const u8 = jit_module
.get_finalized_function(start_func_id
);
176 LAZY_JIT_STATE
.with(|lazy_jit_state
| {
177 let mut lazy_jit_state
= lazy_jit_state
.borrow_mut();
178 assert
!(lazy_jit_state
.is_none());
179 *lazy_jit_state
= Some(JitState { backend_config, jit_module }
);
182 let f
: extern "C" fn(c_int
, *const *const c_char
) -> c_int
=
183 unsafe { ::std::mem::transmute(finalized_start) }
;
185 let (tx
, rx
) = mpsc
::channel();
186 GLOBAL_MESSAGE_SENDER
.set(Mutex
::new(tx
)).unwrap();
188 // Spawn the jitted runtime in a new thread so that this rustc thread can handle messages
189 // (eg to lazily JIT further functions as required)
190 std
::thread
::spawn(move || {
191 let mut argv
= args
.iter().map(|arg
| arg
.as_ptr()).collect
::<Vec
<_
>>();
193 // Push a null pointer as a terminating argument. This is required by POSIX and
194 // useful as some dynamic linkers use it as a marker to jump over.
195 argv
.push(std
::ptr
::null());
197 let ret
= f(args
.len() as c_int
, argv
.as_ptr());
198 std
::process
::exit(ret
);
203 match rx
.recv().unwrap() {
204 // lazy JIT compilation request - compile requested instance and return pointer to result
205 UnsafeMessage
::JitFn { instance_ptr, trampoline_ptr, tx }
=> {
206 tx
.send(jit_fn(instance_ptr
, trampoline_ptr
))
207 .expect("jitted runtime hung up before response to lazy JIT request was sent");
214 extern "C" fn __clif_jit_fn(
215 instance_ptr
: *const Instance
<'
static>,
216 trampoline_ptr
: *const u8,
218 // send the JIT request to the rustc thread, with a channel for the response
219 let (tx
, rx
) = mpsc
::channel();
220 UnsafeMessage
::JitFn { instance_ptr, trampoline_ptr, tx }
222 .expect("rustc thread hung up before lazy JIT request was sent");
224 // block on JIT compilation result
225 rx
.recv().expect("rustc thread hung up before responding to sent lazy JIT request")
228 fn jit_fn(instance_ptr
: *const Instance
<'
static>, trampoline_ptr
: *const u8) -> *const u8 {
229 rustc_middle
::ty
::tls
::with(|tcx
| {
230 // lift is used to ensure the correct lifetime for instance.
231 let instance
= tcx
.lift(unsafe { *instance_ptr }
).unwrap();
233 LAZY_JIT_STATE
.with(|lazy_jit_state
| {
234 let mut lazy_jit_state
= lazy_jit_state
.borrow_mut();
235 let lazy_jit_state
= lazy_jit_state
.as_mut().unwrap();
236 let jit_module
= &mut lazy_jit_state
.jit_module
;
237 let backend_config
= lazy_jit_state
.backend_config
.clone();
239 let name
= tcx
.symbol_name(instance
).name
;
240 let sig
= crate::abi
::get_function_sig(tcx
, jit_module
.isa().triple(), instance
);
241 let func_id
= jit_module
.declare_function(name
, Linkage
::Export
, &sig
).unwrap();
243 let current_ptr
= jit_module
.read_got_entry(func_id
);
245 // If the function's GOT entry has already been updated to point at something other
246 // than the shim trampoline, don't re-jit but just return the new pointer instead.
247 // This does not need synchronization as this code is executed only by a sole rustc
249 if current_ptr
!= trampoline_ptr
{
253 jit_module
.prepare_for_function_redefine(func_id
).unwrap();
255 let mut cx
= crate::CodegenCx
::new(
260 Symbol
::intern("dummy_cgu_name"),
262 tcx
.sess
.time("codegen fn", || crate::base
::codegen_fn(&mut cx
, jit_module
, instance
));
264 assert
!(cx
.global_asm
.is_empty());
265 jit_module
.finalize_definitions();
266 unsafe { cx.unwind_context.register_jit(&jit_module) }
;
267 jit_module
.get_finalized_function(func_id
)
272 fn load_imported_symbols_for_jit(
274 crate_info
: CrateInfo
,
275 ) -> Vec
<(String
, *const u8)> {
276 use rustc_middle
::middle
::dependency_format
::Linkage
;
278 let mut dylib_paths
= Vec
::new();
280 let data
= &crate_info
283 .find(|(crate_type
, _data
)| *crate_type
== rustc_session
::config
::CrateType
::Executable
)
286 for &cnum
in &crate_info
.used_crates
{
287 let src
= &crate_info
.used_crate_source
[&cnum
];
288 match data
[cnum
.as_usize() - 1] {
289 Linkage
::NotLinked
| Linkage
::IncludedFromDylib
=> {}
291 let name
= &crate_info
.crate_name
[&cnum
];
292 let mut err
= sess
.struct_err(&format
!("Can't load static lib {}", name
.as_str()));
293 err
.note("rustc_codegen_cranelift can only load dylibs in JIT mode.");
296 Linkage
::Dynamic
=> {
297 dylib_paths
.push(src
.dylib
.as_ref().unwrap().0.clone());
302 let mut imported_symbols
= Vec
::new();
303 for path
in dylib_paths
{
304 use object
::{Object, ObjectSymbol}
;
305 let lib
= libloading
::Library
::new(&path
).unwrap();
306 let obj
= std
::fs
::read(path
).unwrap();
307 let obj
= object
::File
::parse(&*obj
).unwrap();
308 imported_symbols
.extend(obj
.dynamic_symbols().filter_map(|symbol
| {
309 let name
= symbol
.name().unwrap().to_string();
310 if name
.is_empty() || !symbol
.is_global() || symbol
.is_undefined() {
313 if name
.starts_with("rust_metadata_") {
314 // The metadata is part of a section that is not loaded by the dynamic linker in
318 let dlsym_name
= if cfg
!(target_os
= "macos") {
319 // On macOS `dlsym` expects the name without leading `_`.
320 assert
!(name
.starts_with('_'
), "{:?}", name
);
325 let symbol
: libloading
::Symbol
<'_
, *const u8> =
326 unsafe { lib.get(dlsym_name.as_bytes()) }
.unwrap();
327 Some((name
, *symbol
))
329 std
::mem
::forget(lib
)
332 sess
.abort_if_errors();
337 fn codegen_shim
<'tcx
>(cx
: &mut CodegenCx
<'tcx
>, module
: &mut JITModule
, inst
: Instance
<'tcx
>) {
340 let pointer_type
= module
.target_config().pointer_type();
342 let name
= tcx
.symbol_name(inst
).name
;
343 let sig
= crate::abi
::get_function_sig(tcx
, module
.isa().triple(), inst
);
344 let func_id
= module
.declare_function(name
, Linkage
::Export
, &sig
).unwrap();
346 let instance_ptr
= Box
::into_raw(Box
::new(inst
));
353 call_conv
: module
.target_config().default_call_conv
,
354 params
: vec
![AbiParam
::new(pointer_type
), AbiParam
::new(pointer_type
)],
355 returns
: vec
![AbiParam
::new(pointer_type
)],
360 cx
.cached_context
.clear();
361 let trampoline
= &mut cx
.cached_context
.func
;
362 trampoline
.signature
= sig
.clone();
364 let mut builder_ctx
= FunctionBuilderContext
::new();
365 let mut trampoline_builder
= FunctionBuilder
::new(trampoline
, &mut builder_ctx
);
367 let trampoline_fn
= module
.declare_func_in_func(func_id
, trampoline_builder
.func
);
368 let jit_fn
= module
.declare_func_in_func(jit_fn
, trampoline_builder
.func
);
369 let sig_ref
= trampoline_builder
.func
.import_signature(sig
);
371 let entry_block
= trampoline_builder
.create_block();
372 trampoline_builder
.append_block_params_for_function_params(entry_block
);
373 let fn_args
= trampoline_builder
.func
.dfg
.block_params(entry_block
).to_vec();
375 trampoline_builder
.switch_to_block(entry_block
);
376 let instance_ptr
= trampoline_builder
.ins().iconst(pointer_type
, instance_ptr
as u64 as i64);
377 let trampoline_ptr
= trampoline_builder
.ins().func_addr(pointer_type
, trampoline_fn
);
378 let jitted_fn
= trampoline_builder
.ins().call(jit_fn
, &[instance_ptr
, trampoline_ptr
]);
379 let jitted_fn
= trampoline_builder
.func
.dfg
.inst_results(jitted_fn
)[0];
380 let call_inst
= trampoline_builder
.ins().call_indirect(sig_ref
, jitted_fn
, &fn_args
);
381 let ret_vals
= trampoline_builder
.func
.dfg
.inst_results(call_inst
).to_vec();
382 trampoline_builder
.ins().return_(&ret_vals
);
387 &mut cx
.cached_context
,
388 &mut NullTrapSink {}
,
389 &mut NullStackMapSink {}
,