]>
Commit | Line | Data |
---|---|---|
17df50a5 | 1 | //! The JIT driver uses [`cranelift_jit`] to JIT execute programs without writing any object |
29967ef6 XL |
2 | //! files. |
3 | ||
5869c6ff | 4 | use std::cell::RefCell; |
29967ef6 XL |
5 | use std::ffi::CString; |
6 | use std::os::raw::{c_char, c_int}; | |
136023e0 | 7 | use std::sync::{mpsc, Mutex}; |
29967ef6 XL |
8 | |
9 | use rustc_codegen_ssa::CrateInfo; | |
5869c6ff | 10 | use rustc_middle::mir::mono::MonoItem; |
136023e0 | 11 | use rustc_session::Session; |
a2a8927a | 12 | use rustc_span::Symbol; |
29967ef6 | 13 | |
5869c6ff | 14 | use cranelift_jit::{JITBuilder, JITModule}; |
29967ef6 | 15 | |
923072b8 | 16 | // FIXME use std::sync::OnceLock once it stabilizes |
5e7ed085 FG |
17 | use once_cell::sync::OnceCell; |
18 | ||
6a06907d | 19 | use crate::{prelude::*, BackendConfig}; |
5869c6ff | 20 | use crate::{CodegenCx, CodegenMode}; |
29967ef6 | 21 | |
17df50a5 XL |
22 | struct JitState { |
23 | backend_config: BackendConfig, | |
24 | jit_module: JITModule, | |
5869c6ff XL |
25 | } |
26 | ||
17df50a5 | 27 | thread_local! { |
a2a8927a | 28 | static LAZY_JIT_STATE: RefCell<Option<JitState>> = const { RefCell::new(None) }; |
17df50a5 | 29 | } |
29967ef6 | 30 | |
136023e0 | 31 | /// The Sender owned by the rustc thread |
5e7ed085 | 32 | static GLOBAL_MESSAGE_SENDER: OnceCell<Mutex<mpsc::Sender<UnsafeMessage>>> = OnceCell::new(); |
136023e0 XL |
33 | |
34 | /// A message that is sent from the jitted runtime to the rustc thread. | |
35 | /// Senders are responsible for upholding `Send` semantics. | |
36 | enum UnsafeMessage { | |
37 | /// Request that the specified `Instance` be lazily jitted. | |
38 | /// | |
39 | /// Nothing accessible through `instance_ptr` may be moved or mutated by the sender after | |
40 | /// this message is sent. | |
41 | JitFn { | |
42 | instance_ptr: *const Instance<'static>, | |
43 | trampoline_ptr: *const u8, | |
44 | tx: mpsc::Sender<*const u8>, | |
45 | }, | |
46 | } | |
47 | unsafe impl Send for UnsafeMessage {} | |
48 | ||
49 | impl UnsafeMessage { | |
50 | /// Send the message. | |
51 | fn send(self) -> Result<(), mpsc::SendError<UnsafeMessage>> { | |
52 | thread_local! { | |
53 | /// The Sender owned by the local thread | |
a2a8927a | 54 | static LOCAL_MESSAGE_SENDER: mpsc::Sender<UnsafeMessage> = |
136023e0 XL |
55 | GLOBAL_MESSAGE_SENDER |
56 | .get().unwrap() | |
57 | .lock().unwrap() | |
a2a8927a | 58 | .clone(); |
136023e0 XL |
59 | } |
60 | LOCAL_MESSAGE_SENDER.with(|sender| sender.send(self)) | |
61 | } | |
62 | } | |
63 | ||
f2b60f7d FG |
64 | fn create_jit_module( |
65 | tcx: TyCtxt<'_>, | |
17df50a5 XL |
66 | backend_config: &BackendConfig, |
67 | hotswap: bool, | |
f2b60f7d | 68 | ) -> (JITModule, CodegenCx) { |
136023e0 | 69 | let crate_info = CrateInfo::new(tcx, "dummy_target_cpu".to_string()); |
29967ef6 | 70 | |
17df50a5 XL |
71 | let isa = crate::build_isa(tcx.sess, backend_config); |
72 | let mut jit_builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names()); | |
73 | jit_builder.hotswap(hotswap); | |
cdc7bbd5 | 74 | crate::compiler_builtins::register_functions_for_jit(&mut jit_builder); |
2b03887a | 75 | jit_builder.symbol_lookup_fn(dep_symbol_lookup_fn(tcx.sess, crate_info)); |
923072b8 | 76 | jit_builder.symbol("__clif_jit_fn", clif_jit_fn as *const u8); |
5869c6ff | 77 | let mut jit_module = JITModule::new(jit_builder); |
29967ef6 | 78 | |
a2a8927a XL |
79 | let mut cx = crate::CodegenCx::new( |
80 | tcx, | |
81 | backend_config.clone(), | |
82 | jit_module.isa(), | |
83 | false, | |
84 | Symbol::intern("dummy_cgu_name"), | |
85 | ); | |
17df50a5 XL |
86 | |
87 | crate::allocator::codegen(tcx, &mut jit_module, &mut cx.unwind_context); | |
88 | crate::main_shim::maybe_create_entry_wrapper( | |
89 | tcx, | |
90 | &mut jit_module, | |
91 | &mut cx.unwind_context, | |
92 | true, | |
93 | true, | |
94 | ); | |
95 | ||
96 | (jit_module, cx) | |
97 | } | |
98 | ||
99 | pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { | |
100 | if !tcx.sess.opts.output_types.should_codegen() { | |
101 | tcx.sess.fatal("JIT mode doesn't work with `cargo check`"); | |
102 | } | |
103 | ||
104 | if !tcx.sess.crate_types().contains(&rustc_session::config::CrateType::Executable) { | |
105 | tcx.sess.fatal("can't jit non-executable crate"); | |
106 | } | |
107 | ||
108 | let (mut jit_module, mut cx) = create_jit_module( | |
109 | tcx, | |
110 | &backend_config, | |
111 | matches!(backend_config.codegen_mode, CodegenMode::JitLazy), | |
112 | ); | |
f2b60f7d | 113 | let mut cached_context = Context::new(); |
17df50a5 XL |
114 | |
115 | let (_, cgus) = tcx.collect_and_partition_mono_items(()); | |
29967ef6 XL |
116 | let mono_items = cgus |
117 | .iter() | |
118 | .map(|cgu| cgu.items_in_deterministic_order(tcx).into_iter()) | |
119 | .flatten() | |
120 | .collect::<FxHashMap<_, (_, _)>>() | |
121 | .into_iter() | |
122 | .collect::<Vec<(_, (_, _))>>(); | |
123 | ||
9ffffee4 | 124 | tcx.sess.time("codegen mono items", || { |
17df50a5 | 125 | super::predefine_mono_items(tcx, &mut jit_module, &mono_items); |
cdc7bbd5 | 126 | for (mono_item, _) in mono_items { |
5869c6ff | 127 | match mono_item { |
6a06907d | 128 | MonoItem::Fn(inst) => match backend_config.codegen_mode { |
5869c6ff XL |
129 | CodegenMode::Aot => unreachable!(), |
130 | CodegenMode::Jit => { | |
9ffffee4 FG |
131 | codegen_and_compile_fn( |
132 | tcx, | |
133 | &mut cx, | |
134 | &mut cached_context, | |
135 | &mut jit_module, | |
136 | inst, | |
137 | ); | |
5869c6ff | 138 | } |
f2b60f7d FG |
139 | CodegenMode::JitLazy => { |
140 | codegen_shim(tcx, &mut cx, &mut cached_context, &mut jit_module, inst) | |
141 | } | |
5869c6ff XL |
142 | }, |
143 | MonoItem::Static(def_id) => { | |
17df50a5 | 144 | crate::constant::codegen_static(tcx, &mut jit_module, def_id); |
5869c6ff | 145 | } |
6a06907d | 146 | MonoItem::GlobalAsm(item_id) => { |
17df50a5 | 147 | let item = tcx.hir().item(item_id); |
6a06907d | 148 | tcx.sess.span_fatal(item.span, "Global asm is not supported in JIT mode"); |
5869c6ff | 149 | } |
fc512014 | 150 | } |
5869c6ff XL |
151 | } |
152 | }); | |
153 | ||
17df50a5 | 154 | if !cx.global_asm.is_empty() { |
5869c6ff | 155 | tcx.sess.fatal("Inline asm is not supported in JIT mode"); |
29967ef6 | 156 | } |
5869c6ff | 157 | |
29967ef6 XL |
158 | tcx.sess.abort_if_errors(); |
159 | ||
9c376795 | 160 | jit_module.finalize_definitions().unwrap(); |
17df50a5 | 161 | unsafe { cx.unwind_context.register_jit(&jit_module) }; |
29967ef6 | 162 | |
6a06907d XL |
163 | println!( |
164 | "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed" | |
165 | ); | |
29967ef6 | 166 | |
29967ef6 | 167 | let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string()) |
17df50a5 | 168 | .chain(backend_config.jit_args.iter().map(|arg| &**arg)) |
29967ef6 XL |
169 | .map(|arg| CString::new(arg).unwrap()) |
170 | .collect::<Vec<_>>(); | |
29967ef6 | 171 | |
17df50a5 XL |
172 | let start_sig = Signature { |
173 | params: vec![ | |
174 | AbiParam::new(jit_module.target_config().pointer_type()), | |
175 | AbiParam::new(jit_module.target_config().pointer_type()), | |
176 | ], | |
177 | returns: vec![AbiParam::new(jit_module.target_config().pointer_type() /*isize*/)], | |
136023e0 | 178 | call_conv: jit_module.target_config().default_call_conv, |
17df50a5 XL |
179 | }; |
180 | let start_func_id = jit_module.declare_function("main", Linkage::Import, &start_sig).unwrap(); | |
181 | let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id); | |
182 | ||
183 | LAZY_JIT_STATE.with(|lazy_jit_state| { | |
184 | let mut lazy_jit_state = lazy_jit_state.borrow_mut(); | |
185 | assert!(lazy_jit_state.is_none()); | |
186 | *lazy_jit_state = Some(JitState { backend_config, jit_module }); | |
6a06907d | 187 | }); |
5869c6ff | 188 | |
17df50a5 XL |
189 | let f: extern "C" fn(c_int, *const *const c_char) -> c_int = |
190 | unsafe { ::std::mem::transmute(finalized_start) }; | |
136023e0 XL |
191 | |
192 | let (tx, rx) = mpsc::channel(); | |
193 | GLOBAL_MESSAGE_SENDER.set(Mutex::new(tx)).unwrap(); | |
194 | ||
195 | // Spawn the jitted runtime in a new thread so that this rustc thread can handle messages | |
196 | // (eg to lazily JIT further functions as required) | |
197 | std::thread::spawn(move || { | |
198 | let mut argv = args.iter().map(|arg| arg.as_ptr()).collect::<Vec<_>>(); | |
199 | ||
200 | // Push a null pointer as a terminating argument. This is required by POSIX and | |
201 | // useful as some dynamic linkers use it as a marker to jump over. | |
202 | argv.push(std::ptr::null()); | |
203 | ||
204 | let ret = f(args.len() as c_int, argv.as_ptr()); | |
205 | std::process::exit(ret); | |
206 | }); | |
207 | ||
208 | // Handle messages | |
209 | loop { | |
210 | match rx.recv().unwrap() { | |
211 | // lazy JIT compilation request - compile requested instance and return pointer to result | |
212 | UnsafeMessage::JitFn { instance_ptr, trampoline_ptr, tx } => { | |
213 | tx.send(jit_fn(instance_ptr, trampoline_ptr)) | |
214 | .expect("jitted runtime hung up before response to lazy JIT request was sent"); | |
215 | } | |
216 | } | |
217 | } | |
29967ef6 XL |
218 | } |
219 | ||
9ffffee4 FG |
220 | pub(crate) fn codegen_and_compile_fn<'tcx>( |
221 | tcx: TyCtxt<'tcx>, | |
222 | cx: &mut crate::CodegenCx, | |
223 | cached_context: &mut Context, | |
224 | module: &mut dyn Module, | |
225 | instance: Instance<'tcx>, | |
226 | ) { | |
49aad941 FG |
227 | cranelift_codegen::timing::set_thread_profiler(Box::new(super::MeasuremeProfiler( |
228 | cx.profiler.clone(), | |
229 | ))); | |
230 | ||
9ffffee4 FG |
231 | tcx.prof.generic_activity("codegen and compile fn").run(|| { |
232 | let _inst_guard = | |
233 | crate::PrintOnPanic(|| format!("{:?} {}", instance, tcx.symbol_name(instance).name)); | |
234 | ||
235 | let cached_func = std::mem::replace(&mut cached_context.func, Function::new()); | |
236 | let codegened_func = crate::base::codegen_fn(tcx, cx, cached_func, module, instance); | |
237 | ||
238 | crate::base::compile_fn(cx, cached_context, module, codegened_func); | |
239 | }); | |
240 | } | |
241 | ||
923072b8 | 242 | extern "C" fn clif_jit_fn( |
136023e0 XL |
243 | instance_ptr: *const Instance<'static>, |
244 | trampoline_ptr: *const u8, | |
245 | ) -> *const u8 { | |
246 | // send the JIT request to the rustc thread, with a channel for the response | |
247 | let (tx, rx) = mpsc::channel(); | |
248 | UnsafeMessage::JitFn { instance_ptr, trampoline_ptr, tx } | |
249 | .send() | |
250 | .expect("rustc thread hung up before lazy JIT request was sent"); | |
251 | ||
252 | // block on JIT compilation result | |
253 | rx.recv().expect("rustc thread hung up before responding to sent lazy JIT request") | |
254 | } | |
255 | ||
256 | fn jit_fn(instance_ptr: *const Instance<'static>, trampoline_ptr: *const u8) -> *const u8 { | |
5869c6ff XL |
257 | rustc_middle::ty::tls::with(|tcx| { |
258 | // lift is used to ensure the correct lifetime for instance. | |
259 | let instance = tcx.lift(unsafe { *instance_ptr }).unwrap(); | |
260 | ||
17df50a5 XL |
261 | LAZY_JIT_STATE.with(|lazy_jit_state| { |
262 | let mut lazy_jit_state = lazy_jit_state.borrow_mut(); | |
263 | let lazy_jit_state = lazy_jit_state.as_mut().unwrap(); | |
264 | let jit_module = &mut lazy_jit_state.jit_module; | |
265 | let backend_config = lazy_jit_state.backend_config.clone(); | |
5869c6ff | 266 | |
17df50a5 | 267 | let name = tcx.symbol_name(instance).name; |
9c376795 FG |
268 | let sig = crate::abi::get_function_sig( |
269 | tcx, | |
270 | jit_module.target_config().default_call_conv, | |
271 | instance, | |
272 | ); | |
17df50a5 | 273 | let func_id = jit_module.declare_function(name, Linkage::Export, &sig).unwrap(); |
136023e0 XL |
274 | |
275 | let current_ptr = jit_module.read_got_entry(func_id); | |
276 | ||
277 | // If the function's GOT entry has already been updated to point at something other | |
278 | // than the shim trampoline, don't re-jit but just return the new pointer instead. | |
279 | // This does not need synchronization as this code is executed only by a sole rustc | |
280 | // thread. | |
281 | if current_ptr != trampoline_ptr { | |
282 | return current_ptr; | |
283 | } | |
284 | ||
6a06907d XL |
285 | jit_module.prepare_for_function_redefine(func_id).unwrap(); |
286 | ||
a2a8927a XL |
287 | let mut cx = crate::CodegenCx::new( |
288 | tcx, | |
289 | backend_config, | |
290 | jit_module.isa(), | |
291 | false, | |
292 | Symbol::intern("dummy_cgu_name"), | |
293 | ); | |
9ffffee4 | 294 | codegen_and_compile_fn(tcx, &mut cx, &mut Context::new(), jit_module, instance); |
6a06907d | 295 | |
17df50a5 | 296 | assert!(cx.global_asm.is_empty()); |
9c376795 | 297 | jit_module.finalize_definitions().unwrap(); |
17df50a5 | 298 | unsafe { cx.unwind_context.register_jit(&jit_module) }; |
5869c6ff XL |
299 | jit_module.get_finalized_function(func_id) |
300 | }) | |
301 | }) | |
302 | } | |
303 | ||
2b03887a | 304 | fn dep_symbol_lookup_fn( |
136023e0 XL |
305 | sess: &Session, |
306 | crate_info: CrateInfo, | |
2b03887a | 307 | ) -> Box<dyn Fn(&str) -> Option<*const u8>> { |
29967ef6 XL |
308 | use rustc_middle::middle::dependency_format::Linkage; |
309 | ||
310 | let mut dylib_paths = Vec::new(); | |
311 | ||
136023e0 XL |
312 | let data = &crate_info |
313 | .dependency_formats | |
29967ef6 XL |
314 | .iter() |
315 | .find(|(crate_type, _data)| *crate_type == rustc_session::config::CrateType::Executable) | |
316 | .unwrap() | |
317 | .1; | |
353b0b11 FG |
318 | // `used_crates` is in reverse postorder in terms of dependencies. Reverse the order here to |
319 | // get a postorder which ensures that all dependencies of a dylib are loaded before the dylib | |
320 | // itself. This helps the dynamic linker to find dylibs not in the regular dynamic library | |
321 | // search path. | |
322 | for &cnum in crate_info.used_crates.iter().rev() { | |
29967ef6 XL |
323 | let src = &crate_info.used_crate_source[&cnum]; |
324 | match data[cnum.as_usize() - 1] { | |
325 | Linkage::NotLinked | Linkage::IncludedFromDylib => {} | |
326 | Linkage::Static => { | |
04454e1e FG |
327 | let name = crate_info.crate_name[&cnum]; |
328 | let mut err = sess.struct_err(&format!("Can't load static lib {}", name)); | |
29967ef6 XL |
329 | err.note("rustc_codegen_cranelift can only load dylibs in JIT mode."); |
330 | err.emit(); | |
331 | } | |
332 | Linkage::Dynamic => { | |
333 | dylib_paths.push(src.dylib.as_ref().unwrap().0.clone()); | |
334 | } | |
335 | } | |
336 | } | |
337 | ||
2b03887a FG |
338 | let imported_dylibs = Box::leak( |
339 | dylib_paths | |
340 | .into_iter() | |
341 | .map(|path| unsafe { libloading::Library::new(&path).unwrap() }) | |
342 | .collect::<Box<[_]>>(), | |
343 | ); | |
29967ef6 | 344 | |
136023e0 | 345 | sess.abort_if_errors(); |
29967ef6 | 346 | |
2b03887a FG |
347 | Box::new(move |sym_name| { |
348 | for dylib in &*imported_dylibs { | |
349 | if let Ok(sym) = unsafe { dylib.get::<*const u8>(sym_name.as_bytes()) } { | |
350 | return Some(*sym); | |
351 | } | |
352 | } | |
353 | None | |
354 | }) | |
29967ef6 | 355 | } |
5869c6ff | 356 | |
f2b60f7d FG |
357 | fn codegen_shim<'tcx>( |
358 | tcx: TyCtxt<'tcx>, | |
359 | cx: &mut CodegenCx, | |
360 | cached_context: &mut Context, | |
361 | module: &mut JITModule, | |
362 | inst: Instance<'tcx>, | |
363 | ) { | |
17df50a5 | 364 | let pointer_type = module.target_config().pointer_type(); |
5869c6ff | 365 | |
17df50a5 | 366 | let name = tcx.symbol_name(inst).name; |
9c376795 | 367 | let sig = crate::abi::get_function_sig(tcx, module.target_config().default_call_conv, inst); |
17df50a5 | 368 | let func_id = module.declare_function(name, Linkage::Export, &sig).unwrap(); |
5869c6ff XL |
369 | |
370 | let instance_ptr = Box::into_raw(Box::new(inst)); | |
371 | ||
17df50a5 | 372 | let jit_fn = module |
5869c6ff XL |
373 | .declare_function( |
374 | "__clif_jit_fn", | |
375 | Linkage::Import, | |
376 | &Signature { | |
17df50a5 | 377 | call_conv: module.target_config().default_call_conv, |
136023e0 | 378 | params: vec![AbiParam::new(pointer_type), AbiParam::new(pointer_type)], |
5869c6ff XL |
379 | returns: vec![AbiParam::new(pointer_type)], |
380 | }, | |
381 | ) | |
382 | .unwrap(); | |
383 | ||
f2b60f7d FG |
384 | let context = cached_context; |
385 | context.clear(); | |
386 | let trampoline = &mut context.func; | |
17df50a5 XL |
387 | trampoline.signature = sig.clone(); |
388 | ||
5869c6ff | 389 | let mut builder_ctx = FunctionBuilderContext::new(); |
17df50a5 | 390 | let mut trampoline_builder = FunctionBuilder::new(trampoline, &mut builder_ctx); |
5869c6ff | 391 | |
136023e0 | 392 | let trampoline_fn = module.declare_func_in_func(func_id, trampoline_builder.func); |
17df50a5 | 393 | let jit_fn = module.declare_func_in_func(jit_fn, trampoline_builder.func); |
5869c6ff XL |
394 | let sig_ref = trampoline_builder.func.import_signature(sig); |
395 | ||
396 | let entry_block = trampoline_builder.create_block(); | |
397 | trampoline_builder.append_block_params_for_function_params(entry_block); | |
6a06907d | 398 | let fn_args = trampoline_builder.func.dfg.block_params(entry_block).to_vec(); |
5869c6ff XL |
399 | |
400 | trampoline_builder.switch_to_block(entry_block); | |
6a06907d | 401 | let instance_ptr = trampoline_builder.ins().iconst(pointer_type, instance_ptr as u64 as i64); |
136023e0 XL |
402 | let trampoline_ptr = trampoline_builder.ins().func_addr(pointer_type, trampoline_fn); |
403 | let jitted_fn = trampoline_builder.ins().call(jit_fn, &[instance_ptr, trampoline_ptr]); | |
5869c6ff | 404 | let jitted_fn = trampoline_builder.func.dfg.inst_results(jitted_fn)[0]; |
6a06907d | 405 | let call_inst = trampoline_builder.ins().call_indirect(sig_ref, jitted_fn, &fn_args); |
5869c6ff XL |
406 | let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec(); |
407 | trampoline_builder.ins().return_(&ret_vals); | |
408 | ||
f2b60f7d FG |
409 | module.define_function(func_id, context).unwrap(); |
410 | cx.unwind_context.add_function(func_id, context, module.isa()); | |
5869c6ff | 411 | } |