]>
Commit | Line | Data |
---|---|---|
29967ef6 XL |
1 | //! The JIT driver uses [`cranelift_simplejit`] to JIT execute programs without writing any object |
2 | //! files. | |
3 | ||
5869c6ff | 4 | use std::cell::RefCell; |
29967ef6 XL |
5 | use std::ffi::CString; |
6 | use std::os::raw::{c_char, c_int}; | |
7 | ||
cdc7bbd5 | 8 | use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink}; |
29967ef6 | 9 | use rustc_codegen_ssa::CrateInfo; |
5869c6ff | 10 | use rustc_middle::mir::mono::MonoItem; |
cdc7bbd5 | 11 | use rustc_session::config::EntryFnType; |
29967ef6 | 12 | |
5869c6ff | 13 | use cranelift_jit::{JITBuilder, JITModule}; |
29967ef6 | 14 | |
6a06907d | 15 | use crate::{prelude::*, BackendConfig}; |
5869c6ff | 16 | use crate::{CodegenCx, CodegenMode}; |
29967ef6 | 17 | |
5869c6ff | 18 | thread_local! { |
6a06907d | 19 | pub static BACKEND_CONFIG: RefCell<Option<BackendConfig>> = RefCell::new(None); |
5869c6ff XL |
20 | pub static CURRENT_MODULE: RefCell<Option<JITModule>> = RefCell::new(None); |
21 | } | |
22 | ||
6a06907d | 23 | pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { |
29967ef6 XL |
24 | if !tcx.sess.opts.output_types.should_codegen() { |
25 | tcx.sess.fatal("JIT mode doesn't work with `cargo check`."); | |
26 | } | |
27 | ||
29967ef6 XL |
28 | let imported_symbols = load_imported_symbols_for_jit(tcx); |
29 | ||
6a06907d XL |
30 | let mut jit_builder = |
31 | JITBuilder::with_isa(crate::build_isa(tcx.sess), cranelift_module::default_libcall_names()); | |
32 | jit_builder.hotswap(matches!(backend_config.codegen_mode, CodegenMode::JitLazy)); | |
cdc7bbd5 | 33 | crate::compiler_builtins::register_functions_for_jit(&mut jit_builder); |
29967ef6 | 34 | jit_builder.symbols(imported_symbols); |
5869c6ff | 35 | let mut jit_module = JITModule::new(jit_builder); |
29967ef6 XL |
36 | assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type()); |
37 | ||
29967ef6 XL |
38 | let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE); |
39 | let mono_items = cgus | |
40 | .iter() | |
41 | .map(|cgu| cgu.items_in_deterministic_order(tcx).into_iter()) | |
42 | .flatten() | |
43 | .collect::<FxHashMap<_, (_, _)>>() | |
44 | .into_iter() | |
45 | .collect::<Vec<(_, (_, _))>>(); | |
46 | ||
6a06907d | 47 | let mut cx = crate::CodegenCx::new(tcx, backend_config, &mut jit_module, false); |
29967ef6 | 48 | |
5869c6ff XL |
49 | super::time(tcx, "codegen mono items", || { |
50 | super::predefine_mono_items(&mut cx, &mono_items); | |
cdc7bbd5 | 51 | for (mono_item, _) in mono_items { |
5869c6ff | 52 | match mono_item { |
6a06907d | 53 | MonoItem::Fn(inst) => match backend_config.codegen_mode { |
5869c6ff XL |
54 | CodegenMode::Aot => unreachable!(), |
55 | CodegenMode::Jit => { | |
cdc7bbd5 | 56 | cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst)); |
5869c6ff XL |
57 | } |
58 | CodegenMode::JitLazy => codegen_shim(&mut cx, inst), | |
59 | }, | |
60 | MonoItem::Static(def_id) => { | |
61 | crate::constant::codegen_static(&mut cx.constants_cx, def_id); | |
62 | } | |
6a06907d XL |
63 | MonoItem::GlobalAsm(item_id) => { |
64 | let item = cx.tcx.hir().item(item_id); | |
65 | tcx.sess.span_fatal(item.span, "Global asm is not supported in JIT mode"); | |
5869c6ff | 66 | } |
fc512014 | 67 | } |
5869c6ff XL |
68 | } |
69 | }); | |
70 | ||
6a06907d | 71 | let (global_asm, _debug, mut unwind_context) = |
5869c6ff XL |
72 | tcx.sess.time("finalize CodegenCx", || cx.finalize()); |
73 | jit_module.finalize_definitions(); | |
74 | ||
29967ef6 | 75 | if !global_asm.is_empty() { |
5869c6ff | 76 | tcx.sess.fatal("Inline asm is not supported in JIT mode"); |
29967ef6 | 77 | } |
5869c6ff | 78 | |
29967ef6 XL |
79 | crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context); |
80 | ||
81 | tcx.sess.abort_if_errors(); | |
82 | ||
fc512014 | 83 | jit_module.finalize_definitions(); |
fc512014 | 84 | let _unwind_register_guard = unsafe { unwind_context.register_jit(&jit_module) }; |
29967ef6 | 85 | |
6a06907d XL |
86 | println!( |
87 | "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed" | |
88 | ); | |
29967ef6 | 89 | |
29967ef6 XL |
90 | let args = ::std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new()); |
91 | let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string()) | |
92 | .chain(args.split(' ')) | |
93 | .map(|arg| CString::new(arg).unwrap()) | |
94 | .collect::<Vec<_>>(); | |
95 | let mut argv = args.iter().map(|arg| arg.as_ptr()).collect::<Vec<_>>(); | |
96 | ||
97 | // Push a null pointer as a terminating argument. This is required by POSIX and | |
98 | // useful as some dynamic linkers use it as a marker to jump over. | |
99 | argv.push(std::ptr::null()); | |
100 | ||
6a06907d XL |
101 | BACKEND_CONFIG.with(|tls_backend_config| { |
102 | assert!(tls_backend_config.borrow_mut().replace(backend_config).is_none()) | |
103 | }); | |
5869c6ff | 104 | |
cdc7bbd5 XL |
105 | let (main_def_id, entry_ty) = tcx.entry_fn(LOCAL_CRATE).unwrap(); |
106 | let instance = Instance::mono(tcx, main_def_id).polymorphize(tcx); | |
107 | ||
108 | match entry_ty { | |
109 | EntryFnType::Main => { | |
110 | // FIXME set program arguments somehow | |
29967ef6 | 111 | |
cdc7bbd5 XL |
112 | let main_sig = Signature { |
113 | params: vec![], | |
114 | returns: vec![], | |
115 | call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)), | |
116 | }; | |
117 | let main_func_id = jit_module | |
118 | .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &main_sig) | |
119 | .unwrap(); | |
120 | let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id); | |
121 | ||
122 | CURRENT_MODULE.with(|current_module| { | |
123 | assert!(current_module.borrow_mut().replace(jit_module).is_none()) | |
124 | }); | |
125 | ||
126 | let f: extern "C" fn() = unsafe { ::std::mem::transmute(finalized_main) }; | |
127 | f(); | |
128 | std::process::exit(0); | |
129 | } | |
130 | EntryFnType::Start => { | |
131 | let start_sig = Signature { | |
132 | params: vec![ | |
133 | AbiParam::new(jit_module.target_config().pointer_type()), | |
134 | AbiParam::new(jit_module.target_config().pointer_type()), | |
135 | ], | |
136 | returns: vec![AbiParam::new( | |
137 | jit_module.target_config().pointer_type(), /*isize*/ | |
138 | )], | |
139 | call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)), | |
140 | }; | |
141 | let start_func_id = jit_module | |
142 | .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &start_sig) | |
143 | .unwrap(); | |
144 | let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id); | |
145 | ||
146 | CURRENT_MODULE.with(|current_module| { | |
147 | assert!(current_module.borrow_mut().replace(jit_module).is_none()) | |
148 | }); | |
149 | ||
150 | let f: extern "C" fn(c_int, *const *const c_char) -> c_int = | |
151 | unsafe { ::std::mem::transmute(finalized_start) }; | |
152 | let ret = f(args.len() as c_int, argv.as_ptr()); | |
153 | std::process::exit(ret); | |
154 | } | |
155 | } | |
29967ef6 XL |
156 | } |
157 | ||
5869c6ff XL |
158 | #[no_mangle] |
159 | extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 { | |
160 | rustc_middle::ty::tls::with(|tcx| { | |
161 | // lift is used to ensure the correct lifetime for instance. | |
162 | let instance = tcx.lift(unsafe { *instance_ptr }).unwrap(); | |
163 | ||
164 | CURRENT_MODULE.with(|jit_module| { | |
165 | let mut jit_module = jit_module.borrow_mut(); | |
166 | let jit_module = jit_module.as_mut().unwrap(); | |
6a06907d XL |
167 | let backend_config = |
168 | BACKEND_CONFIG.with(|backend_config| backend_config.borrow().clone().unwrap()); | |
5869c6ff XL |
169 | |
170 | let name = tcx.symbol_name(instance).name.to_string(); | |
6a06907d XL |
171 | let sig = crate::abi::get_function_sig(tcx, jit_module.isa().triple(), instance); |
172 | let func_id = jit_module.declare_function(&name, Linkage::Export, &sig).unwrap(); | |
173 | jit_module.prepare_for_function_redefine(func_id).unwrap(); | |
174 | ||
175 | let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module, false); | |
cdc7bbd5 | 176 | tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, instance)); |
6a06907d XL |
177 | |
178 | let (global_asm, _debug_context, unwind_context) = cx.finalize(); | |
5869c6ff XL |
179 | assert!(global_asm.is_empty()); |
180 | jit_module.finalize_definitions(); | |
181 | std::mem::forget(unsafe { unwind_context.register_jit(&jit_module) }); | |
182 | jit_module.get_finalized_function(func_id) | |
183 | }) | |
184 | }) | |
185 | } | |
186 | ||
29967ef6 XL |
187 | fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> { |
188 | use rustc_middle::middle::dependency_format::Linkage; | |
189 | ||
190 | let mut dylib_paths = Vec::new(); | |
191 | ||
192 | let crate_info = CrateInfo::new(tcx); | |
193 | let formats = tcx.dependency_formats(LOCAL_CRATE); | |
194 | let data = &formats | |
195 | .iter() | |
196 | .find(|(crate_type, _data)| *crate_type == rustc_session::config::CrateType::Executable) | |
197 | .unwrap() | |
198 | .1; | |
199 | for &(cnum, _) in &crate_info.used_crates_dynamic { | |
200 | let src = &crate_info.used_crate_source[&cnum]; | |
201 | match data[cnum.as_usize() - 1] { | |
202 | Linkage::NotLinked | Linkage::IncludedFromDylib => {} | |
203 | Linkage::Static => { | |
204 | let name = tcx.crate_name(cnum); | |
6a06907d XL |
205 | let mut err = |
206 | tcx.sess.struct_err(&format!("Can't load static lib {}", name.as_str())); | |
29967ef6 XL |
207 | err.note("rustc_codegen_cranelift can only load dylibs in JIT mode."); |
208 | err.emit(); | |
209 | } | |
210 | Linkage::Dynamic => { | |
211 | dylib_paths.push(src.dylib.as_ref().unwrap().0.clone()); | |
212 | } | |
213 | } | |
214 | } | |
215 | ||
216 | let mut imported_symbols = Vec::new(); | |
217 | for path in dylib_paths { | |
fc512014 | 218 | use object::{Object, ObjectSymbol}; |
29967ef6 XL |
219 | let lib = libloading::Library::new(&path).unwrap(); |
220 | let obj = std::fs::read(path).unwrap(); | |
221 | let obj = object::File::parse(&obj).unwrap(); | |
fc512014 | 222 | imported_symbols.extend(obj.dynamic_symbols().filter_map(|symbol| { |
29967ef6 XL |
223 | let name = symbol.name().unwrap().to_string(); |
224 | if name.is_empty() || !symbol.is_global() || symbol.is_undefined() { | |
225 | return None; | |
226 | } | |
6a06907d XL |
227 | if name.starts_with("rust_metadata_") { |
228 | // The metadata is part of a section that is not loaded by the dynamic linker in | |
229 | // case of cg_llvm. | |
230 | return None; | |
231 | } | |
29967ef6 XL |
232 | let dlsym_name = if cfg!(target_os = "macos") { |
233 | // On macOS `dlsym` expects the name without leading `_`. | |
234 | assert!(name.starts_with('_'), "{:?}", name); | |
235 | &name[1..] | |
236 | } else { | |
237 | &name | |
238 | }; | |
239 | let symbol: libloading::Symbol<'_, *const u8> = | |
240 | unsafe { lib.get(dlsym_name.as_bytes()) }.unwrap(); | |
241 | Some((name, *symbol)) | |
242 | })); | |
243 | std::mem::forget(lib) | |
244 | } | |
245 | ||
246 | tcx.sess.abort_if_errors(); | |
247 | ||
248 | imported_symbols | |
249 | } | |
5869c6ff | 250 | |
cdc7bbd5 | 251 | fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) { |
5869c6ff XL |
252 | let tcx = cx.tcx; |
253 | ||
254 | let pointer_type = cx.module.target_config().pointer_type(); | |
255 | ||
256 | let name = tcx.symbol_name(inst).name.to_string(); | |
257 | let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), inst); | |
6a06907d | 258 | let func_id = cx.module.declare_function(&name, Linkage::Export, &sig).unwrap(); |
5869c6ff XL |
259 | |
260 | let instance_ptr = Box::into_raw(Box::new(inst)); | |
261 | ||
262 | let jit_fn = cx | |
263 | .module | |
264 | .declare_function( | |
265 | "__clif_jit_fn", | |
266 | Linkage::Import, | |
267 | &Signature { | |
268 | call_conv: cx.module.target_config().default_call_conv, | |
269 | params: vec![AbiParam::new(pointer_type)], | |
270 | returns: vec![AbiParam::new(pointer_type)], | |
271 | }, | |
272 | ) | |
273 | .unwrap(); | |
274 | ||
275 | let mut trampoline = Function::with_name_signature(ExternalName::default(), sig.clone()); | |
276 | let mut builder_ctx = FunctionBuilderContext::new(); | |
277 | let mut trampoline_builder = FunctionBuilder::new(&mut trampoline, &mut builder_ctx); | |
278 | ||
6a06907d | 279 | let jit_fn = cx.module.declare_func_in_func(jit_fn, trampoline_builder.func); |
5869c6ff XL |
280 | let sig_ref = trampoline_builder.func.import_signature(sig); |
281 | ||
282 | let entry_block = trampoline_builder.create_block(); | |
283 | trampoline_builder.append_block_params_for_function_params(entry_block); | |
6a06907d | 284 | let fn_args = trampoline_builder.func.dfg.block_params(entry_block).to_vec(); |
5869c6ff XL |
285 | |
286 | trampoline_builder.switch_to_block(entry_block); | |
6a06907d | 287 | let instance_ptr = trampoline_builder.ins().iconst(pointer_type, instance_ptr as u64 as i64); |
5869c6ff XL |
288 | let jitted_fn = trampoline_builder.ins().call(jit_fn, &[instance_ptr]); |
289 | let jitted_fn = trampoline_builder.func.dfg.inst_results(jitted_fn)[0]; | |
6a06907d | 290 | let call_inst = trampoline_builder.ins().call_indirect(sig_ref, jitted_fn, &fn_args); |
5869c6ff XL |
291 | let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec(); |
292 | trampoline_builder.ins().return_(&ret_vals); | |
293 | ||
294 | cx.module | |
295 | .define_function( | |
296 | func_id, | |
297 | &mut Context::for_function(trampoline), | |
cdc7bbd5 XL |
298 | &mut NullTrapSink {}, |
299 | &mut NullStackMapSink {}, | |
5869c6ff XL |
300 | ) |
301 | .unwrap(); | |
302 | } |