]>
Commit | Line | Data |
---|---|---|
29967ef6 XL |
1 | //! The JIT driver uses [`cranelift_simplejit`] to JIT execute programs without writing any object |
2 | //! files. | |
3 | ||
5869c6ff | 4 | use std::cell::RefCell; |
29967ef6 XL |
5 | use std::ffi::CString; |
6 | use std::os::raw::{c_char, c_int}; | |
7 | ||
8 | use rustc_codegen_ssa::CrateInfo; | |
5869c6ff | 9 | use rustc_middle::mir::mono::MonoItem; |
29967ef6 | 10 | |
5869c6ff | 11 | use cranelift_jit::{JITBuilder, JITModule}; |
29967ef6 | 12 | |
6a06907d | 13 | use crate::{prelude::*, BackendConfig}; |
5869c6ff | 14 | use crate::{CodegenCx, CodegenMode}; |
29967ef6 | 15 | |
5869c6ff | 16 | thread_local! { |
6a06907d | 17 | pub static BACKEND_CONFIG: RefCell<Option<BackendConfig>> = RefCell::new(None); |
5869c6ff XL |
18 | pub static CURRENT_MODULE: RefCell<Option<JITModule>> = RefCell::new(None); |
19 | } | |
20 | ||
6a06907d | 21 | pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { |
29967ef6 XL |
22 | if !tcx.sess.opts.output_types.should_codegen() { |
23 | tcx.sess.fatal("JIT mode doesn't work with `cargo check`."); | |
24 | } | |
25 | ||
29967ef6 XL |
26 | let imported_symbols = load_imported_symbols_for_jit(tcx); |
27 | ||
6a06907d XL |
28 | let mut jit_builder = |
29 | JITBuilder::with_isa(crate::build_isa(tcx.sess), cranelift_module::default_libcall_names()); | |
30 | jit_builder.hotswap(matches!(backend_config.codegen_mode, CodegenMode::JitLazy)); | |
29967ef6 | 31 | jit_builder.symbols(imported_symbols); |
5869c6ff | 32 | let mut jit_module = JITModule::new(jit_builder); |
29967ef6 XL |
33 | assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type()); |
34 | ||
35 | let sig = Signature { | |
36 | params: vec![ | |
37 | AbiParam::new(jit_module.target_config().pointer_type()), | |
38 | AbiParam::new(jit_module.target_config().pointer_type()), | |
39 | ], | |
6a06907d | 40 | returns: vec![AbiParam::new(jit_module.target_config().pointer_type() /*isize*/)], |
29967ef6 XL |
41 | call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)), |
42 | }; | |
6a06907d | 43 | let main_func_id = jit_module.declare_function("main", Linkage::Import, &sig).unwrap(); |
29967ef6 XL |
44 | |
45 | let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE); | |
46 | let mono_items = cgus | |
47 | .iter() | |
48 | .map(|cgu| cgu.items_in_deterministic_order(tcx).into_iter()) | |
49 | .flatten() | |
50 | .collect::<FxHashMap<_, (_, _)>>() | |
51 | .into_iter() | |
52 | .collect::<Vec<(_, (_, _))>>(); | |
53 | ||
6a06907d | 54 | let mut cx = crate::CodegenCx::new(tcx, backend_config, &mut jit_module, false); |
29967ef6 | 55 | |
5869c6ff XL |
56 | super::time(tcx, "codegen mono items", || { |
57 | super::predefine_mono_items(&mut cx, &mono_items); | |
58 | for (mono_item, (linkage, visibility)) in mono_items { | |
59 | let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); | |
60 | match mono_item { | |
6a06907d | 61 | MonoItem::Fn(inst) => match backend_config.codegen_mode { |
5869c6ff XL |
62 | CodegenMode::Aot => unreachable!(), |
63 | CodegenMode::Jit => { | |
6a06907d XL |
64 | cx.tcx |
65 | .sess | |
66 | .time("codegen fn", || crate::base::codegen_fn(&mut cx, inst, linkage)); | |
5869c6ff XL |
67 | } |
68 | CodegenMode::JitLazy => codegen_shim(&mut cx, inst), | |
69 | }, | |
70 | MonoItem::Static(def_id) => { | |
71 | crate::constant::codegen_static(&mut cx.constants_cx, def_id); | |
72 | } | |
6a06907d XL |
73 | MonoItem::GlobalAsm(item_id) => { |
74 | let item = cx.tcx.hir().item(item_id); | |
75 | tcx.sess.span_fatal(item.span, "Global asm is not supported in JIT mode"); | |
5869c6ff | 76 | } |
fc512014 | 77 | } |
5869c6ff XL |
78 | } |
79 | }); | |
80 | ||
6a06907d | 81 | let (global_asm, _debug, mut unwind_context) = |
5869c6ff XL |
82 | tcx.sess.time("finalize CodegenCx", || cx.finalize()); |
83 | jit_module.finalize_definitions(); | |
84 | ||
29967ef6 | 85 | if !global_asm.is_empty() { |
5869c6ff | 86 | tcx.sess.fatal("Inline asm is not supported in JIT mode"); |
29967ef6 | 87 | } |
5869c6ff | 88 | |
6a06907d | 89 | crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context); |
29967ef6 XL |
90 | crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context); |
91 | ||
92 | tcx.sess.abort_if_errors(); | |
93 | ||
fc512014 | 94 | jit_module.finalize_definitions(); |
29967ef6 | 95 | |
fc512014 | 96 | let _unwind_register_guard = unsafe { unwind_context.register_jit(&jit_module) }; |
29967ef6 | 97 | |
fc512014 | 98 | let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id); |
29967ef6 | 99 | |
6a06907d XL |
100 | println!( |
101 | "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed" | |
102 | ); | |
29967ef6 XL |
103 | |
104 | let f: extern "C" fn(c_int, *const *const c_char) -> c_int = | |
105 | unsafe { ::std::mem::transmute(finalized_main) }; | |
106 | ||
107 | let args = ::std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new()); | |
108 | let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string()) | |
109 | .chain(args.split(' ')) | |
110 | .map(|arg| CString::new(arg).unwrap()) | |
111 | .collect::<Vec<_>>(); | |
112 | let mut argv = args.iter().map(|arg| arg.as_ptr()).collect::<Vec<_>>(); | |
113 | ||
114 | // Push a null pointer as a terminating argument. This is required by POSIX and | |
115 | // useful as some dynamic linkers use it as a marker to jump over. | |
116 | argv.push(std::ptr::null()); | |
117 | ||
6a06907d XL |
118 | BACKEND_CONFIG.with(|tls_backend_config| { |
119 | assert!(tls_backend_config.borrow_mut().replace(backend_config).is_none()) | |
120 | }); | |
5869c6ff XL |
121 | CURRENT_MODULE |
122 | .with(|current_module| assert!(current_module.borrow_mut().replace(jit_module).is_none())); | |
123 | ||
29967ef6 XL |
124 | let ret = f(args.len() as c_int, argv.as_ptr()); |
125 | ||
126 | std::process::exit(ret); | |
127 | } | |
128 | ||
5869c6ff XL |
129 | #[no_mangle] |
130 | extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 { | |
131 | rustc_middle::ty::tls::with(|tcx| { | |
132 | // lift is used to ensure the correct lifetime for instance. | |
133 | let instance = tcx.lift(unsafe { *instance_ptr }).unwrap(); | |
134 | ||
135 | CURRENT_MODULE.with(|jit_module| { | |
136 | let mut jit_module = jit_module.borrow_mut(); | |
137 | let jit_module = jit_module.as_mut().unwrap(); | |
6a06907d XL |
138 | let backend_config = |
139 | BACKEND_CONFIG.with(|backend_config| backend_config.borrow().clone().unwrap()); | |
5869c6ff XL |
140 | |
141 | let name = tcx.symbol_name(instance).name.to_string(); | |
6a06907d XL |
142 | let sig = crate::abi::get_function_sig(tcx, jit_module.isa().triple(), instance); |
143 | let func_id = jit_module.declare_function(&name, Linkage::Export, &sig).unwrap(); | |
144 | jit_module.prepare_for_function_redefine(func_id).unwrap(); | |
145 | ||
146 | let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module, false); | |
147 | tcx.sess | |
148 | .time("codegen fn", || crate::base::codegen_fn(&mut cx, instance, Linkage::Export)); | |
149 | ||
150 | let (global_asm, _debug_context, unwind_context) = cx.finalize(); | |
5869c6ff XL |
151 | assert!(global_asm.is_empty()); |
152 | jit_module.finalize_definitions(); | |
153 | std::mem::forget(unsafe { unwind_context.register_jit(&jit_module) }); | |
154 | jit_module.get_finalized_function(func_id) | |
155 | }) | |
156 | }) | |
157 | } | |
158 | ||
29967ef6 XL |
159 | fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> { |
160 | use rustc_middle::middle::dependency_format::Linkage; | |
161 | ||
162 | let mut dylib_paths = Vec::new(); | |
163 | ||
164 | let crate_info = CrateInfo::new(tcx); | |
165 | let formats = tcx.dependency_formats(LOCAL_CRATE); | |
166 | let data = &formats | |
167 | .iter() | |
168 | .find(|(crate_type, _data)| *crate_type == rustc_session::config::CrateType::Executable) | |
169 | .unwrap() | |
170 | .1; | |
171 | for &(cnum, _) in &crate_info.used_crates_dynamic { | |
172 | let src = &crate_info.used_crate_source[&cnum]; | |
173 | match data[cnum.as_usize() - 1] { | |
174 | Linkage::NotLinked | Linkage::IncludedFromDylib => {} | |
175 | Linkage::Static => { | |
176 | let name = tcx.crate_name(cnum); | |
6a06907d XL |
177 | let mut err = |
178 | tcx.sess.struct_err(&format!("Can't load static lib {}", name.as_str())); | |
29967ef6 XL |
179 | err.note("rustc_codegen_cranelift can only load dylibs in JIT mode."); |
180 | err.emit(); | |
181 | } | |
182 | Linkage::Dynamic => { | |
183 | dylib_paths.push(src.dylib.as_ref().unwrap().0.clone()); | |
184 | } | |
185 | } | |
186 | } | |
187 | ||
188 | let mut imported_symbols = Vec::new(); | |
189 | for path in dylib_paths { | |
fc512014 | 190 | use object::{Object, ObjectSymbol}; |
29967ef6 XL |
191 | let lib = libloading::Library::new(&path).unwrap(); |
192 | let obj = std::fs::read(path).unwrap(); | |
193 | let obj = object::File::parse(&obj).unwrap(); | |
fc512014 | 194 | imported_symbols.extend(obj.dynamic_symbols().filter_map(|symbol| { |
29967ef6 XL |
195 | let name = symbol.name().unwrap().to_string(); |
196 | if name.is_empty() || !symbol.is_global() || symbol.is_undefined() { | |
197 | return None; | |
198 | } | |
6a06907d XL |
199 | if name.starts_with("rust_metadata_") { |
200 | // The metadata is part of a section that is not loaded by the dynamic linker in | |
201 | // case of cg_llvm. | |
202 | return None; | |
203 | } | |
29967ef6 XL |
204 | let dlsym_name = if cfg!(target_os = "macos") { |
205 | // On macOS `dlsym` expects the name without leading `_`. | |
206 | assert!(name.starts_with('_'), "{:?}", name); | |
207 | &name[1..] | |
208 | } else { | |
209 | &name | |
210 | }; | |
211 | let symbol: libloading::Symbol<'_, *const u8> = | |
212 | unsafe { lib.get(dlsym_name.as_bytes()) }.unwrap(); | |
213 | Some((name, *symbol)) | |
214 | })); | |
215 | std::mem::forget(lib) | |
216 | } | |
217 | ||
218 | tcx.sess.abort_if_errors(); | |
219 | ||
220 | imported_symbols | |
221 | } | |
5869c6ff | 222 | |
6a06907d | 223 | pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) { |
5869c6ff XL |
224 | let tcx = cx.tcx; |
225 | ||
226 | let pointer_type = cx.module.target_config().pointer_type(); | |
227 | ||
228 | let name = tcx.symbol_name(inst).name.to_string(); | |
229 | let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), inst); | |
6a06907d | 230 | let func_id = cx.module.declare_function(&name, Linkage::Export, &sig).unwrap(); |
5869c6ff XL |
231 | |
232 | let instance_ptr = Box::into_raw(Box::new(inst)); | |
233 | ||
234 | let jit_fn = cx | |
235 | .module | |
236 | .declare_function( | |
237 | "__clif_jit_fn", | |
238 | Linkage::Import, | |
239 | &Signature { | |
240 | call_conv: cx.module.target_config().default_call_conv, | |
241 | params: vec![AbiParam::new(pointer_type)], | |
242 | returns: vec![AbiParam::new(pointer_type)], | |
243 | }, | |
244 | ) | |
245 | .unwrap(); | |
246 | ||
247 | let mut trampoline = Function::with_name_signature(ExternalName::default(), sig.clone()); | |
248 | let mut builder_ctx = FunctionBuilderContext::new(); | |
249 | let mut trampoline_builder = FunctionBuilder::new(&mut trampoline, &mut builder_ctx); | |
250 | ||
6a06907d | 251 | let jit_fn = cx.module.declare_func_in_func(jit_fn, trampoline_builder.func); |
5869c6ff XL |
252 | let sig_ref = trampoline_builder.func.import_signature(sig); |
253 | ||
254 | let entry_block = trampoline_builder.create_block(); | |
255 | trampoline_builder.append_block_params_for_function_params(entry_block); | |
6a06907d | 256 | let fn_args = trampoline_builder.func.dfg.block_params(entry_block).to_vec(); |
5869c6ff XL |
257 | |
258 | trampoline_builder.switch_to_block(entry_block); | |
6a06907d | 259 | let instance_ptr = trampoline_builder.ins().iconst(pointer_type, instance_ptr as u64 as i64); |
5869c6ff XL |
260 | let jitted_fn = trampoline_builder.ins().call(jit_fn, &[instance_ptr]); |
261 | let jitted_fn = trampoline_builder.func.dfg.inst_results(jitted_fn)[0]; | |
6a06907d | 262 | let call_inst = trampoline_builder.ins().call_indirect(sig_ref, jitted_fn, &fn_args); |
5869c6ff XL |
263 | let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec(); |
264 | trampoline_builder.ins().return_(&ret_vals); | |
265 | ||
266 | cx.module | |
267 | .define_function( | |
268 | func_id, | |
269 | &mut Context::for_function(trampoline), | |
270 | &mut cranelift_codegen::binemit::NullTrapSink {}, | |
271 | ) | |
272 | .unwrap(); | |
273 | } |