]>
Commit | Line | Data |
---|---|---|
29967ef6 XL |
1 | //! The JIT driver uses [`cranelift_simplejit`] to JIT execute programs without writing any object |
2 | //! files. | |
3 | ||
5869c6ff | 4 | use std::cell::RefCell; |
29967ef6 XL |
5 | use std::ffi::CString; |
6 | use std::os::raw::{c_char, c_int}; | |
7 | ||
8 | use rustc_codegen_ssa::CrateInfo; | |
5869c6ff | 9 | use rustc_middle::mir::mono::MonoItem; |
29967ef6 | 10 | |
5869c6ff | 11 | use cranelift_jit::{JITBuilder, JITModule}; |
29967ef6 XL |
12 | |
13 | use crate::prelude::*; | |
5869c6ff | 14 | use crate::{CodegenCx, CodegenMode}; |
29967ef6 | 15 | |
5869c6ff XL |
16 | thread_local! { |
17 | pub static CURRENT_MODULE: RefCell<Option<JITModule>> = RefCell::new(None); | |
18 | } | |
19 | ||
20 | pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! { | |
29967ef6 XL |
21 | if !tcx.sess.opts.output_types.should_codegen() { |
22 | tcx.sess.fatal("JIT mode doesn't work with `cargo check`."); | |
23 | } | |
24 | ||
25 | #[cfg(unix)] | |
26 | unsafe { | |
27 | // When not using our custom driver rustc will open us without the RTLD_GLOBAL flag, so | |
28 | // __cg_clif_global_atomic_mutex will not be exported. We fix this by opening ourself again | |
29 | // as global. | |
30 | // FIXME remove once atomic_shim is gone | |
31 | ||
32 | let mut dl_info: libc::Dl_info = std::mem::zeroed(); | |
33 | assert_ne!( | |
34 | libc::dladdr(run_jit as *const libc::c_void, &mut dl_info), | |
35 | 0 | |
36 | ); | |
37 | assert_ne!( | |
38 | libc::dlopen(dl_info.dli_fname, libc::RTLD_NOW | libc::RTLD_GLOBAL), | |
39 | std::ptr::null_mut(), | |
40 | ); | |
41 | } | |
42 | ||
43 | let imported_symbols = load_imported_symbols_for_jit(tcx); | |
44 | ||
5869c6ff XL |
45 | let mut jit_builder = JITBuilder::with_isa( |
46 | crate::build_isa(tcx.sess), | |
29967ef6 XL |
47 | cranelift_module::default_libcall_names(), |
48 | ); | |
5869c6ff | 49 | jit_builder.hotswap(matches!(codegen_mode, CodegenMode::JitLazy)); |
29967ef6 | 50 | jit_builder.symbols(imported_symbols); |
5869c6ff | 51 | let mut jit_module = JITModule::new(jit_builder); |
29967ef6 XL |
52 | assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type()); |
53 | ||
54 | let sig = Signature { | |
55 | params: vec![ | |
56 | AbiParam::new(jit_module.target_config().pointer_type()), | |
57 | AbiParam::new(jit_module.target_config().pointer_type()), | |
58 | ], | |
59 | returns: vec![AbiParam::new( | |
60 | jit_module.target_config().pointer_type(), /*isize*/ | |
61 | )], | |
62 | call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)), | |
63 | }; | |
64 | let main_func_id = jit_module | |
65 | .declare_function("main", Linkage::Import, &sig) | |
66 | .unwrap(); | |
67 | ||
68 | let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE); | |
69 | let mono_items = cgus | |
70 | .iter() | |
71 | .map(|cgu| cgu.items_in_deterministic_order(tcx).into_iter()) | |
72 | .flatten() | |
73 | .collect::<FxHashMap<_, (_, _)>>() | |
74 | .into_iter() | |
75 | .collect::<Vec<(_, (_, _))>>(); | |
76 | ||
5869c6ff | 77 | let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false); |
29967ef6 | 78 | |
5869c6ff XL |
79 | super::time(tcx, "codegen mono items", || { |
80 | super::predefine_mono_items(&mut cx, &mono_items); | |
81 | for (mono_item, (linkage, visibility)) in mono_items { | |
82 | let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); | |
83 | match mono_item { | |
84 | MonoItem::Fn(inst) => match codegen_mode { | |
85 | CodegenMode::Aot => unreachable!(), | |
86 | CodegenMode::Jit => { | |
87 | cx.tcx.sess.time("codegen fn", || { | |
88 | crate::base::codegen_fn(&mut cx, inst, linkage) | |
89 | }); | |
90 | } | |
91 | CodegenMode::JitLazy => codegen_shim(&mut cx, inst), | |
92 | }, | |
93 | MonoItem::Static(def_id) => { | |
94 | crate::constant::codegen_static(&mut cx.constants_cx, def_id); | |
95 | } | |
96 | MonoItem::GlobalAsm(hir_id) => { | |
97 | let item = cx.tcx.hir().expect_item(hir_id); | |
98 | tcx.sess | |
99 | .span_fatal(item.span, "Global asm is not supported in JIT mode"); | |
100 | } | |
fc512014 | 101 | } |
5869c6ff XL |
102 | } |
103 | }); | |
104 | ||
105 | let (mut jit_module, global_asm, _debug, mut unwind_context) = | |
106 | tcx.sess.time("finalize CodegenCx", || cx.finalize()); | |
107 | jit_module.finalize_definitions(); | |
108 | ||
29967ef6 | 109 | if !global_asm.is_empty() { |
5869c6ff | 110 | tcx.sess.fatal("Inline asm is not supported in JIT mode"); |
29967ef6 | 111 | } |
5869c6ff | 112 | |
29967ef6 XL |
113 | crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context, true); |
114 | crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context); | |
115 | ||
116 | tcx.sess.abort_if_errors(); | |
117 | ||
fc512014 | 118 | jit_module.finalize_definitions(); |
29967ef6 | 119 | |
fc512014 | 120 | let _unwind_register_guard = unsafe { unwind_context.register_jit(&jit_module) }; |
29967ef6 | 121 | |
fc512014 | 122 | let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id); |
29967ef6 | 123 | |
5869c6ff | 124 | println!("Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed"); |
29967ef6 XL |
125 | |
126 | let f: extern "C" fn(c_int, *const *const c_char) -> c_int = | |
127 | unsafe { ::std::mem::transmute(finalized_main) }; | |
128 | ||
129 | let args = ::std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new()); | |
130 | let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string()) | |
131 | .chain(args.split(' ')) | |
132 | .map(|arg| CString::new(arg).unwrap()) | |
133 | .collect::<Vec<_>>(); | |
134 | let mut argv = args.iter().map(|arg| arg.as_ptr()).collect::<Vec<_>>(); | |
135 | ||
136 | // Push a null pointer as a terminating argument. This is required by POSIX and | |
137 | // useful as some dynamic linkers use it as a marker to jump over. | |
138 | argv.push(std::ptr::null()); | |
139 | ||
5869c6ff XL |
140 | CURRENT_MODULE |
141 | .with(|current_module| assert!(current_module.borrow_mut().replace(jit_module).is_none())); | |
142 | ||
29967ef6 XL |
143 | let ret = f(args.len() as c_int, argv.as_ptr()); |
144 | ||
145 | std::process::exit(ret); | |
146 | } | |
147 | ||
5869c6ff XL |
148 | #[no_mangle] |
149 | extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 { | |
150 | rustc_middle::ty::tls::with(|tcx| { | |
151 | // lift is used to ensure the correct lifetime for instance. | |
152 | let instance = tcx.lift(unsafe { *instance_ptr }).unwrap(); | |
153 | ||
154 | CURRENT_MODULE.with(|jit_module| { | |
155 | let mut jit_module = jit_module.borrow_mut(); | |
156 | let jit_module = jit_module.as_mut().unwrap(); | |
157 | let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false); | |
158 | ||
159 | let name = tcx.symbol_name(instance).name.to_string(); | |
160 | let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), instance); | |
161 | let func_id = cx | |
162 | .module | |
163 | .declare_function(&name, Linkage::Export, &sig) | |
164 | .unwrap(); | |
165 | cx.module.prepare_for_function_redefine(func_id).unwrap(); | |
166 | ||
167 | tcx.sess.time("codegen fn", || { | |
168 | crate::base::codegen_fn(&mut cx, instance, Linkage::Export) | |
169 | }); | |
170 | ||
171 | let (jit_module, global_asm, _debug_context, unwind_context) = cx.finalize(); | |
172 | assert!(global_asm.is_empty()); | |
173 | jit_module.finalize_definitions(); | |
174 | std::mem::forget(unsafe { unwind_context.register_jit(&jit_module) }); | |
175 | jit_module.get_finalized_function(func_id) | |
176 | }) | |
177 | }) | |
178 | } | |
179 | ||
29967ef6 XL |
180 | fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> { |
181 | use rustc_middle::middle::dependency_format::Linkage; | |
182 | ||
183 | let mut dylib_paths = Vec::new(); | |
184 | ||
185 | let crate_info = CrateInfo::new(tcx); | |
186 | let formats = tcx.dependency_formats(LOCAL_CRATE); | |
187 | let data = &formats | |
188 | .iter() | |
189 | .find(|(crate_type, _data)| *crate_type == rustc_session::config::CrateType::Executable) | |
190 | .unwrap() | |
191 | .1; | |
192 | for &(cnum, _) in &crate_info.used_crates_dynamic { | |
193 | let src = &crate_info.used_crate_source[&cnum]; | |
194 | match data[cnum.as_usize() - 1] { | |
195 | Linkage::NotLinked | Linkage::IncludedFromDylib => {} | |
196 | Linkage::Static => { | |
197 | let name = tcx.crate_name(cnum); | |
198 | let mut err = tcx | |
199 | .sess | |
200 | .struct_err(&format!("Can't load static lib {}", name.as_str())); | |
201 | err.note("rustc_codegen_cranelift can only load dylibs in JIT mode."); | |
202 | err.emit(); | |
203 | } | |
204 | Linkage::Dynamic => { | |
205 | dylib_paths.push(src.dylib.as_ref().unwrap().0.clone()); | |
206 | } | |
207 | } | |
208 | } | |
209 | ||
210 | let mut imported_symbols = Vec::new(); | |
211 | for path in dylib_paths { | |
fc512014 | 212 | use object::{Object, ObjectSymbol}; |
29967ef6 XL |
213 | let lib = libloading::Library::new(&path).unwrap(); |
214 | let obj = std::fs::read(path).unwrap(); | |
215 | let obj = object::File::parse(&obj).unwrap(); | |
fc512014 | 216 | imported_symbols.extend(obj.dynamic_symbols().filter_map(|symbol| { |
29967ef6 XL |
217 | let name = symbol.name().unwrap().to_string(); |
218 | if name.is_empty() || !symbol.is_global() || symbol.is_undefined() { | |
219 | return None; | |
220 | } | |
221 | let dlsym_name = if cfg!(target_os = "macos") { | |
222 | // On macOS `dlsym` expects the name without leading `_`. | |
223 | assert!(name.starts_with('_'), "{:?}", name); | |
224 | &name[1..] | |
225 | } else { | |
226 | &name | |
227 | }; | |
228 | let symbol: libloading::Symbol<'_, *const u8> = | |
229 | unsafe { lib.get(dlsym_name.as_bytes()) }.unwrap(); | |
230 | Some((name, *symbol)) | |
231 | })); | |
232 | std::mem::forget(lib) | |
233 | } | |
234 | ||
235 | tcx.sess.abort_if_errors(); | |
236 | ||
237 | imported_symbols | |
238 | } | |
5869c6ff XL |
239 | |
240 | pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx, impl Module>, inst: Instance<'tcx>) { | |
241 | let tcx = cx.tcx; | |
242 | ||
243 | let pointer_type = cx.module.target_config().pointer_type(); | |
244 | ||
245 | let name = tcx.symbol_name(inst).name.to_string(); | |
246 | let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), inst); | |
247 | let func_id = cx | |
248 | .module | |
249 | .declare_function(&name, Linkage::Export, &sig) | |
250 | .unwrap(); | |
251 | ||
252 | let instance_ptr = Box::into_raw(Box::new(inst)); | |
253 | ||
254 | let jit_fn = cx | |
255 | .module | |
256 | .declare_function( | |
257 | "__clif_jit_fn", | |
258 | Linkage::Import, | |
259 | &Signature { | |
260 | call_conv: cx.module.target_config().default_call_conv, | |
261 | params: vec![AbiParam::new(pointer_type)], | |
262 | returns: vec![AbiParam::new(pointer_type)], | |
263 | }, | |
264 | ) | |
265 | .unwrap(); | |
266 | ||
267 | let mut trampoline = Function::with_name_signature(ExternalName::default(), sig.clone()); | |
268 | let mut builder_ctx = FunctionBuilderContext::new(); | |
269 | let mut trampoline_builder = FunctionBuilder::new(&mut trampoline, &mut builder_ctx); | |
270 | ||
271 | let jit_fn = cx | |
272 | .module | |
273 | .declare_func_in_func(jit_fn, trampoline_builder.func); | |
274 | let sig_ref = trampoline_builder.func.import_signature(sig); | |
275 | ||
276 | let entry_block = trampoline_builder.create_block(); | |
277 | trampoline_builder.append_block_params_for_function_params(entry_block); | |
278 | let fn_args = trampoline_builder | |
279 | .func | |
280 | .dfg | |
281 | .block_params(entry_block) | |
282 | .to_vec(); | |
283 | ||
284 | trampoline_builder.switch_to_block(entry_block); | |
285 | let instance_ptr = trampoline_builder | |
286 | .ins() | |
287 | .iconst(pointer_type, instance_ptr as u64 as i64); | |
288 | let jitted_fn = trampoline_builder.ins().call(jit_fn, &[instance_ptr]); | |
289 | let jitted_fn = trampoline_builder.func.dfg.inst_results(jitted_fn)[0]; | |
290 | let call_inst = trampoline_builder | |
291 | .ins() | |
292 | .call_indirect(sig_ref, jitted_fn, &fn_args); | |
293 | let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec(); | |
294 | trampoline_builder.ins().return_(&ret_vals); | |
295 | ||
296 | cx.module | |
297 | .define_function( | |
298 | func_id, | |
299 | &mut Context::for_function(trampoline), | |
300 | &mut cranelift_codegen::binemit::NullTrapSink {}, | |
301 | ) | |
302 | .unwrap(); | |
303 | } |