]>
Commit | Line | Data |
---|---|---|
fe692bf9 FG |
1 | //! Interpret intrinsics, lang items and `extern "C"` wellknown functions which their implementation |
2 | //! is not available. | |
3 | ||
4 | use std::cmp; | |
5 | ||
add651ee | 6 | use chalk_ir::TyKind; |
781aab86 FG |
7 | use hir_def::{ |
8 | builtin_type::{BuiltinInt, BuiltinUint}, | |
9 | resolver::HasResolver, | |
10 | }; | |
add651ee FG |
11 | use hir_expand::mod_path::ModPath; |
12 | ||
fe692bf9 FG |
13 | use super::*; |
14 | ||
add651ee FG |
15 | mod simd; |
16 | ||
fe692bf9 FG |
17 | macro_rules! from_bytes { |
18 | ($ty:tt, $value:expr) => { | |
19 | ($ty::from_le_bytes(match ($value).try_into() { | |
add651ee | 20 | Ok(it) => it, |
fe692bf9 FG |
21 | Err(_) => return Err(MirEvalError::TypeError("mismatched size")), |
22 | })) | |
23 | }; | |
24 | } | |
25 | ||
26 | macro_rules! not_supported { | |
add651ee FG |
27 | ($it: expr) => { |
28 | return Err(MirEvalError::NotSupported(format!($it))) | |
fe692bf9 FG |
29 | }; |
30 | } | |
31 | ||
32 | impl Evaluator<'_> { | |
33 | pub(super) fn detect_and_exec_special_function( | |
34 | &mut self, | |
35 | def: FunctionId, | |
36 | args: &[IntervalAndTy], | |
37 | generic_args: &Substitution, | |
add651ee | 38 | locals: &Locals, |
fe692bf9 FG |
39 | destination: Interval, |
40 | span: MirSpan, | |
41 | ) -> Result<bool> { | |
add651ee FG |
42 | if self.not_special_fn_cache.borrow().contains(&def) { |
43 | return Ok(false); | |
44 | } | |
fe692bf9 FG |
45 | let function_data = self.db.function_data(def); |
46 | let is_intrinsic = match &function_data.abi { | |
47 | Some(abi) => *abi == Interned::new_str("rust-intrinsic"), | |
48 | None => match def.lookup(self.db.upcast()).container { | |
49 | hir_def::ItemContainerId::ExternBlockId(block) => { | |
50 | let id = block.lookup(self.db.upcast()).id; | |
51 | id.item_tree(self.db.upcast())[id.value].abi.as_deref() | |
52 | == Some("rust-intrinsic") | |
53 | } | |
54 | _ => false, | |
55 | }, | |
56 | }; | |
57 | if is_intrinsic { | |
58 | self.exec_intrinsic( | |
59 | function_data.name.as_text().unwrap_or_default().as_str(), | |
60 | args, | |
61 | generic_args, | |
62 | destination, | |
63 | &locals, | |
64 | span, | |
65 | )?; | |
66 | return Ok(true); | |
67 | } | |
add651ee FG |
68 | let is_platform_intrinsic = match &function_data.abi { |
69 | Some(abi) => *abi == Interned::new_str("platform-intrinsic"), | |
70 | None => match def.lookup(self.db.upcast()).container { | |
71 | hir_def::ItemContainerId::ExternBlockId(block) => { | |
72 | let id = block.lookup(self.db.upcast()).id; | |
73 | id.item_tree(self.db.upcast())[id.value].abi.as_deref() | |
74 | == Some("platform-intrinsic") | |
75 | } | |
76 | _ => false, | |
77 | }, | |
78 | }; | |
79 | if is_platform_intrinsic { | |
80 | self.exec_platform_intrinsic( | |
81 | function_data.name.as_text().unwrap_or_default().as_str(), | |
82 | args, | |
83 | generic_args, | |
84 | destination, | |
85 | &locals, | |
86 | span, | |
87 | )?; | |
88 | return Ok(true); | |
89 | } | |
fe692bf9 FG |
90 | let is_extern_c = match def.lookup(self.db.upcast()).container { |
91 | hir_def::ItemContainerId::ExternBlockId(block) => { | |
92 | let id = block.lookup(self.db.upcast()).id; | |
93 | id.item_tree(self.db.upcast())[id.value].abi.as_deref() == Some("C") | |
94 | } | |
95 | _ => false, | |
96 | }; | |
97 | if is_extern_c { | |
98 | self.exec_extern_c( | |
99 | function_data.name.as_text().unwrap_or_default().as_str(), | |
100 | args, | |
101 | generic_args, | |
102 | destination, | |
103 | &locals, | |
104 | span, | |
105 | )?; | |
106 | return Ok(true); | |
107 | } | |
108 | let alloc_fn = function_data | |
109 | .attrs | |
110 | .iter() | |
add651ee FG |
111 | .filter_map(|it| it.path().as_ident()) |
112 | .filter_map(|it| it.as_str()) | |
113 | .find(|it| { | |
fe692bf9 FG |
114 | [ |
115 | "rustc_allocator", | |
116 | "rustc_deallocator", | |
117 | "rustc_reallocator", | |
118 | "rustc_allocator_zeroed", | |
119 | ] | |
add651ee | 120 | .contains(it) |
fe692bf9 FG |
121 | }); |
122 | if let Some(alloc_fn) = alloc_fn { | |
123 | self.exec_alloc_fn(alloc_fn, args, destination)?; | |
124 | return Ok(true); | |
125 | } | |
add651ee | 126 | if let Some(it) = self.detect_lang_function(def) { |
fe692bf9 | 127 | let arg_bytes = |
add651ee FG |
128 | args.iter().map(|it| Ok(it.get(&self)?.to_owned())).collect::<Result<Vec<_>>>()?; |
129 | let result = self.exec_lang_item(it, generic_args, &arg_bytes, locals, span)?; | |
fe692bf9 FG |
130 | destination.write_from_bytes(self, &result)?; |
131 | return Ok(true); | |
132 | } | |
add651ee FG |
133 | if let ItemContainerId::TraitId(t) = def.lookup(self.db.upcast()).container { |
134 | if self.db.lang_attr(t.into()) == Some(LangItem::Clone) { | |
135 | let [self_ty] = generic_args.as_slice(Interner) else { | |
136 | not_supported!("wrong generic arg count for clone"); | |
137 | }; | |
138 | let Some(self_ty) = self_ty.ty(Interner) else { | |
139 | not_supported!("wrong generic arg kind for clone"); | |
140 | }; | |
141 | // Clone has special impls for tuples and function pointers | |
781aab86 FG |
142 | if matches!( |
143 | self_ty.kind(Interner), | |
144 | TyKind::Function(_) | TyKind::Tuple(..) | TyKind::Closure(..) | |
145 | ) { | |
add651ee FG |
146 | self.exec_clone(def, args, self_ty.clone(), locals, destination, span)?; |
147 | return Ok(true); | |
148 | } | |
149 | // Return early to prevent caching clone as non special fn. | |
150 | return Ok(false); | |
151 | } | |
152 | } | |
153 | self.not_special_fn_cache.borrow_mut().insert(def); | |
fe692bf9 FG |
154 | Ok(false) |
155 | } | |
156 | ||
add651ee FG |
157 | /// Clone has special impls for tuples and function pointers |
158 | fn exec_clone( | |
159 | &mut self, | |
160 | def: FunctionId, | |
161 | args: &[IntervalAndTy], | |
162 | self_ty: Ty, | |
163 | locals: &Locals, | |
164 | destination: Interval, | |
165 | span: MirSpan, | |
166 | ) -> Result<()> { | |
167 | match self_ty.kind(Interner) { | |
168 | TyKind::Function(_) => { | |
169 | let [arg] = args else { | |
170 | not_supported!("wrong arg count for clone"); | |
171 | }; | |
172 | let addr = Address::from_bytes(arg.get(self)?)?; | |
173 | return destination | |
174 | .write_from_interval(self, Interval { addr, size: destination.size }); | |
175 | } | |
781aab86 FG |
176 | TyKind::Closure(id, subst) => { |
177 | let [arg] = args else { | |
178 | not_supported!("wrong arg count for clone"); | |
179 | }; | |
180 | let addr = Address::from_bytes(arg.get(self)?)?; | |
181 | let (closure_owner, _) = self.db.lookup_intern_closure((*id).into()); | |
182 | let infer = self.db.infer(closure_owner); | |
183 | let (captures, _) = infer.closure_info(id); | |
184 | let layout = self.layout(&self_ty)?; | |
185 | let ty_iter = captures.iter().map(|c| c.ty(subst)); | |
186 | self.exec_clone_for_fields(ty_iter, layout, addr, def, locals, destination, span)?; | |
187 | } | |
add651ee FG |
188 | TyKind::Tuple(_, subst) => { |
189 | let [arg] = args else { | |
190 | not_supported!("wrong arg count for clone"); | |
191 | }; | |
192 | let addr = Address::from_bytes(arg.get(self)?)?; | |
193 | let layout = self.layout(&self_ty)?; | |
781aab86 FG |
194 | let ty_iter = subst.iter(Interner).map(|ga| ga.assert_ty_ref(Interner).clone()); |
195 | self.exec_clone_for_fields(ty_iter, layout, addr, def, locals, destination, span)?; | |
add651ee FG |
196 | } |
197 | _ => { | |
198 | self.exec_fn_with_args( | |
199 | def, | |
200 | args, | |
201 | Substitution::from1(Interner, self_ty), | |
202 | locals, | |
203 | destination, | |
204 | None, | |
205 | span, | |
206 | )?; | |
207 | } | |
208 | } | |
209 | Ok(()) | |
210 | } | |
211 | ||
781aab86 FG |
212 | fn exec_clone_for_fields( |
213 | &mut self, | |
214 | ty_iter: impl Iterator<Item = Ty>, | |
215 | layout: Arc<Layout>, | |
216 | addr: Address, | |
217 | def: FunctionId, | |
218 | locals: &Locals, | |
219 | destination: Interval, | |
220 | span: MirSpan, | |
221 | ) -> Result<()> { | |
222 | for (i, ty) in ty_iter.enumerate() { | |
223 | let size = self.layout(&ty)?.size.bytes_usize(); | |
224 | let tmp = self.heap_allocate(self.ptr_size(), self.ptr_size())?; | |
225 | let arg = IntervalAndTy { | |
226 | interval: Interval { addr: tmp, size: self.ptr_size() }, | |
227 | ty: TyKind::Ref(Mutability::Not, static_lifetime(), ty.clone()).intern(Interner), | |
228 | }; | |
229 | let offset = layout.fields.offset(i).bytes_usize(); | |
230 | self.write_memory(tmp, &addr.offset(offset).to_bytes())?; | |
231 | self.exec_clone( | |
232 | def, | |
233 | &[arg], | |
234 | ty, | |
235 | locals, | |
236 | destination.slice(offset..offset + size), | |
237 | span, | |
238 | )?; | |
239 | } | |
240 | Ok(()) | |
241 | } | |
242 | ||
fe692bf9 FG |
243 | fn exec_alloc_fn( |
244 | &mut self, | |
245 | alloc_fn: &str, | |
246 | args: &[IntervalAndTy], | |
247 | destination: Interval, | |
248 | ) -> Result<()> { | |
249 | match alloc_fn { | |
250 | "rustc_allocator_zeroed" | "rustc_allocator" => { | |
251 | let [size, align] = args else { | |
252 | return Err(MirEvalError::TypeError("rustc_allocator args are not provided")); | |
253 | }; | |
254 | let size = from_bytes!(usize, size.get(self)?); | |
255 | let align = from_bytes!(usize, align.get(self)?); | |
add651ee | 256 | let result = self.heap_allocate(size, align)?; |
fe692bf9 FG |
257 | destination.write_from_bytes(self, &result.to_bytes())?; |
258 | } | |
259 | "rustc_deallocator" => { /* no-op for now */ } | |
260 | "rustc_reallocator" => { | |
261 | let [ptr, old_size, align, new_size] = args else { | |
262 | return Err(MirEvalError::TypeError("rustc_allocator args are not provided")); | |
263 | }; | |
fe692bf9 FG |
264 | let old_size = from_bytes!(usize, old_size.get(self)?); |
265 | let new_size = from_bytes!(usize, new_size.get(self)?); | |
add651ee FG |
266 | if old_size >= new_size { |
267 | destination.write_from_interval(self, ptr.interval)?; | |
268 | } else { | |
269 | let ptr = Address::from_bytes(ptr.get(self)?)?; | |
270 | let align = from_bytes!(usize, align.get(self)?); | |
271 | let result = self.heap_allocate(new_size, align)?; | |
272 | Interval { addr: result, size: old_size } | |
273 | .write_from_interval(self, Interval { addr: ptr, size: old_size })?; | |
274 | destination.write_from_bytes(self, &result.to_bytes())?; | |
275 | } | |
fe692bf9 FG |
276 | } |
277 | _ => not_supported!("unknown alloc function"), | |
278 | } | |
279 | Ok(()) | |
280 | } | |
281 | ||
282 | fn detect_lang_function(&self, def: FunctionId) -> Option<LangItem> { | |
283 | use LangItem::*; | |
add651ee | 284 | let candidate = self.db.lang_attr(def.into())?; |
fe692bf9 FG |
285 | // We want to execute these functions with special logic |
286 | if [PanicFmt, BeginPanic, SliceLen, DropInPlace].contains(&candidate) { | |
287 | return Some(candidate); | |
288 | } | |
289 | None | |
290 | } | |
291 | ||
292 | fn exec_lang_item( | |
293 | &mut self, | |
add651ee | 294 | it: LangItem, |
fe692bf9 FG |
295 | generic_args: &Substitution, |
296 | args: &[Vec<u8>], | |
add651ee | 297 | locals: &Locals, |
fe692bf9 FG |
298 | span: MirSpan, |
299 | ) -> Result<Vec<u8>> { | |
300 | use LangItem::*; | |
301 | let mut args = args.iter(); | |
add651ee | 302 | match it { |
fe692bf9 FG |
303 | BeginPanic => Err(MirEvalError::Panic("<unknown-panic-payload>".to_string())), |
304 | PanicFmt => { | |
305 | let message = (|| { | |
781aab86 FG |
306 | let resolver = self |
307 | .db | |
308 | .crate_def_map(self.crate_id) | |
309 | .crate_root() | |
310 | .resolver(self.db.upcast()); | |
add651ee FG |
311 | let Some(format_fn) = resolver.resolve_path_in_value_ns_fully( |
312 | self.db.upcast(), | |
781aab86 FG |
313 | &hir_def::path::Path::from_known_path_with_no_generic( |
314 | ModPath::from_segments( | |
315 | hir_expand::mod_path::PathKind::Abs, | |
316 | [name![std], name![fmt], name![format]].into_iter(), | |
317 | ), | |
318 | ), | |
add651ee FG |
319 | ) else { |
320 | not_supported!("std::fmt::format not found"); | |
321 | }; | |
781aab86 FG |
322 | let hir_def::resolver::ValueNs::FunctionId(format_fn) = format_fn else { |
323 | not_supported!("std::fmt::format is not a function") | |
324 | }; | |
325 | let message_string = self.interpret_mir( | |
326 | self.db | |
327 | .mir_body(format_fn.into()) | |
328 | .map_err(|e| MirEvalError::MirLowerError(format_fn, e))?, | |
329 | args.map(|x| IntervalOrOwned::Owned(x.clone())), | |
330 | )?; | |
331 | let addr = | |
332 | Address::from_bytes(&message_string[self.ptr_size()..2 * self.ptr_size()])?; | |
add651ee | 333 | let size = from_bytes!(usize, message_string[2 * self.ptr_size()..]); |
781aab86 FG |
334 | Ok(std::string::String::from_utf8_lossy(self.read_memory(addr, size)?) |
335 | .into_owned()) | |
fe692bf9 | 336 | })() |
add651ee | 337 | .unwrap_or_else(|e| format!("Failed to render panic format args: {e:?}")); |
fe692bf9 FG |
338 | Err(MirEvalError::Panic(message)) |
339 | } | |
340 | SliceLen => { | |
341 | let arg = args | |
342 | .next() | |
343 | .ok_or(MirEvalError::TypeError("argument of <[T]>::len() is not provided"))?; | |
344 | let ptr_size = arg.len() / 2; | |
345 | Ok(arg[ptr_size..].into()) | |
346 | } | |
347 | DropInPlace => { | |
348 | let ty = | |
add651ee | 349 | generic_args.as_slice(Interner).get(0).and_then(|it| it.ty(Interner)).ok_or( |
fe692bf9 FG |
350 | MirEvalError::TypeError( |
351 | "generic argument of drop_in_place is not provided", | |
352 | ), | |
353 | )?; | |
354 | let arg = args | |
355 | .next() | |
356 | .ok_or(MirEvalError::TypeError("argument of drop_in_place is not provided"))?; | |
357 | self.run_drop_glue_deep( | |
358 | ty.clone(), | |
359 | locals, | |
360 | Address::from_bytes(&arg[0..self.ptr_size()])?, | |
361 | &arg[self.ptr_size()..], | |
362 | span, | |
363 | )?; | |
364 | Ok(vec![]) | |
365 | } | |
add651ee FG |
366 | it => not_supported!("Executing lang item {it:?}"), |
367 | } | |
368 | } | |
369 | ||
370 | fn exec_syscall( | |
371 | &mut self, | |
372 | id: i64, | |
373 | args: &[IntervalAndTy], | |
374 | destination: Interval, | |
375 | _locals: &Locals, | |
376 | _span: MirSpan, | |
377 | ) -> Result<()> { | |
378 | match id { | |
379 | 318 => { | |
380 | // SYS_getrandom | |
381 | let [buf, len, _flags] = args else { | |
382 | return Err(MirEvalError::TypeError("SYS_getrandom args are not provided")); | |
383 | }; | |
384 | let addr = Address::from_bytes(buf.get(self)?)?; | |
385 | let size = from_bytes!(usize, len.get(self)?); | |
386 | for i in 0..size { | |
387 | let rand_byte = self.random_state.rand_u64() as u8; | |
388 | self.write_memory(addr.offset(i), &[rand_byte])?; | |
389 | } | |
390 | destination.write_from_interval(self, len.interval) | |
391 | } | |
392 | _ => { | |
393 | not_supported!("Unknown syscall id {id:?}") | |
394 | } | |
fe692bf9 FG |
395 | } |
396 | } | |
397 | ||
398 | fn exec_extern_c( | |
399 | &mut self, | |
400 | as_str: &str, | |
401 | args: &[IntervalAndTy], | |
402 | _generic_args: &Substitution, | |
403 | destination: Interval, | |
add651ee FG |
404 | locals: &Locals, |
405 | span: MirSpan, | |
fe692bf9 FG |
406 | ) -> Result<()> { |
407 | match as_str { | |
408 | "memcmp" => { | |
409 | let [ptr1, ptr2, size] = args else { | |
410 | return Err(MirEvalError::TypeError("memcmp args are not provided")); | |
411 | }; | |
412 | let addr1 = Address::from_bytes(ptr1.get(self)?)?; | |
413 | let addr2 = Address::from_bytes(ptr2.get(self)?)?; | |
414 | let size = from_bytes!(usize, size.get(self)?); | |
415 | let slice1 = self.read_memory(addr1, size)?; | |
416 | let slice2 = self.read_memory(addr2, size)?; | |
417 | let r: i128 = match slice1.cmp(slice2) { | |
418 | cmp::Ordering::Less => -1, | |
419 | cmp::Ordering::Equal => 0, | |
420 | cmp::Ordering::Greater => 1, | |
421 | }; | |
422 | destination.write_from_bytes(self, &r.to_le_bytes()[..destination.size]) | |
423 | } | |
424 | "write" => { | |
425 | let [fd, ptr, len] = args else { | |
426 | return Err(MirEvalError::TypeError("libc::write args are not provided")); | |
427 | }; | |
428 | let fd = u128::from_le_bytes(pad16(fd.get(self)?, false)); | |
429 | let interval = Interval { | |
430 | addr: Address::from_bytes(ptr.get(self)?)?, | |
431 | size: from_bytes!(usize, len.get(self)?), | |
432 | }; | |
433 | match fd { | |
434 | 1 => { | |
435 | self.write_to_stdout(interval)?; | |
436 | } | |
437 | 2 => { | |
438 | self.write_to_stderr(interval)?; | |
439 | } | |
440 | _ => not_supported!("write to arbitrary file descriptor"), | |
441 | } | |
442 | destination.write_from_interval(self, len.interval)?; | |
443 | Ok(()) | |
444 | } | |
445 | "pthread_key_create" => { | |
446 | let key = self.thread_local_storage.create_key(); | |
447 | let Some(arg0) = args.get(0) else { | |
448 | return Err(MirEvalError::TypeError("pthread_key_create arg0 is not provided")); | |
449 | }; | |
450 | let arg0_addr = Address::from_bytes(arg0.get(self)?)?; | |
451 | let key_ty = if let Some((ty, ..)) = arg0.ty.as_reference_or_ptr() { | |
452 | ty | |
453 | } else { | |
454 | return Err(MirEvalError::TypeError( | |
455 | "pthread_key_create arg0 is not a pointer", | |
456 | )); | |
457 | }; | |
458 | let arg0_interval = Interval::new( | |
459 | arg0_addr, | |
460 | self.size_of_sized(key_ty, locals, "pthread_key_create key arg")?, | |
461 | ); | |
462 | arg0_interval.write_from_bytes(self, &key.to_le_bytes()[0..arg0_interval.size])?; | |
463 | // return 0 as success | |
464 | destination.write_from_bytes(self, &0u64.to_le_bytes()[0..destination.size])?; | |
465 | Ok(()) | |
466 | } | |
467 | "pthread_getspecific" => { | |
468 | let Some(arg0) = args.get(0) else { | |
add651ee FG |
469 | return Err(MirEvalError::TypeError( |
470 | "pthread_getspecific arg0 is not provided", | |
471 | )); | |
fe692bf9 FG |
472 | }; |
473 | let key = from_bytes!(usize, &pad16(arg0.get(self)?, false)[0..8]); | |
474 | let value = self.thread_local_storage.get_key(key)?; | |
475 | destination.write_from_bytes(self, &value.to_le_bytes()[0..destination.size])?; | |
476 | Ok(()) | |
477 | } | |
478 | "pthread_setspecific" => { | |
479 | let Some(arg0) = args.get(0) else { | |
add651ee FG |
480 | return Err(MirEvalError::TypeError( |
481 | "pthread_setspecific arg0 is not provided", | |
482 | )); | |
fe692bf9 FG |
483 | }; |
484 | let key = from_bytes!(usize, &pad16(arg0.get(self)?, false)[0..8]); | |
485 | let Some(arg1) = args.get(1) else { | |
add651ee FG |
486 | return Err(MirEvalError::TypeError( |
487 | "pthread_setspecific arg1 is not provided", | |
488 | )); | |
fe692bf9 FG |
489 | }; |
490 | let value = from_bytes!(u128, pad16(arg1.get(self)?, false)); | |
491 | self.thread_local_storage.set_key(key, value)?; | |
492 | // return 0 as success | |
493 | destination.write_from_bytes(self, &0u64.to_le_bytes()[0..destination.size])?; | |
494 | Ok(()) | |
495 | } | |
496 | "pthread_key_delete" => { | |
497 | // we ignore this currently | |
498 | // return 0 as success | |
499 | destination.write_from_bytes(self, &0u64.to_le_bytes()[0..destination.size])?; | |
500 | Ok(()) | |
501 | } | |
add651ee FG |
502 | "syscall" => { |
503 | let Some((id, rest)) = args.split_first() else { | |
781aab86 | 504 | return Err(MirEvalError::TypeError("syscall arg1 is not provided")); |
add651ee FG |
505 | }; |
506 | let id = from_bytes!(i64, id.get(self)?); | |
507 | self.exec_syscall(id, rest, destination, locals, span) | |
508 | } | |
509 | "sched_getaffinity" => { | |
510 | let [_pid, _set_size, set] = args else { | |
511 | return Err(MirEvalError::TypeError("libc::write args are not provided")); | |
512 | }; | |
513 | let set = Address::from_bytes(set.get(self)?)?; | |
514 | // Only enable core 0 (we are single threaded anyway), which is bitset 0x0000001 | |
515 | self.write_memory(set, &[1])?; | |
516 | // return 0 as success | |
517 | self.write_memory_using_ref(destination.addr, destination.size)?.fill(0); | |
518 | Ok(()) | |
519 | } | |
781aab86 FG |
520 | "getenv" => { |
521 | let [name] = args else { | |
522 | return Err(MirEvalError::TypeError("libc::write args are not provided")); | |
523 | }; | |
524 | let mut name_buf = vec![]; | |
525 | let name = { | |
526 | let mut index = Address::from_bytes(name.get(self)?)?; | |
527 | loop { | |
528 | let byte = self.read_memory(index, 1)?[0]; | |
529 | index = index.offset(1); | |
530 | if byte == 0 { | |
531 | break; | |
532 | } | |
533 | name_buf.push(byte); | |
534 | } | |
535 | String::from_utf8_lossy(&name_buf) | |
536 | }; | |
537 | let value = self.db.crate_graph()[self.crate_id].env.get(&name); | |
538 | match value { | |
539 | None => { | |
540 | // Write null as fail | |
541 | self.write_memory_using_ref(destination.addr, destination.size)?.fill(0); | |
542 | } | |
543 | Some(mut value) => { | |
544 | value.push('\0'); | |
545 | let addr = self.heap_allocate(value.len(), 1)?; | |
546 | self.write_memory(addr, value.as_bytes())?; | |
547 | self.write_memory(destination.addr, &addr.to_bytes())?; | |
548 | } | |
549 | } | |
550 | Ok(()) | |
551 | } | |
fe692bf9 FG |
552 | _ => not_supported!("unknown external function {as_str}"), |
553 | } | |
554 | } | |
555 | ||
add651ee FG |
556 | fn exec_platform_intrinsic( |
557 | &mut self, | |
558 | name: &str, | |
559 | args: &[IntervalAndTy], | |
560 | generic_args: &Substitution, | |
561 | destination: Interval, | |
562 | locals: &Locals, | |
563 | span: MirSpan, | |
564 | ) -> Result<()> { | |
565 | if let Some(name) = name.strip_prefix("simd_") { | |
566 | return self.exec_simd_intrinsic(name, args, generic_args, destination, locals, span); | |
567 | } | |
568 | not_supported!("unknown platform intrinsic {name}"); | |
569 | } | |
570 | ||
fe692bf9 FG |
571 | fn exec_intrinsic( |
572 | &mut self, | |
573 | name: &str, | |
574 | args: &[IntervalAndTy], | |
575 | generic_args: &Substitution, | |
576 | destination: Interval, | |
add651ee | 577 | locals: &Locals, |
fe692bf9 FG |
578 | span: MirSpan, |
579 | ) -> Result<()> { | |
580 | if let Some(name) = name.strip_prefix("atomic_") { | |
581 | return self.exec_atomic_intrinsic(name, args, generic_args, destination, locals, span); | |
582 | } | |
583 | if let Some(name) = name.strip_suffix("f64") { | |
584 | let result = match name { | |
585 | "sqrt" | "sin" | "cos" | "exp" | "exp2" | "log" | "log10" | "log2" | "fabs" | |
586 | | "floor" | "ceil" | "trunc" | "rint" | "nearbyint" | "round" | "roundeven" => { | |
587 | let [arg] = args else { | |
add651ee FG |
588 | return Err(MirEvalError::TypeError( |
589 | "f64 intrinsic signature doesn't match fn (f64) -> f64", | |
590 | )); | |
fe692bf9 FG |
591 | }; |
592 | let arg = from_bytes!(f64, arg.get(self)?); | |
593 | match name { | |
594 | "sqrt" => arg.sqrt(), | |
595 | "sin" => arg.sin(), | |
596 | "cos" => arg.cos(), | |
597 | "exp" => arg.exp(), | |
598 | "exp2" => arg.exp2(), | |
599 | "log" => arg.ln(), | |
600 | "log10" => arg.log10(), | |
601 | "log2" => arg.log2(), | |
602 | "fabs" => arg.abs(), | |
603 | "floor" => arg.floor(), | |
604 | "ceil" => arg.ceil(), | |
605 | "trunc" => arg.trunc(), | |
606 | // FIXME: these rounds should be different, but only `.round()` is stable now. | |
607 | "rint" => arg.round(), | |
608 | "nearbyint" => arg.round(), | |
609 | "round" => arg.round(), | |
610 | "roundeven" => arg.round(), | |
611 | _ => unreachable!(), | |
612 | } | |
613 | } | |
614 | "pow" | "minnum" | "maxnum" | "copysign" => { | |
615 | let [arg1, arg2] = args else { | |
add651ee FG |
616 | return Err(MirEvalError::TypeError( |
617 | "f64 intrinsic signature doesn't match fn (f64, f64) -> f64", | |
618 | )); | |
fe692bf9 FG |
619 | }; |
620 | let arg1 = from_bytes!(f64, arg1.get(self)?); | |
621 | let arg2 = from_bytes!(f64, arg2.get(self)?); | |
622 | match name { | |
623 | "pow" => arg1.powf(arg2), | |
624 | "minnum" => arg1.min(arg2), | |
625 | "maxnum" => arg1.max(arg2), | |
626 | "copysign" => arg1.copysign(arg2), | |
627 | _ => unreachable!(), | |
628 | } | |
629 | } | |
630 | "powi" => { | |
631 | let [arg1, arg2] = args else { | |
add651ee FG |
632 | return Err(MirEvalError::TypeError( |
633 | "powif64 signature doesn't match fn (f64, i32) -> f64", | |
634 | )); | |
fe692bf9 FG |
635 | }; |
636 | let arg1 = from_bytes!(f64, arg1.get(self)?); | |
637 | let arg2 = from_bytes!(i32, arg2.get(self)?); | |
638 | arg1.powi(arg2) | |
639 | } | |
640 | "fma" => { | |
641 | let [arg1, arg2, arg3] = args else { | |
add651ee FG |
642 | return Err(MirEvalError::TypeError( |
643 | "fmaf64 signature doesn't match fn (f64, f64, f64) -> f64", | |
644 | )); | |
fe692bf9 FG |
645 | }; |
646 | let arg1 = from_bytes!(f64, arg1.get(self)?); | |
647 | let arg2 = from_bytes!(f64, arg2.get(self)?); | |
648 | let arg3 = from_bytes!(f64, arg3.get(self)?); | |
649 | arg1.mul_add(arg2, arg3) | |
650 | } | |
651 | _ => not_supported!("unknown f64 intrinsic {name}"), | |
652 | }; | |
653 | return destination.write_from_bytes(self, &result.to_le_bytes()); | |
654 | } | |
655 | if let Some(name) = name.strip_suffix("f32") { | |
656 | let result = match name { | |
657 | "sqrt" | "sin" | "cos" | "exp" | "exp2" | "log" | "log10" | "log2" | "fabs" | |
658 | | "floor" | "ceil" | "trunc" | "rint" | "nearbyint" | "round" | "roundeven" => { | |
659 | let [arg] = args else { | |
add651ee FG |
660 | return Err(MirEvalError::TypeError( |
661 | "f32 intrinsic signature doesn't match fn (f32) -> f32", | |
662 | )); | |
fe692bf9 FG |
663 | }; |
664 | let arg = from_bytes!(f32, arg.get(self)?); | |
665 | match name { | |
666 | "sqrt" => arg.sqrt(), | |
667 | "sin" => arg.sin(), | |
668 | "cos" => arg.cos(), | |
669 | "exp" => arg.exp(), | |
670 | "exp2" => arg.exp2(), | |
671 | "log" => arg.ln(), | |
672 | "log10" => arg.log10(), | |
673 | "log2" => arg.log2(), | |
674 | "fabs" => arg.abs(), | |
675 | "floor" => arg.floor(), | |
676 | "ceil" => arg.ceil(), | |
677 | "trunc" => arg.trunc(), | |
678 | // FIXME: these rounds should be different, but only `.round()` is stable now. | |
679 | "rint" => arg.round(), | |
680 | "nearbyint" => arg.round(), | |
681 | "round" => arg.round(), | |
682 | "roundeven" => arg.round(), | |
683 | _ => unreachable!(), | |
684 | } | |
685 | } | |
686 | "pow" | "minnum" | "maxnum" | "copysign" => { | |
687 | let [arg1, arg2] = args else { | |
add651ee FG |
688 | return Err(MirEvalError::TypeError( |
689 | "f32 intrinsic signature doesn't match fn (f32, f32) -> f32", | |
690 | )); | |
fe692bf9 FG |
691 | }; |
692 | let arg1 = from_bytes!(f32, arg1.get(self)?); | |
693 | let arg2 = from_bytes!(f32, arg2.get(self)?); | |
694 | match name { | |
695 | "pow" => arg1.powf(arg2), | |
696 | "minnum" => arg1.min(arg2), | |
697 | "maxnum" => arg1.max(arg2), | |
698 | "copysign" => arg1.copysign(arg2), | |
699 | _ => unreachable!(), | |
700 | } | |
701 | } | |
702 | "powi" => { | |
703 | let [arg1, arg2] = args else { | |
add651ee FG |
704 | return Err(MirEvalError::TypeError( |
705 | "powif32 signature doesn't match fn (f32, i32) -> f32", | |
706 | )); | |
fe692bf9 FG |
707 | }; |
708 | let arg1 = from_bytes!(f32, arg1.get(self)?); | |
709 | let arg2 = from_bytes!(i32, arg2.get(self)?); | |
710 | arg1.powi(arg2) | |
711 | } | |
712 | "fma" => { | |
713 | let [arg1, arg2, arg3] = args else { | |
add651ee FG |
714 | return Err(MirEvalError::TypeError( |
715 | "fmaf32 signature doesn't match fn (f32, f32, f32) -> f32", | |
716 | )); | |
fe692bf9 FG |
717 | }; |
718 | let arg1 = from_bytes!(f32, arg1.get(self)?); | |
719 | let arg2 = from_bytes!(f32, arg2.get(self)?); | |
720 | let arg3 = from_bytes!(f32, arg3.get(self)?); | |
721 | arg1.mul_add(arg2, arg3) | |
722 | } | |
723 | _ => not_supported!("unknown f32 intrinsic {name}"), | |
724 | }; | |
725 | return destination.write_from_bytes(self, &result.to_le_bytes()); | |
726 | } | |
727 | match name { | |
728 | "size_of" => { | |
781aab86 FG |
729 | let Some(ty) = |
730 | generic_args.as_slice(Interner).get(0).and_then(|it| it.ty(Interner)) | |
add651ee | 731 | else { |
fe692bf9 FG |
732 | return Err(MirEvalError::TypeError("size_of generic arg is not provided")); |
733 | }; | |
734 | let size = self.size_of_sized(ty, locals, "size_of arg")?; | |
735 | destination.write_from_bytes(self, &size.to_le_bytes()[0..destination.size]) | |
736 | } | |
737 | "min_align_of" | "pref_align_of" => { | |
781aab86 FG |
738 | let Some(ty) = |
739 | generic_args.as_slice(Interner).get(0).and_then(|it| it.ty(Interner)) | |
740 | else { | |
fe692bf9 FG |
741 | return Err(MirEvalError::TypeError("align_of generic arg is not provided")); |
742 | }; | |
743 | let align = self.layout(ty)?.align.abi.bytes(); | |
744 | destination.write_from_bytes(self, &align.to_le_bytes()[0..destination.size]) | |
745 | } | |
add651ee | 746 | "size_of_val" => { |
781aab86 FG |
747 | let Some(ty) = |
748 | generic_args.as_slice(Interner).get(0).and_then(|it| it.ty(Interner)) | |
add651ee FG |
749 | else { |
750 | return Err(MirEvalError::TypeError("size_of_val generic arg is not provided")); | |
751 | }; | |
752 | let [arg] = args else { | |
753 | return Err(MirEvalError::TypeError("size_of_val args are not provided")); | |
754 | }; | |
755 | if let Some((size, _)) = self.size_align_of(ty, locals)? { | |
756 | destination.write_from_bytes(self, &size.to_le_bytes()) | |
757 | } else { | |
758 | let metadata = arg.interval.slice(self.ptr_size()..self.ptr_size() * 2); | |
759 | let (size, _) = self.size_align_of_unsized(ty, metadata, locals)?; | |
760 | destination.write_from_bytes(self, &size.to_le_bytes()) | |
761 | } | |
762 | } | |
763 | "min_align_of_val" => { | |
781aab86 FG |
764 | let Some(ty) = |
765 | generic_args.as_slice(Interner).get(0).and_then(|it| it.ty(Interner)) | |
766 | else { | |
767 | return Err(MirEvalError::TypeError( | |
768 | "min_align_of_val generic arg is not provided", | |
769 | )); | |
add651ee FG |
770 | }; |
771 | let [arg] = args else { | |
772 | return Err(MirEvalError::TypeError("min_align_of_val args are not provided")); | |
773 | }; | |
774 | if let Some((_, align)) = self.size_align_of(ty, locals)? { | |
775 | destination.write_from_bytes(self, &align.to_le_bytes()) | |
776 | } else { | |
777 | let metadata = arg.interval.slice(self.ptr_size()..self.ptr_size() * 2); | |
778 | let (_, align) = self.size_align_of_unsized(ty, metadata, locals)?; | |
779 | destination.write_from_bytes(self, &align.to_le_bytes()) | |
780 | } | |
781 | } | |
782 | "type_name" => { | |
781aab86 FG |
783 | let Some(ty) = |
784 | generic_args.as_slice(Interner).get(0).and_then(|it| it.ty(Interner)) | |
add651ee FG |
785 | else { |
786 | return Err(MirEvalError::TypeError("type_name generic arg is not provided")); | |
787 | }; | |
788 | let ty_name = match ty.display_source_code( | |
789 | self.db, | |
790 | locals.body.owner.module(self.db.upcast()), | |
791 | true, | |
792 | ) { | |
793 | Ok(ty_name) => ty_name, | |
794 | // Fallback to human readable display in case of `Err`. Ideally we want to use `display_source_code` to | |
795 | // render full paths. | |
796 | Err(_) => ty.display(self.db).to_string(), | |
797 | }; | |
798 | let len = ty_name.len(); | |
799 | let addr = self.heap_allocate(len, 1)?; | |
800 | self.write_memory(addr, ty_name.as_bytes())?; | |
801 | destination.slice(0..self.ptr_size()).write_from_bytes(self, &addr.to_bytes())?; | |
802 | destination | |
803 | .slice(self.ptr_size()..2 * self.ptr_size()) | |
804 | .write_from_bytes(self, &len.to_le_bytes()) | |
805 | } | |
fe692bf9 | 806 | "needs_drop" => { |
781aab86 FG |
807 | let Some(ty) = |
808 | generic_args.as_slice(Interner).get(0).and_then(|it| it.ty(Interner)) | |
add651ee | 809 | else { |
fe692bf9 FG |
810 | return Err(MirEvalError::TypeError("size_of generic arg is not provided")); |
811 | }; | |
812 | let result = !ty.clone().is_copy(self.db, locals.body.owner); | |
813 | destination.write_from_bytes(self, &[u8::from(result)]) | |
814 | } | |
815 | "ptr_guaranteed_cmp" => { | |
816 | // FIXME: this is wrong for const eval, it should return 2 in some | |
817 | // cases. | |
818 | let [lhs, rhs] = args else { | |
819 | return Err(MirEvalError::TypeError("wrapping_add args are not provided")); | |
820 | }; | |
821 | let ans = lhs.get(self)? == rhs.get(self)?; | |
822 | destination.write_from_bytes(self, &[u8::from(ans)]) | |
823 | } | |
add651ee | 824 | "saturating_add" | "saturating_sub" => { |
fe692bf9 FG |
825 | let [lhs, rhs] = args else { |
826 | return Err(MirEvalError::TypeError("saturating_add args are not provided")); | |
827 | }; | |
828 | let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false)); | |
829 | let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false)); | |
add651ee FG |
830 | let ans = match name { |
831 | "saturating_add" => lhs.saturating_add(rhs), | |
832 | "saturating_sub" => lhs.saturating_sub(rhs), | |
833 | _ => unreachable!(), | |
834 | }; | |
fe692bf9 FG |
835 | let bits = destination.size * 8; |
836 | // FIXME: signed | |
837 | let is_signed = false; | |
838 | let mx: u128 = if is_signed { (1 << (bits - 1)) - 1 } else { (1 << bits) - 1 }; | |
839 | // FIXME: signed | |
840 | let mn: u128 = 0; | |
841 | let ans = cmp::min(mx, cmp::max(mn, ans)); | |
842 | destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]) | |
843 | } | |
844 | "wrapping_add" | "unchecked_add" => { | |
845 | let [lhs, rhs] = args else { | |
846 | return Err(MirEvalError::TypeError("wrapping_add args are not provided")); | |
847 | }; | |
848 | let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false)); | |
849 | let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false)); | |
850 | let ans = lhs.wrapping_add(rhs); | |
851 | destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]) | |
852 | } | |
add651ee FG |
853 | "ptr_offset_from_unsigned" | "ptr_offset_from" => { |
854 | let [lhs, rhs] = args else { | |
855 | return Err(MirEvalError::TypeError("wrapping_sub args are not provided")); | |
856 | }; | |
857 | let lhs = i128::from_le_bytes(pad16(lhs.get(self)?, false)); | |
858 | let rhs = i128::from_le_bytes(pad16(rhs.get(self)?, false)); | |
859 | let ans = lhs.wrapping_sub(rhs); | |
781aab86 FG |
860 | let Some(ty) = |
861 | generic_args.as_slice(Interner).get(0).and_then(|it| it.ty(Interner)) | |
add651ee | 862 | else { |
781aab86 FG |
863 | return Err(MirEvalError::TypeError( |
864 | "ptr_offset_from generic arg is not provided", | |
865 | )); | |
add651ee FG |
866 | }; |
867 | let size = self.size_of_sized(ty, locals, "ptr_offset_from arg")? as i128; | |
868 | let ans = ans / size; | |
869 | destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]) | |
870 | } | |
871 | "wrapping_sub" | "unchecked_sub" => { | |
fe692bf9 FG |
872 | let [lhs, rhs] = args else { |
873 | return Err(MirEvalError::TypeError("wrapping_sub args are not provided")); | |
874 | }; | |
875 | let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false)); | |
876 | let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false)); | |
877 | let ans = lhs.wrapping_sub(rhs); | |
878 | destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]) | |
879 | } | |
880 | "wrapping_mul" | "unchecked_mul" => { | |
881 | let [lhs, rhs] = args else { | |
882 | return Err(MirEvalError::TypeError("wrapping_mul args are not provided")); | |
883 | }; | |
884 | let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false)); | |
885 | let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false)); | |
886 | let ans = lhs.wrapping_mul(rhs); | |
887 | destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]) | |
888 | } | |
add651ee FG |
889 | "wrapping_shl" | "unchecked_shl" => { |
890 | // FIXME: signed | |
891 | let [lhs, rhs] = args else { | |
892 | return Err(MirEvalError::TypeError("unchecked_shl args are not provided")); | |
893 | }; | |
894 | let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false)); | |
895 | let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false)); | |
896 | let ans = lhs.wrapping_shl(rhs as u32); | |
897 | destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]) | |
898 | } | |
899 | "wrapping_shr" | "unchecked_shr" => { | |
900 | // FIXME: signed | |
901 | let [lhs, rhs] = args else { | |
902 | return Err(MirEvalError::TypeError("unchecked_shr args are not provided")); | |
903 | }; | |
904 | let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false)); | |
905 | let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false)); | |
906 | let ans = lhs.wrapping_shr(rhs as u32); | |
907 | destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]) | |
908 | } | |
fe692bf9 FG |
909 | "unchecked_rem" => { |
910 | // FIXME: signed | |
911 | let [lhs, rhs] = args else { | |
912 | return Err(MirEvalError::TypeError("unchecked_rem args are not provided")); | |
913 | }; | |
914 | let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false)); | |
915 | let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false)); | |
916 | let ans = lhs.checked_rem(rhs).ok_or_else(|| { | |
917 | MirEvalError::UndefinedBehavior("unchecked_rem with bad inputs".to_owned()) | |
918 | })?; | |
919 | destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]) | |
920 | } | |
921 | "unchecked_div" | "exact_div" => { | |
922 | // FIXME: signed | |
923 | let [lhs, rhs] = args else { | |
924 | return Err(MirEvalError::TypeError("unchecked_div args are not provided")); | |
925 | }; | |
926 | let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false)); | |
927 | let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false)); | |
928 | let ans = lhs.checked_div(rhs).ok_or_else(|| { | |
929 | MirEvalError::UndefinedBehavior("unchecked_rem with bad inputs".to_owned()) | |
930 | })?; | |
931 | destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]) | |
932 | } | |
933 | "add_with_overflow" | "sub_with_overflow" | "mul_with_overflow" => { | |
934 | let [lhs, rhs] = args else { | |
935 | return Err(MirEvalError::TypeError("const_eval_select args are not provided")); | |
936 | }; | |
937 | let result_ty = TyKind::Tuple( | |
938 | 2, | |
939 | Substitution::from_iter(Interner, [lhs.ty.clone(), TyBuilder::bool()]), | |
940 | ) | |
941 | .intern(Interner); | |
942 | let op_size = | |
943 | self.size_of_sized(&lhs.ty, locals, "operand of add_with_overflow")?; | |
944 | let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false)); | |
945 | let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false)); | |
946 | let (ans, u128overflow) = match name { | |
947 | "add_with_overflow" => lhs.overflowing_add(rhs), | |
948 | "sub_with_overflow" => lhs.overflowing_sub(rhs), | |
949 | "mul_with_overflow" => lhs.overflowing_mul(rhs), | |
950 | _ => unreachable!(), | |
951 | }; | |
952 | let is_overflow = u128overflow | |
add651ee | 953 | || ans.to_le_bytes()[op_size..].iter().any(|&it| it != 0 && it != 255); |
fe692bf9 FG |
954 | let is_overflow = vec![u8::from(is_overflow)]; |
955 | let layout = self.layout(&result_ty)?; | |
956 | let result = self.make_by_layout( | |
957 | layout.size.bytes_usize(), | |
958 | &layout, | |
959 | None, | |
960 | [ans.to_le_bytes()[0..op_size].to_vec(), is_overflow] | |
961 | .into_iter() | |
962 | .map(IntervalOrOwned::Owned), | |
963 | )?; | |
964 | destination.write_from_bytes(self, &result) | |
965 | } | |
966 | "copy" | "copy_nonoverlapping" => { | |
967 | let [src, dst, offset] = args else { | |
add651ee FG |
968 | return Err(MirEvalError::TypeError( |
969 | "copy_nonoverlapping args are not provided", | |
970 | )); | |
fe692bf9 | 971 | }; |
781aab86 FG |
972 | let Some(ty) = |
973 | generic_args.as_slice(Interner).get(0).and_then(|it| it.ty(Interner)) | |
add651ee FG |
974 | else { |
975 | return Err(MirEvalError::TypeError( | |
976 | "copy_nonoverlapping generic arg is not provided", | |
977 | )); | |
fe692bf9 FG |
978 | }; |
979 | let src = Address::from_bytes(src.get(self)?)?; | |
980 | let dst = Address::from_bytes(dst.get(self)?)?; | |
981 | let offset = from_bytes!(usize, offset.get(self)?); | |
982 | let size = self.size_of_sized(ty, locals, "copy_nonoverlapping ptr type")?; | |
983 | let size = offset * size; | |
984 | let src = Interval { addr: src, size }; | |
985 | let dst = Interval { addr: dst, size }; | |
986 | dst.write_from_interval(self, src) | |
987 | } | |
988 | "offset" | "arith_offset" => { | |
989 | let [ptr, offset] = args else { | |
990 | return Err(MirEvalError::TypeError("offset args are not provided")); | |
991 | }; | |
781aab86 FG |
992 | let ty = if name == "offset" { |
993 | let Some(ty0) = | |
994 | generic_args.as_slice(Interner).get(0).and_then(|it| it.ty(Interner)) | |
995 | else { | |
996 | return Err(MirEvalError::TypeError("offset generic arg is not provided")); | |
997 | }; | |
998 | let Some(ty1) = | |
999 | generic_args.as_slice(Interner).get(1).and_then(|it| it.ty(Interner)) | |
1000 | else { | |
1001 | return Err(MirEvalError::TypeError("offset generic arg is not provided")); | |
1002 | }; | |
1003 | if !matches!( | |
1004 | ty1.as_builtin(), | |
1005 | Some( | |
1006 | BuiltinType::Int(BuiltinInt::Isize) | |
1007 | | BuiltinType::Uint(BuiltinUint::Usize) | |
1008 | ) | |
1009 | ) { | |
1010 | return Err(MirEvalError::TypeError( | |
1011 | "offset generic arg is not usize or isize", | |
1012 | )); | |
1013 | } | |
1014 | match ty0.as_raw_ptr() { | |
1015 | Some((ty, _)) => ty, | |
1016 | None => { | |
1017 | return Err(MirEvalError::TypeError( | |
1018 | "offset generic arg is not a raw pointer", | |
1019 | )); | |
1020 | } | |
1021 | } | |
1022 | } else { | |
1023 | let Some(ty) = | |
1024 | generic_args.as_slice(Interner).get(0).and_then(|it| it.ty(Interner)) | |
1025 | else { | |
1026 | return Err(MirEvalError::TypeError( | |
1027 | "arith_offset generic arg is not provided", | |
1028 | )); | |
1029 | }; | |
1030 | ty | |
fe692bf9 FG |
1031 | }; |
1032 | let ptr = u128::from_le_bytes(pad16(ptr.get(self)?, false)); | |
1033 | let offset = u128::from_le_bytes(pad16(offset.get(self)?, false)); | |
1034 | let size = self.size_of_sized(ty, locals, "offset ptr type")? as u128; | |
1035 | let ans = ptr + offset * size; | |
1036 | destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]) | |
1037 | } | |
1038 | "assert_inhabited" | "assert_zero_valid" | "assert_uninit_valid" | "assume" => { | |
1039 | // FIXME: We should actually implement these checks | |
1040 | Ok(()) | |
1041 | } | |
1042 | "forget" => { | |
1043 | // We don't call any drop glue yet, so there is nothing here | |
1044 | Ok(()) | |
1045 | } | |
1046 | "transmute" => { | |
1047 | let [arg] = args else { | |
4b012472 | 1048 | return Err(MirEvalError::TypeError("transmute arg is not provided")); |
fe692bf9 FG |
1049 | }; |
1050 | destination.write_from_interval(self, arg.interval) | |
1051 | } | |
1052 | "likely" | "unlikely" => { | |
1053 | let [arg] = args else { | |
1054 | return Err(MirEvalError::TypeError("likely arg is not provided")); | |
1055 | }; | |
1056 | destination.write_from_interval(self, arg.interval) | |
1057 | } | |
1058 | "ctpop" => { | |
1059 | let [arg] = args else { | |
add651ee | 1060 | return Err(MirEvalError::TypeError("ctpop arg is not provided")); |
fe692bf9 FG |
1061 | }; |
1062 | let result = u128::from_le_bytes(pad16(arg.get(self)?, false)).count_ones(); | |
1063 | destination | |
1064 | .write_from_bytes(self, &(result as u128).to_le_bytes()[0..destination.size]) | |
1065 | } | |
add651ee FG |
1066 | "ctlz" | "ctlz_nonzero" => { |
1067 | let [arg] = args else { | |
4b012472 | 1068 | return Err(MirEvalError::TypeError("ctlz arg is not provided")); |
add651ee FG |
1069 | }; |
1070 | let result = | |
1071 | u128::from_le_bytes(pad16(arg.get(self)?, false)).leading_zeros() as usize; | |
1072 | let result = result - (128 - arg.interval.size * 8); | |
1073 | destination | |
1074 | .write_from_bytes(self, &(result as u128).to_le_bytes()[0..destination.size]) | |
1075 | } | |
fe692bf9 FG |
1076 | "cttz" | "cttz_nonzero" => { |
1077 | let [arg] = args else { | |
add651ee | 1078 | return Err(MirEvalError::TypeError("cttz arg is not provided")); |
fe692bf9 FG |
1079 | }; |
1080 | let result = u128::from_le_bytes(pad16(arg.get(self)?, false)).trailing_zeros(); | |
1081 | destination | |
1082 | .write_from_bytes(self, &(result as u128).to_le_bytes()[0..destination.size]) | |
1083 | } | |
add651ee FG |
1084 | "rotate_left" => { |
1085 | let [lhs, rhs] = args else { | |
1086 | return Err(MirEvalError::TypeError("rotate_left args are not provided")); | |
1087 | }; | |
1088 | let lhs = &lhs.get(self)?[0..destination.size]; | |
1089 | let rhs = rhs.get(self)?[0] as u32; | |
1090 | match destination.size { | |
1091 | 1 => { | |
1092 | let r = from_bytes!(u8, lhs).rotate_left(rhs); | |
1093 | destination.write_from_bytes(self, &r.to_le_bytes()) | |
1094 | } | |
1095 | 2 => { | |
1096 | let r = from_bytes!(u16, lhs).rotate_left(rhs); | |
1097 | destination.write_from_bytes(self, &r.to_le_bytes()) | |
1098 | } | |
1099 | 4 => { | |
1100 | let r = from_bytes!(u32, lhs).rotate_left(rhs); | |
1101 | destination.write_from_bytes(self, &r.to_le_bytes()) | |
1102 | } | |
1103 | 8 => { | |
1104 | let r = from_bytes!(u64, lhs).rotate_left(rhs); | |
1105 | destination.write_from_bytes(self, &r.to_le_bytes()) | |
1106 | } | |
1107 | 16 => { | |
1108 | let r = from_bytes!(u128, lhs).rotate_left(rhs); | |
1109 | destination.write_from_bytes(self, &r.to_le_bytes()) | |
1110 | } | |
1111 | s => not_supported!("destination with size {s} for rotate_left"), | |
1112 | } | |
1113 | } | |
1114 | "rotate_right" => { | |
1115 | let [lhs, rhs] = args else { | |
1116 | return Err(MirEvalError::TypeError("rotate_right args are not provided")); | |
1117 | }; | |
1118 | let lhs = &lhs.get(self)?[0..destination.size]; | |
1119 | let rhs = rhs.get(self)?[0] as u32; | |
1120 | match destination.size { | |
1121 | 1 => { | |
1122 | let r = from_bytes!(u8, lhs).rotate_right(rhs); | |
1123 | destination.write_from_bytes(self, &r.to_le_bytes()) | |
1124 | } | |
1125 | 2 => { | |
1126 | let r = from_bytes!(u16, lhs).rotate_right(rhs); | |
1127 | destination.write_from_bytes(self, &r.to_le_bytes()) | |
1128 | } | |
1129 | 4 => { | |
1130 | let r = from_bytes!(u32, lhs).rotate_right(rhs); | |
1131 | destination.write_from_bytes(self, &r.to_le_bytes()) | |
1132 | } | |
1133 | 8 => { | |
1134 | let r = from_bytes!(u64, lhs).rotate_right(rhs); | |
1135 | destination.write_from_bytes(self, &r.to_le_bytes()) | |
1136 | } | |
1137 | 16 => { | |
1138 | let r = from_bytes!(u128, lhs).rotate_right(rhs); | |
1139 | destination.write_from_bytes(self, &r.to_le_bytes()) | |
1140 | } | |
1141 | s => not_supported!("destination with size {s} for rotate_right"), | |
1142 | } | |
1143 | } | |
1144 | "discriminant_value" => { | |
1145 | let [arg] = args else { | |
1146 | return Err(MirEvalError::TypeError("discriminant_value arg is not provided")); | |
1147 | }; | |
781aab86 FG |
1148 | let Some(ty) = |
1149 | generic_args.as_slice(Interner).get(0).and_then(|it| it.ty(Interner)) | |
add651ee FG |
1150 | else { |
1151 | return Err(MirEvalError::TypeError( | |
1152 | "discriminant_value generic arg is not provided", | |
1153 | )); | |
1154 | }; | |
1155 | let addr = Address::from_bytes(arg.get(self)?)?; | |
1156 | let size = self.size_of_sized(ty, locals, "discriminant_value ptr type")?; | |
1157 | let interval = Interval { addr, size }; | |
1158 | let r = self.compute_discriminant(ty.clone(), interval.get(self)?)?; | |
1159 | destination.write_from_bytes(self, &r.to_le_bytes()[0..destination.size]) | |
1160 | } | |
fe692bf9 FG |
1161 | "const_eval_select" => { |
1162 | let [tuple, const_fn, _] = args else { | |
1163 | return Err(MirEvalError::TypeError("const_eval_select args are not provided")); | |
1164 | }; | |
1165 | let mut args = vec![const_fn.clone()]; | |
1166 | let TyKind::Tuple(_, fields) = tuple.ty.kind(Interner) else { | |
1167 | return Err(MirEvalError::TypeError("const_eval_select arg[0] is not a tuple")); | |
1168 | }; | |
1169 | let layout = self.layout(&tuple.ty)?; | |
1170 | for (i, field) in fields.iter(Interner).enumerate() { | |
1171 | let field = field.assert_ty_ref(Interner).clone(); | |
1172 | let offset = layout.fields.offset(i).bytes_usize(); | |
1173 | let addr = tuple.interval.addr.offset(offset); | |
1174 | args.push(IntervalAndTy::new(addr, field, self, locals)?); | |
1175 | } | |
add651ee FG |
1176 | if let Some(target) = self.db.lang_item(self.crate_id, LangItem::FnOnce) { |
1177 | if let Some(def) = target | |
1178 | .as_trait() | |
1179 | .and_then(|it| self.db.trait_data(it).method_by_name(&name![call_once])) | |
1180 | { | |
1181 | self.exec_fn_trait( | |
1182 | def, | |
1183 | &args, | |
1184 | // FIXME: wrong for manual impls of `FnOnce` | |
1185 | Substitution::empty(Interner), | |
1186 | locals, | |
1187 | destination, | |
1188 | None, | |
1189 | span, | |
1190 | )?; | |
1191 | return Ok(()); | |
1192 | } | |
1193 | } | |
1194 | not_supported!("FnOnce was not available for executing const_eval_select"); | |
1195 | } | |
1196 | "read_via_copy" | "volatile_load" => { | |
1197 | let [arg] = args else { | |
1198 | return Err(MirEvalError::TypeError("read_via_copy args are not provided")); | |
1199 | }; | |
1200 | let addr = Address::from_bytes(arg.interval.get(self)?)?; | |
1201 | destination.write_from_interval(self, Interval { addr, size: destination.size }) | |
1202 | } | |
781aab86 FG |
1203 | "write_via_move" => { |
1204 | let [ptr, val] = args else { | |
1205 | return Err(MirEvalError::TypeError("write_via_move args are not provided")); | |
1206 | }; | |
1207 | let dst = Address::from_bytes(ptr.get(self)?)?; | |
1208 | let Some(ty) = | |
1209 | generic_args.as_slice(Interner).get(0).and_then(|it| it.ty(Interner)) | |
1210 | else { | |
1211 | return Err(MirEvalError::TypeError( | |
1212 | "write_via_copy generic arg is not provided", | |
1213 | )); | |
1214 | }; | |
1215 | let size = self.size_of_sized(ty, locals, "write_via_move ptr type")?; | |
1216 | Interval { addr: dst, size }.write_from_interval(self, val.interval)?; | |
1217 | Ok(()) | |
1218 | } | |
add651ee FG |
1219 | "write_bytes" => { |
1220 | let [dst, val, count] = args else { | |
1221 | return Err(MirEvalError::TypeError("write_bytes args are not provided")); | |
1222 | }; | |
1223 | let count = from_bytes!(usize, count.get(self)?); | |
1224 | let val = from_bytes!(u8, val.get(self)?); | |
781aab86 FG |
1225 | let Some(ty) = |
1226 | generic_args.as_slice(Interner).get(0).and_then(|it| it.ty(Interner)) | |
add651ee | 1227 | else { |
781aab86 | 1228 | return Err(MirEvalError::TypeError("write_bytes generic arg is not provided")); |
add651ee FG |
1229 | }; |
1230 | let dst = Address::from_bytes(dst.get(self)?)?; | |
1231 | let size = self.size_of_sized(ty, locals, "copy_nonoverlapping ptr type")?; | |
1232 | let size = count * size; | |
1233 | self.write_memory_using_ref(dst, size)?.fill(val); | |
1234 | Ok(()) | |
fe692bf9 FG |
1235 | } |
1236 | _ => not_supported!("unknown intrinsic {name}"), | |
1237 | } | |
1238 | } | |
1239 | ||
add651ee FG |
1240 | fn size_align_of_unsized( |
1241 | &mut self, | |
1242 | ty: &Ty, | |
1243 | metadata: Interval, | |
1244 | locals: &Locals, | |
1245 | ) -> Result<(usize, usize)> { | |
1246 | Ok(match ty.kind(Interner) { | |
1247 | TyKind::Str => (from_bytes!(usize, metadata.get(self)?), 1), | |
1248 | TyKind::Slice(inner) => { | |
1249 | let len = from_bytes!(usize, metadata.get(self)?); | |
1250 | let (size, align) = self.size_align_of_sized(inner, locals, "slice inner type")?; | |
1251 | (size * len, align) | |
1252 | } | |
1253 | TyKind::Dyn(_) => self.size_align_of_sized( | |
1254 | self.vtable_map.ty_of_bytes(metadata.get(self)?)?, | |
1255 | locals, | |
1256 | "dyn concrete type", | |
1257 | )?, | |
1258 | TyKind::Adt(id, subst) => { | |
1259 | let id = id.0; | |
1260 | let layout = self.layout_adt(id, subst.clone())?; | |
1261 | let id = match id { | |
1262 | AdtId::StructId(s) => s, | |
1263 | _ => not_supported!("unsized enum or union"), | |
1264 | }; | |
1265 | let field_types = &self.db.field_types(id.into()); | |
1266 | let last_field_ty = | |
1267 | field_types.iter().rev().next().unwrap().1.clone().substitute(Interner, subst); | |
1268 | let sized_part_size = | |
1269 | layout.fields.offset(field_types.iter().count() - 1).bytes_usize(); | |
1270 | let sized_part_align = layout.align.abi.bytes() as usize; | |
1271 | let (unsized_part_size, unsized_part_align) = | |
1272 | self.size_align_of_unsized(&last_field_ty, metadata, locals)?; | |
1273 | let align = sized_part_align.max(unsized_part_align) as isize; | |
1274 | let size = (sized_part_size + unsized_part_size) as isize; | |
1275 | // Must add any necessary padding to `size` | |
1276 | // (to make it a multiple of `align`) before returning it. | |
1277 | // | |
1278 | // Namely, the returned size should be, in C notation: | |
1279 | // | |
1280 | // `size + ((size & (align-1)) ? align : 0)` | |
1281 | // | |
1282 | // emulated via the semi-standard fast bit trick: | |
1283 | // | |
1284 | // `(size + (align-1)) & -align` | |
1285 | let size = (size + (align - 1)) & (-align); | |
1286 | (size as usize, align as usize) | |
1287 | } | |
1288 | _ => not_supported!("unsized type other than str, slice, struct and dyn"), | |
1289 | }) | |
1290 | } | |
1291 | ||
fe692bf9 FG |
1292 | fn exec_atomic_intrinsic( |
1293 | &mut self, | |
1294 | name: &str, | |
1295 | args: &[IntervalAndTy], | |
1296 | generic_args: &Substitution, | |
1297 | destination: Interval, | |
add651ee | 1298 | locals: &Locals, |
fe692bf9 FG |
1299 | _span: MirSpan, |
1300 | ) -> Result<()> { | |
1301 | // We are a single threaded runtime with no UB checking and no optimization, so | |
add651ee FG |
1302 | // we can implement atomic intrinsics as normal functions. |
1303 | ||
1304 | if name.starts_with("singlethreadfence_") || name.starts_with("fence_") { | |
1305 | return Ok(()); | |
1306 | } | |
1307 | ||
1308 | // The rest of atomic intrinsics have exactly one generic arg | |
1309 | ||
1310 | let Some(ty) = generic_args.as_slice(Interner).get(0).and_then(|it| it.ty(Interner)) else { | |
fe692bf9 FG |
1311 | return Err(MirEvalError::TypeError("atomic intrinsic generic arg is not provided")); |
1312 | }; | |
1313 | let Some(arg0) = args.get(0) else { | |
1314 | return Err(MirEvalError::TypeError("atomic intrinsic arg0 is not provided")); | |
1315 | }; | |
1316 | let arg0_addr = Address::from_bytes(arg0.get(self)?)?; | |
1317 | let arg0_interval = | |
1318 | Interval::new(arg0_addr, self.size_of_sized(ty, locals, "atomic intrinsic type arg")?); | |
1319 | if name.starts_with("load_") { | |
1320 | return destination.write_from_interval(self, arg0_interval); | |
1321 | } | |
1322 | let Some(arg1) = args.get(1) else { | |
1323 | return Err(MirEvalError::TypeError("atomic intrinsic arg1 is not provided")); | |
1324 | }; | |
1325 | if name.starts_with("store_") { | |
1326 | return arg0_interval.write_from_interval(self, arg1.interval); | |
1327 | } | |
1328 | if name.starts_with("xchg_") { | |
1329 | destination.write_from_interval(self, arg0_interval)?; | |
1330 | return arg0_interval.write_from_interval(self, arg1.interval); | |
1331 | } | |
1332 | if name.starts_with("xadd_") { | |
1333 | destination.write_from_interval(self, arg0_interval)?; | |
1334 | let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false)); | |
1335 | let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false)); | |
1336 | let ans = lhs.wrapping_add(rhs); | |
1337 | return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]); | |
1338 | } | |
1339 | if name.starts_with("xsub_") { | |
1340 | destination.write_from_interval(self, arg0_interval)?; | |
1341 | let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false)); | |
1342 | let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false)); | |
1343 | let ans = lhs.wrapping_sub(rhs); | |
1344 | return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]); | |
1345 | } | |
1346 | if name.starts_with("and_") { | |
1347 | destination.write_from_interval(self, arg0_interval)?; | |
1348 | let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false)); | |
1349 | let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false)); | |
1350 | let ans = lhs & rhs; | |
1351 | return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]); | |
1352 | } | |
1353 | if name.starts_with("or_") { | |
1354 | destination.write_from_interval(self, arg0_interval)?; | |
1355 | let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false)); | |
1356 | let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false)); | |
1357 | let ans = lhs | rhs; | |
1358 | return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]); | |
1359 | } | |
1360 | if name.starts_with("xor_") { | |
1361 | destination.write_from_interval(self, arg0_interval)?; | |
1362 | let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false)); | |
1363 | let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false)); | |
1364 | let ans = lhs ^ rhs; | |
1365 | return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]); | |
1366 | } | |
1367 | if name.starts_with("nand_") { | |
1368 | destination.write_from_interval(self, arg0_interval)?; | |
1369 | let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false)); | |
1370 | let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false)); | |
1371 | let ans = !(lhs & rhs); | |
1372 | return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]); | |
1373 | } | |
1374 | let Some(arg2) = args.get(2) else { | |
1375 | return Err(MirEvalError::TypeError("atomic intrinsic arg2 is not provided")); | |
1376 | }; | |
1377 | if name.starts_with("cxchg_") || name.starts_with("cxchgweak_") { | |
1378 | let dest = if arg1.get(self)? == arg0_interval.get(self)? { | |
1379 | arg0_interval.write_from_interval(self, arg2.interval)?; | |
1380 | (arg1.interval, true) | |
1381 | } else { | |
1382 | (arg0_interval, false) | |
1383 | }; | |
1384 | let result_ty = TyKind::Tuple( | |
1385 | 2, | |
1386 | Substitution::from_iter(Interner, [ty.clone(), TyBuilder::bool()]), | |
1387 | ) | |
1388 | .intern(Interner); | |
1389 | let layout = self.layout(&result_ty)?; | |
1390 | let result = self.make_by_layout( | |
1391 | layout.size.bytes_usize(), | |
1392 | &layout, | |
1393 | None, | |
1394 | [IntervalOrOwned::Borrowed(dest.0), IntervalOrOwned::Owned(vec![u8::from(dest.1)])] | |
1395 | .into_iter(), | |
1396 | )?; | |
1397 | return destination.write_from_bytes(self, &result); | |
1398 | } | |
1399 | not_supported!("unknown atomic intrinsic {name}"); | |
1400 | } | |
1401 | } |