hir_ty/mir/eval/
shim.rs

1//! Interpret intrinsics, lang items and `extern "C"` wellknown functions which their implementation
2//! is not available.
3//!
4use std::cmp::{self, Ordering};
5
6use hir_def::{attrs::AttrFlags, signatures::FunctionSignature};
7use rustc_abi::ExternAbi;
8use rustc_type_ir::inherent::{GenericArgs as _, IntoKind, SliceLike, Ty as _};
9use stdx::never;
10
11use crate::{
12    drop::{DropGlue, has_drop_glue},
13    mir::eval::{
14        Address, AdtId, Arc, Evaluator, FunctionId, GenericArgs, HasModule, Interval,
15        IntervalAndTy, IntervalOrOwned, IsSigned, ItemContainerId, Layout, Locals, Lookup,
16        MirEvalError, MirSpan, Mutability, Result, Ty, TyKind, from_bytes, not_supported, pad16,
17    },
18    next_solver::Region,
19};
20
21mod simd;
22
23#[derive(Debug, Clone, Copy, PartialEq, Eq)]
24enum EvalLangItem {
25    BeginPanic,
26    SliceLen,
27    DropInPlace,
28}
29
30impl<'a, 'db> Evaluator<'a, 'db> {
31    pub(super) fn detect_and_exec_special_function(
32        &mut self,
33        def: FunctionId,
34        args: &[IntervalAndTy<'db>],
35        generic_args: GenericArgs<'db>,
36        locals: &Locals<'a, 'db>,
37        destination: Interval,
38        span: MirSpan,
39    ) -> Result<'db, bool> {
40        if self.not_special_fn_cache.borrow().contains(&def) {
41            return Ok(false);
42        }
43
44        let function_data = FunctionSignature::of(self.db, def);
45        let attrs = AttrFlags::query(self.db, def.into());
46        let is_intrinsic = FunctionSignature::is_intrinsic(self.db, def);
47
48        if is_intrinsic {
49            return self.exec_intrinsic(
50                function_data.name.as_str(),
51                args,
52                generic_args,
53                destination,
54                locals,
55                span,
56                !function_data.has_body()
57                    || attrs.contains(AttrFlags::RUSTC_INTRINSIC_MUST_BE_OVERRIDDEN),
58            );
59        }
60        let is_extern_c = match def.lookup(self.db).container {
61            hir_def::ItemContainerId::ExternBlockId(block) => {
62                matches!(block.abi(self.db), ExternAbi::C { .. })
63            }
64            _ => false,
65        };
66        if is_extern_c {
67            return self
68                .exec_extern_c(
69                    function_data.name.as_str(),
70                    args,
71                    generic_args,
72                    destination,
73                    locals,
74                    span,
75                )
76                .map(|()| true);
77        }
78
79        if attrs.intersects(
80            AttrFlags::RUSTC_ALLOCATOR
81                | AttrFlags::RUSTC_DEALLOCATOR
82                | AttrFlags::RUSTC_REALLOCATOR
83                | AttrFlags::RUSTC_ALLOCATOR_ZEROED,
84        ) {
85            self.exec_alloc_fn(attrs, args, destination)?;
86            return Ok(true);
87        }
88        if let Some(it) = self.detect_lang_function(def) {
89            let result = self.exec_lang_item(it, generic_args, args, locals, span)?;
90            destination.write_from_bytes(self, &result)?;
91            return Ok(true);
92        }
93        if let ItemContainerId::TraitId(t) = def.lookup(self.db).container
94            && Some(t) == self.lang_items().Clone
95        {
96            let [self_ty] = generic_args.as_slice() else {
97                not_supported!("wrong generic arg count for clone");
98            };
99            let Some(self_ty) = self_ty.ty() else {
100                not_supported!("wrong generic arg kind for clone");
101            };
102            // Clone has special impls for tuples and function pointers
103            if matches!(self_ty.kind(), TyKind::FnPtr(..) | TyKind::Tuple(..) | TyKind::Closure(..))
104            {
105                self.exec_clone(def, args, self_ty, locals, destination, span)?;
106                return Ok(true);
107            }
108            // Return early to prevent caching clone as non special fn.
109            return Ok(false);
110        }
111        self.not_special_fn_cache.borrow_mut().insert(def);
112        Ok(false)
113    }
114
115    pub(super) fn detect_and_redirect_special_function(
116        &mut self,
117        def: FunctionId,
118    ) -> Result<'db, Option<FunctionId>> {
119        // `PanicFmt` is redirected to `ConstPanicFmt`
120        if Some(def) == self.lang_items().PanicFmt {
121            let Some(const_panic_fmt) = self.lang_items().ConstPanicFmt else {
122                not_supported!("const_panic_fmt lang item not found or not a function");
123            };
124            return Ok(Some(const_panic_fmt));
125        }
126        Ok(None)
127    }
128
129    /// Clone has special impls for tuples and function pointers
130    fn exec_clone(
131        &mut self,
132        def: FunctionId,
133        args: &[IntervalAndTy<'db>],
134        self_ty: Ty<'db>,
135        locals: &Locals<'a, 'db>,
136        destination: Interval,
137        span: MirSpan,
138    ) -> Result<'db, ()> {
139        match self_ty.kind() {
140            TyKind::FnPtr(..) => {
141                let [arg] = args else {
142                    not_supported!("wrong arg count for clone");
143                };
144                let addr = Address::from_bytes(arg.get(self)?)?;
145                return destination
146                    .write_from_interval(self, Interval { addr, size: destination.size });
147            }
148            TyKind::Closure(_, closure_args) => self.exec_clone(
149                def,
150                args,
151                closure_args.as_closure().tupled_upvars_ty(),
152                locals,
153                destination,
154                span,
155            )?,
156            TyKind::Tuple(subst) => {
157                let [arg] = args else {
158                    not_supported!("wrong arg count for clone");
159                };
160                let addr = Address::from_bytes(arg.get(self)?)?;
161                let layout = self.layout(self_ty)?;
162                self.exec_clone_for_fields(
163                    subst.iter(),
164                    layout,
165                    addr,
166                    def,
167                    locals,
168                    destination,
169                    span,
170                )?;
171            }
172            _ => {
173                self.exec_fn_with_args(
174                    def,
175                    args,
176                    GenericArgs::new_from_slice(&[self_ty.into()]),
177                    locals,
178                    destination,
179                    None,
180                    span,
181                )?;
182            }
183        }
184        Ok(())
185    }
186
187    fn exec_clone_for_fields(
188        &mut self,
189        ty_iter: impl Iterator<Item = Ty<'db>>,
190        layout: Arc<Layout>,
191        addr: Address,
192        def: FunctionId,
193        locals: &Locals<'a, 'db>,
194        destination: Interval,
195        span: MirSpan,
196    ) -> Result<'db, ()> {
197        for (i, ty) in ty_iter.enumerate() {
198            let size = self.layout(ty)?.size.bytes_usize();
199            let tmp = self.heap_allocate(self.ptr_size(), self.ptr_size())?;
200            let arg = IntervalAndTy {
201                interval: Interval { addr: tmp, size: self.ptr_size() },
202                ty: Ty::new_ref(
203                    self.interner(),
204                    Region::error(self.interner()),
205                    ty,
206                    Mutability::Not,
207                ),
208            };
209            let offset = layout.fields.offset(i).bytes_usize();
210            self.write_memory(tmp, &addr.offset(offset).to_bytes())?;
211            self.exec_clone(
212                def,
213                &[arg],
214                ty,
215                locals,
216                destination.slice(offset..offset + size),
217                span,
218            )?;
219        }
220        Ok(())
221    }
222
223    fn exec_alloc_fn(
224        &mut self,
225        alloc_fn: AttrFlags,
226        args: &[IntervalAndTy<'db>],
227        destination: Interval,
228    ) -> Result<'db, ()> {
229        match alloc_fn {
230            _ if alloc_fn
231                .intersects(AttrFlags::RUSTC_ALLOCATOR_ZEROED | AttrFlags::RUSTC_ALLOCATOR) =>
232            {
233                let [size, align] = args else {
234                    return Err(MirEvalError::InternalError(
235                        "rustc_allocator args are not provided".into(),
236                    ));
237                };
238                let size = from_bytes!(usize, size.get(self)?);
239                let align = from_bytes!(usize, align.get(self)?);
240                let result = self.heap_allocate(size, align)?;
241                destination.write_from_bytes(self, &result.to_bytes())?;
242            }
243            _ if alloc_fn.contains(AttrFlags::RUSTC_DEALLOCATOR) => { /* no-op for now */ }
244            _ if alloc_fn.contains(AttrFlags::RUSTC_REALLOCATOR) => {
245                let [ptr, old_size, align, new_size] = args else {
246                    return Err(MirEvalError::InternalError(
247                        "rustc_allocator args are not provided".into(),
248                    ));
249                };
250                let old_size = from_bytes!(usize, old_size.get(self)?);
251                let new_size = from_bytes!(usize, new_size.get(self)?);
252                if old_size >= new_size {
253                    destination.write_from_interval(self, ptr.interval)?;
254                } else {
255                    let ptr = Address::from_bytes(ptr.get(self)?)?;
256                    let align = from_bytes!(usize, align.get(self)?);
257                    let result = self.heap_allocate(new_size, align)?;
258                    Interval { addr: result, size: old_size }
259                        .write_from_interval(self, Interval { addr: ptr, size: old_size })?;
260                    destination.write_from_bytes(self, &result.to_bytes())?;
261                }
262            }
263            _ => not_supported!("unknown alloc function"),
264        }
265        Ok(())
266    }
267
268    fn detect_lang_function(&self, def: FunctionId) -> Option<EvalLangItem> {
269        use EvalLangItem::*;
270        let lang_items = self.lang_items();
271        let attrs = AttrFlags::query(self.db, def.into());
272
273        if attrs.contains(AttrFlags::RUSTC_CONST_PANIC_STR) {
274            // `#[rustc_const_panic_str]` is treated like `lang = "begin_panic"` by rustc CTFE.
275            return Some(BeginPanic);
276        }
277
278        // We want to execute these functions with special logic
279        // `PanicFmt` is not detected here as it's redirected later.
280        if let Some((_, candidate)) = [
281            (lang_items.BeginPanic, BeginPanic),
282            (lang_items.SliceLen, SliceLen),
283            (lang_items.DropInPlace, DropInPlace),
284        ]
285        .iter()
286        .find(|&(candidate, _)| candidate == Some(def))
287        {
288            return Some(candidate);
289        }
290
291        None
292    }
293
294    fn exec_lang_item(
295        &mut self,
296        it: EvalLangItem,
297        generic_args: GenericArgs<'db>,
298        args: &[IntervalAndTy<'db>],
299        locals: &Locals<'a, 'db>,
300        span: MirSpan,
301    ) -> Result<'db, Vec<u8>> {
302        use EvalLangItem::*;
303        let mut args = args.iter();
304        match it {
305            BeginPanic => {
306                let mut arg = args
307                    .next()
308                    .ok_or(MirEvalError::InternalError(
309                        "argument of BeginPanic is not provided".into(),
310                    ))?
311                    .clone();
312                while let TyKind::Ref(_, ty, _) = arg.ty.kind() {
313                    if ty.is_str() {
314                        let (pointee, metadata) = arg.interval.get(self)?.split_at(self.ptr_size());
315                        let len = from_bytes!(usize, metadata);
316
317                        return {
318                            Err(MirEvalError::Panic(
319                                std::str::from_utf8(
320                                    self.read_memory(Address::from_bytes(pointee)?, len)?,
321                                )
322                                .unwrap()
323                                .to_owned(),
324                            ))
325                        };
326                    }
327                    let size = self.size_of_sized(ty, locals, "begin panic arg")?;
328                    let pointee = arg.interval.get(self)?;
329                    arg = IntervalAndTy {
330                        interval: Interval::new(Address::from_bytes(pointee)?, size),
331                        ty,
332                    };
333                }
334                Err(MirEvalError::Panic(format!("unknown-panic-payload: {:?}", arg.ty.kind())))
335            }
336            SliceLen => {
337                let arg = args.next().ok_or(MirEvalError::InternalError(
338                    "argument of <[T]>::len() is not provided".into(),
339                ))?;
340                let arg = arg.get(self)?;
341                let ptr_size = arg.len() / 2;
342                Ok(arg[ptr_size..].into())
343            }
344            DropInPlace => {
345                let ty = generic_args.as_slice().first().and_then(|it| it.ty()).ok_or(
346                    MirEvalError::InternalError(
347                        "generic argument of drop_in_place is not provided".into(),
348                    ),
349                )?;
350                let arg = args.next().ok_or(MirEvalError::InternalError(
351                    "argument of drop_in_place is not provided".into(),
352                ))?;
353                let arg = arg.interval.get(self)?.to_owned();
354                self.run_drop_glue_deep(
355                    ty,
356                    locals,
357                    Address::from_bytes(&arg[0..self.ptr_size()])?,
358                    &arg[self.ptr_size()..],
359                    span,
360                )?;
361                Ok(vec![])
362            }
363        }
364    }
365
366    fn exec_syscall(
367        &mut self,
368        id: i64,
369        args: &[IntervalAndTy<'db>],
370        destination: Interval,
371        _locals: &Locals<'a, 'db>,
372        _span: MirSpan,
373    ) -> Result<'db, ()> {
374        match id {
375            318 => {
376                // SYS_getrandom
377                let [buf, len, _flags] = args else {
378                    return Err(MirEvalError::InternalError(
379                        "SYS_getrandom args are not provided".into(),
380                    ));
381                };
382                let addr = Address::from_bytes(buf.get(self)?)?;
383                let size = from_bytes!(usize, len.get(self)?);
384                for i in 0..size {
385                    let rand_byte = self.random_state.rand_u64() as u8;
386                    self.write_memory(addr.offset(i), &[rand_byte])?;
387                }
388                destination.write_from_interval(self, len.interval)
389            }
390            _ => {
391                not_supported!("Unknown syscall id {id:?}")
392            }
393        }
394    }
395
396    fn exec_extern_c(
397        &mut self,
398        as_str: &str,
399        args: &[IntervalAndTy<'db>],
400        _generic_args: GenericArgs<'db>,
401        destination: Interval,
402        locals: &Locals<'a, 'db>,
403        span: MirSpan,
404    ) -> Result<'db, ()> {
405        match as_str {
406            "memcmp" => {
407                let [ptr1, ptr2, size] = args else {
408                    return Err(MirEvalError::InternalError("memcmp args are not provided".into()));
409                };
410                let addr1 = Address::from_bytes(ptr1.get(self)?)?;
411                let addr2 = Address::from_bytes(ptr2.get(self)?)?;
412                let size = from_bytes!(usize, size.get(self)?);
413                let slice1 = self.read_memory(addr1, size)?;
414                let slice2 = self.read_memory(addr2, size)?;
415                let r: i128 = match slice1.cmp(slice2) {
416                    cmp::Ordering::Less => -1,
417                    cmp::Ordering::Equal => 0,
418                    cmp::Ordering::Greater => 1,
419                };
420                destination.write_from_bytes(self, &r.to_le_bytes()[..destination.size])
421            }
422            "write" => {
423                let [fd, ptr, len] = args else {
424                    return Err(MirEvalError::InternalError(
425                        "libc::write args are not provided".into(),
426                    ));
427                };
428                let fd = u128::from_le_bytes(pad16(fd.get(self)?, IsSigned::No));
429                let interval = Interval {
430                    addr: Address::from_bytes(ptr.get(self)?)?,
431                    size: from_bytes!(usize, len.get(self)?),
432                };
433                match fd {
434                    1 => {
435                        self.write_to_stdout(interval)?;
436                    }
437                    2 => {
438                        self.write_to_stderr(interval)?;
439                    }
440                    _ => not_supported!("write to arbitrary file descriptor"),
441                }
442                destination.write_from_interval(self, len.interval)?;
443                Ok(())
444            }
445            "pthread_key_create" => {
446                let key = self.thread_local_storage.create_key();
447                let Some(arg0) = args.first() else {
448                    return Err(MirEvalError::InternalError(
449                        "pthread_key_create arg0 is not provided".into(),
450                    ));
451                };
452                let arg0_addr = Address::from_bytes(arg0.get(self)?)?;
453                let key_ty = if let Some((ty, ..)) = arg0.ty.as_reference_or_ptr() {
454                    ty
455                } else {
456                    return Err(MirEvalError::InternalError(
457                        "pthread_key_create arg0 is not a pointer".into(),
458                    ));
459                };
460                let arg0_interval = Interval::new(
461                    arg0_addr,
462                    self.size_of_sized(key_ty, locals, "pthread_key_create key arg")?,
463                );
464                arg0_interval.write_from_bytes(self, &key.to_le_bytes()[0..arg0_interval.size])?;
465                // return 0 as success
466                destination.write_from_bytes(self, &0u64.to_le_bytes()[0..destination.size])?;
467                Ok(())
468            }
469            "pthread_getspecific" => {
470                let Some(arg0) = args.first() else {
471                    return Err(MirEvalError::InternalError(
472                        "pthread_getspecific arg0 is not provided".into(),
473                    ));
474                };
475                let key = from_bytes!(usize, &pad16(arg0.get(self)?, IsSigned::No)[0..8]);
476                let value = self.thread_local_storage.get_key(key)?;
477                destination.write_from_bytes(self, &value.to_le_bytes()[0..destination.size])?;
478                Ok(())
479            }
480            "pthread_setspecific" => {
481                let Some(arg0) = args.first() else {
482                    return Err(MirEvalError::InternalError(
483                        "pthread_setspecific arg0 is not provided".into(),
484                    ));
485                };
486                let key = from_bytes!(usize, &pad16(arg0.get(self)?, IsSigned::No)[0..8]);
487                let Some(arg1) = args.get(1) else {
488                    return Err(MirEvalError::InternalError(
489                        "pthread_setspecific arg1 is not provided".into(),
490                    ));
491                };
492                let value = from_bytes!(u128, pad16(arg1.get(self)?, IsSigned::No));
493                self.thread_local_storage.set_key(key, value)?;
494                // return 0 as success
495                destination.write_from_bytes(self, &0u64.to_le_bytes()[0..destination.size])?;
496                Ok(())
497            }
498            "pthread_key_delete" => {
499                // we ignore this currently
500                // return 0 as success
501                destination.write_from_bytes(self, &0u64.to_le_bytes()[0..destination.size])?;
502                Ok(())
503            }
504            "syscall" => {
505                let Some((id, rest)) = args.split_first() else {
506                    return Err(MirEvalError::InternalError("syscall arg1 is not provided".into()));
507                };
508                let id = from_bytes!(i64, id.get(self)?);
509                self.exec_syscall(id, rest, destination, locals, span)
510            }
511            "sched_getaffinity" => {
512                let [_pid, _set_size, set] = args else {
513                    return Err(MirEvalError::InternalError(
514                        "sched_getaffinity args are not provided".into(),
515                    ));
516                };
517                let set = Address::from_bytes(set.get(self)?)?;
518                // Only enable core 0 (we are single threaded anyway), which is bitset 0x0000001
519                self.write_memory(set, &[1])?;
520                // return 0 as success
521                self.write_memory_using_ref(destination.addr, destination.size)?.fill(0);
522                Ok(())
523            }
524            "getenv" => {
525                let [name] = args else {
526                    return Err(MirEvalError::InternalError("getenv args are not provided".into()));
527                };
528                let mut name_buf = vec![];
529                let name = {
530                    let mut index = Address::from_bytes(name.get(self)?)?;
531                    loop {
532                        let byte = self.read_memory(index, 1)?[0];
533                        index = index.offset(1);
534                        if byte == 0 {
535                            break;
536                        }
537                        name_buf.push(byte);
538                    }
539                    String::from_utf8_lossy(&name_buf)
540                };
541                let value = self.crate_id.env(self.db).get(&name);
542                match value {
543                    None => {
544                        // Write null as fail
545                        self.write_memory_using_ref(destination.addr, destination.size)?.fill(0);
546                    }
547                    Some(mut value) => {
548                        value.push('\0');
549                        let addr = self.heap_allocate(value.len(), 1)?;
550                        self.write_memory(addr, value.as_bytes())?;
551                        self.write_memory(destination.addr, &addr.to_bytes())?;
552                    }
553                }
554                Ok(())
555            }
556            _ => not_supported!("unknown external function {as_str}"),
557        }
558    }
559
560    fn exec_intrinsic(
561        &mut self,
562        name: &str,
563        args: &[IntervalAndTy<'db>],
564        generic_args: GenericArgs<'db>,
565        destination: Interval,
566        locals: &Locals<'a, 'db>,
567        span: MirSpan,
568        needs_override: bool,
569    ) -> Result<'db, bool> {
570        if let Some(name) = name.strip_prefix("atomic_") {
571            return self
572                .exec_atomic_intrinsic(name, args, generic_args, destination, locals, span)
573                .map(|()| true);
574        }
575        if let Some(name) = name.strip_prefix("simd_") {
576            return self
577                .exec_simd_intrinsic(name, args, generic_args, destination, locals, span)
578                .map(|()| true);
579        }
580        // FIXME(#17451): Add `f16` and `f128` intrinsics.
581        if let Some(name) = name.strip_suffix("f64") {
582            let result = match name {
583                "sqrt" | "sin" | "cos" | "exp" | "exp2" | "log" | "log10" | "log2" | "fabs"
584                | "floor" | "ceil" | "trunc" | "rint" | "nearbyint" | "round" | "roundeven" => {
585                    let [arg] = args else {
586                        return Err(MirEvalError::InternalError(
587                            "f64 intrinsic signature doesn't match fn (f64) -> f64".into(),
588                        ));
589                    };
590                    let arg = from_bytes!(f64, arg.get(self)?);
591                    match name {
592                        "sqrt" => arg.sqrt(),
593                        "sin" => arg.sin(),
594                        "cos" => arg.cos(),
595                        "exp" => arg.exp(),
596                        "exp2" => arg.exp2(),
597                        "log" => arg.ln(),
598                        "log10" => arg.log10(),
599                        "log2" => arg.log2(),
600                        "fabs" => arg.abs(),
601                        "floor" => arg.floor(),
602                        "ceil" => arg.ceil(),
603                        "trunc" => arg.trunc(),
604                        // FIXME: these rounds should be different, but only `.round()` is stable now.
605                        "rint" => arg.round(),
606                        "nearbyint" => arg.round(),
607                        "round" => arg.round(),
608                        "roundeven" => arg.round(),
609                        _ => unreachable!(),
610                    }
611                }
612                "pow" | "minnum" | "maxnum" | "copysign" => {
613                    let [arg1, arg2] = args else {
614                        return Err(MirEvalError::InternalError(
615                            "f64 intrinsic signature doesn't match fn (f64, f64) -> f64".into(),
616                        ));
617                    };
618                    let arg1 = from_bytes!(f64, arg1.get(self)?);
619                    let arg2 = from_bytes!(f64, arg2.get(self)?);
620                    match name {
621                        "pow" => arg1.powf(arg2),
622                        "minnum" => arg1.min(arg2),
623                        "maxnum" => arg1.max(arg2),
624                        "copysign" => arg1.copysign(arg2),
625                        _ => unreachable!(),
626                    }
627                }
628                "powi" => {
629                    let [arg1, arg2] = args else {
630                        return Err(MirEvalError::InternalError(
631                            "powif64 signature doesn't match fn (f64, i32) -> f64".into(),
632                        ));
633                    };
634                    let arg1 = from_bytes!(f64, arg1.get(self)?);
635                    let arg2 = from_bytes!(i32, arg2.get(self)?);
636                    arg1.powi(arg2)
637                }
638                "fma" => {
639                    let [arg1, arg2, arg3] = args else {
640                        return Err(MirEvalError::InternalError(
641                            "fmaf64 signature doesn't match fn (f64, f64, f64) -> f64".into(),
642                        ));
643                    };
644                    let arg1 = from_bytes!(f64, arg1.get(self)?);
645                    let arg2 = from_bytes!(f64, arg2.get(self)?);
646                    let arg3 = from_bytes!(f64, arg3.get(self)?);
647                    arg1.mul_add(arg2, arg3)
648                }
649                _ => not_supported!("unknown f64 intrinsic {name}"),
650            };
651            return destination.write_from_bytes(self, &result.to_le_bytes()).map(|()| true);
652        }
653        if let Some(name) = name.strip_suffix("f32") {
654            let result = match name {
655                "sqrt" | "sin" | "cos" | "exp" | "exp2" | "log" | "log10" | "log2" | "fabs"
656                | "floor" | "ceil" | "trunc" | "rint" | "nearbyint" | "round" | "roundeven" => {
657                    let [arg] = args else {
658                        return Err(MirEvalError::InternalError(
659                            "f32 intrinsic signature doesn't match fn (f32) -> f32".into(),
660                        ));
661                    };
662                    let arg = from_bytes!(f32, arg.get(self)?);
663                    match name {
664                        "sqrt" => arg.sqrt(),
665                        "sin" => arg.sin(),
666                        "cos" => arg.cos(),
667                        "exp" => arg.exp(),
668                        "exp2" => arg.exp2(),
669                        "log" => arg.ln(),
670                        "log10" => arg.log10(),
671                        "log2" => arg.log2(),
672                        "fabs" => arg.abs(),
673                        "floor" => arg.floor(),
674                        "ceil" => arg.ceil(),
675                        "trunc" => arg.trunc(),
676                        // FIXME: these rounds should be different, but only `.round()` is stable now.
677                        "rint" => arg.round(),
678                        "nearbyint" => arg.round(),
679                        "round" => arg.round(),
680                        "roundeven" => arg.round(),
681                        _ => unreachable!(),
682                    }
683                }
684                "pow" | "minnum" | "maxnum" | "copysign" => {
685                    let [arg1, arg2] = args else {
686                        return Err(MirEvalError::InternalError(
687                            "f32 intrinsic signature doesn't match fn (f32, f32) -> f32".into(),
688                        ));
689                    };
690                    let arg1 = from_bytes!(f32, arg1.get(self)?);
691                    let arg2 = from_bytes!(f32, arg2.get(self)?);
692                    match name {
693                        "pow" => arg1.powf(arg2),
694                        "minnum" => arg1.min(arg2),
695                        "maxnum" => arg1.max(arg2),
696                        "copysign" => arg1.copysign(arg2),
697                        _ => unreachable!(),
698                    }
699                }
700                "powi" => {
701                    let [arg1, arg2] = args else {
702                        return Err(MirEvalError::InternalError(
703                            "powif32 signature doesn't match fn (f32, i32) -> f32".into(),
704                        ));
705                    };
706                    let arg1 = from_bytes!(f32, arg1.get(self)?);
707                    let arg2 = from_bytes!(i32, arg2.get(self)?);
708                    arg1.powi(arg2)
709                }
710                "fma" => {
711                    let [arg1, arg2, arg3] = args else {
712                        return Err(MirEvalError::InternalError(
713                            "fmaf32 signature doesn't match fn (f32, f32, f32) -> f32".into(),
714                        ));
715                    };
716                    let arg1 = from_bytes!(f32, arg1.get(self)?);
717                    let arg2 = from_bytes!(f32, arg2.get(self)?);
718                    let arg3 = from_bytes!(f32, arg3.get(self)?);
719                    arg1.mul_add(arg2, arg3)
720                }
721                _ => not_supported!("unknown f32 intrinsic {name}"),
722            };
723            return destination.write_from_bytes(self, &result.to_le_bytes()).map(|()| true);
724        }
725        match name {
726            "size_of" => {
727                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
728                    return Err(MirEvalError::InternalError(
729                        "size_of generic arg is not provided".into(),
730                    ));
731                };
732                let size = self.size_of_sized(ty, locals, "size_of arg")?;
733                destination.write_from_bytes(self, &size.to_le_bytes()[0..destination.size])
734            }
735            "align_of" => {
736                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
737                    return Err(MirEvalError::InternalError(
738                        "align_of generic arg is not provided".into(),
739                    ));
740                };
741                let align = self.layout(ty)?.align.bytes();
742                destination.write_from_bytes(self, &align.to_le_bytes()[0..destination.size])
743            }
744            "size_of_val" => {
745                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
746                    return Err(MirEvalError::InternalError(
747                        "size_of_val generic arg is not provided".into(),
748                    ));
749                };
750                let [arg] = args else {
751                    return Err(MirEvalError::InternalError(
752                        "size_of_val args are not provided".into(),
753                    ));
754                };
755                if let Some((size, _)) = self.size_align_of(ty, locals)? {
756                    destination.write_from_bytes(self, &size.to_le_bytes())
757                } else {
758                    let metadata = arg.interval.slice(self.ptr_size()..self.ptr_size() * 2);
759                    let (size, _) = self.size_align_of_unsized(ty, metadata, locals)?;
760                    destination.write_from_bytes(self, &size.to_le_bytes())
761                }
762            }
763            "align_of_val" => {
764                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
765                    return Err(MirEvalError::InternalError(
766                        "align_of_val generic arg is not provided".into(),
767                    ));
768                };
769                let [arg] = args else {
770                    return Err(MirEvalError::InternalError(
771                        "align_of_val args are not provided".into(),
772                    ));
773                };
774                if let Some((_, align)) = self.size_align_of(ty, locals)? {
775                    destination.write_from_bytes(self, &align.to_le_bytes())
776                } else {
777                    let metadata = arg.interval.slice(self.ptr_size()..self.ptr_size() * 2);
778                    let (_, align) = self.size_align_of_unsized(ty, metadata, locals)?;
779                    destination.write_from_bytes(self, &align.to_le_bytes())
780                }
781            }
782            "type_name" => {
783                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
784                    return Err(MirEvalError::InternalError(
785                        "type_name generic arg is not provided".into(),
786                    ));
787                };
788                let ty_name = self.db.type_name(ty, locals.body.owner.module(self.db));
789                let len = ty_name.len();
790                let addr = self.heap_allocate(len, 1)?;
791                self.write_memory(addr, ty_name.as_bytes())?;
792                destination.slice(0..self.ptr_size()).write_from_bytes(self, &addr.to_bytes())?;
793                destination
794                    .slice(self.ptr_size()..2 * self.ptr_size())
795                    .write_from_bytes(self, &len.to_le_bytes())
796            }
797            "needs_drop" => {
798                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
799                    return Err(MirEvalError::InternalError(
800                        "size_of generic arg is not provided".into(),
801                    ));
802                };
803                let result = match has_drop_glue(&self.infcx, ty, self.param_env.param_env) {
804                    DropGlue::HasDropGlue => true,
805                    DropGlue::None => false,
806                    DropGlue::DependOnParams => {
807                        never!("should be fully monomorphized now");
808                        true
809                    }
810                };
811                destination.write_from_bytes(self, &[u8::from(result)])
812            }
813            "ptr_guaranteed_cmp" => {
814                // FIXME: this is wrong for const eval, it should return 2 in some
815                // cases.
816                let [lhs, rhs] = args else {
817                    return Err(MirEvalError::InternalError(
818                        "ptr_guaranteed_cmp args are not provided".into(),
819                    ));
820                };
821                let ans = lhs.get(self)? == rhs.get(self)?;
822                destination.write_from_bytes(self, &[u8::from(ans)])
823            }
824            "saturating_add" | "saturating_sub" => {
825                let [lhs, rhs] = args else {
826                    return Err(MirEvalError::InternalError(
827                        "saturating_add args are not provided".into(),
828                    ));
829                };
830                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, IsSigned::No));
831                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, IsSigned::No));
832                let ans = match name {
833                    "saturating_add" => lhs.saturating_add(rhs),
834                    "saturating_sub" => lhs.saturating_sub(rhs),
835                    _ => unreachable!(),
836                };
837                let bits = destination.size * 8;
838                // FIXME: signed
839                let is_signed = false;
840                let mx: u128 = if is_signed { (1 << (bits - 1)) - 1 } else { (1 << bits) - 1 };
841                // FIXME: signed
842                let mn: u128 = 0;
843                let ans = cmp::min(mx, cmp::max(mn, ans));
844                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])
845            }
846            "wrapping_add" | "unchecked_add" => {
847                let [lhs, rhs] = args else {
848                    return Err(MirEvalError::InternalError(
849                        "wrapping_add args are not provided".into(),
850                    ));
851                };
852                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, IsSigned::No));
853                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, IsSigned::No));
854                let ans = lhs.wrapping_add(rhs);
855                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])
856            }
857            "ptr_offset_from_unsigned" | "ptr_offset_from" => {
858                let [lhs, rhs] = args else {
859                    return Err(MirEvalError::InternalError(
860                        "wrapping_sub args are not provided".into(),
861                    ));
862                };
863                let lhs = i128::from_le_bytes(pad16(lhs.get(self)?, IsSigned::No));
864                let rhs = i128::from_le_bytes(pad16(rhs.get(self)?, IsSigned::No));
865                let ans = lhs.wrapping_sub(rhs);
866                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
867                    return Err(MirEvalError::InternalError(
868                        "ptr_offset_from generic arg is not provided".into(),
869                    ));
870                };
871                let size = self.size_of_sized(ty, locals, "ptr_offset_from arg")? as i128;
872                let ans = ans / size;
873                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])
874            }
875            "wrapping_sub" | "unchecked_sub" => {
876                let [lhs, rhs] = args else {
877                    return Err(MirEvalError::InternalError(
878                        "wrapping_sub args are not provided".into(),
879                    ));
880                };
881                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, IsSigned::No));
882                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, IsSigned::No));
883                let ans = lhs.wrapping_sub(rhs);
884                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])
885            }
886            "wrapping_mul" | "unchecked_mul" => {
887                let [lhs, rhs] = args else {
888                    return Err(MirEvalError::InternalError(
889                        "wrapping_mul args are not provided".into(),
890                    ));
891                };
892                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, IsSigned::No));
893                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, IsSigned::No));
894                let ans = lhs.wrapping_mul(rhs);
895                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])
896            }
897            "wrapping_shl" | "unchecked_shl" => {
898                // FIXME: signed
899                let [lhs, rhs] = args else {
900                    return Err(MirEvalError::InternalError(
901                        "unchecked_shl args are not provided".into(),
902                    ));
903                };
904                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, IsSigned::No));
905                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, IsSigned::No));
906                let ans = lhs.wrapping_shl(rhs as u32);
907                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])
908            }
909            "wrapping_shr" | "unchecked_shr" => {
910                // FIXME: signed
911                let [lhs, rhs] = args else {
912                    return Err(MirEvalError::InternalError(
913                        "unchecked_shr args are not provided".into(),
914                    ));
915                };
916                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, IsSigned::No));
917                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, IsSigned::No));
918                let ans = lhs.wrapping_shr(rhs as u32);
919                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])
920            }
921            "unchecked_rem" => {
922                // FIXME: signed
923                let [lhs, rhs] = args else {
924                    return Err(MirEvalError::InternalError(
925                        "unchecked_rem args are not provided".into(),
926                    ));
927                };
928                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, IsSigned::No));
929                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, IsSigned::No));
930                let ans = lhs.checked_rem(rhs).ok_or_else(|| {
931                    MirEvalError::UndefinedBehavior("unchecked_rem with bad inputs".to_owned())
932                })?;
933                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])
934            }
935            "unchecked_div" | "exact_div" => {
936                // FIXME: signed
937                let [lhs, rhs] = args else {
938                    return Err(MirEvalError::InternalError(
939                        "unchecked_div args are not provided".into(),
940                    ));
941                };
942                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, IsSigned::No));
943                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, IsSigned::No));
944                let ans = lhs.checked_div(rhs).ok_or_else(|| {
945                    MirEvalError::UndefinedBehavior("unchecked_rem with bad inputs".to_owned())
946                })?;
947                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])
948            }
949            "add_with_overflow" | "sub_with_overflow" | "mul_with_overflow" => {
950                let [lhs, rhs] = args else {
951                    return Err(MirEvalError::InternalError(
952                        "const_eval_select args are not provided".into(),
953                    ));
954                };
955                let result_ty = Ty::new_tup_from_iter(
956                    self.interner(),
957                    [lhs.ty, Ty::new_bool(self.interner())].into_iter(),
958                );
959                let op_size = self.size_of_sized(lhs.ty, locals, "operand of add_with_overflow")?;
960                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, IsSigned::No));
961                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, IsSigned::No));
962                let (ans, u128overflow) = match name {
963                    "add_with_overflow" => lhs.overflowing_add(rhs),
964                    "sub_with_overflow" => lhs.overflowing_sub(rhs),
965                    "mul_with_overflow" => lhs.overflowing_mul(rhs),
966                    _ => unreachable!(),
967                };
968                let is_overflow = u128overflow
969                    || ans.to_le_bytes()[op_size..].iter().any(|&it| it != 0 && it != 255);
970                let is_overflow = vec![u8::from(is_overflow)];
971                let layout = self.layout(result_ty)?;
972                let result = self.construct_with_layout(
973                    layout.size.bytes_usize(),
974                    &layout,
975                    None,
976                    [ans.to_le_bytes()[0..op_size].to_vec(), is_overflow]
977                        .into_iter()
978                        .map(IntervalOrOwned::Owned),
979                )?;
980                destination.write_from_bytes(self, &result)
981            }
982            "copy" | "copy_nonoverlapping" => {
983                let [src, dst, offset] = args else {
984                    return Err(MirEvalError::InternalError(
985                        "copy_nonoverlapping args are not provided".into(),
986                    ));
987                };
988                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
989                    return Err(MirEvalError::InternalError(
990                        "copy_nonoverlapping generic arg is not provided".into(),
991                    ));
992                };
993                let src = Address::from_bytes(src.get(self)?)?;
994                let dst = Address::from_bytes(dst.get(self)?)?;
995                let offset = from_bytes!(usize, offset.get(self)?);
996                let size = self.size_of_sized(ty, locals, "copy_nonoverlapping ptr type")?;
997                let size = offset * size;
998                let src = Interval { addr: src, size };
999                let dst = Interval { addr: dst, size };
1000                dst.write_from_interval(self, src)
1001            }
1002            "slice_get_unchecked" => {
1003                let [slice_ptr, index] = args else {
1004                    return Err(MirEvalError::InternalError(
1005                        "slice_get_unchecked args are not provided".into(),
1006                    ));
1007                };
1008                let Some(ty) = generic_args.as_slice().get(2).and_then(|it| it.ty()) else {
1009                    return Err(MirEvalError::InternalError(
1010                        "slice_get_unchecked item type is not provided".into(),
1011                    ));
1012                };
1013                let slice_ptr = slice_ptr.get(self)?;
1014                let ptr_size = self.ptr_size();
1015                let Some(data) = slice_ptr.get(..ptr_size) else {
1016                    return Err(MirEvalError::InternalError(
1017                        "slice_get_unchecked slice pointer is too small".into(),
1018                    ));
1019                };
1020                let Some(len) = slice_ptr.get(ptr_size..2 * ptr_size) else {
1021                    return Err(MirEvalError::InternalError(
1022                        "slice_get_unchecked slice metadata is missing".into(),
1023                    ));
1024                };
1025                let slice_ptr = Address::from_bytes(data)?;
1026                let len = from_bytes!(usize, len);
1027                let index = from_bytes!(usize, index.get(self)?);
1028                if index >= len {
1029                    return Err(MirEvalError::UndefinedBehavior(format!(
1030                        "slice_get_unchecked index {index} is out of bounds for slice of length {len}"
1031                    )));
1032                }
1033                let size = self.size_of_sized(ty, locals, "slice_get_unchecked item type")?;
1034                let offset = index* size;
1035                let addr = slice_ptr.to_usize() + offset;
1036                let addr = Address::from_usize(addr);
1037                destination.write_from_bytes(self, &addr.to_bytes()[..destination.size])
1038            }
1039            "offset" | "arith_offset" => {
1040                let [ptr, offset] = args else {
1041                    return Err(MirEvalError::InternalError("offset args are not provided".into()));
1042                };
1043                let ty = if name == "offset" {
1044                    let Some(ty0) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
1045                        return Err(MirEvalError::InternalError(
1046                            "offset generic arg is not provided".into(),
1047                        ));
1048                    };
1049                    let Some(ty1) = generic_args.as_slice().get(1).and_then(|it| it.ty()) else {
1050                        return Err(MirEvalError::InternalError(
1051                            "offset generic arg is not provided".into(),
1052                        ));
1053                    };
1054                    if !matches!(
1055                        ty1.kind(),
1056                        TyKind::Int(rustc_type_ir::IntTy::Isize)
1057                            | TyKind::Uint(rustc_type_ir::UintTy::Usize)
1058                    ) {
1059                        return Err(MirEvalError::InternalError(
1060                            "offset generic arg is not usize or isize".into(),
1061                        ));
1062                    }
1063                    match ty0.kind() {
1064                        TyKind::RawPtr(ty, _) => ty,
1065                        _ => {
1066                            return Err(MirEvalError::InternalError(
1067                                "offset generic arg is not a raw pointer".into(),
1068                            ));
1069                        }
1070                    }
1071                } else {
1072                    let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
1073                        return Err(MirEvalError::InternalError(
1074                            "arith_offset generic arg is not provided".into(),
1075                        ));
1076                    };
1077                    ty
1078                };
1079                let ptr = u128::from_le_bytes(pad16(ptr.get(self)?, IsSigned::No));
1080                let offset = u128::from_le_bytes(pad16(offset.get(self)?, IsSigned::No));
1081                let size = self.size_of_sized(ty, locals, "offset ptr type")? as u128;
1082                let ans = ptr + offset * size;
1083                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])
1084            }
1085            "assert_inhabited"
1086            | "assert_zero_valid"
1087            | "assert_uninit_valid"
1088            | "assert_mem_uninitialized_valid" => {
1089                // FIXME: We should actually implement these checks
1090                Ok(())
1091            }
1092            "forget" => {
1093                // FIXME
1094                Ok(())
1095            }
1096            "transmute" | "transmute_unchecked" => {
1097                let [arg] = args else {
1098                    return Err(MirEvalError::InternalError(
1099                        "transmute arg is not provided".into(),
1100                    ));
1101                };
1102                destination.write_from_interval(self, arg.interval)
1103            }
1104            "ctpop" => {
1105                let [arg] = args else {
1106                    return Err(MirEvalError::InternalError("ctpop arg is not provided".into()));
1107                };
1108                let result = u128::from_le_bytes(pad16(arg.get(self)?, IsSigned::No)).count_ones();
1109                destination
1110                    .write_from_bytes(self, &(result as u128).to_le_bytes()[0..destination.size])
1111            }
1112            "ctlz" | "ctlz_nonzero" => {
1113                let [arg] = args else {
1114                    return Err(MirEvalError::InternalError("ctlz arg is not provided".into()));
1115                };
1116                let result =
1117                    u128::from_le_bytes(pad16(arg.get(self)?, IsSigned::No)).leading_zeros() as usize;
1118                let result = result - (128 - arg.interval.size * 8);
1119                destination
1120                    .write_from_bytes(self, &(result as u128).to_le_bytes()[0..destination.size])
1121            }
1122            "cttz" | "cttz_nonzero" => {
1123                let [arg] = args else {
1124                    return Err(MirEvalError::InternalError("cttz arg is not provided".into()));
1125                };
1126                let arg: &[u8] = arg.get(self)?;
1127                let bit_count = arg.len() as u32 * 8;
1128                let result = u128::from_le_bytes(pad16(arg, IsSigned::No)).trailing_zeros().min(bit_count);
1129                destination
1130                    .write_from_bytes(self, &(result as u128).to_le_bytes()[0..destination.size])
1131            }
1132            "rotate_left" => {
1133                let [lhs, rhs] = args else {
1134                    return Err(MirEvalError::InternalError(
1135                        "rotate_left args are not provided".into(),
1136                    ));
1137                };
1138                let lhs = &lhs.get(self)?[0..destination.size];
1139                let rhs = rhs.get(self)?[0] as u32;
1140                match destination.size {
1141                    1 => {
1142                        let r = from_bytes!(u8, lhs).rotate_left(rhs);
1143                        destination.write_from_bytes(self, &r.to_le_bytes())
1144                    }
1145                    2 => {
1146                        let r = from_bytes!(u16, lhs).rotate_left(rhs);
1147                        destination.write_from_bytes(self, &r.to_le_bytes())
1148                    }
1149                    4 => {
1150                        let r = from_bytes!(u32, lhs).rotate_left(rhs);
1151                        destination.write_from_bytes(self, &r.to_le_bytes())
1152                    }
1153                    8 => {
1154                        let r = from_bytes!(u64, lhs).rotate_left(rhs);
1155                        destination.write_from_bytes(self, &r.to_le_bytes())
1156                    }
1157                    16 => {
1158                        let r = from_bytes!(u128, lhs).rotate_left(rhs);
1159                        destination.write_from_bytes(self, &r.to_le_bytes())
1160                    }
1161                    s => not_supported!("destination with size {s} for rotate_left"),
1162                }
1163            }
1164            "rotate_right" => {
1165                let [lhs, rhs] = args else {
1166                    return Err(MirEvalError::InternalError(
1167                        "rotate_right args are not provided".into(),
1168                    ));
1169                };
1170                let lhs = &lhs.get(self)?[0..destination.size];
1171                let rhs = rhs.get(self)?[0] as u32;
1172                match destination.size {
1173                    1 => {
1174                        let r = from_bytes!(u8, lhs).rotate_right(rhs);
1175                        destination.write_from_bytes(self, &r.to_le_bytes())
1176                    }
1177                    2 => {
1178                        let r = from_bytes!(u16, lhs).rotate_right(rhs);
1179                        destination.write_from_bytes(self, &r.to_le_bytes())
1180                    }
1181                    4 => {
1182                        let r = from_bytes!(u32, lhs).rotate_right(rhs);
1183                        destination.write_from_bytes(self, &r.to_le_bytes())
1184                    }
1185                    8 => {
1186                        let r = from_bytes!(u64, lhs).rotate_right(rhs);
1187                        destination.write_from_bytes(self, &r.to_le_bytes())
1188                    }
1189                    16 => {
1190                        let r = from_bytes!(u128, lhs).rotate_right(rhs);
1191                        destination.write_from_bytes(self, &r.to_le_bytes())
1192                    }
1193                    s => not_supported!("destination with size {s} for rotate_right"),
1194                }
1195            }
1196            "discriminant_value" => {
1197                let [arg] = args else {
1198                    return Err(MirEvalError::InternalError(
1199                        "discriminant_value arg is not provided".into(),
1200                    ));
1201                };
1202                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
1203                    return Err(MirEvalError::InternalError(
1204                        "discriminant_value generic arg is not provided".into(),
1205                    ));
1206                };
1207                let addr = Address::from_bytes(arg.get(self)?)?;
1208                let size = self.size_of_sized(ty, locals, "discriminant_value ptr type")?;
1209                let interval = Interval { addr, size };
1210                let r = self.compute_discriminant(ty, interval.get(self)?)?;
1211                destination.write_from_bytes(self, &r.to_le_bytes()[0..destination.size])
1212            }
1213            "const_eval_select" => {
1214                let [tuple, const_fn, _] = args else {
1215                    return Err(MirEvalError::InternalError(
1216                        "const_eval_select args are not provided".into(),
1217                    ));
1218                };
1219                let mut args = vec![const_fn.clone()];
1220                let TyKind::Tuple(fields) = tuple.ty.kind() else {
1221                    return Err(MirEvalError::InternalError(
1222                        "const_eval_select arg[0] is not a tuple".into(),
1223                    ));
1224                };
1225                let layout = self.layout(tuple.ty)?;
1226                for (i, field) in fields.iter().enumerate() {
1227                    let offset = layout.fields.offset(i).bytes_usize();
1228                    let addr = tuple.interval.addr.offset(offset);
1229                    args.push(IntervalAndTy::new(addr, field, self, locals)?);
1230                }
1231                if let Some(def) = self.lang_items().FnOnce_call_once {
1232                    self.exec_fn_trait(
1233                        def,
1234                        &args,
1235                        // FIXME: wrong for manual impls of `FnOnce`
1236                        GenericArgs::empty(),
1237                        locals,
1238                        destination,
1239                        None,
1240                        span,
1241                    )?;
1242                    return Ok(true);
1243                }
1244                not_supported!("FnOnce was not available for executing const_eval_select");
1245            }
1246            "read_via_copy" | "volatile_load" => {
1247                let [arg] = args else {
1248                    return Err(MirEvalError::InternalError(
1249                        "read_via_copy args are not provided".into(),
1250                    ));
1251                };
1252                let addr = Address::from_bytes(arg.interval.get(self)?)?;
1253                destination.write_from_interval(self, Interval { addr, size: destination.size })
1254            }
1255            "write_via_move" => {
1256                let [ptr, val] = args else {
1257                    return Err(MirEvalError::InternalError(
1258                        "write_via_move args are not provided".into(),
1259                    ));
1260                };
1261                let dst = Address::from_bytes(ptr.get(self)?)?;
1262                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
1263                    return Err(MirEvalError::InternalError(
1264                        "write_via_copy generic arg is not provided".into(),
1265                    ));
1266                };
1267                let size = self.size_of_sized(ty, locals, "write_via_move ptr type")?;
1268                Interval { addr: dst, size }.write_from_interval(self, val.interval)?;
1269                Ok(())
1270            }
1271            "write_bytes" => {
1272                let [dst, val, count] = args else {
1273                    return Err(MirEvalError::InternalError(
1274                        "write_bytes args are not provided".into(),
1275                    ));
1276                };
1277                let count = from_bytes!(usize, count.get(self)?);
1278                let val = from_bytes!(u8, val.get(self)?);
1279                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
1280                    return Err(MirEvalError::InternalError(
1281                        "write_bytes generic arg is not provided".into(),
1282                    ));
1283                };
1284                let dst = Address::from_bytes(dst.get(self)?)?;
1285                let size = self.size_of_sized(ty, locals, "copy_nonoverlapping ptr type")?;
1286                let size = count * size;
1287                self.write_memory_using_ref(dst, size)?.fill(val);
1288                Ok(())
1289            }
1290            "ptr_metadata" => {
1291                let [ptr] = args else {
1292                    return Err(MirEvalError::InternalError(
1293                        "ptr_metadata args are not provided".into(),
1294                    ));
1295                };
1296                let arg = ptr.interval.get(self)?.to_owned();
1297                let metadata = &arg[self.ptr_size()..];
1298                destination.write_from_bytes(self, metadata)?;
1299                Ok(())
1300            }
1301            "three_way_compare" => {
1302                let [lhs, rhs] = args else {
1303                    return Err(MirEvalError::InternalError(
1304                        "three_way_compare args are not provided".into(),
1305                    ));
1306                };
1307                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
1308                    return Err(MirEvalError::InternalError(
1309                        "three_way_compare generic arg is not provided".into(),
1310                    ));
1311                };
1312                let signed = match ty.kind() {
1313                    TyKind::Int(_) => true,
1314                    TyKind::Uint(_) => false,
1315                    _ => {
1316                        return Err(MirEvalError::InternalError(
1317                            "three_way_compare expects an integral type".into(),
1318                        ));
1319                    }
1320                };
1321                let rhs = rhs.get(self)?;
1322                let lhs = lhs.get(self)?;
1323                let mut result = Ordering::Equal;
1324                for (l, r) in lhs.iter().zip(rhs).rev() {
1325                    let it = l.cmp(r);
1326                    if it != Ordering::Equal {
1327                        result = it;
1328                        break;
1329                    }
1330                }
1331                if signed
1332                    && let Some((&l, &r)) = lhs.iter().zip(rhs).next_back()
1333                    && l != r
1334                {
1335                    result = (l as i8).cmp(&(r as i8));
1336                }
1337                if let Some(e) = self.lang_items().Ordering {
1338                    let ty = self.db.ty(e.into()).skip_binder();
1339                    let r = self.compute_discriminant(ty, &[result as i8 as u8])?;
1340                    destination.write_from_bytes(self, &r.to_le_bytes()[0..destination.size])?;
1341                    Ok(())
1342                } else {
1343                    Err(MirEvalError::InternalError("Ordering enum not found".into()))
1344                }
1345            }
1346            "aggregate_raw_ptr" => {
1347                let [data, meta] = args else {
1348                    return Err(MirEvalError::InternalError(
1349                        "aggregate_raw_ptr args are not provided".into(),
1350                    ));
1351                };
1352                destination.write_from_interval(self, data.interval)?;
1353                Interval {
1354                    addr: destination.addr.offset(data.interval.size),
1355                    size: destination.size - data.interval.size,
1356                }
1357                .write_from_interval(self, meta.interval)?;
1358                Ok(())
1359            }
1360            "fabs" => {
1361                let [arg] = args else {
1362                    return Err(MirEvalError::InternalError(
1363                        "fabs intrinsic signature doesn't match fn (T) -> T".into(),
1364                    ));
1365                };
1366                let mut bytes = arg.get(self)?.to_vec();
1367                if let Some(sign_byte) = bytes.last_mut() {
1368                    *sign_byte &= 0x7f;
1369                }
1370                destination.write_from_bytes(self, &bytes)
1371            }
1372            "unreachable" => {
1373                return Err(MirEvalError::UndefinedBehavior(
1374                    "`unreachable` intrinsic executed".to_owned(),
1375                ));
1376            }
1377            "const_allocate" => {
1378                let [size, align] = args else {
1379                    return Err(MirEvalError::InternalError(
1380                        "const_allocate args are not provided".into(),
1381                    ));
1382                };
1383                let size = from_bytes!(usize, size.get(self)?);
1384                let align = from_bytes!(usize, align.get(self)?);
1385                let result = self.heap_allocate(size, align)?;
1386                destination.write_from_bytes(self, &result.to_bytes())
1387            }
1388            "const_deallocate" => Ok(()),
1389            "caller_location" => {
1390                let Some(location_adt) = self.lang_items().PanicLocation else {
1391                    not_supported!("`caller_location` requires the `panic_location` lang item");
1392                };
1393                let location_ty = self.db.ty(location_adt.into()).skip_binder();
1394                let TyKind::Adt(_, subst) = location_ty.kind() else {
1395                    return Err(MirEvalError::InternalError(
1396                        "`panic_location` lang item is not an ADT".into(),
1397                    ));
1398                };
1399                let layout = self.layout(location_ty)?;
1400                let (file, line, col) = self.caller_location_fields(locals.body.owner, span);
1401                let file_len = file.len();
1402                let file_addr = self.heap_allocate(file_len + 1, 1)?;
1403                self.write_memory(file_addr, file.as_bytes())?;
1404                let ptr_size = self.ptr_size();
1405                let field_types = self.db.field_types(location_adt.into());
1406                let mut line_col = [line, col].into_iter();
1407                let mut fields = Vec::with_capacity(field_types.iter().count());
1408                for (_, field) in field_types.iter() {
1409                    let field_ty = field.ty().instantiate(self.interner(), subst).skip_norm_wip();
1410                    let bytes =
1411                        if matches!(field_ty.kind(), TyKind::Uint(rustc_type_ir::UintTy::U32)) {
1412                            line_col.next().unwrap_or(0).to_le_bytes().to_vec()
1413                        } else {
1414                            let size =
1415                                self.size_of_sized(field_ty, locals, "caller_location field")?;
1416                            if size == ptr_size * 2 {
1417                                // The string slice pointing at the file name: (data pointer, length).
1418                                let mut bytes = file_addr.to_bytes()[..ptr_size].to_vec();
1419                                bytes.extend_from_slice(&file_len.to_le_bytes()[..ptr_size]);
1420                                bytes
1421                            } else {
1422                                vec![0; size]
1423                            }
1424                        };
1425                    fields.push(IntervalOrOwned::Owned(bytes));
1426                }
1427                let location = self.construct_with_layout(
1428                    layout.size.bytes_usize(),
1429                    &layout,
1430                    None,
1431                    fields.into_iter(),
1432                )?;
1433                let location_addr =
1434                    self.heap_allocate(layout.size.bytes_usize(), layout.align.bytes() as usize)?;
1435                self.write_memory(location_addr, &location)?;
1436                destination.write_from_bytes(self, &location_addr.to_bytes()[..ptr_size])
1437            }
1438            "box_new" => {
1439                let ty = generic_args.type_at(0);
1440                let Some((size, align)) = self.size_align_of(ty, locals)? else {
1441                    not_supported!("unsized box initialization");
1442                };
1443                let addr = self.heap_allocate(size, align)?;
1444                self.copy_from_interval(addr, args[0].interval)?;
1445                destination.write_from_bytes(self, &addr.to_bytes()[..self.ptr_size()])
1446            }
1447            _ if needs_override => not_supported!("intrinsic {name} is not implemented"),
1448            _ => return Ok(false),
1449        }
1450        .map(|()| true)
1451    }
1452
1453    fn size_align_of_unsized(
1454        &mut self,
1455        ty: Ty<'db>,
1456        metadata: Interval,
1457        locals: &Locals<'a, 'db>,
1458    ) -> Result<'db, (usize, usize)> {
1459        Ok(match ty.kind() {
1460            TyKind::Str => (from_bytes!(usize, metadata.get(self)?), 1),
1461            TyKind::Slice(inner) => {
1462                let len = from_bytes!(usize, metadata.get(self)?);
1463                let (size, align) = self.size_align_of_sized(inner, locals, "slice inner type")?;
1464                (size * len, align)
1465            }
1466            TyKind::Dynamic(..) => self.size_align_of_sized(
1467                self.vtable_map.ty_of_bytes(metadata.get(self)?)?,
1468                locals,
1469                "dyn concrete type",
1470            )?,
1471            TyKind::Adt(adt_def, subst) => {
1472                let id = adt_def.def_id();
1473                let layout = self.layout_adt(id, subst)?;
1474                let id = match id {
1475                    AdtId::StructId(s) => s,
1476                    _ => not_supported!("unsized enum or union"),
1477                };
1478                let field_types = self.db.field_types(id.into());
1479                let last_field_ty = field_types
1480                    .iter()
1481                    .next_back()
1482                    .unwrap()
1483                    .1
1484                    .ty()
1485                    .instantiate(self.interner(), subst)
1486                    .skip_norm_wip();
1487                let sized_part_size =
1488                    layout.fields.offset(field_types.iter().count() - 1).bytes_usize();
1489                let sized_part_align = layout.align.bytes() as usize;
1490                let (unsized_part_size, unsized_part_align) =
1491                    self.size_align_of_unsized(last_field_ty, metadata, locals)?;
1492                let align = sized_part_align.max(unsized_part_align) as isize;
1493                let size = (sized_part_size + unsized_part_size) as isize;
1494                // Must add any necessary padding to `size`
1495                // (to make it a multiple of `align`) before returning it.
1496                //
1497                // Namely, the returned size should be, in C notation:
1498                //
1499                //   `size + ((size & (align-1)) ? align : 0)`
1500                //
1501                // emulated via the semi-standard fast bit trick:
1502                //
1503                //   `(size + (align-1)) & -align`
1504                let size = (size + (align - 1)) & (-align);
1505                (size as usize, align as usize)
1506            }
1507            _ => not_supported!("unsized type other than str, slice, struct and dyn"),
1508        })
1509    }
1510
1511    fn exec_atomic_intrinsic(
1512        &mut self,
1513        name: &str,
1514        args: &[IntervalAndTy<'db>],
1515        generic_args: GenericArgs<'db>,
1516        destination: Interval,
1517        locals: &Locals<'a, 'db>,
1518        _span: MirSpan,
1519    ) -> Result<'db, ()> {
1520        // We are a single threaded runtime with no UB checking and no optimization, so
1521        // we can implement atomic intrinsics as normal functions.
1522
1523        if name.starts_with("singlethreadfence_") || name.starts_with("fence_") {
1524            return Ok(());
1525        }
1526
1527        // The rest of atomic intrinsics have exactly one generic arg
1528
1529        let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
1530            return Err(MirEvalError::InternalError(
1531                "atomic intrinsic generic arg is not provided".into(),
1532            ));
1533        };
1534        let Some(arg0) = args.first() else {
1535            return Err(MirEvalError::InternalError(
1536                "atomic intrinsic arg0 is not provided".into(),
1537            ));
1538        };
1539        let arg0_addr = Address::from_bytes(arg0.get(self)?)?;
1540        let arg0_interval =
1541            Interval::new(arg0_addr, self.size_of_sized(ty, locals, "atomic intrinsic type arg")?);
1542        if name.starts_with("load_") {
1543            return destination.write_from_interval(self, arg0_interval);
1544        }
1545        let Some(arg1) = args.get(1) else {
1546            return Err(MirEvalError::InternalError(
1547                "atomic intrinsic arg1 is not provided".into(),
1548            ));
1549        };
1550        if name.starts_with("store_") {
1551            return arg0_interval.write_from_interval(self, arg1.interval);
1552        }
1553        if name.starts_with("xchg_") {
1554            destination.write_from_interval(self, arg0_interval)?;
1555            return arg0_interval.write_from_interval(self, arg1.interval);
1556        }
1557        if name.starts_with("xadd_") {
1558            destination.write_from_interval(self, arg0_interval)?;
1559            let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, IsSigned::No));
1560            let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, IsSigned::No));
1561            let ans = lhs.wrapping_add(rhs);
1562            return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]);
1563        }
1564        if name.starts_with("xsub_") {
1565            destination.write_from_interval(self, arg0_interval)?;
1566            let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, IsSigned::No));
1567            let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, IsSigned::No));
1568            let ans = lhs.wrapping_sub(rhs);
1569            return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]);
1570        }
1571        if name.starts_with("and_") {
1572            destination.write_from_interval(self, arg0_interval)?;
1573            let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, IsSigned::No));
1574            let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, IsSigned::No));
1575            let ans = lhs & rhs;
1576            return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]);
1577        }
1578        if name.starts_with("or_") {
1579            destination.write_from_interval(self, arg0_interval)?;
1580            let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, IsSigned::No));
1581            let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, IsSigned::No));
1582            let ans = lhs | rhs;
1583            return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]);
1584        }
1585        if name.starts_with("xor_") {
1586            destination.write_from_interval(self, arg0_interval)?;
1587            let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, IsSigned::No));
1588            let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, IsSigned::No));
1589            let ans = lhs ^ rhs;
1590            return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]);
1591        }
1592        if name.starts_with("nand_") {
1593            destination.write_from_interval(self, arg0_interval)?;
1594            let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, IsSigned::No));
1595            let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, IsSigned::No));
1596            let ans = !(lhs & rhs);
1597            return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]);
1598        }
1599        let Some(arg2) = args.get(2) else {
1600            return Err(MirEvalError::InternalError(
1601                "atomic intrinsic arg2 is not provided".into(),
1602            ));
1603        };
1604        if name.starts_with("cxchg_") || name.starts_with("cxchgweak_") {
1605            let dest = if arg1.get(self)? == arg0_interval.get(self)? {
1606                arg0_interval.write_from_interval(self, arg2.interval)?;
1607                (arg1.interval, true)
1608            } else {
1609                (arg0_interval, false)
1610            };
1611            let result_ty = Ty::new_tup_from_iter(
1612                self.interner(),
1613                [ty, Ty::new_bool(self.interner())].into_iter(),
1614            );
1615            let layout = self.layout(result_ty)?;
1616            let result = self.construct_with_layout(
1617                layout.size.bytes_usize(),
1618                &layout,
1619                None,
1620                [IntervalOrOwned::Borrowed(dest.0), IntervalOrOwned::Owned(vec![u8::from(dest.1)])]
1621                    .into_iter(),
1622            )?;
1623            return destination.write_from_bytes(self, &result);
1624        }
1625        not_supported!("unknown atomic intrinsic {name}");
1626    }
1627}
hir_ty/mir/eval/shim.rs

hir_ty/mir/eval/
shim.rs
