hir_ty/mir/eval/
shim.rs

1//! Interpret intrinsics, lang items and `extern "C"` wellknown functions which their implementation
2//! is not available.
3//!
4use std::cmp::{self, Ordering};
5
6use hir_def::{attrs::AttrFlags, signatures::FunctionSignature};
7use rustc_abi::ExternAbi;
8use rustc_type_ir::inherent::{GenericArgs as _, IntoKind, SliceLike, Ty as _};
9use stdx::never;
10
11use crate::{
12    display::DisplayTarget,
13    drop::{DropGlue, has_drop_glue},
14    mir::eval::{
15        Address, AdtId, Arc, Evaluator, FunctionId, GenericArgs, HasModule, HirDisplay, Interval,
16        IntervalAndTy, IntervalOrOwned, ItemContainerId, Layout, Locals, Lookup, MirEvalError,
17        MirSpan, Mutability, Result, Ty, TyKind, from_bytes, not_supported, pad16,
18    },
19    next_solver::Region,
20};
21
22mod simd;
23
24#[derive(Debug, Clone, Copy, PartialEq, Eq)]
25enum EvalLangItem {
26    BeginPanic,
27    SliceLen,
28    DropInPlace,
29}
30
31impl<'a, 'db> Evaluator<'a, 'db> {
32    pub(super) fn detect_and_exec_special_function(
33        &mut self,
34        def: FunctionId,
35        args: &[IntervalAndTy<'db>],
36        generic_args: GenericArgs<'db>,
37        locals: &Locals<'a, 'db>,
38        destination: Interval,
39        span: MirSpan,
40    ) -> Result<'db, bool> {
41        if self.not_special_fn_cache.borrow().contains(&def) {
42            return Ok(false);
43        }
44
45        let function_data = FunctionSignature::of(self.db, def);
46        let attrs = AttrFlags::query(self.db, def.into());
47        let is_intrinsic = FunctionSignature::is_intrinsic(self.db, def);
48
49        if is_intrinsic {
50            return self.exec_intrinsic(
51                function_data.name.as_str(),
52                args,
53                generic_args,
54                destination,
55                locals,
56                span,
57                !function_data.has_body()
58                    || attrs.contains(AttrFlags::RUSTC_INTRINSIC_MUST_BE_OVERRIDDEN),
59            );
60        }
61        let is_extern_c = match def.lookup(self.db).container {
62            hir_def::ItemContainerId::ExternBlockId(block) => {
63                matches!(block.abi(self.db), ExternAbi::C { .. })
64            }
65            _ => false,
66        };
67        if is_extern_c {
68            return self
69                .exec_extern_c(
70                    function_data.name.as_str(),
71                    args,
72                    generic_args,
73                    destination,
74                    locals,
75                    span,
76                )
77                .map(|()| true);
78        }
79
80        if attrs.intersects(
81            AttrFlags::RUSTC_ALLOCATOR
82                | AttrFlags::RUSTC_DEALLOCATOR
83                | AttrFlags::RUSTC_REALLOCATOR
84                | AttrFlags::RUSTC_ALLOCATOR_ZEROED,
85        ) {
86            self.exec_alloc_fn(attrs, args, destination)?;
87            return Ok(true);
88        }
89        if let Some(it) = self.detect_lang_function(def) {
90            let result = self.exec_lang_item(it, generic_args, args, locals, span)?;
91            destination.write_from_bytes(self, &result)?;
92            return Ok(true);
93        }
94        if let ItemContainerId::TraitId(t) = def.lookup(self.db).container
95            && Some(t) == self.lang_items().Clone
96        {
97            let [self_ty] = generic_args.as_slice() else {
98                not_supported!("wrong generic arg count for clone");
99            };
100            let Some(self_ty) = self_ty.ty() else {
101                not_supported!("wrong generic arg kind for clone");
102            };
103            // Clone has special impls for tuples and function pointers
104            if matches!(self_ty.kind(), TyKind::FnPtr(..) | TyKind::Tuple(..) | TyKind::Closure(..))
105            {
106                self.exec_clone(def, args, self_ty, locals, destination, span)?;
107                return Ok(true);
108            }
109            // Return early to prevent caching clone as non special fn.
110            return Ok(false);
111        }
112        self.not_special_fn_cache.borrow_mut().insert(def);
113        Ok(false)
114    }
115
116    pub(super) fn detect_and_redirect_special_function(
117        &mut self,
118        def: FunctionId,
119    ) -> Result<'db, Option<FunctionId>> {
120        // `PanicFmt` is redirected to `ConstPanicFmt`
121        if Some(def) == self.lang_items().PanicFmt {
122            let Some(const_panic_fmt) = self.lang_items().ConstPanicFmt else {
123                not_supported!("const_panic_fmt lang item not found or not a function");
124            };
125            return Ok(Some(const_panic_fmt));
126        }
127        Ok(None)
128    }
129
130    /// Clone has special impls for tuples and function pointers
131    fn exec_clone(
132        &mut self,
133        def: FunctionId,
134        args: &[IntervalAndTy<'db>],
135        self_ty: Ty<'db>,
136        locals: &Locals<'a, 'db>,
137        destination: Interval,
138        span: MirSpan,
139    ) -> Result<'db, ()> {
140        match self_ty.kind() {
141            TyKind::FnPtr(..) => {
142                let [arg] = args else {
143                    not_supported!("wrong arg count for clone");
144                };
145                let addr = Address::from_bytes(arg.get(self)?)?;
146                return destination
147                    .write_from_interval(self, Interval { addr, size: destination.size });
148            }
149            TyKind::Closure(_, closure_args) => self.exec_clone(
150                def,
151                args,
152                closure_args.as_closure().tupled_upvars_ty(),
153                locals,
154                destination,
155                span,
156            )?,
157            TyKind::Tuple(subst) => {
158                let [arg] = args else {
159                    not_supported!("wrong arg count for clone");
160                };
161                let addr = Address::from_bytes(arg.get(self)?)?;
162                let layout = self.layout(self_ty)?;
163                self.exec_clone_for_fields(
164                    subst.iter(),
165                    layout,
166                    addr,
167                    def,
168                    locals,
169                    destination,
170                    span,
171                )?;
172            }
173            _ => {
174                self.exec_fn_with_args(
175                    def,
176                    args,
177                    GenericArgs::new_from_slice(&[self_ty.into()]),
178                    locals,
179                    destination,
180                    None,
181                    span,
182                )?;
183            }
184        }
185        Ok(())
186    }
187
188    fn exec_clone_for_fields(
189        &mut self,
190        ty_iter: impl Iterator<Item = Ty<'db>>,
191        layout: Arc<Layout>,
192        addr: Address,
193        def: FunctionId,
194        locals: &Locals<'a, 'db>,
195        destination: Interval,
196        span: MirSpan,
197    ) -> Result<'db, ()> {
198        for (i, ty) in ty_iter.enumerate() {
199            let size = self.layout(ty)?.size.bytes_usize();
200            let tmp = self.heap_allocate(self.ptr_size(), self.ptr_size())?;
201            let arg = IntervalAndTy {
202                interval: Interval { addr: tmp, size: self.ptr_size() },
203                ty: Ty::new_ref(
204                    self.interner(),
205                    Region::error(self.interner()),
206                    ty,
207                    Mutability::Not,
208                ),
209            };
210            let offset = layout.fields.offset(i).bytes_usize();
211            self.write_memory(tmp, &addr.offset(offset).to_bytes())?;
212            self.exec_clone(
213                def,
214                &[arg],
215                ty,
216                locals,
217                destination.slice(offset..offset + size),
218                span,
219            )?;
220        }
221        Ok(())
222    }
223
224    fn exec_alloc_fn(
225        &mut self,
226        alloc_fn: AttrFlags,
227        args: &[IntervalAndTy<'db>],
228        destination: Interval,
229    ) -> Result<'db, ()> {
230        match alloc_fn {
231            _ if alloc_fn
232                .intersects(AttrFlags::RUSTC_ALLOCATOR_ZEROED | AttrFlags::RUSTC_ALLOCATOR) =>
233            {
234                let [size, align] = args else {
235                    return Err(MirEvalError::InternalError(
236                        "rustc_allocator args are not provided".into(),
237                    ));
238                };
239                let size = from_bytes!(usize, size.get(self)?);
240                let align = from_bytes!(usize, align.get(self)?);
241                let result = self.heap_allocate(size, align)?;
242                destination.write_from_bytes(self, &result.to_bytes())?;
243            }
244            _ if alloc_fn.contains(AttrFlags::RUSTC_DEALLOCATOR) => { /* no-op for now */ }
245            _ if alloc_fn.contains(AttrFlags::RUSTC_REALLOCATOR) => {
246                let [ptr, old_size, align, new_size] = args else {
247                    return Err(MirEvalError::InternalError(
248                        "rustc_allocator args are not provided".into(),
249                    ));
250                };
251                let old_size = from_bytes!(usize, old_size.get(self)?);
252                let new_size = from_bytes!(usize, new_size.get(self)?);
253                if old_size >= new_size {
254                    destination.write_from_interval(self, ptr.interval)?;
255                } else {
256                    let ptr = Address::from_bytes(ptr.get(self)?)?;
257                    let align = from_bytes!(usize, align.get(self)?);
258                    let result = self.heap_allocate(new_size, align)?;
259                    Interval { addr: result, size: old_size }
260                        .write_from_interval(self, Interval { addr: ptr, size: old_size })?;
261                    destination.write_from_bytes(self, &result.to_bytes())?;
262                }
263            }
264            _ => not_supported!("unknown alloc function"),
265        }
266        Ok(())
267    }
268
269    fn detect_lang_function(&self, def: FunctionId) -> Option<EvalLangItem> {
270        use EvalLangItem::*;
271        let lang_items = self.lang_items();
272        let attrs = AttrFlags::query(self.db, def.into());
273
274        if attrs.contains(AttrFlags::RUSTC_CONST_PANIC_STR) {
275            // `#[rustc_const_panic_str]` is treated like `lang = "begin_panic"` by rustc CTFE.
276            return Some(BeginPanic);
277        }
278
279        // We want to execute these functions with special logic
280        // `PanicFmt` is not detected here as it's redirected later.
281        if let Some((_, candidate)) = [
282            (lang_items.BeginPanic, BeginPanic),
283            (lang_items.SliceLen, SliceLen),
284            (lang_items.DropInPlace, DropInPlace),
285        ]
286        .iter()
287        .find(|&(candidate, _)| candidate == Some(def))
288        {
289            return Some(candidate);
290        }
291
292        None
293    }
294
295    fn exec_lang_item(
296        &mut self,
297        it: EvalLangItem,
298        generic_args: GenericArgs<'db>,
299        args: &[IntervalAndTy<'db>],
300        locals: &Locals<'a, 'db>,
301        span: MirSpan,
302    ) -> Result<'db, Vec<u8>> {
303        use EvalLangItem::*;
304        let mut args = args.iter();
305        match it {
306            BeginPanic => {
307                let mut arg = args
308                    .next()
309                    .ok_or(MirEvalError::InternalError(
310                        "argument of BeginPanic is not provided".into(),
311                    ))?
312                    .clone();
313                while let TyKind::Ref(_, ty, _) = arg.ty.kind() {
314                    if ty.is_str() {
315                        let (pointee, metadata) = arg.interval.get(self)?.split_at(self.ptr_size());
316                        let len = from_bytes!(usize, metadata);
317
318                        return {
319                            Err(MirEvalError::Panic(
320                                std::str::from_utf8(
321                                    self.read_memory(Address::from_bytes(pointee)?, len)?,
322                                )
323                                .unwrap()
324                                .to_owned(),
325                            ))
326                        };
327                    }
328                    let size = self.size_of_sized(ty, locals, "begin panic arg")?;
329                    let pointee = arg.interval.get(self)?;
330                    arg = IntervalAndTy {
331                        interval: Interval::new(Address::from_bytes(pointee)?, size),
332                        ty,
333                    };
334                }
335                Err(MirEvalError::Panic(format!("unknown-panic-payload: {:?}", arg.ty.kind())))
336            }
337            SliceLen => {
338                let arg = args.next().ok_or(MirEvalError::InternalError(
339                    "argument of <[T]>::len() is not provided".into(),
340                ))?;
341                let arg = arg.get(self)?;
342                let ptr_size = arg.len() / 2;
343                Ok(arg[ptr_size..].into())
344            }
345            DropInPlace => {
346                let ty = generic_args.as_slice().first().and_then(|it| it.ty()).ok_or(
347                    MirEvalError::InternalError(
348                        "generic argument of drop_in_place is not provided".into(),
349                    ),
350                )?;
351                let arg = args.next().ok_or(MirEvalError::InternalError(
352                    "argument of drop_in_place is not provided".into(),
353                ))?;
354                let arg = arg.interval.get(self)?.to_owned();
355                self.run_drop_glue_deep(
356                    ty,
357                    locals,
358                    Address::from_bytes(&arg[0..self.ptr_size()])?,
359                    &arg[self.ptr_size()..],
360                    span,
361                )?;
362                Ok(vec![])
363            }
364        }
365    }
366
367    fn exec_syscall(
368        &mut self,
369        id: i64,
370        args: &[IntervalAndTy<'db>],
371        destination: Interval,
372        _locals: &Locals<'a, 'db>,
373        _span: MirSpan,
374    ) -> Result<'db, ()> {
375        match id {
376            318 => {
377                // SYS_getrandom
378                let [buf, len, _flags] = args else {
379                    return Err(MirEvalError::InternalError(
380                        "SYS_getrandom args are not provided".into(),
381                    ));
382                };
383                let addr = Address::from_bytes(buf.get(self)?)?;
384                let size = from_bytes!(usize, len.get(self)?);
385                for i in 0..size {
386                    let rand_byte = self.random_state.rand_u64() as u8;
387                    self.write_memory(addr.offset(i), &[rand_byte])?;
388                }
389                destination.write_from_interval(self, len.interval)
390            }
391            _ => {
392                not_supported!("Unknown syscall id {id:?}")
393            }
394        }
395    }
396
397    fn exec_extern_c(
398        &mut self,
399        as_str: &str,
400        args: &[IntervalAndTy<'db>],
401        _generic_args: GenericArgs<'db>,
402        destination: Interval,
403        locals: &Locals<'a, 'db>,
404        span: MirSpan,
405    ) -> Result<'db, ()> {
406        match as_str {
407            "memcmp" => {
408                let [ptr1, ptr2, size] = args else {
409                    return Err(MirEvalError::InternalError("memcmp args are not provided".into()));
410                };
411                let addr1 = Address::from_bytes(ptr1.get(self)?)?;
412                let addr2 = Address::from_bytes(ptr2.get(self)?)?;
413                let size = from_bytes!(usize, size.get(self)?);
414                let slice1 = self.read_memory(addr1, size)?;
415                let slice2 = self.read_memory(addr2, size)?;
416                let r: i128 = match slice1.cmp(slice2) {
417                    cmp::Ordering::Less => -1,
418                    cmp::Ordering::Equal => 0,
419                    cmp::Ordering::Greater => 1,
420                };
421                destination.write_from_bytes(self, &r.to_le_bytes()[..destination.size])
422            }
423            "write" => {
424                let [fd, ptr, len] = args else {
425                    return Err(MirEvalError::InternalError(
426                        "libc::write args are not provided".into(),
427                    ));
428                };
429                let fd = u128::from_le_bytes(pad16(fd.get(self)?, false));
430                let interval = Interval {
431                    addr: Address::from_bytes(ptr.get(self)?)?,
432                    size: from_bytes!(usize, len.get(self)?),
433                };
434                match fd {
435                    1 => {
436                        self.write_to_stdout(interval)?;
437                    }
438                    2 => {
439                        self.write_to_stderr(interval)?;
440                    }
441                    _ => not_supported!("write to arbitrary file descriptor"),
442                }
443                destination.write_from_interval(self, len.interval)?;
444                Ok(())
445            }
446            "pthread_key_create" => {
447                let key = self.thread_local_storage.create_key();
448                let Some(arg0) = args.first() else {
449                    return Err(MirEvalError::InternalError(
450                        "pthread_key_create arg0 is not provided".into(),
451                    ));
452                };
453                let arg0_addr = Address::from_bytes(arg0.get(self)?)?;
454                let key_ty = if let Some((ty, ..)) = arg0.ty.as_reference_or_ptr() {
455                    ty
456                } else {
457                    return Err(MirEvalError::InternalError(
458                        "pthread_key_create arg0 is not a pointer".into(),
459                    ));
460                };
461                let arg0_interval = Interval::new(
462                    arg0_addr,
463                    self.size_of_sized(key_ty, locals, "pthread_key_create key arg")?,
464                );
465                arg0_interval.write_from_bytes(self, &key.to_le_bytes()[0..arg0_interval.size])?;
466                // return 0 as success
467                destination.write_from_bytes(self, &0u64.to_le_bytes()[0..destination.size])?;
468                Ok(())
469            }
470            "pthread_getspecific" => {
471                let Some(arg0) = args.first() else {
472                    return Err(MirEvalError::InternalError(
473                        "pthread_getspecific arg0 is not provided".into(),
474                    ));
475                };
476                let key = from_bytes!(usize, &pad16(arg0.get(self)?, false)[0..8]);
477                let value = self.thread_local_storage.get_key(key)?;
478                destination.write_from_bytes(self, &value.to_le_bytes()[0..destination.size])?;
479                Ok(())
480            }
481            "pthread_setspecific" => {
482                let Some(arg0) = args.first() else {
483                    return Err(MirEvalError::InternalError(
484                        "pthread_setspecific arg0 is not provided".into(),
485                    ));
486                };
487                let key = from_bytes!(usize, &pad16(arg0.get(self)?, false)[0..8]);
488                let Some(arg1) = args.get(1) else {
489                    return Err(MirEvalError::InternalError(
490                        "pthread_setspecific arg1 is not provided".into(),
491                    ));
492                };
493                let value = from_bytes!(u128, pad16(arg1.get(self)?, false));
494                self.thread_local_storage.set_key(key, value)?;
495                // return 0 as success
496                destination.write_from_bytes(self, &0u64.to_le_bytes()[0..destination.size])?;
497                Ok(())
498            }
499            "pthread_key_delete" => {
500                // we ignore this currently
501                // return 0 as success
502                destination.write_from_bytes(self, &0u64.to_le_bytes()[0..destination.size])?;
503                Ok(())
504            }
505            "syscall" => {
506                let Some((id, rest)) = args.split_first() else {
507                    return Err(MirEvalError::InternalError("syscall arg1 is not provided".into()));
508                };
509                let id = from_bytes!(i64, id.get(self)?);
510                self.exec_syscall(id, rest, destination, locals, span)
511            }
512            "sched_getaffinity" => {
513                let [_pid, _set_size, set] = args else {
514                    return Err(MirEvalError::InternalError(
515                        "sched_getaffinity args are not provided".into(),
516                    ));
517                };
518                let set = Address::from_bytes(set.get(self)?)?;
519                // Only enable core 0 (we are single threaded anyway), which is bitset 0x0000001
520                self.write_memory(set, &[1])?;
521                // return 0 as success
522                self.write_memory_using_ref(destination.addr, destination.size)?.fill(0);
523                Ok(())
524            }
525            "getenv" => {
526                let [name] = args else {
527                    return Err(MirEvalError::InternalError("getenv args are not provided".into()));
528                };
529                let mut name_buf = vec![];
530                let name = {
531                    let mut index = Address::from_bytes(name.get(self)?)?;
532                    loop {
533                        let byte = self.read_memory(index, 1)?[0];
534                        index = index.offset(1);
535                        if byte == 0 {
536                            break;
537                        }
538                        name_buf.push(byte);
539                    }
540                    String::from_utf8_lossy(&name_buf)
541                };
542                let value = self.crate_id.env(self.db).get(&name);
543                match value {
544                    None => {
545                        // Write null as fail
546                        self.write_memory_using_ref(destination.addr, destination.size)?.fill(0);
547                    }
548                    Some(mut value) => {
549                        value.push('\0');
550                        let addr = self.heap_allocate(value.len(), 1)?;
551                        self.write_memory(addr, value.as_bytes())?;
552                        self.write_memory(destination.addr, &addr.to_bytes())?;
553                    }
554                }
555                Ok(())
556            }
557            _ => not_supported!("unknown external function {as_str}"),
558        }
559    }
560
561    fn exec_intrinsic(
562        &mut self,
563        name: &str,
564        args: &[IntervalAndTy<'db>],
565        generic_args: GenericArgs<'db>,
566        destination: Interval,
567        locals: &Locals<'a, 'db>,
568        span: MirSpan,
569        needs_override: bool,
570    ) -> Result<'db, bool> {
571        if let Some(name) = name.strip_prefix("atomic_") {
572            return self
573                .exec_atomic_intrinsic(name, args, generic_args, destination, locals, span)
574                .map(|()| true);
575        }
576        if let Some(name) = name.strip_prefix("simd_") {
577            return self
578                .exec_simd_intrinsic(name, args, generic_args, destination, locals, span)
579                .map(|()| true);
580        }
581        // FIXME(#17451): Add `f16` and `f128` intrinsics.
582        if let Some(name) = name.strip_suffix("f64") {
583            let result = match name {
584                "sqrt" | "sin" | "cos" | "exp" | "exp2" | "log" | "log10" | "log2" | "fabs"
585                | "floor" | "ceil" | "trunc" | "rint" | "nearbyint" | "round" | "roundeven" => {
586                    let [arg] = args else {
587                        return Err(MirEvalError::InternalError(
588                            "f64 intrinsic signature doesn't match fn (f64) -> f64".into(),
589                        ));
590                    };
591                    let arg = from_bytes!(f64, arg.get(self)?);
592                    match name {
593                        "sqrt" => arg.sqrt(),
594                        "sin" => arg.sin(),
595                        "cos" => arg.cos(),
596                        "exp" => arg.exp(),
597                        "exp2" => arg.exp2(),
598                        "log" => arg.ln(),
599                        "log10" => arg.log10(),
600                        "log2" => arg.log2(),
601                        "fabs" => arg.abs(),
602                        "floor" => arg.floor(),
603                        "ceil" => arg.ceil(),
604                        "trunc" => arg.trunc(),
605                        // FIXME: these rounds should be different, but only `.round()` is stable now.
606                        "rint" => arg.round(),
607                        "nearbyint" => arg.round(),
608                        "round" => arg.round(),
609                        "roundeven" => arg.round(),
610                        _ => unreachable!(),
611                    }
612                }
613                "pow" | "minnum" | "maxnum" | "copysign" => {
614                    let [arg1, arg2] = args else {
615                        return Err(MirEvalError::InternalError(
616                            "f64 intrinsic signature doesn't match fn (f64, f64) -> f64".into(),
617                        ));
618                    };
619                    let arg1 = from_bytes!(f64, arg1.get(self)?);
620                    let arg2 = from_bytes!(f64, arg2.get(self)?);
621                    match name {
622                        "pow" => arg1.powf(arg2),
623                        "minnum" => arg1.min(arg2),
624                        "maxnum" => arg1.max(arg2),
625                        "copysign" => arg1.copysign(arg2),
626                        _ => unreachable!(),
627                    }
628                }
629                "powi" => {
630                    let [arg1, arg2] = args else {
631                        return Err(MirEvalError::InternalError(
632                            "powif64 signature doesn't match fn (f64, i32) -> f64".into(),
633                        ));
634                    };
635                    let arg1 = from_bytes!(f64, arg1.get(self)?);
636                    let arg2 = from_bytes!(i32, arg2.get(self)?);
637                    arg1.powi(arg2)
638                }
639                "fma" => {
640                    let [arg1, arg2, arg3] = args else {
641                        return Err(MirEvalError::InternalError(
642                            "fmaf64 signature doesn't match fn (f64, f64, f64) -> f64".into(),
643                        ));
644                    };
645                    let arg1 = from_bytes!(f64, arg1.get(self)?);
646                    let arg2 = from_bytes!(f64, arg2.get(self)?);
647                    let arg3 = from_bytes!(f64, arg3.get(self)?);
648                    arg1.mul_add(arg2, arg3)
649                }
650                _ => not_supported!("unknown f64 intrinsic {name}"),
651            };
652            return destination.write_from_bytes(self, &result.to_le_bytes()).map(|()| true);
653        }
654        if let Some(name) = name.strip_suffix("f32") {
655            let result = match name {
656                "sqrt" | "sin" | "cos" | "exp" | "exp2" | "log" | "log10" | "log2" | "fabs"
657                | "floor" | "ceil" | "trunc" | "rint" | "nearbyint" | "round" | "roundeven" => {
658                    let [arg] = args else {
659                        return Err(MirEvalError::InternalError(
660                            "f32 intrinsic signature doesn't match fn (f32) -> f32".into(),
661                        ));
662                    };
663                    let arg = from_bytes!(f32, arg.get(self)?);
664                    match name {
665                        "sqrt" => arg.sqrt(),
666                        "sin" => arg.sin(),
667                        "cos" => arg.cos(),
668                        "exp" => arg.exp(),
669                        "exp2" => arg.exp2(),
670                        "log" => arg.ln(),
671                        "log10" => arg.log10(),
672                        "log2" => arg.log2(),
673                        "fabs" => arg.abs(),
674                        "floor" => arg.floor(),
675                        "ceil" => arg.ceil(),
676                        "trunc" => arg.trunc(),
677                        // FIXME: these rounds should be different, but only `.round()` is stable now.
678                        "rint" => arg.round(),
679                        "nearbyint" => arg.round(),
680                        "round" => arg.round(),
681                        "roundeven" => arg.round(),
682                        _ => unreachable!(),
683                    }
684                }
685                "pow" | "minnum" | "maxnum" | "copysign" => {
686                    let [arg1, arg2] = args else {
687                        return Err(MirEvalError::InternalError(
688                            "f32 intrinsic signature doesn't match fn (f32, f32) -> f32".into(),
689                        ));
690                    };
691                    let arg1 = from_bytes!(f32, arg1.get(self)?);
692                    let arg2 = from_bytes!(f32, arg2.get(self)?);
693                    match name {
694                        "pow" => arg1.powf(arg2),
695                        "minnum" => arg1.min(arg2),
696                        "maxnum" => arg1.max(arg2),
697                        "copysign" => arg1.copysign(arg2),
698                        _ => unreachable!(),
699                    }
700                }
701                "powi" => {
702                    let [arg1, arg2] = args else {
703                        return Err(MirEvalError::InternalError(
704                            "powif32 signature doesn't match fn (f32, i32) -> f32".into(),
705                        ));
706                    };
707                    let arg1 = from_bytes!(f32, arg1.get(self)?);
708                    let arg2 = from_bytes!(i32, arg2.get(self)?);
709                    arg1.powi(arg2)
710                }
711                "fma" => {
712                    let [arg1, arg2, arg3] = args else {
713                        return Err(MirEvalError::InternalError(
714                            "fmaf32 signature doesn't match fn (f32, f32, f32) -> f32".into(),
715                        ));
716                    };
717                    let arg1 = from_bytes!(f32, arg1.get(self)?);
718                    let arg2 = from_bytes!(f32, arg2.get(self)?);
719                    let arg3 = from_bytes!(f32, arg3.get(self)?);
720                    arg1.mul_add(arg2, arg3)
721                }
722                _ => not_supported!("unknown f32 intrinsic {name}"),
723            };
724            return destination.write_from_bytes(self, &result.to_le_bytes()).map(|()| true);
725        }
726        match name {
727            "size_of" => {
728                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
729                    return Err(MirEvalError::InternalError(
730                        "size_of generic arg is not provided".into(),
731                    ));
732                };
733                let size = self.size_of_sized(ty, locals, "size_of arg")?;
734                destination.write_from_bytes(self, &size.to_le_bytes()[0..destination.size])
735            }
736            // FIXME: `min_align_of` was renamed to `align_of` in Rust 1.89
737            // (https://github.com/rust-lang/rust/pull/142410)
738            "min_align_of" | "align_of" => {
739                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
740                    return Err(MirEvalError::InternalError(
741                        "align_of generic arg is not provided".into(),
742                    ));
743                };
744                let align = self.layout(ty)?.align.bytes();
745                destination.write_from_bytes(self, &align.to_le_bytes()[0..destination.size])
746            }
747            "size_of_val" => {
748                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
749                    return Err(MirEvalError::InternalError(
750                        "size_of_val generic arg is not provided".into(),
751                    ));
752                };
753                let [arg] = args else {
754                    return Err(MirEvalError::InternalError(
755                        "size_of_val args are not provided".into(),
756                    ));
757                };
758                if let Some((size, _)) = self.size_align_of(ty, locals)? {
759                    destination.write_from_bytes(self, &size.to_le_bytes())
760                } else {
761                    let metadata = arg.interval.slice(self.ptr_size()..self.ptr_size() * 2);
762                    let (size, _) = self.size_align_of_unsized(ty, metadata, locals)?;
763                    destination.write_from_bytes(self, &size.to_le_bytes())
764                }
765            }
766            // FIXME: `min_align_of_val` was renamed to `align_of_val` in Rust 1.89
767            // (https://github.com/rust-lang/rust/pull/142410)
768            "min_align_of_val" | "align_of_val" => {
769                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
770                    return Err(MirEvalError::InternalError(
771                        "align_of_val generic arg is not provided".into(),
772                    ));
773                };
774                let [arg] = args else {
775                    return Err(MirEvalError::InternalError(
776                        "align_of_val args are not provided".into(),
777                    ));
778                };
779                if let Some((_, align)) = self.size_align_of(ty, locals)? {
780                    destination.write_from_bytes(self, &align.to_le_bytes())
781                } else {
782                    let metadata = arg.interval.slice(self.ptr_size()..self.ptr_size() * 2);
783                    let (_, align) = self.size_align_of_unsized(ty, metadata, locals)?;
784                    destination.write_from_bytes(self, &align.to_le_bytes())
785                }
786            }
787            "type_name" => {
788                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
789                    return Err(MirEvalError::InternalError(
790                        "type_name generic arg is not provided".into(),
791                    ));
792                };
793                let ty_name = match ty.display_source_code(
794                    self.db,
795                    locals.body.owner.module(self.db),
796                    true,
797                ) {
798                    Ok(ty_name) => ty_name,
799                    // Fallback to human readable display in case of `Err`. Ideally we want to use `display_source_code` to
800                    // render full paths.
801                    Err(_) => {
802                        let krate = locals.body.owner.krate(self.db);
803                        ty.display(self.db, DisplayTarget::from_crate(self.db, krate)).to_string()
804                    }
805                };
806                let len = ty_name.len();
807                let addr = self.heap_allocate(len, 1)?;
808                self.write_memory(addr, ty_name.as_bytes())?;
809                destination.slice(0..self.ptr_size()).write_from_bytes(self, &addr.to_bytes())?;
810                destination
811                    .slice(self.ptr_size()..2 * self.ptr_size())
812                    .write_from_bytes(self, &len.to_le_bytes())
813            }
814            "needs_drop" => {
815                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
816                    return Err(MirEvalError::InternalError(
817                        "size_of generic arg is not provided".into(),
818                    ));
819                };
820                let result = match has_drop_glue(&self.infcx, ty, self.param_env.param_env) {
821                    DropGlue::HasDropGlue => true,
822                    DropGlue::None => false,
823                    DropGlue::DependOnParams => {
824                        never!("should be fully monomorphized now");
825                        true
826                    }
827                };
828                destination.write_from_bytes(self, &[u8::from(result)])
829            }
830            "ptr_guaranteed_cmp" => {
831                // FIXME: this is wrong for const eval, it should return 2 in some
832                // cases.
833                let [lhs, rhs] = args else {
834                    return Err(MirEvalError::InternalError(
835                        "ptr_guaranteed_cmp args are not provided".into(),
836                    ));
837                };
838                let ans = lhs.get(self)? == rhs.get(self)?;
839                destination.write_from_bytes(self, &[u8::from(ans)])
840            }
841            "saturating_add" | "saturating_sub" => {
842                let [lhs, rhs] = args else {
843                    return Err(MirEvalError::InternalError(
844                        "saturating_add args are not provided".into(),
845                    ));
846                };
847                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));
848                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));
849                let ans = match name {
850                    "saturating_add" => lhs.saturating_add(rhs),
851                    "saturating_sub" => lhs.saturating_sub(rhs),
852                    _ => unreachable!(),
853                };
854                let bits = destination.size * 8;
855                // FIXME: signed
856                let is_signed = false;
857                let mx: u128 = if is_signed { (1 << (bits - 1)) - 1 } else { (1 << bits) - 1 };
858                // FIXME: signed
859                let mn: u128 = 0;
860                let ans = cmp::min(mx, cmp::max(mn, ans));
861                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])
862            }
863            "wrapping_add" | "unchecked_add" => {
864                let [lhs, rhs] = args else {
865                    return Err(MirEvalError::InternalError(
866                        "wrapping_add args are not provided".into(),
867                    ));
868                };
869                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));
870                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));
871                let ans = lhs.wrapping_add(rhs);
872                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])
873            }
874            "ptr_offset_from_unsigned" | "ptr_offset_from" => {
875                let [lhs, rhs] = args else {
876                    return Err(MirEvalError::InternalError(
877                        "wrapping_sub args are not provided".into(),
878                    ));
879                };
880                let lhs = i128::from_le_bytes(pad16(lhs.get(self)?, false));
881                let rhs = i128::from_le_bytes(pad16(rhs.get(self)?, false));
882                let ans = lhs.wrapping_sub(rhs);
883                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
884                    return Err(MirEvalError::InternalError(
885                        "ptr_offset_from generic arg is not provided".into(),
886                    ));
887                };
888                let size = self.size_of_sized(ty, locals, "ptr_offset_from arg")? as i128;
889                let ans = ans / size;
890                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])
891            }
892            "wrapping_sub" | "unchecked_sub" => {
893                let [lhs, rhs] = args else {
894                    return Err(MirEvalError::InternalError(
895                        "wrapping_sub args are not provided".into(),
896                    ));
897                };
898                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));
899                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));
900                let ans = lhs.wrapping_sub(rhs);
901                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])
902            }
903            "wrapping_mul" | "unchecked_mul" => {
904                let [lhs, rhs] = args else {
905                    return Err(MirEvalError::InternalError(
906                        "wrapping_mul args are not provided".into(),
907                    ));
908                };
909                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));
910                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));
911                let ans = lhs.wrapping_mul(rhs);
912                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])
913            }
914            "wrapping_shl" | "unchecked_shl" => {
915                // FIXME: signed
916                let [lhs, rhs] = args else {
917                    return Err(MirEvalError::InternalError(
918                        "unchecked_shl args are not provided".into(),
919                    ));
920                };
921                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));
922                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));
923                let ans = lhs.wrapping_shl(rhs as u32);
924                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])
925            }
926            "wrapping_shr" | "unchecked_shr" => {
927                // FIXME: signed
928                let [lhs, rhs] = args else {
929                    return Err(MirEvalError::InternalError(
930                        "unchecked_shr args are not provided".into(),
931                    ));
932                };
933                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));
934                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));
935                let ans = lhs.wrapping_shr(rhs as u32);
936                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])
937            }
938            "unchecked_rem" => {
939                // FIXME: signed
940                let [lhs, rhs] = args else {
941                    return Err(MirEvalError::InternalError(
942                        "unchecked_rem args are not provided".into(),
943                    ));
944                };
945                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));
946                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));
947                let ans = lhs.checked_rem(rhs).ok_or_else(|| {
948                    MirEvalError::UndefinedBehavior("unchecked_rem with bad inputs".to_owned())
949                })?;
950                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])
951            }
952            "unchecked_div" | "exact_div" => {
953                // FIXME: signed
954                let [lhs, rhs] = args else {
955                    return Err(MirEvalError::InternalError(
956                        "unchecked_div args are not provided".into(),
957                    ));
958                };
959                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));
960                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));
961                let ans = lhs.checked_div(rhs).ok_or_else(|| {
962                    MirEvalError::UndefinedBehavior("unchecked_rem with bad inputs".to_owned())
963                })?;
964                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])
965            }
966            "add_with_overflow" | "sub_with_overflow" | "mul_with_overflow" => {
967                let [lhs, rhs] = args else {
968                    return Err(MirEvalError::InternalError(
969                        "const_eval_select args are not provided".into(),
970                    ));
971                };
972                let result_ty = Ty::new_tup_from_iter(
973                    self.interner(),
974                    [lhs.ty, Ty::new_bool(self.interner())].into_iter(),
975                );
976                let op_size = self.size_of_sized(lhs.ty, locals, "operand of add_with_overflow")?;
977                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));
978                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));
979                let (ans, u128overflow) = match name {
980                    "add_with_overflow" => lhs.overflowing_add(rhs),
981                    "sub_with_overflow" => lhs.overflowing_sub(rhs),
982                    "mul_with_overflow" => lhs.overflowing_mul(rhs),
983                    _ => unreachable!(),
984                };
985                let is_overflow = u128overflow
986                    || ans.to_le_bytes()[op_size..].iter().any(|&it| it != 0 && it != 255);
987                let is_overflow = vec![u8::from(is_overflow)];
988                let layout = self.layout(result_ty)?;
989                let result = self.construct_with_layout(
990                    layout.size.bytes_usize(),
991                    &layout,
992                    None,
993                    [ans.to_le_bytes()[0..op_size].to_vec(), is_overflow]
994                        .into_iter()
995                        .map(IntervalOrOwned::Owned),
996                )?;
997                destination.write_from_bytes(self, &result)
998            }
999            "copy" | "copy_nonoverlapping" => {
1000                let [src, dst, offset] = args else {
1001                    return Err(MirEvalError::InternalError(
1002                        "copy_nonoverlapping args are not provided".into(),
1003                    ));
1004                };
1005                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
1006                    return Err(MirEvalError::InternalError(
1007                        "copy_nonoverlapping generic arg is not provided".into(),
1008                    ));
1009                };
1010                let src = Address::from_bytes(src.get(self)?)?;
1011                let dst = Address::from_bytes(dst.get(self)?)?;
1012                let offset = from_bytes!(usize, offset.get(self)?);
1013                let size = self.size_of_sized(ty, locals, "copy_nonoverlapping ptr type")?;
1014                let size = offset * size;
1015                let src = Interval { addr: src, size };
1016                let dst = Interval { addr: dst, size };
1017                dst.write_from_interval(self, src)
1018            }
1019            "offset" | "arith_offset" => {
1020                let [ptr, offset] = args else {
1021                    return Err(MirEvalError::InternalError("offset args are not provided".into()));
1022                };
1023                let ty = if name == "offset" {
1024                    let Some(ty0) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
1025                        return Err(MirEvalError::InternalError(
1026                            "offset generic arg is not provided".into(),
1027                        ));
1028                    };
1029                    let Some(ty1) = generic_args.as_slice().get(1).and_then(|it| it.ty()) else {
1030                        return Err(MirEvalError::InternalError(
1031                            "offset generic arg is not provided".into(),
1032                        ));
1033                    };
1034                    if !matches!(
1035                        ty1.kind(),
1036                        TyKind::Int(rustc_type_ir::IntTy::Isize)
1037                            | TyKind::Uint(rustc_type_ir::UintTy::Usize)
1038                    ) {
1039                        return Err(MirEvalError::InternalError(
1040                            "offset generic arg is not usize or isize".into(),
1041                        ));
1042                    }
1043                    match ty0.kind() {
1044                        TyKind::RawPtr(ty, _) => ty,
1045                        _ => {
1046                            return Err(MirEvalError::InternalError(
1047                                "offset generic arg is not a raw pointer".into(),
1048                            ));
1049                        }
1050                    }
1051                } else {
1052                    let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
1053                        return Err(MirEvalError::InternalError(
1054                            "arith_offset generic arg is not provided".into(),
1055                        ));
1056                    };
1057                    ty
1058                };
1059                let ptr = u128::from_le_bytes(pad16(ptr.get(self)?, false));
1060                let offset = u128::from_le_bytes(pad16(offset.get(self)?, false));
1061                let size = self.size_of_sized(ty, locals, "offset ptr type")? as u128;
1062                let ans = ptr + offset * size;
1063                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])
1064            }
1065            "assert_inhabited"
1066            | "assert_zero_valid"
1067            | "assert_uninit_valid"
1068            | "assert_mem_uninitialized_valid" => {
1069                // FIXME: We should actually implement these checks
1070                Ok(())
1071            }
1072            "forget" => {
1073                // FIXME
1074                Ok(())
1075            }
1076            "transmute" | "transmute_unchecked" => {
1077                let [arg] = args else {
1078                    return Err(MirEvalError::InternalError(
1079                        "transmute arg is not provided".into(),
1080                    ));
1081                };
1082                destination.write_from_interval(self, arg.interval)
1083            }
1084            "ctpop" => {
1085                let [arg] = args else {
1086                    return Err(MirEvalError::InternalError("ctpop arg is not provided".into()));
1087                };
1088                let result = u128::from_le_bytes(pad16(arg.get(self)?, false)).count_ones();
1089                destination
1090                    .write_from_bytes(self, &(result as u128).to_le_bytes()[0..destination.size])
1091            }
1092            "ctlz" | "ctlz_nonzero" => {
1093                let [arg] = args else {
1094                    return Err(MirEvalError::InternalError("ctlz arg is not provided".into()));
1095                };
1096                let result =
1097                    u128::from_le_bytes(pad16(arg.get(self)?, false)).leading_zeros() as usize;
1098                let result = result - (128 - arg.interval.size * 8);
1099                destination
1100                    .write_from_bytes(self, &(result as u128).to_le_bytes()[0..destination.size])
1101            }
1102            "cttz" | "cttz_nonzero" => {
1103                let [arg] = args else {
1104                    return Err(MirEvalError::InternalError("cttz arg is not provided".into()));
1105                };
1106                let arg: &[u8] = arg.get(self)?;
1107                let bit_count = arg.len() as u32 * 8;
1108                let result = u128::from_le_bytes(pad16(arg, false)).trailing_zeros().min(bit_count);
1109                destination
1110                    .write_from_bytes(self, &(result as u128).to_le_bytes()[0..destination.size])
1111            }
1112            "rotate_left" => {
1113                let [lhs, rhs] = args else {
1114                    return Err(MirEvalError::InternalError(
1115                        "rotate_left args are not provided".into(),
1116                    ));
1117                };
1118                let lhs = &lhs.get(self)?[0..destination.size];
1119                let rhs = rhs.get(self)?[0] as u32;
1120                match destination.size {
1121                    1 => {
1122                        let r = from_bytes!(u8, lhs).rotate_left(rhs);
1123                        destination.write_from_bytes(self, &r.to_le_bytes())
1124                    }
1125                    2 => {
1126                        let r = from_bytes!(u16, lhs).rotate_left(rhs);
1127                        destination.write_from_bytes(self, &r.to_le_bytes())
1128                    }
1129                    4 => {
1130                        let r = from_bytes!(u32, lhs).rotate_left(rhs);
1131                        destination.write_from_bytes(self, &r.to_le_bytes())
1132                    }
1133                    8 => {
1134                        let r = from_bytes!(u64, lhs).rotate_left(rhs);
1135                        destination.write_from_bytes(self, &r.to_le_bytes())
1136                    }
1137                    16 => {
1138                        let r = from_bytes!(u128, lhs).rotate_left(rhs);
1139                        destination.write_from_bytes(self, &r.to_le_bytes())
1140                    }
1141                    s => not_supported!("destination with size {s} for rotate_left"),
1142                }
1143            }
1144            "rotate_right" => {
1145                let [lhs, rhs] = args else {
1146                    return Err(MirEvalError::InternalError(
1147                        "rotate_right args are not provided".into(),
1148                    ));
1149                };
1150                let lhs = &lhs.get(self)?[0..destination.size];
1151                let rhs = rhs.get(self)?[0] as u32;
1152                match destination.size {
1153                    1 => {
1154                        let r = from_bytes!(u8, lhs).rotate_right(rhs);
1155                        destination.write_from_bytes(self, &r.to_le_bytes())
1156                    }
1157                    2 => {
1158                        let r = from_bytes!(u16, lhs).rotate_right(rhs);
1159                        destination.write_from_bytes(self, &r.to_le_bytes())
1160                    }
1161                    4 => {
1162                        let r = from_bytes!(u32, lhs).rotate_right(rhs);
1163                        destination.write_from_bytes(self, &r.to_le_bytes())
1164                    }
1165                    8 => {
1166                        let r = from_bytes!(u64, lhs).rotate_right(rhs);
1167                        destination.write_from_bytes(self, &r.to_le_bytes())
1168                    }
1169                    16 => {
1170                        let r = from_bytes!(u128, lhs).rotate_right(rhs);
1171                        destination.write_from_bytes(self, &r.to_le_bytes())
1172                    }
1173                    s => not_supported!("destination with size {s} for rotate_right"),
1174                }
1175            }
1176            "discriminant_value" => {
1177                let [arg] = args else {
1178                    return Err(MirEvalError::InternalError(
1179                        "discriminant_value arg is not provided".into(),
1180                    ));
1181                };
1182                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
1183                    return Err(MirEvalError::InternalError(
1184                        "discriminant_value generic arg is not provided".into(),
1185                    ));
1186                };
1187                let addr = Address::from_bytes(arg.get(self)?)?;
1188                let size = self.size_of_sized(ty, locals, "discriminant_value ptr type")?;
1189                let interval = Interval { addr, size };
1190                let r = self.compute_discriminant(ty, interval.get(self)?)?;
1191                destination.write_from_bytes(self, &r.to_le_bytes()[0..destination.size])
1192            }
1193            "const_eval_select" => {
1194                let [tuple, const_fn, _] = args else {
1195                    return Err(MirEvalError::InternalError(
1196                        "const_eval_select args are not provided".into(),
1197                    ));
1198                };
1199                let mut args = vec![const_fn.clone()];
1200                let TyKind::Tuple(fields) = tuple.ty.kind() else {
1201                    return Err(MirEvalError::InternalError(
1202                        "const_eval_select arg[0] is not a tuple".into(),
1203                    ));
1204                };
1205                let layout = self.layout(tuple.ty)?;
1206                for (i, field) in fields.iter().enumerate() {
1207                    let offset = layout.fields.offset(i).bytes_usize();
1208                    let addr = tuple.interval.addr.offset(offset);
1209                    args.push(IntervalAndTy::new(addr, field, self, locals)?);
1210                }
1211                if let Some(def) = self.lang_items().FnOnce_call_once {
1212                    self.exec_fn_trait(
1213                        def,
1214                        &args,
1215                        // FIXME: wrong for manual impls of `FnOnce`
1216                        GenericArgs::empty(self.interner()),
1217                        locals,
1218                        destination,
1219                        None,
1220                        span,
1221                    )?;
1222                    return Ok(true);
1223                }
1224                not_supported!("FnOnce was not available for executing const_eval_select");
1225            }
1226            "read_via_copy" | "volatile_load" => {
1227                let [arg] = args else {
1228                    return Err(MirEvalError::InternalError(
1229                        "read_via_copy args are not provided".into(),
1230                    ));
1231                };
1232                let addr = Address::from_bytes(arg.interval.get(self)?)?;
1233                destination.write_from_interval(self, Interval { addr, size: destination.size })
1234            }
1235            "write_via_move" => {
1236                let [ptr, val] = args else {
1237                    return Err(MirEvalError::InternalError(
1238                        "write_via_move args are not provided".into(),
1239                    ));
1240                };
1241                let dst = Address::from_bytes(ptr.get(self)?)?;
1242                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
1243                    return Err(MirEvalError::InternalError(
1244                        "write_via_copy generic arg is not provided".into(),
1245                    ));
1246                };
1247                let size = self.size_of_sized(ty, locals, "write_via_move ptr type")?;
1248                Interval { addr: dst, size }.write_from_interval(self, val.interval)?;
1249                Ok(())
1250            }
1251            "write_bytes" => {
1252                let [dst, val, count] = args else {
1253                    return Err(MirEvalError::InternalError(
1254                        "write_bytes args are not provided".into(),
1255                    ));
1256                };
1257                let count = from_bytes!(usize, count.get(self)?);
1258                let val = from_bytes!(u8, val.get(self)?);
1259                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
1260                    return Err(MirEvalError::InternalError(
1261                        "write_bytes generic arg is not provided".into(),
1262                    ));
1263                };
1264                let dst = Address::from_bytes(dst.get(self)?)?;
1265                let size = self.size_of_sized(ty, locals, "copy_nonoverlapping ptr type")?;
1266                let size = count * size;
1267                self.write_memory_using_ref(dst, size)?.fill(val);
1268                Ok(())
1269            }
1270            "ptr_metadata" => {
1271                let [ptr] = args else {
1272                    return Err(MirEvalError::InternalError(
1273                        "ptr_metadata args are not provided".into(),
1274                    ));
1275                };
1276                let arg = ptr.interval.get(self)?.to_owned();
1277                let metadata = &arg[self.ptr_size()..];
1278                destination.write_from_bytes(self, metadata)?;
1279                Ok(())
1280            }
1281            "three_way_compare" => {
1282                let [lhs, rhs] = args else {
1283                    return Err(MirEvalError::InternalError(
1284                        "three_way_compare args are not provided".into(),
1285                    ));
1286                };
1287                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
1288                    return Err(MirEvalError::InternalError(
1289                        "three_way_compare generic arg is not provided".into(),
1290                    ));
1291                };
1292                let signed = match ty.kind() {
1293                    TyKind::Int(_) => true,
1294                    TyKind::Uint(_) => false,
1295                    _ => {
1296                        return Err(MirEvalError::InternalError(
1297                            "three_way_compare expects an integral type".into(),
1298                        ));
1299                    }
1300                };
1301                let rhs = rhs.get(self)?;
1302                let lhs = lhs.get(self)?;
1303                let mut result = Ordering::Equal;
1304                for (l, r) in lhs.iter().zip(rhs).rev() {
1305                    let it = l.cmp(r);
1306                    if it != Ordering::Equal {
1307                        result = it;
1308                        break;
1309                    }
1310                }
1311                if signed
1312                    && let Some((&l, &r)) = lhs.iter().zip(rhs).next_back()
1313                    && l != r
1314                {
1315                    result = (l as i8).cmp(&(r as i8));
1316                }
1317                if let Some(e) = self.lang_items().Ordering {
1318                    let ty = self.db.ty(e.into()).skip_binder();
1319                    let r = self.compute_discriminant(ty, &[result as i8 as u8])?;
1320                    destination.write_from_bytes(self, &r.to_le_bytes()[0..destination.size])?;
1321                    Ok(())
1322                } else {
1323                    Err(MirEvalError::InternalError("Ordering enum not found".into()))
1324                }
1325            }
1326            "aggregate_raw_ptr" => {
1327                let [data, meta] = args else {
1328                    return Err(MirEvalError::InternalError(
1329                        "aggregate_raw_ptr args are not provided".into(),
1330                    ));
1331                };
1332                destination.write_from_interval(self, data.interval)?;
1333                Interval {
1334                    addr: destination.addr.offset(data.interval.size),
1335                    size: destination.size - data.interval.size,
1336                }
1337                .write_from_interval(self, meta.interval)?;
1338                Ok(())
1339            }
1340            "fabs" => {
1341                let [arg] = args else {
1342                    return Err(MirEvalError::InternalError(
1343                        "fabs intrinsic signature doesn't match fn (T) -> T".into(),
1344                    ));
1345                };
1346                let mut bytes = arg.get(self)?.to_vec();
1347                if let Some(sign_byte) = bytes.last_mut() {
1348                    *sign_byte &= 0x7f;
1349                }
1350                destination.write_from_bytes(self, &bytes)
1351            }
1352            "unreachable" => {
1353                return Err(MirEvalError::UndefinedBehavior(
1354                    "`unreachable` intrinsic executed".to_owned(),
1355                ));
1356            }
1357            "const_allocate" => {
1358                let [size, align] = args else {
1359                    return Err(MirEvalError::InternalError(
1360                        "const_allocate args are not provided".into(),
1361                    ));
1362                };
1363                let size = from_bytes!(usize, size.get(self)?);
1364                let align = from_bytes!(usize, align.get(self)?);
1365                let result = self.heap_allocate(size, align)?;
1366                destination.write_from_bytes(self, &result.to_bytes())
1367            }
1368            "const_deallocate" => Ok(()),
1369            "caller_location" => {
1370                let Some(location_adt) = self.lang_items().PanicLocation else {
1371                    not_supported!("`caller_location` requires the `panic_location` lang item");
1372                };
1373                let location_ty = self.db.ty(location_adt.into()).skip_binder();
1374                let TyKind::Adt(_, subst) = location_ty.kind() else {
1375                    return Err(MirEvalError::InternalError(
1376                        "`panic_location` lang item is not an ADT".into(),
1377                    ));
1378                };
1379                let layout = self.layout(location_ty)?;
1380                let (file, line, col) = self.caller_location_fields(locals.body.owner, span);
1381                let file_len = file.len();
1382                let file_addr = self.heap_allocate(file_len + 1, 1)?;
1383                self.write_memory(file_addr, file.as_bytes())?;
1384                let ptr_size = self.ptr_size();
1385                let field_types = self.db.field_types(location_adt.into());
1386                let mut line_col = [line, col].into_iter();
1387                let mut fields = Vec::with_capacity(field_types.iter().count());
1388                for (_, field) in field_types.iter() {
1389                    let field_ty = field.ty().instantiate(self.interner(), subst).skip_norm_wip();
1390                    let bytes =
1391                        if matches!(field_ty.kind(), TyKind::Uint(rustc_type_ir::UintTy::U32)) {
1392                            line_col.next().unwrap_or(0).to_le_bytes().to_vec()
1393                        } else {
1394                            let size =
1395                                self.size_of_sized(field_ty, locals, "caller_location field")?;
1396                            if size == ptr_size * 2 {
1397                                // The string slice pointing at the file name: (data pointer, length).
1398                                let mut bytes = file_addr.to_bytes()[..ptr_size].to_vec();
1399                                bytes.extend_from_slice(&file_len.to_le_bytes()[..ptr_size]);
1400                                bytes
1401                            } else {
1402                                vec![0; size]
1403                            }
1404                        };
1405                    fields.push(IntervalOrOwned::Owned(bytes));
1406                }
1407                let location = self.construct_with_layout(
1408                    layout.size.bytes_usize(),
1409                    &layout,
1410                    None,
1411                    fields.into_iter(),
1412                )?;
1413                let location_addr =
1414                    self.heap_allocate(layout.size.bytes_usize(), layout.align.bytes() as usize)?;
1415                self.write_memory(location_addr, &location)?;
1416                destination.write_from_bytes(self, &location_addr.to_bytes()[..ptr_size])
1417            }
1418            _ if needs_override => not_supported!("intrinsic {name} is not implemented"),
1419            _ => return Ok(false),
1420        }
1421        .map(|()| true)
1422    }
1423
1424    fn size_align_of_unsized(
1425        &mut self,
1426        ty: Ty<'db>,
1427        metadata: Interval,
1428        locals: &Locals<'a, 'db>,
1429    ) -> Result<'db, (usize, usize)> {
1430        Ok(match ty.kind() {
1431            TyKind::Str => (from_bytes!(usize, metadata.get(self)?), 1),
1432            TyKind::Slice(inner) => {
1433                let len = from_bytes!(usize, metadata.get(self)?);
1434                let (size, align) = self.size_align_of_sized(inner, locals, "slice inner type")?;
1435                (size * len, align)
1436            }
1437            TyKind::Dynamic(..) => self.size_align_of_sized(
1438                self.vtable_map.ty_of_bytes(metadata.get(self)?)?,
1439                locals,
1440                "dyn concrete type",
1441            )?,
1442            TyKind::Adt(adt_def, subst) => {
1443                let id = adt_def.def_id();
1444                let layout = self.layout_adt(id, subst)?;
1445                let id = match id {
1446                    AdtId::StructId(s) => s,
1447                    _ => not_supported!("unsized enum or union"),
1448                };
1449                let field_types = self.db.field_types(id.into());
1450                let last_field_ty = field_types
1451                    .iter()
1452                    .next_back()
1453                    .unwrap()
1454                    .1
1455                    .ty()
1456                    .instantiate(self.interner(), subst)
1457                    .skip_norm_wip();
1458                let sized_part_size =
1459                    layout.fields.offset(field_types.iter().count() - 1).bytes_usize();
1460                let sized_part_align = layout.align.bytes() as usize;
1461                let (unsized_part_size, unsized_part_align) =
1462                    self.size_align_of_unsized(last_field_ty, metadata, locals)?;
1463                let align = sized_part_align.max(unsized_part_align) as isize;
1464                let size = (sized_part_size + unsized_part_size) as isize;
1465                // Must add any necessary padding to `size`
1466                // (to make it a multiple of `align`) before returning it.
1467                //
1468                // Namely, the returned size should be, in C notation:
1469                //
1470                //   `size + ((size & (align-1)) ? align : 0)`
1471                //
1472                // emulated via the semi-standard fast bit trick:
1473                //
1474                //   `(size + (align-1)) & -align`
1475                let size = (size + (align - 1)) & (-align);
1476                (size as usize, align as usize)
1477            }
1478            _ => not_supported!("unsized type other than str, slice, struct and dyn"),
1479        })
1480    }
1481
1482    fn exec_atomic_intrinsic(
1483        &mut self,
1484        name: &str,
1485        args: &[IntervalAndTy<'db>],
1486        generic_args: GenericArgs<'db>,
1487        destination: Interval,
1488        locals: &Locals<'a, 'db>,
1489        _span: MirSpan,
1490    ) -> Result<'db, ()> {
1491        // We are a single threaded runtime with no UB checking and no optimization, so
1492        // we can implement atomic intrinsics as normal functions.
1493
1494        if name.starts_with("singlethreadfence_") || name.starts_with("fence_") {
1495            return Ok(());
1496        }
1497
1498        // The rest of atomic intrinsics have exactly one generic arg
1499
1500        let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {
1501            return Err(MirEvalError::InternalError(
1502                "atomic intrinsic generic arg is not provided".into(),
1503            ));
1504        };
1505        let Some(arg0) = args.first() else {
1506            return Err(MirEvalError::InternalError(
1507                "atomic intrinsic arg0 is not provided".into(),
1508            ));
1509        };
1510        let arg0_addr = Address::from_bytes(arg0.get(self)?)?;
1511        let arg0_interval =
1512            Interval::new(arg0_addr, self.size_of_sized(ty, locals, "atomic intrinsic type arg")?);
1513        if name.starts_with("load_") {
1514            return destination.write_from_interval(self, arg0_interval);
1515        }
1516        let Some(arg1) = args.get(1) else {
1517            return Err(MirEvalError::InternalError(
1518                "atomic intrinsic arg1 is not provided".into(),
1519            ));
1520        };
1521        if name.starts_with("store_") {
1522            return arg0_interval.write_from_interval(self, arg1.interval);
1523        }
1524        if name.starts_with("xchg_") {
1525            destination.write_from_interval(self, arg0_interval)?;
1526            return arg0_interval.write_from_interval(self, arg1.interval);
1527        }
1528        if name.starts_with("xadd_") {
1529            destination.write_from_interval(self, arg0_interval)?;
1530            let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false));
1531            let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false));
1532            let ans = lhs.wrapping_add(rhs);
1533            return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]);
1534        }
1535        if name.starts_with("xsub_") {
1536            destination.write_from_interval(self, arg0_interval)?;
1537            let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false));
1538            let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false));
1539            let ans = lhs.wrapping_sub(rhs);
1540            return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]);
1541        }
1542        if name.starts_with("and_") {
1543            destination.write_from_interval(self, arg0_interval)?;
1544            let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false));
1545            let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false));
1546            let ans = lhs & rhs;
1547            return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]);
1548        }
1549        if name.starts_with("or_") {
1550            destination.write_from_interval(self, arg0_interval)?;
1551            let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false));
1552            let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false));
1553            let ans = lhs | rhs;
1554            return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]);
1555        }
1556        if name.starts_with("xor_") {
1557            destination.write_from_interval(self, arg0_interval)?;
1558            let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false));
1559            let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false));
1560            let ans = lhs ^ rhs;
1561            return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]);
1562        }
1563        if name.starts_with("nand_") {
1564            destination.write_from_interval(self, arg0_interval)?;
1565            let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false));
1566            let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false));
1567            let ans = !(lhs & rhs);
1568            return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]);
1569        }
1570        let Some(arg2) = args.get(2) else {
1571            return Err(MirEvalError::InternalError(
1572                "atomic intrinsic arg2 is not provided".into(),
1573            ));
1574        };
1575        if name.starts_with("cxchg_") || name.starts_with("cxchgweak_") {
1576            let dest = if arg1.get(self)? == arg0_interval.get(self)? {
1577                arg0_interval.write_from_interval(self, arg2.interval)?;
1578                (arg1.interval, true)
1579            } else {
1580                (arg0_interval, false)
1581            };
1582            let result_ty = Ty::new_tup_from_iter(
1583                self.interner(),
1584                [ty, Ty::new_bool(self.interner())].into_iter(),
1585            );
1586            let layout = self.layout(result_ty)?;
1587            let result = self.construct_with_layout(
1588                layout.size.bytes_usize(),
1589                &layout,
1590                None,
1591                [IntervalOrOwned::Borrowed(dest.0), IntervalOrOwned::Owned(vec![u8::from(dest.1)])]
1592                    .into_iter(),
1593            )?;
1594            return destination.write_from_bytes(self, &result);
1595        }
1596        not_supported!("unknown atomic intrinsic {name}");
1597    }
1598}
hir_ty/mir/eval/shim.rs

hir_ty/mir/eval/
shim.rs
