Skip to main content

core_simd/
vector.rs

1use core::intrinsics::simd::SimdAlign;
2
3use crate::simd::{
4    Mask, MaskElement,
5    cmp::SimdPartialOrd,
6    num::SimdUint,
7    ptr::{SimdConstPtr, SimdMutPtr},
8};
9
10/// A SIMD vector with the shape of `[T; N]` but the operations of `T`.
11///
12/// `Simd<T, N>` supports the operators (+, *, etc.) that `T` does in "elementwise" fashion.
13/// These take the element at each index from the left-hand side and right-hand side,
14/// perform the operation, then return the result in the same index in a vector of equal size.
15/// However, `Simd` differs from normal iteration and normal arrays:
16/// - `Simd<T, N>` executes `N` operations in a single step with no `break`s
17/// - `Simd<T, N>` can have an alignment greater than `T`, for better mechanical sympathy
18///
19/// By always imposing these constraints on `Simd`, it is easier to compile elementwise operations
20/// into machine instructions that can themselves be executed in parallel.
21///
22/// ```rust
23/// # #![feature(portable_simd)]
24/// # use core::simd::{Simd};
25/// # use core::array;
26/// let a: [i32; 4] = [-2, 0, 2, 4];
27/// let b = [10, 9, 8, 7];
28/// let sum = array::from_fn(|i| a[i] + b[i]);
29/// let prod = array::from_fn(|i| a[i] * b[i]);
30///
31/// // `Simd<T, N>` implements `From<[T; N]>`
32/// let (v, w) = (Simd::from(a), Simd::from(b));
33/// // Which means arrays implement `Into<Simd<T, N>>`.
34/// assert_eq!(v + w, sum.into());
35/// assert_eq!(v * w, prod.into());
36/// ```
37///
38///
39/// `Simd` with integer elements treats operators as wrapping, as if `T` was [`Wrapping<T>`].
40/// Thus, `Simd` does not implement `wrapping_add`, because that is the default behavior.
41/// This means there is no warning on overflows, even in "debug" builds.
42/// For most applications where `Simd` is appropriate, it is "not a bug" to wrap,
43/// and even "debug builds" are unlikely to tolerate the loss of performance.
44/// You may want to consider using explicitly checked arithmetic if such is required.
45/// Division by zero on integers still causes a panic, so
46/// you may want to consider using `f32` or `f64` if that is unacceptable.
47///
48/// [`Wrapping<T>`]: core::num::Wrapping
49///
50/// # Layout
51/// `Simd<T, N>` has a layout similar to `[T; N]` (identical "shapes"), with a greater alignment.
52/// `[T; N]` is aligned to `T`, but `Simd<T, N>` will have an alignment based on both `T` and `N`.
53/// Thus it is sound to [`transmute`] `Simd<T, N>` to `[T; N]` and should optimize to "zero cost",
54/// but the reverse transmutation may require a copy the compiler cannot simply elide.
55///
56/// `N` cannot be 0 and may be at most 64. This limit may be increased in the future.
57///
58/// # ABI "Features"
59/// Due to Rust's safety guarantees, `Simd<T, N>` is currently passed and returned via memory,
60/// not SIMD registers, except as an optimization. Using `#[inline]` on functions that accept
61/// `Simd<T, N>` or return it is recommended, at the cost of code generation time, as
62/// inlining SIMD-using functions can omit a large function prolog or epilog and thus
63/// improve both speed and code size. The need for this may be corrected in the future.
64///
65/// Using `#[inline(always)]` still requires additional care.
66///
67/// # Safe SIMD with Unsafe Rust
68///
69/// Operations with `Simd` are typically safe, but there are many reasons to want to combine SIMD with `unsafe` code.
70/// Care must be taken to respect differences between `Simd` and other types it may be transformed into or derived from.
71/// In particular, the layout of `Simd<T, N>` may be similar to `[T; N]`, and may allow some transmutations,
72/// but references to `[T; N]` are not interchangeable with those to `Simd<T, N>`.
73/// Thus, when using `unsafe` Rust to read and write `Simd<T, N>` through [raw pointers], it is a good idea to first try with
74/// [`read_unaligned`] and [`write_unaligned`]. This is because:
75/// - [`read`] and [`write`] require full alignment (in this case, `Simd<T, N>`'s alignment)
76/// - `Simd<T, N>` is often read from or written to [`[T]`](slice) and other types aligned to `T`
77/// - combining these actions violates the `unsafe` contract and explodes the program into
78///   a puff of **undefined behavior**
79/// - the compiler can implicitly adjust layouts to make unaligned reads or writes fully aligned
80///   if it sees the optimization
81/// - most contemporary processors with "aligned" and "unaligned" read and write instructions
82///   exhibit no performance difference if the "unaligned" variant is aligned at runtime
83///
84/// Less obligations mean unaligned reads and writes are less likely to make the program unsound,
85/// and may be just as fast as stricter alternatives.
86/// When trying to guarantee alignment, [`[T]::as_simd`][as_simd] is an option for
87/// converting `[T]` to `[Simd<T, N>]`, and allows soundly operating on an aligned SIMD body,
88/// but it may cost more time when handling the scalar head and tail.
89/// If these are not enough, it is most ideal to design data structures to be already aligned
90/// to `align_of::<Simd<T, N>>()` before using `unsafe` Rust to read or write.
91/// Other ways to compensate for these facts, like materializing `Simd` to or from an array first,
92/// are handled by safe methods like [`Simd::from_array`] and [`Simd::from_slice`].
93///
94/// [`transmute`]: core::mem::transmute
95/// [raw pointers]: pointer
96/// [`read_unaligned`]: pointer::read_unaligned
97/// [`write_unaligned`]: pointer::write_unaligned
98/// [`read`]: pointer::read
99/// [`write`]: pointer::write
100/// [as_simd]: slice::as_simd
101//
102// NOTE: Accessing the inner array directly in any way (e.g. by using the `.0` field syntax) or
103// directly constructing an instance of the type (i.e. `let vector = Simd(array)`) should be
104// avoided, as it will likely become illegal on `#[repr(simd)]` structs in the future. It also
105// causes rustc to emit illegal LLVM IR in some cases.
106#[repr(simd, packed)]
107#[rustc_simd_monomorphize_lane_limit = "64"]
108pub struct Simd<T, const N: usize>([T; N])
109where
110    T: SimdElement;
111
112impl<T, const N: usize> Simd<T, N>
113where
114    T: SimdElement,
115{
116    /// Number of elements in this vector.
117    pub const LEN: usize = N;
118
119    /// Returns the number of elements in this SIMD vector.
120    ///
121    /// # Examples
122    ///
123    /// ```
124    /// # #![feature(portable_simd)]
125    /// # #[cfg(feature = "as_crate")] use core_simd::simd;
126    /// # #[cfg(not(feature = "as_crate"))] use core::simd;
127    /// # use simd::u32x4;
128    /// let v = u32x4::splat(0);
129    /// assert_eq!(v.len(), 4);
130    /// ```
131    #[inline]
132    #[allow(clippy::len_without_is_empty)]
133    pub const fn len(&self) -> usize {
134        Self::LEN
135    }
136
137    /// Constructs a new SIMD vector with all elements set to the given value.
138    ///
139    /// # Examples
140    ///
141    /// ```
142    /// # #![feature(portable_simd)]
143    /// # #[cfg(feature = "as_crate")] use core_simd::simd;
144    /// # #[cfg(not(feature = "as_crate"))] use core::simd;
145    /// # use simd::u32x4;
146    /// let v = u32x4::splat(8);
147    /// assert_eq!(v.as_array(), &[8, 8, 8, 8]);
148    /// ```
149    #[inline]
150    #[rustc_const_unstable(feature = "portable_simd", issue = "86656")]
151    pub const fn splat(value: T) -> Self {
152        // SAFETY: T is a SimdElement, and the item type of Self.
153        unsafe { core::intrinsics::simd::simd_splat(value) }
154    }
155
156    /// Returns an array reference containing the entire SIMD vector.
157    ///
158    /// For reading a single element, prefer [`Self::get`], which may optimize better.
159    ///
160    /// # Examples
161    ///
162    /// ```
163    /// # #![feature(portable_simd)]
164    /// # use core::simd::{Simd, u64x4};
165    /// let v: u64x4 = Simd::from_array([0, 1, 2, 3]);
166    /// assert_eq!(v.as_array(), &[0, 1, 2, 3]);
167    /// ```
168    #[inline]
169    pub const fn as_array(&self) -> &[T; N] {
170        // SAFETY: `Simd<T, N>` is just an overaligned `[T; N]` with
171        // potential padding at the end, so pointer casting to a
172        // `&[T; N]` is safe.
173        //
174        // NOTE: This deliberately doesn't just use `&self.0`, see the comment
175        // on the struct definition for details.
176        unsafe { &*(self as *const Self as *const [T; N]) }
177    }
178
179    /// Returns a mutable array reference containing the entire SIMD vector.
180    ///
181    /// For updating a single element, prefer [`Self::set`], which may optimize better.
182    #[inline]
183    pub const fn as_mut_array(&mut self) -> &mut [T; N] {
184        // SAFETY: `Simd<T, N>` is just an overaligned `[T; N]` with
185        // potential padding at the end, so pointer casting to a
186        // `&mut [T; N]` is safe.
187        //
188        // NOTE: This deliberately doesn't just use `&mut self.0`, see the comment
189        // on the struct definition for details.
190        unsafe { &mut *(self as *mut Self as *mut [T; N]) }
191    }
192
193    /// Returns the element at `index`.
194    ///
195    /// This may optimize better than reading an element via [`Self::as_array`].
196    ///
197    /// # Panics
198    ///
199    /// Panics if `index` is greater than or equal to the number of elements in the vector.
200    #[inline]
201    #[must_use]
202    #[track_caller]
203    pub fn get(&self, index: usize) -> T {
204        assert!(
205            index < N,
206            "index out of bounds: the len is {N} but the index is {index}"
207        );
208        // safety: index < N
209        unsafe { self.get_unchecked(index) }
210    }
211
212    /// Returns the element at `index` without checking bounds.
213    ///
214    /// # Safety
215    ///
216    /// `index` must be less than the number of elements in the vector.
217    #[inline]
218    #[must_use]
219    pub unsafe fn get_unchecked(&self, index: usize) -> T {
220        // safety: the caller guarantees that index < N
221        unsafe {
222            core::hint::assert_unchecked(index < N);
223            core::intrinsics::simd::simd_extract_dyn(*self, index as u32)
224        }
225    }
226
227    /// Sets the element at `index` to `value`.
228    ///
229    /// This may optimize better than updating an element via [`Self::as_mut_array`].
230    ///
231    /// # Panics
232    ///
233    /// Panics if `index` is greater than or equal to the number of elements in the vector.
234    #[inline]
235    #[track_caller]
236    pub fn set(&mut self, index: usize, value: T) {
237        assert!(
238            index < N,
239            "index out of bounds: the len is {N} but the index is {index}"
240        );
241        // safety: index < N
242        unsafe { self.set_unchecked(index, value) }
243    }
244
245    /// Sets the element at `index` to `value` without checking bounds.
246    ///
247    /// # Safety
248    ///
249    /// `index` must be less than the number of elements in the vector.
250    #[inline]
251    pub unsafe fn set_unchecked(&mut self, index: usize, value: T) {
252        // safety: the caller guarantees that index < N
253        unsafe {
254            core::hint::assert_unchecked(index < N);
255            if cfg!(target_arch = "wasm32") {
256                // work around https://github.com/llvm/llvm-project/issues/190306
257                *self.as_mut_array().get_unchecked_mut(index) = value;
258            } else {
259                *self = core::intrinsics::simd::simd_insert_dyn(*self, index as u32, value);
260            }
261        }
262    }
263
264    /// Loads a vector from an array of `T`.
265    ///
266    /// This function is necessary since `repr(simd)` has padding for non-power-of-2 vectors (at the time of writing).
267    /// With padding, `read_unaligned` will read past the end of an array of N elements.
268    ///
269    /// # Safety
270    /// Reading `ptr` must be safe, as if by `<*const [T; N]>::read`.
271    #[inline]
272    const unsafe fn load(ptr: *const [T; N]) -> Self {
273        // There are potentially simpler ways to write this function, but this should result in
274        // LLVM `load <N x T>`
275
276        let mut tmp = core::mem::MaybeUninit::<Self>::uninit();
277        // SAFETY: `Simd<T, N>` always contains `N` elements of type `T`.  It may have padding
278        // which does not need to be initialized.  The safety of reading `ptr` is ensured by the
279        // caller.
280        unsafe {
281            core::ptr::copy_nonoverlapping(ptr, tmp.as_mut_ptr().cast(), 1);
282            tmp.assume_init()
283        }
284    }
285
286    /// Store a vector to an array of `T`.
287    ///
288    /// See `load` as to why this function is necessary.
289    ///
290    /// # Safety
291    /// Writing to `ptr` must be safe, as if by `<*mut [T; N]>::write`.
292    #[inline]
293    const unsafe fn store(self, ptr: *mut [T; N]) {
294        // There are potentially simpler ways to write this function, but this should result in
295        // LLVM `store <N x T>`
296
297        // Creating a temporary helps LLVM turn the memcpy into a store.
298        let tmp = self;
299        // SAFETY: `Simd<T, N>` always contains `N` elements of type `T`.  The safety of writing
300        // `ptr` is ensured by the caller.
301        unsafe { core::ptr::copy_nonoverlapping(tmp.as_array(), ptr, 1) }
302    }
303
304    /// Converts an array to a SIMD vector.
305    #[inline]
306    pub const fn from_array(array: [T; N]) -> Self {
307        // SAFETY: `&array` is safe to read.
308        //
309        // FIXME: We currently use a pointer load instead of `transmute_copy` because `repr(simd)`
310        // results in padding for non-power-of-2 vectors (so vectors are larger than arrays).
311        //
312        // NOTE: This deliberately doesn't just use `Self(array)`, see the comment
313        // on the struct definition for details.
314        unsafe { Self::load(&array) }
315    }
316
317    /// Converts a SIMD vector to an array.
318    #[inline]
319    pub const fn to_array(self) -> [T; N] {
320        let mut tmp = core::mem::MaybeUninit::uninit();
321        // SAFETY: writing to `tmp` is safe and initializes it.
322        //
323        // FIXME: We currently use a pointer store instead of `transmute_copy` because `repr(simd)`
324        // results in padding for non-power-of-2 vectors (so vectors are larger than arrays).
325        //
326        // NOTE: This deliberately doesn't just use `self.0`, see the comment
327        // on the struct definition for details.
328        unsafe {
329            self.store(tmp.as_mut_ptr());
330            tmp.assume_init()
331        }
332    }
333
334    /// Converts a slice to a SIMD vector containing `slice[..N]`.
335    ///
336    /// # Panics
337    ///
338    /// Panics if the slice's length is less than the vector's `Simd::N`.
339    /// Use `load_or_default` for an alternative that does not panic.
340    ///
341    /// # Example
342    ///
343    /// ```
344    /// # #![feature(portable_simd)]
345    /// # use core::simd::u32x4;
346    /// let source = vec![1, 2, 3, 4, 5, 6];
347    /// let v = u32x4::from_slice(&source);
348    /// assert_eq!(v.as_array(), &[1, 2, 3, 4]);
349    /// ```
350    #[must_use]
351    #[inline]
352    #[track_caller]
353    pub const fn from_slice(slice: &[T]) -> Self {
354        assert!(
355            slice.len() >= Self::LEN,
356            "slice length must be at least the number of elements"
357        );
358        // SAFETY: We just checked that the slice contains
359        // at least `N` elements.
360        unsafe { Self::load(slice.as_ptr().cast()) }
361    }
362
363    /// Writes a SIMD vector to the first `N` elements of a slice.
364    ///
365    /// # Panics
366    ///
367    /// Panics if the slice's length is less than the vector's `Simd::N`.
368    ///
369    /// # Example
370    ///
371    /// ```
372    /// # #![feature(portable_simd)]
373    /// # #[cfg(feature = "as_crate")] use core_simd::simd;
374    /// # #[cfg(not(feature = "as_crate"))] use core::simd;
375    /// # use simd::u32x4;
376    /// let mut dest = vec![0; 6];
377    /// let v = u32x4::from_array([1, 2, 3, 4]);
378    /// v.copy_to_slice(&mut dest);
379    /// assert_eq!(&dest, &[1, 2, 3, 4, 0, 0]);
380    /// ```
381    #[inline]
382    #[track_caller]
383    pub const fn copy_to_slice(self, slice: &mut [T]) {
384        assert!(
385            slice.len() >= Self::LEN,
386            "slice length must be at least the number of elements"
387        );
388        // SAFETY: We just checked that the slice contains
389        // at least `N` elements.
390        unsafe { self.store(slice.as_mut_ptr().cast()) }
391    }
392
393    /// Reads contiguous elements from `slice`. Elements are read so long as they're in-bounds for
394    /// the `slice`. Otherwise, the default value for the element type is returned.
395    ///
396    /// # Examples
397    /// ```
398    /// # #![feature(portable_simd)]
399    /// # #[cfg(feature = "as_crate")] use core_simd::simd;
400    /// # #[cfg(not(feature = "as_crate"))] use core::simd;
401    /// # use simd::Simd;
402    /// let vec: Vec<i32> = vec![10, 11];
403    ///
404    /// let result = Simd::<i32, 4>::load_or_default(&vec);
405    /// assert_eq!(result, Simd::from_array([10, 11, 0, 0]));
406    /// ```
407    #[must_use]
408    #[inline]
409    pub fn load_or_default(slice: &[T]) -> Self
410    where
411        T: Default,
412    {
413        Self::load_or(slice, Default::default())
414    }
415
416    /// Reads contiguous elements from `slice`. Elements are read so long as they're in-bounds for
417    /// the `slice`. Otherwise, the corresponding value from `or` is passed through.
418    ///
419    /// # Examples
420    /// ```
421    /// # #![feature(portable_simd)]
422    /// # #[cfg(feature = "as_crate")] use core_simd::simd;
423    /// # #[cfg(not(feature = "as_crate"))] use core::simd;
424    /// # use simd::Simd;
425    /// let vec: Vec<i32> = vec![10, 11];
426    /// let or = Simd::from_array([-5, -4, -3, -2]);
427    ///
428    /// let result = Simd::load_or(&vec, or);
429    /// assert_eq!(result, Simd::from_array([10, 11, -3, -2]));
430    /// ```
431    #[must_use]
432    #[inline]
433    pub fn load_or(slice: &[T], or: Self) -> Self {
434        Self::load_select(slice, Mask::splat(true), or)
435    }
436
437    /// Reads contiguous elements from `slice`. Each element is read from memory if its
438    /// corresponding element in `enable` is `true`.
439    ///
440    /// When the element is disabled or out of bounds for the slice, that memory location
441    /// is not accessed and the default value for the element type is returned.
442    ///
443    /// # Examples
444    /// ```
445    /// # #![feature(portable_simd)]
446    /// # #[cfg(feature = "as_crate")] use core_simd::simd;
447    /// # #[cfg(not(feature = "as_crate"))] use core::simd;
448    /// # use simd::{Simd, Mask};
449    /// let vec: Vec<i32> = vec![10, 11, 12];
450    /// let enable = Mask::from_array([false, true, true, true]);
451    ///
452    /// let result = Simd::load_select_or_default(&vec, enable);
453    /// assert_eq!(result, Simd::from_array([0, 11, 12, 0]));
454    /// ```
455    #[must_use]
456    #[inline]
457    pub fn load_select_or_default(slice: &[T], enable: Mask<<T as SimdElement>::Mask, N>) -> Self
458    where
459        T: Default,
460    {
461        Self::load_select(slice, enable, Default::default())
462    }
463
464    /// Reads contiguous elements from `slice`. Each element is read from memory if its
465    /// corresponding element in `enable` is `true`.
466    ///
467    /// When the element is disabled or out of bounds for the slice, that memory location
468    /// is not accessed and the corresponding value from `or` is passed through.
469    ///
470    /// # Examples
471    /// ```
472    /// # #![feature(portable_simd)]
473    /// # #[cfg(feature = "as_crate")] use core_simd::simd;
474    /// # #[cfg(not(feature = "as_crate"))] use core::simd;
475    /// # use simd::{Simd, Mask};
476    /// let vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
477    /// let enable = Mask::from_array([true, true, false, true]);
478    /// let or = Simd::from_array([-5, -4, -3, -2]);
479    ///
480    /// let result = Simd::load_select(&vec, enable, or);
481    /// assert_eq!(result, Simd::from_array([10, 11, -3, 13]));
482    /// ```
483    #[must_use]
484    #[inline]
485    pub fn load_select(
486        slice: &[T],
487        mut enable: Mask<<T as SimdElement>::Mask, N>,
488        or: Self,
489    ) -> Self {
490        enable &= mask_up_to(slice.len());
491        // SAFETY: We performed the bounds check by updating the mask. &[T] is properly aligned to
492        // the element.
493        unsafe { Self::load_select_ptr(slice.as_ptr(), enable, or) }
494    }
495
496    /// Reads contiguous elements from `slice`. Each element is read from memory if its
497    /// corresponding element in `enable` is `true`.
498    ///
499    /// When the element is disabled, that memory location is not accessed and the corresponding
500    /// value from `or` is passed through.
501    ///
502    /// # Safety
503    /// Enabled loads must not exceed the length of `slice`.
504    #[must_use]
505    #[inline]
506    pub unsafe fn load_select_unchecked(
507        slice: &[T],
508        enable: Mask<<T as SimdElement>::Mask, N>,
509        or: Self,
510    ) -> Self {
511        let ptr = slice.as_ptr();
512        // SAFETY: The safety of reading elements from `slice` is ensured by the caller.
513        unsafe { Self::load_select_ptr(ptr, enable, or) }
514    }
515
516    /// Reads contiguous elements starting at `ptr`. Each element is read from memory if its
517    /// corresponding element in `enable` is `true`.
518    ///
519    /// When the element is disabled, that memory location is not accessed and the corresponding
520    /// value from `or` is passed through.
521    ///
522    /// # Safety
523    /// Enabled `ptr` elements must be safe to read as if by `core::ptr::read`.
524    #[must_use]
525    #[inline]
526    pub unsafe fn load_select_ptr(
527        ptr: *const T,
528        enable: Mask<<T as SimdElement>::Mask, N>,
529        or: Self,
530    ) -> Self {
531        // SAFETY: The safety of reading elements through `ptr` is ensured by the caller.
532        unsafe {
533            core::intrinsics::simd::simd_masked_load::<_, _, _, { SimdAlign::Element }>(
534                enable.to_simd(),
535                ptr,
536                or,
537            )
538        }
539    }
540
541    /// Reads from potentially discontiguous indices in `slice` to construct a SIMD vector.
542    /// If an index is out-of-bounds, the element is instead selected from the `or` vector.
543    ///
544    /// # Examples
545    /// ```
546    /// # #![feature(portable_simd)]
547    /// # use core::simd::Simd;
548    /// let vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
549    /// let idxs = Simd::from_array([9, 3, 0, 5]);  // Note the index that is out-of-bounds
550    /// let alt = Simd::from_array([-5, -4, -3, -2]);
551    ///
552    /// let result = Simd::gather_or(&vec, idxs, alt);
553    /// assert_eq!(result, Simd::from_array([-5, 13, 10, 15]));
554    /// ```
555    #[must_use]
556    #[inline]
557    pub fn gather_or(slice: &[T], idxs: Simd<usize, N>, or: Self) -> Self {
558        Self::gather_select(slice, Mask::splat(true), idxs, or)
559    }
560
561    /// Reads from indices in `slice` to construct a SIMD vector.
562    /// If an index is out-of-bounds, the element is set to the default given by `T: Default`.
563    ///
564    /// # Examples
565    /// ```
566    /// # #![feature(portable_simd)]
567    /// # use core::simd::Simd;
568    /// let vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
569    /// let idxs = Simd::from_array([9, 3, 0, 5]);  // Note the index that is out-of-bounds
570    ///
571    /// let result = Simd::gather_or_default(&vec, idxs);
572    /// assert_eq!(result, Simd::from_array([0, 13, 10, 15]));
573    /// ```
574    #[must_use]
575    #[inline]
576    pub fn gather_or_default(slice: &[T], idxs: Simd<usize, N>) -> Self
577    where
578        T: Default,
579    {
580        Self::gather_or(slice, idxs, Self::splat(T::default()))
581    }
582
583    /// Reads from indices in `slice` to construct a SIMD vector.
584    /// The mask `enable`s all `true` indices and disables all `false` indices.
585    /// If an index is disabled or is out-of-bounds, the element is selected from the `or` vector.
586    ///
587    /// # Examples
588    /// ```
589    /// # #![feature(portable_simd)]
590    /// # use core::simd::{Simd, Mask};
591    /// let vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
592    /// let idxs = Simd::from_array([9, 3, 0, 5]); // Includes an out-of-bounds index
593    /// let alt = Simd::from_array([-5, -4, -3, -2]);
594    /// let enable = Mask::from_array([true, true, true, false]); // Includes a masked element
595    ///
596    /// let result = Simd::gather_select(&vec, enable, idxs, alt);
597    /// assert_eq!(result, Simd::from_array([-5, 13, 10, -2]));
598    /// ```
599    #[must_use]
600    #[inline]
601    pub fn gather_select(
602        slice: &[T],
603        enable: Mask<isize, N>,
604        idxs: Simd<usize, N>,
605        or: Self,
606    ) -> Self {
607        let enable: Mask<isize, N> = enable & idxs.simd_lt(Simd::splat(slice.len()));
608        // Safety: We have masked-off out-of-bounds indices.
609        unsafe { Self::gather_select_unchecked(slice, enable, idxs, or) }
610    }
611
612    /// Reads from indices in `slice` to construct a SIMD vector.
613    /// The mask `enable`s all `true` indices and disables all `false` indices.
614    /// If an index is disabled, the element is selected from the `or` vector.
615    ///
616    /// # Safety
617    ///
618    /// Calling this function with an `enable`d out-of-bounds index is *[undefined behavior]*
619    /// even if the resulting value is not used.
620    ///
621    /// # Examples
622    /// ```
623    /// # #![feature(portable_simd)]
624    /// # #[cfg(feature = "as_crate")] use core_simd::simd;
625    /// # #[cfg(not(feature = "as_crate"))] use core::simd;
626    /// # use simd::{Simd, cmp::SimdPartialOrd, Mask};
627    /// let vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
628    /// let idxs = Simd::from_array([9, 3, 0, 5]); // Includes an out-of-bounds index
629    /// let alt = Simd::from_array([-5, -4, -3, -2]);
630    /// let enable = Mask::from_array([true, true, true, false]); // Includes a masked element
631    /// // If this mask was used to gather, it would be unsound. Let's fix that.
632    /// let enable = enable & idxs.simd_lt(Simd::splat(vec.len()));
633    ///
634    /// // The out-of-bounds index has been masked, so it's safe to gather now.
635    /// let result = unsafe { Simd::gather_select_unchecked(&vec, enable, idxs, alt) };
636    /// assert_eq!(result, Simd::from_array([-5, 13, 10, -2]));
637    /// ```
638    /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html
639    #[must_use]
640    #[inline]
641    #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
642    pub unsafe fn gather_select_unchecked(
643        slice: &[T],
644        enable: Mask<isize, N>,
645        idxs: Simd<usize, N>,
646        or: Self,
647    ) -> Self {
648        let base_ptr = Simd::<*const T, N>::splat(slice.as_ptr());
649        // Ferris forgive me, I have done pointer arithmetic here.
650        let ptrs = base_ptr.wrapping_add(idxs);
651        // Safety: The caller is responsible for determining the indices are okay to read
652        unsafe { Self::gather_select_ptr(ptrs, enable, or) }
653    }
654
655    /// Reads elementwise from pointers into a SIMD vector.
656    ///
657    /// # Safety
658    ///
659    /// Each read must satisfy the same conditions as [`core::ptr::read`].
660    ///
661    /// # Example
662    /// ```
663    /// # #![feature(portable_simd)]
664    /// # #[cfg(feature = "as_crate")] use core_simd::simd;
665    /// # #[cfg(not(feature = "as_crate"))] use core::simd;
666    /// # use simd::prelude::*;
667    /// let values = [6, 2, 4, 9];
668    /// let offsets = Simd::from_array([1, 0, 0, 3]);
669    /// let source = Simd::splat(values.as_ptr()).wrapping_add(offsets);
670    /// let gathered = unsafe { Simd::gather_ptr(source) };
671    /// assert_eq!(gathered, Simd::from_array([2, 6, 6, 9]));
672    /// ```
673    #[must_use]
674    #[inline]
675    #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
676    pub unsafe fn gather_ptr(source: Simd<*const T, N>) -> Self
677    where
678        T: Default,
679    {
680        // TODO: add an intrinsic that doesn't use a passthru vector, and remove the T: Default bound
681        // Safety: The caller is responsible for upholding all invariants
682        unsafe { Self::gather_select_ptr(source, Mask::splat(true), Self::default()) }
683    }
684
685    /// Conditionally read elementwise from pointers into a SIMD vector.
686    /// The mask `enable`s all `true` pointers and disables all `false` pointers.
687    /// If a pointer is disabled, the element is selected from the `or` vector,
688    /// and no read is performed.
689    ///
690    /// # Safety
691    ///
692    /// Enabled elements must satisfy the same conditions as [`core::ptr::read`].
693    ///
694    /// # Example
695    /// ```
696    /// # #![feature(portable_simd)]
697    /// # #[cfg(feature = "as_crate")] use core_simd::simd;
698    /// # #[cfg(not(feature = "as_crate"))] use core::simd;
699    /// # use simd::prelude::*;
700    /// let values = [6, 2, 4, 9];
701    /// let enable = Mask::from_array([true, true, false, true]);
702    /// let offsets = Simd::from_array([1, 0, 0, 3]);
703    /// let source = Simd::splat(values.as_ptr()).wrapping_add(offsets);
704    /// let gathered = unsafe { Simd::gather_select_ptr(source, enable, Simd::splat(0)) };
705    /// assert_eq!(gathered, Simd::from_array([2, 6, 0, 9]));
706    /// ```
707    #[must_use]
708    #[inline]
709    #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
710    pub unsafe fn gather_select_ptr(
711        source: Simd<*const T, N>,
712        enable: Mask<isize, N>,
713        or: Self,
714    ) -> Self {
715        // Safety: The caller is responsible for upholding all invariants
716        unsafe { core::intrinsics::simd::simd_gather(or, source, enable.to_simd()) }
717    }
718
719    /// Conditionally write contiguous elements to `slice`. The `enable` mask controls
720    /// which elements are written, as long as they're in-bounds of the `slice`.
721    /// If the element is disabled or out of bounds, no memory access to that location
722    /// is made.
723    ///
724    /// # Examples
725    /// ```
726    /// # #![feature(portable_simd)]
727    /// # #[cfg(feature = "as_crate")] use core_simd::simd;
728    /// # #[cfg(not(feature = "as_crate"))] use core::simd;
729    /// # use simd::{Simd, Mask};
730    /// let mut arr = [0i32; 4];
731    /// let write = Simd::from_array([-5, -4, -3, -2]);
732    /// let enable = Mask::from_array([false, true, true, true]);
733    ///
734    /// write.store_select(&mut arr[..3], enable);
735    /// assert_eq!(arr, [0, -4, -3, 0]);
736    /// ```
737    #[inline]
738    pub fn store_select(self, slice: &mut [T], mut enable: Mask<<T as SimdElement>::Mask, N>) {
739        enable &= mask_up_to(slice.len());
740        // SAFETY: We performed the bounds check by updating the mask. &[T] is properly aligned to
741        // the element.
742        unsafe { self.store_select_ptr(slice.as_mut_ptr(), enable) }
743    }
744
745    /// Conditionally write contiguous elements to `slice`. The `enable` mask controls
746    /// which elements are written.
747    ///
748    /// # Safety
749    ///
750    /// Every enabled element must be in bounds for the `slice`.
751    ///
752    /// # Examples
753    /// ```
754    /// # #![feature(portable_simd)]
755    /// # #[cfg(feature = "as_crate")] use core_simd::simd;
756    /// # #[cfg(not(feature = "as_crate"))] use core::simd;
757    /// # use simd::{Simd, Mask};
758    /// let mut arr = [0i32; 4];
759    /// let write = Simd::from_array([-5, -4, -3, -2]);
760    /// let enable = Mask::from_array([false, true, true, true]);
761    ///
762    /// unsafe { write.store_select_unchecked(&mut arr, enable) };
763    /// assert_eq!(arr, [0, -4, -3, -2]);
764    /// ```
765    #[inline]
766    pub unsafe fn store_select_unchecked(
767        self,
768        slice: &mut [T],
769        enable: Mask<<T as SimdElement>::Mask, N>,
770    ) {
771        let ptr = slice.as_mut_ptr();
772        // SAFETY: The safety of writing elements in `slice` is ensured by the caller.
773        unsafe { self.store_select_ptr(ptr, enable) }
774    }
775
776    /// Conditionally write contiguous elements starting from `ptr`.
777    /// The `enable` mask controls which elements are written.
778    /// When disabled, the memory location corresponding to that element is not accessed.
779    ///
780    /// # Safety
781    ///
782    /// Memory addresses for element are calculated [`pointer::wrapping_offset`] and
783    /// each enabled element must satisfy the same conditions as [`core::ptr::write`].
784    #[inline]
785    pub unsafe fn store_select_ptr(self, ptr: *mut T, enable: Mask<<T as SimdElement>::Mask, N>) {
786        // SAFETY: The safety of writing elements through `ptr` is ensured by the caller.
787        unsafe {
788            core::intrinsics::simd::simd_masked_store::<_, _, _, { SimdAlign::Element }>(
789                enable.to_simd(),
790                ptr,
791                self,
792            )
793        }
794    }
795
796    /// Writes the values in a SIMD vector to potentially discontiguous indices in `slice`.
797    /// If an index is out-of-bounds, the write is suppressed without panicking.
798    /// If two elements in the scattered vector would write to the same index
799    /// only the last element is guaranteed to actually be written.
800    ///
801    /// # Examples
802    /// ```
803    /// # #![feature(portable_simd)]
804    /// # use core::simd::Simd;
805    /// let mut vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
806    /// let idxs = Simd::from_array([9, 3, 0, 0]); // Note the duplicate index.
807    /// let vals = Simd::from_array([-27, 82, -41, 124]);
808    ///
809    /// vals.scatter(&mut vec, idxs); // two logical writes means the last wins.
810    /// assert_eq!(vec, vec![124, 11, 12, 82, 14, 15, 16, 17, 18]);
811    /// ```
812    #[inline]
813    pub fn scatter(self, slice: &mut [T], idxs: Simd<usize, N>) {
814        self.scatter_select(slice, Mask::splat(true), idxs)
815    }
816
817    /// Writes values from a SIMD vector to multiple potentially discontiguous indices in `slice`.
818    /// The mask `enable`s all `true` indices and disables all `false` indices.
819    /// If an enabled index is out-of-bounds, the write is suppressed without panicking.
820    /// If two enabled elements in the scattered vector would write to the same index,
821    /// only the last element is guaranteed to actually be written.
822    ///
823    /// # Examples
824    /// ```
825    /// # #![feature(portable_simd)]
826    /// # #[cfg(feature = "as_crate")] use core_simd::simd;
827    /// # #[cfg(not(feature = "as_crate"))] use core::simd;
828    /// # use simd::{Simd, Mask};
829    /// let mut vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
830    /// let idxs = Simd::from_array([9, 3, 0, 0]); // Includes an out-of-bounds index
831    /// let vals = Simd::from_array([-27, 82, -41, 124]);
832    /// let enable = Mask::from_array([true, true, true, false]); // Includes a masked element
833    ///
834    /// vals.scatter_select(&mut vec, enable, idxs); // The last write is masked, thus omitted.
835    /// assert_eq!(vec, vec![-41, 11, 12, 82, 14, 15, 16, 17, 18]);
836    /// ```
837    #[inline]
838    pub fn scatter_select(self, slice: &mut [T], enable: Mask<isize, N>, idxs: Simd<usize, N>) {
839        let enable: Mask<isize, N> = enable & idxs.simd_lt(Simd::splat(slice.len()));
840        // Safety: We have masked-off out-of-bounds indices.
841        unsafe { self.scatter_select_unchecked(slice, enable, idxs) }
842    }
843
844    /// Writes values from a SIMD vector to multiple potentially discontiguous indices in `slice`.
845    /// The mask `enable`s all `true` indices and disables all `false` indices.
846    /// If two enabled elements in the scattered vector would write to the same index,
847    /// only the last element is guaranteed to actually be written.
848    ///
849    /// # Safety
850    ///
851    /// Calling this function with an enabled out-of-bounds index is *[undefined behavior]*,
852    /// and may lead to memory corruption.
853    ///
854    /// # Examples
855    /// ```
856    /// # #![feature(portable_simd)]
857    /// # #[cfg(feature = "as_crate")] use core_simd::simd;
858    /// # #[cfg(not(feature = "as_crate"))] use core::simd;
859    /// # use simd::{Simd, cmp::SimdPartialOrd, Mask};
860    /// let mut vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
861    /// let idxs = Simd::from_array([9, 3, 0, 0]);
862    /// let vals = Simd::from_array([-27, 82, -41, 124]);
863    /// let enable = Mask::from_array([true, true, true, false]); // Masks the final index
864    /// // If this mask was used to scatter, it would be unsound. Let's fix that.
865    /// let enable = enable & idxs.simd_lt(Simd::splat(vec.len()));
866    ///
867    /// // We have masked the OOB index, so it's safe to scatter now.
868    /// unsafe { vals.scatter_select_unchecked(&mut vec, enable, idxs); }
869    /// // The second write to index 0 was masked, thus omitted.
870    /// assert_eq!(vec, vec![-41, 11, 12, 82, 14, 15, 16, 17, 18]);
871    /// ```
872    /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html
873    #[inline]
874    #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
875    pub unsafe fn scatter_select_unchecked(
876        self,
877        slice: &mut [T],
878        enable: Mask<isize, N>,
879        idxs: Simd<usize, N>,
880    ) {
881        // Safety: This block works with *mut T derived from &mut 'a [T],
882        // which means it is delicate in Rust's borrowing model, circa 2021:
883        // &mut 'a [T] asserts uniqueness, so deriving &'a [T] invalidates live *mut Ts!
884        // Even though this block is largely safe methods, it must be exactly this way
885        // to prevent invalidating the raw ptrs while they're live.
886        // Thus, entering this block requires all values to use being already ready:
887        // 0. idxs we want to write to, which are used to construct the mask.
888        // 1. enable, which depends on an initial &'a [T] and the idxs.
889        // 2. actual values to scatter (self).
890        // 3. &mut [T] which will become our base ptr.
891        unsafe {
892            // Now Entering ☢️ *mut T Zone
893            let base_ptr = Simd::<*mut T, N>::splat(slice.as_mut_ptr());
894            // Ferris forgive me, I have done pointer arithmetic here.
895            let ptrs = base_ptr.wrapping_add(idxs);
896            // The ptrs have been bounds-masked to prevent memory-unsafe writes insha'allah
897            self.scatter_select_ptr(ptrs, enable);
898            // Cleared ☢️ *mut T Zone
899        }
900    }
901
902    /// Writes pointers elementwise into a SIMD vector.
903    ///
904    /// # Safety
905    ///
906    /// Each write must satisfy the same conditions as [`core::ptr::write`].
907    ///
908    /// # Example
909    /// ```
910    /// # #![feature(portable_simd)]
911    /// # #[cfg(feature = "as_crate")] use core_simd::simd;
912    /// # #[cfg(not(feature = "as_crate"))] use core::simd;
913    /// # use simd::{Simd, ptr::SimdMutPtr};
914    /// let mut values = [0; 4];
915    /// let offset = Simd::from_array([3, 2, 1, 0]);
916    /// let ptrs = Simd::splat(values.as_mut_ptr()).wrapping_add(offset);
917    /// unsafe { Simd::from_array([6, 3, 5, 7]).scatter_ptr(ptrs); }
918    /// assert_eq!(values, [7, 5, 3, 6]);
919    /// ```
920    #[inline]
921    #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
922    pub unsafe fn scatter_ptr(self, dest: Simd<*mut T, N>) {
923        // Safety: The caller is responsible for upholding all invariants
924        unsafe { self.scatter_select_ptr(dest, Mask::splat(true)) }
925    }
926
927    /// Conditionally write pointers elementwise into a SIMD vector.
928    /// The mask `enable`s all `true` pointers and disables all `false` pointers.
929    /// If a pointer is disabled, the write to its pointee is skipped.
930    ///
931    /// # Safety
932    ///
933    /// Enabled pointers must satisfy the same conditions as [`core::ptr::write`].
934    ///
935    /// # Example
936    /// ```
937    /// # #![feature(portable_simd)]
938    /// # #[cfg(feature = "as_crate")] use core_simd::simd;
939    /// # #[cfg(not(feature = "as_crate"))] use core::simd;
940    /// # use simd::{Mask, Simd, ptr::SimdMutPtr};
941    /// let mut values = [0; 4];
942    /// let offset = Simd::from_array([3, 2, 1, 0]);
943    /// let ptrs = Simd::splat(values.as_mut_ptr()).wrapping_add(offset);
944    /// let enable = Mask::from_array([true, true, false, false]);
945    /// unsafe { Simd::from_array([6, 3, 5, 7]).scatter_select_ptr(ptrs, enable); }
946    /// assert_eq!(values, [0, 0, 3, 6]);
947    /// ```
948    #[inline]
949    #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
950    pub unsafe fn scatter_select_ptr(self, dest: Simd<*mut T, N>, enable: Mask<isize, N>) {
951        // Safety: The caller is responsible for upholding all invariants
952        unsafe { core::intrinsics::simd::simd_scatter(self, dest, enable.to_simd()) }
953    }
954}
955
956impl<T, const N: usize> Copy for Simd<T, N> where T: SimdElement {}
957
958impl<T, const N: usize> Clone for Simd<T, N>
959where
960    T: SimdElement,
961{
962    #[inline]
963    fn clone(&self) -> Self {
964        *self
965    }
966}
967
968impl<T, const N: usize> Default for Simd<T, N>
969where
970    T: SimdElement + Default,
971{
972    #[inline]
973    fn default() -> Self {
974        Self::splat(T::default())
975    }
976}
977
978impl<T, const N: usize> PartialEq for Simd<T, N>
979where
980    T: SimdElement + PartialEq,
981{
982    #[inline]
983    fn eq(&self, other: &Self) -> bool {
984        // Safety: All SIMD vectors are SimdPartialEq, and the comparison produces a valid mask.
985        let mask = unsafe {
986            let tfvec: Simd<<T as SimdElement>::Mask, N> =
987                core::intrinsics::simd::simd_eq(*self, *other);
988            Mask::from_simd_unchecked(tfvec)
989        };
990
991        // Two vectors are equal if all elements are equal when compared elementwise
992        mask.all()
993    }
994
995    #[allow(clippy::partialeq_ne_impl)]
996    #[inline]
997    fn ne(&self, other: &Self) -> bool {
998        // Safety: All SIMD vectors are SimdPartialEq, and the comparison produces a valid mask.
999        let mask = unsafe {
1000            let tfvec: Simd<<T as SimdElement>::Mask, N> =
1001                core::intrinsics::simd::simd_ne(*self, *other);
1002            Mask::from_simd_unchecked(tfvec)
1003        };
1004
1005        // Two vectors are non-equal if any elements are non-equal when compared elementwise
1006        mask.any()
1007    }
1008}
1009
1010/// Lexicographic order. For the SIMD elementwise minimum and maximum, use simd_min and simd_max instead.
1011impl<T, const N: usize> PartialOrd for Simd<T, N>
1012where
1013    T: SimdElement + PartialOrd,
1014{
1015    #[inline]
1016    fn partial_cmp(&self, other: &Self) -> Option<core::cmp::Ordering> {
1017        // TODO use SIMD equality
1018        self.to_array().partial_cmp(other.as_ref())
1019    }
1020}
1021
1022impl<T, const N: usize> Eq for Simd<T, N> where T: SimdElement + Eq {}
1023
1024/// Lexicographic order. For the SIMD elementwise minimum and maximum, use simd_min and simd_max instead.
1025impl<T, const N: usize> Ord for Simd<T, N>
1026where
1027    T: SimdElement + Ord,
1028{
1029    #[inline]
1030    fn cmp(&self, other: &Self) -> core::cmp::Ordering {
1031        // TODO use SIMD equality
1032        self.to_array().cmp(other.as_ref())
1033    }
1034}
1035
1036impl<T, const N: usize> core::hash::Hash for Simd<T, N>
1037where
1038    T: SimdElement + core::hash::Hash,
1039{
1040    #[inline]
1041    fn hash<H>(&self, state: &mut H)
1042    where
1043        H: core::hash::Hasher,
1044    {
1045        self.as_array().hash(state)
1046    }
1047}
1048
1049// array references
1050impl<T, const N: usize> AsRef<[T; N]> for Simd<T, N>
1051where
1052    T: SimdElement,
1053{
1054    #[inline]
1055    fn as_ref(&self) -> &[T; N] {
1056        self.as_array()
1057    }
1058}
1059
1060impl<T, const N: usize> AsMut<[T; N]> for Simd<T, N>
1061where
1062    T: SimdElement,
1063{
1064    #[inline]
1065    fn as_mut(&mut self) -> &mut [T; N] {
1066        self.as_mut_array()
1067    }
1068}
1069
1070// slice references
1071impl<T, const N: usize> AsRef<[T]> for Simd<T, N>
1072where
1073    T: SimdElement,
1074{
1075    #[inline]
1076    fn as_ref(&self) -> &[T] {
1077        self.as_array()
1078    }
1079}
1080
1081impl<T, const N: usize> AsMut<[T]> for Simd<T, N>
1082where
1083    T: SimdElement,
1084{
1085    #[inline]
1086    fn as_mut(&mut self) -> &mut [T] {
1087        self.as_mut_array()
1088    }
1089}
1090
1091// vector/array conversion
1092impl<T, const N: usize> From<[T; N]> for Simd<T, N>
1093where
1094    T: SimdElement,
1095{
1096    #[inline]
1097    fn from(array: [T; N]) -> Self {
1098        Self::from_array(array)
1099    }
1100}
1101
1102impl<T, const N: usize> From<Simd<T, N>> for [T; N]
1103where
1104    T: SimdElement,
1105{
1106    #[inline]
1107    fn from(vector: Simd<T, N>) -> Self {
1108        vector.to_array()
1109    }
1110}
1111
1112impl<T, const N: usize> TryFrom<&[T]> for Simd<T, N>
1113where
1114    T: SimdElement,
1115{
1116    type Error = core::array::TryFromSliceError;
1117
1118    #[inline]
1119    fn try_from(slice: &[T]) -> Result<Self, core::array::TryFromSliceError> {
1120        Ok(Self::from_array(slice.try_into()?))
1121    }
1122}
1123
1124impl<T, const N: usize> TryFrom<&mut [T]> for Simd<T, N>
1125where
1126    T: SimdElement,
1127{
1128    type Error = core::array::TryFromSliceError;
1129
1130    #[inline]
1131    fn try_from(slice: &mut [T]) -> Result<Self, core::array::TryFromSliceError> {
1132        Ok(Self::from_array(slice.try_into()?))
1133    }
1134}
1135
1136/// Marker trait for types that may be used as SIMD vector elements.
1137///
1138/// # Safety
1139/// This trait, when implemented, asserts the compiler can monomorphize
1140/// `#[repr(simd)]` structs with the marked type as an element.
1141/// Strictly, it is valid to impl if the vector will not be miscompiled.
1142/// Practically, it is user-unfriendly to impl it if the vector won't compile,
1143/// even when no soundness guarantees are broken by allowing the user to try.
1144pub impl(self) unsafe trait SimdElement: Copy {
1145    /// The mask element type corresponding to this element type.
1146    type Mask: MaskElement;
1147}
1148
1149// Safety: u8 is a valid SIMD element type, and is supported by this API
1150unsafe impl SimdElement for u8 {
1151    type Mask = i8;
1152}
1153
1154// Safety: u16 is a valid SIMD element type, and is supported by this API
1155unsafe impl SimdElement for u16 {
1156    type Mask = i16;
1157}
1158
1159// Safety: u32 is a valid SIMD element type, and is supported by this API
1160unsafe impl SimdElement for u32 {
1161    type Mask = i32;
1162}
1163
1164// Safety: u64 is a valid SIMD element type, and is supported by this API
1165unsafe impl SimdElement for u64 {
1166    type Mask = i64;
1167}
1168
1169// Safety: usize is a valid SIMD element type, and is supported by this API
1170unsafe impl SimdElement for usize {
1171    type Mask = isize;
1172}
1173
1174// Safety: i8 is a valid SIMD element type, and is supported by this API
1175unsafe impl SimdElement for i8 {
1176    type Mask = i8;
1177}
1178
1179// Safety: i16 is a valid SIMD element type, and is supported by this API
1180unsafe impl SimdElement for i16 {
1181    type Mask = i16;
1182}
1183
1184// Safety: i32 is a valid SIMD element type, and is supported by this API
1185unsafe impl SimdElement for i32 {
1186    type Mask = i32;
1187}
1188
1189// Safety: i64 is a valid SIMD element type, and is supported by this API
1190unsafe impl SimdElement for i64 {
1191    type Mask = i64;
1192}
1193
1194// Safety: isize is a valid SIMD element type, and is supported by this API
1195unsafe impl SimdElement for isize {
1196    type Mask = isize;
1197}
1198
1199// Safety: f16 is a valid SIMD element type, and is supported by this API
1200unsafe impl SimdElement for f16 {
1201    type Mask = i16;
1202}
1203
1204// Safety: f32 is a valid SIMD element type, and is supported by this API
1205unsafe impl SimdElement for f32 {
1206    type Mask = i32;
1207}
1208
1209// Safety: f64 is a valid SIMD element type, and is supported by this API
1210unsafe impl SimdElement for f64 {
1211    type Mask = i64;
1212}
1213
1214// Safety: (thin) const pointers are valid SIMD element types, and are supported by this API
1215//
1216// Fat pointers may be supported in the future.
1217unsafe impl<T> SimdElement for *const T
1218where
1219    T: core::ptr::Pointee<Metadata = ()>,
1220{
1221    type Mask = isize;
1222}
1223
1224// Safety: (thin) mut pointers are valid SIMD element types, and are supported by this API
1225//
1226// Fat pointers may be supported in the future.
1227unsafe impl<T> SimdElement for *mut T
1228where
1229    T: core::ptr::Pointee<Metadata = ()>,
1230{
1231    type Mask = isize;
1232}
1233
1234#[inline]
1235fn lane_indices<const N: usize>() -> Simd<usize, N> {
1236    #![allow(clippy::needless_range_loop)]
1237    let mut index = [0; N];
1238    for i in 0..N {
1239        index[i] = i;
1240    }
1241    Simd::from_array(index)
1242}
1243
1244#[inline]
1245fn mask_up_to<M, const N: usize>(len: usize) -> Mask<M, N>
1246where
1247    M: MaskElement,
1248{
1249    let index = lane_indices::<N>();
1250    let max_value: u64 = M::max_unsigned();
1251    macro_rules! case {
1252        ($ty:ty) => {
1253            if N < <$ty>::MAX as usize && max_value as $ty as u64 == max_value {
1254                return index.cast().simd_lt(Simd::splat(len.min(N) as $ty)).cast();
1255            }
1256        };
1257    }
1258    case!(u8);
1259    case!(u16);
1260    case!(u32);
1261    case!(u64);
1262    index.simd_lt(Simd::splat(len)).cast()
1263}