core_simd/vector.rs
1use core::intrinsics::simd::SimdAlign;
2
3use crate::simd::{
4 Mask, MaskElement,
5 cmp::SimdPartialOrd,
6 num::SimdUint,
7 ptr::{SimdConstPtr, SimdMutPtr},
8};
9
10/// A SIMD vector with the shape of `[T; N]` but the operations of `T`.
11///
12/// `Simd<T, N>` supports the operators (+, *, etc.) that `T` does in "elementwise" fashion.
13/// These take the element at each index from the left-hand side and right-hand side,
14/// perform the operation, then return the result in the same index in a vector of equal size.
15/// However, `Simd` differs from normal iteration and normal arrays:
16/// - `Simd<T, N>` executes `N` operations in a single step with no `break`s
17/// - `Simd<T, N>` can have an alignment greater than `T`, for better mechanical sympathy
18///
19/// By always imposing these constraints on `Simd`, it is easier to compile elementwise operations
20/// into machine instructions that can themselves be executed in parallel.
21///
22/// ```rust
23/// # #![feature(portable_simd)]
24/// # use core::simd::{Simd};
25/// # use core::array;
26/// let a: [i32; 4] = [-2, 0, 2, 4];
27/// let b = [10, 9, 8, 7];
28/// let sum = array::from_fn(|i| a[i] + b[i]);
29/// let prod = array::from_fn(|i| a[i] * b[i]);
30///
31/// // `Simd<T, N>` implements `From<[T; N]>`
32/// let (v, w) = (Simd::from(a), Simd::from(b));
33/// // Which means arrays implement `Into<Simd<T, N>>`.
34/// assert_eq!(v + w, sum.into());
35/// assert_eq!(v * w, prod.into());
36/// ```
37///
38///
39/// `Simd` with integer elements treats operators as wrapping, as if `T` was [`Wrapping<T>`].
40/// Thus, `Simd` does not implement `wrapping_add`, because that is the default behavior.
41/// This means there is no warning on overflows, even in "debug" builds.
42/// For most applications where `Simd` is appropriate, it is "not a bug" to wrap,
43/// and even "debug builds" are unlikely to tolerate the loss of performance.
44/// You may want to consider using explicitly checked arithmetic if such is required.
45/// Division by zero on integers still causes a panic, so
46/// you may want to consider using `f32` or `f64` if that is unacceptable.
47///
48/// [`Wrapping<T>`]: core::num::Wrapping
49///
50/// # Layout
51/// `Simd<T, N>` has a layout similar to `[T; N]` (identical "shapes"), with a greater alignment.
52/// `[T; N]` is aligned to `T`, but `Simd<T, N>` will have an alignment based on both `T` and `N`.
53/// Thus it is sound to [`transmute`] `Simd<T, N>` to `[T; N]` and should optimize to "zero cost",
54/// but the reverse transmutation may require a copy the compiler cannot simply elide.
55///
56/// `N` cannot be 0 and may be at most 64. This limit may be increased in the future.
57///
58/// # ABI "Features"
59/// Due to Rust's safety guarantees, `Simd<T, N>` is currently passed and returned via memory,
60/// not SIMD registers, except as an optimization. Using `#[inline]` on functions that accept
61/// `Simd<T, N>` or return it is recommended, at the cost of code generation time, as
62/// inlining SIMD-using functions can omit a large function prolog or epilog and thus
63/// improve both speed and code size. The need for this may be corrected in the future.
64///
65/// Using `#[inline(always)]` still requires additional care.
66///
67/// # Safe SIMD with Unsafe Rust
68///
69/// Operations with `Simd` are typically safe, but there are many reasons to want to combine SIMD with `unsafe` code.
70/// Care must be taken to respect differences between `Simd` and other types it may be transformed into or derived from.
71/// In particular, the layout of `Simd<T, N>` may be similar to `[T; N]`, and may allow some transmutations,
72/// but references to `[T; N]` are not interchangeable with those to `Simd<T, N>`.
73/// Thus, when using `unsafe` Rust to read and write `Simd<T, N>` through [raw pointers], it is a good idea to first try with
74/// [`read_unaligned`] and [`write_unaligned`]. This is because:
75/// - [`read`] and [`write`] require full alignment (in this case, `Simd<T, N>`'s alignment)
76/// - `Simd<T, N>` is often read from or written to [`[T]`](slice) and other types aligned to `T`
77/// - combining these actions violates the `unsafe` contract and explodes the program into
78/// a puff of **undefined behavior**
79/// - the compiler can implicitly adjust layouts to make unaligned reads or writes fully aligned
80/// if it sees the optimization
81/// - most contemporary processors with "aligned" and "unaligned" read and write instructions
82/// exhibit no performance difference if the "unaligned" variant is aligned at runtime
83///
84/// Less obligations mean unaligned reads and writes are less likely to make the program unsound,
85/// and may be just as fast as stricter alternatives.
86/// When trying to guarantee alignment, [`[T]::as_simd`][as_simd] is an option for
87/// converting `[T]` to `[Simd<T, N>]`, and allows soundly operating on an aligned SIMD body,
88/// but it may cost more time when handling the scalar head and tail.
89/// If these are not enough, it is most ideal to design data structures to be already aligned
90/// to `align_of::<Simd<T, N>>()` before using `unsafe` Rust to read or write.
91/// Other ways to compensate for these facts, like materializing `Simd` to or from an array first,
92/// are handled by safe methods like [`Simd::from_array`] and [`Simd::from_slice`].
93///
94/// [`transmute`]: core::mem::transmute
95/// [raw pointers]: pointer
96/// [`read_unaligned`]: pointer::read_unaligned
97/// [`write_unaligned`]: pointer::write_unaligned
98/// [`read`]: pointer::read
99/// [`write`]: pointer::write
100/// [as_simd]: slice::as_simd
101//
102// NOTE: Accessing the inner array directly in any way (e.g. by using the `.0` field syntax) or
103// directly constructing an instance of the type (i.e. `let vector = Simd(array)`) should be
104// avoided, as it will likely become illegal on `#[repr(simd)]` structs in the future. It also
105// causes rustc to emit illegal LLVM IR in some cases.
106#[repr(simd, packed)]
107#[rustc_simd_monomorphize_lane_limit = "64"]
108pub struct Simd<T, const N: usize>([T; N])
109where
110 T: SimdElement;
111
112impl<T, const N: usize> Simd<T, N>
113where
114 T: SimdElement,
115{
116 /// Number of elements in this vector.
117 pub const LEN: usize = N;
118
119 /// Returns the number of elements in this SIMD vector.
120 ///
121 /// # Examples
122 ///
123 /// ```
124 /// # #![feature(portable_simd)]
125 /// # #[cfg(feature = "as_crate")] use core_simd::simd;
126 /// # #[cfg(not(feature = "as_crate"))] use core::simd;
127 /// # use simd::u32x4;
128 /// let v = u32x4::splat(0);
129 /// assert_eq!(v.len(), 4);
130 /// ```
131 #[inline]
132 #[allow(clippy::len_without_is_empty)]
133 pub const fn len(&self) -> usize {
134 Self::LEN
135 }
136
137 /// Constructs a new SIMD vector with all elements set to the given value.
138 ///
139 /// # Examples
140 ///
141 /// ```
142 /// # #![feature(portable_simd)]
143 /// # #[cfg(feature = "as_crate")] use core_simd::simd;
144 /// # #[cfg(not(feature = "as_crate"))] use core::simd;
145 /// # use simd::u32x4;
146 /// let v = u32x4::splat(8);
147 /// assert_eq!(v.as_array(), &[8, 8, 8, 8]);
148 /// ```
149 #[inline]
150 #[rustc_const_unstable(feature = "portable_simd", issue = "86656")]
151 pub const fn splat(value: T) -> Self {
152 // SAFETY: T is a SimdElement, and the item type of Self.
153 unsafe { core::intrinsics::simd::simd_splat(value) }
154 }
155
156 /// Returns an array reference containing the entire SIMD vector.
157 ///
158 /// For reading a single element, prefer [`Self::get`], which may optimize better.
159 ///
160 /// # Examples
161 ///
162 /// ```
163 /// # #![feature(portable_simd)]
164 /// # use core::simd::{Simd, u64x4};
165 /// let v: u64x4 = Simd::from_array([0, 1, 2, 3]);
166 /// assert_eq!(v.as_array(), &[0, 1, 2, 3]);
167 /// ```
168 #[inline]
169 pub const fn as_array(&self) -> &[T; N] {
170 // SAFETY: `Simd<T, N>` is just an overaligned `[T; N]` with
171 // potential padding at the end, so pointer casting to a
172 // `&[T; N]` is safe.
173 //
174 // NOTE: This deliberately doesn't just use `&self.0`, see the comment
175 // on the struct definition for details.
176 unsafe { &*(self as *const Self as *const [T; N]) }
177 }
178
179 /// Returns a mutable array reference containing the entire SIMD vector.
180 ///
181 /// For updating a single element, prefer [`Self::set`], which may optimize better.
182 #[inline]
183 pub const fn as_mut_array(&mut self) -> &mut [T; N] {
184 // SAFETY: `Simd<T, N>` is just an overaligned `[T; N]` with
185 // potential padding at the end, so pointer casting to a
186 // `&mut [T; N]` is safe.
187 //
188 // NOTE: This deliberately doesn't just use `&mut self.0`, see the comment
189 // on the struct definition for details.
190 unsafe { &mut *(self as *mut Self as *mut [T; N]) }
191 }
192
193 /// Returns the element at `index`.
194 ///
195 /// This may optimize better than reading an element via [`Self::as_array`].
196 ///
197 /// # Panics
198 ///
199 /// Panics if `index` is greater than or equal to the number of elements in the vector.
200 #[inline]
201 #[must_use]
202 #[track_caller]
203 pub fn get(&self, index: usize) -> T {
204 assert!(
205 index < N,
206 "index out of bounds: the len is {N} but the index is {index}"
207 );
208 // safety: index < N
209 unsafe { self.get_unchecked(index) }
210 }
211
212 /// Returns the element at `index` without checking bounds.
213 ///
214 /// # Safety
215 ///
216 /// `index` must be less than the number of elements in the vector.
217 #[inline]
218 #[must_use]
219 pub unsafe fn get_unchecked(&self, index: usize) -> T {
220 // safety: the caller guarantees that index < N
221 unsafe {
222 core::hint::assert_unchecked(index < N);
223 core::intrinsics::simd::simd_extract_dyn(*self, index as u32)
224 }
225 }
226
227 /// Sets the element at `index` to `value`.
228 ///
229 /// This may optimize better than updating an element via [`Self::as_mut_array`].
230 ///
231 /// # Panics
232 ///
233 /// Panics if `index` is greater than or equal to the number of elements in the vector.
234 #[inline]
235 #[track_caller]
236 pub fn set(&mut self, index: usize, value: T) {
237 assert!(
238 index < N,
239 "index out of bounds: the len is {N} but the index is {index}"
240 );
241 // safety: index < N
242 unsafe { self.set_unchecked(index, value) }
243 }
244
245 /// Sets the element at `index` to `value` without checking bounds.
246 ///
247 /// # Safety
248 ///
249 /// `index` must be less than the number of elements in the vector.
250 #[inline]
251 pub unsafe fn set_unchecked(&mut self, index: usize, value: T) {
252 // safety: the caller guarantees that index < N
253 unsafe {
254 core::hint::assert_unchecked(index < N);
255 if cfg!(target_arch = "wasm32") {
256 // work around https://github.com/llvm/llvm-project/issues/190306
257 *self.as_mut_array().get_unchecked_mut(index) = value;
258 } else {
259 *self = core::intrinsics::simd::simd_insert_dyn(*self, index as u32, value);
260 }
261 }
262 }
263
264 /// Loads a vector from an array of `T`.
265 ///
266 /// This function is necessary since `repr(simd)` has padding for non-power-of-2 vectors (at the time of writing).
267 /// With padding, `read_unaligned` will read past the end of an array of N elements.
268 ///
269 /// # Safety
270 /// Reading `ptr` must be safe, as if by `<*const [T; N]>::read`.
271 #[inline]
272 const unsafe fn load(ptr: *const [T; N]) -> Self {
273 // There are potentially simpler ways to write this function, but this should result in
274 // LLVM `load <N x T>`
275
276 let mut tmp = core::mem::MaybeUninit::<Self>::uninit();
277 // SAFETY: `Simd<T, N>` always contains `N` elements of type `T`. It may have padding
278 // which does not need to be initialized. The safety of reading `ptr` is ensured by the
279 // caller.
280 unsafe {
281 core::ptr::copy_nonoverlapping(ptr, tmp.as_mut_ptr().cast(), 1);
282 tmp.assume_init()
283 }
284 }
285
286 /// Store a vector to an array of `T`.
287 ///
288 /// See `load` as to why this function is necessary.
289 ///
290 /// # Safety
291 /// Writing to `ptr` must be safe, as if by `<*mut [T; N]>::write`.
292 #[inline]
293 const unsafe fn store(self, ptr: *mut [T; N]) {
294 // There are potentially simpler ways to write this function, but this should result in
295 // LLVM `store <N x T>`
296
297 // Creating a temporary helps LLVM turn the memcpy into a store.
298 let tmp = self;
299 // SAFETY: `Simd<T, N>` always contains `N` elements of type `T`. The safety of writing
300 // `ptr` is ensured by the caller.
301 unsafe { core::ptr::copy_nonoverlapping(tmp.as_array(), ptr, 1) }
302 }
303
304 /// Converts an array to a SIMD vector.
305 #[inline]
306 pub const fn from_array(array: [T; N]) -> Self {
307 // SAFETY: `&array` is safe to read.
308 //
309 // FIXME: We currently use a pointer load instead of `transmute_copy` because `repr(simd)`
310 // results in padding for non-power-of-2 vectors (so vectors are larger than arrays).
311 //
312 // NOTE: This deliberately doesn't just use `Self(array)`, see the comment
313 // on the struct definition for details.
314 unsafe { Self::load(&array) }
315 }
316
317 /// Converts a SIMD vector to an array.
318 #[inline]
319 pub const fn to_array(self) -> [T; N] {
320 let mut tmp = core::mem::MaybeUninit::uninit();
321 // SAFETY: writing to `tmp` is safe and initializes it.
322 //
323 // FIXME: We currently use a pointer store instead of `transmute_copy` because `repr(simd)`
324 // results in padding for non-power-of-2 vectors (so vectors are larger than arrays).
325 //
326 // NOTE: This deliberately doesn't just use `self.0`, see the comment
327 // on the struct definition for details.
328 unsafe {
329 self.store(tmp.as_mut_ptr());
330 tmp.assume_init()
331 }
332 }
333
334 /// Converts a slice to a SIMD vector containing `slice[..N]`.
335 ///
336 /// # Panics
337 ///
338 /// Panics if the slice's length is less than the vector's `Simd::N`.
339 /// Use `load_or_default` for an alternative that does not panic.
340 ///
341 /// # Example
342 ///
343 /// ```
344 /// # #![feature(portable_simd)]
345 /// # use core::simd::u32x4;
346 /// let source = vec![1, 2, 3, 4, 5, 6];
347 /// let v = u32x4::from_slice(&source);
348 /// assert_eq!(v.as_array(), &[1, 2, 3, 4]);
349 /// ```
350 #[must_use]
351 #[inline]
352 #[track_caller]
353 pub const fn from_slice(slice: &[T]) -> Self {
354 assert!(
355 slice.len() >= Self::LEN,
356 "slice length must be at least the number of elements"
357 );
358 // SAFETY: We just checked that the slice contains
359 // at least `N` elements.
360 unsafe { Self::load(slice.as_ptr().cast()) }
361 }
362
363 /// Writes a SIMD vector to the first `N` elements of a slice.
364 ///
365 /// # Panics
366 ///
367 /// Panics if the slice's length is less than the vector's `Simd::N`.
368 ///
369 /// # Example
370 ///
371 /// ```
372 /// # #![feature(portable_simd)]
373 /// # #[cfg(feature = "as_crate")] use core_simd::simd;
374 /// # #[cfg(not(feature = "as_crate"))] use core::simd;
375 /// # use simd::u32x4;
376 /// let mut dest = vec![0; 6];
377 /// let v = u32x4::from_array([1, 2, 3, 4]);
378 /// v.copy_to_slice(&mut dest);
379 /// assert_eq!(&dest, &[1, 2, 3, 4, 0, 0]);
380 /// ```
381 #[inline]
382 #[track_caller]
383 pub const fn copy_to_slice(self, slice: &mut [T]) {
384 assert!(
385 slice.len() >= Self::LEN,
386 "slice length must be at least the number of elements"
387 );
388 // SAFETY: We just checked that the slice contains
389 // at least `N` elements.
390 unsafe { self.store(slice.as_mut_ptr().cast()) }
391 }
392
393 /// Reads contiguous elements from `slice`. Elements are read so long as they're in-bounds for
394 /// the `slice`. Otherwise, the default value for the element type is returned.
395 ///
396 /// # Examples
397 /// ```
398 /// # #![feature(portable_simd)]
399 /// # #[cfg(feature = "as_crate")] use core_simd::simd;
400 /// # #[cfg(not(feature = "as_crate"))] use core::simd;
401 /// # use simd::Simd;
402 /// let vec: Vec<i32> = vec![10, 11];
403 ///
404 /// let result = Simd::<i32, 4>::load_or_default(&vec);
405 /// assert_eq!(result, Simd::from_array([10, 11, 0, 0]));
406 /// ```
407 #[must_use]
408 #[inline]
409 pub fn load_or_default(slice: &[T]) -> Self
410 where
411 T: Default,
412 {
413 Self::load_or(slice, Default::default())
414 }
415
416 /// Reads contiguous elements from `slice`. Elements are read so long as they're in-bounds for
417 /// the `slice`. Otherwise, the corresponding value from `or` is passed through.
418 ///
419 /// # Examples
420 /// ```
421 /// # #![feature(portable_simd)]
422 /// # #[cfg(feature = "as_crate")] use core_simd::simd;
423 /// # #[cfg(not(feature = "as_crate"))] use core::simd;
424 /// # use simd::Simd;
425 /// let vec: Vec<i32> = vec![10, 11];
426 /// let or = Simd::from_array([-5, -4, -3, -2]);
427 ///
428 /// let result = Simd::load_or(&vec, or);
429 /// assert_eq!(result, Simd::from_array([10, 11, -3, -2]));
430 /// ```
431 #[must_use]
432 #[inline]
433 pub fn load_or(slice: &[T], or: Self) -> Self {
434 Self::load_select(slice, Mask::splat(true), or)
435 }
436
437 /// Reads contiguous elements from `slice`. Each element is read from memory if its
438 /// corresponding element in `enable` is `true`.
439 ///
440 /// When the element is disabled or out of bounds for the slice, that memory location
441 /// is not accessed and the default value for the element type is returned.
442 ///
443 /// # Examples
444 /// ```
445 /// # #![feature(portable_simd)]
446 /// # #[cfg(feature = "as_crate")] use core_simd::simd;
447 /// # #[cfg(not(feature = "as_crate"))] use core::simd;
448 /// # use simd::{Simd, Mask};
449 /// let vec: Vec<i32> = vec![10, 11, 12];
450 /// let enable = Mask::from_array([false, true, true, true]);
451 ///
452 /// let result = Simd::load_select_or_default(&vec, enable);
453 /// assert_eq!(result, Simd::from_array([0, 11, 12, 0]));
454 /// ```
455 #[must_use]
456 #[inline]
457 pub fn load_select_or_default(slice: &[T], enable: Mask<<T as SimdElement>::Mask, N>) -> Self
458 where
459 T: Default,
460 {
461 Self::load_select(slice, enable, Default::default())
462 }
463
464 /// Reads contiguous elements from `slice`. Each element is read from memory if its
465 /// corresponding element in `enable` is `true`.
466 ///
467 /// When the element is disabled or out of bounds for the slice, that memory location
468 /// is not accessed and the corresponding value from `or` is passed through.
469 ///
470 /// # Examples
471 /// ```
472 /// # #![feature(portable_simd)]
473 /// # #[cfg(feature = "as_crate")] use core_simd::simd;
474 /// # #[cfg(not(feature = "as_crate"))] use core::simd;
475 /// # use simd::{Simd, Mask};
476 /// let vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
477 /// let enable = Mask::from_array([true, true, false, true]);
478 /// let or = Simd::from_array([-5, -4, -3, -2]);
479 ///
480 /// let result = Simd::load_select(&vec, enable, or);
481 /// assert_eq!(result, Simd::from_array([10, 11, -3, 13]));
482 /// ```
483 #[must_use]
484 #[inline]
485 pub fn load_select(
486 slice: &[T],
487 mut enable: Mask<<T as SimdElement>::Mask, N>,
488 or: Self,
489 ) -> Self {
490 enable &= mask_up_to(slice.len());
491 // SAFETY: We performed the bounds check by updating the mask. &[T] is properly aligned to
492 // the element.
493 unsafe { Self::load_select_ptr(slice.as_ptr(), enable, or) }
494 }
495
496 /// Reads contiguous elements from `slice`. Each element is read from memory if its
497 /// corresponding element in `enable` is `true`.
498 ///
499 /// When the element is disabled, that memory location is not accessed and the corresponding
500 /// value from `or` is passed through.
501 ///
502 /// # Safety
503 /// Enabled loads must not exceed the length of `slice`.
504 #[must_use]
505 #[inline]
506 pub unsafe fn load_select_unchecked(
507 slice: &[T],
508 enable: Mask<<T as SimdElement>::Mask, N>,
509 or: Self,
510 ) -> Self {
511 let ptr = slice.as_ptr();
512 // SAFETY: The safety of reading elements from `slice` is ensured by the caller.
513 unsafe { Self::load_select_ptr(ptr, enable, or) }
514 }
515
516 /// Reads contiguous elements starting at `ptr`. Each element is read from memory if its
517 /// corresponding element in `enable` is `true`.
518 ///
519 /// When the element is disabled, that memory location is not accessed and the corresponding
520 /// value from `or` is passed through.
521 ///
522 /// # Safety
523 /// Enabled `ptr` elements must be safe to read as if by `core::ptr::read`.
524 #[must_use]
525 #[inline]
526 pub unsafe fn load_select_ptr(
527 ptr: *const T,
528 enable: Mask<<T as SimdElement>::Mask, N>,
529 or: Self,
530 ) -> Self {
531 // SAFETY: The safety of reading elements through `ptr` is ensured by the caller.
532 unsafe {
533 core::intrinsics::simd::simd_masked_load::<_, _, _, { SimdAlign::Element }>(
534 enable.to_simd(),
535 ptr,
536 or,
537 )
538 }
539 }
540
541 /// Reads from potentially discontiguous indices in `slice` to construct a SIMD vector.
542 /// If an index is out-of-bounds, the element is instead selected from the `or` vector.
543 ///
544 /// # Examples
545 /// ```
546 /// # #![feature(portable_simd)]
547 /// # use core::simd::Simd;
548 /// let vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
549 /// let idxs = Simd::from_array([9, 3, 0, 5]); // Note the index that is out-of-bounds
550 /// let alt = Simd::from_array([-5, -4, -3, -2]);
551 ///
552 /// let result = Simd::gather_or(&vec, idxs, alt);
553 /// assert_eq!(result, Simd::from_array([-5, 13, 10, 15]));
554 /// ```
555 #[must_use]
556 #[inline]
557 pub fn gather_or(slice: &[T], idxs: Simd<usize, N>, or: Self) -> Self {
558 Self::gather_select(slice, Mask::splat(true), idxs, or)
559 }
560
561 /// Reads from indices in `slice` to construct a SIMD vector.
562 /// If an index is out-of-bounds, the element is set to the default given by `T: Default`.
563 ///
564 /// # Examples
565 /// ```
566 /// # #![feature(portable_simd)]
567 /// # use core::simd::Simd;
568 /// let vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
569 /// let idxs = Simd::from_array([9, 3, 0, 5]); // Note the index that is out-of-bounds
570 ///
571 /// let result = Simd::gather_or_default(&vec, idxs);
572 /// assert_eq!(result, Simd::from_array([0, 13, 10, 15]));
573 /// ```
574 #[must_use]
575 #[inline]
576 pub fn gather_or_default(slice: &[T], idxs: Simd<usize, N>) -> Self
577 where
578 T: Default,
579 {
580 Self::gather_or(slice, idxs, Self::splat(T::default()))
581 }
582
583 /// Reads from indices in `slice` to construct a SIMD vector.
584 /// The mask `enable`s all `true` indices and disables all `false` indices.
585 /// If an index is disabled or is out-of-bounds, the element is selected from the `or` vector.
586 ///
587 /// # Examples
588 /// ```
589 /// # #![feature(portable_simd)]
590 /// # use core::simd::{Simd, Mask};
591 /// let vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
592 /// let idxs = Simd::from_array([9, 3, 0, 5]); // Includes an out-of-bounds index
593 /// let alt = Simd::from_array([-5, -4, -3, -2]);
594 /// let enable = Mask::from_array([true, true, true, false]); // Includes a masked element
595 ///
596 /// let result = Simd::gather_select(&vec, enable, idxs, alt);
597 /// assert_eq!(result, Simd::from_array([-5, 13, 10, -2]));
598 /// ```
599 #[must_use]
600 #[inline]
601 pub fn gather_select(
602 slice: &[T],
603 enable: Mask<isize, N>,
604 idxs: Simd<usize, N>,
605 or: Self,
606 ) -> Self {
607 let enable: Mask<isize, N> = enable & idxs.simd_lt(Simd::splat(slice.len()));
608 // Safety: We have masked-off out-of-bounds indices.
609 unsafe { Self::gather_select_unchecked(slice, enable, idxs, or) }
610 }
611
612 /// Reads from indices in `slice` to construct a SIMD vector.
613 /// The mask `enable`s all `true` indices and disables all `false` indices.
614 /// If an index is disabled, the element is selected from the `or` vector.
615 ///
616 /// # Safety
617 ///
618 /// Calling this function with an `enable`d out-of-bounds index is *[undefined behavior]*
619 /// even if the resulting value is not used.
620 ///
621 /// # Examples
622 /// ```
623 /// # #![feature(portable_simd)]
624 /// # #[cfg(feature = "as_crate")] use core_simd::simd;
625 /// # #[cfg(not(feature = "as_crate"))] use core::simd;
626 /// # use simd::{Simd, cmp::SimdPartialOrd, Mask};
627 /// let vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
628 /// let idxs = Simd::from_array([9, 3, 0, 5]); // Includes an out-of-bounds index
629 /// let alt = Simd::from_array([-5, -4, -3, -2]);
630 /// let enable = Mask::from_array([true, true, true, false]); // Includes a masked element
631 /// // If this mask was used to gather, it would be unsound. Let's fix that.
632 /// let enable = enable & idxs.simd_lt(Simd::splat(vec.len()));
633 ///
634 /// // The out-of-bounds index has been masked, so it's safe to gather now.
635 /// let result = unsafe { Simd::gather_select_unchecked(&vec, enable, idxs, alt) };
636 /// assert_eq!(result, Simd::from_array([-5, 13, 10, -2]));
637 /// ```
638 /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html
639 #[must_use]
640 #[inline]
641 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
642 pub unsafe fn gather_select_unchecked(
643 slice: &[T],
644 enable: Mask<isize, N>,
645 idxs: Simd<usize, N>,
646 or: Self,
647 ) -> Self {
648 let base_ptr = Simd::<*const T, N>::splat(slice.as_ptr());
649 // Ferris forgive me, I have done pointer arithmetic here.
650 let ptrs = base_ptr.wrapping_add(idxs);
651 // Safety: The caller is responsible for determining the indices are okay to read
652 unsafe { Self::gather_select_ptr(ptrs, enable, or) }
653 }
654
655 /// Reads elementwise from pointers into a SIMD vector.
656 ///
657 /// # Safety
658 ///
659 /// Each read must satisfy the same conditions as [`core::ptr::read`].
660 ///
661 /// # Example
662 /// ```
663 /// # #![feature(portable_simd)]
664 /// # #[cfg(feature = "as_crate")] use core_simd::simd;
665 /// # #[cfg(not(feature = "as_crate"))] use core::simd;
666 /// # use simd::prelude::*;
667 /// let values = [6, 2, 4, 9];
668 /// let offsets = Simd::from_array([1, 0, 0, 3]);
669 /// let source = Simd::splat(values.as_ptr()).wrapping_add(offsets);
670 /// let gathered = unsafe { Simd::gather_ptr(source) };
671 /// assert_eq!(gathered, Simd::from_array([2, 6, 6, 9]));
672 /// ```
673 #[must_use]
674 #[inline]
675 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
676 pub unsafe fn gather_ptr(source: Simd<*const T, N>) -> Self
677 where
678 T: Default,
679 {
680 // TODO: add an intrinsic that doesn't use a passthru vector, and remove the T: Default bound
681 // Safety: The caller is responsible for upholding all invariants
682 unsafe { Self::gather_select_ptr(source, Mask::splat(true), Self::default()) }
683 }
684
685 /// Conditionally read elementwise from pointers into a SIMD vector.
686 /// The mask `enable`s all `true` pointers and disables all `false` pointers.
687 /// If a pointer is disabled, the element is selected from the `or` vector,
688 /// and no read is performed.
689 ///
690 /// # Safety
691 ///
692 /// Enabled elements must satisfy the same conditions as [`core::ptr::read`].
693 ///
694 /// # Example
695 /// ```
696 /// # #![feature(portable_simd)]
697 /// # #[cfg(feature = "as_crate")] use core_simd::simd;
698 /// # #[cfg(not(feature = "as_crate"))] use core::simd;
699 /// # use simd::prelude::*;
700 /// let values = [6, 2, 4, 9];
701 /// let enable = Mask::from_array([true, true, false, true]);
702 /// let offsets = Simd::from_array([1, 0, 0, 3]);
703 /// let source = Simd::splat(values.as_ptr()).wrapping_add(offsets);
704 /// let gathered = unsafe { Simd::gather_select_ptr(source, enable, Simd::splat(0)) };
705 /// assert_eq!(gathered, Simd::from_array([2, 6, 0, 9]));
706 /// ```
707 #[must_use]
708 #[inline]
709 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
710 pub unsafe fn gather_select_ptr(
711 source: Simd<*const T, N>,
712 enable: Mask<isize, N>,
713 or: Self,
714 ) -> Self {
715 // Safety: The caller is responsible for upholding all invariants
716 unsafe { core::intrinsics::simd::simd_gather(or, source, enable.to_simd()) }
717 }
718
719 /// Conditionally write contiguous elements to `slice`. The `enable` mask controls
720 /// which elements are written, as long as they're in-bounds of the `slice`.
721 /// If the element is disabled or out of bounds, no memory access to that location
722 /// is made.
723 ///
724 /// # Examples
725 /// ```
726 /// # #![feature(portable_simd)]
727 /// # #[cfg(feature = "as_crate")] use core_simd::simd;
728 /// # #[cfg(not(feature = "as_crate"))] use core::simd;
729 /// # use simd::{Simd, Mask};
730 /// let mut arr = [0i32; 4];
731 /// let write = Simd::from_array([-5, -4, -3, -2]);
732 /// let enable = Mask::from_array([false, true, true, true]);
733 ///
734 /// write.store_select(&mut arr[..3], enable);
735 /// assert_eq!(arr, [0, -4, -3, 0]);
736 /// ```
737 #[inline]
738 pub fn store_select(self, slice: &mut [T], mut enable: Mask<<T as SimdElement>::Mask, N>) {
739 enable &= mask_up_to(slice.len());
740 // SAFETY: We performed the bounds check by updating the mask. &[T] is properly aligned to
741 // the element.
742 unsafe { self.store_select_ptr(slice.as_mut_ptr(), enable) }
743 }
744
745 /// Conditionally write contiguous elements to `slice`. The `enable` mask controls
746 /// which elements are written.
747 ///
748 /// # Safety
749 ///
750 /// Every enabled element must be in bounds for the `slice`.
751 ///
752 /// # Examples
753 /// ```
754 /// # #![feature(portable_simd)]
755 /// # #[cfg(feature = "as_crate")] use core_simd::simd;
756 /// # #[cfg(not(feature = "as_crate"))] use core::simd;
757 /// # use simd::{Simd, Mask};
758 /// let mut arr = [0i32; 4];
759 /// let write = Simd::from_array([-5, -4, -3, -2]);
760 /// let enable = Mask::from_array([false, true, true, true]);
761 ///
762 /// unsafe { write.store_select_unchecked(&mut arr, enable) };
763 /// assert_eq!(arr, [0, -4, -3, -2]);
764 /// ```
765 #[inline]
766 pub unsafe fn store_select_unchecked(
767 self,
768 slice: &mut [T],
769 enable: Mask<<T as SimdElement>::Mask, N>,
770 ) {
771 let ptr = slice.as_mut_ptr();
772 // SAFETY: The safety of writing elements in `slice` is ensured by the caller.
773 unsafe { self.store_select_ptr(ptr, enable) }
774 }
775
776 /// Conditionally write contiguous elements starting from `ptr`.
777 /// The `enable` mask controls which elements are written.
778 /// When disabled, the memory location corresponding to that element is not accessed.
779 ///
780 /// # Safety
781 ///
782 /// Memory addresses for element are calculated [`pointer::wrapping_offset`] and
783 /// each enabled element must satisfy the same conditions as [`core::ptr::write`].
784 #[inline]
785 pub unsafe fn store_select_ptr(self, ptr: *mut T, enable: Mask<<T as SimdElement>::Mask, N>) {
786 // SAFETY: The safety of writing elements through `ptr` is ensured by the caller.
787 unsafe {
788 core::intrinsics::simd::simd_masked_store::<_, _, _, { SimdAlign::Element }>(
789 enable.to_simd(),
790 ptr,
791 self,
792 )
793 }
794 }
795
796 /// Writes the values in a SIMD vector to potentially discontiguous indices in `slice`.
797 /// If an index is out-of-bounds, the write is suppressed without panicking.
798 /// If two elements in the scattered vector would write to the same index
799 /// only the last element is guaranteed to actually be written.
800 ///
801 /// # Examples
802 /// ```
803 /// # #![feature(portable_simd)]
804 /// # use core::simd::Simd;
805 /// let mut vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
806 /// let idxs = Simd::from_array([9, 3, 0, 0]); // Note the duplicate index.
807 /// let vals = Simd::from_array([-27, 82, -41, 124]);
808 ///
809 /// vals.scatter(&mut vec, idxs); // two logical writes means the last wins.
810 /// assert_eq!(vec, vec![124, 11, 12, 82, 14, 15, 16, 17, 18]);
811 /// ```
812 #[inline]
813 pub fn scatter(self, slice: &mut [T], idxs: Simd<usize, N>) {
814 self.scatter_select(slice, Mask::splat(true), idxs)
815 }
816
817 /// Writes values from a SIMD vector to multiple potentially discontiguous indices in `slice`.
818 /// The mask `enable`s all `true` indices and disables all `false` indices.
819 /// If an enabled index is out-of-bounds, the write is suppressed without panicking.
820 /// If two enabled elements in the scattered vector would write to the same index,
821 /// only the last element is guaranteed to actually be written.
822 ///
823 /// # Examples
824 /// ```
825 /// # #![feature(portable_simd)]
826 /// # #[cfg(feature = "as_crate")] use core_simd::simd;
827 /// # #[cfg(not(feature = "as_crate"))] use core::simd;
828 /// # use simd::{Simd, Mask};
829 /// let mut vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
830 /// let idxs = Simd::from_array([9, 3, 0, 0]); // Includes an out-of-bounds index
831 /// let vals = Simd::from_array([-27, 82, -41, 124]);
832 /// let enable = Mask::from_array([true, true, true, false]); // Includes a masked element
833 ///
834 /// vals.scatter_select(&mut vec, enable, idxs); // The last write is masked, thus omitted.
835 /// assert_eq!(vec, vec![-41, 11, 12, 82, 14, 15, 16, 17, 18]);
836 /// ```
837 #[inline]
838 pub fn scatter_select(self, slice: &mut [T], enable: Mask<isize, N>, idxs: Simd<usize, N>) {
839 let enable: Mask<isize, N> = enable & idxs.simd_lt(Simd::splat(slice.len()));
840 // Safety: We have masked-off out-of-bounds indices.
841 unsafe { self.scatter_select_unchecked(slice, enable, idxs) }
842 }
843
844 /// Writes values from a SIMD vector to multiple potentially discontiguous indices in `slice`.
845 /// The mask `enable`s all `true` indices and disables all `false` indices.
846 /// If two enabled elements in the scattered vector would write to the same index,
847 /// only the last element is guaranteed to actually be written.
848 ///
849 /// # Safety
850 ///
851 /// Calling this function with an enabled out-of-bounds index is *[undefined behavior]*,
852 /// and may lead to memory corruption.
853 ///
854 /// # Examples
855 /// ```
856 /// # #![feature(portable_simd)]
857 /// # #[cfg(feature = "as_crate")] use core_simd::simd;
858 /// # #[cfg(not(feature = "as_crate"))] use core::simd;
859 /// # use simd::{Simd, cmp::SimdPartialOrd, Mask};
860 /// let mut vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
861 /// let idxs = Simd::from_array([9, 3, 0, 0]);
862 /// let vals = Simd::from_array([-27, 82, -41, 124]);
863 /// let enable = Mask::from_array([true, true, true, false]); // Masks the final index
864 /// // If this mask was used to scatter, it would be unsound. Let's fix that.
865 /// let enable = enable & idxs.simd_lt(Simd::splat(vec.len()));
866 ///
867 /// // We have masked the OOB index, so it's safe to scatter now.
868 /// unsafe { vals.scatter_select_unchecked(&mut vec, enable, idxs); }
869 /// // The second write to index 0 was masked, thus omitted.
870 /// assert_eq!(vec, vec![-41, 11, 12, 82, 14, 15, 16, 17, 18]);
871 /// ```
872 /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html
873 #[inline]
874 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
875 pub unsafe fn scatter_select_unchecked(
876 self,
877 slice: &mut [T],
878 enable: Mask<isize, N>,
879 idxs: Simd<usize, N>,
880 ) {
881 // Safety: This block works with *mut T derived from &mut 'a [T],
882 // which means it is delicate in Rust's borrowing model, circa 2021:
883 // &mut 'a [T] asserts uniqueness, so deriving &'a [T] invalidates live *mut Ts!
884 // Even though this block is largely safe methods, it must be exactly this way
885 // to prevent invalidating the raw ptrs while they're live.
886 // Thus, entering this block requires all values to use being already ready:
887 // 0. idxs we want to write to, which are used to construct the mask.
888 // 1. enable, which depends on an initial &'a [T] and the idxs.
889 // 2. actual values to scatter (self).
890 // 3. &mut [T] which will become our base ptr.
891 unsafe {
892 // Now Entering ☢️ *mut T Zone
893 let base_ptr = Simd::<*mut T, N>::splat(slice.as_mut_ptr());
894 // Ferris forgive me, I have done pointer arithmetic here.
895 let ptrs = base_ptr.wrapping_add(idxs);
896 // The ptrs have been bounds-masked to prevent memory-unsafe writes insha'allah
897 self.scatter_select_ptr(ptrs, enable);
898 // Cleared ☢️ *mut T Zone
899 }
900 }
901
902 /// Writes pointers elementwise into a SIMD vector.
903 ///
904 /// # Safety
905 ///
906 /// Each write must satisfy the same conditions as [`core::ptr::write`].
907 ///
908 /// # Example
909 /// ```
910 /// # #![feature(portable_simd)]
911 /// # #[cfg(feature = "as_crate")] use core_simd::simd;
912 /// # #[cfg(not(feature = "as_crate"))] use core::simd;
913 /// # use simd::{Simd, ptr::SimdMutPtr};
914 /// let mut values = [0; 4];
915 /// let offset = Simd::from_array([3, 2, 1, 0]);
916 /// let ptrs = Simd::splat(values.as_mut_ptr()).wrapping_add(offset);
917 /// unsafe { Simd::from_array([6, 3, 5, 7]).scatter_ptr(ptrs); }
918 /// assert_eq!(values, [7, 5, 3, 6]);
919 /// ```
920 #[inline]
921 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
922 pub unsafe fn scatter_ptr(self, dest: Simd<*mut T, N>) {
923 // Safety: The caller is responsible for upholding all invariants
924 unsafe { self.scatter_select_ptr(dest, Mask::splat(true)) }
925 }
926
927 /// Conditionally write pointers elementwise into a SIMD vector.
928 /// The mask `enable`s all `true` pointers and disables all `false` pointers.
929 /// If a pointer is disabled, the write to its pointee is skipped.
930 ///
931 /// # Safety
932 ///
933 /// Enabled pointers must satisfy the same conditions as [`core::ptr::write`].
934 ///
935 /// # Example
936 /// ```
937 /// # #![feature(portable_simd)]
938 /// # #[cfg(feature = "as_crate")] use core_simd::simd;
939 /// # #[cfg(not(feature = "as_crate"))] use core::simd;
940 /// # use simd::{Mask, Simd, ptr::SimdMutPtr};
941 /// let mut values = [0; 4];
942 /// let offset = Simd::from_array([3, 2, 1, 0]);
943 /// let ptrs = Simd::splat(values.as_mut_ptr()).wrapping_add(offset);
944 /// let enable = Mask::from_array([true, true, false, false]);
945 /// unsafe { Simd::from_array([6, 3, 5, 7]).scatter_select_ptr(ptrs, enable); }
946 /// assert_eq!(values, [0, 0, 3, 6]);
947 /// ```
948 #[inline]
949 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
950 pub unsafe fn scatter_select_ptr(self, dest: Simd<*mut T, N>, enable: Mask<isize, N>) {
951 // Safety: The caller is responsible for upholding all invariants
952 unsafe { core::intrinsics::simd::simd_scatter(self, dest, enable.to_simd()) }
953 }
954}
955
956impl<T, const N: usize> Copy for Simd<T, N> where T: SimdElement {}
957
958impl<T, const N: usize> Clone for Simd<T, N>
959where
960 T: SimdElement,
961{
962 #[inline]
963 fn clone(&self) -> Self {
964 *self
965 }
966}
967
968impl<T, const N: usize> Default for Simd<T, N>
969where
970 T: SimdElement + Default,
971{
972 #[inline]
973 fn default() -> Self {
974 Self::splat(T::default())
975 }
976}
977
978impl<T, const N: usize> PartialEq for Simd<T, N>
979where
980 T: SimdElement + PartialEq,
981{
982 #[inline]
983 fn eq(&self, other: &Self) -> bool {
984 // Safety: All SIMD vectors are SimdPartialEq, and the comparison produces a valid mask.
985 let mask = unsafe {
986 let tfvec: Simd<<T as SimdElement>::Mask, N> =
987 core::intrinsics::simd::simd_eq(*self, *other);
988 Mask::from_simd_unchecked(tfvec)
989 };
990
991 // Two vectors are equal if all elements are equal when compared elementwise
992 mask.all()
993 }
994
995 #[allow(clippy::partialeq_ne_impl)]
996 #[inline]
997 fn ne(&self, other: &Self) -> bool {
998 // Safety: All SIMD vectors are SimdPartialEq, and the comparison produces a valid mask.
999 let mask = unsafe {
1000 let tfvec: Simd<<T as SimdElement>::Mask, N> =
1001 core::intrinsics::simd::simd_ne(*self, *other);
1002 Mask::from_simd_unchecked(tfvec)
1003 };
1004
1005 // Two vectors are non-equal if any elements are non-equal when compared elementwise
1006 mask.any()
1007 }
1008}
1009
1010/// Lexicographic order. For the SIMD elementwise minimum and maximum, use simd_min and simd_max instead.
1011impl<T, const N: usize> PartialOrd for Simd<T, N>
1012where
1013 T: SimdElement + PartialOrd,
1014{
1015 #[inline]
1016 fn partial_cmp(&self, other: &Self) -> Option<core::cmp::Ordering> {
1017 // TODO use SIMD equality
1018 self.to_array().partial_cmp(other.as_ref())
1019 }
1020}
1021
1022impl<T, const N: usize> Eq for Simd<T, N> where T: SimdElement + Eq {}
1023
1024/// Lexicographic order. For the SIMD elementwise minimum and maximum, use simd_min and simd_max instead.
1025impl<T, const N: usize> Ord for Simd<T, N>
1026where
1027 T: SimdElement + Ord,
1028{
1029 #[inline]
1030 fn cmp(&self, other: &Self) -> core::cmp::Ordering {
1031 // TODO use SIMD equality
1032 self.to_array().cmp(other.as_ref())
1033 }
1034}
1035
1036impl<T, const N: usize> core::hash::Hash for Simd<T, N>
1037where
1038 T: SimdElement + core::hash::Hash,
1039{
1040 #[inline]
1041 fn hash<H>(&self, state: &mut H)
1042 where
1043 H: core::hash::Hasher,
1044 {
1045 self.as_array().hash(state)
1046 }
1047}
1048
1049// array references
1050impl<T, const N: usize> AsRef<[T; N]> for Simd<T, N>
1051where
1052 T: SimdElement,
1053{
1054 #[inline]
1055 fn as_ref(&self) -> &[T; N] {
1056 self.as_array()
1057 }
1058}
1059
1060impl<T, const N: usize> AsMut<[T; N]> for Simd<T, N>
1061where
1062 T: SimdElement,
1063{
1064 #[inline]
1065 fn as_mut(&mut self) -> &mut [T; N] {
1066 self.as_mut_array()
1067 }
1068}
1069
1070// slice references
1071impl<T, const N: usize> AsRef<[T]> for Simd<T, N>
1072where
1073 T: SimdElement,
1074{
1075 #[inline]
1076 fn as_ref(&self) -> &[T] {
1077 self.as_array()
1078 }
1079}
1080
1081impl<T, const N: usize> AsMut<[T]> for Simd<T, N>
1082where
1083 T: SimdElement,
1084{
1085 #[inline]
1086 fn as_mut(&mut self) -> &mut [T] {
1087 self.as_mut_array()
1088 }
1089}
1090
1091// vector/array conversion
1092impl<T, const N: usize> From<[T; N]> for Simd<T, N>
1093where
1094 T: SimdElement,
1095{
1096 #[inline]
1097 fn from(array: [T; N]) -> Self {
1098 Self::from_array(array)
1099 }
1100}
1101
1102impl<T, const N: usize> From<Simd<T, N>> for [T; N]
1103where
1104 T: SimdElement,
1105{
1106 #[inline]
1107 fn from(vector: Simd<T, N>) -> Self {
1108 vector.to_array()
1109 }
1110}
1111
1112impl<T, const N: usize> TryFrom<&[T]> for Simd<T, N>
1113where
1114 T: SimdElement,
1115{
1116 type Error = core::array::TryFromSliceError;
1117
1118 #[inline]
1119 fn try_from(slice: &[T]) -> Result<Self, core::array::TryFromSliceError> {
1120 Ok(Self::from_array(slice.try_into()?))
1121 }
1122}
1123
1124impl<T, const N: usize> TryFrom<&mut [T]> for Simd<T, N>
1125where
1126 T: SimdElement,
1127{
1128 type Error = core::array::TryFromSliceError;
1129
1130 #[inline]
1131 fn try_from(slice: &mut [T]) -> Result<Self, core::array::TryFromSliceError> {
1132 Ok(Self::from_array(slice.try_into()?))
1133 }
1134}
1135
1136/// Marker trait for types that may be used as SIMD vector elements.
1137///
1138/// # Safety
1139/// This trait, when implemented, asserts the compiler can monomorphize
1140/// `#[repr(simd)]` structs with the marked type as an element.
1141/// Strictly, it is valid to impl if the vector will not be miscompiled.
1142/// Practically, it is user-unfriendly to impl it if the vector won't compile,
1143/// even when no soundness guarantees are broken by allowing the user to try.
1144pub impl(self) unsafe trait SimdElement: Copy {
1145 /// The mask element type corresponding to this element type.
1146 type Mask: MaskElement;
1147}
1148
1149// Safety: u8 is a valid SIMD element type, and is supported by this API
1150unsafe impl SimdElement for u8 {
1151 type Mask = i8;
1152}
1153
1154// Safety: u16 is a valid SIMD element type, and is supported by this API
1155unsafe impl SimdElement for u16 {
1156 type Mask = i16;
1157}
1158
1159// Safety: u32 is a valid SIMD element type, and is supported by this API
1160unsafe impl SimdElement for u32 {
1161 type Mask = i32;
1162}
1163
1164// Safety: u64 is a valid SIMD element type, and is supported by this API
1165unsafe impl SimdElement for u64 {
1166 type Mask = i64;
1167}
1168
1169// Safety: usize is a valid SIMD element type, and is supported by this API
1170unsafe impl SimdElement for usize {
1171 type Mask = isize;
1172}
1173
1174// Safety: i8 is a valid SIMD element type, and is supported by this API
1175unsafe impl SimdElement for i8 {
1176 type Mask = i8;
1177}
1178
1179// Safety: i16 is a valid SIMD element type, and is supported by this API
1180unsafe impl SimdElement for i16 {
1181 type Mask = i16;
1182}
1183
1184// Safety: i32 is a valid SIMD element type, and is supported by this API
1185unsafe impl SimdElement for i32 {
1186 type Mask = i32;
1187}
1188
1189// Safety: i64 is a valid SIMD element type, and is supported by this API
1190unsafe impl SimdElement for i64 {
1191 type Mask = i64;
1192}
1193
1194// Safety: isize is a valid SIMD element type, and is supported by this API
1195unsafe impl SimdElement for isize {
1196 type Mask = isize;
1197}
1198
1199// Safety: f16 is a valid SIMD element type, and is supported by this API
1200unsafe impl SimdElement for f16 {
1201 type Mask = i16;
1202}
1203
1204// Safety: f32 is a valid SIMD element type, and is supported by this API
1205unsafe impl SimdElement for f32 {
1206 type Mask = i32;
1207}
1208
1209// Safety: f64 is a valid SIMD element type, and is supported by this API
1210unsafe impl SimdElement for f64 {
1211 type Mask = i64;
1212}
1213
1214// Safety: (thin) const pointers are valid SIMD element types, and are supported by this API
1215//
1216// Fat pointers may be supported in the future.
1217unsafe impl<T> SimdElement for *const T
1218where
1219 T: core::ptr::Pointee<Metadata = ()>,
1220{
1221 type Mask = isize;
1222}
1223
1224// Safety: (thin) mut pointers are valid SIMD element types, and are supported by this API
1225//
1226// Fat pointers may be supported in the future.
1227unsafe impl<T> SimdElement for *mut T
1228where
1229 T: core::ptr::Pointee<Metadata = ()>,
1230{
1231 type Mask = isize;
1232}
1233
1234#[inline]
1235fn lane_indices<const N: usize>() -> Simd<usize, N> {
1236 #![allow(clippy::needless_range_loop)]
1237 let mut index = [0; N];
1238 for i in 0..N {
1239 index[i] = i;
1240 }
1241 Simd::from_array(index)
1242}
1243
1244#[inline]
1245fn mask_up_to<M, const N: usize>(len: usize) -> Mask<M, N>
1246where
1247 M: MaskElement,
1248{
1249 let index = lane_indices::<N>();
1250 let max_value: u64 = M::max_unsigned();
1251 macro_rules! case {
1252 ($ty:ty) => {
1253 if N < <$ty>::MAX as usize && max_value as $ty as u64 == max_value {
1254 return index.cast().simd_lt(Simd::splat(len.min(N) as $ty)).cast();
1255 }
1256 };
1257 }
1258 case!(u8);
1259 case!(u16);
1260 case!(u32);
1261 case!(u64);
1262 index.simd_lt(Simd::splat(len)).cast()
1263}