Skip to main content

core_simd/
ops.rs

1use crate::simd::{Select, Simd, SimdElement, cmp::SimdPartialEq};
2use core::ops::{Add, Mul};
3use core::ops::{BitAnd, BitOr, BitXor};
4use core::ops::{Div, Rem, Sub};
5use core::ops::{Shl, Shr};
6
7mod assign;
8mod deref;
9mod shift_scalar;
10mod unary;
11
12macro_rules! unsafe_base {
13    ($lhs:ident, $rhs:ident, {$simd_call:ident}, $($_:tt)*) => {
14        // Safety: $lhs and $rhs are vectors
15        unsafe { core::intrinsics::simd::$simd_call($lhs, $rhs) }
16    };
17}
18
19/// SAFETY: This macro should not be used for anything except Shl or Shr, and passed the appropriate shift intrinsic.
20/// It handles performing a bitand in addition to calling the shift operator, so that the result
21/// is well-defined: LLVM can return a poison value if you shl, lshr, or ashr if `rhs >= <Int>::BITS`
22/// At worst, this will maybe add another instruction and cycle,
23/// at best, it may open up more optimization opportunities,
24/// or simply be elided entirely, especially for SIMD ISAs which default to this.
25///
26// FIXME: Consider implementing this in cg_llvm instead?
27// cg_clif defaults to this, and scalar MIR shifts also default to wrapping
28macro_rules! wrap_bitshift {
29    ($lhs:ident, $rhs:ident, {$simd_call:ident}, $int:ident) => {
30        #[allow(clippy::suspicious_arithmetic_impl)]
31        // Safety: $lhs and the bitand result are vectors
32        unsafe {
33            core::intrinsics::simd::$simd_call(
34                $lhs,
35                $rhs.bitand(Simd::splat(<$int>::BITS as $int - 1)),
36            )
37        }
38    };
39}
40
41/// SAFETY: This macro must only be used to impl Div or Rem and given the matching intrinsic.
42/// It guards against LLVM's UB conditions for integer div or rem using masks and selects,
43/// thus guaranteeing a Rust value returns instead.
44///
45/// |                  | LLVM | Rust
46/// | :--------------: | :--- | :----------
47/// | N {/,%} 0        | UB   | panic!()
48/// | <$int>::MIN / -1 | UB   | <$int>::MIN
49/// | <$int>::MIN % -1 | UB   | 0
50///
51macro_rules! int_divrem_guard {
52    (   $lhs:ident,
53        $rhs:ident,
54        {   const PANIC_ZERO: &'static str = $zero:literal;
55            $simd_call:ident, $op:tt
56        },
57        $int:ident ) => {
58        if $rhs.simd_eq(Simd::splat(0 as _)).any() {
59            panic!($zero);
60        } else {
61            // Prevent otherwise-UB overflow on the MIN / -1 case.
62            let rhs = if <$int>::MIN != 0 {
63                // This should, at worst, optimize to a few branchless logical ops
64                // Ideally, this entire conditional should evaporate
65                // Fire LLVM and implement those manually if it doesn't get the hint
66                ($lhs.simd_eq(Simd::splat(<$int>::MIN))
67                // type inference can break here, so cut an SInt to size
68                & $rhs.simd_eq(Simd::splat(-1i64 as _)))
69                .select(Simd::splat(1 as _), $rhs)
70            } else {
71                // Nice base case to make it easy to const-fold away the other branch.
72                $rhs
73            };
74
75            // aarch64 div fails for arbitrary `v % 0`, mod fails when rhs is MIN, for non-powers-of-two
76            // these operations aren't vectorized on aarch64 anyway
77            #[cfg(target_arch = "aarch64")]
78            {
79                let mut out = Simd::splat(0 as _);
80                for i in 0..Self::LEN {
81                    out.set(i, $lhs.get(i) $op rhs.get(i));
82                }
83                out
84            }
85
86            #[cfg(not(target_arch = "aarch64"))]
87            {
88                // Safety: $lhs and rhs are vectors
89                unsafe { core::intrinsics::simd::$simd_call($lhs, rhs) }
90            }
91        }
92    };
93}
94
95macro_rules! for_base_types {
96    (   T = ($($scalar:ident),*);
97        type Lhs = Simd<T, N>;
98        type Rhs = Simd<T, N>;
99        type Output = $out:ty;
100
101        impl $op:ident::$call:ident {
102            $macro_impl:ident $inner:tt
103        }) => {
104            $(
105                impl<const N: usize> $op<Self> for Simd<$scalar, N>
106                where
107                    $scalar: SimdElement,
108                {
109                    type Output = $out;
110
111                    #[inline]
112                    // TODO: only useful for int Div::div, but we hope that this
113                    // will essentially always get inlined anyway.
114                    #[track_caller]
115                    fn $call(self, rhs: Self) -> Self::Output {
116                        $macro_impl!(self, rhs, $inner, $scalar)
117                    }
118                }
119            )*
120    }
121}
122
123// A "TokenTree muncher": takes a set of scalar types `T = {};`
124// type parameters for the ops it implements, `Op::fn` names,
125// and a macro that expands into an expr, substituting in an intrinsic.
126// It passes that to for_base_types, which expands an impl for the types,
127// using the expanded expr in the function, and recurses with itself.
128//
129// tl;dr impls a set of ops::{Traits} for a set of types
130macro_rules! for_base_ops {
131    (
132        T = $types:tt;
133        type Lhs = Simd<T, N>;
134        type Rhs = Simd<T, N>;
135        type Output = $out:ident;
136        impl $op:ident::$call:ident
137            $inner:tt
138        $($rest:tt)*
139    ) => {
140        for_base_types! {
141            T = $types;
142            type Lhs = Simd<T, N>;
143            type Rhs = Simd<T, N>;
144            type Output = $out;
145            impl $op::$call
146                $inner
147        }
148        for_base_ops! {
149            T = $types;
150            type Lhs = Simd<T, N>;
151            type Rhs = Simd<T, N>;
152            type Output = $out;
153            $($rest)*
154        }
155    };
156    ($($done:tt)*) => {
157        // Done.
158    }
159}
160
161// Integers can always accept add, mul, sub, bitand, bitor, and bitxor.
162// For all of these operations, simd_* intrinsics apply wrapping logic.
163for_base_ops! {
164    T = (i8, i16, i32, i64, isize, u8, u16, u32, u64, usize);
165    type Lhs = Simd<T, N>;
166    type Rhs = Simd<T, N>;
167    type Output = Self;
168
169    impl Add::add {
170        unsafe_base { simd_add }
171    }
172
173    impl Mul::mul {
174        unsafe_base { simd_mul }
175    }
176
177    impl Sub::sub {
178        unsafe_base { simd_sub }
179    }
180
181    impl BitAnd::bitand {
182        unsafe_base { simd_and }
183    }
184
185    impl BitOr::bitor {
186        unsafe_base { simd_or }
187    }
188
189    impl BitXor::bitxor {
190        unsafe_base { simd_xor }
191    }
192
193    impl Div::div {
194        int_divrem_guard {
195            const PANIC_ZERO: &'static str = "attempt to divide by zero";
196            simd_div, /
197        }
198    }
199
200    impl Rem::rem {
201        int_divrem_guard {
202            const PANIC_ZERO: &'static str = "attempt to calculate the remainder with a divisor of zero";
203            simd_rem, %
204        }
205    }
206
207    // The only question is how to handle shifts >= <Int>::BITS?
208    // Our current solution uses wrapping logic.
209    impl Shl::shl {
210        wrap_bitshift { simd_shl }
211    }
212
213    impl Shr::shr {
214        wrap_bitshift {
215            // This automatically monomorphizes to lshr or ashr, depending,
216            // so it's fine to use it for both UInts and SInts.
217            simd_shr
218        }
219    }
220}
221
222// We don't need any special precautions here:
223// Floats always accept arithmetic ops, but may become NaN.
224for_base_ops! {
225    T = (f16, f32, f64);
226    type Lhs = Simd<T, N>;
227    type Rhs = Simd<T, N>;
228    type Output = Self;
229
230    impl Add::add {
231        unsafe_base { simd_add }
232    }
233
234    impl Mul::mul {
235        unsafe_base { simd_mul }
236    }
237
238    impl Sub::sub {
239        unsafe_base { simd_sub }
240    }
241
242    impl Div::div {
243        unsafe_base { simd_div }
244    }
245
246    impl Rem::rem {
247        unsafe_base { simd_rem }
248    }
249}