core_simd/ops.rs
1use crate::simd::{Select, Simd, SimdElement, cmp::SimdPartialEq};
2use core::ops::{Add, Mul};
3use core::ops::{BitAnd, BitOr, BitXor};
4use core::ops::{Div, Rem, Sub};
5use core::ops::{Shl, Shr};
6
7mod assign;
8mod deref;
9mod shift_scalar;
10mod unary;
11
12macro_rules! unsafe_base {
13 ($lhs:ident, $rhs:ident, {$simd_call:ident}, $($_:tt)*) => {
14 // Safety: $lhs and $rhs are vectors
15 unsafe { core::intrinsics::simd::$simd_call($lhs, $rhs) }
16 };
17}
18
19/// SAFETY: This macro should not be used for anything except Shl or Shr, and passed the appropriate shift intrinsic.
20/// It handles performing a bitand in addition to calling the shift operator, so that the result
21/// is well-defined: LLVM can return a poison value if you shl, lshr, or ashr if `rhs >= <Int>::BITS`
22/// At worst, this will maybe add another instruction and cycle,
23/// at best, it may open up more optimization opportunities,
24/// or simply be elided entirely, especially for SIMD ISAs which default to this.
25///
26// FIXME: Consider implementing this in cg_llvm instead?
27// cg_clif defaults to this, and scalar MIR shifts also default to wrapping
28macro_rules! wrap_bitshift {
29 ($lhs:ident, $rhs:ident, {$simd_call:ident}, $int:ident) => {
30 #[allow(clippy::suspicious_arithmetic_impl)]
31 // Safety: $lhs and the bitand result are vectors
32 unsafe {
33 core::intrinsics::simd::$simd_call(
34 $lhs,
35 $rhs.bitand(Simd::splat(<$int>::BITS as $int - 1)),
36 )
37 }
38 };
39}
40
41/// SAFETY: This macro must only be used to impl Div or Rem and given the matching intrinsic.
42/// It guards against LLVM's UB conditions for integer div or rem using masks and selects,
43/// thus guaranteeing a Rust value returns instead.
44///
45/// | | LLVM | Rust
46/// | :--------------: | :--- | :----------
47/// | N {/,%} 0 | UB | panic!()
48/// | <$int>::MIN / -1 | UB | <$int>::MIN
49/// | <$int>::MIN % -1 | UB | 0
50///
51macro_rules! int_divrem_guard {
52 ( $lhs:ident,
53 $rhs:ident,
54 { const PANIC_ZERO: &'static str = $zero:literal;
55 $simd_call:ident, $op:tt
56 },
57 $int:ident ) => {
58 if $rhs.simd_eq(Simd::splat(0 as _)).any() {
59 panic!($zero);
60 } else {
61 // Prevent otherwise-UB overflow on the MIN / -1 case.
62 let rhs = if <$int>::MIN != 0 {
63 // This should, at worst, optimize to a few branchless logical ops
64 // Ideally, this entire conditional should evaporate
65 // Fire LLVM and implement those manually if it doesn't get the hint
66 ($lhs.simd_eq(Simd::splat(<$int>::MIN))
67 // type inference can break here, so cut an SInt to size
68 & $rhs.simd_eq(Simd::splat(-1i64 as _)))
69 .select(Simd::splat(1 as _), $rhs)
70 } else {
71 // Nice base case to make it easy to const-fold away the other branch.
72 $rhs
73 };
74
75 // aarch64 div fails for arbitrary `v % 0`, mod fails when rhs is MIN, for non-powers-of-two
76 // these operations aren't vectorized on aarch64 anyway
77 #[cfg(target_arch = "aarch64")]
78 {
79 let mut out = Simd::splat(0 as _);
80 for i in 0..Self::LEN {
81 out.set(i, $lhs.get(i) $op rhs.get(i));
82 }
83 out
84 }
85
86 #[cfg(not(target_arch = "aarch64"))]
87 {
88 // Safety: $lhs and rhs are vectors
89 unsafe { core::intrinsics::simd::$simd_call($lhs, rhs) }
90 }
91 }
92 };
93}
94
95macro_rules! for_base_types {
96 ( T = ($($scalar:ident),*);
97 type Lhs = Simd<T, N>;
98 type Rhs = Simd<T, N>;
99 type Output = $out:ty;
100
101 impl $op:ident::$call:ident {
102 $macro_impl:ident $inner:tt
103 }) => {
104 $(
105 impl<const N: usize> $op<Self> for Simd<$scalar, N>
106 where
107 $scalar: SimdElement,
108 {
109 type Output = $out;
110
111 #[inline]
112 // TODO: only useful for int Div::div, but we hope that this
113 // will essentially always get inlined anyway.
114 #[track_caller]
115 fn $call(self, rhs: Self) -> Self::Output {
116 $macro_impl!(self, rhs, $inner, $scalar)
117 }
118 }
119 )*
120 }
121}
122
123// A "TokenTree muncher": takes a set of scalar types `T = {};`
124// type parameters for the ops it implements, `Op::fn` names,
125// and a macro that expands into an expr, substituting in an intrinsic.
126// It passes that to for_base_types, which expands an impl for the types,
127// using the expanded expr in the function, and recurses with itself.
128//
129// tl;dr impls a set of ops::{Traits} for a set of types
130macro_rules! for_base_ops {
131 (
132 T = $types:tt;
133 type Lhs = Simd<T, N>;
134 type Rhs = Simd<T, N>;
135 type Output = $out:ident;
136 impl $op:ident::$call:ident
137 $inner:tt
138 $($rest:tt)*
139 ) => {
140 for_base_types! {
141 T = $types;
142 type Lhs = Simd<T, N>;
143 type Rhs = Simd<T, N>;
144 type Output = $out;
145 impl $op::$call
146 $inner
147 }
148 for_base_ops! {
149 T = $types;
150 type Lhs = Simd<T, N>;
151 type Rhs = Simd<T, N>;
152 type Output = $out;
153 $($rest)*
154 }
155 };
156 ($($done:tt)*) => {
157 // Done.
158 }
159}
160
161// Integers can always accept add, mul, sub, bitand, bitor, and bitxor.
162// For all of these operations, simd_* intrinsics apply wrapping logic.
163for_base_ops! {
164 T = (i8, i16, i32, i64, isize, u8, u16, u32, u64, usize);
165 type Lhs = Simd<T, N>;
166 type Rhs = Simd<T, N>;
167 type Output = Self;
168
169 impl Add::add {
170 unsafe_base { simd_add }
171 }
172
173 impl Mul::mul {
174 unsafe_base { simd_mul }
175 }
176
177 impl Sub::sub {
178 unsafe_base { simd_sub }
179 }
180
181 impl BitAnd::bitand {
182 unsafe_base { simd_and }
183 }
184
185 impl BitOr::bitor {
186 unsafe_base { simd_or }
187 }
188
189 impl BitXor::bitxor {
190 unsafe_base { simd_xor }
191 }
192
193 impl Div::div {
194 int_divrem_guard {
195 const PANIC_ZERO: &'static str = "attempt to divide by zero";
196 simd_div, /
197 }
198 }
199
200 impl Rem::rem {
201 int_divrem_guard {
202 const PANIC_ZERO: &'static str = "attempt to calculate the remainder with a divisor of zero";
203 simd_rem, %
204 }
205 }
206
207 // The only question is how to handle shifts >= <Int>::BITS?
208 // Our current solution uses wrapping logic.
209 impl Shl::shl {
210 wrap_bitshift { simd_shl }
211 }
212
213 impl Shr::shr {
214 wrap_bitshift {
215 // This automatically monomorphizes to lshr or ashr, depending,
216 // so it's fine to use it for both UInts and SInts.
217 simd_shr
218 }
219 }
220}
221
222// We don't need any special precautions here:
223// Floats always accept arithmetic ops, but may become NaN.
224for_base_ops! {
225 T = (f16, f32, f64);
226 type Lhs = Simd<T, N>;
227 type Rhs = Simd<T, N>;
228 type Output = Self;
229
230 impl Add::add {
231 unsafe_base { simd_add }
232 }
233
234 impl Mul::mul {
235 unsafe_base { simd_mul }
236 }
237
238 impl Sub::sub {
239 unsafe_base { simd_sub }
240 }
241
242 impl Div::div {
243 unsafe_base { simd_div }
244 }
245
246 impl Rem::rem {
247 unsafe_base { simd_rem }
248 }
249}