Skip to main content

core_simd/
swizzle_dyn.rs

1// Allow these FCW: anyone soundly using the intrinsics has to enable
2// the target feature, and that will generate a warning for them.
3#![allow(aarch64_softfloat_neon, x86_softfloat_sse)]
4
5use crate::simd::Simd;
6use core::mem;
7
8impl<const N: usize> Simd<u8, N> {
9    /// Swizzle a vector of bytes according to the index vector.
10    /// Indices within range select the appropriate byte.
11    /// Indices "out of bounds" instead select 0.
12    ///
13    /// Note that the current implementation is selected during build-time
14    /// of the standard library, so `cargo build -Zbuild-std` may be necessary
15    /// to unlock better performance, especially for larger vectors.
16    /// A planned compiler improvement will enable using `#[target_feature]` instead.
17    #[inline]
18    pub fn swizzle_dyn(self, idxs: Simd<u8, N>) -> Self {
19        #![allow(unused_imports, unused_unsafe)]
20        #[cfg(all(
21            target_arch = "arm",
22            target_feature = "v7",
23            target_feature = "neon",
24            target_endian = "little"
25        ))]
26        use core::arch::arm::{uint8x8_t, vtbl1_u8};
27        #[cfg(target_arch = "wasm32")]
28        use core::arch::wasm32 as wasm;
29        #[cfg(target_arch = "wasm64")]
30        use core::arch::wasm64 as wasm;
31        #[cfg(target_arch = "x86")]
32        use core::arch::x86;
33        #[cfg(target_arch = "x86_64")]
34        use core::arch::x86_64 as x86;
35        // SAFETY: Intrinsics covered by cfg
36        unsafe {
37            #[allow(
38                unreachable_patterns,
39                reason = "avoids writing verbose cfg(not), earlier branches take priority"
40            )]
41            match N {
42                // Aarch64
43                #[cfg(all(
44                    any(target_arch = "aarch64", target_arch = "arm64ec"),
45                    target_feature = "neon",
46                    target_endian = "little"
47                ))]
48                8 | 16 | 24 | 32 | 48 | 64 => aarch64_swizzle(self, idxs),
49
50                // 32-bit ARMv7
51                #[cfg(all(
52                    target_arch = "arm",
53                    target_feature = "v7",
54                    target_feature = "neon",
55                    target_endian = "little"
56                ))]
57                16 => transize(armv7_neon_swizzle_u8x16, self, idxs),
58
59                // WASM SIMD128
60                #[cfg(target_feature = "simd128")]
61                16 => transize(wasm::i8x16_swizzle, self, idxs),
62                #[cfg(target_feature = "simd128")]
63                32 => transize(swizzle_dyn_split::<32, 16>, self, idxs),
64
65                // LoongArch64
66                #[cfg(all(target_arch = "loongarch64", target_feature = "lsx"))]
67                16 => transize(loong64_lsx_swizzle, self, idxs),
68                #[cfg(all(target_arch = "loongarch64", target_feature = "lasx"))]
69                32 => transize(loong64_lasx_swizzle, self, idxs),
70                #[cfg(all(target_arch = "loongarch64", target_feature = "lsx"))]
71                32 => transize(swizzle_dyn_split::<32, 16>, self, idxs),
72                #[cfg(all(target_arch = "loongarch64", target_feature = "lasx"))]
73                64 => transize(swizzle_dyn_split::<64, 32>, self, idxs),
74
75                // x86, x86-64
76                #[cfg(target_feature = "ssse3")]
77                16 => transize(x86::_mm_shuffle_epi8, self, zeroing_idxs(idxs)),
78                #[cfg(all(target_feature = "avx512vl", target_feature = "avx512vbmi"))]
79                32 => {
80                    let swizzler = |bytes, idxs| {
81                        // Clamp out-of-range indices to the first byte of a
82                        // second, all-zero table.
83                        let idxs =
84                            x86::_mm256_min_epu8(idxs, Simd::<u8, 32>::splat(N as u8).into());
85                        x86::_mm256_permutex2var_epi8(bytes, idxs, x86::_mm256_setzero_si256())
86                    };
87                    transize(swizzler, self, idxs)
88                }
89                #[cfg(target_feature = "avx2")]
90                32 => transize(avx2_pshufb, self, idxs),
91                #[cfg(target_feature = "ssse3")]
92                32 => transize(swizzle_dyn_split::<32, 16>, self, idxs),
93                // Notable absence: avx512bw pshufb shuffle
94                #[cfg(all(target_feature = "avx512vl", target_feature = "avx512vbmi"))]
95                64 => {
96                    let swizzler = |bytes, idxs| {
97                        // Clamp out-of-range indices to the first byte of a
98                        // second, all-zero table.
99                        let idxs =
100                            x86::_mm512_min_epu8(idxs, Simd::<u8, 64>::splat(N as u8).into());
101                        x86::_mm512_permutex2var_epi8(bytes, idxs, x86::_mm512_setzero_si512())
102                    };
103                    transize(swizzler, self, idxs)
104                }
105                #[cfg(target_feature = "avx2")]
106                64 => transize(swizzle_dyn_split::<64, 32>, self, idxs),
107
108                // scalar fallback
109                _ => {
110                    let mut array = [0; N];
111                    for (i, k) in idxs.to_array().into_iter().enumerate() {
112                        if (k as usize) < N {
113                            array[i] = self.get(k as usize);
114                        };
115                    }
116                    array.into()
117                }
118            }
119        }
120    }
121}
122
123#[allow(dead_code, reason = "only used on some targets/features")]
124/// Implements an arbitrary shuffle over double the native vector width
125/// using 4 native-width shuffles
126fn swizzle_dyn_split<const N: usize, const HALF: usize>(
127    bytes: Simd<u8, N>,
128    idxs: Simd<u8, N>,
129) -> Simd<u8, N> {
130    let table_low = bytes.extract::<0, HALF>();
131    let table_high = bytes.extract::<HALF, HALF>();
132    let idxs_low = idxs.extract::<0, HALF>();
133    let idxs_high = idxs.extract::<HALF, HALF>();
134    let table_high_offset = Simd::<u8, HALF>::splat(HALF as u8);
135
136    let output_low_from_low = table_low.swizzle_dyn(idxs_low);
137    let output_low_from_high = table_high.swizzle_dyn(idxs_low - table_high_offset);
138    let output_low = output_low_from_low | output_low_from_high;
139
140    let output_high_from_low = table_low.swizzle_dyn(idxs_high);
141    let output_high_from_high = table_high.swizzle_dyn(idxs_high - table_high_offset);
142    let output_high = output_high_from_low | output_high_from_high;
143
144    // This is simply a concatenation of two native-sized vectors.
145    // The swizzle does nothing - it maps the elements right back where they already are.
146    // There doesn't seem to be a more direct way to do this as of this writing.
147    // TODO: simplify once a plain `concat` is available.
148    use crate::simd::Swizzle;
149    struct CombineHalves;
150    impl<const N: usize> Swizzle<N> for CombineHalves {
151        const INDEX: [usize; N] = const {
152            let mut index = [0; N];
153            let mut i = 0;
154            while i < N {
155                index[i] = i;
156                i += 1;
157            }
158            index
159        };
160    }
161
162    CombineHalves::concat_swizzle(output_low, output_high)
163}
164
165/// armv7 neon supports swizzling `u8x16` by swizzling two u8x8 blocks
166/// with a u8x8x2 lookup table.
167///
168/// # Safety
169/// This requires armv7 neon to work
170#[cfg(all(
171    target_arch = "arm",
172    target_feature = "v7",
173    target_feature = "neon",
174    target_endian = "little"
175))]
176unsafe fn armv7_neon_swizzle_u8x16(bytes: Simd<u8, 16>, idxs: Simd<u8, 16>) -> Simd<u8, 16> {
177    use core::arch::arm::{uint8x8x2_t, vcombine_u8, vget_high_u8, vget_low_u8, vtbl2_u8};
178    // SAFETY: Caller promised arm neon support
179    unsafe {
180        let bytes = uint8x8x2_t(vget_low_u8(bytes.into()), vget_high_u8(bytes.into()));
181        let lo = vtbl2_u8(bytes, vget_low_u8(idxs.into()));
182        let hi = vtbl2_u8(bytes, vget_high_u8(idxs.into()));
183        vcombine_u8(lo, hi).into()
184    }
185}
186
187/// AArch64 NEON supports swizzling 8, 16, 24, 32, 48 or 64 by stacking multiple TBL instructions.
188///
189/// # Safety
190/// This requires AArch64 NEON to work
191#[cfg(all(
192    any(target_arch = "aarch64", target_arch = "arm64ec"),
193    target_feature = "neon",
194    target_endian = "little"
195))]
196unsafe fn aarch64_swizzle<const N: usize>(bytes: Simd<u8, N>, idxs: Simd<u8, N>) -> Simd<u8, N> {
197    use core::arch::aarch64::*;
198    use core::mem::transmute_copy;
199
200    // SAFETY: Caller promised AArch64 NEON support
201    unsafe {
202        match N {
203            8 => transmute_copy(&vtbl1_u8(transmute_copy(&bytes), transmute_copy(&idxs))),
204            16 => transmute_copy(&vqtbl1q_u8(transmute_copy(&bytes), transmute_copy(&idxs))),
205            24 => {
206                let bytes: uint8x8x3_t = transmute_copy(&bytes);
207                let idxs: uint8x8x3_t = transmute_copy(&idxs);
208
209                let ret0 = vtbl3_u8(bytes, idxs.0);
210                let ret1 = vtbl3_u8(bytes, idxs.1);
211                let ret2 = vtbl3_u8(bytes, idxs.2);
212
213                let ret = uint8x8x3_t(ret0, ret1, ret2);
214                transmute_copy(&ret)
215            }
216            32 => {
217                let bytes: uint8x16x2_t = transmute_copy(&bytes);
218                let idxs: uint8x16x2_t = transmute_copy(&idxs);
219
220                let ret0 = vqtbl2q_u8(bytes, idxs.0);
221                let ret1 = vqtbl2q_u8(bytes, idxs.1);
222
223                let ret = uint8x16x2_t(ret0, ret1);
224                transmute_copy(&ret)
225            }
226            48 => {
227                let bytes: uint8x16x3_t = transmute_copy(&bytes);
228                let idxs: uint8x16x3_t = transmute_copy(&idxs);
229
230                let ret0 = vqtbl3q_u8(bytes, idxs.0);
231                let ret1 = vqtbl3q_u8(bytes, idxs.1);
232                let ret2 = vqtbl3q_u8(bytes, idxs.2);
233
234                let ret = uint8x16x3_t(ret0, ret1, ret2);
235                transmute_copy(&ret)
236            }
237            64 => {
238                let bytes: uint8x16x4_t = transmute_copy(&bytes);
239                let idxs: uint8x16x4_t = transmute_copy(&idxs);
240
241                let ret0 = vqtbl4q_u8(bytes, idxs.0);
242                let ret1 = vqtbl4q_u8(bytes, idxs.1);
243                let ret2 = vqtbl4q_u8(bytes, idxs.2);
244                let ret3 = vqtbl4q_u8(bytes, idxs.3);
245
246                let ret = uint8x16x4_t(ret0, ret1, ret2, ret3);
247                transmute_copy(&ret)
248            }
249            _ => unreachable!(),
250        }
251    }
252}
253
254/// "vpshufb like it was meant to be" on AVX2
255///
256/// # Safety
257/// This requires AVX2 to work
258#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
259#[target_feature(enable = "avx2")]
260#[allow(unused)]
261#[inline]
262#[allow(clippy::let_and_return)]
263unsafe fn avx2_pshufb(bytes: Simd<u8, 32>, idxs: Simd<u8, 32>) -> Simd<u8, 32> {
264    #[cfg(target_arch = "x86")]
265    use core::arch::x86;
266    #[cfg(target_arch = "x86_64")]
267    use core::arch::x86_64 as x86;
268    use x86::_mm256_permute2x128_si256 as avx2_cross_shuffle;
269    use x86::_mm256_shuffle_epi8 as avx2_half_pshufb;
270    // SAFETY: Caller promised AVX2
271    unsafe {
272        let bytes = bytes.into();
273        let indices = idxs.into();
274        let swapped = avx2_cross_shuffle::<0x01>(bytes, bytes);
275
276        // Adding 0x60 preserves the low nibble and bit 4 for valid
277        // indices 0..=31. Larger indices get their high bit set, so
278        // VPSHUFB supplies the required out-of-bounds zeroing.
279        let control = x86::_mm256_adds_epu8(indices, x86::_mm256_set1_epi8(0x60));
280
281        let local = avx2_half_pshufb(bytes, control);
282        let remote = avx2_half_pshufb(swapped, control);
283
284        // In the low lane, adding 0x10 moves the valid index's bit 4
285        // into the sign bit. The high lane has the opposite
286        // local/remote mapping, so adding 0x90 flips the selection.
287        // Out-of-range indices already zeroed both shuffle results,
288        // making the blend selection irrelevant for them.
289        let select_bias = x86::_mm256_set_m128i(x86::_mm_set1_epi8(-112), x86::_mm_set1_epi8(16));
290        let select_remote = x86::_mm256_add_epi8(control, select_bias);
291        x86::_mm256_blendv_epi8(local, remote, select_remote).into()
292    }
293}
294
295/// LoongArch64 LSX supports swizzling `u8x16`
296///
297/// # Safety
298/// This requires LoongArch LSX to work
299#[cfg(all(target_arch = "loongarch64", target_feature = "lsx"))]
300unsafe fn loong64_lsx_swizzle(bytes: Simd<u8, 16>, idxs: Simd<u8, 16>) -> Simd<u8, 16> {
301    use core::arch::loongarch64::{lsx_vand_v, lsx_vshuf_b, lsx_vslei_bu};
302    // SAFETY: Caller promised loongarch lsx support
303    unsafe {
304        let bytes = lsx_vshuf_b(bytes.into(), bytes.into(), idxs.into());
305        let mask = lsx_vslei_bu::<15>(idxs.into());
306        lsx_vand_v(bytes, mask).into()
307    }
308}
309
310/// LoongArch64 LASX supports swizzling `u8x32`
311///
312/// # Safety
313/// This requires LoongArch LASX to work
314#[cfg(all(target_arch = "loongarch64", target_feature = "lasx"))]
315unsafe fn loong64_lasx_swizzle(bytes: Simd<u8, 32>, idxs: Simd<u8, 32>) -> Simd<u8, 32> {
316    use core::arch::loongarch64::{lasx_xvand_v, lasx_xvpermi_q, lasx_xvshuf_b, lasx_xvslei_bu};
317    // SAFETY: Caller promised loongarch lasx support
318    unsafe {
319        let lolo = lasx_xvpermi_q::<0x00>(bytes.into(), bytes.into());
320        let hihi = lasx_xvpermi_q::<0x11>(bytes.into(), bytes.into());
321        let bytes = lasx_xvshuf_b(hihi, lolo, idxs.into());
322        let mask = lasx_xvslei_bu::<31>(idxs.into());
323        lasx_xvand_v(bytes, mask).into()
324    }
325}
326
327/// This sets up a call to an architecture-specific function, and in doing so
328/// it persuades rustc that everything is the correct size. Which it is.
329/// This would not be needed if one could convince Rust that, by matching on N,
330/// N is that value, and thus it would be valid to substitute e.g. 16.
331///
332/// # Safety
333/// The correctness of this function hinges on the sizes agreeing in actuality.
334#[allow(dead_code)]
335#[inline(always)]
336unsafe fn transize<T, const N: usize>(
337    f: unsafe fn(T, T) -> T,
338    a: Simd<u8, N>,
339    b: Simd<u8, N>,
340) -> Simd<u8, N> {
341    // SAFETY: Same obligation to use this function as to use mem::transmute_copy.
342    unsafe { mem::transmute_copy(&f(mem::transmute_copy(&a), mem::transmute_copy(&b))) }
343}
344
345/// Make indices that yield 0 for x86
346#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
347#[allow(unused)]
348#[inline(always)]
349fn zeroing_idxs<const N: usize>(idxs: Simd<u8, N>) -> Simd<u8, N> {
350    // Adding this sets the high bit for indices N..=127, while PSHUFB ignores
351    // the other changed bits. The OR preserves the high bit for indices 128..=255.
352    let zeroing_bits = idxs + Simd::splat((127 - N + 1) as u8);
353    idxs | zeroing_bits
354}