1#![allow(aarch64_softfloat_neon, x86_softfloat_sse)]
4
5use crate::simd::Simd;
6use core::mem;
7
8impl<const N: usize> Simd<u8, N> {
9 #[inline]
18 pub fn swizzle_dyn(self, idxs: Simd<u8, N>) -> Self {
19 #![allow(unused_imports, unused_unsafe)]
20 #[cfg(all(
21 target_arch = "arm",
22 target_feature = "v7",
23 target_feature = "neon",
24 target_endian = "little"
25 ))]
26 use core::arch::arm::{uint8x8_t, vtbl1_u8};
27 #[cfg(target_arch = "wasm32")]
28 use core::arch::wasm32 as wasm;
29 #[cfg(target_arch = "wasm64")]
30 use core::arch::wasm64 as wasm;
31 #[cfg(target_arch = "x86")]
32 use core::arch::x86;
33 #[cfg(target_arch = "x86_64")]
34 use core::arch::x86_64 as x86;
35 unsafe {
37 #[allow(
38 unreachable_patterns,
39 reason = "avoids writing verbose cfg(not), earlier branches take priority"
40 )]
41 match N {
42 #[cfg(all(
44 any(target_arch = "aarch64", target_arch = "arm64ec"),
45 target_feature = "neon",
46 target_endian = "little"
47 ))]
48 8 | 16 | 24 | 32 | 48 | 64 => aarch64_swizzle(self, idxs),
49
50 #[cfg(all(
52 target_arch = "arm",
53 target_feature = "v7",
54 target_feature = "neon",
55 target_endian = "little"
56 ))]
57 16 => transize(armv7_neon_swizzle_u8x16, self, idxs),
58
59 #[cfg(target_feature = "simd128")]
61 16 => transize(wasm::i8x16_swizzle, self, idxs),
62 #[cfg(target_feature = "simd128")]
63 32 => transize(swizzle_dyn_split::<32, 16>, self, idxs),
64
65 #[cfg(all(target_arch = "loongarch64", target_feature = "lsx"))]
67 16 => transize(loong64_lsx_swizzle, self, idxs),
68 #[cfg(all(target_arch = "loongarch64", target_feature = "lasx"))]
69 32 => transize(loong64_lasx_swizzle, self, idxs),
70 #[cfg(all(target_arch = "loongarch64", target_feature = "lsx"))]
71 32 => transize(swizzle_dyn_split::<32, 16>, self, idxs),
72 #[cfg(all(target_arch = "loongarch64", target_feature = "lasx"))]
73 64 => transize(swizzle_dyn_split::<64, 32>, self, idxs),
74
75 #[cfg(target_feature = "ssse3")]
77 16 => transize(x86::_mm_shuffle_epi8, self, zeroing_idxs(idxs)),
78 #[cfg(all(target_feature = "avx512vl", target_feature = "avx512vbmi"))]
79 32 => {
80 let swizzler = |bytes, idxs| {
81 let idxs =
84 x86::_mm256_min_epu8(idxs, Simd::<u8, 32>::splat(N as u8).into());
85 x86::_mm256_permutex2var_epi8(bytes, idxs, x86::_mm256_setzero_si256())
86 };
87 transize(swizzler, self, idxs)
88 }
89 #[cfg(target_feature = "avx2")]
90 32 => transize(avx2_pshufb, self, idxs),
91 #[cfg(target_feature = "ssse3")]
92 32 => transize(swizzle_dyn_split::<32, 16>, self, idxs),
93 #[cfg(all(target_feature = "avx512vl", target_feature = "avx512vbmi"))]
95 64 => {
96 let swizzler = |bytes, idxs| {
97 let idxs =
100 x86::_mm512_min_epu8(idxs, Simd::<u8, 64>::splat(N as u8).into());
101 x86::_mm512_permutex2var_epi8(bytes, idxs, x86::_mm512_setzero_si512())
102 };
103 transize(swizzler, self, idxs)
104 }
105 #[cfg(target_feature = "avx2")]
106 64 => transize(swizzle_dyn_split::<64, 32>, self, idxs),
107
108 _ => {
110 let mut array = [0; N];
111 for (i, k) in idxs.to_array().into_iter().enumerate() {
112 if (k as usize) < N {
113 array[i] = self.get(k as usize);
114 };
115 }
116 array.into()
117 }
118 }
119 }
120 }
121}
122
123#[allow(dead_code, reason = "only used on some targets/features")]
124fn swizzle_dyn_split<const N: usize, const HALF: usize>(
127 bytes: Simd<u8, N>,
128 idxs: Simd<u8, N>,
129) -> Simd<u8, N> {
130 let table_low = bytes.extract::<0, HALF>();
131 let table_high = bytes.extract::<HALF, HALF>();
132 let idxs_low = idxs.extract::<0, HALF>();
133 let idxs_high = idxs.extract::<HALF, HALF>();
134 let table_high_offset = Simd::<u8, HALF>::splat(HALF as u8);
135
136 let output_low_from_low = table_low.swizzle_dyn(idxs_low);
137 let output_low_from_high = table_high.swizzle_dyn(idxs_low - table_high_offset);
138 let output_low = output_low_from_low | output_low_from_high;
139
140 let output_high_from_low = table_low.swizzle_dyn(idxs_high);
141 let output_high_from_high = table_high.swizzle_dyn(idxs_high - table_high_offset);
142 let output_high = output_high_from_low | output_high_from_high;
143
144 use crate::simd::Swizzle;
149 struct CombineHalves;
150 impl<const N: usize> Swizzle<N> for CombineHalves {
151 const INDEX: [usize; N] = const {
152 let mut index = [0; N];
153 let mut i = 0;
154 while i < N {
155 index[i] = i;
156 i += 1;
157 }
158 index
159 };
160 }
161
162 CombineHalves::concat_swizzle(output_low, output_high)
163}
164
165#[cfg(all(
171 target_arch = "arm",
172 target_feature = "v7",
173 target_feature = "neon",
174 target_endian = "little"
175))]
176unsafe fn armv7_neon_swizzle_u8x16(bytes: Simd<u8, 16>, idxs: Simd<u8, 16>) -> Simd<u8, 16> {
177 use core::arch::arm::{uint8x8x2_t, vcombine_u8, vget_high_u8, vget_low_u8, vtbl2_u8};
178 unsafe {
180 let bytes = uint8x8x2_t(vget_low_u8(bytes.into()), vget_high_u8(bytes.into()));
181 let lo = vtbl2_u8(bytes, vget_low_u8(idxs.into()));
182 let hi = vtbl2_u8(bytes, vget_high_u8(idxs.into()));
183 vcombine_u8(lo, hi).into()
184 }
185}
186
187#[cfg(all(
192 any(target_arch = "aarch64", target_arch = "arm64ec"),
193 target_feature = "neon",
194 target_endian = "little"
195))]
196unsafe fn aarch64_swizzle<const N: usize>(bytes: Simd<u8, N>, idxs: Simd<u8, N>) -> Simd<u8, N> {
197 use core::arch::aarch64::*;
198 use core::mem::transmute_copy;
199
200 unsafe {
202 match N {
203 8 => transmute_copy(&vtbl1_u8(transmute_copy(&bytes), transmute_copy(&idxs))),
204 16 => transmute_copy(&vqtbl1q_u8(transmute_copy(&bytes), transmute_copy(&idxs))),
205 24 => {
206 let bytes: uint8x8x3_t = transmute_copy(&bytes);
207 let idxs: uint8x8x3_t = transmute_copy(&idxs);
208
209 let ret0 = vtbl3_u8(bytes, idxs.0);
210 let ret1 = vtbl3_u8(bytes, idxs.1);
211 let ret2 = vtbl3_u8(bytes, idxs.2);
212
213 let ret = uint8x8x3_t(ret0, ret1, ret2);
214 transmute_copy(&ret)
215 }
216 32 => {
217 let bytes: uint8x16x2_t = transmute_copy(&bytes);
218 let idxs: uint8x16x2_t = transmute_copy(&idxs);
219
220 let ret0 = vqtbl2q_u8(bytes, idxs.0);
221 let ret1 = vqtbl2q_u8(bytes, idxs.1);
222
223 let ret = uint8x16x2_t(ret0, ret1);
224 transmute_copy(&ret)
225 }
226 48 => {
227 let bytes: uint8x16x3_t = transmute_copy(&bytes);
228 let idxs: uint8x16x3_t = transmute_copy(&idxs);
229
230 let ret0 = vqtbl3q_u8(bytes, idxs.0);
231 let ret1 = vqtbl3q_u8(bytes, idxs.1);
232 let ret2 = vqtbl3q_u8(bytes, idxs.2);
233
234 let ret = uint8x16x3_t(ret0, ret1, ret2);
235 transmute_copy(&ret)
236 }
237 64 => {
238 let bytes: uint8x16x4_t = transmute_copy(&bytes);
239 let idxs: uint8x16x4_t = transmute_copy(&idxs);
240
241 let ret0 = vqtbl4q_u8(bytes, idxs.0);
242 let ret1 = vqtbl4q_u8(bytes, idxs.1);
243 let ret2 = vqtbl4q_u8(bytes, idxs.2);
244 let ret3 = vqtbl4q_u8(bytes, idxs.3);
245
246 let ret = uint8x16x4_t(ret0, ret1, ret2, ret3);
247 transmute_copy(&ret)
248 }
249 _ => unreachable!(),
250 }
251 }
252}
253
254#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
259#[target_feature(enable = "avx2")]
260#[allow(unused)]
261#[inline]
262#[allow(clippy::let_and_return)]
263unsafe fn avx2_pshufb(bytes: Simd<u8, 32>, idxs: Simd<u8, 32>) -> Simd<u8, 32> {
264 #[cfg(target_arch = "x86")]
265 use core::arch::x86;
266 #[cfg(target_arch = "x86_64")]
267 use core::arch::x86_64 as x86;
268 use x86::_mm256_permute2x128_si256 as avx2_cross_shuffle;
269 use x86::_mm256_shuffle_epi8 as avx2_half_pshufb;
270 unsafe {
272 let bytes = bytes.into();
273 let indices = idxs.into();
274 let swapped = avx2_cross_shuffle::<0x01>(bytes, bytes);
275
276 let control = x86::_mm256_adds_epu8(indices, x86::_mm256_set1_epi8(0x60));
280
281 let local = avx2_half_pshufb(bytes, control);
282 let remote = avx2_half_pshufb(swapped, control);
283
284 let select_bias = x86::_mm256_set_m128i(x86::_mm_set1_epi8(-112), x86::_mm_set1_epi8(16));
290 let select_remote = x86::_mm256_add_epi8(control, select_bias);
291 x86::_mm256_blendv_epi8(local, remote, select_remote).into()
292 }
293}
294
295#[cfg(all(target_arch = "loongarch64", target_feature = "lsx"))]
300unsafe fn loong64_lsx_swizzle(bytes: Simd<u8, 16>, idxs: Simd<u8, 16>) -> Simd<u8, 16> {
301 use core::arch::loongarch64::{lsx_vand_v, lsx_vshuf_b, lsx_vslei_bu};
302 unsafe {
304 let bytes = lsx_vshuf_b(bytes.into(), bytes.into(), idxs.into());
305 let mask = lsx_vslei_bu::<15>(idxs.into());
306 lsx_vand_v(bytes, mask).into()
307 }
308}
309
310#[cfg(all(target_arch = "loongarch64", target_feature = "lasx"))]
315unsafe fn loong64_lasx_swizzle(bytes: Simd<u8, 32>, idxs: Simd<u8, 32>) -> Simd<u8, 32> {
316 use core::arch::loongarch64::{lasx_xvand_v, lasx_xvpermi_q, lasx_xvshuf_b, lasx_xvslei_bu};
317 unsafe {
319 let lolo = lasx_xvpermi_q::<0x00>(bytes.into(), bytes.into());
320 let hihi = lasx_xvpermi_q::<0x11>(bytes.into(), bytes.into());
321 let bytes = lasx_xvshuf_b(hihi, lolo, idxs.into());
322 let mask = lasx_xvslei_bu::<31>(idxs.into());
323 lasx_xvand_v(bytes, mask).into()
324 }
325}
326
327#[allow(dead_code)]
335#[inline(always)]
336unsafe fn transize<T, const N: usize>(
337 f: unsafe fn(T, T) -> T,
338 a: Simd<u8, N>,
339 b: Simd<u8, N>,
340) -> Simd<u8, N> {
341 unsafe { mem::transmute_copy(&f(mem::transmute_copy(&a), mem::transmute_copy(&b))) }
343}
344
345#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
347#[allow(unused)]
348#[inline(always)]
349fn zeroing_idxs<const N: usize>(idxs: Simd<u8, N>) -> Simd<u8, N> {
350 let zeroing_bits = idxs + Simd::splat((127 - N + 1) as u8);
353 idxs | zeroing_bits
354}