parser/
token_set.rs

1//! A bit-set of `SyntaxKind`s.
2
3use crate::SyntaxKind;
4
5/// A bit-set of `SyntaxKind`s
6#[derive(Clone, Copy)]
7pub(crate) struct TokenSet([u64; 3]);
8
9/// `TokenSet`s should only include token `SyntaxKind`s, so the discriminant of any passed/included
10/// `SyntaxKind` must *not* be greater than that of the last token `SyntaxKind`.
11/// See #17037.
12const LAST_TOKEN_KIND_DISCRIMINANT: usize = SyntaxKind::SHEBANG as usize;
13
14impl TokenSet {
15    pub(crate) const EMPTY: TokenSet = TokenSet([0; 3]);
16
17    pub(crate) const fn new(kinds: &[SyntaxKind]) -> TokenSet {
18        let mut res = [0; 3];
19        let mut i = 0;
20        while i < kinds.len() {
21            let discriminant = kinds[i] as usize;
22            debug_assert!(
23                discriminant <= LAST_TOKEN_KIND_DISCRIMINANT,
24                "Expected a token `SyntaxKind`"
25            );
26            let idx = discriminant / 64;
27            res[idx] |= 1 << (discriminant % 64);
28            i += 1;
29        }
30        TokenSet(res)
31    }
32
33    pub(crate) const fn union(self, other: TokenSet) -> TokenSet {
34        TokenSet([self.0[0] | other.0[0], self.0[1] | other.0[1], self.0[2] | other.0[2]])
35    }
36
37    pub(crate) const fn contains(&self, kind: SyntaxKind) -> bool {
38        let discriminant = kind as usize;
39        debug_assert!(
40            discriminant <= LAST_TOKEN_KIND_DISCRIMINANT,
41            "Expected a token `SyntaxKind`"
42        );
43        let idx = discriminant / 64;
44        let mask = 1 << (discriminant % 64);
45        self.0[idx] & mask != 0
46    }
47}
48
49#[test]
50fn token_set_works_for_tokens() {
51    use crate::SyntaxKind::*;
52    let ts = TokenSet::new(&[EOF, SHEBANG]);
53    assert!(ts.contains(EOF));
54    assert!(ts.contains(SHEBANG));
55    assert!(!ts.contains(PLUS));
56}
parser/token_set.rs

parser/
token_set.rs
