parser/
input.rs

1//! See [`Input`].
2
3use edition::Edition;
4
5use crate::SyntaxKind;
6
7#[allow(non_camel_case_types)]
8type bits = u64;
9
10/// Input for the parser -- a sequence of tokens.
11///
12/// As of now, parser doesn't have access to the *text* of the tokens, and makes
13/// decisions based solely on their classification. Unlike `LexerToken`, the
14/// `Tokens` doesn't include whitespace and comments. Main input to the parser.
15///
16/// Struct of arrays internally, but this shouldn't really matter.
17pub struct Input {
18    kind: Vec<SyntaxKind>,
19    joint: Vec<bits>,
20    contextual_kind: Vec<SyntaxKind>,
21    edition: Vec<Edition>,
22}
23
24/// `pub` impl used by callers to create `Tokens`.
25impl Input {
26    #[inline]
27    pub fn with_capacity(capacity: usize) -> Self {
28        Self {
29            kind: Vec::with_capacity(capacity),
30            joint: Vec::with_capacity(capacity / size_of::<bits>()),
31            contextual_kind: Vec::with_capacity(capacity),
32            edition: Vec::with_capacity(capacity),
33        }
34    }
35    #[inline]
36    pub fn push(&mut self, kind: SyntaxKind, edition: Edition) {
37        self.push_impl(kind, SyntaxKind::EOF, edition)
38    }
39    #[inline]
40    pub fn push_ident(&mut self, contextual_kind: SyntaxKind, edition: Edition) {
41        self.push_impl(SyntaxKind::IDENT, contextual_kind, edition)
42    }
43    /// Sets jointness for the last token we've pushed.
44    ///
45    /// This is a separate API rather than an argument to the `push` to make it
46    /// convenient both for textual and mbe tokens. With text, you know whether
47    /// the *previous* token was joint, with mbe, you know whether the *current*
48    /// one is joint. This API allows for styles of usage:
49    ///
50    /// ```ignore
51    /// // In text:
52    /// tokens.was_joint(prev_joint);
53    /// tokens.push(curr);
54    ///
55    /// // In MBE:
56    /// token.push(curr);
57    /// tokens.push(curr_joint)
58    /// ```
59    #[inline]
60    pub fn was_joint(&mut self) {
61        let n = self.len() - 1;
62        let (idx, b_idx) = self.bit_index(n);
63        self.joint[idx] |= 1 << b_idx;
64    }
65    #[inline]
66    fn push_impl(&mut self, kind: SyntaxKind, contextual_kind: SyntaxKind, edition: Edition) {
67        let idx = self.len();
68        if idx.is_multiple_of(bits::BITS as usize) {
69            self.joint.push(0);
70        }
71        self.kind.push(kind);
72        self.contextual_kind.push(contextual_kind);
73        self.edition.push(edition);
74    }
75}
76
77/// pub(crate) impl used by the parser to consume `Tokens`.
78impl Input {
79    pub(crate) fn kind(&self, idx: usize) -> SyntaxKind {
80        self.kind.get(idx).copied().unwrap_or(SyntaxKind::EOF)
81    }
82    pub(crate) fn contextual_kind(&self, idx: usize) -> SyntaxKind {
83        self.contextual_kind.get(idx).copied().unwrap_or(SyntaxKind::EOF)
84    }
85    pub(crate) fn edition(&self, idx: usize) -> Edition {
86        self.edition[idx]
87    }
88    pub(crate) fn is_joint(&self, n: usize) -> bool {
89        let (idx, b_idx) = self.bit_index(n);
90        self.joint[idx] & (1 << b_idx) != 0
91    }
92}
93
94impl Input {
95    fn bit_index(&self, n: usize) -> (usize, usize) {
96        let idx = n / (bits::BITS as usize);
97        let b_idx = n % (bits::BITS as usize);
98        (idx, b_idx)
99    }
100    fn len(&self) -> usize {
101        self.kind.len()
102    }
103}
parser/input.rs

parser/
input.rs
