parser/
output.rs

1//! See [`Output`]
2
3use crate::SyntaxKind;
4
5/// Output of the parser -- a DFS traversal of a concrete syntax tree.
6///
7/// Use the [`Output::iter`] method to iterate over traversal steps and consume
8/// a syntax tree.
9///
10/// In a sense, this is just a sequence of [`SyntaxKind`]-colored parenthesis
11/// interspersed into the original [`crate::Input`]. The output is fundamentally
12/// coordinated with the input and `n_input_tokens` refers to the number of
13/// times [`crate::Input::push`] was called.
14#[derive(Default)]
15pub struct Output {
16    /// 32-bit encoding of events. If LSB is zero, then that's an index into the
17    /// error vector. Otherwise, it's one of the thee other variants, with data encoded as
18    ///
19    /// ```text
20    /// |16 bit kind|8 bit n_input_tokens|4 bit tag|4 bit leftover|
21    /// ``````
22    event: Vec<u32>,
23    error: Vec<String>,
24}
25
26#[derive(Debug)]
27pub enum Step<'a> {
28    Token { kind: SyntaxKind, n_input_tokens: u8 },
29    FloatSplit { ends_in_dot: bool },
30    Enter { kind: SyntaxKind },
31    Exit,
32    Error { msg: &'a str },
33}
34
35impl Output {
36    const EVENT_MASK: u32 = 0b1;
37    const TAG_MASK: u32 = 0x0000_00F0;
38    const N_INPUT_TOKEN_MASK: u32 = 0x0000_FF00;
39    const KIND_MASK: u32 = 0xFFFF_0000;
40
41    const ERROR_SHIFT: u32 = Self::EVENT_MASK.trailing_ones();
42    const TAG_SHIFT: u32 = Self::TAG_MASK.trailing_zeros();
43    const N_INPUT_TOKEN_SHIFT: u32 = Self::N_INPUT_TOKEN_MASK.trailing_zeros();
44    const KIND_SHIFT: u32 = Self::KIND_MASK.trailing_zeros();
45
46    const TOKEN_EVENT: u8 = 0;
47    const ENTER_EVENT: u8 = 1;
48    const EXIT_EVENT: u8 = 2;
49    const SPLIT_EVENT: u8 = 3;
50
51    pub fn iter(&self) -> impl Iterator<Item = Step<'_>> {
52        self.event.iter().map(|&event| {
53            if event & Self::EVENT_MASK == 0 {
54                return Step::Error {
55                    msg: self.error[(event as usize) >> Self::ERROR_SHIFT].as_str(),
56                };
57            }
58            let tag = ((event & Self::TAG_MASK) >> Self::TAG_SHIFT) as u8;
59            match tag {
60                Self::TOKEN_EVENT => {
61                    let kind: SyntaxKind =
62                        (((event & Self::KIND_MASK) >> Self::KIND_SHIFT) as u16).into();
63                    let n_input_tokens =
64                        ((event & Self::N_INPUT_TOKEN_MASK) >> Self::N_INPUT_TOKEN_SHIFT) as u8;
65                    Step::Token { kind, n_input_tokens }
66                }
67                Self::ENTER_EVENT => {
68                    let kind: SyntaxKind =
69                        (((event & Self::KIND_MASK) >> Self::KIND_SHIFT) as u16).into();
70                    Step::Enter { kind }
71                }
72                Self::EXIT_EVENT => Step::Exit,
73                Self::SPLIT_EVENT => {
74                    Step::FloatSplit { ends_in_dot: event & Self::N_INPUT_TOKEN_MASK != 0 }
75                }
76                _ => unreachable!(),
77            }
78        })
79    }
80
81    pub(crate) fn token(&mut self, kind: SyntaxKind, n_tokens: u8) {
82        let e = ((kind as u16 as u32) << Self::KIND_SHIFT)
83            | ((n_tokens as u32) << Self::N_INPUT_TOKEN_SHIFT)
84            | Self::EVENT_MASK;
85        self.event.push(e)
86    }
87
88    pub(crate) fn float_split_hack(&mut self, ends_in_dot: bool) {
89        let e = ((Self::SPLIT_EVENT as u32) << Self::TAG_SHIFT)
90            | ((ends_in_dot as u32) << Self::N_INPUT_TOKEN_SHIFT)
91            | Self::EVENT_MASK;
92        self.event.push(e);
93    }
94
95    pub(crate) fn enter_node(&mut self, kind: SyntaxKind) {
96        let e = ((kind as u16 as u32) << Self::KIND_SHIFT)
97            | ((Self::ENTER_EVENT as u32) << Self::TAG_SHIFT)
98            | Self::EVENT_MASK;
99        self.event.push(e)
100    }
101
102    pub(crate) fn leave_node(&mut self) {
103        let e = ((Self::EXIT_EVENT as u32) << Self::TAG_SHIFT) | Self::EVENT_MASK;
104        self.event.push(e)
105    }
106
107    pub(crate) fn error(&mut self, error: String) {
108        let idx = self.error.len();
109        self.error.push(error);
110        let e = (idx as u32) << Self::ERROR_SHIFT;
111        self.event.push(e);
112    }
113}
parser/output.rs

parser/
output.rs
