tt/
buffer.rs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
//! Stateful iteration over token trees.
//!
//! We use this as the source of tokens for parser.
use crate::{Leaf, Subtree, TokenTree};

#[derive(Copy, Clone, Debug, Eq, PartialEq)]
struct EntryId(usize);

#[derive(Copy, Clone, Debug, Eq, PartialEq)]
struct EntryPtr(
    /// The index of the buffer containing the entry.
    EntryId,
    /// The index of the entry within the buffer.
    usize,
);

/// Internal type which is used instead of `TokenTree` to represent a token tree
/// within a `TokenBuffer`.
#[derive(Debug)]
enum Entry<'t, Span> {
    // Mimicking types from proc-macro.
    Subtree(Option<&'t TokenTree<Span>>, &'t Subtree<Span>, EntryId),
    Leaf(&'t TokenTree<Span>),
    /// End entries contain a pointer to the entry from the containing
    /// token tree, or [`None`] if this is the outermost level.
    End(Option<EntryPtr>),
}

/// A token tree buffer
/// The safe version of `syn` [`TokenBuffer`](https://github.com/dtolnay/syn/blob/6533607f91686545cb034d2838beea338d9d0742/src/buffer.rs#L41)
#[derive(Debug)]
pub struct TokenBuffer<'t, Span> {
    buffers: Vec<Box<[Entry<'t, Span>]>>,
}

trait TokenList<'a, Span> {
    fn entries(
        &self,
    ) -> (Vec<(usize, (&'a Subtree<Span>, Option<&'a TokenTree<Span>>))>, Vec<Entry<'a, Span>>);
}

impl<'a, Span> TokenList<'a, Span> for &'a [TokenTree<Span>] {
    fn entries(
        &self,
    ) -> (Vec<(usize, (&'a Subtree<Span>, Option<&'a TokenTree<Span>>))>, Vec<Entry<'a, Span>>)
    {
        // Must contain everything in tokens and then the Entry::End
        let start_capacity = self.len() + 1;
        let mut entries = Vec::with_capacity(start_capacity);
        let mut children = vec![];
        for (idx, tt) in self.iter().enumerate() {
            match tt {
                TokenTree::Leaf(_) => {
                    entries.push(Entry::Leaf(tt));
                }
                TokenTree::Subtree(subtree) => {
                    entries.push(Entry::End(None));
                    children.push((idx, (subtree, Some(tt))));
                }
            }
        }
        (children, entries)
    }
}

impl<'a, Span> TokenList<'a, Span> for &'a Subtree<Span> {
    fn entries(
        &self,
    ) -> (Vec<(usize, (&'a Subtree<Span>, Option<&'a TokenTree<Span>>))>, Vec<Entry<'a, Span>>)
    {
        // Must contain everything in tokens and then the Entry::End
        let mut entries = vec![];
        let mut children = vec![];
        entries.push(Entry::End(None));
        children.push((0usize, (*self, None)));
        (children, entries)
    }
}

impl<'t, Span> TokenBuffer<'t, Span> {
    pub fn from_tokens(tokens: &'t [TokenTree<Span>]) -> TokenBuffer<'t, Span> {
        Self::new(tokens)
    }

    pub fn from_subtree(subtree: &'t Subtree<Span>) -> TokenBuffer<'t, Span> {
        Self::new(subtree)
    }

    fn new<T: TokenList<'t, Span>>(tokens: T) -> TokenBuffer<'t, Span> {
        let mut buffers = vec![];
        let idx = TokenBuffer::new_inner(tokens, &mut buffers, None);
        assert_eq!(idx, 0);
        TokenBuffer { buffers }
    }

    fn new_inner<T: TokenList<'t, Span>>(
        tokens: T,
        buffers: &mut Vec<Box<[Entry<'t, Span>]>>,
        next: Option<EntryPtr>,
    ) -> usize {
        let (children, mut entries) = tokens.entries();

        entries.push(Entry::End(next));
        let res = buffers.len();
        buffers.push(entries.into_boxed_slice());

        for (child_idx, (subtree, tt)) in children {
            let idx = TokenBuffer::new_inner(
                &*subtree.token_trees,
                buffers,
                Some(EntryPtr(EntryId(res), child_idx + 1)),
            );
            buffers[res].as_mut()[child_idx] = Entry::Subtree(tt, subtree, EntryId(idx));
        }

        res
    }

    /// Creates a cursor referencing the first token in the buffer and able to
    /// traverse until the end of the buffer.
    pub fn begin(&self) -> Cursor<'_, Span> {
        Cursor::create(self, EntryPtr(EntryId(0), 0))
    }

    fn entry(&self, ptr: &EntryPtr) -> Option<&Entry<'_, Span>> {
        let id = ptr.0;
        self.buffers[id.0].get(ptr.1)
    }
}

#[derive(Debug)]
pub enum TokenTreeRef<'a, Span> {
    Subtree(&'a Subtree<Span>, Option<&'a TokenTree<Span>>),
    Leaf(&'a Leaf<Span>, &'a TokenTree<Span>),
}

impl<Span: Copy> TokenTreeRef<'_, Span> {
    pub fn span(&self) -> Span {
        match self {
            TokenTreeRef::Subtree(subtree, _) => subtree.delimiter.open,
            TokenTreeRef::Leaf(leaf, _) => *leaf.span(),
        }
    }
}

impl<Span: Clone> TokenTreeRef<'_, Span> {
    pub fn cloned(&self) -> TokenTree<Span> {
        match self {
            TokenTreeRef::Subtree(subtree, tt) => match tt {
                Some(it) => (*it).clone(),
                None => (*subtree).clone().into(),
            },
            TokenTreeRef::Leaf(_, tt) => (*tt).clone(),
        }
    }
}

/// A safe version of `Cursor` from `syn` crate <https://github.com/dtolnay/syn/blob/6533607f91686545cb034d2838beea338d9d0742/src/buffer.rs#L125>
#[derive(Copy, Clone, Debug)]
pub struct Cursor<'a, Span> {
    buffer: &'a TokenBuffer<'a, Span>,
    ptr: EntryPtr,
}

impl<Span> PartialEq for Cursor<'_, Span> {
    fn eq(&self, other: &Cursor<'_, Span>) -> bool {
        self.ptr == other.ptr && std::ptr::eq(self.buffer, other.buffer)
    }
}

impl<Span> Eq for Cursor<'_, Span> {}

impl<'a, Span> Cursor<'a, Span> {
    /// Check whether it is eof
    pub fn eof(self) -> bool {
        matches!(self.buffer.entry(&self.ptr), None | Some(Entry::End(None)))
    }

    /// If the cursor is pointing at the end of a subtree, returns
    /// the parent subtree
    pub fn end(self) -> Option<&'a Subtree<Span>> {
        match self.entry() {
            Some(Entry::End(Some(ptr))) => {
                let idx = ptr.1;
                if let Some(Entry::Subtree(_, subtree, _)) =
                    self.buffer.entry(&EntryPtr(ptr.0, idx - 1))
                {
                    return Some(subtree);
                }
                None
            }
            _ => None,
        }
    }

    fn entry(&self) -> Option<&'a Entry<'a, Span>> {
        self.buffer.entry(&self.ptr)
    }

    /// If the cursor is pointing at a `Subtree`, returns
    /// a cursor into that subtree
    pub fn subtree(self) -> Option<Cursor<'a, Span>> {
        match self.entry() {
            Some(Entry::Subtree(_, _, entry_id)) => {
                Some(Cursor::create(self.buffer, EntryPtr(*entry_id, 0)))
            }
            _ => None,
        }
    }

    /// If the cursor is pointing at a `TokenTree`, returns it
    pub fn token_tree(self) -> Option<TokenTreeRef<'a, Span>> {
        match self.entry() {
            Some(Entry::Leaf(tt)) => match tt {
                TokenTree::Leaf(leaf) => Some(TokenTreeRef::Leaf(leaf, tt)),
                TokenTree::Subtree(subtree) => Some(TokenTreeRef::Subtree(subtree, Some(tt))),
            },
            Some(Entry::Subtree(tt, subtree, _)) => Some(TokenTreeRef::Subtree(subtree, *tt)),
            Some(Entry::End(_)) | None => None,
        }
    }

    fn create(buffer: &'a TokenBuffer<'_, Span>, ptr: EntryPtr) -> Cursor<'a, Span> {
        Cursor { buffer, ptr }
    }

    /// Bump the cursor
    pub fn bump(self) -> Cursor<'a, Span> {
        if let Some(Entry::End(exit)) = self.buffer.entry(&self.ptr) {
            match exit {
                Some(exit) => Cursor::create(self.buffer, *exit),
                None => self,
            }
        } else {
            Cursor::create(self.buffer, EntryPtr(self.ptr.0, self.ptr.1 + 1))
        }
    }

    /// Bump the cursor, if it is a subtree, returns
    /// a cursor into that subtree
    pub fn bump_subtree(self) -> Cursor<'a, Span> {
        match self.entry() {
            Some(&Entry::Subtree(_, _, entry_id)) => {
                Cursor::create(self.buffer, EntryPtr(entry_id, 0))
            }
            Some(Entry::End(exit)) => match exit {
                Some(exit) => Cursor::create(self.buffer, *exit),
                None => self,
            },
            _ => Cursor::create(self.buffer, EntryPtr(self.ptr.0, self.ptr.1 + 1)),
        }
    }

    /// Check whether it is a top level
    pub fn is_root(&self) -> bool {
        let entry_id = self.ptr.0;
        entry_id.0 == 0
    }
}