proc_macro_api/legacy_protocol/msg/
flat.rs

1//! Serialization-friendly representation of `tt::TopSubtree`.
2//!
3//! It is possible to serialize `TopSubtree` recursively, as a tree, but using
4//! arbitrary-nested trees in JSON is problematic, as they can cause the JSON
5//! parser to overflow the stack.
6//!
7//! Additionally, such implementation would be pretty verbose, and we do care
8//! about performance here a bit.
9//!
10//! So what this module does is dumping a `tt::TopSubtree` into a bunch of flat
11//! array of numbers.
12//!
13//! ```json
14//!  {
15//!    // Array of subtrees, each subtree is represented by 4 numbers:
16//!    // id of delimiter, delimiter kind, index of first child in `token_tree`,
17//!    // index of last child in `token_tree`
18//!    "subtree":[4294967295,0,0,5,2,2,5,5],
19//!    // 2 ints per literal: [token id, index into `text`]
20//!    "literal":[4294967295,1],
21//!    // 3 ints per punct: [token id, char, spacing]
22//!    "punct":[4294967295,64,1],
23//!    // 2 ints per ident: [token id, index into `text`]
24//!    "ident":   [0,0,1,1],
25//!    // children of all subtrees, concatenated. Each child is represented as `index << 2 | tag`
26//!    // where tag denotes one of subtree, literal, punct or ident.
27//!    "token_tree":[3,7,1,4],
28//!    // Strings shared by idents and literals
29//!    "text": ["struct","Foo"]
30//!  }
31//! ```
32//!
33//! We probably should replace most of the code here with bincode someday, but,
34//! as we don't have bincode in Cargo.toml yet, lets stick with serde_json for
35//! the time being.
36
37#[cfg(feature = "sysroot-abi")]
38use proc_macro_srv::TokenStream;
39
40use std::collections::VecDeque;
41
42use intern::Symbol;
43use rustc_hash::FxHashMap;
44use serde_derive::{Deserialize, Serialize};
45use span::{EditionedFileId, ErasedFileAstId, Span, SpanAnchor, SyntaxContext, TextRange};
46
47use crate::{
48    legacy_protocol::SpanId,
49    version::{ENCODE_CLOSE_SPAN_VERSION, EXTENDED_LEAF_DATA},
50};
51
52pub type SpanDataIndexMap =
53    indexmap::IndexSet<Span, std::hash::BuildHasherDefault<rustc_hash::FxHasher>>;
54
55pub fn serialize_span_data_index_map(map: &SpanDataIndexMap) -> Vec<u32> {
56    map.iter()
57        .flat_map(|span| {
58            [
59                span.anchor.file_id.as_u32(),
60                span.anchor.ast_id.into_raw(),
61                span.range.start().into(),
62                span.range.end().into(),
63                span.ctx.into_u32(),
64            ]
65        })
66        .collect()
67}
68
69pub fn deserialize_span_data_index_map(map: &[u32]) -> SpanDataIndexMap {
70    debug_assert!(map.len().is_multiple_of(5));
71    map.chunks_exact(5)
72        .map(|span| {
73            let &[file_id, ast_id, start, end, e] = span else { unreachable!() };
74            Span {
75                anchor: SpanAnchor {
76                    file_id: EditionedFileId::from_raw(file_id),
77                    ast_id: ErasedFileAstId::from_raw(ast_id),
78                },
79                range: TextRange::new(start.into(), end.into()),
80                // SAFETY: We only receive spans from the server. If someone mess up the communication UB can happen,
81                // but that will be their problem.
82                ctx: unsafe { SyntaxContext::from_u32(e) },
83            }
84        })
85        .collect()
86}
87
88#[derive(Serialize, Deserialize, Debug, Clone)]
89pub struct FlatTree {
90    subtree: Vec<u32>,
91    literal: Vec<u32>,
92    punct: Vec<u32>,
93    ident: Vec<u32>,
94    token_tree: Vec<u32>,
95    text: Vec<String>,
96}
97
98struct SubtreeRepr {
99    open: SpanId,
100    close: SpanId,
101    kind: tt::DelimiterKind,
102    tt: [u32; 2],
103}
104
105struct LiteralRepr {
106    id: SpanId,
107    text: u32,
108    suffix: u32,
109    kind: u16,
110}
111
112struct PunctRepr {
113    id: SpanId,
114    char: char,
115    spacing: tt::Spacing,
116}
117
118struct IdentRepr {
119    id: SpanId,
120    text: u32,
121    is_raw: bool,
122}
123
124impl FlatTree {
125    pub fn from_subtree(
126        subtree: tt::SubtreeView<'_, Span>,
127        version: u32,
128        span_data_table: &mut SpanDataIndexMap,
129    ) -> FlatTree {
130        let mut w = Writer::<Span, _> {
131            string_table: FxHashMap::default(),
132            work: VecDeque::new(),
133            span_data_table,
134
135            subtree: Vec::new(),
136            literal: Vec::new(),
137            punct: Vec::new(),
138            ident: Vec::new(),
139            token_tree: Vec::new(),
140            text: Vec::new(),
141            version,
142        };
143        w.write_subtree(subtree);
144
145        FlatTree {
146            subtree: if version >= ENCODE_CLOSE_SPAN_VERSION {
147                write_vec(w.subtree, SubtreeRepr::write_with_close_span)
148            } else {
149                write_vec(w.subtree, SubtreeRepr::write)
150            },
151            literal: if version >= EXTENDED_LEAF_DATA {
152                write_vec(w.literal, LiteralRepr::write_with_kind)
153            } else {
154                write_vec(w.literal, LiteralRepr::write)
155            },
156            punct: write_vec(w.punct, PunctRepr::write),
157            ident: if version >= EXTENDED_LEAF_DATA {
158                write_vec(w.ident, IdentRepr::write_with_rawness)
159            } else {
160                write_vec(w.ident, IdentRepr::write)
161            },
162            token_tree: w.token_tree,
163            text: w.text,
164        }
165    }
166
167    pub fn to_subtree_resolved(
168        self,
169        version: u32,
170        span_data_table: &SpanDataIndexMap,
171    ) -> tt::TopSubtree<Span> {
172        Reader::<Span> {
173            subtree: if version >= ENCODE_CLOSE_SPAN_VERSION {
174                read_vec(self.subtree, SubtreeRepr::read_with_close_span)
175            } else {
176                read_vec(self.subtree, SubtreeRepr::read)
177            },
178            literal: if version >= EXTENDED_LEAF_DATA {
179                read_vec(self.literal, LiteralRepr::read_with_kind)
180            } else {
181                read_vec(self.literal, LiteralRepr::read)
182            },
183            punct: read_vec(self.punct, PunctRepr::read),
184            ident: if version >= EXTENDED_LEAF_DATA {
185                read_vec(self.ident, IdentRepr::read_with_rawness)
186            } else {
187                read_vec(self.ident, IdentRepr::read)
188            },
189            token_tree: self.token_tree,
190            text: self.text,
191            span_data_table,
192            version,
193        }
194        .read_subtree()
195    }
196}
197
198#[cfg(feature = "sysroot-abi")]
199impl FlatTree {
200    pub fn from_tokenstream(
201        tokenstream: proc_macro_srv::TokenStream<Span>,
202        version: u32,
203        call_site: Span,
204        span_data_table: &mut SpanDataIndexMap,
205    ) -> FlatTree {
206        let mut w = Writer::<Span, _> {
207            string_table: FxHashMap::default(),
208            work: VecDeque::new(),
209            span_data_table,
210
211            subtree: Vec::new(),
212            literal: Vec::new(),
213            punct: Vec::new(),
214            ident: Vec::new(),
215            token_tree: Vec::new(),
216            text: Vec::new(),
217            version,
218        };
219        w.write_tokenstream(call_site, &tokenstream);
220
221        FlatTree {
222            subtree: if version >= ENCODE_CLOSE_SPAN_VERSION {
223                write_vec(w.subtree, SubtreeRepr::write_with_close_span)
224            } else {
225                write_vec(w.subtree, SubtreeRepr::write)
226            },
227            literal: if version >= EXTENDED_LEAF_DATA {
228                write_vec(w.literal, LiteralRepr::write_with_kind)
229            } else {
230                write_vec(w.literal, LiteralRepr::write)
231            },
232            punct: write_vec(w.punct, PunctRepr::write),
233            ident: if version >= EXTENDED_LEAF_DATA {
234                write_vec(w.ident, IdentRepr::write_with_rawness)
235            } else {
236                write_vec(w.ident, IdentRepr::write)
237            },
238            token_tree: w.token_tree,
239            text: w.text,
240        }
241    }
242
243    pub fn from_tokenstream_raw<T: SpanTransformer<Table = ()>>(
244        tokenstream: proc_macro_srv::TokenStream<T::Span>,
245        call_site: T::Span,
246        version: u32,
247    ) -> FlatTree {
248        let mut w = Writer::<T, _> {
249            string_table: FxHashMap::default(),
250            work: VecDeque::new(),
251            span_data_table: &mut (),
252
253            subtree: Vec::new(),
254            literal: Vec::new(),
255            punct: Vec::new(),
256            ident: Vec::new(),
257            token_tree: Vec::new(),
258            text: Vec::new(),
259            version,
260        };
261        w.write_tokenstream(call_site, &tokenstream);
262
263        FlatTree {
264            subtree: if version >= ENCODE_CLOSE_SPAN_VERSION {
265                write_vec(w.subtree, SubtreeRepr::write_with_close_span)
266            } else {
267                write_vec(w.subtree, SubtreeRepr::write)
268            },
269            literal: if version >= EXTENDED_LEAF_DATA {
270                write_vec(w.literal, LiteralRepr::write_with_kind)
271            } else {
272                write_vec(w.literal, LiteralRepr::write)
273            },
274            punct: write_vec(w.punct, PunctRepr::write),
275            ident: if version >= EXTENDED_LEAF_DATA {
276                write_vec(w.ident, IdentRepr::write_with_rawness)
277            } else {
278                write_vec(w.ident, IdentRepr::write)
279            },
280            token_tree: w.token_tree,
281            text: w.text,
282        }
283    }
284
285    pub fn to_tokenstream_unresolved<T: SpanTransformer<Table = ()>>(
286        self,
287        version: u32,
288        span_join: impl Fn(T::Span, T::Span) -> T::Span,
289    ) -> proc_macro_srv::TokenStream<T::Span> {
290        Reader::<T> {
291            subtree: if version >= ENCODE_CLOSE_SPAN_VERSION {
292                read_vec(self.subtree, SubtreeRepr::read_with_close_span)
293            } else {
294                read_vec(self.subtree, SubtreeRepr::read)
295            },
296            literal: if version >= EXTENDED_LEAF_DATA {
297                read_vec(self.literal, LiteralRepr::read_with_kind)
298            } else {
299                read_vec(self.literal, LiteralRepr::read)
300            },
301            punct: read_vec(self.punct, PunctRepr::read),
302            ident: if version >= EXTENDED_LEAF_DATA {
303                read_vec(self.ident, IdentRepr::read_with_rawness)
304            } else {
305                read_vec(self.ident, IdentRepr::read)
306            },
307            token_tree: self.token_tree,
308            text: self.text,
309            span_data_table: &(),
310            version,
311        }
312        .read_tokenstream(span_join)
313    }
314
315    pub fn to_tokenstream_resolved(
316        self,
317        version: u32,
318        span_data_table: &SpanDataIndexMap,
319        span_join: impl Fn(Span, Span) -> Span,
320    ) -> proc_macro_srv::TokenStream<Span> {
321        Reader::<Span> {
322            subtree: if version >= ENCODE_CLOSE_SPAN_VERSION {
323                read_vec(self.subtree, SubtreeRepr::read_with_close_span)
324            } else {
325                read_vec(self.subtree, SubtreeRepr::read)
326            },
327            literal: if version >= EXTENDED_LEAF_DATA {
328                read_vec(self.literal, LiteralRepr::read_with_kind)
329            } else {
330                read_vec(self.literal, LiteralRepr::read)
331            },
332            punct: read_vec(self.punct, PunctRepr::read),
333            ident: if version >= EXTENDED_LEAF_DATA {
334                read_vec(self.ident, IdentRepr::read_with_rawness)
335            } else {
336                read_vec(self.ident, IdentRepr::read)
337            },
338            token_tree: self.token_tree,
339            text: self.text,
340            span_data_table,
341            version,
342        }
343        .read_tokenstream(span_join)
344    }
345}
346
347fn read_vec<T, F: Fn([u32; N]) -> T, const N: usize>(xs: Vec<u32>, f: F) -> Vec<T> {
348    let mut chunks = xs.chunks_exact(N);
349    let res = chunks.by_ref().map(|chunk| f(chunk.try_into().unwrap())).collect();
350    assert!(chunks.remainder().is_empty());
351    res
352}
353
354fn write_vec<T, F: Fn(T) -> [u32; N], const N: usize>(xs: Vec<T>, f: F) -> Vec<u32> {
355    xs.into_iter().flat_map(f).collect()
356}
357
358impl SubtreeRepr {
359    fn write(self) -> [u32; 4] {
360        let kind = match self.kind {
361            tt::DelimiterKind::Invisible => 0,
362            tt::DelimiterKind::Parenthesis => 1,
363            tt::DelimiterKind::Brace => 2,
364            tt::DelimiterKind::Bracket => 3,
365        };
366        [self.open.0, kind, self.tt[0], self.tt[1]]
367    }
368    fn read([open, kind, lo, len]: [u32; 4]) -> SubtreeRepr {
369        let kind = match kind {
370            0 => tt::DelimiterKind::Invisible,
371            1 => tt::DelimiterKind::Parenthesis,
372            2 => tt::DelimiterKind::Brace,
373            3 => tt::DelimiterKind::Bracket,
374            other => panic!("bad kind {other}"),
375        };
376        SubtreeRepr { open: SpanId(open), close: SpanId(!0), kind, tt: [lo, len] }
377    }
378    fn write_with_close_span(self) -> [u32; 5] {
379        let kind = match self.kind {
380            tt::DelimiterKind::Invisible => 0,
381            tt::DelimiterKind::Parenthesis => 1,
382            tt::DelimiterKind::Brace => 2,
383            tt::DelimiterKind::Bracket => 3,
384        };
385        [self.open.0, self.close.0, kind, self.tt[0], self.tt[1]]
386    }
387    fn read_with_close_span([open, close, kind, lo, len]: [u32; 5]) -> SubtreeRepr {
388        let kind = match kind {
389            0 => tt::DelimiterKind::Invisible,
390            1 => tt::DelimiterKind::Parenthesis,
391            2 => tt::DelimiterKind::Brace,
392            3 => tt::DelimiterKind::Bracket,
393            other => panic!("bad kind {other}"),
394        };
395        SubtreeRepr { open: SpanId(open), close: SpanId(close), kind, tt: [lo, len] }
396    }
397}
398
399impl LiteralRepr {
400    fn write(self) -> [u32; 2] {
401        [self.id.0, self.text]
402    }
403    fn read([id, text]: [u32; 2]) -> LiteralRepr {
404        LiteralRepr { id: SpanId(id), text, kind: 0, suffix: !0 }
405    }
406    fn write_with_kind(self) -> [u32; 4] {
407        [self.id.0, self.text, self.kind as u32, self.suffix]
408    }
409    fn read_with_kind([id, text, kind, suffix]: [u32; 4]) -> LiteralRepr {
410        LiteralRepr { id: SpanId(id), text, kind: kind as u16, suffix }
411    }
412}
413
414impl PunctRepr {
415    fn write(self) -> [u32; 3] {
416        let spacing = match self.spacing {
417            tt::Spacing::Alone | tt::Spacing::JointHidden => 0,
418            tt::Spacing::Joint => 1,
419        };
420        [self.id.0, self.char as u32, spacing]
421    }
422    fn read([id, char, spacing]: [u32; 3]) -> PunctRepr {
423        let spacing = match spacing {
424            0 => tt::Spacing::Alone,
425            1 => tt::Spacing::Joint,
426            other => panic!("bad spacing {other}"),
427        };
428        PunctRepr { id: SpanId(id), char: char.try_into().unwrap(), spacing }
429    }
430}
431
432impl IdentRepr {
433    fn write(self) -> [u32; 2] {
434        [self.id.0, self.text]
435    }
436    fn read(data: [u32; 2]) -> IdentRepr {
437        IdentRepr { id: SpanId(data[0]), text: data[1], is_raw: false }
438    }
439    fn write_with_rawness(self) -> [u32; 3] {
440        [self.id.0, self.text, self.is_raw as u32]
441    }
442    fn read_with_rawness([id, text, is_raw]: [u32; 3]) -> IdentRepr {
443        IdentRepr { id: SpanId(id), text, is_raw: is_raw == 1 }
444    }
445}
446
447pub trait SpanTransformer {
448    type Table;
449    type Span: Copy;
450    fn token_id_of(table: &mut Self::Table, s: Self::Span) -> SpanId;
451    fn span_for_token_id(table: &Self::Table, id: SpanId) -> Self::Span;
452}
453impl SpanTransformer for SpanId {
454    type Table = ();
455    type Span = Self;
456    fn token_id_of((): &mut Self::Table, token_id: Self::Span) -> SpanId {
457        token_id
458    }
459
460    fn span_for_token_id((): &Self::Table, id: SpanId) -> Self::Span {
461        id
462    }
463}
464impl SpanTransformer for Span {
465    type Table = SpanDataIndexMap;
466    type Span = Self;
467    fn token_id_of(table: &mut Self::Table, span: Self::Span) -> SpanId {
468        SpanId(table.insert_full(span).0 as u32)
469    }
470    fn span_for_token_id(table: &Self::Table, id: SpanId) -> Self::Span {
471        *table.get_index(id.0 as usize).unwrap_or_else(|| &table[0])
472    }
473}
474
475struct Writer<'a, 'span, S: SpanTransformer, W> {
476    work: VecDeque<(usize, usize, W)>,
477    string_table: FxHashMap<std::borrow::Cow<'a, str>, u32>,
478    span_data_table: &'span mut S::Table,
479    version: u32,
480
481    subtree: Vec<SubtreeRepr>,
482    literal: Vec<LiteralRepr>,
483    punct: Vec<PunctRepr>,
484    ident: Vec<IdentRepr>,
485    token_tree: Vec<u32>,
486    text: Vec<String>,
487}
488
489impl<'a, T: SpanTransformer> Writer<'a, '_, T, tt::iter::TtIter<'a, T::Span>> {
490    fn write_subtree(&mut self, root: tt::SubtreeView<'a, T::Span>) {
491        let subtree = root.top_subtree();
492        self.enqueue(subtree, root.iter());
493        while let Some((idx, len, subtree)) = self.work.pop_front() {
494            self.subtree(idx, len, subtree);
495        }
496    }
497
498    fn subtree(&mut self, idx: usize, n_tt: usize, subtree: tt::iter::TtIter<'a, T::Span>) {
499        let mut first_tt = self.token_tree.len();
500        self.token_tree.resize(first_tt + n_tt, !0);
501
502        self.subtree[idx].tt = [first_tt as u32, (first_tt + n_tt) as u32];
503
504        for child in subtree {
505            let idx_tag = match child {
506                tt::iter::TtElement::Subtree(subtree, subtree_iter) => {
507                    let idx = self.enqueue(subtree, subtree_iter);
508                    idx << 2
509                }
510                tt::iter::TtElement::Leaf(leaf) => match leaf {
511                    tt::Leaf::Literal(lit) => {
512                        let idx = self.literal.len() as u32;
513                        let id = self.token_id_of(lit.span);
514                        let (text, suffix) = if self.version >= EXTENDED_LEAF_DATA {
515                            (
516                                self.intern(lit.symbol.as_str()),
517                                lit.suffix.as_ref().map(|s| self.intern(s.as_str())).unwrap_or(!0),
518                            )
519                        } else {
520                            (self.intern_owned(format!("{lit}")), !0)
521                        };
522                        self.literal.push(LiteralRepr {
523                            id,
524                            text,
525                            kind: u16::from_le_bytes(match lit.kind {
526                                tt::LitKind::Err(_) => [0, 0],
527                                tt::LitKind::Byte => [1, 0],
528                                tt::LitKind::Char => [2, 0],
529                                tt::LitKind::Integer => [3, 0],
530                                tt::LitKind::Float => [4, 0],
531                                tt::LitKind::Str => [5, 0],
532                                tt::LitKind::StrRaw(r) => [6, r],
533                                tt::LitKind::ByteStr => [7, 0],
534                                tt::LitKind::ByteStrRaw(r) => [8, r],
535                                tt::LitKind::CStr => [9, 0],
536                                tt::LitKind::CStrRaw(r) => [10, r],
537                            }),
538                            suffix,
539                        });
540                        (idx << 2) | 0b01
541                    }
542                    tt::Leaf::Punct(punct) => {
543                        let idx = self.punct.len() as u32;
544                        let id = self.token_id_of(punct.span);
545                        self.punct.push(PunctRepr { char: punct.char, spacing: punct.spacing, id });
546                        (idx << 2) | 0b10
547                    }
548                    tt::Leaf::Ident(ident) => {
549                        let idx = self.ident.len() as u32;
550                        let id = self.token_id_of(ident.span);
551                        let text = if self.version >= EXTENDED_LEAF_DATA {
552                            self.intern(ident.sym.as_str())
553                        } else if ident.is_raw.yes() {
554                            self.intern_owned(format!("r#{}", ident.sym.as_str(),))
555                        } else {
556                            self.intern(ident.sym.as_str())
557                        };
558                        self.ident.push(IdentRepr { id, text, is_raw: ident.is_raw.yes() });
559                        (idx << 2) | 0b11
560                    }
561                },
562            };
563            self.token_tree[first_tt] = idx_tag;
564            first_tt += 1;
565        }
566    }
567
568    fn enqueue(
569        &mut self,
570        subtree: &'a tt::Subtree<T::Span>,
571        contents: tt::iter::TtIter<'a, T::Span>,
572    ) -> u32 {
573        let idx = self.subtree.len();
574        let open = self.token_id_of(subtree.delimiter.open);
575        let close = self.token_id_of(subtree.delimiter.close);
576        let delimiter_kind = subtree.delimiter.kind;
577        self.subtree.push(SubtreeRepr { open, close, kind: delimiter_kind, tt: [!0, !0] });
578        // FIXME: `count()` walks over the entire iterator.
579        self.work.push_back((idx, contents.clone().count(), contents));
580        idx as u32
581    }
582}
583
584impl<'a, T: SpanTransformer, U> Writer<'a, '_, T, U> {
585    fn token_id_of(&mut self, span: T::Span) -> SpanId {
586        T::token_id_of(self.span_data_table, span)
587    }
588
589    pub(crate) fn intern(&mut self, text: &'a str) -> u32 {
590        let table = &mut self.text;
591        *self.string_table.entry(text.into()).or_insert_with(|| {
592            let idx = table.len();
593            table.push(text.to_owned());
594            idx as u32
595        })
596    }
597
598    pub(crate) fn intern_owned(&mut self, text: String) -> u32 {
599        let table = &mut self.text;
600        *self.string_table.entry(text.clone().into()).or_insert_with(|| {
601            let idx = table.len();
602            table.push(text);
603            idx as u32
604        })
605    }
606}
607
608#[cfg(feature = "sysroot-abi")]
609impl<'a, T: SpanTransformer>
610    Writer<'a, '_, T, Option<proc_macro_srv::TokenStreamIter<'a, T::Span>>>
611{
612    fn write_tokenstream(
613        &mut self,
614        call_site: T::Span,
615        root: &'a proc_macro_srv::TokenStream<T::Span>,
616    ) {
617        let call_site = self.token_id_of(call_site);
618        if let Some(group) = root.as_single_group() {
619            self.enqueue(group);
620        } else {
621            self.subtree.push(SubtreeRepr {
622                open: call_site,
623                close: call_site,
624                kind: tt::DelimiterKind::Invisible,
625                tt: [!0, !0],
626            });
627            self.work.push_back((0, root.len(), Some(root.iter())));
628        }
629        while let Some((idx, len, group)) = self.work.pop_front() {
630            self.group(idx, len, group);
631        }
632    }
633
634    fn group(
635        &mut self,
636        idx: usize,
637        n_tt: usize,
638        group: Option<proc_macro_srv::TokenStreamIter<'a, T::Span>>,
639    ) {
640        let mut first_tt = self.token_tree.len();
641        self.token_tree.resize(first_tt + n_tt, !0);
642
643        self.subtree[idx].tt = [first_tt as u32, (first_tt + n_tt) as u32];
644
645        for tt in group.into_iter().flatten() {
646            let idx_tag = match tt {
647                proc_macro_srv::TokenTree::Group(group) => {
648                    let idx = self.enqueue(group);
649                    idx << 2
650                }
651                proc_macro_srv::TokenTree::Literal(lit) => {
652                    let idx = self.literal.len() as u32;
653                    let id = self.token_id_of(lit.span);
654                    let (text, suffix) = if self.version >= EXTENDED_LEAF_DATA {
655                        (
656                            self.intern(lit.symbol.as_str()),
657                            lit.suffix.as_ref().map(|s| self.intern(s.as_str())).unwrap_or(!0),
658                        )
659                    } else {
660                        (self.intern_owned(proc_macro_srv::literal_to_string(lit)), !0)
661                    };
662                    self.literal.push(LiteralRepr {
663                        id,
664                        text,
665                        kind: u16::from_le_bytes(match lit.kind {
666                            proc_macro_srv::LitKind::ErrWithGuar => [0, 0],
667                            proc_macro_srv::LitKind::Byte => [1, 0],
668                            proc_macro_srv::LitKind::Char => [2, 0],
669                            proc_macro_srv::LitKind::Integer => [3, 0],
670                            proc_macro_srv::LitKind::Float => [4, 0],
671                            proc_macro_srv::LitKind::Str => [5, 0],
672                            proc_macro_srv::LitKind::StrRaw(r) => [6, r],
673                            proc_macro_srv::LitKind::ByteStr => [7, 0],
674                            proc_macro_srv::LitKind::ByteStrRaw(r) => [8, r],
675                            proc_macro_srv::LitKind::CStr => [9, 0],
676                            proc_macro_srv::LitKind::CStrRaw(r) => [10, r],
677                        }),
678                        suffix,
679                    });
680                    (idx << 2) | 0b01
681                }
682                proc_macro_srv::TokenTree::Punct(punct) => {
683                    let idx = self.punct.len() as u32;
684                    let id = self.token_id_of(punct.span);
685                    self.punct.push(PunctRepr {
686                        char: punct.ch as char,
687                        spacing: if punct.joint { tt::Spacing::Joint } else { tt::Spacing::Alone },
688                        id,
689                    });
690                    (idx << 2) | 0b10
691                }
692                proc_macro_srv::TokenTree::Ident(ident) => {
693                    let idx = self.ident.len() as u32;
694                    let id = self.token_id_of(ident.span);
695                    let text = if self.version >= EXTENDED_LEAF_DATA {
696                        self.intern(ident.sym.as_str())
697                    } else if ident.is_raw {
698                        self.intern_owned(format!("r#{}", ident.sym.as_str(),))
699                    } else {
700                        self.intern(ident.sym.as_str())
701                    };
702                    self.ident.push(IdentRepr { id, text, is_raw: ident.is_raw });
703                    (idx << 2) | 0b11
704                }
705            };
706            self.token_tree[first_tt] = idx_tag;
707            first_tt += 1;
708        }
709    }
710
711    fn enqueue(&mut self, group: &'a proc_macro_srv::Group<T::Span>) -> u32 {
712        let idx = self.subtree.len();
713        let open = self.token_id_of(group.span.open);
714        let close = self.token_id_of(group.span.close);
715        let delimiter_kind = match group.delimiter {
716            proc_macro_srv::Delimiter::Parenthesis => tt::DelimiterKind::Parenthesis,
717            proc_macro_srv::Delimiter::Brace => tt::DelimiterKind::Brace,
718            proc_macro_srv::Delimiter::Bracket => tt::DelimiterKind::Bracket,
719            proc_macro_srv::Delimiter::None => tt::DelimiterKind::Invisible,
720        };
721        self.subtree.push(SubtreeRepr { open, close, kind: delimiter_kind, tt: [!0, !0] });
722        self.work.push_back((
723            idx,
724            group.stream.as_ref().map_or(0, |stream| stream.len()),
725            group.stream.as_ref().map(|ts| ts.iter()),
726        ));
727        idx as u32
728    }
729}
730
731struct Reader<'span, S: SpanTransformer> {
732    version: u32,
733    subtree: Vec<SubtreeRepr>,
734    literal: Vec<LiteralRepr>,
735    punct: Vec<PunctRepr>,
736    ident: Vec<IdentRepr>,
737    token_tree: Vec<u32>,
738    text: Vec<String>,
739    span_data_table: &'span S::Table,
740}
741
742impl<T: SpanTransformer> Reader<'_, T> {
743    pub(crate) fn read_subtree(self) -> tt::TopSubtree<T::Span> {
744        let mut res: Vec<Option<(tt::Delimiter<T::Span>, Vec<tt::TokenTree<T::Span>>)>> =
745            vec![None; self.subtree.len()];
746        let read_span = |id| T::span_for_token_id(self.span_data_table, id);
747        for i in (0..self.subtree.len()).rev() {
748            let repr = &self.subtree[i];
749            let token_trees = &self.token_tree[repr.tt[0] as usize..repr.tt[1] as usize];
750            let delimiter = tt::Delimiter {
751                open: read_span(repr.open),
752                close: read_span(repr.close),
753                kind: repr.kind,
754            };
755            let mut s = Vec::new();
756            for &idx_tag in token_trees {
757                let tag = idx_tag & 0b11;
758                let idx = (idx_tag >> 2) as usize;
759                match tag {
760                    // XXX: we iterate subtrees in reverse to guarantee
761                    // that this unwrap doesn't fire.
762                    0b00 => {
763                        let (delimiter, subtree) = res[idx].take().unwrap();
764                        s.push(tt::TokenTree::Subtree(tt::Subtree {
765                            delimiter,
766                            len: subtree.len() as u32,
767                        }));
768                        s.extend(subtree)
769                    }
770                    0b01 => {
771                        use tt::LitKind::*;
772                        let repr = &self.literal[idx];
773                        let text = self.text[repr.text as usize].as_str();
774                        let span = read_span(repr.id);
775                        s.push(
776                            tt::Leaf::Literal(if self.version >= EXTENDED_LEAF_DATA {
777                                tt::Literal {
778                                    symbol: Symbol::intern(text),
779                                    span,
780                                    kind: match u16::to_le_bytes(repr.kind) {
781                                        [0, _] => Err(()),
782                                        [1, _] => Byte,
783                                        [2, _] => Char,
784                                        [3, _] => Integer,
785                                        [4, _] => Float,
786                                        [5, _] => Str,
787                                        [6, r] => StrRaw(r),
788                                        [7, _] => ByteStr,
789                                        [8, r] => ByteStrRaw(r),
790                                        [9, _] => CStr,
791                                        [10, r] => CStrRaw(r),
792                                        _ => unreachable!(),
793                                    },
794                                    suffix: if repr.suffix != !0 {
795                                        Some(Symbol::intern(
796                                            self.text[repr.suffix as usize].as_str(),
797                                        ))
798                                    } else {
799                                        None
800                                    },
801                                }
802                            } else {
803                                tt::token_to_literal(text, span)
804                            })
805                            .into(),
806                        )
807                    }
808                    0b10 => {
809                        let repr = &self.punct[idx];
810                        s.push(
811                            tt::Leaf::Punct(tt::Punct {
812                                char: repr.char,
813                                spacing: repr.spacing,
814                                span: read_span(repr.id),
815                            })
816                            .into(),
817                        )
818                    }
819                    0b11 => {
820                        let repr = &self.ident[idx];
821                        let text = self.text[repr.text as usize].as_str();
822                        let (is_raw, text) = if self.version >= EXTENDED_LEAF_DATA {
823                            (
824                                if repr.is_raw { tt::IdentIsRaw::Yes } else { tt::IdentIsRaw::No },
825                                text,
826                            )
827                        } else {
828                            tt::IdentIsRaw::split_from_symbol(text)
829                        };
830                        s.push(
831                            tt::Leaf::Ident(tt::Ident {
832                                sym: Symbol::intern(text),
833                                span: read_span(repr.id),
834                                is_raw,
835                            })
836                            .into(),
837                        )
838                    }
839                    other => panic!("bad tag: {other}"),
840                }
841            }
842            res[i] = Some((delimiter, s));
843        }
844
845        let (delimiter, mut res) = res[0].take().unwrap();
846        res.insert(0, tt::TokenTree::Subtree(tt::Subtree { delimiter, len: res.len() as u32 }));
847        tt::TopSubtree(res.into_boxed_slice())
848    }
849}
850
851#[cfg(feature = "sysroot-abi")]
852impl<T: SpanTransformer> Reader<'_, T> {
853    pub(crate) fn read_tokenstream(
854        self,
855        span_join: impl Fn(T::Span, T::Span) -> T::Span,
856    ) -> proc_macro_srv::TokenStream<T::Span> {
857        let mut res: Vec<Option<proc_macro_srv::Group<T::Span>>> = vec![None; self.subtree.len()];
858        let read_span = |id| T::span_for_token_id(self.span_data_table, id);
859        for i in (0..self.subtree.len()).rev() {
860            let repr = &self.subtree[i];
861            let token_trees = &self.token_tree[repr.tt[0] as usize..repr.tt[1] as usize];
862
863            let stream = token_trees
864                .iter()
865                .copied()
866                .map(|idx_tag| {
867                    let tag = idx_tag & 0b11;
868                    let idx = (idx_tag >> 2) as usize;
869                    match tag {
870                        // XXX: we iterate subtrees in reverse to guarantee
871                        // that this unwrap doesn't fire.
872                        0b00 => proc_macro_srv::TokenTree::Group(res[idx].take().unwrap()),
873                        0b01 => {
874                            let repr = &self.literal[idx];
875                            let text = self.text[repr.text as usize].as_str();
876                            let span = read_span(repr.id);
877                            proc_macro_srv::TokenTree::Literal(
878                                if self.version >= EXTENDED_LEAF_DATA {
879                                    proc_macro_srv::Literal {
880                                        symbol: Symbol::intern(text),
881                                        span,
882                                        kind: match u16::to_le_bytes(repr.kind) {
883                                            [0, _] => proc_macro_srv::LitKind::ErrWithGuar,
884                                            [1, _] => proc_macro_srv::LitKind::Byte,
885                                            [2, _] => proc_macro_srv::LitKind::Char,
886                                            [3, _] => proc_macro_srv::LitKind::Integer,
887                                            [4, _] => proc_macro_srv::LitKind::Float,
888                                            [5, _] => proc_macro_srv::LitKind::Str,
889                                            [6, r] => proc_macro_srv::LitKind::StrRaw(r),
890                                            [7, _] => proc_macro_srv::LitKind::ByteStr,
891                                            [8, r] => proc_macro_srv::LitKind::ByteStrRaw(r),
892                                            [9, _] => proc_macro_srv::LitKind::CStr,
893                                            [10, r] => proc_macro_srv::LitKind::CStrRaw(r),
894                                            _ => unreachable!(),
895                                        },
896                                        suffix: if repr.suffix != !0 {
897                                            Some(Symbol::intern(
898                                                self.text[repr.suffix as usize].as_str(),
899                                            ))
900                                        } else {
901                                            None
902                                        },
903                                    }
904                                } else {
905                                    proc_macro_srv::literal_from_str(text, span).unwrap_or_else(
906                                        |_| proc_macro_srv::Literal {
907                                            symbol: Symbol::intern("internal error"),
908                                            span,
909                                            kind: proc_macro_srv::LitKind::ErrWithGuar,
910                                            suffix: None,
911                                        },
912                                    )
913                                },
914                            )
915                        }
916                        0b10 => {
917                            let repr = &self.punct[idx];
918                            proc_macro_srv::TokenTree::Punct(proc_macro_srv::Punct {
919                                ch: repr.char as u8,
920                                joint: repr.spacing == tt::Spacing::Joint,
921                                span: read_span(repr.id),
922                            })
923                        }
924                        0b11 => {
925                            let repr = &self.ident[idx];
926                            let text = self.text[repr.text as usize].as_str();
927                            let (is_raw, text) = if self.version >= EXTENDED_LEAF_DATA {
928                                (
929                                    if repr.is_raw {
930                                        tt::IdentIsRaw::Yes
931                                    } else {
932                                        tt::IdentIsRaw::No
933                                    },
934                                    text,
935                                )
936                            } else {
937                                tt::IdentIsRaw::split_from_symbol(text)
938                            };
939                            proc_macro_srv::TokenTree::Ident(proc_macro_srv::Ident {
940                                sym: Symbol::intern(text),
941                                span: read_span(repr.id),
942                                is_raw: is_raw.yes(),
943                            })
944                        }
945                        other => panic!("bad tag: {other}"),
946                    }
947                })
948                .collect::<Vec<_>>();
949            let open = read_span(repr.open);
950            let close = read_span(repr.close);
951            let g = proc_macro_srv::Group {
952                delimiter: match repr.kind {
953                    tt::DelimiterKind::Parenthesis => proc_macro_srv::Delimiter::Parenthesis,
954                    tt::DelimiterKind::Brace => proc_macro_srv::Delimiter::Brace,
955                    tt::DelimiterKind::Bracket => proc_macro_srv::Delimiter::Bracket,
956                    tt::DelimiterKind::Invisible => proc_macro_srv::Delimiter::None,
957                },
958                stream: if stream.is_empty() { None } else { Some(TokenStream::new(stream)) },
959                span: proc_macro_srv::DelimSpan {
960                    open,
961                    close,
962                    // FIXME: The protocol does not yet encode entire spans ...
963                    entire: span_join(open, close),
964                },
965            };
966            res[i] = Some(g);
967        }
968        let group = res[0].take().unwrap();
969        if group.delimiter == proc_macro_srv::Delimiter::None {
970            group.stream.unwrap_or_default()
971        } else {
972            TokenStream::new(vec![proc_macro_srv::TokenTree::Group(group)])
973        }
974    }
975}
proc_macro_api/legacy_protocol/msg/flat.rs

proc_macro_api/legacy_protocol/msg/
flat.rs
