mbe/
parser.rs

1//! Parser recognizes special macro syntax, `$var` and `$(repeat)*`, in token
2//! trees.
3
4use std::sync::Arc;
5
6use arrayvec::ArrayVec;
7use intern::{Symbol, sym};
8use span::{Edition, Span, SyntaxContext};
9use tt::{
10    MAX_GLUED_PUNCT_LEN,
11    iter::{TtElement, TtIter},
12};
13
14use crate::{MacroCallStyle, ParseError};
15
16pub(crate) fn parse_rule_style(src: &mut TtIter<'_, Span>) -> Result<MacroCallStyle, ParseError> {
17    // Skip an optional `unsafe`. This is only actually allowed for `attr`
18    // rules, but we'll let rustc worry about that.
19    if let Some(TtElement::Leaf(tt::Leaf::Ident(ident))) = src.peek()
20        && ident.sym == sym::unsafe_
21    {
22        src.next().expect("already peeked");
23    }
24
25    let kind = match src.peek() {
26        Some(TtElement::Leaf(tt::Leaf::Ident(ident))) if ident.sym == sym::attr => {
27            src.next().expect("already peeked");
28            // FIXME: Add support for `attr(..)` rules with attribute arguments,
29            // which would be inside these parens.
30            src.expect_subtree().map_err(|_| ParseError::expected("expected `()`"))?;
31            MacroCallStyle::Attr
32        }
33        Some(TtElement::Leaf(tt::Leaf::Ident(ident))) if ident.sym == sym::derive => {
34            src.next().expect("already peeked");
35            src.expect_subtree().map_err(|_| ParseError::expected("expected `()`"))?;
36            MacroCallStyle::Derive
37        }
38        _ => MacroCallStyle::FnLike,
39    };
40    Ok(kind)
41}
42
43/// Consider
44///
45/// ```
46/// macro_rules! a_macro {
47///     ($x:expr, $y:expr) => ($y * $x)
48/// }
49/// ```
50///
51/// Stuff to the left of `=>` is a [`MetaTemplate`] pattern (which is matched
52/// with input).
53///
54/// Stuff to the right is a [`MetaTemplate`] template which is used to produce
55/// output.
56#[derive(Clone, Debug, PartialEq, Eq)]
57pub(crate) struct MetaTemplate(pub(crate) Box<[Op]>);
58
59impl MetaTemplate {
60    pub(crate) fn parse_pattern(
61        edition: impl Copy + Fn(SyntaxContext) -> Edition,
62        pattern: TtIter<'_, Span>,
63    ) -> Result<Self, ParseError> {
64        MetaTemplate::parse(edition, pattern, Mode::Pattern)
65    }
66
67    pub(crate) fn parse_template(
68        edition: impl Copy + Fn(SyntaxContext) -> Edition,
69        template: TtIter<'_, Span>,
70    ) -> Result<Self, ParseError> {
71        MetaTemplate::parse(edition, template, Mode::Template)
72    }
73
74    pub(crate) fn iter(&self) -> impl Iterator<Item = &Op> {
75        self.0.iter()
76    }
77
78    fn parse(
79        edition: impl Copy + Fn(SyntaxContext) -> Edition,
80        mut src: TtIter<'_, Span>,
81        mode: Mode,
82    ) -> Result<Self, ParseError> {
83        let mut res = Vec::new();
84        while let Some(first) = src.peek() {
85            let op = next_op(edition, first, &mut src, mode)?;
86            res.push(op);
87        }
88
89        Ok(MetaTemplate(res.into_boxed_slice()))
90    }
91}
92
93#[derive(Clone, Debug, PartialEq, Eq)]
94pub(crate) enum Op {
95    Var {
96        name: Symbol,
97        kind: Option<MetaVarKind>,
98        id: Span,
99    },
100    Ignore {
101        name: Symbol,
102        id: Span,
103    },
104    Index {
105        depth: usize,
106    },
107    Len {
108        depth: usize,
109    },
110    Count {
111        name: Symbol,
112        // FIXME: `usize`` once we drop support for 1.76
113        depth: Option<usize>,
114    },
115    Concat {
116        elements: Box<[ConcatMetaVarExprElem]>,
117        span: Span,
118    },
119    Repeat {
120        tokens: MetaTemplate,
121        kind: RepeatKind,
122        separator: Option<Arc<Separator>>,
123    },
124    Subtree {
125        tokens: MetaTemplate,
126        delimiter: tt::Delimiter<Span>,
127    },
128    Literal(tt::Literal<Span>),
129    Punct(Box<ArrayVec<tt::Punct<Span>, MAX_GLUED_PUNCT_LEN>>),
130    Ident(tt::Ident<Span>),
131}
132
133#[derive(Clone, Debug, PartialEq, Eq)]
134pub(crate) enum ConcatMetaVarExprElem {
135    /// There is NO preceding dollar sign, which means that this identifier should be interpreted
136    /// as a literal.
137    Ident(tt::Ident<Span>),
138    /// There is a preceding dollar sign, which means that this identifier should be expanded
139    /// and interpreted as a variable.
140    Var(tt::Ident<Span>),
141    /// For example, a number or a string.
142    Literal(tt::Literal<Span>),
143}
144
145#[derive(Copy, Clone, Debug, PartialEq, Eq)]
146pub(crate) enum RepeatKind {
147    ZeroOrMore,
148    OneOrMore,
149    ZeroOrOne,
150}
151
152#[derive(Copy, Clone, Debug, PartialEq, Eq)]
153pub(crate) enum ExprKind {
154    // Matches expressions using the post-edition 2024. Was written using
155    // `expr` in edition 2024 or later.
156    Expr,
157    // Matches expressions using the pre-edition 2024 rules.
158    // Either written using `expr` in edition 2021 or earlier or.was written using `expr_2021`.
159    Expr2021,
160}
161
162#[derive(Copy, Clone, Debug, PartialEq, Eq)]
163pub(crate) enum MetaVarKind {
164    Path,
165    Ty,
166    Pat,
167    PatParam,
168    Stmt,
169    Block,
170    Meta,
171    Item,
172    Vis,
173    Expr(ExprKind),
174    Ident,
175    Tt,
176    Lifetime,
177    Literal,
178}
179
180#[derive(Clone, Debug, Eq)]
181pub(crate) enum Separator {
182    Literal(tt::Literal<Span>),
183    Ident(tt::Ident<Span>),
184    Puncts(ArrayVec<tt::Punct<Span>, MAX_GLUED_PUNCT_LEN>),
185    Lifetime(tt::Punct<Span>, tt::Ident<Span>),
186}
187
188// Note that when we compare a Separator, we just care about its textual value.
189impl PartialEq for Separator {
190    fn eq(&self, other: &Separator) -> bool {
191        use Separator::*;
192
193        match (self, other) {
194            (Ident(a), Ident(b)) => a.sym == b.sym,
195            (Literal(a), Literal(b)) => a.symbol == b.symbol,
196            (Puncts(a), Puncts(b)) if a.len() == b.len() => {
197                let a_iter = a.iter().map(|a| a.char);
198                let b_iter = b.iter().map(|b| b.char);
199                a_iter.eq(b_iter)
200            }
201            (Lifetime(_, a), Lifetime(_, b)) => a.sym == b.sym,
202            _ => false,
203        }
204    }
205}
206
207#[derive(Clone, Copy)]
208enum Mode {
209    Pattern,
210    Template,
211}
212
213fn next_op(
214    edition: impl Copy + Fn(SyntaxContext) -> Edition,
215    first_peeked: TtElement<'_, Span>,
216    src: &mut TtIter<'_, Span>,
217    mode: Mode,
218) -> Result<Op, ParseError> {
219    let res = match first_peeked {
220        TtElement::Leaf(tt::Leaf::Punct(p @ tt::Punct { char: '$', .. })) => {
221            src.next().expect("first token already peeked");
222            // Note that the '$' itself is a valid token inside macro_rules.
223            let second = match src.next() {
224                None => {
225                    return Ok(Op::Punct({
226                        let mut res = ArrayVec::new();
227                        res.push(*p);
228                        Box::new(res)
229                    }));
230                }
231                Some(it) => it,
232            };
233            match second {
234                TtElement::Subtree(subtree, mut subtree_iter) => match subtree.delimiter.kind {
235                    tt::DelimiterKind::Parenthesis => {
236                        let (separator, kind) = parse_repeat(src)?;
237                        let tokens = MetaTemplate::parse(edition, subtree_iter, mode)?;
238                        Op::Repeat { tokens, separator: separator.map(Arc::new), kind }
239                    }
240                    tt::DelimiterKind::Brace => match mode {
241                        Mode::Template => parse_metavar_expr(&mut subtree_iter).map_err(|()| {
242                            ParseError::unexpected("invalid metavariable expression")
243                        })?,
244                        Mode::Pattern => {
245                            return Err(ParseError::unexpected(
246                                "`${}` metavariable expressions are not allowed in matchers",
247                            ));
248                        }
249                    },
250                    _ => {
251                        return Err(ParseError::expected(
252                            "expected `$()` repetition or `${}` expression",
253                        ));
254                    }
255                },
256                TtElement::Leaf(leaf) => match leaf {
257                    tt::Leaf::Ident(ident) if ident.sym == sym::crate_ => {
258                        // We simply produce identifier `$crate` here. And it will be resolved when lowering ast to Path.
259                        Op::Ident(tt::Ident {
260                            sym: sym::dollar_crate,
261                            span: ident.span,
262                            is_raw: tt::IdentIsRaw::No,
263                        })
264                    }
265                    tt::Leaf::Ident(ident) => {
266                        let kind = eat_fragment_kind(edition, src, mode)?;
267                        let name = ident.sym.clone();
268                        let id = ident.span;
269                        Op::Var { name, kind, id }
270                    }
271                    tt::Leaf::Literal(lit) if is_boolean_literal(lit) => {
272                        let kind = eat_fragment_kind(edition, src, mode)?;
273                        let name = lit.symbol.clone();
274                        let id = lit.span;
275                        Op::Var { name, kind, id }
276                    }
277                    tt::Leaf::Punct(punct @ tt::Punct { char: '$', .. }) => match mode {
278                        Mode::Pattern => {
279                            return Err(ParseError::unexpected(
280                                "`$$` is not allowed on the pattern side",
281                            ));
282                        }
283                        Mode::Template => Op::Punct({
284                            let mut res = ArrayVec::new();
285                            res.push(*punct);
286                            Box::new(res)
287                        }),
288                    },
289                    tt::Leaf::Punct(_) | tt::Leaf::Literal(_) => {
290                        return Err(ParseError::expected("expected ident"));
291                    }
292                },
293            }
294        }
295
296        TtElement::Leaf(tt::Leaf::Literal(it)) => {
297            src.next().expect("first token already peeked");
298            Op::Literal(it.clone())
299        }
300
301        TtElement::Leaf(tt::Leaf::Ident(it)) => {
302            src.next().expect("first token already peeked");
303            Op::Ident(it.clone())
304        }
305
306        TtElement::Leaf(tt::Leaf::Punct(_)) => {
307            // There's at least one punct so this shouldn't fail.
308            let puncts = src.expect_glued_punct().unwrap();
309            Op::Punct(Box::new(puncts))
310        }
311
312        TtElement::Subtree(subtree, subtree_iter) => {
313            src.next().expect("first token already peeked");
314            let tokens = MetaTemplate::parse(edition, subtree_iter, mode)?;
315            Op::Subtree { tokens, delimiter: subtree.delimiter }
316        }
317    };
318    Ok(res)
319}
320
321fn eat_fragment_kind(
322    edition: impl Copy + Fn(SyntaxContext) -> Edition,
323    src: &mut TtIter<'_, Span>,
324    mode: Mode,
325) -> Result<Option<MetaVarKind>, ParseError> {
326    if let Mode::Pattern = mode {
327        src.expect_char(':').map_err(|()| ParseError::unexpected("missing fragment specifier"))?;
328        let ident = src
329            .expect_ident()
330            .map_err(|()| ParseError::unexpected("missing fragment specifier"))?;
331        let kind = match ident.sym.as_str() {
332            "path" => MetaVarKind::Path,
333            "ty" => MetaVarKind::Ty,
334            "pat" => {
335                if edition(ident.span.ctx).at_least_2021() {
336                    MetaVarKind::Pat
337                } else {
338                    MetaVarKind::PatParam
339                }
340            }
341            "pat_param" => MetaVarKind::PatParam,
342            "stmt" => MetaVarKind::Stmt,
343            "block" => MetaVarKind::Block,
344            "meta" => MetaVarKind::Meta,
345            "item" => MetaVarKind::Item,
346            "vis" => MetaVarKind::Vis,
347            "expr" => {
348                if edition(ident.span.ctx).at_least_2024() {
349                    MetaVarKind::Expr(ExprKind::Expr)
350                } else {
351                    MetaVarKind::Expr(ExprKind::Expr2021)
352                }
353            }
354            "expr_2021" => MetaVarKind::Expr(ExprKind::Expr2021),
355            "ident" => MetaVarKind::Ident,
356            "tt" => MetaVarKind::Tt,
357            "lifetime" => MetaVarKind::Lifetime,
358            "literal" => MetaVarKind::Literal,
359            _ => return Ok(None),
360        };
361        return Ok(Some(kind));
362    };
363    Ok(None)
364}
365
366fn is_boolean_literal(lit: &tt::Literal<Span>) -> bool {
367    matches!(lit.symbol.as_str(), "true" | "false")
368}
369
370fn parse_repeat(src: &mut TtIter<'_, Span>) -> Result<(Option<Separator>, RepeatKind), ParseError> {
371    let mut separator = Separator::Puncts(ArrayVec::new());
372    for tt in src {
373        let tt = match tt {
374            TtElement::Leaf(leaf) => leaf,
375            TtElement::Subtree(..) => return Err(ParseError::InvalidRepeat),
376        };
377        let has_sep = match &separator {
378            Separator::Puncts(puncts) => !puncts.is_empty(),
379            _ => true,
380        };
381        match tt {
382            tt::Leaf::Ident(ident) => match separator {
383                Separator::Puncts(puncts) if puncts.is_empty() => {
384                    separator = Separator::Ident(ident.clone());
385                }
386                Separator::Puncts(puncts) => match puncts.as_slice() {
387                    [tt::Punct { char: '\'', .. }] => {
388                        separator = Separator::Lifetime(puncts[0], ident.clone());
389                    }
390                    _ => return Err(ParseError::InvalidRepeat),
391                },
392                _ => return Err(ParseError::InvalidRepeat),
393            },
394            tt::Leaf::Literal(_) if has_sep => return Err(ParseError::InvalidRepeat),
395            tt::Leaf::Literal(lit) => separator = Separator::Literal(lit.clone()),
396            tt::Leaf::Punct(punct) => {
397                let repeat_kind = match punct.char {
398                    '*' => RepeatKind::ZeroOrMore,
399                    '+' => RepeatKind::OneOrMore,
400                    '?' => RepeatKind::ZeroOrOne,
401                    _ => match &mut separator {
402                        Separator::Puncts(puncts) if puncts.len() < 3 => {
403                            puncts.push(*punct);
404                            continue;
405                        }
406                        _ => return Err(ParseError::InvalidRepeat),
407                    },
408                };
409                return Ok((has_sep.then_some(separator), repeat_kind));
410            }
411        }
412    }
413    Err(ParseError::InvalidRepeat)
414}
415
416fn parse_metavar_expr(src: &mut TtIter<'_, Span>) -> Result<Op, ()> {
417    let func = src.expect_ident()?;
418    let (args, mut args_iter) = src.expect_subtree()?;
419
420    if args.delimiter.kind != tt::DelimiterKind::Parenthesis {
421        return Err(());
422    }
423
424    let op = match &func.sym {
425        s if sym::ignore == *s => {
426            args_iter.expect_dollar()?;
427            let ident = args_iter.expect_ident()?;
428            Op::Ignore { name: ident.sym.clone(), id: ident.span }
429        }
430        s if sym::index == *s => Op::Index { depth: parse_depth(&mut args_iter)? },
431        s if sym::len == *s => Op::Len { depth: parse_depth(&mut args_iter)? },
432        s if sym::count == *s => {
433            args_iter.expect_dollar()?;
434            let ident = args_iter.expect_ident()?;
435            let depth = if try_eat_comma(&mut args_iter) {
436                Some(parse_depth(&mut args_iter)?)
437            } else {
438                None
439            };
440            Op::Count { name: ident.sym.clone(), depth }
441        }
442        s if sym::concat == *s => {
443            let mut elements = Vec::new();
444            while let Some(next) = args_iter.peek() {
445                let element = if let TtElement::Leaf(tt::Leaf::Literal(lit)) = next {
446                    args_iter.next().expect("already peeked");
447                    ConcatMetaVarExprElem::Literal(lit.clone())
448                } else {
449                    let is_var = try_eat_dollar(&mut args_iter);
450                    let ident = args_iter.expect_ident_or_underscore()?.clone();
451
452                    if is_var {
453                        ConcatMetaVarExprElem::Var(ident)
454                    } else {
455                        ConcatMetaVarExprElem::Ident(ident)
456                    }
457                };
458                elements.push(element);
459                if !args_iter.is_empty() {
460                    args_iter.expect_comma()?;
461                }
462            }
463            if elements.len() < 2 {
464                return Err(());
465            }
466            Op::Concat { elements: elements.into_boxed_slice(), span: func.span }
467        }
468        _ => return Err(()),
469    };
470
471    if args_iter.next().is_some() {
472        return Err(());
473    }
474
475    Ok(op)
476}
477
478fn parse_depth(src: &mut TtIter<'_, Span>) -> Result<usize, ()> {
479    if src.is_empty() {
480        Ok(0)
481    } else if let tt::Leaf::Literal(tt::Literal { symbol: text, suffix: None, .. }) =
482        src.expect_literal()?
483    {
484        // Suffixes are not allowed.
485        text.as_str().parse().map_err(|_| ())
486    } else {
487        Err(())
488    }
489}
490
491fn try_eat_comma(src: &mut TtIter<'_, Span>) -> bool {
492    if let Some(TtElement::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', .. }))) = src.peek() {
493        let _ = src.next();
494        return true;
495    }
496    false
497}
498
499fn try_eat_dollar(src: &mut TtIter<'_, Span>) -> bool {
500    if let Some(TtElement::Leaf(tt::Leaf::Punct(tt::Punct { char: '$', .. }))) = src.peek() {
501        let _ = src.next();
502        return true;
503    }
504    false
505}
mbe/parser.rs

mbe/
parser.rs
