parser/
lexed_str.rs

1//! Lexing `&str` into a sequence of Rust tokens.
2//!
3//! Note that strictly speaking the parser in this crate is not required to work
4//! on tokens which originated from text. Macros, eg, can synthesize tokens out
5//! of thin air. So, ideally, lexer should be an orthogonal crate. It is however
6//! convenient to include a text-based lexer here!
7//!
8//! Note that these tokens, unlike the tokens we feed into the parser, do
9//! include info about comments and whitespace.
10
11use std::ops;
12
13use rustc_literal_escaper::{
14    EscapeError, Mode, unescape_byte, unescape_byte_str, unescape_c_str, unescape_char,
15    unescape_str,
16};
17
18use crate::{
19    Edition,
20    SyntaxKind::{self, *},
21    T,
22};
23
24pub struct LexedStr<'a> {
25    text: &'a str,
26    kind: Vec<SyntaxKind>,
27    start: Vec<u32>,
28    error: Vec<LexError>,
29}
30
31struct LexError {
32    msg: String,
33    token: u32,
34}
35
36impl<'a> LexedStr<'a> {
37    pub fn new(edition: Edition, text: &'a str) -> LexedStr<'a> {
38        let _p = tracing::info_span!("LexedStr::new").entered();
39        let mut conv = Converter::new(edition, text);
40        if let Ok(script) = crate::frontmatter::ScriptSource::parse(text) {
41            if let Some(shebang) = script.shebang_span() {
42                conv.push(SHEBANG, shebang.end - shebang.start, Vec::new());
43            }
44            if script.frontmatter().is_some() {
45                conv.push(FRONTMATTER, script.content_span().start - conv.offset, Vec::new());
46            }
47        } else if let Some(shebang_len) = rustc_lexer::strip_shebang(text) {
48            // Leave error reporting to `rustc_lexer`
49            conv.push(SHEBANG, shebang_len, Vec::new());
50        }
51
52        // Re-create the tokenizer from scratch every token because `GuardedStrPrefix` is one token in the lexer
53        // but we want to split it to two in edition <2024.
54        while let Some(token) =
55            rustc_lexer::tokenize(&text[conv.offset..], rustc_lexer::FrontmatterAllowed::No).next()
56        {
57            let token_text = &text[conv.offset..][..token.len as usize];
58
59            conv.extend_token(&token.kind, token_text);
60        }
61
62        conv.finalize_with_eof()
63    }
64
65    pub fn single_token(edition: Edition, text: &'a str) -> Option<(SyntaxKind, Option<String>)> {
66        if text.is_empty() {
67            return None;
68        }
69
70        let token = rustc_lexer::tokenize(text, rustc_lexer::FrontmatterAllowed::No).next()?;
71        if token.len as usize != text.len() {
72            return None;
73        }
74
75        let mut conv = Converter::new(edition, text);
76        conv.extend_token(&token.kind, text);
77        match &*conv.res.kind {
78            [kind] => Some((*kind, conv.res.error.pop().map(|it| it.msg))),
79            _ => None,
80        }
81    }
82
83    pub fn as_str(&self) -> &str {
84        self.text
85    }
86
87    pub fn len(&self) -> usize {
88        self.kind.len() - 1
89    }
90
91    pub fn is_empty(&self) -> bool {
92        self.len() == 0
93    }
94
95    pub fn kind(&self, i: usize) -> SyntaxKind {
96        assert!(i < self.len());
97        self.kind[i]
98    }
99
100    pub fn text(&self, i: usize) -> &str {
101        self.range_text(i..i + 1)
102    }
103
104    pub fn range_text(&self, r: ops::Range<usize>) -> &str {
105        assert!(r.start < r.end && r.end <= self.len());
106        let lo = self.start[r.start] as usize;
107        let hi = self.start[r.end] as usize;
108        &self.text[lo..hi]
109    }
110
111    // Naming is hard.
112    pub fn text_range(&self, i: usize) -> ops::Range<usize> {
113        assert!(i < self.len());
114        let lo = self.start[i] as usize;
115        let hi = self.start[i + 1] as usize;
116        lo..hi
117    }
118    pub fn text_start(&self, i: usize) -> usize {
119        assert!(i <= self.len());
120        self.start[i] as usize
121    }
122    pub fn text_len(&self, i: usize) -> usize {
123        assert!(i < self.len());
124        let r = self.text_range(i);
125        r.end - r.start
126    }
127
128    pub fn error(&self, i: usize) -> Option<&str> {
129        assert!(i < self.len());
130        let err = self.error.binary_search_by_key(&(i as u32), |i| i.token).ok()?;
131        Some(self.error[err].msg.as_str())
132    }
133
134    pub fn errors(&self) -> impl Iterator<Item = (usize, &str)> + '_ {
135        self.error.iter().map(|it| (it.token as usize, it.msg.as_str()))
136    }
137
138    fn push(&mut self, kind: SyntaxKind, offset: usize) {
139        self.kind.push(kind);
140        self.start.push(offset as u32);
141    }
142}
143
144struct Converter<'a> {
145    res: LexedStr<'a>,
146    offset: usize,
147    edition: Edition,
148}
149
150impl<'a> Converter<'a> {
151    fn new(edition: Edition, text: &'a str) -> Self {
152        Self {
153            res: LexedStr { text, kind: Vec::new(), start: Vec::new(), error: Vec::new() },
154            offset: 0,
155            edition,
156        }
157    }
158
159    /// Check for likely unterminated string by analyzing STRING token content
160    fn has_likely_unterminated_string(&self) -> bool {
161        let Some(last_idx) = self.res.kind.len().checked_sub(1) else { return false };
162
163        for i in (0..=last_idx).rev().take(5) {
164            if self.res.kind[i] == STRING {
165                let start = self.res.start[i] as usize;
166                let end = self.res.start.get(i + 1).map(|&s| s as usize).unwrap_or(self.offset);
167                let content = &self.res.text[start..end];
168
169                if content.contains('(') && (content.contains("//") || content.contains(";\n")) {
170                    return true;
171                }
172            }
173        }
174        false
175    }
176
177    fn finalize_with_eof(mut self) -> LexedStr<'a> {
178        self.res.push(EOF, self.offset);
179        self.res
180    }
181
182    fn push(&mut self, kind: SyntaxKind, len: usize, errors: Vec<String>) {
183        self.res.push(kind, self.offset);
184        self.offset += len;
185
186        for msg in errors {
187            if !msg.is_empty() {
188                self.res.error.push(LexError { msg, token: self.res.len() as u32 });
189            }
190        }
191    }
192
193    fn extend_token(&mut self, kind: &rustc_lexer::TokenKind, mut token_text: &str) {
194        // A note on an intended tradeoff:
195        // We drop some useful information here (see patterns with double dots `..`)
196        // Storing that info in `SyntaxKind` is not possible due to its layout requirements of
197        // being `u16` that come from `rowan::SyntaxKind`.
198        let mut errors: Vec<String> = vec![];
199
200        let syntax_kind = {
201            match kind {
202                rustc_lexer::TokenKind::LineComment { doc_style: _ } => COMMENT,
203                rustc_lexer::TokenKind::BlockComment { doc_style: _, terminated } => {
204                    if !terminated {
205                        errors.push(
206                            "Missing trailing `*/` symbols to terminate the block comment".into(),
207                        );
208                    }
209                    COMMENT
210                }
211
212                rustc_lexer::TokenKind::Frontmatter {
213                    has_invalid_preceding_whitespace,
214                    invalid_infostring,
215                } => {
216                    if *has_invalid_preceding_whitespace {
217                        errors.push("invalid preceding whitespace for frontmatter opening".into());
218                    } else if *invalid_infostring {
219                        errors.push("invalid infostring for frontmatter".into());
220                    }
221                    FRONTMATTER
222                }
223
224                rustc_lexer::TokenKind::Whitespace => WHITESPACE,
225
226                rustc_lexer::TokenKind::Ident if token_text == "_" => UNDERSCORE,
227                rustc_lexer::TokenKind::Ident => {
228                    SyntaxKind::from_keyword(token_text, self.edition).unwrap_or(IDENT)
229                }
230                rustc_lexer::TokenKind::InvalidIdent => {
231                    errors.push("Ident contains invalid characters".into());
232                    IDENT
233                }
234
235                rustc_lexer::TokenKind::RawIdent => IDENT,
236
237                rustc_lexer::TokenKind::GuardedStrPrefix if self.edition.at_least_2024() => {
238                    // FIXME: rustc does something better for recovery.
239                    errors.push("Invalid string literal (reserved syntax)".into());
240                    ERROR
241                }
242                rustc_lexer::TokenKind::GuardedStrPrefix => {
243                    // The token is `#"` or `##`, split it into two.
244                    token_text = &token_text[1..];
245                    POUND
246                }
247
248                rustc_lexer::TokenKind::Literal { kind, .. } => {
249                    self.extend_literal(token_text.len(), kind);
250                    return;
251                }
252
253                rustc_lexer::TokenKind::Lifetime { starts_with_number } => {
254                    if *starts_with_number {
255                        errors.push("Lifetime name cannot start with a number".into());
256                    }
257                    LIFETIME_IDENT
258                }
259                rustc_lexer::TokenKind::UnknownPrefixLifetime => {
260                    errors.push("Unknown lifetime prefix".into());
261                    LIFETIME_IDENT
262                }
263                rustc_lexer::TokenKind::RawLifetime => LIFETIME_IDENT,
264
265                rustc_lexer::TokenKind::Semi => T![;],
266                rustc_lexer::TokenKind::Comma => T![,],
267                rustc_lexer::TokenKind::Dot => T![.],
268                rustc_lexer::TokenKind::OpenParen => T!['('],
269                rustc_lexer::TokenKind::CloseParen => T![')'],
270                rustc_lexer::TokenKind::OpenBrace => T!['{'],
271                rustc_lexer::TokenKind::CloseBrace => T!['}'],
272                rustc_lexer::TokenKind::OpenBracket => T!['['],
273                rustc_lexer::TokenKind::CloseBracket => T![']'],
274                rustc_lexer::TokenKind::At => T![@],
275                rustc_lexer::TokenKind::Pound => T![#],
276                rustc_lexer::TokenKind::Tilde => T![~],
277                rustc_lexer::TokenKind::Question => T![?],
278                rustc_lexer::TokenKind::Colon => T![:],
279                rustc_lexer::TokenKind::Dollar => T![$],
280                rustc_lexer::TokenKind::Eq => T![=],
281                rustc_lexer::TokenKind::Bang => T![!],
282                rustc_lexer::TokenKind::Lt => T![<],
283                rustc_lexer::TokenKind::Gt => T![>],
284                rustc_lexer::TokenKind::Minus => T![-],
285                rustc_lexer::TokenKind::And => T![&],
286                rustc_lexer::TokenKind::Or => T![|],
287                rustc_lexer::TokenKind::Plus => T![+],
288                rustc_lexer::TokenKind::Star => T![*],
289                rustc_lexer::TokenKind::Slash => T![/],
290                rustc_lexer::TokenKind::Caret => T![^],
291                rustc_lexer::TokenKind::Percent => T![%],
292                rustc_lexer::TokenKind::Unknown => ERROR,
293                rustc_lexer::TokenKind::UnknownPrefix if token_text == "builtin" => IDENT,
294                rustc_lexer::TokenKind::UnknownPrefix => {
295                    let has_unterminated = self.has_likely_unterminated_string();
296
297                    let error_msg = if has_unterminated {
298                        format!(
299                            "unknown literal prefix `{token_text}` (note: check for unterminated string literal)"
300                        )
301                    } else {
302                        "unknown literal prefix".to_owned()
303                    };
304                    errors.push(error_msg);
305                    IDENT
306                }
307                rustc_lexer::TokenKind::Eof => EOF,
308            }
309        };
310
311        self.push(syntax_kind, token_text.len(), errors);
312    }
313
314    fn extend_literal(&mut self, len: usize, kind: &rustc_lexer::LiteralKind) {
315        let invalid_raw_msg = String::from("Invalid raw string literal");
316
317        let mut errors = vec![];
318        let mut no_end_quote = |c: char, kind: &str| {
319            errors.push(format!("Missing trailing `{c}` symbol to terminate the {kind} literal"));
320        };
321
322        let syntax_kind = match *kind {
323            rustc_lexer::LiteralKind::Int { empty_int, base: _ } => {
324                if empty_int {
325                    errors.push("Missing digits after the integer base prefix".into());
326                }
327                INT_NUMBER
328            }
329            rustc_lexer::LiteralKind::Float { empty_exponent, base: _ } => {
330                if empty_exponent {
331                    errors.push("Missing digits after the exponent symbol".into());
332                }
333                FLOAT_NUMBER
334            }
335            rustc_lexer::LiteralKind::Char { terminated } => {
336                if !terminated {
337                    no_end_quote('\'', "character");
338                } else {
339                    let text = &self.res.text[self.offset + 1..][..len - 1];
340                    let text = &text[..text.rfind('\'').unwrap()];
341                    if let Err(e) = unescape_char(text) {
342                        errors.push(err_to_msg(e, Mode::Char));
343                    }
344                }
345                CHAR
346            }
347            rustc_lexer::LiteralKind::Byte { terminated } => {
348                if !terminated {
349                    no_end_quote('\'', "byte");
350                } else {
351                    let text = &self.res.text[self.offset + 2..][..len - 2];
352                    let text = &text[..text.rfind('\'').unwrap()];
353                    if let Err(e) = unescape_byte(text) {
354                        errors.push(err_to_msg(e, Mode::Byte));
355                    }
356                }
357                BYTE
358            }
359            rustc_lexer::LiteralKind::Str { terminated } => {
360                if !terminated {
361                    no_end_quote('"', "string");
362                } else {
363                    let text = &self.res.text[self.offset + 1..][..len - 1];
364                    let text = &text[..text.rfind('"').unwrap()];
365                    unescape_str(text, |_, res| {
366                        if let Err(e) = res {
367                            errors.push(err_to_msg(e, Mode::Str));
368                        }
369                    });
370                }
371                STRING
372            }
373            rustc_lexer::LiteralKind::ByteStr { terminated } => {
374                if !terminated {
375                    no_end_quote('"', "byte string");
376                } else {
377                    let text = &self.res.text[self.offset + 2..][..len - 2];
378                    let text = &text[..text.rfind('"').unwrap()];
379                    unescape_byte_str(text, |_, res| {
380                        if let Err(e) = res {
381                            errors.push(err_to_msg(e, Mode::ByteStr));
382                        }
383                    });
384                }
385                BYTE_STRING
386            }
387            rustc_lexer::LiteralKind::CStr { terminated } => {
388                if !terminated {
389                    no_end_quote('"', "C string")
390                } else {
391                    let text = &self.res.text[self.offset + 2..][..len - 2];
392                    let text = &text[..text.rfind('"').unwrap()];
393                    unescape_c_str(text, |_, res| {
394                        if let Err(e) = res {
395                            errors.push(err_to_msg(e, Mode::CStr));
396                        }
397                    });
398                }
399                C_STRING
400            }
401            rustc_lexer::LiteralKind::RawStr { n_hashes } => {
402                if n_hashes.is_none() {
403                    errors.push(invalid_raw_msg);
404                }
405                STRING
406            }
407            rustc_lexer::LiteralKind::RawByteStr { n_hashes } => {
408                if n_hashes.is_none() {
409                    errors.push(invalid_raw_msg);
410                }
411                BYTE_STRING
412            }
413            rustc_lexer::LiteralKind::RawCStr { n_hashes } => {
414                if n_hashes.is_none() {
415                    errors.push(invalid_raw_msg);
416                }
417                C_STRING
418            }
419        };
420
421        self.push(syntax_kind, len, errors);
422    }
423}
424
425fn err_to_msg(error: EscapeError, mode: Mode) -> String {
426    match error {
427        EscapeError::ZeroChars => "empty character literal",
428        EscapeError::MoreThanOneChar => "character literal may only contain one codepoint",
429        EscapeError::LoneSlash => "",
430        EscapeError::InvalidEscape if mode == Mode::Byte || mode == Mode::ByteStr => {
431            "unknown byte escape"
432        }
433        EscapeError::InvalidEscape => "unknown character escape",
434        EscapeError::BareCarriageReturn => "",
435        EscapeError::BareCarriageReturnInRawString => "",
436        EscapeError::EscapeOnlyChar if mode == Mode::Byte => "byte constant must be escaped",
437        EscapeError::EscapeOnlyChar => "character constant must be escaped",
438        EscapeError::TooShortHexEscape => "numeric character escape is too short",
439        EscapeError::InvalidCharInHexEscape => "invalid character in numeric character escape",
440        EscapeError::OutOfRangeHexEscape => "out of range hex escape",
441        EscapeError::NoBraceInUnicodeEscape => "incorrect unicode escape sequence",
442        EscapeError::InvalidCharInUnicodeEscape => "invalid character in unicode escape",
443        EscapeError::EmptyUnicodeEscape => "empty unicode escape",
444        EscapeError::UnclosedUnicodeEscape => "unterminated unicode escape",
445        EscapeError::LeadingUnderscoreUnicodeEscape => "invalid start of unicode escape",
446        EscapeError::OverlongUnicodeEscape => "overlong unicode escape",
447        EscapeError::LoneSurrogateUnicodeEscape => "invalid unicode character escape",
448        EscapeError::OutOfRangeUnicodeEscape => "invalid unicode character escape",
449        EscapeError::UnicodeEscapeInByte => "unicode escape in byte string",
450        EscapeError::NonAsciiCharInByte if mode == Mode::Byte => {
451            "non-ASCII character in byte literal"
452        }
453        EscapeError::NonAsciiCharInByte if mode == Mode::ByteStr => {
454            "non-ASCII character in byte string literal"
455        }
456        EscapeError::NonAsciiCharInByte => "non-ASCII character in raw byte string literal",
457        EscapeError::NulInCStr => "null character in C string literal",
458        EscapeError::UnskippedWhitespaceWarning => "",
459        EscapeError::MultipleSkippedLinesWarning => "",
460    }
461    .into()
462}
parser/lexed_str.rs

parser/
lexed_str.rs
