xtask/codegen/
lints.rs

1//! Generates descriptor structures for unstable features from the unstable book
2//! and lints from rustc, rustdoc, and clippy.
3#![allow(clippy::disallowed_types)]
4
5use std::{
6    collections::{HashMap, HashSet, hash_map},
7    fs,
8    path::Path,
9    str::FromStr,
10};
11
12use edition::Edition;
13use stdx::format_to;
14use xshell::{Shell, cmd};
15
16use crate::{
17    codegen::{add_preamble, ensure_file_contents, reformat},
18    project_root,
19    util::list_files,
20};
21
22const DESTINATION: &str = "crates/ide-db/src/generated/lints.rs";
23
24/// This clones rustc repo, and so is not worth to keep up-to-date on a constant basis.
25pub(crate) fn generate(check: bool) {
26    let sh = &Shell::new().unwrap();
27
28    let rust_repo = project_root().join("./target/rust");
29    if rust_repo.exists() {
30        cmd!(sh, "git -C {rust_repo} pull --rebase").run().unwrap();
31    } else {
32        cmd!(sh, "git clone --depth=1 https://github.com/rust-lang/rust {rust_repo}")
33            .run()
34            .unwrap();
35    }
36    // need submodules for Cargo to parse the workspace correctly
37    cmd!(
38        sh,
39        "git -C {rust_repo} submodule update --init --recursive --depth=1 --
40         compiler library src/tools src/doc/book"
41    )
42    .run()
43    .unwrap();
44
45    let mut contents = String::from(
46        r"
47use span::Edition;
48
49use crate::Severity;
50
51#[derive(Clone)]
52pub struct Lint {
53    pub label: &'static str,
54    pub description: &'static str,
55    pub default_severity: Severity,
56    pub warn_since: Option<Edition>,
57    pub deny_since: Option<Edition>,
58}
59
60pub struct LintGroup {
61    pub lint: Lint,
62    pub children: &'static [&'static str],
63}
64
65",
66    );
67
68    generate_lint_descriptor(sh, &mut contents);
69    contents.push('\n');
70
71    let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_owned());
72    let unstable_book = project_root().join("./target/unstable-book-gen");
73    cmd!(
74        sh,
75        "{cargo} run --manifest-path {rust_repo}/src/tools/unstable-book-gen/Cargo.toml --
76         {rust_repo}/library {rust_repo}/compiler {rust_repo}/src {unstable_book}"
77    )
78    .run()
79    .unwrap();
80    generate_feature_descriptor(&mut contents, &unstable_book.join("src"));
81    contents.push('\n');
82
83    let lints_json = project_root().join("./target/clippy_lints.json");
84    cmd!(
85        sh,
86        "curl -f https://raw.githubusercontent.com/rust-lang/rust-clippy/21fd71e3fe6eb063cfb619ecc37b1023f5283894/beta/lints.json --output {lints_json}"
87    )
88    .run()
89    .unwrap();
90    generate_descriptor_clippy(&mut contents, &lints_json);
91
92    let contents = add_preamble(crate::flags::CodegenType::LintDefinitions, reformat(contents));
93
94    let destination = project_root().join(DESTINATION);
95    ensure_file_contents(
96        crate::flags::CodegenType::LintDefinitions,
97        destination.as_path(),
98        &contents,
99        check,
100    );
101}
102
103#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
104enum Severity {
105    Allow,
106    Warn,
107    Deny,
108}
109
110impl std::fmt::Display for Severity {
111    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
112        write!(
113            f,
114            "Severity::{}",
115            match self {
116                Severity::Allow => "Allow",
117                Severity::Warn => "Warning",
118                Severity::Deny => "Error",
119            }
120        )
121    }
122}
123
124impl FromStr for Severity {
125    type Err = &'static str;
126
127    fn from_str(s: &str) -> Result<Self, Self::Err> {
128        match s {
129            "allow" => Ok(Self::Allow),
130            "warn" => Ok(Self::Warn),
131            "deny" => Ok(Self::Deny),
132            _ => Err("invalid severity"),
133        }
134    }
135}
136
137#[derive(Debug)]
138struct Lint {
139    description: String,
140    default_severity: Severity,
141    warn_since: Option<Edition>,
142    deny_since: Option<Edition>,
143}
144
145/// Parses the output of `rustdoc -Whelp` and prints `Lint` and `LintGroup` constants into `buf`.
146///
147/// As of writing, the output of `rustc -Whelp` (not rustdoc) has the following format:
148///
149/// ```text
150/// Lint checks provided by rustc:
151///
152/// name  default  meaning
153/// ----  -------  -------
154///
155/// ...
156///
157/// Lint groups provided by rustc:
158///
159/// name  sub-lints
160/// ----  ---------
161///
162/// ...
163/// ```
164///
165/// `rustdoc -Whelp` (and any other custom `rustc` driver) adds another two
166/// tables after the `rustc` ones, with a different title but the same format.
167fn generate_lint_descriptor(sh: &Shell, buf: &mut String) {
168    fn get_lints_as_text(
169        stdout: &str,
170    ) -> (
171        impl Iterator<Item = (String, &str, Severity)> + '_,
172        impl Iterator<Item = (String, Lint, impl Iterator<Item = String> + '_)> + '_,
173        impl Iterator<Item = (String, &str, Severity)> + '_,
174        impl Iterator<Item = (String, Lint, impl Iterator<Item = String> + '_)> + '_,
175    ) {
176        let lints_pat = "----  -------  -------\n";
177        let lint_groups_pat = "----  ---------\n";
178        let lints = find_and_slice(stdout, lints_pat);
179        let lint_groups = find_and_slice(lints, lint_groups_pat);
180        let lints_rustdoc = find_and_slice(lint_groups, lints_pat);
181        let lint_groups_rustdoc = find_and_slice(lints_rustdoc, lint_groups_pat);
182
183        let lints = lints.lines().take_while(|l| !l.is_empty()).map(|line| {
184            let (name, rest) = line.trim().split_once(char::is_whitespace).unwrap();
185            let (severity, description) = rest.trim().split_once(char::is_whitespace).unwrap();
186            (name.trim().replace('-', "_"), description.trim(), severity.parse().unwrap())
187        });
188        let lint_groups = lint_groups.lines().take_while(|l| !l.is_empty()).map(|line| {
189            let (name, lints) = line.trim().split_once(char::is_whitespace).unwrap();
190            let label = name.trim().replace('-', "_");
191            let lint = Lint {
192                description: format!("lint group for: {}", lints.trim()),
193                default_severity: Severity::Allow,
194                warn_since: None,
195                deny_since: None,
196            };
197            let children = lints
198                .split_ascii_whitespace()
199                .map(|s| s.trim().trim_matches(',').replace('-', "_"));
200            (label, lint, children)
201        });
202
203        let lints_rustdoc = lints_rustdoc.lines().take_while(|l| !l.is_empty()).map(|line| {
204            let (name, rest) = line.trim().split_once(char::is_whitespace).unwrap();
205            let (severity, description) = rest.trim().split_once(char::is_whitespace).unwrap();
206            (name.trim().replace('-', "_"), description.trim(), severity.parse().unwrap())
207        });
208        let lint_groups_rustdoc =
209            lint_groups_rustdoc.lines().take_while(|l| !l.is_empty()).map(|line| {
210                let (name, lints) = line.trim().split_once(char::is_whitespace).unwrap();
211                let label = name.trim().replace('-', "_");
212                let lint = Lint {
213                    description: format!("lint group for: {}", lints.trim()),
214                    default_severity: Severity::Allow,
215                    warn_since: None,
216                    deny_since: None,
217                };
218                let children = lints
219                    .split_ascii_whitespace()
220                    .map(|s| s.trim().trim_matches(',').replace('-', "_"));
221                (label, lint, children)
222            });
223
224        (lints, lint_groups, lints_rustdoc, lint_groups_rustdoc)
225    }
226
227    fn insert_lints<'a>(
228        edition: Edition,
229        lints_map: &mut HashMap<String, Lint>,
230        lint_groups_map: &mut HashMap<String, (Lint, Vec<String>)>,
231        lints: impl Iterator<Item = (String, &'a str, Severity)>,
232        lint_groups: impl Iterator<Item = (String, Lint, impl Iterator<Item = String>)>,
233    ) {
234        for (lint_name, lint_description, lint_severity) in lints {
235            let lint = lints_map.entry(lint_name).or_insert_with(|| Lint {
236                description: lint_description.to_owned(),
237                default_severity: Severity::Allow,
238                warn_since: None,
239                deny_since: None,
240            });
241            if lint_severity == Severity::Warn
242                && lint.warn_since.is_none()
243                && lint.default_severity < Severity::Warn
244            {
245                lint.warn_since = Some(edition);
246            }
247            if lint_severity == Severity::Deny
248                && lint.deny_since.is_none()
249                && lint.default_severity < Severity::Deny
250            {
251                lint.deny_since = Some(edition);
252            }
253        }
254
255        for (group_name, lint, children) in lint_groups {
256            match lint_groups_map.entry(group_name) {
257                hash_map::Entry::Vacant(entry) => {
258                    entry.insert((lint, Vec::from_iter(children)));
259                }
260                hash_map::Entry::Occupied(mut entry) => {
261                    // Overwrite, because some groups (such as edition incompatibility) are changed.
262                    *entry.get_mut() = (lint, Vec::from_iter(children));
263                }
264            }
265        }
266    }
267
268    fn get_lints(
269        sh: &Shell,
270        edition: Edition,
271        lints_map: &mut HashMap<String, Lint>,
272        lint_groups_map: &mut HashMap<String, (Lint, Vec<String>)>,
273        lints_rustdoc_map: &mut HashMap<String, Lint>,
274        lint_groups_rustdoc_map: &mut HashMap<String, (Lint, Vec<String>)>,
275    ) {
276        let edition_str = edition.to_string();
277        let stdout = cmd!(sh, "rustdoc +nightly -Whelp -Zunstable-options --edition={edition_str}")
278            .read()
279            .unwrap();
280        let (lints, lint_groups, lints_rustdoc, lint_groups_rustdoc) = get_lints_as_text(&stdout);
281
282        insert_lints(edition, lints_map, lint_groups_map, lints, lint_groups);
283        insert_lints(
284            edition,
285            lints_rustdoc_map,
286            lint_groups_rustdoc_map,
287            lints_rustdoc,
288            lint_groups_rustdoc,
289        );
290    }
291
292    let basic_lints = cmd!(sh, "rustdoc +nightly -Whelp --edition=2015").read().unwrap();
293    let (lints, lint_groups, lints_rustdoc, lint_groups_rustdoc) = get_lints_as_text(&basic_lints);
294
295    let mut lints = lints
296        .map(|(label, description, severity)| {
297            (
298                label,
299                Lint {
300                    description: description.to_owned(),
301                    default_severity: severity,
302                    warn_since: None,
303                    deny_since: None,
304                },
305            )
306        })
307        .collect::<HashMap<_, _>>();
308    let mut lint_groups = lint_groups
309        .map(|(label, lint, children)| (label, (lint, Vec::from_iter(children))))
310        .collect::<HashMap<_, _>>();
311    let mut lints_rustdoc = lints_rustdoc
312        .map(|(label, description, severity)| {
313            (
314                label,
315                Lint {
316                    description: description.to_owned(),
317                    default_severity: severity,
318                    warn_since: None,
319                    deny_since: None,
320                },
321            )
322        })
323        .collect::<HashMap<_, _>>();
324    let mut lint_groups_rustdoc = lint_groups_rustdoc
325        .map(|(label, lint, children)| (label, (lint, Vec::from_iter(children))))
326        .collect::<HashMap<_, _>>();
327
328    for edition in Edition::iter().skip(1) {
329        get_lints(
330            sh,
331            edition,
332            &mut lints,
333            &mut lint_groups,
334            &mut lints_rustdoc,
335            &mut lint_groups_rustdoc,
336        );
337    }
338
339    let mut lints = Vec::from_iter(lints);
340    lints.sort_unstable_by(|a, b| a.0.cmp(&b.0));
341    let mut lint_groups = Vec::from_iter(lint_groups);
342    lint_groups.sort_unstable_by(|a, b| a.0.cmp(&b.0));
343    let mut lints_rustdoc = Vec::from_iter(lints_rustdoc);
344    lints_rustdoc.sort_unstable_by(|a, b| a.0.cmp(&b.0));
345    let mut lint_groups_rustdoc = Vec::from_iter(lint_groups_rustdoc);
346    lint_groups_rustdoc.sort_unstable_by(|a, b| a.0.cmp(&b.0));
347
348    buf.push_str(r#"pub const DEFAULT_LINTS: &[Lint] = &["#);
349    buf.push('\n');
350
351    let mut known_lints: HashSet<&String> = HashSet::with_capacity(lints.len() + lint_groups.len());
352    for (name, lint) in &lints {
353        push_lint_completion(buf, name, lint);
354        known_lints.insert(name);
355    }
356    for (name, (group, _)) in lint_groups.iter().filter(|(name, _)| !known_lints.contains(name)) {
357        push_lint_completion(buf, name, group);
358    }
359    buf.push_str("];\n\n");
360
361    buf.push_str(r#"pub const DEFAULT_LINT_GROUPS: &[LintGroup] = &["#);
362    for (name, (lint, children)) in &lint_groups {
363        if name == "warnings" {
364            continue;
365        }
366        push_lint_group(buf, name, lint, children);
367    }
368    buf.push('\n');
369    buf.push_str("];\n");
370
371    // rustdoc
372
373    buf.push('\n');
374    buf.push_str(r#"pub const RUSTDOC_LINTS: &[Lint] = &["#);
375    buf.push('\n');
376
377    let mut known_rustdoc_lints: HashSet<&String> =
378        HashSet::with_capacity(lints_rustdoc.len() + lint_groups_rustdoc.len());
379    for (name, lint) in &lints_rustdoc {
380        push_lint_completion(buf, name, lint);
381        known_rustdoc_lints.insert(name);
382    }
383    for (name, (group, _)) in
384        lint_groups_rustdoc.iter().filter(|(name, _)| !known_rustdoc_lints.contains(name))
385    {
386        push_lint_completion(buf, name, group);
387    }
388    buf.push_str("];\n\n");
389
390    buf.push_str(r#"pub const RUSTDOC_LINT_GROUPS: &[LintGroup] = &["#);
391    for (name, (lint, children)) in &lint_groups_rustdoc {
392        push_lint_group(buf, name, lint, children);
393    }
394    buf.push('\n');
395    buf.push_str("];\n");
396}
397
398#[track_caller]
399fn find_and_slice<'a>(i: &'a str, p: &str) -> &'a str {
400    let idx = i.find(p).unwrap();
401    &i[idx + p.len()..]
402}
403
404/// Parses the unstable book `src_dir` and prints a constant with the list of
405/// unstable features into `buf`.
406///
407/// It does this by looking for all `.md` files in the `language-features` and
408/// `library-features` directories, and using the file name as the feature
409/// name, and the file contents as the feature description.
410fn generate_feature_descriptor(buf: &mut String, src_dir: &Path) {
411    let mut features = ["language-features", "library-features"]
412        .into_iter()
413        .flat_map(|it| list_files(&src_dir.join(it)))
414        // Get all `.md` files
415        .filter(|path| path.extension() == Some("md".as_ref()))
416        .map(|path| {
417            let feature_ident = path.file_stem().unwrap().to_str().unwrap().replace('-', "_");
418            let doc = fs::read_to_string(path).unwrap();
419            (feature_ident, doc)
420        })
421        .collect::<Vec<_>>();
422    features.sort_by(|(feature_ident, _), (feature_ident2, _)| feature_ident.cmp(feature_ident2));
423
424    buf.push_str(r#"pub const FEATURES: &[Lint] = &["#);
425    for (feature_ident, doc) in features.into_iter() {
426        let lint = Lint {
427            description: doc,
428            default_severity: Severity::Allow,
429            warn_since: None,
430            deny_since: None,
431        };
432        push_lint_completion(buf, &feature_ident, &lint);
433    }
434    buf.push('\n');
435    buf.push_str("];\n");
436}
437
438#[derive(Debug, Default)]
439struct ClippyLint {
440    help: String,
441    id: String,
442}
443
444fn unescape(s: &str) -> String {
445    s.replace(r#"\""#, "").replace(r#"\n"#, "\n").replace(r#"\r"#, "")
446}
447
448#[allow(clippy::print_stderr)]
449fn generate_descriptor_clippy(buf: &mut String, path: &Path) {
450    let file_content = std::fs::read_to_string(path).unwrap();
451    let mut clippy_lints: Vec<ClippyLint> = Vec::new();
452    let mut clippy_groups: std::collections::BTreeMap<String, Vec<String>> = Default::default();
453
454    for line in file_content.lines().map(str::trim) {
455        if let Some(line) = line.strip_prefix(r#""id": ""#) {
456            let clippy_lint = ClippyLint {
457                id: line.strip_suffix(r#"","#).expect("should be suffixed by comma").into(),
458                help: String::new(),
459            };
460            clippy_lints.push(clippy_lint)
461        } else if let Some(line) = line.strip_prefix(r#""group": ""#) {
462            if let Some(group) = line.strip_suffix("\",") {
463                clippy_groups
464                    .entry(group.to_owned())
465                    .or_default()
466                    .push(clippy_lints.last().unwrap().id.clone());
467            }
468        } else if let Some(line) = line.strip_prefix(r#""docs": ""#) {
469            let header = "### What it does";
470            let line = match line.find(header) {
471                Some(idx) => &line[idx + header.len()..],
472                None => {
473                    let id = &clippy_lints.last().unwrap().id;
474                    // these just don't have the common header
475                    let allowed = ["allow_attributes", "read_line_without_trim"];
476                    if allowed.contains(&id.as_str()) {
477                        line
478                    } else {
479                        eprintln!("\nunexpected clippy prefix for {id}, line={line:?}\n",);
480                        continue;
481                    }
482                }
483            };
484            // Only take the description, any more than this is a lot of additional data we would embed into the exe
485            // which seems unnecessary
486            let up_to = line.find(r#"###"#).expect("no second section found?");
487            let line = &line[..up_to];
488
489            let clippy_lint = clippy_lints.last_mut().expect("clippy lint must already exist");
490            unescape(line).trim().clone_into(&mut clippy_lint.help);
491        }
492    }
493    clippy_lints.sort_by(|lint, lint2| lint.id.cmp(&lint2.id));
494
495    buf.push_str(r#"pub const CLIPPY_LINTS: &[Lint] = &["#);
496    buf.push('\n');
497    for clippy_lint in clippy_lints.into_iter() {
498        let lint_ident = format!("clippy::{}", clippy_lint.id);
499        let lint = Lint {
500            description: clippy_lint.help,
501            // Allow clippy lints by default, not all users want them.
502            default_severity: Severity::Allow,
503            warn_since: None,
504            deny_since: None,
505        };
506        push_lint_completion(buf, &lint_ident, &lint);
507    }
508    buf.push_str("];\n");
509
510    buf.push_str(r#"pub const CLIPPY_LINT_GROUPS: &[LintGroup] = &["#);
511    for (id, children) in clippy_groups {
512        let children = children.iter().map(|id| format!("clippy::{id}")).collect::<Vec<_>>();
513        if !children.is_empty() {
514            let lint_ident = format!("clippy::{id}");
515            let description = format!("lint group for: {}", children.join(", "));
516            let lint = Lint {
517                description,
518                default_severity: Severity::Allow,
519                warn_since: None,
520                deny_since: None,
521            };
522            push_lint_group(buf, &lint_ident, &lint, &children);
523        }
524    }
525    buf.push('\n');
526    buf.push_str("];\n");
527}
528
529fn push_lint_completion(buf: &mut String, name: &str, lint: &Lint) {
530    format_to!(
531        buf,
532        r###"    Lint {{
533        label: "{}",
534        description: r##"{}"##,
535        default_severity: {},
536        warn_since: "###,
537        name,
538        lint.description,
539        lint.default_severity,
540    );
541    match lint.warn_since {
542        Some(edition) => format_to!(buf, "Some(Edition::Edition{edition})"),
543        None => buf.push_str("None"),
544    }
545    format_to!(
546        buf,
547        r###",
548        deny_since: "###
549    );
550    match lint.deny_since {
551        Some(edition) => format_to!(buf, "Some(Edition::Edition{edition})"),
552        None => buf.push_str("None"),
553    }
554    format_to!(
555        buf,
556        r###",
557    }},"###
558    );
559}
560
561fn push_lint_group(buf: &mut String, name: &str, lint: &Lint, children: &[String]) {
562    buf.push_str(
563        r###"    LintGroup {
564        lint:
565        "###,
566    );
567
568    push_lint_completion(buf, name, lint);
569
570    let children = format!(
571        "&[{}]",
572        children.iter().map(|it| format!("\"{it}\"")).collect::<Vec<_>>().join(", ")
573    );
574    format_to!(
575        buf,
576        r###"
577        children: {},
578        }},"###,
579        children,
580    );
581}
xtask/codegen/lints.rs

xtask/codegen/
lints.rs
