xtask/codegen/
lints.rs

1//! Generates descriptor structures for unstable features from the unstable book
2//! and lints from rustc, rustdoc, and clippy.
3
4use std::{
5    collections::{HashMap, HashSet, hash_map},
6    fs,
7    path::Path,
8    str::FromStr,
9};
10
11use edition::Edition;
12use stdx::format_to;
13use xshell::{Shell, cmd};
14
15use crate::{
16    codegen::{add_preamble, ensure_file_contents, reformat},
17    project_root,
18    util::list_files,
19};
20
21const DESTINATION: &str = "crates/ide-db/src/generated/lints.rs";
22
23/// This clones rustc repo, and so is not worth to keep up-to-date on a constant basis.
24pub(crate) fn generate(check: bool) {
25    let sh = &Shell::new().unwrap();
26
27    let rust_repo = project_root().join("./target/rust");
28    if rust_repo.exists() {
29        cmd!(sh, "git -C {rust_repo} pull --rebase").run().unwrap();
30    } else {
31        cmd!(sh, "git clone --depth=1 https://github.com/rust-lang/rust {rust_repo}")
32            .run()
33            .unwrap();
34    }
35    // need submodules for Cargo to parse the workspace correctly
36    cmd!(
37        sh,
38        "git -C {rust_repo} submodule update --init --recursive --depth=1 --
39         compiler library src/tools src/doc/book"
40    )
41    .run()
42    .unwrap();
43
44    let mut contents = String::from(
45        r"
46use span::Edition;
47
48use crate::Severity;
49
50#[derive(Clone)]
51pub struct Lint {
52    pub label: &'static str,
53    pub description: &'static str,
54    pub default_severity: Severity,
55    pub warn_since: Option<Edition>,
56    pub deny_since: Option<Edition>,
57}
58
59pub struct LintGroup {
60    pub lint: Lint,
61    pub children: &'static [&'static str],
62}
63
64",
65    );
66
67    generate_lint_descriptor(sh, &mut contents);
68    contents.push('\n');
69
70    let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_owned());
71    let unstable_book = project_root().join("./target/unstable-book-gen");
72    cmd!(
73        sh,
74        "{cargo} run --manifest-path {rust_repo}/src/tools/unstable-book-gen/Cargo.toml --
75         {rust_repo}/library {rust_repo}/compiler {rust_repo}/src {unstable_book}"
76    )
77    .run()
78    .unwrap();
79    generate_feature_descriptor(&mut contents, &unstable_book.join("src"));
80    contents.push('\n');
81
82    let lints_json = project_root().join("./target/clippy_lints.json");
83    cmd!(
84        sh,
85        "curl -f https://raw.githubusercontent.com/rust-lang/rust-clippy/21fd71e3fe6eb063cfb619ecc37b1023f5283894/beta/lints.json --output {lints_json}"
86    )
87    .run()
88    .unwrap();
89    generate_descriptor_clippy(&mut contents, &lints_json);
90
91    let contents = add_preamble(crate::flags::CodegenType::LintDefinitions, reformat(contents));
92
93    let destination = project_root().join(DESTINATION);
94    ensure_file_contents(
95        crate::flags::CodegenType::LintDefinitions,
96        destination.as_path(),
97        &contents,
98        check,
99    );
100}
101
102#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
103enum Severity {
104    Allow,
105    Warn,
106    Deny,
107}
108
109impl std::fmt::Display for Severity {
110    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
111        write!(
112            f,
113            "Severity::{}",
114            match self {
115                Severity::Allow => "Allow",
116                Severity::Warn => "Warning",
117                Severity::Deny => "Error",
118            }
119        )
120    }
121}
122
123impl FromStr for Severity {
124    type Err = &'static str;
125
126    fn from_str(s: &str) -> Result<Self, Self::Err> {
127        match s {
128            "allow" => Ok(Self::Allow),
129            "warn" => Ok(Self::Warn),
130            "deny" => Ok(Self::Deny),
131            _ => Err("invalid severity"),
132        }
133    }
134}
135
136#[derive(Debug)]
137struct Lint {
138    description: String,
139    default_severity: Severity,
140    warn_since: Option<Edition>,
141    deny_since: Option<Edition>,
142}
143
144/// Parses the output of `rustdoc -Whelp` and prints `Lint` and `LintGroup` constants into `buf`.
145///
146/// As of writing, the output of `rustc -Whelp` (not rustdoc) has the following format:
147///
148/// ```text
149/// Lint checks provided by rustc:
150///
151/// name  default  meaning
152/// ----  -------  -------
153///
154/// ...
155///
156/// Lint groups provided by rustc:
157///
158/// name  sub-lints
159/// ----  ---------
160///
161/// ...
162/// ```
163///
164/// `rustdoc -Whelp` (and any other custom `rustc` driver) adds another two
165/// tables after the `rustc` ones, with a different title but the same format.
166fn generate_lint_descriptor(sh: &Shell, buf: &mut String) {
167    fn get_lints_as_text(
168        stdout: &str,
169    ) -> (
170        impl Iterator<Item = (String, &str, Severity)> + '_,
171        impl Iterator<Item = (String, Lint, impl Iterator<Item = String> + '_)> + '_,
172        impl Iterator<Item = (String, &str, Severity)> + '_,
173        impl Iterator<Item = (String, Lint, impl Iterator<Item = String> + '_)> + '_,
174    ) {
175        let lints_pat = "----  -------  -------\n";
176        let lint_groups_pat = "----  ---------\n";
177        let lints = find_and_slice(stdout, lints_pat);
178        let lint_groups = find_and_slice(lints, lint_groups_pat);
179        let lints_rustdoc = find_and_slice(lint_groups, lints_pat);
180        let lint_groups_rustdoc = find_and_slice(lints_rustdoc, lint_groups_pat);
181
182        let lints = lints.lines().take_while(|l| !l.is_empty()).map(|line| {
183            let (name, rest) = line.trim().split_once(char::is_whitespace).unwrap();
184            let (severity, description) = rest.trim().split_once(char::is_whitespace).unwrap();
185            (name.trim().replace('-', "_"), description.trim(), severity.parse().unwrap())
186        });
187        let lint_groups = lint_groups.lines().take_while(|l| !l.is_empty()).map(|line| {
188            let (name, lints) = line.trim().split_once(char::is_whitespace).unwrap();
189            let label = name.trim().replace('-', "_");
190            let lint = Lint {
191                description: format!("lint group for: {}", lints.trim()),
192                default_severity: Severity::Allow,
193                warn_since: None,
194                deny_since: None,
195            };
196            let children = lints
197                .split_ascii_whitespace()
198                .map(|s| s.trim().trim_matches(',').replace('-', "_"));
199            (label, lint, children)
200        });
201
202        let lints_rustdoc = lints_rustdoc.lines().take_while(|l| !l.is_empty()).map(|line| {
203            let (name, rest) = line.trim().split_once(char::is_whitespace).unwrap();
204            let (severity, description) = rest.trim().split_once(char::is_whitespace).unwrap();
205            (name.trim().replace('-', "_"), description.trim(), severity.parse().unwrap())
206        });
207        let lint_groups_rustdoc =
208            lint_groups_rustdoc.lines().take_while(|l| !l.is_empty()).map(|line| {
209                let (name, lints) = line.trim().split_once(char::is_whitespace).unwrap();
210                let label = name.trim().replace('-', "_");
211                let lint = Lint {
212                    description: format!("lint group for: {}", lints.trim()),
213                    default_severity: Severity::Allow,
214                    warn_since: None,
215                    deny_since: None,
216                };
217                let children = lints
218                    .split_ascii_whitespace()
219                    .map(|s| s.trim().trim_matches(',').replace('-', "_"));
220                (label, lint, children)
221            });
222
223        (lints, lint_groups, lints_rustdoc, lint_groups_rustdoc)
224    }
225
226    fn insert_lints<'a>(
227        edition: Edition,
228        lints_map: &mut HashMap<String, Lint>,
229        lint_groups_map: &mut HashMap<String, (Lint, Vec<String>)>,
230        lints: impl Iterator<Item = (String, &'a str, Severity)>,
231        lint_groups: impl Iterator<Item = (String, Lint, impl Iterator<Item = String>)>,
232    ) {
233        for (lint_name, lint_description, lint_severity) in lints {
234            let lint = lints_map.entry(lint_name).or_insert_with(|| Lint {
235                description: lint_description.to_owned(),
236                default_severity: Severity::Allow,
237                warn_since: None,
238                deny_since: None,
239            });
240            if lint_severity == Severity::Warn
241                && lint.warn_since.is_none()
242                && lint.default_severity < Severity::Warn
243            {
244                lint.warn_since = Some(edition);
245            }
246            if lint_severity == Severity::Deny
247                && lint.deny_since.is_none()
248                && lint.default_severity < Severity::Deny
249            {
250                lint.deny_since = Some(edition);
251            }
252        }
253
254        for (group_name, lint, children) in lint_groups {
255            match lint_groups_map.entry(group_name) {
256                hash_map::Entry::Vacant(entry) => {
257                    entry.insert((lint, Vec::from_iter(children)));
258                }
259                hash_map::Entry::Occupied(mut entry) => {
260                    // Overwrite, because some groups (such as edition incompatibility) are changed.
261                    *entry.get_mut() = (lint, Vec::from_iter(children));
262                }
263            }
264        }
265    }
266
267    fn get_lints(
268        sh: &Shell,
269        edition: Edition,
270        lints_map: &mut HashMap<String, Lint>,
271        lint_groups_map: &mut HashMap<String, (Lint, Vec<String>)>,
272        lints_rustdoc_map: &mut HashMap<String, Lint>,
273        lint_groups_rustdoc_map: &mut HashMap<String, (Lint, Vec<String>)>,
274    ) {
275        let edition_str = edition.to_string();
276        let stdout = cmd!(sh, "rustdoc +nightly -Whelp -Zunstable-options --edition={edition_str}")
277            .read()
278            .unwrap();
279        let (lints, lint_groups, lints_rustdoc, lint_groups_rustdoc) = get_lints_as_text(&stdout);
280
281        insert_lints(edition, lints_map, lint_groups_map, lints, lint_groups);
282        insert_lints(
283            edition,
284            lints_rustdoc_map,
285            lint_groups_rustdoc_map,
286            lints_rustdoc,
287            lint_groups_rustdoc,
288        );
289    }
290
291    let basic_lints = cmd!(sh, "rustdoc +nightly -Whelp --edition=2015").read().unwrap();
292    let (lints, lint_groups, lints_rustdoc, lint_groups_rustdoc) = get_lints_as_text(&basic_lints);
293
294    let mut lints = lints
295        .map(|(label, description, severity)| {
296            (
297                label,
298                Lint {
299                    description: description.to_owned(),
300                    default_severity: severity,
301                    warn_since: None,
302                    deny_since: None,
303                },
304            )
305        })
306        .collect::<HashMap<_, _>>();
307    let mut lint_groups = lint_groups
308        .map(|(label, lint, children)| (label, (lint, Vec::from_iter(children))))
309        .collect::<HashMap<_, _>>();
310    let mut lints_rustdoc = lints_rustdoc
311        .map(|(label, description, severity)| {
312            (
313                label,
314                Lint {
315                    description: description.to_owned(),
316                    default_severity: severity,
317                    warn_since: None,
318                    deny_since: None,
319                },
320            )
321        })
322        .collect::<HashMap<_, _>>();
323    let mut lint_groups_rustdoc = lint_groups_rustdoc
324        .map(|(label, lint, children)| (label, (lint, Vec::from_iter(children))))
325        .collect::<HashMap<_, _>>();
326
327    for edition in Edition::iter().skip(1) {
328        get_lints(
329            sh,
330            edition,
331            &mut lints,
332            &mut lint_groups,
333            &mut lints_rustdoc,
334            &mut lint_groups_rustdoc,
335        );
336    }
337
338    let mut lints = Vec::from_iter(lints);
339    lints.sort_unstable_by(|a, b| a.0.cmp(&b.0));
340    let mut lint_groups = Vec::from_iter(lint_groups);
341    lint_groups.sort_unstable_by(|a, b| a.0.cmp(&b.0));
342    let mut lints_rustdoc = Vec::from_iter(lints_rustdoc);
343    lints_rustdoc.sort_unstable_by(|a, b| a.0.cmp(&b.0));
344    let mut lint_groups_rustdoc = Vec::from_iter(lint_groups_rustdoc);
345    lint_groups_rustdoc.sort_unstable_by(|a, b| a.0.cmp(&b.0));
346
347    buf.push_str(r#"pub const DEFAULT_LINTS: &[Lint] = &["#);
348    buf.push('\n');
349
350    let mut known_lints: HashSet<&String> = HashSet::with_capacity(lints.len() + lint_groups.len());
351    for (name, lint) in &lints {
352        push_lint_completion(buf, name, lint);
353        known_lints.insert(name);
354    }
355    for (name, (group, _)) in lint_groups.iter().filter(|(name, _)| !known_lints.contains(name)) {
356        push_lint_completion(buf, name, group);
357    }
358    buf.push_str("];\n\n");
359
360    buf.push_str(r#"pub const DEFAULT_LINT_GROUPS: &[LintGroup] = &["#);
361    for (name, (lint, children)) in &lint_groups {
362        if name == "warnings" {
363            continue;
364        }
365        push_lint_group(buf, name, lint, children);
366    }
367    buf.push('\n');
368    buf.push_str("];\n");
369
370    // rustdoc
371
372    buf.push('\n');
373    buf.push_str(r#"pub const RUSTDOC_LINTS: &[Lint] = &["#);
374    buf.push('\n');
375
376    let mut known_rustdoc_lints: HashSet<&String> =
377        HashSet::with_capacity(lints_rustdoc.len() + lint_groups_rustdoc.len());
378    for (name, lint) in &lints_rustdoc {
379        push_lint_completion(buf, name, lint);
380        known_rustdoc_lints.insert(name);
381    }
382    for (name, (group, _)) in
383        lint_groups_rustdoc.iter().filter(|(name, _)| !known_rustdoc_lints.contains(name))
384    {
385        push_lint_completion(buf, name, group);
386    }
387    buf.push_str("];\n\n");
388
389    buf.push_str(r#"pub const RUSTDOC_LINT_GROUPS: &[LintGroup] = &["#);
390    for (name, (lint, children)) in &lint_groups_rustdoc {
391        push_lint_group(buf, name, lint, children);
392    }
393    buf.push('\n');
394    buf.push_str("];\n");
395}
396
397#[track_caller]
398fn find_and_slice<'a>(i: &'a str, p: &str) -> &'a str {
399    let idx = i.find(p).unwrap();
400    &i[idx + p.len()..]
401}
402
403/// Parses the unstable book `src_dir` and prints a constant with the list of
404/// unstable features into `buf`.
405///
406/// It does this by looking for all `.md` files in the `language-features` and
407/// `library-features` directories, and using the file name as the feature
408/// name, and the file contents as the feature description.
409fn generate_feature_descriptor(buf: &mut String, src_dir: &Path) {
410    let mut features = ["language-features", "library-features"]
411        .into_iter()
412        .flat_map(|it| list_files(&src_dir.join(it)))
413        // Get all `.md` files
414        .filter(|path| path.extension() == Some("md".as_ref()))
415        .map(|path| {
416            let feature_ident = path.file_stem().unwrap().to_str().unwrap().replace('-', "_");
417            let doc = fs::read_to_string(path).unwrap();
418            (feature_ident, doc)
419        })
420        .collect::<Vec<_>>();
421    features.sort_by(|(feature_ident, _), (feature_ident2, _)| feature_ident.cmp(feature_ident2));
422
423    buf.push_str(r#"pub const FEATURES: &[Lint] = &["#);
424    for (feature_ident, doc) in features.into_iter() {
425        let lint = Lint {
426            description: doc,
427            default_severity: Severity::Allow,
428            warn_since: None,
429            deny_since: None,
430        };
431        push_lint_completion(buf, &feature_ident, &lint);
432    }
433    buf.push('\n');
434    buf.push_str("];\n");
435}
436
437#[derive(Debug, Default)]
438struct ClippyLint {
439    help: String,
440    id: String,
441}
442
443fn unescape(s: &str) -> String {
444    s.replace(r#"\""#, "").replace(r#"\n"#, "\n").replace(r#"\r"#, "")
445}
446
447#[allow(clippy::print_stderr)]
448fn generate_descriptor_clippy(buf: &mut String, path: &Path) {
449    let file_content = std::fs::read_to_string(path).unwrap();
450    let mut clippy_lints: Vec<ClippyLint> = Vec::new();
451    let mut clippy_groups: std::collections::BTreeMap<String, Vec<String>> = Default::default();
452
453    for line in file_content.lines().map(str::trim) {
454        if let Some(line) = line.strip_prefix(r#""id": ""#) {
455            let clippy_lint = ClippyLint {
456                id: line.strip_suffix(r#"","#).expect("should be suffixed by comma").into(),
457                help: String::new(),
458            };
459            clippy_lints.push(clippy_lint)
460        } else if let Some(line) = line.strip_prefix(r#""group": ""#) {
461            if let Some(group) = line.strip_suffix("\",") {
462                clippy_groups
463                    .entry(group.to_owned())
464                    .or_default()
465                    .push(clippy_lints.last().unwrap().id.clone());
466            }
467        } else if let Some(line) = line.strip_prefix(r#""docs": ""#) {
468            let header = "### What it does";
469            let line = match line.find(header) {
470                Some(idx) => &line[idx + header.len()..],
471                None => {
472                    let id = &clippy_lints.last().unwrap().id;
473                    // these just don't have the common header
474                    let allowed = ["allow_attributes", "read_line_without_trim"];
475                    if allowed.contains(&id.as_str()) {
476                        line
477                    } else {
478                        eprintln!("\nunexpected clippy prefix for {id}, line={line:?}\n",);
479                        continue;
480                    }
481                }
482            };
483            // Only take the description, any more than this is a lot of additional data we would embed into the exe
484            // which seems unnecessary
485            let up_to = line.find(r#"###"#).expect("no second section found?");
486            let line = &line[..up_to];
487
488            let clippy_lint = clippy_lints.last_mut().expect("clippy lint must already exist");
489            unescape(line).trim().clone_into(&mut clippy_lint.help);
490        }
491    }
492    clippy_lints.sort_by(|lint, lint2| lint.id.cmp(&lint2.id));
493
494    buf.push_str(r#"pub const CLIPPY_LINTS: &[Lint] = &["#);
495    buf.push('\n');
496    for clippy_lint in clippy_lints.into_iter() {
497        let lint_ident = format!("clippy::{}", clippy_lint.id);
498        let lint = Lint {
499            description: clippy_lint.help,
500            // Allow clippy lints by default, not all users want them.
501            default_severity: Severity::Allow,
502            warn_since: None,
503            deny_since: None,
504        };
505        push_lint_completion(buf, &lint_ident, &lint);
506    }
507    buf.push_str("];\n");
508
509    buf.push_str(r#"pub const CLIPPY_LINT_GROUPS: &[LintGroup] = &["#);
510    for (id, children) in clippy_groups {
511        let children = children.iter().map(|id| format!("clippy::{id}")).collect::<Vec<_>>();
512        if !children.is_empty() {
513            let lint_ident = format!("clippy::{id}");
514            let description = format!("lint group for: {}", children.join(", "));
515            let lint = Lint {
516                description,
517                default_severity: Severity::Allow,
518                warn_since: None,
519                deny_since: None,
520            };
521            push_lint_group(buf, &lint_ident, &lint, &children);
522        }
523    }
524    buf.push('\n');
525    buf.push_str("];\n");
526}
527
528fn push_lint_completion(buf: &mut String, name: &str, lint: &Lint) {
529    format_to!(
530        buf,
531        r###"    Lint {{
532        label: "{}",
533        description: r##"{}"##,
534        default_severity: {},
535        warn_since: "###,
536        name,
537        lint.description,
538        lint.default_severity,
539    );
540    match lint.warn_since {
541        Some(edition) => format_to!(buf, "Some(Edition::Edition{edition})"),
542        None => buf.push_str("None"),
543    }
544    format_to!(
545        buf,
546        r###",
547        deny_since: "###
548    );
549    match lint.deny_since {
550        Some(edition) => format_to!(buf, "Some(Edition::Edition{edition})"),
551        None => buf.push_str("None"),
552    }
553    format_to!(
554        buf,
555        r###",
556    }},"###
557    );
558}
559
560fn push_lint_group(buf: &mut String, name: &str, lint: &Lint, children: &[String]) {
561    buf.push_str(
562        r###"    LintGroup {
563        lint:
564        "###,
565    );
566
567    push_lint_completion(buf, name, lint);
568
569    let children = format!(
570        "&[{}]",
571        children.iter().map(|it| format!("\"{it}\"")).collect::<Vec<_>>().join(", ")
572    );
573    format_to!(
574        buf,
575        r###"
576        children: {},
577        }},"###,
578        children,
579    );
580}
xtask/codegen/lints.rs

xtask/codegen/
lints.rs
