rust_analyzer/cli/
rustc_tests.rs

1//! Run all tests in a project, similar to `cargo test`, but using the mir interpreter.
2
3use std::convert::identity;
4use std::thread::Builder;
5use std::time::{Duration, Instant};
6use std::{cell::RefCell, fs::read_to_string, panic::AssertUnwindSafe, path::PathBuf};
7
8use hir::{ChangeWithProcMacros, Crate};
9use ide::{AnalysisHost, DiagnosticCode, DiagnosticsConfig};
10use ide_db::base_db;
11use itertools::Either;
12use profile::StopWatch;
13use project_model::toolchain_info::{QueryConfig, target_data};
14use project_model::{
15    CargoConfig, ManifestPath, ProjectWorkspace, ProjectWorkspaceKind, RustLibSource,
16    RustSourceWorkspaceConfig, Sysroot,
17};
18
19use load_cargo::{LoadCargoConfig, ProcMacroServerChoice, load_workspace};
20use rustc_hash::FxHashMap;
21use vfs::{AbsPathBuf, FileId};
22use walkdir::WalkDir;
23
24use crate::cli::{Result, flags, report_metric};
25
26struct Tester {
27    host: AnalysisHost,
28    root_file: FileId,
29    pass_count: u64,
30    ignore_count: u64,
31    fail_count: u64,
32    stopwatch: StopWatch,
33}
34
35fn string_to_diagnostic_code_leaky(code: &str) -> DiagnosticCode {
36    thread_local! {
37        static LEAK_STORE: RefCell<FxHashMap<String, DiagnosticCode>> = RefCell::new(FxHashMap::default());
38    }
39    LEAK_STORE.with_borrow_mut(|s| match s.get(code) {
40        Some(c) => *c,
41        None => {
42            let v = DiagnosticCode::RustcHardError(format!("E{code}").leak());
43            s.insert(code.to_owned(), v);
44            v
45        }
46    })
47}
48
49fn detect_errors_from_rustc_stderr_file(p: PathBuf) -> FxHashMap<DiagnosticCode, usize> {
50    let text = read_to_string(p).unwrap();
51    let mut result = FxHashMap::default();
52    {
53        let mut text = &*text;
54        while let Some(p) = text.find("error[E") {
55            text = &text[p + 7..];
56            let code = string_to_diagnostic_code_leaky(&text[..4]);
57            *result.entry(code).or_insert(0) += 1;
58        }
59    }
60    result
61}
62
63impl Tester {
64    fn new() -> Result<Self> {
65        let mut path = AbsPathBuf::assert_utf8(std::env::temp_dir());
66        path.push("ra-rustc-test");
67        let tmp_file = path.join("ra-rustc-test.rs");
68        std::fs::write(&tmp_file, "")?;
69        let cargo_config = CargoConfig {
70            sysroot: Some(RustLibSource::Discover),
71            all_targets: true,
72            set_test: true,
73            ..Default::default()
74        };
75
76        let mut sysroot = Sysroot::discover(tmp_file.parent().unwrap(), &cargo_config.extra_env);
77        let loaded_sysroot =
78            sysroot.load_workspace(&RustSourceWorkspaceConfig::default_cargo(), false, &|_| ());
79        if let Some(loaded_sysroot) = loaded_sysroot {
80            sysroot.set_workspace(loaded_sysroot);
81        }
82
83        let target_data = target_data::get(
84            QueryConfig::Rustc(&sysroot, tmp_file.parent().unwrap().as_ref()),
85            None,
86            &cargo_config.extra_env,
87        );
88
89        let workspace = ProjectWorkspace {
90            kind: ProjectWorkspaceKind::DetachedFile {
91                file: ManifestPath::try_from(tmp_file).unwrap(),
92                cargo: None,
93            },
94            sysroot,
95            rustc_cfg: vec![],
96            toolchain: None,
97            target: target_data.map_err(|it| it.to_string().into()),
98            cfg_overrides: Default::default(),
99            extra_includes: vec![],
100            set_test: true,
101        };
102        let load_cargo_config = LoadCargoConfig {
103            load_out_dirs_from_check: false,
104            with_proc_macro_server: ProcMacroServerChoice::Sysroot,
105            prefill_caches: false,
106            num_worker_threads: 1,
107            proc_macro_processes: 1,
108        };
109        let (db, _vfs, _proc_macro) =
110            load_workspace(workspace, &cargo_config.extra_env, &load_cargo_config)?;
111        let host = AnalysisHost::with_database(db);
112        let db = host.raw_database();
113        let krates = Crate::all(db);
114        let root_crate = krates.iter().cloned().find(|krate| krate.origin(db).is_local()).unwrap();
115        let root_file = root_crate.root_file(db);
116        Ok(Self {
117            host,
118            root_file,
119            pass_count: 0,
120            ignore_count: 0,
121            fail_count: 0,
122            stopwatch: StopWatch::start(),
123        })
124    }
125
126    fn test(&mut self, p: PathBuf) {
127        println!("{}", p.display());
128        if p.parent().unwrap().file_name().unwrap() == "auxiliary" {
129            // These are not tests
130            return;
131        }
132        if IGNORED_TESTS.iter().any(|ig| p.file_name().is_some_and(|x| x == *ig)) {
133            println!("{p:?} IGNORE");
134            self.ignore_count += 1;
135            return;
136        }
137        let stderr_path = p.with_extension("stderr");
138        let expected = if stderr_path.exists() {
139            detect_errors_from_rustc_stderr_file(stderr_path)
140        } else {
141            FxHashMap::default()
142        };
143        let text = read_to_string(&p).unwrap();
144        let mut change = ChangeWithProcMacros::default();
145        // Ignore unstable tests, since they move too fast and we do not intend to support all of them.
146        let mut ignore_test = text.contains("#![feature");
147        // Ignore test with extern crates, as this infra don't support them yet.
148        ignore_test |= text.contains("// aux-build:") || text.contains("// aux-crate:");
149        // Ignore test with extern modules similarly.
150        ignore_test |= text.contains("mod ");
151        // These should work, but they don't, and I don't know why, so ignore them.
152        ignore_test |= text.contains("extern crate proc_macro");
153        let should_have_no_error = text.contains("// check-pass")
154            || text.contains("// build-pass")
155            || text.contains("// run-pass");
156        change.change_file(self.root_file, Some(text));
157        self.host.apply_change(change);
158        let diagnostic_config = DiagnosticsConfig::test_sample();
159
160        let res = std::thread::scope(|s| {
161            let worker = Builder::new()
162                .stack_size(40 * 1024 * 1024)
163                .spawn_scoped(s, {
164                    let diagnostic_config = &diagnostic_config;
165                    let main = std::thread::current();
166                    let analysis = self.host.analysis();
167                    let root_file = self.root_file;
168                    move || {
169                        let res = std::panic::catch_unwind(AssertUnwindSafe(move || {
170                            analysis.full_diagnostics(
171                                diagnostic_config,
172                                ide::AssistResolveStrategy::None,
173                                root_file,
174                            )
175                        }));
176                        main.unpark();
177                        res
178                    }
179                })
180                .unwrap();
181
182            let timeout = Duration::from_secs(5);
183            let now = Instant::now();
184            while now.elapsed() <= timeout && !worker.is_finished() {
185                std::thread::park_timeout(timeout - now.elapsed());
186            }
187
188            if !worker.is_finished() {
189                // attempt to cancel the worker, won't work for chalk hangs unfortunately
190                self.host.trigger_garbage_collection();
191            }
192            worker.join().and_then(identity)
193        });
194        let mut actual = FxHashMap::default();
195        let panicked = match res {
196            Err(e) => Some(Either::Left(e)),
197            Ok(Ok(diags)) => {
198                for diag in diags {
199                    if !matches!(diag.code, DiagnosticCode::RustcHardError(_)) {
200                        continue;
201                    }
202                    if !should_have_no_error && !SUPPORTED_DIAGNOSTICS.contains(&diag.code) {
203                        continue;
204                    }
205                    *actual.entry(diag.code).or_insert(0) += 1;
206                }
207                None
208            }
209            Ok(Err(e)) => Some(Either::Right(e)),
210        };
211        // Ignore tests with diagnostics that we don't emit.
212        ignore_test |= expected.keys().any(|k| !SUPPORTED_DIAGNOSTICS.contains(k));
213        if ignore_test {
214            println!("{p:?} IGNORE");
215            self.ignore_count += 1;
216        } else if let Some(panic) = panicked {
217            match panic {
218                Either::Left(panic) => {
219                    if let Some(msg) = panic
220                        .downcast_ref::<String>()
221                        .map(String::as_str)
222                        .or_else(|| panic.downcast_ref::<&str>().copied())
223                    {
224                        println!("{msg:?} ")
225                    }
226                    println!("{p:?} PANIC");
227                }
228                Either::Right(_) => println!("{p:?} CANCELLED"),
229            }
230            self.fail_count += 1;
231        } else if actual == expected {
232            println!("{p:?} PASS");
233            self.pass_count += 1;
234        } else {
235            println!("{p:?} FAIL");
236            println!("actual   (r-a)   = {actual:?}");
237            println!("expected (rustc) = {expected:?}");
238            self.fail_count += 1;
239        }
240    }
241
242    fn report(&mut self) {
243        println!(
244            "Pass count = {}, Fail count = {}, Ignore count = {}",
245            self.pass_count, self.fail_count, self.ignore_count
246        );
247        println!("Testing time and memory = {}", self.stopwatch.elapsed());
248        report_metric("rustc failed tests", self.fail_count, "#");
249        report_metric("rustc testing time", self.stopwatch.elapsed().time.as_millis() as u64, "ms");
250    }
251}
252
253/// These tests break rust-analyzer (either by panicking or hanging) so we should ignore them.
254const IGNORED_TESTS: &[&str] = &[
255    "trait-with-missing-associated-type-restriction.rs", // #15646
256    "trait-with-missing-associated-type-restriction-fixable.rs", // #15646
257    "resolve-self-in-impl.rs",
258    "basic.rs", // ../rust/tests/ui/associated-type-bounds/return-type-notation/basic.rs
259    "issue-26056.rs",
260    "float-field.rs",
261    "invalid_operator_trait.rs",
262    "type-alias-impl-trait-assoc-dyn.rs",
263    "deeply-nested_closures.rs",    // exponential time
264    "hang-on-deeply-nested-dyn.rs", // exponential time
265    "dyn-rpit-and-let.rs", // unexpected free variable with depth `^1.0` with outer binder ^0
266    "issue-16098.rs",      // Huge recursion limit for macros?
267    "issue-83471.rs", // crates/hir-ty/src/builder.rs:78:9: assertion failed: self.remaining() > 0
268];
269
270const SUPPORTED_DIAGNOSTICS: &[DiagnosticCode] = &[
271    DiagnosticCode::RustcHardError("E0023"),
272    DiagnosticCode::RustcHardError("E0046"),
273    DiagnosticCode::RustcHardError("E0063"),
274    DiagnosticCode::RustcHardError("E0107"),
275    DiagnosticCode::RustcHardError("E0117"),
276    DiagnosticCode::RustcHardError("E0133"),
277    DiagnosticCode::RustcHardError("E0210"),
278    DiagnosticCode::RustcHardError("E0268"),
279    DiagnosticCode::RustcHardError("E0308"),
280    DiagnosticCode::RustcHardError("E0384"),
281    DiagnosticCode::RustcHardError("E0407"),
282    DiagnosticCode::RustcHardError("E0432"),
283    DiagnosticCode::RustcHardError("E0451"),
284    DiagnosticCode::RustcHardError("E0507"),
285    DiagnosticCode::RustcHardError("E0583"),
286    DiagnosticCode::RustcHardError("E0559"),
287    DiagnosticCode::RustcHardError("E0616"),
288    DiagnosticCode::RustcHardError("E0618"),
289    DiagnosticCode::RustcHardError("E0624"),
290    DiagnosticCode::RustcHardError("E0774"),
291    DiagnosticCode::RustcHardError("E0767"),
292    DiagnosticCode::RustcHardError("E0777"),
293];
294
295impl flags::RustcTests {
296    pub fn run(self) -> Result<()> {
297        let mut tester = Tester::new()?;
298        let walk_dir = WalkDir::new(self.rustc_repo.join("tests/ui"));
299        eprintln!("Running tests for tests/ui");
300        for i in walk_dir {
301            let i = i?;
302            let p = i.into_path();
303            if let Some(f) = &self.filter
304                && !p.as_os_str().to_string_lossy().contains(f)
305            {
306                continue;
307            }
308            if p.extension().is_none_or(|x| x != "rs") {
309                continue;
310            }
311            if let Err(e) = std::panic::catch_unwind({
312                let tester = AssertUnwindSafe(&mut tester);
313                let p = p.clone();
314                move || {
315                    let _guard = base_db::DbPanicContext::enter(p.display().to_string());
316                    { tester }.0.test(p);
317                }
318            }) {
319                std::panic::resume_unwind(e);
320            }
321        }
322        tester.report();
323        Ok(())
324    }
325}
rust_analyzer/cli/rustc_tests.rs

rust_analyzer/cli/
rustc_tests.rs
