rust_analyzer/cli/
rustc_tests.rs

Help
1//! Run all tests in a project, similar to `cargo test`, but using the mir interpreter.
2
3use std::convert::identity;
4use std::thread::Builder;
5use std::time::{Duration, Instant};
6use std::{cell::RefCell, fs::read_to_string, panic::AssertUnwindSafe, path::PathBuf};
7
8use hir::{ChangeWithProcMacros, Crate};
9use ide::{AnalysisHost, DiagnosticCode, DiagnosticsConfig};
10use itertools::Either;
11use paths::Utf8PathBuf;
12use profile::StopWatch;
13use project_model::toolchain_info::{QueryConfig, target_data_layout};
14use project_model::{
15    CargoConfig, ManifestPath, ProjectWorkspace, ProjectWorkspaceKind, RustLibSource,
16    RustSourceWorkspaceConfig, Sysroot,
17};
18
19use load_cargo::{LoadCargoConfig, ProcMacroServerChoice, load_workspace};
20use rustc_hash::FxHashMap;
21use triomphe::Arc;
22use vfs::{AbsPathBuf, FileId};
23use walkdir::WalkDir;
24
25use crate::cli::{Result, flags, report_metric};
26
27struct Tester {
28    host: AnalysisHost,
29    root_file: FileId,
30    pass_count: u64,
31    ignore_count: u64,
32    fail_count: u64,
33    stopwatch: StopWatch,
34}
35
36fn string_to_diagnostic_code_leaky(code: &str) -> DiagnosticCode {
37    thread_local! {
38        static LEAK_STORE: RefCell<FxHashMap<String, DiagnosticCode>> = RefCell::new(FxHashMap::default());
39    }
40    LEAK_STORE.with_borrow_mut(|s| match s.get(code) {
41        Some(c) => *c,
42        None => {
43            let v = DiagnosticCode::RustcHardError(format!("E{code}").leak());
44            s.insert(code.to_owned(), v);
45            v
46        }
47    })
48}
49
50fn detect_errors_from_rustc_stderr_file(p: PathBuf) -> FxHashMap<DiagnosticCode, usize> {
51    let text = read_to_string(p).unwrap();
52    let mut result = FxHashMap::default();
53    {
54        let mut text = &*text;
55        while let Some(p) = text.find("error[E") {
56            text = &text[p + 7..];
57            let code = string_to_diagnostic_code_leaky(&text[..4]);
58            *result.entry(code).or_insert(0) += 1;
59        }
60    }
61    result
62}
63
64impl Tester {
65    fn new() -> Result<Self> {
66        let mut path = std::env::temp_dir();
67        path.push("ra-rustc-test.rs");
68        let tmp_file = AbsPathBuf::try_from(Utf8PathBuf::from_path_buf(path).unwrap()).unwrap();
69        std::fs::write(&tmp_file, "")?;
70        let cargo_config = CargoConfig {
71            sysroot: Some(RustLibSource::Discover),
72            all_targets: true,
73            set_test: true,
74            ..Default::default()
75        };
76
77        let mut sysroot = Sysroot::discover(tmp_file.parent().unwrap(), &cargo_config.extra_env);
78        let loaded_sysroot = sysroot.load_workspace(&RustSourceWorkspaceConfig::default_cargo());
79        if let Some(loaded_sysroot) = loaded_sysroot {
80            sysroot.set_workspace(loaded_sysroot);
81        }
82
83        let data_layout = target_data_layout::get(
84            QueryConfig::Rustc(&sysroot, tmp_file.parent().unwrap().as_ref()),
85            None,
86            &cargo_config.extra_env,
87        );
88
89        let workspace = ProjectWorkspace {
90            kind: ProjectWorkspaceKind::DetachedFile {
91                file: ManifestPath::try_from(tmp_file).unwrap(),
92                cargo: None,
93            },
94            sysroot,
95            rustc_cfg: vec![],
96            toolchain: None,
97            target_layout: data_layout.map(Arc::from).map_err(|it| Arc::from(it.to_string())),
98            cfg_overrides: Default::default(),
99            extra_includes: vec![],
100            set_test: true,
101        };
102        let load_cargo_config = LoadCargoConfig {
103            load_out_dirs_from_check: false,
104            with_proc_macro_server: ProcMacroServerChoice::Sysroot,
105            prefill_caches: false,
106        };
107        let (db, _vfs, _proc_macro) =
108            load_workspace(workspace, &cargo_config.extra_env, &load_cargo_config)?;
109        let host = AnalysisHost::with_database(db);
110        let db = host.raw_database();
111        let krates = Crate::all(db);
112        let root_crate = krates.iter().cloned().find(|krate| krate.origin(db).is_local()).unwrap();
113        let root_file = root_crate.root_file(db);
114        Ok(Self {
115            host,
116            root_file,
117            pass_count: 0,
118            ignore_count: 0,
119            fail_count: 0,
120            stopwatch: StopWatch::start(),
121        })
122    }
123
124    fn test(&mut self, p: PathBuf) {
125        println!("{}", p.display());
126        if p.parent().unwrap().file_name().unwrap() == "auxiliary" {
127            // These are not tests
128            return;
129        }
130        if IGNORED_TESTS.iter().any(|ig| p.file_name().is_some_and(|x| x == *ig)) {
131            println!("{p:?} IGNORE");
132            self.ignore_count += 1;
133            return;
134        }
135        let stderr_path = p.with_extension("stderr");
136        let expected = if stderr_path.exists() {
137            detect_errors_from_rustc_stderr_file(stderr_path)
138        } else {
139            FxHashMap::default()
140        };
141        let text = read_to_string(&p).unwrap();
142        let mut change = ChangeWithProcMacros::default();
143        // Ignore unstable tests, since they move too fast and we do not intend to support all of them.
144        let mut ignore_test = text.contains("#![feature");
145        // Ignore test with extern crates, as this infra don't support them yet.
146        ignore_test |= text.contains("// aux-build:") || text.contains("// aux-crate:");
147        // Ignore test with extern modules similarly.
148        ignore_test |= text.contains("mod ");
149        // These should work, but they don't, and I don't know why, so ignore them.
150        ignore_test |= text.contains("extern crate proc_macro");
151        let should_have_no_error = text.contains("// check-pass")
152            || text.contains("// build-pass")
153            || text.contains("// run-pass");
154        change.change_file(self.root_file, Some(text));
155        self.host.apply_change(change);
156        let diagnostic_config = DiagnosticsConfig::test_sample();
157
158        let res = std::thread::scope(|s| {
159            let worker = Builder::new()
160                .stack_size(40 * 1024 * 1024)
161                .spawn_scoped(s, {
162                    let diagnostic_config = &diagnostic_config;
163                    let main = std::thread::current();
164                    let analysis = self.host.analysis();
165                    let root_file = self.root_file;
166                    move || {
167                        let res = std::panic::catch_unwind(AssertUnwindSafe(move || {
168                            analysis.full_diagnostics(
169                                diagnostic_config,
170                                ide::AssistResolveStrategy::None,
171                                root_file,
172                            )
173                        }));
174                        main.unpark();
175                        res
176                    }
177                })
178                .unwrap();
179
180            let timeout = Duration::from_secs(5);
181            let now = Instant::now();
182            while now.elapsed() <= timeout && !worker.is_finished() {
183                std::thread::park_timeout(timeout - now.elapsed());
184            }
185
186            if !worker.is_finished() {
187                // attempt to cancel the worker, won't work for chalk hangs unfortunately
188                self.host.request_cancellation();
189            }
190            worker.join().and_then(identity)
191        });
192        let mut actual = FxHashMap::default();
193        let panicked = match res {
194            Err(e) => Some(Either::Left(e)),
195            Ok(Ok(diags)) => {
196                for diag in diags {
197                    if !matches!(diag.code, DiagnosticCode::RustcHardError(_)) {
198                        continue;
199                    }
200                    if !should_have_no_error && !SUPPORTED_DIAGNOSTICS.contains(&diag.code) {
201                        continue;
202                    }
203                    *actual.entry(diag.code).or_insert(0) += 1;
204                }
205                None
206            }
207            Ok(Err(e)) => Some(Either::Right(e)),
208        };
209        // Ignore tests with diagnostics that we don't emit.
210        ignore_test |= expected.keys().any(|k| !SUPPORTED_DIAGNOSTICS.contains(k));
211        if ignore_test {
212            println!("{p:?} IGNORE");
213            self.ignore_count += 1;
214        } else if let Some(panic) = panicked {
215            match panic {
216                Either::Left(panic) => {
217                    if let Some(msg) = panic
218                        .downcast_ref::<String>()
219                        .map(String::as_str)
220                        .or_else(|| panic.downcast_ref::<&str>().copied())
221                    {
222                        println!("{msg:?} ")
223                    }
224                    println!("{p:?} PANIC");
225                }
226                Either::Right(_) => println!("{p:?} CANCELLED"),
227            }
228            self.fail_count += 1;
229        } else if actual == expected {
230            println!("{p:?} PASS");
231            self.pass_count += 1;
232        } else {
233            println!("{p:?} FAIL");
234            println!("actual   (r-a)   = {actual:?}");
235            println!("expected (rustc) = {expected:?}");
236            self.fail_count += 1;
237        }
238    }
239
240    fn report(&mut self) {
241        println!(
242            "Pass count = {}, Fail count = {}, Ignore count = {}",
243            self.pass_count, self.fail_count, self.ignore_count
244        );
245        println!("Testing time and memory = {}", self.stopwatch.elapsed());
246        report_metric("rustc failed tests", self.fail_count, "#");
247        report_metric("rustc testing time", self.stopwatch.elapsed().time.as_millis() as u64, "ms");
248    }
249}
250
251/// These tests break rust-analyzer (either by panicking or hanging) so we should ignore them.
252const IGNORED_TESTS: &[&str] = &[
253    "trait-with-missing-associated-type-restriction.rs", // #15646
254    "trait-with-missing-associated-type-restriction-fixable.rs", // #15646
255    "resolve-self-in-impl.rs",
256    "basic.rs", // ../rust/tests/ui/associated-type-bounds/return-type-notation/basic.rs
257    "issue-26056.rs",
258    "float-field.rs",
259    "invalid_operator_trait.rs",
260    "type-alias-impl-trait-assoc-dyn.rs",
261    "deeply-nested_closures.rs",    // exponential time
262    "hang-on-deeply-nested-dyn.rs", // exponential time
263    "dyn-rpit-and-let.rs", // unexpected free variable with depth `^1.0` with outer binder ^0
264    "issue-16098.rs",      // Huge recursion limit for macros?
265    "issue-83471.rs", // crates/hir-ty/src/builder.rs:78:9: assertion failed: self.remaining() > 0
266];
267
268const SUPPORTED_DIAGNOSTICS: &[DiagnosticCode] = &[
269    DiagnosticCode::RustcHardError("E0023"),
270    DiagnosticCode::RustcHardError("E0046"),
271    DiagnosticCode::RustcHardError("E0063"),
272    DiagnosticCode::RustcHardError("E0107"),
273    DiagnosticCode::RustcHardError("E0117"),
274    DiagnosticCode::RustcHardError("E0133"),
275    DiagnosticCode::RustcHardError("E0210"),
276    DiagnosticCode::RustcHardError("E0268"),
277    DiagnosticCode::RustcHardError("E0308"),
278    DiagnosticCode::RustcHardError("E0384"),
279    DiagnosticCode::RustcHardError("E0407"),
280    DiagnosticCode::RustcHardError("E0432"),
281    DiagnosticCode::RustcHardError("E0451"),
282    DiagnosticCode::RustcHardError("E0507"),
283    DiagnosticCode::RustcHardError("E0583"),
284    DiagnosticCode::RustcHardError("E0559"),
285    DiagnosticCode::RustcHardError("E0616"),
286    DiagnosticCode::RustcHardError("E0618"),
287    DiagnosticCode::RustcHardError("E0624"),
288    DiagnosticCode::RustcHardError("E0774"),
289    DiagnosticCode::RustcHardError("E0767"),
290    DiagnosticCode::RustcHardError("E0777"),
291];
292
293impl flags::RustcTests {
294    pub fn run(self) -> Result<()> {
295        let mut tester = Tester::new()?;
296        let walk_dir = WalkDir::new(self.rustc_repo.join("tests/ui"));
297        eprintln!("Running tests for tests/ui");
298        for i in walk_dir {
299            let i = i?;
300            let p = i.into_path();
301            if let Some(f) = &self.filter {
302                if !p.as_os_str().to_string_lossy().contains(f) {
303                    continue;
304                }
305            }
306            if p.extension().is_none_or(|x| x != "rs") {
307                continue;
308            }
309            if let Err(e) = std::panic::catch_unwind({
310                let tester = AssertUnwindSafe(&mut tester);
311                let p = p.clone();
312                move || {
313                    let _guard = stdx::panic_context::enter(p.display().to_string());
314                    { tester }.0.test(p);
315                }
316            }) {
317                std::panic::resume_unwind(e);
318            }
319        }
320        tester.report();
321        Ok(())
322    }
323}
rust_analyzer/cli/rustc_tests.rs

rust_analyzer/cli/
rustc_tests.rs
