rust_analyzer/cli/
rustc_tests.rs

1//! Run all tests in a project, similar to `cargo test`, but using the mir interpreter.
2
3use std::convert::identity;
4use std::thread::Builder;
5use std::time::{Duration, Instant};
6use std::{cell::RefCell, fs::read_to_string, panic::AssertUnwindSafe, path::PathBuf};
7
8use hir::{ChangeWithProcMacros, Crate};
9use ide::{AnalysisHost, DiagnosticCode, DiagnosticsConfig};
10use ide_db::base_db;
11use itertools::Either;
12use paths::Utf8PathBuf;
13use profile::StopWatch;
14use project_model::toolchain_info::{QueryConfig, target_data_layout};
15use project_model::{
16    CargoConfig, ManifestPath, ProjectWorkspace, ProjectWorkspaceKind, RustLibSource,
17    RustSourceWorkspaceConfig, Sysroot,
18};
19
20use load_cargo::{LoadCargoConfig, ProcMacroServerChoice, load_workspace};
21use rustc_hash::FxHashMap;
22use triomphe::Arc;
23use vfs::{AbsPathBuf, FileId};
24use walkdir::WalkDir;
25
26use crate::cli::{Result, flags, report_metric};
27
28struct Tester {
29    host: AnalysisHost,
30    root_file: FileId,
31    pass_count: u64,
32    ignore_count: u64,
33    fail_count: u64,
34    stopwatch: StopWatch,
35}
36
37fn string_to_diagnostic_code_leaky(code: &str) -> DiagnosticCode {
38    thread_local! {
39        static LEAK_STORE: RefCell<FxHashMap<String, DiagnosticCode>> = RefCell::new(FxHashMap::default());
40    }
41    LEAK_STORE.with_borrow_mut(|s| match s.get(code) {
42        Some(c) => *c,
43        None => {
44            let v = DiagnosticCode::RustcHardError(format!("E{code}").leak());
45            s.insert(code.to_owned(), v);
46            v
47        }
48    })
49}
50
51fn detect_errors_from_rustc_stderr_file(p: PathBuf) -> FxHashMap<DiagnosticCode, usize> {
52    let text = read_to_string(p).unwrap();
53    let mut result = FxHashMap::default();
54    {
55        let mut text = &*text;
56        while let Some(p) = text.find("error[E") {
57            text = &text[p + 7..];
58            let code = string_to_diagnostic_code_leaky(&text[..4]);
59            *result.entry(code).or_insert(0) += 1;
60        }
61    }
62    result
63}
64
65impl Tester {
66    fn new() -> Result<Self> {
67        let mut path = AbsPathBuf::assert_utf8(std::env::temp_dir());
68        path.push("ra-rustc-test");
69        let tmp_file = path.join("ra-rustc-test.rs");
70        std::fs::write(&tmp_file, "")?;
71        let cargo_config = CargoConfig {
72            sysroot: Some(RustLibSource::Discover),
73            all_targets: true,
74            set_test: true,
75            ..Default::default()
76        };
77
78        let mut sysroot = Sysroot::discover(tmp_file.parent().unwrap(), &cargo_config.extra_env);
79        let loaded_sysroot = sysroot.load_workspace(
80            &RustSourceWorkspaceConfig::default_cargo(),
81            false,
82            &path,
83            &Utf8PathBuf::default(),
84            &|_| (),
85        );
86        if let Some(loaded_sysroot) = loaded_sysroot {
87            sysroot.set_workspace(loaded_sysroot);
88        }
89
90        let data_layout = target_data_layout::get(
91            QueryConfig::Rustc(&sysroot, tmp_file.parent().unwrap().as_ref()),
92            None,
93            &cargo_config.extra_env,
94        );
95
96        let workspace = ProjectWorkspace {
97            kind: ProjectWorkspaceKind::DetachedFile {
98                file: ManifestPath::try_from(tmp_file).unwrap(),
99                cargo: None,
100            },
101            sysroot,
102            rustc_cfg: vec![],
103            toolchain: None,
104            target_layout: data_layout.map(Arc::from).map_err(|it| Arc::from(it.to_string())),
105            cfg_overrides: Default::default(),
106            extra_includes: vec![],
107            set_test: true,
108        };
109        let load_cargo_config = LoadCargoConfig {
110            load_out_dirs_from_check: false,
111            with_proc_macro_server: ProcMacroServerChoice::Sysroot,
112            prefill_caches: false,
113        };
114        let (db, _vfs, _proc_macro) =
115            load_workspace(workspace, &cargo_config.extra_env, &load_cargo_config)?;
116        let host = AnalysisHost::with_database(db);
117        let db = host.raw_database();
118        let krates = Crate::all(db);
119        let root_crate = krates.iter().cloned().find(|krate| krate.origin(db).is_local()).unwrap();
120        let root_file = root_crate.root_file(db);
121        Ok(Self {
122            host,
123            root_file,
124            pass_count: 0,
125            ignore_count: 0,
126            fail_count: 0,
127            stopwatch: StopWatch::start(),
128        })
129    }
130
131    fn test(&mut self, p: PathBuf) {
132        println!("{}", p.display());
133        if p.parent().unwrap().file_name().unwrap() == "auxiliary" {
134            // These are not tests
135            return;
136        }
137        if IGNORED_TESTS.iter().any(|ig| p.file_name().is_some_and(|x| x == *ig)) {
138            println!("{p:?} IGNORE");
139            self.ignore_count += 1;
140            return;
141        }
142        let stderr_path = p.with_extension("stderr");
143        let expected = if stderr_path.exists() {
144            detect_errors_from_rustc_stderr_file(stderr_path)
145        } else {
146            FxHashMap::default()
147        };
148        let text = read_to_string(&p).unwrap();
149        let mut change = ChangeWithProcMacros::default();
150        // Ignore unstable tests, since they move too fast and we do not intend to support all of them.
151        let mut ignore_test = text.contains("#![feature");
152        // Ignore test with extern crates, as this infra don't support them yet.
153        ignore_test |= text.contains("// aux-build:") || text.contains("// aux-crate:");
154        // Ignore test with extern modules similarly.
155        ignore_test |= text.contains("mod ");
156        // These should work, but they don't, and I don't know why, so ignore them.
157        ignore_test |= text.contains("extern crate proc_macro");
158        let should_have_no_error = text.contains("// check-pass")
159            || text.contains("// build-pass")
160            || text.contains("// run-pass");
161        change.change_file(self.root_file, Some(text));
162        self.host.apply_change(change);
163        let diagnostic_config = DiagnosticsConfig::test_sample();
164
165        let res = std::thread::scope(|s| {
166            let worker = Builder::new()
167                .stack_size(40 * 1024 * 1024)
168                .spawn_scoped(s, {
169                    let diagnostic_config = &diagnostic_config;
170                    let main = std::thread::current();
171                    let analysis = self.host.analysis();
172                    let root_file = self.root_file;
173                    move || {
174                        let res = std::panic::catch_unwind(AssertUnwindSafe(move || {
175                            analysis.full_diagnostics(
176                                diagnostic_config,
177                                ide::AssistResolveStrategy::None,
178                                root_file,
179                            )
180                        }));
181                        main.unpark();
182                        res
183                    }
184                })
185                .unwrap();
186
187            let timeout = Duration::from_secs(5);
188            let now = Instant::now();
189            while now.elapsed() <= timeout && !worker.is_finished() {
190                std::thread::park_timeout(timeout - now.elapsed());
191            }
192
193            if !worker.is_finished() {
194                // attempt to cancel the worker, won't work for chalk hangs unfortunately
195                self.host.request_cancellation();
196            }
197            worker.join().and_then(identity)
198        });
199        let mut actual = FxHashMap::default();
200        let panicked = match res {
201            Err(e) => Some(Either::Left(e)),
202            Ok(Ok(diags)) => {
203                for diag in diags {
204                    if !matches!(diag.code, DiagnosticCode::RustcHardError(_)) {
205                        continue;
206                    }
207                    if !should_have_no_error && !SUPPORTED_DIAGNOSTICS.contains(&diag.code) {
208                        continue;
209                    }
210                    *actual.entry(diag.code).or_insert(0) += 1;
211                }
212                None
213            }
214            Ok(Err(e)) => Some(Either::Right(e)),
215        };
216        // Ignore tests with diagnostics that we don't emit.
217        ignore_test |= expected.keys().any(|k| !SUPPORTED_DIAGNOSTICS.contains(k));
218        if ignore_test {
219            println!("{p:?} IGNORE");
220            self.ignore_count += 1;
221        } else if let Some(panic) = panicked {
222            match panic {
223                Either::Left(panic) => {
224                    if let Some(msg) = panic
225                        .downcast_ref::<String>()
226                        .map(String::as_str)
227                        .or_else(|| panic.downcast_ref::<&str>().copied())
228                    {
229                        println!("{msg:?} ")
230                    }
231                    println!("{p:?} PANIC");
232                }
233                Either::Right(_) => println!("{p:?} CANCELLED"),
234            }
235            self.fail_count += 1;
236        } else if actual == expected {
237            println!("{p:?} PASS");
238            self.pass_count += 1;
239        } else {
240            println!("{p:?} FAIL");
241            println!("actual   (r-a)   = {actual:?}");
242            println!("expected (rustc) = {expected:?}");
243            self.fail_count += 1;
244        }
245    }
246
247    fn report(&mut self) {
248        println!(
249            "Pass count = {}, Fail count = {}, Ignore count = {}",
250            self.pass_count, self.fail_count, self.ignore_count
251        );
252        println!("Testing time and memory = {}", self.stopwatch.elapsed());
253        report_metric("rustc failed tests", self.fail_count, "#");
254        report_metric("rustc testing time", self.stopwatch.elapsed().time.as_millis() as u64, "ms");
255    }
256}
257
258/// These tests break rust-analyzer (either by panicking or hanging) so we should ignore them.
259const IGNORED_TESTS: &[&str] = &[
260    "trait-with-missing-associated-type-restriction.rs", // #15646
261    "trait-with-missing-associated-type-restriction-fixable.rs", // #15646
262    "resolve-self-in-impl.rs",
263    "basic.rs", // ../rust/tests/ui/associated-type-bounds/return-type-notation/basic.rs
264    "issue-26056.rs",
265    "float-field.rs",
266    "invalid_operator_trait.rs",
267    "type-alias-impl-trait-assoc-dyn.rs",
268    "deeply-nested_closures.rs",    // exponential time
269    "hang-on-deeply-nested-dyn.rs", // exponential time
270    "dyn-rpit-and-let.rs", // unexpected free variable with depth `^1.0` with outer binder ^0
271    "issue-16098.rs",      // Huge recursion limit for macros?
272    "issue-83471.rs", // crates/hir-ty/src/builder.rs:78:9: assertion failed: self.remaining() > 0
273];
274
275const SUPPORTED_DIAGNOSTICS: &[DiagnosticCode] = &[
276    DiagnosticCode::RustcHardError("E0023"),
277    DiagnosticCode::RustcHardError("E0046"),
278    DiagnosticCode::RustcHardError("E0063"),
279    DiagnosticCode::RustcHardError("E0107"),
280    DiagnosticCode::RustcHardError("E0117"),
281    DiagnosticCode::RustcHardError("E0133"),
282    DiagnosticCode::RustcHardError("E0210"),
283    DiagnosticCode::RustcHardError("E0268"),
284    DiagnosticCode::RustcHardError("E0308"),
285    DiagnosticCode::RustcHardError("E0384"),
286    DiagnosticCode::RustcHardError("E0407"),
287    DiagnosticCode::RustcHardError("E0432"),
288    DiagnosticCode::RustcHardError("E0451"),
289    DiagnosticCode::RustcHardError("E0507"),
290    DiagnosticCode::RustcHardError("E0583"),
291    DiagnosticCode::RustcHardError("E0559"),
292    DiagnosticCode::RustcHardError("E0616"),
293    DiagnosticCode::RustcHardError("E0618"),
294    DiagnosticCode::RustcHardError("E0624"),
295    DiagnosticCode::RustcHardError("E0774"),
296    DiagnosticCode::RustcHardError("E0767"),
297    DiagnosticCode::RustcHardError("E0777"),
298];
299
300impl flags::RustcTests {
301    pub fn run(self) -> Result<()> {
302        let mut tester = Tester::new()?;
303        let walk_dir = WalkDir::new(self.rustc_repo.join("tests/ui"));
304        eprintln!("Running tests for tests/ui");
305        for i in walk_dir {
306            let i = i?;
307            let p = i.into_path();
308            if let Some(f) = &self.filter
309                && !p.as_os_str().to_string_lossy().contains(f)
310            {
311                continue;
312            }
313            if p.extension().is_none_or(|x| x != "rs") {
314                continue;
315            }
316            if let Err(e) = std::panic::catch_unwind({
317                let tester = AssertUnwindSafe(&mut tester);
318                let p = p.clone();
319                move || {
320                    let _guard = base_db::DbPanicContext::enter(p.display().to_string());
321                    { tester }.0.test(p);
322                }
323            }) {
324                std::panic::resume_unwind(e);
325            }
326        }
327        tester.report();
328        Ok(())
329    }
330}
rust_analyzer/cli/rustc_tests.rs

rust_analyzer/cli/
rustc_tests.rs
