project_model/
build_dependencies.rs

1//! Logic to invoke `cargo` for building build-dependencies (build scripts and proc-macros) as well as
2//! executing the build scripts to fetch required dependency information (`OUT_DIR` env var, extra
3//! cfg flags, etc).
4//!
5//! In essence this just invokes `cargo` with the appropriate output format which we consume,
6//! but if enabled we will also use `RUSTC_WRAPPER` to only compile the build scripts and
7//! proc-macros and skip everything else.
8
9use std::{cell::RefCell, io, mem, process::Command};
10
11use base_db::Env;
12use cargo_metadata::{Message, camino::Utf8Path};
13use cfg::CfgAtom;
14use itertools::Itertools;
15use la_arena::ArenaMap;
16use paths::{AbsPath, AbsPathBuf, Utf8PathBuf};
17use rustc_hash::{FxHashMap, FxHashSet};
18use serde::Deserialize as _;
19use stdx::never;
20use toolchain::Tool;
21
22use crate::{
23    CargoConfig, CargoFeatures, CargoWorkspace, InvocationStrategy, ManifestPath, Package, Sysroot,
24    TargetKind, cargo_config_file::make_lockfile_copy,
25    cargo_workspace::MINIMUM_TOOLCHAIN_VERSION_SUPPORTING_LOCKFILE_PATH, utf8_stdout,
26};
27
28/// Output of the build script and proc-macro building steps for a workspace.
29#[derive(Debug, Default, Clone, PartialEq, Eq)]
30pub struct WorkspaceBuildScripts {
31    outputs: ArenaMap<Package, BuildScriptOutput>,
32    error: Option<String>,
33}
34
35#[derive(Debug, Clone, Default, PartialEq, Eq)]
36pub enum ProcMacroDylibPath {
37    Path(AbsPathBuf),
38    DylibNotFound,
39    NotProcMacro,
40    #[default]
41    NotBuilt,
42}
43
44/// Output of the build script and proc-macro building step for a concrete package.
45#[derive(Debug, Clone, Default, PartialEq, Eq)]
46pub(crate) struct BuildScriptOutput {
47    /// List of config flags defined by this package's build script.
48    pub(crate) cfgs: Vec<CfgAtom>,
49    /// List of cargo-related environment variables with their value.
50    ///
51    /// If the package has a build script which defines environment variables,
52    /// they can also be found here.
53    pub(crate) envs: Env,
54    /// Directory where a build script might place its output.
55    pub(crate) out_dir: Option<AbsPathBuf>,
56    /// Path to the proc-macro library file if this package exposes proc-macros.
57    pub(crate) proc_macro_dylib_path: ProcMacroDylibPath,
58}
59
60impl BuildScriptOutput {
61    fn is_empty(&self) -> bool {
62        self.cfgs.is_empty()
63            && self.envs.is_empty()
64            && self.out_dir.is_none()
65            && matches!(
66                self.proc_macro_dylib_path,
67                ProcMacroDylibPath::NotBuilt | ProcMacroDylibPath::NotProcMacro
68            )
69    }
70}
71
72impl WorkspaceBuildScripts {
73    /// Runs the build scripts for the given workspace
74    pub(crate) fn run_for_workspace(
75        config: &CargoConfig,
76        workspace: &CargoWorkspace,
77        progress: &dyn Fn(String),
78        sysroot: &Sysroot,
79        toolchain: Option<&semver::Version>,
80    ) -> io::Result<WorkspaceBuildScripts> {
81        let current_dir = workspace.workspace_root();
82
83        let allowed_features = workspace.workspace_features();
84        let (_guard, cmd) = Self::build_command(
85            config,
86            &allowed_features,
87            workspace.manifest_path(),
88            current_dir,
89            sysroot,
90            toolchain,
91        )?;
92        Self::run_per_ws(cmd, workspace, progress)
93    }
94
95    /// Runs the build scripts by invoking the configured command *once*.
96    /// This populates the outputs for all passed in workspaces.
97    pub(crate) fn run_once(
98        config: &CargoConfig,
99        workspaces: &[&CargoWorkspace],
100        progress: &dyn Fn(String),
101        working_directory: &AbsPathBuf,
102    ) -> io::Result<Vec<WorkspaceBuildScripts>> {
103        assert_eq!(config.invocation_strategy, InvocationStrategy::Once);
104
105        let (_guard, cmd) = Self::build_command(
106            config,
107            &Default::default(),
108            // This is not gonna be used anyways, so just construct a dummy here
109            &ManifestPath::try_from(working_directory.clone()).unwrap(),
110            working_directory,
111            &Sysroot::empty(),
112            None,
113        )?;
114        // NB: Cargo.toml could have been modified between `cargo metadata` and
115        // `cargo check`. We shouldn't assume that package ids we see here are
116        // exactly those from `config`.
117        let mut by_id = FxHashMap::default();
118        // some workspaces might depend on the same crates, so we need to duplicate the outputs
119        // to those collisions
120        let mut collisions = Vec::new();
121        let mut res: Vec<_> = workspaces
122            .iter()
123            .enumerate()
124            .map(|(idx, workspace)| {
125                let mut res = WorkspaceBuildScripts::default();
126                for package in workspace.packages() {
127                    res.outputs.insert(package, BuildScriptOutput::default());
128                    if by_id.contains_key(&workspace[package].id) {
129                        collisions.push((&workspace[package].id, idx, package));
130                    } else {
131                        by_id.insert(workspace[package].id.clone(), (package, idx));
132                    }
133                }
134                res
135            })
136            .collect();
137
138        let errors = Self::run_command(
139            cmd,
140            |package, cb| {
141                if let Some(&(package, workspace)) = by_id.get(package) {
142                    cb(&workspaces[workspace][package].name, &mut res[workspace].outputs[package]);
143                } else {
144                    never!("Received compiler message for unknown package: {}", package);
145                }
146            },
147            progress,
148        )?;
149        res.iter_mut().for_each(|it| it.error.clone_from(&errors));
150        collisions.into_iter().for_each(|(id, workspace, package)| {
151            if let Some(&(p, w)) = by_id.get(id) {
152                res[workspace].outputs[package] = res[w].outputs[p].clone();
153            }
154        });
155
156        if tracing::enabled!(tracing::Level::INFO) {
157            for (idx, workspace) in workspaces.iter().enumerate() {
158                for package in workspace.packages() {
159                    let package_build_data: &mut BuildScriptOutput = &mut res[idx].outputs[package];
160                    if !package_build_data.is_empty() {
161                        tracing::info!("{}: {package_build_data:?}", workspace[package].manifest,);
162                    }
163                }
164            }
165        }
166
167        Ok(res)
168    }
169
170    pub fn error(&self) -> Option<&str> {
171        self.error.as_deref()
172    }
173
174    pub(crate) fn get_output(&self, idx: Package) -> Option<&BuildScriptOutput> {
175        self.outputs.get(idx)
176    }
177
178    /// Assembles build script outputs for the rustc crates via `--print target-libdir`.
179    pub(crate) fn rustc_crates(
180        rustc: &CargoWorkspace,
181        current_dir: &AbsPath,
182        extra_env: &FxHashMap<String, Option<String>>,
183        sysroot: &Sysroot,
184    ) -> Self {
185        let mut bs = WorkspaceBuildScripts::default();
186        for p in rustc.packages() {
187            bs.outputs.insert(p, BuildScriptOutput::default());
188        }
189        let res = (|| {
190            let target_libdir = (|| {
191                let mut cargo_config = sysroot.tool(Tool::Cargo, current_dir, extra_env);
192                cargo_config
193                    .args(["rustc", "-Z", "unstable-options", "--print", "target-libdir"])
194                    .env("RUSTC_BOOTSTRAP", "1");
195                if let Ok(it) = utf8_stdout(&mut cargo_config) {
196                    return Ok(it);
197                }
198                let mut cmd = sysroot.tool(Tool::Rustc, current_dir, extra_env);
199                cmd.args(["--print", "target-libdir"]);
200                utf8_stdout(&mut cmd)
201            })()?;
202
203            let target_libdir = AbsPathBuf::try_from(Utf8PathBuf::from(target_libdir))
204                .map_err(|_| anyhow::format_err!("target-libdir was not an absolute path"))?;
205            tracing::info!("Loading rustc proc-macro paths from {target_libdir}");
206
207            let proc_macro_dylibs: Vec<(String, AbsPathBuf)> = std::fs::read_dir(target_libdir)?
208                .filter_map(|entry| {
209                    let dir_entry = entry.ok()?;
210                    if dir_entry.file_type().ok()?.is_file() {
211                        let path = dir_entry.path();
212                        let extension = path.extension()?;
213                        if extension == std::env::consts::DLL_EXTENSION {
214                            let name = path
215                                .file_stem()?
216                                .to_str()?
217                                .split_once('-')?
218                                .0
219                                .trim_start_matches("lib")
220                                .to_owned();
221                            let path = match Utf8PathBuf::from_path_buf(path) {
222                                Ok(path) => path,
223                                Err(path) => {
224                                    tracing::warn!(
225                                        "Proc-macro dylib path contains non-UTF8 characters: {:?}",
226                                        path.display()
227                                    );
228                                    return None;
229                                }
230                            };
231                            return match AbsPathBuf::try_from(path) {
232                                Ok(path) => Some((name, path)),
233                                Err(path) => {
234                                    tracing::error!(
235                                        "proc-macro dylib path is not absolute: {:?}",
236                                        path
237                                    );
238                                    None
239                                }
240                            };
241                        }
242                    }
243                    None
244                })
245                .collect();
246            for p in rustc.packages() {
247                let package = &rustc[p];
248                bs.outputs[p].proc_macro_dylib_path =
249                    if package.targets.iter().any(|&it| {
250                        matches!(rustc[it].kind, TargetKind::Lib { is_proc_macro: true })
251                    }) {
252                        match proc_macro_dylibs.iter().find(|(name, _)| *name == package.name) {
253                            Some((_, path)) => ProcMacroDylibPath::Path(path.clone()),
254                            _ => ProcMacroDylibPath::DylibNotFound,
255                        }
256                    } else {
257                        ProcMacroDylibPath::NotProcMacro
258                    }
259            }
260
261            if tracing::enabled!(tracing::Level::INFO) {
262                for package in rustc.packages() {
263                    let package_build_data = &bs.outputs[package];
264                    if !package_build_data.is_empty() {
265                        tracing::info!("{}: {package_build_data:?}", rustc[package].manifest,);
266                    }
267                }
268            }
269            Ok(())
270        })();
271        if let Err::<_, anyhow::Error>(e) = res {
272            bs.error = Some(e.to_string());
273        }
274        bs
275    }
276
277    fn run_per_ws(
278        cmd: Command,
279        workspace: &CargoWorkspace,
280        progress: &dyn Fn(String),
281    ) -> io::Result<WorkspaceBuildScripts> {
282        let mut res = WorkspaceBuildScripts::default();
283        let outputs = &mut res.outputs;
284        // NB: Cargo.toml could have been modified between `cargo metadata` and
285        // `cargo check`. We shouldn't assume that package ids we see here are
286        // exactly those from `config`.
287        let mut by_id: FxHashMap<String, Package> = FxHashMap::default();
288        for package in workspace.packages() {
289            outputs.insert(package, BuildScriptOutput::default());
290            by_id.insert(workspace[package].id.clone(), package);
291        }
292
293        res.error = Self::run_command(
294            cmd,
295            |package, cb| {
296                if let Some(&package) = by_id.get(package) {
297                    cb(&workspace[package].name, &mut outputs[package]);
298                } else {
299                    never!(
300                        "Received compiler message for unknown package: {}\n {}",
301                        package,
302                        by_id.keys().join(", ")
303                    );
304                }
305            },
306            progress,
307        )?;
308
309        if tracing::enabled!(tracing::Level::INFO) {
310            for package in workspace.packages() {
311                let package_build_data = &outputs[package];
312                if !package_build_data.is_empty() {
313                    tracing::info!("{}: {package_build_data:?}", workspace[package].manifest,);
314                }
315            }
316        }
317
318        Ok(res)
319    }
320
321    fn run_command(
322        cmd: Command,
323        // ideally this would be something like:
324        // with_output_for: impl FnMut(&str, dyn FnOnce(&mut BuildScriptOutput)),
325        // but owned trait objects aren't a thing
326        mut with_output_for: impl FnMut(&str, &mut dyn FnMut(&str, &mut BuildScriptOutput)),
327        progress: &dyn Fn(String),
328    ) -> io::Result<Option<String>> {
329        let errors = RefCell::new(String::new());
330        let push_err = |err: &str| {
331            let mut e = errors.borrow_mut();
332            e.push_str(err);
333            e.push('\n');
334        };
335
336        tracing::info!("Running build scripts: {:?}", cmd);
337        let output = stdx::process::spawn_with_streaming_output(
338            cmd,
339            &mut |line| {
340                // Copy-pasted from existing cargo_metadata. It seems like we
341                // should be using serde_stacker here?
342                let mut deserializer = serde_json::Deserializer::from_str(line);
343                deserializer.disable_recursion_limit();
344                let message = Message::deserialize(&mut deserializer)
345                    .unwrap_or_else(|_| Message::TextLine(line.to_owned()));
346
347                match message {
348                    Message::BuildScriptExecuted(mut message) => {
349                        with_output_for(&message.package_id.repr, &mut |name, data| {
350                            progress(format!("build script {name} run"));
351                            let cfgs = {
352                                let mut acc = Vec::new();
353                                for cfg in &message.cfgs {
354                                    match crate::parse_cfg(cfg) {
355                                        Ok(it) => acc.push(it),
356                                        Err(err) => {
357                                            push_err(&format!(
358                                                "invalid cfg from cargo-metadata: {err}"
359                                            ));
360                                            return;
361                                        }
362                                    };
363                                }
364                                acc
365                            };
366                            data.envs.extend(message.env.drain(..));
367                            // cargo_metadata crate returns default (empty) path for
368                            // older cargos, which is not absolute, so work around that.
369                            let out_dir = mem::take(&mut message.out_dir);
370                            if !out_dir.as_str().is_empty() {
371                                let out_dir = AbsPathBuf::assert(out_dir);
372                                // inject_cargo_env(package, package_build_data);
373                                data.envs.insert("OUT_DIR", out_dir.as_str());
374                                data.out_dir = Some(out_dir);
375                                data.cfgs = cfgs;
376                            }
377                        });
378                    }
379                    Message::CompilerArtifact(message) => {
380                        with_output_for(&message.package_id.repr, &mut |name, data| {
381                            progress(format!("proc-macro {name} built"));
382                            if data.proc_macro_dylib_path == ProcMacroDylibPath::NotBuilt {
383                                data.proc_macro_dylib_path = ProcMacroDylibPath::NotProcMacro;
384                            }
385                            if !matches!(data.proc_macro_dylib_path, ProcMacroDylibPath::Path(_))
386                                && message
387                                    .target
388                                    .kind
389                                    .contains(&cargo_metadata::TargetKind::ProcMacro)
390                            {
391                                data.proc_macro_dylib_path =
392                                    match message.filenames.iter().find(|file| is_dylib(file)) {
393                                        Some(filename) => {
394                                            let filename = AbsPath::assert(filename);
395                                            ProcMacroDylibPath::Path(filename.to_owned())
396                                        }
397                                        None => ProcMacroDylibPath::DylibNotFound,
398                                    };
399                            }
400                        });
401                    }
402                    Message::CompilerMessage(message) => {
403                        progress(format!("received compiler message for: {}", message.target.name));
404
405                        if let Some(diag) = message.message.rendered.as_deref() {
406                            push_err(diag);
407                        }
408                    }
409                    Message::BuildFinished(_) => {}
410                    Message::TextLine(_) => {}
411                    _ => {}
412                }
413            },
414            &mut |line| {
415                push_err(line);
416            },
417        )?;
418
419        let errors = if !output.status.success() {
420            let errors = errors.into_inner();
421            Some(if errors.is_empty() { "cargo check failed".to_owned() } else { errors })
422        } else {
423            None
424        };
425        Ok(errors)
426    }
427
428    fn build_command(
429        config: &CargoConfig,
430        allowed_features: &FxHashSet<String>,
431        manifest_path: &ManifestPath,
432        current_dir: &AbsPath,
433        sysroot: &Sysroot,
434        toolchain: Option<&semver::Version>,
435    ) -> io::Result<(Option<temp_dir::TempDir>, Command)> {
436        match config.run_build_script_command.as_deref() {
437            Some([program, args @ ..]) => {
438                let mut cmd = toolchain::command(program, current_dir, &config.extra_env);
439                cmd.args(args);
440                Ok((None, cmd))
441            }
442            _ => {
443                let mut requires_unstable_options = false;
444                let mut cmd = sysroot.tool(Tool::Cargo, current_dir, &config.extra_env);
445
446                cmd.args(["check", "--quiet", "--workspace", "--message-format=json"]);
447                cmd.args(&config.extra_args);
448
449                cmd.arg("--manifest-path");
450                cmd.arg(manifest_path);
451
452                if let Some(target_dir) = &config.target_dir {
453                    cmd.arg("--target-dir").arg(target_dir);
454                }
455
456                if let Some(target) = &config.target {
457                    cmd.args(["--target", target]);
458                }
459                let mut temp_dir_guard = None;
460                if toolchain
461                    .is_some_and(|v| *v >= MINIMUM_TOOLCHAIN_VERSION_SUPPORTING_LOCKFILE_PATH)
462                {
463                    let lockfile_path =
464                        <_ as AsRef<Utf8Path>>::as_ref(manifest_path).with_extension("lock");
465                    if let Some((temp_dir, target_lockfile)) = make_lockfile_copy(&lockfile_path) {
466                        requires_unstable_options = true;
467                        temp_dir_guard = Some(temp_dir);
468                        cmd.arg("--lockfile-path");
469                        cmd.arg(target_lockfile.as_str());
470                    }
471                }
472                match &config.features {
473                    CargoFeatures::All => {
474                        cmd.arg("--all-features");
475                    }
476                    CargoFeatures::Selected { features, no_default_features } => {
477                        if *no_default_features {
478                            cmd.arg("--no-default-features");
479                        }
480                        if !features.is_empty() {
481                            cmd.arg("--features");
482                            cmd.arg(
483                                features
484                                    .iter()
485                                    .filter(|&feat| allowed_features.contains(feat))
486                                    .join(","),
487                            );
488                        }
489                    }
490                }
491
492                if manifest_path.is_rust_manifest() {
493                    requires_unstable_options = true;
494                    cmd.arg("-Zscript");
495                }
496
497                cmd.arg("--keep-going");
498
499                // If [`--compile-time-deps` flag](https://github.com/rust-lang/cargo/issues/14434) is
500                // available in current toolchain's cargo, use it to build compile time deps only.
501                const COMP_TIME_DEPS_MIN_TOOLCHAIN_VERSION: semver::Version = semver::Version {
502                    major: 1,
503                    minor: 89,
504                    patch: 0,
505                    pre: semver::Prerelease::EMPTY,
506                    build: semver::BuildMetadata::EMPTY,
507                };
508
509                let cargo_comp_time_deps_available =
510                    toolchain.is_some_and(|v| *v >= COMP_TIME_DEPS_MIN_TOOLCHAIN_VERSION);
511
512                if cargo_comp_time_deps_available {
513                    requires_unstable_options = true;
514                    cmd.arg("--compile-time-deps");
515                    // we can pass this unconditionally, because we won't actually build the
516                    // binaries, and as such, this will succeed even on targets without libtest
517                    cmd.arg("--all-targets");
518                } else {
519                    // --all-targets includes tests, benches and examples in addition to the
520                    // default lib and bins. This is an independent concept from the --target
521                    // flag below.
522                    if config.all_targets {
523                        cmd.arg("--all-targets");
524                    }
525
526                    if config.wrap_rustc_in_build_scripts {
527                        // Setup RUSTC_WRAPPER to point to `rust-analyzer` binary itself. We use
528                        // that to compile only proc macros and build scripts during the initial
529                        // `cargo check`.
530                        // We don't need this if we are using `--compile-time-deps` flag.
531                        let myself = std::env::current_exe()?;
532                        cmd.env("RUSTC_WRAPPER", myself);
533                        cmd.env("RA_RUSTC_WRAPPER", "1");
534                    }
535                }
536                if requires_unstable_options {
537                    cmd.env("__CARGO_TEST_CHANNEL_OVERRIDE_DO_NOT_USE_THIS", "nightly");
538                    cmd.arg("-Zunstable-options");
539                }
540                Ok((temp_dir_guard, cmd))
541            }
542        }
543    }
544}
545
546// FIXME: Find a better way to know if it is a dylib.
547fn is_dylib(path: &Utf8Path) -> bool {
548    match path.extension().map(|e| e.to_owned().to_lowercase()) {
549        None => false,
550        Some(ext) => matches!(ext.as_str(), "dll" | "dylib" | "so"),
551    }
552}
project_model/build_dependencies.rs

project_model/
build_dependencies.rs
