project_model/
build_dependencies.rs

1//! Logic to invoke `cargo` for building build-dependencies (build scripts and proc-macros) as well as
2//! executing the build scripts to fetch required dependency information (`OUT_DIR` env var, extra
3//! cfg flags, etc).
4//!
5//! In essence this just invokes `cargo` with the appropriate output format which we consume,
6//! but if enabled we will also use `RUSTC_WRAPPER` to only compile the build scripts and
7//! proc-macros and skip everything else.
8
9use std::{cell::RefCell, io, mem, process::Command};
10
11use base_db::Env;
12use cargo_metadata::{Message, PackageId, camino::Utf8Path};
13use cfg::CfgAtom;
14use itertools::Itertools;
15use la_arena::ArenaMap;
16use paths::{AbsPath, AbsPathBuf, Utf8PathBuf};
17use rustc_hash::{FxHashMap, FxHashSet};
18use serde::Deserialize as _;
19use stdx::never;
20use toolchain::Tool;
21use triomphe::Arc;
22
23use crate::{
24    CargoConfig, CargoFeatures, CargoWorkspace, InvocationStrategy, ManifestPath, Package, Sysroot,
25    TargetKind, cargo_config_file::make_lockfile_copy,
26    cargo_workspace::MINIMUM_TOOLCHAIN_VERSION_SUPPORTING_LOCKFILE_PATH, utf8_stdout,
27};
28
29/// Output of the build script and proc-macro building steps for a workspace.
30#[derive(Debug, Default, Clone, PartialEq, Eq)]
31pub struct WorkspaceBuildScripts {
32    outputs: ArenaMap<Package, BuildScriptOutput>,
33    error: Option<String>,
34}
35
36#[derive(Debug, Clone, Default, PartialEq, Eq)]
37pub enum ProcMacroDylibPath {
38    Path(AbsPathBuf),
39    DylibNotFound,
40    NotProcMacro,
41    #[default]
42    NotBuilt,
43}
44
45/// Output of the build script and proc-macro building step for a concrete package.
46#[derive(Debug, Clone, Default, PartialEq, Eq)]
47pub(crate) struct BuildScriptOutput {
48    /// List of config flags defined by this package's build script.
49    pub(crate) cfgs: Vec<CfgAtom>,
50    /// List of cargo-related environment variables with their value.
51    ///
52    /// If the package has a build script which defines environment variables,
53    /// they can also be found here.
54    pub(crate) envs: Env,
55    /// Directory where a build script might place its output.
56    pub(crate) out_dir: Option<AbsPathBuf>,
57    /// Path to the proc-macro library file if this package exposes proc-macros.
58    pub(crate) proc_macro_dylib_path: ProcMacroDylibPath,
59}
60
61impl BuildScriptOutput {
62    fn is_empty(&self) -> bool {
63        self.cfgs.is_empty()
64            && self.envs.is_empty()
65            && self.out_dir.is_none()
66            && matches!(
67                self.proc_macro_dylib_path,
68                ProcMacroDylibPath::NotBuilt | ProcMacroDylibPath::NotProcMacro
69            )
70    }
71}
72
73impl WorkspaceBuildScripts {
74    /// Runs the build scripts for the given workspace
75    pub(crate) fn run_for_workspace(
76        config: &CargoConfig,
77        workspace: &CargoWorkspace,
78        progress: &dyn Fn(String),
79        sysroot: &Sysroot,
80        toolchain: Option<&semver::Version>,
81    ) -> io::Result<WorkspaceBuildScripts> {
82        let current_dir = workspace.workspace_root();
83
84        let allowed_features = workspace.workspace_features();
85        let (_guard, cmd) = Self::build_command(
86            config,
87            &allowed_features,
88            workspace.manifest_path(),
89            workspace.target_directory().as_ref(),
90            current_dir,
91            sysroot,
92            toolchain,
93        )?;
94        Self::run_per_ws(cmd, workspace, progress)
95    }
96
97    /// Runs the build scripts by invoking the configured command *once*.
98    /// This populates the outputs for all passed in workspaces.
99    pub(crate) fn run_once(
100        config: &CargoConfig,
101        workspaces: &[&CargoWorkspace],
102        progress: &dyn Fn(String),
103        working_directory: &AbsPathBuf,
104    ) -> io::Result<Vec<WorkspaceBuildScripts>> {
105        assert_eq!(config.invocation_strategy, InvocationStrategy::Once);
106
107        let (_guard, cmd) = Self::build_command(
108            config,
109            &Default::default(),
110            // These are not gonna be used anyways, so just construct a dummy here
111            &ManifestPath::try_from(working_directory.clone()).unwrap(),
112            working_directory.as_ref(),
113            working_directory,
114            &Sysroot::empty(),
115            None,
116        )?;
117        // NB: Cargo.toml could have been modified between `cargo metadata` and
118        // `cargo check`. We shouldn't assume that package ids we see here are
119        // exactly those from `config`.
120        let mut by_id = FxHashMap::default();
121        // some workspaces might depend on the same crates, so we need to duplicate the outputs
122        // to those collisions
123        let mut collisions = Vec::new();
124        let mut res: Vec<_> = workspaces
125            .iter()
126            .enumerate()
127            .map(|(idx, workspace)| {
128                let mut res = WorkspaceBuildScripts::default();
129                for package in workspace.packages() {
130                    res.outputs.insert(package, BuildScriptOutput::default());
131                    if by_id.contains_key(&workspace[package].id) {
132                        collisions.push((&workspace[package].id, idx, package));
133                    } else {
134                        by_id.insert(workspace[package].id.clone(), (package, idx));
135                    }
136                }
137                res
138            })
139            .collect();
140
141        let errors = Self::run_command(
142            cmd,
143            |package, cb| {
144                if let Some(&(package, workspace)) = by_id.get(package) {
145                    cb(&workspaces[workspace][package].name, &mut res[workspace].outputs[package]);
146                } else {
147                    tracing::error!("Received compiler message for unknown package: {}", package);
148                }
149            },
150            progress,
151        )?;
152        res.iter_mut().for_each(|it| it.error.clone_from(&errors));
153        collisions.into_iter().for_each(|(id, workspace, package)| {
154            if let Some(&(p, w)) = by_id.get(id) {
155                res[workspace].outputs[package] = res[w].outputs[p].clone();
156            }
157        });
158
159        if tracing::enabled!(tracing::Level::INFO) {
160            for (idx, workspace) in workspaces.iter().enumerate() {
161                for package in workspace.packages() {
162                    let package_build_data: &mut BuildScriptOutput = &mut res[idx].outputs[package];
163                    if !package_build_data.is_empty() {
164                        tracing::info!("{}: {package_build_data:?}", workspace[package].manifest,);
165                    }
166                }
167            }
168        }
169
170        Ok(res)
171    }
172
173    pub fn error(&self) -> Option<&str> {
174        self.error.as_deref()
175    }
176
177    pub(crate) fn get_output(&self, idx: Package) -> Option<&BuildScriptOutput> {
178        self.outputs.get(idx)
179    }
180
181    /// Assembles build script outputs for the rustc crates via `--print target-libdir`.
182    pub(crate) fn rustc_crates(
183        rustc: &CargoWorkspace,
184        current_dir: &AbsPath,
185        extra_env: &FxHashMap<String, Option<String>>,
186        sysroot: &Sysroot,
187    ) -> Self {
188        let mut bs = WorkspaceBuildScripts::default();
189        for p in rustc.packages() {
190            bs.outputs.insert(p, BuildScriptOutput::default());
191        }
192        let res = (|| {
193            let target_libdir = (|| {
194                let mut cargo_config = sysroot.tool(Tool::Cargo, current_dir, extra_env);
195                cargo_config
196                    .args(["rustc", "-Z", "unstable-options", "--print", "target-libdir"])
197                    .env("RUSTC_BOOTSTRAP", "1");
198                if let Ok(it) = utf8_stdout(&mut cargo_config) {
199                    return Ok(it);
200                }
201                let mut cmd = sysroot.tool(Tool::Rustc, current_dir, extra_env);
202                cmd.args(["--print", "target-libdir"]);
203                utf8_stdout(&mut cmd)
204            })()?;
205
206            let target_libdir = AbsPathBuf::try_from(Utf8PathBuf::from(target_libdir))
207                .map_err(|_| anyhow::format_err!("target-libdir was not an absolute path"))?;
208            tracing::info!("Loading rustc proc-macro paths from {target_libdir}");
209
210            let proc_macro_dylibs: Vec<(String, AbsPathBuf)> = std::fs::read_dir(target_libdir)?
211                .filter_map(|entry| {
212                    let dir_entry = entry.ok()?;
213                    if dir_entry.file_type().ok()?.is_file() {
214                        let path = dir_entry.path();
215                        let extension = path.extension()?;
216                        if extension == std::env::consts::DLL_EXTENSION {
217                            let name = path
218                                .file_stem()?
219                                .to_str()?
220                                .split_once('-')?
221                                .0
222                                .trim_start_matches("lib")
223                                .to_owned();
224                            let path = match Utf8PathBuf::from_path_buf(path) {
225                                Ok(path) => path,
226                                Err(path) => {
227                                    tracing::warn!(
228                                        "Proc-macro dylib path contains non-UTF8 characters: {:?}",
229                                        path.display()
230                                    );
231                                    return None;
232                                }
233                            };
234                            return match AbsPathBuf::try_from(path) {
235                                Ok(path) => Some((name, path)),
236                                Err(path) => {
237                                    tracing::error!(
238                                        "proc-macro dylib path is not absolute: {:?}",
239                                        path
240                                    );
241                                    None
242                                }
243                            };
244                        }
245                    }
246                    None
247                })
248                .collect();
249            for p in rustc.packages() {
250                let package = &rustc[p];
251                bs.outputs[p].proc_macro_dylib_path =
252                    if package.targets.iter().any(|&it| {
253                        matches!(rustc[it].kind, TargetKind::Lib { is_proc_macro: true })
254                    }) {
255                        match proc_macro_dylibs.iter().find(|(name, _)| *name == package.name) {
256                            Some((_, path)) => ProcMacroDylibPath::Path(path.clone()),
257                            _ => ProcMacroDylibPath::DylibNotFound,
258                        }
259                    } else {
260                        ProcMacroDylibPath::NotProcMacro
261                    }
262            }
263
264            if tracing::enabled!(tracing::Level::INFO) {
265                for package in rustc.packages() {
266                    let package_build_data = &bs.outputs[package];
267                    if !package_build_data.is_empty() {
268                        tracing::info!("{}: {package_build_data:?}", rustc[package].manifest,);
269                    }
270                }
271            }
272            Ok(())
273        })();
274        if let Err::<_, anyhow::Error>(e) = res {
275            bs.error = Some(e.to_string());
276        }
277        bs
278    }
279
280    fn run_per_ws(
281        cmd: Command,
282        workspace: &CargoWorkspace,
283        progress: &dyn Fn(String),
284    ) -> io::Result<WorkspaceBuildScripts> {
285        let mut res = WorkspaceBuildScripts::default();
286        let outputs = &mut res.outputs;
287        // NB: Cargo.toml could have been modified between `cargo metadata` and
288        // `cargo check`. We shouldn't assume that package ids we see here are
289        // exactly those from `config`.
290        let mut by_id: FxHashMap<Arc<PackageId>, Package> = FxHashMap::default();
291        for package in workspace.packages() {
292            outputs.insert(package, BuildScriptOutput::default());
293            by_id.insert(workspace[package].id.clone(), package);
294        }
295
296        res.error = Self::run_command(
297            cmd,
298            |package, cb| {
299                if let Some(&package) = by_id.get(package) {
300                    cb(&workspace[package].name, &mut outputs[package]);
301                } else {
302                    never!(
303                        "Received compiler message for unknown package: {}\n {}",
304                        package,
305                        by_id.keys().join(", ")
306                    );
307                }
308            },
309            progress,
310        )?;
311
312        if tracing::enabled!(tracing::Level::INFO) {
313            for package in workspace.packages() {
314                let package_build_data = &outputs[package];
315                if !package_build_data.is_empty() {
316                    tracing::info!("{}: {package_build_data:?}", workspace[package].manifest,);
317                }
318            }
319        }
320
321        Ok(res)
322    }
323
324    fn run_command(
325        cmd: Command,
326        // ideally this would be something like:
327        // with_output_for: impl FnMut(&str, dyn FnOnce(&mut BuildScriptOutput)),
328        // but owned trait objects aren't a thing
329        mut with_output_for: impl FnMut(&PackageId, &mut dyn FnMut(&str, &mut BuildScriptOutput)),
330        progress: &dyn Fn(String),
331    ) -> io::Result<Option<String>> {
332        let errors = RefCell::new(String::new());
333        let push_err = |err: &str| {
334            let mut e = errors.borrow_mut();
335            e.push_str(err);
336            e.push('\n');
337        };
338
339        tracing::info!("Running build scripts: {:?}", cmd);
340        let output = stdx::process::spawn_with_streaming_output(
341            cmd,
342            &mut |line| {
343                // Copy-pasted from existing cargo_metadata. It seems like we
344                // should be using serde_stacker here?
345                let mut deserializer = serde_json::Deserializer::from_str(line);
346                deserializer.disable_recursion_limit();
347                let message = Message::deserialize(&mut deserializer)
348                    .unwrap_or_else(|_| Message::TextLine(line.to_owned()));
349
350                match message {
351                    Message::BuildScriptExecuted(mut message) => {
352                        with_output_for(&message.package_id, &mut |name, data| {
353                            progress(format!("build script {name} run"));
354                            let cfgs = {
355                                let mut acc = Vec::new();
356                                for cfg in &message.cfgs {
357                                    match crate::parse_cfg(cfg) {
358                                        Ok(it) => acc.push(it),
359                                        Err(err) => {
360                                            push_err(&format!(
361                                                "invalid cfg from cargo-metadata: {err}"
362                                            ));
363                                            return;
364                                        }
365                                    };
366                                }
367                                acc
368                            };
369                            data.envs.extend(message.env.drain(..));
370                            // cargo_metadata crate returns default (empty) path for
371                            // older cargos, which is not absolute, so work around that.
372                            let out_dir = mem::take(&mut message.out_dir);
373                            if !out_dir.as_str().is_empty() {
374                                let out_dir = AbsPathBuf::assert(out_dir);
375                                // inject_cargo_env(package, package_build_data);
376                                data.envs.insert("OUT_DIR", out_dir.as_str());
377                                data.out_dir = Some(out_dir);
378                                data.cfgs = cfgs;
379                            }
380                        });
381                    }
382                    Message::CompilerArtifact(message) => {
383                        with_output_for(&message.package_id, &mut |name, data| {
384                            progress(format!("proc-macro {name} built"));
385                            if data.proc_macro_dylib_path == ProcMacroDylibPath::NotBuilt {
386                                data.proc_macro_dylib_path = ProcMacroDylibPath::NotProcMacro;
387                            }
388                            if !matches!(data.proc_macro_dylib_path, ProcMacroDylibPath::Path(_))
389                                && message
390                                    .target
391                                    .kind
392                                    .contains(&cargo_metadata::TargetKind::ProcMacro)
393                            {
394                                data.proc_macro_dylib_path =
395                                    match message.filenames.iter().find(|file| is_dylib(file)) {
396                                        Some(filename) => {
397                                            let filename = AbsPath::assert(filename);
398                                            ProcMacroDylibPath::Path(filename.to_owned())
399                                        }
400                                        None => ProcMacroDylibPath::DylibNotFound,
401                                    };
402                            }
403                        });
404                    }
405                    Message::CompilerMessage(message) => {
406                        progress(format!("received compiler message for: {}", message.target.name));
407
408                        if let Some(diag) = message.message.rendered.as_deref() {
409                            push_err(diag);
410                        }
411                    }
412                    Message::BuildFinished(_) => {}
413                    Message::TextLine(_) => {}
414                    _ => {}
415                }
416            },
417            &mut |line| {
418                push_err(line);
419            },
420        )?;
421
422        let errors = if !output.status.success() {
423            let errors = errors.into_inner();
424            Some(if errors.is_empty() { "cargo check failed".to_owned() } else { errors })
425        } else {
426            None
427        };
428        Ok(errors)
429    }
430
431    fn build_command(
432        config: &CargoConfig,
433        allowed_features: &FxHashSet<String>,
434        manifest_path: &ManifestPath,
435        target_dir: &Utf8Path,
436        current_dir: &AbsPath,
437        sysroot: &Sysroot,
438        toolchain: Option<&semver::Version>,
439    ) -> io::Result<(Option<temp_dir::TempDir>, Command)> {
440        match config.run_build_script_command.as_deref() {
441            Some([program, args @ ..]) => {
442                let mut cmd = toolchain::command(program, current_dir, &config.extra_env);
443                cmd.args(args);
444                Ok((None, cmd))
445            }
446            _ => {
447                let mut requires_unstable_options = false;
448                let mut cmd = sysroot.tool(Tool::Cargo, current_dir, &config.extra_env);
449
450                cmd.args(["check", "--quiet", "--workspace", "--message-format=json"]);
451                cmd.args(&config.extra_args);
452
453                cmd.arg("--manifest-path");
454                cmd.arg(manifest_path);
455
456                if let Some(target_dir) = config.target_dir_config.target_dir(Some(target_dir)) {
457                    cmd.arg("--target-dir");
458                    cmd.arg(target_dir.as_ref());
459                }
460
461                if let Some(target) = &config.target {
462                    cmd.args(["--target", target]);
463                }
464                let mut temp_dir_guard = None;
465                if toolchain
466                    .is_some_and(|v| *v >= MINIMUM_TOOLCHAIN_VERSION_SUPPORTING_LOCKFILE_PATH)
467                {
468                    let lockfile_path =
469                        <_ as AsRef<Utf8Path>>::as_ref(manifest_path).with_extension("lock");
470                    if let Some((temp_dir, target_lockfile)) = make_lockfile_copy(&lockfile_path) {
471                        requires_unstable_options = true;
472                        temp_dir_guard = Some(temp_dir);
473                        cmd.arg("--lockfile-path");
474                        cmd.arg(target_lockfile.as_str());
475                    }
476                }
477                match &config.features {
478                    CargoFeatures::All => {
479                        cmd.arg("--all-features");
480                    }
481                    CargoFeatures::Selected { features, no_default_features } => {
482                        if *no_default_features {
483                            cmd.arg("--no-default-features");
484                        }
485                        if !features.is_empty() {
486                            cmd.arg("--features");
487                            cmd.arg(
488                                features
489                                    .iter()
490                                    .filter(|&feat| allowed_features.contains(feat))
491                                    .join(","),
492                            );
493                        }
494                    }
495                }
496
497                if manifest_path.is_rust_manifest() {
498                    requires_unstable_options = true;
499                    cmd.arg("-Zscript");
500                }
501
502                cmd.arg("--keep-going");
503
504                // If [`--compile-time-deps` flag](https://github.com/rust-lang/cargo/issues/14434) is
505                // available in current toolchain's cargo, use it to build compile time deps only.
506                const COMP_TIME_DEPS_MIN_TOOLCHAIN_VERSION: semver::Version = semver::Version {
507                    major: 1,
508                    minor: 89,
509                    patch: 0,
510                    pre: semver::Prerelease::EMPTY,
511                    build: semver::BuildMetadata::EMPTY,
512                };
513
514                let cargo_comp_time_deps_available =
515                    toolchain.is_some_and(|v| *v >= COMP_TIME_DEPS_MIN_TOOLCHAIN_VERSION);
516
517                if cargo_comp_time_deps_available {
518                    requires_unstable_options = true;
519                    cmd.arg("--compile-time-deps");
520                    // we can pass this unconditionally, because we won't actually build the
521                    // binaries, and as such, this will succeed even on targets without libtest
522                    cmd.arg("--all-targets");
523                } else {
524                    // --all-targets includes tests, benches and examples in addition to the
525                    // default lib and bins. This is an independent concept from the --target
526                    // flag below.
527                    if config.all_targets {
528                        cmd.arg("--all-targets");
529                    }
530
531                    if config.wrap_rustc_in_build_scripts {
532                        // Setup RUSTC_WRAPPER to point to `rust-analyzer` binary itself. We use
533                        // that to compile only proc macros and build scripts during the initial
534                        // `cargo check`.
535                        // We don't need this if we are using `--compile-time-deps` flag.
536                        let myself = std::env::current_exe()?;
537                        cmd.env("RUSTC_WRAPPER", myself);
538                        cmd.env("RA_RUSTC_WRAPPER", "1");
539                    }
540                }
541                if requires_unstable_options {
542                    cmd.env("__CARGO_TEST_CHANNEL_OVERRIDE_DO_NOT_USE_THIS", "nightly");
543                    cmd.arg("-Zunstable-options");
544                }
545                Ok((temp_dir_guard, cmd))
546            }
547        }
548    }
549}
550
551// FIXME: Find a better way to know if it is a dylib.
552fn is_dylib(path: &Utf8Path) -> bool {
553    match path.extension().map(|e| e.to_owned().to_lowercase()) {
554        None => false,
555        Some(ext) => matches!(ext.as_str(), "dll" | "dylib" | "so"),
556    }
557}
project_model/build_dependencies.rs

project_model/
build_dependencies.rs
