project_model/
build_dependencies.rs

1//! Logic to invoke `cargo` for building build-dependencies (build scripts and proc-macros) as well as
2//! executing the build scripts to fetch required dependency information (`OUT_DIR` env var, extra
3//! cfg flags, etc).
4//!
5//! In essence this just invokes `cargo` with the appropriate output format which we consume,
6//! but if enabled we will also use `RUSTC_WRAPPER` to only compile the build scripts and
7//! proc-macros and skip everything else.
8
9use std::{cell::RefCell, io, mem, process::Command};
10
11use base_db::Env;
12use cargo_metadata::{Message, PackageId, camino::Utf8Path};
13use cfg::CfgAtom;
14use itertools::Itertools;
15use la_arena::ArenaMap;
16use paths::{AbsPath, AbsPathBuf, Utf8PathBuf};
17use rustc_hash::{FxHashMap, FxHashSet};
18use serde::Deserialize as _;
19use stdx::never;
20use toolchain::Tool;
21use triomphe::Arc;
22
23use crate::{
24    CargoConfig, CargoFeatures, CargoWorkspace, InvocationStrategy, ManifestPath, Package, Sysroot,
25    TargetKind,
26    cargo_config_file::{LockfileCopy, LockfileUsage, make_lockfile_copy},
27    utf8_stdout,
28};
29
30/// Output of the build script and proc-macro building steps for a workspace.
31#[derive(Debug, Default, Clone, PartialEq, Eq)]
32pub struct WorkspaceBuildScripts {
33    outputs: ArenaMap<Package, BuildScriptOutput>,
34    error: Option<String>,
35}
36
37#[derive(Debug, Clone, Default, PartialEq, Eq)]
38pub enum ProcMacroDylibPath {
39    Path(AbsPathBuf),
40    DylibNotFound,
41    NotProcMacro,
42    #[default]
43    NotBuilt,
44}
45
46/// Output of the build script and proc-macro building step for a concrete package.
47#[derive(Debug, Clone, Default, PartialEq, Eq)]
48pub(crate) struct BuildScriptOutput {
49    /// List of config flags defined by this package's build script.
50    pub(crate) cfgs: Vec<CfgAtom>,
51    /// List of cargo-related environment variables with their value.
52    ///
53    /// If the package has a build script which defines environment variables,
54    /// they can also be found here.
55    pub(crate) envs: Env,
56    /// Directory where a build script might place its output.
57    pub(crate) out_dir: Option<AbsPathBuf>,
58    /// Path to the proc-macro library file if this package exposes proc-macros.
59    pub(crate) proc_macro_dylib_path: ProcMacroDylibPath,
60}
61
62impl BuildScriptOutput {
63    fn is_empty(&self) -> bool {
64        self.cfgs.is_empty()
65            && self.envs.is_empty()
66            && self.out_dir.is_none()
67            && matches!(
68                self.proc_macro_dylib_path,
69                ProcMacroDylibPath::NotBuilt | ProcMacroDylibPath::NotProcMacro
70            )
71    }
72}
73
74impl WorkspaceBuildScripts {
75    /// Runs the build scripts for the given workspace
76    pub(crate) fn run_for_workspace(
77        config: &CargoConfig,
78        workspace: &CargoWorkspace,
79        progress: &dyn Fn(String),
80        sysroot: &Sysroot,
81        toolchain: Option<&semver::Version>,
82    ) -> io::Result<WorkspaceBuildScripts> {
83        let current_dir = workspace.workspace_root();
84
85        let allowed_features = workspace.workspace_features();
86        let (_guard, cmd) = Self::build_command(
87            config,
88            &allowed_features,
89            workspace.manifest_path(),
90            workspace.target_directory().as_ref(),
91            current_dir,
92            sysroot,
93            toolchain,
94        )?;
95        Self::run_per_ws(cmd, workspace, progress)
96    }
97
98    /// Runs the build scripts by invoking the configured command *once*.
99    /// This populates the outputs for all passed in workspaces.
100    pub(crate) fn run_once(
101        config: &CargoConfig,
102        workspaces: &[&CargoWorkspace],
103        progress: &dyn Fn(String),
104        working_directory: &AbsPathBuf,
105    ) -> io::Result<Vec<WorkspaceBuildScripts>> {
106        assert_eq!(config.invocation_strategy, InvocationStrategy::Once);
107
108        let (_guard, cmd) = Self::build_command(
109            config,
110            &Default::default(),
111            // These are not gonna be used anyways, so just construct a dummy here
112            &ManifestPath::try_from(working_directory.clone()).unwrap(),
113            working_directory.as_ref(),
114            working_directory,
115            &Sysroot::empty(),
116            None,
117        )?;
118        // NB: Cargo.toml could have been modified between `cargo metadata` and
119        // `cargo check`. We shouldn't assume that package ids we see here are
120        // exactly those from `config`.
121        let mut by_id = FxHashMap::default();
122        // some workspaces might depend on the same crates, so we need to duplicate the outputs
123        // to those collisions
124        let mut collisions = Vec::new();
125        let mut res: Vec<_> = workspaces
126            .iter()
127            .enumerate()
128            .map(|(idx, workspace)| {
129                let mut res = WorkspaceBuildScripts::default();
130                for package in workspace.packages() {
131                    res.outputs.insert(package, BuildScriptOutput::default());
132                    if by_id.contains_key(&workspace[package].id) {
133                        collisions.push((&workspace[package].id, idx, package));
134                    } else {
135                        by_id.insert(workspace[package].id.clone(), (package, idx));
136                    }
137                }
138                res
139            })
140            .collect();
141
142        let errors = Self::run_command(
143            cmd,
144            |package, cb| {
145                if let Some(&(package, workspace)) = by_id.get(package) {
146                    cb(&workspaces[workspace][package].name, &mut res[workspace].outputs[package]);
147                } else {
148                    tracing::error!("Received compiler message for unknown package: {}", package);
149                }
150            },
151            progress,
152        )?;
153        res.iter_mut().for_each(|it| it.error.clone_from(&errors));
154        collisions.into_iter().for_each(|(id, workspace, package)| {
155            if let Some(&(p, w)) = by_id.get(id) {
156                res[workspace].outputs[package] = res[w].outputs[p].clone();
157            }
158        });
159
160        if tracing::enabled!(tracing::Level::INFO) {
161            for (idx, workspace) in workspaces.iter().enumerate() {
162                for package in workspace.packages() {
163                    let package_build_data: &mut BuildScriptOutput = &mut res[idx].outputs[package];
164                    if !package_build_data.is_empty() {
165                        tracing::info!("{}: {package_build_data:?}", workspace[package].manifest,);
166                    }
167                }
168            }
169        }
170
171        Ok(res)
172    }
173
174    pub fn error(&self) -> Option<&str> {
175        self.error.as_deref()
176    }
177
178    pub(crate) fn get_output(&self, idx: Package) -> Option<&BuildScriptOutput> {
179        self.outputs.get(idx)
180    }
181
182    /// Assembles build script outputs for the rustc crates via `--print target-libdir`.
183    pub(crate) fn rustc_crates(
184        rustc: &CargoWorkspace,
185        current_dir: &AbsPath,
186        extra_env: &FxHashMap<String, Option<String>>,
187        sysroot: &Sysroot,
188    ) -> Self {
189        let mut bs = WorkspaceBuildScripts::default();
190        for p in rustc.packages() {
191            bs.outputs.insert(p, BuildScriptOutput::default());
192        }
193        let res = (|| {
194            let target_libdir = (|| {
195                let mut cargo_config = sysroot.tool(Tool::Cargo, current_dir, extra_env);
196                cargo_config
197                    .args(["rustc", "-Z", "unstable-options", "--print", "target-libdir"])
198                    .env("RUSTC_BOOTSTRAP", "1");
199                if let Ok(it) = utf8_stdout(&mut cargo_config) {
200                    return Ok(it);
201                }
202                let mut cmd = sysroot.tool(Tool::Rustc, current_dir, extra_env);
203                cmd.args(["--print", "target-libdir"]);
204                utf8_stdout(&mut cmd)
205            })()?;
206
207            let target_libdir = AbsPathBuf::try_from(Utf8PathBuf::from(target_libdir))
208                .map_err(|_| anyhow::format_err!("target-libdir was not an absolute path"))?;
209            tracing::info!("Loading rustc proc-macro paths from {target_libdir}");
210
211            let proc_macro_dylibs: Vec<(String, AbsPathBuf)> = std::fs::read_dir(target_libdir)?
212                .filter_map(|entry| {
213                    let dir_entry = entry.ok()?;
214                    if dir_entry.file_type().ok()?.is_file() {
215                        let path = dir_entry.path();
216                        let extension = path.extension()?;
217                        if extension == std::env::consts::DLL_EXTENSION {
218                            let name = path
219                                .file_stem()?
220                                .to_str()?
221                                .split_once('-')?
222                                .0
223                                .trim_start_matches("lib")
224                                .to_owned();
225                            let path = match Utf8PathBuf::from_path_buf(path) {
226                                Ok(path) => path,
227                                Err(path) => {
228                                    tracing::warn!(
229                                        "Proc-macro dylib path contains non-UTF8 characters: {:?}",
230                                        path.display()
231                                    );
232                                    return None;
233                                }
234                            };
235                            return match AbsPathBuf::try_from(path) {
236                                Ok(path) => Some((name, path)),
237                                Err(path) => {
238                                    tracing::error!(
239                                        "proc-macro dylib path is not absolute: {:?}",
240                                        path
241                                    );
242                                    None
243                                }
244                            };
245                        }
246                    }
247                    None
248                })
249                .collect();
250            for p in rustc.packages() {
251                let package = &rustc[p];
252                bs.outputs[p].proc_macro_dylib_path =
253                    if package.targets.iter().any(|&it| {
254                        matches!(rustc[it].kind, TargetKind::Lib { is_proc_macro: true })
255                    }) {
256                        match proc_macro_dylibs.iter().find(|(name, _)| *name == package.name) {
257                            Some((_, path)) => ProcMacroDylibPath::Path(path.clone()),
258                            _ => ProcMacroDylibPath::DylibNotFound,
259                        }
260                    } else {
261                        ProcMacroDylibPath::NotProcMacro
262                    }
263            }
264
265            if tracing::enabled!(tracing::Level::INFO) {
266                for package in rustc.packages() {
267                    let package_build_data = &bs.outputs[package];
268                    if !package_build_data.is_empty() {
269                        tracing::info!("{}: {package_build_data:?}", rustc[package].manifest,);
270                    }
271                }
272            }
273            Ok(())
274        })();
275        if let Err::<_, anyhow::Error>(e) = res {
276            bs.error = Some(e.to_string());
277        }
278        bs
279    }
280
281    fn run_per_ws(
282        cmd: Command,
283        workspace: &CargoWorkspace,
284        progress: &dyn Fn(String),
285    ) -> io::Result<WorkspaceBuildScripts> {
286        let mut res = WorkspaceBuildScripts::default();
287        let outputs = &mut res.outputs;
288        // NB: Cargo.toml could have been modified between `cargo metadata` and
289        // `cargo check`. We shouldn't assume that package ids we see here are
290        // exactly those from `config`.
291        let mut by_id: FxHashMap<Arc<PackageId>, Package> = FxHashMap::default();
292        for package in workspace.packages() {
293            outputs.insert(package, BuildScriptOutput::default());
294            by_id.insert(workspace[package].id.clone(), package);
295        }
296
297        res.error = Self::run_command(
298            cmd,
299            |package, cb| {
300                if let Some(&package) = by_id.get(package) {
301                    cb(&workspace[package].name, &mut outputs[package]);
302                } else {
303                    never!(
304                        "Received compiler message for unknown package: {}\n {}",
305                        package,
306                        by_id.keys().join(", ")
307                    );
308                }
309            },
310            progress,
311        )?;
312
313        if tracing::enabled!(tracing::Level::INFO) {
314            for package in workspace.packages() {
315                let package_build_data = &outputs[package];
316                if !package_build_data.is_empty() {
317                    tracing::info!("{}: {package_build_data:?}", workspace[package].manifest,);
318                }
319            }
320        }
321
322        Ok(res)
323    }
324
325    fn run_command(
326        cmd: Command,
327        // ideally this would be something like:
328        // with_output_for: impl FnMut(&str, dyn FnOnce(&mut BuildScriptOutput)),
329        // but owned trait objects aren't a thing
330        mut with_output_for: impl FnMut(&PackageId, &mut dyn FnMut(&str, &mut BuildScriptOutput)),
331        progress: &dyn Fn(String),
332    ) -> io::Result<Option<String>> {
333        let errors = RefCell::new(String::new());
334        let push_err = |err: &str| {
335            let mut e = errors.borrow_mut();
336            e.push_str(err);
337            e.push('\n');
338        };
339
340        tracing::info!("Running build scripts: {:?}", cmd);
341        let output = stdx::process::spawn_with_streaming_output(
342            cmd,
343            &mut |line| {
344                // Copy-pasted from existing cargo_metadata. It seems like we
345                // should be using serde_stacker here?
346                let mut deserializer = serde_json::Deserializer::from_str(line);
347                deserializer.disable_recursion_limit();
348                let message = Message::deserialize(&mut deserializer)
349                    .unwrap_or_else(|_| Message::TextLine(line.to_owned()));
350
351                match message {
352                    Message::BuildScriptExecuted(mut message) => {
353                        with_output_for(&message.package_id, &mut |name, data| {
354                            progress(format!("build script {name} run"));
355                            let cfgs = {
356                                let mut acc = Vec::new();
357                                for cfg in &message.cfgs {
358                                    match crate::parse_cfg(cfg) {
359                                        Ok(it) => acc.push(it),
360                                        Err(err) => {
361                                            push_err(&format!(
362                                                "invalid cfg from cargo-metadata: {err}"
363                                            ));
364                                            return;
365                                        }
366                                    };
367                                }
368                                acc
369                            };
370                            data.envs.extend(message.env.drain(..));
371                            // cargo_metadata crate returns default (empty) path for
372                            // older cargos, which is not absolute, so work around that.
373                            let out_dir = mem::take(&mut message.out_dir);
374                            if !out_dir.as_str().is_empty() {
375                                let out_dir = AbsPathBuf::assert(out_dir);
376                                // inject_cargo_env(package, package_build_data);
377                                data.envs.insert("OUT_DIR", out_dir.as_str());
378                                data.out_dir = Some(out_dir);
379                                data.cfgs = cfgs;
380                            }
381                        });
382                    }
383                    Message::CompilerArtifact(message) => {
384                        with_output_for(&message.package_id, &mut |name, data| {
385                            if data.proc_macro_dylib_path == ProcMacroDylibPath::NotBuilt {
386                                data.proc_macro_dylib_path = ProcMacroDylibPath::NotProcMacro;
387                            }
388                            if !matches!(data.proc_macro_dylib_path, ProcMacroDylibPath::Path(_))
389                                && message
390                                    .target
391                                    .kind
392                                    .contains(&cargo_metadata::TargetKind::ProcMacro)
393                            {
394                                progress(format!("proc-macro {name} built"));
395                                data.proc_macro_dylib_path =
396                                    match message.filenames.iter().find(|file| is_dylib(file)) {
397                                        Some(filename) => {
398                                            let filename = AbsPath::assert(filename);
399                                            ProcMacroDylibPath::Path(filename.to_owned())
400                                        }
401                                        None => ProcMacroDylibPath::DylibNotFound,
402                                    };
403                            }
404                        });
405                    }
406                    Message::CompilerMessage(message) => {
407                        progress(format!("received compiler message for: {}", message.target.name));
408
409                        if let Some(diag) = message.message.rendered.as_deref() {
410                            push_err(diag);
411                        }
412                    }
413                    Message::BuildFinished(_) => {}
414                    Message::TextLine(_) => {}
415                    _ => {}
416                }
417            },
418            &mut |line| {
419                push_err(line);
420            },
421        )?;
422
423        let errors = if !output.status.success() {
424            let errors = errors.into_inner();
425            Some(if errors.is_empty() { "cargo check failed".to_owned() } else { errors })
426        } else {
427            None
428        };
429        Ok(errors)
430    }
431
432    fn build_command(
433        config: &CargoConfig,
434        allowed_features: &FxHashSet<String>,
435        manifest_path: &ManifestPath,
436        target_dir: &Utf8Path,
437        current_dir: &AbsPath,
438        sysroot: &Sysroot,
439        toolchain: Option<&semver::Version>,
440    ) -> io::Result<(Option<LockfileCopy>, Command)> {
441        match config.run_build_script_command.as_deref() {
442            Some([program, args @ ..]) => {
443                let mut cmd = toolchain::command(program, current_dir, &config.extra_env);
444                cmd.args(args);
445                Ok((None, cmd))
446            }
447            _ => {
448                let mut requires_unstable_options = false;
449                let mut cmd = sysroot.tool(Tool::Cargo, current_dir, &config.extra_env);
450
451                cmd.args(["check", "--quiet", "--workspace", "--message-format=json"]);
452                cmd.args(&config.extra_args);
453
454                cmd.arg("--manifest-path");
455                cmd.arg(manifest_path);
456
457                if let Some(target_dir) = config.target_dir_config.target_dir(Some(target_dir)) {
458                    cmd.arg("--target-dir");
459                    cmd.arg(target_dir.as_ref());
460                }
461
462                if let Some(target) = &config.target {
463                    cmd.args(["--target", target]);
464                }
465                let mut lockfile_copy = None;
466                if let Some(toolchain) = toolchain {
467                    let lockfile_path =
468                        <_ as AsRef<Utf8Path>>::as_ref(manifest_path).with_extension("lock");
469                    lockfile_copy = make_lockfile_copy(toolchain, &lockfile_path);
470                    if let Some(lockfile_copy) = &lockfile_copy {
471                        requires_unstable_options = true;
472                        match lockfile_copy.usage {
473                            LockfileUsage::WithFlag => {
474                                cmd.arg("--lockfile-path");
475                                cmd.arg(lockfile_copy.path.as_str());
476                            }
477                            LockfileUsage::WithEnvVar => {
478                                cmd.arg("-Zlockfile-path");
479                                cmd.env(
480                                    "CARGO_RESOLVER_LOCKFILE_PATH",
481                                    lockfile_copy.path.as_os_str(),
482                                );
483                            }
484                        }
485                    }
486                }
487                match &config.features {
488                    CargoFeatures::All => {
489                        cmd.arg("--all-features");
490                    }
491                    CargoFeatures::Selected { features, no_default_features } => {
492                        if *no_default_features {
493                            cmd.arg("--no-default-features");
494                        }
495                        if !features.is_empty() {
496                            cmd.arg("--features");
497                            cmd.arg(
498                                features
499                                    .iter()
500                                    .filter(|&feat| allowed_features.contains(feat))
501                                    .join(","),
502                            );
503                        }
504                    }
505                }
506
507                if manifest_path.is_rust_manifest() {
508                    requires_unstable_options = true;
509                    cmd.arg("-Zscript");
510                }
511
512                cmd.arg("--keep-going");
513
514                // If [`--compile-time-deps` flag](https://github.com/rust-lang/cargo/issues/14434) is
515                // available in current toolchain's cargo, use it to build compile time deps only.
516                const COMP_TIME_DEPS_MIN_TOOLCHAIN_VERSION: semver::Version = semver::Version {
517                    major: 1,
518                    minor: 89,
519                    patch: 0,
520                    pre: semver::Prerelease::EMPTY,
521                    build: semver::BuildMetadata::EMPTY,
522                };
523
524                let cargo_comp_time_deps_available =
525                    toolchain.is_some_and(|v| *v >= COMP_TIME_DEPS_MIN_TOOLCHAIN_VERSION);
526
527                if cargo_comp_time_deps_available {
528                    requires_unstable_options = true;
529                    cmd.arg("--compile-time-deps");
530                    // we can pass this unconditionally, because we won't actually build the
531                    // binaries, and as such, this will succeed even on targets without libtest
532                    cmd.arg("--all-targets");
533                } else {
534                    // --all-targets includes tests, benches and examples in addition to the
535                    // default lib and bins. This is an independent concept from the --target
536                    // flag below.
537                    if config.all_targets {
538                        cmd.arg("--all-targets");
539                    }
540
541                    if config.wrap_rustc_in_build_scripts {
542                        // Setup RUSTC_WRAPPER to point to `rust-analyzer` binary itself. We use
543                        // that to compile only proc macros and build scripts during the initial
544                        // `cargo check`.
545                        // We don't need this if we are using `--compile-time-deps` flag.
546                        let myself = std::env::current_exe()?;
547                        cmd.env("RUSTC_WRAPPER", myself);
548                        cmd.env("RA_RUSTC_WRAPPER", "1");
549                    }
550                }
551                if requires_unstable_options {
552                    cmd.env("__CARGO_TEST_CHANNEL_OVERRIDE_DO_NOT_USE_THIS", "nightly");
553                    cmd.arg("-Zunstable-options");
554                }
555                Ok((lockfile_copy, cmd))
556            }
557        }
558    }
559}
560
561// FIXME: Find a better way to know if it is a dylib.
562fn is_dylib(path: &Utf8Path) -> bool {
563    match path.extension().map(|e| e.to_owned().to_lowercase()) {
564        None => false,
565        Some(ext) => matches!(ext.as_str(), "dll" | "dylib" | "so"),
566    }
567}
project_model/build_dependencies.rs

project_model/
build_dependencies.rs
