Rollup merge of #137020 - ferrocene:pa-vendor-sources, r=Kobzol

Pass vendored sources from bootstrap to generate-copyright

In addition to doing the vendoring in bootstrap, this PR also loads the list of manifests to parse from bootstrap (instead of hardcoding a smaller list in generate-copyright). This is best reviewed commit-by-commit.

Fixes #136955

Author: matthiaskrgr

src/bootstrap/src/core/build_steps/dist.rs

@@ -21,7 +21,7 @@ use tracing::instrument;
 
 use crate::core::build_steps::doc::DocumentationFormat;
 use crate::core::build_steps::tool::{self, Tool};
-use crate::core::build_steps::vendor::default_paths_to_vendor;
+use crate::core::build_steps::vendor::{VENDOR_DIR, Vendor};
 use crate::core::build_steps::{compile, llvm};
 use crate::core::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
 use crate::core::config::TargetSelection;
@@ -1052,19 +1052,6 @@ impl Step for PlainSourceTarball {
         if builder.config.dist_vendor {
             builder.require_and_update_all_submodules();
 
-            // Vendor all Cargo dependencies
-            let mut cmd = command(&builder.initial_cargo);
-            cmd.arg("vendor").arg("--versioned-dirs");
-
-            for (p, _) in default_paths_to_vendor(builder) {
-                cmd.arg("--sync").arg(p);
-            }
-
-            cmd
-                // Will read the libstd Cargo.toml which uses the unstable `public-dependency` feature.
-                .env("RUSTC_BOOTSTRAP", "1")
-                .current_dir(plain_dst_src);
-
             // Vendor packages that are required by opt-dist to collect PGO profiles.
             let pkgs_for_pgo_training = build_helper::LLVM_PGO_CRATES
                 .iter()
@@ -1076,15 +1063,18 @@ impl Step for PlainSourceTarball {
                     manifest_path.push("Cargo.toml");
                     manifest_path
                 });
-            for manifest_path in pkgs_for_pgo_training {
-                cmd.arg("--sync").arg(manifest_path);
-            }
 
-            let config = cmd.run_capture(builder).stdout();
+            // Vendor all Cargo dependencies
+            let vendor = builder.ensure(Vendor {
+                sync_args: pkgs_for_pgo_training.collect(),
+                versioned_dirs: true,
+                root_dir: plain_dst_src.into(),
+                output_dir: VENDOR_DIR.into(),
+            });
 
             let cargo_config_dir = plain_dst_src.join(".cargo");
             builder.create_dir(&cargo_config_dir);
-            builder.create(&cargo_config_dir.join("config.toml"), &config);
+            builder.create(&cargo_config_dir.join("config.toml"), &vendor.config);
         }
 
         // Delete extraneous directories

src/bootstrap/src/core/build_steps/run.rs

@@ -9,6 +9,7 @@ use crate::Mode;
 use crate::core::build_steps::dist::distdir;
 use crate::core::build_steps::test;
 use crate::core::build_steps::tool::{self, SourceType, Tool};
+use crate::core::build_steps::vendor::{Vendor, default_paths_to_vendor};
 use crate::core::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
 use crate::core::config::TargetSelection;
 use crate::core::config::flags::get_completion;
@@ -212,11 +213,39 @@ impl Step for GenerateCopyright {
         let dest = builder.out.join("COPYRIGHT.html");
         let dest_libstd = builder.out.join("COPYRIGHT-library.html");
 
+        let paths_to_vendor = default_paths_to_vendor(builder);
+        for (_, submodules) in &paths_to_vendor {
+            for submodule in submodules {
+                builder.build.require_submodule(submodule, None);
+            }
+        }
+        let cargo_manifests = paths_to_vendor
+            .into_iter()
+            .map(|(path, _submodules)| path.to_str().unwrap().to_string())
+            .inspect(|path| assert!(!path.contains(','), "{path} contains a comma in its name"))
+            .collect::<Vec<_>>()
+            .join(",");
+
+        let vendored_sources = if let Some(path) = builder.vendored_crates_path() {
+            path
+        } else {
+            let cache_dir = builder.out.join("tmp").join("generate-copyright-vendor");
+            builder.ensure(Vendor {
+                sync_args: Vec::new(),
+                versioned_dirs: true,
+                root_dir: builder.src.clone(),
+                output_dir: cache_dir.clone(),
+            });
+            cache_dir
+        };
+
         let mut cmd = builder.tool_cmd(Tool::GenerateCopyright);
+        cmd.env("CARGO_MANIFESTS", &cargo_manifests);
         cmd.env("LICENSE_METADATA", &license_metadata);
         cmd.env("DEST", &dest);
         cmd.env("DEST_LIBSTD", &dest_libstd);
-        cmd.env("OUT_DIR", &builder.out);
+        cmd.env("SRC_DIR", &builder.src);
+        cmd.env("VENDOR_DIR", &vendored_sources);
         cmd.env("CARGO", &builder.initial_cargo);
         // it is important that generate-copyright runs from the root of the
         // source tree, because it uses relative paths

src/bootstrap/src/core/build_steps/vendor.rs

@@ -4,6 +4,8 @@ use crate::core::build_steps::tool::SUBMODULES_FOR_RUSTBOOK;
 use crate::core::builder::{Builder, RunConfig, ShouldRun, Step};
 use crate::utils::exec::command;
 
+pub const VENDOR_DIR: &str = "vendor";
+
 /// Returns the cargo workspaces to vendor for `x vendor` and dist tarballs.
 ///
 /// Returns a `Vec` of `(path_to_manifest, submodules_required)` where
@@ -29,13 +31,14 @@ pub fn default_paths_to_vendor(builder: &Builder<'_>) -> Vec<(PathBuf, Vec<&'sta
 
 #[derive(Debug, Clone, Hash, PartialEq, Eq)]
 pub(crate) struct Vendor {
-    sync_args: Vec<PathBuf>,
-    versioned_dirs: bool,
-    root_dir: PathBuf,
+    pub(crate) sync_args: Vec<PathBuf>,
+    pub(crate) versioned_dirs: bool,
+    pub(crate) root_dir: PathBuf,
+    pub(crate) output_dir: PathBuf,
 }
 
 impl Step for Vendor {
-    type Output = ();
+    type Output = VendorOutput;
     const DEFAULT: bool = true;
     const ONLY_HOSTS: bool = true;
 
@@ -48,10 +51,13 @@ impl Step for Vendor {
             sync_args: run.builder.config.cmd.vendor_sync_args(),
             versioned_dirs: run.builder.config.cmd.vendor_versioned_dirs(),
             root_dir: run.builder.src.clone(),
+            output_dir: run.builder.src.join(VENDOR_DIR),
         });
     }
 
     fn run(self, builder: &Builder<'_>) -> Self::Output {
+        builder.info(&format!("Vendoring sources to {:?}", self.root_dir));
+
         let mut cmd = command(&builder.initial_cargo);
         cmd.arg("vendor");
 
@@ -81,8 +87,14 @@ impl Step for Vendor {
         // which uses the unstable `public-dependency` feature.
         cmd.env("RUSTC_BOOTSTRAP", "1");
 
-        cmd.current_dir(self.root_dir);
+        cmd.current_dir(self.root_dir).arg(&self.output_dir);
 
-        cmd.run(builder);
+        let config = cmd.run_capture_stdout(builder);
+        VendorOutput { config: config.stdout() }
     }
 }
+
+#[derive(Debug, Clone)]
+pub(crate) struct VendorOutput {
+    pub(crate) config: String,
+}

src/bootstrap/src/core/builder/cargo.rs

@@ -924,8 +924,7 @@ impl Builder<'_> {
 
         if self.config.rust_remap_debuginfo {
             let mut env_var = OsString::new();
-            if self.config.vendor {
-                let vendor = self.build.src.join("vendor");
+            if let Some(vendor) = self.build.vendored_crates_path() {
                 env_var.push(vendor);
                 env_var.push("=/rust/deps");
             } else {

src/bootstrap/src/lib.rs

@@ -50,6 +50,8 @@ pub use utils::change_tracker::{
     CONFIG_CHANGE_HISTORY, find_recent_config_change_ids, human_readable_changes,
 };
 
+use crate::core::build_steps::vendor::VENDOR_DIR;
+
 const LLVM_TOOLS: &[&str] = &[
     "llvm-cov",      // used to generate coverage report
     "llvm-nm",       // used to inspect binaries; it shows symbol names, their sizes and visibility
@@ -791,6 +793,11 @@ impl Build {
         self.out.join(target).join("md-doc")
     }
 
+    /// Path to the vendored Rust crates.
+    fn vendored_crates_path(&self) -> Option<PathBuf> {
+        if self.config.vendor { Some(self.src.join(VENDOR_DIR)) } else { None }
+    }
+
     /// Returns `true` if this is an external version of LLVM not managed by bootstrap.
     /// In particular, we expect llvm sources to be available when this is false.
     ///

src/tools/generate-copyright/src/cargo_metadata.rs

@@ -11,10 +11,6 @@ pub enum Error {
     Io(#[from] std::io::Error),
     #[error("Failed get output from cargo-metadata: {0:?}")]
     GettingMetadata(#[from] cargo_metadata::Error),
-    #[error("Failed to run cargo vendor: {0:?}")]
-    LaunchingVendor(std::io::Error),
-    #[error("Failed to complete cargo vendor")]
-    RunningVendor,
     #[error("Bad path {0:?} whilst scraping files")]
     Scraping(PathBuf),
 }
@@ -43,25 +39,19 @@ pub struct PackageMetadata {
     pub is_in_libstd: Option<bool>,
 }
 
-/// Use `cargo metadata` and `cargo vendor` to get a list of dependencies and their license data.
+/// Use `cargo metadata` to get a list of dependencies and their license data. License files will
+/// also be pulled from the vendor path (generated by bootstrap).
 ///
-/// This will involve running `cargo vendor` into `vendor_path` so we can
-/// grab the license files.
-///
-/// Any dependency with a path beginning with `root_path` is ignored, as we
-/// assume `reuse` has covered it already.
+/// Any dependency with a path beginning with `root_path` is ignored, as we assume `reuse` has
+/// covered it already.
 pub fn get_metadata_and_notices(
     cargo: &Path,
     vendor_path: &Path,
     root_path: &Path,
-    manifest_paths: &[&Path],
+    manifest_paths: &[PathBuf],
 ) -> Result<BTreeMap<Package, PackageMetadata>, Error> {
     let mut output = get_metadata(cargo, root_path, manifest_paths)?;
 
-    // Now do a cargo-vendor and grab everything
-    println!("Vendoring deps into {}...", vendor_path.display());
-    run_cargo_vendor(cargo, &vendor_path, manifest_paths)?;
-
     // Now for each dependency we found, go and grab any important looking files
     for (package, metadata) in output.iter_mut() {
         load_important_files(package, metadata, &vendor_path)?;
@@ -77,7 +67,7 @@ pub fn get_metadata_and_notices(
 pub fn get_metadata(
     cargo: &Path,
     root_path: &Path,
-    manifest_paths: &[&Path],
+    manifest_paths: &[PathBuf],
 ) -> Result<BTreeMap<Package, PackageMetadata>, Error> {
     let mut output = BTreeMap::new();
     // Look at the metadata for each manifest
@@ -113,28 +103,6 @@ pub fn get_metadata(
     Ok(output)
 }
 
-/// Run cargo-vendor, fetching into the given dir
-fn run_cargo_vendor(cargo: &Path, dest: &Path, manifest_paths: &[&Path]) -> Result<(), Error> {
-    let mut vendor_command = std::process::Command::new(cargo);
-    vendor_command.env("RUSTC_BOOTSTRAP", "1");
-    vendor_command.arg("vendor");
-    vendor_command.arg("--quiet");
-    vendor_command.arg("--versioned-dirs");
-    for manifest_path in manifest_paths {
-        vendor_command.arg("-s");
-        vendor_command.arg(manifest_path);
-    }
-    vendor_command.arg(dest);
-
-    let vendor_status = vendor_command.status().map_err(Error::LaunchingVendor)?;
-
-    if !vendor_status.success() {
-        return Err(Error::RunningVendor);
-    }
-
-    Ok(())
-}
-
 /// Add important files off disk into this dependency.
 ///
 /// Maybe one-day Cargo.toml will contain enough information that we don't need

src/tools/generate-copyright/src/main.rs

@@ -17,29 +17,36 @@ mod cargo_metadata;
 fn main() -> Result<(), Error> {
     let dest_file = env_path("DEST")?;
     let libstd_dest_file = env_path("DEST_LIBSTD")?;
-    let out_dir = env_path("OUT_DIR")?;
+    let src_dir = env_path("SRC_DIR")?;
+    let vendor_dir = env_path("VENDOR_DIR")?;
     let cargo = env_path("CARGO")?;
     let license_metadata = env_path("LICENSE_METADATA")?;
 
-    let root_path = std::path::absolute(".")?;
+    let cargo_manifests = env_string("CARGO_MANIFESTS")?
+        .split(",")
+        .map(|manifest| manifest.into())
+        .collect::<Vec<PathBuf>>();
+    let library_manifests = cargo_manifests
+        .iter()
+        .filter(|path| {
+            if let Ok(stripped) = path.strip_prefix(&src_dir) {
+                stripped.starts_with("library")
+            } else {
+                panic!("manifest {path:?} not relative to source dir {src_dir:?}");
+            }
+        })
+        .cloned()
+        .collect::<Vec<_>>();
 
     // Scan Cargo dependencies
-    let mut collected_cargo_metadata = cargo_metadata::get_metadata_and_notices(
-        &cargo,
-        &out_dir.join("vendor"),
-        &root_path,
-        &[
-            Path::new("./Cargo.toml"),
-            Path::new("./src/tools/cargo/Cargo.toml"),
-            Path::new("./library/Cargo.toml"),
-        ],
-    )?;
+    let mut collected_cargo_metadata =
+        cargo_metadata::get_metadata_and_notices(&cargo, &vendor_dir, &src_dir, &cargo_manifests)?;
 
     let library_collected_cargo_metadata = cargo_metadata::get_metadata_and_notices(
         &cargo,
-        &out_dir.join("library-vendor"),
-        &root_path,
-        &[Path::new("./library/Cargo.toml")],
+        &vendor_dir,
+        &src_dir,
+        &library_manifests,
     )?;
 
     for (key, value) in collected_cargo_metadata.iter_mut() {
@@ -54,7 +61,7 @@ fn main() -> Result<(), Error> {
     let library_collected_tree_metadata = Metadata {
         files: collected_tree_metadata
             .files
-            .trim_clone(&Path::new("./library"), &Path::new("."))
+            .trim_clone(&src_dir.join("library"), &src_dir)
             .unwrap(),
     };
 
@@ -193,6 +200,17 @@ struct License {
     copyright: Vec<String>,
 }
 
+/// Grab an environment variable as string, or fail nicely.
+fn env_string(var: &str) -> Result<String, Error> {
+    match std::env::var(var) {
+        Ok(var) => Ok(var),
+        Err(std::env::VarError::NotUnicode(_)) => {
+            anyhow::bail!("environment variable {var} is not utf-8")
+        }
+        Err(std::env::VarError::NotPresent) => anyhow::bail!("missing environment variable {var}"),
+    }
+}
+
 /// Grab an environment variable as a PathBuf, or fail nicely.
 fn env_path(var: &str) -> Result<PathBuf, Error> {
     if let Some(var) = std::env::var_os(var) {