Skip to content

Commit 862df61

Browse files
borsbors
committed
Auto merge of #8823 - ehuss:minimal-download, r=alexcrichton
Avoid some extra downloads with new feature resolver. There are some edge cases with the new feature resolver where it can erroneously trigger a download of a package that is not needed. This is due to the call `is_proc_macro` which has to downloaded the manifest to check if it is a proc-macro. The main change here is to defer calling `is_proc_macro` until after dependencies have been filtered. It also avoids calling `is_proc_macro` if the new feature resolver is enabled, but `decouple_host_deps` and `ignore_inactive_targets` are disabled (such as with `-Z weak-dep-features`), in which case it doesn't matter if it is a proc-macro or not. Fixes #8776
2 parents eaab752 + 3d5a908 commit 862df61

File tree

5 files changed

+273
-11
lines changed

5 files changed

+273
-11
lines changed

‎src/cargo/core/package.rs‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ use serde::Serialize;
1919

2020
use crate::core::compiler::{CompileKind, RustcTargetData};
2121
use crate::core::dependency::DepKind;
22+
use crate::core::resolver::features::ForceAllTargets;
2223
use crate::core::resolver::{HasDevUnits, Resolve};
2324
use crate::core::source::MaybePackage;
2425
use crate::core::{Dependency, Manifest, PackageId, SourceId, Target};
@@ -488,6 +489,7 @@ impl<'cfg> PackageSet<'cfg> {
488489
has_dev_units: HasDevUnits,
489490
requested_kinds: &[CompileKind],
490491
target_data: &RustcTargetData,
492+
force_all_targets: ForceAllTargets,
491493
) -> CargoResult<()> {
492494
fn collect_used_deps(
493495
used: &mut BTreeSet<PackageId>,
@@ -496,6 +498,7 @@ impl<'cfg> PackageSet<'cfg> {
496498
has_dev_units: HasDevUnits,
497499
requested_kinds: &[CompileKind],
498500
target_data: &RustcTargetData,
501+
force_all_targets: ForceAllTargets,
499502
) -> CargoResult<()> {
500503
if !used.insert(pkg_id) {
501504
return Ok(());
@@ -509,12 +512,14 @@ impl<'cfg> PackageSet<'cfg> {
509512
// dependencies are used both for target and host. To tighten this
510513
// up, this function would need to track "for_host" similar to how
511514
// unit dependencies handles it.
512-
let activated = requested_kinds
513-
.iter()
514-
.chain(Some(&CompileKind::Host))
515-
.any(|kind| target_data.dep_platform_activated(dep, *kind));
516-
if !activated {
517-
return false;
515+
if force_all_targets == ForceAllTargets::No {
516+
let activated = requested_kinds
517+
.iter()
518+
.chain(Some(&CompileKind::Host))
519+
.any(|kind| target_data.dep_platform_activated(dep, *kind));
520+
if !activated {
521+
return false;
522+
}
518523
}
519524
true
520525
})
@@ -527,6 +532,7 @@ impl<'cfg> PackageSet<'cfg> {
527532
has_dev_units,
528533
requested_kinds,
529534
target_data,
535+
force_all_targets,
530536
)?;
531537
}
532538
Ok(())
@@ -545,6 +551,7 @@ impl<'cfg> PackageSet<'cfg> {
545551
has_dev_units,
546552
requested_kinds,
547553
target_data,
554+
force_all_targets,
548555
)?;
549556
}
550557
self.get_many(to_download.into_iter())?;

‎src/cargo/core/resolver/features.rs‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -303,6 +303,14 @@ pub struct FeatureResolver<'a, 'cfg> {
303303
/// Keeps track of which packages have had its dependencies processed.
304304
/// Used to avoid cycles, and to speed up processing.
305305
processed_deps: HashSet<(PackageId, bool)>,
306+
/// If this is `true`, then `for_host` needs to be tracked while
307+
/// traversing the graph.
308+
///
309+
/// This is only here to avoid calling `is_proc_macro` when all feature
310+
/// options are disabled (because `is_proc_macro` can trigger downloads).
311+
/// This has to be separate from `FeatureOpts.decouple_host_deps` because
312+
/// `for_host` tracking is also needed for `itarget` to work properly.
313+
track_for_host: bool,
306314
}
307315

308316
impl<'a, 'cfg> FeatureResolver<'a, 'cfg> {
@@ -333,6 +341,7 @@ impl<'a, 'cfg> FeatureResolver<'a, 'cfg> {
333341
opts,
334342
});
335343
}
344+
let track_for_host = opts.decouple_host_deps || opts.ignore_inactive_targets;
336345
let mut r = FeatureResolver {
337346
ws,
338347
target_data,
@@ -343,6 +352,7 @@ impl<'a, 'cfg> FeatureResolver<'a, 'cfg> {
343352
activated_features: HashMap::new(),
344353
activated_dependencies: HashMap::new(),
345354
processed_deps: HashSet::new(),
355+
track_for_host,
346356
};
347357
r.do_resolve(specs, requested_features)?;
348358
log::debug!("features={:#?}", r.activated_features);
@@ -367,7 +377,7 @@ impl<'a, 'cfg> FeatureResolver<'a, 'cfg> {
367377
let member_features = self.ws.members_with_features(specs, requested_features)?;
368378
for (member, requested_features) in &member_features {
369379
let fvs = self.fvs_from_requested(member.package_id(), requested_features);
370-
let for_host = self.is_proc_macro(member.package_id());
380+
let for_host = self.track_for_host && self.is_proc_macro(member.package_id());
371381
self.activate_pkg(member.package_id(), &fvs, for_host)?;
372382
if for_host {
373383
// Also activate without for_host. This is needed if the
@@ -597,7 +607,6 @@ impl<'a, 'cfg> FeatureResolver<'a, 'cfg> {
597607
self.resolve
598608
.deps(pkg_id)
599609
.map(|(dep_id, deps)| {
600-
let is_proc_macro = self.is_proc_macro(dep_id);
601610
let deps = deps
602611
.iter()
603612
.filter(|dep| {
@@ -613,7 +622,8 @@ impl<'a, 'cfg> FeatureResolver<'a, 'cfg> {
613622
true
614623
})
615624
.map(|dep| {
616-
let dep_for_host = for_host || dep.is_build() || is_proc_macro;
625+
let dep_for_host = self.track_for_host
626+
&& (for_host || dep.is_build() || self.is_proc_macro(dep_id));
617627
(dep, dep_for_host)
618628
})
619629
.collect::<Vec<_>>();

‎src/cargo/ops/resolve.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ pub fn resolve_ws_with_opts<'cfg>(
138138
has_dev_units,
139139
requested_targets,
140140
target_data,
141+
force_all_targets,
141142
)?;
142143

143144
let resolved_features = FeatureResolver::resolve(

‎src/cargo/ops/tree/mod.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,8 @@ pub fn build_and_print(ws: &Workspace<'_>, opts: &TreeOptions) -> CargoResult<()
165165
force_all,
166166
)?;
167167
// Download all Packages. Some display formats need to display package metadata.
168+
// This may trigger some unnecessary downloads, but trying to figure out a
169+
// minimal set would be difficult.
168170
let package_map: HashMap<PackageId, &Package> = ws_resolve
169171
.pkg_set
170172
.get_many(ws_resolve.pkg_set.package_ids())?

‎tests/testsuite/features2.rs‎

Lines changed: 244 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
//! Tests for the new feature resolver.
22
33
use cargo_test_support::cross_compile::{self, alternate};
4+
use cargo_test_support::install::cargo_home;
45
use cargo_test_support::paths::CargoPathExt;
56
use cargo_test_support::publish::validate_crate_contents;
67
use cargo_test_support::registry::{Dependency, Package};
@@ -1987,8 +1988,6 @@ fn doc_optional() {
19871988
[DOWNLOADING] crates ...
19881989
[DOWNLOADED] spin v1.0.0 [..]
19891990
[DOWNLOADED] bar v1.0.0 [..]
1990-
[DOWNLOADING] crates ...
1991-
[DOWNLOADED] enabler v1.0.0 [..]
19921991
[DOCUMENTING] bar v1.0.0
19931992
[CHECKING] bar v1.0.0
19941993
[DOCUMENTING] foo v0.1.0 [..]
@@ -1997,3 +1996,246 @@ fn doc_optional() {
19971996
)
19981997
.run();
19991998
}
1999+
2000+
#[cargo_test]
2001+
fn minimal_download() {
2002+
// Various checks that it only downloads the minimum set of dependencies
2003+
// needed in various situations.
2004+
//
2005+
// This checks several permutations of the different
2006+
// host_dep/dev_dep/itarget settings. These 3 are planned to be stabilized
2007+
// together, so there isn't much need to be concerned about how the behave
2008+
// independently. However, there are some cases where they do behave
2009+
// independently. Specifically:
2010+
//
2011+
// * `cargo test` forces dev_dep decoupling to be disabled.
2012+
// * `cargo tree --target=all` forces ignore_inactive_targets off and decouple_dev_deps off.
2013+
// * `cargo tree --target=all -e normal` forces ignore_inactive_targets off.
2014+
//
2015+
// However, `cargo tree` is a little weird because it downloads everything
2016+
// anyways.
2017+
//
2018+
// So to summarize the different permutations:
2019+
//
2020+
// dev_dep | host_dep | itarget | Notes
2021+
// --------|----------|---------|----------------------------
2022+
// | | | -Zfeatures=compare (new resolver should behave same as old)
2023+
// | | ✓ | This scenario should not happen.
2024+
// | ✓ | | `cargo tree --target=all -Zfeatures=all`†
2025+
// | ✓ | ✓ | `cargo test`
2026+
// ✓ | | | This scenario should not happen.
2027+
// ✓ | | ✓ | This scenario should not happen.
2028+
// ✓ | ✓ | | `cargo tree --target=all -e normal -Z features=all`†
2029+
// ✓ | ✓ | ✓ | A normal build.
2030+
//
2031+
// † — However, `cargo tree` downloads everything.
2032+
Package::new("normal", "1.0.0").publish();
2033+
Package::new("normal_pm", "1.0.0").publish();
2034+
Package::new("normal_opt", "1.0.0").publish();
2035+
Package::new("dev_dep", "1.0.0").publish();
2036+
Package::new("dev_dep_pm", "1.0.0").publish();
2037+
Package::new("build_dep", "1.0.0").publish();
2038+
Package::new("build_dep_pm", "1.0.0").publish();
2039+
Package::new("build_dep_opt", "1.0.0").publish();
2040+
2041+
Package::new("itarget_normal", "1.0.0").publish();
2042+
Package::new("itarget_normal_pm", "1.0.0").publish();
2043+
Package::new("itarget_dev_dep", "1.0.0").publish();
2044+
Package::new("itarget_dev_dep_pm", "1.0.0").publish();
2045+
Package::new("itarget_build_dep", "1.0.0").publish();
2046+
Package::new("itarget_build_dep_pm", "1.0.0").publish();
2047+
2048+
let p = project()
2049+
.file(
2050+
"Cargo.toml",
2051+
r#"
2052+
[package]
2053+
name = "foo"
2054+
version = "0.1.0"
2055+
2056+
[dependencies]
2057+
normal = "1.0"
2058+
normal_pm = "1.0"
2059+
normal_opt = { version = "1.0", optional = true }
2060+
2061+
[dev-dependencies]
2062+
dev_dep = "1.0"
2063+
dev_dep_pm = "1.0"
2064+
2065+
[build-dependencies]
2066+
build_dep = "1.0"
2067+
build_dep_pm = "1.0"
2068+
build_dep_opt = { version = "1.0", optional = true }
2069+
2070+
[target.'cfg(whatever)'.dependencies]
2071+
itarget_normal = "1.0"
2072+
itarget_normal_pm = "1.0"
2073+
2074+
[target.'cfg(whatever)'.dev-dependencies]
2075+
itarget_dev_dep = "1.0"
2076+
itarget_dev_dep_pm = "1.0"
2077+
2078+
[target.'cfg(whatever)'.build-dependencies]
2079+
itarget_build_dep = "1.0"
2080+
itarget_build_dep_pm = "1.0"
2081+
"#,
2082+
)
2083+
.file("src/lib.rs", "")
2084+
.file("build.rs", "fn main() {}")
2085+
.build();
2086+
2087+
let clear = || {
2088+
cargo_home().join("registry/cache").rm_rf();
2089+
cargo_home().join("registry/src").rm_rf();
2090+
p.build_dir().rm_rf();
2091+
};
2092+
2093+
// none
2094+
// Should be the same as `-Zfeatures=all`
2095+
p.cargo("check -Zfeatures=compare")
2096+
.masquerade_as_nightly_cargo()
2097+
.with_stderr_unordered(
2098+
"\
2099+
[UPDATING] [..]
2100+
[DOWNLOADING] crates ...
2101+
[DOWNLOADED] normal_pm v1.0.0 [..]
2102+
[DOWNLOADED] normal v1.0.0 [..]
2103+
[DOWNLOADED] build_dep_pm v1.0.0 [..]
2104+
[DOWNLOADED] build_dep v1.0.0 [..]
2105+
[COMPILING] build_dep v1.0.0
2106+
[COMPILING] build_dep_pm v1.0.0
2107+
[CHECKING] normal_pm v1.0.0
2108+
[CHECKING] normal v1.0.0
2109+
[COMPILING] foo v0.1.0 [..]
2110+
[FINISHED] [..]
2111+
",
2112+
)
2113+
.run();
2114+
clear();
2115+
2116+
// all
2117+
p.cargo("check -Zfeatures=all")
2118+
.masquerade_as_nightly_cargo()
2119+
.with_stderr_unordered(
2120+
"\
2121+
[DOWNLOADING] crates ...
2122+
[DOWNLOADED] normal_pm v1.0.0 [..]
2123+
[DOWNLOADED] normal v1.0.0 [..]
2124+
[DOWNLOADED] build_dep_pm v1.0.0 [..]
2125+
[DOWNLOADED] build_dep v1.0.0 [..]
2126+
[COMPILING] build_dep v1.0.0
2127+
[COMPILING] build_dep_pm v1.0.0
2128+
[CHECKING] normal v1.0.0
2129+
[CHECKING] normal_pm v1.0.0
2130+
[COMPILING] foo v0.1.0 [..]
2131+
[FINISHED] [..]
2132+
",
2133+
)
2134+
.run();
2135+
clear();
2136+
2137+
// This disables decouple_dev_deps.
2138+
p.cargo("test --no-run -Zfeatures=all")
2139+
.masquerade_as_nightly_cargo()
2140+
.with_stderr_unordered(
2141+
"\
2142+
[DOWNLOADING] crates ...
2143+
[DOWNLOADED] normal_pm v1.0.0 [..]
2144+
[DOWNLOADED] normal v1.0.0 [..]
2145+
[DOWNLOADED] dev_dep_pm v1.0.0 [..]
2146+
[DOWNLOADED] dev_dep v1.0.0 [..]
2147+
[DOWNLOADED] build_dep_pm v1.0.0 [..]
2148+
[DOWNLOADED] build_dep v1.0.0 [..]
2149+
[COMPILING] build_dep v1.0.0
2150+
[COMPILING] build_dep_pm v1.0.0
2151+
[COMPILING] normal_pm v1.0.0
2152+
[COMPILING] normal v1.0.0
2153+
[COMPILING] dev_dep_pm v1.0.0
2154+
[COMPILING] dev_dep v1.0.0
2155+
[COMPILING] foo v0.1.0 [..]
2156+
[FINISHED] [..]
2157+
",
2158+
)
2159+
.run();
2160+
clear();
2161+
2162+
// This disables itarget, but leaves decouple_dev_deps enabled. However,
2163+
// `cargo tree` downloads everything anyways. Ideally `cargo tree` should
2164+
// be a little smarter, but that would make it a bit more complicated. The
2165+
// two "Downloading" lines are because `download_accessible` doesn't
2166+
// download enough (`cargo tree` really wants everything). Again, that
2167+
// could be cleaner, but is difficult.
2168+
p.cargo("tree -e normal --target=all -Zfeatures=all")
2169+
.masquerade_as_nightly_cargo()
2170+
.with_stderr_unordered(
2171+
"\
2172+
[DOWNLOADING] crates ...
2173+
[DOWNLOADING] crates ...
2174+
[DOWNLOADED] normal v1.0.0 [..]
2175+
[DOWNLOADED] dev_dep v1.0.0 [..]
2176+
[DOWNLOADED] normal_pm v1.0.0 [..]
2177+
[DOWNLOADED] build_dep v1.0.0 [..]
2178+
[DOWNLOADED] dev_dep_pm v1.0.0 [..]
2179+
[DOWNLOADED] build_dep_pm v1.0.0 [..]
2180+
[DOWNLOADED] itarget_normal v1.0.0 [..]
2181+
[DOWNLOADED] itarget_dev_dep v1.0.0 [..]
2182+
[DOWNLOADED] itarget_normal_pm v1.0.0 [..]
2183+
[DOWNLOADED] itarget_build_dep v1.0.0 [..]
2184+
[DOWNLOADED] itarget_dev_dep_pm v1.0.0 [..]
2185+
[DOWNLOADED] itarget_build_dep_pm v1.0.0 [..]
2186+
",
2187+
)
2188+
.with_stdout(
2189+
"\
2190+
foo v0.1.0 ([ROOT]/foo)
2191+
├── itarget_normal v1.0.0
2192+
├── itarget_normal_pm v1.0.0
2193+
├── normal v1.0.0
2194+
└── normal_pm v1.0.0
2195+
",
2196+
)
2197+
.run();
2198+
clear();
2199+
2200+
// This disables itarget and decouple_dev_deps.
2201+
p.cargo("tree --target=all -Zfeatures=all")
2202+
.masquerade_as_nightly_cargo()
2203+
.with_stderr_unordered(
2204+
"\
2205+
[DOWNLOADING] crates ...
2206+
[DOWNLOADED] normal_pm v1.0.0 [..]
2207+
[DOWNLOADED] normal v1.0.0 [..]
2208+
[DOWNLOADED] itarget_normal_pm v1.0.0 [..]
2209+
[DOWNLOADED] itarget_normal v1.0.0 [..]
2210+
[DOWNLOADED] itarget_dev_dep_pm v1.0.0 [..]
2211+
[DOWNLOADED] itarget_dev_dep v1.0.0 [..]
2212+
[DOWNLOADED] itarget_build_dep_pm v1.0.0 [..]
2213+
[DOWNLOADED] itarget_build_dep v1.0.0 [..]
2214+
[DOWNLOADED] dev_dep_pm v1.0.0 [..]
2215+
[DOWNLOADED] dev_dep v1.0.0 [..]
2216+
[DOWNLOADED] build_dep_pm v1.0.0 [..]
2217+
[DOWNLOADED] build_dep v1.0.0 [..]
2218+
",
2219+
)
2220+
.with_stdout(
2221+
"\
2222+
foo v0.1.0 ([ROOT]/foo)
2223+
├── itarget_normal v1.0.0
2224+
├── itarget_normal_pm v1.0.0
2225+
├── normal v1.0.0
2226+
└── normal_pm v1.0.0
2227+
[build-dependencies]
2228+
├── build_dep v1.0.0
2229+
├── build_dep_pm v1.0.0
2230+
├── itarget_build_dep v1.0.0
2231+
└── itarget_build_dep_pm v1.0.0
2232+
[dev-dependencies]
2233+
├── dev_dep v1.0.0
2234+
├── dev_dep_pm v1.0.0
2235+
├── itarget_dev_dep v1.0.0
2236+
└── itarget_dev_dep_pm v1.0.0
2237+
",
2238+
)
2239+
.run();
2240+
clear();
2241+
}

0 commit comments

Comments
 (0)