]> git.proxmox.com Git - cargo.git/blob - src/cargo/ops/resolve.rs
Rewrite new feature resolver to simplify DepKind handling.
[cargo.git] / src / cargo / ops / resolve.rs
1 //! High-level APIs for executing the resolver.
2 //!
3 //! This module provides functions for running the resolver given a workspace.
4 //! There are roughly 3 main functions:
5 //!
6 //! - `resolve_ws`: A simple, high-level function with no options.
7 //! - `resolve_ws_with_opts`: A medium-level function with options like
8 //! user-provided features. This is the most appropriate function to use in
9 //! most cases.
10 //! - `resolve_with_previous`: A low-level function for running the resolver,
11 //! providing the most power and flexibility.
12
13 use crate::core::compiler::{CompileKind, RustcTargetData};
14 use crate::core::registry::PackageRegistry;
15 use crate::core::resolver::features::{FeatureResolver, ResolvedFeatures};
16 use crate::core::resolver::{self, Resolve, ResolveOpts};
17 use crate::core::summary::Summary;
18 use crate::core::Feature;
19 use crate::core::{PackageId, PackageIdSpec, PackageSet, Source, SourceId, Workspace};
20 use crate::ops;
21 use crate::sources::PathSource;
22 use crate::util::errors::{CargoResult, CargoResultExt};
23 use crate::util::profile;
24 use log::{debug, trace};
25 use std::collections::HashSet;
26
27 /// Result for `resolve_ws_with_opts`.
28 pub struct WorkspaceResolve<'a> {
29 /// Packages to be downloaded.
30 pub pkg_set: PackageSet<'a>,
31 /// The resolve for the entire workspace.
32 ///
33 /// This may be `None` for things like `cargo install` and `-Zavoid-dev-deps`.
34 /// This does not include `paths` overrides.
35 pub workspace_resolve: Option<Resolve>,
36 /// The narrowed resolve, with the specific features enabled, and only the
37 /// given package specs requested.
38 pub targeted_resolve: Resolve,
39 /// The features activated per package.
40 pub resolved_features: ResolvedFeatures,
41 }
42
43 const UNUSED_PATCH_WARNING: &str = "\
44 Check that the patched package version and available features are compatible
45 with the dependency requirements. If the patch has a different version from
46 what is locked in the Cargo.lock file, run `cargo update` to use the new
47 version. This may also occur with an optional dependency that is not enabled.";
48
49 /// Resolves all dependencies for the workspace using the previous
50 /// lock file as a guide if present.
51 ///
52 /// This function will also write the result of resolution as a new lock file
53 /// (unless it is an ephemeral workspace such as `cargo install` or `cargo
54 /// package`).
55 ///
56 /// This is a simple interface used by commands like `clean`, `fetch`, and
57 /// `package`, which don't specify any options or features.
58 pub fn resolve_ws<'a>(ws: &Workspace<'a>) -> CargoResult<(PackageSet<'a>, Resolve)> {
59 let mut registry = PackageRegistry::new(ws.config())?;
60 let resolve = resolve_with_registry(ws, &mut registry)?;
61 let packages = get_resolved_packages(&resolve, registry)?;
62 Ok((packages, resolve))
63 }
64
65 /// Resolves dependencies for some packages of the workspace,
66 /// taking into account `paths` overrides and activated features.
67 ///
68 /// This function will also write the result of resolution as a new lock file
69 /// (unless `Workspace::require_optional_deps` is false, such as `cargo
70 /// install` or `-Z avoid-dev-deps`), or it is an ephemeral workspace (`cargo
71 /// install` or `cargo package`).
72 ///
73 /// `specs` may be empty, which indicates it should resolve all workspace
74 /// members. In this case, `opts.all_features` must be `true`.
75 pub fn resolve_ws_with_opts<'a>(
76 ws: &Workspace<'a>,
77 target_data: &RustcTargetData,
78 requested_target: CompileKind,
79 opts: &ResolveOpts,
80 specs: &[PackageIdSpec],
81 has_dev_units: bool,
82 ) -> CargoResult<WorkspaceResolve<'a>> {
83 let mut registry = PackageRegistry::new(ws.config())?;
84 let mut add_patches = true;
85 let resolve = if ws.ignore_lock() {
86 None
87 } else if ws.require_optional_deps() {
88 // First, resolve the root_package's *listed* dependencies, as well as
89 // downloading and updating all remotes and such.
90 let resolve = resolve_with_registry(ws, &mut registry)?;
91 // No need to add patches again, `resolve_with_registry` has done it.
92 add_patches = false;
93
94 // Second, resolve with precisely what we're doing. Filter out
95 // transitive dependencies if necessary, specify features, handle
96 // overrides, etc.
97 let _p = profile::start("resolving with overrides...");
98
99 add_overrides(&mut registry, ws)?;
100
101 for &(ref replace_spec, ref dep) in ws.root_replace() {
102 if !resolve
103 .iter()
104 .any(|r| replace_spec.matches(r) && !dep.matches_id(r))
105 {
106 ws.config()
107 .shell()
108 .warn(format!("package replacement is not used: {}", replace_spec))?
109 }
110 }
111
112 Some(resolve)
113 } else {
114 ops::load_pkg_lockfile(ws)?
115 };
116
117 let resolved_with_overrides = resolve_with_previous(
118 &mut registry,
119 ws,
120 opts,
121 resolve.as_ref(),
122 None,
123 specs,
124 add_patches,
125 )?;
126
127 let pkg_set = get_resolved_packages(&resolved_with_overrides, registry)?;
128
129 let resolved_features = FeatureResolver::resolve(
130 ws,
131 target_data,
132 &resolved_with_overrides,
133 &opts.features,
134 specs,
135 requested_target,
136 has_dev_units,
137 )?;
138
139 Ok(WorkspaceResolve {
140 pkg_set,
141 workspace_resolve: resolve,
142 targeted_resolve: resolved_with_overrides,
143 resolved_features,
144 })
145 }
146
147 fn resolve_with_registry<'cfg>(
148 ws: &Workspace<'cfg>,
149 registry: &mut PackageRegistry<'cfg>,
150 ) -> CargoResult<Resolve> {
151 let prev = ops::load_pkg_lockfile(ws)?;
152 let resolve = resolve_with_previous(
153 registry,
154 ws,
155 &ResolveOpts::everything(),
156 prev.as_ref(),
157 None,
158 &[],
159 true,
160 )?;
161
162 if !ws.is_ephemeral() {
163 ops::write_pkg_lockfile(ws, &resolve)?;
164 }
165 Ok(resolve)
166 }
167
168 /// Resolves all dependencies for a package using an optional previous instance.
169 /// of resolve to guide the resolution process.
170 ///
171 /// This also takes an optional hash set, `to_avoid`, which is a list of package
172 /// IDs that should be avoided when consulting the previous instance of resolve
173 /// (often used in pairings with updates).
174 ///
175 /// The previous resolve normally comes from a lock file. This function does not
176 /// read or write lock files from the filesystem.
177 ///
178 /// `specs` may be empty, which indicates it should resolve all workspace
179 /// members. In this case, `opts.all_features` must be `true`.
180 ///
181 /// If `register_patches` is true, then entries from the `[patch]` table in
182 /// the manifest will be added to the given `PackageRegistry`.
183 pub fn resolve_with_previous<'cfg>(
184 registry: &mut PackageRegistry<'cfg>,
185 ws: &Workspace<'cfg>,
186 opts: &ResolveOpts,
187 previous: Option<&Resolve>,
188 to_avoid: Option<&HashSet<PackageId>>,
189 specs: &[PackageIdSpec],
190 register_patches: bool,
191 ) -> CargoResult<Resolve> {
192 // We only want one Cargo at a time resolving a crate graph since this can
193 // involve a lot of frobbing of the global caches.
194 let _lock = ws.config().acquire_package_cache_lock()?;
195
196 // Here we place an artificial limitation that all non-registry sources
197 // cannot be locked at more than one revision. This means that if a Git
198 // repository provides more than one package, they must all be updated in
199 // step when any of them are updated.
200 //
201 // TODO: this seems like a hokey reason to single out the registry as being
202 // different.
203 let mut to_avoid_sources: HashSet<SourceId> = HashSet::new();
204 if let Some(to_avoid) = to_avoid {
205 to_avoid_sources.extend(
206 to_avoid
207 .iter()
208 .map(|p| p.source_id())
209 .filter(|s| !s.is_registry()),
210 );
211 }
212
213 let keep = |p: &PackageId| {
214 !to_avoid_sources.contains(&p.source_id())
215 && match to_avoid {
216 Some(set) => !set.contains(p),
217 None => true,
218 }
219 };
220
221 // In the case where a previous instance of resolve is available, we
222 // want to lock as many packages as possible to the previous version
223 // without disturbing the graph structure.
224 let mut try_to_use = HashSet::new();
225 if let Some(r) = previous {
226 trace!("previous: {:?}", r);
227 register_previous_locks(ws, registry, r, &keep);
228
229 // Everything in the previous lock file we want to keep is prioritized
230 // in dependency selection if it comes up, aka we want to have
231 // conservative updates.
232 try_to_use.extend(r.iter().filter(keep).inspect(|id| {
233 debug!("attempting to prefer {}", id);
234 }));
235 }
236
237 if register_patches {
238 for (url, patches) in ws.root_patch() {
239 let previous = match previous {
240 Some(r) => r,
241 None => {
242 registry.patch(url, patches)?;
243 continue;
244 }
245 };
246 let patches = patches
247 .iter()
248 .map(|dep| {
249 let unused = previous.unused_patches().iter().cloned();
250 let candidates = previous.iter().chain(unused);
251 match candidates.filter(keep).find(|&id| dep.matches_id(id)) {
252 Some(id) => {
253 let mut dep = dep.clone();
254 dep.lock_to(id);
255 dep
256 }
257 None => dep.clone(),
258 }
259 })
260 .collect::<Vec<_>>();
261 registry.patch(url, &patches)?;
262 }
263
264 registry.lock_patches();
265 }
266
267 for member in ws.members() {
268 registry.add_sources(Some(member.package_id().source_id()))?;
269 }
270
271 let summaries: Vec<(Summary, ResolveOpts)> = ws
272 .members_with_features(specs, &opts.features)?
273 .into_iter()
274 .map(|(member, features)| {
275 let summary = registry.lock(member.summary().clone());
276 (
277 summary,
278 ResolveOpts {
279 dev_deps: opts.dev_deps,
280 features,
281 },
282 )
283 })
284 .collect();
285
286 let root_replace = ws.root_replace();
287
288 let replace = match previous {
289 Some(r) => root_replace
290 .iter()
291 .map(|&(ref spec, ref dep)| {
292 for (&key, &val) in r.replacements().iter() {
293 if spec.matches(key) && dep.matches_id(val) && keep(&val) {
294 let mut dep = dep.clone();
295 dep.lock_to(val);
296 return (spec.clone(), dep);
297 }
298 }
299 (spec.clone(), dep.clone())
300 })
301 .collect::<Vec<_>>(),
302 None => root_replace.to_vec(),
303 };
304
305 ws.preload(registry);
306 let mut resolved = resolver::resolve(
307 &summaries,
308 &replace,
309 registry,
310 &try_to_use,
311 Some(ws.config()),
312 ws.features().require(Feature::public_dependency()).is_ok(),
313 )?;
314 resolved.register_used_patches(&registry.patches());
315 if register_patches {
316 // It would be good if this warning was more targeted and helpful
317 // (such as showing close candidates that failed to match). However,
318 // that's not terribly easy to do, so just show a general help
319 // message.
320 let warnings: Vec<String> = resolved
321 .unused_patches()
322 .iter()
323 .map(|pkgid| format!("Patch `{}` was not used in the crate graph.", pkgid))
324 .collect();
325 if !warnings.is_empty() {
326 ws.config().shell().warn(format!(
327 "{}\n{}",
328 warnings.join("\n"),
329 UNUSED_PATCH_WARNING
330 ))?;
331 }
332 }
333 if let Some(previous) = previous {
334 resolved.merge_from(previous)?;
335 }
336 Ok(resolved)
337 }
338
339 /// Read the `paths` configuration variable to discover all path overrides that
340 /// have been configured.
341 pub fn add_overrides<'a>(
342 registry: &mut PackageRegistry<'a>,
343 ws: &Workspace<'a>,
344 ) -> CargoResult<()> {
345 let config = ws.config();
346 let paths = match config.get_list("paths")? {
347 Some(list) => list,
348 None => return Ok(()),
349 };
350
351 let paths = paths.val.iter().map(|(s, def)| {
352 // The path listed next to the string is the config file in which the
353 // key was located, so we want to pop off the `.cargo/config` component
354 // to get the directory containing the `.cargo` folder.
355 (def.root(config).join(s), def)
356 });
357
358 for (path, definition) in paths {
359 let id = SourceId::for_path(&path)?;
360 let mut source = PathSource::new_recursive(&path, id, ws.config());
361 source.update().chain_err(|| {
362 format!(
363 "failed to update path override `{}` \
364 (defined in `{}`)",
365 path.display(),
366 definition
367 )
368 })?;
369 registry.add_override(Box::new(source));
370 }
371 Ok(())
372 }
373
374 pub fn get_resolved_packages<'a>(
375 resolve: &Resolve,
376 registry: PackageRegistry<'a>,
377 ) -> CargoResult<PackageSet<'a>> {
378 let ids: Vec<PackageId> = resolve.iter().collect();
379 registry.get(&ids)
380 }
381
382 /// In this function we're responsible for informing the `registry` of all
383 /// locked dependencies from the previous lock file we had, `resolve`.
384 ///
385 /// This gets particularly tricky for a couple of reasons. The first is that we
386 /// want all updates to be conservative, so we actually want to take the
387 /// `resolve` into account (and avoid unnecessary registry updates and such).
388 /// the second, however, is that we want to be resilient to updates of
389 /// manifests. For example if a dependency is added or a version is changed we
390 /// want to make sure that we properly re-resolve (conservatively) instead of
391 /// providing an opaque error.
392 ///
393 /// The logic here is somewhat subtle, but there should be more comments below to
394 /// clarify things.
395 ///
396 /// Note that this function, at the time of this writing, is basically the
397 /// entire fix for issue #4127.
398 fn register_previous_locks(
399 ws: &Workspace<'_>,
400 registry: &mut PackageRegistry<'_>,
401 resolve: &Resolve,
402 keep: &dyn Fn(&PackageId) -> bool,
403 ) {
404 let path_pkg = |id: SourceId| {
405 if !id.is_path() {
406 return None;
407 }
408 if let Ok(path) = id.url().to_file_path() {
409 if let Ok(pkg) = ws.load(&path.join("Cargo.toml")) {
410 return Some(pkg);
411 }
412 }
413 None
414 };
415
416 // Ok so we've been passed in a `keep` function which basically says "if I
417 // return `true` then this package wasn't listed for an update on the command
418 // line". That is, if we run `cargo update -p foo` then `keep(bar)` will return
419 // `true`, whereas `keep(foo)` will return `false` (roughly speaking).
420 //
421 // This isn't actually quite what we want, however. Instead we want to
422 // further refine this `keep` function with *all transitive dependencies* of
423 // the packages we're not keeping. For example, consider a case like this:
424 //
425 // * There's a crate `log`.
426 // * There's a crate `serde` which depends on `log`.
427 //
428 // Let's say we then run `cargo update -p serde`. This may *also* want to
429 // update the `log` dependency as our newer version of `serde` may have a
430 // new minimum version required for `log`. Now this isn't always guaranteed
431 // to work. What'll happen here is we *won't* lock the `log` dependency nor
432 // the `log` crate itself, but we will inform the registry "please prefer
433 // this version of `log`". That way if our newer version of serde works with
434 // the older version of `log`, we conservatively won't update `log`. If,
435 // however, nothing else in the dependency graph depends on `log` and the
436 // newer version of `serde` requires a new version of `log` it'll get pulled
437 // in (as we didn't accidentally lock it to an old version).
438 //
439 // Additionally, here we process all path dependencies listed in the previous
440 // resolve. They can not only have their dependencies change but also
441 // the versions of the package change as well. If this ends up happening
442 // then we want to make sure we don't lock a package ID node that doesn't
443 // actually exist. Note that we don't do transitive visits of all the
444 // package's dependencies here as that'll be covered below to poison those
445 // if they changed.
446 let mut avoid_locking = HashSet::new();
447 registry.add_to_yanked_whitelist(resolve.iter().filter(keep));
448 for node in resolve.iter() {
449 if !keep(&node) {
450 add_deps(resolve, node, &mut avoid_locking);
451 } else if let Some(pkg) = path_pkg(node.source_id()) {
452 if pkg.package_id() != node {
453 avoid_locking.insert(node);
454 }
455 }
456 }
457
458 // Ok, but the above loop isn't the entire story! Updates to the dependency
459 // graph can come from two locations, the `cargo update` command or
460 // manifests themselves. For example a manifest on the filesystem may
461 // have been updated to have an updated version requirement on `serde`. In
462 // this case both `keep(serde)` and `keep(log)` return `true` (the `keep`
463 // that's an argument to this function). We, however, don't want to keep
464 // either of those! Otherwise we'll get obscure resolve errors about locked
465 // versions.
466 //
467 // To solve this problem we iterate over all packages with path sources
468 // (aka ones with manifests that are changing) and take a look at all of
469 // their dependencies. If any dependency does not match something in the
470 // previous lock file, then we're guaranteed that the main resolver will
471 // update the source of this dependency no matter what. Knowing this we
472 // poison all packages from the same source, forcing them all to get
473 // updated.
474 //
475 // This may seem like a heavy hammer, and it is! It means that if you change
476 // anything from crates.io then all of crates.io becomes unlocked. Note,
477 // however, that we still want conservative updates. This currently happens
478 // because the first candidate the resolver picks is the previously locked
479 // version, and only if that fails to activate to we move on and try
480 // a different version. (giving the guise of conservative updates)
481 //
482 // For example let's say we had `serde = "0.1"` written in our lock file.
483 // When we later edit this to `serde = "0.1.3"` we don't want to lock serde
484 // at its old version, 0.1.1. Instead we want to allow it to update to
485 // `0.1.3` and update its own dependencies (like above). To do this *all
486 // crates from crates.io* are not locked (aka added to `avoid_locking`).
487 // For dependencies like `log` their previous version in the lock file will
488 // come up first before newer version, if newer version are available.
489 let mut path_deps = ws.members().cloned().collect::<Vec<_>>();
490 let mut visited = HashSet::new();
491 while let Some(member) = path_deps.pop() {
492 if !visited.insert(member.package_id()) {
493 continue;
494 }
495 let is_ws_member = ws.is_member(&member);
496 for dep in member.dependencies() {
497 // If this dependency didn't match anything special then we may want
498 // to poison the source as it may have been added. If this path
499 // dependencies is **not** a workspace member, however, and it's an
500 // optional/non-transitive dependency then it won't be necessarily
501 // be in our lock file. If this shows up then we avoid poisoning
502 // this source as otherwise we'd repeatedly update the registry.
503 //
504 // TODO: this breaks adding an optional dependency in a
505 // non-workspace member and then simultaneously editing the
506 // dependency on that crate to enable the feature. For now,
507 // this bug is better than the always-updating registry though.
508 if !is_ws_member && (dep.is_optional() || !dep.is_transitive()) {
509 continue;
510 }
511
512 // If this is a path dependency, then try to push it onto our
513 // worklist.
514 if let Some(pkg) = path_pkg(dep.source_id()) {
515 path_deps.push(pkg);
516 continue;
517 }
518
519 // If we match *anything* in the dependency graph then we consider
520 // ourselves all ok, and assume that we'll resolve to that.
521 if resolve.iter().any(|id| dep.matches_ignoring_source(id)) {
522 continue;
523 }
524
525 // Ok if nothing matches, then we poison the source of these
526 // dependencies and the previous lock file.
527 debug!(
528 "poisoning {} because {} looks like it changed {}",
529 dep.source_id(),
530 member.package_id(),
531 dep.package_name()
532 );
533 for id in resolve
534 .iter()
535 .filter(|id| id.source_id() == dep.source_id())
536 {
537 add_deps(resolve, id, &mut avoid_locking);
538 }
539 }
540 }
541
542 // Alright now that we've got our new, fresh, shiny, and refined `keep`
543 // function let's put it to action. Take a look at the previous lock file,
544 // filter everything by this callback, and then shove everything else into
545 // the registry as a locked dependency.
546 let keep = |id: &PackageId| keep(id) && !avoid_locking.contains(id);
547
548 for node in resolve.iter().filter(keep) {
549 let deps = resolve
550 .deps_not_replaced(node)
551 .map(|p| p.0)
552 .filter(keep)
553 .collect();
554 registry.register_lock(node, deps);
555 }
556
557 /// Recursively add `node` and all its transitive dependencies to `set`.
558 fn add_deps(resolve: &Resolve, node: PackageId, set: &mut HashSet<PackageId>) {
559 if !set.insert(node) {
560 return;
561 }
562 debug!("ignoring any lock pointing directly at {}", node);
563 for (dep, _) in resolve.deps_not_replaced(node) {
564 add_deps(resolve, dep, set);
565 }
566 }
567 }