src/cargo/core/compiler/context/compilation_files.rs

   1 use std::collections::HashMap;
   2 use std::env;
   3 use std::fmt;
   4 use std::hash::{Hash, Hasher, SipHasher};
   5 use std::path::{Path, PathBuf};
   6 use std::sync::Arc;
   7
   8 use lazycell::LazyCell;
   9 use log::info;
  10
  11 use super::{BuildContext, Context, FileFlavor, Kind, Layout};
  12 use crate::core::compiler::{CompileMode, Unit};
  13 use crate::core::{InternedString, TargetKind, Workspace};
  14 use crate::util::{self, CargoResult};
  15
  16 /// The `Metadata` is a hash used to make unique file names for each unit in a build.
  17 /// For example:
  18 /// - A project may depend on crate `A` and crate `B`, so the package name must be in the file name.
  19 /// - Similarly a project may depend on two versions of `A`, so the version must be in the file name.
  20 /// In general this must include all things that need to be distinguished in different parts of
  21 /// the same build. This is absolutely required or we override things before
  22 /// we get chance to use them.
  23 ///
  24 /// We use a hash because it is an easy way to guarantee
  25 /// that all the inputs can be converted to a valid path.
  26 ///
  27 /// This also acts as the main layer of caching provided by Cargo.
  28 /// For example, we want to cache `cargo build` and `cargo doc` separately, so that running one
  29 /// does not invalidate the artifacts for the other. We do this by including `CompileMode` in the
  30 /// hash, thus the artifacts go in different folders and do not override each other.
  31 /// If we don't add something that we should have, for this reason, we get the
  32 /// correct output but rebuild more than is needed.
  33 ///
  34 /// Some things that need to be tracked to ensure the correct output should definitely *not*
  35 /// go in the `Metadata`. For example, the modification time of a file, should be tracked to make a
  36 /// rebuild when the file changes. However, it would be wasteful to include in the `Metadata`. The
  37 /// old artifacts are never going to be needed again. We can save space by just overwriting them.
  38 /// If we add something that we should not have, for this reason, we get the correct output but take
  39 /// more space than needed. This makes not including something in `Metadata`
  40 /// a form of cache invalidation.
  41 ///
  42 /// Note that the `Fingerprint` is in charge of tracking everything needed to determine if a
  43 /// rebuild is needed.
  44 #[derive(Clone, Hash, Eq, PartialEq, Ord, PartialOrd)]
  45 pub struct Metadata(u64);
  46
  47 impl fmt::Display for Metadata {
  48     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
  49         write!(f, "{:016x}", self.0)
  50     }
  51 }
  52
  53 pub struct CompilationFiles<'a, 'cfg> {
  54     /// The target directory layout for the host (and target if it is the same as host).
  55     pub(super) host: Layout,
  56     /// The target directory layout for the target (if different from then host).
  57     pub(super) target: HashMap<InternedString, Layout>,
  58     /// Additional directory to include a copy of the outputs.
  59     export_dir: Option<PathBuf>,
  60     /// The root targets requested by the user on the command line (does not
  61     /// include dependencies).
  62     roots: Vec<Unit<'a>>,
  63     ws: &'a Workspace<'cfg>,
  64     metas: HashMap<Unit<'a>, Option<Metadata>>,
  65     /// For each Unit, a list all files produced.
  66     outputs: HashMap<Unit<'a>, LazyCell<Arc<Vec<OutputFile>>>>,
  67 }
  68
  69 #[derive(Debug)]
  70 pub struct OutputFile {
  71     /// Absolute path to the file that will be produced by the build process.
  72     pub path: PathBuf,
  73     /// If it should be linked into `target`, and what it should be called
  74     /// (e.g., without metadata).
  75     pub hardlink: Option<PathBuf>,
  76     /// If `--out-dir` is specified, the absolute path to the exported file.
  77     pub export_path: Option<PathBuf>,
  78     /// Type of the file (library / debug symbol / else).
  79     pub flavor: FileFlavor,
  80 }
  81
  82 impl OutputFile {
  83     /// Gets the hard link if present; otherwise, returns the path.
  84     pub fn bin_dst(&self) -> &PathBuf {
  85         match self.hardlink {
  86             Some(ref link_dst) => link_dst,
  87             None => &self.path,
  88         }
  89     }
  90 }
  91
  92 impl<'a, 'cfg: 'a> CompilationFiles<'a, 'cfg> {
  93     pub(super) fn new(
  94         roots: &[Unit<'a>],
  95         host: Layout,
  96         target: HashMap<InternedString, Layout>,
  97         export_dir: Option<PathBuf>,
  98         ws: &'a Workspace<'cfg>,
  99         cx: &Context<'a, 'cfg>,
 100     ) -> CompilationFiles<'a, 'cfg> {
 101         let mut metas = HashMap::new();
 102         for unit in roots {
 103             metadata_of(unit, cx, &mut metas);
 104         }
 105         let outputs = metas
 106             .keys()
 107             .cloned()
 108             .map(|unit| (unit, LazyCell::new()))
 109             .collect();
 110         CompilationFiles {
 111             ws,
 112             host,
 113             target,
 114             export_dir,
 115             roots: roots.to_vec(),
 116             metas,
 117             outputs,
 118         }
 119     }
 120
 121     /// Returns the appropriate directory layout for either a plugin or not.
 122     pub fn layout(&self, kind: Kind) -> &Layout {
 123         match kind {
 124             Kind::Host => &self.host,
 125             Kind::Target(name) => self.target.get(&name).unwrap_or(&self.host),
 126         }
 127     }
 128
 129     /// Gets the metadata for a target in a specific profile.
 130     /// We build to the path `"{filename}-{target_metadata}"`.
 131     /// We use a linking step to link/copy to a predictable filename
 132     /// like `target/debug/libfoo.{a,so,rlib}` and such.
 133     pub fn metadata(&self, unit: &Unit<'a>) -> Option<Metadata> {
 134         self.metas[unit].clone()
 135     }
 136
 137     /// Gets the short hash based only on the `PackageId`.
 138     /// Used for the metadata when `target_metadata` returns `None`.
 139     pub fn target_short_hash(&self, unit: &Unit<'_>) -> String {
 140         let hashable = unit.pkg.package_id().stable_hash(self.ws.root());
 141         util::short_hash(&hashable)
 142     }
 143
 144     /// Returns the appropriate output directory for the specified package and
 145     /// target.
 146     pub fn out_dir(&self, unit: &Unit<'a>) -> PathBuf {
 147         if unit.mode.is_doc() {
 148             self.layout(unit.kind).doc().to_path_buf()
 149         } else if unit.mode.is_doc_test() {
 150             panic!("doc tests do not have an out dir");
 151         } else if unit.target.is_custom_build() {
 152             self.build_script_dir(unit)
 153         } else if unit.target.is_example() {
 154             self.layout(unit.kind).examples().to_path_buf()
 155         } else {
 156             self.deps_dir(unit).to_path_buf()
 157         }
 158     }
 159
 160     pub fn export_dir(&self) -> Option<PathBuf> {
 161         self.export_dir.clone()
 162     }
 163
 164     pub fn pkg_dir(&self, unit: &Unit<'a>) -> String {
 165         let name = unit.pkg.package_id().name();
 166         match self.metas[unit] {
 167             Some(ref meta) => format!("{}-{}", name, meta),
 168             None => format!("{}-{}", name, self.target_short_hash(unit)),
 169         }
 170     }
 171
 172     /// Returns the root of the build output tree for the host
 173     pub fn host_root(&self) -> &Path {
 174         self.host.dest()
 175     }
 176
 177     pub fn host_deps(&self) -> &Path {
 178         self.host.deps()
 179     }
 180
 181     /// Returns the directories where Rust crate dependencies are found for the
 182     /// specified unit.
 183     pub fn deps_dir(&self, unit: &Unit<'_>) -> &Path {
 184         self.layout(unit.kind).deps()
 185     }
 186
 187     pub fn fingerprint_dir(&self, unit: &Unit<'a>) -> PathBuf {
 188         let dir = self.pkg_dir(unit);
 189         self.layout(unit.kind).fingerprint().join(dir)
 190     }
 191
 192     /// Path where compiler output is cached.
 193     pub fn message_cache_path(&self, unit: &Unit<'a>) -> PathBuf {
 194         self.fingerprint_dir(unit).join("output")
 195     }
 196
 197     /// Returns the directory where a compiled build script is stored.
 198     /// `/path/to/target/{debug,release}/build/PKG-HASH`
 199     pub fn build_script_dir(&self, unit: &Unit<'a>) -> PathBuf {
 200         assert!(unit.target.is_custom_build());
 201         assert!(!unit.mode.is_run_custom_build());
 202         let dir = self.pkg_dir(unit);
 203         self.layout(Kind::Host).build().join(dir)
 204     }
 205
 206     /// Returns the directory where information about running a build script
 207     /// is stored.
 208     /// `/path/to/target/{debug,release}/build/PKG-HASH`
 209     pub fn build_script_run_dir(&self, unit: &Unit<'a>) -> PathBuf {
 210         assert!(unit.target.is_custom_build());
 211         assert!(unit.mode.is_run_custom_build());
 212         let dir = self.pkg_dir(unit);
 213         self.layout(unit.kind).build().join(dir)
 214     }
 215
 216     /// Returns the "OUT_DIR" directory for running a build script.
 217     /// `/path/to/target/{debug,release}/build/PKG-HASH/out`
 218     pub fn build_script_out_dir(&self, unit: &Unit<'a>) -> PathBuf {
 219         self.build_script_run_dir(unit).join("out")
 220     }
 221
 222     /// Returns the file stem for a given target/profile combo (with metadata).
 223     pub fn file_stem(&self, unit: &Unit<'a>) -> String {
 224         match self.metas[unit] {
 225             Some(ref metadata) => format!("{}-{}", unit.target.crate_name(), metadata),
 226             None => self.bin_stem(unit),
 227         }
 228     }
 229
 230     pub(super) fn outputs(
 231         &self,
 232         unit: &Unit<'a>,
 233         bcx: &BuildContext<'a, 'cfg>,
 234     ) -> CargoResult<Arc<Vec<OutputFile>>> {
 235         self.outputs[unit]
 236             .try_borrow_with(|| self.calc_outputs(unit, bcx))
 237             .map(Arc::clone)
 238     }
 239
 240     /// Returns the bin stem for a given target (without metadata).
 241     fn bin_stem(&self, unit: &Unit<'_>) -> String {
 242         if unit.target.allows_underscores() {
 243             unit.target.name().to_string()
 244         } else {
 245             unit.target.crate_name()
 246         }
 247     }
 248
 249     /// Returns a tuple with the directory and name of the hard link we expect
 250     /// our target to be copied to. Eg, file_stem may be out_dir/deps/foo-abcdef
 251     /// and link_stem would be out_dir/foo
 252     /// This function returns it in two parts so the caller can add prefix/suffix
 253     /// to filename separately.
 254     ///
 255     /// Returns an `Option` because in some cases we don't want to link
 256     /// (eg a dependent lib).
 257     fn link_stem(&self, unit: &Unit<'a>) -> Option<(PathBuf, String)> {
 258         let out_dir = self.out_dir(unit);
 259         let bin_stem = self.bin_stem(unit); // Stem without metadata.
 260         let file_stem = self.file_stem(unit); // Stem with metadata.
 261
 262         // We currently only lift files up from the `deps` directory. If
 263         // it was compiled into something like `example/` or `doc/` then
 264         // we don't want to link it up.
 265         if out_dir.ends_with("deps") {
 266             // Don't lift up library dependencies.
 267             if unit.target.is_bin() || self.roots.contains(unit) {
 268                 Some((
 269                     out_dir.parent().unwrap().to_owned(),
 270                     if unit.mode.is_any_test() {
 271                         file_stem
 272                     } else {
 273                         bin_stem
 274                     },
 275                 ))
 276             } else {
 277                 None
 278             }
 279         } else if bin_stem == file_stem {
 280             None
 281         } else if out_dir.ends_with("examples") || out_dir.parent().unwrap().ends_with("build") {
 282             Some((out_dir, bin_stem))
 283         } else {
 284             None
 285         }
 286     }
 287
 288     fn calc_outputs(
 289         &self,
 290         unit: &Unit<'a>,
 291         bcx: &BuildContext<'a, 'cfg>,
 292     ) -> CargoResult<Arc<Vec<OutputFile>>> {
 293         let ret = match unit.mode {
 294             CompileMode::Check { .. } => {
 295                 // This may be confusing. rustc outputs a file named `lib*.rmeta`
 296                 // for both libraries and binaries.
 297                 let file_stem = self.file_stem(unit);
 298                 let path = self.out_dir(unit).join(format!("lib{}.rmeta", file_stem));
 299                 vec![OutputFile {
 300                     path,
 301                     hardlink: None,
 302                     export_path: None,
 303                     flavor: FileFlavor::Linkable { rmeta: false },
 304                 }]
 305             }
 306             CompileMode::Doc { .. } => {
 307                 let path = self
 308                     .out_dir(unit)
 309                     .join(unit.target.crate_name())
 310                     .join("index.html");
 311                 vec![OutputFile {
 312                     path,
 313                     hardlink: None,
 314                     export_path: None,
 315                     flavor: FileFlavor::Normal,
 316                 }]
 317             }
 318             CompileMode::RunCustomBuild => {
 319                 // At this time, this code path does not handle build script
 320                 // outputs.
 321                 vec![]
 322             }
 323             CompileMode::Doctest => {
 324                 // Doctests are built in a temporary directory and then
 325                 // deleted. There is the `--persist-doctests` unstable flag,
 326                 // but Cargo does not know about that.
 327                 vec![]
 328             }
 329             CompileMode::Test | CompileMode::Build | CompileMode::Bench => {
 330                 self.calc_outputs_rustc(unit, bcx)?
 331             }
 332         };
 333         info!("Target filenames: {:?}", ret);
 334
 335         Ok(Arc::new(ret))
 336     }
 337
 338     fn calc_outputs_rustc(
 339         &self,
 340         unit: &Unit<'a>,
 341         bcx: &BuildContext<'a, 'cfg>,
 342     ) -> CargoResult<Vec<OutputFile>> {
 343         let mut ret = Vec::new();
 344         let mut unsupported = Vec::new();
 345
 346         let out_dir = self.out_dir(unit);
 347         let link_stem = self.link_stem(unit);
 348         let info = bcx.info(unit.kind);
 349         let file_stem = self.file_stem(unit);
 350
 351         let mut add = |crate_type: &str, flavor: FileFlavor| -> CargoResult<()> {
 352             let crate_type = if crate_type == "lib" {
 353                 "rlib"
 354             } else {
 355                 crate_type
 356             };
 357             let file_types = info.file_types(
 358                 crate_type,
 359                 flavor,
 360                 unit.target.kind(),
 361                 &bcx.target_triple(unit.kind),
 362             )?;
 363
 364             match file_types {
 365                 Some(types) => {
 366                     for file_type in types {
 367                         let path = out_dir.join(file_type.filename(&file_stem));
 368                         let hardlink = link_stem
 369                             .as_ref()
 370                             .map(|&(ref ld, ref ls)| ld.join(file_type.filename(ls)));
 371                         let export_path = if unit.target.is_custom_build() {
 372                             None
 373                         } else {
 374                             self.export_dir.as_ref().and_then(|export_dir| {
 375                                 hardlink.as_ref().and_then(|hardlink| {
 376                                     Some(export_dir.join(hardlink.file_name().unwrap()))
 377                                 })
 378                             })
 379                         };
 380                         ret.push(OutputFile {
 381                             path,
 382                             hardlink,
 383                             export_path,
 384                             flavor: file_type.flavor,
 385                         });
 386                     }
 387                 }
 388                 // Not supported; don't worry about it.
 389                 None => {
 390                     unsupported.push(crate_type.to_string());
 391                 }
 392             }
 393             Ok(())
 394         };
 395         match *unit.target.kind() {
 396             TargetKind::Bin
 397             | TargetKind::CustomBuild
 398             | TargetKind::ExampleBin
 399             | TargetKind::Bench
 400             | TargetKind::Test => {
 401                 add("bin", FileFlavor::Normal)?;
 402             }
 403             TargetKind::Lib(..) | TargetKind::ExampleLib(..) if unit.mode.is_any_test() => {
 404                 add("bin", FileFlavor::Normal)?;
 405             }
 406             TargetKind::ExampleLib(ref kinds) | TargetKind::Lib(ref kinds) => {
 407                 for kind in kinds {
 408                     add(
 409                         kind.crate_type(),
 410                         if kind.linkable() {
 411                             FileFlavor::Linkable { rmeta: false }
 412                         } else {
 413                             FileFlavor::Normal
 414                         },
 415                     )?;
 416                 }
 417                 let path = out_dir.join(format!("lib{}.rmeta", file_stem));
 418                 if !unit.requires_upstream_objects() {
 419                     ret.push(OutputFile {
 420                         path,
 421                         hardlink: None,
 422                         export_path: None,
 423                         flavor: FileFlavor::Linkable { rmeta: true },
 424                     });
 425                 }
 426             }
 427         }
 428         if ret.is_empty() {
 429             if !unsupported.is_empty() {
 430                 failure::bail!(
 431                     "cannot produce {} for `{}` as the target `{}` \
 432                      does not support these crate types",
 433                     unsupported.join(", "),
 434                     unit.pkg,
 435                     bcx.target_triple(unit.kind),
 436                 )
 437             }
 438             failure::bail!(
 439                 "cannot compile `{}` as the target `{}` does not \
 440                  support any of the output crate types",
 441                 unit.pkg,
 442                 bcx.target_triple(unit.kind),
 443             );
 444         }
 445         Ok(ret)
 446     }
 447 }
 448
 449 fn metadata_of<'a, 'cfg>(
 450     unit: &Unit<'a>,
 451     cx: &Context<'a, 'cfg>,
 452     metas: &mut HashMap<Unit<'a>, Option<Metadata>>,
 453 ) -> Option<Metadata> {
 454     if !metas.contains_key(unit) {
 455         let meta = compute_metadata(unit, cx, metas);
 456         metas.insert(*unit, meta);
 457         for unit in cx.dep_targets(unit) {
 458             metadata_of(&unit, cx, metas);
 459         }
 460     }
 461     metas[unit].clone()
 462 }
 463
 464 fn compute_metadata<'a, 'cfg>(
 465     unit: &Unit<'a>,
 466     cx: &Context<'a, 'cfg>,
 467     metas: &mut HashMap<Unit<'a>, Option<Metadata>>,
 468 ) -> Option<Metadata> {
 469     if unit.mode.is_doc_test() {
 470         // Doc tests do not have metadata.
 471         return None;
 472     }
 473     // No metadata for dylibs because of a couple issues:
 474     // - macOS encodes the dylib name in the executable,
 475     // - Windows rustc multiple files of which we can't easily link all of them.
 476     //
 477     // No metadata for bin because of an issue:
 478     // - wasm32 rustc/emcc encodes the `.wasm` name in the `.js` (rust-lang/cargo#4535).
 479     //
 480     // Two exceptions:
 481     // 1) Upstream dependencies (we aren't exporting + need to resolve name conflict),
 482     // 2) `__CARGO_DEFAULT_LIB_METADATA` env var.
 483     //
 484     // Note, however, that the compiler's build system at least wants
 485     // path dependencies (eg libstd) to have hashes in filenames. To account for
 486     // that we have an extra hack here which reads the
 487     // `__CARGO_DEFAULT_LIB_METADATA` environment variable and creates a
 488     // hash in the filename if that's present.
 489     //
 490     // This environment variable should not be relied on! It's
 491     // just here for rustbuild. We need a more principled method
 492     // doing this eventually.
 493     let bcx = &cx.bcx;
 494     let __cargo_default_lib_metadata = env::var("__CARGO_DEFAULT_LIB_METADATA");
 495     if !(unit.mode.is_any_test() || unit.mode.is_check())
 496         && (unit.target.is_dylib()
 497             || unit.target.is_cdylib()
 498             || (unit.target.is_executable() && bcx.target_triple(unit.kind).starts_with("wasm32-")))
 499         && unit.pkg.package_id().source_id().is_path()
 500         && __cargo_default_lib_metadata.is_err()
 501     {
 502         return None;
 503     }
 504
 505     let mut hasher = SipHasher::new_with_keys(0, 0);
 506
 507     // This is a generic version number that can be changed to make
 508     // backwards-incompatible changes to any file structures in the output
 509     // directory. For example, the fingerprint files or the build-script
 510     // output files. Normally cargo updates ship with rustc updates which will
 511     // cause a new hash due to the rustc version changing, but this allows
 512     // cargo to be extra careful to deal with different versions of cargo that
 513     // use the same rustc version.
 514     1.hash(&mut hasher);
 515
 516     // Unique metadata per (name, source, version) triple. This'll allow us
 517     // to pull crates from anywhere without worrying about conflicts.
 518     unit.pkg
 519         .package_id()
 520         .stable_hash(bcx.ws.root())
 521         .hash(&mut hasher);
 522
 523     // Also mix in enabled features to our metadata. This'll ensure that
 524     // when changing feature sets each lib is separately cached.
 525     unit.features.hash(&mut hasher);
 526
 527     // Mix in the target-metadata of all the dependencies of this target.
 528     {
 529         let mut deps_metadata = cx
 530             .dep_targets(unit)
 531             .iter()
 532             .map(|dep| metadata_of(dep, cx, metas))
 533             .collect::<Vec<_>>();
 534         deps_metadata.sort();
 535         deps_metadata.hash(&mut hasher);
 536     }
 537
 538     // Throw in the profile we're compiling with. This helps caching
 539     // `panic=abort` and `panic=unwind` artifacts, additionally with various
 540     // settings like debuginfo and whatnot.
 541     unit.profile.hash(&mut hasher);
 542     unit.mode.hash(&mut hasher);
 543
 544     // Throw in the rustflags we're compiling with.
 545     // This helps when the target directory is a shared cache for projects with different cargo configs,
 546     // or if the user is experimenting with different rustflags manually.
 547     let mut hash_flags = |flags: &[String]| {
 548         // Ignore some flags. These may affect reproducible builds if they affect
 549         // the path. The fingerprint will handle recompilation if these change.
 550         let mut iter = flags.iter();
 551         while let Some(flag) = iter.next() {
 552             if flag.starts_with("--remap-path-prefix=") {
 553                 continue;
 554             }
 555             if flag == "--remap-path-prefix" {
 556                 iter.next();
 557                 continue;
 558             }
 559             flag.hash(&mut hasher);
 560         }
 561     };
 562     if let Some(args) = bcx.extra_args_for(unit) {
 563         // Arguments passed to `cargo rustc`.
 564         hash_flags(args);
 565     }
 566     // Arguments passed in via RUSTFLAGS env var.
 567     let flags = if unit.mode.is_doc() {
 568         bcx.rustdocflags_args(unit)
 569     } else {
 570         bcx.rustflags_args(unit)
 571     };
 572     hash_flags(flags);
 573
 574     // Artifacts compiled for the host should have a different metadata
 575     // piece than those compiled for the target, so make sure we throw in
 576     // the unit's `kind` as well
 577     unit.kind.hash(&mut hasher);
 578
 579     // Finally throw in the target name/kind. This ensures that concurrent
 580     // compiles of targets in the same crate don't collide.
 581     unit.target.name().hash(&mut hasher);
 582     unit.target.kind().hash(&mut hasher);
 583
 584     bcx.rustc.verbose_version.hash(&mut hasher);
 585
 586     // Seed the contents of `__CARGO_DEFAULT_LIB_METADATA` to the hasher if present.
 587     // This should be the release channel, to get a different hash for each channel.
 588     if let Ok(ref channel) = __cargo_default_lib_metadata {
 589         channel.hash(&mut hasher);
 590     }
 591     Some(Metadata(hasher.finish()))
 592 }