benches/benchsuite/benches/resolve.rs

   1 use cargo::core::compiler::{CompileKind, RustcTargetData};
   2 use cargo::core::resolver::features::{CliFeatures, FeatureOpts, FeatureResolver, ForceAllTargets};
   3 use cargo::core::resolver::{HasDevUnits, ResolveBehavior};
   4 use cargo::core::{PackageIdSpec, Workspace};
   5 use cargo::ops::WorkspaceResolve;
   6 use cargo::Config;
   7 use criterion::{criterion_group, criterion_main, Criterion};
   8 use std::fs;
   9 use std::path::{Path, PathBuf};
  10 use std::process::Command;
  11 use url::Url;
  12
  13 // This is an arbitrary commit that existed when I started. This helps
  14 // ensure consistent results. It can be updated if needed, but that can
  15 // make it harder to compare results with older versions of cargo.
  16 const CRATES_IO_COMMIT: &str = "85f7bfd61ea4fee08ec68c468762e886b2aebec6";
  17
  18 fn setup() {
  19     create_home();
  20     create_target_dir();
  21     clone_index();
  22     unpack_workspaces();
  23 }
  24
  25 fn root() -> PathBuf {
  26     let mut p = PathBuf::from(env!("CARGO_TARGET_TMPDIR"));
  27     p.push("bench");
  28     p
  29 }
  30
  31 fn target_dir() -> PathBuf {
  32     let mut p = root();
  33     p.push("target");
  34     p
  35 }
  36
  37 fn cargo_home() -> PathBuf {
  38     let mut p = root();
  39     p.push("chome");
  40     p
  41 }
  42
  43 fn index() -> PathBuf {
  44     let mut p = root();
  45     p.push("index");
  46     p
  47 }
  48
  49 fn workspaces_path() -> PathBuf {
  50     let mut p = root();
  51     p.push("workspaces");
  52     p
  53 }
  54
  55 fn registry_url() -> Url {
  56     Url::from_file_path(index()).unwrap()
  57 }
  58
  59 fn create_home() {
  60     let home = cargo_home();
  61     if !home.exists() {
  62         fs::create_dir_all(&home).unwrap();
  63     }
  64     fs::write(
  65         home.join("config.toml"),
  66         format!(
  67             r#"
  68                 [source.crates-io]
  69                 replace-with = 'local-snapshot'
  70
  71                 [source.local-snapshot]
  72                 registry = '{}'
  73             "#,
  74             registry_url()
  75         ),
  76     )
  77     .unwrap();
  78 }
  79
  80 fn create_target_dir() {
  81     // This is necessary to ensure the .rustc_info.json file is written.
  82     // Otherwise it won't be written, and it is very expensive to create.
  83     if !target_dir().exists() {
  84         std::fs::create_dir_all(target_dir()).unwrap();
  85     }
  86 }
  87
  88 /// This clones crates.io at a specific point in time into tmp/index.
  89 fn clone_index() {
  90     let index = index();
  91     let maybe_git = |command: &str| {
  92         let status = Command::new("git")
  93             .current_dir(&index)
  94             .args(command.split_whitespace().collect::<Vec<_>>())
  95             .status()
  96             .expect("git should be installed");
  97         status.success()
  98     };
  99     let git = |command: &str| {
 100         if !maybe_git(command) {
 101             panic!("failed to run git command: {}", command);
 102         }
 103     };
 104     if index.exists() {
 105         if maybe_git(&format!(
 106             "rev-parse -q --verify {}^{{commit}}",
 107             CRATES_IO_COMMIT
 108         )) {
 109             // Already fetched.
 110             return;
 111         }
 112     } else {
 113         fs::create_dir_all(&index).unwrap();
 114         git("init --bare");
 115         git("remote add origin https://github.com/rust-lang/crates.io-index");
 116     }
 117     git(&format!("fetch origin {}", CRATES_IO_COMMIT));
 118     git("branch -f master FETCH_HEAD");
 119 }
 120
 121 /// This unpacks the compressed workspace skeletons into tmp/workspaces.
 122 fn unpack_workspaces() {
 123     let ws_dir = Path::new(env!("CARGO_MANIFEST_DIR"))
 124         .parent()
 125         .unwrap()
 126         .join("workspaces");
 127     let archives = fs::read_dir(ws_dir)
 128         .unwrap()
 129         .map(|e| e.unwrap().path())
 130         .filter(|p| p.extension() == Some(std::ffi::OsStr::new("tgz")));
 131     for archive in archives {
 132         let name = archive.file_stem().unwrap();
 133         let f = fs::File::open(&archive).unwrap();
 134         let f = flate2::read::GzDecoder::new(f);
 135         let dest = workspaces_path().join(&name);
 136         if dest.exists() {
 137             fs::remove_dir_all(&dest).unwrap();
 138         }
 139         let mut archive = tar::Archive::new(f);
 140         archive.unpack(workspaces_path()).unwrap();
 141     }
 142 }
 143
 144 struct ResolveInfo<'cfg> {
 145     ws: Workspace<'cfg>,
 146     requested_kinds: [CompileKind; 1],
 147     target_data: RustcTargetData<'cfg>,
 148     cli_features: CliFeatures,
 149     specs: Vec<PackageIdSpec>,
 150     has_dev_units: HasDevUnits,
 151     force_all_targets: ForceAllTargets,
 152     ws_resolve: WorkspaceResolve<'cfg>,
 153 }
 154
 155 /// Vec of `(ws_name, ws_root)`.
 156 fn workspaces() -> Vec<(String, PathBuf)> {
 157     // CARGO_BENCH_WORKSPACES can be used to override, otherwise it just uses
 158     // the workspaces in the workspaces directory.
 159     let mut ps: Vec<_> = match std::env::var_os("CARGO_BENCH_WORKSPACES") {
 160         Some(s) => std::env::split_paths(&s).collect(),
 161         None => fs::read_dir(workspaces_path())
 162             .unwrap()
 163             .map(|e| e.unwrap().path())
 164             // These currently fail in most cases on Windows due to long
 165             // filenames in the git checkouts.
 166             .filter(|p| {
 167                 !(cfg!(windows)
 168                     && matches!(p.file_name().unwrap().to_str().unwrap(), "servo" | "tikv"))
 169             })
 170             .collect(),
 171     };
 172     // Sort so it is consistent.
 173     ps.sort();
 174     ps.into_iter()
 175         .map(|p| (p.file_name().unwrap().to_str().unwrap().to_owned(), p))
 176         .collect()
 177 }
 178
 179 /// Helper for resolving a workspace. This will run the resolver once to
 180 /// download everything, and returns all the data structures that are used
 181 /// during resolution.
 182 fn do_resolve<'cfg>(config: &'cfg Config, ws_root: &Path) -> ResolveInfo<'cfg> {
 183     let requested_kinds = [CompileKind::Host];
 184     let ws = cargo::core::Workspace::new(&ws_root.join("Cargo.toml"), config).unwrap();
 185     let target_data = RustcTargetData::new(&ws, &requested_kinds).unwrap();
 186     let cli_features = CliFeatures::from_command_line(&[], false, true).unwrap();
 187     let pkgs = cargo::ops::Packages::Default;
 188     let specs = pkgs.to_package_id_specs(&ws).unwrap();
 189     let has_dev_units = HasDevUnits::Yes;
 190     let force_all_targets = ForceAllTargets::No;
 191     // Do an initial run to download anything necessary so that it does
 192     // not confuse criterion's warmup.
 193     let ws_resolve = cargo::ops::resolve_ws_with_opts(
 194         &ws,
 195         &target_data,
 196         &requested_kinds,
 197         &cli_features,
 198         &specs,
 199         has_dev_units,
 200         force_all_targets,
 201     )
 202     .unwrap();
 203     ResolveInfo {
 204         ws,
 205         requested_kinds,
 206         target_data,
 207         cli_features,
 208         specs,
 209         has_dev_units,
 210         force_all_targets,
 211         ws_resolve,
 212     }
 213 }
 214
 215 /// Creates a new Config.
 216 ///
 217 /// This is separate from `do_resolve` to deal with the ownership and lifetime.
 218 fn make_config(ws_root: &Path) -> Config {
 219     let shell = cargo::core::Shell::new();
 220     let mut config = cargo::util::Config::new(shell, ws_root.to_path_buf(), cargo_home());
 221     // Configure is needed to set the target_dir which is needed to write
 222     // the .rustc_info.json file which is very expensive.
 223     config
 224         .configure(
 225             0,
 226             false,
 227             None,
 228             false,
 229             false,
 230             false,
 231             &Some(target_dir()),
 232             &[],
 233             &[],
 234         )
 235         .unwrap();
 236     config
 237 }
 238
 239 /// Benchmark of the full `resovle_ws_with_opts` which runs the resolver
 240 /// twice, the feature resolver, and more. This is a major component of a
 241 /// regular cargo build.
 242 fn resolve_ws(c: &mut Criterion) {
 243     setup();
 244     let mut group = c.benchmark_group("resolve_ws");
 245     for (ws_name, ws_root) in workspaces() {
 246         let config = make_config(&ws_root);
 247         // The resolver info is initialized only once in a lazy fashion. This
 248         // allows criterion to skip this workspace if the user passes a filter
 249         // on the command-line (like `cargo bench -- resolve_ws/tikv`).
 250         //
 251         // Due to the way criterion works, it tends to only run the inner
 252         // iterator once, and we don't want to call `do_resolve` in every
 253         // "step", since that would just be some useless work.
 254         let mut lazy_info = None;
 255         group.bench_function(&ws_name, |b| {
 256             let ResolveInfo {
 257                 ws,
 258                 requested_kinds,
 259                 target_data,
 260                 cli_features,
 261                 specs,
 262                 has_dev_units,
 263                 force_all_targets,
 264                 ..
 265             } = lazy_info.get_or_insert_with(|| do_resolve(&config, &ws_root));
 266             b.iter(|| {
 267                 cargo::ops::resolve_ws_with_opts(
 268                     ws,
 269                     target_data,
 270                     requested_kinds,
 271                     cli_features,
 272                     specs,
 273                     *has_dev_units,
 274                     *force_all_targets,
 275                 )
 276                 .unwrap();
 277             })
 278         });
 279     }
 280     group.finish();
 281 }
 282
 283 /// Benchmark of the feature resolver.
 284 fn feature_resolver(c: &mut Criterion) {
 285     setup();
 286     let mut group = c.benchmark_group("feature_resolver");
 287     for (ws_name, ws_root) in workspaces() {
 288         let config = make_config(&ws_root);
 289         let mut lazy_info = None;
 290         group.bench_function(&ws_name, |b| {
 291             let ResolveInfo {
 292                 ws,
 293                 requested_kinds,
 294                 target_data,
 295                 cli_features,
 296                 specs,
 297                 has_dev_units,
 298                 ws_resolve,
 299                 ..
 300             } = lazy_info.get_or_insert_with(|| do_resolve(&config, &ws_root));
 301             b.iter(|| {
 302                 let feature_opts = FeatureOpts::new_behavior(ResolveBehavior::V2, *has_dev_units);
 303                 FeatureResolver::resolve(
 304                     ws,
 305                     target_data,
 306                     &ws_resolve.targeted_resolve,
 307                     &ws_resolve.pkg_set,
 308                     cli_features,
 309                     specs,
 310                     requested_kinds,
 311                     feature_opts,
 312                 )
 313                 .unwrap();
 314             })
 315         });
 316     }
 317     group.finish();
 318 }
 319
 320 // Criterion complains about the measurement time being too small, but the
 321 // measurement time doesn't seem important to me, what is more important is
 322 // the number of iterations which defaults to 100, which seems like a
 323 // reasonable default. Otherwise, the measurement time would need to be
 324 // changed per workspace. We wouldn't want to spend 60s on every workspace,
 325 // that would take too long and isn't necessary for the smaller workspaces.
 326 criterion_group!(benches, resolve_ws, feature_resolver);
 327 criterion_main!(benches);