src/bootstrap/dist.rs

   1 // Copyright 2016 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 //! Implementation of the various distribution aspects of the compiler.
  12 //!
  13 //! This module is responsible for creating tarballs of the standard library,
  14 //! compiler, and documentation. This ends up being what we distribute to
  15 //! everyone as well.
  16 //!
  17 //! No tarball is actually created literally in this file, but rather we shell
  18 //! out to `rust-installer` still. This may one day be replaced with bits and
  19 //! pieces of `rustup.rs`!
  20
  21 use std::fs::{self, File};
  22 use std::io::Write;
  23 use std::path::{PathBuf, Path};
  24 use std::process::Command;
  25
  26 use {Build, Compiler};
  27 use util::{cp_r, libdir, is_dylib, cp_filtered, copy};
  28
  29 pub fn package_vers(build: &Build) -> &str {
  30     match &build.config.channel[..] {
  31         "stable" => &build.release,
  32         "beta" => "beta",
  33         "nightly" => "nightly",
  34         _ => &build.release,
  35     }
  36 }
  37
  38 fn distdir(build: &Build) -> PathBuf {
  39     build.out.join("dist")
  40 }
  41
  42 pub fn tmpdir(build: &Build) -> PathBuf {
  43     build.out.join("tmp/dist")
  44 }
  45
  46 /// Builds the `rust-docs` installer component.
  47 ///
  48 /// Slurps up documentation from the `stage`'s `host`.
  49 pub fn docs(build: &Build, stage: u32, host: &str) {
  50     println!("Dist docs stage{} ({})", stage, host);
  51     let name = format!("rust-docs-{}", package_vers(build));
  52     let image = tmpdir(build).join(format!("{}-{}-image", name, name));
  53     let _ = fs::remove_dir_all(&image);
  54
  55     let dst = image.join("share/doc/rust/html");
  56     t!(fs::create_dir_all(&dst));
  57     let src = build.out.join(host).join("doc");
  58     cp_r(&src, &dst);
  59
  60     let mut cmd = Command::new("sh");
  61     cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
  62        .arg("--product-name=Rust-Documentation")
  63        .arg("--rel-manifest-dir=rustlib")
  64        .arg("--success-message=Rust-documentation-is-installed.")
  65        .arg(format!("--image-dir={}", sanitize_sh(&image)))
  66        .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
  67        .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
  68        .arg(format!("--package-name={}-{}", name, host))
  69        .arg("--component-name=rust-docs")
  70        .arg("--legacy-manifest-dirs=rustlib,cargo")
  71        .arg("--bulk-dirs=share/doc/rust/html");
  72     build.run(&mut cmd);
  73     t!(fs::remove_dir_all(&image));
  74
  75     // As part of this step, *also* copy the docs directory to a directory which
  76     // buildbot typically uploads.
  77     if host == build.config.build {
  78         let dst = distdir(build).join("doc").join(&build.package_vers);
  79         t!(fs::create_dir_all(&dst));
  80         cp_r(&src, &dst);
  81     }
  82 }
  83
  84 /// Build the `rust-mingw` installer component.
  85 ///
  86 /// This contains all the bits and pieces to run the MinGW Windows targets
  87 /// without any extra installed software (e.g. we bundle gcc, libraries, etc).
  88 /// Currently just shells out to a python script, but that should be rewritten
  89 /// in Rust.
  90 pub fn mingw(build: &Build, host: &str) {
  91     println!("Dist mingw ({})", host);
  92     let name = format!("rust-mingw-{}", package_vers(build));
  93     let image = tmpdir(build).join(format!("{}-{}-image", name, host));
  94     let _ = fs::remove_dir_all(&image);
  95
  96     // The first argument to the script is a "temporary directory" which is just
  97     // thrown away (this contains the runtime DLLs included in the rustc package
  98     // above) and the second argument is where to place all the MinGW components
  99     // (which is what we want).
 100     //
 101     // FIXME: this script should be rewritten into Rust
 102     let mut cmd = Command::new("python");
 103     cmd.arg(build.src.join("src/etc/make-win-dist.py"))
 104        .arg(tmpdir(build))
 105        .arg(&image)
 106        .arg(host);
 107     build.run(&mut cmd);
 108
 109     let mut cmd = Command::new("sh");
 110     cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
 111        .arg("--product-name=Rust-MinGW")
 112        .arg("--rel-manifest-dir=rustlib")
 113        .arg("--success-message=Rust-MinGW-is-installed.")
 114        .arg(format!("--image-dir={}", sanitize_sh(&image)))
 115        .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
 116        .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
 117        .arg(format!("--package-name={}-{}", name, host))
 118        .arg("--component-name=rust-mingw")
 119        .arg("--legacy-manifest-dirs=rustlib,cargo");
 120     build.run(&mut cmd);
 121     t!(fs::remove_dir_all(&image));
 122 }
 123
 124 /// Creates the `rustc` installer component.
 125 pub fn rustc(build: &Build, stage: u32, host: &str) {
 126     println!("Dist rustc stage{} ({})", stage, host);
 127     let name = format!("rustc-{}", package_vers(build));
 128     let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 129     let _ = fs::remove_dir_all(&image);
 130     let overlay = tmpdir(build).join(format!("{}-{}-overlay", name, host));
 131     let _ = fs::remove_dir_all(&overlay);
 132
 133     // Prepare the rustc "image", what will actually end up getting installed
 134     prepare_image(build, stage, host, &image);
 135
 136     // Prepare the overlay which is part of the tarball but won't actually be
 137     // installed
 138     let cp = |file: &str| {
 139         install(&build.src.join(file), &overlay, 0o644);
 140     };
 141     cp("COPYRIGHT");
 142     cp("LICENSE-APACHE");
 143     cp("LICENSE-MIT");
 144     cp("README.md");
 145     // tiny morsel of metadata is used by rust-packaging
 146     let version = &build.version;
 147     t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
 148
 149     // On MinGW we've got a few runtime DLL dependencies that we need to
 150     // include. The first argument to this script is where to put these DLLs
 151     // (the image we're creating), and the second argument is a junk directory
 152     // to ignore all other MinGW stuff the script creates.
 153     //
 154     // On 32-bit MinGW we're always including a DLL which needs some extra
 155     // licenses to distribute. On 64-bit MinGW we don't actually distribute
 156     // anything requiring us to distribute a license, but it's likely the
 157     // install will *also* include the rust-mingw package, which also needs
 158     // licenses, so to be safe we just include it here in all MinGW packages.
 159     //
 160     // FIXME: this script should be rewritten into Rust
 161     if host.contains("pc-windows-gnu") {
 162         let mut cmd = Command::new("python");
 163         cmd.arg(build.src.join("src/etc/make-win-dist.py"))
 164            .arg(&image)
 165            .arg(tmpdir(build))
 166            .arg(host);
 167         build.run(&mut cmd);
 168
 169         let dst = image.join("share/doc");
 170         t!(fs::create_dir_all(&dst));
 171         cp_r(&build.src.join("src/etc/third-party"), &dst);
 172     }
 173
 174     // Finally, wrap everything up in a nice tarball!
 175     let mut cmd = Command::new("sh");
 176     cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
 177        .arg("--product-name=Rust")
 178        .arg("--rel-manifest-dir=rustlib")
 179        .arg("--success-message=Rust-is-ready-to-roll.")
 180        .arg(format!("--image-dir={}", sanitize_sh(&image)))
 181        .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
 182        .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
 183        .arg(format!("--non-installed-overlay={}", sanitize_sh(&overlay)))
 184        .arg(format!("--package-name={}-{}", name, host))
 185        .arg("--component-name=rustc")
 186        .arg("--legacy-manifest-dirs=rustlib,cargo");
 187     build.run(&mut cmd);
 188     t!(fs::remove_dir_all(&image));
 189     t!(fs::remove_dir_all(&overlay));
 190
 191     fn prepare_image(build: &Build, stage: u32, host: &str, image: &Path) {
 192         let src = build.sysroot(&Compiler::new(stage, host));
 193         let libdir = libdir(host);
 194
 195         // Copy rustc/rustdoc binaries
 196         t!(fs::create_dir_all(image.join("bin")));
 197         cp_r(&src.join("bin"), &image.join("bin"));
 198
 199         // Copy runtime DLLs needed by the compiler
 200         if libdir != "bin" {
 201             for entry in t!(src.join(libdir).read_dir()).map(|e| t!(e)) {
 202                 let name = entry.file_name();
 203                 if let Some(s) = name.to_str() {
 204                     if is_dylib(s) {
 205                         install(&entry.path(), &image.join(libdir), 0o644);
 206                     }
 207                 }
 208             }
 209         }
 210
 211         // Man pages
 212         t!(fs::create_dir_all(image.join("share/man/man1")));
 213         cp_r(&build.src.join("man"), &image.join("share/man/man1"));
 214
 215         // Debugger scripts
 216         debugger_scripts(build, &image, host);
 217
 218         // Misc license info
 219         let cp = |file: &str| {
 220             install(&build.src.join(file), &image.join("share/doc/rust"), 0o644);
 221         };
 222         cp("COPYRIGHT");
 223         cp("LICENSE-APACHE");
 224         cp("LICENSE-MIT");
 225         cp("README.md");
 226     }
 227 }
 228
 229 /// Copies debugger scripts for `host` into the `sysroot` specified.
 230 pub fn debugger_scripts(build: &Build,
 231                         sysroot: &Path,
 232                         host: &str) {
 233     let cp_debugger_script = |file: &str| {
 234         let dst = sysroot.join("lib/rustlib/etc");
 235         t!(fs::create_dir_all(&dst));
 236         install(&build.src.join("src/etc/").join(file), &dst, 0o644);
 237     };
 238     if host.contains("windows-msvc") {
 239         // no debugger scripts
 240     } else {
 241         cp_debugger_script("debugger_pretty_printers_common.py");
 242
 243         // gdb debugger scripts
 244         install(&build.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 245                 0o755);
 246
 247         cp_debugger_script("gdb_load_rust_pretty_printers.py");
 248         cp_debugger_script("gdb_rust_pretty_printing.py");
 249
 250         // lldb debugger scripts
 251         install(&build.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 252                 0o755);
 253
 254         cp_debugger_script("lldb_rust_formatters.py");
 255     }
 256 }
 257
 258 /// Creates the `rust-std` installer component as compiled by `compiler` for the
 259 /// target `target`.
 260 pub fn std(build: &Build, compiler: &Compiler, target: &str) {
 261     println!("Dist std stage{} ({} -> {})", compiler.stage, compiler.host,
 262              target);
 263     let name = format!("rust-std-{}", package_vers(build));
 264     let image = tmpdir(build).join(format!("{}-{}-image", name, target));
 265     let _ = fs::remove_dir_all(&image);
 266
 267     let dst = image.join("lib/rustlib").join(target);
 268     t!(fs::create_dir_all(&dst));
 269     let src = build.sysroot(compiler).join("lib/rustlib");
 270     cp_r(&src.join(target), &dst);
 271
 272     let mut cmd = Command::new("sh");
 273     cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
 274        .arg("--product-name=Rust")
 275        .arg("--rel-manifest-dir=rustlib")
 276        .arg("--success-message=std-is-standing-at-the-ready.")
 277        .arg(format!("--image-dir={}", sanitize_sh(&image)))
 278        .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
 279        .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
 280        .arg(format!("--package-name={}-{}", name, target))
 281        .arg(format!("--component-name=rust-std-{}", target))
 282        .arg("--legacy-manifest-dirs=rustlib,cargo");
 283     build.run(&mut cmd);
 284     t!(fs::remove_dir_all(&image));
 285 }
 286
 287 /// Creates the `rust-src` installer component and the plain source tarball
 288 pub fn rust_src(build: &Build) {
 289     println!("Dist src");
 290     let plain_name = format!("rustc-{}-src", package_vers(build));
 291     let name = format!("rust-src-{}", package_vers(build));
 292     let image = tmpdir(build).join(format!("{}-image", name));
 293     let _ = fs::remove_dir_all(&image);
 294
 295     let dst = image.join("lib/rustlib/src");
 296     let dst_src = dst.join("rust");
 297     let plain_dst_src = dst.join(&plain_name);
 298     t!(fs::create_dir_all(&dst_src));
 299
 300     // This is the set of root paths which will become part of the source package
 301     let src_files = [
 302         "COPYRIGHT",
 303         "LICENSE-APACHE",
 304         "LICENSE-MIT",
 305         "CONTRIBUTING.md",
 306         "README.md",
 307         "RELEASES.md",
 308         "configure",
 309         "Makefile.in"
 310     ];
 311     let src_dirs = [
 312         "man",
 313         "src",
 314         "mk"
 315     ];
 316
 317     let filter_fn = move |path: &Path| {
 318         let spath = match path.to_str() {
 319             Some(path) => path,
 320             None => return false,
 321         };
 322         if spath.ends_with("~") || spath.ends_with(".pyc") {
 323             return false
 324         }
 325         if spath.contains("llvm/test") || spath.contains("llvm\\test") {
 326             if spath.ends_with(".ll") ||
 327                spath.ends_with(".td") ||
 328                spath.ends_with(".s") {
 329                 return false
 330             }
 331         }
 332
 333         let excludes = [
 334             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 335             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 336             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 337             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 338         ];
 339         !path.iter()
 340              .map(|s| s.to_str().unwrap())
 341              .any(|s| excludes.contains(&s))
 342     };
 343
 344     // Copy the directories using our filter
 345     for item in &src_dirs {
 346         let dst = &dst_src.join(item);
 347         t!(fs::create_dir(dst));
 348         cp_filtered(&build.src.join(item), dst, &filter_fn);
 349     }
 350     // Copy the files normally
 351     for item in &src_files {
 352         copy(&build.src.join(item), &dst_src.join(item));
 353     }
 354
 355     // Create source tarball in rust-installer format
 356     let mut cmd = Command::new("sh");
 357     cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
 358        .arg("--product-name=Rust")
 359        .arg("--rel-manifest-dir=rustlib")
 360        .arg("--success-message=Awesome-Source.")
 361        .arg(format!("--image-dir={}", sanitize_sh(&image)))
 362        .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
 363        .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
 364        .arg(format!("--package-name={}", name))
 365        .arg("--component-name=rust-src")
 366        .arg("--legacy-manifest-dirs=rustlib,cargo");
 367     build.run(&mut cmd);
 368
 369     // Rename directory, so that root folder of tarball has the correct name
 370     t!(fs::rename(&dst_src, &plain_dst_src));
 371
 372     // Create the version file
 373     write_file(&plain_dst_src.join("version"), build.version.as_bytes());
 374
 375     // Create plain source tarball
 376     let mut cmd = Command::new("tar");
 377     cmd.arg("-czf").arg(sanitize_sh(&distdir(build).join(&format!("{}.tar.gz", plain_name))))
 378        .arg(&plain_name)
 379        .current_dir(&dst);
 380     build.run(&mut cmd);
 381
 382     t!(fs::remove_dir_all(&image));
 383 }
 384
 385 fn install(src: &Path, dstdir: &Path, perms: u32) {
 386     let dst = dstdir.join(src.file_name().unwrap());
 387     t!(fs::create_dir_all(dstdir));
 388     t!(fs::copy(src, &dst));
 389     chmod(&dst, perms);
 390 }
 391
 392 #[cfg(unix)]
 393 fn chmod(path: &Path, perms: u32) {
 394     use std::os::unix::fs::*;
 395     t!(fs::set_permissions(path, fs::Permissions::from_mode(perms)));
 396 }
 397 #[cfg(windows)]
 398 fn chmod(_path: &Path, _perms: u32) {}
 399
 400 // We have to run a few shell scripts, which choke quite a bit on both `\`
 401 // characters and on `C:\` paths, so normalize both of them away.
 402 pub fn sanitize_sh(path: &Path) -> String {
 403     let path = path.to_str().unwrap().replace("\\", "/");
 404     return change_drive(&path).unwrap_or(path);
 405
 406     fn change_drive(s: &str) -> Option<String> {
 407         let mut ch = s.chars();
 408         let drive = ch.next().unwrap_or('C');
 409         if ch.next() != Some(':') {
 410             return None
 411         }
 412         if ch.next() != Some('/') {
 413             return None
 414         }
 415         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 416     }
 417 }
 418
 419 fn write_file(path: &Path, data: &[u8]) {
 420     let mut vf = t!(fs::File::create(path));
 421     t!(vf.write_all(data));
 422 }