src/tools/build-manifest/src/main.rs

   1 // Copyright 2017 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 extern crate toml;
  12 #[macro_use]
  13 extern crate serde_derive;
  14
  15 use std::collections::BTreeMap;
  16 use std::env;
  17 use std::fs::File;
  18 use std::io::{self, Read, Write};
  19 use std::path::{PathBuf, Path};
  20 use std::process::{Command, Stdio};
  21
  22 static HOSTS: &'static [&'static str] = &[
  23     "aarch64-unknown-linux-gnu",
  24     "arm-unknown-linux-gnueabi",
  25     "arm-unknown-linux-gnueabihf",
  26     "armv7-unknown-linux-gnueabihf",
  27     "i686-apple-darwin",
  28     "i686-pc-windows-gnu",
  29     "i686-pc-windows-msvc",
  30     "i686-unknown-linux-gnu",
  31     "mips-unknown-linux-gnu",
  32     "mips64-unknown-linux-gnuabi64",
  33     "mips64el-unknown-linux-gnuabi64",
  34     "mipsel-unknown-linux-gnu",
  35     "powerpc-unknown-linux-gnu",
  36     "powerpc64-unknown-linux-gnu",
  37     "powerpc64le-unknown-linux-gnu",
  38     "s390x-unknown-linux-gnu",
  39     "x86_64-apple-darwin",
  40     "x86_64-pc-windows-gnu",
  41     "x86_64-pc-windows-msvc",
  42     "x86_64-unknown-freebsd",
  43     "x86_64-unknown-linux-gnu",
  44     "x86_64-unknown-netbsd",
  45 ];
  46
  47 static TARGETS: &'static [&'static str] = &[
  48     "aarch64-apple-ios",
  49     "aarch64-linux-android",
  50     "aarch64-unknown-cloudabi",
  51     "aarch64-unknown-fuchsia",
  52     "aarch64-unknown-linux-gnu",
  53     "aarch64-unknown-linux-musl",
  54     "arm-linux-androideabi",
  55     "arm-unknown-linux-gnueabi",
  56     "arm-unknown-linux-gnueabihf",
  57     "arm-unknown-linux-musleabi",
  58     "arm-unknown-linux-musleabihf",
  59     "armv5te-unknown-linux-gnueabi",
  60     "armv7-apple-ios",
  61     "armv7-linux-androideabi",
  62     "armv7-unknown-cloudabi-eabihf",
  63     "armv7-unknown-linux-gnueabihf",
  64     "armv7-unknown-linux-musleabihf",
  65     "armv7s-apple-ios",
  66     "asmjs-unknown-emscripten",
  67     "i386-apple-ios",
  68     "i586-pc-windows-msvc",
  69     "i586-unknown-linux-gnu",
  70     "i686-apple-darwin",
  71     "i686-linux-android",
  72     "i686-pc-windows-gnu",
  73     "i686-pc-windows-msvc",
  74     "i686-unknown-cloudabi",
  75     "i686-unknown-freebsd",
  76     "i686-unknown-linux-gnu",
  77     "i686-unknown-linux-musl",
  78     "mips-unknown-linux-gnu",
  79     "mips-unknown-linux-musl",
  80     "mips64-unknown-linux-gnuabi64",
  81     "mips64el-unknown-linux-gnuabi64",
  82     "mipsel-unknown-linux-gnu",
  83     "mipsel-unknown-linux-musl",
  84     "powerpc-unknown-linux-gnu",
  85     "powerpc64-unknown-linux-gnu",
  86     "powerpc64le-unknown-linux-gnu",
  87     "s390x-unknown-linux-gnu",
  88     "sparc64-unknown-linux-gnu",
  89     "sparcv9-sun-solaris",
  90     "wasm32-unknown-emscripten",
  91     "wasm32-unknown-unknown",
  92     "x86_64-apple-darwin",
  93     "x86_64-apple-ios",
  94     "x86_64-linux-android",
  95     "x86_64-pc-windows-gnu",
  96     "x86_64-pc-windows-msvc",
  97     "x86_64-rumprun-netbsd",
  98     "x86_64-sun-solaris",
  99     "x86_64-unknown-cloudabi",
 100     "x86_64-unknown-freebsd",
 101     "x86_64-unknown-fuchsia",
 102     "x86_64-unknown-linux-gnu",
 103     "x86_64-unknown-linux-gnux32",
 104     "x86_64-unknown-linux-musl",
 105     "x86_64-unknown-netbsd",
 106     "x86_64-unknown-redox",
 107 ];
 108
 109 static MINGW: &'static [&'static str] = &[
 110     "i686-pc-windows-gnu",
 111     "x86_64-pc-windows-gnu",
 112 ];
 113
 114 #[derive(Serialize)]
 115 #[serde(rename_all = "kebab-case")]
 116 struct Manifest {
 117     manifest_version: String,
 118     date: String,
 119     pkg: BTreeMap<String, Package>,
 120     renames: BTreeMap<String, Rename>
 121 }
 122
 123 #[derive(Serialize)]
 124 struct Package {
 125     version: String,
 126     git_commit_hash: Option<String>,
 127     target: BTreeMap<String, Target>,
 128 }
 129
 130 #[derive(Serialize)]
 131 struct Rename {
 132     to: String,
 133 }
 134
 135 #[derive(Serialize)]
 136 struct Target {
 137     available: bool,
 138     url: Option<String>,
 139     hash: Option<String>,
 140     xz_url: Option<String>,
 141     xz_hash: Option<String>,
 142     components: Option<Vec<Component>>,
 143     extensions: Option<Vec<Component>>,
 144 }
 145
 146 impl Target {
 147     fn unavailable() -> Target {
 148         Target {
 149             available: false,
 150             url: None,
 151             hash: None,
 152             xz_url: None,
 153             xz_hash: None,
 154             components: None,
 155             extensions: None,
 156         }
 157     }
 158 }
 159
 160 #[derive(Serialize)]
 161 struct Component {
 162     pkg: String,
 163     target: String,
 164 }
 165
 166 macro_rules! t {
 167     ($e:expr) => (match $e {
 168         Ok(e) => e,
 169         Err(e) => panic!("{} failed with {}", stringify!($e), e),
 170     })
 171 }
 172
 173 struct Builder {
 174     rust_release: String,
 175     cargo_release: String,
 176     rls_release: String,
 177     rustfmt_release: String,
 178
 179     input: PathBuf,
 180     output: PathBuf,
 181     gpg_passphrase: String,
 182     digests: BTreeMap<String, String>,
 183     s3_address: String,
 184     date: String,
 185
 186     rust_version: Option<String>,
 187     cargo_version: Option<String>,
 188     rls_version: Option<String>,
 189     rustfmt_version: Option<String>,
 190
 191     rust_git_commit_hash: Option<String>,
 192     cargo_git_commit_hash: Option<String>,
 193     rls_git_commit_hash: Option<String>,
 194     rustfmt_git_commit_hash: Option<String>,
 195 }
 196
 197 fn main() {
 198     let mut args = env::args().skip(1);
 199     let input = PathBuf::from(args.next().unwrap());
 200     let output = PathBuf::from(args.next().unwrap());
 201     let date = args.next().unwrap();
 202     let rust_release = args.next().unwrap();
 203     let cargo_release = args.next().unwrap();
 204     let rls_release = args.next().unwrap();
 205     let rustfmt_release = args.next().unwrap();
 206     let s3_address = args.next().unwrap();
 207     let mut passphrase = String::new();
 208     t!(io::stdin().read_to_string(&mut passphrase));
 209
 210     Builder {
 211         rust_release,
 212         cargo_release,
 213         rls_release,
 214         rustfmt_release,
 215
 216         input,
 217         output,
 218         gpg_passphrase: passphrase,
 219         digests: BTreeMap::new(),
 220         s3_address,
 221         date,
 222
 223         rust_version: None,
 224         cargo_version: None,
 225         rls_version: None,
 226         rustfmt_version: None,
 227
 228         rust_git_commit_hash: None,
 229         cargo_git_commit_hash: None,
 230         rls_git_commit_hash: None,
 231         rustfmt_git_commit_hash: None,
 232     }.build();
 233 }
 234
 235 impl Builder {
 236     fn build(&mut self) {
 237         self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu");
 238         self.cargo_version = self.version("cargo", "x86_64-unknown-linux-gnu");
 239         self.rls_version = self.version("rls", "x86_64-unknown-linux-gnu");
 240         self.rustfmt_version = self.version("rustfmt", "x86_64-unknown-linux-gnu");
 241
 242         self.rust_git_commit_hash = self.git_commit_hash("rust", "x86_64-unknown-linux-gnu");
 243         self.cargo_git_commit_hash = self.git_commit_hash("cargo", "x86_64-unknown-linux-gnu");
 244         self.rls_git_commit_hash = self.git_commit_hash("rls", "x86_64-unknown-linux-gnu");
 245         self.rustfmt_git_commit_hash = self.git_commit_hash("rustfmt", "x86_64-unknown-linux-gnu");
 246
 247         self.digest_and_sign();
 248         let manifest = self.build_manifest();
 249         self.write_channel_files(&self.rust_release, &manifest);
 250
 251         if self.rust_release != "beta" && self.rust_release != "nightly" {
 252             self.write_channel_files("stable", &manifest);
 253         }
 254     }
 255
 256     fn digest_and_sign(&mut self) {
 257         for file in t!(self.input.read_dir()).map(|e| t!(e).path()) {
 258             let filename = file.file_name().unwrap().to_str().unwrap();
 259             let digest = self.hash(&file);
 260             self.sign(&file);
 261             assert!(self.digests.insert(filename.to_string(), digest).is_none());
 262         }
 263     }
 264
 265     fn build_manifest(&mut self) -> Manifest {
 266         let mut manifest = Manifest {
 267             manifest_version: "2".to_string(),
 268             date: self.date.to_string(),
 269             pkg: BTreeMap::new(),
 270             renames: BTreeMap::new(),
 271         };
 272
 273         self.package("rustc", &mut manifest.pkg, HOSTS);
 274         self.package("cargo", &mut manifest.pkg, HOSTS);
 275         self.package("rust-mingw", &mut manifest.pkg, MINGW);
 276         self.package("rust-std", &mut manifest.pkg, TARGETS);
 277         self.package("rust-docs", &mut manifest.pkg, TARGETS);
 278         self.package("rust-src", &mut manifest.pkg, &["*"]);
 279         self.package("rls-preview", &mut manifest.pkg, HOSTS);
 280         self.package("rustfmt-preview", &mut manifest.pkg, HOSTS);
 281         self.package("rust-analysis", &mut manifest.pkg, TARGETS);
 282
 283         let rls_present = manifest.pkg.contains_key("rls-preview");
 284         let rustfmt_present = manifest.pkg.contains_key("rustfmt-preview");
 285
 286         if rls_present {
 287             manifest.renames.insert("rls".to_owned(), Rename { to: "rls-preview".to_owned() });
 288         }
 289
 290         let mut pkg = Package {
 291             version: self.cached_version("rust")
 292                          .as_ref()
 293                          .expect("Couldn't find Rust version")
 294                          .clone(),
 295             git_commit_hash: self.cached_git_commit_hash("rust").clone(),
 296             target: BTreeMap::new(),
 297         };
 298         for host in HOSTS {
 299             let filename = self.filename("rust", host);
 300             let digest = match self.digests.remove(&filename) {
 301                 Some(digest) => digest,
 302                 None => {
 303                     pkg.target.insert(host.to_string(), Target::unavailable());
 304                     continue
 305                 }
 306             };
 307             let xz_filename = filename.replace(".tar.gz", ".tar.xz");
 308             let xz_digest = self.digests.remove(&xz_filename);
 309             let mut components = Vec::new();
 310             let mut extensions = Vec::new();
 311
 312             // rustc/rust-std/cargo/docs are all required, and so is rust-mingw
 313             // if it's available for the target.
 314             components.extend(vec![
 315                 Component { pkg: "rustc".to_string(), target: host.to_string() },
 316                 Component { pkg: "rust-std".to_string(), target: host.to_string() },
 317                 Component { pkg: "cargo".to_string(), target: host.to_string() },
 318                 Component { pkg: "rust-docs".to_string(), target: host.to_string() },
 319             ]);
 320             if host.contains("pc-windows-gnu") {
 321                 components.push(Component {
 322                     pkg: "rust-mingw".to_string(),
 323                     target: host.to_string(),
 324                 });
 325             }
 326
 327             if rls_present {
 328                 extensions.push(Component {
 329                     pkg: "rls-preview".to_string(),
 330                     target: host.to_string(),
 331                 });
 332             }
 333             if rustfmt_present {
 334                 extensions.push(Component {
 335                     pkg: "rustfmt-preview".to_string(),
 336                     target: host.to_string(),
 337                 });
 338             }
 339             extensions.push(Component {
 340                 pkg: "rust-analysis".to_string(),
 341                 target: host.to_string(),
 342             });
 343             for target in TARGETS {
 344                 if target != host {
 345                     extensions.push(Component {
 346                         pkg: "rust-std".to_string(),
 347                         target: target.to_string(),
 348                     });
 349                 }
 350             }
 351             extensions.push(Component {
 352                 pkg: "rust-src".to_string(),
 353                 target: "*".to_string(),
 354             });
 355
 356             pkg.target.insert(host.to_string(), Target {
 357                 available: true,
 358                 url: Some(self.url(&filename)),
 359                 hash: Some(digest),
 360                 xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
 361                 xz_hash: xz_digest,
 362                 components: Some(components),
 363                 extensions: Some(extensions),
 364             });
 365         }
 366         manifest.pkg.insert("rust".to_string(), pkg);
 367
 368         return manifest;
 369     }
 370
 371     fn package(&mut self,
 372                pkgname: &str,
 373                dst: &mut BTreeMap<String, Package>,
 374                targets: &[&str]) {
 375         let version = match *self.cached_version(pkgname) {
 376             Some(ref version) => version.clone(),
 377             None => {
 378                 println!("Skipping package {}", pkgname);
 379                 return;
 380             }
 381         };
 382
 383         let targets = targets.iter().map(|name| {
 384             let filename = self.filename(pkgname, name);
 385             let digest = match self.digests.remove(&filename) {
 386                 Some(digest) => digest,
 387                 None => return (name.to_string(), Target::unavailable()),
 388             };
 389             let xz_filename = filename.replace(".tar.gz", ".tar.xz");
 390             let xz_digest = self.digests.remove(&xz_filename);
 391
 392             (name.to_string(), Target {
 393                 available: true,
 394                 url: Some(self.url(&filename)),
 395                 hash: Some(digest),
 396                 xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
 397                 xz_hash: xz_digest,
 398                 components: None,
 399                 extensions: None,
 400             })
 401         }).collect();
 402
 403         dst.insert(pkgname.to_string(), Package {
 404             version,
 405             git_commit_hash: self.cached_git_commit_hash(pkgname).clone(),
 406             target: targets,
 407         });
 408     }
 409
 410     fn url(&self, filename: &str) -> String {
 411         format!("{}/{}/{}",
 412                 self.s3_address,
 413                 self.date,
 414                 filename)
 415     }
 416
 417     fn filename(&self, component: &str, target: &str) -> String {
 418         if component == "rust-src" {
 419             format!("rust-src-{}.tar.gz", self.rust_release)
 420         } else if component == "cargo" {
 421             format!("cargo-{}-{}.tar.gz", self.cargo_release, target)
 422         } else if component == "rls" || component == "rls-preview" {
 423             format!("rls-{}-{}.tar.gz", self.rls_release, target)
 424         } else if component == "rustfmt" || component == "rustfmt-preview" {
 425             format!("rustfmt-{}-{}.tar.gz", self.rustfmt_release, target)
 426         } else {
 427             format!("{}-{}-{}.tar.gz", component, self.rust_release, target)
 428         }
 429     }
 430
 431     fn cached_version(&self, component: &str) -> &Option<String> {
 432         if component == "cargo" {
 433             &self.cargo_version
 434         } else if component == "rls" || component == "rls-preview" {
 435             &self.rls_version
 436         } else if component == "rustfmt" || component == "rustfmt-preview" {
 437             &self.rustfmt_version
 438         } else {
 439             &self.rust_version
 440         }
 441     }
 442
 443     fn cached_git_commit_hash(&self, component: &str) -> &Option<String> {
 444         if component == "cargo" {
 445             &self.cargo_git_commit_hash
 446         } else if component == "rls" || component == "rls-preview" {
 447             &self.rls_git_commit_hash
 448         } else if component == "rustfmt" || component == "rustfmt-preview" {
 449             &self.rustfmt_git_commit_hash
 450         } else {
 451             &self.rust_git_commit_hash
 452         }
 453     }
 454
 455     fn version(&self, component: &str, target: &str) -> Option<String> {
 456         let mut cmd = Command::new("tar");
 457         let filename = self.filename(component, target);
 458         cmd.arg("xf")
 459            .arg(self.input.join(&filename))
 460            .arg(format!("{}/version", filename.replace(".tar.gz", "")))
 461            .arg("-O");
 462         let output = t!(cmd.output());
 463         if output.status.success() {
 464             Some(String::from_utf8_lossy(&output.stdout).trim().to_string())
 465         } else {
 466             // Perhaps we didn't build this package.
 467             None
 468         }
 469     }
 470
 471     fn git_commit_hash(&self, component: &str, target: &str) -> Option<String> {
 472         let mut cmd = Command::new("tar");
 473         let filename = self.filename(component, target);
 474         cmd.arg("xf")
 475            .arg(self.input.join(&filename))
 476            .arg(format!("{}/git-commit-hash", filename.replace(".tar.gz", "")))
 477            .arg("-O");
 478         let output = t!(cmd.output());
 479         if output.status.success() {
 480             Some(String::from_utf8_lossy(&output.stdout).trim().to_string())
 481         } else {
 482             None
 483         }
 484     }
 485
 486     fn hash(&self, path: &Path) -> String {
 487         let sha = t!(Command::new("shasum")
 488                         .arg("-a").arg("256")
 489                         .arg(path.file_name().unwrap())
 490                         .current_dir(path.parent().unwrap())
 491                         .output());
 492         assert!(sha.status.success());
 493
 494         let filename = path.file_name().unwrap().to_str().unwrap();
 495         let sha256 = self.output.join(format!("{}.sha256", filename));
 496         t!(t!(File::create(&sha256)).write_all(&sha.stdout));
 497
 498         let stdout = String::from_utf8_lossy(&sha.stdout);
 499         stdout.split_whitespace().next().unwrap().to_string()
 500     }
 501
 502     fn sign(&self, path: &Path) {
 503         let filename = path.file_name().unwrap().to_str().unwrap();
 504         let asc = self.output.join(format!("{}.asc", filename));
 505         println!("signing: {:?}", path);
 506         let mut cmd = Command::new("gpg");
 507         cmd.arg("--no-tty")
 508             .arg("--yes")
 509             .arg("--passphrase-fd").arg("0")
 510             .arg("--personal-digest-preferences").arg("SHA512")
 511             .arg("--armor")
 512             .arg("--output").arg(&asc)
 513             .arg("--detach-sign").arg(path)
 514             .stdin(Stdio::piped());
 515         let mut child = t!(cmd.spawn());
 516         t!(child.stdin.take().unwrap().write_all(self.gpg_passphrase.as_bytes()));
 517         assert!(t!(child.wait()).success());
 518     }
 519
 520     fn write_channel_files(&self, channel_name: &str, manifest: &Manifest) {
 521         self.write(&toml::to_string(&manifest).unwrap(), channel_name, ".toml");
 522         self.write(&manifest.date, channel_name, "-date.txt");
 523         self.write(manifest.pkg["rust"].git_commit_hash.as_ref().unwrap(),
 524                    channel_name, "-git-commit-hash.txt");
 525     }
 526
 527     fn write(&self, contents: &str, channel_name: &str, suffix: &str) {
 528         let dst = self.output.join(format!("channel-rust-{}{}", channel_name, suffix));
 529         t!(t!(File::create(&dst)).write_all(contents.as_bytes()));
 530         self.hash(&dst);
 531         self.sign(&dst);
 532     }
 533 }