src/tools/build-manifest/src/main.rs

   1 // Copyright 2017 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 extern crate toml;
  12 extern crate rustc_serialize;
  13
  14 use std::collections::{BTreeMap, HashMap};
  15 use std::env;
  16 use std::fs::File;
  17 use std::io::{self, Read, Write};
  18 use std::path::{PathBuf, Path};
  19 use std::process::{Command, Stdio};
  20
  21 static HOSTS: &'static [&'static str] = &[
  22     "aarch64-unknown-linux-gnu",
  23     "arm-unknown-linux-gnueabi",
  24     "arm-unknown-linux-gnueabihf",
  25     "armv7-unknown-linux-gnueabihf",
  26     "i686-apple-darwin",
  27     "i686-pc-windows-gnu",
  28     "i686-pc-windows-msvc",
  29     "i686-unknown-linux-gnu",
  30     "mips-unknown-linux-gnu",
  31     "mips64-unknown-linux-gnuabi64",
  32     "mips64el-unknown-linux-gnuabi64",
  33     "mipsel-unknown-linux-gnu",
  34     "powerpc-unknown-linux-gnu",
  35     "powerpc64-unknown-linux-gnu",
  36     "powerpc64le-unknown-linux-gnu",
  37     "s390x-unknown-linux-gnu",
  38     "x86_64-apple-darwin",
  39     "x86_64-pc-windows-gnu",
  40     "x86_64-pc-windows-msvc",
  41     "x86_64-unknown-freebsd",
  42     "x86_64-unknown-linux-gnu",
  43     "x86_64-unknown-netbsd",
  44 ];
  45
  46 static TARGETS: &'static [&'static str] = &[
  47     "aarch64-apple-ios",
  48     "aarch64-linux-android",
  49     "aarch64-unknown-linux-gnu",
  50     "arm-linux-androideabi",
  51     "arm-unknown-linux-gnueabi",
  52     "arm-unknown-linux-gnueabihf",
  53     "arm-unknown-linux-musleabi",
  54     "arm-unknown-linux-musleabihf",
  55     "armv7-apple-ios",
  56     "armv7-linux-androideabi",
  57     "armv7-unknown-linux-gnueabihf",
  58     "armv7-unknown-linux-musleabihf",
  59     "armv7s-apple-ios",
  60     "asmjs-unknown-emscripten",
  61     "i386-apple-ios",
  62     "i586-pc-windows-msvc",
  63     "i586-unknown-linux-gnu",
  64     "i686-apple-darwin",
  65     "i686-linux-android",
  66     "i686-pc-windows-gnu",
  67     "i686-pc-windows-msvc",
  68     "i686-unknown-freebsd",
  69     "i686-unknown-linux-gnu",
  70     "i686-unknown-linux-musl",
  71     "mips-unknown-linux-gnu",
  72     "mips-unknown-linux-musl",
  73     "mips64-unknown-linux-gnuabi64",
  74     "mips64el-unknown-linux-gnuabi64",
  75     "mipsel-unknown-linux-gnu",
  76     "mipsel-unknown-linux-musl",
  77     "powerpc-unknown-linux-gnu",
  78     "powerpc64-unknown-linux-gnu",
  79     "powerpc64le-unknown-linux-gnu",
  80     "s390x-unknown-linux-gnu",
  81     "wasm32-unknown-emscripten",
  82     "x86_64-apple-darwin",
  83     "x86_64-apple-ios",
  84     "x86_64-pc-windows-gnu",
  85     "x86_64-pc-windows-msvc",
  86     "x86_64-rumprun-netbsd",
  87     "x86_64-unknown-freebsd",
  88     "x86_64-unknown-linux-gnu",
  89     "x86_64-unknown-linux-musl",
  90     "x86_64-unknown-netbsd",
  91 ];
  92
  93 static MINGW: &'static [&'static str] = &[
  94     "i686-pc-windows-gnu",
  95     "x86_64-pc-windows-gnu",
  96 ];
  97
  98 struct Manifest {
  99     manifest_version: String,
 100     date: String,
 101     pkg: HashMap<String, Package>,
 102 }
 103
 104 #[derive(RustcEncodable)]
 105 struct Package {
 106     version: String,
 107     target: HashMap<String, Target>,
 108 }
 109
 110 #[derive(RustcEncodable)]
 111 struct Target {
 112     available: bool,
 113     url: Option<String>,
 114     hash: Option<String>,
 115     components: Option<Vec<Component>>,
 116     extensions: Option<Vec<Component>>,
 117 }
 118
 119 #[derive(RustcEncodable)]
 120 struct Component {
 121     pkg: String,
 122     target: String,
 123 }
 124
 125 macro_rules! t {
 126     ($e:expr) => (match $e {
 127         Ok(e) => e,
 128         Err(e) => panic!("{} failed with {}", stringify!($e), e),
 129     })
 130 }
 131
 132 struct Builder {
 133     channel: String,
 134     input: PathBuf,
 135     output: PathBuf,
 136     gpg_passphrase: String,
 137     digests: HashMap<String, String>,
 138     s3_address: String,
 139     date: String,
 140     rust_version: String,
 141     cargo_version: String,
 142 }
 143
 144 fn main() {
 145     let mut args = env::args().skip(1);
 146     let input = PathBuf::from(args.next().unwrap());
 147     let output = PathBuf::from(args.next().unwrap());
 148     let date = args.next().unwrap();
 149     let channel = args.next().unwrap();
 150     let s3_address = args.next().unwrap();
 151     let mut passphrase = String::new();
 152     t!(io::stdin().read_to_string(&mut passphrase));
 153
 154     Builder {
 155         channel: channel,
 156         input: input,
 157         output: output,
 158         gpg_passphrase: passphrase,
 159         digests: HashMap::new(),
 160         s3_address: s3_address,
 161         date: date,
 162         rust_version: String::new(),
 163         cargo_version: String::new(),
 164     }.build();
 165 }
 166
 167 impl Builder {
 168     fn build(&mut self) {
 169         self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu");
 170         self.cargo_version = self.version("cargo", "x86_64-unknown-linux-gnu");
 171
 172         self.digest_and_sign();
 173         let Manifest { manifest_version, date, pkg } = self.build_manifest();
 174
 175         // Unfortunately we can't use derive(RustcEncodable) here because the
 176         // version field is called `manifest-version`, not `manifest_version`.
 177         // In lieu of that just create the table directly here with a `BTreeMap`
 178         // and wrap it up in a `Value::Table`.
 179         let mut manifest = BTreeMap::new();
 180         manifest.insert("manifest-version".to_string(),
 181                         toml::Value::String(manifest_version));
 182         manifest.insert("date".to_string(), toml::Value::String(date));
 183         manifest.insert("pkg".to_string(), toml::encode(&pkg));
 184         let manifest = toml::Value::Table(manifest).to_string();
 185
 186         let filename = format!("channel-rust-{}.toml", self.channel);
 187         self.write_manifest(&manifest, &filename);
 188
 189         if self.channel != "beta" && self.channel != "nightly" {
 190             self.write_manifest(&manifest, "channel-rust-stable.toml");
 191         }
 192     }
 193
 194     fn digest_and_sign(&mut self) {
 195         for file in t!(self.input.read_dir()).map(|e| t!(e).path()) {
 196             let filename = file.file_name().unwrap().to_str().unwrap();
 197             let digest = self.hash(&file);
 198             self.sign(&file);
 199             assert!(self.digests.insert(filename.to_string(), digest).is_none());
 200         }
 201     }
 202
 203     fn build_manifest(&mut self) -> Manifest {
 204         let mut manifest = Manifest {
 205             manifest_version: "2".to_string(),
 206             date: self.date.to_string(),
 207             pkg: HashMap::new(),
 208         };
 209
 210         self.package("rustc", &mut manifest.pkg, HOSTS);
 211         self.package("cargo", &mut manifest.pkg, HOSTS);
 212         self.package("rust-mingw", &mut manifest.pkg, MINGW);
 213         self.package("rust-std", &mut manifest.pkg, TARGETS);
 214         self.package("rust-docs", &mut manifest.pkg, TARGETS);
 215         self.package("rust-src", &mut manifest.pkg, &["*"]);
 216
 217         let mut pkg = Package {
 218             version: self.cached_version("rust").to_string(),
 219             target: HashMap::new(),
 220         };
 221         for host in HOSTS {
 222             let filename = self.filename("rust", host);
 223             let digest = match self.digests.remove(&filename) {
 224                 Some(digest) => digest,
 225                 None => {
 226                     pkg.target.insert(host.to_string(), Target {
 227                         available: false,
 228                         url: None,
 229                         hash: None,
 230                         components: None,
 231                         extensions: None,
 232                     });
 233                     continue
 234                 }
 235             };
 236             let mut components = Vec::new();
 237             let mut extensions = Vec::new();
 238
 239             // rustc/rust-std/cargo are all required, and so is rust-mingw if it's
 240             // available for the target.
 241             components.extend(vec![
 242                 Component { pkg: "rustc".to_string(), target: host.to_string() },
 243                 Component { pkg: "rust-std".to_string(), target: host.to_string() },
 244                 Component { pkg: "cargo".to_string(), target: host.to_string() },
 245             ]);
 246             if host.contains("pc-windows-gnu") {
 247                 components.push(Component {
 248                     pkg: "rust-mingw".to_string(),
 249                     target: host.to_string(),
 250                 });
 251             }
 252
 253             // Docs, other standard libraries, and the source package are all
 254             // optional.
 255             extensions.push(Component {
 256                 pkg: "rust-docs".to_string(),
 257                 target: host.to_string(),
 258             });
 259             for target in TARGETS {
 260                 if target != host {
 261                     extensions.push(Component {
 262                         pkg: "rust-std".to_string(),
 263                         target: target.to_string(),
 264                     });
 265                 }
 266             }
 267             extensions.push(Component {
 268                 pkg: "rust-src".to_string(),
 269                 target: "*".to_string(),
 270             });
 271
 272             pkg.target.insert(host.to_string(), Target {
 273                 available: true,
 274                 url: Some(self.url("rust", host)),
 275                 hash: Some(to_hex(digest.as_ref())),
 276                 components: Some(components),
 277                 extensions: Some(extensions),
 278             });
 279         }
 280         manifest.pkg.insert("rust".to_string(), pkg);
 281
 282         return manifest
 283     }
 284
 285     fn package(&mut self,
 286                pkgname: &str,
 287                dst: &mut HashMap<String, Package>,
 288                targets: &[&str]) {
 289         let targets = targets.iter().map(|name| {
 290             let filename = self.filename(pkgname, name);
 291             let digest = match self.digests.remove(&filename) {
 292                 Some(digest) => digest,
 293                 None => {
 294                     return (name.to_string(), Target {
 295                         available: false,
 296                         url: None,
 297                         hash: None,
 298                         components: None,
 299                         extensions: None,
 300                     })
 301                 }
 302             };
 303
 304             (name.to_string(), Target {
 305                 available: true,
 306                 url: Some(self.url(pkgname, name)),
 307                 hash: Some(digest),
 308                 components: None,
 309                 extensions: None,
 310             })
 311         }).collect();
 312
 313         dst.insert(pkgname.to_string(), Package {
 314             version: self.cached_version(pkgname).to_string(),
 315             target: targets,
 316         });
 317     }
 318
 319     fn url(&self, component: &str, target: &str) -> String {
 320         format!("{}/{}/{}",
 321                 self.s3_address,
 322                 self.date,
 323                 self.filename(component, target))
 324     }
 325
 326     fn filename(&self, component: &str, target: &str) -> String {
 327         if component == "rust-src" {
 328             format!("rust-src-{}.tar.gz", self.channel)
 329         } else if component == "cargo" {
 330             format!("cargo-nightly-{}.tar.gz", target)
 331         } else {
 332             format!("{}-{}-{}.tar.gz", component, self.channel, target)
 333         }
 334     }
 335
 336     fn cached_version(&self, component: &str) -> &str {
 337         if component == "cargo" {
 338             &self.cargo_version
 339         } else {
 340             &self.rust_version
 341         }
 342     }
 343
 344     fn version(&self, component: &str, target: &str) -> String {
 345         let mut cmd = Command::new("tar");
 346         let filename = self.filename(component, target);
 347         cmd.arg("xf")
 348            .arg(self.input.join(&filename))
 349            .arg(format!("{}/version", filename.replace(".tar.gz", "")))
 350            .arg("-O");
 351         let version = t!(cmd.output());
 352         if !version.status.success() {
 353             panic!("failed to learn version:\n\n{:?}\n\n{}\n\n{}",
 354                    cmd,
 355                    String::from_utf8_lossy(&version.stdout),
 356                    String::from_utf8_lossy(&version.stderr));
 357         }
 358         String::from_utf8_lossy(&version.stdout).trim().to_string()
 359     }
 360
 361     fn hash(&self, path: &Path) -> String {
 362         let sha = t!(Command::new("shasum")
 363                         .arg("-a").arg("256")
 364                         .arg(path.file_name().unwrap())
 365                         .current_dir(path.parent().unwrap())
 366                         .output());
 367         assert!(sha.status.success());
 368
 369         let filename = path.file_name().unwrap().to_str().unwrap();
 370         let sha256 = self.output.join(format!("{}.sha256", filename));
 371         t!(t!(File::create(&sha256)).write_all(&sha.stdout));
 372
 373         let stdout = String::from_utf8_lossy(&sha.stdout);
 374         stdout.split_whitespace().next().unwrap().to_string()
 375     }
 376
 377     fn sign(&self, path: &Path) {
 378         let filename = path.file_name().unwrap().to_str().unwrap();
 379         let asc = self.output.join(format!("{}.asc", filename));
 380         println!("signing: {:?}", path);
 381         let mut cmd = Command::new("gpg");
 382         cmd.arg("--no-tty")
 383             .arg("--yes")
 384             .arg("--passphrase-fd").arg("0")
 385             .arg("--armor")
 386             .arg("--output").arg(&asc)
 387             .arg("--detach-sign").arg(path)
 388             .stdin(Stdio::piped());
 389         let mut child = t!(cmd.spawn());
 390         t!(child.stdin.take().unwrap().write_all(self.gpg_passphrase.as_bytes()));
 391         assert!(t!(child.wait()).success());
 392     }
 393
 394     fn write_manifest(&self, manifest: &str, name: &str) {
 395         let dst = self.output.join(name);
 396         t!(t!(File::create(&dst)).write_all(manifest.as_bytes()));
 397         self.hash(&dst);
 398         self.sign(&dst);
 399     }
 400 }
 401
 402 fn to_hex(digest: &[u8]) -> String {
 403     let mut ret = String::new();
 404     for byte in digest {
 405         ret.push(hex((byte & 0xf0) >> 4));
 406         ret.push(hex(byte & 0xf));
 407     }
 408     return ret;
 409
 410     fn hex(b: u8) -> char {
 411         match b {
 412             0...9 => (b'0' + b) as char,
 413             _ => (b'a' + b - 10) as char,
 414         }
 415     }
 416 }