1 #![deny(rust_2018_idioms)]
6 use std
::collections
::BTreeMap
;
9 use std
::io
::{self, Read, Write}
;
10 use std
::path
::{PathBuf, Path}
;
11 use std
::process
::{Command, Stdio}
;
13 static HOSTS
: &[&str] = &[
14 "aarch64-unknown-linux-gnu",
15 "arm-unknown-linux-gnueabi",
16 "arm-unknown-linux-gnueabihf",
17 "armv7-unknown-linux-gnueabihf",
19 "i686-pc-windows-gnu",
20 "i686-pc-windows-msvc",
21 "i686-unknown-linux-gnu",
22 "mips-unknown-linux-gnu",
23 "mips64-unknown-linux-gnuabi64",
24 "mips64el-unknown-linux-gnuabi64",
25 "mipsel-unknown-linux-gnu",
26 "mipsisa32r6-unknown-linux-gnu",
27 "mipsisa32r6el-unknown-linux-gnu",
28 "mipsisa64r6-unknown-linux-gnuabi64",
29 "mipsisa64r6el-unknown-linux-gnuabi64",
30 "powerpc-unknown-linux-gnu",
31 "powerpc64-unknown-linux-gnu",
32 "powerpc64le-unknown-linux-gnu",
33 "s390x-unknown-linux-gnu",
34 "x86_64-apple-darwin",
35 "x86_64-pc-windows-gnu",
36 "x86_64-pc-windows-msvc",
37 "x86_64-unknown-freebsd",
38 "x86_64-unknown-linux-gnu",
39 "x86_64-unknown-linux-musl",
40 "x86_64-unknown-netbsd",
43 static TARGETS
: &[&str] = &[
46 "aarch64-linux-android",
47 "aarch64-pc-windows-msvc",
48 "aarch64-unknown-cloudabi",
49 "aarch64-unknown-linux-gnu",
50 "aarch64-unknown-linux-musl",
51 "arm-linux-androideabi",
52 "arm-unknown-linux-gnueabi",
53 "arm-unknown-linux-gnueabihf",
54 "arm-unknown-linux-musleabi",
55 "arm-unknown-linux-musleabihf",
56 "armv5te-unknown-linux-gnueabi",
57 "armv5te-unknown-linux-musleabi",
59 "armv7-linux-androideabi",
60 "thumbv7neon-linux-androideabi",
61 "armv7-unknown-linux-gnueabihf",
62 "thumbv7neon-unknown-linux-gnueabihf",
63 "armv7-unknown-linux-musleabihf",
65 "armebv7r-none-eabihf",
69 "asmjs-unknown-emscripten",
71 "i586-pc-windows-msvc",
72 "i586-unknown-linux-gnu",
73 "i586-unknown-linux-musl",
76 "i686-pc-windows-gnu",
77 "i686-pc-windows-msvc",
78 "i686-unknown-freebsd",
79 "i686-unknown-linux-gnu",
80 "i686-unknown-linux-musl",
81 "mips-unknown-linux-gnu",
82 "mips-unknown-linux-musl",
83 "mips64-unknown-linux-gnuabi64",
84 "mips64el-unknown-linux-gnuabi64",
85 "mipsisa32r6-unknown-linux-gnu",
86 "mipsisa32r6el-unknown-linux-gnu",
87 "mipsisa64r6-unknown-linux-gnuabi64",
88 "mipsisa64r6el-unknown-linux-gnuabi64",
89 "mipsel-unknown-linux-gnu",
90 "mipsel-unknown-linux-musl",
91 "nvptx64-nvidia-cuda",
92 "powerpc-unknown-linux-gnu",
93 "powerpc64-unknown-linux-gnu",
94 "powerpc64le-unknown-linux-gnu",
95 "riscv32imc-unknown-none-elf",
96 "riscv32imac-unknown-none-elf",
97 "riscv64imac-unknown-none-elf",
98 "riscv64gc-unknown-none-elf",
99 "s390x-unknown-linux-gnu",
100 "sparc64-unknown-linux-gnu",
101 "sparcv9-sun-solaris",
102 "thumbv6m-none-eabi",
103 "thumbv7em-none-eabi",
104 "thumbv7em-none-eabihf",
105 "thumbv7m-none-eabi",
106 "thumbv8m.base-none-eabi",
107 "thumbv8m.main-none-eabi",
108 "thumbv8m.main-none-eabihf",
109 "wasm32-unknown-emscripten",
110 "wasm32-unknown-unknown",
111 "wasm32-unknown-wasi",
112 "x86_64-apple-darwin",
114 "x86_64-fortanix-unknown-sgx",
116 "x86_64-linux-android",
117 "x86_64-pc-windows-gnu",
118 "x86_64-pc-windows-msvc",
119 "x86_64-rumprun-netbsd",
120 "x86_64-sun-solaris",
121 "x86_64-unknown-cloudabi",
122 "x86_64-unknown-freebsd",
123 "x86_64-unknown-linux-gnu",
124 "x86_64-unknown-linux-gnux32",
125 "x86_64-unknown-linux-musl",
126 "x86_64-unknown-netbsd",
127 "x86_64-unknown-redox",
130 static DOCS_TARGETS
: &[&str] = &[
132 "i686-pc-windows-gnu",
133 "i686-pc-windows-msvc",
134 "i686-unknown-linux-gnu",
135 "x86_64-apple-darwin",
136 "x86_64-pc-windows-gnu",
137 "x86_64-pc-windows-msvc",
138 "x86_64-unknown-linux-gnu",
141 static MINGW
: &[&str] = &[
142 "i686-pc-windows-gnu",
143 "x86_64-pc-windows-gnu",
147 #[serde(rename_all = "kebab-case")]
149 manifest_version
: String
,
151 pkg
: BTreeMap
<String
, Package
>,
152 renames
: BTreeMap
<String
, Rename
>,
153 profiles
: BTreeMap
<String
, Vec
<String
>>,
159 git_commit_hash
: Option
<String
>,
160 target
: BTreeMap
<String
, Target
>,
168 #[derive(Serialize, Default)]
172 hash
: Option
<String
>,
173 xz_url
: Option
<String
>,
174 xz_hash
: Option
<String
>,
175 components
: Option
<Vec
<Component
>>,
176 extensions
: Option
<Vec
<Component
>>,
180 fn unavailable() -> Self { Self::default() }
190 fn from_str(pkg
: &str, target
: &str) -> Self {
191 Self { pkg: pkg.to_string(), target: target.to_string() }
196 ($e
:expr
) => (match $e
{
198 Err(e
) => panic
!("{} failed with {}", stringify
!($e
), e
),
203 rust_release
: String
,
204 cargo_release
: String
,
206 clippy_release
: String
,
207 rustfmt_release
: String
,
208 llvm_tools_release
: String
,
209 lldb_release
: String
,
210 miri_release
: String
,
214 gpg_passphrase
: String
,
215 digests
: BTreeMap
<String
, String
>,
219 rust_version
: Option
<String
>,
220 cargo_version
: Option
<String
>,
221 rls_version
: Option
<String
>,
222 clippy_version
: Option
<String
>,
223 rustfmt_version
: Option
<String
>,
224 llvm_tools_version
: Option
<String
>,
225 lldb_version
: Option
<String
>,
226 miri_version
: Option
<String
>,
228 rust_git_commit_hash
: Option
<String
>,
229 cargo_git_commit_hash
: Option
<String
>,
230 rls_git_commit_hash
: Option
<String
>,
231 clippy_git_commit_hash
: Option
<String
>,
232 rustfmt_git_commit_hash
: Option
<String
>,
233 llvm_tools_git_commit_hash
: Option
<String
>,
234 lldb_git_commit_hash
: Option
<String
>,
235 miri_git_commit_hash
: Option
<String
>,
241 // Avoid signing packages while manually testing
242 // Do NOT set this envvar in CI
243 let should_sign
= env
::var("BUILD_MANIFEST_DISABLE_SIGNING").is_err();
245 // Safety check to ensure signing is always enabled on CI
246 // The CI environment variable is set by both Travis and AppVeyor
247 if !should_sign
&& env
::var("CI").is_ok() {
248 println
!("The 'BUILD_MANIFEST_DISABLE_SIGNING' env var can't be enabled on CI.");
249 println
!("If you're not running this on CI, unset the 'CI' env var.");
253 let mut args
= env
::args().skip(1);
254 let input
= PathBuf
::from(args
.next().unwrap());
255 let output
= PathBuf
::from(args
.next().unwrap());
256 let date
= args
.next().unwrap();
257 let rust_release
= args
.next().unwrap();
258 let s3_address
= args
.next().unwrap();
259 let cargo_release
= args
.next().unwrap();
260 let rls_release
= args
.next().unwrap();
261 let clippy_release
= args
.next().unwrap();
262 let miri_release
= args
.next().unwrap();
263 let rustfmt_release
= args
.next().unwrap();
264 let llvm_tools_release
= args
.next().unwrap();
265 let lldb_release
= args
.next().unwrap();
267 // Do not ask for a passphrase while manually testing
268 let mut passphrase
= String
::new();
270 t
!(io
::stdin().read_to_string(&mut passphrase
));
285 gpg_passphrase
: passphrase
,
286 digests
: BTreeMap
::new(),
293 clippy_version
: None
,
294 rustfmt_version
: None
,
295 llvm_tools_version
: None
,
299 rust_git_commit_hash
: None
,
300 cargo_git_commit_hash
: None
,
301 rls_git_commit_hash
: None
,
302 clippy_git_commit_hash
: None
,
303 rustfmt_git_commit_hash
: None
,
304 llvm_tools_git_commit_hash
: None
,
305 lldb_git_commit_hash
: None
,
306 miri_git_commit_hash
: None
,
312 enum PkgType { RustSrc, Cargo, Rls, Clippy, Rustfmt, LlvmTools, Lldb, Miri, Other }
315 fn from_component(component
: &str) -> Self {
318 "rust-src" => RustSrc
,
320 "rls" | "rls-preview" => Rls
,
321 "clippy" | "clippy-preview" => Clippy
,
322 "rustfmt" | "rustfmt-preview" => Rustfmt
,
323 "llvm-tools" | "llvm-tools-preview" => LlvmTools
,
324 "lldb" | "lldb-preview" => Lldb
,
325 "miri" | "miri-preview" => Miri
,
332 fn build(&mut self) {
333 self.rust_version
= self.version("rust", "x86_64-unknown-linux-gnu");
334 self.cargo_version
= self.version("cargo", "x86_64-unknown-linux-gnu");
335 self.rls_version
= self.version("rls", "x86_64-unknown-linux-gnu");
336 self.clippy_version
= self.version("clippy", "x86_64-unknown-linux-gnu");
337 self.rustfmt_version
= self.version("rustfmt", "x86_64-unknown-linux-gnu");
338 self.llvm_tools_version
= self.version("llvm-tools", "x86_64-unknown-linux-gnu");
339 // lldb is only built for macOS.
340 self.lldb_version
= self.version("lldb", "x86_64-apple-darwin");
341 self.miri_version
= self.version("miri", "x86_64-unknown-linux-gnu");
343 self.rust_git_commit_hash
= self.git_commit_hash("rust", "x86_64-unknown-linux-gnu");
344 self.cargo_git_commit_hash
= self.git_commit_hash("cargo", "x86_64-unknown-linux-gnu");
345 self.rls_git_commit_hash
= self.git_commit_hash("rls", "x86_64-unknown-linux-gnu");
346 self.clippy_git_commit_hash
= self.git_commit_hash("clippy", "x86_64-unknown-linux-gnu");
347 self.rustfmt_git_commit_hash
= self.git_commit_hash("rustfmt", "x86_64-unknown-linux-gnu");
348 self.llvm_tools_git_commit_hash
= self.git_commit_hash("llvm-tools",
349 "x86_64-unknown-linux-gnu");
350 self.lldb_git_commit_hash
= self.git_commit_hash("lldb", "x86_64-unknown-linux-gnu");
351 self.miri_git_commit_hash
= self.git_commit_hash("miri", "x86_64-unknown-linux-gnu");
353 self.digest_and_sign();
354 let manifest
= self.build_manifest();
355 self.write_channel_files(&self.rust_release
, &manifest
);
357 if self.rust_release
!= "beta" && self.rust_release
!= "nightly" {
358 self.write_channel_files("stable", &manifest
);
362 fn digest_and_sign(&mut self) {
363 for file
in t
!(self.input
.read_dir()).map(|e
| t
!(e
).path()) {
364 let filename
= file
.file_name().unwrap().to_str().unwrap();
365 let digest
= self.hash(&file
);
367 assert
!(self.digests
.insert(filename
.to_string(), digest
).is_none());
371 fn build_manifest(&mut self) -> Manifest
{
372 let mut manifest
= Manifest
{
373 manifest_version
: "2".to_string(),
374 date
: self.date
.to_string(),
375 pkg
: BTreeMap
::new(),
376 renames
: BTreeMap
::new(),
377 profiles
: BTreeMap
::new(),
379 self.add_packages_to(&mut manifest
);
380 self.add_profiles_to(&mut manifest
);
381 self.add_renames_to(&mut manifest
);
382 manifest
.pkg
.insert("rust".to_string(), self.rust_package(&manifest
));
386 fn add_packages_to(&mut self, manifest
: &mut Manifest
) {
387 let mut package
= |name
, targets
| self.package(name
, &mut manifest
.pkg
, targets
);
388 package("rustc", HOSTS
);
389 package("cargo", HOSTS
);
390 package("rust-mingw", MINGW
);
391 package("rust-std", TARGETS
);
392 package("rust-docs", DOCS_TARGETS
);
393 package("rust-src", &["*"]);
394 package("rls-preview", HOSTS
);
395 package("clippy-preview", HOSTS
);
396 package("miri-preview", HOSTS
);
397 package("rustfmt-preview", HOSTS
);
398 package("rust-analysis", TARGETS
);
399 package("llvm-tools-preview", TARGETS
);
400 package("lldb-preview", TARGETS
);
403 fn add_profiles_to(&mut self, manifest
: &mut Manifest
) {
404 let mut profile
= |name
, pkgs
| self.profile(name
, &mut manifest
.profiles
, pkgs
);
405 profile("minimal", &["rustc", "cargo", "rust-std", "rust-mingw"]);
406 profile("default", &[
407 "rustc", "cargo", "rust-std", "rust-mingw",
408 "rust-docs", "rustfmt-preview", "clippy-preview"
410 profile("complete", &[
411 "rustc", "cargo", "rust-std", "rust-mingw",
412 "rust-docs", "rustfmt-preview", "clippy-preview",
413 "rls-preview", "rust-src", "llvm-tools-preview",
414 "lldb-preview", "rust-analysis", "miri-preview"
418 fn add_renames_to(&self, manifest
: &mut Manifest
) {
419 let mut rename
= |from
: &str, to
: &str| manifest
.renames
.insert(
421 Rename { to: to.to_owned() }
423 rename("rls", "rls-preview");
424 rename("rustfmt", "rustfmt-preview");
425 rename("clippy", "clippy-preview");
426 rename("miri", "miri-preview");
429 fn rust_package(&mut self, manifest
: &Manifest
) -> Package
{
430 let mut pkg
= Package
{
431 version
: self.cached_version("rust")
433 .expect("Couldn't find Rust version")
435 git_commit_hash
: self.cached_git_commit_hash("rust").clone(),
436 target
: BTreeMap
::new(),
439 if let Some(target
) = self.target_host_combination(host
, &manifest
) {
440 pkg
.target
.insert(host
.to_string(), target
);
442 pkg
.target
.insert(host
.to_string(), Target
::unavailable());
449 fn target_host_combination(&mut self, host
: &str, manifest
: &Manifest
) -> Option
<Target
> {
450 let filename
= self.filename("rust", host
);
451 let digest
= self.digests
.remove(&filename
)?
;
452 let xz_filename
= filename
.replace(".tar.gz", ".tar.xz");
453 let xz_digest
= self.digests
.remove(&xz_filename
);
454 let mut components
= Vec
::new();
455 let mut extensions
= Vec
::new();
457 let host_component
= |pkg
| Component
::from_str(pkg
, host
);
459 // rustc/rust-std/cargo/docs are all required,
460 // and so is rust-mingw if it's available for the target.
461 components
.extend(vec
![
462 host_component("rustc"),
463 host_component("rust-std"),
464 host_component("cargo"),
465 host_component("rust-docs"),
467 if host
.contains("pc-windows-gnu") {
468 components
.push(host_component("rust-mingw"));
471 // Tools are always present in the manifest,
472 // but might be marked as unavailable if they weren't built.
473 extensions
.extend(vec
![
474 host_component("clippy-preview"),
475 host_component("miri-preview"),
476 host_component("rls-preview"),
477 host_component("rustfmt-preview"),
478 host_component("llvm-tools-preview"),
479 host_component("lldb-preview"),
480 host_component("rust-analysis"),
485 .filter(|&&target
| target
!= host
)
486 .map(|target
| Component
::from_str("rust-std", target
))
488 extensions
.push(Component
::from_str("rust-src", "*"));
490 // If the components/extensions don't actually exist for this
491 // particular host/target combination then nix it entirely from our
493 let has_component
= |c
: &Component
| {
497 let pkg
= match manifest
.pkg
.get(&c
.pkg
) {
499 None
=> return false,
501 pkg
.target
.get(&c
.target
).is_some()
503 extensions
.retain(&has_component
);
504 components
.retain(&has_component
);
508 url
: Some(self.url(&filename
)),
510 xz_url
: xz_digest
.as_ref().map(|_
| self.url(&xz_filename
)),
512 components
: Some(components
),
513 extensions
: Some(extensions
),
517 fn profile(&mut self,
519 dst
: &mut BTreeMap
<String
, Vec
<String
>>,
521 dst
.insert(profile_name
.to_owned(), pkgs
.iter().map(|s
| (*s
).to_owned()).collect());
524 fn package(&mut self,
526 dst
: &mut BTreeMap
<String
, Package
>,
528 let (version
, mut is_present
) = self.cached_version(pkgname
)
531 .map(|version
| (version
, true))
532 .unwrap_or_default();
534 // miri needs to build std with xargo, which doesn't allow stable/beta:
535 // <https://github.com/japaric/xargo/pull/204#issuecomment-374888868>
536 if pkgname
== "miri-preview" && self.rust_release
!= "nightly" {
537 is_present
= false; // ignore it
540 let targets
= targets
.iter().map(|name
| {
542 let filename
= self.filename(pkgname
, name
);
543 let digest
= match self.digests
.remove(&filename
) {
544 Some(digest
) => digest
,
545 None
=> return (name
.to_string(), Target
::unavailable()),
547 let xz_filename
= filename
.replace(".tar.gz", ".tar.xz");
548 let xz_digest
= self.digests
.remove(&xz_filename
);
550 (name
.to_string(), Target
{
552 url
: Some(self.url(&filename
)),
554 xz_url
: xz_digest
.as_ref().map(|_
| self.url(&xz_filename
)),
560 // If the component is not present for this build add it anyway but mark it as
561 // unavailable -- this way rustup won't allow upgrades without --force
562 (name
.to_string(), Target
::unavailable())
566 dst
.insert(pkgname
.to_string(), Package
{
568 git_commit_hash
: self.cached_git_commit_hash(pkgname
).clone(),
573 fn url(&self, filename
: &str) -> String
{
580 fn filename(&self, component
: &str, target
: &str) -> String
{
582 match PkgType
::from_component(component
) {
583 RustSrc
=> format
!("rust-src-{}.tar.gz", self.rust_release
),
584 Cargo
=> format
!("cargo-{}-{}.tar.gz", self.cargo_release
, target
),
585 Rls
=> format
!("rls-{}-{}.tar.gz", self.rls_release
, target
),
586 Clippy
=> format
!("clippy-{}-{}.tar.gz", self.clippy_release
, target
),
587 Rustfmt
=> format
!("rustfmt-{}-{}.tar.gz", self.rustfmt_release
, target
),
588 LlvmTools
=> format
!("llvm-tools-{}-{}.tar.gz", self.llvm_tools_release
, target
),
589 Lldb
=> format
!("lldb-{}-{}.tar.gz", self.lldb_release
, target
),
590 Miri
=> format
!("miri-{}-{}.tar.gz", self.miri_release
, target
),
591 Other
=> format
!("{}-{}-{}.tar.gz", component
, self.rust_release
, target
),
595 fn cached_version(&self, component
: &str) -> &Option
<String
> {
597 match PkgType
::from_component(component
) {
598 Cargo
=> &self.cargo_version
,
599 Rls
=> &self.rls_version
,
600 Clippy
=> &self.clippy_version
,
601 Rustfmt
=> &self.rustfmt_version
,
602 LlvmTools
=> &self.llvm_tools_version
,
603 Lldb
=> &self.lldb_version
,
604 Miri
=> &self.miri_version
,
605 _
=> &self.rust_version
,
609 fn cached_git_commit_hash(&self, component
: &str) -> &Option
<String
> {
611 match PkgType
::from_component(component
) {
612 Cargo
=> &self.cargo_git_commit_hash
,
613 Rls
=> &self.rls_git_commit_hash
,
614 Clippy
=> &self.clippy_git_commit_hash
,
615 Rustfmt
=> &self.rustfmt_git_commit_hash
,
616 LlvmTools
=> &self.llvm_tools_git_commit_hash
,
617 Lldb
=> &self.lldb_git_commit_hash
,
618 Miri
=> &self.miri_git_commit_hash
,
619 _
=> &self.rust_git_commit_hash
,
623 fn version(&self, component
: &str, target
: &str) -> Option
<String
> {
624 self.untar(component
, target
, |filename
| format
!("{}/version", filename
))
627 fn git_commit_hash(&self, component
: &str, target
: &str) -> Option
<String
> {
628 self.untar(component
, target
, |filename
| format
!("{}/git-commit-hash", filename
))
631 fn untar
<F
>(&self, component
: &str, target
: &str, dir
: F
) -> Option
<String
>
633 F
: FnOnce(String
) -> String
635 let mut cmd
= Command
::new("tar");
636 let filename
= self.filename(component
, target
);
638 .arg(self.input
.join(&filename
))
639 .arg(dir(filename
.replace(".tar.gz", "")))
641 let output
= t
!(cmd
.output());
642 if output
.status
.success() {
643 Some(String
::from_utf8_lossy(&output
.stdout
).trim().to_string())
649 fn hash(&self, path
: &Path
) -> String
{
650 let sha
= t
!(Command
::new("shasum")
651 .arg("-a").arg("256")
652 .arg(path
.file_name().unwrap())
653 .current_dir(path
.parent().unwrap())
655 assert
!(sha
.status
.success());
657 let filename
= path
.file_name().unwrap().to_str().unwrap();
658 let sha256
= self.output
.join(format
!("{}.sha256", filename
));
659 t
!(fs
::write(&sha256
, &sha
.stdout
));
661 let stdout
= String
::from_utf8_lossy(&sha
.stdout
);
662 stdout
.split_whitespace().next().unwrap().to_string()
665 fn sign(&self, path
: &Path
) {
666 if !self.should_sign
{
670 let filename
= path
.file_name().unwrap().to_str().unwrap();
671 let asc
= self.output
.join(format
!("{}.asc", filename
));
672 println
!("signing: {:?}", path
);
673 let mut cmd
= Command
::new("gpg");
674 cmd
.arg("--pinentry-mode=loopback")
678 .arg("--passphrase-fd").arg("0")
679 .arg("--personal-digest-preferences").arg("SHA512")
681 .arg("--output").arg(&asc
)
682 .arg("--detach-sign").arg(path
)
683 .stdin(Stdio
::piped());
684 let mut child
= t
!(cmd
.spawn());
685 t
!(child
.stdin
.take().unwrap().write_all(self.gpg_passphrase
.as_bytes()));
686 assert
!(t
!(child
.wait()).success());
689 fn write_channel_files(&self, channel_name
: &str, manifest
: &Manifest
) {
690 self.write(&toml
::to_string(&manifest
).unwrap(), channel_name
, ".toml");
691 self.write(&manifest
.date
, channel_name
, "-date.txt");
692 self.write(manifest
.pkg
["rust"].git_commit_hash
.as_ref().unwrap(),
693 channel_name
, "-git-commit-hash.txt");
696 fn write(&self, contents
: &str, channel_name
: &str, suffix
: &str) {
697 let dst
= self.output
.join(format
!("channel-rust-{}{}", channel_name
, suffix
));
698 t
!(fs
::write(&dst
, contents
));