1 //! Build a dist manifest, hash and sign everything.
2 //! This gets called by `promote-release`
3 //! (https://github.com/rust-lang/rust-central-station/tree/master/promote-release)
4 //! via `x.py dist hash-and-sign`; the cmdline arguments are set up
5 //! by rustbuild (in `src/bootstrap/dist.rs`).
11 use crate::checksum
::Checksums
;
12 use crate::manifest
::{Component, Manifest, Package, Rename, Target}
;
13 use crate::versions
::{PkgType, Versions}
;
14 use std
::collections
::{BTreeMap, HashMap, HashSet}
;
16 use std
::fs
::{self, File}
;
17 use std
::path
::{Path, PathBuf}
;
19 static HOSTS
: &[&str] = &[
20 "aarch64-apple-darwin",
21 "aarch64-pc-windows-msvc",
22 "aarch64-unknown-linux-gnu",
23 "aarch64-unknown-linux-musl",
24 "arm-unknown-linux-gnueabi",
25 "arm-unknown-linux-gnueabihf",
26 "armv7-unknown-linux-gnueabihf",
28 "i686-pc-windows-gnu",
29 "i686-pc-windows-msvc",
30 "i686-unknown-linux-gnu",
31 "mips-unknown-linux-gnu",
32 "mips64-unknown-linux-gnuabi64",
33 "mips64el-unknown-linux-gnuabi64",
34 "mipsel-unknown-linux-gnu",
35 "mipsisa32r6-unknown-linux-gnu",
36 "mipsisa32r6el-unknown-linux-gnu",
37 "mipsisa64r6-unknown-linux-gnuabi64",
38 "mipsisa64r6el-unknown-linux-gnuabi64",
39 "powerpc-unknown-linux-gnu",
40 "powerpc64-unknown-linux-gnu",
41 "powerpc64le-unknown-linux-gnu",
42 "riscv64gc-unknown-linux-gnu",
43 "s390x-unknown-linux-gnu",
44 "x86_64-apple-darwin",
45 "x86_64-pc-windows-gnu",
46 "x86_64-pc-windows-msvc",
47 "x86_64-unknown-freebsd",
48 "x86_64-unknown-illumos",
49 "x86_64-unknown-linux-gnu",
50 "x86_64-unknown-linux-musl",
51 "x86_64-unknown-netbsd",
54 static TARGETS
: &[&str] = &[
55 "aarch64-apple-darwin",
57 "aarch64-apple-ios-sim",
59 "aarch64-linux-android",
60 "aarch64-pc-windows-msvc",
61 "aarch64-unknown-hermit",
62 "aarch64-unknown-linux-gnu",
63 "aarch64-unknown-linux-musl",
64 "aarch64-unknown-none",
65 "aarch64-unknown-none-softfloat",
66 "aarch64-unknown-redox",
67 "arm-linux-androideabi",
68 "arm-unknown-linux-gnueabi",
69 "arm-unknown-linux-gnueabihf",
70 "arm-unknown-linux-musleabi",
71 "arm-unknown-linux-musleabihf",
72 "armv5te-unknown-linux-gnueabi",
73 "armv5te-unknown-linux-musleabi",
75 "armv7-linux-androideabi",
76 "thumbv7neon-linux-androideabi",
77 "armv7-unknown-linux-gnueabi",
78 "armv7-unknown-linux-gnueabihf",
80 "thumbv7neon-unknown-linux-gnueabihf",
81 "armv7-unknown-linux-musleabi",
82 "armv7-unknown-linux-musleabihf",
84 "armebv7r-none-eabihf",
88 "asmjs-unknown-emscripten",
92 "i586-pc-windows-msvc",
93 "i586-unknown-linux-gnu",
94 "i586-unknown-linux-musl",
97 "i686-pc-windows-gnu",
98 "i686-pc-windows-msvc",
99 "i686-unknown-freebsd",
100 "i686-unknown-linux-gnu",
101 "i686-unknown-linux-musl",
102 "m68k-unknown-linux-gnu",
103 "mips-unknown-linux-gnu",
104 "mips-unknown-linux-musl",
105 "mips64-unknown-linux-gnuabi64",
106 "mips64-unknown-linux-muslabi64",
107 "mips64el-unknown-linux-gnuabi64",
108 "mips64el-unknown-linux-muslabi64",
109 "mipsisa32r6-unknown-linux-gnu",
110 "mipsisa32r6el-unknown-linux-gnu",
111 "mipsisa64r6-unknown-linux-gnuabi64",
112 "mipsisa64r6el-unknown-linux-gnuabi64",
113 "mipsel-unknown-linux-gnu",
114 "mipsel-unknown-linux-musl",
115 "nvptx64-nvidia-cuda",
116 "powerpc-unknown-linux-gnu",
117 "powerpc64-unknown-linux-gnu",
118 "powerpc64le-unknown-linux-gnu",
119 "riscv32i-unknown-none-elf",
120 "riscv32imc-unknown-none-elf",
121 "riscv32imac-unknown-none-elf",
122 "riscv32gc-unknown-linux-gnu",
123 "riscv64imac-unknown-none-elf",
124 "riscv64gc-unknown-none-elf",
125 "riscv64gc-unknown-linux-gnu",
126 "s390x-unknown-linux-gnu",
127 "sparc64-unknown-linux-gnu",
128 "sparcv9-sun-solaris",
129 "thumbv6m-none-eabi",
130 "thumbv7em-none-eabi",
131 "thumbv7em-none-eabihf",
132 "thumbv7m-none-eabi",
133 "thumbv8m.base-none-eabi",
134 "thumbv8m.main-none-eabi",
135 "thumbv8m.main-none-eabihf",
136 "wasm32-unknown-emscripten",
137 "wasm32-unknown-unknown",
139 "x86_64-apple-darwin",
141 "x86_64-fortanix-unknown-sgx",
143 "x86_64-linux-android",
144 "x86_64-pc-windows-gnu",
145 "x86_64-pc-windows-msvc",
146 "x86_64-sun-solaris",
148 "x86_64-unknown-freebsd",
149 "x86_64-unknown-illumos",
150 "x86_64-unknown-linux-gnu",
151 "x86_64-unknown-linux-gnux32",
152 "x86_64-unknown-linux-musl",
153 "x86_64-unknown-netbsd",
154 "x86_64-unknown-redox",
155 "x86_64-unknown-hermit",
158 static DOCS_TARGETS
: &[&str] = &[
159 "aarch64-unknown-linux-gnu",
161 "i686-pc-windows-gnu",
162 "i686-pc-windows-msvc",
163 "i686-unknown-linux-gnu",
164 "x86_64-apple-darwin",
165 "x86_64-pc-windows-gnu",
166 "x86_64-pc-windows-msvc",
167 "x86_64-unknown-linux-gnu",
168 "x86_64-unknown-linux-musl",
171 static MSI_INSTALLERS
: &[&str] = &[
172 "aarch64-pc-windows-msvc",
173 "i686-pc-windows-gnu",
174 "i686-pc-windows-msvc",
175 "x86_64-pc-windows-gnu",
176 "x86_64-pc-windows-msvc",
179 static PKG_INSTALLERS
: &[&str] = &["x86_64-apple-darwin", "aarch64-apple-darwin"];
181 static MINGW
: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"];
183 static NIGHTLY_ONLY_COMPONENTS
: &[&str] = &["miri-preview", "rust-analyzer-preview"];
189 Err(e
) => panic
!("{} failed with {}", stringify
!($e
), e
),
196 checksums
: Checksums
,
197 shipped_files
: HashSet
<String
>,
206 let num_threads
= if let Some(num
) = env
::var_os("BUILD_MANIFEST_NUM_THREADS") {
207 num
.to_str().unwrap().parse().expect("invalid number for BUILD_MANIFEST_NUM_THREADS")
211 rayon
::ThreadPoolBuilder
::new()
212 .num_threads(num_threads
)
214 .expect("failed to initialize Rayon");
216 let mut args
= env
::args().skip(1);
217 let input
= PathBuf
::from(args
.next().unwrap());
218 let output
= PathBuf
::from(args
.next().unwrap());
219 let date
= args
.next().unwrap();
220 let s3_address
= args
.next().unwrap();
221 let channel
= args
.next().unwrap();
224 versions
: Versions
::new(&channel
, &input
).unwrap(),
225 checksums
: t
!(Checksums
::new()),
226 shipped_files
: HashSet
::new(),
237 fn build(&mut self) {
238 self.check_toolstate();
239 let manifest
= self.build_manifest();
241 let channel
= self.versions
.channel().to_string();
242 self.write_channel_files(&channel
, &manifest
);
243 if channel
== "stable" {
244 // channel-rust-1.XX.YY.toml
245 let rust_version
= self.versions
.rustc_version().to_string();
246 self.write_channel_files(&rust_version
, &manifest
);
248 // channel-rust-1.XX.toml
249 let major_minor
= rust_version
.split('
.'
).take(2).collect
::<Vec
<_
>>().join(".");
250 self.write_channel_files(&major_minor
, &manifest
);
253 if let Some(path
) = std
::env
::var_os("BUILD_MANIFEST_SHIPPED_FILES_PATH") {
254 self.write_shipped_files(&Path
::new(&path
));
257 t
!(self.checksums
.store_cache());
260 /// If a tool does not pass its tests on *any* of Linux and Windows, don't ship
261 /// it on *all* targets, because tools like Miri can "cross-run" programs for
262 /// different targets, for example, run a program for `x86_64-pc-windows-msvc`
263 /// on `x86_64-unknown-linux-gnu`.
264 /// Right now, we do this only for Miri.
265 fn check_toolstate(&mut self) {
266 for file
in &["toolstates-linux.json", "toolstates-windows.json"] {
267 let toolstates
: Option
<HashMap
<String
, String
>> = File
::open(self.input
.join(file
))
269 .and_then(|f
| serde_json
::from_reader(&f
).ok());
270 let toolstates
= toolstates
.unwrap_or_else(|| {
271 println
!("WARNING: `{}` missing/malformed; assuming all tools failed", file
);
272 HashMap
::default() // Use empty map if anything went wrong.
274 // Mark some tools as missing based on toolstate.
275 if toolstates
.get("miri").map(|s
| &*s
as &str) != Some("test-pass") {
276 println
!("Miri tests are not passing, removing component");
277 self.versions
.disable_version(&PkgType
::Miri
);
283 fn build_manifest(&mut self) -> Manifest
{
284 let mut manifest
= Manifest
{
285 manifest_version
: "2".to_string(),
286 date
: self.date
.to_string(),
287 pkg
: BTreeMap
::new(),
288 artifacts
: BTreeMap
::new(),
289 renames
: BTreeMap
::new(),
290 profiles
: BTreeMap
::new(),
292 self.add_packages_to(&mut manifest
);
293 self.add_artifacts_to(&mut manifest
);
294 self.add_profiles_to(&mut manifest
);
295 self.add_renames_to(&mut manifest
);
296 manifest
.pkg
.insert("rust".to_string(), self.rust_package(&manifest
));
298 self.checksums
.fill_missing_checksums(&mut manifest
);
303 fn add_packages_to(&mut self, manifest
: &mut Manifest
) {
304 let mut package
= |name
, targets
| self.package(name
, &mut manifest
.pkg
, targets
);
305 package("rustc", HOSTS
);
306 package("rustc-dev", HOSTS
);
307 package("reproducible-artifacts", HOSTS
);
308 package("rustc-docs", HOSTS
);
309 package("cargo", HOSTS
);
310 package("rust-mingw", MINGW
);
311 package("rust-std", TARGETS
);
312 package("rust-docs", DOCS_TARGETS
);
313 package("rust-src", &["*"]);
314 package("rls-preview", HOSTS
);
315 package("rust-analyzer-preview", HOSTS
);
316 package("clippy-preview", HOSTS
);
317 package("miri-preview", HOSTS
);
318 package("rustfmt-preview", HOSTS
);
319 package("rust-analysis", TARGETS
);
320 package("llvm-tools-preview", TARGETS
);
323 fn add_artifacts_to(&mut self, manifest
: &mut Manifest
) {
324 manifest
.add_artifact("source-code", |artifact
| {
325 let tarball
= self.versions
.tarball_name(&PkgType
::Rustc
, "src").unwrap();
326 artifact
.add_tarball(self, "*", &tarball
);
329 manifest
.add_artifact("installer-msi", |artifact
| {
330 for target
in MSI_INSTALLERS
{
331 let msi
= self.versions
.archive_name(&PkgType
::Rust
, target
, "msi").unwrap();
332 artifact
.add_file(self, target
, &msi
);
336 manifest
.add_artifact("installer-pkg", |artifact
| {
337 for target
in PKG_INSTALLERS
{
338 let pkg
= self.versions
.archive_name(&PkgType
::Rust
, target
, "pkg").unwrap();
339 artifact
.add_file(self, target
, &pkg
);
344 fn add_profiles_to(&mut self, manifest
: &mut Manifest
) {
345 let mut profile
= |name
, pkgs
| self.profile(name
, &mut manifest
.profiles
, pkgs
);
346 profile("minimal", &["rustc", "cargo", "rust-std", "rust-mingw"]);
370 "rust-analyzer-preview",
372 "llvm-tools-preview",
378 // The compiler libraries are not stable for end users, and they're also huge, so we only
379 // `rustc-dev` for nightly users, and only in the "complete" profile. It's still possible
380 // for users to install the additional component manually, if needed.
381 if self.versions
.channel() == "nightly" {
382 self.extend_profile("complete", &mut manifest
.profiles
, &["rustc-dev"]);
383 self.extend_profile("complete", &mut manifest
.profiles
, &["rustc-docs"]);
387 fn add_renames_to(&self, manifest
: &mut Manifest
) {
388 let mut rename
= |from
: &str, to
: &str| {
389 manifest
.renames
.insert(from
.to_owned(), Rename { to: to.to_owned() }
)
391 rename("rls", "rls-preview");
392 rename("rustfmt", "rustfmt-preview");
393 rename("clippy", "clippy-preview");
394 rename("miri", "miri-preview");
397 fn rust_package(&mut self, manifest
: &Manifest
) -> Package
{
398 let version_info
= self.versions
.version(&PkgType
::Rust
).expect("missing Rust tarball");
399 let mut pkg
= Package
{
400 version
: version_info
.version
.expect("missing Rust version"),
401 git_commit_hash
: version_info
.git_commit
,
402 target
: BTreeMap
::new(),
405 if let Some(target
) = self.target_host_combination(host
, &manifest
) {
406 pkg
.target
.insert(host
.to_string(), target
);
408 pkg
.target
.insert(host
.to_string(), Target
::unavailable());
415 fn target_host_combination(&mut self, host
: &str, manifest
: &Manifest
) -> Option
<Target
> {
416 let filename
= self.versions
.tarball_name(&PkgType
::Rust
, host
).unwrap();
418 let mut target
= Target
::from_compressed_tar(self, &filename
);
419 if !target
.available
{
423 let mut components
= Vec
::new();
424 let mut extensions
= Vec
::new();
426 let host_component
= |pkg
| Component
::from_str(pkg
, host
);
428 // rustc/rust-std/cargo/docs are all required,
429 // and so is rust-mingw if it's available for the target.
430 components
.extend(vec
![
431 host_component("rustc"),
432 host_component("rust-std"),
433 host_component("cargo"),
434 host_component("rust-docs"),
436 if host
.contains("pc-windows-gnu") {
437 components
.push(host_component("rust-mingw"));
440 // Tools are always present in the manifest,
441 // but might be marked as unavailable if they weren't built.
442 extensions
.extend(vec
![
443 host_component("clippy-preview"),
444 host_component("miri-preview"),
445 host_component("rls-preview"),
446 host_component("rust-analyzer-preview"),
447 host_component("rustfmt-preview"),
448 host_component("llvm-tools-preview"),
449 host_component("rust-analysis"),
455 .filter(|&&target
| target
!= host
)
456 .map(|target
| Component
::from_str("rust-std", target
)),
458 extensions
.extend(HOSTS
.iter().map(|target
| Component
::from_str("rustc-dev", target
)));
459 extensions
.extend(HOSTS
.iter().map(|target
| Component
::from_str("rustc-docs", target
)));
460 extensions
.push(Component
::from_str("rust-src", "*"));
462 // If the components/extensions don't actually exist for this
463 // particular host/target combination then nix it entirely from our
465 let has_component
= |c
: &Component
| {
469 let pkg
= match manifest
.pkg
.get(&c
.pkg
) {
471 None
=> return false,
473 pkg
.target
.get(&c
.target
).is_some()
475 extensions
.retain(&has_component
);
476 components
.retain(&has_component
);
478 target
.components
= Some(components
);
479 target
.extensions
= Some(extensions
);
486 dst
: &mut BTreeMap
<String
, Vec
<String
>>,
489 dst
.insert(profile_name
.to_owned(), pkgs
.iter().map(|s
| (*s
).to_owned()).collect());
495 dst
: &mut BTreeMap
<String
, Vec
<String
>>,
498 dst
.get_mut(profile_name
)
499 .expect("existing profile")
500 .extend(pkgs
.iter().map(|s
| (*s
).to_owned()));
503 fn package(&mut self, pkgname
: &str, dst
: &mut BTreeMap
<String
, Package
>, targets
: &[&str]) {
504 let version_info
= self
506 .version(&PkgType
::from_component(pkgname
))
507 .expect("failed to load package version");
508 let mut is_present
= version_info
.present
;
510 // Never ship nightly-only components for other trains.
511 if self.versions
.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS
.contains(&pkgname
) {
512 is_present
= false; // Pretend the component is entirely missing.
515 let targets
= targets
518 let target
= if is_present
{
521 .tarball_name(&PkgType
::from_component(pkgname
), name
)
524 Target
::from_compressed_tar(self, &filename
)
526 // If the component is not present for this build add it anyway but mark it as
527 // unavailable -- this way rustup won't allow upgrades without --force
528 Target
::unavailable()
530 (name
.to_string(), target
)
537 version
: version_info
.version
.unwrap_or_default(),
538 git_commit_hash
: version_info
.git_commit
,
544 fn url(&self, path
: &Path
) -> String
{
545 let file_name
= path
.file_name().unwrap().to_str().unwrap();
546 format
!("{}/{}/{}", self.s3_address
, self.date
, file_name
)
549 fn write_channel_files(&mut self, channel_name
: &str, manifest
: &Manifest
) {
550 self.write(&toml
::to_string(&manifest
).unwrap(), channel_name
, ".toml");
551 self.write(&manifest
.date
, channel_name
, "-date.txt");
553 manifest
.pkg
["rust"].git_commit_hash
.as_ref().unwrap(),
555 "-git-commit-hash.txt",
559 fn write(&mut self, contents
: &str, channel_name
: &str, suffix
: &str) {
560 let name
= format
!("channel-rust-{}{}", channel_name
, suffix
);
561 self.shipped_files
.insert(name
.clone());
563 let dst
= self.output
.join(name
);
564 t
!(fs
::write(&dst
, contents
));
567 fn write_shipped_files(&self, path
: &Path
) {
568 let mut files
= self.shipped_files
.iter().map(|s
| s
.as_str()).collect
::<Vec
<_
>>();
570 let content
= format
!("{}\n", files
.join("\n"));
572 t
!(std
::fs
::write(path
, content
.as_bytes()));