]>
git.proxmox.com Git - rustc.git/blob - src/bootstrap/bootstrap.py
1 from __future__
import absolute_import
, division
, print_function
5 import distutils
.version
20 with tempfile
.NamedTemporaryFile(delete
=False) as temp_file
:
21 temp_path
= temp_file
.name
22 with tarfile
.open(temp_path
, "w:xz"):
25 except tarfile
.CompressionError
:
28 def get(base
, url
, path
, checksums
, verbose
=False, do_verify
=True):
29 with tempfile
.NamedTemporaryFile(delete
=False) as temp_file
:
30 temp_path
= temp_file
.name
34 if url
not in checksums
:
35 raise RuntimeError("src/stage0.json doesn't contain a checksum for {}".format(url
))
36 sha256
= checksums
[url
]
37 if os
.path
.exists(path
):
38 if verify(path
, sha256
, False):
40 print("using already-download file", path
)
44 print("ignoring already-download file",
45 path
, "due to failed verification")
47 download(temp_path
, "{}/{}".format(base
, url
), True, verbose
)
48 if do_verify
and not verify(temp_path
, sha256
, verbose
):
49 raise RuntimeError("failed verification")
51 print("moving {} to {}".format(temp_path
, path
))
52 shutil
.move(temp_path
, path
)
54 if os
.path
.isfile(temp_path
):
56 print("removing", temp_path
)
60 def download(path
, url
, probably_big
, verbose
):
63 _download(path
, url
, probably_big
, verbose
, True)
66 print("\nspurious failure, trying again")
67 _download(path
, url
, probably_big
, verbose
, False)
70 def _download(path
, url
, probably_big
, verbose
, exception
):
71 if probably_big
or verbose
:
72 print("downloading {}".format(url
))
73 # see https://serverfault.com/questions/301128/how-to-download
74 if sys
.platform
== 'win32':
75 run(["PowerShell.exe", "/nologo", "-Command",
76 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
77 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url
, path
)],
81 if probably_big
or verbose
:
85 require(["curl", "--version"])
87 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds
88 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
89 "--retry", "3", "-Sf", "-o", path
, url
],
94 def verify(path
, expected
, verbose
):
95 """Check if the sha256 sum of the given path is valid"""
97 print("verifying", path
)
98 with
open(path
, "rb") as source
:
99 found
= hashlib
.sha256(source
.read()).hexdigest()
100 verified
= found
== expected
102 print("invalid checksum:\n"
104 " expected: {}".format(found
, expected
))
108 def unpack(tarball
, tarball_suffix
, dst
, verbose
=False, match
=None):
109 """Unpack the given tarball file"""
110 print("extracting", tarball
)
111 fname
= os
.path
.basename(tarball
).replace(tarball_suffix
, "")
112 with contextlib
.closing(tarfile
.open(tarball
)) as tar
:
113 for member
in tar
.getnames():
114 if "/" not in member
:
116 name
= member
.replace(fname
+ "/", "", 1)
117 if match
is not None and not name
.startswith(match
):
119 name
= name
[len(match
) + 1:]
121 dst_path
= os
.path
.join(dst
, name
)
123 print(" extracting", member
)
124 tar
.extract(member
, dst
)
125 src_path
= os
.path
.join(dst
, member
)
126 if os
.path
.isdir(src_path
) and os
.path
.exists(dst_path
):
128 shutil
.move(src_path
, dst_path
)
129 shutil
.rmtree(os
.path
.join(dst
, fname
))
132 def run(args
, verbose
=False, exception
=False, is_bootstrap
=False, **kwargs
):
133 """Run a child program in a new process"""
135 print("running: " + ' '.join(args
))
137 # Use Popen here instead of call() as it apparently allows powershell on
138 # Windows to not lock up waiting for input presumably.
139 ret
= subprocess
.Popen(args
, **kwargs
)
142 err
= "failed to run: " + ' '.join(args
)
143 if verbose
or exception
:
144 raise RuntimeError(err
)
145 # For most failures, we definitely do want to print this error, or the user will have no
146 # idea what went wrong. But when we've successfully built bootstrap and it failed, it will
147 # have already printed an error above, so there's no need to print the exact command we're
155 def require(cmd
, exit
=True):
156 '''Run a command, returning its output.
158 If `exit` is `True`, exit the process.
159 Otherwise, return None.'''
161 return subprocess
.check_output(cmd
).strip()
162 except (subprocess
.CalledProcessError
, OSError) as exc
:
165 print("error: unable to run `{}`: {}".format(' '.join(cmd
), exc
))
166 print("Please make sure it's installed and in the path.")
170 def format_build_time(duration
):
171 """Return a nicer format for build time
173 >>> format_build_time('300')
176 return str(datetime
.timedelta(seconds
=int(duration
)))
179 def default_build_triple(verbose
):
180 """Build triple as in LLVM"""
181 # If the user already has a host build triple with an existing `rustc`
182 # install, use their preference. This fixes most issues with Windows builds
183 # being detected as GNU instead of MSVC.
184 default_encoding
= sys
.getdefaultencoding()
186 version
= subprocess
.check_output(["rustc", "--version", "--verbose"],
187 stderr
=subprocess
.DEVNULL
)
188 version
= version
.decode(default_encoding
)
189 host
= next(x
for x
in version
.split('\n') if x
.startswith("host: "))
190 triple
= host
.split("host: ")[1]
192 print("detected default triple {}".format(triple
))
194 except Exception as e
:
196 print("rustup not detected: {}".format(e
))
197 print("falling back to auto-detect")
199 required
= sys
.platform
!= 'win32'
200 ostype
= require(["uname", "-s"], exit
=required
)
201 cputype
= require(['uname', '-m'], exit
=required
)
203 # If we do not have `uname`, assume Windows.
204 if ostype
is None or cputype
is None:
205 return 'x86_64-pc-windows-msvc'
207 ostype
= ostype
.decode(default_encoding
)
208 cputype
= cputype
.decode(default_encoding
)
210 # The goal here is to come up with the same triple as LLVM would,
211 # at least for the subset of platforms we're willing to target.
213 'Darwin': 'apple-darwin',
214 'DragonFly': 'unknown-dragonfly',
215 'FreeBSD': 'unknown-freebsd',
216 'Haiku': 'unknown-haiku',
217 'NetBSD': 'unknown-netbsd',
218 'OpenBSD': 'unknown-openbsd'
221 # Consider the direct transformation first and then the special cases
222 if ostype
in ostype_mapper
:
223 ostype
= ostype_mapper
[ostype
]
224 elif ostype
== 'Linux':
225 os_from_sp
= subprocess
.check_output(
226 ['uname', '-o']).strip().decode(default_encoding
)
227 if os_from_sp
== 'Android':
228 ostype
= 'linux-android'
230 ostype
= 'unknown-linux-gnu'
231 elif ostype
== 'SunOS':
232 ostype
= 'pc-solaris'
233 # On Solaris, uname -m will return a machine classification instead
234 # of a cpu type, so uname -p is recommended instead. However, the
235 # output from that option is too generic for our purposes (it will
236 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
237 # must be used instead.
238 cputype
= require(['isainfo', '-k']).decode(default_encoding
)
239 # sparc cpus have sun as a target vendor
240 if 'sparc' in cputype
:
241 ostype
= 'sun-solaris'
242 elif ostype
.startswith('MINGW'):
243 # msys' `uname` does not print gcc configuration, but prints msys
244 # configuration. so we cannot believe `uname -m`:
245 # msys1 is always i686 and msys2 is always x86_64.
246 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
248 ostype
= 'pc-windows-gnu'
250 if os
.environ
.get('MSYSTEM') == 'MINGW64':
252 elif ostype
.startswith('MSYS'):
253 ostype
= 'pc-windows-gnu'
254 elif ostype
.startswith('CYGWIN_NT'):
256 if ostype
.endswith('WOW64'):
258 ostype
= 'pc-windows-gnu'
259 elif sys
.platform
== 'win32':
260 # Some Windows platforms might have a `uname` command that returns a
261 # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
262 # these cases, fall back to using sys.platform.
263 return 'x86_64-pc-windows-msvc'
265 err
= "unknown OS type: {}".format(ostype
)
268 if cputype
== 'powerpc' and ostype
== 'unknown-freebsd':
269 cputype
= subprocess
.check_output(
270 ['uname', '-p']).strip().decode(default_encoding
)
273 'aarch64': 'aarch64',
281 'powerpc': 'powerpc',
282 'powerpc64': 'powerpc64',
283 'powerpc64le': 'powerpc64le',
285 'ppc64': 'powerpc64',
286 'ppc64le': 'powerpc64le',
287 'riscv64': 'riscv64gc',
295 # Consider the direct transformation first and then the special cases
296 if cputype
in cputype_mapper
:
297 cputype
= cputype_mapper
[cputype
]
298 elif cputype
in {'xscale', 'arm'}:
300 if ostype
== 'linux-android':
301 ostype
= 'linux-androideabi'
302 elif ostype
== 'unknown-freebsd':
303 cputype
= subprocess
.check_output(
304 ['uname', '-p']).strip().decode(default_encoding
)
305 ostype
= 'unknown-freebsd'
306 elif cputype
== 'armv6l':
308 if ostype
== 'linux-android':
309 ostype
= 'linux-androideabi'
312 elif cputype
in {'armv7l', 'armv8l'}:
314 if ostype
== 'linux-android':
315 ostype
= 'linux-androideabi'
318 elif cputype
== 'mips':
319 if sys
.byteorder
== 'big':
321 elif sys
.byteorder
== 'little':
324 raise ValueError("unknown byteorder: {}".format(sys
.byteorder
))
325 elif cputype
== 'mips64':
326 if sys
.byteorder
== 'big':
328 elif sys
.byteorder
== 'little':
331 raise ValueError('unknown byteorder: {}'.format(sys
.byteorder
))
332 # only the n64 ABI is supported, indicate it
334 elif cputype
== 'sparc' or cputype
== 'sparcv9' or cputype
== 'sparc64':
337 err
= "unknown cpu type: {}".format(cputype
)
340 return "{}-{}".format(cputype
, ostype
)
343 @contextlib.contextmanager
344 def output(filepath
):
345 tmp
= filepath
+ '.tmp'
346 with
open(tmp
, 'w') as f
:
349 if os
.path
.exists(filepath
):
350 os
.remove(filepath
) # PermissionError/OSError on Win32 if in use
352 shutil
.copy2(tmp
, filepath
)
355 os
.rename(tmp
, filepath
)
358 class Stage0Toolchain
:
359 def __init__(self
, stage0_payload
):
360 self
.date
= stage0_payload
["date"]
361 self
.version
= stage0_payload
["version"]
364 return self
.version
+ "-" + self
.date
367 class RustBuild(object):
368 """Provide all the methods required to build Rust"""
370 self
.checksums_sha256
= {}
371 self
.stage0_compiler
= None
372 self
.stage0_rustfmt
= None
373 self
._download
_url
= ''
377 self
.config_toml
= ''
379 self
.use_locked_deps
= ''
380 self
.use_vendored_sources
= ''
382 self
.git_version
= None
383 self
.nix_deps_dir
= None
384 self
.rustc_commit
= None
386 def download_toolchain(self
, stage0
=True, rustc_channel
=None):
387 """Fetch the build system for Rust, written in Rust
389 This method will build a cache directory, then it will fetch the
390 tarball which has the stage0 compiler used to then bootstrap the Rust
393 Each downloaded tarball is extracted, after that, the script
394 will move all the content to the right place.
396 if rustc_channel
is None:
397 rustc_channel
= self
.stage0_compiler
.version
398 bin_root
= self
.bin_root(stage0
)
400 key
= self
.stage0_compiler
.date
402 key
+= str(self
.rustc_commit
)
403 if self
.rustc(stage0
).startswith(bin_root
) and \
404 (not os
.path
.exists(self
.rustc(stage0
)) or
405 self
.program_out_of_date(self
.rustc_stamp(stage0
), key
)):
406 if os
.path
.exists(bin_root
):
407 shutil
.rmtree(bin_root
)
408 tarball_suffix
= '.tar.xz' if support_xz() else '.tar.gz'
409 filename
= "rust-std-{}-{}{}".format(
410 rustc_channel
, self
.build
, tarball_suffix
)
411 pattern
= "rust-std-{}".format(self
.build
)
412 self
._download
_component
_helper
(filename
, pattern
, tarball_suffix
, stage0
)
413 filename
= "rustc-{}-{}{}".format(rustc_channel
, self
.build
,
415 self
._download
_component
_helper
(filename
, "rustc", tarball_suffix
, stage0
)
416 # download-rustc doesn't need its own cargo, it can just use beta's.
418 filename
= "cargo-{}-{}{}".format(rustc_channel
, self
.build
,
420 self
._download
_component
_helper
(filename
, "cargo", tarball_suffix
)
421 self
.fix_bin_or_dylib("{}/bin/cargo".format(bin_root
))
423 filename
= "rustc-dev-{}-{}{}".format(rustc_channel
, self
.build
, tarball_suffix
)
424 self
._download
_component
_helper
(
425 filename
, "rustc-dev", tarball_suffix
, stage0
428 self
.fix_bin_or_dylib("{}/bin/rustc".format(bin_root
))
429 self
.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root
))
430 lib_dir
= "{}/lib".format(bin_root
)
431 for lib
in os
.listdir(lib_dir
):
432 if lib
.endswith(".so"):
433 self
.fix_bin_or_dylib(os
.path
.join(lib_dir
, lib
))
434 with
output(self
.rustc_stamp(stage0
)) as rust_stamp
:
435 rust_stamp
.write(key
)
437 if self
.rustfmt() and self
.rustfmt().startswith(bin_root
) and (
438 not os
.path
.exists(self
.rustfmt())
439 or self
.program_out_of_date(
440 self
.rustfmt_stamp(),
441 "" if self
.stage0_rustfmt
is None else self
.stage0_rustfmt
.channel()
444 if self
.stage0_rustfmt
is not None:
445 tarball_suffix
= '.tar.xz' if support_xz() else '.tar.gz'
446 filename
= "rustfmt-{}-{}{}".format(
447 self
.stage0_rustfmt
.version
, self
.build
, tarball_suffix
,
449 self
._download
_component
_helper
(
450 filename
, "rustfmt-preview", tarball_suffix
, key
=self
.stage0_rustfmt
.date
452 self
.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root
))
453 self
.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root
))
454 with
output(self
.rustfmt_stamp()) as rustfmt_stamp
:
455 rustfmt_stamp
.write(self
.stage0_rustfmt
.channel())
457 # Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
458 if self
.downloading_llvm() and stage0
:
459 # We want the most recent LLVM submodule update to avoid downloading
460 # LLVM more often than necessary.
462 # This git command finds that commit SHA, looking for bors-authored
463 # commits that modified src/llvm-project or other relevant version
466 # This works even in a repository that has not yet initialized
468 top_level
= subprocess
.check_output([
469 "git", "rev-parse", "--show-toplevel",
470 ]).decode(sys
.getdefaultencoding()).strip()
471 llvm_sha
= subprocess
.check_output([
472 "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
473 "--first-parent", "HEAD",
475 "{}/src/llvm-project".format(top_level
),
476 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level
),
477 # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
478 "{}/src/version".format(top_level
)
479 ]).decode(sys
.getdefaultencoding()).strip()
480 llvm_assertions
= self
.get_toml('assertions', 'llvm') == 'true'
481 llvm_root
= self
.llvm_root()
482 llvm_lib
= os
.path
.join(llvm_root
, "lib")
483 if self
.program_out_of_date(self
.llvm_stamp(), llvm_sha
+ str(llvm_assertions
)):
484 self
._download
_ci
_llvm
(llvm_sha
, llvm_assertions
)
485 for binary
in ["llvm-config", "FileCheck"]:
486 self
.fix_bin_or_dylib(os
.path
.join(llvm_root
, "bin", binary
))
487 for lib
in os
.listdir(llvm_lib
):
488 if lib
.endswith(".so"):
489 self
.fix_bin_or_dylib(os
.path
.join(llvm_lib
, lib
))
490 with
output(self
.llvm_stamp()) as llvm_stamp
:
491 llvm_stamp
.write(llvm_sha
+ str(llvm_assertions
))
493 def downloading_llvm(self
):
494 opt
= self
.get_toml('download-ci-llvm', 'llvm')
495 # This is currently all tier 1 targets and tier 2 targets with host tools
496 # (since others may not have CI artifacts)
497 # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
498 supported_platforms
= [
500 "aarch64-unknown-linux-gnu",
501 "i686-pc-windows-gnu",
502 "i686-pc-windows-msvc",
503 "i686-unknown-linux-gnu",
504 "x86_64-unknown-linux-gnu",
505 "x86_64-apple-darwin",
506 "x86_64-pc-windows-gnu",
507 "x86_64-pc-windows-msvc",
508 # tier 2 with host tools
509 "aarch64-apple-darwin",
510 "aarch64-pc-windows-msvc",
511 "aarch64-unknown-linux-musl",
512 "arm-unknown-linux-gnueabi",
513 "arm-unknown-linux-gnueabihf",
514 "armv7-unknown-linux-gnueabihf",
515 "mips-unknown-linux-gnu",
516 "mips64-unknown-linux-gnuabi64",
517 "mips64el-unknown-linux-gnuabi64",
518 "mipsel-unknown-linux-gnu",
519 "powerpc-unknown-linux-gnu",
520 "powerpc64-unknown-linux-gnu",
521 "powerpc64le-unknown-linux-gnu",
522 "riscv64gc-unknown-linux-gnu",
523 "s390x-unknown-linux-gnu",
524 "x86_64-unknown-freebsd",
525 "x86_64-unknown-illumos",
526 "x86_64-unknown-linux-musl",
527 "x86_64-unknown-netbsd",
529 return opt
== "true" \
530 or (opt
== "if-available" and self
.build
in supported_platforms
)
532 def _download_component_helper(
533 self
, filename
, pattern
, tarball_suffix
, stage0
=True, key
=None
537 key
= self
.stage0_compiler
.date
539 key
= self
.rustc_commit
540 cache_dst
= os
.path
.join(self
.build_dir
, "cache")
541 rustc_cache
= os
.path
.join(cache_dst
, key
)
542 if not os
.path
.exists(rustc_cache
):
543 os
.makedirs(rustc_cache
)
546 base
= self
._download
_url
547 url
= "dist/{}".format(key
)
549 base
= "https://ci-artifacts.rust-lang.org"
550 url
= "rustc-builds/{}".format(self
.rustc_commit
)
551 tarball
= os
.path
.join(rustc_cache
, filename
)
552 if not os
.path
.exists(tarball
):
555 "{}/{}".format(url
, filename
),
557 self
.checksums_sha256
,
558 verbose
=self
.verbose
,
561 unpack(tarball
, tarball_suffix
, self
.bin_root(stage0
), match
=pattern
, verbose
=self
.verbose
)
563 def _download_ci_llvm(self
, llvm_sha
, llvm_assertions
):
565 print("error: could not find commit hash for downloading LLVM")
566 print("help: maybe your repository history is too shallow?")
567 print("help: consider disabling `download-ci-llvm`")
568 print("help: or fetch enough history to include one upstream commit")
570 cache_prefix
= "llvm-{}-{}".format(llvm_sha
, llvm_assertions
)
571 cache_dst
= os
.path
.join(self
.build_dir
, "cache")
572 rustc_cache
= os
.path
.join(cache_dst
, cache_prefix
)
573 if not os
.path
.exists(rustc_cache
):
574 os
.makedirs(rustc_cache
)
576 base
= "https://ci-artifacts.rust-lang.org"
577 url
= "rustc-builds/{}".format(llvm_sha
)
579 url
= url
.replace('rustc-builds', 'rustc-builds-alt')
580 # ci-artifacts are only stored as .xz, not .gz
582 print("error: XZ support is required to download LLVM")
583 print("help: consider disabling `download-ci-llvm` or using python3")
585 tarball_suffix
= '.tar.xz'
586 filename
= "rust-dev-nightly-" + self
.build
+ tarball_suffix
587 tarball
= os
.path
.join(rustc_cache
, filename
)
588 if not os
.path
.exists(tarball
):
591 "{}/{}".format(url
, filename
),
593 self
.checksums_sha256
,
594 verbose
=self
.verbose
,
597 unpack(tarball
, tarball_suffix
, self
.llvm_root(),
599 verbose
=self
.verbose
)
601 def fix_bin_or_dylib(self
, fname
):
602 """Modifies the interpreter section of 'fname' to fix the dynamic linker,
603 or the RPATH section, to fix the dynamic library search path
605 This method is only required on NixOS and uses the PatchELF utility to
606 change the interpreter/RPATH of ELF executables.
608 Please see https://nixos.org/patchelf.html for more information
610 default_encoding
= sys
.getdefaultencoding()
612 ostype
= subprocess
.check_output(
613 ['uname', '-s']).strip().decode(default_encoding
)
614 except subprocess
.CalledProcessError
:
616 except OSError as reason
:
617 if getattr(reason
, 'winerror', None) is not None:
621 if ostype
!= "Linux":
624 # If the user has asked binaries to be patched for Nix, then
625 # don't check for NixOS or `/lib`, just continue to the patching.
626 if self
.get_toml('patch-binaries-for-nix', 'build') != 'true':
627 # Use `/etc/os-release` instead of `/etc/NIXOS`.
628 # The latter one does not exist on NixOS when using tmpfs as root.
630 with
open("/etc/os-release", "r") as f
:
631 if not any(line
.strip() == "ID=nixos" for line
in f
):
633 except FileNotFoundError
:
635 if os
.path
.exists("/lib"):
638 # At this point we're pretty sure the user is running NixOS or
640 nix_os_msg
= "info: you seem to be using Nix. Attempting to patch"
641 print(nix_os_msg
, fname
)
643 # Only build `.nix-deps` once.
644 nix_deps_dir
= self
.nix_deps_dir
646 # Run `nix-build` to "build" each dependency (which will likely reuse
647 # the existing `/nix/store` copy, or at most download a pre-built copy).
649 # Importantly, we create a gc-root called `.nix-deps` in the `build/`
650 # directory, but still reference the actual `/nix/store` path in the rpath
651 # as it makes it significantly more robust against changes to the location of
652 # the `.nix-deps` location.
654 # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
655 # zlib: Needed as a system dependency of `libLLVM-*.so`.
656 # patchelf: Needed for patching ELF binaries (see doc comment above).
657 nix_deps_dir
= "{}/{}".format(self
.build_dir
, ".nix-deps")
659 with (import <nixpkgs> {});
661 name = "rust-stage0-dependencies";
670 subprocess
.check_output([
671 "nix-build", "-E", nix_expr
, "-o", nix_deps_dir
,
673 except subprocess
.CalledProcessError
as reason
:
674 print("warning: failed to call nix-build:", reason
)
676 self
.nix_deps_dir
= nix_deps_dir
678 patchelf
= "{}/bin/patchelf".format(nix_deps_dir
)
680 # Relative default, all binary and dynamic libraries we ship
681 # appear to have this (even when `../lib` is redundant).
683 os
.path
.join(os
.path
.realpath(nix_deps_dir
), "lib")
685 patchelf_args
= ["--set-rpath", ":".join(rpath_entries
)]
686 if not fname
.endswith(".so"):
687 # Finally, set the corret .interp for binaries
688 with
open("{}/nix-support/dynamic-linker".format(nix_deps_dir
)) as dynamic_linker
:
689 patchelf_args
+= ["--set-interpreter", dynamic_linker
.read().rstrip()]
692 subprocess
.check_output([patchelf
] + patchelf_args
+ [fname
])
693 except subprocess
.CalledProcessError
as reason
:
694 print("warning: failed to call patchelf:", reason
)
697 # If `download-rustc` is set, download the most recent commit with CI artifacts
698 def maybe_download_ci_toolchain(self
):
699 # If `download-rustc` is not set, default to rebuilding.
700 download_rustc
= self
.get_toml("download-rustc", section
="rust")
701 if download_rustc
is None or download_rustc
== "false":
703 assert download_rustc
== "true" or download_rustc
== "if-unchanged", download_rustc
705 # Handle running from a directory other than the top level
706 rev_parse
= ["git", "rev-parse", "--show-toplevel"]
707 top_level
= subprocess
.check_output(rev_parse
, universal_newlines
=True).strip()
708 compiler
= "{}/compiler/".format(top_level
)
709 library
= "{}/library/".format(top_level
)
711 # Look for a version to compare to based on the current commit.
712 # Only commits merged by bors will have CI artifacts.
714 "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
715 "--first-parent", "HEAD"
717 commit
= subprocess
.check_output(merge_base
, universal_newlines
=True).strip()
719 print("error: could not find commit hash for downloading rustc")
720 print("help: maybe your repository history is too shallow?")
721 print("help: consider disabling `download-rustc`")
722 print("help: or fetch enough history to include one upstream commit")
725 # Warn if there were changes to the compiler or standard library since the ancestor commit.
726 status
= subprocess
.call(["git", "diff-index", "--quiet", commit
, "--", compiler
, library
])
728 if download_rustc
== "if-unchanged":
730 print("warning: `download-rustc` is enabled, but there are changes to \
731 compiler/ or library/")
734 print("using downloaded stage1 artifacts from CI (commit {})".format(commit
))
735 self
.rustc_commit
= commit
736 # FIXME: support downloading artifacts from the beta channel
737 self
.download_toolchain(False, "nightly")
739 def rustc_stamp(self
, stage0
):
740 """Return the path for .rustc-stamp at the given stage
743 >>> rb.build_dir = "build"
744 >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
746 >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
749 return os
.path
.join(self
.bin_root(stage0
), '.rustc-stamp')
751 def rustfmt_stamp(self
):
752 """Return the path for .rustfmt-stamp
755 >>> rb.build_dir = "build"
756 >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
759 return os
.path
.join(self
.bin_root(True), '.rustfmt-stamp')
761 def llvm_stamp(self
):
762 """Return the path for .rustfmt-stamp
765 >>> rb.build_dir = "build"
766 >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
769 return os
.path
.join(self
.llvm_root(), '.llvm-stamp')
772 def program_out_of_date(self
, stamp_path
, key
):
773 """Check if the given program stamp is out of date"""
774 if not os
.path
.exists(stamp_path
) or self
.clean
:
776 with
open(stamp_path
, 'r') as stamp
:
777 return key
!= stamp
.read()
779 def bin_root(self
, stage0
):
780 """Return the binary root directory for the given stage
783 >>> rb.build_dir = "build"
784 >>> rb.bin_root(True) == os.path.join("build", "stage0")
786 >>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
789 When the 'build' property is given should be a nested directory:
791 >>> rb.build = "devel"
792 >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
799 return os
.path
.join(self
.build_dir
, self
.build
, subdir
)
802 """Return the CI LLVM root directory
805 >>> rb.build_dir = "build"
806 >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
809 When the 'build' property is given should be a nested directory:
811 >>> rb.build = "devel"
812 >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
815 return os
.path
.join(self
.build_dir
, self
.build
, "ci-llvm")
817 def get_toml(self
, key
, section
=None):
818 """Returns the value of the given key in config.toml, otherwise returns None
821 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
822 >>> rb.get_toml("key2")
825 If the key does not exists, the result is None:
827 >>> rb.get_toml("key3") is None
830 Optionally also matches the section the key appears in
832 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
833 >>> rb.get_toml('key', 'a')
835 >>> rb.get_toml('key', 'b')
837 >>> rb.get_toml('key', 'c') is None
840 >>> rb.config_toml = 'key1 = true'
841 >>> rb.get_toml("key1")
846 for line
in self
.config_toml
.splitlines():
847 section_match
= re
.match(r
'^\s*\[(.*)\]\s*$', line
)
848 if section_match
is not None:
849 cur_section
= section_match
.group(1)
851 match
= re
.match(r
'^{}\s*=(.*)$'.format(key
), line
)
852 if match
is not None:
853 value
= match
.group(1)
854 if section
is None or section
== cur_section
:
855 return self
.get_string(value
) or value
.strip()
859 """Return config path for cargo"""
860 return self
.program_config('cargo')
862 def rustc(self
, stage0
):
863 """Return config path for rustc"""
864 return self
.program_config('rustc', stage0
)
867 """Return config path for rustfmt"""
868 if self
.stage0_rustfmt
is None:
870 return self
.program_config('rustfmt')
872 def program_config(self
, program
, stage0
=True):
873 """Return config path for the given program at the given stage
876 >>> rb.config_toml = 'rustc = "rustc"\\n'
877 >>> rb.program_config('rustc')
879 >>> rb.config_toml = ''
880 >>> cargo_path = rb.program_config('cargo', True)
881 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
884 >>> cargo_path = rb.program_config('cargo', False)
885 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
889 config
= self
.get_toml(program
)
891 return os
.path
.expanduser(config
)
892 return os
.path
.join(self
.bin_root(stage0
), "bin", "{}{}".format(
893 program
, self
.exe_suffix()))
896 def get_string(line
):
897 """Return the value between double quotes
899 >>> RustBuild.get_string(' "devel" ')
901 >>> RustBuild.get_string(" 'devel' ")
903 >>> RustBuild.get_string('devel') is None
905 >>> RustBuild.get_string(' "devel ')
908 start
= line
.find('"')
910 end
= start
+ 1 + line
[start
+ 1:].find('"')
911 return line
[start
+ 1:end
]
912 start
= line
.find('\'')
914 end
= start
+ 1 + line
[start
+ 1:].find('\'')
915 return line
[start
+ 1:end
]
920 """Return a suffix for executables"""
921 if sys
.platform
== 'win32':
925 def bootstrap_binary(self
):
926 """Return the path of the bootstrap binary
929 >>> rb.build_dir = "build"
930 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
931 ... "debug", "bootstrap")
934 return os
.path
.join(self
.build_dir
, "bootstrap", "debug", "bootstrap")
936 def build_bootstrap(self
):
937 """Build bootstrap"""
938 build_dir
= os
.path
.join(self
.build_dir
, "bootstrap")
939 if self
.clean
and os
.path
.exists(build_dir
):
940 shutil
.rmtree(build_dir
)
941 env
= os
.environ
.copy()
942 # `CARGO_BUILD_TARGET` breaks bootstrap build.
943 # See also: <https://github.com/rust-lang/rust/issues/70208>.
944 if "CARGO_BUILD_TARGET" in env
:
945 del env
["CARGO_BUILD_TARGET"]
946 env
["CARGO_TARGET_DIR"] = build_dir
947 env
["RUSTC"] = self
.rustc(True)
948 env
["LD_LIBRARY_PATH"] = os
.path
.join(self
.bin_root(True), "lib") + \
949 (os
.pathsep
+ env
["LD_LIBRARY_PATH"]) \
950 if "LD_LIBRARY_PATH" in env
else ""
951 env
["DYLD_LIBRARY_PATH"] = os
.path
.join(self
.bin_root(True), "lib") + \
952 (os
.pathsep
+ env
["DYLD_LIBRARY_PATH"]) \
953 if "DYLD_LIBRARY_PATH" in env
else ""
954 env
["LIBRARY_PATH"] = os
.path
.join(self
.bin_root(True), "lib") + \
955 (os
.pathsep
+ env
["LIBRARY_PATH"]) \
956 if "LIBRARY_PATH" in env
else ""
958 # preserve existing RUSTFLAGS
959 env
.setdefault("RUSTFLAGS", "")
960 build_section
= "target.{}".format(self
.build
)
962 if self
.get_toml("crt-static", build_section
) == "true":
963 target_features
+= ["+crt-static"]
964 elif self
.get_toml("crt-static", build_section
) == "false":
965 target_features
+= ["-crt-static"]
967 env
["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features
))
968 target_linker
= self
.get_toml("linker", build_section
)
969 if target_linker
is not None:
970 env
["RUSTFLAGS"] += " -C linker=" + target_linker
971 env
["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
972 env
["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
973 if self
.get_toml("deny-warnings", "rust") != "false":
974 env
["RUSTFLAGS"] += " -Dwarnings"
976 env
["PATH"] = os
.path
.join(self
.bin_root(True), "bin") + \
977 os
.pathsep
+ env
["PATH"]
978 if not os
.path
.isfile(self
.cargo()):
979 raise Exception("no cargo executable found at `{}`".format(
981 args
= [self
.cargo(), "build", "--manifest-path",
982 os
.path
.join(self
.rust_root
, "src/bootstrap/Cargo.toml")]
983 for _
in range(0, self
.verbose
):
984 args
.append("--verbose")
985 if self
.use_locked_deps
:
986 args
.append("--locked")
987 if self
.use_vendored_sources
:
988 args
.append("--frozen")
989 run(args
, env
=env
, verbose
=self
.verbose
)
991 def build_triple(self
):
992 """Build triple as in LLVM
994 Note that `default_build_triple` is moderately expensive,
995 so use `self.build` where possible.
997 config
= self
.get_toml('build')
1000 return default_build_triple(self
.verbose
)
1002 def check_submodule(self
, module
, slow_submodules
):
1003 if not slow_submodules
:
1004 checked_out
= subprocess
.Popen(["git", "rev-parse", "HEAD"],
1005 cwd
=os
.path
.join(self
.rust_root
, module
),
1006 stdout
=subprocess
.PIPE
)
1011 def update_submodule(self
, module
, checked_out
, recorded_submodules
):
1012 module_path
= os
.path
.join(self
.rust_root
, module
)
1014 if checked_out
is not None:
1015 default_encoding
= sys
.getdefaultencoding()
1016 checked_out
= checked_out
.communicate()[0].decode(default_encoding
).strip()
1017 if recorded_submodules
[module
] == checked_out
:
1020 print("Updating submodule", module
)
1022 run(["git", "submodule", "-q", "sync", module
],
1023 cwd
=self
.rust_root
, verbose
=self
.verbose
)
1025 update_args
= ["git", "submodule", "update", "--init", "--recursive", "--depth=1"]
1026 if self
.git_version
>= distutils
.version
.LooseVersion("2.11.0"):
1027 update_args
.append("--progress")
1028 update_args
.append(module
)
1030 run(update_args
, cwd
=self
.rust_root
, verbose
=self
.verbose
, exception
=True)
1031 except RuntimeError:
1032 print("Failed updating submodule. This is probably due to uncommitted local changes.")
1033 print('Either stash the changes by running "git stash" within the submodule\'s')
1034 print('directory, reset them by running "git reset --hard", or commit them.')
1035 print("To reset all submodules' changes run", end
=" ")
1036 print('"git submodule foreach --recursive git reset --hard".')
1039 run(["git", "reset", "-q", "--hard"],
1040 cwd
=module_path
, verbose
=self
.verbose
)
1041 run(["git", "clean", "-qdfx"],
1042 cwd
=module_path
, verbose
=self
.verbose
)
1044 def update_submodules(self
):
1045 """Update submodules"""
1046 if (not os
.path
.exists(os
.path
.join(self
.rust_root
, ".git"))) or \
1047 self
.get_toml('submodules') == "false":
1050 default_encoding
= sys
.getdefaultencoding()
1052 # check the existence and version of 'git' command
1053 git_version_str
= require(['git', '--version']).split()[2].decode(default_encoding
)
1054 self
.git_version
= distutils
.version
.LooseVersion(git_version_str
)
1056 slow_submodules
= self
.get_toml('fast-submodules') == "false"
1059 print('Unconditionally updating submodules')
1061 print('Updating only changed submodules')
1062 default_encoding
= sys
.getdefaultencoding()
1063 # Only update submodules that are needed to build bootstrap. These are needed because Cargo
1064 # currently requires everything in a workspace to be "locally present" when starting a
1065 # build, and will give a hard error if any Cargo.toml files are missing.
1066 # FIXME: Is there a way to avoid cloning these eagerly? Bootstrap itself doesn't need to
1067 # share a workspace with any tools - maybe it could be excluded from the workspace?
1068 # That will still require cloning the submodules the second you check the standard
1069 # library, though...
1070 # FIXME: Is there a way to avoid hard-coding the submodules required?
1071 # WARNING: keep this in sync with the submodules hard-coded in bootstrap/lib.rs
1073 "src/tools/rust-installer",
1077 "library/backtrace",
1080 filtered_submodules
= []
1081 submodules_names
= []
1082 for module
in submodules
:
1083 check
= self
.check_submodule(module
, slow_submodules
)
1084 filtered_submodules
.append((module
, check
))
1085 submodules_names
.append(module
)
1086 recorded
= subprocess
.Popen(["git", "ls-tree", "HEAD"] + submodules_names
,
1087 cwd
=self
.rust_root
, stdout
=subprocess
.PIPE
)
1088 recorded
= recorded
.communicate()[0].decode(default_encoding
).strip().splitlines()
1089 # { filename: hash }
1090 recorded_submodules
= {}
1091 for data
in recorded
:
1092 # [mode, kind, hash, filename]
1094 recorded_submodules
[data
[3]] = data
[2]
1095 for module
in filtered_submodules
:
1096 self
.update_submodule(module
[0], module
[1], recorded_submodules
)
1097 print("Submodules updated in %.2f seconds" % (time() - start_time
))
1099 def set_dist_environment(self
, url
):
1100 """Set download URL for normal environment"""
1101 if 'RUSTUP_DIST_SERVER' in os
.environ
:
1102 self
._download
_url
= os
.environ
['RUSTUP_DIST_SERVER']
1104 self
._download
_url
= url
1106 def check_vendored_status(self
):
1107 """Check that vendoring is configured properly"""
1108 vendor_dir
= os
.path
.join(self
.rust_root
, 'vendor')
1109 if 'SUDO_USER' in os
.environ
and not self
.use_vendored_sources
:
1110 if os
.environ
.get('USER') != os
.environ
['SUDO_USER']:
1111 self
.use_vendored_sources
= True
1112 print('info: looks like you are running this command under `sudo`')
1113 print(' and so in order to preserve your $HOME this will now')
1114 print(' use vendored sources by default.')
1115 if not os
.path
.exists(vendor_dir
):
1116 print('error: vendoring required, but vendor directory does not exist.')
1117 print(' Run `cargo vendor` without sudo to initialize the '
1118 'vendor directory.')
1119 raise Exception("{} not found".format(vendor_dir
))
1121 if self
.use_vendored_sources
:
1122 config
= ("[source.crates-io]\n"
1123 "replace-with = 'vendored-sources'\n"
1124 "registry = 'https://example.com'\n"
1126 "[source.vendored-sources]\n"
1127 "directory = '{}/vendor'\n"
1128 .format(self
.rust_root
))
1129 if not os
.path
.exists('.cargo'):
1130 os
.makedirs('.cargo')
1131 with
output('.cargo/config') as cargo_config
:
1132 cargo_config
.write(config
)
1134 print('info: using vendored source, but .cargo/config is already present.')
1135 print(' Reusing the current configuration file. But you may want to '
1136 'configure vendoring like this:')
1139 if os
.path
.exists('.cargo'):
1140 shutil
.rmtree('.cargo')
1142 def ensure_vendored(self
):
1143 """Ensure that the vendored sources are available if needed"""
1144 vendor_dir
= os
.path
.join(self
.rust_root
, 'vendor')
1145 # Note that this does not handle updating the vendored dependencies if
1146 # the rust git repository is updated. Normal development usually does
1147 # not use vendoring, so hopefully this isn't too much of a problem.
1148 if self
.use_vendored_sources
and not os
.path
.exists(vendor_dir
):
1152 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1153 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1154 ], verbose
=self
.verbose
, cwd
=self
.rust_root
)
1157 def bootstrap(help_triggered
):
1158 """Configure, fetch, build and run the initial bootstrap"""
1160 # If the user is asking for help, let them know that the whole download-and-build
1161 # process has to happen before anything is printed out.
1163 print("info: Downloading and building bootstrap before processing --help")
1164 print(" command. See src/bootstrap/README.md for help with common")
1167 parser
= argparse
.ArgumentParser(description
='Build rust')
1168 parser
.add_argument('--config')
1169 parser
.add_argument('--build')
1170 parser
.add_argument('--clean', action
='store_true')
1171 parser
.add_argument('-v', '--verbose', action
='count', default
=0)
1173 args
= [a
for a
in sys
.argv
if a
!= '-h' and a
!= '--help']
1174 args
, _
= parser
.parse_known_args(args
)
1176 # Configure initial bootstrap
1178 build
.rust_root
= os
.path
.abspath(os
.path
.join(__file__
, '../../..'))
1179 build
.verbose
= args
.verbose
1180 build
.clean
= args
.clean
1182 # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1184 toml_path
= os
.getenv('RUST_BOOTSTRAP_CONFIG') or args
.config
1185 if not toml_path
and os
.path
.exists('config.toml'):
1186 toml_path
= 'config.toml'
1189 if not os
.path
.exists(toml_path
):
1190 toml_path
= os
.path
.join(build
.rust_root
, toml_path
)
1192 with
open(toml_path
) as config
:
1193 build
.config_toml
= config
.read()
1195 profile
= build
.get_toml('profile')
1196 if profile
is not None:
1197 include_file
= 'config.{}.toml'.format(profile
)
1198 include_dir
= os
.path
.join(build
.rust_root
, 'src', 'bootstrap', 'defaults')
1199 include_path
= os
.path
.join(include_dir
, include_file
)
1200 # HACK: This works because `build.get_toml()` returns the first match it finds for a
1201 # specific key, so appending our defaults at the end allows the user to override them
1202 with
open(include_path
) as included_toml
:
1203 build
.config_toml
+= os
.linesep
+ included_toml
.read()
1205 config_verbose
= build
.get_toml('verbose', 'build')
1206 if config_verbose
is not None:
1207 build
.verbose
= max(build
.verbose
, int(config_verbose
))
1209 build
.use_vendored_sources
= build
.get_toml('vendor', 'build') == 'true'
1211 build
.use_locked_deps
= build
.get_toml('locked-deps', 'build') == 'true'
1213 build
.check_vendored_status()
1215 build_dir
= build
.get_toml('build-dir', 'build') or 'build'
1216 build
.build_dir
= os
.path
.abspath(build_dir
.replace("$ROOT", build
.rust_root
))
1218 with
open(os
.path
.join(build
.rust_root
, "src", "stage0.json")) as f
:
1220 build
.checksums_sha256
= data
["checksums_sha256"]
1221 build
.stage0_compiler
= Stage0Toolchain(data
["compiler"])
1222 if data
.get("rustfmt") is not None:
1223 build
.stage0_rustfmt
= Stage0Toolchain(data
["rustfmt"])
1225 build
.set_dist_environment(data
["dist_server"])
1227 build
.build
= args
.build
or build
.build_triple()
1228 build
.update_submodules()
1230 # Fetch/build the bootstrap
1231 build
.download_toolchain()
1232 # Download the master compiler if `download-rustc` is set
1233 build
.maybe_download_ci_toolchain()
1235 build
.ensure_vendored()
1236 build
.build_bootstrap()
1240 args
= [build
.bootstrap_binary()]
1241 args
.extend(sys
.argv
[1:])
1242 env
= os
.environ
.copy()
1243 env
["BOOTSTRAP_PARENT_ID"] = str(os
.getpid())
1244 env
["BOOTSTRAP_PYTHON"] = sys
.executable
1245 env
["BUILD_DIR"] = build
.build_dir
1246 env
["RUSTC_BOOTSTRAP"] = '1'
1248 env
["BOOTSTRAP_CONFIG"] = toml_path
1249 if build
.rustc_commit
is not None:
1250 env
["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
1251 run(args
, env
=env
, verbose
=build
.verbose
, is_bootstrap
=True)
1255 """Entry point for the bootstrap process"""
1258 # x.py help <cmd> ...
1259 if len(sys
.argv
) > 1 and sys
.argv
[1] == 'help':
1260 sys
.argv
= [sys
.argv
[0], '-h'] + sys
.argv
[2:]
1263 '-h' in sys
.argv
) or ('--help' in sys
.argv
) or (len(sys
.argv
) == 1)
1265 bootstrap(help_triggered
)
1266 if not help_triggered
:
1267 print("Build completed successfully in {}".format(
1268 format_build_time(time() - start_time
)))
1269 except (SystemExit, KeyboardInterrupt) as error
:
1270 if hasattr(error
, 'code') and isinstance(error
.code
, int):
1271 exit_code
= error
.code
1275 if not help_triggered
:
1276 print("Build completed unsuccessfully in {}".format(
1277 format_build_time(time() - start_time
)))
1281 if __name__
== '__main__':