]> git.proxmox.com Git - rustc.git/blob - src/bootstrap/bootstrap.py
New upstream version 1.52.0~beta.3+dfsg1
[rustc.git] / src / bootstrap / bootstrap.py
1 from __future__ import absolute_import, division, print_function
2 import argparse
3 import contextlib
4 import datetime
5 import distutils.version
6 import hashlib
7 import os
8 import re
9 import shutil
10 import subprocess
11 import sys
12 import tarfile
13 import tempfile
14
15 from time import time
16
17 def support_xz():
18 try:
19 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
20 temp_path = temp_file.name
21 with tarfile.open(temp_path, "w:xz"):
22 pass
23 return True
24 except tarfile.CompressionError:
25 return False
26
27 def get(url, path, verbose=False, do_verify=True):
28 suffix = '.sha256'
29 sha_url = url + suffix
30 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
31 temp_path = temp_file.name
32 with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
33 sha_path = sha_file.name
34
35 try:
36 if do_verify:
37 download(sha_path, sha_url, False, verbose)
38 if os.path.exists(path):
39 if verify(path, sha_path, False):
40 if verbose:
41 print("using already-download file", path)
42 return
43 else:
44 if verbose:
45 print("ignoring already-download file",
46 path, "due to failed verification")
47 os.unlink(path)
48 download(temp_path, url, True, verbose)
49 if do_verify and not verify(temp_path, sha_path, verbose):
50 raise RuntimeError("failed verification")
51 if verbose:
52 print("moving {} to {}".format(temp_path, path))
53 shutil.move(temp_path, path)
54 finally:
55 delete_if_present(sha_path, verbose)
56 delete_if_present(temp_path, verbose)
57
58
59 def delete_if_present(path, verbose):
60 """Remove the given file if present"""
61 if os.path.isfile(path):
62 if verbose:
63 print("removing", path)
64 os.unlink(path)
65
66
67 def download(path, url, probably_big, verbose):
68 for _ in range(0, 4):
69 try:
70 _download(path, url, probably_big, verbose, True)
71 return
72 except RuntimeError:
73 print("\nspurious failure, trying again")
74 _download(path, url, probably_big, verbose, False)
75
76
77 def _download(path, url, probably_big, verbose, exception):
78 if probably_big or verbose:
79 print("downloading {}".format(url))
80 # see http://serverfault.com/questions/301128/how-to-download
81 if sys.platform == 'win32':
82 run(["PowerShell.exe", "/nologo", "-Command",
83 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
84 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
85 verbose=verbose,
86 exception=exception)
87 else:
88 if probably_big or verbose:
89 option = "-#"
90 else:
91 option = "-s"
92 require(["curl", "--version"])
93 run(["curl", option,
94 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds
95 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
96 "--retry", "3", "-Sf", "-o", path, url],
97 verbose=verbose,
98 exception=exception)
99
100
101 def verify(path, sha_path, verbose):
102 """Check if the sha256 sum of the given path is valid"""
103 if verbose:
104 print("verifying", path)
105 with open(path, "rb") as source:
106 found = hashlib.sha256(source.read()).hexdigest()
107 with open(sha_path, "r") as sha256sum:
108 expected = sha256sum.readline().split()[0]
109 verified = found == expected
110 if not verified:
111 print("invalid checksum:\n"
112 " found: {}\n"
113 " expected: {}".format(found, expected))
114 return verified
115
116
117 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
118 """Unpack the given tarball file"""
119 print("extracting", tarball)
120 fname = os.path.basename(tarball).replace(tarball_suffix, "")
121 with contextlib.closing(tarfile.open(tarball)) as tar:
122 for member in tar.getnames():
123 if "/" not in member:
124 continue
125 name = member.replace(fname + "/", "", 1)
126 if match is not None and not name.startswith(match):
127 continue
128 name = name[len(match) + 1:]
129
130 dst_path = os.path.join(dst, name)
131 if verbose:
132 print(" extracting", member)
133 tar.extract(member, dst)
134 src_path = os.path.join(dst, member)
135 if os.path.isdir(src_path) and os.path.exists(dst_path):
136 continue
137 shutil.move(src_path, dst_path)
138 shutil.rmtree(os.path.join(dst, fname))
139
140
141 def run(args, verbose=False, exception=False, **kwargs):
142 """Run a child program in a new process"""
143 if verbose:
144 print("running: " + ' '.join(args))
145 sys.stdout.flush()
146 # Use Popen here instead of call() as it apparently allows powershell on
147 # Windows to not lock up waiting for input presumably.
148 ret = subprocess.Popen(args, **kwargs)
149 code = ret.wait()
150 if code != 0:
151 err = "failed to run: " + ' '.join(args)
152 if verbose or exception:
153 raise RuntimeError(err)
154 sys.exit(err)
155
156
157 def require(cmd, exit=True):
158 '''Run a command, returning its output.
159 On error,
160 If `exit` is `True`, exit the process.
161 Otherwise, return None.'''
162 try:
163 return subprocess.check_output(cmd).strip()
164 except (subprocess.CalledProcessError, OSError) as exc:
165 if not exit:
166 return None
167 print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
168 print("Please make sure it's installed and in the path.")
169 sys.exit(1)
170
171
172 def stage0_data(rust_root):
173 """Build a dictionary from stage0.txt"""
174 nightlies = os.path.join(rust_root, "src/stage0.txt")
175 with open(nightlies, 'r') as nightlies:
176 lines = [line.rstrip() for line in nightlies
177 if not line.startswith("#")]
178 return dict([line.split(": ", 1) for line in lines if line])
179
180
181 def format_build_time(duration):
182 """Return a nicer format for build time
183
184 >>> format_build_time('300')
185 '0:05:00'
186 """
187 return str(datetime.timedelta(seconds=int(duration)))
188
189
190 def default_build_triple(verbose):
191 """Build triple as in LLVM"""
192 # If the user already has a host build triple with an existing `rustc`
193 # install, use their preference. This fixes most issues with Windows builds
194 # being detected as GNU instead of MSVC.
195 default_encoding = sys.getdefaultencoding()
196 try:
197 version = subprocess.check_output(["rustc", "--version", "--verbose"],
198 stderr=subprocess.DEVNULL)
199 version = version.decode(default_encoding)
200 host = next(x for x in version.split('\n') if x.startswith("host: "))
201 triple = host.split("host: ")[1]
202 if verbose:
203 print("detected default triple {}".format(triple))
204 return triple
205 except Exception as e:
206 if verbose:
207 print("rustup not detected: {}".format(e))
208 print("falling back to auto-detect")
209
210 required = sys.platform != 'win32'
211 ostype = require(["uname", "-s"], exit=required)
212 cputype = require(['uname', '-m'], exit=required)
213
214 # If we do not have `uname`, assume Windows.
215 if ostype is None or cputype is None:
216 return 'x86_64-pc-windows-msvc'
217
218 ostype = ostype.decode(default_encoding)
219 cputype = cputype.decode(default_encoding)
220
221 # The goal here is to come up with the same triple as LLVM would,
222 # at least for the subset of platforms we're willing to target.
223 ostype_mapper = {
224 'Darwin': 'apple-darwin',
225 'DragonFly': 'unknown-dragonfly',
226 'FreeBSD': 'unknown-freebsd',
227 'Haiku': 'unknown-haiku',
228 'NetBSD': 'unknown-netbsd',
229 'OpenBSD': 'unknown-openbsd'
230 }
231
232 # Consider the direct transformation first and then the special cases
233 if ostype in ostype_mapper:
234 ostype = ostype_mapper[ostype]
235 elif ostype == 'Linux':
236 os_from_sp = subprocess.check_output(
237 ['uname', '-o']).strip().decode(default_encoding)
238 if os_from_sp == 'Android':
239 ostype = 'linux-android'
240 else:
241 ostype = 'unknown-linux-gnu'
242 elif ostype == 'SunOS':
243 ostype = 'pc-solaris'
244 # On Solaris, uname -m will return a machine classification instead
245 # of a cpu type, so uname -p is recommended instead. However, the
246 # output from that option is too generic for our purposes (it will
247 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
248 # must be used instead.
249 cputype = require(['isainfo', '-k']).decode(default_encoding)
250 # sparc cpus have sun as a target vendor
251 if 'sparc' in cputype:
252 ostype = 'sun-solaris'
253 elif ostype.startswith('MINGW'):
254 # msys' `uname` does not print gcc configuration, but prints msys
255 # configuration. so we cannot believe `uname -m`:
256 # msys1 is always i686 and msys2 is always x86_64.
257 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
258 # MINGW64 on x86_64.
259 ostype = 'pc-windows-gnu'
260 cputype = 'i686'
261 if os.environ.get('MSYSTEM') == 'MINGW64':
262 cputype = 'x86_64'
263 elif ostype.startswith('MSYS'):
264 ostype = 'pc-windows-gnu'
265 elif ostype.startswith('CYGWIN_NT'):
266 cputype = 'i686'
267 if ostype.endswith('WOW64'):
268 cputype = 'x86_64'
269 ostype = 'pc-windows-gnu'
270 elif sys.platform == 'win32':
271 # Some Windows platforms might have a `uname` command that returns a
272 # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
273 # these cases, fall back to using sys.platform.
274 return 'x86_64-pc-windows-msvc'
275 else:
276 err = "unknown OS type: {}".format(ostype)
277 sys.exit(err)
278
279 if cputype == 'powerpc' and ostype == 'unknown-freebsd':
280 cputype = subprocess.check_output(
281 ['uname', '-p']).strip().decode(default_encoding)
282 cputype_mapper = {
283 'BePC': 'i686',
284 'aarch64': 'aarch64',
285 'amd64': 'x86_64',
286 'arm64': 'aarch64',
287 'i386': 'i686',
288 'i486': 'i686',
289 'i686': 'i686',
290 'i786': 'i686',
291 'powerpc': 'powerpc',
292 'powerpc64': 'powerpc64',
293 'powerpc64le': 'powerpc64le',
294 'ppc': 'powerpc',
295 'ppc64': 'powerpc64',
296 'ppc64le': 'powerpc64le',
297 's390x': 's390x',
298 'x64': 'x86_64',
299 'x86': 'i686',
300 'x86-64': 'x86_64',
301 'x86_64': 'x86_64'
302 }
303
304 # Consider the direct transformation first and then the special cases
305 if cputype in cputype_mapper:
306 cputype = cputype_mapper[cputype]
307 elif cputype in {'xscale', 'arm'}:
308 cputype = 'arm'
309 if ostype == 'linux-android':
310 ostype = 'linux-androideabi'
311 elif ostype == 'unknown-freebsd':
312 cputype = subprocess.check_output(
313 ['uname', '-p']).strip().decode(default_encoding)
314 ostype = 'unknown-freebsd'
315 elif cputype == 'armv6l':
316 cputype = 'arm'
317 if ostype == 'linux-android':
318 ostype = 'linux-androideabi'
319 else:
320 ostype += 'eabihf'
321 elif cputype in {'armv7l', 'armv8l'}:
322 cputype = 'armv7'
323 if ostype == 'linux-android':
324 ostype = 'linux-androideabi'
325 else:
326 ostype += 'eabihf'
327 elif cputype == 'mips':
328 if sys.byteorder == 'big':
329 cputype = 'mips'
330 elif sys.byteorder == 'little':
331 cputype = 'mipsel'
332 else:
333 raise ValueError("unknown byteorder: {}".format(sys.byteorder))
334 elif cputype == 'mips64':
335 if sys.byteorder == 'big':
336 cputype = 'mips64'
337 elif sys.byteorder == 'little':
338 cputype = 'mips64el'
339 else:
340 raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
341 # only the n64 ABI is supported, indicate it
342 ostype += 'abi64'
343 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
344 pass
345 else:
346 err = "unknown cpu type: {}".format(cputype)
347 sys.exit(err)
348
349 return "{}-{}".format(cputype, ostype)
350
351
352 @contextlib.contextmanager
353 def output(filepath):
354 tmp = filepath + '.tmp'
355 with open(tmp, 'w') as f:
356 yield f
357 try:
358 if os.path.exists(filepath):
359 os.remove(filepath) # PermissionError/OSError on Win32 if in use
360 except OSError:
361 shutil.copy2(tmp, filepath)
362 os.remove(tmp)
363 return
364 os.rename(tmp, filepath)
365
366
367 class RustBuild(object):
368 """Provide all the methods required to build Rust"""
369 def __init__(self):
370 self.date = ''
371 self._download_url = ''
372 self.rustc_channel = ''
373 self.rustfmt_channel = ''
374 self.build = ''
375 self.build_dir = ''
376 self.clean = False
377 self.config_toml = ''
378 self.rust_root = ''
379 self.use_locked_deps = ''
380 self.use_vendored_sources = ''
381 self.verbose = False
382 self.git_version = None
383 self.nix_deps_dir = None
384 self.rustc_commit = None
385
386 def download_stage0(self):
387 """Fetch the build system for Rust, written in Rust
388
389 This method will build a cache directory, then it will fetch the
390 tarball which has the stage0 compiler used to then bootstrap the Rust
391 compiler itself.
392
393 Each downloaded tarball is extracted, after that, the script
394 will move all the content to the right place.
395 """
396 rustc_channel = self.rustc_channel
397 rustfmt_channel = self.rustfmt_channel
398
399 if self.rustc().startswith(self.bin_root()) and \
400 (not os.path.exists(self.rustc()) or
401 self.program_out_of_date(self.rustc_stamp(), self.date + str(self.rustc_commit))):
402 if os.path.exists(self.bin_root()):
403 shutil.rmtree(self.bin_root())
404 download_rustc = self.rustc_commit is not None
405 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
406 filename = "rust-std-{}-{}{}".format(
407 rustc_channel, self.build, tarball_suffix)
408 pattern = "rust-std-{}".format(self.build)
409 self._download_component_helper(filename, pattern, tarball_suffix, download_rustc)
410 filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
411 tarball_suffix)
412 self._download_component_helper(filename, "rustc", tarball_suffix, download_rustc)
413 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
414 tarball_suffix)
415 self._download_component_helper(filename, "cargo", tarball_suffix)
416 if self.rustc_commit is not None:
417 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
418 self._download_component_helper(
419 filename, "rustc-dev", tarball_suffix, download_rustc
420 )
421
422 self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
423 self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
424 self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
425 lib_dir = "{}/lib".format(self.bin_root())
426 for lib in os.listdir(lib_dir):
427 if lib.endswith(".so"):
428 self.fix_bin_or_dylib(os.path.join(lib_dir, lib), rpath_libz=True)
429 with output(self.rustc_stamp()) as rust_stamp:
430 rust_stamp.write(self.date + str(self.rustc_commit))
431
432 if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
433 not os.path.exists(self.rustfmt())
434 or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
435 ):
436 if rustfmt_channel:
437 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
438 [channel, date] = rustfmt_channel.split('-', 1)
439 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
440 self._download_component_helper(
441 filename, "rustfmt-preview", tarball_suffix, key=date
442 )
443 self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
444 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
445 with output(self.rustfmt_stamp()) as rustfmt_stamp:
446 rustfmt_stamp.write(self.rustfmt_channel)
447
448 if self.downloading_llvm():
449 # We want the most recent LLVM submodule update to avoid downloading
450 # LLVM more often than necessary.
451 #
452 # This git command finds that commit SHA, looking for bors-authored
453 # merges that modified src/llvm-project.
454 #
455 # This works even in a repository that has not yet initialized
456 # submodules.
457 top_level = subprocess.check_output([
458 "git", "rev-parse", "--show-toplevel",
459 ]).decode(sys.getdefaultencoding()).strip()
460 llvm_sha = subprocess.check_output([
461 "git", "log", "--author=bors", "--format=%H", "-n1",
462 "-m", "--first-parent",
463 "--",
464 "{}/src/llvm-project".format(top_level),
465 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
466 ]).decode(sys.getdefaultencoding()).strip()
467 llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
468 llvm_root = self.llvm_root()
469 llvm_lib = os.path.join(llvm_root, "lib")
470 if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
471 self._download_ci_llvm(llvm_sha, llvm_assertions)
472 for binary in ["llvm-config", "FileCheck"]:
473 self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary), rpath_libz=True)
474 for lib in os.listdir(llvm_lib):
475 if lib.endswith(".so"):
476 self.fix_bin_or_dylib(os.path.join(llvm_lib, lib), rpath_libz=True)
477 with output(self.llvm_stamp()) as llvm_stamp:
478 llvm_stamp.write(llvm_sha + str(llvm_assertions))
479
480 def downloading_llvm(self):
481 opt = self.get_toml('download-ci-llvm', 'llvm')
482 # This is currently all tier 1 targets (since others may not have CI
483 # artifacts)
484 # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
485 supported_platforms = [
486 "aarch64-unknown-linux-gnu",
487 "i686-pc-windows-gnu",
488 "i686-pc-windows-msvc",
489 "i686-unknown-linux-gnu",
490 "x86_64-unknown-linux-gnu",
491 "x86_64-apple-darwin",
492 "x86_64-pc-windows-gnu",
493 "x86_64-pc-windows-msvc",
494 ]
495 return opt == "true" \
496 or (opt == "if-available" and self.build in supported_platforms)
497
498 def _download_component_helper(
499 self, filename, pattern, tarball_suffix, download_rustc=False, key=None
500 ):
501 if key is None:
502 if download_rustc:
503 key = self.rustc_commit
504 else:
505 key = self.date
506 cache_dst = os.path.join(self.build_dir, "cache")
507 rustc_cache = os.path.join(cache_dst, key)
508 if not os.path.exists(rustc_cache):
509 os.makedirs(rustc_cache)
510
511 if download_rustc:
512 url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(self.rustc_commit)
513 else:
514 url = "{}/dist/{}".format(self._download_url, key)
515 tarball = os.path.join(rustc_cache, filename)
516 if not os.path.exists(tarball):
517 do_verify = not download_rustc
518 get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=do_verify)
519 unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
520
521 def _download_ci_llvm(self, llvm_sha, llvm_assertions):
522 cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
523 cache_dst = os.path.join(self.build_dir, "cache")
524 rustc_cache = os.path.join(cache_dst, cache_prefix)
525 if not os.path.exists(rustc_cache):
526 os.makedirs(rustc_cache)
527
528 url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
529 if llvm_assertions:
530 url = url.replace('rustc-builds', 'rustc-builds-alt')
531 # ci-artifacts are only stored as .xz, not .gz
532 if not support_xz():
533 print("error: XZ support is required to download LLVM")
534 print("help: consider disabling `download-ci-llvm` or using python3")
535 exit(1)
536 tarball_suffix = '.tar.xz'
537 filename = "rust-dev-nightly-" + self.build + tarball_suffix
538 tarball = os.path.join(rustc_cache, filename)
539 if not os.path.exists(tarball):
540 get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
541 unpack(tarball, tarball_suffix, self.llvm_root(),
542 match="rust-dev",
543 verbose=self.verbose)
544
545 def fix_bin_or_dylib(self, fname, rpath_libz=False):
546 """Modifies the interpreter section of 'fname' to fix the dynamic linker,
547 or the RPATH section, to fix the dynamic library search path
548
549 This method is only required on NixOS and uses the PatchELF utility to
550 change the interpreter/RPATH of ELF executables.
551
552 Please see https://nixos.org/patchelf.html for more information
553 """
554 default_encoding = sys.getdefaultencoding()
555 try:
556 ostype = subprocess.check_output(
557 ['uname', '-s']).strip().decode(default_encoding)
558 except subprocess.CalledProcessError:
559 return
560 except OSError as reason:
561 if getattr(reason, 'winerror', None) is not None:
562 return
563 raise reason
564
565 if ostype != "Linux":
566 return
567
568 if not os.path.exists("/etc/NIXOS"):
569 return
570 if os.path.exists("/lib"):
571 return
572
573 # At this point we're pretty sure the user is running NixOS
574 nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
575 print(nix_os_msg, fname)
576
577 # Only build `stage0/.nix-deps` once.
578 nix_deps_dir = self.nix_deps_dir
579 if not nix_deps_dir:
580 nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
581 if not os.path.exists(nix_deps_dir):
582 os.makedirs(nix_deps_dir)
583
584 nix_deps = [
585 # Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
586 "stdenv.cc.bintools",
587
588 # Needed as a system dependency of `libLLVM-*.so`.
589 "zlib",
590
591 # Needed for patching ELF binaries (see doc comment above).
592 "patchelf",
593 ]
594
595 # Run `nix-build` to "build" each dependency (which will likely reuse
596 # the existing `/nix/store` copy, or at most download a pre-built copy).
597 # Importantly, we don't rely on `nix-build` printing the `/nix/store`
598 # path on stdout, but use `-o` to symlink it into `stage0/.nix-deps/$dep`,
599 # ensuring garbage collection will never remove the `/nix/store` path
600 # (which would break our patched binaries that hardcode those paths).
601 for dep in nix_deps:
602 try:
603 subprocess.check_output([
604 "nix-build", "<nixpkgs>",
605 "-A", dep,
606 "-o", "{}/{}".format(nix_deps_dir, dep),
607 ])
608 except subprocess.CalledProcessError as reason:
609 print("warning: failed to call nix-build:", reason)
610 return
611
612 self.nix_deps_dir = nix_deps_dir
613
614 patchelf = "{}/patchelf/bin/patchelf".format(nix_deps_dir)
615 patchelf_args = []
616
617 if rpath_libz:
618 # Patch RPATH to add `zlib` dependency that stems from LLVM
619 dylib_deps = ["zlib"]
620 rpath_entries = [
621 # Relative default, all binary and dynamic libraries we ship
622 # appear to have this (even when `../lib` is redundant).
623 "$ORIGIN/../lib",
624 ] + ["{}/{}/lib".format(nix_deps_dir, dep) for dep in dylib_deps]
625 patchelf_args += ["--set-rpath", ":".join(rpath_entries)]
626 if not fname.endswith(".so"):
627 # Finally, set the corret .interp for binaries
628 bintools_dir = "{}/stdenv.cc.bintools".format(nix_deps_dir)
629 with open("{}/nix-support/dynamic-linker".format(bintools_dir)) as dynamic_linker:
630 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
631
632 try:
633 subprocess.check_output([patchelf] + patchelf_args + [fname])
634 except subprocess.CalledProcessError as reason:
635 print("warning: failed to call patchelf:", reason)
636 return
637
638 # Return the stage1 compiler to download, if any.
639 def maybe_download_rustc(self):
640 # If `download-rustc` is not set, default to rebuilding.
641 if self.get_toml("download-rustc", section="rust") != "true":
642 return None
643
644 # Handle running from a directory other than the top level
645 rev_parse = ["git", "rev-parse", "--show-toplevel"]
646 top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
647 compiler = "{}/compiler/".format(top_level)
648
649 # Look for a version to compare to based on the current commit.
650 # Only commits merged by bors will have CI artifacts.
651 merge_base = ["git", "log", "--author=bors", "--pretty=%H", "-n1"]
652 commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
653
654 # Warn if there were changes to the compiler since the ancestor commit.
655 status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler])
656 if status != 0:
657 print("warning: `download-rustc` is enabled, but there are changes to compiler/")
658
659 return commit
660
661 def rustc_stamp(self):
662 """Return the path for .rustc-stamp
663
664 >>> rb = RustBuild()
665 >>> rb.build_dir = "build"
666 >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
667 True
668 """
669 return os.path.join(self.bin_root(), '.rustc-stamp')
670
671 def rustfmt_stamp(self):
672 """Return the path for .rustfmt-stamp
673
674 >>> rb = RustBuild()
675 >>> rb.build_dir = "build"
676 >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
677 True
678 """
679 return os.path.join(self.bin_root(), '.rustfmt-stamp')
680
681 def llvm_stamp(self):
682 """Return the path for .rustfmt-stamp
683
684 >>> rb = RustBuild()
685 >>> rb.build_dir = "build"
686 >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
687 True
688 """
689 return os.path.join(self.llvm_root(), '.llvm-stamp')
690
691
692 def program_out_of_date(self, stamp_path, key):
693 """Check if the given program stamp is out of date"""
694 if not os.path.exists(stamp_path) or self.clean:
695 return True
696 with open(stamp_path, 'r') as stamp:
697 return key != stamp.read()
698
699 def bin_root(self):
700 """Return the binary root directory
701
702 >>> rb = RustBuild()
703 >>> rb.build_dir = "build"
704 >>> rb.bin_root() == os.path.join("build", "stage0")
705 True
706
707 When the 'build' property is given should be a nested directory:
708
709 >>> rb.build = "devel"
710 >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
711 True
712 """
713 return os.path.join(self.build_dir, self.build, "stage0")
714
715 def llvm_root(self):
716 """Return the CI LLVM root directory
717
718 >>> rb = RustBuild()
719 >>> rb.build_dir = "build"
720 >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
721 True
722
723 When the 'build' property is given should be a nested directory:
724
725 >>> rb.build = "devel"
726 >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
727 True
728 """
729 return os.path.join(self.build_dir, self.build, "ci-llvm")
730
731 def get_toml(self, key, section=None):
732 """Returns the value of the given key in config.toml, otherwise returns None
733
734 >>> rb = RustBuild()
735 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
736 >>> rb.get_toml("key2")
737 'value2'
738
739 If the key does not exists, the result is None:
740
741 >>> rb.get_toml("key3") is None
742 True
743
744 Optionally also matches the section the key appears in
745
746 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
747 >>> rb.get_toml('key', 'a')
748 'value1'
749 >>> rb.get_toml('key', 'b')
750 'value2'
751 >>> rb.get_toml('key', 'c') is None
752 True
753
754 >>> rb.config_toml = 'key1 = true'
755 >>> rb.get_toml("key1")
756 'true'
757 """
758
759 cur_section = None
760 for line in self.config_toml.splitlines():
761 section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
762 if section_match is not None:
763 cur_section = section_match.group(1)
764
765 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
766 if match is not None:
767 value = match.group(1)
768 if section is None or section == cur_section:
769 return self.get_string(value) or value.strip()
770 return None
771
772 def cargo(self):
773 """Return config path for cargo"""
774 return self.program_config('cargo')
775
776 def rustc(self):
777 """Return config path for rustc"""
778 return self.program_config('rustc')
779
780 def rustfmt(self):
781 """Return config path for rustfmt"""
782 if not self.rustfmt_channel:
783 return None
784 return self.program_config('rustfmt')
785
786 def program_config(self, program):
787 """Return config path for the given program
788
789 >>> rb = RustBuild()
790 >>> rb.config_toml = 'rustc = "rustc"\\n'
791 >>> rb.program_config('rustc')
792 'rustc'
793 >>> rb.config_toml = ''
794 >>> cargo_path = rb.program_config('cargo')
795 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
796 ... "bin", "cargo")
797 True
798 """
799 config = self.get_toml(program)
800 if config:
801 return os.path.expanduser(config)
802 return os.path.join(self.bin_root(), "bin", "{}{}".format(
803 program, self.exe_suffix()))
804
805 @staticmethod
806 def get_string(line):
807 """Return the value between double quotes
808
809 >>> RustBuild.get_string(' "devel" ')
810 'devel'
811 >>> RustBuild.get_string(" 'devel' ")
812 'devel'
813 >>> RustBuild.get_string('devel') is None
814 True
815 >>> RustBuild.get_string(' "devel ')
816 ''
817 """
818 start = line.find('"')
819 if start != -1:
820 end = start + 1 + line[start + 1:].find('"')
821 return line[start + 1:end]
822 start = line.find('\'')
823 if start != -1:
824 end = start + 1 + line[start + 1:].find('\'')
825 return line[start + 1:end]
826 return None
827
828 @staticmethod
829 def exe_suffix():
830 """Return a suffix for executables"""
831 if sys.platform == 'win32':
832 return '.exe'
833 return ''
834
835 def bootstrap_binary(self):
836 """Return the path of the bootstrap binary
837
838 >>> rb = RustBuild()
839 >>> rb.build_dir = "build"
840 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
841 ... "debug", "bootstrap")
842 True
843 """
844 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
845
846 def build_bootstrap(self):
847 """Build bootstrap"""
848 build_dir = os.path.join(self.build_dir, "bootstrap")
849 if self.clean and os.path.exists(build_dir):
850 shutil.rmtree(build_dir)
851 env = os.environ.copy()
852 # `CARGO_BUILD_TARGET` breaks bootstrap build.
853 # See also: <https://github.com/rust-lang/rust/issues/70208>.
854 if "CARGO_BUILD_TARGET" in env:
855 del env["CARGO_BUILD_TARGET"]
856 env["CARGO_TARGET_DIR"] = build_dir
857 env["RUSTC"] = self.rustc()
858 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
859 (os.pathsep + env["LD_LIBRARY_PATH"]) \
860 if "LD_LIBRARY_PATH" in env else ""
861 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
862 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
863 if "DYLD_LIBRARY_PATH" in env else ""
864 env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
865 (os.pathsep + env["LIBRARY_PATH"]) \
866 if "LIBRARY_PATH" in env else ""
867 # preserve existing RUSTFLAGS
868 env.setdefault("RUSTFLAGS", "")
869 env["RUSTFLAGS"] += " -Cdebuginfo=2"
870
871 build_section = "target.{}".format(self.build)
872 target_features = []
873 if self.get_toml("crt-static", build_section) == "true":
874 target_features += ["+crt-static"]
875 elif self.get_toml("crt-static", build_section) == "false":
876 target_features += ["-crt-static"]
877 if target_features:
878 env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
879 target_linker = self.get_toml("linker", build_section)
880 if target_linker is not None:
881 env["RUSTFLAGS"] += " -C linker=" + target_linker
882 # cfg(bootstrap): Add `-Wsemicolon_in_expressions_from_macros` after the next beta bump
883 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
884 if self.get_toml("deny-warnings", "rust") != "false":
885 env["RUSTFLAGS"] += " -Dwarnings"
886
887 env["PATH"] = os.path.join(self.bin_root(), "bin") + \
888 os.pathsep + env["PATH"]
889 if not os.path.isfile(self.cargo()):
890 raise Exception("no cargo executable found at `{}`".format(
891 self.cargo()))
892 args = [self.cargo(), "build", "--manifest-path",
893 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
894 for _ in range(1, self.verbose):
895 args.append("--verbose")
896 if self.use_locked_deps:
897 args.append("--locked")
898 if self.use_vendored_sources:
899 args.append("--frozen")
900 run(args, env=env, verbose=self.verbose)
901
902 def build_triple(self):
903 """Build triple as in LLVM
904
905 Note that `default_build_triple` is moderately expensive,
906 so use `self.build` where possible.
907 """
908 config = self.get_toml('build')
909 if config:
910 return config
911 return default_build_triple(self.verbose)
912
913 def check_submodule(self, module, slow_submodules):
914 if not slow_submodules:
915 checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
916 cwd=os.path.join(self.rust_root, module),
917 stdout=subprocess.PIPE)
918 return checked_out
919 else:
920 return None
921
922 def update_submodule(self, module, checked_out, recorded_submodules):
923 module_path = os.path.join(self.rust_root, module)
924
925 if checked_out is not None:
926 default_encoding = sys.getdefaultencoding()
927 checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
928 if recorded_submodules[module] == checked_out:
929 return
930
931 print("Updating submodule", module)
932
933 run(["git", "submodule", "-q", "sync", module],
934 cwd=self.rust_root, verbose=self.verbose)
935
936 update_args = ["git", "submodule", "update", "--init", "--recursive"]
937 if self.git_version >= distutils.version.LooseVersion("2.11.0"):
938 update_args.append("--progress")
939 update_args.append(module)
940 run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
941
942 run(["git", "reset", "-q", "--hard"],
943 cwd=module_path, verbose=self.verbose)
944 run(["git", "clean", "-qdfx"],
945 cwd=module_path, verbose=self.verbose)
946
947 def update_submodules(self):
948 """Update submodules"""
949 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
950 self.get_toml('submodules') == "false":
951 return
952
953 default_encoding = sys.getdefaultencoding()
954
955 # check the existence and version of 'git' command
956 git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
957 self.git_version = distutils.version.LooseVersion(git_version_str)
958
959 slow_submodules = self.get_toml('fast-submodules') == "false"
960 start_time = time()
961 if slow_submodules:
962 print('Unconditionally updating all submodules')
963 else:
964 print('Updating only changed submodules')
965 default_encoding = sys.getdefaultencoding()
966 submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
967 ["git", "config", "--file",
968 os.path.join(self.rust_root, ".gitmodules"),
969 "--get-regexp", "path"]
970 ).decode(default_encoding).splitlines()]
971 filtered_submodules = []
972 submodules_names = []
973 llvm_checked_out = os.path.exists(os.path.join(self.rust_root, "src/llvm-project/.git"))
974 external_llvm_provided = self.get_toml('llvm-config') or self.downloading_llvm()
975 llvm_needed = not self.get_toml('codegen-backends', 'rust') \
976 or "llvm" in self.get_toml('codegen-backends', 'rust')
977 for module in submodules:
978 if module.endswith("llvm-project"):
979 # Don't sync the llvm-project submodule if an external LLVM was
980 # provided, if we are downloading LLVM or if the LLVM backend is
981 # not being built. Also, if the submodule has been initialized
982 # already, sync it anyways so that it doesn't mess up contributor
983 # pull requests.
984 if external_llvm_provided or not llvm_needed:
985 if self.get_toml('lld') != 'true' and not llvm_checked_out:
986 continue
987 check = self.check_submodule(module, slow_submodules)
988 filtered_submodules.append((module, check))
989 submodules_names.append(module)
990 recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
991 cwd=self.rust_root, stdout=subprocess.PIPE)
992 recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
993 recorded_submodules = {}
994 for data in recorded:
995 data = data.split()
996 recorded_submodules[data[3]] = data[2]
997 for module in filtered_submodules:
998 self.update_submodule(module[0], module[1], recorded_submodules)
999 print("Submodules updated in %.2f seconds" % (time() - start_time))
1000
1001 def set_normal_environment(self):
1002 """Set download URL for normal environment"""
1003 if 'RUSTUP_DIST_SERVER' in os.environ:
1004 self._download_url = os.environ['RUSTUP_DIST_SERVER']
1005 else:
1006 self._download_url = 'https://static.rust-lang.org'
1007
1008 def set_dev_environment(self):
1009 """Set download URL for development environment"""
1010 if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
1011 self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
1012 else:
1013 self._download_url = 'https://dev-static.rust-lang.org'
1014
1015 def check_vendored_status(self):
1016 """Check that vendoring is configured properly"""
1017 vendor_dir = os.path.join(self.rust_root, 'vendor')
1018 if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1019 if os.environ.get('USER') != os.environ['SUDO_USER']:
1020 self.use_vendored_sources = True
1021 print('info: looks like you are running this command under `sudo`')
1022 print(' and so in order to preserve your $HOME this will now')
1023 print(' use vendored sources by default.')
1024 if not os.path.exists(vendor_dir):
1025 print('error: vendoring required, but vendor directory does not exist.')
1026 print(' Run `cargo vendor` without sudo to initialize the '
1027 'vendor directory.')
1028 raise Exception("{} not found".format(vendor_dir))
1029
1030 if self.use_vendored_sources:
1031 if not os.path.exists('.cargo'):
1032 os.makedirs('.cargo')
1033 with output('.cargo/config') as cargo_config:
1034 cargo_config.write(
1035 "[source.crates-io]\n"
1036 "replace-with = 'vendored-sources'\n"
1037 "registry = 'https://example.com'\n"
1038 "\n"
1039 "[source.vendored-sources]\n"
1040 "directory = '{}/vendor'\n"
1041 .format(self.rust_root))
1042 else:
1043 if os.path.exists('.cargo'):
1044 shutil.rmtree('.cargo')
1045
1046 def ensure_vendored(self):
1047 """Ensure that the vendored sources are available if needed"""
1048 vendor_dir = os.path.join(self.rust_root, 'vendor')
1049 # Note that this does not handle updating the vendored dependencies if
1050 # the rust git repository is updated. Normal development usually does
1051 # not use vendoring, so hopefully this isn't too much of a problem.
1052 if self.use_vendored_sources and not os.path.exists(vendor_dir):
1053 run([
1054 self.cargo(),
1055 "vendor",
1056 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1057 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1058 ], verbose=self.verbose, cwd=self.rust_root)
1059
1060
1061 def bootstrap(help_triggered):
1062 """Configure, fetch, build and run the initial bootstrap"""
1063
1064 # If the user is asking for help, let them know that the whole download-and-build
1065 # process has to happen before anything is printed out.
1066 if help_triggered:
1067 print("info: Downloading and building bootstrap before processing --help")
1068 print(" command. See src/bootstrap/README.md for help with common")
1069 print(" commands.")
1070
1071 parser = argparse.ArgumentParser(description='Build rust')
1072 parser.add_argument('--config')
1073 parser.add_argument('--build')
1074 parser.add_argument('--clean', action='store_true')
1075 parser.add_argument('-v', '--verbose', action='count', default=0)
1076
1077 args = [a for a in sys.argv if a != '-h' and a != '--help']
1078 args, _ = parser.parse_known_args(args)
1079
1080 # Configure initial bootstrap
1081 build = RustBuild()
1082 build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1083 build.verbose = args.verbose
1084 build.clean = args.clean
1085
1086 # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1087 # exists).
1088 toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1089 if not toml_path and os.path.exists('config.toml'):
1090 toml_path = 'config.toml'
1091
1092 if toml_path:
1093 if not os.path.exists(toml_path):
1094 toml_path = os.path.join(build.rust_root, toml_path)
1095
1096 with open(toml_path) as config:
1097 build.config_toml = config.read()
1098
1099 profile = build.get_toml('profile')
1100 if profile is not None:
1101 include_file = 'config.{}.toml'.format(profile)
1102 include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1103 include_path = os.path.join(include_dir, include_file)
1104 # HACK: This works because `build.get_toml()` returns the first match it finds for a
1105 # specific key, so appending our defaults at the end allows the user to override them
1106 with open(include_path) as included_toml:
1107 build.config_toml += os.linesep + included_toml.read()
1108
1109 config_verbose = build.get_toml('verbose', 'build')
1110 if config_verbose is not None:
1111 build.verbose = max(build.verbose, int(config_verbose))
1112
1113 build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1114
1115 build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1116
1117 build.check_vendored_status()
1118
1119 build_dir = build.get_toml('build-dir', 'build') or 'build'
1120 build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1121
1122 data = stage0_data(build.rust_root)
1123 build.date = data['date']
1124 build.rustc_channel = data['rustc']
1125
1126 if "rustfmt" in data:
1127 build.rustfmt_channel = data['rustfmt']
1128
1129 if 'dev' in data:
1130 build.set_dev_environment()
1131 else:
1132 build.set_normal_environment()
1133
1134 build.build = args.build or build.build_triple()
1135 build.update_submodules()
1136
1137 # Fetch/build the bootstrap
1138 build.rustc_commit = build.maybe_download_rustc()
1139 if build.rustc_commit is not None:
1140 if build.verbose:
1141 commit = build.rustc_commit
1142 print("using downloaded stage1 artifacts from CI (commit {})".format(commit))
1143 # FIXME: support downloading artifacts from the beta channel
1144 build.rustc_channel = "nightly"
1145 build.download_stage0()
1146 sys.stdout.flush()
1147 build.ensure_vendored()
1148 build.build_bootstrap()
1149 sys.stdout.flush()
1150
1151 # Run the bootstrap
1152 args = [build.bootstrap_binary()]
1153 args.extend(sys.argv[1:])
1154 env = os.environ.copy()
1155 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1156 env["BOOTSTRAP_PYTHON"] = sys.executable
1157 env["BUILD_DIR"] = build.build_dir
1158 env["RUSTC_BOOTSTRAP"] = '1'
1159 if toml_path:
1160 env["BOOTSTRAP_CONFIG"] = toml_path
1161 run(args, env=env, verbose=build.verbose)
1162
1163
1164 def main():
1165 """Entry point for the bootstrap process"""
1166 start_time = time()
1167
1168 # x.py help <cmd> ...
1169 if len(sys.argv) > 1 and sys.argv[1] == 'help':
1170 sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1171
1172 help_triggered = (
1173 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1174 try:
1175 bootstrap(help_triggered)
1176 if not help_triggered:
1177 print("Build completed successfully in {}".format(
1178 format_build_time(time() - start_time)))
1179 except (SystemExit, KeyboardInterrupt) as error:
1180 if hasattr(error, 'code') and isinstance(error.code, int):
1181 exit_code = error.code
1182 else:
1183 exit_code = 1
1184 print(error)
1185 if not help_triggered:
1186 print("Build completed unsuccessfully in {}".format(
1187 format_build_time(time() - start_time)))
1188 sys.exit(exit_code)
1189
1190
1191 if __name__ == '__main__':
1192 main()